2023-10-09T05:17:06.102 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-10-09T05:17:06.102 DEBUG:teuthology.run:Teuthology command: teuthology --name teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi --description upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} --archive /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417503 --verbose --owner scheduled_teuthology@teuthology -- /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417503/orig.config.yaml 2023-10-09T05:17:06.163 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T05:17:06.243 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417503 branch: reef description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' first_in_suite: false job_id: '7417503' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic cls tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: a340fb5463aa3715e93e7472d5f60dfdcf184ab3 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 5305 sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:quincy-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: a340fb5463aa3715e93e7472d5f60dfdcf184ab3 targets: smithi064.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= smithi156.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:3.0 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-10-07_14:23:11 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-10-09T05:17:06.244 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa; will attempt to use it 2023-10-09T05:17:06.244 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks 2023-10-09T05:17:06.244 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-10-09T05:17:06.246 INFO:teuthology.task.internal:Checking packages... 2023-10-09T05:17:06.315 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0' 2023-10-09T05:17:06.315 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:17:06.315 INFO:teuthology.packaging:ref: None 2023-10-09T05:17:06.315 INFO:teuthology.packaging:tag: None 2023-10-09T05:17:06.316 INFO:teuthology.packaging:branch: reef 2023-10-09T05:17:06.316 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:17:06.316 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2023-10-09T05:17:06.545 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-610.g397de1b0 2023-10-09T05:17:06.545 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-10-09T05:17:06.567 INFO:teuthology.task.internal:no buildpackages task found 2023-10-09T05:17:06.567 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-10-09T05:17:06.603 INFO:teuthology.task.internal:Saving configuration 2023-10-09T05:17:06.625 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-10-09T05:17:06.632 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T05:17:06.651 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi064.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417503', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 05:11:51.397847', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-09T05:17:06.670 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi156.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417503', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 05:11:51.399717', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-09T05:17:06.670 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-10-09T05:17:06.677 INFO:teuthology.task.internal:roles: ubuntu@smithi064.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-10-09T05:17:06.677 INFO:teuthology.task.internal:roles: ubuntu@smithi156.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-10-09T05:17:06.677 INFO:teuthology.run_tasks:Running task console_log... 2023-10-09T05:17:06.763 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f42616b3af0>, signals=[15]) 2023-10-09T05:17:06.763 INFO:teuthology.run_tasks:Running task internal.connect... 2023-10-09T05:17:06.771 INFO:teuthology.task.internal:Opening connections... 2023-10-09T05:17:06.771 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi064.front.sepia.ceph.com 2023-10-09T05:17:06.772 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi064.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T05:17:06.840 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi156.front.sepia.ceph.com 2023-10-09T05:17:06.841 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi156.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T05:17:06.908 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-10-09T05:17:06.917 DEBUG:teuthology.orchestra.run.smithi064:> uname -m 2023-10-09T05:17:06.938 INFO:teuthology.orchestra.run.smithi064.stdout:x86_64 2023-10-09T05:17:06.938 DEBUG:teuthology.orchestra.run.smithi064:> cat /etc/os-release 2023-10-09T05:17:06.997 INFO:teuthology.orchestra.run.smithi064.stdout:NAME="Red Hat Enterprise Linux" 2023-10-09T05:17:06.997 INFO:teuthology.orchestra.run.smithi064.stdout:VERSION="8.6 (Ootpa)" 2023-10-09T05:17:06.998 INFO:teuthology.orchestra.run.smithi064.stdout:ID="rhel" 2023-10-09T05:17:06.998 INFO:teuthology.orchestra.run.smithi064.stdout:ID_LIKE="fedora" 2023-10-09T05:17:06.998 INFO:teuthology.orchestra.run.smithi064.stdout:VERSION_ID="8.6" 2023-10-09T05:17:06.998 INFO:teuthology.orchestra.run.smithi064.stdout:PLATFORM_ID="platform:el8" 2023-10-09T05:17:06.998 INFO:teuthology.orchestra.run.smithi064.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-10-09T05:17:06.998 INFO:teuthology.orchestra.run.smithi064.stdout:ANSI_COLOR="0;31" 2023-10-09T05:17:06.998 INFO:teuthology.orchestra.run.smithi064.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-10-09T05:17:06.998 INFO:teuthology.orchestra.run.smithi064.stdout:HOME_URL="https://www.redhat.com/" 2023-10-09T05:17:06.998 INFO:teuthology.orchestra.run.smithi064.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-10-09T05:17:06.998 INFO:teuthology.orchestra.run.smithi064.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-10-09T05:17:06.998 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:17:06.999 INFO:teuthology.orchestra.run.smithi064.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-10-09T05:17:06.999 INFO:teuthology.orchestra.run.smithi064.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-10-09T05:17:06.999 INFO:teuthology.orchestra.run.smithi064.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-10-09T05:17:06.999 INFO:teuthology.orchestra.run.smithi064.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-10-09T05:17:06.999 INFO:teuthology.lock.ops:Updating smithi064.front.sepia.ceph.com on lock server 2023-10-09T05:17:07.028 DEBUG:teuthology.orchestra.run.smithi156:> uname -m 2023-10-09T05:17:07.049 INFO:teuthology.orchestra.run.smithi156.stdout:x86_64 2023-10-09T05:17:07.049 DEBUG:teuthology.orchestra.run.smithi156:> cat /etc/os-release 2023-10-09T05:17:07.108 INFO:teuthology.orchestra.run.smithi156.stdout:NAME="Red Hat Enterprise Linux" 2023-10-09T05:17:07.108 INFO:teuthology.orchestra.run.smithi156.stdout:VERSION="8.6 (Ootpa)" 2023-10-09T05:17:07.109 INFO:teuthology.orchestra.run.smithi156.stdout:ID="rhel" 2023-10-09T05:17:07.109 INFO:teuthology.orchestra.run.smithi156.stdout:ID_LIKE="fedora" 2023-10-09T05:17:07.109 INFO:teuthology.orchestra.run.smithi156.stdout:VERSION_ID="8.6" 2023-10-09T05:17:07.109 INFO:teuthology.orchestra.run.smithi156.stdout:PLATFORM_ID="platform:el8" 2023-10-09T05:17:07.109 INFO:teuthology.orchestra.run.smithi156.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-10-09T05:17:07.109 INFO:teuthology.orchestra.run.smithi156.stdout:ANSI_COLOR="0;31" 2023-10-09T05:17:07.109 INFO:teuthology.orchestra.run.smithi156.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-10-09T05:17:07.109 INFO:teuthology.orchestra.run.smithi156.stdout:HOME_URL="https://www.redhat.com/" 2023-10-09T05:17:07.109 INFO:teuthology.orchestra.run.smithi156.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-10-09T05:17:07.110 INFO:teuthology.orchestra.run.smithi156.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-10-09T05:17:07.110 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:17:07.110 INFO:teuthology.orchestra.run.smithi156.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-10-09T05:17:07.110 INFO:teuthology.orchestra.run.smithi156.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-10-09T05:17:07.110 INFO:teuthology.orchestra.run.smithi156.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-10-09T05:17:07.110 INFO:teuthology.orchestra.run.smithi156.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-10-09T05:17:07.111 INFO:teuthology.lock.ops:Updating smithi156.front.sepia.ceph.com on lock server 2023-10-09T05:17:07.145 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-10-09T05:17:07.162 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-10-09T05:17:07.195 INFO:teuthology.task.internal:Checking for old test directory... 2023-10-09T05:17:07.195 DEBUG:teuthology.orchestra.run.smithi064:> test '!' -e /home/ubuntu/cephtest 2023-10-09T05:17:07.198 DEBUG:teuthology.orchestra.run.smithi156:> test '!' -e /home/ubuntu/cephtest 2023-10-09T05:17:07.216 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-10-09T05:17:07.223 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-10-09T05:17:07.223 DEBUG:teuthology.orchestra.run.smithi064:> test -z $(ls -A /var/lib/ceph) 2023-10-09T05:17:07.258 DEBUG:teuthology.orchestra.run.smithi156:> test -z $(ls -A /var/lib/ceph) 2023-10-09T05:17:07.281 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-10-09T05:17:07.340 INFO:teuthology.run_tasks:Running task kernel... 2023-10-09T05:17:07.358 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-10-09T05:17:07.359 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-10-09T05:17:07.359 DEBUG:teuthology.orchestra.run.smithi064:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T05:17:07.360 DEBUG:teuthology.orchestra.run.smithi156:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T05:17:07.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:07.378 DEBUG:teuthology.orchestra.run.smithi156:> uname -r 2023-10-09T05:17:07.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:07.379 DEBUG:teuthology.orchestra.run.smithi064:> uname -r 2023-10-09T05:17:07.436 INFO:teuthology.orchestra.run.smithi156.stdout:4.18.0-372.9.1.el8.x86_64 2023-10-09T05:17:07.437 INFO:teuthology.task.kernel:Running kernel on smithi156: 4.18.0-372.9.1.el8.x86_64 2023-10-09T05:17:07.437 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum install -y kernel 2023-10-09T05:17:07.440 INFO:teuthology.orchestra.run.smithi064.stdout:4.18.0-372.9.1.el8.x86_64 2023-10-09T05:17:07.440 INFO:teuthology.task.kernel:Running kernel on smithi064: 4.18.0-372.9.1.el8.x86_64 2023-10-09T05:17:07.441 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum install -y kernel 2023-10-09T05:17:08.662 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:17:08.662 INFO:teuthology.orchestra.run.smithi064.stdout:Unable to read consumer identity 2023-10-09T05:17:08.666 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:17:08.666 INFO:teuthology.orchestra.run.smithi156.stdout:Unable to read consumer identity 2023-10-09T05:17:10.063 INFO:teuthology.orchestra.run.smithi156.stdout:Copr repo for python3-asyncssh owned by ceph 22 kB/s | 3.5 kB 00:00 2023-10-09T05:17:10.120 INFO:teuthology.orchestra.run.smithi064.stdout:Copr repo for python3-asyncssh owned by ceph 23 kB/s | 3.5 kB 00:00 2023-10-09T05:17:10.781 INFO:teuthology.orchestra.run.smithi156.stdout:Extra Packages for Enterprise Linux 23 MB/s | 16 MB 00:00 2023-10-09T05:17:11.061 INFO:teuthology.orchestra.run.smithi064.stdout:Extra Packages for Enterprise Linux 17 MB/s | 16 MB 00:00 2023-10-09T05:17:14.323 INFO:teuthology.orchestra.run.smithi156.stdout:lab-extras 425 kB/s | 24 kB 00:00 2023-10-09T05:17:14.805 INFO:teuthology.orchestra.run.smithi064.stdout:lab-extras 430 kB/s | 24 kB 00:00 2023-10-09T05:17:15.026 INFO:teuthology.orchestra.run.smithi156.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-10-09T05:17:15.073 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:17:15.074 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2023-10-09T05:17:15.074 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:17:15.182 DEBUG:teuthology.orchestra.run.smithi156:> echo no | sudo yum reinstall kernel || true 2023-10-09T05:17:15.516 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:17:15.516 INFO:teuthology.orchestra.run.smithi156.stdout:Unable to read consumer identity 2023-10-09T05:17:15.532 INFO:teuthology.orchestra.run.smithi064.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-10-09T05:17:15.589 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:17:15.590 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2023-10-09T05:17:15.591 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:17:15.738 DEBUG:teuthology.orchestra.run.smithi064:> echo no | sudo yum reinstall kernel || true 2023-10-09T05:17:15.801 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:00:01 ago on Mon 09 Oct 2023 05:17:14 AM UTC. 2023-10-09T05:17:15.842 INFO:teuthology.orchestra.run.smithi156.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-09T05:17:15.890 INFO:teuthology.orchestra.run.smithi156.stderr:Error: No packages marked for reinstall. 2023-10-09T05:17:15.937 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum reinstall -y kernel || true 2023-10-09T05:17:16.091 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:17:16.091 INFO:teuthology.orchestra.run.smithi064.stdout:Unable to read consumer identity 2023-10-09T05:17:16.266 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:17:16.266 INFO:teuthology.orchestra.run.smithi156.stdout:Unable to read consumer identity 2023-10-09T05:17:16.373 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:00:02 ago on Mon 09 Oct 2023 05:17:14 AM UTC. 2023-10-09T05:17:16.415 INFO:teuthology.orchestra.run.smithi064.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-09T05:17:16.461 INFO:teuthology.orchestra.run.smithi064.stderr:Error: No packages marked for reinstall. 2023-10-09T05:17:16.508 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum reinstall -y kernel || true 2023-10-09T05:17:16.528 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:00:02 ago on Mon 09 Oct 2023 05:17:14 AM UTC. 2023-10-09T05:17:16.568 INFO:teuthology.orchestra.run.smithi156.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-09T05:17:16.605 INFO:teuthology.orchestra.run.smithi156.stderr:Error: No packages marked for reinstall. 2023-10-09T05:17:16.656 DEBUG:teuthology.orchestra.run.smithi156:> rpm -q kernel | sort -rV | head -n 1 2023-10-09T05:17:16.735 INFO:teuthology.orchestra.run.smithi156.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-10-09T05:17:16.736 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-10-09T05:17:16.736 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-10-09T05:17:16.736 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-10-09T05:17:16.736 DEBUG:teuthology.orchestra.run.smithi156:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T05:17:16.766 INFO:teuthology.orchestra.run.smithi156.stdout:ttyS1 2023-10-09T05:17:16.780 DEBUG:teuthology.parallel:result is None 2023-10-09T05:17:16.858 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:17:16.858 INFO:teuthology.orchestra.run.smithi064.stdout:Unable to read consumer identity 2023-10-09T05:17:17.139 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:00:03 ago on Mon 09 Oct 2023 05:17:14 AM UTC. 2023-10-09T05:17:17.182 INFO:teuthology.orchestra.run.smithi064.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-09T05:17:17.210 INFO:teuthology.orchestra.run.smithi064.stderr:Error: No packages marked for reinstall. 2023-10-09T05:17:17.258 DEBUG:teuthology.orchestra.run.smithi064:> rpm -q kernel | sort -rV | head -n 1 2023-10-09T05:17:17.341 INFO:teuthology.orchestra.run.smithi064.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-10-09T05:17:17.342 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-10-09T05:17:17.342 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-10-09T05:17:17.342 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-10-09T05:17:17.343 DEBUG:teuthology.orchestra.run.smithi064:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T05:17:17.375 INFO:teuthology.orchestra.run.smithi064.stdout:ttyS1 2023-10-09T05:17:17.388 DEBUG:teuthology.parallel:result is None 2023-10-09T05:17:17.389 INFO:teuthology.run_tasks:Running task internal.base... 2023-10-09T05:17:17.398 INFO:teuthology.task.internal:Creating test directory... 2023-10-09T05:17:17.398 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T05:17:17.431 DEBUG:teuthology.orchestra.run.smithi156:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T05:17:17.451 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-10-09T05:17:17.458 INFO:teuthology.run_tasks:Running task internal.archive... 2023-10-09T05:17:17.486 INFO:teuthology.task.internal:Creating archive directory... 2023-10-09T05:17:17.486 DEBUG:teuthology.orchestra.run.smithi064:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T05:17:17.488 DEBUG:teuthology.orchestra.run.smithi156:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T05:17:17.515 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-10-09T05:17:17.522 INFO:teuthology.task.internal:Enabling coredump saving... 2023-10-09T05:17:17.522 DEBUG:teuthology.orchestra.run.smithi064:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T05:17:17.546 DEBUG:teuthology.orchestra.run.smithi156:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T05:17:17.591 INFO:teuthology.orchestra.run.smithi064.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T05:17:17.599 INFO:teuthology.orchestra.run.smithi156.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T05:17:17.601 INFO:teuthology.orchestra.run.smithi064.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T05:17:17.610 INFO:teuthology.orchestra.run.smithi156.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T05:17:17.611 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-10-09T05:17:17.619 INFO:teuthology.task.internal:Configuring sudo... 2023-10-09T05:17:17.619 DEBUG:teuthology.orchestra.run.smithi064:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T05:17:17.621 DEBUG:teuthology.orchestra.run.smithi156:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T05:17:17.650 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-10-09T05:17:17.658 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-10-09T05:17:17.659 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T05:17:17.689 DEBUG:teuthology.orchestra.run.smithi156:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T05:17:17.707 DEBUG:teuthology.orchestra.run.smithi064:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T05:17:17.788 DEBUG:teuthology.orchestra.run.smithi064:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T05:17:17.843 DEBUG:teuthology.orchestra.run.smithi064:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T05:17:17.927 DEBUG:teuthology.orchestra.run.smithi064:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T05:17:17.970 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-10-09T05:17:17.970 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T05:17:18.038 DEBUG:teuthology.orchestra.run.smithi156:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T05:17:18.082 DEBUG:teuthology.orchestra.run.smithi156:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T05:17:18.134 DEBUG:teuthology.orchestra.run.smithi156:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T05:17:18.218 DEBUG:teuthology.orchestra.run.smithi156:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T05:17:18.263 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2023-10-09T05:17:18.263 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T05:17:18.331 DEBUG:teuthology.orchestra.run.smithi064:> sudo service rsyslog restart 2023-10-09T05:17:18.333 DEBUG:teuthology.orchestra.run.smithi156:> sudo service rsyslog restart 2023-10-09T05:17:18.415 INFO:teuthology.orchestra.run.smithi064.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T05:17:18.425 INFO:teuthology.orchestra.run.smithi156.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T05:17:18.882 INFO:teuthology.run_tasks:Running task internal.timer... 2023-10-09T05:17:18.889 INFO:teuthology.task.internal:Starting timer... 2023-10-09T05:17:18.890 INFO:teuthology.run_tasks:Running task pcp... 2023-10-09T05:17:18.930 INFO:teuthology.run_tasks:Running task selinux... 2023-10-09T05:17:18.954 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-10-09T05:17:18.955 DEBUG:teuthology.orchestra.run.smithi064:> sudo service auditd rotate 2023-10-09T05:17:19.012 INFO:teuthology.orchestra.run.smithi064.stdout:Rotating logs: 2023-10-09T05:17:19.014 DEBUG:teuthology.orchestra.run.smithi156:> sudo service auditd rotate 2023-10-09T05:17:19.071 INFO:teuthology.orchestra.run.smithi156.stdout:Rotating logs: 2023-10-09T05:17:19.073 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-10-09T05:17:19.073 DEBUG:teuthology.orchestra.run.smithi064:> /usr/sbin/getenforce 2023-10-09T05:17:19.096 INFO:teuthology.orchestra.run.smithi064.stdout:Permissive 2023-10-09T05:17:19.096 DEBUG:teuthology.orchestra.run.smithi156:> /usr/sbin/getenforce 2023-10-09T05:17:19.120 INFO:teuthology.orchestra.run.smithi156.stdout:Permissive 2023-10-09T05:17:19.121 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi064.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi156.front.sepia.ceph.com': 'permissive'} 2023-10-09T05:17:19.121 DEBUG:teuthology.orchestra.run.smithi064:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T05:17:19.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:19.165 DEBUG:teuthology.orchestra.run.smithi156:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T05:17:19.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:17:19.193 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-10-09T05:17:19.193 DEBUG:teuthology.orchestra.run.smithi064:> sudo /usr/sbin/setenforce permissive 2023-10-09T05:17:19.236 DEBUG:teuthology.orchestra.run.smithi156:> sudo /usr/sbin/setenforce permissive 2023-10-09T05:17:19.263 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-10-09T05:17:19.272 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-10-09T05:17:19.287 INFO:teuthology.repo_utils:Fetching main from origin 2023-10-09T05:17:19.420 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-10-09T05:17:19.436 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-10-09T05:17:19.437 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi064.front.sepia.ceph.com,smithi156.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-10-09T05:26:31.237 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi064.front.sepia.ceph.com'), Remote(name='ubuntu@smithi156.front.sepia.ceph.com')] 2023-10-09T05:26:31.238 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi064.front.sepia.ceph.com' 2023-10-09T05:26:31.239 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi064.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T05:26:31.308 DEBUG:teuthology.orchestra.run.smithi064:> true 2023-10-09T05:26:31.376 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi064.front.sepia.ceph.com' 2023-10-09T05:26:31.377 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi156.front.sepia.ceph.com' 2023-10-09T05:26:31.377 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi156.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T05:26:31.446 DEBUG:teuthology.orchestra.run.smithi156:> true 2023-10-09T05:26:31.512 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi156.front.sepia.ceph.com' 2023-10-09T05:26:31.512 INFO:teuthology.run_tasks:Running task clock... 2023-10-09T05:26:31.521 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-10-09T05:26:31.522 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T05:26:31.522 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T05:26:31.524 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T05:26:31.525 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T05:26:31.556 INFO:teuthology.orchestra.run.smithi064.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-10-09T05:26:31.573 INFO:teuthology.orchestra.run.smithi064.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-10-09T05:26:31.584 INFO:teuthology.orchestra.run.smithi156.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-10-09T05:26:31.599 INFO:teuthology.orchestra.run.smithi156.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-10-09T05:26:31.639 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: ntpd: command not found 2023-10-09T05:26:31.651 INFO:teuthology.orchestra.run.smithi064.stdout:506 Cannot talk to daemon 2023-10-09T05:26:31.667 INFO:teuthology.orchestra.run.smithi064.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-10-09T05:26:31.669 INFO:teuthology.orchestra.run.smithi156.stderr:sudo: ntpd: command not found 2023-10-09T05:26:31.681 INFO:teuthology.orchestra.run.smithi156.stdout:506 Cannot talk to daemon 2023-10-09T05:26:31.681 INFO:teuthology.orchestra.run.smithi064.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-10-09T05:26:31.695 INFO:teuthology.orchestra.run.smithi156.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-10-09T05:26:31.709 INFO:teuthology.orchestra.run.smithi156.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-10-09T05:26:31.768 INFO:teuthology.orchestra.run.smithi064.stderr:bash: ntpq: command not found 2023-10-09T05:26:31.771 INFO:teuthology.orchestra.run.smithi064.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T05:26:31.771 INFO:teuthology.orchestra.run.smithi064.stdout:=============================================================================== 2023-10-09T05:26:31.772 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:26:31.772 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:26:31.772 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:26:31.772 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:26:31.778 INFO:teuthology.orchestra.run.smithi156.stderr:bash: ntpq: command not found 2023-10-09T05:26:31.781 INFO:teuthology.orchestra.run.smithi156.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T05:26:31.781 INFO:teuthology.orchestra.run.smithi156.stdout:=============================================================================== 2023-10-09T05:26:31.781 INFO:teuthology.orchestra.run.smithi156.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:26:31.781 INFO:teuthology.orchestra.run.smithi156.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:26:31.781 INFO:teuthology.orchestra.run.smithi156.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:26:31.782 INFO:teuthology.orchestra.run.smithi156.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:26:31.782 INFO:teuthology.run_tasks:Running task pexec... 2023-10-09T05:26:31.792 INFO:teuthology.task.pexec:Executing custom commands... 2023-10-09T05:26:31.793 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi064.front.sepia.ceph.com 2023-10-09T05:26:31.793 DEBUG:teuthology.orchestra.run.smithi064:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-10-09T05:26:31.793 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi156.front.sepia.ceph.com 2023-10-09T05:26:31.793 DEBUG:teuthology.orchestra.run.smithi156:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-10-09T05:26:32.198 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:26:32.204 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:26:33.745 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:48 ago on Mon 09 Oct 2023 05:24:45 AM UTC. 2023-10-09T05:26:33.832 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:28 ago on Mon 09 Oct 2023 05:25:05 AM UTC. 2023-10-09T05:26:35.496 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:26:35.497 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:26:35.497 INFO:teuthology.orchestra.run.smithi156.stdout: Package Architecture Version Repository Size 2023-10-09T05:26:35.497 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:26:35.497 INFO:teuthology.orchestra.run.smithi156.stdout:Resetting modules: 2023-10-09T05:26:35.498 INFO:teuthology.orchestra.run.smithi156.stdout: container-tools 2023-10-09T05:26:35.498 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:26:35.498 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2023-10-09T05:26:35.498 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:26:35.498 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:26:35.788 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:26:35.789 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:26:35.789 INFO:teuthology.orchestra.run.smithi064.stdout: Package Architecture Version Repository Size 2023-10-09T05:26:35.789 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:26:35.789 INFO:teuthology.orchestra.run.smithi064.stdout:Resetting modules: 2023-10-09T05:26:35.789 INFO:teuthology.orchestra.run.smithi064.stdout: container-tools 2023-10-09T05:26:35.789 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:26:35.790 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-10-09T05:26:35.790 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:26:35.790 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:26:36.456 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:51 ago on Mon 09 Oct 2023 05:24:45 AM UTC. 2023-10-09T05:26:36.784 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:31 ago on Mon 09 Oct 2023 05:25:05 AM UTC. 2023-10-09T05:26:42.396 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:26:42.456 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:26:42.868 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:26:42.929 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:26:44.347 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:38 ago on Mon 09 Oct 2023 05:25:05 AM UTC. 2023-10-09T05:26:44.427 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:58 ago on Mon 09 Oct 2023 05:24:45 AM UTC. 2023-10-09T05:26:46.278 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:26:46.322 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:26:46.374 INFO:teuthology.orchestra.run.smithi156.stdout:========================================================================================================================== 2023-10-09T05:26:46.374 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2023-10-09T05:26:46.374 INFO:teuthology.orchestra.run.smithi156.stdout:========================================================================================================================== 2023-10-09T05:26:46.374 INFO:teuthology.orchestra.run.smithi156.stdout:Installing group/module packages: 2023-10-09T05:26:46.375 INFO:teuthology.orchestra.run.smithi156.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-10-09T05:26:46.375 INFO:teuthology.orchestra.run.smithi156.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-10-09T05:26:46.375 INFO:teuthology.orchestra.run.smithi156.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-10-09T05:26:46.375 INFO:teuthology.orchestra.run.smithi156.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-10-09T05:26:46.375 INFO:teuthology.orchestra.run.smithi156.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-10-09T05:26:46.375 INFO:teuthology.orchestra.run.smithi156.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T05:26:46.375 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2023-10-09T05:26:46.375 INFO:teuthology.orchestra.run.smithi156.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-10-09T05:26:46.376 INFO:teuthology.orchestra.run.smithi156.stdout:Downgrading: 2023-10-09T05:26:46.376 INFO:teuthology.orchestra.run.smithi156.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T05:26:46.376 INFO:teuthology.orchestra.run.smithi156.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-10-09T05:26:46.376 INFO:teuthology.orchestra.run.smithi156.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-10-09T05:26:46.376 INFO:teuthology.orchestra.run.smithi156.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-10-09T05:26:46.376 INFO:teuthology.orchestra.run.smithi156.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-10-09T05:26:46.376 INFO:teuthology.orchestra.run.smithi156.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-10-09T05:26:46.376 INFO:teuthology.orchestra.run.smithi156.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-10-09T05:26:46.377 INFO:teuthology.orchestra.run.smithi156.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-10-09T05:26:46.377 INFO:teuthology.orchestra.run.smithi156.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-10-09T05:26:46.377 INFO:teuthology.orchestra.run.smithi156.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-10-09T05:26:46.377 INFO:teuthology.orchestra.run.smithi156.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-10-09T05:26:46.377 INFO:teuthology.orchestra.run.smithi156.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T05:26:46.377 INFO:teuthology.orchestra.run.smithi156.stdout:Installing module profiles: 2023-10-09T05:26:46.377 INFO:teuthology.orchestra.run.smithi156.stdout: container-tools/common 2023-10-09T05:26:46.378 INFO:teuthology.orchestra.run.smithi156.stdout:Enabling module streams: 2023-10-09T05:26:46.378 INFO:teuthology.orchestra.run.smithi156.stdout: container-tools 3.0 2023-10-09T05:26:46.378 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:26:46.378 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2023-10-09T05:26:46.378 INFO:teuthology.orchestra.run.smithi156.stdout:========================================================================================================================== 2023-10-09T05:26:46.378 INFO:teuthology.orchestra.run.smithi156.stdout:Install 7 Packages 2023-10-09T05:26:46.378 INFO:teuthology.orchestra.run.smithi156.stdout:Downgrade 12 Packages 2023-10-09T05:26:46.379 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:26:46.380 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 51 M 2023-10-09T05:26:46.380 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2023-10-09T05:26:46.430 INFO:teuthology.orchestra.run.smithi064.stdout:========================================================================================================================== 2023-10-09T05:26:46.431 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2023-10-09T05:26:46.431 INFO:teuthology.orchestra.run.smithi064.stdout:========================================================================================================================== 2023-10-09T05:26:46.431 INFO:teuthology.orchestra.run.smithi064.stdout:Installing group/module packages: 2023-10-09T05:26:46.431 INFO:teuthology.orchestra.run.smithi064.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-10-09T05:26:46.431 INFO:teuthology.orchestra.run.smithi064.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-10-09T05:26:46.431 INFO:teuthology.orchestra.run.smithi064.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-10-09T05:26:46.431 INFO:teuthology.orchestra.run.smithi064.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-10-09T05:26:46.432 INFO:teuthology.orchestra.run.smithi064.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-10-09T05:26:46.432 INFO:teuthology.orchestra.run.smithi064.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T05:26:46.432 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2023-10-09T05:26:46.432 INFO:teuthology.orchestra.run.smithi064.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-10-09T05:26:46.432 INFO:teuthology.orchestra.run.smithi064.stdout:Downgrading: 2023-10-09T05:26:46.432 INFO:teuthology.orchestra.run.smithi064.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T05:26:46.432 INFO:teuthology.orchestra.run.smithi064.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-10-09T05:26:46.432 INFO:teuthology.orchestra.run.smithi064.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-10-09T05:26:46.432 INFO:teuthology.orchestra.run.smithi064.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-10-09T05:26:46.432 INFO:teuthology.orchestra.run.smithi064.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-10-09T05:26:46.433 INFO:teuthology.orchestra.run.smithi064.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-10-09T05:26:46.433 INFO:teuthology.orchestra.run.smithi064.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-10-09T05:26:46.433 INFO:teuthology.orchestra.run.smithi064.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-10-09T05:26:46.433 INFO:teuthology.orchestra.run.smithi064.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-10-09T05:26:46.433 INFO:teuthology.orchestra.run.smithi064.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-10-09T05:26:46.433 INFO:teuthology.orchestra.run.smithi064.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-10-09T05:26:46.433 INFO:teuthology.orchestra.run.smithi064.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T05:26:46.433 INFO:teuthology.orchestra.run.smithi064.stdout:Installing module profiles: 2023-10-09T05:26:46.433 INFO:teuthology.orchestra.run.smithi064.stdout: container-tools/common 2023-10-09T05:26:46.434 INFO:teuthology.orchestra.run.smithi064.stdout:Enabling module streams: 2023-10-09T05:26:46.434 INFO:teuthology.orchestra.run.smithi064.stdout: container-tools 3.0 2023-10-09T05:26:46.434 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:26:46.434 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-10-09T05:26:46.434 INFO:teuthology.orchestra.run.smithi064.stdout:========================================================================================================================== 2023-10-09T05:26:46.434 INFO:teuthology.orchestra.run.smithi064.stdout:Install 7 Packages 2023-10-09T05:26:46.434 INFO:teuthology.orchestra.run.smithi064.stdout:Downgrade 12 Packages 2023-10-09T05:26:46.434 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:26:46.435 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 51 M 2023-10-09T05:26:46.435 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-10-09T05:26:46.548 INFO:teuthology.orchestra.run.smithi156.stdout:(1/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 294 kB/s | 51 kB 00:00 2023-10-09T05:26:46.618 INFO:teuthology.orchestra.run.smithi156.stdout:(2/19): containers-common-1.2.4-1.module+el8.6. 377 kB/s | 91 kB 00:00 2023-10-09T05:26:46.643 INFO:teuthology.orchestra.run.smithi064.stdout:(1/19): containers-common-1.2.4-1.module+el8.6. 424 kB/s | 91 kB 00:00 2023-10-09T05:26:46.671 INFO:teuthology.orchestra.run.smithi064.stdout:(2/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 214 kB/s | 51 kB 00:00 2023-10-09T05:26:46.683 INFO:teuthology.orchestra.run.smithi156.stdout:(3/19): container-selinux-2.178.0-2.module+el8. 171 kB/s | 53 kB 00:00 2023-10-09T05:26:46.727 INFO:teuthology.orchestra.run.smithi064.stdout:(3/19): container-selinux-2.178.0-2.module+el8. 177 kB/s | 53 kB 00:00 2023-10-09T05:26:46.816 INFO:teuthology.orchestra.run.smithi156.stdout:(4/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 345 kB/s | 69 kB 00:00 2023-10-09T05:26:46.817 INFO:teuthology.orchestra.run.smithi064.stdout:(4/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 460 kB/s | 69 kB 00:00 2023-10-09T05:26:46.858 INFO:teuthology.orchestra.run.smithi156.stdout:(5/19): criu-3.15-1.module+el8.6.0+14874+644362 1.6 MB/s | 511 kB 00:00 2023-10-09T05:26:46.918 INFO:teuthology.orchestra.run.smithi064.stdout:(5/19): criu-3.15-1.module+el8.6.0+14874+644362 1.8 MB/s | 511 kB 00:00 2023-10-09T05:26:46.942 INFO:teuthology.orchestra.run.smithi156.stdout:(6/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 11 MB/s | 2.9 MB 00:00 2023-10-09T05:26:46.975 INFO:teuthology.orchestra.run.smithi156.stdout:(7/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 453 kB/s | 72 kB 00:00 2023-10-09T05:26:46.985 INFO:teuthology.orchestra.run.smithi064.stdout:(6/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 431 kB/s | 72 kB 00:00 2023-10-09T05:26:47.034 INFO:teuthology.orchestra.run.smithi156.stdout:(8/19): podman-docker-3.0.1-9.module+el8.6.0+14 608 kB/s | 56 kB 00:00 2023-10-09T05:26:47.052 INFO:teuthology.orchestra.run.smithi064.stdout:(7/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 8.9 MB/s | 2.9 MB 00:00 2023-10-09T05:26:47.077 INFO:teuthology.orchestra.run.smithi064.stdout:(8/19): podman-docker-3.0.1-9.module+el8.6.0+14 609 kB/s | 56 kB 00:00 2023-10-09T05:26:47.093 INFO:teuthology.orchestra.run.smithi156.stdout:(9/19): conmon-2.0.26-1.module+el8.6.0+14874+64 436 kB/s | 51 kB 00:00 2023-10-09T05:26:47.178 INFO:teuthology.orchestra.run.smithi064.stdout:(9/19): conmon-2.0.26-1.module+el8.6.0+14874+64 406 kB/s | 51 kB 00:00 2023-10-09T05:26:47.292 INFO:teuthology.orchestra.run.smithi156.stdout:(10/19): podman-3.0.1-9.module+el8.6.0+14874+64 25 MB/s | 11 MB 00:00 2023-10-09T05:26:47.377 INFO:teuthology.orchestra.run.smithi156.stdout:(11/19): podman-catatonit-3.0.1-9.module+el8.6. 1.1 MB/s | 322 kB 00:00 2023-10-09T05:26:47.377 INFO:teuthology.orchestra.run.smithi064.stdout:(10/19): podman-3.0.1-9.module+el8.6.0+14874+64 24 MB/s | 11 MB 00:00 2023-10-09T05:26:47.412 INFO:teuthology.orchestra.run.smithi064.stdout:(11/19): podman-catatonit-3.0.1-9.module+el8.6. 1.3 MB/s | 322 kB 00:00 2023-10-09T05:26:47.451 INFO:teuthology.orchestra.run.smithi156.stdout:(12/19): yajl-2.1.0-10.el8.x86_64.rpm 257 kB/s | 41 kB 00:00 2023-10-09T05:26:47.495 INFO:teuthology.orchestra.run.smithi064.stdout:(12/19): yajl-2.1.0-10.el8.x86_64.rpm 349 kB/s | 41 kB 00:00 2023-10-09T05:26:47.694 INFO:teuthology.orchestra.run.smithi156.stdout:(13/19): containernetworking-plugins-0.9.1-1.mo 29 MB/s | 19 MB 00:00 2023-10-09T05:26:47.777 INFO:teuthology.orchestra.run.smithi156.stdout:(14/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 5.5 MB/s | 2.2 MB 00:00 2023-10-09T05:26:47.778 INFO:teuthology.orchestra.run.smithi064.stdout:(13/19): containernetworking-plugins-0.9.1-1.mo 27 MB/s | 19 MB 00:00 2023-10-09T05:26:47.854 INFO:teuthology.orchestra.run.smithi064.stdout:(14/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 5.0 MB/s | 2.2 MB 00:00 2023-10-09T05:26:47.919 INFO:teuthology.orchestra.run.smithi156.stdout:(15/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 14 MB/s | 6.4 MB 00:00 2023-10-09T05:26:47.987 INFO:teuthology.orchestra.run.smithi064.stdout:(15/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 13 MB/s | 6.4 MB 00:00 2023-10-09T05:26:48.063 INFO:teuthology.orchestra.run.smithi064.stdout:(16/19): cockpit-podman-29-2.module+el8.6.0+148 5.1 MB/s | 1.1 MB 00:00 2023-10-09T05:26:48.086 INFO:teuthology.orchestra.run.smithi156.stdout:(16/19): buildah-1.19.9-3.module+el8.6.0+14874+ 17 MB/s | 6.8 MB 00:00 2023-10-09T05:26:48.170 INFO:teuthology.orchestra.run.smithi156.stdout:(17/19): cockpit-podman-29-2.module+el8.6.0+148 2.7 MB/s | 1.1 MB 00:00 2023-10-09T05:26:48.194 INFO:teuthology.orchestra.run.smithi156.stdout:(18/19): crun-0.18-3.module+el8.6.0+14874+64436 672 kB/s | 185 kB 00:00 2023-10-09T05:26:48.371 INFO:teuthology.orchestra.run.smithi064.stdout:(17/19): buildah-1.19.9-3.module+el8.6.0+14874+ 11 MB/s | 6.8 MB 00:00 2023-10-09T05:26:48.386 INFO:teuthology.orchestra.run.smithi156.stdout:(19/19): udica-0.2.4-1.module+el8.6.0+14874+644 168 kB/s | 51 kB 00:00 2023-10-09T05:26:48.387 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:26:48.387 INFO:teuthology.orchestra.run.smithi156.stdout:Total 25 MB/s | 51 MB 00:02 2023-10-09T05:26:48.396 INFO:teuthology.orchestra.run.smithi064.stdout:(18/19): udica-0.2.4-1.module+el8.6.0+14874+644 152 kB/s | 51 kB 00:00 2023-10-09T05:26:48.421 INFO:teuthology.orchestra.run.smithi064.stdout:(19/19): crun-0.18-3.module+el8.6.0+14874+64436 426 kB/s | 185 kB 00:00 2023-10-09T05:26:48.422 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:26:48.422 INFO:teuthology.orchestra.run.smithi064.stdout:Total 25 MB/s | 51 MB 00:01 2023-10-09T05:26:48.862 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2023-10-09T05:26:48.905 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2023-10-09T05:26:48.905 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2023-10-09T05:26:48.914 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-10-09T05:26:48.959 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-10-09T05:26:48.959 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-10-09T05:26:49.333 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2023-10-09T05:26:49.336 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2023-10-09T05:26:49.416 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-10-09T05:26:49.420 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-10-09T05:26:50.487 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2023-10-09T05:26:50.538 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-10-09T05:26:50.663 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-10-09T05:26:50.714 INFO:teuthology.orchestra.run.smithi156.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-10-09T05:26:50.716 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-10-09T05:26:50.723 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-10-09T05:26:50.740 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T05:26:50.886 INFO:teuthology.orchestra.run.smithi064.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-10-09T05:26:50.895 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-10-09T05:26:50.913 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T05:26:50.942 INFO:teuthology.orchestra.run.smithi156.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T05:26:51.119 INFO:teuthology.orchestra.run.smithi064.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T05:27:07.899 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T05:27:08.865 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T05:27:09.659 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-10-09T05:27:09.887 INFO:teuthology.orchestra.run.smithi156.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-10-09T05:27:10.076 INFO:teuthology.orchestra.run.smithi156.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-10-09T05:27:10.243 INFO:teuthology.orchestra.run.smithi156.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-10-09T05:27:10.453 INFO:teuthology.orchestra.run.smithi156.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-10-09T05:27:10.454 INFO:teuthology.orchestra.run.smithi156.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-10-09T05:27:10.454 INFO:teuthology.orchestra.run.smithi156.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-10-09T05:27:10.454 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:27:10.757 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-10-09T05:27:10.963 INFO:teuthology.orchestra.run.smithi064.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-10-09T05:27:11.179 INFO:teuthology.orchestra.run.smithi064.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-10-09T05:27:11.229 INFO:teuthology.orchestra.run.smithi156.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-10-09T05:27:11.341 INFO:teuthology.orchestra.run.smithi064.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-10-09T05:27:11.469 INFO:teuthology.orchestra.run.smithi156.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-10-09T05:27:11.577 INFO:teuthology.orchestra.run.smithi064.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-10-09T05:27:11.577 INFO:teuthology.orchestra.run.smithi064.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-10-09T05:27:11.577 INFO:teuthology.orchestra.run.smithi064.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-10-09T05:27:11.578 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:27:12.436 INFO:teuthology.orchestra.run.smithi064.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-10-09T05:27:12.693 INFO:teuthology.orchestra.run.smithi064.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-10-09T05:27:12.926 INFO:teuthology.orchestra.run.smithi156.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-10-09T05:27:13.221 INFO:teuthology.orchestra.run.smithi156.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-10-09T05:27:13.607 INFO:teuthology.orchestra.run.smithi156.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-10-09T05:27:13.873 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-10-09T05:27:14.227 INFO:teuthology.orchestra.run.smithi064.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-10-09T05:27:14.525 INFO:teuthology.orchestra.run.smithi064.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-10-09T05:27:14.569 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-10-09T05:27:14.916 INFO:teuthology.orchestra.run.smithi064.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-10-09T05:27:15.170 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-10-09T05:27:15.298 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-10-09T05:27:15.551 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-10-09T05:27:15.729 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-10-09T05:27:15.927 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-10-09T05:27:15.981 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-10-09T05:27:16.059 INFO:teuthology.orchestra.run.smithi156.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-10-09T05:27:16.059 INFO:teuthology.orchestra.run.smithi156.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-10-09T05:27:16.377 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-10-09T05:27:16.543 INFO:teuthology.orchestra.run.smithi156.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-10-09T05:27:16.543 INFO:teuthology.orchestra.run.smithi156.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-10-09T05:27:16.720 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-10-09T05:27:16.799 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-10-09T05:27:16.828 INFO:teuthology.orchestra.run.smithi156.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-10-09T05:27:16.987 INFO:teuthology.orchestra.run.smithi156.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-10-09T05:27:17.041 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-10-09T05:27:17.128 INFO:teuthology.orchestra.run.smithi156.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-10-09T05:27:17.262 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-10-09T05:27:17.330 INFO:teuthology.orchestra.run.smithi156.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-10-09T05:27:17.466 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-10-09T05:27:17.478 INFO:teuthology.orchestra.run.smithi156.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-10-09T05:27:17.590 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-10-09T05:27:17.590 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-10-09T05:27:17.619 INFO:teuthology.orchestra.run.smithi156.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-10-09T05:27:17.769 INFO:teuthology.orchestra.run.smithi156.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-10-09T05:27:17.770 INFO:teuthology.orchestra.run.smithi156.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T05:27:17.900 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-10-09T05:27:17.935 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T05:27:18.099 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-10-09T05:27:18.100 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-10-09T05:27:18.267 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-10-09T05:27:18.425 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-10-09T05:27:18.467 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-10-09T05:27:18.568 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-10-09T05:27:18.717 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-10-09T05:27:18.841 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T05:27:18.841 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-10-09T05:27:18.841 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-10-09T05:27:18.841 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-10-09T05:27:18.841 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-10-09T05:27:18.841 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-10-09T05:27:18.842 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-10-09T05:27:18.842 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-10-09T05:27:18.842 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-10-09T05:27:18.842 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-10-09T05:27:18.842 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-10-09T05:27:18.842 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-10-09T05:27:18.842 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-10-09T05:27:18.842 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-10-09T05:27:18.842 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-10-09T05:27:18.842 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-10-09T05:27:18.842 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-10-09T05:27:18.842 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-10-09T05:27:18.842 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-10-09T05:27:18.843 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-10-09T05:27:18.843 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-10-09T05:27:18.843 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-10-09T05:27:18.843 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-10-09T05:27:18.843 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-10-09T05:27:18.843 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-10-09T05:27:18.843 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-10-09T05:27:18.844 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-10-09T05:27:18.844 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-10-09T05:27:18.844 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-10-09T05:27:18.844 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-10-09T05:27:18.844 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-10-09T05:27:18.924 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-10-09T05:27:19.067 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-10-09T05:27:19.242 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-10-09T05:27:19.402 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-10-09T05:27:19.403 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T05:27:19.591 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T05:27:20.144 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-10-09T05:27:20.277 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-10-09T05:27:20.277 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:02:34 ago on Mon 09 Oct 2023 05:24:45 AM UTC. 2023-10-09T05:27:20.544 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T05:27:20.545 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-10-09T05:27:20.545 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-10-09T05:27:20.545 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-10-09T05:27:20.545 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-10-09T05:27:20.545 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-10-09T05:27:20.545 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-10-09T05:27:20.545 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-10-09T05:27:20.546 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-10-09T05:27:20.546 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-10-09T05:27:20.546 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-10-09T05:27:20.546 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-10-09T05:27:20.546 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-10-09T05:27:20.546 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-10-09T05:27:20.546 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-10-09T05:27:20.546 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-10-09T05:27:20.547 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-10-09T05:27:20.547 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-10-09T05:27:20.547 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-10-09T05:27:20.547 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-10-09T05:27:20.547 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-10-09T05:27:20.547 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-10-09T05:27:20.547 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-10-09T05:27:20.547 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-10-09T05:27:20.547 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-10-09T05:27:20.547 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-10-09T05:27:20.548 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-10-09T05:27:20.548 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-10-09T05:27:20.548 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-10-09T05:27:20.548 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-10-09T05:27:20.548 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-10-09T05:27:22.009 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-10-09T05:27:22.009 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:02:16 ago on Mon 09 Oct 2023 05:25:05 AM UTC. 2023-10-09T05:27:22.805 INFO:teuthology.orchestra.run.smithi156.stdout:Installed products updated. 2023-10-09T05:27:22.999 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:27:23.000 INFO:teuthology.orchestra.run.smithi156.stdout:Downgraded: 2023-10-09T05:27:23.000 INFO:teuthology.orchestra.run.smithi156.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:23.000 INFO:teuthology.orchestra.run.smithi156.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-10-09T05:27:23.000 INFO:teuthology.orchestra.run.smithi156.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:23.000 INFO:teuthology.orchestra.run.smithi156.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:23.000 INFO:teuthology.orchestra.run.smithi156.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:23.000 INFO:teuthology.orchestra.run.smithi156.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:23.000 INFO:teuthology.orchestra.run.smithi156.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:23.000 INFO:teuthology.orchestra.run.smithi156.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:23.000 INFO:teuthology.orchestra.run.smithi156.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:23.001 INFO:teuthology.orchestra.run.smithi156.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-10-09T05:27:23.001 INFO:teuthology.orchestra.run.smithi156.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:23.001 INFO:teuthology.orchestra.run.smithi156.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:23.001 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2023-10-09T05:27:23.001 INFO:teuthology.orchestra.run.smithi156.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:23.001 INFO:teuthology.orchestra.run.smithi156.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-10-09T05:27:23.001 INFO:teuthology.orchestra.run.smithi156.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:23.001 INFO:teuthology.orchestra.run.smithi156.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:23.001 INFO:teuthology.orchestra.run.smithi156.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:23.001 INFO:teuthology.orchestra.run.smithi156.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-10-09T05:27:23.001 INFO:teuthology.orchestra.run.smithi156.stdout: yajl-2.1.0-10.el8.x86_64 2023-10-09T05:27:23.002 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:27:23.002 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:27:23.214 DEBUG:teuthology.parallel:result is None 2023-10-09T05:27:24.634 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-10-09T05:27:24.842 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:27:24.842 INFO:teuthology.orchestra.run.smithi064.stdout:Downgraded: 2023-10-09T05:27:24.842 INFO:teuthology.orchestra.run.smithi064.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:24.842 INFO:teuthology.orchestra.run.smithi064.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-10-09T05:27:24.842 INFO:teuthology.orchestra.run.smithi064.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:24.842 INFO:teuthology.orchestra.run.smithi064.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:24.842 INFO:teuthology.orchestra.run.smithi064.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:24.842 INFO:teuthology.orchestra.run.smithi064.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:24.843 INFO:teuthology.orchestra.run.smithi064.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:24.843 INFO:teuthology.orchestra.run.smithi064.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:24.843 INFO:teuthology.orchestra.run.smithi064.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:24.843 INFO:teuthology.orchestra.run.smithi064.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-10-09T05:27:24.843 INFO:teuthology.orchestra.run.smithi064.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:24.843 INFO:teuthology.orchestra.run.smithi064.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:24.843 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-10-09T05:27:24.843 INFO:teuthology.orchestra.run.smithi064.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:24.843 INFO:teuthology.orchestra.run.smithi064.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-10-09T05:27:24.843 INFO:teuthology.orchestra.run.smithi064.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:24.844 INFO:teuthology.orchestra.run.smithi064.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:24.844 INFO:teuthology.orchestra.run.smithi064.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:27:24.844 INFO:teuthology.orchestra.run.smithi064.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-10-09T05:27:24.844 INFO:teuthology.orchestra.run.smithi064.stdout: yajl-2.1.0-10.el8.x86_64 2023-10-09T05:27:24.844 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:27:24.844 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:27:25.039 DEBUG:teuthology.parallel:result is None 2023-10-09T05:27:25.039 INFO:teuthology.run_tasks:Running task install... 2023-10-09T05:27:25.048 DEBUG:teuthology.task.install:project ceph 2023-10-09T05:27:25.048 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}} 2023-10-09T05:27:25.048 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'} 2023-10-09T05:27:25.048 INFO:teuthology.task.install:Using flavor: default 2023-10-09T05:27:25.054 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-10-09T05:27:25.054 INFO:teuthology.task.install:extra packages: [] 2023-10-09T05:27:25.054 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0', 'tag': None, 'wait_for_package': False} 2023-10-09T05:27:25.055 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:27:25.055 INFO:teuthology.packaging:ref: None 2023-10-09T05:27:25.055 INFO:teuthology.packaging:tag: None 2023-10-09T05:27:25.055 INFO:teuthology.packaging:branch: quincy 2023-10-09T05:27:25.055 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:27:25.055 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-10-09T05:27:25.057 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0', 'tag': None, 'wait_for_package': False} 2023-10-09T05:27:25.057 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:27:25.057 INFO:teuthology.packaging:ref: None 2023-10-09T05:27:25.057 INFO:teuthology.packaging:tag: None 2023-10-09T05:27:25.057 INFO:teuthology.packaging:branch: quincy 2023-10-09T05:27:25.057 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:27:25.057 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-10-09T05:27:25.357 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/ 2023-10-09T05:27:25.358 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1303.gb2903cb0 2023-10-09T05:27:25.401 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/ 2023-10-09T05:27:25.402 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1303.gb2903cb0 2023-10-09T05:27:25.465 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-10-09T05:27:25.466 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2023-10-09T05:27:25.466 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-10-09T05:27:25.498 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-10-09T05:27:25.498 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:27:25.498 INFO:teuthology.packaging:ref: None 2023-10-09T05:27:25.498 INFO:teuthology.packaging:tag: None 2023-10-09T05:27:25.499 INFO:teuthology.packaging:branch: quincy 2023-10-09T05:27:25.499 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:27:25.499 DEBUG:teuthology.orchestra.run.smithi156:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-10-09T05:27:25.506 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-10-09T05:27:25.506 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-10-09T05:27:25.506 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-10-09T05:27:25.536 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-10-09T05:27:25.537 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:27:25.537 INFO:teuthology.packaging:ref: None 2023-10-09T05:27:25.537 INFO:teuthology.packaging:tag: None 2023-10-09T05:27:25.537 INFO:teuthology.packaging:branch: quincy 2023-10-09T05:27:25.537 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:27:25.537 DEBUG:teuthology.orchestra.run.smithi064:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-10-09T05:27:25.569 DEBUG:teuthology.orchestra.run.smithi156:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-10-09T05:27:25.607 DEBUG:teuthology.orchestra.run.smithi064:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-10-09T05:27:25.654 DEBUG:teuthology.orchestra.run.smithi156:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-10-09T05:27:25.687 DEBUG:teuthology.orchestra.run.smithi064:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-10-09T05:27:25.722 INFO:teuthology.orchestra.run.smithi156.stdout:check_obsoletes = 1 2023-10-09T05:27:25.724 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum clean all 2023-10-09T05:27:25.753 INFO:teuthology.orchestra.run.smithi064.stdout:check_obsoletes = 1 2023-10-09T05:27:25.755 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum clean all 2023-10-09T05:27:26.101 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:27:26.145 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:27:26.427 INFO:teuthology.orchestra.run.smithi156.stdout:44 files removed 2023-10-09T05:27:26.454 INFO:teuthology.orchestra.run.smithi064.stdout:44 files removed 2023-10-09T05:27:26.474 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-radosgw 2023-10-09T05:27:26.503 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-radosgw 2023-10-09T05:27:26.807 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:27:26.852 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:27:27.621 INFO:teuthology.orchestra.run.smithi156.stdout:ceph packages for x86_64 231 kB/s | 78 kB 00:00 2023-10-09T05:27:27.728 INFO:teuthology.orchestra.run.smithi064.stdout:ceph packages for x86_64 221 kB/s | 78 kB 00:00 2023-10-09T05:27:27.981 INFO:teuthology.orchestra.run.smithi156.stdout:ceph noarch packages 28 kB/s | 9.1 kB 00:00 2023-10-09T05:27:28.046 INFO:teuthology.orchestra.run.smithi064.stdout:ceph noarch packages 32 kB/s | 9.1 kB 00:00 2023-10-09T05:27:28.273 INFO:teuthology.orchestra.run.smithi156.stdout:ceph source packages 5.3 kB/s | 1.4 kB 00:00 2023-10-09T05:27:28.322 INFO:teuthology.orchestra.run.smithi064.stdout:ceph source packages 5.7 kB/s | 1.4 kB 00:00 2023-10-09T05:27:28.447 INFO:teuthology.orchestra.run.smithi064.stdout:Copr repo for python3-asyncssh owned by ceph 33 kB/s | 3.5 kB 00:00 2023-10-09T05:27:29.171 INFO:teuthology.orchestra.run.smithi156.stdout:Copr repo for python3-asyncssh owned by ceph 21 kB/s | 3.5 kB 00:00 2023-10-09T05:27:29.385 INFO:teuthology.orchestra.run.smithi064.stdout:Extra Packages for Enterprise Linux 17 MB/s | 16 MB 00:00 2023-10-09T05:27:30.402 INFO:teuthology.orchestra.run.smithi156.stdout:Extra Packages for Enterprise Linux 8.3 MB/s | 16 MB 00:01 2023-10-09T05:27:33.057 INFO:teuthology.orchestra.run.smithi064.stdout:lab-extras 452 kB/s | 24 kB 00:00 2023-10-09T05:27:33.916 INFO:teuthology.orchestra.run.smithi156.stdout:lab-extras 471 kB/s | 24 kB 00:00 2023-10-09T05:27:34.042 INFO:teuthology.orchestra.run.smithi064.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 58 MB/s | 53 MB 00:00 2023-10-09T05:27:34.835 INFO:teuthology.orchestra.run.smithi156.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 56 MB/s | 47 MB 00:00 2023-10-09T05:27:42.634 INFO:teuthology.orchestra.run.smithi064.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 57 MB/s | 47 MB 00:00 2023-10-09T05:27:42.794 INFO:teuthology.orchestra.run.smithi156.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 63 MB/s | 53 MB 00:00 2023-10-09T05:27:54.065 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:27:54.067 INFO:teuthology.orchestra.run.smithi156.stdout:==================================================================================================== 2023-10-09T05:27:54.067 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2023-10-09T05:27:54.067 INFO:teuthology.orchestra.run.smithi156.stdout:==================================================================================================== 2023-10-09T05:27:54.068 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2023-10-09T05:27:54.068 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-radosgw x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 17 M 2023-10-09T05:27:54.068 INFO:teuthology.orchestra.run.smithi156.stdout:Upgrading: 2023-10-09T05:27:54.068 INFO:teuthology.orchestra.run.smithi156.stdout: librados2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.5 M 2023-10-09T05:27:54.068 INFO:teuthology.orchestra.run.smithi156.stdout: librbd1 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.6 M 2023-10-09T05:27:54.068 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2023-10-09T05:27:54.068 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-base x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 5.2 M 2023-10-09T05:27:54.068 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-common x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 23 M 2023-10-09T05:27:54.068 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-selinux x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 24 k 2023-10-09T05:27:54.068 INFO:teuthology.orchestra.run.smithi156.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-10-09T05:27:54.068 INFO:teuthology.orchestra.run.smithi156.stdout: libcephfs2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 797 k 2023-10-09T05:27:54.068 INFO:teuthology.orchestra.run.smithi156.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-10-09T05:27:54.069 INFO:teuthology.orchestra.run.smithi156.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-10-09T05:27:54.069 INFO:teuthology.orchestra.run.smithi156.stdout: libradosstriper1 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 509 k 2023-10-09T05:27:54.069 INFO:teuthology.orchestra.run.smithi156.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-10-09T05:27:54.069 INFO:teuthology.orchestra.run.smithi156.stdout: librgw2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 5.6 M 2023-10-09T05:27:54.069 INFO:teuthology.orchestra.run.smithi156.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-10-09T05:27:54.069 INFO:teuthology.orchestra.run.smithi156.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-10-09T05:27:54.069 INFO:teuthology.orchestra.run.smithi156.stdout: python3-ceph-argparse x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 45 k 2023-10-09T05:27:54.069 INFO:teuthology.orchestra.run.smithi156.stdout: python3-ceph-common x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 115 k 2023-10-09T05:27:54.069 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cephfs x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 190 k 2023-10-09T05:27:54.069 INFO:teuthology.orchestra.run.smithi156.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-10-09T05:27:54.069 INFO:teuthology.orchestra.run.smithi156.stdout: python3-rados x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 370 k 2023-10-09T05:27:54.070 INFO:teuthology.orchestra.run.smithi156.stdout: python3-rbd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 358 k 2023-10-09T05:27:54.070 INFO:teuthology.orchestra.run.smithi156.stdout: python3-rgw x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 106 k 2023-10-09T05:27:54.070 INFO:teuthology.orchestra.run.smithi156.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-10-09T05:27:54.070 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:27:54.070 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2023-10-09T05:27:54.070 INFO:teuthology.orchestra.run.smithi156.stdout:==================================================================================================== 2023-10-09T05:27:54.070 INFO:teuthology.orchestra.run.smithi156.stdout:Install 21 Packages 2023-10-09T05:27:54.070 INFO:teuthology.orchestra.run.smithi156.stdout:Upgrade 2 Packages 2023-10-09T05:27:54.070 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:27:54.071 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 63 M 2023-10-09T05:27:54.071 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2023-10-09T05:27:54.302 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:27:54.304 INFO:teuthology.orchestra.run.smithi064.stdout:==================================================================================================== 2023-10-09T05:27:54.304 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2023-10-09T05:27:54.305 INFO:teuthology.orchestra.run.smithi064.stdout:==================================================================================================== 2023-10-09T05:27:54.305 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-10-09T05:27:54.305 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-radosgw x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 17 M 2023-10-09T05:27:54.305 INFO:teuthology.orchestra.run.smithi064.stdout:Upgrading: 2023-10-09T05:27:54.305 INFO:teuthology.orchestra.run.smithi064.stdout: librados2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.5 M 2023-10-09T05:27:54.305 INFO:teuthology.orchestra.run.smithi064.stdout: librbd1 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.6 M 2023-10-09T05:27:54.305 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2023-10-09T05:27:54.305 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-base x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 5.2 M 2023-10-09T05:27:54.305 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-common x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 23 M 2023-10-09T05:27:54.305 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-selinux x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 24 k 2023-10-09T05:27:54.305 INFO:teuthology.orchestra.run.smithi064.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-10-09T05:27:54.306 INFO:teuthology.orchestra.run.smithi064.stdout: libcephfs2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 797 k 2023-10-09T05:27:54.306 INFO:teuthology.orchestra.run.smithi064.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-10-09T05:27:54.306 INFO:teuthology.orchestra.run.smithi064.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-10-09T05:27:54.306 INFO:teuthology.orchestra.run.smithi064.stdout: libradosstriper1 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 509 k 2023-10-09T05:27:54.306 INFO:teuthology.orchestra.run.smithi064.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-10-09T05:27:54.306 INFO:teuthology.orchestra.run.smithi064.stdout: librgw2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 5.6 M 2023-10-09T05:27:54.306 INFO:teuthology.orchestra.run.smithi064.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-10-09T05:27:54.306 INFO:teuthology.orchestra.run.smithi064.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-10-09T05:27:54.306 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ceph-argparse x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 45 k 2023-10-09T05:27:54.306 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ceph-common x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 115 k 2023-10-09T05:27:54.306 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cephfs x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 190 k 2023-10-09T05:27:54.306 INFO:teuthology.orchestra.run.smithi064.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-10-09T05:27:54.307 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rados x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 370 k 2023-10-09T05:27:54.307 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rbd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 358 k 2023-10-09T05:27:54.307 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rgw x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 106 k 2023-10-09T05:27:54.307 INFO:teuthology.orchestra.run.smithi064.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-10-09T05:27:54.307 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:27:54.307 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-10-09T05:27:54.307 INFO:teuthology.orchestra.run.smithi064.stdout:==================================================================================================== 2023-10-09T05:27:54.307 INFO:teuthology.orchestra.run.smithi064.stdout:Install 21 Packages 2023-10-09T05:27:54.307 INFO:teuthology.orchestra.run.smithi064.stdout:Upgrade 2 Packages 2023-10-09T05:27:54.307 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:27:54.308 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 63 M 2023-10-09T05:27:54.308 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-10-09T05:27:54.986 INFO:teuthology.orchestra.run.smithi156.stdout:(1/23): ceph-base-17.2.6-1303.gb2903cb0.el8.x86 5.8 MB/s | 5.2 MB 00:00 2023-10-09T05:27:55.070 INFO:teuthology.orchestra.run.smithi156.stdout:(2/23): ceph-selinux-17.2.6-1303.gb2903cb0.el8. 292 kB/s | 24 kB 00:00 2023-10-09T05:27:55.721 INFO:teuthology.orchestra.run.smithi156.stdout:(3/23): libcephfs2-17.2.6-1303.gb2903cb0.el8.x8 1.2 MB/s | 797 kB 00:00 2023-10-09T05:27:56.128 INFO:teuthology.orchestra.run.smithi156.stdout:(4/23): ceph-radosgw-17.2.6-1303.gb2903cb0.el8. 8.3 MB/s | 17 MB 00:02 2023-10-09T05:27:56.254 INFO:teuthology.orchestra.run.smithi156.stdout:(5/23): libradosstriper1-17.2.6-1303.gb2903cb0. 952 kB/s | 509 kB 00:00 2023-10-09T05:27:56.355 INFO:teuthology.orchestra.run.smithi156.stdout:(6/23): python3-ceph-argparse-17.2.6-1303.gb290 447 kB/s | 45 kB 00:00 2023-10-09T05:27:56.447 INFO:teuthology.orchestra.run.smithi156.stdout:(7/23): python3-ceph-common-17.2.6-1303.gb2903c 1.2 MB/s | 115 kB 00:00 2023-10-09T05:27:56.738 INFO:teuthology.orchestra.run.smithi156.stdout:(8/23): ceph-common-17.2.6-1303.gb2903cb0.el8.x 8.9 MB/s | 23 MB 00:02 2023-10-09T05:27:56.743 INFO:teuthology.orchestra.run.smithi064.stdout:(1/23): ceph-base-17.2.6-1303.gb2903cb0.el8.x86 2.2 MB/s | 5.2 MB 00:02 2023-10-09T05:27:56.763 INFO:teuthology.orchestra.run.smithi156.stdout:(9/23): python3-cephfs-17.2.6-1303.gb2903cb0.el 600 kB/s | 190 kB 00:00 2023-10-09T05:27:56.810 INFO:teuthology.orchestra.run.smithi064.stdout:(2/23): ceph-selinux-17.2.6-1303.gb2903cb0.el8. 366 kB/s | 24 kB 00:00 2023-10-09T05:27:56.847 INFO:teuthology.orchestra.run.smithi156.stdout:(10/23): librgw2-17.2.6-1303.gb2903cb0.el8.x86_ 7.7 MB/s | 5.6 MB 00:00 2023-10-09T05:27:56.872 INFO:teuthology.orchestra.run.smithi156.stdout:(11/23): python3-rados-17.2.6-1303.gb2903cb0.el 2.7 MB/s | 370 kB 00:00 2023-10-09T05:27:56.906 INFO:teuthology.orchestra.run.smithi156.stdout:(12/23): python3-rgw-17.2.6-1303.gb2903cb0.el8. 1.8 MB/s | 106 kB 00:00 2023-10-09T05:27:56.940 INFO:teuthology.orchestra.run.smithi156.stdout:(13/23): python3-rbd-17.2.6-1303.gb2903cb0.el8. 2.0 MB/s | 358 kB 00:00 2023-10-09T05:27:56.954 INFO:teuthology.orchestra.run.smithi064.stdout:(3/23): libcephfs2-17.2.6-1303.gb2903cb0.el8.x8 5.5 MB/s | 797 kB 00:00 2023-10-09T05:27:57.052 INFO:teuthology.orchestra.run.smithi064.stdout:(4/23): libradosstriper1-17.2.6-1303.gb2903cb0. 5.0 MB/s | 509 kB 00:00 2023-10-09T05:27:57.182 INFO:teuthology.orchestra.run.smithi156.stdout:(14/23): liboath-2.6.2-3.el8.x86_64.rpm 213 kB/s | 59 kB 00:00 2023-10-09T05:27:57.232 INFO:teuthology.orchestra.run.smithi156.stdout:(15/23): libunwind-1.3.1-3.el8.x86_64.rpm 261 kB/s | 75 kB 00:00 2023-10-09T05:27:57.290 INFO:teuthology.orchestra.run.smithi156.stdout:(16/23): gperftools-libs-2.7-9.el8.x86_64.rpm 733 kB/s | 306 kB 00:00 2023-10-09T05:27:57.307 INFO:teuthology.orchestra.run.smithi156.stdout:(17/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 3.4 MB/s | 259 kB 00:00 2023-10-09T05:27:57.374 INFO:teuthology.orchestra.run.smithi156.stdout:(18/23): librdkafka-0.11.4-3.el8.x86_64.rpm 5.6 MB/s | 354 kB 00:00 2023-10-09T05:27:57.391 INFO:teuthology.orchestra.run.smithi156.stdout:(19/23): python3-prettytable-0.7.2-14.el8.noarc 437 kB/s | 44 kB 00:00 2023-10-09T05:27:57.425 INFO:teuthology.orchestra.run.smithi156.stdout:(20/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 934 kB/s | 47 kB 00:00 2023-10-09T05:27:57.542 INFO:teuthology.orchestra.run.smithi156.stdout:(21/23): thrift-0.13.0-2.el8.x86_64.rpm 4.8 MB/s | 1.7 MB 00:00 2023-10-09T05:27:57.594 INFO:teuthology.orchestra.run.smithi064.stdout:(5/23): librgw2-17.2.6-1303.gb2903cb0.el8.x86_6 10 MB/s | 5.6 MB 00:00 2023-10-09T05:27:57.661 INFO:teuthology.orchestra.run.smithi064.stdout:(6/23): python3-ceph-argparse-17.2.6-1303.gb290 674 kB/s | 45 kB 00:00 2023-10-09T05:27:57.728 INFO:teuthology.orchestra.run.smithi064.stdout:(7/23): python3-ceph-common-17.2.6-1303.gb2903c 1.7 MB/s | 115 kB 00:00 2023-10-09T05:27:57.803 INFO:teuthology.orchestra.run.smithi064.stdout:(8/23): python3-cephfs-17.2.6-1303.gb2903cb0.el 2.5 MB/s | 190 kB 00:00 2023-10-09T05:27:57.825 INFO:teuthology.orchestra.run.smithi156.stdout:(22/23): librbd1-17.2.6-1303.gb2903cb0.el8.x86_ 9.0 MB/s | 3.6 MB 00:00 2023-10-09T05:27:57.887 INFO:teuthology.orchestra.run.smithi064.stdout:(9/23): python3-rados-17.2.6-1303.gb2903cb0.el8 4.3 MB/s | 370 kB 00:00 2023-10-09T05:27:57.962 INFO:teuthology.orchestra.run.smithi064.stdout:(10/23): python3-rbd-17.2.6-1303.gb2903cb0.el8. 4.7 MB/s | 358 kB 00:00 2023-10-09T05:27:58.029 INFO:teuthology.orchestra.run.smithi064.stdout:(11/23): python3-rgw-17.2.6-1303.gb2903cb0.el8. 1.6 MB/s | 106 kB 00:00 2023-10-09T05:27:58.188 INFO:teuthology.orchestra.run.smithi064.stdout:(12/23): gperftools-libs-2.7-9.el8.x86_64.rpm 1.9 MB/s | 306 kB 00:00 2023-10-09T05:27:58.238 INFO:teuthology.orchestra.run.smithi064.stdout:(13/23): liboath-2.6.2-3.el8.x86_64.rpm 1.2 MB/s | 59 kB 00:00 2023-10-09T05:27:58.242 INFO:teuthology.orchestra.run.smithi156.stdout:(23/23): librados2-17.2.6-1303.gb2903cb0.el8.x8 4.1 MB/s | 3.5 MB 00:00 2023-10-09T05:27:58.247 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:27:58.247 INFO:teuthology.orchestra.run.smithi156.stdout:Total 15 MB/s | 63 MB 00:04 2023-10-09T05:27:58.289 INFO:teuthology.orchestra.run.smithi064.stdout:(14/23): libunwind-1.3.1-3.el8.x86_64.rpm 1.5 MB/s | 75 kB 00:00 2023-10-09T05:27:58.293 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2023-10-09T05:27:58.340 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2023-10-09T05:27:58.340 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2023-10-09T05:27:58.405 INFO:teuthology.orchestra.run.smithi064.stdout:(15/23): thrift-0.13.0-2.el8.x86_64.rpm 15 MB/s | 1.7 MB 00:00 2023-10-09T05:27:58.464 INFO:teuthology.orchestra.run.smithi064.stdout:(16/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 805 kB/s | 47 kB 00:00 2023-10-09T05:27:58.506 INFO:teuthology.orchestra.run.smithi064.stdout:(17/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 6.1 MB/s | 259 kB 00:00 2023-10-09T05:27:58.539 INFO:teuthology.orchestra.run.smithi064.stdout:(18/23): python3-prettytable-0.7.2-14.el8.noarc 1.3 MB/s | 44 kB 00:00 2023-10-09T05:27:58.616 INFO:teuthology.orchestra.run.smithi064.stdout:(19/23): librdkafka-0.11.4-3.el8.x86_64.rpm 4.5 MB/s | 354 kB 00:00 2023-10-09T05:27:58.686 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2023-10-09T05:27:58.690 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2023-10-09T05:27:58.965 INFO:teuthology.orchestra.run.smithi064.stdout:(20/23): librados2-17.2.6-1303.gb2903cb0.el8.x8 9.9 MB/s | 3.5 MB 00:00 2023-10-09T05:27:59.257 INFO:teuthology.orchestra.run.smithi064.stdout:(21/23): librbd1-17.2.6-1303.gb2903cb0.el8.x86_ 12 MB/s | 3.6 MB 00:00 2023-10-09T05:27:59.490 INFO:teuthology.orchestra.run.smithi064.stdout:(22/23): ceph-radosgw-17.2.6-1303.gb2903cb0.el8 3.3 MB/s | 17 MB 00:05 2023-10-09T05:27:59.761 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2023-10-09T05:27:59.937 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-09T05:28:00.317 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-09T05:28:00.450 INFO:teuthology.orchestra.run.smithi064.stdout:(23/23): ceph-common-17.2.6-1303.gb2903cb0.el8. 3.9 MB/s | 23 MB 00:06 2023-10-09T05:28:00.452 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:28:00.452 INFO:teuthology.orchestra.run.smithi064.stdout:Total 10 MB/s | 63 MB 00:06 2023-10-09T05:28:00.461 INFO:teuthology.orchestra.run.smithi156.stdout: Upgrading : librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T05:28:00.502 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-10-09T05:28:00.551 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-10-09T05:28:00.551 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-10-09T05:28:00.559 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T05:28:00.759 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/25 2023-10-09T05:28:00.919 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-10-09T05:28:00.923 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-10-09T05:28:00.978 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-10-09T05:28:01.145 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T05:28:01.546 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T05:28:01.737 INFO:teuthology.orchestra.run.smithi156.stdout: Upgrading : librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/25 2023-10-09T05:28:01.800 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/25 2023-10-09T05:28:01.991 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 7/25 2023-10-09T05:28:01.997 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-10-09T05:28:02.120 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 8/25 2023-10-09T05:28:02.194 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-09T05:28:02.330 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 8/25 2023-10-09T05:28:02.605 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-09T05:28:02.793 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T05:28:02.892 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T05:28:03.027 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-10-09T05:28:03.057 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/25 2023-10-09T05:28:03.178 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T05:28:03.247 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T05:28:03.297 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-10-09T05:28:03.372 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8. 11/25 2023-10-09T05:28:03.476 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T05:28:03.547 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 12/25 2023-10-09T05:28:03.751 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 13/25 2023-10-09T05:28:03.922 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T05:28:03.949 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 14/25 2023-10-09T05:28:04.086 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 15/25 2023-10-09T05:28:04.101 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/25 2023-10-09T05:28:04.160 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 15/25 2023-10-09T05:28:04.207 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/25 2023-10-09T05:28:04.271 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-10-09T05:28:04.385 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-10-09T05:28:04.456 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-10-09T05:28:04.791 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-10-09T05:28:04.897 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-10-09T05:28:05.382 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-10-09T05:28:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-10-09T05:28:05.578 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x8 19/25 2023-10-09T05:28:05.693 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T05:28:05.760 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T05:28:05.938 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8. 11/25 2023-10-09T05:28:06.105 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 12/25 2023-10-09T05:28:06.338 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 13/25 2023-10-09T05:28:06.534 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 14/25 2023-10-09T05:28:06.726 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 15/25 2023-10-09T05:28:06.799 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 15/25 2023-10-09T05:28:06.937 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-10-09T05:28:07.153 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-10-09T05:28:07.384 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-10-09T05:28:07.560 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x8 19/25 2023-10-09T05:28:09.334 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T05:28:09.569 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T05:28:10.134 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T05:28:10.417 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 21/25 2023-10-09T05:28:10.575 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 21/25 2023-10-09T05:28:10.935 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T05:28:11.860 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T05:28:12.084 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T05:28:12.701 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T05:28:12.975 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 21/25 2023-10-09T05:28:13.063 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 21/25 2023-10-09T05:28:13.242 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T05:28:29.483 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T05:28:29.483 INFO:teuthology.orchestra.run.smithi156.stdout:skipping the directory /sys 2023-10-09T05:28:29.483 INFO:teuthology.orchestra.run.smithi156.stdout:skipping the directory /proc 2023-10-09T05:28:29.483 INFO:teuthology.orchestra.run.smithi156.stdout:skipping the directory /dev 2023-10-09T05:28:29.483 INFO:teuthology.orchestra.run.smithi156.stdout:skipping the directory /run 2023-10-09T05:28:29.483 INFO:teuthology.orchestra.run.smithi156.stdout:skipping the directory /mnt 2023-10-09T05:28:29.483 INFO:teuthology.orchestra.run.smithi156.stdout:skipping the directory /var/tmp 2023-10-09T05:28:29.483 INFO:teuthology.orchestra.run.smithi156.stdout:skipping the directory /home 2023-10-09T05:28:29.484 INFO:teuthology.orchestra.run.smithi156.stdout:skipping the directory /tmp 2023-10-09T05:28:29.484 INFO:teuthology.orchestra.run.smithi156.stdout:skipping the directory /dev 2023-10-09T05:28:29.484 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:28:31.155 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 23/25 2023-10-09T05:28:31.235 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 23/25 2023-10-09T05:28:31.235 INFO:teuthology.orchestra.run.smithi156.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T05:28:31.451 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T05:28:31.451 INFO:teuthology.orchestra.run.smithi156.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T05:28:32.086 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T05:28:32.087 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/25 2023-10-09T05:28:32.087 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T05:28:32.087 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/25 2023-10-09T05:28:32.087 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 4/25 2023-10-09T05:28:32.087 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T05:28:32.087 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 6/25 2023-10-09T05:28:32.087 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 7/25 2023-10-09T05:28:32.087 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8. 8/25 2023-10-09T05:28:32.087 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x8 9/25 2023-10-09T05:28:32.088 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T05:28:32.088 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 11/25 2023-10-09T05:28:32.088 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 12/25 2023-10-09T05:28:32.088 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 13/25 2023-10-09T05:28:32.088 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-10-09T05:28:32.088 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-10-09T05:28:32.088 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-10-09T05:28:32.088 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-10-09T05:28:32.088 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 18/25 2023-10-09T05:28:32.088 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 19/25 2023-10-09T05:28:32.088 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 20/25 2023-10-09T05:28:32.088 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 21/25 2023-10-09T05:28:32.089 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T05:28:32.089 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-10-09T05:28:32.089 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 24/25 2023-10-09T05:28:32.785 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T05:28:32.785 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /sys 2023-10-09T05:28:32.785 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /proc 2023-10-09T05:28:32.786 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /dev 2023-10-09T05:28:32.786 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /run 2023-10-09T05:28:32.786 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /mnt 2023-10-09T05:28:32.786 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /var/tmp 2023-10-09T05:28:32.786 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /home 2023-10-09T05:28:32.786 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /tmp 2023-10-09T05:28:32.786 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /dev 2023-10-09T05:28:32.786 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:28:33.502 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T05:28:33.503 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:00:51 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:28:34.538 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 23/25 2023-10-09T05:28:34.620 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 23/25 2023-10-09T05:28:34.621 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T05:28:34.875 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T05:28:34.875 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T05:28:35.530 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T05:28:35.530 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/25 2023-10-09T05:28:35.530 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T05:28:35.530 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/25 2023-10-09T05:28:35.530 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 4/25 2023-10-09T05:28:35.530 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T05:28:35.530 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 6/25 2023-10-09T05:28:35.531 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 7/25 2023-10-09T05:28:35.531 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8. 8/25 2023-10-09T05:28:35.531 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x8 9/25 2023-10-09T05:28:35.531 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T05:28:35.531 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 11/25 2023-10-09T05:28:35.531 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 12/25 2023-10-09T05:28:35.531 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 13/25 2023-10-09T05:28:35.532 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-10-09T05:28:35.532 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-10-09T05:28:35.532 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-10-09T05:28:35.532 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-10-09T05:28:35.532 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-10-09T05:28:35.532 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-10-09T05:28:35.532 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-10-09T05:28:35.532 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-10-09T05:28:35.532 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T05:28:35.532 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-10-09T05:28:35.532 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 24/25 2023-10-09T05:28:35.793 INFO:teuthology.orchestra.run.smithi156.stdout:Installed products updated. 2023-10-09T05:28:35.977 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:28:35.977 INFO:teuthology.orchestra.run.smithi156.stdout:Upgraded: 2023-10-09T05:28:35.977 INFO:teuthology.orchestra.run.smithi156.stdout: librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:35.978 INFO:teuthology.orchestra.run.smithi156.stdout: librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:35.978 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2023-10-09T05:28:35.978 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:35.978 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:35.978 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:35.978 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:35.978 INFO:teuthology.orchestra.run.smithi156.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-10-09T05:28:35.978 INFO:teuthology.orchestra.run.smithi156.stdout: libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:35.978 INFO:teuthology.orchestra.run.smithi156.stdout: liboath-2.6.2-3.el8.x86_64 2023-10-09T05:28:35.978 INFO:teuthology.orchestra.run.smithi156.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-10-09T05:28:35.978 INFO:teuthology.orchestra.run.smithi156.stdout: libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:35.979 INFO:teuthology.orchestra.run.smithi156.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-10-09T05:28:35.979 INFO:teuthology.orchestra.run.smithi156.stdout: librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:35.979 INFO:teuthology.orchestra.run.smithi156.stdout: libunwind-1.3.1-3.el8.x86_64 2023-10-09T05:28:35.979 INFO:teuthology.orchestra.run.smithi156.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-10-09T05:28:35.979 INFO:teuthology.orchestra.run.smithi156.stdout: python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:35.979 INFO:teuthology.orchestra.run.smithi156.stdout: python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:35.979 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:35.979 INFO:teuthology.orchestra.run.smithi156.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-10-09T05:28:35.979 INFO:teuthology.orchestra.run.smithi156.stdout: python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:35.979 INFO:teuthology.orchestra.run.smithi156.stdout: python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:35.979 INFO:teuthology.orchestra.run.smithi156.stdout: python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:35.979 INFO:teuthology.orchestra.run.smithi156.stdout: thrift-0.13.0-2.el8.x86_64 2023-10-09T05:28:35.980 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:28:35.980 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:28:36.146 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-test 2023-10-09T05:28:36.499 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:28:36.917 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T05:28:36.917 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:00:54 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:28:37.410 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:00:55 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:28:38.867 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:28:38.868 INFO:teuthology.orchestra.run.smithi156.stdout:============================================================================================ 2023-10-09T05:28:38.868 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2023-10-09T05:28:38.868 INFO:teuthology.orchestra.run.smithi156.stdout:============================================================================================ 2023-10-09T05:28:38.868 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2023-10-09T05:28:38.868 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-test x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 52 M 2023-10-09T05:28:38.869 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2023-10-09T05:28:38.869 INFO:teuthology.orchestra.run.smithi156.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-10-09T05:28:38.869 INFO:teuthology.orchestra.run.smithi156.stdout: libcephsqlite x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 175 k 2023-10-09T05:28:38.869 INFO:teuthology.orchestra.run.smithi156.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-10-09T05:28:38.869 INFO:teuthology.orchestra.run.smithi156.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-10-09T05:28:38.869 INFO:teuthology.orchestra.run.smithi156.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-10-09T05:28:38.869 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:28:38.869 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2023-10-09T05:28:38.869 INFO:teuthology.orchestra.run.smithi156.stdout:============================================================================================ 2023-10-09T05:28:38.869 INFO:teuthology.orchestra.run.smithi156.stdout:Install 6 Packages 2023-10-09T05:28:38.869 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:28:38.870 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 53 M 2023-10-09T05:28:38.870 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 233 M 2023-10-09T05:28:38.870 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2023-10-09T05:28:38.909 INFO:teuthology.orchestra.run.smithi156.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.8 MB/s | 70 kB 00:00 2023-10-09T05:28:39.009 INFO:teuthology.orchestra.run.smithi156.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 3.4 MB/s | 323 kB 00:00 2023-10-09T05:28:39.059 INFO:teuthology.orchestra.run.smithi156.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 3.7 MB/s | 187 kB 00:00 2023-10-09T05:28:39.076 INFO:teuthology.orchestra.run.smithi156.stdout:(4/6): libcephsqlite-17.2.6-1303.gb2903cb0.el8. 849 kB/s | 175 kB 00:00 2023-10-09T05:28:39.110 INFO:teuthology.orchestra.run.smithi156.stdout:(5/6): jq-1.6-3.el8.x86_64.rpm 3.9 MB/s | 202 kB 00:00 2023-10-09T05:28:39.368 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-10-09T05:28:39.574 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:28:39.574 INFO:teuthology.orchestra.run.smithi064.stdout:Upgraded: 2023-10-09T05:28:39.574 INFO:teuthology.orchestra.run.smithi064.stdout: librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:39.574 INFO:teuthology.orchestra.run.smithi064.stdout: librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:39.574 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-10-09T05:28:39.574 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:39.575 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:39.575 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:39.575 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:39.575 INFO:teuthology.orchestra.run.smithi064.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-10-09T05:28:39.575 INFO:teuthology.orchestra.run.smithi064.stdout: libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:39.575 INFO:teuthology.orchestra.run.smithi064.stdout: liboath-2.6.2-3.el8.x86_64 2023-10-09T05:28:39.575 INFO:teuthology.orchestra.run.smithi064.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-10-09T05:28:39.575 INFO:teuthology.orchestra.run.smithi064.stdout: libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:39.576 INFO:teuthology.orchestra.run.smithi064.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-10-09T05:28:39.576 INFO:teuthology.orchestra.run.smithi064.stdout: librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:39.576 INFO:teuthology.orchestra.run.smithi064.stdout: libunwind-1.3.1-3.el8.x86_64 2023-10-09T05:28:39.576 INFO:teuthology.orchestra.run.smithi064.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-10-09T05:28:39.576 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:39.576 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:39.576 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:39.576 INFO:teuthology.orchestra.run.smithi064.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-10-09T05:28:39.576 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:39.577 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:39.577 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:39.577 INFO:teuthology.orchestra.run.smithi064.stdout: thrift-0.13.0-2.el8.x86_64 2023-10-09T05:28:39.577 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:28:39.577 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:28:39.781 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-test 2023-10-09T05:28:40.150 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:28:41.028 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:00:59 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:28:41.610 INFO:teuthology.orchestra.run.smithi156.stdout:(6/6): ceph-test-17.2.6-1303.gb2903cb0.el8.x86_ 19 MB/s | 52 MB 00:02 2023-10-09T05:28:41.612 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:28:41.613 INFO:teuthology.orchestra.run.smithi156.stdout:Total 19 MB/s | 53 MB 00:02 2023-10-09T05:28:41.649 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2023-10-09T05:28:41.670 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2023-10-09T05:28:41.670 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2023-10-09T05:28:41.944 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2023-10-09T05:28:41.948 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2023-10-09T05:28:42.424 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2023-10-09T05:28:42.532 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:28:42.533 INFO:teuthology.orchestra.run.smithi064.stdout:============================================================================================ 2023-10-09T05:28:42.533 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2023-10-09T05:28:42.533 INFO:teuthology.orchestra.run.smithi064.stdout:============================================================================================ 2023-10-09T05:28:42.533 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-10-09T05:28:42.533 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-test x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 52 M 2023-10-09T05:28:42.533 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2023-10-09T05:28:42.534 INFO:teuthology.orchestra.run.smithi064.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-10-09T05:28:42.534 INFO:teuthology.orchestra.run.smithi064.stdout: libcephsqlite x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 175 k 2023-10-09T05:28:42.534 INFO:teuthology.orchestra.run.smithi064.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-10-09T05:28:42.534 INFO:teuthology.orchestra.run.smithi064.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-10-09T05:28:42.534 INFO:teuthology.orchestra.run.smithi064.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-10-09T05:28:42.534 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:28:42.534 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-10-09T05:28:42.534 INFO:teuthology.orchestra.run.smithi064.stdout:============================================================================================ 2023-10-09T05:28:42.534 INFO:teuthology.orchestra.run.smithi064.stdout:Install 6 Packages 2023-10-09T05:28:42.534 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:28:42.535 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 53 M 2023-10-09T05:28:42.535 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 233 M 2023-10-09T05:28:42.535 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-10-09T05:28:42.569 INFO:teuthology.orchestra.run.smithi064.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 2.0 MB/s | 70 kB 00:00 2023-10-09T05:28:42.580 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-09T05:28:42.653 INFO:teuthology.orchestra.run.smithi064.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 4.2 MB/s | 323 kB 00:00 2023-10-09T05:28:42.671 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-09T05:28:42.720 INFO:teuthology.orchestra.run.smithi064.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 2.7 MB/s | 187 kB 00:00 2023-10-09T05:28:42.762 INFO:teuthology.orchestra.run.smithi064.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 4.8 MB/s | 202 kB 00:00 2023-10-09T05:28:42.787 INFO:teuthology.orchestra.run.smithi064.stdout:(5/6): libcephsqlite-17.2.6-1303.gb2903cb0.el8. 697 kB/s | 175 kB 00:00 2023-10-09T05:28:42.874 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-10-09T05:28:43.063 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-10-09T05:28:43.236 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-10-09T05:28:43.371 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/6 2023-10-09T05:28:45.062 INFO:teuthology.orchestra.run.smithi064.stdout:(6/6): ceph-test-17.2.6-1303.gb2903cb0.el8.x86_ 20 MB/s | 52 MB 00:02 2023-10-09T05:28:45.064 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:28:45.065 INFO:teuthology.orchestra.run.smithi064.stdout:Total 21 MB/s | 53 MB 00:02 2023-10-09T05:28:45.102 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-10-09T05:28:45.125 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-10-09T05:28:45.125 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-10-09T05:28:45.414 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-10-09T05:28:45.418 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-10-09T05:28:45.885 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-10-09T05:28:46.029 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-09T05:28:46.115 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-09T05:28:46.344 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-10-09T05:28:46.489 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-10-09T05:28:46.698 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-10-09T05:28:46.839 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/6 2023-10-09T05:28:48.275 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/6 2023-10-09T05:28:48.509 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/6 2023-10-09T05:28:48.732 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/6 2023-10-09T05:28:48.732 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/6 2023-10-09T05:28:48.732 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/6 2023-10-09T05:28:48.733 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-10-09T05:28:48.733 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-10-09T05:28:48.733 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-10-09T05:28:49.822 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-10-09T05:28:49.822 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:07 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:28:52.096 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/6 2023-10-09T05:28:52.239 INFO:teuthology.orchestra.run.smithi156.stdout:Installed products updated. 2023-10-09T05:28:52.342 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/6 2023-10-09T05:28:52.420 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:28:52.420 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2023-10-09T05:28:52.420 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:52.420 INFO:teuthology.orchestra.run.smithi156.stdout: jq-1.6-3.el8.x86_64 2023-10-09T05:28:52.421 INFO:teuthology.orchestra.run.smithi156.stdout: libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:52.421 INFO:teuthology.orchestra.run.smithi156.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-10-09T05:28:52.421 INFO:teuthology.orchestra.run.smithi156.stdout: socat-1.7.4.1-1.el8.x86_64 2023-10-09T05:28:52.421 INFO:teuthology.orchestra.run.smithi156.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-10-09T05:28:52.421 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:28:52.421 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:28:52.556 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph 2023-10-09T05:28:52.575 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/6 2023-10-09T05:28:52.575 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/6 2023-10-09T05:28:52.576 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/6 2023-10-09T05:28:52.576 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-10-09T05:28:52.576 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-10-09T05:28:52.576 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-10-09T05:28:52.891 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:28:53.687 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:11 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:28:53.705 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-10-09T05:28:53.706 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:11 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:28:55.119 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:28:55.122 INFO:teuthology.orchestra.run.smithi156.stdout:======================================================================================================= 2023-10-09T05:28:55.122 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2023-10-09T05:28:55.122 INFO:teuthology.orchestra.run.smithi156.stdout:======================================================================================================= 2023-10-09T05:28:55.122 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2023-10-09T05:28:55.122 INFO:teuthology.orchestra.run.smithi156.stdout: ceph x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 6.4 k 2023-10-09T05:28:55.122 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2023-10-09T05:28:55.123 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mds x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 2.3 M 2023-10-09T05:28:55.123 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 1.6 M 2023-10-09T05:28:55.123 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 238 k 2023-10-09T05:28:55.123 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mon x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.9 M 2023-10-09T05:28:55.123 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-osd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 17 M 2023-10-09T05:28:55.123 INFO:teuthology.orchestra.run.smithi156.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-10-09T05:28:55.123 INFO:teuthology.orchestra.run.smithi156.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-10-09T05:28:55.123 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-10-09T05:28:55.123 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-10-09T05:28:55.123 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-10-09T05:28:55.123 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-10-09T05:28:55.124 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-10-09T05:28:55.124 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-10-09T05:28:55.124 INFO:teuthology.orchestra.run.smithi156.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-10-09T05:28:55.124 INFO:teuthology.orchestra.run.smithi156.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-10-09T05:28:55.124 INFO:teuthology.orchestra.run.smithi156.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-10-09T05:28:55.124 INFO:teuthology.orchestra.run.smithi156.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-10-09T05:28:55.124 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-10-09T05:28:55.124 INFO:teuthology.orchestra.run.smithi156.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-10-09T05:28:55.124 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-10-09T05:28:55.124 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-10-09T05:28:55.124 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-10-09T05:28:55.124 INFO:teuthology.orchestra.run.smithi156.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-10-09T05:28:55.125 INFO:teuthology.orchestra.run.smithi156.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-10-09T05:28:55.125 INFO:teuthology.orchestra.run.smithi156.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-10-09T05:28:55.125 INFO:teuthology.orchestra.run.smithi156.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-10-09T05:28:55.125 INFO:teuthology.orchestra.run.smithi156.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-10-09T05:28:55.125 INFO:teuthology.orchestra.run.smithi156.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-10-09T05:28:55.125 INFO:teuthology.orchestra.run.smithi156.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-10-09T05:28:55.125 INFO:teuthology.orchestra.run.smithi156.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-10-09T05:28:55.125 INFO:teuthology.orchestra.run.smithi156.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-10-09T05:28:55.125 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:28:55.125 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2023-10-09T05:28:55.125 INFO:teuthology.orchestra.run.smithi156.stdout:======================================================================================================= 2023-10-09T05:28:55.125 INFO:teuthology.orchestra.run.smithi156.stdout:Install 32 Packages 2023-10-09T05:28:55.126 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:28:55.126 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 28 M 2023-10-09T05:28:55.126 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 110 M 2023-10-09T05:28:55.126 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2023-10-09T05:28:55.295 INFO:teuthology.orchestra.run.smithi156.stdout:(1/32): ceph-17.2.6-1303.gb2903cb0.el8.x86_64.r 57 kB/s | 6.4 kB 00:00 2023-10-09T05:28:55.494 INFO:teuthology.orchestra.run.smithi156.stdout:(2/32): ceph-mgr-17.2.6-1303.gb2903cb0.el8.x86_ 5.0 MB/s | 1.6 MB 00:00 2023-10-09T05:28:55.779 INFO:teuthology.orchestra.run.smithi156.stdout:(3/32): ceph-mon-17.2.6-1303.gb2903cb0.el8.x86_ 8.1 MB/s | 3.9 MB 00:00 2023-10-09T05:28:55.862 INFO:teuthology.orchestra.run.smithi156.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1303.gb290 2.8 MB/s | 238 kB 00:00 2023-10-09T05:28:56.129 INFO:teuthology.orchestra.run.smithi156.stdout:(5/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 163 kB/s | 44 kB 00:00 2023-10-09T05:28:56.170 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-10-09T05:28:56.213 INFO:teuthology.orchestra.run.smithi156.stdout:(6/32): ceph-mds-17.2.6-1303.gb2903cb0.el8.x86_ 2.2 MB/s | 2.3 MB 00:01 2023-10-09T05:28:56.368 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:28:56.368 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-10-09T05:28:56.369 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:56.369 INFO:teuthology.orchestra.run.smithi064.stdout: jq-1.6-3.el8.x86_64 2023-10-09T05:28:56.369 INFO:teuthology.orchestra.run.smithi064.stdout: libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:28:56.369 INFO:teuthology.orchestra.run.smithi064.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-10-09T05:28:56.369 INFO:teuthology.orchestra.run.smithi064.stdout: socat-1.7.4.1-1.el8.x86_64 2023-10-09T05:28:56.369 INFO:teuthology.orchestra.run.smithi064.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-10-09T05:28:56.369 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:28:56.369 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:28:56.397 INFO:teuthology.orchestra.run.smithi156.stdout:(7/32): ceph-osd-17.2.6-1303.gb2903cb0.el8.x86_ 19 MB/s | 17 MB 00:00 2023-10-09T05:28:56.430 INFO:teuthology.orchestra.run.smithi156.stdout:(8/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 616 kB/s | 185 kB 00:00 2023-10-09T05:28:56.505 INFO:teuthology.orchestra.run.smithi156.stdout:(9/32): python3-jaraco-6.2-6.el8.noarch.rpm 148 kB/s | 11 kB 00:00 2023-10-09T05:28:56.555 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph 2023-10-09T05:28:56.580 INFO:teuthology.orchestra.run.smithi156.stdout:(10/32): python3-jaraco-functools-2.0-4.el8.noa 236 kB/s | 18 kB 00:00 2023-10-09T05:28:56.664 INFO:teuthology.orchestra.run.smithi156.stdout:(11/32): python3-logutils-0.3.5-11.el8.noarch.r 584 kB/s | 49 kB 00:00 2023-10-09T05:28:56.698 INFO:teuthology.orchestra.run.smithi156.stdout:(12/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 356 kB/s | 173 kB 00:00 2023-10-09T05:28:56.749 INFO:teuthology.orchestra.run.smithi156.stdout:(13/32): python3-more-itertools-7.2.0-3.el8.noa 703 kB/s | 59 kB 00:00 2023-10-09T05:28:56.798 INFO:teuthology.orchestra.run.smithi156.stdout:(14/32): python3-cherrypy-18.4.0-1.el8.noarch.r 968 kB/s | 384 kB 00:00 2023-10-09T05:28:56.824 INFO:teuthology.orchestra.run.smithi156.stdout:(15/32): python3-portend-2.6-1.el8.noarch.rpm 207 kB/s | 16 kB 00:00 2023-10-09T05:28:56.849 INFO:teuthology.orchestra.run.smithi156.stdout:(16/32): python3-pecan-1.3.2-9.el8.noarch.rpm 1.8 MB/s | 283 kB 00:00 2023-10-09T05:28:56.874 INFO:teuthology.orchestra.run.smithi156.stdout:(17/32): python3-simplegeneric-0.8.1-17.el8.noa 248 kB/s | 19 kB 00:00 2023-10-09T05:28:56.899 INFO:teuthology.orchestra.run.smithi156.stdout:(18/32): python3-singledispatch-3.4.0.3-18.el8. 318 kB/s | 24 kB 00:00 2023-10-09T05:28:56.910 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:28:56.925 INFO:teuthology.orchestra.run.smithi156.stdout:(19/32): python3-tempora-1.14.1-5.el8.noarch.rp 387 kB/s | 29 kB 00:00 2023-10-09T05:28:56.950 INFO:teuthology.orchestra.run.smithi156.stdout:(20/32): python3-trustme-0.6.0-4.el8.noarch.rpm 365 kB/s | 27 kB 00:00 2023-10-09T05:28:57.018 INFO:teuthology.orchestra.run.smithi156.stdout:(21/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 2.7 MB/s | 251 kB 00:00 2023-10-09T05:28:57.043 INFO:teuthology.orchestra.run.smithi156.stdout:(22/32): python3-waitress-1.4.3-1.el8.noarch.rp 1.7 MB/s | 255 kB 00:00 2023-10-09T05:28:57.068 INFO:teuthology.orchestra.run.smithi156.stdout:(23/32): python3-webtest-2.0.33-1.el8.noarch.rp 726 kB/s | 85 kB 00:00 2023-10-09T05:28:57.094 INFO:teuthology.orchestra.run.smithi156.stdout:(24/32): python3-zc-lockfile-2.0-2.el8.noarch.r 304 kB/s | 23 kB 00:00 2023-10-09T05:28:57.160 INFO:teuthology.orchestra.run.smithi156.stdout:(25/32): python3-pytz-2017.2-9.el8.noarch.rpm 462 kB/s | 54 kB 00:00 2023-10-09T05:28:57.186 INFO:teuthology.orchestra.run.smithi156.stdout:(26/32): python3-markupsafe-0.23-19.el8.x86_64. 331 kB/s | 39 kB 00:00 2023-10-09T05:28:57.211 INFO:teuthology.orchestra.run.smithi156.stdout:(27/32): python3-mako-1.0.6-13.el8.noarch.rpm 1.4 MB/s | 157 kB 00:00 2023-10-09T05:28:57.244 INFO:teuthology.orchestra.run.smithi156.stdout:(28/32): python3-werkzeug-0.12.2-4.el8.noarch.r 5.3 MB/s | 457 kB 00:00 2023-10-09T05:28:57.263 INFO:teuthology.orchestra.run.smithi156.stdout:(29/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 1.3 MB/s | 103 kB 00:00 2023-10-09T05:28:57.287 INFO:teuthology.orchestra.run.smithi156.stdout:(30/32): python3-pycparser-2.14-14.el8.noarch.r 1.4 MB/s | 109 kB 00:00 2023-10-09T05:28:57.313 INFO:teuthology.orchestra.run.smithi156.stdout:(31/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 4.6 MB/s | 238 kB 00:00 2023-10-09T05:28:57.339 INFO:teuthology.orchestra.run.smithi156.stdout:(32/32): python3-cryptography-3.2.1-5.el8.x86_6 5.9 MB/s | 559 kB 00:00 2023-10-09T05:28:57.343 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:28:57.344 INFO:teuthology.orchestra.run.smithi156.stdout:Total 13 MB/s | 28 MB 00:02 2023-10-09T05:28:57.432 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2023-10-09T05:28:57.467 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2023-10-09T05:28:57.467 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2023-10-09T05:28:57.671 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2023-10-09T05:28:57.676 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2023-10-09T05:28:57.769 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:15 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:28:58.757 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2023-10-09T05:28:58.950 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-10-09T05:28:59.239 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-10-09T05:28:59.296 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:28:59.300 INFO:teuthology.orchestra.run.smithi064.stdout:======================================================================================================= 2023-10-09T05:28:59.300 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2023-10-09T05:28:59.300 INFO:teuthology.orchestra.run.smithi064.stdout:======================================================================================================= 2023-10-09T05:28:59.300 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-10-09T05:28:59.300 INFO:teuthology.orchestra.run.smithi064.stdout: ceph x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 6.4 k 2023-10-09T05:28:59.300 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2023-10-09T05:28:59.300 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mds x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 2.3 M 2023-10-09T05:28:59.301 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 1.6 M 2023-10-09T05:28:59.301 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 238 k 2023-10-09T05:28:59.301 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mon x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.9 M 2023-10-09T05:28:59.301 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-osd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 17 M 2023-10-09T05:28:59.301 INFO:teuthology.orchestra.run.smithi064.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-10-09T05:28:59.301 INFO:teuthology.orchestra.run.smithi064.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-10-09T05:28:59.301 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-10-09T05:28:59.301 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-10-09T05:28:59.301 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-10-09T05:28:59.301 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-10-09T05:28:59.301 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-10-09T05:28:59.301 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-10-09T05:28:59.302 INFO:teuthology.orchestra.run.smithi064.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-10-09T05:28:59.302 INFO:teuthology.orchestra.run.smithi064.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-10-09T05:28:59.302 INFO:teuthology.orchestra.run.smithi064.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-10-09T05:28:59.302 INFO:teuthology.orchestra.run.smithi064.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-10-09T05:28:59.302 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-10-09T05:28:59.302 INFO:teuthology.orchestra.run.smithi064.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-10-09T05:28:59.302 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-10-09T05:28:59.302 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-10-09T05:28:59.302 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-10-09T05:28:59.302 INFO:teuthology.orchestra.run.smithi064.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-10-09T05:28:59.302 INFO:teuthology.orchestra.run.smithi064.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-10-09T05:28:59.303 INFO:teuthology.orchestra.run.smithi064.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-10-09T05:28:59.303 INFO:teuthology.orchestra.run.smithi064.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-10-09T05:28:59.303 INFO:teuthology.orchestra.run.smithi064.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-10-09T05:28:59.303 INFO:teuthology.orchestra.run.smithi064.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-10-09T05:28:59.303 INFO:teuthology.orchestra.run.smithi064.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-10-09T05:28:59.303 INFO:teuthology.orchestra.run.smithi064.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-10-09T05:28:59.303 INFO:teuthology.orchestra.run.smithi064.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-10-09T05:28:59.303 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:28:59.303 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-10-09T05:28:59.303 INFO:teuthology.orchestra.run.smithi064.stdout:======================================================================================================= 2023-10-09T05:28:59.303 INFO:teuthology.orchestra.run.smithi064.stdout:Install 32 Packages 2023-10-09T05:28:59.303 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:28:59.304 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 28 M 2023-10-09T05:28:59.304 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 110 M 2023-10-09T05:28:59.304 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-10-09T05:28:59.462 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 3/32 2023-10-09T05:28:59.788 INFO:teuthology.orchestra.run.smithi064.stdout:(1/32): ceph-17.2.6-1303.gb2903cb0.el8.x86_64.r 53 kB/s | 6.4 kB 00:00 2023-10-09T05:28:59.843 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 4/32 2023-10-09T05:29:00.013 INFO:teuthology.orchestra.run.smithi064.stdout:(2/32): ceph-mds-17.2.6-1303.gb2903cb0.el8.x86_ 6.7 MB/s | 2.3 MB 00:00 2023-10-09T05:29:00.047 INFO:teuthology.orchestra.run.smithi064.stdout:(3/32): ceph-mgr-17.2.6-1303.gb2903cb0.el8.x86_ 4.1 MB/s | 1.6 MB 00:00 2023-10-09T05:29:00.113 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 5/32 2023-10-09T05:29:00.130 INFO:teuthology.orchestra.run.smithi064.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1303.gb290 2.8 MB/s | 238 kB 00:00 2023-10-09T05:29:00.276 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 6/32 2023-10-09T05:29:00.297 INFO:teuthology.orchestra.run.smithi064.stdout:(5/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 262 kB/s | 44 kB 00:00 2023-10-09T05:29:00.397 INFO:teuthology.orchestra.run.smithi064.stdout:(6/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 1.8 MB/s | 185 kB 00:00 2023-10-09T05:29:00.456 INFO:teuthology.orchestra.run.smithi064.stdout:(7/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 2.9 MB/s | 173 kB 00:00 2023-10-09T05:29:00.461 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 7/32 2023-10-09T05:29:00.531 INFO:teuthology.orchestra.run.smithi064.stdout:(8/32): python3-cherrypy-18.4.0-1.el8.noarch.rp 5.0 MB/s | 384 kB 00:00 2023-10-09T05:29:00.607 INFO:teuthology.orchestra.run.smithi064.stdout:(9/32): python3-jaraco-6.2-6.el8.noarch.rpm 148 kB/s | 11 kB 00:00 2023-10-09T05:29:00.648 INFO:teuthology.orchestra.run.smithi064.stdout:(10/32): python3-jaraco-functools-2.0-4.el8.noa 424 kB/s | 18 kB 00:00 2023-10-09T05:29:00.715 INFO:teuthology.orchestra.run.smithi064.stdout:(11/32): ceph-mon-17.2.6-1303.gb2903cb0.el8.x86 4.2 MB/s | 3.9 MB 00:00 2023-10-09T05:29:00.732 INFO:teuthology.orchestra.run.smithi064.stdout:(12/32): python3-logutils-0.3.5-11.el8.noarch.r 583 kB/s | 49 kB 00:00 2023-10-09T05:29:00.745 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 8/32 2023-10-09T05:29:00.916 INFO:teuthology.orchestra.run.smithi064.stdout:(13/32): ceph-osd-17.2.6-1303.gb2903cb0.el8.x86 19 MB/s | 17 MB 00:00 2023-10-09T05:29:00.930 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 9/32 2023-10-09T05:29:00.969 INFO:teuthology.orchestra.run.smithi064.stdout:(14/32): python3-pecan-1.3.2-9.el8.noarch.rpm 1.2 MB/s | 283 kB 00:00 2023-10-09T05:29:01.017 INFO:teuthology.orchestra.run.smithi064.stdout:(15/32): python3-more-itertools-7.2.0-3.el8.noa 195 kB/s | 59 kB 00:00 2023-10-09T05:29:01.034 INFO:teuthology.orchestra.run.smithi064.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 313 kB/s | 19 kB 00:00 2023-10-09T05:29:01.059 INFO:teuthology.orchestra.run.smithi064.stdout:(17/32): python3-singledispatch-3.4.0.3-18.el8. 583 kB/s | 24 kB 00:00 2023-10-09T05:29:01.076 INFO:teuthology.orchestra.run.smithi064.stdout:(18/32): python3-portend-2.6-1.el8.noarch.rpm 98 kB/s | 16 kB 00:00 2023-10-09T05:29:01.093 INFO:teuthology.orchestra.run.smithi064.stdout:(19/32): python3-tempora-1.14.1-5.el8.noarch.rp 496 kB/s | 29 kB 00:00 2023-10-09T05:29:01.110 INFO:teuthology.orchestra.run.smithi064.stdout:(20/32): python3-trustme-0.6.0-4.el8.noarch.rpm 547 kB/s | 27 kB 00:00 2023-10-09T05:29:01.135 INFO:teuthology.orchestra.run.smithi064.stdout:(21/32): python3-waitress-1.4.3-1.el8.noarch.rp 4.2 MB/s | 255 kB 00:00 2023-10-09T05:29:01.162 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 10/32 2023-10-09T05:29:01.168 INFO:teuthology.orchestra.run.smithi064.stdout:(22/32): python3-webtest-2.0.33-1.el8.noarch.rp 1.4 MB/s | 85 kB 00:00 2023-10-09T05:29:01.194 INFO:teuthology.orchestra.run.smithi064.stdout:(23/32): python3-zc-lockfile-2.0-2.el8.noarch.r 393 kB/s | 23 kB 00:00 2023-10-09T05:29:01.227 INFO:teuthology.orchestra.run.smithi064.stdout:(24/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 1.8 MB/s | 251 kB 00:00 2023-10-09T05:29:01.286 INFO:teuthology.orchestra.run.smithi064.stdout:(25/32): python3-pycparser-2.14-14.el8.noarch.r 933 kB/s | 109 kB 00:00 2023-10-09T05:29:01.311 INFO:teuthology.orchestra.run.smithi064.stdout:(26/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 3.5 MB/s | 238 kB 00:00 2023-10-09T05:29:01.355 INFO:teuthology.orchestra.run.smithi064.stdout:(27/32): python3-pytz-2017.2-9.el8.noarch.rpm 785 kB/s | 54 kB 00:00 2023-10-09T05:29:01.387 INFO:teuthology.orchestra.run.smithi064.stdout:(28/32): python3-markupsafe-0.23-19.el8.x86_64. 517 kB/s | 39 kB 00:00 2023-10-09T05:29:01.412 INFO:teuthology.orchestra.run.smithi064.stdout:(29/32): python3-cryptography-3.2.1-5.el8.x86_6 2.5 MB/s | 559 kB 00:00 2023-10-09T05:29:01.414 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 11/32 2023-10-09T05:29:01.438 INFO:teuthology.orchestra.run.smithi064.stdout:(30/32): python3-mako-1.0.6-13.el8.noarch.rpm 1.9 MB/s | 157 kB 00:00 2023-10-09T05:29:01.463 INFO:teuthology.orchestra.run.smithi064.stdout:(31/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 2.0 MB/s | 103 kB 00:00 2023-10-09T05:29:01.488 INFO:teuthology.orchestra.run.smithi064.stdout:(32/32): python3-werkzeug-0.12.2-4.el8.noarch.r 4.4 MB/s | 457 kB 00:00 2023-10-09T05:29:01.492 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:29:01.492 INFO:teuthology.orchestra.run.smithi064.stdout:Total 13 MB/s | 28 MB 00:02 2023-10-09T05:29:01.590 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-10-09T05:29:01.609 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 12/32 2023-10-09T05:29:01.629 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-10-09T05:29:01.630 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-10-09T05:29:01.818 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-10-09T05:29:01.848 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-10-09T05:29:01.852 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-10-09T05:29:01.983 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-10-09T05:29:02.139 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-10-09T05:29:02.302 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-10-09T05:29:02.454 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-10-09T05:29:02.582 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-10-09T05:29:02.801 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-10-09T05:29:03.020 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-10-09T05:29:03.059 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-10-09T05:29:03.223 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-10-09T05:29:03.261 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-10-09T05:29:03.475 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-10-09T05:29:03.582 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-10-09T05:29:03.721 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-10-09T05:29:03.831 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-10-09T05:29:03.942 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-10-09T05:29:04.049 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-10-09T05:29:04.225 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-10-09T05:29:04.273 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-10-09T05:29:04.499 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-10-09T05:29:04.732 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-10-09T05:29:05.107 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-10-09T05:29:05.113 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-10-09T05:29:05.389 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-10-09T05:29:05.448 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8. 27/32 2023-10-09T05:29:05.551 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-10-09T05:29:05.616 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 28/32 2023-10-09T05:29:05.753 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-10-09T05:29:05.935 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-10-09T05:29:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-10-09T05:29:06.375 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-10-09T05:29:06.590 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-10-09T05:29:06.748 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-10-09T05:29:06.938 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-10-09T05:29:07.094 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 28/32 2023-10-09T05:29:07.099 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-10-09T05:29:07.316 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 29/32 2023-10-09T05:29:07.368 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-10-09T05:29:07.718 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 29/32 2023-10-09T05:29:07.868 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 30/32 2023-10-09T05:29:07.969 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-10-09T05:29:08.084 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 30/32 2023-10-09T05:29:08.242 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 31/32 2023-10-09T05:29:08.261 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 31/32 2023-10-09T05:29:08.390 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 32/32 2023-10-09T05:29:08.441 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-10-09T05:29:08.710 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-10-09T05:29:08.742 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 32/32 2023-10-09T05:29:08.743 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/32 2023-10-09T05:29:08.743 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/32 2023-10-09T05:29:08.743 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/32 2023-10-09T05:29:08.743 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 4/32 2023-10-09T05:29:08.743 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/32 2023-10-09T05:29:08.743 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8. 6/32 2023-10-09T05:29:08.743 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-10-09T05:29:08.743 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-10-09T05:29:08.743 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-10-09T05:29:08.744 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-10-09T05:29:08.744 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-10-09T05:29:08.744 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-10-09T05:29:08.744 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-10-09T05:29:08.744 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-10-09T05:29:08.744 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-10-09T05:29:08.744 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-10-09T05:29:08.744 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-10-09T05:29:08.744 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-10-09T05:29:08.744 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-10-09T05:29:08.744 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-10-09T05:29:08.744 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-10-09T05:29:08.745 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-10-09T05:29:08.745 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-10-09T05:29:08.745 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-10-09T05:29:08.745 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 25/32 2023-10-09T05:29:08.745 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 26/32 2023-10-09T05:29:08.745 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 27/32 2023-10-09T05:29:08.745 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 28/32 2023-10-09T05:29:08.745 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 29/32 2023-10-09T05:29:08.745 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 30/32 2023-10-09T05:29:08.745 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 31/32 2023-10-09T05:29:08.994 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-10-09T05:29:09.247 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-10-09T05:29:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-10-09T05:29:09.879 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-10-09T05:29:10.240 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8. 27/32 2023-10-09T05:29:10.251 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 32/32 2023-10-09T05:29:10.251 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:28 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:29:10.458 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 28/32 2023-10-09T05:29:12.030 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 28/32 2023-10-09T05:29:12.258 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 29/32 2023-10-09T05:29:12.679 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 29/32 2023-10-09T05:29:12.842 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 30/32 2023-10-09T05:29:13.071 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 30/32 2023-10-09T05:29:13.225 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 31/32 2023-10-09T05:29:13.244 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 31/32 2023-10-09T05:29:13.399 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 32/32 2023-10-09T05:29:13.776 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 32/32 2023-10-09T05:29:13.776 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/32 2023-10-09T05:29:13.776 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/32 2023-10-09T05:29:13.776 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/32 2023-10-09T05:29:13.776 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 4/32 2023-10-09T05:29:13.776 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/32 2023-10-09T05:29:13.776 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8. 6/32 2023-10-09T05:29:13.777 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-10-09T05:29:13.777 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-10-09T05:29:13.777 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-10-09T05:29:13.777 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-10-09T05:29:13.777 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-10-09T05:29:13.777 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-10-09T05:29:13.777 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-10-09T05:29:13.777 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-10-09T05:29:13.777 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-10-09T05:29:13.777 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-10-09T05:29:13.777 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-10-09T05:29:13.778 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-10-09T05:29:13.778 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-10-09T05:29:13.778 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-10-09T05:29:13.778 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-10-09T05:29:13.778 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-10-09T05:29:13.778 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-10-09T05:29:13.778 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-10-09T05:29:13.778 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-10-09T05:29:13.778 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-10-09T05:29:13.778 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-10-09T05:29:13.779 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-10-09T05:29:13.779 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-10-09T05:29:13.779 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-10-09T05:29:13.779 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-10-09T05:29:14.392 INFO:teuthology.orchestra.run.smithi156.stdout:Installed products updated. 2023-10-09T05:29:14.578 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:29:14.578 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2023-10-09T05:29:14.578 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:29:14.578 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:29:14.578 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:29:14.578 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:29:14.578 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:29:14.579 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:29:14.579 INFO:teuthology.orchestra.run.smithi156.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-10-09T05:29:14.579 INFO:teuthology.orchestra.run.smithi156.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-10-09T05:29:14.579 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-10-09T05:29:14.579 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-10-09T05:29:14.579 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-10-09T05:29:14.579 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-10-09T05:29:14.579 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jaraco-6.2-6.el8.noarch 2023-10-09T05:29:14.579 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-10-09T05:29:14.579 INFO:teuthology.orchestra.run.smithi156.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-10-09T05:29:14.579 INFO:teuthology.orchestra.run.smithi156.stdout: python3-mako-1.0.6-13.el8.noarch 2023-10-09T05:29:14.579 INFO:teuthology.orchestra.run.smithi156.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-10-09T05:29:14.580 INFO:teuthology.orchestra.run.smithi156.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-10-09T05:29:14.580 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-10-09T05:29:14.580 INFO:teuthology.orchestra.run.smithi156.stdout: python3-portend-2.6-1.el8.noarch 2023-10-09T05:29:14.580 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-10-09T05:29:14.580 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pycparser-2.14-14.el8.noarch 2023-10-09T05:29:14.580 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pytz-2017.2-9.el8.noarch 2023-10-09T05:29:14.580 INFO:teuthology.orchestra.run.smithi156.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-10-09T05:29:14.580 INFO:teuthology.orchestra.run.smithi156.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-10-09T05:29:14.580 INFO:teuthology.orchestra.run.smithi156.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-10-09T05:29:14.580 INFO:teuthology.orchestra.run.smithi156.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-10-09T05:29:14.580 INFO:teuthology.orchestra.run.smithi156.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-10-09T05:29:14.580 INFO:teuthology.orchestra.run.smithi156.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-10-09T05:29:14.581 INFO:teuthology.orchestra.run.smithi156.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-10-09T05:29:14.581 INFO:teuthology.orchestra.run.smithi156.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-10-09T05:29:14.581 INFO:teuthology.orchestra.run.smithi156.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-10-09T05:29:14.581 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:29:14.581 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:29:14.748 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-base 2023-10-09T05:29:15.099 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:29:15.260 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-10-09T05:29:15.260 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:33 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:29:15.946 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:33 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:29:17.199 INFO:teuthology.orchestra.run.smithi156.stdout:Package ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:29:17.309 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:29:17.311 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2023-10-09T05:29:17.311 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:29:17.420 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install cephadm 2023-10-09T05:29:17.765 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:29:18.902 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:36 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:29:18.959 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-10-09T05:29:19.168 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:29:19.168 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-10-09T05:29:19.168 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:29:19.168 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:29:19.168 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:29:19.168 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:29:19.168 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:29:19.168 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:29:19.168 INFO:teuthology.orchestra.run.smithi064.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-10-09T05:29:19.168 INFO:teuthology.orchestra.run.smithi064.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-10-09T05:29:19.169 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-10-09T05:29:19.169 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-10-09T05:29:19.169 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-10-09T05:29:19.169 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-10-09T05:29:19.169 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-6.2-6.el8.noarch 2023-10-09T05:29:19.169 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-10-09T05:29:19.169 INFO:teuthology.orchestra.run.smithi064.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-10-09T05:29:19.169 INFO:teuthology.orchestra.run.smithi064.stdout: python3-mako-1.0.6-13.el8.noarch 2023-10-09T05:29:19.169 INFO:teuthology.orchestra.run.smithi064.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-10-09T05:29:19.169 INFO:teuthology.orchestra.run.smithi064.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-10-09T05:29:19.169 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-10-09T05:29:19.169 INFO:teuthology.orchestra.run.smithi064.stdout: python3-portend-2.6-1.el8.noarch 2023-10-09T05:29:19.170 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-10-09T05:29:19.170 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pycparser-2.14-14.el8.noarch 2023-10-09T05:29:19.170 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pytz-2017.2-9.el8.noarch 2023-10-09T05:29:19.170 INFO:teuthology.orchestra.run.smithi064.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-10-09T05:29:19.170 INFO:teuthology.orchestra.run.smithi064.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-10-09T05:29:19.170 INFO:teuthology.orchestra.run.smithi064.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-10-09T05:29:19.170 INFO:teuthology.orchestra.run.smithi064.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-10-09T05:29:19.170 INFO:teuthology.orchestra.run.smithi064.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-10-09T05:29:19.170 INFO:teuthology.orchestra.run.smithi064.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-10-09T05:29:19.170 INFO:teuthology.orchestra.run.smithi064.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-10-09T05:29:19.170 INFO:teuthology.orchestra.run.smithi064.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-10-09T05:29:19.170 INFO:teuthology.orchestra.run.smithi064.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-10-09T05:29:19.170 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:29:19.171 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:29:20.268 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:29:20.268 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:29:20.268 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2023-10-09T05:29:20.268 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:29:20.269 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2023-10-09T05:29:20.269 INFO:teuthology.orchestra.run.smithi156.stdout: cephadm noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 80 k 2023-10-09T05:29:20.269 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:29:20.269 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2023-10-09T05:29:20.269 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:29:20.269 INFO:teuthology.orchestra.run.smithi156.stdout:Install 1 Package 2023-10-09T05:29:20.269 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:29:20.270 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 80 k 2023-10-09T05:29:20.270 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 364 k 2023-10-09T05:29:20.270 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2023-10-09T05:29:20.785 INFO:teuthology.orchestra.run.smithi156.stdout:cephadm-17.2.6-1303.gb2903cb0.el8.noarch.rpm 156 kB/s | 80 kB 00:00 2023-10-09T05:29:20.786 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:29:20.786 INFO:teuthology.orchestra.run.smithi156.stdout:Total 155 kB/s | 80 kB 00:00 2023-10-09T05:29:20.786 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2023-10-09T05:29:20.794 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2023-10-09T05:29:20.795 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2023-10-09T05:29:20.796 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2023-10-09T05:29:20.800 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2023-10-09T05:29:20.843 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2023-10-09T05:29:21.096 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-base 2023-10-09T05:29:21.467 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:29:22.361 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:40 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:29:23.255 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T05:29:23.452 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T05:29:23.619 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T05:29:23.761 INFO:teuthology.orchestra.run.smithi064.stdout:Package ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:29:23.878 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:29:23.879 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2023-10-09T05:29:23.880 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:29:24.007 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install cephadm 2023-10-09T05:29:24.373 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:29:24.691 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T05:29:24.691 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:42 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:29:25.869 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:43 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:29:27.402 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:29:27.403 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:29:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2023-10-09T05:29:27.403 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:29:27.403 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-10-09T05:29:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: cephadm noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 80 k 2023-10-09T05:29:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:29:27.403 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-10-09T05:29:27.404 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:29:27.404 INFO:teuthology.orchestra.run.smithi064.stdout:Install 1 Package 2023-10-09T05:29:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:29:27.404 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 80 k 2023-10-09T05:29:27.404 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 364 k 2023-10-09T05:29:27.404 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-10-09T05:29:27.490 INFO:teuthology.orchestra.run.smithi156.stdout:Installed products updated. 2023-10-09T05:29:27.561 INFO:teuthology.orchestra.run.smithi064.stdout:cephadm-17.2.6-1303.gb2903cb0.el8.noarch.rpm 511 kB/s | 80 kB 00:00 2023-10-09T05:29:27.562 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:29:27.562 INFO:teuthology.orchestra.run.smithi064.stdout:Total 507 kB/s | 80 kB 00:00 2023-10-09T05:29:27.562 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-10-09T05:29:27.571 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-10-09T05:29:27.571 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-10-09T05:29:27.573 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-10-09T05:29:27.577 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-10-09T05:29:27.672 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:29:27.672 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2023-10-09T05:29:27.672 INFO:teuthology.orchestra.run.smithi156.stdout: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:29:27.672 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:29:27.672 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:29:27.731 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-10-09T05:29:27.810 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-immutable-object-cache 2023-10-09T05:29:28.167 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:29:29.089 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:47 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:29:29.557 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T05:29:29.761 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T05:29:29.935 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T05:29:30.478 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:29:30.479 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:29:30.479 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repo Size 2023-10-09T05:29:30.479 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:29:30.479 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2023-10-09T05:29:30.479 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 161 k 2023-10-09T05:29:30.479 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:29:30.479 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2023-10-09T05:29:30.479 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:29:30.479 INFO:teuthology.orchestra.run.smithi156.stdout:Install 1 Package 2023-10-09T05:29:30.480 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:29:30.480 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 161 k 2023-10-09T05:29:30.480 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 490 k 2023-10-09T05:29:30.480 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2023-10-09T05:29:30.667 INFO:teuthology.orchestra.run.smithi156.stdout:ceph-immutable-object-cache-17.2.6-1303.gb2903c 866 kB/s | 161 kB 00:00 2023-10-09T05:29:30.667 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:29:30.667 INFO:teuthology.orchestra.run.smithi156.stdout:Total 860 kB/s | 161 kB 00:00 2023-10-09T05:29:30.668 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2023-10-09T05:29:30.678 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2023-10-09T05:29:30.678 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2023-10-09T05:29:30.729 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2023-10-09T05:29:30.733 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2023-10-09T05:29:30.851 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2023-10-09T05:29:31.012 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T05:29:31.012 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:49 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:29:31.030 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T05:29:31.379 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T05:29:32.402 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T05:29:32.402 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:50 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:29:33.591 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-10-09T05:29:33.788 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:29:33.788 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-10-09T05:29:33.788 INFO:teuthology.orchestra.run.smithi064.stdout: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:29:33.788 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:29:33.789 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:29:33.957 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-immutable-object-cache 2023-10-09T05:29:34.338 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:29:34.988 INFO:teuthology.orchestra.run.smithi156.stdout:Installed products updated. 2023-10-09T05:29:35.169 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:29:35.169 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2023-10-09T05:29:35.169 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:29:35.169 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:29:35.170 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:29:35.303 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:53 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:29:35.305 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-mgr 2023-10-09T05:29:35.654 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:29:36.532 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:54 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:29:36.824 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:29:36.825 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:29:36.825 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repo Size 2023-10-09T05:29:36.825 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:29:36.825 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-10-09T05:29:36.825 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 161 k 2023-10-09T05:29:36.825 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:29:36.825 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-10-09T05:29:36.826 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:29:36.826 INFO:teuthology.orchestra.run.smithi064.stdout:Install 1 Package 2023-10-09T05:29:36.826 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:29:36.826 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 161 k 2023-10-09T05:29:36.826 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 490 k 2023-10-09T05:29:36.826 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-10-09T05:29:37.009 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-immutable-object-cache-17.2.6-1303.gb2903c 880 kB/s | 161 kB 00:00 2023-10-09T05:29:37.010 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:29:37.010 INFO:teuthology.orchestra.run.smithi064.stdout:Total 874 kB/s | 161 kB 00:00 2023-10-09T05:29:37.010 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-10-09T05:29:37.022 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-10-09T05:29:37.022 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-10-09T05:29:37.076 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-10-09T05:29:37.080 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-10-09T05:29:37.193 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-10-09T05:29:37.389 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T05:29:37.751 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T05:29:37.862 INFO:teuthology.orchestra.run.smithi156.stdout:Package ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:29:37.971 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:29:37.972 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2023-10-09T05:29:37.973 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:29:38.081 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-mgr-dashboard 2023-10-09T05:29:38.428 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:29:38.851 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T05:29:38.851 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:56 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:29:40.047 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:58 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:29:41.480 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:29:41.481 INFO:teuthology.orchestra.run.smithi156.stdout:=================================================================================================== 2023-10-09T05:29:41.481 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2023-10-09T05:29:41.481 INFO:teuthology.orchestra.run.smithi156.stdout:=================================================================================================== 2023-10-09T05:29:41.481 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2023-10-09T05:29:41.481 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 1.7 M 2023-10-09T05:29:41.481 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2023-10-09T05:29:41.481 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 23 k 2023-10-09T05:29:41.481 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 14 k 2023-10-09T05:29:41.481 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-09T05:29:41.482 INFO:teuthology.orchestra.run.smithi156.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-10-09T05:29:41.482 INFO:teuthology.orchestra.run.smithi156.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-10-09T05:29:41.482 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:29:41.482 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2023-10-09T05:29:41.482 INFO:teuthology.orchestra.run.smithi156.stdout:=================================================================================================== 2023-10-09T05:29:41.482 INFO:teuthology.orchestra.run.smithi156.stdout:Install 6 Packages 2023-10-09T05:29:41.482 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:29:41.483 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 2.0 M 2023-10-09T05:29:41.483 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 8.0 M 2023-10-09T05:29:41.483 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2023-10-09T05:29:41.665 INFO:teuthology.orchestra.run.smithi156.stdout:(1/6): ceph-prometheus-alerts-17.2.6-1303.gb290 97 kB/s | 14 kB 00:00 2023-10-09T05:29:41.681 INFO:teuthology.orchestra.run.smithi156.stdout:(2/6): ceph-grafana-dashboards-17.2.6-1303.gb29 138 kB/s | 23 kB 00:00 2023-10-09T05:29:41.873 INFO:teuthology.orchestra.run.smithi156.stdout:(3/6): ceph-mgr-dashboard-17.2.6-1303.gb2903cb0 4.7 MB/s | 1.7 MB 00:00 2023-10-09T05:29:41.899 INFO:teuthology.orchestra.run.smithi156.stdout:(4/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 144 kB/s | 34 kB 00:00 2023-10-09T05:29:42.049 INFO:teuthology.orchestra.run.smithi156.stdout:(5/6): python3-routes-2.4.1-12.el8.noarch.rpm 535 kB/s | 196 kB 00:00 2023-10-09T05:29:42.065 INFO:teuthology.orchestra.run.smithi156.stdout:(6/6): python3-jwt-1.6.1-2.el8.noarch.rpm 222 kB/s | 43 kB 00:00 2023-10-09T05:29:42.067 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:29:42.067 INFO:teuthology.orchestra.run.smithi156.stdout:Total 3.4 MB/s | 2.0 MB 00:00 2023-10-09T05:29:42.078 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2023-10-09T05:29:42.087 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2023-10-09T05:29:42.087 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2023-10-09T05:29:42.143 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2023-10-09T05:29:42.147 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2023-10-09T05:29:42.422 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2023-10-09T05:29:42.621 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-10-09T05:29:42.801 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-10-09T05:29:42.972 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-10-09T05:29:43.093 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.n 4/6 2023-10-09T05:29:43.353 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-10-09T05:29:43.415 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8. 5/6 2023-10-09T05:29:43.549 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:29:43.549 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-10-09T05:29:43.549 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:29:43.549 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:29:43.549 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:29:43.640 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 6/6 2023-10-09T05:29:43.686 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-mgr 2023-10-09T05:29:43.880 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 6/6 2023-10-09T05:29:43.880 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8. 1/6 2023-10-09T05:29:43.881 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 2/6 2023-10-09T05:29:43.881 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.n 3/6 2023-10-09T05:29:43.881 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-10-09T05:29:43.881 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-10-09T05:29:44.064 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:29:44.929 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-10-09T05:29:44.930 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:02:02 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:29:45.707 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:02:03 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:29:47.149 INFO:teuthology.orchestra.run.smithi064.stdout:Package ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:29:47.274 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:29:47.275 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2023-10-09T05:29:47.275 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:29:47.401 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-mgr-dashboard 2023-10-09T05:29:47.766 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:29:49.443 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:02:07 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:29:50.995 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:29:50.997 INFO:teuthology.orchestra.run.smithi064.stdout:=================================================================================================== 2023-10-09T05:29:50.997 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2023-10-09T05:29:50.997 INFO:teuthology.orchestra.run.smithi064.stdout:=================================================================================================== 2023-10-09T05:29:50.997 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-10-09T05:29:50.997 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 1.7 M 2023-10-09T05:29:50.997 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2023-10-09T05:29:50.997 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 23 k 2023-10-09T05:29:50.997 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 14 k 2023-10-09T05:29:50.997 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-09T05:29:50.997 INFO:teuthology.orchestra.run.smithi064.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-10-09T05:29:50.998 INFO:teuthology.orchestra.run.smithi064.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-10-09T05:29:50.998 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:29:50.998 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-10-09T05:29:50.998 INFO:teuthology.orchestra.run.smithi064.stdout:=================================================================================================== 2023-10-09T05:29:50.998 INFO:teuthology.orchestra.run.smithi064.stdout:Install 6 Packages 2023-10-09T05:29:50.998 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:29:50.998 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 2.0 M 2023-10-09T05:29:50.999 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 8.0 M 2023-10-09T05:29:50.999 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-10-09T05:29:51.580 INFO:teuthology.orchestra.run.smithi064.stdout:(1/6): ceph-prometheus-alerts-17.2.6-1303.gb290 61 kB/s | 14 kB 00:00 2023-10-09T05:29:51.605 INFO:teuthology.orchestra.run.smithi064.stdout:(2/6): ceph-grafana-dashboards-17.2.6-1303.gb29 87 kB/s | 23 kB 00:00 2023-10-09T05:29:51.781 INFO:teuthology.orchestra.run.smithi064.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 168 kB/s | 34 kB 00:00 2023-10-09T05:29:51.831 INFO:teuthology.orchestra.run.smithi064.stdout:(4/6): python3-routes-2.4.1-12.el8.noarch.rpm 872 kB/s | 196 kB 00:00 2023-10-09T05:29:51.856 INFO:teuthology.orchestra.run.smithi064.stdout:(5/6): python3-jwt-1.6.1-2.el8.noarch.rpm 566 kB/s | 43 kB 00:00 2023-10-09T05:29:51.906 INFO:teuthology.orchestra.run.smithi064.stdout:(6/6): ceph-mgr-dashboard-17.2.6-1303.gb2903cb0 3.0 MB/s | 1.7 MB 00:00 2023-10-09T05:29:51.908 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:29:51.908 INFO:teuthology.orchestra.run.smithi064.stdout:Total 2.2 MB/s | 2.0 MB 00:00 2023-10-09T05:29:51.921 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-10-09T05:29:51.931 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-10-09T05:29:51.932 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-10-09T05:29:51.995 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-10-09T05:29:51.999 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-10-09T05:29:52.253 INFO:teuthology.orchestra.run.smithi156.stdout:Installed products updated. 2023-10-09T05:29:52.341 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-10-09T05:29:52.434 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:29:52.434 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2023-10-09T05:29:52.434 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:29:52.434 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:29:52.434 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:29:52.434 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-10-09T05:29:52.434 INFO:teuthology.orchestra.run.smithi156.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-10-09T05:29:52.434 INFO:teuthology.orchestra.run.smithi156.stdout: python3-routes-2.4.1-12.el8.noarch 2023-10-09T05:29:52.435 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:29:52.435 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:29:52.563 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-10-09T05:29:52.568 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-mgr-diskprediction-local 2023-10-09T05:29:52.753 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-10-09T05:29:52.916 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:29:52.956 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-10-09T05:29:53.085 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.n 4/6 2023-10-09T05:29:53.717 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8. 5/6 2023-10-09T05:29:53.913 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 6/6 2023-10-09T05:29:54.146 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 6/6 2023-10-09T05:29:54.146 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8. 1/6 2023-10-09T05:29:54.146 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 2/6 2023-10-09T05:29:54.146 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.n 3/6 2023-10-09T05:29:54.146 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-10-09T05:29:54.146 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-10-09T05:29:54.355 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:02:12 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:29:55.229 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-10-09T05:29:55.229 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:02:13 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:29:55.722 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:29:55.722 INFO:teuthology.orchestra.run.smithi156.stdout:======================================================================================================================= 2023-10-09T05:29:55.722 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2023-10-09T05:29:55.722 INFO:teuthology.orchestra.run.smithi156.stdout:======================================================================================================================= 2023-10-09T05:29:55.722 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2023-10-09T05:29:55.722 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 7.4 M 2023-10-09T05:29:55.722 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2023-10-09T05:29:55.723 INFO:teuthology.orchestra.run.smithi156.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-10-09T05:29:55.723 INFO:teuthology.orchestra.run.smithi156.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-10-09T05:29:55.723 INFO:teuthology.orchestra.run.smithi156.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-10-09T05:29:55.723 INFO:teuthology.orchestra.run.smithi156.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-10-09T05:29:55.723 INFO:teuthology.orchestra.run.smithi156.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-10-09T05:29:55.723 INFO:teuthology.orchestra.run.smithi156.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-10-09T05:29:55.723 INFO:teuthology.orchestra.run.smithi156.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-10-09T05:29:55.723 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:29:55.724 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2023-10-09T05:29:55.724 INFO:teuthology.orchestra.run.smithi156.stdout:======================================================================================================================= 2023-10-09T05:29:55.724 INFO:teuthology.orchestra.run.smithi156.stdout:Install 8 Packages 2023-10-09T05:29:55.724 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:29:55.724 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 35 M 2023-10-09T05:29:55.724 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 210 M 2023-10-09T05:29:55.724 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2023-10-09T05:29:56.063 INFO:teuthology.orchestra.run.smithi156.stdout:(1/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 11 MB/s | 3.7 MB 00:00 2023-10-09T05:29:56.180 INFO:teuthology.orchestra.run.smithi156.stdout:(2/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 1.9 MB/s | 225 kB 00:00 2023-10-09T05:29:56.430 INFO:teuthology.orchestra.run.smithi156.stdout:(3/8): python3-scipy-1.0.0-21.module+el8.5.0+10 19 MB/s | 14 MB 00:00 2023-10-09T05:29:56.539 INFO:teuthology.orchestra.run.smithi156.stdout:(4/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 14 MB/s | 4.9 MB 00:00 2023-10-09T05:29:56.631 INFO:teuthology.orchestra.run.smithi156.stdout:(5/8): ceph-mgr-diskprediction-local-17.2.6-130 8.1 MB/s | 7.4 MB 00:00 2023-10-09T05:29:56.747 INFO:teuthology.orchestra.run.smithi156.stdout:(6/8): openblas-0.3.15-3.el8.x86_64.rpm 15 MB/s | 4.7 MB 00:00 2023-10-09T05:29:56.781 INFO:teuthology.orchestra.run.smithi156.stdout:(7/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 2.6 MB/s | 644 kB 00:00 2023-10-09T05:29:56.814 INFO:teuthology.orchestra.run.smithi156.stdout:(8/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 933 kB/s | 171 kB 00:00 2023-10-09T05:29:56.815 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:29:56.816 INFO:teuthology.orchestra.run.smithi156.stdout:Total 32 MB/s | 35 MB 00:01 2023-10-09T05:29:57.052 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2023-10-09T05:29:57.070 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2023-10-09T05:29:57.070 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2023-10-09T05:29:57.362 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2023-10-09T05:29:57.365 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2023-10-09T05:29:57.895 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2023-10-09T05:29:58.050 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-09T05:29:58.191 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-09T05:29:58.326 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T05:29:58.947 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T05:29:59.101 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-09T05:29:59.697 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-09T05:29:59.883 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-10-09T05:30:00.166 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-10-09T05:30:00.366 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:30:00.366 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-10-09T05:30:00.366 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:30:00.366 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:30:00.366 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:30:00.366 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-10-09T05:30:00.366 INFO:teuthology.orchestra.run.smithi064.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-10-09T05:30:00.366 INFO:teuthology.orchestra.run.smithi064.stdout: python3-routes-2.4.1-12.el8.noarch 2023-10-09T05:30:00.366 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:30:00.367 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:30:00.509 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-mgr-diskprediction-local 2023-10-09T05:30:00.556 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-10-09T05:30:00.861 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-10-09T05:30:00.879 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:30:01.075 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T05:30:02.451 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:02:20 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:30:02.828 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T05:30:03.976 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:30:03.978 INFO:teuthology.orchestra.run.smithi064.stdout:======================================================================================================================= 2023-10-09T05:30:03.978 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2023-10-09T05:30:03.978 INFO:teuthology.orchestra.run.smithi064.stdout:======================================================================================================================= 2023-10-09T05:30:03.978 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-10-09T05:30:03.978 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 7.4 M 2023-10-09T05:30:03.978 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2023-10-09T05:30:03.978 INFO:teuthology.orchestra.run.smithi064.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-10-09T05:30:03.978 INFO:teuthology.orchestra.run.smithi064.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-10-09T05:30:03.978 INFO:teuthology.orchestra.run.smithi064.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-10-09T05:30:03.978 INFO:teuthology.orchestra.run.smithi064.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-10-09T05:30:03.979 INFO:teuthology.orchestra.run.smithi064.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-10-09T05:30:03.979 INFO:teuthology.orchestra.run.smithi064.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-10-09T05:30:03.979 INFO:teuthology.orchestra.run.smithi064.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-10-09T05:30:03.979 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:30:03.979 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-10-09T05:30:03.979 INFO:teuthology.orchestra.run.smithi064.stdout:======================================================================================================================= 2023-10-09T05:30:03.979 INFO:teuthology.orchestra.run.smithi064.stdout:Install 8 Packages 2023-10-09T05:30:03.979 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:30:03.980 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 35 M 2023-10-09T05:30:03.980 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 210 M 2023-10-09T05:30:03.980 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-10-09T05:30:04.075 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-10-09T05:30:04.137 INFO:teuthology.orchestra.run.smithi064.stdout:(1/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 4.0 MB/s | 644 kB 00:00 2023-10-09T05:30:04.170 INFO:teuthology.orchestra.run.smithi064.stdout:(2/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 904 kB/s | 171 kB 00:00 2023-10-09T05:30:04.225 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 8/8 2023-10-09T05:30:04.437 INFO:teuthology.orchestra.run.smithi064.stdout:(3/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 14 MB/s | 3.7 MB 00:00 2023-10-09T05:30:04.505 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 8/8 2023-10-09T05:30:04.505 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 1/8 2023-10-09T05:30:04.505 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 2/8 2023-10-09T05:30:04.505 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 3/8 2023-10-09T05:30:04.505 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 4/8 2023-10-09T05:30:04.505 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 5/8 2023-10-09T05:30:04.505 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 6/8 2023-10-09T05:30:04.505 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 7/8 2023-10-09T05:30:04.588 INFO:teuthology.orchestra.run.smithi064.stdout:(4/8): python3-scipy-1.0.0-21.module+el8.5.0+10 30 MB/s | 14 MB 00:00 2023-10-09T05:30:04.612 INFO:teuthology.orchestra.run.smithi064.stdout:(5/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 1.3 MB/s | 225 kB 00:00 2023-10-09T05:30:04.705 INFO:teuthology.orchestra.run.smithi064.stdout:(6/8): ceph-mgr-diskprediction-local-17.2.6-130 10 MB/s | 7.4 MB 00:00 2023-10-09T05:30:04.871 INFO:teuthology.orchestra.run.smithi064.stdout:(7/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 17 MB/s | 4.9 MB 00:00 2023-10-09T05:30:04.980 INFO:teuthology.orchestra.run.smithi064.stdout:(8/8): openblas-0.3.15-3.el8.x86_64.rpm 13 MB/s | 4.7 MB 00:00 2023-10-09T05:30:04.981 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:30:04.981 INFO:teuthology.orchestra.run.smithi064.stdout:Total 35 MB/s | 35 MB 00:01 2023-10-09T05:30:05.228 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-10-09T05:30:05.247 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-10-09T05:30:05.247 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-10-09T05:30:05.553 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 8/8 2023-10-09T05:30:05.553 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:02:23 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:30:05.557 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-10-09T05:30:05.561 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-10-09T05:30:06.120 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-10-09T05:30:06.293 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-09T05:30:06.430 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-09T05:30:06.568 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T05:30:07.224 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T05:30:07.444 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-09T05:30:08.090 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-09T05:30:08.301 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-10-09T05:30:09.039 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-10-09T05:30:09.327 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-10-09T05:30:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T05:30:11.430 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T05:30:12.217 INFO:teuthology.orchestra.run.smithi156.stdout:Installed products updated. 2023-10-09T05:30:12.404 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:30:12.405 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2023-10-09T05:30:12.405 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:30:12.405 INFO:teuthology.orchestra.run.smithi156.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-10-09T05:30:12.405 INFO:teuthology.orchestra.run.smithi156.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-10-09T05:30:12.405 INFO:teuthology.orchestra.run.smithi156.stdout: openblas-0.3.15-3.el8.x86_64 2023-10-09T05:30:12.405 INFO:teuthology.orchestra.run.smithi156.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-10-09T05:30:12.405 INFO:teuthology.orchestra.run.smithi156.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-10-09T05:30:12.405 INFO:teuthology.orchestra.run.smithi156.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-10-09T05:30:12.405 INFO:teuthology.orchestra.run.smithi156.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-10-09T05:30:12.405 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:30:12.405 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:30:12.555 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-mgr-rook 2023-10-09T05:30:12.747 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-10-09T05:30:12.904 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:30:12.934 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 8/8 2023-10-09T05:30:13.213 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 8/8 2023-10-09T05:30:13.213 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 1/8 2023-10-09T05:30:13.213 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T05:30:13.213 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-10-09T05:30:13.214 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-10-09T05:30:13.214 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-10-09T05:30:13.214 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T05:30:13.214 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-10-09T05:30:14.301 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-10-09T05:30:14.301 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:02:32 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:30:14.418 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:02:32 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:30:15.790 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:30:15.792 INFO:teuthology.orchestra.run.smithi156.stdout:======================================================================================================== 2023-10-09T05:30:15.792 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2023-10-09T05:30:15.792 INFO:teuthology.orchestra.run.smithi156.stdout:======================================================================================================== 2023-10-09T05:30:15.792 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2023-10-09T05:30:15.792 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-rook noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 47 k 2023-10-09T05:30:15.792 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2023-10-09T05:30:15.793 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-10-09T05:30:15.793 INFO:teuthology.orchestra.run.smithi156.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-10-09T05:30:15.793 INFO:teuthology.orchestra.run.smithi156.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-10-09T05:30:15.793 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-10-09T05:30:15.793 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-10-09T05:30:15.793 INFO:teuthology.orchestra.run.smithi156.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-10-09T05:30:15.793 INFO:teuthology.orchestra.run.smithi156.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-10-09T05:30:15.793 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-10-09T05:30:15.793 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-10-09T05:30:15.793 INFO:teuthology.orchestra.run.smithi156.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-09T05:30:15.793 INFO:teuthology.orchestra.run.smithi156.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-10-09T05:30:15.794 INFO:teuthology.orchestra.run.smithi156.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-10-09T05:30:15.794 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:30:15.794 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2023-10-09T05:30:15.794 INFO:teuthology.orchestra.run.smithi156.stdout:======================================================================================================== 2023-10-09T05:30:15.794 INFO:teuthology.orchestra.run.smithi156.stdout:Install 13 Packages 2023-10-09T05:30:15.794 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:30:15.794 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 1.9 M 2023-10-09T05:30:15.795 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 25 M 2023-10-09T05:30:15.795 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2023-10-09T05:30:16.354 INFO:teuthology.orchestra.run.smithi156.stdout:(1/13): python3-certifi-2018.10.15-7.el8.noarch 51 kB/s | 16 kB 00:00 2023-10-09T05:30:16.379 INFO:teuthology.orchestra.run.smithi156.stdout:(2/13): ceph-mgr-rook-17.2.6-1303.gb2903cb0.el8 137 kB/s | 47 kB 00:00 2023-10-09T05:30:16.404 INFO:teuthology.orchestra.run.smithi156.stdout:(3/13): python3-cachetools-3.1.1-4.el8.noarch.r 89 kB/s | 33 kB 00:00 2023-10-09T05:30:16.596 INFO:teuthology.orchestra.run.smithi156.stdout:(4/13): python3-google-auth-1.1.1-10.el8.noarch 335 kB/s | 81 kB 00:00 2023-10-09T05:30:16.755 INFO:teuthology.orchestra.run.smithi156.stdout:(5/13): python3-websocket-client-0.56.0-5.el8.n 387 kB/s | 61 kB 00:00 2023-10-09T05:30:16.780 INFO:teuthology.orchestra.run.smithi156.stdout:(6/13): python3-rsa-4.9-2.el8.noarch.rpm 171 kB/s | 64 kB 00:00 2023-10-09T05:30:16.922 INFO:teuthology.orchestra.run.smithi156.stdout:(7/13): python3-pyasn1-modules-0.3.7-6.el8.noar 661 kB/s | 110 kB 00:00 2023-10-09T05:30:17.014 INFO:teuthology.orchestra.run.smithi156.stdout:(8/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 114 kB/s | 27 kB 00:00 2023-10-09T05:30:17.056 INFO:teuthology.orchestra.run.smithi156.stdout:(9/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 945 kB/s | 126 kB 00:00 2023-10-09T05:30:17.090 INFO:teuthology.orchestra.run.smithi156.stdout:(10/13): python3-kubernetes-11.0.0-6.el8.noarch 1.7 MB/s | 1.2 MB 00:00 2023-10-09T05:30:17.182 INFO:teuthology.orchestra.run.smithi156.stdout:(11/13): python3-jsonpointer-1.10-11.el8.noarch 119 kB/s | 20 kB 00:00 2023-10-09T05:30:17.207 INFO:teuthology.orchestra.run.smithi156.stdout:(12/13): python3-requests-oauthlib-1.0.0-1.el8. 284 kB/s | 43 kB 00:00 2023-10-09T05:30:17.249 INFO:teuthology.orchestra.run.smithi156.stdout:(13/13): python3-oauthlib-2.1.0-1.el8.noarch.rp 977 kB/s | 155 kB 00:00 2023-10-09T05:30:17.250 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:30:17.250 INFO:teuthology.orchestra.run.smithi156.stdout:Total 1.3 MB/s | 1.9 MB 00:01 2023-10-09T05:30:17.316 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2023-10-09T05:30:17.333 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2023-10-09T05:30:17.333 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2023-10-09T05:30:17.405 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2023-10-09T05:30:17.409 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2023-10-09T05:30:17.864 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2023-10-09T05:30:18.047 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-10-09T05:30:18.283 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-10-09T05:30:18.500 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-10-09T05:30:18.735 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 4/13 2023-10-09T05:30:18.921 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 5/13 2023-10-09T05:30:19.080 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 6/13 2023-10-09T05:30:19.233 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 7/13 2023-10-09T05:30:19.421 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-10-09T05:30:19.586 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-10-09T05:30:19.759 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-10-09T05:30:20.841 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-10-09T05:30:21.424 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-10-09T05:30:21.573 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 13/13 2023-10-09T05:30:21.817 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 13/13 2023-10-09T05:30:21.817 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 1/13 2023-10-09T05:30:21.817 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-10-09T05:30:21.817 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-10-09T05:30:21.817 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-10-09T05:30:21.817 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-10-09T05:30:21.817 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-10-09T05:30:21.817 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-10-09T05:30:21.818 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 8/13 2023-10-09T05:30:21.818 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 9/13 2023-10-09T05:30:21.818 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 10/13 2023-10-09T05:30:21.818 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 11/13 2023-10-09T05:30:21.818 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 12/13 2023-10-09T05:30:22.945 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 13/13 2023-10-09T05:30:22.946 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:02:40 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:30:23.683 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-10-09T05:30:23.888 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:30:23.888 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-10-09T05:30:23.888 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:30:23.889 INFO:teuthology.orchestra.run.smithi064.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-10-09T05:30:23.889 INFO:teuthology.orchestra.run.smithi064.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-10-09T05:30:23.889 INFO:teuthology.orchestra.run.smithi064.stdout: openblas-0.3.15-3.el8.x86_64 2023-10-09T05:30:23.889 INFO:teuthology.orchestra.run.smithi064.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-10-09T05:30:23.889 INFO:teuthology.orchestra.run.smithi064.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-10-09T05:30:23.889 INFO:teuthology.orchestra.run.smithi064.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-10-09T05:30:23.889 INFO:teuthology.orchestra.run.smithi064.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-10-09T05:30:23.889 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:30:23.889 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:30:24.148 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-mgr-rook 2023-10-09T05:30:24.516 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:30:25.967 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:02:43 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:30:27.494 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:30:27.495 INFO:teuthology.orchestra.run.smithi064.stdout:======================================================================================================== 2023-10-09T05:30:27.495 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2023-10-09T05:30:27.495 INFO:teuthology.orchestra.run.smithi064.stdout:======================================================================================================== 2023-10-09T05:30:27.496 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-10-09T05:30:27.496 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-rook noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 47 k 2023-10-09T05:30:27.496 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2023-10-09T05:30:27.496 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-10-09T05:30:27.496 INFO:teuthology.orchestra.run.smithi064.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-10-09T05:30:27.496 INFO:teuthology.orchestra.run.smithi064.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-10-09T05:30:27.496 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-10-09T05:30:27.496 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-10-09T05:30:27.496 INFO:teuthology.orchestra.run.smithi064.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-10-09T05:30:27.496 INFO:teuthology.orchestra.run.smithi064.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-10-09T05:30:27.496 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-10-09T05:30:27.497 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-10-09T05:30:27.497 INFO:teuthology.orchestra.run.smithi064.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-09T05:30:27.497 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-10-09T05:30:27.497 INFO:teuthology.orchestra.run.smithi064.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-10-09T05:30:27.497 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:30:27.497 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-10-09T05:30:27.497 INFO:teuthology.orchestra.run.smithi064.stdout:======================================================================================================== 2023-10-09T05:30:27.497 INFO:teuthology.orchestra.run.smithi064.stdout:Install 13 Packages 2023-10-09T05:30:27.497 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:30:27.498 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 1.9 M 2023-10-09T05:30:27.498 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 25 M 2023-10-09T05:30:27.498 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-10-09T05:30:27.901 INFO:teuthology.orchestra.run.smithi064.stdout:(1/13): python3-certifi-2018.10.15-7.el8.noarch 52 kB/s | 16 kB 00:00 2023-10-09T05:30:27.927 INFO:teuthology.orchestra.run.smithi064.stdout:(2/13): python3-cachetools-3.1.1-4.el8.noarch.r 97 kB/s | 33 kB 00:00 2023-10-09T05:30:27.969 INFO:teuthology.orchestra.run.smithi064.stdout:(3/13): ceph-mgr-rook-17.2.6-1303.gb2903cb0.el8 124 kB/s | 47 kB 00:00 2023-10-09T05:30:28.252 INFO:teuthology.orchestra.run.smithi064.stdout:(4/13): python3-google-auth-1.1.1-10.el8.noarch 232 kB/s | 81 kB 00:00 2023-10-09T05:30:28.411 INFO:teuthology.orchestra.run.smithi064.stdout:(5/13): python3-websocket-client-0.56.0-5.el8.n 387 kB/s | 61 kB 00:00 2023-10-09T05:30:28.445 INFO:teuthology.orchestra.run.smithi064.stdout:(6/13): python3-rsa-4.9-2.el8.noarch.rpm 135 kB/s | 64 kB 00:00 2023-10-09T05:30:28.570 INFO:teuthology.orchestra.run.smithi064.stdout:(7/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 1.3 MB/s | 155 kB 00:00 2023-10-09T05:30:28.586 INFO:teuthology.orchestra.run.smithi064.stdout:(8/13): python3-requests-oauthlib-1.0.0-1.el8.n 243 kB/s | 43 kB 00:00 2023-10-09T05:30:28.695 INFO:teuthology.orchestra.run.smithi064.stdout:(9/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 245 kB/s | 27 kB 00:00 2023-10-09T05:30:28.720 INFO:teuthology.orchestra.run.smithi064.stdout:(10/13): python3-pyasn1-modules-0.3.7-6.el8.noa 732 kB/s | 110 kB 00:00 2023-10-09T05:30:28.754 INFO:teuthology.orchestra.run.smithi064.stdout:(11/13): python3-kubernetes-11.0.0-6.el8.noarch 1.4 MB/s | 1.2 MB 00:00 2023-10-09T05:30:28.863 INFO:teuthology.orchestra.run.smithi064.stdout:(12/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 757 kB/s | 126 kB 00:00 2023-10-09T05:30:28.879 INFO:teuthology.orchestra.run.smithi064.stdout:(13/13): python3-jsonpointer-1.10-11.el8.noarch 126 kB/s | 20 kB 00:00 2023-10-09T05:30:28.880 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:30:28.880 INFO:teuthology.orchestra.run.smithi064.stdout:Total 1.4 MB/s | 1.9 MB 00:01 2023-10-09T05:30:28.949 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-10-09T05:30:28.967 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-10-09T05:30:28.967 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-10-09T05:30:29.045 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-10-09T05:30:29.049 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-10-09T05:30:29.571 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-10-09T05:30:29.775 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-10-09T05:30:30.012 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-10-09T05:30:30.179 INFO:teuthology.orchestra.run.smithi156.stdout:Installed products updated. 2023-10-09T05:30:30.213 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-10-09T05:30:30.360 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:30:30.360 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2023-10-09T05:30:30.360 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:30:30.360 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-10-09T05:30:30.361 INFO:teuthology.orchestra.run.smithi156.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-10-09T05:30:30.361 INFO:teuthology.orchestra.run.smithi156.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-10-09T05:30:30.361 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-10-09T05:30:30.361 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-10-09T05:30:30.361 INFO:teuthology.orchestra.run.smithi156.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-10-09T05:30:30.361 INFO:teuthology.orchestra.run.smithi156.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-10-09T05:30:30.361 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-10-09T05:30:30.361 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-10-09T05:30:30.361 INFO:teuthology.orchestra.run.smithi156.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-10-09T05:30:30.361 INFO:teuthology.orchestra.run.smithi156.stdout: python3-rsa-4.9-2.el8.noarch 2023-10-09T05:30:30.361 INFO:teuthology.orchestra.run.smithi156.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-10-09T05:30:30.361 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:30:30.362 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:30:30.364 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-10-09T05:30:30.498 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-mgr-cephadm 2023-10-09T05:30:30.565 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-10-09T05:30:30.779 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-10-09T05:30:30.854 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:30:31.011 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-10-09T05:30:31.214 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-10-09T05:30:31.415 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-10-09T05:30:31.617 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-10-09T05:30:32.477 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:02:50 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:30:32.787 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-10-09T05:30:33.033 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-10-09T05:30:33.173 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 13/13 2023-10-09T05:30:33.444 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 13/13 2023-10-09T05:30:33.444 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 1/13 2023-10-09T05:30:33.444 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-10-09T05:30:33.444 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-10-09T05:30:33.444 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-10-09T05:30:33.444 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-10-09T05:30:33.447 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-10-09T05:30:33.447 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-10-09T05:30:33.447 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-10-09T05:30:33.447 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-10-09T05:30:33.448 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-10-09T05:30:33.448 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-10-09T05:30:33.448 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-10-09T05:30:33.872 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:30:33.873 INFO:teuthology.orchestra.run.smithi156.stdout:=============================================================================================== 2023-10-09T05:30:33.873 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2023-10-09T05:30:33.873 INFO:teuthology.orchestra.run.smithi156.stdout:=============================================================================================== 2023-10-09T05:30:33.873 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2023-10-09T05:30:33.873 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 121 k 2023-10-09T05:30:33.873 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2023-10-09T05:30:33.874 INFO:teuthology.orchestra.run.smithi156.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-10-09T05:30:33.874 INFO:teuthology.orchestra.run.smithi156.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-10-09T05:30:33.874 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-10-09T05:30:33.874 INFO:teuthology.orchestra.run.smithi156.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-10-09T05:30:33.874 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:30:33.874 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2023-10-09T05:30:33.874 INFO:teuthology.orchestra.run.smithi156.stdout:=============================================================================================== 2023-10-09T05:30:33.874 INFO:teuthology.orchestra.run.smithi156.stdout:Install 5 Packages 2023-10-09T05:30:33.875 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:30:33.875 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 6.0 M 2023-10-09T05:30:33.875 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 26 M 2023-10-09T05:30:33.876 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2023-10-09T05:30:34.259 INFO:teuthology.orchestra.run.smithi156.stdout:(1/5): ceph-mgr-cephadm-17.2.6-1303.gb2903cb0.e 628 kB/s | 121 kB 00:00 2023-10-09T05:30:34.292 INFO:teuthology.orchestra.run.smithi156.stdout:(2/5): python3-natsort-7.1.1-2.el8.noarch.rpm 266 kB/s | 60 kB 00:00 2023-10-09T05:30:34.461 INFO:teuthology.orchestra.run.smithi156.stdout:(3/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 1.2 MB/s | 476 kB 00:00 2023-10-09T05:30:34.493 INFO:teuthology.orchestra.run.smithi156.stdout:(4/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 2.2 MB/s | 538 kB 00:00 2023-10-09T05:30:34.584 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-10-09T05:30:34.585 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:02:52 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:30:34.743 INFO:teuthology.orchestra.run.smithi156.stdout:(5/5): python3-babel-2.5.1-7.el8.noarch.rpm 11 MB/s | 4.8 MB 00:00 2023-10-09T05:30:34.744 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:30:34.745 INFO:teuthology.orchestra.run.smithi156.stdout:Total 6.8 MB/s | 6.0 MB 00:00 2023-10-09T05:30:34.798 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2023-10-09T05:30:34.809 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2023-10-09T05:30:34.809 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2023-10-09T05:30:34.888 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2023-10-09T05:30:34.892 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2023-10-09T05:30:35.831 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2023-10-09T05:30:36.173 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-10-09T05:30:36.446 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-10-09T05:30:36.767 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-10-09T05:30:37.021 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-10-09T05:30:37.228 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 5/5 2023-10-09T05:30:37.478 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 5/5 2023-10-09T05:30:37.478 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/5 2023-10-09T05:30:37.478 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-10-09T05:30:37.478 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-10-09T05:30:37.478 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-10-09T05:30:38.540 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-10-09T05:30:38.540 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:02:56 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:30:41.879 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-10-09T05:30:42.080 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:30:42.081 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-10-09T05:30:42.081 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:30:42.081 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-10-09T05:30:42.081 INFO:teuthology.orchestra.run.smithi064.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-10-09T05:30:42.081 INFO:teuthology.orchestra.run.smithi064.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-10-09T05:30:42.082 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-10-09T05:30:42.082 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-10-09T05:30:42.082 INFO:teuthology.orchestra.run.smithi064.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-10-09T05:30:42.082 INFO:teuthology.orchestra.run.smithi064.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-10-09T05:30:42.082 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-10-09T05:30:42.083 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-10-09T05:30:42.083 INFO:teuthology.orchestra.run.smithi064.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-10-09T05:30:42.083 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rsa-4.9-2.el8.noarch 2023-10-09T05:30:42.083 INFO:teuthology.orchestra.run.smithi064.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-10-09T05:30:42.083 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:30:42.084 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:30:42.226 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-mgr-cephadm 2023-10-09T05:30:42.602 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:30:44.092 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:02 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:30:45.478 INFO:teuthology.orchestra.run.smithi156.stdout:Installed products updated. 2023-10-09T05:30:45.619 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:30:45.620 INFO:teuthology.orchestra.run.smithi064.stdout:=============================================================================================== 2023-10-09T05:30:45.620 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2023-10-09T05:30:45.620 INFO:teuthology.orchestra.run.smithi064.stdout:=============================================================================================== 2023-10-09T05:30:45.621 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-10-09T05:30:45.621 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 121 k 2023-10-09T05:30:45.621 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2023-10-09T05:30:45.621 INFO:teuthology.orchestra.run.smithi064.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-10-09T05:30:45.621 INFO:teuthology.orchestra.run.smithi064.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-10-09T05:30:45.621 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-10-09T05:30:45.621 INFO:teuthology.orchestra.run.smithi064.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-10-09T05:30:45.621 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:30:45.621 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-10-09T05:30:45.621 INFO:teuthology.orchestra.run.smithi064.stdout:=============================================================================================== 2023-10-09T05:30:45.622 INFO:teuthology.orchestra.run.smithi064.stdout:Install 5 Packages 2023-10-09T05:30:45.622 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:30:45.622 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 6.0 M 2023-10-09T05:30:45.622 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 26 M 2023-10-09T05:30:45.622 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-10-09T05:30:45.662 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:30:45.662 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2023-10-09T05:30:45.663 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:30:45.663 INFO:teuthology.orchestra.run.smithi156.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-10-09T05:30:45.663 INFO:teuthology.orchestra.run.smithi156.stdout: python3-babel-2.5.1-7.el8.noarch 2023-10-09T05:30:45.663 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-10-09T05:30:45.663 INFO:teuthology.orchestra.run.smithi156.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-10-09T05:30:45.663 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:30:45.663 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:30:45.812 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-fuse 2023-10-09T05:30:45.866 INFO:teuthology.orchestra.run.smithi064.stdout:(1/5): ceph-mgr-cephadm-17.2.6-1303.gb2903cb0.e 667 kB/s | 121 kB 00:00 2023-10-09T05:30:46.006 INFO:teuthology.orchestra.run.smithi064.stdout:(2/5): python3-natsort-7.1.1-2.el8.noarch.rpm 186 kB/s | 60 kB 00:00 2023-10-09T05:30:46.064 INFO:teuthology.orchestra.run.smithi064.stdout:(3/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 2.6 MB/s | 538 kB 00:00 2023-10-09T05:30:46.166 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:30:46.290 INFO:teuthology.orchestra.run.smithi064.stdout:(4/5): python3-babel-2.5.1-7.el8.noarch.rpm 17 MB/s | 4.8 MB 00:00 2023-10-09T05:30:46.314 INFO:teuthology.orchestra.run.smithi064.stdout:(5/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 752 kB/s | 476 kB 00:00 2023-10-09T05:30:46.315 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:30:46.315 INFO:teuthology.orchestra.run.smithi064.stdout:Total 8.6 MB/s | 6.0 MB 00:00 2023-10-09T05:30:46.370 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-10-09T05:30:46.382 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-10-09T05:30:46.382 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-10-09T05:30:46.490 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-10-09T05:30:46.536 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-10-09T05:30:47.755 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:03:05 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:30:47.946 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-10-09T05:30:48.325 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-10-09T05:30:48.591 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-10-09T05:30:48.920 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-10-09T05:30:49.131 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:30:49.133 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:30:49.133 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2023-10-09T05:30:49.133 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:30:49.133 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2023-10-09T05:30:49.133 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-fuse 2023-10-09T05:30:49.133 INFO:teuthology.orchestra.run.smithi156.stdout: x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 876 k 2023-10-09T05:30:49.133 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2023-10-09T05:30:49.133 INFO:teuthology.orchestra.run.smithi156.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-10-09T05:30:49.133 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:30:49.133 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2023-10-09T05:30:49.134 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:30:49.134 INFO:teuthology.orchestra.run.smithi156.stdout:Install 2 Packages 2023-10-09T05:30:49.134 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:30:49.134 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 959 k 2023-10-09T05:30:49.134 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 3.0 M 2023-10-09T05:30:49.134 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2023-10-09T05:30:49.183 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-10-09T05:30:49.293 INFO:teuthology.orchestra.run.smithi156.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 517 kB/s | 83 kB 00:00 2023-10-09T05:30:49.379 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 5/5 2023-10-09T05:30:49.614 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 5/5 2023-10-09T05:30:49.614 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/5 2023-10-09T05:30:49.614 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-10-09T05:30:49.614 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-10-09T05:30:49.614 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-10-09T05:30:49.752 INFO:teuthology.orchestra.run.smithi156.stdout:(2/2): ceph-fuse-17.2.6-1303.gb2903cb0.el8.x86_ 1.4 MB/s | 876 kB 00:00 2023-10-09T05:30:49.752 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:30:49.752 INFO:teuthology.orchestra.run.smithi156.stdout:Total 1.5 MB/s | 959 kB 00:00 2023-10-09T05:30:49.764 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2023-10-09T05:30:49.785 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2023-10-09T05:30:49.785 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2023-10-09T05:30:49.859 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2023-10-09T05:30:49.863 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2023-10-09T05:30:50.024 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2023-10-09T05:30:50.265 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-10-09T05:30:50.424 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/2 2023-10-09T05:30:50.709 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-10-09T05:30:50.709 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:08 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:30:50.760 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/2 2023-10-09T05:30:50.760 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/2 2023-10-09T05:30:51.853 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-10-09T05:30:51.853 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:03:09 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:30:55.647 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-10-09T05:30:55.848 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:30:55.848 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-10-09T05:30:55.849 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:30:55.849 INFO:teuthology.orchestra.run.smithi064.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-10-09T05:30:55.849 INFO:teuthology.orchestra.run.smithi064.stdout: python3-babel-2.5.1-7.el8.noarch 2023-10-09T05:30:55.849 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-10-09T05:30:55.849 INFO:teuthology.orchestra.run.smithi064.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-10-09T05:30:55.849 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:30:55.849 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:30:56.006 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-fuse 2023-10-09T05:30:56.382 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:30:57.237 INFO:teuthology.orchestra.run.smithi156.stdout:Installed products updated. 2023-10-09T05:30:57.419 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:30:57.419 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2023-10-09T05:30:57.419 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-10-09T05:30:57.419 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:30:57.420 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:30:57.554 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install librados-devel 2023-10-09T05:30:57.909 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:30:57.950 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:15 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:30:59.360 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:03:17 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:30:59.450 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:30:59.451 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:30:59.451 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2023-10-09T05:30:59.451 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:30:59.451 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-10-09T05:30:59.452 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-fuse 2023-10-09T05:30:59.452 INFO:teuthology.orchestra.run.smithi064.stdout: x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 876 k 2023-10-09T05:30:59.452 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2023-10-09T05:30:59.452 INFO:teuthology.orchestra.run.smithi064.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-10-09T05:30:59.452 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:30:59.452 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-10-09T05:30:59.452 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:30:59.452 INFO:teuthology.orchestra.run.smithi064.stdout:Install 2 Packages 2023-10-09T05:30:59.452 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:30:59.453 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 959 k 2023-10-09T05:30:59.453 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 3.0 M 2023-10-09T05:30:59.453 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-10-09T05:30:59.665 INFO:teuthology.orchestra.run.smithi064.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 391 kB/s | 83 kB 00:00 2023-10-09T05:30:59.698 INFO:teuthology.orchestra.run.smithi064.stdout:(2/2): ceph-fuse-17.2.6-1303.gb2903cb0.el8.x86_ 3.5 MB/s | 876 kB 00:00 2023-10-09T05:30:59.699 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:30:59.699 INFO:teuthology.orchestra.run.smithi064.stdout:Total 3.8 MB/s | 959 kB 00:00 2023-10-09T05:30:59.711 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-10-09T05:30:59.733 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-10-09T05:30:59.733 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-10-09T05:30:59.811 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-10-09T05:30:59.815 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-10-09T05:31:00.005 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-10-09T05:31:00.262 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-10-09T05:31:00.410 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/2 2023-10-09T05:31:00.729 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:31:00.730 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:31:00.730 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repo Size 2023-10-09T05:31:00.730 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:31:00.730 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2023-10-09T05:31:00.730 INFO:teuthology.orchestra.run.smithi156.stdout: librados-devel x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 130 k 2023-10-09T05:31:00.730 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:31:00.731 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2023-10-09T05:31:00.731 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:31:00.731 INFO:teuthology.orchestra.run.smithi156.stdout:Install 1 Package 2023-10-09T05:31:00.731 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:31:00.731 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 130 k 2023-10-09T05:31:00.731 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 464 k 2023-10-09T05:31:00.731 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2023-10-09T05:31:00.762 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/2 2023-10-09T05:31:00.762 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/2 2023-10-09T05:31:00.917 INFO:teuthology.orchestra.run.smithi156.stdout:librados-devel-17.2.6-1303.gb2903cb0.el8.x86_64 699 kB/s | 130 kB 00:00 2023-10-09T05:31:00.917 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:31:00.918 INFO:teuthology.orchestra.run.smithi156.stdout:Total 694 kB/s | 130 kB 00:00 2023-10-09T05:31:00.918 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2023-10-09T05:31:00.923 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2023-10-09T05:31:00.923 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2023-10-09T05:31:00.977 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2023-10-09T05:31:00.981 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2023-10-09T05:31:01.091 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2023-10-09T05:31:01.246 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:31:01.462 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:31:01.849 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-10-09T05:31:01.849 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:19 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:31:02.510 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:31:02.510 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:03:20 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:31:09.671 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-10-09T05:31:09.675 INFO:teuthology.orchestra.run.smithi156.stdout:Installed products updated. 2023-10-09T05:31:09.862 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:31:09.863 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2023-10-09T05:31:09.863 INFO:teuthology.orchestra.run.smithi156.stdout: librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:31:09.863 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:31:09.863 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:31:09.870 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:31:09.871 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-10-09T05:31:09.871 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-10-09T05:31:09.871 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:31:09.871 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:31:10.018 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install librados-devel 2023-10-09T05:31:10.111 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install libcephfs2 2023-10-09T05:31:10.397 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:31:10.477 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:31:12.002 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:29 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:31:12.033 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:03:30 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:31:13.286 INFO:teuthology.orchestra.run.smithi156.stdout:Package libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:31:13.394 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:31:13.396 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2023-10-09T05:31:13.396 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:31:13.499 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:31:13.500 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:31:13.501 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repo Size 2023-10-09T05:31:13.501 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:31:13.501 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-10-09T05:31:13.501 INFO:teuthology.orchestra.run.smithi064.stdout: librados-devel x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 130 k 2023-10-09T05:31:13.501 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:31:13.501 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-10-09T05:31:13.501 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:31:13.501 INFO:teuthology.orchestra.run.smithi064.stdout:Install 1 Package 2023-10-09T05:31:13.501 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:31:13.501 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 130 k 2023-10-09T05:31:13.501 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 464 k 2023-10-09T05:31:13.502 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-10-09T05:31:13.518 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install libcephfs-devel 2023-10-09T05:31:13.688 INFO:teuthology.orchestra.run.smithi064.stdout:librados-devel-17.2.6-1303.gb2903cb0.el8.x86_64 697 kB/s | 130 kB 00:00 2023-10-09T05:31:13.689 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:31:13.689 INFO:teuthology.orchestra.run.smithi064.stdout:Total 692 kB/s | 130 kB 00:00 2023-10-09T05:31:13.689 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-10-09T05:31:13.696 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-10-09T05:31:13.696 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-10-09T05:31:13.752 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-10-09T05:31:13.756 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-10-09T05:31:13.874 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:31:13.877 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-10-09T05:31:14.032 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:31:14.277 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:31:15.330 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:03:33 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:31:15.363 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:31:15.363 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:33 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:31:16.700 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:31:16.701 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:31:16.701 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repo Size 2023-10-09T05:31:16.701 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:31:16.701 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2023-10-09T05:31:16.701 INFO:teuthology.orchestra.run.smithi156.stdout: libcephfs-devel x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 25 k 2023-10-09T05:31:16.701 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:31:16.701 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2023-10-09T05:31:16.701 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:31:16.701 INFO:teuthology.orchestra.run.smithi156.stdout:Install 1 Package 2023-10-09T05:31:16.701 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:31:16.702 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 25 k 2023-10-09T05:31:16.702 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 109 k 2023-10-09T05:31:16.702 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2023-10-09T05:31:17.033 INFO:teuthology.orchestra.run.smithi156.stdout:libcephfs-devel-17.2.6-1303.gb2903cb0.el8.x86_6 75 kB/s | 25 kB 00:00 2023-10-09T05:31:17.033 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:31:17.034 INFO:teuthology.orchestra.run.smithi156.stdout:Total 75 kB/s | 25 kB 00:00 2023-10-09T05:31:17.034 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2023-10-09T05:31:17.037 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2023-10-09T05:31:17.037 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2023-10-09T05:31:17.039 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2023-10-09T05:31:17.043 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2023-10-09T05:31:17.095 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2023-10-09T05:31:17.260 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:31:17.463 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:31:18.471 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:31:18.471 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:03:36 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:31:24.051 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-10-09T05:31:24.253 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:31:24.253 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-10-09T05:31:24.253 INFO:teuthology.orchestra.run.smithi064.stdout: librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:31:24.253 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:31:24.253 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:31:24.387 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install libcephfs2 2023-10-09T05:31:24.766 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:31:25.586 INFO:teuthology.orchestra.run.smithi156.stdout:Installed products updated. 2023-10-09T05:31:25.773 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:31:25.773 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2023-10-09T05:31:25.773 INFO:teuthology.orchestra.run.smithi156.stdout: libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:31:25.773 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:31:25.773 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:31:25.910 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install librados2 2023-10-09T05:31:26.266 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:31:26.292 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:44 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:31:27.692 INFO:teuthology.orchestra.run.smithi064.stdout:Package libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:31:27.812 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:31:27.813 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2023-10-09T05:31:27.813 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:31:27.906 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:03:45 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:31:27.934 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install libcephfs-devel 2023-10-09T05:31:28.311 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:31:29.249 INFO:teuthology.orchestra.run.smithi156.stdout:Package librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:31:29.358 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:31:29.359 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2023-10-09T05:31:29.359 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:31:29.479 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install librbd1 2023-10-09T05:31:29.835 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:31:29.941 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:47 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:31:31.297 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:03:49 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:31:31.481 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:31:31.481 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:31:31.481 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repo Size 2023-10-09T05:31:31.482 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:31:31.482 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-10-09T05:31:31.482 INFO:teuthology.orchestra.run.smithi064.stdout: libcephfs-devel x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 25 k 2023-10-09T05:31:31.482 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:31:31.482 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-10-09T05:31:31.482 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:31:31.482 INFO:teuthology.orchestra.run.smithi064.stdout:Install 1 Package 2023-10-09T05:31:31.482 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:31:31.483 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 25 k 2023-10-09T05:31:31.483 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 109 k 2023-10-09T05:31:31.483 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-10-09T05:31:31.694 INFO:teuthology.orchestra.run.smithi064.stdout:libcephfs-devel-17.2.6-1303.gb2903cb0.el8.x86_6 118 kB/s | 25 kB 00:00 2023-10-09T05:31:31.694 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:31:31.694 INFO:teuthology.orchestra.run.smithi064.stdout:Total 117 kB/s | 25 kB 00:00 2023-10-09T05:31:31.695 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-10-09T05:31:31.698 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-10-09T05:31:31.699 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-10-09T05:31:31.701 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-10-09T05:31:31.705 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-10-09T05:31:31.752 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-10-09T05:31:31.921 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:31:32.135 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:31:32.566 INFO:teuthology.orchestra.run.smithi156.stdout:Package librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:31:32.672 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:31:32.679 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2023-10-09T05:31:32.679 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:31:32.783 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install python3-rados 2023-10-09T05:31:33.141 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:31:33.188 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:31:33.188 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:51 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:31:34.717 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:03:52 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:31:35.959 INFO:teuthology.orchestra.run.smithi156.stdout:Package python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:31:36.067 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:31:36.068 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2023-10-09T05:31:36.068 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:31:36.178 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install python3-rgw 2023-10-09T05:31:36.530 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:31:38.106 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:03:56 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:31:38.114 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-10-09T05:31:38.317 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:31:38.317 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-10-09T05:31:38.317 INFO:teuthology.orchestra.run.smithi064.stdout: libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:31:38.318 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:31:38.318 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:31:38.447 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install librados2 2023-10-09T05:31:38.823 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:31:39.355 INFO:teuthology.orchestra.run.smithi156.stdout:Package python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:31:39.462 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:31:39.463 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2023-10-09T05:31:39.463 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:31:39.586 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install python3-cephfs 2023-10-09T05:31:39.940 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:31:40.376 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:58 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:31:41.607 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:03:59 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:31:41.768 INFO:teuthology.orchestra.run.smithi064.stdout:Package librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:31:41.885 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:31:41.886 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2023-10-09T05:31:41.887 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:31:42.006 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install librbd1 2023-10-09T05:31:42.382 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:31:42.926 INFO:teuthology.orchestra.run.smithi156.stdout:Package python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:31:43.036 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:31:43.037 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2023-10-09T05:31:43.038 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:31:43.161 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install python3-rbd 2023-10-09T05:31:43.514 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:31:43.655 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:04:01 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:31:45.058 INFO:teuthology.orchestra.run.smithi064.stdout:Package librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:31:45.062 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:04:03 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:31:45.180 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:31:45.182 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2023-10-09T05:31:45.182 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:31:45.396 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install python3-rados 2023-10-09T05:31:45.772 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:31:46.380 INFO:teuthology.orchestra.run.smithi156.stdout:Package python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:31:46.491 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:31:46.492 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2023-10-09T05:31:46.492 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:31:46.603 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install rbd-fuse 2023-10-09T05:31:46.955 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:31:47.304 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:04:05 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:31:48.458 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:04:06 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:31:48.727 INFO:teuthology.orchestra.run.smithi064.stdout:Package python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:31:48.846 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:31:48.849 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2023-10-09T05:31:48.850 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:31:48.968 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install python3-rgw 2023-10-09T05:31:49.343 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:31:49.864 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:31:49.864 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:31:49.865 INFO:teuthology.orchestra.run.smithi156.stdout: Package Architecture Version Repository Size 2023-10-09T05:31:49.865 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:31:49.865 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2023-10-09T05:31:49.865 INFO:teuthology.orchestra.run.smithi156.stdout: rbd-fuse x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 90 k 2023-10-09T05:31:49.865 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:31:49.865 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2023-10-09T05:31:49.865 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:31:49.865 INFO:teuthology.orchestra.run.smithi156.stdout:Install 1 Package 2023-10-09T05:31:49.865 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:31:49.866 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 90 k 2023-10-09T05:31:49.866 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 264 k 2023-10-09T05:31:49.866 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2023-10-09T05:31:50.035 INFO:teuthology.orchestra.run.smithi156.stdout:rbd-fuse-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 537 kB/s | 90 kB 00:00 2023-10-09T05:31:50.035 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:31:50.035 INFO:teuthology.orchestra.run.smithi156.stdout:Total 533 kB/s | 90 kB 00:00 2023-10-09T05:31:50.035 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2023-10-09T05:31:50.044 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2023-10-09T05:31:50.044 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2023-10-09T05:31:50.099 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2023-10-09T05:31:50.103 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2023-10-09T05:31:50.213 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2023-10-09T05:31:50.372 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:31:50.609 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:31:50.910 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:04:08 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:31:51.675 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:31:51.675 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:04:09 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:31:52.283 INFO:teuthology.orchestra.run.smithi064.stdout:Package python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:31:52.402 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:31:52.404 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2023-10-09T05:31:52.404 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:31:52.519 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install python3-cephfs 2023-10-09T05:31:52.894 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:31:54.557 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:04:12 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:31:55.963 INFO:teuthology.orchestra.run.smithi064.stdout:Package python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:31:56.082 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:31:56.083 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2023-10-09T05:31:56.083 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:31:56.203 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install python3-rbd 2023-10-09T05:31:56.580 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:31:58.265 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:04:16 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:31:59.679 INFO:teuthology.orchestra.run.smithi064.stdout:Package python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:31:59.795 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:31:59.796 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2023-10-09T05:31:59.796 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:31:59.911 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install rbd-fuse 2023-10-09T05:32:00.289 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:32:00.409 INFO:teuthology.orchestra.run.smithi156.stdout:Installed products updated. 2023-10-09T05:32:00.588 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:32:00.588 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2023-10-09T05:32:00.588 INFO:teuthology.orchestra.run.smithi156.stdout: rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:32:00.589 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:32:00.589 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:32:00.719 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install rbd-mirror 2023-10-09T05:32:01.075 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:32:02.067 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:04:20 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:32:02.588 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:04:20 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:32:03.605 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:32:03.606 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:32:03.606 INFO:teuthology.orchestra.run.smithi064.stdout: Package Architecture Version Repository Size 2023-10-09T05:32:03.606 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:32:03.606 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-10-09T05:32:03.606 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-fuse x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 90 k 2023-10-09T05:32:03.606 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:32:03.607 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-10-09T05:32:03.607 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:32:03.607 INFO:teuthology.orchestra.run.smithi064.stdout:Install 1 Package 2023-10-09T05:32:03.607 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:32:03.607 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 90 k 2023-10-09T05:32:03.608 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 264 k 2023-10-09T05:32:03.608 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-10-09T05:32:03.967 INFO:teuthology.orchestra.run.smithi064.stdout:rbd-fuse-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 250 kB/s | 90 kB 00:00 2023-10-09T05:32:03.968 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:32:03.968 INFO:teuthology.orchestra.run.smithi064.stdout:Total 250 kB/s | 90 kB 00:00 2023-10-09T05:32:03.968 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-10-09T05:32:03.979 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-10-09T05:32:03.980 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-10-09T05:32:04.019 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:32:04.020 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:32:04.020 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repo Size 2023-10-09T05:32:04.020 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:32:04.020 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2023-10-09T05:32:04.020 INFO:teuthology.orchestra.run.smithi156.stdout: rbd-mirror x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.5 M 2023-10-09T05:32:04.020 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:32:04.020 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2023-10-09T05:32:04.020 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:32:04.020 INFO:teuthology.orchestra.run.smithi156.stdout:Install 1 Package 2023-10-09T05:32:04.021 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:32:04.021 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 3.5 M 2023-10-09T05:32:04.021 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 16 M 2023-10-09T05:32:04.021 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2023-10-09T05:32:04.039 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-10-09T05:32:04.042 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-10-09T05:32:04.159 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-10-09T05:32:04.404 INFO:teuthology.orchestra.run.smithi156.stdout:rbd-mirror-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 9.0 MB/s | 3.5 MB 00:00 2023-10-09T05:32:04.404 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:32:04.404 INFO:teuthology.orchestra.run.smithi156.stdout:Total 9.0 MB/s | 3.5 MB 00:00 2023-10-09T05:32:04.404 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2023-10-09T05:32:04.415 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2023-10-09T05:32:04.416 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2023-10-09T05:32:04.442 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:32:04.482 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2023-10-09T05:32:04.485 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2023-10-09T05:32:04.684 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:32:04.926 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2023-10-09T05:32:05.097 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:32:05.465 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:32:05.770 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:32:05.770 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:04:23 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:32:06.523 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:32:06.523 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:04:24 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:32:13.066 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-10-09T05:32:13.264 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:32:13.264 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-10-09T05:32:13.264 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:32:13.264 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:32:13.265 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:32:13.370 INFO:teuthology.orchestra.run.smithi156.stdout:Installed products updated. 2023-10-09T05:32:13.408 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install rbd-mirror 2023-10-09T05:32:13.550 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:32:13.550 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2023-10-09T05:32:13.550 INFO:teuthology.orchestra.run.smithi156.stdout: rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:32:13.550 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:32:13.550 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:32:13.689 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install rbd-nbd 2023-10-09T05:32:13.783 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:32:14.047 INFO:teuthology.orchestra.run.smithi156.stdout:Updating Subscription Management repositories. 2023-10-09T05:32:15.503 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:04:33 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:32:15.657 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:04:33 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:32:17.045 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:32:17.046 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:32:17.046 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repo Size 2023-10-09T05:32:17.046 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:32:17.046 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-10-09T05:32:17.046 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-mirror x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.5 M 2023-10-09T05:32:17.046 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:32:17.046 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-10-09T05:32:17.046 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:32:17.047 INFO:teuthology.orchestra.run.smithi064.stdout:Install 1 Package 2023-10-09T05:32:17.047 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:32:17.047 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 3.5 M 2023-10-09T05:32:17.047 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 16 M 2023-10-09T05:32:17.047 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-10-09T05:32:17.092 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2023-10-09T05:32:17.093 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:32:17.093 INFO:teuthology.orchestra.run.smithi156.stdout: Package Architecture Version Repository Size 2023-10-09T05:32:17.093 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:32:17.093 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2023-10-09T05:32:17.093 INFO:teuthology.orchestra.run.smithi156.stdout: rbd-nbd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 173 k 2023-10-09T05:32:17.094 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:32:17.094 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2023-10-09T05:32:17.094 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2023-10-09T05:32:17.094 INFO:teuthology.orchestra.run.smithi156.stdout:Install 1 Package 2023-10-09T05:32:17.094 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:32:17.095 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 173 k 2023-10-09T05:32:17.095 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 552 k 2023-10-09T05:32:17.095 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2023-10-09T05:32:17.280 INFO:teuthology.orchestra.run.smithi156.stdout:rbd-nbd-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 932 kB/s | 173 kB 00:00 2023-10-09T05:32:17.280 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:32:17.281 INFO:teuthology.orchestra.run.smithi156.stdout:Total 926 kB/s | 173 kB 00:00 2023-10-09T05:32:17.281 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2023-10-09T05:32:17.291 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2023-10-09T05:32:17.292 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2023-10-09T05:32:17.346 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2023-10-09T05:32:17.350 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2023-10-09T05:32:17.406 INFO:teuthology.orchestra.run.smithi064.stdout:rbd-mirror-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 9.6 MB/s | 3.5 MB 00:00 2023-10-09T05:32:17.406 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:32:17.406 INFO:teuthology.orchestra.run.smithi064.stdout:Total 9.6 MB/s | 3.5 MB 00:00 2023-10-09T05:32:17.407 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-10-09T05:32:17.420 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-10-09T05:32:17.420 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-10-09T05:32:17.467 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2023-10-09T05:32:17.491 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-10-09T05:32:17.495 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-10-09T05:32:17.601 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:32:17.821 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:32:17.958 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-10-09T05:32:18.150 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:32:18.509 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:32:18.859 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:32:18.860 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:04:36 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:32:19.584 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:32:19.584 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:04:37 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:32:26.976 INFO:teuthology.orchestra.run.smithi156.stdout:Installed products updated. 2023-10-09T05:32:27.159 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:32:27.159 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2023-10-09T05:32:27.159 INFO:teuthology.orchestra.run.smithi156.stdout: rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:32:27.160 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:32:27.160 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2023-10-09T05:32:27.291 DEBUG:teuthology.parallel:result is None 2023-10-09T05:32:28.576 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-10-09T05:32:28.775 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:32:28.775 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-10-09T05:32:28.776 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:32:28.776 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:32:28.776 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:32:28.918 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install rbd-nbd 2023-10-09T05:32:29.297 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-10-09T05:32:30.870 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:04:48 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:32:32.449 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-10-09T05:32:32.449 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:32:32.450 INFO:teuthology.orchestra.run.smithi064.stdout: Package Architecture Version Repository Size 2023-10-09T05:32:32.450 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:32:32.450 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-10-09T05:32:32.450 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-nbd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 173 k 2023-10-09T05:32:32.450 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:32:32.450 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-10-09T05:32:32.450 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-10-09T05:32:32.450 INFO:teuthology.orchestra.run.smithi064.stdout:Install 1 Package 2023-10-09T05:32:32.450 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:32:32.451 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 173 k 2023-10-09T05:32:32.451 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 552 k 2023-10-09T05:32:32.451 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-10-09T05:32:32.749 INFO:teuthology.orchestra.run.smithi064.stdout:rbd-nbd-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 578 kB/s | 173 kB 00:00 2023-10-09T05:32:32.750 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:32:32.750 INFO:teuthology.orchestra.run.smithi064.stdout:Total 576 kB/s | 173 kB 00:00 2023-10-09T05:32:32.750 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-10-09T05:32:32.763 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-10-09T05:32:32.763 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-10-09T05:32:32.821 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-10-09T05:32:32.826 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-10-09T05:32:32.947 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-10-09T05:32:33.088 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:32:33.315 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:32:34.414 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:32:34.414 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:04:52 ago on Mon 09 Oct 2023 05:27:42 AM UTC. 2023-10-09T05:32:40.901 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-10-09T05:32:41.100 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:32:41.100 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-10-09T05:32:41.100 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:32:41.100 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:32:41.100 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-10-09T05:32:41.273 DEBUG:teuthology.parallel:result is None 2023-10-09T05:32:41.273 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:32:41.273 INFO:teuthology.packaging:ref: None 2023-10-09T05:32:41.274 INFO:teuthology.packaging:tag: None 2023-10-09T05:32:41.274 INFO:teuthology.packaging:branch: quincy 2023-10-09T05:32:41.274 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:32:41.274 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-10-09T05:32:41.635 DEBUG:teuthology.orchestra.run.smithi064:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-10-09T05:32:41.659 INFO:teuthology.orchestra.run.smithi064.stdout:17.2.6-1303.gb2903cb0.el8 2023-10-09T05:32:41.659 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1303.gb2903cb0.el8 2023-10-09T05:32:41.659 INFO:teuthology.task.install:The correct ceph version 17.2.6-1303.gb2903cb0 is installed. 2023-10-09T05:32:41.660 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:32:41.660 INFO:teuthology.packaging:ref: None 2023-10-09T05:32:41.660 INFO:teuthology.packaging:tag: None 2023-10-09T05:32:41.660 INFO:teuthology.packaging:branch: quincy 2023-10-09T05:32:41.660 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:32:41.660 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-10-09T05:32:41.916 DEBUG:teuthology.orchestra.run.smithi156:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-10-09T05:32:41.943 INFO:teuthology.orchestra.run.smithi156.stdout:17.2.6-1303.gb2903cb0.el8 2023-10-09T05:32:41.943 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1303.gb2903cb0.el8 2023-10-09T05:32:41.943 INFO:teuthology.task.install:The correct ceph version 17.2.6-1303.gb2903cb0 is installed. 2023-10-09T05:32:41.944 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-10-09T05:32:41.944 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-10-09T05:32:41.944 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-09T05:32:41.976 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2023-10-09T05:32:41.976 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-09T05:32:42.016 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-10-09T05:32:42.016 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-10-09T05:32:42.017 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/usr/bin/daemon-helper 2023-10-09T05:32:42.047 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-09T05:32:42.117 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2023-10-09T05:32:42.117 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/usr/bin/daemon-helper 2023-10-09T05:32:42.146 DEBUG:teuthology.orchestra.run.smithi156:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-09T05:32:42.214 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-10-09T05:32:42.215 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-10-09T05:32:42.215 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-09T05:32:42.244 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-09T05:32:42.312 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2023-10-09T05:32:42.312 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-09T05:32:42.340 DEBUG:teuthology.orchestra.run.smithi156:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-09T05:32:42.407 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-10-09T05:32:42.408 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-10-09T05:32:42.408 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/usr/bin/stdin-killer 2023-10-09T05:32:42.436 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-09T05:32:42.504 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2023-10-09T05:32:42.504 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/usr/bin/stdin-killer 2023-10-09T05:32:42.531 DEBUG:teuthology.orchestra.run.smithi156:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-09T05:32:42.598 INFO:teuthology.run_tasks:Running task cephadm... 2023-10-09T05:32:42.671 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'} 2023-10-09T05:32:42.671 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2023-10-09T05:32:42.671 INFO:tasks.cephadm:Cluster fsid is 44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:32:42.671 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-10-09T05:32:42.672 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.64', 'mon.c': '[v2:172.21.15.64:3301,v1:172.21.15.64:6790]', 'mon.b': '172.21.15.156'} 2023-10-09T05:32:42.672 INFO:tasks.cephadm:First mon is mon.a on smithi064 2023-10-09T05:32:42.672 INFO:tasks.cephadm:First mgr is y 2023-10-09T05:32:42.672 INFO:tasks.cephadm:Normalizing hostnames... 2023-10-09T05:32:42.672 DEBUG:teuthology.orchestra.run.smithi064:> sudo hostname $(hostname -s) 2023-10-09T05:32:42.702 DEBUG:teuthology.orchestra.run.smithi156:> sudo hostname $(hostname -s) 2023-10-09T05:32:42.729 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2023-10-09T05:32:42.730 DEBUG:teuthology.orchestra.run.smithi064:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T05:32:42.973 INFO:teuthology.orchestra.run.smithi064.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366887 Oct 9 05:32 /home/ubuntu/cephtest/cephadm 2023-10-09T05:32:42.977 DEBUG:teuthology.orchestra.run.smithi156:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T05:32:43.251 INFO:teuthology.orchestra.run.smithi156.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366887 Oct 9 05:32 /home/ubuntu/cephtest/cephadm 2023-10-09T05:32:43.252 DEBUG:teuthology.orchestra.run.smithi064:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T05:32:43.274 DEBUG:teuthology.orchestra.run.smithi156:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T05:32:43.311 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2023-10-09T05:32:43.311 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-10-09T05:32:43.317 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-10-09T05:32:44.284 INFO:teuthology.orchestra.run.smithi156.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-10-09T05:32:44.522 INFO:teuthology.orchestra.run.smithi064.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-10-09T05:33:29.856 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:33:29.856 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph_version": "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)", 2023-10-09T05:33:29.856 INFO:teuthology.orchestra.run.smithi156.stdout: "image_id": "5517396c50d4b07181c79c353f571b455dbdd80614d068cc9a6e42eb27b7bccf", 2023-10-09T05:33:29.856 INFO:teuthology.orchestra.run.smithi156.stdout: "repo_digests": [ 2023-10-09T05:33:29.856 INFO:teuthology.orchestra.run.smithi156.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:67105d45415a7f6878d82dd27e27caaa40ebf0ca6488030995660f1cad5fce9d" 2023-10-09T05:33:29.857 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:33:29.857 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:33:30.523 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:33:30.523 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_version": "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)", 2023-10-09T05:33:30.523 INFO:teuthology.orchestra.run.smithi064.stdout: "image_id": "5517396c50d4b07181c79c353f571b455dbdd80614d068cc9a6e42eb27b7bccf", 2023-10-09T05:33:30.523 INFO:teuthology.orchestra.run.smithi064.stdout: "repo_digests": [ 2023-10-09T05:33:30.523 INFO:teuthology.orchestra.run.smithi064.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:67105d45415a7f6878d82dd27e27caaa40ebf0ca6488030995660f1cad5fce9d" 2023-10-09T05:33:30.523 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:33:30.523 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:33:30.547 DEBUG:teuthology.orchestra.run.smithi064:> sudo mkdir -p /etc/ceph 2023-10-09T05:33:30.581 DEBUG:teuthology.orchestra.run.smithi156:> sudo mkdir -p /etc/ceph 2023-10-09T05:33:30.617 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 777 /etc/ceph 2023-10-09T05:33:30.650 DEBUG:teuthology.orchestra.run.smithi156:> sudo chmod 777 /etc/ceph 2023-10-09T05:33:30.687 INFO:tasks.cephadm:Writing seed config... 2023-10-09T05:33:30.688 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-10-09T05:33:30.688 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-10-09T05:33:30.688 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-10-09T05:33:30.688 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-10-09T05:33:30.689 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-10-09T05:33:30.689 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2023-10-09T05:33:30.689 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-10-09T05:33:30.689 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-10-09T05:33:30.689 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-10-09T05:33:30.689 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-10-09T05:33:30.689 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-10-09T05:33:30.690 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-10-09T05:33:30.690 DEBUG:teuthology.orchestra.run.smithi064:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-10-09T05:33:30.708 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 44c77acc-6665-11ee-8db5-212e2dc638e7 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-10-09T05:33:30.708 DEBUG:teuthology.orchestra.run.smithi064:mon.a> sudo journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mon.a.service 2023-10-09T05:33:30.750 DEBUG:teuthology.orchestra.run.smithi064:mgr.y> sudo journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mgr.y.service 2023-10-09T05:33:30.778 INFO:journalctl@ceph.mon.a.smithi064.stdout:-- Logs begin at Mon 2023-10-09 05:16:16 UTC. -- 2023-10-09T05:33:30.779 INFO:tasks.cephadm:Bootstrapping... 2023-10-09T05:33:30.779 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.64 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-10-09T05:33:30.806 INFO:journalctl@ceph.mgr.y.smithi064.stdout:-- Logs begin at Mon 2023-10-09 05:16:16 UTC. -- 2023-10-09T05:33:30.989 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:33:30.990 INFO:teuthology.orchestra.run.smithi064.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', '44c77acc-6665-11ee-8db5-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.64', '--skip-admin-label'] 2023-10-09T05:33:31.055 INFO:teuthology.orchestra.run.smithi064.stdout:/bin/podman: stdout 3.0.1 2023-10-09T05:33:31.056 INFO:teuthology.orchestra.run.smithi064.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2023-10-09T05:33:31.056 INFO:teuthology.orchestra.run.smithi064.stdout:Verifying podman|docker is present... 2023-10-09T05:33:31.112 INFO:teuthology.orchestra.run.smithi064.stdout:/bin/podman: stdout 3.0.1 2023-10-09T05:33:31.112 INFO:teuthology.orchestra.run.smithi064.stdout:Verifying lvm2 is present... 2023-10-09T05:33:31.112 INFO:teuthology.orchestra.run.smithi064.stdout:Verifying time synchronization is in place... 2023-10-09T05:33:31.119 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-10-09T05:33:31.119 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T05:33:31.125 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-10-09T05:33:31.125 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout inactive 2023-10-09T05:33:31.131 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout enabled 2023-10-09T05:33:31.137 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout active 2023-10-09T05:33:31.137 INFO:teuthology.orchestra.run.smithi064.stdout:Unit chronyd.service is enabled and running 2023-10-09T05:33:31.137 INFO:teuthology.orchestra.run.smithi064.stdout:Repeating the final host check... 2023-10-09T05:33:31.192 INFO:teuthology.orchestra.run.smithi064.stdout:/bin/podman: stdout 3.0.1 2023-10-09T05:33:31.192 INFO:teuthology.orchestra.run.smithi064.stdout:podman (/bin/podman) version 3.0.1 is present 2023-10-09T05:33:31.192 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl is present 2023-10-09T05:33:31.192 INFO:teuthology.orchestra.run.smithi064.stdout:lvcreate is present 2023-10-09T05:33:31.198 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-10-09T05:33:31.198 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T05:33:31.204 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-10-09T05:33:31.204 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout inactive 2023-10-09T05:33:31.211 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout enabled 2023-10-09T05:33:31.218 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout active 2023-10-09T05:33:31.218 INFO:teuthology.orchestra.run.smithi064.stdout:Unit chronyd.service is enabled and running 2023-10-09T05:33:31.218 INFO:teuthology.orchestra.run.smithi064.stdout:Host looks OK 2023-10-09T05:33:31.218 INFO:teuthology.orchestra.run.smithi064.stdout:Cluster fsid: 44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:33:31.218 INFO:teuthology.orchestra.run.smithi064.stdout:Acquiring lock 140448381524960 on /run/cephadm/44c77acc-6665-11ee-8db5-212e2dc638e7.lock 2023-10-09T05:33:31.218 INFO:teuthology.orchestra.run.smithi064.stdout:Lock 140448381524960 acquired on /run/cephadm/44c77acc-6665-11ee-8db5-212e2dc638e7.lock 2023-10-09T05:33:31.219 INFO:teuthology.orchestra.run.smithi064.stdout:Verifying IP 172.21.15.64 port 3300 ... 2023-10-09T05:33:31.219 INFO:teuthology.orchestra.run.smithi064.stdout:Verifying IP 172.21.15.64 port 6789 ... 2023-10-09T05:33:31.219 INFO:teuthology.orchestra.run.smithi064.stdout:Base mon IP(s) is [172.21.15.64:3300, 172.21.15.64:6789], mon addrv is [v2:172.21.15.64:3300,v1:172.21.15.64:6789] 2023-10-09T05:33:31.222 INFO:teuthology.orchestra.run.smithi064.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp metric 100 2023-10-09T05:33:31.223 INFO:teuthology.orchestra.run.smithi064.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.64 metric 100 2023-10-09T05:33:31.225 INFO:teuthology.orchestra.run.smithi064.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2023-10-09T05:33:31.225 INFO:teuthology.orchestra.run.smithi064.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2023-10-09T05:33:31.225 INFO:teuthology.orchestra.run.smithi064.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2023-10-09T05:33:31.228 INFO:teuthology.orchestra.run.smithi064.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-10-09T05:33:31.228 INFO:teuthology.orchestra.run.smithi064.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2023-10-09T05:33:31.228 INFO:teuthology.orchestra.run.smithi064.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-10-09T05:33:31.228 INFO:teuthology.orchestra.run.smithi064.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2023-10-09T05:33:31.228 INFO:teuthology.orchestra.run.smithi064.stdout:/sbin/ip: stdout inet6 fe80::3661:aaef:c525:ac7a/64 scope link noprefixroute 2023-10-09T05:33:31.228 INFO:teuthology.orchestra.run.smithi064.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-10-09T05:33:31.229 INFO:teuthology.orchestra.run.smithi064.stdout:Mon IP `172.21.15.64` is in CIDR network `172.21.0.0/20` 2023-10-09T05:33:31.229 INFO:teuthology.orchestra.run.smithi064.stdout:Mon IP `172.21.15.64` is in CIDR network `172.21.0.0/20` 2023-10-09T05:33:31.229 INFO:teuthology.orchestra.run.smithi064.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-10-09T05:33:31.229 INFO:teuthology.orchestra.run.smithi064.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-10-09T05:33:31.230 INFO:teuthology.orchestra.run.smithi064.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-10-09T05:33:31.669 INFO:teuthology.orchestra.run.smithi064.stdout:/bin/podman: stdout 5517396c50d4b07181c79c353f571b455dbdd80614d068cc9a6e42eb27b7bccf 2023-10-09T05:33:31.669 INFO:teuthology.orchestra.run.smithi064.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2023-10-09T05:33:31.669 INFO:teuthology.orchestra.run.smithi064.stdout:/bin/podman: stderr Getting image source signatures 2023-10-09T05:33:31.669 INFO:teuthology.orchestra.run.smithi064.stdout:/bin/podman: stderr Copying blob sha256:1b55aded3a89dccf5f05a82d583393d22b3476851fe870b413c77ca909f5f760 2023-10-09T05:33:31.669 INFO:teuthology.orchestra.run.smithi064.stdout:/bin/podman: stderr Copying blob sha256:be9ed931e0f930233626f1c4d225977fbcc33e056e1a8fc67c7f63553d9455a2 2023-10-09T05:33:31.670 INFO:teuthology.orchestra.run.smithi064.stdout:/bin/podman: stderr Copying config sha256:5517396c50d4b07181c79c353f571b455dbdd80614d068cc9a6e42eb27b7bccf 2023-10-09T05:33:31.670 INFO:teuthology.orchestra.run.smithi064.stdout:/bin/podman: stderr Writing manifest to image destination 2023-10-09T05:33:31.670 INFO:teuthology.orchestra.run.smithi064.stdout:/bin/podman: stderr Storing signatures 2023-10-09T05:33:32.867 INFO:teuthology.orchestra.run.smithi064.stdout:ceph: stdout ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable) 2023-10-09T05:33:32.868 INFO:teuthology.orchestra.run.smithi064.stdout:Ceph version: ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable) 2023-10-09T05:33:32.868 INFO:teuthology.orchestra.run.smithi064.stdout:Extracting ceph user uid/gid from container image... 2023-10-09T05:33:33.966 INFO:teuthology.orchestra.run.smithi064.stdout:stat: stdout 167 167 2023-10-09T05:33:33.966 INFO:teuthology.orchestra.run.smithi064.stdout:Creating initial keys... 2023-10-09T05:33:34.953 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph-authtool: stdout AQCukCNlQgLiExAAY6wNKymc/UnauUoJNr+Y6Q== 2023-10-09T05:33:36.042 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph-authtool: stdout AQCvkCNlm+XZExAAHXbb4JWB9vNPkWPo/9s10A== 2023-10-09T05:33:36.949 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph-authtool: stdout AQCwkCNlI0DdFRAAFw+ssRUkQP+9MsehlY9Btg== 2023-10-09T05:33:36.950 INFO:teuthology.orchestra.run.smithi064.stdout:Creating initial monmap... 2023-10-09T05:33:37.958 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T05:33:37.958 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2023-10-09T05:33:37.958 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:33:37.959 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T05:33:37.959 INFO:teuthology.orchestra.run.smithi064.stdout:monmaptool for a [v2:172.21.15.64:3300,v1:172.21.15.64:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T05:33:37.959 INFO:teuthology.orchestra.run.smithi064.stdout:setting min_mon_release = octopus 2023-10-09T05:33:37.959 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/monmaptool: set fsid to 44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:33:37.959 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T05:33:37.959 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:33:37.959 INFO:teuthology.orchestra.run.smithi064.stdout:Creating mon... 2023-10-09T05:33:38.914 INFO:teuthology.orchestra.run.smithi064.stdout:create mon.a on 2023-10-09T05:33:39.017 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-10-09T05:33:39.109 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-10-09T05:33:39.206 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-44c77acc-6665-11ee-8db5-212e2dc638e7.target → /etc/systemd/system/ceph-44c77acc-6665-11ee-8db5-212e2dc638e7.target. 2023-10-09T05:33:39.206 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-44c77acc-6665-11ee-8db5-212e2dc638e7.target → /etc/systemd/system/ceph-44c77acc-6665-11ee-8db5-212e2dc638e7.target. 2023-10-09T05:33:39.338 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mon.a 2023-10-09T05:33:39.338 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stderr Failed to reset failed state of unit ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mon.a.service: Unit ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mon.a.service not loaded. 2023-10-09T05:33:39.427 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-44c77acc-6665-11ee-8db5-212e2dc638e7.target.wants/ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mon.a.service → /etc/systemd/system/ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@.service. 2023-10-09T05:33:39.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:39 smithi064 systemd[1]: Starting Ceph mon.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:33:40.012 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-09T05:33:40.012 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout disabled 2023-10-09T05:33:40.021 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-09T05:33:40.021 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout inactive 2023-10-09T05:33:40.021 INFO:teuthology.orchestra.run.smithi064.stdout:firewalld.service is not enabled 2023-10-09T05:33:40.022 INFO:teuthology.orchestra.run.smithi064.stdout:Not possible to enable service . firewalld.service is not available 2023-10-09T05:33:40.022 INFO:teuthology.orchestra.run.smithi064.stdout:Waiting for mon to start... 2023-10-09T05:33:40.022 INFO:teuthology.orchestra.run.smithi064.stdout:Waiting for mon... 2023-10-09T05:33:40.260 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:39 smithi064 bash[100055]: 8ae361a5512b39c50e937ef7fad2e510ab96b6309499ef95b3f05272fd700cdf 2023-10-09T05:33:40.260 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:40 smithi064 systemd[1]: Started Ceph mon.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:33:40.260 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:40 smithi064 ceph-mon[100120]: mkfs 44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:33:40.260 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:40 smithi064 ceph-mon[100120]: mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T05:33:41.367 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout cluster: 2023-10-09T05:33:41.368 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout id: 44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:33:41.368 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2023-10-09T05:33:41.368 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2023-10-09T05:33:41.368 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout services: 2023-10-09T05:33:41.368 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.58703s) 2023-10-09T05:33:41.368 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2023-10-09T05:33:41.368 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2023-10-09T05:33:41.368 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2023-10-09T05:33:41.368 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout data: 2023-10-09T05:33:41.368 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2023-10-09T05:33:41.368 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2023-10-09T05:33:41.369 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2023-10-09T05:33:41.369 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout pgs: 2023-10-09T05:33:41.369 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2023-10-09T05:33:41.369 INFO:teuthology.orchestra.run.smithi064.stdout:mon is available 2023-10-09T05:33:41.369 INFO:teuthology.orchestra.run.smithi064.stdout:Assimilating anything we can from ceph.conf... 2023-10-09T05:33:41.372 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:41 smithi064 ceph-mon[100120]: mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T05:33:41.372 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:41 smithi064 ceph-mon[100120]: monmap e1: 1 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0]} removed_ranks: {} 2023-10-09T05:33:41.372 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:41 smithi064 ceph-mon[100120]: fsmap 2023-10-09T05:33:41.372 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:41 smithi064 ceph-mon[100120]: osdmap e1: 0 total, 0 up, 0 in 2023-10-09T05:33:41.372 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:41 smithi064 ceph-mon[100120]: mgrmap e1: no daemons active 2023-10-09T05:33:41.372 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:41 smithi064 ceph-mon[100120]: from='client.? 172.21.15.64:0/3528680007' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-10-09T05:33:42.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:42 smithi064 ceph-mon[100120]: from='client.? 172.21.15.64:0/688579693' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T05:33:42.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:42 smithi064 ceph-mon[100120]: from='client.? 172.21.15.64:0/688579693' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-10-09T05:33:42.576 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2023-10-09T05:33:42.576 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout [global] 2023-10-09T05:33:42.576 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout fsid = 44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:33:42.576 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.64:3300,v1:172.21.15.64:6789] 2023-10-09T05:33:42.576 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-10-09T05:33:42.576 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-10-09T05:33:42.576 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-10-09T05:33:42.576 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-10-09T05:33:42.577 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2023-10-09T05:33:42.577 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout [mgr] 2023-10-09T05:33:42.577 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-10-09T05:33:42.577 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2023-10-09T05:33:42.577 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout [osd] 2023-10-09T05:33:42.577 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-10-09T05:33:42.577 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-10-09T05:33:42.577 INFO:teuthology.orchestra.run.smithi064.stdout:Generating new minimal ceph.conf... 2023-10-09T05:33:43.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:43 smithi064 ceph-mon[100120]: from='client.? 172.21.15.64:0/2027566410' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:33:43.732 INFO:teuthology.orchestra.run.smithi064.stdout:Restarting the monitor... 2023-10-09T05:33:44.206 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:43 smithi064 systemd[1]: Stopping Ceph mon.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:33:44.206 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:43 smithi064 conmon[100096]: 2023-10-09T05:33:43.968+0000 7f931fdf4700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T05:33:44.207 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:43 smithi064 conmon[100096]: 2023-10-09T05:33:43.968+0000 7f931fdf4700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-10-09T05:33:44.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:44 smithi064 bash[100679]: 8ae361a5512b39c50e937ef7fad2e510ab96b6309499ef95b3f05272fd700cdf 2023-10-09T05:33:44.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:44 smithi064 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T05:33:44.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:44 smithi064 systemd[1]: Stopped Ceph mon.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:33:44.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:44 smithi064 systemd[1]: Starting Ceph mon.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:33:45.301 INFO:teuthology.orchestra.run.smithi064.stdout:Setting mon public_network to 172.21.0.0/20 2023-10-09T05:33:45.534 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T05:33:45.534 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable), process ceph-mon, pid 7 2023-10-09T05:33:45.534 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: pidfile_write: ignore empty --pid-file 2023-10-09T05:33:45.534 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: load: jerasure load: lrc 2023-10-09T05:33:45.534 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: RocksDB version: 6.15.5 2023-10-09T05:33:45.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T05:33:45.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Compile date Oct 8 2023 2023-10-09T05:33:45.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: DB SUMMARY 2023-10-09T05:33:45.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: DB Session ID: 4TJ2IMYOHOO919316LU5 2023-10-09T05:33:45.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: CURRENT file: CURRENT 2023-10-09T05:33:45.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T05:33:45.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2023-10-09T05:33:45.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2023-10-09T05:33:45.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83581 ; 2023-10-09T05:33:45.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.error_if_exists: 0 2023-10-09T05:33:45.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.create_if_missing: 0 2023-10-09T05:33:45.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.paranoid_checks: 1 2023-10-09T05:33:45.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T05:33:45.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.env: 0x560a3fbafa80 2023-10-09T05:33:45.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.fs: Posix File System 2023-10-09T05:33:45.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.info_log: 0x560a408ac5a0 2023-10-09T05:33:45.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T05:33:45.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.statistics: (nil) 2023-10-09T05:33:45.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.use_fsync: 0 2023-10-09T05:33:45.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_log_file_size: 0 2023-10-09T05:33:45.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T05:33:45.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T05:33:45.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T05:33:45.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T05:33:45.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.allow_fallocate: 1 2023-10-09T05:33:45.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T05:33:45.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T05:33:45.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.use_direct_reads: 0 2023-10-09T05:33:45.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T05:33:45.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T05:33:45.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.db_log_dir: 2023-10-09T05:33:45.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-10-09T05:33:45.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T05:33:45.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T05:33:45.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T05:33:45.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T05:33:45.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T05:33:45.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T05:33:45.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T05:33:45.539 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T05:33:45.539 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.write_buffer_manager: 0x560a4161e960 2023-10-09T05:33:45.539 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T05:33:45.539 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T05:33:45.539 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T05:33:45.539 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T05:33:45.539 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T05:33:45.539 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T05:33:45.539 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T05:33:45.539 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T05:33:45.539 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T05:33:45.540 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.unordered_write: 0 2023-10-09T05:33:45.540 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T05:33:45.540 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T05:33:45.540 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T05:33:45.540 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T05:33:45.540 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.row_cache: None 2023-10-09T05:33:45.540 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.wal_filter: None 2023-10-09T05:33:45.540 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T05:33:45.540 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T05:33:45.540 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.preserve_deletes: 0 2023-10-09T05:33:45.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.two_write_queues: 0 2023-10-09T05:33:45.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T05:33:45.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.atomic_flush: 0 2023-10-09T05:33:45.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T05:33:45.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T05:33:45.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T05:33:45.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.log_readahead_size: 0 2023-10-09T05:33:45.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T05:33:45.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T05:33:45.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T05:33:45.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T05:33:45.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T05:33:45.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T05:33:45.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_background_jobs: 2 2023-10-09T05:33:45.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_background_compactions: -1 2023-10-09T05:33:45.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_subcompactions: 1 2023-10-09T05:33:45.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T05:33:45.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T05:33:45.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T05:33:45.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T05:33:45.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T05:33:45.543 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T05:33:45.543 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T05:33:45.543 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T05:33:45.543 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_open_files: -1 2023-10-09T05:33:45.543 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T05:33:45.543 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T05:33:45.543 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T05:33:45.543 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T05:33:45.543 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_background_flushes: -1 2023-10-09T05:33:45.543 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Compression algorithms supported: 2023-10-09T05:33:45.543 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T05:33:45.544 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: kZSTD supported: 0 2023-10-09T05:33:45.544 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: kXpressCompression supported: 0 2023-10-09T05:33:45.544 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T05:33:45.544 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: kZlibCompression supported: 1 2023-10-09T05:33:45.544 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: kSnappyCompression supported: 1 2023-10-09T05:33:45.544 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: kLZ4Compression supported: 1 2023-10-09T05:33:45.544 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: kBZip2Compression supported: 0 2023-10-09T05:33:45.544 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T05:33:45.544 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2023-10-09T05:33:45.544 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-10-09T05:33:45.544 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T05:33:45.545 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.merge_operator: 2023-10-09T05:33:45.545 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.compaction_filter: None 2023-10-09T05:33:45.545 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T05:33:45.545 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T05:33:45.545 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T05:33:45.545 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T05:33:45.545 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560a408fefb8) 2023-10-09T05:33:45.545 INFO:journalctl@ceph.mon.a.smithi064.stdout: cache_index_and_filter_blocks: 1 2023-10-09T05:33:45.545 INFO:journalctl@ceph.mon.a.smithi064.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T05:33:45.545 INFO:journalctl@ceph.mon.a.smithi064.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T05:33:45.545 INFO:journalctl@ceph.mon.a.smithi064.stdout: pin_top_level_index_and_filter: 1 2023-10-09T05:33:45.545 INFO:journalctl@ceph.mon.a.smithi064.stdout: index_type: 0 2023-10-09T05:33:45.546 INFO:journalctl@ceph.mon.a.smithi064.stdout: data_block_index_type: 0 2023-10-09T05:33:45.546 INFO:journalctl@ceph.mon.a.smithi064.stdout: index_shortening: 1 2023-10-09T05:33:45.546 INFO:journalctl@ceph.mon.a.smithi064.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T05:33:45.546 INFO:journalctl@ceph.mon.a.smithi064.stdout: hash_index_allow_collision: 1 2023-10-09T05:33:45.546 INFO:journalctl@ceph.mon.a.smithi064.stdout: checksum: 1 2023-10-09T05:33:45.546 INFO:journalctl@ceph.mon.a.smithi064.stdout: no_block_cache: 0 2023-10-09T05:33:45.546 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_cache: 0x560a40931090 2023-10-09T05:33:45.546 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_cache_name: BinnedLRUCache 2023-10-09T05:33:45.546 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_cache_options: 2023-10-09T05:33:45.546 INFO:journalctl@ceph.mon.a.smithi064.stdout: capacity : 536870912 2023-10-09T05:33:45.547 INFO:journalctl@ceph.mon.a.smithi064.stdout: num_shard_bits : 4 2023-10-09T05:33:45.547 INFO:journalctl@ceph.mon.a.smithi064.stdout: strict_capacity_limit : 0 2023-10-09T05:33:45.547 INFO:journalctl@ceph.mon.a.smithi064.stdout: high_pri_pool_ratio: 0.000 2023-10-09T05:33:45.547 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_cache_compressed: (nil) 2023-10-09T05:33:45.547 INFO:journalctl@ceph.mon.a.smithi064.stdout: persistent_cache: (nil) 2023-10-09T05:33:45.547 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_size: 4096 2023-10-09T05:33:45.547 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_size_deviation: 10 2023-10-09T05:33:45.547 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_restart_interval: 16 2023-10-09T05:33:45.547 INFO:journalctl@ceph.mon.a.smithi064.stdout: index_block_restart_interval: 1 2023-10-09T05:33:45.547 INFO:journalctl@ceph.mon.a.smithi064.stdout: metadata_block_size: 4096 2023-10-09T05:33:45.547 INFO:journalctl@ceph.mon.a.smithi064.stdout: partition_filters: 0 2023-10-09T05:33:45.547 INFO:journalctl@ceph.mon.a.smithi064.stdout: use_delta_encoding: 1 2023-10-09T05:33:45.548 INFO:journalctl@ceph.mon.a.smithi064.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T05:33:45.548 INFO:journalctl@ceph.mon.a.smithi064.stdout: whole_key_filtering: 1 2023-10-09T05:33:45.548 INFO:journalctl@ceph.mon.a.smithi064.stdout: verify_compression: 0 2023-10-09T05:33:45.548 INFO:journalctl@ceph.mon.a.smithi064.stdout: read_amp_bytes_per_bit: 0 2023-10-09T05:33:45.548 INFO:journalctl@ceph.mon.a.smithi064.stdout: format_version: 4 2023-10-09T05:33:45.548 INFO:journalctl@ceph.mon.a.smithi064.stdout: enable_index_compression: 1 2023-10-09T05:33:45.548 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_align: 0 2023-10-09T05:33:45.548 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T05:33:45.548 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T05:33:45.548 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.compression: NoCompression 2023-10-09T05:33:45.548 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T05:33:45.549 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T05:33:45.549 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T05:33:45.549 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.num_levels: 7 2023-10-09T05:33:45.549 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T05:33:45.549 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T05:33:45.549 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T05:33:45.549 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T05:33:45.549 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T05:33:45.549 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T05:33:45.549 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T05:33:45.549 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:33:45.550 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T05:33:45.550 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T05:33:45.550 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T05:33:45.550 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T05:33:45.550 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T05:33:45.550 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T05:33:45.550 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:33:45.550 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T05:33:45.550 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T05:33:45.550 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T05:33:45.550 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T05:33:45.551 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T05:33:45.551 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T05:33:45.551 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T05:33:45.551 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T05:33:45.551 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T05:33:45.551 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T05:33:45.551 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T05:33:45.551 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T05:33:45.551 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T05:33:45.551 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T05:33:45.551 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T05:33:45.551 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T05:33:45.552 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T05:33:45.552 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T05:33:45.552 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T05:33:45.552 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.arena_block_size: 4194304 2023-10-09T05:33:45.552 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T05:33:45.552 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T05:33:45.552 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T05:33:45.552 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T05:33:45.552 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T05:33:45.552 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T05:33:45.553 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T05:33:45.553 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T05:33:45.553 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T05:33:45.553 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T05:33:45.553 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T05:33:45.553 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T05:33:45.553 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T05:33:45.554 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T05:33:45.554 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.table_properties_collectors: 2023-10-09T05:33:45.554 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.inplace_update_support: 0 2023-10-09T05:33:45.554 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T05:33:45.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T05:33:45.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T05:33:45.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T05:33:45.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.bloom_locality: 0 2023-10-09T05:33:45.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.max_successive_merges: 0 2023-10-09T05:33:45.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T05:33:45.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T05:33:45.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T05:33:45.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T05:33:45.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.ttl: 2592000 2023-10-09T05:33:45.556 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T05:33:45.556 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.enable_blob_files: false 2023-10-09T05:33:45.556 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.min_blob_size: 0 2023-10-09T05:33:45.556 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T05:33:45.556 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T05:33:45.556 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T05:33:45.556 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T05:33:45.556 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T05:33:45.556 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2023-10-09T05:33:45.556 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2023-10-09T05:33:45.556 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696829625287864, "job": 1, "event": "recovery_started", "wal_files": [10]} 2023-10-09T05:33:45.556 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2023-10-09T05:33:45.557 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T05:33:45.557 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696829625288582, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80128, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78396, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10136, "raw_average_key_size": 47, "raw_value_size": 72604, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696829625, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "c2b484cd-9d2d-4f9f-83c4-d113cbfb15fa", "db_session_id": "4TJ2IMYOHOO919316LU5"}} 2023-10-09T05:33:45.557 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2023-10-09T05:33:45.557 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696829625289526, "job": 1, "event": "recovery_finished"} 2023-10-09T05:33:45.557 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T05:33:45.557 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x560a408c5180 2023-10-09T05:33:45.557 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: DB pointer 0x560a4099a000 2023-10-09T05:33:45.557 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-10-09T05:33:45.557 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-10-09T05:33:45.557 INFO:journalctl@ceph.mon.a.smithi064.stdout: ** DB Stats ** 2023-10-09T05:33:45.558 INFO:journalctl@ceph.mon.a.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:33:45.558 INFO:journalctl@ceph.mon.a.smithi064.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T05:33:45.558 INFO:journalctl@ceph.mon.a.smithi064.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:33:45.558 INFO:journalctl@ceph.mon.a.smithi064.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:33:45.558 INFO:journalctl@ceph.mon.a.smithi064.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T05:33:45.558 INFO:journalctl@ceph.mon.a.smithi064.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T05:33:45.558 INFO:journalctl@ceph.mon.a.smithi064.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:33:45.558 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-10-09T05:33:45.558 INFO:journalctl@ceph.mon.a.smithi064.stdout: ** Compaction Stats [default] ** 2023-10-09T05:33:45.558 INFO:journalctl@ceph.mon.a.smithi064.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:33:45.559 INFO:journalctl@ceph.mon.a.smithi064.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:33:45.559 INFO:journalctl@ceph.mon.a.smithi064.stdout: L0 2/0 80.01 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 170.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:33:45.559 INFO:journalctl@ceph.mon.a.smithi064.stdout: Sum 2/0 80.01 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 170.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:33:45.559 INFO:journalctl@ceph.mon.a.smithi064.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 170.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:33:45.559 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-10-09T05:33:45.559 INFO:journalctl@ceph.mon.a.smithi064.stdout: ** Compaction Stats [default] ** 2023-10-09T05:33:45.559 INFO:journalctl@ceph.mon.a.smithi064.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:33:45.559 INFO:journalctl@ceph.mon.a.smithi064.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:33:45.559 INFO:journalctl@ceph.mon.a.smithi064.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 170.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:33:45.559 INFO:journalctl@ceph.mon.a.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:33:45.560 INFO:journalctl@ceph.mon.a.smithi064.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:33:45.560 INFO:journalctl@ceph.mon.a.smithi064.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:33:45.560 INFO:journalctl@ceph.mon.a.smithi064.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:33:45.560 INFO:journalctl@ceph.mon.a.smithi064.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:33:45.560 INFO:journalctl@ceph.mon.a.smithi064.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:33:45.560 INFO:journalctl@ceph.mon.a.smithi064.stdout: Cumulative compaction: 0.00 GB write, 20.65 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:33:45.560 INFO:journalctl@ceph.mon.a.smithi064.stdout: Interval compaction: 0.00 GB write, 20.65 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:33:45.560 INFO:journalctl@ceph.mon.a.smithi064.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:33:45.560 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-10-09T05:33:45.560 INFO:journalctl@ceph.mon.a.smithi064.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:33:45.560 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-10-09T05:33:45.561 INFO:journalctl@ceph.mon.a.smithi064.stdout: ** Compaction Stats [default] ** 2023-10-09T05:33:45.561 INFO:journalctl@ceph.mon.a.smithi064.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:33:45.561 INFO:journalctl@ceph.mon.a.smithi064.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:33:45.561 INFO:journalctl@ceph.mon.a.smithi064.stdout: L0 2/0 80.01 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 170.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:33:45.561 INFO:journalctl@ceph.mon.a.smithi064.stdout: Sum 2/0 80.01 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 170.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:33:45.561 INFO:journalctl@ceph.mon.a.smithi064.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T05:33:45.561 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-10-09T05:33:45.561 INFO:journalctl@ceph.mon.a.smithi064.stdout: ** Compaction Stats [default] ** 2023-10-09T05:33:45.561 INFO:journalctl@ceph.mon.a.smithi064.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:33:45.561 INFO:journalctl@ceph.mon.a.smithi064.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:33:45.562 INFO:journalctl@ceph.mon.a.smithi064.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 170.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:33:45.562 INFO:journalctl@ceph.mon.a.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:33:45.562 INFO:journalctl@ceph.mon.a.smithi064.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:33:45.562 INFO:journalctl@ceph.mon.a.smithi064.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:33:45.562 INFO:journalctl@ceph.mon.a.smithi064.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:33:45.562 INFO:journalctl@ceph.mon.a.smithi064.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:33:45.562 INFO:journalctl@ceph.mon.a.smithi064.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:33:45.562 INFO:journalctl@ceph.mon.a.smithi064.stdout: Cumulative compaction: 0.00 GB write, 20.48 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:33:45.562 INFO:journalctl@ceph.mon.a.smithi064.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:33:45.562 INFO:journalctl@ceph.mon.a.smithi064.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:33:45.562 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-10-09T05:33:45.563 INFO:journalctl@ceph.mon.a.smithi064.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:33:45.563 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: starting mon.a rank 0 at public addrs [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] at bind addrs [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:33:45.563 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: mon.a@-1(???) e1 preinit fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:33:45.563 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: mon.a@-1(???).mds e1 new map 2023-10-09T05:33:45.563 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: mon.a@-1(???).mds e1 print_map 2023-10-09T05:33:45.563 INFO:journalctl@ceph.mon.a.smithi064.stdout: e1 2023-10-09T05:33:45.563 INFO:journalctl@ceph.mon.a.smithi064.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T05:33:45.563 INFO:journalctl@ceph.mon.a.smithi064.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T05:33:45.563 INFO:journalctl@ceph.mon.a.smithi064.stdout: legacy client fscid: -1 2023-10-09T05:33:45.564 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-10-09T05:33:45.564 INFO:journalctl@ceph.mon.a.smithi064.stdout: No filesystems configured 2023-10-09T05:33:45.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T05:33:45.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:33:45.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:33:45.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:33:45.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-10-09T05:33:45.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: expand_channel_meta expand map: {default=false} 2023-10-09T05:33:45.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: expand_channel_meta from 'false' to 'false' 2023-10-09T05:33:45.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: expand_channel_meta expanded map: {default=false} 2023-10-09T05:33:45.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: expand_channel_meta expand map: {default=info} 2023-10-09T05:33:45.565 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: expand_channel_meta from 'info' to 'info' 2023-10-09T05:33:45.565 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: expand_channel_meta expanded map: {default=info} 2023-10-09T05:33:45.565 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: expand_channel_meta expand map: {default=daemon} 2023-10-09T05:33:45.565 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T05:33:45.565 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: expand_channel_meta expanded map: {default=daemon} 2023-10-09T05:33:45.565 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: expand_channel_meta expand map: {default=debug} 2023-10-09T05:33:45.565 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: expand_channel_meta from 'debug' to 'debug' 2023-10-09T05:33:45.565 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: expand_channel_meta expanded map: {default=debug} 2023-10-09T05:33:45.565 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 bash[101005]: c9fda800131015cec90ab377329ba7c26657b63c07194b89591bd6a5fa591809 2023-10-09T05:33:45.565 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 systemd[1]: Started Ceph mon.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:33:45.565 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T05:33:45.566 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: monmap e1: 1 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0]} removed_ranks: {} 2023-10-09T05:33:45.566 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: fsmap 2023-10-09T05:33:45.566 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: osdmap e1: 0 total, 0 up, 0 in 2023-10-09T05:33:45.566 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:45 smithi064 ceph-mon[101070]: mgrmap e1: no daemons active 2023-10-09T05:33:46.448 INFO:teuthology.orchestra.run.smithi064.stdout:Wrote config to /etc/ceph/ceph.conf 2023-10-09T05:33:46.449 INFO:teuthology.orchestra.run.smithi064.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-10-09T05:33:46.449 INFO:teuthology.orchestra.run.smithi064.stdout:Creating mgr... 2023-10-09T05:33:46.449 INFO:teuthology.orchestra.run.smithi064.stdout:Verifying port 9283 ... 2023-10-09T05:33:46.582 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mgr.y 2023-10-09T05:33:46.582 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stderr Failed to reset failed state of unit ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mgr.y.service: Unit ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mgr.y.service not loaded. 2023-10-09T05:33:46.675 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-44c77acc-6665-11ee-8db5-212e2dc638e7.target.wants/ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mgr.y.service → /etc/systemd/system/ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@.service. 2023-10-09T05:33:46.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:46 smithi064 systemd[1]: Starting Ceph mgr.y for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:33:47.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:46 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/714749640' entity='client.admin' 2023-10-09T05:33:47.273 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-09T05:33:47.273 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout disabled 2023-10-09T05:33:47.282 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-09T05:33:47.283 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout inactive 2023-10-09T05:33:47.283 INFO:teuthology.orchestra.run.smithi064.stdout:firewalld.service is not enabled 2023-10-09T05:33:47.283 INFO:teuthology.orchestra.run.smithi064.stdout:Not possible to enable service . firewalld.service is not available 2023-10-09T05:33:47.293 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-09T05:33:47.293 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout disabled 2023-10-09T05:33:47.301 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-09T05:33:47.301 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout inactive 2023-10-09T05:33:47.301 INFO:teuthology.orchestra.run.smithi064.stdout:firewalld.service is not enabled 2023-10-09T05:33:47.301 INFO:teuthology.orchestra.run.smithi064.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2023-10-09T05:33:47.301 INFO:teuthology.orchestra.run.smithi064.stdout:Waiting for mgr to start... 2023-10-09T05:33:47.301 INFO:teuthology.orchestra.run.smithi064.stdout:Waiting for mgr... 2023-10-09T05:33:47.506 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:47 smithi064 bash[101457]: 1f3d7bc3f2423c8c180ded78b0d0120e89ca42215efd12ce3b17c46bc7ebdeca 2023-10-09T05:33:47.506 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:47 smithi064 systemd[1]: Started Ceph mgr.y for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:33:47.507 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:47 smithi064 conmon[101498]: 2023-10-09T05:33:47.409+0000 7fdf571aa1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:33:47.507 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:47 smithi064 conmon[101498]: 2023-10-09T05:33:47.505+0000 7fdf571aa1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:33:47.942 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:47 smithi064 conmon[101498]: 2023-10-09T05:33:47.625+0000 7fdf571aa1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:33:47.942 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:47 smithi064 conmon[101498]: 2023-10-09T05:33:47.736+0000 7fdf571aa1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:33:47.942 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:47 smithi064 conmon[101498]: 2023-10-09T05:33:47.940+0000 7fdf571aa1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:33:48.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:48 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1270366204' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T05:33:48.583 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2023-10-09T05:33:48.583 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout { 2023-10-09T05:33:48.583 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "fsid": "44c77acc-6665-11ee-8db5-212e2dc638e7", 2023-10-09T05:33:48.583 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T05:33:48.583 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T05:33:48.584 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T05:33:48.584 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T05:33:48.584 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:48.584 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T05:33:48.584 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T05:33:48.584 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 0 2023-10-09T05:33:48.584 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout ], 2023-10-09T05:33:48.584 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T05:33:48.584 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T05:33:48.584 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout ], 2023-10-09T05:33:48.584 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2023-10-09T05:33:48.585 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T05:33:48.585 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:33:48.585 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-10-09T05:33:48.585 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T05:33:48.585 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:48.585 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T05:33:48.585 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:33:48.585 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T05:33:48.585 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T05:33:48.585 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T05:33:48.585 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T05:33:48.585 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T05:33:48.586 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T05:33:48.589 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:48.589 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T05:33:48.589 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T05:33:48.589 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T05:33:48.589 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T05:33:48.589 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T05:33:48.589 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T05:33:48.589 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T05:33:48.590 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T05:33:48.590 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T05:33:48.590 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:48.590 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T05:33:48.590 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:33:48.590 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T05:33:48.590 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T05:33:48.590 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:48.590 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T05:33:48.590 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "available": false, 2023-10-09T05:33:48.591 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T05:33:48.591 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T05:33:48.591 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T05:33:48.591 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T05:33:48.591 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T05:33:48.591 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout ], 2023-10-09T05:33:48.591 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T05:33:48.591 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:48.591 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T05:33:48.593 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:33:48.593 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T05:33:40.038001+0000", 2023-10-09T05:33:48.593 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T05:33:48.593 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:48.593 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T05:33:48.593 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout } 2023-10-09T05:33:48.593 INFO:teuthology.orchestra.run.smithi064.stdout:mgr not available, waiting (1/15)... 2023-10-09T05:33:48.609 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:48 smithi064 conmon[101498]: 2023-10-09T05:33:48.350+0000 7fdf571aa1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:33:48.609 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:48 smithi064 conmon[101498]: 2023-10-09T05:33:48.457+0000 7fdf571aa1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:33:48.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:48 smithi064 conmon[101498]: 2023-10-09T05:33:48.607+0000 7fdf571aa1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:33:48.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:48 smithi064 conmon[101498]: 2023-10-09T05:33:48.720+0000 7fdf571aa1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:33:49.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:49 smithi064 conmon[101498]: 2023-10-09T05:33:49.227+0000 7fdf571aa1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:33:50.220 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:49 smithi064 conmon[101498]: 2023-10-09T05:33:49.917+0000 7fdf571aa1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:33:50.220 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:50 smithi064 conmon[101498]: 2023-10-09T05:33:50.106+0000 7fdf571aa1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:33:51.404 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:51 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/857904502' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T05:33:51.405 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:51 smithi064 conmon[101498]: 2023-10-09T05:33:51.073+0000 7fdf571aa1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:33:51.405 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:51 smithi064 conmon[101498]: 2023-10-09T05:33:51.182+0000 7fdf571aa1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:33:51.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:51 smithi064 conmon[101498]: 2023-10-09T05:33:51.408+0000 7fdf571aa1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:33:51.764 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2023-10-09T05:33:51.769 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout { 2023-10-09T05:33:51.769 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "fsid": "44c77acc-6665-11ee-8db5-212e2dc638e7", 2023-10-09T05:33:51.769 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T05:33:51.770 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T05:33:51.770 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T05:33:51.770 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T05:33:51.770 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:51.770 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T05:33:51.770 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T05:33:51.770 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 0 2023-10-09T05:33:51.770 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout ], 2023-10-09T05:33:51.770 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T05:33:51.770 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T05:33:51.770 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout ], 2023-10-09T05:33:51.770 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "quorum_age": 5, 2023-10-09T05:33:51.771 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T05:33:51.771 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:33:51.771 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-10-09T05:33:51.771 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T05:33:51.771 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:51.771 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T05:33:51.771 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:33:51.771 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T05:33:51.771 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T05:33:51.771 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T05:33:51.771 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T05:33:51.771 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T05:33:51.772 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T05:33:51.772 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:51.772 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T05:33:51.772 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T05:33:51.772 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T05:33:51.772 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T05:33:51.772 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T05:33:51.772 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T05:33:51.772 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T05:33:51.772 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T05:33:51.772 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T05:33:51.772 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:51.772 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T05:33:51.773 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:33:51.773 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T05:33:51.773 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T05:33:51.773 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:51.773 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T05:33:51.773 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "available": false, 2023-10-09T05:33:51.773 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T05:33:51.773 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T05:33:51.773 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T05:33:51.773 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T05:33:51.773 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T05:33:51.773 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout ], 2023-10-09T05:33:51.774 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T05:33:51.774 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:51.774 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T05:33:51.774 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:33:51.774 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T05:33:40.038001+0000", 2023-10-09T05:33:51.774 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T05:33:51.774 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:51.774 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T05:33:51.774 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout } 2023-10-09T05:33:51.774 INFO:teuthology.orchestra.run.smithi064.stdout:mgr not available, waiting (2/15)... 2023-10-09T05:33:52.185 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:51 smithi064 conmon[101498]: 2023-10-09T05:33:51.750+0000 7fdf571aa1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:33:52.185 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:51 smithi064 conmon[101498]: 2023-10-09T05:33:51.854+0000 7fdf571aa1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:33:52.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:52 smithi064 conmon[101498]: 2023-10-09T05:33:52.183+0000 7fdf571aa1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:33:52.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:52 smithi064 conmon[101498]: 2023-10-09T05:33:52.502+0000 7fdf571aa1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:33:53.715 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:53 smithi064 conmon[101498]: 2023-10-09T05:33:53.323+0000 7fdf571aa1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:33:53.715 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:53 smithi064 conmon[101498]: 2023-10-09T05:33:53.478+0000 7fdf571aa1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:33:53.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:53 smithi064 conmon[101498]: 2023-10-09T05:33:53.714+0000 7fdf571aa1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:33:53.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:53 smithi064 conmon[101498]: 2023-10-09T05:33:53.855+0000 7fdf571aa1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:33:54.407 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:54 smithi064 conmon[101498]: 2023-10-09T05:33:54.130+0000 7fdf571aa1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:33:54.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:54 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1552292777' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T05:33:54.960 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2023-10-09T05:33:54.960 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout { 2023-10-09T05:33:54.960 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "fsid": "44c77acc-6665-11ee-8db5-212e2dc638e7", 2023-10-09T05:33:54.960 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T05:33:54.960 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T05:33:54.960 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T05:33:54.960 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T05:33:54.960 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:54.961 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T05:33:54.961 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T05:33:54.961 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 0 2023-10-09T05:33:54.961 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout ], 2023-10-09T05:33:54.961 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T05:33:54.961 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T05:33:54.961 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout ], 2023-10-09T05:33:54.961 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "quorum_age": 9, 2023-10-09T05:33:54.961 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T05:33:54.961 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:33:54.961 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-10-09T05:33:54.962 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T05:33:54.962 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:54.962 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T05:33:54.962 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:33:54.962 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T05:33:54.962 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T05:33:54.962 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T05:33:54.962 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T05:33:54.965 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T05:33:54.965 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T05:33:54.965 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:54.966 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T05:33:54.966 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T05:33:54.966 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T05:33:54.966 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T05:33:54.966 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T05:33:54.966 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T05:33:54.966 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T05:33:54.966 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T05:33:54.967 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T05:33:54.967 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:54.967 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T05:33:54.967 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:33:54.967 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T05:33:54.967 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T05:33:54.967 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:54.967 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T05:33:54.967 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "available": false, 2023-10-09T05:33:54.967 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T05:33:54.967 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T05:33:54.968 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T05:33:54.968 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T05:33:54.968 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T05:33:54.968 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout ], 2023-10-09T05:33:54.968 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T05:33:54.968 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:54.968 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T05:33:54.968 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:33:54.968 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T05:33:40.038001+0000", 2023-10-09T05:33:54.968 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T05:33:54.968 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:54.969 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T05:33:54.969 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout } 2023-10-09T05:33:54.969 INFO:teuthology.orchestra.run.smithi064.stdout:mgr not available, waiting (3/15)... 2023-10-09T05:33:55.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:33:55 smithi064 conmon[101498]: 2023-10-09T05:33:55.178+0000 7fdf571aa1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:33:55.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:55 smithi064 ceph-mon[101070]: Activating manager daemon y 2023-10-09T05:33:55.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:55 smithi064 ceph-mon[101070]: mgrmap e2: y(active, starting, since 0.0027703s) 2023-10-09T05:33:55.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:55 smithi064 ceph-mon[101070]: from='mgr.14100 172.21.15.64:0/3825583079' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:33:55.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:55 smithi064 ceph-mon[101070]: from='mgr.14100 172.21.15.64:0/3825583079' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:33:55.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:55 smithi064 ceph-mon[101070]: from='mgr.14100 172.21.15.64:0/3825583079' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:33:55.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:55 smithi064 ceph-mon[101070]: from='mgr.14100 172.21.15.64:0/3825583079' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:33:55.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:55 smithi064 ceph-mon[101070]: from='mgr.14100 172.21.15.64:0/3825583079' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:33:55.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:55 smithi064 ceph-mon[101070]: Manager daemon y is now available 2023-10-09T05:33:55.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:55 smithi064 ceph-mon[101070]: from='mgr.14100 172.21.15.64:0/3825583079' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:33:55.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:55 smithi064 ceph-mon[101070]: from='mgr.14100 172.21.15.64:0/3825583079' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:33:55.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:55 smithi064 ceph-mon[101070]: from='mgr.14100 172.21.15.64:0/3825583079' entity='mgr.y' 2023-10-09T05:33:55.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:55 smithi064 ceph-mon[101070]: from='mgr.14100 172.21.15.64:0/3825583079' entity='mgr.y' 2023-10-09T05:33:55.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:55 smithi064 ceph-mon[101070]: from='mgr.14100 172.21.15.64:0/3825583079' entity='mgr.y' 2023-10-09T05:33:57.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:57 smithi064 ceph-mon[101070]: mgrmap e3: y(active, since 1.00876s) 2023-10-09T05:33:58.448 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2023-10-09T05:33:58.448 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout { 2023-10-09T05:33:58.449 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "fsid": "44c77acc-6665-11ee-8db5-212e2dc638e7", 2023-10-09T05:33:58.449 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T05:33:58.449 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T05:33:58.449 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T05:33:58.449 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T05:33:58.449 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:58.449 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T05:33:58.449 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T05:33:58.449 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 0 2023-10-09T05:33:58.449 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout ], 2023-10-09T05:33:58.450 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T05:33:58.450 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T05:33:58.450 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout ], 2023-10-09T05:33:58.450 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "quorum_age": 12, 2023-10-09T05:33:58.450 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T05:33:58.450 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:33:58.450 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-10-09T05:33:58.450 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T05:33:58.450 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:58.450 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T05:33:58.451 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:33:58.451 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T05:33:58.451 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T05:33:58.451 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T05:33:58.451 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T05:33:58.451 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T05:33:58.451 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T05:33:58.454 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:58.454 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T05:33:58.454 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T05:33:58.454 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T05:33:58.454 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T05:33:58.455 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T05:33:58.455 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T05:33:58.455 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T05:33:58.455 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T05:33:58.455 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T05:33:58.455 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:58.455 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T05:33:58.455 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:33:58.455 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T05:33:58.455 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T05:33:58.455 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:58.456 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T05:33:58.456 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "available": true, 2023-10-09T05:33:58.456 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T05:33:58.456 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T05:33:58.456 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T05:33:58.456 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T05:33:58.456 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T05:33:58.456 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout ], 2023-10-09T05:33:58.456 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T05:33:58.456 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:58.456 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T05:33:58.457 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:33:58.457 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T05:33:40.038001+0000", 2023-10-09T05:33:58.457 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T05:33:58.457 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:33:58.457 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T05:33:58.457 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout } 2023-10-09T05:33:58.457 INFO:teuthology.orchestra.run.smithi064.stdout:mgr is available 2023-10-09T05:33:58.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:58 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/2334450810' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T05:33:59.468 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:33:59 smithi064 ceph-mon[101070]: mgrmap e4: y(active, since 3s) 2023-10-09T05:33:59.864 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2023-10-09T05:33:59.864 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout [global] 2023-10-09T05:33:59.864 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout fsid = 44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:33:59.864 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-10-09T05:33:59.864 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-10-09T05:33:59.864 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-10-09T05:33:59.864 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-10-09T05:33:59.864 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2023-10-09T05:33:59.865 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout [mgr] 2023-10-09T05:33:59.865 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-10-09T05:33:59.865 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 2023-10-09T05:33:59.865 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout [osd] 2023-10-09T05:33:59.865 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-10-09T05:33:59.865 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-10-09T05:33:59.865 INFO:teuthology.orchestra.run.smithi064.stdout:Enabling cephadm module... 2023-10-09T05:34:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:00 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/2784180077' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T05:34:01.580 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:01 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3760738030' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-10-09T05:34:01.580 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:01 smithi064 conmon[101498]: ignoring --setuser ceph since I am not root 2023-10-09T05:34:01.580 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:01 smithi064 conmon[101498]: ignoring --setgroup ceph since I am not root 2023-10-09T05:34:01.580 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:01 smithi064 conmon[101498]: 2023-10-09T05:34:01.487+0000 7f25cdbb91c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:34:01.580 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:01 smithi064 conmon[101498]: 2023-10-09T05:34:01.579+0000 7f25cdbb91c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:34:01.885 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:01 smithi064 conmon[101498]: 2023-10-09T05:34:01.685+0000 7f25cdbb91c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:34:01.885 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:01 smithi064 conmon[101498]: 2023-10-09T05:34:01.793+0000 7f25cdbb91c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:34:02.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:02 smithi064 conmon[101498]: 2023-10-09T05:34:02.002+0000 7f25cdbb91c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:34:02.671 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:02 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3760738030' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-10-09T05:34:02.671 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:02 smithi064 ceph-mon[101070]: mgrmap e5: y(active, since 6s) 2023-10-09T05:34:02.671 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:02 smithi064 conmon[101498]: 2023-10-09T05:34:02.416+0000 7f25cdbb91c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:34:02.671 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:02 smithi064 conmon[101498]: 2023-10-09T05:34:02.522+0000 7f25cdbb91c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:34:02.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:02 smithi064 conmon[101498]: 2023-10-09T05:34:02.671+0000 7f25cdbb91c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:34:02.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:02 smithi064 conmon[101498]: 2023-10-09T05:34:02.783+0000 7f25cdbb91c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:34:03.333 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout { 2023-10-09T05:34:03.333 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 5, 2023-10-09T05:34:03.333 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "available": true, 2023-10-09T05:34:03.334 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-10-09T05:34:03.334 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-10-09T05:34:03.334 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout } 2023-10-09T05:34:03.334 INFO:teuthology.orchestra.run.smithi064.stdout:Waiting for the mgr to restart... 2023-10-09T05:34:03.334 INFO:teuthology.orchestra.run.smithi064.stdout:Waiting for mgr epoch 5... 2023-10-09T05:34:03.558 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:03 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3399121986' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T05:34:03.559 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:03 smithi064 conmon[101498]: 2023-10-09T05:34:03.297+0000 7f25cdbb91c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:34:04.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:03 smithi064 conmon[101498]: 2023-10-09T05:34:03.995+0000 7f25cdbb91c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:34:04.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:04 smithi064 conmon[101498]: 2023-10-09T05:34:04.184+0000 7f25cdbb91c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:34:05.453 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:05 smithi064 conmon[101498]: 2023-10-09T05:34:05.132+0000 7f25cdbb91c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:34:05.453 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:05 smithi064 conmon[101498]: 2023-10-09T05:34:05.240+0000 7f25cdbb91c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:34:05.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:05 smithi064 conmon[101498]: 2023-10-09T05:34:05.453+0000 7f25cdbb91c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:34:06.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:05 smithi064 conmon[101498]: 2023-10-09T05:34:05.793+0000 7f25cdbb91c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:34:06.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:05 smithi064 conmon[101498]: 2023-10-09T05:34:05.896+0000 7f25cdbb91c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:34:06.557 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:06 smithi064 conmon[101498]: 2023-10-09T05:34:06.233+0000 7f25cdbb91c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:34:06.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:06 smithi064 conmon[101498]: 2023-10-09T05:34:06.557+0000 7f25cdbb91c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:34:07.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:07 smithi064 conmon[101498]: 2023-10-09T05:34:07.411+0000 7f25cdbb91c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:34:07.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:07 smithi064 conmon[101498]: 2023-10-09T05:34:07.568+0000 7f25cdbb91c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:34:08.213 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:07 smithi064 conmon[101498]: 2023-10-09T05:34:07.805+0000 7f25cdbb91c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:34:08.213 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:07 smithi064 conmon[101498]: 2023-10-09T05:34:07.943+0000 7f25cdbb91c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:34:08.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:08 smithi064 conmon[101498]: 2023-10-09T05:34:08.212+0000 7f25cdbb91c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:34:09.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:09 smithi064 conmon[101498]: 2023-10-09T05:34:09.267+0000 7f25cdbb91c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:34:09.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:09 smithi064 ceph-mon[101070]: Active manager daemon y restarted 2023-10-09T05:34:09.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:09 smithi064 ceph-mon[101070]: Activating manager daemon y 2023-10-09T05:34:09.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:09 smithi064 ceph-mon[101070]: osdmap e2: 0 total, 0 up, 0 in 2023-10-09T05:34:09.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:09 smithi064 ceph-mon[101070]: mgrmap e6: y(active, starting, since 0.00311502s) 2023-10-09T05:34:09.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:09 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:34:09.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:09 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:34:09.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:09 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:34:09.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:09 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:34:09.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:09 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:34:09.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:09 smithi064 ceph-mon[101070]: Manager daemon y is now available 2023-10-09T05:34:09.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:09 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' 2023-10-09T05:34:09.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:09 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' 2023-10-09T05:34:09.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:09 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:34:09.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:09 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:34:10.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:10 smithi064 ceph-mon[101070]: Found migration_current of "None". Setting to last migration. 2023-10-09T05:34:10.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:10 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:34:10.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:10 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:34:10.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:10 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:34:10.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:10 smithi064 ceph-mon[101070]: mgrmap e7: y(active, since 1.00519s) 2023-10-09T05:34:10.886 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout { 2023-10-09T05:34:10.887 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2023-10-09T05:34:10.887 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "initialized": true 2023-10-09T05:34:10.887 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout } 2023-10-09T05:34:10.887 INFO:teuthology.orchestra.run.smithi064.stdout:mgr epoch 5 is available 2023-10-09T05:34:10.887 INFO:teuthology.orchestra.run.smithi064.stdout:Setting orchestrator backend to cephadm... 2023-10-09T05:34:11.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:11 smithi064 ceph-mon[101070]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T05:34:11.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:11 smithi064 ceph-mon[101070]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T05:34:12.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:12 smithi064 ceph-mon[101070]: [09/Oct/2023:05:34:11] ENGINE Bus STARTING 2023-10-09T05:34:12.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:12 smithi064 ceph-mon[101070]: mgrmap e8: y(active, since 2s) 2023-10-09T05:34:12.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:12 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' 2023-10-09T05:34:12.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:12 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:34:12.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:12 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' 2023-10-09T05:34:12.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:12 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:34:13.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:13 smithi064 ceph-mon[101070]: [09/Oct/2023:05:34:11] ENGINE Serving on https://172.21.15.64:7150 2023-10-09T05:34:13.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:13 smithi064 ceph-mon[101070]: [09/Oct/2023:05:34:11] ENGINE Bus STARTED 2023-10-09T05:34:13.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:13 smithi064 ceph-mon[101070]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:34:13.661 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout value unchanged 2023-10-09T05:34:13.661 INFO:teuthology.orchestra.run.smithi064.stdout:Generating ssh key... 2023-10-09T05:34:14.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:14 smithi064 ceph-mon[101070]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:34:14.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:14 smithi064 conmon[101498]: Generating public/private rsa key pair. 2023-10-09T05:34:14.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:14 smithi064 conmon[101498]: Your identification has been saved in /tmp/tmpc3ns60xn/key. 2023-10-09T05:34:14.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:14 smithi064 conmon[101498]: Your public key has been saved in /tmp/tmpc3ns60xn/key.pub. 2023-10-09T05:34:14.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:14 smithi064 conmon[101498]: The key fingerprint is: 2023-10-09T05:34:14.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:14 smithi064 conmon[101498]: SHA256:p+5xuBJhXgNx+Kk7siZPmmZARiynhYNPtRkFOncdMss ceph-44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:34:14.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:14 smithi064 conmon[101498]: The key's randomart image is: 2023-10-09T05:34:14.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:14 smithi064 conmon[101498]: +---[RSA 3072]----+ 2023-10-09T05:34:14.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:14 smithi064 conmon[101498]: |o. .+o=oo | 2023-10-09T05:34:14.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:14 smithi064 conmon[101498]: |=o+. =o* . | 2023-10-09T05:34:14.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:14 smithi064 conmon[101498]: |oBo + Eo.. | 2023-10-09T05:34:14.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:14 smithi064 conmon[101498]: |.o.o .o = | 2023-10-09T05:34:14.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:14 smithi064 conmon[101498]: |o o +S.. | 2023-10-09T05:34:14.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:14 smithi064 conmon[101498]: |. + + | 2023-10-09T05:34:14.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:14 smithi064 conmon[101498]: | . . o+ . | 2023-10-09T05:34:14.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:14 smithi064 conmon[101498]: | ++o +. + | 2023-10-09T05:34:14.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:14 smithi064 conmon[101498]: | oo+oo ++ | 2023-10-09T05:34:14.971 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:14 smithi064 conmon[101498]: +----[SHA256]-----+ 2023-10-09T05:34:15.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:15 smithi064 ceph-mon[101070]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:34:15.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:15 smithi064 ceph-mon[101070]: Generating ssh key... 2023-10-09T05:34:15.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:15 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' 2023-10-09T05:34:15.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:15 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' 2023-10-09T05:34:16.677 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDOVvCT+IlGdG6ca+krEzW131PnODdwwyK4i1/cm9omWa6Va/rTEjd/6/eJsGQdS08vnFD/GAcDvDj8lryV84hug2/nz25lzi+4RTWWpk/cxn56vPLDHcSV2eNNmvnh9kFnfMsmBQ3DbB2+Dw+IJqX0D1AE4qGupbNkGCqTMOKEM01w0hGrbBjAjpgL0aWWLkWvBALS0V3HRiZzf4tiEQhw11iIffx1GLONBQjrNGEiImtbw/kZ11vzq/Yi6AW09jJ7tUWIAQamrCRqsVjKvKF9n25ro7ZFoGOXiWPtuhFp53GsmMbTyq0+tsvNYUUfGpBOAmkZbFW/byiZNT2dFyvIK56csAEvN+daeKFWQeNYUS1JKZ0PGphNLtz3PvXQmsNTBoPQHw9zs3qvJgCP15O8r8CT674y776TLsA4tABU7DIQFeEeUWTkHG12vY7vVVqoD0go3jjk0RaIvgDTju4qbSqmrJHQfmoMbGZNz9nwECmZjPTZePUwZofGk6+eG/s= ceph-44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:34:16.677 INFO:teuthology.orchestra.run.smithi064.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-10-09T05:34:16.677 INFO:teuthology.orchestra.run.smithi064.stdout:Adding key to root@localhost authorized_keys... 2023-10-09T05:34:16.679 INFO:teuthology.orchestra.run.smithi064.stdout:Adding host smithi064... 2023-10-09T05:34:16.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:16 smithi064 ceph-mon[101070]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:34:18.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:18 smithi064 ceph-mon[101070]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi064", "addr": "172.21.15.64", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:34:18.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:18 smithi064 ceph-mon[101070]: Deploying cephadm binary to smithi064 2023-10-09T05:34:20.044 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout Added host 'smithi064' with addr '172.21.15.64' 2023-10-09T05:34:20.044 INFO:teuthology.orchestra.run.smithi064.stdout:Deploying unmanaged mon service... 2023-10-09T05:34:20.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:20 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' 2023-10-09T05:34:20.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:20 smithi064 ceph-mon[101070]: Added host smithi064 2023-10-09T05:34:20.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:20 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:34:21.712 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2023-10-09T05:34:21.712 INFO:teuthology.orchestra.run.smithi064.stdout:Deploying unmanaged mgr service... 2023-10-09T05:34:22.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:21 smithi064 ceph-mon[101070]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:34:22.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:21 smithi064 ceph-mon[101070]: Saving service mon spec with placement count:5 2023-10-09T05:34:22.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:21 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' 2023-10-09T05:34:22.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:21 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' 2023-10-09T05:34:23.301 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2023-10-09T05:34:23.625 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:23 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' 2023-10-09T05:34:23.626 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:23 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' 2023-10-09T05:34:24.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:24 smithi064 ceph-mon[101070]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:34:24.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:24 smithi064 ceph-mon[101070]: Saving service mgr spec with placement count:2 2023-10-09T05:34:24.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:24 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' 2023-10-09T05:34:25.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:25 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1648202932' entity='client.admin' 2023-10-09T05:34:25.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:25 smithi064 ceph-mon[101070]: from='mgr.14120 172.21.15.64:0/371003000' entity='mgr.y' 2023-10-09T05:34:26.663 INFO:teuthology.orchestra.run.smithi064.stdout:Enabling the dashboard module... 2023-10-09T05:34:26.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:26 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/958198372' entity='client.admin' 2023-10-09T05:34:28.060 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:27 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/487916110' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-10-09T05:34:28.061 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:27 smithi064 conmon[101498]: ignoring --setuser ceph since I am not root 2023-10-09T05:34:28.061 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:27 smithi064 conmon[101498]: ignoring --setgroup ceph since I am not root 2023-10-09T05:34:28.061 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:27 smithi064 conmon[101498]: 2023-10-09T05:34:27.963+0000 7fca4c0411c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:34:28.443 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:28 smithi064 conmon[101498]: 2023-10-09T05:34:28.059+0000 7fca4c0411c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:34:28.443 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:28 smithi064 conmon[101498]: 2023-10-09T05:34:28.165+0000 7fca4c0411c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:34:28.443 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:28 smithi064 conmon[101498]: 2023-10-09T05:34:28.275+0000 7fca4c0411c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:34:28.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:28 smithi064 conmon[101498]: 2023-10-09T05:34:28.494+0000 7fca4c0411c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:34:29.016 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:28 smithi064 conmon[101498]: 2023-10-09T05:34:28.908+0000 7fca4c0411c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:34:29.016 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:28 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/487916110' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-10-09T05:34:29.016 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:28 smithi064 ceph-mon[101070]: mgrmap e9: y(active, since 18s) 2023-10-09T05:34:29.283 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:29 smithi064 conmon[101498]: 2023-10-09T05:34:29.015+0000 7fca4c0411c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:34:29.283 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:29 smithi064 conmon[101498]: 2023-10-09T05:34:29.164+0000 7fca4c0411c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:34:29.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:29 smithi064 conmon[101498]: 2023-10-09T05:34:29.281+0000 7fca4c0411c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:34:29.759 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout { 2023-10-09T05:34:29.759 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "epoch": 9, 2023-10-09T05:34:29.759 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "available": true, 2023-10-09T05:34:29.760 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-10-09T05:34:29.760 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-10-09T05:34:29.760 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout } 2023-10-09T05:34:29.760 INFO:teuthology.orchestra.run.smithi064.stdout:Waiting for the mgr to restart... 2023-10-09T05:34:29.760 INFO:teuthology.orchestra.run.smithi064.stdout:Waiting for mgr epoch 9... 2023-10-09T05:34:30.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:29 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1506054471' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T05:34:30.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:29 smithi064 conmon[101498]: 2023-10-09T05:34:29.800+0000 7fca4c0411c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:34:30.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:30 smithi064 conmon[101498]: 2023-10-09T05:34:30.501+0000 7fca4c0411c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:34:30.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:30 smithi064 conmon[101498]: 2023-10-09T05:34:30.690+0000 7fca4c0411c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:34:31.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:31 smithi064 conmon[101498]: 2023-10-09T05:34:31.646+0000 7fca4c0411c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:34:31.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:31 smithi064 conmon[101498]: 2023-10-09T05:34:31.755+0000 7fca4c0411c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:34:32.315 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:31 smithi064 conmon[101498]: 2023-10-09T05:34:31.972+0000 7fca4c0411c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:34:32.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:32 smithi064 conmon[101498]: 2023-10-09T05:34:32.314+0000 7fca4c0411c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:34:32.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:32 smithi064 conmon[101498]: 2023-10-09T05:34:32.418+0000 7fca4c0411c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:34:33.069 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:32 smithi064 conmon[101498]: 2023-10-09T05:34:32.747+0000 7fca4c0411c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:34:33.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:33 smithi064 conmon[101498]: 2023-10-09T05:34:33.068+0000 7fca4c0411c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:34:34.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:33 smithi064 conmon[101498]: 2023-10-09T05:34:33.891+0000 7fca4c0411c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:34:34.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:34 smithi064 conmon[101498]: 2023-10-09T05:34:34.048+0000 7fca4c0411c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:34:34.699 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:34 smithi064 conmon[101498]: 2023-10-09T05:34:34.288+0000 7fca4c0411c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:34:34.699 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:34 smithi064 conmon[101498]: 2023-10-09T05:34:34.427+0000 7fca4c0411c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:34:34.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:34 smithi064 conmon[101498]: 2023-10-09T05:34:34.698+0000 7fca4c0411c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:34:36.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:34:35 smithi064 conmon[101498]: 2023-10-09T05:34:35.729+0000 7fca4c0411c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:34:36.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:36 smithi064 ceph-mon[101070]: Active manager daemon y restarted 2023-10-09T05:34:36.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:36 smithi064 ceph-mon[101070]: Activating manager daemon y 2023-10-09T05:34:36.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:36 smithi064 ceph-mon[101070]: osdmap e3: 0 total, 0 up, 0 in 2023-10-09T05:34:36.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:36 smithi064 ceph-mon[101070]: mgrmap e10: y(active, starting, since 0.00319407s) 2023-10-09T05:34:36.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:36 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:34:36.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:36 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:34:36.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:36 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:34:36.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:36 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:34:36.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:36 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:34:36.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:36 smithi064 ceph-mon[101070]: Manager daemon y is now available 2023-10-09T05:34:36.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:36 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:34:36.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:36 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:34:36.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:36 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:34:36.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:36 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:34:37.383 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout { 2023-10-09T05:34:37.383 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2023-10-09T05:34:37.383 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout "initialized": true 2023-10-09T05:34:37.383 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout } 2023-10-09T05:34:37.384 INFO:teuthology.orchestra.run.smithi064.stdout:mgr epoch 9 is available 2023-10-09T05:34:37.384 INFO:teuthology.orchestra.run.smithi064.stdout:Generating a dashboard self-signed certificate... 2023-10-09T05:34:38.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:37 smithi064 ceph-mon[101070]: [09/Oct/2023:05:34:36] ENGINE Bus STARTING 2023-10-09T05:34:38.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:37 smithi064 ceph-mon[101070]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T05:34:38.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:37 smithi064 ceph-mon[101070]: mgrmap e11: y(active, since 1.00787s) 2023-10-09T05:34:38.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:37 smithi064 ceph-mon[101070]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T05:34:38.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:37 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:34:38.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:37 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:34:38.891 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2023-10-09T05:34:38.891 INFO:teuthology.orchestra.run.smithi064.stdout:Creating initial admin user... 2023-10-09T05:34:39.073 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:38 smithi064 ceph-mon[101070]: [09/Oct/2023:05:34:36] ENGINE Serving on https://172.21.15.64:7150 2023-10-09T05:34:39.074 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:38 smithi064 ceph-mon[101070]: [09/Oct/2023:05:34:36] ENGINE Bus STARTED 2023-10-09T05:34:39.074 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:38 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:34:39.074 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:38 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:34:39.074 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:38 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:34:39.074 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:38 smithi064 ceph-mon[101070]: mgrmap e12: y(active, since 2s) 2023-10-09T05:34:40.072 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:39 smithi064 ceph-mon[101070]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:34:40.532 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$v4Id06TX0gk5g/eGwT7CdOO2LEcMrG6fK5txNhnCcL8JNjivFB6TG", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1696829680, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-10-09T05:34:40.532 INFO:teuthology.orchestra.run.smithi064.stdout:Fetching dashboard port number... 2023-10-09T05:34:40.787 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:40 smithi064 ceph-mon[101070]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:34:40.787 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:40 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:34:41.932 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stdout 8443 2023-10-09T05:34:41.942 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-09T05:34:41.942 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout disabled 2023-10-09T05:34:41.950 INFO:teuthology.orchestra.run.smithi064.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-09T05:34:41.950 INFO:teuthology.orchestra.run.smithi064.stdout:systemctl: stdout inactive 2023-10-09T05:34:41.950 INFO:teuthology.orchestra.run.smithi064.stdout:firewalld.service is not enabled 2023-10-09T05:34:41.950 INFO:teuthology.orchestra.run.smithi064.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2023-10-09T05:34:41.952 INFO:teuthology.orchestra.run.smithi064.stdout:Ceph Dashboard is now available at: 2023-10-09T05:34:41.952 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:34:41.952 INFO:teuthology.orchestra.run.smithi064.stdout: URL: https://smithi064:8443/ 2023-10-09T05:34:41.952 INFO:teuthology.orchestra.run.smithi064.stdout: User: admin 2023-10-09T05:34:41.952 INFO:teuthology.orchestra.run.smithi064.stdout: Password: wygmw65kdg 2023-10-09T05:34:41.953 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:34:41.953 INFO:teuthology.orchestra.run.smithi064.stdout:Saving cluster configuration to /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config directory 2023-10-09T05:34:41.953 INFO:teuthology.orchestra.run.smithi064.stdout:Enabling autotune for osd_memory_target 2023-10-09T05:34:42.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:41 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1706253855' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-10-09T05:34:43.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:43 smithi064 ceph-mon[101070]: mgrmap e13: y(active, since 7s) 2023-10-09T05:34:44.957 INFO:teuthology.orchestra.run.smithi064.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2023-10-09T05:34:44.957 INFO:teuthology.orchestra.run.smithi064.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-10-09T05:34:44.957 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:34:44.957 INFO:teuthology.orchestra.run.smithi064.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-10-09T05:34:44.958 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:34:44.958 INFO:teuthology.orchestra.run.smithi064.stdout:Or, if you are only running a single cluster on this host: 2023-10-09T05:34:44.958 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:34:44.958 INFO:teuthology.orchestra.run.smithi064.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2023-10-09T05:34:44.958 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:34:44.958 INFO:teuthology.orchestra.run.smithi064.stdout:Please consider enabling telemetry to help improve Ceph: 2023-10-09T05:34:44.958 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:34:44.958 INFO:teuthology.orchestra.run.smithi064.stdout: ceph telemetry on 2023-10-09T05:34:44.958 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:34:44.958 INFO:teuthology.orchestra.run.smithi064.stdout:For more information see: 2023-10-09T05:34:44.958 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:34:44.959 INFO:teuthology.orchestra.run.smithi064.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2023-10-09T05:34:44.959 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:34:44.959 INFO:teuthology.orchestra.run.smithi064.stdout:Bootstrap complete. 2023-10-09T05:34:45.010 INFO:tasks.cephadm:Fetching config... 2023-10-09T05:34:45.011 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-10-09T05:34:45.011 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-10-09T05:34:45.028 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-10-09T05:34:45.028 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-10-09T05:34:45.029 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-10-09T05:34:45.087 INFO:tasks.cephadm:Fetching mon keyring... 2023-10-09T05:34:45.087 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-10-09T05:34:45.087 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-10-09T05:34:45.156 INFO:tasks.cephadm:Fetching pub ssh key... 2023-10-09T05:34:45.156 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-10-09T05:34:45.157 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-10-09T05:34:45.214 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-10-09T05:34:45.214 DEBUG:teuthology.orchestra.run.smithi064:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDOVvCT+IlGdG6ca+krEzW131PnODdwwyK4i1/cm9omWa6Va/rTEjd/6/eJsGQdS08vnFD/GAcDvDj8lryV84hug2/nz25lzi+4RTWWpk/cxn56vPLDHcSV2eNNmvnh9kFnfMsmBQ3DbB2+Dw+IJqX0D1AE4qGupbNkGCqTMOKEM01w0hGrbBjAjpgL0aWWLkWvBALS0V3HRiZzf4tiEQhw11iIffx1GLONBQjrNGEiImtbw/kZ11vzq/Yi6AW09jJ7tUWIAQamrCRqsVjKvKF9n25ro7ZFoGOXiWPtuhFp53GsmMbTyq0+tsvNYUUfGpBOAmkZbFW/byiZNT2dFyvIK56csAEvN+daeKFWQeNYUS1JKZ0PGphNLtz3PvXQmsNTBoPQHw9zs3qvJgCP15O8r8CT674y776TLsA4tABU7DIQFeEeUWTkHG12vY7vVVqoD0go3jjk0RaIvgDTju4qbSqmrJHQfmoMbGZNz9nwECmZjPTZePUwZofGk6+eG/s= ceph-44c77acc-6665-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T05:34:45.293 INFO:teuthology.orchestra.run.smithi064.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDOVvCT+IlGdG6ca+krEzW131PnODdwwyK4i1/cm9omWa6Va/rTEjd/6/eJsGQdS08vnFD/GAcDvDj8lryV84hug2/nz25lzi+4RTWWpk/cxn56vPLDHcSV2eNNmvnh9kFnfMsmBQ3DbB2+Dw+IJqX0D1AE4qGupbNkGCqTMOKEM01w0hGrbBjAjpgL0aWWLkWvBALS0V3HRiZzf4tiEQhw11iIffx1GLONBQjrNGEiImtbw/kZ11vzq/Yi6AW09jJ7tUWIAQamrCRqsVjKvKF9n25ro7ZFoGOXiWPtuhFp53GsmMbTyq0+tsvNYUUfGpBOAmkZbFW/byiZNT2dFyvIK56csAEvN+daeKFWQeNYUS1JKZ0PGphNLtz3PvXQmsNTBoPQHw9zs3qvJgCP15O8r8CT674y776TLsA4tABU7DIQFeEeUWTkHG12vY7vVVqoD0go3jjk0RaIvgDTju4qbSqmrJHQfmoMbGZNz9nwECmZjPTZePUwZofGk6+eG/s= ceph-44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:34:45.305 DEBUG:teuthology.orchestra.run.smithi156:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDOVvCT+IlGdG6ca+krEzW131PnODdwwyK4i1/cm9omWa6Va/rTEjd/6/eJsGQdS08vnFD/GAcDvDj8lryV84hug2/nz25lzi+4RTWWpk/cxn56vPLDHcSV2eNNmvnh9kFnfMsmBQ3DbB2+Dw+IJqX0D1AE4qGupbNkGCqTMOKEM01w0hGrbBjAjpgL0aWWLkWvBALS0V3HRiZzf4tiEQhw11iIffx1GLONBQjrNGEiImtbw/kZ11vzq/Yi6AW09jJ7tUWIAQamrCRqsVjKvKF9n25ro7ZFoGOXiWPtuhFp53GsmMbTyq0+tsvNYUUfGpBOAmkZbFW/byiZNT2dFyvIK56csAEvN+daeKFWQeNYUS1JKZ0PGphNLtz3PvXQmsNTBoPQHw9zs3qvJgCP15O8r8CT674y776TLsA4tABU7DIQFeEeUWTkHG12vY7vVVqoD0go3jjk0RaIvgDTju4qbSqmrJHQfmoMbGZNz9nwECmZjPTZePUwZofGk6+eG/s= ceph-44c77acc-6665-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T05:34:45.364 INFO:teuthology.orchestra.run.smithi156.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDOVvCT+IlGdG6ca+krEzW131PnODdwwyK4i1/cm9omWa6Va/rTEjd/6/eJsGQdS08vnFD/GAcDvDj8lryV84hug2/nz25lzi+4RTWWpk/cxn56vPLDHcSV2eNNmvnh9kFnfMsmBQ3DbB2+Dw+IJqX0D1AE4qGupbNkGCqTMOKEM01w0hGrbBjAjpgL0aWWLkWvBALS0V3HRiZzf4tiEQhw11iIffx1GLONBQjrNGEiImtbw/kZ11vzq/Yi6AW09jJ7tUWIAQamrCRqsVjKvKF9n25ro7ZFoGOXiWPtuhFp53GsmMbTyq0+tsvNYUUfGpBOAmkZbFW/byiZNT2dFyvIK56csAEvN+daeKFWQeNYUS1JKZ0PGphNLtz3PvXQmsNTBoPQHw9zs3qvJgCP15O8r8CT674y776TLsA4tABU7DIQFeEeUWTkHG12vY7vVVqoD0go3jjk0RaIvgDTju4qbSqmrJHQfmoMbGZNz9nwECmZjPTZePUwZofGk6+eG/s= ceph-44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:34:45.377 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-10-09T05:34:45.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:45 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/4006748094' entity='client.admin' 2023-10-09T05:34:45.732 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:48.468 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-10-09T05:34:48.468 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-10-09T05:34:49.126 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:48 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/2868460520' entity='client.admin' 2023-10-09T05:34:49.227 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:50.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:50 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:34:50.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:50 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:34:50.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:50 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi064", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:34:50.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:50 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:34:50.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:50 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:34:50.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:50 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:34:51.372 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi156 2023-10-09T05:34:51.372 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2023-10-09T05:34:51.372 DEBUG:teuthology.orchestra.run.smithi156:> dd of=/etc/ceph/ceph.conf 2023-10-09T05:34:51.392 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2023-10-09T05:34:51.392 DEBUG:teuthology.orchestra.run.smithi156:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:34:51.452 INFO:tasks.cephadm:Adding host smithi156 to orchestrator... 2023-10-09T05:34:51.452 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph orch host add smithi156 2023-10-09T05:34:51.819 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:52.013 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:51 smithi064 ceph-mon[101070]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:34:52.013 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:51 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:34:52.013 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:51 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:34:52.013 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:51 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:34:52.013 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:51 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:34:52.013 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:51 smithi064 ceph-mon[101070]: Updating smithi064:/etc/ceph/ceph.conf 2023-10-09T05:34:52.956 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:52 smithi064 ceph-mon[101070]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:34:52.956 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:52 smithi064 ceph-mon[101070]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:34:52.956 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:52 smithi064 ceph-mon[101070]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:34:52.956 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:52 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:34:52.957 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:52 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:34:52.957 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:52 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:34:54.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:53 smithi064 ceph-mon[101070]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi156", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:34:55.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:54 smithi064 ceph-mon[101070]: Deploying cephadm binary to smithi156 2023-10-09T05:34:55.311 INFO:teuthology.orchestra.run.smithi064.stdout:Added host 'smithi156' with addr '172.21.15.156' 2023-10-09T05:34:55.878 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph orch host ls --format=json 2023-10-09T05:34:56.225 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:34:56.578 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:56 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:34:56.578 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:56 smithi064 ceph-mon[101070]: Added host smithi156 2023-10-09T05:34:56.578 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:56 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:34:56.578 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:56 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:34:56.578 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:56 smithi064 ceph-mon[101070]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:34:56.579 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:56 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:34:57.781 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:34:57.781 INFO:teuthology.orchestra.run.smithi064.stdout:[{"addr": "172.21.15.64", "hostname": "smithi064", "labels": [], "status": ""}, {"addr": "172.21.15.156", "hostname": "smithi156", "labels": [], "status": ""}] 2023-10-09T05:34:58.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:57 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:34:58.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:57 smithi064 ceph-mon[101070]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:34:58.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:34:57 smithi064 ceph-mon[101070]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:34:58.378 INFO:tasks.cephadm:Setting crush tunables to default 2023-10-09T05:34:58.379 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd crush tunables default 2023-10-09T05:34:58.729 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:01.134 INFO:teuthology.orchestra.run.smithi064.stderr:adjusted tunables profile to default 2023-10-09T05:35:01.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:00 smithi064 ceph-mon[101070]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:01.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:00 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:01.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:00 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:01.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:00 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:01.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:00 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:01.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:00 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi156", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:35:01.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:00 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:01.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:00 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:35:01.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:00 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/2715771670' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-10-09T05:35:01.702 INFO:tasks.cephadm:Adding mon.a on smithi064 2023-10-09T05:35:01.702 INFO:tasks.cephadm:Adding mon.c on smithi064 2023-10-09T05:35:01.703 INFO:tasks.cephadm:Adding mon.b on smithi156 2023-10-09T05:35:01.703 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph orch apply mon '3;smithi064:172.21.15.64=a;smithi064:[v2:172.21.15.64:3301,v1:172.21.15.64:6790]=c;smithi156:172.21.15.156=b' 2023-10-09T05:35:01.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:01 smithi064 ceph-mon[101070]: Updating smithi156:/etc/ceph/ceph.conf 2023-10-09T05:35:01.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:01 smithi064 ceph-mon[101070]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:35:01.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:01 smithi064 ceph-mon[101070]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:35:01.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:01 smithi064 ceph-mon[101070]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:35:01.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:01 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:01.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:01 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:01.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:01 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:01.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:01 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/2715771670' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-09T05:35:01.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:01 smithi064 ceph-mon[101070]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:35:02.065 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:35:03.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:02 smithi064 ceph-mon[101070]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:03.699 INFO:teuthology.orchestra.run.smithi156.stdout:Scheduled mon update... 2023-10-09T05:35:04.342 DEBUG:teuthology.orchestra.run.smithi064:mon.c> sudo journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mon.c.service 2023-10-09T05:35:04.345 DEBUG:teuthology.orchestra.run.smithi156:mon.b> sudo journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mon.b.service 2023-10-09T05:35:04.347 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T05:35:04.349 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T05:35:04.376 INFO:journalctl@ceph.mon.b.smithi156.stdout:-- Logs begin at Mon 2023-10-09 05:16:12 UTC. -- 2023-10-09T05:35:04.382 INFO:journalctl@ceph.mon.c.smithi064.stdout:-- Logs begin at Mon 2023-10-09 05:16:16 UTC. -- 2023-10-09T05:35:04.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:04 smithi064 ceph-mon[101070]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi064:172.21.15.64=a;smithi064:[v2:172.21.15.64:3301,v1:172.21.15.64:6790]=c;smithi156:172.21.15.156=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:35:04.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:04 smithi064 ceph-mon[101070]: Saving service mon spec with placement smithi064:172.21.15.64=a;smithi064:[v2:172.21.15.64:3301,v1:172.21.15.64:6790]=c;smithi156:172.21.15.156=b;count:3 2023-10-09T05:35:04.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:04 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:04.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:04 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:04.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:04 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:04.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:04 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:35:04.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:04 smithi064 ceph-mon[101070]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:04.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:04 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:04.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:04 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:35:04.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:04 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:04.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:04 smithi064 ceph-mon[101070]: Deploying daemon mon.b on smithi156 2023-10-09T05:35:05.323 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:35:06.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:06 smithi064 ceph-mon[101070]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:07.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 systemd[1]: Starting Ceph mon.b for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:35:07.728 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:35:07.728 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":1,"fsid":"44c77acc-6665-11ee-8db5-212e2dc638e7","modified":"2023-10-09T05:33:37.342634Z","created":"2023-10-09T05:33:37.342634Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:3300","nonce":0},{"type":"v1","addr":"172.21.15.64:6789","nonce":0}]},"addr":"172.21.15.64:6789/0","public_addr":"172.21.15.64:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-10-09T05:35:07.730 INFO:teuthology.orchestra.run.smithi156.stderr:dumped monmap epoch 1 2023-10-09T05:35:08.061 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T05:35:08.061 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable), process ceph-mon, pid 7 2023-10-09T05:35:08.061 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: pidfile_write: ignore empty --pid-file 2023-10-09T05:35:08.062 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 bash[105911]: c60a890df99eb0a3df217e512882753954791ae5c057840ae022e5a4f1d80d64 2023-10-09T05:35:08.062 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 systemd[1]: Started Ceph mon.b for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:35:08.062 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: load: jerasure load: lrc 2023-10-09T05:35:08.062 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: RocksDB version: 6.15.5 2023-10-09T05:35:08.062 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T05:35:08.062 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Compile date Oct 8 2023 2023-10-09T05:35:08.062 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: DB SUMMARY 2023-10-09T05:35:08.062 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: DB Session ID: CQLBNJ57ZY370Y6T8GE2 2023-10-09T05:35:08.062 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: CURRENT file: CURRENT 2023-10-09T05:35:08.063 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T05:35:08.063 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-10-09T05:35:08.063 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2023-10-09T05:35:08.063 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2023-10-09T05:35:08.063 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.error_if_exists: 0 2023-10-09T05:35:08.063 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.create_if_missing: 0 2023-10-09T05:35:08.063 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.paranoid_checks: 1 2023-10-09T05:35:08.063 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T05:35:08.063 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.env: 0x5610afd9ea80 2023-10-09T05:35:08.063 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.fs: Posix File System 2023-10-09T05:35:08.064 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.info_log: 0x5610b0ec65a0 2023-10-09T05:35:08.064 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T05:35:08.064 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.statistics: (nil) 2023-10-09T05:35:08.064 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.use_fsync: 0 2023-10-09T05:35:08.064 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_log_file_size: 0 2023-10-09T05:35:08.064 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T05:35:08.064 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T05:35:08.064 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T05:35:08.064 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T05:35:08.064 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.allow_fallocate: 1 2023-10-09T05:35:08.065 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T05:35:08.065 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T05:35:08.065 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.use_direct_reads: 0 2023-10-09T05:35:08.065 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T05:35:08.065 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T05:35:08.065 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.db_log_dir: 2023-10-09T05:35:08.065 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2023-10-09T05:35:08.065 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T05:35:08.065 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T05:35:08.066 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T05:35:08.066 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T05:35:08.066 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T05:35:08.066 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T05:35:08.066 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T05:35:08.066 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T05:35:08.066 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.write_buffer_manager: 0x5610b1c38960 2023-10-09T05:35:08.066 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T05:35:08.066 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T05:35:08.066 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T05:35:08.067 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T05:35:08.067 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T05:35:08.067 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T05:35:08.067 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T05:35:08.067 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T05:35:08.067 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T05:35:08.067 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.unordered_write: 0 2023-10-09T05:35:08.067 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T05:35:08.067 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T05:35:08.067 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T05:35:08.067 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T05:35:08.068 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.row_cache: None 2023-10-09T05:35:08.068 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.wal_filter: None 2023-10-09T05:35:08.068 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T05:35:08.068 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T05:35:08.068 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.preserve_deletes: 0 2023-10-09T05:35:08.068 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.two_write_queues: 0 2023-10-09T05:35:08.068 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T05:35:08.068 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.atomic_flush: 0 2023-10-09T05:35:08.068 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T05:35:08.068 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T05:35:08.069 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T05:35:08.069 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.log_readahead_size: 0 2023-10-09T05:35:08.069 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T05:35:08.069 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T05:35:08.069 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T05:35:08.069 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T05:35:08.069 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T05:35:08.069 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T05:35:08.069 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_background_jobs: 2 2023-10-09T05:35:08.070 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_background_compactions: -1 2023-10-09T05:35:08.070 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_subcompactions: 1 2023-10-09T05:35:08.070 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T05:35:08.070 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T05:35:08.070 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T05:35:08.070 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T05:35:08.071 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T05:35:08.071 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T05:35:08.071 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T05:35:08.071 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T05:35:08.071 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_open_files: -1 2023-10-09T05:35:08.071 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T05:35:08.071 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T05:35:08.071 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T05:35:08.071 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T05:35:08.071 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_background_flushes: -1 2023-10-09T05:35:08.071 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Compression algorithms supported: 2023-10-09T05:35:08.072 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T05:35:08.072 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: kZSTD supported: 0 2023-10-09T05:35:08.072 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: kXpressCompression supported: 0 2023-10-09T05:35:08.072 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T05:35:08.072 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: kZlibCompression supported: 1 2023-10-09T05:35:08.072 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: kSnappyCompression supported: 1 2023-10-09T05:35:08.072 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: kLZ4Compression supported: 1 2023-10-09T05:35:08.073 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: kBZip2Compression supported: 0 2023-10-09T05:35:08.073 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T05:35:08.073 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2023-10-09T05:35:08.073 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-10-09T05:35:08.073 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T05:35:08.073 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.merge_operator: 2023-10-09T05:35:08.073 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.compaction_filter: None 2023-10-09T05:35:08.073 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T05:35:08.073 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T05:35:08.073 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T05:35:08.074 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T05:35:08.074 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5610b0f18fb8) 2023-10-09T05:35:08.074 INFO:journalctl@ceph.mon.b.smithi156.stdout: cache_index_and_filter_blocks: 1 2023-10-09T05:35:08.074 INFO:journalctl@ceph.mon.b.smithi156.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T05:35:08.074 INFO:journalctl@ceph.mon.b.smithi156.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T05:35:08.074 INFO:journalctl@ceph.mon.b.smithi156.stdout: pin_top_level_index_and_filter: 1 2023-10-09T05:35:08.074 INFO:journalctl@ceph.mon.b.smithi156.stdout: index_type: 0 2023-10-09T05:35:08.074 INFO:journalctl@ceph.mon.b.smithi156.stdout: data_block_index_type: 0 2023-10-09T05:35:08.075 INFO:journalctl@ceph.mon.b.smithi156.stdout: index_shortening: 1 2023-10-09T05:35:08.075 INFO:journalctl@ceph.mon.b.smithi156.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T05:35:08.075 INFO:journalctl@ceph.mon.b.smithi156.stdout: hash_index_allow_collision: 1 2023-10-09T05:35:08.075 INFO:journalctl@ceph.mon.b.smithi156.stdout: checksum: 1 2023-10-09T05:35:08.075 INFO:journalctl@ceph.mon.b.smithi156.stdout: no_block_cache: 0 2023-10-09T05:35:08.075 INFO:journalctl@ceph.mon.b.smithi156.stdout: block_cache: 0x5610b0f4b090 2023-10-09T05:35:08.075 INFO:journalctl@ceph.mon.b.smithi156.stdout: block_cache_name: BinnedLRUCache 2023-10-09T05:35:08.075 INFO:journalctl@ceph.mon.b.smithi156.stdout: block_cache_options: 2023-10-09T05:35:08.075 INFO:journalctl@ceph.mon.b.smithi156.stdout: capacity : 536870912 2023-10-09T05:35:08.076 INFO:journalctl@ceph.mon.b.smithi156.stdout: num_shard_bits : 4 2023-10-09T05:35:08.076 INFO:journalctl@ceph.mon.b.smithi156.stdout: strict_capacity_limit : 0 2023-10-09T05:35:08.076 INFO:journalctl@ceph.mon.b.smithi156.stdout: high_pri_pool_ratio: 0.000 2023-10-09T05:35:08.076 INFO:journalctl@ceph.mon.b.smithi156.stdout: block_cache_compressed: (nil) 2023-10-09T05:35:08.076 INFO:journalctl@ceph.mon.b.smithi156.stdout: persistent_cache: (nil) 2023-10-09T05:35:08.076 INFO:journalctl@ceph.mon.b.smithi156.stdout: block_size: 4096 2023-10-09T05:35:08.076 INFO:journalctl@ceph.mon.b.smithi156.stdout: block_size_deviation: 10 2023-10-09T05:35:08.077 INFO:journalctl@ceph.mon.b.smithi156.stdout: block_restart_interval: 16 2023-10-09T05:35:08.077 INFO:journalctl@ceph.mon.b.smithi156.stdout: index_block_restart_interval: 1 2023-10-09T05:35:08.077 INFO:journalctl@ceph.mon.b.smithi156.stdout: metadata_block_size: 4096 2023-10-09T05:35:08.077 INFO:journalctl@ceph.mon.b.smithi156.stdout: partition_filters: 0 2023-10-09T05:35:08.077 INFO:journalctl@ceph.mon.b.smithi156.stdout: use_delta_encoding: 1 2023-10-09T05:35:08.077 INFO:journalctl@ceph.mon.b.smithi156.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T05:35:08.077 INFO:journalctl@ceph.mon.b.smithi156.stdout: whole_key_filtering: 1 2023-10-09T05:35:08.077 INFO:journalctl@ceph.mon.b.smithi156.stdout: verify_compression: 0 2023-10-09T05:35:08.077 INFO:journalctl@ceph.mon.b.smithi156.stdout: read_amp_bytes_per_bit: 0 2023-10-09T05:35:08.077 INFO:journalctl@ceph.mon.b.smithi156.stdout: format_version: 4 2023-10-09T05:35:08.078 INFO:journalctl@ceph.mon.b.smithi156.stdout: enable_index_compression: 1 2023-10-09T05:35:08.078 INFO:journalctl@ceph.mon.b.smithi156.stdout: block_align: 0 2023-10-09T05:35:08.078 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T05:35:08.078 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T05:35:08.078 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.compression: NoCompression 2023-10-09T05:35:08.078 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T05:35:08.078 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T05:35:08.078 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T05:35:08.079 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.num_levels: 7 2023-10-09T05:35:08.079 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T05:35:08.079 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T05:35:08.079 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T05:35:08.079 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T05:35:08.079 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T05:35:08.079 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T05:35:08.079 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T05:35:08.080 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:35:08.080 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T05:35:08.080 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T05:35:08.080 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T05:35:08.080 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T05:35:08.080 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T05:35:08.081 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T05:35:08.081 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:35:08.081 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T05:35:08.081 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T05:35:08.081 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T05:35:08.081 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T05:35:08.082 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T05:35:08.082 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T05:35:08.082 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T05:35:08.082 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T05:35:08.082 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T05:35:08.082 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T05:35:08.082 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T05:35:08.082 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T05:35:08.083 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T05:35:08.083 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T05:35:08.083 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T05:35:08.083 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T05:35:08.083 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T05:35:08.083 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T05:35:08.083 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T05:35:08.083 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.arena_block_size: 4194304 2023-10-09T05:35:08.083 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T05:35:08.083 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T05:35:08.084 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T05:35:08.084 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T05:35:08.084 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T05:35:08.084 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T05:35:08.084 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T05:35:08.084 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T05:35:08.084 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T05:35:08.084 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T05:35:08.084 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T05:35:08.084 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T05:35:08.085 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T05:35:08.085 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T05:35:08.085 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.table_properties_collectors: 2023-10-09T05:35:08.085 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.inplace_update_support: 0 2023-10-09T05:35:08.085 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T05:35:08.085 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T05:35:08.085 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T05:35:08.085 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T05:35:08.085 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.bloom_locality: 0 2023-10-09T05:35:08.085 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.max_successive_merges: 0 2023-10-09T05:35:08.086 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T05:35:08.086 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T05:35:08.086 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T05:35:08.086 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T05:35:08.086 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.ttl: 2592000 2023-10-09T05:35:08.086 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T05:35:08.086 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.enable_blob_files: false 2023-10-09T05:35:08.086 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.min_blob_size: 0 2023-10-09T05:35:08.086 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T05:35:08.086 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T05:35:08.087 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T05:35:08.087 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T05:35:08.087 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T05:35:08.087 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-10-09T05:35:08.087 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-10-09T05:35:08.087 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696829707831696, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-10-09T05:35:08.087 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-10-09T05:35:08.087 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T05:35:08.087 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696829707831978, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696829707, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "c54cd4cb-8fcf-43c7-b340-d88169a4b04e", "db_session_id": "CQLBNJ57ZY370Y6T8GE2"}} 2023-10-09T05:35:08.087 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-10-09T05:35:08.088 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696829707832378, "job": 1, "event": "recovery_finished"} 2023-10-09T05:35:08.088 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T05:35:08.088 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5610b0edf180 2023-10-09T05:35:08.088 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: DB pointer 0x5610b0fb4000 2023-10-09T05:35:08.088 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-10-09T05:35:08.088 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-10-09T05:35:08.088 INFO:journalctl@ceph.mon.b.smithi156.stdout: ** DB Stats ** 2023-10-09T05:35:08.088 INFO:journalctl@ceph.mon.b.smithi156.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:35:08.088 INFO:journalctl@ceph.mon.b.smithi156.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T05:35:08.089 INFO:journalctl@ceph.mon.b.smithi156.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:35:08.089 INFO:journalctl@ceph.mon.b.smithi156.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:35:08.089 INFO:journalctl@ceph.mon.b.smithi156.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T05:35:08.089 INFO:journalctl@ceph.mon.b.smithi156.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T05:35:08.089 INFO:journalctl@ceph.mon.b.smithi156.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:35:08.089 INFO:journalctl@ceph.mon.b.smithi156.stdout: 2023-10-09T05:35:08.089 INFO:journalctl@ceph.mon.b.smithi156.stdout: ** Compaction Stats [default] ** 2023-10-09T05:35:08.089 INFO:journalctl@ceph.mon.b.smithi156.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:35:08.089 INFO:journalctl@ceph.mon.b.smithi156.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:35:08.089 INFO:journalctl@ceph.mon.b.smithi156.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2023-10-09T05:35:08.090 INFO:journalctl@ceph.mon.b.smithi156.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2023-10-09T05:35:08.090 INFO:journalctl@ceph.mon.b.smithi156.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2023-10-09T05:35:08.090 INFO:journalctl@ceph.mon.b.smithi156.stdout: 2023-10-09T05:35:08.090 INFO:journalctl@ceph.mon.b.smithi156.stdout: ** Compaction Stats [default] ** 2023-10-09T05:35:08.090 INFO:journalctl@ceph.mon.b.smithi156.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:35:08.090 INFO:journalctl@ceph.mon.b.smithi156.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:35:08.090 INFO:journalctl@ceph.mon.b.smithi156.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2023-10-09T05:35:08.090 INFO:journalctl@ceph.mon.b.smithi156.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:35:08.090 INFO:journalctl@ceph.mon.b.smithi156.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:35:08.090 INFO:journalctl@ceph.mon.b.smithi156.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:35:08.091 INFO:journalctl@ceph.mon.b.smithi156.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:35:08.091 INFO:journalctl@ceph.mon.b.smithi156.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:35:08.091 INFO:journalctl@ceph.mon.b.smithi156.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:35:08.091 INFO:journalctl@ceph.mon.b.smithi156.stdout: Cumulative compaction: 0.00 GB write, 0.48 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:35:08.091 INFO:journalctl@ceph.mon.b.smithi156.stdout: Interval compaction: 0.00 GB write, 0.48 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:35:08.091 INFO:journalctl@ceph.mon.b.smithi156.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:35:08.091 INFO:journalctl@ceph.mon.b.smithi156.stdout: 2023-10-09T05:35:08.091 INFO:journalctl@ceph.mon.b.smithi156.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:35:08.091 INFO:journalctl@ceph.mon.b.smithi156.stdout: 2023-10-09T05:35:08.092 INFO:journalctl@ceph.mon.b.smithi156.stdout: ** Compaction Stats [default] ** 2023-10-09T05:35:08.092 INFO:journalctl@ceph.mon.b.smithi156.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:35:08.092 INFO:journalctl@ceph.mon.b.smithi156.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:35:08.092 INFO:journalctl@ceph.mon.b.smithi156.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2023-10-09T05:35:08.092 INFO:journalctl@ceph.mon.b.smithi156.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2023-10-09T05:35:08.092 INFO:journalctl@ceph.mon.b.smithi156.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T05:35:08.092 INFO:journalctl@ceph.mon.b.smithi156.stdout: 2023-10-09T05:35:08.092 INFO:journalctl@ceph.mon.b.smithi156.stdout: ** Compaction Stats [default] ** 2023-10-09T05:35:08.092 INFO:journalctl@ceph.mon.b.smithi156.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:35:08.092 INFO:journalctl@ceph.mon.b.smithi156.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:35:08.093 INFO:journalctl@ceph.mon.b.smithi156.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2023-10-09T05:35:08.093 INFO:journalctl@ceph.mon.b.smithi156.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:35:08.093 INFO:journalctl@ceph.mon.b.smithi156.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:35:08.093 INFO:journalctl@ceph.mon.b.smithi156.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:35:08.093 INFO:journalctl@ceph.mon.b.smithi156.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:35:08.093 INFO:journalctl@ceph.mon.b.smithi156.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:35:08.093 INFO:journalctl@ceph.mon.b.smithi156.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:35:08.093 INFO:journalctl@ceph.mon.b.smithi156.stdout: Cumulative compaction: 0.00 GB write, 0.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:35:08.093 INFO:journalctl@ceph.mon.b.smithi156.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:35:08.094 INFO:journalctl@ceph.mon.b.smithi156.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:35:08.094 INFO:journalctl@ceph.mon.b.smithi156.stdout: 2023-10-09T05:35:08.094 INFO:journalctl@ceph.mon.b.smithi156.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:35:08.094 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: mon.b does not exist in monmap, will attempt to join an existing cluster 2023-10-09T05:35:08.094 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: using public_addr v2:172.21.15.156:0/0 -> [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] 2023-10-09T05:35:08.094 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: starting mon.b rank -1 at public addrs [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] at bind addrs [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:35:08.094 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: mon.b@-1(???) e0 preinit fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:35:08.094 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: mon.b@-1(synchronizing).mds e1 new map 2023-10-09T05:35:08.094 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: mon.b@-1(synchronizing).mds e1 print_map 2023-10-09T05:35:08.094 INFO:journalctl@ceph.mon.b.smithi156.stdout: e1 2023-10-09T05:35:08.095 INFO:journalctl@ceph.mon.b.smithi156.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T05:35:08.095 INFO:journalctl@ceph.mon.b.smithi156.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T05:35:08.095 INFO:journalctl@ceph.mon.b.smithi156.stdout: legacy client fscid: -1 2023-10-09T05:35:08.095 INFO:journalctl@ceph.mon.b.smithi156.stdout: 2023-10-09T05:35:08.095 INFO:journalctl@ceph.mon.b.smithi156.stdout: No filesystems configured 2023-10-09T05:35:08.095 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-10-09T05:35:08.095 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-10-09T05:35:08.095 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-10-09T05:35:08.095 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-10-09T05:35:08.095 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-10-09T05:35:08.096 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-10-09T05:35:08.096 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T05:35:08.096 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:35:08.096 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:35:08.096 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:35:08.096 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:08.096 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:08.096 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:35:08.096 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:08.096 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:08.097 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:08.097 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:08.097 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:08.097 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi156", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:35:08.097 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:08.097 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:35:08.097 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/2715771670' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-10-09T05:35:08.097 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: Updating smithi156:/etc/ceph/ceph.conf 2023-10-09T05:35:08.097 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:35:08.097 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:35:08.098 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:35:08.098 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:08.098 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:08.098 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:08.098 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/2715771670' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-09T05:35:08.098 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:35:08.098 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:08.098 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi064:172.21.15.64=a;smithi064:[v2:172.21.15.64:3301,v1:172.21.15.64:6790]=c;smithi156:172.21.15.156=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:35:08.098 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: Saving service mon spec with placement smithi064:172.21.15.64=a;smithi064:[v2:172.21.15.64:3301,v1:172.21.15.64:6790]=c;smithi156:172.21.15.156=b;count:3 2023-10-09T05:35:08.098 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:08.099 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:08.099 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:08.099 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:35:08.099 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:08.099 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:08.099 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:35:08.099 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:08.099 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: Deploying daemon mon.b on smithi156 2023-10-09T05:35:08.099 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:08.099 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: from='client.? 172.21.15.156:0/2267448357' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:35:08.100 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-09T05:35:08.100 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:07 smithi156 ceph-mon[105979]: mon.b@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete 2023-10-09T05:35:08.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:07 smithi064 ceph-mon[101070]: from='client.? 172.21.15.156:0/2267448357' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:35:09.637 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T05:35:09.638 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T05:35:09.981 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:35:13.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:12 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:35:13.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:12 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:35:13.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:12 smithi064 ceph-mon[101070]: mon.a calling monitor election 2023-10-09T05:35:13.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:12 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:35:13.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:12 smithi064 ceph-mon[101070]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:13.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:12 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:35:13.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:12 smithi064 ceph-mon[101070]: mon.b calling monitor election 2023-10-09T05:35:13.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:12 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:35:13.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:12 smithi064 ceph-mon[101070]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:13.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:12 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:35:13.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:12 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:35:13.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:12 smithi064 ceph-mon[101070]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T05:35:13.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:12 smithi064 ceph-mon[101070]: monmap e2: 2 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0]} removed_ranks: {} 2023-10-09T05:35:13.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:12 smithi064 ceph-mon[101070]: fsmap 2023-10-09T05:35:13.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:12 smithi064 ceph-mon[101070]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:35:13.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:12 smithi064 ceph-mon[101070]: mgrmap e13: y(active, since 37s) 2023-10-09T05:35:13.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:12 smithi064 ceph-mon[101070]: overall HEALTH_OK 2023-10-09T05:35:13.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:12 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:13.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:12 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:13.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:12 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:13.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:12 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:35:13.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:12 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:13.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:12 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:35:13.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:12 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:35:13.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:12 smithi156 ceph-mon[105979]: mon.a calling monitor election 2023-10-09T05:35:13.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:12 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:35:13.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:12 smithi156 ceph-mon[105979]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:13.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:12 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:35:13.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:12 smithi156 ceph-mon[105979]: mon.b calling monitor election 2023-10-09T05:35:13.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:12 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:35:13.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:12 smithi156 ceph-mon[105979]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:13.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:12 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:35:13.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:12 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:35:13.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:12 smithi156 ceph-mon[105979]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T05:35:13.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:12 smithi156 ceph-mon[105979]: monmap e2: 2 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0]} removed_ranks: {} 2023-10-09T05:35:13.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:12 smithi156 ceph-mon[105979]: fsmap 2023-10-09T05:35:13.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:12 smithi156 ceph-mon[105979]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:35:13.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:12 smithi156 ceph-mon[105979]: mgrmap e13: y(active, since 37s) 2023-10-09T05:35:13.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:12 smithi156 ceph-mon[105979]: overall HEALTH_OK 2023-10-09T05:35:13.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:12 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:13.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:12 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:13.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:12 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:13.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:12 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:35:13.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:12 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:14.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:13 smithi064 ceph-mon[101070]: Deploying daemon mon.c on smithi064 2023-10-09T05:35:14.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:13 smithi064 ceph-mon[101070]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:14.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:13 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:35:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:13 smithi156 ceph-mon[105979]: Deploying daemon mon.c on smithi064 2023-10-09T05:35:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:13 smithi156 ceph-mon[105979]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:13 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:35:14.562 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:35:14.562 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":2,"fsid":"44c77acc-6665-11ee-8db5-212e2dc638e7","modified":"2023-10-09T05:35:07.857261Z","created":"2023-10-09T05:33:37.342634Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:3300","nonce":0},{"type":"v1","addr":"172.21.15.64:6789","nonce":0}]},"addr":"172.21.15.64:6789/0","public_addr":"172.21.15.64:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:3300","nonce":0},{"type":"v1","addr":"172.21.15.156:6789","nonce":0}]},"addr":"172.21.15.156:6789/0","public_addr":"172.21.15.156:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-10-09T05:35:14.564 INFO:teuthology.orchestra.run.smithi156.stderr:dumped monmap epoch 2 2023-10-09T05:35:15.249 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:14 smithi064 ceph-mon[101070]: from='client.? 172.21.15.156:0/3358280317' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:35:15.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:14 smithi156 ceph-mon[105979]: from='client.? 172.21.15.156:0/3358280317' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:35:15.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 systemd[1]: Starting Ceph mon.c for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:35:16.107 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T05:35:16.107 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T05:35:16.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T05:35:16.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable), process ceph-mon, pid 7 2023-10-09T05:35:16.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: pidfile_write: ignore empty --pid-file 2023-10-09T05:35:16.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 bash[118288]: b6790a9456bae7dc3eaff42c53aa886f88fad291308ad3e786e951c2d3d255e8 2023-10-09T05:35:16.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: load: jerasure load: lrc 2023-10-09T05:35:16.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: RocksDB version: 6.15.5 2023-10-09T05:35:16.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T05:35:16.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Compile date Oct 8 2023 2023-10-09T05:35:16.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: DB SUMMARY 2023-10-09T05:35:16.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: DB Session ID: LJCYL3JW37HHV5XLRAW5 2023-10-09T05:35:16.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: CURRENT file: CURRENT 2023-10-09T05:35:16.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T05:35:16.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-10-09T05:35:16.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-10-09T05:35:16.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2023-10-09T05:35:16.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.error_if_exists: 0 2023-10-09T05:35:16.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.create_if_missing: 0 2023-10-09T05:35:16.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.paranoid_checks: 1 2023-10-09T05:35:16.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T05:35:16.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.env: 0x56107b2cda80 2023-10-09T05:35:16.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.fs: Posix File System 2023-10-09T05:35:16.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.info_log: 0x56107d5de5a0 2023-10-09T05:35:16.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T05:35:16.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.statistics: (nil) 2023-10-09T05:35:16.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.use_fsync: 0 2023-10-09T05:35:16.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_log_file_size: 0 2023-10-09T05:35:16.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T05:35:16.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T05:35:16.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T05:35:16.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T05:35:16.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.allow_fallocate: 1 2023-10-09T05:35:16.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T05:35:16.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T05:35:16.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.use_direct_reads: 0 2023-10-09T05:35:16.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T05:35:16.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T05:35:16.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.db_log_dir: 2023-10-09T05:35:16.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2023-10-09T05:35:16.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T05:35:16.224 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T05:35:16.224 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T05:35:16.224 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T05:35:16.224 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T05:35:16.225 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T05:35:16.225 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T05:35:16.225 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T05:35:16.225 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.write_buffer_manager: 0x56107e350930 2023-10-09T05:35:16.225 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T05:35:16.225 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T05:35:16.226 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T05:35:16.226 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T05:35:16.226 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T05:35:16.226 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T05:35:16.226 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T05:35:16.227 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T05:35:16.227 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T05:35:16.227 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.unordered_write: 0 2023-10-09T05:35:16.227 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T05:35:16.227 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T05:35:16.227 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T05:35:16.228 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T05:35:16.228 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.row_cache: None 2023-10-09T05:35:16.228 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.wal_filter: None 2023-10-09T05:35:16.228 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T05:35:16.228 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T05:35:16.228 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.preserve_deletes: 0 2023-10-09T05:35:16.228 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.two_write_queues: 0 2023-10-09T05:35:16.229 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T05:35:16.229 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.atomic_flush: 0 2023-10-09T05:35:16.229 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T05:35:16.229 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T05:35:16.229 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T05:35:16.229 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.log_readahead_size: 0 2023-10-09T05:35:16.229 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T05:35:16.230 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T05:35:16.230 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T05:35:16.230 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T05:35:16.230 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T05:35:16.230 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T05:35:16.230 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_background_jobs: 2 2023-10-09T05:35:16.231 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_background_compactions: -1 2023-10-09T05:35:16.231 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_subcompactions: 1 2023-10-09T05:35:16.231 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T05:35:16.231 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T05:35:16.231 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T05:35:16.232 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T05:35:16.232 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T05:35:16.232 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T05:35:16.232 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T05:35:16.232 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T05:35:16.232 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_open_files: -1 2023-10-09T05:35:16.232 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T05:35:16.232 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T05:35:16.233 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T05:35:16.233 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T05:35:16.233 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_background_flushes: -1 2023-10-09T05:35:16.233 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Compression algorithms supported: 2023-10-09T05:35:16.233 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T05:35:16.233 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: kZSTD supported: 0 2023-10-09T05:35:16.233 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: kXpressCompression supported: 0 2023-10-09T05:35:16.233 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T05:35:16.233 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: kZlibCompression supported: 1 2023-10-09T05:35:16.233 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: kSnappyCompression supported: 1 2023-10-09T05:35:16.233 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: kLZ4Compression supported: 1 2023-10-09T05:35:16.233 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: kBZip2Compression supported: 0 2023-10-09T05:35:16.234 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T05:35:16.234 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2023-10-09T05:35:16.234 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-10-09T05:35:16.234 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T05:35:16.234 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.merge_operator: 2023-10-09T05:35:16.234 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.compaction_filter: None 2023-10-09T05:35:16.234 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T05:35:16.234 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T05:35:16.234 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T05:35:16.234 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T05:35:16.234 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56107d630fb8) 2023-10-09T05:35:16.234 INFO:journalctl@ceph.mon.c.smithi064.stdout: cache_index_and_filter_blocks: 1 2023-10-09T05:35:16.235 INFO:journalctl@ceph.mon.c.smithi064.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T05:35:16.235 INFO:journalctl@ceph.mon.c.smithi064.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T05:35:16.235 INFO:journalctl@ceph.mon.c.smithi064.stdout: pin_top_level_index_and_filter: 1 2023-10-09T05:35:16.235 INFO:journalctl@ceph.mon.c.smithi064.stdout: index_type: 0 2023-10-09T05:35:16.235 INFO:journalctl@ceph.mon.c.smithi064.stdout: data_block_index_type: 0 2023-10-09T05:35:16.235 INFO:journalctl@ceph.mon.c.smithi064.stdout: index_shortening: 1 2023-10-09T05:35:16.235 INFO:journalctl@ceph.mon.c.smithi064.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T05:35:16.235 INFO:journalctl@ceph.mon.c.smithi064.stdout: hash_index_allow_collision: 1 2023-10-09T05:35:16.235 INFO:journalctl@ceph.mon.c.smithi064.stdout: checksum: 1 2023-10-09T05:35:16.235 INFO:journalctl@ceph.mon.c.smithi064.stdout: no_block_cache: 0 2023-10-09T05:35:16.235 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_cache: 0x56107d663090 2023-10-09T05:35:16.235 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_cache_name: BinnedLRUCache 2023-10-09T05:35:16.236 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_cache_options: 2023-10-09T05:35:16.236 INFO:journalctl@ceph.mon.c.smithi064.stdout: capacity : 536870912 2023-10-09T05:35:16.236 INFO:journalctl@ceph.mon.c.smithi064.stdout: num_shard_bits : 4 2023-10-09T05:35:16.236 INFO:journalctl@ceph.mon.c.smithi064.stdout: strict_capacity_limit : 0 2023-10-09T05:35:16.236 INFO:journalctl@ceph.mon.c.smithi064.stdout: high_pri_pool_ratio: 0.000 2023-10-09T05:35:16.236 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_cache_compressed: (nil) 2023-10-09T05:35:16.236 INFO:journalctl@ceph.mon.c.smithi064.stdout: persistent_cache: (nil) 2023-10-09T05:35:16.236 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_size: 4096 2023-10-09T05:35:16.236 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_size_deviation: 10 2023-10-09T05:35:16.236 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_restart_interval: 16 2023-10-09T05:35:16.237 INFO:journalctl@ceph.mon.c.smithi064.stdout: index_block_restart_interval: 1 2023-10-09T05:35:16.237 INFO:journalctl@ceph.mon.c.smithi064.stdout: metadata_block_size: 4096 2023-10-09T05:35:16.237 INFO:journalctl@ceph.mon.c.smithi064.stdout: partition_filters: 0 2023-10-09T05:35:16.237 INFO:journalctl@ceph.mon.c.smithi064.stdout: use_delta_encoding: 1 2023-10-09T05:35:16.237 INFO:journalctl@ceph.mon.c.smithi064.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T05:35:16.237 INFO:journalctl@ceph.mon.c.smithi064.stdout: whole_key_filtering: 1 2023-10-09T05:35:16.237 INFO:journalctl@ceph.mon.c.smithi064.stdout: verify_compression: 0 2023-10-09T05:35:16.237 INFO:journalctl@ceph.mon.c.smithi064.stdout: read_amp_bytes_per_bit: 0 2023-10-09T05:35:16.237 INFO:journalctl@ceph.mon.c.smithi064.stdout: format_version: 4 2023-10-09T05:35:16.237 INFO:journalctl@ceph.mon.c.smithi064.stdout: enable_index_compression: 1 2023-10-09T05:35:16.237 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_align: 0 2023-10-09T05:35:16.237 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T05:35:16.237 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T05:35:16.238 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.compression: NoCompression 2023-10-09T05:35:16.238 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T05:35:16.238 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T05:35:16.238 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T05:35:16.238 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.num_levels: 7 2023-10-09T05:35:16.238 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T05:35:16.238 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T05:35:16.238 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T05:35:16.238 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T05:35:16.238 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T05:35:16.238 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T05:35:16.239 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T05:35:16.239 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:35:16.239 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T05:35:16.239 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T05:35:16.239 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T05:35:16.239 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T05:35:16.239 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T05:35:16.239 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T05:35:16.239 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:35:16.239 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T05:35:16.239 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T05:35:16.239 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T05:35:16.239 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T05:35:16.240 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T05:35:16.240 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T05:35:16.240 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T05:35:16.240 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T05:35:16.240 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T05:35:16.240 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T05:35:16.240 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T05:35:16.240 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T05:35:16.240 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T05:35:16.240 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T05:35:16.240 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T05:35:16.240 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T05:35:16.241 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T05:35:16.241 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T05:35:16.241 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T05:35:16.241 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.arena_block_size: 4194304 2023-10-09T05:35:16.241 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T05:35:16.241 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T05:35:16.241 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T05:35:16.241 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T05:35:16.241 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T05:35:16.241 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T05:35:16.241 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T05:35:16.241 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T05:35:16.242 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T05:35:16.242 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T05:35:16.242 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T05:35:16.242 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T05:35:16.242 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T05:35:16.242 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T05:35:16.242 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.table_properties_collectors: 2023-10-09T05:35:16.242 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.inplace_update_support: 0 2023-10-09T05:35:16.242 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T05:35:16.242 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T05:35:16.242 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T05:35:16.243 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T05:35:16.243 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.bloom_locality: 0 2023-10-09T05:35:16.243 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.max_successive_merges: 0 2023-10-09T05:35:16.243 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T05:35:16.243 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T05:35:16.243 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T05:35:16.243 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T05:35:16.243 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.ttl: 2592000 2023-10-09T05:35:16.243 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T05:35:16.243 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.enable_blob_files: false 2023-10-09T05:35:16.243 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.min_blob_size: 0 2023-10-09T05:35:16.243 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T05:35:16.243 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T05:35:16.244 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T05:35:16.244 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T05:35:16.244 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T05:35:16.244 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-10-09T05:35:16.244 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-10-09T05:35:16.244 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 systemd[1]: Started Ceph mon.c for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:35:16.244 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696829715946769, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-10-09T05:35:16.244 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-10-09T05:35:16.244 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T05:35:16.244 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696829715947172, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696829715, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "9d335a72-2f19-456c-b884-2132a5374b29", "db_session_id": "LJCYL3JW37HHV5XLRAW5"}} 2023-10-09T05:35:16.244 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-10-09T05:35:16.244 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696829715947713, "job": 1, "event": "recovery_finished"} 2023-10-09T05:35:16.245 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T05:35:16.245 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x56107d5f7180 2023-10-09T05:35:16.245 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: DB pointer 0x56107d6cc000 2023-10-09T05:35:16.245 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-10-09T05:35:16.245 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-10-09T05:35:16.245 INFO:journalctl@ceph.mon.c.smithi064.stdout: ** DB Stats ** 2023-10-09T05:35:16.245 INFO:journalctl@ceph.mon.c.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:35:16.245 INFO:journalctl@ceph.mon.c.smithi064.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T05:35:16.245 INFO:journalctl@ceph.mon.c.smithi064.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:35:16.245 INFO:journalctl@ceph.mon.c.smithi064.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:35:16.245 INFO:journalctl@ceph.mon.c.smithi064.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T05:35:16.245 INFO:journalctl@ceph.mon.c.smithi064.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T05:35:16.246 INFO:journalctl@ceph.mon.c.smithi064.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:35:16.246 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-10-09T05:35:16.246 INFO:journalctl@ceph.mon.c.smithi064.stdout: ** Compaction Stats [default] ** 2023-10-09T05:35:16.246 INFO:journalctl@ceph.mon.c.smithi064.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:35:16.246 INFO:journalctl@ceph.mon.c.smithi064.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:35:16.246 INFO:journalctl@ceph.mon.c.smithi064.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:35:16.246 INFO:journalctl@ceph.mon.c.smithi064.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:35:16.246 INFO:journalctl@ceph.mon.c.smithi064.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:35:16.246 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-10-09T05:35:16.246 INFO:journalctl@ceph.mon.c.smithi064.stdout: ** Compaction Stats [default] ** 2023-10-09T05:35:16.246 INFO:journalctl@ceph.mon.c.smithi064.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:35:16.246 INFO:journalctl@ceph.mon.c.smithi064.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:35:16.247 INFO:journalctl@ceph.mon.c.smithi064.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:35:16.247 INFO:journalctl@ceph.mon.c.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:35:16.247 INFO:journalctl@ceph.mon.c.smithi064.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:35:16.247 INFO:journalctl@ceph.mon.c.smithi064.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:35:16.247 INFO:journalctl@ceph.mon.c.smithi064.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:35:16.247 INFO:journalctl@ceph.mon.c.smithi064.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:35:16.247 INFO:journalctl@ceph.mon.c.smithi064.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:35:16.247 INFO:journalctl@ceph.mon.c.smithi064.stdout: Cumulative compaction: 0.00 GB write, 0.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:35:16.247 INFO:journalctl@ceph.mon.c.smithi064.stdout: Interval compaction: 0.00 GB write, 0.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:35:16.247 INFO:journalctl@ceph.mon.c.smithi064.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:35:16.247 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-10-09T05:35:16.248 INFO:journalctl@ceph.mon.c.smithi064.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:35:16.248 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-10-09T05:35:16.248 INFO:journalctl@ceph.mon.c.smithi064.stdout: ** Compaction Stats [default] ** 2023-10-09T05:35:16.248 INFO:journalctl@ceph.mon.c.smithi064.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:35:16.248 INFO:journalctl@ceph.mon.c.smithi064.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:35:16.248 INFO:journalctl@ceph.mon.c.smithi064.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:35:16.248 INFO:journalctl@ceph.mon.c.smithi064.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:35:16.248 INFO:journalctl@ceph.mon.c.smithi064.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T05:35:16.248 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-10-09T05:35:16.248 INFO:journalctl@ceph.mon.c.smithi064.stdout: ** Compaction Stats [default] ** 2023-10-09T05:35:16.248 INFO:journalctl@ceph.mon.c.smithi064.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:35:16.248 INFO:journalctl@ceph.mon.c.smithi064.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:35:16.248 INFO:journalctl@ceph.mon.c.smithi064.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:35:16.249 INFO:journalctl@ceph.mon.c.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:35:16.249 INFO:journalctl@ceph.mon.c.smithi064.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:35:16.249 INFO:journalctl@ceph.mon.c.smithi064.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:35:16.249 INFO:journalctl@ceph.mon.c.smithi064.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:35:16.249 INFO:journalctl@ceph.mon.c.smithi064.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:35:16.249 INFO:journalctl@ceph.mon.c.smithi064.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:35:16.249 INFO:journalctl@ceph.mon.c.smithi064.stdout: Cumulative compaction: 0.00 GB write, 0.40 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:35:16.249 INFO:journalctl@ceph.mon.c.smithi064.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:35:16.249 INFO:journalctl@ceph.mon.c.smithi064.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:35:16.249 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-10-09T05:35:16.249 INFO:journalctl@ceph.mon.c.smithi064.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:35:16.249 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: mon.c does not exist in monmap, will attempt to join an existing cluster 2023-10-09T05:35:16.249 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: using public_addrv [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] 2023-10-09T05:35:16.250 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: starting mon.c rank -1 at public addrs [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] at bind addrs [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:35:16.250 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: mon.c@-1(???) e0 preinit fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:35:16.250 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: mon.c@-1(synchronizing).mds e1 new map 2023-10-09T05:35:16.250 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: mon.c@-1(synchronizing).mds e1 print_map 2023-10-09T05:35:16.250 INFO:journalctl@ceph.mon.c.smithi064.stdout: e1 2023-10-09T05:35:16.250 INFO:journalctl@ceph.mon.c.smithi064.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T05:35:16.250 INFO:journalctl@ceph.mon.c.smithi064.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T05:35:16.250 INFO:journalctl@ceph.mon.c.smithi064.stdout: legacy client fscid: -1 2023-10-09T05:35:16.250 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-10-09T05:35:16.250 INFO:journalctl@ceph.mon.c.smithi064.stdout: No filesystems configured 2023-10-09T05:35:16.250 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-10-09T05:35:16.251 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-10-09T05:35:16.251 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-10-09T05:35:16.251 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-10-09T05:35:16.251 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-10-09T05:35:16.251 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-10-09T05:35:16.251 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T05:35:16.251 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:35:16.251 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:35:16.251 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:35:16.252 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:16.252 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:16.252 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:35:16.252 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:16.252 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:16.252 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:16.253 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:16.253 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:16.253 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi156", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:35:16.253 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:16.253 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:35:16.253 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/2715771670' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-10-09T05:35:16.253 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: Updating smithi156:/etc/ceph/ceph.conf 2023-10-09T05:35:16.254 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:35:16.254 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:35:16.254 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:35:16.254 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:16.254 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:16.254 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:16.254 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/2715771670' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-09T05:35:16.254 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:35:16.254 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:16.254 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi064:172.21.15.64=a;smithi064:[v2:172.21.15.64:3301,v1:172.21.15.64:6790]=c;smithi156:172.21.15.156=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:35:16.255 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: Saving service mon spec with placement smithi064:172.21.15.64=a;smithi064:[v2:172.21.15.64:3301,v1:172.21.15.64:6790]=c;smithi156:172.21.15.156=b;count:3 2023-10-09T05:35:16.255 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:16.255 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:16.255 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:16.255 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:35:16.256 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:16.256 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:16.256 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:35:16.256 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:16.256 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: Deploying daemon mon.b on smithi156 2023-10-09T05:35:16.256 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:16.256 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='client.? 172.21.15.156:0/2267448357' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:35:16.256 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:35:16.256 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:35:16.256 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: mon.a calling monitor election 2023-10-09T05:35:16.256 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:35:16.257 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:16.257 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:35:16.257 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: mon.b calling monitor election 2023-10-09T05:35:16.257 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:35:16.257 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:16.257 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:35:16.257 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:35:16.257 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T05:35:16.257 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: monmap e2: 2 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0]} removed_ranks: {} 2023-10-09T05:35:16.257 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: fsmap 2023-10-09T05:35:16.257 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:35:16.257 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: mgrmap e13: y(active, since 37s) 2023-10-09T05:35:16.258 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: overall HEALTH_OK 2023-10-09T05:35:16.258 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:16.258 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:16.258 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:16.258 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:35:16.258 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:16.258 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: Deploying daemon mon.c on smithi064 2023-10-09T05:35:16.258 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:16.258 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:35:16.258 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: from='client.? 172.21.15.156:0/3358280317' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:35:16.258 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-09T05:35:16.259 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:15 smithi064 ceph-mon[118354]: mon.c@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete 2023-10-09T05:35:16.450 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:35:21.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:35:21.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: mon.a calling monitor election 2023-10-09T05:35:21.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: mon.b calling monitor election 2023-10-09T05:35:21.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:35:21.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:35:21.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:35:21.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:21.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:35:21.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:35:21.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:21.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:35:21.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:35:21.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T05:35:21.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: monmap e3: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} 2023-10-09T05:35:21.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: fsmap 2023-10-09T05:35:21.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:35:21.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: mgrmap e13: y(active, since 45s) 2023-10-09T05:35:21.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-09T05:35:21.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-09T05:35:21.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-09T05:35:21.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: mon.c (rank 2) addr [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] is down (out of quorum) 2023-10-09T05:35:21.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:21.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:21.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:21.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:21 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:21.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:35:21.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: mon.a calling monitor election 2023-10-09T05:35:21.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: mon.b calling monitor election 2023-10-09T05:35:21.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:35:21.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:35:21.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:35:21.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:21.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:35:21.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:35:21.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:21.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:35:21.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:35:21.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T05:35:21.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: monmap e3: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} 2023-10-09T05:35:21.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: fsmap 2023-10-09T05:35:21.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:35:21.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: mgrmap e13: y(active, since 45s) 2023-10-09T05:35:21.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-09T05:35:21.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-09T05:35:21.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-09T05:35:21.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: mon.c (rank 2) addr [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] is down (out of quorum) 2023-10-09T05:35:21.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:21.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:21.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:21.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:21 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:21.629 INFO:teuthology.orchestra.run.smithi156.stdout: 2023-10-09T05:35:21.629 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":3,"fsid":"44c77acc-6665-11ee-8db5-212e2dc638e7","modified":"2023-10-09T05:35:15.974764Z","created":"2023-10-09T05:33:37.342634Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:3300","nonce":0},{"type":"v1","addr":"172.21.15.64:6789","nonce":0}]},"addr":"172.21.15.64:6789/0","public_addr":"172.21.15.64:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:3300","nonce":0},{"type":"v1","addr":"172.21.15.156:6789","nonce":0}]},"addr":"172.21.15.156:6789/0","public_addr":"172.21.15.156:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:3301","nonce":0},{"type":"v1","addr":"172.21.15.64:6790","nonce":0}]},"addr":"172.21.15.64:6790/0","public_addr":"172.21.15.64:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-10-09T05:35:21.632 INFO:teuthology.orchestra.run.smithi156.stderr:dumped monmap epoch 3 2023-10-09T05:35:22.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:22 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:22.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:22 smithi156 ceph-mon[105979]: from='client.? 172.21.15.156:0/2716458174' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:35:22.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:22 smithi156 ceph-mon[105979]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:22.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:22 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:35:22.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:22.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[101070]: from='client.? 172.21.15.156:0/2716458174' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:35:22.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[101070]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:22.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:35:22.531 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-10-09T05:35:22.532 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph config generate-minimal-conf 2023-10-09T05:35:22.899 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:23.276 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:35:23.276 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: mon.a calling monitor election 2023-10-09T05:35:23.277 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: mon.b calling monitor election 2023-10-09T05:35:23.277 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:35:23.277 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:35:23.277 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:35:23.277 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:23.277 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:35:23.277 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:35:23.277 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:23.277 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:35:23.277 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:35:23.277 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T05:35:23.277 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: monmap e3: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} 2023-10-09T05:35:23.278 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: fsmap 2023-10-09T05:35:23.278 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:35:23.278 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: mgrmap e13: y(active, since 45s) 2023-10-09T05:35:23.278 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-09T05:35:23.278 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-09T05:35:23.278 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-09T05:35:23.278 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: mon.c (rank 2) addr [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] is down (out of quorum) 2023-10-09T05:35:23.278 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:23.278 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:23.278 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:23.278 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:23.279 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:23.279 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: from='client.? 172.21.15.156:0/2716458174' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:35:23.279 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:23.279 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:22 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:35:24.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[101070]: mon.c calling monitor election 2023-10-09T05:35:24.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[101070]: mon.c calling monitor election 2023-10-09T05:35:24.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[101070]: mon.b calling monitor election 2023-10-09T05:35:24.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[101070]: mon.a calling monitor election 2023-10-09T05:35:24.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[101070]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:35:24.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[101070]: monmap e3: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} 2023-10-09T05:35:24.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[101070]: fsmap 2023-10-09T05:35:24.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[101070]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:35:24.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[101070]: mgrmap e13: y(active, since 47s) 2023-10-09T05:35:24.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[101070]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T05:35:24.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[101070]: Cluster is now healthy 2023-10-09T05:35:24.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[101070]: overall HEALTH_OK 2023-10-09T05:35:24.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[101070]: Reconfiguring mon.a (unknown last config time)... 2023-10-09T05:35:24.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:35:24.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:35:24.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:24.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[101070]: Reconfiguring daemon mon.a on smithi064 2023-10-09T05:35:24.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[101070]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:24.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[118354]: mon.c calling monitor election 2023-10-09T05:35:24.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[118354]: mon.c calling monitor election 2023-10-09T05:35:24.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[118354]: mon.b calling monitor election 2023-10-09T05:35:24.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[118354]: mon.a calling monitor election 2023-10-09T05:35:24.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[118354]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:35:24.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[118354]: monmap e3: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} 2023-10-09T05:35:24.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[118354]: fsmap 2023-10-09T05:35:24.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[118354]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:35:24.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[118354]: mgrmap e13: y(active, since 47s) 2023-10-09T05:35:24.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[118354]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T05:35:24.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[118354]: Cluster is now healthy 2023-10-09T05:35:24.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[118354]: overall HEALTH_OK 2023-10-09T05:35:24.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[118354]: Reconfiguring mon.a (unknown last config time)... 2023-10-09T05:35:24.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:35:24.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:35:24.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:24.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[118354]: Reconfiguring daemon mon.a on smithi064 2023-10-09T05:35:24.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:23 smithi064 ceph-mon[118354]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:24.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:23 smithi156 ceph-mon[105979]: mon.c calling monitor election 2023-10-09T05:35:24.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:23 smithi156 ceph-mon[105979]: mon.c calling monitor election 2023-10-09T05:35:24.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:23 smithi156 ceph-mon[105979]: mon.b calling monitor election 2023-10-09T05:35:24.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:23 smithi156 ceph-mon[105979]: mon.a calling monitor election 2023-10-09T05:35:24.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:23 smithi156 ceph-mon[105979]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:35:24.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:23 smithi156 ceph-mon[105979]: monmap e3: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} 2023-10-09T05:35:24.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:23 smithi156 ceph-mon[105979]: fsmap 2023-10-09T05:35:24.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:23 smithi156 ceph-mon[105979]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:35:24.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:23 smithi156 ceph-mon[105979]: mgrmap e13: y(active, since 47s) 2023-10-09T05:35:24.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:23 smithi156 ceph-mon[105979]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T05:35:24.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:23 smithi156 ceph-mon[105979]: Cluster is now healthy 2023-10-09T05:35:24.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:23 smithi156 ceph-mon[105979]: overall HEALTH_OK 2023-10-09T05:35:24.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:23 smithi156 ceph-mon[105979]: Reconfiguring mon.a (unknown last config time)... 2023-10-09T05:35:24.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:23 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:35:24.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:23 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:35:24.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:23 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:24.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:23 smithi156 ceph-mon[105979]: Reconfiguring daemon mon.a on smithi064 2023-10-09T05:35:24.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:23 smithi156 ceph-mon[105979]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:25.006 INFO:teuthology.orchestra.run.smithi064.stdout:# minimal ceph.conf for 44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:35:25.007 INFO:teuthology.orchestra.run.smithi064.stdout:[global] 2023-10-09T05:35:25.007 INFO:teuthology.orchestra.run.smithi064.stdout: fsid = 44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:35:25.007 INFO:teuthology.orchestra.run.smithi064.stdout: mon_host = [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] 2023-10-09T05:35:25.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:24 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:35:25.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:24 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:35:25.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:24 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:35:25.556 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-10-09T05:35:25.556 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-10-09T05:35:25.556 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T05:35:25.587 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-10-09T05:35:25.587 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:35:25.654 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2023-10-09T05:35:25.654 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T05:35:25.691 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2023-10-09T05:35:25.691 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:35:25.762 INFO:tasks.cephadm:Adding mgr.y on smithi064 2023-10-09T05:35:25.763 INFO:tasks.cephadm:Adding mgr.x on smithi156 2023-10-09T05:35:25.763 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph orch apply mgr '2;smithi064=y;smithi156=x' 2023-10-09T05:35:26.075 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:25 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/4144832857' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:26.075 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:25 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:26.075 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:25 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:26.075 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:25 smithi064 ceph-mon[101070]: Reconfiguring mon.c (monmap changed)... 2023-10-09T05:35:26.075 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:25 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:35:26.075 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:25 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:35:26.075 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:25 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:26.075 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:25 smithi064 ceph-mon[101070]: Reconfiguring daemon mon.c on smithi064 2023-10-09T05:35:26.075 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:25 smithi064 ceph-mon[101070]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:26.076 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:25 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:26.076 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:25 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/4144832857' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:26.076 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:25 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:26.076 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:25 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:26.076 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:25 smithi064 ceph-mon[118354]: Reconfiguring mon.c (monmap changed)... 2023-10-09T05:35:26.076 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:25 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:35:26.076 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:25 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:35:26.076 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:25 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:26.076 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:25 smithi064 ceph-mon[118354]: Reconfiguring daemon mon.c on smithi064 2023-10-09T05:35:26.076 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:25 smithi064 ceph-mon[118354]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:26.076 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:25 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:26.132 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:35:26.307 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:25 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/4144832857' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:26.307 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:25 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:26.307 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:25 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:26.307 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:25 smithi156 ceph-mon[105979]: Reconfiguring mon.c (monmap changed)... 2023-10-09T05:35:26.307 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:25 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:35:26.307 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:25 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:35:26.307 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:25 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:26.307 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:25 smithi156 ceph-mon[105979]: Reconfiguring daemon mon.c on smithi064 2023-10-09T05:35:26.308 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:25 smithi156 ceph-mon[105979]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:26.308 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:25 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:27.698 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:27 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:27.698 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:27 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:27.698 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:27 smithi156 ceph-mon[105979]: Reconfiguring mon.b (monmap changed)... 2023-10-09T05:35:27.698 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:27 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:35:27.698 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:27 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:35:27.699 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:27 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:27.699 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:27 smithi156 ceph-mon[105979]: Reconfiguring daemon mon.b on smithi156 2023-10-09T05:35:27.833 INFO:teuthology.orchestra.run.smithi156.stdout:Scheduled mgr update... 2023-10-09T05:35:27.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:27 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:27.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:27 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:27.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:27 smithi064 ceph-mon[101070]: Reconfiguring mon.b (monmap changed)... 2023-10-09T05:35:27.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:27 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:35:27.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:27 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:35:27.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:27 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:27.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:27 smithi064 ceph-mon[101070]: Reconfiguring daemon mon.b on smithi156 2023-10-09T05:35:27.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:27 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:27.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:27 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:27.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:27 smithi064 ceph-mon[118354]: Reconfiguring mon.b (monmap changed)... 2023-10-09T05:35:27.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:27 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:35:27.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:27 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:35:27.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:27 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:27.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:27 smithi064 ceph-mon[118354]: Reconfiguring daemon mon.b on smithi156 2023-10-09T05:35:28.595 DEBUG:teuthology.orchestra.run.smithi156:mgr.x> sudo journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mgr.x.service 2023-10-09T05:35:28.598 INFO:tasks.cephadm:Deploying OSDs... 2023-10-09T05:35:28.598 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-10-09T05:35:28.598 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T05:35:28.620 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T05:35:28.620 DEBUG:teuthology.orchestra.run.smithi064:> stat /dev/vg_nvme/lv_1 2023-10-09T05:35:28.627 INFO:journalctl@ceph.mgr.x.smithi156.stdout:-- Logs begin at Mon 2023-10-09 05:16:12 UTC. -- 2023-10-09T05:35:28.681 INFO:teuthology.orchestra.run.smithi064.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T05:35:28.681 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:35:28.681 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 6h/6d Inode: 394252 Links: 1 2023-10-09T05:35:28.682 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:35:28.682 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T05:35:28.682 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2023-10-09 05:35:15.337042538 +0000 2023-10-09T05:35:28.682 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2023-10-09 05:26:00.670809351 +0000 2023-10-09T05:35:28.682 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2023-10-09 05:26:00.670809351 +0000 2023-10-09T05:35:28.682 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: - 2023-10-09T05:35:28.682 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T05:35:28.756 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records in 2023-10-09T05:35:28.756 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records out 2023-10-09T05:35:28.756 INFO:teuthology.orchestra.run.smithi064.stderr:512 bytes copied, 0.000259756 s, 2.0 MB/s 2023-10-09T05:35:28.758 DEBUG:teuthology.orchestra.run.smithi064:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T05:35:28.818 DEBUG:teuthology.orchestra.run.smithi064:> stat /dev/vg_nvme/lv_2 2023-10-09T05:35:28.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:28 smithi156 ceph-mon[105979]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:28.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:28 smithi156 ceph-mon[105979]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi064=y;smithi156=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:35:28.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:28 smithi156 ceph-mon[105979]: Saving service mgr spec with placement smithi064=y;smithi156=x;count:2 2023-10-09T05:35:28.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:28 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:28.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:28 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:28 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:28 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:28 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:28 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:35:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:28 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:28 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:35:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:28 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T05:35:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:28 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:35:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:28 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:28.877 INFO:teuthology.orchestra.run.smithi064.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T05:35:28.877 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:35:28.877 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 6h/6d Inode: 391588 Links: 1 2023-10-09T05:35:28.877 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:35:28.877 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T05:35:28.877 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2023-10-09 05:35:15.337042538 +0000 2023-10-09T05:35:28.877 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2023-10-09 05:26:01.193799215 +0000 2023-10-09T05:35:28.877 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2023-10-09 05:26:01.193799215 +0000 2023-10-09T05:35:28.877 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: - 2023-10-09T05:35:28.878 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T05:35:28.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[101070]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:28.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[101070]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi064=y;smithi156=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:35:28.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[101070]: Saving service mgr spec with placement smithi064=y;smithi156=x;count:2 2023-10-09T05:35:28.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:28.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:28.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:28.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:28.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:28.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:35:28.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:28.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:35:28.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T05:35:28.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:35:28.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:28.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[118354]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:28.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[118354]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi064=y;smithi156=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:35:28.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[118354]: Saving service mgr spec with placement smithi064=y;smithi156=x;count:2 2023-10-09T05:35:28.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:28.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:28.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:28.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:28.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:28.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:35:28.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:28.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:35:28.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T05:35:28.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:35:28.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:28 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:28.945 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records in 2023-10-09T05:35:28.945 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records out 2023-10-09T05:35:28.945 INFO:teuthology.orchestra.run.smithi064.stderr:512 bytes copied, 0.000223435 s, 2.3 MB/s 2023-10-09T05:35:28.946 DEBUG:teuthology.orchestra.run.smithi064:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T05:35:29.006 DEBUG:teuthology.orchestra.run.smithi064:> stat /dev/vg_nvme/lv_3 2023-10-09T05:35:29.065 INFO:teuthology.orchestra.run.smithi064.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T05:35:29.066 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:35:29.066 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 6h/6d Inode: 394345 Links: 1 2023-10-09T05:35:29.066 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:35:29.066 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T05:35:29.066 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2023-10-09 05:35:15.337042538 +0000 2023-10-09T05:35:29.066 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2023-10-09 05:26:01.700789390 +0000 2023-10-09T05:35:29.066 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2023-10-09 05:26:01.700789390 +0000 2023-10-09T05:35:29.066 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: - 2023-10-09T05:35:29.067 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T05:35:29.132 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records in 2023-10-09T05:35:29.133 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records out 2023-10-09T05:35:29.133 INFO:teuthology.orchestra.run.smithi064.stderr:512 bytes copied, 0.000228589 s, 2.2 MB/s 2023-10-09T05:35:29.134 DEBUG:teuthology.orchestra.run.smithi064:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T05:35:29.192 DEBUG:teuthology.orchestra.run.smithi064:> stat /dev/vg_nvme/lv_4 2023-10-09T05:35:29.251 INFO:teuthology.orchestra.run.smithi064.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T05:35:29.251 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:35:29.251 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 6h/6d Inode: 394407 Links: 1 2023-10-09T05:35:29.251 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:35:29.251 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T05:35:29.251 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2023-10-09 05:35:15.337042538 +0000 2023-10-09T05:35:29.251 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2023-10-09 05:26:02.209779527 +0000 2023-10-09T05:35:29.251 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2023-10-09 05:26:02.209779527 +0000 2023-10-09T05:35:29.252 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: - 2023-10-09T05:35:29.252 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T05:35:29.317 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records in 2023-10-09T05:35:29.317 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records out 2023-10-09T05:35:29.317 INFO:teuthology.orchestra.run.smithi064.stderr:512 bytes copied, 0.000196699 s, 2.6 MB/s 2023-10-09T05:35:29.318 DEBUG:teuthology.orchestra.run.smithi064:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T05:35:29.378 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2023-10-09T05:35:29.378 DEBUG:teuthology.orchestra.run.smithi156:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T05:35:29.395 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T05:35:29.395 DEBUG:teuthology.orchestra.run.smithi156:> stat /dev/vg_nvme/lv_1 2023-10-09T05:35:29.451 INFO:teuthology.orchestra.run.smithi156.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T05:35:29.452 INFO:teuthology.orchestra.run.smithi156.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:35:29.452 INFO:teuthology.orchestra.run.smithi156.stdout:Device: 6h/6d Inode: 388044 Links: 1 2023-10-09T05:35:29.452 INFO:teuthology.orchestra.run.smithi156.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:35:29.452 INFO:teuthology.orchestra.run.smithi156.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T05:35:29.452 INFO:teuthology.orchestra.run.smithi156.stdout:Access: 2023-10-09 05:35:07.211250559 +0000 2023-10-09T05:35:29.452 INFO:teuthology.orchestra.run.smithi156.stdout:Modify: 2023-10-09 05:25:38.146890058 +0000 2023-10-09T05:35:29.452 INFO:teuthology.orchestra.run.smithi156.stdout:Change: 2023-10-09 05:25:38.146890058 +0000 2023-10-09T05:35:29.452 INFO:teuthology.orchestra.run.smithi156.stdout: Birth: - 2023-10-09T05:35:29.453 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T05:35:29.518 INFO:teuthology.orchestra.run.smithi156.stderr:1+0 records in 2023-10-09T05:35:29.518 INFO:teuthology.orchestra.run.smithi156.stderr:1+0 records out 2023-10-09T05:35:29.518 INFO:teuthology.orchestra.run.smithi156.stderr:512 bytes copied, 0.000118756 s, 4.3 MB/s 2023-10-09T05:35:29.519 DEBUG:teuthology.orchestra.run.smithi156:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T05:35:29.577 DEBUG:teuthology.orchestra.run.smithi156:> stat /dev/vg_nvme/lv_2 2023-10-09T05:35:29.635 INFO:teuthology.orchestra.run.smithi156.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T05:35:29.635 INFO:teuthology.orchestra.run.smithi156.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:35:29.635 INFO:teuthology.orchestra.run.smithi156.stdout:Device: 6h/6d Inode: 395344 Links: 1 2023-10-09T05:35:29.635 INFO:teuthology.orchestra.run.smithi156.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:35:29.635 INFO:teuthology.orchestra.run.smithi156.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T05:35:29.635 INFO:teuthology.orchestra.run.smithi156.stdout:Access: 2023-10-09 05:35:07.212250545 +0000 2023-10-09T05:35:29.635 INFO:teuthology.orchestra.run.smithi156.stdout:Modify: 2023-10-09 05:25:38.656882535 +0000 2023-10-09T05:35:29.636 INFO:teuthology.orchestra.run.smithi156.stdout:Change: 2023-10-09 05:25:38.656882535 +0000 2023-10-09T05:35:29.636 INFO:teuthology.orchestra.run.smithi156.stdout: Birth: - 2023-10-09T05:35:29.636 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T05:35:29.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:29 smithi064 ceph-mon[101070]: Deploying daemon mgr.x on smithi156 2023-10-09T05:35:29.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:29 smithi064 ceph-mon[118354]: Deploying daemon mgr.x on smithi156 2023-10-09T05:35:29.743 INFO:teuthology.orchestra.run.smithi156.stderr:1+0 records in 2023-10-09T05:35:29.743 INFO:teuthology.orchestra.run.smithi156.stderr:1+0 records out 2023-10-09T05:35:29.743 INFO:teuthology.orchestra.run.smithi156.stderr:512 bytes copied, 0.000138895 s, 3.7 MB/s 2023-10-09T05:35:29.744 DEBUG:teuthology.orchestra.run.smithi156:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T05:35:29.766 DEBUG:teuthology.orchestra.run.smithi156:> stat /dev/vg_nvme/lv_3 2023-10-09T05:35:29.825 INFO:teuthology.orchestra.run.smithi156.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T05:35:29.825 INFO:teuthology.orchestra.run.smithi156.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:35:29.825 INFO:teuthology.orchestra.run.smithi156.stdout:Device: 6h/6d Inode: 395410 Links: 1 2023-10-09T05:35:29.825 INFO:teuthology.orchestra.run.smithi156.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:35:29.825 INFO:teuthology.orchestra.run.smithi156.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T05:35:29.825 INFO:teuthology.orchestra.run.smithi156.stdout:Access: 2023-10-09 05:35:29.816917241 +0000 2023-10-09T05:35:29.825 INFO:teuthology.orchestra.run.smithi156.stdout:Modify: 2023-10-09 05:25:39.156875160 +0000 2023-10-09T05:35:29.825 INFO:teuthology.orchestra.run.smithi156.stdout:Change: 2023-10-09 05:25:39.156875160 +0000 2023-10-09T05:35:29.826 INFO:teuthology.orchestra.run.smithi156.stdout: Birth: - 2023-10-09T05:35:29.826 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T05:35:29.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:29 smithi156 ceph-mon[105979]: Deploying daemon mgr.x on smithi156 2023-10-09T05:35:29.902 INFO:teuthology.orchestra.run.smithi156.stderr:1+0 records in 2023-10-09T05:35:29.903 INFO:teuthology.orchestra.run.smithi156.stderr:1+0 records out 2023-10-09T05:35:29.903 INFO:teuthology.orchestra.run.smithi156.stderr:512 bytes copied, 0.000100395 s, 5.1 MB/s 2023-10-09T05:35:29.904 DEBUG:teuthology.orchestra.run.smithi156:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T05:35:29.977 DEBUG:teuthology.orchestra.run.smithi156:> stat /dev/vg_nvme/lv_4 2023-10-09T05:35:30.044 INFO:teuthology.orchestra.run.smithi156.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T05:35:30.045 INFO:teuthology.orchestra.run.smithi156.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:35:30.045 INFO:teuthology.orchestra.run.smithi156.stdout:Device: 6h/6d Inode: 390752 Links: 1 2023-10-09T05:35:30.045 INFO:teuthology.orchestra.run.smithi156.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:35:30.045 INFO:teuthology.orchestra.run.smithi156.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T05:35:30.045 INFO:teuthology.orchestra.run.smithi156.stdout:Access: 2023-10-09 05:35:29.817917226 +0000 2023-10-09T05:35:30.045 INFO:teuthology.orchestra.run.smithi156.stdout:Modify: 2023-10-09 05:25:39.640868021 +0000 2023-10-09T05:35:30.045 INFO:teuthology.orchestra.run.smithi156.stdout:Change: 2023-10-09 05:25:39.640868021 +0000 2023-10-09T05:35:30.045 INFO:teuthology.orchestra.run.smithi156.stdout: Birth: - 2023-10-09T05:35:30.046 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T05:35:30.114 INFO:teuthology.orchestra.run.smithi156.stderr:1+0 records in 2023-10-09T05:35:30.115 INFO:teuthology.orchestra.run.smithi156.stderr:1+0 records out 2023-10-09T05:35:30.115 INFO:teuthology.orchestra.run.smithi156.stderr:512 bytes copied, 9.8427e-05 s, 5.2 MB/s 2023-10-09T05:35:30.116 DEBUG:teuthology.orchestra.run.smithi156:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T05:35:30.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:29 smithi156 systemd[1]: Starting Ceph mgr.x for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:35:30.140 INFO:tasks.cephadm:Deploying osd.0 on smithi064 with /dev/vg_nvme/lv_4... 2023-10-09T05:35:30.140 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T05:35:30.506 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:30.748 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:30 smithi156 ceph-mon[105979]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:30.748 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:30 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:30.748 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:30 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:30.749 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:30 smithi156 ceph-mgr[109223]: -- 172.21.15.156:0/1265949354 <== mon.1 v2:172.21.15.156:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e0e31c49c0 con 0x55e0e319c800 2023-10-09T05:35:30.749 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:30 smithi156 bash[109115]: 2efef4f1cb6cf0213bd8c4743ec3470f4e7b4a9abaf756b0d08ec07cad5bce07 2023-10-09T05:35:30.749 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:30 smithi156 systemd[1]: Started Ceph mgr.x for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:35:30.749 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:30 smithi156 conmon[109199]: 2023-10-09T05:35:30.578+0000 7f059546a1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:35:30.750 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:30 smithi156 conmon[109199]: 2023-10-09T05:35:30.655+0000 7f059546a1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:35:30.750 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:30 smithi156 conmon[109199]: 2023-10-09T05:35:30.747+0000 7f059546a1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:35:30.893 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:30 smithi064 ceph-mon[101070]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:30.893 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:30 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:30.893 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:30 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:30.893 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:30 smithi064 ceph-mon[118354]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:30.893 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:30 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:30.893 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:30 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:31.031 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:30 smithi156 conmon[109199]: 2023-10-09T05:35:30.844+0000 7f059546a1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:35:31.031 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:31 smithi156 conmon[109199]: 2023-10-09T05:35:31.030+0000 7f059546a1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:35:31.707 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:31 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:31.707 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:31 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:31.708 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:31 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:31.708 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:31 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:31.708 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:31 smithi156 conmon[109199]: 2023-10-09T05:35:31.378+0000 7f059546a1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:35:31.708 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:31 smithi156 conmon[109199]: 2023-10-09T05:35:31.471+0000 7f059546a1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:35:31.708 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:31 smithi156 conmon[109199]: 2023-10-09T05:35:31.610+0000 7f059546a1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:35:31.708 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:31 smithi156 conmon[109199]: 2023-10-09T05:35:31.706+0000 7f059546a1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:35:31.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:31 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:31.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:31 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:31.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:31 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:31.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:31 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:31.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:31 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:31.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:31 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:31.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:31 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:31.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:31 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:32.251 INFO:teuthology.orchestra.run.smithi064.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T05:35:32.251 INFO:teuthology.orchestra.run.smithi064.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T05:35:32.251 INFO:teuthology.orchestra.run.smithi064.stderr: stderr: 10+0 records in 2023-10-09T05:35:32.251 INFO:teuthology.orchestra.run.smithi064.stderr:10+0 records out 2023-10-09T05:35:32.251 INFO:teuthology.orchestra.run.smithi064.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0131434 s, 798 MB/s 2023-10-09T05:35:32.251 INFO:teuthology.orchestra.run.smithi064.stderr: stderr: 2023-10-09T05:35:32.252 INFO:teuthology.orchestra.run.smithi064.stderr:--> Zapping successful for: 2023-10-09T05:35:32.620 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:32 smithi156 conmon[109199]: 2023-10-09T05:35:32.129+0000 7f059546a1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:35:32.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:32 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:32.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:32 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:32.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:32 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:32.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:32 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:35:32.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:32 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:32.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:32 smithi064 ceph-mon[118354]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:32.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:32 smithi064 ceph-mon[118354]: Reconfiguring mgr.y (unknown last config time)... 2023-10-09T05:35:32.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:32 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:35:32.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:32 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:35:32.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:32 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:32.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:32 smithi064 ceph-mon[118354]: Reconfiguring daemon mgr.y on smithi064 2023-10-09T05:35:32.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:32 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:32.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:32 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:32.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:32 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:32.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:32 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:35:32.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:32 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:32.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:32 smithi064 ceph-mon[101070]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:32.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:32 smithi064 ceph-mon[101070]: Reconfiguring mgr.y (unknown last config time)... 2023-10-09T05:35:32.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:32 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:35:32.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:32 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:35:32.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:32 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:32.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:32 smithi064 ceph-mon[101070]: Reconfiguring daemon mgr.y on smithi064 2023-10-09T05:35:33.021 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi064:vg_nvme/lv_4 2023-10-09T05:35:33.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:32 smithi156 conmon[109199]: 2023-10-09T05:35:32.699+0000 7f059546a1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:35:33.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:32 smithi156 conmon[109199]: 2023-10-09T05:35:32.868+0000 7f059546a1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:35:33.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:32 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:33.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:32 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:33.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:32 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:33.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:32 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:35:33.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:32 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:33.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:32 smithi156 ceph-mon[105979]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:33.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:32 smithi156 ceph-mon[105979]: Reconfiguring mgr.y (unknown last config time)... 2023-10-09T05:35:33.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:32 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:35:33.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:32 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:35:33.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:32 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:33.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:32 smithi156 ceph-mon[105979]: Reconfiguring daemon mgr.y on smithi064 2023-10-09T05:35:33.551 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:33.896 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:33 smithi156 conmon[109199]: 2023-10-09T05:35:33.640+0000 7f059546a1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:35:33.896 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:33 smithi156 conmon[109199]: 2023-10-09T05:35:33.728+0000 7f059546a1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:35:33.897 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:33 smithi156 conmon[109199]: 2023-10-09T05:35:33.895+0000 7f059546a1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:35:34.496 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:34 smithi156 conmon[109199]: 2023-10-09T05:35:34.165+0000 7f059546a1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:35:34.496 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:34 smithi156 conmon[109199]: 2023-10-09T05:35:34.251+0000 7f059546a1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:35:34.778 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:34 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:34.778 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:34 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:34.778 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:34 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:34.778 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:34 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:34.778 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:34 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:35:34.779 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:34 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:34.779 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:34 smithi064 ceph-mon[118354]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:34.779 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:34 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:34.779 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:34 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:34.779 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:34 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:34.779 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:34 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:34.779 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:34 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:35:34.779 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:34 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:34.779 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:34 smithi064 ceph-mon[101070]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:34.792 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:34 smithi156 conmon[109199]: 2023-10-09T05:35:34.539+0000 7f059546a1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:35:34.792 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:34 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:34.792 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:34 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:34.793 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:34 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:34.793 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:34 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:34.793 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:34 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:35:34.793 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:34 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:34.793 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:34 smithi156 ceph-mon[105979]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:35.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:34 smithi156 conmon[109199]: 2023-10-09T05:35:34.792+0000 7f059546a1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:35:35.772 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:35 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:35:35.772 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:35 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:35:35.772 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:35 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:35.772 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:35 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:35:35.773 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:35 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:35:35.773 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:35 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:35.828 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:35 smithi156 conmon[109199]: 2023-10-09T05:35:35.514+0000 7f059546a1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:35:35.828 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:35 smithi156 conmon[109199]: 2023-10-09T05:35:35.643+0000 7f059546a1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:35:35.828 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:35 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:35:35.828 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:35 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:35:35.829 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:35 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:36.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:35 smithi156 conmon[109199]: 2023-10-09T05:35:35.827+0000 7f059546a1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:35:36.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:35 smithi156 conmon[109199]: 2023-10-09T05:35:35.939+0000 7f059546a1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:35:36.518 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:36 smithi156 conmon[109199]: 2023-10-09T05:35:36.152+0000 7f059546a1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:35:36.825 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:36 smithi064 ceph-mon[101070]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:35:36.825 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:36 smithi064 ceph-mon[101070]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:36.826 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:36 smithi064 ceph-mon[118354]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:35:36.826 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:36 smithi064 ceph-mon[118354]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:36.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:36 smithi156 ceph-mon[105979]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:35:36.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:36 smithi156 ceph-mon[105979]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:37.370 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:35:37 smithi156 conmon[109199]: 2023-10-09T05:35:37.001+0000 7f059546a1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:35:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:37 smithi156 ceph-mon[105979]: Standby manager daemon x started 2023-10-09T05:35:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:37 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.156:0/1575689427' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:35:37.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:37 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.156:0/1575689427' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:35:37.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:37 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.156:0/1575689427' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:35:37.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:37 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.156:0/1575689427' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:35:37.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:37 smithi064 ceph-mon[118354]: Standby manager daemon x started 2023-10-09T05:35:37.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:37 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.156:0/1575689427' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:35:37.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:37 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.156:0/1575689427' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:35:37.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:37 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.156:0/1575689427' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:35:37.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:37 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.156:0/1575689427' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:35:37.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:37 smithi064 ceph-mon[101070]: Standby manager daemon x started 2023-10-09T05:35:37.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:37 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.156:0/1575689427' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:35:37.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:37 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.156:0/1575689427' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:35:37.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:37 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.156:0/1575689427' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:35:37.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:37 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.156:0/1575689427' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:35:38.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:38 smithi156 ceph-mon[105979]: mgrmap e14: y(active, since 61s), standbys: x 2023-10-09T05:35:38.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:38 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:35:38.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:38 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/4290085062' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "85d0857a-ca32-43d2-8cec-bbd916db9f98"}]: dispatch 2023-10-09T05:35:38.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:38 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/4290085062' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "85d0857a-ca32-43d2-8cec-bbd916db9f98"}]': finished 2023-10-09T05:35:38.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:38 smithi156 ceph-mon[105979]: osdmap e5: 1 total, 0 up, 1 in 2023-10-09T05:35:38.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:38 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:35:38.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:38 smithi156 ceph-mon[105979]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:38.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:38 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3174300714' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:35:38.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:38 smithi064 ceph-mon[101070]: mgrmap e14: y(active, since 61s), standbys: x 2023-10-09T05:35:38.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:38 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:35:38.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:38 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/4290085062' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "85d0857a-ca32-43d2-8cec-bbd916db9f98"}]: dispatch 2023-10-09T05:35:38.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:38 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/4290085062' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "85d0857a-ca32-43d2-8cec-bbd916db9f98"}]': finished 2023-10-09T05:35:38.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:38 smithi064 ceph-mon[101070]: osdmap e5: 1 total, 0 up, 1 in 2023-10-09T05:35:38.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:38 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:35:38.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:38 smithi064 ceph-mon[101070]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:38.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:38 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3174300714' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:35:38.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:38 smithi064 ceph-mon[118354]: mgrmap e14: y(active, since 61s), standbys: x 2023-10-09T05:35:38.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:38 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:35:38.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:38 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/4290085062' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "85d0857a-ca32-43d2-8cec-bbd916db9f98"}]: dispatch 2023-10-09T05:35:38.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:38 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/4290085062' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "85d0857a-ca32-43d2-8cec-bbd916db9f98"}]': finished 2023-10-09T05:35:38.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:38 smithi064 ceph-mon[118354]: osdmap e5: 1 total, 0 up, 1 in 2023-10-09T05:35:38.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:38 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:35:38.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:38 smithi064 ceph-mon[118354]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:38.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:38 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3174300714' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:35:41.111 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:40 smithi064 ceph-mon[101070]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:41.112 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:40 smithi064 ceph-mon[118354]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:41.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:40 smithi156 ceph-mon[105979]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:43.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:42 smithi156 ceph-mon[105979]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:43.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:42 smithi064 ceph-mon[101070]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:43.210 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:42 smithi064 ceph-mon[118354]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:44.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:43 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T05:35:44.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:43 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:44.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:43 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T05:35:44.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:43 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:44.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:43 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T05:35:44.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:43 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:44.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:44 smithi064 ceph-mon[101070]: Deploying daemon osd.0 on smithi064 2023-10-09T05:35:44.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:44 smithi064 ceph-mon[101070]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:44.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:44 smithi064 ceph-mon[118354]: Deploying daemon osd.0 on smithi064 2023-10-09T05:35:44.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:44 smithi064 ceph-mon[118354]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:45.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:44 smithi156 ceph-mon[105979]: Deploying daemon osd.0 on smithi064 2023-10-09T05:35:45.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:44 smithi156 ceph-mon[105979]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:46.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:46 smithi156 ceph-mon[105979]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:46.684 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:46 smithi064 ceph-mon[101070]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:46.685 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:46 smithi064 ceph-mon[118354]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:48.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:48 smithi064 ceph-mon[101070]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:48.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:48 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:48.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:48 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:48.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:48 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:48.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:48 smithi064 ceph-mon[118354]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:48.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:48 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:48.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:48 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:48.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:48 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:49.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:48 smithi156 ceph-mon[105979]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:49.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:48 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:49.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:48 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:49.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:48 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:51.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:50 smithi156 ceph-mon[105979]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:51.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:50 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:51.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:50 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:51.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:50 smithi064 ceph-mon[101070]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:51.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:50 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:51.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:50 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:51.204 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:50 smithi064 ceph-mon[118354]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:51.204 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:50 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:51.204 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:50 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:51.801 INFO:teuthology.orchestra.run.smithi064.stdout:Created osd(s) 0 on host 'smithi064' 2023-10-09T05:35:52.022 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:51 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:52.022 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:51 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:35:52.022 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:51 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:52.022 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:51 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:52.022 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:51 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:52.022 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:51 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:52.023 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:51 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:52.023 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:51 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:35:52.023 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:51 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:52.023 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:51 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:52.023 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:51 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:52.023 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:51 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:52.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:51 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:52.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:51 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:35:52.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:51 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:52.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:51 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:52.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:51 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:52.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:51 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:35:52.443 DEBUG:teuthology.orchestra.run.smithi064:osd.0> sudo journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.0.service 2023-10-09T05:35:52.445 INFO:tasks.cephadm:Deploying osd.1 on smithi064 with /dev/vg_nvme/lv_3... 2023-10-09T05:35:52.446 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T05:35:52.479 INFO:journalctl@ceph.osd.0.smithi064.stdout:-- Logs begin at Mon 2023-10-09 05:16:16 UTC. -- 2023-10-09T05:35:53.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:52 smithi156 ceph-mon[105979]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:53.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:52 smithi156 ceph-mon[105979]: from='osd.0 [v2:172.21.15.64:6802/2019939615,v1:172.21.15.64:6803/2019939615]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:35:53.121 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:52 smithi064 ceph-mon[118354]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:53.121 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:52 smithi064 ceph-mon[118354]: from='osd.0 [v2:172.21.15.64:6802/2019939615,v1:172.21.15.64:6803/2019939615]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:35:53.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:52 smithi064 ceph-mon[101070]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:53.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:52 smithi064 ceph-mon[101070]: from='osd.0 [v2:172.21.15.64:6802/2019939615,v1:172.21.15.64:6803/2019939615]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:35:53.533 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:53.848 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:53 smithi064 ceph-mon[101070]: from='osd.0 [v2:172.21.15.64:6802/2019939615,v1:172.21.15.64:6803/2019939615]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T05:35:53.849 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:53 smithi064 ceph-mon[101070]: osdmap e6: 1 total, 0 up, 1 in 2023-10-09T05:35:53.849 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:53 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:35:53.849 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:53 smithi064 ceph-mon[101070]: from='osd.0 [v2:172.21.15.64:6802/2019939615,v1:172.21.15.64:6803/2019939615]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:35:53.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:53 smithi064 ceph-mon[118354]: from='osd.0 [v2:172.21.15.64:6802/2019939615,v1:172.21.15.64:6803/2019939615]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T05:35:53.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:53 smithi064 ceph-mon[118354]: osdmap e6: 1 total, 0 up, 1 in 2023-10-09T05:35:53.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:53 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:35:53.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:53 smithi064 ceph-mon[118354]: from='osd.0 [v2:172.21.15.64:6802/2019939615,v1:172.21.15.64:6803/2019939615]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:35:53.850 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:35:53 smithi064 conmon[124618]: 2023-10-09T05:35:53.825+0000 7fe07e778700 -1 osd.0 0 waiting for initial osdmap 2023-10-09T05:35:53.850 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:35:53 smithi064 conmon[124618]: 2023-10-09T05:35:53.829+0000 7fe07756f700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:35:54.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:53 smithi156 ceph-mon[105979]: from='osd.0 [v2:172.21.15.64:6802/2019939615,v1:172.21.15.64:6803/2019939615]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T05:35:54.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:53 smithi156 ceph-mon[105979]: osdmap e6: 1 total, 0 up, 1 in 2023-10-09T05:35:54.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:53 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:35:54.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:53 smithi156 ceph-mon[105979]: from='osd.0 [v2:172.21.15.64:6802/2019939615,v1:172.21.15.64:6803/2019939615]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:35:55.097 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:54 smithi064 ceph-mon[101070]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:55.098 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:54 smithi064 ceph-mon[101070]: from='osd.0 [v2:172.21.15.64:6802/2019939615,v1:172.21.15.64:6803/2019939615]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2023-10-09T05:35:55.098 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:54 smithi064 ceph-mon[101070]: osdmap e7: 1 total, 0 up, 1 in 2023-10-09T05:35:55.098 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:54 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:35:55.098 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:54 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:35:55.098 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:54 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:35:55.098 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:54 smithi064 ceph-mon[118354]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:55.098 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:54 smithi064 ceph-mon[118354]: from='osd.0 [v2:172.21.15.64:6802/2019939615,v1:172.21.15.64:6803/2019939615]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2023-10-09T05:35:55.098 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:54 smithi064 ceph-mon[118354]: osdmap e7: 1 total, 0 up, 1 in 2023-10-09T05:35:55.099 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:54 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:35:55.099 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:54 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:35:55.099 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:54 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:35:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:54 smithi156 ceph-mon[105979]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:54 smithi156 ceph-mon[105979]: from='osd.0 [v2:172.21.15.64:6802/2019939615,v1:172.21.15.64:6803/2019939615]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2023-10-09T05:35:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:54 smithi156 ceph-mon[105979]: osdmap e7: 1 total, 0 up, 1 in 2023-10-09T05:35:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:54 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:35:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:54 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:35:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:54 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:35:55.805 INFO:teuthology.orchestra.run.smithi064.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T05:35:55.805 INFO:teuthology.orchestra.run.smithi064.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T05:35:55.805 INFO:teuthology.orchestra.run.smithi064.stderr: stderr: 10+0 records in 2023-10-09T05:35:55.805 INFO:teuthology.orchestra.run.smithi064.stderr:10+0 records out 2023-10-09T05:35:55.805 INFO:teuthology.orchestra.run.smithi064.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0154685 s, 678 MB/s 2023-10-09T05:35:55.806 INFO:teuthology.orchestra.run.smithi064.stderr:--> Zapping successful for: 2023-10-09T05:35:55.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:55 smithi064 ceph-mon[118354]: purged_snaps scrub starts 2023-10-09T05:35:55.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:55 smithi064 ceph-mon[118354]: purged_snaps scrub ok 2023-10-09T05:35:55.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:55 smithi064 ceph-mon[118354]: osd.0 [v2:172.21.15.64:6802/2019939615,v1:172.21.15.64:6803/2019939615] boot 2023-10-09T05:35:55.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:55 smithi064 ceph-mon[118354]: osdmap e8: 1 total, 1 up, 1 in 2023-10-09T05:35:55.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:55 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:35:55.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:55 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:55.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:55 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:55.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:55 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:35:55.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:55 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:55.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:55 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:55.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:55 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:35:55.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:55 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:55.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:55 smithi064 ceph-mon[101070]: purged_snaps scrub starts 2023-10-09T05:35:55.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:55 smithi064 ceph-mon[101070]: purged_snaps scrub ok 2023-10-09T05:35:55.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:55 smithi064 ceph-mon[101070]: osd.0 [v2:172.21.15.64:6802/2019939615,v1:172.21.15.64:6803/2019939615] boot 2023-10-09T05:35:55.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:55 smithi064 ceph-mon[101070]: osdmap e8: 1 total, 1 up, 1 in 2023-10-09T05:35:55.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:55 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:35:55.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:55 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:55.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:55 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:55.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:55 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:35:55.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:55 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:55.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:55 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:55.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:55 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:35:55.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:55 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:56.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:55 smithi156 ceph-mon[105979]: purged_snaps scrub starts 2023-10-09T05:35:56.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:55 smithi156 ceph-mon[105979]: purged_snaps scrub ok 2023-10-09T05:35:56.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:55 smithi156 ceph-mon[105979]: osd.0 [v2:172.21.15.64:6802/2019939615,v1:172.21.15.64:6803/2019939615] boot 2023-10-09T05:35:56.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:55 smithi156 ceph-mon[105979]: osdmap e8: 1 total, 1 up, 1 in 2023-10-09T05:35:56.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:55 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:35:56.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:55 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:56.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:55 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:56.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:55 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:35:56.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:55 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:56.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:55 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:56.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:55 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:35:56.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:55 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:35:56.370 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi064:vg_nvme/lv_3 2023-10-09T05:35:56.721 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:35:57.088 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:56 smithi064 ceph-mon[101070]: Detected new or changed devices on smithi064 2023-10-09T05:35:57.088 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:56 smithi064 ceph-mon[101070]: Adjusting osd_memory_target on smithi064 to 16144M 2023-10-09T05:35:57.088 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:56 smithi064 ceph-mon[101070]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:57.088 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:56 smithi064 ceph-mon[101070]: osdmap e9: 1 total, 1 up, 1 in 2023-10-09T05:35:57.089 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:56 smithi064 ceph-mon[118354]: Detected new or changed devices on smithi064 2023-10-09T05:35:57.089 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:56 smithi064 ceph-mon[118354]: Adjusting osd_memory_target on smithi064 to 16144M 2023-10-09T05:35:57.089 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:56 smithi064 ceph-mon[118354]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:57.089 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:56 smithi064 ceph-mon[118354]: osdmap e9: 1 total, 1 up, 1 in 2023-10-09T05:35:57.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:56 smithi156 ceph-mon[105979]: Detected new or changed devices on smithi064 2023-10-09T05:35:57.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:56 smithi156 ceph-mon[105979]: Adjusting osd_memory_target on smithi064 to 16144M 2023-10-09T05:35:57.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:56 smithi156 ceph-mon[105979]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:35:57.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:56 smithi156 ceph-mon[105979]: osdmap e9: 1 total, 1 up, 1 in 2023-10-09T05:35:58.023 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:57 smithi064 ceph-mon[101070]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:35:58.024 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:57 smithi064 ceph-mon[118354]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:35:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:57 smithi156 ceph-mon[105979]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:35:59.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:58 smithi156 ceph-mon[105979]: from='client.14247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:35:59.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:58 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:35:59.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:58 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:35:59.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:58 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:59.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:58 smithi064 ceph-mon[101070]: from='client.14247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:35:59.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:58 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:35:59.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:58 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:35:59.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:58 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:35:59.209 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:58 smithi064 ceph-mon[118354]: from='client.14247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:35:59.209 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:58 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:35:59.209 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:58 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:35:59.209 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:58 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:00.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:35:59 smithi156 ceph-mon[105979]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:00.130 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:35:59 smithi064 ceph-mon[101070]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:00.130 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:35:59 smithi064 ceph-mon[118354]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:01.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:01 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/2243887577' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "05570845-2cd8-4e24-a2f5-f46cf67aa31a"}]: dispatch 2023-10-09T05:36:01.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:01 smithi156 ceph-mon[105979]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "05570845-2cd8-4e24-a2f5-f46cf67aa31a"}]: dispatch 2023-10-09T05:36:01.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:01 smithi156 ceph-mon[105979]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "05570845-2cd8-4e24-a2f5-f46cf67aa31a"}]': finished 2023-10-09T05:36:01.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:01 smithi156 ceph-mon[105979]: osdmap e10: 2 total, 1 up, 2 in 2023-10-09T05:36:01.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:01 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:36:01.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:01 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/2243887577' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "05570845-2cd8-4e24-a2f5-f46cf67aa31a"}]: dispatch 2023-10-09T05:36:01.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:01 smithi064 ceph-mon[101070]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "05570845-2cd8-4e24-a2f5-f46cf67aa31a"}]: dispatch 2023-10-09T05:36:01.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:01 smithi064 ceph-mon[101070]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "05570845-2cd8-4e24-a2f5-f46cf67aa31a"}]': finished 2023-10-09T05:36:01.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:01 smithi064 ceph-mon[101070]: osdmap e10: 2 total, 1 up, 2 in 2023-10-09T05:36:01.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:01 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:36:01.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:01 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/2243887577' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "05570845-2cd8-4e24-a2f5-f46cf67aa31a"}]: dispatch 2023-10-09T05:36:01.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:01 smithi064 ceph-mon[118354]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "05570845-2cd8-4e24-a2f5-f46cf67aa31a"}]: dispatch 2023-10-09T05:36:01.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:01 smithi064 ceph-mon[118354]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "05570845-2cd8-4e24-a2f5-f46cf67aa31a"}]': finished 2023-10-09T05:36:01.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:01 smithi064 ceph-mon[118354]: osdmap e10: 2 total, 1 up, 2 in 2023-10-09T05:36:01.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:01 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:36:02.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:02 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/1487343344' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:36:02.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:02 smithi156 ceph-mon[105979]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:02.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:02 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1487343344' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:36:02.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:02 smithi064 ceph-mon[101070]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:02.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:02 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/1487343344' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:36:02.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:02 smithi064 ceph-mon[118354]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:05.113 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:04 smithi064 ceph-mon[101070]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:05.113 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:04 smithi064 ceph-mon[118354]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:05.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:04 smithi156 ceph-mon[105979]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:06 smithi064 ceph-mon[101070]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:06.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:06 smithi064 ceph-mon[118354]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:06.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:06 smithi156 ceph-mon[105979]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:07 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T05:36:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:07 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:07.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:07 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T05:36:07.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:07 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:07.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T05:36:07.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:08.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:08 smithi064 ceph-mon[101070]: Deploying daemon osd.1 on smithi064 2023-10-09T05:36:08.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:08 smithi064 ceph-mon[101070]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:08.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:08 smithi064 ceph-mon[118354]: Deploying daemon osd.1 on smithi064 2023-10-09T05:36:08.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:08 smithi064 ceph-mon[118354]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:08.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:08 smithi156 ceph-mon[105979]: Deploying daemon osd.1 on smithi064 2023-10-09T05:36:08.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:08 smithi156 ceph-mon[105979]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:11.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:10 smithi156 ceph-mon[105979]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:11.211 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:10 smithi064 ceph-mon[101070]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:11.212 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:10 smithi064 ceph-mon[118354]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:12.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:12 smithi064 ceph-mon[101070]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:12.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:12 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:12.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:12 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:36:12.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:12 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:12.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:12 smithi064 ceph-mon[118354]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:12.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:12 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:12.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:12 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:36:12.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:12 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:13.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:12 smithi156 ceph-mon[105979]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:13.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:12 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:13.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:12 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:36:13.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:12 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:15.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:14 smithi156 ceph-mon[105979]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:15.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:14 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:15.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:14 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:15.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:14 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:15.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:14 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:36:15.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:14 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:15.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:14 smithi064 ceph-mon[101070]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:15.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:14 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:15.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:14 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:15.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:14 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:15.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:14 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:36:15.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:14 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:15.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:14 smithi064 ceph-mon[118354]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:15.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:14 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:15.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:14 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:15.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:14 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:15.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:14 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:36:15.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:14 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:15.342 INFO:teuthology.orchestra.run.smithi064.stdout:Created osd(s) 1 on host 'smithi064' 2023-10-09T05:36:15.974 DEBUG:teuthology.orchestra.run.smithi064:osd.1> sudo journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.1.service 2023-10-09T05:36:15.976 INFO:tasks.cephadm:Deploying osd.2 on smithi064 with /dev/vg_nvme/lv_2... 2023-10-09T05:36:15.976 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-09T05:36:16.011 INFO:journalctl@ceph.osd.1.smithi064.stdout:-- Logs begin at Mon 2023-10-09 05:16:16 UTC. -- 2023-10-09T05:36:16.560 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:16 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:16.560 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:16 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:16.560 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:16 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:36:16.560 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:16 smithi064 ceph-mon[101070]: from='osd.1 [v2:172.21.15.64:6810/1958566614,v1:172.21.15.64:6811/1958566614]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:36:16.560 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:16 smithi064 ceph-mon[101070]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:16.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:16 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:16.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:16 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:16.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:16 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:36:16.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:16 smithi064 ceph-mon[118354]: from='osd.1 [v2:172.21.15.64:6810/1958566614,v1:172.21.15.64:6811/1958566614]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:36:16.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:16 smithi064 ceph-mon[118354]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:16.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:16 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:16.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:16 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:16.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:16 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:36:16.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:16 smithi156 ceph-mon[105979]: from='osd.1 [v2:172.21.15.64:6810/1958566614,v1:172.21.15.64:6811/1958566614]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:36:16.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:16 smithi156 ceph-mon[105979]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:16.981 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:17.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:17 smithi156 ceph-mon[105979]: from='osd.1 [v2:172.21.15.64:6810/1958566614,v1:172.21.15.64:6811/1958566614]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T05:36:17.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:17 smithi156 ceph-mon[105979]: osdmap e11: 2 total, 1 up, 2 in 2023-10-09T05:36:17.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:17 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:36:17.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:17 smithi156 ceph-mon[105979]: from='osd.1 [v2:172.21.15.64:6810/1958566614,v1:172.21.15.64:6811/1958566614]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:36:17.644 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:17 smithi064 ceph-mon[101070]: from='osd.1 [v2:172.21.15.64:6810/1958566614,v1:172.21.15.64:6811/1958566614]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T05:36:17.644 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:17 smithi064 ceph-mon[101070]: osdmap e11: 2 total, 1 up, 2 in 2023-10-09T05:36:17.644 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:17 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:36:17.644 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:17 smithi064 ceph-mon[101070]: from='osd.1 [v2:172.21.15.64:6810/1958566614,v1:172.21.15.64:6811/1958566614]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:36:17.644 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:17 smithi064 ceph-mon[118354]: from='osd.1 [v2:172.21.15.64:6810/1958566614,v1:172.21.15.64:6811/1958566614]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T05:36:17.645 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:17 smithi064 ceph-mon[118354]: osdmap e11: 2 total, 1 up, 2 in 2023-10-09T05:36:17.645 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:17 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:36:17.645 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:17 smithi064 ceph-mon[118354]: from='osd.1 [v2:172.21.15.64:6810/1958566614,v1:172.21.15.64:6811/1958566614]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:36:17.645 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:36:17 smithi064 conmon[133568]: 2023-10-09T05:36:17.345+0000 7f10360ae700 -1 osd.1 0 waiting for initial osdmap 2023-10-09T05:36:17.645 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:36:17 smithi064 conmon[133568]: 2023-10-09T05:36:17.349+0000 7f10326ac700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:36:18.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:18 smithi064 ceph-mon[118354]: from='osd.1 [v2:172.21.15.64:6810/1958566614,v1:172.21.15.64:6811/1958566614]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2023-10-09T05:36:18.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:18 smithi064 ceph-mon[118354]: osdmap e12: 2 total, 1 up, 2 in 2023-10-09T05:36:18.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:18 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:36:18.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:18 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:36:18.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:18 smithi064 ceph-mon[118354]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:18.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:18 smithi064 ceph-mon[101070]: from='osd.1 [v2:172.21.15.64:6810/1958566614,v1:172.21.15.64:6811/1958566614]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2023-10-09T05:36:18.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:18 smithi064 ceph-mon[101070]: osdmap e12: 2 total, 1 up, 2 in 2023-10-09T05:36:18.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:18 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:36:18.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:18 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:36:18.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:18 smithi064 ceph-mon[101070]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:18.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:18 smithi156 ceph-mon[105979]: from='osd.1 [v2:172.21.15.64:6810/1958566614,v1:172.21.15.64:6811/1958566614]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2023-10-09T05:36:18.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:18 smithi156 ceph-mon[105979]: osdmap e12: 2 total, 1 up, 2 in 2023-10-09T05:36:18.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:18 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:36:18.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:18 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:36:18.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:18 smithi156 ceph-mon[105979]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:36:19.116 INFO:teuthology.orchestra.run.smithi064.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-09T05:36:19.116 INFO:teuthology.orchestra.run.smithi064.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-09T05:36:19.116 INFO:teuthology.orchestra.run.smithi064.stderr: stderr: 10+0 records in 2023-10-09T05:36:19.116 INFO:teuthology.orchestra.run.smithi064.stderr:10+0 records out 2023-10-09T05:36:19.116 INFO:teuthology.orchestra.run.smithi064.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0110177 s, 952 MB/s 2023-10-09T05:36:19.116 INFO:teuthology.orchestra.run.smithi064.stderr:--> Zapping successful for: 2023-10-09T05:36:19.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:19 smithi064 ceph-mon[101070]: purged_snaps scrub starts 2023-10-09T05:36:19.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:19 smithi064 ceph-mon[101070]: purged_snaps scrub ok 2023-10-09T05:36:19.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:19 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:36:19.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:19 smithi064 ceph-mon[101070]: osd.1 [v2:172.21.15.64:6810/1958566614,v1:172.21.15.64:6811/1958566614] boot 2023-10-09T05:36:19.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:19 smithi064 ceph-mon[101070]: osdmap e13: 2 total, 2 up, 2 in 2023-10-09T05:36:19.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:19 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:36:19.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:19 smithi064 ceph-mon[118354]: purged_snaps scrub starts 2023-10-09T05:36:19.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:19 smithi064 ceph-mon[118354]: purged_snaps scrub ok 2023-10-09T05:36:19.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:19 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:36:19.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:19 smithi064 ceph-mon[118354]: osd.1 [v2:172.21.15.64:6810/1958566614,v1:172.21.15.64:6811/1958566614] boot 2023-10-09T05:36:19.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:19 smithi064 ceph-mon[118354]: osdmap e13: 2 total, 2 up, 2 in 2023-10-09T05:36:19.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:19 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:36:19.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:19 smithi156 ceph-mon[105979]: purged_snaps scrub starts 2023-10-09T05:36:19.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:19 smithi156 ceph-mon[105979]: purged_snaps scrub ok 2023-10-09T05:36:19.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:19 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:36:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:19 smithi156 ceph-mon[105979]: osd.1 [v2:172.21.15.64:6810/1958566614,v1:172.21.15.64:6811/1958566614] boot 2023-10-09T05:36:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:19 smithi156 ceph-mon[105979]: osdmap e13: 2 total, 2 up, 2 in 2023-10-09T05:36:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:19 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:36:19.749 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi064:vg_nvme/lv_2 2023-10-09T05:36:20.103 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:20.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:20 smithi156 ceph-mon[105979]: osdmap e14: 2 total, 2 up, 2 in 2023-10-09T05:36:20.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:20 smithi156 ceph-mon[105979]: Detected new or changed devices on smithi064 2023-10-09T05:36:20.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:20 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:20.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:20 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:20.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:20 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:36:20.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:20 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:36:20.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:20 smithi156 ceph-mon[105979]: Adjusting osd_memory_target on smithi064 to 8072M 2023-10-09T05:36:20.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:20 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:20.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:20 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:20.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:20 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:36:20.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:20 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:20.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:20 smithi156 ceph-mon[105979]: pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:20.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:20 smithi064 ceph-mon[101070]: osdmap e14: 2 total, 2 up, 2 in 2023-10-09T05:36:20.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:20 smithi064 ceph-mon[101070]: Detected new or changed devices on smithi064 2023-10-09T05:36:20.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:20 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:20.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:20 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:20.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:20 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:36:20.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:20 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:36:20.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:20 smithi064 ceph-mon[101070]: Adjusting osd_memory_target on smithi064 to 8072M 2023-10-09T05:36:20.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:20 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:20.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:20 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:20.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:20 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:36:20.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:20 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:20.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:20 smithi064 ceph-mon[101070]: pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:20.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:20 smithi064 ceph-mon[118354]: osdmap e14: 2 total, 2 up, 2 in 2023-10-09T05:36:20.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:20 smithi064 ceph-mon[118354]: Detected new or changed devices on smithi064 2023-10-09T05:36:20.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:20 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:20.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:20 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:20.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:20 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:36:20.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:20 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:36:20.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:20 smithi064 ceph-mon[118354]: Adjusting osd_memory_target on smithi064 to 8072M 2023-10-09T05:36:20.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:20 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:20.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:20 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:20.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:20 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:36:20.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:20 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:20.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:20 smithi064 ceph-mon[118354]: pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:22.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:21 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:36:22.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:21 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:36:22.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:21 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:22.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:21 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:36:22.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:21 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:36:22.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:21 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:22.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:21 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:36:22.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:21 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:36:22.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:21 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:22.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:22 smithi064 ceph-mon[118354]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:36:22.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:22 smithi064 ceph-mon[118354]: pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:22.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:22 smithi064 ceph-mon[101070]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:36:22.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:22 smithi064 ceph-mon[101070]: pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:23.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:22 smithi156 ceph-mon[105979]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:36:23.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:22 smithi156 ceph-mon[105979]: pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:25.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:24 smithi156 ceph-mon[105979]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:25.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:24 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/550795129' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0f306460-9f08-4f15-b573-3587f63be61c"}]: dispatch 2023-10-09T05:36:25.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:24 smithi156 ceph-mon[105979]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0f306460-9f08-4f15-b573-3587f63be61c"}]: dispatch 2023-10-09T05:36:25.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:24 smithi156 ceph-mon[105979]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0f306460-9f08-4f15-b573-3587f63be61c"}]': finished 2023-10-09T05:36:25.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:24 smithi156 ceph-mon[105979]: osdmap e15: 3 total, 2 up, 3 in 2023-10-09T05:36:25.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:24 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:36:25.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:24 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/2941314588' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:36:25.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:24 smithi064 ceph-mon[101070]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:25.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:24 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/550795129' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0f306460-9f08-4f15-b573-3587f63be61c"}]: dispatch 2023-10-09T05:36:25.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:24 smithi064 ceph-mon[101070]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0f306460-9f08-4f15-b573-3587f63be61c"}]: dispatch 2023-10-09T05:36:25.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:24 smithi064 ceph-mon[101070]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0f306460-9f08-4f15-b573-3587f63be61c"}]': finished 2023-10-09T05:36:25.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:24 smithi064 ceph-mon[101070]: osdmap e15: 3 total, 2 up, 3 in 2023-10-09T05:36:25.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:24 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:36:25.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:24 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/2941314588' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:36:25.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:24 smithi064 ceph-mon[118354]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:25.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:24 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/550795129' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0f306460-9f08-4f15-b573-3587f63be61c"}]: dispatch 2023-10-09T05:36:25.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:24 smithi064 ceph-mon[118354]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0f306460-9f08-4f15-b573-3587f63be61c"}]: dispatch 2023-10-09T05:36:25.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:24 smithi064 ceph-mon[118354]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0f306460-9f08-4f15-b573-3587f63be61c"}]': finished 2023-10-09T05:36:25.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:24 smithi064 ceph-mon[118354]: osdmap e15: 3 total, 2 up, 3 in 2023-10-09T05:36:25.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:24 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:36:25.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:24 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/2941314588' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:36:26.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:26 smithi156 ceph-mon[105979]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:26.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:26 smithi064 ceph-mon[101070]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:26.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:26 smithi064 ceph-mon[118354]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:28.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:28 smithi064 ceph-mon[101070]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:28.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:28 smithi064 ceph-mon[118354]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:29.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:28 smithi156 ceph-mon[105979]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:31.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:30 smithi156 ceph-mon[105979]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:31.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:30 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:36:31.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:30 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:31.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:30 smithi064 ceph-mon[101070]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:31.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:30 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:36:31.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:30 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:31.200 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:30 smithi064 ceph-mon[118354]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:31.200 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:30 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:36:31.200 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:30 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:31.934 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:31 smithi064 ceph-mon[101070]: Deploying daemon osd.2 on smithi064 2023-10-09T05:36:31.934 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:31 smithi064 ceph-mon[118354]: Deploying daemon osd.2 on smithi064 2023-10-09T05:36:32.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:31 smithi156 ceph-mon[105979]: Deploying daemon osd.2 on smithi064 2023-10-09T05:36:33.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:32 smithi156 ceph-mon[105979]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:33.137 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:32 smithi064 ceph-mon[101070]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:33.138 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:32 smithi064 ceph-mon[118354]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:35.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:34 smithi156 ceph-mon[105979]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:35.136 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:34 smithi064 ceph-mon[101070]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:35.137 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:34 smithi064 ceph-mon[118354]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:36.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:36 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:36.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:36 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:36.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:36 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:36:36.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:36 smithi064 ceph-mon[101070]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:36.424 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:36 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:36.424 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:36 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:36.424 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:36 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:36:36.424 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:36 smithi064 ceph-mon[118354]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:36.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:36 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:36.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:36 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:36.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:36 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:36:36.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:36 smithi156 ceph-mon[105979]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:38.160 INFO:teuthology.orchestra.run.smithi064.stdout:Created osd(s) 2 on host 'smithi064' 2023-10-09T05:36:38.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:38 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:38.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:38 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:38.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:38 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:38.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:38 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:36:38.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:38 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:38.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:38 smithi064 ceph-mon[101070]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:38.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:38 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:38.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:38 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:38.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:38 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:36:38.375 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:38 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:38.375 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:38 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:38.375 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:38 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:38.375 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:38 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:36:38.375 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:38 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:38.375 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:38 smithi064 ceph-mon[118354]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:38.375 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:38 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:38.375 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:38 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:38.375 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:38 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:36:38.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:38 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:38.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:38 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:38.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:38 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:38.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:38 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:36:38.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:38 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:38.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:38 smithi156 ceph-mon[105979]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:38.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:38 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:38.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:38 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:38.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:38 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:36:38.770 DEBUG:teuthology.orchestra.run.smithi064:osd.2> sudo journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.2.service 2023-10-09T05:36:38.773 INFO:tasks.cephadm:Deploying osd.3 on smithi064 with /dev/vg_nvme/lv_1... 2023-10-09T05:36:38.773 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-09T05:36:38.804 INFO:journalctl@ceph.osd.2.smithi064.stdout:-- Logs begin at Mon 2023-10-09 05:16:16 UTC. -- 2023-10-09T05:36:39.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:39 smithi064 ceph-mon[101070]: from='osd.2 [v2:172.21.15.64:6818/2050077610,v1:172.21.15.64:6819/2050077610]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:36:39.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:39 smithi064 ceph-mon[101070]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:36:39.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:39 smithi064 ceph-mon[118354]: from='osd.2 [v2:172.21.15.64:6818/2050077610,v1:172.21.15.64:6819/2050077610]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:36:39.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:39 smithi064 ceph-mon[118354]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:36:39.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:39 smithi156 ceph-mon[105979]: from='osd.2 [v2:172.21.15.64:6818/2050077610,v1:172.21.15.64:6819/2050077610]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:36:39.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:39 smithi156 ceph-mon[105979]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:36:39.930 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:40.469 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:36:40 smithi064 conmon[143096]: 2023-10-09T05:36:40.354+0000 7f9c28f5a700 -1 osd.2 0 waiting for initial osdmap 2023-10-09T05:36:40.469 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:36:40 smithi064 conmon[143096]: 2023-10-09T05:36:40.358+0000 7f9c25d59700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:36:40.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:40 smithi064 ceph-mon[118354]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T05:36:40.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:40 smithi064 ceph-mon[118354]: osdmap e16: 3 total, 2 up, 3 in 2023-10-09T05:36:40.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:40 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:36:40.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:40 smithi064 ceph-mon[118354]: from='osd.2 [v2:172.21.15.64:6818/2050077610,v1:172.21.15.64:6819/2050077610]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:36:40.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:40 smithi064 ceph-mon[118354]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:36:40.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:40 smithi064 ceph-mon[118354]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:40.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:40 smithi064 ceph-mon[101070]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T05:36:40.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:40 smithi064 ceph-mon[101070]: osdmap e16: 3 total, 2 up, 3 in 2023-10-09T05:36:40.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:40 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:36:40.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:40 smithi064 ceph-mon[101070]: from='osd.2 [v2:172.21.15.64:6818/2050077610,v1:172.21.15.64:6819/2050077610]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:36:40.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:40 smithi064 ceph-mon[101070]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:36:40.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:40 smithi064 ceph-mon[101070]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:40.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:40 smithi156 ceph-mon[105979]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T05:36:40.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:40 smithi156 ceph-mon[105979]: osdmap e16: 3 total, 2 up, 3 in 2023-10-09T05:36:40.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:40 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:36:40.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:40 smithi156 ceph-mon[105979]: from='osd.2 [v2:172.21.15.64:6818/2050077610,v1:172.21.15.64:6819/2050077610]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:36:40.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:40 smithi156 ceph-mon[105979]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:36:40.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:40 smithi156 ceph-mon[105979]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:36:41.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:41 smithi064 ceph-mon[101070]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2023-10-09T05:36:41.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:41 smithi064 ceph-mon[101070]: osdmap e17: 3 total, 2 up, 3 in 2023-10-09T05:36:41.543 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:41 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:36:41.543 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:41 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:36:41.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:41 smithi064 ceph-mon[118354]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2023-10-09T05:36:41.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:41 smithi064 ceph-mon[118354]: osdmap e17: 3 total, 2 up, 3 in 2023-10-09T05:36:41.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:41 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:36:41.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:41 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:36:41.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:41 smithi156 ceph-mon[105979]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2023-10-09T05:36:41.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:41 smithi156 ceph-mon[105979]: osdmap e17: 3 total, 2 up, 3 in 2023-10-09T05:36:41.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:41 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:36:41.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:41 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:36:42.073 INFO:teuthology.orchestra.run.smithi064.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-09T05:36:42.073 INFO:teuthology.orchestra.run.smithi064.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-09T05:36:42.073 INFO:teuthology.orchestra.run.smithi064.stderr: stderr: 10+0 records in 2023-10-09T05:36:42.073 INFO:teuthology.orchestra.run.smithi064.stderr:10+0 records out 2023-10-09T05:36:42.074 INFO:teuthology.orchestra.run.smithi064.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0125103 s, 838 MB/s 2023-10-09T05:36:42.074 INFO:teuthology.orchestra.run.smithi064.stderr:--> Zapping successful for: 2023-10-09T05:36:42.381 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:42 smithi064 ceph-mon[101070]: purged_snaps scrub starts 2023-10-09T05:36:42.382 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:42 smithi064 ceph-mon[101070]: purged_snaps scrub ok 2023-10-09T05:36:42.382 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:42 smithi064 ceph-mon[101070]: osd.2 [v2:172.21.15.64:6818/2050077610,v1:172.21.15.64:6819/2050077610] boot 2023-10-09T05:36:42.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:42 smithi156 ceph-mon[105979]: purged_snaps scrub starts 2023-10-09T05:36:42.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:42 smithi156 ceph-mon[105979]: purged_snaps scrub ok 2023-10-09T05:36:42.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:42 smithi156 ceph-mon[105979]: osd.2 [v2:172.21.15.64:6818/2050077610,v1:172.21.15.64:6819/2050077610] boot 2023-10-09T05:36:42.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:42 smithi156 ceph-mon[105979]: osdmap e18: 3 total, 3 up, 3 in 2023-10-09T05:36:42.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:42 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:36:42.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:42 smithi156 ceph-mon[105979]: pgmap v71: 0 pgs: ; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:36:42.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:42 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T05:36:42.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:42 smithi064 ceph-mon[101070]: osdmap e18: 3 total, 3 up, 3 in 2023-10-09T05:36:42.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:42 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:36:42.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:42 smithi064 ceph-mon[101070]: pgmap v71: 0 pgs: ; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:36:42.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:42 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T05:36:42.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:42 smithi064 ceph-mon[118354]: purged_snaps scrub starts 2023-10-09T05:36:42.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:42 smithi064 ceph-mon[118354]: purged_snaps scrub ok 2023-10-09T05:36:42.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:42 smithi064 ceph-mon[118354]: osd.2 [v2:172.21.15.64:6818/2050077610,v1:172.21.15.64:6819/2050077610] boot 2023-10-09T05:36:42.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:42 smithi064 ceph-mon[118354]: osdmap e18: 3 total, 3 up, 3 in 2023-10-09T05:36:42.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:42 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:36:42.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:42 smithi064 ceph-mon[118354]: pgmap v71: 0 pgs: ; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:36:42.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:42 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T05:36:42.821 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi064:vg_nvme/lv_1 2023-10-09T05:36:43.173 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:36:43.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:43 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T05:36:43.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:43 smithi156 ceph-mon[105979]: osdmap e19: 3 total, 3 up, 3 in 2023-10-09T05:36:43.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:43 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:36:43.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:43 smithi156 ceph-mon[105979]: Detected new or changed devices on smithi064 2023-10-09T05:36:43.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:43 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:43.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:43 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:43.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:43 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:36:43.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:43 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:36:43.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:43 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:36:43.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:43 smithi156 ceph-mon[105979]: Adjusting osd_memory_target on smithi064 to 5381M 2023-10-09T05:36:43.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:43 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:43.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:43 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:43.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:43 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:36:43.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:43 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:43.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T05:36:43.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[118354]: osdmap e19: 3 total, 3 up, 3 in 2023-10-09T05:36:43.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:36:43.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[118354]: Detected new or changed devices on smithi064 2023-10-09T05:36:43.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:43.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:43.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:36:43.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:36:43.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:36:43.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[118354]: Adjusting osd_memory_target on smithi064 to 5381M 2023-10-09T05:36:43.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:43.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:43.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:36:43.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:43.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T05:36:43.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[101070]: osdmap e19: 3 total, 3 up, 3 in 2023-10-09T05:36:43.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:36:43.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[101070]: Detected new or changed devices on smithi064 2023-10-09T05:36:43.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:43.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:43.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:36:43.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:36:43.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:36:43.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[101070]: Adjusting osd_memory_target on smithi064 to 5381M 2023-10-09T05:36:43.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:43.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:43.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:36:43.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:43 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:44.219 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:36:43 smithi064 sudo[149163]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T05:36:44.219 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:36:43 smithi064 sudo[149163]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T05:36:44.219 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:36:43 smithi064 sudo[149163]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T05:36:44.219 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:36:43 smithi064 sudo[149163]: pam_unix(sudo:session): session closed for user root 2023-10-09T05:36:44.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:44 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:36:44.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:44 smithi064 ceph-mon[101070]: osdmap e20: 3 total, 3 up, 3 in 2023-10-09T05:36:44.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:44 smithi064 ceph-mon[101070]: pgmap v74: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:36:44.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:44 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:36:44.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:44 smithi064 ceph-mon[118354]: osdmap e20: 3 total, 3 up, 3 in 2023-10-09T05:36:44.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:44 smithi064 ceph-mon[118354]: pgmap v74: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:36:44.591 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:36:44 smithi064 sudo[149183]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T05:36:44.591 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:36:44 smithi064 sudo[149183]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T05:36:44.591 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:36:44 smithi064 sudo[149183]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T05:36:44.591 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:36:44 smithi064 sudo[149183]: pam_unix(sudo:session): session closed for user root 2023-10-09T05:36:44.849 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:36:44 smithi064 sudo[149271]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T05:36:44.849 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:36:44 smithi064 sudo[149271]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T05:36:44.849 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:36:44 smithi064 sudo[149271]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T05:36:44.849 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:36:44 smithi064 sudo[149271]: pam_unix(sudo:session): session closed for user root 2023-10-09T05:36:44.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:44 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:36:44.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:44 smithi156 ceph-mon[105979]: osdmap e20: 3 total, 3 up, 3 in 2023-10-09T05:36:44.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:44 smithi156 ceph-mon[105979]: pgmap v74: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:36:45.641 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:36:45 smithi064 sudo[149376]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T05:36:45.642 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:36:45 smithi064 sudo[149376]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T05:36:45.642 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:36:45 smithi064 sudo[149376]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T05:36:45.642 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:36:45 smithi064 sudo[149376]: pam_unix(sudo:session): session closed for user root 2023-10-09T05:36:45.642 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:45 smithi064 ceph-mon[101070]: osdmap e21: 3 total, 3 up, 3 in 2023-10-09T05:36:45.642 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:45 smithi064 ceph-mon[101070]: mgrmap e15: y(active, since 2m), standbys: x 2023-10-09T05:36:45.642 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:45 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:45.642 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:45 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:36:45.643 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:45 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:36:45.643 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:45 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:45.643 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:45 smithi064 ceph-mon[118354]: osdmap e21: 3 total, 3 up, 3 in 2023-10-09T05:36:45.643 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:45 smithi064 ceph-mon[118354]: mgrmap e15: y(active, since 2m), standbys: x 2023-10-09T05:36:45.643 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:45 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:45.643 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:45 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:36:45.643 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:45 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:36:45.643 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:45 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:45.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:45 smithi156 ceph-mon[105979]: osdmap e21: 3 total, 3 up, 3 in 2023-10-09T05:36:45.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:45 smithi156 ceph-mon[105979]: mgrmap e15: y(active, since 2m), standbys: x 2023-10-09T05:36:45.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:45 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:45.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:45 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:36:45.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:45 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:36:45.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:45 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:45.969 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:36:45 smithi064 sudo[149517]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T05:36:45.969 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:36:45 smithi064 sudo[149517]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T05:36:45.969 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:36:45 smithi064 sudo[149517]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T05:36:45.969 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:36:45 smithi064 sudo[149517]: pam_unix(sudo:session): session closed for user root 2023-10-09T05:36:46.469 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:36:46 smithi064 sudo[149573]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T05:36:46.469 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:36:46 smithi064 sudo[149573]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T05:36:46.469 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:36:46 smithi064 sudo[149573]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T05:36:46.469 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:36:46 smithi064 sudo[149573]: pam_unix(sudo:session): session closed for user root 2023-10-09T05:36:46.808 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:46 smithi064 sudo[149632]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T05:36:46.808 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:46 smithi064 sudo[149632]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T05:36:46.808 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:46 smithi064 sudo[149632]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T05:36:46.808 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:46 smithi064 sudo[149632]: pam_unix(sudo:session): session closed for user root 2023-10-09T05:36:47.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:47 smithi064 sudo[149665]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T05:36:47.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:47 smithi064 sudo[149665]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T05:36:47.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:47 smithi064 sudo[149665]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T05:36:47.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:47 smithi064 sudo[149665]: pam_unix(sudo:session): session closed for user root 2023-10-09T05:36:47.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:47 smithi064 ceph-mon[101070]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:36:47.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:47 smithi064 ceph-mon[101070]: pgmap v76: 1 pgs: 1 active+clean; 449 KiB data, 872 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:36:47.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:47 smithi064 ceph-mon[101070]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:36:47.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:47 smithi064 ceph-mon[118354]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:36:47.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:47 smithi064 ceph-mon[118354]: pgmap v76: 1 pgs: 1 active+clean; 449 KiB data, 872 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:36:47.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:47 smithi064 ceph-mon[118354]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:36:47.618 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:47 smithi156 sudo[110096]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T05:36:47.618 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:47 smithi156 sudo[110096]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T05:36:47.618 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:47 smithi156 sudo[110096]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T05:36:47.618 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:47 smithi156 sudo[110096]: pam_unix(sudo:session): session closed for user root 2023-10-09T05:36:47.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:47 smithi156 sudo[110099]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T05:36:47.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:47 smithi156 sudo[110099]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T05:36:47.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:47 smithi156 sudo[110099]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T05:36:47.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:47 smithi156 sudo[110099]: pam_unix(sudo:session): session closed for user root 2023-10-09T05:36:47.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:47 smithi156 ceph-mon[105979]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:36:47.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:47 smithi156 ceph-mon[105979]: pgmap v76: 1 pgs: 1 active+clean; 449 KiB data, 872 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:36:47.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:47 smithi156 ceph-mon[105979]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:36:48.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 sudo[149735]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T05:36:48.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 sudo[149735]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T05:36:48.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 sudo[149735]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T05:36:48.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 sudo[149735]: pam_unix(sudo:session): session closed for user root 2023-10-09T05:36:48.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:48 smithi156 ceph-mon[105979]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:36:48.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:48 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:36:48.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:48 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:36:48.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:48 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:36:48.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:48 smithi156 ceph-mon[105979]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:36:48.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:48 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:36:48.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:48 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:36:48.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:48 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:36:48.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:48 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3150520147' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4bf38edd-e555-402e-ba4b-c751de06aa6a"}]: dispatch 2023-10-09T05:36:48.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:48 smithi156 ceph-mon[105979]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4bf38edd-e555-402e-ba4b-c751de06aa6a"}]: dispatch 2023-10-09T05:36:48.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:48 smithi156 ceph-mon[105979]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:36:48.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:48 smithi156 ceph-mon[105979]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4bf38edd-e555-402e-ba4b-c751de06aa6a"}]': finished 2023-10-09T05:36:48.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:48 smithi156 ceph-mon[105979]: osdmap e22: 4 total, 3 up, 4 in 2023-10-09T05:36:48.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:48 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:36:48.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:48 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:48.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:48 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:36:48.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:48 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:36:48.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:48 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:36:48.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:48 smithi156 ceph-mon[105979]: pgmap v78: 1 pgs: 1 active+clean; 449 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:36:48.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 sudo[149788]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T05:36:48.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 sudo[149788]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T05:36:48.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 sudo[149788]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T05:36:48.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 sudo[149788]: pam_unix(sudo:session): session closed for user root 2023-10-09T05:36:48.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[118354]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:36:48.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:36:48.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:36:48.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:36:48.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[118354]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:36:48.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:36:48.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:36:48.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:36:48.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3150520147' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4bf38edd-e555-402e-ba4b-c751de06aa6a"}]: dispatch 2023-10-09T05:36:48.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[118354]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4bf38edd-e555-402e-ba4b-c751de06aa6a"}]: dispatch 2023-10-09T05:36:48.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[118354]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:36:48.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[118354]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4bf38edd-e555-402e-ba4b-c751de06aa6a"}]': finished 2023-10-09T05:36:48.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[118354]: osdmap e22: 4 total, 3 up, 4 in 2023-10-09T05:36:48.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:36:48.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:48.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:36:48.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:36:48.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:36:48.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[118354]: pgmap v78: 1 pgs: 1 active+clean; 449 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:36:48.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[101070]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:36:48.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:36:48.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:36:48.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:36:48.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[101070]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:36:48.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:36:48.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:36:48.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:36:48.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3150520147' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4bf38edd-e555-402e-ba4b-c751de06aa6a"}]: dispatch 2023-10-09T05:36:48.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[101070]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4bf38edd-e555-402e-ba4b-c751de06aa6a"}]: dispatch 2023-10-09T05:36:48.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[101070]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:36:48.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[101070]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4bf38edd-e555-402e-ba4b-c751de06aa6a"}]': finished 2023-10-09T05:36:48.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[101070]: osdmap e22: 4 total, 3 up, 4 in 2023-10-09T05:36:48.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:36:48.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:36:48.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:36:48.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:36:48.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:36:48.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:48 smithi064 ceph-mon[101070]: pgmap v78: 1 pgs: 1 active+clean; 449 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:36:49.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:49 smithi156 ceph-mon[105979]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:36:49.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:49 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/1600891518' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:36:49.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:49 smithi156 ceph-mon[105979]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:36:49.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:49 smithi064 ceph-mon[101070]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:36:49.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:49 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1600891518' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:36:49.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:49 smithi064 ceph-mon[101070]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:36:49.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:49 smithi064 ceph-mon[118354]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:36:49.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:49 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/1600891518' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:36:49.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:49 smithi064 ceph-mon[118354]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:36:50.847 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:50 smithi064 ceph-mon[101070]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 5.0 KiB/s rd, 242 KiB/s wr, 16 op/s 2023-10-09T05:36:50.848 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:50 smithi064 ceph-mon[118354]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 5.0 KiB/s rd, 242 KiB/s wr, 16 op/s 2023-10-09T05:36:50.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:50 smithi156 ceph-mon[105979]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 5.0 KiB/s rd, 242 KiB/s wr, 16 op/s 2023-10-09T05:36:53.008 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:52 smithi064 ceph-mon[101070]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.6 KiB/s rd, 177 KiB/s wr, 12 op/s 2023-10-09T05:36:53.008 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:52 smithi064 ceph-mon[118354]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.6 KiB/s rd, 177 KiB/s wr, 12 op/s 2023-10-09T05:36:53.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:52 smithi156 ceph-mon[105979]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.6 KiB/s rd, 177 KiB/s wr, 12 op/s 2023-10-09T05:36:54.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:54 smithi064 ceph-mon[101070]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.9 KiB/s rd, 139 KiB/s wr, 9 op/s 2023-10-09T05:36:54.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:54 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:36:54.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:54 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:54.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:54 smithi064 ceph-mon[118354]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.9 KiB/s rd, 139 KiB/s wr, 9 op/s 2023-10-09T05:36:54.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:54 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:36:54.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:54 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:54 smithi156 ceph-mon[105979]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.9 KiB/s rd, 139 KiB/s wr, 9 op/s 2023-10-09T05:36:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:54 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:36:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:54 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:36:55.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:55 smithi064 ceph-mon[101070]: Deploying daemon osd.3 on smithi064 2023-10-09T05:36:55.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:55 smithi064 ceph-mon[118354]: Deploying daemon osd.3 on smithi064 2023-10-09T05:36:56.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:55 smithi156 ceph-mon[105979]: Deploying daemon osd.3 on smithi064 2023-10-09T05:36:56.960 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:56 smithi064 ceph-mon[101070]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2023-10-09T05:36:56.960 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:56 smithi064 ceph-mon[118354]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2023-10-09T05:36:57.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:56 smithi156 ceph-mon[105979]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2023-10-09T05:36:59.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:36:58 smithi156 ceph-mon[105979]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.7 KiB/s rd, 129 KiB/s wr, 8 op/s 2023-10-09T05:36:59.127 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:36:58 smithi064 ceph-mon[101070]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.7 KiB/s rd, 129 KiB/s wr, 8 op/s 2023-10-09T05:36:59.128 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:36:58 smithi064 ceph-mon[118354]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.7 KiB/s rd, 129 KiB/s wr, 8 op/s 2023-10-09T05:37:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:00 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:00 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:00 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:37:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:00 smithi064 ceph-mon[101070]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2023-10-09T05:37:00.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:00 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:00.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:00 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:00.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:00 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:37:00.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:00 smithi064 ceph-mon[118354]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2023-10-09T05:37:00.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:00 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:00.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:00 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:00.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:00 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:37:00.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:00 smithi156 ceph-mon[105979]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2023-10-09T05:37:02.108 INFO:teuthology.orchestra.run.smithi064.stdout:Created osd(s) 3 on host 'smithi064' 2023-10-09T05:37:02.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:02 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:02.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:02 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:02.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:02 smithi064 ceph-mon[101070]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:37:02.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:02 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:02.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:02 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:37:02.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:02 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:02.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:02 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:02.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:02 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:02.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:02 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:37:02.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:02 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:02.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:02 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:02.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:02 smithi064 ceph-mon[118354]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:37:02.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:02 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:02.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:02 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:37:02.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:02 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:02.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:02 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:02.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:02 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:02.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:02 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:37:02.775 DEBUG:teuthology.orchestra.run.smithi064:osd.3> sudo journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.3.service 2023-10-09T05:37:02.778 INFO:tasks.cephadm:Deploying osd.4 on smithi156 with /dev/vg_nvme/lv_4... 2023-10-09T05:37:02.778 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T05:37:02.811 INFO:journalctl@ceph.osd.3.smithi064.stdout:-- Logs begin at Mon 2023-10-09 05:16:16 UTC. -- 2023-10-09T05:37:02.811 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:02 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:02.811 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:02 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:02.811 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:02 smithi156 ceph-mon[105979]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:37:02.811 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:02 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:02.811 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:02 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:37:02.811 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:02 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:02.811 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:02 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:02.811 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:02 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:02.811 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:02 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:37:03.120 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:37:03.607 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:03 smithi156 ceph-mon[105979]: from='osd.3 [v2:172.21.15.64:6826/3037106123,v1:172.21.15.64:6827/3037106123]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:37:03.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:03 smithi064 ceph-mon[101070]: from='osd.3 [v2:172.21.15.64:6826/3037106123,v1:172.21.15.64:6827/3037106123]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:37:03.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:03 smithi064 ceph-mon[118354]: from='osd.3 [v2:172.21.15.64:6826/3037106123,v1:172.21.15.64:6827/3037106123]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:37:04.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:04 smithi064 ceph-mon[101070]: from='osd.3 [v2:172.21.15.64:6826/3037106123,v1:172.21.15.64:6827/3037106123]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T05:37:04.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:04 smithi064 ceph-mon[101070]: osdmap e23: 4 total, 3 up, 4 in 2023-10-09T05:37:04.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:04 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:37:04.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:04 smithi064 ceph-mon[101070]: from='osd.3 [v2:172.21.15.64:6826/3037106123,v1:172.21.15.64:6827/3037106123]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:37:04.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:04 smithi064 ceph-mon[101070]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:37:04.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:04 smithi064 ceph-mon[118354]: from='osd.3 [v2:172.21.15.64:6826/3037106123,v1:172.21.15.64:6827/3037106123]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T05:37:04.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:04 smithi064 ceph-mon[118354]: osdmap e23: 4 total, 3 up, 4 in 2023-10-09T05:37:04.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:04 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:37:04.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:04 smithi064 ceph-mon[118354]: from='osd.3 [v2:172.21.15.64:6826/3037106123,v1:172.21.15.64:6827/3037106123]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:37:04.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:04 smithi064 ceph-mon[118354]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:37:04.721 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 05:37:04 smithi064 conmon[152299]: 2023-10-09T05:37:04.384+0000 7fe209b71700 -1 osd.3 0 waiting for initial osdmap 2023-10-09T05:37:04.721 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 05:37:04 smithi064 conmon[152299]: 2023-10-09T05:37:04.388+0000 7fe20416b700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:37:04.727 INFO:teuthology.orchestra.run.smithi156.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T05:37:04.727 INFO:teuthology.orchestra.run.smithi156.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T05:37:04.727 INFO:teuthology.orchestra.run.smithi156.stderr: stderr: 10+0 records in 2023-10-09T05:37:04.727 INFO:teuthology.orchestra.run.smithi156.stderr:10+0 records out 2023-10-09T05:37:04.727 INFO:teuthology.orchestra.run.smithi156.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00848203 s, 1.2 GB/s 2023-10-09T05:37:04.728 INFO:teuthology.orchestra.run.smithi156.stderr:--> Zapping successful for: 2023-10-09T05:37:04.767 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:04 smithi156 ceph-mon[105979]: from='osd.3 [v2:172.21.15.64:6826/3037106123,v1:172.21.15.64:6827/3037106123]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T05:37:04.767 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:04 smithi156 ceph-mon[105979]: osdmap e23: 4 total, 3 up, 4 in 2023-10-09T05:37:04.767 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:04 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:37:04.767 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:04 smithi156 ceph-mon[105979]: from='osd.3 [v2:172.21.15.64:6826/3037106123,v1:172.21.15.64:6827/3037106123]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:37:04.767 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:04 smithi156 ceph-mon[105979]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:37:05.356 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi156:vg_nvme/lv_4 2023-10-09T05:37:05.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:05 smithi156 ceph-mon[105979]: purged_snaps scrub starts 2023-10-09T05:37:05.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:05 smithi156 ceph-mon[105979]: purged_snaps scrub ok 2023-10-09T05:37:05.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:05 smithi156 ceph-mon[105979]: from='osd.3 [v2:172.21.15.64:6826/3037106123,v1:172.21.15.64:6827/3037106123]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2023-10-09T05:37:05.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:05 smithi156 ceph-mon[105979]: osdmap e24: 4 total, 3 up, 4 in 2023-10-09T05:37:05.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:05 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:37:05.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:05 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:37:05.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:05 smithi064 ceph-mon[101070]: purged_snaps scrub starts 2023-10-09T05:37:05.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:05 smithi064 ceph-mon[101070]: purged_snaps scrub ok 2023-10-09T05:37:05.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:05 smithi064 ceph-mon[101070]: from='osd.3 [v2:172.21.15.64:6826/3037106123,v1:172.21.15.64:6827/3037106123]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2023-10-09T05:37:05.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:05 smithi064 ceph-mon[101070]: osdmap e24: 4 total, 3 up, 4 in 2023-10-09T05:37:05.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:05 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:37:05.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:05 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:37:05.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:05 smithi064 ceph-mon[118354]: purged_snaps scrub starts 2023-10-09T05:37:05.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:05 smithi064 ceph-mon[118354]: purged_snaps scrub ok 2023-10-09T05:37:05.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:05 smithi064 ceph-mon[118354]: from='osd.3 [v2:172.21.15.64:6826/3037106123,v1:172.21.15.64:6827/3037106123]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2023-10-09T05:37:05.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:05 smithi064 ceph-mon[118354]: osdmap e24: 4 total, 3 up, 4 in 2023-10-09T05:37:05.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:05 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:37:05.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:05 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:37:05.732 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:37:06.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:06 smithi156 ceph-mon[105979]: osd.3 [v2:172.21.15.64:6826/3037106123,v1:172.21.15.64:6827/3037106123] boot 2023-10-09T05:37:06.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:06 smithi156 ceph-mon[105979]: osdmap e25: 4 total, 4 up, 4 in 2023-10-09T05:37:06.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:06 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:37:06.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:06 smithi156 ceph-mon[105979]: Detected new or changed devices on smithi064 2023-10-09T05:37:06.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:06 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:06.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:06 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:06.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:06 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:37:06.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:06 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:37:06.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:06 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:37:06.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:06 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:37:06.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:06 smithi156 ceph-mon[105979]: Adjusting osd_memory_target on smithi064 to 4036M 2023-10-09T05:37:06.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:06 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:06.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:06 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:06.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:06 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:37:06.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:06 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:06.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:06 smithi156 ceph-mon[105979]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:37:06.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[101070]: osd.3 [v2:172.21.15.64:6826/3037106123,v1:172.21.15.64:6827/3037106123] boot 2023-10-09T05:37:06.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[101070]: osdmap e25: 4 total, 4 up, 4 in 2023-10-09T05:37:06.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:37:06.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[101070]: Detected new or changed devices on smithi064 2023-10-09T05:37:06.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:06.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:37:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:37:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:37:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:37:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[101070]: Adjusting osd_memory_target on smithi064 to 4036M 2023-10-09T05:37:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:37:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[101070]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:37:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[118354]: osd.3 [v2:172.21.15.64:6826/3037106123,v1:172.21.15.64:6827/3037106123] boot 2023-10-09T05:37:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[118354]: osdmap e25: 4 total, 4 up, 4 in 2023-10-09T05:37:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:37:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[118354]: Detected new or changed devices on smithi064 2023-10-09T05:37:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:37:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:37:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:37:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:37:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[118354]: Adjusting osd_memory_target on smithi064 to 4036M 2023-10-09T05:37:06.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:06.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:06.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:37:06.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:06.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:06 smithi064 ceph-mon[118354]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:37:07.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:07 smithi064 ceph-mon[101070]: osdmap e26: 4 total, 4 up, 4 in 2023-10-09T05:37:07.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:07 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:37:07.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:07 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:37:07.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:07 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:07.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:07 smithi064 ceph-mon[118354]: osdmap e26: 4 total, 4 up, 4 in 2023-10-09T05:37:07.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:07 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:37:07.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:07 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:37:07.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:07 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:08.015 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:07 smithi156 ceph-mon[105979]: osdmap e26: 4 total, 4 up, 4 in 2023-10-09T05:37:08.015 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:37:08.015 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:37:08.015 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:08.863 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:08 smithi156 ceph-mon[105979]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:37:08.863 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:08 smithi156 ceph-mon[105979]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:08.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:08 smithi064 ceph-mon[101070]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:37:08.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:08 smithi064 ceph-mon[101070]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:08.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:08 smithi064 ceph-mon[118354]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:37:08.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:08 smithi064 ceph-mon[118354]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:09.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:09 smithi064 ceph-mon[118354]: from='client.? 172.21.15.156:0/1061838941' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "060e8766-7bf6-42f1-bebe-dae6828f7fb9"}]: dispatch 2023-10-09T05:37:09.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:09 smithi064 ceph-mon[118354]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "060e8766-7bf6-42f1-bebe-dae6828f7fb9"}]: dispatch 2023-10-09T05:37:09.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:09 smithi064 ceph-mon[118354]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "060e8766-7bf6-42f1-bebe-dae6828f7fb9"}]': finished 2023-10-09T05:37:09.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:09 smithi064 ceph-mon[118354]: osdmap e27: 5 total, 4 up, 5 in 2023-10-09T05:37:09.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:09 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:37:09.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:09 smithi064 ceph-mon[101070]: from='client.? 172.21.15.156:0/1061838941' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "060e8766-7bf6-42f1-bebe-dae6828f7fb9"}]: dispatch 2023-10-09T05:37:09.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:09 smithi064 ceph-mon[101070]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "060e8766-7bf6-42f1-bebe-dae6828f7fb9"}]: dispatch 2023-10-09T05:37:09.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:09 smithi064 ceph-mon[101070]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "060e8766-7bf6-42f1-bebe-dae6828f7fb9"}]': finished 2023-10-09T05:37:09.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:09 smithi064 ceph-mon[101070]: osdmap e27: 5 total, 4 up, 5 in 2023-10-09T05:37:09.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:09 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:37:10.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:09 smithi156 ceph-mon[105979]: from='client.? 172.21.15.156:0/1061838941' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "060e8766-7bf6-42f1-bebe-dae6828f7fb9"}]: dispatch 2023-10-09T05:37:10.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:09 smithi156 ceph-mon[105979]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "060e8766-7bf6-42f1-bebe-dae6828f7fb9"}]: dispatch 2023-10-09T05:37:10.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:09 smithi156 ceph-mon[105979]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "060e8766-7bf6-42f1-bebe-dae6828f7fb9"}]': finished 2023-10-09T05:37:10.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:09 smithi156 ceph-mon[105979]: osdmap e27: 5 total, 4 up, 5 in 2023-10-09T05:37:10.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:09 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:37:10.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:10 smithi064 ceph-mon[118354]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:10.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:10 smithi064 ceph-mon[118354]: from='client.? 172.21.15.156:0/3344584600' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:37:10.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:10 smithi064 ceph-mon[101070]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:10.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:10 smithi064 ceph-mon[101070]: from='client.? 172.21.15.156:0/3344584600' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:37:11.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:10 smithi156 ceph-mon[105979]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:11.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:10 smithi156 ceph-mon[105979]: from='client.? 172.21.15.156:0/3344584600' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:37:13.085 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:12 smithi156 ceph-mon[105979]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:13.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:12 smithi064 ceph-mon[101070]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:13.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:12 smithi064 ceph-mon[118354]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:15.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:14 smithi156 ceph-mon[105979]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:15.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:14 smithi064 ceph-mon[101070]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:15.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:14 smithi064 ceph-mon[118354]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:16.463 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:16 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T05:37:16.464 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:16 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:16.464 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:16 smithi156 ceph-mon[105979]: Deploying daemon osd.4 on smithi156 2023-10-09T05:37:16.464 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:16 smithi156 ceph-mon[105979]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:16 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T05:37:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:16 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:16 smithi064 ceph-mon[101070]: Deploying daemon osd.4 on smithi156 2023-10-09T05:37:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:16 smithi064 ceph-mon[101070]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:16 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T05:37:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:16 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:16 smithi064 ceph-mon[118354]: Deploying daemon osd.4 on smithi156 2023-10-09T05:37:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:16 smithi064 ceph-mon[118354]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:19.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:18 smithi156 ceph-mon[105979]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:19.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:18 smithi064 ceph-mon[101070]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:19.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:18 smithi064 ceph-mon[118354]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:21.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:20 smithi156 ceph-mon[105979]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:21.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:20 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:21.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:20 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:21.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:20 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:37:21.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:20 smithi064 ceph-mon[101070]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:21.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:20 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:21.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:20 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:21.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:20 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:37:21.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:20 smithi064 ceph-mon[118354]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:21.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:20 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:21.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:20 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:21.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:20 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:37:23.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:22 smithi156 ceph-mon[105979]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:23.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:22 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:23.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:22 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:23.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:22 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:23.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:22 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:37:23.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:22 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:23.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:22 smithi064 ceph-mon[101070]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:23.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:22 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:23.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:22 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:23.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:22 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:23.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:22 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:37:23.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:22 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:23.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:22 smithi064 ceph-mon[118354]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:23.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:22 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:23.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:22 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:23.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:22 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:23.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:22 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:37:23.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:22 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:23.280 INFO:teuthology.orchestra.run.smithi156.stdout:Created osd(s) 4 on host 'smithi156' 2023-10-09T05:37:23.818 DEBUG:teuthology.orchestra.run.smithi156:osd.4> sudo journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.4.service 2023-10-09T05:37:23.820 INFO:tasks.cephadm:Deploying osd.5 on smithi156 with /dev/vg_nvme/lv_3... 2023-10-09T05:37:23.820 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T05:37:23.850 INFO:journalctl@ceph.osd.4.smithi156.stdout:-- Logs begin at Mon 2023-10-09 05:16:12 UTC. -- 2023-10-09T05:37:24.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:24 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:24.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:24 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:24.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:24 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:37:24.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:24 smithi156 ceph-mon[105979]: from='osd.4 [v2:172.21.15.156:6800/657648738,v1:172.21.15.156:6801/657648738]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:37:24.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:24 smithi156 ceph-mon[105979]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:37:24.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:24 smithi156 ceph-mon[105979]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:24.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:24 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:24.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:24 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:24.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:24 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:37:24.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:24 smithi064 ceph-mon[101070]: from='osd.4 [v2:172.21.15.156:6800/657648738,v1:172.21.15.156:6801/657648738]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:37:24.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:24 smithi064 ceph-mon[101070]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:37:24.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:24 smithi064 ceph-mon[101070]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:24.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:24 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:24.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:24 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:24.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:24 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:37:24.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:24 smithi064 ceph-mon[118354]: from='osd.4 [v2:172.21.15.156:6800/657648738,v1:172.21.15.156:6801/657648738]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:37:24.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:24 smithi064 ceph-mon[118354]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:37:24.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:24 smithi064 ceph-mon[118354]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:24.773 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:37:25.421 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:25 smithi156 ceph-mon[105979]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T05:37:25.421 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:25 smithi156 ceph-mon[105979]: osdmap e28: 5 total, 4 up, 5 in 2023-10-09T05:37:25.422 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:25 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:37:25.422 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:25 smithi156 ceph-mon[105979]: from='osd.4 [v2:172.21.15.156:6800/657648738,v1:172.21.15.156:6801/657648738]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T05:37:25.422 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:25 smithi156 ceph-mon[105979]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T05:37:25.422 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 05:37:25 smithi156 conmon[114036]: 2023-10-09T05:37:25.260+0000 7f1209ef4700 -1 osd.4 0 waiting for initial osdmap 2023-10-09T05:37:25.422 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 05:37:25 smithi156 conmon[114036]: 2023-10-09T05:37:25.264+0000 7f1203ced700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:37:25.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:25 smithi064 ceph-mon[101070]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T05:37:25.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:25 smithi064 ceph-mon[101070]: osdmap e28: 5 total, 4 up, 5 in 2023-10-09T05:37:25.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:25 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:37:25.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:25 smithi064 ceph-mon[101070]: from='osd.4 [v2:172.21.15.156:6800/657648738,v1:172.21.15.156:6801/657648738]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T05:37:25.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:25 smithi064 ceph-mon[101070]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T05:37:25.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:25 smithi064 ceph-mon[118354]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T05:37:25.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:25 smithi064 ceph-mon[118354]: osdmap e28: 5 total, 4 up, 5 in 2023-10-09T05:37:25.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:25 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:37:25.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:25 smithi064 ceph-mon[118354]: from='osd.4 [v2:172.21.15.156:6800/657648738,v1:172.21.15.156:6801/657648738]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T05:37:25.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:25 smithi064 ceph-mon[118354]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T05:37:26.437 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:26 smithi156 ceph-mon[105979]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2023-10-09T05:37:26.438 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:26 smithi156 ceph-mon[105979]: osdmap e29: 5 total, 4 up, 5 in 2023-10-09T05:37:26.438 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:26 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:37:26.438 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:26 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:37:26.438 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:26 smithi156 ceph-mon[105979]: osd.4 [v2:172.21.15.156:6800/657648738,v1:172.21.15.156:6801/657648738] boot 2023-10-09T05:37:26.438 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:26 smithi156 ceph-mon[105979]: osdmap e30: 5 total, 5 up, 5 in 2023-10-09T05:37:26.438 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:26 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:37:26.438 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:26 smithi156 ceph-mon[105979]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:26.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:26 smithi064 ceph-mon[101070]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2023-10-09T05:37:26.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:26 smithi064 ceph-mon[101070]: osdmap e29: 5 total, 4 up, 5 in 2023-10-09T05:37:26.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:26 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:37:26.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:26 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:37:26.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:26 smithi064 ceph-mon[101070]: osd.4 [v2:172.21.15.156:6800/657648738,v1:172.21.15.156:6801/657648738] boot 2023-10-09T05:37:26.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:26 smithi064 ceph-mon[101070]: osdmap e30: 5 total, 5 up, 5 in 2023-10-09T05:37:26.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:26 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:37:26.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:26 smithi064 ceph-mon[101070]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:26 smithi064 ceph-mon[118354]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2023-10-09T05:37:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:26 smithi064 ceph-mon[118354]: osdmap e29: 5 total, 4 up, 5 in 2023-10-09T05:37:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:26 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:37:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:26 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:37:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:26 smithi064 ceph-mon[118354]: osd.4 [v2:172.21.15.156:6800/657648738,v1:172.21.15.156:6801/657648738] boot 2023-10-09T05:37:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:26 smithi064 ceph-mon[118354]: osdmap e30: 5 total, 5 up, 5 in 2023-10-09T05:37:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:26 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:37:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:26 smithi064 ceph-mon[118354]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:37:26.729 INFO:teuthology.orchestra.run.smithi156.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T05:37:26.729 INFO:teuthology.orchestra.run.smithi156.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T05:37:26.729 INFO:teuthology.orchestra.run.smithi156.stderr: stderr: 10+0 records in 2023-10-09T05:37:26.729 INFO:teuthology.orchestra.run.smithi156.stderr:10+0 records out 2023-10-09T05:37:26.729 INFO:teuthology.orchestra.run.smithi156.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00814592 s, 1.3 GB/s 2023-10-09T05:37:26.729 INFO:teuthology.orchestra.run.smithi156.stderr:--> Zapping successful for: 2023-10-09T05:37:27.322 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi156:vg_nvme/lv_3 2023-10-09T05:37:27.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:27 smithi156 ceph-mon[105979]: purged_snaps scrub starts 2023-10-09T05:37:27.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:27 smithi156 ceph-mon[105979]: purged_snaps scrub ok 2023-10-09T05:37:27.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:27 smithi156 ceph-mon[105979]: osdmap e31: 5 total, 5 up, 5 in 2023-10-09T05:37:27.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:27 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:27.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:27 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:27.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:27 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:37:27.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:27 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:27.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:27 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:27.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:27 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:37:27.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:27 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:27.660 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:37:27.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:27 smithi064 ceph-mon[101070]: purged_snaps scrub starts 2023-10-09T05:37:27.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:27 smithi064 ceph-mon[101070]: purged_snaps scrub ok 2023-10-09T05:37:27.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:27 smithi064 ceph-mon[101070]: osdmap e31: 5 total, 5 up, 5 in 2023-10-09T05:37:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:27 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:27 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:27 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:37:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:27 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:27 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:27 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:37:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:27 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:27.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:27 smithi064 ceph-mon[118354]: purged_snaps scrub starts 2023-10-09T05:37:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:27 smithi064 ceph-mon[118354]: purged_snaps scrub ok 2023-10-09T05:37:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:27 smithi064 ceph-mon[118354]: osdmap e31: 5 total, 5 up, 5 in 2023-10-09T05:37:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:27 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:27 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:27 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:37:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:27 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:27 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:27 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:37:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:27 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:28.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:28 smithi156 ceph-mon[105979]: Detected new or changed devices on smithi156 2023-10-09T05:37:28.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:28 smithi156 ceph-mon[105979]: Adjusting osd_memory_target on smithi156 to 17168M 2023-10-09T05:37:28.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:28 smithi156 ceph-mon[105979]: osdmap e32: 5 total, 5 up, 5 in 2023-10-09T05:37:28.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:28 smithi156 ceph-mon[105979]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:37:28.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:28 smithi064 ceph-mon[101070]: Detected new or changed devices on smithi156 2023-10-09T05:37:28.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:28 smithi064 ceph-mon[101070]: Adjusting osd_memory_target on smithi156 to 17168M 2023-10-09T05:37:28.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:28 smithi064 ceph-mon[101070]: osdmap e32: 5 total, 5 up, 5 in 2023-10-09T05:37:28.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:28 smithi064 ceph-mon[101070]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:37:28.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:28 smithi064 ceph-mon[118354]: Detected new or changed devices on smithi156 2023-10-09T05:37:28.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:28 smithi064 ceph-mon[118354]: Adjusting osd_memory_target on smithi156 to 17168M 2023-10-09T05:37:28.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:28 smithi064 ceph-mon[118354]: osdmap e32: 5 total, 5 up, 5 in 2023-10-09T05:37:28.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:28 smithi064 ceph-mon[118354]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:37:29.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:29 smithi064 ceph-mon[101070]: osdmap e33: 5 total, 5 up, 5 in 2023-10-09T05:37:29.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:29 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:37:29.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:29 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:37:29.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:29 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:29.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:29 smithi064 ceph-mon[118354]: osdmap e33: 5 total, 5 up, 5 in 2023-10-09T05:37:29.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:29 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:37:29.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:29 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:37:29.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:29 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:29.827 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:29 smithi156 ceph-mon[105979]: osdmap e33: 5 total, 5 up, 5 in 2023-10-09T05:37:29.827 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:29 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:37:29.827 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:29 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:37:29.827 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:29 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:30.558 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:30 smithi156 ceph-mon[105979]: from='client.24232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:37:30.558 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:30 smithi156 ceph-mon[105979]: osdmap e34: 5 total, 5 up, 5 in 2023-10-09T05:37:30.558 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:30 smithi156 ceph-mon[105979]: pgmap v111: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-10-09T05:37:30.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:30 smithi064 ceph-mon[101070]: from='client.24232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:37:30.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:30 smithi064 ceph-mon[101070]: osdmap e34: 5 total, 5 up, 5 in 2023-10-09T05:37:30.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:30 smithi064 ceph-mon[101070]: pgmap v111: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-10-09T05:37:30.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:30 smithi064 ceph-mon[118354]: from='client.24232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:37:30.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:30 smithi064 ceph-mon[118354]: osdmap e34: 5 total, 5 up, 5 in 2023-10-09T05:37:30.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:30 smithi064 ceph-mon[118354]: pgmap v111: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-10-09T05:37:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:31 smithi156 ceph-mon[105979]: from='client.? 172.21.15.156:0/2197344822' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6b5cf38a-6792-4ac7-9694-268601260ba8"}]: dispatch 2023-10-09T05:37:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:31 smithi156 ceph-mon[105979]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6b5cf38a-6792-4ac7-9694-268601260ba8"}]: dispatch 2023-10-09T05:37:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:31 smithi156 ceph-mon[105979]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6b5cf38a-6792-4ac7-9694-268601260ba8"}]': finished 2023-10-09T05:37:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:31 smithi156 ceph-mon[105979]: osdmap e35: 6 total, 5 up, 6 in 2023-10-09T05:37:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:31 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:37:31.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:31 smithi064 ceph-mon[101070]: from='client.? 172.21.15.156:0/2197344822' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6b5cf38a-6792-4ac7-9694-268601260ba8"}]: dispatch 2023-10-09T05:37:31.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:31 smithi064 ceph-mon[101070]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6b5cf38a-6792-4ac7-9694-268601260ba8"}]: dispatch 2023-10-09T05:37:31.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:31 smithi064 ceph-mon[101070]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6b5cf38a-6792-4ac7-9694-268601260ba8"}]': finished 2023-10-09T05:37:31.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:31 smithi064 ceph-mon[101070]: osdmap e35: 6 total, 5 up, 6 in 2023-10-09T05:37:31.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:31 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:37:31.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:31 smithi064 ceph-mon[118354]: from='client.? 172.21.15.156:0/2197344822' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6b5cf38a-6792-4ac7-9694-268601260ba8"}]: dispatch 2023-10-09T05:37:31.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:31 smithi064 ceph-mon[118354]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6b5cf38a-6792-4ac7-9694-268601260ba8"}]: dispatch 2023-10-09T05:37:31.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:31 smithi064 ceph-mon[118354]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6b5cf38a-6792-4ac7-9694-268601260ba8"}]': finished 2023-10-09T05:37:31.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:31 smithi064 ceph-mon[118354]: osdmap e35: 6 total, 5 up, 6 in 2023-10-09T05:37:31.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:31 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:37:32.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:32 smithi156 ceph-mon[105979]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 131 KiB/s, 0 objects/s recovering 2023-10-09T05:37:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:32 smithi156 ceph-mon[105979]: from='client.? 172.21.15.156:0/1298932127' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:37:32.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:32 smithi064 ceph-mon[101070]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 131 KiB/s, 0 objects/s recovering 2023-10-09T05:37:32.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:32 smithi064 ceph-mon[101070]: from='client.? 172.21.15.156:0/1298932127' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:37:32.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:32 smithi064 ceph-mon[118354]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 131 KiB/s, 0 objects/s recovering 2023-10-09T05:37:32.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:32 smithi064 ceph-mon[118354]: from='client.? 172.21.15.156:0/1298932127' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:37:35.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:34 smithi156 ceph-mon[105979]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T05:37:35.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:34 smithi064 ceph-mon[101070]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T05:37:35.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:34 smithi064 ceph-mon[118354]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T05:37:36.503 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:36 smithi156 ceph-mon[105979]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 78 KiB/s, 0 objects/s recovering 2023-10-09T05:37:36.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:36 smithi064 ceph-mon[101070]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 78 KiB/s, 0 objects/s recovering 2023-10-09T05:37:36.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:36 smithi064 ceph-mon[118354]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 78 KiB/s, 0 objects/s recovering 2023-10-09T05:37:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:37 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T05:37:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:37 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:37.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:37 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T05:37:37.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:37 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:37.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:37 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T05:37:37.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:37 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:38.698 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:38 smithi156 ceph-mon[105979]: Deploying daemon osd.5 on smithi156 2023-10-09T05:37:38.698 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:38 smithi156 ceph-mon[105979]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:37:38.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:38 smithi064 ceph-mon[101070]: Deploying daemon osd.5 on smithi156 2023-10-09T05:37:38.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:38 smithi064 ceph-mon[101070]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:37:38.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:38 smithi064 ceph-mon[118354]: Deploying daemon osd.5 on smithi156 2023-10-09T05:37:38.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:38 smithi064 ceph-mon[118354]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:37:41.118 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:40 smithi156 ceph-mon[105979]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:37:41.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:40 smithi064 ceph-mon[101070]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:37:41.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:40 smithi064 ceph-mon[118354]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:37:42.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:42 smithi156 ceph-mon[105979]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:37:42.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:42 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:42.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:42 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:42.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:42 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:37:43.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:42 smithi064 ceph-mon[101070]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:37:43.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:42 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:43.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:42 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:43.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:42 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:37:43.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:42 smithi064 ceph-mon[118354]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:37:43.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:42 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:43.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:42 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:43.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:42 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:37:44.932 INFO:teuthology.orchestra.run.smithi156.stdout:Created osd(s) 5 on host 'smithi156' 2023-10-09T05:37:45.118 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:44 smithi156 ceph-mon[105979]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:37:45.118 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:44 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:45.118 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:44 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:45.118 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:44 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:45.118 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:44 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:37:45.118 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:44 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:45.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:44 smithi064 ceph-mon[118354]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:37:45.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:44 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:45.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:44 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:45.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:44 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:45.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:44 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:37:45.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:44 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:45.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:44 smithi064 ceph-mon[101070]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:37:45.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:44 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:45.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:44 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:45.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:44 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:45.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:44 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:37:45.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:44 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:45.506 DEBUG:teuthology.orchestra.run.smithi156:osd.5> sudo journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.5.service 2023-10-09T05:37:45.508 INFO:tasks.cephadm:Deploying osd.6 on smithi156 with /dev/vg_nvme/lv_2... 2023-10-09T05:37:45.508 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-09T05:37:45.538 INFO:journalctl@ceph.osd.5.smithi156.stdout:-- Logs begin at Mon 2023-10-09 05:16:12 UTC. -- 2023-10-09T05:37:46.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:45 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:46.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:45 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:46.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:45 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:37:46.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:45 smithi156 ceph-mon[105979]: from='osd.5 [v2:172.21.15.156:6808/726279450,v1:172.21.15.156:6809/726279450]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T05:37:46.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:45 smithi156 ceph-mon[105979]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T05:37:46.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:45 smithi156 ceph-mon[105979]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:37:46.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:45 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:46.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:45 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:46.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:45 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:37:46.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:45 smithi064 ceph-mon[101070]: from='osd.5 [v2:172.21.15.156:6808/726279450,v1:172.21.15.156:6809/726279450]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T05:37:46.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:45 smithi064 ceph-mon[101070]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T05:37:46.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:45 smithi064 ceph-mon[101070]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:37:46.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:45 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:46.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:45 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:46.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:45 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:37:46.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:45 smithi064 ceph-mon[118354]: from='osd.5 [v2:172.21.15.156:6808/726279450,v1:172.21.15.156:6809/726279450]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T05:37:46.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:45 smithi064 ceph-mon[118354]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T05:37:46.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:45 smithi064 ceph-mon[118354]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:37:46.531 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:37:46.995 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:46 smithi156 ceph-mon[105979]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T05:37:46.995 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:46 smithi156 ceph-mon[105979]: osdmap e36: 6 total, 5 up, 6 in 2023-10-09T05:37:46.995 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:46 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:37:46.995 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:46 smithi156 ceph-mon[105979]: from='osd.5 [v2:172.21.15.156:6808/726279450,v1:172.21.15.156:6809/726279450]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T05:37:46.995 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:46 smithi156 ceph-mon[105979]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T05:37:46.995 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 05:37:46 smithi156 conmon[123547]: 2023-10-09T05:37:46.903+0000 7fd2f9c24700 -1 osd.5 0 waiting for initial osdmap 2023-10-09T05:37:46.996 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 05:37:46 smithi156 conmon[123547]: 2023-10-09T05:37:46.907+0000 7fd2f4a1f700 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:37:47.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:46 smithi064 ceph-mon[101070]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T05:37:47.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:46 smithi064 ceph-mon[101070]: osdmap e36: 6 total, 5 up, 6 in 2023-10-09T05:37:47.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:46 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:37:47.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:46 smithi064 ceph-mon[101070]: from='osd.5 [v2:172.21.15.156:6808/726279450,v1:172.21.15.156:6809/726279450]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T05:37:47.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:46 smithi064 ceph-mon[101070]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T05:37:47.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:46 smithi064 ceph-mon[118354]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T05:37:47.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:46 smithi064 ceph-mon[118354]: osdmap e36: 6 total, 5 up, 6 in 2023-10-09T05:37:47.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:46 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:37:47.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:46 smithi064 ceph-mon[118354]: from='osd.5 [v2:172.21.15.156:6808/726279450,v1:172.21.15.156:6809/726279450]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T05:37:47.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:46 smithi064 ceph-mon[118354]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T05:37:48.002 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:47 smithi156 ceph-mon[105979]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2023-10-09T05:37:48.002 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:47 smithi156 ceph-mon[105979]: osdmap e37: 6 total, 5 up, 6 in 2023-10-09T05:37:48.003 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:47 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:37:48.003 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:47 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:37:48.003 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:47 smithi156 ceph-mon[105979]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:37:48.003 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:47 smithi156 ceph-mon[105979]: osd.5 [v2:172.21.15.156:6808/726279450,v1:172.21.15.156:6809/726279450] boot 2023-10-09T05:37:48.003 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:47 smithi156 ceph-mon[105979]: osdmap e38: 6 total, 6 up, 6 in 2023-10-09T05:37:48.003 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:47 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:37:48.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:47 smithi064 ceph-mon[101070]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2023-10-09T05:37:48.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:47 smithi064 ceph-mon[101070]: osdmap e37: 6 total, 5 up, 6 in 2023-10-09T05:37:48.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:47 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:37:48.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:47 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:37:48.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:47 smithi064 ceph-mon[101070]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:37:48.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:47 smithi064 ceph-mon[101070]: osd.5 [v2:172.21.15.156:6808/726279450,v1:172.21.15.156:6809/726279450] boot 2023-10-09T05:37:48.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:47 smithi064 ceph-mon[101070]: osdmap e38: 6 total, 6 up, 6 in 2023-10-09T05:37:48.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:47 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:37:48.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:47 smithi064 ceph-mon[118354]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2023-10-09T05:37:48.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:47 smithi064 ceph-mon[118354]: osdmap e37: 6 total, 5 up, 6 in 2023-10-09T05:37:48.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:47 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:37:48.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:47 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:37:48.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:47 smithi064 ceph-mon[118354]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:37:48.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:47 smithi064 ceph-mon[118354]: osd.5 [v2:172.21.15.156:6808/726279450,v1:172.21.15.156:6809/726279450] boot 2023-10-09T05:37:48.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:47 smithi064 ceph-mon[118354]: osdmap e38: 6 total, 6 up, 6 in 2023-10-09T05:37:48.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:47 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:37:48.710 INFO:teuthology.orchestra.run.smithi156.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-09T05:37:48.710 INFO:teuthology.orchestra.run.smithi156.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-09T05:37:48.710 INFO:teuthology.orchestra.run.smithi156.stderr: stderr: 10+0 records in 2023-10-09T05:37:48.711 INFO:teuthology.orchestra.run.smithi156.stderr:10+0 records out 2023-10-09T05:37:48.711 INFO:teuthology.orchestra.run.smithi156.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00822313 s, 1.3 GB/s 2023-10-09T05:37:48.711 INFO:teuthology.orchestra.run.smithi156.stderr:--> Zapping successful for: 2023-10-09T05:37:49.332 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi156:vg_nvme/lv_2 2023-10-09T05:37:49.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[101070]: purged_snaps scrub starts 2023-10-09T05:37:49.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[101070]: purged_snaps scrub ok 2023-10-09T05:37:49.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[101070]: Detected new or changed devices on smithi156 2023-10-09T05:37:49.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:49.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:49.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:37:49.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:37:49.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[101070]: Adjusting osd_memory_target on smithi156 to 8584M 2023-10-09T05:37:49.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:49.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:49.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:37:49.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:49.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[101070]: osdmap e39: 6 total, 6 up, 6 in 2023-10-09T05:37:49.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[118354]: purged_snaps scrub starts 2023-10-09T05:37:49.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[118354]: purged_snaps scrub ok 2023-10-09T05:37:49.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[118354]: Detected new or changed devices on smithi156 2023-10-09T05:37:49.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:49.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:49.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:37:49.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:37:49.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[118354]: Adjusting osd_memory_target on smithi156 to 8584M 2023-10-09T05:37:49.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:49.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:49.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:37:49.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:49.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:49 smithi064 ceph-mon[118354]: osdmap e39: 6 total, 6 up, 6 in 2023-10-09T05:37:49.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:49 smithi156 ceph-mon[105979]: purged_snaps scrub starts 2023-10-09T05:37:49.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:49 smithi156 ceph-mon[105979]: purged_snaps scrub ok 2023-10-09T05:37:49.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:49 smithi156 ceph-mon[105979]: Detected new or changed devices on smithi156 2023-10-09T05:37:49.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:49 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:49.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:49 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:49.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:49 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:37:49.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:49 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:37:49.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:49 smithi156 ceph-mon[105979]: Adjusting osd_memory_target on smithi156 to 8584M 2023-10-09T05:37:49.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:49 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:49.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:49 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:49.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:49 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:37:49.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:49 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:37:49.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:49 smithi156 ceph-mon[105979]: osdmap e39: 6 total, 6 up, 6 in 2023-10-09T05:37:49.667 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:37:50.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:50 smithi064 ceph-mon[101070]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:37:50.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:50 smithi064 ceph-mon[101070]: osdmap e40: 6 total, 6 up, 6 in 2023-10-09T05:37:50.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:50 smithi064 ceph-mon[118354]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:37:50.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:50 smithi064 ceph-mon[118354]: osdmap e40: 6 total, 6 up, 6 in 2023-10-09T05:37:50.592 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:50 smithi156 ceph-mon[105979]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:37:50.592 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:50 smithi156 ceph-mon[105979]: osdmap e40: 6 total, 6 up, 6 in 2023-10-09T05:37:51.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:51 smithi156 ceph-mon[105979]: osdmap e41: 6 total, 6 up, 6 in 2023-10-09T05:37:51.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:51 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:37:51.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:51 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:37:51.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:51 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:51.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:51 smithi064 ceph-mon[101070]: osdmap e41: 6 total, 6 up, 6 in 2023-10-09T05:37:51.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:51 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:37:51.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:51 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:37:51.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:51 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:51.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:51 smithi064 ceph-mon[118354]: osdmap e41: 6 total, 6 up, 6 in 2023-10-09T05:37:51.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:51 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:37:51.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:51 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:37:51.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:51 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:37:52.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:52 smithi156 ceph-mon[105979]: from='client.24259 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:37:52.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:52 smithi156 ceph-mon[105979]: osdmap e42: 6 total, 6 up, 6 in 2023-10-09T05:37:52.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:52 smithi156 ceph-mon[105979]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:37:52.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:52 smithi064 ceph-mon[101070]: from='client.24259 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:37:52.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:52 smithi064 ceph-mon[101070]: osdmap e42: 6 total, 6 up, 6 in 2023-10-09T05:37:52.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:52 smithi064 ceph-mon[101070]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:37:52.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:52 smithi064 ceph-mon[118354]: from='client.24259 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:37:52.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:52 smithi064 ceph-mon[118354]: osdmap e42: 6 total, 6 up, 6 in 2023-10-09T05:37:52.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:52 smithi064 ceph-mon[118354]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:37:53.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:53 smithi156 ceph-mon[105979]: from='client.? 172.21.15.156:0/475403574' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "01deea2d-a225-40b8-a54b-61eee57da527"}]: dispatch 2023-10-09T05:37:53.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:53 smithi156 ceph-mon[105979]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "01deea2d-a225-40b8-a54b-61eee57da527"}]: dispatch 2023-10-09T05:37:53.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:53 smithi156 ceph-mon[105979]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "01deea2d-a225-40b8-a54b-61eee57da527"}]': finished 2023-10-09T05:37:53.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:53 smithi156 ceph-mon[105979]: osdmap e43: 7 total, 6 up, 7 in 2023-10-09T05:37:53.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:53 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:37:53.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:53 smithi064 ceph-mon[101070]: from='client.? 172.21.15.156:0/475403574' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "01deea2d-a225-40b8-a54b-61eee57da527"}]: dispatch 2023-10-09T05:37:53.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:53 smithi064 ceph-mon[101070]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "01deea2d-a225-40b8-a54b-61eee57da527"}]: dispatch 2023-10-09T05:37:53.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:53 smithi064 ceph-mon[101070]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "01deea2d-a225-40b8-a54b-61eee57da527"}]': finished 2023-10-09T05:37:53.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:53 smithi064 ceph-mon[101070]: osdmap e43: 7 total, 6 up, 7 in 2023-10-09T05:37:53.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:53 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:37:53.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:53 smithi064 ceph-mon[118354]: from='client.? 172.21.15.156:0/475403574' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "01deea2d-a225-40b8-a54b-61eee57da527"}]: dispatch 2023-10-09T05:37:53.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:53 smithi064 ceph-mon[118354]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "01deea2d-a225-40b8-a54b-61eee57da527"}]: dispatch 2023-10-09T05:37:53.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:53 smithi064 ceph-mon[118354]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "01deea2d-a225-40b8-a54b-61eee57da527"}]': finished 2023-10-09T05:37:53.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:53 smithi064 ceph-mon[118354]: osdmap e43: 7 total, 6 up, 7 in 2023-10-09T05:37:53.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:53 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:37:54.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:54 smithi156 ceph-mon[105979]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:37:54.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:54 smithi156 ceph-mon[105979]: from='client.? 172.21.15.156:0/2829993000' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:37:54.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:54 smithi064 ceph-mon[101070]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:37:54.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:54 smithi064 ceph-mon[101070]: from='client.? 172.21.15.156:0/2829993000' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:37:54.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:54 smithi064 ceph-mon[118354]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:37:54.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:54 smithi064 ceph-mon[118354]: from='client.? 172.21.15.156:0/2829993000' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:37:56.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:56 smithi156 ceph-mon[105979]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:37:56.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:56 smithi064 ceph-mon[101070]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:37:56.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:56 smithi064 ceph-mon[118354]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:37:59.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:58 smithi156 ceph-mon[105979]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:37:59.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:58 smithi064 ceph-mon[101070]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:37:59.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:58 smithi064 ceph-mon[118354]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:38:00.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:59 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T05:38:00.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:37:59 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:00.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:59 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T05:38:00.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:37:59 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:00.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:59 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T05:38:00.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:37:59 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:00.960 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:00 smithi156 ceph-mon[105979]: Deploying daemon osd.6 on smithi156 2023-10-09T05:38:00.960 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:00 smithi156 ceph-mon[105979]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:38:01.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:00 smithi064 ceph-mon[101070]: Deploying daemon osd.6 on smithi156 2023-10-09T05:38:01.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:00 smithi064 ceph-mon[101070]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:38:01.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:00 smithi064 ceph-mon[118354]: Deploying daemon osd.6 on smithi156 2023-10-09T05:38:01.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:00 smithi064 ceph-mon[118354]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:38:03.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:02 smithi156 ceph-mon[105979]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:38:03.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:02 smithi064 ceph-mon[101070]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:38:03.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:02 smithi064 ceph-mon[118354]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:38:05.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:04 smithi156 ceph-mon[105979]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:38:05.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:04 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:05.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:04 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:05.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:04 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:38:05.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:04 smithi064 ceph-mon[101070]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:38:05.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:04 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:05.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:04 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:05.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:04 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:38:05.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:04 smithi064 ceph-mon[118354]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:38:05.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:04 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:05.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:04 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:05.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:04 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:38:06.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:06 smithi156 ceph-mon[105979]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:38:06.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:06 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:06.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:06 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:06.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:06 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:06.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:06 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:38:06.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:06 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:06.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:06 smithi064 ceph-mon[118354]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:38:06.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:06 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:06.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:06 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:06.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:06 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:06.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:06 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:38:06.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:06 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:06 smithi064 ceph-mon[101070]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:38:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:06 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:06 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:06 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:06 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:38:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:06 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:06.727 INFO:teuthology.orchestra.run.smithi156.stdout:Created osd(s) 6 on host 'smithi156' 2023-10-09T05:38:07.302 DEBUG:teuthology.orchestra.run.smithi156:osd.6> sudo journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.6.service 2023-10-09T05:38:07.305 INFO:tasks.cephadm:Deploying osd.7 on smithi156 with /dev/vg_nvme/lv_1... 2023-10-09T05:38:07.305 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-09T05:38:07.340 INFO:journalctl@ceph.osd.6.smithi156.stdout:-- Logs begin at Mon 2023-10-09 05:16:12 UTC. -- 2023-10-09T05:38:07.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:07.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:07.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:07 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:38:07.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:07 smithi156 ceph-mon[105979]: from='osd.6 [v2:172.21.15.156:6816/2369965428,v1:172.21.15.156:6817/2369965428]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T05:38:07.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:07 smithi156 ceph-mon[105979]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T05:38:07.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:07 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:07.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:07 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:07.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:07 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:38:07.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:07 smithi064 ceph-mon[101070]: from='osd.6 [v2:172.21.15.156:6816/2369965428,v1:172.21.15.156:6817/2369965428]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T05:38:07.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:07 smithi064 ceph-mon[101070]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T05:38:07.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:07 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:07.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:07 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:07.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:07 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:38:07.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:07 smithi064 ceph-mon[118354]: from='osd.6 [v2:172.21.15.156:6816/2369965428,v1:172.21.15.156:6817/2369965428]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T05:38:07.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:07 smithi064 ceph-mon[118354]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T05:38:08.288 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:38:08.922 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:08 smithi156 ceph-mon[105979]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T05:38:08.923 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:08 smithi156 ceph-mon[105979]: osdmap e44: 7 total, 6 up, 7 in 2023-10-09T05:38:08.923 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:08 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:38:08.923 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:08 smithi156 ceph-mon[105979]: from='osd.6 [v2:172.21.15.156:6816/2369965428,v1:172.21.15.156:6817/2369965428]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T05:38:08.923 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:08 smithi156 ceph-mon[105979]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T05:38:08.923 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:08 smithi156 ceph-mon[105979]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:38:08.923 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 05:38:08 smithi156 conmon[132493]: 2023-10-09T05:38:08.722+0000 7fa68159e700 -1 osd.6 0 waiting for initial osdmap 2023-10-09T05:38:08.923 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 05:38:08 smithi156 conmon[132493]: 2023-10-09T05:38:08.729+0000 7fa67cb9a700 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:38:08.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:08 smithi064 ceph-mon[101070]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T05:38:08.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:08 smithi064 ceph-mon[101070]: osdmap e44: 7 total, 6 up, 7 in 2023-10-09T05:38:08.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:08 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:38:08.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:08 smithi064 ceph-mon[101070]: from='osd.6 [v2:172.21.15.156:6816/2369965428,v1:172.21.15.156:6817/2369965428]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T05:38:08.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:08 smithi064 ceph-mon[101070]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T05:38:08.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:08 smithi064 ceph-mon[101070]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:38:08.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:08 smithi064 ceph-mon[118354]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T05:38:08.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:08 smithi064 ceph-mon[118354]: osdmap e44: 7 total, 6 up, 7 in 2023-10-09T05:38:08.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:08 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:38:08.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:08 smithi064 ceph-mon[118354]: from='osd.6 [v2:172.21.15.156:6816/2369965428,v1:172.21.15.156:6817/2369965428]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T05:38:08.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:08 smithi064 ceph-mon[118354]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T05:38:08.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:08 smithi064 ceph-mon[118354]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:38:09.964 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:09 smithi156 ceph-mon[105979]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2023-10-09T05:38:09.964 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:09 smithi156 ceph-mon[105979]: osdmap e45: 7 total, 6 up, 7 in 2023-10-09T05:38:09.964 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:09 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:38:09.964 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:09 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:38:09.964 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:09 smithi156 ceph-mon[105979]: osd.6 [v2:172.21.15.156:6816/2369965428,v1:172.21.15.156:6817/2369965428] boot 2023-10-09T05:38:10.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:09 smithi064 ceph-mon[101070]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2023-10-09T05:38:10.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:09 smithi064 ceph-mon[101070]: osdmap e45: 7 total, 6 up, 7 in 2023-10-09T05:38:10.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:09 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:38:10.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:09 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:38:10.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:09 smithi064 ceph-mon[101070]: osd.6 [v2:172.21.15.156:6816/2369965428,v1:172.21.15.156:6817/2369965428] boot 2023-10-09T05:38:10.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:09 smithi064 ceph-mon[118354]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2023-10-09T05:38:10.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:09 smithi064 ceph-mon[118354]: osdmap e45: 7 total, 6 up, 7 in 2023-10-09T05:38:10.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:09 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:38:10.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:09 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:38:10.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:09 smithi064 ceph-mon[118354]: osd.6 [v2:172.21.15.156:6816/2369965428,v1:172.21.15.156:6817/2369965428] boot 2023-10-09T05:38:10.229 INFO:teuthology.orchestra.run.smithi156.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-09T05:38:10.229 INFO:teuthology.orchestra.run.smithi156.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-09T05:38:10.230 INFO:teuthology.orchestra.run.smithi156.stderr: stderr: 10+0 records in 2023-10-09T05:38:10.230 INFO:teuthology.orchestra.run.smithi156.stderr:10+0 records out 2023-10-09T05:38:10.230 INFO:teuthology.orchestra.run.smithi156.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00829462 s, 1.3 GB/s 2023-10-09T05:38:10.230 INFO:teuthology.orchestra.run.smithi156.stderr:--> Zapping successful for: 2023-10-09T05:38:10.787 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi156:vg_nvme/lv_1 2023-10-09T05:38:11.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:10 smithi156 ceph-mon[105979]: purged_snaps scrub starts 2023-10-09T05:38:11.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:10 smithi156 ceph-mon[105979]: purged_snaps scrub ok 2023-10-09T05:38:11.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:10 smithi156 ceph-mon[105979]: osdmap e46: 7 total, 7 up, 7 in 2023-10-09T05:38:11.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:10 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:38:11.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:10 smithi156 ceph-mon[105979]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:38:11.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:10 smithi156 ceph-mon[105979]: osdmap e47: 7 total, 7 up, 7 in 2023-10-09T05:38:11.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:10 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:11.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:10 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:11.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:10 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:38:11.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:10 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:38:11.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:10 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:38:11.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:10 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:11.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:10 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:11.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:10 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:38:11.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:10 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:11.126 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:38:11.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[101070]: purged_snaps scrub starts 2023-10-09T05:38:11.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[101070]: purged_snaps scrub ok 2023-10-09T05:38:11.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[101070]: osdmap e46: 7 total, 7 up, 7 in 2023-10-09T05:38:11.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:38:11.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[101070]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:38:11.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[101070]: osdmap e47: 7 total, 7 up, 7 in 2023-10-09T05:38:11.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:11.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:11.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:38:11.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:38:11.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:38:11.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:11.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:11.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:38:11.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:11.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[118354]: purged_snaps scrub starts 2023-10-09T05:38:11.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[118354]: purged_snaps scrub ok 2023-10-09T05:38:11.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[118354]: osdmap e46: 7 total, 7 up, 7 in 2023-10-09T05:38:11.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:38:11.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[118354]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:38:11.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[118354]: osdmap e47: 7 total, 7 up, 7 in 2023-10-09T05:38:11.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:11.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:11.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:38:11.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:38:11.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:38:11.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:11.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:11.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:38:11.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:10 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:12.029 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:11 smithi156 ceph-mon[105979]: Detected new or changed devices on smithi156 2023-10-09T05:38:12.029 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:11 smithi156 ceph-mon[105979]: Adjusting osd_memory_target on smithi156 to 5722M 2023-10-09T05:38:12.029 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:11 smithi156 ceph-mon[105979]: osdmap e48: 7 total, 7 up, 7 in 2023-10-09T05:38:12.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:11 smithi064 ceph-mon[101070]: Detected new or changed devices on smithi156 2023-10-09T05:38:12.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:11 smithi064 ceph-mon[101070]: Adjusting osd_memory_target on smithi156 to 5722M 2023-10-09T05:38:12.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:11 smithi064 ceph-mon[101070]: osdmap e48: 7 total, 7 up, 7 in 2023-10-09T05:38:12.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:11 smithi064 ceph-mon[118354]: Detected new or changed devices on smithi156 2023-10-09T05:38:12.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:11 smithi064 ceph-mon[118354]: Adjusting osd_memory_target on smithi156 to 5722M 2023-10-09T05:38:12.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:11 smithi064 ceph-mon[118354]: osdmap e48: 7 total, 7 up, 7 in 2023-10-09T05:38:13.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:12 smithi156 ceph-mon[105979]: pgmap v146: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:38:13.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:12 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:38:13.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:12 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:38:13.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:12 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:13.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:12 smithi064 ceph-mon[101070]: pgmap v146: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:38:13.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:12 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:38:13.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:12 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:38:13.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:12 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:13.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:12 smithi064 ceph-mon[118354]: pgmap v146: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:38:13.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:12 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:38:13.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:12 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:38:13.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:12 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:13.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:13 smithi156 ceph-mon[105979]: from='client.24286 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:38:13.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:13 smithi156 ceph-mon[105979]: osdmap e49: 7 total, 7 up, 7 in 2023-10-09T05:38:14.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:13 smithi064 ceph-mon[101070]: from='client.24286 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:38:14.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:13 smithi064 ceph-mon[101070]: osdmap e49: 7 total, 7 up, 7 in 2023-10-09T05:38:14.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:13 smithi064 ceph-mon[118354]: from='client.24286 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:38:14.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:13 smithi064 ceph-mon[118354]: osdmap e49: 7 total, 7 up, 7 in 2023-10-09T05:38:15.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:14 smithi156 ceph-mon[105979]: osdmap e50: 7 total, 7 up, 7 in 2023-10-09T05:38:15.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:14 smithi156 ceph-mon[105979]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:38:15.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:14 smithi064 ceph-mon[101070]: osdmap e50: 7 total, 7 up, 7 in 2023-10-09T05:38:15.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:14 smithi064 ceph-mon[101070]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:38:15.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:14 smithi064 ceph-mon[118354]: osdmap e50: 7 total, 7 up, 7 in 2023-10-09T05:38:15.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:14 smithi064 ceph-mon[118354]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:38:16.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:15 smithi156 ceph-mon[105979]: from='client.? 172.21.15.156:0/566821969' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "693e93b0-c235-47fe-9172-85af68e38634"}]: dispatch 2023-10-09T05:38:16.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:15 smithi156 ceph-mon[105979]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "693e93b0-c235-47fe-9172-85af68e38634"}]: dispatch 2023-10-09T05:38:16.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:15 smithi156 ceph-mon[105979]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "693e93b0-c235-47fe-9172-85af68e38634"}]': finished 2023-10-09T05:38:16.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:15 smithi156 ceph-mon[105979]: osdmap e51: 8 total, 7 up, 8 in 2023-10-09T05:38:16.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:15 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:38:16.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:15 smithi156 ceph-mon[105979]: from='client.? 172.21.15.156:0/596949299' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:38:16.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:15 smithi064 ceph-mon[101070]: from='client.? 172.21.15.156:0/566821969' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "693e93b0-c235-47fe-9172-85af68e38634"}]: dispatch 2023-10-09T05:38:16.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:15 smithi064 ceph-mon[101070]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "693e93b0-c235-47fe-9172-85af68e38634"}]: dispatch 2023-10-09T05:38:16.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:15 smithi064 ceph-mon[101070]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "693e93b0-c235-47fe-9172-85af68e38634"}]': finished 2023-10-09T05:38:16.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:15 smithi064 ceph-mon[101070]: osdmap e51: 8 total, 7 up, 8 in 2023-10-09T05:38:16.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:15 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:38:16.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:15 smithi064 ceph-mon[101070]: from='client.? 172.21.15.156:0/596949299' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:38:16.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:15 smithi064 ceph-mon[118354]: from='client.? 172.21.15.156:0/566821969' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "693e93b0-c235-47fe-9172-85af68e38634"}]: dispatch 2023-10-09T05:38:16.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:15 smithi064 ceph-mon[118354]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "693e93b0-c235-47fe-9172-85af68e38634"}]: dispatch 2023-10-09T05:38:16.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:15 smithi064 ceph-mon[118354]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "693e93b0-c235-47fe-9172-85af68e38634"}]': finished 2023-10-09T05:38:16.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:15 smithi064 ceph-mon[118354]: osdmap e51: 8 total, 7 up, 8 in 2023-10-09T05:38:16.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:15 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:38:16.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:15 smithi064 ceph-mon[118354]: from='client.? 172.21.15.156:0/596949299' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:38:17.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:16 smithi156 ceph-mon[105979]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 130 KiB/s, 0 objects/s recovering 2023-10-09T05:38:17.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:16 smithi064 ceph-mon[101070]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 130 KiB/s, 0 objects/s recovering 2023-10-09T05:38:17.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:16 smithi064 ceph-mon[118354]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 130 KiB/s, 0 objects/s recovering 2023-10-09T05:38:19.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:18 smithi156 ceph-mon[105979]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T05:38:19.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:18 smithi064 ceph-mon[101070]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T05:38:19.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:18 smithi064 ceph-mon[118354]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T05:38:21.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:20 smithi156 ceph-mon[105979]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2023-10-09T05:38:21.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:20 smithi064 ceph-mon[101070]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2023-10-09T05:38:21.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:20 smithi064 ceph-mon[118354]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2023-10-09T05:38:22.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:21 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T05:38:22.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:21 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:22.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:21 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T05:38:22.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:21 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:22.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:21 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T05:38:22.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:21 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:23.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:22 smithi156 ceph-mon[105979]: Deploying daemon osd.7 on smithi156 2023-10-09T05:38:23.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:22 smithi156 ceph-mon[105979]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T05:38:23.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:22 smithi064 ceph-mon[101070]: Deploying daemon osd.7 on smithi156 2023-10-09T05:38:23.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:22 smithi064 ceph-mon[101070]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T05:38:23.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:22 smithi064 ceph-mon[118354]: Deploying daemon osd.7 on smithi156 2023-10-09T05:38:23.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:22 smithi064 ceph-mon[118354]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T05:38:25.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:24 smithi156 ceph-mon[105979]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T05:38:25.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:24 smithi064 ceph-mon[101070]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T05:38:25.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:24 smithi064 ceph-mon[118354]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T05:38:26.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:26 smithi156 ceph-mon[105979]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:38:26.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:26 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:26.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:26 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:26.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:26 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:38:26.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:26 smithi064 ceph-mon[101070]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:38:26.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:26 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:26.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:26 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:26.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:26 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:38:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:26 smithi064 ceph-mon[118354]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:38:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:26 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:26 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:26 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:38:28.819 INFO:teuthology.orchestra.run.smithi156.stdout:Created osd(s) 7 on host 'smithi156' 2023-10-09T05:38:29.098 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:28 smithi156 ceph-mon[105979]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:38:29.098 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:28 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:29.099 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:28 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:29.099 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:28 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:29.099 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:28 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:38:29.099 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:28 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:29.099 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:28 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:29.099 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:28 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:29.099 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:28 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:38:29.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:28 smithi064 ceph-mon[101070]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:38:29.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:28 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:29.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:28 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:29.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:28 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:29.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:28 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:38:29.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:28 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:29.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:28 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:29.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:28 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:29.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:28 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:38:29.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:28 smithi064 ceph-mon[118354]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:38:29.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:28 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:29.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:28 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:29.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:28 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:29.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:28 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:38:29.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:28 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:29.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:28 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:29.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:28 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:29.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:28 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:38:29.349 DEBUG:teuthology.orchestra.run.smithi156:osd.7> sudo journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.7.service 2023-10-09T05:38:29.351 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-10-09T05:38:29.352 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-09T05:38:29.381 INFO:journalctl@ceph.osd.7.smithi156.stdout:-- Logs begin at Mon 2023-10-09 05:16:12 UTC. -- 2023-10-09T05:38:29.718 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:29.960 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:29 smithi156 ceph-mon[105979]: from='osd.7 [v2:172.21.15.156:6824/2767546322,v1:172.21.15.156:6825/2767546322]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:38:29.960 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:29 smithi156 ceph-mon[105979]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:38:30.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:29 smithi064 ceph-mon[101070]: from='osd.7 [v2:172.21.15.156:6824/2767546322,v1:172.21.15.156:6825/2767546322]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:38:30.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:29 smithi064 ceph-mon[101070]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:38:30.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:29 smithi064 ceph-mon[118354]: from='osd.7 [v2:172.21.15.156:6824/2767546322,v1:172.21.15.156:6825/2767546322]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:38:30.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:29 smithi064 ceph-mon[118354]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:38:31.005 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:30 smithi064 ceph-mon[101070]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:38:31.005 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:30 smithi064 ceph-mon[101070]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T05:38:31.005 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:30 smithi064 ceph-mon[101070]: osdmap e52: 8 total, 7 up, 8 in 2023-10-09T05:38:31.006 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:30 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:38:31.006 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:30 smithi064 ceph-mon[101070]: from='osd.7 [v2:172.21.15.156:6824/2767546322,v1:172.21.15.156:6825/2767546322]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T05:38:31.006 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:30 smithi064 ceph-mon[101070]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T05:38:31.006 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:30 smithi064 ceph-mon[118354]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:38:31.006 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:30 smithi064 ceph-mon[118354]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T05:38:31.006 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:30 smithi064 ceph-mon[118354]: osdmap e52: 8 total, 7 up, 8 in 2023-10-09T05:38:31.006 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:30 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:38:31.006 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:30 smithi064 ceph-mon[118354]: from='osd.7 [v2:172.21.15.156:6824/2767546322,v1:172.21.15.156:6825/2767546322]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T05:38:31.007 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:30 smithi064 ceph-mon[118354]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T05:38:31.120 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 05:38:30 smithi156 conmon[141346]: 2023-10-09T05:38:30.868+0000 7f5ca286c700 -1 osd.7 0 waiting for initial osdmap 2023-10-09T05:38:31.120 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 05:38:30 smithi156 conmon[141346]: 2023-10-09T05:38:30.876+0000 7f5c9b663700 -1 osd.7 53 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:38:31.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:30 smithi156 ceph-mon[105979]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:38:31.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:30 smithi156 ceph-mon[105979]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T05:38:31.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:30 smithi156 ceph-mon[105979]: osdmap e52: 8 total, 7 up, 8 in 2023-10-09T05:38:31.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:30 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:38:31.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:30 smithi156 ceph-mon[105979]: from='osd.7 [v2:172.21.15.156:6824/2767546322,v1:172.21.15.156:6825/2767546322]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T05:38:31.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:30 smithi156 ceph-mon[105979]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T05:38:31.390 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:38:32.109 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":53,"num_osds":8,"num_up_osds":7,"osd_up_since":1696829889,"num_in_osds":8,"osd_in_since":1696829894,"num_remapped_pgs":0} 2023-10-09T05:38:32.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:31 smithi064 ceph-mon[118354]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2023-10-09T05:38:32.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:31 smithi064 ceph-mon[118354]: osdmap e53: 8 total, 7 up, 8 in 2023-10-09T05:38:32.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:31 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:38:32.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:31 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:38:32.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:31 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/1727933207' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T05:38:32.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:31 smithi064 ceph-mon[118354]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:38:32.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:31 smithi064 ceph-mon[118354]: osd.7 [v2:172.21.15.156:6824/2767546322,v1:172.21.15.156:6825/2767546322] boot 2023-10-09T05:38:32.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:31 smithi064 ceph-mon[118354]: osdmap e54: 8 total, 8 up, 8 in 2023-10-09T05:38:32.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:31 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:38:32.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:31 smithi064 ceph-mon[101070]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2023-10-09T05:38:32.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:31 smithi064 ceph-mon[101070]: osdmap e53: 8 total, 7 up, 8 in 2023-10-09T05:38:32.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:31 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:38:32.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:31 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:38:32.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:31 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1727933207' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T05:38:32.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:31 smithi064 ceph-mon[101070]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:38:32.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:31 smithi064 ceph-mon[101070]: osd.7 [v2:172.21.15.156:6824/2767546322,v1:172.21.15.156:6825/2767546322] boot 2023-10-09T05:38:32.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:31 smithi064 ceph-mon[101070]: osdmap e54: 8 total, 8 up, 8 in 2023-10-09T05:38:32.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:31 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:38:32.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:31 smithi156 ceph-mon[105979]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2023-10-09T05:38:32.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:31 smithi156 ceph-mon[105979]: osdmap e53: 8 total, 7 up, 8 in 2023-10-09T05:38:32.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:31 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:38:32.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:31 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:38:32.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:31 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/1727933207' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T05:38:32.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:31 smithi156 ceph-mon[105979]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:38:32.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:31 smithi156 ceph-mon[105979]: osd.7 [v2:172.21.15.156:6824/2767546322,v1:172.21.15.156:6825/2767546322] boot 2023-10-09T05:38:32.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:31 smithi156 ceph-mon[105979]: osdmap e54: 8 total, 8 up, 8 in 2023-10-09T05:38:32.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:31 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:38:33.110 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-09T05:38:33.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:33 smithi156 ceph-mon[105979]: purged_snaps scrub starts 2023-10-09T05:38:33.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:33 smithi156 ceph-mon[105979]: purged_snaps scrub ok 2023-10-09T05:38:33.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:33 smithi156 ceph-mon[105979]: Detected new or changed devices on smithi156 2023-10-09T05:38:33.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:33 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:33.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:33 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:33.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:33 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:38:33.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:33 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:38:33.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:33 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:38:33.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:33 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:38:33.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:33 smithi156 ceph-mon[105979]: Adjusting osd_memory_target on smithi156 to 4292M 2023-10-09T05:38:33.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:33 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:33.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:33 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:33.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:33 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:38:33.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:33 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:33.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:33 smithi156 ceph-mon[105979]: osdmap e55: 8 total, 8 up, 8 in 2023-10-09T05:38:33.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[101070]: purged_snaps scrub starts 2023-10-09T05:38:33.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[101070]: purged_snaps scrub ok 2023-10-09T05:38:33.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[101070]: Detected new or changed devices on smithi156 2023-10-09T05:38:33.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:33.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:33.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:38:33.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:38:33.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:38:33.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:38:33.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[101070]: Adjusting osd_memory_target on smithi156 to 4292M 2023-10-09T05:38:33.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:33.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:33.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:38:33.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:33.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[101070]: osdmap e55: 8 total, 8 up, 8 in 2023-10-09T05:38:33.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[118354]: purged_snaps scrub starts 2023-10-09T05:38:33.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[118354]: purged_snaps scrub ok 2023-10-09T05:38:33.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[118354]: Detected new or changed devices on smithi156 2023-10-09T05:38:33.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:33.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:33.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:38:33.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:38:33.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:38:33.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:38:33.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[118354]: Adjusting osd_memory_target on smithi156 to 4292M 2023-10-09T05:38:33.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:33.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:33.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:38:33.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:33.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:33 smithi064 ceph-mon[118354]: osdmap e55: 8 total, 8 up, 8 in 2023-10-09T05:38:33.477 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:34.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:34 smithi156 ceph-mon[105979]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:38:34.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:34 smithi156 ceph-mon[105979]: osdmap e56: 8 total, 8 up, 8 in 2023-10-09T05:38:34.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:34 smithi064 ceph-mon[101070]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:38:34.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:34 smithi064 ceph-mon[101070]: osdmap e56: 8 total, 8 up, 8 in 2023-10-09T05:38:34.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:34 smithi064 ceph-mon[118354]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:38:34.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:34 smithi064 ceph-mon[118354]: osdmap e56: 8 total, 8 up, 8 in 2023-10-09T05:38:35.192 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:38:35.824 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":57,"num_osds":8,"num_up_osds":8,"osd_up_since":1696829911,"num_in_osds":8,"osd_in_since":1696829894,"num_remapped_pgs":0} 2023-10-09T05:38:35.825 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:38:36.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:35 smithi156 ceph-mon[105979]: osdmap e57: 8 total, 8 up, 8 in 2023-10-09T05:38:36.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:35 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/753714661' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T05:38:36.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:35 smithi156 ceph-mon[105979]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:38:36.182 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:36.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:35 smithi064 ceph-mon[101070]: osdmap e57: 8 total, 8 up, 8 in 2023-10-09T05:38:36.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:35 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/753714661' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T05:38:36.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:35 smithi064 ceph-mon[101070]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:38:36.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:35 smithi064 ceph-mon[118354]: osdmap e57: 8 total, 8 up, 8 in 2023-10-09T05:38:36.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:35 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/753714661' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T05:38:36.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:35 smithi064 ceph-mon[118354]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:38:37.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:36 smithi156 ceph-mon[105979]: osdmap e58: 8 total, 8 up, 8 in 2023-10-09T05:38:37.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:36 smithi064 ceph-mon[101070]: osdmap e58: 8 total, 8 up, 8 in 2023-10-09T05:38:37.213 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:36 smithi064 ceph-mon[118354]: osdmap e58: 8 total, 8 up, 8 in 2023-10-09T05:38:37.867 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:38:37.867 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":58,"fsid":"44c77acc-6665-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:33:40.037198+0000","modified":"2023-10-09T05:38:35.861670+0000","last_up_change":"2023-10-09T05:38:31.856610+0000","last_in_change":"2023-10-09T05:38:14.961182+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:36:42.276784+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"85d0857a-ca32-43d2-8cec-bbd916db9f98","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2019939615},{"type":"v1","addr":"172.21.15.64:6803","nonce":2019939615}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2019939615},{"type":"v1","addr":"172.21.15.64:6805","nonce":2019939615}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2019939615},{"type":"v1","addr":"172.21.15.64:6809","nonce":2019939615}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2019939615},{"type":"v1","addr":"172.21.15.64:6807","nonce":2019939615}]},"public_addr":"172.21.15.64:6803/2019939615","cluster_addr":"172.21.15.64:6805/2019939615","heartbeat_back_addr":"172.21.15.64:6809/2019939615","heartbeat_front_addr":"172.21.15.64:6807/2019939615","state":["exists","up"]},{"osd":1,"uuid":"05570845-2cd8-4e24-a2f5-f46cf67aa31a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":1958566614},{"type":"v1","addr":"172.21.15.64:6811","nonce":1958566614}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":1958566614},{"type":"v1","addr":"172.21.15.64:6813","nonce":1958566614}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1958566614},{"type":"v1","addr":"172.21.15.64:6817","nonce":1958566614}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":1958566614},{"type":"v1","addr":"172.21.15.64:6815","nonce":1958566614}]},"public_addr":"172.21.15.64:6811/1958566614","cluster_addr":"172.21.15.64:6813/1958566614","heartbeat_back_addr":"172.21.15.64:6817/1958566614","heartbeat_front_addr":"172.21.15.64:6815/1958566614","state":["exists","up"]},{"osd":2,"uuid":"0f306460-9f08-4f15-b573-3587f63be61c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":2050077610},{"type":"v1","addr":"172.21.15.64:6819","nonce":2050077610}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":2050077610},{"type":"v1","addr":"172.21.15.64:6821","nonce":2050077610}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2050077610},{"type":"v1","addr":"172.21.15.64:6825","nonce":2050077610}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":2050077610},{"type":"v1","addr":"172.21.15.64:6823","nonce":2050077610}]},"public_addr":"172.21.15.64:6819/2050077610","cluster_addr":"172.21.15.64:6821/2050077610","heartbeat_back_addr":"172.21.15.64:6825/2050077610","heartbeat_front_addr":"172.21.15.64:6823/2050077610","state":["exists","up"]},{"osd":3,"uuid":"4bf38edd-e555-402e-ba4b-c751de06aa6a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":3037106123},{"type":"v1","addr":"172.21.15.64:6827","nonce":3037106123}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":3037106123},{"type":"v1","addr":"172.21.15.64:6829","nonce":3037106123}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":3037106123},{"type":"v1","addr":"172.21.15.64:6833","nonce":3037106123}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":3037106123},{"type":"v1","addr":"172.21.15.64:6831","nonce":3037106123}]},"public_addr":"172.21.15.64:6827/3037106123","cluster_addr":"172.21.15.64:6829/3037106123","heartbeat_back_addr":"172.21.15.64:6833/3037106123","heartbeat_front_addr":"172.21.15.64:6831/3037106123","state":["exists","up"]},{"osd":4,"uuid":"060e8766-7bf6-42f1-bebe-dae6828f7fb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6800","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6801","nonce":657648738}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6803","nonce":657648738}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6807","nonce":657648738}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6805","nonce":657648738}]},"public_addr":"172.21.15.156:6801/657648738","cluster_addr":"172.21.15.156:6803/657648738","heartbeat_back_addr":"172.21.15.156:6807/657648738","heartbeat_front_addr":"172.21.15.156:6805/657648738","state":["exists","up"]},{"osd":5,"uuid":"6b5cf38a-6792-4ac7-9694-268601260ba8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6809","nonce":726279450}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6811","nonce":726279450}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6815","nonce":726279450}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6813","nonce":726279450}]},"public_addr":"172.21.15.156:6809/726279450","cluster_addr":"172.21.15.156:6811/726279450","heartbeat_back_addr":"172.21.15.156:6815/726279450","heartbeat_front_addr":"172.21.15.156:6813/726279450","state":["exists","up"]},{"osd":6,"uuid":"01deea2d-a225-40b8-a54b-61eee57da527","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6817","nonce":2369965428}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6819","nonce":2369965428}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6823","nonce":2369965428}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6821","nonce":2369965428}]},"public_addr":"172.21.15.156:6817/2369965428","cluster_addr":"172.21.15.156:6819/2369965428","heartbeat_back_addr":"172.21.15.156:6823/2369965428","heartbeat_front_addr":"172.21.15.156:6821/2369965428","state":["exists","up"]},{"osd":7,"uuid":"693e93b0-c235-47fe-9172-85af68e38634","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6825","nonce":2767546322}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6827","nonce":2767546322}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6831","nonce":2767546322}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6829","nonce":2767546322}]},"public_addr":"172.21.15.156:6825/2767546322","cluster_addr":"172.21.15.156:6827/2767546322","heartbeat_back_addr":"172.21.15.156:6831/2767546322","heartbeat_front_addr":"172.21.15.156:6829/2767546322","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:35:52.928919+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:36:16.479793+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:36:39.503123+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:03.340353+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:24.645080+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:46.312540+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:38:08.346971+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:38:30.073761+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:0/1900245998":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6801/1833729701":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6800/1833729701":"2023-10-10T05:34:35.744792+0000","172.21.15.64:0/1562509700":"2023-10-10T05:34:35.744792+0000","172.21.15.64:0/3346006057":"2023-10-10T05:34:09.281655+0000","172.21.15.64:0/2171400226":"2023-10-10T05:34:09.281655+0000","172.21.15.64:6800/3793931367":"2023-10-10T05:34:09.281655+0000","172.21.15.64:0/4054596417":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6801/3793931367":"2023-10-10T05:34:09.281655+0000","172.21.15.64:0/514741379":"2023-10-10T05:34:09.281655+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:38:38.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:37 smithi064 ceph-mon[101070]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 117 KiB/s, 0 objects/s recovering 2023-10-09T05:38:38.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:37 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1830549159' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:38:38.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:37 smithi064 ceph-mon[118354]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 117 KiB/s, 0 objects/s recovering 2023-10-09T05:38:38.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:37 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/1830549159' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:38:38.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:37 smithi156 ceph-mon[105979]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 117 KiB/s, 0 objects/s recovering 2023-10-09T05:38:38.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:37 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/1830549159' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:38:38.564 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-10-09T05:36:42.276784+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2023-10-09T05:38:38.565 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool get .mgr pg_num 2023-10-09T05:38:38.914 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:38:40.633 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2023-10-09T05:38:40.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:40 smithi064 ceph-mon[101070]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T05:38:40.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:40 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/4179591896' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-10-09T05:38:40.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:40 smithi064 ceph-mon[118354]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T05:38:40.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:40 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/4179591896' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-10-09T05:38:41.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:40 smithi156 ceph-mon[105979]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T05:38:41.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:40 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/4179591896' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-10-09T05:38:41.347 INFO:tasks.cephadm:Adding prometheus.a on smithi156 2023-10-09T05:38:41.347 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph orch apply prometheus '1;smithi156=a' 2023-10-09T05:38:41.703 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:38:42.931 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:42 smithi156 ceph-mon[105979]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2023-10-09T05:38:43.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:42 smithi064 ceph-mon[101070]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2023-10-09T05:38:43.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:42 smithi064 ceph-mon[118354]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2023-10-09T05:38:43.269 INFO:teuthology.orchestra.run.smithi156.stdout:Scheduled prometheus update... 2023-10-09T05:38:43.864 DEBUG:teuthology.orchestra.run.smithi156:prometheus.a> sudo journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@prometheus.a.service 2023-10-09T05:38:43.866 INFO:tasks.cephadm:Adding node-exporter.a on smithi064 2023-10-09T05:38:43.866 INFO:tasks.cephadm:Adding node-exporter.b on smithi156 2023-10-09T05:38:43.866 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi064=a;smithi156=b' 2023-10-09T05:38:43.897 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:-- Logs begin at Mon 2023-10-09 05:16:12 UTC. -- 2023-10-09T05:38:44.237 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:38:44.582 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:44 smithi156 ceph-mon[105979]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi156=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:38:44.582 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:44 smithi156 ceph-mon[105979]: Saving service prometheus spec with placement smithi156=a;count:1 2023-10-09T05:38:44.582 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:44 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:44.583 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:44 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:38:44.583 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:44 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:44.583 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:44 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:38:44.583 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:44 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:44.583 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:44 smithi156 ceph-mon[105979]: Deploying daemon prometheus.a on smithi156 2023-10-09T05:38:44.583 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:44 smithi156 ceph-mon[105979]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2023-10-09T05:38:44.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:44 smithi064 ceph-mon[101070]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi156=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:38:44.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:44 smithi064 ceph-mon[101070]: Saving service prometheus spec with placement smithi156=a;count:1 2023-10-09T05:38:44.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:44 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:44.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:44 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:38:44.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:44 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:44.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:44 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:38:44.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:44 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:44.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:44 smithi064 ceph-mon[101070]: Deploying daemon prometheus.a on smithi156 2023-10-09T05:38:44.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:44 smithi064 ceph-mon[101070]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2023-10-09T05:38:44.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:44 smithi064 ceph-mon[118354]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi156=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:38:44.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:44 smithi064 ceph-mon[118354]: Saving service prometheus spec with placement smithi156=a;count:1 2023-10-09T05:38:44.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:44 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:44.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:44 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:38:44.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:44 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:38:44.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:44 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:38:44.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:44 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:44.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:44 smithi064 ceph-mon[118354]: Deploying daemon prometheus.a on smithi156 2023-10-09T05:38:44.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:44 smithi064 ceph-mon[118354]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2023-10-09T05:38:46.515 INFO:teuthology.orchestra.run.smithi156.stdout:Scheduled node-exporter update... 2023-10-09T05:38:46.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:46 smithi156 ceph-mon[105979]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T05:38:46.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:46 smithi064 ceph-mon[101070]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T05:38:46.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:46 smithi064 ceph-mon[118354]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T05:38:47.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:47 smithi156 ceph-mon[105979]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi064=a;smithi156=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:38:47.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:47 smithi156 ceph-mon[105979]: Saving service node-exporter spec with placement smithi064=a;smithi156=b;count:2 2023-10-09T05:38:47.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:47 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:47.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:47 smithi064 ceph-mon[101070]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi064=a;smithi156=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:38:47.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:47 smithi064 ceph-mon[101070]: Saving service node-exporter spec with placement smithi064=a;smithi156=b;count:2 2023-10-09T05:38:47.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:47 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:47.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:47 smithi064 ceph-mon[118354]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi064=a;smithi156=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:38:47.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:47 smithi064 ceph-mon[118354]: Saving service node-exporter spec with placement smithi064=a;smithi156=b;count:2 2023-10-09T05:38:47.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:47 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:48.007 DEBUG:teuthology.orchestra.run.smithi064:node-exporter.a> sudo journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@node-exporter.a.service 2023-10-09T05:38:48.009 DEBUG:teuthology.orchestra.run.smithi156:node-exporter.b> sudo journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@node-exporter.b.service 2023-10-09T05:38:48.011 INFO:tasks.cephadm:Adding alertmanager.a on smithi064 2023-10-09T05:38:48.012 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi064=a' 2023-10-09T05:38:48.044 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:-- Logs begin at Mon 2023-10-09 05:16:12 UTC. -- 2023-10-09T05:38:48.047 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:-- Logs begin at Mon 2023-10-09 05:16:16 UTC. -- 2023-10-09T05:38:48.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:48 smithi156 ceph-mon[105979]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-09T05:38:48.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:48 smithi064 ceph-mon[101070]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-09T05:38:48.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:48 smithi064 ceph-mon[118354]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-09T05:38:50.373 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:38:51.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:50 smithi156 ceph-mon[105979]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:38:51.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:50 smithi064 ceph-mon[101070]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:38:51.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:50 smithi064 ceph-mon[118354]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:38:53.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:52 smithi156 ceph-mon[105979]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:38:53.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:52 smithi064 ceph-mon[101070]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:38:53.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:52 smithi064 ceph-mon[118354]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:38:55.007 INFO:teuthology.orchestra.run.smithi156.stdout:Scheduled alertmanager update... 2023-10-09T05:38:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:54 smithi156 ceph-mon[105979]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:38:55.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:54 smithi064 ceph-mon[101070]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:38:55.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:54 smithi064 ceph-mon[118354]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:38:55.771 DEBUG:teuthology.orchestra.run.smithi064:alertmanager.a> sudo journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@alertmanager.a.service 2023-10-09T05:38:55.774 INFO:tasks.cephadm:Adding grafana.a on smithi156 2023-10-09T05:38:55.774 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph orch apply grafana '1;smithi156=a' 2023-10-09T05:38:55.808 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:-- Logs begin at Mon 2023-10-09 05:16:16 UTC. -- 2023-10-09T05:38:56.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:56 smithi064 ceph-mon[101070]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi064=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:38:56.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:56 smithi064 ceph-mon[101070]: Saving service alertmanager spec with placement smithi064=a;count:1 2023-10-09T05:38:56.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:56 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:56.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:56 smithi064 ceph-mon[101070]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:38:56.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:56 smithi064 ceph-mon[118354]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi064=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:38:56.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:56 smithi064 ceph-mon[118354]: Saving service alertmanager spec with placement smithi064=a;count:1 2023-10-09T05:38:56.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:56 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:56.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:56 smithi064 ceph-mon[118354]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:38:56.339 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:56 smithi156 ceph-mon[105979]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi064=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:38:56.339 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:56 smithi156 ceph-mon[105979]: Saving service alertmanager spec with placement smithi064=a;count:1 2023-10-09T05:38:56.339 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:56 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:56.339 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:56 smithi156 ceph-mon[105979]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:38:56.339 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:38:55 smithi156 systemd[1]: Starting Ceph prometheus.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:38:56.471 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:38:56.821 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:38:56 smithi156 conmon[149396]: ts=2023-10-09T05:38:56.461Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-10-09T05:38:56.821 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:38:56 smithi156 conmon[149396]: ts=2023-10-09T05:38:56.461Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-10-09T05:38:56.821 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:38:56 smithi156 conmon[149396]: ts=2023-10-09T05:38:56.461Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi156 (none))" 2023-10-09T05:38:56.821 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:38:56 smithi156 conmon[149396]: ts=2023-10-09T05:38:56.461Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T05:38:56.821 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:38:56 smithi156 conmon[149396]: ts=2023-10-09T05:38:56.461Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T05:38:56.821 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:38:56 smithi156 conmon[149396]: ts=2023-10-09T05:38:56.462Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T05:38:56.821 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:38:56 smithi156 conmon[149396]: ts=2023-10-09T05:38:56.462Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-10-09T05:38:56.821 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:38:56 smithi156 bash[149277]: 7dc2d972520016764a5f34e34b2154879e137b132445c7d6497e7c2d74d1856f 2023-10-09T05:38:56.822 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:38:56 smithi156 conmon[149396]: ts=2023-10-09T05:38:56.464Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-10-09T05:38:56.822 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:38:56 smithi156 conmon[149396]: ts=2023-10-09T05:38:56.465Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-10-09T05:38:56.822 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:38:56 smithi156 conmon[149396]: ts=2023-10-09T05:38:56.466Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T05:38:56.822 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:38:56 smithi156 conmon[149396]: ts=2023-10-09T05:38:56.466Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.709µs 2023-10-09T05:38:56.822 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:38:56 smithi156 conmon[149396]: ts=2023-10-09T05:38:56.466Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T05:38:56.822 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:38:56 smithi156 systemd[1]: Started Ceph prometheus.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:38:56.822 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:38:56 smithi156 conmon[149396]: ts=2023-10-09T05:38:56.466Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2023-10-09T05:38:56.822 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:38:56 smithi156 conmon[149396]: ts=2023-10-09T05:38:56.466Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=26.858µs wal_replay_duration=418.629µs wbl_replay_duration=141ns total_replay_duration=463.741µs 2023-10-09T05:38:56.822 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:38:56 smithi156 conmon[149396]: ts=2023-10-09T05:38:56.467Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T05:38:56.822 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:38:56 smithi156 conmon[149396]: ts=2023-10-09T05:38:56.467Z caller=main.go:1029 level=info msg="TSDB started" 2023-10-09T05:38:56.822 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:38:56 smithi156 conmon[149396]: ts=2023-10-09T05:38:56.467Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T05:38:56.823 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:38:56 smithi156 conmon[149396]: ts=2023-10-09T05:38:56.484Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.352319ms db_storage=820ns remote_storage=1.496µs web_handler=331ns query_engine=636ns scrape=409.954µs scrape_sd=19.3µs notify=693ns notify_sd=1.32µs rules=15.743718ms tracing=8.22µs 2023-10-09T05:38:56.823 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:38:56 smithi156 conmon[149396]: ts=2023-10-09T05:38:56.484Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-10-09T05:38:56.823 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:38:56 smithi156 conmon[149396]: ts=2023-10-09T05:38:56.484Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-10-09T05:38:57.825 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:57 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:57.825 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:57 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:57.825 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:57 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:57.825 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:57 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T05:38:57.825 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:38:57 smithi156 conmon[109199]: ignoring --setuser ceph since I am not root 2023-10-09T05:38:57.826 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:38:57 smithi156 conmon[109199]: ignoring --setgroup ceph since I am not root 2023-10-09T05:38:57.826 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:38:57 smithi156 ceph-mgr[109223]: -- 172.21.15.156:0/3866367247 <== mon.2 v2:172.21.15.64:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55861d9849c0 con 0x55861d95cc00 2023-10-09T05:38:57.826 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:38:57 smithi156 conmon[109199]: 2023-10-09T05:38:57.742+0000 7f73adae01c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:38:57.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:57 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:57.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:57 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:57.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:57 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:57.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:57 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T05:38:57.856 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:57 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:57.856 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:57 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:57.856 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:57 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' 2023-10-09T05:38:57.856 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:57 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T05:38:57.856 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:38:57 smithi064 conmon[101498]: ignoring --setuser ceph since I am not root 2023-10-09T05:38:57.856 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:38:57 smithi064 conmon[101498]: ignoring --setgroup ceph since I am not root 2023-10-09T05:38:57.857 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:38:57 smithi064 ceph-mgr[101522]: -- 172.21.15.64:0/1809991532 <== mon.0 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c9efa229c0 con 0x55c9ef9fa800 2023-10-09T05:38:57.857 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:38:57 smithi064 conmon[101498]: 2023-10-09T05:38:57.760+0000 7f82d52131c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:38:57.857 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:38:57 smithi064 conmon[101498]: 2023-10-09T05:38:57.853+0000 7f82d52131c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:38:58.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:38:57 smithi156 conmon[109199]: 2023-10-09T05:38:57.823+0000 7f73adae01c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:38:58.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:38:57 smithi156 conmon[109199]: 2023-10-09T05:38:57.915+0000 7f73adae01c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:38:58.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:38:58 smithi156 conmon[109199]: 2023-10-09T05:38:58.010+0000 7f73adae01c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:38:58.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:38:57 smithi064 conmon[101498]: 2023-10-09T05:38:57.960+0000 7f82d52131c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:38:58.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:38:58 smithi064 conmon[101498]: 2023-10-09T05:38:58.069+0000 7f82d52131c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:38:58.485 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:38:58 smithi156 conmon[109199]: 2023-10-09T05:38:58.178+0000 7f73adae01c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:38:58.571 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:38:58 smithi064 conmon[101498]: 2023-10-09T05:38:58.271+0000 7f82d52131c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:38:58.792 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:58 smithi156 ceph-mon[105979]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T05:38:58.792 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:38:58 smithi156 ceph-mon[105979]: mgrmap e16: y(active, since 4m), standbys: x 2023-10-09T05:38:58.793 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:38:58 smithi156 conmon[109199]: 2023-10-09T05:38:58.484+0000 7f73adae01c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:38:58.793 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:38:58 smithi156 conmon[109199]: 2023-10-09T05:38:58.571+0000 7f73adae01c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:38:58.793 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:38:58 smithi156 conmon[109199]: 2023-10-09T05:38:58.699+0000 7f73adae01c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:38:58.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:58 smithi064 ceph-mon[101070]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T05:38:58.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:38:58 smithi064 ceph-mon[101070]: mgrmap e16: y(active, since 4m), standbys: x 2023-10-09T05:38:58.923 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:58 smithi064 ceph-mon[118354]: from='mgr.14152 172.21.15.64:0/125586780' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T05:38:58.923 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:38:58 smithi064 ceph-mon[118354]: mgrmap e16: y(active, since 4m), standbys: x 2023-10-09T05:38:58.924 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:38:58 smithi064 conmon[101498]: 2023-10-09T05:38:58.665+0000 7f82d52131c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:38:58.924 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:38:58 smithi064 conmon[101498]: 2023-10-09T05:38:58.772+0000 7f82d52131c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:38:58.924 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:38:58 smithi064 conmon[101498]: 2023-10-09T05:38:58.921+0000 7f82d52131c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:38:59.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:38:58 smithi156 conmon[109199]: 2023-10-09T05:38:58.791+0000 7f73adae01c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:38:59.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:38:59 smithi064 conmon[101498]: 2023-10-09T05:38:59.034+0000 7f82d52131c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:38:59.620 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:38:59 smithi156 conmon[109199]: 2023-10-09T05:38:59.209+0000 7f73adae01c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:38:59.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:38:59 smithi064 conmon[101498]: 2023-10-09T05:38:59.541+0000 7f82d52131c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:39:00.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:38:59 smithi156 conmon[109199]: 2023-10-09T05:38:59.767+0000 7f73adae01c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:39:00.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:38:59 smithi156 conmon[109199]: 2023-10-09T05:38:59.935+0000 7f73adae01c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:39:00.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:00 smithi064 conmon[101498]: 2023-10-09T05:39:00.221+0000 7f82d52131c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:39:00.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:00 smithi064 conmon[101498]: 2023-10-09T05:39:00.411+0000 7f82d52131c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:39:00.961 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:39:00 smithi156 conmon[109199]: 2023-10-09T05:39:00.703+0000 7f73adae01c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:39:00.962 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:39:00 smithi156 conmon[109199]: 2023-10-09T05:39:00.792+0000 7f73adae01c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:39:01.231 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:39:00 smithi156 conmon[109199]: 2023-10-09T05:39:00.961+0000 7f73adae01c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:39:01.604 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:39:01 smithi156 conmon[109199]: 2023-10-09T05:39:01.231+0000 7f73adae01c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:39:01.604 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:39:01 smithi156 conmon[109199]: 2023-10-09T05:39:01.316+0000 7f73adae01c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:39:01.717 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:01 smithi064 conmon[101498]: 2023-10-09T05:39:01.387+0000 7f82d52131c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:39:01.717 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:01 smithi064 conmon[101498]: 2023-10-09T05:39:01.497+0000 7f82d52131c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:39:01.858 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:39:01 smithi156 conmon[109199]: 2023-10-09T05:39:01.603+0000 7f73adae01c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:39:02.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:39:01 smithi156 conmon[109199]: 2023-10-09T05:39:01.857+0000 7f73adae01c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:39:02.471 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:01 smithi064 conmon[101498]: 2023-10-09T05:39:01.716+0000 7f82d52131c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:39:02.471 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:02 smithi064 conmon[101498]: 2023-10-09T05:39:02.064+0000 7f82d52131c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:39:02.472 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:02 smithi064 conmon[101498]: 2023-10-09T05:39:02.169+0000 7f82d52131c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:39:02.826 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:02 smithi064 conmon[101498]: 2023-10-09T05:39:02.499+0000 7f82d52131c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:39:02.870 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:39:02 smithi156 conmon[109199]: 2023-10-09T05:39:02.562+0000 7f73adae01c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:39:02.870 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:39:02 smithi156 conmon[109199]: 2023-10-09T05:39:02.691+0000 7f73adae01c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:39:03.200 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:39:02 smithi156 conmon[109199]: 2023-10-09T05:39:02.876+0000 7f73adae01c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:39:03.200 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:39:02 smithi156 conmon[109199]: 2023-10-09T05:39:02.988+0000 7f73adae01c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:39:03.620 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:39:03 smithi156 conmon[109199]: 2023-10-09T05:39:03.199+0000 7f73adae01c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:39:03.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:02 smithi064 conmon[101498]: 2023-10-09T05:39:02.824+0000 7f82d52131c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:39:03.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:03 smithi064 conmon[101498]: 2023-10-09T05:39:03.657+0000 7f82d52131c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:39:03.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:03 smithi064 conmon[101498]: 2023-10-09T05:39:03.816+0000 7f82d52131c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:39:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:04 smithi156 ceph-mon[105979]: Standby manager daemon x restarted 2023-10-09T05:39:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:04 smithi156 ceph-mon[105979]: Standby manager daemon x started 2023-10-09T05:39:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:04 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.156:0/128159220' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:39:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:04 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.156:0/128159220' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:39:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:04 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.156:0/128159220' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:39:04.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:04 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.156:0/128159220' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:39:04.371 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:39:04 smithi156 conmon[109199]: 2023-10-09T05:39:04.046+0000 7f73adae01c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:39:04.371 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:39:04 smithi156 conmon[109199]: [09/Oct/2023:05:39:04] ENGINE Bus STARTING 2023-10-09T05:39:04.371 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:39:04 smithi156 conmon[109199]: CherryPy Checker: 2023-10-09T05:39:04.371 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:39:04 smithi156 conmon[109199]: The Application mounted at '' has an empty config. 2023-10-09T05:39:04.371 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:39:04 smithi156 conmon[109199]: 2023-10-09T05:39:04.371 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:39:04 smithi156 conmon[109199]: [09/Oct/2023:05:39:04] ENGINE Serving on http://:::9283 2023-10-09T05:39:04.371 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:39:04 smithi156 conmon[109199]: [09/Oct/2023:05:39:04] ENGINE Bus STARTED 2023-10-09T05:39:04.468 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:04 smithi064 ceph-mon[101070]: Standby manager daemon x restarted 2023-10-09T05:39:04.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:04 smithi064 ceph-mon[101070]: Standby manager daemon x started 2023-10-09T05:39:04.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:04 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.156:0/128159220' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:39:04.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:04 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.156:0/128159220' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:39:04.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:04 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.156:0/128159220' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:39:04.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:04 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.156:0/128159220' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:39:04.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:04 smithi064 ceph-mon[118354]: Standby manager daemon x restarted 2023-10-09T05:39:04.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:04 smithi064 ceph-mon[118354]: Standby manager daemon x started 2023-10-09T05:39:04.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:04 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.156:0/128159220' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:39:04.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:04 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.156:0/128159220' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:39:04.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:04 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.156:0/128159220' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:39:04.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:04 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.156:0/128159220' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:39:04.471 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:04 smithi064 conmon[101498]: 2023-10-09T05:39:04.057+0000 7f82d52131c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:39:04.471 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:04 smithi064 conmon[101498]: 2023-10-09T05:39:04.198+0000 7f82d52131c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:39:04.471 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:04 smithi064 conmon[101498]: 2023-10-09T05:39:04.468+0000 7f82d52131c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:39:05.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:05 smithi064 ceph-mon[101070]: mgrmap e17: y(active, since 4m), standbys: x 2023-10-09T05:39:05.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:05 smithi064 ceph-mon[118354]: mgrmap e17: y(active, since 4m), standbys: x 2023-10-09T05:39:05.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:05 smithi156 ceph-mon[105979]: mgrmap e17: y(active, since 4m), standbys: x 2023-10-09T05:39:05.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:05 smithi064 conmon[101498]: 2023-10-09T05:39:05.510+0000 7f82d52131c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:39:05.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:05 smithi064 conmon[101498]: [09/Oct/2023:05:39:05] ENGINE Bus STARTING 2023-10-09T05:39:05.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:05 smithi064 conmon[101498]: CherryPy Checker: 2023-10-09T05:39:05.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:05 smithi064 conmon[101498]: The Application mounted at '' has an empty config. 2023-10-09T05:39:05.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:05 smithi064 conmon[101498]: 2023-10-09T05:39:05.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:05 smithi064 conmon[101498]: [09/Oct/2023:05:39:05] ENGINE Serving on http://:::9283 2023-10-09T05:39:05.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:05 smithi064 conmon[101498]: [09/Oct/2023:05:39:05] ENGINE Bus STARTED 2023-10-09T05:39:06.548 INFO:teuthology.orchestra.run.smithi156.stdout:Scheduled grafana update... 2023-10-09T05:39:06.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: Active manager daemon y restarted 2023-10-09T05:39:06.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: Activating manager daemon y 2023-10-09T05:39:06.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: osdmap e59: 8 total, 8 up, 8 in 2023-10-09T05:39:06.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: mgrmap e18: y(active, starting, since 0.00689971s), standbys: x 2023-10-09T05:39:06.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:39:06.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:39:06.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:39:06.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:39:06.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:39:06.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:39:06.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:39:06.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:39:06.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:39:06.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:39:06.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:39:06.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:39:06.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:39:06.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:39:06.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:39:06.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:39:06.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: Manager daemon y is now available 2023-10-09T05:39:06.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:39:06.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:39:06.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:06.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:06.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:39:06.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:39:06.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: Active manager daemon y restarted 2023-10-09T05:39:06.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: Activating manager daemon y 2023-10-09T05:39:06.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: osdmap e59: 8 total, 8 up, 8 in 2023-10-09T05:39:06.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: mgrmap e18: y(active, starting, since 0.00689971s), standbys: x 2023-10-09T05:39:06.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:39:06.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:39:06.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:39:06.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:39:06.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:39:06.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:39:06.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:39:06.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:39:06.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:39:06.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:39:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:39:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:39:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:39:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:39:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:39:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:39:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: Manager daemon y is now available 2023-10-09T05:39:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:39:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:39:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:39:06.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:39:06.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: Active manager daemon y restarted 2023-10-09T05:39:06.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: Activating manager daemon y 2023-10-09T05:39:06.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: osdmap e59: 8 total, 8 up, 8 in 2023-10-09T05:39:06.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: mgrmap e18: y(active, starting, since 0.00689971s), standbys: x 2023-10-09T05:39:06.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:39:06.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:39:06.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:39:06.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:39:06.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:39:06.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:39:06.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:39:06.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:39:06.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:39:06.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:39:06.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:39:06.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:39:06.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:39:06.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:39:06.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:39:06.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:39:06.724 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: Manager daemon y is now available 2023-10-09T05:39:06.724 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:39:06.724 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:39:06.724 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:06.724 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:06.724 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:39:06.724 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:39:07.332 DEBUG:teuthology.orchestra.run.smithi156:grafana.a> sudo journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@grafana.a.service 2023-10-09T05:39:07.334 INFO:tasks.cephadm:Setting up client nodes... 2023-10-09T05:39:07.335 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T05:39:07.367 INFO:journalctl@ceph.grafana.a.smithi156.stdout:-- Logs begin at Mon 2023-10-09 05:16:12 UTC. -- 2023-10-09T05:39:07.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:07 smithi156 ceph-mon[105979]: mgrmap e19: y(active, since 1.00993s), standbys: x 2023-10-09T05:39:07.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:07 smithi156 ceph-mon[105979]: Saving service grafana spec with placement smithi156=a;count:1 2023-10-09T05:39:07.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:07 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:07.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:07 smithi156 ceph-mon[105979]: [09/Oct/2023:05:39:06] ENGINE Bus STARTING 2023-10-09T05:39:07.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:07 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:07.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:07 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:07.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:07 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:07.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:07 smithi064 ceph-mon[101070]: mgrmap e19: y(active, since 1.00993s), standbys: x 2023-10-09T05:39:07.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:07 smithi064 ceph-mon[101070]: Saving service grafana spec with placement smithi156=a;count:1 2023-10-09T05:39:07.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:07 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:07.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:07 smithi064 ceph-mon[101070]: [09/Oct/2023:05:39:06] ENGINE Bus STARTING 2023-10-09T05:39:07.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:07 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:07.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:07 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:07.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:07 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:07.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:07 smithi064 ceph-mon[118354]: mgrmap e19: y(active, since 1.00993s), standbys: x 2023-10-09T05:39:07.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:07 smithi064 ceph-mon[118354]: Saving service grafana spec with placement smithi156=a;count:1 2023-10-09T05:39:07.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:07 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:07.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:07 smithi064 ceph-mon[118354]: [09/Oct/2023:05:39:06] ENGINE Bus STARTING 2023-10-09T05:39:07.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:07 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:07.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:07 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:07.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:07 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:07.724 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:08.545 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:08 smithi064 ceph-mon[101070]: [09/Oct/2023:05:39:06] ENGINE Serving on https://172.21.15.64:7150 2023-10-09T05:39:08.545 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:08 smithi064 ceph-mon[101070]: [09/Oct/2023:05:39:06] ENGINE Bus STARTED 2023-10-09T05:39:08.545 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:08 smithi064 ceph-mon[101070]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:08.545 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:08 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:08.545 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:08 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:08.546 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:08 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:08.546 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:08 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:08.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:08 smithi156 ceph-mon[105979]: [09/Oct/2023:05:39:06] ENGINE Serving on https://172.21.15.64:7150 2023-10-09T05:39:08.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:08 smithi156 ceph-mon[105979]: [09/Oct/2023:05:39:06] ENGINE Bus STARTED 2023-10-09T05:39:08.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:08 smithi156 ceph-mon[105979]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:08.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:08 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:08.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:08 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:08.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:08 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:08.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:08 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:08.955 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:08 smithi064 ceph-mon[118354]: [09/Oct/2023:05:39:06] ENGINE Serving on https://172.21.15.64:7150 2023-10-09T05:39:08.955 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:08 smithi064 ceph-mon[118354]: [09/Oct/2023:05:39:06] ENGINE Bus STARTED 2023-10-09T05:39:08.955 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:08 smithi064 ceph-mon[118354]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:08.956 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:08 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:08.956 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:08 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:08.956 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:08 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:08.956 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:08 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:09.482 INFO:teuthology.orchestra.run.smithi064.stdout:[client.0] 2023-10-09T05:39:09.483 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQD9kSNlbTmXHBAAHAzya+EnL8E8aBMqfO3m5w== 2023-10-09T05:39:09.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:09 smithi156 ceph-mon[105979]: mgrmap e20: y(active, since 3s), standbys: x 2023-10-09T05:39:09.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:09 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:09.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:09 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:09.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:09 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:39:09.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:09 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:39:09.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:09 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:39:09.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:09 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:39:09.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:09 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:09.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:09 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:39:09.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:09 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:39:09.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:09 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/1320617044' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:39:09.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:09 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/1320617044' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T05:39:09.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:09 smithi064 ceph-mon[101070]: mgrmap e20: y(active, since 3s), standbys: x 2023-10-09T05:39:09.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:09 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:09.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:09 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:09.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:09 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:39:09.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:09 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:39:09.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:09 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:39:09.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:09 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:39:09.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:09 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:09.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:09 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:39:09.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:09 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:39:09.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:09 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1320617044' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:39:09.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:09 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1320617044' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T05:39:09.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:09 smithi064 ceph-mon[118354]: mgrmap e20: y(active, since 3s), standbys: x 2023-10-09T05:39:09.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:09 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:09.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:09 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:09.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:09 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:39:09.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:09 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:39:09.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:09 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:39:09.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:09 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:39:09.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:09 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:09.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:09 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:39:09.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:09 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:39:09.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:09 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/1320617044' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:39:09.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:09 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/1320617044' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T05:39:10.361 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-10-09T05:39:10.362 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-10-09T05:39:10.362 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-10-09T05:39:10.408 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T05:39:10.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[101070]: Adjusting osd_memory_target on smithi156 to 4036M 2023-10-09T05:39:10.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[101070]: Updating smithi064:/etc/ceph/ceph.conf 2023-10-09T05:39:10.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[101070]: Updating smithi156:/etc/ceph/ceph.conf 2023-10-09T05:39:10.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[101070]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:39:10.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[101070]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:39:10.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[101070]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:39:10.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[101070]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:39:10.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[101070]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:10.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:10.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:10.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:10.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:10.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:10.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[118354]: Adjusting osd_memory_target on smithi156 to 4036M 2023-10-09T05:39:10.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[118354]: Updating smithi064:/etc/ceph/ceph.conf 2023-10-09T05:39:10.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[118354]: Updating smithi156:/etc/ceph/ceph.conf 2023-10-09T05:39:10.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[118354]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:39:10.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[118354]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:39:10.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[118354]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:39:10.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[118354]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:39:10.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[118354]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:10.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:10.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:10.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:10.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:10.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:10 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:10.747 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:39:10.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:10 smithi156 ceph-mon[105979]: Adjusting osd_memory_target on smithi156 to 4036M 2023-10-09T05:39:10.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:10 smithi156 ceph-mon[105979]: Updating smithi064:/etc/ceph/ceph.conf 2023-10-09T05:39:10.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:10 smithi156 ceph-mon[105979]: Updating smithi156:/etc/ceph/ceph.conf 2023-10-09T05:39:10.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:10 smithi156 ceph-mon[105979]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:39:10.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:10 smithi156 ceph-mon[105979]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:39:10.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:10 smithi156 ceph-mon[105979]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:39:10.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:10 smithi156 ceph-mon[105979]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:39:10.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:10 smithi156 ceph-mon[105979]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:10.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:10 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:10.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:10 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:10.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:10 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:10.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:10 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:10.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:10 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:11.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:11 smithi156 ceph-mon[105979]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:39:11.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:11 smithi156 ceph-mon[105979]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:39:11.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:11 smithi156 ceph-mon[105979]: Deploying daemon alertmanager.a on smithi064 2023-10-09T05:39:11.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:11 smithi064 ceph-mon[101070]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:39:11.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:11 smithi064 ceph-mon[101070]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:39:11.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:11 smithi064 ceph-mon[101070]: Deploying daemon alertmanager.a on smithi064 2023-10-09T05:39:11.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:11 smithi064 ceph-mon[118354]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:39:11.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:11 smithi064 ceph-mon[118354]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:39:11.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:11 smithi064 ceph-mon[118354]: Deploying daemon alertmanager.a on smithi064 2023-10-09T05:39:12.439 INFO:teuthology.orchestra.run.smithi156.stdout:[client.1] 2023-10-09T05:39:12.440 INFO:teuthology.orchestra.run.smithi156.stdout: key = AQAAkiNlBfX2GRAAQzrkjD446kJcfJSF0N6b7g== 2023-10-09T05:39:12.768 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:12 smithi156 ceph-mon[105979]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:12.768 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:12 smithi156 ceph-mon[105979]: from='client.? 172.21.15.156:0/755244628' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:39:12.768 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:12 smithi156 ceph-mon[105979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:39:12.769 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:12 smithi156 ceph-mon[105979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T05:39:12.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:12 smithi064 ceph-mon[101070]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:12.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:12 smithi064 ceph-mon[101070]: from='client.? 172.21.15.156:0/755244628' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:39:12.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:12 smithi064 ceph-mon[101070]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:39:12.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:12 smithi064 ceph-mon[101070]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T05:39:12.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:12 smithi064 ceph-mon[118354]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:12.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:12 smithi064 ceph-mon[118354]: from='client.? 172.21.15.156:0/755244628' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:39:12.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:12 smithi064 ceph-mon[118354]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:39:12.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:12 smithi064 ceph-mon[118354]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T05:39:13.011 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2023-10-09T05:39:13.011 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-10-09T05:39:13.011 DEBUG:teuthology.orchestra.run.smithi156:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-10-09T05:39:13.039 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:39:12 smithi156 conmon[109199]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:39:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:39:13.055 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-10-09T05:39:13.056 DEBUG:teuthology.orchestra.run.smithi064:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-10-09T05:39:13.415 INFO:teuthology.misc.health.smithi064.stdout: 2023-10-09T05:39:13.415 INFO:teuthology.misc.health.smithi064.stdout:{"epoch":59,"fsid":"44c77acc-6665-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:33:40.037198+0000","modified":"2023-10-09T05:39:05.525376+0000","last_up_change":"2023-10-09T05:38:31.856610+0000","last_in_change":"2023-10-09T05:38:14.961182+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:36:42.276784+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"85d0857a-ca32-43d2-8cec-bbd916db9f98","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2019939615},{"type":"v1","addr":"172.21.15.64:6803","nonce":2019939615}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2019939615},{"type":"v1","addr":"172.21.15.64:6805","nonce":2019939615}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2019939615},{"type":"v1","addr":"172.21.15.64:6809","nonce":2019939615}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2019939615},{"type":"v1","addr":"172.21.15.64:6807","nonce":2019939615}]},"public_addr":"172.21.15.64:6803/2019939615","cluster_addr":"172.21.15.64:6805/2019939615","heartbeat_back_addr":"172.21.15.64:6809/2019939615","heartbeat_front_addr":"172.21.15.64:6807/2019939615","state":["exists","up"]},{"osd":1,"uuid":"05570845-2cd8-4e24-a2f5-f46cf67aa31a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":1958566614},{"type":"v1","addr":"172.21.15.64:6811","nonce":1958566614}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":1958566614},{"type":"v1","addr":"172.21.15.64:6813","nonce":1958566614}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1958566614},{"type":"v1","addr":"172.21.15.64:6817","nonce":1958566614}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":1958566614},{"type":"v1","addr":"172.21.15.64:6815","nonce":1958566614}]},"public_addr":"172.21.15.64:6811/1958566614","cluster_addr":"172.21.15.64:6813/1958566614","heartbeat_back_addr":"172.21.15.64:6817/1958566614","heartbeat_front_addr":"172.21.15.64:6815/1958566614","state":["exists","up"]},{"osd":2,"uuid":"0f306460-9f08-4f15-b573-3587f63be61c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":2050077610},{"type":"v1","addr":"172.21.15.64:6819","nonce":2050077610}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":2050077610},{"type":"v1","addr":"172.21.15.64:6821","nonce":2050077610}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2050077610},{"type":"v1","addr":"172.21.15.64:6825","nonce":2050077610}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":2050077610},{"type":"v1","addr":"172.21.15.64:6823","nonce":2050077610}]},"public_addr":"172.21.15.64:6819/2050077610","cluster_addr":"172.21.15.64:6821/2050077610","heartbeat_back_addr":"172.21.15.64:6825/2050077610","heartbeat_front_addr":"172.21.15.64:6823/2050077610","state":["exists","up"]},{"osd":3,"uuid":"4bf38edd-e555-402e-ba4b-c751de06aa6a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":3037106123},{"type":"v1","addr":"172.21.15.64:6827","nonce":3037106123}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":3037106123},{"type":"v1","addr":"172.21.15.64:6829","nonce":3037106123}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":3037106123},{"type":"v1","addr":"172.21.15.64:6833","nonce":3037106123}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":3037106123},{"type":"v1","addr":"172.21.15.64:6831","nonce":3037106123}]},"public_addr":"172.21.15.64:6827/3037106123","cluster_addr":"172.21.15.64:6829/3037106123","heartbeat_back_addr":"172.21.15.64:6833/3037106123","heartbeat_front_addr":"172.21.15.64:6831/3037106123","state":["exists","up"]},{"osd":4,"uuid":"060e8766-7bf6-42f1-bebe-dae6828f7fb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6800","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6801","nonce":657648738}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6803","nonce":657648738}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6807","nonce":657648738}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6805","nonce":657648738}]},"public_addr":"172.21.15.156:6801/657648738","cluster_addr":"172.21.15.156:6803/657648738","heartbeat_back_addr":"172.21.15.156:6807/657648738","heartbeat_front_addr":"172.21.15.156:6805/657648738","state":["exists","up"]},{"osd":5,"uuid":"6b5cf38a-6792-4ac7-9694-268601260ba8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6809","nonce":726279450}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6811","nonce":726279450}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6815","nonce":726279450}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6813","nonce":726279450}]},"public_addr":"172.21.15.156:6809/726279450","cluster_addr":"172.21.15.156:6811/726279450","heartbeat_back_addr":"172.21.15.156:6815/726279450","heartbeat_front_addr":"172.21.15.156:6813/726279450","state":["exists","up"]},{"osd":6,"uuid":"01deea2d-a225-40b8-a54b-61eee57da527","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6817","nonce":2369965428}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6819","nonce":2369965428}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6823","nonce":2369965428}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6821","nonce":2369965428}]},"public_addr":"172.21.15.156:6817/2369965428","cluster_addr":"172.21.15.156:6819/2369965428","heartbeat_back_addr":"172.21.15.156:6823/2369965428","heartbeat_front_addr":"172.21.15.156:6821/2369965428","state":["exists","up"]},{"osd":7,"uuid":"693e93b0-c235-47fe-9172-85af68e38634","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6825","nonce":2767546322}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6827","nonce":2767546322}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6831","nonce":2767546322}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6829","nonce":2767546322}]},"public_addr":"172.21.15.156:6825/2767546322","cluster_addr":"172.21.15.156:6827/2767546322","heartbeat_back_addr":"172.21.15.156:6831/2767546322","heartbeat_front_addr":"172.21.15.156:6829/2767546322","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:35:52.928919+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:36:16.479793+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:36:39.503123+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:03.340353+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:24.645080+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:46.312540+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:38:08.346971+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:38:30.073761+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:0/2115451128":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/3707401615":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/1900245998":"2023-10-10T05:34:35.744792+0000","172.21.15.64:0/1336186767":"2023-10-10T05:39:05.525329+0000","172.21.15.64:6801/1833729701":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6800/1833729701":"2023-10-10T05:34:35.744792+0000","172.21.15.64:0/1562509700":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6800/3627852432":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/3346006057":"2023-10-10T05:34:09.281655+0000","172.21.15.64:0/2171400226":"2023-10-10T05:34:09.281655+0000","172.21.15.64:6800/3793931367":"2023-10-10T05:34:09.281655+0000","172.21.15.64:0/3185620259":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/4054596417":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6801/3627852432":"2023-10-10T05:39:05.525329+0000","172.21.15.64:6801/3793931367":"2023-10-10T05:34:09.281655+0000","172.21.15.64:0/514741379":"2023-10-10T05:34:09.281655+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:39:13.428 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-10-09T05:39:13.428 INFO:tasks.cephadm:Creating RBD pool 2023-10-09T05:39:13.428 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-10-09T05:39:13.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:13 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/43440046' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:39:13.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:13 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/43440046' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:39:13.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:13 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/43440046' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:39:14.022 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:14.827 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:14 smithi064 ceph-mon[101070]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:14.828 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:14 smithi064 ceph-mon[118354]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:14.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:14 smithi156 ceph-mon[105979]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:16 smithi064 ceph-mon[101070]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:16.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:16 smithi064 ceph-mon[118354]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:16 smithi156 ceph-mon[105979]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:17.957 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'rbd' created 2023-10-09T05:39:18.057 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:17 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3991958978' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T05:39:18.058 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:17 smithi064 ceph-mon[101070]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:18.058 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:17 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3991958978' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T05:39:18.058 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:17 smithi064 ceph-mon[118354]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:18.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:17 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3991958978' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T05:39:18.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:17 smithi156 ceph-mon[105979]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:18.559 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-10-09T05:39:19.199 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:19.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:18 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3991958978' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T05:39:19.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:18 smithi064 ceph-mon[101070]: osdmap e60: 8 total, 8 up, 8 in 2023-10-09T05:39:19.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:18 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3991958978' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T05:39:19.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:18 smithi064 ceph-mon[118354]: osdmap e60: 8 total, 8 up, 8 in 2023-10-09T05:39:19.220 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:39:19 smithi064 systemd[1]: Started Ceph alertmanager.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:39:19.220 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:39:19 smithi064 conmon[163377]: ts=2023-10-09T05:39:19.146Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T05:39:19.220 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:39:19 smithi064 conmon[163377]: ts=2023-10-09T05:39:19.147Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T05:39:19.220 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:39:19 smithi064 conmon[163377]: ts=2023-10-09T05:39:19.151Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-10-09T05:39:19.220 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:39:19 smithi064 conmon[163377]: ts=2023-10-09T05:39:19.151Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-10-09T05:39:19.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:18 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3991958978' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T05:39:19.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:18 smithi156 ceph-mon[105979]: osdmap e60: 8 total, 8 up, 8 in 2023-10-09T05:39:20.116 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:19 smithi064 ceph-mon[101070]: osdmap e61: 8 total, 8 up, 8 in 2023-10-09T05:39:20.116 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:19 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:20.116 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:19 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:20.116 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:19 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:20.116 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:19 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:20.116 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:19 smithi064 ceph-mon[101070]: Deploying daemon node-exporter.a on smithi064 2023-10-09T05:39:20.116 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:19 smithi064 ceph-mon[101070]: pgmap v12: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:20.116 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:19 smithi064 ceph-mon[118354]: osdmap e61: 8 total, 8 up, 8 in 2023-10-09T05:39:20.117 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:19 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:20.117 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:19 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:20.117 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:19 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:20.117 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:19 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:20.117 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:19 smithi064 ceph-mon[118354]: Deploying daemon node-exporter.a on smithi064 2023-10-09T05:39:20.117 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:19 smithi064 ceph-mon[118354]: pgmap v12: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:19 smithi156 ceph-mon[105979]: osdmap e61: 8 total, 8 up, 8 in 2023-10-09T05:39:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:19 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:19 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:20.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:19 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:20.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:19 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:20.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:19 smithi156 ceph-mon[105979]: Deploying daemon node-exporter.a on smithi064 2023-10-09T05:39:20.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:19 smithi156 ceph-mon[105979]: pgmap v12: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:20.964 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:20 smithi064 bash[164032]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-10-09T05:39:21.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:20 smithi064 ceph-mon[101070]: osdmap e62: 8 total, 8 up, 8 in 2023-10-09T05:39:21.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:20 smithi064 ceph-mon[101070]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T05:39:21.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:20 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:21.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:20 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:21.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:20 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:21.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:20 smithi064 ceph-mon[118354]: osdmap e62: 8 total, 8 up, 8 in 2023-10-09T05:39:21.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:20 smithi064 ceph-mon[118354]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T05:39:21.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:20 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:21.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:20 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:21.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:20 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:21.221 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:21 smithi064 bash[164032]: Getting image source signatures 2023-10-09T05:39:21.221 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:39:21 smithi064 conmon[163377]: ts=2023-10-09T05:39:21.116Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000896793s 2023-10-09T05:39:21.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:20 smithi156 ceph-mon[105979]: osdmap e62: 8 total, 8 up, 8 in 2023-10-09T05:39:21.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:20 smithi156 ceph-mon[105979]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T05:39:21.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:20 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:21.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:20 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:21.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:20 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:21.719 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:21 smithi064 bash[164032]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-10-09T05:39:21.719 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:21 smithi064 bash[164032]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-10-09T05:39:21.719 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:21 smithi064 bash[164032]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-10-09T05:39:21.976 INFO:teuthology.orchestra.run.smithi064.stderr:enabled application 'rbd' on pool 'rbd' 2023-10-09T05:39:22.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:21 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/629764325' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:39:22.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:21 smithi064 ceph-mon[101070]: pgmap v14: 9 pgs: 3 active+clean, 6 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:22.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:21 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/629764325' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:39:22.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:21 smithi064 ceph-mon[118354]: pgmap v14: 9 pgs: 3 active+clean, 6 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:22.220 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:21 smithi064 bash[164032]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-10-09T05:39:22.220 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:21 smithi064 bash[164032]: Writing manifest to image destination 2023-10-09T05:39:22.220 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:21 smithi064 bash[164032]: Storing signatures 2023-10-09T05:39:22.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:21 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/629764325' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:39:22.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:21 smithi156 ceph-mon[105979]: pgmap v14: 9 pgs: 3 active+clean, 6 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:23.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:22 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/629764325' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:39:23.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:22 smithi156 ceph-mon[105979]: osdmap e63: 8 total, 8 up, 8 in 2023-10-09T05:39:23.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:39:22 smithi156 conmon[109199]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:39:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:39:23.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:22 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/629764325' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:39:23.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:22 smithi064 ceph-mon[118354]: osdmap e63: 8 total, 8 up, 8 in 2023-10-09T05:39:23.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:22 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/629764325' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:39:23.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:22 smithi064 ceph-mon[101070]: osdmap e63: 8 total, 8 up, 8 in 2023-10-09T05:39:23.674 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T05:39:23.675 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T05:39:23.675 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T05:39:24.173 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:24.359 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:23 smithi064 ceph-mon[101070]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:24.360 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:23 smithi064 ceph-mon[118354]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:24.360 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.077Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-10-09T05:39:24.360 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.077Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-10-09T05:39:24.360 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.077Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-10-09T05:39:24.361 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.077Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T05:39:24.361 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.077Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-10-09T05:39:24.361 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.077Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-10-09T05:39:24.361 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-10-09T05:39:24.361 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=arp 2023-10-09T05:39:24.361 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=bcache 2023-10-09T05:39:24.362 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=bonding 2023-10-09T05:39:24.362 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=btrfs 2023-10-09T05:39:24.362 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=conntrack 2023-10-09T05:39:24.362 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=cpu 2023-10-09T05:39:24.362 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-10-09T05:39:24.362 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=diskstats 2023-10-09T05:39:24.363 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=dmi 2023-10-09T05:39:24.363 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=edac 2023-10-09T05:39:24.363 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=entropy 2023-10-09T05:39:24.363 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-10-09T05:39:24.363 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=filefd 2023-10-09T05:39:24.363 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=filesystem 2023-10-09T05:39:24.363 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=hwmon 2023-10-09T05:39:24.363 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=infiniband 2023-10-09T05:39:24.364 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=ipvs 2023-10-09T05:39:24.364 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=loadavg 2023-10-09T05:39:24.364 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=mdadm 2023-10-09T05:39:24.364 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=meminfo 2023-10-09T05:39:24.364 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=netclass 2023-10-09T05:39:24.364 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=netdev 2023-10-09T05:39:24.364 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=netstat 2023-10-09T05:39:24.364 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=nfs 2023-10-09T05:39:24.365 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=nfsd 2023-10-09T05:39:24.365 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=nvme 2023-10-09T05:39:24.365 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=os 2023-10-09T05:39:24.365 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-10-09T05:39:24.365 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=pressure 2023-10-09T05:39:24.365 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=rapl 2023-10-09T05:39:24.365 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=schedstat 2023-10-09T05:39:24.366 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=selinux 2023-10-09T05:39:24.366 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=sockstat 2023-10-09T05:39:24.366 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=softnet 2023-10-09T05:39:24.366 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=stat 2023-10-09T05:39:24.366 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=tapestats 2023-10-09T05:39:24.366 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=textfile 2023-10-09T05:39:24.367 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-10-09T05:39:24.367 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=time 2023-10-09T05:39:24.367 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-10-09T05:39:24.367 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=uname 2023-10-09T05:39:24.367 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=vmstat 2023-10-09T05:39:24.367 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=xfs 2023-10-09T05:39:24.368 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.078Z caller=node_exporter.go:117 level=info collector=zfs 2023-10-09T05:39:24.368 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.079Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-10-09T05:39:24.368 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 conmon[164263]: ts=2023-10-09T05:39:24.079Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-10-09T05:39:24.368 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 bash[164032]: 59bb208a1e32d93cd1848aa8775c51bb0e3807a3a3ab0c70faa34a2da9095b41 2023-10-09T05:39:24.368 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:39:24 smithi064 systemd[1]: Started Ceph node-exporter.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:39:24.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:23 smithi156 ceph-mon[105979]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:24.977 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:24 smithi156 systemd[1]: Starting Ceph node-exporter.b for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:39:25.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:24 smithi064 ceph-mon[101070]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T05:39:25.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:24 smithi064 ceph-mon[101070]: Cluster is now healthy 2023-10-09T05:39:25.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:24 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:25.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:24 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:25.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:24 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:25.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:24 smithi064 ceph-mon[101070]: Deploying daemon node-exporter.b on smithi156 2023-10-09T05:39:25.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:24 smithi064 ceph-mon[118354]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T05:39:25.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:24 smithi064 ceph-mon[118354]: Cluster is now healthy 2023-10-09T05:39:25.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:24 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:25.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:24 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:25.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:24 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:25.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:24 smithi064 ceph-mon[118354]: Deploying daemon node-exporter.b on smithi156 2023-10-09T05:39:25.370 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:25 smithi156 bash[152427]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-10-09T05:39:25.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:24 smithi156 ceph-mon[105979]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T05:39:25.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:24 smithi156 ceph-mon[105979]: Cluster is now healthy 2023-10-09T05:39:25.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:24 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:25.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:24 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:25.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:24 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:25.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:24 smithi156 ceph-mon[105979]: Deploying daemon node-exporter.b on smithi156 2023-10-09T05:39:25.870 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:25 smithi156 bash[152427]: Getting image source signatures 2023-10-09T05:39:25.870 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:25 smithi156 bash[152427]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-10-09T05:39:25.870 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:25 smithi156 bash[152427]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-10-09T05:39:25.870 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:25 smithi156 bash[152427]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-10-09T05:39:25.953 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:39:26.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:26 smithi156 ceph-mon[105979]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:26.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:26 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/1212161367' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:39:26.621 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:26 smithi156 bash[152427]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-10-09T05:39:26.621 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:26 smithi156 bash[152427]: Writing manifest to image destination 2023-10-09T05:39:26.621 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:26 smithi156 bash[152427]: Storing signatures 2023-10-09T05:39:26.642 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:26 smithi064 ceph-mon[101070]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:26.642 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:26 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1212161367' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:39:26.643 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:26 smithi064 ceph-mon[118354]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:26.643 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:26 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/1212161367' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:39:26.658 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":20,"active_gid":24341,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":984813922},{"type":"v1","addr":"172.21.15.64:6801","nonce":984813922}]},"active_addr":"172.21.15.64:6801/984813922","active_change":"2023-10-09T05:39:05.525488+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24346,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":3038747921}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":1571829871}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":3029574061}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":1951412611}]}]} 2023-10-09T05:39:26.663 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T05:39:26.663 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T05:39:26.663 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:39:27.024 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:27.843 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:27 smithi064 conmon[101498]: [09/Oct/2023:05:39:27] ENGINE Bus STOPPING 2023-10-09T05:39:28.120 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.647Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-10-09T05:39:28.121 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.647Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-10-09T05:39:28.121 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.647Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-10-09T05:39:28.121 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.647Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T05:39:28.121 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.647Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-10-09T05:39:28.121 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-10-09T05:39:28.121 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-10-09T05:39:28.121 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=arp 2023-10-09T05:39:28.121 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=bcache 2023-10-09T05:39:28.121 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=bonding 2023-10-09T05:39:28.121 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=btrfs 2023-10-09T05:39:28.121 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=conntrack 2023-10-09T05:39:28.122 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=cpu 2023-10-09T05:39:28.122 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-10-09T05:39:28.122 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=diskstats 2023-10-09T05:39:28.122 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=dmi 2023-10-09T05:39:28.122 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=edac 2023-10-09T05:39:28.122 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=entropy 2023-10-09T05:39:28.122 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-10-09T05:39:28.122 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=filefd 2023-10-09T05:39:28.122 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=filesystem 2023-10-09T05:39:28.122 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=hwmon 2023-10-09T05:39:28.122 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=infiniband 2023-10-09T05:39:28.123 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=ipvs 2023-10-09T05:39:28.123 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=loadavg 2023-10-09T05:39:28.123 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=mdadm 2023-10-09T05:39:28.123 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=meminfo 2023-10-09T05:39:28.123 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=netclass 2023-10-09T05:39:28.123 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=netdev 2023-10-09T05:39:28.123 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=netstat 2023-10-09T05:39:28.123 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=nfs 2023-10-09T05:39:28.123 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=nfsd 2023-10-09T05:39:28.123 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=nvme 2023-10-09T05:39:28.123 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=os 2023-10-09T05:39:28.123 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-10-09T05:39:28.124 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=pressure 2023-10-09T05:39:28.124 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=rapl 2023-10-09T05:39:28.124 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=schedstat 2023-10-09T05:39:28.124 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=selinux 2023-10-09T05:39:28.124 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=sockstat 2023-10-09T05:39:28.124 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=softnet 2023-10-09T05:39:28.124 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=stat 2023-10-09T05:39:28.124 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=tapestats 2023-10-09T05:39:28.124 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=textfile 2023-10-09T05:39:28.124 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-10-09T05:39:28.124 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=time 2023-10-09T05:39:28.124 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-10-09T05:39:28.125 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=uname 2023-10-09T05:39:28.125 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=vmstat 2023-10-09T05:39:28.125 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=xfs 2023-10-09T05:39:28.125 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=node_exporter.go:117 level=info collector=zfs 2023-10-09T05:39:28.125 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-10-09T05:39:28.125 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 conmon[152503]: ts=2023-10-09T05:39:27.648Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-10-09T05:39:28.125 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 bash[152427]: fead60fdc013821bcb81d67888ea028bf8a07b84730304bc55a3ee99544b0c45 2023-10-09T05:39:28.125 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:39:27 smithi156 systemd[1]: Started Ceph node-exporter.b for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:39:28.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:27 smithi064 conmon[101498]: [09/Oct/2023:05:39:27] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:39:28.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:27 smithi064 conmon[101498]: [09/Oct/2023:05:39:27] ENGINE Bus STOPPED 2023-10-09T05:39:28.220 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:27 smithi064 conmon[101498]: [09/Oct/2023:05:39:27] ENGINE Bus STARTING 2023-10-09T05:39:28.220 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:27 smithi064 conmon[101498]: [09/Oct/2023:05:39:27] ENGINE Serving on http://:::9283 2023-10-09T05:39:28.220 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:39:27 smithi064 conmon[101498]: [09/Oct/2023:05:39:27] ENGINE Bus STARTED 2023-10-09T05:39:28.623 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:39:28.624 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":63,"fsid":"44c77acc-6665-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:33:40.037198+0000","modified":"2023-10-09T05:39:21.964065+0000","last_up_change":"2023-10-09T05:38:31.856610+0000","last_in_change":"2023-10-09T05:38:14.961182+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:36:42.276784+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:39:17.895056+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"85d0857a-ca32-43d2-8cec-bbd916db9f98","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2019939615},{"type":"v1","addr":"172.21.15.64:6803","nonce":2019939615}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2019939615},{"type":"v1","addr":"172.21.15.64:6805","nonce":2019939615}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2019939615},{"type":"v1","addr":"172.21.15.64:6809","nonce":2019939615}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2019939615},{"type":"v1","addr":"172.21.15.64:6807","nonce":2019939615}]},"public_addr":"172.21.15.64:6803/2019939615","cluster_addr":"172.21.15.64:6805/2019939615","heartbeat_back_addr":"172.21.15.64:6809/2019939615","heartbeat_front_addr":"172.21.15.64:6807/2019939615","state":["exists","up"]},{"osd":1,"uuid":"05570845-2cd8-4e24-a2f5-f46cf67aa31a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":1958566614},{"type":"v1","addr":"172.21.15.64:6811","nonce":1958566614}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":1958566614},{"type":"v1","addr":"172.21.15.64:6813","nonce":1958566614}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1958566614},{"type":"v1","addr":"172.21.15.64:6817","nonce":1958566614}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":1958566614},{"type":"v1","addr":"172.21.15.64:6815","nonce":1958566614}]},"public_addr":"172.21.15.64:6811/1958566614","cluster_addr":"172.21.15.64:6813/1958566614","heartbeat_back_addr":"172.21.15.64:6817/1958566614","heartbeat_front_addr":"172.21.15.64:6815/1958566614","state":["exists","up"]},{"osd":2,"uuid":"0f306460-9f08-4f15-b573-3587f63be61c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":2050077610},{"type":"v1","addr":"172.21.15.64:6819","nonce":2050077610}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":2050077610},{"type":"v1","addr":"172.21.15.64:6821","nonce":2050077610}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2050077610},{"type":"v1","addr":"172.21.15.64:6825","nonce":2050077610}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":2050077610},{"type":"v1","addr":"172.21.15.64:6823","nonce":2050077610}]},"public_addr":"172.21.15.64:6819/2050077610","cluster_addr":"172.21.15.64:6821/2050077610","heartbeat_back_addr":"172.21.15.64:6825/2050077610","heartbeat_front_addr":"172.21.15.64:6823/2050077610","state":["exists","up"]},{"osd":3,"uuid":"4bf38edd-e555-402e-ba4b-c751de06aa6a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":3037106123},{"type":"v1","addr":"172.21.15.64:6827","nonce":3037106123}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":3037106123},{"type":"v1","addr":"172.21.15.64:6829","nonce":3037106123}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":3037106123},{"type":"v1","addr":"172.21.15.64:6833","nonce":3037106123}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":3037106123},{"type":"v1","addr":"172.21.15.64:6831","nonce":3037106123}]},"public_addr":"172.21.15.64:6827/3037106123","cluster_addr":"172.21.15.64:6829/3037106123","heartbeat_back_addr":"172.21.15.64:6833/3037106123","heartbeat_front_addr":"172.21.15.64:6831/3037106123","state":["exists","up"]},{"osd":4,"uuid":"060e8766-7bf6-42f1-bebe-dae6828f7fb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6800","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6801","nonce":657648738}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6803","nonce":657648738}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6807","nonce":657648738}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6805","nonce":657648738}]},"public_addr":"172.21.15.156:6801/657648738","cluster_addr":"172.21.15.156:6803/657648738","heartbeat_back_addr":"172.21.15.156:6807/657648738","heartbeat_front_addr":"172.21.15.156:6805/657648738","state":["exists","up"]},{"osd":5,"uuid":"6b5cf38a-6792-4ac7-9694-268601260ba8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6809","nonce":726279450}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6811","nonce":726279450}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6815","nonce":726279450}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6813","nonce":726279450}]},"public_addr":"172.21.15.156:6809/726279450","cluster_addr":"172.21.15.156:6811/726279450","heartbeat_back_addr":"172.21.15.156:6815/726279450","heartbeat_front_addr":"172.21.15.156:6813/726279450","state":["exists","up"]},{"osd":6,"uuid":"01deea2d-a225-40b8-a54b-61eee57da527","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6817","nonce":2369965428}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6819","nonce":2369965428}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6823","nonce":2369965428}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6821","nonce":2369965428}]},"public_addr":"172.21.15.156:6817/2369965428","cluster_addr":"172.21.15.156:6819/2369965428","heartbeat_back_addr":"172.21.15.156:6823/2369965428","heartbeat_front_addr":"172.21.15.156:6821/2369965428","state":["exists","up"]},{"osd":7,"uuid":"693e93b0-c235-47fe-9172-85af68e38634","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6825","nonce":2767546322}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6827","nonce":2767546322}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6831","nonce":2767546322}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6829","nonce":2767546322}]},"public_addr":"172.21.15.156:6825/2767546322","cluster_addr":"172.21.15.156:6827/2767546322","heartbeat_back_addr":"172.21.15.156:6831/2767546322","heartbeat_front_addr":"172.21.15.156:6829/2767546322","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:35:52.928919+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:36:16.479793+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:36:39.503123+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:03.340353+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:24.645080+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:46.312540+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:38:08.346971+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:38:30.073761+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:0/2115451128":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/3707401615":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/1900245998":"2023-10-10T05:34:35.744792+0000","172.21.15.64:0/1336186767":"2023-10-10T05:39:05.525329+0000","172.21.15.64:6801/1833729701":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6800/1833729701":"2023-10-10T05:34:35.744792+0000","172.21.15.64:0/1562509700":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6800/3627852432":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/3346006057":"2023-10-10T05:34:09.281655+0000","172.21.15.64:0/2171400226":"2023-10-10T05:34:09.281655+0000","172.21.15.64:6800/3793931367":"2023-10-10T05:34:09.281655+0000","172.21.15.64:0/3185620259":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/4054596417":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6801/3627852432":"2023-10-10T05:39:05.525329+0000","172.21.15.64:6801/3793931367":"2023-10-10T05:34:09.281655+0000","172.21.15.64:0/514741379":"2023-10-10T05:34:09.281655+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:39:28.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:28 smithi156 ceph-mon[105979]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:28.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:28 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:28.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:28 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:28 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:28 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:28 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:28 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:28 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:28 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T05:39:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:28 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:28.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:28 smithi064 ceph-mon[101070]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:28.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:28 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:28.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:28 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:28.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:28 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:28.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:28 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:28.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:28 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:28.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:28 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:28.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:28 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:28.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:28 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T05:39:28.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:28 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:28.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:28 smithi064 ceph-mon[118354]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:28.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:28 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:28.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:28 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:28.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:28 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:28.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:28 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:28.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:28 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:28.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:28 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:28.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:28 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:28.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:28 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T05:39:28.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:28 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:29.308 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T05:39:29.309 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:39:29.469 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:39:29 smithi064 conmon[163377]: ts=2023-10-09T05:39:29.118Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003171746s 2023-10-09T05:39:29.670 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:29.858 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:29 smithi064 ceph-mon[101070]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T05:39:29.858 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:29 smithi064 ceph-mon[101070]: Deploying daemon grafana.a on smithi156 2023-10-09T05:39:29.858 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:29 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1311645224' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:39:29.859 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:29 smithi064 ceph-mon[118354]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T05:39:29.859 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:29 smithi064 ceph-mon[118354]: Deploying daemon grafana.a on smithi156 2023-10-09T05:39:29.859 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:29 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/1311645224' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:39:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:29 smithi156 ceph-mon[105979]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T05:39:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:29 smithi156 ceph-mon[105979]: Deploying daemon grafana.a on smithi156 2023-10-09T05:39:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:29 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/1311645224' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:39:30.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:30 smithi156 ceph-mon[105979]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:30.918 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:30 smithi064 ceph-mon[101070]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:30.919 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:30 smithi064 ceph-mon[118354]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:31.315 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:39:31.315 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":63,"fsid":"44c77acc-6665-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:33:40.037198+0000","modified":"2023-10-09T05:39:21.964065+0000","last_up_change":"2023-10-09T05:38:31.856610+0000","last_in_change":"2023-10-09T05:38:14.961182+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:36:42.276784+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:39:17.895056+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"85d0857a-ca32-43d2-8cec-bbd916db9f98","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2019939615},{"type":"v1","addr":"172.21.15.64:6803","nonce":2019939615}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2019939615},{"type":"v1","addr":"172.21.15.64:6805","nonce":2019939615}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2019939615},{"type":"v1","addr":"172.21.15.64:6809","nonce":2019939615}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2019939615},{"type":"v1","addr":"172.21.15.64:6807","nonce":2019939615}]},"public_addr":"172.21.15.64:6803/2019939615","cluster_addr":"172.21.15.64:6805/2019939615","heartbeat_back_addr":"172.21.15.64:6809/2019939615","heartbeat_front_addr":"172.21.15.64:6807/2019939615","state":["exists","up"]},{"osd":1,"uuid":"05570845-2cd8-4e24-a2f5-f46cf67aa31a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":1958566614},{"type":"v1","addr":"172.21.15.64:6811","nonce":1958566614}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":1958566614},{"type":"v1","addr":"172.21.15.64:6813","nonce":1958566614}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1958566614},{"type":"v1","addr":"172.21.15.64:6817","nonce":1958566614}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":1958566614},{"type":"v1","addr":"172.21.15.64:6815","nonce":1958566614}]},"public_addr":"172.21.15.64:6811/1958566614","cluster_addr":"172.21.15.64:6813/1958566614","heartbeat_back_addr":"172.21.15.64:6817/1958566614","heartbeat_front_addr":"172.21.15.64:6815/1958566614","state":["exists","up"]},{"osd":2,"uuid":"0f306460-9f08-4f15-b573-3587f63be61c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":2050077610},{"type":"v1","addr":"172.21.15.64:6819","nonce":2050077610}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":2050077610},{"type":"v1","addr":"172.21.15.64:6821","nonce":2050077610}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2050077610},{"type":"v1","addr":"172.21.15.64:6825","nonce":2050077610}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":2050077610},{"type":"v1","addr":"172.21.15.64:6823","nonce":2050077610}]},"public_addr":"172.21.15.64:6819/2050077610","cluster_addr":"172.21.15.64:6821/2050077610","heartbeat_back_addr":"172.21.15.64:6825/2050077610","heartbeat_front_addr":"172.21.15.64:6823/2050077610","state":["exists","up"]},{"osd":3,"uuid":"4bf38edd-e555-402e-ba4b-c751de06aa6a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":3037106123},{"type":"v1","addr":"172.21.15.64:6827","nonce":3037106123}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":3037106123},{"type":"v1","addr":"172.21.15.64:6829","nonce":3037106123}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":3037106123},{"type":"v1","addr":"172.21.15.64:6833","nonce":3037106123}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":3037106123},{"type":"v1","addr":"172.21.15.64:6831","nonce":3037106123}]},"public_addr":"172.21.15.64:6827/3037106123","cluster_addr":"172.21.15.64:6829/3037106123","heartbeat_back_addr":"172.21.15.64:6833/3037106123","heartbeat_front_addr":"172.21.15.64:6831/3037106123","state":["exists","up"]},{"osd":4,"uuid":"060e8766-7bf6-42f1-bebe-dae6828f7fb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6800","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6801","nonce":657648738}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6803","nonce":657648738}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6807","nonce":657648738}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6805","nonce":657648738}]},"public_addr":"172.21.15.156:6801/657648738","cluster_addr":"172.21.15.156:6803/657648738","heartbeat_back_addr":"172.21.15.156:6807/657648738","heartbeat_front_addr":"172.21.15.156:6805/657648738","state":["exists","up"]},{"osd":5,"uuid":"6b5cf38a-6792-4ac7-9694-268601260ba8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6809","nonce":726279450}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6811","nonce":726279450}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6815","nonce":726279450}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6813","nonce":726279450}]},"public_addr":"172.21.15.156:6809/726279450","cluster_addr":"172.21.15.156:6811/726279450","heartbeat_back_addr":"172.21.15.156:6815/726279450","heartbeat_front_addr":"172.21.15.156:6813/726279450","state":["exists","up"]},{"osd":6,"uuid":"01deea2d-a225-40b8-a54b-61eee57da527","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6817","nonce":2369965428}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6819","nonce":2369965428}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6823","nonce":2369965428}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6821","nonce":2369965428}]},"public_addr":"172.21.15.156:6817/2369965428","cluster_addr":"172.21.15.156:6819/2369965428","heartbeat_back_addr":"172.21.15.156:6823/2369965428","heartbeat_front_addr":"172.21.15.156:6821/2369965428","state":["exists","up"]},{"osd":7,"uuid":"693e93b0-c235-47fe-9172-85af68e38634","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6825","nonce":2767546322}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6827","nonce":2767546322}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6831","nonce":2767546322}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6829","nonce":2767546322}]},"public_addr":"172.21.15.156:6825/2767546322","cluster_addr":"172.21.15.156:6827/2767546322","heartbeat_back_addr":"172.21.15.156:6831/2767546322","heartbeat_front_addr":"172.21.15.156:6829/2767546322","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:35:52.928919+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:36:16.479793+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:36:39.503123+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:03.340353+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:24.645080+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:46.312540+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:38:08.346971+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:38:30.073761+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:0/2115451128":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/3707401615":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/1900245998":"2023-10-10T05:34:35.744792+0000","172.21.15.64:0/1336186767":"2023-10-10T05:39:05.525329+0000","172.21.15.64:6801/1833729701":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6800/1833729701":"2023-10-10T05:34:35.744792+0000","172.21.15.64:0/1562509700":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6800/3627852432":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/3346006057":"2023-10-10T05:34:09.281655+0000","172.21.15.64:0/2171400226":"2023-10-10T05:34:09.281655+0000","172.21.15.64:6800/3793931367":"2023-10-10T05:34:09.281655+0000","172.21.15.64:0/3185620259":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/4054596417":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6801/3627852432":"2023-10-10T05:39:05.525329+0000","172.21.15.64:6801/3793931367":"2023-10-10T05:34:09.281655+0000","172.21.15.64:0/514741379":"2023-10-10T05:34:09.281655+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:39:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:31 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:31 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/424163655' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:39:31.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:31 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:31.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:31 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/424163655' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:39:31.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:31 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:31.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:31 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/424163655' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:39:32.049 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T05:39:32.049 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T05:39:32.050 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T05:39:32.050 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T05:39:32.051 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T05:39:32.051 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T05:39:32.051 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T05:39:32.052 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T05:39:32.675 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:32.815 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:32.816 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:32.900 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:32 smithi064 ceph-mon[101070]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:32.901 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:32 smithi064 ceph-mon[118354]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:32.984 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:33.100 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:33.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:32 smithi156 ceph-mon[105979]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:33.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:39:32 smithi156 conmon[109199]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:39:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:39:33.776 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:34.850 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:34 smithi064 ceph-mon[101070]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:34.851 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:34 smithi064 ceph-mon[118354]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:35.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:34 smithi156 ceph-mon[105979]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:35.295 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:36.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:36 smithi156 ceph-mon[105979]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:36.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:36 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:36.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:36 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:36.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:36 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:36.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:36 smithi064 ceph-mon[101070]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:36.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:36 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:36.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:36 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:36.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:36 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:36.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:36 smithi064 ceph-mon[118354]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:36.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:36 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:36.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:36 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:36.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:36 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:37.687 INFO:teuthology.orchestra.run.smithi064.stdout:34359738414 2023-10-09T05:39:37.687 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T05:39:38.056 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:38.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:38 smithi064 ceph-mon[101070]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:38.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:38 smithi064 ceph-mon[118354]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:38.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:38 smithi156 ceph-mon[105979]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:40.892 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:40 smithi064 ceph-mon[101070]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:40.892 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:40 smithi064 ceph-mon[118354]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:41.051 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:40 smithi156 ceph-mon[105979]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:41.146 INFO:teuthology.orchestra.run.smithi064.stdout:163208757271 2023-10-09T05:39:41.146 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T05:39:41.211 INFO:teuthology.orchestra.run.smithi064.stdout:231928233999 2023-10-09T05:39:41.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T05:39:42.272 INFO:teuthology.orchestra.run.smithi064.stdout:197568495635 2023-10-09T05:39:42.272 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T05:39:42.387 INFO:teuthology.orchestra.run.smithi064.stdout:107374182432 2023-10-09T05:39:42.387 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T05:39:42.856 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:42 smithi064 ceph-mon[101070]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:42.857 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:42 smithi064 ceph-mon[118354]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:43.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:42 smithi156 ceph-mon[105979]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:43.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:39:42 smithi156 conmon[109199]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:39:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:39:43.985 INFO:teuthology.orchestra.run.smithi064.stdout:55834574891 2023-10-09T05:39:43.986 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T05:39:44.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:44 smithi064 ceph-mon[101070]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:44.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:44 smithi064 ceph-mon[118354]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:44.916 INFO:teuthology.orchestra.run.smithi064.stdout:128849018909 2023-10-09T05:39:44.917 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T05:39:44.946 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:44.972 INFO:teuthology.orchestra.run.smithi064.stdout:77309411365 2023-10-09T05:39:44.972 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T05:39:45.063 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:45.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:44 smithi156 ceph-mon[105979]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:45.149 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:45.275 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:45.366 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:45.367 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:46.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:46 smithi064 ceph-mon[101070]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:46.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:46 smithi064 ceph-mon[118354]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:46.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:46 smithi156 ceph-mon[105979]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:47.402 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:47.787 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:48.597 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:48 smithi156 systemd[1]: Starting Ceph grafana.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:39:48.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:48 smithi156 ceph-mon[105979]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:48.956 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:48 smithi064 ceph-mon[101070]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:48.956 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:48 smithi064 ceph-mon[118354]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:49.292 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 bash[153184]: dbad77de7b52a3c79dc39c2a19b5c7109dd3372599e716a9c62b69bb8bdf5e8e 2023-10-09T05:39:49.315 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 systemd[1]: Started Ceph grafana.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:39:49.315 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=settings t=2023-10-09T05:39:49.145952387Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-10-09T05:39:49.315 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=settings t=2023-10-09T05:39:49.146161885Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-10-09T05:39:49.315 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=settings t=2023-10-09T05:39:49.146180826Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-10-09T05:39:49.315 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=settings t=2023-10-09T05:39:49.146187652Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-10-09T05:39:49.315 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=settings t=2023-10-09T05:39:49.146193673Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-09T05:39:49.315 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=settings t=2023-10-09T05:39:49.146199652Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-09T05:39:49.315 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=settings t=2023-10-09T05:39:49.146205543Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-09T05:39:49.315 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=settings t=2023-10-09T05:39:49.146211305Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-09T05:39:49.316 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=settings t=2023-10-09T05:39:49.146217612Z level=info msg="Path Home" path=/usr/share/grafana 2023-10-09T05:39:49.316 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=settings t=2023-10-09T05:39:49.146223583Z level=info msg="Path Data" path=/var/lib/grafana 2023-10-09T05:39:49.316 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=settings t=2023-10-09T05:39:49.146229095Z level=info msg="Path Logs" path=/var/log/grafana 2023-10-09T05:39:49.316 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=settings t=2023-10-09T05:39:49.1462345Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-10-09T05:39:49.316 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=settings t=2023-10-09T05:39:49.14624017Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-10-09T05:39:49.316 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=settings t=2023-10-09T05:39:49.146246031Z level=info msg="App mode production" 2023-10-09T05:39:49.316 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=sqlstore t=2023-10-09T05:39:49.146328893Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-10-09T05:39:49.316 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=sqlstore t=2023-10-09T05:39:49.146345667Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-09T05:39:49.316 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:49.147210541Z level=info msg="Starting DB migrations" 2023-10-09T05:39:49.316 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:49.147285204Z level=info msg="Executing migration" id="create migration_log table" 2023-10-09T05:39:49.316 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:49.183793648Z level=info msg="Executing migration" id="create user table" 2023-10-09T05:39:49.316 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:49.233938719Z level=info msg="Executing migration" id="add unique index user.login" 2023-10-09T05:39:49.317 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:49.292532045Z level=info msg="Executing migration" id="add unique index user.email" 2023-10-09T05:39:49.553 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:49.33494036Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2023-10-09T05:39:49.553 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:49.360149971Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2023-10-09T05:39:49.553 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:49.385389592Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2023-10-09T05:39:49.553 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:49.410772567Z level=info msg="Executing migration" id="create user table v2" 2023-10-09T05:39:49.553 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:49.435970003Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2023-10-09T05:39:49.554 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:49.477878028Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2023-10-09T05:39:49.554 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:49.519991787Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2023-10-09T05:39:49.554 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:49.553563527Z level=info msg="Executing migration" id="Drop old table user_v1" 2023-10-09T05:39:49.858 INFO:teuthology.orchestra.run.smithi064.stdout:34359738417 2023-10-09T05:39:49.870 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:49.57895274Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2023-10-09T05:39:49.870 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:49.612472156Z level=info msg="Executing migration" id="Update user table charset" 2023-10-09T05:39:49.870 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:49.654526389Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2023-10-09T05:39:49.870 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:49.721345628Z level=info msg="Executing migration" id="Add missing user data" 2023-10-09T05:39:49.870 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:49.746585809Z level=info msg="Executing migration" id="Add is_disabled column to user" 2023-10-09T05:39:49.871 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:49.771795638Z level=info msg="Executing migration" id="Add index user.login/user.email" 2023-10-09T05:39:49.871 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:49.797116377Z level=info msg="Executing migration" id="Add is_service_account column to user" 2023-10-09T05:39:50.140 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:49.87228207Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2023-10-09T05:39:50.140 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:49 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:49.956132868Z level=info msg="Executing migration" id="create temp user table v1-7" 2023-10-09T05:39:50.140 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:50 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:50.006586041Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2023-10-09T05:39:50.140 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:50 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:50.081943065Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2023-10-09T05:39:50.140 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:50 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:50.140 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:50 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:50.141 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:50 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:50.141 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:50 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:50.141 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:50 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:39:50.141 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:50 smithi156 ceph-mon[105979]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:50.141 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:50 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/901225481' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:39:50.273 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:50 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:50.274 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:50 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:50.274 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:50 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:50.274 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:50 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:50.274 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:50 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:39:50.274 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:50 smithi064 ceph-mon[101070]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:50.274 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:50 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/901225481' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:39:50.274 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:50 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:50.274 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:50 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:50.274 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:50 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:50.275 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:50 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:50.275 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:50 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:39:50.275 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:50 smithi064 ceph-mon[118354]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:50.275 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:50 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/901225481' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:39:50.408 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:50 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:50.140657994Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2023-10-09T05:39:50.409 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:50 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:50.225158081Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2023-10-09T05:39:50.409 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:50 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:50.283592062Z level=info msg="Executing migration" id="Update temp_user table charset" 2023-10-09T05:39:50.409 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:50 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:50.325586741Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2023-10-09T05:39:50.409 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:50 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:50.36755412Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2023-10-09T05:39:50.409 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:50 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:50.40935045Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2023-10-09T05:39:50.492 INFO:teuthology.orchestra.run.smithi064.stdout:231928234001 2023-10-09T05:39:50.564 INFO:teuthology.orchestra.run.smithi064.stdout:163208757273 2023-10-09T05:39:50.667 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:50 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:50.434655928Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2023-10-09T05:39:50.668 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:50 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:50.476547788Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-10-09T05:39:50.668 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:50 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:50.501786627Z level=info msg="Executing migration" id="create temp_user v2" 2023-10-09T05:39:50.668 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:50 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:50.526980862Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2023-10-09T05:39:50.668 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:50 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:50.552216469Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2023-10-09T05:39:50.668 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:50 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:50.602769243Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2023-10-09T05:39:50.668 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:50 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:50.668412286Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2023-10-09T05:39:50.929 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:50 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:50.711991778Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2023-10-09T05:39:50.929 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:50 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:50.780207022Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2023-10-09T05:39:50.929 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:50 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:50.840210712Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2023-10-09T05:39:50.930 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:50 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:50.879569Z level=info msg="Executing migration" id="create star table" 2023-10-09T05:39:51.027 INFO:teuthology.orchestra.run.smithi064.stdout:55834574892 2023-10-09T05:39:51.197 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:50 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:50.93005234Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2023-10-09T05:39:51.197 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:50 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:50.997117262Z level=info msg="Executing migration" id="create org table v1" 2023-10-09T05:39:51.198 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:51 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:51.064076516Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2023-10-09T05:39:51.198 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:51 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:51.138521053Z level=info msg="Executing migration" id="create org_user table v1" 2023-10-09T05:39:51.455 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:51 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3387434441' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:39:51.455 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:51 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/4140042741' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:39:51.455 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:51 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:51.456 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:51 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:51.456 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:51 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/648812930' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:39:51.456 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:51 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:51.19820811Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2023-10-09T05:39:51.456 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:51 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:51.223661554Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2023-10-09T05:39:51.456 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:51 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:51.265561394Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2023-10-09T05:39:51.456 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:51 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:51.290815824Z level=info msg="Executing migration" id="Update org table charset" 2023-10-09T05:39:51.456 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:51 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:51.315973387Z level=info msg="Executing migration" id="Update org_user table charset" 2023-10-09T05:39:51.456 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:51 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:51.341301796Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2023-10-09T05:39:51.456 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:51 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:51.400030943Z level=info msg="Executing migration" id="create dashboard table" 2023-10-09T05:39:51.457 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:51 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:51.455936648Z level=info msg="Executing migration" id="add index dashboard.account_id" 2023-10-09T05:39:51.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:51 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3387434441' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:39:51.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:51 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/4140042741' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:39:51.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:51 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:51.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:51 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:51.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:51 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/648812930' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:39:51.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:51 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3387434441' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:39:51.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:51 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/4140042741' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:39:51.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:51 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:39:51.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:51 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:51.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:51 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/648812930' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:39:51.753 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:51 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:51.492459225Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2023-10-09T05:39:51.753 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:51 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:51.534527978Z level=info msg="Executing migration" id="create dashboard_tag table" 2023-10-09T05:39:51.753 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:51 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:51.584940299Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2023-10-09T05:39:51.753 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:51 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:51.671769563Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-10-09T05:39:51.753 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:51 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:51.753800985Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2023-10-09T05:39:52.072 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:51 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:51.786335265Z level=info msg="Executing migration" id="create dashboard v2" 2023-10-09T05:39:52.072 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:51 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:51.828145107Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2023-10-09T05:39:52.072 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:51 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:51.853446845Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2023-10-09T05:39:52.072 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:51 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:51.870344533Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2023-10-09T05:39:52.072 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:51 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:51.887163931Z level=info msg="Executing migration" id="drop table dashboard_v1" 2023-10-09T05:39:52.072 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:51 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:51.904046151Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2023-10-09T05:39:52.072 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:51 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:51.929329272Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2023-10-09T05:39:52.072 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:51 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:51.996702559Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2023-10-09T05:39:52.370 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.072384361Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2023-10-09T05:39:52.370 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.12305863Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2023-10-09T05:39:52.370 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.189884789Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2023-10-09T05:39:52.370 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.214628812Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2023-10-09T05:39:52.370 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.256581454Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2023-10-09T05:39:52.371 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.321654791Z level=info msg="Executing migration" id="Update dashboard table charset" 2023-10-09T05:39:52.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:52 smithi064 ceph-mon[101070]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:52.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:52 smithi064 ceph-mon[118354]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:52.633 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:52 smithi156 ceph-mon[105979]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:52.634 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.371332202Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2023-10-09T05:39:52.634 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.433069959Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2023-10-09T05:39:52.634 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.475157328Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2023-10-09T05:39:52.634 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.51688611Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2023-10-09T05:39:52.634 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.53388936Z level=info msg="Executing migration" id="Add column uid in dashboard" 2023-10-09T05:39:52.634 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.592132689Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2023-10-09T05:39:52.634 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.634354863Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2023-10-09T05:39:52.886 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[109199]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:39:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:39:52.887 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.659752067Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2023-10-09T05:39:52.887 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.676643381Z level=info msg="Executing migration" id="Update dashboard title length" 2023-10-09T05:39:52.887 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.710214435Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2023-10-09T05:39:52.887 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.727118467Z level=info msg="Executing migration" id="create dashboard_provisioning" 2023-10-09T05:39:52.888 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.744133403Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-10-09T05:39:52.888 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.786188124Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2023-10-09T05:39:52.888 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.819933693Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-10-09T05:39:52.888 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.853355512Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-10-09T05:39:52.888 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.887084209Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2023-10-09T05:39:52.970 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574891 got 55834574892 for osd.1 2023-10-09T05:39:52.970 DEBUG:teuthology.parallel:result is None 2023-10-09T05:39:53.001 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738414 got 34359738417 for osd.0 2023-10-09T05:39:53.001 DEBUG:teuthology.parallel:result is None 2023-10-09T05:39:53.007 INFO:teuthology.orchestra.run.smithi064.stdout:107374182435 2023-10-09T05:39:53.163 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.929177439Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2023-10-09T05:39:53.164 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.962904288Z level=info msg="Executing migration" id="Add check_sum column" 2023-10-09T05:39:53.164 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:52 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:52.979737285Z level=info msg="Executing migration" id="Add index for dashboard_title" 2023-10-09T05:39:53.164 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.004848988Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2023-10-09T05:39:53.164 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.030158483Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2023-10-09T05:39:53.164 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.072096507Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2023-10-09T05:39:53.164 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.097243368Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2023-10-09T05:39:53.164 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.122374841Z level=info msg="Executing migration" id="create data_source table" 2023-10-09T05:39:53.164 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.164310988Z level=info msg="Executing migration" id="add index data_source.account_id" 2023-10-09T05:39:53.249 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928233999 got 231928234001 for osd.7 2023-10-09T05:39:53.249 DEBUG:teuthology.parallel:result is None 2023-10-09T05:39:53.353 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757271 got 163208757273 for osd.5 2023-10-09T05:39:53.353 DEBUG:teuthology.parallel:result is None 2023-10-09T05:39:53.432 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.189479299Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2023-10-09T05:39:53.433 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.214918151Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2023-10-09T05:39:53.433 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.240071512Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2023-10-09T05:39:53.433 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.265235016Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2023-10-09T05:39:53.433 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.307214083Z level=info msg="Executing migration" id="create data_source table v2" 2023-10-09T05:39:53.433 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.33228175Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2023-10-09T05:39:53.433 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.357574453Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2023-10-09T05:39:53.434 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.382930187Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2023-10-09T05:39:53.434 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.407971288Z level=info msg="Executing migration" id="Add column with_credentials" 2023-10-09T05:39:53.434 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.433173193Z level=info msg="Executing migration" id="Add secure json data column" 2023-10-09T05:39:53.554 INFO:teuthology.orchestra.run.smithi064.stdout:197568495637 2023-10-09T05:39:53.684 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:53 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:53.684 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:53 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:53.684 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:53 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3247617775' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:39:53.684 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:53 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/1935231123' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:39:53.685 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.458387627Z level=info msg="Executing migration" id="Update data_source table charset" 2023-10-09T05:39:53.685 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.483800968Z level=info msg="Executing migration" id="Update initial version to 1" 2023-10-09T05:39:53.685 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.545535776Z level=info msg="Executing migration" id="Add read_only data column" 2023-10-09T05:39:53.685 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.592643988Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2023-10-09T05:39:53.685 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.617835782Z level=info msg="Executing migration" id="Update json_data with nulls" 2023-10-09T05:39:53.686 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.643088236Z level=info msg="Executing migration" id="Add uid column" 2023-10-09T05:39:53.686 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.684953675Z level=info msg="Executing migration" id="Update uid value" 2023-10-09T05:39:53.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:53 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:53.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:53 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:53.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:53 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3247617775' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:39:53.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:53 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1935231123' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:39:53.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:53 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:53.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:53 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:39:53.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:53 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3247617775' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:39:53.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:53 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/1935231123' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:39:53.970 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.72729278Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2023-10-09T05:39:53.970 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.761109848Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2023-10-09T05:39:53.971 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.786329393Z level=info msg="Executing migration" id="create api_key table" 2023-10-09T05:39:53.971 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.811502024Z level=info msg="Executing migration" id="add index api_key.account_id" 2023-10-09T05:39:53.971 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.836808938Z level=info msg="Executing migration" id="add index api_key.key" 2023-10-09T05:39:53.971 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.878895989Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2023-10-09T05:39:53.971 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.90399323Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2023-10-09T05:39:53.971 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.929372927Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2023-10-09T05:39:53.971 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:53 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:53.971077296Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2023-10-09T05:39:54.230 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.013122228Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2023-10-09T05:39:54.231 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.055090978Z level=info msg="Executing migration" id="create api_key table v2" 2023-10-09T05:39:54.231 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.097018556Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2023-10-09T05:39:54.231 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.138967063Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2023-10-09T05:39:54.231 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.180807878Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2023-10-09T05:39:54.231 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.206201953Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2023-10-09T05:39:54.231 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.231364296Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2023-10-09T05:39:54.501 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.264990048Z level=info msg="Executing migration" id="Update api_key table charset" 2023-10-09T05:39:54.501 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.290189053Z level=info msg="Executing migration" id="Add expires to api_key table" 2023-10-09T05:39:54.501 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.332215479Z level=info msg="Executing migration" id="Add service account foreign key" 2023-10-09T05:39:54.501 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.374110743Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2023-10-09T05:39:54.501 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.399301509Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2023-10-09T05:39:54.501 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.424385408Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2023-10-09T05:39:54.501 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.449698135Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2023-10-09T05:39:54.513 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495635 got 197568495637 for osd.6 2023-10-09T05:39:54.513 DEBUG:teuthology.parallel:result is None 2023-10-09T05:39:54.769 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:54 smithi156 ceph-mon[105979]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:54.769 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.516568438Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2023-10-09T05:39:54.770 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.533675355Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2023-10-09T05:39:54.770 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.550662964Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2023-10-09T05:39:54.770 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.567527833Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-10-09T05:39:54.770 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.584258459Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2023-10-09T05:39:54.770 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.601226125Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2023-10-09T05:39:54.770 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.634726264Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2023-10-09T05:39:54.770 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.651510349Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2023-10-09T05:39:54.770 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.668486895Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2023-10-09T05:39:54.770 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.710430981Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-10-09T05:39:54.770 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.727329149Z level=info msg="Executing migration" id="create quota table v1" 2023-10-09T05:39:54.770 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.744318295Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2023-10-09T05:39:54.771 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.769725821Z level=info msg="Executing migration" id="Update quota table charset" 2023-10-09T05:39:54.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:54 smithi064 ceph-mon[101070]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:54.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:54 smithi064 ceph-mon[118354]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:55.029 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.820322563Z level=info msg="Executing migration" id="create plugin_setting table" 2023-10-09T05:39:55.030 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.845373147Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-10-09T05:39:55.030 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.870701076Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2023-10-09T05:39:55.030 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.895993707Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2023-10-09T05:39:55.030 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.937813755Z level=info msg="Executing migration" id="create session table" 2023-10-09T05:39:55.030 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:54 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:54.979745892Z level=info msg="Executing migration" id="Drop old table playlist table" 2023-10-09T05:39:55.030 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.005040586Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2023-10-09T05:39:55.030 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.030215409Z level=info msg="Executing migration" id="create playlist table v2" 2023-10-09T05:39:55.298 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.080700993Z level=info msg="Executing migration" id="create playlist item table v2" 2023-10-09T05:39:55.299 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.106018129Z level=info msg="Executing migration" id="Update playlist table charset" 2023-10-09T05:39:55.299 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.131197Z level=info msg="Executing migration" id="Update playlist_item table charset" 2023-10-09T05:39:55.299 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.173168417Z level=info msg="Executing migration" id="drop preferences table v2" 2023-10-09T05:39:55.299 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.190046715Z level=info msg="Executing migration" id="drop preferences table v3" 2023-10-09T05:39:55.299 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.206876303Z level=info msg="Executing migration" id="create preferences table v3" 2023-10-09T05:39:55.299 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.223650631Z level=info msg="Executing migration" id="Update preferences table charset" 2023-10-09T05:39:55.299 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.240415618Z level=info msg="Executing migration" id="Add column team_id in preferences" 2023-10-09T05:39:55.299 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.265828766Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2023-10-09T05:39:55.299 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.299156564Z level=info msg="Executing migration" id="Add column week_start in preferences" 2023-10-09T05:39:55.355 INFO:teuthology.orchestra.run.smithi064.stdout:77309411367 2023-10-09T05:39:55.550 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:55 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3853936410' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:39:55.550 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.324813407Z level=info msg="Executing migration" id="Add column preferences.json_data" 2023-10-09T05:39:55.551 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.366572679Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2023-10-09T05:39:55.551 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.408595708Z level=info msg="Executing migration" id="Add preferences index org_id" 2023-10-09T05:39:55.551 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.483736884Z level=info msg="Executing migration" id="Add preferences index user_id" 2023-10-09T05:39:55.551 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.525715705Z level=info msg="Executing migration" id="create alert table v1" 2023-10-09T05:39:55.551 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.551000859Z level=info msg="Executing migration" id="add index alert org_id & id " 2023-10-09T05:39:55.575 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:55 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3853936410' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:39:55.575 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:55 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3853936410' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:39:55.802 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182432 got 107374182435 for osd.3 2023-10-09T05:39:55.802 DEBUG:teuthology.parallel:result is None 2023-10-09T05:39:55.811 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.576322064Z level=info msg="Executing migration" id="add index alert state" 2023-10-09T05:39:55.811 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.618230742Z level=info msg="Executing migration" id="add index alert dashboard_id" 2023-10-09T05:39:55.811 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.660271172Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2023-10-09T05:39:55.811 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.685410403Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-10-09T05:39:55.811 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.710634967Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-10-09T05:39:55.811 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.735803373Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-10-09T05:39:55.811 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.76096132Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2023-10-09T05:39:55.811 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.786473666Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-10-09T05:39:55.811 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.811691227Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2023-10-09T05:39:55.983 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411365 got 77309411367 for osd.2 2023-10-09T05:39:55.983 DEBUG:teuthology.parallel:result is None 2023-10-09T05:39:56.096 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.853914443Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2023-10-09T05:39:56.097 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.895908967Z level=info msg="Executing migration" id="create alert_notification table v1" 2023-10-09T05:39:56.097 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.921062838Z level=info msg="Executing migration" id="Add column is_default" 2023-10-09T05:39:56.097 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.962938051Z level=info msg="Executing migration" id="Add column frequency" 2023-10-09T05:39:56.097 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:55 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:55.988086302Z level=info msg="Executing migration" id="Add column send_reminder" 2023-10-09T05:39:56.097 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.013282475Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2023-10-09T05:39:56.097 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.05532361Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2023-10-09T05:39:56.097 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.097257052Z level=info msg="Executing migration" id="Update alert table charset" 2023-10-09T05:39:56.349 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.130875978Z level=info msg="Executing migration" id="Update alert_notification table charset" 2023-10-09T05:39:56.349 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.147606574Z level=info msg="Executing migration" id="create notification_journal table v1" 2023-10-09T05:39:56.349 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.181488728Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2023-10-09T05:39:56.349 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.198278611Z level=info msg="Executing migration" id="drop alert_notification_journal" 2023-10-09T05:39:56.349 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.215160818Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2023-10-09T05:39:56.350 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.232101922Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-10-09T05:39:56.350 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.248772485Z level=info msg="Executing migration" id="Add for to alert table" 2023-10-09T05:39:56.350 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.265717843Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2023-10-09T05:39:56.350 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.299236034Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2023-10-09T05:39:56.350 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.332776789Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2023-10-09T05:39:56.350 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.349838552Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2023-10-09T05:39:56.610 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:56 smithi156 ceph-mon[105979]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:56.610 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.422464421Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2023-10-09T05:39:56.610 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.458905622Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2023-10-09T05:39:56.610 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.475874193Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2023-10-09T05:39:56.610 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.492709196Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2023-10-09T05:39:56.610 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.526398374Z level=info msg="Executing migration" id="Drop old annotation table v4" 2023-10-09T05:39:56.611 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.56007087Z level=info msg="Executing migration" id="create annotation table v5" 2023-10-09T05:39:56.611 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.577096667Z level=info msg="Executing migration" id="add index annotation 0 v3" 2023-10-09T05:39:56.611 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.610505453Z level=info msg="Executing migration" id="add index annotation 1 v3" 2023-10-09T05:39:56.641 INFO:teuthology.orchestra.run.smithi064.stdout:128849018911 2023-10-09T05:39:56.846 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:56 smithi064 ceph-mon[101070]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:56.847 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:56 smithi064 ceph-mon[118354]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:56.862 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.644244379Z level=info msg="Executing migration" id="add index annotation 2 v3" 2023-10-09T05:39:56.862 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.677788012Z level=info msg="Executing migration" id="add index annotation 3 v3" 2023-10-09T05:39:56.862 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.694924049Z level=info msg="Executing migration" id="add index annotation 4 v3" 2023-10-09T05:39:56.862 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.745155028Z level=info msg="Executing migration" id="Update annotation table charset" 2023-10-09T05:39:56.862 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.770335058Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2023-10-09T05:39:56.862 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.795447176Z level=info msg="Executing migration" id="Drop category_id index" 2023-10-09T05:39:56.862 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.820873131Z level=info msg="Executing migration" id="Add column tags to annotation table" 2023-10-09T05:39:56.862 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.862740916Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2023-10-09T05:39:57.121 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.888122031Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2023-10-09T05:39:57.122 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.913207806Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-10-09T05:39:57.122 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.938412594Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-10-09T05:39:57.122 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.963652188Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2023-10-09T05:39:57.122 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:56 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:56.988899555Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-10-09T05:39:57.122 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.030923214Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2023-10-09T05:39:57.122 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.072836125Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2023-10-09T05:39:57.122 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.097870516Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2023-10-09T05:39:57.407 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.148344997Z level=info msg="Executing migration" id="Add created time to annotation table" 2023-10-09T05:39:57.408 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.19032223Z level=info msg="Executing migration" id="Add updated time to annotation table" 2023-10-09T05:39:57.408 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.215506584Z level=info msg="Executing migration" id="Add index for created in annotation table" 2023-10-09T05:39:57.408 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.257591204Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2023-10-09T05:39:57.408 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.282678684Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2023-10-09T05:39:57.408 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.307954019Z level=info msg="Executing migration" id="Add epoch_end column" 2023-10-09T05:39:57.408 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.349840274Z level=info msg="Executing migration" id="Add index for epoch_end" 2023-10-09T05:39:57.408 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.408341947Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2023-10-09T05:39:57.446 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018909 got 128849018911 for osd.4 2023-10-09T05:39:57.446 DEBUG:teuthology.parallel:result is None 2023-10-09T05:39:57.446 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T05:39:57.447 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:39:57.693 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:57 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/2721679322' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:39:57.693 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.450450591Z level=info msg="Executing migration" id="Move region to single row" 2023-10-09T05:39:57.694 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.475628954Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2023-10-09T05:39:57.694 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.500888986Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-10-09T05:39:57.694 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.543063616Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-10-09T05:39:57.694 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.584939369Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2023-10-09T05:39:57.694 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.610007387Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2023-10-09T05:39:57.694 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.651916854Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2023-10-09T05:39:57.694 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.694059643Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2023-10-09T05:39:57.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:57 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/2721679322' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:39:57.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:57 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/2721679322' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:39:57.971 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.7359369Z level=info msg="Executing migration" id="create test_data table" 2023-10-09T05:39:57.971 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.753057522Z level=info msg="Executing migration" id="create dashboard_version table v1" 2023-10-09T05:39:57.971 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.786668819Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2023-10-09T05:39:57.971 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.811813432Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-10-09T05:39:57.971 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.853798691Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2023-10-09T05:39:57.971 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.895909726Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2023-10-09T05:39:57.971 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.921208596Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2023-10-09T05:39:57.972 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.937844358Z level=info msg="Executing migration" id="create team table" 2023-10-09T05:39:57.972 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.971569579Z level=info msg="Executing migration" id="add index team.org_id" 2023-10-09T05:39:57.976 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:39:58.257 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:57 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:57.996982954Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2023-10-09T05:39:58.257 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.013752894Z level=info msg="Executing migration" id="create team member table" 2023-10-09T05:39:58.257 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.030626232Z level=info msg="Executing migration" id="add index team_member.org_id" 2023-10-09T05:39:58.257 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.064415824Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2023-10-09T05:39:58.257 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.081124882Z level=info msg="Executing migration" id="add index team_member.team_id" 2023-10-09T05:39:58.257 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.123190263Z level=info msg="Executing migration" id="Add column email to team table" 2023-10-09T05:39:58.257 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.148400488Z level=info msg="Executing migration" id="Add column external to team_member table" 2023-10-09T05:39:58.257 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.190392836Z level=info msg="Executing migration" id="Add column permission to team_member table" 2023-10-09T05:39:58.258 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.215514904Z level=info msg="Executing migration" id="create dashboard acl table" 2023-10-09T05:39:58.258 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.257535287Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2023-10-09T05:39:58.519 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.308056859Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2023-10-09T05:39:58.520 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.374996924Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2023-10-09T05:39:58.520 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.416770157Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2023-10-09T05:39:58.520 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.442166522Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2023-10-09T05:39:58.520 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.467235419Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2023-10-09T05:39:58.520 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.492281329Z level=info msg="Executing migration" id="add index dashboard_permission" 2023-10-09T05:39:58.554 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:39:58 smithi064 ceph-mon[101070]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:58.555 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:39:58 smithi064 ceph-mon[118354]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:58.776 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:39:58 smithi156 ceph-mon[105979]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:39:58.777 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.534225626Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2023-10-09T05:39:58.777 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.567963899Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2023-10-09T05:39:58.777 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.609834888Z level=info msg="Executing migration" id="create tag table" 2023-10-09T05:39:58.777 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.659817195Z level=info msg="Executing migration" id="add index tag.key_value" 2023-10-09T05:39:58.777 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.693515567Z level=info msg="Executing migration" id="create login attempt table" 2023-10-09T05:39:58.777 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.743737127Z level=info msg="Executing migration" id="add index login_attempt.username" 2023-10-09T05:39:58.777 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.777308668Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2023-10-09T05:39:59.037 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.80251666Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-10-09T05:39:59.037 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.827803216Z level=info msg="Executing migration" id="create login_attempt v2" 2023-10-09T05:39:59.037 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.869854307Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2023-10-09T05:39:59.037 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.903464379Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2023-10-09T05:39:59.037 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.92862414Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2023-10-09T05:39:59.037 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.970496245Z level=info msg="Executing migration" id="create user auth table" 2023-10-09T05:39:59.037 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:58 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:58.995847097Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-10-09T05:39:59.038 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.037729381Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2023-10-09T05:39:59.330 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.088190804Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2023-10-09T05:39:59.331 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.113324953Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2023-10-09T05:39:59.331 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.155279078Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2023-10-09T05:39:59.331 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.197367889Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2023-10-09T05:39:59.331 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.222355881Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2023-10-09T05:39:59.331 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.264257193Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2023-10-09T05:39:59.331 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.331268752Z level=info msg="Executing migration" id="create server_lock table" 2023-10-09T05:39:59.582 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.381687638Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2023-10-09T05:39:59.582 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.423801331Z level=info msg="Executing migration" id="create user auth token table" 2023-10-09T05:39:59.583 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.465516835Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2023-10-09T05:39:59.583 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.50740081Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2023-10-09T05:39:59.583 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.532661413Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2023-10-09T05:39:59.583 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.557775534Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2023-10-09T05:39:59.583 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.583031637Z level=info msg="Executing migration" id="create cache_data table" 2023-10-09T05:39:59.834 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.624984477Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2023-10-09T05:39:59.834 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.650226404Z level=info msg="Executing migration" id="create short_url table v1" 2023-10-09T05:39:59.834 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.692259221Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2023-10-09T05:39:59.834 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.717293813Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2023-10-09T05:39:59.834 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.742400197Z level=info msg="Executing migration" id="delete alert_definition table" 2023-10-09T05:39:59.834 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.767730186Z level=info msg="Executing migration" id="recreate alert_definition table" 2023-10-09T05:39:59.835 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.80955579Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2023-10-09T05:39:59.835 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.834718984Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2023-10-09T05:40:00.086 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.877038064Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2023-10-09T05:40:00.086 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.918736082Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2023-10-09T05:40:00.086 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.960731426Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2023-10-09T05:40:00.086 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:39:59 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:39:59.985896171Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2023-10-09T05:40:00.086 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.011283865Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2023-10-09T05:40:00.086 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.053155187Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2023-10-09T05:40:00.087 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.086761103Z level=info msg="Executing migration" id="drop alert_definition table" 2023-10-09T05:40:00.100 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:40:00.101 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T05:40:00.352 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.137287962Z level=info msg="Executing migration" id="delete alert_definition_version table" 2023-10-09T05:40:00.352 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.162388771Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2023-10-09T05:40:00.352 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.204344911Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-10-09T05:40:00.352 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.229408388Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-10-09T05:40:00.352 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.296380186Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2023-10-09T05:40:00.353 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.321598229Z level=info msg="Executing migration" id="drop alert_definition_version table" 2023-10-09T05:40:00.606 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.373556954Z level=info msg="Executing migration" id="create alert_instance table" 2023-10-09T05:40:00.607 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.414120746Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-10-09T05:40:00.607 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.439326079Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2023-10-09T05:40:00.607 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.464392283Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2023-10-09T05:40:00.607 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.489640519Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-10-09T05:40:00.607 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.514993261Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2023-10-09T05:40:00.607 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.540034771Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2023-10-09T05:40:00.607 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.565295115Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2023-10-09T05:40:00.607 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.607222104Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-10-09T05:40:00.608 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:00 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:00.608 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:00 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:00.608 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:00 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:40:00.608 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:00 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:40:00.608 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:00 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:00.608 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:00 smithi156 ceph-mon[105979]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T05:40:00.608 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:00 smithi156 ceph-mon[105979]: Reconfiguring daemon alertmanager.a on smithi064 2023-10-09T05:40:00.608 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:00 smithi156 ceph-mon[105979]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:00.608 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:00 smithi156 ceph-mon[105979]: overall HEALTH_OK 2023-10-09T05:40:00.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:00 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:00.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:00 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:00.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:00 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:40:00.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:00 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:40:00.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:00 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:00.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:00 smithi064 ceph-mon[101070]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T05:40:00.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:00 smithi064 ceph-mon[101070]: Reconfiguring daemon alertmanager.a on smithi064 2023-10-09T05:40:00.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:00 smithi064 ceph-mon[101070]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:00.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:00 smithi064 ceph-mon[101070]: overall HEALTH_OK 2023-10-09T05:40:00.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:00 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:00.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:00 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:00.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:00 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:40:00.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:00 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:40:00.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:00 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:00.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:00 smithi064 ceph-mon[118354]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T05:40:00.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:00 smithi064 ceph-mon[118354]: Reconfiguring daemon alertmanager.a on smithi064 2023-10-09T05:40:00.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:00 smithi064 ceph-mon[118354]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:00.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:00 smithi064 ceph-mon[118354]: overall HEALTH_OK 2023-10-09T05:40:00.867 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.657679288Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2023-10-09T05:40:00.867 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.699585107Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2023-10-09T05:40:00.867 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.725000158Z level=info msg="Executing migration" id="create alert_rule table" 2023-10-09T05:40:00.867 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.750138416Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2023-10-09T05:40:00.867 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.775271538Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2023-10-09T05:40:00.867 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.800345763Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-10-09T05:40:00.867 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.825491756Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2023-10-09T05:40:00.868 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.867607629Z level=info msg="Executing migration" id="add column for to alert_rule" 2023-10-09T05:40:01.120 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.893113176Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2023-10-09T05:40:01.120 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.934868427Z level=info msg="Executing migration" id="add column labels to alert_rule" 2023-10-09T05:40:01.120 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:00 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:00.976686932Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2023-10-09T05:40:01.120 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.00200963Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-10-09T05:40:01.120 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.027209179Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2023-10-09T05:40:01.120 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.068933888Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2023-10-09T05:40:01.120 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.110797994Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-10-09T05:40:01.412 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:01 smithi156 ceph-mon[105979]: from='client.24521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:40:01.413 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.161434748Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2023-10-09T05:40:01.413 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.236935252Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2023-10-09T05:40:01.413 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.278997892Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2023-10-09T05:40:01.413 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.320891744Z level=info msg="Executing migration" id="create alert_rule_version table" 2023-10-09T05:40:01.414 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.346086845Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-10-09T05:40:01.414 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.371131887Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-10-09T05:40:01.414 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.413172646Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2023-10-09T05:40:01.681 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.455039175Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2023-10-09T05:40:01.681 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.480169828Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2023-10-09T05:40:01.681 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.505297078Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2023-10-09T05:40:01.681 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.530458567Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2023-10-09T05:40:01.681 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.572585241Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2023-10-09T05:40:01.681 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.614468946Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2023-10-09T05:40:01.681 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.639536764Z level=info msg="Executing migration" id=create_alert_configuration_table 2023-10-09T05:40:01.682 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.681496218Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2023-10-09T05:40:01.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:01 smithi064 ceph-mon[101070]: from='client.24521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:40:01.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:01 smithi064 ceph-mon[118354]: from='client.24521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:40:01.811 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":34,"stamp":"2023-10-09T05:39:59.546753+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2384952,"kb_used_data":6648,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747576264,"statfs":{"total":767960285184,"available":765518094336,"internally_reserved":0,"allocated":6807552,"data_stored":3875447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001934"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.609674+0000","last_change":"2023-10-09T05:39:18.963184+0000","last_active":"2023-10-09T05:39:20.609674+0000","last_peered":"2023-10-09T05:39:20.609674+0000","last_clean":"2023-10-09T05:39:20.609674+0000","last_became_active":"2023-10-09T05:39:18.962837+0000","last_became_peered":"2023-10-09T05:39:18.962837+0000","last_unstale":"2023-10-09T05:39:20.609674+0000","last_undegraded":"2023-10-09T05:39:20.609674+0000","last_fullsized":"2023-10-09T05:39:20.609674+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:05.818061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.609344+0000","last_change":"2023-10-09T05:39:19.405012+0000","last_active":"2023-10-09T05:39:20.609344+0000","last_peered":"2023-10-09T05:39:20.609344+0000","last_clean":"2023-10-09T05:39:20.609344+0000","last_became_active":"2023-10-09T05:39:19.404658+0000","last_became_peered":"2023-10-09T05:39:19.404658+0000","last_unstale":"2023-10-09T05:39:20.609344+0000","last_undegraded":"2023-10-09T05:39:20.609344+0000","last_fullsized":"2023-10-09T05:39:20.609344+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:24.956847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:39:19.964197+0000","last_change":"2023-10-09T05:39:19.405346+0000","last_active":"2023-10-09T05:39:19.964197+0000","last_peered":"2023-10-09T05:39:19.964197+0000","last_clean":"2023-10-09T05:39:19.964197+0000","last_became_active":"2023-10-09T05:39:19.404876+0000","last_became_peered":"2023-10-09T05:39:19.404876+0000","last_unstale":"2023-10-09T05:39:19.964197+0000","last_undegraded":"2023-10-09T05:39:19.964197+0000","last_fullsized":"2023-10-09T05:39:19.964197+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:28.442496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:39:19.964131+0000","last_change":"2023-10-09T05:39:18.964130+0000","last_active":"2023-10-09T05:39:19.964131+0000","last_peered":"2023-10-09T05:39:19.964131+0000","last_clean":"2023-10-09T05:39:19.964131+0000","last_became_active":"2023-10-09T05:39:18.963726+0000","last_became_peered":"2023-10-09T05:39:18.963726+0000","last_unstale":"2023-10-09T05:39:19.964131+0000","last_undegraded":"2023-10-09T05:39:19.964131+0000","last_fullsized":"2023-10-09T05:39:19.964131+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:13.428159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":271,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:39:19.964133+0000","last_change":"2023-10-09T05:38:35.869176+0000","last_active":"2023-10-09T05:39:19.964133+0000","last_peered":"2023-10-09T05:39:19.964133+0000","last_clean":"2023-10-09T05:39:19.964133+0000","last_became_active":"2023-10-09T05:38:35.868750+0000","last_became_peered":"2023-10-09T05:38:35.868750+0000","last_unstale":"2023-10-09T05:39:19.964133+0000","last_undegraded":"2023-10-09T05:39:19.964133+0000","last_fullsized":"2023-10-09T05:39:19.964133+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:20.200381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.054447+0000","last_change":"2023-10-09T05:39:18.963327+0000","last_active":"2023-10-09T05:39:20.054447+0000","last_peered":"2023-10-09T05:39:20.054447+0000","last_clean":"2023-10-09T05:39:20.054447+0000","last_became_active":"2023-10-09T05:39:18.962933+0000","last_became_peered":"2023-10-09T05:39:18.962933+0000","last_unstale":"2023-10-09T05:39:20.054447+0000","last_undegraded":"2023-10-09T05:39:20.054447+0000","last_fullsized":"2023-10-09T05:39:20.054447+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:43.427745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:39:19.964232+0000","last_change":"2023-10-09T05:39:19.362360+0000","last_active":"2023-10-09T05:39:19.964232+0000","last_peered":"2023-10-09T05:39:19.964232+0000","last_clean":"2023-10-09T05:39:19.964232+0000","last_became_active":"2023-10-09T05:39:19.361972+0000","last_became_peered":"2023-10-09T05:39:19.361972+0000","last_unstale":"2023-10-09T05:39:19.964232+0000","last_undegraded":"2023-10-09T05:39:19.964232+0000","last_fullsized":"2023-10-09T05:39:19.964232+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:06.194167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.054444+0000","last_change":"2023-10-09T05:39:18.963544+0000","last_active":"2023-10-09T05:39:20.054444+0000","last_peered":"2023-10-09T05:39:20.054444+0000","last_clean":"2023-10-09T05:39:20.054444+0000","last_became_active":"2023-10-09T05:39:18.962981+0000","last_became_peered":"2023-10-09T05:39:18.962981+0000","last_unstale":"2023-10-09T05:39:20.054444+0000","last_undegraded":"2023-10-09T05:39:20.054444+0000","last_fullsized":"2023-10-09T05:39:20.054444+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:11.589793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.609337+0000","last_change":"2023-10-09T05:39:18.964305+0000","last_active":"2023-10-09T05:39:20.609337+0000","last_peered":"2023-10-09T05:39:20.609337+0000","last_clean":"2023-10-09T05:39:20.609337+0000","last_became_active":"2023-10-09T05:39:18.963935+0000","last_became_peered":"2023-10-09T05:39:18.963935+0000","last_unstale":"2023-10-09T05:39:20.609337+0000","last_undegraded":"2023-10-09T05:39:20.609337+0000","last_fullsized":"2023-10-09T05:39:20.609337+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:33.105059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234003,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298480,"kb_used_data":1192,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446672,"statfs":{"total":95995035648,"available":95689392128,"internally_reserved":0,"allocated":1220608,"data_stored":853268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:36 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.38200000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:39:36 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.59999999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:39:36 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.0780000000000001,"5min":1.0780000000000001,"15min":1.0780000000000001},"last":0.35999999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:39:36 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.46100000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:39:36 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.628},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.40500000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:39:36 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.65300000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:39:36 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.52200000000000002}]}]},{"osd":6,"up_from":46,"seq":197568495639,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298476,"kb_used_data":1188,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446676,"statfs":{"total":95995035648,"available":95689396224,"internally_reserved":0,"allocated":1216512,"data_stored":853003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:10 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.439},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.44700000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:39:10 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.38},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.497}]},{"osd":2,"last update":"Mon Oct 9 05:39:10 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.42999999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:39:10 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.36799999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:39:10 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.30399999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:39:10 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.41799999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:39:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.35799999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574894,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297900,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447252,"statfs":{"total":95995035648,"available":95689986048,"internally_reserved":0,"allocated":626688,"data_stored":262958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:25 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.34899999999999998},"min":{"1min":0.26600000000000001,"5min":0.219,"15min":0.219},"max":{"1min":0.47699999999999998,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.372,"5min":0.36299999999999999,"15min":0.34899999999999998},"min":{"1min":0.25600000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.53100000000000003,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.39000000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:39:45 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.27200000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.59399999999999997,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.378},{"interface":"front","average":{"1min":0.377,"5min":0.39900000000000002,"15min":0.40200000000000002},"min":{"1min":0.27800000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.625,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.30499999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:39:09 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.433,"15min":0.437},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.55200000000000005,"5min":0.624,"15min":0.624},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.47499999999999998,"15min":0.47699999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.51800000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:39:28 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.57599999999999996,"15min":0.57799999999999996},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.57499999999999996,"15min":0.57299999999999995},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.83599999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.52900000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:39:50 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.59699999999999998,"15min":0.60399999999999998},"min":{"1min":0.39600000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.80600000000000005,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.622,"5min":0.60299999999999998,"15min":0.59999999999999998},"min":{"1min":0.41299999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.7290000000000001,"5min":1.7290000000000001,"15min":1.7290000000000001},"last":0.621}]},{"osd":6,"last update":"Mon Oct 9 05:39:12 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.57199999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:39:33 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.63500000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738419,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298480,"kb_used_data":1192,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446672,"statfs":{"total":95995035648,"available":95689392128,"internally_reserved":0,"allocated":1220608,"data_stored":853326,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:39:27 2023","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.36199999999999999,"15min":0.34499999999999997},"min":{"1min":0.26400000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.46500000000000002,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.19600000000000001},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.373,"15min":0.35299999999999998},"min":{"1min":0.25600000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.54800000000000004,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.25900000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:39:44 2023","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.433,"15min":0.43099999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.55,"5min":1.55,"15min":1.55},"last":0.25},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.436,"15min":0.437},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.58699999999999997,"5min":0.62,"15min":0.62},"last":0.29999999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:39:09 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.504,"15min":0.50900000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.70199999999999996,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.495,"5min":0.52700000000000002,"15min":0.53300000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.78700000000000003,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.33400000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:39:27 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.61599999999999999,"15min":0.622},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.84499999999999997,"5min":0.875,"15min":0.875},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.60499999999999998,"15min":0.60699999999999998},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.503}]},{"osd":5,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.63900000000000001,"15min":0.65100000000000002},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.72699999999999998,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.879,"5min":0.72999999999999998,"15min":0.70499999999999996},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":5.6539999999999999,"5min":5.6539999999999999,"15min":5.6539999999999999},"last":0.54300000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:39:09 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"min":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.46600000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:39:35 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.439},{"interface":"front","average":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":6.4660000000000002,"5min":6.4660000000000002,"15min":6.4660000000000002},"last":0.53200000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411368,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:42 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.371,"15min":0.36499999999999999},"min":{"1min":0.25600000000000001,"5min":0.255,"15min":0.255},"max":{"1min":0.64800000000000002,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.40500000000000003,"15min":0.38200000000000001},"min":{"1min":0.26400000000000001,"5min":0.254,"15min":0.254},"max":{"1min":2.746,"5min":2.746,"15min":2.746},"last":0.32700000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:39:42 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.38400000000000001,"15min":0.374},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.52900000000000003,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.434,"15min":0.41799999999999998},"min":{"1min":0.28899999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.929,"5min":1.929,"15min":1.929},"last":0.309}]},{"osd":3,"last update":"Mon Oct 9 05:39:08 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.46700000000000003,"15min":0.47099999999999997},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.46100000000000002,"15min":0.46400000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.58099999999999996,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.372}]},{"osd":4,"last update":"Mon Oct 9 05:39:32 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.60099999999999998,"15min":0.59899999999999998},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.60199999999999998,"15min":0.60299999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.77900000000000003,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.81200000000000006}]},{"osd":5,"last update":"Mon Oct 9 05:39:49 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.60399999999999998,"15min":0.60899999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.79500000000000004,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.61799999999999999,"15min":0.61299999999999999},"min":{"1min":0.46999999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.65700000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:39:13 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.68200000000000005}]},{"osd":7,"last update":"Mon Oct 9 05:39:32 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.83399999999999996},{"interface":"front","average":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":2.8210000000000002,"5min":2.8210000000000002,"15min":2.8210000000000002},"last":0.64500000000000002}]}]},{"osd":3,"up_from":25,"seq":107374182436,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:08 2023","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.58399999999999996,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.437,"5min":0.42799999999999999,"15min":0.42599999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.59699999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.27200000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:39:08 2023","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.46700000000000003,"15min":0.47199999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.314},{"interface":"front","average":{"1min":0.439,"5min":0.47099999999999997,"15min":0.47699999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.35899999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:39:08 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.50900000000000001,"15min":0.51700000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.47199999999999998,"15min":0.47399999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.29099999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:39:33 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.58599999999999997,"15min":0.58199999999999996},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.63100000000000001,"15min":0.623},"min":{"1min":0.48599999999999999,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.008,"5min":1.008,"15min":1.008},"last":0.50600000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:38:51 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"min":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.41599999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:39:13 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.45900000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:39:33 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.53600000000000003}]}]},{"osd":4,"up_from":30,"seq":128849018912,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:34 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.443,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.68899999999999995,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.438},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.58899999999999997,"15min":0.60799999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.629,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.68700000000000006}]},{"osd":1,"last update":"Mon Oct 9 05:39:34 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.56799999999999995,"15min":0.58099999999999996},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.71299999999999997,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.57999999999999996,"15min":0.58699999999999997},"min":{"1min":0.39700000000000002,"5min":0.377,"15min":0.377},"max":{"1min":0.84999999999999998,"5min":1.006,"15min":1.006},"last":0.55400000000000005}]},{"osd":2,"last update":"Mon Oct 9 05:39:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.57899999999999996,"15min":0.58899999999999997},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.79900000000000004,"5min":1.026,"15min":1.026},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.56599999999999995,"15min":0.57199999999999995},"min":{"1min":0.39900000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.52300000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:39:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.58399999999999996,"15min":0.59399999999999997},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.76400000000000001,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.626},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.57799999999999996,"15min":0.58299999999999996},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.77600000000000002,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.65100000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:39:53 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.435,"15min":0.42699999999999999},"min":{"1min":0.32100000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.44900000000000001,"15min":0.438},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.76700000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.61299999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:39:34 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.79900000000000004},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.78300000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757275,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.51400000000000001,"15min":0.51700000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.54800000000000004,"15min":0.52600000000000002},"min":{"1min":0.40300000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":3.2149999999999999,"5min":3.2149999999999999,"15min":3.2149999999999999},"last":0.40300000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.51600000000000001,"15min":0.51800000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.56299999999999994,"15min":0.53900000000000003},"min":{"1min":0.40699999999999997,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":3.2589999999999999,"5min":3.2589999999999999,"15min":3.2589999999999999},"last":0.49399999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.66900000000000004,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.44},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.53300000000000003,"15min":0.53600000000000003},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.34899999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.52400000000000002,"15min":0.52200000000000002},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.56100000000000005,"15min":0.55900000000000005},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.52700000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.42699999999999999,"15min":0.39500000000000002},"min":{"1min":0.314,"5min":0.312,"15min":0.312},"max":{"1min":3.5430000000000001,"5min":3.5430000000000001,"15min":3.5430000000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.41399999999999998,"15min":0.39700000000000002},"min":{"1min":0.36799999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.58899999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:39:12 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.56499999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:39:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.2869999999999999,"5min":3.2869999999999999,"15min":3.2869999999999999},"last":0.54100000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:40:01.813 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:40:01.966 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.731943162Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-10-09T05:40:01.966 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.757080443Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2023-10-09T05:40:01.966 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.782440176Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2023-10-09T05:40:01.967 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.807408172Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2023-10-09T05:40:01.967 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.849383393Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2023-10-09T05:40:01.967 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.87454704Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2023-10-09T05:40:01.967 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.899845865Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2023-10-09T05:40:01.967 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.925045995Z level=info msg="Executing migration" id="create provenance_type table" 2023-10-09T05:40:01.967 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:01 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:01.966907587Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2023-10-09T05:40:02.234 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.01740469Z level=info msg="Executing migration" id="create alert_image table" 2023-10-09T05:40:02.234 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.059158059Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2023-10-09T05:40:02.235 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.101247529Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2023-10-09T05:40:02.235 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.143115781Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2023-10-09T05:40:02.235 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.210074269Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2023-10-09T05:40:02.235 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.23519296Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2023-10-09T05:40:02.520 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:02 smithi156 ceph-mon[105979]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:02.520 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.235314817Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2023-10-09T05:40:02.520 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.285642502Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2023-10-09T05:40:02.520 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.310914075Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2023-10-09T05:40:02.520 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.335923729Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2023-10-09T05:40:02.521 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.336154254Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2023-10-09T05:40:02.521 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.336325592Z level=info msg="alerts found to migrate" alerts=0 2023-10-09T05:40:02.521 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.361264476Z level=info msg="Executing migration" id="create library_element table v1" 2023-10-09T05:40:02.521 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.403012319Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2023-10-09T05:40:02.521 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.453748409Z level=info msg="Executing migration" id="create library_element_connection table v1" 2023-10-09T05:40:02.521 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.478881898Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2023-10-09T05:40:02.521 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.52072294Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2023-10-09T05:40:02.718 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:02.718 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:02 smithi064 ceph-mon[101070]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:02.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:02 smithi064 ceph-mon[118354]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:02.719 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:40:02 smithi064 systemd[1]: Stopping Ceph alertmanager.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:40:02.788 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[109199]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:40:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:40:02.788 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.579615757Z level=info msg="Executing migration" id="increase max description length to 2048" 2023-10-09T05:40:02.788 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.629630234Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2023-10-09T05:40:02.788 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.663418993Z level=info msg="Executing migration" id="create data_keys table" 2023-10-09T05:40:02.788 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.713483593Z level=info msg="Executing migration" id="create secrets table" 2023-10-09T05:40:02.789 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.763736543Z level=info msg="Executing migration" id="rename data_keys name column to id" 2023-10-09T05:40:02.789 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.78878882Z level=info msg="Executing migration" id="add name column into data_keys" 2023-10-09T05:40:03.057 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.831008315Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2023-10-09T05:40:03.057 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.872947393Z level=info msg="Executing migration" id="rename data_keys name column to label" 2023-10-09T05:40:03.057 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.8981397Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2023-10-09T05:40:03.057 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.923308074Z level=info msg="Executing migration" id="create kv_store table v1" 2023-10-09T05:40:03.057 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.965415073Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2023-10-09T05:40:03.057 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:02 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:02.990386248Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2023-10-09T05:40:03.057 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.01547277Z level=info msg="Executing migration" id="create permission table" 2023-10-09T05:40:03.057 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.057518746Z level=info msg="Executing migration" id="add unique index permission.role_id" 2023-10-09T05:40:03.206 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:40:02 smithi064 conmon[163377]: ts=2023-10-09T05:40:02.913Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-10-09T05:40:03.207 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:40:03 smithi064 bash[173979]: 2d6f440aa702396dc31af1baad957fe84e3dda5d6bf5cab61c2668ca14241755 2023-10-09T05:40:03.316 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.082670961Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2023-10-09T05:40:03.317 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.108009879Z level=info msg="Executing migration" id="create role table" 2023-10-09T05:40:03.317 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.183301516Z level=info msg="Executing migration" id="add column display_name" 2023-10-09T05:40:03.317 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.208342209Z level=info msg="Executing migration" id="add column group_name" 2023-10-09T05:40:03.317 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.250404394Z level=info msg="Executing migration" id="add index role.org_id" 2023-10-09T05:40:03.317 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.275387726Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2023-10-09T05:40:03.317 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.317345507Z level=info msg="Executing migration" id="add index role_org_id_uid" 2023-10-09T05:40:03.585 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.351118047Z level=info msg="Executing migration" id="create team role table" 2023-10-09T05:40:03.586 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.376378546Z level=info msg="Executing migration" id="add index team_role.org_id" 2023-10-09T05:40:03.586 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.40150895Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2023-10-09T05:40:03.586 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.426710387Z level=info msg="Executing migration" id="add index team_role.team_id" 2023-10-09T05:40:03.586 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.451899183Z level=info msg="Executing migration" id="create user role table" 2023-10-09T05:40:03.586 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.477103287Z level=info msg="Executing migration" id="add index user_role.org_id" 2023-10-09T05:40:03.586 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.502456205Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2023-10-09T05:40:03.586 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.536129511Z level=info msg="Executing migration" id="add index user_role.user_id" 2023-10-09T05:40:03.586 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.561255692Z level=info msg="Executing migration" id="create builtin role table" 2023-10-09T05:40:03.846 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.58629336Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2023-10-09T05:40:03.846 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.637238392Z level=info msg="Executing migration" id="add index builtin_role.name" 2023-10-09T05:40:03.846 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.679194556Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2023-10-09T05:40:03.846 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.721038379Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2023-10-09T05:40:03.847 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.746277044Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2023-10-09T05:40:03.847 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.804962804Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2023-10-09T05:40:03.847 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.846880576Z level=info msg="Executing migration" id="add unique index role.uid" 2023-10-09T05:40:04.120 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.905652019Z level=info msg="Executing migration" id="create seed assignment table" 2023-10-09T05:40:04.120 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.930735847Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2023-10-09T05:40:04.120 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.972550454Z level=info msg="Executing migration" id="add column hidden to role table" 2023-10-09T05:40:04.120 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:03 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:03.998074536Z level=info msg="Executing migration" id="create query_history table v1" 2023-10-09T05:40:04.121 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.023209385Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2023-10-09T05:40:04.121 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.064985009Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2023-10-09T05:40:04.417 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.148756167Z level=info msg="Executing migration" id="teams permissions migration" 2023-10-09T05:40:04.417 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.190443736Z level=info msg="Executing migration" id="dashboard permissions" 2023-10-09T05:40:04.417 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.232482851Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2023-10-09T05:40:04.417 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.274415472Z level=info msg="Executing migration" id="drop managed folder create actions" 2023-10-09T05:40:04.417 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.299588304Z level=info msg="Executing migration" id="alerting notification permissions" 2023-10-09T05:40:04.417 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.324904358Z level=info msg="Executing migration" id="create query_history_star table v1" 2023-10-09T05:40:04.417 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.350157525Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2023-10-09T05:40:04.418 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.375236358Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2023-10-09T05:40:04.418 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.417204496Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2023-10-09T05:40:04.677 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:04 smithi156 ceph-mon[105979]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:04.677 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.4678633Z level=info msg="Executing migration" id="create correlation table v1" 2023-10-09T05:40:04.678 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.493209808Z level=info msg="Executing migration" id="add index correlations.uid" 2023-10-09T05:40:04.678 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.518420659Z level=info msg="Executing migration" id="add index correlations.source_uid" 2023-10-09T05:40:04.678 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.543455987Z level=info msg="Executing migration" id="add correlation config column" 2023-10-09T05:40:04.678 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.568662909Z level=info msg="Executing migration" id="create entity_events table" 2023-10-09T05:40:04.678 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.610759388Z level=info msg="Executing migration" id="create dashboard public config v1" 2023-10-09T05:40:04.678 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.635947982Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2023-10-09T05:40:04.678 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.636098737Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2023-10-09T05:40:04.678 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.677839932Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-10-09T05:40:04.678 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.677987857Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-10-09T05:40:04.700 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:04 smithi064 ceph-mon[101070]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:04.701 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:04 smithi064 ceph-mon[118354]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:04.701 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:40:04 smithi064 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T05:40:04.701 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:40:04 smithi064 systemd[1]: Stopped Ceph alertmanager.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:40:04.701 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:40:04 smithi064 systemd[1]: Starting Ceph alertmanager.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:40:04.963 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.728314283Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2023-10-09T05:40:04.963 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.762442029Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2023-10-09T05:40:04.964 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.804138169Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2023-10-09T05:40:04.964 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.829338201Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-10-09T05:40:04.964 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.854472213Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2023-10-09T05:40:04.964 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.879706826Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-10-09T05:40:04.964 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.921823319Z level=info msg="Executing migration" id="Drop public config table" 2023-10-09T05:40:04.964 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:04 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:04.963677245Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2023-10-09T05:40:05.112 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:40:05.116 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T05:40:05.232 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:05 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:05.005637849Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2023-10-09T05:40:05.232 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:05 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:05.072566859Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-10-09T05:40:05.233 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:05 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:05.097830615Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2023-10-09T05:40:05.233 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:05 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:05.122897597Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2023-10-09T05:40:05.233 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:05 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:05.148232298Z level=info msg="Executing migration" id="add annotations_enabled column" 2023-10-09T05:40:05.233 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:05 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:05.190064234Z level=info msg="Executing migration" id="add time_selection_enabled column" 2023-10-09T05:40:05.233 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:05 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:05.23200081Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2023-10-09T05:40:05.469 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:40:05 smithi064 conmon[174526]: ts=2023-10-09T05:40:05.144Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-10-09T05:40:05.469 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:40:05 smithi064 conmon[174526]: ts=2023-10-09T05:40:05.144Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-10-09T05:40:05.470 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:40:05 smithi064 conmon[174526]: ts=2023-10-09T05:40:05.145Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.64 port=9094 2023-10-09T05:40:05.470 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:40:05 smithi064 conmon[174526]: ts=2023-10-09T05:40:05.146Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T05:40:05.470 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:40:05 smithi064 bash[174448]: 51110671bb411847bbc25ea8af102e2d1069ab374c2b359f96a83395b9d4ea91 2023-10-09T05:40:05.470 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:40:05 smithi064 systemd[1]: Started Ceph alertmanager.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:40:05.470 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:40:05 smithi064 conmon[174526]: ts=2023-10-09T05:40:05.176Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T05:40:05.470 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:40:05 smithi064 conmon[174526]: ts=2023-10-09T05:40:05.176Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T05:40:05.470 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:40:05 smithi064 conmon[174526]: ts=2023-10-09T05:40:05.179Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-10-09T05:40:05.471 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:40:05 smithi064 conmon[174526]: ts=2023-10-09T05:40:05.179Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-10-09T05:40:05.485 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:05 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:05.282425131Z level=info msg="Executing migration" id="add share column" 2023-10-09T05:40:05.485 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:05 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:05.324396087Z level=info msg="Executing migration" id="create default alerting folders" 2023-10-09T05:40:05.485 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:05 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:05.349742569Z level=info msg="Executing migration" id="create file table" 2023-10-09T05:40:05.485 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:05 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:05.374941Z level=info msg="Executing migration" id="file table idx: path natural pk" 2023-10-09T05:40:05.486 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:05 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:05.416826019Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2023-10-09T05:40:05.486 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:05 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:05.441985582Z level=info msg="Executing migration" id="create file_meta table" 2023-10-09T05:40:05.486 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:05 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:05.483927581Z level=info msg="Executing migration" id="file table idx: path key" 2023-10-09T05:40:05.745 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:05 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:05.534582309Z level=info msg="Executing migration" id="set path collation in file table" 2023-10-09T05:40:05.745 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:05 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:05.576374038Z level=info msg="Executing migration" id="managed permissions migration" 2023-10-09T05:40:05.745 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:05 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:05.618817912Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2023-10-09T05:40:05.745 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:05 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:05.664822702Z level=info msg="Executing migration" id="RBAC action name migrator" 2023-10-09T05:40:05.745 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:05 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:05.745596128Z level=info msg="Executing migration" id="Add UID column to playlist" 2023-10-09T05:40:05.870 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":36,"stamp":"2023-10-09T05:40:03.547649+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2384952,"kb_used_data":6648,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747576264,"statfs":{"total":767960285184,"available":765518094336,"internally_reserved":0,"allocated":6807552,"data_stored":3875447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002170"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.609674+0000","last_change":"2023-10-09T05:39:18.963184+0000","last_active":"2023-10-09T05:39:20.609674+0000","last_peered":"2023-10-09T05:39:20.609674+0000","last_clean":"2023-10-09T05:39:20.609674+0000","last_became_active":"2023-10-09T05:39:18.962837+0000","last_became_peered":"2023-10-09T05:39:18.962837+0000","last_unstale":"2023-10-09T05:39:20.609674+0000","last_undegraded":"2023-10-09T05:39:20.609674+0000","last_fullsized":"2023-10-09T05:39:20.609674+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:05.818061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.609344+0000","last_change":"2023-10-09T05:39:19.405012+0000","last_active":"2023-10-09T05:39:20.609344+0000","last_peered":"2023-10-09T05:39:20.609344+0000","last_clean":"2023-10-09T05:39:20.609344+0000","last_became_active":"2023-10-09T05:39:19.404658+0000","last_became_peered":"2023-10-09T05:39:19.404658+0000","last_unstale":"2023-10-09T05:39:20.609344+0000","last_undegraded":"2023-10-09T05:39:20.609344+0000","last_fullsized":"2023-10-09T05:39:20.609344+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:24.956847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:39:19.964197+0000","last_change":"2023-10-09T05:39:19.405346+0000","last_active":"2023-10-09T05:39:19.964197+0000","last_peered":"2023-10-09T05:39:19.964197+0000","last_clean":"2023-10-09T05:39:19.964197+0000","last_became_active":"2023-10-09T05:39:19.404876+0000","last_became_peered":"2023-10-09T05:39:19.404876+0000","last_unstale":"2023-10-09T05:39:19.964197+0000","last_undegraded":"2023-10-09T05:39:19.964197+0000","last_fullsized":"2023-10-09T05:39:19.964197+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:28.442496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:39:19.964131+0000","last_change":"2023-10-09T05:39:18.964130+0000","last_active":"2023-10-09T05:39:19.964131+0000","last_peered":"2023-10-09T05:39:19.964131+0000","last_clean":"2023-10-09T05:39:19.964131+0000","last_became_active":"2023-10-09T05:39:18.963726+0000","last_became_peered":"2023-10-09T05:39:18.963726+0000","last_unstale":"2023-10-09T05:39:19.964131+0000","last_undegraded":"2023-10-09T05:39:19.964131+0000","last_fullsized":"2023-10-09T05:39:19.964131+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:13.428159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":271,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:39:19.964133+0000","last_change":"2023-10-09T05:38:35.869176+0000","last_active":"2023-10-09T05:39:19.964133+0000","last_peered":"2023-10-09T05:39:19.964133+0000","last_clean":"2023-10-09T05:39:19.964133+0000","last_became_active":"2023-10-09T05:38:35.868750+0000","last_became_peered":"2023-10-09T05:38:35.868750+0000","last_unstale":"2023-10-09T05:39:19.964133+0000","last_undegraded":"2023-10-09T05:39:19.964133+0000","last_fullsized":"2023-10-09T05:39:19.964133+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:20.200381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.054447+0000","last_change":"2023-10-09T05:39:18.963327+0000","last_active":"2023-10-09T05:39:20.054447+0000","last_peered":"2023-10-09T05:39:20.054447+0000","last_clean":"2023-10-09T05:39:20.054447+0000","last_became_active":"2023-10-09T05:39:18.962933+0000","last_became_peered":"2023-10-09T05:39:18.962933+0000","last_unstale":"2023-10-09T05:39:20.054447+0000","last_undegraded":"2023-10-09T05:39:20.054447+0000","last_fullsized":"2023-10-09T05:39:20.054447+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:43.427745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:39:19.964232+0000","last_change":"2023-10-09T05:39:19.362360+0000","last_active":"2023-10-09T05:39:19.964232+0000","last_peered":"2023-10-09T05:39:19.964232+0000","last_clean":"2023-10-09T05:39:19.964232+0000","last_became_active":"2023-10-09T05:39:19.361972+0000","last_became_peered":"2023-10-09T05:39:19.361972+0000","last_unstale":"2023-10-09T05:39:19.964232+0000","last_undegraded":"2023-10-09T05:39:19.964232+0000","last_fullsized":"2023-10-09T05:39:19.964232+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:06.194167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.054444+0000","last_change":"2023-10-09T05:39:18.963544+0000","last_active":"2023-10-09T05:39:20.054444+0000","last_peered":"2023-10-09T05:39:20.054444+0000","last_clean":"2023-10-09T05:39:20.054444+0000","last_became_active":"2023-10-09T05:39:18.962981+0000","last_became_peered":"2023-10-09T05:39:18.962981+0000","last_unstale":"2023-10-09T05:39:20.054444+0000","last_undegraded":"2023-10-09T05:39:20.054444+0000","last_fullsized":"2023-10-09T05:39:20.054444+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:11.589793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:39:20.609337+0000","last_change":"2023-10-09T05:39:18.964305+0000","last_active":"2023-10-09T05:39:20.609337+0000","last_peered":"2023-10-09T05:39:20.609337+0000","last_clean":"2023-10-09T05:39:20.609337+0000","last_became_active":"2023-10-09T05:39:18.963935+0000","last_became_peered":"2023-10-09T05:39:18.963935+0000","last_unstale":"2023-10-09T05:39:20.609337+0000","last_undegraded":"2023-10-09T05:39:20.609337+0000","last_fullsized":"2023-10-09T05:39:20.609337+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:33.105059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234004,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298480,"kb_used_data":1192,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446672,"statfs":{"total":95995035648,"available":95689392128,"internally_reserved":0,"allocated":1220608,"data_stored":853268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:36 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.505},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.628}]},{"osd":1,"last update":"Mon Oct 9 05:39:36 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.55800000000000005}]},{"osd":2,"last update":"Mon Oct 9 05:39:36 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.0780000000000001,"5min":1.0780000000000001,"15min":1.0780000000000001},"last":0.57199999999999995}]},{"osd":3,"last update":"Mon Oct 9 05:39:36 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.53300000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:39:36 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.45800000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:39:36 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.64400000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:39:36 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.51800000000000002}]}]},{"osd":6,"up_from":46,"seq":197568495640,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298476,"kb_used_data":1188,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446676,"statfs":{"total":95995035648,"available":95689396224,"internally_reserved":0,"allocated":1216512,"data_stored":853003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:10 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.51100000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:39:10 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.61399999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:39:10 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.52700000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:39:10 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.502}]},{"osd":4,"last update":"Mon Oct 9 05:39:10 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.56599999999999995}]},{"osd":5,"last update":"Mon Oct 9 05:39:10 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.65400000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:39:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.71299999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574895,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297900,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447252,"statfs":{"total":95995035648,"available":95689986048,"internally_reserved":0,"allocated":626688,"data_stored":262958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:25 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.34899999999999998},"min":{"1min":0.26600000000000001,"5min":0.219,"15min":0.219},"max":{"1min":0.47699999999999998,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.372,"5min":0.36299999999999999,"15min":0.34899999999999998},"min":{"1min":0.25600000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.53100000000000003,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.29399999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:39:45 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.27200000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.59399999999999997,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.377,"5min":0.39900000000000002,"15min":0.40200000000000002},"min":{"1min":0.27800000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.625,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.30099999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:39:09 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.433,"15min":0.437},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.55200000000000005,"5min":0.624,"15min":0.624},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.47499999999999998,"15min":0.47699999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.38800000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:39:28 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.57599999999999996,"15min":0.57799999999999996},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.57499999999999996,"15min":0.57299999999999995},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.83599999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.68300000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:39:50 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.59699999999999998,"15min":0.60399999999999998},"min":{"1min":0.39600000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.80600000000000005,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.622,"5min":0.60299999999999998,"15min":0.59999999999999998},"min":{"1min":0.41299999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.7290000000000001,"5min":1.7290000000000001,"15min":1.7290000000000001},"last":0.69499999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:39:12 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.64800000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:39:33 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.625},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.71199999999999997}]}]},{"osd":0,"up_from":8,"seq":34359738420,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298480,"kb_used_data":1192,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446672,"statfs":{"total":95995035648,"available":95689392128,"internally_reserved":0,"allocated":1220608,"data_stored":853326,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:39:27 2023","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.36199999999999999,"15min":0.34499999999999997},"min":{"1min":0.26400000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.46500000000000002,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.373,"15min":0.35299999999999998},"min":{"1min":0.25600000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.54800000000000004,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.50900000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:39:44 2023","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.433,"15min":0.43099999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.55,"5min":1.55,"15min":1.55},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.436,"15min":0.437},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.58699999999999997,"5min":0.62,"15min":0.62},"last":0.42999999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:39:09 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.504,"15min":0.50900000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.70199999999999996,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.495,"5min":0.52700000000000002,"15min":0.53300000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.78700000000000003,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.47999999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:39:27 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.61599999999999999,"15min":0.622},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.84499999999999997,"5min":0.875,"15min":0.875},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.60499999999999998,"15min":0.60699999999999998},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.56299999999999994}]},{"osd":5,"last update":"Mon Oct 9 05:39:48 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.63900000000000001,"15min":0.65100000000000002},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.72699999999999998,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.879,"5min":0.72999999999999998,"15min":0.70499999999999996},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":5.6539999999999999,"5min":5.6539999999999999,"15min":5.6539999999999999},"last":0.64400000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:39:09 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"min":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.63}]},{"osd":7,"last update":"Mon Oct 9 05:39:35 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":6.4660000000000002,"5min":6.4660000000000002,"15min":6.4660000000000002},"last":0.65300000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411369,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:42 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.371,"15min":0.36499999999999999},"min":{"1min":0.25600000000000001,"5min":0.255,"15min":0.255},"max":{"1min":0.64800000000000002,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.40500000000000003,"15min":0.38200000000000001},"min":{"1min":0.26400000000000001,"5min":0.254,"15min":0.254},"max":{"1min":2.746,"5min":2.746,"15min":2.746},"last":0.311}]},{"osd":1,"last update":"Mon Oct 9 05:39:42 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.38400000000000001,"15min":0.374},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.52900000000000003,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.434,"15min":0.41799999999999998},"min":{"1min":0.28899999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.929,"5min":1.929,"15min":1.929},"last":0.36599999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:39:08 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.46700000000000003,"15min":0.47099999999999997},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.46100000000000002,"15min":0.46400000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.58099999999999996,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.46000000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:39:32 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.60099999999999998,"15min":0.59899999999999998},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.60199999999999998,"15min":0.60299999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.77900000000000003,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.54100000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:39:49 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.60399999999999998,"15min":0.60899999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.79500000000000004,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.61799999999999999,"15min":0.61299999999999999},"min":{"1min":0.46999999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.65200000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:39:13 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.48799999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:39:32 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":2.8210000000000002,"5min":2.8210000000000002,"15min":2.8210000000000002},"last":0.50600000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182437,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:08 2023","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.58399999999999996,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.26000000000000001},{"interface":"front","average":{"1min":0.437,"5min":0.42799999999999999,"15min":0.42599999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.59699999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.307}]},{"osd":1,"last update":"Mon Oct 9 05:39:08 2023","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.46700000000000003,"15min":0.47199999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.439,"5min":0.47099999999999997,"15min":0.47699999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.33600000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:39:08 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.50900000000000001,"15min":0.51700000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.47199999999999998,"15min":0.47399999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.32200000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:39:33 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.58599999999999997,"15min":0.58199999999999996},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.63100000000000001,"15min":0.623},"min":{"1min":0.48599999999999999,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.008,"5min":1.008,"15min":1.008},"last":0.432}]},{"osd":5,"last update":"Mon Oct 9 05:39:56 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.65600000000000003,"15min":0.66800000000000004},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.70799999999999996,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.60499999999999998,"15min":0.61199999999999999},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.71799999999999997,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.47899999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:39:13 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.53200000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:39:33 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.55400000000000005}]}]},{"osd":4,"up_from":30,"seq":128849018913,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:34 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.443,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.68899999999999995,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.58899999999999997,"15min":0.60799999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.629,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.64200000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:39:34 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.56799999999999995,"15min":0.58099999999999996},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.71299999999999997,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.57999999999999996,"15min":0.58699999999999997},"min":{"1min":0.39700000000000002,"5min":0.377,"15min":0.377},"max":{"1min":0.84999999999999998,"5min":1.006,"15min":1.006},"last":0.374}]},{"osd":2,"last update":"Mon Oct 9 05:39:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.57899999999999996,"15min":0.58899999999999997},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.79900000000000004,"5min":1.026,"15min":1.026},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.56599999999999995,"15min":0.57199999999999995},"min":{"1min":0.39900000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.60799999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:39:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.58399999999999996,"15min":0.59399999999999997},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.76400000000000001,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.57799999999999996,"15min":0.58299999999999996},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.77600000000000002,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.625}]},{"osd":5,"last update":"Mon Oct 9 05:39:53 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.435,"15min":0.42699999999999999},"min":{"1min":0.32100000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.44900000000000001,"15min":0.438},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.57899999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:39:11 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.51000000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:39:34 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.66700000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757276,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.51400000000000001,"15min":0.51700000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.54800000000000004,"15min":0.52600000000000002},"min":{"1min":0.40300000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":3.2149999999999999,"5min":3.2149999999999999,"15min":3.2149999999999999},"last":0.60699999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.51600000000000001,"15min":0.51800000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.56299999999999994,"15min":0.53900000000000003},"min":{"1min":0.40699999999999997,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":3.2589999999999999,"5min":3.2589999999999999,"15min":3.2589999999999999},"last":0.53200000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.66900000000000004,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.53300000000000003,"15min":0.53600000000000003},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.67400000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.52400000000000002,"15min":0.52200000000000002},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.56100000000000005,"15min":0.55900000000000005},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.625}]},{"osd":4,"last update":"Mon Oct 9 05:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.42699999999999999,"15min":0.39500000000000002},"min":{"1min":0.314,"5min":0.312,"15min":0.312},"max":{"1min":3.5430000000000001,"5min":3.5430000000000001,"15min":3.5430000000000001},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.41399999999999998,"15min":0.39700000000000002},"min":{"1min":0.36799999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.42499999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:39:12 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.49299999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:39:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.2869999999999999,"5min":3.2869999999999999,"15min":3.2869999999999999},"last":0.57099999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:40:05.873 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T05:40:05.874 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T05:40:05.874 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T05:40:05.874 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T05:40:05.996 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:05 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:05.812669319Z level=info msg="Executing migration" id="Update uid column values in playlist" 2023-10-09T05:40:05.997 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:05 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:05.869257706Z level=info msg="Executing migration" id="Add index for uid in playlist" 2023-10-09T05:40:05.997 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:05 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:05.94444889Z level=info msg="Executing migration" id="update group index for alert rules" 2023-10-09T05:40:06.234 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:06.318 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:06 smithi156 ceph-mon[105979]: from='client.14631 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:40:06.319 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:06.319 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:06.319 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:06 smithi156 ceph-mon[105979]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T05:40:06.319 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:06 smithi156 ceph-mon[105979]: Reconfiguring daemon prometheus.a on smithi156 2023-10-09T05:40:06.319 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:06 smithi156 ceph-mon[105979]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:06.319 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:06.319 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:05 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:05.99708188Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2023-10-09T05:40:06.319 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:06 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:06.072726385Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2023-10-09T05:40:06.319 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:06 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:06.123147604Z level=info msg="Executing migration" id="add action column to seed_assignment" 2023-10-09T05:40:06.320 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:06 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:06.139995843Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2023-10-09T05:40:06.320 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:06 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:06.190428737Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2023-10-09T05:40:06.320 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:06 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:06.243264615Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2023-10-09T05:40:06.320 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:06 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:06.319196879Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2023-10-09T05:40:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:06 smithi064 ceph-mon[101070]: from='client.14631 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:40:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:06 smithi064 ceph-mon[101070]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T05:40:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:06 smithi064 ceph-mon[101070]: Reconfiguring daemon prometheus.a on smithi156 2023-10-09T05:40:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:06 smithi064 ceph-mon[101070]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:06.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:06 smithi064 ceph-mon[118354]: from='client.14631 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:40:06.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:06.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:06.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:06 smithi064 ceph-mon[118354]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T05:40:06.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:06 smithi064 ceph-mon[118354]: Reconfiguring daemon prometheus.a on smithi156 2023-10-09T05:40:06.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:06 smithi064 ceph-mon[118354]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:06.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:06.620 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:06 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:06.41114651Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2023-10-09T05:40:07.060 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:06 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:06.654905832Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2023-10-09T05:40:07.060 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:06 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:06.829661904Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2023-10-09T05:40:07.329 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:07 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:07.060824853Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2023-10-09T05:40:07.329 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:07 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:07.144801424Z level=info msg="Executing migration" id="create folder table" 2023-10-09T05:40:07.329 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:07 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:07.229143806Z level=info msg="Executing migration" id="Add index for parent_uid" 2023-10-09T05:40:07.330 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:07 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:07.287897921Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2023-10-09T05:40:07.330 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:07 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:07.329966817Z level=info msg="Executing migration" id="Update folder title length" 2023-10-09T05:40:07.469 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:40:07 smithi064 conmon[174526]: ts=2023-10-09T05:40:07.147Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000260647s 2023-10-09T05:40:07.580 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:07 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:07.405995282Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2023-10-09T05:40:07.580 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:07 smithi156 conmon[153225]: logger=migrator t=2023-10-09T05:40:07.452475321Z level=info msg="migrations completed" performed=485 skipped=0 duration=18.305206378s 2023-10-09T05:40:07.580 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:07 smithi156 conmon[153225]: logger=sqlstore t=2023-10-09T05:40:07.452964855Z level=info msg="Created default organization" 2023-10-09T05:40:07.580 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:07 smithi156 conmon[153225]: logger=plugin.loader t=2023-10-09T05:40:07.569741418Z level=info msg="Plugin registered" pluginID=input 2023-10-09T05:40:07.870 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:07 smithi156 conmon[153225]: logger=plugin.loader t=2023-10-09T05:40:07.599908577Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-10-09T05:40:07.870 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:07 smithi156 conmon[153225]: logger=plugin.loader t=2023-10-09T05:40:07.59995271Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-10-09T05:40:07.870 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:07 smithi156 conmon[153225]: logger=secrets t=2023-10-09T05:40:07.600008805Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-10-09T05:40:07.870 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:07 smithi156 conmon[153225]: logger=query_data t=2023-10-09T05:40:07.602339022Z level=info msg="Query Service initialization" 2023-10-09T05:40:07.870 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:07 smithi156 conmon[153225]: logger=live.push_http t=2023-10-09T05:40:07.605984034Z level=info msg="Live Push Gateway initialization" 2023-10-09T05:40:07.978 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:40:07.978 INFO:teuthology.orchestra.run.smithi064.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T05:40:08.561 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T05:40:08.561 INFO:tasks.cephadm:Setup complete, yielding 2023-10-09T05:40:08.562 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-10-09T05:40:08.572 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi064.front.sepia.ceph.com 2023-10-09T05:40:08.572 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-10-09T05:40:08.601 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:08 smithi156 systemd[1]: Stopping Ceph prometheus.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:40:08.601 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[153225]: logger=infra.usagestats.collector t=2023-10-09T05:40:08.405713628Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-10-09T05:40:08.601 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[153225]: logger=provisioning.datasources t=2023-10-09T05:40:08.406074795Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-10-09T05:40:08.601 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[153225]: logger=provisioning.datasources t=2023-10-09T05:40:08.439405774Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-09T05:40:08.602 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[153225]: logger=provisioning.datasources t=2023-10-09T05:40:08.504374815Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2023-10-09T05:40:08.602 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[153225]: logger=provisioning.alerting t=2023-10-09T05:40:08.552901056Z level=info msg="starting to provision alerting" 2023-10-09T05:40:08.602 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[153225]: logger=provisioning.alerting t=2023-10-09T05:40:08.552934572Z level=info msg="finished to provision alerting" 2023-10-09T05:40:08.602 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[153225]: logger=grafanaStorageLogger t=2023-10-09T05:40:08.553143432Z level=info msg="storage starting" 2023-10-09T05:40:08.602 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[153225]: logger=ngalert.state.manager t=2023-10-09T05:40:08.553542947Z level=info msg="Warming state cache for startup" 2023-10-09T05:40:08.602 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[153225]: logger=http.server t=2023-10-09T05:40:08.55518164Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-10-09T05:40:08.602 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[153225]: logger=sqlstore.transactions t=2023-10-09T05:40:08.564379378Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-10-09T05:40:08.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:08 smithi156 ceph-mon[105979]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:08.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:08 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3636881497' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:40:08.871 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[149396]: ts=2023-10-09T05:40:08.642Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T05:40:08.871 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[149396]: ts=2023-10-09T05:40:08.642Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-10-09T05:40:08.871 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[149396]: ts=2023-10-09T05:40:08.642Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-10-09T05:40:08.871 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[149396]: ts=2023-10-09T05:40:08.642Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T05:40:08.871 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[149396]: ts=2023-10-09T05:40:08.642Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-10-09T05:40:08.871 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[149396]: ts=2023-10-09T05:40:08.642Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-10-09T05:40:08.871 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[149396]: ts=2023-10-09T05:40:08.642Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T05:40:08.871 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[149396]: ts=2023-10-09T05:40:08.642Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-10-09T05:40:08.871 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[149396]: ts=2023-10-09T05:40:08.642Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-10-09T05:40:08.872 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[149396]: ts=2023-10-09T05:40:08.642Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T05:40:08.872 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[149396]: ts=2023-10-09T05:40:08.642Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-10-09T05:40:08.872 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[149396]: ts=2023-10-09T05:40:08.642Z caller=main.go:1147 level=info msg="See you next time!" 2023-10-09T05:40:08.872 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[153225]: logger=ngalert.state.manager t=2023-10-09T05:40:08.6998621Z level=info msg="State cache has been initialized" states=0 duration=146.313518ms 2023-10-09T05:40:08.872 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[153225]: logger=ticker t=2023-10-09T05:40:08.700206755Z level=info msg=starting first_tick=2023-10-09T05:40:10Z 2023-10-09T05:40:08.872 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[153225]: logger=ngalert.multiorg.alertmanager t=2023-10-09T05:40:08.700189857Z level=info msg="starting MultiOrg Alertmanager" 2023-10-09T05:40:08.872 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:40:08 smithi156 conmon[153225]: logger=sqlstore.transactions t=2023-10-09T05:40:08.811248485Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-10-09T05:40:08.931 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:08.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:08 smithi064 ceph-mon[101070]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:08.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:08 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3636881497' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:40:08.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:08 smithi064 ceph-mon[118354]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:08.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:08 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3636881497' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:40:09.620 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:09 smithi156 bash[154771]: 7dc2d972520016764a5f34e34b2154879e137b132445c7d6497e7c2d74d1856f 2023-10-09T05:40:09.620 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:09 smithi156 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T05:40:09.620 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:09 smithi156 systemd[1]: Stopped Ceph prometheus.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:40:09.620 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:09 smithi156 systemd[1]: Starting Ceph prometheus.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:40:10.265 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:40:10 smithi064 conmon[101498]: [09/Oct/2023:05:40:10] ENGINE Bus STOPPING 2023-10-09T05:40:10.265 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:40:10 smithi064 conmon[101498]: [09/Oct/2023:05:40:10] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:40:10.265 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:40:10 smithi064 conmon[101498]: [09/Oct/2023:05:40:10] ENGINE Bus STOPPED 2023-10-09T05:40:10.265 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:40:10 smithi064 conmon[101498]: [09/Oct/2023:05:40:10] ENGINE Bus STARTING 2023-10-09T05:40:10.370 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:10 smithi156 bash[155105]: d46b6df5107c15c1115e96f578b6fd1a60a27672491d3e080e0257f24e71add8 2023-10-09T05:40:10.370 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:10 smithi156 conmon[155147]: ts=2023-10-09T05:40:10.102Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-10-09T05:40:10.370 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:10 smithi156 conmon[155147]: ts=2023-10-09T05:40:10.102Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-10-09T05:40:10.371 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:10 smithi156 conmon[155147]: ts=2023-10-09T05:40:10.102Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi156 (none))" 2023-10-09T05:40:10.371 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:10 smithi156 conmon[155147]: ts=2023-10-09T05:40:10.102Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T05:40:10.371 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:10 smithi156 conmon[155147]: ts=2023-10-09T05:40:10.102Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T05:40:10.371 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:10 smithi156 conmon[155147]: ts=2023-10-09T05:40:10.104Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T05:40:10.371 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:10 smithi156 conmon[155147]: ts=2023-10-09T05:40:10.105Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-10-09T05:40:10.371 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:10 smithi156 conmon[155147]: ts=2023-10-09T05:40:10.106Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-10-09T05:40:10.371 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:10 smithi156 conmon[155147]: ts=2023-10-09T05:40:10.106Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-10-09T05:40:10.371 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:10 smithi156 systemd[1]: Started Ceph prometheus.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:40:10.371 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:10 smithi156 conmon[155147]: ts=2023-10-09T05:40:10.108Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T05:40:10.371 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:10 smithi156 conmon[155147]: ts=2023-10-09T05:40:10.108Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.615µs 2023-10-09T05:40:10.371 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:10 smithi156 conmon[155147]: ts=2023-10-09T05:40:10.108Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T05:40:10.372 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:10 smithi156 conmon[155147]: ts=2023-10-09T05:40:10.108Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-10-09T05:40:10.372 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:10 smithi156 conmon[155147]: ts=2023-10-09T05:40:10.109Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-10-09T05:40:10.372 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:10 smithi156 conmon[155147]: ts=2023-10-09T05:40:10.109Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=21.617µs wal_replay_duration=704.161µs wbl_replay_duration=128ns total_replay_duration=747.677µs 2023-10-09T05:40:10.372 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:10 smithi156 conmon[155147]: ts=2023-10-09T05:40:10.110Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T05:40:10.372 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:10 smithi156 conmon[155147]: ts=2023-10-09T05:40:10.110Z caller=main.go:1029 level=info msg="TSDB started" 2023-10-09T05:40:10.372 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:10 smithi156 conmon[155147]: ts=2023-10-09T05:40:10.110Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T05:40:10.372 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:10 smithi156 conmon[155147]: ts=2023-10-09T05:40:10.128Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.594723ms db_storage=930ns remote_storage=1.75µs web_handler=498ns query_engine=821ns scrape=641.319µs scrape_sd=38.521µs notify=27.893µs notify_sd=8.757µs rules=17.573958ms tracing=6.527µs 2023-10-09T05:40:10.372 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:10 smithi156 conmon[155147]: ts=2023-10-09T05:40:10.128Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-10-09T05:40:10.372 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:40:10 smithi156 conmon[155147]: ts=2023-10-09T05:40:10.128Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-10-09T05:40:10.533 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:40:10 smithi064 conmon[101498]: [09/Oct/2023:05:40:10] ENGINE Serving on http://:::9283 2023-10-09T05:40:10.533 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:40:10 smithi064 conmon[101498]: [09/Oct/2023:05:40:10] ENGINE Bus STARTED 2023-10-09T05:40:10.533 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:40:10 smithi064 conmon[101498]: [09/Oct/2023:05:40:10] ENGINE Bus STOPPING 2023-10-09T05:40:10.533 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:40:10 smithi064 conmon[101498]: [09/Oct/2023:05:40:10] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:40:10.533 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:40:10 smithi064 conmon[101498]: [09/Oct/2023:05:40:10] ENGINE Bus STOPPED 2023-10-09T05:40:10.534 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:40:10 smithi064 conmon[101498]: [09/Oct/2023:05:40:10] ENGINE Bus STARTING 2023-10-09T05:40:10.534 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:40:10 smithi064 conmon[101498]: [09/Oct/2023:05:40:10] ENGINE Serving on http://:::9283 2023-10-09T05:40:10.534 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:40:10 smithi064 conmon[101498]: [09/Oct/2023:05:40:10] ENGINE Bus STARTED 2023-10-09T05:40:10.534 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:40:10 smithi064 conmon[101498]: [09/Oct/2023:05:40:10] ENGINE Bus STOPPING 2023-10-09T05:40:10.534 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:40:10 smithi064 conmon[101498]: [09/Oct/2023:05:40:10] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:40:10.534 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:40:10 smithi064 conmon[101498]: [09/Oct/2023:05:40:10] ENGINE Bus STOPPED 2023-10-09T05:40:10.534 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:40:10 smithi064 conmon[101498]: [09/Oct/2023:05:40:10] ENGINE Bus STARTING 2023-10-09T05:40:10.534 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:40:10 smithi064 conmon[101498]: [09/Oct/2023:05:40:10] ENGINE Serving on http://:::9283 2023-10-09T05:40:10.534 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:40:10 smithi064 conmon[101498]: [09/Oct/2023:05:40:10] ENGINE Bus STARTED 2023-10-09T05:40:10.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:10 smithi156 ceph-mon[105979]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:10.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:10 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:10.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:10 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:10.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:10 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:40:10.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:10 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi064:9093"}]: dispatch 2023-10-09T05:40:10.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:10 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:10.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:10 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:40:10.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:10 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi156.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:40:10.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:10 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:10.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:10 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:40:10.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:10 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi156.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T05:40:10.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:10 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:10.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:10 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:40:10.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[101070]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:10.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:10.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:10.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:40:10.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi064:9093"}]: dispatch 2023-10-09T05:40:10.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:10.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:40:10.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi156.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:40:10.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:10.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:40:10.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi156.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T05:40:10.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:10.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:40:10.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[118354]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:10.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:10.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:10.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:40:10.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi064:9093"}]: dispatch 2023-10-09T05:40:10.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:10.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:40:10.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi156.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:40:10.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:10.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:40:10.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi156.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T05:40:10.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:10.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:10 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:40:11.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:11 smithi156 ceph-mon[105979]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:40:11.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:11 smithi156 ceph-mon[105979]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi064:9093"}]: dispatch 2023-10-09T05:40:11.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:11 smithi156 ceph-mon[105979]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:40:11.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:11 smithi156 ceph-mon[105979]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi156.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:40:11.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:11 smithi156 ceph-mon[105979]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:40:11.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:11 smithi156 ceph-mon[105979]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi156.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T05:40:11.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:11 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T05:40:11.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:11 smithi064 ceph-mon[101070]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:40:11.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:11 smithi064 ceph-mon[101070]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi064:9093"}]: dispatch 2023-10-09T05:40:11.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:11 smithi064 ceph-mon[101070]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:40:11.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:11 smithi064 ceph-mon[101070]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi156.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:40:11.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:11 smithi064 ceph-mon[101070]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:40:11.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:11 smithi064 ceph-mon[101070]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi156.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T05:40:11.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:11 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T05:40:11.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:11 smithi064 ceph-mon[118354]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:40:11.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:11 smithi064 ceph-mon[118354]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi064:9093"}]: dispatch 2023-10-09T05:40:11.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:11 smithi064 ceph-mon[118354]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:40:11.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:11 smithi064 ceph-mon[118354]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi156.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:40:11.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:11 smithi064 ceph-mon[118354]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:40:11.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:11 smithi064 ceph-mon[118354]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi156.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T05:40:11.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:11 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T05:40:12.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:12 smithi064 ceph-mon[118354]: from='client.14643 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:40:12.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:12 smithi064 ceph-mon[118354]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:12.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:12 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T05:40:12.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:12 smithi064 ceph-mon[118354]: osdmap e64: 8 total, 8 up, 8 in 2023-10-09T05:40:12.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:12 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T05:40:12.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:12 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:12.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:12 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:12.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:12 smithi064 ceph-mon[101070]: from='client.14643 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:40:12.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:12 smithi064 ceph-mon[101070]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:12.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:12 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T05:40:12.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:12 smithi064 ceph-mon[101070]: osdmap e64: 8 total, 8 up, 8 in 2023-10-09T05:40:12.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:12 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T05:40:12.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:12 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:12.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:12 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:12.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:12 smithi064 conmon[101046]: 2023-10-09T05:40:12.634+0000 7fbea77de700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T05:40:13.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:12 smithi156 ceph-mon[105979]: from='client.14643 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:40:13.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:12 smithi156 ceph-mon[105979]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:13.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:12 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T05:40:13.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:12 smithi156 ceph-mon[105979]: osdmap e64: 8 total, 8 up, 8 in 2023-10-09T05:40:13.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:12 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T05:40:13.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:12 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:13.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:12 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:13.455 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-10-09T05:40:13.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T05:40:13.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[118354]: osdmap e65: 8 total, 8 up, 8 in 2023-10-09T05:40:13.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T05:40:13.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[118354]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T05:40:13.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[118354]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T05:40:13.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[118354]: osdmap e66: 8 total, 8 up, 8 in 2023-10-09T05:40:13.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T05:40:13.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[118354]: fsmap foo:0 2023-10-09T05:40:13.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:13.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:13.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:13.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:40:13.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:40:13.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:13.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.gmrwsy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:40:13.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.gmrwsy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T05:40:13.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:40:13.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T05:40:13.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[101070]: osdmap e65: 8 total, 8 up, 8 in 2023-10-09T05:40:13.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T05:40:13.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[101070]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T05:40:13.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[101070]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T05:40:13.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[101070]: osdmap e66: 8 total, 8 up, 8 in 2023-10-09T05:40:13.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T05:40:13.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[101070]: fsmap foo:0 2023-10-09T05:40:13.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:13.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:13.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:13.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:40:13.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:40:13.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:13.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.gmrwsy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:40:13.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.gmrwsy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T05:40:13.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:13 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:40:13.847 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:14.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:13 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T05:40:14.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:13 smithi156 ceph-mon[105979]: osdmap e65: 8 total, 8 up, 8 in 2023-10-09T05:40:14.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:13 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T05:40:14.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:13 smithi156 ceph-mon[105979]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T05:40:14.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:13 smithi156 ceph-mon[105979]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T05:40:14.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:13 smithi156 ceph-mon[105979]: osdmap e66: 8 total, 8 up, 8 in 2023-10-09T05:40:14.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:13 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T05:40:14.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:13 smithi156 ceph-mon[105979]: fsmap foo:0 2023-10-09T05:40:14.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:13 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:14.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:13 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:14.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:13 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:14.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:13 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:40:14.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:13 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:40:14.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:13 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:14.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:13 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.gmrwsy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:40:14.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:13 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.gmrwsy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T05:40:14.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:13 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:40:14.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:14 smithi064 ceph-mon[101070]: Saving service mds.foo spec with placement count:2 2023-10-09T05:40:14.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:14 smithi064 ceph-mon[101070]: Deploying daemon mds.foo.smithi064.gmrwsy on smithi064 2023-10-09T05:40:14.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:14 smithi064 ceph-mon[101070]: pgmap v44: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:14.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:14 smithi064 ceph-mon[101070]: osdmap e67: 8 total, 8 up, 8 in 2023-10-09T05:40:14.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:14 smithi064 ceph-mon[118354]: Saving service mds.foo spec with placement count:2 2023-10-09T05:40:14.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:14 smithi064 ceph-mon[118354]: Deploying daemon mds.foo.smithi064.gmrwsy on smithi064 2023-10-09T05:40:14.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:14 smithi064 ceph-mon[118354]: pgmap v44: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:14.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:14 smithi064 ceph-mon[118354]: osdmap e67: 8 total, 8 up, 8 in 2023-10-09T05:40:15.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:14 smithi156 ceph-mon[105979]: Saving service mds.foo spec with placement count:2 2023-10-09T05:40:15.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:14 smithi156 ceph-mon[105979]: Deploying daemon mds.foo.smithi064.gmrwsy on smithi064 2023-10-09T05:40:15.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:14 smithi156 ceph-mon[105979]: pgmap v44: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:15.124 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:14 smithi156 ceph-mon[105979]: osdmap e67: 8 total, 8 up, 8 in 2023-10-09T05:40:15.469 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:40:15 smithi064 conmon[174526]: ts=2023-10-09T05:40:15.149Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002425889s 2023-10-09T05:40:15.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:15 smithi064 ceph-mon[101070]: osdmap e68: 8 total, 8 up, 8 in 2023-10-09T05:40:15.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:15 smithi064 ceph-mon[101070]: pgmap v47: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:15.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:15 smithi064 ceph-mon[118354]: osdmap e68: 8 total, 8 up, 8 in 2023-10-09T05:40:15.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:15 smithi064 ceph-mon[118354]: pgmap v47: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:16.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:15 smithi156 ceph-mon[105979]: osdmap e68: 8 total, 8 up, 8 in 2023-10-09T05:40:16.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:15 smithi156 ceph-mon[105979]: pgmap v47: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:17.224 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-10-09T05:40:17.393 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:40:17 smithi064 conmon[101498]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:40:17] "GET /metrics HTTP/1.1" 200 203225 "" "Prometheus/2.43.0" 2023-10-09T05:40:17.610 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:18.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[118354]: mds.? [v2:172.21.15.64:6834/3975184368,v1:172.21.15.64:6835/3975184368] up:boot 2023-10-09T05:40:18.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[118354]: daemon mds.foo.smithi064.gmrwsy assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T05:40:18.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[118354]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T05:40:18.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[118354]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T05:40:18.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[118354]: Cluster is now healthy 2023-10-09T05:40:18.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[118354]: fsmap foo:0 1 up:standby 2023-10-09T05:40:18.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.gmrwsy"}]: dispatch 2023-10-09T05:40:18.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[118354]: fsmap foo:1 {0=foo.smithi064.gmrwsy=up:creating} 2023-10-09T05:40:18.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:18.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:18.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[118354]: daemon mds.foo.smithi064.gmrwsy is now active in filesystem foo as rank 0 2023-10-09T05:40:18.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:18.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.pizbdw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:40:18.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.pizbdw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T05:40:18.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:40:18.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[118354]: Deploying daemon mds.foo.smithi156.pizbdw on smithi156 2023-10-09T05:40:18.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[118354]: pgmap v48: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:18.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[101070]: mds.? [v2:172.21.15.64:6834/3975184368,v1:172.21.15.64:6835/3975184368] up:boot 2023-10-09T05:40:18.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[101070]: daemon mds.foo.smithi064.gmrwsy assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T05:40:18.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[101070]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T05:40:18.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[101070]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T05:40:18.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[101070]: Cluster is now healthy 2023-10-09T05:40:18.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[101070]: fsmap foo:0 1 up:standby 2023-10-09T05:40:18.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.gmrwsy"}]: dispatch 2023-10-09T05:40:18.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[101070]: fsmap foo:1 {0=foo.smithi064.gmrwsy=up:creating} 2023-10-09T05:40:18.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:18.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:18.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[101070]: daemon mds.foo.smithi064.gmrwsy is now active in filesystem foo as rank 0 2023-10-09T05:40:18.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:18.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.pizbdw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:40:18.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.pizbdw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T05:40:18.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:40:18.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[101070]: Deploying daemon mds.foo.smithi156.pizbdw on smithi156 2023-10-09T05:40:18.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:18 smithi064 ceph-mon[101070]: pgmap v48: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:18.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:18 smithi156 ceph-mon[105979]: mds.? [v2:172.21.15.64:6834/3975184368,v1:172.21.15.64:6835/3975184368] up:boot 2023-10-09T05:40:18.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:18 smithi156 ceph-mon[105979]: daemon mds.foo.smithi064.gmrwsy assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T05:40:18.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:18 smithi156 ceph-mon[105979]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T05:40:18.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:18 smithi156 ceph-mon[105979]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T05:40:18.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:18 smithi156 ceph-mon[105979]: Cluster is now healthy 2023-10-09T05:40:18.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:18 smithi156 ceph-mon[105979]: fsmap foo:0 1 up:standby 2023-10-09T05:40:18.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:18 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.gmrwsy"}]: dispatch 2023-10-09T05:40:18.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:18 smithi156 ceph-mon[105979]: fsmap foo:1 {0=foo.smithi064.gmrwsy=up:creating} 2023-10-09T05:40:18.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:18 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:18.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:18 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:18.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:18 smithi156 ceph-mon[105979]: daemon mds.foo.smithi064.gmrwsy is now active in filesystem foo as rank 0 2023-10-09T05:40:18.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:18 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:18.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:18 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.pizbdw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:40:18.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:18 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.pizbdw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T05:40:18.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:18 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:40:18.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:18 smithi156 ceph-mon[105979]: Deploying daemon mds.foo.smithi156.pizbdw on smithi156 2023-10-09T05:40:18.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:18 smithi156 ceph-mon[105979]: pgmap v48: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:19.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:19 smithi064 ceph-mon[118354]: mds.? [v2:172.21.15.64:6834/3975184368,v1:172.21.15.64:6835/3975184368] up:active 2023-10-09T05:40:19.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:19 smithi064 ceph-mon[118354]: fsmap foo:1 {0=foo.smithi064.gmrwsy=up:active} 2023-10-09T05:40:19.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:19 smithi064 ceph-mon[101070]: mds.? [v2:172.21.15.64:6834/3975184368,v1:172.21.15.64:6835/3975184368] up:active 2023-10-09T05:40:19.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:19 smithi064 ceph-mon[101070]: fsmap foo:1 {0=foo.smithi064.gmrwsy=up:active} 2023-10-09T05:40:19.757 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:19 smithi156 ceph-mon[105979]: mds.? [v2:172.21.15.64:6834/3975184368,v1:172.21.15.64:6835/3975184368] up:active 2023-10-09T05:40:19.758 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:19 smithi156 ceph-mon[105979]: fsmap foo:1 {0=foo.smithi064.gmrwsy=up:active} 2023-10-09T05:40:20.109 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-09T05:40:20.119 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T05:40:20.119 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T05:40:20.119 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T05:40:20.591 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:20.843 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:20 smithi064 ceph-mon[118354]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:20.843 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:20 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:20.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:20 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:20.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:20 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:20.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:20 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:20.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:20 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:20.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:20 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:40:20.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:20 smithi064 ceph-mon[118354]: mds.? [v2:172.21.15.156:6832/4137868430,v1:172.21.15.156:6833/4137868430] up:boot 2023-10-09T05:40:20.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:20 smithi064 ceph-mon[118354]: fsmap foo:1 {0=foo.smithi064.gmrwsy=up:active} 1 up:standby 2023-10-09T05:40:20.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:20 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.pizbdw"}]: dispatch 2023-10-09T05:40:20.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:20 smithi064 ceph-mon[101070]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:20.845 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:20 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:20.845 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:20 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:20.845 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:20 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:20.845 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:20 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:20.845 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:20 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:20.845 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:20 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:40:20.845 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:20 smithi064 ceph-mon[101070]: mds.? [v2:172.21.15.156:6832/4137868430,v1:172.21.15.156:6833/4137868430] up:boot 2023-10-09T05:40:20.845 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:20 smithi064 ceph-mon[101070]: fsmap foo:1 {0=foo.smithi064.gmrwsy=up:active} 1 up:standby 2023-10-09T05:40:20.845 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:20 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.pizbdw"}]: dispatch 2023-10-09T05:40:20.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:20 smithi156 ceph-mon[105979]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:20.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:20 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:20.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:20 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:20.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:20 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:20.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:20 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:20.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:20 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:20.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:20 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:40:20.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:20 smithi156 ceph-mon[105979]: mds.? [v2:172.21.15.156:6832/4137868430,v1:172.21.15.156:6833/4137868430] up:boot 2023-10-09T05:40:20.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:20 smithi156 ceph-mon[105979]: fsmap foo:1 {0=foo.smithi064.gmrwsy=up:active} 1 up:standby 2023-10-09T05:40:20.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:20 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.pizbdw"}]: dispatch 2023-10-09T05:40:21.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:21 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:21.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:21 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:21.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:21 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:21.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:21 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:21.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:21 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:21.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:21 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:22.761 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:40:22.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:22 smithi064 ceph-mon[101070]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2023-10-09T05:40:22.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:22 smithi064 ceph-mon[101070]: mds.? [v2:172.21.15.64:6834/3975184368,v1:172.21.15.64:6835/3975184368] up:active 2023-10-09T05:40:22.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:22 smithi064 ceph-mon[101070]: fsmap foo:1 {0=foo.smithi064.gmrwsy=up:active} 1 up:standby 2023-10-09T05:40:22.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:22 smithi064 ceph-mon[118354]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2023-10-09T05:40:22.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:22 smithi064 ceph-mon[118354]: mds.? [v2:172.21.15.64:6834/3975184368,v1:172.21.15.64:6835/3975184368] up:active 2023-10-09T05:40:22.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:22 smithi064 ceph-mon[118354]: fsmap foo:1 {0=foo.smithi064.gmrwsy=up:active} 1 up:standby 2023-10-09T05:40:23.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:40:22 smithi156 conmon[109199]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:40:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:40:23.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:22 smithi156 ceph-mon[105979]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2023-10-09T05:40:23.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:22 smithi156 ceph-mon[105979]: mds.? [v2:172.21.15.64:6834/3975184368,v1:172.21.15.64:6835/3975184368] up:active 2023-10-09T05:40:23.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:22 smithi156 ceph-mon[105979]: fsmap foo:1 {0=foo.smithi064.gmrwsy=up:active} 1 up:standby 2023-10-09T05:40:23.563 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":20,"active_gid":24341,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":984813922},{"type":"v1","addr":"172.21.15.64:6801","nonce":984813922}]},"active_addr":"172.21.15.64:6801/984813922","active_change":"2023-10-09T05:39:05.525488+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24346,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":3038747921}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":1571829871}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":3029574061}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":1951412611}]}]} 2023-10-09T05:40:23.568 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T05:40:23.569 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T05:40:23.569 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:40:23.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:23 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/51905246' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:40:23.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:23 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:23.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:23 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:23.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:23 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/51905246' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:40:23.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:23 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:23.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:23 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:23.994 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:24.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:23 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/51905246' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:40:24.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:23 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:24.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:23 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:24.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:24 smithi064 ceph-mon[101070]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2023-10-09T05:40:24.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:24 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:24.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:24 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:24.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:24 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:40:24.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:24 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:40:24.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:24 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:24.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:24 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:40:24.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:24 smithi064 ceph-mon[118354]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2023-10-09T05:40:24.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:24 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:24.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:24 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:24.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:24 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:40:24.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:24 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:40:24.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:24 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:24.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:24 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:40:25.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:24 smithi156 ceph-mon[105979]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2023-10-09T05:40:25.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:24 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:25.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:24 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:25.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:24 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:40:25.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:24 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:40:25.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:24 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:25.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:24 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:40:25.812 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:40:25.813 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":68,"fsid":"44c77acc-6665-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:33:40.037198+0000","modified":"2023-10-09T05:40:14.644959+0000","last_up_change":"2023-10-09T05:38:31.856610+0000","last_in_change":"2023-10-09T05:38:14.961182+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:36:42.276784+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:39:17.895056+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T05:40:11.204791+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T05:40:12.067097+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"85d0857a-ca32-43d2-8cec-bbd916db9f98","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2019939615},{"type":"v1","addr":"172.21.15.64:6803","nonce":2019939615}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2019939615},{"type":"v1","addr":"172.21.15.64:6805","nonce":2019939615}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2019939615},{"type":"v1","addr":"172.21.15.64:6809","nonce":2019939615}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2019939615},{"type":"v1","addr":"172.21.15.64:6807","nonce":2019939615}]},"public_addr":"172.21.15.64:6803/2019939615","cluster_addr":"172.21.15.64:6805/2019939615","heartbeat_back_addr":"172.21.15.64:6809/2019939615","heartbeat_front_addr":"172.21.15.64:6807/2019939615","state":["exists","up"]},{"osd":1,"uuid":"05570845-2cd8-4e24-a2f5-f46cf67aa31a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":1958566614},{"type":"v1","addr":"172.21.15.64:6811","nonce":1958566614}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":1958566614},{"type":"v1","addr":"172.21.15.64:6813","nonce":1958566614}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1958566614},{"type":"v1","addr":"172.21.15.64:6817","nonce":1958566614}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":1958566614},{"type":"v1","addr":"172.21.15.64:6815","nonce":1958566614}]},"public_addr":"172.21.15.64:6811/1958566614","cluster_addr":"172.21.15.64:6813/1958566614","heartbeat_back_addr":"172.21.15.64:6817/1958566614","heartbeat_front_addr":"172.21.15.64:6815/1958566614","state":["exists","up"]},{"osd":2,"uuid":"0f306460-9f08-4f15-b573-3587f63be61c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":2050077610},{"type":"v1","addr":"172.21.15.64:6819","nonce":2050077610}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":2050077610},{"type":"v1","addr":"172.21.15.64:6821","nonce":2050077610}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2050077610},{"type":"v1","addr":"172.21.15.64:6825","nonce":2050077610}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":2050077610},{"type":"v1","addr":"172.21.15.64:6823","nonce":2050077610}]},"public_addr":"172.21.15.64:6819/2050077610","cluster_addr":"172.21.15.64:6821/2050077610","heartbeat_back_addr":"172.21.15.64:6825/2050077610","heartbeat_front_addr":"172.21.15.64:6823/2050077610","state":["exists","up"]},{"osd":3,"uuid":"4bf38edd-e555-402e-ba4b-c751de06aa6a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":3037106123},{"type":"v1","addr":"172.21.15.64:6827","nonce":3037106123}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":3037106123},{"type":"v1","addr":"172.21.15.64:6829","nonce":3037106123}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":3037106123},{"type":"v1","addr":"172.21.15.64:6833","nonce":3037106123}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":3037106123},{"type":"v1","addr":"172.21.15.64:6831","nonce":3037106123}]},"public_addr":"172.21.15.64:6827/3037106123","cluster_addr":"172.21.15.64:6829/3037106123","heartbeat_back_addr":"172.21.15.64:6833/3037106123","heartbeat_front_addr":"172.21.15.64:6831/3037106123","state":["exists","up"]},{"osd":4,"uuid":"060e8766-7bf6-42f1-bebe-dae6828f7fb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6800","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6801","nonce":657648738}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6803","nonce":657648738}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6807","nonce":657648738}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6805","nonce":657648738}]},"public_addr":"172.21.15.156:6801/657648738","cluster_addr":"172.21.15.156:6803/657648738","heartbeat_back_addr":"172.21.15.156:6807/657648738","heartbeat_front_addr":"172.21.15.156:6805/657648738","state":["exists","up"]},{"osd":5,"uuid":"6b5cf38a-6792-4ac7-9694-268601260ba8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6809","nonce":726279450}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6811","nonce":726279450}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6815","nonce":726279450}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6813","nonce":726279450}]},"public_addr":"172.21.15.156:6809/726279450","cluster_addr":"172.21.15.156:6811/726279450","heartbeat_back_addr":"172.21.15.156:6815/726279450","heartbeat_front_addr":"172.21.15.156:6813/726279450","state":["exists","up"]},{"osd":6,"uuid":"01deea2d-a225-40b8-a54b-61eee57da527","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6817","nonce":2369965428}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6819","nonce":2369965428}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6823","nonce":2369965428}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6821","nonce":2369965428}]},"public_addr":"172.21.15.156:6817/2369965428","cluster_addr":"172.21.15.156:6819/2369965428","heartbeat_back_addr":"172.21.15.156:6823/2369965428","heartbeat_front_addr":"172.21.15.156:6821/2369965428","state":["exists","up"]},{"osd":7,"uuid":"693e93b0-c235-47fe-9172-85af68e38634","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6825","nonce":2767546322}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6827","nonce":2767546322}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6831","nonce":2767546322}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6829","nonce":2767546322}]},"public_addr":"172.21.15.156:6825/2767546322","cluster_addr":"172.21.15.156:6827/2767546322","heartbeat_back_addr":"172.21.15.156:6831/2767546322","heartbeat_front_addr":"172.21.15.156:6829/2767546322","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:35:52.928919+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:36:16.479793+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:36:39.503123+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:03.340353+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:24.645080+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:46.312540+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:38:08.346971+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:38:30.073761+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:0/2115451128":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/3707401615":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/1900245998":"2023-10-10T05:34:35.744792+0000","172.21.15.64:0/1336186767":"2023-10-10T05:39:05.525329+0000","172.21.15.64:6801/1833729701":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6800/1833729701":"2023-10-10T05:34:35.744792+0000","172.21.15.64:0/1562509700":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6800/3627852432":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/3346006057":"2023-10-10T05:34:09.281655+0000","172.21.15.64:0/2171400226":"2023-10-10T05:34:09.281655+0000","172.21.15.64:6800/3793931367":"2023-10-10T05:34:09.281655+0000","172.21.15.64:0/3185620259":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/4054596417":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6801/3627852432":"2023-10-10T05:39:05.525329+0000","172.21.15.64:6801/3793931367":"2023-10-10T05:34:09.281655+0000","172.21.15.64:0/514741379":"2023-10-10T05:34:09.281655+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:40:26.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:25 smithi064 ceph-mon[118354]: mds.? [v2:172.21.15.156:6832/4137868430,v1:172.21.15.156:6833/4137868430] up:standby 2023-10-09T05:40:26.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:25 smithi064 ceph-mon[118354]: fsmap foo:1 {0=foo.smithi064.gmrwsy=up:active} 1 up:standby 2023-10-09T05:40:26.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:25 smithi064 ceph-mon[118354]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2023-10-09T05:40:26.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:25 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/1442197463' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:40:26.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:25 smithi064 ceph-mon[101070]: mds.? [v2:172.21.15.156:6832/4137868430,v1:172.21.15.156:6833/4137868430] up:standby 2023-10-09T05:40:26.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:25 smithi064 ceph-mon[101070]: fsmap foo:1 {0=foo.smithi064.gmrwsy=up:active} 1 up:standby 2023-10-09T05:40:26.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:25 smithi064 ceph-mon[101070]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2023-10-09T05:40:26.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:25 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1442197463' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:40:26.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:25 smithi156 ceph-mon[105979]: mds.? [v2:172.21.15.156:6832/4137868430,v1:172.21.15.156:6833/4137868430] up:standby 2023-10-09T05:40:26.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:25 smithi156 ceph-mon[105979]: fsmap foo:1 {0=foo.smithi064.gmrwsy=up:active} 1 up:standby 2023-10-09T05:40:26.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:25 smithi156 ceph-mon[105979]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2023-10-09T05:40:26.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:25 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/1442197463' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:40:26.490 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T05:40:26.490 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:40:26.894 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:27.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:40:27 smithi064 conmon[101498]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:40:27] "GET /metrics HTTP/1.1" 200 238419 "" "Prometheus/2.43.0" 2023-10-09T05:40:28.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:27 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:28.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:27 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:28.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:27 smithi064 ceph-mon[101070]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T05:40:28.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:27 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:28.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:27 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:28.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:27 smithi064 ceph-mon[118354]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T05:40:28.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:27 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:28.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:27 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:28.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:27 smithi156 ceph-mon[105979]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T05:40:28.909 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:40:28.909 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":68,"fsid":"44c77acc-6665-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:33:40.037198+0000","modified":"2023-10-09T05:40:14.644959+0000","last_up_change":"2023-10-09T05:38:31.856610+0000","last_in_change":"2023-10-09T05:38:14.961182+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:36:42.276784+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:39:17.895056+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T05:40:11.204791+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T05:40:12.067097+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"85d0857a-ca32-43d2-8cec-bbd916db9f98","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2019939615},{"type":"v1","addr":"172.21.15.64:6803","nonce":2019939615}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2019939615},{"type":"v1","addr":"172.21.15.64:6805","nonce":2019939615}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2019939615},{"type":"v1","addr":"172.21.15.64:6809","nonce":2019939615}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2019939615},{"type":"v1","addr":"172.21.15.64:6807","nonce":2019939615}]},"public_addr":"172.21.15.64:6803/2019939615","cluster_addr":"172.21.15.64:6805/2019939615","heartbeat_back_addr":"172.21.15.64:6809/2019939615","heartbeat_front_addr":"172.21.15.64:6807/2019939615","state":["exists","up"]},{"osd":1,"uuid":"05570845-2cd8-4e24-a2f5-f46cf67aa31a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":1958566614},{"type":"v1","addr":"172.21.15.64:6811","nonce":1958566614}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":1958566614},{"type":"v1","addr":"172.21.15.64:6813","nonce":1958566614}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":1958566614},{"type":"v1","addr":"172.21.15.64:6817","nonce":1958566614}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":1958566614},{"type":"v1","addr":"172.21.15.64:6815","nonce":1958566614}]},"public_addr":"172.21.15.64:6811/1958566614","cluster_addr":"172.21.15.64:6813/1958566614","heartbeat_back_addr":"172.21.15.64:6817/1958566614","heartbeat_front_addr":"172.21.15.64:6815/1958566614","state":["exists","up"]},{"osd":2,"uuid":"0f306460-9f08-4f15-b573-3587f63be61c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":2050077610},{"type":"v1","addr":"172.21.15.64:6819","nonce":2050077610}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":2050077610},{"type":"v1","addr":"172.21.15.64:6821","nonce":2050077610}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":2050077610},{"type":"v1","addr":"172.21.15.64:6825","nonce":2050077610}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":2050077610},{"type":"v1","addr":"172.21.15.64:6823","nonce":2050077610}]},"public_addr":"172.21.15.64:6819/2050077610","cluster_addr":"172.21.15.64:6821/2050077610","heartbeat_back_addr":"172.21.15.64:6825/2050077610","heartbeat_front_addr":"172.21.15.64:6823/2050077610","state":["exists","up"]},{"osd":3,"uuid":"4bf38edd-e555-402e-ba4b-c751de06aa6a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":3037106123},{"type":"v1","addr":"172.21.15.64:6827","nonce":3037106123}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":3037106123},{"type":"v1","addr":"172.21.15.64:6829","nonce":3037106123}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":3037106123},{"type":"v1","addr":"172.21.15.64:6833","nonce":3037106123}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":3037106123},{"type":"v1","addr":"172.21.15.64:6831","nonce":3037106123}]},"public_addr":"172.21.15.64:6827/3037106123","cluster_addr":"172.21.15.64:6829/3037106123","heartbeat_back_addr":"172.21.15.64:6833/3037106123","heartbeat_front_addr":"172.21.15.64:6831/3037106123","state":["exists","up"]},{"osd":4,"uuid":"060e8766-7bf6-42f1-bebe-dae6828f7fb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6800","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6801","nonce":657648738}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6803","nonce":657648738}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6807","nonce":657648738}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6805","nonce":657648738}]},"public_addr":"172.21.15.156:6801/657648738","cluster_addr":"172.21.15.156:6803/657648738","heartbeat_back_addr":"172.21.15.156:6807/657648738","heartbeat_front_addr":"172.21.15.156:6805/657648738","state":["exists","up"]},{"osd":5,"uuid":"6b5cf38a-6792-4ac7-9694-268601260ba8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6809","nonce":726279450}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6811","nonce":726279450}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6815","nonce":726279450}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6813","nonce":726279450}]},"public_addr":"172.21.15.156:6809/726279450","cluster_addr":"172.21.15.156:6811/726279450","heartbeat_back_addr":"172.21.15.156:6815/726279450","heartbeat_front_addr":"172.21.15.156:6813/726279450","state":["exists","up"]},{"osd":6,"uuid":"01deea2d-a225-40b8-a54b-61eee57da527","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6817","nonce":2369965428}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6819","nonce":2369965428}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6823","nonce":2369965428}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6821","nonce":2369965428}]},"public_addr":"172.21.15.156:6817/2369965428","cluster_addr":"172.21.15.156:6819/2369965428","heartbeat_back_addr":"172.21.15.156:6823/2369965428","heartbeat_front_addr":"172.21.15.156:6821/2369965428","state":["exists","up"]},{"osd":7,"uuid":"693e93b0-c235-47fe-9172-85af68e38634","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6825","nonce":2767546322}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6827","nonce":2767546322}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6831","nonce":2767546322}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6829","nonce":2767546322}]},"public_addr":"172.21.15.156:6825/2767546322","cluster_addr":"172.21.15.156:6827/2767546322","heartbeat_back_addr":"172.21.15.156:6831/2767546322","heartbeat_front_addr":"172.21.15.156:6829/2767546322","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:35:52.928919+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:36:16.479793+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:36:39.503123+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:03.340353+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:24.645080+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:46.312540+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:38:08.346971+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:38:30.073761+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:0/2115451128":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/3707401615":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/1900245998":"2023-10-10T05:34:35.744792+0000","172.21.15.64:0/1336186767":"2023-10-10T05:39:05.525329+0000","172.21.15.64:6801/1833729701":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6800/1833729701":"2023-10-10T05:34:35.744792+0000","172.21.15.64:0/1562509700":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6800/3627852432":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/3346006057":"2023-10-10T05:34:09.281655+0000","172.21.15.64:0/2171400226":"2023-10-10T05:34:09.281655+0000","172.21.15.64:6800/3793931367":"2023-10-10T05:34:09.281655+0000","172.21.15.64:0/3185620259":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/4054596417":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6801/3627852432":"2023-10-10T05:39:05.525329+0000","172.21.15.64:6801/3793931367":"2023-10-10T05:34:09.281655+0000","172.21.15.64:0/514741379":"2023-10-10T05:34:09.281655+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:40:29.625 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T05:40:29.626 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T05:40:29.626 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T05:40:29.626 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T05:40:29.627 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T05:40:29.627 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T05:40:29.628 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T05:40:29.628 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T05:40:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:29 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:29 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:29 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:40:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:29 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:40:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:29 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:29.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:29 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/651071229' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:40:29.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:29 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:29.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:29 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:29.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:29 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:40:29.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:29 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:40:29.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:29 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:29.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:29 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/651071229' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:40:29.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:29 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:29.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:29 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:29.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:29 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:40:29.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:29 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:40:29.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:29 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:29.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:29 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/651071229' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:40:30.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:30 smithi156 ceph-mon[105979]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T05:40:30.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:30 smithi064 ceph-mon[101070]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T05:40:30.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:30 smithi064 ceph-mon[118354]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T05:40:30.975 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:31.106 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:31.133 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:31.232 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:31.304 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:31.305 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:31.435 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:31.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:31 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:31.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:31 smithi064 ceph-mon[101070]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T05:40:31.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:31 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:31.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:31 smithi064 ceph-mon[118354]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T05:40:32.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:31 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:40:32.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:31 smithi156 ceph-mon[105979]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T05:40:32.323 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:33.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:40:32 smithi156 conmon[109199]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:40:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:40:34.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:34 smithi156 ceph-mon[105979]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:40:34.918 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:34 smithi064 ceph-mon[101070]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:40:34.919 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:34 smithi064 ceph-mon[118354]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:40:36.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:36 smithi064 ceph-mon[101070]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:40:36.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:36 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:36.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:36 smithi064 ceph-mon[118354]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:40:36.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:36 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:36.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:36 smithi156 ceph-mon[105979]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:40:36.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:36 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:37.196 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:40:37 smithi064 conmon[101498]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:40:37] "GET /metrics HTTP/1.1" 200 238746 "" "Prometheus/2.43.0" 2023-10-09T05:40:37.370 INFO:teuthology.orchestra.run.smithi064.stdout:34359738427 2023-10-09T05:40:37.371 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T05:40:38.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:38 smithi064 ceph-mon[101070]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:40:38.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:38 smithi064 ceph-mon[118354]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:40:38.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:38 smithi156 ceph-mon[105979]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:40:40.322 INFO:teuthology.orchestra.run.smithi064.stdout:77309411376 2023-10-09T05:40:40.323 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T05:40:40.350 INFO:teuthology.orchestra.run.smithi064.stdout:55834574903 2023-10-09T05:40:40.351 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T05:40:40.535 INFO:teuthology.orchestra.run.smithi064.stdout:128849018921 2023-10-09T05:40:40.536 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T05:40:40.809 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:40 smithi064 ceph-mon[118354]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:40.810 INFO:teuthology.orchestra.run.smithi064.stdout:197568495648 2023-10-09T05:40:40.810 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T05:40:40.810 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:40 smithi064 ceph-mon[101070]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:40.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:40 smithi156 ceph-mon[105979]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:41.706 INFO:teuthology.orchestra.run.smithi064.stdout:163208757284 2023-10-09T05:40:41.707 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T05:40:42.494 INFO:teuthology.orchestra.run.smithi064.stdout:107374182445 2023-10-09T05:40:42.497 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T05:40:42.712 INFO:teuthology.orchestra.run.smithi064.stdout:231928234013 2023-10-09T05:40:42.712 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T05:40:42.790 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:42 smithi064 ceph-mon[101070]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:42.790 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:42 smithi064 ceph-mon[118354]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:42.841 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:42.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:42 smithi156 ceph-mon[105979]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:42.870 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:40:42 smithi156 conmon[109199]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:40:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:40:43.092 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:43.144 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:43.302 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:44.101 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:44.226 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:44.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:44 smithi064 ceph-mon[101070]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:44.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:44 smithi064 ceph-mon[118354]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:44.810 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:44.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:44 smithi156 ceph-mon[105979]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:46.123 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:46.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:46 smithi064 ceph-mon[101070]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:46.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:46 smithi064 ceph-mon[118354]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:46.830 INFO:teuthology.orchestra.run.smithi064.stdout:55834574904 2023-10-09T05:40:46.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:46 smithi156 ceph-mon[105979]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:47.195 INFO:teuthology.orchestra.run.smithi064.stdout:34359738429 2023-10-09T05:40:47.400 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:40:47 smithi064 conmon[101498]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:40:47] "GET /metrics HTTP/1.1" 200 238746 "" "Prometheus/2.43.0" 2023-10-09T05:40:47.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:47 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/208865055' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:40:47.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:47 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3753467053' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:40:47.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:47 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/208865055' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:40:47.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:47 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3753467053' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:40:47.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:47 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/208865055' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:40:47.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:47 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3753467053' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:40:48.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:48 smithi064 ceph-mon[101070]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:48.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:48 smithi064 ceph-mon[118354]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:48.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:48 smithi156 ceph-mon[105979]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:49.096 INFO:teuthology.orchestra.run.smithi064.stdout:163208757286 2023-10-09T05:40:49.310 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738427 got 34359738429 for osd.0 2023-10-09T05:40:49.310 DEBUG:teuthology.parallel:result is None 2023-10-09T05:40:49.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:49 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3868304929' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:40:49.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:49 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3868304929' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:40:49.870 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574903 got 55834574904 for osd.1 2023-10-09T05:40:49.870 DEBUG:teuthology.parallel:result is None 2023-10-09T05:40:49.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:49 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3868304929' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:40:50.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:50 smithi064 ceph-mon[101070]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:50.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:50 smithi064 ceph-mon[118354]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:50.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:50 smithi156 ceph-mon[105979]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:51.217 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757284 got 163208757286 for osd.5 2023-10-09T05:40:51.218 DEBUG:teuthology.parallel:result is None 2023-10-09T05:40:51.641 INFO:teuthology.orchestra.run.smithi064.stdout:231928234014 2023-10-09T05:40:51.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:51 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:51.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:51 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:51.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:51 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:40:52.091 INFO:teuthology.orchestra.run.smithi064.stdout:197568495650 2023-10-09T05:40:52.157 INFO:teuthology.orchestra.run.smithi064.stdout:107374182447 2023-10-09T05:40:52.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:52 smithi064 ceph-mon[101070]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:52.692 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:52 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3271720123' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:40:52.692 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:52 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/529188873' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:40:52.692 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:52 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1848585292' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:40:52.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:52 smithi064 ceph-mon[118354]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:52.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:52 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3271720123' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:40:52.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:52 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/529188873' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:40:52.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:52 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/1848585292' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:40:52.704 INFO:teuthology.orchestra.run.smithi064.stdout:77309411380 2023-10-09T05:40:52.764 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:52 smithi156 ceph-mon[105979]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:52.764 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:52 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3271720123' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:40:52.764 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:52 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/529188873' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:40:52.764 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:52 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/1848585292' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:40:53.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:40:52 smithi156 conmon[109199]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:40:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:40:53.422 INFO:teuthology.orchestra.run.smithi064.stdout:128849018924 2023-10-09T05:40:53.483 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234013 got 231928234014 for osd.7 2023-10-09T05:40:53.483 DEBUG:teuthology.parallel:result is None 2023-10-09T05:40:53.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:53 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/2088756838' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:40:53.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:53 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/2088756838' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:40:53.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:53 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/2088756838' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:40:54.665 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411376 got 77309411380 for osd.2 2023-10-09T05:40:54.665 DEBUG:teuthology.parallel:result is None 2023-10-09T05:40:54.700 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:54 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3361153235' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:40:54.700 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:54 smithi064 ceph-mon[101070]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:54.700 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:54 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3361153235' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:40:54.700 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:54 smithi064 ceph-mon[118354]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:54.701 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018921 got 128849018924 for osd.4 2023-10-09T05:40:54.701 DEBUG:teuthology.parallel:result is None 2023-10-09T05:40:54.740 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182445 got 107374182447 for osd.3 2023-10-09T05:40:54.740 DEBUG:teuthology.parallel:result is None 2023-10-09T05:40:54.821 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495648 got 197568495650 for osd.6 2023-10-09T05:40:54.821 DEBUG:teuthology.parallel:result is None 2023-10-09T05:40:54.821 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T05:40:54.822 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:40:54.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:54 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3361153235' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:40:54.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:54 smithi156 ceph-mon[105979]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:55.183 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:56.600 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:56 smithi064 ceph-mon[101070]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:56.600 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:56 smithi064 ceph-mon[118354]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:56.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:56 smithi156 ceph-mon[105979]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:57.003 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:40:57.005 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T05:40:57.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:40:57 smithi064 conmon[101498]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:40:57] "GET /metrics HTTP/1.1" 200 238712 "" "Prometheus/2.43.0" 2023-10-09T05:40:57.679 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":67,"stamp":"2023-10-09T05:40:55.558788+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385528,"kb_used_data":7224,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747575688,"statfs":{"total":767960285184,"available":765517504512,"internally_reserved":0,"allocated":7397376,"data_stored":4173037,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002594"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.671995+0000","last_change":"2023-10-09T05:40:12.676860+0000","last_active":"2023-10-09T05:40:13.671995+0000","last_peered":"2023-10-09T05:40:13.671995+0000","last_clean":"2023-10-09T05:40:13.671995+0000","last_became_active":"2023-10-09T05:40:12.676172+0000","last_became_peered":"2023-10-09T05:40:12.676172+0000","last_unstale":"2023-10-09T05:40:13.671995+0000","last_undegraded":"2023-10-09T05:40:13.671995+0000","last_fullsized":"2023-10-09T05:40:13.671995+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:26:11.807891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.464478+0000","last_change":"2023-10-09T05:40:12.627433+0000","last_active":"2023-10-09T05:40:17.464478+0000","last_peered":"2023-10-09T05:40:17.464478+0000","last_clean":"2023-10-09T05:40:17.464478+0000","last_became_active":"2023-10-09T05:40:12.626888+0000","last_became_peered":"2023-10-09T05:40:12.626888+0000","last_unstale":"2023-10-09T05:40:17.464478+0000","last_undegraded":"2023-10-09T05:40:17.464478+0000","last_fullsized":"2023-10-09T05:40:17.464478+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:22.990244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.312970+0000","last_change":"2023-10-09T05:40:12.660488+0000","last_active":"2023-10-09T05:40:14.312970+0000","last_peered":"2023-10-09T05:40:14.312970+0000","last_clean":"2023-10-09T05:40:14.312970+0000","last_became_active":"2023-10-09T05:40:12.660226+0000","last_became_peered":"2023-10-09T05:40:12.660226+0000","last_unstale":"2023-10-09T05:40:14.312970+0000","last_undegraded":"2023-10-09T05:40:14.312970+0000","last_fullsized":"2023-10-09T05:40:14.312970+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:11:18.728422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:15.718722+0000","last_change":"2023-10-09T05:40:12.637232+0000","last_active":"2023-10-09T05:40:15.718722+0000","last_peered":"2023-10-09T05:40:15.718722+0000","last_clean":"2023-10-09T05:40:15.718722+0000","last_became_active":"2023-10-09T05:40:12.636729+0000","last_became_peered":"2023-10-09T05:40:12.636729+0000","last_unstale":"2023-10-09T05:40:15.718722+0000","last_undegraded":"2023-10-09T05:40:15.718722+0000","last_fullsized":"2023-10-09T05:40:15.718722+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:16.890317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.672232+0000","last_change":"2023-10-09T05:40:12.660135+0000","last_active":"2023-10-09T05:40:13.672232+0000","last_peered":"2023-10-09T05:40:13.672232+0000","last_clean":"2023-10-09T05:40:13.672232+0000","last_became_active":"2023-10-09T05:40:12.659919+0000","last_became_peered":"2023-10-09T05:40:12.659919+0000","last_unstale":"2023-10-09T05:40:13.672232+0000","last_undegraded":"2023-10-09T05:40:13.672232+0000","last_fullsized":"2023-10-09T05:40:13.672232+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:02.925104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.465321+0000","last_change":"2023-10-09T05:40:12.633195+0000","last_active":"2023-10-09T05:40:17.465321+0000","last_peered":"2023-10-09T05:40:17.465321+0000","last_clean":"2023-10-09T05:40:17.465321+0000","last_became_active":"2023-10-09T05:40:12.632942+0000","last_became_peered":"2023-10-09T05:40:12.632942+0000","last_unstale":"2023-10-09T05:40:17.465321+0000","last_undegraded":"2023-10-09T05:40:17.465321+0000","last_fullsized":"2023-10-09T05:40:17.465321+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:04.470685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.672328+0000","last_change":"2023-10-09T05:40:12.659241+0000","last_active":"2023-10-09T05:40:13.672328+0000","last_peered":"2023-10-09T05:40:13.672328+0000","last_clean":"2023-10-09T05:40:13.672328+0000","last_became_active":"2023-10-09T05:40:12.658955+0000","last_became_peered":"2023-10-09T05:40:12.658955+0000","last_unstale":"2023-10-09T05:40:13.672328+0000","last_undegraded":"2023-10-09T05:40:13.672328+0000","last_fullsized":"2023-10-09T05:40:13.672328+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:19.986933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.464130+0000","last_change":"2023-10-09T05:40:12.631949+0000","last_active":"2023-10-09T05:40:17.464130+0000","last_peered":"2023-10-09T05:40:17.464130+0000","last_clean":"2023-10-09T05:40:17.464130+0000","last_became_active":"2023-10-09T05:40:12.631660+0000","last_became_peered":"2023-10-09T05:40:12.631660+0000","last_unstale":"2023-10-09T05:40:17.464130+0000","last_undegraded":"2023-10-09T05:40:17.464130+0000","last_fullsized":"2023-10-09T05:40:17.464130+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:36.314119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.655035+0000","last_change":"2023-10-09T05:40:12.659881+0000","last_active":"2023-10-09T05:40:13.655035+0000","last_peered":"2023-10-09T05:40:13.655035+0000","last_clean":"2023-10-09T05:40:13.655035+0000","last_became_active":"2023-10-09T05:40:12.659588+0000","last_became_peered":"2023-10-09T05:40:12.659588+0000","last_unstale":"2023-10-09T05:40:13.655035+0000","last_undegraded":"2023-10-09T05:40:13.655035+0000","last_fullsized":"2023-10-09T05:40:13.655035+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:09.897771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:15.633572+0000","last_change":"2023-10-09T05:40:12.631499+0000","last_active":"2023-10-09T05:40:15.633572+0000","last_peered":"2023-10-09T05:40:15.633572+0000","last_clean":"2023-10-09T05:40:15.633572+0000","last_became_active":"2023-10-09T05:40:12.630754+0000","last_became_peered":"2023-10-09T05:40:12.630754+0000","last_unstale":"2023-10-09T05:40:15.633572+0000","last_undegraded":"2023-10-09T05:40:15.633572+0000","last_fullsized":"2023-10-09T05:40:15.633572+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:51.146061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.672188+0000","last_change":"2023-10-09T05:40:12.676587+0000","last_active":"2023-10-09T05:40:13.672188+0000","last_peered":"2023-10-09T05:40:13.672188+0000","last_clean":"2023-10-09T05:40:13.672188+0000","last_became_active":"2023-10-09T05:40:12.676330+0000","last_became_peered":"2023-10-09T05:40:12.676330+0000","last_unstale":"2023-10-09T05:40:13.672188+0000","last_undegraded":"2023-10-09T05:40:13.672188+0000","last_fullsized":"2023-10-09T05:40:13.672188+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:46.659556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:15.633602+0000","last_change":"2023-10-09T05:40:12.625851+0000","last_active":"2023-10-09T05:40:15.633602+0000","last_peered":"2023-10-09T05:40:15.633602+0000","last_clean":"2023-10-09T05:40:15.633602+0000","last_became_active":"2023-10-09T05:40:12.625486+0000","last_became_peered":"2023-10-09T05:40:12.625486+0000","last_unstale":"2023-10-09T05:40:15.633602+0000","last_undegraded":"2023-10-09T05:40:15.633602+0000","last_fullsized":"2023-10-09T05:40:15.633602+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:08.394037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.648913+0000","last_change":"2023-10-09T05:39:18.963184+0000","last_active":"2023-10-09T05:40:13.648913+0000","last_peered":"2023-10-09T05:40:13.648913+0000","last_clean":"2023-10-09T05:40:13.648913+0000","last_became_active":"2023-10-09T05:39:18.962837+0000","last_became_peered":"2023-10-09T05:39:18.962837+0000","last_unstale":"2023-10-09T05:40:13.648913+0000","last_undegraded":"2023-10-09T05:40:13.648913+0000","last_fullsized":"2023-10-09T05:40:13.648913+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:05.818061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.312841+0000","last_change":"2023-10-09T05:40:13.112991+0000","last_active":"2023-10-09T05:40:14.312841+0000","last_peered":"2023-10-09T05:40:14.312841+0000","last_clean":"2023-10-09T05:40:14.312841+0000","last_became_active":"2023-10-09T05:40:13.112646+0000","last_became_peered":"2023-10-09T05:40:13.112646+0000","last_unstale":"2023-10-09T05:40:14.312841+0000","last_undegraded":"2023-10-09T05:40:14.312841+0000","last_fullsized":"2023-10-09T05:40:14.312841+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:00.533988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.654183+0000","last_change":"2023-10-09T05:39:19.405012+0000","last_active":"2023-10-09T05:40:13.654183+0000","last_peered":"2023-10-09T05:40:13.654183+0000","last_clean":"2023-10-09T05:40:13.654183+0000","last_became_active":"2023-10-09T05:39:19.404658+0000","last_became_peered":"2023-10-09T05:39:19.404658+0000","last_unstale":"2023-10-09T05:40:13.654183+0000","last_undegraded":"2023-10-09T05:40:13.654183+0000","last_fullsized":"2023-10-09T05:40:13.654183+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:24.956847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.464933+0000","last_change":"2023-10-09T05:40:12.630020+0000","last_active":"2023-10-09T05:40:17.464933+0000","last_peered":"2023-10-09T05:40:17.464933+0000","last_clean":"2023-10-09T05:40:17.464933+0000","last_became_active":"2023-10-09T05:40:12.629746+0000","last_became_peered":"2023-10-09T05:40:12.629746+0000","last_unstale":"2023-10-09T05:40:17.464933+0000","last_undegraded":"2023-10-09T05:40:17.464933+0000","last_fullsized":"2023-10-09T05:40:17.464933+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:05.159248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.306200+0000","last_change":"2023-10-09T05:40:13.112384+0000","last_active":"2023-10-09T05:40:14.306200+0000","last_peered":"2023-10-09T05:40:14.306200+0000","last_clean":"2023-10-09T05:40:14.306200+0000","last_became_active":"2023-10-09T05:40:13.112113+0000","last_became_peered":"2023-10-09T05:40:13.112113+0000","last_unstale":"2023-10-09T05:40:14.306200+0000","last_undegraded":"2023-10-09T05:40:14.306200+0000","last_fullsized":"2023-10-09T05:40:14.306200+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:00.125258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.654277+0000","last_change":"2023-10-09T05:40:12.640616+0000","last_active":"2023-10-09T05:40:13.654277+0000","last_peered":"2023-10-09T05:40:13.654277+0000","last_clean":"2023-10-09T05:40:13.654277+0000","last_became_active":"2023-10-09T05:40:12.640313+0000","last_became_peered":"2023-10-09T05:40:12.640313+0000","last_unstale":"2023-10-09T05:40:13.654277+0000","last_undegraded":"2023-10-09T05:40:13.654277+0000","last_fullsized":"2023-10-09T05:40:13.654277+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:48.607096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.361720+0000","last_change":"2023-10-09T05:39:19.405346+0000","last_active":"2023-10-09T05:40:14.361720+0000","last_peered":"2023-10-09T05:40:14.361720+0000","last_clean":"2023-10-09T05:40:14.361720+0000","last_became_active":"2023-10-09T05:39:19.404876+0000","last_became_peered":"2023-10-09T05:39:19.404876+0000","last_unstale":"2023-10-09T05:40:14.361720+0000","last_undegraded":"2023-10-09T05:40:14.361720+0000","last_fullsized":"2023-10-09T05:40:14.361720+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:28.442496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.306389+0000","last_change":"2023-10-09T05:40:12.653425+0000","last_active":"2023-10-09T05:40:14.306389+0000","last_peered":"2023-10-09T05:40:14.306389+0000","last_clean":"2023-10-09T05:40:14.306389+0000","last_became_active":"2023-10-09T05:40:12.652985+0000","last_became_peered":"2023-10-09T05:40:12.652985+0000","last_unstale":"2023-10-09T05:40:14.306389+0000","last_undegraded":"2023-10-09T05:40:14.306389+0000","last_fullsized":"2023-10-09T05:40:14.306389+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:52.785830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.306166+0000","last_change":"2023-10-09T05:40:12.630146+0000","last_active":"2023-10-09T05:40:14.306166+0000","last_peered":"2023-10-09T05:40:14.306166+0000","last_clean":"2023-10-09T05:40:14.306166+0000","last_became_active":"2023-10-09T05:40:12.629702+0000","last_became_peered":"2023-10-09T05:40:12.629702+0000","last_unstale":"2023-10-09T05:40:14.306166+0000","last_undegraded":"2023-10-09T05:40:14.306166+0000","last_fullsized":"2023-10-09T05:40:14.306166+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:31.056565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.361539+0000","last_change":"2023-10-09T05:39:18.964130+0000","last_active":"2023-10-09T05:40:14.361539+0000","last_peered":"2023-10-09T05:40:14.361539+0000","last_clean":"2023-10-09T05:40:14.361539+0000","last_became_active":"2023-10-09T05:39:18.963726+0000","last_became_peered":"2023-10-09T05:39:18.963726+0000","last_unstale":"2023-10-09T05:40:14.361539+0000","last_undegraded":"2023-10-09T05:40:14.361539+0000","last_fullsized":"2023-10-09T05:40:14.361539+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:13.428159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.363514+0000","last_change":"2023-10-09T05:40:14.363125+0000","last_active":"2023-10-09T05:40:14.363514+0000","last_peered":"2023-10-09T05:40:14.363514+0000","last_clean":"2023-10-09T05:40:14.363514+0000","last_became_active":"2023-10-09T05:40:14.362723+0000","last_became_peered":"2023-10-09T05:40:14.362723+0000","last_unstale":"2023-10-09T05:40:14.363514+0000","last_undegraded":"2023-10-09T05:40:14.363514+0000","last_fullsized":"2023-10-09T05:40:14.363514+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:30.741318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.313063+0000","last_change":"2023-10-09T05:40:12.633103+0000","last_active":"2023-10-09T05:40:14.313063+0000","last_peered":"2023-10-09T05:40:14.313063+0000","last_clean":"2023-10-09T05:40:14.313063+0000","last_became_active":"2023-10-09T05:40:12.632831+0000","last_became_peered":"2023-10-09T05:40:12.632831+0000","last_unstale":"2023-10-09T05:40:14.313063+0000","last_undegraded":"2023-10-09T05:40:14.313063+0000","last_fullsized":"2023-10-09T05:40:14.313063+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:32.796053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":276,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.361600+0000","last_change":"2023-10-09T05:38:35.869176+0000","last_active":"2023-10-09T05:40:14.361600+0000","last_peered":"2023-10-09T05:40:14.361600+0000","last_clean":"2023-10-09T05:40:14.361600+0000","last_became_active":"2023-10-09T05:38:35.868750+0000","last_became_peered":"2023-10-09T05:38:35.868750+0000","last_unstale":"2023-10-09T05:40:14.361600+0000","last_undegraded":"2023-10-09T05:40:14.361600+0000","last_fullsized":"2023-10-09T05:40:14.361600+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:20.200381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.646859+0000","last_change":"2023-10-09T05:39:18.963327+0000","last_active":"2023-10-09T05:40:13.646859+0000","last_peered":"2023-10-09T05:40:13.646859+0000","last_clean":"2023-10-09T05:40:13.646859+0000","last_became_active":"2023-10-09T05:39:18.962933+0000","last_became_peered":"2023-10-09T05:39:18.962933+0000","last_unstale":"2023-10-09T05:40:13.646859+0000","last_undegraded":"2023-10-09T05:40:13.646859+0000","last_fullsized":"2023-10-09T05:40:13.646859+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:43.427745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.654564+0000","last_change":"2023-10-09T05:40:12.665952+0000","last_active":"2023-10-09T05:40:13.654564+0000","last_peered":"2023-10-09T05:40:13.654564+0000","last_clean":"2023-10-09T05:40:13.654564+0000","last_became_active":"2023-10-09T05:40:12.665664+0000","last_became_peered":"2023-10-09T05:40:12.665664+0000","last_unstale":"2023-10-09T05:40:13.654564+0000","last_undegraded":"2023-10-09T05:40:13.654564+0000","last_fullsized":"2023-10-09T05:40:13.654564+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:58.953498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.654253+0000","last_change":"2023-10-09T05:40:12.633205+0000","last_active":"2023-10-09T05:40:13.654253+0000","last_peered":"2023-10-09T05:40:13.654253+0000","last_clean":"2023-10-09T05:40:13.654253+0000","last_became_active":"2023-10-09T05:40:12.632846+0000","last_became_peered":"2023-10-09T05:40:12.632846+0000","last_unstale":"2023-10-09T05:40:13.654253+0000","last_undegraded":"2023-10-09T05:40:13.654253+0000","last_fullsized":"2023-10-09T05:40:13.654253+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:16.854263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.654499+0000","last_change":"2023-10-09T05:39:19.362360+0000","last_active":"2023-10-09T05:40:13.654499+0000","last_peered":"2023-10-09T05:40:13.654499+0000","last_clean":"2023-10-09T05:40:13.654499+0000","last_became_active":"2023-10-09T05:39:19.361972+0000","last_became_peered":"2023-10-09T05:39:19.361972+0000","last_unstale":"2023-10-09T05:40:13.654499+0000","last_undegraded":"2023-10-09T05:40:13.654499+0000","last_fullsized":"2023-10-09T05:40:13.654499+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:06.194167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.654225+0000","last_change":"2023-10-09T05:40:12.664058+0000","last_active":"2023-10-09T05:40:13.654225+0000","last_peered":"2023-10-09T05:40:13.654225+0000","last_clean":"2023-10-09T05:40:13.654225+0000","last_became_active":"2023-10-09T05:40:12.663759+0000","last_became_peered":"2023-10-09T05:40:12.663759+0000","last_unstale":"2023-10-09T05:40:13.654225+0000","last_undegraded":"2023-10-09T05:40:13.654225+0000","last_fullsized":"2023-10-09T05:40:13.654225+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:51.023928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.672315+0000","last_change":"2023-10-09T05:40:12.630487+0000","last_active":"2023-10-09T05:40:13.672315+0000","last_peered":"2023-10-09T05:40:13.672315+0000","last_clean":"2023-10-09T05:40:13.672315+0000","last_became_active":"2023-10-09T05:40:12.630135+0000","last_became_peered":"2023-10-09T05:40:12.630135+0000","last_unstale":"2023-10-09T05:40:13.672315+0000","last_undegraded":"2023-10-09T05:40:13.672315+0000","last_fullsized":"2023-10-09T05:40:13.672315+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:59.045086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.646524+0000","last_change":"2023-10-09T05:39:18.963544+0000","last_active":"2023-10-09T05:40:13.646524+0000","last_peered":"2023-10-09T05:40:13.646524+0000","last_clean":"2023-10-09T05:40:13.646524+0000","last_became_active":"2023-10-09T05:39:18.962981+0000","last_became_peered":"2023-10-09T05:39:18.962981+0000","last_unstale":"2023-10-09T05:40:13.646524+0000","last_undegraded":"2023-10-09T05:40:13.646524+0000","last_fullsized":"2023-10-09T05:40:13.646524+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:11.589793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.654223+0000","last_change":"2023-10-09T05:40:12.665302+0000","last_active":"2023-10-09T05:40:13.654223+0000","last_peered":"2023-10-09T05:40:13.654223+0000","last_clean":"2023-10-09T05:40:13.654223+0000","last_became_active":"2023-10-09T05:40:12.665037+0000","last_became_peered":"2023-10-09T05:40:12.665037+0000","last_unstale":"2023-10-09T05:40:13.654223+0000","last_undegraded":"2023-10-09T05:40:13.654223+0000","last_fullsized":"2023-10-09T05:40:13.654223+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:10.300607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.646806+0000","last_change":"2023-10-09T05:40:12.627754+0000","last_active":"2023-10-09T05:40:13.646806+0000","last_peered":"2023-10-09T05:40:13.646806+0000","last_clean":"2023-10-09T05:40:13.646806+0000","last_became_active":"2023-10-09T05:40:12.627490+0000","last_became_peered":"2023-10-09T05:40:12.627490+0000","last_unstale":"2023-10-09T05:40:13.646806+0000","last_undegraded":"2023-10-09T05:40:13.646806+0000","last_fullsized":"2023-10-09T05:40:13.646806+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:47.650326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.654372+0000","last_change":"2023-10-09T05:39:18.964305+0000","last_active":"2023-10-09T05:40:13.654372+0000","last_peered":"2023-10-09T05:40:13.654372+0000","last_clean":"2023-10-09T05:40:13.654372+0000","last_became_active":"2023-10-09T05:39:18.963935+0000","last_became_peered":"2023-10-09T05:39:18.963935+0000","last_unstale":"2023-10-09T05:40:13.654372+0000","last_undegraded":"2023-10-09T05:40:13.654372+0000","last_fullsized":"2023-10-09T05:40:13.654372+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:33.105059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.654634+0000","last_change":"2023-10-09T05:40:12.662743+0000","last_active":"2023-10-09T05:40:13.654634+0000","last_peered":"2023-10-09T05:40:13.654634+0000","last_clean":"2023-10-09T05:40:13.654634+0000","last_became_active":"2023-10-09T05:40:12.662488+0000","last_became_peered":"2023-10-09T05:40:12.662488+0000","last_unstale":"2023-10-09T05:40:13.654634+0000","last_undegraded":"2023-10-09T05:40:13.654634+0000","last_fullsized":"2023-10-09T05:40:13.654634+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:25.038412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.312860+0000","last_change":"2023-10-09T05:40:12.638512+0000","last_active":"2023-10-09T05:40:14.312860+0000","last_peered":"2023-10-09T05:40:14.312860+0000","last_clean":"2023-10-09T05:40:14.312860+0000","last_became_active":"2023-10-09T05:40:12.638030+0000","last_became_peered":"2023-10-09T05:40:12.638030+0000","last_unstale":"2023-10-09T05:40:14.312860+0000","last_undegraded":"2023-10-09T05:40:14.312860+0000","last_fullsized":"2023-10-09T05:40:14.312860+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:24.393305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.672582+0000","last_change":"2023-10-09T05:40:12.676961+0000","last_active":"2023-10-09T05:40:13.672582+0000","last_peered":"2023-10-09T05:40:13.672582+0000","last_clean":"2023-10-09T05:40:13.672582+0000","last_became_active":"2023-10-09T05:40:12.676453+0000","last_became_peered":"2023-10-09T05:40:12.676453+0000","last_unstale":"2023-10-09T05:40:13.672582+0000","last_undegraded":"2023-10-09T05:40:13.672582+0000","last_fullsized":"2023-10-09T05:40:13.672582+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:41.173802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.672204+0000","last_change":"2023-10-09T05:40:12.626069+0000","last_active":"2023-10-09T05:40:13.672204+0000","last_peered":"2023-10-09T05:40:13.672204+0000","last_clean":"2023-10-09T05:40:13.672204+0000","last_became_active":"2023-10-09T05:40:12.625380+0000","last_became_peered":"2023-10-09T05:40:12.625380+0000","last_unstale":"2023-10-09T05:40:13.672204+0000","last_undegraded":"2023-10-09T05:40:13.672204+0000","last_fullsized":"2023-10-09T05:40:13.672204+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:32:11.601924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.672690+0000","last_change":"2023-10-09T05:40:12.652676+0000","last_active":"2023-10-09T05:40:13.672690+0000","last_peered":"2023-10-09T05:40:13.672690+0000","last_clean":"2023-10-09T05:40:13.672690+0000","last_became_active":"2023-10-09T05:40:12.652437+0000","last_became_peered":"2023-10-09T05:40:12.652437+0000","last_unstale":"2023-10-09T05:40:13.672690+0000","last_undegraded":"2023-10-09T05:40:13.672690+0000","last_fullsized":"2023-10-09T05:40:13.672690+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:12.886016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.648820+0000","last_change":"2023-10-09T05:40:12.637136+0000","last_active":"2023-10-09T05:40:13.648820+0000","last_peered":"2023-10-09T05:40:13.648820+0000","last_clean":"2023-10-09T05:40:13.648820+0000","last_became_active":"2023-10-09T05:40:12.636812+0000","last_became_peered":"2023-10-09T05:40:12.636812+0000","last_unstale":"2023-10-09T05:40:13.648820+0000","last_undegraded":"2023-10-09T05:40:13.648820+0000","last_fullsized":"2023-10-09T05:40:13.648820+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:59.890646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.672415+0000","last_change":"2023-10-09T05:40:12.676584+0000","last_active":"2023-10-09T05:40:13.672415+0000","last_peered":"2023-10-09T05:40:13.672415+0000","last_clean":"2023-10-09T05:40:13.672415+0000","last_became_active":"2023-10-09T05:40:12.676241+0000","last_became_peered":"2023-10-09T05:40:12.676241+0000","last_unstale":"2023-10-09T05:40:13.672415+0000","last_undegraded":"2023-10-09T05:40:13.672415+0000","last_fullsized":"2023-10-09T05:40:13.672415+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:20:52.246349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.465840+0000","last_change":"2023-10-09T05:40:12.625178+0000","last_active":"2023-10-09T05:40:17.465840+0000","last_peered":"2023-10-09T05:40:17.465840+0000","last_clean":"2023-10-09T05:40:17.465840+0000","last_became_active":"2023-10-09T05:40:12.624841+0000","last_became_peered":"2023-10-09T05:40:12.624841+0000","last_unstale":"2023-10-09T05:40:17.465840+0000","last_undegraded":"2023-10-09T05:40:17.465840+0000","last_fullsized":"2023-10-09T05:40:17.465840+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:11.715344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.306314+0000","last_change":"2023-10-09T05:40:12.656403+0000","last_active":"2023-10-09T05:40:14.306314+0000","last_peered":"2023-10-09T05:40:14.306314+0000","last_clean":"2023-10-09T05:40:14.306314+0000","last_became_active":"2023-10-09T05:40:12.655733+0000","last_became_peered":"2023-10-09T05:40:12.655733+0000","last_unstale":"2023-10-09T05:40:14.306314+0000","last_undegraded":"2023-10-09T05:40:14.306314+0000","last_fullsized":"2023-10-09T05:40:14.306314+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:30.078067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.646654+0000","last_change":"2023-10-09T05:40:12.634263+0000","last_active":"2023-10-09T05:40:13.646654+0000","last_peered":"2023-10-09T05:40:13.646654+0000","last_clean":"2023-10-09T05:40:13.646654+0000","last_became_active":"2023-10-09T05:40:12.633644+0000","last_became_peered":"2023-10-09T05:40:12.633644+0000","last_unstale":"2023-10-09T05:40:13.646654+0000","last_undegraded":"2023-10-09T05:40:13.646654+0000","last_fullsized":"2023-10-09T05:40:13.646654+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:57.270857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.362272+0000","last_change":"2023-10-09T05:40:14.362061+0000","last_active":"2023-10-09T05:40:14.362272+0000","last_peered":"2023-10-09T05:40:14.362272+0000","last_clean":"2023-10-09T05:40:14.362272+0000","last_became_active":"2023-10-09T05:40:14.361712+0000","last_became_peered":"2023-10-09T05:40:14.361712+0000","last_unstale":"2023-10-09T05:40:14.362272+0000","last_undegraded":"2023-10-09T05:40:14.362272+0000","last_fullsized":"2023-10-09T05:40:14.362272+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:07.409309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:15.676122+0000","last_change":"2023-10-09T05:40:12.632083+0000","last_active":"2023-10-09T05:40:15.676122+0000","last_peered":"2023-10-09T05:40:15.676122+0000","last_clean":"2023-10-09T05:40:15.676122+0000","last_became_active":"2023-10-09T05:40:12.631715+0000","last_became_peered":"2023-10-09T05:40:12.631715+0000","last_unstale":"2023-10-09T05:40:15.676122+0000","last_undegraded":"2023-10-09T05:40:15.676122+0000","last_fullsized":"2023-10-09T05:40:15.676122+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:17.641378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.672098+0000","last_change":"2023-10-09T05:40:12.659398+0000","last_active":"2023-10-09T05:40:13.672098+0000","last_peered":"2023-10-09T05:40:13.672098+0000","last_clean":"2023-10-09T05:40:13.672098+0000","last_became_active":"2023-10-09T05:40:12.659094+0000","last_became_peered":"2023-10-09T05:40:12.659094+0000","last_unstale":"2023-10-09T05:40:13.672098+0000","last_undegraded":"2023-10-09T05:40:13.672098+0000","last_fullsized":"2023-10-09T05:40:13.672098+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:32:04.616532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.465478+0000","last_change":"2023-10-09T05:40:12.633917+0000","last_active":"2023-10-09T05:40:17.465478+0000","last_peered":"2023-10-09T05:40:17.465478+0000","last_clean":"2023-10-09T05:40:17.465478+0000","last_became_active":"2023-10-09T05:40:12.633637+0000","last_became_peered":"2023-10-09T05:40:12.633637+0000","last_unstale":"2023-10-09T05:40:17.465478+0000","last_undegraded":"2023-10-09T05:40:17.465478+0000","last_fullsized":"2023-10-09T05:40:17.465478+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:10.531016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.362919+0000","last_change":"2023-10-09T05:40:14.362515+0000","last_active":"2023-10-09T05:40:14.362919+0000","last_peered":"2023-10-09T05:40:14.362919+0000","last_clean":"2023-10-09T05:40:14.362919+0000","last_became_active":"2023-10-09T05:40:14.362184+0000","last_became_peered":"2023-10-09T05:40:14.362184+0000","last_unstale":"2023-10-09T05:40:14.362919+0000","last_undegraded":"2023-10-09T05:40:14.362919+0000","last_fullsized":"2023-10-09T05:40:14.362919+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:37.681193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:15.676207+0000","last_change":"2023-10-09T05:40:12.633085+0000","last_active":"2023-10-09T05:40:15.676207+0000","last_peered":"2023-10-09T05:40:15.676207+0000","last_clean":"2023-10-09T05:40:15.676207+0000","last_became_active":"2023-10-09T05:40:12.632853+0000","last_became_peered":"2023-10-09T05:40:12.632853+0000","last_unstale":"2023-10-09T05:40:15.676207+0000","last_undegraded":"2023-10-09T05:40:15.676207+0000","last_fullsized":"2023-10-09T05:40:15.676207+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:06.203927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.313310+0000","last_change":"2023-10-09T05:40:13.112524+0000","last_active":"2023-10-09T05:40:14.313310+0000","last_peered":"2023-10-09T05:40:14.313310+0000","last_clean":"2023-10-09T05:40:14.313310+0000","last_became_active":"2023-10-09T05:40:13.112241+0000","last_became_peered":"2023-10-09T05:40:13.112241+0000","last_unstale":"2023-10-09T05:40:14.313310+0000","last_undegraded":"2023-10-09T05:40:14.313310+0000","last_fullsized":"2023-10-09T05:40:14.313310+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:27:19.649999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":20,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.471709+0000","last_change":"2023-10-09T05:40:12.636208+0000","last_active":"2023-10-09T05:40:17.471709+0000","last_peered":"2023-10-09T05:40:17.471709+0000","last_clean":"2023-10-09T05:40:17.471709+0000","last_became_active":"2023-10-09T05:40:12.635908+0000","last_became_peered":"2023-10-09T05:40:12.635908+0000","last_unstale":"2023-10-09T05:40:17.471709+0000","last_undegraded":"2023-10-09T05:40:17.471709+0000","last_fullsized":"2023-10-09T05:40:17.471709+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:04.914952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.306271+0000","last_change":"2023-10-09T05:40:13.112221+0000","last_active":"2023-10-09T05:40:14.306271+0000","last_peered":"2023-10-09T05:40:14.306271+0000","last_clean":"2023-10-09T05:40:14.306271+0000","last_became_active":"2023-10-09T05:40:13.111921+0000","last_became_peered":"2023-10-09T05:40:13.111921+0000","last_unstale":"2023-10-09T05:40:14.306271+0000","last_undegraded":"2023-10-09T05:40:14.306271+0000","last_fullsized":"2023-10-09T05:40:14.306271+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:01:57.541469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.361682+0000","last_change":"2023-10-09T05:40:12.631893+0000","last_active":"2023-10-09T05:40:14.361682+0000","last_peered":"2023-10-09T05:40:14.361682+0000","last_clean":"2023-10-09T05:40:14.361682+0000","last_became_active":"2023-10-09T05:40:12.631592+0000","last_became_peered":"2023-10-09T05:40:12.631592+0000","last_unstale":"2023-10-09T05:40:14.361682+0000","last_undegraded":"2023-10-09T05:40:14.361682+0000","last_fullsized":"2023-10-09T05:40:14.361682+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:18:12.517403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.362867+0000","last_change":"2023-10-09T05:40:14.362514+0000","last_active":"2023-10-09T05:40:14.362867+0000","last_peered":"2023-10-09T05:40:14.362867+0000","last_clean":"2023-10-09T05:40:14.362867+0000","last_became_active":"2023-10-09T05:40:14.362188+0000","last_became_peered":"2023-10-09T05:40:14.362188+0000","last_unstale":"2023-10-09T05:40:14.362867+0000","last_undegraded":"2023-10-09T05:40:14.362867+0000","last_fullsized":"2023-10-09T05:40:14.362867+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:20:16.864500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.306199+0000","last_change":"2023-10-09T05:40:12.625243+0000","last_active":"2023-10-09T05:40:14.306199+0000","last_peered":"2023-10-09T05:40:14.306199+0000","last_clean":"2023-10-09T05:40:14.306199+0000","last_became_active":"2023-10-09T05:40:12.624960+0000","last_became_peered":"2023-10-09T05:40:12.624960+0000","last_unstale":"2023-10-09T05:40:14.306199+0000","last_undegraded":"2023-10-09T05:40:14.306199+0000","last_fullsized":"2023-10-09T05:40:14.306199+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:31.906798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.313272+0000","last_change":"2023-10-09T05:40:13.112786+0000","last_active":"2023-10-09T05:40:14.313272+0000","last_peered":"2023-10-09T05:40:14.313272+0000","last_clean":"2023-10-09T05:40:14.313272+0000","last_became_active":"2023-10-09T05:40:13.112019+0000","last_became_peered":"2023-10-09T05:40:13.112019+0000","last_unstale":"2023-10-09T05:40:14.313272+0000","last_undegraded":"2023-10-09T05:40:14.313272+0000","last_fullsized":"2023-10-09T05:40:14.313272+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:21.357286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.465902+0000","last_change":"2023-10-09T05:40:12.629501+0000","last_active":"2023-10-09T05:40:17.465902+0000","last_peered":"2023-10-09T05:40:17.465902+0000","last_clean":"2023-10-09T05:40:17.465902+0000","last_became_active":"2023-10-09T05:40:12.629196+0000","last_became_peered":"2023-10-09T05:40:12.629196+0000","last_unstale":"2023-10-09T05:40:17.465902+0000","last_undegraded":"2023-10-09T05:40:17.465902+0000","last_fullsized":"2023-10-09T05:40:17.465902+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:14.739294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.672193+0000","last_change":"2023-10-09T05:40:12.676722+0000","last_active":"2023-10-09T05:40:13.672193+0000","last_peered":"2023-10-09T05:40:13.672193+0000","last_clean":"2023-10-09T05:40:13.672193+0000","last_became_active":"2023-10-09T05:40:12.676301+0000","last_became_peered":"2023-10-09T05:40:12.676301+0000","last_unstale":"2023-10-09T05:40:13.672193+0000","last_undegraded":"2023-10-09T05:40:13.672193+0000","last_fullsized":"2023-10-09T05:40:13.672193+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:35.706368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.467140+0000","last_change":"2023-10-09T05:40:12.630786+0000","last_active":"2023-10-09T05:40:17.467140+0000","last_peered":"2023-10-09T05:40:17.467140+0000","last_clean":"2023-10-09T05:40:17.467140+0000","last_became_active":"2023-10-09T05:40:12.630528+0000","last_became_peered":"2023-10-09T05:40:12.630528+0000","last_unstale":"2023-10-09T05:40:17.467140+0000","last_undegraded":"2023-10-09T05:40:17.467140+0000","last_fullsized":"2023-10-09T05:40:17.467140+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:54.427346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.654121+0000","last_change":"2023-10-09T05:40:12.676256+0000","last_active":"2023-10-09T05:40:13.654121+0000","last_peered":"2023-10-09T05:40:13.654121+0000","last_clean":"2023-10-09T05:40:13.654121+0000","last_became_active":"2023-10-09T05:40:12.675931+0000","last_became_peered":"2023-10-09T05:40:12.675931+0000","last_unstale":"2023-10-09T05:40:13.654121+0000","last_undegraded":"2023-10-09T05:40:13.654121+0000","last_fullsized":"2023-10-09T05:40:13.654121+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:16.028244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.465838+0000","last_change":"2023-10-09T05:40:12.632736+0000","last_active":"2023-10-09T05:40:17.465838+0000","last_peered":"2023-10-09T05:40:17.465838+0000","last_clean":"2023-10-09T05:40:17.465838+0000","last_became_active":"2023-10-09T05:40:12.632460+0000","last_became_peered":"2023-10-09T05:40:12.632460+0000","last_unstale":"2023-10-09T05:40:17.465838+0000","last_undegraded":"2023-10-09T05:40:17.465838+0000","last_fullsized":"2023-10-09T05:40:17.465838+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:42.594033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.312981+0000","last_change":"2023-10-09T05:40:13.112751+0000","last_active":"2023-10-09T05:40:14.312981+0000","last_peered":"2023-10-09T05:40:14.312981+0000","last_clean":"2023-10-09T05:40:14.312981+0000","last_became_active":"2023-10-09T05:40:13.112431+0000","last_became_peered":"2023-10-09T05:40:13.112431+0000","last_unstale":"2023-10-09T05:40:14.312981+0000","last_undegraded":"2023-10-09T05:40:14.312981+0000","last_fullsized":"2023-10-09T05:40:14.312981+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:21.368942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.646640+0000","last_change":"2023-10-09T05:40:12.632865+0000","last_active":"2023-10-09T05:40:13.646640+0000","last_peered":"2023-10-09T05:40:13.646640+0000","last_clean":"2023-10-09T05:40:13.646640+0000","last_became_active":"2023-10-09T05:40:12.632578+0000","last_became_peered":"2023-10-09T05:40:12.632578+0000","last_unstale":"2023-10-09T05:40:13.646640+0000","last_undegraded":"2023-10-09T05:40:13.646640+0000","last_fullsized":"2023-10-09T05:40:13.646640+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:52:00.690041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.312961+0000","last_change":"2023-10-09T05:40:12.676413+0000","last_active":"2023-10-09T05:40:14.312961+0000","last_peered":"2023-10-09T05:40:14.312961+0000","last_clean":"2023-10-09T05:40:14.312961+0000","last_became_active":"2023-10-09T05:40:12.676072+0000","last_became_peered":"2023-10-09T05:40:12.676072+0000","last_unstale":"2023-10-09T05:40:14.312961+0000","last_undegraded":"2023-10-09T05:40:14.312961+0000","last_fullsized":"2023-10-09T05:40:14.312961+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:50.470411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.306494+0000","last_change":"2023-10-09T05:40:12.627611+0000","last_active":"2023-10-09T05:40:14.306494+0000","last_peered":"2023-10-09T05:40:14.306494+0000","last_clean":"2023-10-09T05:40:14.306494+0000","last_became_active":"2023-10-09T05:40:12.627213+0000","last_became_peered":"2023-10-09T05:40:12.627213+0000","last_unstale":"2023-10-09T05:40:14.306494+0000","last_undegraded":"2023-10-09T05:40:14.306494+0000","last_fullsized":"2023-10-09T05:40:14.306494+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:38.640471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.655967+0000","last_change":"2023-10-09T05:40:13.655774+0000","last_active":"2023-10-09T05:40:13.655967+0000","last_peered":"2023-10-09T05:40:13.655967+0000","last_clean":"2023-10-09T05:40:13.655967+0000","last_became_active":"2023-10-09T05:40:13.655490+0000","last_became_peered":"2023-10-09T05:40:13.655490+0000","last_unstale":"2023-10-09T05:40:13.655967+0000","last_undegraded":"2023-10-09T05:40:13.655967+0000","last_fullsized":"2023-10-09T05:40:13.655967+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:22.052294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.312993+0000","last_change":"2023-10-09T05:40:12.638336+0000","last_active":"2023-10-09T05:40:14.312993+0000","last_peered":"2023-10-09T05:40:14.312993+0000","last_clean":"2023-10-09T05:40:14.312993+0000","last_became_active":"2023-10-09T05:40:12.638039+0000","last_became_peered":"2023-10-09T05:40:12.638039+0000","last_unstale":"2023-10-09T05:40:14.312993+0000","last_undegraded":"2023-10-09T05:40:14.312993+0000","last_fullsized":"2023-10-09T05:40:14.312993+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:15.707770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.306305+0000","last_change":"2023-10-09T05:40:13.112218+0000","last_active":"2023-10-09T05:40:14.306305+0000","last_peered":"2023-10-09T05:40:14.306305+0000","last_clean":"2023-10-09T05:40:14.306305+0000","last_became_active":"2023-10-09T05:40:13.111890+0000","last_became_peered":"2023-10-09T05:40:13.111890+0000","last_unstale":"2023-10-09T05:40:14.306305+0000","last_undegraded":"2023-10-09T05:40:14.306305+0000","last_fullsized":"2023-10-09T05:40:14.306305+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:14.567445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.654208+0000","last_change":"2023-10-09T05:40:12.636600+0000","last_active":"2023-10-09T05:40:13.654208+0000","last_peered":"2023-10-09T05:40:13.654208+0000","last_clean":"2023-10-09T05:40:13.654208+0000","last_became_active":"2023-10-09T05:40:12.636312+0000","last_became_peered":"2023-10-09T05:40:12.636312+0000","last_unstale":"2023-10-09T05:40:13.654208+0000","last_undegraded":"2023-10-09T05:40:13.654208+0000","last_fullsized":"2023-10-09T05:40:13.654208+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:21.080726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.313250+0000","last_change":"2023-10-09T05:40:12.663793+0000","last_active":"2023-10-09T05:40:14.313250+0000","last_peered":"2023-10-09T05:40:14.313250+0000","last_clean":"2023-10-09T05:40:14.313250+0000","last_became_active":"2023-10-09T05:40:12.663487+0000","last_became_peered":"2023-10-09T05:40:12.663487+0000","last_unstale":"2023-10-09T05:40:14.313250+0000","last_undegraded":"2023-10-09T05:40:14.313250+0000","last_fullsized":"2023-10-09T05:40:14.313250+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:26.824199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:15.665630+0000","last_change":"2023-10-09T05:40:12.636780+0000","last_active":"2023-10-09T05:40:15.665630+0000","last_peered":"2023-10-09T05:40:15.665630+0000","last_clean":"2023-10-09T05:40:15.665630+0000","last_became_active":"2023-10-09T05:40:12.636503+0000","last_became_peered":"2023-10-09T05:40:12.636503+0000","last_unstale":"2023-10-09T05:40:15.665630+0000","last_undegraded":"2023-10-09T05:40:15.665630+0000","last_fullsized":"2023-10-09T05:40:15.665630+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:52:03.334368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234015,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298568,"kb_used_data":1280,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446584,"statfs":{"total":95995035648,"available":95689302016,"internally_reserved":0,"allocated":1310720,"data_stored":891354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.62,"5min":0.57199999999999995,"15min":0.56399999999999995},"min":{"1min":0.34100000000000003,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":2.629,"5min":2.629,"15min":2.629},"last":0.54700000000000004}]},{"osd":1,"last update":"Mon Oct 9 05:40:39 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.53500000000000003,"15min":0.52300000000000002},"min":{"1min":0.34499999999999997,"5min":0.313,"15min":0.313},"max":{"1min":2.5550000000000002,"5min":2.5550000000000002,"15min":2.5550000000000002},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.58899999999999997,"15min":0.59799999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":1.091,"5min":1.091,"15min":1.091},"last":0.42599999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:40:39 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.54700000000000004,"15min":0.55100000000000005},"min":{"1min":0.35499999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.113,"5min":1.113,"15min":1.113},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.56100000000000005,"15min":0.55200000000000005},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":3.202,"5min":3.202,"15min":3.202},"last":0.40500000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:40:39 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.56100000000000005,"15min":0.54700000000000004},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":3.2309999999999999,"5min":3.2309999999999999,"15min":3.2309999999999999},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.623,"5min":0.58199999999999996,"15min":0.57499999999999996},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.657,"5min":2.657,"15min":2.657},"last":0.44500000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:40:39 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.50700000000000001,"15min":0.51100000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":1.135,"5min":1.135,"15min":1.135},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.502,"15min":0.48599999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.165,"5min":3.165,"15min":3.165},"last":0.49099999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:40:39 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.51400000000000001,"15min":0.5},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":2.5990000000000002,"5min":2.5990000000000002,"15min":2.5990000000000002},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.54000000000000004,"15min":0.53500000000000003},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.0660000000000001,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.47399999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:40:39 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.55200000000000005,"15min":0.53600000000000003},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.8170000000000002,"5min":2.8170000000000002,"15min":2.8170000000000002},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.54800000000000004,"15min":0.52800000000000002},"min":{"1min":0.376,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":3.2589999999999999,"5min":3.2589999999999999,"15min":3.2589999999999999},"last":0.61599999999999999}]}]},{"osd":6,"up_from":46,"seq":197568495651,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1264,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1294336,"data_stored":890943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:12 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.505,"15min":0.501},"min":{"1min":0.38200000000000001,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.54300000000000004,"15min":0.54700000000000004},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.55500000000000005}]},{"osd":1,"last update":"Mon Oct 9 05:40:12 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.54100000000000004,"15min":0.53900000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.442}]},{"osd":2,"last update":"Mon Oct 9 05:40:12 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.67700000000000005,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.52400000000000002,"15min":0.52100000000000002},"min":{"1min":0.41799999999999998,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.58399999999999996}]},{"osd":3,"last update":"Mon Oct 9 05:40:12 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.53400000000000003,"15min":0.53100000000000003},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.625},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.57599999999999996}]},{"osd":4,"last update":"Mon Oct 9 05:40:12 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.438,"15min":0.44},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.55700000000000005,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.47399999999999998,"15min":0.47199999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.67300000000000004,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.66700000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:40:12 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.44900000000000001,"15min":0.44700000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.65700000000000003,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.501,"15min":0.497},"min":{"1min":0.32900000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.70699999999999996,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.69299999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:40:39 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.53500000000000003,"15min":0.53700000000000003},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58399999999999996,"15min":0.58299999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.71699999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574906,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297960,"kb_used_data":672,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447192,"statfs":{"total":95995035648,"available":95689924608,"internally_reserved":0,"allocated":688128,"data_stored":299292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:25 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35399999999999998,"15min":0.34999999999999998},"min":{"1min":0.224,"5min":0.219,"15min":0.219},"max":{"1min":0.59399999999999997,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.36799999999999999,"15min":0.34999999999999998},"min":{"1min":0.27500000000000002,"5min":0.22,"15min":0.22},"max":{"1min":0.55800000000000005,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.374}]},{"osd":2,"last update":"Mon Oct 9 05:40:47 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.40999999999999998,"15min":0.41299999999999998},"min":{"1min":0.28000000000000003,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.70099999999999996,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.39100000000000001,"15min":0.39900000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.29199999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:40:13 2023","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.41799999999999998,"15min":0.432},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.46500000000000002,"15min":0.47399999999999998},"min":{"1min":0.34599999999999997,"5min":0.31,"15min":0.31},"max":{"1min":0.59599999999999997,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.39600000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:40:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57499999999999996,"15min":0.57799999999999996},"min":{"1min":0.46600000000000003,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.78000000000000003,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.57099999999999995,"15min":0.57199999999999995},"min":{"1min":0.433,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.80300000000000005,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.80400000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:39:50 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.59699999999999998,"15min":0.60399999999999998},"min":{"1min":0.39600000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.80600000000000005,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.622,"5min":0.60299999999999998,"15min":0.59999999999999998},"min":{"1min":0.41299999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.7290000000000001,"5min":1.7290000000000001,"15min":1.7290000000000001},"last":0.84399999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:40:13 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.61199999999999999,"15min":0.61799999999999999},"min":{"1min":0.47199999999999998,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.68500000000000005,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.74299999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:40:34 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.58699999999999997,"15min":0.58299999999999996},"min":{"1min":0.46400000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.61899999999999999,"15min":0.621},"min":{"1min":0.48899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.76700000000000002,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.78800000000000003}]}]},{"osd":0,"up_from":8,"seq":34359738431,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298548,"kb_used_data":1260,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446604,"statfs":{"total":95995035648,"available":95689322496,"internally_reserved":0,"allocated":1290240,"data_stored":890236,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:40:27 2023","interfaces":[{"interface":"back","average":{"1min":0.32800000000000001,"5min":0.35999999999999999,"15min":0.34399999999999997},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.52400000000000002,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.27700000000000002},{"interface":"front","average":{"1min":0.37,"5min":0.378,"15min":0.35499999999999998},"min":{"1min":0.25900000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.41599999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:40:46 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.42299999999999999,"15min":0.42699999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.68999999999999995,"5min":1.55,"15min":1.55},"last":0.312},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.42499999999999999,"15min":0.433},"min":{"1min":0.26600000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.46300000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:40:12 2023","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.47699999999999998,"15min":0.5},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.49199999999999999,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.5,"15min":0.52400000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.74099999999999999,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.51000000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:40:27 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.60399999999999998,"15min":0.61799999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.70299999999999996,"5min":0.875,"15min":0.875},"last":0.59499999999999997},{"interface":"front","average":{"1min":1.1619999999999999,"5min":0.71499999999999997,"15min":0.64300000000000002},"min":{"1min":0.40999999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":6.5250000000000004,"5min":6.5250000000000004,"15min":6.5250000000000004},"last":0.65700000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.63,"15min":0.64800000000000002},"min":{"1min":0.45300000000000001,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.71199999999999997,"15min":0.69899999999999995},"min":{"1min":0.437,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.83499999999999996,"5min":5.6539999999999999,"15min":5.6539999999999999},"last":0.63100000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:40:12 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.65500000000000003,"15min":0.66800000000000004},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.73899999999999999,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.60899999999999999,"15min":0.621},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.73499999999999999,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.56100000000000005}]},{"osd":7,"last update":"Mon Oct 9 05:40:35 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":0.68500000000000005,"15min":0.65500000000000003},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":5.2779999999999996,"5min":5.2779999999999996,"15min":5.2779999999999996},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.86199999999999999,"5min":0.90000000000000002,"15min":0.90600000000000003},"min":{"1min":0.48399999999999999,"5min":0.379,"15min":0.379},"max":{"1min":5.3029999999999999,"5min":6.4660000000000002,"15min":6.4660000000000002},"last":0.60799999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411380,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":684,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":700416,"data_stored":300145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:42 2023","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.375,"15min":0.36599999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.23499999999999999},{"interface":"front","average":{"1min":0.38,"5min":0.40699999999999997,"15min":0.38300000000000001},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":1.2110000000000001,"5min":2.746,"15min":2.746},"last":0.34699999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:40:42 2023","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.38800000000000001,"15min":0.376},"min":{"1min":0.23799999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.373,"5min":0.42599999999999999,"15min":0.41599999999999998},"min":{"1min":0.25800000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.59399999999999997,"5min":1.929,"15min":1.929},"last":0.317}]},{"osd":3,"last update":"Mon Oct 9 05:40:11 2023","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.46100000000000002,"15min":0.46899999999999997},"min":{"1min":0.307,"5min":0.246,"15min":0.246},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.373},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.496,"15min":0.47599999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.895,"5min":2.895,"15min":2.895},"last":0.29299999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:40:34 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.60399999999999998,"15min":0.59999999999999998},"min":{"1min":0.40200000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.77800000000000002,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.60499999999999998,"15min":0.60399999999999998},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.63300000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:39:49 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.60399999999999998,"15min":0.60899999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.79500000000000004,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.61799999999999999,"15min":0.61299999999999999},"min":{"1min":0.46999999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.623}]},{"osd":6,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.65800000000000003,"15min":0.66000000000000003},"min":{"1min":0.46200000000000002,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.83699999999999997,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.77600000000000002},{"interface":"front","average":{"1min":0.751,"5min":0.65100000000000002,"15min":0.63400000000000001},"min":{"1min":0.434,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":2.8650000000000002,"5min":2.8650000000000002,"15min":2.8650000000000002},"last":0.61399999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:40:34 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.623,"15min":0.621},"min":{"1min":0.46999999999999997,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.73899999999999999,"15min":0.755},"min":{"1min":0.47799999999999998,"5min":0.439,"15min":0.439},"max":{"1min":0.91800000000000004,"5min":2.8210000000000002,"15min":2.8210000000000002},"last":0.78400000000000003}]}]},{"osd":3,"up_from":25,"seq":107374182448,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297968,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447184,"statfs":{"total":95995035648,"available":95689916416,"internally_reserved":0,"allocated":696320,"data_stored":299647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:10 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.40999999999999998,"15min":0.41799999999999998},"min":{"1min":0.26000000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.58299999999999996,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.27200000000000002,"5min":0.251,"15min":0.251},"max":{"1min":0.66800000000000004,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.40899999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:40:10 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.44800000000000001,"15min":0.46600000000000003},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.61599999999999999,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.38},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.46000000000000002,"15min":0.47299999999999998},"min":{"1min":0.33400000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.70199999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.29199999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:40:10 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.49199999999999999,"15min":0.51100000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.64100000000000001,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.45900000000000002,"15min":0.46999999999999997},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.68500000000000005,"5min":0.746,"15min":0.746},"last":0.33100000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:40:33 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.58899999999999997,"15min":0.58299999999999996},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.625,"15min":0.621},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.84499999999999997,"5min":1.008,"15min":1.008},"last":0.61199999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:39:56 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.65600000000000003,"15min":0.66800000000000004},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.70799999999999996,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.60499999999999998,"15min":0.61199999999999999},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.71799999999999997,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.60299999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.626,"15min":0.628},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.63800000000000001,"15min":0.63600000000000001},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.88500000000000001,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.626}]},{"osd":7,"last update":"Mon Oct 9 05:40:33 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.64300000000000002,"15min":0.65300000000000002},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.66100000000000003,"15min":0.66400000000000003},"min":{"1min":0.50900000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.64700000000000002}]}]},{"osd":4,"up_from":30,"seq":128849018924,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297988,"kb_used_data":700,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447164,"statfs":{"total":95995035648,"available":95689895936,"internally_reserved":0,"allocated":716800,"data_stored":301239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:35 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.54100000000000004,"15min":0.54800000000000004},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.69099999999999995,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.57499999999999996,"15min":0.60299999999999998},"min":{"1min":0.35999999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.75700000000000001,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.46300000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:40:35 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.55500000000000005,"15min":0.57699999999999996},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.74299999999999999,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.56699999999999995,"15min":0.58299999999999996},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.70099999999999996,"5min":1.006,"15min":1.006},"last":0.748}]},{"osd":2,"last update":"Mon Oct 9 05:40:35 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.56599999999999995,"15min":0.58399999999999996},"min":{"1min":0.41999999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.73899999999999999,"5min":1.026,"15min":1.026},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.56200000000000006,"15min":0.56999999999999995},"min":{"1min":0.40600000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.79000000000000004,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.64300000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:40:35 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57999999999999996,"15min":0.59199999999999997},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.751,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.56699999999999995,"15min":0.57899999999999996},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.71599999999999997,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.621}]},{"osd":5,"last update":"Mon Oct 9 05:39:53 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.435,"15min":0.42699999999999999},"min":{"1min":0.32100000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.44900000000000001,"15min":0.438},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.68500000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:40:12 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.47599999999999998,"15min":0.46700000000000003},"min":{"1min":0.34999999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.50900000000000001,"15min":0.501},"min":{"1min":0.35599999999999998,"5min":0.315,"15min":0.315},"max":{"1min":0.77100000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.66000000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:40:35 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.55100000000000005,"15min":0.54300000000000004},"min":{"1min":0.41599999999999998,"5min":0.37,"15min":0.37},"max":{"1min":0.79900000000000004,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.57799999999999996,"15min":0.57599999999999996},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.78300000000000003,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.71699999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757287,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":684,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":700416,"data_stored":300181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.51400000000000001,"15min":0.51700000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.54800000000000004,"15min":0.52600000000000002},"min":{"1min":0.40300000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":3.2149999999999999,"5min":3.2149999999999999,"15min":3.2149999999999999},"last":0.53400000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.51600000000000001,"15min":0.51800000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.56299999999999994,"15min":0.53900000000000003},"min":{"1min":0.40699999999999997,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":3.2589999999999999,"5min":3.2589999999999999,"15min":3.2589999999999999},"last":0.626}]},{"osd":2,"last update":"Mon Oct 9 05:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.66900000000000004,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.53300000000000003,"15min":0.53600000000000003},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.59199999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.52400000000000002,"15min":0.52200000000000002},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.56100000000000005,"15min":0.55900000000000005},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.55400000000000005}]},{"osd":4,"last update":"Mon Oct 9 05:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.42699999999999999,"15min":0.39500000000000002},"min":{"1min":0.314,"5min":0.312,"15min":0.312},"max":{"1min":3.5430000000000001,"5min":3.5430000000000001,"15min":3.5430000000000001},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.41399999999999998,"15min":0.39700000000000002},"min":{"1min":0.36799999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.69499999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:40:13 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.45100000000000001,"15min":0.44500000000000001},"min":{"1min":0.33200000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.61099999999999999,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.46899999999999997,"15min":0.45900000000000002},"min":{"1min":0.33100000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.66500000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:40:37 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.53500000000000003,"15min":0.52900000000000003},"min":{"1min":0.35999999999999999,"5min":0.318,"15min":0.318},"max":{"1min":0.73099999999999998,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.64400000000000002,"15min":0.66100000000000003},"min":{"1min":0.38500000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.68500000000000005,"5min":3.2869999999999999,"15min":3.2869999999999999},"last":0.71399999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:40:57.685 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:40:58.047 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:40:58.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:58 smithi156 ceph-mon[105979]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:40:58.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:40:58 smithi156 ceph-mon[105979]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:58.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:58 smithi064 ceph-mon[101070]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:40:58.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:40:58 smithi064 ceph-mon[101070]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:58.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:58 smithi064 ceph-mon[118354]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:40:58.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:40:58 smithi064 ceph-mon[118354]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:40:59.862 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:40:59.864 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T05:41:00.529 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":69,"stamp":"2023-10-09T05:40:59.560077+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385528,"kb_used_data":7224,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747575688,"statfs":{"total":767960285184,"available":765517504512,"internally_reserved":0,"allocated":7397376,"data_stored":4173037,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003133"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.671995+0000","last_change":"2023-10-09T05:40:12.676860+0000","last_active":"2023-10-09T05:40:13.671995+0000","last_peered":"2023-10-09T05:40:13.671995+0000","last_clean":"2023-10-09T05:40:13.671995+0000","last_became_active":"2023-10-09T05:40:12.676172+0000","last_became_peered":"2023-10-09T05:40:12.676172+0000","last_unstale":"2023-10-09T05:40:13.671995+0000","last_undegraded":"2023-10-09T05:40:13.671995+0000","last_fullsized":"2023-10-09T05:40:13.671995+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:26:11.807891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.464478+0000","last_change":"2023-10-09T05:40:12.627433+0000","last_active":"2023-10-09T05:40:17.464478+0000","last_peered":"2023-10-09T05:40:17.464478+0000","last_clean":"2023-10-09T05:40:17.464478+0000","last_became_active":"2023-10-09T05:40:12.626888+0000","last_became_peered":"2023-10-09T05:40:12.626888+0000","last_unstale":"2023-10-09T05:40:17.464478+0000","last_undegraded":"2023-10-09T05:40:17.464478+0000","last_fullsized":"2023-10-09T05:40:17.464478+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:22.990244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.312970+0000","last_change":"2023-10-09T05:40:12.660488+0000","last_active":"2023-10-09T05:40:14.312970+0000","last_peered":"2023-10-09T05:40:14.312970+0000","last_clean":"2023-10-09T05:40:14.312970+0000","last_became_active":"2023-10-09T05:40:12.660226+0000","last_became_peered":"2023-10-09T05:40:12.660226+0000","last_unstale":"2023-10-09T05:40:14.312970+0000","last_undegraded":"2023-10-09T05:40:14.312970+0000","last_fullsized":"2023-10-09T05:40:14.312970+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:11:18.728422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:15.718722+0000","last_change":"2023-10-09T05:40:12.637232+0000","last_active":"2023-10-09T05:40:15.718722+0000","last_peered":"2023-10-09T05:40:15.718722+0000","last_clean":"2023-10-09T05:40:15.718722+0000","last_became_active":"2023-10-09T05:40:12.636729+0000","last_became_peered":"2023-10-09T05:40:12.636729+0000","last_unstale":"2023-10-09T05:40:15.718722+0000","last_undegraded":"2023-10-09T05:40:15.718722+0000","last_fullsized":"2023-10-09T05:40:15.718722+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:16.890317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.672232+0000","last_change":"2023-10-09T05:40:12.660135+0000","last_active":"2023-10-09T05:40:13.672232+0000","last_peered":"2023-10-09T05:40:13.672232+0000","last_clean":"2023-10-09T05:40:13.672232+0000","last_became_active":"2023-10-09T05:40:12.659919+0000","last_became_peered":"2023-10-09T05:40:12.659919+0000","last_unstale":"2023-10-09T05:40:13.672232+0000","last_undegraded":"2023-10-09T05:40:13.672232+0000","last_fullsized":"2023-10-09T05:40:13.672232+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:02.925104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.465321+0000","last_change":"2023-10-09T05:40:12.633195+0000","last_active":"2023-10-09T05:40:17.465321+0000","last_peered":"2023-10-09T05:40:17.465321+0000","last_clean":"2023-10-09T05:40:17.465321+0000","last_became_active":"2023-10-09T05:40:12.632942+0000","last_became_peered":"2023-10-09T05:40:12.632942+0000","last_unstale":"2023-10-09T05:40:17.465321+0000","last_undegraded":"2023-10-09T05:40:17.465321+0000","last_fullsized":"2023-10-09T05:40:17.465321+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:04.470685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.672328+0000","last_change":"2023-10-09T05:40:12.659241+0000","last_active":"2023-10-09T05:40:13.672328+0000","last_peered":"2023-10-09T05:40:13.672328+0000","last_clean":"2023-10-09T05:40:13.672328+0000","last_became_active":"2023-10-09T05:40:12.658955+0000","last_became_peered":"2023-10-09T05:40:12.658955+0000","last_unstale":"2023-10-09T05:40:13.672328+0000","last_undegraded":"2023-10-09T05:40:13.672328+0000","last_fullsized":"2023-10-09T05:40:13.672328+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:19.986933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.464130+0000","last_change":"2023-10-09T05:40:12.631949+0000","last_active":"2023-10-09T05:40:17.464130+0000","last_peered":"2023-10-09T05:40:17.464130+0000","last_clean":"2023-10-09T05:40:17.464130+0000","last_became_active":"2023-10-09T05:40:12.631660+0000","last_became_peered":"2023-10-09T05:40:12.631660+0000","last_unstale":"2023-10-09T05:40:17.464130+0000","last_undegraded":"2023-10-09T05:40:17.464130+0000","last_fullsized":"2023-10-09T05:40:17.464130+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:36.314119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.655035+0000","last_change":"2023-10-09T05:40:12.659881+0000","last_active":"2023-10-09T05:40:13.655035+0000","last_peered":"2023-10-09T05:40:13.655035+0000","last_clean":"2023-10-09T05:40:13.655035+0000","last_became_active":"2023-10-09T05:40:12.659588+0000","last_became_peered":"2023-10-09T05:40:12.659588+0000","last_unstale":"2023-10-09T05:40:13.655035+0000","last_undegraded":"2023-10-09T05:40:13.655035+0000","last_fullsized":"2023-10-09T05:40:13.655035+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:09.897771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:15.633572+0000","last_change":"2023-10-09T05:40:12.631499+0000","last_active":"2023-10-09T05:40:15.633572+0000","last_peered":"2023-10-09T05:40:15.633572+0000","last_clean":"2023-10-09T05:40:15.633572+0000","last_became_active":"2023-10-09T05:40:12.630754+0000","last_became_peered":"2023-10-09T05:40:12.630754+0000","last_unstale":"2023-10-09T05:40:15.633572+0000","last_undegraded":"2023-10-09T05:40:15.633572+0000","last_fullsized":"2023-10-09T05:40:15.633572+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:51.146061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.672188+0000","last_change":"2023-10-09T05:40:12.676587+0000","last_active":"2023-10-09T05:40:13.672188+0000","last_peered":"2023-10-09T05:40:13.672188+0000","last_clean":"2023-10-09T05:40:13.672188+0000","last_became_active":"2023-10-09T05:40:12.676330+0000","last_became_peered":"2023-10-09T05:40:12.676330+0000","last_unstale":"2023-10-09T05:40:13.672188+0000","last_undegraded":"2023-10-09T05:40:13.672188+0000","last_fullsized":"2023-10-09T05:40:13.672188+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:46.659556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:15.633602+0000","last_change":"2023-10-09T05:40:12.625851+0000","last_active":"2023-10-09T05:40:15.633602+0000","last_peered":"2023-10-09T05:40:15.633602+0000","last_clean":"2023-10-09T05:40:15.633602+0000","last_became_active":"2023-10-09T05:40:12.625486+0000","last_became_peered":"2023-10-09T05:40:12.625486+0000","last_unstale":"2023-10-09T05:40:15.633602+0000","last_undegraded":"2023-10-09T05:40:15.633602+0000","last_fullsized":"2023-10-09T05:40:15.633602+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:08.394037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.648913+0000","last_change":"2023-10-09T05:39:18.963184+0000","last_active":"2023-10-09T05:40:13.648913+0000","last_peered":"2023-10-09T05:40:13.648913+0000","last_clean":"2023-10-09T05:40:13.648913+0000","last_became_active":"2023-10-09T05:39:18.962837+0000","last_became_peered":"2023-10-09T05:39:18.962837+0000","last_unstale":"2023-10-09T05:40:13.648913+0000","last_undegraded":"2023-10-09T05:40:13.648913+0000","last_fullsized":"2023-10-09T05:40:13.648913+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:05.818061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.312841+0000","last_change":"2023-10-09T05:40:13.112991+0000","last_active":"2023-10-09T05:40:14.312841+0000","last_peered":"2023-10-09T05:40:14.312841+0000","last_clean":"2023-10-09T05:40:14.312841+0000","last_became_active":"2023-10-09T05:40:13.112646+0000","last_became_peered":"2023-10-09T05:40:13.112646+0000","last_unstale":"2023-10-09T05:40:14.312841+0000","last_undegraded":"2023-10-09T05:40:14.312841+0000","last_fullsized":"2023-10-09T05:40:14.312841+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:00.533988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.654183+0000","last_change":"2023-10-09T05:39:19.405012+0000","last_active":"2023-10-09T05:40:13.654183+0000","last_peered":"2023-10-09T05:40:13.654183+0000","last_clean":"2023-10-09T05:40:13.654183+0000","last_became_active":"2023-10-09T05:39:19.404658+0000","last_became_peered":"2023-10-09T05:39:19.404658+0000","last_unstale":"2023-10-09T05:40:13.654183+0000","last_undegraded":"2023-10-09T05:40:13.654183+0000","last_fullsized":"2023-10-09T05:40:13.654183+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:24.956847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.464933+0000","last_change":"2023-10-09T05:40:12.630020+0000","last_active":"2023-10-09T05:40:17.464933+0000","last_peered":"2023-10-09T05:40:17.464933+0000","last_clean":"2023-10-09T05:40:17.464933+0000","last_became_active":"2023-10-09T05:40:12.629746+0000","last_became_peered":"2023-10-09T05:40:12.629746+0000","last_unstale":"2023-10-09T05:40:17.464933+0000","last_undegraded":"2023-10-09T05:40:17.464933+0000","last_fullsized":"2023-10-09T05:40:17.464933+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:05.159248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.306200+0000","last_change":"2023-10-09T05:40:13.112384+0000","last_active":"2023-10-09T05:40:14.306200+0000","last_peered":"2023-10-09T05:40:14.306200+0000","last_clean":"2023-10-09T05:40:14.306200+0000","last_became_active":"2023-10-09T05:40:13.112113+0000","last_became_peered":"2023-10-09T05:40:13.112113+0000","last_unstale":"2023-10-09T05:40:14.306200+0000","last_undegraded":"2023-10-09T05:40:14.306200+0000","last_fullsized":"2023-10-09T05:40:14.306200+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:00.125258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.654277+0000","last_change":"2023-10-09T05:40:12.640616+0000","last_active":"2023-10-09T05:40:13.654277+0000","last_peered":"2023-10-09T05:40:13.654277+0000","last_clean":"2023-10-09T05:40:13.654277+0000","last_became_active":"2023-10-09T05:40:12.640313+0000","last_became_peered":"2023-10-09T05:40:12.640313+0000","last_unstale":"2023-10-09T05:40:13.654277+0000","last_undegraded":"2023-10-09T05:40:13.654277+0000","last_fullsized":"2023-10-09T05:40:13.654277+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:48.607096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.361720+0000","last_change":"2023-10-09T05:39:19.405346+0000","last_active":"2023-10-09T05:40:14.361720+0000","last_peered":"2023-10-09T05:40:14.361720+0000","last_clean":"2023-10-09T05:40:14.361720+0000","last_became_active":"2023-10-09T05:39:19.404876+0000","last_became_peered":"2023-10-09T05:39:19.404876+0000","last_unstale":"2023-10-09T05:40:14.361720+0000","last_undegraded":"2023-10-09T05:40:14.361720+0000","last_fullsized":"2023-10-09T05:40:14.361720+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:28.442496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.306389+0000","last_change":"2023-10-09T05:40:12.653425+0000","last_active":"2023-10-09T05:40:14.306389+0000","last_peered":"2023-10-09T05:40:14.306389+0000","last_clean":"2023-10-09T05:40:14.306389+0000","last_became_active":"2023-10-09T05:40:12.652985+0000","last_became_peered":"2023-10-09T05:40:12.652985+0000","last_unstale":"2023-10-09T05:40:14.306389+0000","last_undegraded":"2023-10-09T05:40:14.306389+0000","last_fullsized":"2023-10-09T05:40:14.306389+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:52.785830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.306166+0000","last_change":"2023-10-09T05:40:12.630146+0000","last_active":"2023-10-09T05:40:14.306166+0000","last_peered":"2023-10-09T05:40:14.306166+0000","last_clean":"2023-10-09T05:40:14.306166+0000","last_became_active":"2023-10-09T05:40:12.629702+0000","last_became_peered":"2023-10-09T05:40:12.629702+0000","last_unstale":"2023-10-09T05:40:14.306166+0000","last_undegraded":"2023-10-09T05:40:14.306166+0000","last_fullsized":"2023-10-09T05:40:14.306166+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:31.056565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.361539+0000","last_change":"2023-10-09T05:39:18.964130+0000","last_active":"2023-10-09T05:40:14.361539+0000","last_peered":"2023-10-09T05:40:14.361539+0000","last_clean":"2023-10-09T05:40:14.361539+0000","last_became_active":"2023-10-09T05:39:18.963726+0000","last_became_peered":"2023-10-09T05:39:18.963726+0000","last_unstale":"2023-10-09T05:40:14.361539+0000","last_undegraded":"2023-10-09T05:40:14.361539+0000","last_fullsized":"2023-10-09T05:40:14.361539+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:13.428159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.363514+0000","last_change":"2023-10-09T05:40:14.363125+0000","last_active":"2023-10-09T05:40:14.363514+0000","last_peered":"2023-10-09T05:40:14.363514+0000","last_clean":"2023-10-09T05:40:14.363514+0000","last_became_active":"2023-10-09T05:40:14.362723+0000","last_became_peered":"2023-10-09T05:40:14.362723+0000","last_unstale":"2023-10-09T05:40:14.363514+0000","last_undegraded":"2023-10-09T05:40:14.363514+0000","last_fullsized":"2023-10-09T05:40:14.363514+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:30.741318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.313063+0000","last_change":"2023-10-09T05:40:12.633103+0000","last_active":"2023-10-09T05:40:14.313063+0000","last_peered":"2023-10-09T05:40:14.313063+0000","last_clean":"2023-10-09T05:40:14.313063+0000","last_became_active":"2023-10-09T05:40:12.632831+0000","last_became_peered":"2023-10-09T05:40:12.632831+0000","last_unstale":"2023-10-09T05:40:14.313063+0000","last_undegraded":"2023-10-09T05:40:14.313063+0000","last_fullsized":"2023-10-09T05:40:14.313063+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:32.796053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":276,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.361600+0000","last_change":"2023-10-09T05:38:35.869176+0000","last_active":"2023-10-09T05:40:14.361600+0000","last_peered":"2023-10-09T05:40:14.361600+0000","last_clean":"2023-10-09T05:40:14.361600+0000","last_became_active":"2023-10-09T05:38:35.868750+0000","last_became_peered":"2023-10-09T05:38:35.868750+0000","last_unstale":"2023-10-09T05:40:14.361600+0000","last_undegraded":"2023-10-09T05:40:14.361600+0000","last_fullsized":"2023-10-09T05:40:14.361600+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:20.200381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.646859+0000","last_change":"2023-10-09T05:39:18.963327+0000","last_active":"2023-10-09T05:40:13.646859+0000","last_peered":"2023-10-09T05:40:13.646859+0000","last_clean":"2023-10-09T05:40:13.646859+0000","last_became_active":"2023-10-09T05:39:18.962933+0000","last_became_peered":"2023-10-09T05:39:18.962933+0000","last_unstale":"2023-10-09T05:40:13.646859+0000","last_undegraded":"2023-10-09T05:40:13.646859+0000","last_fullsized":"2023-10-09T05:40:13.646859+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:43.427745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.654564+0000","last_change":"2023-10-09T05:40:12.665952+0000","last_active":"2023-10-09T05:40:13.654564+0000","last_peered":"2023-10-09T05:40:13.654564+0000","last_clean":"2023-10-09T05:40:13.654564+0000","last_became_active":"2023-10-09T05:40:12.665664+0000","last_became_peered":"2023-10-09T05:40:12.665664+0000","last_unstale":"2023-10-09T05:40:13.654564+0000","last_undegraded":"2023-10-09T05:40:13.654564+0000","last_fullsized":"2023-10-09T05:40:13.654564+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:58.953498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.654253+0000","last_change":"2023-10-09T05:40:12.633205+0000","last_active":"2023-10-09T05:40:13.654253+0000","last_peered":"2023-10-09T05:40:13.654253+0000","last_clean":"2023-10-09T05:40:13.654253+0000","last_became_active":"2023-10-09T05:40:12.632846+0000","last_became_peered":"2023-10-09T05:40:12.632846+0000","last_unstale":"2023-10-09T05:40:13.654253+0000","last_undegraded":"2023-10-09T05:40:13.654253+0000","last_fullsized":"2023-10-09T05:40:13.654253+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:16.854263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.654499+0000","last_change":"2023-10-09T05:39:19.362360+0000","last_active":"2023-10-09T05:40:13.654499+0000","last_peered":"2023-10-09T05:40:13.654499+0000","last_clean":"2023-10-09T05:40:13.654499+0000","last_became_active":"2023-10-09T05:39:19.361972+0000","last_became_peered":"2023-10-09T05:39:19.361972+0000","last_unstale":"2023-10-09T05:40:13.654499+0000","last_undegraded":"2023-10-09T05:40:13.654499+0000","last_fullsized":"2023-10-09T05:40:13.654499+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:06.194167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.654225+0000","last_change":"2023-10-09T05:40:12.664058+0000","last_active":"2023-10-09T05:40:13.654225+0000","last_peered":"2023-10-09T05:40:13.654225+0000","last_clean":"2023-10-09T05:40:13.654225+0000","last_became_active":"2023-10-09T05:40:12.663759+0000","last_became_peered":"2023-10-09T05:40:12.663759+0000","last_unstale":"2023-10-09T05:40:13.654225+0000","last_undegraded":"2023-10-09T05:40:13.654225+0000","last_fullsized":"2023-10-09T05:40:13.654225+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:51.023928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.672315+0000","last_change":"2023-10-09T05:40:12.630487+0000","last_active":"2023-10-09T05:40:13.672315+0000","last_peered":"2023-10-09T05:40:13.672315+0000","last_clean":"2023-10-09T05:40:13.672315+0000","last_became_active":"2023-10-09T05:40:12.630135+0000","last_became_peered":"2023-10-09T05:40:12.630135+0000","last_unstale":"2023-10-09T05:40:13.672315+0000","last_undegraded":"2023-10-09T05:40:13.672315+0000","last_fullsized":"2023-10-09T05:40:13.672315+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:59.045086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.646524+0000","last_change":"2023-10-09T05:39:18.963544+0000","last_active":"2023-10-09T05:40:13.646524+0000","last_peered":"2023-10-09T05:40:13.646524+0000","last_clean":"2023-10-09T05:40:13.646524+0000","last_became_active":"2023-10-09T05:39:18.962981+0000","last_became_peered":"2023-10-09T05:39:18.962981+0000","last_unstale":"2023-10-09T05:40:13.646524+0000","last_undegraded":"2023-10-09T05:40:13.646524+0000","last_fullsized":"2023-10-09T05:40:13.646524+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:11.589793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.654223+0000","last_change":"2023-10-09T05:40:12.665302+0000","last_active":"2023-10-09T05:40:13.654223+0000","last_peered":"2023-10-09T05:40:13.654223+0000","last_clean":"2023-10-09T05:40:13.654223+0000","last_became_active":"2023-10-09T05:40:12.665037+0000","last_became_peered":"2023-10-09T05:40:12.665037+0000","last_unstale":"2023-10-09T05:40:13.654223+0000","last_undegraded":"2023-10-09T05:40:13.654223+0000","last_fullsized":"2023-10-09T05:40:13.654223+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:10.300607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.646806+0000","last_change":"2023-10-09T05:40:12.627754+0000","last_active":"2023-10-09T05:40:13.646806+0000","last_peered":"2023-10-09T05:40:13.646806+0000","last_clean":"2023-10-09T05:40:13.646806+0000","last_became_active":"2023-10-09T05:40:12.627490+0000","last_became_peered":"2023-10-09T05:40:12.627490+0000","last_unstale":"2023-10-09T05:40:13.646806+0000","last_undegraded":"2023-10-09T05:40:13.646806+0000","last_fullsized":"2023-10-09T05:40:13.646806+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:47.650326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.654372+0000","last_change":"2023-10-09T05:39:18.964305+0000","last_active":"2023-10-09T05:40:13.654372+0000","last_peered":"2023-10-09T05:40:13.654372+0000","last_clean":"2023-10-09T05:40:13.654372+0000","last_became_active":"2023-10-09T05:39:18.963935+0000","last_became_peered":"2023-10-09T05:39:18.963935+0000","last_unstale":"2023-10-09T05:40:13.654372+0000","last_undegraded":"2023-10-09T05:40:13.654372+0000","last_fullsized":"2023-10-09T05:40:13.654372+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:33.105059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.654634+0000","last_change":"2023-10-09T05:40:12.662743+0000","last_active":"2023-10-09T05:40:13.654634+0000","last_peered":"2023-10-09T05:40:13.654634+0000","last_clean":"2023-10-09T05:40:13.654634+0000","last_became_active":"2023-10-09T05:40:12.662488+0000","last_became_peered":"2023-10-09T05:40:12.662488+0000","last_unstale":"2023-10-09T05:40:13.654634+0000","last_undegraded":"2023-10-09T05:40:13.654634+0000","last_fullsized":"2023-10-09T05:40:13.654634+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:25.038412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.312860+0000","last_change":"2023-10-09T05:40:12.638512+0000","last_active":"2023-10-09T05:40:14.312860+0000","last_peered":"2023-10-09T05:40:14.312860+0000","last_clean":"2023-10-09T05:40:14.312860+0000","last_became_active":"2023-10-09T05:40:12.638030+0000","last_became_peered":"2023-10-09T05:40:12.638030+0000","last_unstale":"2023-10-09T05:40:14.312860+0000","last_undegraded":"2023-10-09T05:40:14.312860+0000","last_fullsized":"2023-10-09T05:40:14.312860+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:24.393305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.672582+0000","last_change":"2023-10-09T05:40:12.676961+0000","last_active":"2023-10-09T05:40:13.672582+0000","last_peered":"2023-10-09T05:40:13.672582+0000","last_clean":"2023-10-09T05:40:13.672582+0000","last_became_active":"2023-10-09T05:40:12.676453+0000","last_became_peered":"2023-10-09T05:40:12.676453+0000","last_unstale":"2023-10-09T05:40:13.672582+0000","last_undegraded":"2023-10-09T05:40:13.672582+0000","last_fullsized":"2023-10-09T05:40:13.672582+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:41.173802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.672204+0000","last_change":"2023-10-09T05:40:12.626069+0000","last_active":"2023-10-09T05:40:13.672204+0000","last_peered":"2023-10-09T05:40:13.672204+0000","last_clean":"2023-10-09T05:40:13.672204+0000","last_became_active":"2023-10-09T05:40:12.625380+0000","last_became_peered":"2023-10-09T05:40:12.625380+0000","last_unstale":"2023-10-09T05:40:13.672204+0000","last_undegraded":"2023-10-09T05:40:13.672204+0000","last_fullsized":"2023-10-09T05:40:13.672204+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:32:11.601924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.672690+0000","last_change":"2023-10-09T05:40:12.652676+0000","last_active":"2023-10-09T05:40:13.672690+0000","last_peered":"2023-10-09T05:40:13.672690+0000","last_clean":"2023-10-09T05:40:13.672690+0000","last_became_active":"2023-10-09T05:40:12.652437+0000","last_became_peered":"2023-10-09T05:40:12.652437+0000","last_unstale":"2023-10-09T05:40:13.672690+0000","last_undegraded":"2023-10-09T05:40:13.672690+0000","last_fullsized":"2023-10-09T05:40:13.672690+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:12.886016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.648820+0000","last_change":"2023-10-09T05:40:12.637136+0000","last_active":"2023-10-09T05:40:13.648820+0000","last_peered":"2023-10-09T05:40:13.648820+0000","last_clean":"2023-10-09T05:40:13.648820+0000","last_became_active":"2023-10-09T05:40:12.636812+0000","last_became_peered":"2023-10-09T05:40:12.636812+0000","last_unstale":"2023-10-09T05:40:13.648820+0000","last_undegraded":"2023-10-09T05:40:13.648820+0000","last_fullsized":"2023-10-09T05:40:13.648820+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:59.890646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.672415+0000","last_change":"2023-10-09T05:40:12.676584+0000","last_active":"2023-10-09T05:40:13.672415+0000","last_peered":"2023-10-09T05:40:13.672415+0000","last_clean":"2023-10-09T05:40:13.672415+0000","last_became_active":"2023-10-09T05:40:12.676241+0000","last_became_peered":"2023-10-09T05:40:12.676241+0000","last_unstale":"2023-10-09T05:40:13.672415+0000","last_undegraded":"2023-10-09T05:40:13.672415+0000","last_fullsized":"2023-10-09T05:40:13.672415+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:20:52.246349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.465840+0000","last_change":"2023-10-09T05:40:12.625178+0000","last_active":"2023-10-09T05:40:17.465840+0000","last_peered":"2023-10-09T05:40:17.465840+0000","last_clean":"2023-10-09T05:40:17.465840+0000","last_became_active":"2023-10-09T05:40:12.624841+0000","last_became_peered":"2023-10-09T05:40:12.624841+0000","last_unstale":"2023-10-09T05:40:17.465840+0000","last_undegraded":"2023-10-09T05:40:17.465840+0000","last_fullsized":"2023-10-09T05:40:17.465840+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:11.715344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.306314+0000","last_change":"2023-10-09T05:40:12.656403+0000","last_active":"2023-10-09T05:40:14.306314+0000","last_peered":"2023-10-09T05:40:14.306314+0000","last_clean":"2023-10-09T05:40:14.306314+0000","last_became_active":"2023-10-09T05:40:12.655733+0000","last_became_peered":"2023-10-09T05:40:12.655733+0000","last_unstale":"2023-10-09T05:40:14.306314+0000","last_undegraded":"2023-10-09T05:40:14.306314+0000","last_fullsized":"2023-10-09T05:40:14.306314+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:30.078067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.646654+0000","last_change":"2023-10-09T05:40:12.634263+0000","last_active":"2023-10-09T05:40:13.646654+0000","last_peered":"2023-10-09T05:40:13.646654+0000","last_clean":"2023-10-09T05:40:13.646654+0000","last_became_active":"2023-10-09T05:40:12.633644+0000","last_became_peered":"2023-10-09T05:40:12.633644+0000","last_unstale":"2023-10-09T05:40:13.646654+0000","last_undegraded":"2023-10-09T05:40:13.646654+0000","last_fullsized":"2023-10-09T05:40:13.646654+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:57.270857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.362272+0000","last_change":"2023-10-09T05:40:14.362061+0000","last_active":"2023-10-09T05:40:14.362272+0000","last_peered":"2023-10-09T05:40:14.362272+0000","last_clean":"2023-10-09T05:40:14.362272+0000","last_became_active":"2023-10-09T05:40:14.361712+0000","last_became_peered":"2023-10-09T05:40:14.361712+0000","last_unstale":"2023-10-09T05:40:14.362272+0000","last_undegraded":"2023-10-09T05:40:14.362272+0000","last_fullsized":"2023-10-09T05:40:14.362272+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:07.409309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:15.676122+0000","last_change":"2023-10-09T05:40:12.632083+0000","last_active":"2023-10-09T05:40:15.676122+0000","last_peered":"2023-10-09T05:40:15.676122+0000","last_clean":"2023-10-09T05:40:15.676122+0000","last_became_active":"2023-10-09T05:40:12.631715+0000","last_became_peered":"2023-10-09T05:40:12.631715+0000","last_unstale":"2023-10-09T05:40:15.676122+0000","last_undegraded":"2023-10-09T05:40:15.676122+0000","last_fullsized":"2023-10-09T05:40:15.676122+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:17.641378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.672098+0000","last_change":"2023-10-09T05:40:12.659398+0000","last_active":"2023-10-09T05:40:13.672098+0000","last_peered":"2023-10-09T05:40:13.672098+0000","last_clean":"2023-10-09T05:40:13.672098+0000","last_became_active":"2023-10-09T05:40:12.659094+0000","last_became_peered":"2023-10-09T05:40:12.659094+0000","last_unstale":"2023-10-09T05:40:13.672098+0000","last_undegraded":"2023-10-09T05:40:13.672098+0000","last_fullsized":"2023-10-09T05:40:13.672098+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:32:04.616532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.465478+0000","last_change":"2023-10-09T05:40:12.633917+0000","last_active":"2023-10-09T05:40:17.465478+0000","last_peered":"2023-10-09T05:40:17.465478+0000","last_clean":"2023-10-09T05:40:17.465478+0000","last_became_active":"2023-10-09T05:40:12.633637+0000","last_became_peered":"2023-10-09T05:40:12.633637+0000","last_unstale":"2023-10-09T05:40:17.465478+0000","last_undegraded":"2023-10-09T05:40:17.465478+0000","last_fullsized":"2023-10-09T05:40:17.465478+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:10.531016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.362919+0000","last_change":"2023-10-09T05:40:14.362515+0000","last_active":"2023-10-09T05:40:14.362919+0000","last_peered":"2023-10-09T05:40:14.362919+0000","last_clean":"2023-10-09T05:40:14.362919+0000","last_became_active":"2023-10-09T05:40:14.362184+0000","last_became_peered":"2023-10-09T05:40:14.362184+0000","last_unstale":"2023-10-09T05:40:14.362919+0000","last_undegraded":"2023-10-09T05:40:14.362919+0000","last_fullsized":"2023-10-09T05:40:14.362919+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:37.681193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:15.676207+0000","last_change":"2023-10-09T05:40:12.633085+0000","last_active":"2023-10-09T05:40:15.676207+0000","last_peered":"2023-10-09T05:40:15.676207+0000","last_clean":"2023-10-09T05:40:15.676207+0000","last_became_active":"2023-10-09T05:40:12.632853+0000","last_became_peered":"2023-10-09T05:40:12.632853+0000","last_unstale":"2023-10-09T05:40:15.676207+0000","last_undegraded":"2023-10-09T05:40:15.676207+0000","last_fullsized":"2023-10-09T05:40:15.676207+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:06.203927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.313310+0000","last_change":"2023-10-09T05:40:13.112524+0000","last_active":"2023-10-09T05:40:14.313310+0000","last_peered":"2023-10-09T05:40:14.313310+0000","last_clean":"2023-10-09T05:40:14.313310+0000","last_became_active":"2023-10-09T05:40:13.112241+0000","last_became_peered":"2023-10-09T05:40:13.112241+0000","last_unstale":"2023-10-09T05:40:14.313310+0000","last_undegraded":"2023-10-09T05:40:14.313310+0000","last_fullsized":"2023-10-09T05:40:14.313310+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:27:19.649999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":20,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.471709+0000","last_change":"2023-10-09T05:40:12.636208+0000","last_active":"2023-10-09T05:40:17.471709+0000","last_peered":"2023-10-09T05:40:17.471709+0000","last_clean":"2023-10-09T05:40:17.471709+0000","last_became_active":"2023-10-09T05:40:12.635908+0000","last_became_peered":"2023-10-09T05:40:12.635908+0000","last_unstale":"2023-10-09T05:40:17.471709+0000","last_undegraded":"2023-10-09T05:40:17.471709+0000","last_fullsized":"2023-10-09T05:40:17.471709+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:04.914952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.306271+0000","last_change":"2023-10-09T05:40:13.112221+0000","last_active":"2023-10-09T05:40:14.306271+0000","last_peered":"2023-10-09T05:40:14.306271+0000","last_clean":"2023-10-09T05:40:14.306271+0000","last_became_active":"2023-10-09T05:40:13.111921+0000","last_became_peered":"2023-10-09T05:40:13.111921+0000","last_unstale":"2023-10-09T05:40:14.306271+0000","last_undegraded":"2023-10-09T05:40:14.306271+0000","last_fullsized":"2023-10-09T05:40:14.306271+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:01:57.541469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.361682+0000","last_change":"2023-10-09T05:40:12.631893+0000","last_active":"2023-10-09T05:40:14.361682+0000","last_peered":"2023-10-09T05:40:14.361682+0000","last_clean":"2023-10-09T05:40:14.361682+0000","last_became_active":"2023-10-09T05:40:12.631592+0000","last_became_peered":"2023-10-09T05:40:12.631592+0000","last_unstale":"2023-10-09T05:40:14.361682+0000","last_undegraded":"2023-10-09T05:40:14.361682+0000","last_fullsized":"2023-10-09T05:40:14.361682+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:18:12.517403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.362867+0000","last_change":"2023-10-09T05:40:14.362514+0000","last_active":"2023-10-09T05:40:14.362867+0000","last_peered":"2023-10-09T05:40:14.362867+0000","last_clean":"2023-10-09T05:40:14.362867+0000","last_became_active":"2023-10-09T05:40:14.362188+0000","last_became_peered":"2023-10-09T05:40:14.362188+0000","last_unstale":"2023-10-09T05:40:14.362867+0000","last_undegraded":"2023-10-09T05:40:14.362867+0000","last_fullsized":"2023-10-09T05:40:14.362867+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:20:16.864500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.306199+0000","last_change":"2023-10-09T05:40:12.625243+0000","last_active":"2023-10-09T05:40:14.306199+0000","last_peered":"2023-10-09T05:40:14.306199+0000","last_clean":"2023-10-09T05:40:14.306199+0000","last_became_active":"2023-10-09T05:40:12.624960+0000","last_became_peered":"2023-10-09T05:40:12.624960+0000","last_unstale":"2023-10-09T05:40:14.306199+0000","last_undegraded":"2023-10-09T05:40:14.306199+0000","last_fullsized":"2023-10-09T05:40:14.306199+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:31.906798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.313272+0000","last_change":"2023-10-09T05:40:13.112786+0000","last_active":"2023-10-09T05:40:14.313272+0000","last_peered":"2023-10-09T05:40:14.313272+0000","last_clean":"2023-10-09T05:40:14.313272+0000","last_became_active":"2023-10-09T05:40:13.112019+0000","last_became_peered":"2023-10-09T05:40:13.112019+0000","last_unstale":"2023-10-09T05:40:14.313272+0000","last_undegraded":"2023-10-09T05:40:14.313272+0000","last_fullsized":"2023-10-09T05:40:14.313272+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:21.357286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.465902+0000","last_change":"2023-10-09T05:40:12.629501+0000","last_active":"2023-10-09T05:40:17.465902+0000","last_peered":"2023-10-09T05:40:17.465902+0000","last_clean":"2023-10-09T05:40:17.465902+0000","last_became_active":"2023-10-09T05:40:12.629196+0000","last_became_peered":"2023-10-09T05:40:12.629196+0000","last_unstale":"2023-10-09T05:40:17.465902+0000","last_undegraded":"2023-10-09T05:40:17.465902+0000","last_fullsized":"2023-10-09T05:40:17.465902+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:14.739294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.672193+0000","last_change":"2023-10-09T05:40:12.676722+0000","last_active":"2023-10-09T05:40:13.672193+0000","last_peered":"2023-10-09T05:40:13.672193+0000","last_clean":"2023-10-09T05:40:13.672193+0000","last_became_active":"2023-10-09T05:40:12.676301+0000","last_became_peered":"2023-10-09T05:40:12.676301+0000","last_unstale":"2023-10-09T05:40:13.672193+0000","last_undegraded":"2023-10-09T05:40:13.672193+0000","last_fullsized":"2023-10-09T05:40:13.672193+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:35.706368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.467140+0000","last_change":"2023-10-09T05:40:12.630786+0000","last_active":"2023-10-09T05:40:17.467140+0000","last_peered":"2023-10-09T05:40:17.467140+0000","last_clean":"2023-10-09T05:40:17.467140+0000","last_became_active":"2023-10-09T05:40:12.630528+0000","last_became_peered":"2023-10-09T05:40:12.630528+0000","last_unstale":"2023-10-09T05:40:17.467140+0000","last_undegraded":"2023-10-09T05:40:17.467140+0000","last_fullsized":"2023-10-09T05:40:17.467140+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:54.427346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.654121+0000","last_change":"2023-10-09T05:40:12.676256+0000","last_active":"2023-10-09T05:40:13.654121+0000","last_peered":"2023-10-09T05:40:13.654121+0000","last_clean":"2023-10-09T05:40:13.654121+0000","last_became_active":"2023-10-09T05:40:12.675931+0000","last_became_peered":"2023-10-09T05:40:12.675931+0000","last_unstale":"2023-10-09T05:40:13.654121+0000","last_undegraded":"2023-10-09T05:40:13.654121+0000","last_fullsized":"2023-10-09T05:40:13.654121+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:16.028244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:17.465838+0000","last_change":"2023-10-09T05:40:12.632736+0000","last_active":"2023-10-09T05:40:17.465838+0000","last_peered":"2023-10-09T05:40:17.465838+0000","last_clean":"2023-10-09T05:40:17.465838+0000","last_became_active":"2023-10-09T05:40:12.632460+0000","last_became_peered":"2023-10-09T05:40:12.632460+0000","last_unstale":"2023-10-09T05:40:17.465838+0000","last_undegraded":"2023-10-09T05:40:17.465838+0000","last_fullsized":"2023-10-09T05:40:17.465838+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:42.594033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.312981+0000","last_change":"2023-10-09T05:40:13.112751+0000","last_active":"2023-10-09T05:40:14.312981+0000","last_peered":"2023-10-09T05:40:14.312981+0000","last_clean":"2023-10-09T05:40:14.312981+0000","last_became_active":"2023-10-09T05:40:13.112431+0000","last_became_peered":"2023-10-09T05:40:13.112431+0000","last_unstale":"2023-10-09T05:40:14.312981+0000","last_undegraded":"2023-10-09T05:40:14.312981+0000","last_fullsized":"2023-10-09T05:40:14.312981+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:21.368942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.646640+0000","last_change":"2023-10-09T05:40:12.632865+0000","last_active":"2023-10-09T05:40:13.646640+0000","last_peered":"2023-10-09T05:40:13.646640+0000","last_clean":"2023-10-09T05:40:13.646640+0000","last_became_active":"2023-10-09T05:40:12.632578+0000","last_became_peered":"2023-10-09T05:40:12.632578+0000","last_unstale":"2023-10-09T05:40:13.646640+0000","last_undegraded":"2023-10-09T05:40:13.646640+0000","last_fullsized":"2023-10-09T05:40:13.646640+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:52:00.690041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.312961+0000","last_change":"2023-10-09T05:40:12.676413+0000","last_active":"2023-10-09T05:40:14.312961+0000","last_peered":"2023-10-09T05:40:14.312961+0000","last_clean":"2023-10-09T05:40:14.312961+0000","last_became_active":"2023-10-09T05:40:12.676072+0000","last_became_peered":"2023-10-09T05:40:12.676072+0000","last_unstale":"2023-10-09T05:40:14.312961+0000","last_undegraded":"2023-10-09T05:40:14.312961+0000","last_fullsized":"2023-10-09T05:40:14.312961+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:50.470411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.306494+0000","last_change":"2023-10-09T05:40:12.627611+0000","last_active":"2023-10-09T05:40:14.306494+0000","last_peered":"2023-10-09T05:40:14.306494+0000","last_clean":"2023-10-09T05:40:14.306494+0000","last_became_active":"2023-10-09T05:40:12.627213+0000","last_became_peered":"2023-10-09T05:40:12.627213+0000","last_unstale":"2023-10-09T05:40:14.306494+0000","last_undegraded":"2023-10-09T05:40:14.306494+0000","last_fullsized":"2023-10-09T05:40:14.306494+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:38.640471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.655967+0000","last_change":"2023-10-09T05:40:13.655774+0000","last_active":"2023-10-09T05:40:13.655967+0000","last_peered":"2023-10-09T05:40:13.655967+0000","last_clean":"2023-10-09T05:40:13.655967+0000","last_became_active":"2023-10-09T05:40:13.655490+0000","last_became_peered":"2023-10-09T05:40:13.655490+0000","last_unstale":"2023-10-09T05:40:13.655967+0000","last_undegraded":"2023-10-09T05:40:13.655967+0000","last_fullsized":"2023-10-09T05:40:13.655967+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:22.052294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.312993+0000","last_change":"2023-10-09T05:40:12.638336+0000","last_active":"2023-10-09T05:40:14.312993+0000","last_peered":"2023-10-09T05:40:14.312993+0000","last_clean":"2023-10-09T05:40:14.312993+0000","last_became_active":"2023-10-09T05:40:12.638039+0000","last_became_peered":"2023-10-09T05:40:12.638039+0000","last_unstale":"2023-10-09T05:40:14.312993+0000","last_undegraded":"2023-10-09T05:40:14.312993+0000","last_fullsized":"2023-10-09T05:40:14.312993+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:15.707770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.306305+0000","last_change":"2023-10-09T05:40:13.112218+0000","last_active":"2023-10-09T05:40:14.306305+0000","last_peered":"2023-10-09T05:40:14.306305+0000","last_clean":"2023-10-09T05:40:14.306305+0000","last_became_active":"2023-10-09T05:40:13.111890+0000","last_became_peered":"2023-10-09T05:40:13.111890+0000","last_unstale":"2023-10-09T05:40:14.306305+0000","last_undegraded":"2023-10-09T05:40:14.306305+0000","last_fullsized":"2023-10-09T05:40:14.306305+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:14.567445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:13.654208+0000","last_change":"2023-10-09T05:40:12.636600+0000","last_active":"2023-10-09T05:40:13.654208+0000","last_peered":"2023-10-09T05:40:13.654208+0000","last_clean":"2023-10-09T05:40:13.654208+0000","last_became_active":"2023-10-09T05:40:12.636312+0000","last_became_peered":"2023-10-09T05:40:12.636312+0000","last_unstale":"2023-10-09T05:40:13.654208+0000","last_undegraded":"2023-10-09T05:40:13.654208+0000","last_fullsized":"2023-10-09T05:40:13.654208+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:21.080726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:40:14.313250+0000","last_change":"2023-10-09T05:40:12.663793+0000","last_active":"2023-10-09T05:40:14.313250+0000","last_peered":"2023-10-09T05:40:14.313250+0000","last_clean":"2023-10-09T05:40:14.313250+0000","last_became_active":"2023-10-09T05:40:12.663487+0000","last_became_peered":"2023-10-09T05:40:12.663487+0000","last_unstale":"2023-10-09T05:40:14.313250+0000","last_undegraded":"2023-10-09T05:40:14.313250+0000","last_fullsized":"2023-10-09T05:40:14.313250+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:26.824199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:40:15.665630+0000","last_change":"2023-10-09T05:40:12.636780+0000","last_active":"2023-10-09T05:40:15.665630+0000","last_peered":"2023-10-09T05:40:15.665630+0000","last_clean":"2023-10-09T05:40:15.665630+0000","last_became_active":"2023-10-09T05:40:12.636503+0000","last_became_peered":"2023-10-09T05:40:12.636503+0000","last_unstale":"2023-10-09T05:40:15.665630+0000","last_undegraded":"2023-10-09T05:40:15.665630+0000","last_fullsized":"2023-10-09T05:40:15.665630+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:52:03.334368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234016,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298568,"kb_used_data":1280,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446584,"statfs":{"total":95995035648,"available":95689302016,"internally_reserved":0,"allocated":1310720,"data_stored":891354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.62,"5min":0.57199999999999995,"15min":0.56399999999999995},"min":{"1min":0.34100000000000003,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":2.629,"5min":2.629,"15min":2.629},"last":0.50600000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:40:39 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.53500000000000003,"15min":0.52300000000000002},"min":{"1min":0.34499999999999997,"5min":0.313,"15min":0.313},"max":{"1min":2.5550000000000002,"5min":2.5550000000000002,"15min":2.5550000000000002},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.58899999999999997,"15min":0.59799999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":1.091,"5min":1.091,"15min":1.091},"last":0.59699999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:40:39 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.54700000000000004,"15min":0.55100000000000005},"min":{"1min":0.35499999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.113,"5min":1.113,"15min":1.113},"last":0.629},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.56100000000000005,"15min":0.55200000000000005},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":3.202,"5min":3.202,"15min":3.202},"last":0.49099999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:40:39 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.56100000000000005,"15min":0.54700000000000004},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":3.2309999999999999,"5min":3.2309999999999999,"15min":3.2309999999999999},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.623,"5min":0.58199999999999996,"15min":0.57499999999999996},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.657,"5min":2.657,"15min":2.657},"last":0.65600000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:40:39 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.50700000000000001,"15min":0.51100000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":1.135,"5min":1.135,"15min":1.135},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.502,"15min":0.48599999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.165,"5min":3.165,"15min":3.165},"last":0.52100000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:40:39 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.51400000000000001,"15min":0.5},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":2.5990000000000002,"5min":2.5990000000000002,"15min":2.5990000000000002},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.54000000000000004,"15min":0.53500000000000003},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.0660000000000001,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.53600000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:40:39 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.55200000000000005,"15min":0.53600000000000003},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.8170000000000002,"5min":2.8170000000000002,"15min":2.8170000000000002},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.54800000000000004,"15min":0.52800000000000002},"min":{"1min":0.376,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":3.2589999999999999,"5min":3.2589999999999999,"15min":3.2589999999999999},"last":0.57899999999999996}]}]},{"osd":6,"up_from":46,"seq":197568495652,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1264,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1294336,"data_stored":890943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:12 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.505,"15min":0.501},"min":{"1min":0.38200000000000001,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.54300000000000004,"15min":0.54700000000000004},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.57999999999999996}]},{"osd":1,"last update":"Mon Oct 9 05:40:12 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.54100000000000004,"15min":0.53900000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.54000000000000004}]},{"osd":2,"last update":"Mon Oct 9 05:40:12 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.67700000000000005,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.52400000000000002,"15min":0.52100000000000002},"min":{"1min":0.41799999999999998,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.41599999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:40:12 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.53400000000000003,"15min":0.53100000000000003},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.496},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.435}]},{"osd":4,"last update":"Mon Oct 9 05:40:12 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.438,"15min":0.44},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.55700000000000005,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.47399999999999998,"15min":0.47199999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.67300000000000004,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.375}]},{"osd":5,"last update":"Mon Oct 9 05:40:12 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.44900000000000001,"15min":0.44700000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.65700000000000003,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.501,"15min":0.497},"min":{"1min":0.32900000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.70699999999999996,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.45500000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:40:39 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.53500000000000003,"15min":0.53700000000000003},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58399999999999996,"15min":0.58299999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.61399999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574907,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297960,"kb_used_data":672,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447192,"statfs":{"total":95995035648,"available":95689924608,"internally_reserved":0,"allocated":688128,"data_stored":299292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:25 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35399999999999998,"15min":0.34999999999999998},"min":{"1min":0.224,"5min":0.219,"15min":0.219},"max":{"1min":0.59399999999999997,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.222},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.36799999999999999,"15min":0.34999999999999998},"min":{"1min":0.27500000000000002,"5min":0.22,"15min":0.22},"max":{"1min":0.55800000000000005,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.27400000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:40:47 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.40999999999999998,"15min":0.41299999999999998},"min":{"1min":0.28000000000000003,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.70099999999999996,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.39100000000000001,"15min":0.39900000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.34699999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:40:13 2023","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.41799999999999998,"15min":0.432},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.312},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.46500000000000002,"15min":0.47399999999999998},"min":{"1min":0.34599999999999997,"5min":0.31,"15min":0.31},"max":{"1min":0.59599999999999997,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.33800000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:40:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57499999999999996,"15min":0.57799999999999996},"min":{"1min":0.46600000000000003,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.78000000000000003,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.57099999999999995,"15min":0.57199999999999995},"min":{"1min":0.433,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.80300000000000005,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.91500000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:40:53 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.59399999999999997,"15min":0.60299999999999998},"min":{"1min":0.44800000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.78000000000000003,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.754},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60399999999999998,"15min":0.59999999999999998},"min":{"1min":0.41199999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.84399999999999997,"5min":1.7290000000000001,"15min":1.7290000000000001},"last":0.78700000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:40:13 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.61199999999999999,"15min":0.61799999999999999},"min":{"1min":0.47199999999999998,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.68500000000000005,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.623}]},{"osd":7,"last update":"Mon Oct 9 05:40:34 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.58699999999999997,"15min":0.58299999999999996},"min":{"1min":0.46400000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.83899999999999997},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.61899999999999999,"15min":0.621},"min":{"1min":0.48899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.76700000000000002,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.73399999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738432,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298548,"kb_used_data":1260,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446604,"statfs":{"total":95995035648,"available":95689322496,"internally_reserved":0,"allocated":1290240,"data_stored":890236,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:40:27 2023","interfaces":[{"interface":"back","average":{"1min":0.32800000000000001,"5min":0.35999999999999999,"15min":0.34399999999999997},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.52400000000000002,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.37,"5min":0.378,"15min":0.35499999999999998},"min":{"1min":0.25900000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.32600000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:40:46 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.42299999999999999,"15min":0.42699999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.68999999999999995,"5min":1.55,"15min":1.55},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.42499999999999999,"15min":0.433},"min":{"1min":0.26600000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.40500000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:40:12 2023","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.47699999999999998,"15min":0.5},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.49199999999999999,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.5,"15min":0.52400000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.74099999999999999,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.373}]},{"osd":4,"last update":"Mon Oct 9 05:40:27 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.60399999999999998,"15min":0.61799999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.70299999999999996,"5min":0.875,"15min":0.875},"last":0.68300000000000005},{"interface":"front","average":{"1min":1.1619999999999999,"5min":0.71499999999999997,"15min":0.64300000000000002},"min":{"1min":0.40999999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":6.5250000000000004,"5min":6.5250000000000004,"15min":6.5250000000000004},"last":0.55600000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:40:50 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.63,"15min":0.64800000000000002},"min":{"1min":0.45300000000000001,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.71199999999999997,"15min":0.69899999999999995},"min":{"1min":0.437,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.83499999999999996,"5min":5.6539999999999999,"15min":5.6539999999999999},"last":0.80900000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:40:12 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.65500000000000003,"15min":0.66800000000000004},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.73899999999999999,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.60899999999999999,"15min":0.621},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.73499999999999999,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.69399999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:40:35 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":0.68500000000000005,"15min":0.65500000000000003},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":5.2779999999999996,"5min":5.2779999999999996,"15min":5.2779999999999996},"last":0.76600000000000001},{"interface":"front","average":{"1min":0.86199999999999999,"5min":0.90000000000000002,"15min":0.90600000000000003},"min":{"1min":0.48399999999999999,"5min":0.379,"15min":0.379},"max":{"1min":5.3029999999999999,"5min":6.4660000000000002,"15min":6.4660000000000002},"last":0.73299999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411381,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":684,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":700416,"data_stored":300145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:42 2023","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.375,"15min":0.36599999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.38,"5min":0.40699999999999997,"15min":0.38300000000000001},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":1.2110000000000001,"5min":2.746,"15min":2.746},"last":0.33700000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:40:42 2023","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.38800000000000001,"15min":0.376},"min":{"1min":0.23799999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.38},{"interface":"front","average":{"1min":0.373,"5min":0.42599999999999999,"15min":0.41599999999999998},"min":{"1min":0.25800000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.59399999999999997,"5min":1.929,"15min":1.929},"last":0.31}]},{"osd":3,"last update":"Mon Oct 9 05:40:11 2023","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.46100000000000002,"15min":0.46899999999999997},"min":{"1min":0.307,"5min":0.246,"15min":0.246},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.496,"15min":0.47599999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.895,"5min":2.895,"15min":2.895},"last":0.36899999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:40:34 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.60399999999999998,"15min":0.59999999999999998},"min":{"1min":0.40200000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.77800000000000002,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.60499999999999998,"15min":0.60399999999999998},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.59799999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:40:53 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60199999999999998,"15min":0.60799999999999998},"min":{"1min":0.44700000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.628,"15min":0.61599999999999999},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.0009999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.501}]},{"osd":6,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.65800000000000003,"15min":0.66000000000000003},"min":{"1min":0.46200000000000002,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.83699999999999997,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.751,"5min":0.65100000000000002,"15min":0.63400000000000001},"min":{"1min":0.434,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":2.8650000000000002,"5min":2.8650000000000002,"15min":2.8650000000000002},"last":0.60999999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:40:34 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.623,"15min":0.621},"min":{"1min":0.46999999999999997,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.625},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.73899999999999999,"15min":0.755},"min":{"1min":0.47799999999999998,"5min":0.439,"15min":0.439},"max":{"1min":0.91800000000000004,"5min":2.8210000000000002,"15min":2.8210000000000002},"last":0.69999999999999996}]}]},{"osd":3,"up_from":25,"seq":107374182449,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297968,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447184,"statfs":{"total":95995035648,"available":95689916416,"internally_reserved":0,"allocated":696320,"data_stored":299647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:10 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.40999999999999998,"15min":0.41799999999999998},"min":{"1min":0.26000000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.58299999999999996,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.316},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.27200000000000002,"5min":0.251,"15min":0.251},"max":{"1min":0.66800000000000004,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.35099999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:40:10 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.44800000000000001,"15min":0.46600000000000003},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.61599999999999999,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.46000000000000002,"15min":0.47299999999999998},"min":{"1min":0.33400000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.70199999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.39600000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:40:10 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.49199999999999999,"15min":0.51100000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.64100000000000001,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.442},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.45900000000000002,"15min":0.46999999999999997},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.68500000000000005,"5min":0.746,"15min":0.746},"last":0.28199999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:40:33 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.58899999999999997,"15min":0.58299999999999996},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.625,"15min":0.621},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.84499999999999997,"5min":1.008,"15min":1.008},"last":0.63600000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:39:56 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.65600000000000003,"15min":0.66800000000000004},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.70799999999999996,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.60499999999999998,"15min":0.61199999999999999},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.71799999999999997,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.74199999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:40:16 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.626,"15min":0.628},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.63800000000000001,"15min":0.63600000000000001},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.88500000000000001,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.64800000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:40:33 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.64300000000000002,"15min":0.65300000000000002},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.66100000000000003,"15min":0.66400000000000003},"min":{"1min":0.50900000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.59999999999999998}]}]},{"osd":4,"up_from":30,"seq":128849018925,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297988,"kb_used_data":700,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447164,"statfs":{"total":95995035648,"available":95689895936,"internally_reserved":0,"allocated":716800,"data_stored":301239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:35 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.54100000000000004,"15min":0.54800000000000004},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.69099999999999995,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.57499999999999996,"15min":0.60299999999999998},"min":{"1min":0.35999999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.75700000000000001,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.58299999999999996}]},{"osd":1,"last update":"Mon Oct 9 05:40:35 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.55500000000000005,"15min":0.57699999999999996},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.74299999999999999,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.56699999999999995,"15min":0.58299999999999996},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.70099999999999996,"5min":1.006,"15min":1.006},"last":0.59999999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:40:35 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.56599999999999995,"15min":0.58399999999999996},"min":{"1min":0.41999999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.73899999999999999,"5min":1.026,"15min":1.026},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.56200000000000006,"15min":0.56999999999999995},"min":{"1min":0.40600000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.79000000000000004,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.76100000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:40:35 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57999999999999996,"15min":0.59199999999999997},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.751,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.82299999999999995},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.56699999999999995,"15min":0.57899999999999996},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.71599999999999997,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.71099999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:40:54 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.44500000000000001,"15min":0.43099999999999999},"min":{"1min":0.35499999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.60399999999999998,"5min":0.877,"15min":0.877},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.47199999999999998,"15min":0.44600000000000001},"min":{"1min":0.37,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.76700000000000002,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.85899999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:40:12 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.47599999999999998,"15min":0.46700000000000003},"min":{"1min":0.34999999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.751},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.50900000000000001,"15min":0.501},"min":{"1min":0.35599999999999998,"5min":0.315,"15min":0.315},"max":{"1min":0.77100000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.78800000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:40:35 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.55100000000000005,"15min":0.54300000000000004},"min":{"1min":0.41599999999999998,"5min":0.37,"15min":0.37},"max":{"1min":0.79900000000000004,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.88400000000000001},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.57799999999999996,"15min":0.57599999999999996},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.78300000000000003,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.80900000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757288,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":684,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":700416,"data_stored":300181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:40:54 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.505,"15min":0.51400000000000001},"min":{"1min":0.373,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.61799999999999999,"5min":0.752,"15min":0.752},"last":0.497},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.54900000000000004,"15min":0.52700000000000002},"min":{"1min":0.41899999999999998,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.64500000000000002,"5min":3.2149999999999999,"15min":3.2149999999999999},"last":0.47099999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:40:54 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.51400000000000001,"15min":0.51700000000000002},"min":{"1min":0.36199999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.56799999999999995,"15min":0.54100000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.3400000000000001,"5min":3.2589999999999999,"15min":3.2589999999999999},"last":0.54200000000000004}]},{"osd":2,"last update":"Mon Oct 9 05:40:54 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.52800000000000002,"15min":0.53400000000000003},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.627,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53200000000000003,"15min":0.53600000000000003},"min":{"1min":0.41299999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.67400000000000004,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.56000000000000005}]},{"osd":3,"last update":"Mon Oct 9 05:40:54 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52400000000000002,"15min":0.52200000000000002},"min":{"1min":0.38100000000000001,"5min":0.37,"15min":0.37},"max":{"1min":0.68999999999999995,"5min":0.754,"15min":0.754},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.55900000000000005,"15min":0.55800000000000005},"min":{"1min":0.435,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.74299999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.61499999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:40:54 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.44700000000000001,"15min":0.40100000000000002},"min":{"1min":0.33300000000000002,"5min":0.312,"15min":0.312},"max":{"1min":0.68899999999999995,"5min":3.5430000000000001,"15min":3.5430000000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.439,"15min":0.40500000000000003},"min":{"1min":0.30099999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.66700000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:40:13 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.45100000000000001,"15min":0.44500000000000001},"min":{"1min":0.33200000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.61099999999999999,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.46899999999999997,"15min":0.45900000000000002},"min":{"1min":0.33100000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.65500000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:40:37 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.53500000000000003,"15min":0.52900000000000003},"min":{"1min":0.35999999999999999,"5min":0.318,"15min":0.318},"max":{"1min":0.73099999999999998,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.64400000000000002,"15min":0.66100000000000003},"min":{"1min":0.38500000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.68500000000000005,"5min":3.2869999999999999,"15min":3.2869999999999999},"last":0.69399999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:41:00.535 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T05:41:00.535 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T05:41:00.535 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T05:41:00.535 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T05:41:00.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:00 smithi156 ceph-mon[105979]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:00.901 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:00.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:00 smithi064 ceph-mon[101070]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:00.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:00 smithi064 ceph-mon[118354]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:01.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:01 smithi156 ceph-mon[105979]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:41:01.949 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:01 smithi064 ceph-mon[101070]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:41:01.950 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:01 smithi064 ceph-mon[118354]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:41:02.616 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:41:02.617 INFO:teuthology.orchestra.run.smithi064.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T05:41:02.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:02 smithi064 ceph-mon[101070]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:02.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:02 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/681165745' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:41:02.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:02 smithi064 ceph-mon[118354]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:02.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:02 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/681165745' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:41:03.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:41:02 smithi156 conmon[109199]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:41:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:41:03.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:02 smithi156 ceph-mon[105979]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:03.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:02 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/681165745' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:41:03.303 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T05:41:03.303 INFO:teuthology.run_tasks:Running task print... 2023-10-09T05:41:03.312 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-10-09T05:41:03.313 INFO:teuthology.run_tasks:Running task parallel... 2023-10-09T05:41:03.343 INFO:teuthology.task.parallel:starting parallel... 2023-10-09T05:41:03.344 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T05:41:03.344 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T05:41:03.348 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-10-09T05:41:03.348 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T05:41:03.348 DEBUG:teuthology.orchestra.run.smithi064:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T05:41:03.348 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T05:41:03.349 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-10-09T05:41:03.349 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi064.front.sepia.ceph.com 2023-10-09T05:41:03.349 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-10-09T05:41:03.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:41:03.368 INFO:teuthology.orchestra.run.smithi064.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T05:41:03.368 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T05:41:03.425 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T05:41:03.425 DEBUG:teuthology.orchestra.run.smithi064:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T05:41:03.482 INFO:tasks.workunit:timeout=3h 2023-10-09T05:41:03.482 INFO:tasks.workunit:cleanup=True 2023-10-09T05:41:03.482 DEBUG:teuthology.orchestra.run.smithi064:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T05:41:03.539 INFO:tasks.workunit.client.0.smithi064.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T05:41:03.714 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:04.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:04 smithi064 ceph-mon[101070]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:04.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:04 smithi064 ceph-mon[118354]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:05.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:04 smithi156 ceph-mon[105979]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:05.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:41:05 smithi064 conmon[101498]: [09/Oct/2023:05:41:05] ENGINE Bus STOPPING 2023-10-09T05:41:05.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:41:05 smithi064 conmon[101498]: [09/Oct/2023:05:41:05] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:41:05.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:41:05 smithi064 conmon[101498]: [09/Oct/2023:05:41:05] ENGINE Bus STOPPED 2023-10-09T05:41:05.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:41:05 smithi064 conmon[101498]: [09/Oct/2023:05:41:05] ENGINE Bus STARTING 2023-10-09T05:41:05.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:41:05 smithi064 conmon[101498]: [09/Oct/2023:05:41:05] ENGINE Serving on http://:::9283 2023-10-09T05:41:05.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:41:05 smithi064 conmon[101498]: [09/Oct/2023:05:41:05] ENGINE Bus STARTED 2023-10-09T05:41:06.178 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph config set global log_to_journald false --force' 2023-10-09T05:41:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:06 smithi064 ceph-mon[101070]: from='client.? ' entity='client.admin' 2023-10-09T05:41:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:41:06.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:41:06.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:41:06.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:41:06.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:06 smithi064 ceph-mon[101070]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:06.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T05:41:06.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2023-10-09T05:41:06.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T05:41:06.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:06.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:06 smithi064 ceph-mon[118354]: from='client.? ' entity='client.admin' 2023-10-09T05:41:06.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:41:06.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:41:06.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:41:06.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:41:06.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:06 smithi064 ceph-mon[118354]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:06.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T05:41:06.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2023-10-09T05:41:06.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T05:41:06.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:06.616 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:06.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:06 smithi156 ceph-mon[105979]: from='client.? ' entity='client.admin' 2023-10-09T05:41:06.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:41:06.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:41:06.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:41:06.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:41:06.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:06 smithi156 ceph-mon[105979]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:06.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T05:41:06.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2023-10-09T05:41:06.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T05:41:06.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:07.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:41:07 smithi064 conmon[101498]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:41:07] "GET /metrics HTTP/1.1" 200 238708 "" "Prometheus/2.43.0" 2023-10-09T05:41:07.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:07 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T05:41:07.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:07 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2023-10-09T05:41:07.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:07 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-10-09T05:41:07.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:07 smithi064 ceph-mon[101070]: osdmap e69: 8 total, 8 up, 8 in 2023-10-09T05:41:07.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:07 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T05:41:07.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:07 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2023-10-09T05:41:07.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:07 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-10-09T05:41:07.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:07 smithi064 ceph-mon[118354]: osdmap e69: 8 total, 8 up, 8 in 2023-10-09T05:41:07.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:07 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T05:41:07.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:07 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2023-10-09T05:41:07.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:07 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-10-09T05:41:07.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:07 smithi156 ceph-mon[105979]: osdmap e69: 8 total, 8 up, 8 in 2023-10-09T05:41:08.285 DEBUG:teuthology.orchestra.run.smithi064:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T05:41:08.411 INFO:tasks.workunit.client.0.smithi064.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T05:41:08.414 INFO:tasks.workunit.client.0.smithi064.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T05:41:08.414 INFO:tasks.workunit.client.0.smithi064.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T05:41:08.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:08 smithi064 ceph-mon[101070]: osdmap e70: 8 total, 8 up, 8 in 2023-10-09T05:41:08.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:08 smithi064 ceph-mon[101070]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:08.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:08 smithi064 ceph-mon[118354]: osdmap e70: 8 total, 8 up, 8 in 2023-10-09T05:41:08.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:08 smithi064 ceph-mon[118354]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:08.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:08 smithi156 ceph-mon[105979]: osdmap e70: 8 total, 8 up, 8 in 2023-10-09T05:41:08.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:08 smithi156 ceph-mon[105979]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:09.265 INFO:tasks.workunit.client.0.smithi064.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T05:41:09.315 INFO:tasks.workunit.client.0.smithi064.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T05:41:09.353 INFO:tasks.workunit.client.0.smithi064.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T05:41:09.355 INFO:tasks.workunit.client.0.smithi064.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T05:41:09.355 INFO:tasks.workunit.client.0.smithi064.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T05:41:09.392 INFO:tasks.workunit.client.0.smithi064.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T05:41:09.395 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-10-09T05:41:09.395 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T05:41:09.413 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-10-09T05:41:09.414 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-10-09T05:41:09.415 DEBUG:teuthology.orchestra.run.smithi064:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-10-09T05:41:09.485 INFO:tasks.workunit.client.0.smithi064.stdout:Running main() from gmock_main.cc 2023-10-09T05:41:09.485 INFO:tasks.workunit.client.0.smithi064.stdout:[==========] Running 70 tests from 1 test suite. 2023-10-09T05:41:09.485 INFO:tasks.workunit.client.0.smithi064.stdout:[----------] Global test environment set-up. 2023-10-09T05:41:09.486 INFO:tasks.workunit.client.0.smithi064.stdout:[----------] 70 tests from TestClsRbd 2023-10-09T05:41:10.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:10 smithi156 ceph-mon[105979]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:10.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:10 smithi064 ceph-mon[101070]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:10.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:10 smithi064 ceph-mon[118354]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:11.007 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_all_features 2023-10-09T05:41:11.329 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_all_features (321 ms) 2023-10-09T05:41:11.329 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.copyup 2023-10-09T05:41:11.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:11 smithi156 ceph-mon[105979]: osdmap e71: 8 total, 8 up, 8 in 2023-10-09T05:41:11.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:11 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/583033683' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-189816-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:41:11.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:11 smithi156 ceph-mon[105979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-189816-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:41:11.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:11 smithi156 ceph-mon[105979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-189816-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T05:41:11.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:11 smithi156 ceph-mon[105979]: osdmap e72: 8 total, 8 up, 8 in 2023-10-09T05:41:11.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:11 smithi064 ceph-mon[101070]: osdmap e71: 8 total, 8 up, 8 in 2023-10-09T05:41:11.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:11 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/583033683' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-189816-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:41:11.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:11 smithi064 ceph-mon[101070]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-189816-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:41:11.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:11 smithi064 ceph-mon[101070]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-189816-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T05:41:11.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:11 smithi064 ceph-mon[101070]: osdmap e72: 8 total, 8 up, 8 in 2023-10-09T05:41:11.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:11 smithi064 ceph-mon[118354]: osdmap e71: 8 total, 8 up, 8 in 2023-10-09T05:41:11.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:11 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/583033683' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-189816-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:41:11.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:11 smithi064 ceph-mon[118354]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-189816-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:41:11.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:11 smithi064 ceph-mon[118354]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-189816-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T05:41:11.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:11 smithi064 ceph-mon[118354]: osdmap e72: 8 total, 8 up, 8 in 2023-10-09T05:41:11.523 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.copyup (194 ms) 2023-10-09T05:41:11.523 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-10-09T05:41:11.537 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.sparse_copyup (14 ms) 2023-10-09T05:41:11.537 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-10-09T05:41:11.545 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_and_set_id (8 ms) 2023-10-09T05:41:11.545 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-10-09T05:41:11.555 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.add_remove_child (11 ms) 2023-10-09T05:41:11.556 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.directory_methods 2023-10-09T05:41:11.587 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.directory_methods (32 ms) 2023-10-09T05:41:11.588 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.create 2023-10-09T05:41:11.601 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.create (13 ms) 2023-10-09T05:41:11.601 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_features 2023-10-09T05:41:11.602 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2023-10-09T05:41:11.602 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-10-09T05:41:11.604 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2023-10-09T05:41:11.604 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-10-09T05:41:11.606 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_create_timestamp (1 ms) 2023-10-09T05:41:11.606 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-10-09T05:41:11.607 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_access_timestamp (2 ms) 2023-10-09T05:41:11.607 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-10-09T05:41:11.608 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2023-10-09T05:41:11.608 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-10-09T05:41:11.612 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_data_pool (4 ms) 2023-10-09T05:41:11.612 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_size 2023-10-09T05:41:11.615 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_size (3 ms) 2023-10-09T05:41:11.616 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.set_size 2023-10-09T05:41:11.620 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.set_size (5 ms) 2023-10-09T05:41:11.620 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.protection_status 2023-10-09T05:41:11.637 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.protection_status (17 ms) 2023-10-09T05:41:11.637 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-10-09T05:41:11.644 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.snapshot_limits (7 ms) 2023-10-09T05:41:11.644 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.parents_v1 2023-10-09T05:41:11.684 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.parents_v1 (40 ms) 2023-10-09T05:41:11.684 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.parents_v2 2023-10-09T05:41:11.713 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.parents_v2 (28 ms) 2023-10-09T05:41:11.713 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.snapshots 2023-10-09T05:41:11.734 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.snapshots (22 ms) 2023-10-09T05:41:11.734 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.snapid_race 2023-10-09T05:41:11.738 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.snapid_race (4 ms) 2023-10-09T05:41:11.738 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.stripingv2 2023-10-09T05:41:11.747 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.stripingv2 (8 ms) 2023-10-09T05:41:11.747 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.object_map_save 2023-10-09T05:41:11.748 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.object_map_save (2 ms) 2023-10-09T05:41:11.748 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-10-09T05:41:11.753 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.object_map_resize (5 ms) 2023-10-09T05:41:11.753 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.object_map_update 2023-10-09T05:41:11.758 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.object_map_update (5 ms) 2023-10-09T05:41:11.758 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-10-09T05:41:11.758 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-10-09T05:41:11.758 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-10-09T05:41:11.762 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.object_map_snap_add (4 ms) 2023-10-09T05:41:11.762 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-10-09T05:41:11.770 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.object_map_snap_remove (7 ms) 2023-10-09T05:41:11.770 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.flags 2023-10-09T05:41:11.778 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.flags (6 ms) 2023-10-09T05:41:11.778 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.metadata 2023-10-09T05:41:11.799 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.metadata (24 ms) 2023-10-09T05:41:11.800 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.set_features 2023-10-09T05:41:11.807 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.set_features (8 ms) 2023-10-09T05:41:11.807 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.mirror 2023-10-09T05:41:11.820 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-10-09T05:41:11.839 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.mirror (31 ms) 2023-10-09T05:41:11.839 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.mirror_image 2023-10-09T05:41:11.854 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.mirror_image (16 ms) 2023-10-09T05:41:11.854 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-10-09T05:41:12.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:12 smithi064 ceph-mon[101070]: pgmap v79: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 803 KiB/s wr, 5 op/s 2023-10-09T05:41:12.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:12 smithi064 ceph-mon[118354]: pgmap v79: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 803 KiB/s wr, 5 op/s 2023-10-09T05:41:12.250 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:12.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:12 smithi156 ceph-mon[105979]: pgmap v79: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 803 KiB/s wr, 5 op/s 2023-10-09T05:41:13.015 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:41:12 smithi156 conmon[109199]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:41:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:41:13.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:13 smithi064 ceph-mon[101070]: osdmap e73: 8 total, 8 up, 8 in 2023-10-09T05:41:13.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:13 smithi064 ceph-mon[118354]: osdmap e73: 8 total, 8 up, 8 in 2023-10-09T05:41:13.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:13 smithi156 ceph-mon[105979]: osdmap e73: 8 total, 8 up, 8 in 2023-10-09T05:41:14.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:14 smithi064 ceph-mon[101070]: pgmap v81: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 684 KiB/s wr, 4 op/s 2023-10-09T05:41:14.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:14 smithi064 ceph-mon[118354]: pgmap v81: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 684 KiB/s wr, 4 op/s 2023-10-09T05:41:14.291 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.mirror_image_status (2436 ms) 2023-10-09T05:41:14.291 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-10-09T05:41:14.334 INFO:teuthology.orchestra.run.smithi064.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:41:14.367 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.mirror_image_map (77 ms) 2023-10-09T05:41:14.367 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-10-09T05:41:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:14 smithi156 ceph-mon[105979]: pgmap v81: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 684 KiB/s wr, 4 op/s 2023-10-09T05:41:14.376 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.mirror_instances (9 ms) 2023-10-09T05:41:14.376 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-10-09T05:41:14.389 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.mirror_snapshot (13 ms) 2023-10-09T05:41:14.390 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-10-09T05:41:14.392 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_dir_list (3 ms) 2023-10-09T05:41:14.392 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-10-09T05:41:14.394 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2023-10-09T05:41:14.394 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-10-09T05:41:14.397 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.dir_add_already_existing (2 ms) 2023-10-09T05:41:14.397 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-10-09T05:41:14.403 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_dir_rename (6 ms) 2023-10-09T05:41:14.403 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-10-09T05:41:14.406 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_dir_remove (4 ms) 2023-10-09T05:41:14.406 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-10-09T05:41:14.410 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2023-10-09T05:41:14.410 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_image_add 2023-10-09T05:41:14.413 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_image_add (3 ms) 2023-10-09T05:41:14.413 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-10-09T05:41:14.416 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2023-10-09T05:41:14.416 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_image_list 2023-10-09T05:41:14.420 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_image_list (4 ms) 2023-10-09T05:41:14.420 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-10-09T05:41:14.425 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_image_clean (5 ms) 2023-10-09T05:41:14.426 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.image_group_add 2023-10-09T05:41:14.428 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.image_group_add (3 ms) 2023-10-09T05:41:14.429 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-10-09T05:41:14.432 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.image_group_remove (4 ms) 2023-10-09T05:41:14.432 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.image_group_get 2023-10-09T05:41:14.435 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.image_group_get (3 ms) 2023-10-09T05:41:14.435 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-10-09T05:41:14.437 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2023-10-09T05:41:14.437 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-10-09T05:41:14.439 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2023-10-09T05:41:14.439 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-10-09T05:41:14.442 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (2 ms) 2023-10-09T05:41:14.442 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-10-09T05:41:14.445 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2023-10-09T05:41:14.445 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-10-09T05:41:14.447 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_set (3 ms) 2023-10-09T05:41:14.447 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-10-09T05:41:14.450 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2023-10-09T05:41:14.450 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-10-09T05:41:14.468 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (18 ms) 2023-10-09T05:41:14.468 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-10-09T05:41:14.684 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (215 ms) 2023-10-09T05:41:14.684 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-10-09T05:41:14.687 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_remove (4 ms) 2023-10-09T05:41:14.687 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-10-09T05:41:14.689 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2023-10-09T05:41:14.689 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.trash_methods 2023-10-09T05:41:14.696 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.trash_methods (6 ms) 2023-10-09T05:41:14.696 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.op_features 2023-10-09T05:41:14.703 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.op_features (8 ms) 2023-10-09T05:41:14.703 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.clone_parent 2023-10-09T05:41:14.723 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.clone_parent (20 ms) 2023-10-09T05:41:14.724 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-10-09T05:41:14.733 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.clone_parent_ns (9 ms) 2023-10-09T05:41:14.733 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.clone_child 2023-10-09T05:41:14.744 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.clone_child (12 ms) 2023-10-09T05:41:14.745 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-10-09T05:41:14.758 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.namespace_methods (14 ms) 2023-10-09T05:41:14.758 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.migration 2023-10-09T05:41:14.768 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.migration (10 ms) 2023-10-09T05:41:14.768 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.migration_v1 2023-10-09T05:41:14.779 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.migration_v1 (11 ms) 2023-10-09T05:41:14.779 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-10-09T05:41:15.304 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T05:41:15.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:15 smithi156 ceph-mon[105979]: from='client.14820 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:41:15.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:15 smithi156 ceph-mon[105979]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:41:15.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:15 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:41:15.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:15 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:41:15.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:15 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:41:15.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:15 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:41:15.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:15 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:41:15.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:15 smithi064 ceph-mon[118354]: from='client.14820 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:41:15.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:15 smithi064 ceph-mon[118354]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:41:15.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:15 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:41:15.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:15 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:41:15.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:15 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:41:15.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:15 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:41:15.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:15 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:41:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:15 smithi064 ceph-mon[101070]: from='client.14820 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:41:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:15 smithi064 ceph-mon[101070]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:41:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:15 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:41:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:15 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:41:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:15 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:41:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:15 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:41:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:15 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:41:15.729 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:16.350 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1570 ms) 2023-10-09T05:41:16.350 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.sparsify 2023-10-09T05:41:16.395 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.sparsify (45 ms) 2023-10-09T05:41:16.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:16 smithi156 ceph-mon[105979]: osdmap e74: 8 total, 8 up, 8 in 2023-10-09T05:41:16.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:16 smithi156 ceph-mon[105979]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:41:16.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:16 smithi156 ceph-mon[105979]: pgmap v83: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 738 KiB/s wr, 5 op/s 2023-10-09T05:41:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:16 smithi064 ceph-mon[101070]: osdmap e74: 8 total, 8 up, 8 in 2023-10-09T05:41:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:16 smithi064 ceph-mon[101070]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:41:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:16 smithi064 ceph-mon[101070]: pgmap v83: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 738 KiB/s wr, 5 op/s 2023-10-09T05:41:16.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:16 smithi064 ceph-mon[118354]: osdmap e74: 8 total, 8 up, 8 in 2023-10-09T05:41:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:16 smithi064 ceph-mon[118354]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:41:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:16 smithi064 ceph-mon[118354]: pgmap v83: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 738 KiB/s wr, 5 op/s 2023-10-09T05:41:17.163 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:41:17 smithi064 conmon[101498]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:41:17] "GET /metrics HTTP/1.1" 200 238708 "" "Prometheus/2.43.0" 2023-10-09T05:41:17.381 INFO:tasks.workunit.client.0.smithi064.stdout:[----------] 70 tests from TestClsRbd (5387 ms total) 2023-10-09T05:41:17.381 INFO:tasks.workunit.client.0.smithi064.stdout: 2023-10-09T05:41:17.381 INFO:tasks.workunit.client.0.smithi064.stdout:[----------] Global test environment tear-down 2023-10-09T05:41:17.381 INFO:tasks.workunit.client.0.smithi064.stdout:[==========] 70 tests from 1 test suite ran. (7895 ms total) 2023-10-09T05:41:17.381 INFO:tasks.workunit.client.0.smithi064.stdout:[ PASSED ] 70 tests. 2023-10-09T05:41:17.382 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T05:41:17.383 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T05:41:17.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:17 smithi064 ceph-mon[101070]: osdmap e75: 8 total, 8 up, 8 in 2023-10-09T05:41:17.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:17 smithi064 ceph-mon[118354]: osdmap e75: 8 total, 8 up, 8 in 2023-10-09T05:41:17.419 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-10-09T05:41:17.419 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T05:41:17.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:17 smithi156 ceph-mon[105979]: osdmap e75: 8 total, 8 up, 8 in 2023-10-09T05:41:17.679 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T05:41:17.679 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (72s) 49s ago 118s 16.1M - 0.25.0 c8568f914cd2 51110671bb41 2023-10-09T05:41:17.679 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi156 *:3000 running (89s) 50s ago 88s 88.4M - 9.4.7 2c41d148cca3 dbad77de7b52 2023-10-09T05:41:17.679 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.gmrwsy smithi064 running (61s) 49s ago 60s 14.9M - 17.2.6-1303-gb2903cb0 5517396c50d4 e938ff2e972d 2023-10-09T05:41:17.679 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi156.pizbdw smithi156 running (58s) 50s ago 57s 15.6M - 17.2.6-1303-gb2903cb0 5517396c50d4 e8906e08c938 2023-10-09T05:41:17.680 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi156 *:8443 running (5m) 50s ago 5m 439M - 17.2.6-1303-gb2903cb0 5517396c50d4 2efef4f1cb6c 2023-10-09T05:41:17.680 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:9283 running (7m) 49s ago 7m 493M - 17.2.6-1303-gb2903cb0 5517396c50d4 1f3d7bc3f242 2023-10-09T05:41:17.680 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (7m) 49s ago 7m 79.5M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 c9fda8001310 2023-10-09T05:41:17.680 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi156 running (6m) 50s ago 6m 57.8M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 c60a890df99e 2023-10-09T05:41:17.680 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (6m) 49s ago 6m 59.2M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 b6790a9456ba 2023-10-09T05:41:17.680 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (113s) 49s ago 113s 16.7M - 1.5.0 0da6a335fe13 59bb208a1e32 2023-10-09T05:41:17.680 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi156 *:9100 running (110s) 50s ago 109s 16.8M - 1.5.0 0da6a335fe13 fead60fdc013 2023-10-09T05:41:17.680 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (5m) 49s ago 5m 57.0M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 bd3c343a2f7c 2023-10-09T05:41:17.680 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (5m) 49s ago 5m 57.8M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 854146c77f6d 2023-10-09T05:41:17.680 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (4m) 49s ago 4m 50.9M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 75a274ca9b6e 2023-10-09T05:41:17.680 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (4m) 49s ago 4m 50.8M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 711a9677648c 2023-10-09T05:41:17.681 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi156 running (3m) 50s ago 3m 51.7M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 72313d594357 2023-10-09T05:41:17.681 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi156 running (3m) 50s ago 3m 48.7M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 40d9335e2ab7 2023-10-09T05:41:17.681 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi156 running (3m) 50s ago 3m 48.2M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 c8a3e3228b89 2023-10-09T05:41:17.681 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi156 running (2m) 50s ago 2m 53.4M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 df93983742a3 2023-10-09T05:41:17.681 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi156 *:9095 running (67s) 50s ago 2m 36.3M - 2.43.0 a07b618ecd1d d46b6df5107c 2023-10-09T05:41:17.743 DEBUG:teuthology.parallel:result is None 2023-10-09T05:41:17.743 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T05:41:17.773 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T05:41:17.773 DEBUG:teuthology.orchestra.run.smithi064:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T05:41:17.856 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T05:41:17.856 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T05:41:17.856 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-10-09T05:41:17.856 DEBUG:teuthology.parallel:result is None 2023-10-09T05:41:18.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:18 smithi064 ceph-mon[101070]: osdmap e76: 8 total, 8 up, 8 in 2023-10-09T05:41:18.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:18 smithi064 ceph-mon[101070]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:18.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:18 smithi064 ceph-mon[118354]: osdmap e76: 8 total, 8 up, 8 in 2023-10-09T05:41:18.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:18 smithi064 ceph-mon[118354]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:18.820 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for minority of mons to upgrade' 2023-10-09T05:41:18.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:18 smithi156 ceph-mon[105979]: osdmap e76: 8 total, 8 up, 8 in 2023-10-09T05:41:18.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:18 smithi156 ceph-mon[105979]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:19.205 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:19.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:19 smithi064 ceph-mon[101070]: from='client.24725 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:41:19.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:19 smithi064 ceph-mon[118354]: from='client.24725 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:41:19.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:19 smithi156 ceph-mon[105979]: from='client.24725 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:41:20.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:20 smithi064 ceph-mon[101070]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:20.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:20 smithi064 ceph-mon[118354]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:20.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:20 smithi156 ceph-mon[105979]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:21.059 INFO:teuthology.orchestra.run.smithi064.stdout:wait for minority of mons to upgrade 2023-10-09T05:41:21.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:21 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:21.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:21 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:21.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:21 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:21.918 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-10-09T05:41:22.320 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:41:22.709 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:22 smithi064 ceph-mon[101070]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:22.800 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:22 smithi064 ceph-mon[118354]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:22.800 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:22 smithi156 ceph-mon[105979]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:23.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:41:22 smithi156 conmon[109199]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:41:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:41:24.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:24 smithi156 ceph-mon[105979]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:24.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:24 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/4292925465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:24.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:24 smithi064 ceph-mon[101070]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:24.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:24 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/4292925465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:24.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:24 smithi064 ceph-mon[118354]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:24.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:24 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/4292925465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:26.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:26 smithi064 ceph-mon[101070]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:26.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:26 smithi064 ceph-mon[118354]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:26.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:26 smithi156 ceph-mon[105979]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:27.403 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:41:27 smithi064 conmon[101498]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:41:27] "GET /metrics HTTP/1.1" 200 239647 "" "Prometheus/2.43.0" 2023-10-09T05:41:27.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:27 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/325908716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:27.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:27 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/325908716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:27 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/325908716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:28.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:28 smithi064 ceph-mon[101070]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:28.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:28 smithi064 ceph-mon[118354]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:28.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:28 smithi156 ceph-mon[105979]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:29 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/969344815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:29.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:29 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/969344815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:29.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:29 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/969344815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:30.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:30 smithi156 ceph-mon[105979]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:30.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:30 smithi064 ceph-mon[101070]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:30.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:30 smithi064 ceph-mon[118354]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:32.870 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:41:32 smithi156 conmon[109199]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:41:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:41:32.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:32 smithi156 ceph-mon[105979]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:32.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:32 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3658456781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:32.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:32 smithi064 ceph-mon[101070]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:32.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:32 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3658456781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:32.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:32 smithi064 ceph-mon[118354]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:32.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:32 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3658456781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:34.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:34 smithi064 ceph-mon[101070]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:34.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:34 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/79471817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:34.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:34 smithi064 ceph-mon[118354]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:34.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:34 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/79471817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:35.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:34 smithi156 ceph-mon[105979]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:35.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:34 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/79471817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:36.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:36 smithi064 ceph-mon[101070]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:36.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:36 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:36.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:36 smithi064 ceph-mon[118354]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:36.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:36 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:36.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:36 smithi156 ceph-mon[105979]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:36.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:36 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:37.405 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:41:37 smithi064 conmon[101498]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:41:37] "GET /metrics HTTP/1.1" 200 239636 "" "Prometheus/2.43.0" 2023-10-09T05:41:37.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:37 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1210188570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:37.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:37 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/1210188570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:37 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/1210188570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:38.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:38 smithi064 ceph-mon[118354]: pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:38.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:38 smithi064 ceph-mon[101070]: pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:38.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:38 smithi156 ceph-mon[105979]: pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:39.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:39 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3800982916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:39.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:39 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3800982916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:39.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:39 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3800982916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:40.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:40 smithi064 ceph-mon[101070]: pgmap v97: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:40.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:40 smithi064 ceph-mon[118354]: pgmap v97: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:40.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:40 smithi156 ceph-mon[105979]: pgmap v97: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:42.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:42 smithi064 ceph-mon[101070]: pgmap v98: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:42.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:42 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/2094217191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:42.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:42 smithi064 ceph-mon[118354]: pgmap v98: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:42.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:42 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/2094217191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:43.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:41:42 smithi156 conmon[109199]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:41:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:41:43.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:42 smithi156 ceph-mon[105979]: pgmap v98: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:43.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:42 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/2094217191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:44.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:44 smithi064 ceph-mon[101070]: pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:44.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:44 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3191815841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:44.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:44 smithi064 ceph-mon[118354]: pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:44.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:44 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3191815841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:45.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:44 smithi156 ceph-mon[105979]: pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:45.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:44 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3191815841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:46.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:46 smithi064 ceph-mon[101070]: pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:46.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:46 smithi064 ceph-mon[118354]: pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:46.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:46 smithi156 ceph-mon[105979]: pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:47.410 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:41:47 smithi064 conmon[101498]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:41:47] "GET /metrics HTTP/1.1" 200 239636 "" "Prometheus/2.43.0" 2023-10-09T05:41:47.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:47 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/2075593671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:47.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:47 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/2075593671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:47.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:47 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/2075593671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:48.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:48 smithi064 ceph-mon[101070]: pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:48.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:48 smithi064 ceph-mon[118354]: pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:48.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:48 smithi156 ceph-mon[105979]: pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:49.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:49 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3327340139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:49.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:49 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3327340139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:49.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:49 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3327340139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:50.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:50 smithi064 ceph-mon[101070]: pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:50.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:50 smithi064 ceph-mon[118354]: pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:50.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:50 smithi156 ceph-mon[105979]: pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:51.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:51 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:51.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:51 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:41:51.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:51 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:41:51.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:51 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:41:51.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:51 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/200017210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:51.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:51 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:51.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:51 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:41:51.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:51 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:41:51.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:51 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:41:51.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:51 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/200017210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:51.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:51 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:41:51.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:51 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:41:51.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:51 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:41:51.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:51 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:41:51.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:51 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/200017210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:52.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:52 smithi064 ceph-mon[118354]: Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T05:41:52.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:52 smithi064 ceph-mon[118354]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T05:41:52.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:52 smithi064 ceph-mon[118354]: Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T05:41:52.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:52 smithi064 ceph-mon[118354]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi156 2023-10-09T05:41:52.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:52 smithi064 ceph-mon[118354]: pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:52.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:52 smithi064 ceph-mon[101070]: Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T05:41:52.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:52 smithi064 ceph-mon[101070]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T05:41:52.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:52 smithi064 ceph-mon[101070]: Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T05:41:52.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:52 smithi064 ceph-mon[101070]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi156 2023-10-09T05:41:52.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:52 smithi064 ceph-mon[101070]: pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:52.764 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:52 smithi156 ceph-mon[105979]: Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T05:41:52.764 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:52 smithi156 ceph-mon[105979]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T05:41:52.764 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:52 smithi156 ceph-mon[105979]: Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T05:41:52.764 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:52 smithi156 ceph-mon[105979]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi156 2023-10-09T05:41:52.764 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:52 smithi156 ceph-mon[105979]: pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:53.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:41:52 smithi156 conmon[109199]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:41:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:41:54.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:54 smithi064 ceph-mon[101070]: pgmap v104: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:54.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:54 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/4264570081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:54.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:54 smithi064 ceph-mon[118354]: pgmap v104: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:54.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:54 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/4264570081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:54 smithi156 ceph-mon[105979]: pgmap v104: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:54 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/4264570081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:56.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:56 smithi064 ceph-mon[118354]: pgmap v105: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:56.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:56 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/2223624205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:56.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:56 smithi064 ceph-mon[101070]: pgmap v105: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:56.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:56 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/2223624205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:56.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:56 smithi156 ceph-mon[105979]: pgmap v105: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:56.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:56 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/2223624205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:57.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:41:57 smithi064 conmon[101498]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:41:57] "GET /metrics HTTP/1.1" 200 239598 "" "Prometheus/2.43.0" 2023-10-09T05:41:58.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:58 smithi064 ceph-mon[118354]: pgmap v106: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:58.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:41:58 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/2725386604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:58.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:58 smithi064 ceph-mon[101070]: pgmap v106: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:58.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:41:58 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/2725386604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:41:59.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:58 smithi156 ceph-mon[105979]: pgmap v106: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:41:59.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:41:58 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/2725386604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:00.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:00 smithi064 ceph-mon[101070]: pgmap v107: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:00.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:00 smithi064 ceph-mon[118354]: pgmap v107: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:01.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:00 smithi156 ceph-mon[105979]: pgmap v107: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:01.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:01 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3603426758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:01.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:01 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3603426758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:02.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:01 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3603426758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:02.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:02 smithi064 ceph-mon[101070]: pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:02.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:02 smithi064 ceph-mon[118354]: pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:03.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:02 smithi156 ceph-mon[105979]: pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:03.121 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:02 smithi156 conmon[109199]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:42:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:42:03.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:03 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1629461319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:03.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:03 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/1629461319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:04.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:03 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/1629461319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:04.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:04 smithi064 ceph-mon[101070]: pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:04.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:04 smithi064 ceph-mon[118354]: pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:05.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:04 smithi156 ceph-mon[105979]: pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:06.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:06 smithi064 ceph-mon[101070]: pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:06.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:06 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:06.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:06 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1330229166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:06.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:06 smithi064 ceph-mon[118354]: pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:06.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:06 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:06.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:06 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/1330229166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:06.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:06 smithi156 ceph-mon[105979]: pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:06.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:06 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:06.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:06 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/1330229166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:07.473 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:07 smithi064 conmon[101498]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:42:07] "GET /metrics HTTP/1.1" 200 239606 "" "Prometheus/2.43.0" 2023-10-09T05:42:08.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:08 smithi064 ceph-mon[101070]: pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:08.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:08 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1711239477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:08.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:08 smithi064 ceph-mon[118354]: pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:08.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:08 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/1711239477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:09.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:08 smithi156 ceph-mon[105979]: pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:09.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:08 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/1711239477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:10.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:10 smithi064 ceph-mon[101070]: pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:10.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:10 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/974147862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:10.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:10 smithi064 ceph-mon[118354]: pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:10.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:10 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/974147862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:11.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:10 smithi156 ceph-mon[105979]: pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:11.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:10 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/974147862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:12.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:12 smithi064 ceph-mon[101070]: pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:12.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:12 smithi064 ceph-mon[118354]: pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:13.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:12 smithi156 ceph-mon[105979]: pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:13.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:12 smithi156 conmon[109199]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:42:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:42:13.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:13 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3977410447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:13.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:13 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3977410447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:14.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:13 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3977410447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:14.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:14 smithi064 ceph-mon[101070]: pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:14.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:14 smithi064 ceph-mon[118354]: pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:15.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:14 smithi156 ceph-mon[105979]: pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:16.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:16 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3196302671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:16.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:16 smithi064 ceph-mon[118354]: pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:16 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3196302671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:16 smithi064 ceph-mon[101070]: pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:16 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3196302671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:16 smithi156 ceph-mon[105979]: pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:17.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:17 smithi064 conmon[101498]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:42:17] "GET /metrics HTTP/1.1" 200 239606 "" "Prometheus/2.43.0" 2023-10-09T05:42:18.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:18 smithi064 ceph-mon[101070]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:18.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:18 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3994825208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:18.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:18 smithi064 ceph-mon[118354]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:18.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:18 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3994825208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:19.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:18 smithi156 ceph-mon[105979]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:19.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:18 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3994825208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:20.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:20 smithi064 ceph-mon[101070]: pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:20.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:20 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1527567201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:20.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:20 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:20.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:20 smithi064 ceph-mon[118354]: pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:20.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:20 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/1527567201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:20.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:20 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:21.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:20 smithi156 ceph-mon[105979]: pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:21.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:20 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/1527567201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:21.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:20 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:22.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:22 smithi064 ceph-mon[101070]: pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:22.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:22 smithi064 ceph-mon[118354]: pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:23.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:22 smithi156 ceph-mon[105979]: pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:23.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:22 smithi156 conmon[109199]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:42:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:42:23.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:23 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1737903494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:23.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:23 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/1737903494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:24.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:23 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/1737903494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:24.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:24 smithi064 ceph-mon[101070]: pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:24.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:24 smithi064 ceph-mon[118354]: pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:25.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:24 smithi156 ceph-mon[105979]: pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:25.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:25 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/982296000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:25.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:25 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/982296000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:26.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:25 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/982296000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:26.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:26 smithi064 ceph-mon[101070]: pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:26.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:26 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:26.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:26 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:42:26.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:26 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:42:26.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:26 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:42:26.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:26 smithi064 ceph-mon[118354]: pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:26.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:26 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:26.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:26 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:42:26.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:26 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:42:26.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:26 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:42:27.055 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:26 smithi156 ceph-mon[105979]: pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:27.055 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:26 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:27.055 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:26 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:42:27.055 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:26 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:42:27.056 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:26 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:42:27.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:27 smithi064 conmon[101498]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:42:27] "GET /metrics HTTP/1.1" 200 239606 "" "Prometheus/2.43.0" 2023-10-09T05:42:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:27 smithi156 ceph-mon[105979]: Upgrade: Updating mgr.x 2023-10-09T05:42:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:27 smithi156 ceph-mon[105979]: Deploying daemon mgr.x on smithi156 2023-10-09T05:42:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:27 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3582073587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:27 smithi156 ceph-mon[105979]: pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:27.871 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:27 smithi156 systemd[1]: Stopping Ceph mgr.x for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:42:27.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:27 smithi064 ceph-mon[101070]: Upgrade: Updating mgr.x 2023-10-09T05:42:27.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:27 smithi064 ceph-mon[101070]: Deploying daemon mgr.x on smithi156 2023-10-09T05:42:27.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:27 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3582073587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:27.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:27 smithi064 ceph-mon[101070]: pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:27.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:27 smithi064 ceph-mon[118354]: Upgrade: Updating mgr.x 2023-10-09T05:42:27.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:27 smithi064 ceph-mon[118354]: Deploying daemon mgr.x on smithi156 2023-10-09T05:42:27.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:27 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3582073587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:27.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:27 smithi064 ceph-mon[118354]: pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:28.682 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:28 smithi156 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T05:42:28.682 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:28 smithi156 bash[160074]: 2efef4f1cb6cf0213bd8c4743ec3470f4e7b4a9abaf756b0d08ec07cad5bce07 2023-10-09T05:42:28.682 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:28 smithi156 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T05:42:28.682 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:28 smithi156 systemd[1]: Stopped Ceph mgr.x for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:42:29.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:28 smithi156 systemd[1]: Starting Ceph mgr.x for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:42:29.566 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:29 smithi156 bash[160427]: 5085268da26c896a58f83a3262c3a55517f9bb5ce5d893c5c34e2c11d55c44ae 2023-10-09T05:42:29.566 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:29 smithi156 ceph-mgr[160492]: -- 172.21.15.156:0/4211433247 <== mon.1 v2:172.21.15.156:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55da1e7af1e0 con 0x55da1e786c00 2023-10-09T05:42:29.566 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:29 smithi156 systemd[1]: Started Ceph mgr.x for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:42:29.566 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:29 smithi156 conmon[160468]: 2023-10-09T05:42:29.387+0000 7fb06c72c200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:42:29.567 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:29 smithi156 conmon[160468]: 2023-10-09T05:42:29.472+0000 7fb06c72c200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:42:29.567 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:29 smithi156 conmon[160468]: 2023-10-09T05:42:29.565+0000 7fb06c72c200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:42:29.867 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:29 smithi156 conmon[160468]: 2023-10-09T05:42:29.673+0000 7fb06c72c200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:42:29.868 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:29 smithi156 conmon[160468]: 2023-10-09T05:42:29.867+0000 7fb06c72c200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:42:30.620 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:30 smithi156 conmon[160468]: 2023-10-09T05:42:30.265+0000 7fb06c72c200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:42:30.620 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:30 smithi156 conmon[160468]: 2023-10-09T05:42:30.418+0000 7fb06c72c200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:42:30.620 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:30 smithi156 conmon[160468]: 2023-10-09T05:42:30.512+0000 7fb06c72c200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:42:30.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:30 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:30.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:30 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:30.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:30 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:42:30.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:30 smithi156 ceph-mon[105979]: pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:30.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:30 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/140291505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:30.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:30 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:30.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:30 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:30.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:30 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:42:30.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:30 smithi064 ceph-mon[101070]: pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:30.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:30 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/140291505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:30.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:30 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:30.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:30 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:30.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:30 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:42:30.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:30 smithi064 ceph-mon[118354]: pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:30.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:30 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/140291505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:31.105 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:30 smithi156 conmon[160468]: 2023-10-09T05:42:30.621+0000 7fb06c72c200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:42:31.370 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:31 smithi156 conmon[160468]: 2023-10-09T05:42:31.104+0000 7fb06c72c200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:42:32.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:31 smithi156 conmon[160468]: 2023-10-09T05:42:31.850+0000 7fb06c72c200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:42:32.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:31 smithi156 conmon[160468]: 2023-10-09T05:42:31.938+0000 7fb06c72c200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:42:32.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:32 smithi064 ceph-mon[101070]: pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:32.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:32 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:32.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:32 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:32.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:32 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/2287508697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:32.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:32 smithi064 ceph-mon[118354]: pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:32.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:32 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:32.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:32 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:32.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:32 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/2287508697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:33.072 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:32 smithi156 conmon[160468]: 2023-10-09T05:42:32.783+0000 7fb06c72c200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:42:33.072 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:32 smithi156 conmon[160468]: 2023-10-09T05:42:32.882+0000 7fb06c72c200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:42:33.073 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:32 smithi156 ceph-mon[105979]: pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:33.073 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:32 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:33.073 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:32 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:33.073 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:32 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/2287508697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:33.370 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:33 smithi156 conmon[160468]: 2023-10-09T05:42:33.071+0000 7fb06c72c200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:42:33.742 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:33 smithi156 conmon[160468]: 2023-10-09T05:42:33.374+0000 7fb06c72c200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:42:33.743 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:33 smithi156 conmon[160468]: 2023-10-09T05:42:33.620+0000 7fb06c72c200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:42:34.089 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:33 smithi156 conmon[160468]: 2023-10-09T05:42:33.781+0000 7fb06c72c200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:42:34.089 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:33 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:34.090 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:33 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:34.090 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:33 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:34.090 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:33 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:34.090 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:33 smithi156 ceph-mon[105979]: pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:33 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:33 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:33 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:33 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:33 smithi064 ceph-mon[101070]: pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:33 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:33 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:33 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:33 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:33 smithi064 ceph-mon[118354]: pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:34.344 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:34 smithi156 conmon[160468]: 2023-10-09T05:42:34.088+0000 7fb06c72c200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:42:35.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:34 smithi156 conmon[160468]: 2023-10-09T05:42:34.836+0000 7fb06c72c200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:42:35.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:34 smithi156 conmon[160468]: 2023-10-09T05:42:34.970+0000 7fb06c72c200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:42:35.428 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:35 smithi156 conmon[160468]: 2023-10-09T05:42:35.172+0000 7fb06c72c200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:42:35.428 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:35 smithi156 conmon[160468]: 2023-10-09T05:42:35.305+0000 7fb06c72c200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:42:35.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:35 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/4213408882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:35.871 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:35 smithi156 conmon[160468]: 2023-10-09T05:42:35.543+0000 7fb06c72c200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:42:35.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:35 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/4213408882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:35.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:35 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/4213408882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:36.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:36 smithi156 ceph-mon[105979]: pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:36.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:36 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:36.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:36 smithi156 ceph-mon[105979]: Standby manager daemon x restarted 2023-10-09T05:42:36.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:36 smithi156 ceph-mon[105979]: Standby manager daemon x started 2023-10-09T05:42:36.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:36 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:42:36.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:36 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:42:36.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:36 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:42:36.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:36 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:42:36.871 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:36 smithi156 conmon[160468]: 2023-10-09T05:42:36.457+0000 7fb06c72c200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:42:36.871 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:36 smithi156 conmon[160468]: [09/Oct/2023:05:42:36] ENGINE Bus STARTING 2023-10-09T05:42:36.871 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:36 smithi156 conmon[160468]: CherryPy Checker: 2023-10-09T05:42:36.871 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:36 smithi156 conmon[160468]: The Application mounted at '' has an empty config. 2023-10-09T05:42:36.872 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:36 smithi156 conmon[160468]: 2023-10-09T05:42:36.872 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:36 smithi156 conmon[160468]: [09/Oct/2023:05:42:36] ENGINE Serving on http://:::9283 2023-10-09T05:42:36.872 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:36 smithi156 conmon[160468]: [09/Oct/2023:05:42:36] ENGINE Bus STARTED 2023-10-09T05:42:36.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:36 smithi064 ceph-mon[101070]: pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:36.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:36 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:36.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:36 smithi064 ceph-mon[101070]: Standby manager daemon x restarted 2023-10-09T05:42:36.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:36 smithi064 ceph-mon[101070]: Standby manager daemon x started 2023-10-09T05:42:36.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:36 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:42:36.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:36 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:42:36.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:36 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:42:36.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:36 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:42:36.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:36 smithi064 ceph-mon[118354]: pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:36.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:36 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:36.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:36 smithi064 ceph-mon[118354]: Standby manager daemon x restarted 2023-10-09T05:42:36.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:36 smithi064 ceph-mon[118354]: Standby manager daemon x started 2023-10-09T05:42:36.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:36 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:42:36.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:36 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:42:36.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:36 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:42:36.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:36 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:42:37.321 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:37 smithi156 conmon[160468]: [09/Oct/2023:05:42:37] ENGINE Bus STOPPING 2023-10-09T05:42:37.321 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:37 smithi156 conmon[160468]: [09/Oct/2023:05:42:37] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:42:37.322 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:37 smithi156 conmon[160468]: [09/Oct/2023:05:42:37] ENGINE Bus STOPPED 2023-10-09T05:42:37.322 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:37 smithi156 conmon[160468]: [09/Oct/2023:05:42:37] ENGINE Bus STARTING 2023-10-09T05:42:37.368 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:37 smithi064 conmon[101498]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:42:37] "GET /metrics HTTP/1.1" 200 239604 "" "Prometheus/2.43.0" 2023-10-09T05:42:37.368 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:37 smithi064 conmon[101498]: 2023-10-09T05:42:37.232+0000 7f82ca71e700 -1 mgr handle_mgr_map I was active but no longer am 2023-10-09T05:42:37.368 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:37 smithi064 conmon[101498]: ignoring --setuser ceph since I am not root 2023-10-09T05:42:37.368 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:37 smithi064 conmon[101498]: ignoring --setgroup ceph since I am not root 2023-10-09T05:42:37.368 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mgr[101522]: -- 172.21.15.64:0/1884054079 <== mon.0 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563f9ad769c0 con 0x563f9ad4ec00 2023-10-09T05:42:37.620 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:37 smithi156 conmon[160468]: 2023-10-09T05:42:37.376+0000 7fafc6b97700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:42:37.620 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:37 smithi156 conmon[160468]: 2023-10-09T05:42:37.376+0000 7fafc6b97700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:42:37.620 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:37 smithi156 conmon[160468]: 2023-10-09T05:42:37.376+0000 7fafc6b97700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:42:37.621 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:37 smithi156 conmon[160468]: 2023-10-09T05:42:37.376+0000 7fafc6b97700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:42:37.621 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:37 smithi156 conmon[160468]: 2023-10-09T05:42:37.376+0000 7fafc6b97700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:42:37.621 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:37 smithi156 conmon[160468]: 2023-10-09T05:42:37.379+0000 7fafcd3a4700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:42:37.621 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:37 smithi156 conmon[160468]: 2023-10-09T05:42:37.379+0000 7fafcd3a4700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:42:37.621 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:37 smithi156 conmon[160468]: 2023-10-09T05:42:37.379+0000 7fafcd3a4700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:42:37.621 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:37 smithi156 conmon[160468]: 2023-10-09T05:42:37.379+0000 7fafcd3a4700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:42:37.621 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:37 smithi156 conmon[160468]: 2023-10-09T05:42:37.379+0000 7fafcd3a4700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:42:37.621 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:37 smithi156 conmon[160468]: [09/Oct/2023:05:42:37] ENGINE Serving on http://:::9283 2023-10-09T05:42:37.621 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:37 smithi156 conmon[160468]: [09/Oct/2023:05:42:37] ENGINE Bus STARTED 2023-10-09T05:42:37.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: mgrmap e21: y(active, since 3m), standbys: x 2023-10-09T05:42:37.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:37.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:37.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:42:37.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:42:37.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:37.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3401837144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:37.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:42:37.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T05:42:37.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: osdmap e77: 8 total, 8 up, 8 in 2023-10-09T05:42:37.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T05:42:37.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: mgrmap e22: x(active, starting, since 0.00547937s) 2023-10-09T05:42:37.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:42:37.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:42:37.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:42:37.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.gmrwsy"}]: dispatch 2023-10-09T05:42:37.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.pizbdw"}]: dispatch 2023-10-09T05:42:37.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:42:37.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:42:37.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:42:37.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:42:37.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:42:37.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:42:37.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:42:37.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:42:37.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:42:37.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:42:37.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:42:37.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:42:37.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: Manager daemon x is now available 2023-10-09T05:42:37.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:37.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:42:37.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:42:37.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:37.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:42:37.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:42:37.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T05:42:37.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:37 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T05:42:37.627 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:37 smithi064 conmon[101498]: 2023-10-09T05:42:37.420+0000 7f2913fee1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:42:37.627 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:37 smithi064 conmon[101498]: 2023-10-09T05:42:37.516+0000 7f2913fee1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:42:37.628 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: mgrmap e21: y(active, since 3m), standbys: x 2023-10-09T05:42:37.628 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:37.628 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:37.628 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:42:37.628 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:42:37.628 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:37.628 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3401837144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:37.629 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:42:37.629 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T05:42:37.629 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: osdmap e77: 8 total, 8 up, 8 in 2023-10-09T05:42:37.629 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T05:42:37.629 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: mgrmap e22: x(active, starting, since 0.00547937s) 2023-10-09T05:42:37.630 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:42:37.630 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:42:37.630 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:42:37.631 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.gmrwsy"}]: dispatch 2023-10-09T05:42:37.631 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.pizbdw"}]: dispatch 2023-10-09T05:42:37.631 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:42:37.631 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:42:37.631 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:42:37.631 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:42:37.631 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:42:37.632 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:42:37.632 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:42:37.632 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:42:37.632 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:42:37.632 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:42:37.632 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:42:37.632 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: mgrmap e21: y(active, since 3m), standbys: x 2023-10-09T05:42:37.632 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:37.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:37.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:42:37.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:42:37.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' 2023-10-09T05:42:37.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3401837144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:37.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:42:37.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T05:42:37.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: osdmap e77: 8 total, 8 up, 8 in 2023-10-09T05:42:37.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24341 172.21.15.64:0/4190892671' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T05:42:37.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: mgrmap e22: x(active, starting, since 0.00547937s) 2023-10-09T05:42:37.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:42:37.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:42:37.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:42:37.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.gmrwsy"}]: dispatch 2023-10-09T05:42:37.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.pizbdw"}]: dispatch 2023-10-09T05:42:37.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:42:37.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:42:37.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:42:37.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:42:37.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:42:37.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:42:37.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:42:37.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:42:37.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:42:37.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:42:37.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:42:37.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:42:37.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: Manager daemon x is now available 2023-10-09T05:42:37.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:37.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:42:37.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:42:37.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:37.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:42:37.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:42:37.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T05:42:37.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T05:42:37.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:42:37.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: Manager daemon x is now available 2023-10-09T05:42:37.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:37.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:42:37.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:42:37.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:37.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:42:37.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:42:37.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T05:42:37.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:37 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T05:42:37.951 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:37 smithi064 conmon[101498]: 2023-10-09T05:42:37.627+0000 7f2913fee1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:42:37.951 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:37 smithi064 conmon[101498]: 2023-10-09T05:42:37.742+0000 7f2913fee1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:42:37.951 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:37 smithi064 conmon[101498]: 2023-10-09T05:42:37.950+0000 7f2913fee1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:42:38.611 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:38 smithi064 conmon[101498]: 2023-10-09T05:42:38.351+0000 7f2913fee1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:42:38.639 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:38 smithi064 conmon[101498]: 2023-10-09T05:42:38.459+0000 7f2913fee1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:42:38.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:38 smithi064 conmon[101498]: 2023-10-09T05:42:38.610+0000 7f2913fee1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:42:38.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:38 smithi064 conmon[101498]: 2023-10-09T05:42:38.725+0000 7f2913fee1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:42:39.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:39 smithi064 ceph-mon[118354]: Deploying cephadm binary to smithi156 2023-10-09T05:42:39.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:39 smithi064 ceph-mon[118354]: Deploying cephadm binary to smithi064 2023-10-09T05:42:39.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:39 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:39.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:39 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:39.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:39 smithi064 ceph-mon[118354]: mgrmap e23: x(active, since 1.00882s) 2023-10-09T05:42:39.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:39 smithi064 ceph-mon[118354]: pgmap v3: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:39.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:39 smithi064 ceph-mon[101070]: Deploying cephadm binary to smithi156 2023-10-09T05:42:39.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:39 smithi064 ceph-mon[101070]: Deploying cephadm binary to smithi064 2023-10-09T05:42:39.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:39 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:39.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:39 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:39.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:39 smithi064 ceph-mon[101070]: mgrmap e23: x(active, since 1.00882s) 2023-10-09T05:42:39.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:39 smithi064 ceph-mon[101070]: pgmap v3: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:39.470 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:39 smithi064 conmon[101498]: 2023-10-09T05:42:39.245+0000 7f2913fee1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:42:39.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:39 smithi156 ceph-mon[105979]: Deploying cephadm binary to smithi156 2023-10-09T05:42:39.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:39 smithi156 ceph-mon[105979]: Deploying cephadm binary to smithi064 2023-10-09T05:42:39.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:39 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:39.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:39 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:39.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:39 smithi156 ceph-mon[105979]: mgrmap e23: x(active, since 1.00882s) 2023-10-09T05:42:39.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:39 smithi156 ceph-mon[105979]: pgmap v3: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:40.247 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:40 smithi064 ceph-mon[118354]: [09/Oct/2023:05:42:38] ENGINE Bus STARTING 2023-10-09T05:42:40.247 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:40 smithi064 ceph-mon[118354]: [09/Oct/2023:05:42:38] ENGINE Serving on http://172.21.15.156:8765 2023-10-09T05:42:40.247 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:40 smithi064 ceph-mon[118354]: [09/Oct/2023:05:42:38] ENGINE Serving on https://172.21.15.156:7150 2023-10-09T05:42:40.247 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:40 smithi064 ceph-mon[118354]: [09/Oct/2023:05:42:38] ENGINE Bus STARTED 2023-10-09T05:42:40.247 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:40 smithi064 ceph-mon[118354]: pgmap v4: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:40.247 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:40 smithi064 ceph-mon[118354]: mgrmap e24: x(active, since 2s) 2023-10-09T05:42:40.247 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:40 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3298155299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:40.248 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:39 smithi064 conmon[101498]: 2023-10-09T05:42:39.979+0000 7f2913fee1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:42:40.248 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:40 smithi064 conmon[101498]: 2023-10-09T05:42:40.173+0000 7f2913fee1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:42:40.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:40 smithi156 ceph-mon[105979]: [09/Oct/2023:05:42:38] ENGINE Bus STARTING 2023-10-09T05:42:40.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:40 smithi156 ceph-mon[105979]: [09/Oct/2023:05:42:38] ENGINE Serving on http://172.21.15.156:8765 2023-10-09T05:42:40.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:40 smithi156 ceph-mon[105979]: [09/Oct/2023:05:42:38] ENGINE Serving on https://172.21.15.156:7150 2023-10-09T05:42:40.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:40 smithi156 ceph-mon[105979]: [09/Oct/2023:05:42:38] ENGINE Bus STARTED 2023-10-09T05:42:40.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:40 smithi156 ceph-mon[105979]: pgmap v4: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:40.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:40 smithi156 ceph-mon[105979]: mgrmap e24: x(active, since 2s) 2023-10-09T05:42:40.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:40 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3298155299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:40.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:40 smithi064 ceph-mon[101070]: [09/Oct/2023:05:42:38] ENGINE Bus STARTING 2023-10-09T05:42:40.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:40 smithi064 ceph-mon[101070]: [09/Oct/2023:05:42:38] ENGINE Serving on http://172.21.15.156:8765 2023-10-09T05:42:40.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:40 smithi064 ceph-mon[101070]: [09/Oct/2023:05:42:38] ENGINE Serving on https://172.21.15.156:7150 2023-10-09T05:42:40.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:40 smithi064 ceph-mon[101070]: [09/Oct/2023:05:42:38] ENGINE Bus STARTED 2023-10-09T05:42:40.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:40 smithi064 ceph-mon[101070]: pgmap v4: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:40.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:40 smithi064 ceph-mon[101070]: mgrmap e24: x(active, since 2s) 2023-10-09T05:42:40.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:40 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3298155299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:41.576 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:41 smithi064 ceph-mon[101070]: pgmap v5: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:41.577 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:41 smithi064 ceph-mon[118354]: pgmap v5: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:41.577 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:41 smithi064 conmon[101498]: 2023-10-09T05:42:41.229+0000 7f2913fee1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:42:41.577 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:41 smithi064 conmon[101498]: 2023-10-09T05:42:41.344+0000 7f2913fee1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:42:41.577 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:41 smithi064 conmon[101498]: 2023-10-09T05:42:41.576+0000 7f2913fee1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:42:41.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:41 smithi156 ceph-mon[105979]: pgmap v5: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:41.958 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:41 smithi064 conmon[101498]: 2023-10-09T05:42:41.957+0000 7f2913fee1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:42:42.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:42 smithi064 conmon[101498]: 2023-10-09T05:42:42.071+0000 7f2913fee1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:42:42.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:42 smithi064 conmon[101498]: 2023-10-09T05:42:42.421+0000 7f2913fee1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:42:42.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:42.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:42.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1223001076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:42.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:42.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:42.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:42.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:42.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:42.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:42.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:42.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:42.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:42.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:42.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:42.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:42.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:42.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:42.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:42.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/1223001076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:42.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:42.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:42.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:42.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:42.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:42.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:42.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:42.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:42.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:42.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:42.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:42.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:42.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:42 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:42.870 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:42 smithi156 conmon[160468]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:42:42] "GET /metrics HTTP/1.1" 200 30957 "" "Prometheus/2.43.0" 2023-10-09T05:42:42.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:42 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:42.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:42 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:42.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:42 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/1223001076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:42.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:42 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:42.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:42 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:42.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:42 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:42.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:42 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:42.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:42 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:42.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:42 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:42.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:42 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:42.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:42 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:42.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:42 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:42.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:42 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:42.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:42 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:42.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:42 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:42.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:42 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:43.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:42 smithi064 conmon[101498]: 2023-10-09T05:42:42.751+0000 7f2913fee1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:42:43.964 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:43 smithi064 conmon[101498]: 2023-10-09T05:42:43.595+0000 7f2913fee1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:42:43.964 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:43 smithi064 conmon[101498]: 2023-10-09T05:42:43.756+0000 7f2913fee1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:42:44.124 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:43 smithi156 ceph-mon[105979]: Adjusting osd_memory_target on smithi064 to 2500M 2023-10-09T05:42:44.124 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:43 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:44.124 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:43 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:44.125 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:43 smithi156 ceph-mon[105979]: pgmap v6: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:44.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:43 smithi064 ceph-mon[101070]: Adjusting osd_memory_target on smithi064 to 2500M 2023-10-09T05:42:44.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:43 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:44.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:43 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:44.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:43 smithi064 ceph-mon[101070]: pgmap v6: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:44.220 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:44 smithi064 conmon[101498]: 2023-10-09T05:42:44.003+0000 7f2913fee1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:42:44.220 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:44 smithi064 conmon[101498]: 2023-10-09T05:42:44.148+0000 7f2913fee1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:42:44.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:43 smithi064 ceph-mon[118354]: Adjusting osd_memory_target on smithi064 to 2500M 2023-10-09T05:42:44.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:43 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:44.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:43 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:44.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:43 smithi064 ceph-mon[118354]: pgmap v6: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:44.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:44 smithi064 conmon[101498]: 2023-10-09T05:42:44.425+0000 7f2913fee1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:42:45.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:44 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3338914675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:45.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:44 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3338914675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:45.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:44 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3338914675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:45.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:45 smithi064 conmon[101498]: 2023-10-09T05:42:45.479+0000 7f2913fee1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:42:45.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:45 smithi064 conmon[101498]: [09/Oct/2023:05:42:45] ENGINE Bus STARTING 2023-10-09T05:42:45.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:45 smithi064 conmon[101498]: CherryPy Checker: 2023-10-09T05:42:45.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:45 smithi064 conmon[101498]: The Application mounted at '' has an empty config. 2023-10-09T05:42:45.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:45 smithi064 conmon[101498]: 2023-10-09T05:42:45.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:45 smithi064 conmon[101498]: [09/Oct/2023:05:42:45] ENGINE Serving on http://:::9283 2023-10-09T05:42:45.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:45 smithi064 conmon[101498]: [09/Oct/2023:05:42:45] ENGINE Bus STARTED 2023-10-09T05:42:46.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:45 smithi156 ceph-mon[105979]: pgmap v7: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:46.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:45 smithi156 ceph-mon[105979]: Standby manager daemon y started 2023-10-09T05:42:46.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:45 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.64:0/1754895882' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T05:42:46.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:45 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.64:0/1754895882' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:42:46.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:45 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.64:0/1754895882' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T05:42:46.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:45 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.64:0/1754895882' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:42:46.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:45 smithi064 ceph-mon[101070]: pgmap v7: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:46.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:45 smithi064 ceph-mon[101070]: Standby manager daemon y started 2023-10-09T05:42:46.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:45 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.64:0/1754895882' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T05:42:46.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:45 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.64:0/1754895882' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:42:46.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:45 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.64:0/1754895882' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T05:42:46.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:45 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.64:0/1754895882' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:42:46.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:45 smithi064 ceph-mon[118354]: pgmap v7: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:46.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:45 smithi064 ceph-mon[118354]: Standby manager daemon y started 2023-10-09T05:42:46.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:45 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.64:0/1754895882' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T05:42:46.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:45 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.64:0/1754895882' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:42:46.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:45 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.64:0/1754895882' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T05:42:46.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:45 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.64:0/1754895882' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:42:47.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:42:47 smithi064 conmon[101498]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:42:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:42:47.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:47.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:47.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:47.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:47.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:47.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:47.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:47.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:47.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:47.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:47.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[101070]: Adjusting osd_memory_target on smithi156 to 2500M 2023-10-09T05:42:47.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:47.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:42:47.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:42:47.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[101070]: Updating smithi064:/etc/ceph/ceph.conf 2023-10-09T05:42:47.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[101070]: Updating smithi156:/etc/ceph/ceph.conf 2023-10-09T05:42:47.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[101070]: mgrmap e25: x(active, since 9s), standbys: y 2023-10-09T05:42:47.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:42:47.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1400804561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:47.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:47.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:47.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:47.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:47.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:47.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:47.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:47.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:47.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:47.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:47.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:47.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:47.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:47.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:47.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:47.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[118354]: Adjusting osd_memory_target on smithi156 to 2500M 2023-10-09T05:42:47.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:47.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:42:47.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:42:47.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[118354]: Updating smithi064:/etc/ceph/ceph.conf 2023-10-09T05:42:47.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[118354]: Updating smithi156:/etc/ceph/ceph.conf 2023-10-09T05:42:47.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[118354]: mgrmap e25: x(active, since 9s), standbys: y 2023-10-09T05:42:47.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:42:47.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/1400804561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:47.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:47.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:47.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:47.474 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:47.474 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:47 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:47.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:47 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:47.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:47 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:47.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:47 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:47.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:47 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:47.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:47 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:47.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:47 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:47.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:47 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:47.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:47 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:47.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:47 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:47.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:47 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:42:47.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:47 smithi156 ceph-mon[105979]: Adjusting osd_memory_target on smithi156 to 2500M 2023-10-09T05:42:47.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:47 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:47.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:47 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:42:47.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:47 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:42:47.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:47 smithi156 ceph-mon[105979]: Updating smithi064:/etc/ceph/ceph.conf 2023-10-09T05:42:47.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:47 smithi156 ceph-mon[105979]: Updating smithi156:/etc/ceph/ceph.conf 2023-10-09T05:42:47.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:47 smithi156 ceph-mon[105979]: mgrmap e25: x(active, since 9s), standbys: y 2023-10-09T05:42:47.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:47 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:42:47.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:47 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/1400804561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:47.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:47 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:47.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:47 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:47.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:47 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:47.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:47 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:47.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:47 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:48.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:48 smithi064 ceph-mon[118354]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:42:48.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:48 smithi064 ceph-mon[118354]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:42:48.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:48 smithi064 ceph-mon[118354]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:42:48.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:48 smithi064 ceph-mon[118354]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:42:48.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:48 smithi064 ceph-mon[118354]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:42:48.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:48 smithi064 ceph-mon[118354]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:42:48.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:48 smithi064 ceph-mon[118354]: pgmap v8: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:48.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:48 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:48.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:48 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:48.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:48 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:48.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:48 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:48.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:48 smithi064 ceph-mon[101070]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:42:48.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:48 smithi064 ceph-mon[101070]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:42:48.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:48 smithi064 ceph-mon[101070]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:42:48.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:48 smithi064 ceph-mon[101070]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:42:48.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:48 smithi064 ceph-mon[101070]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:42:48.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:48 smithi064 ceph-mon[101070]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:42:48.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:48 smithi064 ceph-mon[101070]: pgmap v8: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:48.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:48 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:48.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:48 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:48.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:48 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:48.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:48 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:48.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:48 smithi156 ceph-mon[105979]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:42:48.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:48 smithi156 ceph-mon[105979]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:42:48.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:48 smithi156 ceph-mon[105979]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:42:48.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:48 smithi156 ceph-mon[105979]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:42:48.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:48 smithi156 ceph-mon[105979]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:42:48.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:48 smithi156 ceph-mon[105979]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:42:48.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:48 smithi156 ceph-mon[105979]: pgmap v8: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:48.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:48 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:48.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:48 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:48.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:48 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:48.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:48 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:49.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:49 smithi064 ceph-mon[101070]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T05:42:49.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:49 smithi064 ceph-mon[101070]: Reconfiguring daemon alertmanager.a on smithi064 2023-10-09T05:42:49.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:49 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/2367652460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:49.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:49 smithi064 ceph-mon[118354]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T05:42:49.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:49 smithi064 ceph-mon[118354]: Reconfiguring daemon alertmanager.a on smithi064 2023-10-09T05:42:49.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:49 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/2367652460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:49.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:49 smithi156 ceph-mon[105979]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T05:42:49.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:49 smithi156 ceph-mon[105979]: Reconfiguring daemon alertmanager.a on smithi064 2023-10-09T05:42:49.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:49 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/2367652460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:50.249 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:42:50 smithi064 systemd[1]: Stopping Ceph alertmanager.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:42:50.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:50 smithi064 ceph-mon[101070]: pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:50.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:50 smithi064 ceph-mon[118354]: pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:50.591 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:42:50 smithi064 conmon[174526]: ts=2023-10-09T05:42:50.248Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-10-09T05:42:50.591 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:42:50 smithi064 bash[196673]: 51110671bb411847bbc25ea8af102e2d1069ab374c2b359f96a83395b9d4ea91 2023-10-09T05:42:50.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:50 smithi156 ceph-mon[105979]: pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:51.424 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:42:51 smithi064 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T05:42:51.424 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:42:51 smithi064 systemd[1]: Stopped Ceph alertmanager.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:42:51.424 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:42:51 smithi064 systemd[1]: Starting Ceph alertmanager.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:42:51.685 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:51 smithi064 ceph-mon[101070]: pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:51.686 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:51 smithi064 ceph-mon[118354]: pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:51.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:51 smithi156 ceph-mon[105979]: pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:51.969 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:42:51 smithi064 conmon[197067]: ts=2023-10-09T05:42:51.683Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-10-09T05:42:51.969 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:42:51 smithi064 conmon[197067]: ts=2023-10-09T05:42:51.683Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-10-09T05:42:51.969 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:42:51 smithi064 conmon[197067]: ts=2023-10-09T05:42:51.684Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.64 port=9094 2023-10-09T05:42:51.970 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:42:51 smithi064 conmon[197067]: ts=2023-10-09T05:42:51.685Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T05:42:51.970 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:42:51 smithi064 conmon[197067]: ts=2023-10-09T05:42:51.714Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T05:42:51.970 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:42:51 smithi064 conmon[197067]: ts=2023-10-09T05:42:51.714Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T05:42:51.971 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:42:51 smithi064 conmon[197067]: ts=2023-10-09T05:42:51.717Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-10-09T05:42:51.971 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:42:51 smithi064 conmon[197067]: ts=2023-10-09T05:42:51.717Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-10-09T05:42:51.971 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:42:51 smithi064 bash[197006]: 386f8a4e2221913e01416d141066eeb8280f62201286ec81d1b08f71ca497bb4 2023-10-09T05:42:51.971 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:42:51 smithi064 systemd[1]: Started Ceph alertmanager.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:42:52.374 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:52 smithi064 systemd[1]: Stopping Ceph node-exporter.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:42:52.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:52 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/747160425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:52.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:52 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:52.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:52 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:52.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:52 smithi064 ceph-mon[118354]: Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T05:42:52.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:52 smithi064 ceph-mon[118354]: Reconfiguring daemon node-exporter.a on smithi064 2023-10-09T05:42:52.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:52 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:52.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:52 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/747160425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:52.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:52 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:52.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:52 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:52.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:52 smithi064 ceph-mon[101070]: Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T05:42:52.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:52 smithi064 ceph-mon[101070]: Reconfiguring daemon node-exporter.a on smithi064 2023-10-09T05:42:52.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:52 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:52.766 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:52 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/747160425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:52.766 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:52 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:52.766 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:52 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:52.766 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:52 smithi156 ceph-mon[105979]: Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T05:42:52.766 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:52 smithi156 ceph-mon[105979]: Reconfiguring daemon node-exporter.a on smithi064 2023-10-09T05:42:52.766 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:52 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:42:53.121 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:42:52 smithi156 conmon[160468]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:42:52] "GET /metrics HTTP/1.1" 200 33801 "" "Prometheus/2.43.0" 2023-10-09T05:42:53.219 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:52 smithi064 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T05:42:53.219 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:52 smithi064 bash[197302]: 59bb208a1e32d93cd1848aa8775c51bb0e3807a3a3ab0c70faa34a2da9095b41 2023-10-09T05:42:53.219 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-09T05:42:53.219 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 systemd[1]: Stopped Ceph node-exporter.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:42:53.219 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 systemd[1]: Starting Ceph node-exporter.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:42:53.686 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:53 smithi064 ceph-mon[101070]: pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:53.686 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:53 smithi064 ceph-mon[118354]: pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:53.687 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.635Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-10-09T05:42:53.687 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.635Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-10-09T05:42:53.687 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.636Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-10-09T05:42:53.687 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.636Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-10-09T05:42:53.687 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-10-09T05:42:53.687 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T05:42:53.687 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-10-09T05:42:53.688 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=arp 2023-10-09T05:42:53.688 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=bcache 2023-10-09T05:42:53.688 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=bonding 2023-10-09T05:42:53.688 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=btrfs 2023-10-09T05:42:53.688 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=conntrack 2023-10-09T05:42:53.688 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=cpu 2023-10-09T05:42:53.688 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-10-09T05:42:53.688 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=diskstats 2023-10-09T05:42:53.688 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=dmi 2023-10-09T05:42:53.688 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=edac 2023-10-09T05:42:53.688 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=entropy 2023-10-09T05:42:53.688 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-10-09T05:42:53.689 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=filefd 2023-10-09T05:42:53.689 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=filesystem 2023-10-09T05:42:53.689 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=hwmon 2023-10-09T05:42:53.689 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=infiniband 2023-10-09T05:42:53.689 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=ipvs 2023-10-09T05:42:53.689 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=loadavg 2023-10-09T05:42:53.689 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=mdadm 2023-10-09T05:42:53.689 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=meminfo 2023-10-09T05:42:53.689 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=netclass 2023-10-09T05:42:53.689 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=netdev 2023-10-09T05:42:53.689 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=netstat 2023-10-09T05:42:53.690 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=nfs 2023-10-09T05:42:53.690 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=nfsd 2023-10-09T05:42:53.690 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=nvme 2023-10-09T05:42:53.690 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=os 2023-10-09T05:42:53.690 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-10-09T05:42:53.690 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=pressure 2023-10-09T05:42:53.690 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=rapl 2023-10-09T05:42:53.690 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=schedstat 2023-10-09T05:42:53.690 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=selinux 2023-10-09T05:42:53.690 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=sockstat 2023-10-09T05:42:53.690 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=softnet 2023-10-09T05:42:53.690 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=stat 2023-10-09T05:42:53.691 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=tapestats 2023-10-09T05:42:53.691 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=textfile 2023-10-09T05:42:53.691 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-10-09T05:42:53.691 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=time 2023-10-09T05:42:53.691 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-10-09T05:42:53.691 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=uname 2023-10-09T05:42:53.691 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=vmstat 2023-10-09T05:42:53.691 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=xfs 2023-10-09T05:42:53.691 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=node_exporter.go:117 level=info collector=zfs 2023-10-09T05:42:53.691 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-10-09T05:42:53.691 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197670]: ts=2023-10-09T05:42:53.637Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-10-09T05:42:53.691 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 bash[197629]: afb28622babd191733cc0f1f8c57cb6135b975673afec1be3854b9e591e706da 2023-10-09T05:42:53.692 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 05:42:53 smithi064 systemd[1]: Started Ceph node-exporter.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:42:53.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:53 smithi156 ceph-mon[105979]: pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:53.969 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:42:53 smithi064 conmon[197067]: ts=2023-10-09T05:42:53.686Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000896716s 2023-10-09T05:42:54.767 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:54 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:54.768 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:54 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:54.768 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:54 smithi156 ceph-mon[105979]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T05:42:54.768 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:54 smithi156 ceph-mon[105979]: Reconfiguring daemon prometheus.a on smithi156 2023-10-09T05:42:54.768 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:54 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3750009644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:54.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:54 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:54.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:54 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:54.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:54 smithi064 ceph-mon[101070]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T05:42:54.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:54 smithi064 ceph-mon[101070]: Reconfiguring daemon prometheus.a on smithi156 2023-10-09T05:42:54.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:54 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3750009644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:54.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:54 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:54.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:54 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:54.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:54 smithi064 ceph-mon[118354]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T05:42:54.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:54 smithi064 ceph-mon[118354]: Reconfiguring daemon prometheus.a on smithi156 2023-10-09T05:42:54.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:54 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3750009644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:55.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:55 smithi156 ceph-mon[105979]: pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:55.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:55 smithi064 ceph-mon[101070]: pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:55.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:55 smithi064 ceph-mon[118354]: pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:56.648 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:56 smithi156 systemd[1]: Stopping Ceph prometheus.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:42:56.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:56 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3272712565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:56.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:56 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3272712565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:57.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:56 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3272712565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:57.121 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:56 smithi156 conmon[155147]: ts=2023-10-09T05:42:56.648Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T05:42:57.121 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:56 smithi156 conmon[155147]: ts=2023-10-09T05:42:56.648Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-10-09T05:42:57.121 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:56 smithi156 conmon[155147]: ts=2023-10-09T05:42:56.648Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-10-09T05:42:57.121 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:56 smithi156 conmon[155147]: ts=2023-10-09T05:42:56.648Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T05:42:57.121 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:56 smithi156 conmon[155147]: ts=2023-10-09T05:42:56.648Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-10-09T05:42:57.122 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:56 smithi156 conmon[155147]: ts=2023-10-09T05:42:56.648Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-10-09T05:42:57.122 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:56 smithi156 conmon[155147]: ts=2023-10-09T05:42:56.648Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T05:42:57.122 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:56 smithi156 conmon[155147]: ts=2023-10-09T05:42:56.648Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-10-09T05:42:57.122 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:56 smithi156 conmon[155147]: ts=2023-10-09T05:42:56.648Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-10-09T05:42:57.123 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:56 smithi156 conmon[155147]: ts=2023-10-09T05:42:56.649Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T05:42:57.123 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:56 smithi156 conmon[155147]: ts=2023-10-09T05:42:56.649Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-10-09T05:42:57.123 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:56 smithi156 conmon[155147]: ts=2023-10-09T05:42:56.649Z caller=main.go:1147 level=info msg="See you next time!" 2023-10-09T05:42:57.620 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 bash[173609]: d46b6df5107c15c1115e96f578b6fd1a60a27672491d3e080e0257f24e71add8 2023-10-09T05:42:57.620 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T05:42:57.620 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 systemd[1]: Stopped Ceph prometheus.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:42:57.621 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 systemd[1]: Starting Ceph prometheus.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:42:57.945 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:57 smithi156 ceph-mon[105979]: pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:57.945 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 bash[173941]: 09f4a99cecde4dfb4b12caf287185d9881feb3dfbb641d13ad9a2b176f62b3c2 2023-10-09T05:42:57.946 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 conmon[173982]: ts=2023-10-09T05:42:57.925Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-10-09T05:42:57.946 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 conmon[173982]: ts=2023-10-09T05:42:57.925Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-10-09T05:42:57.946 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 conmon[173982]: ts=2023-10-09T05:42:57.925Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi156 (none))" 2023-10-09T05:42:57.946 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 conmon[173982]: ts=2023-10-09T05:42:57.925Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T05:42:57.946 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 conmon[173982]: ts=2023-10-09T05:42:57.925Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T05:42:57.946 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 conmon[173982]: ts=2023-10-09T05:42:57.927Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T05:42:57.946 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 conmon[173982]: ts=2023-10-09T05:42:57.927Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-10-09T05:42:57.946 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 conmon[173982]: ts=2023-10-09T05:42:57.928Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-10-09T05:42:57.946 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 conmon[173982]: ts=2023-10-09T05:42:57.928Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-10-09T05:42:57.946 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 conmon[173982]: ts=2023-10-09T05:42:57.929Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T05:42:57.947 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 conmon[173982]: ts=2023-10-09T05:42:57.929Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.507µs 2023-10-09T05:42:57.947 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 conmon[173982]: ts=2023-10-09T05:42:57.929Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T05:42:57.947 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 conmon[173982]: ts=2023-10-09T05:42:57.930Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-10-09T05:42:57.947 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 systemd[1]: Started Ceph prometheus.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:42:57.947 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 conmon[173982]: ts=2023-10-09T05:42:57.946Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-10-09T05:42:57.947 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 conmon[173982]: ts=2023-10-09T05:42:57.946Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-10-09T05:42:57.947 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 conmon[173982]: ts=2023-10-09T05:42:57.946Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.225µs wal_replay_duration=16.317501ms wbl_replay_duration=129ns total_replay_duration=16.355991ms 2023-10-09T05:42:57.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:57 smithi064 ceph-mon[101070]: pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:57.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:57 smithi064 ceph-mon[118354]: pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:42:58.304 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 conmon[173982]: ts=2023-10-09T05:42:57.949Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T05:42:58.304 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 conmon[173982]: ts=2023-10-09T05:42:57.949Z caller=main.go:1029 level=info msg="TSDB started" 2023-10-09T05:42:58.305 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 conmon[173982]: ts=2023-10-09T05:42:57.949Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T05:42:58.305 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 conmon[173982]: ts=2023-10-09T05:42:57.966Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.030876ms db_storage=1.068µs remote_storage=1.59µs web_handler=472ns query_engine=853ns scrape=610.584µs scrape_sd=149.06µs notify=16.803µs notify_sd=13.707µs rules=15.919744ms tracing=7.073µs 2023-10-09T05:42:58.305 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 conmon[173982]: ts=2023-10-09T05:42:57.966Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-10-09T05:42:58.305 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:42:57 smithi156 conmon[173982]: ts=2023-10-09T05:42:57.966Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-10-09T05:42:58.620 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:58 smithi156 systemd[1]: Stopping Ceph node-exporter.b for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:42:59.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:58 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:59.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:58 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:59.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:58 smithi064 ceph-mon[101070]: Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T05:42:59.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:58 smithi064 ceph-mon[101070]: Reconfiguring daemon node-exporter.b on smithi156 2023-10-09T05:42:59.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:58 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1264487022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:59.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:58 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:59.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:58 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:59.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:58 smithi064 ceph-mon[118354]: Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T05:42:59.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:58 smithi064 ceph-mon[118354]: Reconfiguring daemon node-exporter.b on smithi156 2023-10-09T05:42:59.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:58 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/1264487022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:42:59.370 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-09T05:42:59.370 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 bash[174214]: fead60fdc013821bcb81d67888ea028bf8a07b84730304bc55a3ee99544b0c45 2023-10-09T05:42:59.370 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-09T05:42:59.370 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 systemd[1]: Stopped Ceph node-exporter.b for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:42:59.371 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 systemd[1]: Starting Ceph node-exporter.b for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:42:59.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:58 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:59.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:58 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:42:59.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:58 smithi156 ceph-mon[105979]: Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T05:42:59.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:58 smithi156 ceph-mon[105979]: Reconfiguring daemon node-exporter.b on smithi156 2023-10-09T05:42:59.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:58 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/1264487022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:00.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:59 smithi156 ceph-mon[105979]: pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:00.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:59 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:00.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:42:59 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:00.121 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.759Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-10-09T05:43:00.121 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.759Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-10-09T05:43:00.121 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-10-09T05:43:00.121 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-10-09T05:43:00.121 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-10-09T05:43:00.121 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T05:43:00.121 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-10-09T05:43:00.121 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=arp 2023-10-09T05:43:00.121 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=bcache 2023-10-09T05:43:00.122 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=bonding 2023-10-09T05:43:00.122 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=btrfs 2023-10-09T05:43:00.122 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=conntrack 2023-10-09T05:43:00.122 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=cpu 2023-10-09T05:43:00.122 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-10-09T05:43:00.122 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=diskstats 2023-10-09T05:43:00.122 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=dmi 2023-10-09T05:43:00.122 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=edac 2023-10-09T05:43:00.122 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=entropy 2023-10-09T05:43:00.122 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-10-09T05:43:00.122 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=filefd 2023-10-09T05:43:00.122 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=filesystem 2023-10-09T05:43:00.123 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=hwmon 2023-10-09T05:43:00.123 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=infiniband 2023-10-09T05:43:00.123 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=ipvs 2023-10-09T05:43:00.123 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=loadavg 2023-10-09T05:43:00.123 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=mdadm 2023-10-09T05:43:00.123 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=meminfo 2023-10-09T05:43:00.123 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=netclass 2023-10-09T05:43:00.123 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=netdev 2023-10-09T05:43:00.123 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=netstat 2023-10-09T05:43:00.123 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=nfs 2023-10-09T05:43:00.123 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=nfsd 2023-10-09T05:43:00.123 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=nvme 2023-10-09T05:43:00.124 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=os 2023-10-09T05:43:00.124 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-10-09T05:43:00.124 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=pressure 2023-10-09T05:43:00.124 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=rapl 2023-10-09T05:43:00.124 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=schedstat 2023-10-09T05:43:00.124 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=selinux 2023-10-09T05:43:00.124 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=sockstat 2023-10-09T05:43:00.124 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=softnet 2023-10-09T05:43:00.124 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=stat 2023-10-09T05:43:00.124 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=tapestats 2023-10-09T05:43:00.124 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=textfile 2023-10-09T05:43:00.124 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-10-09T05:43:00.125 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=time 2023-10-09T05:43:00.125 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-10-09T05:43:00.125 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=uname 2023-10-09T05:43:00.125 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=vmstat 2023-10-09T05:43:00.125 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=xfs 2023-10-09T05:43:00.125 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=node_exporter.go:117 level=info collector=zfs 2023-10-09T05:43:00.125 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-10-09T05:43:00.125 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 conmon[174579]: ts=2023-10-09T05:42:59.760Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-10-09T05:43:00.125 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 bash[174539]: 1e8c1e28630a753867da56381edc65a074f68c80a2c269177d693db75749592d 2023-10-09T05:43:00.125 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 05:42:59 smithi156 systemd[1]: Started Ceph node-exporter.b for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:43:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:59 smithi064 ceph-mon[101070]: pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:59 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:42:59 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:00.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:59 smithi064 ceph-mon[118354]: pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:00.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:59 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:00.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:42:59 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:01.366 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:00 smithi156 ceph-mon[105979]: Reconfiguring grafana.a (dependencies changed)... 2023-10-09T05:43:01.366 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:00 smithi156 ceph-mon[105979]: Reconfiguring daemon grafana.a on smithi156 2023-10-09T05:43:01.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:00 smithi064 ceph-mon[101070]: Reconfiguring grafana.a (dependencies changed)... 2023-10-09T05:43:01.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:00 smithi064 ceph-mon[101070]: Reconfiguring daemon grafana.a on smithi156 2023-10-09T05:43:01.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:00 smithi064 ceph-mon[118354]: Reconfiguring grafana.a (dependencies changed)... 2023-10-09T05:43:01.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:00 smithi064 ceph-mon[118354]: Reconfiguring daemon grafana.a on smithi156 2023-10-09T05:43:01.969 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:43:01 smithi064 conmon[197067]: ts=2023-10-09T05:43:01.689Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003167371s 2023-10-09T05:43:02.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:01 smithi156 ceph-mon[105979]: pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:02.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:01 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/2490831566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:02.371 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:02 smithi156 systemd[1]: Stopping Ceph grafana.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:43:02.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:01 smithi064 ceph-mon[101070]: pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:02.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:01 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/2490831566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:02.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:01 smithi064 ceph-mon[118354]: pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:02.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:01 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/2490831566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:03.323 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:02 smithi156 bash[175049]: dbad77de7b52a3c79dc39c2a19b5c7109dd3372599e716a9c62b69bb8bdf5e8e 2023-10-09T05:43:03.323 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:02 smithi156 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T05:43:03.323 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-10-09T05:43:03.323 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 systemd[1]: Stopped Ceph grafana.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:43:03.324 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 systemd[1]: Starting Ceph grafana.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:43:03.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:03 smithi156 ceph-mon[105979]: pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:03.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:03 smithi064 ceph-mon[101070]: pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:03.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:03 smithi064 ceph-mon[118354]: pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:04.021 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[160468]: [09/Oct/2023:05:43:03] ENGINE Bus STOPPING 2023-10-09T05:43:04.021 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[160468]: [09/Oct/2023:05:43:03] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:43:04.021 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[160468]: [09/Oct/2023:05:43:03] ENGINE Bus STOPPED 2023-10-09T05:43:04.021 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[160468]: [09/Oct/2023:05:43:03] ENGINE Bus STARTING 2023-10-09T05:43:04.022 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[160468]: [09/Oct/2023:05:43:03] ENGINE Serving on http://:::9283 2023-10-09T05:43:04.022 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[160468]: [09/Oct/2023:05:43:03] ENGINE Bus STARTED 2023-10-09T05:43:04.022 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[160468]: [09/Oct/2023:05:43:03] ENGINE Bus STOPPING 2023-10-09T05:43:04.022 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[160468]: [09/Oct/2023:05:43:03] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:43:04.022 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[160468]: [09/Oct/2023:05:43:03] ENGINE Bus STOPPED 2023-10-09T05:43:04.022 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[160468]: [09/Oct/2023:05:43:03] ENGINE Bus STARTING 2023-10-09T05:43:04.022 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:04 smithi156 conmon[160468]: [09/Oct/2023:05:43:04] ENGINE Serving on http://:::9283 2023-10-09T05:43:04.022 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:04 smithi156 conmon[160468]: [09/Oct/2023:05:43:04] ENGINE Bus STARTED 2023-10-09T05:43:04.023 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 bash[175378]: 365834e48ffb84a2d21b6af747a8343a0bdbfd78c5c7b4b33b418c4fc093050d 2023-10-09T05:43:04.023 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 systemd[1]: Started Ceph grafana.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:43:04.023 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=settings t=2023-10-09T05:43:03.832989453Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-10-09T05:43:04.023 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=settings t=2023-10-09T05:43:03.833192854Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-10-09T05:43:04.023 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=settings t=2023-10-09T05:43:03.833212822Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-10-09T05:43:04.023 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=settings t=2023-10-09T05:43:03.833222241Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-10-09T05:43:04.023 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=settings t=2023-10-09T05:43:03.833228357Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-09T05:43:04.023 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=settings t=2023-10-09T05:43:03.83323449Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-09T05:43:04.023 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=settings t=2023-10-09T05:43:03.833240199Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-09T05:43:04.023 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=settings t=2023-10-09T05:43:03.833246092Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-09T05:43:04.023 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=settings t=2023-10-09T05:43:03.833252003Z level=info msg="Path Home" path=/usr/share/grafana 2023-10-09T05:43:04.024 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=settings t=2023-10-09T05:43:03.833257638Z level=info msg="Path Data" path=/var/lib/grafana 2023-10-09T05:43:04.024 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=settings t=2023-10-09T05:43:03.833263082Z level=info msg="Path Logs" path=/var/log/grafana 2023-10-09T05:43:04.024 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=settings t=2023-10-09T05:43:03.833268509Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-10-09T05:43:04.024 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=settings t=2023-10-09T05:43:03.833274272Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-10-09T05:43:04.024 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=settings t=2023-10-09T05:43:03.833280015Z level=info msg="App mode production" 2023-10-09T05:43:04.024 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=sqlstore t=2023-10-09T05:43:03.833342041Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-10-09T05:43:04.024 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=sqlstore t=2023-10-09T05:43:03.833376578Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-09T05:43:04.024 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=migrator t=2023-10-09T05:43:03.847250918Z level=info msg="Starting DB migrations" 2023-10-09T05:43:04.024 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=migrator t=2023-10-09T05:43:03.851815663Z level=info msg="migrations completed" performed=0 skipped=481 duration=382.273µs 2023-10-09T05:43:04.024 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=sqlstore t=2023-10-09T05:43:03.852179064Z level=info msg="Created default organization" 2023-10-09T05:43:04.024 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=plugin.loader t=2023-10-09T05:43:03.913209814Z level=info msg="Plugin registered" pluginID=input 2023-10-09T05:43:04.025 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=plugin.loader t=2023-10-09T05:43:03.941989335Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-10-09T05:43:04.025 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=plugin.loader t=2023-10-09T05:43:03.942025027Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-10-09T05:43:04.025 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=secrets t=2023-10-09T05:43:03.942084672Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-10-09T05:43:04.025 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=query_data t=2023-10-09T05:43:03.943808697Z level=info msg="Query Service initialization" 2023-10-09T05:43:04.025 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:03 smithi156 conmon[175419]: logger=live.push_http t=2023-10-09T05:43:03.94608676Z level=info msg="Live Push Gateway initialization" 2023-10-09T05:43:04.324 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:04 smithi156 conmon[160468]: [09/Oct/2023:05:43:04] ENGINE Bus STOPPING 2023-10-09T05:43:04.324 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:04 smithi156 conmon[160468]: [09/Oct/2023:05:43:04] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:43:04.324 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:04 smithi156 conmon[160468]: [09/Oct/2023:05:43:04] ENGINE Bus STOPPED 2023-10-09T05:43:04.324 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:04 smithi156 conmon[160468]: [09/Oct/2023:05:43:04] ENGINE Bus STARTING 2023-10-09T05:43:04.324 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:04 smithi156 conmon[160468]: [09/Oct/2023:05:43:04] ENGINE Serving on http://:::9283 2023-10-09T05:43:04.325 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:04 smithi156 conmon[160468]: [09/Oct/2023:05:43:04] ENGINE Bus STARTED 2023-10-09T05:43:04.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:04 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/4046731049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:04.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:04 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:04.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:04 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:04.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:04 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:43:04.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:04 smithi156 ceph-mon[105979]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:43:04.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:04 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi064.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T05:43:04.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:04 smithi156 ceph-mon[105979]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi064.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T05:43:04.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:04 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:04.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:04 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:43:04.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:04 smithi156 ceph-mon[105979]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:43:04.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:04 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi156:9095"}]: dispatch 2023-10-09T05:43:04.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:04 smithi156 ceph-mon[105979]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi156:9095"}]: dispatch 2023-10-09T05:43:04.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:04 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:04.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:04 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:43:04.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:04 smithi156 ceph-mon[105979]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:43:04.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:04 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi156:3000"}]: dispatch 2023-10-09T05:43:04.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:04 smithi156 ceph-mon[105979]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi156:3000"}]: dispatch 2023-10-09T05:43:04.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:04 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:04.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:04 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:04.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:04 smithi156 ceph-mon[105979]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T05:43:04.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:04 smithi156 ceph-mon[105979]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T05:43:04.637 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/4046731049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:04.637 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:04.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:04.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:43:04.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[101070]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:43:04.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi064.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T05:43:04.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[101070]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi064.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T05:43:04.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:04.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:43:04.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[101070]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:43:04.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi156:9095"}]: dispatch 2023-10-09T05:43:04.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[101070]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi156:9095"}]: dispatch 2023-10-09T05:43:04.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:04.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:43:04.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[101070]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:43:04.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi156:3000"}]: dispatch 2023-10-09T05:43:04.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[101070]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi156:3000"}]: dispatch 2023-10-09T05:43:04.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:04.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:04.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[101070]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T05:43:04.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[101070]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T05:43:04.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/4046731049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:04.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:04.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:04.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:43:04.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[118354]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:43:04.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi064.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T05:43:04.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[118354]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi064.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T05:43:04.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:04.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:43:04.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[118354]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:43:04.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi156:9095"}]: dispatch 2023-10-09T05:43:04.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[118354]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi156:9095"}]: dispatch 2023-10-09T05:43:04.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:04.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:43:04.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[118354]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:43:04.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi156:3000"}]: dispatch 2023-10-09T05:43:04.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[118354]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi156:3000"}]: dispatch 2023-10-09T05:43:04.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:04.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:04.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[118354]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T05:43:04.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:04 smithi064 ceph-mon[118354]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T05:43:04.958 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:04 smithi156 conmon[175419]: logger=infra.usagestats.collector t=2023-10-09T05:43:04.673326212Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-10-09T05:43:04.958 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:04 smithi156 conmon[175419]: logger=provisioning.datasources t=2023-10-09T05:43:04.82184965Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2023-10-09T05:43:04.958 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:04 smithi156 conmon[175419]: logger=provisioning.datasources t=2023-10-09T05:43:04.821872975Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2023-10-09T05:43:04.958 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:04 smithi156 conmon[175419]: logger=provisioning.datasources t=2023-10-09T05:43:04.832156315Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-09T05:43:05.328 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:04 smithi156 conmon[175419]: logger=provisioning.alerting t=2023-10-09T05:43:04.957585372Z level=info msg="starting to provision alerting" 2023-10-09T05:43:05.328 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:04 smithi156 conmon[175419]: logger=provisioning.alerting t=2023-10-09T05:43:04.957625446Z level=info msg="finished to provision alerting" 2023-10-09T05:43:05.328 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:04 smithi156 conmon[175419]: logger=grafanaStorageLogger t=2023-10-09T05:43:04.957825529Z level=info msg="storage starting" 2023-10-09T05:43:05.328 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:04 smithi156 conmon[175419]: logger=ngalert.state.manager t=2023-10-09T05:43:04.957975792Z level=info msg="Warming state cache for startup" 2023-10-09T05:43:05.328 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:04 smithi156 conmon[175419]: logger=http.server t=2023-10-09T05:43:04.959787052Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-10-09T05:43:05.329 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:05 smithi156 conmon[175419]: logger=ngalert.state.manager t=2023-10-09T05:43:05.005502189Z level=info msg="State cache has been initialized" states=0 duration=47.525689ms 2023-10-09T05:43:05.329 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:05 smithi156 conmon[175419]: logger=ticker t=2023-10-09T05:43:05.005574982Z level=info msg=starting first_tick=2023-10-09T05:43:10Z 2023-10-09T05:43:05.329 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:43:05 smithi156 conmon[175419]: logger=ngalert.multiorg.alertmanager t=2023-10-09T05:43:05.005590438Z level=info msg="starting MultiOrg Alertmanager" 2023-10-09T05:43:05.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:05 smithi156 ceph-mon[105979]: pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:05.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:05 smithi064 ceph-mon[101070]: pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:05.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:05 smithi064 ceph-mon[118354]: pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:06.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:06 smithi064 ceph-mon[101070]: Upgrade: Updating mgr.y 2023-10-09T05:43:06.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:06 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:06.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:06 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:43:06.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:06 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:43:06.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:06 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:43:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:06 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:43:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:06 smithi064 ceph-mon[101070]: Deploying daemon mgr.y on smithi064 2023-10-09T05:43:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:06 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/59393576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:06.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:06 smithi064 ceph-mon[118354]: Upgrade: Updating mgr.y 2023-10-09T05:43:06.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:06 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:06.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:06 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:43:06.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:06 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:43:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:06 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:43:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:06 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:43:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:06 smithi064 ceph-mon[118354]: Deploying daemon mgr.y on smithi064 2023-10-09T05:43:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:06 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/59393576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:06.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:06 smithi156 ceph-mon[105979]: Upgrade: Updating mgr.y 2023-10-09T05:43:06.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:06 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:06.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:06 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:43:06.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:06 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:43:06.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:06 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:43:06.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:06 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:43:06.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:06 smithi156 ceph-mon[105979]: Deploying daemon mgr.y on smithi064 2023-10-09T05:43:06.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:06 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/59393576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:07.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:06 smithi064 systemd[1]: Stopping Ceph mgr.y for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:43:07.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:07 smithi064 ceph-mon[101070]: pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:07.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:07 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:07.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:07 smithi064 ceph-mon[118354]: pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:07.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:07 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:07.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:07 smithi156 ceph-mon[105979]: pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:07.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:07 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:08.076 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:07 smithi064 bash[198442]: 1f3d7bc3f2423c8c180ded78b0d0120e89ca42215efd12ce3b17c46bc7ebdeca 2023-10-09T05:43:08.076 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:07 smithi064 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-10-09T05:43:08.076 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:07 smithi064 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-10-09T05:43:08.076 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:07 smithi064 systemd[1]: Stopped Ceph mgr.y for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:43:08.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:08 smithi064 systemd[1]: Starting Ceph mgr.y for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:43:08.956 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:08 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/2578661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:08.956 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:08 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/2578661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:08.957 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:08 smithi064 ceph-mgr[198886]: -- 172.21.15.64:0/2160574120 <== mon.2 v2:172.21.15.64:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5577e7b551e0 con 0x5577e7b2c800 2023-10-09T05:43:08.957 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:08 smithi064 bash[198818]: 2bc7549ee2c14c1252103620ea10ed15dd121c366d423c95149400879aa0dce1 2023-10-09T05:43:08.957 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:08 smithi064 systemd[1]: Started Ceph mgr.y for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:43:08.957 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:08 smithi064 conmon[198859]: 2023-10-09T05:43:08.856+0000 7f4316829200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:43:08.957 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:08 smithi064 conmon[198859]: 2023-10-09T05:43:08.954+0000 7f4316829200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:43:09.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:08 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/2578661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:09.206 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:09 smithi064 conmon[198859]: 2023-10-09T05:43:09.071+0000 7f4316829200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:43:09.207 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:09 smithi064 conmon[198859]: 2023-10-09T05:43:09.205+0000 7f4316829200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:43:09.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:09 smithi064 conmon[198859]: 2023-10-09T05:43:09.437+0000 7f4316829200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:43:10.119 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:09 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:10.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:09 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:10.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:09 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:10.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:09 smithi064 ceph-mon[118354]: pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:10.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:09 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:10.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:09 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:10.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:09 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:10.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:09 smithi064 ceph-mon[101070]: pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:10.163 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:09 smithi064 conmon[198859]: 2023-10-09T05:43:09.957+0000 7f4316829200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:43:10.164 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:09 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:10.164 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:09 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:10.164 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:09 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:10.164 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:09 smithi156 ceph-mon[105979]: pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:10.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:10 smithi064 conmon[198859]: 2023-10-09T05:43:10.117+0000 7f4316829200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:43:10.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:10 smithi064 conmon[198859]: 2023-10-09T05:43:10.226+0000 7f4316829200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:43:10.470 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:10 smithi064 conmon[198859]: 2023-10-09T05:43:10.354+0000 7f4316829200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:43:11.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:10 smithi156 conmon[160468]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:43:10] "GET /metrics HTTP/1.1" 200 33796 "" "Prometheus/2.43.0" 2023-10-09T05:43:11.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:10 smithi064 conmon[198859]: 2023-10-09T05:43:10.944+0000 7f4316829200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:43:11.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:11 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/1350015515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:11.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:11 smithi064 ceph-mon[118354]: pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:11.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:11 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1350015515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:11.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:11 smithi064 ceph-mon[101070]: pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:11.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:11 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/1350015515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:11.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:11 smithi156 ceph-mon[105979]: pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:12.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:11 smithi064 conmon[198859]: 2023-10-09T05:43:11.789+0000 7f4316829200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:43:12.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:11 smithi064 conmon[198859]: 2023-10-09T05:43:11.894+0000 7f4316829200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:43:12.967 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:12 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:12.967 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:12 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:12.967 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:12 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:12.967 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:12 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:12.967 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:12 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:12.968 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:12 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:12.968 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:12 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:12.968 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:12 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:12.968 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:12 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:12.968 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:12 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:12.968 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:12 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:12.968 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:12 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:12.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:12 smithi064 conmon[198859]: 2023-10-09T05:43:12.937+0000 7f4316829200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:43:13.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:12 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:13.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:12 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:13.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:12 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:13.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:12 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:13.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:12 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:13.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:12 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:13.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:13 smithi064 conmon[198859]: 2023-10-09T05:43:13.063+0000 7f4316829200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:43:13.675 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:13 smithi064 conmon[198859]: 2023-10-09T05:43:13.309+0000 7f4316829200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:43:13.676 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:43:13 smithi064 conmon[197067]: ts=2023-10-09T05:43:13.508Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi064.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.64:8443: connect: connection refused" 2023-10-09T05:43:13.934 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:13 smithi064 ceph-mon[101070]: pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:13.934 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:13 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1674146554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:13.934 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:13 smithi064 ceph-mon[118354]: pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:13.934 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:13 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/1674146554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:13.935 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:13 smithi064 conmon[198859]: 2023-10-09T05:43:13.674+0000 7f4316829200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:43:14.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:13 smithi156 ceph-mon[105979]: pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:14.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:13 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/1674146554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:14.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:13 smithi064 conmon[198859]: 2023-10-09T05:43:13.933+0000 7f4316829200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:43:14.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:14 smithi064 conmon[198859]: 2023-10-09T05:43:14.141+0000 7f4316829200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:43:14.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:14 smithi064 conmon[198859]: 2023-10-09T05:43:14.494+0000 7f4316829200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:43:15.611 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:15 smithi064 ceph-mon[101070]: pgmap v22: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:15.612 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:15 smithi064 ceph-mon[118354]: pgmap v22: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:15.612 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:15 smithi064 conmon[198859]: 2023-10-09T05:43:15.363+0000 7f4316829200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:43:15.612 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:15 smithi064 conmon[198859]: 2023-10-09T05:43:15.534+0000 7f4316829200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:43:15.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:15 smithi156 ceph-mon[105979]: pgmap v22: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:15.940 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:15 smithi064 conmon[198859]: 2023-10-09T05:43:15.796+0000 7f4316829200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:43:16.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:15 smithi064 conmon[198859]: 2023-10-09T05:43:15.938+0000 7f4316829200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:43:16.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:16 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/4176035856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:16.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:16 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:16.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:16 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:16.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:16 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:43:16.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:16 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:43:16.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:16 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:16.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:16 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:16.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:16 smithi156 ceph-mon[105979]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T05:43:16.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:16 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T05:43:16.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:16 smithi156 ceph-mon[105979]: osdmap e78: 8 total, 8 up, 8 in 2023-10-09T05:43:16.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:16 smithi156 ceph-mon[105979]: from='mgr.24841 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T05:43:16.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:16 smithi156 ceph-mon[105979]: mgrmap e26: y(active, starting, since 0.00742096s) 2023-10-09T05:43:16.622 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:16 smithi156 conmon[160468]: 2023-10-09T05:43:16.307+0000 7fb05fdd9700 -1 mgr handle_mgr_map I was active but no longer am 2023-10-09T05:43:16.622 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:16 smithi156 conmon[160468]: ignoring --setuser ceph since I am not root 2023-10-09T05:43:16.622 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:16 smithi156 conmon[160468]: ignoring --setgroup ceph since I am not root 2023-10-09T05:43:16.622 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:16 smithi156 ceph-mgr[160492]: -- 172.21.15.156:0/18288350 <== mon.1 v2:172.21.15.156:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e840d5b1e0 con 0x55e840d32c00 2023-10-09T05:43:16.622 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:16 smithi156 conmon[160468]: 2023-10-09T05:43:16.475+0000 7f3fbc20a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:43:16.622 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:16 smithi156 conmon[160468]: 2023-10-09T05:43:16.554+0000 7f3fbc20a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:43:16.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:16 smithi064 conmon[198859]: 2023-10-09T05:43:16.226+0000 7f4316829200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:43:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:16 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/4176035856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:16 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:16 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:16 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:43:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:16 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:43:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:16 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:16 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:16 smithi064 ceph-mon[101070]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T05:43:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:16 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T05:43:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:16 smithi064 ceph-mon[101070]: osdmap e78: 8 total, 8 up, 8 in 2023-10-09T05:43:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:16 smithi064 ceph-mon[101070]: from='mgr.24841 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T05:43:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:16 smithi064 ceph-mon[101070]: mgrmap e26: y(active, starting, since 0.00742096s) 2023-10-09T05:43:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:16 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/4176035856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:16.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:16 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:16.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:16 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:16.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:16 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:43:16.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:16 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:43:16.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:16 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' 2023-10-09T05:43:16.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:16 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:16.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:16 smithi064 ceph-mon[118354]: from='mgr.24841 172.21.15.156:0/2653203134' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T05:43:16.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:16 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T05:43:16.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:16 smithi064 ceph-mon[118354]: osdmap e78: 8 total, 8 up, 8 in 2023-10-09T05:43:16.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:16 smithi064 ceph-mon[118354]: from='mgr.24841 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T05:43:16.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:16 smithi064 ceph-mon[118354]: mgrmap e26: y(active, starting, since 0.00742096s) 2023-10-09T05:43:16.910 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:16 smithi156 conmon[160468]: 2023-10-09T05:43:16.646+0000 7f3fbc20a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:43:16.911 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:16 smithi156 conmon[160468]: 2023-10-09T05:43:16.741+0000 7f3fbc20a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:43:17.275 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:16 smithi156 conmon[160468]: 2023-10-09T05:43:16.910+0000 7f3fbc20a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:43:17.584 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:17 smithi156 conmon[160468]: 2023-10-09T05:43:17.274+0000 7f3fbc20a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:43:17.584 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:17 smithi156 conmon[160468]: 2023-10-09T05:43:17.408+0000 7f3fbc20a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:43:17.584 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:17 smithi156 conmon[160468]: 2023-10-09T05:43:17.498+0000 7f3fbc20a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:43:17.584 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:17 smithi064 conmon[198859]: 2023-10-09T05:43:17.304+0000 7f4316829200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:43:17.584 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:17 smithi064 conmon[198859]: [09/Oct/2023:05:43:17] ENGINE Bus STARTING 2023-10-09T05:43:17.584 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:17 smithi064 conmon[198859]: CherryPy Checker: 2023-10-09T05:43:17.585 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:17 smithi064 conmon[198859]: The Application mounted at '' has an empty config. 2023-10-09T05:43:17.585 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:17 smithi064 conmon[198859]: 2023-10-09T05:43:17.585 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:17 smithi064 conmon[198859]: [09/Oct/2023:05:43:17] ENGINE Serving on http://:::9283 2023-10-09T05:43:17.585 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:17 smithi064 conmon[198859]: [09/Oct/2023:05:43:17] ENGINE Bus STARTED 2023-10-09T05:43:17.585 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:17 smithi064 conmon[198859]: [09/Oct/2023:05:43:17] ENGINE Bus STOPPING 2023-10-09T05:43:17.585 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:17 smithi064 conmon[198859]: [09/Oct/2023:05:43:17] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:43:17.585 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:17 smithi064 conmon[198859]: [09/Oct/2023:05:43:17] ENGINE Bus STOPPED 2023-10-09T05:43:17.585 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:17 smithi064 conmon[198859]: [09/Oct/2023:05:43:17] ENGINE Bus STARTING 2023-10-09T05:43:17.585 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: Active manager daemon y restarted 2023-10-09T05:43:17.586 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: Activating manager daemon y 2023-10-09T05:43:17.586 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: osdmap e79: 8 total, 8 up, 8 in 2023-10-09T05:43:17.586 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: mgrmap e27: y(active, starting, since 0.00520766s) 2023-10-09T05:43:17.586 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T05:43:17.586 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T05:43:17.586 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:43:17.586 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:43:17.586 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:43:17.586 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.gmrwsy"}]: dispatch 2023-10-09T05:43:17.586 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.pizbdw"}]: dispatch 2023-10-09T05:43:17.586 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:43:17.587 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:43:17.587 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:43:17.587 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:43:17.587 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:43:17.587 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:43:17.587 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:43:17.587 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:43:17.587 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:43:17.587 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:43:17.587 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:43:17.587 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:43:17.588 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:43:17.588 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T05:43:17.588 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:17 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:43:17.588 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: Active manager daemon y restarted 2023-10-09T05:43:17.588 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: Activating manager daemon y 2023-10-09T05:43:17.588 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: osdmap e79: 8 total, 8 up, 8 in 2023-10-09T05:43:17.588 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: mgrmap e27: y(active, starting, since 0.00520766s) 2023-10-09T05:43:17.588 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T05:43:17.588 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T05:43:17.588 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:43:17.588 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:43:17.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:43:17.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.gmrwsy"}]: dispatch 2023-10-09T05:43:17.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.pizbdw"}]: dispatch 2023-10-09T05:43:17.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:43:17.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:43:17.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:43:17.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:43:17.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:43:17.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:43:17.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:43:17.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:43:17.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:43:17.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:43:17.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:43:17.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:43:17.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:43:17.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T05:43:17.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:43:17.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: Active manager daemon y restarted 2023-10-09T05:43:17.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: Activating manager daemon y 2023-10-09T05:43:17.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: osdmap e79: 8 total, 8 up, 8 in 2023-10-09T05:43:17.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: mgrmap e27: y(active, starting, since 0.00520766s) 2023-10-09T05:43:17.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T05:43:17.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T05:43:17.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:43:17.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:43:17.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:43:17.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.gmrwsy"}]: dispatch 2023-10-09T05:43:17.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.pizbdw"}]: dispatch 2023-10-09T05:43:17.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:43:17.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:43:17.592 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:43:17.592 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:43:17.592 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:43:17.592 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:43:17.592 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:43:17.592 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:43:17.592 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:43:17.592 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:43:17.592 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:43:17.592 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:43:17.592 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:43:17.592 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T05:43:17.593 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:17 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:43:17.870 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:17 smithi156 conmon[160468]: 2023-10-09T05:43:17.603+0000 7f3fbc20a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:43:17.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:17 smithi064 conmon[198859]: 2023-10-09T05:43:17.589+0000 7f4273c9a700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:43:17.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:17 smithi064 conmon[198859]: 2023-10-09T05:43:17.589+0000 7f4273c9a700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:43:17.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:17 smithi064 conmon[198859]: 2023-10-09T05:43:17.589+0000 7f4273c9a700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:43:17.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:17 smithi064 conmon[198859]: 2023-10-09T05:43:17.589+0000 7f4273c9a700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:43:17.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:17 smithi064 conmon[198859]: 2023-10-09T05:43:17.589+0000 7f4273c9a700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:43:17.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:17 smithi064 conmon[198859]: 2023-10-09T05:43:17.589+0000 7f4270c94700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:43:17.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:17 smithi064 conmon[198859]: 2023-10-09T05:43:17.589+0000 7f4270c94700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:43:17.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:17 smithi064 conmon[198859]: 2023-10-09T05:43:17.589+0000 7f4270c94700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:43:17.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:17 smithi064 conmon[198859]: 2023-10-09T05:43:17.589+0000 7f4270c94700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:43:17.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:17 smithi064 conmon[198859]: 2023-10-09T05:43:17.589+0000 7f4270c94700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:43:17.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:17 smithi064 conmon[198859]: [09/Oct/2023:05:43:17] ENGINE Serving on http://:::9283 2023-10-09T05:43:17.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:17 smithi064 conmon[198859]: [09/Oct/2023:05:43:17] ENGINE Bus STARTED 2023-10-09T05:43:18.370 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:18 smithi156 conmon[160468]: 2023-10-09T05:43:18.042+0000 7f3fbc20a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:43:18.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:18 smithi064 ceph-mon[101070]: Manager daemon y is now available 2023-10-09T05:43:18.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:18 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:18.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:18 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:18.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:18 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:18.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:18 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:43:18.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:18 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:43:18.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:18 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/2636862876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:18.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:18 smithi064 ceph-mon[118354]: Manager daemon y is now available 2023-10-09T05:43:18.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:18 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:18.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:18 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:18.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:18 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:18.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:18 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:43:18.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:18 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:43:18.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:18 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/2636862876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:18.774 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:18 smithi156 ceph-mon[105979]: Manager daemon y is now available 2023-10-09T05:43:18.774 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:18 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:18.774 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:18 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:18.774 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:18 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:18.774 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:18 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:43:18.774 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:18 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:43:18.774 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:18 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/2636862876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:18.774 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:18 smithi156 conmon[173982]: ts=2023-10-09T05:43:18.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:43:19.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:18 smithi156 conmon[160468]: 2023-10-09T05:43:18.772+0000 7f3fbc20a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:43:19.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:18 smithi156 conmon[160468]: 2023-10-09T05:43:18.875+0000 7f3fbc20a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:43:19.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:19 smithi064 ceph-mon[118354]: mgrmap e28: y(active, since 1.13631s) 2023-10-09T05:43:19.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:19 smithi064 ceph-mon[118354]: pgmap v3: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:19.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:19 smithi064 ceph-mon[101070]: mgrmap e28: y(active, since 1.13631s) 2023-10-09T05:43:19.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:19 smithi064 ceph-mon[101070]: pgmap v3: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:19.756 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:19 smithi156 ceph-mon[105979]: mgrmap e28: y(active, since 1.13631s) 2023-10-09T05:43:19.756 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:19 smithi156 ceph-mon[105979]: pgmap v3: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:20.047 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:19 smithi156 conmon[160468]: 2023-10-09T05:43:19.755+0000 7f3fbc20a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:43:20.047 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:19 smithi156 conmon[160468]: 2023-10-09T05:43:19.856+0000 7f3fbc20a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:43:20.219 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 05:43:19 smithi064 conmon[197067]: ts=2023-10-09T05:43:19.964Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=6 2023-10-09T05:43:20.369 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:20 smithi156 conmon[160468]: 2023-10-09T05:43:20.046+0000 7f3fbc20a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:43:20.620 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:20 smithi156 conmon[160468]: 2023-10-09T05:43:20.368+0000 7f3fbc20a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:43:20.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:20 smithi156 ceph-mon[105979]: [09/Oct/2023:05:43:19] ENGINE Bus STARTING 2023-10-09T05:43:20.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:20 smithi156 ceph-mon[105979]: [09/Oct/2023:05:43:19] ENGINE Serving on http://172.21.15.64:8765 2023-10-09T05:43:20.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:20 smithi156 ceph-mon[105979]: pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:20.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:20 smithi156 ceph-mon[105979]: [09/Oct/2023:05:43:19] ENGINE Serving on https://172.21.15.64:7150 2023-10-09T05:43:20.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:20 smithi156 ceph-mon[105979]: [09/Oct/2023:05:43:19] ENGINE Bus STARTED 2023-10-09T05:43:20.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:20 smithi156 ceph-mon[105979]: mgrmap e29: y(active, since 2s) 2023-10-09T05:43:20.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:20 smithi064 ceph-mon[101070]: [09/Oct/2023:05:43:19] ENGINE Bus STARTING 2023-10-09T05:43:20.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:20 smithi064 ceph-mon[101070]: [09/Oct/2023:05:43:19] ENGINE Serving on http://172.21.15.64:8765 2023-10-09T05:43:20.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:20 smithi064 ceph-mon[101070]: pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:20.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:20 smithi064 ceph-mon[101070]: [09/Oct/2023:05:43:19] ENGINE Serving on https://172.21.15.64:7150 2023-10-09T05:43:20.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:20 smithi064 ceph-mon[101070]: [09/Oct/2023:05:43:19] ENGINE Bus STARTED 2023-10-09T05:43:20.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:20 smithi064 ceph-mon[101070]: mgrmap e29: y(active, since 2s) 2023-10-09T05:43:20.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:20 smithi064 ceph-mon[118354]: [09/Oct/2023:05:43:19] ENGINE Bus STARTING 2023-10-09T05:43:20.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:20 smithi064 ceph-mon[118354]: [09/Oct/2023:05:43:19] ENGINE Serving on http://172.21.15.64:8765 2023-10-09T05:43:20.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:20 smithi064 ceph-mon[118354]: pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:20.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:20 smithi064 ceph-mon[118354]: [09/Oct/2023:05:43:19] ENGINE Serving on https://172.21.15.64:7150 2023-10-09T05:43:20.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:20 smithi064 ceph-mon[118354]: [09/Oct/2023:05:43:19] ENGINE Bus STARTED 2023-10-09T05:43:20.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:20 smithi064 ceph-mon[118354]: mgrmap e29: y(active, since 2s) 2023-10-09T05:43:21.077 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:20 smithi156 conmon[160468]: 2023-10-09T05:43:20.619+0000 7f3fbc20a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:43:21.077 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:20 smithi156 conmon[160468]: 2023-10-09T05:43:20.782+0000 7f3fbc20a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:43:21.370 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:21 smithi156 conmon[160468]: 2023-10-09T05:43:21.076+0000 7f3fbc20a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:43:21.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:21 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/95869872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:21.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:21 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:21.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:21 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:21.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:21 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:21.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:21 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:21.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:21 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/95869872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:21.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:21 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:21.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:21 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:21.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:21 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:21.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:21 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:21.844 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:21 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/95869872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:21.844 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:21 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:21.844 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:21 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:21.844 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:21 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:21.844 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:21 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:22.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:21 smithi156 conmon[160468]: 2023-10-09T05:43:21.843+0000 7f3fbc20a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:43:22.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:21 smithi156 conmon[160468]: 2023-10-09T05:43:21.979+0000 7f3fbc20a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:43:22.513 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:22 smithi156 conmon[160468]: 2023-10-09T05:43:22.176+0000 7f3fbc20a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:43:22.513 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:22 smithi156 conmon[160468]: 2023-10-09T05:43:22.292+0000 7f3fbc20a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:43:22.870 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:22 smithi156 conmon[160468]: 2023-10-09T05:43:22.513+0000 7f3fbc20a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:43:22.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:22 smithi156 ceph-mon[105979]: pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:22.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:22 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:22.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:22 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:22.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:22 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:22.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:22 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:22.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:22 smithi064 ceph-mon[101070]: pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:22.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:22 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:22.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:22 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:22.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:22 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:22 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:22.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:22 smithi064 ceph-mon[118354]: pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:22.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:22 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:22.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:22 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:22.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:22 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:22.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:22 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:23.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:23 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/1206833151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:23.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:23 smithi064 ceph-mon[118354]: Standby manager daemon x started 2023-10-09T05:43:23.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:23 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.156:0/3900128658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:43:23.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:23 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.156:0/3900128658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:43:23.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:23 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.156:0/3900128658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:43:23.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:23 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.156:0/3900128658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:43:23.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:23 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1206833151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:23.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:23 smithi064 ceph-mon[101070]: Standby manager daemon x started 2023-10-09T05:43:23.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:23 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.156:0/3900128658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:43:23.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:23 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.156:0/3900128658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:43:23.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:23 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.156:0/3900128658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:43:23.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:23 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.156:0/3900128658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:43:23.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:23 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/1206833151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:23.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:23 smithi156 ceph-mon[105979]: Standby manager daemon x started 2023-10-09T05:43:23.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:23 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.156:0/3900128658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:43:23.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:23 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.156:0/3900128658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:43:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:23 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.156:0/3900128658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:43:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:23 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.156:0/3900128658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:43:23.871 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:23 smithi156 conmon[160468]: 2023-10-09T05:43:23.386+0000 7f3fbc20a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:43:23.871 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:23 smithi156 conmon[160468]: [09/Oct/2023:05:43:23] ENGINE Bus STARTING 2023-10-09T05:43:23.871 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:23 smithi156 conmon[160468]: CherryPy Checker: 2023-10-09T05:43:23.871 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:23 smithi156 conmon[160468]: The Application mounted at '' has an empty config. 2023-10-09T05:43:23.871 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:23 smithi156 conmon[160468]: 2023-10-09T05:43:23.871 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:23 smithi156 conmon[160468]: [09/Oct/2023:05:43:23] ENGINE Serving on http://:::9283 2023-10-09T05:43:23.871 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:23 smithi156 conmon[160468]: [09/Oct/2023:05:43:23] ENGINE Bus STARTED 2023-10-09T05:43:24.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:24 smithi064 ceph-mon[101070]: pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:24.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:24 smithi064 ceph-mon[101070]: mgrmap e30: y(active, since 6s), standbys: x 2023-10-09T05:43:24.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:24 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:43:24.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:24 smithi064 ceph-mon[118354]: pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:24.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:24 smithi064 ceph-mon[118354]: mgrmap e30: y(active, since 6s), standbys: x 2023-10-09T05:43:24.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:24 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:43:24.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:24 smithi156 ceph-mon[105979]: pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:24.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:24 smithi156 ceph-mon[105979]: mgrmap e30: y(active, since 6s), standbys: x 2023-10-09T05:43:24.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:24 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:43:26.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:26 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:26.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:26 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:26.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:26 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:43:26.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:26 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:43:26.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:26 smithi064 ceph-mon[101070]: Updating smithi064:/etc/ceph/ceph.conf 2023-10-09T05:43:26.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:26 smithi064 ceph-mon[101070]: Updating smithi156:/etc/ceph/ceph.conf 2023-10-09T05:43:26.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:26 smithi064 ceph-mon[101070]: pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:26.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:26 smithi064 ceph-mon[101070]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:43:26.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:26 smithi064 ceph-mon[101070]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:43:26.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:26 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/453448689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:26.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:26 smithi064 ceph-mon[101070]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:43:26.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:26 smithi064 ceph-mon[101070]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:43:26.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:26 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:26.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:26 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:26.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:26 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:43:26.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:26 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:43:26.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:26 smithi064 ceph-mon[118354]: Updating smithi064:/etc/ceph/ceph.conf 2023-10-09T05:43:26.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:26 smithi064 ceph-mon[118354]: Updating smithi156:/etc/ceph/ceph.conf 2023-10-09T05:43:26.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:26 smithi064 ceph-mon[118354]: pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:26.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:26 smithi064 ceph-mon[118354]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:43:26.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:26 smithi064 ceph-mon[118354]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:43:26.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:26 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/453448689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:26.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:26 smithi064 ceph-mon[118354]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:43:26.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:26 smithi064 ceph-mon[118354]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:43:26.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:26 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:26.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:26 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:26.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:26 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:43:26.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:26 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:43:26.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:26 smithi156 ceph-mon[105979]: Updating smithi064:/etc/ceph/ceph.conf 2023-10-09T05:43:26.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:26 smithi156 ceph-mon[105979]: Updating smithi156:/etc/ceph/ceph.conf 2023-10-09T05:43:26.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:26 smithi156 ceph-mon[105979]: pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:26.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:26 smithi156 ceph-mon[105979]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:43:26.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:26 smithi156 ceph-mon[105979]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:43:26.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:26 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/453448689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:26.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:26 smithi156 ceph-mon[105979]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:43:26.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:26 smithi156 ceph-mon[105979]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:43:27.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:27 smithi156 ceph-mon[105979]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:43:27.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:27 smithi156 ceph-mon[105979]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:43:27.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:27 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:27.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:27 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:27.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:27 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:27.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:27 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:27.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:27 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:27.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:27 smithi156 ceph-mon[105979]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T05:43:27.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:27 smithi156 ceph-mon[105979]: Reconfiguring daemon prometheus.a on smithi156 2023-10-09T05:43:27.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:27 smithi064 ceph-mon[101070]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:43:27.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:27 smithi064 ceph-mon[101070]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:43:27.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:27 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:27.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:27 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:27.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:27 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:27.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:27 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:27.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:27 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:27.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:27 smithi064 ceph-mon[101070]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T05:43:27.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:27 smithi064 ceph-mon[101070]: Reconfiguring daemon prometheus.a on smithi156 2023-10-09T05:43:27.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:27 smithi064 ceph-mon[118354]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:43:27.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:27 smithi064 ceph-mon[118354]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:43:27.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:27 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:27.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:27 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:27.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:27 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:27.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:27 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:27.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:27 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:27.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:27 smithi064 ceph-mon[118354]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T05:43:27.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:27 smithi064 ceph-mon[118354]: Reconfiguring daemon prometheus.a on smithi156 2023-10-09T05:43:28.246 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:28 smithi156 ceph-mon[105979]: pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:28.246 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:28 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/799293857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:28.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:28 smithi064 ceph-mon[118354]: pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:28.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:28 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/799293857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:28.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:28 smithi064 ceph-mon[101070]: pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:28.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:28 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/799293857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:29.271 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:29 smithi156 systemd[1]: Stopping Ceph prometheus.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:43:29.620 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:29 smithi156 conmon[173982]: ts=2023-10-09T05:43:29.270Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T05:43:29.620 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:29 smithi156 conmon[173982]: ts=2023-10-09T05:43:29.271Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-10-09T05:43:29.621 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:29 smithi156 conmon[173982]: ts=2023-10-09T05:43:29.271Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-10-09T05:43:29.621 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:29 smithi156 conmon[173982]: ts=2023-10-09T05:43:29.271Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T05:43:29.621 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:29 smithi156 conmon[173982]: ts=2023-10-09T05:43:29.271Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-10-09T05:43:29.621 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:29 smithi156 conmon[173982]: ts=2023-10-09T05:43:29.271Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T05:43:29.621 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:29 smithi156 conmon[173982]: ts=2023-10-09T05:43:29.271Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-10-09T05:43:29.621 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:29 smithi156 conmon[173982]: ts=2023-10-09T05:43:29.271Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-10-09T05:43:29.621 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:29 smithi156 conmon[173982]: ts=2023-10-09T05:43:29.271Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-10-09T05:43:29.621 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:29 smithi156 conmon[173982]: ts=2023-10-09T05:43:29.272Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T05:43:29.621 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:29 smithi156 conmon[173982]: ts=2023-10-09T05:43:29.272Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-10-09T05:43:29.621 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:29 smithi156 conmon[173982]: ts=2023-10-09T05:43:29.272Z caller=main.go:1147 level=info msg="See you next time!" 2023-10-09T05:43:30.120 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:29 smithi156 bash[179414]: 09f4a99cecde4dfb4b12caf287185d9881feb3dfbb641d13ad9a2b176f62b3c2 2023-10-09T05:43:30.121 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:29 smithi156 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T05:43:30.121 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:29 smithi156 systemd[1]: Stopped Ceph prometheus.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:43:30.121 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:29 smithi156 systemd[1]: Starting Ceph prometheus.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:43:30.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:30 smithi064 ceph-mon[101070]: pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:30.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:30 smithi064 ceph-mon[118354]: pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:30.689 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:30 smithi064 conmon[198859]: [09/Oct/2023:05:43:30] ENGINE Bus STOPPING 2023-10-09T05:43:30.689 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:30 smithi064 conmon[198859]: [09/Oct/2023:05:43:30] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:43:30.689 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:30 smithi064 conmon[198859]: [09/Oct/2023:05:43:30] ENGINE Bus STOPPED 2023-10-09T05:43:30.689 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:30 smithi064 conmon[198859]: [09/Oct/2023:05:43:30] ENGINE Bus STARTING 2023-10-09T05:43:30.689 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:30 smithi064 conmon[198859]: [09/Oct/2023:05:43:30] ENGINE Serving on http://:::9283 2023-10-09T05:43:30.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:30 smithi156 ceph-mon[105979]: pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:30.871 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 bash[179742]: 712dad4d0e3b982c758749e31ce4510c395baba943d49bc0fe9caa8ab00b46cf 2023-10-09T05:43:30.871 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 conmon[179783]: ts=2023-10-09T05:43:30.495Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-10-09T05:43:30.871 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 conmon[179783]: ts=2023-10-09T05:43:30.495Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-10-09T05:43:30.871 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 conmon[179783]: ts=2023-10-09T05:43:30.495Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi156 (none))" 2023-10-09T05:43:30.871 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 conmon[179783]: ts=2023-10-09T05:43:30.495Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T05:43:30.871 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 conmon[179783]: ts=2023-10-09T05:43:30.495Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T05:43:30.871 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 conmon[179783]: ts=2023-10-09T05:43:30.497Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T05:43:30.871 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 conmon[179783]: ts=2023-10-09T05:43:30.497Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-10-09T05:43:30.871 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 conmon[179783]: ts=2023-10-09T05:43:30.499Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-10-09T05:43:30.871 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 conmon[179783]: ts=2023-10-09T05:43:30.499Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-10-09T05:43:30.871 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 conmon[179783]: ts=2023-10-09T05:43:30.501Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T05:43:30.872 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 conmon[179783]: ts=2023-10-09T05:43:30.501Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.016µs 2023-10-09T05:43:30.872 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 conmon[179783]: ts=2023-10-09T05:43:30.501Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T05:43:30.872 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 conmon[179783]: ts=2023-10-09T05:43:30.501Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-10-09T05:43:30.872 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 systemd[1]: Started Ceph prometheus.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:43:30.872 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 conmon[179783]: ts=2023-10-09T05:43:30.518Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-10-09T05:43:30.872 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 conmon[179783]: ts=2023-10-09T05:43:30.520Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-10-09T05:43:30.872 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 conmon[179783]: ts=2023-10-09T05:43:30.520Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-10-09T05:43:30.872 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 conmon[179783]: ts=2023-10-09T05:43:30.520Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=33.447µs wal_replay_duration=19.389506ms wbl_replay_duration=131ns total_replay_duration=19.451322ms 2023-10-09T05:43:30.872 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 conmon[179783]: ts=2023-10-09T05:43:30.523Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T05:43:30.872 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 conmon[179783]: ts=2023-10-09T05:43:30.523Z caller=main.go:1029 level=info msg="TSDB started" 2023-10-09T05:43:30.872 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 conmon[179783]: ts=2023-10-09T05:43:30.524Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T05:43:30.873 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 conmon[179783]: ts=2023-10-09T05:43:30.539Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.800993ms db_storage=1.325µs remote_storage=1.654µs web_handler=590ns query_engine=1.095µs scrape=778.841µs scrape_sd=143.54µs notify=25.889µs notify_sd=28.049µs rules=14.34399ms tracing=6.923µs 2023-10-09T05:43:30.873 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 conmon[179783]: ts=2023-10-09T05:43:30.540Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-10-09T05:43:30.873 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:30 smithi156 conmon[179783]: ts=2023-10-09T05:43:30.540Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-10-09T05:43:30.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:30 smithi064 conmon[198859]: [09/Oct/2023:05:43:30] ENGINE Bus STARTED 2023-10-09T05:43:31.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:31 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/2444751251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:31.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:31 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:31.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:31 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:31.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:31 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:43:31.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:31 smithi156 ceph-mon[105979]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:43:31.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:31 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi156.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:43:31.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:31 smithi156 ceph-mon[105979]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi156.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:43:31.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:31 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:31.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:31 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:31.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:31 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/2444751251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:31.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:31 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:31.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:31 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:31.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:31 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:43:31.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:31 smithi064 ceph-mon[101070]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:43:31.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:31 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi156.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:43:31.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:31 smithi064 ceph-mon[101070]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi156.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:43:31.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:31 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:31.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:31 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:31.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:31 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/2444751251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:31.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:31 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:31.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:31 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:31.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:31 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:43:31.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:31 smithi064 ceph-mon[118354]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:43:31.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:31 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi156.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:43:31.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:31 smithi064 ceph-mon[118354]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi156.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:43:31.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:31 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:31.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:31 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:32.862 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:32 smithi156 ceph-mon[105979]: pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:32.862 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:32 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:43:32.862 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:32 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:43:32.862 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:32 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:43:32.862 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:32 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:32.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:32 smithi064 ceph-mon[101070]: pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:32.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:32 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:43:32.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:32 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:43:32.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:32 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:43:32.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:32 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:32.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:32 smithi064 ceph-mon[118354]: pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:32.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:32 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:43:32.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:32 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:43:32.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:32 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:43:32.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:32 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:33.579 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:33 smithi156 systemd[1]: Stopping Ceph mgr.x for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:43:33.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:33 smithi156 ceph-mon[105979]: Upgrade: Updating mgr.x 2023-10-09T05:43:33.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:33 smithi156 ceph-mon[105979]: Deploying daemon mgr.x on smithi156 2023-10-09T05:43:33.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:33 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/2047295474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:33.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:33 smithi064 ceph-mon[101070]: Upgrade: Updating mgr.x 2023-10-09T05:43:33.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:33 smithi064 ceph-mon[101070]: Deploying daemon mgr.x on smithi156 2023-10-09T05:43:33.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:33 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/2047295474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:33.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:33 smithi064 ceph-mon[118354]: Upgrade: Updating mgr.x 2023-10-09T05:43:33.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:33 smithi064 ceph-mon[118354]: Deploying daemon mgr.x on smithi156 2023-10-09T05:43:33.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:33 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/2047295474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:34.453 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:34 smithi156 bash[180418]: 5085268da26c896a58f83a3262c3a55517f9bb5ce5d893c5c34e2c11d55c44ae 2023-10-09T05:43:34.454 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:34 smithi156 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T05:43:34.454 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:34 smithi156 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T05:43:34.454 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:34 smithi156 systemd[1]: Stopped Ceph mgr.x for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:43:34.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:34 smithi156 ceph-mon[105979]: pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:34.870 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:34 smithi156 systemd[1]: Starting Ceph mgr.x for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:43:34.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:34 smithi064 ceph-mon[101070]: pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:34.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:34 smithi064 ceph-mon[118354]: pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:35.356 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:35 smithi156 ceph-mgr[180836]: -- 172.21.15.156:0/431226013 <== mon.1 v2:172.21.15.156:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e798acb1e0 con 0x55e798aa2800 2023-10-09T05:43:35.357 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:35 smithi156 bash[180771]: d0092a415024eed937d34e74943b54d858db4b0a65f4e4cea20d4368e794cc5d 2023-10-09T05:43:35.357 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:35 smithi156 systemd[1]: Started Ceph mgr.x for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:43:35.357 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:35 smithi156 conmon[180812]: 2023-10-09T05:43:35.172+0000 7f3d5be36200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:43:35.357 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:35 smithi156 conmon[180812]: 2023-10-09T05:43:35.256+0000 7f3d5be36200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:43:35.357 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:35 smithi156 conmon[180812]: 2023-10-09T05:43:35.355+0000 7f3d5be36200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:43:35.620 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:35 smithi156 conmon[180812]: 2023-10-09T05:43:35.470+0000 7f3d5be36200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:43:36.074 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:35 smithi156 conmon[180812]: 2023-10-09T05:43:35.671+0000 7f3d5be36200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:43:36.370 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:36 smithi156 conmon[180812]: 2023-10-09T05:43:36.073+0000 7f3d5be36200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:43:36.370 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:36 smithi156 conmon[180812]: 2023-10-09T05:43:36.220+0000 7f3d5be36200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:43:36.370 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:36 smithi156 conmon[180812]: 2023-10-09T05:43:36.315+0000 7f3d5be36200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:43:36.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:36 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:36.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:36 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:36.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:36 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:36.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:36 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/655543187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:36.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:36 smithi156 ceph-mon[105979]: pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:36.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:36 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:36.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:36 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:36.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:36 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:36.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:36 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/655543187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:36.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:36 smithi064 ceph-mon[101070]: pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:36.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:36 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:36.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:36 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:36.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:36 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:36.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:36 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/655543187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:36.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:36 smithi064 ceph-mon[118354]: pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:36.870 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:36 smithi156 conmon[180812]: 2023-10-09T05:43:36.417+0000 7f3d5be36200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:43:37.370 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:36 smithi156 conmon[180812]: 2023-10-09T05:43:36.908+0000 7f3d5be36200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:43:38.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:37 smithi156 conmon[180812]: 2023-10-09T05:43:37.639+0000 7f3d5be36200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:43:38.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:37 smithi156 conmon[180812]: 2023-10-09T05:43:37.735+0000 7f3d5be36200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:43:38.717 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:38 smithi156 conmon[180812]: 2023-10-09T05:43:38.617+0000 7f3d5be36200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:43:38.718 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:38 smithi156 ceph-mon[105979]: pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:38.718 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:38 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/2755319766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:38.718 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:38 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:38.718 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:38 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:38.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:38 smithi064 ceph-mon[101070]: pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:38.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:38 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/2755319766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:38.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:38 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:38.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:38 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:38.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:38 smithi064 ceph-mon[118354]: pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:38.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:38 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/2755319766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:38.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:38 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:38.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:38 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:39.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:38 smithi156 conmon[180812]: 2023-10-09T05:43:38.716+0000 7f3d5be36200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:43:39.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:38 smithi156 conmon[180812]: 2023-10-09T05:43:38.902+0000 7f3d5be36200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:43:39.477 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:39 smithi156 conmon[180812]: 2023-10-09T05:43:39.223+0000 7f3d5be36200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:43:39.771 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:39 smithi156 conmon[180812]: 2023-10-09T05:43:39.476+0000 7f3d5be36200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:43:39.771 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:39 smithi156 conmon[180812]: 2023-10-09T05:43:39.638+0000 7f3d5be36200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:43:40.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:39 smithi156 conmon[180812]: 2023-10-09T05:43:39.944+0000 7f3d5be36200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:43:40.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:40 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:40.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:40 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:40.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:40 smithi156 ceph-mon[105979]: pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:40.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:40 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:40.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:40 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:40.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:40 smithi064 ceph-mon[101070]: pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:40.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:40 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:40.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:40 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:40.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:40 smithi064 ceph-mon[118354]: pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:41.015 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:40 smithi156 conmon[180812]: 2023-10-09T05:43:40.703+0000 7f3d5be36200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:43:41.015 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:40 smithi156 conmon[180812]: 2023-10-09T05:43:40.847+0000 7f3d5be36200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:43:41.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:41 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/2424665713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:41.371 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:41 smithi156 conmon[180812]: 2023-10-09T05:43:41.063+0000 7f3d5be36200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:43:41.371 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:41 smithi156 conmon[180812]: 2023-10-09T05:43:41.191+0000 7f3d5be36200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:43:41.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:41 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/2424665713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:41.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:41 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/2424665713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:41.760 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:41 smithi156 conmon[180812]: 2023-10-09T05:43:41.427+0000 7f3d5be36200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:43:42.346 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:42 smithi156 ceph-mon[105979]: pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:42.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:42 smithi064 ceph-mon[101070]: pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:42.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:42 smithi064 ceph-mon[118354]: pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:42.620 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:42 smithi156 conmon[180812]: 2023-10-09T05:43:42.345+0000 7f3d5be36200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:43:42.621 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:42 smithi156 conmon[180812]: [09/Oct/2023:05:43:42] ENGINE Bus STARTING 2023-10-09T05:43:42.621 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:42 smithi156 conmon[180812]: CherryPy Checker: 2023-10-09T05:43:42.621 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:42 smithi156 conmon[180812]: The Application mounted at '' has an empty config. 2023-10-09T05:43:42.621 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:42 smithi156 conmon[180812]: 2023-10-09T05:43:42.621 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:42 smithi156 conmon[180812]: [09/Oct/2023:05:43:42] ENGINE Serving on http://:::9283 2023-10-09T05:43:42.621 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:42 smithi156 conmon[180812]: [09/Oct/2023:05:43:42] ENGINE Bus STARTED 2023-10-09T05:43:43.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:43.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:43.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:43:43.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:43:43.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:43.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[118354]: Standby manager daemon x restarted 2023-10-09T05:43:43.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[118354]: Standby manager daemon x started 2023-10-09T05:43:43.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.156:0/1436398995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:43:43.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.156:0/1436398995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:43:43.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.156:0/1436398995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:43:43.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.156:0/1436398995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:43:43.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:43.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:43:43.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[118354]: Upgrade: Setting container_image for all mgr 2023-10-09T05:43:43.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:43.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T05:43:43.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T05:43:43.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T05:43:43.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T05:43:43.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T05:43:43.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T05:43:43.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[118354]: Upgrade: It appears safe to stop mon.a 2023-10-09T05:43:43.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3593000523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:43.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:43.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:43.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:43:43.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:43:43.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:43.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[101070]: Standby manager daemon x restarted 2023-10-09T05:43:43.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[101070]: Standby manager daemon x started 2023-10-09T05:43:43.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.156:0/1436398995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:43:43.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.156:0/1436398995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:43:43.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.156:0/1436398995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:43:43.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[101070]: from='mgr.? 172.21.15.156:0/1436398995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:43:43.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:43.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:43:43.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[101070]: Upgrade: Setting container_image for all mgr 2023-10-09T05:43:43.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:43.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T05:43:43.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T05:43:43.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T05:43:43.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T05:43:43.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T05:43:43.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T05:43:43.474 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[101070]: Upgrade: It appears safe to stop mon.a 2023-10-09T05:43:43.474 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:43 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/3593000523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:43.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:43 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:43.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:43 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:43.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:43 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:43:43.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:43 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:43:43.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:43 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:43.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:43 smithi156 ceph-mon[105979]: Standby manager daemon x restarted 2023-10-09T05:43:43.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:43 smithi156 ceph-mon[105979]: Standby manager daemon x started 2023-10-09T05:43:43.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:43 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.156:0/1436398995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:43:43.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:43 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.156:0/1436398995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:43:43.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:43 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.156:0/1436398995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:43:43.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:43 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.156:0/1436398995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:43:43.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:43 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:43.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:43 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:43:43.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:43 smithi156 ceph-mon[105979]: Upgrade: Setting container_image for all mgr 2023-10-09T05:43:43.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:43 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:43.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:43 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T05:43:43.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:43 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T05:43:43.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:43 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T05:43:43.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:43 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T05:43:43.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:43 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T05:43:43.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:43 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T05:43:43.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:43 smithi156 ceph-mon[105979]: Upgrade: It appears safe to stop mon.a 2023-10-09T05:43:43.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:43 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3593000523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:44.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:44 smithi156 ceph-mon[105979]: mgrmap e31: y(active, since 25s), standbys: x 2023-10-09T05:43:44.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:44 smithi156 ceph-mon[105979]: pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:44.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:44 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:44.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:44 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:43:44.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:44 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:43:44.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:44 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:43:44.699 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:44 smithi064 ceph-mon[101070]: mgrmap e31: y(active, since 25s), standbys: x 2023-10-09T05:43:44.699 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:44 smithi064 ceph-mon[101070]: pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:44.699 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:44 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:44.699 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:44 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:43:44.699 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:44 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:43:44.699 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:44 smithi064 ceph-mon[101070]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:43:44.700 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:44 smithi064 ceph-mon[118354]: mgrmap e31: y(active, since 25s), standbys: x 2023-10-09T05:43:44.700 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:44 smithi064 ceph-mon[118354]: pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:44.700 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:44 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' 2023-10-09T05:43:44.700 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:44 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:43:44.700 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:44 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:43:44.700 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:44 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:43:45.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:45 smithi064 ceph-mon[101070]: Upgrade: Updating mon.a 2023-10-09T05:43:45.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:45 smithi064 ceph-mon[101070]: Deploying daemon mon.a on smithi064 2023-10-09T05:43:45.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:45 smithi064 ceph-mon[101070]: from='client.? 172.21.15.64:0/1766177366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:45.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:45 smithi064 systemd[1]: Stopping Ceph mon.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:43:45.471 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:43:45] "GET /metrics HTTP/1.1" 200 33800 "" "Prometheus/2.43.0" 2023-10-09T05:43:45.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:45 smithi064 ceph-mon[118354]: Upgrade: Updating mon.a 2023-10-09T05:43:45.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:45 smithi064 ceph-mon[118354]: Deploying daemon mon.a on smithi064 2023-10-09T05:43:45.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:45 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/1766177366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:45.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:45 smithi156 ceph-mon[105979]: Upgrade: Updating mon.a 2023-10-09T05:43:45.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:45 smithi156 ceph-mon[105979]: Deploying daemon mon.a on smithi064 2023-10-09T05:43:45.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:45 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/1766177366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:45.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:45 smithi064 conmon[101046]: 2023-10-09T05:43:45.516+0000 7fbeacfe9700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T05:43:45.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:45 smithi064 conmon[101046]: 2023-10-09T05:43:45.516+0000 7fbeacfe9700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-10-09T05:43:45.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:45 smithi064 conmon[198859]: [09/Oct/2023:05:43:45] ENGINE Bus STOPPING 2023-10-09T05:43:45.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:45 smithi064 conmon[198859]: [09/Oct/2023:05:43:45] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:43:45.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:45 smithi064 conmon[198859]: [09/Oct/2023:05:43:45] ENGINE Bus STOPPED 2023-10-09T05:43:45.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:45 smithi064 conmon[198859]: [09/Oct/2023:05:43:45] ENGINE Bus STARTING 2023-10-09T05:43:45.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:45 smithi064 conmon[198859]: [09/Oct/2023:05:43:45] ENGINE Serving on http://:::9283 2023-10-09T05:43:45.970 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:45 smithi064 conmon[198859]: [09/Oct/2023:05:43:45] ENGINE Bus STARTED 2023-10-09T05:43:46.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 bash[212318]: c9fda800131015cec90ab377329ba7c26657b63c07194b89591bd6a5fa591809 2023-10-09T05:43:46.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T05:43:46.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 systemd[1]: Stopped Ceph mon.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:43:46.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 systemd[1]: Starting Ceph mon.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:43:47.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T05:43:47.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T05:43:47.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: pidfile_write: ignore empty --pid-file 2023-10-09T05:43:47.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: load: jerasure load: lrc 2023-10-09T05:43:47.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: RocksDB version: 7.9.2 2023-10-09T05:43:47.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Git sha 0 2023-10-09T05:43:47.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T05:43:47.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: DB SUMMARY 2023-10-09T05:43:47.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: DB Session ID: 8R51IHPWO45WJDVMY61R 2023-10-09T05:43:47.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: CURRENT file: CURRENT 2023-10-09T05:43:47.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T05:43:47.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1171 Bytes 2023-10-09T05:43:47.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2023-10-09T05:43:47.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 295550 ; 2023-10-09T05:43:47.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.error_if_exists: 0 2023-10-09T05:43:47.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.create_if_missing: 0 2023-10-09T05:43:47.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.paranoid_checks: 1 2023-10-09T05:43:47.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T05:43:47.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T05:43:47.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T05:43:47.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.env: 0x56398eeb6720 2023-10-09T05:43:47.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.fs: PosixFileSystem 2023-10-09T05:43:47.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.info_log: 0x563991db5420 2023-10-09T05:43:47.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T05:43:47.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.statistics: (nil) 2023-10-09T05:43:47.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.use_fsync: 0 2023-10-09T05:43:47.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_log_file_size: 0 2023-10-09T05:43:47.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T05:43:47.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T05:43:47.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T05:43:47.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T05:43:47.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.allow_fallocate: 1 2023-10-09T05:43:47.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T05:43:47.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T05:43:47.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.use_direct_reads: 0 2023-10-09T05:43:47.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T05:43:47.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T05:43:47.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.db_log_dir: 2023-10-09T05:43:47.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.wal_dir: 2023-10-09T05:43:47.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T05:43:47.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T05:43:47.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T05:43:47.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T05:43:47.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T05:43:47.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T05:43:47.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T05:43:47.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T05:43:47.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.write_buffer_manager: 0x563991044c80 2023-10-09T05:43:47.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T05:43:47.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T05:43:47.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T05:43:47.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T05:43:47.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T05:43:47.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T05:43:47.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T05:43:47.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T05:43:47.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.unordered_write: 0 2023-10-09T05:43:47.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T05:43:47.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T05:43:47.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T05:43:47.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T05:43:47.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.row_cache: None 2023-10-09T05:43:47.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.wal_filter: None 2023-10-09T05:43:47.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T05:43:47.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T05:43:47.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.two_write_queues: 0 2023-10-09T05:43:47.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T05:43:47.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.wal_compression: 0 2023-10-09T05:43:47.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.atomic_flush: 0 2023-10-09T05:43:47.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T05:43:47.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T05:43:47.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T05:43:47.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.log_readahead_size: 0 2023-10-09T05:43:47.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T05:43:47.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T05:43:47.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T05:43:47.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T05:43:47.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T05:43:47.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T05:43:47.227 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T05:43:47.227 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_background_jobs: 2 2023-10-09T05:43:47.227 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_background_compactions: -1 2023-10-09T05:43:47.227 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_subcompactions: 1 2023-10-09T05:43:47.227 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T05:43:47.227 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T05:43:47.228 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T05:43:47.228 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T05:43:47.228 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T05:43:47.228 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T05:43:47.228 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T05:43:47.228 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T05:43:47.228 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_open_files: -1 2023-10-09T05:43:47.228 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T05:43:47.228 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T05:43:47.228 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T05:43:47.228 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T05:43:47.228 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_background_flushes: -1 2023-10-09T05:43:47.229 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Compression algorithms supported: 2023-10-09T05:43:47.229 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T05:43:47.229 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: kZSTD supported: 0 2023-10-09T05:43:47.229 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: kXpressCompression supported: 0 2023-10-09T05:43:47.229 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T05:43:47.229 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: kZlibCompression supported: 1 2023-10-09T05:43:47.229 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: kSnappyCompression supported: 1 2023-10-09T05:43:47.229 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: kLZ4Compression supported: 1 2023-10-09T05:43:47.229 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: kBZip2Compression supported: 0 2023-10-09T05:43:47.229 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T05:43:47.229 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T05:43:47.229 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2023-10-09T05:43:47.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T05:43:47.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T05:43:47.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.merge_operator: 2023-10-09T05:43:47.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.compaction_filter: None 2023-10-09T05:43:47.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T05:43:47.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T05:43:47.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T05:43:47.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T05:43:47.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x563991db5520) 2023-10-09T05:43:47.230 INFO:journalctl@ceph.mon.a.smithi064.stdout: cache_index_and_filter_blocks: 1 2023-10-09T05:43:47.230 INFO:journalctl@ceph.mon.a.smithi064.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T05:43:47.230 INFO:journalctl@ceph.mon.a.smithi064.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T05:43:47.231 INFO:journalctl@ceph.mon.a.smithi064.stdout: pin_top_level_index_and_filter: 1 2023-10-09T05:43:47.231 INFO:journalctl@ceph.mon.a.smithi064.stdout: index_type: 0 2023-10-09T05:43:47.231 INFO:journalctl@ceph.mon.a.smithi064.stdout: data_block_index_type: 0 2023-10-09T05:43:47.231 INFO:journalctl@ceph.mon.a.smithi064.stdout: index_shortening: 1 2023-10-09T05:43:47.231 INFO:journalctl@ceph.mon.a.smithi064.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T05:43:47.231 INFO:journalctl@ceph.mon.a.smithi064.stdout: checksum: 4 2023-10-09T05:43:47.231 INFO:journalctl@ceph.mon.a.smithi064.stdout: no_block_cache: 0 2023-10-09T05:43:47.231 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_cache: 0x5639910c7350 2023-10-09T05:43:47.231 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_cache_name: BinnedLRUCache 2023-10-09T05:43:47.231 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_cache_options: 2023-10-09T05:43:47.231 INFO:journalctl@ceph.mon.a.smithi064.stdout: capacity : 536870912 2023-10-09T05:43:47.231 INFO:journalctl@ceph.mon.a.smithi064.stdout: num_shard_bits : 4 2023-10-09T05:43:47.232 INFO:journalctl@ceph.mon.a.smithi064.stdout: strict_capacity_limit : 0 2023-10-09T05:43:47.232 INFO:journalctl@ceph.mon.a.smithi064.stdout: high_pri_pool_ratio: 0.000 2023-10-09T05:43:47.232 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_cache_compressed: (nil) 2023-10-09T05:43:47.232 INFO:journalctl@ceph.mon.a.smithi064.stdout: persistent_cache: (nil) 2023-10-09T05:43:47.232 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_size: 4096 2023-10-09T05:43:47.232 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_size_deviation: 10 2023-10-09T05:43:47.232 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_restart_interval: 16 2023-10-09T05:43:47.232 INFO:journalctl@ceph.mon.a.smithi064.stdout: index_block_restart_interval: 1 2023-10-09T05:43:47.232 INFO:journalctl@ceph.mon.a.smithi064.stdout: metadata_block_size: 4096 2023-10-09T05:43:47.232 INFO:journalctl@ceph.mon.a.smithi064.stdout: partition_filters: 0 2023-10-09T05:43:47.232 INFO:journalctl@ceph.mon.a.smithi064.stdout: use_delta_encoding: 1 2023-10-09T05:43:47.232 INFO:journalctl@ceph.mon.a.smithi064.stdout: filter_policy: bloomfilter 2023-10-09T05:43:47.232 INFO:journalctl@ceph.mon.a.smithi064.stdout: whole_key_filtering: 1 2023-10-09T05:43:47.233 INFO:journalctl@ceph.mon.a.smithi064.stdout: verify_compression: 0 2023-10-09T05:43:47.233 INFO:journalctl@ceph.mon.a.smithi064.stdout: read_amp_bytes_per_bit: 0 2023-10-09T05:43:47.233 INFO:journalctl@ceph.mon.a.smithi064.stdout: format_version: 5 2023-10-09T05:43:47.233 INFO:journalctl@ceph.mon.a.smithi064.stdout: enable_index_compression: 1 2023-10-09T05:43:47.233 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_align: 0 2023-10-09T05:43:47.233 INFO:journalctl@ceph.mon.a.smithi064.stdout: max_auto_readahead_size: 262144 2023-10-09T05:43:47.233 INFO:journalctl@ceph.mon.a.smithi064.stdout: prepopulate_block_cache: 0 2023-10-09T05:43:47.233 INFO:journalctl@ceph.mon.a.smithi064.stdout: initial_auto_readahead_size: 8192 2023-10-09T05:43:47.233 INFO:journalctl@ceph.mon.a.smithi064.stdout: num_file_reads_for_auto_readahead: 2 2023-10-09T05:43:47.233 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T05:43:47.233 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T05:43:47.233 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.compression: NoCompression 2023-10-09T05:43:47.234 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T05:43:47.234 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T05:43:47.234 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T05:43:47.234 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.num_levels: 7 2023-10-09T05:43:47.234 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T05:43:47.235 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T05:43:47.235 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T05:43:47.235 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T05:43:47.235 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T05:43:47.235 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T05:43:47.235 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T05:43:47.235 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:43:47.235 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T05:43:47.235 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T05:43:47.235 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T05:43:47.236 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T05:43:47.236 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T05:43:47.236 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T05:43:47.236 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T05:43:47.236 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T05:43:47.236 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:43:47.236 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T05:43:47.236 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T05:43:47.236 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T05:43:47.236 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T05:43:47.236 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T05:43:47.236 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T05:43:47.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T05:43:47.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T05:43:47.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T05:43:47.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T05:43:47.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T05:43:47.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T05:43:47.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T05:43:47.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T05:43:47.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T05:43:47.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T05:43:47.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T05:43:47.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T05:43:47.238 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T05:43:47.238 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T05:43:47.238 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T05:43:47.238 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T05:43:47.238 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.arena_block_size: 1048576 2023-10-09T05:43:47.238 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T05:43:47.238 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T05:43:47.238 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T05:43:47.238 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T05:43:47.238 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T05:43:47.238 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T05:43:47.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T05:43:47.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T05:43:47.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T05:43:47.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T05:43:47.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T05:43:47.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T05:43:47.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T05:43:47.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.table_properties_collectors: 2023-10-09T05:43:47.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.inplace_update_support: 0 2023-10-09T05:43:47.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T05:43:47.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T05:43:47.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T05:43:47.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T05:43:47.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.bloom_locality: 0 2023-10-09T05:43:47.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.max_successive_merges: 0 2023-10-09T05:43:47.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T05:43:47.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T05:43:47.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T05:43:47.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T05:43:47.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.ttl: 2592000 2023-10-09T05:43:47.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T05:43:47.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T05:43:47.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T05:43:47.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.enable_blob_files: false 2023-10-09T05:43:47.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.min_blob_size: 0 2023-10-09T05:43:47.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T05:43:47.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T05:43:47.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T05:43:47.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T05:43:47.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T05:43:47.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T05:43:47.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.blob_file_starting_level: 0 2023-10-09T05:43:47.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T05:43:47.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2023-10-09T05:43:47.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T05:43:47.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12431, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T05:43:47.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2023-10-09T05:43:47.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: c2b484cd-9d2d-4f9f-83c4-d113cbfb15fa 2023-10-09T05:43:47.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696830226936808, "job": 1, "event": "recovery_started", "wal_files": [31]} 2023-10-09T05:43:47.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2023-10-09T05:43:47.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696830226938499, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 294439, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12429, "largest_seqno": 12595, "table_properties": {"data_size": 292783, "index_size": 388, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 261, "raw_key_size": 2699, "raw_average_key_size": 27, "raw_value_size": 290638, "raw_average_value_size": 2935, "num_data_blocks": 16, "num_entries": 99, "num_filter_entries": 99, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696830226, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "c2b484cd-9d2d-4f9f-83c4-d113cbfb15fa", "db_session_id": "8R51IHPWO45WJDVMY61R", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2023-10-09T05:43:47.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696830226938572, "job": 1, "event": "recovery_finished"} 2023-10-09T05:43:47.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2023-10-09T05:43:47.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T05:43:47.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T05:43:47.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x563991162000 2023-10-09T05:43:47.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: DB pointer 0x563991150000 2023-10-09T05:43:47.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T05:43:47.243 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T05:43:47.243 INFO:journalctl@ceph.mon.a.smithi064.stdout: ** DB Stats ** 2023-10-09T05:43:47.243 INFO:journalctl@ceph.mon.a.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:43:47.243 INFO:journalctl@ceph.mon.a.smithi064.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T05:43:47.243 INFO:journalctl@ceph.mon.a.smithi064.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:43:47.243 INFO:journalctl@ceph.mon.a.smithi064.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:43:47.243 INFO:journalctl@ceph.mon.a.smithi064.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T05:43:47.243 INFO:journalctl@ceph.mon.a.smithi064.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:43:47.243 INFO:journalctl@ceph.mon.a.smithi064.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:43:47.243 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-10-09T05:43:47.243 INFO:journalctl@ceph.mon.a.smithi064.stdout: ** Compaction Stats [default] ** 2023-10-09T05:43:47.243 INFO:journalctl@ceph.mon.a.smithi064.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T05:43:47.244 INFO:journalctl@ceph.mon.a.smithi064.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T05:43:47.244 INFO:journalctl@ceph.mon.a.smithi064.stdout: L0 1/0 287.54 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 280.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-10-09T05:43:47.244 INFO:journalctl@ceph.mon.a.smithi064.stdout: L6 1/0 6.33 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T05:43:47.244 INFO:journalctl@ceph.mon.a.smithi064.stdout: Sum 2/0 6.61 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 280.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-10-09T05:43:47.244 INFO:journalctl@ceph.mon.a.smithi064.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 280.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-10-09T05:43:47.244 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-10-09T05:43:47.244 INFO:journalctl@ceph.mon.a.smithi064.stdout: ** Compaction Stats [default] ** 2023-10-09T05:43:47.244 INFO:journalctl@ceph.mon.a.smithi064.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T05:43:47.244 INFO:journalctl@ceph.mon.a.smithi064.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:43:47.244 INFO:journalctl@ceph.mon.a.smithi064.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 280.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-10-09T05:43:47.244 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-10-09T05:43:47.244 INFO:journalctl@ceph.mon.a.smithi064.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T05:43:47.245 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-10-09T05:43:47.245 INFO:journalctl@ceph.mon.a.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:43:47.245 INFO:journalctl@ceph.mon.a.smithi064.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:43:47.245 INFO:journalctl@ceph.mon.a.smithi064.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:43:47.245 INFO:journalctl@ceph.mon.a.smithi064.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:43:47.245 INFO:journalctl@ceph.mon.a.smithi064.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:43:47.245 INFO:journalctl@ceph.mon.a.smithi064.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:43:47.245 INFO:journalctl@ceph.mon.a.smithi064.stdout: Cumulative compaction: 0.00 GB write, 72.04 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:43:47.245 INFO:journalctl@ceph.mon.a.smithi064.stdout: Interval compaction: 0.00 GB write, 72.04 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:43:47.245 INFO:journalctl@ceph.mon.a.smithi064.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:43:47.245 INFO:journalctl@ceph.mon.a.smithi064.stdout: Block cache BinnedLRUCache@0x5639910c7350#7 capacity: 512.00 MB usage: 0.80 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2023-10-09T05:43:47.246 INFO:journalctl@ceph.mon.a.smithi064.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.31 KB,5.96046e-05%) IndexBlock(1,0.48 KB,9.23872e-05%) Misc(1,0.00 KB,0%) 2023-10-09T05:43:47.246 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-10-09T05:43:47.246 INFO:journalctl@ceph.mon.a.smithi064.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:43:47.246 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: starting mon.a rank 0 at public addrs [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] at bind addrs [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:43:47.246 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: mon.a@-1(???) e3 preinit fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:43:47.246 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: mon.a@-1(???).mds e8 new map 2023-10-09T05:43:47.246 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: mon.a@-1(???).mds e8 print_map 2023-10-09T05:43:47.246 INFO:journalctl@ceph.mon.a.smithi064.stdout: e8 2023-10-09T05:43:47.246 INFO:journalctl@ceph.mon.a.smithi064.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T05:43:47.246 INFO:journalctl@ceph.mon.a.smithi064.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T05:43:47.246 INFO:journalctl@ceph.mon.a.smithi064.stdout: legacy client fscid: 1 2023-10-09T05:43:47.246 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-10-09T05:43:47.247 INFO:journalctl@ceph.mon.a.smithi064.stdout: Filesystem 'foo' (1) 2023-10-09T05:43:47.247 INFO:journalctl@ceph.mon.a.smithi064.stdout: fs_name foo 2023-10-09T05:43:47.247 INFO:journalctl@ceph.mon.a.smithi064.stdout: epoch 7 2023-10-09T05:43:47.247 INFO:journalctl@ceph.mon.a.smithi064.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T05:43:47.247 INFO:journalctl@ceph.mon.a.smithi064.stdout: created 2023-10-09T05:40:12.634524+0000 2023-10-09T05:43:47.247 INFO:journalctl@ceph.mon.a.smithi064.stdout: modified 2023-10-09T05:40:21.631615+0000 2023-10-09T05:43:47.247 INFO:journalctl@ceph.mon.a.smithi064.stdout: tableserver 0 2023-10-09T05:43:47.247 INFO:journalctl@ceph.mon.a.smithi064.stdout: root 0 2023-10-09T05:43:47.247 INFO:journalctl@ceph.mon.a.smithi064.stdout: session_timeout 60 2023-10-09T05:43:47.247 INFO:journalctl@ceph.mon.a.smithi064.stdout: session_autoclose 300 2023-10-09T05:43:47.247 INFO:journalctl@ceph.mon.a.smithi064.stdout: max_file_size 1099511627776 2023-10-09T05:43:47.247 INFO:journalctl@ceph.mon.a.smithi064.stdout: required_client_features {} 2023-10-09T05:43:47.248 INFO:journalctl@ceph.mon.a.smithi064.stdout: last_failure 0 2023-10-09T05:43:47.248 INFO:journalctl@ceph.mon.a.smithi064.stdout: last_failure_osd_epoch 0 2023-10-09T05:43:47.248 INFO:journalctl@ceph.mon.a.smithi064.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T05:43:47.248 INFO:journalctl@ceph.mon.a.smithi064.stdout: max_mds 1 2023-10-09T05:43:47.248 INFO:journalctl@ceph.mon.a.smithi064.stdout: in 0 2023-10-09T05:43:47.248 INFO:journalctl@ceph.mon.a.smithi064.stdout: up {0=24551} 2023-10-09T05:43:47.248 INFO:journalctl@ceph.mon.a.smithi064.stdout: failed 2023-10-09T05:43:47.248 INFO:journalctl@ceph.mon.a.smithi064.stdout: damaged 2023-10-09T05:43:47.248 INFO:journalctl@ceph.mon.a.smithi064.stdout: stopped 2023-10-09T05:43:47.248 INFO:journalctl@ceph.mon.a.smithi064.stdout: data_pools [4] 2023-10-09T05:43:47.248 INFO:journalctl@ceph.mon.a.smithi064.stdout: metadata_pool 3 2023-10-09T05:43:47.248 INFO:journalctl@ceph.mon.a.smithi064.stdout: inline_data disabled 2023-10-09T05:43:47.249 INFO:journalctl@ceph.mon.a.smithi064.stdout: balancer 2023-10-09T05:43:47.249 INFO:journalctl@ceph.mon.a.smithi064.stdout: bal_rank_mask -1 2023-10-09T05:43:47.249 INFO:journalctl@ceph.mon.a.smithi064.stdout: standby_count_wanted 1 2023-10-09T05:43:47.249 INFO:journalctl@ceph.mon.a.smithi064.stdout: [mds.foo.smithi064.gmrwsy{0:24551} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.64:6834/3975184368,v1:172.21.15.64:6835/3975184368] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T05:43:47.249 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-10-09T05:43:47.249 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-10-09T05:43:47.249 INFO:journalctl@ceph.mon.a.smithi064.stdout: Standby daemons: 2023-10-09T05:43:47.249 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-10-09T05:43:47.249 INFO:journalctl@ceph.mon.a.smithi064.stdout: [mds.foo.smithi156.pizbdw{-1:24556} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.156:6832/4137868430,v1:172.21.15.156:6833/4137868430] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T05:43:47.249 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: mon.a@-1(???).osd e79 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T05:43:47.249 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: mon.a@-1(???).osd e79 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T05:43:47.249 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: mon.a@-1(???).osd e79 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T05:43:47.250 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: mon.a@-1(???).osd e79 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T05:43:47.250 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2023-10-09T05:43:47.250 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: expand_channel_meta expand map: {default=false} 2023-10-09T05:43:47.250 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: expand_channel_meta from 'false' to 'false' 2023-10-09T05:43:47.250 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: expand_channel_meta expanded map: {default=false} 2023-10-09T05:43:47.250 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: expand_channel_meta expand map: {default=info} 2023-10-09T05:43:47.250 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: expand_channel_meta from 'info' to 'info' 2023-10-09T05:43:47.250 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: expand_channel_meta expanded map: {default=info} 2023-10-09T05:43:47.250 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: expand_channel_meta expand map: {default=daemon} 2023-10-09T05:43:47.250 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T05:43:47.250 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: expand_channel_meta expanded map: {default=daemon} 2023-10-09T05:43:47.250 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: expand_channel_meta expand map: {default=debug} 2023-10-09T05:43:47.250 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: expand_channel_meta from 'debug' to 'debug' 2023-10-09T05:43:47.251 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 ceph-mon[212750]: expand_channel_meta expanded map: {default=debug} 2023-10-09T05:43:47.251 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 bash[212685]: 5d3854b1b1eb71aa2bb759f4396c9cf41c8f6c54083f19a840cd320cab1b52e2 2023-10-09T05:43:47.251 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:46 smithi064 systemd[1]: Started Ceph mon.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:43:49.120 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:43:48 smithi156 conmon[179783]: ts=2023-10-09T05:43:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:43:50.283 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:50 smithi064 conmon[198859]: ignoring --setuser ceph since I am not root 2023-10-09T05:43:50.283 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:50 smithi064 conmon[198859]: ignoring --setgroup ceph since I am not root 2023-10-09T05:43:50.283 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mgr[198886]: -- 172.21.15.64:0/1350909281 <== mon.0 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5556215031e0 con 0x5556214dac00 2023-10-09T05:43:50.283 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:50 smithi064 conmon[198859]: 2023-10-09T05:43:50.186+0000 7fcbcdb55200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:43:50.283 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:50 smithi064 conmon[198859]: 2023-10-09T05:43:50.282+0000 7fcbcdb55200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:43:50.283 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[212750]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:43:50.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[212750]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:43:50.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[212750]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:43:50.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[212750]: pgmap v18: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:50.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3337086789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:50.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[212750]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:50.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[212750]: pgmap v19: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:50.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/800939313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:50.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[212750]: mon.a calling monitor election 2023-10-09T05:43:50.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[212750]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:43:50.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[212750]: monmap e3: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} 2023-10-09T05:43:50.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[212750]: fsmap foo:1 {0=foo.smithi064.gmrwsy=up:active} 1 up:standby 2023-10-09T05:43:50.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[212750]: osdmap e79: 8 total, 8 up, 8 in 2023-10-09T05:43:50.285 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[212750]: mgrmap e31: y(active, since 32s), standbys: x 2023-10-09T05:43:50.285 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[212750]: mgrmap e32: y(active, since 32s), standbys: x 2023-10-09T05:43:50.285 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[212750]: overall HEALTH_OK 2023-10-09T05:43:50.285 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[212750]: from='mgr.15141 ' entity='' 2023-10-09T05:43:50.285 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:43:50.285 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:43:50.285 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:43:50.285 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[118354]: pgmap v18: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:50.285 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/3337086789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:50.285 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[118354]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:50.286 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[118354]: pgmap v19: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:50.286 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/800939313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:50.286 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[118354]: mon.a calling monitor election 2023-10-09T05:43:50.286 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[118354]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:43:50.286 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[118354]: monmap e3: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} 2023-10-09T05:43:50.286 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[118354]: fsmap foo:1 {0=foo.smithi064.gmrwsy=up:active} 1 up:standby 2023-10-09T05:43:50.286 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[118354]: osdmap e79: 8 total, 8 up, 8 in 2023-10-09T05:43:50.286 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[118354]: mgrmap e31: y(active, since 32s), standbys: x 2023-10-09T05:43:50.286 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[118354]: mgrmap e32: y(active, since 32s), standbys: x 2023-10-09T05:43:50.286 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[118354]: overall HEALTH_OK 2023-10-09T05:43:50.286 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:50 smithi064 ceph-mon[118354]: from='mgr.15141 ' entity='' 2023-10-09T05:43:50.330 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:50 smithi156 conmon[180812]: ignoring --setuser ceph since I am not root 2023-10-09T05:43:50.330 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:50 smithi156 conmon[180812]: ignoring --setgroup ceph since I am not root 2023-10-09T05:43:50.330 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:50 smithi156 ceph-mgr[180836]: -- 172.21.15.156:0/679665591 <== mon.1 v2:172.21.15.156:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5582428671e0 con 0x55824283f000 2023-10-09T05:43:50.331 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:50 smithi156 conmon[180812]: 2023-10-09T05:43:50.159+0000 7f21ef6ef200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:43:50.331 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:50 smithi156 conmon[180812]: 2023-10-09T05:43:50.239+0000 7f21ef6ef200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:43:50.331 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:50 smithi156 conmon[180812]: 2023-10-09T05:43:50.329+0000 7f21ef6ef200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:43:50.331 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:50 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:43:50.331 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:50 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:43:50.331 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:50 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:43:50.331 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:50 smithi156 ceph-mon[105979]: pgmap v18: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:50.331 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:50 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/3337086789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:50.332 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:50 smithi156 ceph-mon[105979]: from='mgr.15141 172.21.15.64:0/1577907743' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:50.332 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:50 smithi156 ceph-mon[105979]: pgmap v19: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:43:50.332 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:50 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/800939313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:50.332 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:50 smithi156 ceph-mon[105979]: mon.a calling monitor election 2023-10-09T05:43:50.332 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:50 smithi156 ceph-mon[105979]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:43:50.332 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:50 smithi156 ceph-mon[105979]: monmap e3: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} 2023-10-09T05:43:50.332 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:50 smithi156 ceph-mon[105979]: fsmap foo:1 {0=foo.smithi064.gmrwsy=up:active} 1 up:standby 2023-10-09T05:43:50.332 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:50 smithi156 ceph-mon[105979]: osdmap e79: 8 total, 8 up, 8 in 2023-10-09T05:43:50.332 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:50 smithi156 ceph-mon[105979]: mgrmap e31: y(active, since 32s), standbys: x 2023-10-09T05:43:50.332 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:50 smithi156 ceph-mon[105979]: mgrmap e32: y(active, since 32s), standbys: x 2023-10-09T05:43:50.333 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:50 smithi156 ceph-mon[105979]: overall HEALTH_OK 2023-10-09T05:43:50.333 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:50 smithi156 ceph-mon[105979]: from='mgr.15141 ' entity='' 2023-10-09T05:43:50.600 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:50 smithi156 conmon[180812]: 2023-10-09T05:43:50.431+0000 7f21ef6ef200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:43:50.713 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:50 smithi064 conmon[198859]: 2023-10-09T05:43:50.392+0000 7fcbcdb55200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:43:50.714 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:50 smithi064 conmon[198859]: 2023-10-09T05:43:50.504+0000 7fcbcdb55200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:43:50.714 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:50 smithi064 conmon[198859]: 2023-10-09T05:43:50.713+0000 7fcbcdb55200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:43:50.870 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:50 smithi156 conmon[180812]: 2023-10-09T05:43:50.599+0000 7f21ef6ef200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:43:51.279 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:50 smithi156 conmon[180812]: 2023-10-09T05:43:50.964+0000 7f21ef6ef200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:43:51.279 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:51 smithi156 conmon[180812]: 2023-10-09T05:43:51.095+0000 7f21ef6ef200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:43:51.279 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:51 smithi156 conmon[180812]: 2023-10-09T05:43:51.183+0000 7f21ef6ef200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:43:51.452 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:51 smithi064 conmon[198859]: 2023-10-09T05:43:51.188+0000 7fcbcdb55200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:43:51.452 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:51 smithi064 conmon[198859]: 2023-10-09T05:43:51.342+0000 7fcbcdb55200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:43:51.620 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:51 smithi156 conmon[180812]: 2023-10-09T05:43:51.278+0000 7f21ef6ef200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:43:51.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:51 smithi064 conmon[198859]: 2023-10-09T05:43:51.451+0000 7fcbcdb55200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:43:51.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:51 smithi064 conmon[198859]: 2023-10-09T05:43:51.568+0000 7fcbcdb55200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:43:52.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:51 smithi156 conmon[180812]: 2023-10-09T05:43:51.702+0000 7f21ef6ef200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:43:52.209 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 1 2023-10-09T05:43:52.407 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:52 smithi064 conmon[198859]: 2023-10-09T05:43:52.092+0000 7fcbcdb55200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:43:52.620 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:52 smithi156 conmon[180812]: 2023-10-09T05:43:52.360+0000 7f21ef6ef200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:43:52.620 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:52 smithi156 conmon[180812]: 2023-10-09T05:43:52.443+0000 7f21ef6ef200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:43:52.956 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T05:43:53.219 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:52 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/4248731677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:53.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:52 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4248731677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:53.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:52 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/4248731677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:43:53.220 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:52 smithi064 conmon[198859]: 2023-10-09T05:43:52.892+0000 7fcbcdb55200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:43:53.220 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:53 smithi064 conmon[198859]: 2023-10-09T05:43:53.002+0000 7fcbcdb55200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:43:53.325 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:43:53.483 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:53 smithi156 conmon[180812]: 2023-10-09T05:43:53.218+0000 7f21ef6ef200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:43:53.483 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:53 smithi156 conmon[180812]: 2023-10-09T05:43:53.311+0000 7f21ef6ef200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:43:53.764 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:53 smithi156 conmon[180812]: 2023-10-09T05:43:53.482+0000 7f21ef6ef200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:43:54.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:53 smithi156 conmon[180812]: 2023-10-09T05:43:53.764+0000 7f21ef6ef200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:43:54.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:53 smithi156 conmon[180812]: 2023-10-09T05:43:53.993+0000 7f21ef6ef200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:43:54.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:54 smithi064 conmon[198859]: 2023-10-09T05:43:54.032+0000 7fcbcdb55200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:43:54.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:54 smithi064 conmon[198859]: 2023-10-09T05:43:54.146+0000 7fcbcdb55200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:43:54.408 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:54 smithi156 conmon[180812]: 2023-10-09T05:43:54.141+0000 7f21ef6ef200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:43:54.666 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:54 smithi064 conmon[198859]: 2023-10-09T05:43:54.373+0000 7fcbcdb55200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:43:54.870 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:54 smithi156 conmon[180812]: 2023-10-09T05:43:54.408+0000 7f21ef6ef200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:43:54.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:54 smithi064 conmon[198859]: 2023-10-09T05:43:54.748+0000 7fcbcdb55200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:43:55.370 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:55 smithi156 conmon[180812]: 2023-10-09T05:43:55.119+0000 7f21ef6ef200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:43:55.370 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:55 smithi156 conmon[180812]: 2023-10-09T05:43:55.251+0000 7f21ef6ef200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:43:55.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:55 smithi064 conmon[198859]: 2023-10-09T05:43:55.011+0000 7fcbcdb55200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:43:55.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:55 smithi064 conmon[198859]: 2023-10-09T05:43:55.199+0000 7fcbcdb55200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:43:55.779 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:55 smithi156 conmon[180812]: 2023-10-09T05:43:55.447+0000 7f21ef6ef200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:43:55.779 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:55 smithi156 conmon[180812]: 2023-10-09T05:43:55.560+0000 7f21ef6ef200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:43:55.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:55 smithi064 conmon[198859]: 2023-10-09T05:43:55.538+0000 7fcbcdb55200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:43:56.120 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:55 smithi156 conmon[180812]: 2023-10-09T05:43:55.778+0000 7f21ef6ef200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:43:56.720 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:56 smithi064 conmon[198859]: 2023-10-09T05:43:56.381+0000 7fcbcdb55200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:43:56.720 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:56 smithi064 conmon[198859]: 2023-10-09T05:43:56.543+0000 7fcbcdb55200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:43:57.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:56 smithi156 ceph-mon[105979]: Standby manager daemon x restarted 2023-10-09T05:43:57.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:56 smithi156 ceph-mon[105979]: Standby manager daemon x started 2023-10-09T05:43:57.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:56 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.156:0/775718415' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:43:57.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:56 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.156:0/775718415' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:43:57.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:56 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.156:0/775718415' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:43:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:56 smithi156 ceph-mon[105979]: from='mgr.? 172.21.15.156:0/775718415' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:43:57.121 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:56 smithi156 conmon[180812]: 2023-10-09T05:43:56.649+0000 7f21ef6ef200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:43:57.121 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:56 smithi156 conmon[180812]: [09/Oct/2023:05:43:56] ENGINE Bus STARTING 2023-10-09T05:43:57.121 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:56 smithi156 conmon[180812]: CherryPy Checker: 2023-10-09T05:43:57.121 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:56 smithi156 conmon[180812]: The Application mounted at '' has an empty config. 2023-10-09T05:43:57.121 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:56 smithi156 conmon[180812]: 2023-10-09T05:43:57.121 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:56 smithi156 conmon[180812]: [09/Oct/2023:05:43:56] ENGINE Serving on http://:::9283 2023-10-09T05:43:57.121 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 05:43:56 smithi156 conmon[180812]: [09/Oct/2023:05:43:56] ENGINE Bus STARTED 2023-10-09T05:43:57.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:56 smithi064 ceph-mon[212750]: Standby manager daemon x restarted 2023-10-09T05:43:57.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:56 smithi064 ceph-mon[212750]: Standby manager daemon x started 2023-10-09T05:43:57.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:56 smithi064 ceph-mon[212750]: from='mgr.? 172.21.15.156:0/775718415' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:43:57.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:56 smithi064 ceph-mon[212750]: from='mgr.? 172.21.15.156:0/775718415' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:43:57.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:56 smithi064 ceph-mon[212750]: from='mgr.? 172.21.15.156:0/775718415' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:43:57.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:56 smithi064 ceph-mon[212750]: from='mgr.? 172.21.15.156:0/775718415' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:43:57.220 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:56 smithi064 conmon[198859]: 2023-10-09T05:43:56.800+0000 7fcbcdb55200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:43:57.220 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:56 smithi064 conmon[198859]: 2023-10-09T05:43:56.942+0000 7fcbcdb55200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:43:57.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:56 smithi064 ceph-mon[118354]: Standby manager daemon x restarted 2023-10-09T05:43:57.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:56 smithi064 ceph-mon[118354]: Standby manager daemon x started 2023-10-09T05:43:57.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:56 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.156:0/775718415' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:43:57.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:56 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.156:0/775718415' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:43:57.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:56 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.156:0/775718415' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:43:57.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:56 smithi064 ceph-mon[118354]: from='mgr.? 172.21.15.156:0/775718415' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:43:57.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:57 smithi064 conmon[198859]: 2023-10-09T05:43:57.229+0000 7fcbcdb55200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:43:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:57 smithi156 ceph-mon[105979]: mgrmap e33: y(active, since 39s), standbys: x 2023-10-09T05:43:58.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:57 smithi064 ceph-mon[212750]: mgrmap e33: y(active, since 39s), standbys: x 2023-10-09T05:43:58.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:57 smithi064 ceph-mon[118354]: mgrmap e33: y(active, since 39s), standbys: x 2023-10-09T05:43:58.552 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 conmon[118330]: 2023-10-09T05:43:58.316+0000 7f135b948700 -1 mon.c@2(peon).osd e80 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T05:43:58.552 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:58 smithi064 conmon[198859]: 2023-10-09T05:43:58.299+0000 7fcbcdb55200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:43:58.552 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:58 smithi064 conmon[198859]: [09/Oct/2023:05:43:58] ENGINE Bus STARTING 2023-10-09T05:43:58.553 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:58 smithi064 conmon[198859]: 2023-10-09T05:43:58.477+0000 7fcb27e80700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:43:58.553 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:58 smithi064 conmon[198859]: 2023-10-09T05:43:58.477+0000 7fcb27e80700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:43:58.553 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:58 smithi064 conmon[198859]: 2023-10-09T05:43:58.477+0000 7fcb27e80700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:43:58.553 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:58 smithi064 conmon[198859]: 2023-10-09T05:43:58.477+0000 7fcb27e80700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:43:58.553 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:58 smithi064 conmon[198859]: 2023-10-09T05:43:58.477+0000 7fcb27e80700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:43:58.553 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:58 smithi064 conmon[198859]: 2023-10-09T05:43:58.479+0000 7fcb2ce8a700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:43:58.553 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:58 smithi064 conmon[198859]: 2023-10-09T05:43:58.479+0000 7fcb2ce8a700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:43:58.553 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:58 smithi064 conmon[198859]: 2023-10-09T05:43:58.479+0000 7fcb2ce8a700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:43:58.553 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:58 smithi064 conmon[198859]: 2023-10-09T05:43:58.479+0000 7fcb2ce8a700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:43:58.553 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:58 smithi064 conmon[198859]: 2023-10-09T05:43:58.479+0000 7fcb2ce8a700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T05:43:58.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 conmon[105952]: 2023-10-09T05:43:58.317+0000 7fea8504b700 -1 mon.b@1(peon).osd e80 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T05:43:58.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:58 smithi064 conmon[198859]: CherryPy Checker: 2023-10-09T05:43:58.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:58 smithi064 conmon[198859]: The Application mounted at '' has an empty config. 2023-10-09T05:43:58.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:58 smithi064 conmon[198859]: 2023-10-09T05:43:58.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:58 smithi064 conmon[198859]: [09/Oct/2023:05:43:58] ENGINE Serving on http://:::9283 2023-10-09T05:43:58.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:58 smithi064 conmon[198859]: [09/Oct/2023:05:43:58] ENGINE Bus STARTED 2023-10-09T05:43:58.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: Active manager daemon y restarted 2023-10-09T05:43:58.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: Activating manager daemon y 2023-10-09T05:43:58.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T05:43:58.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: mgrmap e34: y(active, starting, since 0.00633368s), standbys: x 2023-10-09T05:43:58.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: failed to encode map e80 with expected crc 2023-10-09T05:43:58.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: failed to encode map e80 with expected crc 2023-10-09T05:43:58.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:43:58.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:43:58.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:43:58.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.gmrwsy"}]: dispatch 2023-10-09T05:43:58.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.pizbdw"}]: dispatch 2023-10-09T05:43:58.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: failed to encode map e80 with expected crc 2023-10-09T05:43:58.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:43:58.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:43:58.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:43:58.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:43:58.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:43:58.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:43:58.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:43:58.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:43:58.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:43:58.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:43:58.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:43:58.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:43:58.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:43:58.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: Manager daemon y is now available 2023-10-09T05:43:58.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:58.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:58.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:58.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: Active manager daemon y restarted 2023-10-09T05:43:58.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: Activating manager daemon y 2023-10-09T05:43:58.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T05:43:58.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: mgrmap e34: y(active, starting, since 0.00633368s), standbys: x 2023-10-09T05:43:58.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: failed to encode map e80 with expected crc 2023-10-09T05:43:58.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: failed to encode map e80 with expected crc 2023-10-09T05:43:58.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:43:58.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:43:58.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:43:58.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.gmrwsy"}]: dispatch 2023-10-09T05:43:58.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.pizbdw"}]: dispatch 2023-10-09T05:43:58.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: failed to encode map e80 with expected crc 2023-10-09T05:43:58.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:43:58.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:43:58.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:43:58.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:43:58.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:43:58.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:43:58.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:43:58.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:43:58.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:43:58.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:43:58.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:43:58.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:43:58.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:43:58.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: Manager daemon y is now available 2023-10-09T05:43:58.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:58.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:58.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:58.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:43:58.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:43:58.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: failed to encode map e80 with expected crc 2023-10-09T05:43:58.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:43:58.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:43:58.976 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:43:58.976 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:43:58.977 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: failed to encode map e80 with expected crc 2023-10-09T05:43:58.977 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:43:58.977 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:58 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:43:59.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: Active manager daemon y restarted 2023-10-09T05:43:59.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: Activating manager daemon y 2023-10-09T05:43:59.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T05:43:59.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: mgrmap e34: y(active, starting, since 0.00633368s), standbys: x 2023-10-09T05:43:59.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: failed to encode map e80 with expected crc 2023-10-09T05:43:59.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: failed to encode map e80 with expected crc 2023-10-09T05:43:59.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:43:59.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:43:59.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:43:59.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.gmrwsy"}]: dispatch 2023-10-09T05:43:59.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.pizbdw"}]: dispatch 2023-10-09T05:43:59.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: failed to encode map e80 with expected crc 2023-10-09T05:43:59.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:43:59.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:43:59.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:43:59.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:43:59.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:43:59.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:43:59.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:43:59.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:43:59.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:43:59.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:43:59.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:43:59.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:43:59.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:43:59.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: Manager daemon y is now available 2023-10-09T05:43:59.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:59.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:43:59.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:43:59.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:43:59.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:43:59.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: failed to encode map e80 with expected crc 2023-10-09T05:43:59.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:43:59.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:58 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:43:59.345 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T05:43:59.345 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (68s) - 4m 18.6M - 0.25.0 c8568f914cd2 386f8a4e2221 2023-10-09T05:43:59.345 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi156 *:3000 running (55s) - 4m 85.8M - 9.4.7 2c41d148cca3 365834e48ffb 2023-10-09T05:43:59.346 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.gmrwsy smithi064 running (3m) - 3m 16.7M - 17.2.6-1303-gb2903cb0 5517396c50d4 e938ff2e972d 2023-10-09T05:43:59.346 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi156.pizbdw smithi156 running (3m) - 3m 17.0M - 17.2.6-1303-gb2903cb0 5517396c50d4 e8906e08c938 2023-10-09T05:43:59.346 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi156 *:8443,9283,8765 running (24s) - 8m 45.6M - 18.2.0-598-gb4fa663f 3060897fdf97 d0092a415024 2023-10-09T05:43:59.346 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:8443,9283,8765 running (50s) - 10m 475M - 18.2.0-598-gb4fa663f 3060897fdf97 2bc7549ee2c1 2023-10-09T05:43:59.346 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (10m) - 10m 100M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 c9fda8001310 2023-10-09T05:43:59.347 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi156 running (8m) - 8m 73.4M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 c60a890df99e 2023-10-09T05:43:59.347 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (8m) - 8m 66.1M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 b6790a9456ba 2023-10-09T05:43:59.347 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (66s) - 4m 8070k - 1.5.0 0da6a335fe13 afb28622babd 2023-10-09T05:43:59.347 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi156 *:9100 running (59s) - 4m 16.0M - 1.5.0 0da6a335fe13 1e8c1e28630a 2023-10-09T05:43:59.347 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (8m) - 8m 74.5M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 bd3c343a2f7c 2023-10-09T05:43:59.348 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (7m) - 7m 75.9M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 854146c77f6d 2023-10-09T05:43:59.348 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (7m) - 7m 147M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 75a274ca9b6e 2023-10-09T05:43:59.348 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (7m) - 7m 66.1M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 711a9677648c 2023-10-09T05:43:59.348 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi156 running (6m) - 6m 75.4M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 72313d594357 2023-10-09T05:43:59.348 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi156 running (6m) - 6m 99.3M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 40d9335e2ab7 2023-10-09T05:43:59.349 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi156 running (5m) - 5m 64.2M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 c8a3e3228b89 2023-10-09T05:43:59.349 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi156 running (5m) - 5m 104M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 df93983742a3 2023-10-09T05:43:59.349 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi156 *:9095 running (29s) - 5m 36.7M - 2.43.0 a07b618ecd1d 712dad4d0e3b 2023-10-09T05:43:59.614 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:43:59 smithi064 conmon[198859]: 2023-10-09T05:43:59.327+0000 7fcb872b4700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-10-09T05:43:59.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:59 smithi064 ceph-mon[212750]: failed to encode map e80 with expected crc 2023-10-09T05:43:59.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:59 smithi064 ceph-mon[212750]: [09/Oct/2023:05:43:58] ENGINE Bus STARTING 2023-10-09T05:43:59.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:59 smithi064 ceph-mon[212750]: [09/Oct/2023:05:43:58] ENGINE Serving on https://172.21.15.64:7150 2023-10-09T05:43:59.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:59 smithi064 ceph-mon[212750]: [09/Oct/2023:05:43:59] ENGINE Serving on http://172.21.15.64:8765 2023-10-09T05:43:59.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:59 smithi064 ceph-mon[212750]: [09/Oct/2023:05:43:59] ENGINE Bus STARTED 2023-10-09T05:43:59.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:43:59 smithi064 ceph-mon[212750]: mgrmap e35: y(active, since 1.01341s), standbys: x 2023-10-09T05:43:59.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:59 smithi064 ceph-mon[118354]: failed to encode map e80 with expected crc 2023-10-09T05:43:59.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:59 smithi064 ceph-mon[118354]: [09/Oct/2023:05:43:58] ENGINE Bus STARTING 2023-10-09T05:43:59.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:59 smithi064 ceph-mon[118354]: [09/Oct/2023:05:43:58] ENGINE Serving on https://172.21.15.64:7150 2023-10-09T05:43:59.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:59 smithi064 ceph-mon[118354]: [09/Oct/2023:05:43:59] ENGINE Serving on http://172.21.15.64:8765 2023-10-09T05:43:59.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:59 smithi064 ceph-mon[118354]: [09/Oct/2023:05:43:59] ENGINE Bus STARTED 2023-10-09T05:43:59.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:43:59 smithi064 ceph-mon[118354]: mgrmap e35: y(active, since 1.01341s), standbys: x 2023-10-09T05:44:00.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:59 smithi156 ceph-mon[105979]: failed to encode map e80 with expected crc 2023-10-09T05:44:00.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:59 smithi156 ceph-mon[105979]: [09/Oct/2023:05:43:58] ENGINE Bus STARTING 2023-10-09T05:44:00.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:59 smithi156 ceph-mon[105979]: [09/Oct/2023:05:43:58] ENGINE Serving on https://172.21.15.64:7150 2023-10-09T05:44:00.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:59 smithi156 ceph-mon[105979]: [09/Oct/2023:05:43:59] ENGINE Serving on http://172.21.15.64:8765 2023-10-09T05:44:00.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:59 smithi156 ceph-mon[105979]: [09/Oct/2023:05:43:59] ENGINE Bus STARTED 2023-10-09T05:44:00.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:43:59 smithi156 ceph-mon[105979]: mgrmap e35: y(active, since 1.01341s), standbys: x 2023-10-09T05:44:00.251 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T05:44:00.760 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:00.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:00 smithi064 ceph-mon[212750]: from='client.34119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:44:00.984 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:00 smithi064 ceph-mon[212750]: pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:00.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:00 smithi064 ceph-mon[118354]: from='client.34119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:44:00.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:00 smithi064 ceph-mon[118354]: pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:01.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:00 smithi156 ceph-mon[105979]: from='client.34119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:44:01.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:00 smithi156 ceph-mon[105979]: pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:02.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:01 smithi156 ceph-mon[105979]: pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:02.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:01 smithi156 ceph-mon[105979]: mgrmap e36: y(active, since 2s), standbys: x 2023-10-09T05:44:02.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:01 smithi064 ceph-mon[212750]: pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:02.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:01 smithi064 ceph-mon[212750]: mgrmap e36: y(active, since 2s), standbys: x 2023-10-09T05:44:02.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:01 smithi064 ceph-mon[118354]: pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:02.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:01 smithi064 ceph-mon[118354]: mgrmap e36: y(active, since 2s), standbys: x 2023-10-09T05:44:02.852 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:02 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:02.853 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:02 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:02.853 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:02 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:02.853 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:02 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:02.853 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:02 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:02.853 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:02 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:03.033 INFO:teuthology.orchestra.run.smithi064.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:44:03.106 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:02 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:03.106 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:02 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:03.107 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:02 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:03.107 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:02 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:03.107 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:02 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:03.107 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:02 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:03.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:02 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:03.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:02 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:03.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:02 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:03.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:02 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:03.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:02 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:03.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:02 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:03.736 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T05:44:04.093 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:44:04.283 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:04 smithi156 ceph-mon[105979]: pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:04.283 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:04 smithi156 ceph-mon[105979]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:44:04.283 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:04 smithi156 ceph-mon[105979]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:44:04.283 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:04 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:04.284 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:04 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:04.284 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:04 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:04.312 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:04 smithi064 ceph-mon[212750]: pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:04.312 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:04 smithi064 ceph-mon[212750]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:44:04.313 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:04 smithi064 ceph-mon[212750]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:44:04.313 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:04 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:04.313 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:04 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:04.313 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:04 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:04.313 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:04 smithi064 ceph-mon[118354]: pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:04.313 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:04 smithi064 ceph-mon[118354]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:44:04.313 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:04 smithi064 ceph-mon[118354]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:44:04.314 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:04 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:04.314 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:04 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:04.314 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:04 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:05.399 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:44:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:44:05] "GET /metrics HTTP/1.1" 200 31162 "" "Prometheus/2.43.0" 2023-10-09T05:44:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:06 smithi156 ceph-mon[105979]: pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:06 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:06 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:06 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:44:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:06 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:44:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:06 smithi064 ceph-mon[212750]: pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:06 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:06 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:06 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:44:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:06 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:44:06.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:06 smithi064 ceph-mon[118354]: pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:06.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:06 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:06.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:06 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:06.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:06 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:44:06.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:06 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:44:07.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:07 smithi156 ceph-mon[105979]: Updating smithi064:/etc/ceph/ceph.conf 2023-10-09T05:44:07.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:07 smithi156 ceph-mon[105979]: Updating smithi156:/etc/ceph/ceph.conf 2023-10-09T05:44:07.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:07 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:07.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:07 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:07.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:07 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:07 smithi064 ceph-mon[212750]: Updating smithi064:/etc/ceph/ceph.conf 2023-10-09T05:44:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:07 smithi064 ceph-mon[212750]: Updating smithi156:/etc/ceph/ceph.conf 2023-10-09T05:44:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:07 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:07 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:07 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:07 smithi064 ceph-mon[118354]: Updating smithi064:/etc/ceph/ceph.conf 2023-10-09T05:44:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:07 smithi064 ceph-mon[118354]: Updating smithi156:/etc/ceph/ceph.conf 2023-10-09T05:44:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:07 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:07 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:07 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:08.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:08 smithi156 ceph-mon[105979]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:44:08.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:08 smithi156 ceph-mon[105979]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:44:08.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:08 smithi156 ceph-mon[105979]: pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:08.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:08 smithi156 ceph-mon[105979]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:44:08.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:08 smithi156 ceph-mon[105979]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:44:08.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:08 smithi156 ceph-mon[105979]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:44:08.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:08 smithi156 ceph-mon[105979]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:44:08.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:08 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:08.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:08 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:08.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:08 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:44:08.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:08 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:44:08.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:08 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:44:08.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:08 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:08.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[212750]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:44:08.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[212750]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:44:08.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[212750]: pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:08.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[212750]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:44:08.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[212750]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:44:08.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[212750]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:44:08.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[212750]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:44:08.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:08.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:08.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:44:08.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:44:08.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:44:08.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:08.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[118354]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:44:08.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[118354]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:44:08.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[118354]: pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:08.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[118354]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:44:08.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[118354]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:44:08.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[118354]: Updating smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:44:08.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[118354]: Updating smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:44:08.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:08.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:08.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:44:08.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:44:08.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:44:08.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:08 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:44:09.120 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:44:08 smithi156 conmon[179783]: ts=2023-10-09T05:44:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:44:10.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:10 smithi156 ceph-mon[105979]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:10.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:10 smithi064 ceph-mon[212750]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:10.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:10 smithi064 ceph-mon[118354]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:12.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:12 smithi156 ceph-mon[105979]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:12.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:12 smithi064 ceph-mon[212750]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:12.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:12 smithi064 ceph-mon[118354]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:14 smithi156 ceph-mon[105979]: pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:14 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:14.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:14 smithi064 ceph-mon[212750]: pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:14.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:14.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:14 smithi064 ceph-mon[118354]: pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:14.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:14 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:15.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:44:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:44:15] "GET /metrics HTTP/1.1" 200 33792 "" "Prometheus/2.43.0" 2023-10-09T05:44:16.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:16 smithi156 ceph-mon[105979]: pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:16.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:16 smithi064 ceph-mon[212750]: pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:16.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:16 smithi064 ceph-mon[118354]: pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:18.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:18 smithi156 ceph-mon[105979]: pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:18.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:18 smithi064 ceph-mon[212750]: pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:18.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:18 smithi064 ceph-mon[118354]: pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:19.120 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:44:18 smithi156 conmon[179783]: ts=2023-10-09T05:44:18.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:44:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:20 smithi156 ceph-mon[105979]: pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:20.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:20 smithi064 ceph-mon[212750]: pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:20.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:20 smithi064 ceph-mon[118354]: pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:21.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:21 smithi156 ceph-mon[105979]: pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:21.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:21 smithi064 ceph-mon[212750]: pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:21.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:21 smithi064 ceph-mon[118354]: pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:24.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:23 smithi064 ceph-mon[212750]: pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:24.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:23 smithi064 ceph-mon[118354]: pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:24.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:23 smithi156 ceph-mon[105979]: pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:25.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:44:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:44:25] "GET /metrics HTTP/1.1" 200 33792 "" "Prometheus/2.43.0" 2023-10-09T05:44:26.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:25 smithi064 ceph-mon[212750]: pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:26.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:25 smithi064 ceph-mon[118354]: pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:26.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:25 smithi156 ceph-mon[105979]: pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:28.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:27 smithi064 ceph-mon[212750]: pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:28.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:27 smithi064 ceph-mon[118354]: pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:28.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:27 smithi156 ceph-mon[105979]: pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:28.969 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:44:28 smithi156 conmon[179783]: ts=2023-10-09T05:44:28.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:44:29.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:28 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:29.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:28 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:29.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:28 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:30.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:29 smithi156 ceph-mon[105979]: pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:30.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:29 smithi064 ceph-mon[212750]: pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:30.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:29 smithi064 ceph-mon[118354]: pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:32.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:31 smithi156 ceph-mon[105979]: pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:32.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:31 smithi064 ceph-mon[212750]: pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:32.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:31 smithi064 ceph-mon[118354]: pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:34.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:33 smithi156 ceph-mon[105979]: pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:34.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:33 smithi064 ceph-mon[212750]: pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:34.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:33 smithi064 ceph-mon[118354]: pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:35.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:44:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:44:35] "GET /metrics HTTP/1.1" 200 33792 "" "Prometheus/2.43.0" 2023-10-09T05:44:36.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:35 smithi156 ceph-mon[105979]: pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:36.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:35 smithi064 ceph-mon[212750]: pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:36.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:35 smithi064 ceph-mon[118354]: pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:38.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:37 smithi156 ceph-mon[105979]: pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:38.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:37 smithi064 ceph-mon[212750]: pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:38.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:37 smithi064 ceph-mon[118354]: pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:39.120 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:44:38 smithi156 conmon[179783]: ts=2023-10-09T05:44:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:44:40.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:39 smithi156 ceph-mon[105979]: pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:40.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:39 smithi064 ceph-mon[212750]: pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:40.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:39 smithi064 ceph-mon[118354]: pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:42.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:41 smithi156 ceph-mon[105979]: pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:42.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:41 smithi064 ceph-mon[212750]: pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:42.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:41 smithi064 ceph-mon[118354]: pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:44.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:44 smithi156 ceph-mon[105979]: pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:44.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:44 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:44.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:44 smithi064 ceph-mon[118354]: pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:44.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:44 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:44.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:44 smithi064 ceph-mon[212750]: pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:44.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:45.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:44:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:44:45] "GET /metrics HTTP/1.1" 200 33791 "" "Prometheus/2.43.0" 2023-10-09T05:44:46.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:46 smithi156 ceph-mon[105979]: pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:46.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:46 smithi064 ceph-mon[212750]: pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:46.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:46 smithi064 ceph-mon[118354]: pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:48.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:48 smithi156 ceph-mon[105979]: pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:48.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:48 smithi064 ceph-mon[212750]: pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:48.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:48 smithi064 ceph-mon[118354]: pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:49.120 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:44:48 smithi156 conmon[179783]: ts=2023-10-09T05:44:48.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:44:50.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:50 smithi156 ceph-mon[105979]: pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:50.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:50 smithi064 ceph-mon[212750]: pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:50.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:50 smithi064 ceph-mon[118354]: pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:52.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:52 smithi156 ceph-mon[105979]: pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:52.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:52 smithi064 ceph-mon[212750]: pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:52.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:52 smithi064 ceph-mon[118354]: pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:54.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:54 smithi156 ceph-mon[105979]: pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:54.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:54 smithi064 ceph-mon[212750]: pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:54.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:54 smithi064 ceph-mon[118354]: pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:55.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:44:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:44:55] "GET /metrics HTTP/1.1" 200 33791 "" "Prometheus/2.43.0" 2023-10-09T05:44:56.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:56 smithi156 ceph-mon[105979]: pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:56.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:56 smithi064 ceph-mon[212750]: pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:56.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:56 smithi064 ceph-mon[118354]: pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:58.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:58 smithi156 ceph-mon[105979]: pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:58.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:58 smithi064 ceph-mon[212750]: pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:58.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:58 smithi064 ceph-mon[118354]: pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:44:59.028 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:44:58 smithi156 conmon[179783]: ts=2023-10-09T05:44:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:44:59.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:44:59 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:59.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:44:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:44:59.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:44:59 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:00.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:00 smithi156 ceph-mon[105979]: pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:00 smithi064 ceph-mon[212750]: pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:00.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:00 smithi064 ceph-mon[118354]: pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:02.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:02 smithi156 ceph-mon[105979]: pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:02.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:02 smithi064 ceph-mon[212750]: pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:02.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:02 smithi064 ceph-mon[118354]: pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:04 smithi156 ceph-mon[105979]: pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:04.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:04 smithi064 ceph-mon[212750]: pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:04.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:04 smithi064 ceph-mon[118354]: pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:05.630 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:45:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:45:05] "GET /metrics HTTP/1.1" 200 33789 "" "Prometheus/2.43.0" 2023-10-09T05:45:05.989 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T05:45:06.367 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:06 smithi156 ceph-mon[105979]: pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:06 smithi064 ceph-mon[212750]: pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:06.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:06 smithi064 ceph-mon[118354]: pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:08.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:08 smithi064 ceph-mon[212750]: pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:08.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:08 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:45:08.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:08 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:08.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:08 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:08.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:08 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:08.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:08 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:08.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:08 smithi064 ceph-mon[118354]: pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:08.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:08 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:45:08.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:08 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:08.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:08 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:08.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:08 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:08.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:08 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:08.313 INFO:teuthology.orchestra.run.smithi064.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:45:08.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:08 smithi156 ceph-mon[105979]: pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:08.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:08 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:45:08.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:08 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:08.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:08 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:08.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:08 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:08.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:08 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:09.121 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:45:08 smithi156 conmon[179783]: ts=2023-10-09T05:45:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:45:09.351 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for majority of mons to upgrade' 2023-10-09T05:45:09.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:09 smithi156 ceph-mon[105979]: from='client.25241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:45:09.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:09 smithi156 ceph-mon[105979]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:45:09.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:09 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:09.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:09 smithi156 ceph-mon[105979]: pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:09.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:09 smithi064 ceph-mon[212750]: from='client.25241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:45:09.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:09 smithi064 ceph-mon[212750]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:45:09.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:09 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:09.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:09 smithi064 ceph-mon[212750]: pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:09.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:09 smithi064 ceph-mon[118354]: from='client.25241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:45:09.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:09 smithi064 ceph-mon[118354]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:45:09.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:09 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:09.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:09 smithi064 ceph-mon[118354]: pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:09.824 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:11.925 INFO:teuthology.orchestra.run.smithi064.stdout:wait for majority of mons to upgrade 2023-10-09T05:45:11.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:11 smithi064 ceph-mon[212750]: pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:11.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:11 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:11.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:11 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:11.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:11 smithi064 ceph-mon[118354]: pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:11.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:11 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:11.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:11 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:12.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:11 smithi156 ceph-mon[105979]: pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:12.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:11 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:12.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:11 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:12.610 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-10-09T05:45:12.984 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:13.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:13 smithi156 ceph-mon[105979]: pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:13.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:13 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:13.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:13 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:13.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:13 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:13.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:13 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:45:13.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:13 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:13.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:13 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:45:13.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:13 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:45:13.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:13 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T05:45:13.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:13 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T05:45:13.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:13 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:13.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:13 smithi064 ceph-mon[212750]: pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:13.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:13 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:13.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:13 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:13.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:13 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:13.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:13 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:45:13.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:13 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:13.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:13 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:45:13.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:13 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:45:13.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:13 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T05:45:13.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:13 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T05:45:13.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:13 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:13.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:13 smithi064 ceph-mon[118354]: pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:13.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:13 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:13.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:13 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:13.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:13 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:13.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:13 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:45:13.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:13 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:13.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:13 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:45:13.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:13 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:45:13.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:13 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T05:45:13.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:13 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T05:45:13.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:13 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:14.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:14 smithi064 ceph-mon[212750]: Upgrade: It appears safe to stop mon.c 2023-10-09T05:45:14.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:14 smithi064 ceph-mon[118354]: Upgrade: It appears safe to stop mon.c 2023-10-09T05:45:14.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:14 smithi156 ceph-mon[105979]: Upgrade: It appears safe to stop mon.c 2023-10-09T05:45:15.563 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:45:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:45:15] "GET /metrics HTTP/1.1" 200 33790 "" "Prometheus/2.43.0" 2023-10-09T05:45:15.564 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:15 smithi064 ceph-mon[118354]: pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:15.564 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:15 smithi064 ceph-mon[118354]: from='client.? 172.21.15.64:0/191869258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:45:15.564 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:15 smithi064 ceph-mon[118354]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:15.564 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:15 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:45:15.565 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:15 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:45:15.565 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:15 smithi064 ceph-mon[118354]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:15.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:15 smithi156 ceph-mon[105979]: pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:15.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:15 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/191869258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:45:15.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:15 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:15 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:45:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:15 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:45:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:15 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:15.925 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:15 smithi064 ceph-mon[212750]: pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:15.926 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:15 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/191869258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:45:15.926 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:15 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:15.926 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:15 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:45:15.926 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:15 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:45:15.926 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:15 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:16 smithi156 ceph-mon[105979]: Upgrade: Updating mon.c 2023-10-09T05:45:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:16 smithi156 ceph-mon[105979]: Deploying daemon mon.c on smithi064 2023-10-09T05:45:16.953 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:16 smithi064 ceph-mon[118354]: Upgrade: Updating mon.c 2023-10-09T05:45:16.953 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:16 smithi064 ceph-mon[118354]: Deploying daemon mon.c on smithi064 2023-10-09T05:45:16.953 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:16 smithi064 systemd[1]: Stopping Ceph mon.c for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:45:16.954 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:16 smithi064 conmon[118330]: 2023-10-09T05:45:16.949+0000 7f1361153700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T05:45:16.954 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:16 smithi064 conmon[118330]: 2023-10-09T05:45:16.949+0000 7f1361153700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-10-09T05:45:16.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:16 smithi064 ceph-mon[212750]: Upgrade: Updating mon.c 2023-10-09T05:45:16.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:16 smithi064 ceph-mon[212750]: Deploying daemon mon.c on smithi064 2023-10-09T05:45:17.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:45:16 smithi064 conmon[198859]: [09/Oct/2023:05:45:16] ENGINE Bus STOPPING 2023-10-09T05:45:17.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:45:16 smithi064 conmon[198859]: [09/Oct/2023:05:45:16] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:45:17.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:45:16 smithi064 conmon[198859]: [09/Oct/2023:05:45:16] ENGINE Bus STOPPED 2023-10-09T05:45:17.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:45:16 smithi064 conmon[198859]: [09/Oct/2023:05:45:16] ENGINE Bus STARTING 2023-10-09T05:45:17.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:45:17 smithi064 conmon[198859]: [09/Oct/2023:05:45:17] ENGINE Serving on http://:::9283 2023-10-09T05:45:17.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:45:17 smithi064 conmon[198859]: [09/Oct/2023:05:45:17] ENGINE Bus STARTED 2023-10-09T05:45:17.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:17 smithi064 bash[219580]: b6790a9456bae7dc3eaff42c53aa886f88fad291308ad3e786e951c2d3d255e8 2023-10-09T05:45:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:17 smithi064 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mon.c.service: Succeeded. 2023-10-09T05:45:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:17 smithi064 systemd[1]: Stopped Ceph mon.c for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:45:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 systemd[1]: Starting Ceph mon.c for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:45:18.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T05:45:18.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T05:45:18.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: pidfile_write: ignore empty --pid-file 2023-10-09T05:45:18.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: load: jerasure load: lrc 2023-10-09T05:45:18.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: RocksDB version: 7.9.2 2023-10-09T05:45:18.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Git sha 0 2023-10-09T05:45:18.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T05:45:18.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: DB SUMMARY 2023-10-09T05:45:18.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: DB Session ID: VGQ2NKDSN42ZP7LLVBOG 2023-10-09T05:45:18.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: CURRENT file: CURRENT 2023-10-09T05:45:18.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T05:45:18.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1096 Bytes 2023-10-09T05:45:18.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2023-10-09T05:45:18.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 3915241 ; 2023-10-09T05:45:18.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.error_if_exists: 0 2023-10-09T05:45:18.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.create_if_missing: 0 2023-10-09T05:45:18.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.paranoid_checks: 1 2023-10-09T05:45:18.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T05:45:18.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T05:45:18.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T05:45:18.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.env: 0x560f66953720 2023-10-09T05:45:18.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.fs: PosixFileSystem 2023-10-09T05:45:18.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.info_log: 0x560f6823d420 2023-10-09T05:45:18.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T05:45:18.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.statistics: (nil) 2023-10-09T05:45:18.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.use_fsync: 0 2023-10-09T05:45:18.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_log_file_size: 0 2023-10-09T05:45:18.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T05:45:18.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T05:45:18.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T05:45:18.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T05:45:18.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.allow_fallocate: 1 2023-10-09T05:45:18.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T05:45:18.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T05:45:18.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.use_direct_reads: 0 2023-10-09T05:45:18.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T05:45:18.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T05:45:18.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.db_log_dir: 2023-10-09T05:45:18.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.wal_dir: 2023-10-09T05:45:18.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T05:45:18.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T05:45:18.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T05:45:18.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T05:45:18.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T05:45:18.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T05:45:18.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T05:45:18.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T05:45:18.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.write_buffer_manager: 0x560f674ccc80 2023-10-09T05:45:18.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T05:45:18.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T05:45:18.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T05:45:18.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T05:45:18.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T05:45:18.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T05:45:18.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T05:45:18.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T05:45:18.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.unordered_write: 0 2023-10-09T05:45:18.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T05:45:18.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T05:45:18.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T05:45:18.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T05:45:18.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.row_cache: None 2023-10-09T05:45:18.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.wal_filter: None 2023-10-09T05:45:18.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T05:45:18.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T05:45:18.977 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.two_write_queues: 0 2023-10-09T05:45:18.977 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T05:45:18.977 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.wal_compression: 0 2023-10-09T05:45:18.977 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.atomic_flush: 0 2023-10-09T05:45:18.977 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T05:45:18.977 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T05:45:18.977 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T05:45:18.977 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.log_readahead_size: 0 2023-10-09T05:45:18.978 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T05:45:18.978 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T05:45:18.978 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T05:45:18.978 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T05:45:18.978 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T05:45:18.978 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T05:45:18.978 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T05:45:18.978 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_background_jobs: 2 2023-10-09T05:45:18.978 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_background_compactions: -1 2023-10-09T05:45:18.978 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_subcompactions: 1 2023-10-09T05:45:18.978 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T05:45:18.978 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T05:45:18.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T05:45:18.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T05:45:18.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T05:45:18.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T05:45:18.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T05:45:18.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T05:45:18.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_open_files: -1 2023-10-09T05:45:18.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T05:45:18.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T05:45:18.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T05:45:18.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T05:45:18.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_background_flushes: -1 2023-10-09T05:45:18.980 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Compression algorithms supported: 2023-10-09T05:45:18.980 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T05:45:18.980 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: kZSTD supported: 0 2023-10-09T05:45:18.980 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: kXpressCompression supported: 0 2023-10-09T05:45:18.980 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T05:45:18.980 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: kZlibCompression supported: 1 2023-10-09T05:45:18.980 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: kSnappyCompression supported: 1 2023-10-09T05:45:18.980 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: kLZ4Compression supported: 1 2023-10-09T05:45:18.980 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: kBZip2Compression supported: 0 2023-10-09T05:45:18.980 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T05:45:18.980 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T05:45:18.980 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2023-10-09T05:45:18.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T05:45:18.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T05:45:18.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.merge_operator: 2023-10-09T05:45:18.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.compaction_filter: None 2023-10-09T05:45:18.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T05:45:18.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T05:45:18.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T05:45:18.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T05:45:18.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560f6823d520) 2023-10-09T05:45:18.981 INFO:journalctl@ceph.mon.c.smithi064.stdout: cache_index_and_filter_blocks: 1 2023-10-09T05:45:18.981 INFO:journalctl@ceph.mon.c.smithi064.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T05:45:18.981 INFO:journalctl@ceph.mon.c.smithi064.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T05:45:18.982 INFO:journalctl@ceph.mon.c.smithi064.stdout: pin_top_level_index_and_filter: 1 2023-10-09T05:45:18.982 INFO:journalctl@ceph.mon.c.smithi064.stdout: index_type: 0 2023-10-09T05:45:18.982 INFO:journalctl@ceph.mon.c.smithi064.stdout: data_block_index_type: 0 2023-10-09T05:45:18.982 INFO:journalctl@ceph.mon.c.smithi064.stdout: index_shortening: 1 2023-10-09T05:45:18.982 INFO:journalctl@ceph.mon.c.smithi064.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T05:45:18.982 INFO:journalctl@ceph.mon.c.smithi064.stdout: checksum: 4 2023-10-09T05:45:18.982 INFO:journalctl@ceph.mon.c.smithi064.stdout: no_block_cache: 0 2023-10-09T05:45:18.982 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_cache: 0x560f6754f1f0 2023-10-09T05:45:18.982 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_cache_name: BinnedLRUCache 2023-10-09T05:45:18.982 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_cache_options: 2023-10-09T05:45:18.982 INFO:journalctl@ceph.mon.c.smithi064.stdout: capacity : 536870912 2023-10-09T05:45:18.982 INFO:journalctl@ceph.mon.c.smithi064.stdout: num_shard_bits : 4 2023-10-09T05:45:18.983 INFO:journalctl@ceph.mon.c.smithi064.stdout: strict_capacity_limit : 0 2023-10-09T05:45:18.983 INFO:journalctl@ceph.mon.c.smithi064.stdout: high_pri_pool_ratio: 0.000 2023-10-09T05:45:18.983 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_cache_compressed: (nil) 2023-10-09T05:45:18.983 INFO:journalctl@ceph.mon.c.smithi064.stdout: persistent_cache: (nil) 2023-10-09T05:45:18.983 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_size: 4096 2023-10-09T05:45:18.983 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_size_deviation: 10 2023-10-09T05:45:18.983 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_restart_interval: 16 2023-10-09T05:45:18.983 INFO:journalctl@ceph.mon.c.smithi064.stdout: index_block_restart_interval: 1 2023-10-09T05:45:18.983 INFO:journalctl@ceph.mon.c.smithi064.stdout: metadata_block_size: 4096 2023-10-09T05:45:18.983 INFO:journalctl@ceph.mon.c.smithi064.stdout: partition_filters: 0 2023-10-09T05:45:18.983 INFO:journalctl@ceph.mon.c.smithi064.stdout: use_delta_encoding: 1 2023-10-09T05:45:18.983 INFO:journalctl@ceph.mon.c.smithi064.stdout: filter_policy: bloomfilter 2023-10-09T05:45:18.984 INFO:journalctl@ceph.mon.c.smithi064.stdout: whole_key_filtering: 1 2023-10-09T05:45:18.984 INFO:journalctl@ceph.mon.c.smithi064.stdout: verify_compression: 0 2023-10-09T05:45:18.984 INFO:journalctl@ceph.mon.c.smithi064.stdout: read_amp_bytes_per_bit: 0 2023-10-09T05:45:18.984 INFO:journalctl@ceph.mon.c.smithi064.stdout: format_version: 5 2023-10-09T05:45:18.984 INFO:journalctl@ceph.mon.c.smithi064.stdout: enable_index_compression: 1 2023-10-09T05:45:18.984 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_align: 0 2023-10-09T05:45:18.984 INFO:journalctl@ceph.mon.c.smithi064.stdout: max_auto_readahead_size: 262144 2023-10-09T05:45:18.984 INFO:journalctl@ceph.mon.c.smithi064.stdout: prepopulate_block_cache: 0 2023-10-09T05:45:18.984 INFO:journalctl@ceph.mon.c.smithi064.stdout: initial_auto_readahead_size: 8192 2023-10-09T05:45:18.984 INFO:journalctl@ceph.mon.c.smithi064.stdout: num_file_reads_for_auto_readahead: 2 2023-10-09T05:45:18.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T05:45:18.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T05:45:18.985 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.compression: NoCompression 2023-10-09T05:45:18.985 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T05:45:18.985 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T05:45:18.985 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T05:45:18.985 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.num_levels: 7 2023-10-09T05:45:18.985 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T05:45:18.985 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T05:45:18.985 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T05:45:18.985 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T05:45:18.985 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T05:45:18.985 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T05:45:18.985 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T05:45:18.986 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:45:18.986 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T05:45:18.986 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T05:45:18.986 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T05:45:18.986 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T05:45:18.986 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T05:45:18.986 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T05:45:18.986 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T05:45:18.986 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T05:45:18.986 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:45:18.986 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T05:45:18.987 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T05:45:18.987 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T05:45:18.987 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T05:45:18.987 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T05:45:18.987 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T05:45:18.987 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T05:45:18.987 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T05:45:18.987 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T05:45:18.987 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T05:45:18.987 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T05:45:18.987 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T05:45:18.987 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T05:45:18.988 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T05:45:18.988 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T05:45:18.988 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T05:45:18.988 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T05:45:18.988 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T05:45:18.988 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T05:45:18.988 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T05:45:18.988 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T05:45:18.988 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T05:45:18.988 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.arena_block_size: 1048576 2023-10-09T05:45:18.988 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T05:45:18.988 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T05:45:18.989 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T05:45:18.989 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T05:45:18.990 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T05:45:18.990 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T05:45:18.990 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T05:45:18.990 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T05:45:18.990 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T05:45:18.990 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T05:45:18.990 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T05:45:18.990 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T05:45:18.990 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T05:45:18.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.table_properties_collectors: 2023-10-09T05:45:18.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.inplace_update_support: 0 2023-10-09T05:45:18.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T05:45:18.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T05:45:18.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T05:45:18.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T05:45:18.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.bloom_locality: 0 2023-10-09T05:45:18.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.max_successive_merges: 0 2023-10-09T05:45:18.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T05:45:18.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T05:45:18.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T05:45:18.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T05:45:18.992 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.ttl: 2592000 2023-10-09T05:45:18.992 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T05:45:18.992 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T05:45:18.992 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T05:45:18.992 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.enable_blob_files: false 2023-10-09T05:45:18.992 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.min_blob_size: 0 2023-10-09T05:45:18.992 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T05:45:18.992 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T05:45:18.992 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T05:45:18.992 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T05:45:18.992 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T05:45:18.992 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T05:45:18.993 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.blob_file_starting_level: 0 2023-10-09T05:45:18.993 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T05:45:18.993 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-10-09T05:45:18.993 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T05:45:18.993 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12962, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T05:45:18.993 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-10-09T05:45:18.993 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 9d335a72-2f19-456c-b884-2132a5374b29 2023-10-09T05:45:18.993 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696830318584682, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-10-09T05:45:18.993 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-10-09T05:45:18.993 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696830318594181, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 2465123, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12964, "largest_seqno": 14487, "table_properties": {"data_size": 2458933, "index_size": 3763, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1413, "raw_key_size": 12807, "raw_average_key_size": 22, "raw_value_size": 2446879, "raw_average_value_size": 4369, "num_data_blocks": 171, "num_entries": 560, "num_filter_entries": 560, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696830318, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "9d335a72-2f19-456c-b884-2132a5374b29", "db_session_id": "VGQ2NKDSN42ZP7LLVBOG", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-10-09T05:45:18.993 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696830318594265, "job": 1, "event": "recovery_finished"} 2023-10-09T05:45:18.994 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-10-09T05:45:18.994 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T05:45:18.994 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T05:45:18.994 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x560f675ea000 2023-10-09T05:45:18.994 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: DB pointer 0x560f675d8000 2023-10-09T05:45:18.994 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T05:45:18.994 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T05:45:18.994 INFO:journalctl@ceph.mon.c.smithi064.stdout: ** DB Stats ** 2023-10-09T05:45:18.994 INFO:journalctl@ceph.mon.c.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:45:18.994 INFO:journalctl@ceph.mon.c.smithi064.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T05:45:18.994 INFO:journalctl@ceph.mon.c.smithi064.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:45:18.994 INFO:journalctl@ceph.mon.c.smithi064.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:45:18.995 INFO:journalctl@ceph.mon.c.smithi064.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T05:45:18.995 INFO:journalctl@ceph.mon.c.smithi064.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:45:18.995 INFO:journalctl@ceph.mon.c.smithi064.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:45:18.995 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-10-09T05:45:18.995 INFO:journalctl@ceph.mon.c.smithi064.stdout: ** Compaction Stats [default] ** 2023-10-09T05:45:18.995 INFO:journalctl@ceph.mon.c.smithi064.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T05:45:18.995 INFO:journalctl@ceph.mon.c.smithi064.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T05:45:18.995 INFO:journalctl@ceph.mon.c.smithi064.stdout: L0 1/0 2.35 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 677.7 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T05:45:18.995 INFO:journalctl@ceph.mon.c.smithi064.stdout: L6 1/0 6.33 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T05:45:18.995 INFO:journalctl@ceph.mon.c.smithi064.stdout: Sum 2/0 8.68 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 677.7 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T05:45:18.995 INFO:journalctl@ceph.mon.c.smithi064.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 677.7 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T05:45:18.995 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-10-09T05:45:18.996 INFO:journalctl@ceph.mon.c.smithi064.stdout: ** Compaction Stats [default] ** 2023-10-09T05:45:18.996 INFO:journalctl@ceph.mon.c.smithi064.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T05:45:18.996 INFO:journalctl@ceph.mon.c.smithi064.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:45:18.996 INFO:journalctl@ceph.mon.c.smithi064.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 677.7 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T05:45:18.996 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-10-09T05:45:18.996 INFO:journalctl@ceph.mon.c.smithi064.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T05:45:18.996 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-10-09T05:45:18.996 INFO:journalctl@ceph.mon.c.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:45:18.996 INFO:journalctl@ceph.mon.c.smithi064.stdout: Flush(GB): cumulative 0.002, interval 0.002 2023-10-09T05:45:18.996 INFO:journalctl@ceph.mon.c.smithi064.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:45:18.996 INFO:journalctl@ceph.mon.c.smithi064.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:45:18.996 INFO:journalctl@ceph.mon.c.smithi064.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:45:18.997 INFO:journalctl@ceph.mon.c.smithi064.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:45:18.997 INFO:journalctl@ceph.mon.c.smithi064.stdout: Cumulative compaction: 0.00 GB write, 166.32 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:45:18.997 INFO:journalctl@ceph.mon.c.smithi064.stdout: Interval compaction: 0.00 GB write, 166.32 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:45:18.997 INFO:journalctl@ceph.mon.c.smithi064.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:45:18.997 INFO:journalctl@ceph.mon.c.smithi064.stdout: Block cache BinnedLRUCache@0x560f6754f1f0#7 capacity: 512.00 MB usage: 5.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2023-10-09T05:45:18.997 INFO:journalctl@ceph.mon.c.smithi064.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2023-10-09T05:45:18.997 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-10-09T05:45:18.997 INFO:journalctl@ceph.mon.c.smithi064.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:45:18.997 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: starting mon.c rank 2 at public addrs [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] at bind addrs [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:45:18.997 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: mon.c@-1(???) e3 preinit fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:45:18.997 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: mon.c@-1(???).mds e8 new map 2023-10-09T05:45:18.998 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: mon.c@-1(???).mds e8 print_map 2023-10-09T05:45:18.998 INFO:journalctl@ceph.mon.c.smithi064.stdout: e8 2023-10-09T05:45:18.998 INFO:journalctl@ceph.mon.c.smithi064.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T05:45:18.998 INFO:journalctl@ceph.mon.c.smithi064.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T05:45:18.998 INFO:journalctl@ceph.mon.c.smithi064.stdout: legacy client fscid: 1 2023-10-09T05:45:18.998 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-10-09T05:45:18.998 INFO:journalctl@ceph.mon.c.smithi064.stdout: Filesystem 'foo' (1) 2023-10-09T05:45:18.998 INFO:journalctl@ceph.mon.c.smithi064.stdout: fs_name foo 2023-10-09T05:45:18.998 INFO:journalctl@ceph.mon.c.smithi064.stdout: epoch 7 2023-10-09T05:45:18.998 INFO:journalctl@ceph.mon.c.smithi064.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T05:45:18.998 INFO:journalctl@ceph.mon.c.smithi064.stdout: created 2023-10-09T05:40:12.634524+0000 2023-10-09T05:45:18.998 INFO:journalctl@ceph.mon.c.smithi064.stdout: modified 2023-10-09T05:40:21.631615+0000 2023-10-09T05:45:18.999 INFO:journalctl@ceph.mon.c.smithi064.stdout: tableserver 0 2023-10-09T05:45:18.999 INFO:journalctl@ceph.mon.c.smithi064.stdout: root 0 2023-10-09T05:45:18.999 INFO:journalctl@ceph.mon.c.smithi064.stdout: session_timeout 60 2023-10-09T05:45:18.999 INFO:journalctl@ceph.mon.c.smithi064.stdout: session_autoclose 300 2023-10-09T05:45:18.999 INFO:journalctl@ceph.mon.c.smithi064.stdout: max_file_size 1099511627776 2023-10-09T05:45:18.999 INFO:journalctl@ceph.mon.c.smithi064.stdout: required_client_features {} 2023-10-09T05:45:18.999 INFO:journalctl@ceph.mon.c.smithi064.stdout: last_failure 0 2023-10-09T05:45:18.999 INFO:journalctl@ceph.mon.c.smithi064.stdout: last_failure_osd_epoch 0 2023-10-09T05:45:18.999 INFO:journalctl@ceph.mon.c.smithi064.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T05:45:18.999 INFO:journalctl@ceph.mon.c.smithi064.stdout: max_mds 1 2023-10-09T05:45:18.999 INFO:journalctl@ceph.mon.c.smithi064.stdout: in 0 2023-10-09T05:45:18.999 INFO:journalctl@ceph.mon.c.smithi064.stdout: up {0=24551} 2023-10-09T05:45:19.000 INFO:journalctl@ceph.mon.c.smithi064.stdout: failed 2023-10-09T05:45:19.000 INFO:journalctl@ceph.mon.c.smithi064.stdout: damaged 2023-10-09T05:45:19.000 INFO:journalctl@ceph.mon.c.smithi064.stdout: stopped 2023-10-09T05:45:19.000 INFO:journalctl@ceph.mon.c.smithi064.stdout: data_pools [4] 2023-10-09T05:45:19.000 INFO:journalctl@ceph.mon.c.smithi064.stdout: metadata_pool 3 2023-10-09T05:45:19.000 INFO:journalctl@ceph.mon.c.smithi064.stdout: inline_data disabled 2023-10-09T05:45:19.000 INFO:journalctl@ceph.mon.c.smithi064.stdout: balancer 2023-10-09T05:45:19.000 INFO:journalctl@ceph.mon.c.smithi064.stdout: bal_rank_mask -1 2023-10-09T05:45:19.000 INFO:journalctl@ceph.mon.c.smithi064.stdout: standby_count_wanted 1 2023-10-09T05:45:19.000 INFO:journalctl@ceph.mon.c.smithi064.stdout: [mds.foo.smithi064.gmrwsy{0:24551} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.64:6834/3975184368,v1:172.21.15.64:6835/3975184368] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T05:45:19.000 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-10-09T05:45:19.000 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-10-09T05:45:19.001 INFO:journalctl@ceph.mon.c.smithi064.stdout: Standby daemons: 2023-10-09T05:45:19.001 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-10-09T05:45:19.001 INFO:journalctl@ceph.mon.c.smithi064.stdout: [mds.foo.smithi156.pizbdw{-1:24556} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.156:6832/4137868430,v1:172.21.15.156:6833/4137868430] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T05:45:19.001 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: mon.c@-1(???).osd e80 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T05:45:19.001 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: mon.c@-1(???).osd e80 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T05:45:19.001 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: mon.c@-1(???).osd e80 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T05:45:19.001 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: mon.c@-1(???).osd e80 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T05:45:19.001 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: mon.c@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2023-10-09T05:45:19.001 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 bash[219979]: af10058879c1fabd0719b057e1e100a54872f5bd391aae294618c969e15d492c 2023-10-09T05:45:19.001 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: expand_channel_meta expand map: {default=false} 2023-10-09T05:45:19.001 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: expand_channel_meta from 'false' to 'false' 2023-10-09T05:45:19.001 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: expand_channel_meta expanded map: {default=false} 2023-10-09T05:45:19.002 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: expand_channel_meta expand map: {default=info} 2023-10-09T05:45:19.002 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: expand_channel_meta from 'info' to 'info' 2023-10-09T05:45:19.002 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: expand_channel_meta expanded map: {default=info} 2023-10-09T05:45:19.002 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: expand_channel_meta expand map: {default=daemon} 2023-10-09T05:45:19.002 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T05:45:19.002 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: expand_channel_meta expanded map: {default=daemon} 2023-10-09T05:45:19.002 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: expand_channel_meta expand map: {default=debug} 2023-10-09T05:45:19.002 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: expand_channel_meta from 'debug' to 'debug' 2023-10-09T05:45:19.002 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 ceph-mon[220044]: expand_channel_meta expanded map: {default=debug} 2023-10-09T05:45:19.002 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:18 smithi064 systemd[1]: Started Ceph mon.c for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:45:19.121 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:45:18 smithi156 conmon[179783]: ts=2023-10-09T05:45:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:45:25.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:24 smithi064 ceph-mon[212750]: mon.c calling monitor election 2023-10-09T05:45:25.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:24 smithi064 ceph-mon[212750]: mon.a calling monitor election 2023-10-09T05:45:25.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:24 smithi064 ceph-mon[212750]: mon.b calling monitor election 2023-10-09T05:45:25.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:24 smithi064 ceph-mon[212750]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:45:25.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:24 smithi064 ceph-mon[212750]: monmap e3: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} 2023-10-09T05:45:25.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:24 smithi064 ceph-mon[212750]: fsmap foo:1 {0=foo.smithi064.gmrwsy=up:active} 1 up:standby 2023-10-09T05:45:25.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:24 smithi064 ceph-mon[212750]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T05:45:25.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:24 smithi064 ceph-mon[212750]: mgrmap e36: y(active, since 86s), standbys: x 2023-10-09T05:45:25.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:24 smithi064 ceph-mon[212750]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T05:45:25.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:24 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T05:45:25.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:24 smithi064 ceph-mon[212750]: overall HEALTH_OK 2023-10-09T05:45:25.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:24 smithi064 ceph-mon[220044]: mon.c calling monitor election 2023-10-09T05:45:25.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:24 smithi064 ceph-mon[220044]: mon.a calling monitor election 2023-10-09T05:45:25.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:24 smithi064 ceph-mon[220044]: mon.b calling monitor election 2023-10-09T05:45:25.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:24 smithi064 ceph-mon[220044]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:45:25.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:24 smithi064 ceph-mon[220044]: monmap e3: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} 2023-10-09T05:45:25.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:24 smithi064 ceph-mon[220044]: fsmap foo:1 {0=foo.smithi064.gmrwsy=up:active} 1 up:standby 2023-10-09T05:45:25.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:24 smithi064 ceph-mon[220044]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T05:45:25.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:24 smithi064 ceph-mon[220044]: mgrmap e36: y(active, since 86s), standbys: x 2023-10-09T05:45:25.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:24 smithi064 ceph-mon[220044]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T05:45:25.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:24 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T05:45:25.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:24 smithi064 ceph-mon[220044]: overall HEALTH_OK 2023-10-09T05:45:25.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:24 smithi156 ceph-mon[105979]: mon.c calling monitor election 2023-10-09T05:45:25.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:24 smithi156 ceph-mon[105979]: mon.a calling monitor election 2023-10-09T05:45:25.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:24 smithi156 ceph-mon[105979]: mon.b calling monitor election 2023-10-09T05:45:25.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:24 smithi156 ceph-mon[105979]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:45:25.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:24 smithi156 ceph-mon[105979]: monmap e3: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} 2023-10-09T05:45:25.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:24 smithi156 ceph-mon[105979]: fsmap foo:1 {0=foo.smithi064.gmrwsy=up:active} 1 up:standby 2023-10-09T05:45:25.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:24 smithi156 ceph-mon[105979]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T05:45:25.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:24 smithi156 ceph-mon[105979]: mgrmap e36: y(active, since 86s), standbys: x 2023-10-09T05:45:25.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:24 smithi156 ceph-mon[105979]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T05:45:25.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:24 smithi156 ceph-mon[105979]: Cluster is now healthy 2023-10-09T05:45:25.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:24 smithi156 ceph-mon[105979]: overall HEALTH_OK 2023-10-09T05:45:25.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:45:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:45:25] "GET /metrics HTTP/1.1" 200 33790 "" "Prometheus/2.43.0" 2023-10-09T05:45:26.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:25 smithi064 ceph-mon[212750]: pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:26.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:25 smithi064 ceph-mon[220044]: pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:26.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:25 smithi156 ceph-mon[105979]: pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:26.677 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 2 2023-10-09T05:45:27.445 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T05:45:27.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:27 smithi064 ceph-mon[212750]: pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:27.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:27 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:27.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:27 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:27.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:27 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2505833253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:45:27.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:27 smithi064 ceph-mon[220044]: pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:27.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:27 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:27.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:27 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:27.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:27 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2505833253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:45:27.818 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:27.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:27 smithi156 ceph-mon[105979]: pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:27.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:27 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:27.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:27 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:27 smithi156 ceph-mon[105979]: from='client.? 172.21.15.64:0/2505833253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:45:28.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:45:28 smithi064 conmon[198859]: 2023-10-09T05:45:28.605+0000 7fcb872b4700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-10-09T05:45:29.035 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:28 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:29.035 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:28 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:29.035 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:28 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:29.035 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:28 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:29.035 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:28 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:29.035 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:28 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:29.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:28 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:29.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:28 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:29.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:28 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:29.121 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:45:28 smithi156 conmon[179783]: ts=2023-10-09T05:45:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:45:30.000 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:29 smithi064 ceph-mon[212750]: pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:30.001 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:29 smithi064 ceph-mon[220044]: pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:30.054 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T05:45:30.055 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (2m) - 6m 29.7M - 0.25.0 c8568f914cd2 386f8a4e2221 2023-10-09T05:45:30.055 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi156 *:3000 running (2m) - 5m 87.0M - 9.4.7 2c41d148cca3 365834e48ffb 2023-10-09T05:45:30.055 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.gmrwsy smithi064 running (5m) - 5m 17.9M - 17.2.6-1303-gb2903cb0 5517396c50d4 e938ff2e972d 2023-10-09T05:45:30.055 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi156.pizbdw smithi156 running (5m) - 5m 17.5M - 17.2.6-1303-gb2903cb0 5517396c50d4 e8906e08c938 2023-10-09T05:45:30.055 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi156 *:8443,9283,8765 running (115s) - 9m 447M - 18.2.0-598-gb4fa663f 3060897fdf97 d0092a415024 2023-10-09T05:45:30.055 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:8443,9283,8765 running (2m) - 11m 503M - 18.2.0-598-gb4fa663f 3060897fdf97 2bc7549ee2c1 2023-10-09T05:45:30.055 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (103s) - 11m 48.8M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 5d3854b1b1eb 2023-10-09T05:45:30.055 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi156 running (10m) - 10m 81.1M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 c60a890df99e 2023-10-09T05:45:30.055 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (11s) - 10m 23.8M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 af10058879c1 2023-10-09T05:45:30.055 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (2m) - 6m 17.8M - 1.5.0 0da6a335fe13 afb28622babd 2023-10-09T05:45:30.055 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi156 *:9100 running (2m) - 6m 18.5M - 1.5.0 0da6a335fe13 1e8c1e28630a 2023-10-09T05:45:30.056 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (9m) - 9m 80.0M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 bd3c343a2f7c 2023-10-09T05:45:30.056 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (9m) - 9m 81.5M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 854146c77f6d 2023-10-09T05:45:30.056 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (8m) - 8m 151M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 75a274ca9b6e 2023-10-09T05:45:30.056 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (8m) - 8m 71.8M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 711a9677648c 2023-10-09T05:45:30.056 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi156 running (8m) - 8m 80.3M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 72313d594357 2023-10-09T05:45:30.056 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi156 running (7m) - 7m 102M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 40d9335e2ab7 2023-10-09T05:45:30.056 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi156 running (7m) - 7m 68.0M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 c8a3e3228b89 2023-10-09T05:45:30.056 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi156 running (7m) - 7m 108M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 df93983742a3 2023-10-09T05:45:30.056 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi156 *:9095 running (119s) - 6m 50.6M - 2.43.0 a07b618ecd1d 712dad4d0e3b 2023-10-09T05:45:30.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:29 smithi156 ceph-mon[105979]: pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:30.883 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T05:45:30.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:30 smithi064 ceph-mon[212750]: mgrmap e37: y(active, since 91s), standbys: x 2023-10-09T05:45:30.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:30 smithi064 ceph-mon[212750]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:45:30.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:30 smithi064 ceph-mon[220044]: mgrmap e37: y(active, since 91s), standbys: x 2023-10-09T05:45:30.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:30 smithi064 ceph-mon[220044]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:45:31.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:30 smithi156 ceph-mon[105979]: mgrmap e37: y(active, since 91s), standbys: x 2023-10-09T05:45:31.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:30 smithi156 ceph-mon[105979]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:45:31.249 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:32.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:31 smithi156 ceph-mon[105979]: pgmap v49: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:45:32.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:31 smithi064 ceph-mon[212750]: pgmap v49: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:45:32.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:31 smithi064 ceph-mon[220044]: pgmap v49: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:45:33.431 INFO:teuthology.orchestra.run.smithi064.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:45:34.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:33 smithi156 ceph-mon[105979]: pgmap v50: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:45:34.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:33 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:34.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:33 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:34.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:33 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:34.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:33 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:45:34.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:33 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:34.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:33 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:45:34.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:33 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:45:34.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:33 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T05:45:34.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:33 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T05:45:34.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:33 smithi156 ceph-mon[105979]: Upgrade: It appears safe to stop mon.b 2023-10-09T05:45:34.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:33 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:34.218 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:33 smithi064 ceph-mon[212750]: pgmap v50: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:45:34.218 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:33 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:34.218 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:33 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:34.218 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:33 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:34.218 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:33 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:45:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:33 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:33 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:45:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:33 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:45:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:33 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T05:45:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:33 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T05:45:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:33 smithi064 ceph-mon[212750]: Upgrade: It appears safe to stop mon.b 2023-10-09T05:45:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:33 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:34.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:33 smithi064 ceph-mon[220044]: pgmap v50: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:45:34.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:33 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:33 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:33 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:33 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:45:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:33 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:33 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:45:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:33 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:45:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:33 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T05:45:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:33 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T05:45:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:33 smithi064 ceph-mon[220044]: Upgrade: It appears safe to stop mon.b 2023-10-09T05:45:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:33 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:34.239 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T05:45:34.646 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:45:35.046 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:34 smithi064 ceph-mon[212750]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:45:35.046 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:34 smithi064 ceph-mon[212750]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:45:35.046 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:34 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:35.046 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:34 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:45:35.046 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:34 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:45:35.047 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:34 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:35.047 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:34 smithi064 ceph-mon[220044]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:45:35.047 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:34 smithi064 ceph-mon[220044]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:45:35.047 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:34 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:35.047 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:34 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:45:35.047 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:34 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:45:35.047 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:34 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:35.100 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:34 smithi156 ceph-mon[105979]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:45:35.101 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:34 smithi156 ceph-mon[105979]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:45:35.101 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:34 smithi156 ceph-mon[105979]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:35.101 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:34 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:45:35.101 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:34 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:45:35.101 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:34 smithi156 ceph-mon[105979]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:35.699 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:45:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:45:35] "GET /metrics HTTP/1.1" 200 33788 "" "Prometheus/2.43.0" 2023-10-09T05:45:35.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:35 smithi156 systemd[1]: Stopping Ceph mon.b for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:45:35.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:35 smithi156 ceph-mon[105979]: Upgrade: Updating mon.b 2023-10-09T05:45:35.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:35 smithi156 ceph-mon[105979]: Deploying daemon mon.b on smithi156 2023-10-09T05:45:35.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:35 smithi156 ceph-mon[105979]: pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:45:35.961 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:35 smithi064 ceph-mon[212750]: Upgrade: Updating mon.b 2023-10-09T05:45:35.961 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:35 smithi064 ceph-mon[212750]: Deploying daemon mon.b on smithi156 2023-10-09T05:45:35.961 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:35 smithi064 ceph-mon[212750]: pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:45:35.962 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:35 smithi064 ceph-mon[220044]: Upgrade: Updating mon.b 2023-10-09T05:45:35.962 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:35 smithi064 ceph-mon[220044]: Deploying daemon mon.b on smithi156 2023-10-09T05:45:35.962 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:35 smithi064 ceph-mon[220044]: pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:45:36.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:45:35 smithi064 conmon[198859]: [09/Oct/2023:05:45:35] ENGINE Bus STOPPING 2023-10-09T05:45:36.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:45:36 smithi064 conmon[198859]: [09/Oct/2023:05:45:36] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:45:36.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:45:36 smithi064 conmon[198859]: [09/Oct/2023:05:45:36] ENGINE Bus STOPPED 2023-10-09T05:45:36.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:45:36 smithi064 conmon[198859]: [09/Oct/2023:05:45:36] ENGINE Bus STARTING 2023-10-09T05:45:36.220 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:45:36 smithi064 conmon[198859]: [09/Oct/2023:05:45:36] ENGINE Serving on http://:::9283 2023-10-09T05:45:36.220 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:45:36 smithi064 conmon[198859]: [09/Oct/2023:05:45:36] ENGINE Bus STARTED 2023-10-09T05:45:36.292 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:35 smithi156 conmon[105952]: 2023-10-09T05:45:35.979+0000 7fea8a856700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T05:45:36.292 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:35 smithi156 conmon[105952]: 2023-10-09T05:45:35.979+0000 7fea8a856700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-10-09T05:45:36.293 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:36 smithi156 bash[194678]: c60a890df99eb0a3df217e512882753954791ae5c057840ae022e5a4f1d80d64 2023-10-09T05:45:37.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:36 smithi156 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mon.b.service: Succeeded. 2023-10-09T05:45:37.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:36 smithi156 systemd[1]: Stopped Ceph mon.b for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:45:37.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:36 smithi156 systemd[1]: Starting Ceph mon.b for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:45:37.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T05:45:37.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T05:45:37.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: pidfile_write: ignore empty --pid-file 2023-10-09T05:45:37.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 bash[195036]: 0cd87bdd0b75fbcb75e31a2d7352d6615047f9c02dc6ddc7ca146e5d79098584 2023-10-09T05:45:37.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: load: jerasure load: lrc 2023-10-09T05:45:37.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: RocksDB version: 7.9.2 2023-10-09T05:45:37.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Git sha 0 2023-10-09T05:45:37.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T05:45:37.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: DB SUMMARY 2023-10-09T05:45:37.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: DB Session ID: 5ERIWDSDOURNJ2VBSSNU 2023-10-09T05:45:37.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: CURRENT file: CURRENT 2023-10-09T05:45:37.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T05:45:37.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1096 Bytes 2023-10-09T05:45:37.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2023-10-09T05:45:37.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 4651455 ; 2023-10-09T05:45:37.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.error_if_exists: 0 2023-10-09T05:45:37.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.create_if_missing: 0 2023-10-09T05:45:37.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.paranoid_checks: 1 2023-10-09T05:45:37.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T05:45:37.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T05:45:37.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T05:45:37.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.env: 0x55c1e2c72720 2023-10-09T05:45:37.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.fs: PosixFileSystem 2023-10-09T05:45:37.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.info_log: 0x55c1e4947420 2023-10-09T05:45:37.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T05:45:37.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.statistics: (nil) 2023-10-09T05:45:37.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.use_fsync: 0 2023-10-09T05:45:37.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_log_file_size: 0 2023-10-09T05:45:37.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T05:45:37.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T05:45:37.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T05:45:37.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T05:45:37.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.allow_fallocate: 1 2023-10-09T05:45:37.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T05:45:37.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T05:45:37.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.use_direct_reads: 0 2023-10-09T05:45:37.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T05:45:37.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T05:45:37.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.db_log_dir: 2023-10-09T05:45:37.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.wal_dir: 2023-10-09T05:45:37.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T05:45:37.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T05:45:37.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T05:45:37.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T05:45:37.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T05:45:37.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T05:45:37.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T05:45:37.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T05:45:37.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.write_buffer_manager: 0x55c1e3bd6c80 2023-10-09T05:45:37.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T05:45:37.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T05:45:37.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T05:45:37.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T05:45:37.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T05:45:37.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T05:45:37.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T05:45:37.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T05:45:37.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.unordered_write: 0 2023-10-09T05:45:37.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T05:45:37.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T05:45:37.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T05:45:37.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T05:45:37.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.row_cache: None 2023-10-09T05:45:37.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.wal_filter: None 2023-10-09T05:45:37.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T05:45:37.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T05:45:37.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.two_write_queues: 0 2023-10-09T05:45:37.627 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T05:45:37.627 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.wal_compression: 0 2023-10-09T05:45:37.627 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.atomic_flush: 0 2023-10-09T05:45:37.627 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T05:45:37.627 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T05:45:37.627 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T05:45:37.627 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.log_readahead_size: 0 2023-10-09T05:45:37.627 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T05:45:37.629 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T05:45:37.629 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T05:45:37.629 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T05:45:37.629 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T05:45:37.629 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T05:45:37.629 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T05:45:37.629 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_background_jobs: 2 2023-10-09T05:45:37.630 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_background_compactions: -1 2023-10-09T05:45:37.630 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_subcompactions: 1 2023-10-09T05:45:37.630 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T05:45:37.630 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T05:45:37.630 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T05:45:37.630 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T05:45:37.630 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T05:45:37.630 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T05:45:37.630 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T05:45:37.630 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T05:45:37.630 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_open_files: -1 2023-10-09T05:45:37.630 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T05:45:37.631 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T05:45:37.631 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T05:45:37.631 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T05:45:37.631 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_background_flushes: -1 2023-10-09T05:45:37.631 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Compression algorithms supported: 2023-10-09T05:45:37.631 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T05:45:37.631 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: kZSTD supported: 0 2023-10-09T05:45:37.631 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: kXpressCompression supported: 0 2023-10-09T05:45:37.631 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T05:45:37.631 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: kZlibCompression supported: 1 2023-10-09T05:45:37.631 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: kSnappyCompression supported: 1 2023-10-09T05:45:37.631 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: kLZ4Compression supported: 1 2023-10-09T05:45:37.632 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: kBZip2Compression supported: 0 2023-10-09T05:45:37.632 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T05:45:37.632 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T05:45:37.632 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2023-10-09T05:45:37.632 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T05:45:37.632 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T05:45:37.632 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.merge_operator: 2023-10-09T05:45:37.632 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.compaction_filter: None 2023-10-09T05:45:37.632 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T05:45:37.632 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T05:45:37.632 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T05:45:37.632 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T05:45:37.633 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c1e4947520) 2023-10-09T05:45:37.633 INFO:journalctl@ceph.mon.b.smithi156.stdout: cache_index_and_filter_blocks: 1 2023-10-09T05:45:37.633 INFO:journalctl@ceph.mon.b.smithi156.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T05:45:37.633 INFO:journalctl@ceph.mon.b.smithi156.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T05:45:37.633 INFO:journalctl@ceph.mon.b.smithi156.stdout: pin_top_level_index_and_filter: 1 2023-10-09T05:45:37.633 INFO:journalctl@ceph.mon.b.smithi156.stdout: index_type: 0 2023-10-09T05:45:37.633 INFO:journalctl@ceph.mon.b.smithi156.stdout: data_block_index_type: 0 2023-10-09T05:45:37.633 INFO:journalctl@ceph.mon.b.smithi156.stdout: index_shortening: 1 2023-10-09T05:45:37.633 INFO:journalctl@ceph.mon.b.smithi156.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T05:45:37.633 INFO:journalctl@ceph.mon.b.smithi156.stdout: checksum: 4 2023-10-09T05:45:37.633 INFO:journalctl@ceph.mon.b.smithi156.stdout: no_block_cache: 0 2023-10-09T05:45:37.633 INFO:journalctl@ceph.mon.b.smithi156.stdout: block_cache: 0x55c1e3c59350 2023-10-09T05:45:37.634 INFO:journalctl@ceph.mon.b.smithi156.stdout: block_cache_name: BinnedLRUCache 2023-10-09T05:45:37.634 INFO:journalctl@ceph.mon.b.smithi156.stdout: block_cache_options: 2023-10-09T05:45:37.634 INFO:journalctl@ceph.mon.b.smithi156.stdout: capacity : 536870912 2023-10-09T05:45:37.634 INFO:journalctl@ceph.mon.b.smithi156.stdout: num_shard_bits : 4 2023-10-09T05:45:37.634 INFO:journalctl@ceph.mon.b.smithi156.stdout: strict_capacity_limit : 0 2023-10-09T05:45:37.634 INFO:journalctl@ceph.mon.b.smithi156.stdout: high_pri_pool_ratio: 0.000 2023-10-09T05:45:37.634 INFO:journalctl@ceph.mon.b.smithi156.stdout: block_cache_compressed: (nil) 2023-10-09T05:45:37.634 INFO:journalctl@ceph.mon.b.smithi156.stdout: persistent_cache: (nil) 2023-10-09T05:45:37.634 INFO:journalctl@ceph.mon.b.smithi156.stdout: block_size: 4096 2023-10-09T05:45:37.634 INFO:journalctl@ceph.mon.b.smithi156.stdout: block_size_deviation: 10 2023-10-09T05:45:37.634 INFO:journalctl@ceph.mon.b.smithi156.stdout: block_restart_interval: 16 2023-10-09T05:45:37.634 INFO:journalctl@ceph.mon.b.smithi156.stdout: index_block_restart_interval: 1 2023-10-09T05:45:37.635 INFO:journalctl@ceph.mon.b.smithi156.stdout: metadata_block_size: 4096 2023-10-09T05:45:37.635 INFO:journalctl@ceph.mon.b.smithi156.stdout: partition_filters: 0 2023-10-09T05:45:37.635 INFO:journalctl@ceph.mon.b.smithi156.stdout: use_delta_encoding: 1 2023-10-09T05:45:37.635 INFO:journalctl@ceph.mon.b.smithi156.stdout: filter_policy: bloomfilter 2023-10-09T05:45:37.635 INFO:journalctl@ceph.mon.b.smithi156.stdout: whole_key_filtering: 1 2023-10-09T05:45:37.635 INFO:journalctl@ceph.mon.b.smithi156.stdout: verify_compression: 0 2023-10-09T05:45:37.635 INFO:journalctl@ceph.mon.b.smithi156.stdout: read_amp_bytes_per_bit: 0 2023-10-09T05:45:37.635 INFO:journalctl@ceph.mon.b.smithi156.stdout: format_version: 5 2023-10-09T05:45:37.635 INFO:journalctl@ceph.mon.b.smithi156.stdout: enable_index_compression: 1 2023-10-09T05:45:37.635 INFO:journalctl@ceph.mon.b.smithi156.stdout: block_align: 0 2023-10-09T05:45:37.635 INFO:journalctl@ceph.mon.b.smithi156.stdout: max_auto_readahead_size: 262144 2023-10-09T05:45:37.635 INFO:journalctl@ceph.mon.b.smithi156.stdout: prepopulate_block_cache: 0 2023-10-09T05:45:37.636 INFO:journalctl@ceph.mon.b.smithi156.stdout: initial_auto_readahead_size: 8192 2023-10-09T05:45:37.636 INFO:journalctl@ceph.mon.b.smithi156.stdout: num_file_reads_for_auto_readahead: 2 2023-10-09T05:45:37.636 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T05:45:37.636 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T05:45:37.636 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.compression: NoCompression 2023-10-09T05:45:37.636 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T05:45:37.636 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T05:45:37.636 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T05:45:37.636 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.num_levels: 7 2023-10-09T05:45:37.636 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T05:45:37.636 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T05:45:37.636 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T05:45:37.637 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T05:45:37.637 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T05:45:37.637 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T05:45:37.638 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T05:45:37.638 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:45:37.638 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T05:45:37.638 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T05:45:37.638 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T05:45:37.638 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T05:45:37.638 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T05:45:37.638 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T05:45:37.639 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T05:45:37.639 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T05:45:37.639 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:45:37.639 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T05:45:37.639 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T05:45:37.639 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T05:45:37.639 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T05:45:37.639 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T05:45:37.639 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T05:45:37.639 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T05:45:37.639 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T05:45:37.639 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T05:45:37.640 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T05:45:37.640 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T05:45:37.640 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T05:45:37.640 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T05:45:37.640 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T05:45:37.640 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T05:45:37.640 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T05:45:37.640 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T05:45:37.640 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T05:45:37.640 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T05:45:37.640 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T05:45:37.641 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T05:45:37.641 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T05:45:37.641 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.arena_block_size: 1048576 2023-10-09T05:45:37.641 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T05:45:37.641 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T05:45:37.641 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T05:45:37.641 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T05:45:37.641 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T05:45:37.641 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T05:45:37.641 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T05:45:37.641 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T05:45:37.641 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T05:45:37.642 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T05:45:37.642 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T05:45:37.642 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T05:45:37.642 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T05:45:37.642 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.table_properties_collectors: 2023-10-09T05:45:37.642 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.inplace_update_support: 0 2023-10-09T05:45:37.642 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T05:45:37.642 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T05:45:37.642 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T05:45:37.642 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T05:45:37.642 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.bloom_locality: 0 2023-10-09T05:45:37.642 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.max_successive_merges: 0 2023-10-09T05:45:37.643 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T05:45:37.643 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T05:45:37.643 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T05:45:37.643 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T05:45:37.643 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.ttl: 2592000 2023-10-09T05:45:37.643 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T05:45:37.643 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T05:45:37.643 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T05:45:37.643 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.enable_blob_files: false 2023-10-09T05:45:37.643 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.min_blob_size: 0 2023-10-09T05:45:37.643 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T05:45:37.643 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T05:45:37.644 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T05:45:37.644 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T05:45:37.644 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T05:45:37.644 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T05:45:37.644 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.blob_file_starting_level: 0 2023-10-09T05:45:37.644 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T05:45:37.644 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-10-09T05:45:37.644 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T05:45:37.644 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13088, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T05:45:37.644 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-10-09T05:45:37.644 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: c54cd4cb-8fcf-43c7-b340-d88169a4b04e 2023-10-09T05:45:37.645 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696830337368573, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-10-09T05:45:37.645 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-10-09T05:45:37.645 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 systemd[1]: Started Ceph mon.b for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:45:37.645 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696830337379856, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 2895815, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13090, "largest_seqno": 14950, "table_properties": {"data_size": 2888599, "index_size": 4469, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1733, "raw_key_size": 15226, "raw_average_key_size": 22, "raw_value_size": 2874314, "raw_average_value_size": 4302, "num_data_blocks": 203, "num_entries": 668, "num_filter_entries": 668, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696830337, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "c54cd4cb-8fcf-43c7-b340-d88169a4b04e", "db_session_id": "5ERIWDSDOURNJ2VBSSNU", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-10-09T05:45:37.645 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696830337379954, "job": 1, "event": "recovery_finished"} 2023-10-09T05:45:37.645 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-10-09T05:45:37.645 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T05:45:37.645 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T05:45:37.645 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55c1e3cf4000 2023-10-09T05:45:37.645 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: DB pointer 0x55c1e3ce2000 2023-10-09T05:45:37.645 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T05:45:37.646 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T05:45:37.646 INFO:journalctl@ceph.mon.b.smithi156.stdout: ** DB Stats ** 2023-10-09T05:45:37.646 INFO:journalctl@ceph.mon.b.smithi156.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:45:37.646 INFO:journalctl@ceph.mon.b.smithi156.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T05:45:37.646 INFO:journalctl@ceph.mon.b.smithi156.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:45:37.646 INFO:journalctl@ceph.mon.b.smithi156.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:45:37.646 INFO:journalctl@ceph.mon.b.smithi156.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T05:45:37.646 INFO:journalctl@ceph.mon.b.smithi156.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:45:37.646 INFO:journalctl@ceph.mon.b.smithi156.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:45:37.646 INFO:journalctl@ceph.mon.b.smithi156.stdout: 2023-10-09T05:45:37.646 INFO:journalctl@ceph.mon.b.smithi156.stdout: ** Compaction Stats [default] ** 2023-10-09T05:45:37.647 INFO:journalctl@ceph.mon.b.smithi156.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T05:45:37.647 INFO:journalctl@ceph.mon.b.smithi156.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T05:45:37.647 INFO:journalctl@ceph.mon.b.smithi156.stdout: L0 1/0 2.76 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 600.6 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-09T05:45:37.647 INFO:journalctl@ceph.mon.b.smithi156.stdout: L6 1/0 6.33 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T05:45:37.647 INFO:journalctl@ceph.mon.b.smithi156.stdout: Sum 2/0 9.09 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 600.6 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-09T05:45:37.647 INFO:journalctl@ceph.mon.b.smithi156.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 600.6 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-09T05:45:37.647 INFO:journalctl@ceph.mon.b.smithi156.stdout: 2023-10-09T05:45:37.647 INFO:journalctl@ceph.mon.b.smithi156.stdout: ** Compaction Stats [default] ** 2023-10-09T05:45:37.647 INFO:journalctl@ceph.mon.b.smithi156.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T05:45:37.647 INFO:journalctl@ceph.mon.b.smithi156.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:45:37.647 INFO:journalctl@ceph.mon.b.smithi156.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 600.6 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-09T05:45:37.647 INFO:journalctl@ceph.mon.b.smithi156.stdout: 2023-10-09T05:45:37.648 INFO:journalctl@ceph.mon.b.smithi156.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T05:45:37.648 INFO:journalctl@ceph.mon.b.smithi156.stdout: 2023-10-09T05:45:37.648 INFO:journalctl@ceph.mon.b.smithi156.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:45:37.648 INFO:journalctl@ceph.mon.b.smithi156.stdout: Flush(GB): cumulative 0.003, interval 0.003 2023-10-09T05:45:37.648 INFO:journalctl@ceph.mon.b.smithi156.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:45:37.648 INFO:journalctl@ceph.mon.b.smithi156.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:45:37.648 INFO:journalctl@ceph.mon.b.smithi156.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:45:37.648 INFO:journalctl@ceph.mon.b.smithi156.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:45:37.648 INFO:journalctl@ceph.mon.b.smithi156.stdout: Cumulative compaction: 0.00 GB write, 200.99 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:45:37.649 INFO:journalctl@ceph.mon.b.smithi156.stdout: Interval compaction: 0.00 GB write, 200.99 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:45:37.649 INFO:journalctl@ceph.mon.b.smithi156.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:45:37.649 INFO:journalctl@ceph.mon.b.smithi156.stdout: Block cache BinnedLRUCache@0x55c1e3c59350#7 capacity: 512.00 MB usage: 6.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.3e-05 secs_since: 0 2023-10-09T05:45:37.649 INFO:journalctl@ceph.mon.b.smithi156.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,4.58 KB,0.000873208%) Misc(1,0.00 KB,0%) 2023-10-09T05:45:37.649 INFO:journalctl@ceph.mon.b.smithi156.stdout: 2023-10-09T05:45:37.649 INFO:journalctl@ceph.mon.b.smithi156.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:45:37.649 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: starting mon.b rank 1 at public addrs [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] at bind addrs [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:45:37.649 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: mon.b@-1(???) e3 preinit fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:45:37.649 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: mon.b@-1(???).mds e8 new map 2023-10-09T05:45:37.649 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: mon.b@-1(???).mds e8 print_map 2023-10-09T05:45:37.649 INFO:journalctl@ceph.mon.b.smithi156.stdout: e8 2023-10-09T05:45:37.650 INFO:journalctl@ceph.mon.b.smithi156.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T05:45:37.650 INFO:journalctl@ceph.mon.b.smithi156.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T05:45:37.650 INFO:journalctl@ceph.mon.b.smithi156.stdout: legacy client fscid: 1 2023-10-09T05:45:37.650 INFO:journalctl@ceph.mon.b.smithi156.stdout: 2023-10-09T05:45:37.650 INFO:journalctl@ceph.mon.b.smithi156.stdout: Filesystem 'foo' (1) 2023-10-09T05:45:37.650 INFO:journalctl@ceph.mon.b.smithi156.stdout: fs_name foo 2023-10-09T05:45:37.650 INFO:journalctl@ceph.mon.b.smithi156.stdout: epoch 7 2023-10-09T05:45:37.650 INFO:journalctl@ceph.mon.b.smithi156.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T05:45:37.650 INFO:journalctl@ceph.mon.b.smithi156.stdout: created 2023-10-09T05:40:12.634524+0000 2023-10-09T05:45:37.650 INFO:journalctl@ceph.mon.b.smithi156.stdout: modified 2023-10-09T05:40:21.631615+0000 2023-10-09T05:45:37.650 INFO:journalctl@ceph.mon.b.smithi156.stdout: tableserver 0 2023-10-09T05:45:37.650 INFO:journalctl@ceph.mon.b.smithi156.stdout: root 0 2023-10-09T05:45:37.651 INFO:journalctl@ceph.mon.b.smithi156.stdout: session_timeout 60 2023-10-09T05:45:37.651 INFO:journalctl@ceph.mon.b.smithi156.stdout: session_autoclose 300 2023-10-09T05:45:37.651 INFO:journalctl@ceph.mon.b.smithi156.stdout: max_file_size 1099511627776 2023-10-09T05:45:37.651 INFO:journalctl@ceph.mon.b.smithi156.stdout: required_client_features {} 2023-10-09T05:45:37.651 INFO:journalctl@ceph.mon.b.smithi156.stdout: last_failure 0 2023-10-09T05:45:37.651 INFO:journalctl@ceph.mon.b.smithi156.stdout: last_failure_osd_epoch 0 2023-10-09T05:45:37.651 INFO:journalctl@ceph.mon.b.smithi156.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T05:45:37.651 INFO:journalctl@ceph.mon.b.smithi156.stdout: max_mds 1 2023-10-09T05:45:37.651 INFO:journalctl@ceph.mon.b.smithi156.stdout: in 0 2023-10-09T05:45:37.651 INFO:journalctl@ceph.mon.b.smithi156.stdout: up {0=24551} 2023-10-09T05:45:37.651 INFO:journalctl@ceph.mon.b.smithi156.stdout: failed 2023-10-09T05:45:37.652 INFO:journalctl@ceph.mon.b.smithi156.stdout: damaged 2023-10-09T05:45:37.652 INFO:journalctl@ceph.mon.b.smithi156.stdout: stopped 2023-10-09T05:45:37.652 INFO:journalctl@ceph.mon.b.smithi156.stdout: data_pools [4] 2023-10-09T05:45:37.652 INFO:journalctl@ceph.mon.b.smithi156.stdout: metadata_pool 3 2023-10-09T05:45:37.652 INFO:journalctl@ceph.mon.b.smithi156.stdout: inline_data disabled 2023-10-09T05:45:37.652 INFO:journalctl@ceph.mon.b.smithi156.stdout: balancer 2023-10-09T05:45:37.652 INFO:journalctl@ceph.mon.b.smithi156.stdout: bal_rank_mask -1 2023-10-09T05:45:37.652 INFO:journalctl@ceph.mon.b.smithi156.stdout: standby_count_wanted 1 2023-10-09T05:45:37.652 INFO:journalctl@ceph.mon.b.smithi156.stdout: [mds.foo.smithi064.gmrwsy{0:24551} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.64:6834/3975184368,v1:172.21.15.64:6835/3975184368] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T05:45:37.653 INFO:journalctl@ceph.mon.b.smithi156.stdout: 2023-10-09T05:45:37.653 INFO:journalctl@ceph.mon.b.smithi156.stdout: 2023-10-09T05:45:37.653 INFO:journalctl@ceph.mon.b.smithi156.stdout: Standby daemons: 2023-10-09T05:45:37.653 INFO:journalctl@ceph.mon.b.smithi156.stdout: 2023-10-09T05:45:37.653 INFO:journalctl@ceph.mon.b.smithi156.stdout: [mds.foo.smithi156.pizbdw{-1:24556} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.156:6832/4137868430,v1:172.21.15.156:6833/4137868430] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T05:45:37.653 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: mon.b@-1(???).osd e80 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T05:45:37.653 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: mon.b@-1(???).osd e80 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T05:45:37.653 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: mon.b@-1(???).osd e80 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T05:45:37.653 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: mon.b@-1(???).osd e80 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T05:45:37.653 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: mon.b@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2023-10-09T05:45:37.653 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: expand_channel_meta expand map: {default=false} 2023-10-09T05:45:37.653 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: expand_channel_meta from 'false' to 'false' 2023-10-09T05:45:37.654 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: expand_channel_meta expanded map: {default=false} 2023-10-09T05:45:37.654 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: expand_channel_meta expand map: {default=info} 2023-10-09T05:45:37.654 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: expand_channel_meta from 'info' to 'info' 2023-10-09T05:45:37.654 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: expand_channel_meta expanded map: {default=info} 2023-10-09T05:45:37.655 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: expand_channel_meta expand map: {default=daemon} 2023-10-09T05:45:37.655 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T05:45:37.655 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: expand_channel_meta expanded map: {default=daemon} 2023-10-09T05:45:37.655 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: expand_channel_meta expand map: {default=debug} 2023-10-09T05:45:37.655 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: expand_channel_meta from 'debug' to 'debug' 2023-10-09T05:45:37.655 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:37 smithi156 ceph-mon[195101]: expand_channel_meta expanded map: {default=debug} 2023-10-09T05:45:39.120 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:45:38 smithi156 conmon[179783]: ts=2023-10-09T05:45:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:45:39.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[212750]: overall HEALTH_OK 2023-10-09T05:45:39.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[212750]: mon.b calling monitor election 2023-10-09T05:45:39.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:45:39.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[212750]: mon.c calling monitor election 2023-10-09T05:45:39.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[212750]: mon.a calling monitor election 2023-10-09T05:45:39.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:45:39.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:45:39.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[212750]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:45:39.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[212750]: monmap e4: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} 2023-10-09T05:45:39.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[212750]: fsmap foo:1 {0=foo.smithi064.gmrwsy=up:active} 1 up:standby 2023-10-09T05:45:39.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[212750]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T05:45:39.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[212750]: mgrmap e37: y(active, since 99s), standbys: x 2023-10-09T05:45:39.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[212750]: overall HEALTH_OK 2023-10-09T05:45:39.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='' 2023-10-09T05:45:39.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:39.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:45:39.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[212750]: pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:45:39.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[220044]: overall HEALTH_OK 2023-10-09T05:45:39.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[220044]: mon.b calling monitor election 2023-10-09T05:45:39.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:45:39.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[220044]: mon.c calling monitor election 2023-10-09T05:45:39.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[220044]: mon.a calling monitor election 2023-10-09T05:45:39.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:45:39.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:45:39.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[220044]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:45:39.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[220044]: monmap e4: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} 2023-10-09T05:45:39.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[220044]: fsmap foo:1 {0=foo.smithi064.gmrwsy=up:active} 1 up:standby 2023-10-09T05:45:39.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[220044]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T05:45:39.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[220044]: mgrmap e37: y(active, since 99s), standbys: x 2023-10-09T05:45:39.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[220044]: overall HEALTH_OK 2023-10-09T05:45:39.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='' 2023-10-09T05:45:39.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:39.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:45:39.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:39 smithi064 ceph-mon[220044]: pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:45:39.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:39 smithi156 ceph-mon[195101]: overall HEALTH_OK 2023-10-09T05:45:39.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:39 smithi156 ceph-mon[195101]: mon.b calling monitor election 2023-10-09T05:45:39.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:39 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:45:39.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:39 smithi156 ceph-mon[195101]: mon.c calling monitor election 2023-10-09T05:45:39.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:39 smithi156 ceph-mon[195101]: mon.a calling monitor election 2023-10-09T05:45:39.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:39 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:45:39.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:39 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:45:39.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:39 smithi156 ceph-mon[195101]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:45:39.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:39 smithi156 ceph-mon[195101]: monmap e4: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} 2023-10-09T05:45:39.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:39 smithi156 ceph-mon[195101]: fsmap foo:1 {0=foo.smithi064.gmrwsy=up:active} 1 up:standby 2023-10-09T05:45:39.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:39 smithi156 ceph-mon[195101]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T05:45:39.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:39 smithi156 ceph-mon[195101]: mgrmap e37: y(active, since 99s), standbys: x 2023-10-09T05:45:39.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:39 smithi156 ceph-mon[195101]: overall HEALTH_OK 2023-10-09T05:45:39.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:39 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='' 2023-10-09T05:45:39.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:39 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:39.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:39 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:45:39.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:39 smithi156 ceph-mon[195101]: pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:45:41.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:41 smithi064 ceph-mon[212750]: pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:45:41.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:41 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:41.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:41 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:41.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:41 smithi064 ceph-mon[220044]: pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:45:41.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:41 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:41.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:41 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:41.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:41 smithi156 ceph-mon[195101]: pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T05:45:41.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:41 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:41.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:41 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:42.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:45:42 smithi064 conmon[198859]: 2023-10-09T05:45:42.388+0000 7fcb872b4700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-10-09T05:45:43.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:43 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:43.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:43 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:43.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:43 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:43.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:43 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:43.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:43 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:43.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:43 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:44.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:44 smithi156 ceph-mon[195101]: pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:44.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:44.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:44 smithi064 ceph-mon[212750]: pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:44.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:44.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:44 smithi064 ceph-mon[220044]: pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:44.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:45.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:45:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:45:45] "GET /metrics HTTP/1.1" 200 33784 "" "Prometheus/2.43.0" 2023-10-09T05:45:46.140 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:46 smithi064 ceph-mon[212750]: pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:46.141 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:46 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:46.141 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:46 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:46.141 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:46 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:46.141 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:46 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:45:46.141 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:46 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:46.141 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:46 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:45:46.141 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:46 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:45:46.141 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:46 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:46.142 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:46 smithi064 ceph-mon[220044]: pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:46.142 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:46 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:46.142 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:46 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:46.142 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:46 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:46.142 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:46 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:45:46.142 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:46 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:46.142 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:46 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:45:46.143 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:46 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:45:46.143 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:46 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:46.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:46 smithi156 ceph-mon[195101]: pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:46.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:46 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:46.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:46 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:46.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:46 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:46.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:46 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:45:46.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:46 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:46.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:46 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:45:46.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:46 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:45:46.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:46 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:47.342 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:47 smithi064 ceph-mon[212750]: Reconfiguring mon.a (monmap changed)... 2023-10-09T05:45:47.342 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:47 smithi064 ceph-mon[212750]: Reconfiguring daemon mon.a on smithi064 2023-10-09T05:45:47.342 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:47 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:47.343 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:47 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:47.343 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:47 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:45:47.343 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:47 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:45:47.343 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:47 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:45:47.343 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:47 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:47.343 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:47 smithi064 ceph-mon[220044]: Reconfiguring mon.a (monmap changed)... 2023-10-09T05:45:47.343 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:47 smithi064 ceph-mon[220044]: Reconfiguring daemon mon.a on smithi064 2023-10-09T05:45:47.343 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:47 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:47.343 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:47 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:47.344 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:47 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:45:47.344 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:47 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:45:47.344 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:47 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:45:47.344 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:47 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:47 smithi156 ceph-mon[195101]: Reconfiguring mon.a (monmap changed)... 2023-10-09T05:45:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:47 smithi156 ceph-mon[195101]: Reconfiguring daemon mon.a on smithi064 2023-10-09T05:45:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:47 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:47 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:47.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:47 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:45:47.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:47 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:45:47.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:47 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:45:47.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:47 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:48.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:48 smithi064 ceph-mon[212750]: pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:48.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:48 smithi064 ceph-mon[212750]: Reconfiguring mgr.y (monmap changed)... 2023-10-09T05:45:48.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:48 smithi064 ceph-mon[212750]: Reconfiguring daemon mgr.y on smithi064 2023-10-09T05:45:48.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:48 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:48.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:48 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:48.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:48 smithi064 ceph-mon[220044]: pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:48.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:48 smithi064 ceph-mon[220044]: Reconfiguring mgr.y (monmap changed)... 2023-10-09T05:45:48.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:48 smithi064 ceph-mon[220044]: Reconfiguring daemon mgr.y on smithi064 2023-10-09T05:45:48.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:48 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:48.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:48 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:48.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:48 smithi156 ceph-mon[195101]: pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:48.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:48 smithi156 ceph-mon[195101]: Reconfiguring mgr.y (monmap changed)... 2023-10-09T05:45:48.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:48 smithi156 ceph-mon[195101]: Reconfiguring daemon mgr.y on smithi064 2023-10-09T05:45:48.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:48 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:48.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:48 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:49.025 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:45:48 smithi156 conmon[179783]: ts=2023-10-09T05:45:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:45:49.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:49 smithi064 ceph-mon[212750]: Reconfiguring mon.c (monmap changed)... 2023-10-09T05:45:49.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:49 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:45:49.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:49 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:45:49.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:49 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:49.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:49 smithi064 ceph-mon[212750]: Reconfiguring daemon mon.c on smithi064 2023-10-09T05:45:49.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:49 smithi064 ceph-mon[220044]: Reconfiguring mon.c (monmap changed)... 2023-10-09T05:45:49.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:49 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:45:49.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:49 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:45:49.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:49 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:49.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:49 smithi064 ceph-mon[220044]: Reconfiguring daemon mon.c on smithi064 2023-10-09T05:45:49.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:49 smithi156 ceph-mon[195101]: Reconfiguring mon.c (monmap changed)... 2023-10-09T05:45:49.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:49 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:45:49.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:49 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:45:49.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:49 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:49.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:49 smithi156 ceph-mon[195101]: Reconfiguring daemon mon.c on smithi064 2023-10-09T05:45:50.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:50 smithi064 ceph-mon[212750]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:50.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:50 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:50.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:50 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:50.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:50 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T05:45:50.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:50 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:50.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:50 smithi064 ceph-mon[220044]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:50.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:50 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:50.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:50 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:50.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:50 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T05:45:50.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:50 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:50.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:50 smithi156 ceph-mon[195101]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:50.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:50 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:50.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:50 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:50.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:50 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T05:45:50.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:50 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:51.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:51 smithi156 ceph-mon[195101]: Reconfiguring osd.0 (monmap changed)... 2023-10-09T05:45:51.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:51 smithi156 ceph-mon[195101]: Reconfiguring daemon osd.0 on smithi064 2023-10-09T05:45:51.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:51 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:51.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:51 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:51.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:51 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T05:45:51.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:51 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:51.401 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:51 smithi064 ceph-mon[212750]: Reconfiguring osd.0 (monmap changed)... 2023-10-09T05:45:51.401 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:51 smithi064 ceph-mon[212750]: Reconfiguring daemon osd.0 on smithi064 2023-10-09T05:45:51.401 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:51 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:51.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:51 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:51.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:51 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T05:45:51.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:51 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:51.402 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:51 smithi064 ceph-mon[220044]: Reconfiguring osd.0 (monmap changed)... 2023-10-09T05:45:51.402 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:51 smithi064 ceph-mon[220044]: Reconfiguring daemon osd.0 on smithi064 2023-10-09T05:45:51.402 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:51 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:51.402 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:51 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:51.402 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:51 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T05:45:51.402 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:51 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:52.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:52 smithi156 ceph-mon[195101]: pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:52.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:52 smithi156 ceph-mon[195101]: Reconfiguring osd.1 (monmap changed)... 2023-10-09T05:45:52.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:52 smithi156 ceph-mon[195101]: Reconfiguring daemon osd.1 on smithi064 2023-10-09T05:45:52.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:52 smithi064 ceph-mon[220044]: pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:52.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:52 smithi064 ceph-mon[220044]: Reconfiguring osd.1 (monmap changed)... 2023-10-09T05:45:52.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:52 smithi064 ceph-mon[220044]: Reconfiguring daemon osd.1 on smithi064 2023-10-09T05:45:52.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:52 smithi064 ceph-mon[212750]: pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:52.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:52 smithi064 ceph-mon[212750]: Reconfiguring osd.1 (monmap changed)... 2023-10-09T05:45:52.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:52 smithi064 ceph-mon[212750]: Reconfiguring daemon osd.1 on smithi064 2023-10-09T05:45:53.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:53 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:53.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:53 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:53.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:53 smithi064 ceph-mon[212750]: Reconfiguring osd.2 (monmap changed)... 2023-10-09T05:45:53.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:53 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:45:53.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:53 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:53.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:53 smithi064 ceph-mon[212750]: Reconfiguring daemon osd.2 on smithi064 2023-10-09T05:45:53.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:53 smithi064 ceph-mon[212750]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:53.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:53 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:53.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:53 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:53.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:53 smithi064 ceph-mon[220044]: Reconfiguring osd.2 (monmap changed)... 2023-10-09T05:45:53.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:53 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:45:53.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:53 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:53.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:53 smithi064 ceph-mon[220044]: Reconfiguring daemon osd.2 on smithi064 2023-10-09T05:45:53.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:53 smithi064 ceph-mon[220044]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:53.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:53 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:53.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:53 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:53.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:53 smithi156 ceph-mon[195101]: Reconfiguring osd.2 (monmap changed)... 2023-10-09T05:45:53.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:53 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:45:53.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:53 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:53.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:53 smithi156 ceph-mon[195101]: Reconfiguring daemon osd.2 on smithi064 2023-10-09T05:45:53.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:53 smithi156 ceph-mon[195101]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:54.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:54 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:54.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:54 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:54.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:54 smithi064 ceph-mon[212750]: Reconfiguring osd.3 (monmap changed)... 2023-10-09T05:45:54.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:54 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:45:54.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:54 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:54.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:54 smithi064 ceph-mon[212750]: Reconfiguring daemon osd.3 on smithi064 2023-10-09T05:45:54.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:54 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:54.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:54 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:54.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:54 smithi064 ceph-mon[220044]: Reconfiguring osd.3 (monmap changed)... 2023-10-09T05:45:54.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:54 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:45:54.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:54 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:54.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:54 smithi064 ceph-mon[220044]: Reconfiguring daemon osd.3 on smithi064 2023-10-09T05:45:54.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:54 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:54.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:54 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:54.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:54 smithi156 ceph-mon[195101]: Reconfiguring osd.3 (monmap changed)... 2023-10-09T05:45:54.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:54 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:45:54.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:54 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:54.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:54 smithi156 ceph-mon[195101]: Reconfiguring daemon osd.3 on smithi064 2023-10-09T05:45:55.664 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:45:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:45:55] "GET /metrics HTTP/1.1" 200 33784 "" "Prometheus/2.43.0" 2023-10-09T05:45:56.294 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:55 smithi064 ceph-mon[212750]: pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:56.295 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:55 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:56.295 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:55 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:56.295 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:55 smithi064 ceph-mon[212750]: Reconfiguring mds.foo.smithi064.gmrwsy (monmap changed)... 2023-10-09T05:45:56.295 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:55 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.gmrwsy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:45:56.295 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:55 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.gmrwsy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:45:56.295 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:55 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:56.295 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:55 smithi064 ceph-mon[212750]: Reconfiguring daemon mds.foo.smithi064.gmrwsy on smithi064 2023-10-09T05:45:56.295 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:55 smithi064 ceph-mon[220044]: pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:56.295 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:55 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:56.296 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:55 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:56.296 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:55 smithi064 ceph-mon[220044]: Reconfiguring mds.foo.smithi064.gmrwsy (monmap changed)... 2023-10-09T05:45:56.296 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:55 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.gmrwsy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:45:56.296 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:55 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.gmrwsy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:45:56.296 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:55 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:56.296 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:55 smithi064 ceph-mon[220044]: Reconfiguring daemon mds.foo.smithi064.gmrwsy on smithi064 2023-10-09T05:45:56.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:55 smithi156 ceph-mon[195101]: pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:56.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:55 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:56.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:55 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:56.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:55 smithi156 ceph-mon[195101]: Reconfiguring mds.foo.smithi064.gmrwsy (monmap changed)... 2023-10-09T05:45:56.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:55 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.gmrwsy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:45:56.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:55 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.gmrwsy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:45:56.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:55 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:56.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:55 smithi156 ceph-mon[195101]: Reconfiguring daemon mds.foo.smithi064.gmrwsy on smithi064 2023-10-09T05:45:57.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:57 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:57.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:57 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:57.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:57 smithi156 ceph-mon[195101]: pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:57.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:57 smithi156 ceph-mon[195101]: Reconfiguring mon.b (monmap changed)... 2023-10-09T05:45:57.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:57 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:45:57.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:57 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:45:57.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:57 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:57.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:57 smithi156 ceph-mon[195101]: Reconfiguring daemon mon.b on smithi156 2023-10-09T05:45:57.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:57 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:57.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:57 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:57.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:57 smithi064 ceph-mon[212750]: pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:57.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:57 smithi064 ceph-mon[212750]: Reconfiguring mon.b (monmap changed)... 2023-10-09T05:45:57.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:57 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:45:57.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:57 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:45:57.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:57 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:57.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:57 smithi064 ceph-mon[212750]: Reconfiguring daemon mon.b on smithi156 2023-10-09T05:45:57.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:57 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:57.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:57 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:57.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:57 smithi064 ceph-mon[220044]: pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:45:57.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:57 smithi064 ceph-mon[220044]: Reconfiguring mon.b (monmap changed)... 2023-10-09T05:45:57.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:57 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:45:57.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:57 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:45:57.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:57 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:57.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:57 smithi064 ceph-mon[220044]: Reconfiguring daemon mon.b on smithi156 2023-10-09T05:45:58.870 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:45:58 smithi156 conmon[179783]: ts=2023-10-09T05:45:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:45:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:58 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:58 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:58 smithi156 ceph-mon[195101]: Reconfiguring mgr.x (monmap changed)... 2023-10-09T05:45:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:58 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:45:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:58 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:45:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:58 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:45:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:58 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:58 smithi156 ceph-mon[195101]: Reconfiguring daemon mgr.x on smithi156 2023-10-09T05:45:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:45:58 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:58.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:58 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:58.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:58 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:58.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:58 smithi064 ceph-mon[212750]: Reconfiguring mgr.x (monmap changed)... 2023-10-09T05:45:58.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:45:58.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:58 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:45:58.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:45:58.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:58.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:58 smithi064 ceph-mon[212750]: Reconfiguring daemon mgr.x on smithi156 2023-10-09T05:45:58.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:45:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:45:58.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:58 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:58.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:58 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:45:58.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:58 smithi064 ceph-mon[220044]: Reconfiguring mgr.x (monmap changed)... 2023-10-09T05:45:58.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:58 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:45:58.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:58 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:45:58.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:58 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:45:58.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:58 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:45:58.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:58 smithi064 ceph-mon[220044]: Reconfiguring daemon mgr.x on smithi156 2023-10-09T05:45:58.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:45:58 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:00.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:00 smithi156 ceph-mon[195101]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:00.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:00 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:00.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:00 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:00.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:00 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T05:46:00.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:00 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:00 smithi064 ceph-mon[212750]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:00 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:00 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:00 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T05:46:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:00 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:00.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:00 smithi064 ceph-mon[220044]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:00.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:00 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:00.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:00 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:00.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:00 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T05:46:00.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:00 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:01.079 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:01 smithi156 ceph-mon[195101]: Reconfiguring osd.4 (monmap changed)... 2023-10-09T05:46:01.079 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:01 smithi156 ceph-mon[195101]: Reconfiguring daemon osd.4 on smithi156 2023-10-09T05:46:01.079 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:01 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:01.079 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:01 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:01.079 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:01 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T05:46:01.080 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:01 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:01.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:01 smithi064 ceph-mon[212750]: Reconfiguring osd.4 (monmap changed)... 2023-10-09T05:46:01.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:01 smithi064 ceph-mon[212750]: Reconfiguring daemon osd.4 on smithi156 2023-10-09T05:46:01.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:01 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:01.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:01 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:01.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:01 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T05:46:01.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:01 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:01.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:01 smithi064 ceph-mon[220044]: Reconfiguring osd.4 (monmap changed)... 2023-10-09T05:46:01.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:01 smithi064 ceph-mon[220044]: Reconfiguring daemon osd.4 on smithi156 2023-10-09T05:46:01.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:01 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:01.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:01 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:01.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:01 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T05:46:01.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:01 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:02.287 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:02 smithi156 ceph-mon[195101]: Reconfiguring osd.5 (monmap changed)... 2023-10-09T05:46:02.287 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:02 smithi156 ceph-mon[195101]: Reconfiguring daemon osd.5 on smithi156 2023-10-09T05:46:02.287 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:02 smithi156 ceph-mon[195101]: pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:02.287 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:02 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:02.287 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:02 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:02.288 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:02 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T05:46:02.288 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:02 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:02.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:02 smithi064 ceph-mon[212750]: Reconfiguring osd.5 (monmap changed)... 2023-10-09T05:46:02.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:02 smithi064 ceph-mon[212750]: Reconfiguring daemon osd.5 on smithi156 2023-10-09T05:46:02.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:02 smithi064 ceph-mon[212750]: pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:02.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:02 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:02.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:02 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:02.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:02 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T05:46:02.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:02 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:02.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:02 smithi064 ceph-mon[220044]: Reconfiguring osd.5 (monmap changed)... 2023-10-09T05:46:02.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:02 smithi064 ceph-mon[220044]: Reconfiguring daemon osd.5 on smithi156 2023-10-09T05:46:02.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:02 smithi064 ceph-mon[220044]: pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:02.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:02 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:02.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:02 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:02.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:02 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T05:46:02.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:02 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:03.284 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:03 smithi156 ceph-mon[195101]: Reconfiguring osd.6 (monmap changed)... 2023-10-09T05:46:03.284 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:03 smithi156 ceph-mon[195101]: Reconfiguring daemon osd.6 on smithi156 2023-10-09T05:46:03.285 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:03 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:03.285 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:03 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:03.285 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:03 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T05:46:03.285 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:03 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:03.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:03 smithi064 ceph-mon[212750]: Reconfiguring osd.6 (monmap changed)... 2023-10-09T05:46:03.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:03 smithi064 ceph-mon[212750]: Reconfiguring daemon osd.6 on smithi156 2023-10-09T05:46:03.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:03 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:03.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:03 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:03.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:03 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T05:46:03.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:03 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:03.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:03 smithi064 ceph-mon[220044]: Reconfiguring osd.6 (monmap changed)... 2023-10-09T05:46:03.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:03 smithi064 ceph-mon[220044]: Reconfiguring daemon osd.6 on smithi156 2023-10-09T05:46:03.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:03 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:03.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:03 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:03.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:03 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T05:46:03.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:03 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:04 smithi156 ceph-mon[195101]: pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:04 smithi156 ceph-mon[195101]: Reconfiguring osd.7 (monmap changed)... 2023-10-09T05:46:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:04 smithi156 ceph-mon[195101]: Reconfiguring daemon osd.7 on smithi156 2023-10-09T05:46:04.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:04 smithi064 ceph-mon[212750]: pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:04.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:04 smithi064 ceph-mon[212750]: Reconfiguring osd.7 (monmap changed)... 2023-10-09T05:46:04.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:04 smithi064 ceph-mon[212750]: Reconfiguring daemon osd.7 on smithi156 2023-10-09T05:46:04.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:04 smithi064 ceph-mon[220044]: pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:04.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:04 smithi064 ceph-mon[220044]: Reconfiguring osd.7 (monmap changed)... 2023-10-09T05:46:04.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:04 smithi064 ceph-mon[220044]: Reconfiguring daemon osd.7 on smithi156 2023-10-09T05:46:05.296 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:05 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:05.296 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:05 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:05.296 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:05 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.pizbdw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:46:05.296 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:05 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.pizbdw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:46:05.297 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:05 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:05.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:46:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:46:05] "GET /metrics HTTP/1.1" 200 33782 "" "Prometheus/2.43.0" 2023-10-09T05:46:05.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:05 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:05.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:05 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:05.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:05 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.pizbdw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:46:05.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:05 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.pizbdw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:46:05.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:05 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:05.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:05 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:05.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:05 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:05.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:05 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.pizbdw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:46:05.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:05 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.pizbdw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:46:05.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:05 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:06 smithi156 ceph-mon[195101]: Reconfiguring mds.foo.smithi156.pizbdw (monmap changed)... 2023-10-09T05:46:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:06 smithi156 ceph-mon[195101]: Reconfiguring daemon mds.foo.smithi156.pizbdw on smithi156 2023-10-09T05:46:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:06 smithi156 ceph-mon[195101]: pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:06 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:06.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:06 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:06.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:06 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:46:06.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:06 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:06.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:06 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:46:06.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:06 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:06 smithi064 ceph-mon[212750]: Reconfiguring mds.foo.smithi156.pizbdw (monmap changed)... 2023-10-09T05:46:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:06 smithi064 ceph-mon[212750]: Reconfiguring daemon mds.foo.smithi156.pizbdw on smithi156 2023-10-09T05:46:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:06 smithi064 ceph-mon[212750]: pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:06 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:06 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:06.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:06 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:46:06.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:06 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:06.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:06 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:46:06.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:06 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:06.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:06 smithi064 ceph-mon[220044]: Reconfiguring mds.foo.smithi156.pizbdw (monmap changed)... 2023-10-09T05:46:06.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:06 smithi064 ceph-mon[220044]: Reconfiguring daemon mds.foo.smithi156.pizbdw on smithi156 2023-10-09T05:46:06.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:06 smithi064 ceph-mon[220044]: pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:06.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:06 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:06.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:06 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:06.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:06 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:46:06.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:06 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:06.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:06 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:46:06.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:06 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:08.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:08 smithi156 ceph-mon[195101]: pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:08.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:08 smithi064 ceph-mon[212750]: pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:08.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:08 smithi064 ceph-mon[220044]: pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:09.120 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:46:08 smithi156 conmon[179783]: ts=2023-10-09T05:46:08.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:46:09.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:09 smithi064 ceph-mon[212750]: pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:09.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:09 smithi064 ceph-mon[220044]: pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:09.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:09 smithi156 ceph-mon[195101]: pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:11.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:11 smithi064 ceph-mon[212750]: pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:11.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:11 smithi064 ceph-mon[220044]: pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:11.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:11 smithi156 ceph-mon[195101]: pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:13 smithi156 ceph-mon[195101]: pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:13 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:14.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:13 smithi064 ceph-mon[212750]: pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:14.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:13 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:14.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:13 smithi064 ceph-mon[220044]: pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:14.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:13 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:15.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:46:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:46:15] "GET /metrics HTTP/1.1" 200 33796 "" "Prometheus/2.43.0" 2023-10-09T05:46:16.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:15 smithi156 ceph-mon[195101]: pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:16.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:15 smithi064 ceph-mon[212750]: pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:16.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:15 smithi064 ceph-mon[220044]: pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:18.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:17 smithi156 ceph-mon[195101]: pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:18.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:17 smithi064 ceph-mon[212750]: pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:18.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:17 smithi064 ceph-mon[220044]: pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:19.120 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:46:18 smithi156 conmon[179783]: ts=2023-10-09T05:46:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:46:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:19 smithi156 ceph-mon[195101]: pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:20.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:19 smithi064 ceph-mon[212750]: pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:20.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:19 smithi064 ceph-mon[220044]: pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:22.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:21 smithi156 ceph-mon[195101]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:22.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:21 smithi064 ceph-mon[212750]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:22.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:21 smithi064 ceph-mon[220044]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:24.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:23 smithi156 ceph-mon[195101]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:24.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:23 smithi064 ceph-mon[212750]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:24.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:23 smithi064 ceph-mon[220044]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:25.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:46:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:46:25] "GET /metrics HTTP/1.1" 200 33796 "" "Prometheus/2.43.0" 2023-10-09T05:46:26.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:25 smithi156 ceph-mon[195101]: pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:26.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:25 smithi064 ceph-mon[212750]: pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:26.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:25 smithi064 ceph-mon[220044]: pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:28.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:28 smithi156 ceph-mon[195101]: pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:28.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:28 smithi064 ceph-mon[212750]: pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:28.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:28 smithi064 ceph-mon[220044]: pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:29.005 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:46:28 smithi156 conmon[179783]: ts=2023-10-09T05:46:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:46:29.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:29.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:29.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:30.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:30 smithi156 ceph-mon[195101]: pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:30.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:30 smithi064 ceph-mon[212750]: pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:30.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:30 smithi064 ceph-mon[220044]: pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:32.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:32 smithi156 ceph-mon[195101]: pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:32.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:32 smithi064 ceph-mon[212750]: pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:32.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:32 smithi064 ceph-mon[220044]: pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:34.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:34 smithi156 ceph-mon[195101]: pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:34.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:34 smithi064 ceph-mon[212750]: pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:34.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:34 smithi064 ceph-mon[220044]: pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:35.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:46:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:46:35] "GET /metrics HTTP/1.1" 200 33793 "" "Prometheus/2.43.0" 2023-10-09T05:46:36.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:36 smithi156 ceph-mon[195101]: pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:36.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:36 smithi064 ceph-mon[212750]: pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:36.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:36 smithi064 ceph-mon[220044]: pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:36.611 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T05:46:36.976 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:38.286 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:38 smithi064 ceph-mon[212750]: pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:38.286 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:38 smithi064 ceph-mon[220044]: pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:38.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:38 smithi156 ceph-mon[195101]: pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:38.733 INFO:teuthology.orchestra.run.smithi064.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:46:39.121 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:46:38 smithi156 conmon[179783]: ts=2023-10-09T05:46:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:46:39.482 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for all mons to upgrade' 2023-10-09T05:46:40.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:39 smithi156 ceph-mon[195101]: pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:40.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:39 smithi156 ceph-mon[195101]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:46:40.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:39 smithi156 ceph-mon[195101]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:46:40.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:39 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:40.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:39 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:46:40.123 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:40.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:39 smithi064 ceph-mon[212750]: pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:40.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:39 smithi064 ceph-mon[212750]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:46:40.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:39 smithi064 ceph-mon[212750]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:46:40.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:39 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:40.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:39 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:46:40.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:39 smithi064 ceph-mon[220044]: pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:40.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:39 smithi064 ceph-mon[220044]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:46:40.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:39 smithi064 ceph-mon[220044]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:46:40.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:39 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:40.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:39 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:46:41.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:41 smithi064 ceph-mon[212750]: pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:41.868 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:41 smithi064 ceph-mon[220044]: pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:41.990 INFO:teuthology.orchestra.run.smithi064.stdout:wait for all mons to upgrade 2023-10-09T05:46:42.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:41 smithi156 ceph-mon[195101]: pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:43.008 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-10-09T05:46:43.440 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:44.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:43 smithi064 ceph-mon[212750]: pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:44.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:43 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:44.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:43 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:44.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:43 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:44.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:43 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:44.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:43 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:46:44.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:43 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:44.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:43 smithi064 ceph-mon[220044]: pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:44.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:43 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:44.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:43 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:44.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:43 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:44.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:43 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:44.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:43 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:46:44.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:43 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:44.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:43 smithi156 ceph-mon[195101]: pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:44.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:43 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:44.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:43 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:44.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:43 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:44.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:43 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:44.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:43 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:46:44.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:43 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:45.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:46:45.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:46:45.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:46:45.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[212750]: Upgrade: Setting container_image for all mon 2023-10-09T05:46:45.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:45.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T05:46:45.304 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T05:46:45.304 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T05:46:45.304 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T05:46:45.304 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T05:46:45.304 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T05:46:45.304 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T05:46:45.304 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T05:46:45.304 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T05:46:45.304 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:46:45.304 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[212750]: Upgrade: Setting container_image for all crash 2023-10-09T05:46:45.304 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:45.304 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T05:46:45.305 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[212750]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T05:46:45.305 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[212750]: Upgrade: osd.0 is safe to restart 2023-10-09T05:46:45.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:46:45.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:46:45.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:46:45.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[220044]: Upgrade: Setting container_image for all mon 2023-10-09T05:46:45.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:45.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T05:46:45.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T05:46:45.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T05:46:45.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T05:46:45.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T05:46:45.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T05:46:45.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T05:46:45.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T05:46:45.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T05:46:45.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:46:45.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[220044]: Upgrade: Setting container_image for all crash 2023-10-09T05:46:45.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:45.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T05:46:45.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[220044]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T05:46:45.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:44 smithi064 ceph-mon[220044]: Upgrade: osd.0 is safe to restart 2023-10-09T05:46:45.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:46:45.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:46:45.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:46:45.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:44 smithi156 ceph-mon[195101]: Upgrade: Setting container_image for all mon 2023-10-09T05:46:45.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:45.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T05:46:45.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T05:46:45.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T05:46:45.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T05:46:45.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T05:46:45.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T05:46:45.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T05:46:45.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T05:46:45.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T05:46:45.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:46:45.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:44 smithi156 ceph-mon[195101]: Upgrade: Setting container_image for all crash 2023-10-09T05:46:45.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:45.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T05:46:45.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:44 smithi156 ceph-mon[195101]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T05:46:45.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:44 smithi156 ceph-mon[195101]: Upgrade: osd.0 is safe to restart 2023-10-09T05:46:45.556 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 3 2023-10-09T05:46:45.573 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:46:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:46:45] "GET /metrics HTTP/1.1" 200 33788 "" "Prometheus/2.43.0" 2023-10-09T05:46:46.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:46 smithi064 ceph-mon[212750]: pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:46.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:46 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2258390030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:46:46.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:46 smithi064 ceph-mon[220044]: pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:46.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:46 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2258390030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:46:46.311 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T05:46:46.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:46 smithi156 ceph-mon[195101]: pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:46.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:46 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2258390030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:46:47.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:47 smithi064 ceph-mon[220044]: Upgrade: Updating osd.0 2023-10-09T05:46:47.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:47 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:47.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:47 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T05:46:47.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:47 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:47.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:47 smithi064 ceph-mon[212750]: Upgrade: Updating osd.0 2023-10-09T05:46:47.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:47 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:47.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:47 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T05:46:47.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:47 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:47.354 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:47 smithi156 ceph-mon[195101]: Upgrade: Updating osd.0 2023-10-09T05:46:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:47 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:47 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T05:46:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:47 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:46:48.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:48 smithi064 ceph-mon[212750]: Deploying daemon osd.0 on smithi064 2023-10-09T05:46:48.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:48 smithi064 ceph-mon[212750]: pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:48.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:48 smithi064 ceph-mon[220044]: Deploying daemon osd.0 on smithi064 2023-10-09T05:46:48.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:48 smithi064 ceph-mon[220044]: pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:48.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:48 smithi156 ceph-mon[195101]: Deploying daemon osd.0 on smithi064 2023-10-09T05:46:48.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:48 smithi156 ceph-mon[195101]: pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:49.094 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:46:48 smithi156 conmon[179783]: ts=2023-10-09T05:46:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:46:49.363 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:49 smithi064 ceph-mon[212750]: pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:49.363 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:49 smithi064 ceph-mon[220044]: pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:49.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:49 smithi156 ceph-mon[195101]: pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:49.671 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:46:49 smithi064 systemd[1]: Stopping Ceph osd.0 for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:46:49.815 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T05:46:49.815 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (3m) - 7m 29.7M - 0.25.0 c8568f914cd2 386f8a4e2221 2023-10-09T05:46:49.815 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi156 *:3000 running (3m) - 7m 87.4M - 9.4.7 2c41d148cca3 365834e48ffb 2023-10-09T05:46:49.815 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.gmrwsy smithi064 running (6m) - 6m 18.3M - 17.2.6-1303-gb2903cb0 5517396c50d4 e938ff2e972d 2023-10-09T05:46:49.816 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi156.pizbdw smithi156 running (6m) - 6m 17.9M - 17.2.6-1303-gb2903cb0 5517396c50d4 e8906e08c938 2023-10-09T05:46:49.816 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi156 *:8443,9283,8765 running (3m) - 11m 448M - 18.2.0-598-gb4fa663f 3060897fdf97 d0092a415024 2023-10-09T05:46:49.816 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:8443,9283,8765 running (3m) - 13m 508M - 18.2.0-598-gb4fa663f 3060897fdf97 2bc7549ee2c1 2023-10-09T05:46:49.816 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (3m) - 13m 61.5M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 5d3854b1b1eb 2023-10-09T05:46:49.816 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi156 running (72s) - 11m 28.8M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 0cd87bdd0b75 2023-10-09T05:46:49.816 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (91s) - 11m 43.5M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 af10058879c1 2023-10-09T05:46:49.816 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (3m) - 7m 18.0M - 1.5.0 0da6a335fe13 afb28622babd 2023-10-09T05:46:49.816 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi156 *:9100 running (3m) - 7m 18.7M - 1.5.0 0da6a335fe13 1e8c1e28630a 2023-10-09T05:46:49.816 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (11m) - 11m 83.5M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 bd3c343a2f7c 2023-10-09T05:46:49.816 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (10m) - 10m 84.6M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 854146c77f6d 2023-10-09T05:46:49.816 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (10m) - 10m 154M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 75a274ca9b6e 2023-10-09T05:46:49.816 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (9m) - 9m 75.2M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 711a9677648c 2023-10-09T05:46:49.816 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi156 running (9m) - 9m 82.1M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 72313d594357 2023-10-09T05:46:49.817 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi156 running (9m) - 9m 103M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 40d9335e2ab7 2023-10-09T05:46:49.817 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi156 running (8m) - 8m 69.2M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 c8a3e3228b89 2023-10-09T05:46:49.817 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi156 running (8m) - 8m 110M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 df93983742a3 2023-10-09T05:46:49.817 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi156 *:9095 running (3m) - 7m 53.0M - 2.43.0 a07b618ecd1d 712dad4d0e3b 2023-10-09T05:46:49.969 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:46:49 smithi064 conmon[124618]: 2023-10-09T05:46:49.670+0000 7fe07f77a700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T05:46:49.969 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:46:49 smithi064 conmon[124618]: 2023-10-09T05:46:49.670+0000 7fe07f77a700 -1 osd.0 80 *** Got signal Terminated *** 2023-10-09T05:46:49.969 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:46:49 smithi064 conmon[124618]: 2023-10-09T05:46:49.670+0000 7fe07f77a700 -1 osd.0 80 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:46:50.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:50 smithi156 ceph-mon[195101]: osd.0 marked itself down and dead 2023-10-09T05:46:50.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:50 smithi156 ceph-mon[195101]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:46:50.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:50 smithi064 ceph-mon[212750]: osd.0 marked itself down and dead 2023-10-09T05:46:50.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:50 smithi064 ceph-mon[212750]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:46:50.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:50 smithi064 ceph-mon[220044]: osd.0 marked itself down and dead 2023-10-09T05:46:50.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:50 smithi064 ceph-mon[220044]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:46:50.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T05:46:50.950 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:46:50 smithi064 bash[232553]: bd3c343a2f7cfbfb335a484719e7abe050179f9f113437cc461313f8da1e467a 2023-10-09T05:46:51.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:51 smithi064 ceph-mon[220044]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:46:51.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:51 smithi064 ceph-mon[220044]: failed to encode map e81 with expected crc 2023-10-09T05:46:51.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:51 smithi064 ceph-mon[220044]: failed to encode map e81 with expected crc 2023-10-09T05:46:51.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:51 smithi064 ceph-mon[220044]: osdmap e81: 8 total, 7 up, 8 in 2023-10-09T05:46:51.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:51 smithi064 ceph-mon[220044]: pgmap v90: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:51.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:51 smithi064 ceph-mon[212750]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:46:51.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:51 smithi064 ceph-mon[212750]: failed to encode map e81 with expected crc 2023-10-09T05:46:51.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:51 smithi064 ceph-mon[212750]: failed to encode map e81 with expected crc 2023-10-09T05:46:51.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:51 smithi064 ceph-mon[212750]: osdmap e81: 8 total, 7 up, 8 in 2023-10-09T05:46:51.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:51 smithi064 ceph-mon[212750]: pgmap v90: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:51.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:51 smithi156 ceph-mon[195101]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:46:51.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:51 smithi156 ceph-mon[195101]: failed to encode map e81 with expected crc 2023-10-09T05:46:51.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:51 smithi156 ceph-mon[195101]: failed to encode map e81 with expected crc 2023-10-09T05:46:51.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:51 smithi156 ceph-mon[195101]: osdmap e81: 8 total, 7 up, 8 in 2023-10-09T05:46:51.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:51 smithi156 ceph-mon[195101]: pgmap v90: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:51.425 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:52.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:52 smithi156 ceph-mon[195101]: failed to encode map e82 with expected crc 2023-10-09T05:46:52.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:52 smithi156 ceph-mon[195101]: failed to encode map e82 with expected crc 2023-10-09T05:46:52.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:52 smithi156 ceph-mon[195101]: osdmap e82: 8 total, 7 up, 8 in 2023-10-09T05:46:52.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:52 smithi156 ceph-mon[195101]: failed to encode map e82 with expected crc 2023-10-09T05:46:52.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:52 smithi156 ceph-mon[195101]: failed to encode map e82 with expected crc 2023-10-09T05:46:52.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:52 smithi156 ceph-mon[195101]: failed to encode map e82 with expected crc 2023-10-09T05:46:52.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:52 smithi156 ceph-mon[195101]: failed to encode map e82 with expected crc 2023-10-09T05:46:52.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:52 smithi156 ceph-mon[195101]: failed to encode map e82 with expected crc 2023-10-09T05:46:52.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:52 smithi156 ceph-mon[195101]: failed to encode map e82 with expected crc 2023-10-09T05:46:52.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:52 smithi064 ceph-mon[220044]: failed to encode map e82 with expected crc 2023-10-09T05:46:52.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:52 smithi064 ceph-mon[220044]: failed to encode map e82 with expected crc 2023-10-09T05:46:52.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:52 smithi064 ceph-mon[220044]: osdmap e82: 8 total, 7 up, 8 in 2023-10-09T05:46:52.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:52 smithi064 ceph-mon[220044]: failed to encode map e82 with expected crc 2023-10-09T05:46:52.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:52 smithi064 ceph-mon[220044]: failed to encode map e82 with expected crc 2023-10-09T05:46:52.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:52 smithi064 ceph-mon[220044]: failed to encode map e82 with expected crc 2023-10-09T05:46:52.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:52 smithi064 ceph-mon[220044]: failed to encode map e82 with expected crc 2023-10-09T05:46:52.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:52 smithi064 ceph-mon[220044]: failed to encode map e82 with expected crc 2023-10-09T05:46:52.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:52 smithi064 ceph-mon[220044]: failed to encode map e82 with expected crc 2023-10-09T05:46:52.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:52 smithi064 ceph-mon[212750]: failed to encode map e82 with expected crc 2023-10-09T05:46:52.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:52 smithi064 ceph-mon[212750]: failed to encode map e82 with expected crc 2023-10-09T05:46:52.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:52 smithi064 ceph-mon[212750]: osdmap e82: 8 total, 7 up, 8 in 2023-10-09T05:46:52.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:52 smithi064 ceph-mon[212750]: failed to encode map e82 with expected crc 2023-10-09T05:46:52.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:52 smithi064 ceph-mon[212750]: failed to encode map e82 with expected crc 2023-10-09T05:46:52.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:52 smithi064 ceph-mon[212750]: failed to encode map e82 with expected crc 2023-10-09T05:46:52.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:52 smithi064 ceph-mon[212750]: failed to encode map e82 with expected crc 2023-10-09T05:46:52.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:52 smithi064 ceph-mon[212750]: failed to encode map e82 with expected crc 2023-10-09T05:46:52.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:52 smithi064 ceph-mon[212750]: failed to encode map e82 with expected crc 2023-10-09T05:46:53.282 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:53 smithi064 ceph-mon[212750]: pgmap v92: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:53.283 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:53 smithi064 ceph-mon[220044]: pgmap v92: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:53.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:53 smithi156 ceph-mon[195101]: pgmap v92: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:46:53.496 INFO:teuthology.orchestra.run.smithi064.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:46:54.191 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:46:54 smithi064 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.0.service: Succeeded. 2023-10-09T05:46:54.191 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:46:54 smithi064 systemd[1]: Stopped Ceph osd.0 for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:46:54.319 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T05:46:54.469 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:46:54 smithi064 systemd[1]: Starting Ceph osd.0 for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:46:54.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:54 smithi156 ceph-mon[195101]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:46:54.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:54 smithi156 ceph-mon[195101]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:46:54.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:54 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:54.918 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:46:54.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:54 smithi064 ceph-mon[212750]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:46:54.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:54 smithi064 ceph-mon[212750]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:46:54.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:54 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:54.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:54 smithi064 ceph-mon[220044]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:46:54.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:54 smithi064 ceph-mon[220044]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:46:54.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:54 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:55.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:46:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:46:55] "GET /metrics HTTP/1.1" 200 33788 "" "Prometheus/2.43.0" 2023-10-09T05:46:55.470 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:46:55 smithi064 bash[233673]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T05:46:55.470 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:46:55 smithi064 bash[233673]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T05:46:55.470 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:46:55 smithi064 bash[233673]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-10-09T05:46:55.470 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:46:55 smithi064 bash[233673]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-10-09T05:46:55.470 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:46:55 smithi064 bash[233673]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-10-09T05:46:55.470 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:46:55 smithi064 bash[233673]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-10-09T05:46:55.470 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:46:55 smithi064 bash[233673]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T05:46:55.470 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:46:55 smithi064 bash[233673]: --> ceph-volume lvm activate successful for osd ID: 0 2023-10-09T05:46:55.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:55 smithi156 ceph-mon[195101]: pgmap v93: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T05:46:55.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:55 smithi156 ceph-mon[195101]: Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:46:55.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:55 smithi064 ceph-mon[212750]: pgmap v93: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T05:46:55.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:55 smithi064 ceph-mon[212750]: Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:46:55.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:55 smithi064 ceph-mon[220044]: pgmap v93: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T05:46:55.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:55 smithi064 ceph-mon[220044]: Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:46:57.097 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:46:57 smithi064 ceph-osd[234473]: -- 172.21.15.64:0/1901991943 <== mon.2 v2:172.21.15.64:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55bffba064e0 con 0x55bffb9d9000 2023-10-09T05:46:57.469 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:46:57 smithi064 bash[234353]: 5779746d3c04380519a1a4a1c4335de455b3843e92b9c225c70fdc6c56bf1364 2023-10-09T05:46:57.469 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:46:57 smithi064 systemd[1]: Started Ceph osd.0 for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:46:57.969 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:46:57 smithi064 conmon[234442]: 2023-10-09T05:46:57.613+0000 7fe5341a4640 -1 Falling back to public interface 2023-10-09T05:46:58.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:57 smithi156 ceph-mon[195101]: pgmap v94: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T05:46:58.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:57 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:58.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:57 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:58.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:57 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:46:58.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:57 smithi064 ceph-mon[212750]: pgmap v94: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T05:46:58.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:57 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:58.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:57 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:58.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:57 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:46:58.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:57 smithi064 ceph-mon[220044]: pgmap v94: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T05:46:58.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:57 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:58.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:57 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:58.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:57 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:46:59.120 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:46:58 smithi156 conmon[179783]: ts=2023-10-09T05:46:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:46:59.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:59 smithi064 ceph-mon[212750]: pgmap v95: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T05:46:59.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:59 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:59.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:46:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:59.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:59 smithi064 ceph-mon[220044]: pgmap v95: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T05:46:59.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:59 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:59.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:46:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:46:59.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:59 smithi156 ceph-mon[195101]: pgmap v95: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T05:46:59.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:59 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:46:59.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:46:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:00.719 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:47:00 smithi064 conmon[234442]: 2023-10-09T05:47:00.388+0000 7fe5341a4640 -1 osd.0 80 log_to_monitors true 2023-10-09T05:47:01.105 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:00 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:01.105 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:00 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:01.106 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:00 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:01.106 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:00 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:01.106 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:00 smithi064 ceph-mon[212750]: from='osd.0 [v2:172.21.15.64:6802/3368956113,v1:172.21.15.64:6803/3368956113]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:47:01.106 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:00 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:01.106 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:00 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:01.106 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:00 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:01.106 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:00 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:01.106 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:00 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:01.106 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:00 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:01.107 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:00 smithi064 ceph-mon[220044]: from='osd.0 [v2:172.21.15.64:6802/3368956113,v1:172.21.15.64:6803/3368956113]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:47:01.107 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:00 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:01.107 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:00 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:01.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:00 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:01.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:00 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:01.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:00 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:01.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:00 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:01.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:00 smithi156 ceph-mon[195101]: from='osd.0 [v2:172.21.15.64:6802/3368956113,v1:172.21.15.64:6803/3368956113]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:47:01.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:00 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:01.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:00 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:01.449 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:47:01 smithi064 conmon[234442]: 2023-10-09T05:47:01.234+0000 7fe521e09700 -1 osd.0 80 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:47:02.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:01 smithi156 ceph-mon[195101]: pgmap v96: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T05:47:02.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:01 smithi156 ceph-mon[195101]: failed to encode map e83 with expected crc 2023-10-09T05:47:02.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:01 smithi156 ceph-mon[195101]: failed to encode map e83 with expected crc 2023-10-09T05:47:02.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:01 smithi156 ceph-mon[195101]: from='osd.0 [v2:172.21.15.64:6802/3368956113,v1:172.21.15.64:6803/3368956113]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T05:47:02.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:01 smithi156 ceph-mon[195101]: osdmap e83: 8 total, 7 up, 8 in 2023-10-09T05:47:02.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:01 smithi156 ceph-mon[195101]: from='osd.0 [v2:172.21.15.64:6802/3368956113,v1:172.21.15.64:6803/3368956113]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:47:02.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:01 smithi064 ceph-mon[212750]: pgmap v96: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T05:47:02.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:01 smithi064 ceph-mon[212750]: failed to encode map e83 with expected crc 2023-10-09T05:47:02.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:01 smithi064 ceph-mon[212750]: failed to encode map e83 with expected crc 2023-10-09T05:47:02.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:01 smithi064 ceph-mon[212750]: from='osd.0 [v2:172.21.15.64:6802/3368956113,v1:172.21.15.64:6803/3368956113]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T05:47:02.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:01 smithi064 ceph-mon[212750]: osdmap e83: 8 total, 7 up, 8 in 2023-10-09T05:47:02.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:01 smithi064 ceph-mon[212750]: from='osd.0 [v2:172.21.15.64:6802/3368956113,v1:172.21.15.64:6803/3368956113]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:47:02.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:01 smithi064 ceph-mon[220044]: pgmap v96: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T05:47:02.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:01 smithi064 ceph-mon[220044]: failed to encode map e83 with expected crc 2023-10-09T05:47:02.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:01 smithi064 ceph-mon[220044]: failed to encode map e83 with expected crc 2023-10-09T05:47:02.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:01 smithi064 ceph-mon[220044]: from='osd.0 [v2:172.21.15.64:6802/3368956113,v1:172.21.15.64:6803/3368956113]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T05:47:02.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:01 smithi064 ceph-mon[220044]: osdmap e83: 8 total, 7 up, 8 in 2023-10-09T05:47:02.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:01 smithi064 ceph-mon[220044]: from='osd.0 [v2:172.21.15.64:6802/3368956113,v1:172.21.15.64:6803/3368956113]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:47:03.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:02 smithi156 ceph-mon[195101]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:47:03.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:02 smithi156 ceph-mon[195101]: failed to encode map e84 with expected crc 2023-10-09T05:47:03.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:02 smithi156 ceph-mon[195101]: failed to encode map e83 with expected crc 2023-10-09T05:47:03.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:02 smithi156 ceph-mon[195101]: osd.0 [v2:172.21.15.64:6802/3368956113,v1:172.21.15.64:6803/3368956113] boot 2023-10-09T05:47:03.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:02 smithi156 ceph-mon[195101]: osdmap e84: 8 total, 8 up, 8 in 2023-10-09T05:47:03.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:02 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:47:03.397 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:02 smithi064 ceph-mon[220044]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:47:03.397 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:02 smithi064 ceph-mon[220044]: failed to encode map e84 with expected crc 2023-10-09T05:47:03.397 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:02 smithi064 ceph-mon[220044]: failed to encode map e83 with expected crc 2023-10-09T05:47:03.397 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:02 smithi064 ceph-mon[220044]: osd.0 [v2:172.21.15.64:6802/3368956113,v1:172.21.15.64:6803/3368956113] boot 2023-10-09T05:47:03.397 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:02 smithi064 ceph-mon[220044]: osdmap e84: 8 total, 8 up, 8 in 2023-10-09T05:47:03.397 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:02 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:47:03.397 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:02 smithi064 ceph-mon[212750]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:47:03.397 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:02 smithi064 ceph-mon[212750]: failed to encode map e84 with expected crc 2023-10-09T05:47:03.398 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:02 smithi064 ceph-mon[212750]: failed to encode map e83 with expected crc 2023-10-09T05:47:03.398 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:02 smithi064 ceph-mon[212750]: osd.0 [v2:172.21.15.64:6802/3368956113,v1:172.21.15.64:6803/3368956113] boot 2023-10-09T05:47:03.398 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:02 smithi064 ceph-mon[212750]: osdmap e84: 8 total, 8 up, 8 in 2023-10-09T05:47:03.398 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:02 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:47:04.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[212750]: pgmap v99: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T05:47:04.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[212750]: failed to encode map e85 with expected crc 2023-10-09T05:47:04.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[212750]: failed to encode map e85 with expected crc 2023-10-09T05:47:04.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[212750]: osdmap e85: 8 total, 8 up, 8 in 2023-10-09T05:47:04.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[212750]: failed to encode map e85 with expected crc 2023-10-09T05:47:04.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[212750]: failed to encode map e85 with expected crc 2023-10-09T05:47:04.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[212750]: failed to encode map e85 with expected crc 2023-10-09T05:47:04.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[212750]: failed to encode map e85 with expected crc 2023-10-09T05:47:04.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[212750]: failed to encode map e85 with expected crc 2023-10-09T05:47:04.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[212750]: failed to encode map e85 with expected crc 2023-10-09T05:47:04.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:04.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:04.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:47:04.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:47:04.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:04.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[220044]: pgmap v99: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T05:47:04.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[220044]: failed to encode map e85 with expected crc 2023-10-09T05:47:04.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[220044]: failed to encode map e85 with expected crc 2023-10-09T05:47:04.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[220044]: osdmap e85: 8 total, 8 up, 8 in 2023-10-09T05:47:04.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[220044]: failed to encode map e85 with expected crc 2023-10-09T05:47:04.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[220044]: failed to encode map e85 with expected crc 2023-10-09T05:47:04.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[220044]: failed to encode map e85 with expected crc 2023-10-09T05:47:04.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[220044]: failed to encode map e85 with expected crc 2023-10-09T05:47:04.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[220044]: failed to encode map e85 with expected crc 2023-10-09T05:47:04.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[220044]: failed to encode map e85 with expected crc 2023-10-09T05:47:04.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:04.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:04.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:47:04.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:47:04.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:03 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:03 smithi156 ceph-mon[195101]: pgmap v99: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T05:47:04.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:03 smithi156 ceph-mon[195101]: failed to encode map e85 with expected crc 2023-10-09T05:47:04.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:03 smithi156 ceph-mon[195101]: failed to encode map e85 with expected crc 2023-10-09T05:47:04.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:03 smithi156 ceph-mon[195101]: osdmap e85: 8 total, 8 up, 8 in 2023-10-09T05:47:04.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:03 smithi156 ceph-mon[195101]: failed to encode map e85 with expected crc 2023-10-09T05:47:04.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:03 smithi156 ceph-mon[195101]: failed to encode map e85 with expected crc 2023-10-09T05:47:04.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:03 smithi156 ceph-mon[195101]: failed to encode map e85 with expected crc 2023-10-09T05:47:04.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:03 smithi156 ceph-mon[195101]: failed to encode map e85 with expected crc 2023-10-09T05:47:04.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:03 smithi156 ceph-mon[195101]: failed to encode map e85 with expected crc 2023-10-09T05:47:04.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:03 smithi156 ceph-mon[195101]: failed to encode map e85 with expected crc 2023-10-09T05:47:04.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:03 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:04.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:03 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:04.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:03 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:47:04.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:03 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:47:04.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:03 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:05.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:04 smithi064 ceph-mon[212750]: failed to encode map e85 with expected crc 2023-10-09T05:47:05.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:04 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:47:05.304 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:04 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:47:05.304 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:04 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:47:05.304 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:04 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:05.304 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:04 smithi064 ceph-mon[220044]: failed to encode map e85 with expected crc 2023-10-09T05:47:05.304 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:04 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:47:05.304 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:04 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:47:05.304 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:04 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:47:05.304 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:04 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:05.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:04 smithi156 ceph-mon[195101]: failed to encode map e85 with expected crc 2023-10-09T05:47:05.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:04 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:47:05.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:04 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:47:05.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:04 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:47:05.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:04 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:05.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:47:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:47:05] "GET /metrics HTTP/1.1" 200 33921 "" "Prometheus/2.43.0" 2023-10-09T05:47:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:05 smithi156 ceph-mon[195101]: pgmap v101: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T05:47:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:05 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:47:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:05 smithi064 ceph-mon[212750]: pgmap v101: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T05:47:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:05 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:47:06.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:05 smithi064 ceph-mon[220044]: pgmap v101: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T05:47:06.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:05 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:47:08.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:07 smithi156 ceph-mon[195101]: pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:08.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:07 smithi156 ceph-mon[195101]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2023-10-09T05:47:08.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:07 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T05:47:08.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:07 smithi064 ceph-mon[212750]: pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:08.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:07 smithi064 ceph-mon[212750]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2023-10-09T05:47:08.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:07 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T05:47:08.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:07 smithi064 ceph-mon[220044]: pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:08.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:07 smithi064 ceph-mon[220044]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2023-10-09T05:47:08.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:07 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T05:47:09.120 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:47:08 smithi156 conmon[179783]: ts=2023-10-09T05:47:08.661Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2023-10-09T05:47:09.120 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:47:08 smithi156 conmon[179783]: ts=2023-10-09T05:47:08.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:47:10.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:09 smithi156 ceph-mon[195101]: pgmap v103: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:10.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:09 smithi064 ceph-mon[212750]: pgmap v103: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:10.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:09 smithi064 ceph-mon[220044]: pgmap v103: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:12.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:11 smithi156 ceph-mon[195101]: pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:12.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:11 smithi064 ceph-mon[212750]: pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:12.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:11 smithi064 ceph-mon[220044]: pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:14 smithi156 ceph-mon[195101]: pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:14.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:14 smithi064 ceph-mon[220044]: pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:14.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:14.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:14.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:14.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:13 smithi064 ceph-mon[212750]: pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:14.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:13 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:14.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:13 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:14.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:47:15.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:47:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:47:15] "GET /metrics HTTP/1.1" 200 33914 "" "Prometheus/2.43.0" 2023-10-09T05:47:16.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:16 smithi156 ceph-mon[195101]: pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:16.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:16 smithi064 ceph-mon[212750]: pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:16.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:16 smithi064 ceph-mon[220044]: pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:18.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:18 smithi156 ceph-mon[195101]: pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:18.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:18 smithi064 ceph-mon[212750]: pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:18.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:18 smithi064 ceph-mon[220044]: pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:19.120 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:47:18 smithi156 conmon[179783]: ts=2023-10-09T05:47:18.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:47:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:20 smithi156 ceph-mon[195101]: pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:20.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:20 smithi064 ceph-mon[212750]: pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:20.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:20 smithi064 ceph-mon[220044]: pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:22.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:22 smithi156 ceph-mon[195101]: pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:22.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:22 smithi064 ceph-mon[212750]: pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:22.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:22 smithi064 ceph-mon[220044]: pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:24.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:24 smithi156 ceph-mon[195101]: pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:24.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:24 smithi064 ceph-mon[212750]: pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:24.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:24 smithi064 ceph-mon[220044]: pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:25.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:47:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:47:25] "GET /metrics HTTP/1.1" 200 33914 "" "Prometheus/2.43.0" 2023-10-09T05:47:26.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:26 smithi156 ceph-mon[195101]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:26.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:26 smithi064 ceph-mon[212750]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:26.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:26 smithi064 ceph-mon[220044]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:28.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:28 smithi156 ceph-mon[195101]: pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:28.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:28 smithi064 ceph-mon[212750]: pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:28.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:28 smithi064 ceph-mon[220044]: pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:29.030 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:47:28 smithi156 conmon[179783]: ts=2023-10-09T05:47:28.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:47:29.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:29.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:29.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:30.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:30 smithi156 ceph-mon[195101]: pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:30.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:30 smithi064 ceph-mon[212750]: pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:30.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:30 smithi064 ceph-mon[220044]: pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:32.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:32 smithi156 ceph-mon[195101]: pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:32.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:32 smithi064 ceph-mon[212750]: pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:32.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:32 smithi064 ceph-mon[220044]: pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:34.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:34 smithi156 ceph-mon[195101]: pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:34.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:34 smithi064 ceph-mon[212750]: pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:34.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:34 smithi064 ceph-mon[220044]: pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:35.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:47:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:47:35] "GET /metrics HTTP/1.1" 200 33912 "" "Prometheus/2.43.0" 2023-10-09T05:47:36.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:36 smithi156 ceph-mon[195101]: pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:36.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:36 smithi064 ceph-mon[212750]: pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:36.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:36 smithi064 ceph-mon[220044]: pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:38.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:38 smithi156 ceph-mon[195101]: pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:38.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:38 smithi064 ceph-mon[212750]: pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:38.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:38 smithi064 ceph-mon[220044]: pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:39.120 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:47:38 smithi156 conmon[179783]: ts=2023-10-09T05:47:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:47:40.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:40 smithi156 ceph-mon[195101]: pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:40.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:40 smithi064 ceph-mon[212750]: pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:40.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:40 smithi064 ceph-mon[220044]: pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:42.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:42 smithi156 ceph-mon[195101]: pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:42.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:42 smithi064 ceph-mon[212750]: pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:42.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:42 smithi064 ceph-mon[220044]: pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:44.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:44 smithi156 ceph-mon[195101]: pgmap v120: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:44.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:44.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:44 smithi064 ceph-mon[212750]: pgmap v120: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:44.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:44.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:44 smithi064 ceph-mon[220044]: pgmap v120: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:44.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:45.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:47:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:47:45] "GET /metrics HTTP/1.1" 200 33919 "" "Prometheus/2.43.0" 2023-10-09T05:47:46.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:46 smithi156 ceph-mon[195101]: pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:46.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:46 smithi064 ceph-mon[212750]: pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:46.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:46 smithi064 ceph-mon[220044]: pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:48.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:48 smithi156 ceph-mon[195101]: pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:48.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:48 smithi064 ceph-mon[212750]: pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:48.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:48 smithi064 ceph-mon[220044]: pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:49.120 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 05:47:48 smithi156 conmon[179783]: ts=2023-10-09T05:47:48.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi156.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T05:47:50.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:50 smithi156 ceph-mon[195101]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:50.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:50 smithi064 ceph-mon[212750]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:50.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:50 smithi064 ceph-mon[220044]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:52.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:52 smithi156 ceph-mon[195101]: pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:52.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:52 smithi064 ceph-mon[212750]: pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:52.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:52 smithi064 ceph-mon[220044]: pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:54.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:54 smithi156 ceph-mon[195101]: pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:54.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:54 smithi064 ceph-mon[212750]: pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:54.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:54 smithi064 ceph-mon[220044]: pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:55.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:47:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:47:55] "GET /metrics HTTP/1.1" 200 33919 "" "Prometheus/2.43.0" 2023-10-09T05:47:56.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:56 smithi156 ceph-mon[195101]: pgmap v126: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:56.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:56 smithi064 ceph-mon[220044]: pgmap v126: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:56.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:56 smithi064 ceph-mon[212750]: pgmap v126: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:57.441 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T05:47:57.805 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:47:58.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:58 smithi156 ceph-mon[195101]: pgmap v127: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:58.419 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:58 smithi064 ceph-mon[212750]: pgmap v127: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:58.419 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:58 smithi064 ceph-mon[220044]: pgmap v127: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:47:59.141 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:47:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:59.142 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:47:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:59.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:47:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:47:59.592 INFO:teuthology.orchestra.run.smithi064.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:48:00.295 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for half of osds to upgrade' 2023-10-09T05:48:00.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:00 smithi156 ceph-mon[195101]: pgmap v128: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:00.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:00 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:00.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:00 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:48:00.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:00 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:48:00.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:00 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:48:00.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:00 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:00.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:00 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:48:00.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:00 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:00.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:00 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:00.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:00 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:00.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:00 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T05:48:00.456 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:00 smithi064 ceph-mon[212750]: pgmap v128: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:00.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:00 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:00.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:00 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:48:00.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:00 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:48:00.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:00 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:48:00.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:00 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:00.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:00 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:48:00.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:00 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:00.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:00 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:00.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:00 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:00.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:00 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T05:48:00.458 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:00 smithi064 ceph-mon[220044]: pgmap v128: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:00.458 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:00 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:00.458 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:00 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:48:00.458 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:00 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:48:00.458 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:00 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:48:00.458 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:00 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:00.458 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:00 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:48:00.459 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:00 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:00.459 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:00 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:00.459 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:00 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:00.459 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:00 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T05:48:00.706 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:48:01.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:01 smithi156 ceph-mon[195101]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:48:01.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:01 smithi156 ceph-mon[195101]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:48:01.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:01 smithi156 ceph-mon[195101]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T05:48:01.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:01 smithi156 ceph-mon[195101]: Upgrade: osd.1 is safe to restart 2023-10-09T05:48:01.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:01 smithi064 ceph-mon[220044]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:48:01.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:01 smithi064 ceph-mon[220044]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:48:01.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:01 smithi064 ceph-mon[220044]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T05:48:01.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:01 smithi064 ceph-mon[220044]: Upgrade: osd.1 is safe to restart 2023-10-09T05:48:01.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:01 smithi064 ceph-mon[212750]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:48:01.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:01 smithi064 ceph-mon[212750]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:48:01.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:01 smithi064 ceph-mon[212750]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T05:48:01.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:01 smithi064 ceph-mon[212750]: Upgrade: osd.1 is safe to restart 2023-10-09T05:48:02.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:02 smithi156 ceph-mon[195101]: pgmap v129: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:02.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:02 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:02.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:02 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T05:48:02.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:02 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:48:02.397 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:02 smithi064 ceph-mon[212750]: pgmap v129: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:02.397 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:02 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:02.398 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:02 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T05:48:02.398 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:02 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:48:02.398 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:02 smithi064 ceph-mon[220044]: pgmap v129: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:02.398 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:02 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:02.398 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:02 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T05:48:02.398 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:02 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:48:02.703 INFO:teuthology.orchestra.run.smithi064.stdout:wait for half of osds to upgrade 2023-10-09T05:48:03.364 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:03 smithi064 ceph-mon[212750]: Upgrade: Updating osd.1 2023-10-09T05:48:03.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:03 smithi064 ceph-mon[212750]: Deploying daemon osd.1 on smithi064 2023-10-09T05:48:03.365 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:03 smithi064 ceph-mon[220044]: Upgrade: Updating osd.1 2023-10-09T05:48:03.365 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:03 smithi064 ceph-mon[220044]: Deploying daemon osd.1 on smithi064 2023-10-09T05:48:03.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:03 smithi156 ceph-mon[195101]: Upgrade: Updating osd.1 2023-10-09T05:48:03.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:03 smithi156 ceph-mon[195101]: Deploying daemon osd.1 on smithi064 2023-10-09T05:48:03.694 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-10-09T05:48:04.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:04 smithi064 ceph-mon[212750]: pgmap v130: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:04.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:04 smithi064 ceph-mon[220044]: pgmap v130: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:04.220 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:48:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:04 smithi156 ceph-mon[195101]: pgmap v130: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:05.719 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:48:05 smithi064 systemd[1]: Stopping Ceph osd.1 for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:48:05.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:48:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:48:05] "GET /metrics HTTP/1.1" 200 33910 "" "Prometheus/2.43.0" 2023-10-09T05:48:06.073 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:48:05 smithi064 conmon[133568]: 2023-10-09T05:48:05.853+0000 7f10370b0700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T05:48:06.074 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:48:05 smithi064 conmon[133568]: 2023-10-09T05:48:05.853+0000 7f10370b0700 -1 osd.1 85 *** Got signal Terminated *** 2023-10-09T05:48:06.074 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:48:05 smithi064 conmon[133568]: 2023-10-09T05:48:05.853+0000 7f10370b0700 -1 osd.1 85 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:48:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:06 smithi156 ceph-mon[195101]: pgmap v131: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:06 smithi156 ceph-mon[195101]: osd.1 marked itself down and dead 2023-10-09T05:48:06.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:06 smithi064 ceph-mon[212750]: pgmap v131: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:06.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:06 smithi064 ceph-mon[212750]: osd.1 marked itself down and dead 2023-10-09T05:48:06.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:06 smithi064 ceph-mon[220044]: pgmap v131: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:06.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:06 smithi064 ceph-mon[220044]: osd.1 marked itself down and dead 2023-10-09T05:48:06.969 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:48:06 smithi064 bash[241772]: 854146c77f6d65ac2477243f62bef9a3a5cab660669ca06be9080cba277d7d1d 2023-10-09T05:48:07.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:07 smithi156 ceph-mon[195101]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:48:07.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:07 smithi156 ceph-mon[195101]: failed to encode map e86 with expected crc 2023-10-09T05:48:07.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:07 smithi156 ceph-mon[195101]: osdmap e86: 8 total, 7 up, 8 in 2023-10-09T05:48:07.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:07 smithi156 ceph-mon[195101]: failed to encode map e86 with expected crc 2023-10-09T05:48:07.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:07 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2273547997' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:07 smithi064 ceph-mon[212750]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:48:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:07 smithi064 ceph-mon[212750]: failed to encode map e86 with expected crc 2023-10-09T05:48:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:07 smithi064 ceph-mon[212750]: osdmap e86: 8 total, 7 up, 8 in 2023-10-09T05:48:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:07 smithi064 ceph-mon[212750]: failed to encode map e86 with expected crc 2023-10-09T05:48:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:07 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2273547997' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:07 smithi064 ceph-mon[220044]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:48:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:07 smithi064 ceph-mon[220044]: failed to encode map e86 with expected crc 2023-10-09T05:48:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:07 smithi064 ceph-mon[220044]: osdmap e86: 8 total, 7 up, 8 in 2023-10-09T05:48:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:07 smithi064 ceph-mon[220044]: failed to encode map e86 with expected crc 2023-10-09T05:48:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:07 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2273547997' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:08.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:08 smithi064 ceph-mon[212750]: pgmap v133: 73 pgs: 4 peering, 7 stale+active+clean, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:08.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:08 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 1 pg inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T05:48:08.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:08 smithi064 ceph-mon[212750]: failed to encode map e87 with expected crc 2023-10-09T05:48:08.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:08 smithi064 ceph-mon[212750]: osdmap e87: 8 total, 7 up, 8 in 2023-10-09T05:48:08.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:08 smithi064 ceph-mon[212750]: failed to encode map e87 with expected crc 2023-10-09T05:48:08.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:08 smithi064 ceph-mon[212750]: failed to encode map e87 with expected crc 2023-10-09T05:48:08.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:08 smithi064 ceph-mon[212750]: failed to encode map e87 with expected crc 2023-10-09T05:48:08.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:08 smithi064 ceph-mon[212750]: failed to encode map e87 with expected crc 2023-10-09T05:48:08.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:08 smithi064 ceph-mon[212750]: failed to encode map e87 with expected crc 2023-10-09T05:48:08.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:08 smithi064 ceph-mon[212750]: failed to encode map e87 with expected crc 2023-10-09T05:48:08.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:08 smithi064 ceph-mon[212750]: failed to encode map e87 with expected crc 2023-10-09T05:48:08.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:08 smithi064 ceph-mon[220044]: pgmap v133: 73 pgs: 4 peering, 7 stale+active+clean, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:08.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:08 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 1 pg inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T05:48:08.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:08 smithi064 ceph-mon[220044]: failed to encode map e87 with expected crc 2023-10-09T05:48:08.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:08 smithi064 ceph-mon[220044]: osdmap e87: 8 total, 7 up, 8 in 2023-10-09T05:48:08.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:08 smithi064 ceph-mon[220044]: failed to encode map e87 with expected crc 2023-10-09T05:48:08.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:08 smithi064 ceph-mon[220044]: failed to encode map e87 with expected crc 2023-10-09T05:48:08.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:08 smithi064 ceph-mon[220044]: failed to encode map e87 with expected crc 2023-10-09T05:48:08.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:08 smithi064 ceph-mon[220044]: failed to encode map e87 with expected crc 2023-10-09T05:48:08.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:08 smithi064 ceph-mon[220044]: failed to encode map e87 with expected crc 2023-10-09T05:48:08.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:08 smithi064 ceph-mon[220044]: failed to encode map e87 with expected crc 2023-10-09T05:48:08.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:08 smithi064 ceph-mon[220044]: failed to encode map e87 with expected crc 2023-10-09T05:48:08.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:08 smithi156 ceph-mon[195101]: pgmap v133: 73 pgs: 4 peering, 7 stale+active+clean, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:08.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:08 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 1 pg inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T05:48:08.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:08 smithi156 ceph-mon[195101]: failed to encode map e87 with expected crc 2023-10-09T05:48:08.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:08 smithi156 ceph-mon[195101]: osdmap e87: 8 total, 7 up, 8 in 2023-10-09T05:48:08.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:08 smithi156 ceph-mon[195101]: failed to encode map e87 with expected crc 2023-10-09T05:48:08.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:08 smithi156 ceph-mon[195101]: failed to encode map e87 with expected crc 2023-10-09T05:48:08.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:08 smithi156 ceph-mon[195101]: failed to encode map e87 with expected crc 2023-10-09T05:48:08.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:08 smithi156 ceph-mon[195101]: failed to encode map e87 with expected crc 2023-10-09T05:48:08.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:08 smithi156 ceph-mon[195101]: failed to encode map e87 with expected crc 2023-10-09T05:48:08.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:08 smithi156 ceph-mon[195101]: failed to encode map e87 with expected crc 2023-10-09T05:48:08.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:08 smithi156 ceph-mon[195101]: failed to encode map e87 with expected crc 2023-10-09T05:48:08.969 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:48:08 smithi064 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.1.service: Succeeded. 2023-10-09T05:48:08.969 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:48:08 smithi064 systemd[1]: Stopped Ceph osd.1 for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:48:08.969 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:48:08 smithi064 systemd[1]: Starting Ceph osd.1 for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:48:09.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:09 smithi156 ceph-mon[195101]: pgmap v135: 73 pgs: 4 peering, 7 stale+active+clean, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:09.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:09 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3919058981' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:09.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:09 smithi064 ceph-mon[212750]: pgmap v135: 73 pgs: 4 peering, 7 stale+active+clean, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:09.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:09 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3919058981' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:09.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:09 smithi064 ceph-mon[220044]: pgmap v135: 73 pgs: 4 peering, 7 stale+active+clean, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:09.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:09 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3919058981' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:09.969 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:48:09 smithi064 bash[242547]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T05:48:09.969 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:48:09 smithi064 bash[242547]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T05:48:09.969 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:48:09 smithi064 bash[242547]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-10-09T05:48:09.969 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:48:09 smithi064 bash[242547]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-10-09T05:48:09.969 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:48:09 smithi064 bash[242547]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-10-09T05:48:09.970 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:48:09 smithi064 bash[242547]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-10-09T05:48:09.970 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:48:09 smithi064 bash[242547]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T05:48:09.970 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:48:09 smithi064 bash[242547]: --> ceph-volume lvm activate successful for osd ID: 1 2023-10-09T05:48:10.860 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:10 smithi064 ceph-mon[212750]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:48:10.860 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:10 smithi064 ceph-mon[220044]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:48:10.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:10 smithi156 ceph-mon[195101]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:48:11.219 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:48:10 smithi064 ceph-osd[243041]: -- 172.21.15.64:0/2386671025 <== mon.0 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5615002664e0 con 0x561500239400 2023-10-09T05:48:11.219 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:48:10 smithi064 bash[242973]: 2e7bf325c8048e148312d76fb93be30813c876c855b8e0f706e97310b6ee1c08 2023-10-09T05:48:11.219 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:48:10 smithi064 systemd[1]: Started Ceph osd.1 for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:48:11.719 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:48:11 smithi064 conmon[243014]: 2023-10-09T05:48:11.464+0000 7f0e4603a640 -1 Falling back to public interface 2023-10-09T05:48:12.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:11 smithi156 ceph-mon[195101]: pgmap v136: 73 pgs: 2 active+undersized+degraded, 4 peering, 26 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T05:48:12.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:11 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:12.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:11 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:12.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:11 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:48:12.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:11 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1523164531' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:12.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:11 smithi064 ceph-mon[220044]: pgmap v136: 73 pgs: 2 active+undersized+degraded, 4 peering, 26 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T05:48:12.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:11 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:12.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:11 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:12.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:11 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:48:12.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:11 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1523164531' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:12.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:11 smithi064 ceph-mon[212750]: pgmap v136: 73 pgs: 2 active+undersized+degraded, 4 peering, 26 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T05:48:12.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:11 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:12.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:11 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:12.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:11 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:48:12.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:11 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1523164531' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:13.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:13 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 4 pgs peering) 2023-10-09T05:48:13.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:13 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 4 pgs peering) 2023-10-09T05:48:13.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:13 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 4 pgs peering) 2023-10-09T05:48:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:14 smithi156 ceph-mon[195101]: pgmap v137: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T05:48:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:48:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:14 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1361199555' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:14.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:14.469 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:48:14 smithi064 conmon[243014]: 2023-10-09T05:48:14.219+0000 7f0e4603a640 -1 osd.1 85 log_to_monitors true 2023-10-09T05:48:14.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:14 smithi064 ceph-mon[212750]: pgmap v137: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T05:48:14.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:14.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:48:14.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:14 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1361199555' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:14.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:14.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:14.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:14 smithi064 ceph-mon[220044]: pgmap v137: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T05:48:14.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:14.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:48:14.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:14 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1361199555' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:14.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:14.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:15.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:15 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:15.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:15 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:15.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:15 smithi156 ceph-mon[195101]: from='osd.1 [v2:172.21.15.64:6810/3188144294,v1:172.21.15.64:6811/3188144294]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:48:15.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:15 smithi156 ceph-mon[195101]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:48:15.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:15 smithi156 ceph-mon[195101]: pgmap v138: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T05:48:15.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:15 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:15.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:15 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:15.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:48:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:48:15] "GET /metrics HTTP/1.1" 200 33916 "" "Prometheus/2.43.0" 2023-10-09T05:48:15.470 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 05:48:15 smithi064 conmon[243014]: 2023-10-09T05:48:15.175+0000 7f0e354a2700 -1 osd.1 85 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:48:15.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:15 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:15.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:15 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:15.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:15 smithi064 ceph-mon[212750]: from='osd.1 [v2:172.21.15.64:6810/3188144294,v1:172.21.15.64:6811/3188144294]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:48:15.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:15 smithi064 ceph-mon[212750]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:48:15.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:15 smithi064 ceph-mon[212750]: pgmap v138: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T05:48:15.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:15 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:15.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:15 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:15.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:15 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:15.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:15 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:15.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:15 smithi064 ceph-mon[220044]: from='osd.1 [v2:172.21.15.64:6810/3188144294,v1:172.21.15.64:6811/3188144294]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:48:15.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:15 smithi064 ceph-mon[220044]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:48:15.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:15 smithi064 ceph-mon[220044]: pgmap v138: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T05:48:15.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:15 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:15.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:15 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:16.169 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:16 smithi156 ceph-mon[195101]: failed to encode map e88 with expected crc 2023-10-09T05:48:16.169 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:16 smithi156 ceph-mon[195101]: failed to encode map e88 with expected crc 2023-10-09T05:48:16.169 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:16 smithi156 ceph-mon[195101]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T05:48:16.169 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:16 smithi156 ceph-mon[195101]: osdmap e88: 8 total, 7 up, 8 in 2023-10-09T05:48:16.169 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:16 smithi156 ceph-mon[195101]: from='osd.1 [v2:172.21.15.64:6810/3188144294,v1:172.21.15.64:6811/3188144294]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:48:16.169 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:16 smithi156 ceph-mon[195101]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:48:16.169 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:16 smithi156 ceph-mon[195101]: failed to encode map e88 with expected crc 2023-10-09T05:48:16.169 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:16 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2953317381' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:16.289 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:16 smithi064 ceph-mon[212750]: failed to encode map e88 with expected crc 2023-10-09T05:48:16.289 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:16 smithi064 ceph-mon[212750]: failed to encode map e88 with expected crc 2023-10-09T05:48:16.290 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:16 smithi064 ceph-mon[212750]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T05:48:16.290 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:16 smithi064 ceph-mon[212750]: osdmap e88: 8 total, 7 up, 8 in 2023-10-09T05:48:16.290 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:16 smithi064 ceph-mon[212750]: from='osd.1 [v2:172.21.15.64:6810/3188144294,v1:172.21.15.64:6811/3188144294]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:48:16.290 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:16 smithi064 ceph-mon[212750]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:48:16.290 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:16 smithi064 ceph-mon[212750]: failed to encode map e88 with expected crc 2023-10-09T05:48:16.290 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:16 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2953317381' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:16.290 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:16 smithi064 ceph-mon[220044]: failed to encode map e88 with expected crc 2023-10-09T05:48:16.290 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:16 smithi064 ceph-mon[220044]: failed to encode map e88 with expected crc 2023-10-09T05:48:16.290 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:16 smithi064 ceph-mon[220044]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T05:48:16.291 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:16 smithi064 ceph-mon[220044]: osdmap e88: 8 total, 7 up, 8 in 2023-10-09T05:48:16.291 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:16 smithi064 ceph-mon[220044]: from='osd.1 [v2:172.21.15.64:6810/3188144294,v1:172.21.15.64:6811/3188144294]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:48:16.291 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:16 smithi064 ceph-mon[220044]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:48:16.291 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:16 smithi064 ceph-mon[220044]: failed to encode map e88 with expected crc 2023-10-09T05:48:16.291 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:16 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2953317381' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:17.183 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:17 smithi156 ceph-mon[195101]: failed to encode map e88 with expected crc 2023-10-09T05:48:17.183 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:17 smithi156 ceph-mon[195101]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:48:17.183 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:17 smithi156 ceph-mon[195101]: failed to encode map e89 with expected crc 2023-10-09T05:48:17.183 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:17 smithi156 ceph-mon[195101]: osd.1 [v2:172.21.15.64:6810/3188144294,v1:172.21.15.64:6811/3188144294] boot 2023-10-09T05:48:17.183 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:17 smithi156 ceph-mon[195101]: osdmap e89: 8 total, 8 up, 8 in 2023-10-09T05:48:17.183 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:17 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:48:17.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:17 smithi064 ceph-mon[212750]: failed to encode map e88 with expected crc 2023-10-09T05:48:17.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:17 smithi064 ceph-mon[212750]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:48:17.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:17 smithi064 ceph-mon[212750]: failed to encode map e89 with expected crc 2023-10-09T05:48:17.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:17 smithi064 ceph-mon[212750]: osd.1 [v2:172.21.15.64:6810/3188144294,v1:172.21.15.64:6811/3188144294] boot 2023-10-09T05:48:17.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:17 smithi064 ceph-mon[212750]: osdmap e89: 8 total, 8 up, 8 in 2023-10-09T05:48:17.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:17 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:48:17.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:17 smithi064 ceph-mon[212750]: pgmap v141: 73 pgs: 3 peering, 2 active+undersized+degraded, 27 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T05:48:17.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:17 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T05:48:17.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:17 smithi064 ceph-mon[212750]: osdmap e90: 8 total, 8 up, 8 in 2023-10-09T05:48:17.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:17 smithi064 ceph-mon[220044]: failed to encode map e88 with expected crc 2023-10-09T05:48:17.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:17 smithi064 ceph-mon[220044]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:48:17.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:17 smithi064 ceph-mon[220044]: failed to encode map e89 with expected crc 2023-10-09T05:48:17.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:17 smithi064 ceph-mon[220044]: osd.1 [v2:172.21.15.64:6810/3188144294,v1:172.21.15.64:6811/3188144294] boot 2023-10-09T05:48:17.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:17 smithi064 ceph-mon[220044]: osdmap e89: 8 total, 8 up, 8 in 2023-10-09T05:48:17.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:17 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:48:17.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:17 smithi064 ceph-mon[220044]: pgmap v141: 73 pgs: 3 peering, 2 active+undersized+degraded, 27 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T05:48:17.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:17 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T05:48:17.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:17 smithi064 ceph-mon[220044]: osdmap e90: 8 total, 8 up, 8 in 2023-10-09T05:48:17.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:17 smithi156 ceph-mon[195101]: pgmap v141: 73 pgs: 3 peering, 2 active+undersized+degraded, 27 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T05:48:17.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:17 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T05:48:17.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:17 smithi156 ceph-mon[195101]: osdmap e90: 8 total, 8 up, 8 in 2023-10-09T05:48:18.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:48:18 smithi064 conmon[198859]: 2023-10-09T05:48:18.156+0000 7fcb872b4700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T05:48:18.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[220044]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[220044]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[220044]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[220044]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[220044]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[220044]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[220044]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[220044]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[220044]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[220044]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[220044]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[220044]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[220044]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[220044]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:18.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:18.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:48:18.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:48:18.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:18.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:48:18.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:18.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:18.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:18.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:48:18.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[212750]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[212750]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[212750]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[212750]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[212750]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[212750]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[212750]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[212750]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[212750]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[212750]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[212750]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[212750]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[212750]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[212750]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:18.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:18.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:48:18.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:48:18.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:18.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:48:18.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:18.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:18.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:18.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:18 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:48:18.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:18 smithi156 ceph-mon[195101]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:18 smithi156 ceph-mon[195101]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:18 smithi156 ceph-mon[195101]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:18 smithi156 ceph-mon[195101]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:18 smithi156 ceph-mon[195101]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:18 smithi156 ceph-mon[195101]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:18 smithi156 ceph-mon[195101]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:18 smithi156 ceph-mon[195101]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:18 smithi156 ceph-mon[195101]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:18 smithi156 ceph-mon[195101]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:18 smithi156 ceph-mon[195101]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:18 smithi156 ceph-mon[195101]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:18 smithi156 ceph-mon[195101]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:18 smithi156 ceph-mon[195101]: failed to encode map e90 with expected crc 2023-10-09T05:48:18.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:18 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:18.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:18 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:18.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:18 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:48:18.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:18 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:48:18.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:18 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:18.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:18 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:48:18.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:18 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:18.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:18 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:18.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:18 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:18.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:18 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:48:19.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:19 smithi064 ceph-mon[220044]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:48:19.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:19 smithi064 ceph-mon[220044]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T05:48:19.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:19 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1412550202' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:19.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:19 smithi064 ceph-mon[220044]: pgmap v143: 73 pgs: 3 peering, 2 active+undersized+degraded, 27 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T05:48:19.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:19 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:19.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:19 smithi064 ceph-mon[212750]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:48:19.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:19 smithi064 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T05:48:19.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:19 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1412550202' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:19.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:19 smithi064 ceph-mon[212750]: pgmap v143: 73 pgs: 3 peering, 2 active+undersized+degraded, 27 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T05:48:19.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:19 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:19.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:19 smithi156 ceph-mon[195101]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:48:19.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:19 smithi156 ceph-mon[195101]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T05:48:19.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:19 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1412550202' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:19 smithi156 ceph-mon[195101]: pgmap v143: 73 pgs: 3 peering, 2 active+undersized+degraded, 27 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T05:48:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:19 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:20.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:20 smithi156 ceph-mon[195101]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-10-09T05:48:20.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:20 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T05:48:20.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:20 smithi064 ceph-mon[212750]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-10-09T05:48:20.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:20 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T05:48:20.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:20 smithi064 ceph-mon[220044]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-10-09T05:48:20.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:20 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T05:48:21.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:21 smithi156 ceph-mon[195101]: pgmap v144: 73 pgs: 3 peering, 70 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:21.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:21 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3636357637' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:21.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:21 smithi064 ceph-mon[220044]: pgmap v144: 73 pgs: 3 peering, 70 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:21.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:21 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3636357637' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:21.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:21 smithi064 ceph-mon[212750]: pgmap v144: 73 pgs: 3 peering, 70 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:21.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:21 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3636357637' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:23.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:23 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2576698588' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:23.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:23 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2576698588' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:23.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:23 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2576698588' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:24.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:24 smithi156 ceph-mon[195101]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:24.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:24 smithi064 ceph-mon[212750]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:24.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:24 smithi064 ceph-mon[220044]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:25.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:48:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:48:25] "GET /metrics HTTP/1.1" 200 33916 "" "Prometheus/2.43.0" 2023-10-09T05:48:26.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:26 smithi156 ceph-mon[195101]: pgmap v146: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:26.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:26 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2656356894' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:26.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:26 smithi064 ceph-mon[212750]: pgmap v146: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:26.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:26 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2656356894' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:26.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:26 smithi064 ceph-mon[220044]: pgmap v146: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:26.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:26 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2656356894' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:28.295 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:28 smithi156 ceph-mon[195101]: pgmap v147: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:28.296 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:28 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3627651239' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:28.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:28 smithi064 ceph-mon[212750]: pgmap v147: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:28.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:28 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3627651239' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:28.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:28 smithi064 ceph-mon[220044]: pgmap v147: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:28.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:28 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3627651239' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:29.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:29 smithi064 ceph-mon[212750]: pgmap v148: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:29.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:29 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:29.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:48:29.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:29 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:29.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:29 smithi064 ceph-mon[220044]: pgmap v148: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:29.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:29 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:29.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:48:29.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:29 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:29 smithi156 ceph-mon[195101]: pgmap v148: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:29 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:48:29.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:29 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:30.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:30 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1559987229' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:30.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:30 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1559987229' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:30.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:30 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1559987229' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:31.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:31 smithi064 ceph-mon[212750]: pgmap v149: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:31.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:31 smithi064 ceph-mon[220044]: pgmap v149: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:31 smithi156 ceph-mon[195101]: pgmap v149: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:33.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:33 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2835183347' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:33.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:33 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2835183347' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:33.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:33 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2835183347' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:34.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:34 smithi156 ceph-mon[195101]: pgmap v150: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:34.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:34 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:48:34.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:34 smithi064 ceph-mon[220044]: pgmap v150: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:34.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:34 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:48:34.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:34 smithi064 ceph-mon[212750]: pgmap v150: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:34.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:34 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:48:35.285 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:35 smithi064 ceph-mon[212750]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:48:35.286 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:35 smithi064 ceph-mon[212750]: Upgrade: osd.2 is safe to restart 2023-10-09T05:48:35.286 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:35 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1586755738' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:35.286 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:35 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:35.286 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:35 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:48:35.286 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:35 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:48:35.287 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:35 smithi064 ceph-mon[220044]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:48:35.287 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:35 smithi064 ceph-mon[220044]: Upgrade: osd.2 is safe to restart 2023-10-09T05:48:35.287 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:35 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1586755738' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:35.287 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:35 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:35.287 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:35 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:48:35.287 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:35 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:48:35.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:35 smithi156 ceph-mon[195101]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T05:48:35.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:35 smithi156 ceph-mon[195101]: Upgrade: osd.2 is safe to restart 2023-10-09T05:48:35.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:35 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1586755738' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:35.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:35 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:35.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:35 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:48:35.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:35 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:48:35.651 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:48:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:48:35] "GET /metrics HTTP/1.1" 200 33912 "" "Prometheus/2.43.0" 2023-10-09T05:48:36.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:36 smithi156 ceph-mon[195101]: pgmap v151: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:36.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:36 smithi156 ceph-mon[195101]: Upgrade: Updating osd.2 2023-10-09T05:48:36.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:36 smithi156 ceph-mon[195101]: Deploying daemon osd.2 on smithi064 2023-10-09T05:48:36.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:36 smithi064 ceph-mon[212750]: pgmap v151: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:36.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:36 smithi064 ceph-mon[212750]: Upgrade: Updating osd.2 2023-10-09T05:48:36.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:36 smithi064 ceph-mon[212750]: Deploying daemon osd.2 on smithi064 2023-10-09T05:48:36.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:36 smithi064 ceph-mon[220044]: pgmap v151: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:36.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:36 smithi064 ceph-mon[220044]: Upgrade: Updating osd.2 2023-10-09T05:48:36.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:36 smithi064 ceph-mon[220044]: Deploying daemon osd.2 on smithi064 2023-10-09T05:48:37.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:37 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4278275024' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:37.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:37 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4278275024' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:37.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:37 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4278275024' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:38.029 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:48:37 smithi064 systemd[1]: Stopping Ceph osd.2 for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:48:38.029 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:48:37 smithi064 conmon[143096]: 2023-10-09T05:48:37.972+0000 7f9c29f5c700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T05:48:38.029 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:48:37 smithi064 conmon[143096]: 2023-10-09T05:48:37.972+0000 7f9c29f5c700 -1 osd.2 90 *** Got signal Terminated *** 2023-10-09T05:48:38.030 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:48:37 smithi064 conmon[143096]: 2023-10-09T05:48:37.972+0000 7f9c29f5c700 -1 osd.2 90 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:48:38.304 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:38 smithi156 ceph-mon[195101]: pgmap v152: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:38.304 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:38 smithi156 ceph-mon[195101]: osd.2 marked itself down and dead 2023-10-09T05:48:38.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:38 smithi064 ceph-mon[220044]: pgmap v152: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:38.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:38 smithi064 ceph-mon[220044]: osd.2 marked itself down and dead 2023-10-09T05:48:38.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:38 smithi064 ceph-mon[212750]: pgmap v152: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:38.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:38 smithi064 ceph-mon[212750]: osd.2 marked itself down and dead 2023-10-09T05:48:39.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:39 smithi156 ceph-mon[195101]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:48:39.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:39 smithi156 ceph-mon[195101]: failed to encode map e91 with expected crc 2023-10-09T05:48:39.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:39 smithi156 ceph-mon[195101]: failed to encode map e91 with expected crc 2023-10-09T05:48:39.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:39 smithi156 ceph-mon[195101]: osdmap e91: 8 total, 7 up, 8 in 2023-10-09T05:48:39.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:39 smithi156 ceph-mon[195101]: failed to encode map e91 with expected crc 2023-10-09T05:48:39.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:39 smithi156 ceph-mon[195101]: failed to encode map e91 with expected crc 2023-10-09T05:48:39.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:39 smithi156 ceph-mon[195101]: failed to encode map e91 with expected crc 2023-10-09T05:48:39.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:39 smithi156 ceph-mon[195101]: failed to encode map e91 with expected crc 2023-10-09T05:48:39.469 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:48:39 smithi064 bash[249653]: 75a274ca9b6e1583e0311c16c3b8103f58db364572d553f2b598b7b269510c21 2023-10-09T05:48:39.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:39 smithi064 ceph-mon[220044]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:48:39.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:39 smithi064 ceph-mon[220044]: failed to encode map e91 with expected crc 2023-10-09T05:48:39.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:39 smithi064 ceph-mon[220044]: failed to encode map e91 with expected crc 2023-10-09T05:48:39.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:39 smithi064 ceph-mon[220044]: osdmap e91: 8 total, 7 up, 8 in 2023-10-09T05:48:39.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:39 smithi064 ceph-mon[220044]: failed to encode map e91 with expected crc 2023-10-09T05:48:39.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:39 smithi064 ceph-mon[220044]: failed to encode map e91 with expected crc 2023-10-09T05:48:39.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:39 smithi064 ceph-mon[220044]: failed to encode map e91 with expected crc 2023-10-09T05:48:39.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:39 smithi064 ceph-mon[220044]: failed to encode map e91 with expected crc 2023-10-09T05:48:39.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:39 smithi064 ceph-mon[212750]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:48:39.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:39 smithi064 ceph-mon[212750]: failed to encode map e91 with expected crc 2023-10-09T05:48:39.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:39 smithi064 ceph-mon[212750]: failed to encode map e91 with expected crc 2023-10-09T05:48:39.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:39 smithi064 ceph-mon[212750]: osdmap e91: 8 total, 7 up, 8 in 2023-10-09T05:48:39.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:39 smithi064 ceph-mon[212750]: failed to encode map e91 with expected crc 2023-10-09T05:48:39.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:39 smithi064 ceph-mon[212750]: failed to encode map e91 with expected crc 2023-10-09T05:48:39.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:39 smithi064 ceph-mon[212750]: failed to encode map e91 with expected crc 2023-10-09T05:48:39.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:39 smithi064 ceph-mon[212750]: failed to encode map e91 with expected crc 2023-10-09T05:48:40.333 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:40 smithi064 ceph-mon[212750]: pgmap v154: 73 pgs: 3 stale+active+clean, 70 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:40.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:40 smithi064 ceph-mon[212750]: failed to encode map e92 with expected crc 2023-10-09T05:48:40.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:40 smithi064 ceph-mon[212750]: failed to encode map e92 with expected crc 2023-10-09T05:48:40.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:40 smithi064 ceph-mon[212750]: osdmap e92: 8 total, 7 up, 8 in 2023-10-09T05:48:40.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:40 smithi064 ceph-mon[212750]: failed to encode map e92 with expected crc 2023-10-09T05:48:40.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:40 smithi064 ceph-mon[212750]: failed to encode map e92 with expected crc 2023-10-09T05:48:40.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:40 smithi064 ceph-mon[212750]: failed to encode map e92 with expected crc 2023-10-09T05:48:40.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:40 smithi064 ceph-mon[212750]: failed to encode map e92 with expected crc 2023-10-09T05:48:40.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:40 smithi064 ceph-mon[212750]: failed to encode map e92 with expected crc 2023-10-09T05:48:40.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:40 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2120903321' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:40.335 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:40 smithi064 ceph-mon[220044]: pgmap v154: 73 pgs: 3 stale+active+clean, 70 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:40.335 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:40 smithi064 ceph-mon[220044]: failed to encode map e92 with expected crc 2023-10-09T05:48:40.335 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:40 smithi064 ceph-mon[220044]: failed to encode map e92 with expected crc 2023-10-09T05:48:40.335 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:40 smithi064 ceph-mon[220044]: osdmap e92: 8 total, 7 up, 8 in 2023-10-09T05:48:40.335 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:40 smithi064 ceph-mon[220044]: failed to encode map e92 with expected crc 2023-10-09T05:48:40.335 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:40 smithi064 ceph-mon[220044]: failed to encode map e92 with expected crc 2023-10-09T05:48:40.335 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:40 smithi064 ceph-mon[220044]: failed to encode map e92 with expected crc 2023-10-09T05:48:40.335 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:40 smithi064 ceph-mon[220044]: failed to encode map e92 with expected crc 2023-10-09T05:48:40.335 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:40 smithi064 ceph-mon[220044]: failed to encode map e92 with expected crc 2023-10-09T05:48:40.336 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:40 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2120903321' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:40.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:40 smithi156 ceph-mon[195101]: pgmap v154: 73 pgs: 3 stale+active+clean, 70 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:48:40.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:40 smithi156 ceph-mon[195101]: failed to encode map e92 with expected crc 2023-10-09T05:48:40.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:40 smithi156 ceph-mon[195101]: failed to encode map e92 with expected crc 2023-10-09T05:48:40.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:40 smithi156 ceph-mon[195101]: osdmap e92: 8 total, 7 up, 8 in 2023-10-09T05:48:40.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:40 smithi156 ceph-mon[195101]: failed to encode map e92 with expected crc 2023-10-09T05:48:40.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:40 smithi156 ceph-mon[195101]: failed to encode map e92 with expected crc 2023-10-09T05:48:40.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:40 smithi156 ceph-mon[195101]: failed to encode map e92 with expected crc 2023-10-09T05:48:40.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:40 smithi156 ceph-mon[195101]: failed to encode map e92 with expected crc 2023-10-09T05:48:40.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:40 smithi156 ceph-mon[195101]: failed to encode map e92 with expected crc 2023-10-09T05:48:40.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:40 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2120903321' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:41.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:41 smithi064 ceph-mon[212750]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:48:41.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:41 smithi064 ceph-mon[220044]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:48:41.220 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:48:40 smithi064 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.2.service: Succeeded. 2023-10-09T05:48:41.220 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:48:40 smithi064 systemd[1]: Stopped Ceph osd.2 for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:48:41.220 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:48:40 smithi064 systemd[1]: Starting Ceph osd.2 for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:48:41.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:41 smithi156 ceph-mon[195101]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:48:42.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:42 smithi156 ceph-mon[195101]: pgmap v156: 73 pgs: 1 active+undersized+degraded, 16 active+undersized, 56 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T05:48:42.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:42 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/887908535' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:42.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:42 smithi064 ceph-mon[212750]: pgmap v156: 73 pgs: 1 active+undersized+degraded, 16 active+undersized, 56 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T05:48:42.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:42 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/887908535' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:42.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:42 smithi064 ceph-mon[220044]: pgmap v156: 73 pgs: 1 active+undersized+degraded, 16 active+undersized, 56 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T05:48:42.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:42 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/887908535' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:42.471 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:48:41 smithi064 bash[250304]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T05:48:42.471 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:48:41 smithi064 bash[250304]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T05:48:42.471 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:48:41 smithi064 bash[250304]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-10-09T05:48:42.471 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:48:41 smithi064 bash[250304]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-10-09T05:48:42.471 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:48:41 smithi064 bash[250304]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-10-09T05:48:42.471 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:48:41 smithi064 bash[250304]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-10-09T05:48:42.471 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:48:41 smithi064 bash[250304]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T05:48:42.471 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:48:41 smithi064 bash[250304]: --> ceph-volume lvm activate successful for osd ID: 2 2023-10-09T05:48:43.719 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:48:43 smithi064 ceph-osd[250820]: -- 172.21.15.64:0/753581908 <== mon.2 v2:172.21.15.64:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x558664c864e0 con 0x558664c58c00 2023-10-09T05:48:43.719 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:48:43 smithi064 bash[250755]: 152098b1dc2d3960d2f8f2afcca75cfd06b76ec28308f0b9ec842a6894973a66 2023-10-09T05:48:43.719 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:48:43 smithi064 systemd[1]: Started Ceph osd.2 for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:48:44.100 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:48:43 smithi064 conmon[250796]: 2023-10-09T05:48:43.760+0000 7f29b42bf640 -1 Falling back to public interface 2023-10-09T05:48:44.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:44 smithi156 ceph-mon[195101]: pgmap v157: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:48:44.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:44.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:44.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:48:44.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:44.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:48:44.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:44 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1941722891' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:44.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:44 smithi064 ceph-mon[212750]: pgmap v157: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:48:44.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:44.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:44.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:48:44.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:44.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:48:44.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:44 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1941722891' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:44.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:44 smithi064 ceph-mon[220044]: pgmap v157: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:48:44.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:44.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:44.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:48:44.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:44.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:48:44.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:44 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1941722891' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:45.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:48:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:48:45] "GET /metrics HTTP/1.1" 200 33907 "" "Prometheus/2.43.0" 2023-10-09T05:48:46.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:46 smithi156 ceph-mon[195101]: pgmap v158: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:48:46.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:46 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:46.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:46 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:46.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:46 smithi064 ceph-mon[212750]: pgmap v158: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:48:46.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:46 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:46.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:46 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:46.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:46 smithi064 ceph-mon[220044]: pgmap v158: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:48:46.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:46 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:46.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:46 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:46.969 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:48:46 smithi064 conmon[250796]: 2023-10-09T05:48:46.550+0000 7f29b42bf640 -1 osd.2 90 log_to_monitors true 2023-10-09T05:48:47.368 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 05:48:47 smithi064 conmon[250796]: 2023-10-09T05:48:47.134+0000 7f29a572b700 -1 osd.2 90 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:48:47.368 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:47 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1216712832' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:47.369 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:47 smithi064 ceph-mon[220044]: pgmap v159: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:48:47.369 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:47 smithi064 ceph-mon[220044]: from='osd.2 [v2:172.21.15.64:6818/3375454963,v1:172.21.15.64:6819/3375454963]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:48:47.369 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:47 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:47.369 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:47 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:47.369 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:47 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:48:47.369 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:47 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1216712832' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:47.369 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:47 smithi064 ceph-mon[212750]: pgmap v159: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:48:47.369 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:47 smithi064 ceph-mon[212750]: from='osd.2 [v2:172.21.15.64:6818/3375454963,v1:172.21.15.64:6819/3375454963]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:48:47.369 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:47 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:47.369 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:47 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:47.370 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:47 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:48:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:47 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1216712832' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:47 smithi156 ceph-mon[195101]: pgmap v159: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:48:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:47 smithi156 ceph-mon[195101]: from='osd.2 [v2:172.21.15.64:6818/3375454963,v1:172.21.15.64:6819/3375454963]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:48:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:47 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:47 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:47 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T05:48:48.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:48 smithi156 ceph-mon[195101]: failed to encode map e93 with expected crc 2023-10-09T05:48:48.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:48 smithi156 ceph-mon[195101]: from='osd.2 [v2:172.21.15.64:6818/3375454963,v1:172.21.15.64:6819/3375454963]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T05:48:48.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:48 smithi156 ceph-mon[195101]: osdmap e93: 8 total, 7 up, 8 in 2023-10-09T05:48:48.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:48 smithi156 ceph-mon[195101]: from='osd.2 [v2:172.21.15.64:6818/3375454963,v1:172.21.15.64:6819/3375454963]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:48:48.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:48 smithi156 ceph-mon[195101]: failed to encode map e93 with expected crc 2023-10-09T05:48:48.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:48 smithi156 ceph-mon[195101]: failed to encode map e93 with expected crc 2023-10-09T05:48:48.443 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:48 smithi064 ceph-mon[212750]: failed to encode map e93 with expected crc 2023-10-09T05:48:48.443 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:48 smithi064 ceph-mon[212750]: from='osd.2 [v2:172.21.15.64:6818/3375454963,v1:172.21.15.64:6819/3375454963]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T05:48:48.443 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:48 smithi064 ceph-mon[212750]: osdmap e93: 8 total, 7 up, 8 in 2023-10-09T05:48:48.443 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:48 smithi064 ceph-mon[212750]: from='osd.2 [v2:172.21.15.64:6818/3375454963,v1:172.21.15.64:6819/3375454963]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:48:48.443 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:48 smithi064 ceph-mon[212750]: failed to encode map e93 with expected crc 2023-10-09T05:48:48.444 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:48 smithi064 ceph-mon[212750]: failed to encode map e93 with expected crc 2023-10-09T05:48:48.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:48 smithi064 ceph-mon[220044]: failed to encode map e93 with expected crc 2023-10-09T05:48:48.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:48 smithi064 ceph-mon[220044]: from='osd.2 [v2:172.21.15.64:6818/3375454963,v1:172.21.15.64:6819/3375454963]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T05:48:48.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:48 smithi064 ceph-mon[220044]: osdmap e93: 8 total, 7 up, 8 in 2023-10-09T05:48:48.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:48 smithi064 ceph-mon[220044]: from='osd.2 [v2:172.21.15.64:6818/3375454963,v1:172.21.15.64:6819/3375454963]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:48:48.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:48 smithi064 ceph-mon[220044]: failed to encode map e93 with expected crc 2023-10-09T05:48:48.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:48 smithi064 ceph-mon[220044]: failed to encode map e93 with expected crc 2023-10-09T05:48:49.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:49 smithi064 ceph-mon[220044]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:48:49.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:49 smithi064 ceph-mon[220044]: failed to encode map e94 with expected crc 2023-10-09T05:48:49.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:49 smithi064 ceph-mon[220044]: failed to encode map e94 with expected crc 2023-10-09T05:48:49.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:49 smithi064 ceph-mon[220044]: osd.2 [v2:172.21.15.64:6818/3375454963,v1:172.21.15.64:6819/3375454963] boot 2023-10-09T05:48:49.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:49 smithi064 ceph-mon[220044]: osdmap e94: 8 total, 8 up, 8 in 2023-10-09T05:48:49.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:49 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:48:49.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:49 smithi064 ceph-mon[220044]: pgmap v162: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:48:49.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:49 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/418091451' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:49.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:49 smithi064 ceph-mon[212750]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:48:49.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:49 smithi064 ceph-mon[212750]: failed to encode map e94 with expected crc 2023-10-09T05:48:49.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:49 smithi064 ceph-mon[212750]: failed to encode map e94 with expected crc 2023-10-09T05:48:49.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:49 smithi064 ceph-mon[212750]: osd.2 [v2:172.21.15.64:6818/3375454963,v1:172.21.15.64:6819/3375454963] boot 2023-10-09T05:48:49.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:49 smithi064 ceph-mon[212750]: osdmap e94: 8 total, 8 up, 8 in 2023-10-09T05:48:49.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:49 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:48:49.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:49 smithi064 ceph-mon[212750]: pgmap v162: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:48:49.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:49 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/418091451' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:49.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:49 smithi156 ceph-mon[195101]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:48:49.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:49 smithi156 ceph-mon[195101]: failed to encode map e94 with expected crc 2023-10-09T05:48:49.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:49 smithi156 ceph-mon[195101]: failed to encode map e94 with expected crc 2023-10-09T05:48:49.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:49 smithi156 ceph-mon[195101]: osd.2 [v2:172.21.15.64:6818/3375454963,v1:172.21.15.64:6819/3375454963] boot 2023-10-09T05:48:49.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:49 smithi156 ceph-mon[195101]: osdmap e94: 8 total, 8 up, 8 in 2023-10-09T05:48:49.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:49 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:48:49.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:49 smithi156 ceph-mon[195101]: pgmap v162: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:48:49.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:49 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/418091451' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:50.401 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:50 smithi064 ceph-mon[212750]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:50 smithi064 ceph-mon[212750]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:50 smithi064 ceph-mon[212750]: osdmap e95: 8 total, 8 up, 8 in 2023-10-09T05:48:50.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:50 smithi064 ceph-mon[212750]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:50 smithi064 ceph-mon[212750]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:50 smithi064 ceph-mon[212750]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:50 smithi064 ceph-mon[212750]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:50 smithi064 ceph-mon[212750]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:50 smithi064 ceph-mon[212750]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:50 smithi064 ceph-mon[212750]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:50 smithi064 ceph-mon[212750]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:50 smithi064 ceph-mon[212750]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.403 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:50 smithi064 ceph-mon[220044]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.403 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:50 smithi064 ceph-mon[220044]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.403 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:50 smithi064 ceph-mon[220044]: osdmap e95: 8 total, 8 up, 8 in 2023-10-09T05:48:50.403 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:50 smithi064 ceph-mon[220044]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.403 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:50 smithi064 ceph-mon[220044]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.403 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:50 smithi064 ceph-mon[220044]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.403 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:50 smithi064 ceph-mon[220044]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.403 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:50 smithi064 ceph-mon[220044]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:50 smithi064 ceph-mon[220044]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:50 smithi064 ceph-mon[220044]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:50 smithi064 ceph-mon[220044]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:50 smithi064 ceph-mon[220044]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:50 smithi156 ceph-mon[195101]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:50 smithi156 ceph-mon[195101]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:50 smithi156 ceph-mon[195101]: osdmap e95: 8 total, 8 up, 8 in 2023-10-09T05:48:50.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:50 smithi156 ceph-mon[195101]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:50 smithi156 ceph-mon[195101]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:50 smithi156 ceph-mon[195101]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:50 smithi156 ceph-mon[195101]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:50 smithi156 ceph-mon[195101]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:50 smithi156 ceph-mon[195101]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:50 smithi156 ceph-mon[195101]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:50 smithi156 ceph-mon[195101]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:50 smithi156 ceph-mon[195101]: failed to encode map e95 with expected crc 2023-10-09T05:48:50.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:48:50 smithi064 conmon[198859]: 2023-10-09T05:48:50.400+0000 7fcb872b4700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-10-09T05:48:51.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:51 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:51.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:51 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:51.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:51 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:48:51.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:51 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:48:51.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:51 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:51.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:51 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:48:51.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:51 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:51.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:51 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:51.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:51 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:51.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:51 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:48:51.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:51 smithi156 ceph-mon[195101]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:48:51.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:51 smithi156 ceph-mon[195101]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-10-09T05:48:51.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:51 smithi156 ceph-mon[195101]: pgmap v164: 73 pgs: 1 active+undersized+degraded, 4 active+undersized, 68 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T05:48:51.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:51 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3727843733' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:51.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:51.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:51.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:48:51.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:48:51.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:51.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:48:51.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:51.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:51.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:51.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:48:51.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[212750]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:48:51.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-10-09T05:48:51.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[212750]: pgmap v164: 73 pgs: 1 active+undersized+degraded, 4 active+undersized, 68 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T05:48:51.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3727843733' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:51.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:51.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:51.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:48:51.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:48:51.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:51.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:48:51.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:51.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:51.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:48:51.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:48:51.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[220044]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:48:51.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[220044]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-10-09T05:48:51.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[220044]: pgmap v164: 73 pgs: 1 active+undersized+degraded, 4 active+undersized, 68 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T05:48:51.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:51 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3727843733' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:52.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:52 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:48:52.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:52 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:48:52.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:52 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:48:53.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:53 smithi156 ceph-mon[195101]: pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:48:53.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:53 smithi064 ceph-mon[212750]: pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:48:53.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:53 smithi064 ceph-mon[220044]: pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:48:54.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:54 smithi156 ceph-mon[195101]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-10-09T05:48:54.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:54 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T05:48:54.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:54 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2879747947' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:54.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:54 smithi064 ceph-mon[220044]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-10-09T05:48:54.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:54 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T05:48:54.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:54 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2879747947' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:54.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:54 smithi064 ceph-mon[212750]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-10-09T05:48:54.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:54 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T05:48:54.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:54 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2879747947' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:55.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:55 smithi156 ceph-mon[195101]: pgmap v166: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:48:55.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:55 smithi064 ceph-mon[212750]: pgmap v166: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:48:55.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:55 smithi064 ceph-mon[220044]: pgmap v166: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:48:55.720 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:48:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:48:55] "GET /metrics HTTP/1.1" 200 33907 "" "Prometheus/2.43.0" 2023-10-09T05:48:56.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:56 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2823453781' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:56.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:56 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2823453781' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:56.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:56 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2823453781' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:58.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:58 smithi156 ceph-mon[195101]: pgmap v167: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:48:58.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:58 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3594580332' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:58.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:58 smithi064 ceph-mon[220044]: pgmap v167: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:48:58.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:58 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3594580332' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:58.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:58 smithi064 ceph-mon[212750]: pgmap v167: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:48:58.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:58 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3594580332' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:48:59.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:59 smithi064 ceph-mon[220044]: pgmap v168: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:48:59.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:59 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:59.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:48:59.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:48:59 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:59.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:59 smithi064 ceph-mon[212750]: pgmap v168: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:48:59.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:59 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:59.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:48:59.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:48:59 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:59.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:59 smithi156 ceph-mon[195101]: pgmap v168: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:48:59.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:59 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:48:59.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:48:59.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:48:59 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:00.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:00 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1919837101' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:00.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:00 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1919837101' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:00.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:00 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1919837101' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:01.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:01 smithi064 ceph-mon[212750]: pgmap v169: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:01.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:01 smithi064 ceph-mon[220044]: pgmap v169: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:01.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:01 smithi156 ceph-mon[195101]: pgmap v169: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:03.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:03 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3337028799' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:03.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:03 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3337028799' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:03.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:03 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3337028799' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:04 smithi156 ceph-mon[195101]: pgmap v170: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:04.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:04 smithi064 ceph-mon[212750]: pgmap v170: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:04.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:04 smithi064 ceph-mon[220044]: pgmap v170: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:05.301 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:05 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1391048327' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:05.301 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:05 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1391048327' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:05.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:05 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1391048327' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:05.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:49:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:49:05] "GET /metrics HTTP/1.1" 200 33911 "" "Prometheus/2.43.0" 2023-10-09T05:49:06.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:06 smithi064 ceph-mon[212750]: pgmap v171: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:06.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:06 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:49:06.208 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:06 smithi064 ceph-mon[220044]: pgmap v171: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:06.208 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:06 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:49:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:06 smithi156 ceph-mon[195101]: pgmap v171: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:06 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:49:07.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:07 smithi064 ceph-mon[212750]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:49:07.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:07 smithi064 ceph-mon[212750]: Upgrade: osd.3 is safe to restart 2023-10-09T05:49:07.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:07 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:07.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:07 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:49:07.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:07 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:49:07.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:07 smithi064 ceph-mon[220044]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:49:07.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:07 smithi064 ceph-mon[220044]: Upgrade: osd.3 is safe to restart 2023-10-09T05:49:07.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:07 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:07.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:07 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:49:07.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:07 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:49:07.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:07 smithi156 ceph-mon[195101]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T05:49:07.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:07 smithi156 ceph-mon[195101]: Upgrade: osd.3 is safe to restart 2023-10-09T05:49:07.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:07 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:07.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:07 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:49:07.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:07 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:49:08.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:08 smithi156 ceph-mon[195101]: pgmap v172: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:08.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:08 smithi156 ceph-mon[195101]: Upgrade: Updating osd.3 2023-10-09T05:49:08.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:08 smithi156 ceph-mon[195101]: Deploying daemon osd.3 on smithi064 2023-10-09T05:49:08.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:08 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3501690807' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:08.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:08 smithi064 ceph-mon[212750]: pgmap v172: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:08.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:08 smithi064 ceph-mon[212750]: Upgrade: Updating osd.3 2023-10-09T05:49:08.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:08 smithi064 ceph-mon[212750]: Deploying daemon osd.3 on smithi064 2023-10-09T05:49:08.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:08 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3501690807' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:08.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:08 smithi064 ceph-mon[220044]: pgmap v172: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:08.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:08 smithi064 ceph-mon[220044]: Upgrade: Updating osd.3 2023-10-09T05:49:08.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:08 smithi064 ceph-mon[220044]: Deploying daemon osd.3 on smithi064 2023-10-09T05:49:08.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:08 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3501690807' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:10.037 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 05:49:09 smithi064 systemd[1]: Stopping Ceph osd.3 for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:49:10.037 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 05:49:09 smithi064 conmon[152299]: 2023-10-09T05:49:09.979+0000 7fe20ab73700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T05:49:10.038 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 05:49:09 smithi064 conmon[152299]: 2023-10-09T05:49:09.979+0000 7fe20ab73700 -1 osd.3 95 *** Got signal Terminated *** 2023-10-09T05:49:10.038 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 05:49:09 smithi064 conmon[152299]: 2023-10-09T05:49:09.979+0000 7fe20ab73700 -1 osd.3 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:49:10.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:10 smithi156 ceph-mon[195101]: pgmap v173: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:10.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:10 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2790564095' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:10.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:10 smithi156 ceph-mon[195101]: osd.3 marked itself down and dead 2023-10-09T05:49:10.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:10 smithi064 ceph-mon[212750]: pgmap v173: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:10.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:10 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2790564095' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:10.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:10 smithi064 ceph-mon[212750]: osd.3 marked itself down and dead 2023-10-09T05:49:10.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:10 smithi064 ceph-mon[220044]: pgmap v173: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:10.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:10 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2790564095' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:10.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:10 smithi064 ceph-mon[220044]: osd.3 marked itself down and dead 2023-10-09T05:49:10.969 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 05:49:10 smithi064 bash[258027]: 711a9677648c6da6d6565ea1a3d3cbaff3340dddcb67e508ba6c1e5c1b73e92e 2023-10-09T05:49:11.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:11 smithi156 ceph-mon[195101]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:49:11.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:11 smithi156 ceph-mon[195101]: failed to encode map e96 with expected crc 2023-10-09T05:49:11.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:11 smithi156 ceph-mon[195101]: osdmap e96: 8 total, 7 up, 8 in 2023-10-09T05:49:11.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:11 smithi156 ceph-mon[195101]: failed to encode map e96 with expected crc 2023-10-09T05:49:11.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:11 smithi156 ceph-mon[195101]: failed to encode map e96 with expected crc 2023-10-09T05:49:11.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:11 smithi064 ceph-mon[212750]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:49:11.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:11 smithi064 ceph-mon[212750]: failed to encode map e96 with expected crc 2023-10-09T05:49:11.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:11 smithi064 ceph-mon[212750]: osdmap e96: 8 total, 7 up, 8 in 2023-10-09T05:49:11.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:11 smithi064 ceph-mon[212750]: failed to encode map e96 with expected crc 2023-10-09T05:49:11.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:11 smithi064 ceph-mon[212750]: failed to encode map e96 with expected crc 2023-10-09T05:49:11.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:11 smithi064 ceph-mon[220044]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:49:11.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:11 smithi064 ceph-mon[220044]: failed to encode map e96 with expected crc 2023-10-09T05:49:11.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:11 smithi064 ceph-mon[220044]: osdmap e96: 8 total, 7 up, 8 in 2023-10-09T05:49:11.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:11 smithi064 ceph-mon[220044]: failed to encode map e96 with expected crc 2023-10-09T05:49:11.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:11 smithi064 ceph-mon[220044]: failed to encode map e96 with expected crc 2023-10-09T05:49:12.305 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:12 smithi064 ceph-mon[212750]: pgmap v175: 73 pgs: 6 peering, 9 stale+active+clean, 58 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:12.305 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:12 smithi064 ceph-mon[212750]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.305 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:12 smithi064 ceph-mon[212750]: osdmap e97: 8 total, 7 up, 8 in 2023-10-09T05:49:12.305 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:12 smithi064 ceph-mon[212750]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.305 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:12 smithi064 ceph-mon[212750]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.305 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:12 smithi064 ceph-mon[212750]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.306 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:12 smithi064 ceph-mon[212750]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.306 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:12 smithi064 ceph-mon[212750]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.306 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:12 smithi064 ceph-mon[212750]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.306 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:12 smithi064 ceph-mon[212750]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.306 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:12 smithi064 ceph-mon[212750]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.306 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:12 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/977897050' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:12.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:12 smithi064 ceph-mon[220044]: pgmap v175: 73 pgs: 6 peering, 9 stale+active+clean, 58 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:12.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:12 smithi064 ceph-mon[220044]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:12 smithi064 ceph-mon[220044]: osdmap e97: 8 total, 7 up, 8 in 2023-10-09T05:49:12.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:12 smithi064 ceph-mon[220044]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.307 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:12 smithi064 ceph-mon[220044]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.307 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:12 smithi064 ceph-mon[220044]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.307 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:12 smithi064 ceph-mon[220044]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.307 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:12 smithi064 ceph-mon[220044]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.307 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:12 smithi064 ceph-mon[220044]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.307 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:12 smithi064 ceph-mon[220044]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.307 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:12 smithi064 ceph-mon[220044]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.307 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:12 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/977897050' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:12.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:12 smithi156 ceph-mon[195101]: pgmap v175: 73 pgs: 6 peering, 9 stale+active+clean, 58 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:12.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:12 smithi156 ceph-mon[195101]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:12 smithi156 ceph-mon[195101]: osdmap e97: 8 total, 7 up, 8 in 2023-10-09T05:49:12.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:12 smithi156 ceph-mon[195101]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:12 smithi156 ceph-mon[195101]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:12 smithi156 ceph-mon[195101]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:12 smithi156 ceph-mon[195101]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:12 smithi156 ceph-mon[195101]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:12 smithi156 ceph-mon[195101]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:12 smithi156 ceph-mon[195101]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:12 smithi156 ceph-mon[195101]: failed to encode map e97 with expected crc 2023-10-09T05:49:12.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:12 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/977897050' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:13.101 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 05:49:12 smithi064 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.3.service: Succeeded. 2023-10-09T05:49:13.101 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 05:49:12 smithi064 systemd[1]: Stopped Ceph osd.3 for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:49:13.101 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 05:49:12 smithi064 systemd[1]: Starting Ceph osd.3 for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:49:13.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:13 smithi156 ceph-mon[195101]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:49:13.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:13 smithi064 ceph-mon[212750]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:49:13.418 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:13 smithi064 ceph-mon[220044]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T05:49:14.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:14 smithi064 ceph-mon[212750]: pgmap v177: 73 pgs: 1 active+undersized+degraded, 6 peering, 6 stale+active+clean, 9 active+undersized, 51 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T05:49:14.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:14.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:14.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:14 smithi064 ceph-mon[220044]: pgmap v177: 73 pgs: 1 active+undersized+degraded, 6 peering, 6 stale+active+clean, 9 active+undersized, 51 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T05:49:14.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:14.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:14.220 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 05:49:13 smithi064 bash[258692]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T05:49:14.220 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 05:49:13 smithi064 bash[258692]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T05:49:14.221 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 05:49:13 smithi064 bash[258692]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-10-09T05:49:14.221 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 05:49:13 smithi064 bash[258692]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-10-09T05:49:14.221 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 05:49:13 smithi064 bash[258692]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-10-09T05:49:14.221 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 05:49:13 smithi064 bash[258692]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-10-09T05:49:14.221 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 05:49:13 smithi064 bash[258692]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T05:49:14.221 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 05:49:13 smithi064 bash[258692]: --> ceph-volume lvm activate successful for osd ID: 3 2023-10-09T05:49:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:14 smithi156 ceph-mon[195101]: pgmap v177: 73 pgs: 1 active+undersized+degraded, 6 peering, 6 stale+active+clean, 9 active+undersized, 51 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T05:49:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:15.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:15 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4060024637' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:15.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:15 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4060024637' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:15.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:15 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4060024637' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:15.470 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:49:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:49:15] "GET /metrics HTTP/1.1" 200 33909 "" "Prometheus/2.43.0" 2023-10-09T05:49:15.470 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 05:49:15 smithi064 bash[259145]: 08e3f2a3f251f126f0ef7fde23e949b597a03b668178087b9fee5ca7d2ff3107 2023-10-09T05:49:15.470 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 05:49:15 smithi064 systemd[1]: Started Ceph osd.3 for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:49:15.470 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 05:49:15 smithi064 ceph-osd[259210]: -- 172.21.15.64:0/4283606225 <== mon.0 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5591dc62c4e0 con 0x5591dc5fec00 2023-10-09T05:49:15.969 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 05:49:15 smithi064 conmon[259186]: 2023-10-09T05:49:15.674+0000 7fe299abb640 -1 Falling back to public interface 2023-10-09T05:49:16.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:16 smithi156 ceph-mon[195101]: pgmap v178: 73 pgs: 3 active+undersized+degraded, 6 peering, 22 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:49:16.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:16 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:16.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:16 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:16.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:16 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:49:16.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:16 smithi064 ceph-mon[212750]: pgmap v178: 73 pgs: 3 active+undersized+degraded, 6 peering, 22 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:49:16.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:16 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:16.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:16 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:16.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:16 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:49:16.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:16 smithi064 ceph-mon[220044]: pgmap v178: 73 pgs: 3 active+undersized+degraded, 6 peering, 22 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:49:16.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:16 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:16.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:16 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:16.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:16 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:49:17.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:17 smithi156 ceph-mon[195101]: pgmap v179: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:49:17.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:17 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/938389742' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:17.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:17 smithi064 ceph-mon[212750]: pgmap v179: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:49:17.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:17 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/938389742' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:17.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:17 smithi064 ceph-mon[220044]: pgmap v179: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:49:17.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:17 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/938389742' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:18.719 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 05:49:18 smithi064 conmon[259186]: 2023-10-09T05:49:18.452+0000 7fe299abb640 -1 osd.3 95 log_to_monitors true 2023-10-09T05:49:19.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:18 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:19.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:18 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:19.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:18 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:19.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:18 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:19.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:18 smithi064 ceph-mon[212750]: from='osd.3 [v2:172.21.15.64:6826/1043927146,v1:172.21.15.64:6827/1043927146]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:49:19.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:18 smithi064 ceph-mon[212750]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:49:19.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:18 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:19.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:18 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:19.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:18 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:19.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:18 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:19.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:18 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:19.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:18 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:19.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:18 smithi064 ceph-mon[220044]: from='osd.3 [v2:172.21.15.64:6826/1043927146,v1:172.21.15.64:6827/1043927146]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:49:19.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:18 smithi064 ceph-mon[220044]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:49:19.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:18 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:19.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:18 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:19.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:18 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:19.408 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:18 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:19.408 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:18 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:19.408 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:18 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:19.408 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:18 smithi156 ceph-mon[195101]: from='osd.3 [v2:172.21.15.64:6826/1043927146,v1:172.21.15.64:6827/1043927146]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:49:19.408 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:18 smithi156 ceph-mon[195101]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:49:19.408 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:18 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:19.408 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:18 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:19.670 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 05:49:19 smithi064 conmon[259186]: 2023-10-09T05:49:19.439+0000 7fe28a726700 -1 osd.3 95 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:49:20.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:19 smithi064 ceph-mon[220044]: pgmap v180: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:49:20.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:19 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3621292713' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:20.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:19 smithi064 ceph-mon[220044]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T05:49:20.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:19 smithi064 ceph-mon[220044]: osdmap e98: 8 total, 7 up, 8 in 2023-10-09T05:49:20.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:19 smithi064 ceph-mon[220044]: from='osd.3 [v2:172.21.15.64:6826/1043927146,v1:172.21.15.64:6827/1043927146]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:49:20.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:19 smithi064 ceph-mon[220044]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:49:20.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:19 smithi064 ceph-mon[212750]: pgmap v180: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:49:20.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:19 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3621292713' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:20.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:19 smithi064 ceph-mon[212750]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T05:49:20.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:19 smithi064 ceph-mon[212750]: osdmap e98: 8 total, 7 up, 8 in 2023-10-09T05:49:20.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:19 smithi064 ceph-mon[212750]: from='osd.3 [v2:172.21.15.64:6826/1043927146,v1:172.21.15.64:6827/1043927146]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:49:20.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:19 smithi064 ceph-mon[212750]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:49:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:19 smithi156 ceph-mon[195101]: pgmap v180: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:49:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:19 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3621292713' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:19 smithi156 ceph-mon[195101]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T05:49:20.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:19 smithi156 ceph-mon[195101]: osdmap e98: 8 total, 7 up, 8 in 2023-10-09T05:49:20.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:19 smithi156 ceph-mon[195101]: from='osd.3 [v2:172.21.15.64:6826/1043927146,v1:172.21.15.64:6827/1043927146]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:49:20.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:19 smithi156 ceph-mon[195101]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:49:21.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:20 smithi064 ceph-mon[212750]: failed to encode map e98 with expected crc 2023-10-09T05:49:21.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:20 smithi064 ceph-mon[212750]: failed to encode map e98 with expected crc 2023-10-09T05:49:21.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:20 smithi064 ceph-mon[212750]: failed to encode map e98 with expected crc 2023-10-09T05:49:21.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:20 smithi064 ceph-mon[212750]: failed to encode map e98 with expected crc 2023-10-09T05:49:21.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:20 smithi064 ceph-mon[212750]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:49:21.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:20 smithi064 ceph-mon[212750]: failed to encode map e99 with expected crc 2023-10-09T05:49:21.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:20 smithi064 ceph-mon[212750]: osd.3 [v2:172.21.15.64:6826/1043927146,v1:172.21.15.64:6827/1043927146] boot 2023-10-09T05:49:21.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:20 smithi064 ceph-mon[212750]: osdmap e99: 8 total, 8 up, 8 in 2023-10-09T05:49:21.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:20 smithi064 ceph-mon[212750]: failed to encode map e99 with expected crc 2023-10-09T05:49:21.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:20 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:49:21.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:20 smithi064 ceph-mon[212750]: failed to encode map e99 with expected crc 2023-10-09T05:49:21.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:20 smithi064 ceph-mon[212750]: failed to encode map e99 with expected crc 2023-10-09T05:49:21.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:20 smithi064 ceph-mon[220044]: failed to encode map e98 with expected crc 2023-10-09T05:49:21.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:20 smithi064 ceph-mon[220044]: failed to encode map e98 with expected crc 2023-10-09T05:49:21.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:20 smithi064 ceph-mon[220044]: failed to encode map e98 with expected crc 2023-10-09T05:49:21.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:20 smithi064 ceph-mon[220044]: failed to encode map e98 with expected crc 2023-10-09T05:49:21.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:20 smithi064 ceph-mon[220044]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:49:21.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:20 smithi064 ceph-mon[220044]: failed to encode map e99 with expected crc 2023-10-09T05:49:21.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:20 smithi064 ceph-mon[220044]: osd.3 [v2:172.21.15.64:6826/1043927146,v1:172.21.15.64:6827/1043927146] boot 2023-10-09T05:49:21.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:20 smithi064 ceph-mon[220044]: osdmap e99: 8 total, 8 up, 8 in 2023-10-09T05:49:21.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:20 smithi064 ceph-mon[220044]: failed to encode map e99 with expected crc 2023-10-09T05:49:21.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:20 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:49:21.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:20 smithi064 ceph-mon[220044]: failed to encode map e99 with expected crc 2023-10-09T05:49:21.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:20 smithi064 ceph-mon[220044]: failed to encode map e99 with expected crc 2023-10-09T05:49:21.332 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 4 2023-10-09T05:49:21.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:20 smithi156 ceph-mon[195101]: failed to encode map e98 with expected crc 2023-10-09T05:49:21.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:20 smithi156 ceph-mon[195101]: failed to encode map e98 with expected crc 2023-10-09T05:49:21.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:20 smithi156 ceph-mon[195101]: failed to encode map e98 with expected crc 2023-10-09T05:49:21.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:20 smithi156 ceph-mon[195101]: failed to encode map e98 with expected crc 2023-10-09T05:49:21.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:20 smithi156 ceph-mon[195101]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:49:21.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:20 smithi156 ceph-mon[195101]: failed to encode map e99 with expected crc 2023-10-09T05:49:21.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:20 smithi156 ceph-mon[195101]: osd.3 [v2:172.21.15.64:6826/1043927146,v1:172.21.15.64:6827/1043927146] boot 2023-10-09T05:49:21.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:20 smithi156 ceph-mon[195101]: osdmap e99: 8 total, 8 up, 8 in 2023-10-09T05:49:21.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:20 smithi156 ceph-mon[195101]: failed to encode map e99 with expected crc 2023-10-09T05:49:21.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:20 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:49:21.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:20 smithi156 ceph-mon[195101]: failed to encode map e99 with expected crc 2023-10-09T05:49:21.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:20 smithi156 ceph-mon[195101]: failed to encode map e99 with expected crc 2023-10-09T05:49:22.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:21 smithi064 ceph-mon[212750]: failed to encode map e98 with expected crc 2023-10-09T05:49:22.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:21 smithi064 ceph-mon[212750]: failed to encode map e99 with expected crc 2023-10-09T05:49:22.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:21 smithi064 ceph-mon[212750]: pgmap v183: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:49:22.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:21 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2992952890' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:22.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:21 smithi064 ceph-mon[212750]: osdmap e100: 8 total, 8 up, 8 in 2023-10-09T05:49:22.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:21 smithi064 ceph-mon[212750]: failed to encode map e100 with expected crc 2023-10-09T05:49:22.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:21 smithi064 ceph-mon[212750]: failed to encode map e100 with expected crc 2023-10-09T05:49:22.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:21 smithi064 ceph-mon[212750]: failed to encode map e100 with expected crc 2023-10-09T05:49:22.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:21 smithi064 ceph-mon[220044]: failed to encode map e98 with expected crc 2023-10-09T05:49:22.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:21 smithi064 ceph-mon[220044]: failed to encode map e99 with expected crc 2023-10-09T05:49:22.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:21 smithi064 ceph-mon[220044]: pgmap v183: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:49:22.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:21 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2992952890' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:22.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:21 smithi064 ceph-mon[220044]: osdmap e100: 8 total, 8 up, 8 in 2023-10-09T05:49:22.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:21 smithi064 ceph-mon[220044]: failed to encode map e100 with expected crc 2023-10-09T05:49:22.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:21 smithi064 ceph-mon[220044]: failed to encode map e100 with expected crc 2023-10-09T05:49:22.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:21 smithi064 ceph-mon[220044]: failed to encode map e100 with expected crc 2023-10-09T05:49:22.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:21 smithi156 ceph-mon[195101]: failed to encode map e98 with expected crc 2023-10-09T05:49:22.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:21 smithi156 ceph-mon[195101]: failed to encode map e99 with expected crc 2023-10-09T05:49:22.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:21 smithi156 ceph-mon[195101]: pgmap v183: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:49:22.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:21 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2992952890' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T05:49:22.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:21 smithi156 ceph-mon[195101]: osdmap e100: 8 total, 8 up, 8 in 2023-10-09T05:49:22.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:21 smithi156 ceph-mon[195101]: failed to encode map e100 with expected crc 2023-10-09T05:49:22.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:21 smithi156 ceph-mon[195101]: failed to encode map e100 with expected crc 2023-10-09T05:49:22.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:21 smithi156 ceph-mon[195101]: failed to encode map e100 with expected crc 2023-10-09T05:49:22.444 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T05:49:22.871 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:22.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:49:22 smithi064 conmon[198859]: 2023-10-09T05:49:22.932+0000 7fcb872b4700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-09T05:49:22.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:22 smithi064 ceph-mon[212750]: failed to encode map e100 with expected crc 2023-10-09T05:49:22.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:22 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:22 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:22 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:22 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:49:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:22 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:49:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:22 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:22 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:49:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:22 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:49:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:22 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:49:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:22 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:49:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:22 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T05:49:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:22 smithi064 ceph-mon[220044]: failed to encode map e100 with expected crc 2023-10-09T05:49:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:22 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:22 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:22 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:22 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:49:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:22 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:49:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:22 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:22 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:49:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:22 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:49:22.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:22 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:49:22.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:22 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:49:22.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:22 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T05:49:23.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:22 smithi156 ceph-mon[195101]: failed to encode map e100 with expected crc 2023-10-09T05:49:23.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:22 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T05:49:23.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:22 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:23.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:22 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:23.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:22 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:49:23.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:22 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:49:23.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:22 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:23.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:22 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:49:23.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:22 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:49:23.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:22 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:49:23.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:22 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:49:23.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:22 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T05:49:24.314 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:24 smithi064 ceph-mon[212750]: pgmap v185: 73 pgs: 5 peering, 3 active+undersized+degraded, 21 active+undersized, 44 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:49:24.314 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:24 smithi064 ceph-mon[212750]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T05:49:24.314 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:24 smithi064 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-09T05:49:24.314 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:24 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:24.315 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:24 smithi064 ceph-mon[220044]: pgmap v185: 73 pgs: 5 peering, 3 active+undersized+degraded, 21 active+undersized, 44 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:49:24.315 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:24 smithi064 ceph-mon[220044]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T05:49:24.315 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:24 smithi064 ceph-mon[220044]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-09T05:49:24.315 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:24 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:24.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:24 smithi156 ceph-mon[195101]: pgmap v185: 73 pgs: 5 peering, 3 active+undersized+degraded, 21 active+undersized, 44 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T05:49:24.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:24 smithi156 ceph-mon[195101]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T05:49:24.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:24 smithi156 ceph-mon[195101]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-09T05:49:24.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:24 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:24.739 INFO:teuthology.orchestra.run.smithi064.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:49:25.520 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T05:49:25.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:49:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:49:25] "GET /metrics HTTP/1.1" 200 33909 "" "Prometheus/2.43.0" 2023-10-09T05:49:25.886 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:26.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:25 smithi156 ceph-mon[195101]: pgmap v186: 73 pgs: 5 peering, 2 active+undersized+degraded, 13 active+undersized, 53 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T05:49:26.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:25 smithi156 ceph-mon[195101]: from='client.44363 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:49:26.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:25 smithi156 ceph-mon[195101]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:49:26.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:25 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:26.126 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:25 smithi064 ceph-mon[212750]: pgmap v186: 73 pgs: 5 peering, 2 active+undersized+degraded, 13 active+undersized, 53 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T05:49:26.126 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:25 smithi064 ceph-mon[212750]: from='client.44363 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:49:26.126 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:25 smithi064 ceph-mon[212750]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:49:26.126 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:25 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:26.126 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:25 smithi064 ceph-mon[220044]: pgmap v186: 73 pgs: 5 peering, 2 active+undersized+degraded, 13 active+undersized, 53 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T05:49:26.127 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:25 smithi064 ceph-mon[220044]: from='client.44363 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:49:26.127 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:25 smithi064 ceph-mon[220044]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T05:49:26.127 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:25 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:27.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:26 smithi156 ceph-mon[195101]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-10-09T05:49:27.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:26 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T05:49:27.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:26 smithi064 ceph-mon[212750]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-10-09T05:49:27.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:26 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T05:49:27.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:26 smithi064 ceph-mon[220044]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-10-09T05:49:27.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:26 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T05:49:27.714 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T05:49:27.714 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (6m) - 10m 29.7M - 0.25.0 c8568f914cd2 386f8a4e2221 2023-10-09T05:49:27.714 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi156 *:3000 running (6m) - 9m 93.7M - 9.4.7 2c41d148cca3 365834e48ffb 2023-10-09T05:49:27.714 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.gmrwsy smithi064 running (9m) - 9m 19.0M - 17.2.6-1303-gb2903cb0 5517396c50d4 e938ff2e972d 2023-10-09T05:49:27.714 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi156.pizbdw smithi156 running (9m) - 9m 18.6M - 17.2.6-1303-gb2903cb0 5517396c50d4 e8906e08c938 2023-10-09T05:49:27.714 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi156 *:8443,9283,8765 running (5m) - 13m 450M - 18.2.0-598-gb4fa663f 3060897fdf97 d0092a415024 2023-10-09T05:49:27.714 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:8443,9283,8765 running (6m) - 15m 516M - 18.2.0-598-gb4fa663f 3060897fdf97 2bc7549ee2c1 2023-10-09T05:49:27.715 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (5m) - 15m 81.7M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 5d3854b1b1eb 2023-10-09T05:49:27.715 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi156 running (3m) - 14m 56.6M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 0cd87bdd0b75 2023-10-09T05:49:27.715 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (4m) - 14m 62.5M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 af10058879c1 2023-10-09T05:49:27.715 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (6m) - 10m 18.6M - 1.5.0 0da6a335fe13 afb28622babd 2023-10-09T05:49:27.715 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi156 *:9100 running (6m) - 10m 19.0M - 1.5.0 0da6a335fe13 1e8c1e28630a 2023-10-09T05:49:27.715 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (2m) - 13m 61.5M 2500M 18.2.0-598-gb4fa663f 3060897fdf97 5779746d3c04 2023-10-09T05:49:27.715 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (77s) - 13m 54.0M 2500M 18.2.0-598-gb4fa663f 3060897fdf97 2e7bf325c804 2023-10-09T05:49:27.715 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (44s) - 12m 48.8M 2500M 18.2.0-598-gb4fa663f 3060897fdf97 152098b1dc2d 2023-10-09T05:49:27.715 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (12s) - 12m 15.0M 2500M 18.2.0-598-gb4fa663f 3060897fdf97 08e3f2a3f251 2023-10-09T05:49:27.715 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi156 running (12m) - 12m 104M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 72313d594357 2023-10-09T05:49:27.715 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi156 running (11m) - 11m 119M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 40d9335e2ab7 2023-10-09T05:49:27.715 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi156 running (11m) - 11m 85.6M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 c8a3e3228b89 2023-10-09T05:49:27.716 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi156 running (11m) - 11m 128M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 df93983742a3 2023-10-09T05:49:27.716 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi156 *:9095 running (5m) - 10m 62.9M - 2.43.0 a07b618ecd1d 712dad4d0e3b 2023-10-09T05:49:28.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:28 smithi156 ceph-mon[195101]: pgmap v187: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:28.450 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:28 smithi064 ceph-mon[212750]: pgmap v187: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:28.450 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:28 smithi064 ceph-mon[220044]: pgmap v187: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:28.467 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T05:49:28.827 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:29.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:29 smithi156 ceph-mon[195101]: from='client.44369 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:49:29.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:29 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:29.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:29.435 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:29 smithi064 ceph-mon[212750]: from='client.44369 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:49:29.435 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:29 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:29.435 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:29.435 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:29 smithi064 ceph-mon[220044]: from='client.44369 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:49:29.435 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:29 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:29.435 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:30.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:30 smithi064 ceph-mon[212750]: pgmap v188: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:30.195 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:30 smithi064 ceph-mon[220044]: pgmap v188: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:30.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:30 smithi156 ceph-mon[195101]: pgmap v188: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:30.630 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T05:49:30.630 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (6m) - 10m 29.7M - 0.25.0 c8568f914cd2 386f8a4e2221 2023-10-09T05:49:30.630 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi156 *:3000 running (6m) - 9m 93.7M - 9.4.7 2c41d148cca3 365834e48ffb 2023-10-09T05:49:30.631 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.gmrwsy smithi064 running (9m) - 9m 19.0M - 17.2.6-1303-gb2903cb0 5517396c50d4 e938ff2e972d 2023-10-09T05:49:30.631 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi156.pizbdw smithi156 running (9m) - 9m 18.6M - 17.2.6-1303-gb2903cb0 5517396c50d4 e8906e08c938 2023-10-09T05:49:30.631 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi156 *:8443,9283,8765 running (5m) - 14m 450M - 18.2.0-598-gb4fa663f 3060897fdf97 d0092a415024 2023-10-09T05:49:30.631 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:8443,9283,8765 running (6m) - 15m 516M - 18.2.0-598-gb4fa663f 3060897fdf97 2bc7549ee2c1 2023-10-09T05:49:30.631 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (5m) - 15m 81.7M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 5d3854b1b1eb 2023-10-09T05:49:30.631 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi156 running (3m) - 14m 56.6M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 0cd87bdd0b75 2023-10-09T05:49:30.631 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (4m) - 14m 62.5M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 af10058879c1 2023-10-09T05:49:30.631 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (6m) - 10m 18.6M - 1.5.0 0da6a335fe13 afb28622babd 2023-10-09T05:49:30.631 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi156 *:9100 running (6m) - 10m 19.0M - 1.5.0 0da6a335fe13 1e8c1e28630a 2023-10-09T05:49:30.632 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (2m) - 13m 61.5M 2500M 18.2.0-598-gb4fa663f 3060897fdf97 5779746d3c04 2023-10-09T05:49:30.632 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (79s) - 13m 54.0M 2500M 18.2.0-598-gb4fa663f 3060897fdf97 2e7bf325c804 2023-10-09T05:49:30.632 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (47s) - 12m 48.8M 2500M 18.2.0-598-gb4fa663f 3060897fdf97 152098b1dc2d 2023-10-09T05:49:30.632 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (15s) - 12m 15.0M 2500M 18.2.0-598-gb4fa663f 3060897fdf97 08e3f2a3f251 2023-10-09T05:49:30.632 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi156 running (12m) - 12m 104M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 72313d594357 2023-10-09T05:49:30.632 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi156 running (11m) - 11m 119M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 40d9335e2ab7 2023-10-09T05:49:30.632 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi156 running (11m) - 11m 85.6M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 c8a3e3228b89 2023-10-09T05:49:30.632 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi156 running (11m) - 11m 128M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 df93983742a3 2023-10-09T05:49:30.632 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi156 *:9095 running (6m) - 10m 62.9M - 2.43.0 a07b618ecd1d 712dad4d0e3b 2023-10-09T05:49:31.335 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph versions' 2023-10-09T05:49:31.697 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:32.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:32 smithi064 ceph-mon[220044]: pgmap v189: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:32.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:32 smithi064 ceph-mon[220044]: from='client.34467 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:49:32.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:32 smithi064 ceph-mon[212750]: pgmap v189: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:32.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:32 smithi064 ceph-mon[212750]: from='client.34467 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:49:32.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:32 smithi156 ceph-mon[195101]: pgmap v189: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:32.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:32 smithi156 ceph-mon[195101]: from='client.34467 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:49:33.576 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:49:33.577 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": { 2023-10-09T05:49:33.577 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 3 2023-10-09T05:49:33.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:49:33.577 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": { 2023-10-09T05:49:33.577 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 2 2023-10-09T05:49:33.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:49:33.577 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": { 2023-10-09T05:49:33.577 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)": 4, 2023-10-09T05:49:33.577 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 4 2023-10-09T05:49:33.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:49:33.578 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": { 2023-10-09T05:49:33.578 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)": 2 2023-10-09T05:49:33.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:49:33.578 INFO:teuthology.orchestra.run.smithi064.stdout: "overall": { 2023-10-09T05:49:33.578 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)": 6, 2023-10-09T05:49:33.578 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 9 2023-10-09T05:49:33.578 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:49:33.578 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:49:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:34 smithi064 ceph-mon[212750]: pgmap v190: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:34 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:34 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1254305936' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:49:34.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:34 smithi064 ceph-mon[220044]: pgmap v190: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:34.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:34 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:34.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:34 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1254305936' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:49:34.324 DEBUG:teuthology.parallel:result is None 2023-10-09T05:49:34.325 INFO:teuthology.run_tasks:Running task print... 2023-10-09T05:49:34.334 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-10-09T05:49:34.334 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-09T05:49:34.383 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T05:49:34.384 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T05:49:34.384 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T05:49:34.385 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:34 smithi156 ceph-mon[195101]: pgmap v190: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:34.385 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:34 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:34.385 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:34 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1254305936' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T05:49:34.747 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:35.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:49:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:49:35] "GET /metrics HTTP/1.1" 200 33906 "" "Prometheus/2.43.0" 2023-10-09T05:49:36.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:36 smithi156 ceph-mon[195101]: pgmap v191: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:36.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:36 smithi064 ceph-mon[212750]: pgmap v191: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:36.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:36 smithi064 ceph-mon[220044]: pgmap v191: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:36.509 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:49:37.208 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":37,"active_gid":34107,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":2746282109},{"type":"v1","addr":"172.21.15.64:6801","nonce":2746282109}]},"active_addr":"172.21.15.64:6801/2746282109","active_change":"2023-10-09T05:43:58.314966+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":34101,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":80,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":2654283623}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":3651316655}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":1958498304}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":3831963688}]}]} 2023-10-09T05:49:37.213 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T05:49:37.213 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T05:49:37.213 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:49:37.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:37 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4175871793' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:49:37.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:37 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4175871793' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:49:37.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:37 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4175871793' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:49:37.579 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:38.175 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:38 smithi064 ceph-mon[212750]: pgmap v192: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:38.175 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:38 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:49:38.175 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:38 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:49:38.176 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:38 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:49:38.176 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:38 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:38.176 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:38 smithi064 ceph-mon[220044]: pgmap v192: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:38.176 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:38 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:49:38.176 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:38 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:49:38.176 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:38 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:49:38.176 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:38 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:38.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:38 smithi156 ceph-mon[195101]: pgmap v192: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:38.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:38 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:49:38.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:38 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:49:38.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:38 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:49:38.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:38 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:39.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:39 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:49:39.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:39 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:49:39.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:39 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:49:39.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:39 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:39.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:39 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:49:39.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:39 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:49:39.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:39 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:49:39.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:39 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:39.339 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:49:39.339 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":100,"fsid":"44c77acc-6665-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:33:40.037198+0000","modified":"2023-10-09T05:49:21.406990+0000","last_up_change":"2023-10-09T05:49:20.404859+0000","last_in_change":"2023-10-09T05:38:14.961182+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:36:42.276784+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:39:17.895056+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T05:40:11.204791+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T05:40:12.067097+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"85d0857a-ca32-43d2-8cec-bbd916db9f98","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":80,"up_from":84,"up_thru":99,"down_at":81,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":3368956113},{"type":"v1","addr":"172.21.15.64:6803","nonce":3368956113}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":3368956113},{"type":"v1","addr":"172.21.15.64:6805","nonce":3368956113}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":3368956113},{"type":"v1","addr":"172.21.15.64:6809","nonce":3368956113}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":3368956113},{"type":"v1","addr":"172.21.15.64:6807","nonce":3368956113}]},"public_addr":"172.21.15.64:6803/3368956113","cluster_addr":"172.21.15.64:6805/3368956113","heartbeat_back_addr":"172.21.15.64:6809/3368956113","heartbeat_front_addr":"172.21.15.64:6807/3368956113","state":["exists","up"]},{"osd":1,"uuid":"05570845-2cd8-4e24-a2f5-f46cf67aa31a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":85,"up_from":89,"up_thru":99,"down_at":86,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":3188144294},{"type":"v1","addr":"172.21.15.64:6811","nonce":3188144294}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":3188144294},{"type":"v1","addr":"172.21.15.64:6813","nonce":3188144294}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":3188144294},{"type":"v1","addr":"172.21.15.64:6817","nonce":3188144294}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":3188144294},{"type":"v1","addr":"172.21.15.64:6815","nonce":3188144294}]},"public_addr":"172.21.15.64:6811/3188144294","cluster_addr":"172.21.15.64:6813/3188144294","heartbeat_back_addr":"172.21.15.64:6817/3188144294","heartbeat_front_addr":"172.21.15.64:6815/3188144294","state":["exists","up"]},{"osd":2,"uuid":"0f306460-9f08-4f15-b573-3587f63be61c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":90,"up_from":94,"up_thru":99,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":3375454963},{"type":"v1","addr":"172.21.15.64:6819","nonce":3375454963}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":3375454963},{"type":"v1","addr":"172.21.15.64:6821","nonce":3375454963}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":3375454963},{"type":"v1","addr":"172.21.15.64:6825","nonce":3375454963}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":3375454963},{"type":"v1","addr":"172.21.15.64:6823","nonce":3375454963}]},"public_addr":"172.21.15.64:6819/3375454963","cluster_addr":"172.21.15.64:6821/3375454963","heartbeat_back_addr":"172.21.15.64:6825/3375454963","heartbeat_front_addr":"172.21.15.64:6823/3375454963","state":["exists","up"]},{"osd":3,"uuid":"4bf38edd-e555-402e-ba4b-c751de06aa6a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":95,"up_from":99,"up_thru":99,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":1043927146},{"type":"v1","addr":"172.21.15.64:6827","nonce":1043927146}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":1043927146},{"type":"v1","addr":"172.21.15.64:6829","nonce":1043927146}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":1043927146},{"type":"v1","addr":"172.21.15.64:6833","nonce":1043927146}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":1043927146},{"type":"v1","addr":"172.21.15.64:6831","nonce":1043927146}]},"public_addr":"172.21.15.64:6827/1043927146","cluster_addr":"172.21.15.64:6829/1043927146","heartbeat_back_addr":"172.21.15.64:6833/1043927146","heartbeat_front_addr":"172.21.15.64:6831/1043927146","state":["exists","up"]},{"osd":4,"uuid":"060e8766-7bf6-42f1-bebe-dae6828f7fb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":99,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6800","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6801","nonce":657648738}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6803","nonce":657648738}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6807","nonce":657648738}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6805","nonce":657648738}]},"public_addr":"172.21.15.156:6801/657648738","cluster_addr":"172.21.15.156:6803/657648738","heartbeat_back_addr":"172.21.15.156:6807/657648738","heartbeat_front_addr":"172.21.15.156:6805/657648738","state":["exists","up"]},{"osd":5,"uuid":"6b5cf38a-6792-4ac7-9694-268601260ba8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":99,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6809","nonce":726279450}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6811","nonce":726279450}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6815","nonce":726279450}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6813","nonce":726279450}]},"public_addr":"172.21.15.156:6809/726279450","cluster_addr":"172.21.15.156:6811/726279450","heartbeat_back_addr":"172.21.15.156:6815/726279450","heartbeat_front_addr":"172.21.15.156:6813/726279450","state":["exists","up"]},{"osd":6,"uuid":"01deea2d-a225-40b8-a54b-61eee57da527","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":96,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6817","nonce":2369965428}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6819","nonce":2369965428}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6823","nonce":2369965428}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6821","nonce":2369965428}]},"public_addr":"172.21.15.156:6817/2369965428","cluster_addr":"172.21.15.156:6819/2369965428","heartbeat_back_addr":"172.21.15.156:6823/2369965428","heartbeat_front_addr":"172.21.15.156:6821/2369965428","state":["exists","up"]},{"osd":7,"uuid":"693e93b0-c235-47fe-9172-85af68e38634","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":99,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6825","nonce":2767546322}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6827","nonce":2767546322}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6831","nonce":2767546322}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6829","nonce":2767546322}]},"public_addr":"172.21.15.156:6825/2767546322","cluster_addr":"172.21.15.156:6827/2767546322","heartbeat_back_addr":"172.21.15.156:6831/2767546322","heartbeat_front_addr":"172.21.15.156:6829/2767546322","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:35:52.928919+0000","dead_epoch":80},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:36:16.479793+0000","dead_epoch":85},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:36:39.503123+0000","dead_epoch":90},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:03.340353+0000","dead_epoch":95},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:24.645080+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:46.312540+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:38:08.346971+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:38:30.073761+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.7","mappings":[{"from":1,"to":2}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:0/390193900":"2023-10-10T05:43:58.314769+0000","172.21.15.64:0/1951412611":"2023-10-10T05:42:37.227748+0000","172.21.15.156:0/698211702":"2023-10-10T05:43:16.303186+0000","172.21.15.64:0/514741379":"2023-10-10T05:34:09.281655+0000","172.21.15.64:0/3707401615":"2023-10-10T05:39:05.525329+0000","172.21.15.64:6801/3793931367":"2023-10-10T05:34:09.281655+0000","172.21.15.64:6800/3793931367":"2023-10-10T05:34:09.281655+0000","172.21.15.64:0/1562509700":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6800/1111090977":"2023-10-10T05:43:58.314769+0000","172.21.15.64:0/2324320735":"2023-10-10T05:43:58.314769+0000","172.21.15.64:0/1766272185":"2023-10-10T05:43:58.314769+0000","172.21.15.64:6801/984813922":"2023-10-10T05:42:37.227748+0000","172.21.15.64:0/3185620259":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/4054596417":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6800/3627852432":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/1571829871":"2023-10-10T05:42:37.227748+0000","172.21.15.64:6800/984813922":"2023-10-10T05:42:37.227748+0000","172.21.15.64:0/3038747921":"2023-10-10T05:42:37.227748+0000","172.21.15.64:0/2171400226":"2023-10-10T05:34:09.281655+0000","172.21.15.64:0/3346006057":"2023-10-10T05:34:09.281655+0000","172.21.15.156:0/1526589339":"2023-10-10T05:43:16.303186+0000","172.21.15.156:0/2936069725":"2023-10-10T05:43:16.303186+0000","172.21.15.64:0/1336186767":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/1900245998":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6801/1111090977":"2023-10-10T05:43:58.314769+0000","172.21.15.64:6801/3627852432":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/584000461":"2023-10-10T05:43:58.314769+0000","172.21.15.64:6801/1833729701":"2023-10-10T05:34:35.744792+0000","172.21.15.64:0/3029574061":"2023-10-10T05:42:37.227748+0000","172.21.15.156:6834/3461673683":"2023-10-10T05:43:16.303186+0000","172.21.15.64:0/479050685":"2023-10-10T05:43:58.314769+0000","172.21.15.156:0/4113996764":"2023-10-10T05:43:16.303186+0000","172.21.15.156:6835/3461673683":"2023-10-10T05:43:16.303186+0000","172.21.15.156:0/1457402607":"2023-10-10T05:43:16.303186+0000","172.21.15.156:0/3281795067":"2023-10-10T05:43:16.303186+0000","172.21.15.64:0/2115451128":"2023-10-10T05:39:05.525329+0000","172.21.15.64:6800/1833729701":"2023-10-10T05:34:35.744792+0000","172.21.15.64:0/2736837524":"2023-10-10T05:43:58.314769+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:49:39.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:39 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:49:39.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:39 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:49:39.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:39 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:49:39.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:39 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:49:40.075 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T05:49:40.076 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:49:40.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:40 smithi156 ceph-mon[195101]: pgmap v193: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:40.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:40 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2055529143' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:49:40.435 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:40.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:40 smithi064 ceph-mon[212750]: pgmap v193: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:40.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:40 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2055529143' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:49:40.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:40 smithi064 ceph-mon[220044]: pgmap v193: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:40.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:40 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2055529143' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:49:42.225 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:49:42.225 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":100,"fsid":"44c77acc-6665-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:33:40.037198+0000","modified":"2023-10-09T05:49:21.406990+0000","last_up_change":"2023-10-09T05:49:20.404859+0000","last_in_change":"2023-10-09T05:38:14.961182+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:36:42.276784+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:39:17.895056+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T05:40:11.204791+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T05:40:12.067097+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"85d0857a-ca32-43d2-8cec-bbd916db9f98","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":80,"up_from":84,"up_thru":99,"down_at":81,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":3368956113},{"type":"v1","addr":"172.21.15.64:6803","nonce":3368956113}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":3368956113},{"type":"v1","addr":"172.21.15.64:6805","nonce":3368956113}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":3368956113},{"type":"v1","addr":"172.21.15.64:6809","nonce":3368956113}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":3368956113},{"type":"v1","addr":"172.21.15.64:6807","nonce":3368956113}]},"public_addr":"172.21.15.64:6803/3368956113","cluster_addr":"172.21.15.64:6805/3368956113","heartbeat_back_addr":"172.21.15.64:6809/3368956113","heartbeat_front_addr":"172.21.15.64:6807/3368956113","state":["exists","up"]},{"osd":1,"uuid":"05570845-2cd8-4e24-a2f5-f46cf67aa31a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":85,"up_from":89,"up_thru":99,"down_at":86,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":3188144294},{"type":"v1","addr":"172.21.15.64:6811","nonce":3188144294}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":3188144294},{"type":"v1","addr":"172.21.15.64:6813","nonce":3188144294}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":3188144294},{"type":"v1","addr":"172.21.15.64:6817","nonce":3188144294}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":3188144294},{"type":"v1","addr":"172.21.15.64:6815","nonce":3188144294}]},"public_addr":"172.21.15.64:6811/3188144294","cluster_addr":"172.21.15.64:6813/3188144294","heartbeat_back_addr":"172.21.15.64:6817/3188144294","heartbeat_front_addr":"172.21.15.64:6815/3188144294","state":["exists","up"]},{"osd":2,"uuid":"0f306460-9f08-4f15-b573-3587f63be61c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":90,"up_from":94,"up_thru":99,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":3375454963},{"type":"v1","addr":"172.21.15.64:6819","nonce":3375454963}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":3375454963},{"type":"v1","addr":"172.21.15.64:6821","nonce":3375454963}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":3375454963},{"type":"v1","addr":"172.21.15.64:6825","nonce":3375454963}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":3375454963},{"type":"v1","addr":"172.21.15.64:6823","nonce":3375454963}]},"public_addr":"172.21.15.64:6819/3375454963","cluster_addr":"172.21.15.64:6821/3375454963","heartbeat_back_addr":"172.21.15.64:6825/3375454963","heartbeat_front_addr":"172.21.15.64:6823/3375454963","state":["exists","up"]},{"osd":3,"uuid":"4bf38edd-e555-402e-ba4b-c751de06aa6a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":95,"up_from":99,"up_thru":99,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":1043927146},{"type":"v1","addr":"172.21.15.64:6827","nonce":1043927146}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":1043927146},{"type":"v1","addr":"172.21.15.64:6829","nonce":1043927146}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":1043927146},{"type":"v1","addr":"172.21.15.64:6833","nonce":1043927146}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":1043927146},{"type":"v1","addr":"172.21.15.64:6831","nonce":1043927146}]},"public_addr":"172.21.15.64:6827/1043927146","cluster_addr":"172.21.15.64:6829/1043927146","heartbeat_back_addr":"172.21.15.64:6833/1043927146","heartbeat_front_addr":"172.21.15.64:6831/1043927146","state":["exists","up"]},{"osd":4,"uuid":"060e8766-7bf6-42f1-bebe-dae6828f7fb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":99,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6800","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6801","nonce":657648738}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6803","nonce":657648738}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6807","nonce":657648738}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":657648738},{"type":"v1","addr":"172.21.15.156:6805","nonce":657648738}]},"public_addr":"172.21.15.156:6801/657648738","cluster_addr":"172.21.15.156:6803/657648738","heartbeat_back_addr":"172.21.15.156:6807/657648738","heartbeat_front_addr":"172.21.15.156:6805/657648738","state":["exists","up"]},{"osd":5,"uuid":"6b5cf38a-6792-4ac7-9694-268601260ba8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":99,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6809","nonce":726279450}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6811","nonce":726279450}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6815","nonce":726279450}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":726279450},{"type":"v1","addr":"172.21.15.156:6813","nonce":726279450}]},"public_addr":"172.21.15.156:6809/726279450","cluster_addr":"172.21.15.156:6811/726279450","heartbeat_back_addr":"172.21.15.156:6815/726279450","heartbeat_front_addr":"172.21.15.156:6813/726279450","state":["exists","up"]},{"osd":6,"uuid":"01deea2d-a225-40b8-a54b-61eee57da527","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":96,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6817","nonce":2369965428}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6819","nonce":2369965428}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6823","nonce":2369965428}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6821","nonce":2369965428}]},"public_addr":"172.21.15.156:6817/2369965428","cluster_addr":"172.21.15.156:6819/2369965428","heartbeat_back_addr":"172.21.15.156:6823/2369965428","heartbeat_front_addr":"172.21.15.156:6821/2369965428","state":["exists","up"]},{"osd":7,"uuid":"693e93b0-c235-47fe-9172-85af68e38634","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":99,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6825","nonce":2767546322}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6827","nonce":2767546322}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6831","nonce":2767546322}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6829","nonce":2767546322}]},"public_addr":"172.21.15.156:6825/2767546322","cluster_addr":"172.21.15.156:6827/2767546322","heartbeat_back_addr":"172.21.15.156:6831/2767546322","heartbeat_front_addr":"172.21.15.156:6829/2767546322","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:35:52.928919+0000","dead_epoch":80},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:36:16.479793+0000","dead_epoch":85},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:36:39.503123+0000","dead_epoch":90},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:03.340353+0000","dead_epoch":95},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:24.645080+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:46.312540+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:38:08.346971+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:38:30.073761+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.7","mappings":[{"from":1,"to":2}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.156:0/3281795067":"2023-10-10T05:43:16.303186+0000","172.21.15.64:0/2115451128":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/1951412611":"2023-10-10T05:42:37.227748+0000","172.21.15.156:0/698211702":"2023-10-10T05:43:16.303186+0000","172.21.15.64:0/514741379":"2023-10-10T05:34:09.281655+0000","172.21.15.64:6800/3793931367":"2023-10-10T05:34:09.281655+0000","172.21.15.64:6801/984813922":"2023-10-10T05:42:37.227748+0000","172.21.15.64:0/3185620259":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/4054596417":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6800/3627852432":"2023-10-10T05:39:05.525329+0000","172.21.15.64:6800/1111090977":"2023-10-10T05:43:58.314769+0000","172.21.15.64:0/1562509700":"2023-10-10T05:34:35.744792+0000","172.21.15.64:0/1571829871":"2023-10-10T05:42:37.227748+0000","172.21.15.64:6800/984813922":"2023-10-10T05:42:37.227748+0000","172.21.15.64:0/3038747921":"2023-10-10T05:42:37.227748+0000","172.21.15.64:0/390193900":"2023-10-10T05:43:58.314769+0000","172.21.15.156:0/2936069725":"2023-10-10T05:43:16.303186+0000","172.21.15.64:0/3346006057":"2023-10-10T05:34:09.281655+0000","172.21.15.156:0/1526589339":"2023-10-10T05:43:16.303186+0000","172.21.15.64:0/1900245998":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6801/1111090977":"2023-10-10T05:43:58.314769+0000","172.21.15.64:0/1336186767":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/2171400226":"2023-10-10T05:34:09.281655+0000","172.21.15.64:6801/3793931367":"2023-10-10T05:34:09.281655+0000","172.21.15.64:0/3707401615":"2023-10-10T05:39:05.525329+0000","172.21.15.64:6801/3627852432":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/584000461":"2023-10-10T05:43:58.314769+0000","172.21.15.64:6800/1833729701":"2023-10-10T05:34:35.744792+0000","172.21.15.64:0/2736837524":"2023-10-10T05:43:58.314769+0000","172.21.15.64:6801/1833729701":"2023-10-10T05:34:35.744792+0000","172.21.15.64:0/3029574061":"2023-10-10T05:42:37.227748+0000","172.21.15.156:6834/3461673683":"2023-10-10T05:43:16.303186+0000","172.21.15.64:0/479050685":"2023-10-10T05:43:58.314769+0000","172.21.15.156:0/4113996764":"2023-10-10T05:43:16.303186+0000","172.21.15.156:6835/3461673683":"2023-10-10T05:43:16.303186+0000","172.21.15.64:0/1766272185":"2023-10-10T05:43:58.314769+0000","172.21.15.64:0/2324320735":"2023-10-10T05:43:58.314769+0000","172.21.15.156:0/1457402607":"2023-10-10T05:43:16.303186+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:49:42.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:42 smithi156 ceph-mon[195101]: pgmap v194: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:42.425 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:42 smithi064 ceph-mon[212750]: pgmap v194: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:42.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:42 smithi064 ceph-mon[220044]: pgmap v194: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:42.984 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T05:49:42.984 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T05:49:42.984 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T05:49:42.985 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T05:49:42.985 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T05:49:42.986 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T05:49:42.986 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T05:49:42.987 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T05:49:43.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:43 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2433828211' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:49:43.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:43 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2433828211' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:49:43.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:43 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2433828211' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:49:44.202 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:44.304 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:44.411 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:44.412 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:44.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:44 smithi064 ceph-mon[212750]: pgmap v195: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:44.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:44.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:44 smithi064 ceph-mon[220044]: pgmap v195: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:44.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:44.504 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:44.566 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:44.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:44 smithi156 ceph-mon[195101]: pgmap v195: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:44.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:44.867 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:45.094 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:45.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:45 smithi064 ceph-mon[212750]: pgmap v196: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:45.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:49:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:49:45] "GET /metrics HTTP/1.1" 200 33904 "" "Prometheus/2.43.0" 2023-10-09T05:49:45.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:45 smithi064 ceph-mon[220044]: pgmap v196: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:45.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:45 smithi156 ceph-mon[195101]: pgmap v196: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:48.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:48 smithi064 ceph-mon[212750]: pgmap v197: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:48.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:48 smithi064 ceph-mon[220044]: pgmap v197: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:48.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:48 smithi156 ceph-mon[195101]: pgmap v197: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:50.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:50 smithi064 ceph-mon[212750]: pgmap v198: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:50.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:50 smithi064 ceph-mon[220044]: pgmap v198: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:50.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:50 smithi156 ceph-mon[195101]: pgmap v198: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:51.688 INFO:teuthology.orchestra.run.smithi064.stdout:128849019032 2023-10-09T05:49:51.689 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T05:49:52.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:52 smithi156 ceph-mon[195101]: pgmap v199: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:52.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:52 smithi064 ceph-mon[212750]: pgmap v199: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:52.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:52 smithi064 ceph-mon[220044]: pgmap v199: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:53.258 INFO:teuthology.orchestra.run.smithi064.stdout:231928234123 2023-10-09T05:49:53.258 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T05:49:54.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:54 smithi156 ceph-mon[195101]: pgmap v200: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:54.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:54 smithi064 ceph-mon[212750]: pgmap v200: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:54.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:54 smithi064 ceph-mon[220044]: pgmap v200: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:54.629 INFO:teuthology.orchestra.run.smithi064.stdout:425201762312 2023-10-09T05:49:54.629 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T05:49:55.253 INFO:teuthology.orchestra.run.smithi064.stdout:382252089365 2023-10-09T05:49:55.253 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T05:49:55.345 INFO:teuthology.orchestra.run.smithi064.stdout:163208757395 2023-10-09T05:49:55.345 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T05:49:55.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:49:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:49:55] "GET /metrics HTTP/1.1" 200 33904 "" "Prometheus/2.43.0" 2023-10-09T05:49:56.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:56 smithi064 ceph-mon[212750]: pgmap v201: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:56.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:56 smithi064 ceph-mon[220044]: pgmap v201: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:56.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:56 smithi156 ceph-mon[195101]: pgmap v201: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:56.466 INFO:teuthology.orchestra.run.smithi064.stdout:197568495759 2023-10-09T05:49:56.467 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T05:49:56.538 INFO:teuthology.orchestra.run.smithi064.stdout:403726925839 2023-10-09T05:49:56.539 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T05:49:57.285 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:57.316 INFO:teuthology.orchestra.run.smithi064.stdout:360777252901 2023-10-09T05:49:57.317 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T05:49:57.337 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:57.427 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:57.482 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:57.482 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:57.483 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:57.816 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:49:58.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:58 smithi156 ceph-mon[195101]: pgmap v202: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:58.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:58 smithi064 ceph-mon[212750]: pgmap v202: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:58.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:58 smithi064 ceph-mon[220044]: pgmap v202: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:49:59.101 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:49:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:59.101 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:49:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:49:59.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:49:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:00.366 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:00 smithi064 ceph-mon[212750]: pgmap v203: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:00.366 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:00 smithi064 ceph-mon[212750]: overall HEALTH_OK 2023-10-09T05:50:00.366 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:00 smithi064 ceph-mon[220044]: pgmap v203: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:00.367 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:00 smithi064 ceph-mon[220044]: overall HEALTH_OK 2023-10-09T05:50:00.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:00 smithi156 ceph-mon[195101]: pgmap v203: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:00.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:00 smithi156 ceph-mon[195101]: overall HEALTH_OK 2023-10-09T05:50:01.788 INFO:teuthology.orchestra.run.smithi064.stdout:425201762314 2023-10-09T05:50:02.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:02 smithi064 ceph-mon[212750]: pgmap v204: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:02.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:02 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/384281103' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:50:02.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:02 smithi064 ceph-mon[220044]: pgmap v204: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:02.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:02 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/384281103' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:50:02.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:02 smithi156 ceph-mon[195101]: pgmap v204: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:02.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:02 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/384281103' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:50:03.449 INFO:tasks.cephadm.ceph_manager.ceph:need seq 425201762312 got 425201762314 for osd.3 2023-10-09T05:50:03.449 DEBUG:teuthology.parallel:result is None 2023-10-09T05:50:03.454 INFO:teuthology.orchestra.run.smithi064.stdout:403726925841 2023-10-09T05:50:03.669 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:04.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:04 smithi064 ceph-mon[212750]: pgmap v205: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:04.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:04 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/71470644' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:50:04.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:04 smithi064 ceph-mon[220044]: pgmap v205: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:04.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:04 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/71470644' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:50:04.367 INFO:teuthology.orchestra.run.smithi064.stdout:163208757397 2023-10-09T05:50:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:04 smithi156 ceph-mon[195101]: pgmap v205: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:04 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/71470644' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:50:05.230 INFO:tasks.cephadm.ceph_manager.ceph:need seq 403726925839 got 403726925841 for osd.2 2023-10-09T05:50:05.230 DEBUG:teuthology.parallel:result is None 2023-10-09T05:50:05.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:05 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2649551513' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:50:05.380 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:05 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2649551513' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:50:05.381 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:05 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2649551513' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:50:05.381 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:50:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:50:05] "GET /metrics HTTP/1.1" 200 33905 "" "Prometheus/2.43.0" 2023-10-09T05:50:05.416 INFO:teuthology.orchestra.run.smithi064.stdout:382252089367 2023-10-09T05:50:05.636 INFO:teuthology.orchestra.run.smithi064.stdout:231928234126 2023-10-09T05:50:06.046 INFO:teuthology.orchestra.run.smithi064.stdout:197568495762 2023-10-09T05:50:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:06 smithi156 ceph-mon[195101]: pgmap v206: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:06 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1654490965' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:50:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:06 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/865731835' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:50:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:06 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2541567781' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:50:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:06 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2364116123' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:50:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:06 smithi064 ceph-mon[212750]: pgmap v206: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:06 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1654490965' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:50:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:06 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/865731835' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:50:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:06 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2541567781' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:50:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:06 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2364116123' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:50:06.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:06 smithi064 ceph-mon[220044]: pgmap v206: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:06.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:06 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1654490965' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:50:06.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:06 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/865731835' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:50:06.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:06 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2541567781' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:50:06.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:06 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2364116123' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:50:06.561 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757395 got 163208757397 for osd.5 2023-10-09T05:50:06.561 DEBUG:teuthology.parallel:result is None 2023-10-09T05:50:07.173 INFO:teuthology.orchestra.run.smithi064.stdout:128849019035 2023-10-09T05:50:07.472 INFO:tasks.cephadm.ceph_manager.ceph:need seq 382252089365 got 382252089367 for osd.1 2023-10-09T05:50:07.473 DEBUG:teuthology.parallel:result is None 2023-10-09T05:50:07.502 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495759 got 197568495762 for osd.6 2023-10-09T05:50:07.502 DEBUG:teuthology.parallel:result is None 2023-10-09T05:50:08.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:08 smithi064 ceph-mon[212750]: pgmap v207: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:08.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:08 smithi064 ceph-mon[220044]: pgmap v207: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:08.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:08 smithi156 ceph-mon[195101]: pgmap v207: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:08.464 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234123 got 231928234126 for osd.7 2023-10-09T05:50:08.464 DEBUG:teuthology.parallel:result is None 2023-10-09T05:50:08.478 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019032 got 128849019035 for osd.4 2023-10-09T05:50:08.479 DEBUG:teuthology.parallel:result is None 2023-10-09T05:50:08.829 INFO:teuthology.orchestra.run.smithi064.stdout:360777252903 2023-10-09T05:50:09.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:09 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3810312013' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:50:09.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:09 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3810312013' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:50:09.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:09 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3810312013' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:50:09.612 INFO:tasks.cephadm.ceph_manager.ceph:need seq 360777252901 got 360777252903 for osd.0 2023-10-09T05:50:09.613 DEBUG:teuthology.parallel:result is None 2023-10-09T05:50:09.613 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T05:50:09.613 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:50:09.988 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:10.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:10 smithi156 ceph-mon[195101]: pgmap v208: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:10.397 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:10 smithi064 ceph-mon[212750]: pgmap v208: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:10.397 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:10 smithi064 ceph-mon[220044]: pgmap v208: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:11.819 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:50:11.820 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T05:50:12.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:12 smithi064 ceph-mon[212750]: pgmap v209: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:12.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:12 smithi064 ceph-mon[220044]: pgmap v209: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:12.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:12 smithi156 ceph-mon[195101]: pgmap v209: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:12.580 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":209,"stamp":"2023-10-09T05:50:10.431487+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1333612,"kb_used_data":9836,"kb_used_omap":126,"kb_used_meta":1323585,"kb_avail":748627604,"statfs":{"total":767960285184,"available":766594666496,"internally_reserved":0,"allocated":10072064,"data_stored":6185824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1355351675},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002806"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.413173+0000","last_change":"2023-10-09T05:48:17.172491+0000","last_active":"2023-10-09T05:49:20.413173+0000","last_peered":"2023-10-09T05:49:20.413173+0000","last_clean":"2023-10-09T05:49:20.413173+0000","last_became_active":"2023-10-09T05:48:17.172154+0000","last_became_peered":"2023-10-09T05:48:17.172154+0000","last_unstale":"2023-10-09T05:49:20.413173+0000","last_undegraded":"2023-10-09T05:49:20.413173+0000","last_fullsized":"2023-10-09T05:49:20.413173+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:24:40.551774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":57,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.416576+0000","last_change":"2023-10-09T05:48:49.139798+0000","last_active":"2023-10-09T05:49:20.416576+0000","last_peered":"2023-10-09T05:49:20.416576+0000","last_clean":"2023-10-09T05:49:20.416576+0000","last_became_active":"2023-10-09T05:48:49.139442+0000","last_became_peered":"2023-10-09T05:48:49.139442+0000","last_unstale":"2023-10-09T05:49:20.416576+0000","last_undegraded":"2023-10-09T05:49:20.416576+0000","last_fullsized":"2023-10-09T05:49:20.416576+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:25.034620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":84,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.421384+0000","last_change":"2023-10-09T05:49:21.421384+0000","last_active":"2023-10-09T05:49:21.421384+0000","last_peered":"2023-10-09T05:49:21.421384+0000","last_clean":"2023-10-09T05:49:21.421384+0000","last_became_active":"2023-10-09T05:49:21.420886+0000","last_became_peered":"2023-10-09T05:49:21.420886+0000","last_unstale":"2023-10-09T05:49:21.421384+0000","last_undegraded":"2023-10-09T05:49:21.421384+0000","last_fullsized":"2023-10-09T05:49:21.421384+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:33:24.415387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":69,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.430192+0000","last_change":"2023-10-09T05:49:21.430192+0000","last_active":"2023-10-09T05:49:21.430192+0000","last_peered":"2023-10-09T05:49:21.430192+0000","last_clean":"2023-10-09T05:49:21.430192+0000","last_became_active":"2023-10-09T05:49:21.429857+0000","last_became_peered":"2023-10-09T05:49:21.429857+0000","last_unstale":"2023-10-09T05:49:21.430192+0000","last_undegraded":"2023-10-09T05:49:21.430192+0000","last_fullsized":"2023-10-09T05:49:21.430192+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:45.887757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":76,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426615+0000","last_change":"2023-10-09T05:49:21.426615+0000","last_active":"2023-10-09T05:49:21.426615+0000","last_peered":"2023-10-09T05:49:21.426615+0000","last_clean":"2023-10-09T05:49:21.426615+0000","last_became_active":"2023-10-09T05:49:21.426245+0000","last_became_peered":"2023-10-09T05:49:21.426245+0000","last_unstale":"2023-10-09T05:49:21.426615+0000","last_undegraded":"2023-10-09T05:49:21.426615+0000","last_fullsized":"2023-10-09T05:49:21.426615+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:47.642455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":45,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.425017+0000","last_change":"2023-10-09T05:40:12.633195+0000","last_active":"2023-10-09T05:49:20.425017+0000","last_peered":"2023-10-09T05:49:20.425017+0000","last_clean":"2023-10-09T05:49:20.425017+0000","last_became_active":"2023-10-09T05:40:12.632942+0000","last_became_peered":"2023-10-09T05:40:12.632942+0000","last_unstale":"2023-10-09T05:49:20.425017+0000","last_undegraded":"2023-10-09T05:49:20.425017+0000","last_fullsized":"2023-10-09T05:49:20.425017+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:04.470685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":76,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426304+0000","last_change":"2023-10-09T05:49:21.426304+0000","last_active":"2023-10-09T05:49:21.426304+0000","last_peered":"2023-10-09T05:49:21.426304+0000","last_clean":"2023-10-09T05:49:21.426304+0000","last_became_active":"2023-10-09T05:49:21.425949+0000","last_became_peered":"2023-10-09T05:49:21.425949+0000","last_unstale":"2023-10-09T05:49:21.426304+0000","last_undegraded":"2023-10-09T05:49:21.426304+0000","last_fullsized":"2023-10-09T05:49:21.426304+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:11.209520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.424610+0000","last_change":"2023-10-09T05:48:17.173906+0000","last_active":"2023-10-09T05:49:20.424610+0000","last_peered":"2023-10-09T05:49:20.424610+0000","last_clean":"2023-10-09T05:49:20.424610+0000","last_became_active":"2023-10-09T05:48:17.173564+0000","last_became_peered":"2023-10-09T05:48:17.173564+0000","last_unstale":"2023-10-09T05:49:20.424610+0000","last_undegraded":"2023-10-09T05:49:20.424610+0000","last_fullsized":"2023-10-09T05:49:20.424610+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:37.582247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":81,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.430604+0000","last_change":"2023-10-09T05:49:21.430604+0000","last_active":"2023-10-09T05:49:21.430604+0000","last_peered":"2023-10-09T05:49:21.430604+0000","last_clean":"2023-10-09T05:49:21.430604+0000","last_became_active":"2023-10-09T05:49:21.430220+0000","last_became_peered":"2023-10-09T05:49:21.430220+0000","last_unstale":"2023-10-09T05:49:21.430604+0000","last_undegraded":"2023-10-09T05:49:21.430604+0000","last_fullsized":"2023-10-09T05:49:21.430604+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:06:01.882964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":42,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.417852+0000","last_change":"2023-10-09T05:47:03.233346+0000","last_active":"2023-10-09T05:49:21.417852+0000","last_peered":"2023-10-09T05:49:21.417852+0000","last_clean":"2023-10-09T05:49:21.417852+0000","last_became_active":"2023-10-09T05:47:03.231300+0000","last_became_peered":"2023-10-09T05:47:03.231300+0000","last_unstale":"2023-10-09T05:49:21.417852+0000","last_undegraded":"2023-10-09T05:49:21.417852+0000","last_fullsized":"2023-10-09T05:49:21.417852+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:22.050814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":42,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.414600+0000","last_change":"2023-10-09T05:40:12.676587+0000","last_active":"2023-10-09T05:49:20.414600+0000","last_peered":"2023-10-09T05:49:20.414600+0000","last_clean":"2023-10-09T05:49:20.414600+0000","last_became_active":"2023-10-09T05:40:12.676330+0000","last_became_peered":"2023-10-09T05:40:12.676330+0000","last_unstale":"2023-10-09T05:49:20.414600+0000","last_undegraded":"2023-10-09T05:49:20.414600+0000","last_fullsized":"2023-10-09T05:49:20.414600+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:46.659556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":57,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.416515+0000","last_change":"2023-10-09T05:48:17.174996+0000","last_active":"2023-10-09T05:49:20.416515+0000","last_peered":"2023-10-09T05:49:20.416515+0000","last_clean":"2023-10-09T05:49:20.416515+0000","last_became_active":"2023-10-09T05:48:17.174668+0000","last_became_peered":"2023-10-09T05:48:17.174668+0000","last_unstale":"2023-10-09T05:49:20.416515+0000","last_undegraded":"2023-10-09T05:49:20.416515+0000","last_fullsized":"2023-10-09T05:49:20.416515+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:49.712686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.421123+0000","last_change":"2023-10-09T05:48:49.137565+0000","last_active":"2023-10-09T05:49:20.421123+0000","last_peered":"2023-10-09T05:49:20.421123+0000","last_clean":"2023-10-09T05:49:20.421123+0000","last_became_active":"2023-10-09T05:48:49.137099+0000","last_became_peered":"2023-10-09T05:48:49.137099+0000","last_unstale":"2023-10-09T05:49:20.421123+0000","last_undegraded":"2023-10-09T05:49:20.421123+0000","last_fullsized":"2023-10-09T05:49:20.421123+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:28.103821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":58,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.423248+0000","last_change":"2023-10-09T05:49:21.423248+0000","last_active":"2023-10-09T05:49:21.423248+0000","last_peered":"2023-10-09T05:49:21.423248+0000","last_clean":"2023-10-09T05:49:21.423248+0000","last_became_active":"2023-10-09T05:49:21.422985+0000","last_became_peered":"2023-10-09T05:49:21.422985+0000","last_unstale":"2023-10-09T05:49:21.423248+0000","last_undegraded":"2023-10-09T05:49:21.423248+0000","last_fullsized":"2023-10-09T05:49:21.423248+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:02.431791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.409445+0000","last_change":"2023-10-09T05:48:17.174101+0000","last_active":"2023-10-09T05:49:20.409445+0000","last_peered":"2023-10-09T05:49:20.409445+0000","last_clean":"2023-10-09T05:49:20.409445+0000","last_became_active":"2023-10-09T05:48:17.173752+0000","last_became_peered":"2023-10-09T05:48:17.173752+0000","last_unstale":"2023-10-09T05:49:20.409445+0000","last_undegraded":"2023-10-09T05:49:20.409445+0000","last_fullsized":"2023-10-09T05:49:20.409445+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:45:16.222083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":61,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.423414+0000","last_change":"2023-10-09T05:49:21.423414+0000","last_active":"2023-10-09T05:49:21.423414+0000","last_peered":"2023-10-09T05:49:21.423414+0000","last_clean":"2023-10-09T05:49:21.423414+0000","last_became_active":"2023-10-09T05:49:21.423085+0000","last_became_peered":"2023-10-09T05:49:21.423085+0000","last_unstale":"2023-10-09T05:49:21.423414+0000","last_undegraded":"2023-10-09T05:49:21.423414+0000","last_fullsized":"2023-10-09T05:49:21.423414+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:51.777782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.414717+0000","last_change":"2023-10-09T05:47:03.233460+0000","last_active":"2023-10-09T05:49:20.414717+0000","last_peered":"2023-10-09T05:49:20.414717+0000","last_clean":"2023-10-09T05:49:20.414717+0000","last_became_active":"2023-10-09T05:47:03.231685+0000","last_became_peered":"2023-10-09T05:47:03.231685+0000","last_unstale":"2023-10-09T05:49:20.414717+0000","last_undegraded":"2023-10-09T05:49:20.414717+0000","last_fullsized":"2023-10-09T05:49:20.414717+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:10.790717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.409203+0000","last_change":"2023-10-09T05:48:49.139759+0000","last_active":"2023-10-09T05:49:20.409203+0000","last_peered":"2023-10-09T05:49:20.409203+0000","last_clean":"2023-10-09T05:49:20.409203+0000","last_became_active":"2023-10-09T05:48:49.139346+0000","last_became_peered":"2023-10-09T05:48:49.139346+0000","last_unstale":"2023-10-09T05:49:20.409203+0000","last_undegraded":"2023-10-09T05:49:20.409203+0000","last_fullsized":"2023-10-09T05:49:20.409203+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:51.980711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.420851+0000","last_change":"2023-10-09T05:47:03.230117+0000","last_active":"2023-10-09T05:49:20.420851+0000","last_peered":"2023-10-09T05:49:20.420851+0000","last_clean":"2023-10-09T05:49:20.420851+0000","last_became_active":"2023-10-09T05:47:03.229660+0000","last_became_peered":"2023-10-09T05:47:03.229660+0000","last_unstale":"2023-10-09T05:49:20.420851+0000","last_undegraded":"2023-10-09T05:49:20.420851+0000","last_fullsized":"2023-10-09T05:49:20.420851+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:07:21.669282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.416527+0000","last_change":"2023-10-09T05:48:49.134851+0000","last_active":"2023-10-09T05:49:20.416527+0000","last_peered":"2023-10-09T05:49:20.416527+0000","last_clean":"2023-10-09T05:49:20.416527+0000","last_became_active":"2023-10-09T05:48:49.134532+0000","last_became_peered":"2023-10-09T05:48:49.134532+0000","last_unstale":"2023-10-09T05:49:20.416527+0000","last_undegraded":"2023-10-09T05:49:20.416527+0000","last_fullsized":"2023-10-09T05:49:20.416527+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:23.395907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.422120+0000","last_change":"2023-10-09T05:49:21.422120+0000","last_active":"2023-10-09T05:49:21.422120+0000","last_peered":"2023-10-09T05:49:21.422120+0000","last_clean":"2023-10-09T05:49:21.422120+0000","last_became_active":"2023-10-09T05:49:21.421835+0000","last_became_peered":"2023-10-09T05:49:21.421835+0000","last_unstale":"2023-10-09T05:49:21.422120+0000","last_undegraded":"2023-10-09T05:49:21.422120+0000","last_fullsized":"2023-10-09T05:49:21.422120+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:56.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":81,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.418771+0000","last_change":"2023-10-09T05:48:17.175668+0000","last_active":"2023-10-09T05:49:20.418771+0000","last_peered":"2023-10-09T05:49:20.418771+0000","last_clean":"2023-10-09T05:49:20.418771+0000","last_became_active":"2023-10-09T05:48:17.175344+0000","last_became_peered":"2023-10-09T05:48:17.175344+0000","last_unstale":"2023-10-09T05:49:20.418771+0000","last_undegraded":"2023-10-09T05:49:20.418771+0000","last_fullsized":"2023-10-09T05:49:20.418771+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:43.949807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":42,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.419433+0000","last_change":"2023-10-09T05:40:14.363125+0000","last_active":"2023-10-09T05:49:20.419433+0000","last_peered":"2023-10-09T05:49:20.419433+0000","last_clean":"2023-10-09T05:49:20.419433+0000","last_became_active":"2023-10-09T05:40:14.362723+0000","last_became_peered":"2023-10-09T05:40:14.362723+0000","last_unstale":"2023-10-09T05:49:20.419433+0000","last_undegraded":"2023-10-09T05:49:20.419433+0000","last_fullsized":"2023-10-09T05:49:20.419433+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:30.741318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426544+0000","last_change":"2023-10-09T05:49:21.426544+0000","last_active":"2023-10-09T05:49:21.426544+0000","last_peered":"2023-10-09T05:49:21.426544+0000","last_clean":"2023-10-09T05:49:21.426544+0000","last_became_active":"2023-10-09T05:49:21.426253+0000","last_became_peered":"2023-10-09T05:49:21.426253+0000","last_unstale":"2023-10-09T05:49:21.426544+0000","last_undegraded":"2023-10-09T05:49:21.426544+0000","last_fullsized":"2023-10-09T05:49:21.426544+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:32:40.322649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":441,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.418674+0000","last_change":"2023-10-09T05:47:03.234961+0000","last_active":"2023-10-09T05:49:20.418674+0000","last_peered":"2023-10-09T05:49:20.418674+0000","last_clean":"2023-10-09T05:49:20.418674+0000","last_became_active":"2023-10-09T05:47:03.234618+0000","last_became_peered":"2023-10-09T05:47:03.234618+0000","last_unstale":"2023-10-09T05:49:20.418674+0000","last_undegraded":"2023-10-09T05:49:20.418674+0000","last_fullsized":"2023-10-09T05:49:20.418674+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:01.472111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.422069+0000","last_change":"2023-10-09T05:48:49.141207+0000","last_active":"2023-10-09T05:49:20.422069+0000","last_peered":"2023-10-09T05:49:20.422069+0000","last_clean":"2023-10-09T05:49:20.422069+0000","last_became_active":"2023-10-09T05:48:49.140798+0000","last_became_peered":"2023-10-09T05:48:49.140798+0000","last_unstale":"2023-10-09T05:49:20.422069+0000","last_undegraded":"2023-10-09T05:49:20.422069+0000","last_fullsized":"2023-10-09T05:49:20.422069+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:44:13.125030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.414450+0000","last_change":"2023-10-09T05:48:49.139757+0000","last_active":"2023-10-09T05:49:20.414450+0000","last_peered":"2023-10-09T05:49:20.414450+0000","last_clean":"2023-10-09T05:49:20.414450+0000","last_became_active":"2023-10-09T05:48:49.139311+0000","last_became_peered":"2023-10-09T05:48:49.139311+0000","last_unstale":"2023-10-09T05:49:20.414450+0000","last_undegraded":"2023-10-09T05:49:20.414450+0000","last_fullsized":"2023-10-09T05:49:20.414450+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:22.445381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.409547+0000","last_change":"2023-10-09T05:48:49.137529+0000","last_active":"2023-10-09T05:49:20.409547+0000","last_peered":"2023-10-09T05:49:20.409547+0000","last_clean":"2023-10-09T05:49:20.409547+0000","last_became_active":"2023-10-09T05:48:49.137212+0000","last_became_peered":"2023-10-09T05:48:49.137212+0000","last_unstale":"2023-10-09T05:49:20.409547+0000","last_undegraded":"2023-10-09T05:49:20.409547+0000","last_fullsized":"2023-10-09T05:49:20.409547+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:29.740386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":82,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.429787+0000","last_change":"2023-10-09T05:49:21.429787+0000","last_active":"2023-10-09T05:49:21.429787+0000","last_peered":"2023-10-09T05:49:21.429787+0000","last_clean":"2023-10-09T05:49:21.429787+0000","last_became_active":"2023-10-09T05:49:21.429304+0000","last_became_peered":"2023-10-09T05:49:21.429304+0000","last_unstale":"2023-10-09T05:49:21.429787+0000","last_undegraded":"2023-10-09T05:49:21.429787+0000","last_fullsized":"2023-10-09T05:49:21.429787+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:28:24.515861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":73,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.429250+0000","last_change":"2023-10-09T05:49:21.429250+0000","last_active":"2023-10-09T05:49:21.429250+0000","last_peered":"2023-10-09T05:49:21.429250+0000","last_clean":"2023-10-09T05:49:21.429250+0000","last_became_active":"2023-10-09T05:49:21.428946+0000","last_became_peered":"2023-10-09T05:49:21.428946+0000","last_unstale":"2023-10-09T05:49:21.429250+0000","last_undegraded":"2023-10-09T05:49:21.429250+0000","last_fullsized":"2023-10-09T05:49:21.429250+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:32.978473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.414007+0000","last_change":"2023-10-09T05:47:03.237144+0000","last_active":"2023-10-09T05:49:20.414007+0000","last_peered":"2023-10-09T05:49:20.414007+0000","last_clean":"2023-10-09T05:49:20.414007+0000","last_became_active":"2023-10-09T05:47:03.236793+0000","last_became_peered":"2023-10-09T05:47:03.236793+0000","last_unstale":"2023-10-09T05:49:20.414007+0000","last_undegraded":"2023-10-09T05:49:20.414007+0000","last_fullsized":"2023-10-09T05:49:20.414007+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:00:44.273589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.422361+0000","last_change":"2023-10-09T05:48:17.172780+0000","last_active":"2023-10-09T05:49:20.422361+0000","last_peered":"2023-10-09T05:49:20.422361+0000","last_clean":"2023-10-09T05:49:20.422361+0000","last_became_active":"2023-10-09T05:48:17.172460+0000","last_became_peered":"2023-10-09T05:48:17.172460+0000","last_unstale":"2023-10-09T05:49:20.422361+0000","last_undegraded":"2023-10-09T05:49:20.422361+0000","last_fullsized":"2023-10-09T05:49:20.422361+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:41.981302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":32,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.412397+0000","last_change":"2023-10-09T05:48:17.172596+0000","last_active":"2023-10-09T05:49:20.412397+0000","last_peered":"2023-10-09T05:49:20.412397+0000","last_clean":"2023-10-09T05:49:20.412397+0000","last_became_active":"2023-10-09T05:48:17.172287+0000","last_became_peered":"2023-10-09T05:48:17.172287+0000","last_unstale":"2023-10-09T05:49:20.412397+0000","last_undegraded":"2023-10-09T05:49:20.412397+0000","last_fullsized":"2023-10-09T05:49:20.412397+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:53.440906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":77,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.422980+0000","last_change":"2023-10-09T05:49:21.422980+0000","last_active":"2023-10-09T05:49:21.422980+0000","last_peered":"2023-10-09T05:49:21.422980+0000","last_clean":"2023-10-09T05:49:21.422980+0000","last_became_active":"2023-10-09T05:49:21.422349+0000","last_became_peered":"2023-10-09T05:49:21.422349+0000","last_unstale":"2023-10-09T05:49:21.422980+0000","last_undegraded":"2023-10-09T05:49:21.422980+0000","last_fullsized":"2023-10-09T05:49:21.422980+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:03.118182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":72,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.409864+0000","last_change":"2023-10-09T05:48:17.170845+0000","last_active":"2023-10-09T05:49:20.409864+0000","last_peered":"2023-10-09T05:49:20.409864+0000","last_clean":"2023-10-09T05:49:20.409864+0000","last_became_active":"2023-10-09T05:48:17.170321+0000","last_became_peered":"2023-10-09T05:48:17.170321+0000","last_unstale":"2023-10-09T05:49:20.409864+0000","last_undegraded":"2023-10-09T05:49:20.409864+0000","last_fullsized":"2023-10-09T05:49:20.409864+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:41.854157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":47,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.431778+0000","last_change":"2023-10-09T05:49:21.431778+0000","last_active":"2023-10-09T05:49:21.431778+0000","last_peered":"2023-10-09T05:49:21.431778+0000","last_clean":"2023-10-09T05:49:21.431778+0000","last_became_active":"2023-10-09T05:49:21.431483+0000","last_became_peered":"2023-10-09T05:49:21.431483+0000","last_unstale":"2023-10-09T05:49:21.431778+0000","last_undegraded":"2023-10-09T05:49:21.431778+0000","last_fullsized":"2023-10-09T05:49:21.431778+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:29:39.293436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":64,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.427586+0000","last_change":"2023-10-09T05:49:21.427586+0000","last_active":"2023-10-09T05:49:21.427586+0000","last_peered":"2023-10-09T05:49:21.427586+0000","last_clean":"2023-10-09T05:49:21.427586+0000","last_became_active":"2023-10-09T05:49:21.426908+0000","last_became_peered":"2023-10-09T05:49:21.426908+0000","last_unstale":"2023-10-09T05:49:21.427586+0000","last_undegraded":"2023-10-09T05:49:21.427586+0000","last_fullsized":"2023-10-09T05:49:21.427586+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:50.043430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.414177+0000","last_change":"2023-10-09T05:48:17.173636+0000","last_active":"2023-10-09T05:49:20.414177+0000","last_peered":"2023-10-09T05:49:20.414177+0000","last_clean":"2023-10-09T05:49:20.414177+0000","last_became_active":"2023-10-09T05:48:17.173329+0000","last_became_peered":"2023-10-09T05:48:17.173329+0000","last_unstale":"2023-10-09T05:49:20.414177+0000","last_undegraded":"2023-10-09T05:49:20.414177+0000","last_fullsized":"2023-10-09T05:49:20.414177+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:26:36.634285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.414234+0000","last_change":"2023-10-09T05:48:49.141287+0000","last_active":"2023-10-09T05:49:20.414234+0000","last_peered":"2023-10-09T05:49:20.414234+0000","last_clean":"2023-10-09T05:49:20.414234+0000","last_became_active":"2023-10-09T05:48:49.140950+0000","last_became_peered":"2023-10-09T05:48:49.140950+0000","last_unstale":"2023-10-09T05:49:20.414234+0000","last_undegraded":"2023-10-09T05:49:20.414234+0000","last_fullsized":"2023-10-09T05:49:20.414234+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:35.527434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":76,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.413852+0000","last_change":"2023-10-09T05:48:49.138043+0000","last_active":"2023-10-09T05:49:20.413852+0000","last_peered":"2023-10-09T05:49:20.413852+0000","last_clean":"2023-10-09T05:49:20.413852+0000","last_became_active":"2023-10-09T05:48:49.137722+0000","last_became_peered":"2023-10-09T05:48:49.137722+0000","last_unstale":"2023-10-09T05:49:20.413852+0000","last_undegraded":"2023-10-09T05:49:20.413852+0000","last_fullsized":"2023-10-09T05:49:20.413852+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:38.270179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.420336+0000","last_change":"2023-10-09T05:48:17.179214+0000","last_active":"2023-10-09T05:49:20.420336+0000","last_peered":"2023-10-09T05:49:20.420336+0000","last_clean":"2023-10-09T05:49:20.420336+0000","last_became_active":"2023-10-09T05:48:17.178918+0000","last_became_peered":"2023-10-09T05:48:17.178918+0000","last_unstale":"2023-10-09T05:49:20.420336+0000","last_undegraded":"2023-10-09T05:49:20.420336+0000","last_fullsized":"2023-10-09T05:49:20.420336+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:58:07.653506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":59,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.430742+0000","last_change":"2023-10-09T05:49:21.430742+0000","last_active":"2023-10-09T05:49:21.430742+0000","last_peered":"2023-10-09T05:49:21.430742+0000","last_clean":"2023-10-09T05:49:21.430742+0000","last_became_active":"2023-10-09T05:49:21.430233+0000","last_became_peered":"2023-10-09T05:49:21.430233+0000","last_unstale":"2023-10-09T05:49:21.430742+0000","last_undegraded":"2023-10-09T05:49:21.430742+0000","last_fullsized":"2023-10-09T05:49:21.430742+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:10.541302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":61,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.421396+0000","last_change":"2023-10-09T05:49:21.421396+0000","last_active":"2023-10-09T05:49:21.421396+0000","last_peered":"2023-10-09T05:49:21.421396+0000","last_clean":"2023-10-09T05:49:21.421396+0000","last_became_active":"2023-10-09T05:49:21.421129+0000","last_became_peered":"2023-10-09T05:49:21.421129+0000","last_unstale":"2023-10-09T05:49:21.421396+0000","last_undegraded":"2023-10-09T05:49:21.421396+0000","last_fullsized":"2023-10-09T05:49:21.421396+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:14.365625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.415371+0000","last_change":"2023-10-09T05:48:17.172849+0000","last_active":"2023-10-09T05:49:20.415371+0000","last_peered":"2023-10-09T05:49:20.415371+0000","last_clean":"2023-10-09T05:49:20.415371+0000","last_became_active":"2023-10-09T05:48:17.172536+0000","last_became_peered":"2023-10-09T05:48:17.172536+0000","last_unstale":"2023-10-09T05:49:20.415371+0000","last_undegraded":"2023-10-09T05:49:20.415371+0000","last_fullsized":"2023-10-09T05:49:20.415371+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:52.891228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":60,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.429234+0000","last_change":"2023-10-09T05:49:21.429234+0000","last_active":"2023-10-09T05:49:21.429234+0000","last_peered":"2023-10-09T05:49:21.429234+0000","last_clean":"2023-10-09T05:49:21.429234+0000","last_became_active":"2023-10-09T05:49:21.428961+0000","last_became_peered":"2023-10-09T05:49:21.428961+0000","last_unstale":"2023-10-09T05:49:21.429234+0000","last_undegraded":"2023-10-09T05:49:21.429234+0000","last_fullsized":"2023-10-09T05:49:21.429234+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:18.029914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.420226+0000","last_change":"2023-10-09T05:48:49.139877+0000","last_active":"2023-10-09T05:49:20.420226+0000","last_peered":"2023-10-09T05:49:20.420226+0000","last_clean":"2023-10-09T05:49:20.420226+0000","last_became_active":"2023-10-09T05:48:49.139479+0000","last_became_peered":"2023-10-09T05:48:49.139479+0000","last_unstale":"2023-10-09T05:49:20.420226+0000","last_undegraded":"2023-10-09T05:49:20.420226+0000","last_fullsized":"2023-10-09T05:49:20.420226+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:14:16.097335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":44,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.423108+0000","last_change":"2023-10-09T05:40:12.632083+0000","last_active":"2023-10-09T05:49:20.423108+0000","last_peered":"2023-10-09T05:49:20.423108+0000","last_clean":"2023-10-09T05:49:20.423108+0000","last_became_active":"2023-10-09T05:40:12.631715+0000","last_became_peered":"2023-10-09T05:40:12.631715+0000","last_unstale":"2023-10-09T05:49:20.423108+0000","last_undegraded":"2023-10-09T05:49:20.423108+0000","last_fullsized":"2023-10-09T05:49:20.423108+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:17.641378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":76,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426392+0000","last_change":"2023-10-09T05:49:21.426392+0000","last_active":"2023-10-09T05:49:21.426392+0000","last_peered":"2023-10-09T05:49:21.426392+0000","last_clean":"2023-10-09T05:49:21.426392+0000","last_became_active":"2023-10-09T05:49:21.426056+0000","last_became_peered":"2023-10-09T05:49:21.426056+0000","last_unstale":"2023-10-09T05:49:21.426392+0000","last_undegraded":"2023-10-09T05:49:21.426392+0000","last_fullsized":"2023-10-09T05:49:21.426392+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:09.862053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.420742+0000","last_change":"2023-10-09T05:48:17.173680+0000","last_active":"2023-10-09T05:49:20.420742+0000","last_peered":"2023-10-09T05:49:20.420742+0000","last_clean":"2023-10-09T05:49:20.420742+0000","last_became_active":"2023-10-09T05:48:17.173322+0000","last_became_peered":"2023-10-09T05:48:17.173322+0000","last_unstale":"2023-10-09T05:49:20.420742+0000","last_undegraded":"2023-10-09T05:49:20.420742+0000","last_fullsized":"2023-10-09T05:49:20.420742+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:41.005172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":42,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.421877+0000","last_change":"2023-10-09T05:40:14.362515+0000","last_active":"2023-10-09T05:49:20.421877+0000","last_peered":"2023-10-09T05:49:20.421877+0000","last_clean":"2023-10-09T05:49:20.421877+0000","last_became_active":"2023-10-09T05:40:14.362184+0000","last_became_peered":"2023-10-09T05:40:14.362184+0000","last_unstale":"2023-10-09T05:49:20.421877+0000","last_undegraded":"2023-10-09T05:49:20.421877+0000","last_fullsized":"2023-10-09T05:49:20.421877+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:37.681193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.423481+0000","last_change":"2023-10-09T05:47:03.230162+0000","last_active":"2023-10-09T05:49:20.423481+0000","last_peered":"2023-10-09T05:49:20.423481+0000","last_clean":"2023-10-09T05:49:20.423481+0000","last_became_active":"2023-10-09T05:47:03.229695+0000","last_became_peered":"2023-10-09T05:47:03.229695+0000","last_unstale":"2023-10-09T05:49:20.423481+0000","last_undegraded":"2023-10-09T05:49:20.423481+0000","last_fullsized":"2023-10-09T05:49:20.423481+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:13.194065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.427270+0000","last_change":"2023-10-09T05:49:21.427270+0000","last_active":"2023-10-09T05:49:21.427270+0000","last_peered":"2023-10-09T05:49:21.427270+0000","last_clean":"2023-10-09T05:49:21.427270+0000","last_became_active":"2023-10-09T05:49:21.427014+0000","last_became_peered":"2023-10-09T05:49:21.427014+0000","last_unstale":"2023-10-09T05:49:21.427270+0000","last_undegraded":"2023-10-09T05:49:21.427270+0000","last_fullsized":"2023-10-09T05:49:21.427270+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:36:30.822627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":68,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.416655+0000","last_change":"2023-10-09T05:47:03.230438+0000","last_active":"2023-10-09T05:49:20.416655+0000","last_peered":"2023-10-09T05:49:20.416655+0000","last_clean":"2023-10-09T05:49:20.416655+0000","last_became_active":"2023-10-09T05:47:03.229346+0000","last_became_peered":"2023-10-09T05:47:03.229346+0000","last_unstale":"2023-10-09T05:49:20.416655+0000","last_undegraded":"2023-10-09T05:49:20.416655+0000","last_fullsized":"2023-10-09T05:49:20.416655+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:54.948861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426926+0000","last_change":"2023-10-09T05:49:21.426431+0000","last_active":"2023-10-09T05:49:21.426926+0000","last_peered":"2023-10-09T05:49:21.426926+0000","last_clean":"2023-10-09T05:49:21.426926+0000","last_became_active":"2023-10-09T05:49:21.426011+0000","last_became_peered":"2023-10-09T05:49:21.426011+0000","last_unstale":"2023-10-09T05:49:21.426926+0000","last_undegraded":"2023-10-09T05:49:21.426926+0000","last_fullsized":"2023-10-09T05:49:21.426926+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:27.350168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":43,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.419867+0000","last_change":"2023-10-09T05:40:12.631893+0000","last_active":"2023-10-09T05:49:20.419867+0000","last_peered":"2023-10-09T05:49:20.419867+0000","last_clean":"2023-10-09T05:49:20.419867+0000","last_became_active":"2023-10-09T05:40:12.631592+0000","last_became_peered":"2023-10-09T05:40:12.631592+0000","last_unstale":"2023-10-09T05:49:20.419867+0000","last_undegraded":"2023-10-09T05:49:20.419867+0000","last_fullsized":"2023-10-09T05:49:20.419867+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:18:12.517403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":59,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.423218+0000","last_change":"2023-10-09T05:49:21.423218+0000","last_active":"2023-10-09T05:49:21.423218+0000","last_peered":"2023-10-09T05:49:21.423218+0000","last_clean":"2023-10-09T05:49:21.423218+0000","last_became_active":"2023-10-09T05:49:21.422706+0000","last_became_peered":"2023-10-09T05:49:21.422706+0000","last_unstale":"2023-10-09T05:49:21.423218+0000","last_undegraded":"2023-10-09T05:49:21.423218+0000","last_fullsized":"2023-10-09T05:49:21.423218+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:31.571220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426167+0000","last_change":"2023-10-09T05:49:21.426167+0000","last_active":"2023-10-09T05:49:21.426167+0000","last_peered":"2023-10-09T05:49:21.426167+0000","last_clean":"2023-10-09T05:49:21.426167+0000","last_became_active":"2023-10-09T05:49:21.425874+0000","last_became_peered":"2023-10-09T05:49:21.425874+0000","last_unstale":"2023-10-09T05:49:21.426167+0000","last_undegraded":"2023-10-09T05:49:21.426167+0000","last_fullsized":"2023-10-09T05:49:21.426167+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:59.642587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.428386+0000","last_change":"2023-10-09T05:49:21.428386+0000","last_active":"2023-10-09T05:49:21.428386+0000","last_peered":"2023-10-09T05:49:21.428386+0000","last_clean":"2023-10-09T05:49:21.428386+0000","last_became_active":"2023-10-09T05:49:21.428131+0000","last_became_peered":"2023-10-09T05:49:21.428131+0000","last_unstale":"2023-10-09T05:49:21.428386+0000","last_undegraded":"2023-10-09T05:49:21.428386+0000","last_fullsized":"2023-10-09T05:49:21.428386+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:12.858458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.419185+0000","last_change":"2023-10-09T05:48:49.139681+0000","last_active":"2023-10-09T05:49:20.419185+0000","last_peered":"2023-10-09T05:49:20.419185+0000","last_clean":"2023-10-09T05:49:20.419185+0000","last_became_active":"2023-10-09T05:48:49.139317+0000","last_became_peered":"2023-10-09T05:48:49.139317+0000","last_unstale":"2023-10-09T05:49:20.419185+0000","last_undegraded":"2023-10-09T05:49:20.419185+0000","last_fullsized":"2023-10-09T05:49:20.419185+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:34.769043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.413441+0000","last_change":"2023-10-09T05:48:17.173137+0000","last_active":"2023-10-09T05:49:20.413441+0000","last_peered":"2023-10-09T05:49:20.413441+0000","last_clean":"2023-10-09T05:49:20.413441+0000","last_became_active":"2023-10-09T05:48:17.172891+0000","last_became_peered":"2023-10-09T05:48:17.172891+0000","last_unstale":"2023-10-09T05:49:20.413441+0000","last_undegraded":"2023-10-09T05:49:20.413441+0000","last_fullsized":"2023-10-09T05:49:20.413441+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:21.393939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":65,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.413929+0000","last_change":"2023-10-09T05:40:12.630786+0000","last_active":"2023-10-09T05:49:20.413929+0000","last_peered":"2023-10-09T05:49:20.413929+0000","last_clean":"2023-10-09T05:49:20.413929+0000","last_became_active":"2023-10-09T05:40:12.630528+0000","last_became_peered":"2023-10-09T05:40:12.630528+0000","last_unstale":"2023-10-09T05:49:20.413929+0000","last_undegraded":"2023-10-09T05:49:20.413929+0000","last_fullsized":"2023-10-09T05:49:20.413929+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:54.427346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.412474+0000","last_change":"2023-10-09T05:48:49.138162+0000","last_active":"2023-10-09T05:49:20.412474+0000","last_peered":"2023-10-09T05:49:20.412474+0000","last_clean":"2023-10-09T05:49:20.412474+0000","last_became_active":"2023-10-09T05:48:49.137824+0000","last_became_peered":"2023-10-09T05:48:49.137824+0000","last_unstale":"2023-10-09T05:49:20.412474+0000","last_undegraded":"2023-10-09T05:49:20.412474+0000","last_fullsized":"2023-10-09T05:49:20.412474+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:43:26.642768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.430469+0000","last_change":"2023-10-09T05:49:21.430469+0000","last_active":"2023-10-09T05:49:21.430469+0000","last_peered":"2023-10-09T05:49:21.430469+0000","last_clean":"2023-10-09T05:49:21.430469+0000","last_became_active":"2023-10-09T05:49:21.430107+0000","last_became_peered":"2023-10-09T05:49:21.430107+0000","last_unstale":"2023-10-09T05:49:21.430469+0000","last_undegraded":"2023-10-09T05:49:21.430469+0000","last_fullsized":"2023-10-09T05:49:21.430469+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:08.815343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.425819+0000","last_change":"2023-10-09T05:49:21.425819+0000","last_active":"2023-10-09T05:49:21.425819+0000","last_peered":"2023-10-09T05:49:21.425819+0000","last_clean":"2023-10-09T05:49:21.425819+0000","last_became_active":"2023-10-09T05:49:21.425498+0000","last_became_peered":"2023-10-09T05:49:21.425498+0000","last_unstale":"2023-10-09T05:49:21.425819+0000","last_undegraded":"2023-10-09T05:49:21.425819+0000","last_fullsized":"2023-10-09T05:49:21.425819+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:21.443055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.421834+0000","last_change":"2023-10-09T05:48:17.176541+0000","last_active":"2023-10-09T05:49:20.421834+0000","last_peered":"2023-10-09T05:49:20.421834+0000","last_clean":"2023-10-09T05:49:20.421834+0000","last_became_active":"2023-10-09T05:48:17.176213+0000","last_became_peered":"2023-10-09T05:48:17.176213+0000","last_unstale":"2023-10-09T05:49:20.421834+0000","last_undegraded":"2023-10-09T05:49:20.421834+0000","last_fullsized":"2023-10-09T05:49:20.421834+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:23.599435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.427804+0000","last_change":"2023-10-09T05:49:21.427804+0000","last_active":"2023-10-09T05:49:21.427804+0000","last_peered":"2023-10-09T05:49:21.427804+0000","last_clean":"2023-10-09T05:49:21.427804+0000","last_became_active":"2023-10-09T05:49:21.427569+0000","last_became_peered":"2023-10-09T05:49:21.427569+0000","last_unstale":"2023-10-09T05:49:21.427804+0000","last_undegraded":"2023-10-09T05:49:21.427804+0000","last_fullsized":"2023-10-09T05:49:21.427804+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.791826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":53,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426616+0000","last_change":"2023-10-09T05:49:21.426149+0000","last_active":"2023-10-09T05:49:21.426616+0000","last_peered":"2023-10-09T05:49:21.426616+0000","last_clean":"2023-10-09T05:49:21.426616+0000","last_became_active":"2023-10-09T05:49:21.425805+0000","last_became_peered":"2023-10-09T05:49:21.425805+0000","last_unstale":"2023-10-09T05:49:21.426616+0000","last_undegraded":"2023-10-09T05:49:21.426616+0000","last_fullsized":"2023-10-09T05:49:21.426616+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:39:57.117762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.421197+0000","last_change":"2023-10-09T05:48:17.179524+0000","last_active":"2023-10-09T05:49:20.421197+0000","last_peered":"2023-10-09T05:49:20.421197+0000","last_clean":"2023-10-09T05:49:20.421197+0000","last_became_active":"2023-10-09T05:48:17.179233+0000","last_became_peered":"2023-10-09T05:48:17.179233+0000","last_unstale":"2023-10-09T05:49:20.421197+0000","last_undegraded":"2023-10-09T05:49:20.421197+0000","last_fullsized":"2023-10-09T05:49:20.421197+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:24.135263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":81,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.427745+0000","last_change":"2023-10-09T05:49:21.427745+0000","last_active":"2023-10-09T05:49:21.427745+0000","last_peered":"2023-10-09T05:49:21.427745+0000","last_clean":"2023-10-09T05:49:21.427745+0000","last_became_active":"2023-10-09T05:49:21.427292+0000","last_became_peered":"2023-10-09T05:49:21.427292+0000","last_unstale":"2023-10-09T05:49:21.427745+0000","last_undegraded":"2023-10-09T05:49:21.427745+0000","last_fullsized":"2023-10-09T05:49:21.427745+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:04:24.929346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.427476+0000","last_change":"2023-10-09T05:49:21.427476+0000","last_active":"2023-10-09T05:49:21.427476+0000","last_peered":"2023-10-09T05:49:21.427476+0000","last_clean":"2023-10-09T05:49:21.427476+0000","last_became_active":"2023-10-09T05:49:21.426417+0000","last_became_peered":"2023-10-09T05:49:21.426417+0000","last_unstale":"2023-10-09T05:49:21.427476+0000","last_undegraded":"2023-10-09T05:49:21.427476+0000","last_fullsized":"2023-10-09T05:49:21.427476+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:45:03.848962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.431711+0000","last_change":"2023-10-09T05:49:21.431711+0000","last_active":"2023-10-09T05:49:21.431711+0000","last_peered":"2023-10-09T05:49:21.431711+0000","last_clean":"2023-10-09T05:49:21.431711+0000","last_became_active":"2023-10-09T05:49:21.431441+0000","last_became_peered":"2023-10-09T05:49:21.431441+0000","last_unstale":"2023-10-09T05:49:21.431711+0000","last_undegraded":"2023-10-09T05:49:21.431711+0000","last_fullsized":"2023-10-09T05:49:21.431711+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:26.617516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.427978+0000","last_change":"2023-10-09T05:49:21.427978+0000","last_active":"2023-10-09T05:49:21.427978+0000","last_peered":"2023-10-09T05:49:21.427978+0000","last_clean":"2023-10-09T05:49:21.427978+0000","last_became_active":"2023-10-09T05:49:21.427703+0000","last_became_peered":"2023-10-09T05:49:21.427703+0000","last_unstale":"2023-10-09T05:49:21.427978+0000","last_undegraded":"2023-10-09T05:49:21.427978+0000","last_fullsized":"2023-10-09T05:49:21.427978+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:53:04.116683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":78,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.413362+0000","last_change":"2023-10-09T05:48:49.142429+0000","last_active":"2023-10-09T05:49:20.413362+0000","last_peered":"2023-10-09T05:49:20.413362+0000","last_clean":"2023-10-09T05:49:20.413362+0000","last_became_active":"2023-10-09T05:48:49.141966+0000","last_became_peered":"2023-10-09T05:48:49.141966+0000","last_unstale":"2023-10-09T05:49:20.413362+0000","last_undegraded":"2023-10-09T05:49:20.413362+0000","last_fullsized":"2023-10-09T05:49:20.413362+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:52.195431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234127,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311160,"kb_used_data":1584,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93433992,"statfs":{"total":95995035648,"available":95676407808,"internally_reserved":0,"allocated":1622016,"data_stored":1140338,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:49:07 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.745,"15min":0.77900000000000003},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.75900000000000001,"5min":4.8940000000000001,"15min":4.8940000000000001},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.59599999999999997,"15min":0.60699999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.73999999999999999,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.57199999999999995}]},{"osd":1,"last update":"Mon Oct 9 05:49:17 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.376},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.64500000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:49:50 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.47199999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:48:53 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.57299999999999995,"15min":0.82899999999999996},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.77100000000000002,"5min":0.82199999999999995,"15min":95.108000000000004},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58199999999999996,"15min":0.83199999999999996},"min":{"1min":0.42999999999999999,"5min":0.36199999999999999,"15min":0.32800000000000001},"max":{"1min":0.749,"5min":0.94099999999999995,"15min":87.664000000000001},"last":0.51500000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:49:56 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.48399999999999999,"15min":0.48699999999999999},"min":{"1min":0.313,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":1.135},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.51200000000000001,"15min":0.498},"min":{"1min":0.378,"5min":0.27400000000000002,"15min":0.27100000000000002},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":3.165},"last":0.55700000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:49:56 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.499,"15min":0.499},"min":{"1min":0.33200000000000002,"5min":0.28199999999999997,"15min":0.248},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":2.5990000000000002},"last":0.496},{"interface":"front","average":{"1min":0.622,"5min":0.55700000000000005,"15min":0.54000000000000004},"min":{"1min":0.38800000000000001,"5min":0.34000000000000002,"15min":0.28100000000000003},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":1.0660000000000001},"last":0.621}]},{"osd":6,"last update":"Mon Oct 9 05:49:56 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.55900000000000005,"15min":0.54300000000000004},"min":{"1min":0.42399999999999999,"5min":0.317,"15min":0.29599999999999999},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":2.8170000000000002},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.57299999999999995,"15min":0.55100000000000005},"min":{"1min":0.41899999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":3.2589999999999999},"last":0.58499999999999996}]}]},{"osd":6,"up_from":46,"seq":197568495763,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298856,"kb_used_data":1568,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446296,"statfs":{"total":95995035648,"available":95689007104,"internally_reserved":0,"allocated":1605632,"data_stored":1139927,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:49:07 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.58899999999999997,"15min":0.59599999999999997},"min":{"1min":0.38400000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.58899999999999997,"15min":0.59499999999999997},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.38600000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:49:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.45900000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:49:50 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.49199999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:48:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.59999999999999998,"15min":0.56200000000000006},"min":{"1min":0.46100000000000002,"5min":0.38300000000000001,"15min":0.318},"max":{"1min":0.79000000000000004,"5min":0.85199999999999998,"15min":1.1100000000000001},"last":0.442},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.57799999999999996,"15min":0.55600000000000005},"min":{"1min":0.40699999999999997,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.77400000000000002,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.56799999999999995}]},{"osd":4,"last update":"Mon Oct 9 05:49:28 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.48499999999999999,"15min":0.45400000000000001},"min":{"1min":0.35499999999999998,"5min":0.309,"15min":0.26300000000000001},"max":{"1min":0.67300000000000004,"5min":0.71899999999999997,"15min":0.75600000000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.51700000000000002,"15min":0.48899999999999999},"min":{"1min":0.33400000000000002,"5min":0.29099999999999998,"15min":0.28599999999999998},"max":{"1min":0.75900000000000001,"5min":0.80500000000000005,"15min":0.82799999999999996},"last":0.61199999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:49:28 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.50900000000000001,"15min":0.47199999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.27700000000000002},"max":{"1min":0.77500000000000002,"5min":0.78200000000000003,"15min":0.81299999999999994},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.54500000000000004,"15min":0.51800000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32200000000000001},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85799999999999998},"last":0.51000000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:49:58 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.54700000000000004,"15min":0.53400000000000003},"min":{"1min":0.39800000000000002,"5min":0.316,"15min":0.316},"max":{"1min":0.64800000000000002,"5min":0.78800000000000003,"15min":0.79700000000000004},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.626,"5min":0.627,"15min":0.59799999999999998},"min":{"1min":0.39800000000000002,"5min":0.39600000000000002,"15min":0.35799999999999998},"max":{"1min":0.82299999999999995,"5min":0.88200000000000001,"15min":0.88300000000000001},"last":0.63900000000000001}]}]},{"osd":1,"up_from":89,"seq":382252089369,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28492,"kb_used_data":1028,"kb_used_omap":9,"kb_used_meta":27446,"kb_avail":93716660,"statfs":{"total":95995035648,"available":95965859840,"internally_reserved":0,"allocated":1052672,"data_stored":555584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":28105705},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:49:18 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"last":0.316},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.36099999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.5119999999999996,"5min":4.5119999999999996,"15min":4.5119999999999996},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.6139999999999999,"5min":3.6139999999999999,"15min":3.6139999999999999},"last":0.29599999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.374}]},{"osd":4,"last update":"Mon Oct 9 05:49:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.64900000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:49:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.59999999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:49:18 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.70199999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:49:18 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.67500000000000004}]}]},{"osd":0,"up_from":84,"seq":360777252903,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29160,"kb_used_data":1632,"kb_used_omap":28,"kb_used_meta":27491,"kb_avail":93715992,"statfs":{"total":95995035648,"available":95965175808,"internally_reserved":0,"allocated":1671168,"data_stored":1149588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":28151632},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.625,"5min":7.625,"15min":7.625},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":3.8130000000000002,"5min":3.8130000000000002,"15min":3.8130000000000002},"last":0.40600000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:49:49 2023","interfaces":[{"interface":"back","average":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.5640000000000001,"5min":7.5640000000000001,"15min":7.5640000000000001},"last":0.36299999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:49:04 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.502,"15min":0.51400000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.51000000000000001,"15min":0.52000000000000002},"min":{"1min":0.33600000000000002,"5min":0.307,"15min":0.307},"max":{"1min":0.63,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.38}]},{"osd":4,"last update":"Mon Oct 9 05:50:04 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.67500000000000004,"15min":0.69499999999999995},"min":{"1min":0.42499999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.69799999999999995,"15min":0.72199999999999998},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.90100000000000002,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.71799999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:50:04 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.66800000000000004,"15min":0.68600000000000005},"min":{"1min":0.42199999999999999,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.622,"5min":0.68200000000000005,"15min":0.71599999999999997},"min":{"1min":0.498,"5min":0.376,"15min":0.376},"max":{"1min":1.27,"5min":1.27,"15min":1.27},"last":0.53100000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:50:04 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.68999999999999995,"15min":0.72499999999999998},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.71799999999999997,"15min":0.746},"min":{"1min":0.47499999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.995,"5min":0.995,"15min":0.995},"last":0.74199999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:50:04 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.75700000000000001,"15min":0.78900000000000003},"min":{"1min":0.47799999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.016,"5min":1.016,"15min":1.016},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.73699999999999999,"15min":0.76500000000000001},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.59299999999999997}]}]},{"osd":2,"up_from":94,"seq":403726925842,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28620,"kb_used_data":1028,"kb_used_omap":81,"kb_used_meta":27502,"kb_avail":93716532,"statfs":{"total":95995035648,"available":95965728768,"internally_reserved":0,"allocated":1052672,"data_stored":554321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":28162266},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.318},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.30399999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.23799999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34799999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.68999999999999995}]},{"osd":5,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.55800000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.51500000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.66900000000000004}]}]},{"osd":3,"up_from":99,"seq":425201762316,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28468,"kb_used_data":1004,"kb_used_omap":7,"kb_used_meta":27448,"kb_avail":93716684,"statfs":{"total":95995035648,"available":95965884416,"internally_reserved":0,"allocated":1028096,"data_stored":549876,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":28107368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45500000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.434}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48599999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70599999999999996}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66000000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68000000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60799999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019036,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302388,"kb_used_data":1004,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93442764,"statfs":{"total":95995035648,"available":95685390336,"internally_reserved":0,"allocated":1028096,"data_stored":550223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:49:07 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.58999999999999997,"15min":0.60199999999999998},"min":{"1min":0.42199999999999999,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.64400000000000002,"5min":0.75,"15min":0.75},"last":0.502},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.629,"15min":0.64200000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.78800000000000003,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.54700000000000004}]},{"osd":1,"last update":"Mon Oct 9 05:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.59299999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:49:50 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.032,"5min":1.032,"15min":1.032},"last":0.66100000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:48:56 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.60299999999999998,"15min":0.58699999999999997},"min":{"1min":0.443,"5min":0.38,"15min":0.32600000000000001},"max":{"1min":0.76100000000000001,"5min":0.81200000000000006,"15min":0.995},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.59499999999999997,"15min":0.57899999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.35899999999999999},"max":{"1min":0.71199999999999997,"5min":0.83599999999999997,"15min":0.89800000000000002},"last":0.68100000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:49:07 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.48299999999999998,"15min":0.45700000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.66400000000000003,"5min":0.83399999999999996,"15min":0.90500000000000003},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.52300000000000002,"15min":0.48899999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.88100000000000001},"last":0.44500000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:49:27 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.50900000000000001,"15min":0.495},"min":{"1min":0.34100000000000003,"5min":0.31,"15min":0.27400000000000002},"max":{"1min":0.64300000000000002,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.55700000000000005,"15min":0.53200000000000003},"min":{"1min":0.34799999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.72499999999999998,"5min":0.86199999999999999,"15min":0.89900000000000002},"last":0.64300000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:50:00 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.58399999999999996,"15min":0.56799999999999995},"min":{"1min":0.41699999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":1.046},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.58699999999999997,"15min":0.58399999999999996},"min":{"1min":0.48199999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.95799999999999996},"last":0.53300000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757399,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306468,"kb_used_data":988,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93438684,"statfs":{"total":95995035648,"available":95681212416,"internally_reserved":0,"allocated":1011712,"data_stored":545967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:50:07 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.57299999999999995,"15min":0.58999999999999997},"min":{"1min":0.35599999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.81100000000000005,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.59199999999999997,"15min":0.61499999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.68899999999999995,"5min":1,"15min":1},"last":0.65100000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:49:19 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.59899999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:49:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.68500000000000005}]},{"osd":3,"last update":"Mon Oct 9 05:49:07 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.59599999999999997,"15min":1.46},"min":{"1min":0.373,"5min":0.373,"15min":0.36699999999999999},"max":{"1min":0.88700000000000001,"5min":0.98499999999999999,"15min":247.672},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.61399999999999999,"15min":1.4930000000000001},"min":{"1min":0.48399999999999999,"5min":0.36899999999999999,"15min":0.36799999999999999},"max":{"1min":0.82299999999999995,"5min":0.88600000000000001,"15min":247.90000000000001},"last":0.50600000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:50:07 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.48799999999999999,"15min":0.45600000000000002},"min":{"1min":0.311,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.754,"5min":0.754,"15min":3.5430000000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.50800000000000001,"15min":0.47599999999999998},"min":{"1min":0.372,"5min":0.29999999999999999,"15min":0.28999999999999998},"max":{"1min":0.66600000000000004,"5min":0.72699999999999998,"15min":1.6990000000000001},"last":0.57199999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:49:24 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.53000000000000003,"15min":0.48799999999999999},"min":{"1min":0.41899999999999998,"5min":0.307,"15min":0.29899999999999999},"max":{"1min":0.69799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.622,"5min":0.57799999999999996,"15min":0.52200000000000002},"min":{"1min":0.39300000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.80600000000000005,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.67000000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:50:03 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.60799999999999998,"15min":0.56299999999999994},"min":{"1min":0.435,"5min":0.32700000000000001,"15min":0.315},"max":{"1min":0.81000000000000005,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.58799999999999997,"15min":0.59799999999999998},"min":{"1min":0.47399999999999998,"5min":0.35099999999999998,"15min":0.34899999999999998},"max":{"1min":0.79400000000000004,"5min":0.79700000000000004,"15min":3.2869999999999999},"last":0.54200000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:50:12.586 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:50:12.945 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:13.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:13 smithi156 ceph-mon[195101]: from='client.34587 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:50:13.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:13 smithi064 ceph-mon[212750]: from='client.34587 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:50:13.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:13 smithi064 ceph-mon[220044]: from='client.34587 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:50:14.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:14 smithi064 ceph-mon[212750]: pgmap v210: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:14.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:14.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:14 smithi064 ceph-mon[220044]: pgmap v210: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:14.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:14 smithi156 ceph-mon[195101]: pgmap v210: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:14.631 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:50:14.633 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T05:50:15.420 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":211,"stamp":"2023-10-09T05:50:14.432447+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1333612,"kb_used_data":9836,"kb_used_omap":126,"kb_used_meta":1323585,"kb_avail":748627604,"statfs":{"total":767960285184,"available":766594666496,"internally_reserved":0,"allocated":10072064,"data_stored":6185824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1355351675},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002848"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.413173+0000","last_change":"2023-10-09T05:48:17.172491+0000","last_active":"2023-10-09T05:49:20.413173+0000","last_peered":"2023-10-09T05:49:20.413173+0000","last_clean":"2023-10-09T05:49:20.413173+0000","last_became_active":"2023-10-09T05:48:17.172154+0000","last_became_peered":"2023-10-09T05:48:17.172154+0000","last_unstale":"2023-10-09T05:49:20.413173+0000","last_undegraded":"2023-10-09T05:49:20.413173+0000","last_fullsized":"2023-10-09T05:49:20.413173+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:24:40.551774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":57,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.416576+0000","last_change":"2023-10-09T05:48:49.139798+0000","last_active":"2023-10-09T05:49:20.416576+0000","last_peered":"2023-10-09T05:49:20.416576+0000","last_clean":"2023-10-09T05:49:20.416576+0000","last_became_active":"2023-10-09T05:48:49.139442+0000","last_became_peered":"2023-10-09T05:48:49.139442+0000","last_unstale":"2023-10-09T05:49:20.416576+0000","last_undegraded":"2023-10-09T05:49:20.416576+0000","last_fullsized":"2023-10-09T05:49:20.416576+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:25.034620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":84,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.421384+0000","last_change":"2023-10-09T05:49:21.421384+0000","last_active":"2023-10-09T05:49:21.421384+0000","last_peered":"2023-10-09T05:49:21.421384+0000","last_clean":"2023-10-09T05:49:21.421384+0000","last_became_active":"2023-10-09T05:49:21.420886+0000","last_became_peered":"2023-10-09T05:49:21.420886+0000","last_unstale":"2023-10-09T05:49:21.421384+0000","last_undegraded":"2023-10-09T05:49:21.421384+0000","last_fullsized":"2023-10-09T05:49:21.421384+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:33:24.415387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":69,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.430192+0000","last_change":"2023-10-09T05:49:21.430192+0000","last_active":"2023-10-09T05:49:21.430192+0000","last_peered":"2023-10-09T05:49:21.430192+0000","last_clean":"2023-10-09T05:49:21.430192+0000","last_became_active":"2023-10-09T05:49:21.429857+0000","last_became_peered":"2023-10-09T05:49:21.429857+0000","last_unstale":"2023-10-09T05:49:21.430192+0000","last_undegraded":"2023-10-09T05:49:21.430192+0000","last_fullsized":"2023-10-09T05:49:21.430192+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:45.887757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":76,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426615+0000","last_change":"2023-10-09T05:49:21.426615+0000","last_active":"2023-10-09T05:49:21.426615+0000","last_peered":"2023-10-09T05:49:21.426615+0000","last_clean":"2023-10-09T05:49:21.426615+0000","last_became_active":"2023-10-09T05:49:21.426245+0000","last_became_peered":"2023-10-09T05:49:21.426245+0000","last_unstale":"2023-10-09T05:49:21.426615+0000","last_undegraded":"2023-10-09T05:49:21.426615+0000","last_fullsized":"2023-10-09T05:49:21.426615+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:47.642455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":45,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.425017+0000","last_change":"2023-10-09T05:40:12.633195+0000","last_active":"2023-10-09T05:49:20.425017+0000","last_peered":"2023-10-09T05:49:20.425017+0000","last_clean":"2023-10-09T05:49:20.425017+0000","last_became_active":"2023-10-09T05:40:12.632942+0000","last_became_peered":"2023-10-09T05:40:12.632942+0000","last_unstale":"2023-10-09T05:49:20.425017+0000","last_undegraded":"2023-10-09T05:49:20.425017+0000","last_fullsized":"2023-10-09T05:49:20.425017+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:04.470685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":76,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426304+0000","last_change":"2023-10-09T05:49:21.426304+0000","last_active":"2023-10-09T05:49:21.426304+0000","last_peered":"2023-10-09T05:49:21.426304+0000","last_clean":"2023-10-09T05:49:21.426304+0000","last_became_active":"2023-10-09T05:49:21.425949+0000","last_became_peered":"2023-10-09T05:49:21.425949+0000","last_unstale":"2023-10-09T05:49:21.426304+0000","last_undegraded":"2023-10-09T05:49:21.426304+0000","last_fullsized":"2023-10-09T05:49:21.426304+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:11.209520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.424610+0000","last_change":"2023-10-09T05:48:17.173906+0000","last_active":"2023-10-09T05:49:20.424610+0000","last_peered":"2023-10-09T05:49:20.424610+0000","last_clean":"2023-10-09T05:49:20.424610+0000","last_became_active":"2023-10-09T05:48:17.173564+0000","last_became_peered":"2023-10-09T05:48:17.173564+0000","last_unstale":"2023-10-09T05:49:20.424610+0000","last_undegraded":"2023-10-09T05:49:20.424610+0000","last_fullsized":"2023-10-09T05:49:20.424610+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:37.582247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":81,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.430604+0000","last_change":"2023-10-09T05:49:21.430604+0000","last_active":"2023-10-09T05:49:21.430604+0000","last_peered":"2023-10-09T05:49:21.430604+0000","last_clean":"2023-10-09T05:49:21.430604+0000","last_became_active":"2023-10-09T05:49:21.430220+0000","last_became_peered":"2023-10-09T05:49:21.430220+0000","last_unstale":"2023-10-09T05:49:21.430604+0000","last_undegraded":"2023-10-09T05:49:21.430604+0000","last_fullsized":"2023-10-09T05:49:21.430604+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:06:01.882964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":42,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.417852+0000","last_change":"2023-10-09T05:47:03.233346+0000","last_active":"2023-10-09T05:49:21.417852+0000","last_peered":"2023-10-09T05:49:21.417852+0000","last_clean":"2023-10-09T05:49:21.417852+0000","last_became_active":"2023-10-09T05:47:03.231300+0000","last_became_peered":"2023-10-09T05:47:03.231300+0000","last_unstale":"2023-10-09T05:49:21.417852+0000","last_undegraded":"2023-10-09T05:49:21.417852+0000","last_fullsized":"2023-10-09T05:49:21.417852+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:22.050814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":42,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.414600+0000","last_change":"2023-10-09T05:40:12.676587+0000","last_active":"2023-10-09T05:49:20.414600+0000","last_peered":"2023-10-09T05:49:20.414600+0000","last_clean":"2023-10-09T05:49:20.414600+0000","last_became_active":"2023-10-09T05:40:12.676330+0000","last_became_peered":"2023-10-09T05:40:12.676330+0000","last_unstale":"2023-10-09T05:49:20.414600+0000","last_undegraded":"2023-10-09T05:49:20.414600+0000","last_fullsized":"2023-10-09T05:49:20.414600+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:46.659556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":57,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.416515+0000","last_change":"2023-10-09T05:48:17.174996+0000","last_active":"2023-10-09T05:49:20.416515+0000","last_peered":"2023-10-09T05:49:20.416515+0000","last_clean":"2023-10-09T05:49:20.416515+0000","last_became_active":"2023-10-09T05:48:17.174668+0000","last_became_peered":"2023-10-09T05:48:17.174668+0000","last_unstale":"2023-10-09T05:49:20.416515+0000","last_undegraded":"2023-10-09T05:49:20.416515+0000","last_fullsized":"2023-10-09T05:49:20.416515+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:49.712686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.421123+0000","last_change":"2023-10-09T05:48:49.137565+0000","last_active":"2023-10-09T05:49:20.421123+0000","last_peered":"2023-10-09T05:49:20.421123+0000","last_clean":"2023-10-09T05:49:20.421123+0000","last_became_active":"2023-10-09T05:48:49.137099+0000","last_became_peered":"2023-10-09T05:48:49.137099+0000","last_unstale":"2023-10-09T05:49:20.421123+0000","last_undegraded":"2023-10-09T05:49:20.421123+0000","last_fullsized":"2023-10-09T05:49:20.421123+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:28.103821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":58,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.423248+0000","last_change":"2023-10-09T05:49:21.423248+0000","last_active":"2023-10-09T05:49:21.423248+0000","last_peered":"2023-10-09T05:49:21.423248+0000","last_clean":"2023-10-09T05:49:21.423248+0000","last_became_active":"2023-10-09T05:49:21.422985+0000","last_became_peered":"2023-10-09T05:49:21.422985+0000","last_unstale":"2023-10-09T05:49:21.423248+0000","last_undegraded":"2023-10-09T05:49:21.423248+0000","last_fullsized":"2023-10-09T05:49:21.423248+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:02.431791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.409445+0000","last_change":"2023-10-09T05:48:17.174101+0000","last_active":"2023-10-09T05:49:20.409445+0000","last_peered":"2023-10-09T05:49:20.409445+0000","last_clean":"2023-10-09T05:49:20.409445+0000","last_became_active":"2023-10-09T05:48:17.173752+0000","last_became_peered":"2023-10-09T05:48:17.173752+0000","last_unstale":"2023-10-09T05:49:20.409445+0000","last_undegraded":"2023-10-09T05:49:20.409445+0000","last_fullsized":"2023-10-09T05:49:20.409445+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:45:16.222083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":61,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.423414+0000","last_change":"2023-10-09T05:49:21.423414+0000","last_active":"2023-10-09T05:49:21.423414+0000","last_peered":"2023-10-09T05:49:21.423414+0000","last_clean":"2023-10-09T05:49:21.423414+0000","last_became_active":"2023-10-09T05:49:21.423085+0000","last_became_peered":"2023-10-09T05:49:21.423085+0000","last_unstale":"2023-10-09T05:49:21.423414+0000","last_undegraded":"2023-10-09T05:49:21.423414+0000","last_fullsized":"2023-10-09T05:49:21.423414+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:51.777782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.414717+0000","last_change":"2023-10-09T05:47:03.233460+0000","last_active":"2023-10-09T05:49:20.414717+0000","last_peered":"2023-10-09T05:49:20.414717+0000","last_clean":"2023-10-09T05:49:20.414717+0000","last_became_active":"2023-10-09T05:47:03.231685+0000","last_became_peered":"2023-10-09T05:47:03.231685+0000","last_unstale":"2023-10-09T05:49:20.414717+0000","last_undegraded":"2023-10-09T05:49:20.414717+0000","last_fullsized":"2023-10-09T05:49:20.414717+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:10.790717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.409203+0000","last_change":"2023-10-09T05:48:49.139759+0000","last_active":"2023-10-09T05:49:20.409203+0000","last_peered":"2023-10-09T05:49:20.409203+0000","last_clean":"2023-10-09T05:49:20.409203+0000","last_became_active":"2023-10-09T05:48:49.139346+0000","last_became_peered":"2023-10-09T05:48:49.139346+0000","last_unstale":"2023-10-09T05:49:20.409203+0000","last_undegraded":"2023-10-09T05:49:20.409203+0000","last_fullsized":"2023-10-09T05:49:20.409203+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:51.980711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.420851+0000","last_change":"2023-10-09T05:47:03.230117+0000","last_active":"2023-10-09T05:49:20.420851+0000","last_peered":"2023-10-09T05:49:20.420851+0000","last_clean":"2023-10-09T05:49:20.420851+0000","last_became_active":"2023-10-09T05:47:03.229660+0000","last_became_peered":"2023-10-09T05:47:03.229660+0000","last_unstale":"2023-10-09T05:49:20.420851+0000","last_undegraded":"2023-10-09T05:49:20.420851+0000","last_fullsized":"2023-10-09T05:49:20.420851+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:07:21.669282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.416527+0000","last_change":"2023-10-09T05:48:49.134851+0000","last_active":"2023-10-09T05:49:20.416527+0000","last_peered":"2023-10-09T05:49:20.416527+0000","last_clean":"2023-10-09T05:49:20.416527+0000","last_became_active":"2023-10-09T05:48:49.134532+0000","last_became_peered":"2023-10-09T05:48:49.134532+0000","last_unstale":"2023-10-09T05:49:20.416527+0000","last_undegraded":"2023-10-09T05:49:20.416527+0000","last_fullsized":"2023-10-09T05:49:20.416527+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:23.395907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.422120+0000","last_change":"2023-10-09T05:49:21.422120+0000","last_active":"2023-10-09T05:49:21.422120+0000","last_peered":"2023-10-09T05:49:21.422120+0000","last_clean":"2023-10-09T05:49:21.422120+0000","last_became_active":"2023-10-09T05:49:21.421835+0000","last_became_peered":"2023-10-09T05:49:21.421835+0000","last_unstale":"2023-10-09T05:49:21.422120+0000","last_undegraded":"2023-10-09T05:49:21.422120+0000","last_fullsized":"2023-10-09T05:49:21.422120+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:56.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":81,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.418771+0000","last_change":"2023-10-09T05:48:17.175668+0000","last_active":"2023-10-09T05:49:20.418771+0000","last_peered":"2023-10-09T05:49:20.418771+0000","last_clean":"2023-10-09T05:49:20.418771+0000","last_became_active":"2023-10-09T05:48:17.175344+0000","last_became_peered":"2023-10-09T05:48:17.175344+0000","last_unstale":"2023-10-09T05:49:20.418771+0000","last_undegraded":"2023-10-09T05:49:20.418771+0000","last_fullsized":"2023-10-09T05:49:20.418771+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:43.949807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":42,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.419433+0000","last_change":"2023-10-09T05:40:14.363125+0000","last_active":"2023-10-09T05:49:20.419433+0000","last_peered":"2023-10-09T05:49:20.419433+0000","last_clean":"2023-10-09T05:49:20.419433+0000","last_became_active":"2023-10-09T05:40:14.362723+0000","last_became_peered":"2023-10-09T05:40:14.362723+0000","last_unstale":"2023-10-09T05:49:20.419433+0000","last_undegraded":"2023-10-09T05:49:20.419433+0000","last_fullsized":"2023-10-09T05:49:20.419433+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:30.741318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426544+0000","last_change":"2023-10-09T05:49:21.426544+0000","last_active":"2023-10-09T05:49:21.426544+0000","last_peered":"2023-10-09T05:49:21.426544+0000","last_clean":"2023-10-09T05:49:21.426544+0000","last_became_active":"2023-10-09T05:49:21.426253+0000","last_became_peered":"2023-10-09T05:49:21.426253+0000","last_unstale":"2023-10-09T05:49:21.426544+0000","last_undegraded":"2023-10-09T05:49:21.426544+0000","last_fullsized":"2023-10-09T05:49:21.426544+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:32:40.322649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":441,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.418674+0000","last_change":"2023-10-09T05:47:03.234961+0000","last_active":"2023-10-09T05:49:20.418674+0000","last_peered":"2023-10-09T05:49:20.418674+0000","last_clean":"2023-10-09T05:49:20.418674+0000","last_became_active":"2023-10-09T05:47:03.234618+0000","last_became_peered":"2023-10-09T05:47:03.234618+0000","last_unstale":"2023-10-09T05:49:20.418674+0000","last_undegraded":"2023-10-09T05:49:20.418674+0000","last_fullsized":"2023-10-09T05:49:20.418674+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:01.472111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.422069+0000","last_change":"2023-10-09T05:48:49.141207+0000","last_active":"2023-10-09T05:49:20.422069+0000","last_peered":"2023-10-09T05:49:20.422069+0000","last_clean":"2023-10-09T05:49:20.422069+0000","last_became_active":"2023-10-09T05:48:49.140798+0000","last_became_peered":"2023-10-09T05:48:49.140798+0000","last_unstale":"2023-10-09T05:49:20.422069+0000","last_undegraded":"2023-10-09T05:49:20.422069+0000","last_fullsized":"2023-10-09T05:49:20.422069+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:44:13.125030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.414450+0000","last_change":"2023-10-09T05:48:49.139757+0000","last_active":"2023-10-09T05:49:20.414450+0000","last_peered":"2023-10-09T05:49:20.414450+0000","last_clean":"2023-10-09T05:49:20.414450+0000","last_became_active":"2023-10-09T05:48:49.139311+0000","last_became_peered":"2023-10-09T05:48:49.139311+0000","last_unstale":"2023-10-09T05:49:20.414450+0000","last_undegraded":"2023-10-09T05:49:20.414450+0000","last_fullsized":"2023-10-09T05:49:20.414450+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:22.445381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.409547+0000","last_change":"2023-10-09T05:48:49.137529+0000","last_active":"2023-10-09T05:49:20.409547+0000","last_peered":"2023-10-09T05:49:20.409547+0000","last_clean":"2023-10-09T05:49:20.409547+0000","last_became_active":"2023-10-09T05:48:49.137212+0000","last_became_peered":"2023-10-09T05:48:49.137212+0000","last_unstale":"2023-10-09T05:49:20.409547+0000","last_undegraded":"2023-10-09T05:49:20.409547+0000","last_fullsized":"2023-10-09T05:49:20.409547+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:29.740386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":82,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.429787+0000","last_change":"2023-10-09T05:49:21.429787+0000","last_active":"2023-10-09T05:49:21.429787+0000","last_peered":"2023-10-09T05:49:21.429787+0000","last_clean":"2023-10-09T05:49:21.429787+0000","last_became_active":"2023-10-09T05:49:21.429304+0000","last_became_peered":"2023-10-09T05:49:21.429304+0000","last_unstale":"2023-10-09T05:49:21.429787+0000","last_undegraded":"2023-10-09T05:49:21.429787+0000","last_fullsized":"2023-10-09T05:49:21.429787+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:28:24.515861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":73,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.429250+0000","last_change":"2023-10-09T05:49:21.429250+0000","last_active":"2023-10-09T05:49:21.429250+0000","last_peered":"2023-10-09T05:49:21.429250+0000","last_clean":"2023-10-09T05:49:21.429250+0000","last_became_active":"2023-10-09T05:49:21.428946+0000","last_became_peered":"2023-10-09T05:49:21.428946+0000","last_unstale":"2023-10-09T05:49:21.429250+0000","last_undegraded":"2023-10-09T05:49:21.429250+0000","last_fullsized":"2023-10-09T05:49:21.429250+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:32.978473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.414007+0000","last_change":"2023-10-09T05:47:03.237144+0000","last_active":"2023-10-09T05:49:20.414007+0000","last_peered":"2023-10-09T05:49:20.414007+0000","last_clean":"2023-10-09T05:49:20.414007+0000","last_became_active":"2023-10-09T05:47:03.236793+0000","last_became_peered":"2023-10-09T05:47:03.236793+0000","last_unstale":"2023-10-09T05:49:20.414007+0000","last_undegraded":"2023-10-09T05:49:20.414007+0000","last_fullsized":"2023-10-09T05:49:20.414007+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:00:44.273589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.422361+0000","last_change":"2023-10-09T05:48:17.172780+0000","last_active":"2023-10-09T05:49:20.422361+0000","last_peered":"2023-10-09T05:49:20.422361+0000","last_clean":"2023-10-09T05:49:20.422361+0000","last_became_active":"2023-10-09T05:48:17.172460+0000","last_became_peered":"2023-10-09T05:48:17.172460+0000","last_unstale":"2023-10-09T05:49:20.422361+0000","last_undegraded":"2023-10-09T05:49:20.422361+0000","last_fullsized":"2023-10-09T05:49:20.422361+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:41.981302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":32,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.412397+0000","last_change":"2023-10-09T05:48:17.172596+0000","last_active":"2023-10-09T05:49:20.412397+0000","last_peered":"2023-10-09T05:49:20.412397+0000","last_clean":"2023-10-09T05:49:20.412397+0000","last_became_active":"2023-10-09T05:48:17.172287+0000","last_became_peered":"2023-10-09T05:48:17.172287+0000","last_unstale":"2023-10-09T05:49:20.412397+0000","last_undegraded":"2023-10-09T05:49:20.412397+0000","last_fullsized":"2023-10-09T05:49:20.412397+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:53.440906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":77,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.422980+0000","last_change":"2023-10-09T05:49:21.422980+0000","last_active":"2023-10-09T05:49:21.422980+0000","last_peered":"2023-10-09T05:49:21.422980+0000","last_clean":"2023-10-09T05:49:21.422980+0000","last_became_active":"2023-10-09T05:49:21.422349+0000","last_became_peered":"2023-10-09T05:49:21.422349+0000","last_unstale":"2023-10-09T05:49:21.422980+0000","last_undegraded":"2023-10-09T05:49:21.422980+0000","last_fullsized":"2023-10-09T05:49:21.422980+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:03.118182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":72,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.409864+0000","last_change":"2023-10-09T05:48:17.170845+0000","last_active":"2023-10-09T05:49:20.409864+0000","last_peered":"2023-10-09T05:49:20.409864+0000","last_clean":"2023-10-09T05:49:20.409864+0000","last_became_active":"2023-10-09T05:48:17.170321+0000","last_became_peered":"2023-10-09T05:48:17.170321+0000","last_unstale":"2023-10-09T05:49:20.409864+0000","last_undegraded":"2023-10-09T05:49:20.409864+0000","last_fullsized":"2023-10-09T05:49:20.409864+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:41.854157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":47,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.431778+0000","last_change":"2023-10-09T05:49:21.431778+0000","last_active":"2023-10-09T05:49:21.431778+0000","last_peered":"2023-10-09T05:49:21.431778+0000","last_clean":"2023-10-09T05:49:21.431778+0000","last_became_active":"2023-10-09T05:49:21.431483+0000","last_became_peered":"2023-10-09T05:49:21.431483+0000","last_unstale":"2023-10-09T05:49:21.431778+0000","last_undegraded":"2023-10-09T05:49:21.431778+0000","last_fullsized":"2023-10-09T05:49:21.431778+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:29:39.293436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":64,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.427586+0000","last_change":"2023-10-09T05:49:21.427586+0000","last_active":"2023-10-09T05:49:21.427586+0000","last_peered":"2023-10-09T05:49:21.427586+0000","last_clean":"2023-10-09T05:49:21.427586+0000","last_became_active":"2023-10-09T05:49:21.426908+0000","last_became_peered":"2023-10-09T05:49:21.426908+0000","last_unstale":"2023-10-09T05:49:21.427586+0000","last_undegraded":"2023-10-09T05:49:21.427586+0000","last_fullsized":"2023-10-09T05:49:21.427586+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:50.043430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.414177+0000","last_change":"2023-10-09T05:48:17.173636+0000","last_active":"2023-10-09T05:49:20.414177+0000","last_peered":"2023-10-09T05:49:20.414177+0000","last_clean":"2023-10-09T05:49:20.414177+0000","last_became_active":"2023-10-09T05:48:17.173329+0000","last_became_peered":"2023-10-09T05:48:17.173329+0000","last_unstale":"2023-10-09T05:49:20.414177+0000","last_undegraded":"2023-10-09T05:49:20.414177+0000","last_fullsized":"2023-10-09T05:49:20.414177+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:26:36.634285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.414234+0000","last_change":"2023-10-09T05:48:49.141287+0000","last_active":"2023-10-09T05:49:20.414234+0000","last_peered":"2023-10-09T05:49:20.414234+0000","last_clean":"2023-10-09T05:49:20.414234+0000","last_became_active":"2023-10-09T05:48:49.140950+0000","last_became_peered":"2023-10-09T05:48:49.140950+0000","last_unstale":"2023-10-09T05:49:20.414234+0000","last_undegraded":"2023-10-09T05:49:20.414234+0000","last_fullsized":"2023-10-09T05:49:20.414234+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:35.527434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":76,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.413852+0000","last_change":"2023-10-09T05:48:49.138043+0000","last_active":"2023-10-09T05:49:20.413852+0000","last_peered":"2023-10-09T05:49:20.413852+0000","last_clean":"2023-10-09T05:49:20.413852+0000","last_became_active":"2023-10-09T05:48:49.137722+0000","last_became_peered":"2023-10-09T05:48:49.137722+0000","last_unstale":"2023-10-09T05:49:20.413852+0000","last_undegraded":"2023-10-09T05:49:20.413852+0000","last_fullsized":"2023-10-09T05:49:20.413852+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:38.270179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.420336+0000","last_change":"2023-10-09T05:48:17.179214+0000","last_active":"2023-10-09T05:49:20.420336+0000","last_peered":"2023-10-09T05:49:20.420336+0000","last_clean":"2023-10-09T05:49:20.420336+0000","last_became_active":"2023-10-09T05:48:17.178918+0000","last_became_peered":"2023-10-09T05:48:17.178918+0000","last_unstale":"2023-10-09T05:49:20.420336+0000","last_undegraded":"2023-10-09T05:49:20.420336+0000","last_fullsized":"2023-10-09T05:49:20.420336+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:58:07.653506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":59,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.430742+0000","last_change":"2023-10-09T05:49:21.430742+0000","last_active":"2023-10-09T05:49:21.430742+0000","last_peered":"2023-10-09T05:49:21.430742+0000","last_clean":"2023-10-09T05:49:21.430742+0000","last_became_active":"2023-10-09T05:49:21.430233+0000","last_became_peered":"2023-10-09T05:49:21.430233+0000","last_unstale":"2023-10-09T05:49:21.430742+0000","last_undegraded":"2023-10-09T05:49:21.430742+0000","last_fullsized":"2023-10-09T05:49:21.430742+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:10.541302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":61,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.421396+0000","last_change":"2023-10-09T05:49:21.421396+0000","last_active":"2023-10-09T05:49:21.421396+0000","last_peered":"2023-10-09T05:49:21.421396+0000","last_clean":"2023-10-09T05:49:21.421396+0000","last_became_active":"2023-10-09T05:49:21.421129+0000","last_became_peered":"2023-10-09T05:49:21.421129+0000","last_unstale":"2023-10-09T05:49:21.421396+0000","last_undegraded":"2023-10-09T05:49:21.421396+0000","last_fullsized":"2023-10-09T05:49:21.421396+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:14.365625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.415371+0000","last_change":"2023-10-09T05:48:17.172849+0000","last_active":"2023-10-09T05:49:20.415371+0000","last_peered":"2023-10-09T05:49:20.415371+0000","last_clean":"2023-10-09T05:49:20.415371+0000","last_became_active":"2023-10-09T05:48:17.172536+0000","last_became_peered":"2023-10-09T05:48:17.172536+0000","last_unstale":"2023-10-09T05:49:20.415371+0000","last_undegraded":"2023-10-09T05:49:20.415371+0000","last_fullsized":"2023-10-09T05:49:20.415371+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:52.891228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":60,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.429234+0000","last_change":"2023-10-09T05:49:21.429234+0000","last_active":"2023-10-09T05:49:21.429234+0000","last_peered":"2023-10-09T05:49:21.429234+0000","last_clean":"2023-10-09T05:49:21.429234+0000","last_became_active":"2023-10-09T05:49:21.428961+0000","last_became_peered":"2023-10-09T05:49:21.428961+0000","last_unstale":"2023-10-09T05:49:21.429234+0000","last_undegraded":"2023-10-09T05:49:21.429234+0000","last_fullsized":"2023-10-09T05:49:21.429234+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:18.029914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.420226+0000","last_change":"2023-10-09T05:48:49.139877+0000","last_active":"2023-10-09T05:49:20.420226+0000","last_peered":"2023-10-09T05:49:20.420226+0000","last_clean":"2023-10-09T05:49:20.420226+0000","last_became_active":"2023-10-09T05:48:49.139479+0000","last_became_peered":"2023-10-09T05:48:49.139479+0000","last_unstale":"2023-10-09T05:49:20.420226+0000","last_undegraded":"2023-10-09T05:49:20.420226+0000","last_fullsized":"2023-10-09T05:49:20.420226+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:14:16.097335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":44,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.423108+0000","last_change":"2023-10-09T05:40:12.632083+0000","last_active":"2023-10-09T05:49:20.423108+0000","last_peered":"2023-10-09T05:49:20.423108+0000","last_clean":"2023-10-09T05:49:20.423108+0000","last_became_active":"2023-10-09T05:40:12.631715+0000","last_became_peered":"2023-10-09T05:40:12.631715+0000","last_unstale":"2023-10-09T05:49:20.423108+0000","last_undegraded":"2023-10-09T05:49:20.423108+0000","last_fullsized":"2023-10-09T05:49:20.423108+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:17.641378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":76,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426392+0000","last_change":"2023-10-09T05:49:21.426392+0000","last_active":"2023-10-09T05:49:21.426392+0000","last_peered":"2023-10-09T05:49:21.426392+0000","last_clean":"2023-10-09T05:49:21.426392+0000","last_became_active":"2023-10-09T05:49:21.426056+0000","last_became_peered":"2023-10-09T05:49:21.426056+0000","last_unstale":"2023-10-09T05:49:21.426392+0000","last_undegraded":"2023-10-09T05:49:21.426392+0000","last_fullsized":"2023-10-09T05:49:21.426392+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:09.862053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.420742+0000","last_change":"2023-10-09T05:48:17.173680+0000","last_active":"2023-10-09T05:49:20.420742+0000","last_peered":"2023-10-09T05:49:20.420742+0000","last_clean":"2023-10-09T05:49:20.420742+0000","last_became_active":"2023-10-09T05:48:17.173322+0000","last_became_peered":"2023-10-09T05:48:17.173322+0000","last_unstale":"2023-10-09T05:49:20.420742+0000","last_undegraded":"2023-10-09T05:49:20.420742+0000","last_fullsized":"2023-10-09T05:49:20.420742+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:41.005172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":42,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.421877+0000","last_change":"2023-10-09T05:40:14.362515+0000","last_active":"2023-10-09T05:49:20.421877+0000","last_peered":"2023-10-09T05:49:20.421877+0000","last_clean":"2023-10-09T05:49:20.421877+0000","last_became_active":"2023-10-09T05:40:14.362184+0000","last_became_peered":"2023-10-09T05:40:14.362184+0000","last_unstale":"2023-10-09T05:49:20.421877+0000","last_undegraded":"2023-10-09T05:49:20.421877+0000","last_fullsized":"2023-10-09T05:49:20.421877+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:37.681193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.423481+0000","last_change":"2023-10-09T05:47:03.230162+0000","last_active":"2023-10-09T05:49:20.423481+0000","last_peered":"2023-10-09T05:49:20.423481+0000","last_clean":"2023-10-09T05:49:20.423481+0000","last_became_active":"2023-10-09T05:47:03.229695+0000","last_became_peered":"2023-10-09T05:47:03.229695+0000","last_unstale":"2023-10-09T05:49:20.423481+0000","last_undegraded":"2023-10-09T05:49:20.423481+0000","last_fullsized":"2023-10-09T05:49:20.423481+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:13.194065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.427270+0000","last_change":"2023-10-09T05:49:21.427270+0000","last_active":"2023-10-09T05:49:21.427270+0000","last_peered":"2023-10-09T05:49:21.427270+0000","last_clean":"2023-10-09T05:49:21.427270+0000","last_became_active":"2023-10-09T05:49:21.427014+0000","last_became_peered":"2023-10-09T05:49:21.427014+0000","last_unstale":"2023-10-09T05:49:21.427270+0000","last_undegraded":"2023-10-09T05:49:21.427270+0000","last_fullsized":"2023-10-09T05:49:21.427270+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:36:30.822627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":68,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.416655+0000","last_change":"2023-10-09T05:47:03.230438+0000","last_active":"2023-10-09T05:49:20.416655+0000","last_peered":"2023-10-09T05:49:20.416655+0000","last_clean":"2023-10-09T05:49:20.416655+0000","last_became_active":"2023-10-09T05:47:03.229346+0000","last_became_peered":"2023-10-09T05:47:03.229346+0000","last_unstale":"2023-10-09T05:49:20.416655+0000","last_undegraded":"2023-10-09T05:49:20.416655+0000","last_fullsized":"2023-10-09T05:49:20.416655+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:54.948861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426926+0000","last_change":"2023-10-09T05:49:21.426431+0000","last_active":"2023-10-09T05:49:21.426926+0000","last_peered":"2023-10-09T05:49:21.426926+0000","last_clean":"2023-10-09T05:49:21.426926+0000","last_became_active":"2023-10-09T05:49:21.426011+0000","last_became_peered":"2023-10-09T05:49:21.426011+0000","last_unstale":"2023-10-09T05:49:21.426926+0000","last_undegraded":"2023-10-09T05:49:21.426926+0000","last_fullsized":"2023-10-09T05:49:21.426926+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:27.350168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":43,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.419867+0000","last_change":"2023-10-09T05:40:12.631893+0000","last_active":"2023-10-09T05:49:20.419867+0000","last_peered":"2023-10-09T05:49:20.419867+0000","last_clean":"2023-10-09T05:49:20.419867+0000","last_became_active":"2023-10-09T05:40:12.631592+0000","last_became_peered":"2023-10-09T05:40:12.631592+0000","last_unstale":"2023-10-09T05:49:20.419867+0000","last_undegraded":"2023-10-09T05:49:20.419867+0000","last_fullsized":"2023-10-09T05:49:20.419867+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:18:12.517403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":59,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.423218+0000","last_change":"2023-10-09T05:49:21.423218+0000","last_active":"2023-10-09T05:49:21.423218+0000","last_peered":"2023-10-09T05:49:21.423218+0000","last_clean":"2023-10-09T05:49:21.423218+0000","last_became_active":"2023-10-09T05:49:21.422706+0000","last_became_peered":"2023-10-09T05:49:21.422706+0000","last_unstale":"2023-10-09T05:49:21.423218+0000","last_undegraded":"2023-10-09T05:49:21.423218+0000","last_fullsized":"2023-10-09T05:49:21.423218+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:31.571220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426167+0000","last_change":"2023-10-09T05:49:21.426167+0000","last_active":"2023-10-09T05:49:21.426167+0000","last_peered":"2023-10-09T05:49:21.426167+0000","last_clean":"2023-10-09T05:49:21.426167+0000","last_became_active":"2023-10-09T05:49:21.425874+0000","last_became_peered":"2023-10-09T05:49:21.425874+0000","last_unstale":"2023-10-09T05:49:21.426167+0000","last_undegraded":"2023-10-09T05:49:21.426167+0000","last_fullsized":"2023-10-09T05:49:21.426167+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:59.642587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.428386+0000","last_change":"2023-10-09T05:49:21.428386+0000","last_active":"2023-10-09T05:49:21.428386+0000","last_peered":"2023-10-09T05:49:21.428386+0000","last_clean":"2023-10-09T05:49:21.428386+0000","last_became_active":"2023-10-09T05:49:21.428131+0000","last_became_peered":"2023-10-09T05:49:21.428131+0000","last_unstale":"2023-10-09T05:49:21.428386+0000","last_undegraded":"2023-10-09T05:49:21.428386+0000","last_fullsized":"2023-10-09T05:49:21.428386+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:12.858458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.419185+0000","last_change":"2023-10-09T05:48:49.139681+0000","last_active":"2023-10-09T05:49:20.419185+0000","last_peered":"2023-10-09T05:49:20.419185+0000","last_clean":"2023-10-09T05:49:20.419185+0000","last_became_active":"2023-10-09T05:48:49.139317+0000","last_became_peered":"2023-10-09T05:48:49.139317+0000","last_unstale":"2023-10-09T05:49:20.419185+0000","last_undegraded":"2023-10-09T05:49:20.419185+0000","last_fullsized":"2023-10-09T05:49:20.419185+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:34.769043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.413441+0000","last_change":"2023-10-09T05:48:17.173137+0000","last_active":"2023-10-09T05:49:20.413441+0000","last_peered":"2023-10-09T05:49:20.413441+0000","last_clean":"2023-10-09T05:49:20.413441+0000","last_became_active":"2023-10-09T05:48:17.172891+0000","last_became_peered":"2023-10-09T05:48:17.172891+0000","last_unstale":"2023-10-09T05:49:20.413441+0000","last_undegraded":"2023-10-09T05:49:20.413441+0000","last_fullsized":"2023-10-09T05:49:20.413441+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:21.393939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":65,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.413929+0000","last_change":"2023-10-09T05:40:12.630786+0000","last_active":"2023-10-09T05:49:20.413929+0000","last_peered":"2023-10-09T05:49:20.413929+0000","last_clean":"2023-10-09T05:49:20.413929+0000","last_became_active":"2023-10-09T05:40:12.630528+0000","last_became_peered":"2023-10-09T05:40:12.630528+0000","last_unstale":"2023-10-09T05:49:20.413929+0000","last_undegraded":"2023-10-09T05:49:20.413929+0000","last_fullsized":"2023-10-09T05:49:20.413929+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:54.427346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.412474+0000","last_change":"2023-10-09T05:48:49.138162+0000","last_active":"2023-10-09T05:49:20.412474+0000","last_peered":"2023-10-09T05:49:20.412474+0000","last_clean":"2023-10-09T05:49:20.412474+0000","last_became_active":"2023-10-09T05:48:49.137824+0000","last_became_peered":"2023-10-09T05:48:49.137824+0000","last_unstale":"2023-10-09T05:49:20.412474+0000","last_undegraded":"2023-10-09T05:49:20.412474+0000","last_fullsized":"2023-10-09T05:49:20.412474+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:43:26.642768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.430469+0000","last_change":"2023-10-09T05:49:21.430469+0000","last_active":"2023-10-09T05:49:21.430469+0000","last_peered":"2023-10-09T05:49:21.430469+0000","last_clean":"2023-10-09T05:49:21.430469+0000","last_became_active":"2023-10-09T05:49:21.430107+0000","last_became_peered":"2023-10-09T05:49:21.430107+0000","last_unstale":"2023-10-09T05:49:21.430469+0000","last_undegraded":"2023-10-09T05:49:21.430469+0000","last_fullsized":"2023-10-09T05:49:21.430469+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:08.815343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.425819+0000","last_change":"2023-10-09T05:49:21.425819+0000","last_active":"2023-10-09T05:49:21.425819+0000","last_peered":"2023-10-09T05:49:21.425819+0000","last_clean":"2023-10-09T05:49:21.425819+0000","last_became_active":"2023-10-09T05:49:21.425498+0000","last_became_peered":"2023-10-09T05:49:21.425498+0000","last_unstale":"2023-10-09T05:49:21.425819+0000","last_undegraded":"2023-10-09T05:49:21.425819+0000","last_fullsized":"2023-10-09T05:49:21.425819+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:21.443055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.421834+0000","last_change":"2023-10-09T05:48:17.176541+0000","last_active":"2023-10-09T05:49:20.421834+0000","last_peered":"2023-10-09T05:49:20.421834+0000","last_clean":"2023-10-09T05:49:20.421834+0000","last_became_active":"2023-10-09T05:48:17.176213+0000","last_became_peered":"2023-10-09T05:48:17.176213+0000","last_unstale":"2023-10-09T05:49:20.421834+0000","last_undegraded":"2023-10-09T05:49:20.421834+0000","last_fullsized":"2023-10-09T05:49:20.421834+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:23.599435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.427804+0000","last_change":"2023-10-09T05:49:21.427804+0000","last_active":"2023-10-09T05:49:21.427804+0000","last_peered":"2023-10-09T05:49:21.427804+0000","last_clean":"2023-10-09T05:49:21.427804+0000","last_became_active":"2023-10-09T05:49:21.427569+0000","last_became_peered":"2023-10-09T05:49:21.427569+0000","last_unstale":"2023-10-09T05:49:21.427804+0000","last_undegraded":"2023-10-09T05:49:21.427804+0000","last_fullsized":"2023-10-09T05:49:21.427804+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.791826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":53,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426616+0000","last_change":"2023-10-09T05:49:21.426149+0000","last_active":"2023-10-09T05:49:21.426616+0000","last_peered":"2023-10-09T05:49:21.426616+0000","last_clean":"2023-10-09T05:49:21.426616+0000","last_became_active":"2023-10-09T05:49:21.425805+0000","last_became_peered":"2023-10-09T05:49:21.425805+0000","last_unstale":"2023-10-09T05:49:21.426616+0000","last_undegraded":"2023-10-09T05:49:21.426616+0000","last_fullsized":"2023-10-09T05:49:21.426616+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:39:57.117762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.421197+0000","last_change":"2023-10-09T05:48:17.179524+0000","last_active":"2023-10-09T05:49:20.421197+0000","last_peered":"2023-10-09T05:49:20.421197+0000","last_clean":"2023-10-09T05:49:20.421197+0000","last_became_active":"2023-10-09T05:48:17.179233+0000","last_became_peered":"2023-10-09T05:48:17.179233+0000","last_unstale":"2023-10-09T05:49:20.421197+0000","last_undegraded":"2023-10-09T05:49:20.421197+0000","last_fullsized":"2023-10-09T05:49:20.421197+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:24.135263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":81,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.427745+0000","last_change":"2023-10-09T05:49:21.427745+0000","last_active":"2023-10-09T05:49:21.427745+0000","last_peered":"2023-10-09T05:49:21.427745+0000","last_clean":"2023-10-09T05:49:21.427745+0000","last_became_active":"2023-10-09T05:49:21.427292+0000","last_became_peered":"2023-10-09T05:49:21.427292+0000","last_unstale":"2023-10-09T05:49:21.427745+0000","last_undegraded":"2023-10-09T05:49:21.427745+0000","last_fullsized":"2023-10-09T05:49:21.427745+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:04:24.929346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.427476+0000","last_change":"2023-10-09T05:49:21.427476+0000","last_active":"2023-10-09T05:49:21.427476+0000","last_peered":"2023-10-09T05:49:21.427476+0000","last_clean":"2023-10-09T05:49:21.427476+0000","last_became_active":"2023-10-09T05:49:21.426417+0000","last_became_peered":"2023-10-09T05:49:21.426417+0000","last_unstale":"2023-10-09T05:49:21.427476+0000","last_undegraded":"2023-10-09T05:49:21.427476+0000","last_fullsized":"2023-10-09T05:49:21.427476+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:45:03.848962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.431711+0000","last_change":"2023-10-09T05:49:21.431711+0000","last_active":"2023-10-09T05:49:21.431711+0000","last_peered":"2023-10-09T05:49:21.431711+0000","last_clean":"2023-10-09T05:49:21.431711+0000","last_became_active":"2023-10-09T05:49:21.431441+0000","last_became_peered":"2023-10-09T05:49:21.431441+0000","last_unstale":"2023-10-09T05:49:21.431711+0000","last_undegraded":"2023-10-09T05:49:21.431711+0000","last_fullsized":"2023-10-09T05:49:21.431711+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:26.617516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.427978+0000","last_change":"2023-10-09T05:49:21.427978+0000","last_active":"2023-10-09T05:49:21.427978+0000","last_peered":"2023-10-09T05:49:21.427978+0000","last_clean":"2023-10-09T05:49:21.427978+0000","last_became_active":"2023-10-09T05:49:21.427703+0000","last_became_peered":"2023-10-09T05:49:21.427703+0000","last_unstale":"2023-10-09T05:49:21.427978+0000","last_undegraded":"2023-10-09T05:49:21.427978+0000","last_fullsized":"2023-10-09T05:49:21.427978+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:53:04.116683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":78,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.413362+0000","last_change":"2023-10-09T05:48:49.142429+0000","last_active":"2023-10-09T05:49:20.413362+0000","last_peered":"2023-10-09T05:49:20.413362+0000","last_clean":"2023-10-09T05:49:20.413362+0000","last_became_active":"2023-10-09T05:48:49.141966+0000","last_became_peered":"2023-10-09T05:48:49.141966+0000","last_unstale":"2023-10-09T05:49:20.413362+0000","last_undegraded":"2023-10-09T05:49:20.413362+0000","last_fullsized":"2023-10-09T05:49:20.413362+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:52.195431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234128,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311160,"kb_used_data":1584,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93433992,"statfs":{"total":95995035648,"available":95676407808,"internally_reserved":0,"allocated":1622016,"data_stored":1140338,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.69199999999999995,"15min":0.76100000000000001},"min":{"1min":0.38500000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.74299999999999999,"5min":4.8940000000000001,"15min":4.8940000000000001},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.57999999999999996,"15min":0.60199999999999998},"min":{"1min":0.34799999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.749,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.47199999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:49:17 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.51300000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:49:50 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.442},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.42499999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:48:53 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.57299999999999995,"15min":0.82899999999999996},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.77100000000000002,"5min":0.82199999999999995,"15min":95.108000000000004},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58199999999999996,"15min":0.83199999999999996},"min":{"1min":0.42999999999999999,"5min":0.36199999999999999,"15min":0.32800000000000001},"max":{"1min":0.749,"5min":0.94099999999999995,"15min":87.664000000000001},"last":0.53700000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:49:56 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.48399999999999999,"15min":0.48699999999999999},"min":{"1min":0.313,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":1.135},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.51200000000000001,"15min":0.498},"min":{"1min":0.378,"5min":0.27400000000000002,"15min":0.27100000000000002},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":3.165},"last":0.60899999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:49:56 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.499,"15min":0.499},"min":{"1min":0.33200000000000002,"5min":0.28199999999999997,"15min":0.248},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":2.5990000000000002},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.622,"5min":0.55700000000000005,"15min":0.54000000000000004},"min":{"1min":0.38800000000000001,"5min":0.34000000000000002,"15min":0.28100000000000003},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":1.0660000000000001},"last":0.63700000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:49:56 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.55900000000000005,"15min":0.54300000000000004},"min":{"1min":0.42399999999999999,"5min":0.317,"15min":0.29599999999999999},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":2.8170000000000002},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.57299999999999995,"15min":0.55100000000000005},"min":{"1min":0.41899999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":3.2589999999999999},"last":0.65200000000000002}]}]},{"osd":6,"up_from":46,"seq":197568495764,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298856,"kb_used_data":1568,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446296,"statfs":{"total":95995035648,"available":95689007104,"internally_reserved":0,"allocated":1605632,"data_stored":1139927,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.57999999999999996,"15min":0.59299999999999997},"min":{"1min":0.40300000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77000000000000002,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.57099999999999995,"15min":0.58899999999999997},"min":{"1min":0.38600000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.67800000000000005,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.63300000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:49:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.51100000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:49:50 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.58099999999999996}]},{"osd":3,"last update":"Mon Oct 9 05:48:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.59999999999999998,"15min":0.56200000000000006},"min":{"1min":0.46100000000000002,"5min":0.38300000000000001,"15min":0.318},"max":{"1min":0.79000000000000004,"5min":0.85199999999999998,"15min":1.1100000000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.57799999999999996,"15min":0.55600000000000005},"min":{"1min":0.40699999999999997,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.77400000000000002,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.745}]},{"osd":4,"last update":"Mon Oct 9 05:49:28 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.48499999999999999,"15min":0.45400000000000001},"min":{"1min":0.35499999999999998,"5min":0.309,"15min":0.26300000000000001},"max":{"1min":0.67300000000000004,"5min":0.71899999999999997,"15min":0.75600000000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.51700000000000002,"15min":0.48899999999999999},"min":{"1min":0.33400000000000002,"5min":0.29099999999999998,"15min":0.28599999999999998},"max":{"1min":0.75900000000000001,"5min":0.80500000000000005,"15min":0.82799999999999996},"last":0.60999999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:49:28 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.50900000000000001,"15min":0.47199999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.27700000000000002},"max":{"1min":0.77500000000000002,"5min":0.78200000000000003,"15min":0.81299999999999994},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.54500000000000004,"15min":0.51800000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32200000000000001},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85799999999999998},"last":0.72699999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:49:58 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.54700000000000004,"15min":0.53400000000000003},"min":{"1min":0.39800000000000002,"5min":0.316,"15min":0.316},"max":{"1min":0.64800000000000002,"5min":0.78800000000000003,"15min":0.79700000000000004},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.626,"5min":0.627,"15min":0.59799999999999998},"min":{"1min":0.39800000000000002,"5min":0.39600000000000002,"15min":0.35799999999999998},"max":{"1min":0.82299999999999995,"5min":0.88200000000000001,"15min":0.88300000000000001},"last":0.76600000000000001}]}]},{"osd":1,"up_from":89,"seq":382252089369,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28492,"kb_used_data":1028,"kb_used_omap":9,"kb_used_meta":27446,"kb_avail":93716660,"statfs":{"total":95995035648,"available":95965859840,"internally_reserved":0,"allocated":1052672,"data_stored":555584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":28105705},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:49:18 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"last":0.316},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.36099999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.5119999999999996,"5min":4.5119999999999996,"15min":4.5119999999999996},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.6139999999999999,"5min":3.6139999999999999,"15min":3.6139999999999999},"last":0.29599999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.374}]},{"osd":4,"last update":"Mon Oct 9 05:49:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.64900000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:49:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.59999999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:49:18 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.70199999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:49:18 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.67500000000000004}]}]},{"osd":0,"up_from":84,"seq":360777252904,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29160,"kb_used_data":1632,"kb_used_omap":28,"kb_used_meta":27491,"kb_avail":93715992,"statfs":{"total":95995035648,"available":95965175808,"internally_reserved":0,"allocated":1671168,"data_stored":1149588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":28151632},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.625,"5min":7.625,"15min":7.625},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":3.8130000000000002,"5min":3.8130000000000002,"15min":3.8130000000000002},"last":0.42899999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:49:49 2023","interfaces":[{"interface":"back","average":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.5640000000000001,"5min":7.5640000000000001,"15min":7.5640000000000001},"last":0.44600000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:49:04 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.502,"15min":0.51400000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.501},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.51000000000000001,"15min":0.52000000000000002},"min":{"1min":0.33600000000000002,"5min":0.307,"15min":0.307},"max":{"1min":0.63,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.57299999999999995}]},{"osd":4,"last update":"Mon Oct 9 05:50:04 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.67500000000000004,"15min":0.69499999999999995},"min":{"1min":0.42499999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.69799999999999995,"15min":0.72199999999999998},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.90100000000000002,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.68600000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:50:04 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.66800000000000004,"15min":0.68600000000000005},"min":{"1min":0.42199999999999999,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.622,"5min":0.68200000000000005,"15min":0.71599999999999997},"min":{"1min":0.498,"5min":0.376,"15min":0.376},"max":{"1min":1.27,"5min":1.27,"15min":1.27},"last":0.78100000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:50:04 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.68999999999999995,"15min":0.72499999999999998},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.71799999999999997,"15min":0.746},"min":{"1min":0.47499999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.995,"5min":0.995,"15min":0.995},"last":0.67900000000000005}]},{"osd":7,"last update":"Mon Oct 9 05:50:04 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.75700000000000001,"15min":0.78900000000000003},"min":{"1min":0.47799999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.016,"5min":1.016,"15min":1.016},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.73699999999999999,"15min":0.76500000000000001},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.79900000000000004}]}]},{"osd":2,"up_from":94,"seq":403726925843,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28620,"kb_used_data":1028,"kb_used_omap":81,"kb_used_meta":27502,"kb_avail":93716532,"statfs":{"total":95995035648,"available":95965728768,"internally_reserved":0,"allocated":1052672,"data_stored":554321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":28162266},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.39500000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.317},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.30399999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38200000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.68500000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.54900000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.55800000000000005}]},{"osd":7,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.64800000000000002}]}]},{"osd":3,"up_from":99,"seq":425201762317,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28468,"kb_used_data":1004,"kb_used_omap":7,"kb_used_meta":27448,"kb_avail":93716684,"statfs":{"total":95995035648,"available":95965884416,"internally_reserved":0,"allocated":1028096,"data_stored":549876,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":28107368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29099999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42299999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41099999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52800000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49099999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56799999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53800000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019037,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302388,"kb_used_data":1004,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93442764,"statfs":{"total":95995035648,"available":95685390336,"internally_reserved":0,"allocated":1028096,"data_stored":550223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:50:10 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.57699999999999996,"15min":0.59799999999999998},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.61499999999999999,"15min":0.63700000000000001},"min":{"1min":0.35799999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.502}]},{"osd":1,"last update":"Mon Oct 9 05:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.59599999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:49:50 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.433},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.032,"5min":1.032,"15min":1.032},"last":0.68500000000000005}]},{"osd":3,"last update":"Mon Oct 9 05:48:56 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.60299999999999998,"15min":0.58699999999999997},"min":{"1min":0.443,"5min":0.38,"15min":0.32600000000000001},"max":{"1min":0.76100000000000001,"5min":0.81200000000000006,"15min":0.995},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.59499999999999997,"15min":0.57899999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.35899999999999999},"max":{"1min":0.71199999999999997,"5min":0.83599999999999997,"15min":0.89800000000000002},"last":0.54800000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:50:10 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.47599999999999998,"15min":0.46000000000000002},"min":{"1min":0.36899999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.66300000000000003,"5min":0.83399999999999996,"15min":0.90500000000000003},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.53000000000000003,"15min":0.498},"min":{"1min":0.33100000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.78000000000000003,"5min":0.82199999999999995,"15min":0.88100000000000001},"last":0.61699999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:49:27 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.50900000000000001,"15min":0.495},"min":{"1min":0.34100000000000003,"5min":0.31,"15min":0.27400000000000002},"max":{"1min":0.64300000000000002,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.55700000000000005,"15min":0.53200000000000003},"min":{"1min":0.34799999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.72499999999999998,"5min":0.86199999999999999,"15min":0.89900000000000002},"last":0.51400000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:50:00 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.58399999999999996,"15min":0.56799999999999995},"min":{"1min":0.41699999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":1.046},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.58699999999999997,"15min":0.58399999999999996},"min":{"1min":0.48199999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.95799999999999996},"last":0.64600000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757400,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306468,"kb_used_data":988,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93438684,"statfs":{"total":95995035648,"available":95681212416,"internally_reserved":0,"allocated":1011712,"data_stored":545967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:50:07 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.57299999999999995,"15min":0.58999999999999997},"min":{"1min":0.35599999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.81100000000000005,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.59199999999999997,"15min":0.61499999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.68899999999999995,"5min":1,"15min":1},"last":0.61399999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:49:19 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.50700000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:49:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.59399999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:49:07 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.59599999999999997,"15min":1.46},"min":{"1min":0.373,"5min":0.373,"15min":0.36699999999999999},"max":{"1min":0.88700000000000001,"5min":0.98499999999999999,"15min":247.672},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.61399999999999999,"15min":1.4930000000000001},"min":{"1min":0.48399999999999999,"5min":0.36899999999999999,"15min":0.36799999999999999},"max":{"1min":0.82299999999999995,"5min":0.88600000000000001,"15min":247.90000000000001},"last":0.57999999999999996}]},{"osd":4,"last update":"Mon Oct 9 05:50:07 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.48799999999999999,"15min":0.45600000000000002},"min":{"1min":0.311,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.754,"5min":0.754,"15min":3.5430000000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.50800000000000001,"15min":0.47599999999999998},"min":{"1min":0.372,"5min":0.29999999999999999,"15min":0.28999999999999998},"max":{"1min":0.66600000000000004,"5min":0.72699999999999998,"15min":1.6990000000000001},"last":0.39800000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:49:24 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.53000000000000003,"15min":0.48799999999999999},"min":{"1min":0.41899999999999998,"5min":0.307,"15min":0.29899999999999999},"max":{"1min":0.69799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.622,"5min":0.57799999999999996,"15min":0.52200000000000002},"min":{"1min":0.39300000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.80600000000000005,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.623}]},{"osd":7,"last update":"Mon Oct 9 05:50:03 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.60799999999999998,"15min":0.56299999999999994},"min":{"1min":0.435,"5min":0.32700000000000001,"15min":0.315},"max":{"1min":0.81000000000000005,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.58799999999999997,"15min":0.59799999999999998},"min":{"1min":0.47399999999999998,"5min":0.35099999999999998,"15min":0.34899999999999998},"max":{"1min":0.79400000000000004,"5min":0.79700000000000004,"15min":3.2869999999999999},"last":0.63700000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:50:15.425 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T05:50:15.425 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T05:50:15.425 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T05:50:15.426 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T05:50:15.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:50:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:50:15] "GET /metrics HTTP/1.1" 200 33904 "" "Prometheus/2.43.0" 2023-10-09T05:50:15.790 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:16.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:16 smithi156 ceph-mon[195101]: pgmap v211: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:16.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:16 smithi156 ceph-mon[195101]: from='client.34593 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:50:16.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:16 smithi064 ceph-mon[220044]: pgmap v211: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:16.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:16 smithi064 ceph-mon[220044]: from='client.34593 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:50:16.446 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:16 smithi064 ceph-mon[212750]: pgmap v211: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:16.446 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:16 smithi064 ceph-mon[212750]: from='client.34593 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:50:17.715 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:50:17.715 INFO:teuthology.orchestra.run.smithi064.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T05:50:18.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:18 smithi156 ceph-mon[195101]: pgmap v212: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:18.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:18 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4178827194' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:50:18.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:18 smithi064 ceph-mon[212750]: pgmap v212: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:18.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:18 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4178827194' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:50:18.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:18 smithi064 ceph-mon[220044]: pgmap v212: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:18.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:18 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4178827194' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:50:18.453 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T05:50:18.453 INFO:teuthology.run_tasks:Running task print... 2023-10-09T05:50:18.463 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-10-09T05:50:18.463 INFO:teuthology.run_tasks:Running task parallel... 2023-10-09T05:50:18.494 INFO:teuthology.task.parallel:starting parallel... 2023-10-09T05:50:18.494 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T05:50:18.494 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-10-09T05:50:18.496 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-09T05:50:18.496 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'reef', 'sha1': 'a340fb5463aa3715e93e7472d5f60dfdcf184ab3'}} 2023-10-09T05:50:18.496 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-09T05:50:18.496 INFO:tasks.thrashosds:Beginning thrashosds... 2023-10-09T05:50:18.496 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T05:50:18.496 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:50:18.858 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:20.156 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:20 smithi064 ceph-mon[212750]: pgmap v213: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:20.156 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:20 smithi064 ceph-mon[220044]: pgmap v213: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:20 smithi156 ceph-mon[195101]: pgmap v213: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:20.583 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:50:20.584 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T05:50:21.327 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":214,"stamp":"2023-10-09T05:50:20.433630+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1333612,"kb_used_data":9836,"kb_used_omap":126,"kb_used_meta":1323585,"kb_avail":748627604,"statfs":{"total":767960285184,"available":766594666496,"internally_reserved":0,"allocated":10072064,"data_stored":6185824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1355351675},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.413173+0000","last_change":"2023-10-09T05:48:17.172491+0000","last_active":"2023-10-09T05:49:20.413173+0000","last_peered":"2023-10-09T05:49:20.413173+0000","last_clean":"2023-10-09T05:49:20.413173+0000","last_became_active":"2023-10-09T05:48:17.172154+0000","last_became_peered":"2023-10-09T05:48:17.172154+0000","last_unstale":"2023-10-09T05:49:20.413173+0000","last_undegraded":"2023-10-09T05:49:20.413173+0000","last_fullsized":"2023-10-09T05:49:20.413173+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:24:40.551774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":57,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.416576+0000","last_change":"2023-10-09T05:48:49.139798+0000","last_active":"2023-10-09T05:49:20.416576+0000","last_peered":"2023-10-09T05:49:20.416576+0000","last_clean":"2023-10-09T05:49:20.416576+0000","last_became_active":"2023-10-09T05:48:49.139442+0000","last_became_peered":"2023-10-09T05:48:49.139442+0000","last_unstale":"2023-10-09T05:49:20.416576+0000","last_undegraded":"2023-10-09T05:49:20.416576+0000","last_fullsized":"2023-10-09T05:49:20.416576+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:25.034620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":84,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.421384+0000","last_change":"2023-10-09T05:49:21.421384+0000","last_active":"2023-10-09T05:49:21.421384+0000","last_peered":"2023-10-09T05:49:21.421384+0000","last_clean":"2023-10-09T05:49:21.421384+0000","last_became_active":"2023-10-09T05:49:21.420886+0000","last_became_peered":"2023-10-09T05:49:21.420886+0000","last_unstale":"2023-10-09T05:49:21.421384+0000","last_undegraded":"2023-10-09T05:49:21.421384+0000","last_fullsized":"2023-10-09T05:49:21.421384+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:33:24.415387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":69,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.430192+0000","last_change":"2023-10-09T05:49:21.430192+0000","last_active":"2023-10-09T05:49:21.430192+0000","last_peered":"2023-10-09T05:49:21.430192+0000","last_clean":"2023-10-09T05:49:21.430192+0000","last_became_active":"2023-10-09T05:49:21.429857+0000","last_became_peered":"2023-10-09T05:49:21.429857+0000","last_unstale":"2023-10-09T05:49:21.430192+0000","last_undegraded":"2023-10-09T05:49:21.430192+0000","last_fullsized":"2023-10-09T05:49:21.430192+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:45.887757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":76,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426615+0000","last_change":"2023-10-09T05:49:21.426615+0000","last_active":"2023-10-09T05:49:21.426615+0000","last_peered":"2023-10-09T05:49:21.426615+0000","last_clean":"2023-10-09T05:49:21.426615+0000","last_became_active":"2023-10-09T05:49:21.426245+0000","last_became_peered":"2023-10-09T05:49:21.426245+0000","last_unstale":"2023-10-09T05:49:21.426615+0000","last_undegraded":"2023-10-09T05:49:21.426615+0000","last_fullsized":"2023-10-09T05:49:21.426615+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:47.642455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":45,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.425017+0000","last_change":"2023-10-09T05:40:12.633195+0000","last_active":"2023-10-09T05:49:20.425017+0000","last_peered":"2023-10-09T05:49:20.425017+0000","last_clean":"2023-10-09T05:49:20.425017+0000","last_became_active":"2023-10-09T05:40:12.632942+0000","last_became_peered":"2023-10-09T05:40:12.632942+0000","last_unstale":"2023-10-09T05:49:20.425017+0000","last_undegraded":"2023-10-09T05:49:20.425017+0000","last_fullsized":"2023-10-09T05:49:20.425017+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:04.470685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":76,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426304+0000","last_change":"2023-10-09T05:49:21.426304+0000","last_active":"2023-10-09T05:49:21.426304+0000","last_peered":"2023-10-09T05:49:21.426304+0000","last_clean":"2023-10-09T05:49:21.426304+0000","last_became_active":"2023-10-09T05:49:21.425949+0000","last_became_peered":"2023-10-09T05:49:21.425949+0000","last_unstale":"2023-10-09T05:49:21.426304+0000","last_undegraded":"2023-10-09T05:49:21.426304+0000","last_fullsized":"2023-10-09T05:49:21.426304+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:11.209520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.424610+0000","last_change":"2023-10-09T05:48:17.173906+0000","last_active":"2023-10-09T05:49:20.424610+0000","last_peered":"2023-10-09T05:49:20.424610+0000","last_clean":"2023-10-09T05:49:20.424610+0000","last_became_active":"2023-10-09T05:48:17.173564+0000","last_became_peered":"2023-10-09T05:48:17.173564+0000","last_unstale":"2023-10-09T05:49:20.424610+0000","last_undegraded":"2023-10-09T05:49:20.424610+0000","last_fullsized":"2023-10-09T05:49:20.424610+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:37.582247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":81,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.430604+0000","last_change":"2023-10-09T05:49:21.430604+0000","last_active":"2023-10-09T05:49:21.430604+0000","last_peered":"2023-10-09T05:49:21.430604+0000","last_clean":"2023-10-09T05:49:21.430604+0000","last_became_active":"2023-10-09T05:49:21.430220+0000","last_became_peered":"2023-10-09T05:49:21.430220+0000","last_unstale":"2023-10-09T05:49:21.430604+0000","last_undegraded":"2023-10-09T05:49:21.430604+0000","last_fullsized":"2023-10-09T05:49:21.430604+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:06:01.882964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":42,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.417852+0000","last_change":"2023-10-09T05:47:03.233346+0000","last_active":"2023-10-09T05:49:21.417852+0000","last_peered":"2023-10-09T05:49:21.417852+0000","last_clean":"2023-10-09T05:49:21.417852+0000","last_became_active":"2023-10-09T05:47:03.231300+0000","last_became_peered":"2023-10-09T05:47:03.231300+0000","last_unstale":"2023-10-09T05:49:21.417852+0000","last_undegraded":"2023-10-09T05:49:21.417852+0000","last_fullsized":"2023-10-09T05:49:21.417852+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:22.050814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":42,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.414600+0000","last_change":"2023-10-09T05:40:12.676587+0000","last_active":"2023-10-09T05:49:20.414600+0000","last_peered":"2023-10-09T05:49:20.414600+0000","last_clean":"2023-10-09T05:49:20.414600+0000","last_became_active":"2023-10-09T05:40:12.676330+0000","last_became_peered":"2023-10-09T05:40:12.676330+0000","last_unstale":"2023-10-09T05:49:20.414600+0000","last_undegraded":"2023-10-09T05:49:20.414600+0000","last_fullsized":"2023-10-09T05:49:20.414600+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:46.659556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":57,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.416515+0000","last_change":"2023-10-09T05:48:17.174996+0000","last_active":"2023-10-09T05:49:20.416515+0000","last_peered":"2023-10-09T05:49:20.416515+0000","last_clean":"2023-10-09T05:49:20.416515+0000","last_became_active":"2023-10-09T05:48:17.174668+0000","last_became_peered":"2023-10-09T05:48:17.174668+0000","last_unstale":"2023-10-09T05:49:20.416515+0000","last_undegraded":"2023-10-09T05:49:20.416515+0000","last_fullsized":"2023-10-09T05:49:20.416515+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:49.712686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.421123+0000","last_change":"2023-10-09T05:48:49.137565+0000","last_active":"2023-10-09T05:49:20.421123+0000","last_peered":"2023-10-09T05:49:20.421123+0000","last_clean":"2023-10-09T05:49:20.421123+0000","last_became_active":"2023-10-09T05:48:49.137099+0000","last_became_peered":"2023-10-09T05:48:49.137099+0000","last_unstale":"2023-10-09T05:49:20.421123+0000","last_undegraded":"2023-10-09T05:49:20.421123+0000","last_fullsized":"2023-10-09T05:49:20.421123+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:28.103821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":58,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.423248+0000","last_change":"2023-10-09T05:49:21.423248+0000","last_active":"2023-10-09T05:49:21.423248+0000","last_peered":"2023-10-09T05:49:21.423248+0000","last_clean":"2023-10-09T05:49:21.423248+0000","last_became_active":"2023-10-09T05:49:21.422985+0000","last_became_peered":"2023-10-09T05:49:21.422985+0000","last_unstale":"2023-10-09T05:49:21.423248+0000","last_undegraded":"2023-10-09T05:49:21.423248+0000","last_fullsized":"2023-10-09T05:49:21.423248+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:02.431791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.409445+0000","last_change":"2023-10-09T05:48:17.174101+0000","last_active":"2023-10-09T05:49:20.409445+0000","last_peered":"2023-10-09T05:49:20.409445+0000","last_clean":"2023-10-09T05:49:20.409445+0000","last_became_active":"2023-10-09T05:48:17.173752+0000","last_became_peered":"2023-10-09T05:48:17.173752+0000","last_unstale":"2023-10-09T05:49:20.409445+0000","last_undegraded":"2023-10-09T05:49:20.409445+0000","last_fullsized":"2023-10-09T05:49:20.409445+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:45:16.222083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":61,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.423414+0000","last_change":"2023-10-09T05:49:21.423414+0000","last_active":"2023-10-09T05:49:21.423414+0000","last_peered":"2023-10-09T05:49:21.423414+0000","last_clean":"2023-10-09T05:49:21.423414+0000","last_became_active":"2023-10-09T05:49:21.423085+0000","last_became_peered":"2023-10-09T05:49:21.423085+0000","last_unstale":"2023-10-09T05:49:21.423414+0000","last_undegraded":"2023-10-09T05:49:21.423414+0000","last_fullsized":"2023-10-09T05:49:21.423414+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:51.777782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.414717+0000","last_change":"2023-10-09T05:47:03.233460+0000","last_active":"2023-10-09T05:49:20.414717+0000","last_peered":"2023-10-09T05:49:20.414717+0000","last_clean":"2023-10-09T05:49:20.414717+0000","last_became_active":"2023-10-09T05:47:03.231685+0000","last_became_peered":"2023-10-09T05:47:03.231685+0000","last_unstale":"2023-10-09T05:49:20.414717+0000","last_undegraded":"2023-10-09T05:49:20.414717+0000","last_fullsized":"2023-10-09T05:49:20.414717+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:10.790717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.423612+0000","last_change":"2023-10-09T05:48:49.139759+0000","last_active":"2023-10-09T05:49:21.423612+0000","last_peered":"2023-10-09T05:49:21.423612+0000","last_clean":"2023-10-09T05:49:21.423612+0000","last_became_active":"2023-10-09T05:48:49.139346+0000","last_became_peered":"2023-10-09T05:48:49.139346+0000","last_unstale":"2023-10-09T05:49:21.423612+0000","last_undegraded":"2023-10-09T05:49:21.423612+0000","last_fullsized":"2023-10-09T05:49:21.423612+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:51.980711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.420851+0000","last_change":"2023-10-09T05:47:03.230117+0000","last_active":"2023-10-09T05:49:20.420851+0000","last_peered":"2023-10-09T05:49:20.420851+0000","last_clean":"2023-10-09T05:49:20.420851+0000","last_became_active":"2023-10-09T05:47:03.229660+0000","last_became_peered":"2023-10-09T05:47:03.229660+0000","last_unstale":"2023-10-09T05:49:20.420851+0000","last_undegraded":"2023-10-09T05:49:20.420851+0000","last_fullsized":"2023-10-09T05:49:20.420851+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:07:21.669282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.416527+0000","last_change":"2023-10-09T05:48:49.134851+0000","last_active":"2023-10-09T05:49:20.416527+0000","last_peered":"2023-10-09T05:49:20.416527+0000","last_clean":"2023-10-09T05:49:20.416527+0000","last_became_active":"2023-10-09T05:48:49.134532+0000","last_became_peered":"2023-10-09T05:48:49.134532+0000","last_unstale":"2023-10-09T05:49:20.416527+0000","last_undegraded":"2023-10-09T05:49:20.416527+0000","last_fullsized":"2023-10-09T05:49:20.416527+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:23.395907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.422120+0000","last_change":"2023-10-09T05:49:21.422120+0000","last_active":"2023-10-09T05:49:21.422120+0000","last_peered":"2023-10-09T05:49:21.422120+0000","last_clean":"2023-10-09T05:49:21.422120+0000","last_became_active":"2023-10-09T05:49:21.421835+0000","last_became_peered":"2023-10-09T05:49:21.421835+0000","last_unstale":"2023-10-09T05:49:21.422120+0000","last_undegraded":"2023-10-09T05:49:21.422120+0000","last_fullsized":"2023-10-09T05:49:21.422120+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:56.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":81,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.418771+0000","last_change":"2023-10-09T05:48:17.175668+0000","last_active":"2023-10-09T05:49:20.418771+0000","last_peered":"2023-10-09T05:49:20.418771+0000","last_clean":"2023-10-09T05:49:20.418771+0000","last_became_active":"2023-10-09T05:48:17.175344+0000","last_became_peered":"2023-10-09T05:48:17.175344+0000","last_unstale":"2023-10-09T05:49:20.418771+0000","last_undegraded":"2023-10-09T05:49:20.418771+0000","last_fullsized":"2023-10-09T05:49:20.418771+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:43.949807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":42,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.419433+0000","last_change":"2023-10-09T05:40:14.363125+0000","last_active":"2023-10-09T05:49:20.419433+0000","last_peered":"2023-10-09T05:49:20.419433+0000","last_clean":"2023-10-09T05:49:20.419433+0000","last_became_active":"2023-10-09T05:40:14.362723+0000","last_became_peered":"2023-10-09T05:40:14.362723+0000","last_unstale":"2023-10-09T05:49:20.419433+0000","last_undegraded":"2023-10-09T05:49:20.419433+0000","last_fullsized":"2023-10-09T05:49:20.419433+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:30.741318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426544+0000","last_change":"2023-10-09T05:49:21.426544+0000","last_active":"2023-10-09T05:49:21.426544+0000","last_peered":"2023-10-09T05:49:21.426544+0000","last_clean":"2023-10-09T05:49:21.426544+0000","last_became_active":"2023-10-09T05:49:21.426253+0000","last_became_peered":"2023-10-09T05:49:21.426253+0000","last_unstale":"2023-10-09T05:49:21.426544+0000","last_undegraded":"2023-10-09T05:49:21.426544+0000","last_fullsized":"2023-10-09T05:49:21.426544+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:32:40.322649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":441,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.418674+0000","last_change":"2023-10-09T05:47:03.234961+0000","last_active":"2023-10-09T05:49:20.418674+0000","last_peered":"2023-10-09T05:49:20.418674+0000","last_clean":"2023-10-09T05:49:20.418674+0000","last_became_active":"2023-10-09T05:47:03.234618+0000","last_became_peered":"2023-10-09T05:47:03.234618+0000","last_unstale":"2023-10-09T05:49:20.418674+0000","last_undegraded":"2023-10-09T05:49:20.418674+0000","last_fullsized":"2023-10-09T05:49:20.418674+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:01.472111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.422069+0000","last_change":"2023-10-09T05:48:49.141207+0000","last_active":"2023-10-09T05:49:20.422069+0000","last_peered":"2023-10-09T05:49:20.422069+0000","last_clean":"2023-10-09T05:49:20.422069+0000","last_became_active":"2023-10-09T05:48:49.140798+0000","last_became_peered":"2023-10-09T05:48:49.140798+0000","last_unstale":"2023-10-09T05:49:20.422069+0000","last_undegraded":"2023-10-09T05:49:20.422069+0000","last_fullsized":"2023-10-09T05:49:20.422069+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:44:13.125030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.414450+0000","last_change":"2023-10-09T05:48:49.139757+0000","last_active":"2023-10-09T05:49:20.414450+0000","last_peered":"2023-10-09T05:49:20.414450+0000","last_clean":"2023-10-09T05:49:20.414450+0000","last_became_active":"2023-10-09T05:48:49.139311+0000","last_became_peered":"2023-10-09T05:48:49.139311+0000","last_unstale":"2023-10-09T05:49:20.414450+0000","last_undegraded":"2023-10-09T05:49:20.414450+0000","last_fullsized":"2023-10-09T05:49:20.414450+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:22.445381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.422046+0000","last_change":"2023-10-09T05:48:49.137529+0000","last_active":"2023-10-09T05:49:21.422046+0000","last_peered":"2023-10-09T05:49:21.422046+0000","last_clean":"2023-10-09T05:49:21.422046+0000","last_became_active":"2023-10-09T05:48:49.137212+0000","last_became_peered":"2023-10-09T05:48:49.137212+0000","last_unstale":"2023-10-09T05:49:21.422046+0000","last_undegraded":"2023-10-09T05:49:21.422046+0000","last_fullsized":"2023-10-09T05:49:21.422046+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:29.740386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":82,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.429787+0000","last_change":"2023-10-09T05:49:21.429787+0000","last_active":"2023-10-09T05:49:21.429787+0000","last_peered":"2023-10-09T05:49:21.429787+0000","last_clean":"2023-10-09T05:49:21.429787+0000","last_became_active":"2023-10-09T05:49:21.429304+0000","last_became_peered":"2023-10-09T05:49:21.429304+0000","last_unstale":"2023-10-09T05:49:21.429787+0000","last_undegraded":"2023-10-09T05:49:21.429787+0000","last_fullsized":"2023-10-09T05:49:21.429787+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:28:24.515861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":73,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.429250+0000","last_change":"2023-10-09T05:49:21.429250+0000","last_active":"2023-10-09T05:49:21.429250+0000","last_peered":"2023-10-09T05:49:21.429250+0000","last_clean":"2023-10-09T05:49:21.429250+0000","last_became_active":"2023-10-09T05:49:21.428946+0000","last_became_peered":"2023-10-09T05:49:21.428946+0000","last_unstale":"2023-10-09T05:49:21.429250+0000","last_undegraded":"2023-10-09T05:49:21.429250+0000","last_fullsized":"2023-10-09T05:49:21.429250+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:32.978473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.414007+0000","last_change":"2023-10-09T05:47:03.237144+0000","last_active":"2023-10-09T05:49:20.414007+0000","last_peered":"2023-10-09T05:49:20.414007+0000","last_clean":"2023-10-09T05:49:20.414007+0000","last_became_active":"2023-10-09T05:47:03.236793+0000","last_became_peered":"2023-10-09T05:47:03.236793+0000","last_unstale":"2023-10-09T05:49:20.414007+0000","last_undegraded":"2023-10-09T05:49:20.414007+0000","last_fullsized":"2023-10-09T05:49:20.414007+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:00:44.273589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.422361+0000","last_change":"2023-10-09T05:48:17.172780+0000","last_active":"2023-10-09T05:49:20.422361+0000","last_peered":"2023-10-09T05:49:20.422361+0000","last_clean":"2023-10-09T05:49:20.422361+0000","last_became_active":"2023-10-09T05:48:17.172460+0000","last_became_peered":"2023-10-09T05:48:17.172460+0000","last_unstale":"2023-10-09T05:49:20.422361+0000","last_undegraded":"2023-10-09T05:49:20.422361+0000","last_fullsized":"2023-10-09T05:49:20.422361+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:41.981302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.421018+0000","last_change":"2023-10-09T05:48:17.172596+0000","last_active":"2023-10-09T05:49:21.421018+0000","last_peered":"2023-10-09T05:49:21.421018+0000","last_clean":"2023-10-09T05:49:21.421018+0000","last_became_active":"2023-10-09T05:48:17.172287+0000","last_became_peered":"2023-10-09T05:48:17.172287+0000","last_unstale":"2023-10-09T05:49:21.421018+0000","last_undegraded":"2023-10-09T05:49:21.421018+0000","last_fullsized":"2023-10-09T05:49:21.421018+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:53.440906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":77,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.422980+0000","last_change":"2023-10-09T05:49:21.422980+0000","last_active":"2023-10-09T05:49:21.422980+0000","last_peered":"2023-10-09T05:49:21.422980+0000","last_clean":"2023-10-09T05:49:21.422980+0000","last_became_active":"2023-10-09T05:49:21.422349+0000","last_became_peered":"2023-10-09T05:49:21.422349+0000","last_unstale":"2023-10-09T05:49:21.422980+0000","last_undegraded":"2023-10-09T05:49:21.422980+0000","last_fullsized":"2023-10-09T05:49:21.422980+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:03.118182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":73,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.423855+0000","last_change":"2023-10-09T05:48:17.170845+0000","last_active":"2023-10-09T05:49:21.423855+0000","last_peered":"2023-10-09T05:49:21.423855+0000","last_clean":"2023-10-09T05:49:21.423855+0000","last_became_active":"2023-10-09T05:48:17.170321+0000","last_became_peered":"2023-10-09T05:48:17.170321+0000","last_unstale":"2023-10-09T05:49:21.423855+0000","last_undegraded":"2023-10-09T05:49:21.423855+0000","last_fullsized":"2023-10-09T05:49:21.423855+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:41.854157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":47,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.431778+0000","last_change":"2023-10-09T05:49:21.431778+0000","last_active":"2023-10-09T05:49:21.431778+0000","last_peered":"2023-10-09T05:49:21.431778+0000","last_clean":"2023-10-09T05:49:21.431778+0000","last_became_active":"2023-10-09T05:49:21.431483+0000","last_became_peered":"2023-10-09T05:49:21.431483+0000","last_unstale":"2023-10-09T05:49:21.431778+0000","last_undegraded":"2023-10-09T05:49:21.431778+0000","last_fullsized":"2023-10-09T05:49:21.431778+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:29:39.293436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":64,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.427586+0000","last_change":"2023-10-09T05:49:21.427586+0000","last_active":"2023-10-09T05:49:21.427586+0000","last_peered":"2023-10-09T05:49:21.427586+0000","last_clean":"2023-10-09T05:49:21.427586+0000","last_became_active":"2023-10-09T05:49:21.426908+0000","last_became_peered":"2023-10-09T05:49:21.426908+0000","last_unstale":"2023-10-09T05:49:21.427586+0000","last_undegraded":"2023-10-09T05:49:21.427586+0000","last_fullsized":"2023-10-09T05:49:21.427586+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:50.043430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.414177+0000","last_change":"2023-10-09T05:48:17.173636+0000","last_active":"2023-10-09T05:49:20.414177+0000","last_peered":"2023-10-09T05:49:20.414177+0000","last_clean":"2023-10-09T05:49:20.414177+0000","last_became_active":"2023-10-09T05:48:17.173329+0000","last_became_peered":"2023-10-09T05:48:17.173329+0000","last_unstale":"2023-10-09T05:49:20.414177+0000","last_undegraded":"2023-10-09T05:49:20.414177+0000","last_fullsized":"2023-10-09T05:49:20.414177+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:26:36.634285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.414234+0000","last_change":"2023-10-09T05:48:49.141287+0000","last_active":"2023-10-09T05:49:20.414234+0000","last_peered":"2023-10-09T05:49:20.414234+0000","last_clean":"2023-10-09T05:49:20.414234+0000","last_became_active":"2023-10-09T05:48:49.140950+0000","last_became_peered":"2023-10-09T05:48:49.140950+0000","last_unstale":"2023-10-09T05:49:20.414234+0000","last_undegraded":"2023-10-09T05:49:20.414234+0000","last_fullsized":"2023-10-09T05:49:20.414234+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:35.527434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":76,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.413852+0000","last_change":"2023-10-09T05:48:49.138043+0000","last_active":"2023-10-09T05:49:20.413852+0000","last_peered":"2023-10-09T05:49:20.413852+0000","last_clean":"2023-10-09T05:49:20.413852+0000","last_became_active":"2023-10-09T05:48:49.137722+0000","last_became_peered":"2023-10-09T05:48:49.137722+0000","last_unstale":"2023-10-09T05:49:20.413852+0000","last_undegraded":"2023-10-09T05:49:20.413852+0000","last_fullsized":"2023-10-09T05:49:20.413852+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:38.270179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.420336+0000","last_change":"2023-10-09T05:48:17.179214+0000","last_active":"2023-10-09T05:49:20.420336+0000","last_peered":"2023-10-09T05:49:20.420336+0000","last_clean":"2023-10-09T05:49:20.420336+0000","last_became_active":"2023-10-09T05:48:17.178918+0000","last_became_peered":"2023-10-09T05:48:17.178918+0000","last_unstale":"2023-10-09T05:49:20.420336+0000","last_undegraded":"2023-10-09T05:49:20.420336+0000","last_fullsized":"2023-10-09T05:49:20.420336+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:58:07.653506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":59,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.430742+0000","last_change":"2023-10-09T05:49:21.430742+0000","last_active":"2023-10-09T05:49:21.430742+0000","last_peered":"2023-10-09T05:49:21.430742+0000","last_clean":"2023-10-09T05:49:21.430742+0000","last_became_active":"2023-10-09T05:49:21.430233+0000","last_became_peered":"2023-10-09T05:49:21.430233+0000","last_unstale":"2023-10-09T05:49:21.430742+0000","last_undegraded":"2023-10-09T05:49:21.430742+0000","last_fullsized":"2023-10-09T05:49:21.430742+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:10.541302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":61,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.421396+0000","last_change":"2023-10-09T05:49:21.421396+0000","last_active":"2023-10-09T05:49:21.421396+0000","last_peered":"2023-10-09T05:49:21.421396+0000","last_clean":"2023-10-09T05:49:21.421396+0000","last_became_active":"2023-10-09T05:49:21.421129+0000","last_became_peered":"2023-10-09T05:49:21.421129+0000","last_unstale":"2023-10-09T05:49:21.421396+0000","last_undegraded":"2023-10-09T05:49:21.421396+0000","last_fullsized":"2023-10-09T05:49:21.421396+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:14.365625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.415371+0000","last_change":"2023-10-09T05:48:17.172849+0000","last_active":"2023-10-09T05:49:20.415371+0000","last_peered":"2023-10-09T05:49:20.415371+0000","last_clean":"2023-10-09T05:49:20.415371+0000","last_became_active":"2023-10-09T05:48:17.172536+0000","last_became_peered":"2023-10-09T05:48:17.172536+0000","last_unstale":"2023-10-09T05:49:20.415371+0000","last_undegraded":"2023-10-09T05:49:20.415371+0000","last_fullsized":"2023-10-09T05:49:20.415371+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:52.891228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":60,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.429234+0000","last_change":"2023-10-09T05:49:21.429234+0000","last_active":"2023-10-09T05:49:21.429234+0000","last_peered":"2023-10-09T05:49:21.429234+0000","last_clean":"2023-10-09T05:49:21.429234+0000","last_became_active":"2023-10-09T05:49:21.428961+0000","last_became_peered":"2023-10-09T05:49:21.428961+0000","last_unstale":"2023-10-09T05:49:21.429234+0000","last_undegraded":"2023-10-09T05:49:21.429234+0000","last_fullsized":"2023-10-09T05:49:21.429234+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:18.029914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.420226+0000","last_change":"2023-10-09T05:48:49.139877+0000","last_active":"2023-10-09T05:49:20.420226+0000","last_peered":"2023-10-09T05:49:20.420226+0000","last_clean":"2023-10-09T05:49:20.420226+0000","last_became_active":"2023-10-09T05:48:49.139479+0000","last_became_peered":"2023-10-09T05:48:49.139479+0000","last_unstale":"2023-10-09T05:49:20.420226+0000","last_undegraded":"2023-10-09T05:49:20.420226+0000","last_fullsized":"2023-10-09T05:49:20.420226+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:14:16.097335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":44,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.423108+0000","last_change":"2023-10-09T05:40:12.632083+0000","last_active":"2023-10-09T05:49:20.423108+0000","last_peered":"2023-10-09T05:49:20.423108+0000","last_clean":"2023-10-09T05:49:20.423108+0000","last_became_active":"2023-10-09T05:40:12.631715+0000","last_became_peered":"2023-10-09T05:40:12.631715+0000","last_unstale":"2023-10-09T05:49:20.423108+0000","last_undegraded":"2023-10-09T05:49:20.423108+0000","last_fullsized":"2023-10-09T05:49:20.423108+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:17.641378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":76,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426392+0000","last_change":"2023-10-09T05:49:21.426392+0000","last_active":"2023-10-09T05:49:21.426392+0000","last_peered":"2023-10-09T05:49:21.426392+0000","last_clean":"2023-10-09T05:49:21.426392+0000","last_became_active":"2023-10-09T05:49:21.426056+0000","last_became_peered":"2023-10-09T05:49:21.426056+0000","last_unstale":"2023-10-09T05:49:21.426392+0000","last_undegraded":"2023-10-09T05:49:21.426392+0000","last_fullsized":"2023-10-09T05:49:21.426392+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:09.862053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.420742+0000","last_change":"2023-10-09T05:48:17.173680+0000","last_active":"2023-10-09T05:49:20.420742+0000","last_peered":"2023-10-09T05:49:20.420742+0000","last_clean":"2023-10-09T05:49:20.420742+0000","last_became_active":"2023-10-09T05:48:17.173322+0000","last_became_peered":"2023-10-09T05:48:17.173322+0000","last_unstale":"2023-10-09T05:49:20.420742+0000","last_undegraded":"2023-10-09T05:49:20.420742+0000","last_fullsized":"2023-10-09T05:49:20.420742+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:41.005172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":42,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.421877+0000","last_change":"2023-10-09T05:40:14.362515+0000","last_active":"2023-10-09T05:49:20.421877+0000","last_peered":"2023-10-09T05:49:20.421877+0000","last_clean":"2023-10-09T05:49:20.421877+0000","last_became_active":"2023-10-09T05:40:14.362184+0000","last_became_peered":"2023-10-09T05:40:14.362184+0000","last_unstale":"2023-10-09T05:49:20.421877+0000","last_undegraded":"2023-10-09T05:49:20.421877+0000","last_fullsized":"2023-10-09T05:49:20.421877+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:37.681193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.423481+0000","last_change":"2023-10-09T05:47:03.230162+0000","last_active":"2023-10-09T05:49:20.423481+0000","last_peered":"2023-10-09T05:49:20.423481+0000","last_clean":"2023-10-09T05:49:20.423481+0000","last_became_active":"2023-10-09T05:47:03.229695+0000","last_became_peered":"2023-10-09T05:47:03.229695+0000","last_unstale":"2023-10-09T05:49:20.423481+0000","last_undegraded":"2023-10-09T05:49:20.423481+0000","last_fullsized":"2023-10-09T05:49:20.423481+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:13.194065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.427270+0000","last_change":"2023-10-09T05:49:21.427270+0000","last_active":"2023-10-09T05:49:21.427270+0000","last_peered":"2023-10-09T05:49:21.427270+0000","last_clean":"2023-10-09T05:49:21.427270+0000","last_became_active":"2023-10-09T05:49:21.427014+0000","last_became_peered":"2023-10-09T05:49:21.427014+0000","last_unstale":"2023-10-09T05:49:21.427270+0000","last_undegraded":"2023-10-09T05:49:21.427270+0000","last_fullsized":"2023-10-09T05:49:21.427270+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:36:30.822627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":68,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.416655+0000","last_change":"2023-10-09T05:47:03.230438+0000","last_active":"2023-10-09T05:49:20.416655+0000","last_peered":"2023-10-09T05:49:20.416655+0000","last_clean":"2023-10-09T05:49:20.416655+0000","last_became_active":"2023-10-09T05:47:03.229346+0000","last_became_peered":"2023-10-09T05:47:03.229346+0000","last_unstale":"2023-10-09T05:49:20.416655+0000","last_undegraded":"2023-10-09T05:49:20.416655+0000","last_fullsized":"2023-10-09T05:49:20.416655+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:54.948861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426926+0000","last_change":"2023-10-09T05:49:21.426431+0000","last_active":"2023-10-09T05:49:21.426926+0000","last_peered":"2023-10-09T05:49:21.426926+0000","last_clean":"2023-10-09T05:49:21.426926+0000","last_became_active":"2023-10-09T05:49:21.426011+0000","last_became_peered":"2023-10-09T05:49:21.426011+0000","last_unstale":"2023-10-09T05:49:21.426926+0000","last_undegraded":"2023-10-09T05:49:21.426926+0000","last_fullsized":"2023-10-09T05:49:21.426926+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:27.350168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":43,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.419867+0000","last_change":"2023-10-09T05:40:12.631893+0000","last_active":"2023-10-09T05:49:20.419867+0000","last_peered":"2023-10-09T05:49:20.419867+0000","last_clean":"2023-10-09T05:49:20.419867+0000","last_became_active":"2023-10-09T05:40:12.631592+0000","last_became_peered":"2023-10-09T05:40:12.631592+0000","last_unstale":"2023-10-09T05:49:20.419867+0000","last_undegraded":"2023-10-09T05:49:20.419867+0000","last_fullsized":"2023-10-09T05:49:20.419867+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:18:12.517403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":59,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.423218+0000","last_change":"2023-10-09T05:49:21.423218+0000","last_active":"2023-10-09T05:49:21.423218+0000","last_peered":"2023-10-09T05:49:21.423218+0000","last_clean":"2023-10-09T05:49:21.423218+0000","last_became_active":"2023-10-09T05:49:21.422706+0000","last_became_peered":"2023-10-09T05:49:21.422706+0000","last_unstale":"2023-10-09T05:49:21.423218+0000","last_undegraded":"2023-10-09T05:49:21.423218+0000","last_fullsized":"2023-10-09T05:49:21.423218+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:31.571220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426167+0000","last_change":"2023-10-09T05:49:21.426167+0000","last_active":"2023-10-09T05:49:21.426167+0000","last_peered":"2023-10-09T05:49:21.426167+0000","last_clean":"2023-10-09T05:49:21.426167+0000","last_became_active":"2023-10-09T05:49:21.425874+0000","last_became_peered":"2023-10-09T05:49:21.425874+0000","last_unstale":"2023-10-09T05:49:21.426167+0000","last_undegraded":"2023-10-09T05:49:21.426167+0000","last_fullsized":"2023-10-09T05:49:21.426167+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:59.642587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.428386+0000","last_change":"2023-10-09T05:49:21.428386+0000","last_active":"2023-10-09T05:49:21.428386+0000","last_peered":"2023-10-09T05:49:21.428386+0000","last_clean":"2023-10-09T05:49:21.428386+0000","last_became_active":"2023-10-09T05:49:21.428131+0000","last_became_peered":"2023-10-09T05:49:21.428131+0000","last_unstale":"2023-10-09T05:49:21.428386+0000","last_undegraded":"2023-10-09T05:49:21.428386+0000","last_fullsized":"2023-10-09T05:49:21.428386+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:12.858458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.419185+0000","last_change":"2023-10-09T05:48:49.139681+0000","last_active":"2023-10-09T05:49:20.419185+0000","last_peered":"2023-10-09T05:49:20.419185+0000","last_clean":"2023-10-09T05:49:20.419185+0000","last_became_active":"2023-10-09T05:48:49.139317+0000","last_became_peered":"2023-10-09T05:48:49.139317+0000","last_unstale":"2023-10-09T05:49:20.419185+0000","last_undegraded":"2023-10-09T05:49:20.419185+0000","last_fullsized":"2023-10-09T05:49:20.419185+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:34.769043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.413441+0000","last_change":"2023-10-09T05:48:17.173137+0000","last_active":"2023-10-09T05:49:20.413441+0000","last_peered":"2023-10-09T05:49:20.413441+0000","last_clean":"2023-10-09T05:49:20.413441+0000","last_became_active":"2023-10-09T05:48:17.172891+0000","last_became_peered":"2023-10-09T05:48:17.172891+0000","last_unstale":"2023-10-09T05:49:20.413441+0000","last_undegraded":"2023-10-09T05:49:20.413441+0000","last_fullsized":"2023-10-09T05:49:20.413441+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:21.393939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":65,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.413929+0000","last_change":"2023-10-09T05:40:12.630786+0000","last_active":"2023-10-09T05:49:20.413929+0000","last_peered":"2023-10-09T05:49:20.413929+0000","last_clean":"2023-10-09T05:49:20.413929+0000","last_became_active":"2023-10-09T05:40:12.630528+0000","last_became_peered":"2023-10-09T05:40:12.630528+0000","last_unstale":"2023-10-09T05:49:20.413929+0000","last_undegraded":"2023-10-09T05:49:20.413929+0000","last_fullsized":"2023-10-09T05:49:20.413929+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:54.427346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.412474+0000","last_change":"2023-10-09T05:48:49.138162+0000","last_active":"2023-10-09T05:49:20.412474+0000","last_peered":"2023-10-09T05:49:20.412474+0000","last_clean":"2023-10-09T05:49:20.412474+0000","last_became_active":"2023-10-09T05:48:49.137824+0000","last_became_peered":"2023-10-09T05:48:49.137824+0000","last_unstale":"2023-10-09T05:49:20.412474+0000","last_undegraded":"2023-10-09T05:49:20.412474+0000","last_fullsized":"2023-10-09T05:49:20.412474+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:43:26.642768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.430469+0000","last_change":"2023-10-09T05:49:21.430469+0000","last_active":"2023-10-09T05:49:21.430469+0000","last_peered":"2023-10-09T05:49:21.430469+0000","last_clean":"2023-10-09T05:49:21.430469+0000","last_became_active":"2023-10-09T05:49:21.430107+0000","last_became_peered":"2023-10-09T05:49:21.430107+0000","last_unstale":"2023-10-09T05:49:21.430469+0000","last_undegraded":"2023-10-09T05:49:21.430469+0000","last_fullsized":"2023-10-09T05:49:21.430469+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:08.815343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.425819+0000","last_change":"2023-10-09T05:49:21.425819+0000","last_active":"2023-10-09T05:49:21.425819+0000","last_peered":"2023-10-09T05:49:21.425819+0000","last_clean":"2023-10-09T05:49:21.425819+0000","last_became_active":"2023-10-09T05:49:21.425498+0000","last_became_peered":"2023-10-09T05:49:21.425498+0000","last_unstale":"2023-10-09T05:49:21.425819+0000","last_undegraded":"2023-10-09T05:49:21.425819+0000","last_fullsized":"2023-10-09T05:49:21.425819+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:21.443055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.421834+0000","last_change":"2023-10-09T05:48:17.176541+0000","last_active":"2023-10-09T05:49:20.421834+0000","last_peered":"2023-10-09T05:49:20.421834+0000","last_clean":"2023-10-09T05:49:20.421834+0000","last_became_active":"2023-10-09T05:48:17.176213+0000","last_became_peered":"2023-10-09T05:48:17.176213+0000","last_unstale":"2023-10-09T05:49:20.421834+0000","last_undegraded":"2023-10-09T05:49:20.421834+0000","last_fullsized":"2023-10-09T05:49:20.421834+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:23.599435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.427804+0000","last_change":"2023-10-09T05:49:21.427804+0000","last_active":"2023-10-09T05:49:21.427804+0000","last_peered":"2023-10-09T05:49:21.427804+0000","last_clean":"2023-10-09T05:49:21.427804+0000","last_became_active":"2023-10-09T05:49:21.427569+0000","last_became_peered":"2023-10-09T05:49:21.427569+0000","last_unstale":"2023-10-09T05:49:21.427804+0000","last_undegraded":"2023-10-09T05:49:21.427804+0000","last_fullsized":"2023-10-09T05:49:21.427804+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.791826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":53,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426616+0000","last_change":"2023-10-09T05:49:21.426149+0000","last_active":"2023-10-09T05:49:21.426616+0000","last_peered":"2023-10-09T05:49:21.426616+0000","last_clean":"2023-10-09T05:49:21.426616+0000","last_became_active":"2023-10-09T05:49:21.425805+0000","last_became_peered":"2023-10-09T05:49:21.425805+0000","last_unstale":"2023-10-09T05:49:21.426616+0000","last_undegraded":"2023-10-09T05:49:21.426616+0000","last_fullsized":"2023-10-09T05:49:21.426616+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:39:57.117762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.421197+0000","last_change":"2023-10-09T05:48:17.179524+0000","last_active":"2023-10-09T05:49:20.421197+0000","last_peered":"2023-10-09T05:49:20.421197+0000","last_clean":"2023-10-09T05:49:20.421197+0000","last_became_active":"2023-10-09T05:48:17.179233+0000","last_became_peered":"2023-10-09T05:48:17.179233+0000","last_unstale":"2023-10-09T05:49:20.421197+0000","last_undegraded":"2023-10-09T05:49:20.421197+0000","last_fullsized":"2023-10-09T05:49:20.421197+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:24.135263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":81,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.427745+0000","last_change":"2023-10-09T05:49:21.427745+0000","last_active":"2023-10-09T05:49:21.427745+0000","last_peered":"2023-10-09T05:49:21.427745+0000","last_clean":"2023-10-09T05:49:21.427745+0000","last_became_active":"2023-10-09T05:49:21.427292+0000","last_became_peered":"2023-10-09T05:49:21.427292+0000","last_unstale":"2023-10-09T05:49:21.427745+0000","last_undegraded":"2023-10-09T05:49:21.427745+0000","last_fullsized":"2023-10-09T05:49:21.427745+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:04:24.929346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.427476+0000","last_change":"2023-10-09T05:49:21.427476+0000","last_active":"2023-10-09T05:49:21.427476+0000","last_peered":"2023-10-09T05:49:21.427476+0000","last_clean":"2023-10-09T05:49:21.427476+0000","last_became_active":"2023-10-09T05:49:21.426417+0000","last_became_peered":"2023-10-09T05:49:21.426417+0000","last_unstale":"2023-10-09T05:49:21.427476+0000","last_undegraded":"2023-10-09T05:49:21.427476+0000","last_fullsized":"2023-10-09T05:49:21.427476+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:45:03.848962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.431711+0000","last_change":"2023-10-09T05:49:21.431711+0000","last_active":"2023-10-09T05:49:21.431711+0000","last_peered":"2023-10-09T05:49:21.431711+0000","last_clean":"2023-10-09T05:49:21.431711+0000","last_became_active":"2023-10-09T05:49:21.431441+0000","last_became_peered":"2023-10-09T05:49:21.431441+0000","last_unstale":"2023-10-09T05:49:21.431711+0000","last_undegraded":"2023-10-09T05:49:21.431711+0000","last_fullsized":"2023-10-09T05:49:21.431711+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:26.617516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.427978+0000","last_change":"2023-10-09T05:49:21.427978+0000","last_active":"2023-10-09T05:49:21.427978+0000","last_peered":"2023-10-09T05:49:21.427978+0000","last_clean":"2023-10-09T05:49:21.427978+0000","last_became_active":"2023-10-09T05:49:21.427703+0000","last_became_peered":"2023-10-09T05:49:21.427703+0000","last_unstale":"2023-10-09T05:49:21.427978+0000","last_undegraded":"2023-10-09T05:49:21.427978+0000","last_fullsized":"2023-10-09T05:49:21.427978+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:53:04.116683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":78,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.413362+0000","last_change":"2023-10-09T05:48:49.142429+0000","last_active":"2023-10-09T05:49:20.413362+0000","last_peered":"2023-10-09T05:49:20.413362+0000","last_clean":"2023-10-09T05:49:20.413362+0000","last_became_active":"2023-10-09T05:48:49.141966+0000","last_became_peered":"2023-10-09T05:48:49.141966+0000","last_unstale":"2023-10-09T05:49:20.413362+0000","last_undegraded":"2023-10-09T05:49:20.413362+0000","last_fullsized":"2023-10-09T05:49:20.413362+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:52.195431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234129,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311160,"kb_used_data":1584,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93433992,"statfs":{"total":95995035648,"available":95676407808,"internally_reserved":0,"allocated":1622016,"data_stored":1140338,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.69199999999999995,"15min":0.76100000000000001},"min":{"1min":0.38500000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.74299999999999999,"5min":4.8940000000000001,"15min":4.8940000000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.57999999999999996,"15min":0.60199999999999998},"min":{"1min":0.34799999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.749,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.66300000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.52300000000000002,"15min":0.52400000000000002},"min":{"1min":0.376,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.61199999999999999,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.55400000000000005,"15min":0.55200000000000005},"min":{"1min":0.37,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.71599999999999997,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.40799999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:49:50 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.53200000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:48:53 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.57299999999999995,"15min":0.82899999999999996},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.77100000000000002,"5min":0.82199999999999995,"15min":95.108000000000004},"last":0.629},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58199999999999996,"15min":0.83199999999999996},"min":{"1min":0.42999999999999999,"5min":0.36199999999999999,"15min":0.32800000000000001},"max":{"1min":0.749,"5min":0.94099999999999995,"15min":87.664000000000001},"last":0.374}]},{"osd":4,"last update":"Mon Oct 9 05:49:56 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.48399999999999999,"15min":0.48699999999999999},"min":{"1min":0.313,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":1.135},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.51200000000000001,"15min":0.498},"min":{"1min":0.378,"5min":0.27400000000000002,"15min":0.27100000000000002},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":3.165},"last":0.42099999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:49:56 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.499,"15min":0.499},"min":{"1min":0.33200000000000002,"5min":0.28199999999999997,"15min":0.248},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":2.5990000000000002},"last":0.435},{"interface":"front","average":{"1min":0.622,"5min":0.55700000000000005,"15min":0.54000000000000004},"min":{"1min":0.38800000000000001,"5min":0.34000000000000002,"15min":0.28100000000000003},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":1.0660000000000001},"last":0.68999999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:49:56 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.55900000000000005,"15min":0.54300000000000004},"min":{"1min":0.42399999999999999,"5min":0.317,"15min":0.29599999999999999},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":2.8170000000000002},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.57299999999999995,"15min":0.55100000000000005},"min":{"1min":0.41899999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":3.2589999999999999},"last":0.59799999999999998}]}]},{"osd":6,"up_from":46,"seq":197568495765,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298856,"kb_used_data":1568,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446296,"statfs":{"total":95995035648,"available":95689007104,"internally_reserved":0,"allocated":1605632,"data_stored":1139927,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.57999999999999996,"15min":0.59299999999999997},"min":{"1min":0.40300000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77000000000000002,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.57099999999999995,"15min":0.58899999999999997},"min":{"1min":0.38600000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.67800000000000005,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.33500000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:49:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.45500000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:49:50 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.39900000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:48:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.59999999999999998,"15min":0.56200000000000006},"min":{"1min":0.46100000000000002,"5min":0.38300000000000001,"15min":0.318},"max":{"1min":0.79000000000000004,"5min":0.85199999999999998,"15min":1.1100000000000001},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.57799999999999996,"15min":0.55600000000000005},"min":{"1min":0.40699999999999997,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.77400000000000002,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.433}]},{"osd":4,"last update":"Mon Oct 9 05:49:28 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.48499999999999999,"15min":0.45400000000000001},"min":{"1min":0.35499999999999998,"5min":0.309,"15min":0.26300000000000001},"max":{"1min":0.67300000000000004,"5min":0.71899999999999997,"15min":0.75600000000000001},"last":0.629},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.51700000000000002,"15min":0.48899999999999999},"min":{"1min":0.33400000000000002,"5min":0.29099999999999998,"15min":0.28599999999999998},"max":{"1min":0.75900000000000001,"5min":0.80500000000000005,"15min":0.82799999999999996},"last":0.69699999999999995}]},{"osd":5,"last update":"Mon Oct 9 05:49:28 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.50900000000000001,"15min":0.47199999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.27700000000000002},"max":{"1min":0.77500000000000002,"5min":0.78200000000000003,"15min":0.81299999999999994},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.54500000000000004,"15min":0.51800000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32200000000000001},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85799999999999998},"last":0.73199999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:49:58 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.54700000000000004,"15min":0.53400000000000003},"min":{"1min":0.39800000000000002,"5min":0.316,"15min":0.316},"max":{"1min":0.64800000000000002,"5min":0.78800000000000003,"15min":0.79700000000000004},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.626,"5min":0.627,"15min":0.59799999999999998},"min":{"1min":0.39800000000000002,"5min":0.39600000000000002,"15min":0.35799999999999998},"max":{"1min":0.82299999999999995,"5min":0.88200000000000001,"15min":0.88300000000000001},"last":0.76800000000000002}]}]},{"osd":1,"up_from":89,"seq":382252089371,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28492,"kb_used_data":1028,"kb_used_omap":9,"kb_used_meta":27446,"kb_avail":93716660,"statfs":{"total":95995035648,"available":95965859840,"internally_reserved":0,"allocated":1052672,"data_stored":555584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":28105705},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:50:19 2023","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.35599999999999998,"15min":0.35799999999999998},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.27400000000000002},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.39700000000000002,"15min":0.40200000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.29599999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.5119999999999996,"5min":4.5119999999999996,"15min":4.5119999999999996},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.6139999999999999,"5min":3.6139999999999999,"15min":3.6139999999999999},"last":0.33900000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40100000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:50:19 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.59599999999999997,"15min":0.59299999999999997},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.028,"5min":1.028,"15min":1.028},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60999999999999999,"15min":0.61099999999999999},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.76100000000000001,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.57499999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:50:19 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.45000000000000001,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.498},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.60699999999999998,"15min":0.60899999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.58699999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:50:19 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.61299999999999999,"15min":0.61899999999999999},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.626,"5min":0.63400000000000001,"15min":0.63500000000000001},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.94599999999999995,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.51100000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:50:19 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.65600000000000003,"15min":0.66300000000000003},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.002,"5min":1.002,"15min":1.002},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.82499999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.54200000000000004}]}]},{"osd":0,"up_from":84,"seq":360777252905,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29160,"kb_used_data":1632,"kb_used_omap":28,"kb_used_meta":27491,"kb_avail":93715992,"statfs":{"total":95995035648,"available":95965175808,"internally_reserved":0,"allocated":1671168,"data_stored":1149588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":28151632},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.625,"5min":7.625,"15min":7.625},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":3.8130000000000002,"5min":3.8130000000000002,"15min":3.8130000000000002},"last":0.33600000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:49:49 2023","interfaces":[{"interface":"back","average":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.5640000000000001,"5min":7.5640000000000001,"15min":7.5640000000000001},"last":0.35399999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:49:04 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.502,"15min":0.51400000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.51000000000000001,"15min":0.52000000000000002},"min":{"1min":0.33600000000000002,"5min":0.307,"15min":0.307},"max":{"1min":0.63,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.36699999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:50:04 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.67500000000000004,"15min":0.69499999999999995},"min":{"1min":0.42499999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.69799999999999995,"15min":0.72199999999999998},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.90100000000000002,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.72899999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:50:04 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.66800000000000004,"15min":0.68600000000000005},"min":{"1min":0.42199999999999999,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.622,"5min":0.68200000000000005,"15min":0.71599999999999997},"min":{"1min":0.498,"5min":0.376,"15min":0.376},"max":{"1min":1.27,"5min":1.27,"15min":1.27},"last":0.63900000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:50:04 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.68999999999999995,"15min":0.72499999999999998},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.753},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.71799999999999997,"15min":0.746},"min":{"1min":0.47499999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.995,"5min":0.995,"15min":0.995},"last":0.70599999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:50:04 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.75700000000000001,"15min":0.78900000000000003},"min":{"1min":0.47799999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.016,"5min":1.016,"15min":1.016},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.73699999999999999,"15min":0.76500000000000001},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.78600000000000003}]}]},{"osd":2,"up_from":94,"seq":403726925844,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28620,"kb_used_data":1028,"kb_used_omap":81,"kb_used_meta":27502,"kb_avail":93716532,"statfs":{"total":95995035648,"available":95965728768,"internally_reserved":0,"allocated":1052672,"data_stored":554321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":28162266},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.36799999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.30599999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28100000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.62},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.59899999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.56999999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.63100000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.498}]}]},{"osd":3,"up_from":99,"seq":425201762318,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28468,"kb_used_data":1004,"kb_used_omap":7,"kb_used_meta":27448,"kb_avail":93716684,"statfs":{"total":95995035648,"available":95965884416,"internally_reserved":0,"allocated":1028096,"data_stored":549876,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":28107368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.37}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38600000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55100000000000005}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46300000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57699999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56200000000000006}]}]},{"osd":4,"up_from":30,"seq":128849019038,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302388,"kb_used_data":1004,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93442764,"statfs":{"total":95995035648,"available":95685390336,"internally_reserved":0,"allocated":1028096,"data_stored":550223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:50:10 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.57699999999999996,"15min":0.59799999999999998},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.61499999999999999,"15min":0.63700000000000001},"min":{"1min":0.35799999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.67000000000000004}]},{"osd":1,"last update":"Mon Oct 9 05:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.57299999999999995}]},{"osd":2,"last update":"Mon Oct 9 05:49:50 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.032,"5min":1.032,"15min":1.032},"last":0.61099999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:48:56 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.60299999999999998,"15min":0.58699999999999997},"min":{"1min":0.443,"5min":0.38,"15min":0.32600000000000001},"max":{"1min":0.76100000000000001,"5min":0.81200000000000006,"15min":0.995},"last":0.627},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.59499999999999997,"15min":0.57899999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.35899999999999999},"max":{"1min":0.71199999999999997,"5min":0.83599999999999997,"15min":0.89800000000000002},"last":0.59699999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:50:10 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.47599999999999998,"15min":0.46000000000000002},"min":{"1min":0.36899999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.66300000000000003,"5min":0.83399999999999996,"15min":0.90500000000000003},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.53000000000000003,"15min":0.498},"min":{"1min":0.33100000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.78000000000000003,"5min":0.82199999999999995,"15min":0.88100000000000001},"last":0.42799999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:49:27 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.50900000000000001,"15min":0.495},"min":{"1min":0.34100000000000003,"5min":0.31,"15min":0.27400000000000002},"max":{"1min":0.64300000000000002,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.55700000000000005,"15min":0.53200000000000003},"min":{"1min":0.34799999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.72499999999999998,"5min":0.86199999999999999,"15min":0.89900000000000002},"last":0.51400000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:50:00 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.58399999999999996,"15min":0.56799999999999995},"min":{"1min":0.41699999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":1.046},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.58699999999999997,"15min":0.58399999999999996},"min":{"1min":0.48199999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.95799999999999996},"last":0.53200000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757401,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306468,"kb_used_data":988,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93438684,"statfs":{"total":95995035648,"available":95681212416,"internally_reserved":0,"allocated":1011712,"data_stored":545967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:50:07 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.57299999999999995,"15min":0.58999999999999997},"min":{"1min":0.35599999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.81100000000000005,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.59199999999999997,"15min":0.61499999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.68899999999999995,"5min":1,"15min":1},"last":0.379}]},{"osd":1,"last update":"Mon Oct 9 05:49:19 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.52000000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:49:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.59199999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:49:07 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.59599999999999997,"15min":1.46},"min":{"1min":0.373,"5min":0.373,"15min":0.36699999999999999},"max":{"1min":0.88700000000000001,"5min":0.98499999999999999,"15min":247.672},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.61399999999999999,"15min":1.4930000000000001},"min":{"1min":0.48399999999999999,"5min":0.36899999999999999,"15min":0.36799999999999999},"max":{"1min":0.82299999999999995,"5min":0.88600000000000001,"15min":247.90000000000001},"last":0.63}]},{"osd":4,"last update":"Mon Oct 9 05:50:07 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.48799999999999999,"15min":0.45600000000000002},"min":{"1min":0.311,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.754,"5min":0.754,"15min":3.5430000000000001},"last":0.62},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.50800000000000001,"15min":0.47599999999999998},"min":{"1min":0.372,"5min":0.29999999999999999,"15min":0.28999999999999998},"max":{"1min":0.66600000000000004,"5min":0.72699999999999998,"15min":1.6990000000000001},"last":0.57499999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:49:24 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.53000000000000003,"15min":0.48799999999999999},"min":{"1min":0.41899999999999998,"5min":0.307,"15min":0.29899999999999999},"max":{"1min":0.69799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.438},{"interface":"front","average":{"1min":0.622,"5min":0.57799999999999996,"15min":0.52200000000000002},"min":{"1min":0.39300000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.80600000000000005,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.66300000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:50:03 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.60799999999999998,"15min":0.56299999999999994},"min":{"1min":0.435,"5min":0.32700000000000001,"15min":0.315},"max":{"1min":0.81000000000000005,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.58799999999999997,"15min":0.59799999999999998},"min":{"1min":0.47399999999999998,"5min":0.35099999999999998,"15min":0.34899999999999998},"max":{"1min":0.79400000000000004,"5min":0.79700000000000004,"15min":3.2869999999999999},"last":0.65000000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:50:21.332 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:50:21.692 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:22.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:22 smithi064 ceph-mon[212750]: pgmap v214: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:22.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:22 smithi064 ceph-mon[212750]: from='client.34605 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:50:22.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:22 smithi064 ceph-mon[220044]: pgmap v214: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:22.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:22 smithi064 ceph-mon[220044]: from='client.34605 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:50:22.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:22 smithi156 ceph-mon[195101]: pgmap v214: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:22.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:22 smithi156 ceph-mon[195101]: from='client.34605 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:50:23.428 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:50:23.429 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T05:50:24.177 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":215,"stamp":"2023-10-09T05:50:22.434099+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1333612,"kb_used_data":9836,"kb_used_omap":126,"kb_used_meta":1323585,"kb_avail":748627604,"statfs":{"total":767960285184,"available":766594666496,"internally_reserved":0,"allocated":10072064,"data_stored":6185824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1355351675},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002611"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.413173+0000","last_change":"2023-10-09T05:48:17.172491+0000","last_active":"2023-10-09T05:49:20.413173+0000","last_peered":"2023-10-09T05:49:20.413173+0000","last_clean":"2023-10-09T05:49:20.413173+0000","last_became_active":"2023-10-09T05:48:17.172154+0000","last_became_peered":"2023-10-09T05:48:17.172154+0000","last_unstale":"2023-10-09T05:49:20.413173+0000","last_undegraded":"2023-10-09T05:49:20.413173+0000","last_fullsized":"2023-10-09T05:49:20.413173+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:24:40.551774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":57,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.416576+0000","last_change":"2023-10-09T05:48:49.139798+0000","last_active":"2023-10-09T05:49:20.416576+0000","last_peered":"2023-10-09T05:49:20.416576+0000","last_clean":"2023-10-09T05:49:20.416576+0000","last_became_active":"2023-10-09T05:48:49.139442+0000","last_became_peered":"2023-10-09T05:48:49.139442+0000","last_unstale":"2023-10-09T05:49:20.416576+0000","last_undegraded":"2023-10-09T05:49:20.416576+0000","last_fullsized":"2023-10-09T05:49:20.416576+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:25.034620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":84,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.421384+0000","last_change":"2023-10-09T05:49:21.421384+0000","last_active":"2023-10-09T05:49:21.421384+0000","last_peered":"2023-10-09T05:49:21.421384+0000","last_clean":"2023-10-09T05:49:21.421384+0000","last_became_active":"2023-10-09T05:49:21.420886+0000","last_became_peered":"2023-10-09T05:49:21.420886+0000","last_unstale":"2023-10-09T05:49:21.421384+0000","last_undegraded":"2023-10-09T05:49:21.421384+0000","last_fullsized":"2023-10-09T05:49:21.421384+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:33:24.415387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":69,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.430192+0000","last_change":"2023-10-09T05:49:21.430192+0000","last_active":"2023-10-09T05:49:21.430192+0000","last_peered":"2023-10-09T05:49:21.430192+0000","last_clean":"2023-10-09T05:49:21.430192+0000","last_became_active":"2023-10-09T05:49:21.429857+0000","last_became_peered":"2023-10-09T05:49:21.429857+0000","last_unstale":"2023-10-09T05:49:21.430192+0000","last_undegraded":"2023-10-09T05:49:21.430192+0000","last_fullsized":"2023-10-09T05:49:21.430192+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:45.887757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":76,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426615+0000","last_change":"2023-10-09T05:49:21.426615+0000","last_active":"2023-10-09T05:49:21.426615+0000","last_peered":"2023-10-09T05:49:21.426615+0000","last_clean":"2023-10-09T05:49:21.426615+0000","last_became_active":"2023-10-09T05:49:21.426245+0000","last_became_peered":"2023-10-09T05:49:21.426245+0000","last_unstale":"2023-10-09T05:49:21.426615+0000","last_undegraded":"2023-10-09T05:49:21.426615+0000","last_fullsized":"2023-10-09T05:49:21.426615+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:47.642455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":45,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.425017+0000","last_change":"2023-10-09T05:40:12.633195+0000","last_active":"2023-10-09T05:49:20.425017+0000","last_peered":"2023-10-09T05:49:20.425017+0000","last_clean":"2023-10-09T05:49:20.425017+0000","last_became_active":"2023-10-09T05:40:12.632942+0000","last_became_peered":"2023-10-09T05:40:12.632942+0000","last_unstale":"2023-10-09T05:49:20.425017+0000","last_undegraded":"2023-10-09T05:49:20.425017+0000","last_fullsized":"2023-10-09T05:49:20.425017+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:04.470685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":76,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426304+0000","last_change":"2023-10-09T05:49:21.426304+0000","last_active":"2023-10-09T05:49:21.426304+0000","last_peered":"2023-10-09T05:49:21.426304+0000","last_clean":"2023-10-09T05:49:21.426304+0000","last_became_active":"2023-10-09T05:49:21.425949+0000","last_became_peered":"2023-10-09T05:49:21.425949+0000","last_unstale":"2023-10-09T05:49:21.426304+0000","last_undegraded":"2023-10-09T05:49:21.426304+0000","last_fullsized":"2023-10-09T05:49:21.426304+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:11.209520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.424610+0000","last_change":"2023-10-09T05:48:17.173906+0000","last_active":"2023-10-09T05:49:20.424610+0000","last_peered":"2023-10-09T05:49:20.424610+0000","last_clean":"2023-10-09T05:49:20.424610+0000","last_became_active":"2023-10-09T05:48:17.173564+0000","last_became_peered":"2023-10-09T05:48:17.173564+0000","last_unstale":"2023-10-09T05:49:20.424610+0000","last_undegraded":"2023-10-09T05:49:20.424610+0000","last_fullsized":"2023-10-09T05:49:20.424610+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:37.582247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":81,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.430604+0000","last_change":"2023-10-09T05:49:21.430604+0000","last_active":"2023-10-09T05:49:21.430604+0000","last_peered":"2023-10-09T05:49:21.430604+0000","last_clean":"2023-10-09T05:49:21.430604+0000","last_became_active":"2023-10-09T05:49:21.430220+0000","last_became_peered":"2023-10-09T05:49:21.430220+0000","last_unstale":"2023-10-09T05:49:21.430604+0000","last_undegraded":"2023-10-09T05:49:21.430604+0000","last_fullsized":"2023-10-09T05:49:21.430604+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:06:01.882964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":42,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.417852+0000","last_change":"2023-10-09T05:47:03.233346+0000","last_active":"2023-10-09T05:49:21.417852+0000","last_peered":"2023-10-09T05:49:21.417852+0000","last_clean":"2023-10-09T05:49:21.417852+0000","last_became_active":"2023-10-09T05:47:03.231300+0000","last_became_peered":"2023-10-09T05:47:03.231300+0000","last_unstale":"2023-10-09T05:49:21.417852+0000","last_undegraded":"2023-10-09T05:49:21.417852+0000","last_fullsized":"2023-10-09T05:49:21.417852+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:22.050814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":42,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.414600+0000","last_change":"2023-10-09T05:40:12.676587+0000","last_active":"2023-10-09T05:49:20.414600+0000","last_peered":"2023-10-09T05:49:20.414600+0000","last_clean":"2023-10-09T05:49:20.414600+0000","last_became_active":"2023-10-09T05:40:12.676330+0000","last_became_peered":"2023-10-09T05:40:12.676330+0000","last_unstale":"2023-10-09T05:49:20.414600+0000","last_undegraded":"2023-10-09T05:49:20.414600+0000","last_fullsized":"2023-10-09T05:49:20.414600+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:46.659556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":57,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.416515+0000","last_change":"2023-10-09T05:48:17.174996+0000","last_active":"2023-10-09T05:49:20.416515+0000","last_peered":"2023-10-09T05:49:20.416515+0000","last_clean":"2023-10-09T05:49:20.416515+0000","last_became_active":"2023-10-09T05:48:17.174668+0000","last_became_peered":"2023-10-09T05:48:17.174668+0000","last_unstale":"2023-10-09T05:49:20.416515+0000","last_undegraded":"2023-10-09T05:49:20.416515+0000","last_fullsized":"2023-10-09T05:49:20.416515+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:49.712686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.421123+0000","last_change":"2023-10-09T05:48:49.137565+0000","last_active":"2023-10-09T05:49:20.421123+0000","last_peered":"2023-10-09T05:49:20.421123+0000","last_clean":"2023-10-09T05:49:20.421123+0000","last_became_active":"2023-10-09T05:48:49.137099+0000","last_became_peered":"2023-10-09T05:48:49.137099+0000","last_unstale":"2023-10-09T05:49:20.421123+0000","last_undegraded":"2023-10-09T05:49:20.421123+0000","last_fullsized":"2023-10-09T05:49:20.421123+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:28.103821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":58,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.423248+0000","last_change":"2023-10-09T05:49:21.423248+0000","last_active":"2023-10-09T05:49:21.423248+0000","last_peered":"2023-10-09T05:49:21.423248+0000","last_clean":"2023-10-09T05:49:21.423248+0000","last_became_active":"2023-10-09T05:49:21.422985+0000","last_became_peered":"2023-10-09T05:49:21.422985+0000","last_unstale":"2023-10-09T05:49:21.423248+0000","last_undegraded":"2023-10-09T05:49:21.423248+0000","last_fullsized":"2023-10-09T05:49:21.423248+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:02.431791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.409445+0000","last_change":"2023-10-09T05:48:17.174101+0000","last_active":"2023-10-09T05:49:20.409445+0000","last_peered":"2023-10-09T05:49:20.409445+0000","last_clean":"2023-10-09T05:49:20.409445+0000","last_became_active":"2023-10-09T05:48:17.173752+0000","last_became_peered":"2023-10-09T05:48:17.173752+0000","last_unstale":"2023-10-09T05:49:20.409445+0000","last_undegraded":"2023-10-09T05:49:20.409445+0000","last_fullsized":"2023-10-09T05:49:20.409445+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:45:16.222083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":61,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.423414+0000","last_change":"2023-10-09T05:49:21.423414+0000","last_active":"2023-10-09T05:49:21.423414+0000","last_peered":"2023-10-09T05:49:21.423414+0000","last_clean":"2023-10-09T05:49:21.423414+0000","last_became_active":"2023-10-09T05:49:21.423085+0000","last_became_peered":"2023-10-09T05:49:21.423085+0000","last_unstale":"2023-10-09T05:49:21.423414+0000","last_undegraded":"2023-10-09T05:49:21.423414+0000","last_fullsized":"2023-10-09T05:49:21.423414+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:51.777782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.414717+0000","last_change":"2023-10-09T05:47:03.233460+0000","last_active":"2023-10-09T05:49:20.414717+0000","last_peered":"2023-10-09T05:49:20.414717+0000","last_clean":"2023-10-09T05:49:20.414717+0000","last_became_active":"2023-10-09T05:47:03.231685+0000","last_became_peered":"2023-10-09T05:47:03.231685+0000","last_unstale":"2023-10-09T05:49:20.414717+0000","last_undegraded":"2023-10-09T05:49:20.414717+0000","last_fullsized":"2023-10-09T05:49:20.414717+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:10.790717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.423612+0000","last_change":"2023-10-09T05:48:49.139759+0000","last_active":"2023-10-09T05:49:21.423612+0000","last_peered":"2023-10-09T05:49:21.423612+0000","last_clean":"2023-10-09T05:49:21.423612+0000","last_became_active":"2023-10-09T05:48:49.139346+0000","last_became_peered":"2023-10-09T05:48:49.139346+0000","last_unstale":"2023-10-09T05:49:21.423612+0000","last_undegraded":"2023-10-09T05:49:21.423612+0000","last_fullsized":"2023-10-09T05:49:21.423612+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:51.980711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.420851+0000","last_change":"2023-10-09T05:47:03.230117+0000","last_active":"2023-10-09T05:49:20.420851+0000","last_peered":"2023-10-09T05:49:20.420851+0000","last_clean":"2023-10-09T05:49:20.420851+0000","last_became_active":"2023-10-09T05:47:03.229660+0000","last_became_peered":"2023-10-09T05:47:03.229660+0000","last_unstale":"2023-10-09T05:49:20.420851+0000","last_undegraded":"2023-10-09T05:49:20.420851+0000","last_fullsized":"2023-10-09T05:49:20.420851+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:07:21.669282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.416527+0000","last_change":"2023-10-09T05:48:49.134851+0000","last_active":"2023-10-09T05:49:20.416527+0000","last_peered":"2023-10-09T05:49:20.416527+0000","last_clean":"2023-10-09T05:49:20.416527+0000","last_became_active":"2023-10-09T05:48:49.134532+0000","last_became_peered":"2023-10-09T05:48:49.134532+0000","last_unstale":"2023-10-09T05:49:20.416527+0000","last_undegraded":"2023-10-09T05:49:20.416527+0000","last_fullsized":"2023-10-09T05:49:20.416527+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:23.395907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.422120+0000","last_change":"2023-10-09T05:49:21.422120+0000","last_active":"2023-10-09T05:49:21.422120+0000","last_peered":"2023-10-09T05:49:21.422120+0000","last_clean":"2023-10-09T05:49:21.422120+0000","last_became_active":"2023-10-09T05:49:21.421835+0000","last_became_peered":"2023-10-09T05:49:21.421835+0000","last_unstale":"2023-10-09T05:49:21.422120+0000","last_undegraded":"2023-10-09T05:49:21.422120+0000","last_fullsized":"2023-10-09T05:49:21.422120+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:56.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":81,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.418771+0000","last_change":"2023-10-09T05:48:17.175668+0000","last_active":"2023-10-09T05:49:20.418771+0000","last_peered":"2023-10-09T05:49:20.418771+0000","last_clean":"2023-10-09T05:49:20.418771+0000","last_became_active":"2023-10-09T05:48:17.175344+0000","last_became_peered":"2023-10-09T05:48:17.175344+0000","last_unstale":"2023-10-09T05:49:20.418771+0000","last_undegraded":"2023-10-09T05:49:20.418771+0000","last_fullsized":"2023-10-09T05:49:20.418771+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:43.949807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":42,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.419433+0000","last_change":"2023-10-09T05:40:14.363125+0000","last_active":"2023-10-09T05:49:20.419433+0000","last_peered":"2023-10-09T05:49:20.419433+0000","last_clean":"2023-10-09T05:49:20.419433+0000","last_became_active":"2023-10-09T05:40:14.362723+0000","last_became_peered":"2023-10-09T05:40:14.362723+0000","last_unstale":"2023-10-09T05:49:20.419433+0000","last_undegraded":"2023-10-09T05:49:20.419433+0000","last_fullsized":"2023-10-09T05:49:20.419433+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:30.741318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426544+0000","last_change":"2023-10-09T05:49:21.426544+0000","last_active":"2023-10-09T05:49:21.426544+0000","last_peered":"2023-10-09T05:49:21.426544+0000","last_clean":"2023-10-09T05:49:21.426544+0000","last_became_active":"2023-10-09T05:49:21.426253+0000","last_became_peered":"2023-10-09T05:49:21.426253+0000","last_unstale":"2023-10-09T05:49:21.426544+0000","last_undegraded":"2023-10-09T05:49:21.426544+0000","last_fullsized":"2023-10-09T05:49:21.426544+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:32:40.322649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":441,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.418674+0000","last_change":"2023-10-09T05:47:03.234961+0000","last_active":"2023-10-09T05:49:20.418674+0000","last_peered":"2023-10-09T05:49:20.418674+0000","last_clean":"2023-10-09T05:49:20.418674+0000","last_became_active":"2023-10-09T05:47:03.234618+0000","last_became_peered":"2023-10-09T05:47:03.234618+0000","last_unstale":"2023-10-09T05:49:20.418674+0000","last_undegraded":"2023-10-09T05:49:20.418674+0000","last_fullsized":"2023-10-09T05:49:20.418674+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:01.472111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.422069+0000","last_change":"2023-10-09T05:48:49.141207+0000","last_active":"2023-10-09T05:49:20.422069+0000","last_peered":"2023-10-09T05:49:20.422069+0000","last_clean":"2023-10-09T05:49:20.422069+0000","last_became_active":"2023-10-09T05:48:49.140798+0000","last_became_peered":"2023-10-09T05:48:49.140798+0000","last_unstale":"2023-10-09T05:49:20.422069+0000","last_undegraded":"2023-10-09T05:49:20.422069+0000","last_fullsized":"2023-10-09T05:49:20.422069+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:44:13.125030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.414450+0000","last_change":"2023-10-09T05:48:49.139757+0000","last_active":"2023-10-09T05:49:20.414450+0000","last_peered":"2023-10-09T05:49:20.414450+0000","last_clean":"2023-10-09T05:49:20.414450+0000","last_became_active":"2023-10-09T05:48:49.139311+0000","last_became_peered":"2023-10-09T05:48:49.139311+0000","last_unstale":"2023-10-09T05:49:20.414450+0000","last_undegraded":"2023-10-09T05:49:20.414450+0000","last_fullsized":"2023-10-09T05:49:20.414450+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:22.445381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.422046+0000","last_change":"2023-10-09T05:48:49.137529+0000","last_active":"2023-10-09T05:49:21.422046+0000","last_peered":"2023-10-09T05:49:21.422046+0000","last_clean":"2023-10-09T05:49:21.422046+0000","last_became_active":"2023-10-09T05:48:49.137212+0000","last_became_peered":"2023-10-09T05:48:49.137212+0000","last_unstale":"2023-10-09T05:49:21.422046+0000","last_undegraded":"2023-10-09T05:49:21.422046+0000","last_fullsized":"2023-10-09T05:49:21.422046+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:29.740386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":82,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.429787+0000","last_change":"2023-10-09T05:49:21.429787+0000","last_active":"2023-10-09T05:49:21.429787+0000","last_peered":"2023-10-09T05:49:21.429787+0000","last_clean":"2023-10-09T05:49:21.429787+0000","last_became_active":"2023-10-09T05:49:21.429304+0000","last_became_peered":"2023-10-09T05:49:21.429304+0000","last_unstale":"2023-10-09T05:49:21.429787+0000","last_undegraded":"2023-10-09T05:49:21.429787+0000","last_fullsized":"2023-10-09T05:49:21.429787+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:28:24.515861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":73,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.429250+0000","last_change":"2023-10-09T05:49:21.429250+0000","last_active":"2023-10-09T05:49:21.429250+0000","last_peered":"2023-10-09T05:49:21.429250+0000","last_clean":"2023-10-09T05:49:21.429250+0000","last_became_active":"2023-10-09T05:49:21.428946+0000","last_became_peered":"2023-10-09T05:49:21.428946+0000","last_unstale":"2023-10-09T05:49:21.429250+0000","last_undegraded":"2023-10-09T05:49:21.429250+0000","last_fullsized":"2023-10-09T05:49:21.429250+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:32.978473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.414007+0000","last_change":"2023-10-09T05:47:03.237144+0000","last_active":"2023-10-09T05:49:20.414007+0000","last_peered":"2023-10-09T05:49:20.414007+0000","last_clean":"2023-10-09T05:49:20.414007+0000","last_became_active":"2023-10-09T05:47:03.236793+0000","last_became_peered":"2023-10-09T05:47:03.236793+0000","last_unstale":"2023-10-09T05:49:20.414007+0000","last_undegraded":"2023-10-09T05:49:20.414007+0000","last_fullsized":"2023-10-09T05:49:20.414007+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:00:44.273589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.422361+0000","last_change":"2023-10-09T05:48:17.172780+0000","last_active":"2023-10-09T05:49:20.422361+0000","last_peered":"2023-10-09T05:49:20.422361+0000","last_clean":"2023-10-09T05:49:20.422361+0000","last_became_active":"2023-10-09T05:48:17.172460+0000","last_became_peered":"2023-10-09T05:48:17.172460+0000","last_unstale":"2023-10-09T05:49:20.422361+0000","last_undegraded":"2023-10-09T05:49:20.422361+0000","last_fullsized":"2023-10-09T05:49:20.422361+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:41.981302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.421018+0000","last_change":"2023-10-09T05:48:17.172596+0000","last_active":"2023-10-09T05:49:21.421018+0000","last_peered":"2023-10-09T05:49:21.421018+0000","last_clean":"2023-10-09T05:49:21.421018+0000","last_became_active":"2023-10-09T05:48:17.172287+0000","last_became_peered":"2023-10-09T05:48:17.172287+0000","last_unstale":"2023-10-09T05:49:21.421018+0000","last_undegraded":"2023-10-09T05:49:21.421018+0000","last_fullsized":"2023-10-09T05:49:21.421018+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:53.440906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":77,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.422980+0000","last_change":"2023-10-09T05:49:21.422980+0000","last_active":"2023-10-09T05:49:21.422980+0000","last_peered":"2023-10-09T05:49:21.422980+0000","last_clean":"2023-10-09T05:49:21.422980+0000","last_became_active":"2023-10-09T05:49:21.422349+0000","last_became_peered":"2023-10-09T05:49:21.422349+0000","last_unstale":"2023-10-09T05:49:21.422980+0000","last_undegraded":"2023-10-09T05:49:21.422980+0000","last_fullsized":"2023-10-09T05:49:21.422980+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:03.118182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":73,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.423855+0000","last_change":"2023-10-09T05:48:17.170845+0000","last_active":"2023-10-09T05:49:21.423855+0000","last_peered":"2023-10-09T05:49:21.423855+0000","last_clean":"2023-10-09T05:49:21.423855+0000","last_became_active":"2023-10-09T05:48:17.170321+0000","last_became_peered":"2023-10-09T05:48:17.170321+0000","last_unstale":"2023-10-09T05:49:21.423855+0000","last_undegraded":"2023-10-09T05:49:21.423855+0000","last_fullsized":"2023-10-09T05:49:21.423855+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:41.854157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":47,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.431778+0000","last_change":"2023-10-09T05:49:21.431778+0000","last_active":"2023-10-09T05:49:21.431778+0000","last_peered":"2023-10-09T05:49:21.431778+0000","last_clean":"2023-10-09T05:49:21.431778+0000","last_became_active":"2023-10-09T05:49:21.431483+0000","last_became_peered":"2023-10-09T05:49:21.431483+0000","last_unstale":"2023-10-09T05:49:21.431778+0000","last_undegraded":"2023-10-09T05:49:21.431778+0000","last_fullsized":"2023-10-09T05:49:21.431778+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:29:39.293436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":64,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.427586+0000","last_change":"2023-10-09T05:49:21.427586+0000","last_active":"2023-10-09T05:49:21.427586+0000","last_peered":"2023-10-09T05:49:21.427586+0000","last_clean":"2023-10-09T05:49:21.427586+0000","last_became_active":"2023-10-09T05:49:21.426908+0000","last_became_peered":"2023-10-09T05:49:21.426908+0000","last_unstale":"2023-10-09T05:49:21.427586+0000","last_undegraded":"2023-10-09T05:49:21.427586+0000","last_fullsized":"2023-10-09T05:49:21.427586+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:50.043430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.414177+0000","last_change":"2023-10-09T05:48:17.173636+0000","last_active":"2023-10-09T05:49:20.414177+0000","last_peered":"2023-10-09T05:49:20.414177+0000","last_clean":"2023-10-09T05:49:20.414177+0000","last_became_active":"2023-10-09T05:48:17.173329+0000","last_became_peered":"2023-10-09T05:48:17.173329+0000","last_unstale":"2023-10-09T05:49:20.414177+0000","last_undegraded":"2023-10-09T05:49:20.414177+0000","last_fullsized":"2023-10-09T05:49:20.414177+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:26:36.634285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.414234+0000","last_change":"2023-10-09T05:48:49.141287+0000","last_active":"2023-10-09T05:49:20.414234+0000","last_peered":"2023-10-09T05:49:20.414234+0000","last_clean":"2023-10-09T05:49:20.414234+0000","last_became_active":"2023-10-09T05:48:49.140950+0000","last_became_peered":"2023-10-09T05:48:49.140950+0000","last_unstale":"2023-10-09T05:49:20.414234+0000","last_undegraded":"2023-10-09T05:49:20.414234+0000","last_fullsized":"2023-10-09T05:49:20.414234+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:35.527434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":76,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.413852+0000","last_change":"2023-10-09T05:48:49.138043+0000","last_active":"2023-10-09T05:49:20.413852+0000","last_peered":"2023-10-09T05:49:20.413852+0000","last_clean":"2023-10-09T05:49:20.413852+0000","last_became_active":"2023-10-09T05:48:49.137722+0000","last_became_peered":"2023-10-09T05:48:49.137722+0000","last_unstale":"2023-10-09T05:49:20.413852+0000","last_undegraded":"2023-10-09T05:49:20.413852+0000","last_fullsized":"2023-10-09T05:49:20.413852+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:38.270179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.420336+0000","last_change":"2023-10-09T05:48:17.179214+0000","last_active":"2023-10-09T05:49:20.420336+0000","last_peered":"2023-10-09T05:49:20.420336+0000","last_clean":"2023-10-09T05:49:20.420336+0000","last_became_active":"2023-10-09T05:48:17.178918+0000","last_became_peered":"2023-10-09T05:48:17.178918+0000","last_unstale":"2023-10-09T05:49:20.420336+0000","last_undegraded":"2023-10-09T05:49:20.420336+0000","last_fullsized":"2023-10-09T05:49:20.420336+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:58:07.653506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":59,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.430742+0000","last_change":"2023-10-09T05:49:21.430742+0000","last_active":"2023-10-09T05:49:21.430742+0000","last_peered":"2023-10-09T05:49:21.430742+0000","last_clean":"2023-10-09T05:49:21.430742+0000","last_became_active":"2023-10-09T05:49:21.430233+0000","last_became_peered":"2023-10-09T05:49:21.430233+0000","last_unstale":"2023-10-09T05:49:21.430742+0000","last_undegraded":"2023-10-09T05:49:21.430742+0000","last_fullsized":"2023-10-09T05:49:21.430742+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:10.541302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":61,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.421396+0000","last_change":"2023-10-09T05:49:21.421396+0000","last_active":"2023-10-09T05:49:21.421396+0000","last_peered":"2023-10-09T05:49:21.421396+0000","last_clean":"2023-10-09T05:49:21.421396+0000","last_became_active":"2023-10-09T05:49:21.421129+0000","last_became_peered":"2023-10-09T05:49:21.421129+0000","last_unstale":"2023-10-09T05:49:21.421396+0000","last_undegraded":"2023-10-09T05:49:21.421396+0000","last_fullsized":"2023-10-09T05:49:21.421396+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:14.365625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.415371+0000","last_change":"2023-10-09T05:48:17.172849+0000","last_active":"2023-10-09T05:49:20.415371+0000","last_peered":"2023-10-09T05:49:20.415371+0000","last_clean":"2023-10-09T05:49:20.415371+0000","last_became_active":"2023-10-09T05:48:17.172536+0000","last_became_peered":"2023-10-09T05:48:17.172536+0000","last_unstale":"2023-10-09T05:49:20.415371+0000","last_undegraded":"2023-10-09T05:49:20.415371+0000","last_fullsized":"2023-10-09T05:49:20.415371+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:52.891228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":60,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.429234+0000","last_change":"2023-10-09T05:49:21.429234+0000","last_active":"2023-10-09T05:49:21.429234+0000","last_peered":"2023-10-09T05:49:21.429234+0000","last_clean":"2023-10-09T05:49:21.429234+0000","last_became_active":"2023-10-09T05:49:21.428961+0000","last_became_peered":"2023-10-09T05:49:21.428961+0000","last_unstale":"2023-10-09T05:49:21.429234+0000","last_undegraded":"2023-10-09T05:49:21.429234+0000","last_fullsized":"2023-10-09T05:49:21.429234+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:18.029914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.420226+0000","last_change":"2023-10-09T05:48:49.139877+0000","last_active":"2023-10-09T05:49:20.420226+0000","last_peered":"2023-10-09T05:49:20.420226+0000","last_clean":"2023-10-09T05:49:20.420226+0000","last_became_active":"2023-10-09T05:48:49.139479+0000","last_became_peered":"2023-10-09T05:48:49.139479+0000","last_unstale":"2023-10-09T05:49:20.420226+0000","last_undegraded":"2023-10-09T05:49:20.420226+0000","last_fullsized":"2023-10-09T05:49:20.420226+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:14:16.097335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":44,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.423108+0000","last_change":"2023-10-09T05:40:12.632083+0000","last_active":"2023-10-09T05:49:20.423108+0000","last_peered":"2023-10-09T05:49:20.423108+0000","last_clean":"2023-10-09T05:49:20.423108+0000","last_became_active":"2023-10-09T05:40:12.631715+0000","last_became_peered":"2023-10-09T05:40:12.631715+0000","last_unstale":"2023-10-09T05:49:20.423108+0000","last_undegraded":"2023-10-09T05:49:20.423108+0000","last_fullsized":"2023-10-09T05:49:20.423108+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:17.641378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":76,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426392+0000","last_change":"2023-10-09T05:49:21.426392+0000","last_active":"2023-10-09T05:49:21.426392+0000","last_peered":"2023-10-09T05:49:21.426392+0000","last_clean":"2023-10-09T05:49:21.426392+0000","last_became_active":"2023-10-09T05:49:21.426056+0000","last_became_peered":"2023-10-09T05:49:21.426056+0000","last_unstale":"2023-10-09T05:49:21.426392+0000","last_undegraded":"2023-10-09T05:49:21.426392+0000","last_fullsized":"2023-10-09T05:49:21.426392+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:09.862053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.420742+0000","last_change":"2023-10-09T05:48:17.173680+0000","last_active":"2023-10-09T05:49:20.420742+0000","last_peered":"2023-10-09T05:49:20.420742+0000","last_clean":"2023-10-09T05:49:20.420742+0000","last_became_active":"2023-10-09T05:48:17.173322+0000","last_became_peered":"2023-10-09T05:48:17.173322+0000","last_unstale":"2023-10-09T05:49:20.420742+0000","last_undegraded":"2023-10-09T05:49:20.420742+0000","last_fullsized":"2023-10-09T05:49:20.420742+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:41.005172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":42,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.421877+0000","last_change":"2023-10-09T05:40:14.362515+0000","last_active":"2023-10-09T05:49:20.421877+0000","last_peered":"2023-10-09T05:49:20.421877+0000","last_clean":"2023-10-09T05:49:20.421877+0000","last_became_active":"2023-10-09T05:40:14.362184+0000","last_became_peered":"2023-10-09T05:40:14.362184+0000","last_unstale":"2023-10-09T05:49:20.421877+0000","last_undegraded":"2023-10-09T05:49:20.421877+0000","last_fullsized":"2023-10-09T05:49:20.421877+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:37.681193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.423481+0000","last_change":"2023-10-09T05:47:03.230162+0000","last_active":"2023-10-09T05:49:20.423481+0000","last_peered":"2023-10-09T05:49:20.423481+0000","last_clean":"2023-10-09T05:49:20.423481+0000","last_became_active":"2023-10-09T05:47:03.229695+0000","last_became_peered":"2023-10-09T05:47:03.229695+0000","last_unstale":"2023-10-09T05:49:20.423481+0000","last_undegraded":"2023-10-09T05:49:20.423481+0000","last_fullsized":"2023-10-09T05:49:20.423481+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:13.194065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.427270+0000","last_change":"2023-10-09T05:49:21.427270+0000","last_active":"2023-10-09T05:49:21.427270+0000","last_peered":"2023-10-09T05:49:21.427270+0000","last_clean":"2023-10-09T05:49:21.427270+0000","last_became_active":"2023-10-09T05:49:21.427014+0000","last_became_peered":"2023-10-09T05:49:21.427014+0000","last_unstale":"2023-10-09T05:49:21.427270+0000","last_undegraded":"2023-10-09T05:49:21.427270+0000","last_fullsized":"2023-10-09T05:49:21.427270+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:36:30.822627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":68,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.416655+0000","last_change":"2023-10-09T05:47:03.230438+0000","last_active":"2023-10-09T05:49:20.416655+0000","last_peered":"2023-10-09T05:49:20.416655+0000","last_clean":"2023-10-09T05:49:20.416655+0000","last_became_active":"2023-10-09T05:47:03.229346+0000","last_became_peered":"2023-10-09T05:47:03.229346+0000","last_unstale":"2023-10-09T05:49:20.416655+0000","last_undegraded":"2023-10-09T05:49:20.416655+0000","last_fullsized":"2023-10-09T05:49:20.416655+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:54.948861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426926+0000","last_change":"2023-10-09T05:49:21.426431+0000","last_active":"2023-10-09T05:49:21.426926+0000","last_peered":"2023-10-09T05:49:21.426926+0000","last_clean":"2023-10-09T05:49:21.426926+0000","last_became_active":"2023-10-09T05:49:21.426011+0000","last_became_peered":"2023-10-09T05:49:21.426011+0000","last_unstale":"2023-10-09T05:49:21.426926+0000","last_undegraded":"2023-10-09T05:49:21.426926+0000","last_fullsized":"2023-10-09T05:49:21.426926+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:27.350168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":43,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.419867+0000","last_change":"2023-10-09T05:40:12.631893+0000","last_active":"2023-10-09T05:49:20.419867+0000","last_peered":"2023-10-09T05:49:20.419867+0000","last_clean":"2023-10-09T05:49:20.419867+0000","last_became_active":"2023-10-09T05:40:12.631592+0000","last_became_peered":"2023-10-09T05:40:12.631592+0000","last_unstale":"2023-10-09T05:49:20.419867+0000","last_undegraded":"2023-10-09T05:49:20.419867+0000","last_fullsized":"2023-10-09T05:49:20.419867+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:18:12.517403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":59,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.423218+0000","last_change":"2023-10-09T05:49:21.423218+0000","last_active":"2023-10-09T05:49:21.423218+0000","last_peered":"2023-10-09T05:49:21.423218+0000","last_clean":"2023-10-09T05:49:21.423218+0000","last_became_active":"2023-10-09T05:49:21.422706+0000","last_became_peered":"2023-10-09T05:49:21.422706+0000","last_unstale":"2023-10-09T05:49:21.423218+0000","last_undegraded":"2023-10-09T05:49:21.423218+0000","last_fullsized":"2023-10-09T05:49:21.423218+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:31.571220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426167+0000","last_change":"2023-10-09T05:49:21.426167+0000","last_active":"2023-10-09T05:49:21.426167+0000","last_peered":"2023-10-09T05:49:21.426167+0000","last_clean":"2023-10-09T05:49:21.426167+0000","last_became_active":"2023-10-09T05:49:21.425874+0000","last_became_peered":"2023-10-09T05:49:21.425874+0000","last_unstale":"2023-10-09T05:49:21.426167+0000","last_undegraded":"2023-10-09T05:49:21.426167+0000","last_fullsized":"2023-10-09T05:49:21.426167+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:59.642587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.428386+0000","last_change":"2023-10-09T05:49:21.428386+0000","last_active":"2023-10-09T05:49:21.428386+0000","last_peered":"2023-10-09T05:49:21.428386+0000","last_clean":"2023-10-09T05:49:21.428386+0000","last_became_active":"2023-10-09T05:49:21.428131+0000","last_became_peered":"2023-10-09T05:49:21.428131+0000","last_unstale":"2023-10-09T05:49:21.428386+0000","last_undegraded":"2023-10-09T05:49:21.428386+0000","last_fullsized":"2023-10-09T05:49:21.428386+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:12.858458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.419185+0000","last_change":"2023-10-09T05:48:49.139681+0000","last_active":"2023-10-09T05:49:20.419185+0000","last_peered":"2023-10-09T05:49:20.419185+0000","last_clean":"2023-10-09T05:49:20.419185+0000","last_became_active":"2023-10-09T05:48:49.139317+0000","last_became_peered":"2023-10-09T05:48:49.139317+0000","last_unstale":"2023-10-09T05:49:20.419185+0000","last_undegraded":"2023-10-09T05:49:20.419185+0000","last_fullsized":"2023-10-09T05:49:20.419185+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:34.769043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.413441+0000","last_change":"2023-10-09T05:48:17.173137+0000","last_active":"2023-10-09T05:49:20.413441+0000","last_peered":"2023-10-09T05:49:20.413441+0000","last_clean":"2023-10-09T05:49:20.413441+0000","last_became_active":"2023-10-09T05:48:17.172891+0000","last_became_peered":"2023-10-09T05:48:17.172891+0000","last_unstale":"2023-10-09T05:49:20.413441+0000","last_undegraded":"2023-10-09T05:49:20.413441+0000","last_fullsized":"2023-10-09T05:49:20.413441+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:21.393939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":65,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.413929+0000","last_change":"2023-10-09T05:40:12.630786+0000","last_active":"2023-10-09T05:49:20.413929+0000","last_peered":"2023-10-09T05:49:20.413929+0000","last_clean":"2023-10-09T05:49:20.413929+0000","last_became_active":"2023-10-09T05:40:12.630528+0000","last_became_peered":"2023-10-09T05:40:12.630528+0000","last_unstale":"2023-10-09T05:49:20.413929+0000","last_undegraded":"2023-10-09T05:49:20.413929+0000","last_fullsized":"2023-10-09T05:49:20.413929+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:54.427346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.412474+0000","last_change":"2023-10-09T05:48:49.138162+0000","last_active":"2023-10-09T05:49:20.412474+0000","last_peered":"2023-10-09T05:49:20.412474+0000","last_clean":"2023-10-09T05:49:20.412474+0000","last_became_active":"2023-10-09T05:48:49.137824+0000","last_became_peered":"2023-10-09T05:48:49.137824+0000","last_unstale":"2023-10-09T05:49:20.412474+0000","last_undegraded":"2023-10-09T05:49:20.412474+0000","last_fullsized":"2023-10-09T05:49:20.412474+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:43:26.642768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.430469+0000","last_change":"2023-10-09T05:49:21.430469+0000","last_active":"2023-10-09T05:49:21.430469+0000","last_peered":"2023-10-09T05:49:21.430469+0000","last_clean":"2023-10-09T05:49:21.430469+0000","last_became_active":"2023-10-09T05:49:21.430107+0000","last_became_peered":"2023-10-09T05:49:21.430107+0000","last_unstale":"2023-10-09T05:49:21.430469+0000","last_undegraded":"2023-10-09T05:49:21.430469+0000","last_fullsized":"2023-10-09T05:49:21.430469+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:08.815343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.425819+0000","last_change":"2023-10-09T05:49:21.425819+0000","last_active":"2023-10-09T05:49:21.425819+0000","last_peered":"2023-10-09T05:49:21.425819+0000","last_clean":"2023-10-09T05:49:21.425819+0000","last_became_active":"2023-10-09T05:49:21.425498+0000","last_became_peered":"2023-10-09T05:49:21.425498+0000","last_unstale":"2023-10-09T05:49:21.425819+0000","last_undegraded":"2023-10-09T05:49:21.425819+0000","last_fullsized":"2023-10-09T05:49:21.425819+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:21.443055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.421834+0000","last_change":"2023-10-09T05:48:17.176541+0000","last_active":"2023-10-09T05:49:20.421834+0000","last_peered":"2023-10-09T05:49:20.421834+0000","last_clean":"2023-10-09T05:49:20.421834+0000","last_became_active":"2023-10-09T05:48:17.176213+0000","last_became_peered":"2023-10-09T05:48:17.176213+0000","last_unstale":"2023-10-09T05:49:20.421834+0000","last_undegraded":"2023-10-09T05:49:20.421834+0000","last_fullsized":"2023-10-09T05:49:20.421834+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:23.599435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.427804+0000","last_change":"2023-10-09T05:49:21.427804+0000","last_active":"2023-10-09T05:49:21.427804+0000","last_peered":"2023-10-09T05:49:21.427804+0000","last_clean":"2023-10-09T05:49:21.427804+0000","last_became_active":"2023-10-09T05:49:21.427569+0000","last_became_peered":"2023-10-09T05:49:21.427569+0000","last_unstale":"2023-10-09T05:49:21.427804+0000","last_undegraded":"2023-10-09T05:49:21.427804+0000","last_fullsized":"2023-10-09T05:49:21.427804+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.791826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":53,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.426616+0000","last_change":"2023-10-09T05:49:21.426149+0000","last_active":"2023-10-09T05:49:21.426616+0000","last_peered":"2023-10-09T05:49:21.426616+0000","last_clean":"2023-10-09T05:49:21.426616+0000","last_became_active":"2023-10-09T05:49:21.425805+0000","last_became_peered":"2023-10-09T05:49:21.425805+0000","last_unstale":"2023-10-09T05:49:21.426616+0000","last_undegraded":"2023-10-09T05:49:21.426616+0000","last_fullsized":"2023-10-09T05:49:21.426616+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:39:57.117762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.421197+0000","last_change":"2023-10-09T05:48:17.179524+0000","last_active":"2023-10-09T05:49:20.421197+0000","last_peered":"2023-10-09T05:49:20.421197+0000","last_clean":"2023-10-09T05:49:20.421197+0000","last_became_active":"2023-10-09T05:48:17.179233+0000","last_became_peered":"2023-10-09T05:48:17.179233+0000","last_unstale":"2023-10-09T05:49:20.421197+0000","last_undegraded":"2023-10-09T05:49:20.421197+0000","last_fullsized":"2023-10-09T05:49:20.421197+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:24.135263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":81,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.427745+0000","last_change":"2023-10-09T05:49:21.427745+0000","last_active":"2023-10-09T05:49:21.427745+0000","last_peered":"2023-10-09T05:49:21.427745+0000","last_clean":"2023-10-09T05:49:21.427745+0000","last_became_active":"2023-10-09T05:49:21.427292+0000","last_became_peered":"2023-10-09T05:49:21.427292+0000","last_unstale":"2023-10-09T05:49:21.427745+0000","last_undegraded":"2023-10-09T05:49:21.427745+0000","last_fullsized":"2023-10-09T05:49:21.427745+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:04:24.929346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.427476+0000","last_change":"2023-10-09T05:49:21.427476+0000","last_active":"2023-10-09T05:49:21.427476+0000","last_peered":"2023-10-09T05:49:21.427476+0000","last_clean":"2023-10-09T05:49:21.427476+0000","last_became_active":"2023-10-09T05:49:21.426417+0000","last_became_peered":"2023-10-09T05:49:21.426417+0000","last_unstale":"2023-10-09T05:49:21.427476+0000","last_undegraded":"2023-10-09T05:49:21.427476+0000","last_fullsized":"2023-10-09T05:49:21.427476+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:45:03.848962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.431711+0000","last_change":"2023-10-09T05:49:21.431711+0000","last_active":"2023-10-09T05:49:21.431711+0000","last_peered":"2023-10-09T05:49:21.431711+0000","last_clean":"2023-10-09T05:49:21.431711+0000","last_became_active":"2023-10-09T05:49:21.431441+0000","last_became_peered":"2023-10-09T05:49:21.431441+0000","last_unstale":"2023-10-09T05:49:21.431711+0000","last_undegraded":"2023-10-09T05:49:21.431711+0000","last_fullsized":"2023-10-09T05:49:21.431711+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:26.617516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T05:49:21.427978+0000","last_change":"2023-10-09T05:49:21.427978+0000","last_active":"2023-10-09T05:49:21.427978+0000","last_peered":"2023-10-09T05:49:21.427978+0000","last_clean":"2023-10-09T05:49:21.427978+0000","last_became_active":"2023-10-09T05:49:21.427703+0000","last_became_peered":"2023-10-09T05:49:21.427703+0000","last_unstale":"2023-10-09T05:49:21.427978+0000","last_undegraded":"2023-10-09T05:49:21.427978+0000","last_fullsized":"2023-10-09T05:49:21.427978+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:53:04.116683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":78,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T05:49:20.413362+0000","last_change":"2023-10-09T05:48:49.142429+0000","last_active":"2023-10-09T05:49:20.413362+0000","last_peered":"2023-10-09T05:49:20.413362+0000","last_clean":"2023-10-09T05:49:20.413362+0000","last_became_active":"2023-10-09T05:48:49.141966+0000","last_became_peered":"2023-10-09T05:48:49.141966+0000","last_unstale":"2023-10-09T05:49:20.413362+0000","last_undegraded":"2023-10-09T05:49:20.413362+0000","last_fullsized":"2023-10-09T05:49:20.413362+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:52.195431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234129,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311160,"kb_used_data":1584,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93433992,"statfs":{"total":95995035648,"available":95676407808,"internally_reserved":0,"allocated":1622016,"data_stored":1140338,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.69199999999999995,"15min":0.76100000000000001},"min":{"1min":0.38500000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.74299999999999999,"5min":4.8940000000000001,"15min":4.8940000000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.57999999999999996,"15min":0.60199999999999998},"min":{"1min":0.34799999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.749,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.66300000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.52300000000000002,"15min":0.52400000000000002},"min":{"1min":0.376,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.61199999999999999,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.55400000000000005,"15min":0.55200000000000005},"min":{"1min":0.37,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.71599999999999997,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.40799999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:49:50 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.53200000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:48:53 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.57299999999999995,"15min":0.82899999999999996},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.77100000000000002,"5min":0.82199999999999995,"15min":95.108000000000004},"last":0.629},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58199999999999996,"15min":0.83199999999999996},"min":{"1min":0.42999999999999999,"5min":0.36199999999999999,"15min":0.32800000000000001},"max":{"1min":0.749,"5min":0.94099999999999995,"15min":87.664000000000001},"last":0.374}]},{"osd":4,"last update":"Mon Oct 9 05:49:56 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.48399999999999999,"15min":0.48699999999999999},"min":{"1min":0.313,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":1.135},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.51200000000000001,"15min":0.498},"min":{"1min":0.378,"5min":0.27400000000000002,"15min":0.27100000000000002},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":3.165},"last":0.42099999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:49:56 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.499,"15min":0.499},"min":{"1min":0.33200000000000002,"5min":0.28199999999999997,"15min":0.248},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":2.5990000000000002},"last":0.435},{"interface":"front","average":{"1min":0.622,"5min":0.55700000000000005,"15min":0.54000000000000004},"min":{"1min":0.38800000000000001,"5min":0.34000000000000002,"15min":0.28100000000000003},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":1.0660000000000001},"last":0.68999999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:49:56 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.55900000000000005,"15min":0.54300000000000004},"min":{"1min":0.42399999999999999,"5min":0.317,"15min":0.29599999999999999},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":2.8170000000000002},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.57299999999999995,"15min":0.55100000000000005},"min":{"1min":0.41899999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":3.2589999999999999},"last":0.59799999999999998}]}]},{"osd":6,"up_from":46,"seq":197568495765,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298856,"kb_used_data":1568,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446296,"statfs":{"total":95995035648,"available":95689007104,"internally_reserved":0,"allocated":1605632,"data_stored":1139927,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.57999999999999996,"15min":0.59299999999999997},"min":{"1min":0.40300000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77000000000000002,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.57099999999999995,"15min":0.58899999999999997},"min":{"1min":0.38600000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.67800000000000005,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.33500000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:49:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.45500000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:49:50 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.39900000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:48:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.59999999999999998,"15min":0.56200000000000006},"min":{"1min":0.46100000000000002,"5min":0.38300000000000001,"15min":0.318},"max":{"1min":0.79000000000000004,"5min":0.85199999999999998,"15min":1.1100000000000001},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.57799999999999996,"15min":0.55600000000000005},"min":{"1min":0.40699999999999997,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.77400000000000002,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.433}]},{"osd":4,"last update":"Mon Oct 9 05:49:28 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.48499999999999999,"15min":0.45400000000000001},"min":{"1min":0.35499999999999998,"5min":0.309,"15min":0.26300000000000001},"max":{"1min":0.67300000000000004,"5min":0.71899999999999997,"15min":0.75600000000000001},"last":0.629},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.51700000000000002,"15min":0.48899999999999999},"min":{"1min":0.33400000000000002,"5min":0.29099999999999998,"15min":0.28599999999999998},"max":{"1min":0.75900000000000001,"5min":0.80500000000000005,"15min":0.82799999999999996},"last":0.69699999999999995}]},{"osd":5,"last update":"Mon Oct 9 05:49:28 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.50900000000000001,"15min":0.47199999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.27700000000000002},"max":{"1min":0.77500000000000002,"5min":0.78200000000000003,"15min":0.81299999999999994},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.54500000000000004,"15min":0.51800000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32200000000000001},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85799999999999998},"last":0.73199999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:49:58 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.54700000000000004,"15min":0.53400000000000003},"min":{"1min":0.39800000000000002,"5min":0.316,"15min":0.316},"max":{"1min":0.64800000000000002,"5min":0.78800000000000003,"15min":0.79700000000000004},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.626,"5min":0.627,"15min":0.59799999999999998},"min":{"1min":0.39800000000000002,"5min":0.39600000000000002,"15min":0.35799999999999998},"max":{"1min":0.82299999999999995,"5min":0.88200000000000001,"15min":0.88300000000000001},"last":0.76800000000000002}]}]},{"osd":1,"up_from":89,"seq":382252089371,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28492,"kb_used_data":1028,"kb_used_omap":9,"kb_used_meta":27446,"kb_avail":93716660,"statfs":{"total":95995035648,"available":95965859840,"internally_reserved":0,"allocated":1052672,"data_stored":555584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":28105705},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:50:19 2023","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.35599999999999998,"15min":0.35799999999999998},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.27400000000000002},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.39700000000000002,"15min":0.40200000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.29599999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.5119999999999996,"5min":4.5119999999999996,"15min":4.5119999999999996},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.6139999999999999,"5min":3.6139999999999999,"15min":3.6139999999999999},"last":0.33900000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40100000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:50:19 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.59599999999999997,"15min":0.59299999999999997},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.028,"5min":1.028,"15min":1.028},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60999999999999999,"15min":0.61099999999999999},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.76100000000000001,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.57499999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:50:19 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.45000000000000001,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.498},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.60699999999999998,"15min":0.60899999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.58699999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:50:19 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.61299999999999999,"15min":0.61899999999999999},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.626,"5min":0.63400000000000001,"15min":0.63500000000000001},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.94599999999999995,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.51100000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:50:19 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.65600000000000003,"15min":0.66300000000000003},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.002,"5min":1.002,"15min":1.002},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.82499999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.54200000000000004}]}]},{"osd":0,"up_from":84,"seq":360777252906,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29160,"kb_used_data":1632,"kb_used_omap":28,"kb_used_meta":27491,"kb_avail":93715992,"statfs":{"total":95995035648,"available":95965175808,"internally_reserved":0,"allocated":1671168,"data_stored":1149588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":28151632},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.625,"5min":7.625,"15min":7.625},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":3.8130000000000002,"5min":3.8130000000000002,"15min":3.8130000000000002},"last":0.32400000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:49:49 2023","interfaces":[{"interface":"back","average":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.314},{"interface":"front","average":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.5640000000000001,"5min":7.5640000000000001,"15min":7.5640000000000001},"last":0.35799999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:49:04 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.502,"15min":0.51400000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.51000000000000001,"15min":0.52000000000000002},"min":{"1min":0.33600000000000002,"5min":0.307,"15min":0.307},"max":{"1min":0.63,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.371}]},{"osd":4,"last update":"Mon Oct 9 05:50:04 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.67500000000000004,"15min":0.69499999999999995},"min":{"1min":0.42499999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.69799999999999995,"15min":0.72199999999999998},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.90100000000000002,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.66800000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:50:04 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.66800000000000004,"15min":0.68600000000000005},"min":{"1min":0.42199999999999999,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.496},{"interface":"front","average":{"1min":0.622,"5min":0.68200000000000005,"15min":0.71599999999999997},"min":{"1min":0.498,"5min":0.376,"15min":0.376},"max":{"1min":1.27,"5min":1.27,"15min":1.27},"last":0.63300000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:50:04 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.68999999999999995,"15min":0.72499999999999998},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.622},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.71799999999999997,"15min":0.746},"min":{"1min":0.47499999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.995,"5min":0.995,"15min":0.995},"last":0.72799999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:50:04 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.75700000000000001,"15min":0.78900000000000003},"min":{"1min":0.47799999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.016,"5min":1.016,"15min":1.016},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.73699999999999999,"15min":0.76500000000000001},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.73799999999999999}]}]},{"osd":2,"up_from":94,"seq":403726925845,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28620,"kb_used_data":1028,"kb_used_omap":81,"kb_used_meta":27502,"kb_avail":93716532,"statfs":{"total":95995035648,"available":95965728768,"internally_reserved":0,"allocated":1052672,"data_stored":554321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":28162266},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.501},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.40899999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.45000000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.435}]},{"osd":4,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.63100000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.72799999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.75800000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.748}]}]},{"osd":3,"up_from":99,"seq":425201762318,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28468,"kb_used_data":1004,"kb_used_omap":7,"kb_used_meta":27448,"kb_avail":93716684,"statfs":{"total":95995035648,"available":95965884416,"internally_reserved":0,"allocated":1028096,"data_stored":549876,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":28107368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.37}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38600000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55100000000000005}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46300000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57699999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56200000000000006}]}]},{"osd":4,"up_from":30,"seq":128849019038,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302388,"kb_used_data":1004,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93442764,"statfs":{"total":95995035648,"available":95685390336,"internally_reserved":0,"allocated":1028096,"data_stored":550223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:50:10 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.57699999999999996,"15min":0.59799999999999998},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.61499999999999999,"15min":0.63700000000000001},"min":{"1min":0.35799999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.67000000000000004}]},{"osd":1,"last update":"Mon Oct 9 05:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.57299999999999995}]},{"osd":2,"last update":"Mon Oct 9 05:49:50 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.032,"5min":1.032,"15min":1.032},"last":0.61099999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:48:56 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.60299999999999998,"15min":0.58699999999999997},"min":{"1min":0.443,"5min":0.38,"15min":0.32600000000000001},"max":{"1min":0.76100000000000001,"5min":0.81200000000000006,"15min":0.995},"last":0.627},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.59499999999999997,"15min":0.57899999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.35899999999999999},"max":{"1min":0.71199999999999997,"5min":0.83599999999999997,"15min":0.89800000000000002},"last":0.59699999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:50:10 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.47599999999999998,"15min":0.46000000000000002},"min":{"1min":0.36899999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.66300000000000003,"5min":0.83399999999999996,"15min":0.90500000000000003},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.53000000000000003,"15min":0.498},"min":{"1min":0.33100000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.78000000000000003,"5min":0.82199999999999995,"15min":0.88100000000000001},"last":0.42799999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:49:27 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.50900000000000001,"15min":0.495},"min":{"1min":0.34100000000000003,"5min":0.31,"15min":0.27400000000000002},"max":{"1min":0.64300000000000002,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.55700000000000005,"15min":0.53200000000000003},"min":{"1min":0.34799999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.72499999999999998,"5min":0.86199999999999999,"15min":0.89900000000000002},"last":0.51400000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:50:00 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.58399999999999996,"15min":0.56799999999999995},"min":{"1min":0.41699999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":1.046},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.58699999999999997,"15min":0.58399999999999996},"min":{"1min":0.48199999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.95799999999999996},"last":0.53200000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757401,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306468,"kb_used_data":988,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93438684,"statfs":{"total":95995035648,"available":95681212416,"internally_reserved":0,"allocated":1011712,"data_stored":545967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:50:07 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.57299999999999995,"15min":0.58999999999999997},"min":{"1min":0.35599999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.81100000000000005,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.59199999999999997,"15min":0.61499999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.68899999999999995,"5min":1,"15min":1},"last":0.379}]},{"osd":1,"last update":"Mon Oct 9 05:49:19 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.52000000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:49:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.59199999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:49:07 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.59599999999999997,"15min":1.46},"min":{"1min":0.373,"5min":0.373,"15min":0.36699999999999999},"max":{"1min":0.88700000000000001,"5min":0.98499999999999999,"15min":247.672},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.61399999999999999,"15min":1.4930000000000001},"min":{"1min":0.48399999999999999,"5min":0.36899999999999999,"15min":0.36799999999999999},"max":{"1min":0.82299999999999995,"5min":0.88600000000000001,"15min":247.90000000000001},"last":0.63}]},{"osd":4,"last update":"Mon Oct 9 05:50:07 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.48799999999999999,"15min":0.45600000000000002},"min":{"1min":0.311,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.754,"5min":0.754,"15min":3.5430000000000001},"last":0.62},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.50800000000000001,"15min":0.47599999999999998},"min":{"1min":0.372,"5min":0.29999999999999999,"15min":0.28999999999999998},"max":{"1min":0.66600000000000004,"5min":0.72699999999999998,"15min":1.6990000000000001},"last":0.57499999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:49:24 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.53000000000000003,"15min":0.48799999999999999},"min":{"1min":0.41899999999999998,"5min":0.307,"15min":0.29899999999999999},"max":{"1min":0.69799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.438},{"interface":"front","average":{"1min":0.622,"5min":0.57799999999999996,"15min":0.52200000000000002},"min":{"1min":0.39300000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.80600000000000005,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.66300000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:50:03 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.60799999999999998,"15min":0.56299999999999994},"min":{"1min":0.435,"5min":0.32700000000000001,"15min":0.315},"max":{"1min":0.81000000000000005,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.58799999999999997,"15min":0.59799999999999998},"min":{"1min":0.47399999999999998,"5min":0.35099999999999998,"15min":0.34899999999999998},"max":{"1min":0.79400000000000004,"5min":0.79700000000000004,"15min":3.2869999999999999},"last":0.65000000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:50:24.183 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T05:50:24.183 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd dump 2023-10-09T05:50:24.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:24 smithi156 ceph-mon[195101]: pgmap v215: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:24.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:24 smithi064 ceph-mon[212750]: pgmap v215: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:24.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:24 smithi064 ceph-mon[220044]: pgmap v215: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:24.542 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:25.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:25 smithi156 ceph-mon[195101]: from='client.34611 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:50:25.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:25 smithi064 ceph-mon[212750]: from='client.34611 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:50:25.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:25 smithi064 ceph-mon[220044]: from='client.34611 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:50:25.470 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:50:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:50:25] "GET /metrics HTTP/1.1" 200 33904 "" "Prometheus/2.43.0" 2023-10-09T05:50:26.266 INFO:teuthology.orchestra.run.smithi064.stdout:epoch 100 2023-10-09T05:50:26.266 INFO:teuthology.orchestra.run.smithi064.stdout:fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 2023-10-09T05:50:26.266 INFO:teuthology.orchestra.run.smithi064.stdout:created 2023-10-09T05:33:40.037198+0000 2023-10-09T05:50:26.266 INFO:teuthology.orchestra.run.smithi064.stdout:modified 2023-10-09T05:49:21.406990+0000 2023-10-09T05:50:26.266 INFO:teuthology.orchestra.run.smithi064.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-10-09T05:50:26.266 INFO:teuthology.orchestra.run.smithi064.stdout:crush_version 22 2023-10-09T05:50:26.267 INFO:teuthology.orchestra.run.smithi064.stdout:full_ratio 0.95 2023-10-09T05:50:26.267 INFO:teuthology.orchestra.run.smithi064.stdout:backfillfull_ratio 0.9 2023-10-09T05:50:26.267 INFO:teuthology.orchestra.run.smithi064.stdout:nearfull_ratio 0.85 2023-10-09T05:50:26.267 INFO:teuthology.orchestra.run.smithi064.stdout:require_min_compat_client luminous 2023-10-09T05:50:26.267 INFO:teuthology.orchestra.run.smithi064.stdout:min_compat_client luminous 2023-10-09T05:50:26.267 INFO:teuthology.orchestra.run.smithi064.stdout:require_osd_release quincy 2023-10-09T05:50:26.267 INFO:teuthology.orchestra.run.smithi064.stdout:stretch_mode_enabled false 2023-10-09T05:50:26.267 INFO:teuthology.orchestra.run.smithi064.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 21 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2023-10-09T05:50:26.267 INFO:teuthology.orchestra.run.smithi064.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 63 flags hashpspool stripe_width 0 application rbd read_balance_score 2.00 2023-10-09T05:50:26.267 INFO:teuthology.orchestra.run.smithi064.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 67 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-10-09T05:50:26.268 INFO:teuthology.orchestra.run.smithi064.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 68 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2023-10-09T05:50:26.268 INFO:teuthology.orchestra.run.smithi064.stdout:max_osd 8 2023-10-09T05:50:26.268 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 up in weight 1 up_from 84 up_thru 99 down_at 81 last_clean_interval [8,80) [v2:172.21.15.64:6802/3368956113,v1:172.21.15.64:6803/3368956113] [v2:172.21.15.64:6804/3368956113,v1:172.21.15.64:6805/3368956113] exists,up 85d0857a-ca32-43d2-8cec-bbd916db9f98 2023-10-09T05:50:26.268 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 up in weight 1 up_from 89 up_thru 99 down_at 86 last_clean_interval [13,85) [v2:172.21.15.64:6810/3188144294,v1:172.21.15.64:6811/3188144294] [v2:172.21.15.64:6812/3188144294,v1:172.21.15.64:6813/3188144294] exists,up 05570845-2cd8-4e24-a2f5-f46cf67aa31a 2023-10-09T05:50:26.268 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 up in weight 1 up_from 94 up_thru 99 down_at 91 last_clean_interval [18,90) [v2:172.21.15.64:6818/3375454963,v1:172.21.15.64:6819/3375454963] [v2:172.21.15.64:6820/3375454963,v1:172.21.15.64:6821/3375454963] exists,up 0f306460-9f08-4f15-b573-3587f63be61c 2023-10-09T05:50:26.268 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 up in weight 1 up_from 99 up_thru 99 down_at 96 last_clean_interval [25,95) [v2:172.21.15.64:6826/1043927146,v1:172.21.15.64:6827/1043927146] [v2:172.21.15.64:6828/1043927146,v1:172.21.15.64:6829/1043927146] exists,up 4bf38edd-e555-402e-ba4b-c751de06aa6a 2023-10-09T05:50:26.268 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 up in weight 1 up_from 30 up_thru 99 down_at 0 last_clean_interval [0,0) [v2:172.21.15.156:6800/657648738,v1:172.21.15.156:6801/657648738] [v2:172.21.15.156:6802/657648738,v1:172.21.15.156:6803/657648738] exists,up 060e8766-7bf6-42f1-bebe-dae6828f7fb9 2023-10-09T05:50:26.268 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 up in weight 1 up_from 38 up_thru 99 down_at 0 last_clean_interval [0,0) [v2:172.21.15.156:6808/726279450,v1:172.21.15.156:6809/726279450] [v2:172.21.15.156:6810/726279450,v1:172.21.15.156:6811/726279450] exists,up 6b5cf38a-6792-4ac7-9694-268601260ba8 2023-10-09T05:50:26.268 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 up in weight 1 up_from 46 up_thru 96 down_at 0 last_clean_interval [0,0) [v2:172.21.15.156:6816/2369965428,v1:172.21.15.156:6817/2369965428] [v2:172.21.15.156:6818/2369965428,v1:172.21.15.156:6819/2369965428] exists,up 01deea2d-a225-40b8-a54b-61eee57da527 2023-10-09T05:50:26.268 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 up in weight 1 up_from 54 up_thru 99 down_at 0 last_clean_interval [0,0) [v2:172.21.15.156:6824/2767546322,v1:172.21.15.156:6825/2767546322] [v2:172.21.15.156:6826/2767546322,v1:172.21.15.156:6827/2767546322] exists,up 693e93b0-c235-47fe-9172-85af68e38634 2023-10-09T05:50:26.268 INFO:teuthology.orchestra.run.smithi064.stdout:pg_upmap_items 4.4 [1,5] 2023-10-09T05:50:26.268 INFO:teuthology.orchestra.run.smithi064.stdout:pg_upmap_items 4.7 [1,2] 2023-10-09T05:50:26.269 INFO:teuthology.orchestra.run.smithi064.stdout:pg_upmap_items 4.a [1,2] 2023-10-09T05:50:26.269 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/390193900 expires 2023-10-10T05:43:58.314769+0000 2023-10-09T05:50:26.269 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/1951412611 expires 2023-10-10T05:42:37.227748+0000 2023-10-09T05:50:26.269 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.156:0/698211702 expires 2023-10-10T05:43:16.303186+0000 2023-10-09T05:50:26.269 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/514741379 expires 2023-10-10T05:34:09.281655+0000 2023-10-09T05:50:26.269 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/3707401615 expires 2023-10-10T05:39:05.525329+0000 2023-10-09T05:50:26.269 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6801/3793931367 expires 2023-10-10T05:34:09.281655+0000 2023-10-09T05:50:26.269 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6800/3793931367 expires 2023-10-10T05:34:09.281655+0000 2023-10-09T05:50:26.269 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/1562509700 expires 2023-10-10T05:34:35.744792+0000 2023-10-09T05:50:26.269 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6800/1111090977 expires 2023-10-10T05:43:58.314769+0000 2023-10-09T05:50:26.269 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/2324320735 expires 2023-10-10T05:43:58.314769+0000 2023-10-09T05:50:26.269 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/1766272185 expires 2023-10-10T05:43:58.314769+0000 2023-10-09T05:50:26.270 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6801/984813922 expires 2023-10-10T05:42:37.227748+0000 2023-10-09T05:50:26.270 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/3185620259 expires 2023-10-10T05:39:05.525329+0000 2023-10-09T05:50:26.270 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/4054596417 expires 2023-10-10T05:34:35.744792+0000 2023-10-09T05:50:26.270 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6800/3627852432 expires 2023-10-10T05:39:05.525329+0000 2023-10-09T05:50:26.270 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/1571829871 expires 2023-10-10T05:42:37.227748+0000 2023-10-09T05:50:26.270 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6800/984813922 expires 2023-10-10T05:42:37.227748+0000 2023-10-09T05:50:26.270 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/3038747921 expires 2023-10-10T05:42:37.227748+0000 2023-10-09T05:50:26.270 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/2171400226 expires 2023-10-10T05:34:09.281655+0000 2023-10-09T05:50:26.270 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/3346006057 expires 2023-10-10T05:34:09.281655+0000 2023-10-09T05:50:26.270 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.156:0/1526589339 expires 2023-10-10T05:43:16.303186+0000 2023-10-09T05:50:26.270 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.156:0/2936069725 expires 2023-10-10T05:43:16.303186+0000 2023-10-09T05:50:26.271 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/1336186767 expires 2023-10-10T05:39:05.525329+0000 2023-10-09T05:50:26.271 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/1900245998 expires 2023-10-10T05:34:35.744792+0000 2023-10-09T05:50:26.271 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6801/1111090977 expires 2023-10-10T05:43:58.314769+0000 2023-10-09T05:50:26.271 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6801/3627852432 expires 2023-10-10T05:39:05.525329+0000 2023-10-09T05:50:26.271 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/584000461 expires 2023-10-10T05:43:58.314769+0000 2023-10-09T05:50:26.271 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6801/1833729701 expires 2023-10-10T05:34:35.744792+0000 2023-10-09T05:50:26.271 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/3029574061 expires 2023-10-10T05:42:37.227748+0000 2023-10-09T05:50:26.271 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.156:6834/3461673683 expires 2023-10-10T05:43:16.303186+0000 2023-10-09T05:50:26.271 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/479050685 expires 2023-10-10T05:43:58.314769+0000 2023-10-09T05:50:26.271 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.156:0/4113996764 expires 2023-10-10T05:43:16.303186+0000 2023-10-09T05:50:26.271 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.156:6835/3461673683 expires 2023-10-10T05:43:16.303186+0000 2023-10-09T05:50:26.271 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.156:0/1457402607 expires 2023-10-10T05:43:16.303186+0000 2023-10-09T05:50:26.272 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.156:0/3281795067 expires 2023-10-10T05:43:16.303186+0000 2023-10-09T05:50:26.272 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/2115451128 expires 2023-10-10T05:39:05.525329+0000 2023-10-09T05:50:26.272 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6800/1833729701 expires 2023-10-10T05:34:35.744792+0000 2023-10-09T05:50:26.272 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/2736837524 expires 2023-10-10T05:43:58.314769+0000 2023-10-09T05:50:26.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:26 smithi156 ceph-mon[195101]: pgmap v216: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:26.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:26 smithi064 ceph-mon[212750]: pgmap v216: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:26.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:26 smithi064 ceph-mon[220044]: pgmap v216: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:27.035 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 84 up_thru 99 down_at 81 last_clean_interval [8,80) [v2:172.21.15.64:6802/3368956113,v1:172.21.15.64:6803/3368956113] [v2:172.21.15.64:6804/3368956113,v1:172.21.15.64:6805/3368956113] exists,up 85d0857a-ca32-43d2-8cec-bbd916db9f98', 'osd.1 up in weight 1 up_from 89 up_thru 99 down_at 86 last_clean_interval [13,85) [v2:172.21.15.64:6810/3188144294,v1:172.21.15.64:6811/3188144294] [v2:172.21.15.64:6812/3188144294,v1:172.21.15.64:6813/3188144294] exists,up 05570845-2cd8-4e24-a2f5-f46cf67aa31a', 'osd.2 up in weight 1 up_from 94 up_thru 99 down_at 91 last_clean_interval [18,90) [v2:172.21.15.64:6818/3375454963,v1:172.21.15.64:6819/3375454963] [v2:172.21.15.64:6820/3375454963,v1:172.21.15.64:6821/3375454963] exists,up 0f306460-9f08-4f15-b573-3587f63be61c', 'osd.3 up in weight 1 up_from 99 up_thru 99 down_at 96 last_clean_interval [25,95) [v2:172.21.15.64:6826/1043927146,v1:172.21.15.64:6827/1043927146] [v2:172.21.15.64:6828/1043927146,v1:172.21.15.64:6829/1043927146] exists,up 4bf38edd-e555-402e-ba4b-c751de06aa6a', 'osd.4 up in weight 1 up_from 30 up_thru 99 down_at 0 last_clean_interval [0,0) [v2:172.21.15.156:6800/657648738,v1:172.21.15.156:6801/657648738] [v2:172.21.15.156:6802/657648738,v1:172.21.15.156:6803/657648738] exists,up 060e8766-7bf6-42f1-bebe-dae6828f7fb9', 'osd.5 up in weight 1 up_from 38 up_thru 99 down_at 0 last_clean_interval [0,0) [v2:172.21.15.156:6808/726279450,v1:172.21.15.156:6809/726279450] [v2:172.21.15.156:6810/726279450,v1:172.21.15.156:6811/726279450] exists,up 6b5cf38a-6792-4ac7-9694-268601260ba8', 'osd.6 up in weight 1 up_from 46 up_thru 96 down_at 0 last_clean_interval [0,0) [v2:172.21.15.156:6816/2369965428,v1:172.21.15.156:6817/2369965428] [v2:172.21.15.156:6818/2369965428,v1:172.21.15.156:6819/2369965428] exists,up 01deea2d-a225-40b8-a54b-61eee57da527', 'osd.7 up in weight 1 up_from 54 up_thru 99 down_at 0 last_clean_interval [0,0) [v2:172.21.15.156:6824/2767546322,v1:172.21.15.156:6825/2767546322] [v2:172.21.15.156:6826/2767546322,v1:172.21.15.156:6827/2767546322] exists,up 693e93b0-c235-47fe-9172-85af68e38634'] 2023-10-09T05:50:27.035 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon mon.a config show 2023-10-09T05:50:27.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:27 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/280513364' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:50:27.397 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:27.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:27 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/280513364' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:50:27.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:27 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/280513364' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:50:28.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:28 smithi156 ceph-mon[195101]: pgmap v217: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:28.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:28 smithi064 ceph-mon[212750]: pgmap v217: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:28.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:28 smithi064 ceph-mon[220044]: pgmap v217: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:28.948 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:50:28.948 INFO:teuthology.orchestra.run.smithi064.stdout: "name": "mon.a", 2023-10-09T05:50:28.948 INFO:teuthology.orchestra.run.smithi064.stdout: "cluster": "ceph", 2023-10-09T05:50:28.948 INFO:teuthology.orchestra.run.smithi064.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-10-09T05:50:28.948 INFO:teuthology.orchestra.run.smithi064.stdout: "admin_socket_mode": "", 2023-10-09T05:50:28.948 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-10-09T05:50:28.948 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_client_required": "cephx, none", 2023-10-09T05:50:28.949 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_cluster_required": "cephx", 2023-10-09T05:50:28.949 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_debug": "true", 2023-10-09T05:50:28.949 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-10-09T05:50:28.949 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-10-09T05:50:28.949 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_service_required": "cephx", 2023-10-09T05:50:28.949 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_service_ticket_ttl": "240.000000", 2023-10-09T05:50:28.949 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_supported": "", 2023-10-09T05:50:28.949 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_aio": "true", 2023-10-09T05:50:28.949 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_aio_max_queue_depth": "1024", 2023-10-09T05:50:28.949 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_aio_poll_ms": "250", 2023-10-09T05:50:28.949 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_aio_reap_max": "16", 2023-10-09T05:50:28.949 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_async_discard": "false", 2023-10-09T05:50:28.950 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_block_size": "4096", 2023-10-09T05:50:28.950 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_debug_aio": "false", 2023-10-09T05:50:28.950 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-10-09T05:50:28.950 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-10-09T05:50:28.950 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_debug_inflight_ios": "false", 2023-10-09T05:50:28.950 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_enable_discard": "false", 2023-10-09T05:50:28.950 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_flock_retry": "3", 2023-10-09T05:50:28.950 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_flock_retry_interval": "0.100000", 2023-10-09T05:50:28.950 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_inject_crash": "0", 2023-10-09T05:50:28.950 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_inject_crash_flush_delay": "2", 2023-10-09T05:50:28.950 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_ioring": "false", 2023-10-09T05:50:28.950 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_ioring_hipri": "false", 2023-10-09T05:50:28.951 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_ioring_sqthread_poll": "false", 2023-10-09T05:50:28.951 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-10-09T05:50:28.951 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_read_buffer_alignment": "4096", 2023-10-09T05:50:28.951 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-10-09T05:50:28.951 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_type": "", 2023-10-09T05:50:28.951 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_alloc_size": "1048576", 2023-10-09T05:50:28.951 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_allocator": "hybrid", 2023-10-09T05:50:28.951 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_buffered_io": "true", 2023-10-09T05:50:28.951 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_check_for_zeros": "false", 2023-10-09T05:50:28.951 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_check_volume_selector_often": "false", 2023-10-09T05:50:28.951 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-10-09T05:50:28.951 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_compact_log_sync": "false", 2023-10-09T05:50:28.952 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-10-09T05:50:28.952 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-10-09T05:50:28.952 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_log_compact_min_size": "16777216", 2023-10-09T05:50:28.952 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_log_replay_check_allocations": "true", 2023-10-09T05:50:28.952 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_max_log_runway": "4194304", 2023-10-09T05:50:28.952 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_max_prefetch": "1048576", 2023-10-09T05:50:28.952 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_min_flush_size": "524288", 2023-10-09T05:50:28.952 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_min_log_runway": "1048576", 2023-10-09T05:50:28.952 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_replay_recovery": "false", 2023-10-09T05:50:28.952 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-10-09T05:50:28.952 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_shared_alloc_size": "65536", 2023-10-09T05:50:28.952 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_sync_write": "false", 2023-10-09T05:50:28.953 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-10-09T05:50:28.953 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-10-09T05:50:28.953 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-10-09T05:50:28.953 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_allocation_from_file": "true", 2023-10-09T05:50:28.953 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_allocator": "hybrid", 2023-10-09T05:50:28.953 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-10-09T05:50:28.953 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-10-09T05:50:28.953 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-10-09T05:50:28.953 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-10-09T05:50:28.953 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-10-09T05:50:28.953 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-10-09T05:50:28.953 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_blobid_prealloc": "10240", 2023-10-09T05:50:28.953 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_create": "true", 2023-10-09T05:50:28.954 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_db_create": "false", 2023-10-09T05:50:28.954 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_db_path": "", 2023-10-09T05:50:28.954 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_db_size": "0", 2023-10-09T05:50:28.954 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_path": "", 2023-10-09T05:50:28.954 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_preallocate_file": "false", 2023-10-09T05:50:28.954 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_size": "107374182400", 2023-10-09T05:50:28.954 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_wal_create": "false", 2023-10-09T05:50:28.954 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_wal_path": "", 2023-10-09T05:50:28.954 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_wal_size": "100663296", 2023-10-09T05:50:28.954 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_bluefs": "true", 2023-10-09T05:50:28.954 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-10-09T05:50:28.954 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_bluefs_env_mirror": "false", 2023-10-09T05:50:28.955 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-10-09T05:50:28.955 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-10-09T05:50:28.955 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T05:50:28.955 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T05:50:28.955 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-10-09T05:50:28.955 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T05:50:28.955 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_autotune": "true", 2023-10-09T05:50:28.955 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-10-09T05:50:28.955 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-10-09T05:50:28.955 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-10-09T05:50:28.955 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-10-09T05:50:28.955 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_size": "0", 2023-10-09T05:50:28.955 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-10-09T05:50:28.956 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-10-09T05:50:28.956 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-10-09T05:50:28.956 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-10-09T05:50:28.956 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_type": "2q", 2023-10-09T05:50:28.956 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-10-09T05:50:28.956 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_clone_cow": "true", 2023-10-09T05:50:28.956 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_algorithm": "snappy", 2023-10-09T05:50:28.956 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_max_blob_size": "0", 2023-10-09T05:50:28.956 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-10-09T05:50:28.956 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-10-09T05:50:28.956 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_min_blob_size": "0", 2023-10-09T05:50:28.956 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-10-09T05:50:28.957 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-10-09T05:50:28.957 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_mode": "none", 2023-10-09T05:50:28.957 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-10-09T05:50:28.957 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_csum_type": "crc32c", 2023-10-09T05:50:28.957 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_enforce_settings": "default", 2023-10-09T05:50:28.957 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_freelist": "false", 2023-10-09T05:50:28.957 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_fsck_abort": "false", 2023-10-09T05:50:28.957 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-10-09T05:50:28.957 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_inject_bug21040": "false", 2023-10-09T05:50:28.957 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-10-09T05:50:28.957 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_inject_read_err": "false", 2023-10-09T05:50:28.957 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_legacy_omap": "false", 2023-10-09T05:50:28.957 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-10-09T05:50:28.958 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-10-09T05:50:28.958 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-10-09T05:50:28.958 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-10-09T05:50:28.958 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_prefill": "0.000000", 2023-10-09T05:50:28.958 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-10-09T05:50:28.958 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-10-09T05:50:28.958 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-10-09T05:50:28.958 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_small_allocations": "0", 2023-10-09T05:50:28.958 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-10-09T05:50:28.958 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_default_buffered_read": "true", 2023-10-09T05:50:28.958 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_default_buffered_write": "false", 2023-10-09T05:50:28.958 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_deferred_batch_ops": "0", 2023-10-09T05:50:28.959 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-10-09T05:50:28.959 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-10-09T05:50:28.959 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-10-09T05:50:28.959 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-10-09T05:50:28.959 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-10-09T05:50:28.959 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-10-09T05:50:28.959 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-10-09T05:50:28.959 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fail_eio": "false", 2023-10-09T05:50:28.959 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-10-09T05:50:28.959 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-10-09T05:50:28.959 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-10-09T05:50:28.959 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-10-09T05:50:28.960 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_on_mkfs": "true", 2023-10-09T05:50:28.960 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-10-09T05:50:28.960 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_on_mount": "false", 2023-10-09T05:50:28.960 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-10-09T05:50:28.960 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_on_umount": "false", 2023-10-09T05:50:28.960 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-10-09T05:50:28.960 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-10-09T05:50:28.960 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-10-09T05:50:28.960 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-10-09T05:50:28.960 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-10-09T05:50:28.960 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-10-09T05:50:28.960 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-10-09T05:50:28.961 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-10-09T05:50:28.961 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_ignore_data_csum": "false", 2023-10-09T05:50:28.961 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-10-09T05:50:28.961 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_kvbackend": "rocksdb", 2023-10-09T05:50:28.961 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-10-09T05:50:28.961 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-10-09T05:50:28.961 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_log_op_age": "5.000000", 2023-10-09T05:50:28.961 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_max_alloc_size": "0", 2023-10-09T05:50:28.961 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_max_blob_size": "0", 2023-10-09T05:50:28.961 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-10-09T05:50:28.961 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-10-09T05:50:28.962 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_max_defer_interval": "3.000000", 2023-10-09T05:50:28.962 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_max_deferred_txc": "32", 2023-10-09T05:50:28.962 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_min_alloc_size": "0", 2023-10-09T05:50:28.962 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-10-09T05:50:28.962 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-10-09T05:50:28.962 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_nid_prealloc": "1024", 2023-10-09T05:50:28.962 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_prefer_deferred_size": "0", 2023-10-09T05:50:28.962 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-10-09T05:50:28.962 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-10-09T05:50:28.962 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_qfsck_on_mount": "true", 2023-10-09T05:50:28.962 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_retry_disk_reads": "3", 2023-10-09T05:50:28.962 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_rocksdb_cf": "true", 2023-10-09T05:50:28.963 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-10-09T05:50:28.963 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-10-09T05:50:28.963 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_rocksdb_options_annex": "", 2023-10-09T05:50:28.963 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_spdk_coremask": "0x1", 2023-10-09T05:50:28.963 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_spdk_io_sleep": "5", 2023-10-09T05:50:28.963 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_spdk_max_io_completion": "0", 2023-10-09T05:50:28.963 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_spdk_mem": "512", 2023-10-09T05:50:28.963 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_sync_submit_transaction": "false", 2023-10-09T05:50:28.963 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_throttle_bytes": "67108864", 2023-10-09T05:50:28.964 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_throttle_cost_per_io": "0", 2023-10-09T05:50:28.964 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-10-09T05:50:28.964 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-10-09T05:50:28.964 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-10-09T05:50:28.964 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-10-09T05:50:28.964 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_tracing": "false", 2023-10-09T05:50:28.964 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-10-09T05:50:28.964 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-10-09T05:50:28.964 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_volume_selection_reserved": "0", 2023-10-09T05:50:28.964 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-10-09T05:50:28.965 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-10-09T05:50:28.965 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-10-09T05:50:28.965 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-10-09T05:50:28.965 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-10-09T05:50:28.965 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-10-09T05:50:28.965 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_zero_block_detection": "false", 2023-10-09T05:50:28.965 INFO:teuthology.orchestra.run.smithi064.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-10-09T05:50:28.965 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_action_update_interval": "2", 2023-10-09T05:50:28.965 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-10-09T05:50:28.965 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-10-09T05:50:28.965 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-10-09T05:50:28.966 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-10-09T05:50:28.966 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_mount_timeout": "10", 2023-10-09T05:50:28.966 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-10-09T05:50:28.966 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-10-09T05:50:28.966 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-10-09T05:50:28.966 INFO:teuthology.orchestra.run.smithi064.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-10-09T05:50:28.966 INFO:teuthology.orchestra.run.smithi064.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-10-09T05:50:28.966 INFO:teuthology.orchestra.run.smithi064.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-10-09T05:50:28.966 INFO:teuthology.orchestra.run.smithi064.stdout: "cephx_cluster_require_signatures": "false", 2023-10-09T05:50:28.966 INFO:teuthology.orchestra.run.smithi064.stdout: "cephx_cluster_require_version": "2", 2023-10-09T05:50:28.966 INFO:teuthology.orchestra.run.smithi064.stdout: "cephx_require_signatures": "false", 2023-10-09T05:50:28.967 INFO:teuthology.orchestra.run.smithi064.stdout: "cephx_require_version": "2", 2023-10-09T05:50:28.967 INFO:teuthology.orchestra.run.smithi064.stdout: "cephx_service_require_signatures": "false", 2023-10-09T05:50:28.967 INFO:teuthology.orchestra.run.smithi064.stdout: "cephx_service_require_version": "2", 2023-10-09T05:50:28.967 INFO:teuthology.orchestra.run.smithi064.stdout: "cephx_sign_messages": "true", 2023-10-09T05:50:28.967 INFO:teuthology.orchestra.run.smithi064.stdout: "chdir": "", 2023-10-09T05:50:28.967 INFO:teuthology.orchestra.run.smithi064.stdout: "client_acl_type": "", 2023-10-09T05:50:28.967 INFO:teuthology.orchestra.run.smithi064.stdout: "client_asio_thread_count": "2", 2023-10-09T05:50:28.967 INFO:teuthology.orchestra.run.smithi064.stdout: "client_cache_mid": "0.750000", 2023-10-09T05:50:28.967 INFO:teuthology.orchestra.run.smithi064.stdout: "client_cache_size": "16384", 2023-10-09T05:50:28.967 INFO:teuthology.orchestra.run.smithi064.stdout: "client_caps_release_delay": "5", 2023-10-09T05:50:28.967 INFO:teuthology.orchestra.run.smithi064.stdout: "client_check_pool_perm": "true", 2023-10-09T05:50:28.967 INFO:teuthology.orchestra.run.smithi064.stdout: "client_collect_and_send_global_metrics": "false", 2023-10-09T05:50:28.967 INFO:teuthology.orchestra.run.smithi064.stdout: "client_debug_force_sync_read": "false", 2023-10-09T05:50:28.968 INFO:teuthology.orchestra.run.smithi064.stdout: "client_debug_getattr_caps": "false", 2023-10-09T05:50:28.968 INFO:teuthology.orchestra.run.smithi064.stdout: "client_debug_inject_tick_delay": "0", 2023-10-09T05:50:28.968 INFO:teuthology.orchestra.run.smithi064.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-10-09T05:50:28.968 INFO:teuthology.orchestra.run.smithi064.stdout: "client_die_on_failed_remount": "false", 2023-10-09T05:50:28.968 INFO:teuthology.orchestra.run.smithi064.stdout: "client_dirsize_rbytes": "true", 2023-10-09T05:50:28.968 INFO:teuthology.orchestra.run.smithi064.stdout: "client_force_lazyio": "false", 2023-10-09T05:50:28.968 INFO:teuthology.orchestra.run.smithi064.stdout: "client_fs": "", 2023-10-09T05:50:28.968 INFO:teuthology.orchestra.run.smithi064.stdout: "client_inject_fixed_oldest_tid": "false", 2023-10-09T05:50:28.968 INFO:teuthology.orchestra.run.smithi064.stdout: "client_inject_release_failure": "false", 2023-10-09T05:50:28.968 INFO:teuthology.orchestra.run.smithi064.stdout: "client_max_inline_size": "4096", 2023-10-09T05:50:28.968 INFO:teuthology.orchestra.run.smithi064.stdout: "client_max_retries_on_remount_failure": "5", 2023-10-09T05:50:28.968 INFO:teuthology.orchestra.run.smithi064.stdout: "client_mds_namespace": "", 2023-10-09T05:50:28.969 INFO:teuthology.orchestra.run.smithi064.stdout: "client_metadata": "", 2023-10-09T05:50:28.969 INFO:teuthology.orchestra.run.smithi064.stdout: "client_mount_gid": "-1", 2023-10-09T05:50:28.969 INFO:teuthology.orchestra.run.smithi064.stdout: "client_mount_timeout": "300", 2023-10-09T05:50:28.969 INFO:teuthology.orchestra.run.smithi064.stdout: "client_mount_uid": "-1", 2023-10-09T05:50:28.969 INFO:teuthology.orchestra.run.smithi064.stdout: "client_mountpoint": "/", 2023-10-09T05:50:28.969 INFO:teuthology.orchestra.run.smithi064.stdout: "client_notify_timeout": "10", 2023-10-09T05:50:28.969 INFO:teuthology.orchestra.run.smithi064.stdout: "client_oc": "true", 2023-10-09T05:50:28.969 INFO:teuthology.orchestra.run.smithi064.stdout: "client_oc_max_dirty": "104857600", 2023-10-09T05:50:28.969 INFO:teuthology.orchestra.run.smithi064.stdout: "client_oc_max_dirty_age": "5.000000", 2023-10-09T05:50:28.969 INFO:teuthology.orchestra.run.smithi064.stdout: "client_oc_max_objects": "1000", 2023-10-09T05:50:28.969 INFO:teuthology.orchestra.run.smithi064.stdout: "client_oc_size": "209715200", 2023-10-09T05:50:28.969 INFO:teuthology.orchestra.run.smithi064.stdout: "client_oc_target_dirty": "8388608", 2023-10-09T05:50:28.970 INFO:teuthology.orchestra.run.smithi064.stdout: "client_permissions": "true", 2023-10-09T05:50:28.970 INFO:teuthology.orchestra.run.smithi064.stdout: "client_quota": "true", 2023-10-09T05:50:28.970 INFO:teuthology.orchestra.run.smithi064.stdout: "client_quota_df": "true", 2023-10-09T05:50:28.970 INFO:teuthology.orchestra.run.smithi064.stdout: "client_readahead_max_bytes": "0", 2023-10-09T05:50:28.970 INFO:teuthology.orchestra.run.smithi064.stdout: "client_readahead_max_periods": "4", 2023-10-09T05:50:28.970 INFO:teuthology.orchestra.run.smithi064.stdout: "client_readahead_min": "131072", 2023-10-09T05:50:28.970 INFO:teuthology.orchestra.run.smithi064.stdout: "client_reconnect_stale": "false", 2023-10-09T05:50:28.970 INFO:teuthology.orchestra.run.smithi064.stdout: "client_shutdown_timeout": "30", 2023-10-09T05:50:28.970 INFO:teuthology.orchestra.run.smithi064.stdout: "client_snapdir": ".snap", 2023-10-09T05:50:28.970 INFO:teuthology.orchestra.run.smithi064.stdout: "client_tick_interval": "1", 2023-10-09T05:50:28.970 INFO:teuthology.orchestra.run.smithi064.stdout: "client_trace": "", 2023-10-09T05:50:28.970 INFO:teuthology.orchestra.run.smithi064.stdout: "client_try_dentry_invalidate": "false", 2023-10-09T05:50:28.971 INFO:teuthology.orchestra.run.smithi064.stdout: "client_use_faked_inos": "false", 2023-10-09T05:50:28.971 INFO:teuthology.orchestra.run.smithi064.stdout: "client_use_random_mds": "false", 2023-10-09T05:50:28.971 INFO:teuthology.orchestra.run.smithi064.stdout: "clog_to_graylog": "false", 2023-10-09T05:50:28.971 INFO:teuthology.orchestra.run.smithi064.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-10-09T05:50:28.971 INFO:teuthology.orchestra.run.smithi064.stdout: "clog_to_graylog_port": "12201", 2023-10-09T05:50:28.971 INFO:teuthology.orchestra.run.smithi064.stdout: "clog_to_monitors": "default=true", 2023-10-09T05:50:28.971 INFO:teuthology.orchestra.run.smithi064.stdout: "clog_to_syslog": "false", 2023-10-09T05:50:28.971 INFO:teuthology.orchestra.run.smithi064.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-10-09T05:50:28.971 INFO:teuthology.orchestra.run.smithi064.stdout: "clog_to_syslog_level": "info", 2023-10-09T05:50:28.971 INFO:teuthology.orchestra.run.smithi064.stdout: "cluster_addr": "-", 2023-10-09T05:50:28.971 INFO:teuthology.orchestra.run.smithi064.stdout: "cluster_network": "", 2023-10-09T05:50:28.971 INFO:teuthology.orchestra.run.smithi064.stdout: "cluster_network_interface": "", 2023-10-09T05:50:28.972 INFO:teuthology.orchestra.run.smithi064.stdout: "compressor_zlib_isal": "false", 2023-10-09T05:50:28.972 INFO:teuthology.orchestra.run.smithi064.stdout: "compressor_zlib_level": "5", 2023-10-09T05:50:28.972 INFO:teuthology.orchestra.run.smithi064.stdout: "compressor_zlib_winsize": "-15", 2023-10-09T05:50:28.972 INFO:teuthology.orchestra.run.smithi064.stdout: "compressor_zstd_level": "1", 2023-10-09T05:50:28.972 INFO:teuthology.orchestra.run.smithi064.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c", 2023-10-09T05:50:28.972 INFO:teuthology.orchestra.run.smithi064.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-10-09T05:50:28.972 INFO:teuthology.orchestra.run.smithi064.stdout: "crimson_alien_op_num_threads": "6", 2023-10-09T05:50:28.972 INFO:teuthology.orchestra.run.smithi064.stdout: "crimson_alien_thread_cpu_cores": "", 2023-10-09T05:50:28.972 INFO:teuthology.orchestra.run.smithi064.stdout: "crimson_osd_obc_lru_size": "10", 2023-10-09T05:50:28.972 INFO:teuthology.orchestra.run.smithi064.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-10-09T05:50:28.972 INFO:teuthology.orchestra.run.smithi064.stdout: "crimson_seastar_smp": "1", 2023-10-09T05:50:28.973 INFO:teuthology.orchestra.run.smithi064.stdout: "crush_location": "", 2023-10-09T05:50:28.973 INFO:teuthology.orchestra.run.smithi064.stdout: "crush_location_hook": "", 2023-10-09T05:50:28.973 INFO:teuthology.orchestra.run.smithi064.stdout: "crush_location_hook_timeout": "10", 2023-10-09T05:50:28.973 INFO:teuthology.orchestra.run.smithi064.stdout: "daemonize": "false", 2023-10-09T05:50:28.973 INFO:teuthology.orchestra.run.smithi064.stdout: "daos_pool": "tank", 2023-10-09T05:50:28.973 INFO:teuthology.orchestra.run.smithi064.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-10-09T05:50:28.973 INFO:teuthology.orchestra.run.smithi064.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-10-09T05:50:28.973 INFO:teuthology.orchestra.run.smithi064.stdout: "dbstore_db_name_prefix": "dbstore", 2023-10-09T05:50:28.973 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_alienstore": "0/5", 2023-10-09T05:50:28.973 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_allow_any_pool_priority": "false", 2023-10-09T05:50:28.973 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_asok": "1/5", 2023-10-09T05:50:28.973 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_asok_assert_abort": "false", 2023-10-09T05:50:28.974 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_asserts_on_shutdown": "true", 2023-10-09T05:50:28.974 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_auth": "1/5", 2023-10-09T05:50:28.974 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_bdev": "1/3", 2023-10-09T05:50:28.974 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_bluefs": "1/5", 2023-10-09T05:50:28.974 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_bluestore": "1/5", 2023-10-09T05:50:28.974 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_buffer": "0/1", 2023-10-09T05:50:28.974 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_ceph_exporter": "1/5", 2023-10-09T05:50:28.974 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_cephfs_mirror": "0/5", 2023-10-09T05:50:28.974 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_cephsqlite": "0/5", 2023-10-09T05:50:28.974 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_client": "0/5", 2023-10-09T05:50:28.974 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_compressor": "1/5", 2023-10-09T05:50:28.975 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_context": "0/1", 2023-10-09T05:50:28.975 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_crush": "1/1", 2023-10-09T05:50:28.975 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_crypto": "1/5", 2023-10-09T05:50:28.975 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_cyanstore": "0/5", 2023-10-09T05:50:28.975 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_deliberately_leak_memory": "false", 2023-10-09T05:50:28.975 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_disable_randomized_ping": "false", 2023-10-09T05:50:28.975 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_dpdk": "1/5", 2023-10-09T05:50:28.975 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_eventtrace": "1/5", 2023-10-09T05:50:28.975 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_filer": "0/1", 2023-10-09T05:50:28.975 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_filestore": "1/3", 2023-10-09T05:50:28.975 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_finisher": "1/1", 2023-10-09T05:50:28.975 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_fuse": "1/5", 2023-10-09T05:50:28.976 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_heartbeat_testing_span": "0", 2023-10-09T05:50:28.976 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_heartbeatmap": "1/5", 2023-10-09T05:50:28.976 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_immutable_obj_cache": "0/5", 2023-10-09T05:50:28.976 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_javaclient": "1/5", 2023-10-09T05:50:28.976 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_journal": "1/3", 2023-10-09T05:50:28.976 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_journaler": "0/5", 2023-10-09T05:50:28.976 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_kstore": "1/5", 2023-10-09T05:50:28.976 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_leveldb": "4/5", 2023-10-09T05:50:28.976 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_lockdep": "0/1", 2023-10-09T05:50:28.976 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mclock": "1/5", 2023-10-09T05:50:28.976 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mds": "1/5", 2023-10-09T05:50:28.977 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mds_balancer": "1/5", 2023-10-09T05:50:28.977 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mds_locker": "1/5", 2023-10-09T05:50:28.977 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mds_log": "1/5", 2023-10-09T05:50:28.977 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mds_log_expire": "1/5", 2023-10-09T05:50:28.977 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mds_migrator": "1/5", 2023-10-09T05:50:28.977 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_memstore": "1/5", 2023-10-09T05:50:28.977 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mgr": "2/5", 2023-10-09T05:50:28.977 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mgrc": "1/5", 2023-10-09T05:50:28.977 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mon": "20/20", 2023-10-09T05:50:28.977 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_monc": "0/10", 2023-10-09T05:50:28.977 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_ms": "1/1", 2023-10-09T05:50:28.977 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_none": "0/5", 2023-10-09T05:50:28.978 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_objclass": "0/5", 2023-10-09T05:50:28.978 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_objectcacher": "0/5", 2023-10-09T05:50:28.978 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_objecter": "0/1", 2023-10-09T05:50:28.978 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_optracker": "0/5", 2023-10-09T05:50:28.978 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_osd": "1/5", 2023-10-09T05:50:28.978 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_paxos": "20/20", 2023-10-09T05:50:28.978 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_perfcounter": "1/5", 2023-10-09T05:50:28.978 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_prioritycache": "1/5", 2023-10-09T05:50:28.978 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rados": "0/5", 2023-10-09T05:50:28.978 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rbd": "0/5", 2023-10-09T05:50:28.978 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rbd_mirror": "0/5", 2023-10-09T05:50:28.979 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rbd_pwl": "0/5", 2023-10-09T05:50:28.979 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rbd_replay": "0/5", 2023-10-09T05:50:28.979 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_refs": "0/0", 2023-10-09T05:50:28.979 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_reserver": "1/1", 2023-10-09T05:50:28.979 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rgw": "1/5", 2023-10-09T05:50:28.979 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rgw_access": "1/5", 2023-10-09T05:50:28.979 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rgw_datacache": "1/5", 2023-10-09T05:50:28.979 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rgw_dbstore": "1/5", 2023-10-09T05:50:28.979 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rgw_flight": "1/5", 2023-10-09T05:50:28.979 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rgw_sync": "1/5", 2023-10-09T05:50:28.979 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rocksdb": "4/5", 2023-10-09T05:50:28.979 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore": "0/5", 2023-10-09T05:50:28.980 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_backref": "0/5", 2023-10-09T05:50:28.980 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_cache": "0/5", 2023-10-09T05:50:28.980 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_cleaner": "0/5", 2023-10-09T05:50:28.980 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_device": "0/5", 2023-10-09T05:50:28.980 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_epm": "0/5", 2023-10-09T05:50:28.980 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-10-09T05:50:28.980 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_journal": "0/5", 2023-10-09T05:50:28.980 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_lba": "0/5", 2023-10-09T05:50:28.980 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_odata": "0/5", 2023-10-09T05:50:28.980 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_omap": "0/5", 2023-10-09T05:50:28.980 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_onode": "0/5", 2023-10-09T05:50:28.980 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_t": "0/5", 2023-10-09T05:50:28.981 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_tm": "0/5", 2023-10-09T05:50:28.981 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_striper": "0/1", 2023-10-09T05:50:28.981 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_test": "0/5", 2023-10-09T05:50:28.981 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_throttle": "1/1", 2023-10-09T05:50:28.981 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_timer": "0/1", 2023-10-09T05:50:28.981 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_tp": "0/5", 2023-10-09T05:50:28.981 INFO:teuthology.orchestra.run.smithi064.stdout: "device_failure_prediction_mode": "none", 2023-10-09T05:50:28.981 INFO:teuthology.orchestra.run.smithi064.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-10-09T05:50:28.981 INFO:teuthology.orchestra.run.smithi064.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-10-09T05:50:28.981 INFO:teuthology.orchestra.run.smithi064.stdout: "err_to_graylog": "false", 2023-10-09T05:50:28.982 INFO:teuthology.orchestra.run.smithi064.stdout: "err_to_journald": "false", 2023-10-09T05:50:28.982 INFO:teuthology.orchestra.run.smithi064.stdout: "err_to_stderr": "true", 2023-10-09T05:50:28.982 INFO:teuthology.orchestra.run.smithi064.stdout: "err_to_syslog": "false", 2023-10-09T05:50:28.982 INFO:teuthology.orchestra.run.smithi064.stdout: "event_tracing": "false", 2023-10-09T05:50:28.982 INFO:teuthology.orchestra.run.smithi064.stdout: "exporter_addr": "0.0.0.0", 2023-10-09T05:50:28.982 INFO:teuthology.orchestra.run.smithi064.stdout: "exporter_http_port": "9926", 2023-10-09T05:50:28.982 INFO:teuthology.orchestra.run.smithi064.stdout: "exporter_prio_limit": "5", 2023-10-09T05:50:28.982 INFO:teuthology.orchestra.run.smithi064.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-10-09T05:50:28.982 INFO:teuthology.orchestra.run.smithi064.stdout: "exporter_sort_metrics": "true", 2023-10-09T05:50:28.982 INFO:teuthology.orchestra.run.smithi064.stdout: "exporter_stats_period": "5", 2023-10-09T05:50:28.982 INFO:teuthology.orchestra.run.smithi064.stdout: "fake_statfs_for_testing": "0", 2023-10-09T05:50:28.982 INFO:teuthology.orchestra.run.smithi064.stdout: "fatal_signal_handlers": "true", 2023-10-09T05:50:28.983 INFO:teuthology.orchestra.run.smithi064.stdout: "filer_max_purge_ops": "10", 2023-10-09T05:50:28.983 INFO:teuthology.orchestra.run.smithi064.stdout: "filer_max_truncate_ops": "128", 2023-10-09T05:50:28.983 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_apply_finisher_threads": "1", 2023-10-09T05:50:28.983 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_blackhole": "false", 2023-10-09T05:50:28.983 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_btrfs_clone_range": "true", 2023-10-09T05:50:28.983 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_btrfs_snap": "true", 2023-10-09T05:50:28.983 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_caller_concurrency": "10", 2023-10-09T05:50:28.983 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_collect_device_partition_information": "true", 2023-10-09T05:50:28.983 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_commit_timeout": "600.000000", 2023-10-09T05:50:28.983 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_debug_inject_read_err": "false", 2023-10-09T05:50:28.983 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_debug_omap_check": "false", 2023-10-09T05:50:28.983 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_debug_random_read_err": "0.000000", 2023-10-09T05:50:28.984 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_debug_verify_split": "false", 2023-10-09T05:50:28.984 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_dump_file": "", 2023-10-09T05:50:28.984 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-10-09T05:50:28.984 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-10-09T05:50:28.984 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_fadvise": "true", 2023-10-09T05:50:28.984 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_fail_eio": "true", 2023-10-09T05:50:28.984 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_fd_cache_shards": "16", 2023-10-09T05:50:28.984 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_fd_cache_size": "128", 2023-10-09T05:50:28.984 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_fiemap": "false", 2023-10-09T05:50:28.984 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_fiemap_threshold": "4096", 2023-10-09T05:50:28.984 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-10-09T05:50:28.984 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_index_retry_probability": "0.000000", 2023-10-09T05:50:28.985 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_inject_stall": "0", 2023-10-09T05:50:28.985 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_journal_parallel": "false", 2023-10-09T05:50:28.985 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_journal_trailing": "false", 2023-10-09T05:50:28.985 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_journal_writeahead": "false", 2023-10-09T05:50:28.985 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_kill_at": "0", 2023-10-09T05:50:28.985 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-10-09T05:50:28.985 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_inline_xattr_size": "0", 2023-10-09T05:50:28.985 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-10-09T05:50:28.985 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-10-09T05:50:28.985 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-10-09T05:50:28.985 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_inline_xattrs": "0", 2023-10-09T05:50:28.985 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-10-09T05:50:28.986 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_inline_xattrs_other": "2", 2023-10-09T05:50:28.986 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-10-09T05:50:28.986 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_sync_interval": "5.000000", 2023-10-09T05:50:28.986 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_xattr_value_size": "0", 2023-10-09T05:50:28.986 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-10-09T05:50:28.986 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-10-09T05:50:28.986 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-10-09T05:50:28.986 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_merge_threshold": "-10", 2023-10-09T05:50:28.986 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_min_sync_interval": "0.010000", 2023-10-09T05:50:28.986 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_odsync_write": "false", 2023-10-09T05:50:28.986 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_omap_backend": "rocksdb", 2023-10-09T05:50:28.986 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_omap_backend_path": "", 2023-10-09T05:50:28.987 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_omap_header_cache_size": "1024", 2023-10-09T05:50:28.987 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_ondisk_finisher_threads": "1", 2023-10-09T05:50:28.987 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-10-09T05:50:28.987 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_op_thread_timeout": "60", 2023-10-09T05:50:28.987 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_op_threads": "2", 2023-10-09T05:50:28.987 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_punch_hole": "false", 2023-10-09T05:50:28.987 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-10-09T05:50:28.987 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-10-09T05:50:28.987 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-10-09T05:50:28.987 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-10-09T05:50:28.987 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-10-09T05:50:28.988 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_max_bytes": "104857600", 2023-10-09T05:50:28.988 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-10-09T05:50:28.988 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-10-09T05:50:28.988 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-10-09T05:50:28.988 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_max_ops": "50", 2023-10-09T05:50:28.988 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-10-09T05:50:28.988 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_seek_data_hole": "false", 2023-10-09T05:50:28.988 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_sloppy_crc": "false", 2023-10-09T05:50:28.988 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-10-09T05:50:28.988 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_splice": "false", 2023-10-09T05:50:28.988 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_split_multiple": "2", 2023-10-09T05:50:28.989 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_split_rand_factor": "20", 2023-10-09T05:50:28.989 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_update_to": "1000", 2023-10-09T05:50:28.989 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-10-09T05:50:28.989 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-10-09T05:50:28.989 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-10-09T05:50:28.989 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-10-09T05:50:28.989 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-10-09T05:50:28.989 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-10-09T05:50:28.989 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_enable": "true", 2023-10-09T05:50:28.989 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-10-09T05:50:28.989 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-10-09T05:50:28.989 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-10-09T05:50:28.990 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-10-09T05:50:28.990 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-10-09T05:50:28.990 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-10-09T05:50:28.990 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_xfs_extsize": "false", 2023-10-09T05:50:28.990 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_zfs_snap": "false", 2023-10-09T05:50:28.990 INFO:teuthology.orchestra.run.smithi064.stdout: "fio_dir": "/tmp/fio", 2023-10-09T05:50:28.990 INFO:teuthology.orchestra.run.smithi064.stdout: "fsid": "44c77acc-6665-11ee-8db5-212e2dc638e7", 2023-10-09T05:50:28.990 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_allow_other": "true", 2023-10-09T05:50:28.990 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_atomic_o_trunc": "true", 2023-10-09T05:50:28.990 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_big_writes": "true", 2023-10-09T05:50:28.990 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_debug": "false", 2023-10-09T05:50:28.991 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_default_permissions": "false", 2023-10-09T05:50:28.991 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_disable_pagecache": "false", 2023-10-09T05:50:28.991 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_max_write": "0", 2023-10-09T05:50:28.991 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_multithreaded": "true", 2023-10-09T05:50:28.991 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_require_active_mds": "true", 2023-10-09T05:50:28.991 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_set_user_groups": "true", 2023-10-09T05:50:28.991 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_splice_move": "true", 2023-10-09T05:50:28.991 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_splice_read": "true", 2023-10-09T05:50:28.991 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_splice_write": "true", 2023-10-09T05:50:28.991 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_syncfs_on_mksnap": "true", 2023-10-09T05:50:28.991 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_use_invalidate_cb": "true", 2023-10-09T05:50:28.991 INFO:teuthology.orchestra.run.smithi064.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-10-09T05:50:28.992 INFO:teuthology.orchestra.run.smithi064.stdout: "gss_target_name": "ceph", 2023-10-09T05:50:28.992 INFO:teuthology.orchestra.run.smithi064.stdout: "heartbeat_file": "", 2023-10-09T05:50:28.992 INFO:teuthology.orchestra.run.smithi064.stdout: "heartbeat_inject_failure": "0", 2023-10-09T05:50:28.992 INFO:teuthology.orchestra.run.smithi064.stdout: "heartbeat_interval": "5", 2023-10-09T05:50:28.992 INFO:teuthology.orchestra.run.smithi064.stdout: "host": "smithi064", 2023-10-09T05:50:28.992 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-10-09T05:50:28.992 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-10-09T05:50:28.992 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_max_size": "1073741824", 2023-10-09T05:50:28.992 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-10-09T05:50:28.992 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-10-09T05:50:28.992 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-10-09T05:50:28.993 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-10-09T05:50:28.993 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-10-09T05:50:28.993 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-10-09T05:50:28.993 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-10-09T05:50:28.993 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-10-09T05:50:28.993 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-10-09T05:50:28.993 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_watermark": "0.900000", 2023-10-09T05:50:28.993 INFO:teuthology.orchestra.run.smithi064.stdout: "inject_early_sigterm": "false", 2023-10-09T05:50:28.993 INFO:teuthology.orchestra.run.smithi064.stdout: "jaeger_agent_port": "6799", 2023-10-09T05:50:28.994 INFO:teuthology.orchestra.run.smithi064.stdout: "jaeger_tracing_enable": "false", 2023-10-09T05:50:28.994 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_aio": "true", 2023-10-09T05:50:28.994 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_align_min_size": "65536", 2023-10-09T05:50:28.994 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_block_align": "true", 2023-10-09T05:50:28.994 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_block_size": "4096", 2023-10-09T05:50:28.994 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_dio": "true", 2023-10-09T05:50:28.994 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_discard": "false", 2023-10-09T05:50:28.994 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_force_aio": "false", 2023-10-09T05:50:28.994 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_ignore_corruption": "false", 2023-10-09T05:50:28.994 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_max_write_bytes": "10485760", 2023-10-09T05:50:28.994 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_max_write_entries": "100", 2023-10-09T05:50:28.995 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_replay_from": "0", 2023-10-09T05:50:28.995 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_throttle_high_multiple": "0.000000", 2023-10-09T05:50:28.995 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-10-09T05:50:28.995 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-10-09T05:50:28.995 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_throttle_max_multiple": "0.000000", 2023-10-09T05:50:28.995 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_write_header_frequency": "0", 2023-10-09T05:50:28.995 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_zero_on_create": "false", 2023-10-09T05:50:28.995 INFO:teuthology.orchestra.run.smithi064.stdout: "journaler_prefetch_periods": "10", 2023-10-09T05:50:28.995 INFO:teuthology.orchestra.run.smithi064.stdout: "journaler_prezero_periods": "5", 2023-10-09T05:50:28.995 INFO:teuthology.orchestra.run.smithi064.stdout: "journaler_write_head_interval": "15", 2023-10-09T05:50:28.996 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "", 2023-10-09T05:50:28.996 INFO:teuthology.orchestra.run.smithi064.stdout: "keyfile": "", 2023-10-09T05:50:28.996 INFO:teuthology.orchestra.run.smithi064.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-10-09T05:50:28.996 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_backend": "rocksdb", 2023-10-09T05:50:28.996 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_default_stripe_size": "65536", 2023-10-09T05:50:28.996 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_fsck_on_mount": "false", 2023-10-09T05:50:28.996 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_fsck_on_mount_deep": "true", 2023-10-09T05:50:28.996 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_max_bytes": "67108864", 2023-10-09T05:50:28.996 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_max_ops": "512", 2023-10-09T05:50:28.997 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_nid_prealloc": "1024", 2023-10-09T05:50:28.997 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_onode_map_size": "1024", 2023-10-09T05:50:28.997 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-10-09T05:50:28.997 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_sync_submit_transaction": "false", 2023-10-09T05:50:28.997 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_sync_transaction": "false", 2023-10-09T05:50:28.997 INFO:teuthology.orchestra.run.smithi064.stdout: "leveldb_block_size": "65536", 2023-10-09T05:50:28.997 INFO:teuthology.orchestra.run.smithi064.stdout: "leveldb_bloom_size": "0", 2023-10-09T05:50:28.997 INFO:teuthology.orchestra.run.smithi064.stdout: "leveldb_cache_size": "536870912", 2023-10-09T05:50:28.997 INFO:teuthology.orchestra.run.smithi064.stdout: "leveldb_compact_on_mount": "false", 2023-10-09T05:50:28.997 INFO:teuthology.orchestra.run.smithi064.stdout: "leveldb_compression": "false", 2023-10-09T05:50:28.997 INFO:teuthology.orchestra.run.smithi064.stdout: "leveldb_log": "", 2023-10-09T05:50:28.998 INFO:teuthology.orchestra.run.smithi064.stdout: "leveldb_log_to_ceph_log": "true", 2023-10-09T05:50:28.998 INFO:teuthology.orchestra.run.smithi064.stdout: "leveldb_max_open_files": "0", 2023-10-09T05:50:28.998 INFO:teuthology.orchestra.run.smithi064.stdout: "leveldb_paranoid": "false", 2023-10-09T05:50:28.998 INFO:teuthology.orchestra.run.smithi064.stdout: "leveldb_write_buffer_size": "33554432", 2023-10-09T05:50:28.998 INFO:teuthology.orchestra.run.smithi064.stdout: "librados_thread_count": "2", 2023-10-09T05:50:28.998 INFO:teuthology.orchestra.run.smithi064.stdout: "lockdep": "false", 2023-10-09T05:50:28.998 INFO:teuthology.orchestra.run.smithi064.stdout: "lockdep_force_backtrace": "false", 2023-10-09T05:50:28.998 INFO:teuthology.orchestra.run.smithi064.stdout: "log_coarse_timestamps": "true", 2023-10-09T05:50:28.998 INFO:teuthology.orchestra.run.smithi064.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-10-09T05:50:28.998 INFO:teuthology.orchestra.run.smithi064.stdout: "log_flush_on_exit": "false", 2023-10-09T05:50:28.999 INFO:teuthology.orchestra.run.smithi064.stdout: "log_graylog_host": "127.0.0.1", 2023-10-09T05:50:28.999 INFO:teuthology.orchestra.run.smithi064.stdout: "log_graylog_port": "12201", 2023-10-09T05:50:28.999 INFO:teuthology.orchestra.run.smithi064.stdout: "log_max_new": "1000", 2023-10-09T05:50:28.999 INFO:teuthology.orchestra.run.smithi064.stdout: "log_max_recent": "10000", 2023-10-09T05:50:28.999 INFO:teuthology.orchestra.run.smithi064.stdout: "log_stderr_prefix": "", 2023-10-09T05:50:28.999 INFO:teuthology.orchestra.run.smithi064.stdout: "log_stop_at_utilization": "0.970000", 2023-10-09T05:50:28.999 INFO:teuthology.orchestra.run.smithi064.stdout: "log_to_file": "true", 2023-10-09T05:50:28.999 INFO:teuthology.orchestra.run.smithi064.stdout: "log_to_graylog": "false", 2023-10-09T05:50:28.999 INFO:teuthology.orchestra.run.smithi064.stdout: "log_to_journald": "false", 2023-10-09T05:50:28.999 INFO:teuthology.orchestra.run.smithi064.stdout: "log_to_stderr": "false", 2023-10-09T05:50:28.999 INFO:teuthology.orchestra.run.smithi064.stdout: "log_to_syslog": "false", 2023-10-09T05:50:29.000 INFO:teuthology.orchestra.run.smithi064.stdout: "max_rotating_auth_attempts": "10", 2023-10-09T05:50:29.000 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-10-09T05:50:29.000 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_action_on_write_error": "1", 2023-10-09T05:50:29.000 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_alternate_name_max": "8192", 2023-10-09T05:50:29.000 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_asio_thread_count": "2", 2023-10-09T05:50:29.000 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_export_pin": "true", 2023-10-09T05:50:29.000 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_fragment_dirs": "true", 2023-10-09T05:50:29.000 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-10-09T05:50:29.000 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_fragment_interval": "5", 2023-10-09T05:50:29.000 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_fragment_size_max": "100000", 2023-10-09T05:50:29.000 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_idle_threshold": "0.000000", 2023-10-09T05:50:29.000 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_interval": "10", 2023-10-09T05:50:29.001 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_max": "-1", 2023-10-09T05:50:29.001 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_max_until": "-1", 2023-10-09T05:50:29.001 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_merge_size": "50", 2023-10-09T05:50:29.001 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_midchunk": "0.300000", 2023-10-09T05:50:29.001 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_min_rebalance": "0.100000", 2023-10-09T05:50:29.001 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_min_start": "0.200000", 2023-10-09T05:50:29.001 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_minchunk": "0.001000", 2023-10-09T05:50:29.001 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_mode": "0", 2023-10-09T05:50:29.001 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_need_max": "1.200000", 2023-10-09T05:50:29.001 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_need_min": "0.800000", 2023-10-09T05:50:29.001 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-10-09T05:50:29.002 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_sample_interval": "3.000000", 2023-10-09T05:50:29.002 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_split_bits": "3", 2023-10-09T05:50:29.002 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_split_rd": "25000.000000", 2023-10-09T05:50:29.002 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_split_size": "10000", 2023-10-09T05:50:29.002 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_split_wr": "10000.000000", 2023-10-09T05:50:29.002 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_target_decay": "10.000000", 2023-10-09T05:50:29.002 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-10-09T05:50:29.002 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_beacon_grace": "15.000000", 2023-10-09T05:50:29.002 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_beacon_interval": "4.000000", 2023-10-09T05:50:29.002 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_beacon_mon_down_grace": "60", 2023-10-09T05:50:29.002 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_memory_limit": "4294967296", 2023-10-09T05:50:29.003 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_mid": "0.700000", 2023-10-09T05:50:29.003 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_release_free_interval": "10", 2023-10-09T05:50:29.003 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_reservation": "0.050000", 2023-10-09T05:50:29.003 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-10-09T05:50:29.003 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_trim_interval": "1", 2023-10-09T05:50:29.003 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_trim_threshold": "262144", 2023-10-09T05:50:29.003 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-10-09T05:50:29.003 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-10-09T05:50:29.003 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_client_delegate_inos_pct": "50", 2023-10-09T05:50:29.003 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_client_prealloc_inos": "1000", 2023-10-09T05:50:29.003 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-10-09T05:50:29.003 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_connect_bootstrapping": "false", 2023-10-09T05:50:29.004 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_damage_table_max_entries": "10000", 2023-10-09T05:50:29.004 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-10-09T05:50:29.004 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_debug_auth_pins": "false", 2023-10-09T05:50:29.004 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_debug_frag": "false", 2023-10-09T05:50:29.004 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_debug_scatterstat": "false", 2023-10-09T05:50:29.004 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_debug_subtrees": "false", 2023-10-09T05:50:29.004 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_decay_halflife": "5.000000", 2023-10-09T05:50:29.004 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_default_dir_hash": "2", 2023-10-09T05:50:29.004 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_defer_session_stale": "true", 2023-10-09T05:50:29.004 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_deny_all_reconnect": "false", 2023-10-09T05:50:29.004 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dir_keys_per_op": "16384", 2023-10-09T05:50:29.004 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dir_max_commit_size": "10", 2023-10-09T05:50:29.005 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dir_max_entries": "0", 2023-10-09T05:50:29.005 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dir_prefetch": "true", 2023-10-09T05:50:29.005 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dirstat_min_interval": "1.000000", 2023-10-09T05:50:29.005 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dump_cache_after_rejoin": "false", 2023-10-09T05:50:29.005 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dump_cache_on_map": "false", 2023-10-09T05:50:29.005 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dump_cache_threshold_file": "0", 2023-10-09T05:50:29.005 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-10-09T05:50:29.005 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_early_reply": "true", 2023-10-09T05:50:29.005 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_enable_op_tracker": "true", 2023-10-09T05:50:29.005 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_enforce_unique_name": "true", 2023-10-09T05:50:29.005 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_export_ephemeral_distributed": "true", 2023-10-09T05:50:29.005 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-10-09T05:50:29.006 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_export_ephemeral_random": "true", 2023-10-09T05:50:29.006 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-10-09T05:50:29.006 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-10-09T05:50:29.006 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_forward_all_requests_to_auth": "false", 2023-10-09T05:50:29.006 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-10-09T05:50:29.006 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-10-09T05:50:29.006 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-10-09T05:50:29.006 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-10-09T05:50:29.006 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_health_cache_threshold": "1.500000", 2023-10-09T05:50:29.006 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_health_summarize_threshold": "10", 2023-10-09T05:50:29.006 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_heartbeat_grace": "15.000000", 2023-10-09T05:50:29.007 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_heartbeat_reset_grace": "1000", 2023-10-09T05:50:29.007 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_inject_health_dummy": "false", 2023-10-09T05:50:29.007 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-10-09T05:50:29.007 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_inject_migrator_session_race": "false", 2023-10-09T05:50:29.007 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-10-09T05:50:29.007 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-10-09T05:50:29.007 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-10-09T05:50:29.007 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_join_fs": "", 2023-10-09T05:50:29.007 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_journal_format": "1", 2023-10-09T05:50:29.007 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_create_at": "0", 2023-10-09T05:50:29.007 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_export_at": "0", 2023-10-09T05:50:29.007 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_import_at": "0", 2023-10-09T05:50:29.007 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_journal_at": "0", 2023-10-09T05:50:29.008 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_journal_expire_at": "0", 2023-10-09T05:50:29.008 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_journal_replay_at": "0", 2023-10-09T05:50:29.008 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_link_at": "0", 2023-10-09T05:50:29.008 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_mdstable_at": "0", 2023-10-09T05:50:29.008 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_openc_at": "0", 2023-10-09T05:50:29.008 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_rename_at": "0", 2023-10-09T05:50:29.008 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-10-09T05:50:29.008 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_events_per_segment": "1024", 2023-10-09T05:50:29.008 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_max_events": "-1", 2023-10-09T05:50:29.008 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_max_segments": "128", 2023-10-09T05:50:29.009 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_pause": "false", 2023-10-09T05:50:29.009 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_segment_size": "0", 2023-10-09T05:50:29.009 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_skip_corrupt_events": "false", 2023-10-09T05:50:29.009 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_warn_factor": "2.000000", 2023-10-09T05:50:29.009 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_caps_per_client": "1048576", 2023-10-09T05:50:29.009 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_completed_flushes": "100000", 2023-10-09T05:50:29.010 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_completed_requests": "100000", 2023-10-09T05:50:29.010 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_export_size": "20971520", 2023-10-09T05:50:29.010 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_file_recover": "32", 2023-10-09T05:50:29.010 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_purge_files": "64", 2023-10-09T05:50:29.010 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_purge_ops": "8192", 2023-10-09T05:50:29.010 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-10-09T05:50:29.010 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-10-09T05:50:29.011 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_snaps_per_dir": "100", 2023-10-09T05:50:29.011 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_xattr_pairs_size": "65536", 2023-10-09T05:50:29.011 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_metrics_update_interval": "2", 2023-10-09T05:50:29.011 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_min_caps_per_client": "100", 2023-10-09T05:50:29.011 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_min_caps_working_set": "10000", 2023-10-09T05:50:29.011 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-10-09T05:50:29.011 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_numa_node": "-1", 2023-10-09T05:50:29.011 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-10-09T05:50:29.011 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_op_complaint_time": "30.000000", 2023-10-09T05:50:29.011 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_op_history_duration": "600", 2023-10-09T05:50:29.011 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_op_history_size": "20", 2023-10-09T05:50:29.012 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_op_log_threshold": "5", 2023-10-09T05:50:29.012 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_ping_grace": "15", 2023-10-09T05:50:29.012 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_ping_interval": "5", 2023-10-09T05:50:29.012 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-10-09T05:50:29.012 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-10-09T05:50:29.012 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_recall_max_caps": "30000", 2023-10-09T05:50:29.012 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-10-09T05:50:29.012 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_recall_max_decay_threshold": "131072", 2023-10-09T05:50:29.012 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-10-09T05:50:29.012 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_recall_warning_threshold": "262144", 2023-10-09T05:50:29.012 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_reconnect_timeout": "45.000000", 2023-10-09T05:50:29.013 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_replay_interval": "1.000000", 2023-10-09T05:50:29.013 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-10-09T05:50:29.013 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-10-09T05:50:29.013 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_root_ino_gid": "0", 2023-10-09T05:50:29.013 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_root_ino_uid": "0", 2023-10-09T05:50:29.013 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-10-09T05:50:29.013 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_session_blocklist_on_evict": "true", 2023-10-09T05:50:29.013 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_session_blocklist_on_timeout": "true", 2023-10-09T05:50:29.013 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-10-09T05:50:29.013 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-10-09T05:50:29.013 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-10-09T05:50:29.013 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-10-09T05:50:29.014 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-10-09T05:50:29.014 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-10-09T05:50:29.014 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_shutdown_check": "0", 2023-10-09T05:50:29.014 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_skip_ino": "0", 2023-10-09T05:50:29.014 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_sleep_rank_change": "0.000000", 2023-10-09T05:50:29.014 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_snap_max_uid": "4294967294", 2023-10-09T05:50:29.014 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_snap_min_uid": "0", 2023-10-09T05:50:29.014 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_snap_rstat": "false", 2023-10-09T05:50:29.014 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_standby_replay_damaged": "false", 2023-10-09T05:50:29.014 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_symlink_recovery": "true", 2023-10-09T05:50:29.014 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_task_status_update_interval": "2.000000", 2023-10-09T05:50:29.015 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_thrash_exports": "0", 2023-10-09T05:50:29.015 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_thrash_fragments": "0", 2023-10-09T05:50:29.015 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_tick_interval": "5.000000", 2023-10-09T05:50:29.015 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_valgrind_exit": "false", 2023-10-09T05:50:29.015 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_verify_backtrace": "1", 2023-10-09T05:50:29.015 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_verify_scatter": "false", 2023-10-09T05:50:29.015 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_wipe_ino_prealloc": "false", 2023-10-09T05:50:29.015 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_wipe_sessions": "false", 2023-10-09T05:50:29.015 INFO:teuthology.orchestra.run.smithi064.stdout: "mempool_debug": "false", 2023-10-09T05:50:29.015 INFO:teuthology.orchestra.run.smithi064.stdout: "memstore_debug_omit_block_device_write": "false", 2023-10-09T05:50:29.015 INFO:teuthology.orchestra.run.smithi064.stdout: "memstore_device_bytes": "1073741824", 2023-10-09T05:50:29.015 INFO:teuthology.orchestra.run.smithi064.stdout: "memstore_page_set": "false", 2023-10-09T05:50:29.016 INFO:teuthology.orchestra.run.smithi064.stdout: "memstore_page_size": "65536", 2023-10-09T05:50:29.016 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_client_bytes": "134217728", 2023-10-09T05:50:29.016 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_client_messages": "512", 2023-10-09T05:50:29.016 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-10-09T05:50:29.016 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_connect_retry_interval": "1.000000", 2023-10-09T05:50:29.016 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-10-09T05:50:29.016 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-10-09T05:50:29.016 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_disabled_modules": "", 2023-10-09T05:50:29.016 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-10-09T05:50:29.016 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_max_pg_num_change": "128", 2023-10-09T05:50:29.016 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_mds_bytes": "134217728", 2023-10-09T05:50:29.016 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_mds_messages": "128", 2023-10-09T05:50:29.016 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-10-09T05:50:29.017 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_mon_bytes": "134217728", 2023-10-09T05:50:29.017 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_mon_messages": "128", 2023-10-09T05:50:29.017 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_osd_bytes": "536870912", 2023-10-09T05:50:29.017 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_osd_messages": "8192", 2023-10-09T05:50:29.017 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_pool": "true", 2023-10-09T05:50:29.017 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_service_beacon_grace": "60.000000", 2023-10-09T05:50:29.017 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_standby_modules": "true", 2023-10-09T05:50:29.017 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_stats_period": "5", 2023-10-09T05:50:29.017 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_stats_threshold": "5", 2023-10-09T05:50:29.017 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_tick_period": "2", 2023-10-09T05:50:29.017 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-10-09T05:50:29.017 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_accept_timeout_factor": "2.000000", 2023-10-09T05:50:29.018 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_allow_pool_delete": "true", 2023-10-09T05:50:29.018 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_allow_pool_size_one": "false", 2023-10-09T05:50:29.018 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_auth_validate_all_caps": "true", 2023-10-09T05:50:29.018 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-10-09T05:50:29.018 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-10-09T05:50:29.018 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_bytes": "104857600", 2023-10-09T05:50:29.018 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_directed_command_retry": "2", 2023-10-09T05:50:29.018 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_hunt_interval": "3.000000", 2023-10-09T05:50:29.018 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-10-09T05:50:29.018 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-10-09T05:50:29.019 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-10-09T05:50:29.019 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_hunt_parallel": "3", 2023-10-09T05:50:29.019 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_log_interval": "1.000000", 2023-10-09T05:50:29.019 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-10-09T05:50:29.019 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_ping_interval": "10.000000", 2023-10-09T05:50:29.019 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_ping_timeout": "30.000000", 2023-10-09T05:50:29.019 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_clock_drift_allowed": "1.000000", 2023-10-09T05:50:29.019 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-10-09T05:50:29.019 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-10-09T05:50:29.019 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_file_level": "debug", 2023-10-09T05:50:29.019 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_file": "false", 2023-10-09T05:50:29.019 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_graylog": "false", 2023-10-09T05:50:29.020 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-10-09T05:50:29.020 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-10-09T05:50:29.020 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_journald": "true", 2023-10-09T05:50:29.020 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_stderr": "false", 2023-10-09T05:50:29.020 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-10-09T05:50:29.020 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-10-09T05:50:29.020 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-10-09T05:50:29.020 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_compact_on_bootstrap": "false", 2023-10-09T05:50:29.020 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_compact_on_start": "false", 2023-10-09T05:50:29.020 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_compact_on_trim": "true", 2023-10-09T05:50:29.020 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_con_tracker_persist_interval": "10", 2023-10-09T05:50:29.020 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_con_tracker_score_halflife": "43200", 2023-10-09T05:50:29.021 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_config_key_max_entry_size": "65536", 2023-10-09T05:50:29.021 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cpu_threads": "4", 2023-10-09T05:50:29.021 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_crush_min_required_version": "hammer", 2023-10-09T05:50:29.021 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_daemon_bytes": "419430400", 2023-10-09T05:50:29.021 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-10-09T05:50:29.021 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_data_avail_crit": "5", 2023-10-09T05:50:29.021 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_data_avail_warn": "5", 2023-10-09T05:50:29.021 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_data_size_warn": "16106127360", 2023-10-09T05:50:29.021 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_block_osdmap_trim": "false", 2023-10-09T05:50:29.021 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-10-09T05:50:29.021 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_dump_json": "false", 2023-10-09T05:50:29.021 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-10-09T05:50:29.022 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_dump_transactions": "false", 2023-10-09T05:50:29.022 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_extra_checks": "false", 2023-10-09T05:50:29.022 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-10-09T05:50:29.022 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-10-09T05:50:29.022 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_no_require_quincy": "false", 2023-10-09T05:50:29.022 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_no_require_reef": "false", 2023-10-09T05:50:29.022 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-10-09T05:50:29.022 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_delta_reset_interval": "10.000000", 2023-10-09T05:50:29.022 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_dns_srv_name": "ceph-mon", 2023-10-09T05:50:29.022 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_down_mkfs_grace": "60", 2023-10-09T05:50:29.022 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_election_default_strategy": "1", 2023-10-09T05:50:29.023 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_election_timeout": "5.000000", 2023-10-09T05:50:29.023 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-10-09T05:50:29.023 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_elector_ping_divisor": "2", 2023-10-09T05:50:29.023 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_elector_ping_timeout": "2.000000", 2023-10-09T05:50:29.023 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_enable_op_tracker": "true", 2023-10-09T05:50:29.023 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_fake_pool_delete": "false", 2023-10-09T05:50:29.023 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_force_quorum_join": "false", 2023-10-09T05:50:29.023 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_globalid_prealloc": "10000", 2023-10-09T05:50:29.023 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_health_detail_to_clog": "true", 2023-10-09T05:50:29.023 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_health_log_update_period": "5", 2023-10-09T05:50:29.023 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_health_max_detail": "50", 2023-10-09T05:50:29.023 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_health_to_clog": "true", 2023-10-09T05:50:29.024 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_health_to_clog_interval": "600", 2023-10-09T05:50:29.024 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-10-09T05:50:29.024 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_host": "[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]", 2023-10-09T05:50:29.024 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_host_override": "", 2023-10-09T05:50:29.024 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_initial_members": "", 2023-10-09T05:50:29.024 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-10-09T05:50:29.024 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-10-09T05:50:29.024 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-10-09T05:50:29.024 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-10-09T05:50:29.024 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_keyvaluedb": "rocksdb", 2023-10-09T05:50:29.024 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_lease": "5.000000", 2023-10-09T05:50:29.024 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-10-09T05:50:29.025 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-10-09T05:50:29.025 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_log_full_interval": "50", 2023-10-09T05:50:29.025 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_log_max": "10000", 2023-10-09T05:50:29.025 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_log_max_summary": "50", 2023-10-09T05:50:29.025 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_log_entries_per_event": "4096", 2023-10-09T05:50:29.025 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_log_epochs": "500", 2023-10-09T05:50:29.025 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_mdsmap_epochs": "500", 2023-10-09T05:50:29.025 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_mgrmap_epochs": "500", 2023-10-09T05:50:29.025 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_osd": "10000", 2023-10-09T05:50:29.025 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_pg_per_osd": "10000", 2023-10-09T05:50:29.025 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_pool_pg_num": "65536", 2023-10-09T05:50:29.025 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-10-09T05:50:29.026 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-10-09T05:50:29.026 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mds_force_trim_to": "0", 2023-10-09T05:50:29.026 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mds_skip_sanity": "true", 2023-10-09T05:50:29.026 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_memory_autotune": "true", 2023-10-09T05:50:29.026 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_memory_target": "2147483648", 2023-10-09T05:50:29.026 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mgr_beacon_grace": "30", 2023-10-09T05:50:29.026 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-10-09T05:50:29.026 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mgr_digest_period": "5", 2023-10-09T05:50:29.026 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mgr_inactive_grace": "60", 2023-10-09T05:50:29.026 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mgr_mkfs_grace": "240", 2023-10-09T05:50:29.026 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-10-09T05:50:29.027 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_min_osdmap_epochs": "500", 2023-10-09T05:50:29.027 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_op_complaint_time": "30", 2023-10-09T05:50:29.027 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_op_history_duration": "600", 2023-10-09T05:50:29.027 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_op_history_size": "20", 2023-10-09T05:50:29.027 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_op_history_slow_op_size": "20", 2023-10-09T05:50:29.027 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_op_history_slow_op_threshold": "10", 2023-10-09T05:50:29.027 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_op_log_threshold": "5", 2023-10-09T05:50:29.027 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-10-09T05:50:29.027 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-10-09T05:50:29.027 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-10-09T05:50:29.028 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_auto_mark_in": "false", 2023-10-09T05:50:29.028 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_auto_mark_new_in": "true", 2023-10-09T05:50:29.028 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-10-09T05:50:29.028 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-10-09T05:50:29.028 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_cache_size": "500", 2023-10-09T05:50:29.028 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_cache_size_min": "134217728", 2023-10-09T05:50:29.028 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_crush_smoke_test": "true", 2023-10-09T05:50:29.028 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_destroyed_out_interval": "600", 2023-10-09T05:50:29.028 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_down_out_interval": "600", 2023-10-09T05:50:29.028 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-10-09T05:50:29.028 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-10-09T05:50:29.028 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_force_trim_to": "0", 2023-10-09T05:50:29.029 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_full_ratio": "0.950000", 2023-10-09T05:50:29.029 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-10-09T05:50:29.029 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_laggy_halflife": "3600", 2023-10-09T05:50:29.029 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_laggy_max_interval": "300", 2023-10-09T05:50:29.029 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_laggy_weight": "0.300000", 2023-10-09T05:50:29.029 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-10-09T05:50:29.029 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_max_creating_pgs": "1024", 2023-10-09T05:50:29.029 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_max_initial_pgs": "1024", 2023-10-09T05:50:29.029 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_min_down_reporters": "2", 2023-10-09T05:50:29.029 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-10-09T05:50:29.029 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-10-09T05:50:29.029 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-10-09T05:50:29.030 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_prime_pg_temp": "true", 2023-10-09T05:50:29.030 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-10-09T05:50:29.030 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-10-09T05:50:29.030 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_report_timeout": "900", 2023-10-09T05:50:29.030 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-10-09T05:50:29.030 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-10-09T05:50:29.030 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_warn_num_repaired": "10", 2023-10-09T05:50:29.030 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_warn_op_age": "32.000000", 2023-10-09T05:50:29.030 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-10-09T05:50:29.030 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osdmap_full_prune_interval": "10", 2023-10-09T05:50:29.030 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osdmap_full_prune_min": "10000", 2023-10-09T05:50:29.031 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-10-09T05:50:29.031 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-10-09T05:50:29.031 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_pg_stuck_threshold": "60", 2023-10-09T05:50:29.031 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-10-09T05:50:29.031 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_pg_warn_min_objects": "10000", 2023-10-09T05:50:29.031 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_pg_warn_min_per_osd": "0", 2023-10-09T05:50:29.031 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-10-09T05:50:29.031 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_pool_quota_crit_threshold": "0", 2023-10-09T05:50:29.031 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_pool_quota_warn_threshold": "0", 2023-10-09T05:50:29.031 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_probe_timeout": "2.000000", 2023-10-09T05:50:29.031 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_reweight_max_change": "0.050000", 2023-10-09T05:50:29.031 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_reweight_max_osds": "4", 2023-10-09T05:50:29.032 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-10-09T05:50:29.032 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-10-09T05:50:29.032 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-10-09T05:50:29.032 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-10-09T05:50:29.032 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-10-09T05:50:29.032 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_scrub_interval": "86400", 2023-10-09T05:50:29.032 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_scrub_max_keys": "100", 2023-10-09T05:50:29.032 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_scrub_timeout": "300", 2023-10-09T05:50:29.032 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_session_timeout": "300", 2023-10-09T05:50:29.032 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_smart_report_timeout": "5", 2023-10-09T05:50:29.032 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_stat_smooth_intervals": "6", 2023-10-09T05:50:29.032 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-10-09T05:50:29.032 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_stretch_pool_min_size": "2", 2023-10-09T05:50:29.033 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_stretch_pool_size": "4", 2023-10-09T05:50:29.033 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-10-09T05:50:29.033 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_subscribe_interval": "86400.000000", 2023-10-09T05:50:29.033 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_sync_debug": "false", 2023-10-09T05:50:29.033 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_sync_max_payload_keys": "2000", 2023-10-09T05:50:29.033 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_sync_max_payload_size": "1048576", 2023-10-09T05:50:29.033 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_sync_provider_kill_at": "0", 2023-10-09T05:50:29.033 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_sync_requester_kill_at": "0", 2023-10-09T05:50:29.033 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_sync_timeout": "60.000000", 2023-10-09T05:50:29.033 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_target_pg_per_osd": "100", 2023-10-09T05:50:29.033 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_tick_interval": "5", 2023-10-09T05:50:29.034 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_timecheck_interval": "300.000000", 2023-10-09T05:50:29.034 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-10-09T05:50:29.034 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_use_min_delay_socket": "false", 2023-10-09T05:50:29.034 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_older_version_delay": "604800", 2023-10-09T05:50:29.034 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-10-09T05:50:29.034 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-10-09T05:50:29.034 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-10-09T05:50:29.034 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_filestore_osds": "true", 2023-10-09T05:50:29.034 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-10-09T05:50:29.034 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-10-09T05:50:29.035 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-10-09T05:50:29.035 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_misplaced": "false", 2023-10-09T05:50:29.035 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-10-09T05:50:29.035 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_older_version": "true", 2023-10-09T05:50:29.035 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-10-09T05:50:29.035 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_pool_no_app": "true", 2023-10-09T05:50:29.035 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-10-09T05:50:29.035 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-10-09T05:50:29.035 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-10-09T05:50:29.035 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-10-09T05:50:29.035 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_too_few_osds": "false", 2023-10-09T05:50:29.036 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-10-09T05:50:29.036 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-10-09T05:50:29.036 INFO:teuthology.orchestra.run.smithi064.stdout: "monmap": "", 2023-10-09T05:50:29.036 INFO:teuthology.orchestra.run.smithi064.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-09T05:50:29.036 INFO:teuthology.orchestra.run.smithi064.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-10-09T05:50:29.036 INFO:teuthology.orchestra.run.smithi064.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-10-09T05:50:29.036 INFO:teuthology.orchestra.run.smithi064.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-09T05:50:29.036 INFO:teuthology.orchestra.run.smithi064.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-10-09T05:50:29.036 INFO:teuthology.orchestra.run.smithi064.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-10-09T05:50:29.036 INFO:teuthology.orchestra.run.smithi064.stdout: "motr_tracing_enabled": "false", 2023-10-09T05:50:29.036 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_op_threads": "3", 2023-10-09T05:50:29.037 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_buffer_size": "131072", 2023-10-09T05:50:29.037 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_cm": "false", 2023-10-09T05:50:29.037 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_device_name": "", 2023-10-09T05:50:29.037 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_dscp": "96", 2023-10-09T05:50:29.037 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-10-09T05:50:29.037 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_gid_idx": "0", 2023-10-09T05:50:29.037 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_local_gid": "", 2023-10-09T05:50:29.037 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_polling_us": "1000", 2023-10-09T05:50:29.037 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_port_num": "1", 2023-10-09T05:50:29.037 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-10-09T05:50:29.037 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-10-09T05:50:29.038 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_roce_ver": "1", 2023-10-09T05:50:29.038 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_send_buffers": "1024", 2023-10-09T05:50:29.038 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_sl": "3", 2023-10-09T05:50:29.038 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_support_srq": "true", 2023-10-09T05:50:29.038 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_type": "ib", 2023-10-09T05:50:29.038 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_reap_threshold": "5", 2023-10-09T05:50:29.038 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_before_connect": "false", 2023-10-09T05:50:29.038 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_ipv4": "true", 2023-10-09T05:50:29.038 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_ipv6": "false", 2023-10-09T05:50:29.038 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_msgr1": "true", 2023-10-09T05:50:29.038 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_msgr2": "true", 2023-10-09T05:50:29.039 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_port_max": "7568", 2023-10-09T05:50:29.039 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_port_min": "6800", 2023-10-09T05:50:29.039 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_prefer_ipv4": "false", 2023-10-09T05:50:29.039 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_retry_count": "3", 2023-10-09T05:50:29.039 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_retry_delay": "5", 2023-10-09T05:50:29.039 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_blackhole_client": "false", 2023-10-09T05:50:29.039 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_blackhole_mds": "false", 2023-10-09T05:50:29.039 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_blackhole_mgr": "false", 2023-10-09T05:50:29.039 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_blackhole_mon": "false", 2023-10-09T05:50:29.039 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_blackhole_osd": "false", 2023-10-09T05:50:29.039 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_client_mode": "crc secure", 2023-10-09T05:50:29.039 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_cluster_mode": "crc secure", 2023-10-09T05:50:29.040 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_cluster_type": "", 2023-10-09T05:50:29.040 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_compress_secure": "false", 2023-10-09T05:50:29.040 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_connection_idle_timeout": "900", 2023-10-09T05:50:29.040 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_connection_ready_timeout": "10", 2023-10-09T05:50:29.040 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_crc_data": "true", 2023-10-09T05:50:29.040 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_crc_header": "true", 2023-10-09T05:50:29.040 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_die_on_bad_msg": "false", 2023-10-09T05:50:29.040 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_die_on_bug": "true", 2023-10-09T05:50:29.040 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_die_on_old_message": "true", 2023-10-09T05:50:29.040 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_die_on_skipped_message": "false", 2023-10-09T05:50:29.040 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_die_on_unhandled_msg": "false", 2023-10-09T05:50:29.041 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-10-09T05:50:29.041 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_coremask": "0xF", 2023-10-09T05:50:29.041 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-10-09T05:50:29.041 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_devs_allowlist": "", 2023-10-09T05:50:29.041 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_enable_tso": "true", 2023-10-09T05:50:29.041 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-10-09T05:50:29.041 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-10-09T05:50:29.041 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_hugepages": "", 2023-10-09T05:50:29.041 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_hw_flow_control": "true", 2023-10-09T05:50:29.041 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-10-09T05:50:29.041 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_lro": "true", 2023-10-09T05:50:29.042 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_memory_channel": "4", 2023-10-09T05:50:29.042 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-10-09T05:50:29.042 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_pmd": "", 2023-10-09T05:50:29.042 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_port_id": "0", 2023-10-09T05:50:29.042 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-10-09T05:50:29.042 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dump_corrupt_message_level": "1", 2023-10-09T05:50:29.042 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dump_on_send": "false", 2023-10-09T05:50:29.042 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_initial_backoff": "0.200000", 2023-10-09T05:50:29.042 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_inject_delay_max": "1.000000", 2023-10-09T05:50:29.043 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_inject_delay_probability": "0.000000", 2023-10-09T05:50:29.043 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_inject_delay_type": "", 2023-10-09T05:50:29.043 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_inject_internal_delays": "0.000000", 2023-10-09T05:50:29.043 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_inject_network_congestion": "0", 2023-10-09T05:50:29.043 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_inject_socket_failures": "0", 2023-10-09T05:50:29.043 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_learn_addr_from_peer": "true", 2023-10-09T05:50:29.043 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_max_accept_failures": "4", 2023-10-09T05:50:29.043 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_max_backoff": "15.000000", 2023-10-09T05:50:29.043 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_mon_client_mode": "secure crc", 2023-10-09T05:50:29.043 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_mon_cluster_mode": "secure crc", 2023-10-09T05:50:29.043 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_mon_service_mode": "secure crc", 2023-10-09T05:50:29.044 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_osd_compress_min_size": "1024", 2023-10-09T05:50:29.044 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_osd_compress_mode": "none", 2023-10-09T05:50:29.044 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_osd_compression_algorithm": "snappy", 2023-10-09T05:50:29.044 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-10-09T05:50:29.044 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_pq_min_cost": "65536", 2023-10-09T05:50:29.044 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_public_type": "", 2023-10-09T05:50:29.044 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_service_mode": "crc secure", 2023-10-09T05:50:29.044 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_tcp_listen_backlog": "512", 2023-10-09T05:50:29.044 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_tcp_nodelay": "true", 2023-10-09T05:50:29.044 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-10-09T05:50:29.044 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_tcp_rcvbuf": "0", 2023-10-09T05:50:29.045 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_type": "async+posix", 2023-10-09T05:50:29.045 INFO:teuthology.orchestra.run.smithi064.stdout: "no_config_file": "false", 2023-10-09T05:50:29.045 INFO:teuthology.orchestra.run.smithi064.stdout: "objecter_completion_locks_per_session": "32", 2023-10-09T05:50:29.045 INFO:teuthology.orchestra.run.smithi064.stdout: "objecter_debug_inject_relock_delay": "false", 2023-10-09T05:50:29.045 INFO:teuthology.orchestra.run.smithi064.stdout: "objecter_inflight_op_bytes": "104857600", 2023-10-09T05:50:29.045 INFO:teuthology.orchestra.run.smithi064.stdout: "objecter_inflight_ops": "1024", 2023-10-09T05:50:29.045 INFO:teuthology.orchestra.run.smithi064.stdout: "objecter_inject_no_watch_ping": "false", 2023-10-09T05:50:29.045 INFO:teuthology.orchestra.run.smithi064.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-10-09T05:50:29.045 INFO:teuthology.orchestra.run.smithi064.stdout: "objecter_tick_interval": "5.000000", 2023-10-09T05:50:29.045 INFO:teuthology.orchestra.run.smithi064.stdout: "objecter_timeout": "10.000000", 2023-10-09T05:50:29.045 INFO:teuthology.orchestra.run.smithi064.stdout: "objectstore_blackhole": "false", 2023-10-09T05:50:29.046 INFO:teuthology.orchestra.run.smithi064.stdout: "openssl_engine_opts": "", 2023-10-09T05:50:29.046 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_agent_delay_time": "5.000000", 2023-10-09T05:50:29.046 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_agent_hist_halflife": "1000", 2023-10-09T05:50:29.046 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_agent_max_low_ops": "2", 2023-10-09T05:50:29.046 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_agent_max_ops": "4", 2023-10-09T05:50:29.046 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-10-09T05:50:29.046 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_agent_quantize_effort": "0.100000", 2023-10-09T05:50:29.046 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_agent_slop": "0.020000", 2023-10-09T05:50:29.046 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-10-09T05:50:29.046 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_allow_recovery_below_min_size": "true", 2023-10-09T05:50:29.046 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_asio_thread_count": "2", 2023-10-09T05:50:29.047 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_async_recovery_min_cost": "100", 2023-10-09T05:50:29.047 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_auto_mark_unfound_lost": "false", 2023-10-09T05:50:29.047 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_backfill_retry_interval": "30.000000", 2023-10-09T05:50:29.047 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_backfill_scan_max": "512", 2023-10-09T05:50:29.047 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_backfill_scan_min": "64", 2023-10-09T05:50:29.047 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_backoff_on_degraded": "false", 2023-10-09T05:50:29.047 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_backoff_on_peering": "false", 2023-10-09T05:50:29.047 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_backoff_on_unfound": "true", 2023-10-09T05:50:29.047 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_beacon_report_interval": "300", 2023-10-09T05:50:29.047 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_bench_duration": "30", 2023-10-09T05:50:29.047 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-10-09T05:50:29.047 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_bench_max_block_size": "67108864", 2023-10-09T05:50:29.048 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_bench_small_size_max_iops": "100", 2023-10-09T05:50:29.048 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_blkin_trace_all": "false", 2023-10-09T05:50:29.048 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_blocked_scrub_grace_period": "120", 2023-10-09T05:50:29.048 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-10-09T05:50:29.048 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-10-09T05:50:29.048 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-10-09T05:50:29.048 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_check_for_log_corruption": "false", 2023-10-09T05:50:29.048 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-10-09T05:50:29.048 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-09T05:50:29.048 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-10-09T05:50:29.048 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-09T05:50:29.049 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_class_update_on_start": "true", 2023-10-09T05:50:29.049 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_client_message_cap": "256", 2023-10-09T05:50:29.049 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_client_message_size_cap": "524288000", 2023-10-09T05:50:29.049 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_client_op_priority": "63", 2023-10-09T05:50:29.049 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_client_watch_timeout": "30", 2023-10-09T05:50:29.049 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_command_max_records": "256", 2023-10-09T05:50:29.049 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_compact_on_start": "false", 2023-10-09T05:50:29.049 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-10-09T05:50:29.049 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_crush_chooseleaf_type": "1", 2023-10-09T05:50:29.049 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_crush_initial_weight": "-1.000000", 2023-10-09T05:50:29.049 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_crush_update_on_start": "true", 2023-10-09T05:50:29.050 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_crush_update_weight_set": "true", 2023-10-09T05:50:29.050 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-10-09T05:50:29.050 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-10-09T05:50:29.050 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-10-09T05:50:29.050 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_drop_ping_duration": "0", 2023-10-09T05:50:29.050 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-10-09T05:50:29.050 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_feed_pullee": "-1", 2023-10-09T05:50:29.050 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-10-09T05:50:29.050 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-10-09T05:50:29.050 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-10-09T05:50:29.050 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_misdirected_ops": "false", 2023-10-09T05:50:29.051 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_no_acting_change": "false", 2023-10-09T05:50:29.051 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_no_purge_strays": "false", 2023-10-09T05:50:29.051 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_op_order": "false", 2023-10-09T05:50:29.051 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_pg_log_writeout": "false", 2023-10-09T05:50:29.051 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_pretend_recovery_active": "false", 2023-10-09T05:50:29.051 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-10-09T05:50:29.051 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-10-09T05:50:29.051 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_shutdown": "false", 2023-10-09T05:50:29.051 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-10-09T05:50:29.051 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-10-09T05:50:29.051 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_verify_cached_snaps": "false", 2023-10-09T05:50:29.052 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_verify_missing_on_start": "false", 2023-10-09T05:50:29.052 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_verify_snaps": "false", 2023-10-09T05:50:29.052 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-10-09T05:50:29.052 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-10-09T05:50:29.052 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_deep_scrub_keys": "1024", 2023-10-09T05:50:29.052 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-10-09T05:50:29.052 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-10-09T05:50:29.052 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-10-09T05:50:29.052 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_deep_scrub_stride": "524288", 2023-10-09T05:50:29.052 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-10-09T05:50:29.052 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_default_data_pool_replay_window": "45", 2023-10-09T05:50:29.053 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_default_notify_timeout": "30", 2023-10-09T05:50:29.053 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_delete_sleep": "0.000000", 2023-10-09T05:50:29.053 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-10-09T05:50:29.053 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-10-09T05:50:29.053 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-10-09T05:50:29.053 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_discard_disconnected_ops": "true", 2023-10-09T05:50:29.053 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_enable_op_tracker": "true", 2023-10-09T05:50:29.053 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-10-09T05:50:29.053 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_extblkdev_plugins": "vdo", 2023-10-09T05:50:29.053 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-10-09T05:50:29.053 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-10-09T05:50:29.054 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_fast_info": "true", 2023-10-09T05:50:29.054 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_fast_shutdown": "true", 2023-10-09T05:50:29.054 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-10-09T05:50:29.054 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_fast_shutdown_timeout": "15", 2023-10-09T05:50:29.054 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-10-09T05:50:29.054 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-10-09T05:50:29.054 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-10-09T05:50:29.054 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_function_tracing": "false", 2023-10-09T05:50:29.054 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_heartbeat_grace": "20", 2023-10-09T05:50:29.054 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_heartbeat_interval": "6", 2023-10-09T05:50:29.054 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-10-09T05:50:29.055 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_heartbeat_min_peers": "10", 2023-10-09T05:50:29.055 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_heartbeat_min_size": "2000", 2023-10-09T05:50:29.055 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_heartbeat_stale": "600", 2023-10-09T05:50:29.055 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-10-09T05:50:29.055 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_hit_set_max_size": "100000", 2023-10-09T05:50:29.055 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_hit_set_min_size": "1000", 2023-10-09T05:50:29.055 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-10-09T05:50:29.055 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-10-09T05:50:29.056 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-10-09T05:50:29.056 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-10-09T05:50:29.056 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-10-09T05:50:29.056 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_journal_flush_on_shutdown": "true", 2023-10-09T05:50:29.056 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_journal_size": "5120", 2023-10-09T05:50:29.056 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_kill_backfill_at": "0", 2023-10-09T05:50:29.056 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_loop_before_reset_tphandle": "64", 2023-10-09T05:50:29.056 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_map_cache_size": "50", 2023-10-09T05:50:29.056 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_map_dedup": "true", 2023-10-09T05:50:29.056 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_map_message_max": "40", 2023-10-09T05:50:29.057 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_map_message_max_bytes": "10485760", 2023-10-09T05:50:29.057 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_map_share_max_epochs": "40", 2023-10-09T05:50:29.057 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_attr_name_len": "100", 2023-10-09T05:50:29.057 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_attr_size": "0", 2023-10-09T05:50:29.057 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_backfills": "1", 2023-10-09T05:50:29.057 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_markdown_count": "5", 2023-10-09T05:50:29.057 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_markdown_period": "600", 2023-10-09T05:50:29.057 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_object_name_len": "2048", 2023-10-09T05:50:29.057 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_object_namespace_len": "256", 2023-10-09T05:50:29.057 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_object_size": "134217728", 2023-10-09T05:50:29.057 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-10-09T05:50:29.058 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_omap_entries_per_request": "1024", 2023-10-09T05:50:29.058 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_pg_blocked_by": "16", 2023-10-09T05:50:29.058 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_pg_log_entries": "10000", 2023-10-09T05:50:29.058 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-10-09T05:50:29.058 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_pgls": "1024", 2023-10-09T05:50:29.058 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_push_cost": "8388608", 2023-10-09T05:50:29.058 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_push_objects": "10", 2023-10-09T05:50:29.058 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_scrubs": "1", 2023-10-09T05:50:29.058 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-10-09T05:50:29.058 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_trimming_pgs": "2", 2023-10-09T05:50:29.058 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_write_op_reply_len": "64", 2023-10-09T05:50:29.059 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_write_size": "90", 2023-10-09T05:50:29.059 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-10-09T05:50:29.059 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-10-09T05:50:29.059 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-10-09T05:50:29.059 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-10-09T05:50:29.059 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-10-09T05:50:29.059 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-10-09T05:50:29.059 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-10-09T05:50:29.059 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_override_recovery_settings": "false", 2023-10-09T05:50:29.059 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_profile": "balanced", 2023-10-09T05:50:29.059 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-10-09T05:50:29.059 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-10-09T05:50:29.060 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-10-09T05:50:29.060 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-10-09T05:50:29.060 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-10-09T05:50:29.060 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-10-09T05:50:29.060 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-10-09T05:50:29.060 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-10-09T05:50:29.060 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-10-09T05:50:29.060 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-10-09T05:50:29.060 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_skip_benchmark": "false", 2023-10-09T05:50:29.060 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_memory_base": "805306368", 2023-10-09T05:50:29.061 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_memory_cache_min": "134217728", 2023-10-09T05:50:29.061 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-10-09T05:50:29.061 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-10-09T05:50:29.061 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_memory_target": "4294967296", 2023-10-09T05:50:29.061 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_memory_target_autotune": "false", 2023-10-09T05:50:29.061 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-10-09T05:50:29.061 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_min_pg_log_entries": "250", 2023-10-09T05:50:29.061 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_min_recovery_priority": "0", 2023-10-09T05:50:29.061 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mon_heartbeat_interval": "30", 2023-10-09T05:50:29.061 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-10-09T05:50:29.061 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mon_report_interval": "5", 2023-10-09T05:50:29.061 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mon_report_max_in_flight": "2", 2023-10-09T05:50:29.062 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-10-09T05:50:29.062 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_num_cache_shards": "32", 2023-10-09T05:50:29.062 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_num_op_tracker_shard": "32", 2023-10-09T05:50:29.062 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_numa_auto_affinity": "true", 2023-10-09T05:50:29.062 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_numa_node": "-1", 2023-10-09T05:50:29.062 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_numa_prefer_iface": "true", 2023-10-09T05:50:29.062 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-10-09T05:50:29.062 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_objecter_finishers": "1", 2023-10-09T05:50:29.062 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_objectstore": "bluestore", 2023-10-09T05:50:29.062 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_objectstore_fuse": "false", 2023-10-09T05:50:29.062 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_objectstore_tracing": "false", 2023-10-09T05:50:29.063 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_complaint_time": "30.000000", 2023-10-09T05:50:29.063 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_history_duration": "600", 2023-10-09T05:50:29.063 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_history_size": "20", 2023-10-09T05:50:29.063 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_history_slow_op_size": "20", 2023-10-09T05:50:29.063 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-10-09T05:50:29.063 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_log_threshold": "5", 2023-10-09T05:50:29.063 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_num_shards": "0", 2023-10-09T05:50:29.063 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_num_shards_hdd": "5", 2023-10-09T05:50:29.063 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_num_shards_ssd": "8", 2023-10-09T05:50:29.063 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_num_threads_per_shard": "0", 2023-10-09T05:50:29.063 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-10-09T05:50:29.064 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-10-09T05:50:29.064 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-10-09T05:50:29.064 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_pq_min_cost": "65536", 2023-10-09T05:50:29.064 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_queue": "mclock_scheduler", 2023-10-09T05:50:29.064 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_queue_cut_off": "high", 2023-10-09T05:50:29.064 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_thread_suicide_timeout": "150", 2023-10-09T05:50:29.064 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_thread_timeout": "15", 2023-10-09T05:50:29.064 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_open_classes_on_start": "true", 2023-10-09T05:50:29.064 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_os_flags": "0", 2023-10-09T05:50:29.064 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_peering_op_priority": "255", 2023-10-09T05:50:29.064 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_delete_cost": "1048576", 2023-10-09T05:50:29.064 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_delete_priority": "5", 2023-10-09T05:50:29.065 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-10-09T05:50:29.065 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-10-09T05:50:29.065 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_log_dups_tracked": "3000", 2023-10-09T05:50:29.065 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_log_trim_max": "10000", 2023-10-09T05:50:29.065 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_log_trim_min": "100", 2023-10-09T05:50:29.065 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-10-09T05:50:29.065 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_object_context_cache_count": "64", 2023-10-09T05:50:29.065 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_stat_report_interval_max": "500", 2023-10-09T05:50:29.065 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-10-09T05:50:29.065 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-10-09T05:50:29.065 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-10-09T05:50:29.066 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-10-09T05:50:29.066 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-10-09T05:50:29.066 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-10-09T05:50:29.066 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_crimson": "false", 2023-10-09T05:50:29.066 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_crush_rule": "-1", 2023-10-09T05:50:29.066 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_ec_fast_read": "false", 2023-10-09T05:50:29.066 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-10-09T05:50:29.066 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_flag_bulk": "false", 2023-10-09T05:50:29.066 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-10-09T05:50:29.066 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_flag_nodelete": "false", 2023-10-09T05:50:29.066 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-10-09T05:50:29.066 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-10-09T05:50:29.067 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_flags": "0", 2023-10-09T05:50:29.067 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-10-09T05:50:29.067 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_min_size": "0", 2023-10-09T05:50:29.067 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-10-09T05:50:29.067 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_pg_num": "32", 2023-10-09T05:50:29.067 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_pgp_num": "0", 2023-10-09T05:50:29.067 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-10-09T05:50:29.067 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_size": "3", 2023-10-09T05:50:29.067 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_type": "replicated", 2023-10-09T05:50:29.067 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-10-09T05:50:29.067 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_use_gmt_hitset": "true", 2023-10-09T05:50:29.068 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_push_per_object_cost": "1000", 2023-10-09T05:50:29.068 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_read_ec_check_for_errors": "false", 2023-10-09T05:50:29.068 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recover_clone_overlap": "true", 2023-10-09T05:50:29.068 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recover_clone_overlap_limit": "10", 2023-10-09T05:50:29.068 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_cost": "20971520", 2023-10-09T05:50:29.068 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_delay_start": "0.000000", 2023-10-09T05:50:29.068 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_max_active": "0", 2023-10-09T05:50:29.068 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_max_active_hdd": "3", 2023-10-09T05:50:29.068 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_max_active_ssd": "10", 2023-10-09T05:50:29.068 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_max_chunk": "8388608", 2023-10-09T05:50:29.069 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-10-09T05:50:29.069 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_max_single_start": "1", 2023-10-09T05:50:29.069 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_op_priority": "3", 2023-10-09T05:50:29.069 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_op_warn_multiple": "16", 2023-10-09T05:50:29.069 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_priority": "5", 2023-10-09T05:50:29.069 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_retry_interval": "30.000000", 2023-10-09T05:50:29.069 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_sleep": "0.000000", 2023-10-09T05:50:29.069 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-10-09T05:50:29.069 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-10-09T05:50:29.069 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-10-09T05:50:29.069 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_repair_during_recovery": "false", 2023-10-09T05:50:29.069 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_requested_scrub_priority": "120", 2023-10-09T05:50:29.070 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-10-09T05:50:29.070 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_rollback_to_cluster_snap": "", 2023-10-09T05:50:29.070 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_auto_repair": "false", 2023-10-09T05:50:29.070 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-10-09T05:50:29.070 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-10-09T05:50:29.070 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_begin_hour": "0", 2023-10-09T05:50:29.070 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_begin_week_day": "0", 2023-10-09T05:50:29.070 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_chunk_max": "25", 2023-10-09T05:50:29.070 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_chunk_min": "5", 2023-10-09T05:50:29.070 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_cost": "52428800", 2023-10-09T05:50:29.070 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_during_recovery": "false", 2023-10-09T05:50:29.071 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_end_hour": "0", 2023-10-09T05:50:29.071 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_end_week_day": "0", 2023-10-09T05:50:29.071 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_event_cost": "4096", 2023-10-09T05:50:29.071 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-10-09T05:50:29.071 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-10-09T05:50:29.071 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_invalid_stats": "true", 2023-10-09T05:50:29.071 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_load_threshold": "0.500000", 2023-10-09T05:50:29.071 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_max_interval": "604800.000000", 2023-10-09T05:50:29.071 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_max_preemptions": "5", 2023-10-09T05:50:29.071 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_min_interval": "86400.000000", 2023-10-09T05:50:29.071 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_priority": "5", 2023-10-09T05:50:29.072 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_reservation_timeout": "5000", 2023-10-09T05:50:29.072 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_sleep": "0.000000", 2023-10-09T05:50:29.072 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-10-09T05:50:29.072 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-10-09T05:50:29.072 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-10-09T05:50:29.072 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_shutdown_pgref_assert": "false", 2023-10-09T05:50:29.072 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_skip_data_digest": "false", 2023-10-09T05:50:29.072 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_smart_report_timeout": "5", 2023-10-09T05:50:29.072 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_snap_trim_cost": "1048576", 2023-10-09T05:50:29.072 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_snap_trim_priority": "5", 2023-10-09T05:50:29.072 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_snap_trim_sleep": "0.000000", 2023-10-09T05:50:29.073 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-10-09T05:50:29.073 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-10-09T05:50:29.073 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-10-09T05:50:29.073 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-10-09T05:50:29.073 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_stats_update_period_scrubbing": "15", 2023-10-09T05:50:29.073 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-10-09T05:50:29.073 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_target_transaction_size": "30", 2023-10-09T05:50:29.073 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-10-09T05:50:29.073 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-10-09T05:50:29.073 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-10-09T05:50:29.073 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-10-09T05:50:29.073 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-10-09T05:50:29.074 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-10-09T05:50:29.074 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-10-09T05:50:29.074 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_default_cache_mode": "writeback", 2023-10-09T05:50:29.074 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-10-09T05:50:29.074 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-10-09T05:50:29.074 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tracing": "false", 2023-10-09T05:50:29.074 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_use_stale_snap": "false", 2023-10-09T05:50:29.074 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-10-09T05:50:29.074 INFO:teuthology.orchestra.run.smithi064.stdout: "osdc_blkin_trace_all": "false", 2023-10-09T05:50:29.074 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_kill_at": "0", 2023-10-09T05:50:29.074 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_max_join_drift": "10", 2023-10-09T05:50:29.074 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_min": "500", 2023-10-09T05:50:29.075 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_min_wait": "0.050000", 2023-10-09T05:50:29.075 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_propose_interval": "1.000000", 2023-10-09T05:50:29.075 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_service_trim_max": "500", 2023-10-09T05:50:29.075 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_service_trim_max_multiplier": "20", 2023-10-09T05:50:29.075 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_service_trim_min": "250", 2023-10-09T05:50:29.075 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_stash_full_interval": "25", 2023-10-09T05:50:29.075 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_trim_max": "500", 2023-10-09T05:50:29.075 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_trim_min": "250", 2023-10-09T05:50:29.075 INFO:teuthology.orchestra.run.smithi064.stdout: "perf": "true", 2023-10-09T05:50:29.075 INFO:teuthology.orchestra.run.smithi064.stdout: "pid_file": "", 2023-10-09T05:50:29.075 INFO:teuthology.orchestra.run.smithi064.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-10-09T05:50:29.075 INFO:teuthology.orchestra.run.smithi064.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-10-09T05:50:29.076 INFO:teuthology.orchestra.run.smithi064.stdout: "public_addr": "-", 2023-10-09T05:50:29.076 INFO:teuthology.orchestra.run.smithi064.stdout: "public_addrv": "", 2023-10-09T05:50:29.076 INFO:teuthology.orchestra.run.smithi064.stdout: "public_bind_addr": "-", 2023-10-09T05:50:29.076 INFO:teuthology.orchestra.run.smithi064.stdout: "public_network": "172.21.0.0/20", 2023-10-09T05:50:29.076 INFO:teuthology.orchestra.run.smithi064.stdout: "public_network_interface": "", 2023-10-09T05:50:29.076 INFO:teuthology.orchestra.run.smithi064.stdout: "qat_compressor_enabled": "false", 2023-10-09T05:50:29.076 INFO:teuthology.orchestra.run.smithi064.stdout: "qat_compressor_session_max_number": "256", 2023-10-09T05:50:29.076 INFO:teuthology.orchestra.run.smithi064.stdout: "rados_mon_op_timeout": "0", 2023-10-09T05:50:29.076 INFO:teuthology.orchestra.run.smithi064.stdout: "rados_osd_op_timeout": "0", 2023-10-09T05:50:29.076 INFO:teuthology.orchestra.run.smithi064.stdout: "rados_tracing": "false", 2023-10-09T05:50:29.076 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_atime_update_interval": "60", 2023-10-09T05:50:29.077 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-10-09T05:50:29.077 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_balance_parent_reads": "false", 2023-10-09T05:50:29.077 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_balance_snap_reads": "false", 2023-10-09T05:50:29.077 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_blkin_trace_all": "false", 2023-10-09T05:50:29.077 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_blocklist_expire_seconds": "0", 2023-10-09T05:50:29.077 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_blocklist_on_break_lock": "true", 2023-10-09T05:50:29.077 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache": "true", 2023-10-09T05:50:29.077 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache_block_writes_upfront": "false", 2023-10-09T05:50:29.077 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache_max_dirty": "25165824", 2023-10-09T05:50:29.077 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-10-09T05:50:29.077 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache_max_dirty_object": "0", 2023-10-09T05:50:29.077 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache_policy": "writearound", 2023-10-09T05:50:29.078 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache_size": "33554432", 2023-10-09T05:50:29.078 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache_target_dirty": "16777216", 2023-10-09T05:50:29.078 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-10-09T05:50:29.078 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_clone_copy_on_read": "false", 2023-10-09T05:50:29.078 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_compression_hint": "none", 2023-10-09T05:50:29.078 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_concurrent_management_ops": "10", 2023-10-09T05:50:29.078 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-10-09T05:50:29.078 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_clone_format": "auto", 2023-10-09T05:50:29.078 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_data_pool": "", 2023-10-09T05:50:29.078 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_features": "61", 2023-10-09T05:50:29.078 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_format": "2", 2023-10-09T05:50:29.078 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_map_options": "", 2023-10-09T05:50:29.079 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_order": "22", 2023-10-09T05:50:29.079 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_pool": "rbd", 2023-10-09T05:50:29.079 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-10-09T05:50:29.079 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_stripe_count": "0", 2023-10-09T05:50:29.079 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_stripe_unit": "0", 2023-10-09T05:50:29.079 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_disable_zero_copy_writes": "true", 2023-10-09T05:50:29.079 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_discard_granularity_bytes": "65536", 2023-10-09T05:50:29.079 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-10-09T05:50:29.079 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_enable_alloc_hint": "true", 2023-10-09T05:50:29.079 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-10-09T05:50:29.079 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_io_scheduler": "simple", 2023-10-09T05:50:29.079 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-10-09T05:50:29.079 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_commit_age": "5.000000", 2023-10-09T05:50:29.080 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-10-09T05:50:29.080 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-10-09T05:50:29.080 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-10-09T05:50:29.080 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-10-09T05:50:29.080 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_object_flush_interval": "0", 2023-10-09T05:50:29.080 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-10-09T05:50:29.080 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-10-09T05:50:29.080 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_order": "24", 2023-10-09T05:50:29.080 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_pool": "", 2023-10-09T05:50:29.080 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_splay_width": "4", 2023-10-09T05:50:29.080 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_localize_parent_reads": "false", 2023-10-09T05:50:29.080 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_localize_snap_reads": "false", 2023-10-09T05:50:29.081 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-10-09T05:50:29.081 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-10-09T05:50:29.081 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-10-09T05:50:29.081 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-10-09T05:50:29.081 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-10-09T05:50:29.081 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-10-09T05:50:29.081 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_image_policy_type": "simple", 2023-10-09T05:50:29.081 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-10-09T05:50:29.081 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-10-09T05:50:29.081 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-10-09T05:50:29.081 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-10-09T05:50:29.081 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-10-09T05:50:29.082 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-10-09T05:50:29.082 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-10-09T05:50:29.082 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_memory_autotune": "true", 2023-10-09T05:50:29.082 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_memory_base": "805306368", 2023-10-09T05:50:29.082 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-10-09T05:50:29.082 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-10-09T05:50:29.082 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-10-09T05:50:29.082 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-10-09T05:50:29.082 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_memory_target": "4294967296", 2023-10-09T05:50:29.082 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-10-09T05:50:29.082 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-10-09T05:50:29.082 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-10-09T05:50:29.083 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirroring_delete_delay": "0", 2023-10-09T05:50:29.083 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-10-09T05:50:29.083 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirroring_replay_delay": "0", 2023-10-09T05:50:29.083 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-10-09T05:50:29.083 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-10-09T05:50:29.083 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_move_to_trash_on_remove": "false", 2023-10-09T05:50:29.083 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-10-09T05:50:29.083 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mtime_update_interval": "60", 2023-10-09T05:50:29.083 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_non_blocking_aio": "true", 2023-10-09T05:50:29.083 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_op_thread_timeout": "60", 2023-10-09T05:50:29.083 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_op_threads": "1", 2023-10-09T05:50:29.083 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_parent_cache_enabled": "false", 2023-10-09T05:50:29.084 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_persistent_cache_mode": "disabled", 2023-10-09T05:50:29.084 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_persistent_cache_path": "/tmp", 2023-10-09T05:50:29.084 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_persistent_cache_size": "1073741824", 2023-10-09T05:50:29.084 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_plugins": "", 2023-10-09T05:50:29.084 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_bps_burst": "0", 2023-10-09T05:50:29.084 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-10-09T05:50:29.084 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_bps_limit": "0", 2023-10-09T05:50:29.084 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_exclude_ops": "0", 2023-10-09T05:50:29.084 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_iops_burst": "0", 2023-10-09T05:50:29.084 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-10-09T05:50:29.084 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_iops_limit": "0", 2023-10-09T05:50:29.084 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_read_bps_burst": "0", 2023-10-09T05:50:29.085 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-10-09T05:50:29.085 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_read_bps_limit": "0", 2023-10-09T05:50:29.085 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_read_iops_burst": "0", 2023-10-09T05:50:29.085 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-10-09T05:50:29.085 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_read_iops_limit": "0", 2023-10-09T05:50:29.085 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_schedule_tick_min": "50", 2023-10-09T05:50:29.085 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_write_bps_burst": "0", 2023-10-09T05:50:29.085 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-10-09T05:50:29.085 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_write_bps_limit": "0", 2023-10-09T05:50:29.085 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_write_iops_burst": "0", 2023-10-09T05:50:29.085 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-10-09T05:50:29.085 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_write_iops_limit": "0", 2023-10-09T05:50:29.086 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_quiesce_notification_attempts": "10", 2023-10-09T05:50:29.086 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_read_from_replica_policy": "default", 2023-10-09T05:50:29.086 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-10-09T05:50:29.086 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_readahead_max_bytes": "524288", 2023-10-09T05:50:29.086 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_readahead_trigger_requests": "10", 2023-10-09T05:50:29.086 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_request_timed_out_seconds": "30", 2023-10-09T05:50:29.086 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_skip_partial_discard": "true", 2023-10-09T05:50:29.086 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-10-09T05:50:29.086 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_tracing": "false", 2023-10-09T05:50:29.086 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_validate_names": "true", 2023-10-09T05:50:29.086 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_validate_pool": "true", 2023-10-09T05:50:29.086 INFO:teuthology.orchestra.run.smithi064.stdout: "restapi_base_url": "", 2023-10-09T05:50:29.087 INFO:teuthology.orchestra.run.smithi064.stdout: "restapi_log_level": "", 2023-10-09T05:50:29.087 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_acl_grants_max_num": "100", 2023-10-09T05:50:29.087 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_admin_entry": "admin", 2023-10-09T05:50:29.087 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-10-09T05:50:29.087 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_backend_store": "rados", 2023-10-09T05:50:29.087 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_barbican_url": "", 2023-10-09T05:50:29.087 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_beast_enable_async": "true", 2023-10-09T05:50:29.087 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-10-09T05:50:29.087 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-10-09T05:50:29.087 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_bucket_index_max_aio": "128", 2023-10-09T05:50:29.087 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-10-09T05:50:29.087 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-10-09T05:50:29.087 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_bucket_quota_ttl": "600", 2023-10-09T05:50:29.088 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-10-09T05:50:29.088 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_cache_enabled": "true", 2023-10-09T05:50:29.088 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_cache_expiry_interval": "900", 2023-10-09T05:50:29.088 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_cache_lru_size": "10000", 2023-10-09T05:50:29.088 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_config_store": "rados", 2023-10-09T05:50:29.088 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_content_length_compat": "false", 2023-10-09T05:50:29.088 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_copy_obj_progress": "true", 2023-10-09T05:50:29.088 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-10-09T05:50:29.088 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_cors_rules_max_num": "100", 2023-10-09T05:50:29.088 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_cross_domain_policy": "", 2023-10-09T05:50:29.088 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_default_encryption_key": "", 2023-10-09T05:50:29.088 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_kmip_addr": "", 2023-10-09T05:50:29.089 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_kmip_ca_path": "", 2023-10-09T05:50:29.089 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_kmip_client_cert": "", 2023-10-09T05:50:29.089 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_kmip_client_key": "", 2023-10-09T05:50:29.089 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-10-09T05:50:29.089 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_kmip_password": "", 2023-10-09T05:50:29.089 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-10-09T05:50:29.089 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_kmip_username": "", 2023-10-09T05:50:29.089 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_require_ssl": "true", 2023-10-09T05:50:29.089 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-10-09T05:50:29.089 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-10-09T05:50:29.089 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-10-09T05:50:29.090 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-10-09T05:50:29.090 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-10-09T05:50:29.090 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-10-09T05:50:29.090 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-10-09T05:50:29.090 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-10-09T05:50:29.090 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-10-09T05:50:29.090 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-10-09T05:50:29.090 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-10-09T05:50:29.090 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-10-09T05:50:29.090 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-10-09T05:50:29.090 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-10-09T05:50:29.091 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_suppress_logs": "true", 2023-10-09T05:50:29.091 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_addr": "", 2023-10-09T05:50:29.091 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_auth": "token", 2023-10-09T05:50:29.091 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_namespace": "", 2023-10-09T05:50:29.091 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_prefix": "", 2023-10-09T05:50:29.091 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-10-09T05:50:29.091 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-10-09T05:50:29.091 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-10-09T05:50:29.091 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-10-09T05:50:29.091 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_token_file": "", 2023-10-09T05:50:29.091 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-10-09T05:50:29.091 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_curl_buffersize": "524288", 2023-10-09T05:50:29.091 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_curl_low_speed_limit": "1024", 2023-10-09T05:50:29.092 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_curl_low_speed_time": "300", 2023-10-09T05:50:29.092 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_curl_tcp_keepalive": "0", 2023-10-09T05:50:29.092 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-10-09T05:50:29.092 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-10-09T05:50:29.092 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-10-09T05:50:29.092 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-10-09T05:50:29.092 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-10-09T05:50:29.092 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d3n_l1_fadvise": "4", 2023-10-09T05:50:29.092 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-10-09T05:50:29.092 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-10-09T05:50:29.092 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-10-09T05:50:29.092 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-10-09T05:50:29.092 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_data_log_changes_size": "1000", 2023-10-09T05:50:29.093 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_data_log_num_shards": "128", 2023-10-09T05:50:29.093 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-10-09T05:50:29.093 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_data_log_window": "30", 2023-10-09T05:50:29.093 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_data_notify_interval_msec": "0", 2023-10-09T05:50:29.093 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_data_sync_poll_interval": "20", 2023-10-09T05:50:29.093 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_data_sync_spawn_window": "20", 2023-10-09T05:50:29.093 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-10-09T05:50:29.093 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-10-09T05:50:29.093 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_default_data_log_backing": "fifo", 2023-10-09T05:50:29.093 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-10-09T05:50:29.093 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_default_region_info_oid": "default.region", 2023-10-09T05:50:29.093 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-10-09T05:50:29.094 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-10-09T05:50:29.094 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_defer_to_bucket_acls": "", 2023-10-09T05:50:29.094 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-10-09T05:50:29.094 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-10-09T05:50:29.094 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-10-09T05:50:29.094 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-10-09T05:50:29.094 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-10-09T05:50:29.094 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-10-09T05:50:29.094 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-10-09T05:50:29.094 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-10-09T05:50:29.094 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_data_res": "500.000000", 2023-10-09T05:50:29.094 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-10-09T05:50:29.095 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-10-09T05:50:29.095 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-10-09T05:50:29.095 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-10-09T05:50:29.095 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dns_name": "", 2023-10-09T05:50:29.095 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dns_s3website_name": "", 2023-10-09T05:50:29.095 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dynamic_resharding": "true", 2023-10-09T05:50:29.095 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-10-09T05:50:29.095 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enable_gc_threads": "true", 2023-10-09T05:50:29.095 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enable_lc_threads": "true", 2023-10-09T05:50:29.095 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enable_ops_log": "false", 2023-10-09T05:50:29.095 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enable_quota_threads": "true", 2023-10-09T05:50:29.095 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enable_static_website": "false", 2023-10-09T05:50:29.096 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enable_usage_log": "false", 2023-10-09T05:50:29.096 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enforce_swift_acls": "true", 2023-10-09T05:50:29.096 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_exit_timeout_secs": "120", 2023-10-09T05:50:29.096 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_expose_bucket": "false", 2023-10-09T05:50:29.096 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_extended_http_attrs": "", 2023-10-09T05:50:29.096 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_filter": "none", 2023-10-09T05:50:29.096 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-10-09T05:50:29.096 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_frontends": "beast port=7480", 2023-10-09T05:50:29.096 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_max_concurrent_io": "10", 2023-10-09T05:50:29.096 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_max_deferred": "50", 2023-10-09T05:50:29.096 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-10-09T05:50:29.096 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_max_objs": "32", 2023-10-09T05:50:29.096 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_max_queue_size": "134213632", 2023-10-09T05:50:29.097 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_max_trim_chunk": "16", 2023-10-09T05:50:29.097 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_obj_min_wait": "7200", 2023-10-09T05:50:29.097 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_processor_max_time": "3600", 2023-10-09T05:50:29.097 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_processor_period": "3600", 2023-10-09T05:50:29.097 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-10-09T05:50:29.097 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_get_obj_window_size": "16777216", 2023-10-09T05:50:29.097 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_healthcheck_disabling_path": "", 2023-10-09T05:50:29.097 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ignore_get_invalid_range": "false", 2023-10-09T05:50:29.097 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_init_timeout": "300", 2023-10-09T05:50:29.097 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-10-09T05:50:29.097 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-10-09T05:50:29.097 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-10-09T05:50:29.098 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-10-09T05:50:29.098 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_admin_domain": "", 2023-10-09T05:50:29.098 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_admin_password": "", 2023-10-09T05:50:29.098 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_admin_password_path": "", 2023-10-09T05:50:29.098 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_admin_project": "", 2023-10-09T05:50:29.098 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_admin_tenant": "", 2023-10-09T05:50:29.098 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_admin_token": "", 2023-10-09T05:50:29.098 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_admin_token_path": "", 2023-10-09T05:50:29.098 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_admin_user": "", 2023-10-09T05:50:29.098 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_api_version": "2", 2023-10-09T05:50:29.099 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_barbican_domain": "", 2023-10-09T05:50:29.099 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_barbican_password": "", 2023-10-09T05:50:29.099 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_barbican_project": "", 2023-10-09T05:50:29.099 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_barbican_tenant": "", 2023-10-09T05:50:29.099 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_barbican_user": "", 2023-10-09T05:50:29.099 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-10-09T05:50:29.099 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_implicit_tenants": "false", 2023-10-09T05:50:29.099 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-10-09T05:50:29.099 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_service_token_enabled": "false", 2023-10-09T05:50:29.099 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_token_cache_size": "10000", 2023-10-09T05:50:29.100 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_url": "", 2023-10-09T05:50:29.100 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_verify_ssl": "true", 2023-10-09T05:50:29.100 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lc_debug_interval": "-1", 2023-10-09T05:50:29.100 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lc_lock_max_time": "90", 2023-10-09T05:50:29.100 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lc_max_objs": "32", 2023-10-09T05:50:29.100 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lc_max_rules": "1000", 2023-10-09T05:50:29.100 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lc_max_worker": "3", 2023-10-09T05:50:29.100 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lc_max_wp_worker": "3", 2023-10-09T05:50:29.100 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lc_thread_delay": "0", 2023-10-09T05:50:29.100 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-10-09T05:50:29.100 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ldap_dnattr": "uid", 2023-10-09T05:50:29.100 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-10-09T05:50:29.101 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ldap_searchfilter": "", 2023-10-09T05:50:29.101 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-10-09T05:50:29.101 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ldap_uri": "ldaps://", 2023-10-09T05:50:29.101 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-10-09T05:50:29.101 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-10-09T05:50:29.101 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-10-09T05:50:29.101 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_log_http_headers": "", 2023-10-09T05:50:29.101 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_log_nonexistent_bucket": "false", 2023-10-09T05:50:29.101 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-10-09T05:50:29.101 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_log_object_name_utc": "false", 2023-10-09T05:50:29.101 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-10-09T05:50:29.101 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_attr_name_len": "0", 2023-10-09T05:50:29.102 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_attr_size": "0", 2023-10-09T05:50:29.102 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_attrs_num_in_req": "0", 2023-10-09T05:50:29.102 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_chunk_size": "4194304", 2023-10-09T05:50:29.102 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_concurrent_requests": "1024", 2023-10-09T05:50:29.102 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-10-09T05:50:29.102 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_dynamic_shards": "1999", 2023-10-09T05:50:29.102 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_listing_results": "1000", 2023-10-09T05:50:29.102 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_notify_retries": "10", 2023-10-09T05:50:29.102 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_objs_per_shard": "100000", 2023-10-09T05:50:29.102 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_put_param_size": "1048576", 2023-10-09T05:50:29.102 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_put_size": "5368709120", 2023-10-09T05:50:29.103 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_slo_entries": "1000", 2023-10-09T05:50:29.103 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_md_log_max_shards": "64", 2023-10-09T05:50:29.103 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_md_notify_interval_msec": "200", 2023-10-09T05:50:29.103 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_meta_sync_poll_interval": "20", 2023-10-09T05:50:29.103 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_meta_sync_spawn_window": "20", 2023-10-09T05:50:29.103 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-10-09T05:50:29.103 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_mp_lock_max_time": "600", 2023-10-09T05:50:29.103 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-10-09T05:50:29.103 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_multipart_min_part_size": "5242880", 2023-10-09T05:50:29.104 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-10-09T05:50:29.104 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-10-09T05:50:29.104 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_fhcache_size": "2017", 2023-10-09T05:50:29.104 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-10-09T05:50:29.104 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-10-09T05:50:29.104 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_lru_lanes": "5", 2023-10-09T05:50:29.104 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_max_gc": "300", 2023-10-09T05:50:29.104 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-10-09T05:50:29.104 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_run_gc_threads": "false", 2023-10-09T05:50:29.104 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_run_lc_threads": "false", 2023-10-09T05:50:29.104 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_run_quota_threads": "false", 2023-10-09T05:50:29.104 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_run_sync_thread": "false", 2023-10-09T05:50:29.105 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-10-09T05:50:29.105 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-10-09T05:50:29.105 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_num_async_rados_threads": "32", 2023-10-09T05:50:29.105 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_num_control_oids": "8", 2023-10-09T05:50:29.105 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_numa_node": "-1", 2023-10-09T05:50:29.105 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_obj_stripe_size": "4194304", 2023-10-09T05:50:29.105 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-10-09T05:50:29.105 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_objexp_chunk_size": "100", 2023-10-09T05:50:29.105 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_objexp_gc_interval": "600", 2023-10-09T05:50:29.105 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_objexp_hints_num_shards": "127", 2023-10-09T05:50:29.105 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-10-09T05:50:29.106 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-10-09T05:50:29.106 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_op_thread_timeout": "600", 2023-10-09T05:50:29.106 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_op_tracing": "false", 2023-10-09T05:50:29.106 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_opa_token": "", 2023-10-09T05:50:29.106 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_opa_url": "", 2023-10-09T05:50:29.106 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_opa_verify_ssl": "true", 2023-10-09T05:50:29.106 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-10-09T05:50:29.106 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-10-09T05:50:29.106 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ops_log_rados": "false", 2023-10-09T05:50:29.106 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ops_log_socket_path": "", 2023-10-09T05:50:29.106 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-10-09T05:50:29.106 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-10-09T05:50:29.107 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-10-09T05:50:29.107 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_period_push_interval": "2.000000", 2023-10-09T05:50:29.107 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_period_push_interval_max": "30.000000", 2023-10-09T05:50:29.107 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_period_root_pool": ".rgw.root", 2023-10-09T05:50:29.107 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-10-09T05:50:29.107 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_print_continue": "true", 2023-10-09T05:50:29.107 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_print_prohibited_content_length": "false", 2023-10-09T05:50:29.107 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-10-09T05:50:29.107 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-10-09T05:50:29.107 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-10-09T05:50:29.107 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-10-09T05:50:29.107 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_rados_tracing": "false", 2023-10-09T05:50:29.108 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_realm": "", 2023-10-09T05:50:29.108 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_realm_id": "", 2023-10-09T05:50:29.108 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-10-09T05:50:29.108 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_region": "", 2023-10-09T05:50:29.108 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_region_root_pool": ".rgw.root", 2023-10-09T05:50:29.108 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_relaxed_region_enforcement": "false", 2023-10-09T05:50:29.108 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-10-09T05:50:29.108 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-10-09T05:50:29.108 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_request_uri": "", 2023-10-09T05:50:29.108 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_reshard_batch_size": "64", 2023-10-09T05:50:29.108 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-10-09T05:50:29.108 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_reshard_max_aio": "128", 2023-10-09T05:50:29.109 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_reshard_num_logs": "16", 2023-10-09T05:50:29.109 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_reshard_thread_interval": "600", 2023-10-09T05:50:29.109 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_resolve_cname": "false", 2023-10-09T05:50:29.109 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_rest_getusage_op_compat": "false", 2023-10-09T05:50:29.109 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_run_sync_thread": "true", 2023-10-09T05:50:29.109 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-10-09T05:50:29.109 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_s3_auth_use_keystone": "false", 2023-10-09T05:50:29.109 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_s3_auth_use_ldap": "false", 2023-10-09T05:50:29.109 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_s3_auth_use_rados": "true", 2023-10-09T05:50:29.109 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_s3_auth_use_sts": "false", 2023-10-09T05:50:29.109 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-10-09T05:50:29.109 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_s3_success_create_obj_status": "0", 2023-10-09T05:50:29.110 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-10-09T05:50:29.110 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_scheduler_type": "throttler", 2023-10-09T05:50:29.110 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_script_uri": "", 2023-10-09T05:50:29.110 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_service_provider_name": "", 2023-10-09T05:50:29.110 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-10-09T05:50:29.110 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sts_client_id": "", 2023-10-09T05:50:29.110 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sts_client_secret": "", 2023-10-09T05:50:29.110 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sts_entry": "sts", 2023-10-09T05:50:29.110 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sts_key": "sts", 2023-10-09T05:50:29.110 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sts_max_session_duration": "43200", 2023-10-09T05:50:29.110 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sts_min_session_duration": "900", 2023-10-09T05:50:29.111 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sts_token_introspection_url": "", 2023-10-09T05:50:29.111 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_account_in_url": "false", 2023-10-09T05:50:29.111 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_auth_entry": "auth", 2023-10-09T05:50:29.111 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_auth_url": "", 2023-10-09T05:50:29.111 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_custom_header": "", 2023-10-09T05:50:29.111 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_enforce_content_length": "false", 2023-10-09T05:50:29.111 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_need_stats": "true", 2023-10-09T05:50:29.111 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_tenant_name": "", 2023-10-09T05:50:29.111 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_token_expiration": "86400", 2023-10-09T05:50:29.111 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_url": "", 2023-10-09T05:50:29.111 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_url_prefix": "swift", 2023-10-09T05:50:29.111 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_versioning_enabled": "false", 2023-10-09T05:50:29.112 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-10-09T05:50:29.112 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-10-09T05:50:29.112 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_lease_period": "120", 2023-10-09T05:50:29.112 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-10-09T05:50:29.112 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_log_trim_interval": "1200", 2023-10-09T05:50:29.112 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-10-09T05:50:29.112 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-10-09T05:50:29.112 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-10-09T05:50:29.112 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_obj_etag_verify": "false", 2023-10-09T05:50:29.112 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_trace_history_size": "4096", 2023-10-09T05:50:29.112 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-10-09T05:50:29.113 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-10-09T05:50:29.113 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_thread_pool_size": "512", 2023-10-09T05:50:29.113 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_torrent_comment": "", 2023-10-09T05:50:29.113 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_torrent_createby": "", 2023-10-09T05:50:29.113 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_torrent_encoding": "", 2023-10-09T05:50:29.113 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_torrent_flag": "false", 2023-10-09T05:50:29.113 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_torrent_origin": "", 2023-10-09T05:50:29.113 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_torrent_sha_unit": "524288", 2023-10-09T05:50:29.113 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_torrent_tracker": "", 2023-10-09T05:50:29.113 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_trust_forwarded_https": "false", 2023-10-09T05:50:29.113 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-10-09T05:50:29.114 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_usage_log_tick_interval": "30", 2023-10-09T05:50:29.114 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_usage_max_shards": "32", 2023-10-09T05:50:29.114 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_usage_max_user_shards": "1", 2023-10-09T05:50:29.114 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_use_opa_authz": "false", 2023-10-09T05:50:29.114 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-10-09T05:50:29.114 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_default_quota_max_size": "-1", 2023-10-09T05:50:29.114 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_max_buckets": "1000", 2023-10-09T05:50:29.114 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_policies_max_num": "100", 2023-10-09T05:50:29.114 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-10-09T05:50:29.114 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-10-09T05:50:29.114 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_quota_sync_interval": "86400", 2023-10-09T05:50:29.114 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-10-09T05:50:29.115 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_unique_email": "true", 2023-10-09T05:50:29.115 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_verify_ssl": "true", 2023-10-09T05:50:29.115 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_website_routing_rules_max_num": "50", 2023-10-09T05:50:29.115 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_zone": "", 2023-10-09T05:50:29.115 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_zone_id": "", 2023-10-09T05:50:29.115 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-10-09T05:50:29.115 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_zonegroup": "", 2023-10-09T05:50:29.115 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_zonegroup_id": "", 2023-10-09T05:50:29.115 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-10-09T05:50:29.115 INFO:teuthology.orchestra.run.smithi064.stdout: "rgwlc_auto_session_clear": "true", 2023-10-09T05:50:29.115 INFO:teuthology.orchestra.run.smithi064.stdout: "rgwlc_skip_bucket_step": "false", 2023-10-09T05:50:29.115 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_block_size": "4096", 2023-10-09T05:50:29.115 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-10-09T05:50:29.116 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-10-09T05:50:29.116 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-10-09T05:50:29.116 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-10-09T05:50:29.116 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cache_shard_bits": "4", 2023-10-09T05:50:29.116 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cache_size": "536870912", 2023-10-09T05:50:29.116 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cache_type": "binned_lru", 2023-10-09T05:50:29.116 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-10-09T05:50:29.116 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-10-09T05:50:29.116 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-10-09T05:50:29.116 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_collect_compaction_stats": "false", 2023-10-09T05:50:29.116 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_collect_extended_stats": "false", 2023-10-09T05:50:29.116 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_collect_memory_stats": "false", 2023-10-09T05:50:29.117 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-10-09T05:50:29.117 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_index_type": "binary_search", 2023-10-09T05:50:29.117 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_log_to_ceph_log": "true", 2023-10-09T05:50:29.117 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_metadata_block_size": "4096", 2023-10-09T05:50:29.117 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_partition_filters": "false", 2023-10-09T05:50:29.117 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_perf": "false", 2023-10-09T05:50:29.117 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-10-09T05:50:29.117 INFO:teuthology.orchestra.run.smithi064.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-10-09T05:50:29.117 INFO:teuthology.orchestra.run.smithi064.stdout: "rotating_keys_renewal_timeout": "10", 2023-10-09T05:50:29.117 INFO:teuthology.orchestra.run.smithi064.stdout: "run_dir": "/var/run/ceph", 2023-10-09T05:50:29.117 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_block_create": "true", 2023-10-09T05:50:29.117 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_cache_lru_size": "67108864", 2023-10-09T05:50:29.118 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_cbjournal_size": "5368709120", 2023-10-09T05:50:29.118 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_default_max_object_size": "16777216", 2023-10-09T05:50:29.118 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-10-09T05:50:29.118 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_device_size": "53687091200", 2023-10-09T05:50:29.118 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_journal_batch_capacity": "16", 2023-10-09T05:50:29.118 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-10-09T05:50:29.118 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-10-09T05:50:29.118 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_journal_iodepth_limit": "5", 2023-10-09T05:50:29.118 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_main_device_type": "SSD", 2023-10-09T05:50:29.118 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_max_concurrent_transactions": "8", 2023-10-09T05:50:29.118 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-10-09T05:50:29.118 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-10-09T05:50:29.119 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-10-09T05:50:29.119 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-10-09T05:50:29.119 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_segment_size": "67108864", 2023-10-09T05:50:29.119 INFO:teuthology.orchestra.run.smithi064.stdout: "set_keepcaps": "false", 2023-10-09T05:50:29.119 INFO:teuthology.orchestra.run.smithi064.stdout: "setgroup": "ceph", 2023-10-09T05:50:29.119 INFO:teuthology.orchestra.run.smithi064.stdout: "setuser": "ceph", 2023-10-09T05:50:29.119 INFO:teuthology.orchestra.run.smithi064.stdout: "setuser_match_path": "", 2023-10-09T05:50:29.119 INFO:teuthology.orchestra.run.smithi064.stdout: "target_max_misplaced_ratio": "0.050000", 2023-10-09T05:50:29.119 INFO:teuthology.orchestra.run.smithi064.stdout: "thp": "false", 2023-10-09T05:50:29.119 INFO:teuthology.orchestra.run.smithi064.stdout: "threadpool_default_timeout": "60", 2023-10-09T05:50:29.119 INFO:teuthology.orchestra.run.smithi064.stdout: "threadpool_empty_queue_max_wait": "2", 2023-10-09T05:50:29.119 INFO:teuthology.orchestra.run.smithi064.stdout: "throttler_perf_counter": "true" 2023-10-09T05:50:29.120 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:50:29.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:29.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:29.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:29.644 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-10-09T05:50:30.002 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:30.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:30 smithi064 ceph-mon[212750]: pgmap v218: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:30.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:30 smithi064 ceph-mon[220044]: pgmap v218: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:30.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:30 smithi156 ceph-mon[195101]: pgmap v218: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:31.271 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:31 smithi064 ceph-mon[212750]: pgmap v219: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:31.271 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:31 smithi064 ceph-mon[220044]: pgmap v219: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:31.521 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a: {} 2023-10-09T05:50:31.532 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b: {} 2023-10-09T05:50:31.542 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c: {} 2023-10-09T05:50:31.543 INFO:teuthology.orchestra.run.smithi064.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T05:50:31.543 INFO:teuthology.orchestra.run.smithi064.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T05:50:31.543 INFO:teuthology.orchestra.run.smithi064.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T05:50:31.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:31 smithi156 ceph-mon[195101]: pgmap v219: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:32.216 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-10-09T05:50:32.216 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-10-09T05:50:32.217 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T05:50:32.218 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T05:50:32.219 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-10-09T05:50:32.219 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-10-09T05:50:32.219 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-10-09T05:50:32.219 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:50:32.219 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:50:32.220 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-10-09T05:50:32.220 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:32.220 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-10-09T05:50:32.220 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:50:32.221 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-10-09T05:50:32.221 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:50:32.221 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-10-09T05:50:32.221 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:50:32.274 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:32.324 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:32.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:32.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:32.431 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:32.467 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:32 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3425873000' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:50:32.467 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:32 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3425873000' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:50:32.471 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:32.532 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:32.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:32.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:32.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:32 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3425873000' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:50:32.638 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:32.649 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:32.670 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:32.719 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:32.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:32.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:32.777 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:32.827 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:32.854 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:32.897 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:32.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:32.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:33.004 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:33.030 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:33.072 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:33.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:33.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:33.145 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:33.147 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:33.177 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:50:33.202 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:33.244 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:50:33.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:33.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:50:33.349 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:50:33.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:33 smithi156 ceph-mon[195101]: pgmap v220: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:33.380 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:33.389 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:33 smithi064 ceph-mon[212750]: pgmap v220: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:33.389 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:33 smithi064 ceph-mon[220044]: pgmap v220: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:33.679 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:50:33.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:33.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:50:33.782 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:50:33.809 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:33.852 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:50:33.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:33.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:50:33.957 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:50:33.986 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:34.385 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:50:34.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:34.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:50:34.490 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:34.526 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:35.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:35 smithi064 ceph-mon[212750]: pgmap v221: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:35.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:35 smithi064 ceph-mon[220044]: pgmap v221: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:35.720 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:50:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:50:35] "GET /metrics HTTP/1.1" 200 33906 "" "Prometheus/2.43.0" 2023-10-09T05:50:35.743 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:35.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:35.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:35.847 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:35.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:35 smithi156 ceph-mon[195101]: pgmap v221: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:35.878 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:35.928 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:35.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:35.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:36.034 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:36.063 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:36.107 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:36.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:36.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:36.134 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:50:36.134 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T05:50:36.134 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2023-10-09T05:50:36.134 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:50:36.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:36.255 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:36.459 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:36.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:36.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:36.504 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T05:50:36.563 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:36.591 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:36.637 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:36.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:36.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:36.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:36 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1432188396' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:50:36.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:36 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1432188396' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:50:36.741 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:36.770 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:36 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1432188396' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:50:36.773 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:36.822 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:36.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:36.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:36.929 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:36.957 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:37.002 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:37.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:37.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:37.109 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:37.136 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:37.179 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:37.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:37.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:37.284 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:37.310 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:37.351 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:37.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:37.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:37.458 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:50:37.489 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:37.587 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:50:37.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:37.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:50:37.604 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:50:37.691 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:37.719 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:37.770 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:37 smithi064 ceph-mon[212750]: pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:37.771 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:37 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:50:37.771 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:37 smithi064 ceph-mon[212750]: failed to encode map e101 with expected crc 2023-10-09T05:50:37.771 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:37 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1432188396' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:50:37.771 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:37 smithi064 ceph-mon[212750]: osdmap e101: 8 total, 8 up, 8 in 2023-10-09T05:50:37.771 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:37 smithi064 ceph-mon[212750]: failed to encode map e101 with expected crc 2023-10-09T05:50:37.771 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:37 smithi064 ceph-mon[220044]: pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:37.771 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:37 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:50:37.772 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:37 smithi064 ceph-mon[220044]: failed to encode map e101 with expected crc 2023-10-09T05:50:37.772 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:37 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1432188396' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:50:37.772 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:37 smithi064 ceph-mon[220044]: osdmap e101: 8 total, 8 up, 8 in 2023-10-09T05:50:37.772 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:37 smithi064 ceph-mon[220044]: failed to encode map e101 with expected crc 2023-10-09T05:50:37.823 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:37.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:37.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:37 smithi156 ceph-mon[195101]: pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:37 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:50:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:37 smithi156 ceph-mon[195101]: failed to encode map e101 with expected crc 2023-10-09T05:50:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:37 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1432188396' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:50:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:37 smithi156 ceph-mon[195101]: osdmap e101: 8 total, 8 up, 8 in 2023-10-09T05:50:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:37 smithi156 ceph-mon[195101]: failed to encode map e101 with expected crc 2023-10-09T05:50:37.929 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:37.958 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:38.003 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:38.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:38.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:38.109 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:38.144 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:38.169 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:38.207 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:38.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:38.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:38.212 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:50:38.236 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:50:38.270 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:50:38.293 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:50:38.311 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:38.321 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:50:38.341 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:50:38.341 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:38.364 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:50:38.385 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:50:38.387 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:50:38.387 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:50:38.387 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:50:38.387 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:50:38.387 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:50:38.387 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:50:38.387 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:50:38.387 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:50:38.776 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_0' created 2023-10-09T05:50:38.950 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:38.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:38.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:38.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:38 smithi064 ceph-mon[212750]: failed to encode map e101 with expected crc 2023-10-09T05:50:38.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:38 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/845502693' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:38.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:38 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:50:38.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:38 smithi064 ceph-mon[220044]: failed to encode map e101 with expected crc 2023-10-09T05:50:38.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:38 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/845502693' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:38.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:38 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:50:39.055 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:39.083 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:39.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:38 smithi156 ceph-mon[195101]: failed to encode map e101 with expected crc 2023-10-09T05:50:39.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:38 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/845502693' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:39.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:38 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:50:39.860 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:39 smithi064 ceph-mon[212750]: failed to encode map e101 with expected crc 2023-10-09T05:50:39.861 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:39 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4129028284' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T05:50:39.861 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:39 smithi064 ceph-mon[212750]: pgmap v224: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:39.861 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:39 smithi064 ceph-mon[212750]: failed to encode map e102 with expected crc 2023-10-09T05:50:39.861 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:39 smithi064 ceph-mon[212750]: failed to encode map e102 with expected crc 2023-10-09T05:50:39.861 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:39 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4129028284' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T05:50:39.862 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:39 smithi064 ceph-mon[212750]: osdmap e102: 8 total, 8 up, 8 in 2023-10-09T05:50:39.862 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:39 smithi064 ceph-mon[212750]: failed to encode map e102 with expected crc 2023-10-09T05:50:39.862 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:39 smithi064 ceph-mon[212750]: failed to encode map e102 with expected crc 2023-10-09T05:50:39.863 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:39 smithi064 ceph-mon[212750]: failed to encode map e102 with expected crc 2023-10-09T05:50:39.863 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:39 smithi064 ceph-mon[212750]: failed to encode map e102 with expected crc 2023-10-09T05:50:39.864 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:39 smithi064 ceph-mon[220044]: failed to encode map e101 with expected crc 2023-10-09T05:50:39.864 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:39 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4129028284' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T05:50:39.864 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:39 smithi064 ceph-mon[220044]: pgmap v224: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:39.864 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:39 smithi064 ceph-mon[220044]: failed to encode map e102 with expected crc 2023-10-09T05:50:39.865 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:39 smithi064 ceph-mon[220044]: failed to encode map e102 with expected crc 2023-10-09T05:50:39.865 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:39 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4129028284' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T05:50:39.865 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:39 smithi064 ceph-mon[220044]: osdmap e102: 8 total, 8 up, 8 in 2023-10-09T05:50:39.865 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:39 smithi064 ceph-mon[220044]: failed to encode map e102 with expected crc 2023-10-09T05:50:39.865 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:39 smithi064 ceph-mon[220044]: failed to encode map e102 with expected crc 2023-10-09T05:50:39.866 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:39 smithi064 ceph-mon[220044]: failed to encode map e102 with expected crc 2023-10-09T05:50:39.866 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:39 smithi064 ceph-mon[220044]: failed to encode map e102 with expected crc 2023-10-09T05:50:40.015 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:50:40.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:39 smithi156 ceph-mon[195101]: failed to encode map e101 with expected crc 2023-10-09T05:50:40.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:39 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4129028284' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T05:50:40.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:39 smithi156 ceph-mon[195101]: pgmap v224: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:40.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:39 smithi156 ceph-mon[195101]: failed to encode map e102 with expected crc 2023-10-09T05:50:40.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:39 smithi156 ceph-mon[195101]: failed to encode map e102 with expected crc 2023-10-09T05:50:40.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:39 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4129028284' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T05:50:40.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:39 smithi156 ceph-mon[195101]: osdmap e102: 8 total, 8 up, 8 in 2023-10-09T05:50:40.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:39 smithi156 ceph-mon[195101]: failed to encode map e102 with expected crc 2023-10-09T05:50:40.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:39 smithi156 ceph-mon[195101]: failed to encode map e102 with expected crc 2023-10-09T05:50:40.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:39 smithi156 ceph-mon[195101]: failed to encode map e102 with expected crc 2023-10-09T05:50:40.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:39 smithi156 ceph-mon[195101]: failed to encode map e102 with expected crc 2023-10-09T05:50:40.196 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:40.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:40.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:40.300 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:50:40.330 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:40.378 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:50:40.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:40.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:50:40.436 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-10-09T05:50:40.483 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:50:40.519 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:40.627 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:50:40.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:40.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:50:40.731 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:40.760 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:40.774 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:40.825 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:40.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:40.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:40.888 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:50:40.888 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:50:40.930 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:50:40.960 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:40.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:40 smithi064 ceph-mon[212750]: failed to encode map e102 with expected crc 2023-10-09T05:50:40.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:40 smithi064 ceph-mon[212750]: failed to encode map e102 with expected crc 2023-10-09T05:50:40.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:40 smithi064 ceph-mon[212750]: failed to encode map e103 with expected crc 2023-10-09T05:50:40.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:40 smithi064 ceph-mon[212750]: osdmap e103: 8 total, 8 up, 8 in 2023-10-09T05:50:40.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:40 smithi064 ceph-mon[212750]: failed to encode map e103 with expected crc 2023-10-09T05:50:40.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:40 smithi064 ceph-mon[212750]: failed to encode map e103 with expected crc 2023-10-09T05:50:40.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:40 smithi064 ceph-mon[212750]: failed to encode map e103 with expected crc 2023-10-09T05:50:40.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:40 smithi064 ceph-mon[220044]: failed to encode map e102 with expected crc 2023-10-09T05:50:40.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:40 smithi064 ceph-mon[220044]: failed to encode map e102 with expected crc 2023-10-09T05:50:40.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:40 smithi064 ceph-mon[220044]: failed to encode map e103 with expected crc 2023-10-09T05:50:40.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:40 smithi064 ceph-mon[220044]: osdmap e103: 8 total, 8 up, 8 in 2023-10-09T05:50:40.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:40 smithi064 ceph-mon[220044]: failed to encode map e103 with expected crc 2023-10-09T05:50:40.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:40 smithi064 ceph-mon[220044]: failed to encode map e103 with expected crc 2023-10-09T05:50:40.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:40 smithi064 ceph-mon[220044]: failed to encode map e103 with expected crc 2023-10-09T05:50:41.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:40 smithi156 ceph-mon[195101]: failed to encode map e102 with expected crc 2023-10-09T05:50:41.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:40 smithi156 ceph-mon[195101]: failed to encode map e102 with expected crc 2023-10-09T05:50:41.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:40 smithi156 ceph-mon[195101]: failed to encode map e103 with expected crc 2023-10-09T05:50:41.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:40 smithi156 ceph-mon[195101]: osdmap e103: 8 total, 8 up, 8 in 2023-10-09T05:50:41.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:40 smithi156 ceph-mon[195101]: failed to encode map e103 with expected crc 2023-10-09T05:50:41.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:40 smithi156 ceph-mon[195101]: failed to encode map e103 with expected crc 2023-10-09T05:50:41.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:40 smithi156 ceph-mon[195101]: failed to encode map e103 with expected crc 2023-10-09T05:50:41.224 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:50:41.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:41.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:50:41.328 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:41.358 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:41.400 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:41.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:41.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:41.505 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:50:41.531 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:41.579 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:50:41.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:41.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:50:41.609 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:50:41.686 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:50:41.717 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:41.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:41 smithi064 ceph-mon[212750]: failed to encode map e103 with expected crc 2023-10-09T05:50:41.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:41 smithi064 ceph-mon[212750]: pgmap v227: 89 pgs: 11 creating+peering, 3 unknown, 75 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:41.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:41 smithi064 ceph-mon[212750]: osdmap e104: 8 total, 8 up, 8 in 2023-10-09T05:50:41.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:41 smithi064 ceph-mon[212750]: failed to encode map e104 with expected crc 2023-10-09T05:50:41.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:41 smithi064 ceph-mon[212750]: failed to encode map e104 with expected crc 2023-10-09T05:50:41.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:41 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:50:41.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:41 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:50:41.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:41 smithi064 ceph-mon[220044]: failed to encode map e103 with expected crc 2023-10-09T05:50:41.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:41 smithi064 ceph-mon[220044]: pgmap v227: 89 pgs: 11 creating+peering, 3 unknown, 75 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:41.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:41 smithi064 ceph-mon[220044]: osdmap e104: 8 total, 8 up, 8 in 2023-10-09T05:50:41.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:41 smithi064 ceph-mon[220044]: failed to encode map e104 with expected crc 2023-10-09T05:50:41.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:41 smithi064 ceph-mon[220044]: failed to encode map e104 with expected crc 2023-10-09T05:50:41.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:41 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:50:41.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:41 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:50:42.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:41 smithi156 ceph-mon[195101]: failed to encode map e103 with expected crc 2023-10-09T05:50:42.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:41 smithi156 ceph-mon[195101]: pgmap v227: 89 pgs: 11 creating+peering, 3 unknown, 75 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:42.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:41 smithi156 ceph-mon[195101]: osdmap e104: 8 total, 8 up, 8 in 2023-10-09T05:50:42.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:41 smithi156 ceph-mon[195101]: failed to encode map e104 with expected crc 2023-10-09T05:50:42.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:41 smithi156 ceph-mon[195101]: failed to encode map e104 with expected crc 2023-10-09T05:50:42.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:41 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:50:42.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:41 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:50:42.676 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:50:42.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:42.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:50:42.689 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:42.755 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:50:42.782 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:42.826 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:42.877 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:42.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:42.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:42.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:42 smithi064 ceph-mon[212750]: failed to encode map e104 with expected crc 2023-10-09T05:50:42.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:42 smithi064 ceph-mon[212750]: failed to encode map e104 with expected crc 2023-10-09T05:50:42.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:42 smithi064 ceph-mon[212750]: failed to encode map e104 with expected crc 2023-10-09T05:50:42.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:42 smithi064 ceph-mon[220044]: failed to encode map e104 with expected crc 2023-10-09T05:50:42.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:42 smithi064 ceph-mon[220044]: failed to encode map e104 with expected crc 2023-10-09T05:50:42.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:42 smithi064 ceph-mon[220044]: failed to encode map e104 with expected crc 2023-10-09T05:50:42.981 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:50:42.999 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:43.011 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:43.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:42 smithi156 ceph-mon[195101]: failed to encode map e104 with expected crc 2023-10-09T05:50:43.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:42 smithi156 ceph-mon[195101]: failed to encode map e104 with expected crc 2023-10-09T05:50:43.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:42 smithi156 ceph-mon[195101]: failed to encode map e104 with expected crc 2023-10-09T05:50:43.619 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:50:43.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:43.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:50:43.723 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:50:43.756 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:43.805 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:50:43.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:43.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:50:43.912 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:43.942 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:44.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:44 smithi064 ceph-mon[212750]: pgmap v229: 89 pgs: 11 creating+peering, 78 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:44.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:50:44.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:44.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:44 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/46217148' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:50:44.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:44 smithi064 ceph-mon[220044]: pgmap v229: 89 pgs: 11 creating+peering, 78 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:44.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:50:44.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:44.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:44 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/46217148' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:50:44.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:44 smithi156 ceph-mon[195101]: pgmap v229: 89 pgs: 11 creating+peering, 78 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:44.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:50:44.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:44.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:44 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/46217148' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:50:44.426 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T05:50:44.988 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:44.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:44.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:45.092 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:45.123 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:45.172 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:45.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:45.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:45.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:45 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:50:45.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:45 smithi064 ceph-mon[220044]: failed to encode map e105 with expected crc 2023-10-09T05:50:45.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:45 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/46217148' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:50:45.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:45 smithi064 ceph-mon[220044]: osdmap e105: 8 total, 8 up, 8 in 2023-10-09T05:50:45.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:45 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:50:45.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:45 smithi064 ceph-mon[212750]: failed to encode map e105 with expected crc 2023-10-09T05:50:45.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:45 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/46217148' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:50:45.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:45 smithi064 ceph-mon[212750]: osdmap e105: 8 total, 8 up, 8 in 2023-10-09T05:50:45.281 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:45.313 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:45.363 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:45.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:45.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:45.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:45 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:50:45.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:45 smithi156 ceph-mon[195101]: failed to encode map e105 with expected crc 2023-10-09T05:50:45.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:45 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/46217148' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:50:45.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:45 smithi156 ceph-mon[195101]: osdmap e105: 8 total, 8 up, 8 in 2023-10-09T05:50:45.467 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:45.496 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:45.541 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:45.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:45.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:45.572 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:50:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:50:45] "GET /metrics HTTP/1.1" 200 36264 "" "Prometheus/2.43.0" 2023-10-09T05:50:45.649 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:45.683 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:45.718 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:45.737 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:45.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:45.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:45.849 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:45.879 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:46.062 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:50:46.080 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:50:46.100 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:50:46.108 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:46 smithi064 ceph-mon[220044]: pgmap v231: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:46.108 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:46 smithi064 ceph-mon[220044]: failed to encode map e105 with expected crc 2023-10-09T05:50:46.108 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:46 smithi064 ceph-mon[220044]: failed to encode map e105 with expected crc 2023-10-09T05:50:46.108 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:46 smithi064 ceph-mon[220044]: failed to encode map e105 with expected crc 2023-10-09T05:50:46.108 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:46 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3021973309' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:50:46.109 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:46 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:50:46.109 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:46 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3808783655' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:46.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:46 smithi064 ceph-mon[212750]: pgmap v231: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:46.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:46 smithi064 ceph-mon[212750]: failed to encode map e105 with expected crc 2023-10-09T05:50:46.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:46 smithi064 ceph-mon[212750]: failed to encode map e105 with expected crc 2023-10-09T05:50:46.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:46 smithi064 ceph-mon[212750]: failed to encode map e105 with expected crc 2023-10-09T05:50:46.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:46 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3021973309' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:50:46.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:46 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:50:46.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:46 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3808783655' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:46.121 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:50:46.126 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:46.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:46.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:46.146 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:50:46.169 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:50:46.191 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:50:46.211 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:50:46.213 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:50:46.213 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:50:46.213 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:50:46.213 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:50:46.213 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:50:46.213 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:50:46.213 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:50:46.213 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:50:46.231 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:46.262 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:46.310 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:46.314 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:46 smithi156 ceph-mon[195101]: pgmap v231: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:46.314 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:46 smithi156 ceph-mon[195101]: failed to encode map e105 with expected crc 2023-10-09T05:50:46.314 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:46 smithi156 ceph-mon[195101]: failed to encode map e105 with expected crc 2023-10-09T05:50:46.314 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:46 smithi156 ceph-mon[195101]: failed to encode map e105 with expected crc 2023-10-09T05:50:46.314 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:46 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3021973309' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:50:46.314 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:46 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:50:46.314 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:46 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3808783655' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:46.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:46.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:46.416 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:46.518 INFO:teuthology.orchestra.run.smithi064.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-10-09T05:50:46.547 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:46.593 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:46.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:46.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:46.700 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:46.730 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:47 smithi156 ceph-mon[195101]: failed to encode map e105 with expected crc 2023-10-09T05:50:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:47 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:50:47.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:47 smithi156 ceph-mon[195101]: osdmap e106: 8 total, 8 up, 8 in 2023-10-09T05:50:47.447 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:47 smithi064 ceph-mon[212750]: failed to encode map e105 with expected crc 2023-10-09T05:50:47.447 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:47 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:50:47.447 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:47 smithi064 ceph-mon[212750]: osdmap e106: 8 total, 8 up, 8 in 2023-10-09T05:50:47.447 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:47 smithi064 ceph-mon[220044]: failed to encode map e105 with expected crc 2023-10-09T05:50:47.448 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:47 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:50:47.448 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:47 smithi064 ceph-mon[220044]: osdmap e106: 8 total, 8 up, 8 in 2023-10-09T05:50:47.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:50:48.269 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:48.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:48.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:48.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:48 smithi156 ceph-mon[195101]: pgmap v233: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:48.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:48 smithi156 ceph-mon[195101]: failed to encode map e106 with expected crc 2023-10-09T05:50:48.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:48 smithi156 ceph-mon[195101]: failed to encode map e106 with expected crc 2023-10-09T05:50:48.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:48 smithi156 ceph-mon[195101]: failed to encode map e106 with expected crc 2023-10-09T05:50:48.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:48 smithi156 ceph-mon[195101]: failed to encode map e106 with expected crc 2023-10-09T05:50:48.372 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:50:48.403 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:48.437 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:48.453 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:50:48.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:48.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:50:48.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:48 smithi064 ceph-mon[212750]: pgmap v233: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:48.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:48 smithi064 ceph-mon[212750]: failed to encode map e106 with expected crc 2023-10-09T05:50:48.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:48 smithi064 ceph-mon[212750]: failed to encode map e106 with expected crc 2023-10-09T05:50:48.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:48 smithi064 ceph-mon[212750]: failed to encode map e106 with expected crc 2023-10-09T05:50:48.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:48 smithi064 ceph-mon[212750]: failed to encode map e106 with expected crc 2023-10-09T05:50:48.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:48 smithi064 ceph-mon[220044]: pgmap v233: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:48.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:48 smithi064 ceph-mon[220044]: failed to encode map e106 with expected crc 2023-10-09T05:50:48.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:48 smithi064 ceph-mon[220044]: failed to encode map e106 with expected crc 2023-10-09T05:50:48.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:48 smithi064 ceph-mon[220044]: failed to encode map e106 with expected crc 2023-10-09T05:50:48.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:48 smithi064 ceph-mon[220044]: failed to encode map e106 with expected crc 2023-10-09T05:50:48.560 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:48.591 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:48.907 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:50:48.907 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T05:50:48.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T05:50:48.907 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [] 2023-10-09T05:50:48.907 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:50:48.960 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:48.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:48.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:49.063 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:50:49.094 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:49.252 DEBUG:teuthology.orchestra.run.smithi064:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-10-09T05:50:49.253 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T05:50:49.253 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:50:49.257 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T05:50:49.257 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T05:50:49.342 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:50:49.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:49.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:50:49.347 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:hints = 1 2023-10-09T05:50:49.347 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T05:50:49.347 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object prefix: benchmark_data_smithi064_284193 2023-10-09T05:50:49.347 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:50:49.347 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 0 0 0 0 0 0 - 0 2023-10-09T05:50:49.447 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:49.464 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:50:49.509 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:50.347 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 1 16 1979 1963 122.673 122.688 0.00356284 0.00813176 2023-10-09T05:50:50.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:50 smithi156 ceph-mon[195101]: pgmap v234: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:50.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:50 smithi064 ceph-mon[220044]: pgmap v234: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:50.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:50 smithi064 ceph-mon[212750]: pgmap v234: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T05:50:50.896 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:50:51.130 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:51.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:51.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:51.228 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:51.252 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:50:51.286 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:51.329 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:51.341 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:50:51.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:51.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:50:51.348 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 2 16 4495 4479 139.854 157.25 0.00482085 0.00710771 2023-10-09T05:50:51.446 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:50:51.482 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:51.540 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:50:51.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:51.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:50:51.646 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:51.718 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:52.162 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:52.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:52.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:52.272 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:50:52.307 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:52.348 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 3 16 6846 6830 142.199 146.938 0.00315472 0.00702072 2023-10-09T05:50:52.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:52 smithi156 ceph-mon[195101]: pgmap v235: 89 pgs: 89 active+clean; 18 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s wr, 35 op/s 2023-10-09T05:50:52.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:52 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:50:52.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:52 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:50:52.381 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:50:52.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:52.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:50:52.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:52 smithi064 ceph-mon[220044]: pgmap v235: 89 pgs: 89 active+clean; 18 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s wr, 35 op/s 2023-10-09T05:50:52.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:52 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:50:52.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:52 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:50:52.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:52 smithi064 ceph-mon[212750]: pgmap v235: 89 pgs: 89 active+clean; 18 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s wr, 35 op/s 2023-10-09T05:50:52.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:52 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:50:52.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:52 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:50:52.490 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:52.499 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:50:52.561 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:53.321 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T05:50:53.348 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:53 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:50:53.348 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:53 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:50:53.348 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:53 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:50:53.348 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:53 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4264909399' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:50:53.348 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:53 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:50:53.349 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 4 16 10039 10023 156.53 199.562 0.00365477 0.00637731 2023-10-09T05:50:53.349 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:53 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:50:53.349 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:53 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:50:53.349 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:53 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:50:53.349 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:53 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4264909399' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:50:53.349 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:53 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:50:53.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:53 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:50:53.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:53 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:50:53.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:53 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:50:53.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:53 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4264909399' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:50:53.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:53 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:50:54.349 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 5 16 11845 11829 147.8 112.875 0.00345115 0.00676069 2023-10-09T05:50:54.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:54 smithi156 ceph-mon[195101]: pgmap v236: 89 pgs: 89 active+clean; 74 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s wr, 146 op/s 2023-10-09T05:50:54.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:54 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:50:54.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:54 smithi156 ceph-mon[195101]: failed to encode map e107 with expected crc 2023-10-09T05:50:54.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:54 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:50:54.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:54 smithi156 ceph-mon[195101]: osdmap e107: 8 total, 8 up, 8 in 2023-10-09T05:50:54.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:54 smithi156 ceph-mon[195101]: failed to encode map e107 with expected crc 2023-10-09T05:50:54.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:54 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:50:54.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:54 smithi064 ceph-mon[212750]: pgmap v236: 89 pgs: 89 active+clean; 74 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s wr, 146 op/s 2023-10-09T05:50:54.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:54 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:50:54.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:54 smithi064 ceph-mon[212750]: failed to encode map e107 with expected crc 2023-10-09T05:50:54.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:54 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:50:54.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:54 smithi064 ceph-mon[212750]: osdmap e107: 8 total, 8 up, 8 in 2023-10-09T05:50:54.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:54 smithi064 ceph-mon[212750]: failed to encode map e107 with expected crc 2023-10-09T05:50:54.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:54 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:50:54.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:54 smithi064 ceph-mon[220044]: pgmap v236: 89 pgs: 89 active+clean; 74 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s wr, 146 op/s 2023-10-09T05:50:54.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:54 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:50:54.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:54 smithi064 ceph-mon[220044]: failed to encode map e107 with expected crc 2023-10-09T05:50:54.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:54 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:50:54.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:54 smithi064 ceph-mon[220044]: osdmap e107: 8 total, 8 up, 8 in 2023-10-09T05:50:54.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:54 smithi064 ceph-mon[220044]: failed to encode map e107 with expected crc 2023-10-09T05:50:54.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:54 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:50:54.982 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:54.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:54.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:55.089 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:50:55.128 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:55.204 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:50:55.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:55.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:50:55.310 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:50:55.349 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 6 16 14756 14740 153.471 181.938 0.00561414 0.0065115 2023-10-09T05:50:55.377 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:55.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:50:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:50:55] "GET /metrics HTTP/1.1" 200 36264 "" "Prometheus/2.43.0" 2023-10-09T05:50:55.987 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:50:55.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:55.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:50:56.097 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:50:56.188 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:56.279 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:56 smithi064 ceph-mon[212750]: pgmap v238: 89 pgs: 89 active+clean; 587 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T05:50:56.279 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:56 smithi064 ceph-mon[220044]: pgmap v238: 89 pgs: 89 active+clean; 587 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T05:50:56.349 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 7 16 17195 17179 153.322 152.438 0.00524551 0.00651505 2023-10-09T05:50:56.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:56 smithi156 ceph-mon[195101]: pgmap v238: 89 pgs: 89 active+clean; 587 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T05:50:57.006 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:50:57.006 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:50:57.006 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:50:57.006 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:50:57.007 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:50:57.031 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:50:57.067 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:50:57.109 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:50:57.112 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:50:57.112 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:50:57.113 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:50:57.113 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:50:57.113 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:50:57.113 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:50:57.113 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:50:57.113 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:50:57.349 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 8 16 19487 19471 152.061 143.25 0.00432131 0.00656868 2023-10-09T05:50:57.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:57 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/327055848' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:57.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:57 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/327055848' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:57.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:57 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/327055848' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:50:57.937 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:50:57.953 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T05:50:58.226 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:50:58.262 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:58 smithi064 ceph-mon[212750]: pgmap v239: 89 pgs: 89 active+clean; 785 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T05:50:58.262 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:58 smithi064 ceph-mon[220044]: pgmap v239: 89 pgs: 89 active+clean; 785 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T05:50:58.350 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 9 16 21488 21472 149.051 125.062 0.00818209 0.00670339 2023-10-09T05:50:58.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:58 smithi156 ceph-mon[195101]: pgmap v239: 89 pgs: 89 active+clean; 785 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T05:50:58.534 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:50:58.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:58.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:50:58.641 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:50:58.669 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:50:59.352 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 10 16 23274 23258 145.288 111.625 0.0122612 0.00687506 2023-10-09T05:50:59.359 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:50:59.369 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:59 smithi064 ceph-mon[212750]: from='client.44603 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:50:59.369 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:50:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:59.370 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:59 smithi064 ceph-mon[220044]: from='client.44603 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:50:59.370 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:50:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:59.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:50:59.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:50:59.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:59 smithi156 ceph-mon[195101]: from='client.44603 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:50:59.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:50:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:50:59.471 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:50:59.540 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:00.212 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:00.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:00.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:00.316 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":239,"stamp":"2023-10-09T05:50:56.446241+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":823007136,"num_objects":12573,"num_object_clones":0,"num_object_copies":37719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12573,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":12849,"num_write_kb":805581,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":12765,"ondisk_log_size":12765,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3408452,"kb_used_data":2035524,"kb_used_omap":126,"kb_used_meta":1372737,"kb_avail":746552764,"statfs":{"total":767960285184,"available":764470030336,"internally_reserved":0,"allocated":2084376576,"data_stored":2080395930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1405683323},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[8,1,5,0,0,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":822411264,"num_objects":12549,"num_object_clones":0,"num_object_copies":37647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12549,"num_write_kb":803136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006051"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":85,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.853392+0000","last_change":"2023-10-09T05:48:49.142429+0000","last_active":"2023-10-09T05:50:46.853392+0000","last_peered":"2023-10-09T05:50:46.853392+0000","last_clean":"2023-10-09T05:50:46.853392+0000","last_became_active":"2023-10-09T05:48:49.141966+0000","last_became_peered":"2023-10-09T05:48:49.141966+0000","last_unstale":"2023-10-09T05:50:46.853392+0000","last_undegraded":"2023-10-09T05:50:46.853392+0000","last_fullsized":"2023-10-09T05:50:46.853392+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:52.195431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":70,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.426692+0000","last_change":"2023-10-09T05:49:21.427978+0000","last_active":"2023-10-09T05:50:46.426692+0000","last_peered":"2023-10-09T05:50:46.426692+0000","last_clean":"2023-10-09T05:50:46.426692+0000","last_became_active":"2023-10-09T05:49:21.427703+0000","last_became_peered":"2023-10-09T05:49:21.427703+0000","last_unstale":"2023-10-09T05:50:46.426692+0000","last_undegraded":"2023-10-09T05:50:46.426692+0000","last_fullsized":"2023-10-09T05:50:46.426692+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:53:04.116683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":55,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.426958+0000","last_change":"2023-10-09T05:49:21.431711+0000","last_active":"2023-10-09T05:50:46.426958+0000","last_peered":"2023-10-09T05:50:46.426958+0000","last_clean":"2023-10-09T05:50:46.426958+0000","last_became_active":"2023-10-09T05:49:21.431441+0000","last_became_peered":"2023-10-09T05:49:21.431441+0000","last_unstale":"2023-10-09T05:50:46.426958+0000","last_undegraded":"2023-10-09T05:50:46.426958+0000","last_fullsized":"2023-10-09T05:50:46.426958+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:26.617516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":60,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.849202+0000","last_change":"2023-10-09T05:49:21.427476+0000","last_active":"2023-10-09T05:50:46.849202+0000","last_peered":"2023-10-09T05:50:46.849202+0000","last_clean":"2023-10-09T05:50:46.849202+0000","last_became_active":"2023-10-09T05:49:21.426417+0000","last_became_peered":"2023-10-09T05:49:21.426417+0000","last_unstale":"2023-10-09T05:50:46.849202+0000","last_undegraded":"2023-10-09T05:50:46.849202+0000","last_fullsized":"2023-10-09T05:50:46.849202+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:45:03.848962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":88,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.426068+0000","last_change":"2023-10-09T05:49:21.427745+0000","last_active":"2023-10-09T05:50:46.426068+0000","last_peered":"2023-10-09T05:50:46.426068+0000","last_clean":"2023-10-09T05:50:46.426068+0000","last_became_active":"2023-10-09T05:49:21.427292+0000","last_became_peered":"2023-10-09T05:49:21.427292+0000","last_unstale":"2023-10-09T05:50:46.426068+0000","last_undegraded":"2023-10-09T05:50:46.426068+0000","last_fullsized":"2023-10-09T05:50:46.426068+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:04:24.929346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":66,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.852382+0000","last_change":"2023-10-09T05:48:17.179524+0000","last_active":"2023-10-09T05:50:46.852382+0000","last_peered":"2023-10-09T05:50:46.852382+0000","last_clean":"2023-10-09T05:50:46.852382+0000","last_became_active":"2023-10-09T05:48:17.179233+0000","last_became_peered":"2023-10-09T05:48:17.179233+0000","last_unstale":"2023-10-09T05:50:46.852382+0000","last_undegraded":"2023-10-09T05:50:46.852382+0000","last_fullsized":"2023-10-09T05:50:46.852382+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:24.135263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":60,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.849287+0000","last_change":"2023-10-09T05:49:21.426149+0000","last_active":"2023-10-09T05:50:46.849287+0000","last_peered":"2023-10-09T05:50:46.849287+0000","last_clean":"2023-10-09T05:50:46.849287+0000","last_became_active":"2023-10-09T05:49:21.425805+0000","last_became_peered":"2023-10-09T05:49:21.425805+0000","last_unstale":"2023-10-09T05:50:46.849287+0000","last_undegraded":"2023-10-09T05:50:46.849287+0000","last_fullsized":"2023-10-09T05:50:46.849287+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:39:57.117762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":70,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.425983+0000","last_change":"2023-10-09T05:49:21.427804+0000","last_active":"2023-10-09T05:50:46.425983+0000","last_peered":"2023-10-09T05:50:46.425983+0000","last_clean":"2023-10-09T05:50:46.425983+0000","last_became_active":"2023-10-09T05:49:21.427569+0000","last_became_peered":"2023-10-09T05:49:21.427569+0000","last_unstale":"2023-10-09T05:50:46.425983+0000","last_undegraded":"2023-10-09T05:50:46.425983+0000","last_fullsized":"2023-10-09T05:50:46.425983+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:48.791826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":67,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.852869+0000","last_change":"2023-10-09T05:48:17.176541+0000","last_active":"2023-10-09T05:50:46.852869+0000","last_peered":"2023-10-09T05:50:46.852869+0000","last_clean":"2023-10-09T05:50:46.852869+0000","last_became_active":"2023-10-09T05:48:17.176213+0000","last_became_peered":"2023-10-09T05:48:17.176213+0000","last_unstale":"2023-10-09T05:50:46.852869+0000","last_undegraded":"2023-10-09T05:50:46.852869+0000","last_fullsized":"2023-10-09T05:50:46.852869+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:23.599435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":29,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.426499+0000","last_change":"2023-10-09T05:49:21.425819+0000","last_active":"2023-10-09T05:50:46.426499+0000","last_peered":"2023-10-09T05:50:46.426499+0000","last_clean":"2023-10-09T05:50:46.426499+0000","last_became_active":"2023-10-09T05:49:21.425498+0000","last_became_peered":"2023-10-09T05:49:21.425498+0000","last_unstale":"2023-10-09T05:50:46.426499+0000","last_undegraded":"2023-10-09T05:50:46.426499+0000","last_fullsized":"2023-10-09T05:50:46.426499+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:21.443055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":70,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.853434+0000","last_change":"2023-10-09T05:49:21.430469+0000","last_active":"2023-10-09T05:50:46.853434+0000","last_peered":"2023-10-09T05:50:46.853434+0000","last_clean":"2023-10-09T05:50:46.853434+0000","last_became_active":"2023-10-09T05:49:21.430107+0000","last_became_peered":"2023-10-09T05:49:21.430107+0000","last_unstale":"2023-10-09T05:50:46.853434+0000","last_undegraded":"2023-10-09T05:50:46.853434+0000","last_fullsized":"2023-10-09T05:50:46.853434+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:08.815343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":55,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.427295+0000","last_change":"2023-10-09T05:48:49.138162+0000","last_active":"2023-10-09T05:50:46.427295+0000","last_peered":"2023-10-09T05:50:46.427295+0000","last_clean":"2023-10-09T05:50:46.427295+0000","last_became_active":"2023-10-09T05:48:49.137824+0000","last_became_peered":"2023-10-09T05:48:49.137824+0000","last_unstale":"2023-10-09T05:50:46.427295+0000","last_undegraded":"2023-10-09T05:50:46.427295+0000","last_fullsized":"2023-10-09T05:50:46.427295+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:43:26.642768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":72,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.852817+0000","last_change":"2023-10-09T05:40:12.630786+0000","last_active":"2023-10-09T05:50:46.852817+0000","last_peered":"2023-10-09T05:50:46.852817+0000","last_clean":"2023-10-09T05:50:46.852817+0000","last_became_active":"2023-10-09T05:40:12.630528+0000","last_became_peered":"2023-10-09T05:40:12.630528+0000","last_unstale":"2023-10-09T05:50:46.852817+0000","last_undegraded":"2023-10-09T05:50:46.852817+0000","last_fullsized":"2023-10-09T05:50:46.852817+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:54.427346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":66,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.853032+0000","last_change":"2023-10-09T05:48:17.173137+0000","last_active":"2023-10-09T05:50:46.853032+0000","last_peered":"2023-10-09T05:50:46.853032+0000","last_clean":"2023-10-09T05:50:46.853032+0000","last_became_active":"2023-10-09T05:48:17.172891+0000","last_became_peered":"2023-10-09T05:48:17.172891+0000","last_unstale":"2023-10-09T05:50:46.853032+0000","last_undegraded":"2023-10-09T05:50:46.853032+0000","last_fullsized":"2023-10-09T05:50:46.853032+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:21.393939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":69,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.853473+0000","last_change":"2023-10-09T05:48:49.139681+0000","last_active":"2023-10-09T05:50:46.853473+0000","last_peered":"2023-10-09T05:50:46.853473+0000","last_clean":"2023-10-09T05:50:46.853473+0000","last_became_active":"2023-10-09T05:48:49.139317+0000","last_became_peered":"2023-10-09T05:48:49.139317+0000","last_unstale":"2023-10-09T05:50:46.853473+0000","last_undegraded":"2023-10-09T05:50:46.853473+0000","last_fullsized":"2023-10-09T05:50:46.853473+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:34.769043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":70,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.426753+0000","last_change":"2023-10-09T05:49:21.428386+0000","last_active":"2023-10-09T05:50:46.426753+0000","last_peered":"2023-10-09T05:50:46.426753+0000","last_clean":"2023-10-09T05:50:46.426753+0000","last_became_active":"2023-10-09T05:49:21.428131+0000","last_became_peered":"2023-10-09T05:49:21.428131+0000","last_unstale":"2023-10-09T05:50:46.426753+0000","last_undegraded":"2023-10-09T05:50:46.426753+0000","last_fullsized":"2023-10-09T05:50:46.426753+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:12.858458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":60,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.849090+0000","last_change":"2023-10-09T05:49:21.426167+0000","last_active":"2023-10-09T05:50:46.849090+0000","last_peered":"2023-10-09T05:50:46.849090+0000","last_clean":"2023-10-09T05:50:46.849090+0000","last_became_active":"2023-10-09T05:49:21.425874+0000","last_became_peered":"2023-10-09T05:49:21.425874+0000","last_unstale":"2023-10-09T05:50:46.849090+0000","last_undegraded":"2023-10-09T05:50:46.849090+0000","last_fullsized":"2023-10-09T05:50:46.849090+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:59.642587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":66,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.852521+0000","last_change":"2023-10-09T05:49:21.423218+0000","last_active":"2023-10-09T05:50:46.852521+0000","last_peered":"2023-10-09T05:50:46.852521+0000","last_clean":"2023-10-09T05:50:46.852521+0000","last_became_active":"2023-10-09T05:49:21.422706+0000","last_became_peered":"2023-10-09T05:49:21.422706+0000","last_unstale":"2023-10-09T05:50:46.852521+0000","last_undegraded":"2023-10-09T05:50:46.852521+0000","last_fullsized":"2023-10-09T05:50:46.852521+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:31.571220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":50,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.853575+0000","last_change":"2023-10-09T05:40:12.631893+0000","last_active":"2023-10-09T05:50:46.853575+0000","last_peered":"2023-10-09T05:50:46.853575+0000","last_clean":"2023-10-09T05:50:46.853575+0000","last_became_active":"2023-10-09T05:40:12.631592+0000","last_became_peered":"2023-10-09T05:40:12.631592+0000","last_unstale":"2023-10-09T05:50:46.853575+0000","last_undegraded":"2023-10-09T05:50:46.853575+0000","last_fullsized":"2023-10-09T05:50:46.853575+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:18:12.517403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":60,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.849112+0000","last_change":"2023-10-09T05:49:21.426431+0000","last_active":"2023-10-09T05:50:46.849112+0000","last_peered":"2023-10-09T05:50:46.849112+0000","last_clean":"2023-10-09T05:50:46.849112+0000","last_became_active":"2023-10-09T05:49:21.426011+0000","last_became_peered":"2023-10-09T05:49:21.426011+0000","last_unstale":"2023-10-09T05:50:46.849112+0000","last_undegraded":"2023-10-09T05:50:46.849112+0000","last_fullsized":"2023-10-09T05:50:46.849112+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:27.350168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":75,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.851758+0000","last_change":"2023-10-09T05:47:03.230438+0000","last_active":"2023-10-09T05:50:46.851758+0000","last_peered":"2023-10-09T05:50:46.851758+0000","last_clean":"2023-10-09T05:50:46.851758+0000","last_became_active":"2023-10-09T05:47:03.229346+0000","last_became_peered":"2023-10-09T05:47:03.229346+0000","last_unstale":"2023-10-09T05:50:46.851758+0000","last_undegraded":"2023-10-09T05:50:46.851758+0000","last_fullsized":"2023-10-09T05:50:46.851758+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:54.948861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":70,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.427840+0000","last_change":"2023-10-09T05:49:21.427270+0000","last_active":"2023-10-09T05:50:46.427840+0000","last_peered":"2023-10-09T05:50:46.427840+0000","last_clean":"2023-10-09T05:50:46.427840+0000","last_became_active":"2023-10-09T05:49:21.427014+0000","last_became_peered":"2023-10-09T05:49:21.427014+0000","last_unstale":"2023-10-09T05:50:46.427840+0000","last_undegraded":"2023-10-09T05:50:46.427840+0000","last_fullsized":"2023-10-09T05:50:46.427840+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:36:30.822627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"107'756","reported_seq":769,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:50:54.355982+0000","last_change":"2023-10-09T05:50:40.123354+0000","last_active":"2023-10-09T05:50:54.355982+0000","last_peered":"2023-10-09T05:50:54.355982+0000","last_clean":"2023-10-09T05:50:54.355982+0000","last_became_active":"2023-10-09T05:50:40.122998+0000","last_became_peered":"2023-10-09T05:50:40.122998+0000","last_unstale":"2023-10-09T05:50:54.355982+0000","last_undegraded":"2023-10-09T05:50:54.355982+0000","last_fullsized":"2023-10-09T05:50:54.355982+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":756,"log_dups_size":0,"ondisk_log_size":756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:53.689023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49545216,"num_objects":756,"num_object_clones":0,"num_object_copies":2268,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":756,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":756,"num_write_kb":48384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":68,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.851974+0000","last_change":"2023-10-09T05:47:03.230162+0000","last_active":"2023-10-09T05:50:46.851974+0000","last_peered":"2023-10-09T05:50:46.851974+0000","last_clean":"2023-10-09T05:50:46.851974+0000","last_became_active":"2023-10-09T05:47:03.229695+0000","last_became_peered":"2023-10-09T05:47:03.229695+0000","last_unstale":"2023-10-09T05:50:46.851974+0000","last_undegraded":"2023-10-09T05:50:46.851974+0000","last_fullsized":"2023-10-09T05:50:46.851974+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:13.194065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":49,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.851558+0000","last_change":"2023-10-09T05:40:14.362515+0000","last_active":"2023-10-09T05:50:46.851558+0000","last_peered":"2023-10-09T05:50:46.851558+0000","last_clean":"2023-10-09T05:50:46.851558+0000","last_became_active":"2023-10-09T05:40:14.362184+0000","last_became_peered":"2023-10-09T05:40:14.362184+0000","last_unstale":"2023-10-09T05:50:46.851558+0000","last_undegraded":"2023-10-09T05:50:46.851558+0000","last_fullsized":"2023-10-09T05:50:46.851558+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:37.681193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"107'752","reported_seq":766,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:50:54.418177+0000","last_change":"2023-10-09T05:50:39.775544+0000","last_active":"2023-10-09T05:50:54.418177+0000","last_peered":"2023-10-09T05:50:54.418177+0000","last_clean":"2023-10-09T05:50:54.418177+0000","last_became_active":"2023-10-09T05:50:39.775272+0000","last_became_peered":"2023-10-09T05:50:39.775272+0000","last_unstale":"2023-10-09T05:50:54.418177+0000","last_undegraded":"2023-10-09T05:50:54.418177+0000","last_fullsized":"2023-10-09T05:50:54.418177+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":752,"log_dups_size":0,"ondisk_log_size":752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:57.841743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49283072,"num_objects":752,"num_object_clones":0,"num_object_copies":2256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":752,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":752,"num_write_kb":48128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":69,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.853665+0000","last_change":"2023-10-09T05:48:17.173680+0000","last_active":"2023-10-09T05:50:46.853665+0000","last_peered":"2023-10-09T05:50:46.853665+0000","last_clean":"2023-10-09T05:50:46.853665+0000","last_became_active":"2023-10-09T05:48:17.173322+0000","last_became_peered":"2023-10-09T05:48:17.173322+0000","last_unstale":"2023-10-09T05:50:46.853665+0000","last_undegraded":"2023-10-09T05:50:46.853665+0000","last_fullsized":"2023-10-09T05:50:46.853665+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:41.005172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":83,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.852072+0000","last_change":"2023-10-09T05:49:21.426392+0000","last_active":"2023-10-09T05:50:46.852072+0000","last_peered":"2023-10-09T05:50:46.852072+0000","last_clean":"2023-10-09T05:50:46.852072+0000","last_became_active":"2023-10-09T05:49:21.426056+0000","last_became_peered":"2023-10-09T05:49:21.426056+0000","last_unstale":"2023-10-09T05:50:46.852072+0000","last_undegraded":"2023-10-09T05:50:46.852072+0000","last_fullsized":"2023-10-09T05:50:46.852072+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:09.862053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"107'778","reported_seq":791,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:50:54.372302+0000","last_change":"2023-10-09T05:50:39.777752+0000","last_active":"2023-10-09T05:50:54.372302+0000","last_peered":"2023-10-09T05:50:54.372302+0000","last_clean":"2023-10-09T05:50:54.372302+0000","last_became_active":"2023-10-09T05:50:39.777526+0000","last_became_peered":"2023-10-09T05:50:39.777526+0000","last_unstale":"2023-10-09T05:50:54.372302+0000","last_undegraded":"2023-10-09T05:50:54.372302+0000","last_fullsized":"2023-10-09T05:50:54.372302+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":778,"log_dups_size":0,"ondisk_log_size":778,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:19.849370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50987008,"num_objects":778,"num_object_clones":0,"num_object_copies":2334,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":778,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":778,"num_write_kb":49792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":51,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.853546+0000","last_change":"2023-10-09T05:40:12.632083+0000","last_active":"2023-10-09T05:50:46.853546+0000","last_peered":"2023-10-09T05:50:46.853546+0000","last_clean":"2023-10-09T05:50:46.853546+0000","last_became_active":"2023-10-09T05:40:12.631715+0000","last_became_peered":"2023-10-09T05:40:12.631715+0000","last_unstale":"2023-10-09T05:50:46.853546+0000","last_undegraded":"2023-10-09T05:50:46.853546+0000","last_fullsized":"2023-10-09T05:50:46.853546+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:17.641378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":76,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.852647+0000","last_change":"2023-10-09T05:48:49.139877+0000","last_active":"2023-10-09T05:50:46.852647+0000","last_peered":"2023-10-09T05:50:46.852647+0000","last_clean":"2023-10-09T05:50:46.852647+0000","last_became_active":"2023-10-09T05:48:49.139479+0000","last_became_peered":"2023-10-09T05:48:49.139479+0000","last_unstale":"2023-10-09T05:50:46.852647+0000","last_undegraded":"2023-10-09T05:50:46.852647+0000","last_fullsized":"2023-10-09T05:50:46.852647+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:14:16.097335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"107'1018","reported_seq":1032,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:50:56.230926+0000","last_change":"2023-10-09T05:50:39.776363+0000","last_active":"2023-10-09T05:50:56.230926+0000","last_peered":"2023-10-09T05:50:56.230926+0000","last_clean":"2023-10-09T05:50:56.230926+0000","last_became_active":"2023-10-09T05:50:39.776072+0000","last_became_peered":"2023-10-09T05:50:39.776072+0000","last_unstale":"2023-10-09T05:50:56.230926+0000","last_undegraded":"2023-10-09T05:50:56.230926+0000","last_fullsized":"2023-10-09T05:50:56.230926+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":1018,"log_dups_size":0,"ondisk_log_size":1018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:13.359764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66715648,"num_objects":1018,"num_object_clones":0,"num_object_copies":3054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1018,"num_write_kb":65152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":67,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.852172+0000","last_change":"2023-10-09T05:49:21.429234+0000","last_active":"2023-10-09T05:50:46.852172+0000","last_peered":"2023-10-09T05:50:46.852172+0000","last_clean":"2023-10-09T05:50:46.852172+0000","last_became_active":"2023-10-09T05:49:21.428961+0000","last_became_peered":"2023-10-09T05:49:21.428961+0000","last_unstale":"2023-10-09T05:50:46.852172+0000","last_undegraded":"2023-10-09T05:50:46.852172+0000","last_fullsized":"2023-10-09T05:50:46.852172+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:18.029914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":60,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.849285+0000","last_change":"2023-10-09T05:48:17.172849+0000","last_active":"2023-10-09T05:50:46.849285+0000","last_peered":"2023-10-09T05:50:46.849285+0000","last_clean":"2023-10-09T05:50:46.849285+0000","last_became_active":"2023-10-09T05:48:17.172536+0000","last_became_peered":"2023-10-09T05:48:17.172536+0000","last_unstale":"2023-10-09T05:50:46.849285+0000","last_undegraded":"2023-10-09T05:50:46.849285+0000","last_fullsized":"2023-10-09T05:50:46.849285+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:52.891228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"107'739","reported_seq":752,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:50:54.352697+0000","last_change":"2023-10-09T05:50:39.779893+0000","last_active":"2023-10-09T05:50:54.352697+0000","last_peered":"2023-10-09T05:50:54.352697+0000","last_clean":"2023-10-09T05:50:54.352697+0000","last_became_active":"2023-10-09T05:50:39.779586+0000","last_became_peered":"2023-10-09T05:50:39.779586+0000","last_unstale":"2023-10-09T05:50:54.352697+0000","last_undegraded":"2023-10-09T05:50:54.352697+0000","last_fullsized":"2023-10-09T05:50:54.352697+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":739,"log_dups_size":0,"ondisk_log_size":739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:25.761165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48431104,"num_objects":739,"num_object_clones":0,"num_object_copies":2217,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":739,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":739,"num_write_kb":47296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":68,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.426258+0000","last_change":"2023-10-09T05:49:21.421396+0000","last_active":"2023-10-09T05:50:46.426258+0000","last_peered":"2023-10-09T05:50:46.426258+0000","last_clean":"2023-10-09T05:50:46.426258+0000","last_became_active":"2023-10-09T05:49:21.421129+0000","last_became_peered":"2023-10-09T05:49:21.421129+0000","last_unstale":"2023-10-09T05:50:46.426258+0000","last_undegraded":"2023-10-09T05:50:46.426258+0000","last_fullsized":"2023-10-09T05:50:46.426258+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:14.365625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":66,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.852699+0000","last_change":"2023-10-09T05:49:21.430742+0000","last_active":"2023-10-09T05:50:46.852699+0000","last_peered":"2023-10-09T05:50:46.852699+0000","last_clean":"2023-10-09T05:50:46.852699+0000","last_became_active":"2023-10-09T05:49:21.430233+0000","last_became_peered":"2023-10-09T05:49:21.430233+0000","last_unstale":"2023-10-09T05:50:46.852699+0000","last_undegraded":"2023-10-09T05:50:46.852699+0000","last_fullsized":"2023-10-09T05:50:46.852699+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:10.541302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"106'380","reported_seq":392,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:52.127066+0000","last_change":"2023-10-09T05:50:39.778792+0000","last_active":"2023-10-09T05:50:52.127066+0000","last_peered":"2023-10-09T05:50:52.127066+0000","last_clean":"2023-10-09T05:50:52.127066+0000","last_became_active":"2023-10-09T05:50:39.778492+0000","last_became_peered":"2023-10-09T05:50:39.778492+0000","last_unstale":"2023-10-09T05:50:52.127066+0000","last_undegraded":"2023-10-09T05:50:52.127066+0000","last_fullsized":"2023-10-09T05:50:52.127066+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":380,"log_dups_size":0,"ondisk_log_size":380,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:23:10.212354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24903680,"num_objects":380,"num_object_clones":0,"num_object_copies":1140,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":380,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":380,"num_write_kb":24320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":67,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.852785+0000","last_change":"2023-10-09T05:48:17.179214+0000","last_active":"2023-10-09T05:50:46.852785+0000","last_peered":"2023-10-09T05:50:46.852785+0000","last_clean":"2023-10-09T05:50:46.852785+0000","last_became_active":"2023-10-09T05:48:17.178918+0000","last_became_peered":"2023-10-09T05:48:17.178918+0000","last_unstale":"2023-10-09T05:50:46.852785+0000","last_undegraded":"2023-10-09T05:50:46.852785+0000","last_fullsized":"2023-10-09T05:50:46.852785+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:58:07.653506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":83,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.852610+0000","last_change":"2023-10-09T05:48:49.138043+0000","last_active":"2023-10-09T05:50:46.852610+0000","last_peered":"2023-10-09T05:50:46.852610+0000","last_clean":"2023-10-09T05:50:46.852610+0000","last_became_active":"2023-10-09T05:48:49.137722+0000","last_became_peered":"2023-10-09T05:48:49.137722+0000","last_unstale":"2023-10-09T05:50:46.852610+0000","last_undegraded":"2023-10-09T05:50:46.852610+0000","last_fullsized":"2023-10-09T05:50:46.852610+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:38.270179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"107'755","reported_seq":769,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:50:54.420019+0000","last_change":"2023-10-09T05:50:40.123200+0000","last_active":"2023-10-09T05:50:54.420019+0000","last_peered":"2023-10-09T05:50:54.420019+0000","last_clean":"2023-10-09T05:50:54.420019+0000","last_became_active":"2023-10-09T05:50:40.122891+0000","last_became_peered":"2023-10-09T05:50:40.122891+0000","last_unstale":"2023-10-09T05:50:54.420019+0000","last_undegraded":"2023-10-09T05:50:54.420019+0000","last_fullsized":"2023-10-09T05:50:54.420019+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":755,"log_dups_size":0,"ondisk_log_size":755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:09.955514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49479680,"num_objects":755,"num_object_clones":0,"num_object_copies":2265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":755,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":755,"num_write_kb":48320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":67,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.851865+0000","last_change":"2023-10-09T05:48:49.141287+0000","last_active":"2023-10-09T05:50:46.851865+0000","last_peered":"2023-10-09T05:50:46.851865+0000","last_clean":"2023-10-09T05:50:46.851865+0000","last_became_active":"2023-10-09T05:48:49.140950+0000","last_became_peered":"2023-10-09T05:48:49.140950+0000","last_unstale":"2023-10-09T05:50:46.851865+0000","last_undegraded":"2023-10-09T05:50:46.851865+0000","last_fullsized":"2023-10-09T05:50:46.851865+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:35.527434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":66,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.853098+0000","last_change":"2023-10-09T05:48:17.173636+0000","last_active":"2023-10-09T05:50:46.853098+0000","last_peered":"2023-10-09T05:50:46.853098+0000","last_clean":"2023-10-09T05:50:46.853098+0000","last_became_active":"2023-10-09T05:48:17.173329+0000","last_became_peered":"2023-10-09T05:48:17.173329+0000","last_unstale":"2023-10-09T05:50:46.853098+0000","last_undegraded":"2023-10-09T05:50:46.853098+0000","last_fullsized":"2023-10-09T05:50:46.853098+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:26:36.634285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"107'746","reported_seq":759,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:50:54.362661+0000","last_change":"2023-10-09T05:50:39.780876+0000","last_active":"2023-10-09T05:50:54.362661+0000","last_peered":"2023-10-09T05:50:54.362661+0000","last_clean":"2023-10-09T05:50:54.362661+0000","last_became_active":"2023-10-09T05:50:39.780630+0000","last_became_peered":"2023-10-09T05:50:39.780630+0000","last_unstale":"2023-10-09T05:50:54.362661+0000","last_undegraded":"2023-10-09T05:50:54.362661+0000","last_fullsized":"2023-10-09T05:50:54.362661+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":746,"log_dups_size":0,"ondisk_log_size":746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:30.687520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48889856,"num_objects":746,"num_object_clones":0,"num_object_copies":2238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":746,"num_write_kb":47744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":71,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.426141+0000","last_change":"2023-10-09T05:49:21.427586+0000","last_active":"2023-10-09T05:50:46.426141+0000","last_peered":"2023-10-09T05:50:46.426141+0000","last_clean":"2023-10-09T05:50:46.426141+0000","last_became_active":"2023-10-09T05:49:21.426908+0000","last_became_peered":"2023-10-09T05:49:21.426908+0000","last_unstale":"2023-10-09T05:50:46.426141+0000","last_undegraded":"2023-10-09T05:50:46.426141+0000","last_fullsized":"2023-10-09T05:50:46.426141+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:50.043430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":55,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.427876+0000","last_change":"2023-10-09T05:49:21.431778+0000","last_active":"2023-10-09T05:50:46.427876+0000","last_peered":"2023-10-09T05:50:46.427876+0000","last_clean":"2023-10-09T05:50:46.427876+0000","last_became_active":"2023-10-09T05:49:21.431483+0000","last_became_peered":"2023-10-09T05:49:21.431483+0000","last_unstale":"2023-10-09T05:50:46.427876+0000","last_undegraded":"2023-10-09T05:50:46.427876+0000","last_fullsized":"2023-10-09T05:50:46.427876+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:29:39.293436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":80,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.427746+0000","last_change":"2023-10-09T05:48:17.170845+0000","last_active":"2023-10-09T05:50:46.427746+0000","last_peered":"2023-10-09T05:50:46.427746+0000","last_clean":"2023-10-09T05:50:46.427746+0000","last_became_active":"2023-10-09T05:48:17.170321+0000","last_became_peered":"2023-10-09T05:48:17.170321+0000","last_unstale":"2023-10-09T05:50:46.427746+0000","last_undegraded":"2023-10-09T05:50:46.427746+0000","last_fullsized":"2023-10-09T05:50:46.427746+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:41.854157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":84,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.852607+0000","last_change":"2023-10-09T05:49:21.422980+0000","last_active":"2023-10-09T05:50:46.852607+0000","last_peered":"2023-10-09T05:50:46.852607+0000","last_clean":"2023-10-09T05:50:46.852607+0000","last_became_active":"2023-10-09T05:49:21.422349+0000","last_became_peered":"2023-10-09T05:49:21.422349+0000","last_unstale":"2023-10-09T05:50:46.852607+0000","last_undegraded":"2023-10-09T05:50:46.852607+0000","last_fullsized":"2023-10-09T05:50:46.852607+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:03.118182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":40,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.427212+0000","last_change":"2023-10-09T05:48:17.172596+0000","last_active":"2023-10-09T05:50:46.427212+0000","last_peered":"2023-10-09T05:50:46.427212+0000","last_clean":"2023-10-09T05:50:46.427212+0000","last_became_active":"2023-10-09T05:48:17.172287+0000","last_became_peered":"2023-10-09T05:48:17.172287+0000","last_unstale":"2023-10-09T05:50:46.427212+0000","last_undegraded":"2023-10-09T05:50:46.427212+0000","last_fullsized":"2023-10-09T05:50:46.427212+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:53.440906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"107'1052","reported_seq":1064,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:50:56.243316+0000","last_change":"2023-10-09T05:50:39.776406+0000","last_active":"2023-10-09T05:50:56.243316+0000","last_peered":"2023-10-09T05:50:56.243316+0000","last_clean":"2023-10-09T05:50:56.243316+0000","last_became_active":"2023-10-09T05:50:39.776142+0000","last_became_peered":"2023-10-09T05:50:39.776142+0000","last_unstale":"2023-10-09T05:50:56.243316+0000","last_undegraded":"2023-10-09T05:50:56.243316+0000","last_fullsized":"2023-10-09T05:50:56.243316+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":1052,"log_dups_size":0,"ondisk_log_size":1052,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:47.840623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68943872,"num_objects":1052,"num_object_clones":0,"num_object_copies":3156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1052,"num_write_kb":67328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"107'786","reported_seq":800,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:50:54.427650+0000","last_change":"2023-10-09T05:50:39.778177+0000","last_active":"2023-10-09T05:50:54.427650+0000","last_peered":"2023-10-09T05:50:54.427650+0000","last_clean":"2023-10-09T05:50:54.427650+0000","last_became_active":"2023-10-09T05:50:39.777888+0000","last_became_peered":"2023-10-09T05:50:39.777888+0000","last_unstale":"2023-10-09T05:50:54.427650+0000","last_undegraded":"2023-10-09T05:50:54.427650+0000","last_fullsized":"2023-10-09T05:50:54.427650+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":786,"log_dups_size":0,"ondisk_log_size":786,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:25.454960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51511296,"num_objects":786,"num_object_clones":0,"num_object_copies":2358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":786,"num_write_kb":50304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":71,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.851849+0000","last_change":"2023-10-09T05:48:17.172780+0000","last_active":"2023-10-09T05:50:46.851849+0000","last_peered":"2023-10-09T05:50:46.851849+0000","last_clean":"2023-10-09T05:50:46.851849+0000","last_became_active":"2023-10-09T05:48:17.172460+0000","last_became_peered":"2023-10-09T05:48:17.172460+0000","last_unstale":"2023-10-09T05:50:46.851849+0000","last_undegraded":"2023-10-09T05:50:46.851849+0000","last_fullsized":"2023-10-09T05:50:46.851849+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:41.981302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":67,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.853162+0000","last_change":"2023-10-09T05:47:03.237144+0000","last_active":"2023-10-09T05:50:46.853162+0000","last_peered":"2023-10-09T05:50:46.853162+0000","last_clean":"2023-10-09T05:50:46.853162+0000","last_became_active":"2023-10-09T05:47:03.236793+0000","last_became_peered":"2023-10-09T05:47:03.236793+0000","last_unstale":"2023-10-09T05:50:46.853162+0000","last_undegraded":"2023-10-09T05:50:46.853162+0000","last_fullsized":"2023-10-09T05:50:46.853162+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:00:44.273589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":80,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.852075+0000","last_change":"2023-10-09T05:49:21.429250+0000","last_active":"2023-10-09T05:50:46.852075+0000","last_peered":"2023-10-09T05:50:46.852075+0000","last_clean":"2023-10-09T05:50:46.852075+0000","last_became_active":"2023-10-09T05:49:21.428946+0000","last_became_peered":"2023-10-09T05:49:21.428946+0000","last_unstale":"2023-10-09T05:50:46.852075+0000","last_undegraded":"2023-10-09T05:50:46.852075+0000","last_fullsized":"2023-10-09T05:50:46.852075+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:32.978473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.5","version":"107'697","reported_seq":710,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:50:54.357005+0000","last_change":"2023-10-09T05:50:40.123175+0000","last_active":"2023-10-09T05:50:54.357005+0000","last_peered":"2023-10-09T05:50:54.357005+0000","last_clean":"2023-10-09T05:50:54.357005+0000","last_became_active":"2023-10-09T05:50:40.122928+0000","last_became_peered":"2023-10-09T05:50:40.122928+0000","last_unstale":"2023-10-09T05:50:54.357005+0000","last_undegraded":"2023-10-09T05:50:54.357005+0000","last_fullsized":"2023-10-09T05:50:54.357005+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":697,"log_dups_size":0,"ondisk_log_size":697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:16.111485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45678592,"num_objects":697,"num_object_clones":0,"num_object_copies":2091,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":697,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":697,"num_write_kb":44608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":89,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.426376+0000","last_change":"2023-10-09T05:49:21.429787+0000","last_active":"2023-10-09T05:50:46.426376+0000","last_peered":"2023-10-09T05:50:46.426376+0000","last_clean":"2023-10-09T05:50:46.426376+0000","last_became_active":"2023-10-09T05:49:21.429304+0000","last_became_peered":"2023-10-09T05:49:21.429304+0000","last_unstale":"2023-10-09T05:50:46.426376+0000","last_undegraded":"2023-10-09T05:50:46.426376+0000","last_fullsized":"2023-10-09T05:50:46.426376+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:28:24.515861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":55,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.426729+0000","last_change":"2023-10-09T05:48:49.137529+0000","last_active":"2023-10-09T05:50:46.426729+0000","last_peered":"2023-10-09T05:50:46.426729+0000","last_clean":"2023-10-09T05:50:46.426729+0000","last_became_active":"2023-10-09T05:48:49.137212+0000","last_became_peered":"2023-10-09T05:48:49.137212+0000","last_unstale":"2023-10-09T05:50:46.426729+0000","last_undegraded":"2023-10-09T05:50:46.426729+0000","last_fullsized":"2023-10-09T05:50:46.426729+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:29.740386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":65,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-09T05:50:45.468275+0000","last_change":"2023-10-09T05:48:49.139757+0000","last_active":"2023-10-09T05:50:45.468275+0000","last_peered":"2023-10-09T05:50:45.468275+0000","last_clean":"2023-10-09T05:50:45.468275+0000","last_became_active":"2023-10-09T05:48:49.139311+0000","last_became_peered":"2023-10-09T05:48:49.139311+0000","last_unstale":"2023-10-09T05:50:45.468275+0000","last_undegraded":"2023-10-09T05:50:45.468275+0000","last_fullsized":"2023-10-09T05:50:45.468275+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:22.445381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.7","version":"107'735","reported_seq":748,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:50:54.357912+0000","last_change":"2023-10-09T05:50:39.778086+0000","last_active":"2023-10-09T05:50:54.357912+0000","last_peered":"2023-10-09T05:50:54.357912+0000","last_clean":"2023-10-09T05:50:54.357912+0000","last_became_active":"2023-10-09T05:50:39.777777+0000","last_became_peered":"2023-10-09T05:50:39.777777+0000","last_unstale":"2023-10-09T05:50:54.357912+0000","last_undegraded":"2023-10-09T05:50:54.357912+0000","last_fullsized":"2023-10-09T05:50:54.357912+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":735,"log_dups_size":0,"ondisk_log_size":735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:02.350198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48168960,"num_objects":735,"num_object_clones":0,"num_object_copies":2205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":735,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":735,"num_write_kb":47040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":71,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.852013+0000","last_change":"2023-10-09T05:48:49.141207+0000","last_active":"2023-10-09T05:50:46.852013+0000","last_peered":"2023-10-09T05:50:46.852013+0000","last_clean":"2023-10-09T05:50:46.852013+0000","last_became_active":"2023-10-09T05:48:49.140798+0000","last_became_peered":"2023-10-09T05:48:49.140798+0000","last_unstale":"2023-10-09T05:50:46.852013+0000","last_undegraded":"2023-10-09T05:50:46.852013+0000","last_fullsized":"2023-10-09T05:50:46.852013+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:44:13.125030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":448,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.852338+0000","last_change":"2023-10-09T05:47:03.234961+0000","last_active":"2023-10-09T05:50:46.852338+0000","last_peered":"2023-10-09T05:50:46.852338+0000","last_clean":"2023-10-09T05:50:46.852338+0000","last_became_active":"2023-10-09T05:47:03.234618+0000","last_became_peered":"2023-10-09T05:47:03.234618+0000","last_unstale":"2023-10-09T05:50:46.852338+0000","last_undegraded":"2023-10-09T05:50:46.852338+0000","last_fullsized":"2023-10-09T05:50:46.852338+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:01.472111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":29,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.428034+0000","last_change":"2023-10-09T05:49:21.426544+0000","last_active":"2023-10-09T05:50:46.428034+0000","last_peered":"2023-10-09T05:50:46.428034+0000","last_clean":"2023-10-09T05:50:46.428034+0000","last_became_active":"2023-10-09T05:49:21.426253+0000","last_became_peered":"2023-10-09T05:49:21.426253+0000","last_unstale":"2023-10-09T05:50:46.428034+0000","last_undegraded":"2023-10-09T05:50:46.428034+0000","last_fullsized":"2023-10-09T05:50:46.428034+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:32:40.322649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":49,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.853064+0000","last_change":"2023-10-09T05:40:14.363125+0000","last_active":"2023-10-09T05:50:46.853064+0000","last_peered":"2023-10-09T05:50:46.853064+0000","last_clean":"2023-10-09T05:50:46.853064+0000","last_became_active":"2023-10-09T05:40:14.362723+0000","last_became_peered":"2023-10-09T05:40:14.362723+0000","last_unstale":"2023-10-09T05:50:46.853064+0000","last_undegraded":"2023-10-09T05:50:46.853064+0000","last_fullsized":"2023-10-09T05:50:46.853064+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:30.741318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":88,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.852401+0000","last_change":"2023-10-09T05:48:17.175668+0000","last_active":"2023-10-09T05:50:46.852401+0000","last_peered":"2023-10-09T05:50:46.852401+0000","last_clean":"2023-10-09T05:50:46.852401+0000","last_became_active":"2023-10-09T05:48:17.175344+0000","last_became_peered":"2023-10-09T05:48:17.175344+0000","last_unstale":"2023-10-09T05:50:46.852401+0000","last_undegraded":"2023-10-09T05:50:46.852401+0000","last_fullsized":"2023-10-09T05:50:46.852401+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:43.949807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":60,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.848919+0000","last_change":"2023-10-09T05:49:21.422120+0000","last_active":"2023-10-09T05:50:46.848919+0000","last_peered":"2023-10-09T05:50:46.848919+0000","last_clean":"2023-10-09T05:50:46.848919+0000","last_became_active":"2023-10-09T05:49:21.421835+0000","last_became_peered":"2023-10-09T05:49:21.421835+0000","last_unstale":"2023-10-09T05:50:46.848919+0000","last_undegraded":"2023-10-09T05:50:46.848919+0000","last_fullsized":"2023-10-09T05:50:46.848919+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:56.262510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":75,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.849289+0000","last_change":"2023-10-09T05:48:49.134851+0000","last_active":"2023-10-09T05:50:46.849289+0000","last_peered":"2023-10-09T05:50:46.849289+0000","last_clean":"2023-10-09T05:50:46.849289+0000","last_became_active":"2023-10-09T05:48:49.134532+0000","last_became_peered":"2023-10-09T05:48:49.134532+0000","last_unstale":"2023-10-09T05:50:46.849289+0000","last_undegraded":"2023-10-09T05:50:46.849289+0000","last_fullsized":"2023-10-09T05:50:46.849289+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:23.395907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"107'921","reported_seq":935,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:50:55.182983+0000","last_change":"2023-10-09T05:50:39.779775+0000","last_active":"2023-10-09T05:50:55.182983+0000","last_peered":"2023-10-09T05:50:55.182983+0000","last_clean":"2023-10-09T05:50:55.182983+0000","last_became_active":"2023-10-09T05:50:39.779424+0000","last_became_peered":"2023-10-09T05:50:39.779424+0000","last_unstale":"2023-10-09T05:50:55.182983+0000","last_undegraded":"2023-10-09T05:50:55.182983+0000","last_fullsized":"2023-10-09T05:50:55.182983+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":921,"log_dups_size":0,"ondisk_log_size":921,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:37.657767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60358656,"num_objects":921,"num_object_clones":0,"num_object_copies":2763,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":921,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":921,"num_write_kb":58944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":71,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.853481+0000","last_change":"2023-10-09T05:47:03.230117+0000","last_active":"2023-10-09T05:50:46.853481+0000","last_peered":"2023-10-09T05:50:46.853481+0000","last_clean":"2023-10-09T05:50:46.853481+0000","last_became_active":"2023-10-09T05:47:03.229660+0000","last_became_peered":"2023-10-09T05:47:03.229660+0000","last_unstale":"2023-10-09T05:50:46.853481+0000","last_undegraded":"2023-10-09T05:50:46.853481+0000","last_fullsized":"2023-10-09T05:50:46.853481+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:07:21.669282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":55,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.427516+0000","last_change":"2023-10-09T05:48:49.139759+0000","last_active":"2023-10-09T05:50:46.427516+0000","last_peered":"2023-10-09T05:50:46.427516+0000","last_clean":"2023-10-09T05:50:46.427516+0000","last_became_active":"2023-10-09T05:48:49.139346+0000","last_became_peered":"2023-10-09T05:48:49.139346+0000","last_unstale":"2023-10-09T05:50:46.427516+0000","last_undegraded":"2023-10-09T05:50:46.427516+0000","last_fullsized":"2023-10-09T05:50:46.427516+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:51.980711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":45,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.849196+0000","last_change":"2023-10-09T05:47:03.233460+0000","last_active":"2023-10-09T05:50:46.849196+0000","last_peered":"2023-10-09T05:50:46.849196+0000","last_clean":"2023-10-09T05:50:46.849196+0000","last_became_active":"2023-10-09T05:47:03.231685+0000","last_became_peered":"2023-10-09T05:47:03.231685+0000","last_unstale":"2023-10-09T05:50:46.849196+0000","last_undegraded":"2023-10-09T05:50:46.849196+0000","last_fullsized":"2023-10-09T05:50:46.849196+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:10.790717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"107'907","reported_seq":921,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:50:55.181643+0000","last_change":"2023-10-09T05:50:40.123199+0000","last_active":"2023-10-09T05:50:55.181643+0000","last_peered":"2023-10-09T05:50:55.181643+0000","last_clean":"2023-10-09T05:50:55.181643+0000","last_became_active":"2023-10-09T05:50:40.122866+0000","last_became_peered":"2023-10-09T05:50:40.122866+0000","last_unstale":"2023-10-09T05:50:55.181643+0000","last_undegraded":"2023-10-09T05:50:55.181643+0000","last_fullsized":"2023-10-09T05:50:55.181643+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":907,"log_dups_size":0,"ondisk_log_size":907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:03.898397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59441152,"num_objects":907,"num_object_clones":0,"num_object_copies":2721,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":907,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":907,"num_write_kb":58048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":68,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.427894+0000","last_change":"2023-10-09T05:49:21.423414+0000","last_active":"2023-10-09T05:50:46.427894+0000","last_peered":"2023-10-09T05:50:46.427894+0000","last_clean":"2023-10-09T05:50:46.427894+0000","last_became_active":"2023-10-09T05:49:21.423085+0000","last_became_peered":"2023-10-09T05:49:21.423085+0000","last_unstale":"2023-10-09T05:50:46.427894+0000","last_undegraded":"2023-10-09T05:50:46.427894+0000","last_fullsized":"2023-10-09T05:50:46.427894+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:51.777782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":40,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.427873+0000","last_change":"2023-10-09T05:48:17.174101+0000","last_active":"2023-10-09T05:50:46.427873+0000","last_peered":"2023-10-09T05:50:46.427873+0000","last_clean":"2023-10-09T05:50:46.427873+0000","last_became_active":"2023-10-09T05:48:17.173752+0000","last_became_peered":"2023-10-09T05:48:17.173752+0000","last_unstale":"2023-10-09T05:50:46.427873+0000","last_undegraded":"2023-10-09T05:50:46.427873+0000","last_fullsized":"2023-10-09T05:50:46.427873+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:45:16.222083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":65,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.426211+0000","last_change":"2023-10-09T05:49:21.423248+0000","last_active":"2023-10-09T05:50:46.426211+0000","last_peered":"2023-10-09T05:50:46.426211+0000","last_clean":"2023-10-09T05:50:46.426211+0000","last_became_active":"2023-10-09T05:49:21.422985+0000","last_became_peered":"2023-10-09T05:49:21.422985+0000","last_unstale":"2023-10-09T05:50:46.426211+0000","last_undegraded":"2023-10-09T05:50:46.426211+0000","last_fullsized":"2023-10-09T05:50:46.426211+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:02.431791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"107'797","reported_seq":810,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:50:54.356884+0000","last_change":"2023-10-09T05:50:39.779923+0000","last_active":"2023-10-09T05:50:54.356884+0000","last_peered":"2023-10-09T05:50:54.356884+0000","last_clean":"2023-10-09T05:50:54.356884+0000","last_became_active":"2023-10-09T05:50:39.779669+0000","last_became_peered":"2023-10-09T05:50:39.779669+0000","last_unstale":"2023-10-09T05:50:54.356884+0000","last_undegraded":"2023-10-09T05:50:54.356884+0000","last_fullsized":"2023-10-09T05:50:54.356884+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":797,"log_dups_size":0,"ondisk_log_size":797,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:26:38.454592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52232192,"num_objects":797,"num_object_clones":0,"num_object_copies":2391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":797,"num_write_kb":51008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":71,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.852302+0000","last_change":"2023-10-09T05:48:49.137565+0000","last_active":"2023-10-09T05:50:46.852302+0000","last_peered":"2023-10-09T05:50:46.852302+0000","last_clean":"2023-10-09T05:50:46.852302+0000","last_became_active":"2023-10-09T05:48:49.137099+0000","last_became_peered":"2023-10-09T05:48:49.137099+0000","last_unstale":"2023-10-09T05:50:46.852302+0000","last_undegraded":"2023-10-09T05:50:46.852302+0000","last_fullsized":"2023-10-09T05:50:46.852302+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:28.103821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":64,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.849278+0000","last_change":"2023-10-09T05:48:17.174996+0000","last_active":"2023-10-09T05:50:46.849278+0000","last_peered":"2023-10-09T05:50:46.849278+0000","last_clean":"2023-10-09T05:50:46.849278+0000","last_became_active":"2023-10-09T05:48:17.174668+0000","last_became_peered":"2023-10-09T05:48:17.174668+0000","last_unstale":"2023-10-09T05:50:46.849278+0000","last_undegraded":"2023-10-09T05:50:46.849278+0000","last_fullsized":"2023-10-09T05:50:46.849278+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:49.712686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":49,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.851936+0000","last_change":"2023-10-09T05:40:12.676587+0000","last_active":"2023-10-09T05:50:46.851936+0000","last_peered":"2023-10-09T05:50:46.851936+0000","last_clean":"2023-10-09T05:50:46.851936+0000","last_became_active":"2023-10-09T05:40:12.676330+0000","last_became_peered":"2023-10-09T05:40:12.676330+0000","last_unstale":"2023-10-09T05:50:46.851936+0000","last_undegraded":"2023-10-09T05:50:46.851936+0000","last_fullsized":"2023-10-09T05:50:46.851936+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:46.659556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"107'730","reported_seq":743,"reported_epoch":107,"state":"active+clean","last_fresh":"2023-10-09T05:50:54.359909+0000","last_change":"2023-10-09T05:50:40.123172+0000","last_active":"2023-10-09T05:50:54.359909+0000","last_peered":"2023-10-09T05:50:54.359909+0000","last_clean":"2023-10-09T05:50:54.359909+0000","last_became_active":"2023-10-09T05:50:40.122867+0000","last_became_peered":"2023-10-09T05:50:40.122867+0000","last_unstale":"2023-10-09T05:50:54.359909+0000","last_undegraded":"2023-10-09T05:50:54.359909+0000","last_fullsized":"2023-10-09T05:50:54.359909+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":730,"log_dups_size":0,"ondisk_log_size":730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:16.038279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47841280,"num_objects":730,"num_object_clones":0,"num_object_copies":2190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":730,"num_write_kb":46720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":49,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.849339+0000","last_change":"2023-10-09T05:47:03.233346+0000","last_active":"2023-10-09T05:50:46.849339+0000","last_peered":"2023-10-09T05:50:46.849339+0000","last_clean":"2023-10-09T05:50:46.849339+0000","last_became_active":"2023-10-09T05:47:03.231300+0000","last_became_peered":"2023-10-09T05:47:03.231300+0000","last_unstale":"2023-10-09T05:50:46.849339+0000","last_undegraded":"2023-10-09T05:50:46.849339+0000","last_fullsized":"2023-10-09T05:50:46.849339+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:22.050814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":87,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-09T05:50:45.467784+0000","last_change":"2023-10-09T05:49:21.430604+0000","last_active":"2023-10-09T05:50:45.467784+0000","last_peered":"2023-10-09T05:50:45.467784+0000","last_clean":"2023-10-09T05:50:45.467784+0000","last_became_active":"2023-10-09T05:49:21.430220+0000","last_became_peered":"2023-10-09T05:49:21.430220+0000","last_unstale":"2023-10-09T05:50:45.467784+0000","last_undegraded":"2023-10-09T05:50:45.467784+0000","last_fullsized":"2023-10-09T05:50:45.467784+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:06:01.882964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":69,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.852267+0000","last_change":"2023-10-09T05:48:17.173906+0000","last_active":"2023-10-09T05:50:46.852267+0000","last_peered":"2023-10-09T05:50:46.852267+0000","last_clean":"2023-10-09T05:50:46.852267+0000","last_became_active":"2023-10-09T05:48:17.173564+0000","last_became_peered":"2023-10-09T05:48:17.173564+0000","last_unstale":"2023-10-09T05:50:46.852267+0000","last_undegraded":"2023-10-09T05:50:46.852267+0000","last_fullsized":"2023-10-09T05:50:46.852267+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:37.582247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":83,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.853252+0000","last_change":"2023-10-09T05:49:21.426304+0000","last_active":"2023-10-09T05:50:46.853252+0000","last_peered":"2023-10-09T05:50:46.853252+0000","last_clean":"2023-10-09T05:50:46.853252+0000","last_became_active":"2023-10-09T05:49:21.425949+0000","last_became_peered":"2023-10-09T05:49:21.425949+0000","last_unstale":"2023-10-09T05:50:46.853252+0000","last_undegraded":"2023-10-09T05:50:46.853252+0000","last_fullsized":"2023-10-09T05:50:46.853252+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:11.209520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":52,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.852700+0000","last_change":"2023-10-09T05:40:12.633195+0000","last_active":"2023-10-09T05:50:46.852700+0000","last_peered":"2023-10-09T05:50:46.852700+0000","last_clean":"2023-10-09T05:50:46.852700+0000","last_became_active":"2023-10-09T05:40:12.632942+0000","last_became_peered":"2023-10-09T05:40:12.632942+0000","last_unstale":"2023-10-09T05:50:46.852700+0000","last_undegraded":"2023-10-09T05:50:46.852700+0000","last_fullsized":"2023-10-09T05:50:46.852700+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:04.470685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":83,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.853493+0000","last_change":"2023-10-09T05:49:21.426615+0000","last_active":"2023-10-09T05:50:46.853493+0000","last_peered":"2023-10-09T05:50:46.853493+0000","last_clean":"2023-10-09T05:50:46.853493+0000","last_became_active":"2023-10-09T05:49:21.426245+0000","last_became_peered":"2023-10-09T05:49:21.426245+0000","last_unstale":"2023-10-09T05:50:46.853493+0000","last_undegraded":"2023-10-09T05:50:46.853493+0000","last_fullsized":"2023-10-09T05:50:46.853493+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:47.642455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":76,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.426377+0000","last_change":"2023-10-09T05:49:21.430192+0000","last_active":"2023-10-09T05:50:46.426377+0000","last_peered":"2023-10-09T05:50:46.426377+0000","last_clean":"2023-10-09T05:50:46.426377+0000","last_became_active":"2023-10-09T05:49:21.429857+0000","last_became_peered":"2023-10-09T05:49:21.429857+0000","last_unstale":"2023-10-09T05:50:46.426377+0000","last_undegraded":"2023-10-09T05:50:46.426377+0000","last_fullsized":"2023-10-09T05:50:46.426377+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:45.887757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":91,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.426567+0000","last_change":"2023-10-09T05:49:21.421384+0000","last_active":"2023-10-09T05:50:46.426567+0000","last_peered":"2023-10-09T05:50:46.426567+0000","last_clean":"2023-10-09T05:50:46.426567+0000","last_became_active":"2023-10-09T05:49:21.420886+0000","last_became_peered":"2023-10-09T05:49:21.420886+0000","last_unstale":"2023-10-09T05:50:46.426567+0000","last_undegraded":"2023-10-09T05:50:46.426567+0000","last_fullsized":"2023-10-09T05:50:46.426567+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:33:24.415387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":64,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.849186+0000","last_change":"2023-10-09T05:48:49.139798+0000","last_active":"2023-10-09T05:50:46.849186+0000","last_peered":"2023-10-09T05:50:46.849186+0000","last_clean":"2023-10-09T05:50:46.849186+0000","last_became_active":"2023-10-09T05:48:49.139442+0000","last_became_peered":"2023-10-09T05:48:49.139442+0000","last_unstale":"2023-10-09T05:50:46.849186+0000","last_undegraded":"2023-10-09T05:50:46.849186+0000","last_fullsized":"2023-10-09T05:50:46.849186+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:25.034620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":66,"reported_epoch":106,"state":"active+clean","last_fresh":"2023-10-09T05:50:46.851768+0000","last_change":"2023-10-09T05:48:17.172491+0000","last_active":"2023-10-09T05:50:46.851768+0000","last_peered":"2023-10-09T05:50:46.851768+0000","last_clean":"2023-10-09T05:50:46.851768+0000","last_became_active":"2023-10-09T05:48:17.172154+0000","last_became_peered":"2023-10-09T05:48:17.172154+0000","last_unstale":"2023-10-09T05:50:46.851768+0000","last_undegraded":"2023-10-09T05:50:46.851768+0000","last_fullsized":"2023-10-09T05:50:46.851768+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:24:40.551774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":822411264,"num_objects":12549,"num_object_clones":0,"num_object_copies":37647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12549,"num_write_kb":803136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2297561088,"data_stored":2297561088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":12549,"ondisk_log_size":12549,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234136,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":553520,"kb_used_data":248040,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93191632,"statfs":{"total":95995035648,"available":95428231168,"internally_reserved":0,"allocated":253992960,"data_stored":253510227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.69199999999999995,"15min":0.76100000000000001},"min":{"1min":0.38500000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.74299999999999999,"5min":4.8940000000000001,"15min":4.8940000000000001},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.57999999999999996,"15min":0.60199999999999998},"min":{"1min":0.34799999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.749,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.34899999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.52300000000000002,"15min":0.52400000000000002},"min":{"1min":0.376,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.61199999999999999,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.372},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.55400000000000005,"15min":0.55200000000000005},"min":{"1min":0.37,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.71599999999999997,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.69999999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:50:52 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":0.70199999999999996,"15min":0.58399999999999996},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":16.791,"5min":16.791,"15min":16.791},"last":0.438},{"interface":"front","average":{"1min":1.452,"5min":0.78200000000000003,"15min":0.67000000000000004},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":16.777000000000001,"5min":16.777000000000001,"15min":16.777000000000001},"last":0.53500000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:50:22 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":2.661},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.499}]},{"osd":4,"last update":"Mon Oct 9 05:49:56 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.48399999999999999,"15min":0.48699999999999999},"min":{"1min":0.313,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":1.135},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.51200000000000001,"15min":0.498},"min":{"1min":0.378,"5min":0.27400000000000002,"15min":0.27100000000000002},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":3.165},"last":0.52600000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:49:56 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.499,"15min":0.499},"min":{"1min":0.33200000000000002,"5min":0.28199999999999997,"15min":0.248},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":2.5990000000000002},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.622,"5min":0.55700000000000005,"15min":0.54000000000000004},"min":{"1min":0.38800000000000001,"5min":0.34000000000000002,"15min":0.28100000000000003},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":1.0660000000000001},"last":0.60699999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:49:56 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.55900000000000005,"15min":0.54300000000000004},"min":{"1min":0.42399999999999999,"5min":0.317,"15min":0.29599999999999999},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":2.8170000000000002},"last":0.62},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.57299999999999995,"15min":0.55100000000000005},"min":{"1min":0.41899999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":3.2589999999999999},"last":0.55600000000000005}]}]},{"osd":6,"up_from":46,"seq":197568495772,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":538848,"kb_used_data":225176,"kb_used_omap":0,"kb_used_meta":313664,"kb_avail":93206304,"statfs":{"total":95995035648,"available":95443255296,"internally_reserved":0,"allocated":230580224,"data_stored":230113464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321191936},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.57999999999999996,"15min":0.59299999999999997},"min":{"1min":0.40300000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77000000000000002,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.57099999999999995,"15min":0.58899999999999997},"min":{"1min":0.38600000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.67800000000000005,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.31900000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:50:23 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.58399999999999996,"15min":0.58999999999999997},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.68899999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.57899999999999996,"15min":0.58599999999999997},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.66700000000000004,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.504}]},{"osd":2,"last update":"Mon Oct 9 05:50:50 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.53600000000000003,"15min":0.54000000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.55500000000000005,"15min":0.55700000000000005},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.55800000000000005}]},{"osd":3,"last update":"Mon Oct 9 05:50:23 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.59799999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:50:32 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48499999999999999,"15min":0.45700000000000002},"min":{"1min":0.34200000000000003,"5min":0.309,"15min":0.26300000000000001},"max":{"1min":0.65800000000000003,"5min":0.71899999999999997,"15min":0.75600000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.52400000000000002,"15min":0.495},"min":{"1min":0.36199999999999999,"5min":0.29099999999999998,"15min":0.28599999999999998},"max":{"1min":0.73799999999999999,"5min":0.80500000000000005,"15min":0.82799999999999996},"last":0.48899999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:50:32 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.51800000000000002,"15min":0.47999999999999998},"min":{"1min":0.35899999999999999,"5min":0.32600000000000001,"15min":0.27700000000000002},"max":{"1min":0.70699999999999996,"5min":0.77500000000000002,"15min":0.81299999999999994},"last":0.371},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.55900000000000005,"15min":0.52500000000000002},"min":{"1min":0.38600000000000001,"5min":0.32400000000000001,"15min":0.32200000000000001},"max":{"1min":0.79700000000000004,"5min":0.85099999999999998,"15min":0.85799999999999998},"last":0.57099999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:49:58 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.54700000000000004,"15min":0.53400000000000003},"min":{"1min":0.39800000000000002,"5min":0.316,"15min":0.316},"max":{"1min":0.64800000000000002,"5min":0.78800000000000003,"15min":0.79700000000000004},"last":0.439},{"interface":"front","average":{"1min":0.626,"5min":0.627,"15min":0.59799999999999998},"min":{"1min":0.39800000000000002,"5min":0.39600000000000002,"15min":0.35799999999999998},"max":{"1min":0.82299999999999995,"5min":0.88200000000000001,"15min":0.88300000000000001},"last":0.61099999999999999}]}]},{"osd":1,"up_from":89,"seq":382252089378,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":329248,"kb_used_data":289496,"kb_used_omap":9,"kb_used_meta":39734,"kb_avail":93415904,"statfs":{"total":95995035648,"available":95657885696,"internally_reserved":0,"allocated":296443904,"data_stored":295920481,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":40688617},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:50:19 2023","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.35599999999999998,"15min":0.35799999999999998},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":1.284},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.39700000000000002,"15min":0.40200000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":1.248}]},{"osd":2,"last update":"Mon Oct 9 05:50:53 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.57999999999999996,"15min":0.61299999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.754,"5min":4.5119999999999996,"15min":4.5119999999999996},"last":1.0640000000000001},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.504,"15min":0.52200000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.79700000000000004,"5min":3.6139999999999999,"15min":3.6139999999999999},"last":0.251}]},{"osd":3,"last update":"Mon Oct 9 05:50:23 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.32900000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:50:19 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.59599999999999997,"15min":0.59299999999999997},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.028,"5min":1.028,"15min":1.028},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60999999999999999,"15min":0.61099999999999999},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.76100000000000001,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.58099999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:50:19 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.45000000000000001,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":1.3680000000000001},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.60699999999999998,"15min":0.60899999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":1.347}]},{"osd":6,"last update":"Mon Oct 9 05:50:19 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.61299999999999999,"15min":0.61899999999999999},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.626,"5min":0.63400000000000001,"15min":0.63500000000000001},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.94599999999999995,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":1.3979999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:50:19 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.65600000000000003,"15min":0.66300000000000003},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.002,"5min":1.002,"15min":1.002},"last":1.5369999999999999},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.82499999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":1.5589999999999999}]}]},{"osd":0,"up_from":84,"seq":360777252913,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":281588,"kb_used_data":241772,"kb_used_omap":28,"kb_used_meta":39779,"kb_avail":93463564,"statfs":{"total":95995035648,"available":95706689536,"internally_reserved":0,"allocated":247574528,"data_stored":247034143,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":40734544},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:50:23 2023","interfaces":[{"interface":"back","average":{"1min":0.34999999999999998,"5min":0.622,"15min":0.66700000000000004},"min":{"1min":0.214,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.60399999999999998,"5min":7.625,"15min":7.625},"last":0.90900000000000003},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.498,"15min":0.52000000000000002},"min":{"1min":0.24099999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.72699999999999998,"5min":3.8130000000000002,"15min":3.8130000000000002},"last":1.006}]},{"osd":2,"last update":"Mon Oct 9 05:50:49 2023","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.33300000000000002,"15min":0.33200000000000002},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":0.56000000000000005,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":1.9830000000000001},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.66300000000000003,"15min":0.70299999999999996},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.71199999999999997,"5min":7.5640000000000001,"15min":7.5640000000000001},"last":2.1659999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:50:23 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":2.093},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":1.036}]},{"osd":4,"last update":"Mon Oct 9 05:50:04 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.67500000000000004,"15min":0.69499999999999995},"min":{"1min":0.42499999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.95799999999999996},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.69799999999999995,"15min":0.72199999999999998},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.90100000000000002,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.877}]},{"osd":5,"last update":"Mon Oct 9 05:50:04 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.66800000000000004,"15min":0.68600000000000005},"min":{"1min":0.42199999999999999,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":2.1320000000000001},{"interface":"front","average":{"1min":0.622,"5min":0.68200000000000005,"15min":0.71599999999999997},"min":{"1min":0.498,"5min":0.376,"15min":0.376},"max":{"1min":1.27,"5min":1.27,"15min":1.27},"last":0.64000000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:50:04 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.68999999999999995,"15min":0.72499999999999998},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.93300000000000005},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.71799999999999997,"15min":0.746},"min":{"1min":0.47499999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.995,"5min":0.995,"15min":0.995},"last":2.0489999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:50:04 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.75700000000000001,"15min":0.78900000000000003},"min":{"1min":0.47799999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.016,"5min":1.016,"15min":1.016},"last":2.1909999999999998},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.73699999999999999,"15min":0.76500000000000001},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":1.163}]}]},{"osd":2,"up_from":94,"seq":403726925851,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":220880,"kb_used_data":185096,"kb_used_omap":81,"kb_used_meta":35694,"kb_avail":93524272,"statfs":{"total":95995035648,"available":95768854528,"internally_reserved":0,"allocated":189538304,"data_stored":189021242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":36550874},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.33600000000000002,"5min":0.39800000000000002,"15min":0.40799999999999997},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.55300000000000005,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.34799999999999998,"5min":0.40200000000000002,"15min":0.41099999999999998},"min":{"1min":0.247,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.497,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.28599999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.41599999999999998,"15min":0.42599999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.50700000000000001,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.39700000000000002,"15min":0.40500000000000003},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.51500000000000001,"5min":0.63,"15min":0.63},"last":0.42399999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:50:22 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.35399999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.66400000000000003,"15min":0.67200000000000004},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.623},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.69599999999999995,"15min":0.70599999999999996},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.87,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.72499999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.66900000000000004,"15min":0.67500000000000004},"min":{"1min":0.45800000000000002,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.88300000000000001,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.65600000000000003,"15min":0.66600000000000004},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.84899999999999998,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.66700000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.65000000000000002,"15min":0.66100000000000003},"min":{"1min":0.41899999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.80300000000000005,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.68100000000000005,"15min":0.68700000000000006},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.88800000000000001,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.67800000000000005}]},{"osd":7,"last update":"Mon Oct 9 05:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.66200000000000003,"15min":0.66900000000000004},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.82499999999999996,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.69399999999999995,"15min":0.70299999999999996},"min":{"1min":0.47399999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.82899999999999996,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.71099999999999997}]}]},{"osd":3,"up_from":99,"seq":425201762325,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":461384,"kb_used_data":433920,"kb_used_omap":7,"kb_used_meta":27448,"kb_avail":93283768,"statfs":{"total":95995035648,"available":95522578432,"internally_reserved":0,"allocated":444334080,"data_stored":443829525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":28107368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:50:21 2023","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"last":1.0009999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:50:21 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.37,"15min":0.37},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":1.405}]},{"osd":2,"last update":"Mon Oct 9 05:50:21 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.376,"15min":0.376},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":1.78},{"interface":"front","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":2.7400000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:50:21 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":1.2070000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.54300000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:50:21 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":1.627},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.95899999999999996,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":1.494}]},{"osd":6,"last update":"Mon Oct 9 05:50:21 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":1.542}]},{"osd":7,"last update":"Mon Oct 9 05:50:21 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":1.597},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":1.242}]}]},{"osd":4,"up_from":30,"seq":128849019045,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":514796,"kb_used_data":205220,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93230356,"statfs":{"total":95995035648,"available":95467884544,"internally_reserved":0,"allocated":210145280,"data_stored":209666352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:50:10 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.57699999999999996,"15min":0.59799999999999998},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.61499999999999999,"15min":0.63700000000000001},"min":{"1min":0.35799999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.40899999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:50:23 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.57699999999999996,"15min":0.56999999999999995},"min":{"1min":0.40100000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.996,"5min":0.996,"15min":0.996},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.52600000000000002,"15min":0.52000000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.34899999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:50:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":0.873,"15min":0.67500000000000004},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":33.262,"5min":33.262,"15min":33.262},"last":33.262},{"interface":"front","average":{"1min":2.0990000000000002,"5min":0.96599999999999997,"15min":0.77700000000000002},"min":{"1min":0.40500000000000003,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":33.240000000000002,"5min":33.240000000000002,"15min":33.240000000000002},"last":33.240000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:50:20 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.44}]},{"osd":5,"last update":"Mon Oct 9 05:50:10 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.47599999999999998,"15min":0.46000000000000002},"min":{"1min":0.36899999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.66300000000000003,"5min":0.83399999999999996,"15min":0.90500000000000003},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.53000000000000003,"15min":0.498},"min":{"1min":0.33100000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.78000000000000003,"5min":0.82199999999999995,"15min":0.88100000000000001},"last":0.377}]},{"osd":6,"last update":"Mon Oct 9 05:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.50900000000000001,"15min":0.5},"min":{"1min":0.36699999999999999,"5min":0.31,"15min":0.27400000000000002},"max":{"1min":0.70999999999999996,"5min":0.77100000000000002,"15min":0.83299999999999996},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.53500000000000003},"min":{"1min":0.35999999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.70299999999999996,"5min":0.86199999999999999,"15min":0.89900000000000002},"last":0.36499999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:50:00 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.58399999999999996,"15min":0.56799999999999995},"min":{"1min":0.41699999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":1.046},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.58699999999999997,"15min":0.58399999999999996},"min":{"1min":0.48199999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.95799999999999996},"last":0.45100000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757408,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":508188,"kb_used_data":206804,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93236964,"statfs":{"total":95995035648,"available":95474651136,"internally_reserved":0,"allocated":211767296,"data_stored":211300496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:50:07 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.57299999999999995,"15min":0.58999999999999997},"min":{"1min":0.35599999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.81100000000000005,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.59199999999999997,"15min":0.61499999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.68899999999999995,"5min":1,"15min":1},"last":1.427}]},{"osd":1,"last update":"Mon Oct 9 05:50:22 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.70499999999999996,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.41299999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.76700000000000002,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":1.131}]},{"osd":2,"last update":"Mon Oct 9 05:50:50 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.57599999999999996,"15min":0.58299999999999996},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.72199999999999998,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.58599999999999997,"15min":0.58699999999999997},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.234,"5min":1.234,"15min":1.234},"last":1.234}]},{"osd":3,"last update":"Mon Oct 9 05:50:22 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.5609999999999999,"5min":1.5609999999999999,"15min":1.5609999999999999},"last":0.745}]},{"osd":4,"last update":"Mon Oct 9 05:50:07 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.48799999999999999,"15min":0.45600000000000002},"min":{"1min":0.311,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.754,"5min":0.754,"15min":3.5430000000000001},"last":1.044},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.50800000000000001,"15min":0.47599999999999998},"min":{"1min":0.372,"5min":0.29999999999999999,"15min":0.28999999999999998},"max":{"1min":0.66600000000000004,"5min":0.72699999999999998,"15min":1.6990000000000001},"last":0.65300000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:50:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.53800000000000003,"15min":0.495},"min":{"1min":0.438,"5min":0.307,"15min":0.29899999999999999},"max":{"1min":0.68999999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.83399999999999996},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.57599999999999996,"15min":0.53100000000000003},"min":{"1min":0.35299999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.78000000000000003,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.76600000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:50:03 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.60799999999999998,"15min":0.56299999999999994},"min":{"1min":0.435,"5min":0.32700000000000001,"15min":0.315},"max":{"1min":0.81000000000000005,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.92500000000000004},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.58799999999999997,"15min":0.59799999999999998},"min":{"1min":0.47399999999999998,"5min":0.35099999999999998,"15min":0.34899999999999998},"max":{"1min":0.79400000000000004,"5min":0.79700000000000004,"15min":3.2869999999999999},"last":1.3939999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":275513344,"data_stored":275513344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":296943616,"data_stored":296943616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":205586432,"data_stored":205586432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":497483776,"data_stored":497483776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":245432320,"data_stored":245432320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":245825536,"data_stored":245825536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":240386048,"data_stored":240386048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":290390016,"data_stored":290390016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:51:00.324 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:51:00.324 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 11 2023-10-09T05:51:00.324 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:00.351 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 11 16 24567 24551 139.428 80.8125 0.00168498 0.00708968 2023-10-09T05:51:00.354 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:00.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:00 smithi064 ceph-mon[212750]: pgmap v240: 89 pgs: 89 active+clean; 834 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:51:00.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:00 smithi064 ceph-mon[220044]: pgmap v240: 89 pgs: 89 active+clean; 834 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:51:00.515 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:00.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:00.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:00.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:00 smithi156 ceph-mon[195101]: pgmap v240: 89 pgs: 89 active+clean; 834 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:51:00.622 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:00.657 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:00.684 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:00.717 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:00.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:00.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:00.822 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:00.837 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:51:00.837 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:51:00.879 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:00.993 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:00.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:00.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:01.097 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:01.135 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:01.202 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:01.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:01.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:01.309 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:01.352 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 12 16 26306 26290 136.867 108.688 0.00689701 0.0073007 2023-10-09T05:51:01.358 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:01.405 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:51:01.438 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:01.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:01.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:01.547 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:01.633 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:02.136 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:02.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:02.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:02.245 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:02.303 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:02.353 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 13 16 28059 28043 134.75 109.562 0.00914789 0.00741443 2023-10-09T05:51:02.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:02 smithi064 ceph-mon[212750]: pgmap v241: 89 pgs: 89 active+clean; 1.4 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-10-09T05:51:02.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:02 smithi064 ceph-mon[220044]: pgmap v241: 89 pgs: 89 active+clean; 1.4 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-10-09T05:51:02.568 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:51:02.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:02 smithi156 ceph-mon[195101]: pgmap v241: 89 pgs: 89 active+clean; 1.4 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-10-09T05:51:02.634 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:02.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:02.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:02.743 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:02.781 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:02.844 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:02.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:02.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:02.950 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:03.005 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:03.354 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 14 16 29797 29781 132.876 108.625 0.00643769 0.00752099 2023-10-09T05:51:03.365 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:03.367 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:03 smithi064 ceph-mon[220044]: pgmap v242: 89 pgs: 89 active+clean; 1.5 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-10-09T05:51:03.367 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:03 smithi064 ceph-mon[212750]: pgmap v242: 89 pgs: 89 active+clean; 1.5 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-10-09T05:51:03.384 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:03.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:03.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:03.499 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:03.520 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:03.529 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:03 smithi156 ceph-mon[195101]: pgmap v242: 89 pgs: 89 active+clean; 1.5 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-10-09T05:51:03.533 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:03.597 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:03.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:03.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:03.655 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:03.702 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:03.742 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:03.812 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:03.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:03.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:03.919 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:03.998 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:04.150 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:04.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:04.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:04.264 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:04.301 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:04.356 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 15 16 31506 31490 131.127 106.812 0.00435306 0.00761995 2023-10-09T05:51:04.372 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:04.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:04.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:04.480 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:04.575 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:04.839 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:04.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:04.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:04.949 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:05.004 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:05.158 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:05.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:05.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:05.265 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:05.306 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:05.356 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 16 16 33305 33289 129.959 112.438 0.00563221 0.00768977 2023-10-09T05:51:05.384 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:05.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:05.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:05.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:51:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:51:05] "GET /metrics HTTP/1.1" 200 36317 "" "Prometheus/2.43.0" 2023-10-09T05:51:05.489 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:05.552 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:05.605 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T05:51:05.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:05 smithi156 ceph-mon[195101]: pgmap v243: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 160 MiB/s wr, 2.56k op/s 2023-10-09T05:51:05.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:05 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3037949670' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:51:05.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:05 smithi064 ceph-mon[212750]: pgmap v243: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 160 MiB/s wr, 2.56k op/s 2023-10-09T05:51:05.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:05 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3037949670' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:51:05.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:05 smithi064 ceph-mon[220044]: pgmap v243: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 160 MiB/s wr, 2.56k op/s 2023-10-09T05:51:05.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:05 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3037949670' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:51:06.356 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 17 16 34858 34842 128.025 97.0625 0.00177706 0.00779815 2023-10-09T05:51:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:06 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:51:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:06 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T05:51:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:06 smithi064 ceph-mon[212750]: failed to encode map e108 with expected crc 2023-10-09T05:51:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:06 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3037949670' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:51:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:06 smithi064 ceph-mon[212750]: osdmap e108: 8 total, 8 up, 8 in 2023-10-09T05:51:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:06 smithi064 ceph-mon[212750]: failed to encode map e108 with expected crc 2023-10-09T05:51:06.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:06 smithi064 ceph-mon[212750]: failed to encode map e108 with expected crc 2023-10-09T05:51:06.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:06 smithi064 ceph-mon[212750]: failed to encode map e108 with expected crc 2023-10-09T05:51:06.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:06 smithi064 ceph-mon[212750]: failed to encode map e108 with expected crc 2023-10-09T05:51:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:06 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:51:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:06 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T05:51:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:06 smithi064 ceph-mon[220044]: failed to encode map e108 with expected crc 2023-10-09T05:51:06.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:06 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3037949670' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:51:06.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:06 smithi064 ceph-mon[220044]: osdmap e108: 8 total, 8 up, 8 in 2023-10-09T05:51:06.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:06 smithi064 ceph-mon[220044]: failed to encode map e108 with expected crc 2023-10-09T05:51:06.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:06 smithi064 ceph-mon[220044]: failed to encode map e108 with expected crc 2023-10-09T05:51:06.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:06 smithi064 ceph-mon[220044]: failed to encode map e108 with expected crc 2023-10-09T05:51:06.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:06 smithi064 ceph-mon[220044]: failed to encode map e108 with expected crc 2023-10-09T05:51:06.823 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:06.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:06.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:06.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:06 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:51:06.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:06 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T05:51:06.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:06 smithi156 ceph-mon[195101]: failed to encode map e108 with expected crc 2023-10-09T05:51:06.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:06 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3037949670' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:51:06.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:06 smithi156 ceph-mon[195101]: osdmap e108: 8 total, 8 up, 8 in 2023-10-09T05:51:06.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:06 smithi156 ceph-mon[195101]: failed to encode map e108 with expected crc 2023-10-09T05:51:06.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:06 smithi156 ceph-mon[195101]: failed to encode map e108 with expected crc 2023-10-09T05:51:06.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:06 smithi156 ceph-mon[195101]: failed to encode map e108 with expected crc 2023-10-09T05:51:06.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:06 smithi156 ceph-mon[195101]: failed to encode map e108 with expected crc 2023-10-09T05:51:06.933 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:07.082 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:07.123 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:07.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:07.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:07.230 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:07.289 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:07.356 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 18 16 36017 36001 124.937 72.4375 0.0147653 0.00799774 2023-10-09T05:51:07.719 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:07.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:07.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:07.831 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:07.877 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:07.934 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:07.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:07.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:08.039 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:08.094 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:08.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:08 smithi156 ceph-mon[195101]: pgmap v245: 89 pgs: 89 active+clean; 1.9 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2023-10-09T05:51:08.356 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 19 16 37556 37540 123.424 96.1875 0.00823336 0.0080974 2023-10-09T05:51:08.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:08 smithi064 ceph-mon[212750]: pgmap v245: 89 pgs: 89 active+clean; 1.9 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2023-10-09T05:51:08.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:08 smithi064 ceph-mon[220044]: pgmap v245: 89 pgs: 89 active+clean; 1.9 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2023-10-09T05:51:08.535 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:08.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:08.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:08.641 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:08.713 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:09.036 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:09.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:09.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:09.144 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:09.207 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:09.334 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:09.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:09.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:09.356 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-10-09T05:51:09.357271+0000 min lat: 0.00106234 max lat: 0.246418 avg lat: 0.00817152 2023-10-09T05:51:09.356 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:51:09.356 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 20 16 39167 39151 122.287 100.688 0.00774833 0.00817152 2023-10-09T05:51:09.387 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:51:09.440 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:09.440 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:51:09.480 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:51:09.481 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:09.527 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:51:09.543 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:09.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:09.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:09.583 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:51:09.634 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:51:09.649 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:09.680 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:51:09.688 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:09.717 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:51:09.717 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:51:09.718 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:51:09.718 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:51:09.718 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:51:09.718 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:51:09.718 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:51:09.718 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:51:09.718 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:51:09.733 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:51:09.745 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:09.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:09.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:09.855 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:09.888 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:09.924 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:51:09.925 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T05:51:09.925 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T05:51:09.925 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T05:51:09.925 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:51:09.939 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:09.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:09.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:10.044 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:10.072 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:10 smithi156 ceph-mon[195101]: pgmap v246: 89 pgs: 89 active+clean; 2.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-10-09T05:51:10.073 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:10 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1559027455' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:10.073 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:10 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1460802856' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T05:51:10.073 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:10 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T05:51:10.086 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 1 pg_num to 11 2023-10-09T05:51:10.124 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:10.356 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 21 16 40491 40475 120.405 82.75 0.00827484 0.0083005 2023-10-09T05:51:10.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:10 smithi064 ceph-mon[220044]: pgmap v246: 89 pgs: 89 active+clean; 2.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-10-09T05:51:10.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:10 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1559027455' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:10.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:10 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1460802856' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T05:51:10.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:10 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T05:51:10.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:10 smithi064 ceph-mon[212750]: pgmap v246: 89 pgs: 89 active+clean; 2.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-10-09T05:51:10.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:10 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1559027455' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:10.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:10 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1460802856' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T05:51:10.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:10 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T05:51:11.361 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 22 16 41791 41775 118.603 81.25 0.00597227 0.00842683 2023-10-09T05:51:11.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:11 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-10-09T05:51:11.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:11 smithi156 ceph-mon[195101]: osdmap e109: 8 total, 8 up, 8 in 2023-10-09T05:51:11.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:11 smithi156 ceph-mon[195101]: failed to encode map e109 with expected crc 2023-10-09T05:51:11.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:11 smithi156 ceph-mon[195101]: failed to encode map e109 with expected crc 2023-10-09T05:51:11.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:11 smithi156 ceph-mon[195101]: failed to encode map e109 with expected crc 2023-10-09T05:51:11.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:11 smithi156 ceph-mon[195101]: failed to encode map e109 with expected crc 2023-10-09T05:51:11.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:11 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-10-09T05:51:11.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:11 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-10-09T05:51:11.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:11 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-10-09T05:51:11.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:11 smithi064 ceph-mon[220044]: osdmap e109: 8 total, 8 up, 8 in 2023-10-09T05:51:11.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:11 smithi064 ceph-mon[220044]: failed to encode map e109 with expected crc 2023-10-09T05:51:11.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:11 smithi064 ceph-mon[220044]: failed to encode map e109 with expected crc 2023-10-09T05:51:11.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:11 smithi064 ceph-mon[220044]: failed to encode map e109 with expected crc 2023-10-09T05:51:11.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:11 smithi064 ceph-mon[220044]: failed to encode map e109 with expected crc 2023-10-09T05:51:11.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:11 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-10-09T05:51:11.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:11 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-10-09T05:51:11.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:11 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-10-09T05:51:11.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:11 smithi064 ceph-mon[212750]: osdmap e109: 8 total, 8 up, 8 in 2023-10-09T05:51:11.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:11 smithi064 ceph-mon[212750]: failed to encode map e109 with expected crc 2023-10-09T05:51:11.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:11 smithi064 ceph-mon[212750]: failed to encode map e109 with expected crc 2023-10-09T05:51:11.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:11 smithi064 ceph-mon[212750]: failed to encode map e109 with expected crc 2023-10-09T05:51:11.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:11 smithi064 ceph-mon[212750]: failed to encode map e109 with expected crc 2023-10-09T05:51:11.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:11 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-10-09T05:51:11.442 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:11 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-10-09T05:51:11.699 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:11.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:11.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:11.805 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:11.830 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:11.871 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:11.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:11.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:11.978 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:12.034 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:12.084 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:12 smithi156 ceph-mon[195101]: pgmap v248: 89 pgs: 89 active+clean; 2.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-10-09T05:51:12.084 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:12 smithi156 ceph-mon[195101]: failed to encode map e110 with expected crc 2023-10-09T05:51:12.084 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:12 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-10-09T05:51:12.084 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:12 smithi156 ceph-mon[195101]: osdmap e110: 8 total, 8 up, 8 in 2023-10-09T05:51:12.289 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:12 smithi064 ceph-mon[212750]: pgmap v248: 89 pgs: 89 active+clean; 2.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-10-09T05:51:12.289 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:12 smithi064 ceph-mon[212750]: failed to encode map e110 with expected crc 2023-10-09T05:51:12.289 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:12 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-10-09T05:51:12.289 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:12 smithi064 ceph-mon[212750]: osdmap e110: 8 total, 8 up, 8 in 2023-10-09T05:51:12.290 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:12 smithi064 ceph-mon[220044]: pgmap v248: 89 pgs: 89 active+clean; 2.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-10-09T05:51:12.290 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:12 smithi064 ceph-mon[220044]: failed to encode map e110 with expected crc 2023-10-09T05:51:12.290 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:12 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-10-09T05:51:12.291 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:12 smithi064 ceph-mon[220044]: osdmap e110: 8 total, 8 up, 8 in 2023-10-09T05:51:12.363 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 23 16 43016 43000 116.776 76.5625 0.00647536 0.00855916 2023-10-09T05:51:12.461 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:12.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:12.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:12.552 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:51:12.573 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:12.609 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:12.668 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:12.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:12.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:12.678 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:12.776 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:12.907 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:13.034 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:13.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:13.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:13.139 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:13.211 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:13 smithi064 ceph-mon[220044]: osdmap e111: 8 total, 8 up, 8 in 2023-10-09T05:51:13.211 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:13 smithi064 ceph-mon[220044]: failed to encode map e111 with expected crc 2023-10-09T05:51:13.211 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:13 smithi064 ceph-mon[220044]: failed to encode map e111 with expected crc 2023-10-09T05:51:13.211 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:13 smithi064 ceph-mon[220044]: failed to encode map e111 with expected crc 2023-10-09T05:51:13.211 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:13 smithi064 ceph-mon[220044]: failed to encode map e111 with expected crc 2023-10-09T05:51:13.211 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:13 smithi064 ceph-mon[212750]: osdmap e111: 8 total, 8 up, 8 in 2023-10-09T05:51:13.212 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:13 smithi064 ceph-mon[212750]: failed to encode map e111 with expected crc 2023-10-09T05:51:13.212 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:13 smithi064 ceph-mon[212750]: failed to encode map e111 with expected crc 2023-10-09T05:51:13.212 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:13 smithi064 ceph-mon[212750]: failed to encode map e111 with expected crc 2023-10-09T05:51:13.212 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:13 smithi064 ceph-mon[212750]: failed to encode map e111 with expected crc 2023-10-09T05:51:13.214 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:13.361 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 24 16 44465 44449 115.684 90.5625 0.00846584 0.00863965 2023-10-09T05:51:13.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:13 smithi156 ceph-mon[195101]: osdmap e111: 8 total, 8 up, 8 in 2023-10-09T05:51:13.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:13 smithi156 ceph-mon[195101]: failed to encode map e111 with expected crc 2023-10-09T05:51:13.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:13 smithi156 ceph-mon[195101]: failed to encode map e111 with expected crc 2023-10-09T05:51:13.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:13 smithi156 ceph-mon[195101]: failed to encode map e111 with expected crc 2023-10-09T05:51:13.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:13 smithi156 ceph-mon[195101]: failed to encode map e111 with expected crc 2023-10-09T05:51:13.424 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:13.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:13.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:13.536 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:13.582 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:13.659 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:13.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:13.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:13.766 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:13.807 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:13.849 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:13.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:13.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:13.954 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:13.983 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:14.027 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:14.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:14.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:14.131 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:14.178 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:14.253 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:14.257 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:51:14.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:14.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:14.325 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:14 smithi064 ceph-mon[212750]: 1.0 scrub starts 2023-10-09T05:51:14.325 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:14 smithi064 ceph-mon[212750]: 1.0 scrub ok 2023-10-09T05:51:14.325 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:14 smithi064 ceph-mon[212750]: pgmap v251: 99 pgs: 10 unknown, 89 active+clean; 2.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:51:14.325 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:51:14.325 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:51:14.325 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:14 smithi064 ceph-mon[220044]: 1.0 scrub starts 2023-10-09T05:51:14.325 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:14 smithi064 ceph-mon[220044]: 1.0 scrub ok 2023-10-09T05:51:14.325 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:14 smithi064 ceph-mon[220044]: pgmap v251: 99 pgs: 10 unknown, 89 active+clean; 2.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:51:14.325 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:51:14.326 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:51:14.359 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:14.362 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 25 16 45591 45575 113.872 70.375 0.0143264 0.00877699 2023-10-09T05:51:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:14 smithi156 ceph-mon[195101]: 1.0 scrub starts 2023-10-09T05:51:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:14 smithi156 ceph-mon[195101]: 1.0 scrub ok 2023-10-09T05:51:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:14 smithi156 ceph-mon[195101]: pgmap v251: 99 pgs: 10 unknown, 89 active+clean; 2.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:51:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:51:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:51:14.416 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:14.553 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:14.590 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:14.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:14.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:14.697 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:14.730 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:14.804 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:14.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:14.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:14.913 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:14.949 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:15.020 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:15.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:15.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:15.129 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:15.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:15 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-10-09T05:51:15.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:15 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-10-09T05:51:15.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:15 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-10-09T05:51:15.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:15 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-10-09T05:51:15.197 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:15.361 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 26 16 46962 46946 112.788 85.6875 0.00595442 0.00886174 2023-10-09T05:51:15.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:15 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-10-09T05:51:15.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:15 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-10-09T05:51:15.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:51:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:51:15] "GET /metrics HTTP/1.1" 200 36321 "" "Prometheus/2.43.0" 2023-10-09T05:51:16.009 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:16.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:16.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:16.114 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:16.188 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:16.361 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 27 16 48305 48289 111.72 83.9375 0.0201076 0.00894666 2023-10-09T05:51:16.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:16 smithi064 ceph-mon[212750]: pgmap v252: 99 pgs: 99 active+clean; 2.7 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T05:51:16.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:16 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-10-09T05:51:16.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:16 smithi064 ceph-mon[212750]: osdmap e112: 8 total, 8 up, 8 in 2023-10-09T05:51:16.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:16 smithi064 ceph-mon[212750]: failed to encode map e112 with expected crc 2023-10-09T05:51:16.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:16 smithi064 ceph-mon[220044]: pgmap v252: 99 pgs: 99 active+clean; 2.7 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T05:51:16.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:16 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-10-09T05:51:16.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:16 smithi064 ceph-mon[220044]: osdmap e112: 8 total, 8 up, 8 in 2023-10-09T05:51:16.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:16 smithi064 ceph-mon[220044]: failed to encode map e112 with expected crc 2023-10-09T05:51:16.502 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:16.552 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:16.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:16.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:16.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:16 smithi156 ceph-mon[195101]: pgmap v252: 99 pgs: 99 active+clean; 2.7 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T05:51:16.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:16 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-10-09T05:51:16.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:16 smithi156 ceph-mon[195101]: osdmap e112: 8 total, 8 up, 8 in 2023-10-09T05:51:16.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:16 smithi156 ceph-mon[195101]: failed to encode map e112 with expected crc 2023-10-09T05:51:16.672 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:16.732 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:17.151 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:17 smithi064 ceph-mon[212750]: osdmap e113: 8 total, 8 up, 8 in 2023-10-09T05:51:17.151 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:17 smithi064 ceph-mon[212750]: failed to encode map e113 with expected crc 2023-10-09T05:51:17.152 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:17 smithi064 ceph-mon[212750]: failed to encode map e113 with expected crc 2023-10-09T05:51:17.152 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:17 smithi064 ceph-mon[212750]: failed to encode map e113 with expected crc 2023-10-09T05:51:17.152 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:17 smithi064 ceph-mon[212750]: failed to encode map e113 with expected crc 2023-10-09T05:51:17.152 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:17 smithi064 ceph-mon[212750]: failed to encode map e113 with expected crc 2023-10-09T05:51:17.152 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:17 smithi064 ceph-mon[212750]: failed to encode map e113 with expected crc 2023-10-09T05:51:17.152 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:17 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/382256196' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:51:17.152 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:17 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:51:17.185 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T05:51:17.222 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:17.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:17.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:17.290 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:51:17.291 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:51:17.291 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:51:17.291 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd out 1 2023-10-09T05:51:17.330 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:17.361 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 28 16 49705 49689 110.855 87.5 0.0101363 0.00901521 2023-10-09T05:51:17.366 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:17.433 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:17.442 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:17 smithi156 ceph-mon[195101]: osdmap e113: 8 total, 8 up, 8 in 2023-10-09T05:51:17.443 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:17 smithi156 ceph-mon[195101]: failed to encode map e113 with expected crc 2023-10-09T05:51:17.443 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:17 smithi156 ceph-mon[195101]: failed to encode map e113 with expected crc 2023-10-09T05:51:17.443 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:17 smithi156 ceph-mon[195101]: failed to encode map e113 with expected crc 2023-10-09T05:51:17.443 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:17 smithi156 ceph-mon[195101]: failed to encode map e113 with expected crc 2023-10-09T05:51:17.443 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:17 smithi156 ceph-mon[195101]: failed to encode map e113 with expected crc 2023-10-09T05:51:17.443 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:17 smithi156 ceph-mon[195101]: failed to encode map e113 with expected crc 2023-10-09T05:51:17.443 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:17 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/382256196' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:51:17.443 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:17 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:51:17.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:17.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:17.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:17 smithi064 ceph-mon[220044]: osdmap e113: 8 total, 8 up, 8 in 2023-10-09T05:51:17.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:17 smithi064 ceph-mon[220044]: failed to encode map e113 with expected crc 2023-10-09T05:51:17.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:17 smithi064 ceph-mon[220044]: failed to encode map e113 with expected crc 2023-10-09T05:51:17.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:17 smithi064 ceph-mon[220044]: failed to encode map e113 with expected crc 2023-10-09T05:51:17.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:17 smithi064 ceph-mon[220044]: failed to encode map e113 with expected crc 2023-10-09T05:51:17.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:17 smithi064 ceph-mon[220044]: failed to encode map e113 with expected crc 2023-10-09T05:51:17.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:17 smithi064 ceph-mon[220044]: failed to encode map e113 with expected crc 2023-10-09T05:51:17.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:17 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/382256196' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:51:17.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:17 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:51:17.545 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:17.657 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:18.362 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 29 16 50694 50678 109.164 61.8125 0.00257288 0.00913205 2023-10-09T05:51:18.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[212750]: pgmap v255: 99 pgs: 6 remapped+peering, 93 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T05:51:18.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 6 pgs peering (PG_AVAILABILITY) 2023-10-09T05:51:18.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:51:18.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:51:18.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[212750]: osdmap e114: 8 total, 8 up, 8 in 2023-10-09T05:51:18.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[212750]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[212750]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[212750]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[212750]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[212750]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[212750]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[212750]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[212750]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[212750]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[212750]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[220044]: pgmap v255: 99 pgs: 6 remapped+peering, 93 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T05:51:18.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 6 pgs peering (PG_AVAILABILITY) 2023-10-09T05:51:18.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:51:18.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:51:18.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[220044]: osdmap e114: 8 total, 8 up, 8 in 2023-10-09T05:51:18.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[220044]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[220044]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[220044]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[220044]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[220044]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[220044]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[220044]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[220044]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[220044]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:18 smithi064 ceph-mon[220044]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:18 smithi156 ceph-mon[195101]: pgmap v255: 99 pgs: 6 remapped+peering, 93 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T05:51:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:18 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 6 pgs peering (PG_AVAILABILITY) 2023-10-09T05:51:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:18 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:51:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:18 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:51:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:18 smithi156 ceph-mon[195101]: osdmap e114: 8 total, 8 up, 8 in 2023-10-09T05:51:18.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:18 smithi156 ceph-mon[195101]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:18 smithi156 ceph-mon[195101]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:18 smithi156 ceph-mon[195101]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:18 smithi156 ceph-mon[195101]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:18 smithi156 ceph-mon[195101]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:18 smithi156 ceph-mon[195101]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:18 smithi156 ceph-mon[195101]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:18 smithi156 ceph-mon[195101]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:18 smithi156 ceph-mon[195101]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:18 smithi156 ceph-mon[195101]: failed to encode map e114 with expected crc 2023-10-09T05:51:18.717 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:18.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:18.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:18.827 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:18.859 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:18.916 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:18.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:18.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:18.938 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:51:18.939 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T05:51:18.939 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T05:51:18.939 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T05:51:18.939 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.939 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:1475 6.1 6:8bd87812:::benchmark_data_smithi064_284193_object1474:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-10-09T05:51:18.939 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:50:49.851776+0000", 2023-10-09T05:51:18.939 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 29.083453516999999, 2023-10-09T05:51:18.939 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.144418886, 2023-10-09T05:51:18.939 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:18.940 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:18.940 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:18.940 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:18.940 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:18.940 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 1475 2023-10-09T05:51:18.940 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.940 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:18.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.940 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:18.941 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.851776+0000", 2023-10-09T05:51:18.941 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:18.941 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.941 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.941 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:18.941 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.851776+0000", 2023-10-09T05:51:18.941 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:18.942 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.942 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.942 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:18.942 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.851774+0000", 2023-10-09T05:51:18.942 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:51:18.942 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.942 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.942 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:18.943 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.851800+0000", 2023-10-09T05:51:18.943 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5831999999999999e-05 2023-10-09T05:51:18.943 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.943 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.943 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:18.943 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.851810+0000", 2023-10-09T05:51:18.943 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.012e-05 2023-10-09T05:51:18.943 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.943 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.944 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:18.944 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.851817+0000", 2023-10-09T05:51:18.944 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.72e-06 2023-10-09T05:51:18.944 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.944 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.944 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:18.944 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.851861+0000", 2023-10-09T05:51:18.944 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3738e-05 2023-10-09T05:51:18.944 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.945 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.945 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:51:18.945 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.852001+0000", 2023-10-09T05:51:18.945 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014039799999999999 2023-10-09T05:51:18.945 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.945 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.945 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:51:18.945 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.852141+0000", 2023-10-09T05:51:18.945 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000139634 2023-10-09T05:51:18.946 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.946 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.946 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:51:18.946 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.996003+0000", 2023-10-09T05:51:18.946 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14386222600000001 2023-10-09T05:51:18.946 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.946 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.947 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:18.947 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.996052+0000", 2023-10-09T05:51:18.947 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9314999999999998e-05 2023-10-09T05:51:18.947 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.947 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.947 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:18.947 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.996095+0000", 2023-10-09T05:51:18.947 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2447000000000002e-05 2023-10-09T05:51:18.947 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.948 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.948 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:51:18.948 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.996154+0000", 2023-10-09T05:51:18.948 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9209000000000002e-05 2023-10-09T05:51:18.948 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.948 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.948 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:51:18.948 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.996195+0000", 2023-10-09T05:51:18.948 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1065000000000001e-05 2023-10-09T05:51:18.948 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:18.948 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:18.948 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:18.949 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.949 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.949 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:1477 6.1 6:86c2513c:::benchmark_data_smithi064_284193_object1476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-10-09T05:51:18.949 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:50:49.851989+0000", 2023-10-09T05:51:18.949 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 29.083240315000001, 2023-10-09T05:51:18.949 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14470181600000001, 2023-10-09T05:51:18.949 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:18.949 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:18.949 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:18.949 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:18.949 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:18.949 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 1477 2023-10-09T05:51:18.950 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.950 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:18.950 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.950 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:18.950 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.851989+0000", 2023-10-09T05:51:18.950 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:18.950 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.950 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.950 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:18.950 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.851989+0000", 2023-10-09T05:51:18.950 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:18.951 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.951 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.951 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:18.951 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.851988+0000", 2023-10-09T05:51:18.951 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T05:51:18.951 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.951 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.951 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:18.951 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.852014+0000", 2023-10-09T05:51:18.951 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5391000000000001e-05 2023-10-09T05:51:18.951 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.951 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.952 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:18.952 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.852027+0000", 2023-10-09T05:51:18.952 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3028e-05 2023-10-09T05:51:18.952 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.952 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.952 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:18.952 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.852035+0000", 2023-10-09T05:51:18.952 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9119999999999998e-06 2023-10-09T05:51:18.952 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.952 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.952 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:18.952 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.995445+0000", 2023-10-09T05:51:18.953 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14341036400000001 2023-10-09T05:51:18.953 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.953 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.953 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:51:18.953 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.995560+0000", 2023-10-09T05:51:18.953 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000115152 2023-10-09T05:51:18.953 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.953 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.953 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:51:18.953 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.995739+0000", 2023-10-09T05:51:18.953 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000178417 2023-10-09T05:51:18.953 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.953 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.954 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:18.954 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.996501+0000", 2023-10-09T05:51:18.954 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00076291100000000001 2023-10-09T05:51:18.954 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.954 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.954 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:51:18.954 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.996518+0000", 2023-10-09T05:51:18.954 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6867999999999998e-05 2023-10-09T05:51:18.954 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.954 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.954 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:18.955 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.996606+0000", 2023-10-09T05:51:18.955 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7291000000000002e-05 2023-10-09T05:51:18.955 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.955 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.955 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:51:18.955 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.996654+0000", 2023-10-09T05:51:18.955 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8726000000000001e-05 2023-10-09T05:51:18.955 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.955 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.955 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:51:18.955 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.996691+0000", 2023-10-09T05:51:18.955 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6801000000000002e-05 2023-10-09T05:51:18.956 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:18.956 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:18.956 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:18.956 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.956 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.956 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.44588.0:1474 6.e e106/102 6:7363f9e5:::benchmark_data_smithi064_284193_object1473:head v 106'100, mlcod=106'98)", 2023-10-09T05:51:18.956 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:50:49.852278+0000", 2023-10-09T05:51:18.956 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 29.082951225999999, 2023-10-09T05:51:18.956 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14364628400000001, 2023-10-09T05:51:18.956 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:18.956 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:18.956 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:18.957 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.957 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:18.957 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.852278+0000", 2023-10-09T05:51:18.957 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:18.957 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.957 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.957 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:18.957 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.852278+0000", 2023-10-09T05:51:18.957 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:18.957 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.958 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.958 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:18.958 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.852269+0000", 2023-10-09T05:51:18.958 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999909 2023-10-09T05:51:18.958 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.958 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.958 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:18.958 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.852341+0000", 2023-10-09T05:51:18.958 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1391999999999997e-05 2023-10-09T05:51:18.958 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.958 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.958 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:18.959 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.994976+0000", 2023-10-09T05:51:18.959 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14263563300000001 2023-10-09T05:51:18.959 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.959 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.959 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:18.959 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.994989+0000", 2023-10-09T05:51:18.959 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2454e-05 2023-10-09T05:51:18.959 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.959 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.959 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:18.959 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.995038+0000", 2023-10-09T05:51:18.959 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8588999999999998e-05 2023-10-09T05:51:18.960 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.960 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.960 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:51:18.960 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.995065+0000", 2023-10-09T05:51:18.960 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7494000000000001e-05 2023-10-09T05:51:18.960 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.960 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.960 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:51:18.960 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.995892+0000", 2023-10-09T05:51:18.960 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00082715700000000002 2023-10-09T05:51:18.960 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.960 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.961 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:51:18.961 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.995925+0000", 2023-10-09T05:51:18.961 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2508000000000001e-05 2023-10-09T05:51:18.961 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:18.961 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:18.961 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:18.961 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.961 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.961 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:1482 6.4 6:2c21a7cb:::benchmark_data_smithi064_284193_object1481:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-10-09T05:51:18.961 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:50:49.853577+0000", 2023-10-09T05:51:18.961 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 29.081652523999999, 2023-10-09T05:51:18.961 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14290936100000001, 2023-10-09T05:51:18.962 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:18.962 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:18.962 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:18.962 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:18.962 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:18.962 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 1482 2023-10-09T05:51:18.962 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.962 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:18.962 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.962 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:18.962 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.853577+0000", 2023-10-09T05:51:18.963 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:18.963 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.963 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.963 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:18.963 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.853577+0000", 2023-10-09T05:51:18.963 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:18.963 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.963 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.963 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:18.963 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.853575+0000", 2023-10-09T05:51:18.963 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:51:18.963 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.964 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.964 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:18.964 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.853603+0000", 2023-10-09T05:51:18.964 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7888000000000001e-05 2023-10-09T05:51:18.964 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.964 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.964 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:18.964 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.994980+0000", 2023-10-09T05:51:18.964 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14137707399999999 2023-10-09T05:51:18.964 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.964 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.964 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:18.965 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.994996+0000", 2023-10-09T05:51:18.965 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5410999999999999e-05 2023-10-09T05:51:18.965 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.965 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.965 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:18.965 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.995052+0000", 2023-10-09T05:51:18.965 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5856000000000002e-05 2023-10-09T05:51:18.965 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.965 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.965 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:51:18.965 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.995227+0000", 2023-10-09T05:51:18.965 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017571000000000001 2023-10-09T05:51:18.965 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.966 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.966 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:51:18.966 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.995395+0000", 2023-10-09T05:51:18.966 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016755200000000001 2023-10-09T05:51:18.966 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.966 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.966 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:51:18.966 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.995935+0000", 2023-10-09T05:51:18.966 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00054027800000000005 2023-10-09T05:51:18.966 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.966 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.967 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:18.967 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.996366+0000", 2023-10-09T05:51:18.967 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00043044200000000001 2023-10-09T05:51:18.967 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.967 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.967 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:18.967 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.996409+0000", 2023-10-09T05:51:18.967 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3115999999999999e-05 2023-10-09T05:51:18.967 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.968 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.968 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:51:18.968 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.996448+0000", 2023-10-09T05:51:18.968 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9304000000000002e-05 2023-10-09T05:51:18.968 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.968 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.968 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:51:18.968 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:49.996486+0000", 2023-10-09T05:51:18.968 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8331999999999998e-05 2023-10-09T05:51:18.968 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:18.968 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:18.969 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:18.969 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.969 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.969 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:1862 6.4 6:2de52d73:::benchmark_data_smithi064_284193_object1861:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-10-09T05:51:18.969 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:50:50.172073+0000", 2023-10-09T05:51:18.969 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 28.763157107000001, 2023-10-09T05:51:18.969 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.140519529, 2023-10-09T05:51:18.969 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:18.969 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:18.970 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:18.970 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:18.970 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:18.970 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 1862 2023-10-09T05:51:18.970 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.970 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:18.970 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.970 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:18.970 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:50.172073+0000", 2023-10-09T05:51:18.970 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:18.971 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.971 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.971 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:18.971 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:50.172073+0000", 2023-10-09T05:51:18.971 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:18.971 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.971 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.971 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:18.971 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:50.172072+0000", 2023-10-09T05:51:18.971 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T05:51:18.971 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.972 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.972 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:18.972 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:50.172093+0000", 2023-10-09T05:51:18.972 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1673000000000001e-05 2023-10-09T05:51:18.972 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.972 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.972 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:18.972 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:50.172105+0000", 2023-10-09T05:51:18.972 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1549000000000001e-05 2023-10-09T05:51:18.972 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.973 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.973 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:18.973 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:50.172113+0000", 2023-10-09T05:51:18.973 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6000000000000001e-06 2023-10-09T05:51:18.973 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.973 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.973 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:18.973 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:50.172221+0000", 2023-10-09T05:51:18.973 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108912 2023-10-09T05:51:18.973 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.974 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.974 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:51:18.974 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:50.172349+0000", 2023-10-09T05:51:18.974 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000127205 2023-10-09T05:51:18.974 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.974 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.974 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:51:18.974 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:50.172481+0000", 2023-10-09T05:51:18.974 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013274700000000001 2023-10-09T05:51:18.975 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.975 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.975 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:51:18.975 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:50.172945+0000", 2023-10-09T05:51:18.975 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00046408499999999999 2023-10-09T05:51:18.975 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.975 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.975 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:18.975 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:50.174113+0000", 2023-10-09T05:51:18.975 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001167773 2023-10-09T05:51:18.976 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.976 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.976 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:18.976 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:50.312496+0000", 2023-10-09T05:51:18.976 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.138382598 2023-10-09T05:51:18.976 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.976 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.976 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:51:18.976 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:50.312541+0000", 2023-10-09T05:51:18.976 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5272000000000003e-05 2023-10-09T05:51:18.977 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.977 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.977 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:51:18.977 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:50.312592+0000", 2023-10-09T05:51:18.977 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0918999999999998e-05 2023-10-09T05:51:18.977 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:18.977 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:18.977 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:18.978 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.978 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.978 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:10248 6.1 6:8e3b0a79:::benchmark_data_smithi064_284193_object10247:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e107)", 2023-10-09T05:51:18.978 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:50:53.461561+0000", 2023-10-09T05:51:18.978 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 25.473668272000001, 2023-10-09T05:51:18.978 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15830158699999999, 2023-10-09T05:51:18.978 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:18.978 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:18.979 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:18.979 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:18.979 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:18.979 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 10248 2023-10-09T05:51:18.979 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.979 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:18.980 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.980 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:18.980 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.461561+0000", 2023-10-09T05:51:18.980 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:18.980 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.980 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.980 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:18.980 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.461561+0000", 2023-10-09T05:51:18.980 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:18.980 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.980 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.981 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:18.981 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.461559+0000", 2023-10-09T05:51:18.981 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:51:18.981 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.981 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.981 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:18.981 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.461590+0000", 2023-10-09T05:51:18.981 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0920000000000002e-05 2023-10-09T05:51:18.981 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.981 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.981 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:18.982 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.461610+0000", 2023-10-09T05:51:18.982 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9854e-05 2023-10-09T05:51:18.982 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.982 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.982 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:18.982 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.461625+0000", 2023-10-09T05:51:18.982 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4922e-05 2023-10-09T05:51:18.982 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.982 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.982 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:18.982 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.461708+0000", 2023-10-09T05:51:18.983 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3436999999999994e-05 2023-10-09T05:51:18.983 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.983 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.983 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:51:18.983 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.461863+0000", 2023-10-09T05:51:18.983 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015487000000000001 2023-10-09T05:51:18.983 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.983 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.983 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:51:18.983 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.462029+0000", 2023-10-09T05:51:18.983 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000166117 2023-10-09T05:51:18.984 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.984 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.984 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:51:18.984 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.463227+0000", 2023-10-09T05:51:18.984 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001197481 2023-10-09T05:51:18.984 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.984 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.984 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:18.984 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.465087+0000", 2023-10-09T05:51:18.984 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001860064 2023-10-09T05:51:18.984 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.984 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.985 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:18.985 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.619761+0000", 2023-10-09T05:51:18.985 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15467361499999999 2023-10-09T05:51:18.985 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.985 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.985 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:51:18.985 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.619818+0000", 2023-10-09T05:51:18.985 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7052999999999997e-05 2023-10-09T05:51:18.985 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.985 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.985 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:51:18.985 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.619863+0000", 2023-10-09T05:51:18.986 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5272999999999998e-05 2023-10-09T05:51:18.986 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:18.986 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:18.986 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:18.986 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.986 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.986 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:10251 6.1 6:87e18e8b:::benchmark_data_smithi064_284193_object10250:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e107)", 2023-10-09T05:51:18.986 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:50:53.463262+0000", 2023-10-09T05:51:18.986 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 25.471967312, 2023-10-09T05:51:18.986 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.157190407, 2023-10-09T05:51:18.987 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:18.987 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:18.987 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:18.987 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:18.987 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:18.987 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 10251 2023-10-09T05:51:18.987 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.987 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:18.987 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.987 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:18.988 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.463262+0000", 2023-10-09T05:51:18.988 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:18.988 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.988 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.988 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:18.988 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.463262+0000", 2023-10-09T05:51:18.988 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:18.988 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.988 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.988 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:18.988 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.463261+0000", 2023-10-09T05:51:18.988 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:51:18.989 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.989 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.989 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:18.989 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.463285+0000", 2023-10-09T05:51:18.989 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4036000000000002e-05 2023-10-09T05:51:18.989 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.989 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.989 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:18.989 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.463298+0000", 2023-10-09T05:51:18.989 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2621e-05 2023-10-09T05:51:18.989 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.989 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.990 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:18.990 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.463306+0000", 2023-10-09T05:51:18.990 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9719999999999992e-06 2023-10-09T05:51:18.990 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.990 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.990 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:18.990 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.463342+0000", 2023-10-09T05:51:18.990 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6817999999999999e-05 2023-10-09T05:51:18.990 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.990 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.990 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:51:18.990 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.463491+0000", 2023-10-09T05:51:18.991 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014854399999999999 2023-10-09T05:51:18.991 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.991 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.991 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:51:18.991 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.463644+0000", 2023-10-09T05:51:18.991 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000152798 2023-10-09T05:51:18.991 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.991 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.991 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:51:18.991 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.465118+0000", 2023-10-09T05:51:18.991 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014739460000000001 2023-10-09T05:51:18.992 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.992 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.992 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:18.992 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.466361+0000", 2023-10-09T05:51:18.992 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0012430049999999999 2023-10-09T05:51:18.992 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.992 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.992 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:18.992 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.620373+0000", 2023-10-09T05:51:18.992 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15401226900000001 2023-10-09T05:51:18.992 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.993 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.993 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:51:18.993 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.620418+0000", 2023-10-09T05:51:18.993 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5101000000000001e-05 2023-10-09T05:51:18.993 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.993 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.993 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:51:18.993 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.620453+0000", 2023-10-09T05:51:18.993 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4626999999999999e-05 2023-10-09T05:51:18.993 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:18.993 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:18.993 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:18.994 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.994 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.994 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:10253 6.1 6:8bf45efa:::benchmark_data_smithi064_284193_object10252:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e107)", 2023-10-09T05:51:18.994 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:50:53.463388+0000", 2023-10-09T05:51:18.994 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 25.47184171, 2023-10-09T05:51:18.994 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.157181346, 2023-10-09T05:51:18.994 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:18.994 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:18.994 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:18.994 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:18.994 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:18.995 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 10253 2023-10-09T05:51:18.995 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.995 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:18.995 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.995 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:18.995 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.463388+0000", 2023-10-09T05:51:18.995 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:18.995 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.995 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.995 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:18.995 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.463388+0000", 2023-10-09T05:51:18.995 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:18.996 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.996 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.996 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:18.996 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.463387+0000", 2023-10-09T05:51:18.996 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T05:51:18.996 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.996 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.996 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:18.996 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.463412+0000", 2023-10-09T05:51:18.996 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4539999999999999e-05 2023-10-09T05:51:18.996 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.996 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.996 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:18.997 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.463423+0000", 2023-10-09T05:51:18.997 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1375999999999999e-05 2023-10-09T05:51:18.997 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.997 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.997 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:18.997 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.463431+0000", 2023-10-09T05:51:18.997 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2660000000000001e-06 2023-10-09T05:51:18.997 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.997 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.997 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:18.997 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.464239+0000", 2023-10-09T05:51:18.997 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00080781799999999997 2023-10-09T05:51:18.998 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.998 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.998 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:51:18.998 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.464376+0000", 2023-10-09T05:51:18.998 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000137095 2023-10-09T05:51:18.998 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.998 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.998 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:51:18.998 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.464517+0000", 2023-10-09T05:51:18.998 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014058000000000001 2023-10-09T05:51:18.998 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.998 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.999 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:51:18.999 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.465858+0000", 2023-10-09T05:51:18.999 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001341091 2023-10-09T05:51:18.999 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.999 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.999 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:18.999 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.467158+0000", 2023-10-09T05:51:18.999 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013000049999999999 2023-10-09T05:51:18.999 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:18.999 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:18.999 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:18.999 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.620494+0000", 2023-10-09T05:51:18.999 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15333635400000001 2023-10-09T05:51:19.000 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.000 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.000 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:51:19.000 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.620536+0000", 2023-10-09T05:51:19.000 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1857999999999998e-05 2023-10-09T05:51:19.000 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.000 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.000 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:51:19.000 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:50:53.620569+0000", 2023-10-09T05:51:19.000 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3272999999999998e-05 2023-10-09T05:51:19.000 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.000 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:19.001 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.001 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.001 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.001 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:41353 6.1 6:8e79ac5a:::benchmark_data_smithi064_284193_object41352:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T05:51:19.001 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:10.983434+0000", 2023-10-09T05:51:19.001 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 7.9517959249999999, 2023-10-09T05:51:19.001 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15601717500000001, 2023-10-09T05:51:19.001 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:19.001 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:19.001 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:19.002 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:19.002 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:19.002 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 41353 2023-10-09T05:51:19.002 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.002 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:19.002 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.002 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:19.002 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:10.983434+0000", 2023-10-09T05:51:19.002 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:19.002 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.003 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.003 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:19.003 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:10.983434+0000", 2023-10-09T05:51:19.003 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:19.003 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.003 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.003 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:19.003 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:10.983432+0000", 2023-10-09T05:51:19.003 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:51:19.003 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.003 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.004 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:19.004 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:10.983455+0000", 2023-10-09T05:51:19.004 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2989e-05 2023-10-09T05:51:19.004 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.004 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.004 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:19.004 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:10.983476+0000", 2023-10-09T05:51:19.004 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.05e-05 2023-10-09T05:51:19.004 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.004 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.005 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:19.005 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:10.983482+0000", 2023-10-09T05:51:19.005 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0639999999999999e-06 2023-10-09T05:51:19.005 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.005 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.005 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:19.005 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:10.983527+0000", 2023-10-09T05:51:19.005 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4820000000000001e-05 2023-10-09T05:51:19.005 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.005 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.005 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:51:19.005 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:10.983660+0000", 2023-10-09T05:51:19.006 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013310500000000001 2023-10-09T05:51:19.006 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.006 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.006 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:51:19.006 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:10.983837+0000", 2023-10-09T05:51:19.006 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017731800000000001 2023-10-09T05:51:19.006 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.006 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.006 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:51:19.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:10.985404+0000", 2023-10-09T05:51:19.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00156639 2023-10-09T05:51:19.007 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.007 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.007 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:19.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:10.985480+0000", 2023-10-09T05:51:19.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6297999999999994e-05 2023-10-09T05:51:19.007 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.007 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.007 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:19.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:11.139356+0000", 2023-10-09T05:51:19.008 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15387646199999999 2023-10-09T05:51:19.008 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.008 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.008 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:51:19.008 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:11.139410+0000", 2023-10-09T05:51:19.008 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4092000000000003e-05 2023-10-09T05:51:19.008 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.008 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.008 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:51:19.009 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:11.139451+0000", 2023-10-09T05:51:19.009 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0420000000000003e-05 2023-10-09T05:51:19.009 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.009 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:19.009 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.009 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.009 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.009 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:45186 6.4 6:2fac37d8:::benchmark_data_smithi064_284193_object45185:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T05:51:19.009 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:13.801568+0000", 2023-10-09T05:51:19.010 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 5.1336618820000002, 2023-10-09T05:51:19.010 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33509184800000003, 2023-10-09T05:51:19.010 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:19.010 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:19.010 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:19.010 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:19.010 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:19.010 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 45186 2023-10-09T05:51:19.011 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.011 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:19.011 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.011 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:19.011 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.801568+0000", 2023-10-09T05:51:19.011 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:19.011 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.011 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.011 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:19.012 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.801568+0000", 2023-10-09T05:51:19.012 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:19.012 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.012 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.012 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:19.012 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.801565+0000", 2023-10-09T05:51:19.012 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:51:19.012 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.012 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.013 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:19.013 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.801605+0000", 2023-10-09T05:51:19.013 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9807e-05 2023-10-09T05:51:19.013 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.013 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.013 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:19.013 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.801629+0000", 2023-10-09T05:51:19.013 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3261e-05 2023-10-09T05:51:19.013 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.014 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.014 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:19.014 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.801638+0000", 2023-10-09T05:51:19.014 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2750000000000001e-06 2023-10-09T05:51:19.014 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.014 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.014 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:19.014 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.801686+0000", 2023-10-09T05:51:19.014 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8412000000000003e-05 2023-10-09T05:51:19.014 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.014 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.015 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:51:19.015 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.801823+0000", 2023-10-09T05:51:19.015 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013631599999999999 2023-10-09T05:51:19.015 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.015 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.015 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:51:19.015 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.801982+0000", 2023-10-09T05:51:19.015 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000159877 2023-10-09T05:51:19.015 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.015 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.015 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:51:19.015 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.804641+0000", 2023-10-09T05:51:19.015 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026580750000000002 2023-10-09T05:51:19.016 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.016 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:19.016 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.809742+0000", 2023-10-09T05:51:19.016 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051013059999999999 2023-10-09T05:51:19.016 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.016 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:19.016 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:14.136563+0000", 2023-10-09T05:51:19.016 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32682160599999999 2023-10-09T05:51:19.016 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.017 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:51:19.017 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:14.136613+0000", 2023-10-09T05:51:19.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9347e-05 2023-10-09T05:51:19.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.017 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.017 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:51:19.017 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:14.136660+0000", 2023-10-09T05:51:19.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6795000000000002e-05 2023-10-09T05:51:19.017 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.017 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:19.017 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.018 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.018 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:45187 6.4 6:2d200818:::benchmark_data_smithi064_284193_object45186:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T05:51:19.018 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:13.801653+0000", 2023-10-09T05:51:19.018 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 5.1335763940000003, 2023-10-09T05:51:19.018 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33823780799999997, 2023-10-09T05:51:19.018 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:19.018 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:19.018 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:19.018 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:19.018 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:19.018 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 45187 2023-10-09T05:51:19.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.019 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:19.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.019 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:19.019 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.801653+0000", 2023-10-09T05:51:19.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:19.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.019 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:19.019 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.801653+0000", 2023-10-09T05:51:19.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:19.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:19.020 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.801653+0000", 2023-10-09T05:51:19.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:51:19.020 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:19.020 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.801671+0000", 2023-10-09T05:51:19.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8698999999999998e-05 2023-10-09T05:51:19.020 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:19.021 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.801679+0000", 2023-10-09T05:51:19.021 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7989999999999994e-06 2023-10-09T05:51:19.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.021 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.021 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:19.021 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.801685+0000", 2023-10-09T05:51:19.021 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7760000000000003e-06 2023-10-09T05:51:19.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.021 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.021 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:19.022 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.803198+0000", 2023-10-09T05:51:19.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001513286 2023-10-09T05:51:19.022 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.022 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.022 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:51:19.022 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.803330+0000", 2023-10-09T05:51:19.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000131545 2023-10-09T05:51:19.022 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.022 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.022 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:51:19.022 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.803462+0000", 2023-10-09T05:51:19.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013186799999999999 2023-10-09T05:51:19.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:51:19.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.804648+0000", 2023-10-09T05:51:19.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001185979 2023-10-09T05:51:19.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:19.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.809781+0000", 2023-10-09T05:51:19.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051335570000000004 2023-10-09T05:51:19.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:19.024 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:14.139801+0000", 2023-10-09T05:51:19.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33001976100000002 2023-10-09T05:51:19.024 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.024 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.024 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:51:19.024 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:14.139851+0000", 2023-10-09T05:51:19.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0086000000000001e-05 2023-10-09T05:51:19.024 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.024 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.024 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:51:19.024 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:14.139891+0000", 2023-10-09T05:51:19.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0040999999999998e-05 2023-10-09T05:51:19.025 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.025 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:19.025 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.025 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:45193 6.4 6:25fcc336:::benchmark_data_smithi064_284193_object45192:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T05:51:19.025 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:13.806348+0000", 2023-10-09T05:51:19.025 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 5.1288815019999996, 2023-10-09T05:51:19.025 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33364052399999999, 2023-10-09T05:51:19.025 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:19.025 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:19.025 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:19.026 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:19.026 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:19.026 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 45193 2023-10-09T05:51:19.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.026 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:19.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.026 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:19.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.806348+0000", 2023-10-09T05:51:19.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:19.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:19.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.806348+0000", 2023-10-09T05:51:19.027 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:19.027 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:19.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.806347+0000", 2023-10-09T05:51:19.027 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T05:51:19.027 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:19.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.806368+0000", 2023-10-09T05:51:19.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0767e-05 2023-10-09T05:51:19.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:19.028 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.806381+0000", 2023-10-09T05:51:19.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.322e-05 2023-10-09T05:51:19.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:19.028 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.806387+0000", 2023-10-09T05:51:19.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1279999999999996e-06 2023-10-09T05:51:19.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.029 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:19.029 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.806424+0000", 2023-10-09T05:51:19.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7311999999999998e-05 2023-10-09T05:51:19.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.029 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:51:19.029 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.806519+0000", 2023-10-09T05:51:19.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4040000000000001e-05 2023-10-09T05:51:19.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.029 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:51:19.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.806650+0000", 2023-10-09T05:51:19.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013160599999999999 2023-10-09T05:51:19.030 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.030 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:51:19.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.808124+0000", 2023-10-09T05:51:19.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014742690000000001 2023-10-09T05:51:19.030 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.030 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:19.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.809822+0000", 2023-10-09T05:51:19.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001697624 2023-10-09T05:51:19.031 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.031 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.031 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:19.031 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:14.139925+0000", 2023-10-09T05:51:19.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.330102648 2023-10-09T05:51:19.031 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.031 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.031 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:51:19.031 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:14.139960+0000", 2023-10-09T05:51:19.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5385000000000001e-05 2023-10-09T05:51:19.032 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.032 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.032 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:51:19.032 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:14.139989+0000", 2023-10-09T05:51:19.032 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8561e-05 2023-10-09T05:51:19.032 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.032 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:19.032 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.032 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.032 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.032 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:47819 6.1 6:8ddabdc6:::benchmark_data_smithi064_284193_object47818:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T05:51:19.032 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:15.890889+0000", 2023-10-09T05:51:19.033 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 3.0443403199999999, 2023-10-09T05:51:19.033 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.17175396100000001, 2023-10-09T05:51:19.033 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:19.033 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:19.033 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:19.033 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:19.033 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:19.033 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 47819 2023-10-09T05:51:19.033 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.033 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:19.034 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.034 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:19.034 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.890889+0000", 2023-10-09T05:51:19.034 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:19.034 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.034 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.034 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:19.034 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.890889+0000", 2023-10-09T05:51:19.034 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:19.034 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.035 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:19.035 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.890889+0000", 2023-10-09T05:51:19.035 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:51:19.035 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.035 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:19.035 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.890908+0000", 2023-10-09T05:51:19.035 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9432999999999998e-05 2023-10-09T05:51:19.035 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.035 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:19.036 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.890915+0000", 2023-10-09T05:51:19.036 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5189999999999998e-06 2023-10-09T05:51:19.036 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.036 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.036 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:19.036 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.890920+0000", 2023-10-09T05:51:19.036 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8899999999999998e-06 2023-10-09T05:51:19.036 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.036 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.037 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:19.037 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.894765+0000", 2023-10-09T05:51:19.037 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038448520000000002 2023-10-09T05:51:19.037 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.037 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.037 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:51:19.037 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.894892+0000", 2023-10-09T05:51:19.037 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012762900000000001 2023-10-09T05:51:19.037 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.038 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.038 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:51:19.038 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.894995+0000", 2023-10-09T05:51:19.038 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000102849 2023-10-09T05:51:19.038 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.038 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.038 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:51:19.038 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.896333+0000", 2023-10-09T05:51:19.038 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001337932 2023-10-09T05:51:19.038 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.039 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.039 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:19.039 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.896517+0000", 2023-10-09T05:51:19.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000183936 2023-10-09T05:51:19.039 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.039 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.039 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:19.039 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.062556+0000", 2023-10-09T05:51:19.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16603905199999999 2023-10-09T05:51:19.039 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.040 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.040 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:51:19.040 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.062600+0000", 2023-10-09T05:51:19.040 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4252000000000003e-05 2023-10-09T05:51:19.040 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.040 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.040 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:51:19.040 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.062643+0000", 2023-10-09T05:51:19.040 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3074000000000001e-05 2023-10-09T05:51:19.040 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.040 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:19.041 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.041 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.041 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.041 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:47822 6.4 6:2222fea0:::benchmark_data_smithi064_284193_object47821:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T05:51:19.041 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:15.893824+0000", 2023-10-09T05:51:19.041 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 3.041405374, 2023-10-09T05:51:19.041 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16793796999999999, 2023-10-09T05:51:19.041 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:19.041 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:19.042 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:19.042 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:19.042 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:19.042 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 47822 2023-10-09T05:51:19.042 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.042 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:19.042 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.042 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:19.042 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.893824+0000", 2023-10-09T05:51:19.043 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:19.043 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.043 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:19.043 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.893824+0000", 2023-10-09T05:51:19.043 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:19.043 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.043 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:19.043 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.893822+0000", 2023-10-09T05:51:19.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:51:19.044 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.044 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.044 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:19.044 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.893848+0000", 2023-10-09T05:51:19.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5219999999999999e-05 2023-10-09T05:51:19.044 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.044 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.045 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:19.045 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.893861+0000", 2023-10-09T05:51:19.045 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3256e-05 2023-10-09T05:51:19.045 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.045 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.045 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:19.045 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.893869+0000", 2023-10-09T05:51:19.045 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3350000000000007e-06 2023-10-09T05:51:19.045 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.046 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.046 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:19.046 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.893921+0000", 2023-10-09T05:51:19.046 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1416999999999999e-05 2023-10-09T05:51:19.046 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.046 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.046 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:51:19.046 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.894039+0000", 2023-10-09T05:51:19.046 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118465 2023-10-09T05:51:19.046 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.046 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.046 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:51:19.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.894167+0000", 2023-10-09T05:51:19.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0001281 2023-10-09T05:51:19.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:51:19.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.895878+0000", 2023-10-09T05:51:19.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0017110879999999999 2023-10-09T05:51:19.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:19.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.896415+0000", 2023-10-09T05:51:19.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00053715900000000005 2023-10-09T05:51:19.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.048 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:19.048 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.061675+0000", 2023-10-09T05:51:19.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16525967599999999 2023-10-09T05:51:19.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.048 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:51:19.048 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.061725+0000", 2023-10-09T05:51:19.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9427e-05 2023-10-09T05:51:19.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.049 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:51:19.049 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.061762+0000", 2023-10-09T05:51:19.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7697999999999997e-05 2023-10-09T05:51:19.049 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.049 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:19.049 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.049 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.049 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.049 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:47823 6.1 6:806eba3c:::benchmark_data_smithi064_284193_object47822:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T05:51:19.049 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:15.896757+0000", 2023-10-09T05:51:19.049 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 3.0384728509999999, 2023-10-09T05:51:19.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16603305500000001, 2023-10-09T05:51:19.050 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:19.050 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:19.050 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:19.050 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:19.050 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:19.050 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 47823 2023-10-09T05:51:19.050 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.050 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:19.050 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.050 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:19.050 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.896757+0000", 2023-10-09T05:51:19.050 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:19.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.051 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:19.051 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.896757+0000", 2023-10-09T05:51:19.051 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:19.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.051 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:19.051 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.896755+0000", 2023-10-09T05:51:19.051 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:51:19.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.052 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:19.052 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.896798+0000", 2023-10-09T05:51:19.052 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2740000000000001e-05 2023-10-09T05:51:19.052 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.052 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.052 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:19.052 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.896810+0000", 2023-10-09T05:51:19.052 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1623e-05 2023-10-09T05:51:19.052 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.052 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.052 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:19.053 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.896818+0000", 2023-10-09T05:51:19.053 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9529999999999999e-06 2023-10-09T05:51:19.053 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.053 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.053 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:19.053 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.896867+0000", 2023-10-09T05:51:19.053 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9620999999999998e-05 2023-10-09T05:51:19.053 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.053 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.053 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:51:19.053 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.897001+0000", 2023-10-09T05:51:19.053 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000133275 2023-10-09T05:51:19.053 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.054 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.054 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:51:19.054 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.897098+0000", 2023-10-09T05:51:19.054 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7448999999999995e-05 2023-10-09T05:51:19.054 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.054 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.054 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:19.054 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.062522+0000", 2023-10-09T05:51:19.054 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.165423821 2023-10-09T05:51:19.054 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.054 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.054 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:51:19.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.062668+0000", 2023-10-09T05:51:19.055 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014595100000000001 2023-10-09T05:51:19.055 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.055 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.055 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:19.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.062715+0000", 2023-10-09T05:51:19.055 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7212999999999997e-05 2023-10-09T05:51:19.055 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.055 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.055 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:51:19.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.062755+0000", 2023-10-09T05:51:19.056 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0550999999999999e-05 2023-10-09T05:51:19.056 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.056 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.056 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:51:19.056 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.062790+0000", 2023-10-09T05:51:19.056 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4323999999999999e-05 2023-10-09T05:51:19.056 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.056 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:19.056 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.056 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.056 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.056 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:47825 6.4 6:27f88302:::benchmark_data_smithi064_284193_object47824:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T05:51:19.057 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:15.896831+0000", 2023-10-09T05:51:19.057 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 3.0383982139999999, 2023-10-09T05:51:19.057 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.165737829, 2023-10-09T05:51:19.057 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:19.057 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:19.057 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:19.057 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:19.057 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:19.057 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 47825 2023-10-09T05:51:19.057 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.057 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:19.058 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.058 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:19.058 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.896831+0000", 2023-10-09T05:51:19.058 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:19.058 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.058 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.058 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:19.058 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.896831+0000", 2023-10-09T05:51:19.058 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:19.058 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.058 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.058 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:19.059 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.896831+0000", 2023-10-09T05:51:19.059 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:51:19.059 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.059 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.059 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:19.059 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.896863+0000", 2023-10-09T05:51:19.059 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1890999999999997e-05 2023-10-09T05:51:19.059 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.059 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.059 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:19.059 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.896873+0000", 2023-10-09T05:51:19.060 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0118999999999999e-05 2023-10-09T05:51:19.060 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.060 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.060 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:19.060 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.896879+0000", 2023-10-09T05:51:19.060 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2099999999999998e-06 2023-10-09T05:51:19.060 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.060 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.060 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:19.060 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.896930+0000", 2023-10-09T05:51:19.060 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0481e-05 2023-10-09T05:51:19.060 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.061 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.061 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:51:19.061 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.897059+0000", 2023-10-09T05:51:19.061 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012962299999999999 2023-10-09T05:51:19.061 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.061 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.061 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:51:19.061 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.897203+0000", 2023-10-09T05:51:19.061 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014381999999999999 2023-10-09T05:51:19.061 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.061 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.061 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:19.062 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.060766+0000", 2023-10-09T05:51:19.062 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.163563135 2023-10-09T05:51:19.062 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.062 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.062 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:19.062 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.061795+0000", 2023-10-09T05:51:19.062 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001029203 2023-10-09T05:51:19.062 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.062 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.062 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:51:19.062 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.062472+0000", 2023-10-09T05:51:19.063 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00067720499999999995 2023-10-09T05:51:19.063 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.063 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.063 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:51:19.063 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.062532+0000", 2023-10-09T05:51:19.063 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9038e-05 2023-10-09T05:51:19.063 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.063 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.063 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:51:19.063 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.062569+0000", 2023-10-09T05:51:19.063 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7691000000000002e-05 2023-10-09T05:51:19.063 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.064 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:19.064 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.064 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.064 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.064 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:47829 6.1 6:85d0c3c4:::benchmark_data_smithi064_284193_object47828:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T05:51:19.064 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:15.896895+0000", 2023-10-09T05:51:19.064 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 3.0383343219999999, 2023-10-09T05:51:19.064 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.167576962, 2023-10-09T05:51:19.064 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:19.064 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:19.064 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:19.064 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:19.065 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:19.065 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 47829 2023-10-09T05:51:19.065 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.065 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:19.065 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.065 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:19.065 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.896895+0000", 2023-10-09T05:51:19.065 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:19.065 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.065 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.065 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:19.065 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.896895+0000", 2023-10-09T05:51:19.066 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:19.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:19.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.896895+0000", 2023-10-09T05:51:19.066 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:51:19.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:19.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.896914+0000", 2023-10-09T05:51:19.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9267999999999999e-05 2023-10-09T05:51:19.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:19.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.896921+0000", 2023-10-09T05:51:19.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0620000000000003e-06 2023-10-09T05:51:19.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:19.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.896926+0000", 2023-10-09T05:51:19.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2209999999999996e-06 2023-10-09T05:51:19.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.068 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:19.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.898453+0000", 2023-10-09T05:51:19.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0015271460000000001 2023-10-09T05:51:19.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.068 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:51:19.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.898590+0000", 2023-10-09T05:51:19.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013694400000000001 2023-10-09T05:51:19.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.069 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:51:19.069 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.060796+0000", 2023-10-09T05:51:19.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16220544100000001 2023-10-09T05:51:19.069 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.069 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:19.069 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.064332+0000", 2023-10-09T05:51:19.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035363410000000001 2023-10-09T05:51:19.069 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.069 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:19.069 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.064371+0000", 2023-10-09T05:51:19.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8943000000000003e-05 2023-10-09T05:51:19.070 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.070 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:51:19.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.064382+0000", 2023-10-09T05:51:19.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0566e-05 2023-10-09T05:51:19.070 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.070 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:51:19.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.064436+0000", 2023-10-09T05:51:19.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4806999999999999e-05 2023-10-09T05:51:19.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.071 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:51:19.071 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.064472+0000", 2023-10-09T05:51:19.071 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5791999999999999e-05 2023-10-09T05:51:19.071 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.071 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:19.071 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.072 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.072 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.072 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.44588.0:47823 6.1 e112/102 ondisk, result = 0)", 2023-10-09T05:51:19.072 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:15.898741+0000", 2023-10-09T05:51:19.072 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 3.036488726, 2023-10-09T05:51:19.072 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16378421900000001, 2023-10-09T05:51:19.072 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:19.072 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T05:51:19.072 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:19.072 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.072 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:19.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.898741+0000", 2023-10-09T05:51:19.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:19.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.073 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:19.073 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.898741+0000", 2023-10-09T05:51:19.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:19.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:19.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.898740+0000", 2023-10-09T05:51:19.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T05:51:19.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.074 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:19.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.898745+0000", 2023-10-09T05:51:19.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3329999999999996e-06 2023-10-09T05:51:19.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.074 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:19.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.060692+0000", 2023-10-09T05:51:19.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.161946969 2023-10-09T05:51:19.075 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.075 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:19.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.060709+0000", 2023-10-09T05:51:19.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6799999999999998e-05 2023-10-09T05:51:19.075 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.076 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:19.076 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.062499+0000", 2023-10-09T05:51:19.076 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001790411 2023-10-09T05:51:19.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.076 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.076 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:51:19.076 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.062515+0000", 2023-10-09T05:51:19.076 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5571000000000001e-05 2023-10-09T05:51:19.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.077 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.077 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:51:19.077 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.062525+0000", 2023-10-09T05:51:19.077 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0039e-05 2023-10-09T05:51:19.077 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.077 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:19.077 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.077 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.077 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.078 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.44588.0:47825 6.4 e112/102 ondisk, result = 0)", 2023-10-09T05:51:19.078 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:15.900101+0000", 2023-10-09T05:51:19.078 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 3.035128174, 2023-10-09T05:51:19.078 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.160667692, 2023-10-09T05:51:19.078 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:19.078 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T05:51:19.078 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:19.078 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.078 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:19.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.900101+0000", 2023-10-09T05:51:19.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:19.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:19.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.900101+0000", 2023-10-09T05:51:19.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:19.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:19.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.900101+0000", 2023-10-09T05:51:19.080 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:51:19.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:19.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:15.900105+0000", 2023-10-09T05:51:19.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4159999999999997e-06 2023-10-09T05:51:19.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:19.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.060695+0000", 2023-10-09T05:51:19.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16058985000000001 2023-10-09T05:51:19.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:19.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.060710+0000", 2023-10-09T05:51:19.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4467999999999999e-05 2023-10-09T05:51:19.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.082 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.082 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:19.082 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.060751+0000", 2023-10-09T05:51:19.082 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1730000000000002e-05 2023-10-09T05:51:19.082 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.082 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.082 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:51:19.082 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.060760+0000", 2023-10-09T05:51:19.082 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2490000000000001e-06 2023-10-09T05:51:19.082 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.082 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.083 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:51:19.083 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:16.060769+0000", 2023-10-09T05:51:19.083 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5990000000000008e-06 2023-10-09T05:51:19.083 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.083 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:19.083 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.083 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.083 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.083 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:50495 6.1 6:840c54c9:::benchmark_data_smithi064_284193_object50494:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e114)", 2023-10-09T05:51:19.083 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:18.053000+0000", 2023-10-09T05:51:19.083 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.88222940800000005, 2023-10-09T05:51:19.084 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.13522258700000001, 2023-10-09T05:51:19.084 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:19.084 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:51:19.084 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:19.084 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:19.084 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:19.084 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 50495 2023-10-09T05:51:19.084 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.084 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:19.084 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.084 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:19.085 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:18.053000+0000", 2023-10-09T05:51:19.085 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:19.085 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.085 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.085 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:19.085 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:18.053000+0000", 2023-10-09T05:51:19.085 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:19.085 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.085 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.085 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:19.085 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:18.053000+0000", 2023-10-09T05:51:19.085 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:51:19.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.086 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:19.086 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:18.053020+0000", 2023-10-09T05:51:19.086 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0786999999999999e-05 2023-10-09T05:51:19.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.086 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:19.086 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:18.053028+0000", 2023-10-09T05:51:19.086 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5900000000000002e-06 2023-10-09T05:51:19.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:19.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:18.053034+0000", 2023-10-09T05:51:19.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1299999999999998e-06 2023-10-09T05:51:19.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:19.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:18.055349+0000", 2023-10-09T05:51:19.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002314534 2023-10-09T05:51:19.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:51:19.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:18.055489+0000", 2023-10-09T05:51:19.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000140426 2023-10-09T05:51:19.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:51:19.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:18.055652+0000", 2023-10-09T05:51:19.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000162769 2023-10-09T05:51:19.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:19.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:18.061591+0000", 2023-10-09T05:51:19.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059392020000000002 2023-10-09T05:51:19.089 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.089 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.089 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:51:19.089 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:18.061598+0000", 2023-10-09T05:51:19.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6139999999999997e-06 2023-10-09T05:51:19.089 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.089 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.089 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:51:19.089 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:18.188143+0000", 2023-10-09T05:51:19.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.126545355 2023-10-09T05:51:19.090 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.090 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.090 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:51:19.090 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:18.188185+0000", 2023-10-09T05:51:19.090 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2228999999999998e-05 2023-10-09T05:51:19.090 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:19.090 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:19.090 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:51:19.090 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:18.188223+0000", 2023-10-09T05:51:19.090 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7614000000000003e-05 2023-10-09T05:51:19.091 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.091 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:19.091 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.091 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:19.091 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:19.091 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:51:19.091 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:19.143 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:19.217 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:19.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:19.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:19.323 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:19.362 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 30 16 51861 51845 107.954 72.9375 0.0068538 0.00925837 2023-10-09T05:51:19.398 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:19.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:19 smithi064 ceph-mon[212750]: failed to encode map e115 with expected crc 2023-10-09T05:51:19.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:19 smithi064 ceph-mon[212750]: osdmap e115: 8 total, 8 up, 8 in 2023-10-09T05:51:19.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:19 smithi064 ceph-mon[212750]: failed to encode map e115 with expected crc 2023-10-09T05:51:19.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:19 smithi064 ceph-mon[212750]: failed to encode map e115 with expected crc 2023-10-09T05:51:19.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:19 smithi064 ceph-mon[212750]: failed to encode map e115 with expected crc 2023-10-09T05:51:19.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:19 smithi064 ceph-mon[212750]: failed to encode map e115 with expected crc 2023-10-09T05:51:19.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:19 smithi064 ceph-mon[212750]: failed to encode map e115 with expected crc 2023-10-09T05:51:19.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:19 smithi064 ceph-mon[212750]: failed to encode map e115 with expected crc 2023-10-09T05:51:19.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:19 smithi064 ceph-mon[212750]: failed to encode map e115 with expected crc 2023-10-09T05:51:19.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:19 smithi064 ceph-mon[212750]: pgmap v258: 99 pgs: 6 remapped+peering, 93 active+clean; 2.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 32 MiB/s wr, 506 op/s 2023-10-09T05:51:19.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:19 smithi064 ceph-mon[220044]: failed to encode map e115 with expected crc 2023-10-09T05:51:19.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:19 smithi064 ceph-mon[220044]: osdmap e115: 8 total, 8 up, 8 in 2023-10-09T05:51:19.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:19 smithi064 ceph-mon[220044]: failed to encode map e115 with expected crc 2023-10-09T05:51:19.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:19 smithi064 ceph-mon[220044]: failed to encode map e115 with expected crc 2023-10-09T05:51:19.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:19 smithi064 ceph-mon[220044]: failed to encode map e115 with expected crc 2023-10-09T05:51:19.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:19 smithi064 ceph-mon[220044]: failed to encode map e115 with expected crc 2023-10-09T05:51:19.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:19 smithi064 ceph-mon[220044]: failed to encode map e115 with expected crc 2023-10-09T05:51:19.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:19 smithi064 ceph-mon[220044]: failed to encode map e115 with expected crc 2023-10-09T05:51:19.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:19 smithi064 ceph-mon[220044]: failed to encode map e115 with expected crc 2023-10-09T05:51:19.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:19 smithi064 ceph-mon[220044]: pgmap v258: 99 pgs: 6 remapped+peering, 93 active+clean; 2.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 32 MiB/s wr, 506 op/s 2023-10-09T05:51:19.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:19 smithi156 ceph-mon[195101]: failed to encode map e115 with expected crc 2023-10-09T05:51:19.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:19 smithi156 ceph-mon[195101]: osdmap e115: 8 total, 8 up, 8 in 2023-10-09T05:51:19.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:19 smithi156 ceph-mon[195101]: failed to encode map e115 with expected crc 2023-10-09T05:51:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:19 smithi156 ceph-mon[195101]: failed to encode map e115 with expected crc 2023-10-09T05:51:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:19 smithi156 ceph-mon[195101]: failed to encode map e115 with expected crc 2023-10-09T05:51:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:19 smithi156 ceph-mon[195101]: failed to encode map e115 with expected crc 2023-10-09T05:51:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:19 smithi156 ceph-mon[195101]: failed to encode map e115 with expected crc 2023-10-09T05:51:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:19 smithi156 ceph-mon[195101]: failed to encode map e115 with expected crc 2023-10-09T05:51:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:19 smithi156 ceph-mon[195101]: failed to encode map e115 with expected crc 2023-10-09T05:51:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:19 smithi156 ceph-mon[195101]: pgmap v258: 99 pgs: 6 remapped+peering, 93 active+clean; 2.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 32 MiB/s wr, 506 op/s 2023-10-09T05:51:19.831 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:19.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:19.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:19.940 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:19.980 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:20.050 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:20.050 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:20.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:20.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:20.081 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:51:20.156 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:20.221 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:20.362 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 31 16 53132 53116 107.034 79.4375 0.00602068 0.00933825 2023-10-09T05:51:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:20 smithi156 ceph-mon[195101]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:20 smithi156 ceph-mon[195101]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:20 smithi156 ceph-mon[195101]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:20 smithi156 ceph-mon[195101]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:20 smithi156 ceph-mon[195101]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:20 smithi156 ceph-mon[195101]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:20 smithi156 ceph-mon[195101]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:20 smithi156 ceph-mon[195101]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:20 smithi156 ceph-mon[195101]: osdmap e116: 8 total, 8 up, 8 in 2023-10-09T05:51:20.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:20 smithi156 ceph-mon[195101]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:20 smithi156 ceph-mon[195101]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:20 smithi156 ceph-mon[195101]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:20 smithi156 ceph-mon[195101]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:20 smithi156 ceph-mon[195101]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:20 smithi156 ceph-mon[195101]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:20 smithi156 ceph-mon[195101]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.398 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:51:20.456 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:51:20.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[212750]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[212750]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[212750]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[212750]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[212750]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[212750]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[212750]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[212750]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[212750]: osdmap e116: 8 total, 8 up, 8 in 2023-10-09T05:51:20.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[212750]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[212750]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[212750]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[212750]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[212750]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[212750]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[212750]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[220044]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[220044]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[220044]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[220044]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[220044]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[220044]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[220044]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[220044]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[220044]: osdmap e116: 8 total, 8 up, 8 in 2023-10-09T05:51:20.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[220044]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[220044]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[220044]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[220044]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[220044]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[220044]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:20 smithi064 ceph-mon[220044]: failed to encode map e116 with expected crc 2023-10-09T05:51:20.500 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:51:20.501 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:20.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:20.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:20.549 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:51:20.574 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:51:20.611 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:20.616 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:51:20.658 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:51:20.689 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:20.694 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:51:20.704 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:51:20.704 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:51:20.704 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:51:20.705 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:51:20.705 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:51:20.705 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:51:20.705 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:51:20.705 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:51:21.364 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 32 16 54476 54460 106.31 84 0.00349242 0.0094023 2023-10-09T05:51:21.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:21 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4174623492' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:21.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:21 smithi064 ceph-mon[212750]: pgmap v260: 99 pgs: 99 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 18 KiB/s rd, 91 MiB/s wr, 1.48k op/s; 170 KiB/s, 0 objects/s recovering 2023-10-09T05:51:21.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:21 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4174623492' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:21.474 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:21 smithi064 ceph-mon[220044]: pgmap v260: 99 pgs: 99 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 18 KiB/s rd, 91 MiB/s wr, 1.48k op/s; 170 KiB/s, 0 objects/s recovering 2023-10-09T05:51:21.528 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:21.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:21.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:21.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:21 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4174623492' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:21.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:21 smithi156 ceph-mon[195101]: pgmap v260: 99 pgs: 99 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 18 KiB/s rd, 91 MiB/s wr, 1.48k op/s; 170 KiB/s, 0 objects/s recovering 2023-10-09T05:51:21.632 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:21.659 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:21.685 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:51:22.073 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:22.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:22.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:22.178 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:22.242 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:22.364 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 33 16 55514 55498 105.055 64.875 0.00267356 0.00950144 2023-10-09T05:51:22.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:22 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs peering) 2023-10-09T05:51:22.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:22 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs peering) 2023-10-09T05:51:22.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:22 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs peering) 2023-10-09T05:51:22.777 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:22.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:22.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:22.813 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:22.887 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:22.922 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:22.985 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:22.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:22.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:23.023 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:23.092 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:23.131 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:23.198 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:23.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:23.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:23.307 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:23.365 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 34 16 56709 56693 104.162 74.6875 0.00563983 0.00959472 2023-10-09T05:51:23.386 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:23 smithi064 ceph-mon[220044]: pgmap v261: 99 pgs: 99 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 96 KiB/s, 0 objects/s recovering 2023-10-09T05:51:23.389 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:23 smithi064 ceph-mon[212750]: pgmap v261: 99 pgs: 99 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 96 KiB/s, 0 objects/s recovering 2023-10-09T05:51:23.390 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:23.550 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:23.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:23.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:23.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:23 smithi156 ceph-mon[195101]: pgmap v261: 99 pgs: 99 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 96 KiB/s, 0 objects/s recovering 2023-10-09T05:51:23.668 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:23.734 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:24.364 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 35 16 57911 57895 103.332 75.125 0.00423843 0.0096526 2023-10-09T05:51:24.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:24 smithi064 conmon[212726]: 2023-10-09T05:51:24.208+0000 7f8823f23700 -1 mon.a@0(leader).osd e116 definitely_dead 0 2023-10-09T05:51:24.546 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:24.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:24.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:24.570 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:51:24.603 INFO:teuthology.orchestra.run.smithi064.stderr:marked out osd.1. 2023-10-09T05:51:24.652 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:24.677 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:24.718 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:24.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:24.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:24.825 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:24.856 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:24 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:51:24.856 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:24 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3075118031' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:51:24.856 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:24 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:51:24.856 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:24 smithi064 ceph-mon[212750]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T05:51:24.856 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:24 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:51:24.856 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:24 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3075118031' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:51:24.856 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:24 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:51:24.856 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:24 smithi064 ceph-mon[220044]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T05:51:24.858 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:24.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:24 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:51:24.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:24 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3075118031' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:51:24.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:24 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:51:24.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:24 smithi156 ceph-mon[195101]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T05:51:25.305 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:25.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:25.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:25.364 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 36 16 58138 58122 100.857 14.1875 0.00268081 0.00970175 2023-10-09T05:51:25.409 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:25.434 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:51:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:51:25] "GET /metrics HTTP/1.1" 200 36321 "" "Prometheus/2.43.0" 2023-10-09T05:51:25.437 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:25.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:25 smithi064 ceph-mon[212750]: pgmap v262: 99 pgs: 99 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 79 KiB/s, 0 objects/s recovering 2023-10-09T05:51:25.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:25 smithi064 ceph-mon[212750]: failed to encode map e117 with expected crc 2023-10-09T05:51:25.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:25 smithi064 ceph-mon[212750]: failed to encode map e117 with expected crc 2023-10-09T05:51:25.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:25 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T05:51:25.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:25 smithi064 ceph-mon[212750]: failed to encode map e117 with expected crc 2023-10-09T05:51:25.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:25 smithi064 ceph-mon[212750]: osdmap e117: 8 total, 8 up, 7 in 2023-10-09T05:51:25.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:25 smithi064 ceph-mon[212750]: failed to encode map e117 with expected crc 2023-10-09T05:51:25.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:25 smithi064 ceph-mon[212750]: failed to encode map e117 with expected crc 2023-10-09T05:51:25.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:25 smithi064 ceph-mon[212750]: failed to encode map e117 with expected crc 2023-10-09T05:51:25.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:25 smithi064 ceph-mon[212750]: failed to encode map e117 with expected crc 2023-10-09T05:51:25.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:25 smithi064 ceph-mon[220044]: pgmap v262: 99 pgs: 99 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 79 KiB/s, 0 objects/s recovering 2023-10-09T05:51:25.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:25 smithi064 ceph-mon[220044]: failed to encode map e117 with expected crc 2023-10-09T05:51:25.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:25 smithi064 ceph-mon[220044]: failed to encode map e117 with expected crc 2023-10-09T05:51:25.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:25 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T05:51:25.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:25 smithi064 ceph-mon[220044]: failed to encode map e117 with expected crc 2023-10-09T05:51:25.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:25 smithi064 ceph-mon[220044]: osdmap e117: 8 total, 8 up, 7 in 2023-10-09T05:51:25.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:25 smithi064 ceph-mon[220044]: failed to encode map e117 with expected crc 2023-10-09T05:51:25.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:25 smithi064 ceph-mon[220044]: failed to encode map e117 with expected crc 2023-10-09T05:51:25.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:25 smithi064 ceph-mon[220044]: failed to encode map e117 with expected crc 2023-10-09T05:51:25.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:25 smithi064 ceph-mon[220044]: failed to encode map e117 with expected crc 2023-10-09T05:51:25.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:25 smithi156 ceph-mon[195101]: pgmap v262: 99 pgs: 99 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 79 KiB/s, 0 objects/s recovering 2023-10-09T05:51:25.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:25 smithi156 ceph-mon[195101]: failed to encode map e117 with expected crc 2023-10-09T05:51:25.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:25 smithi156 ceph-mon[195101]: failed to encode map e117 with expected crc 2023-10-09T05:51:25.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:25 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T05:51:25.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:25 smithi156 ceph-mon[195101]: failed to encode map e117 with expected crc 2023-10-09T05:51:25.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:25 smithi156 ceph-mon[195101]: osdmap e117: 8 total, 8 up, 7 in 2023-10-09T05:51:25.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:25 smithi156 ceph-mon[195101]: failed to encode map e117 with expected crc 2023-10-09T05:51:25.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:25 smithi156 ceph-mon[195101]: failed to encode map e117 with expected crc 2023-10-09T05:51:25.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:25 smithi156 ceph-mon[195101]: failed to encode map e117 with expected crc 2023-10-09T05:51:25.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:25 smithi156 ceph-mon[195101]: failed to encode map e117 with expected crc 2023-10-09T05:51:25.889 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:25.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:25.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:25.991 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:26.023 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:26.126 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:26.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:26.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:26.229 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:26.261 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:26.294 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:26.364 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 37 16 58138 58122 98.1323 0 - 0.00970175 2023-10-09T05:51:26.376 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:51:26.376 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:51:26.410 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:26.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:26.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:26.515 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:26.547 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:26.620 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:26.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:26.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:26.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[220044]: osdmap e118: 8 total, 8 up, 7 in 2023-10-09T05:51:26.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[220044]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[220044]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[220044]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[220044]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[220044]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[220044]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[220044]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[220044]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[220044]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[220044]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[220044]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[220044]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[220044]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[220044]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[220044]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[220044]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[220044]: osdmap e119: 8 total, 8 up, 7 in 2023-10-09T05:51:26.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[212750]: osdmap e118: 8 total, 8 up, 7 in 2023-10-09T05:51:26.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[212750]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[212750]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[212750]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[212750]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[212750]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[212750]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[212750]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[212750]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[212750]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[212750]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[212750]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[212750]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[212750]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[212750]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[212750]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[212750]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:26 smithi064 ceph-mon[212750]: osdmap e119: 8 total, 8 up, 7 in 2023-10-09T05:51:26.751 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:26.797 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:26.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:26 smithi156 ceph-mon[195101]: osdmap e118: 8 total, 8 up, 7 in 2023-10-09T05:51:26.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:26 smithi156 ceph-mon[195101]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:26 smithi156 ceph-mon[195101]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:26 smithi156 ceph-mon[195101]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:26 smithi156 ceph-mon[195101]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:26 smithi156 ceph-mon[195101]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:26 smithi156 ceph-mon[195101]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:26 smithi156 ceph-mon[195101]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:26 smithi156 ceph-mon[195101]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:26 smithi156 ceph-mon[195101]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:26 smithi156 ceph-mon[195101]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:26 smithi156 ceph-mon[195101]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:26 smithi156 ceph-mon[195101]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.875 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:26 smithi156 ceph-mon[195101]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.875 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:26 smithi156 ceph-mon[195101]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.875 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:26 smithi156 ceph-mon[195101]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.875 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:26 smithi156 ceph-mon[195101]: failed to encode map e118 with expected crc 2023-10-09T05:51:26.875 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:26 smithi156 ceph-mon[195101]: osdmap e119: 8 total, 8 up, 7 in 2023-10-09T05:51:27.075 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T05:51:27.364 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 38 16 58734 58718 96.5307 18.625 0.0197723 0.0103537 2023-10-09T05:51:27.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:27 smithi156 ceph-mon[195101]: pgmap v265: 99 pgs: 99 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s; 79 KiB/s, 0 objects/s recovering 2023-10-09T05:51:27.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:27 smithi156 ceph-mon[195101]: failed to encode map e119 with expected crc 2023-10-09T05:51:27.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:27 smithi156 ceph-mon[195101]: failed to encode map e119 with expected crc 2023-10-09T05:51:27.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:27 smithi156 ceph-mon[195101]: failed to encode map e119 with expected crc 2023-10-09T05:51:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:27 smithi156 ceph-mon[195101]: failed to encode map e119 with expected crc 2023-10-09T05:51:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:27 smithi156 ceph-mon[195101]: failed to encode map e119 with expected crc 2023-10-09T05:51:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:27 smithi156 ceph-mon[195101]: failed to encode map e119 with expected crc 2023-10-09T05:51:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:27 smithi156 ceph-mon[195101]: failed to encode map e119 with expected crc 2023-10-09T05:51:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:27 smithi156 ceph-mon[195101]: failed to encode map e119 with expected crc 2023-10-09T05:51:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:27 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1836721622' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:51:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:27 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:51:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:27 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1836721622' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:51:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:27 smithi156 ceph-mon[195101]: osdmap e120: 8 total, 8 up, 7 in 2023-10-09T05:51:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:27 smithi156 ceph-mon[195101]: failed to encode map e120 with expected crc 2023-10-09T05:51:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:27 smithi156 ceph-mon[195101]: failed to encode map e120 with expected crc 2023-10-09T05:51:27.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:27 smithi156 ceph-mon[195101]: failed to encode map e120 with expected crc 2023-10-09T05:51:27.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:27 smithi156 ceph-mon[195101]: failed to encode map e120 with expected crc 2023-10-09T05:51:27.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:27 smithi156 ceph-mon[195101]: failed to encode map e120 with expected crc 2023-10-09T05:51:27.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:27 smithi156 ceph-mon[195101]: failed to encode map e120 with expected crc 2023-10-09T05:51:27.892 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:27.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:27.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:27.896 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[220044]: pgmap v265: 99 pgs: 99 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s; 79 KiB/s, 0 objects/s recovering 2023-10-09T05:51:27.896 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[220044]: failed to encode map e119 with expected crc 2023-10-09T05:51:27.897 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[220044]: failed to encode map e119 with expected crc 2023-10-09T05:51:27.897 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[220044]: failed to encode map e119 with expected crc 2023-10-09T05:51:27.897 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[220044]: failed to encode map e119 with expected crc 2023-10-09T05:51:27.897 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[220044]: failed to encode map e119 with expected crc 2023-10-09T05:51:27.897 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[220044]: failed to encode map e119 with expected crc 2023-10-09T05:51:27.897 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[220044]: failed to encode map e119 with expected crc 2023-10-09T05:51:27.897 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[220044]: failed to encode map e119 with expected crc 2023-10-09T05:51:27.897 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1836721622' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:51:27.897 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:51:27.897 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1836721622' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:51:27.897 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[220044]: osdmap e120: 8 total, 8 up, 7 in 2023-10-09T05:51:27.897 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[220044]: failed to encode map e120 with expected crc 2023-10-09T05:51:27.898 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[220044]: failed to encode map e120 with expected crc 2023-10-09T05:51:27.898 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[220044]: failed to encode map e120 with expected crc 2023-10-09T05:51:27.898 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[220044]: failed to encode map e120 with expected crc 2023-10-09T05:51:27.898 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[220044]: failed to encode map e120 with expected crc 2023-10-09T05:51:27.898 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[220044]: failed to encode map e120 with expected crc 2023-10-09T05:51:27.898 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[212750]: pgmap v265: 99 pgs: 99 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s; 79 KiB/s, 0 objects/s recovering 2023-10-09T05:51:27.898 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[212750]: failed to encode map e119 with expected crc 2023-10-09T05:51:27.898 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[212750]: failed to encode map e119 with expected crc 2023-10-09T05:51:27.899 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[212750]: failed to encode map e119 with expected crc 2023-10-09T05:51:27.899 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[212750]: failed to encode map e119 with expected crc 2023-10-09T05:51:27.899 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[212750]: failed to encode map e119 with expected crc 2023-10-09T05:51:27.899 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[212750]: failed to encode map e119 with expected crc 2023-10-09T05:51:27.899 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[212750]: failed to encode map e119 with expected crc 2023-10-09T05:51:27.899 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[212750]: failed to encode map e119 with expected crc 2023-10-09T05:51:27.899 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1836721622' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:51:27.899 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:51:27.899 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1836721622' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:51:27.899 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[212750]: osdmap e120: 8 total, 8 up, 7 in 2023-10-09T05:51:27.899 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[212750]: failed to encode map e120 with expected crc 2023-10-09T05:51:27.900 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[212750]: failed to encode map e120 with expected crc 2023-10-09T05:51:27.900 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[212750]: failed to encode map e120 with expected crc 2023-10-09T05:51:27.900 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[212750]: failed to encode map e120 with expected crc 2023-10-09T05:51:27.900 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[212750]: failed to encode map e120 with expected crc 2023-10-09T05:51:27.900 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:27 smithi064 ceph-mon[212750]: failed to encode map e120 with expected crc 2023-10-09T05:51:27.998 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:28.024 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:28.069 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:28.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:28.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:28.178 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:28.204 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:51:28.230 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:28.335 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:28.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:28.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:28.365 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 39 16 59685 59669 95.5781 59.4375 0.00948852 0.0104582 2023-10-09T05:51:28.444 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:28.489 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:28.601 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:28.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:28.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:28.709 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:28.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:28 smithi064 ceph-mon[212750]: failed to encode map e120 with expected crc 2023-10-09T05:51:28.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:28 smithi064 ceph-mon[212750]: osdmap e121: 8 total, 8 up, 7 in 2023-10-09T05:51:28.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:28 smithi064 ceph-mon[212750]: failed to encode map e121 with expected crc 2023-10-09T05:51:28.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:28 smithi064 ceph-mon[212750]: failed to encode map e121 with expected crc 2023-10-09T05:51:28.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:28 smithi064 ceph-mon[212750]: failed to encode map e121 with expected crc 2023-10-09T05:51:28.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:28 smithi064 ceph-mon[212750]: failed to encode map e121 with expected crc 2023-10-09T05:51:28.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:28 smithi064 ceph-mon[212750]: failed to encode map e121 with expected crc 2023-10-09T05:51:28.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:28 smithi064 ceph-mon[212750]: failed to encode map e121 with expected crc 2023-10-09T05:51:28.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:28 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:51:28.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:28 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:51:28.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:28 smithi064 ceph-mon[220044]: failed to encode map e120 with expected crc 2023-10-09T05:51:28.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:28 smithi064 ceph-mon[220044]: osdmap e121: 8 total, 8 up, 7 in 2023-10-09T05:51:28.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:28 smithi064 ceph-mon[220044]: failed to encode map e121 with expected crc 2023-10-09T05:51:28.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:28 smithi064 ceph-mon[220044]: failed to encode map e121 with expected crc 2023-10-09T05:51:28.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:28 smithi064 ceph-mon[220044]: failed to encode map e121 with expected crc 2023-10-09T05:51:28.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:28 smithi064 ceph-mon[220044]: failed to encode map e121 with expected crc 2023-10-09T05:51:28.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:28 smithi064 ceph-mon[220044]: failed to encode map e121 with expected crc 2023-10-09T05:51:28.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:28 smithi064 ceph-mon[220044]: failed to encode map e121 with expected crc 2023-10-09T05:51:28.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:28 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:51:28.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:28 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:51:28.733 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:28.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:28 smithi156 ceph-mon[195101]: failed to encode map e120 with expected crc 2023-10-09T05:51:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:28 smithi156 ceph-mon[195101]: osdmap e121: 8 total, 8 up, 7 in 2023-10-09T05:51:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:28 smithi156 ceph-mon[195101]: failed to encode map e121 with expected crc 2023-10-09T05:51:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:28 smithi156 ceph-mon[195101]: failed to encode map e121 with expected crc 2023-10-09T05:51:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:28 smithi156 ceph-mon[195101]: failed to encode map e121 with expected crc 2023-10-09T05:51:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:28 smithi156 ceph-mon[195101]: failed to encode map e121 with expected crc 2023-10-09T05:51:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:28 smithi156 ceph-mon[195101]: failed to encode map e121 with expected crc 2023-10-09T05:51:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:28 smithi156 ceph-mon[195101]: failed to encode map e121 with expected crc 2023-10-09T05:51:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:28 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:51:28.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:28 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:51:28.872 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:28.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:28.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:28.989 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:29.018 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:29.353 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:29.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:29.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:29.365 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-10-09T05:51:29.366076+0000 min lat: 0.00106234 max lat: 2.26368 avg lat: 0.0105775 2023-10-09T05:51:29.365 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:51:29.365 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 40 16 60521 60505 94.4952 52.25 0.00411673 0.0105775 2023-10-09T05:51:29.463 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:29.509 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:29.589 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:29.603 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:29.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:29.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:29.719 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:29.786 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:29 smithi156 ceph-mon[195101]: failed to encode map e121 with expected crc 2023-10-09T05:51:29.786 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:29 smithi156 ceph-mon[195101]: pgmap v269: 99 pgs: 99 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 11 MiB/s wr, 183 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T05:51:29.788 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:29.866 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:29 smithi064 ceph-mon[212750]: failed to encode map e121 with expected crc 2023-10-09T05:51:29.866 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:29 smithi064 ceph-mon[212750]: pgmap v269: 99 pgs: 99 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 11 MiB/s wr, 183 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T05:51:29.867 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:29 smithi064 ceph-mon[220044]: failed to encode map e121 with expected crc 2023-10-09T05:51:29.867 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:29 smithi064 ceph-mon[220044]: pgmap v269: 99 pgs: 99 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 11 MiB/s wr, 183 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T05:51:29.908 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:29.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:29.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:30.026 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:30.094 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:30.213 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:30.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:30.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:30.227 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:51:30.268 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:51:30.306 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:51:30.327 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:30.340 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:51:30.365 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 41 16 61532 61516 93.7316 63.1875 0.0116014 0.0106633 2023-10-09T05:51:30.380 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:51:30.381 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:30.429 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:51:30.456 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:30.462 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:51:30.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:30.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:30.470 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:51:30.502 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:51:30.508 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:51:30.508 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:51:30.508 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:51:30.508 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:51:30.508 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:51:30.508 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:51:30.508 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:51:30.508 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:51:30.565 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:51:30.648 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:30.789 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:51:30.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:30.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:51:30.900 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:30.936 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:30 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2804216362' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:30.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:30 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2804216362' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:30.940 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:31.030 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:51:31.030 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:51:31.030 INFO:tasks.thrashosds.thrasher:Killing osd 0, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:51:31.030 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-10-09T05:51:31.030 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.0 2023-10-09T05:51:31.126 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:30 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2804216362' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:31.219 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:51:31 smithi064 systemd[1]: Stopping Ceph osd.0 for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:51:31.365 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 42 16 62386 62370 92.771 53.375 0.0113173 0.0107713 2023-10-09T05:51:31.676 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:31.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:31.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:31.792 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:31.845 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:31.932 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:31.949 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:31 smithi156 ceph-mon[195101]: pgmap v270: 99 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 44 MiB/s wr, 699 op/s; 10851/180372 objects degraded (6.016%); 7058/180372 objects misplaced (3.913%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:51:31.949 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:31 smithi156 ceph-mon[195101]: Health check failed: Degraded data redundancy: 10851/180372 objects degraded (6.016%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T05:51:31.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:31.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:31.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:31 smithi064 ceph-mon[220044]: pgmap v270: 99 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 44 MiB/s wr, 699 op/s; 10851/180372 objects degraded (6.016%); 7058/180372 objects misplaced (3.913%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:51:31.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:31 smithi064 ceph-mon[220044]: Health check failed: Degraded data redundancy: 10851/180372 objects degraded (6.016%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T05:51:31.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:31 smithi064 ceph-mon[212750]: pgmap v270: 99 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 44 MiB/s wr, 699 op/s; 10851/180372 objects degraded (6.016%); 7058/180372 objects misplaced (3.913%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:51:31.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:31 smithi064 ceph-mon[212750]: Health check failed: Degraded data redundancy: 10851/180372 objects degraded (6.016%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T05:51:32.050 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:32.134 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:32.307 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:32.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:32.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:32.367 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 43 16 63285 63269 91.9167 56.1875 0.0137273 0.010873 2023-10-09T05:51:32.422 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:32.431 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:32.486 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:32.604 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:32.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:32.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:32.718 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:32.786 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:51:32 smithi064 conmon[234442]: 2023-10-09T05:51:32.528+0000 7fe52a81d700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T05:51:32.786 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:51:32 smithi064 conmon[234442]: 2023-10-09T05:51:32.528+0000 7fe52a81d700 -1 osd.0 121 *** Got signal Terminated *** 2023-10-09T05:51:32.786 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:51:32 smithi064 conmon[234442]: 2023-10-09T05:51:32.528+0000 7fe52a81d700 -1 osd.0 121 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:51:32.797 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:33.048 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:51:33.048 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T05:51:33.048 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T05:51:33.048 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T05:51:33.048 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:51:33.113 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:33.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:33.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:33.226 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:33.253 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:33.298 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:33.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:33.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:33.367 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 44 16 64070 64054 90.943 49.0625 0.00331788 0.0109372 2023-10-09T05:51:33.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:33 smithi156 ceph-mon[195101]: osd.0 marked itself down and dead 2023-10-09T05:51:33.404 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:33.430 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:33 smithi064 ceph-mon[212750]: osd.0 marked itself down and dead 2023-10-09T05:51:33.431 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:33 smithi064 ceph-mon[220044]: osd.0 marked itself down and dead 2023-10-09T05:51:33.434 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:33.534 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:33.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:33.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:33.638 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:33.672 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:34.289 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:51:34.367 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 45 16 64200 64184 89.1033 8.125 0.00355851 0.0110941 2023-10-09T05:51:34.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:34 smithi156 ceph-mon[195101]: pgmap v271: 99 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.7 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 39 MiB/s wr, 621 op/s; 10851/182847 objects degraded (5.934%); 7058/182847 objects misplaced (3.860%); 2.4 MiB/s, 39 objects/s recovering 2023-10-09T05:51:34.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:34 smithi156 ceph-mon[195101]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:51:34.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:34 smithi156 ceph-mon[195101]: osdmap e122: 8 total, 7 up, 7 in 2023-10-09T05:51:34.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:34 smithi156 ceph-mon[195101]: failed to encode map e122 with expected crc 2023-10-09T05:51:34.455 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:34 smithi064 ceph-mon[212750]: pgmap v271: 99 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.7 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 39 MiB/s wr, 621 op/s; 10851/182847 objects degraded (5.934%); 7058/182847 objects misplaced (3.860%); 2.4 MiB/s, 39 objects/s recovering 2023-10-09T05:51:34.455 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:34 smithi064 ceph-mon[212750]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:51:34.455 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:34 smithi064 ceph-mon[212750]: osdmap e122: 8 total, 7 up, 7 in 2023-10-09T05:51:34.455 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:34 smithi064 ceph-mon[212750]: failed to encode map e122 with expected crc 2023-10-09T05:51:34.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:34 smithi064 ceph-mon[220044]: pgmap v271: 99 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.7 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 39 MiB/s wr, 621 op/s; 10851/182847 objects degraded (5.934%); 7058/182847 objects misplaced (3.860%); 2.4 MiB/s, 39 objects/s recovering 2023-10-09T05:51:34.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:34 smithi064 ceph-mon[220044]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:51:34.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:34 smithi064 ceph-mon[220044]: osdmap e122: 8 total, 7 up, 7 in 2023-10-09T05:51:34.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:34 smithi064 ceph-mon[220044]: failed to encode map e122 with expected crc 2023-10-09T05:51:34.456 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:51:34 smithi064 bash[293195]: 5779746d3c04380519a1a4a1c4335de455b3843e92b9c225c70fdc6c56bf1364 2023-10-09T05:51:35.335 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:35.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:35.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:35.367 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 46 16 64396 64380 87.4331 12.25 0.0281055 0.0114313 2023-10-09T05:51:35.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:35 smithi156 ceph-mon[195101]: failed to encode map e123 with expected crc 2023-10-09T05:51:35.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:35 smithi156 ceph-mon[195101]: failed to encode map e123 with expected crc 2023-10-09T05:51:35.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:35 smithi156 ceph-mon[195101]: failed to encode map e123 with expected crc 2023-10-09T05:51:35.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:35 smithi156 ceph-mon[195101]: osdmap e123: 8 total, 7 up, 7 in 2023-10-09T05:51:35.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:35 smithi156 ceph-mon[195101]: failed to encode map e123 with expected crc 2023-10-09T05:51:35.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:35 smithi156 ceph-mon[195101]: failed to encode map e123 with expected crc 2023-10-09T05:51:35.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:35 smithi156 ceph-mon[195101]: failed to encode map e123 with expected crc 2023-10-09T05:51:35.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:35 smithi156 ceph-mon[195101]: failed to encode map e123 with expected crc 2023-10-09T05:51:35.442 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:35.450 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:35 smithi064 ceph-mon[212750]: failed to encode map e123 with expected crc 2023-10-09T05:51:35.450 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:35 smithi064 ceph-mon[212750]: failed to encode map e123 with expected crc 2023-10-09T05:51:35.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:35 smithi064 ceph-mon[212750]: failed to encode map e123 with expected crc 2023-10-09T05:51:35.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:35 smithi064 ceph-mon[212750]: osdmap e123: 8 total, 7 up, 7 in 2023-10-09T05:51:35.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:35 smithi064 ceph-mon[212750]: failed to encode map e123 with expected crc 2023-10-09T05:51:35.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:35 smithi064 ceph-mon[212750]: failed to encode map e123 with expected crc 2023-10-09T05:51:35.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:35 smithi064 ceph-mon[212750]: failed to encode map e123 with expected crc 2023-10-09T05:51:35.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:35 smithi064 ceph-mon[212750]: failed to encode map e123 with expected crc 2023-10-09T05:51:35.451 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:35 smithi064 ceph-mon[220044]: failed to encode map e123 with expected crc 2023-10-09T05:51:35.451 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:35 smithi064 ceph-mon[220044]: failed to encode map e123 with expected crc 2023-10-09T05:51:35.451 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:35 smithi064 ceph-mon[220044]: failed to encode map e123 with expected crc 2023-10-09T05:51:35.451 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:35 smithi064 ceph-mon[220044]: osdmap e123: 8 total, 7 up, 7 in 2023-10-09T05:51:35.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:35 smithi064 ceph-mon[220044]: failed to encode map e123 with expected crc 2023-10-09T05:51:35.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:35 smithi064 ceph-mon[220044]: failed to encode map e123 with expected crc 2023-10-09T05:51:35.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:35 smithi064 ceph-mon[220044]: failed to encode map e123 with expected crc 2023-10-09T05:51:35.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:35 smithi064 ceph-mon[220044]: failed to encode map e123 with expected crc 2023-10-09T05:51:35.452 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:51:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:51:35] "GET /metrics HTTP/1.1" 200 36330 "" "Prometheus/2.43.0" 2023-10-09T05:51:35.489 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:35.610 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:35.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:35.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:35.731 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:35.784 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:35.962 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:51:36.151 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:36.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:36.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:36.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:36 smithi064 ceph-mon[220044]: pgmap v274: 99 pgs: 1 remapped+peering, 57 active+clean, 24 active+undersized, 2 active+recovering+undersized+remapped, 1 stale+active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s; 19273/192477 objects degraded (10.013%); 6441/192477 objects misplaced (3.346%); 8.3 MiB/s, 133 objects/s recovering 2023-10-09T05:51:36.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:36 smithi064 ceph-mon[220044]: failed to encode map e124 with expected crc 2023-10-09T05:51:36.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:36 smithi064 ceph-mon[220044]: failed to encode map e124 with expected crc 2023-10-09T05:51:36.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:36 smithi064 ceph-mon[220044]: osdmap e124: 8 total, 7 up, 7 in 2023-10-09T05:51:36.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:36 smithi064 ceph-mon[220044]: failed to encode map e124 with expected crc 2023-10-09T05:51:36.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:36 smithi064 ceph-mon[220044]: failed to encode map e124 with expected crc 2023-10-09T05:51:36.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:36 smithi064 ceph-mon[212750]: pgmap v274: 99 pgs: 1 remapped+peering, 57 active+clean, 24 active+undersized, 2 active+recovering+undersized+remapped, 1 stale+active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s; 19273/192477 objects degraded (10.013%); 6441/192477 objects misplaced (3.346%); 8.3 MiB/s, 133 objects/s recovering 2023-10-09T05:51:36.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:36 smithi064 ceph-mon[212750]: failed to encode map e124 with expected crc 2023-10-09T05:51:36.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:36 smithi064 ceph-mon[212750]: failed to encode map e124 with expected crc 2023-10-09T05:51:36.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:36 smithi064 ceph-mon[212750]: osdmap e124: 8 total, 7 up, 7 in 2023-10-09T05:51:36.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:36 smithi064 ceph-mon[212750]: failed to encode map e124 with expected crc 2023-10-09T05:51:36.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:36 smithi064 ceph-mon[212750]: failed to encode map e124 with expected crc 2023-10-09T05:51:36.257 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:36.349 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:36.367 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 47 16 65358 65342 86.8521 60.125 0.0142598 0.0115083 2023-10-09T05:51:36.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:36 smithi156 ceph-mon[195101]: pgmap v274: 99 pgs: 1 remapped+peering, 57 active+clean, 24 active+undersized, 2 active+recovering+undersized+remapped, 1 stale+active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s; 19273/192477 objects degraded (10.013%); 6441/192477 objects misplaced (3.346%); 8.3 MiB/s, 133 objects/s recovering 2023-10-09T05:51:36.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:36 smithi156 ceph-mon[195101]: failed to encode map e124 with expected crc 2023-10-09T05:51:36.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:36 smithi156 ceph-mon[195101]: failed to encode map e124 with expected crc 2023-10-09T05:51:36.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:36 smithi156 ceph-mon[195101]: osdmap e124: 8 total, 7 up, 7 in 2023-10-09T05:51:36.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:36 smithi156 ceph-mon[195101]: failed to encode map e124 with expected crc 2023-10-09T05:51:36.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:36 smithi156 ceph-mon[195101]: failed to encode map e124 with expected crc 2023-10-09T05:51:36.439 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:36.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:36.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:36.544 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:36.599 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:36.656 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:36.721 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:36.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:36.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:36.839 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:36.911 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:37.000 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:37.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:37.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:37.112 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:37.180 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:37.311 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:37.368 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 48 16 66286 66270 86.2511 58 0.00977151 0.0115868 2023-10-09T05:51:37.369 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:37 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 19273/192477 objects degraded (10.013%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T05:51:37.370 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:37 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 19273/192477 objects degraded (10.013%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T05:51:37.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:37 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 19273/192477 objects degraded (10.013%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T05:51:37.779 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:37.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:37.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:37.887 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:37.981 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:38.163 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T05:51:38.368 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 49 15 67339 67324 85.8353 65.875 0.0359395 0.0116449 2023-10-09T05:51:38.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:38 smithi064 ceph-mon[212750]: pgmap v276: 99 pgs: 1 remapped+peering, 57 active+clean, 24 active+undersized, 2 active+recovering+undersized+remapped, 1 stale+active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 42 MiB/s wr, 672 op/s; 19273/192477 objects degraded (10.013%); 6441/192477 objects misplaced (3.346%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T05:51:38.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:38 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2720927762' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:51:38.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:38 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:51:38.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:38 smithi064 ceph-mon[220044]: pgmap v276: 99 pgs: 1 remapped+peering, 57 active+clean, 24 active+undersized, 2 active+recovering+undersized+remapped, 1 stale+active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 42 MiB/s wr, 672 op/s; 19273/192477 objects degraded (10.013%); 6441/192477 objects misplaced (3.346%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T05:51:38.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:38 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2720927762' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:51:38.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:38 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:51:38.564 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:38.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:38.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:38.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:38 smithi156 ceph-mon[195101]: pgmap v276: 99 pgs: 1 remapped+peering, 57 active+clean, 24 active+undersized, 2 active+recovering+undersized+remapped, 1 stale+active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 42 MiB/s wr, 672 op/s; 19273/192477 objects degraded (10.013%); 6441/192477 objects misplaced (3.346%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T05:51:38.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:38 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2720927762' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:51:38.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:38 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:51:38.671 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:38.746 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:38.846 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:38.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:38.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:38.960 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:38.987 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:39.050 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:39.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:39.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:39.155 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:39.220 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:39.307 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:39.320 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:39 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:51:39.321 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:39 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:51:39.322 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:39 smithi156 ceph-mon[195101]: osdmap e125: 8 total, 7 up, 7 in 2023-10-09T05:51:39.322 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:39 smithi156 ceph-mon[195101]: failed to encode map e125 with expected crc 2023-10-09T05:51:39.323 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:39 smithi156 ceph-mon[195101]: failed to encode map e125 with expected crc 2023-10-09T05:51:39.323 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:39 smithi156 ceph-mon[195101]: failed to encode map e125 with expected crc 2023-10-09T05:51:39.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:39.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:39.369 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 50 16 68234 68218 85.2348 55.875 0.0322214 0.0117273 2023-10-09T05:51:39.424 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:39.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:39 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:51:39.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:39 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:51:39.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:39 smithi064 ceph-mon[220044]: osdmap e125: 8 total, 7 up, 7 in 2023-10-09T05:51:39.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:39 smithi064 ceph-mon[220044]: failed to encode map e125 with expected crc 2023-10-09T05:51:39.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:39 smithi064 ceph-mon[220044]: failed to encode map e125 with expected crc 2023-10-09T05:51:39.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:39 smithi064 ceph-mon[220044]: failed to encode map e125 with expected crc 2023-10-09T05:51:39.475 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:39 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:51:39.475 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:39 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:51:39.475 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:39 smithi064 ceph-mon[212750]: osdmap e125: 8 total, 7 up, 7 in 2023-10-09T05:51:39.475 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:39 smithi064 ceph-mon[212750]: failed to encode map e125 with expected crc 2023-10-09T05:51:39.475 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:39 smithi064 ceph-mon[212750]: failed to encode map e125 with expected crc 2023-10-09T05:51:39.475 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:39 smithi064 ceph-mon[212750]: failed to encode map e125 with expected crc 2023-10-09T05:51:39.491 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:39.602 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:39.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:39.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:39.714 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:39.779 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:39.882 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:39.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:39.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:40.003 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:40.064 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:40.166 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:40.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:40.180 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:40 smithi156 ceph-mon[195101]: pgmap v278: 99 pgs: 1 remapped+peering, 53 active+clean, 29 active+undersized, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 40 MiB/s wr, 633 op/s; 19273/193044 objects degraded (9.984%); 6441/193044 objects misplaced (3.337%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T05:51:40.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:40.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:40 smithi064 ceph-mon[212750]: pgmap v278: 99 pgs: 1 remapped+peering, 53 active+clean, 29 active+undersized, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 40 MiB/s wr, 633 op/s; 19273/193044 objects degraded (9.984%); 6441/193044 objects misplaced (3.337%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T05:51:40.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:40 smithi064 ceph-mon[220044]: pgmap v278: 99 pgs: 1 remapped+peering, 53 active+clean, 29 active+undersized, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 40 MiB/s wr, 633 op/s; 19273/193044 objects degraded (9.984%); 6441/193044 objects misplaced (3.337%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T05:51:40.282 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:40.343 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:40.369 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 51 15 69197 69182 84.7449 60.25 0.0176298 0.0117941 2023-10-09T05:51:41.224 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:41.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:41.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:41.332 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:41.369 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 52 16 70233 70217 84.3592 64.6875 0.00815245 0.0118479 2023-10-09T05:51:41.400 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:41.458 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:41.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:41.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:41.563 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:41.625 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:41.818 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:51:41.818 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:51:41.818 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:51:41.818 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:51:41.818 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:51:41.818 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:51:41.819 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:51:41.819 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-10-09T05:51:41.819 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:51:41.819 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:51:41.819 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:51:41.819 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:51:41.819 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:51:41.819 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:51:41.819 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:51:41.857 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.0.service' 2023-10-09T05:51:41.911 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:51:41 smithi064 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.0.service: Succeeded. 2023-10-09T05:51:41.911 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:51:41 smithi064 systemd[1]: Stopped Ceph osd.0 for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:51:41.980 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:51:41.980 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-10-09T05:51:41.986 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:51:42.037 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:42.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:42.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:42.148 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:42.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:42 smithi064 ceph-mon[212750]: pgmap v279: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 29 active+undersized, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s; 27299/204462 objects degraded (13.352%); 6698/204462 objects misplaced (3.276%); 11 MiB/s, 181 objects/s recovering 2023-10-09T05:51:42.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:42 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1138556926' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:42.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:42 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 27299/204462 objects degraded (13.352%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:51:42.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:42 smithi064 ceph-mon[220044]: pgmap v279: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 29 active+undersized, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s; 27299/204462 objects degraded (13.352%); 6698/204462 objects misplaced (3.276%); 11 MiB/s, 181 objects/s recovering 2023-10-09T05:51:42.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:42 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1138556926' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:42.168 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:42 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 27299/204462 objects degraded (13.352%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:51:42.272 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:42.369 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 53 16 71098 71082 83.7877 54.0625 0.0310883 0.01193 2023-10-09T05:51:42.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:42 smithi156 ceph-mon[195101]: pgmap v279: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 29 active+undersized, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s; 27299/204462 objects degraded (13.352%); 6698/204462 objects misplaced (3.276%); 11 MiB/s, 181 objects/s recovering 2023-10-09T05:51:42.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:42 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1138556926' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:42.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:42 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 27299/204462 objects degraded (13.352%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:51:42.880 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:42.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:42.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:42.984 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:43.008 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T05:51:43.008 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T05:51:43.054 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:43.169 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:43.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:43.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:43.267 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:43.283 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:43.326 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:43 smithi156 ceph-mon[195101]: pgmap v280: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 29 active+undersized, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 33 MiB/s wr, 535 op/s; 27299/205332 objects degraded (13.295%); 6698/205332 objects misplaced (3.262%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T05:51:43.329 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:43.360 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:51:43.360 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:51:43.369 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 54 16 71976 71960 83.2524 54.875 0.0370534 0.0120065 2023-10-09T05:51:43.426 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:43.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:43.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:43.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:43 smithi064 ceph-mon[220044]: pgmap v280: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 29 active+undersized, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 33 MiB/s wr, 535 op/s; 27299/205332 objects degraded (13.295%); 6698/205332 objects misplaced (3.262%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T05:51:43.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:43 smithi064 ceph-mon[212750]: pgmap v280: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 29 active+undersized, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 33 MiB/s wr, 535 op/s; 27299/205332 objects degraded (13.295%); 6698/205332 objects misplaced (3.262%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T05:51:43.540 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:43.604 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:43.708 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:43.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:43.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:43.827 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:43.873 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:44.369 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 55 16 72945 72929 82.8399 60.5625 0.0104612 0.0120667 2023-10-09T05:51:44.370 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:44.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:44.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:44.476 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:44.546 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:51:44.552 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:44.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:51:44.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:51:44.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:51:44.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:51:44.794 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:44.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:44.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:44.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:51:44.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:51:44.897 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:44.940 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:44.998 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:45.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:45.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:45.009 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:51:45.102 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:45.123 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:51:45.130 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:45.196 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:45.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:45.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:45.301 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:45.369 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 56 16 73719 73703 82.2245 48.375 0.00900388 0.0121563 2023-10-09T05:51:45.384 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:45.450 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:45 smithi064 ceph-mon[212750]: pgmap v281: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 1 active+recovering+undersized+remapped, 29 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 58 MiB/s wr, 930 op/s; 28506/218601 objects degraded (13.040%); 6464/218601 objects misplaced (2.957%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T05:51:45.450 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:51:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:51:45] "GET /metrics HTTP/1.1" 200 36352 "" "Prometheus/2.43.0" 2023-10-09T05:51:45.451 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:45 smithi064 ceph-mon[220044]: pgmap v281: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 1 active+recovering+undersized+remapped, 29 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 58 MiB/s wr, 930 op/s; 28506/218601 objects degraded (13.040%); 6464/218601 objects misplaced (2.957%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T05:51:45.518 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:45.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:45.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:45.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:45 smithi156 ceph-mon[195101]: pgmap v281: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 1 active+recovering+undersized+remapped, 29 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 58 MiB/s wr, 930 op/s; 28506/218601 objects degraded (13.040%); 6464/218601 objects misplaced (2.957%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T05:51:45.623 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:45.668 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:45.673 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:45.810 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:45.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:45.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:45.914 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:45.982 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:46.177 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:46.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:46.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:46.281 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:51:46.370 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 57 16 74639 74623 81.7908 57.5 0.0183236 0.0122215 2023-10-09T05:51:46.376 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:46.488 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:51:46.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:46.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:51:46.602 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:46.653 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:46.721 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:46.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:46.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:46.825 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:46.874 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:46.943 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:46.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:46.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:46.981 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [0] live_osds: [1, 2, 3, 4, 5, 6, 7] 2023-10-09T05:51:46.982 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:51:46.982 INFO:tasks.thrashosds.thrasher:Reviving osd 0 2023-10-09T05:51:46.982 INFO:tasks.cephadm.osd.0:Restarting osd.0 (starting--it wasn't running)... 2023-10-09T05:51:46.982 DEBUG:teuthology.orchestra.run.smithi064:osd.0> sudo journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.0.service 2023-10-09T05:51:46.985 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl start ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.0 2023-10-09T05:51:47.043 INFO:journalctl@ceph.osd.0.smithi064.stdout:-- Logs begin at Mon 2023-10-09 05:16:16 UTC. -- 2023-10-09T05:51:47.056 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:47.113 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:47.198 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:47.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:47.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:47.221 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:51:47 smithi064 systemd[1]: Starting Ceph osd.0 for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T05:51:47.303 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:47.339 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:47.369 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 58 16 75565 75549 81.3785 57.875 0.00997523 0.0122823 2023-10-09T05:51:47.554 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T05:51:47.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:47 smithi064 ceph-mon[212750]: pgmap v282: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 1 active+recovering+undersized+remapped, 29 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 54 MiB/s wr, 870 op/s; 28506/218601 objects degraded (13.040%); 6464/218601 objects misplaced (2.957%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T05:51:47.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:47 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1186651249' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:51:47.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:47 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 28506/218601 objects degraded (13.040%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:51:47.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:47 smithi064 ceph-mon[220044]: pgmap v282: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 1 active+recovering+undersized+remapped, 29 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 54 MiB/s wr, 870 op/s; 28506/218601 objects degraded (13.040%); 6464/218601 objects misplaced (2.957%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T05:51:47.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:47 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1186651249' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:51:47.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:47 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 28506/218601 objects degraded (13.040%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:51:47.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:47 smithi156 ceph-mon[195101]: pgmap v282: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 1 active+recovering+undersized+remapped, 29 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 54 MiB/s wr, 870 op/s; 28506/218601 objects degraded (13.040%); 6464/218601 objects misplaced (2.957%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T05:51:47.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:47 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1186651249' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:51:47.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:47 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 28506/218601 objects degraded (13.040%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:51:48.206 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:48.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:48.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:48.311 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:48.366 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:48.369 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 59 16 76388 76372 80.871 51.4375 0.0162515 0.0123605 2023-10-09T05:51:48.506 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:48.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:48.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:48.620 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:48.689 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:48.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:48 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:51:48.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:48 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1186651249' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:51:48.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:48 smithi064 ceph-mon[220044]: osdmap e126: 8 total, 7 up, 7 in 2023-10-09T05:51:48.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:48 smithi064 ceph-mon[220044]: failed to encode map e126 with expected crc 2023-10-09T05:51:48.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:48 smithi064 ceph-mon[220044]: failed to encode map e126 with expected crc 2023-10-09T05:51:48.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:48 smithi064 ceph-mon[220044]: failed to encode map e126 with expected crc 2023-10-09T05:51:48.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:48 smithi064 ceph-mon[220044]: failed to encode map e126 with expected crc 2023-10-09T05:51:48.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:48 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:51:48.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:48 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1186651249' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:51:48.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:48 smithi064 ceph-mon[212750]: osdmap e126: 8 total, 7 up, 7 in 2023-10-09T05:51:48.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:48 smithi064 ceph-mon[212750]: failed to encode map e126 with expected crc 2023-10-09T05:51:48.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:48 smithi064 ceph-mon[212750]: failed to encode map e126 with expected crc 2023-10-09T05:51:48.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:48 smithi064 ceph-mon[212750]: failed to encode map e126 with expected crc 2023-10-09T05:51:48.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:48 smithi064 ceph-mon[212750]: failed to encode map e126 with expected crc 2023-10-09T05:51:48.793 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:48.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:48.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:48.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:48 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:51:48.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:48 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1186651249' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:51:48.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:48 smithi156 ceph-mon[195101]: osdmap e126: 8 total, 7 up, 7 in 2023-10-09T05:51:48.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:48 smithi156 ceph-mon[195101]: failed to encode map e126 with expected crc 2023-10-09T05:51:48.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:48 smithi156 ceph-mon[195101]: failed to encode map e126 with expected crc 2023-10-09T05:51:48.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:48 smithi156 ceph-mon[195101]: failed to encode map e126 with expected crc 2023-10-09T05:51:48.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:48 smithi156 ceph-mon[195101]: failed to encode map e126 with expected crc 2023-10-09T05:51:48.905 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:48.986 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:49.091 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:51:49.092 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:49.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:49.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:49.121 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:51:49.146 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:51:49.179 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:51:49.203 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:49.221 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:51:49.275 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:51:49.287 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:49.326 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:51:49.327 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-10-09T05:51:49.327 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:51:49.327 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:51:49.327 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:51:49.327 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:51:49.327 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:51:49.327 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:51:49.327 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:51:49.370 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-10-09T05:51:49.370641+0000 min lat: 0.00106234 max lat: 2.26368 avg lat: 0.0124203 2023-10-09T05:51:49.370 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:51:49.370 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 60 16 77307 77291 80.4804 57.4375 0.0115268 0.0124203 2023-10-09T05:51:49.382 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:49.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:49.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:49.486 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:49.537 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:49.636 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:49.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:49 smithi064 ceph-mon[220044]: pgmap v284: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 1 active+recovering+undersized+remapped, 29 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 55 MiB/s wr, 878 op/s; 28506/219420 objects degraded (12.992%); 6464/219420 objects misplaced (2.946%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T05:51:49.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:49 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4164175699' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:49.641 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:49 smithi064 ceph-mon[212750]: pgmap v284: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 1 active+recovering+undersized+remapped, 29 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 55 MiB/s wr, 878 op/s; 28506/219420 objects degraded (12.992%); 6464/219420 objects misplaced (2.946%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T05:51:49.641 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:49 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4164175699' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:49.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:49.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:49.742 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:49.773 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:49.820 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:49.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:49.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:49.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:49 smithi156 ceph-mon[195101]: pgmap v284: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 1 active+recovering+undersized+remapped, 29 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 55 MiB/s wr, 878 op/s; 28506/219420 objects degraded (12.992%); 6464/219420 objects misplaced (2.946%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T05:51:49.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:49 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4164175699' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:51:49.923 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:49.950 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:49.969 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:51:49 smithi064 bash[296906]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T05:51:49.969 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:51:49 smithi064 bash[296906]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T05:51:49.969 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:51:49 smithi064 bash[296906]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-10-09T05:51:49.969 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:51:49 smithi064 bash[296906]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-10-09T05:51:49.969 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:51:49 smithi064 bash[296906]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-10-09T05:51:49.969 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:51:49 smithi064 bash[296906]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-10-09T05:51:49.969 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:51:49 smithi064 bash[296906]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T05:51:49.970 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:51:49 smithi064 bash[296906]: --> ceph-volume lvm activate successful for osd ID: 0 2023-10-09T05:51:50.071 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:50.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:50.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:50.180 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:50.254 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:50.370 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 61 16 77921 77905 79.7903 38.375 0.00403762 0.0125257 2023-10-09T05:51:50.392 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:50.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:50.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:50.497 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:51:50.500 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:50.588 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:50.918 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:50.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:50.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:50.939 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T05:51:50.940 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T05:51:51.037 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:51.110 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:51.140 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:51.217 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:51.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:51.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:51.332 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:51.370 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 62 16 78712 78696 79.3008 49.4375 0.016971 0.0126049 2023-10-09T05:51:51.384 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:51.511 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:51.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:51.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:51.617 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:51.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:51 smithi064 ceph-mon[220044]: pgmap v285: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 1 active+recovering+undersized+remapped, 29 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 57 MiB/s wr, 908 op/s; 29563/231714 objects degraded (12.758%); 6062/231714 objects misplaced (2.616%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T05:51:51.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:51 smithi064 ceph-mon[212750]: pgmap v285: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 1 active+recovering+undersized+remapped, 29 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 57 MiB/s wr, 908 op/s; 29563/231714 objects degraded (12.758%); 6062/231714 objects misplaced (2.616%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T05:51:51.671 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:51.830 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:51:51.830 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T05:51:51.830 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:51:51.830 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.44588.0:79100 6.7 e126/102)", 2023-10-09T05:51:51.830 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:51.821303+0000", 2023-10-09T05:51:51.830 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0028307279999999998, 2023-10-09T05:51:51.830 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0028669139999999999, 2023-10-09T05:51:51.831 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:51:51.831 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "queued for pg", 2023-10-09T05:51:51.831 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:51:51.831 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:51:51.831 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:51:51.831 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:51.821303+0000", 2023-10-09T05:51:51.831 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:51:51.831 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:51:51.831 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:51:51.831 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:51:51.832 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:51.821303+0000", 2023-10-09T05:51:51.832 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:51:51.832 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:51:51.832 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:51:51.832 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:51:51.832 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:51.821302+0000", 2023-10-09T05:51:51.832 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:51:51.832 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:51:51.832 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:51:51.832 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:51:51.833 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:51.821328+0000", 2023-10-09T05:51:51.833 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.6438999999999999e-05 2023-10-09T05:51:51.833 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:51:51.833 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:51:51.833 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:51:51.833 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:51.821345+0000", 2023-10-09T05:51:51.833 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6636000000000001e-05 2023-10-09T05:51:51.833 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:51:51.834 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:51:51.834 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:51:51.834 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:51.821352+0000", 2023-10-09T05:51:51.834 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8730000000000001e-06 2023-10-09T05:51:51.834 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:51:51.834 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:51:51.834 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:51:51.835 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:51:51.835 INFO:teuthology.orchestra.run.smithi156.stdout: ], 2023-10-09T05:51:51.835 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 1 2023-10-09T05:51:51.835 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:51:51.855 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:51 smithi156 ceph-mon[195101]: pgmap v285: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 1 active+recovering+undersized+remapped, 29 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 57 MiB/s wr, 908 op/s; 29563/231714 objects degraded (12.758%); 6062/231714 objects misplaced (2.616%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T05:51:51.965 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:51:52.061 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:52.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:52.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:52.165 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:52.219 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:51:51 smithi064 ceph-osd[298026]: -- 172.21.15.64:0/2207154578 <== mon.0 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56435ca544e0 con 0x56435ca27000 2023-10-09T05:51:52.219 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:51:51 smithi064 bash[297852]: 866dc7ecde6084c1e76264c4c155a65c17c31a9cffdadde94c346398acca4379 2023-10-09T05:51:52.219 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:51:51 smithi064 systemd[1]: Started Ceph osd.0 for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T05:51:52.231 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:52.356 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:52.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:52.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:52.370 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 63 16 79595 79579 78.9181 55.1875 0.014186 0.0126661 2023-10-09T05:51:52.468 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:51:52.499 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:52.719 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:51:52 smithi064 conmon[297966]: 2023-10-09T05:51:52.481+0000 7fb96b40a640 -1 Falling back to public interface 2023-10-09T05:51:52.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:52 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 29563/231714 objects degraded (12.758%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:51:52.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:52 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:51:52.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:52 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 29563/231714 objects degraded (12.758%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:51:52.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:52 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:51:52.854 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:51:52.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:52.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:51:52.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:52 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 29563/231714 objects degraded (12.758%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:51:52.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:52 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:51:52.941 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:51:52.964 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:53.077 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:53.174 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:53.200 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:53.203 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T05:51:53.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:53.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:53.308 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:53.358 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:53.373 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 64 16 80155 80139 78.2288 35 0.02002 0.0127756 2023-10-09T05:51:53.659 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:53.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:53.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:53.770 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:53.817 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:53.937 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:53.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:53.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:54.056 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:54.113 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:54 smithi156 ceph-mon[195101]: pgmap v286: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 1 active+recovering+undersized+remapped, 29 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 57 MiB/s wr, 906 op/s; 29563/232533 objects degraded (12.713%); 6062/232533 objects misplaced (2.607%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T05:51:54.114 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:54 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1181021138' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:51:54.114 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:54 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:51:54.118 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T05:51:54.121 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:54.149 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:51:54.373 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 65 16 80916 80900 77.7571 47.5625 0.00858991 0.0128551 2023-10-09T05:51:54.423 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:54 smithi064 ceph-mon[212750]: pgmap v286: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 1 active+recovering+undersized+remapped, 29 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 57 MiB/s wr, 906 op/s; 29563/232533 objects degraded (12.713%); 6062/232533 objects misplaced (2.607%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T05:51:54.423 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:54 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1181021138' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:51:54.423 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:54 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:51:54.424 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:54 smithi064 ceph-mon[220044]: pgmap v286: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 1 active+recovering+undersized+remapped, 29 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 57 MiB/s wr, 906 op/s; 29563/232533 objects degraded (12.713%); 6062/232533 objects misplaced (2.607%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T05:51:54.424 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:54 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1181021138' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:51:54.424 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:54 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:51:55.132 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:55.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:55.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:55.240 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:55.298 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:55 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:51:55.298 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:55 smithi156 ceph-mon[195101]: failed to encode map e127 with expected crc 2023-10-09T05:51:55.298 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:55 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:51:55.298 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:55 smithi156 ceph-mon[195101]: osdmap e127: 8 total, 7 up, 7 in 2023-10-09T05:51:55.298 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:55 smithi156 ceph-mon[195101]: failed to encode map e127 with expected crc 2023-10-09T05:51:55.298 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:55 smithi156 ceph-mon[195101]: from='osd.0 [v2:172.21.15.64:6802/1524397216,v1:172.21.15.64:6803/1524397216]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:51:55.308 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:55.373 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 66 16 81741 81725 77.3602 51.5625 0.018417 0.0129207 2023-10-09T05:51:55.469 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:51:55 smithi064 conmon[297966]: 2023-10-09T05:51:55.016+0000 7fb96b40a640 -1 osd.0 121 log_to_monitors true 2023-10-09T05:51:55.469 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 05:51:55 smithi064 conmon[297966]: 2023-10-09T05:51:55.161+0000 7fb95b073700 -1 osd.0 121 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:51:55.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:55 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:51:55.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:55 smithi064 ceph-mon[212750]: failed to encode map e127 with expected crc 2023-10-09T05:51:55.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:55 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:51:55.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:55 smithi064 ceph-mon[212750]: osdmap e127: 8 total, 7 up, 7 in 2023-10-09T05:51:55.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:55 smithi064 ceph-mon[212750]: failed to encode map e127 with expected crc 2023-10-09T05:51:55.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:55 smithi064 ceph-mon[212750]: from='osd.0 [v2:172.21.15.64:6802/1524397216,v1:172.21.15.64:6803/1524397216]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:51:55.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:55 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:51:55.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:55 smithi064 ceph-mon[220044]: failed to encode map e127 with expected crc 2023-10-09T05:51:55.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:55 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:51:55.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:55 smithi064 ceph-mon[220044]: osdmap e127: 8 total, 7 up, 7 in 2023-10-09T05:51:55.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:55 smithi064 ceph-mon[220044]: failed to encode map e127 with expected crc 2023-10-09T05:51:55.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:55 smithi064 ceph-mon[220044]: from='osd.0 [v2:172.21.15.64:6802/1524397216,v1:172.21.15.64:6803/1524397216]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:51:55.471 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:51:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:51:55] "GET /metrics HTTP/1.1" 200 36352 "" "Prometheus/2.43.0" 2023-10-09T05:51:55.929 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:55.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:55.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:56.034 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:56.153 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:56.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:56 smithi064 ceph-mon[212750]: pgmap v288: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 1 active+recovering+undersized+remapped, 29 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 55 MiB/s wr, 886 op/s; 30260/239889 objects degraded (12.614%); 5839/239889 objects misplaced (2.434%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T05:51:56.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:56 smithi064 ceph-mon[212750]: failed to encode map e128 with expected crc 2023-10-09T05:51:56.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:56 smithi064 ceph-mon[212750]: from='osd.0 [v2:172.21.15.64:6802/1524397216,v1:172.21.15.64:6803/1524397216]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T05:51:56.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:56 smithi064 ceph-mon[212750]: osdmap e128: 8 total, 7 up, 7 in 2023-10-09T05:51:56.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:56 smithi064 ceph-mon[212750]: from='osd.0 [v2:172.21.15.64:6802/1524397216,v1:172.21.15.64:6803/1524397216]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:51:56.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:56 smithi064 ceph-mon[212750]: failed to encode map e128 with expected crc 2023-10-09T05:51:56.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:56 smithi064 ceph-mon[212750]: failed to encode map e128 with expected crc 2023-10-09T05:51:56.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:56 smithi064 ceph-mon[212750]: failed to encode map e128 with expected crc 2023-10-09T05:51:56.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:56 smithi064 ceph-mon[212750]: failed to encode map e128 with expected crc 2023-10-09T05:51:56.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:56 smithi064 ceph-mon[220044]: pgmap v288: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 1 active+recovering+undersized+remapped, 29 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 55 MiB/s wr, 886 op/s; 30260/239889 objects degraded (12.614%); 5839/239889 objects misplaced (2.434%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T05:51:56.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:56 smithi064 ceph-mon[220044]: failed to encode map e128 with expected crc 2023-10-09T05:51:56.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:56 smithi064 ceph-mon[220044]: from='osd.0 [v2:172.21.15.64:6802/1524397216,v1:172.21.15.64:6803/1524397216]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T05:51:56.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:56 smithi064 ceph-mon[220044]: osdmap e128: 8 total, 7 up, 7 in 2023-10-09T05:51:56.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:56 smithi064 ceph-mon[220044]: from='osd.0 [v2:172.21.15.64:6802/1524397216,v1:172.21.15.64:6803/1524397216]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:51:56.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:56 smithi064 ceph-mon[220044]: failed to encode map e128 with expected crc 2023-10-09T05:51:56.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:56 smithi064 ceph-mon[220044]: failed to encode map e128 with expected crc 2023-10-09T05:51:56.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:56 smithi064 ceph-mon[220044]: failed to encode map e128 with expected crc 2023-10-09T05:51:56.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:56 smithi064 ceph-mon[220044]: failed to encode map e128 with expected crc 2023-10-09T05:51:56.291 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:56.293 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:56 smithi156 ceph-mon[195101]: pgmap v288: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 1 active+recovering+undersized+remapped, 29 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 55 MiB/s wr, 886 op/s; 30260/239889 objects degraded (12.614%); 5839/239889 objects misplaced (2.434%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T05:51:56.293 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:56 smithi156 ceph-mon[195101]: failed to encode map e128 with expected crc 2023-10-09T05:51:56.293 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:56 smithi156 ceph-mon[195101]: from='osd.0 [v2:172.21.15.64:6802/1524397216,v1:172.21.15.64:6803/1524397216]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T05:51:56.293 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:56 smithi156 ceph-mon[195101]: osdmap e128: 8 total, 7 up, 7 in 2023-10-09T05:51:56.294 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:56 smithi156 ceph-mon[195101]: from='osd.0 [v2:172.21.15.64:6802/1524397216,v1:172.21.15.64:6803/1524397216]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T05:51:56.294 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:56 smithi156 ceph-mon[195101]: failed to encode map e128 with expected crc 2023-10-09T05:51:56.294 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:56 smithi156 ceph-mon[195101]: failed to encode map e128 with expected crc 2023-10-09T05:51:56.294 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:56 smithi156 ceph-mon[195101]: failed to encode map e128 with expected crc 2023-10-09T05:51:56.294 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:56 smithi156 ceph-mon[195101]: failed to encode map e128 with expected crc 2023-10-09T05:51:56.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:56.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:56.305 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:56.373 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 67 16 82142 82126 76.5799 25.0625 0.00310087 0.0130182 2023-10-09T05:51:56.395 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:51:56.446 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:56.649 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:51:56.649 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T05:51:56.649 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.649 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82093 6.0 6.a3a5430 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:51:56.649 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.146555+0000", 2023-10-09T05:51:56.649 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.50199363100000005, 2023-10-09T05:51:56.650 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.50201932000000005, 2023-10-09T05:51:56.650 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:56.650 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "delayed", 2023-10-09T05:51:56.650 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:56.650 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:56.650 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:56.650 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82093 2023-10-09T05:51:56.650 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.650 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:56.650 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.650 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:56.651 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.146555+0000", 2023-10-09T05:51:56.651 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:56.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.651 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.651 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:56.651 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.146555+0000", 2023-10-09T05:51:56.651 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:56.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.651 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.652 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:56.652 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.146553+0000", 2023-10-09T05:51:56.652 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:51:56.652 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.652 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:56.652 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.146607+0000", 2023-10-09T05:51:56.652 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4317000000000002e-05 2023-10-09T05:51:56.652 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.652 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:56.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.146636+0000", 2023-10-09T05:51:56.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8294999999999999e-05 2023-10-09T05:51:56.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.653 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.653 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:56.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.146647+0000", 2023-10-09T05:51:56.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1513e-05 2023-10-09T05:51:56.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.653 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.653 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:56.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.150385+0000", 2023-10-09T05:51:56.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037372640000000001 2023-10-09T05:51:56.654 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.654 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.654 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:51:56.654 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.150398+0000", 2023-10-09T05:51:56.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.34e-05 2023-10-09T05:51:56.654 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:56.654 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:56.654 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:56.654 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.655 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.655 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82107 6.0 6.432cde0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:51:56.655 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.173816+0000", 2023-10-09T05:51:56.655 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.47473225899999999, 2023-10-09T05:51:56.655 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.47481975700000001, 2023-10-09T05:51:56.655 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:56.655 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "delayed", 2023-10-09T05:51:56.655 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:56.655 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:56.655 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:56.655 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82107 2023-10-09T05:51:56.655 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.656 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:56.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.656 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:56.656 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173816+0000", 2023-10-09T05:51:56.656 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:56.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.656 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:56.656 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173816+0000", 2023-10-09T05:51:56.656 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:56.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:56.657 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173814+0000", 2023-10-09T05:51:56.657 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:51:56.657 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.657 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:56.657 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173867+0000", 2023-10-09T05:51:56.657 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2862999999999999e-05 2023-10-09T05:51:56.657 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.657 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:56.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173887+0000", 2023-10-09T05:51:56.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0347e-05 2023-10-09T05:51:56.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.658 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:56.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173907+0000", 2023-10-09T05:51:56.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9782000000000001e-05 2023-10-09T05:51:56.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.658 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:56.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173983+0000", 2023-10-09T05:51:56.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6056000000000005e-05 2023-10-09T05:51:56.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:51:56.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173996+0000", 2023-10-09T05:51:56.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3237999999999999e-05 2023-10-09T05:51:56.659 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:56.659 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:56.659 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:56.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.659 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82109 6.0 6.cee0ea60 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:51:56.659 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.183890+0000", 2023-10-09T05:51:56.660 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.464658126, 2023-10-09T05:51:56.660 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.46478641799999998, 2023-10-09T05:51:56.660 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:56.660 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "delayed", 2023-10-09T05:51:56.660 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:56.660 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:56.660 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:56.660 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82109 2023-10-09T05:51:56.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.660 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:56.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.661 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:56.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.183890+0000", 2023-10-09T05:51:56.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:56.661 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.661 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.661 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:56.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.183890+0000", 2023-10-09T05:51:56.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:56.661 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.661 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.661 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:56.662 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.183889+0000", 2023-10-09T05:51:56.662 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:51:56.662 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.662 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.662 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:56.662 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.184035+0000", 2023-10-09T05:51:56.662 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000145375 2023-10-09T05:51:56.662 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.662 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.662 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:56.662 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.184048+0000", 2023-10-09T05:51:56.663 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3730999999999999e-05 2023-10-09T05:51:56.663 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.663 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.663 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:56.663 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.184060+0000", 2023-10-09T05:51:56.663 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1338000000000001e-05 2023-10-09T05:51:56.663 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.663 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.663 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:56.663 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.184099+0000", 2023-10-09T05:51:56.663 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9183000000000001e-05 2023-10-09T05:51:56.663 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.664 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.664 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:51:56.664 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.184113+0000", 2023-10-09T05:51:56.664 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3750999999999999e-05 2023-10-09T05:51:56.664 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:56.664 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:56.664 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:56.664 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.664 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.664 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82116 6.9 6.318f1fa9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:51:56.664 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.194331+0000", 2023-10-09T05:51:56.665 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.45421742300000001, 2023-10-09T05:51:56.665 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.45438440600000002, 2023-10-09T05:51:56.665 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:56.665 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "delayed", 2023-10-09T05:51:56.665 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:56.665 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:56.665 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:56.665 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82116 2023-10-09T05:51:56.665 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.665 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:56.665 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.665 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:56.666 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194331+0000", 2023-10-09T05:51:56.666 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:56.666 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.666 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.666 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:56.666 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194331+0000", 2023-10-09T05:51:56.666 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:56.666 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.666 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.666 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:56.666 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194329+0000", 2023-10-09T05:51:56.667 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:51:56.667 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.667 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.667 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:56.667 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194387+0000", 2023-10-09T05:51:56.667 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7734e-05 2023-10-09T05:51:56.667 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.667 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.667 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:56.668 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194406+0000", 2023-10-09T05:51:56.668 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9304e-05 2023-10-09T05:51:56.668 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.668 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.668 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:56.668 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194417+0000", 2023-10-09T05:51:56.668 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0803e-05 2023-10-09T05:51:56.668 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.668 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.668 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:56.668 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194490+0000", 2023-10-09T05:51:56.669 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2939000000000001e-05 2023-10-09T05:51:56.669 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.669 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.669 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:51:56.669 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194504+0000", 2023-10-09T05:51:56.669 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4358e-05 2023-10-09T05:51:56.669 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:56.669 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:56.669 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:56.669 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.669 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.669 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82117 6.9 6.b6ae8c09 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:51:56.670 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.200001+0000", 2023-10-09T05:51:56.670 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.448547317, 2023-10-09T05:51:56.670 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.44876030700000002, 2023-10-09T05:51:56.670 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:56.670 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "delayed", 2023-10-09T05:51:56.670 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:56.670 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:56.670 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:56.670 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82117 2023-10-09T05:51:56.670 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.670 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:56.671 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.671 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:56.671 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200001+0000", 2023-10-09T05:51:56.671 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:56.671 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.671 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.671 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:56.671 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200001+0000", 2023-10-09T05:51:56.671 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:56.671 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.671 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.671 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:56.672 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.199999+0000", 2023-10-09T05:51:56.672 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:51:56.672 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.672 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.672 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:56.672 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200060+0000", 2023-10-09T05:51:56.672 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0708999999999998e-05 2023-10-09T05:51:56.672 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.672 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.672 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:56.672 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200090+0000", 2023-10-09T05:51:56.672 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0085000000000001e-05 2023-10-09T05:51:56.673 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.673 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.673 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:56.673 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200101+0000", 2023-10-09T05:51:56.673 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.06e-05 2023-10-09T05:51:56.673 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.673 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.673 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:56.673 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200179+0000", 2023-10-09T05:51:56.673 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7848000000000004e-05 2023-10-09T05:51:56.673 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.673 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.674 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:51:56.674 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200192+0000", 2023-10-09T05:51:56.674 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3943e-05 2023-10-09T05:51:56.674 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:56.674 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:56.674 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:56.674 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.674 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.674 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82118 6.9 6.aad68289 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:51:56.674 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.200117+0000", 2023-10-09T05:51:56.674 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.44843181599999998, 2023-10-09T05:51:56.674 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.44868296699999999, 2023-10-09T05:51:56.675 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:56.675 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "delayed", 2023-10-09T05:51:56.675 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:56.675 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:56.675 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:56.675 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82118 2023-10-09T05:51:56.675 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.675 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:56.675 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.675 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:56.675 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200117+0000", 2023-10-09T05:51:56.675 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:56.676 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.676 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.676 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:56.676 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200117+0000", 2023-10-09T05:51:56.676 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:56.676 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.676 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.676 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:56.676 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200116+0000", 2023-10-09T05:51:56.676 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:51:56.676 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.677 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.677 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:56.677 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200162+0000", 2023-10-09T05:51:56.677 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5788999999999998e-05 2023-10-09T05:51:56.677 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.677 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.677 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:56.677 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200174+0000", 2023-10-09T05:51:56.677 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1905999999999999e-05 2023-10-09T05:51:56.677 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.677 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.677 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:56.678 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200181+0000", 2023-10-09T05:51:56.678 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2690000000000002e-06 2023-10-09T05:51:56.678 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.678 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.678 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:56.678 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200224+0000", 2023-10-09T05:51:56.678 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2839000000000001e-05 2023-10-09T05:51:56.678 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.678 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.678 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:51:56.678 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200233+0000", 2023-10-09T05:51:56.678 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2490000000000007e-06 2023-10-09T05:51:56.679 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:56.679 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:56.679 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:56.679 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.679 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.679 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82125 6.0 6.dade3890 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:51:56.679 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.207895+0000", 2023-10-09T05:51:56.679 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.440653613, 2023-10-09T05:51:56.679 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.44094213100000001, 2023-10-09T05:51:56.679 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:56.679 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "delayed", 2023-10-09T05:51:56.679 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:56.680 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:56.680 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:56.680 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82125 2023-10-09T05:51:56.680 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.680 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:56.680 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.680 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:56.680 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.207895+0000", 2023-10-09T05:51:56.680 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:56.680 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.680 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.680 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:56.680 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.207895+0000", 2023-10-09T05:51:56.681 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:56.681 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.681 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.681 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:56.681 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.207893+0000", 2023-10-09T05:51:56.681 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:51:56.681 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.681 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.681 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:56.681 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.207955+0000", 2023-10-09T05:51:56.681 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2361999999999997e-05 2023-10-09T05:51:56.681 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.682 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.682 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:56.682 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.207982+0000", 2023-10-09T05:51:56.682 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6838e-05 2023-10-09T05:51:56.682 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.682 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.682 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:56.682 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.207995+0000", 2023-10-09T05:51:56.682 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2992e-05 2023-10-09T05:51:56.682 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.682 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.683 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:56.683 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.208064+0000", 2023-10-09T05:51:56.683 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9366e-05 2023-10-09T05:51:56.683 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.683 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.683 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:51:56.683 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.208080+0000", 2023-10-09T05:51:56.683 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5690000000000001e-05 2023-10-09T05:51:56.683 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:56.683 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:56.683 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:56.683 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.684 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.684 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82137 6.0 6.90177890 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:51:56.684 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.236426+0000", 2023-10-09T05:51:56.684 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.41212243999999998, 2023-10-09T05:51:56.684 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.41245018700000002, 2023-10-09T05:51:56.684 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:56.684 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "delayed", 2023-10-09T05:51:56.684 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:56.684 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:56.684 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:56.684 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82137 2023-10-09T05:51:56.684 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.685 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:56.685 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.685 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:56.685 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236426+0000", 2023-10-09T05:51:56.685 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:56.685 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.685 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.685 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:56.685 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236426+0000", 2023-10-09T05:51:56.685 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:56.685 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.685 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.686 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:56.686 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236424+0000", 2023-10-09T05:51:56.686 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:51:56.686 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.686 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.686 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:56.686 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236475+0000", 2023-10-09T05:51:56.686 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0819999999999998e-05 2023-10-09T05:51:56.686 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.686 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.686 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:56.686 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236499+0000", 2023-10-09T05:51:56.687 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3669e-05 2023-10-09T05:51:56.687 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.687 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.687 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:56.687 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236510+0000", 2023-10-09T05:51:56.687 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1258e-05 2023-10-09T05:51:56.687 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.687 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.687 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:56.687 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236585+0000", 2023-10-09T05:51:56.687 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4828999999999998e-05 2023-10-09T05:51:56.687 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.688 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.688 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:51:56.688 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236601+0000", 2023-10-09T05:51:56.688 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6308999999999999e-05 2023-10-09T05:51:56.688 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:56.688 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:56.688 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:56.688 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.688 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.688 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82139 6.0 6.c02e1490 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:51:56.688 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.237890+0000", 2023-10-09T05:51:56.688 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.41065906899999999, 2023-10-09T05:51:56.689 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.41102389, 2023-10-09T05:51:56.689 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:56.689 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "delayed", 2023-10-09T05:51:56.689 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:56.689 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:56.689 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:56.689 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82139 2023-10-09T05:51:56.689 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.689 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:56.689 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.689 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:56.689 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.237890+0000", 2023-10-09T05:51:56.690 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:56.690 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.690 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.690 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:56.690 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.237890+0000", 2023-10-09T05:51:56.690 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:56.690 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.690 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.690 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:56.690 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.237889+0000", 2023-10-09T05:51:56.690 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T05:51:56.690 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.691 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.691 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:56.691 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.237927+0000", 2023-10-09T05:51:56.691 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8469e-05 2023-10-09T05:51:56.691 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.691 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.691 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:56.691 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.237940+0000", 2023-10-09T05:51:56.691 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2994e-05 2023-10-09T05:51:56.691 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.691 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.691 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:56.692 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.237947+0000", 2023-10-09T05:51:56.692 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.652e-06 2023-10-09T05:51:56.692 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.692 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.692 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:56.692 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.237996+0000", 2023-10-09T05:51:56.692 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8798000000000003e-05 2023-10-09T05:51:56.692 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.692 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.692 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:51:56.692 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.238008+0000", 2023-10-09T05:51:56.692 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2527000000000001e-05 2023-10-09T05:51:56.693 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:56.693 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:56.693 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:56.693 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.693 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.693 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82142 6.9 6.3a5bcf49 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:51:56.693 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.245410+0000", 2023-10-09T05:51:56.693 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.403138306, 2023-10-09T05:51:56.693 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.40353999699999998, 2023-10-09T05:51:56.693 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:51:56.693 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "delayed", 2023-10-09T05:51:56.693 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:51:56.694 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:51:56.694 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:51:56.694 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82142 2023-10-09T05:51:56.694 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.694 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:51:56.694 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.694 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:51:56.694 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.245410+0000", 2023-10-09T05:51:56.694 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:56.694 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.694 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.695 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:51:56.695 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.245410+0000", 2023-10-09T05:51:56.695 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:51:56.695 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.695 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.695 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:51:56.695 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.245409+0000", 2023-10-09T05:51:56.695 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:51:56.695 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.695 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.695 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:51:56.695 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.245459+0000", 2023-10-09T05:51:56.696 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0399000000000003e-05 2023-10-09T05:51:56.696 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.696 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.696 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:51:56.696 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.245479+0000", 2023-10-09T05:51:56.696 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9783999999999999e-05 2023-10-09T05:51:56.696 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.696 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.696 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:51:56.696 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.245488+0000", 2023-10-09T05:51:56.696 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3899999999999999e-06 2023-10-09T05:51:56.697 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.697 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.697 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:51:56.697 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.245546+0000", 2023-10-09T05:51:56.697 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7602999999999997e-05 2023-10-09T05:51:56.697 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:51:56.697 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:51:56.697 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:51:56.697 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.245555+0000", 2023-10-09T05:51:56.697 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2439999999999999e-06 2023-10-09T05:51:56.697 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:56.698 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:51:56.698 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:56.698 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:51:56.698 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-10-09T05:51:56.698 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 10 2023-10-09T05:51:56.698 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:51:56.801 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:51:56.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:56.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:51:56.849 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:51:56.849 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T05:51:56.850 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T05:51:56.850 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T05:51:56.850 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:51:56.904 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:56.942 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:57.307 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:57 smithi156 ceph-mon[195101]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:51:57.307 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:57 smithi156 ceph-mon[195101]: osd.0 [v2:172.21.15.64:6802/1524397216,v1:172.21.15.64:6803/1524397216] boot 2023-10-09T05:51:57.307 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:57 smithi156 ceph-mon[195101]: osdmap e129: 8 total, 8 up, 7 in 2023-10-09T05:51:57.307 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:57 smithi156 ceph-mon[195101]: failed to encode map e129 with expected crc 2023-10-09T05:51:57.307 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:57 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:51:57.308 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:57 smithi156 ceph-mon[195101]: failed to encode map e129 with expected crc 2023-10-09T05:51:57.308 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:57 smithi156 ceph-mon[195101]: failed to encode map e129 with expected crc 2023-10-09T05:51:57.308 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:57 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 30260/239889 objects degraded (12.614%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:51:57.308 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:57 smithi156 ceph-mon[195101]: osdmap e130: 8 total, 8 up, 7 in 2023-10-09T05:51:57.373 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 68 16 82142 82126 75.454 0 - 0.0130182 2023-10-09T05:51:57.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:57 smithi064 ceph-mon[212750]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:51:57.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:57 smithi064 ceph-mon[212750]: osd.0 [v2:172.21.15.64:6802/1524397216,v1:172.21.15.64:6803/1524397216] boot 2023-10-09T05:51:57.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:57 smithi064 ceph-mon[212750]: osdmap e129: 8 total, 8 up, 7 in 2023-10-09T05:51:57.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:57 smithi064 ceph-mon[212750]: failed to encode map e129 with expected crc 2023-10-09T05:51:57.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:57 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:51:57.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:57 smithi064 ceph-mon[212750]: failed to encode map e129 with expected crc 2023-10-09T05:51:57.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:57 smithi064 ceph-mon[212750]: failed to encode map e129 with expected crc 2023-10-09T05:51:57.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:57 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 30260/239889 objects degraded (12.614%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:51:57.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:57 smithi064 ceph-mon[212750]: osdmap e130: 8 total, 8 up, 7 in 2023-10-09T05:51:57.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:57 smithi064 ceph-mon[220044]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:51:57.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:57 smithi064 ceph-mon[220044]: osd.0 [v2:172.21.15.64:6802/1524397216,v1:172.21.15.64:6803/1524397216] boot 2023-10-09T05:51:57.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:57 smithi064 ceph-mon[220044]: osdmap e129: 8 total, 8 up, 7 in 2023-10-09T05:51:57.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:57 smithi064 ceph-mon[220044]: failed to encode map e129 with expected crc 2023-10-09T05:51:57.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:57 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:51:57.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:57 smithi064 ceph-mon[220044]: failed to encode map e129 with expected crc 2023-10-09T05:51:57.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:57 smithi064 ceph-mon[220044]: failed to encode map e129 with expected crc 2023-10-09T05:51:57.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:57 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 30260/239889 objects degraded (12.614%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:51:57.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:57 smithi064 ceph-mon[220044]: osdmap e130: 8 total, 8 up, 7 in 2023-10-09T05:51:57.708 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:57.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:57.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:57.811 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:57.850 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:57.898 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T05:51:58.052 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:58.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:58.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:58.160 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:58.198 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:58.206 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:51:58.246 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[212750]: pgmap v291: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 1 active+recovering+undersized+remapped, 29 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 38 MiB/s wr, 606 op/s; 30475/242631 objects degraded (12.560%); 5651/242631 objects misplaced (2.329%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T05:51:58.247 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[212750]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.247 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[212750]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.247 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[212750]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.247 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[212750]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.247 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[212750]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.247 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[212750]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.247 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[212750]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.247 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[212750]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.247 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[212750]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.247 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[212750]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.248 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[212750]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.248 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[212750]: osdmap e131: 8 total, 8 up, 7 in 2023-10-09T05:51:58.248 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[220044]: pgmap v291: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 1 active+recovering+undersized+remapped, 29 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 38 MiB/s wr, 606 op/s; 30475/242631 objects degraded (12.560%); 5651/242631 objects misplaced (2.329%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T05:51:58.248 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[220044]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.248 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[220044]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.248 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[220044]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.248 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[220044]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.248 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[220044]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.248 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[220044]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.248 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[220044]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.249 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[220044]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.249 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[220044]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.249 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[220044]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.249 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[220044]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.249 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:58 smithi064 ceph-mon[220044]: osdmap e131: 8 total, 8 up, 7 in 2023-10-09T05:51:58.257 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:51:58.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:58.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:51:58.273 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:58 smithi156 ceph-mon[195101]: pgmap v291: 99 pgs: 1 active+recovering+degraded+remapped, 53 active+clean, 1 active+recovering+undersized+remapped, 29 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 38 MiB/s wr, 606 op/s; 30475/242631 objects degraded (12.560%); 5651/242631 objects misplaced (2.329%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T05:51:58.274 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:58 smithi156 ceph-mon[195101]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.274 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:58 smithi156 ceph-mon[195101]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.274 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:58 smithi156 ceph-mon[195101]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.274 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:58 smithi156 ceph-mon[195101]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.274 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:58 smithi156 ceph-mon[195101]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.274 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:58 smithi156 ceph-mon[195101]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.274 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:58 smithi156 ceph-mon[195101]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.274 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:58 smithi156 ceph-mon[195101]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.274 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:58 smithi156 ceph-mon[195101]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.275 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:58 smithi156 ceph-mon[195101]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.275 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:58 smithi156 ceph-mon[195101]: failed to encode map e130 with expected crc 2023-10-09T05:51:58.275 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:58 smithi156 ceph-mon[195101]: osdmap e131: 8 total, 8 up, 7 in 2023-10-09T05:51:58.371 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:51:58.374 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 69 16 82240 82224 74.4483 3.0625 0.0173856 0.0134275 2023-10-09T05:51:58.453 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:58.651 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:51:58.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:58.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:51:58.686 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:51:58.758 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:51:58.828 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:58.993 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:51:58.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:58.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:51:59.014 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:51:59.097 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:51:59.180 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:59.304 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:59 smithi156 ceph-mon[195101]: failed to encode map e131 with expected crc 2023-10-09T05:51:59.304 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:59 smithi156 ceph-mon[195101]: failed to encode map e131 with expected crc 2023-10-09T05:51:59.304 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:59 smithi156 ceph-mon[195101]: failed to encode map e131 with expected crc 2023-10-09T05:51:59.304 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:59 smithi156 ceph-mon[195101]: failed to encode map e131 with expected crc 2023-10-09T05:51:59.304 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:59 smithi156 ceph-mon[195101]: failed to encode map e131 with expected crc 2023-10-09T05:51:59.304 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:59 smithi156 ceph-mon[195101]: failed to encode map e131 with expected crc 2023-10-09T05:51:59.304 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:59 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:51:59.304 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:51:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:51:59.322 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:51:59.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:51:59.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:51:59.374 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 70 16 83006 82990 74.0687 47.875 0.0310194 0.013495 2023-10-09T05:51:59.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:59 smithi064 ceph-mon[212750]: failed to encode map e131 with expected crc 2023-10-09T05:51:59.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:59 smithi064 ceph-mon[212750]: failed to encode map e131 with expected crc 2023-10-09T05:51:59.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:59 smithi064 ceph-mon[212750]: failed to encode map e131 with expected crc 2023-10-09T05:51:59.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:59 smithi064 ceph-mon[212750]: failed to encode map e131 with expected crc 2023-10-09T05:51:59.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:59 smithi064 ceph-mon[212750]: failed to encode map e131 with expected crc 2023-10-09T05:51:59.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:59 smithi064 ceph-mon[212750]: failed to encode map e131 with expected crc 2023-10-09T05:51:59.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:59 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:51:59.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:51:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:51:59.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:59 smithi064 ceph-mon[220044]: failed to encode map e131 with expected crc 2023-10-09T05:51:59.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:59 smithi064 ceph-mon[220044]: failed to encode map e131 with expected crc 2023-10-09T05:51:59.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:59 smithi064 ceph-mon[220044]: failed to encode map e131 with expected crc 2023-10-09T05:51:59.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:59 smithi064 ceph-mon[220044]: failed to encode map e131 with expected crc 2023-10-09T05:51:59.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:59 smithi064 ceph-mon[220044]: failed to encode map e131 with expected crc 2023-10-09T05:51:59.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:59 smithi064 ceph-mon[220044]: failed to encode map e131 with expected crc 2023-10-09T05:51:59.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:59 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:51:59.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:51:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:51:59.426 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:51:59.427 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:51:59.465 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:51:59.504 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:51:59.509 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:51:59.538 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:51:59.574 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:51:59.618 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:51:59.661 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:51:59.701 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:51:59.716 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:51:59.717 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:51:59.717 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:51:59.717 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:51:59.717 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:51:59.717 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:51:59.717 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:51:59.717 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:52:00.281 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:00.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:00.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:00.374 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 71 16 83716 83700 73.6506 44.375 0.0133022 0.0135726 2023-10-09T05:52:00.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:00.426 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:00 smithi064 ceph-mon[220044]: pgmap v294: 99 pgs: 2 remapped+peering, 1 active+recovering+degraded+remapped, 57 active+clean, 1 active+recovering+undersized+remapped, 25 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 10 active+undersized+degraded; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 19 MiB/s wr, 311 op/s; 20299/243627 objects degraded (8.332%); 5651/243627 objects misplaced (2.320%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:52:00.427 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:00 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3418339668' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:00.429 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:00 smithi064 ceph-mon[212750]: pgmap v294: 99 pgs: 2 remapped+peering, 1 active+recovering+degraded+remapped, 57 active+clean, 1 active+recovering+undersized+remapped, 25 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 10 active+undersized+degraded; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 19 MiB/s wr, 311 op/s; 20299/243627 objects degraded (8.332%); 5651/243627 objects misplaced (2.320%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:52:00.429 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:00 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3418339668' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:00.434 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:00.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:00 smithi156 ceph-mon[195101]: pgmap v294: 99 pgs: 2 remapped+peering, 1 active+recovering+degraded+remapped, 57 active+clean, 1 active+recovering+undersized+remapped, 25 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 10 active+undersized+degraded; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 19 MiB/s wr, 311 op/s; 20299/243627 objects degraded (8.332%); 5651/243627 objects misplaced (2.320%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:52:00.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:00 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3418339668' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:01.138 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T05:52:01.139 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:52:01.367 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:01.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:01.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:01.374 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 72 16 84480 84464 73.2909 47.75 0.0224586 0.0136386 2023-10-09T05:52:01.470 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:01.519 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:01.794 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:01.799 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:01 smithi064 ceph-mon[212750]: pgmap v295: 99 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 91 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 24 MiB/s wr, 377 op/s; 15541/248700 objects degraded (6.249%); 2056/248700 objects misplaced (0.827%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T05:52:01.799 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:01 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:01.799 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:01 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:01.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:01.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:01.804 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:01 smithi064 ceph-mon[220044]: pgmap v295: 99 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 91 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 24 MiB/s wr, 377 op/s; 15541/248700 objects degraded (6.249%); 2056/248700 objects misplaced (0.827%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T05:52:01.804 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:01 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:01.804 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:01 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:01.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:01 smithi156 ceph-mon[195101]: pgmap v295: 99 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 91 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 24 MiB/s wr, 377 op/s; 15541/248700 objects degraded (6.249%); 2056/248700 objects misplaced (0.827%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T05:52:01.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:01 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:01.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:01 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:01.901 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:01.976 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:02.065 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:52:02.141 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:02.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:02.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:02.248 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:02.289 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:02.375 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 73 16 85279 85263 72.971 49.9375 0.0283974 0.013699 2023-10-09T05:52:02.473 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:02.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:02.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:02.579 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:02.654 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:02.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:02 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 15541/248700 objects degraded (6.249%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:02.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:02 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 15541/248700 objects degraded (6.249%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:02.735 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:52:02.754 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:02.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:02.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:02.859 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:02.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:02 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 15541/248700 objects degraded (6.249%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:02.951 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:03.234 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:03.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:03.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:03.343 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:03.375 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 74 16 86104 86088 72.6817 51.5625 0.012483 0.013755 2023-10-09T05:52:03.403 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:03.493 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:03.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:03.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:03.605 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:03.617 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:52:03.687 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T05:52:03.687 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:52:03.687 INFO:tasks.thrashosds.thrasher:Adding osd 1 2023-10-09T05:52:03.688 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd in 1 2023-10-09T05:52:03.688 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:03.834 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:03.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:03.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:03.938 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:03.985 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:04.143 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T05:52:04.238 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:04 smithi156 ceph-mon[195101]: pgmap v296: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 25 MiB/s wr, 404 op/s; 17893/250329 objects degraded (7.148%); 2056/250329 objects misplaced (0.821%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T05:52:04.238 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:04 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1769318794' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:52:04.239 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:04 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:52:04.239 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:04 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:04.239 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:04 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:04.274 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:04.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:04.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:04.347 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:04 smithi064 ceph-mon[212750]: pgmap v296: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 25 MiB/s wr, 404 op/s; 17893/250329 objects degraded (7.148%); 2056/250329 objects misplaced (0.821%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T05:52:04.348 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:04 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1769318794' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:52:04.348 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:04 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:52:04.348 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:04 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:04.348 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:04 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:04.348 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:04 smithi064 ceph-mon[220044]: pgmap v296: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 25 MiB/s wr, 404 op/s; 17893/250329 objects degraded (7.148%); 2056/250329 objects misplaced (0.821%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T05:52:04.348 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:04 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1769318794' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:52:04.348 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:04 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:52:04.348 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:04 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:04.349 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:04 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:04.375 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 75 16 86839 86823 72.3252 45.9375 0.0149874 0.013822 2023-10-09T05:52:04.395 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:04.458 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:04.565 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:04.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:04.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:04.671 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:04.766 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:04.901 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:04.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:04.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:04.974 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T05:52:04.974 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:52:05.015 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:05.069 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:05.132 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:05.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:05.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:05.235 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:05.284 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:05.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:05 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:52:05.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:05 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:05.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:05 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:52:05.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:05 smithi156 ceph-mon[195101]: osdmap e132: 8 total, 8 up, 7 in 2023-10-09T05:52:05.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:05 smithi156 ceph-mon[195101]: failed to encode map e132 with expected crc 2023-10-09T05:52:05.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:05 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:52:05.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:05 smithi156 ceph-mon[195101]: failed to encode map e132 with expected crc 2023-10-09T05:52:05.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:05 smithi156 ceph-mon[195101]: failed to encode map e132 with expected crc 2023-10-09T05:52:05.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:05 smithi156 ceph-mon[195101]: failed to encode map e132 with expected crc 2023-10-09T05:52:05.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:05 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:05.375 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 76 16 87648 87632 72.0389 50.5625 0.0136902 0.0138766 2023-10-09T05:52:05.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:52:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:52:05] "GET /metrics HTTP/1.1" 200 36205 "" "Prometheus/2.43.0" 2023-10-09T05:52:05.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:05 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:52:05.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:05 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:05.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:05 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:52:05.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:05 smithi064 ceph-mon[220044]: osdmap e132: 8 total, 8 up, 7 in 2023-10-09T05:52:05.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:05 smithi064 ceph-mon[220044]: failed to encode map e132 with expected crc 2023-10-09T05:52:05.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:05 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:52:05.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:05 smithi064 ceph-mon[220044]: failed to encode map e132 with expected crc 2023-10-09T05:52:05.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:05 smithi064 ceph-mon[220044]: failed to encode map e132 with expected crc 2023-10-09T05:52:05.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:05 smithi064 ceph-mon[220044]: failed to encode map e132 with expected crc 2023-10-09T05:52:05.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:05 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:05.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:05 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:52:05.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:05 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:05.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:05 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:52:05.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:05 smithi064 ceph-mon[212750]: osdmap e132: 8 total, 8 up, 7 in 2023-10-09T05:52:05.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:05 smithi064 ceph-mon[212750]: failed to encode map e132 with expected crc 2023-10-09T05:52:05.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:05 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:52:05.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:05 smithi064 ceph-mon[212750]: failed to encode map e132 with expected crc 2023-10-09T05:52:05.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:05 smithi064 ceph-mon[212750]: failed to encode map e132 with expected crc 2023-10-09T05:52:05.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:05 smithi064 ceph-mon[212750]: failed to encode map e132 with expected crc 2023-10-09T05:52:05.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:05 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:05.501 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:05.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:05.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:05.604 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:05.684 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:05.767 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:05.824 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:05.887 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:05.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:05.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:05.924 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:52:05.992 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:06.090 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:06.168 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:06 smithi064 ceph-mon[220044]: pgmap v298: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 671 op/s; 17893/257547 objects degraded (6.947%); 2056/257547 objects misplaced (0.798%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T05:52:06.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:06 smithi064 ceph-mon[212750]: pgmap v298: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 671 op/s; 17893/257547 objects degraded (6.947%); 2056/257547 objects misplaced (0.798%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T05:52:06.269 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:06.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:06.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:06.279 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:06 smithi156 ceph-mon[195101]: pgmap v298: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 671 op/s; 17893/257547 objects degraded (6.947%); 2056/257547 objects misplaced (0.798%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T05:52:06.375 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 77 16 88337 88321 71.6626 43.0625 0.011195 0.0139497 2023-10-09T05:52:06.380 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:06.471 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:06.581 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:06.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:06.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:06.659 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:52:06.701 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:06.756 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:06.917 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:06.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:06.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:07.030 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:07.094 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:07.287 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:07 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 17893/257547 objects degraded (6.947%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:07.375 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 78 16 89051 89035 71.316 44.625 0.0104647 0.0140164 2023-10-09T05:52:07.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:07 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 17893/257547 objects degraded (6.947%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:07.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:07 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 17893/257547 objects degraded (6.947%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:07.622 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:52:08.093 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:08.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:08.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:08.204 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:08.281 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:08.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:08 smithi156 ceph-mon[195101]: pgmap v299: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 40 MiB/s wr, 637 op/s; 17893/259701 objects degraded (6.890%); 1796/259701 objects misplaced (0.692%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T05:52:08.376 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 79 16 89718 89702 70.9406 41.6875 0.020021 0.0140911 2023-10-09T05:52:08.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:08 smithi064 ceph-mon[212750]: pgmap v299: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 40 MiB/s wr, 637 op/s; 17893/259701 objects degraded (6.890%); 1796/259701 objects misplaced (0.692%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T05:52:08.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:08 smithi064 ceph-mon[220044]: pgmap v299: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 40 MiB/s wr, 637 op/s; 17893/259701 objects degraded (6.890%); 1796/259701 objects misplaced (0.692%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T05:52:08.490 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:08.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:08.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:08.599 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:08.681 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:08.789 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:08.801 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T05:52:08.801 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:52:08.801 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:08.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:08.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:08.918 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:09.027 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:09.098 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:52:09.159 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:52:09.186 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:52:09.199 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:09.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:09.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:09.229 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:52:09.253 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:52:09.270 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:09 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1195969199' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:09.270 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:09 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1195969199' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:09.271 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:09 smithi064 conmon[212726]: 2023-10-09T05:52:09.193+0000 7f8823f23700 -1 mon.a@0(leader).osd e132 definitely_dead 0 2023-10-09T05:52:09.310 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:09.339 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:09.378 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-10-09T05:52:09.379306+0000 min lat: 0.00106234 max lat: 2.26368 avg lat: 0.0141437 2023-10-09T05:52:09.379 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:52:09.379 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 80 16 90398 90382 70.5829 42.5 0.156712 0.0141437 2023-10-09T05:52:09.411 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:52:09.458 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:52:09.505 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:52:09.518 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:52:09.518 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:52:09.518 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:52:09.518 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:52:09.518 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:52:09.518 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:52:09.518 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:52:09.518 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:52:09.539 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:09.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:09.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:09.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:09 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1195969199' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:09.645 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:09.726 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:52:09.727 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:09.873 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:09.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:09.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:09.996 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:10.076 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:10.125 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:10.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:10.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:10.203 INFO:teuthology.orchestra.run.smithi064.stderr:marked in osd.1. 2023-10-09T05:52:10.230 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:10.272 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:10.312 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:52:10.326 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:10.330 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:10 smithi156 ceph-mon[195101]: pgmap v300: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 38 MiB/s wr, 608 op/s; 17893/261885 objects degraded (6.832%); 1796/261885 objects misplaced (0.686%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T05:52:10.330 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:10 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/903971169' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T05:52:10.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:10.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:10.378 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 81 16 91061 91045 70.2231 41.4375 0.00389563 0.0142088 2023-10-09T05:52:10.432 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:10.459 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:10.468 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:10 smithi064 ceph-mon[212750]: pgmap v300: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 38 MiB/s wr, 608 op/s; 17893/261885 objects degraded (6.832%); 1796/261885 objects misplaced (0.686%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T05:52:10.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:10 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/903971169' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T05:52:10.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:10 smithi064 ceph-mon[220044]: pgmap v300: 99 pgs: 1 active+recovering+undersized+remapped, 91 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 38 MiB/s wr, 608 op/s; 17893/261885 objects degraded (6.832%); 1796/261885 objects misplaced (0.686%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T05:52:10.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:10 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/903971169' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T05:52:10.558 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:10.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:10.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:10.662 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:10.693 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:11.379 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 82 16 91061 91045 69.3669 0 - 0.0142088 2023-10-09T05:52:11.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:11 smithi064 ceph-mon[220044]: failed to encode map e133 with expected crc 2023-10-09T05:52:11.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:11 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/903971169' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T05:52:11.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:11 smithi064 ceph-mon[220044]: osdmap e133: 8 total, 8 up, 8 in 2023-10-09T05:52:11.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:11 smithi064 ceph-mon[220044]: failed to encode map e133 with expected crc 2023-10-09T05:52:11.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:11 smithi064 ceph-mon[220044]: failed to encode map e133 with expected crc 2023-10-09T05:52:11.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:11 smithi064 ceph-mon[220044]: failed to encode map e133 with expected crc 2023-10-09T05:52:11.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:11 smithi064 ceph-mon[220044]: failed to encode map e133 with expected crc 2023-10-09T05:52:11.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:11 smithi064 ceph-mon[220044]: pgmap v302: 99 pgs: 10 peering, 1 unknown, 1 active+recovering+undersized+remapped, 80 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 838 op/s; 17893/270516 objects degraded (6.614%); 1524/270516 objects misplaced (0.563%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T05:52:11.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:11 smithi064 ceph-mon[212750]: failed to encode map e133 with expected crc 2023-10-09T05:52:11.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:11 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/903971169' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T05:52:11.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:11 smithi064 ceph-mon[212750]: osdmap e133: 8 total, 8 up, 8 in 2023-10-09T05:52:11.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:11 smithi064 ceph-mon[212750]: failed to encode map e133 with expected crc 2023-10-09T05:52:11.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:11 smithi064 ceph-mon[212750]: failed to encode map e133 with expected crc 2023-10-09T05:52:11.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:11 smithi064 ceph-mon[212750]: failed to encode map e133 with expected crc 2023-10-09T05:52:11.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:11 smithi064 ceph-mon[212750]: failed to encode map e133 with expected crc 2023-10-09T05:52:11.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:11 smithi064 ceph-mon[212750]: pgmap v302: 99 pgs: 10 peering, 1 unknown, 1 active+recovering+undersized+remapped, 80 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 838 op/s; 17893/270516 objects degraded (6.614%); 1524/270516 objects misplaced (0.563%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T05:52:11.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:11 smithi156 ceph-mon[195101]: failed to encode map e133 with expected crc 2023-10-09T05:52:11.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:11 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/903971169' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T05:52:11.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:11 smithi156 ceph-mon[195101]: osdmap e133: 8 total, 8 up, 8 in 2023-10-09T05:52:11.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:11 smithi156 ceph-mon[195101]: failed to encode map e133 with expected crc 2023-10-09T05:52:11.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:11 smithi156 ceph-mon[195101]: failed to encode map e133 with expected crc 2023-10-09T05:52:11.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:11 smithi156 ceph-mon[195101]: failed to encode map e133 with expected crc 2023-10-09T05:52:11.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:11 smithi156 ceph-mon[195101]: failed to encode map e133 with expected crc 2023-10-09T05:52:11.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:11 smithi156 ceph-mon[195101]: pgmap v302: 99 pgs: 10 peering, 1 unknown, 1 active+recovering+undersized+remapped, 80 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 838 op/s; 17893/270516 objects degraded (6.614%); 1524/270516 objects misplaced (0.563%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T05:52:11.689 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:11.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:11.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:11.801 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:11.889 INFO:tasks.thrashosds.thrasher:Added osd 1 2023-10-09T05:52:11.910 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:12.049 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:12.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:12.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:12.162 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:12.194 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:52:12.194 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[212750]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.194 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[212750]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.194 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[212750]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.194 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[212750]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[212750]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[212750]: osdmap e134: 8 total, 8 up, 8 in 2023-10-09T05:52:12.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[212750]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[212750]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[212750]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[212750]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[212750]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 17893/270516 objects degraded (6.614%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:12.214 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:12.326 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:12.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:12.335 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:12 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:52:12.336 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:12 smithi156 ceph-mon[195101]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.336 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:12 smithi156 ceph-mon[195101]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.336 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:12 smithi156 ceph-mon[195101]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.336 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:12 smithi156 ceph-mon[195101]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.336 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:12 smithi156 ceph-mon[195101]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.336 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:12 smithi156 ceph-mon[195101]: osdmap e134: 8 total, 8 up, 8 in 2023-10-09T05:52:12.336 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:12 smithi156 ceph-mon[195101]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.336 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:12 smithi156 ceph-mon[195101]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.336 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:12 smithi156 ceph-mon[195101]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.336 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:12 smithi156 ceph-mon[195101]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.336 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:12 smithi156 ceph-mon[195101]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.337 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:12 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 17893/270516 objects degraded (6.614%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:12.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:12.379 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 83 16 91797 91781 69.0854 23 0.00848131 0.014469 2023-10-09T05:52:12.390 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:52:12.390 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T05:52:12.390 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T05:52:12.390 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T05:52:12.390 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.390 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64028 6.d 6:b2a5011c:::benchmark_data_smithi064_284193_object64027:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:52:12.391 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.138056+0000", 2023-10-09T05:52:12.391 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 39.247676179000003, 2023-10-09T05:52:12.391 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0914284259999998, 2023-10-09T05:52:12.391 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:12.391 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:12.391 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:12.391 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:12.391 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:12.391 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64028 2023-10-09T05:52:12.391 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.391 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:12.391 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.392 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:12.392 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.138056+0000", 2023-10-09T05:52:12.392 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.392 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.392 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.392 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:12.392 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.138056+0000", 2023-10-09T05:52:12.392 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.392 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.392 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.392 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:12.392 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.138055+0000", 2023-10-09T05:52:12.393 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:12.393 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.393 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.393 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:12.393 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.138098+0000", 2023-10-09T05:52:12.393 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3702000000000003e-05 2023-10-09T05:52:12.393 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.393 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.393 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:12.393 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.138112+0000", 2023-10-09T05:52:12.393 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3393999999999999e-05 2023-10-09T05:52:12.394 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.394 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.394 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:12.394 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.138119+0000", 2023-10-09T05:52:12.394 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.905e-06 2023-10-09T05:52:12.394 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.394 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.394 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.394 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.138179+0000", 2023-10-09T05:52:12.394 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0426e-05 2023-10-09T05:52:12.394 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.394 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.395 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.395 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.138187+0000", 2023-10-09T05:52:12.395 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.3680000000000002e-06 2023-10-09T05:52:12.395 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.395 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.395 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.395 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111163+0000", 2023-10-09T05:52:12.395 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.97297587900000004 2023-10-09T05:52:12.395 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.395 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.395 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.396 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111172+0000", 2023-10-09T05:52:12.396 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.4999999999999999e-06 2023-10-09T05:52:12.396 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.396 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.396 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.396 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.204580+0000", 2023-10-09T05:52:12.396 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0934087079999999 2023-10-09T05:52:12.396 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.396 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.396 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:12.396 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.204699+0000", 2023-10-09T05:52:12.396 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000118097 2023-10-09T05:52:12.397 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.397 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.397 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:52:12.397 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.204810+0000", 2023-10-09T05:52:12.397 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011183099999999999 2023-10-09T05:52:12.397 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.397 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.397 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:12.397 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.211455+0000", 2023-10-09T05:52:12.397 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0066445150000000001 2023-10-09T05:52:12.397 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.397 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.398 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.398 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.227513+0000", 2023-10-09T05:52:12.398 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.016057795999999999 2023-10-09T05:52:12.398 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.398 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.398 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.398 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.227594+0000", 2023-10-09T05:52:12.398 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.174e-05 2023-10-09T05:52:12.398 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.398 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.398 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:12.399 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.227636+0000", 2023-10-09T05:52:12.399 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2018999999999999e-05 2023-10-09T05:52:12.399 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.399 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.399 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:12.399 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.229485+0000", 2023-10-09T05:52:12.399 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0018480969999999999 2023-10-09T05:52:12.399 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.399 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:12.399 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.399 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.399 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.400 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64042 6.d 6:b4a627c6:::benchmark_data_smithi064_284193_object64041:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:52:12.400 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.148881+0000", 2023-10-09T05:52:12.400 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 39.236851588, 2023-10-09T05:52:12.400 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.083266171, 2023-10-09T05:52:12.400 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:12.400 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:12.400 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:12.400 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:12.400 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:12.400 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64042 2023-10-09T05:52:12.400 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.400 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:12.401 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.401 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:12.401 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.148881+0000", 2023-10-09T05:52:12.401 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.401 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.401 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.401 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:12.401 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.148881+0000", 2023-10-09T05:52:12.401 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.401 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.401 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.401 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:12.402 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.148879+0000", 2023-10-09T05:52:12.402 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:12.402 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.402 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.402 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:12.402 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.148915+0000", 2023-10-09T05:52:12.402 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5583999999999997e-05 2023-10-09T05:52:12.402 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.402 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.402 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:12.402 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.148925+0000", 2023-10-09T05:52:12.403 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.9150000000000008e-06 2023-10-09T05:52:12.403 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.403 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.403 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:12.403 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.148930+0000", 2023-10-09T05:52:12.403 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2100000000000001e-06 2023-10-09T05:52:12.403 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.403 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.403 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.403 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.149088+0000", 2023-10-09T05:52:12.403 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015745000000000001 2023-10-09T05:52:12.403 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.404 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.404 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.404 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.149096+0000", 2023-10-09T05:52:12.404 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.5560000000000008e-06 2023-10-09T05:52:12.404 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.404 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.404 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.404 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111207+0000", 2023-10-09T05:52:12.405 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96211053099999999 2023-10-09T05:52:12.405 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.405 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.405 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.405 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111214+0000", 2023-10-09T05:52:12.405 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3189999999999997e-06 2023-10-09T05:52:12.405 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.405 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.405 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.405 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.206599+0000", 2023-10-09T05:52:12.406 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.095385359 2023-10-09T05:52:12.406 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.406 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.406 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:12.406 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.206696+0000", 2023-10-09T05:52:12.406 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.6280000000000007e-05 2023-10-09T05:52:12.406 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.406 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.406 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:52:12.406 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.206803+0000", 2023-10-09T05:52:12.407 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000107219 2023-10-09T05:52:12.407 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.407 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.407 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:12.409 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.211465+0000", 2023-10-09T05:52:12.409 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0046617569999999999 2023-10-09T05:52:12.409 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.409 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.409 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.409 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.227565+0000", 2023-10-09T05:52:12.409 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.016100023000000001 2023-10-09T05:52:12.409 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.409 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.409 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.409 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.229614+0000", 2023-10-09T05:52:12.410 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0020497720000000001 2023-10-09T05:52:12.410 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.410 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.410 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:12.410 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.229681+0000", 2023-10-09T05:52:12.410 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6727e-05 2023-10-09T05:52:12.410 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.410 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.410 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:12.410 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.232147+0000", 2023-10-09T05:52:12.410 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0024657419999999999 2023-10-09T05:52:12.411 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.411 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:12.411 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.411 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.411 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.412 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64051 6.d 6:b4ee6d36:::benchmark_data_smithi064_284193_object64050:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:52:12.412 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.160656+0000", 2023-10-09T05:52:12.412 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 39.225075863999997, 2023-10-09T05:52:12.412 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0740513780000001, 2023-10-09T05:52:12.412 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:12.412 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:12.412 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:12.412 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:12.412 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:12.412 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64051 2023-10-09T05:52:12.413 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.413 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:12.413 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.413 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:12.413 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.160656+0000", 2023-10-09T05:52:12.413 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.413 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.413 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.413 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:12.413 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.160656+0000", 2023-10-09T05:52:12.414 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.414 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.414 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.414 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:12.414 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.160656+0000", 2023-10-09T05:52:12.414 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:52:12.414 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.414 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.415 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:12.415 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.160686+0000", 2023-10-09T05:52:12.415 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0577000000000002e-05 2023-10-09T05:52:12.415 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.415 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.415 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:12.415 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.160694+0000", 2023-10-09T05:52:12.415 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4800000000000004e-06 2023-10-09T05:52:12.415 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.416 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.416 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:12.416 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.160698+0000", 2023-10-09T05:52:12.416 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2119999999999997e-06 2023-10-09T05:52:12.416 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.416 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.416 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.416 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.160752+0000", 2023-10-09T05:52:12.416 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4283999999999997e-05 2023-10-09T05:52:12.416 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.417 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.417 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.417 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.160762+0000", 2023-10-09T05:52:12.417 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.4539999999999996e-06 2023-10-09T05:52:12.417 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.417 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.417 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.417 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111247+0000", 2023-10-09T05:52:12.417 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.95048511800000002 2023-10-09T05:52:12.418 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.418 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.418 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.418 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111254+0000", 2023-10-09T05:52:12.418 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4229999999999999e-06 2023-10-09T05:52:12.418 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.418 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.418 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.418 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.207847+0000", 2023-10-09T05:52:12.419 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.096592206 2023-10-09T05:52:12.419 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.419 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.419 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:12.419 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.207986+0000", 2023-10-09T05:52:12.419 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000138961 2023-10-09T05:52:12.419 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.420 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.420 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:52:12.420 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.208089+0000", 2023-10-09T05:52:12.420 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010323900000000001 2023-10-09T05:52:12.420 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.420 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.420 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:12.420 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.213812+0000", 2023-10-09T05:52:12.420 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0057226880000000001 2023-10-09T05:52:12.421 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.421 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.421 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.421 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.229558+0000", 2023-10-09T05:52:12.421 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015746514999999999 2023-10-09T05:52:12.421 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.421 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.421 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.421 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.232222+0000", 2023-10-09T05:52:12.421 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0026640000000000001 2023-10-09T05:52:12.421 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.422 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.422 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:12.422 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.232292+0000", 2023-10-09T05:52:12.422 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9648999999999997e-05 2023-10-09T05:52:12.423 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.423 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.423 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:12.423 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.234708+0000", 2023-10-09T05:52:12.423 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0024161170000000002 2023-10-09T05:52:12.423 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.423 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:12.424 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.424 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.425 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.425 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64056 6.d 6:bb5661cc:::benchmark_data_smithi064_284193_object64055:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:52:12.426 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.165123+0000", 2023-10-09T05:52:12.426 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 39.220609684999999, 2023-10-09T05:52:12.426 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0726746980000001, 2023-10-09T05:52:12.426 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:12.426 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:12.427 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:12.427 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:12.427 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:12.427 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64056 2023-10-09T05:52:12.427 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.427 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:12.428 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.428 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:12.428 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.165123+0000", 2023-10-09T05:52:12.428 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.428 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.428 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.428 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:12.428 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.165123+0000", 2023-10-09T05:52:12.429 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.429 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.429 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.429 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:12.429 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.165122+0000", 2023-10-09T05:52:12.429 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:52:12.429 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.429 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.429 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:12.429 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.165150+0000", 2023-10-09T05:52:12.429 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7784999999999999e-05 2023-10-09T05:52:12.429 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.430 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.430 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:12.430 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.165158+0000", 2023-10-09T05:52:12.430 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1100000000000003e-06 2023-10-09T05:52:12.430 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.430 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.430 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:12.430 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.165163+0000", 2023-10-09T05:52:12.430 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5889999999999996e-06 2023-10-09T05:52:12.430 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.430 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.431 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.431 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.165204+0000", 2023-10-09T05:52:12.431 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0961000000000003e-05 2023-10-09T05:52:12.431 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.431 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.431 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.431 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.165211+0000", 2023-10-09T05:52:12.431 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1849999999999998e-06 2023-10-09T05:52:12.431 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.431 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.432 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.432 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111284+0000", 2023-10-09T05:52:12.432 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.94607339800000001 2023-10-09T05:52:12.432 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.432 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.432 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.432 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111289+0000", 2023-10-09T05:52:12.432 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1630000000000004e-06 2023-10-09T05:52:12.432 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.432 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.432 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.432 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.209079+0000", 2023-10-09T05:52:12.433 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.097789296 2023-10-09T05:52:12.433 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.433 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.433 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:12.433 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.209170+0000", 2023-10-09T05:52:12.433 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1420000000000002e-05 2023-10-09T05:52:12.433 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.433 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.433 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:52:12.433 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.209274+0000", 2023-10-09T05:52:12.433 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000103553 2023-10-09T05:52:12.434 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.434 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.434 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:12.434 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.213841+0000", 2023-10-09T05:52:12.434 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0045676650000000003 2023-10-09T05:52:12.434 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.434 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.434 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.434 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.234777+0000", 2023-10-09T05:52:12.434 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.020935958000000001 2023-10-09T05:52:12.434 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.435 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.435 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.435 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.234979+0000", 2023-10-09T05:52:12.435 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00020143200000000001 2023-10-09T05:52:12.435 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.435 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.435 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:12.435 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.235052+0000", 2023-10-09T05:52:12.435 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3369999999999997e-05 2023-10-09T05:52:12.435 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.435 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.435 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:12.436 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.237797+0000", 2023-10-09T05:52:12.436 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00274537 2023-10-09T05:52:12.436 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.436 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:12.436 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.436 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.436 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.436 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64062 6.d 6:bfc7fb96:::benchmark_data_smithi064_284193_object64061:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:52:12.436 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.176528+0000", 2023-10-09T05:52:12.436 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 39.209203955, 2023-10-09T05:52:12.436 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.063752225, 2023-10-09T05:52:12.437 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:12.437 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:12.437 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:12.437 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:12.437 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:12.437 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64062 2023-10-09T05:52:12.437 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.437 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:12.437 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.437 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:12.437 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.176528+0000", 2023-10-09T05:52:12.437 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.438 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.438 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.438 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:12.438 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.176528+0000", 2023-10-09T05:52:12.438 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.438 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.438 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.438 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:12.441 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.176527+0000", 2023-10-09T05:52:12.441 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:52:12.441 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.441 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.441 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:12.441 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.176580+0000", 2023-10-09T05:52:12.442 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2935000000000001e-05 2023-10-09T05:52:12.442 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.442 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.442 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:12.442 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.176592+0000", 2023-10-09T05:52:12.442 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1481000000000001e-05 2023-10-09T05:52:12.443 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.443 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.443 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:12.443 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.176598+0000", 2023-10-09T05:52:12.443 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9540000000000001e-06 2023-10-09T05:52:12.443 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.443 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.444 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.444 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.176646+0000", 2023-10-09T05:52:12.444 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7753999999999997e-05 2023-10-09T05:52:12.444 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.444 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.444 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.444 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.176653+0000", 2023-10-09T05:52:12.445 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8259999999999993e-06 2023-10-09T05:52:12.445 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.445 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.445 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.445 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111323+0000", 2023-10-09T05:52:12.445 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.93466992699999996 2023-10-09T05:52:12.445 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.446 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.446 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.446 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111329+0000", 2023-10-09T05:52:12.446 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7699999999999998e-06 2023-10-09T05:52:12.446 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.446 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.447 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.447 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.210262+0000", 2023-10-09T05:52:12.447 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.098933036 2023-10-09T05:52:12.447 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.447 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.447 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:12.447 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.210386+0000", 2023-10-09T05:52:12.448 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012342000000000001 2023-10-09T05:52:12.448 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.448 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.448 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:52:12.448 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.210493+0000", 2023-10-09T05:52:12.448 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000106988 2023-10-09T05:52:12.450 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.450 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.450 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:12.450 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.216295+0000", 2023-10-09T05:52:12.450 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0058020320000000004 2023-10-09T05:52:12.450 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.450 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.450 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.450 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.234811+0000", 2023-10-09T05:52:12.450 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018516469000000001 2023-10-09T05:52:12.451 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.451 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.451 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.451 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.237872+0000", 2023-10-09T05:52:12.451 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.003061032 2023-10-09T05:52:12.451 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.451 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.451 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:12.451 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.237919+0000", 2023-10-09T05:52:12.451 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7448999999999999e-05 2023-10-09T05:52:12.451 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.451 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.452 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:12.452 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.240281+0000", 2023-10-09T05:52:12.452 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0023610879999999999 2023-10-09T05:52:12.452 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.452 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:12.452 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.452 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.452 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.452 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64070 6.d 6:bb0f4d69:::benchmark_data_smithi064_284193_object64069:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:52:12.452 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.203536+0000", 2023-10-09T05:52:12.452 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 39.182196617000002, 2023-10-09T05:52:12.452 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.039812205, 2023-10-09T05:52:12.453 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:12.453 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:12.453 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:12.453 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:12.453 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:12.453 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64070 2023-10-09T05:52:12.453 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.453 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:12.453 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.453 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:12.453 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.203536+0000", 2023-10-09T05:52:12.453 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.454 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.458 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.458 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:12.458 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.203536+0000", 2023-10-09T05:52:12.458 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.458 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.459 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.459 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:12.459 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.203534+0000", 2023-10-09T05:52:12.459 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:12.459 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.459 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.459 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:12.459 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.203598+0000", 2023-10-09T05:52:12.459 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4401999999999998e-05 2023-10-09T05:52:12.459 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.459 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.459 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:12.460 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.203617+0000", 2023-10-09T05:52:12.460 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8428999999999999e-05 2023-10-09T05:52:12.460 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.460 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.460 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:12.460 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.203624+0000", 2023-10-09T05:52:12.460 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5000000000000002e-06 2023-10-09T05:52:12.460 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.460 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.460 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.460 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.203672+0000", 2023-10-09T05:52:12.460 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7793000000000001e-05 2023-10-09T05:52:12.461 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.461 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.461 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.461 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.203679+0000", 2023-10-09T05:52:12.461 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7750000000000004e-06 2023-10-09T05:52:12.461 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.461 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.461 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.461 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111355+0000", 2023-10-09T05:52:12.461 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.90767584199999995 2023-10-09T05:52:12.461 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.461 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.462 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.462 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111360+0000", 2023-10-09T05:52:12.462 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1970000000000004e-06 2023-10-09T05:52:12.462 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.462 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.462 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.462 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.211500+0000", 2023-10-09T05:52:12.462 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1001402220000001 2023-10-09T05:52:12.462 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.462 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.462 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:12.463 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.211590+0000", 2023-10-09T05:52:12.463 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9363000000000005e-05 2023-10-09T05:52:12.463 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.463 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.463 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:52:12.463 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.211698+0000", 2023-10-09T05:52:12.463 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000108789 2023-10-09T05:52:12.463 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.463 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.463 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:12.463 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.216302+0000", 2023-10-09T05:52:12.463 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0046034869999999999 2023-10-09T05:52:12.464 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.464 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.464 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.464 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.240331+0000", 2023-10-09T05:52:12.464 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.024028898 2023-10-09T05:52:12.464 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.464 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.464 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.464 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.240426+0000", 2023-10-09T05:52:12.464 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.5624000000000005e-05 2023-10-09T05:52:12.464 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.464 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.465 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:12.465 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.240483+0000", 2023-10-09T05:52:12.465 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6329999999999998e-05 2023-10-09T05:52:12.465 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.465 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.465 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:12.465 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.243348+0000", 2023-10-09T05:52:12.465 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002865214 2023-10-09T05:52:12.465 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.465 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:12.465 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.465 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.466 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.466 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64074 6.d 6:b61b5635:::benchmark_data_smithi064_284193_object64073:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-10-09T05:52:12.466 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:34.202267+0000", 2023-10-09T05:52:12.466 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 38.183464989999997, 2023-10-09T05:52:12.466 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.044023162, 2023-10-09T05:52:12.466 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:12.466 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:12.466 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:12.466 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:12.466 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:12.466 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64074 2023-10-09T05:52:12.466 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.467 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:12.467 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.467 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:12.467 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.202267+0000", 2023-10-09T05:52:12.467 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.467 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.467 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.467 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:12.467 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.202267+0000", 2023-10-09T05:52:12.467 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.467 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.467 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.467 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:12.468 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.202266+0000", 2023-10-09T05:52:12.468 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:12.468 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.468 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.468 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:12.468 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.203900+0000", 2023-10-09T05:52:12.468 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0016342100000000001 2023-10-09T05:52:12.468 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.468 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.468 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:12.468 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.203913+0000", 2023-10-09T05:52:12.468 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2914999999999999e-05 2023-10-09T05:52:12.469 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.469 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.469 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:12.469 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.203920+0000", 2023-10-09T05:52:12.469 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.443e-06 2023-10-09T05:52:12.469 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.469 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.469 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.469 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220752+0000", 2023-10-09T05:52:12.469 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.016832783 2023-10-09T05:52:12.469 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.469 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.470 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.470 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220761+0000", 2023-10-09T05:52:12.470 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6410000000000008e-06 2023-10-09T05:52:12.470 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.470 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.470 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.470 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.212668+0000", 2023-10-09T05:52:12.470 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99190736599999996 2023-10-09T05:52:12.470 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.470 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.470 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:12.470 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.212794+0000", 2023-10-09T05:52:12.470 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012561199999999999 2023-10-09T05:52:12.471 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.471 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.471 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:52:12.471 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.212886+0000", 2023-10-09T05:52:12.471 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2281000000000006e-05 2023-10-09T05:52:12.471 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.471 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.471 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:12.471 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.220148+0000", 2023-10-09T05:52:12.471 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0072622019999999997 2023-10-09T05:52:12.471 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.471 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.472 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.472 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.240385+0000", 2023-10-09T05:52:12.472 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.020236849000000001 2023-10-09T05:52:12.472 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.472 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.472 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.472 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.243402+0000", 2023-10-09T05:52:12.472 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0030166049999999999 2023-10-09T05:52:12.472 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.472 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.472 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:12.472 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.243449+0000", 2023-10-09T05:52:12.473 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7592000000000001e-05 2023-10-09T05:52:12.473 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.473 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.473 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:12.473 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.246290+0000", 2023-10-09T05:52:12.473 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0028410290000000001 2023-10-09T05:52:12.473 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.473 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:12.473 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.473 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.473 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.473 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64081 6.d 6:be2ec322:::benchmark_data_smithi064_284193_object64080:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-10-09T05:52:12.474 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:34.204234+0000", 2023-10-09T05:52:12.474 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 38.181498167999997, 2023-10-09T05:52:12.474 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0455772409999999, 2023-10-09T05:52:12.474 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:12.474 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:12.474 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:12.474 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:12.474 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:12.474 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64081 2023-10-09T05:52:12.474 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.474 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:12.475 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.475 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:12.475 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204234+0000", 2023-10-09T05:52:12.475 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.475 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.475 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.475 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:12.475 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204234+0000", 2023-10-09T05:52:12.475 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.475 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.475 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.475 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:12.476 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204233+0000", 2023-10-09T05:52:12.476 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:52:12.476 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.476 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.476 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:12.476 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204263+0000", 2023-10-09T05:52:12.476 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9601e-05 2023-10-09T05:52:12.476 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.476 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.476 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:12.476 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204271+0000", 2023-10-09T05:52:12.476 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8860000000000004e-06 2023-10-09T05:52:12.476 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.477 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.477 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:12.477 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204277+0000", 2023-10-09T05:52:12.477 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0140000000000004e-06 2023-10-09T05:52:12.477 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.477 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.477 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.477 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220790+0000", 2023-10-09T05:52:12.477 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.016513109000000002 2023-10-09T05:52:12.477 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.477 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.477 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.478 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220796+0000", 2023-10-09T05:52:12.478 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0090000000000004e-06 2023-10-09T05:52:12.478 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.478 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.478 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.478 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.213877+0000", 2023-10-09T05:52:12.478 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99308116199999996 2023-10-09T05:52:12.478 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.478 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.478 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:12.478 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.213975+0000", 2023-10-09T05:52:12.478 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7417999999999995e-05 2023-10-09T05:52:12.479 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.479 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.479 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:52:12.479 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.214074+0000", 2023-10-09T05:52:12.479 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.9256e-05 2023-10-09T05:52:12.479 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.479 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.479 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:12.479 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.220163+0000", 2023-10-09T05:52:12.479 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0060896270000000002 2023-10-09T05:52:12.479 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.479 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.480 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.480 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.246362+0000", 2023-10-09T05:52:12.480 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.026198430000000002 2023-10-09T05:52:12.480 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.480 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.480 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.480 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.246630+0000", 2023-10-09T05:52:12.480 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00026832000000000001 2023-10-09T05:52:12.480 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.480 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.480 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:12.480 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.246689+0000", 2023-10-09T05:52:12.481 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8675000000000003e-05 2023-10-09T05:52:12.481 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.481 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.481 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:12.481 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.249811+0000", 2023-10-09T05:52:12.481 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.003122465 2023-10-09T05:52:12.481 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.481 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:12.481 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.481 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.481 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.481 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64089 6.d 6:b58e5e7c:::benchmark_data_smithi064_284193_object64088:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-10-09T05:52:12.482 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:34.213048+0000", 2023-10-09T05:52:12.482 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 38.172684525000001, 2023-10-09T05:52:12.482 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0395069539999999, 2023-10-09T05:52:12.482 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:12.482 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:12.482 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:12.482 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:12.482 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:12.482 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64089 2023-10-09T05:52:12.482 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.482 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:12.482 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.483 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:12.483 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.213048+0000", 2023-10-09T05:52:12.483 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.483 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.483 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.483 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:12.483 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.213048+0000", 2023-10-09T05:52:12.483 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.483 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.483 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.483 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:12.483 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.213047+0000", 2023-10-09T05:52:12.484 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:52:12.484 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.484 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.484 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:12.484 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.213081+0000", 2023-10-09T05:52:12.484 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.4516e-05 2023-10-09T05:52:12.484 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.484 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.484 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:12.484 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.213094+0000", 2023-10-09T05:52:12.484 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2349e-05 2023-10-09T05:52:12.484 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.484 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.485 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:12.485 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.213099+0000", 2023-10-09T05:52:12.485 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7429999999999999e-06 2023-10-09T05:52:12.485 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.485 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.485 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.485 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220817+0000", 2023-10-09T05:52:12.485 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0077176060000000001 2023-10-09T05:52:12.485 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.485 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.485 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.485 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220823+0000", 2023-10-09T05:52:12.486 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6699999999999999e-06 2023-10-09T05:52:12.486 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.486 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.486 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.486 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.215043+0000", 2023-10-09T05:52:12.486 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99422076999999998 2023-10-09T05:52:12.486 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.486 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.486 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:12.486 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.215180+0000", 2023-10-09T05:52:12.486 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000136131 2023-10-09T05:52:12.486 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.487 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.487 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:52:12.487 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.215271+0000", 2023-10-09T05:52:12.487 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1936999999999997e-05 2023-10-09T05:52:12.487 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.487 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.487 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:12.487 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.222469+0000", 2023-10-09T05:52:12.487 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0071980409999999996 2023-10-09T05:52:12.487 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.487 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.487 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.488 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.246413+0000", 2023-10-09T05:52:12.488 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.023943939000000001 2023-10-09T05:52:12.488 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.488 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.488 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.488 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.249860+0000", 2023-10-09T05:52:12.488 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0034468910000000001 2023-10-09T05:52:12.488 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.488 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.488 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:12.488 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.249897+0000", 2023-10-09T05:52:12.488 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7141999999999998e-05 2023-10-09T05:52:12.489 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.489 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.489 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:12.489 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.252555+0000", 2023-10-09T05:52:12.489 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002657263 2023-10-09T05:52:12.489 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.489 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:12.489 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.489 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.489 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.489 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64093 6.d 6:b073101d:::benchmark_data_smithi064_284193_object64092:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-10-09T05:52:12.489 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:34.220415+0000", 2023-10-09T05:52:12.490 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 38.165316955999998, 2023-10-09T05:52:12.490 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.035115665, 2023-10-09T05:52:12.490 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:12.490 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:12.490 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:12.490 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:12.490 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:12.490 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64093 2023-10-09T05:52:12.490 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.490 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:12.490 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.491 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:12.491 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220415+0000", 2023-10-09T05:52:12.491 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.491 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.491 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.491 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:12.491 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220415+0000", 2023-10-09T05:52:12.491 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.491 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.491 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.491 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:12.491 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220409+0000", 2023-10-09T05:52:12.492 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999938 2023-10-09T05:52:12.492 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.492 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.492 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:12.492 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220472+0000", 2023-10-09T05:52:12.492 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3229000000000001e-05 2023-10-09T05:52:12.492 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.492 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.492 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:12.492 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220484+0000", 2023-10-09T05:52:12.492 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1467999999999999e-05 2023-10-09T05:52:12.492 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.493 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.493 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:12.493 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220491+0000", 2023-10-09T05:52:12.493 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9809999999999997e-06 2023-10-09T05:52:12.493 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.493 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.493 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.493 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220841+0000", 2023-10-09T05:52:12.493 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00035016700000000002 2023-10-09T05:52:12.493 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.493 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.493 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.493 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220846+0000", 2023-10-09T05:52:12.494 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2079999999999999e-06 2023-10-09T05:52:12.494 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.494 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.494 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.494 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.216340+0000", 2023-10-09T05:52:12.494 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99549343000000001 2023-10-09T05:52:12.494 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.494 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.494 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:12.494 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.216439+0000", 2023-10-09T05:52:12.494 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.8918000000000004e-05 2023-10-09T05:52:12.495 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.495 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.495 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:52:12.495 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.216552+0000", 2023-10-09T05:52:12.495 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000113495 2023-10-09T05:52:12.495 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.495 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.495 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:12.495 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.222481+0000", 2023-10-09T05:52:12.495 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0059289099999999999 2023-10-09T05:52:12.495 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.495 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.496 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.496 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.246451+0000", 2023-10-09T05:52:12.496 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.02396968 2023-10-09T05:52:12.496 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.496 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.496 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.496 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.252814+0000", 2023-10-09T05:52:12.496 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0063631360000000001 2023-10-09T05:52:12.496 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.496 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.496 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:12.496 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.252851+0000", 2023-10-09T05:52:12.497 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7106e-05 2023-10-09T05:52:12.497 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.497 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.497 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:12.497 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.255531+0000", 2023-10-09T05:52:12.497 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002680095 2023-10-09T05:52:12.497 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.497 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:12.497 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.497 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.497 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.497 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:82074 6.a 6:58f4f3d0:::benchmark_data_smithi064_284193_object82073:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T05:52:12.498 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:56.097757+0000", 2023-10-09T05:52:12.498 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 16.287975041999999, 2023-10-09T05:52:12.498 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.158105742, 2023-10-09T05:52:12.498 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:12.498 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T05:52:12.498 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:12.498 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:12.498 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:12.498 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 82074 2023-10-09T05:52:12.498 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.498 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:12.498 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.499 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:12.499 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.097757+0000", 2023-10-09T05:52:12.499 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.499 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.499 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.499 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:12.499 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.097757+0000", 2023-10-09T05:52:12.499 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.499 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.499 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.499 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:12.499 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.097756+0000", 2023-10-09T05:52:12.500 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:12.500 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.500 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.500 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:12.500 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.097799+0000", 2023-10-09T05:52:12.500 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3735000000000001e-05 2023-10-09T05:52:12.500 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.500 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.500 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:12.500 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.097817+0000", 2023-10-09T05:52:12.500 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7289e-05 2023-10-09T05:52:12.500 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.501 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.501 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:12.501 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.097824+0000", 2023-10-09T05:52:12.501 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1319999999999996e-06 2023-10-09T05:52:12.501 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.501 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.501 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.501 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.100302+0000", 2023-10-09T05:52:12.501 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0024780340000000001 2023-10-09T05:52:12.501 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.501 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.501 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:12.502 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.100452+0000", 2023-10-09T05:52:12.502 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000149863 2023-10-09T05:52:12.502 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.502 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.502 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 6", 2023-10-09T05:52:12.502 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.100616+0000", 2023-10-09T05:52:12.502 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000164673 2023-10-09T05:52:12.502 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.502 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.502 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:12.502 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.106430+0000", 2023-10-09T05:52:12.502 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0058139039999999999 2023-10-09T05:52:12.503 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.503 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.503 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.503 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.144922+0000", 2023-10-09T05:52:12.503 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.038491535 2023-10-09T05:52:12.503 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.503 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.503 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.503 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.144937+0000", 2023-10-09T05:52:12.503 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5568000000000001e-05 2023-10-09T05:52:12.503 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.504 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.504 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.504 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088545+0000", 2023-10-09T05:52:12.504 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.94360759900000002 2023-10-09T05:52:12.504 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.504 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.504 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.504 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088550+0000", 2023-10-09T05:52:12.504 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2449999999999998e-06 2023-10-09T05:52:12.504 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.504 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.504 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.505 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.255794+0000", 2023-10-09T05:52:12.505 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1672438190000001 2023-10-09T05:52:12.505 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.505 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.505 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:12.505 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.255863+0000", 2023-10-09T05:52:12.505 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8866000000000001e-05 2023-10-09T05:52:12.505 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.505 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:12.505 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.505 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.505 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.506 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:82079 6.a 6:59a1f462:::benchmark_data_smithi064_284193_object82078:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T05:52:12.506 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:56.101890+0000", 2023-10-09T05:52:12.506 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 16.283842443000001, 2023-10-09T05:52:12.506 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.154915967, 2023-10-09T05:52:12.506 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:12.506 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T05:52:12.506 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:12.506 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:12.506 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:12.506 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 82079 2023-10-09T05:52:12.506 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.507 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:12.507 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.507 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:12.507 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.101890+0000", 2023-10-09T05:52:12.507 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.507 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.507 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.507 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:12.507 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.101890+0000", 2023-10-09T05:52:12.507 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.507 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.507 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.508 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:12.508 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.101889+0000", 2023-10-09T05:52:12.508 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:52:12.508 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.508 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.508 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:12.508 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.101925+0000", 2023-10-09T05:52:12.508 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5763999999999998e-05 2023-10-09T05:52:12.508 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.508 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.508 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:12.508 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.101932+0000", 2023-10-09T05:52:12.509 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7290000000000003e-06 2023-10-09T05:52:12.509 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.509 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.509 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:12.509 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.101937+0000", 2023-10-09T05:52:12.509 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.972e-06 2023-10-09T05:52:12.509 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.509 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.509 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.509 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.103569+0000", 2023-10-09T05:52:12.509 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0016325479999999999 2023-10-09T05:52:12.509 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.510 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.510 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:12.510 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.103679+0000", 2023-10-09T05:52:12.510 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000109225 2023-10-09T05:52:12.510 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.510 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.510 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 6", 2023-10-09T05:52:12.510 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.103810+0000", 2023-10-09T05:52:12.510 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013134799999999999 2023-10-09T05:52:12.510 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.510 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.510 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:12.511 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.113355+0000", 2023-10-09T05:52:12.511 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.009545118 2023-10-09T05:52:12.511 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.511 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.511 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.511 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.147616+0000", 2023-10-09T05:52:12.511 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.034261319999999998 2023-10-09T05:52:12.511 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.511 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.511 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.511 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.147626+0000", 2023-10-09T05:52:12.511 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7899999999999994e-06 2023-10-09T05:52:12.512 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.512 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.512 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.512 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088568+0000", 2023-10-09T05:52:12.512 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.94094197000000002 2023-10-09T05:52:12.512 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.512 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.512 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.512 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088574+0000", 2023-10-09T05:52:12.512 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3439999999999997e-06 2023-10-09T05:52:12.512 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.512 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.513 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.513 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.256728+0000", 2023-10-09T05:52:12.513 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.168153088 2023-10-09T05:52:12.513 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.513 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.513 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:12.513 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.256806+0000", 2023-10-09T05:52:12.513 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8255000000000002e-05 2023-10-09T05:52:12.513 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.513 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:12.513 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.513 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.514 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.514 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:82095 6.d 6:b2648f7f:::benchmark_data_smithi064_284193_object82094:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:52:12.514 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:56.144382+0000", 2023-10-09T05:52:12.514 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 16.241350131000001, 2023-10-09T05:52:12.514 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1392636949999999, 2023-10-09T05:52:12.514 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:12.514 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:12.514 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:12.514 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:12.514 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:12.514 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 82095 2023-10-09T05:52:12.515 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.515 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:12.515 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.515 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:12.515 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.144382+0000", 2023-10-09T05:52:12.515 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.515 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.515 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.515 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:12.515 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.144382+0000", 2023-10-09T05:52:12.515 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.515 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.515 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.516 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:12.516 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.144382+0000", 2023-10-09T05:52:12.516 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:52:12.516 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.516 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.516 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:12.516 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.144417+0000", 2023-10-09T05:52:12.516 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5509999999999997e-05 2023-10-09T05:52:12.516 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.516 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.516 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:12.516 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.144426+0000", 2023-10-09T05:52:12.517 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6719999999999993e-06 2023-10-09T05:52:12.517 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.517 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.517 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:12.517 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.144431+0000", 2023-10-09T05:52:12.517 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2750000000000003e-06 2023-10-09T05:52:12.517 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.517 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.517 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.517 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.189174+0000", 2023-10-09T05:52:12.517 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.044742523999999999 2023-10-09T05:52:12.517 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.518 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.518 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.518 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.189184+0000", 2023-10-09T05:52:12.518 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0798999999999999e-05 2023-10-09T05:52:12.518 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.518 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.518 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.518 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088004+0000", 2023-10-09T05:52:12.518 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.89882000699999998 2023-10-09T05:52:12.518 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.518 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.518 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.519 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088016+0000", 2023-10-09T05:52:12.519 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1809e-05 2023-10-09T05:52:12.519 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.519 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.519 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.519 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.266944+0000", 2023-10-09T05:52:12.519 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.178928057 2023-10-09T05:52:12.519 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.519 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.519 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:12.519 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.267103+0000", 2023-10-09T05:52:12.519 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015863100000000001 2023-10-09T05:52:12.520 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.520 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.520 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:52:12.520 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.267262+0000", 2023-10-09T05:52:12.520 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000159388 2023-10-09T05:52:12.520 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.520 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.520 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:12.520 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.275961+0000", 2023-10-09T05:52:12.520 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0086985629999999994 2023-10-09T05:52:12.520 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.520 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.520 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.521 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.280428+0000", 2023-10-09T05:52:12.521 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0044668620000000003 2023-10-09T05:52:12.521 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.521 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.521 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.521 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.281192+0000", 2023-10-09T05:52:12.521 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00076383899999999999 2023-10-09T05:52:12.521 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.521 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.521 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:12.521 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.281240+0000", 2023-10-09T05:52:12.521 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8152999999999998e-05 2023-10-09T05:52:12.522 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.522 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.522 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:12.522 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.283646+0000", 2023-10-09T05:52:12.522 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0024061540000000002 2023-10-09T05:52:12.522 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.522 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:12.522 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.522 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.522 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.522 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:82106 6.d 6:b7051b26:::benchmark_data_smithi064_284193_object82105:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:52:12.523 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:56.173869+0000", 2023-10-09T05:52:12.523 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 16.211863524999998, 2023-10-09T05:52:12.523 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1196443020000002, 2023-10-09T05:52:12.523 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:12.523 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:12.523 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:12.523 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:12.523 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:12.523 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 82106 2023-10-09T05:52:12.523 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.523 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:12.523 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.523 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:12.524 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.173869+0000", 2023-10-09T05:52:12.524 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.524 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.524 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.524 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:12.524 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.173869+0000", 2023-10-09T05:52:12.524 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.524 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.524 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.524 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:12.524 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.173867+0000", 2023-10-09T05:52:12.524 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T05:52:12.525 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.525 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.525 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:12.525 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.173923+0000", 2023-10-09T05:52:12.525 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.647e-05 2023-10-09T05:52:12.525 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.525 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.525 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:12.525 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.173949+0000", 2023-10-09T05:52:12.525 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.5079000000000002e-05 2023-10-09T05:52:12.525 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.525 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.526 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:12.526 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.173955+0000", 2023-10-09T05:52:12.526 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4300000000000003e-06 2023-10-09T05:52:12.526 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.526 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.526 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.526 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.189208+0000", 2023-10-09T05:52:12.526 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015253040000000001 2023-10-09T05:52:12.526 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.526 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.526 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.526 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.189215+0000", 2023-10-09T05:52:12.527 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0999999999999998e-06 2023-10-09T05:52:12.527 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.527 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.527 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.527 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088046+0000", 2023-10-09T05:52:12.527 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.89883056900000002 2023-10-09T05:52:12.527 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.527 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.527 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.527 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088053+0000", 2023-10-09T05:52:12.527 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5909999999999998e-06 2023-10-09T05:52:12.527 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.528 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.528 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.528 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.269849+0000", 2023-10-09T05:52:12.528 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1817954580000001 2023-10-09T05:52:12.528 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.528 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.528 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:12.528 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.269978+0000", 2023-10-09T05:52:12.528 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012922500000000001 2023-10-09T05:52:12.528 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.528 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.528 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:52:12.529 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.270106+0000", 2023-10-09T05:52:12.529 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012807000000000001 2023-10-09T05:52:12.529 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.529 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.529 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:12.529 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.275969+0000", 2023-10-09T05:52:12.529 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0058634300000000002 2023-10-09T05:52:12.529 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.529 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.529 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.529 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.288964+0000", 2023-10-09T05:52:12.529 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012994916 2023-10-09T05:52:12.529 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.530 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.530 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.530 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.290167+0000", 2023-10-09T05:52:12.530 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0012025320000000001 2023-10-09T05:52:12.530 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.530 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.530 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:12.530 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.290229+0000", 2023-10-09T05:52:12.530 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2274999999999996e-05 2023-10-09T05:52:12.530 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.530 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.531 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:12.531 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.293513+0000", 2023-10-09T05:52:12.531 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0032839380000000001 2023-10-09T05:52:12.531 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.531 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:12.531 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.531 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.531 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.531 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:82129 6.a 6:5e1e98b2:::benchmark_data_smithi064_284193_object82128:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:52:12.531 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:56.214518+0000", 2023-10-09T05:52:12.531 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 16.171214635999998, 2023-10-09T05:52:12.531 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.051940192, 2023-10-09T05:52:12.532 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:12.532 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:12.532 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:12.532 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:12.532 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:12.532 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 82129 2023-10-09T05:52:12.532 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.532 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:12.532 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.532 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:12.532 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.214518+0000", 2023-10-09T05:52:12.532 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.532 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.533 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.533 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:12.533 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.214518+0000", 2023-10-09T05:52:12.533 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.533 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.533 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.533 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:12.533 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.214513+0000", 2023-10-09T05:52:12.533 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999948 2023-10-09T05:52:12.533 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.533 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.533 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:12.534 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.214557+0000", 2023-10-09T05:52:12.534 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4127e-05 2023-10-09T05:52:12.534 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.534 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.534 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:12.534 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.214580+0000", 2023-10-09T05:52:12.534 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3390000000000001e-05 2023-10-09T05:52:12.534 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.534 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.534 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:12.534 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.214589+0000", 2023-10-09T05:52:12.534 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9719999999999998e-06 2023-10-09T05:52:12.535 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.535 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.535 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.535 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.216231+0000", 2023-10-09T05:52:12.535 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001641637 2023-10-09T05:52:12.535 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.535 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.535 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.535 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.216242+0000", 2023-10-09T05:52:12.535 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0929e-05 2023-10-09T05:52:12.535 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.535 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.536 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.536 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088597+0000", 2023-10-09T05:52:12.536 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.87235592100000003 2023-10-09T05:52:12.536 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.536 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.536 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.536 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088604+0000", 2023-10-09T05:52:12.536 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7649999999999996e-06 2023-10-09T05:52:12.536 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.536 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.536 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.536 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.256832+0000", 2023-10-09T05:52:12.537 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1682279229999999 2023-10-09T05:52:12.537 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.537 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.537 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:12.537 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.256991+0000", 2023-10-09T05:52:12.537 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015927700000000001 2023-10-09T05:52:12.537 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.537 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.537 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:52:12.537 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.257137+0000", 2023-10-09T05:52:12.537 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014602500000000001 2023-10-09T05:52:12.537 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.538 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.538 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:12.538 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.263290+0000", 2023-10-09T05:52:12.538 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0061524459999999998 2023-10-09T05:52:12.538 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.538 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.538 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.538 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.265238+0000", 2023-10-09T05:52:12.538 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001948537 2023-10-09T05:52:12.538 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.538 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.538 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.539 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.265424+0000", 2023-10-09T05:52:12.539 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000185164 2023-10-09T05:52:12.539 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.539 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.539 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:12.539 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.265488+0000", 2023-10-09T05:52:12.539 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4632000000000001e-05 2023-10-09T05:52:12.539 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.539 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.539 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:12.539 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.266458+0000", 2023-10-09T05:52:12.539 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00096960600000000003 2023-10-09T05:52:12.540 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.540 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:12.540 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.540 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.540 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.540 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:82138 6.a 6:5fd97b9e:::benchmark_data_smithi064_284193_object82137:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:52:12.540 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:56.236652+0000", 2023-10-09T05:52:12.540 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 16.149080033000001, 2023-10-09T05:52:12.540 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0342095279999999, 2023-10-09T05:52:12.540 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:12.540 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:12.540 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:12.541 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:12.541 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:12.541 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 82138 2023-10-09T05:52:12.541 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.541 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:12.541 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.541 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:12.541 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.236652+0000", 2023-10-09T05:52:12.541 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.541 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.541 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.541 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:12.542 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.236652+0000", 2023-10-09T05:52:12.542 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.542 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.542 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.542 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:12.542 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.236650+0000", 2023-10-09T05:52:12.542 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T05:52:12.542 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.542 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.542 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:12.542 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.236699+0000", 2023-10-09T05:52:12.542 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8606000000000002e-05 2023-10-09T05:52:12.543 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.543 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.543 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:12.543 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.236723+0000", 2023-10-09T05:52:12.543 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4097999999999999e-05 2023-10-09T05:52:12.543 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.543 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.543 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:12.543 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.236732+0000", 2023-10-09T05:52:12.543 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.5709999999999998e-06 2023-10-09T05:52:12.543 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.543 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.544 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.544 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.236803+0000", 2023-10-09T05:52:12.544 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1162999999999996e-05 2023-10-09T05:52:12.544 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.544 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.544 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.544 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.236815+0000", 2023-10-09T05:52:12.544 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2274e-05 2023-10-09T05:52:12.544 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.544 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.544 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.544 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.089210+0000", 2023-10-09T05:52:12.545 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.852395291 2023-10-09T05:52:12.545 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.545 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.545 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.545 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.089216+0000", 2023-10-09T05:52:12.545 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6060000000000002e-06 2023-10-09T05:52:12.545 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.545 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.545 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.545 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.260084+0000", 2023-10-09T05:52:12.545 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.170867879 2023-10-09T05:52:12.545 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.546 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.546 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:12.546 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.260256+0000", 2023-10-09T05:52:12.546 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00017194700000000001 2023-10-09T05:52:12.546 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.546 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.546 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:52:12.546 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.260449+0000", 2023-10-09T05:52:12.546 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00019292300000000001 2023-10-09T05:52:12.546 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.546 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.546 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:12.547 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.265252+0000", 2023-10-09T05:52:12.547 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0048034929999999998 2023-10-09T05:52:12.547 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.547 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.547 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.547 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.265322+0000", 2023-10-09T05:52:12.547 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9429000000000002e-05 2023-10-09T05:52:12.547 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.547 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.547 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.547 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.269814+0000", 2023-10-09T05:52:12.547 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0044918750000000002 2023-10-09T05:52:12.548 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.548 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.548 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:12.548 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.269876+0000", 2023-10-09T05:52:12.548 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2621999999999998e-05 2023-10-09T05:52:12.548 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.548 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.548 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:12.548 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.270862+0000", 2023-10-09T05:52:12.548 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00098564500000000005 2023-10-09T05:52:12.548 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.548 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:12.549 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.549 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.549 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.549 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:91002 6.4 6:2cef0a59:::benchmark_data_smithi064_284193_object91001:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-10-09T05:52:12.549 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:10.173814+0000", 2023-10-09T05:52:12.549 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 2.211918453, 2023-10-09T05:52:12.549 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.336934939, 2023-10-09T05:52:12.549 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:12.549 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T05:52:12.549 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:12.549 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:12.549 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:12.550 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 91002 2023-10-09T05:52:12.550 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.550 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:12.550 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.550 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:12.550 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.173814+0000", 2023-10-09T05:52:12.550 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.550 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.550 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.550 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:12.550 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.173814+0000", 2023-10-09T05:52:12.551 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.551 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.551 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.551 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:12.551 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.173813+0000", 2023-10-09T05:52:12.551 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:52:12.551 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.551 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.551 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:12.551 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.173843+0000", 2023-10-09T05:52:12.551 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9298999999999998e-05 2023-10-09T05:52:12.551 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.552 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.552 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:12.552 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.173850+0000", 2023-10-09T05:52:12.552 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.413e-06 2023-10-09T05:52:12.552 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.552 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.552 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:12.552 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.173854+0000", 2023-10-09T05:52:12.552 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4159999999999997e-06 2023-10-09T05:52:12.552 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.552 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.552 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.552 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.177830+0000", 2023-10-09T05:52:12.553 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0039752570000000003 2023-10-09T05:52:12.553 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.553 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.553 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:12.553 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.178005+0000", 2023-10-09T05:52:12.553 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00017501100000000001 2023-10-09T05:52:12.553 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.553 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.553 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:52:12.553 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.178180+0000", 2023-10-09T05:52:12.553 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00017503899999999999 2023-10-09T05:52:12.554 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.554 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.554 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.554 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.212537+0000", 2023-10-09T05:52:12.554 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.034357681000000001 2023-10-09T05:52:12.554 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.554 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.554 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.554 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.212546+0000", 2023-10-09T05:52:12.554 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.3909999999999998e-06 2023-10-09T05:52:12.554 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.554 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.555 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.555 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.488951+0000", 2023-10-09T05:52:12.555 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.276405051 2023-10-09T05:52:12.555 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.555 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.555 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for degraded object", 2023-10-09T05:52:12.555 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.489540+0000", 2023-10-09T05:52:12.555 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00058895699999999998 2023-10-09T05:52:12.555 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.555 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.555 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.555 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.510637+0000", 2023-10-09T05:52:12.556 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.021097202999999998 2023-10-09T05:52:12.556 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.556 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.556 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:12.556 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.510749+0000", 2023-10-09T05:52:12.556 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011176299999999999 2023-10-09T05:52:12.556 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.556 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:12.556 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.556 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.556 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.557 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:91017 6.d 6:b3504a53:::benchmark_data_smithi064_284193_object91016:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-10-09T05:52:12.557 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:10.210870+0000", 2023-10-09T05:52:12.557 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 2.174862444, 2023-10-09T05:52:12.557 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2371660760000001, 2023-10-09T05:52:12.557 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:12.557 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:12.557 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:12.557 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:12.557 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:12.557 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 91017 2023-10-09T05:52:12.557 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.557 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:12.557 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.558 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:12.558 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210870+0000", 2023-10-09T05:52:12.558 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.558 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.558 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.558 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:12.558 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210870+0000", 2023-10-09T05:52:12.558 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.558 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.558 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.558 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:12.558 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210869+0000", 2023-10-09T05:52:12.559 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:52:12.559 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.559 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.559 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:12.559 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210904+0000", 2023-10-09T05:52:12.559 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.4656999999999997e-05 2023-10-09T05:52:12.559 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.559 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.559 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:12.559 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210914+0000", 2023-10-09T05:52:12.559 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0464e-05 2023-10-09T05:52:12.559 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.560 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.560 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:12.560 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210920+0000", 2023-10-09T05:52:12.560 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6380000000000001e-06 2023-10-09T05:52:12.560 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.560 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.560 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.560 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210970+0000", 2023-10-09T05:52:12.560 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0062999999999998e-05 2023-10-09T05:52:12.560 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.560 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.560 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.561 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210979+0000", 2023-10-09T05:52:12.561 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.7299999999999994e-06 2023-10-09T05:52:12.561 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.561 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.561 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.561 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.441103+0000", 2023-10-09T05:52:12.561 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2301243880000001 2023-10-09T05:52:12.561 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.561 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.561 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:12.561 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.441224+0000", 2023-10-09T05:52:12.561 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012059 2023-10-09T05:52:12.562 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.562 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.562 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:52:12.562 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.441351+0000", 2023-10-09T05:52:12.562 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012714700000000001 2023-10-09T05:52:12.562 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.562 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.562 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:12.562 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.445558+0000", 2023-10-09T05:52:12.562 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0042071510000000001 2023-10-09T05:52:12.562 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.563 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.563 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.563 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.446131+0000", 2023-10-09T05:52:12.563 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00057308200000000004 2023-10-09T05:52:12.563 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.563 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.563 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.563 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.447453+0000", 2023-10-09T05:52:12.563 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0013219460000000001 2023-10-09T05:52:12.563 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.563 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.563 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:12.563 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.447493+0000", 2023-10-09T05:52:12.564 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9920999999999999e-05 2023-10-09T05:52:12.564 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.564 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.564 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:12.564 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.448036+0000", 2023-10-09T05:52:12.564 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00054314100000000002 2023-10-09T05:52:12.564 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.564 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:12.564 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.564 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.564 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.564 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:91019 6.4 6:25997699:::benchmark_data_smithi064_284193_object91018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-10-09T05:52:12.565 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:10.215587+0000", 2023-10-09T05:52:12.565 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 2.1701455620000001, 2023-10-09T05:52:12.565 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2962565879999999, 2023-10-09T05:52:12.565 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:12.565 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:12.565 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:12.565 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:12.565 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:12.565 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 91019 2023-10-09T05:52:12.565 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.565 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:12.565 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.566 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:12.566 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215587+0000", 2023-10-09T05:52:12.566 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.566 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.566 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.566 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:12.566 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215587+0000", 2023-10-09T05:52:12.566 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.566 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.566 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.566 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:12.566 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215585+0000", 2023-10-09T05:52:12.567 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:12.567 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.567 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.567 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:12.567 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215691+0000", 2023-10-09T05:52:12.567 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000105311 2023-10-09T05:52:12.567 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.567 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.567 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:12.567 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215710+0000", 2023-10-09T05:52:12.567 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.893e-05 2023-10-09T05:52:12.567 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.568 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.568 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:12.568 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215717+0000", 2023-10-09T05:52:12.568 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8950000000000001e-06 2023-10-09T05:52:12.568 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.568 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.568 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.568 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.216473+0000", 2023-10-09T05:52:12.568 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00075626899999999995 2023-10-09T05:52:12.568 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.568 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.568 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.569 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.216492+0000", 2023-10-09T05:52:12.569 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9072999999999998e-05 2023-10-09T05:52:12.569 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.569 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.569 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.569 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.489588+0000", 2023-10-09T05:52:12.569 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.273096518 2023-10-09T05:52:12.569 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.569 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.569 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:12.569 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.489698+0000", 2023-10-09T05:52:12.569 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000109848 2023-10-09T05:52:12.569 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.570 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.570 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:52:12.570 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.489836+0000", 2023-10-09T05:52:12.570 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013729700000000001 2023-10-09T05:52:12.570 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.570 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.570 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:12.570 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.496824+0000", 2023-10-09T05:52:12.570 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0069883929999999999 2023-10-09T05:52:12.570 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.570 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.571 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.571 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.504295+0000", 2023-10-09T05:52:12.571 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0074707799999999998 2023-10-09T05:52:12.571 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.571 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.571 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.571 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.510792+0000", 2023-10-09T05:52:12.571 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00649718 2023-10-09T05:52:12.571 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.571 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.571 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:12.571 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.510853+0000", 2023-10-09T05:52:12.572 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0868000000000001e-05 2023-10-09T05:52:12.572 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.572 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.572 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:12.572 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.511843+0000", 2023-10-09T05:52:12.572 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00099058400000000008 2023-10-09T05:52:12.572 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.572 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:12.572 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.572 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.572 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.572 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:91052 6.d 6:b9656c98:::benchmark_data_smithi064_284193_object91051:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-10-09T05:52:12.573 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:10.282349+0000", 2023-10-09T05:52:12.573 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 2.103383526, 2023-10-09T05:52:12.573 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.174734331, 2023-10-09T05:52:12.573 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:12.573 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:12.573 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:12.573 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:12.573 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:12.573 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 91052 2023-10-09T05:52:12.573 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.573 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:12.573 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.573 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:12.574 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.282349+0000", 2023-10-09T05:52:12.574 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.574 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.574 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.574 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:12.574 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.282349+0000", 2023-10-09T05:52:12.574 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:12.574 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.574 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.574 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:12.574 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.282347+0000", 2023-10-09T05:52:12.575 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T05:52:12.575 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.575 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.575 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:12.575 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.282415+0000", 2023-10-09T05:52:12.575 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8465000000000003e-05 2023-10-09T05:52:12.575 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.575 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.575 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:12.575 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.282431+0000", 2023-10-09T05:52:12.575 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5739e-05 2023-10-09T05:52:12.575 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.576 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.576 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:12.576 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.282439+0000", 2023-10-09T05:52:12.576 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5900000000000002e-06 2023-10-09T05:52:12.576 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.576 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.576 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.576 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.282507+0000", 2023-10-09T05:52:12.576 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7891000000000004e-05 2023-10-09T05:52:12.576 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.576 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.576 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:12.576 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.282519+0000", 2023-10-09T05:52:12.577 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2887e-05 2023-10-09T05:52:12.577 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.577 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.577 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:12.577 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.443395+0000", 2023-10-09T05:52:12.577 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.160875715 2023-10-09T05:52:12.577 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.577 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.577 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:12.577 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.443472+0000", 2023-10-09T05:52:12.577 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7158999999999998e-05 2023-10-09T05:52:12.577 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.577 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.578 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:52:12.578 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.443574+0000", 2023-10-09T05:52:12.578 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000102141 2023-10-09T05:52:12.578 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.578 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.578 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:12.578 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.445569+0000", 2023-10-09T05:52:12.578 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001994928 2023-10-09T05:52:12.578 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.578 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.578 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.579 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.448070+0000", 2023-10-09T05:52:12.579 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0025002280000000002 2023-10-09T05:52:12.579 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.579 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.579 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:12.579 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.456279+0000", 2023-10-09T05:52:12.579 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0082089169999999996 2023-10-09T05:52:12.579 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.579 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.579 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:12.579 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.456318+0000", 2023-10-09T05:52:12.579 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9218000000000003e-05 2023-10-09T05:52:12.580 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:12.580 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:12.580 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:12.580 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.457083+0000", 2023-10-09T05:52:12.580 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00076537899999999995 2023-10-09T05:52:12.580 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.580 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:12.580 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.580 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:12.580 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:12.580 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:52:12.581 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:12.581 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:52:12.581 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[220044]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[220044]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[220044]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[220044]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[220044]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[220044]: osdmap e134: 8 total, 8 up, 8 in 2023-10-09T05:52:12.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[220044]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[220044]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[220044]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[220044]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[220044]: failed to encode map e134 with expected crc 2023-10-09T05:52:12.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:12 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 17893/270516 objects degraded (6.614%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:12.668 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:12.793 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:12.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:12.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:12.909 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:12.983 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:13.076 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:13.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:13.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:13.179 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:13.207 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:13.236 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T05:52:13.382 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 84 16 92504 92488 68.7867 44.1875 0.0297401 0.0145323 2023-10-09T05:52:13.513 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:13.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:13 smithi064 ceph-mon[212750]: failed to encode map e135 with expected crc 2023-10-09T05:52:13.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:13 smithi064 ceph-mon[212750]: osdmap e135: 8 total, 8 up, 8 in 2023-10-09T05:52:13.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:13 smithi064 ceph-mon[212750]: failed to encode map e135 with expected crc 2023-10-09T05:52:13.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:13 smithi064 ceph-mon[212750]: failed to encode map e135 with expected crc 2023-10-09T05:52:13.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:13 smithi064 ceph-mon[212750]: failed to encode map e135 with expected crc 2023-10-09T05:52:13.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:13 smithi064 ceph-mon[212750]: failed to encode map e135 with expected crc 2023-10-09T05:52:13.523 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:13 smithi064 ceph-mon[212750]: pgmap v305: 99 pgs: 10 peering, 1 unknown, 1 active+recovering+undersized+remapped, 80 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 43 MiB/s wr, 686 op/s; 17886/272124 objects degraded (6.573%); 1524/272124 objects misplaced (0.560%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:52:13.523 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:13 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/669200720' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:52:13.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:13.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:13.524 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:13 smithi064 ceph-mon[220044]: failed to encode map e135 with expected crc 2023-10-09T05:52:13.524 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:13 smithi064 ceph-mon[220044]: osdmap e135: 8 total, 8 up, 8 in 2023-10-09T05:52:13.524 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:13 smithi064 ceph-mon[220044]: failed to encode map e135 with expected crc 2023-10-09T05:52:13.524 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:13 smithi064 ceph-mon[220044]: failed to encode map e135 with expected crc 2023-10-09T05:52:13.525 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:13 smithi064 ceph-mon[220044]: failed to encode map e135 with expected crc 2023-10-09T05:52:13.525 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:13 smithi064 ceph-mon[220044]: failed to encode map e135 with expected crc 2023-10-09T05:52:13.525 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:13 smithi064 ceph-mon[220044]: pgmap v305: 99 pgs: 10 peering, 1 unknown, 1 active+recovering+undersized+remapped, 80 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 43 MiB/s wr, 686 op/s; 17886/272124 objects degraded (6.573%); 1524/272124 objects misplaced (0.560%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:52:13.525 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:13 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/669200720' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:52:13.572 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:13 smithi156 ceph-mon[195101]: failed to encode map e135 with expected crc 2023-10-09T05:52:13.572 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:13 smithi156 ceph-mon[195101]: osdmap e135: 8 total, 8 up, 8 in 2023-10-09T05:52:13.572 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:13 smithi156 ceph-mon[195101]: failed to encode map e135 with expected crc 2023-10-09T05:52:13.572 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:13 smithi156 ceph-mon[195101]: failed to encode map e135 with expected crc 2023-10-09T05:52:13.572 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:13 smithi156 ceph-mon[195101]: failed to encode map e135 with expected crc 2023-10-09T05:52:13.572 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:13 smithi156 ceph-mon[195101]: failed to encode map e135 with expected crc 2023-10-09T05:52:13.573 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:13 smithi156 ceph-mon[195101]: pgmap v305: 99 pgs: 10 peering, 1 unknown, 1 active+recovering+undersized+remapped, 80 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 43 MiB/s wr, 686 op/s; 17886/272124 objects degraded (6.573%); 1524/272124 objects misplaced (0.560%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:52:13.573 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:13 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/669200720' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:52:13.583 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:52:13.594 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:52:13.625 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:13.708 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:13.839 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:13.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:13.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:13.943 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:13.971 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:14.015 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:52:14.318 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:14.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:14.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:14.383 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 85 15 93184 93169 68.4782 42.5625 0.0130744 0.0145973 2023-10-09T05:52:14.436 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:14.517 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:14 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:52:14.517 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:14 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/669200720' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:52:14.517 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:14 smithi064 ceph-mon[220044]: osdmap e136: 8 total, 8 up, 8 in 2023-10-09T05:52:14.517 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:14 smithi064 ceph-mon[220044]: failed to encode map e136 with expected crc 2023-10-09T05:52:14.517 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:14 smithi064 ceph-mon[220044]: failed to encode map e136 with expected crc 2023-10-09T05:52:14.517 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:14 smithi064 ceph-mon[220044]: failed to encode map e136 with expected crc 2023-10-09T05:52:14.517 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:14 smithi064 ceph-mon[220044]: failed to encode map e136 with expected crc 2023-10-09T05:52:14.518 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:14 smithi064 ceph-mon[220044]: failed to encode map e136 with expected crc 2023-10-09T05:52:14.518 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:14.518 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:52:14.518 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:14 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:52:14.518 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:14 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/669200720' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:52:14.518 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:14 smithi064 ceph-mon[212750]: osdmap e136: 8 total, 8 up, 8 in 2023-10-09T05:52:14.518 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:14 smithi064 ceph-mon[212750]: failed to encode map e136 with expected crc 2023-10-09T05:52:14.518 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:14 smithi064 ceph-mon[212750]: failed to encode map e136 with expected crc 2023-10-09T05:52:14.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:14 smithi064 ceph-mon[212750]: failed to encode map e136 with expected crc 2023-10-09T05:52:14.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:14 smithi064 ceph-mon[212750]: failed to encode map e136 with expected crc 2023-10-09T05:52:14.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:14 smithi064 ceph-mon[212750]: failed to encode map e136 with expected crc 2023-10-09T05:52:14.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:14.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:52:14.531 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:14.579 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:14 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:52:14.579 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:14 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/669200720' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:52:14.579 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:14 smithi156 ceph-mon[195101]: osdmap e136: 8 total, 8 up, 8 in 2023-10-09T05:52:14.579 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:14 smithi156 ceph-mon[195101]: failed to encode map e136 with expected crc 2023-10-09T05:52:14.580 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:14 smithi156 ceph-mon[195101]: failed to encode map e136 with expected crc 2023-10-09T05:52:14.580 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:14 smithi156 ceph-mon[195101]: failed to encode map e136 with expected crc 2023-10-09T05:52:14.580 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:14 smithi156 ceph-mon[195101]: failed to encode map e136 with expected crc 2023-10-09T05:52:14.580 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:14 smithi156 ceph-mon[195101]: failed to encode map e136 with expected crc 2023-10-09T05:52:14.580 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:14.580 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:52:14.616 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:14.643 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:14.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:14.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:14.760 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:14.842 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:14.971 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:14.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:14.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:15.086 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:15.118 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:15.170 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:15.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:15.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:15.274 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:15.319 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:15.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:15 smithi156 ceph-mon[195101]: osdmap e137: 8 total, 8 up, 8 in 2023-10-09T05:52:15.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:15 smithi156 ceph-mon[195101]: failed to encode map e137 with expected crc 2023-10-09T05:52:15.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:15 smithi156 ceph-mon[195101]: failed to encode map e137 with expected crc 2023-10-09T05:52:15.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:15 smithi156 ceph-mon[195101]: failed to encode map e137 with expected crc 2023-10-09T05:52:15.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:15 smithi156 ceph-mon[195101]: failed to encode map e137 with expected crc 2023-10-09T05:52:15.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:15 smithi156 ceph-mon[195101]: failed to encode map e137 with expected crc 2023-10-09T05:52:15.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:15 smithi156 ceph-mon[195101]: pgmap v308: 99 pgs: 10 peering, 1 unknown, 2 active+recovering+undersized+remapped, 79 active+clean, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 36 MiB/s wr, 568 op/s; 9485/277335 objects degraded (3.420%); 2999/277335 objects misplaced (1.081%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T05:52:15.384 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 86 16 93884 93868 68.1887 43.6875 0.00796845 0.0146594 2023-10-09T05:52:15.410 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:15.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:15.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:15.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:52:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:52:15] "GET /metrics HTTP/1.1" 200 36424 "" "Prometheus/2.43.0" 2023-10-09T05:52:15.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:15 smithi064 ceph-mon[212750]: osdmap e137: 8 total, 8 up, 8 in 2023-10-09T05:52:15.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:15 smithi064 ceph-mon[212750]: failed to encode map e137 with expected crc 2023-10-09T05:52:15.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:15 smithi064 ceph-mon[212750]: failed to encode map e137 with expected crc 2023-10-09T05:52:15.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:15 smithi064 ceph-mon[212750]: failed to encode map e137 with expected crc 2023-10-09T05:52:15.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:15 smithi064 ceph-mon[212750]: failed to encode map e137 with expected crc 2023-10-09T05:52:15.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:15 smithi064 ceph-mon[212750]: failed to encode map e137 with expected crc 2023-10-09T05:52:15.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:15 smithi064 ceph-mon[212750]: pgmap v308: 99 pgs: 10 peering, 1 unknown, 2 active+recovering+undersized+remapped, 79 active+clean, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 36 MiB/s wr, 568 op/s; 9485/277335 objects degraded (3.420%); 2999/277335 objects misplaced (1.081%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T05:52:15.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:15 smithi064 ceph-mon[220044]: osdmap e137: 8 total, 8 up, 8 in 2023-10-09T05:52:15.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:15 smithi064 ceph-mon[220044]: failed to encode map e137 with expected crc 2023-10-09T05:52:15.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:15 smithi064 ceph-mon[220044]: failed to encode map e137 with expected crc 2023-10-09T05:52:15.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:15 smithi064 ceph-mon[220044]: failed to encode map e137 with expected crc 2023-10-09T05:52:15.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:15 smithi064 ceph-mon[220044]: failed to encode map e137 with expected crc 2023-10-09T05:52:15.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:15 smithi064 ceph-mon[220044]: failed to encode map e137 with expected crc 2023-10-09T05:52:15.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:15 smithi064 ceph-mon[220044]: pgmap v308: 99 pgs: 10 peering, 1 unknown, 2 active+recovering+undersized+remapped, 79 active+clean, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 36 MiB/s wr, 568 op/s; 9485/277335 objects degraded (3.420%); 2999/277335 objects misplaced (1.081%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T05:52:15.514 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:15.561 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:15.608 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:15.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:15.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:15.710 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:15.753 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:15.846 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:15.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:15.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:15.952 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:16.033 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:16.115 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:52:16.115 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T05:52:16.116 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.116 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:94370 6.1 6:8e813a16:::benchmark_data_smithi064_284193_object94369:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2023-10-09T05:52:16.116 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:16.087079+0000", 2023-10-09T05:52:16.116 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.01820041, 2023-10-09T05:52:16.116 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018230183, 2023-10-09T05:52:16.116 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:16.116 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:16.116 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:16.116 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:16.116 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:16.117 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 94370 2023-10-09T05:52:16.117 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.117 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:16.117 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.117 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:16.117 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.087079+0000", 2023-10-09T05:52:16.117 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:16.117 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.117 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.117 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:16.118 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.087079+0000", 2023-10-09T05:52:16.118 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:16.118 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.118 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.118 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:16.118 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.087078+0000", 2023-10-09T05:52:16.118 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:52:16.118 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.118 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.118 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:16.119 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.087115+0000", 2023-10-09T05:52:16.119 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6573999999999999e-05 2023-10-09T05:52:16.119 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.119 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.119 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:16.119 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.087123+0000", 2023-10-09T05:52:16.119 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1599999999999998e-06 2023-10-09T05:52:16.119 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.119 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.119 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:16.120 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.087128+0000", 2023-10-09T05:52:16.120 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7570000000000004e-06 2023-10-09T05:52:16.120 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.120 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.120 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:16.120 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.090895+0000", 2023-10-09T05:52:16.120 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0037667540000000002 2023-10-09T05:52:16.120 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.120 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.120 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:16.121 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.091063+0000", 2023-10-09T05:52:16.121 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00016821800000000001 2023-10-09T05:52:16.121 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.121 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.121 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:52:16.121 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.091253+0000", 2023-10-09T05:52:16.121 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000190032 2023-10-09T05:52:16.121 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.121 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.122 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:16.122 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.095858+0000", 2023-10-09T05:52:16.122 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0046053149999999996 2023-10-09T05:52:16.122 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.122 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.122 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:16.122 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.103124+0000", 2023-10-09T05:52:16.122 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0072656300000000004 2023-10-09T05:52:16.122 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.122 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.122 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:16.123 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.104518+0000", 2023-10-09T05:52:16.123 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001394368 2023-10-09T05:52:16.123 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.123 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.123 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:16.123 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.104579+0000", 2023-10-09T05:52:16.123 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0851999999999999e-05 2023-10-09T05:52:16.123 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:16.123 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:16.123 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:16.123 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.124 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.124 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop_reply(client.44588.0:94370 6.1 e137/133 ondisk, result = 0)", 2023-10-09T05:52:16.124 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:16.103373+0000", 2023-10-09T05:52:16.124 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0019065740000000001, 2023-10-09T05:52:16.124 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0020301709999999999, 2023-10-09T05:52:16.124 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:16.124 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T05:52:16.124 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:16.124 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.124 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:16.124 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.103373+0000", 2023-10-09T05:52:16.125 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:16.125 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.125 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.125 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:16.125 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.103373+0000", 2023-10-09T05:52:16.125 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:16.125 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.125 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.125 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:16.125 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.103373+0000", 2023-10-09T05:52:16.126 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967296 2023-10-09T05:52:16.126 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.126 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.126 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:16.126 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.103375+0000", 2023-10-09T05:52:16.126 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4090000000000001e-06 2023-10-09T05:52:16.126 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.126 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.126 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:16.126 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.103381+0000", 2023-10-09T05:52:16.126 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.558e-06 2023-10-09T05:52:16.127 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.127 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.127 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:16.127 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.103384+0000", 2023-10-09T05:52:16.127 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.518e-06 2023-10-09T05:52:16.127 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.127 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.127 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:16.127 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.104499+0000", 2023-10-09T05:52:16.127 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0011150789999999999 2023-10-09T05:52:16.127 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.128 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.128 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:16.128 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.104510+0000", 2023-10-09T05:52:16.128 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0677e-05 2023-10-09T05:52:16.128 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:16.128 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:16.128 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:16.128 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.128 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.128 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:94382 6.1 6.26ea5761 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e137)", 2023-10-09T05:52:16.128 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:16.104229+0000", 2023-10-09T05:52:16.128 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0010502109999999999, 2023-10-09T05:52:16.129 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001218383, 2023-10-09T05:52:16.129 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:16.129 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "queued for pg", 2023-10-09T05:52:16.129 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:16.129 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:16.129 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:16.129 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 94382 2023-10-09T05:52:16.129 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.129 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:16.129 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.129 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:16.129 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.104229+0000", 2023-10-09T05:52:16.130 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:16.130 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.130 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.130 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:16.130 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.104229+0000", 2023-10-09T05:52:16.130 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:16.130 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.130 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.130 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:16.130 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.104228+0000", 2023-10-09T05:52:16.130 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:16.130 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.131 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.131 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:16.131 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.104342+0000", 2023-10-09T05:52:16.131 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000114466 2023-10-09T05:52:16.131 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.131 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.131 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:16.131 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.104360+0000", 2023-10-09T05:52:16.131 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7884e-05 2023-10-09T05:52:16.131 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.131 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.131 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:16.132 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.104367+0000", 2023-10-09T05:52:16.132 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0099999999999998e-06 2023-10-09T05:52:16.132 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:16.132 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:16.132 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:16.132 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.132 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.132 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.44588.0:94379 6.a e137/130)", 2023-10-09T05:52:16.132 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:16.105452+0000", 2023-10-09T05:52:16.132 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 4294967295.9998274, 2023-10-09T05:52:16.133 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3130000000000001e-05, 2023-10-09T05:52:16.133 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:16.133 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "queued for pg", 2023-10-09T05:52:16.133 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:16.133 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.133 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:16.133 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.105452+0000", 2023-10-09T05:52:16.133 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:16.133 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.133 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.133 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:16.133 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.105452+0000", 2023-10-09T05:52:16.134 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:16.134 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.134 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.134 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:16.134 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.105451+0000", 2023-10-09T05:52:16.134 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:52:16.134 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.134 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.134 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:16.134 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.105480+0000", 2023-10-09T05:52:16.134 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9183000000000002e-05 2023-10-09T05:52:16.134 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.135 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.135 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:16.135 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.105493+0000", 2023-10-09T05:52:16.135 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2626000000000001e-05 2023-10-09T05:52:16.135 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:16.135 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:16.135 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:16.135 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:16.105499+0000", 2023-10-09T05:52:16.135 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4710000000000004e-06 2023-10-09T05:52:16.135 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:16.135 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:16.135 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:16.136 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:16.136 INFO:teuthology.orchestra.run.smithi156.stdout: ], 2023-10-09T05:52:16.136 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 4 2023-10-09T05:52:16.136 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:52:16.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:16.137 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:16.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:16.238 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:16.302 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:16.388 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:16.389 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 87 16 94622 94606 67.9308 46.125 0.0162633 0.0147157 2023-10-09T05:52:16.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:16.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:16.427 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:52:16.494 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:16.529 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:16.585 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:16.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:16.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:16.689 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:16.732 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:16.795 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:16.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:16.806 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:16 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:52:16.807 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:16 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:52:16.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:16 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:52:16.890 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T05:52:16.890 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:52:16.890 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T05:52:16.891 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-10-09T05:52:16.908 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:16.919 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:52:16.967 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:52:17.014 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:52:17.048 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:52:17.063 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:17.125 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:52:17.154 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:52:17.199 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:52:17.207 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:52:17.243 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:52:17.249 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:17.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:17.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:17.283 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:52:17.283 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:52:17.283 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:52:17.283 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:52:17.284 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:52:17.284 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:52:17.284 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:52:17.284 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:52:17.286 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:17.355 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:17.389 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 88 16 95260 95244 67.612 39.875 0.0331817 0.0147841 2023-10-09T05:52:17.421 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:17.509 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:17.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:17.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:17.595 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:17 smithi064 ceph-mon[212750]: pgmap v309: 99 pgs: 1 active+recovering+undersized+remapped, 90 active+clean, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 16 KiB/s rd, 33 MiB/s wr, 538 op/s; 11526/278856 objects degraded (4.133%); 1475/278856 objects misplaced (0.529%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T05:52:17.596 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:17 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2217821188' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:17.596 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:17 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 11526/278856 objects degraded (4.133%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:17.617 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:17.678 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:17.773 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:52:17.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:17 smithi156 ceph-mon[195101]: pgmap v309: 99 pgs: 1 active+recovering+undersized+remapped, 90 active+clean, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 16 KiB/s rd, 33 MiB/s wr, 538 op/s; 11526/278856 objects degraded (4.133%); 1475/278856 objects misplaced (0.529%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T05:52:17.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:17 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2217821188' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:17.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:17 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 11526/278856 objects degraded (4.133%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:17.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:17 smithi064 ceph-mon[220044]: pgmap v309: 99 pgs: 1 active+recovering+undersized+remapped, 90 active+clean, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 16 KiB/s rd, 33 MiB/s wr, 538 op/s; 11526/278856 objects degraded (4.133%); 1475/278856 objects misplaced (0.529%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T05:52:17.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:17 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2217821188' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:17.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:17 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 11526/278856 objects degraded (4.133%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:18.215 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:18.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:18.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:18.320 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:18.390 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 89 16 95987 95971 67.3622 45.4375 0.0203405 0.0148393 2023-10-09T05:52:18.398 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:18.596 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:18.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:18.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:18.704 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:18.719 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:18.744 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:18.805 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:18.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:18.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:18.909 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:18.950 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:19.023 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:19.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:19.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:19.127 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:19.156 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:19.200 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:19.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:19.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:19.302 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:19.363 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:19.393 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total time run: 90.0122 2023-10-09T05:52:19.394 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total writes made: 96540 2023-10-09T05:52:19.394 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Write size: 65536 2023-10-09T05:52:19.394 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object size: 65536 2023-10-09T05:52:19.394 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Bandwidth (MB/sec): 67.0326 2023-10-09T05:52:19.394 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Bandwidth: 38.5502 2023-10-09T05:52:19.394 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max bandwidth (MB/sec): 199.562 2023-10-09T05:52:19.394 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min bandwidth (MB/sec): 0 2023-10-09T05:52:19.394 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average IOPS: 1072 2023-10-09T05:52:19.394 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev IOPS: 616.802 2023-10-09T05:52:19.395 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max IOPS: 3193 2023-10-09T05:52:19.395 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min IOPS: 0 2023-10-09T05:52:19.395 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average Latency(s): 0.014914 2023-10-09T05:52:19.395 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Latency(s): 0.0503381 2023-10-09T05:52:19.395 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max latency(s): 2.26368 2023-10-09T05:52:19.395 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min latency(s): 0.00106234 2023-10-09T05:52:19.400 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T05:52:19.771 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:19.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:19.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:19.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:19 smithi156 ceph-mon[195101]: pgmap v310: 99 pgs: 1 active+recovering+undersized+remapped, 90 active+clean, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 13 KiB/s rd, 35 MiB/s wr, 565 op/s; 11526/280935 objects degraded (4.103%); 1475/280935 objects misplaced (0.525%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T05:52:19.881 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:19.916 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T05:52:19.916 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:52:19.943 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:19.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:19 smithi064 ceph-mon[220044]: pgmap v310: 99 pgs: 1 active+recovering+undersized+remapped, 90 active+clean, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 13 KiB/s rd, 35 MiB/s wr, 565 op/s; 11526/280935 objects degraded (4.103%); 1475/280935 objects misplaced (0.525%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T05:52:19.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:19 smithi064 ceph-mon[212750]: pgmap v310: 99 pgs: 1 active+recovering+undersized+remapped, 90 active+clean, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 13 KiB/s rd, 35 MiB/s wr, 565 op/s; 11526/280935 objects degraded (4.103%); 1475/280935 objects misplaced (0.525%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T05:52:20.121 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:20.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:20.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:20.229 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:20.286 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:20.361 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:52:20.363 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:20.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:20.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:20.468 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:20.494 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:20.536 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:20.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:20.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:20.643 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:20.746 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:20.901 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:52:20.945 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:20.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:20.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:21.056 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:21.110 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:21.182 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:21.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:21.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:21.288 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:21.376 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:21.436 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:52:21.569 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:21.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:21.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:21.626 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:21 smithi064 ceph-mon[212750]: pgmap v311: 99 pgs: 1 active+recovering+undersized+remapped, 90 active+clean, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 10 KiB/s rd, 44 MiB/s wr, 715 op/s; 11526/288882 objects degraded (3.990%); 1065/288882 objects misplaced (0.369%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T05:52:21.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:21 smithi064 ceph-mon[220044]: pgmap v311: 99 pgs: 1 active+recovering+undersized+remapped, 90 active+clean, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 10 KiB/s rd, 44 MiB/s wr, 715 op/s; 11526/288882 objects degraded (3.990%); 1065/288882 objects misplaced (0.369%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T05:52:21.682 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:21.736 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:21.789 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:21.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:21 smithi156 ceph-mon[195101]: pgmap v311: 99 pgs: 1 active+recovering+undersized+remapped, 90 active+clean, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 10 KiB/s rd, 44 MiB/s wr, 715 op/s; 11526/288882 objects degraded (3.990%); 1065/288882 objects misplaced (0.369%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T05:52:21.906 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:21.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:21.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:22.022 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:22.064 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:22.140 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:22.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:22.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:22.243 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:22.330 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:22.554 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:22.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:22.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:22.630 INFO:teuthology.orchestra.run.smithi064.stdout:moved 21 / 297 (7.07071%) 2023-10-09T05:52:22.631 INFO:teuthology.orchestra.run.smithi064.stdout:avg 37.125 2023-10-09T05:52:22.631 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 5.59994 -> 6.67903 (expected baseline 5.69951) 2023-10-09T05:52:22.631 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.5 with 29 -> 33 pgs (0.781145 -> 0.888889 * mean) 2023-10-09T05:52:22.631 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.3 with 46 -> 32 pgs (1.23906 -> 0.861953 * mean) 2023-10-09T05:52:22.631 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:52:22.631 INFO:teuthology.orchestra.run.smithi064.stdout:oload 110 2023-10-09T05:52:22.631 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 3 2023-10-09T05:52:22.631 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-10-09T05:52:22.631 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0268 2023-10-09T05:52:22.631 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.0295 2023-10-09T05:52:22.632 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 weight 1.0000 -> 0.6332 2023-10-09T05:52:22.632 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 weight 1.0000 -> 0.7587 2023-10-09T05:52:22.658 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T05:52:22.664 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:22.693 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:22.763 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:22.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:22.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:22.866 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:22.893 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:22 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3073799708' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:52:22.893 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:22 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 11526/288882 objects degraded (3.990%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:22.894 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:22 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'49722','3':'41495'}"}]: dispatch 2023-10-09T05:52:22.894 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:22 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'49722','3':'41495'}"}]: dispatch 2023-10-09T05:52:22.894 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:22 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3073799708' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:52:22.895 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:22 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 11526/288882 objects degraded (3.990%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:22.895 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:22 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'49722','3':'41495'}"}]: dispatch 2023-10-09T05:52:22.895 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:22 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'49722','3':'41495'}"}]: dispatch 2023-10-09T05:52:22.896 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:22.946 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:22.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:22.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:23.047 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:22 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3073799708' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:52:23.047 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:22 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 11526/288882 objects degraded (3.990%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:23.047 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:22 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'49722','3':'41495'}"}]: dispatch 2023-10-09T05:52:23.047 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:22 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'49722','3':'41495'}"}]: dispatch 2023-10-09T05:52:23.050 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:23.078 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:23.250 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T05:52:23.250 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:52:23.479 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:23.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:23.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:23.583 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:23.612 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:23.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[212750]: from='client.54601 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:23.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[212750]: pgmap v312: 99 pgs: 1 active+recovering+undersized+remapped, 90 active+clean, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 9.1 KiB/s rd, 40 MiB/s wr, 687 op/s; 11526/288384 objects degraded (3.997%); 1065/288384 objects misplaced (0.369%); 6.2 MiB/s, 100 objects/s recovering 2023-10-09T05:52:23.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:52:23.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3073799708' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:52:23.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'49722','3':'41495'}"}]': finished 2023-10-09T05:52:23.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[212750]: osdmap e138: 8 total, 8 up, 8 in 2023-10-09T05:52:23.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[212750]: failed to encode map e138 with expected crc 2023-10-09T05:52:23.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[212750]: failed to encode map e138 with expected crc 2023-10-09T05:52:23.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[212750]: failed to encode map e138 with expected crc 2023-10-09T05:52:23.648 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[212750]: failed to encode map e138 with expected crc 2023-10-09T05:52:23.648 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[212750]: failed to encode map e138 with expected crc 2023-10-09T05:52:23.648 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[212750]: failed to encode map e138 with expected crc 2023-10-09T05:52:23.648 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[212750]: failed to encode map e138 with expected crc 2023-10-09T05:52:23.648 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[212750]: osdmap e139: 8 total, 8 up, 8 in 2023-10-09T05:52:23.648 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[220044]: from='client.54601 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:23.648 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[220044]: pgmap v312: 99 pgs: 1 active+recovering+undersized+remapped, 90 active+clean, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 9.1 KiB/s rd, 40 MiB/s wr, 687 op/s; 11526/288384 objects degraded (3.997%); 1065/288384 objects misplaced (0.369%); 6.2 MiB/s, 100 objects/s recovering 2023-10-09T05:52:23.648 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:52:23.648 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3073799708' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:52:23.648 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'49722','3':'41495'}"}]': finished 2023-10-09T05:52:23.649 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[220044]: osdmap e138: 8 total, 8 up, 8 in 2023-10-09T05:52:23.649 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[220044]: failed to encode map e138 with expected crc 2023-10-09T05:52:23.649 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[220044]: failed to encode map e138 with expected crc 2023-10-09T05:52:23.649 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[220044]: failed to encode map e138 with expected crc 2023-10-09T05:52:23.649 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[220044]: failed to encode map e138 with expected crc 2023-10-09T05:52:23.649 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[220044]: failed to encode map e138 with expected crc 2023-10-09T05:52:23.649 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[220044]: failed to encode map e138 with expected crc 2023-10-09T05:52:23.649 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[220044]: failed to encode map e138 with expected crc 2023-10-09T05:52:23.649 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:23 smithi064 ceph-mon[220044]: osdmap e139: 8 total, 8 up, 8 in 2023-10-09T05:52:23.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:23 smithi156 ceph-mon[195101]: from='client.54601 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:23.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:23 smithi156 ceph-mon[195101]: pgmap v312: 99 pgs: 1 active+recovering+undersized+remapped, 90 active+clean, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 9.1 KiB/s rd, 40 MiB/s wr, 687 op/s; 11526/288384 objects degraded (3.997%); 1065/288384 objects misplaced (0.369%); 6.2 MiB/s, 100 objects/s recovering 2023-10-09T05:52:23.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:23 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:52:23.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:23 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3073799708' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:52:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:23 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'49722','3':'41495'}"}]': finished 2023-10-09T05:52:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:23 smithi156 ceph-mon[195101]: osdmap e138: 8 total, 8 up, 8 in 2023-10-09T05:52:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:23 smithi156 ceph-mon[195101]: failed to encode map e138 with expected crc 2023-10-09T05:52:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:23 smithi156 ceph-mon[195101]: failed to encode map e138 with expected crc 2023-10-09T05:52:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:23 smithi156 ceph-mon[195101]: failed to encode map e138 with expected crc 2023-10-09T05:52:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:23 smithi156 ceph-mon[195101]: failed to encode map e138 with expected crc 2023-10-09T05:52:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:23 smithi156 ceph-mon[195101]: failed to encode map e138 with expected crc 2023-10-09T05:52:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:23 smithi156 ceph-mon[195101]: failed to encode map e138 with expected crc 2023-10-09T05:52:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:23 smithi156 ceph-mon[195101]: failed to encode map e138 with expected crc 2023-10-09T05:52:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:23 smithi156 ceph-mon[195101]: osdmap e139: 8 total, 8 up, 8 in 2023-10-09T05:52:24.218 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:52:24.335 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:24.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:24.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:24.352 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T05:52:24.353 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-10-09T05:52:24.439 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:24.474 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:24.579 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:52:24.611 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:24.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:24.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:24.715 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:24.786 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:24.853 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:24.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:24.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:24.957 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:24.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[220044]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[220044]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[220044]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[220044]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[220044]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[220044]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[220044]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[220044]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[220044]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[220044]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[220044]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[220044]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[220044]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[220044]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[220044]: osdmap e140: 8 total, 8 up, 8 in 2023-10-09T05:52:24.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[212750]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[212750]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[212750]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[212750]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[212750]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[212750]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[212750]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[212750]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[212750]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[212750]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[212750]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[212750]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[212750]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[212750]: failed to encode map e139 with expected crc 2023-10-09T05:52:24.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:24 smithi064 ceph-mon[212750]: osdmap e140: 8 total, 8 up, 8 in 2023-10-09T05:52:25.019 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:24 smithi156 ceph-mon[195101]: failed to encode map e139 with expected crc 2023-10-09T05:52:25.019 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:24 smithi156 ceph-mon[195101]: failed to encode map e139 with expected crc 2023-10-09T05:52:25.019 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:24 smithi156 ceph-mon[195101]: failed to encode map e139 with expected crc 2023-10-09T05:52:25.019 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:24 smithi156 ceph-mon[195101]: failed to encode map e139 with expected crc 2023-10-09T05:52:25.019 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:24 smithi156 ceph-mon[195101]: failed to encode map e139 with expected crc 2023-10-09T05:52:25.019 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:24 smithi156 ceph-mon[195101]: failed to encode map e139 with expected crc 2023-10-09T05:52:25.020 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:24 smithi156 ceph-mon[195101]: failed to encode map e139 with expected crc 2023-10-09T05:52:25.020 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:24 smithi156 ceph-mon[195101]: failed to encode map e139 with expected crc 2023-10-09T05:52:25.020 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:24 smithi156 ceph-mon[195101]: failed to encode map e139 with expected crc 2023-10-09T05:52:25.020 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:24 smithi156 ceph-mon[195101]: failed to encode map e139 with expected crc 2023-10-09T05:52:25.020 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:24 smithi156 ceph-mon[195101]: failed to encode map e139 with expected crc 2023-10-09T05:52:25.020 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:24 smithi156 ceph-mon[195101]: failed to encode map e139 with expected crc 2023-10-09T05:52:25.020 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:24 smithi156 ceph-mon[195101]: failed to encode map e139 with expected crc 2023-10-09T05:52:25.020 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:24 smithi156 ceph-mon[195101]: failed to encode map e139 with expected crc 2023-10-09T05:52:25.020 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:24 smithi156 ceph-mon[195101]: osdmap e140: 8 total, 8 up, 8 in 2023-10-09T05:52:25.029 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:25.089 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:52:25.108 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:25.126 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:25.127 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:52:25.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:25.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:25.166 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:52:25.197 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:52:25.221 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:52:25.231 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:25.255 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:52:25.261 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:25.285 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:52:25.305 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:52:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:52:25] "GET /metrics HTTP/1.1" 200 36424 "" "Prometheus/2.43.0" 2023-10-09T05:52:25.310 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:52:25.316 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:25.316 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:52:25.316 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:52:25.316 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:52:25.316 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:52:25.316 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:52:25.317 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:52:25.317 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:52:25.317 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:52:25.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:25.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:25.420 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:25.450 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:25.499 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:25.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:25.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:25.607 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:25.638 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:25.704 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:25.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:25.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:25.808 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:25.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:25 smithi064 ceph-mon[212750]: pgmap v315: 99 pgs: 1 active+recovering+undersized+remapped, 6 remapped+peering, 85 active+clean, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 28 MiB/s wr, 680 op/s; 9425/284196 objects degraded (3.316%); 415/284196 objects misplaced (0.146%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T05:52:25.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:25 smithi064 ceph-mon[212750]: failed to encode map e140 with expected crc 2023-10-09T05:52:25.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:25 smithi064 ceph-mon[212750]: failed to encode map e140 with expected crc 2023-10-09T05:52:25.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:25 smithi064 ceph-mon[212750]: failed to encode map e140 with expected crc 2023-10-09T05:52:25.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:25 smithi064 ceph-mon[212750]: failed to encode map e140 with expected crc 2023-10-09T05:52:25.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:25 smithi064 ceph-mon[212750]: failed to encode map e140 with expected crc 2023-10-09T05:52:25.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:25 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2397395456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:25.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:25 smithi064 ceph-mon[212750]: osdmap e141: 8 total, 8 up, 8 in 2023-10-09T05:52:25.843 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:25 smithi064 ceph-mon[220044]: pgmap v315: 99 pgs: 1 active+recovering+undersized+remapped, 6 remapped+peering, 85 active+clean, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 28 MiB/s wr, 680 op/s; 9425/284196 objects degraded (3.316%); 415/284196 objects misplaced (0.146%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T05:52:25.843 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:25 smithi064 ceph-mon[220044]: failed to encode map e140 with expected crc 2023-10-09T05:52:25.843 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:25 smithi064 ceph-mon[220044]: failed to encode map e140 with expected crc 2023-10-09T05:52:25.843 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:25 smithi064 ceph-mon[220044]: failed to encode map e140 with expected crc 2023-10-09T05:52:25.843 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:25 smithi064 ceph-mon[220044]: failed to encode map e140 with expected crc 2023-10-09T05:52:25.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:25 smithi064 ceph-mon[220044]: failed to encode map e140 with expected crc 2023-10-09T05:52:25.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:25 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2397395456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:25.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:25 smithi064 ceph-mon[220044]: osdmap e141: 8 total, 8 up, 8 in 2023-10-09T05:52:25.846 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:26.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:25 smithi156 ceph-mon[195101]: pgmap v315: 99 pgs: 1 active+recovering+undersized+remapped, 6 remapped+peering, 85 active+clean, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 28 MiB/s wr, 680 op/s; 9425/284196 objects degraded (3.316%); 415/284196 objects misplaced (0.146%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T05:52:26.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:25 smithi156 ceph-mon[195101]: failed to encode map e140 with expected crc 2023-10-09T05:52:26.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:25 smithi156 ceph-mon[195101]: failed to encode map e140 with expected crc 2023-10-09T05:52:26.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:25 smithi156 ceph-mon[195101]: failed to encode map e140 with expected crc 2023-10-09T05:52:26.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:25 smithi156 ceph-mon[195101]: failed to encode map e140 with expected crc 2023-10-09T05:52:26.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:25 smithi156 ceph-mon[195101]: failed to encode map e140 with expected crc 2023-10-09T05:52:26.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:25 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2397395456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:26.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:25 smithi156 ceph-mon[195101]: osdmap e141: 8 total, 8 up, 8 in 2023-10-09T05:52:26.270 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:26.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:26.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:26.376 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:26.404 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:26.453 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:26.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:26.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:26.557 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:26.558 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:52:26.596 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:26.918 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:26.921 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:26 smithi064 ceph-mon[212750]: failed to encode map e141 with expected crc 2023-10-09T05:52:26.921 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:26 smithi064 ceph-mon[212750]: failed to encode map e141 with expected crc 2023-10-09T05:52:26.921 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:26 smithi064 ceph-mon[212750]: failed to encode map e141 with expected crc 2023-10-09T05:52:26.921 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:26 smithi064 ceph-mon[212750]: failed to encode map e141 with expected crc 2023-10-09T05:52:26.921 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:26 smithi064 ceph-mon[212750]: failed to encode map e141 with expected crc 2023-10-09T05:52:26.921 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:26 smithi064 ceph-mon[212750]: osdmap e142: 8 total, 8 up, 8 in 2023-10-09T05:52:26.921 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:26 smithi064 ceph-mon[220044]: failed to encode map e141 with expected crc 2023-10-09T05:52:26.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:26 smithi064 ceph-mon[220044]: failed to encode map e141 with expected crc 2023-10-09T05:52:26.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:26 smithi064 ceph-mon[220044]: failed to encode map e141 with expected crc 2023-10-09T05:52:26.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:26 smithi064 ceph-mon[220044]: failed to encode map e141 with expected crc 2023-10-09T05:52:26.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:26 smithi064 ceph-mon[220044]: failed to encode map e141 with expected crc 2023-10-09T05:52:26.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:26 smithi064 ceph-mon[220044]: osdmap e142: 8 total, 8 up, 8 in 2023-10-09T05:52:26.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:26.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:27.024 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:27.051 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:26 smithi156 ceph-mon[195101]: failed to encode map e141 with expected crc 2023-10-09T05:52:27.051 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:26 smithi156 ceph-mon[195101]: failed to encode map e141 with expected crc 2023-10-09T05:52:27.051 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:26 smithi156 ceph-mon[195101]: failed to encode map e141 with expected crc 2023-10-09T05:52:27.051 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:26 smithi156 ceph-mon[195101]: failed to encode map e141 with expected crc 2023-10-09T05:52:27.051 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:26 smithi156 ceph-mon[195101]: failed to encode map e141 with expected crc 2023-10-09T05:52:27.051 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:26 smithi156 ceph-mon[195101]: osdmap e142: 8 total, 8 up, 8 in 2023-10-09T05:52:27.054 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:27.141 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:27.156 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:27.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:27.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:27.266 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:27.303 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:27.411 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:52:27.411 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T05:52:27.411 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 0 2023-10-09T05:52:27.411 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:52:27.486 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:27.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:27.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:27.597 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:27.642 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:27.697 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:27 smithi156 ceph-mon[195101]: pgmap v318: 99 pgs: 1 active+recovering+undersized+remapped, 6 remapped+peering, 85 active+clean, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 426 op/s; 9425/283263 objects degraded (3.327%); 415/283263 objects misplaced (0.147%); 10 MiB/s, 162 objects/s recovering 2023-10-09T05:52:27.697 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:27 smithi156 ceph-mon[195101]: failed to encode map e142 with expected crc 2023-10-09T05:52:27.697 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:27 smithi156 ceph-mon[195101]: failed to encode map e142 with expected crc 2023-10-09T05:52:27.697 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:27 smithi156 ceph-mon[195101]: failed to encode map e142 with expected crc 2023-10-09T05:52:27.697 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:27 smithi156 ceph-mon[195101]: failed to encode map e142 with expected crc 2023-10-09T05:52:27.697 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:27 smithi156 ceph-mon[195101]: failed to encode map e142 with expected crc 2023-10-09T05:52:27.698 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:27 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 9425/283263 objects degraded (3.327%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:27.862 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:27 smithi064 ceph-mon[212750]: pgmap v318: 99 pgs: 1 active+recovering+undersized+remapped, 6 remapped+peering, 85 active+clean, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 426 op/s; 9425/283263 objects degraded (3.327%); 415/283263 objects misplaced (0.147%); 10 MiB/s, 162 objects/s recovering 2023-10-09T05:52:27.862 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:27 smithi064 ceph-mon[212750]: failed to encode map e142 with expected crc 2023-10-09T05:52:27.862 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:27 smithi064 ceph-mon[212750]: failed to encode map e142 with expected crc 2023-10-09T05:52:27.862 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:27 smithi064 ceph-mon[212750]: failed to encode map e142 with expected crc 2023-10-09T05:52:27.862 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:27 smithi064 ceph-mon[212750]: failed to encode map e142 with expected crc 2023-10-09T05:52:27.862 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:27 smithi064 ceph-mon[212750]: failed to encode map e142 with expected crc 2023-10-09T05:52:27.862 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:27 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 9425/283263 objects degraded (3.327%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:27.862 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:27 smithi064 ceph-mon[220044]: pgmap v318: 99 pgs: 1 active+recovering+undersized+remapped, 6 remapped+peering, 85 active+clean, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 426 op/s; 9425/283263 objects degraded (3.327%); 415/283263 objects misplaced (0.147%); 10 MiB/s, 162 objects/s recovering 2023-10-09T05:52:27.863 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:27 smithi064 ceph-mon[220044]: failed to encode map e142 with expected crc 2023-10-09T05:52:27.863 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:27 smithi064 ceph-mon[220044]: failed to encode map e142 with expected crc 2023-10-09T05:52:27.863 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:27 smithi064 ceph-mon[220044]: failed to encode map e142 with expected crc 2023-10-09T05:52:27.863 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:27 smithi064 ceph-mon[220044]: failed to encode map e142 with expected crc 2023-10-09T05:52:27.863 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:27 smithi064 ceph-mon[220044]: failed to encode map e142 with expected crc 2023-10-09T05:52:27.863 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:27 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 9425/283263 objects degraded (3.327%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:27.916 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:27.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:27.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:28.020 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:28.054 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:28.110 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:28.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:28.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:28.214 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:28.246 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:28.297 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:28.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:28.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:28.400 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:28.428 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:28.478 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:28.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:28.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:28.582 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:28.611 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:28.660 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:28.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:28.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:28.722 INFO:teuthology.orchestra.run.smithi064.stdout:moved 8 / 297 (2.6936%) 2023-10-09T05:52:28.722 INFO:teuthology.orchestra.run.smithi064.stdout:avg 37.125 2023-10-09T05:52:28.722 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 6.67903 -> 8.53577 (expected baseline 5.69951) 2023-10-09T05:52:28.722 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.2 with 25 -> 22 pgs (0.673401 -> 0.592593 * mean) 2023-10-09T05:52:28.722 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.1 with 45 -> 46 pgs (1.21212 -> 1.23906 * mean) 2023-10-09T05:52:28.723 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:52:28.723 INFO:teuthology.orchestra.run.smithi064.stdout:oload 110 2023-10-09T05:52:28.723 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 0.05 2023-10-09T05:52:28.723 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-10-09T05:52:28.723 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0264 2023-10-09T05:52:28.723 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.0290 2023-10-09T05:52:28.723 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 weight 0.6332 -> 0.5832 2023-10-09T05:52:28.723 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 weight 0.7587 -> 0.7087 2023-10-09T05:52:28.735 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:52:28.764 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:28.793 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:28.839 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:28.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:28.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:28.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:28 smithi156 ceph-mon[195101]: failed to encode map e143 with expected crc 2023-10-09T05:52:28.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:28 smithi156 ceph-mon[195101]: osdmap e143: 8 total, 8 up, 8 in 2023-10-09T05:52:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:28 smithi156 ceph-mon[195101]: failed to encode map e143 with expected crc 2023-10-09T05:52:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:28 smithi156 ceph-mon[195101]: failed to encode map e143 with expected crc 2023-10-09T05:52:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:28 smithi156 ceph-mon[195101]: from='client.34797 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:28 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'46446','3':'38219'}"}]: dispatch 2023-10-09T05:52:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:28 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'46446','3':'38219'}"}]: dispatch 2023-10-09T05:52:28.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:28 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:28.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:28 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:52:28.945 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:28.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:28 smithi064 ceph-mon[212750]: failed to encode map e143 with expected crc 2023-10-09T05:52:28.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:28 smithi064 ceph-mon[212750]: osdmap e143: 8 total, 8 up, 8 in 2023-10-09T05:52:28.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:28 smithi064 ceph-mon[212750]: failed to encode map e143 with expected crc 2023-10-09T05:52:28.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:28 smithi064 ceph-mon[212750]: failed to encode map e143 with expected crc 2023-10-09T05:52:28.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:28 smithi064 ceph-mon[212750]: from='client.34797 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:28.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:28 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'46446','3':'38219'}"}]: dispatch 2023-10-09T05:52:28.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:28 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'46446','3':'38219'}"}]: dispatch 2023-10-09T05:52:28.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:28 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:28.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:28 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:52:28.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:28 smithi064 ceph-mon[220044]: failed to encode map e143 with expected crc 2023-10-09T05:52:28.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:28 smithi064 ceph-mon[220044]: osdmap e143: 8 total, 8 up, 8 in 2023-10-09T05:52:28.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:28 smithi064 ceph-mon[220044]: failed to encode map e143 with expected crc 2023-10-09T05:52:28.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:28 smithi064 ceph-mon[220044]: failed to encode map e143 with expected crc 2023-10-09T05:52:28.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:28 smithi064 ceph-mon[220044]: from='client.34797 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:28.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:28 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'46446','3':'38219'}"}]: dispatch 2023-10-09T05:52:28.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:28 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'46446','3':'38219'}"}]: dispatch 2023-10-09T05:52:28.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:28 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:28.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:28 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:52:29.008 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:29.077 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:29.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:29.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:29.181 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:29.224 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:29.462 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:29.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:29.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:29.566 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:29.586 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T05:52:29.587 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 3.0 2023-10-09T05:52:29.608 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:29.694 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:29.697 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:29.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:29.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:29.727 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:29 smithi064 ceph-mon[212750]: pgmap v321: 99 pgs: 1 active+recovering+undersized+remapped, 6 remapped+peering, 85 active+clean, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 85 op/s; 9425/283173 objects degraded (3.328%); 415/283173 objects misplaced (0.147%) 2023-10-09T05:52:29.727 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:29 smithi064 ceph-mon[212750]: failed to encode map e144 with expected crc 2023-10-09T05:52:29.728 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:29 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'46446','3':'38219'}"}]': finished 2023-10-09T05:52:29.728 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:29 smithi064 ceph-mon[212750]: osdmap e144: 8 total, 8 up, 8 in 2023-10-09T05:52:29.728 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:29 smithi064 ceph-mon[212750]: failed to encode map e144 with expected crc 2023-10-09T05:52:29.728 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:29 smithi064 ceph-mon[212750]: failed to encode map e144 with expected crc 2023-10-09T05:52:29.728 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:29 smithi064 ceph-mon[212750]: failed to encode map e144 with expected crc 2023-10-09T05:52:29.728 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:29 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2121632754' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:52:29.729 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:29 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:52:29.729 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:29 smithi064 ceph-mon[220044]: pgmap v321: 99 pgs: 1 active+recovering+undersized+remapped, 6 remapped+peering, 85 active+clean, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 85 op/s; 9425/283173 objects degraded (3.328%); 415/283173 objects misplaced (0.147%) 2023-10-09T05:52:29.729 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:29 smithi064 ceph-mon[220044]: failed to encode map e144 with expected crc 2023-10-09T05:52:29.729 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:29 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'46446','3':'38219'}"}]': finished 2023-10-09T05:52:29.729 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:29 smithi064 ceph-mon[220044]: osdmap e144: 8 total, 8 up, 8 in 2023-10-09T05:52:29.730 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:29 smithi064 ceph-mon[220044]: failed to encode map e144 with expected crc 2023-10-09T05:52:29.730 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:29 smithi064 ceph-mon[220044]: failed to encode map e144 with expected crc 2023-10-09T05:52:29.730 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:29 smithi064 ceph-mon[220044]: failed to encode map e144 with expected crc 2023-10-09T05:52:29.730 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:29 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2121632754' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:52:29.730 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:29 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:52:29.740 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T05:52:29.801 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:29.832 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:29.981 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:29.985 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:29 smithi156 ceph-mon[195101]: pgmap v321: 99 pgs: 1 active+recovering+undersized+remapped, 6 remapped+peering, 85 active+clean, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 85 op/s; 9425/283173 objects degraded (3.328%); 415/283173 objects misplaced (0.147%) 2023-10-09T05:52:29.985 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:29 smithi156 ceph-mon[195101]: failed to encode map e144 with expected crc 2023-10-09T05:52:29.985 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:29 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'46446','3':'38219'}"}]': finished 2023-10-09T05:52:29.986 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:29 smithi156 ceph-mon[195101]: osdmap e144: 8 total, 8 up, 8 in 2023-10-09T05:52:29.986 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:29 smithi156 ceph-mon[195101]: failed to encode map e144 with expected crc 2023-10-09T05:52:29.986 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:29 smithi156 ceph-mon[195101]: failed to encode map e144 with expected crc 2023-10-09T05:52:29.986 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:29 smithi156 ceph-mon[195101]: failed to encode map e144 with expected crc 2023-10-09T05:52:29.986 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:29 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2121632754' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:52:29.986 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:29 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:52:29.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:29.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:30.058 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:52:30.088 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:30.123 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:30.459 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:52:30.478 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:30.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:30.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:30.582 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:30.621 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:30.808 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:30 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:52:30.808 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:30 smithi064 ceph-mon[212750]: failed to encode map e145 with expected crc 2023-10-09T05:52:30.808 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:30 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:52:30.808 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:30 smithi064 ceph-mon[212750]: osdmap e145: 8 total, 8 up, 8 in 2023-10-09T05:52:30.808 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:30 smithi064 ceph-mon[212750]: failed to encode map e145 with expected crc 2023-10-09T05:52:30.808 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:30 smithi064 ceph-mon[212750]: failed to encode map e145 with expected crc 2023-10-09T05:52:30.808 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:30 smithi064 ceph-mon[212750]: failed to encode map e145 with expected crc 2023-10-09T05:52:30.809 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:30 smithi064 ceph-mon[212750]: failed to encode map e145 with expected crc 2023-10-09T05:52:30.809 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:30 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:52:30.809 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:30 smithi064 ceph-mon[220044]: failed to encode map e145 with expected crc 2023-10-09T05:52:30.809 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:30 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:52:30.809 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:30 smithi064 ceph-mon[220044]: osdmap e145: 8 total, 8 up, 8 in 2023-10-09T05:52:30.809 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:30 smithi064 ceph-mon[220044]: failed to encode map e145 with expected crc 2023-10-09T05:52:30.809 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:30 smithi064 ceph-mon[220044]: failed to encode map e145 with expected crc 2023-10-09T05:52:30.809 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:30 smithi064 ceph-mon[220044]: failed to encode map e145 with expected crc 2023-10-09T05:52:30.809 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:30 smithi064 ceph-mon[220044]: failed to encode map e145 with expected crc 2023-10-09T05:52:30.844 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:30.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:30.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:30.948 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:30.982 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:30 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:52:30.982 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:30 smithi156 ceph-mon[195101]: failed to encode map e145 with expected crc 2023-10-09T05:52:30.983 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:30 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:52:30.983 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:30 smithi156 ceph-mon[195101]: osdmap e145: 8 total, 8 up, 8 in 2023-10-09T05:52:30.983 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:30 smithi156 ceph-mon[195101]: failed to encode map e145 with expected crc 2023-10-09T05:52:30.983 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:30 smithi156 ceph-mon[195101]: failed to encode map e145 with expected crc 2023-10-09T05:52:30.983 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:30 smithi156 ceph-mon[195101]: failed to encode map e145 with expected crc 2023-10-09T05:52:30.983 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:30 smithi156 ceph-mon[195101]: failed to encode map e145 with expected crc 2023-10-09T05:52:30.991 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:31.042 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:31.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:31.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:31.147 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:31.176 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:31.224 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:31.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:31.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:31.330 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:31.366 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:31.377 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:31.449 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:31.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:31.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:31.553 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:31.581 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:31.634 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:31.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:31.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:31.737 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:31.764 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:31.816 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:31.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:31.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:31 smithi156 ceph-mon[195101]: pgmap v324: 99 pgs: 1 unknown, 3 peering, 86 active+clean, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 12 op/s; 14985/283113 objects degraded (5.293%); 11123/283113 objects misplaced (3.929%); 16 MiB/s, 252 objects/s recovering 2023-10-09T05:52:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:31 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:52:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:31 smithi156 ceph-mon[195101]: failed to encode map e146 with expected crc 2023-10-09T05:52:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:31 smithi156 ceph-mon[195101]: osdmap e146: 8 total, 8 up, 8 in 2023-10-09T05:52:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:31 smithi156 ceph-mon[195101]: failed to encode map e146 with expected crc 2023-10-09T05:52:31.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:31 smithi156 ceph-mon[195101]: failed to encode map e146 with expected crc 2023-10-09T05:52:31.919 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:31.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:31 smithi064 ceph-mon[212750]: pgmap v324: 99 pgs: 1 unknown, 3 peering, 86 active+clean, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 12 op/s; 14985/283113 objects degraded (5.293%); 11123/283113 objects misplaced (3.929%); 16 MiB/s, 252 objects/s recovering 2023-10-09T05:52:31.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:31 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:52:31.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:31 smithi064 ceph-mon[212750]: failed to encode map e146 with expected crc 2023-10-09T05:52:31.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:31 smithi064 ceph-mon[212750]: osdmap e146: 8 total, 8 up, 8 in 2023-10-09T05:52:31.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:31 smithi064 ceph-mon[212750]: failed to encode map e146 with expected crc 2023-10-09T05:52:31.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:31 smithi064 ceph-mon[212750]: failed to encode map e146 with expected crc 2023-10-09T05:52:31.955 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:31 smithi064 ceph-mon[220044]: pgmap v324: 99 pgs: 1 unknown, 3 peering, 86 active+clean, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 12 op/s; 14985/283113 objects degraded (5.293%); 11123/283113 objects misplaced (3.929%); 16 MiB/s, 252 objects/s recovering 2023-10-09T05:52:31.955 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:31 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:52:31.955 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:31 smithi064 ceph-mon[220044]: failed to encode map e146 with expected crc 2023-10-09T05:52:31.955 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:31 smithi064 ceph-mon[220044]: osdmap e146: 8 total, 8 up, 8 in 2023-10-09T05:52:31.955 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:31 smithi064 ceph-mon[220044]: failed to encode map e146 with expected crc 2023-10-09T05:52:31.955 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:31 smithi064 ceph-mon[220044]: failed to encode map e146 with expected crc 2023-10-09T05:52:31.958 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:32.167 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:32.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:32.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:32.273 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:32.300 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:32.351 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:32.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:32.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:32.458 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:32.485 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:32.533 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:32.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:32.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:32.636 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:32.668 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:32.675 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:52:32.787 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:32.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:32.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:32.892 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:32.930 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:32 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2507141401' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:32.930 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:32 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 14985/283113 objects degraded (5.293%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:32.930 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:32 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2507141401' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:32.930 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:32 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 14985/283113 objects degraded (5.293%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:32.938 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:33.007 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:33.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:33.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:33.112 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:33.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:32 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2507141401' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:52:33.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:32 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 14985/283113 objects degraded (5.293%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:33.152 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:33.331 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:33.377 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:33.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:33.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:33.484 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:33.526 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:33.658 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:33.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:33.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:33.672 INFO:teuthology.orchestra.run.smithi064.stdout:moved 0 / 297 (0%) 2023-10-09T05:52:33.672 INFO:teuthology.orchestra.run.smithi064.stdout:avg 37.125 2023-10-09T05:52:33.672 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 8.53577 -> 8.53577 (expected baseline 5.69951) 2023-10-09T05:52:33.673 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.2 with 22 -> 22 pgs (0.592593 -> 0.592593 * mean) 2023-10-09T05:52:33.673 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.1 with 46 -> 46 pgs (1.23906 -> 1.23906 * mean) 2023-10-09T05:52:33.673 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:52:33.673 INFO:teuthology.orchestra.run.smithi064.stdout:oload 1000 2023-10-09T05:52:33.673 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 3 2023-10-09T05:52:33.673 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-10-09T05:52:33.673 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0264 2023-10-09T05:52:33.673 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.2640 2023-10-09T05:52:33.675 INFO:teuthology.orchestra.run.smithi064.stderr:no change 2023-10-09T05:52:33.765 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:33.803 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:34 smithi064 ceph-mon[212750]: pgmap v326: 99 pgs: 1 unknown, 4 peering, 85 active+clean, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 4 op/s; 14985/283113 objects degraded (5.293%); 11123/283113 objects misplaced (3.929%); 13 MiB/s, 210 objects/s recovering 2023-10-09T05:52:34.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:34 smithi064 ceph-mon[220044]: pgmap v326: 99 pgs: 1 unknown, 4 peering, 85 active+clean, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 4 op/s; 14985/283113 objects degraded (5.293%); 11123/283113 objects misplaced (3.929%); 13 MiB/s, 210 objects/s recovering 2023-10-09T05:52:34.237 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:34 smithi156 ceph-mon[195101]: pgmap v326: 99 pgs: 1 unknown, 4 peering, 85 active+clean, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 4 op/s; 14985/283113 objects degraded (5.293%); 11123/283113 objects misplaced (3.929%); 13 MiB/s, 210 objects/s recovering 2023-10-09T05:52:34.462 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:52:34.462 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T05:52:34.462 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T05:52:34.463 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T05:52:34.463 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:52:34.472 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:34.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:34.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:34.577 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:34.611 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:34.706 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T05:52:34.706 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-10-09T05:52:34.952 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:34.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:34.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:35.057 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:35.106 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:35.285 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:35.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:35.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:35.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:35 smithi156 ceph-mon[195101]: from='client.54625 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:35.390 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:35.425 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:35.449 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:35.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:35 smithi064 ceph-mon[212750]: from='client.54625 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:35.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:35 smithi064 ceph-mon[220044]: from='client.54625 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:35.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:52:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:52:35] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.43.0" 2023-10-09T05:52:35.475 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:35.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:35.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:35.580 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:35.609 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:35.661 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:35.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:35.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:35.766 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:35.794 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:35.840 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:35.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:35.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:35.943 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:35.992 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:36.098 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:36.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:36.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:36.136 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T05:52:36.204 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:36.235 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:36.281 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:36.282 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:36 smithi064 ceph-mon[212750]: pgmap v327: 99 pgs: 1 unknown, 1 peering, 87 active+clean, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 3 op/s; 21007/283113 objects degraded (7.420%); 10436/283113 objects misplaced (3.686%); 18 MiB/s, 282 objects/s recovering 2023-10-09T05:52:36.282 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:36 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T05:52:36.282 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:36 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3659728609' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:52:36.282 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:36 smithi064 ceph-mon[220044]: pgmap v327: 99 pgs: 1 unknown, 1 peering, 87 active+clean, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 3 op/s; 21007/283113 objects degraded (7.420%); 10436/283113 objects misplaced (3.686%); 18 MiB/s, 282 objects/s recovering 2023-10-09T05:52:36.283 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:36 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T05:52:36.283 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:36 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3659728609' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:52:36.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:36.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:36.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:36 smithi156 ceph-mon[195101]: pgmap v327: 99 pgs: 1 unknown, 1 peering, 87 active+clean, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 3 op/s; 21007/283113 objects degraded (7.420%); 10436/283113 objects misplaced (3.686%); 18 MiB/s, 282 objects/s recovering 2023-10-09T05:52:36.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:36 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T05:52:36.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:36 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3659728609' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:52:36.385 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:36.419 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:37.108 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:37.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:37.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:37.214 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:37.242 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:37.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:37 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:52:37.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:37 smithi156 ceph-mon[195101]: failed to encode map e147 with expected crc 2023-10-09T05:52:37.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:37 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3659728609' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:52:37.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:37 smithi156 ceph-mon[195101]: osdmap e147: 8 total, 8 up, 8 in 2023-10-09T05:52:37.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:37 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 21007/283113 objects degraded (7.420%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:37.386 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:37 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:52:37.386 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:37 smithi064 ceph-mon[212750]: failed to encode map e147 with expected crc 2023-10-09T05:52:37.386 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:37 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3659728609' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:52:37.386 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:37 smithi064 ceph-mon[212750]: osdmap e147: 8 total, 8 up, 8 in 2023-10-09T05:52:37.386 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:37 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 21007/283113 objects degraded (7.420%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:37.387 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:37 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:52:37.387 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:37 smithi064 ceph-mon[220044]: failed to encode map e147 with expected crc 2023-10-09T05:52:37.387 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:37 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3659728609' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:52:37.387 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:37 smithi064 ceph-mon[220044]: osdmap e147: 8 total, 8 up, 8 in 2023-10-09T05:52:37.387 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:37 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 21007/283113 objects degraded (7.420%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:37.387 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:37.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:37.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:37.449 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:52:37.492 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:37.539 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:37.604 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:37.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:37.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:37.708 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:37.749 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:37.813 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:37.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:37.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:37.841 INFO:teuthology.orchestra.run.smithi064.stdout:moved 0 / 297 (0%) 2023-10-09T05:52:37.841 INFO:teuthology.orchestra.run.smithi064.stdout:avg 37.125 2023-10-09T05:52:37.841 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 8.53577 -> 8.53577 (expected baseline 5.69951) 2023-10-09T05:52:37.841 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.2 with 22 -> 22 pgs (0.592593 -> 0.592593 * mean) 2023-10-09T05:52:37.841 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.1 with 46 -> 46 pgs (1.23906 -> 1.23906 * mean) 2023-10-09T05:52:37.841 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:52:37.842 INFO:teuthology.orchestra.run.smithi064.stdout:oload 1000 2023-10-09T05:52:37.842 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 1 2023-10-09T05:52:37.842 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-10-09T05:52:37.842 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0264 2023-10-09T05:52:37.842 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.2644 2023-10-09T05:52:37.844 INFO:teuthology.orchestra.run.smithi064.stderr:no change 2023-10-09T05:52:37.856 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:52:37.918 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:37.964 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:38.069 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:38.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:38.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:38.173 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:38.204 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:38.253 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:38.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:38.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:38.340 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:38 smithi156 ceph-mon[195101]: pgmap v329: 99 pgs: 1 peering, 87 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 27148/283113 objects degraded (9.589%); 10436/283113 objects misplaced (3.686%); 6.4 MiB/s, 101 objects/s recovering 2023-10-09T05:52:38.356 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:38.384 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:38 smithi064 ceph-mon[212750]: pgmap v329: 99 pgs: 1 peering, 87 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 27148/283113 objects degraded (9.589%); 10436/283113 objects misplaced (3.686%); 6.4 MiB/s, 101 objects/s recovering 2023-10-09T05:52:38.384 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:38 smithi064 ceph-mon[220044]: pgmap v329: 99 pgs: 1 peering, 87 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 27148/283113 objects degraded (9.589%); 10436/283113 objects misplaced (3.686%); 6.4 MiB/s, 101 objects/s recovering 2023-10-09T05:52:38.387 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:38.604 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:38.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:38.609 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T05:52:38.609 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-10-09T05:52:38.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:38.711 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:38.752 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:38.818 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:38.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:38.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:38.922 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:38.961 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:39.024 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:39.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:39.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:39.123 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:39.129 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:39.158 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:39.208 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:39.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:39.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:39.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:39 smithi064 ceph-mon[212750]: from='client.34821 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:39.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:39 smithi064 ceph-mon[220044]: from='client.34821 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:39.229 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:52:39.311 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:39.351 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:39.441 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:39.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:39.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:39.546 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:39.571 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:39 smithi156 ceph-mon[195101]: from='client.34821 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:39.574 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:39.622 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:39.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:39.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:39.726 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:39.754 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:39.804 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:39.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:39.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:39.908 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:39.943 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:39.998 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:40.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:40.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:40.102 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:40.138 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:40.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:40 smithi064 ceph-mon[212750]: pgmap v330: 99 pgs: 1 peering, 87 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 27148/283113 objects degraded (9.589%); 10436/283113 objects misplaced (3.686%); 5.4 MiB/s, 85 objects/s recovering 2023-10-09T05:52:40.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:40 smithi064 ceph-mon[220044]: pgmap v330: 99 pgs: 1 peering, 87 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 27148/283113 objects degraded (9.589%); 10436/283113 objects misplaced (3.686%); 5.4 MiB/s, 85 objects/s recovering 2023-10-09T05:52:40.335 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:40.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:40.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:40.439 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:40.485 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:40.526 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:40.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:40 smithi156 ceph-mon[195101]: pgmap v330: 99 pgs: 1 peering, 87 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 27148/283113 objects degraded (9.589%); 10436/283113 objects misplaced (3.686%); 5.4 MiB/s, 85 objects/s recovering 2023-10-09T05:52:40.649 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:40.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:40.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:40.753 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:40.782 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:40.833 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:40.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:40.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:40.939 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:40.968 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:41.118 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:41.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:41.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:41.168 INFO:teuthology.orchestra.run.smithi064.stdout:moved 0 / 297 (0%) 2023-10-09T05:52:41.168 INFO:teuthology.orchestra.run.smithi064.stdout:avg 37.125 2023-10-09T05:52:41.169 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 8.53577 -> 8.53577 (expected baseline 5.69951) 2023-10-09T05:52:41.169 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.2 with 22 -> 22 pgs (0.592593 -> 0.592593 * mean) 2023-10-09T05:52:41.169 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.1 with 46 -> 46 pgs (1.23906 -> 1.23906 * mean) 2023-10-09T05:52:41.169 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:52:41.169 INFO:teuthology.orchestra.run.smithi064.stdout:oload 1000 2023-10-09T05:52:41.169 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 0.05 2023-10-09T05:52:41.169 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-10-09T05:52:41.169 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0265 2023-10-09T05:52:41.169 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.2650 2023-10-09T05:52:41.171 INFO:teuthology.orchestra.run.smithi064.stderr:no change 2023-10-09T05:52:41.225 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:41.268 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:41.612 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:52:41.612 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T05:52:41.612 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T05:52:41.612 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T05:52:41.612 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.612 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:45183 6.5 6:aac00114:::benchmark_data_smithi064_284193_object45182:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T05:52:41.612 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:13.801557+0000", 2023-10-09T05:52:41.612 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 87.806968214999998, 2023-10-09T05:52:41.612 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.33244016100000001, 2023-10-09T05:52:41.613 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:41.613 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:41.613 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:41.613 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:41.613 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:41.613 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 45183 2023-10-09T05:52:41.613 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.613 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:41.613 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.613 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:41.613 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801557+0000", 2023-10-09T05:52:41.613 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.614 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.614 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.614 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:41.614 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801557+0000", 2023-10-09T05:52:41.614 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.614 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.614 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.614 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:41.614 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801555+0000", 2023-10-09T05:52:41.614 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:41.614 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.615 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.615 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:41.615 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801742+0000", 2023-10-09T05:52:41.615 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000186954 2023-10-09T05:52:41.615 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.615 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.615 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:41.615 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801755+0000", 2023-10-09T05:52:41.615 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2575000000000001e-05 2023-10-09T05:52:41.615 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.615 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.615 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:41.616 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801762+0000", 2023-10-09T05:52:41.616 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.827e-06 2023-10-09T05:52:41.616 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.616 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.616 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.616 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801828+0000", 2023-10-09T05:52:41.616 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6237999999999998e-05 2023-10-09T05:52:41.616 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.616 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.616 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:41.616 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801958+0000", 2023-10-09T05:52:41.616 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013046299999999999 2023-10-09T05:52:41.617 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.617 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.617 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:52:41.617 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.802104+0000", 2023-10-09T05:52:41.617 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000145465 2023-10-09T05:52:41.617 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.617 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.617 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.617 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.804037+0000", 2023-10-09T05:52:41.617 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001933113 2023-10-09T05:52:41.617 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.617 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.618 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:41.618 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.805890+0000", 2023-10-09T05:52:41.618 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00185311 2023-10-09T05:52:41.618 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.618 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.618 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.618 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:14.133916+0000", 2023-10-09T05:52:41.618 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.32802647899999998 2023-10-09T05:52:41.618 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.618 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.618 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:41.618 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:14.133958+0000", 2023-10-09T05:52:41.619 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1396999999999997e-05 2023-10-09T05:52:41.619 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.619 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.619 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:41.619 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:14.133997+0000", 2023-10-09T05:52:41.619 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9158e-05 2023-10-09T05:52:41.619 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.619 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:41.619 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.619 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.619 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.619 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:45190 6.8 6:1669d52f:::benchmark_data_smithi064_284193_object45189:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T05:52:41.620 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:13.804266+0000", 2023-10-09T05:52:41.620 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 87.804259102000003, 2023-10-09T05:52:41.620 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.32912121900000002, 2023-10-09T05:52:41.620 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:41.620 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:41.620 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:41.620 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:41.620 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:41.620 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 45190 2023-10-09T05:52:41.620 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.620 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:41.620 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.620 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:41.621 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.804266+0000", 2023-10-09T05:52:41.621 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.621 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.621 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.621 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:41.621 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.804266+0000", 2023-10-09T05:52:41.621 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.621 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.621 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.621 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:41.621 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.804264+0000", 2023-10-09T05:52:41.621 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:41.622 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.622 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.622 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:41.622 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.804311+0000", 2023-10-09T05:52:41.622 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6177999999999997e-05 2023-10-09T05:52:41.622 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.622 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.622 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:41.622 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.804328+0000", 2023-10-09T05:52:41.622 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6969e-05 2023-10-09T05:52:41.622 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.622 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.623 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:41.623 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.804334+0000", 2023-10-09T05:52:41.623 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1580000000000002e-06 2023-10-09T05:52:41.623 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.623 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.623 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.623 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.805072+0000", 2023-10-09T05:52:41.623 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00073821900000000003 2023-10-09T05:52:41.623 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.623 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.623 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:41.623 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.805186+0000", 2023-10-09T05:52:41.624 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011408099999999999 2023-10-09T05:52:41.624 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.624 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.624 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:52:41.624 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.805312+0000", 2023-10-09T05:52:41.624 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0001259 2023-10-09T05:52:41.624 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.624 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.624 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:41.624 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.806929+0000", 2023-10-09T05:52:41.624 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0016169839999999999 2023-10-09T05:52:41.625 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.625 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.625 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.625 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.808641+0000", 2023-10-09T05:52:41.625 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0017123469999999999 2023-10-09T05:52:41.625 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.625 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.625 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.625 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:14.133307+0000", 2023-10-09T05:52:41.625 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.32466545299999999 2023-10-09T05:52:41.625 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.625 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.626 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:41.626 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:14.133342+0000", 2023-10-09T05:52:41.626 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5429999999999998e-05 2023-10-09T05:52:41.626 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.626 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.626 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:41.626 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:14.133387+0000", 2023-10-09T05:52:41.626 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5037000000000003e-05 2023-10-09T05:52:41.626 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.626 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:41.626 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.627 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.627 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.627 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64016 6.9 6:964728b5:::benchmark_data_smithi064_284193_object64015:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:52:41.627 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.109619+0000", 2023-10-09T05:52:41.627 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 68.498906008999995, 2023-10-09T05:52:41.627 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0912698940000001, 2023-10-09T05:52:41.627 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:41.627 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:41.627 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:41.627 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:41.627 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:41.627 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64016 2023-10-09T05:52:41.628 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.628 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:41.628 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.628 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:41.628 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.109619+0000", 2023-10-09T05:52:41.628 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.628 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.628 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.628 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:41.628 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.109619+0000", 2023-10-09T05:52:41.628 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.628 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.629 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.629 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:41.629 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.109619+0000", 2023-10-09T05:52:41.629 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:52:41.629 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.629 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.629 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:41.629 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.109650+0000", 2023-10-09T05:52:41.629 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1087999999999998e-05 2023-10-09T05:52:41.629 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.629 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.629 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:41.630 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.109657+0000", 2023-10-09T05:52:41.630 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4290000000000004e-06 2023-10-09T05:52:41.630 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.630 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.630 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:41.630 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.109662+0000", 2023-10-09T05:52:41.630 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4510000000000002e-06 2023-10-09T05:52:41.630 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.630 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.630 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.630 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.115072+0000", 2023-10-09T05:52:41.631 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0054107180000000001 2023-10-09T05:52:41.631 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.631 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.631 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:41.631 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.115080+0000", 2023-10-09T05:52:41.631 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3710000000000002e-06 2023-10-09T05:52:41.631 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.631 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.631 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.631 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.189454+0000", 2023-10-09T05:52:41.631 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.074374384 2023-10-09T05:52:41.631 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.632 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.632 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:41.632 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.190617+0000", 2023-10-09T05:52:41.632 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0011633150000000001 2023-10-09T05:52:41.632 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.632 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.632 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2", 2023-10-09T05:52:41.632 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.190779+0000", 2023-10-09T05:52:41.632 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00016118799999999999 2023-10-09T05:52:41.632 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.632 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.633 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:41.633 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.198531+0000", 2023-10-09T05:52:41.633 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0077525600000000004 2023-10-09T05:52:41.633 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.633 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.633 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.633 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200740+0000", 2023-10-09T05:52:41.633 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002208785 2023-10-09T05:52:41.633 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.633 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.633 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:41.633 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200795+0000", 2023-10-09T05:52:41.634 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5225000000000001e-05 2023-10-09T05:52:41.634 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.634 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.634 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:41.634 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200889+0000", 2023-10-09T05:52:41.634 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3837000000000003e-05 2023-10-09T05:52:41.634 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.634 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:41.634 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.634 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.634 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.635 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64021 6.9 6:9b197d79:::benchmark_data_smithi064_284193_object64020:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:52:41.635 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.123178+0000", 2023-10-09T05:52:41.635 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 68.485347226000002, 2023-10-09T05:52:41.635 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0778894430000001, 2023-10-09T05:52:41.635 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:41.635 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:41.635 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:41.635 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:41.635 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:41.635 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64021 2023-10-09T05:52:41.635 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.635 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:41.635 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.636 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:41.636 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.123178+0000", 2023-10-09T05:52:41.636 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.636 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.636 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.636 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:41.636 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.123178+0000", 2023-10-09T05:52:41.636 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.636 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.636 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.636 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:41.636 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.123177+0000", 2023-10-09T05:52:41.637 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:41.637 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.637 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.637 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:41.637 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.123212+0000", 2023-10-09T05:52:41.637 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5641e-05 2023-10-09T05:52:41.637 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.637 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.637 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:41.637 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.123221+0000", 2023-10-09T05:52:41.637 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0909999999999999e-06 2023-10-09T05:52:41.638 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.638 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.638 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:41.638 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.123227+0000", 2023-10-09T05:52:41.638 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4469999999999997e-06 2023-10-09T05:52:41.638 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.638 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.638 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.638 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.124899+0000", 2023-10-09T05:52:41.638 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001671797 2023-10-09T05:52:41.638 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.638 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.639 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:41.639 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.124905+0000", 2023-10-09T05:52:41.639 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8650000000000003e-06 2023-10-09T05:52:41.639 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.639 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.639 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.639 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.192517+0000", 2023-10-09T05:52:41.639 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0676111340000001 2023-10-09T05:52:41.639 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.639 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.639 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:41.640 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.192607+0000", 2023-10-09T05:52:41.640 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0493000000000002e-05 2023-10-09T05:52:41.640 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.640 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.640 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2", 2023-10-09T05:52:41.640 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.192721+0000", 2023-10-09T05:52:41.640 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000114003 2023-10-09T05:52:41.640 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.640 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.640 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:41.640 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200480+0000", 2023-10-09T05:52:41.641 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0077586089999999996 2023-10-09T05:52:41.641 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.641 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.641 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.641 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200926+0000", 2023-10-09T05:52:41.641 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00044662899999999998 2023-10-09T05:52:41.641 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.641 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.641 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:41.641 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200976+0000", 2023-10-09T05:52:41.641 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9607999999999997e-05 2023-10-09T05:52:41.641 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.641 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.642 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:41.642 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.201067+0000", 2023-10-09T05:52:41.642 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1414000000000003e-05 2023-10-09T05:52:41.642 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.642 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:41.642 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.642 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.642 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.642 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64026 6.9 6:95eb16a1:::benchmark_data_smithi064_284193_object64025:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:52:41.642 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.131052+0000", 2023-10-09T05:52:41.642 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 68.477472763999998, 2023-10-09T05:52:41.643 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0701834699999999, 2023-10-09T05:52:41.643 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:41.643 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:41.643 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:41.643 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:41.643 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:41.643 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64026 2023-10-09T05:52:41.643 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.643 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:41.643 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.643 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:41.643 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.131052+0000", 2023-10-09T05:52:41.644 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.644 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.644 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.644 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:41.644 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.131052+0000", 2023-10-09T05:52:41.644 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.644 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.644 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.644 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:41.644 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.131051+0000", 2023-10-09T05:52:41.644 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:41.644 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.645 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.645 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:41.645 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.131088+0000", 2023-10-09T05:52:41.645 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6916999999999999e-05 2023-10-09T05:52:41.645 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.645 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.645 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:41.645 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.131101+0000", 2023-10-09T05:52:41.645 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2877999999999999e-05 2023-10-09T05:52:41.645 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.645 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.646 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:41.646 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.131107+0000", 2023-10-09T05:52:41.646 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5899999999999996e-06 2023-10-09T05:52:41.646 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.646 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.646 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.646 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.131156+0000", 2023-10-09T05:52:41.646 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8696000000000003e-05 2023-10-09T05:52:41.646 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.646 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.646 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:41.646 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.131164+0000", 2023-10-09T05:52:41.647 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9340000000000006e-06 2023-10-09T05:52:41.647 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.647 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.647 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.647 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.194260+0000", 2023-10-09T05:52:41.647 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.063095543 2023-10-09T05:52:41.647 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.647 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.647 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:41.647 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.194404+0000", 2023-10-09T05:52:41.647 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000144058 2023-10-09T05:52:41.647 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2", 2023-10-09T05:52:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.194588+0000", 2023-10-09T05:52:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00018420399999999999 2023-10-09T05:52:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200488+0000", 2023-10-09T05:52:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0059001640000000003 2023-10-09T05:52:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.201102+0000", 2023-10-09T05:52:41.649 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00061416900000000002 2023-10-09T05:52:41.649 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.649 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.649 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:41.649 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.201150+0000", 2023-10-09T05:52:41.649 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7922e-05 2023-10-09T05:52:41.649 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.649 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.649 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:41.649 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.201236+0000", 2023-10-09T05:52:41.649 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.5679000000000004e-05 2023-10-09T05:52:41.649 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.650 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:41.650 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.650 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.650 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.650 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64027 6.9 6:98f248d6:::benchmark_data_smithi064_284193_object64026:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:52:41.650 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.133112+0000", 2023-10-09T05:52:41.650 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 68.475413458000006, 2023-10-09T05:52:41.650 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0717281329999999, 2023-10-09T05:52:41.650 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:41.650 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:41.650 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:41.650 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:41.651 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:41.651 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64027 2023-10-09T05:52:41.651 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.651 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:41.651 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.651 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:41.651 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133112+0000", 2023-10-09T05:52:41.651 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.651 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.651 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.651 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:41.651 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133112+0000", 2023-10-09T05:52:41.652 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.652 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.652 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.652 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:41.652 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133110+0000", 2023-10-09T05:52:41.652 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:52:41.652 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.652 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.652 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:41.652 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133165+0000", 2023-10-09T05:52:41.652 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4601999999999997e-05 2023-10-09T05:52:41.652 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.653 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.653 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:41.653 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133174+0000", 2023-10-09T05:52:41.653 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6899999999999998e-06 2023-10-09T05:52:41.653 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.653 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.653 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:41.653 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133179+0000", 2023-10-09T05:52:41.653 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9509999999999997e-06 2023-10-09T05:52:41.653 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.653 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.654 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.654 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133222+0000", 2023-10-09T05:52:41.654 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3266000000000002e-05 2023-10-09T05:52:41.654 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.654 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.654 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:41.654 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133229+0000", 2023-10-09T05:52:41.654 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.486e-06 2023-10-09T05:52:41.654 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.654 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.654 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.655 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.196460+0000", 2023-10-09T05:52:41.655 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0632303270000001 2023-10-09T05:52:41.655 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.655 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.655 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:41.655 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.196575+0000", 2023-10-09T05:52:41.655 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000114918 2023-10-09T05:52:41.655 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.655 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.655 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2", 2023-10-09T05:52:41.655 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.196709+0000", 2023-10-09T05:52:41.656 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013413100000000001 2023-10-09T05:52:41.656 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.656 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.656 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.656 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.203829+0000", 2023-10-09T05:52:41.656 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0071203179999999996 2023-10-09T05:52:41.656 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.656 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.656 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:41.656 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204716+0000", 2023-10-09T05:52:41.656 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00088727100000000002 2023-10-09T05:52:41.657 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.657 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.657 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:41.657 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204766+0000", 2023-10-09T05:52:41.657 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9855999999999998e-05 2023-10-09T05:52:41.657 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.657 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.657 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:41.657 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204840+0000", 2023-10-09T05:52:41.657 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3491999999999994e-05 2023-10-09T05:52:41.658 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.658 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:41.658 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.658 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.658 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.658 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64047 6.9 6:98194c7b:::benchmark_data_smithi064_284193_object64046:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:52:41.658 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.155174+0000", 2023-10-09T05:52:41.658 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 68.453351358000006, 2023-10-09T05:52:41.658 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0497767680000001, 2023-10-09T05:52:41.658 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:41.658 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:41.658 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:41.659 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:41.659 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:41.659 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64047 2023-10-09T05:52:41.659 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.659 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:41.659 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.659 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:41.659 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.155174+0000", 2023-10-09T05:52:41.659 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.659 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.659 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.659 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:41.660 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.155174+0000", 2023-10-09T05:52:41.660 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.660 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.660 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.660 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:41.660 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.155172+0000", 2023-10-09T05:52:41.660 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:41.660 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.660 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.660 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:41.660 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.155288+0000", 2023-10-09T05:52:41.661 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011592299999999999 2023-10-09T05:52:41.661 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.661 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.661 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:41.661 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.155304+0000", 2023-10-09T05:52:41.661 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6515999999999999e-05 2023-10-09T05:52:41.661 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.661 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.661 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:41.661 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.155311+0000", 2023-10-09T05:52:41.661 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5470000000000002e-06 2023-10-09T05:52:41.661 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.662 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.662 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.662 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.155364+0000", 2023-10-09T05:52:41.662 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3075000000000003e-05 2023-10-09T05:52:41.662 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.662 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.662 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:52:41.662 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.155375+0000", 2023-10-09T05:52:41.662 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0631e-05 2023-10-09T05:52:41.662 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.662 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.662 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.663 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.198594+0000", 2023-10-09T05:52:41.663 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0432188309999999 2023-10-09T05:52:41.663 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.663 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.663 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:41.663 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.198721+0000", 2023-10-09T05:52:41.663 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000127398 2023-10-09T05:52:41.663 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.663 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.663 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2", 2023-10-09T05:52:41.663 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.198812+0000", 2023-10-09T05:52:41.664 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1410999999999996e-05 2023-10-09T05:52:41.664 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.664 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.664 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.664 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.203873+0000", 2023-10-09T05:52:41.664 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0050606719999999996 2023-10-09T05:52:41.664 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.664 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.664 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:41.664 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204849+0000", 2023-10-09T05:52:41.664 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00097625800000000003 2023-10-09T05:52:41.664 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.664 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.665 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:41.665 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204887+0000", 2023-10-09T05:52:41.665 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7339999999999998e-05 2023-10-09T05:52:41.665 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.665 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.665 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:41.665 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204950+0000", 2023-10-09T05:52:41.665 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3805000000000003e-05 2023-10-09T05:52:41.665 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.665 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:41.665 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.665 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.666 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.666 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:77504 6.5 6:af5f1d5a:::benchmark_data_smithi064_284193_object77503:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:52:41.666 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:49.602465+0000", 2023-10-09T05:52:41.666 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 52.006060245, 2023-10-09T05:52:41.666 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.36206133400000001, 2023-10-09T05:52:41.666 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:41.666 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:41.666 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:41.666 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:41.666 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:41.666 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 77504 2023-10-09T05:52:41.667 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.667 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:41.667 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.667 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:41.667 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602465+0000", 2023-10-09T05:52:41.667 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.667 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.667 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.667 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:41.667 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602465+0000", 2023-10-09T05:52:41.667 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.668 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.668 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.668 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:41.668 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602463+0000", 2023-10-09T05:52:41.668 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:41.668 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.668 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.668 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:41.668 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602503+0000", 2023-10-09T05:52:41.668 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0176000000000003e-05 2023-10-09T05:52:41.668 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.669 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.669 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:41.669 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602519+0000", 2023-10-09T05:52:41.669 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5979999999999999e-05 2023-10-09T05:52:41.669 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.669 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.669 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:41.669 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602534+0000", 2023-10-09T05:52:41.669 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4303e-05 2023-10-09T05:52:41.669 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.669 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.670 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.670 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602595+0000", 2023-10-09T05:52:41.670 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1381000000000001e-05 2023-10-09T05:52:41.670 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.670 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.670 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:41.670 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602714+0000", 2023-10-09T05:52:41.670 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000119098 2023-10-09T05:52:41.670 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.670 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.670 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:52:41.671 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602851+0000", 2023-10-09T05:52:41.671 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000137093 2023-10-09T05:52:41.671 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.671 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.671 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:41.671 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.605415+0000", 2023-10-09T05:52:41.671 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002564055 2023-10-09T05:52:41.671 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.671 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.671 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.671 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.607058+0000", 2023-10-09T05:52:41.671 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0016429680000000001 2023-10-09T05:52:41.672 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.672 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.672 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.672 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964454+0000", 2023-10-09T05:52:41.672 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.35739558999999999 2023-10-09T05:52:41.672 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.672 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.672 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:41.672 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964494+0000", 2023-10-09T05:52:41.672 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0250000000000003e-05 2023-10-09T05:52:41.672 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.673 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.673 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:41.673 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964526+0000", 2023-10-09T05:52:41.673 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1959999999999999e-05 2023-10-09T05:52:41.673 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.673 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:41.673 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.673 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.673 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.673 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:77509 6.5 6:ae2c471a:::benchmark_data_smithi064_284193_object77508:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:52:41.673 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:49.610882+0000", 2023-10-09T05:52:41.673 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 51.997643005999997, 2023-10-09T05:52:41.674 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.353839925, 2023-10-09T05:52:41.674 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:41.674 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:41.674 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:41.674 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:41.674 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:41.674 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 77509 2023-10-09T05:52:41.674 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.674 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:41.674 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.674 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:41.675 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.610882+0000", 2023-10-09T05:52:41.675 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.675 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.675 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.675 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:41.675 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.610882+0000", 2023-10-09T05:52:41.675 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.675 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.675 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.675 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:41.675 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.610881+0000", 2023-10-09T05:52:41.675 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:41.675 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.675 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.676 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:41.676 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.610960+0000", 2023-10-09T05:52:41.676 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9631999999999999e-05 2023-10-09T05:52:41.676 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.676 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.676 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:41.676 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.610979+0000", 2023-10-09T05:52:41.676 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9011000000000001e-05 2023-10-09T05:52:41.676 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.676 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.676 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:41.677 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.610987+0000", 2023-10-09T05:52:41.677 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3289999999999996e-06 2023-10-09T05:52:41.677 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.677 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.677 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.677 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.611048+0000", 2023-10-09T05:52:41.677 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0708999999999998e-05 2023-10-09T05:52:41.677 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.677 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.677 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:41.677 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.611203+0000", 2023-10-09T05:52:41.677 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015526900000000001 2023-10-09T05:52:41.678 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.678 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.678 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:52:41.678 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.611355+0000", 2023-10-09T05:52:41.678 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015201200000000001 2023-10-09T05:52:41.678 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.678 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.678 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:41.678 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.613876+0000", 2023-10-09T05:52:41.678 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0025209579999999998 2023-10-09T05:52:41.678 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.679 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.679 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.679 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.615859+0000", 2023-10-09T05:52:41.679 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001983744 2023-10-09T05:52:41.679 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.679 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.679 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.679 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964635+0000", 2023-10-09T05:52:41.679 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.34877526800000003 2023-10-09T05:52:41.679 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.679 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.680 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:41.680 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964685+0000", 2023-10-09T05:52:41.680 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0586e-05 2023-10-09T05:52:41.680 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.680 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.680 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:41.680 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964722+0000", 2023-10-09T05:52:41.680 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6636e-05 2023-10-09T05:52:41.680 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.680 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:41.680 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.680 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.681 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.681 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:77517 6.5 6:a52e5624:::benchmark_data_smithi064_284193_object77516:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:52:41.681 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:49.620459+0000", 2023-10-09T05:52:41.681 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 51.988066207999999, 2023-10-09T05:52:41.681 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.34437174100000001, 2023-10-09T05:52:41.681 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:41.681 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:41.681 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:41.681 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:41.681 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:41.681 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 77517 2023-10-09T05:52:41.682 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.682 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:41.682 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.682 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:41.682 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620459+0000", 2023-10-09T05:52:41.682 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.682 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.682 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.682 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:41.682 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620459+0000", 2023-10-09T05:52:41.682 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.682 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.683 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.683 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:41.683 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620458+0000", 2023-10-09T05:52:41.683 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:52:41.683 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.683 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.683 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:41.683 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620525+0000", 2023-10-09T05:52:41.683 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7404999999999996e-05 2023-10-09T05:52:41.683 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.683 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.683 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:41.683 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620539+0000", 2023-10-09T05:52:41.684 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3997e-05 2023-10-09T05:52:41.684 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.684 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.684 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:41.684 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620545+0000", 2023-10-09T05:52:41.684 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7189999999999998e-06 2023-10-09T05:52:41.684 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.684 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.684 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.684 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620612+0000", 2023-10-09T05:52:41.684 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7325999999999999e-05 2023-10-09T05:52:41.684 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.685 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.685 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:41.685 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620750+0000", 2023-10-09T05:52:41.685 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000137059 2023-10-09T05:52:41.685 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.685 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.685 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:52:41.685 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620901+0000", 2023-10-09T05:52:41.685 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015133200000000001 2023-10-09T05:52:41.685 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.685 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.685 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:41.686 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.623123+0000", 2023-10-09T05:52:41.686 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0022225970000000002 2023-10-09T05:52:41.686 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.686 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.686 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.686 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.623990+0000", 2023-10-09T05:52:41.686 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00086624800000000002 2023-10-09T05:52:41.686 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.686 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.686 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.686 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964757+0000", 2023-10-09T05:52:41.687 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.34076741799999999 2023-10-09T05:52:41.687 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.687 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.687 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:41.687 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964800+0000", 2023-10-09T05:52:41.687 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2741000000000003e-05 2023-10-09T05:52:41.687 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.687 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.687 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:41.687 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964831+0000", 2023-10-09T05:52:41.687 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0738999999999998e-05 2023-10-09T05:52:41.687 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.688 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:41.688 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.688 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.688 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.688 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:77534 6.3 6:c39d213f:::benchmark_data_smithi064_284193_object77533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:52:41.688 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:49.644389+0000", 2023-10-09T05:52:41.688 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 51.964136377999999, 2023-10-09T05:52:41.688 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.33628388399999998, 2023-10-09T05:52:41.688 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:41.688 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:41.688 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:41.689 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:41.689 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:41.689 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 77534 2023-10-09T05:52:41.689 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.689 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:41.689 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.689 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:41.689 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644389+0000", 2023-10-09T05:52:41.689 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.689 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.689 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.689 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:41.690 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644389+0000", 2023-10-09T05:52:41.690 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.690 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.690 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.690 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:41.690 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644388+0000", 2023-10-09T05:52:41.690 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:52:41.690 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.690 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.690 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:41.690 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644424+0000", 2023-10-09T05:52:41.690 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5824999999999997e-05 2023-10-09T05:52:41.691 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.691 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.691 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:41.691 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644434+0000", 2023-10-09T05:52:41.691 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0461e-05 2023-10-09T05:52:41.691 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.691 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.691 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:41.691 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644440+0000", 2023-10-09T05:52:41.691 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1100000000000002e-06 2023-10-09T05:52:41.691 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.691 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.691 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.692 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644481+0000", 2023-10-09T05:52:41.692 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1902e-05 2023-10-09T05:52:41.692 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.692 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.692 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:41.692 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644613+0000", 2023-10-09T05:52:41.692 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000131815 2023-10-09T05:52:41.692 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.692 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.692 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:52:41.693 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644762+0000", 2023-10-09T05:52:41.693 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000148777 2023-10-09T05:52:41.693 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.693 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.693 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:41.693 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.649170+0000", 2023-10-09T05:52:41.693 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0044076310000000004 2023-10-09T05:52:41.693 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.693 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.693 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.693 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.650165+0000", 2023-10-09T05:52:41.693 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00099568399999999994 2023-10-09T05:52:41.694 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.694 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.694 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.694 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.980566+0000", 2023-10-09T05:52:41.694 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.33040050599999998 2023-10-09T05:52:41.694 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.694 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.694 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:41.694 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.980621+0000", 2023-10-09T05:52:41.694 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5392000000000001e-05 2023-10-09T05:52:41.694 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.694 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.695 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:41.695 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.980673+0000", 2023-10-09T05:52:41.695 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1316999999999997e-05 2023-10-09T05:52:41.695 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.695 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:41.695 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.695 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.695 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.695 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:77535 6.3 6:c58d58f2:::benchmark_data_smithi064_284193_object77534:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:52:41.695 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:49.644455+0000", 2023-10-09T05:52:41.695 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 51.964069713999997, 2023-10-09T05:52:41.696 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.33631617699999999, 2023-10-09T05:52:41.696 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:41.696 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:41.696 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:41.696 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:41.696 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:41.696 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 77535 2023-10-09T05:52:41.696 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.696 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:41.696 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.696 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:41.696 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644455+0000", 2023-10-09T05:52:41.697 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.697 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.697 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.697 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:41.697 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644455+0000", 2023-10-09T05:52:41.697 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.697 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.697 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.697 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:41.697 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644455+0000", 2023-10-09T05:52:41.697 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:52:41.697 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.698 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.698 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:41.698 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644491+0000", 2023-10-09T05:52:41.698 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6570999999999999e-05 2023-10-09T05:52:41.698 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.698 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.698 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:41.698 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644507+0000", 2023-10-09T05:52:41.698 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5880999999999999e-05 2023-10-09T05:52:41.698 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.698 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.698 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:41.699 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644512+0000", 2023-10-09T05:52:41.699 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0880000000000002e-06 2023-10-09T05:52:41.699 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.699 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.699 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.699 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.647155+0000", 2023-10-09T05:52:41.699 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0026430009999999999 2023-10-09T05:52:41.699 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.699 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.699 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:41.699 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.647283+0000", 2023-10-09T05:52:41.699 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012812300000000001 2023-10-09T05:52:41.699 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.700 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.700 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:52:41.700 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.647407+0000", 2023-10-09T05:52:41.700 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012368899999999999 2023-10-09T05:52:41.700 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.700 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.700 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:41.700 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.649330+0000", 2023-10-09T05:52:41.700 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0019226779999999999 2023-10-09T05:52:41.700 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.700 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.701 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.701 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.650266+0000", 2023-10-09T05:52:41.701 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00093634999999999996 2023-10-09T05:52:41.701 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.701 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.701 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.701 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.980707+0000", 2023-10-09T05:52:41.701 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.33044109100000002 2023-10-09T05:52:41.701 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.701 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.701 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:41.701 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.980743+0000", 2023-10-09T05:52:41.701 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5800000000000003e-05 2023-10-09T05:52:41.702 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.702 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.702 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:41.702 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.980772+0000", 2023-10-09T05:52:41.702 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.8580000000000001e-05 2023-10-09T05:52:41.702 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.702 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:41.702 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.702 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.702 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.702 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:79589 6.5 6:a0fa1d08:::benchmark_data_smithi064_284193_object79588:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:52:41.703 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:52.363694+0000", 2023-10-09T05:52:41.703 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 49.244831507000001, 2023-10-09T05:52:41.703 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.26210419000000001, 2023-10-09T05:52:41.703 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:41.703 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:41.703 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:41.703 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:41.703 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:41.703 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 79589 2023-10-09T05:52:41.703 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.703 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:41.703 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.704 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:41.704 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.363694+0000", 2023-10-09T05:52:41.704 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.704 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.704 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.704 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:41.704 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.363694+0000", 2023-10-09T05:52:41.704 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.704 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.704 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.704 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:41.704 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.363693+0000", 2023-10-09T05:52:41.705 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:52:41.705 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.705 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.705 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:41.705 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.363741+0000", 2023-10-09T05:52:41.705 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8436999999999997e-05 2023-10-09T05:52:41.705 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.705 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.705 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:41.705 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.363750+0000", 2023-10-09T05:52:41.706 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1820000000000006e-06 2023-10-09T05:52:41.706 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.706 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.706 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:41.706 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.363754+0000", 2023-10-09T05:52:41.706 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6120000000000001e-06 2023-10-09T05:52:41.706 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.706 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.706 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.706 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.366906+0000", 2023-10-09T05:52:41.706 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0031518340000000001 2023-10-09T05:52:41.706 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.707 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.707 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:41.707 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.367025+0000", 2023-10-09T05:52:41.707 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000118854 2023-10-09T05:52:41.707 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.707 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.707 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:52:41.707 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.367157+0000", 2023-10-09T05:52:41.707 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013233 2023-10-09T05:52:41.707 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.707 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.708 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:41.708 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.372713+0000", 2023-10-09T05:52:41.708 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0055557940000000002 2023-10-09T05:52:41.708 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.708 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.708 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.708 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.376095+0000", 2023-10-09T05:52:41.708 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0033821039999999999 2023-10-09T05:52:41.708 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.708 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.708 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.709 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.625686+0000", 2023-10-09T05:52:41.709 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24959123 2023-10-09T05:52:41.709 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.709 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.709 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:41.709 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.625752+0000", 2023-10-09T05:52:41.709 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6228000000000004e-05 2023-10-09T05:52:41.709 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.709 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.709 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:41.709 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.625798+0000", 2023-10-09T05:52:41.710 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5246e-05 2023-10-09T05:52:41.710 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.710 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:41.710 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.710 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.710 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.710 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:79591 6.5 6:a2ce5c7b:::benchmark_data_smithi064_284193_object79590:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:52:41.710 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:52.366733+0000", 2023-10-09T05:52:41.710 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 49.241792529000001, 2023-10-09T05:52:41.710 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.25915231599999999, 2023-10-09T05:52:41.710 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:41.711 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:41.711 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:41.711 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:41.711 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:41.711 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 79591 2023-10-09T05:52:41.711 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.711 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:41.711 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.711 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:41.711 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.366733+0000", 2023-10-09T05:52:41.711 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.712 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.712 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.712 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:41.712 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.366733+0000", 2023-10-09T05:52:41.712 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.712 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.712 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.712 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:41.712 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.366731+0000", 2023-10-09T05:52:41.712 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:52:41.712 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.713 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.713 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:41.713 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.366762+0000", 2023-10-09T05:52:41.713 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0255000000000001e-05 2023-10-09T05:52:41.713 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.713 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.713 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:41.713 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.366770+0000", 2023-10-09T05:52:41.713 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6680000000000007e-06 2023-10-09T05:52:41.713 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.713 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.714 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:41.714 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.366775+0000", 2023-10-09T05:52:41.714 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4710000000000001e-06 2023-10-09T05:52:41.714 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.714 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.714 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.714 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.369642+0000", 2023-10-09T05:52:41.714 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0028674389999999998 2023-10-09T05:52:41.714 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.714 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.714 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:41.715 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.369755+0000", 2023-10-09T05:52:41.715 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011286900000000001 2023-10-09T05:52:41.715 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.715 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.715 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:52:41.715 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.369850+0000", 2023-10-09T05:52:41.715 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.4812e-05 2023-10-09T05:52:41.715 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.715 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.715 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:41.715 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.375836+0000", 2023-10-09T05:52:41.716 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0059865750000000001 2023-10-09T05:52:41.716 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.716 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.716 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.716 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.377143+0000", 2023-10-09T05:52:41.716 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001306967 2023-10-09T05:52:41.716 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.716 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.716 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.716 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.625835+0000", 2023-10-09T05:52:41.716 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24869169799999999 2023-10-09T05:52:41.716 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.716 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.717 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:41.717 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.625865+0000", 2023-10-09T05:52:41.717 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9937000000000002e-05 2023-10-09T05:52:41.717 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.717 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.717 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:41.717 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.625885+0000", 2023-10-09T05:52:41.717 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9755e-05 2023-10-09T05:52:41.717 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.717 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:41.718 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.718 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.718 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.718 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:79595 6.8 6:1929f855:::benchmark_data_smithi064_284193_object79594:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:52:41.718 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:52.372938+0000", 2023-10-09T05:52:41.718 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 49.235587135000003, 2023-10-09T05:52:41.718 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.25287009199999999, 2023-10-09T05:52:41.718 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:41.718 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:41.718 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:41.718 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:41.718 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:41.719 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 79595 2023-10-09T05:52:41.719 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.719 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:41.719 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.719 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:41.719 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.372938+0000", 2023-10-09T05:52:41.719 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.719 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.719 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.719 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:41.719 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.372938+0000", 2023-10-09T05:52:41.720 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.720 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.720 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.720 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:41.720 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.372937+0000", 2023-10-09T05:52:41.720 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:52:41.720 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.720 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.720 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:41.720 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.372969+0000", 2023-10-09T05:52:41.720 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.2190999999999997e-05 2023-10-09T05:52:41.720 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.721 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.721 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:41.721 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.372976+0000", 2023-10-09T05:52:41.721 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6680000000000004e-06 2023-10-09T05:52:41.721 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.721 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.721 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:41.721 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.372979+0000", 2023-10-09T05:52:41.721 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7369999999999998e-06 2023-10-09T05:52:41.721 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.721 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.722 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.722 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.373034+0000", 2023-10-09T05:52:41.722 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5637999999999998e-05 2023-10-09T05:52:41.722 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.722 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.722 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:41.722 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.373151+0000", 2023-10-09T05:52:41.722 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000116282 2023-10-09T05:52:41.722 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.722 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.722 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:52:41.723 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.373279+0000", 2023-10-09T05:52:41.723 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012823000000000001 2023-10-09T05:52:41.723 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.723 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.723 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:41.723 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.376056+0000", 2023-10-09T05:52:41.723 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0027769019999999999 2023-10-09T05:52:41.723 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.723 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.723 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.723 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.377172+0000", 2023-10-09T05:52:41.724 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0011158730000000001 2023-10-09T05:52:41.724 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.724 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.724 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.724 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.625736+0000", 2023-10-09T05:52:41.724 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24856469 2023-10-09T05:52:41.724 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.724 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.724 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:41.724 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.625772+0000", 2023-10-09T05:52:41.724 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5989999999999999e-05 2023-10-09T05:52:41.725 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.725 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.725 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:41.725 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.625808+0000", 2023-10-09T05:52:41.725 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5621999999999999e-05 2023-10-09T05:52:41.725 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.725 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:41.725 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.725 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.725 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.726 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.44588.0:81811 6.1 e128/118 6:880e97e1:::benchmark_data_smithi064_284193_object81810:head v 128'5158, mlcod=0'0)", 2023-10-09T05:52:41.726 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:55.436474+0000", 2023-10-09T05:52:41.726 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 46.172051308, 2023-10-09T05:52:41.726 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24507322600000001, 2023-10-09T05:52:41.726 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:41.726 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:41.726 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:41.726 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.726 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:41.726 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.436474+0000", 2023-10-09T05:52:41.726 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.727 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.727 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.727 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:41.727 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.436474+0000", 2023-10-09T05:52:41.727 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.727 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.727 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.727 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:41.727 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.436473+0000", 2023-10-09T05:52:41.727 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:52:41.727 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.727 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.728 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:41.728 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.436497+0000", 2023-10-09T05:52:41.728 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3646e-05 2023-10-09T05:52:41.728 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.728 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.728 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:41.728 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.436507+0000", 2023-10-09T05:52:41.728 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0539e-05 2023-10-09T05:52:41.728 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.728 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.728 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:41.729 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.436513+0000", 2023-10-09T05:52:41.729 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3689999999999998e-06 2023-10-09T05:52:41.729 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.729 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.729 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.729 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.440741+0000", 2023-10-09T05:52:41.729 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0042279179999999998 2023-10-09T05:52:41.729 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.729 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.729 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:41.730 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.440763+0000", 2023-10-09T05:52:41.730 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2685e-05 2023-10-09T05:52:41.730 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.730 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.730 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:41.730 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.681509+0000", 2023-10-09T05:52:41.730 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24074568599999999 2023-10-09T05:52:41.730 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.730 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.730 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:41.730 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.681547+0000", 2023-10-09T05:52:41.730 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7879000000000001e-05 2023-10-09T05:52:41.731 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.731 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:41.731 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.731 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.731 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.731 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:81814 6.9 6:90e965d6:::benchmark_data_smithi064_284193_object81813:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T05:52:41.731 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:55.439384+0000", 2023-10-09T05:52:41.731 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 46.169140871000003, 2023-10-09T05:52:41.731 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24229829899999999, 2023-10-09T05:52:41.731 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:41.731 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:41.732 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:41.732 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:41.732 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:41.732 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 81814 2023-10-09T05:52:41.732 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.732 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:41.732 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.732 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:41.732 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.439384+0000", 2023-10-09T05:52:41.732 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.732 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.732 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.733 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:41.733 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.439384+0000", 2023-10-09T05:52:41.733 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.733 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.733 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.733 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:41.733 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.439383+0000", 2023-10-09T05:52:41.733 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:52:41.733 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.733 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.733 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:41.733 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.439436+0000", 2023-10-09T05:52:41.733 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3418999999999998e-05 2023-10-09T05:52:41.734 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.734 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.734 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:41.734 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.439448+0000", 2023-10-09T05:52:41.734 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1861999999999999e-05 2023-10-09T05:52:41.734 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.734 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.734 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:41.734 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.439454+0000", 2023-10-09T05:52:41.734 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9869999999999996e-06 2023-10-09T05:52:41.734 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.734 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.735 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.735 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.440728+0000", 2023-10-09T05:52:41.735 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001273864 2023-10-09T05:52:41.735 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.735 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.735 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:41.735 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.440849+0000", 2023-10-09T05:52:41.735 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000120986 2023-10-09T05:52:41.735 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.735 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.735 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2", 2023-10-09T05:52:41.736 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.440983+0000", 2023-10-09T05:52:41.736 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013372199999999999 2023-10-09T05:52:41.736 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.736 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.736 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:41.736 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.449915+0000", 2023-10-09T05:52:41.736 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0089326160000000009 2023-10-09T05:52:41.736 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.736 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.736 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.736 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.681544+0000", 2023-10-09T05:52:41.736 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.231628475 2023-10-09T05:52:41.736 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.737 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.737 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:41.737 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.681595+0000", 2023-10-09T05:52:41.737 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1721999999999997e-05 2023-10-09T05:52:41.737 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.737 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.737 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:41.737 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.681682+0000", 2023-10-09T05:52:41.737 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.7125000000000004e-05 2023-10-09T05:52:41.737 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.737 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:41.737 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.738 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.738 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.738 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.44588.0:81818 6.1 e128/118 6:853e1ef7:::benchmark_data_smithi064_284193_object81817:head v 128'5159, mlcod=0'0)", 2023-10-09T05:52:41.738 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:55.443887+0000", 2023-10-09T05:52:41.738 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 46.164637741999996, 2023-10-09T05:52:41.738 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24230628000000001, 2023-10-09T05:52:41.738 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:41.738 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:41.738 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:41.738 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.738 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:41.739 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.443887+0000", 2023-10-09T05:52:41.739 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.739 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.739 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.739 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:41.739 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.443887+0000", 2023-10-09T05:52:41.739 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.739 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.739 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.739 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:41.739 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.443887+0000", 2023-10-09T05:52:41.739 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:52:41.740 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.740 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.740 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:41.740 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.443910+0000", 2023-10-09T05:52:41.740 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3011e-05 2023-10-09T05:52:41.740 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.740 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.740 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:41.740 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.443920+0000", 2023-10-09T05:52:41.740 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0465999999999999e-05 2023-10-09T05:52:41.740 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.740 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.740 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:41.740 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.443926+0000", 2023-10-09T05:52:41.741 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7429999999999999e-06 2023-10-09T05:52:41.741 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.741 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.741 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.741 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.446417+0000", 2023-10-09T05:52:41.741 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0024909649999999999 2023-10-09T05:52:41.741 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.741 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.741 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:41.741 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.446436+0000", 2023-10-09T05:52:41.741 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8998999999999999e-05 2023-10-09T05:52:41.741 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.742 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.742 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:41.742 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.686156+0000", 2023-10-09T05:52:41.742 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.23971962699999999 2023-10-09T05:52:41.742 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.742 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.742 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:41.742 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.686194+0000", 2023-10-09T05:52:41.742 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8049000000000001e-05 2023-10-09T05:52:41.742 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.742 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:41.742 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.743 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.743 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.743 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.44588.0:81822 6.b e128/118 6:d664fd33:::benchmark_data_smithi064_284193_object81821:head v 128'5197, mlcod=107'1276)", 2023-10-09T05:52:41.743 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:55.451714+0000", 2023-10-09T05:52:41.743 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 46.156811050000002, 2023-10-09T05:52:41.743 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.239798436, 2023-10-09T05:52:41.743 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:41.743 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:41.743 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:41.743 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.743 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:41.743 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.451714+0000", 2023-10-09T05:52:41.744 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.744 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.744 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.744 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:41.744 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.451714+0000", 2023-10-09T05:52:41.744 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.744 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.744 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.744 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:41.744 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.451713+0000", 2023-10-09T05:52:41.744 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:52:41.744 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.745 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.745 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:41.745 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.451740+0000", 2023-10-09T05:52:41.745 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.6672000000000001e-05 2023-10-09T05:52:41.745 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.745 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.745 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:41.745 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.678418+0000", 2023-10-09T05:52:41.745 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.226678458 2023-10-09T05:52:41.745 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.745 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.745 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:41.746 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.678430+0000", 2023-10-09T05:52:41.746 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.136e-05 2023-10-09T05:52:41.746 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.746 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.746 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.746 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.682221+0000", 2023-10-09T05:52:41.746 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0037916410000000001 2023-10-09T05:52:41.746 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.746 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.746 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:41.746 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.682249+0000", 2023-10-09T05:52:41.746 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7319999999999999e-05 2023-10-09T05:52:41.747 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.747 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.747 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:41.747 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.691482+0000", 2023-10-09T05:52:41.747 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0092338920000000005 2023-10-09T05:52:41.747 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.747 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.747 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:41.747 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.691512+0000", 2023-10-09T05:52:41.747 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9944999999999999e-05 2023-10-09T05:52:41.747 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.747 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:41.748 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.748 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.748 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.748 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:81827 6.9 6:9293b997:::benchmark_data_smithi064_284193_object81826:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T05:52:41.748 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:55.452859+0000", 2023-10-09T05:52:41.748 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 46.155665663999997, 2023-10-09T05:52:41.748 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24494559699999999, 2023-10-09T05:52:41.748 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:52:41.748 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:52:41.748 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:52:41.748 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:52:41.748 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:52:41.749 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 81827 2023-10-09T05:52:41.749 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.749 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:52:41.749 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.749 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:52:41.749 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.452859+0000", 2023-10-09T05:52:41.749 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.749 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.749 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.749 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:52:41.749 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.452859+0000", 2023-10-09T05:52:41.749 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:52:41.750 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.750 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.750 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:52:41.750 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.452858+0000", 2023-10-09T05:52:41.750 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:52:41.750 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.750 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.750 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:52:41.750 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.452889+0000", 2023-10-09T05:52:41.750 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1037000000000001e-05 2023-10-09T05:52:41.750 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.751 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.751 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:52:41.751 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.680046+0000", 2023-10-09T05:52:41.751 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.227156793 2023-10-09T05:52:41.751 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.751 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.751 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:52:41.751 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.680058+0000", 2023-10-09T05:52:41.751 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.182e-05 2023-10-09T05:52:41.751 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.751 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.751 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:52:41.752 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.686253+0000", 2023-10-09T05:52:41.752 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0061949459999999998 2023-10-09T05:52:41.752 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.752 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.752 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:52:41.752 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.686375+0000", 2023-10-09T05:52:41.752 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012215199999999999 2023-10-09T05:52:41.752 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.752 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.752 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2", 2023-10-09T05:52:41.752 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.686521+0000", 2023-10-09T05:52:41.752 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000146203 2023-10-09T05:52:41.752 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.753 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.753 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:52:41.753 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.697632+0000", 2023-10-09T05:52:41.753 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011110372 2023-10-09T05:52:41.753 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.753 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.753 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:52:41.753 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.697673+0000", 2023-10-09T05:52:41.753 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1384999999999998e-05 2023-10-09T05:52:41.753 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.753 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.753 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:52:41.754 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.697715+0000", 2023-10-09T05:52:41.754 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1755000000000003e-05 2023-10-09T05:52:41.754 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:52:41.754 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:52:41.754 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:52:41.754 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.697805+0000", 2023-10-09T05:52:41.754 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0086000000000005e-05 2023-10-09T05:52:41.754 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.754 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:41.754 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.754 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:52:41.754 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:52:41.755 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:52:42.147 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:42.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:42.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:42.229 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:52:42.252 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:42.293 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:42.361 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:42.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:42.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:42.465 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:42.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:42 smithi064 ceph-mon[220044]: pgmap v331: 99 pgs: 1 peering, 87 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 27148/283113 objects degraded (9.589%); 9699/283113 objects misplaced (3.426%); 9.1 MiB/s, 146 objects/s recovering 2023-10-09T05:52:42.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:42 smithi064 ceph-mon[220044]: from='client.44750 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:42.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:42 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 27148/283113 objects degraded (9.589%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:42.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:42 smithi064 ceph-mon[212750]: pgmap v331: 99 pgs: 1 peering, 87 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 27148/283113 objects degraded (9.589%); 9699/283113 objects misplaced (3.426%); 9.1 MiB/s, 146 objects/s recovering 2023-10-09T05:52:42.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:42 smithi064 ceph-mon[212750]: from='client.44750 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:42.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:42 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 27148/283113 objects degraded (9.589%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:42.494 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:42 smithi156 ceph-mon[195101]: pgmap v331: 99 pgs: 1 peering, 87 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 27148/283113 objects degraded (9.589%); 9699/283113 objects misplaced (3.426%); 9.1 MiB/s, 146 objects/s recovering 2023-10-09T05:52:42.494 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:42 smithi156 ceph-mon[195101]: from='client.44750 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:42.495 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:42 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 27148/283113 objects degraded (9.589%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:42.498 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:42.550 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:42.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:42.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:42.656 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:42.687 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:42.748 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:42.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:42.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:42.852 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:42.889 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:42.947 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:42.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:42.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:43.051 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:43.082 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:43.132 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:43.134 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:43 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1997797185' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:52:43.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:43.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:43.160 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T05:52:43.236 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:43.265 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:43.318 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:43.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:43.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:43.422 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:43.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:43 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1997797185' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:52:43.458 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:43.510 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:43.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:43.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:43.617 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:43.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:43 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1997797185' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:52:43.647 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:43.937 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:43.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:43.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:44.042 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:44.072 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:44.123 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:44.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:44.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:44.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:44 smithi064 ceph-mon[212750]: pgmap v332: 99 pgs: 1 activating+undersized+degraded+remapped, 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 33220/283089 objects degraded (11.735%); 9699/283089 objects misplaced (3.426%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T05:52:44.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:44 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:52:44.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:44 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1997797185' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:52:44.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:44 smithi064 ceph-mon[212750]: osdmap e148: 8 total, 8 up, 8 in 2023-10-09T05:52:44.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:44 smithi064 ceph-mon[212750]: failed to encode map e148 with expected crc 2023-10-09T05:52:44.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:44 smithi064 ceph-mon[212750]: failed to encode map e148 with expected crc 2023-10-09T05:52:44.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:44 smithi064 ceph-mon[212750]: failed to encode map e148 with expected crc 2023-10-09T05:52:44.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:44 smithi064 ceph-mon[212750]: failed to encode map e148 with expected crc 2023-10-09T05:52:44.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:44.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:52:44.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:44 smithi064 ceph-mon[220044]: pgmap v332: 99 pgs: 1 activating+undersized+degraded+remapped, 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 33220/283089 objects degraded (11.735%); 9699/283089 objects misplaced (3.426%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T05:52:44.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:44 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:52:44.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:44 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1997797185' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:52:44.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:44 smithi064 ceph-mon[220044]: osdmap e148: 8 total, 8 up, 8 in 2023-10-09T05:52:44.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:44 smithi064 ceph-mon[220044]: failed to encode map e148 with expected crc 2023-10-09T05:52:44.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:44 smithi064 ceph-mon[220044]: failed to encode map e148 with expected crc 2023-10-09T05:52:44.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:44 smithi064 ceph-mon[220044]: failed to encode map e148 with expected crc 2023-10-09T05:52:44.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:44 smithi064 ceph-mon[220044]: failed to encode map e148 with expected crc 2023-10-09T05:52:44.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:44.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:52:44.226 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:44.256 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:44.309 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:44.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:44.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:44.413 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:44.440 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:44 smithi156 ceph-mon[195101]: pgmap v332: 99 pgs: 1 activating+undersized+degraded+remapped, 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 33220/283089 objects degraded (11.735%); 9699/283089 objects misplaced (3.426%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T05:52:44.440 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:44 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:52:44.440 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:44 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1997797185' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:52:44.440 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:44 smithi156 ceph-mon[195101]: osdmap e148: 8 total, 8 up, 8 in 2023-10-09T05:52:44.440 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:44 smithi156 ceph-mon[195101]: failed to encode map e148 with expected crc 2023-10-09T05:52:44.440 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:44 smithi156 ceph-mon[195101]: failed to encode map e148 with expected crc 2023-10-09T05:52:44.440 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:44 smithi156 ceph-mon[195101]: failed to encode map e148 with expected crc 2023-10-09T05:52:44.440 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:44 smithi156 ceph-mon[195101]: failed to encode map e148 with expected crc 2023-10-09T05:52:44.440 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:44.441 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:52:44.441 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:44.499 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:44.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:44.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:44.509 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:52:44.609 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:44.642 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:44.696 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:44.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:44.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:44.800 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:44.827 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:44.875 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:44.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:44.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:44.909 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:44.982 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:45.011 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:45.061 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:45.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:45.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:45.165 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:45.192 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:45.243 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:45.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:45.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:45.304 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:52:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:52:45] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.43.0" 2023-10-09T05:52:45.348 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:45.379 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:45.538 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:45.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:45.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:45.643 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:45.674 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:45.723 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:45.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:45.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:45.827 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:45.856 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:45.998 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:52:46.044 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:46.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:46.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:46.148 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:46.186 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:46.303 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:46.305 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:46 smithi064 ceph-mon[212750]: pgmap v334: 99 pgs: 1 activating+undersized+degraded+remapped, 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 3 op/s; 33220/283032 objects degraded (11.737%); 8916/283032 objects misplaced (3.150%); 11 MiB/s, 181 objects/s recovering 2023-10-09T05:52:46.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:46 smithi064 ceph-mon[220044]: pgmap v334: 99 pgs: 1 activating+undersized+degraded+remapped, 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 3 op/s; 33220/283032 objects degraded (11.737%); 8916/283032 objects misplaced (3.150%); 11 MiB/s, 181 objects/s recovering 2023-10-09T05:52:46.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:46.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:46.407 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:46.431 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:46 smithi156 ceph-mon[195101]: pgmap v334: 99 pgs: 1 activating+undersized+degraded+remapped, 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 3 op/s; 33220/283032 objects degraded (11.737%); 8916/283032 objects misplaced (3.150%); 11 MiB/s, 181 objects/s recovering 2023-10-09T05:52:46.434 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:46.483 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:46.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:46.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:46.592 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:46.619 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:46.649 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:52:46.649 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:52:46.659 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:46.672 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:46.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:46.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:46.777 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:46.815 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:46.865 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:46.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:46.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:46.969 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:46.998 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:47.050 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:47.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:47.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:47.154 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:47.182 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:47.232 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:47.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:47.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:47.340 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:47.345 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T05:52:47.345 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:52:47.345 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-10-09T05:52:47.346 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:52:47.367 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:47 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 33220/283032 objects degraded (11.737%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:47.368 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:47 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 33220/283032 objects degraded (11.737%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:47.371 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:47.532 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:52:47.534 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:47.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:47.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:47.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:47 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 33220/283032 objects degraded (11.737%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:47.638 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:47.667 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:47.714 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:47.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:47.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:47.822 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:47.852 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:48.127 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:48.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:48.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:48.231 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:48.271 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:48.341 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:48.430 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:48.432 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:48 smithi064 ceph-mon[212750]: pgmap v335: 99 pgs: 1 activating+undersized+degraded+remapped, 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 2 op/s; 33220/283029 objects degraded (11.737%); 8916/283029 objects misplaced (3.150%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T05:52:48.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:48.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:48.434 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:48 smithi064 ceph-mon[220044]: pgmap v335: 99 pgs: 1 activating+undersized+degraded+remapped, 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 2 op/s; 33220/283029 objects degraded (11.737%); 8916/283029 objects misplaced (3.150%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T05:52:48.535 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:48.559 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:48 smithi156 ceph-mon[195101]: pgmap v335: 99 pgs: 1 activating+undersized+degraded+remapped, 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 2 op/s; 33220/283029 objects degraded (11.737%); 8916/283029 objects misplaced (3.150%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T05:52:48.564 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:48.613 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:48.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:48.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:48.719 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:48.738 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:48.747 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:48.794 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:48.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:48.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:48.900 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:48.928 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:48.979 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:48.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:48.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:49.084 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:49.126 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:49.163 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T05:52:49.247 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:49.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:49.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:49.351 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:49.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:49 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3329689365' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:52:49.378 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:49.426 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:49.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:49.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:49.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:49 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3329689365' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:52:49.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:49 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3329689365' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:52:49.539 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:49.571 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:49.987 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:49.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:49.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:50.090 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:50.123 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:50.392 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:50 smithi064 ceph-mon[212750]: pgmap v336: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 3 op/s; 33220/283017 objects degraded (11.738%); 8916/283017 objects misplaced (3.150%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T05:52:50.392 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:50 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:52:50.392 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:50 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3329689365' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:52:50.393 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:50 smithi064 ceph-mon[212750]: osdmap e149: 8 total, 8 up, 8 in 2023-10-09T05:52:50.393 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:50 smithi064 ceph-mon[212750]: failed to encode map e149 with expected crc 2023-10-09T05:52:50.393 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:50 smithi064 ceph-mon[212750]: failed to encode map e149 with expected crc 2023-10-09T05:52:50.393 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:50 smithi064 ceph-mon[212750]: failed to encode map e149 with expected crc 2023-10-09T05:52:50.393 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:50 smithi064 ceph-mon[212750]: failed to encode map e149 with expected crc 2023-10-09T05:52:50.393 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:50 smithi064 ceph-mon[220044]: pgmap v336: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 3 op/s; 33220/283017 objects degraded (11.738%); 8916/283017 objects misplaced (3.150%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T05:52:50.393 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:50 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:52:50.393 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:50 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3329689365' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:52:50.393 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:50 smithi064 ceph-mon[220044]: osdmap e149: 8 total, 8 up, 8 in 2023-10-09T05:52:50.394 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:50 smithi064 ceph-mon[220044]: failed to encode map e149 with expected crc 2023-10-09T05:52:50.394 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:50 smithi064 ceph-mon[220044]: failed to encode map e149 with expected crc 2023-10-09T05:52:50.394 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:50 smithi064 ceph-mon[220044]: failed to encode map e149 with expected crc 2023-10-09T05:52:50.394 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:50 smithi064 ceph-mon[220044]: failed to encode map e149 with expected crc 2023-10-09T05:52:50.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:50 smithi156 ceph-mon[195101]: pgmap v336: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 3 op/s; 33220/283017 objects degraded (11.738%); 8916/283017 objects misplaced (3.150%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T05:52:50.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:50 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:52:50.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:50 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3329689365' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:52:50.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:50 smithi156 ceph-mon[195101]: osdmap e149: 8 total, 8 up, 8 in 2023-10-09T05:52:50.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:50 smithi156 ceph-mon[195101]: failed to encode map e149 with expected crc 2023-10-09T05:52:50.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:50 smithi156 ceph-mon[195101]: failed to encode map e149 with expected crc 2023-10-09T05:52:50.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:50 smithi156 ceph-mon[195101]: failed to encode map e149 with expected crc 2023-10-09T05:52:50.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:50 smithi156 ceph-mon[195101]: failed to encode map e149 with expected crc 2023-10-09T05:52:50.633 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:50.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:50.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:50.739 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:50.770 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:51.080 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:51.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:51.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:51.171 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:52:51.171 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:52:51.186 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:51.215 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:51.263 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:51.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:51.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:51.369 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:51.406 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:51.467 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:51.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:51.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:51.571 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:51.609 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:51.665 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:51.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:51.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:51.772 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:52:51.774 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T05:52:51.774 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:51.803 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:51.852 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:51.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:51.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:51.960 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:51.990 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:52.040 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:52.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:52.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:52.150 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:52:52.186 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:52.466 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:52 smithi064 ceph-mon[212750]: pgmap v338: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 2 op/s; 33220/283017 objects degraded (11.738%); 8074/283017 objects misplaced (2.853%); 13 MiB/s, 203 objects/s recovering 2023-10-09T05:52:52.466 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:52 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 33220/283017 objects degraded (11.738%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:52.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:52 smithi064 ceph-mon[220044]: pgmap v338: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 2 op/s; 33220/283017 objects degraded (11.738%); 8074/283017 objects misplaced (2.853%); 13 MiB/s, 203 objects/s recovering 2023-10-09T05:52:52.467 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:52 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 33220/283017 objects degraded (11.738%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:52.505 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:52:52.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:52 smithi156 ceph-mon[195101]: pgmap v338: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 2 op/s; 33220/283017 objects degraded (11.738%); 8074/283017 objects misplaced (2.853%); 13 MiB/s, 203 objects/s recovering 2023-10-09T05:52:52.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:52 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 33220/283017 objects degraded (11.738%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:52.727 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":338,"stamp":"2023-10-09T05:52:50.491713+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6181504531,"num_objects":94339,"num_object_clones":0,"num_object_copies":283017,"num_objects_missing_on_primary":1173,"num_objects_missing":1173,"num_objects_degraded":33220,"num_objects_misplaced":8074,"num_objects_unfound":0,"num_objects_dirty":94339,"num_whiteouts":0,"num_read":445,"num_read_kb":833,"num_write":99046,"num_write_kb":6180787,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9314,"num_bytes_recovered":611423673,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100593,"ondisk_log_size":100593,"up":297,"acting":286,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":325,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19988344,"kb_used_data":18451704,"kb_used_omap":127,"kb_used_meta":1536448,"kb_avail":729972872,"statfs":{"total":767960285184,"available":747492220928,"internally_reserved":0,"allocated":18894544896,"data_stored":18889413773,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":130887,"internal_metadata":1573322937},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-1572864,"num_objects":-24,"num_object_clones":0,"num_object_copies":-72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1625,"num_objects_unfound":0,"num_objects_dirty":-24,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":24,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1625,"num_bytes_recovered":106496000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002363"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":164,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.229644+0000","last_change":"2023-10-09T05:52:11.412762+0000","last_active":"2023-10-09T05:52:43.229644+0000","last_peered":"2023-10-09T05:52:43.229644+0000","last_clean":"2023-10-09T05:52:43.229644+0000","last_became_active":"2023-10-09T05:52:11.412360+0000","last_became_peered":"2023-10-09T05:52:11.412360+0000","last_unstale":"2023-10-09T05:52:43.229644+0000","last_undegraded":"2023-10-09T05:52:43.229644+0000","last_fullsized":"2023-10-09T05:52:43.229644+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:28.459463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":131,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.144287+0000","last_change":"2023-10-09T05:52:11.417209+0000","last_active":"2023-10-09T05:52:43.144287+0000","last_peered":"2023-10-09T05:52:43.144287+0000","last_clean":"2023-10-09T05:52:43.144287+0000","last_became_active":"2023-10-09T05:52:11.416952+0000","last_became_peered":"2023-10-09T05:52:11.416952+0000","last_unstale":"2023-10-09T05:52:43.144287+0000","last_undegraded":"2023-10-09T05:52:43.144287+0000","last_fullsized":"2023-10-09T05:52:43.144287+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:32.958103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:52:13.234549+0000","last_change":"2023-10-09T05:52:11.410566+0000","last_active":"2023-10-09T05:52:13.234549+0000","last_peered":"2023-10-09T05:52:13.234549+0000","last_clean":"2023-10-09T05:52:13.234549+0000","last_became_active":"2023-10-09T05:52:11.410252+0000","last_became_peered":"2023-10-09T05:52:11.410252+0000","last_unstale":"2023-10-09T05:52:13.234549+0000","last_undegraded":"2023-10-09T05:52:13.234549+0000","last_fullsized":"2023-10-09T05:52:13.234549+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:06.878658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":93,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:52:36.134428+0000","last_change":"2023-10-09T05:52:23.639987+0000","last_active":"2023-10-09T05:52:36.134428+0000","last_peered":"2023-10-09T05:52:36.134428+0000","last_clean":"2023-10-09T05:52:36.134428+0000","last_became_active":"2023-10-09T05:52:23.639671+0000","last_became_peered":"2023-10-09T05:52:23.639671+0000","last_unstale":"2023-10-09T05:52:36.134428+0000","last_undegraded":"2023-10-09T05:52:36.134428+0000","last_fullsized":"2023-10-09T05:52:36.134428+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:19.677942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":164,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.143994+0000","last_change":"2023-10-09T05:52:11.221586+0000","last_active":"2023-10-09T05:52:43.143994+0000","last_peered":"2023-10-09T05:52:43.143994+0000","last_clean":"2023-10-09T05:52:43.143994+0000","last_became_active":"2023-10-09T05:52:11.221307+0000","last_became_peered":"2023-10-09T05:52:11.221307+0000","last_unstale":"2023-10-09T05:52:43.143994+0000","last_undegraded":"2023-10-09T05:52:43.143994+0000","last_fullsized":"2023-10-09T05:52:43.143994+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:48:04.204923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":145,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.225122+0000","last_change":"2023-10-09T05:52:11.218711+0000","last_active":"2023-10-09T05:52:43.225122+0000","last_peered":"2023-10-09T05:52:43.225122+0000","last_clean":"2023-10-09T05:52:43.225122+0000","last_became_active":"2023-10-09T05:52:11.218513+0000","last_became_peered":"2023-10-09T05:52:11.218513+0000","last_unstale":"2023-10-09T05:52:43.225122+0000","last_undegraded":"2023-10-09T05:52:43.225122+0000","last_fullsized":"2023-10-09T05:52:43.225122+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:02.240130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":83,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:52:36.134735+0000","last_change":"2023-10-09T05:51:57.202627+0000","last_active":"2023-10-09T05:52:36.134735+0000","last_peered":"2023-10-09T05:52:36.134735+0000","last_clean":"2023-10-09T05:52:36.134735+0000","last_became_active":"2023-10-09T05:51:57.097707+0000","last_became_peered":"2023-10-09T05:51:57.097707+0000","last_unstale":"2023-10-09T05:52:36.134735+0000","last_undegraded":"2023-10-09T05:52:36.134735+0000","last_fullsized":"2023-10-09T05:52:36.134735+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:02.682328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":131,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.144053+0000","last_change":"2023-10-09T05:52:11.210606+0000","last_active":"2023-10-09T05:52:43.144053+0000","last_peered":"2023-10-09T05:52:43.144053+0000","last_clean":"2023-10-09T05:52:43.144053+0000","last_became_active":"2023-10-09T05:52:11.209886+0000","last_became_peered":"2023-10-09T05:52:11.209886+0000","last_unstale":"2023-10-09T05:52:43.144053+0000","last_undegraded":"2023-10-09T05:52:43.144053+0000","last_fullsized":"2023-10-09T05:52:43.144053+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:56.049681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":129,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.223147+0000","last_change":"2023-10-09T05:52:11.214597+0000","last_active":"2023-10-09T05:52:43.223147+0000","last_peered":"2023-10-09T05:52:43.223147+0000","last_clean":"2023-10-09T05:52:43.223147+0000","last_became_active":"2023-10-09T05:52:11.211759+0000","last_became_peered":"2023-10-09T05:52:11.211759+0000","last_unstale":"2023-10-09T05:52:43.223147+0000","last_undegraded":"2023-10-09T05:52:43.223147+0000","last_fullsized":"2023-10-09T05:52:43.223147+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:32.429536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":72,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.144114+0000","last_change":"2023-10-09T05:49:21.425819+0000","last_active":"2023-10-09T05:52:43.144114+0000","last_peered":"2023-10-09T05:52:43.144114+0000","last_clean":"2023-10-09T05:52:43.144114+0000","last_became_active":"2023-10-09T05:49:21.425498+0000","last_became_peered":"2023-10-09T05:49:21.425498+0000","last_unstale":"2023-10-09T05:52:43.144114+0000","last_undegraded":"2023-10-09T05:52:43.144114+0000","last_fullsized":"2023-10-09T05:52:43.144114+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:21.443055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":112,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.223928+0000","last_change":"2023-10-09T05:49:21.430469+0000","last_active":"2023-10-09T05:52:43.223928+0000","last_peered":"2023-10-09T05:52:43.223928+0000","last_clean":"2023-10-09T05:52:43.223928+0000","last_became_active":"2023-10-09T05:49:21.430107+0000","last_became_peered":"2023-10-09T05:49:21.430107+0000","last_unstale":"2023-10-09T05:52:43.223928+0000","last_undegraded":"2023-10-09T05:52:43.223928+0000","last_fullsized":"2023-10-09T05:52:43.223928+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:08.815343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:52:13.234684+0000","last_change":"2023-10-09T05:52:11.224890+0000","last_active":"2023-10-09T05:52:13.234684+0000","last_peered":"2023-10-09T05:52:13.234684+0000","last_clean":"2023-10-09T05:52:13.234684+0000","last_became_active":"2023-10-09T05:52:11.224591+0000","last_became_peered":"2023-10-09T05:52:11.224591+0000","last_unstale":"2023-10-09T05:52:13.234684+0000","last_undegraded":"2023-10-09T05:52:13.234684+0000","last_fullsized":"2023-10-09T05:52:13.234684+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:53.788492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":114,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.229996+0000","last_change":"2023-10-09T05:40:12.630786+0000","last_active":"2023-10-09T05:52:43.229996+0000","last_peered":"2023-10-09T05:52:43.229996+0000","last_clean":"2023-10-09T05:52:43.229996+0000","last_became_active":"2023-10-09T05:40:12.630528+0000","last_became_peered":"2023-10-09T05:40:12.630528+0000","last_unstale":"2023-10-09T05:52:43.229996+0000","last_undegraded":"2023-10-09T05:52:43.229996+0000","last_fullsized":"2023-10-09T05:52:43.229996+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:54.427346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":128,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.229518+0000","last_change":"2023-10-09T05:52:11.412779+0000","last_active":"2023-10-09T05:52:43.229518+0000","last_peered":"2023-10-09T05:52:43.229518+0000","last_clean":"2023-10-09T05:52:43.229518+0000","last_became_active":"2023-10-09T05:52:11.412489+0000","last_became_peered":"2023-10-09T05:52:11.412489+0000","last_unstale":"2023-10-09T05:52:43.229518+0000","last_undegraded":"2023-10-09T05:52:43.229518+0000","last_fullsized":"2023-10-09T05:52:43.229518+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:39:37.025949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":111,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.223589+0000","last_change":"2023-10-09T05:48:49.139681+0000","last_active":"2023-10-09T05:52:43.223589+0000","last_peered":"2023-10-09T05:52:43.223589+0000","last_clean":"2023-10-09T05:52:43.223589+0000","last_became_active":"2023-10-09T05:48:49.139317+0000","last_became_peered":"2023-10-09T05:48:49.139317+0000","last_unstale":"2023-10-09T05:52:43.223589+0000","last_undegraded":"2023-10-09T05:52:43.223589+0000","last_fullsized":"2023-10-09T05:52:43.223589+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:34.769043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":131,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.144408+0000","last_change":"2023-10-09T05:52:11.210197+0000","last_active":"2023-10-09T05:52:43.144408+0000","last_peered":"2023-10-09T05:52:43.144408+0000","last_clean":"2023-10-09T05:52:43.144408+0000","last_became_active":"2023-10-09T05:52:11.209895+0000","last_became_peered":"2023-10-09T05:52:11.209895+0000","last_unstale":"2023-10-09T05:52:43.144408+0000","last_undegraded":"2023-10-09T05:52:43.144408+0000","last_fullsized":"2023-10-09T05:52:43.144408+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:13.043187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":93,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:52:36.133766+0000","last_change":"2023-10-09T05:52:23.640711+0000","last_active":"2023-10-09T05:52:36.133766+0000","last_peered":"2023-10-09T05:52:36.133766+0000","last_clean":"2023-10-09T05:52:36.133766+0000","last_became_active":"2023-10-09T05:52:23.640420+0000","last_became_peered":"2023-10-09T05:52:23.640420+0000","last_unstale":"2023-10-09T05:52:36.133766+0000","last_undegraded":"2023-10-09T05:52:36.133766+0000","last_fullsized":"2023-10-09T05:52:36.133766+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:00:36.382963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":108,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.225190+0000","last_change":"2023-10-09T05:49:21.423218+0000","last_active":"2023-10-09T05:52:43.225190+0000","last_peered":"2023-10-09T05:52:43.225190+0000","last_clean":"2023-10-09T05:52:43.225190+0000","last_became_active":"2023-10-09T05:49:21.422706+0000","last_became_peered":"2023-10-09T05:49:21.422706+0000","last_unstale":"2023-10-09T05:52:43.225190+0000","last_undegraded":"2023-10-09T05:52:43.225190+0000","last_fullsized":"2023-10-09T05:52:43.225190+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:31.571220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":92,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.223435+0000","last_change":"2023-10-09T05:40:12.631893+0000","last_active":"2023-10-09T05:52:43.223435+0000","last_peered":"2023-10-09T05:52:43.223435+0000","last_clean":"2023-10-09T05:52:43.223435+0000","last_became_active":"2023-10-09T05:40:12.631592+0000","last_became_peered":"2023-10-09T05:40:12.631592+0000","last_unstale":"2023-10-09T05:52:43.223435+0000","last_undegraded":"2023-10-09T05:52:43.223435+0000","last_fullsized":"2023-10-09T05:52:43.223435+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:18:12.517403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":83,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:52:36.134275+0000","last_change":"2023-10-09T05:51:57.212052+0000","last_active":"2023-10-09T05:52:36.134275+0000","last_peered":"2023-10-09T05:52:36.134275+0000","last_clean":"2023-10-09T05:52:36.134275+0000","last_became_active":"2023-10-09T05:51:57.207796+0000","last_became_peered":"2023-10-09T05:51:57.207796+0000","last_unstale":"2023-10-09T05:52:36.134275+0000","last_undegraded":"2023-10-09T05:52:36.134275+0000","last_fullsized":"2023-10-09T05:52:36.134275+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:58.484959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":134,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.223283+0000","last_change":"2023-10-09T05:51:57.091639+0000","last_active":"2023-10-09T05:52:43.223283+0000","last_peered":"2023-10-09T05:52:43.223283+0000","last_clean":"2023-10-09T05:52:43.223283+0000","last_became_active":"2023-10-09T05:51:57.091284+0000","last_became_peered":"2023-10-09T05:51:57.091284+0000","last_unstale":"2023-10-09T05:52:43.223283+0000","last_undegraded":"2023-10-09T05:52:43.223283+0000","last_fullsized":"2023-10-09T05:52:43.223283+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:10.553453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":130,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.223386+0000","last_change":"2023-10-09T05:52:23.637255+0000","last_active":"2023-10-09T05:52:43.223386+0000","last_peered":"2023-10-09T05:52:43.223386+0000","last_clean":"2023-10-09T05:52:43.223386+0000","last_became_active":"2023-10-09T05:52:23.636918+0000","last_became_peered":"2023-10-09T05:52:23.636918+0000","last_unstale":"2023-10-09T05:52:43.223386+0000","last_undegraded":"2023-10-09T05:52:43.223386+0000","last_fullsized":"2023-10-09T05:52:43.223386+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:43.616064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.a","version":"140'6175","reported_seq":6800,"reported_epoch":148,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:43.225727+0000","last_change":"2023-10-09T05:52:11.342223+0000","last_active":"2023-10-09T05:52:43.225727+0000","last_peered":"2023-10-09T05:52:43.225727+0000","last_clean":"2023-10-09T05:51:33.088844+0000","last_became_active":"2023-10-09T05:51:58.249120+0000","last_became_peered":"2023-10-09T05:51:58.249120+0000","last_unstale":"2023-10-09T05:52:43.225727+0000","last_undegraded":"2023-10-09T05:51:58.095514+0000","last_fullsized":"2023-10-09T05:51:58.082441+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6175,"log_dups_size":0,"ondisk_log_size":6175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:16:45.784914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383451136,"num_objects":5851,"num_object_clones":0,"num_object_copies":17553,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":809,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5851,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6175,"num_write_kb":384832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":276,"num_bytes_recovered":18087936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":5042}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":127,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.223264+0000","last_change":"2023-10-09T05:51:57.209892+0000","last_active":"2023-10-09T05:52:43.223264+0000","last_peered":"2023-10-09T05:52:43.223264+0000","last_clean":"2023-10-09T05:52:43.223264+0000","last_became_active":"2023-10-09T05:51:57.209354+0000","last_became_peered":"2023-10-09T05:51:57.209354+0000","last_unstale":"2023-10-09T05:52:43.223264+0000","last_undegraded":"2023-10-09T05:52:43.223264+0000","last_fullsized":"2023-10-09T05:52:43.223264+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:41:28.128976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":91,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.223434+0000","last_change":"2023-10-09T05:40:14.362515+0000","last_active":"2023-10-09T05:52:43.223434+0000","last_peered":"2023-10-09T05:52:43.223434+0000","last_clean":"2023-10-09T05:52:43.223434+0000","last_became_active":"2023-10-09T05:40:14.362184+0000","last_became_peered":"2023-10-09T05:40:14.362184+0000","last_unstale":"2023-10-09T05:52:43.223434+0000","last_undegraded":"2023-10-09T05:52:43.223434+0000","last_fullsized":"2023-10-09T05:52:43.223434+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:37.681193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"147'6256","reported_seq":10769,"reported_epoch":148,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:43.146115+0000","last_change":"2023-10-09T05:52:11.493867+0000","last_active":"2023-10-09T05:52:43.146115+0000","last_peered":"2023-10-09T05:52:43.146115+0000","last_clean":"2023-10-09T05:51:24.569821+0000","last_became_active":"2023-10-09T05:52:11.486930+0000","last_became_peered":"2023-10-09T05:52:11.486930+0000","last_unstale":"2023-10-09T05:52:43.146115+0000","last_undegraded":"2023-10-09T05:52:11.207681+0000","last_fullsized":"2023-10-09T05:52:11.192685+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6256,"log_dups_size":0,"ondisk_log_size":6256,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:37:50.679348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389939200,"num_objects":5950,"num_object_clones":0,"num_object_copies":17850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2041,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5950,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6256,"num_write_kb":390592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2221,"num_bytes_recovered":145555456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,6,7","objects":2041},{"shards":"3,7","objects":3909}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":159,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.224134+0000","last_change":"2023-10-09T05:52:26.421671+0000","last_active":"2023-10-09T05:52:43.224134+0000","last_peered":"2023-10-09T05:52:43.224134+0000","last_clean":"2023-10-09T05:52:43.224134+0000","last_became_active":"2023-10-09T05:52:11.353717+0000","last_became_peered":"2023-10-09T05:52:11.353717+0000","last_unstale":"2023-10-09T05:52:43.224134+0000","last_undegraded":"2023-10-09T05:52:43.224134+0000","last_fullsized":"2023-10-09T05:52:43.224134+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:08.999775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":155,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.229610+0000","last_change":"2023-10-09T05:52:23.642420+0000","last_active":"2023-10-09T05:52:43.229610+0000","last_peered":"2023-10-09T05:52:43.229610+0000","last_clean":"2023-10-09T05:52:43.229610+0000","last_became_active":"2023-10-09T05:52:23.641015+0000","last_became_peered":"2023-10-09T05:52:23.641015+0000","last_unstale":"2023-10-09T05:52:43.229610+0000","last_undegraded":"2023-10-09T05:52:43.229610+0000","last_fullsized":"2023-10-09T05:52:43.229610+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:14.092876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"147'6150","reported_seq":6205,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.224487+0000","last_change":"2023-10-09T05:50:39.777752+0000","last_active":"2023-10-09T05:52:43.224487+0000","last_peered":"2023-10-09T05:52:43.224487+0000","last_clean":"2023-10-09T05:52:43.224487+0000","last_became_active":"2023-10-09T05:50:39.777526+0000","last_became_peered":"2023-10-09T05:50:39.777526+0000","last_unstale":"2023-10-09T05:52:43.224487+0000","last_undegraded":"2023-10-09T05:52:43.224487+0000","last_fullsized":"2023-10-09T05:52:43.224487+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6150,"log_dups_size":0,"ondisk_log_size":6150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:19.849370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378929152,"num_objects":5782,"num_object_clones":0,"num_object_copies":17346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6150,"num_write_kb":381824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":93,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.223821+0000","last_change":"2023-10-09T05:40:12.632083+0000","last_active":"2023-10-09T05:52:43.223821+0000","last_peered":"2023-10-09T05:52:43.223821+0000","last_clean":"2023-10-09T05:52:43.223821+0000","last_became_active":"2023-10-09T05:40:12.631715+0000","last_became_peered":"2023-10-09T05:40:12.631715+0000","last_unstale":"2023-10-09T05:52:43.223821+0000","last_undegraded":"2023-10-09T05:52:43.223821+0000","last_fullsized":"2023-10-09T05:52:43.223821+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:17.641378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":138,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.223252+0000","last_change":"2023-10-09T05:52:11.213297+0000","last_active":"2023-10-09T05:52:43.223252+0000","last_peered":"2023-10-09T05:52:43.223252+0000","last_clean":"2023-10-09T05:52:43.223252+0000","last_became_active":"2023-10-09T05:52:11.212974+0000","last_became_peered":"2023-10-09T05:52:11.212974+0000","last_unstale":"2023-10-09T05:52:43.223252+0000","last_undegraded":"2023-10-09T05:52:43.223252+0000","last_fullsized":"2023-10-09T05:52:43.223252+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:26:39.083838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"147'6212","reported_seq":6292,"reported_epoch":147,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:42.715154+0000","last_change":"2023-10-09T05:52:42.557805+0000","last_active":"2023-10-09T05:52:42.715154+0000","last_peered":"2023-10-09T05:52:42.715154+0000","last_clean":"2023-10-09T05:51:33.020184+0000","last_became_active":"2023-10-09T05:52:42.288853+0000","last_became_peered":"2023-10-09T05:52:42.288853+0000","last_unstale":"2023-10-09T05:52:42.715154+0000","last_undegraded":"2023-10-09T05:52:42.125072+0000","last_fullsized":"2023-10-09T05:52:24.631025+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6212,"log_dups_size":0,"ondisk_log_size":6212,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:54:46.897489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388759552,"num_objects":5932,"num_object_clones":0,"num_object_copies":17796,"num_objects_missing_on_primary":1173,"num_objects_missing":1173,"num_objects_degraded":7245,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5932,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6212,"num_write_kb":388608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,7","objects":4759},{"shards":"0,7","objects":-3},{"shards":"2,7","objects":1173}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":109,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.224584+0000","last_change":"2023-10-09T05:49:21.429234+0000","last_active":"2023-10-09T05:52:43.224584+0000","last_peered":"2023-10-09T05:52:43.224584+0000","last_clean":"2023-10-09T05:52:43.224584+0000","last_became_active":"2023-10-09T05:49:21.428961+0000","last_became_peered":"2023-10-09T05:49:21.428961+0000","last_unstale":"2023-10-09T05:52:43.224584+0000","last_undegraded":"2023-10-09T05:52:43.224584+0000","last_fullsized":"2023-10-09T05:52:43.224584+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:18.029914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":120,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:52:36.133864+0000","last_change":"2023-10-09T05:52:11.345261+0000","last_active":"2023-10-09T05:52:36.133864+0000","last_peered":"2023-10-09T05:52:36.133864+0000","last_clean":"2023-10-09T05:52:36.133864+0000","last_became_active":"2023-10-09T05:52:11.345031+0000","last_became_peered":"2023-10-09T05:52:11.345031+0000","last_unstale":"2023-10-09T05:52:36.133864+0000","last_undegraded":"2023-10-09T05:52:36.133864+0000","last_fullsized":"2023-10-09T05:52:36.133864+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:07.748821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":520,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.223242+0000","last_change":"2023-10-09T05:51:19.197324+0000","last_active":"2023-10-09T05:52:43.223242+0000","last_peered":"2023-10-09T05:52:43.223242+0000","last_clean":"2023-10-09T05:52:43.223242+0000","last_became_active":"2023-10-09T05:51:19.197052+0000","last_became_peered":"2023-10-09T05:51:19.197052+0000","last_unstale":"2023-10-09T05:52:43.223242+0000","last_undegraded":"2023-10-09T05:52:43.223242+0000","last_fullsized":"2023-10-09T05:52:43.223242+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.e","version":"140'6148","reported_seq":7418,"reported_epoch":148,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:43.233284+0000","last_change":"2023-10-09T05:52:24.892066+0000","last_active":"2023-10-09T05:52:43.233284+0000","last_peered":"2023-10-09T05:52:43.233284+0000","last_clean":"2023-10-09T05:51:24.573410+0000","last_became_active":"2023-10-09T05:52:24.814156+0000","last_became_peered":"2023-10-09T05:52:24.814156+0000","last_unstale":"2023-10-09T05:52:43.233284+0000","last_undegraded":"2023-10-09T05:52:24.800936+0000","last_fullsized":"2023-10-09T05:52:24.635559+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6148,"log_dups_size":0,"ondisk_log_size":6148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:35.696825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382205952,"num_objects":5832,"num_object_clones":0,"num_object_copies":17496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5556,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6148,"num_write_kb":383360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":591,"num_bytes_recovered":38731776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"1,2,4","objects":3298},{"shards":"1,4","objects":433},{"shards":"2,4,5","objects":2101}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":126,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.144705+0000","last_change":"2023-10-09T05:51:57.092524+0000","last_active":"2023-10-09T05:52:43.144705+0000","last_peered":"2023-10-09T05:52:43.144705+0000","last_clean":"2023-10-09T05:52:43.144705+0000","last_became_active":"2023-10-09T05:51:57.092273+0000","last_became_peered":"2023-10-09T05:51:57.092273+0000","last_unstale":"2023-10-09T05:52:43.144705+0000","last_undegraded":"2023-10-09T05:52:43.144705+0000","last_fullsized":"2023-10-09T05:52:43.144705+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:56.479664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":108,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.229958+0000","last_change":"2023-10-09T05:49:21.430742+0000","last_active":"2023-10-09T05:52:43.229958+0000","last_peered":"2023-10-09T05:52:43.229958+0000","last_clean":"2023-10-09T05:52:43.229958+0000","last_became_active":"2023-10-09T05:49:21.430233+0000","last_became_peered":"2023-10-09T05:49:21.430233+0000","last_unstale":"2023-10-09T05:52:43.229958+0000","last_undegraded":"2023-10-09T05:52:43.229958+0000","last_fullsized":"2023-10-09T05:52:43.229958+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:10.541302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":453,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.223357+0000","last_change":"2023-10-09T05:51:19.193204+0000","last_active":"2023-10-09T05:52:43.223357+0000","last_peered":"2023-10-09T05:52:43.223357+0000","last_clean":"2023-10-09T05:52:43.223357+0000","last_became_active":"2023-10-09T05:51:19.192660+0000","last_became_peered":"2023-10-09T05:51:19.192660+0000","last_unstale":"2023-10-09T05:52:43.223357+0000","last_undegraded":"2023-10-09T05:52:43.223357+0000","last_fullsized":"2023-10-09T05:52:43.223357+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.f","version":"138'6281","reported_seq":6339,"reported_epoch":148,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:43.148660+0000","last_change":"2023-10-09T05:52:31.004567+0000","last_active":"2023-10-09T05:52:43.148660+0000","last_peered":"2023-10-09T05:52:43.148660+0000","last_clean":"2023-10-09T05:52:22.632757+0000","last_became_active":"2023-10-09T05:52:30.992047+0000","last_became_peered":"2023-10-09T05:52:30.992047+0000","last_unstale":"2023-10-09T05:52:43.148660+0000","last_undegraded":"2023-10-09T05:52:30.768811+0000","last_fullsized":"2023-10-09T05:52:30.724937+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6281,"log_dups_size":0,"ondisk_log_size":6281,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:40.469088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393281536,"num_objects":6001,"num_object_clones":0,"num_object_copies":18003,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6141,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6001,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6281,"num_write_kb":393024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":6001}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":129,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.223476+0000","last_change":"2023-10-09T05:52:11.344035+0000","last_active":"2023-10-09T05:52:43.223476+0000","last_peered":"2023-10-09T05:52:43.223476+0000","last_clean":"2023-10-09T05:52:43.223476+0000","last_became_active":"2023-10-09T05:52:11.343800+0000","last_became_peered":"2023-10-09T05:52:11.343800+0000","last_unstale":"2023-10-09T05:52:43.223476+0000","last_undegraded":"2023-10-09T05:52:43.223476+0000","last_fullsized":"2023-10-09T05:52:43.223476+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:07.588214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":145,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.229746+0000","last_change":"2023-10-09T05:52:11.419040+0000","last_active":"2023-10-09T05:52:43.229746+0000","last_peered":"2023-10-09T05:52:43.229746+0000","last_clean":"2023-10-09T05:52:43.229746+0000","last_became_active":"2023-10-09T05:52:11.418708+0000","last_became_peered":"2023-10-09T05:52:11.418708+0000","last_unstale":"2023-10-09T05:52:43.229746+0000","last_undegraded":"2023-10-09T05:52:43.229746+0000","last_fullsized":"2023-10-09T05:52:43.229746+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:22.969249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"147'6191","reported_seq":10102,"reported_epoch":149,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:49.439957+0000","last_change":"2023-10-09T05:52:24.904553+0000","last_active":"2023-10-09T05:52:49.439957+0000","last_peered":"2023-10-09T05:52:49.439957+0000","last_clean":"2023-10-09T05:52:22.567157+0000","last_became_active":"2023-10-09T05:52:24.871228+0000","last_became_peered":"2023-10-09T05:52:24.871228+0000","last_unstale":"2023-10-09T05:52:49.439957+0000","last_undegraded":"2023-10-09T05:52:24.677767+0000","last_fullsized":"2023-10-09T05:52:24.631791+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6191,"log_dups_size":0,"ondisk_log_size":6191,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:42.864852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":385024000,"num_objects":5875,"num_object_clones":0,"num_object_copies":17625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":3958,"num_objects_unfound":0,"num_objects_dirty":5875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6191,"num_write_kb":386112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1923,"num_bytes_recovered":125894656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5],"avail_no_missing":["6","3","5"],"object_location_counts":[{"shards":"3,5,6","objects":3958},{"shards":"5,6","objects":1916}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":109,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.229323+0000","last_change":"2023-10-09T05:48:49.141287+0000","last_active":"2023-10-09T05:52:43.229323+0000","last_peered":"2023-10-09T05:52:43.229323+0000","last_clean":"2023-10-09T05:52:43.229323+0000","last_became_active":"2023-10-09T05:48:49.140950+0000","last_became_peered":"2023-10-09T05:48:49.140950+0000","last_unstale":"2023-10-09T05:52:43.229323+0000","last_undegraded":"2023-10-09T05:52:43.229323+0000","last_fullsized":"2023-10-09T05:52:43.229323+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:35.527434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":128,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.230112+0000","last_change":"2023-10-09T05:52:11.412851+0000","last_active":"2023-10-09T05:52:43.230112+0000","last_peered":"2023-10-09T05:52:43.230112+0000","last_clean":"2023-10-09T05:52:43.230112+0000","last_became_active":"2023-10-09T05:52:11.412489+0000","last_became_peered":"2023-10-09T05:52:11.412489+0000","last_unstale":"2023-10-09T05:52:43.230112+0000","last_undegraded":"2023-10-09T05:52:43.230112+0000","last_fullsized":"2023-10-09T05:52:43.230112+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:30:34.286409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":453,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.225237+0000","last_change":"2023-10-09T05:51:19.195984+0000","last_active":"2023-10-09T05:52:43.225237+0000","last_peered":"2023-10-09T05:52:43.225237+0000","last_clean":"2023-10-09T05:52:43.225237+0000","last_became_active":"2023-10-09T05:51:19.195632+0000","last_became_peered":"2023-10-09T05:51:19.195632+0000","last_unstale":"2023-10-09T05:52:43.225237+0000","last_undegraded":"2023-10-09T05:52:43.225237+0000","last_fullsized":"2023-10-09T05:52:43.225237+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.d","version":"147'6155","reported_seq":11383,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.225043+0000","last_change":"2023-10-09T05:52:27.708167+0000","last_active":"2023-10-09T05:52:43.225043+0000","last_peered":"2023-10-09T05:52:43.225043+0000","last_clean":"2023-10-09T05:52:43.225043+0000","last_became_active":"2023-10-09T05:52:27.701168+0000","last_became_peered":"2023-10-09T05:52:27.701168+0000","last_unstale":"2023-10-09T05:52:43.225043+0000","last_undegraded":"2023-10-09T05:52:43.225043+0000","last_fullsized":"2023-10-09T05:52:43.225043+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6155,"log_dups_size":0,"ondisk_log_size":6155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:17.406250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383254552,"num_objects":5849,"num_object_clones":0,"num_object_copies":17547,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5849,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6155,"num_write_kb":384065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2559,"num_bytes_recovered":167706624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":120,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:52:13.229403+0000","last_change":"2023-10-09T05:52:11.211488+0000","last_active":"2023-10-09T05:52:13.229403+0000","last_peered":"2023-10-09T05:52:13.229403+0000","last_clean":"2023-10-09T05:52:13.229403+0000","last_became_active":"2023-10-09T05:52:11.210352+0000","last_became_peered":"2023-10-09T05:52:11.210352+0000","last_unstale":"2023-10-09T05:52:13.229403+0000","last_undegraded":"2023-10-09T05:52:13.229403+0000","last_fullsized":"2023-10-09T05:52:13.229403+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:21:08.194392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":14,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:52:13.236176+0000","last_change":"2023-10-09T05:52:11.409470+0000","last_active":"2023-10-09T05:52:13.236176+0000","last_peered":"2023-10-09T05:52:13.236176+0000","last_clean":"2023-10-09T05:52:13.236176+0000","last_became_active":"2023-10-09T05:52:11.409125+0000","last_became_peered":"2023-10-09T05:52:11.409125+0000","last_unstale":"2023-10-09T05:52:13.236176+0000","last_undegraded":"2023-10-09T05:52:13.236176+0000","last_fullsized":"2023-10-09T05:52:13.236176+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:39:35.220346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":482,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.225425+0000","last_change":"2023-10-09T05:52:26.673774+0000","last_active":"2023-10-09T05:52:43.225425+0000","last_peered":"2023-10-09T05:52:43.225425+0000","last_clean":"2023-10-09T05:52:43.225425+0000","last_became_active":"2023-10-09T05:52:26.673393+0000","last_became_peered":"2023-10-09T05:52:26.673393+0000","last_unstale":"2023-10-09T05:52:43.225425+0000","last_undegraded":"2023-10-09T05:52:43.225425+0000","last_fullsized":"2023-10-09T05:52:43.225425+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:52:13.234879+0000","last_change":"2023-10-09T05:52:11.226922+0000","last_active":"2023-10-09T05:52:13.234879+0000","last_peered":"2023-10-09T05:52:13.234879+0000","last_clean":"2023-10-09T05:52:13.234879+0000","last_became_active":"2023-10-09T05:52:11.226605+0000","last_became_peered":"2023-10-09T05:52:11.226605+0000","last_unstale":"2023-10-09T05:52:13.234879+0000","last_undegraded":"2023-10-09T05:52:13.234879+0000","last_fullsized":"2023-10-09T05:52:13.234879+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:58.006329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":136,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.225273+0000","last_change":"2023-10-09T05:52:29.735372+0000","last_active":"2023-10-09T05:52:43.225273+0000","last_peered":"2023-10-09T05:52:43.225273+0000","last_clean":"2023-10-09T05:52:43.225273+0000","last_became_active":"2023-10-09T05:52:29.735026+0000","last_became_peered":"2023-10-09T05:52:29.735026+0000","last_unstale":"2023-10-09T05:52:43.225273+0000","last_undegraded":"2023-10-09T05:52:43.225273+0000","last_fullsized":"2023-10-09T05:52:43.225273+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:53.849406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:52:13.234715+0000","last_change":"2023-10-09T05:52:11.407985+0000","last_active":"2023-10-09T05:52:13.234715+0000","last_peered":"2023-10-09T05:52:13.234715+0000","last_clean":"2023-10-09T05:52:13.234715+0000","last_became_active":"2023-10-09T05:52:11.407681+0000","last_became_peered":"2023-10-09T05:52:11.407681+0000","last_unstale":"2023-10-09T05:52:13.234715+0000","last_undegraded":"2023-10-09T05:52:13.234715+0000","last_fullsized":"2023-10-09T05:52:13.234715+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:54.384669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"147'6238","reported_seq":6303,"reported_epoch":147,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:42.730992+0000","last_change":"2023-10-09T05:51:58.262986+0000","last_active":"2023-10-09T05:52:42.730992+0000","last_peered":"2023-10-09T05:52:42.730992+0000","last_clean":"2023-10-09T05:51:33.087654+0000","last_became_active":"2023-10-09T05:51:58.252808+0000","last_became_peered":"2023-10-09T05:51:58.252808+0000","last_unstale":"2023-10-09T05:52:42.730992+0000","last_undegraded":"2023-10-09T05:51:58.088414+0000","last_fullsized":"2023-10-09T05:51:58.077844+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6238,"log_dups_size":0,"ondisk_log_size":6238,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:00.878041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387579904,"num_objects":5914,"num_object_clones":0,"num_object_copies":17742,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1172,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5914,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6238,"num_write_kb":388864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":4742}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":470,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.223801+0000","last_change":"2023-10-09T05:51:57.209470+0000","last_active":"2023-10-09T05:52:43.223801+0000","last_peered":"2023-10-09T05:52:43.223801+0000","last_clean":"2023-10-09T05:52:43.223801+0000","last_became_active":"2023-10-09T05:51:57.208993+0000","last_became_peered":"2023-10-09T05:51:57.208993+0000","last_unstale":"2023-10-09T05:52:43.223801+0000","last_undegraded":"2023-10-09T05:52:43.223801+0000","last_fullsized":"2023-10-09T05:52:43.223801+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.6","version":"147'6200","reported_seq":6262,"reported_epoch":148,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:43.232899+0000","last_change":"2023-10-09T05:52:30.983730+0000","last_active":"2023-10-09T05:52:43.232899+0000","last_peered":"2023-10-09T05:52:43.232899+0000","last_clean":"2023-10-09T05:52:24.946135+0000","last_became_active":"2023-10-09T05:52:30.971337+0000","last_became_peered":"2023-10-09T05:52:30.971337+0000","last_unstale":"2023-10-09T05:52:43.232899+0000","last_undegraded":"2023-10-09T05:52:30.773579+0000","last_fullsized":"2023-10-09T05:52:30.746434+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6200,"log_dups_size":0,"ondisk_log_size":6200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:04:00.772909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382992384,"num_objects":5844,"num_object_clones":0,"num_object_copies":17532,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6022,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5844,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6200,"num_write_kb":385408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":5844},{"shards":"4,7","objects":-1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":150,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.225120+0000","last_change":"2023-10-09T05:52:11.217079+0000","last_active":"2023-10-09T05:52:43.225120+0000","last_peered":"2023-10-09T05:52:43.225120+0000","last_clean":"2023-10-09T05:52:43.225120+0000","last_became_active":"2023-10-09T05:52:11.216737+0000","last_became_peered":"2023-10-09T05:52:11.216737+0000","last_unstale":"2023-10-09T05:52:43.225120+0000","last_undegraded":"2023-10-09T05:52:43.225120+0000","last_fullsized":"2023-10-09T05:52:43.225120+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:04.493525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":126,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.229683+0000","last_change":"2023-10-09T05:51:57.203142+0000","last_active":"2023-10-09T05:52:43.229683+0000","last_peered":"2023-10-09T05:52:43.229683+0000","last_clean":"2023-10-09T05:52:43.229683+0000","last_became_active":"2023-10-09T05:51:57.202850+0000","last_became_peered":"2023-10-09T05:51:57.202850+0000","last_unstale":"2023-10-09T05:52:43.229683+0000","last_undegraded":"2023-10-09T05:52:43.229683+0000","last_fullsized":"2023-10-09T05:52:43.229683+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:51.090737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:52:13.234803+0000","last_change":"2023-10-09T05:52:11.232219+0000","last_active":"2023-10-09T05:52:13.234803+0000","last_peered":"2023-10-09T05:52:13.234803+0000","last_clean":"2023-10-09T05:52:13.234803+0000","last_became_active":"2023-10-09T05:52:11.231851+0000","last_became_peered":"2023-10-09T05:52:11.231851+0000","last_unstale":"2023-10-09T05:52:13.234803+0000","last_undegraded":"2023-10-09T05:52:13.234803+0000","last_fullsized":"2023-10-09T05:52:13.234803+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:02.962989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":537,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.223572+0000","last_change":"2023-10-09T05:51:57.221078+0000","last_active":"2023-10-09T05:52:43.223572+0000","last_peered":"2023-10-09T05:52:43.223572+0000","last_clean":"2023-10-09T05:52:43.223572+0000","last_became_active":"2023-10-09T05:51:57.220754+0000","last_became_peered":"2023-10-09T05:51:57.220754+0000","last_unstale":"2023-10-09T05:52:43.223572+0000","last_undegraded":"2023-10-09T05:52:43.223572+0000","last_fullsized":"2023-10-09T05:52:43.223572+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.5","version":"147'6070","reported_seq":6125,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.224408+0000","last_change":"2023-10-09T05:50:40.123175+0000","last_active":"2023-10-09T05:52:43.224408+0000","last_peered":"2023-10-09T05:52:43.224408+0000","last_clean":"2023-10-09T05:52:43.224408+0000","last_became_active":"2023-10-09T05:50:40.122928+0000","last_became_peered":"2023-10-09T05:50:40.122928+0000","last_unstale":"2023-10-09T05:52:43.224408+0000","last_undegraded":"2023-10-09T05:52:43.224408+0000","last_fullsized":"2023-10-09T05:52:43.224408+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6070,"log_dups_size":0,"ondisk_log_size":6070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:16.111485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375783424,"num_objects":5734,"num_object_clones":0,"num_object_copies":17202,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6070,"num_write_kb":377728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":155,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:52:36.131922+0000","last_change":"2023-10-09T05:52:23.638460+0000","last_active":"2023-10-09T05:52:36.131922+0000","last_peered":"2023-10-09T05:52:36.131922+0000","last_clean":"2023-10-09T05:52:36.131922+0000","last_became_active":"2023-10-09T05:52:23.637993+0000","last_became_peered":"2023-10-09T05:52:23.637993+0000","last_unstale":"2023-10-09T05:52:36.131922+0000","last_undegraded":"2023-10-09T05:52:36.131922+0000","last_fullsized":"2023-10-09T05:52:36.131922+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:35.191837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:52:13.234500+0000","last_change":"2023-10-09T05:52:11.209763+0000","last_active":"2023-10-09T05:52:13.234500+0000","last_peered":"2023-10-09T05:52:13.234500+0000","last_clean":"2023-10-09T05:52:13.234500+0000","last_became_active":"2023-10-09T05:52:11.209472+0000","last_became_peered":"2023-10-09T05:52:11.209472+0000","last_unstale":"2023-10-09T05:52:13.234500+0000","last_undegraded":"2023-10-09T05:52:13.234500+0000","last_fullsized":"2023-10-09T05:52:13.234500+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:05.160815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:52:13.236186+0000","last_change":"2023-10-09T05:52:11.234634+0000","last_active":"2023-10-09T05:52:13.236186+0000","last_peered":"2023-10-09T05:52:13.236186+0000","last_clean":"2023-10-09T05:52:13.236186+0000","last_became_active":"2023-10-09T05:52:11.234307+0000","last_became_peered":"2023-10-09T05:52:11.234307+0000","last_unstale":"2023-10-09T05:52:13.236186+0000","last_undegraded":"2023-10-09T05:52:13.236186+0000","last_fullsized":"2023-10-09T05:52:13.236186+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:37.064308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"138'6091","reported_seq":6146,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.223898+0000","last_change":"2023-10-09T05:50:39.778086+0000","last_active":"2023-10-09T05:52:43.223898+0000","last_peered":"2023-10-09T05:52:43.223898+0000","last_clean":"2023-10-09T05:52:43.223898+0000","last_became_active":"2023-10-09T05:50:39.777777+0000","last_became_peered":"2023-10-09T05:50:39.777777+0000","last_unstale":"2023-10-09T05:52:43.223898+0000","last_undegraded":"2023-10-09T05:52:43.223898+0000","last_fullsized":"2023-10-09T05:52:43.223898+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6091,"log_dups_size":0,"ondisk_log_size":6091,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:02.350198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382009344,"num_objects":5829,"num_object_clones":0,"num_object_copies":17487,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5829,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6091,"num_write_kb":381440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":113,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.223291+0000","last_change":"2023-10-09T05:48:49.141207+0000","last_active":"2023-10-09T05:52:43.223291+0000","last_peered":"2023-10-09T05:52:43.223291+0000","last_clean":"2023-10-09T05:52:43.223291+0000","last_became_active":"2023-10-09T05:48:49.140798+0000","last_became_peered":"2023-10-09T05:48:49.140798+0000","last_unstale":"2023-10-09T05:52:43.223291+0000","last_undegraded":"2023-10-09T05:52:43.223291+0000","last_fullsized":"2023-10-09T05:52:43.223291+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:44:13.125030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":528,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.224212+0000","last_change":"2023-10-09T05:51:57.111994+0000","last_active":"2023-10-09T05:52:43.224212+0000","last_peered":"2023-10-09T05:52:43.224212+0000","last_clean":"2023-10-09T05:52:43.224212+0000","last_became_active":"2023-10-09T05:51:57.111617+0000","last_became_peered":"2023-10-09T05:51:57.111617+0000","last_unstale":"2023-10-09T05:52:43.224212+0000","last_undegraded":"2023-10-09T05:52:43.224212+0000","last_fullsized":"2023-10-09T05:52:43.224212+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:51:12.346243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:51:12.346243+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:45.587863+0000","scrub_duration":0.0027096379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":36,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.226142+0000","last_change":"2023-10-09T05:52:23.648626+0000","last_active":"2023-10-09T05:52:43.226142+0000","last_peered":"2023-10-09T05:52:43.226142+0000","last_clean":"2023-10-09T05:52:43.226142+0000","last_became_active":"2023-10-09T05:52:23.646143+0000","last_became_peered":"2023-10-09T05:52:23.646143+0000","last_unstale":"2023-10-09T05:52:43.226142+0000","last_undegraded":"2023-10-09T05:52:43.226142+0000","last_fullsized":"2023-10-09T05:52:43.226142+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:51:04.733282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":91,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.224409+0000","last_change":"2023-10-09T05:40:14.363125+0000","last_active":"2023-10-09T05:52:43.224409+0000","last_peered":"2023-10-09T05:52:43.224409+0000","last_clean":"2023-10-09T05:52:43.224409+0000","last_became_active":"2023-10-09T05:40:14.362723+0000","last_became_peered":"2023-10-09T05:40:14.362723+0000","last_unstale":"2023-10-09T05:52:43.224409+0000","last_undegraded":"2023-10-09T05:52:43.224409+0000","last_fullsized":"2023-10-09T05:52:43.224409+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:30.741318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":422,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T05:52:14.275995+0000","last_change":"2023-10-09T05:52:14.275995+0000","last_active":"2023-10-09T05:52:14.275995+0000","last_peered":"2023-10-09T05:52:14.275995+0000","last_clean":"2023-10-09T05:52:14.275995+0000","last_became_active":"2023-10-09T05:52:14.275584+0000","last_became_peered":"2023-10-09T05:52:14.275584+0000","last_unstale":"2023-10-09T05:52:14.275995+0000","last_undegraded":"2023-10-09T05:52:14.275995+0000","last_fullsized":"2023-10-09T05:52:14.275995+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":167,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.224101+0000","last_change":"2023-10-09T05:52:11.209995+0000","last_active":"2023-10-09T05:52:43.224101+0000","last_peered":"2023-10-09T05:52:43.224101+0000","last_clean":"2023-10-09T05:52:43.224101+0000","last_became_active":"2023-10-09T05:52:11.209715+0000","last_became_peered":"2023-10-09T05:52:11.209715+0000","last_unstale":"2023-10-09T05:52:43.224101+0000","last_undegraded":"2023-10-09T05:52:43.224101+0000","last_fullsized":"2023-10-09T05:52:43.224101+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:13.217575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":87,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:52:42.205946+0000","last_change":"2023-10-09T05:52:23.643245+0000","last_active":"2023-10-09T05:52:42.205946+0000","last_peered":"2023-10-09T05:52:42.205946+0000","last_clean":"2023-10-09T05:52:42.205946+0000","last_became_active":"2023-10-09T05:52:23.642947+0000","last_became_peered":"2023-10-09T05:52:23.642947+0000","last_unstale":"2023-10-09T05:52:42.205946+0000","last_undegraded":"2023-10-09T05:52:42.205946+0000","last_fullsized":"2023-10-09T05:52:42.205946+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:52.260774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":145,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:52:36.134495+0000","last_change":"2023-10-09T05:52:29.734403+0000","last_active":"2023-10-09T05:52:36.134495+0000","last_peered":"2023-10-09T05:52:36.134495+0000","last_clean":"2023-10-09T05:52:36.134495+0000","last_became_active":"2023-10-09T05:52:29.734046+0000","last_became_peered":"2023-10-09T05:52:29.734046+0000","last_unstale":"2023-10-09T05:52:36.134495+0000","last_undegraded":"2023-10-09T05:52:36.134495+0000","last_fullsized":"2023-10-09T05:52:36.134495+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:53.346403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"137'6054","reported_seq":6118,"reported_epoch":137,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:19.308671+0000","last_change":"2023-10-09T05:52:11.492779+0000","last_active":"2023-10-09T05:52:19.308671+0000","last_peered":"2023-10-09T05:52:19.308671+0000","last_clean":"2023-10-09T05:51:24.557317+0000","last_became_active":"2023-10-09T05:52:11.482621+0000","last_became_peered":"2023-10-09T05:52:11.482621+0000","last_unstale":"2023-10-09T05:52:19.308671+0000","last_undegraded":"2023-10-09T05:52:11.263063+0000","last_fullsized":"2023-10-09T05:52:11.198072+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6054,"log_dups_size":0,"ondisk_log_size":6054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:15.893356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396754944,"num_objects":6054,"num_object_clones":0,"num_object_copies":18162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2109,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6054,"num_write_kb":387456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":3945},{"shards":"3,5,6","objects":2109}],"blocked_by":[],"up_primary":1,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":499,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.224824+0000","last_change":"2023-10-09T05:52:26.679997+0000","last_active":"2023-10-09T05:52:43.224824+0000","last_peered":"2023-10-09T05:52:43.224824+0000","last_clean":"2023-10-09T05:52:43.224824+0000","last_became_active":"2023-10-09T05:52:26.679641+0000","last_became_peered":"2023-10-09T05:52:26.679641+0000","last_unstale":"2023-10-09T05:52:43.224824+0000","last_undegraded":"2023-10-09T05:52:43.224824+0000","last_fullsized":"2023-10-09T05:52:43.224824+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":130,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.224320+0000","last_change":"2023-10-09T05:51:57.112837+0000","last_active":"2023-10-09T05:52:43.224320+0000","last_peered":"2023-10-09T05:52:43.224320+0000","last_clean":"2023-10-09T05:52:43.224320+0000","last_became_active":"2023-10-09T05:51:57.112133+0000","last_became_peered":"2023-10-09T05:51:57.112133+0000","last_unstale":"2023-10-09T05:52:43.224320+0000","last_undegraded":"2023-10-09T05:52:43.224320+0000","last_fullsized":"2023-10-09T05:52:43.224320+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:27:43.538434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:52:13.235434+0000","last_change":"2023-10-09T05:52:11.406954+0000","last_active":"2023-10-09T05:52:13.235434+0000","last_peered":"2023-10-09T05:52:13.235434+0000","last_clean":"2023-10-09T05:52:13.235434+0000","last_became_active":"2023-10-09T05:52:11.405333+0000","last_became_peered":"2023-10-09T05:52:11.405333+0000","last_unstale":"2023-10-09T05:52:13.235434+0000","last_undegraded":"2023-10-09T05:52:13.235434+0000","last_fullsized":"2023-10-09T05:52:13.235434+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:37.142582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":53,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:52:36.133975+0000","last_change":"2023-10-09T05:51:57.210829+0000","last_active":"2023-10-09T05:52:36.133975+0000","last_peered":"2023-10-09T05:52:36.133975+0000","last_clean":"2023-10-09T05:52:36.133975+0000","last_became_active":"2023-10-09T05:51:57.209398+0000","last_became_peered":"2023-10-09T05:51:57.209398+0000","last_unstale":"2023-10-09T05:52:36.133975+0000","last_undegraded":"2023-10-09T05:52:36.133975+0000","last_fullsized":"2023-10-09T05:52:36.133975+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:44.351726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"137'6109","reported_seq":6171,"reported_epoch":137,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:19.415918+0000","last_change":"2023-10-09T05:52:11.496179+0000","last_active":"2023-10-09T05:52:19.415918+0000","last_peered":"2023-10-09T05:52:19.415918+0000","last_clean":"2023-10-09T05:51:24.563560+0000","last_became_active":"2023-10-09T05:52:11.487841+0000","last_became_peered":"2023-10-09T05:52:11.487841+0000","last_unstale":"2023-10-09T05:52:19.415918+0000","last_undegraded":"2023-10-09T05:52:11.214602+0000","last_fullsized":"2023-10-09T05:52:11.201105+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6109,"log_dups_size":0,"ondisk_log_size":6109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:32.512460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399966208,"num_objects":6103,"num_object_clones":0,"num_object_copies":18309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2121,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6109,"num_write_kb":390784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":3982},{"shards":"2,6,7","objects":2121}],"blocked_by":[],"up_primary":1,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":473,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.229653+0000","last_change":"2023-10-09T05:52:26.676938+0000","last_active":"2023-10-09T05:52:43.229653+0000","last_peered":"2023-10-09T05:52:43.229653+0000","last_clean":"2023-10-09T05:52:43.229653+0000","last_became_active":"2023-10-09T05:52:26.676588+0000","last_became_peered":"2023-10-09T05:52:26.676588+0000","last_unstale":"2023-10-09T05:52:43.229653+0000","last_undegraded":"2023-10-09T05:52:43.229653+0000","last_fullsized":"2023-10-09T05:52:43.229653+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":126,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.144935+0000","last_change":"2023-10-09T05:51:57.112493+0000","last_active":"2023-10-09T05:52:43.144935+0000","last_peered":"2023-10-09T05:52:43.144935+0000","last_clean":"2023-10-09T05:52:43.144935+0000","last_became_active":"2023-10-09T05:51:57.112153+0000","last_became_peered":"2023-10-09T05:51:57.112153+0000","last_unstale":"2023-10-09T05:52:43.144935+0000","last_undegraded":"2023-10-09T05:52:43.144935+0000","last_fullsized":"2023-10-09T05:52:43.144935+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:29.925231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":14,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:52:13.236030+0000","last_change":"2023-10-09T05:52:11.415624+0000","last_active":"2023-10-09T05:52:13.236030+0000","last_peered":"2023-10-09T05:52:13.236030+0000","last_clean":"2023-10-09T05:52:13.236030+0000","last_became_active":"2023-10-09T05:52:11.415174+0000","last_became_peered":"2023-10-09T05:52:11.415174+0000","last_unstale":"2023-10-09T05:52:13.236030+0000","last_undegraded":"2023-10-09T05:52:13.236030+0000","last_fullsized":"2023-10-09T05:52:13.236030+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:13.657987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":118,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.224353+0000","last_change":"2023-10-09T05:52:23.639628+0000","last_active":"2023-10-09T05:52:43.224353+0000","last_peered":"2023-10-09T05:52:43.224353+0000","last_clean":"2023-10-09T05:52:43.224353+0000","last_became_active":"2023-10-09T05:52:23.639286+0000","last_became_peered":"2023-10-09T05:52:23.639286+0000","last_unstale":"2023-10-09T05:52:43.224353+0000","last_undegraded":"2023-10-09T05:52:43.224353+0000","last_fullsized":"2023-10-09T05:52:43.224353+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:21.450509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.2","version":"147'6207","reported_seq":9714,"reported_epoch":149,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:52:49.450899+0000","last_change":"2023-10-09T05:52:24.898238+0000","last_active":"2023-10-09T05:52:49.450899+0000","last_peered":"2023-10-09T05:52:49.450899+0000","last_clean":"2023-10-09T05:52:22.629746+0000","last_became_active":"2023-10-09T05:52:24.861945+0000","last_became_peered":"2023-10-09T05:52:24.861945+0000","last_unstale":"2023-10-09T05:52:49.450899+0000","last_undegraded":"2023-10-09T05:52:24.671738+0000","last_fullsized":"2023-10-09T05:52:24.632223+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6207,"log_dups_size":0,"ondisk_log_size":6207,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:37.391764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382140416,"num_objects":5831,"num_object_clones":0,"num_object_copies":17493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":4116,"num_objects_unfound":0,"num_objects_dirty":5831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6207,"num_write_kb":385216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1721,"num_bytes_recovered":112721920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"2,3,4","objects":4116},{"shards":"2,4","objects":1712}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":452,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.144176+0000","last_change":"2023-10-09T05:51:19.193833+0000","last_active":"2023-10-09T05:52:43.144176+0000","last_peered":"2023-10-09T05:52:43.144176+0000","last_clean":"2023-10-09T05:52:43.144176+0000","last_became_active":"2023-10-09T05:51:19.192996+0000","last_became_peered":"2023-10-09T05:51:19.192996+0000","last_unstale":"2023-10-09T05:52:43.144176+0000","last_undegraded":"2023-10-09T05:52:43.144176+0000","last_fullsized":"2023-10-09T05:52:43.144176+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":488690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":113,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.225351+0000","last_change":"2023-10-09T05:48:49.137565+0000","last_active":"2023-10-09T05:52:43.225351+0000","last_peered":"2023-10-09T05:52:43.225351+0000","last_clean":"2023-10-09T05:52:43.225351+0000","last_became_active":"2023-10-09T05:48:49.137099+0000","last_became_peered":"2023-10-09T05:48:49.137099+0000","last_unstale":"2023-10-09T05:52:43.225351+0000","last_undegraded":"2023-10-09T05:52:43.225351+0000","last_fullsized":"2023-10-09T05:52:43.225351+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:28.103821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":133,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:52:36.134356+0000","last_change":"2023-10-09T05:52:11.411584+0000","last_active":"2023-10-09T05:52:36.134356+0000","last_peered":"2023-10-09T05:52:36.134356+0000","last_clean":"2023-10-09T05:52:36.134356+0000","last_became_active":"2023-10-09T05:52:11.402696+0000","last_became_peered":"2023-10-09T05:52:11.402696+0000","last_unstale":"2023-10-09T05:52:36.134356+0000","last_undegraded":"2023-10-09T05:52:36.134356+0000","last_fullsized":"2023-10-09T05:52:36.134356+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:22.747118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":91,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.229351+0000","last_change":"2023-10-09T05:40:12.676587+0000","last_active":"2023-10-09T05:52:43.229351+0000","last_peered":"2023-10-09T05:52:43.229351+0000","last_clean":"2023-10-09T05:52:43.229351+0000","last_became_active":"2023-10-09T05:40:12.676330+0000","last_became_peered":"2023-10-09T05:40:12.676330+0000","last_unstale":"2023-10-09T05:52:43.229351+0000","last_undegraded":"2023-10-09T05:52:43.229351+0000","last_fullsized":"2023-10-09T05:52:43.229351+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:46.659556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"147'6230","reported_seq":6285,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.223809+0000","last_change":"2023-10-09T05:50:40.123172+0000","last_active":"2023-10-09T05:52:43.223809+0000","last_peered":"2023-10-09T05:52:43.223809+0000","last_clean":"2023-10-09T05:52:43.223809+0000","last_became_active":"2023-10-09T05:50:40.122867+0000","last_became_peered":"2023-10-09T05:50:40.122867+0000","last_unstale":"2023-10-09T05:52:43.223809+0000","last_undegraded":"2023-10-09T05:52:43.223809+0000","last_fullsized":"2023-10-09T05:52:43.223809+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6230,"log_dups_size":0,"ondisk_log_size":6230,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:16.038279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388890624,"num_objects":5934,"num_object_clones":0,"num_object_copies":17802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6230,"num_write_kb":389248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":57,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:52:36.133732+0000","last_change":"2023-10-09T05:51:57.209458+0000","last_active":"2023-10-09T05:52:36.133732+0000","last_peered":"2023-10-09T05:52:36.133732+0000","last_clean":"2023-10-09T05:52:36.133732+0000","last_became_active":"2023-10-09T05:51:57.208070+0000","last_became_peered":"2023-10-09T05:51:57.208070+0000","last_unstale":"2023-10-09T05:52:36.133732+0000","last_undegraded":"2023-10-09T05:52:36.133732+0000","last_fullsized":"2023-10-09T05:52:36.133732+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:30.631238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":152,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T05:52:13.229824+0000","last_change":"2023-10-09T05:52:11.219426+0000","last_active":"2023-10-09T05:52:13.229824+0000","last_peered":"2023-10-09T05:52:13.229824+0000","last_clean":"2023-10-09T05:52:13.229824+0000","last_became_active":"2023-10-09T05:52:11.218976+0000","last_became_peered":"2023-10-09T05:52:11.218976+0000","last_unstale":"2023-10-09T05:52:13.229824+0000","last_undegraded":"2023-10-09T05:52:13.229824+0000","last_fullsized":"2023-10-09T05:52:13.229824+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:32:05.636670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":139,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.224438+0000","last_change":"2023-10-09T05:52:11.422464+0000","last_active":"2023-10-09T05:52:43.224438+0000","last_peered":"2023-10-09T05:52:43.224438+0000","last_clean":"2023-10-09T05:52:43.224438+0000","last_became_active":"2023-10-09T05:52:11.341825+0000","last_became_peered":"2023-10-09T05:52:11.341825+0000","last_unstale":"2023-10-09T05:52:43.224438+0000","last_undegraded":"2023-10-09T05:52:43.224438+0000","last_fullsized":"2023-10-09T05:52:43.224438+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:48.933107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":145,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.229766+0000","last_change":"2023-10-09T05:52:11.425354+0000","last_active":"2023-10-09T05:52:43.229766+0000","last_peered":"2023-10-09T05:52:43.229766+0000","last_clean":"2023-10-09T05:52:43.229766+0000","last_became_active":"2023-10-09T05:52:11.425006+0000","last_became_peered":"2023-10-09T05:52:11.425006+0000","last_unstale":"2023-10-09T05:52:43.229766+0000","last_undegraded":"2023-10-09T05:52:43.229766+0000","last_fullsized":"2023-10-09T05:52:43.229766+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:07.273992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":94,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.225373+0000","last_change":"2023-10-09T05:40:12.633195+0000","last_active":"2023-10-09T05:52:43.225373+0000","last_peered":"2023-10-09T05:52:43.225373+0000","last_clean":"2023-10-09T05:52:43.225373+0000","last_became_active":"2023-10-09T05:40:12.632942+0000","last_became_peered":"2023-10-09T05:40:12.632942+0000","last_unstale":"2023-10-09T05:52:43.225373+0000","last_undegraded":"2023-10-09T05:52:43.225373+0000","last_fullsized":"2023-10-09T05:52:43.225373+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:04.470685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":142,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.229553+0000","last_change":"2023-10-09T05:51:57.093104+0000","last_active":"2023-10-09T05:52:43.229553+0000","last_peered":"2023-10-09T05:52:43.229553+0000","last_clean":"2023-10-09T05:52:43.229553+0000","last_became_active":"2023-10-09T05:51:57.092810+0000","last_became_peered":"2023-10-09T05:51:57.092810+0000","last_unstale":"2023-10-09T05:52:43.229553+0000","last_undegraded":"2023-10-09T05:52:43.229553+0000","last_fullsized":"2023-10-09T05:52:43.229553+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:51.192227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":27,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.225017+0000","last_change":"2023-10-09T05:52:29.736624+0000","last_active":"2023-10-09T05:52:43.225017+0000","last_peered":"2023-10-09T05:52:43.225017+0000","last_clean":"2023-10-09T05:52:43.225017+0000","last_became_active":"2023-10-09T05:52:29.736263+0000","last_became_peered":"2023-10-09T05:52:29.736263+0000","last_unstale":"2023-10-09T05:52:43.225017+0000","last_undegraded":"2023-10-09T05:52:43.225017+0000","last_fullsized":"2023-10-09T05:52:43.225017+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:23:56.032785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":136,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:52:42.262355+0000","last_change":"2023-10-09T05:52:42.261890+0000","last_active":"2023-10-09T05:52:42.262355+0000","last_peered":"2023-10-09T05:52:42.262355+0000","last_clean":"2023-10-09T05:52:42.262355+0000","last_became_active":"2023-10-09T05:52:42.261521+0000","last_became_peered":"2023-10-09T05:52:42.261521+0000","last_unstale":"2023-10-09T05:52:42.262355+0000","last_undegraded":"2023-10-09T05:52:42.262355+0000","last_fullsized":"2023-10-09T05:52:42.262355+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:59.208145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":97,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T05:52:36.133810+0000","last_change":"2023-10-09T05:52:23.654749+0000","last_active":"2023-10-09T05:52:36.133810+0000","last_peered":"2023-10-09T05:52:36.133810+0000","last_clean":"2023-10-09T05:52:36.133810+0000","last_became_active":"2023-10-09T05:52:23.643044+0000","last_became_peered":"2023-10-09T05:52:23.643044+0000","last_unstale":"2023-10-09T05:52:36.133810+0000","last_undegraded":"2023-10-09T05:52:36.133810+0000","last_fullsized":"2023-10-09T05:52:36.133810+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:23:08.898078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":128,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T05:52:43.229311+0000","last_change":"2023-10-09T05:52:11.411838+0000","last_active":"2023-10-09T05:52:43.229311+0000","last_peered":"2023-10-09T05:52:43.229311+0000","last_clean":"2023-10-09T05:52:43.229311+0000","last_became_active":"2023-10-09T05:52:11.411506+0000","last_became_peered":"2023-10-09T05:52:11.411506+0000","last_unstale":"2023-10-09T05:52:43.229311+0000","last_undegraded":"2023-10-09T05:52:43.229311+0000","last_fullsized":"2023-10-09T05:52:43.229311+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:55.135008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":6180962328,"num_objects":94315,"num_object_clones":0,"num_object_copies":282945,"num_objects_missing_on_primary":1173,"num_objects_missing":1173,"num_objects_degraded":33220,"num_objects_misplaced":8074,"num_objects_unfound":0,"num_objects_dirty":94315,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":98767,"num_write_kb":6178561,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9299,"num_bytes_recovered":609222656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18869137408,"data_stored":18869125192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":98767,"ondisk_log_size":98767,"up":48,"acting":37,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":536699,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":444,"num_read_kb":832,"num_write":218,"num_write_kb":2197,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2200463,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234159,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2812500,"kb_used_data":2511116,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":90932652,"statfs":{"total":95995035648,"available":93115035648,"internally_reserved":0,"allocated":2571382784,"data_stored":2570814654,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:51:11 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.629,"15min":0.73999999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":0.77100000000000002,"5min":4.8940000000000001,"15min":4.8940000000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.57599999999999996,"15min":0.59999999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.915,"5min":1.915,"15min":1.915},"last":0.39000000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:52:18 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":0.77900000000000003,"15min":0.60899999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.9859999999999998,"5min":7.9859999999999998,"15min":7.9859999999999998},"last":0.88300000000000001},{"interface":"front","average":{"1min":1.534,"5min":0.78700000000000003,"15min":0.63},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":5.8129999999999997,"5min":5.8129999999999997,"15min":5.8129999999999997},"last":0.374}]},{"osd":2,"last update":"Mon Oct 9 05:51:55 2023","interfaces":[{"interface":"back","average":{"1min":4.0700000000000003,"5min":1.4099999999999999,"15min":0.81999999999999995},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":50.57,"5min":50.57,"15min":50.57},"last":0.47599999999999998},{"interface":"front","average":{"1min":4.7329999999999997,"5min":1.6060000000000001,"15min":0.94499999999999995},"min":{"1min":0.38700000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":50.536999999999999,"5min":50.536999999999999,"15min":50.536999999999999},"last":0.80800000000000005}]},{"osd":3,"last update":"Mon Oct 9 05:52:24 2023","interfaces":[{"interface":"back","average":{"1min":15.843,"5min":5.6440000000000001,"15min":2.2919999999999998},"min":{"1min":0.41599999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":147.54900000000001,"5min":147.54900000000001,"15min":147.54900000000001},"last":0.82599999999999996},{"interface":"front","average":{"1min":15.071,"5min":5.4699999999999998,"15min":2.214},"min":{"1min":0.40400000000000003,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":144.065,"5min":144.065,"15min":144.065},"last":0.56499999999999995}]},{"osd":4,"last update":"Mon Oct 9 05:52:01 2023","interfaces":[{"interface":"back","average":{"1min":1.522,"5min":0.71199999999999997,"15min":0.55600000000000005},"min":{"1min":0.27000000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":9.5329999999999995,"5min":9.5329999999999995,"15min":9.5329999999999995},"last":0.77700000000000002},{"interface":"front","average":{"1min":2.339,"5min":0.89200000000000002,"15min":0.624},"min":{"1min":0.29099999999999998,"5min":0.27400000000000002,"15min":0.27100000000000002},"max":{"1min":7.6120000000000001,"5min":7.6120000000000001,"15min":7.6120000000000001},"last":0.50800000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:52:01 2023","interfaces":[{"interface":"back","average":{"1min":2.2160000000000002,"5min":0.84499999999999997,"15min":0.61399999999999999},"min":{"1min":0.33500000000000002,"5min":0.29499999999999998,"15min":0.248},"max":{"1min":10.795999999999999,"5min":10.795999999999999,"15min":10.795999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":1.103,"5min":0.69099999999999995,"15min":0.58299999999999996},"min":{"1min":0.307,"5min":0.307,"15min":0.28100000000000003},"max":{"1min":3.2429999999999999,"5min":3.2429999999999999,"15min":3.2429999999999999},"last":0.96299999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:52:01 2023","interfaces":[{"interface":"back","average":{"1min":9.9489999999999998,"5min":2.4449999999999998,"15min":1.171},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.29599999999999999},"max":{"1min":131.94399999999999,"5min":131.94399999999999,"15min":131.94399999999999},"last":0.44800000000000001},{"interface":"front","average":{"1min":10.265000000000001,"5min":2.5339999999999998,"15min":1.204},"min":{"1min":0.41799999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":135.23400000000001,"5min":135.23400000000001,"15min":135.23400000000001},"last":0.63}]}]},{"osd":6,"up_from":46,"seq":197568495795,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2700924,"kb_used_data":2395444,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":91044228,"statfs":{"total":95995035648,"available":93229289472,"internally_reserved":0,"allocated":2452934656,"data_stored":2452382915,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:51:12 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":0.62,"15min":0.60599999999999998},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.7930000000000001,"5min":2.7930000000000001,"15min":2.7930000000000001},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.73499999999999999,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.823,"5min":2.823,"15min":2.823},"last":0.41599999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:52:30 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":0.75800000000000001,"15min":0.64800000000000002},"min":{"1min":0.25800000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.7229999999999999,"5min":3.7949999999999999,"15min":3.7949999999999999},"last":1.609},{"interface":"front","average":{"1min":1.863,"5min":0.88800000000000001,"15min":0.68899999999999995},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":9.9670000000000005,"5min":9.9670000000000005,"15min":9.9670000000000005},"last":0.73299999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:51:54 2023","interfaces":[{"interface":"back","average":{"1min":1.6679999999999999,"5min":0.76100000000000001,"15min":0.61499999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":5.4459999999999997,"5min":5.4459999999999997,"15min":5.4459999999999997},"last":0.33100000000000002},{"interface":"front","average":{"1min":1.403,"5min":0.72399999999999998,"15min":0.61299999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":9.9390000000000001,"5min":9.9390000000000001,"15min":9.9390000000000001},"last":0.52300000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:52:30 2023","interfaces":[{"interface":"back","average":{"1min":2.6779999999999999,"5min":1.0760000000000001,"15min":0.72799999999999998},"min":{"1min":0.33200000000000002,"5min":0.311,"15min":0.311},"max":{"1min":10.19,"5min":10.19,"15min":10.19},"last":0.36699999999999999},{"interface":"front","average":{"1min":3.0270000000000001,"5min":1.087,"15min":0.747},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":11.914,"5min":11.914,"15min":11.914},"last":0.45700000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:52:40 2023","interfaces":[{"interface":"back","average":{"1min":2.1139999999999999,"5min":0.93799999999999994,"15min":0.61499999999999999},"min":{"1min":0.28000000000000003,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.9100000000000001,"5min":7.9100000000000001,"15min":7.9100000000000001},"last":0.38200000000000001},{"interface":"front","average":{"1min":1.8440000000000001,"5min":0.83799999999999997,"15min":0.60799999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.3300000000000001,"5min":7.3300000000000001,"15min":7.3300000000000001},"last":1.4450000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:52:40 2023","interfaces":[{"interface":"back","average":{"1min":2.3959999999999999,"5min":0.91300000000000003,"15min":0.62},"min":{"1min":0.30199999999999999,"5min":0.29799999999999999,"15min":0.27700000000000002},"max":{"1min":13.879,"5min":13.879,"15min":13.879},"last":0.495},{"interface":"front","average":{"1min":2.0129999999999999,"5min":0.97899999999999998,"15min":0.67300000000000004},"min":{"1min":0.32100000000000001,"5min":0.307,"15min":0.307},"max":{"1min":7.29,"5min":7.29,"15min":7.29},"last":0.40500000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:52:02 2023","interfaces":[{"interface":"back","average":{"1min":3.7269999999999999,"5min":1.1870000000000001,"15min":0.751},"min":{"1min":0.34300000000000003,"5min":0.316,"15min":0.316},"max":{"1min":19.914999999999999,"5min":19.914999999999999,"15min":19.914999999999999},"last":5.4969999999999999},{"interface":"front","average":{"1min":2.5990000000000002,"5min":1.0489999999999999,"15min":0.746},"min":{"1min":0.42199999999999999,"5min":0.39600000000000002,"15min":0.35799999999999998},"max":{"1min":7.0309999999999997,"5min":7.0309999999999997,"15min":7.0309999999999997},"last":5.5099999999999998}]}]},{"osd":1,"up_from":89,"seq":382252089395,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1433392,"kb_used_data":1365736,"kb_used_omap":9,"kb_used_meta":67638,"kb_avail":92311760,"statfs":{"total":95995035648,"available":94527242240,"internally_reserved":0,"allocated":1398513664,"data_stored":1397841570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":69262313},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:51:19 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.42299999999999999,"15min":0.38100000000000001},"min":{"1min":0.217,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.391,"5min":3.391,"15min":3.391},"last":0.23499999999999999},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.438,"15min":0.41599999999999998},"min":{"1min":0.314,"5min":0.253,"15min":0.253},"max":{"1min":1.248,"5min":1.248,"15min":1.248},"last":0.29399999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:51:58 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":0.68200000000000005,"15min":0.64700000000000002},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":5.0789999999999997,"5min":5.0789999999999997,"15min":5.0789999999999997},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.91500000000000004,"5min":0.58099999999999996,"15min":0.54700000000000004},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":3.7690000000000001,"5min":3.7690000000000001,"15min":3.7690000000000001},"last":0.88300000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:51:23 2023","interfaces":[{"interface":"back","average":{"1min":12.706,"5min":2.863,"15min":1.2230000000000001},"min":{"1min":0.27400000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":284.06099999999998,"5min":284.06099999999998,"15min":284.06099999999998},"last":0.46899999999999997},{"interface":"front","average":{"1min":12.707000000000001,"5min":2.867,"15min":1.2270000000000001},"min":{"1min":0.317,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":284.97300000000001,"5min":284.97300000000001,"15min":284.97300000000001},"last":0.54100000000000004}]},{"osd":4,"last update":"Mon Oct 9 05:51:19 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.67200000000000004,"15min":0.61799999999999999},"min":{"1min":0.34300000000000003,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":4.1580000000000004,"5min":4.1580000000000004,"15min":4.1580000000000004},"last":1.5580000000000001},{"interface":"front","average":{"1min":0.754,"5min":0.63800000000000001,"15min":0.62},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":3.4220000000000002,"5min":3.4220000000000002,"15min":3.4220000000000002},"last":0.33100000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:51:19 2023","interfaces":[{"interface":"back","average":{"1min":0.77000000000000002,"5min":0.63200000000000001,"15min":0.60799999999999998},"min":{"1min":0.438,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.9419999999999999,"5min":1.9419999999999999,"15min":1.9419999999999999},"last":3.4079999999999999},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.68300000000000005,"15min":0.63400000000000001},"min":{"1min":0.38100000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.2859999999999996,"5min":4.2859999999999996,"15min":4.2859999999999996},"last":0.94399999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:51:19 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.64400000000000002,"15min":0.629},"min":{"1min":0.40100000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.9059999999999999,"5min":1.9059999999999999,"15min":1.9059999999999999},"last":0.499},{"interface":"front","average":{"1min":0.98599999999999999,"5min":0.70399999999999996,"15min":0.65800000000000003},"min":{"1min":0.41199999999999998,"5min":0.371,"15min":0.371},"max":{"1min":4.2539999999999996,"5min":4.2539999999999996,"15min":4.2539999999999996},"last":1.6539999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:51:19 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":0.73099999999999998,"15min":0.68799999999999994},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":4.21,"5min":4.21,"15min":4.21},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.67700000000000005,"15min":0.65100000000000002},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":3.4500000000000002,"5min":3.4500000000000002,"15min":3.4500000000000002},"last":0.60999999999999999}]}]},{"osd":0,"up_from":129,"seq":554050781192,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1350228,"kb_used_data":1306700,"kb_used_omap":29,"kb_used_meta":43490,"kb_avail":92394924,"statfs":{"total":95995035648,"available":94612402176,"internally_reserved":0,"allocated":1338060800,"data_stored":1337316513,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30322,"internal_metadata":44534158},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45100000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30199999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33400000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44900000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60899999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57399999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55200000000000005}]}]},{"osd":2,"up_from":94,"seq":403726925874,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3298164,"kb_used_data":3186092,"kb_used_omap":81,"kb_used_meta":111982,"kb_avail":90446988,"statfs":{"total":95995035648,"available":92617715712,"internally_reserved":0,"allocated":3262558208,"data_stored":3261840141,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":114669786},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.33600000000000002,"5min":0.39800000000000002,"15min":0.40799999999999997},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.55300000000000005,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.34799999999999998,"5min":0.40200000000000002,"15min":0.41099999999999998},"min":{"1min":0.247,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.497,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.30099999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:51:53 2023","interfaces":[{"interface":"back","average":{"1min":9.6929999999999996,"5min":2.2679999999999998,"15min":1.0429999999999999},"min":{"1min":0.25900000000000001,"5min":0.253,"15min":0.253},"max":{"1min":163.24199999999999,"5min":163.24199999999999,"15min":163.24199999999999},"last":0.85299999999999998},{"interface":"front","average":{"1min":9.6519999999999992,"5min":2.246,"15min":1.0209999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":163.27000000000001,"5min":163.27000000000001,"15min":163.27000000000001},"last":0.751}]},{"osd":3,"last update":"Mon Oct 9 05:52:24 2023","interfaces":[{"interface":"back","average":{"1min":2.1640000000000001,"5min":2.855,"15min":1.22},"min":{"1min":0.249,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":8.8000000000000007,"5min":170.07599999999999,"15min":170.07599999999999},"last":0.441},{"interface":"front","average":{"1min":2.302,"5min":1.5429999999999999,"15min":0.80900000000000005},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":11.596,"5min":63.600000000000001,"15min":63.600000000000001},"last":0.36199999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:51:53 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":0.80500000000000005,"15min":0.71899999999999997},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.9329999999999998,"5min":3.9329999999999998,"15min":3.9329999999999998},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.522,"5min":0.85799999999999998,"15min":0.76000000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.5709999999999997,"5min":6.5709999999999997,"15min":6.5709999999999997},"last":0.47599999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:51:53 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":0.73499999999999999,"15min":0.69699999999999995},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":2.8460000000000001,"5min":2.8460000000000001,"15min":2.8460000000000001},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.4570000000000001,"5min":0.81299999999999994,"15min":0.71799999999999997},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.9809999999999999,"5min":3.9809999999999999,"15min":3.9809999999999999},"last":0.434}]},{"osd":6,"last update":"Mon Oct 9 05:51:53 2023","interfaces":[{"interface":"back","average":{"1min":1.375,"5min":0.79100000000000004,"15min":0.70799999999999996},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.5289999999999999,"5min":6.5289999999999999,"15min":6.5289999999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.0720000000000001,"5min":0.75700000000000001,"15min":0.71299999999999997},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":4.1420000000000003,"5min":4.1420000000000003,"15min":4.1420000000000003},"last":0.60399999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:51:53 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":0.753,"15min":0.69899999999999995},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.645,"5min":2.645,"15min":2.645},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.446,"5min":0.84199999999999997,"15min":0.752},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":4.0199999999999996,"5min":4.0199999999999996,"15min":4.0199999999999996},"last":0.45300000000000001}]}]},{"osd":3,"up_from":99,"seq":425201762348,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3857308,"kb_used_data":3757652,"kb_used_omap":7,"kb_used_meta":99640,"kb_avail":89887844,"statfs":{"total":95995035648,"available":92045152256,"internally_reserved":0,"allocated":3847835648,"data_stored":3847086136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":102031976},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:51:23 2023","interfaces":[{"interface":"back","average":{"1min":9.6880000000000006,"5min":2.2210000000000001,"15min":0.97699999999999998},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":191.352,"5min":191.352,"15min":191.352},"last":0.373},{"interface":"front","average":{"1min":9.5329999999999995,"5min":2.181,"15min":0.95599999999999996},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":191.405,"5min":191.405,"15min":191.405},"last":0.33800000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:52:23 2023","interfaces":[{"interface":"back","average":{"1min":2.2109999999999999,"5min":1.9510000000000001,"15min":0.89700000000000002},"min":{"1min":0.253,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":9.2579999999999991,"5min":127.226,"15min":127.226},"last":0.502},{"interface":"front","average":{"1min":2.2959999999999998,"5min":2.0470000000000002,"15min":0.94699999999999995},"min":{"1min":0.26300000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":15.680999999999999,"5min":127.468,"15min":127.468},"last":0.48499999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:52:23 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":1.282,"15min":0.67800000000000005},"min":{"1min":0.26600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.0350000000000001,"5min":62.938000000000002,"15min":62.938000000000002},"last":0.32500000000000001},{"interface":"front","average":{"1min":2.0369999999999999,"5min":1.5629999999999999,"15min":0.80900000000000005},"min":{"1min":0.32500000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":15.821999999999999,"5min":62.43,"15min":62.43},"last":0.40500000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:52:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8080000000000001,"5min":0.93400000000000005,"15min":0.71199999999999997},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.7439999999999998,"5min":7.7439999999999998,"15min":7.7439999999999998},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.5389999999999999,"5min":0.80700000000000005,"15min":0.68000000000000005},"min":{"1min":0.32100000000000001,"5min":0.311,"15min":0.311},"max":{"1min":6.2000000000000002,"5min":6.2000000000000002,"15min":6.2000000000000002},"last":0.46800000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:52:23 2023","interfaces":[{"interface":"back","average":{"1min":1.623,"5min":0.83199999999999996,"15min":0.67900000000000005},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":5.5860000000000003,"5min":5.5860000000000003,"15min":5.5860000000000003},"last":0.505},{"interface":"front","average":{"1min":1.3779999999999999,"5min":0.81100000000000005,"15min":0.69499999999999995},"min":{"1min":0.34300000000000003,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.2839999999999998,"5min":4.2839999999999998,"15min":4.2839999999999998},"last":0.58199999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:52:23 2023","interfaces":[{"interface":"back","average":{"1min":2.1339999999999999,"5min":0.94599999999999995,"15min":0.73699999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":12.798999999999999,"5min":12.798999999999999,"15min":12.798999999999999},"last":0.53700000000000003},{"interface":"front","average":{"1min":2.0609999999999999,"5min":0.98099999999999998,"15min":0.78000000000000003},"min":{"1min":0.39900000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":12.773,"5min":12.773,"15min":12.773},"last":0.51700000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:52:23 2023","interfaces":[{"interface":"back","average":{"1min":1.954,"5min":0.95799999999999996,"15min":0.73899999999999999},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":7.4580000000000002,"5min":7.4580000000000002,"15min":7.4580000000000002},"last":0.55200000000000005},{"interface":"front","average":{"1min":1.841,"5min":0.96199999999999997,"15min":0.75800000000000001},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":7.8399999999999999,"5min":7.8399999999999999,"15min":7.8399999999999999},"last":0.56799999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019068,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2176340,"kb_used_data":1879052,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":91568812,"statfs":{"total":95995035648,"available":93766463488,"internally_reserved":0,"allocated":1924149248,"data_stored":1923581371,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:51:11 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.56399999999999995,"15min":0.59399999999999997},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":2.2109999999999999,"5min":2.2109999999999999,"15min":2.2109999999999999},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.61499999999999999,"15min":0.63700000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":2.6219999999999999,"5min":2.6219999999999999,"15min":2.6219999999999999},"last":0.45900000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:52:27 2023","interfaces":[{"interface":"back","average":{"1min":2.4769999999999999,"5min":0.99199999999999999,"15min":0.70799999999999996},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":9.2390000000000008,"5min":9.2390000000000008,"15min":9.2390000000000008},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.837,"5min":0.85299999999999998,"15min":0.629},"min":{"1min":0.29699999999999999,"5min":0.255,"15min":0.255},"max":{"1min":8.2629999999999999,"5min":8.2629999999999999,"15min":8.2629999999999999},"last":0.621}]},{"osd":2,"last update":"Mon Oct 9 05:51:54 2023","interfaces":[{"interface":"back","average":{"1min":1.7529999999999999,"5min":1.1080000000000001,"15min":0.754},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.9240000000000004,"5min":33.262,"15min":33.262},"last":0.41899999999999998},{"interface":"front","average":{"1min":2.0409999999999999,"5min":1.2370000000000001,"15min":0.86699999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":9.0030000000000001,"5min":33.240000000000002,"15min":33.240000000000002},"last":0.38500000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.883,"5min":1.3859999999999999,"15min":0.85399999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.3490000000000002,"5min":40.531999999999996,"15min":40.531999999999996},"last":0.55300000000000005},{"interface":"front","average":{"1min":2.2170000000000001,"5min":1.506,"15min":0.90200000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":9.9779999999999998,"5min":39.905999999999999,"15min":39.905999999999999},"last":0.52300000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:52:16 2023","interfaces":[{"interface":"back","average":{"1min":2.0289999999999999,"5min":0.80500000000000005,"15min":0.57699999999999996},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.012,"5min":10.012,"15min":10.012},"last":0.54200000000000004},{"interface":"front","average":{"1min":2.7069999999999999,"5min":0.99099999999999999,"15min":0.66000000000000003},"min":{"1min":0.30499999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":12.489000000000001,"5min":12.489000000000001,"15min":12.489000000000001},"last":0.34599999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:52:34 2023","interfaces":[{"interface":"back","average":{"1min":2.3450000000000002,"5min":0.95599999999999996,"15min":0.65500000000000003},"min":{"1min":0.35299999999999998,"5min":0.29799999999999999,"15min":0.27400000000000002},"max":{"1min":11.234999999999999,"5min":11.234999999999999,"15min":11.234999999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":1.4570000000000001,"5min":0.73999999999999999,"15min":0.60599999999999998},"min":{"1min":0.314,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":9.923,"5min":9.923,"15min":9.923},"last":0.504}]},{"osd":7,"last update":"Mon Oct 9 05:52:03 2023","interfaces":[{"interface":"back","average":{"1min":2.516,"5min":1.0029999999999999,"15min":0.70999999999999996},"min":{"1min":0.32900000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.9619999999999997,"5min":8.9619999999999997,"15min":8.9619999999999997},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.7549999999999999,"5min":0.82299999999999995,"15min":0.66400000000000003},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":8.3819999999999997,"5min":8.3819999999999997,"15min":8.3819999999999997},"last":0.68100000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757431,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2359488,"kb_used_data":2049912,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":91385664,"statfs":{"total":95995035648,"available":93578919936,"internally_reserved":0,"allocated":2099109888,"data_stored":2098550473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:51:11 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.55500000000000005,"15min":0.58399999999999996},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.84799999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.58099999999999996,"15min":0.61099999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.427,"5min":1.427,"15min":1.427},"last":0.46300000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:52:24 2023","interfaces":[{"interface":"back","average":{"1min":2.0670000000000002,"5min":1.913,"15min":0.995},"min":{"1min":0.27000000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.069,"5min":114.247,"15min":114.247},"last":0.59399999999999997},{"interface":"front","average":{"1min":1.4950000000000001,"5min":1.8600000000000001,"15min":0.995},"min":{"1min":0.30399999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.9800000000000004,"5min":114.678,"15min":114.678},"last":0.34399999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:51:50 2023","interfaces":[{"interface":"back","average":{"1min":1.6240000000000001,"5min":0.78400000000000003,"15min":0.65200000000000002},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":6.0469999999999997,"5min":6.0469999999999997,"15min":6.0469999999999997},"last":0.55100000000000005},{"interface":"front","average":{"1min":1.5009999999999999,"5min":0.76800000000000002,"15min":0.64800000000000002},"min":{"1min":0.40100000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":10.161,"5min":10.161,"15min":10.161},"last":0.38400000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:52:24 2023","interfaces":[{"interface":"back","average":{"1min":19.015000000000001,"5min":6.7089999999999996,"15min":2.597},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":174.22200000000001,"5min":272.81299999999999,"15min":272.81299999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":19.501999999999999,"5min":6.944,"15min":2.7429999999999999},"min":{"1min":0.36799999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":174.339,"5min":273.00999999999999,"15min":273.00999999999999},"last":1.262}]},{"osd":4,"last update":"Mon Oct 9 05:52:12 2023","interfaces":[{"interface":"back","average":{"1min":1.6220000000000001,"5min":0.72699999999999998,"15min":0.54900000000000004},"min":{"1min":0.27500000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.2050000000000001,"5min":6.2050000000000001,"15min":6.2050000000000001},"last":0.34599999999999997},{"interface":"front","average":{"1min":1.5149999999999999,"5min":0.71299999999999997,"15min":0.56100000000000005},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":13.050000000000001,"5min":13.050000000000001,"15min":13.050000000000001},"last":0.39700000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:52:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":0.73299999999999998,"15min":0.57099999999999995},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":10.606999999999999,"5min":10.606999999999999,"15min":10.606999999999999},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.891,"5min":0.878,"15min":0.64600000000000002},"min":{"1min":0.30299999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.9770000000000003,"5min":5.9770000000000003,"15min":5.9770000000000003},"last":0.58999999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:52:07 2023","interfaces":[{"interface":"back","average":{"1min":1.954,"5min":0.874,"15min":0.66400000000000003},"min":{"1min":0.36899999999999999,"5min":0.32700000000000001,"15min":0.315},"max":{"1min":6.1600000000000001,"5min":6.1600000000000001,"15min":6.1600000000000001},"last":0.61299999999999999},{"interface":"front","average":{"1min":1.218,"5min":0.73199999999999998,"15min":0.63500000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":5.032,"5min":5.032,"15min":5.032},"last":0.496}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1474,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1336348672,"data_stored":1336344600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1383403520,"data_stored":1383399448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3260612608,"data_stored":3260612608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3845521408,"data_stored":3845521408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1922629632,"data_stored":1922629632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2097025024,"data_stored":2097020952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2453733376,"data_stored":2453733376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2569863168,"data_stored":2569863168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:52:52.734 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 11 2023-10-09T05:52:52.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:52.736 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:52:52.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:52:52.836 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:52.864 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:52.883 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:52:52.914 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:52.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:52.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:52.948 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:53.019 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:53.054 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:53.105 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:53.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:53.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:53.208 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:53.236 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:53.285 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:53.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:53.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:53.372 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:53 smithi064 ceph-mon[212750]: from='client.34845 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:52:53.372 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:53 smithi064 ceph-mon[212750]: osdmap e150: 8 total, 8 up, 8 in 2023-10-09T05:52:53.372 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:53 smithi064 ceph-mon[212750]: failed to encode map e150 with expected crc 2023-10-09T05:52:53.372 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:53 smithi064 ceph-mon[212750]: failed to encode map e150 with expected crc 2023-10-09T05:52:53.372 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:53 smithi064 ceph-mon[220044]: from='client.34845 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:52:53.372 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:53 smithi064 ceph-mon[220044]: osdmap e150: 8 total, 8 up, 8 in 2023-10-09T05:52:53.373 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:53 smithi064 ceph-mon[220044]: failed to encode map e150 with expected crc 2023-10-09T05:52:53.373 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:53 smithi064 ceph-mon[220044]: failed to encode map e150 with expected crc 2023-10-09T05:52:53.398 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:53.432 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:53.538 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:53.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:53.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:53.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:53 smithi156 ceph-mon[195101]: from='client.34845 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:52:53.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:53 smithi156 ceph-mon[195101]: osdmap e150: 8 total, 8 up, 8 in 2023-10-09T05:52:53.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:53 smithi156 ceph-mon[195101]: failed to encode map e150 with expected crc 2023-10-09T05:52:53.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:53 smithi156 ceph-mon[195101]: failed to encode map e150 with expected crc 2023-10-09T05:52:53.644 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:53.674 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:53.721 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:53.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:53.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:53.825 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:53.851 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:53.900 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:53.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:53.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:53.942 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:52:53.963 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:52:53.985 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:52:54.009 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:54.012 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:52:54.033 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:52:54.038 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:54.053 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:52:54.077 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:52:54.079 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:52:54.079 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:52:54.079 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:52:54.079 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:52:54.079 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:52:54.079 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:52:54.079 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:52:54.080 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:52:54.090 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:54.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:54.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:54.163 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:54.196 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:54.236 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:54.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:54 smithi064 ceph-mon[212750]: pgmap v340: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 33220/283014 objects degraded (11.738%); 8074/283014 objects misplaced (2.853%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T05:52:54.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:54 smithi064 ceph-mon[220044]: pgmap v340: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 33220/283014 objects degraded (11.738%); 8074/283014 objects misplaced (2.853%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T05:52:54.480 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:54.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:54.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:54.586 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:54.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:54 smithi156 ceph-mon[195101]: pgmap v340: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 33220/283014 objects degraded (11.738%); 8074/283014 objects misplaced (2.853%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T05:52:54.620 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:55.118 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:55.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:55 smithi064 ceph-mon[212750]: pgmap v341: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped+laggy, 2 peering, 86 active+clean, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 44 op/s; 41150/281964 objects degraded (14.594%); 7156/281964 objects misplaced (2.538%); 14 MiB/s, 219 objects/s recovering 2023-10-09T05:52:55.304 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:55 smithi064 ceph-mon[220044]: pgmap v341: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped+laggy, 2 peering, 86 active+clean, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 44 op/s; 41150/281964 objects degraded (14.594%); 7156/281964 objects misplaced (2.538%); 14 MiB/s, 219 objects/s recovering 2023-10-09T05:52:55.304 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:52:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:52:55] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.43.0" 2023-10-09T05:52:55.389 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:55.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:55.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:55.497 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:55.536 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:55.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:55 smithi156 ceph-mon[195101]: pgmap v341: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped+laggy, 2 peering, 86 active+clean, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 44 op/s; 41150/281964 objects degraded (14.594%); 7156/281964 objects misplaced (2.538%); 14 MiB/s, 219 objects/s recovering 2023-10-09T05:52:55.791 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:52:55.792 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:52:55.811 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:55.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:55.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:55.915 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:55.953 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:56.275 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:52:56.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:56.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:52:56.392 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:52:56.440 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:56.501 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:52:56.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:56.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:52:56.607 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:56.652 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:56.892 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:56.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:56.893 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:52:56.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:56.995 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:57.034 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:57.157 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:52:57.424 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:57.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:57.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:57.529 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:52:57.559 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:57.608 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:52:57.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:57.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:52:57.713 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:52:57.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:57 smithi064 ceph-mon[212750]: pgmap v342: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped+laggy, 2 peering, 86 active+clean, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 45 op/s; 41150/281916 objects degraded (14.597%); 7156/281916 objects misplaced (2.538%); 14 MiB/s, 219 objects/s recovering 2023-10-09T05:52:57.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:57 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 41150/281916 objects degraded (14.597%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:57.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:57 smithi064 ceph-mon[220044]: pgmap v342: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped+laggy, 2 peering, 86 active+clean, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 45 op/s; 41150/281916 objects degraded (14.597%); 7156/281916 objects misplaced (2.538%); 14 MiB/s, 219 objects/s recovering 2023-10-09T05:52:57.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:57 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 41150/281916 objects degraded (14.597%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:57.743 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:57.793 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:52:57.800 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:57 smithi156 ceph-mon[195101]: pgmap v342: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped+laggy, 2 peering, 86 active+clean, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 45 op/s; 41150/281916 objects degraded (14.597%); 7156/281916 objects misplaced (2.538%); 14 MiB/s, 219 objects/s recovering 2023-10-09T05:52:57.800 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:57 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 41150/281916 objects degraded (14.597%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T05:52:57.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:57.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:52:57.895 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:57.902 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:57.966 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:58.033 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:58.044 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:58.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:58.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:58.148 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:52:58.178 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:58.230 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:52:58.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:58.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:52:58.336 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:52:58.375 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:58.492 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:52:58.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:58.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:52:58.550 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:58 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3271666723' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T05:52:58.550 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:58 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T05:52:58.550 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:58 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:58.550 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:58 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:52:58.584 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 1 pgp_num to 11 2023-10-09T05:52:58.598 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:52:58.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:58 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3271666723' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T05:52:58.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:58 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T05:52:58.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:58 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:58.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:58 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:52:58.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:58 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3271666723' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T05:52:58.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:58 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T05:52:58.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:58 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:52:58.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:52:58.660 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:59.327 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:52:59.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:52:59.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:52:59.430 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:52:59.467 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:52:59.581 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T05:52:59.605 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:59 smithi064 ceph-mon[212750]: pgmap v343: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped+laggy, 2 peering, 86 active+clean, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 44 op/s; 41150/281760 objects degraded (14.605%); 7156/281760 objects misplaced (2.540%); 12 MiB/s, 188 objects/s recovering 2023-10-09T05:52:59.606 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:59 smithi064 ceph-mon[212750]: failed to encode map e151 with expected crc 2023-10-09T05:52:59.606 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:59 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-10-09T05:52:59.606 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:59 smithi064 ceph-mon[212750]: osdmap e151: 8 total, 8 up, 8 in 2023-10-09T05:52:59.606 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:59 smithi064 ceph-mon[212750]: failed to encode map e151 with expected crc 2023-10-09T05:52:59.606 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:59 smithi064 ceph-mon[212750]: failed to encode map e151 with expected crc 2023-10-09T05:52:59.606 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:59 smithi064 ceph-mon[212750]: failed to encode map e151 with expected crc 2023-10-09T05:52:59.606 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:59 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2952184022' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:52:59.606 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:52:59 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:52:59.606 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:59 smithi064 ceph-mon[220044]: pgmap v343: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped+laggy, 2 peering, 86 active+clean, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 44 op/s; 41150/281760 objects degraded (14.605%); 7156/281760 objects misplaced (2.540%); 12 MiB/s, 188 objects/s recovering 2023-10-09T05:52:59.607 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:59 smithi064 ceph-mon[220044]: failed to encode map e151 with expected crc 2023-10-09T05:52:59.607 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:59 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-10-09T05:52:59.607 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:59 smithi064 ceph-mon[220044]: osdmap e151: 8 total, 8 up, 8 in 2023-10-09T05:52:59.607 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:59 smithi064 ceph-mon[220044]: failed to encode map e151 with expected crc 2023-10-09T05:52:59.607 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:59 smithi064 ceph-mon[220044]: failed to encode map e151 with expected crc 2023-10-09T05:52:59.607 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:59 smithi064 ceph-mon[220044]: failed to encode map e151 with expected crc 2023-10-09T05:52:59.607 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:59 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2952184022' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:52:59.607 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:52:59 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:52:59.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:59 smithi156 ceph-mon[195101]: pgmap v343: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped+laggy, 2 peering, 86 active+clean, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 44 op/s; 41150/281760 objects degraded (14.605%); 7156/281760 objects misplaced (2.540%); 12 MiB/s, 188 objects/s recovering 2023-10-09T05:52:59.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:59 smithi156 ceph-mon[195101]: failed to encode map e151 with expected crc 2023-10-09T05:52:59.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:59 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-10-09T05:52:59.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:59 smithi156 ceph-mon[195101]: osdmap e151: 8 total, 8 up, 8 in 2023-10-09T05:52:59.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:59 smithi156 ceph-mon[195101]: failed to encode map e151 with expected crc 2023-10-09T05:52:59.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:59 smithi156 ceph-mon[195101]: failed to encode map e151 with expected crc 2023-10-09T05:52:59.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:59 smithi156 ceph-mon[195101]: failed to encode map e151 with expected crc 2023-10-09T05:52:59.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:59 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2952184022' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:52:59.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:52:59 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:53:00.110 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:00.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:00.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:00.216 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:00.248 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:00.299 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:00.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:00.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:00.410 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:00.442 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:00.696 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:00 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:00.696 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:00 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:53:00.696 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:00 smithi064 ceph-mon[212750]: osdmap e152: 8 total, 8 up, 8 in 2023-10-09T05:53:00.696 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:00 smithi064 ceph-mon[212750]: failed to encode map e152 with expected crc 2023-10-09T05:53:00.696 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:00 smithi064 ceph-mon[212750]: failed to encode map e152 with expected crc 2023-10-09T05:53:00.696 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:00 smithi064 ceph-mon[212750]: failed to encode map e152 with expected crc 2023-10-09T05:53:00.696 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:00 smithi064 ceph-mon[212750]: failed to encode map e152 with expected crc 2023-10-09T05:53:00.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:00 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:00.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:00 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:53:00.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:00 smithi064 ceph-mon[220044]: osdmap e152: 8 total, 8 up, 8 in 2023-10-09T05:53:00.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:00 smithi064 ceph-mon[220044]: failed to encode map e152 with expected crc 2023-10-09T05:53:00.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:00 smithi064 ceph-mon[220044]: failed to encode map e152 with expected crc 2023-10-09T05:53:00.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:00 smithi064 ceph-mon[220044]: failed to encode map e152 with expected crc 2023-10-09T05:53:00.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:00 smithi064 ceph-mon[220044]: failed to encode map e152 with expected crc 2023-10-09T05:53:00.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:00 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:00.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:00 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:53:00.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:00 smithi156 ceph-mon[195101]: osdmap e152: 8 total, 8 up, 8 in 2023-10-09T05:53:00.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:00 smithi156 ceph-mon[195101]: failed to encode map e152 with expected crc 2023-10-09T05:53:00.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:00 smithi156 ceph-mon[195101]: failed to encode map e152 with expected crc 2023-10-09T05:53:00.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:00 smithi156 ceph-mon[195101]: failed to encode map e152 with expected crc 2023-10-09T05:53:00.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:00 smithi156 ceph-mon[195101]: failed to encode map e152 with expected crc 2023-10-09T05:53:01.156 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:01.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:01.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:01.261 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:01.290 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:01.343 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:01.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:01.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:01.455 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:01.498 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:01.623 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:01.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:01.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:01.717 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:53:01.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:01 smithi064 ceph-mon[212750]: pgmap v346: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 41148/279993 objects degraded (14.696%); 6252/279993 objects misplaced (2.233%); 14 MiB/s, 227 objects/s recovering 2023-10-09T05:53:01.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:01 smithi064 ceph-mon[220044]: pgmap v346: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 41148/279993 objects degraded (14.696%); 6252/279993 objects misplaced (2.233%); 14 MiB/s, 227 objects/s recovering 2023-10-09T05:53:01.727 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:01.737 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:53:01.756 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:01.761 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:53:01.783 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:53:01.804 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:53:01.812 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:01.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:01.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:01.834 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:53:01.866 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:53:01.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:01 smithi156 ceph-mon[195101]: pgmap v346: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 41148/279993 objects degraded (14.696%); 6252/279993 objects misplaced (2.233%); 14 MiB/s, 227 objects/s recovering 2023-10-09T05:53:01.896 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:53:01.899 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:53:01.899 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:53:01.899 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:53:01.899 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:53:01.899 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:53:01.899 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:53:01.899 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:53:01.899 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:53:01.918 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:01.947 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:02.001 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:02.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:02.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:02.107 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:02.136 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:02.173 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:53:02.173 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T05:53:02.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.173 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:100220 6.1 6.b072f6c1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-10-09T05:53:02.173 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:53:02.017855+0000", 2023-10-09T05:53:02.173 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.153811699, 2023-10-09T05:53:02.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15384531700000001, 2023-10-09T05:53:02.173 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:53:02.174 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-10-09T05:53:02.174 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:53:02.174 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:53:02.174 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:53:02.174 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 100220 2023-10-09T05:53:02.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.174 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:53:02.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:53:02.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.017855+0000", 2023-10-09T05:53:02.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:53:02.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:53:02.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.017855+0000", 2023-10-09T05:53:02.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:53:02.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:53:02.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.017853+0000", 2023-10-09T05:53:02.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:53:02.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:53:02.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.017863+0000", 2023-10-09T05:53:02.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0162e-05 2023-10-09T05:53:02.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:53:02.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.017882+0000", 2023-10-09T05:53:02.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8974000000000002e-05 2023-10-09T05:53:02.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:53:02.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.017894+0000", 2023-10-09T05:53:02.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1756999999999999e-05 2023-10-09T05:53:02.177 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.177 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:53:02.177 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.177 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.44588.0:100156 6.1 e152/139)", 2023-10-09T05:53:02.177 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:53:02.050398+0000", 2023-10-09T05:53:02.177 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.121268915, 2023-10-09T05:53:02.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.121371162, 2023-10-09T05:53:02.177 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:53:02.178 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-10-09T05:53:02.178 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:53:02.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:53:02.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.050398+0000", 2023-10-09T05:53:02.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:53:02.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:53:02.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.050398+0000", 2023-10-09T05:53:02.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:53:02.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:53:02.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.050396+0000", 2023-10-09T05:53:02.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:53:02.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:53:02.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.050409+0000", 2023-10-09T05:53:02.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3621999999999999e-05 2023-10-09T05:53:02.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:53:02.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.050427+0000", 2023-10-09T05:53:02.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7537000000000002e-05 2023-10-09T05:53:02.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:53:02.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.050440+0000", 2023-10-09T05:53:02.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2625e-05 2023-10-09T05:53:02.180 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.180 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:53:02.180 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.180 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.44588.0:100156 6.1 e152/139)", 2023-10-09T05:53:02.181 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:53:02.053394+0000", 2023-10-09T05:53:02.181 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.11827228200000001, 2023-10-09T05:53:02.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.118425399, 2023-10-09T05:53:02.181 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:53:02.181 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-10-09T05:53:02.181 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:53:02.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:53:02.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.053394+0000", 2023-10-09T05:53:02.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:53:02.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:53:02.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.053394+0000", 2023-10-09T05:53:02.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:53:02.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:53:02.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.053389+0000", 2023-10-09T05:53:02.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999938 2023-10-09T05:53:02.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:53:02.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.053402+0000", 2023-10-09T05:53:02.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3573e-05 2023-10-09T05:53:02.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:53:02.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.053422+0000", 2023-10-09T05:53:02.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0092999999999999e-05 2023-10-09T05:53:02.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:53:02.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.053432+0000", 2023-10-09T05:53:02.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5980000000000003e-06 2023-10-09T05:53:02.183 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.184 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:53:02.184 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.184 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.44588.0:100149 6.4 e152/139)", 2023-10-09T05:53:02.184 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:53:02.093798+0000", 2023-10-09T05:53:02.184 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.077868645, 2023-10-09T05:53:02.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.078073624999999994, 2023-10-09T05:53:02.184 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:53:02.184 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-10-09T05:53:02.184 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:53:02.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:53:02.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.093798+0000", 2023-10-09T05:53:02.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:53:02.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:53:02.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.093798+0000", 2023-10-09T05:53:02.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:53:02.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:53:02.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.093797+0000", 2023-10-09T05:53:02.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T05:53:02.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:53:02.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.093804+0000", 2023-10-09T05:53:02.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.469e-06 2023-10-09T05:53:02.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:53:02.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.093825+0000", 2023-10-09T05:53:02.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0202000000000001e-05 2023-10-09T05:53:02.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:53:02.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.093834+0000", 2023-10-09T05:53:02.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0529999999999996e-06 2023-10-09T05:53:02.187 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.187 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:53:02.187 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.187 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.44588.0:100149 6.4 e152/139)", 2023-10-09T05:53:02.187 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:53:02.097599+0000", 2023-10-09T05:53:02.188 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.074067425000000006, 2023-10-09T05:53:02.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.074323299999999995, 2023-10-09T05:53:02.188 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:53:02.188 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-10-09T05:53:02.188 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:53:02.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:53:02.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.097599+0000", 2023-10-09T05:53:02.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:53:02.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:53:02.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.097599+0000", 2023-10-09T05:53:02.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:53:02.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:53:02.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.097598+0000", 2023-10-09T05:53:02.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T05:53:02.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:53:02.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.097605+0000", 2023-10-09T05:53:02.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2879999999999998e-06 2023-10-09T05:53:02.190 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:53:02.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.097636+0000", 2023-10-09T05:53:02.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1371999999999997e-05 2023-10-09T05:53:02.190 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:53:02.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.097643+0000", 2023-10-09T05:53:02.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1289999999999998e-06 2023-10-09T05:53:02.190 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.191 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:53:02.191 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.191 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:100231 6.4 6.1286f0d4 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-10-09T05:53:02.191 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:53:02.108397+0000", 2023-10-09T05:53:02.191 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.063270236999999993, 2023-10-09T05:53:02.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.063580986000000006, 2023-10-09T05:53:02.191 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:53:02.191 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-10-09T05:53:02.191 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:53:02.191 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:53:02.192 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:53:02.192 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 100231 2023-10-09T05:53:02.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.192 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:53:02.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:53:02.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.108397+0000", 2023-10-09T05:53:02.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:53:02.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:53:02.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.108397+0000", 2023-10-09T05:53:02.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:53:02.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:53:02.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.108394+0000", 2023-10-09T05:53:02.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:53:02.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:53:02.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.108404+0000", 2023-10-09T05:53:02.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2380000000000003e-06 2023-10-09T05:53:02.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:53:02.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.108422+0000", 2023-10-09T05:53:02.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8345000000000001e-05 2023-10-09T05:53:02.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:53:02.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.108431+0000", 2023-10-09T05:53:02.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2499999999999995e-06 2023-10-09T05:53:02.194 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.194 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:53:02.194 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.195 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:100234 6.4 6.d7849d94 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-10-09T05:53:02.195 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:53:02.157558+0000", 2023-10-09T05:53:02.195 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.01410842, 2023-10-09T05:53:02.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014479646000000001, 2023-10-09T05:53:02.195 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:53:02.195 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-10-09T05:53:02.195 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:53:02.195 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:53:02.195 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:53:02.195 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 100234 2023-10-09T05:53:02.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.196 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:53:02.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:53:02.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.157558+0000", 2023-10-09T05:53:02.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:53:02.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:53:02.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.157558+0000", 2023-10-09T05:53:02.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:53:02.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:53:02.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.157556+0000", 2023-10-09T05:53:02.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:53:02.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:53:02.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.157567+0000", 2023-10-09T05:53:02.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0975000000000001e-05 2023-10-09T05:53:02.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:53:02.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.157585+0000", 2023-10-09T05:53:02.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7835000000000001e-05 2023-10-09T05:53:02.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:53:02.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.157598+0000", 2023-10-09T05:53:02.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3451e-05 2023-10-09T05:53:02.198 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.198 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:53:02.198 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.198 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.44588.0:100153 6.4 e152/139)", 2023-10-09T05:53:02.199 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:53:02.166350+0000", 2023-10-09T05:53:02.199 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0053163719999999998, 2023-10-09T05:53:02.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0057433450000000004, 2023-10-09T05:53:02.199 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:53:02.199 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-10-09T05:53:02.199 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:53:02.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:53:02.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.166350+0000", 2023-10-09T05:53:02.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:53:02.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:53:02.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.166350+0000", 2023-10-09T05:53:02.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:53:02.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:53:02.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.166348+0000", 2023-10-09T05:53:02.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:53:02.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:53:02.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.166360+0000", 2023-10-09T05:53:02.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2163e-05 2023-10-09T05:53:02.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:53:02.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.166379+0000", 2023-10-09T05:53:02.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8427000000000002e-05 2023-10-09T05:53:02.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:53:02.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.166389+0000", 2023-10-09T05:53:02.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0229e-05 2023-10-09T05:53:02.201 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.202 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:53:02.202 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.202 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.44588.0:100153 6.4 e152/139)", 2023-10-09T05:53:02.202 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:53:02.166713+0000", 2023-10-09T05:53:02.202 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0049533290000000002, 2023-10-09T05:53:02.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0054284490000000001, 2023-10-09T05:53:02.202 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:53:02.202 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-10-09T05:53:02.202 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:53:02.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:53:02.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.166713+0000", 2023-10-09T05:53:02.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:53:02.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:53:02.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.166713+0000", 2023-10-09T05:53:02.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:53:02.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:53:02.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.166712+0000", 2023-10-09T05:53:02.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T05:53:02.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:53:02.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.166721+0000", 2023-10-09T05:53:02.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3040000000000005e-06 2023-10-09T05:53:02.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:53:02.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.166737+0000", 2023-10-09T05:53:02.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6257999999999999e-05 2023-10-09T05:53:02.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.205 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:53:02.205 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:02.166746+0000", 2023-10-09T05:53:02.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5290000000000008e-06 2023-10-09T05:53:02.205 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.205 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:53:02.205 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.205 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:100203 6.4 6.cebf5b74 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-10-09T05:53:02.205 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:53:01.803279+0000", 2023-10-09T05:53:02.206 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.36838793199999997, 2023-10-09T05:53:02.206 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.36891476699999998, 2023-10-09T05:53:02.206 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:53:02.206 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-10-09T05:53:02.206 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:53:02.206 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:53:02.206 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:53:02.206 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 100203 2023-10-09T05:53:02.206 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.206 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:53:02.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.206 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:53:02.207 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:01.803279+0000", 2023-10-09T05:53:02.207 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:53:02.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.207 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:53:02.207 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:01.803279+0000", 2023-10-09T05:53:02.207 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:53:02.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.207 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:53:02.207 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:01.803277+0000", 2023-10-09T05:53:02.207 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:53:02.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:53:02.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:01.803286+0000", 2023-10-09T05:53:02.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.969e-06 2023-10-09T05:53:02.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:53:02.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:01.803300+0000", 2023-10-09T05:53:02.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3532e-05 2023-10-09T05:53:02.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:53:02.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:01.803312+0000", 2023-10-09T05:53:02.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2488e-05 2023-10-09T05:53:02.209 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.209 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:53:02.209 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.209 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:100206 6.4 6.223afd44 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-10-09T05:53:02.209 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:53:01.814365+0000", 2023-10-09T05:53:02.209 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.35730212900000002, 2023-10-09T05:53:02.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.35791783999999999, 2023-10-09T05:53:02.210 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:53:02.210 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-10-09T05:53:02.210 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:53:02.210 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:53:02.210 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:53:02.210 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 100206 2023-10-09T05:53:02.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.210 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:53:02.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.210 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:53:02.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:01.814365+0000", 2023-10-09T05:53:02.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:53:02.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.211 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:53:02.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:01.814365+0000", 2023-10-09T05:53:02.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:53:02.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.211 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:53:02.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:01.814363+0000", 2023-10-09T05:53:02.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:53:02.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:53:02.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:01.814372+0000", 2023-10-09T05:53:02.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5629999999999998e-06 2023-10-09T05:53:02.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:53:02.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:01.814390+0000", 2023-10-09T05:53:02.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7711999999999999e-05 2023-10-09T05:53:02.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:53:02.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:01.814403+0000", 2023-10-09T05:53:02.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3175e-05 2023-10-09T05:53:02.213 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.213 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:53:02.213 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.213 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:100207 6.4 6.d7239344 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-10-09T05:53:02.213 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:53:01.830897+0000", 2023-10-09T05:53:02.213 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.34076990499999998, 2023-10-09T05:53:02.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34144205100000002, 2023-10-09T05:53:02.214 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:53:02.214 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-10-09T05:53:02.214 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:53:02.214 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:53:02.214 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:53:02.214 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 100207 2023-10-09T05:53:02.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.214 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:53:02.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.214 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:53:02.214 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:01.830897+0000", 2023-10-09T05:53:02.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:53:02.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:53:02.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:01.830897+0000", 2023-10-09T05:53:02.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:53:02.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:53:02.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:01.830895+0000", 2023-10-09T05:53:02.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:53:02.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:53:02.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:01.830904+0000", 2023-10-09T05:53:02.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7350000000000002e-06 2023-10-09T05:53:02.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:53:02.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:01.830917+0000", 2023-10-09T05:53:02.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3324e-05 2023-10-09T05:53:02.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:53:02.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:53:02.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:53:02.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:01.830929+0000", 2023-10-09T05:53:02.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2130000000000001e-05 2023-10-09T05:53:02.217 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.217 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:53:02.217 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.217 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:53:02.217 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-10-09T05:53:02.217 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 12 2023-10-09T05:53:02.217 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:53:02.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:02.218 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:02.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:02.319 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:02.356 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:02.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:02 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3682323771' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:02.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:02 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 41148/279993 objects degraded (14.696%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:02.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:02 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3682323771' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:02.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:02 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 41148/279993 objects degraded (14.696%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:02.801 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:53:02.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:02 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3682323771' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:02.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:02 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 41148/279993 objects degraded (14.696%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:03.329 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:53:03.372 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:03.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:03.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:03.484 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:03.519 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:03.533 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:03.579 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:03.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:03.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:03.687 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:03.724 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:03.781 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:03.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:03.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:03.887 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:03.929 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:03.992 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:03.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:03.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:04.095 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:04.111 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:04 smithi064 ceph-mon[212750]: pgmap v347: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 91 op/s; 41148/279759 objects degraded (14.708%); 6252/279759 objects misplaced (2.235%); 7.1 MiB/s, 113 objects/s recovering 2023-10-09T05:53:04.127 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:04.188 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:04.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:04.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:04.296 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:04.331 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:04 smithi156 ceph-mon[195101]: pgmap v347: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 91 op/s; 41148/279759 objects degraded (14.708%); 6252/279759 objects misplaced (2.235%); 7.1 MiB/s, 113 objects/s recovering 2023-10-09T05:53:04.382 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:04.384 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:04 smithi064 ceph-mon[220044]: pgmap v347: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 91 op/s; 41148/279759 objects degraded (14.708%); 6252/279759 objects misplaced (2.235%); 7.1 MiB/s, 113 objects/s recovering 2023-10-09T05:53:04.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:04.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:04.486 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:04.518 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:04.580 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:04.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:04.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:04.685 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:04.722 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:04.745 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:04.831 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:04.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:04.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:04.940 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:04.988 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:05.122 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:05.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:05.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:05.193 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T05:53:05.194 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:53:05.194 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T05:53:05.194 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:53:05.224 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:05.254 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:05 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:05.254 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:05 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:05.264 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:53:04 smithi156 conmon[175419]: logger=sqlstore.transactions t=2023-10-09T05:53:04.968654243Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-10-09T05:53:05.264 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:53:05 smithi156 conmon[175419]: logger=cleanup t=2023-10-09T05:53:05.155013456Z level=info msg="Completed cleanup jobs" duration=196.858826ms 2023-10-09T05:53:05.265 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:05 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:05.268 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:05.386 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:05.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:05.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:05.407 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:53:05.491 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:05.522 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:05.577 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:05.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:05.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:05.684 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:05.702 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:53:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:53:05] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.43.0" 2023-10-09T05:53:05.722 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:05.792 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:05.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:05.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:05.901 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:05.943 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:06.054 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:06.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:06.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:06.158 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:06.159 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T05:53:06.202 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:06.307 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:06.313 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:06 smithi156 ceph-mon[195101]: pgmap v348: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 150 op/s; 41148/278295 objects degraded (14.786%); 5298/278295 objects misplaced (1.904%); 15 MiB/s, 232 objects/s recovering 2023-10-09T05:53:06.313 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:06 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3222842676' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:06.313 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:06 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:06.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:06.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:06.415 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:06.467 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:06 smithi064 ceph-mon[220044]: pgmap v348: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 150 op/s; 41148/278295 objects degraded (14.786%); 5298/278295 objects misplaced (1.904%); 15 MiB/s, 232 objects/s recovering 2023-10-09T05:53:06.468 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:06 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3222842676' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:06.468 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:06 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:06.468 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:06 smithi064 ceph-mon[212750]: pgmap v348: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 150 op/s; 41148/278295 objects degraded (14.786%); 5298/278295 objects misplaced (1.904%); 15 MiB/s, 232 objects/s recovering 2023-10-09T05:53:06.468 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:06 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3222842676' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:06.468 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:06 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:06.472 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:06.831 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:06.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:06.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:06.935 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:06.975 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:07.036 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:07.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:07.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:07.048 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:53:07.048 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T05:53:07.048 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T05:53:07.048 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T05:53:07.048 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:53:07.142 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:07.148 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:07 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:07.148 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:07 smithi064 ceph-mon[212750]: failed to encode map e153 with expected crc 2023-10-09T05:53:07.148 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:07 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:53:07.148 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:07 smithi064 ceph-mon[212750]: osdmap e153: 8 total, 8 up, 8 in 2023-10-09T05:53:07.148 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:07 smithi064 ceph-mon[212750]: failed to encode map e153 with expected crc 2023-10-09T05:53:07.148 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:07 smithi064 ceph-mon[212750]: failed to encode map e153 with expected crc 2023-10-09T05:53:07.148 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:07 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 41148/278295 objects degraded (14.786%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:07.177 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:07.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:07 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:07.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:07 smithi156 ceph-mon[195101]: failed to encode map e153 with expected crc 2023-10-09T05:53:07.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:07 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:53:07.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:07 smithi156 ceph-mon[195101]: osdmap e153: 8 total, 8 up, 8 in 2023-10-09T05:53:07.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:07 smithi156 ceph-mon[195101]: failed to encode map e153 with expected crc 2023-10-09T05:53:07.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:07 smithi156 ceph-mon[195101]: failed to encode map e153 with expected crc 2023-10-09T05:53:07.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:07 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 41148/278295 objects degraded (14.786%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:07.429 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:07.432 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:07 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:07.432 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:07 smithi064 ceph-mon[220044]: failed to encode map e153 with expected crc 2023-10-09T05:53:07.432 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:07 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:53:07.432 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:07 smithi064 ceph-mon[220044]: osdmap e153: 8 total, 8 up, 8 in 2023-10-09T05:53:07.432 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:07 smithi064 ceph-mon[220044]: failed to encode map e153 with expected crc 2023-10-09T05:53:07.432 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:07 smithi064 ceph-mon[220044]: failed to encode map e153 with expected crc 2023-10-09T05:53:07.433 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:07 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 41148/278295 objects degraded (14.786%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:07.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:07.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:07.534 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:07.568 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:07.642 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:07.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:07.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:07.751 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:07.790 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:08.002 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:08.035 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:53:08.085 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:08.097 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:08.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:08.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:08.213 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:08.303 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:08.387 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:08 smithi064 ceph-mon[220044]: pgmap v350: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 171 op/s; 41148/277692 objects degraded (14.818%); 5298/277692 objects misplaced (1.908%); 15 MiB/s, 234 objects/s recovering 2023-10-09T05:53:08.388 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:08 smithi064 ceph-mon[212750]: pgmap v350: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 171 op/s; 41148/277692 objects degraded (14.818%); 5298/277692 objects misplaced (1.908%); 15 MiB/s, 234 objects/s recovering 2023-10-09T05:53:08.617 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:08.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:08 smithi156 ceph-mon[195101]: pgmap v350: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 171 op/s; 41148/277692 objects degraded (14.818%); 5298/277692 objects misplaced (1.908%); 15 MiB/s, 234 objects/s recovering 2023-10-09T05:53:08.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:08.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:08.724 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:08.759 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:08.816 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:08.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:08.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:08.925 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:08.955 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:09.012 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:09.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:09.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:09.120 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:09.162 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:09.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:09 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:09.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:09 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:09.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:09 smithi064 ceph-mon[212750]: pgmap v351: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 97 op/s; 41148/277386 objects degraded (14.834%); 5298/277386 objects misplaced (1.910%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T05:53:09.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:09 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:09.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:09 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:09.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:09 smithi064 ceph-mon[220044]: pgmap v351: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 97 op/s; 41148/277386 objects degraded (14.834%); 5298/277386 objects misplaced (1.910%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T05:53:09.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:09 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:09.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:09 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:09.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:09 smithi156 ceph-mon[195101]: pgmap v351: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 97 op/s; 41148/277386 objects degraded (14.834%); 5298/277386 objects misplaced (1.910%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T05:53:09.921 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:09.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:09.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:09.936 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:53:10.025 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:10.026 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:10.115 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:10.265 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:10.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:10.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:10.375 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:10.408 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:10.461 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:10.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:10.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:10.570 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:10.604 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:10.660 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:10.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:10.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:10.770 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:10.824 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:10.858 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:53:10.870 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T05:53:10.973 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:53:10.999 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:53:11.026 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:53:11.051 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:53:11.087 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:53:11.103 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:53:11.137 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:53:11.138 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:11.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:11.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:11.165 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:53:11.167 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:53:11.167 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:53:11.167 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:53:11.167 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:53:11.167 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:53:11.168 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:53:11.168 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:53:11.168 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:53:11.252 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:11.292 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:11.347 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:11.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:11.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:11.454 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:11.484 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:11.535 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:11.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:11.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:11.640 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:11.685 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:11.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:11 smithi156 ceph-mon[195101]: pgmap v352: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 142 op/s; 41148/275724 objects degraded (14.924%); 4288/275724 objects misplaced (1.555%); 12 MiB/s, 196 objects/s recovering 2023-10-09T05:53:11.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:11 smithi156 ceph-mon[195101]: from='client.44792 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:53:11.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:11 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1366992394' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:11.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:11 smithi064 ceph-mon[212750]: pgmap v352: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 142 op/s; 41148/275724 objects degraded (14.924%); 4288/275724 objects misplaced (1.555%); 12 MiB/s, 196 objects/s recovering 2023-10-09T05:53:11.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:11 smithi064 ceph-mon[212750]: from='client.44792 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:53:11.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:11 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1366992394' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:11.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:11 smithi064 ceph-mon[220044]: pgmap v352: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 142 op/s; 41148/275724 objects degraded (14.924%); 4288/275724 objects misplaced (1.555%); 12 MiB/s, 196 objects/s recovering 2023-10-09T05:53:11.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:11 smithi064 ceph-mon[220044]: from='client.44792 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:53:11.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:11 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1366992394' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:12.249 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":352,"stamp":"2023-10-09T05:53:10.497974+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6022186515,"num_objects":91908,"num_object_clones":0,"num_object_copies":275724,"num_objects_missing_on_primary":5403,"num_objects_missing":5403,"num_objects_degraded":41148,"num_objects_misplaced":4288,"num_objects_unfound":0,"num_objects_dirty":91908,"num_whiteouts":0,"num_read":445,"num_read_kb":833,"num_write":101477,"num_write_kb":6180787,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13102,"num_bytes_recovered":859542969,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":103024,"ondisk_log_size":103024,"up":297,"acting":286,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":332,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19853408,"kb_used_data":18320800,"kb_used_omap":127,"kb_used_meta":1532352,"kb_avail":730107808,"statfs":{"total":767960285184,"available":747630395392,"internally_reserved":0,"allocated":18760499200,"data_stored":18755079479,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":130887,"internal_metadata":1569128633},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,1,0,1,4,3,2,4,6],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-93257728,"num_objects":-1423,"num_object_clones":0,"num_object_copies":-4269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1964,"num_objects_unfound":0,"num_objects_dirty":-1423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1423,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1964,"num_bytes_recovered":128712704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002913"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":168,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.584111+0000","last_change":"2023-10-09T05:52:11.412762+0000","last_active":"2023-10-09T05:52:59.584111+0000","last_peered":"2023-10-09T05:52:59.584111+0000","last_clean":"2023-10-09T05:52:59.584111+0000","last_became_active":"2023-10-09T05:52:11.412360+0000","last_became_peered":"2023-10-09T05:52:11.412360+0000","last_unstale":"2023-10-09T05:52:59.584111+0000","last_undegraded":"2023-10-09T05:52:59.584111+0000","last_fullsized":"2023-10-09T05:52:59.584111+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:28.459463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":135,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.567213+0000","last_change":"2023-10-09T05:52:11.417209+0000","last_active":"2023-10-09T05:52:59.567213+0000","last_peered":"2023-10-09T05:52:59.567213+0000","last_clean":"2023-10-09T05:52:59.567213+0000","last_became_active":"2023-10-09T05:52:11.416952+0000","last_became_peered":"2023-10-09T05:52:11.416952+0000","last_unstale":"2023-10-09T05:52:59.567213+0000","last_undegraded":"2023-10-09T05:52:59.567213+0000","last_fullsized":"2023-10-09T05:52:59.567213+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:32.958103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":23,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.589215+0000","last_change":"2023-10-09T05:52:11.410566+0000","last_active":"2023-10-09T05:52:59.589215+0000","last_peered":"2023-10-09T05:52:59.589215+0000","last_clean":"2023-10-09T05:52:59.589215+0000","last_became_active":"2023-10-09T05:52:11.410252+0000","last_became_peered":"2023-10-09T05:52:11.410252+0000","last_unstale":"2023-10-09T05:52:59.589215+0000","last_undegraded":"2023-10-09T05:52:59.589215+0000","last_fullsized":"2023-10-09T05:52:59.589215+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:06.878658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":98,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.603943+0000","last_change":"2023-10-09T05:52:23.639987+0000","last_active":"2023-10-09T05:52:59.603943+0000","last_peered":"2023-10-09T05:52:59.603943+0000","last_clean":"2023-10-09T05:52:59.603943+0000","last_became_active":"2023-10-09T05:52:23.639671+0000","last_became_peered":"2023-10-09T05:52:23.639671+0000","last_unstale":"2023-10-09T05:52:59.603943+0000","last_undegraded":"2023-10-09T05:52:59.603943+0000","last_fullsized":"2023-10-09T05:52:59.603943+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:19.677942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":168,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.567499+0000","last_change":"2023-10-09T05:52:11.221586+0000","last_active":"2023-10-09T05:52:59.567499+0000","last_peered":"2023-10-09T05:52:59.567499+0000","last_clean":"2023-10-09T05:52:59.567499+0000","last_became_active":"2023-10-09T05:52:11.221307+0000","last_became_peered":"2023-10-09T05:52:11.221307+0000","last_unstale":"2023-10-09T05:52:59.567499+0000","last_undegraded":"2023-10-09T05:52:59.567499+0000","last_fullsized":"2023-10-09T05:52:59.567499+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:48:04.204923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":149,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.588140+0000","last_change":"2023-10-09T05:52:11.218711+0000","last_active":"2023-10-09T05:52:59.588140+0000","last_peered":"2023-10-09T05:52:59.588140+0000","last_clean":"2023-10-09T05:52:59.588140+0000","last_became_active":"2023-10-09T05:52:11.218513+0000","last_became_peered":"2023-10-09T05:52:11.218513+0000","last_unstale":"2023-10-09T05:52:59.588140+0000","last_undegraded":"2023-10-09T05:52:59.588140+0000","last_fullsized":"2023-10-09T05:52:59.588140+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:02.240130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":88,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.604422+0000","last_change":"2023-10-09T05:51:57.202627+0000","last_active":"2023-10-09T05:52:59.604422+0000","last_peered":"2023-10-09T05:52:59.604422+0000","last_clean":"2023-10-09T05:52:59.604422+0000","last_became_active":"2023-10-09T05:51:57.097707+0000","last_became_peered":"2023-10-09T05:51:57.097707+0000","last_unstale":"2023-10-09T05:52:59.604422+0000","last_undegraded":"2023-10-09T05:52:59.604422+0000","last_fullsized":"2023-10-09T05:52:59.604422+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:02.682328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":135,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.567585+0000","last_change":"2023-10-09T05:52:11.210606+0000","last_active":"2023-10-09T05:52:59.567585+0000","last_peered":"2023-10-09T05:52:59.567585+0000","last_clean":"2023-10-09T05:52:59.567585+0000","last_became_active":"2023-10-09T05:52:11.209886+0000","last_became_peered":"2023-10-09T05:52:11.209886+0000","last_unstale":"2023-10-09T05:52:59.567585+0000","last_undegraded":"2023-10-09T05:52:59.567585+0000","last_fullsized":"2023-10-09T05:52:59.567585+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:56.049681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":133,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.601297+0000","last_change":"2023-10-09T05:52:11.214597+0000","last_active":"2023-10-09T05:52:59.601297+0000","last_peered":"2023-10-09T05:52:59.601297+0000","last_clean":"2023-10-09T05:52:59.601297+0000","last_became_active":"2023-10-09T05:52:11.211759+0000","last_became_peered":"2023-10-09T05:52:11.211759+0000","last_unstale":"2023-10-09T05:52:59.601297+0000","last_undegraded":"2023-10-09T05:52:59.601297+0000","last_fullsized":"2023-10-09T05:52:59.601297+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:32.429536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":76,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.567008+0000","last_change":"2023-10-09T05:49:21.425819+0000","last_active":"2023-10-09T05:52:59.567008+0000","last_peered":"2023-10-09T05:52:59.567008+0000","last_clean":"2023-10-09T05:52:59.567008+0000","last_became_active":"2023-10-09T05:49:21.425498+0000","last_became_peered":"2023-10-09T05:49:21.425498+0000","last_unstale":"2023-10-09T05:52:59.567008+0000","last_undegraded":"2023-10-09T05:52:59.567008+0000","last_fullsized":"2023-10-09T05:52:59.567008+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:21.443055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":116,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.575599+0000","last_change":"2023-10-09T05:49:21.430469+0000","last_active":"2023-10-09T05:52:59.575599+0000","last_peered":"2023-10-09T05:52:59.575599+0000","last_clean":"2023-10-09T05:52:59.575599+0000","last_became_active":"2023-10-09T05:49:21.430107+0000","last_became_peered":"2023-10-09T05:49:21.430107+0000","last_unstale":"2023-10-09T05:52:59.575599+0000","last_undegraded":"2023-10-09T05:52:59.575599+0000","last_fullsized":"2023-10-09T05:52:59.575599+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:08.815343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":30,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.571255+0000","last_change":"2023-10-09T05:52:11.224890+0000","last_active":"2023-10-09T05:52:59.571255+0000","last_peered":"2023-10-09T05:52:59.571255+0000","last_clean":"2023-10-09T05:52:59.571255+0000","last_became_active":"2023-10-09T05:52:11.224591+0000","last_became_peered":"2023-10-09T05:52:11.224591+0000","last_unstale":"2023-10-09T05:52:59.571255+0000","last_undegraded":"2023-10-09T05:52:59.571255+0000","last_fullsized":"2023-10-09T05:52:59.571255+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:53.788492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":118,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.584391+0000","last_change":"2023-10-09T05:40:12.630786+0000","last_active":"2023-10-09T05:52:59.584391+0000","last_peered":"2023-10-09T05:52:59.584391+0000","last_clean":"2023-10-09T05:52:59.584391+0000","last_became_active":"2023-10-09T05:40:12.630528+0000","last_became_peered":"2023-10-09T05:40:12.630528+0000","last_unstale":"2023-10-09T05:52:59.584391+0000","last_undegraded":"2023-10-09T05:52:59.584391+0000","last_fullsized":"2023-10-09T05:52:59.584391+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:54.427346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":132,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.583992+0000","last_change":"2023-10-09T05:52:11.412779+0000","last_active":"2023-10-09T05:52:59.583992+0000","last_peered":"2023-10-09T05:52:59.583992+0000","last_clean":"2023-10-09T05:52:59.583992+0000","last_became_active":"2023-10-09T05:52:11.412489+0000","last_became_peered":"2023-10-09T05:52:11.412489+0000","last_unstale":"2023-10-09T05:52:59.583992+0000","last_undegraded":"2023-10-09T05:52:59.583992+0000","last_fullsized":"2023-10-09T05:52:59.583992+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:39:37.025949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":115,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.575242+0000","last_change":"2023-10-09T05:48:49.139681+0000","last_active":"2023-10-09T05:52:59.575242+0000","last_peered":"2023-10-09T05:52:59.575242+0000","last_clean":"2023-10-09T05:52:59.575242+0000","last_became_active":"2023-10-09T05:48:49.139317+0000","last_became_peered":"2023-10-09T05:48:49.139317+0000","last_unstale":"2023-10-09T05:52:59.575242+0000","last_undegraded":"2023-10-09T05:52:59.575242+0000","last_fullsized":"2023-10-09T05:52:59.575242+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:34.769043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":135,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.566748+0000","last_change":"2023-10-09T05:52:11.210197+0000","last_active":"2023-10-09T05:52:59.566748+0000","last_peered":"2023-10-09T05:52:59.566748+0000","last_clean":"2023-10-09T05:52:59.566748+0000","last_became_active":"2023-10-09T05:52:11.209895+0000","last_became_peered":"2023-10-09T05:52:11.209895+0000","last_unstale":"2023-10-09T05:52:59.566748+0000","last_undegraded":"2023-10-09T05:52:59.566748+0000","last_fullsized":"2023-10-09T05:52:59.566748+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:13.043187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":98,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.604824+0000","last_change":"2023-10-09T05:52:23.640711+0000","last_active":"2023-10-09T05:52:59.604824+0000","last_peered":"2023-10-09T05:52:59.604824+0000","last_clean":"2023-10-09T05:52:59.604824+0000","last_became_active":"2023-10-09T05:52:23.640420+0000","last_became_peered":"2023-10-09T05:52:23.640420+0000","last_unstale":"2023-10-09T05:52:59.604824+0000","last_undegraded":"2023-10-09T05:52:59.604824+0000","last_fullsized":"2023-10-09T05:52:59.604824+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:00:36.382963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":112,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.603240+0000","last_change":"2023-10-09T05:49:21.423218+0000","last_active":"2023-10-09T05:52:59.603240+0000","last_peered":"2023-10-09T05:52:59.603240+0000","last_clean":"2023-10-09T05:52:59.603240+0000","last_became_active":"2023-10-09T05:49:21.422706+0000","last_became_peered":"2023-10-09T05:49:21.422706+0000","last_unstale":"2023-10-09T05:52:59.603240+0000","last_undegraded":"2023-10-09T05:52:59.603240+0000","last_fullsized":"2023-10-09T05:52:59.603240+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:31.571220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":96,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.575200+0000","last_change":"2023-10-09T05:40:12.631893+0000","last_active":"2023-10-09T05:52:59.575200+0000","last_peered":"2023-10-09T05:52:59.575200+0000","last_clean":"2023-10-09T05:52:59.575200+0000","last_became_active":"2023-10-09T05:40:12.631592+0000","last_became_peered":"2023-10-09T05:40:12.631592+0000","last_unstale":"2023-10-09T05:52:59.575200+0000","last_undegraded":"2023-10-09T05:52:59.575200+0000","last_fullsized":"2023-10-09T05:52:59.575200+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:18:12.517403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":88,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.603783+0000","last_change":"2023-10-09T05:51:57.212052+0000","last_active":"2023-10-09T05:52:59.603783+0000","last_peered":"2023-10-09T05:52:59.603783+0000","last_clean":"2023-10-09T05:52:59.603783+0000","last_became_active":"2023-10-09T05:51:57.207796+0000","last_became_peered":"2023-10-09T05:51:57.207796+0000","last_unstale":"2023-10-09T05:52:59.603783+0000","last_undegraded":"2023-10-09T05:52:59.603783+0000","last_fullsized":"2023-10-09T05:52:59.603783+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:58.484959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":138,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.586440+0000","last_change":"2023-10-09T05:51:57.091639+0000","last_active":"2023-10-09T05:52:59.586440+0000","last_peered":"2023-10-09T05:52:59.586440+0000","last_clean":"2023-10-09T05:52:59.586440+0000","last_became_active":"2023-10-09T05:51:57.091284+0000","last_became_peered":"2023-10-09T05:51:57.091284+0000","last_unstale":"2023-10-09T05:52:59.586440+0000","last_undegraded":"2023-10-09T05:52:59.586440+0000","last_fullsized":"2023-10-09T05:52:59.586440+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:10.553453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":134,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.575836+0000","last_change":"2023-10-09T05:52:23.637255+0000","last_active":"2023-10-09T05:52:59.575836+0000","last_peered":"2023-10-09T05:52:59.575836+0000","last_clean":"2023-10-09T05:52:59.575836+0000","last_became_active":"2023-10-09T05:52:23.636918+0000","last_became_peered":"2023-10-09T05:52:23.636918+0000","last_unstale":"2023-10-09T05:52:59.575836+0000","last_undegraded":"2023-10-09T05:52:59.575836+0000","last_fullsized":"2023-10-09T05:52:59.575836+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:43.616064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.a","version":"153'6306","reported_seq":6935,"reported_epoch":153,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:53:08.868893+0000","last_change":"2023-10-09T05:52:11.342223+0000","last_active":"2023-10-09T05:53:08.868893+0000","last_peered":"2023-10-09T05:53:08.868893+0000","last_clean":"2023-10-09T05:51:33.088844+0000","last_became_active":"2023-10-09T05:51:58.249120+0000","last_became_peered":"2023-10-09T05:51:58.249120+0000","last_unstale":"2023-10-09T05:53:08.868893+0000","last_undegraded":"2023-10-09T05:51:58.095514+0000","last_fullsized":"2023-10-09T05:51:58.082441+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6306,"log_dups_size":0,"ondisk_log_size":6306,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:16:45.784914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374865920,"num_objects":5720,"num_object_clones":0,"num_object_copies":17160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":809,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6306,"num_write_kb":384832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":276,"num_bytes_recovered":18087936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":4911}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":131,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.575767+0000","last_change":"2023-10-09T05:51:57.209892+0000","last_active":"2023-10-09T05:52:59.575767+0000","last_peered":"2023-10-09T05:52:59.575767+0000","last_clean":"2023-10-09T05:52:59.575767+0000","last_became_active":"2023-10-09T05:51:57.209354+0000","last_became_peered":"2023-10-09T05:51:57.209354+0000","last_unstale":"2023-10-09T05:52:59.575767+0000","last_undegraded":"2023-10-09T05:52:59.575767+0000","last_fullsized":"2023-10-09T05:52:59.575767+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:41:28.128976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":95,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.600199+0000","last_change":"2023-10-09T05:40:14.362515+0000","last_active":"2023-10-09T05:52:59.600199+0000","last_peered":"2023-10-09T05:52:59.600199+0000","last_clean":"2023-10-09T05:52:59.600199+0000","last_became_active":"2023-10-09T05:40:14.362184+0000","last_became_peered":"2023-10-09T05:40:14.362184+0000","last_unstale":"2023-10-09T05:52:59.600199+0000","last_undegraded":"2023-10-09T05:52:59.600199+0000","last_fullsized":"2023-10-09T05:52:59.600199+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:37.681193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"153'6409","reported_seq":10927,"reported_epoch":153,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:53:09.514908+0000","last_change":"2023-10-09T05:52:11.493867+0000","last_active":"2023-10-09T05:53:09.514908+0000","last_peered":"2023-10-09T05:53:09.514908+0000","last_clean":"2023-10-09T05:51:24.569821+0000","last_became_active":"2023-10-09T05:52:11.486930+0000","last_became_peered":"2023-10-09T05:52:11.486930+0000","last_unstale":"2023-10-09T05:53:09.514908+0000","last_undegraded":"2023-10-09T05:52:11.207681+0000","last_fullsized":"2023-10-09T05:52:11.192685+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6409,"log_dups_size":0,"ondisk_log_size":6409,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:37:50.679348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379912192,"num_objects":5797,"num_object_clones":0,"num_object_copies":17391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2041,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6409,"num_write_kb":390592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2221,"num_bytes_recovered":145555456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,6,7","objects":2041},{"shards":"3,7","objects":3756}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":163,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.576062+0000","last_change":"2023-10-09T05:52:26.421671+0000","last_active":"2023-10-09T05:52:59.576062+0000","last_peered":"2023-10-09T05:52:59.576062+0000","last_clean":"2023-10-09T05:52:59.576062+0000","last_became_active":"2023-10-09T05:52:11.353717+0000","last_became_peered":"2023-10-09T05:52:11.353717+0000","last_unstale":"2023-10-09T05:52:59.576062+0000","last_undegraded":"2023-10-09T05:52:59.576062+0000","last_fullsized":"2023-10-09T05:52:59.576062+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:08.999775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":159,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.583947+0000","last_change":"2023-10-09T05:52:23.642420+0000","last_active":"2023-10-09T05:52:59.583947+0000","last_peered":"2023-10-09T05:52:59.583947+0000","last_clean":"2023-10-09T05:52:59.583947+0000","last_became_active":"2023-10-09T05:52:23.641015+0000","last_became_peered":"2023-10-09T05:52:23.641015+0000","last_unstale":"2023-10-09T05:52:59.583947+0000","last_undegraded":"2023-10-09T05:52:59.583947+0000","last_fullsized":"2023-10-09T05:52:59.583947+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:14.092876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"153'6287","reported_seq":6346,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.172869+0000","last_change":"2023-10-09T05:50:39.777752+0000","last_active":"2023-10-09T05:53:09.172869+0000","last_peered":"2023-10-09T05:53:09.172869+0000","last_clean":"2023-10-09T05:53:09.172869+0000","last_became_active":"2023-10-09T05:50:39.777526+0000","last_became_peered":"2023-10-09T05:50:39.777526+0000","last_unstale":"2023-10-09T05:53:09.172869+0000","last_undegraded":"2023-10-09T05:53:09.172869+0000","last_fullsized":"2023-10-09T05:53:09.172869+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6287,"log_dups_size":0,"ondisk_log_size":6287,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:19.849370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369950720,"num_objects":5645,"num_object_clones":0,"num_object_copies":16935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6287,"num_write_kb":381824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":97,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.575449+0000","last_change":"2023-10-09T05:40:12.632083+0000","last_active":"2023-10-09T05:52:59.575449+0000","last_peered":"2023-10-09T05:52:59.575449+0000","last_clean":"2023-10-09T05:52:59.575449+0000","last_became_active":"2023-10-09T05:40:12.631715+0000","last_became_peered":"2023-10-09T05:40:12.631715+0000","last_unstale":"2023-10-09T05:52:59.575449+0000","last_undegraded":"2023-10-09T05:52:59.575449+0000","last_fullsized":"2023-10-09T05:52:59.575449+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:17.641378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":142,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.587999+0000","last_change":"2023-10-09T05:52:11.213297+0000","last_active":"2023-10-09T05:52:59.587999+0000","last_peered":"2023-10-09T05:52:59.587999+0000","last_clean":"2023-10-09T05:52:59.587999+0000","last_became_active":"2023-10-09T05:52:11.212974+0000","last_became_peered":"2023-10-09T05:52:11.212974+0000","last_unstale":"2023-10-09T05:52:59.587999+0000","last_undegraded":"2023-10-09T05:52:59.587999+0000","last_fullsized":"2023-10-09T05:52:59.587999+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:26:39.083838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"153'6309","reported_seq":6394,"reported_epoch":153,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:53:07.220765+0000","last_change":"2023-10-09T05:52:42.557805+0000","last_active":"2023-10-09T05:53:07.220765+0000","last_peered":"2023-10-09T05:53:07.220765+0000","last_clean":"2023-10-09T05:51:33.020184+0000","last_became_active":"2023-10-09T05:52:42.288853+0000","last_became_peered":"2023-10-09T05:52:42.288853+0000","last_unstale":"2023-10-09T05:53:07.220765+0000","last_undegraded":"2023-10-09T05:52:42.125072+0000","last_fullsized":"2023-10-09T05:52:24.631025+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6309,"log_dups_size":0,"ondisk_log_size":6309,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:54:46.897489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382402560,"num_objects":5835,"num_object_clones":0,"num_object_copies":17505,"num_objects_missing_on_primary":1173,"num_objects_missing":1173,"num_objects_degraded":7245,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6309,"num_write_kb":388608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,7","objects":4662},{"shards":"0,7","objects":-100},{"shards":"2,7","objects":1173}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":113,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.601225+0000","last_change":"2023-10-09T05:49:21.429234+0000","last_active":"2023-10-09T05:52:59.601225+0000","last_peered":"2023-10-09T05:52:59.601225+0000","last_clean":"2023-10-09T05:52:59.601225+0000","last_became_active":"2023-10-09T05:49:21.428961+0000","last_became_peered":"2023-10-09T05:49:21.428961+0000","last_unstale":"2023-10-09T05:52:59.601225+0000","last_undegraded":"2023-10-09T05:52:59.601225+0000","last_fullsized":"2023-10-09T05:52:59.601225+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:18.029914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":125,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.603354+0000","last_change":"2023-10-09T05:52:11.345261+0000","last_active":"2023-10-09T05:52:59.603354+0000","last_peered":"2023-10-09T05:52:59.603354+0000","last_clean":"2023-10-09T05:52:59.603354+0000","last_became_active":"2023-10-09T05:52:11.345031+0000","last_became_peered":"2023-10-09T05:52:11.345031+0000","last_unstale":"2023-10-09T05:52:59.603354+0000","last_undegraded":"2023-10-09T05:52:59.603354+0000","last_fullsized":"2023-10-09T05:52:59.603354+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:07.748821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":524,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.574973+0000","last_change":"2023-10-09T05:51:19.197324+0000","last_active":"2023-10-09T05:52:59.574973+0000","last_peered":"2023-10-09T05:52:59.574973+0000","last_clean":"2023-10-09T05:52:59.574973+0000","last_became_active":"2023-10-09T05:51:19.197052+0000","last_became_peered":"2023-10-09T05:51:19.197052+0000","last_unstale":"2023-10-09T05:52:59.574973+0000","last_undegraded":"2023-10-09T05:52:59.574973+0000","last_fullsized":"2023-10-09T05:52:59.574973+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.e","version":"153'6274","reported_seq":7548,"reported_epoch":153,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:53:09.161313+0000","last_change":"2023-10-09T05:52:24.892066+0000","last_active":"2023-10-09T05:53:09.161313+0000","last_peered":"2023-10-09T05:53:09.161313+0000","last_clean":"2023-10-09T05:51:24.573410+0000","last_became_active":"2023-10-09T05:52:24.814156+0000","last_became_peered":"2023-10-09T05:52:24.814156+0000","last_unstale":"2023-10-09T05:53:09.161313+0000","last_undegraded":"2023-10-09T05:52:24.800936+0000","last_fullsized":"2023-10-09T05:52:24.635559+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6274,"log_dups_size":0,"ondisk_log_size":6274,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:35.696825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373948416,"num_objects":5706,"num_object_clones":0,"num_object_copies":17118,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5556,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5706,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6274,"num_write_kb":383360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":591,"num_bytes_recovered":38731776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"1,2,4","objects":3298},{"shards":"1,4","objects":307},{"shards":"2,4,5","objects":2101}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":130,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.570581+0000","last_change":"2023-10-09T05:51:57.092524+0000","last_active":"2023-10-09T05:52:59.570581+0000","last_peered":"2023-10-09T05:52:59.570581+0000","last_clean":"2023-10-09T05:52:59.570581+0000","last_became_active":"2023-10-09T05:51:57.092273+0000","last_became_peered":"2023-10-09T05:51:57.092273+0000","last_unstale":"2023-10-09T05:52:59.570581+0000","last_undegraded":"2023-10-09T05:52:59.570581+0000","last_fullsized":"2023-10-09T05:52:59.570581+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:56.479664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":112,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.584313+0000","last_change":"2023-10-09T05:49:21.430742+0000","last_active":"2023-10-09T05:52:59.584313+0000","last_peered":"2023-10-09T05:52:59.584313+0000","last_clean":"2023-10-09T05:52:59.584313+0000","last_became_active":"2023-10-09T05:49:21.430233+0000","last_became_peered":"2023-10-09T05:49:21.430233+0000","last_unstale":"2023-10-09T05:52:59.584313+0000","last_undegraded":"2023-10-09T05:52:59.584313+0000","last_fullsized":"2023-10-09T05:52:59.584313+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:10.541302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":457,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.600200+0000","last_change":"2023-10-09T05:51:19.193204+0000","last_active":"2023-10-09T05:52:59.600200+0000","last_peered":"2023-10-09T05:52:59.600200+0000","last_clean":"2023-10-09T05:52:59.600200+0000","last_became_active":"2023-10-09T05:51:19.192660+0000","last_became_peered":"2023-10-09T05:51:19.192660+0000","last_unstale":"2023-10-09T05:52:59.600200+0000","last_undegraded":"2023-10-09T05:52:59.600200+0000","last_fullsized":"2023-10-09T05:52:59.600200+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.f","version":"153'6411","reported_seq":6473,"reported_epoch":153,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:53:09.290864+0000","last_change":"2023-10-09T05:52:31.004567+0000","last_active":"2023-10-09T05:53:09.290864+0000","last_peered":"2023-10-09T05:53:09.290864+0000","last_clean":"2023-10-09T05:52:22.632757+0000","last_became_active":"2023-10-09T05:52:30.992047+0000","last_became_peered":"2023-10-09T05:52:30.992047+0000","last_unstale":"2023-10-09T05:53:09.290864+0000","last_undegraded":"2023-10-09T05:52:30.768811+0000","last_fullsized":"2023-10-09T05:52:30.724937+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6411,"log_dups_size":0,"ondisk_log_size":6411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:40.469088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384761856,"num_objects":5871,"num_object_clones":0,"num_object_copies":17613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6141,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6411,"num_write_kb":393024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":5871},{"shards":"3,4","objects":-130}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":133,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.587927+0000","last_change":"2023-10-09T05:52:11.344035+0000","last_active":"2023-10-09T05:52:59.587927+0000","last_peered":"2023-10-09T05:52:59.587927+0000","last_clean":"2023-10-09T05:52:59.587927+0000","last_became_active":"2023-10-09T05:52:11.343800+0000","last_became_peered":"2023-10-09T05:52:11.343800+0000","last_unstale":"2023-10-09T05:52:59.587927+0000","last_undegraded":"2023-10-09T05:52:59.587927+0000","last_fullsized":"2023-10-09T05:52:59.587927+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:07.588214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":149,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.584442+0000","last_change":"2023-10-09T05:52:11.419040+0000","last_active":"2023-10-09T05:52:59.584442+0000","last_peered":"2023-10-09T05:52:59.584442+0000","last_clean":"2023-10-09T05:52:59.584442+0000","last_became_active":"2023-10-09T05:52:11.418708+0000","last_became_peered":"2023-10-09T05:52:11.418708+0000","last_unstale":"2023-10-09T05:52:59.584442+0000","last_undegraded":"2023-10-09T05:52:59.584442+0000","last_fullsized":"2023-10-09T05:52:59.584442+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:22.969249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"153'6342","reported_seq":14313,"reported_epoch":153,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:53:09.451723+0000","last_change":"2023-10-09T05:52:24.904553+0000","last_active":"2023-10-09T05:53:09.451723+0000","last_peered":"2023-10-09T05:53:09.451723+0000","last_clean":"2023-10-09T05:52:22.567157+0000","last_became_active":"2023-10-09T05:52:24.871228+0000","last_became_peered":"2023-10-09T05:52:24.871228+0000","last_unstale":"2023-10-09T05:53:09.451723+0000","last_undegraded":"2023-10-09T05:52:24.677767+0000","last_fullsized":"2023-10-09T05:52:24.631791+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6342,"log_dups_size":0,"ondisk_log_size":6342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:42.864852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375128064,"num_objects":5724,"num_object_clones":0,"num_object_copies":17172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":1930,"num_objects_unfound":0,"num_objects_dirty":5724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6342,"num_write_kb":386112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3951,"num_bytes_recovered":258801664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5],"avail_no_missing":["6","3","5"],"object_location_counts":[{"shards":"3,5,6","objects":1930},{"shards":"5,6","objects":3793}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":113,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.583832+0000","last_change":"2023-10-09T05:48:49.141287+0000","last_active":"2023-10-09T05:52:59.583832+0000","last_peered":"2023-10-09T05:52:59.583832+0000","last_clean":"2023-10-09T05:52:59.583832+0000","last_became_active":"2023-10-09T05:48:49.140950+0000","last_became_peered":"2023-10-09T05:48:49.140950+0000","last_unstale":"2023-10-09T05:52:59.583832+0000","last_undegraded":"2023-10-09T05:52:59.583832+0000","last_fullsized":"2023-10-09T05:52:59.583832+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:35.527434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":132,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.584918+0000","last_change":"2023-10-09T05:52:11.412851+0000","last_active":"2023-10-09T05:52:59.584918+0000","last_peered":"2023-10-09T05:52:59.584918+0000","last_clean":"2023-10-09T05:52:59.584918+0000","last_became_active":"2023-10-09T05:52:11.412489+0000","last_became_peered":"2023-10-09T05:52:11.412489+0000","last_unstale":"2023-10-09T05:52:59.584918+0000","last_undegraded":"2023-10-09T05:52:59.584918+0000","last_fullsized":"2023-10-09T05:52:59.584918+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:30:34.286409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":457,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.600423+0000","last_change":"2023-10-09T05:51:19.195984+0000","last_active":"2023-10-09T05:52:59.600423+0000","last_peered":"2023-10-09T05:52:59.600423+0000","last_clean":"2023-10-09T05:52:59.600423+0000","last_became_active":"2023-10-09T05:51:19.195632+0000","last_became_peered":"2023-10-09T05:51:19.195632+0000","last_unstale":"2023-10-09T05:52:59.600423+0000","last_undegraded":"2023-10-09T05:52:59.600423+0000","last_fullsized":"2023-10-09T05:52:59.600423+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.d","version":"153'6303","reported_seq":11535,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:53:08.576713+0000","last_change":"2023-10-09T05:52:27.708167+0000","last_active":"2023-10-09T05:53:08.576713+0000","last_peered":"2023-10-09T05:53:08.576713+0000","last_clean":"2023-10-09T05:53:08.576713+0000","last_became_active":"2023-10-09T05:52:27.701168+0000","last_became_peered":"2023-10-09T05:52:27.701168+0000","last_unstale":"2023-10-09T05:53:08.576713+0000","last_undegraded":"2023-10-09T05:53:08.576713+0000","last_fullsized":"2023-10-09T05:53:08.576713+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6303,"log_dups_size":0,"ondisk_log_size":6303,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:17.406250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373555224,"num_objects":5701,"num_object_clones":0,"num_object_copies":17103,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5701,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6303,"num_write_kb":384065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2559,"num_bytes_recovered":167706624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":134,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.571124+0000","last_change":"2023-10-09T05:52:23.637306+0000","last_active":"2023-10-09T05:52:59.571124+0000","last_peered":"2023-10-09T05:52:59.571124+0000","last_clean":"2023-10-09T05:52:59.571124+0000","last_became_active":"2023-10-09T05:52:23.636975+0000","last_became_peered":"2023-10-09T05:52:23.636975+0000","last_unstale":"2023-10-09T05:52:59.571124+0000","last_undegraded":"2023-10-09T05:52:59.571124+0000","last_fullsized":"2023-10-09T05:52:59.571124+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:23.820825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":30,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.572924+0000","last_change":"2023-10-09T05:52:11.409470+0000","last_active":"2023-10-09T05:52:59.572924+0000","last_peered":"2023-10-09T05:52:59.572924+0000","last_clean":"2023-10-09T05:52:59.572924+0000","last_became_active":"2023-10-09T05:52:11.409125+0000","last_became_peered":"2023-10-09T05:52:11.409125+0000","last_unstale":"2023-10-09T05:52:59.572924+0000","last_undegraded":"2023-10-09T05:52:59.572924+0000","last_fullsized":"2023-10-09T05:52:59.572924+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:39:35.220346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":486,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.588099+0000","last_change":"2023-10-09T05:52:26.673774+0000","last_active":"2023-10-09T05:52:59.588099+0000","last_peered":"2023-10-09T05:52:59.588099+0000","last_clean":"2023-10-09T05:52:59.588099+0000","last_became_active":"2023-10-09T05:52:26.673393+0000","last_became_peered":"2023-10-09T05:52:26.673393+0000","last_unstale":"2023-10-09T05:52:59.588099+0000","last_undegraded":"2023-10-09T05:52:59.588099+0000","last_fullsized":"2023-10-09T05:52:59.588099+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":21,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.699491+0000","last_change":"2023-10-09T05:52:11.226922+0000","last_active":"2023-10-09T05:52:59.699491+0000","last_peered":"2023-10-09T05:52:59.699491+0000","last_clean":"2023-10-09T05:52:59.699491+0000","last_became_active":"2023-10-09T05:52:11.226605+0000","last_became_peered":"2023-10-09T05:52:11.226605+0000","last_unstale":"2023-10-09T05:52:59.699491+0000","last_undegraded":"2023-10-09T05:52:59.699491+0000","last_fullsized":"2023-10-09T05:52:59.699491+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:58.006329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":140,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.603314+0000","last_change":"2023-10-09T05:52:29.735372+0000","last_active":"2023-10-09T05:52:59.603314+0000","last_peered":"2023-10-09T05:52:59.603314+0000","last_clean":"2023-10-09T05:52:59.603314+0000","last_became_active":"2023-10-09T05:52:29.735026+0000","last_became_peered":"2023-10-09T05:52:29.735026+0000","last_unstale":"2023-10-09T05:52:59.603314+0000","last_undegraded":"2023-10-09T05:52:59.603314+0000","last_fullsized":"2023-10-09T05:52:59.603314+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:53.849406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":30,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.570905+0000","last_change":"2023-10-09T05:52:11.407985+0000","last_active":"2023-10-09T05:52:59.570905+0000","last_peered":"2023-10-09T05:52:59.570905+0000","last_clean":"2023-10-09T05:52:59.570905+0000","last_became_active":"2023-10-09T05:52:11.407681+0000","last_became_peered":"2023-10-09T05:52:11.407681+0000","last_unstale":"2023-10-09T05:52:59.570905+0000","last_undegraded":"2023-10-09T05:52:59.570905+0000","last_fullsized":"2023-10-09T05:52:59.570905+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:54.384669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"153'6374","reported_seq":6444,"reported_epoch":153,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:53:07.279972+0000","last_change":"2023-10-09T05:51:58.262986+0000","last_active":"2023-10-09T05:53:07.279972+0000","last_peered":"2023-10-09T05:53:07.279972+0000","last_clean":"2023-10-09T05:51:33.087654+0000","last_became_active":"2023-10-09T05:51:58.252808+0000","last_became_peered":"2023-10-09T05:51:58.252808+0000","last_unstale":"2023-10-09T05:53:07.279972+0000","last_undegraded":"2023-10-09T05:51:58.088414+0000","last_fullsized":"2023-10-09T05:51:58.077844+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6374,"log_dups_size":0,"ondisk_log_size":6374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:00.878041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378667008,"num_objects":5778,"num_object_clones":0,"num_object_copies":17334,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1172,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5778,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6374,"num_write_kb":388864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":4606}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":474,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.600321+0000","last_change":"2023-10-09T05:51:57.209470+0000","last_active":"2023-10-09T05:52:59.600321+0000","last_peered":"2023-10-09T05:52:59.600321+0000","last_clean":"2023-10-09T05:52:59.600321+0000","last_became_active":"2023-10-09T05:51:57.208993+0000","last_became_peered":"2023-10-09T05:51:57.208993+0000","last_unstale":"2023-10-09T05:52:59.600321+0000","last_undegraded":"2023-10-09T05:52:59.600321+0000","last_fullsized":"2023-10-09T05:52:59.600321+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.6","version":"153'6340","reported_seq":6406,"reported_epoch":153,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:53:09.271214+0000","last_change":"2023-10-09T05:52:30.983730+0000","last_active":"2023-10-09T05:53:09.271214+0000","last_peered":"2023-10-09T05:53:09.271214+0000","last_clean":"2023-10-09T05:52:24.946135+0000","last_became_active":"2023-10-09T05:52:30.971337+0000","last_became_peered":"2023-10-09T05:52:30.971337+0000","last_unstale":"2023-10-09T05:53:09.271214+0000","last_undegraded":"2023-10-09T05:52:30.773579+0000","last_fullsized":"2023-10-09T05:52:30.746434+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6340,"log_dups_size":0,"ondisk_log_size":6340,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:04:00.772909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373817344,"num_objects":5704,"num_object_clones":0,"num_object_copies":17112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6022,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6340,"num_write_kb":385408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":5704},{"shards":"4,7","objects":-141}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":154,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.600629+0000","last_change":"2023-10-09T05:52:11.217079+0000","last_active":"2023-10-09T05:52:59.600629+0000","last_peered":"2023-10-09T05:52:59.600629+0000","last_clean":"2023-10-09T05:52:59.600629+0000","last_became_active":"2023-10-09T05:52:11.216737+0000","last_became_peered":"2023-10-09T05:52:11.216737+0000","last_unstale":"2023-10-09T05:52:59.600629+0000","last_undegraded":"2023-10-09T05:52:59.600629+0000","last_fullsized":"2023-10-09T05:52:59.600629+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:04.493525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":130,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.584131+0000","last_change":"2023-10-09T05:51:57.203142+0000","last_active":"2023-10-09T05:52:59.584131+0000","last_peered":"2023-10-09T05:52:59.584131+0000","last_clean":"2023-10-09T05:52:59.584131+0000","last_became_active":"2023-10-09T05:51:57.202850+0000","last_became_peered":"2023-10-09T05:51:57.202850+0000","last_unstale":"2023-10-09T05:52:59.584131+0000","last_undegraded":"2023-10-09T05:52:59.584131+0000","last_fullsized":"2023-10-09T05:52:59.584131+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:51.090737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":30,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.699439+0000","last_change":"2023-10-09T05:52:23.645442+0000","last_active":"2023-10-09T05:52:59.699439+0000","last_peered":"2023-10-09T05:52:59.699439+0000","last_clean":"2023-10-09T05:52:59.699439+0000","last_became_active":"2023-10-09T05:52:23.645170+0000","last_became_peered":"2023-10-09T05:52:23.645170+0000","last_unstale":"2023-10-09T05:52:59.699439+0000","last_undegraded":"2023-10-09T05:52:59.699439+0000","last_fullsized":"2023-10-09T05:52:59.699439+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:26.222480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":541,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.575113+0000","last_change":"2023-10-09T05:51:57.221078+0000","last_active":"2023-10-09T05:52:59.575113+0000","last_peered":"2023-10-09T05:52:59.575113+0000","last_clean":"2023-10-09T05:52:59.575113+0000","last_became_active":"2023-10-09T05:51:57.220754+0000","last_became_peered":"2023-10-09T05:51:57.220754+0000","last_unstale":"2023-10-09T05:52:59.575113+0000","last_undegraded":"2023-10-09T05:52:59.575113+0000","last_fullsized":"2023-10-09T05:52:59.575113+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.5","version":"153'6197","reported_seq":6256,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.274338+0000","last_change":"2023-10-09T05:50:40.123175+0000","last_active":"2023-10-09T05:53:09.274338+0000","last_peered":"2023-10-09T05:53:09.274338+0000","last_clean":"2023-10-09T05:53:09.274338+0000","last_became_active":"2023-10-09T05:50:40.122928+0000","last_became_peered":"2023-10-09T05:50:40.122928+0000","last_unstale":"2023-10-09T05:53:09.274338+0000","last_undegraded":"2023-10-09T05:53:09.274338+0000","last_fullsized":"2023-10-09T05:53:09.274338+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6197,"log_dups_size":0,"ondisk_log_size":6197,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:16.111485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367460352,"num_objects":5607,"num_object_clones":0,"num_object_copies":16821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6197,"num_write_kb":377728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":161,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.623616+0000","last_change":"2023-10-09T05:52:23.638460+0000","last_active":"2023-10-09T05:52:59.623616+0000","last_peered":"2023-10-09T05:52:59.623616+0000","last_clean":"2023-10-09T05:52:59.623616+0000","last_became_active":"2023-10-09T05:52:23.637993+0000","last_became_peered":"2023-10-09T05:52:23.637993+0000","last_unstale":"2023-10-09T05:52:59.623616+0000","last_undegraded":"2023-10-09T05:52:59.623616+0000","last_fullsized":"2023-10-09T05:52:59.623616+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:35.191837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":30,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.571049+0000","last_change":"2023-10-09T05:52:11.209763+0000","last_active":"2023-10-09T05:52:59.571049+0000","last_peered":"2023-10-09T05:52:59.571049+0000","last_clean":"2023-10-09T05:52:59.571049+0000","last_became_active":"2023-10-09T05:52:11.209472+0000","last_became_peered":"2023-10-09T05:52:11.209472+0000","last_unstale":"2023-10-09T05:52:59.571049+0000","last_undegraded":"2023-10-09T05:52:59.571049+0000","last_fullsized":"2023-10-09T05:52:59.571049+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:05.160815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":30,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.572993+0000","last_change":"2023-10-09T05:52:11.234634+0000","last_active":"2023-10-09T05:52:59.572993+0000","last_peered":"2023-10-09T05:52:59.572993+0000","last_clean":"2023-10-09T05:52:59.572993+0000","last_became_active":"2023-10-09T05:52:11.234307+0000","last_became_peered":"2023-10-09T05:52:11.234307+0000","last_unstale":"2023-10-09T05:52:59.572993+0000","last_undegraded":"2023-10-09T05:52:59.572993+0000","last_fullsized":"2023-10-09T05:52:59.572993+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:37.064308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"153'6216","reported_seq":6275,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.257551+0000","last_change":"2023-10-09T05:50:39.778086+0000","last_active":"2023-10-09T05:53:09.257551+0000","last_peered":"2023-10-09T05:53:09.257551+0000","last_clean":"2023-10-09T05:53:09.257551+0000","last_became_active":"2023-10-09T05:50:39.777777+0000","last_became_peered":"2023-10-09T05:50:39.777777+0000","last_unstale":"2023-10-09T05:53:09.257551+0000","last_undegraded":"2023-10-09T05:53:09.257551+0000","last_fullsized":"2023-10-09T05:53:09.257551+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6216,"log_dups_size":0,"ondisk_log_size":6216,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:02.350198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373817344,"num_objects":5704,"num_object_clones":0,"num_object_copies":17112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6216,"num_write_kb":381440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":117,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.600320+0000","last_change":"2023-10-09T05:48:49.141207+0000","last_active":"2023-10-09T05:52:59.600320+0000","last_peered":"2023-10-09T05:52:59.600320+0000","last_clean":"2023-10-09T05:52:59.600320+0000","last_became_active":"2023-10-09T05:48:49.140798+0000","last_became_peered":"2023-10-09T05:48:49.140798+0000","last_unstale":"2023-10-09T05:52:59.600320+0000","last_undegraded":"2023-10-09T05:52:59.600320+0000","last_fullsized":"2023-10-09T05:52:59.600320+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:44:13.125030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":532,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.575387+0000","last_change":"2023-10-09T05:51:57.111994+0000","last_active":"2023-10-09T05:52:59.575387+0000","last_peered":"2023-10-09T05:52:59.575387+0000","last_clean":"2023-10-09T05:52:59.575387+0000","last_became_active":"2023-10-09T05:51:57.111617+0000","last_became_peered":"2023-10-09T05:51:57.111617+0000","last_unstale":"2023-10-09T05:52:59.575387+0000","last_undegraded":"2023-10-09T05:52:59.575387+0000","last_fullsized":"2023-10-09T05:52:59.575387+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:51:12.346243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:51:12.346243+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:45.587863+0000","scrub_duration":0.0027096379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":40,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.600681+0000","last_change":"2023-10-09T05:52:23.648626+0000","last_active":"2023-10-09T05:52:59.600681+0000","last_peered":"2023-10-09T05:52:59.600681+0000","last_clean":"2023-10-09T05:52:59.600681+0000","last_became_active":"2023-10-09T05:52:23.646143+0000","last_became_peered":"2023-10-09T05:52:23.646143+0000","last_unstale":"2023-10-09T05:52:59.600681+0000","last_undegraded":"2023-10-09T05:52:59.600681+0000","last_fullsized":"2023-10-09T05:52:59.600681+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:51:04.733282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":95,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.587278+0000","last_change":"2023-10-09T05:40:14.363125+0000","last_active":"2023-10-09T05:52:59.587278+0000","last_peered":"2023-10-09T05:52:59.587278+0000","last_clean":"2023-10-09T05:52:59.587278+0000","last_became_active":"2023-10-09T05:40:14.362723+0000","last_became_peered":"2023-10-09T05:40:14.362723+0000","last_unstale":"2023-10-09T05:52:59.587278+0000","last_undegraded":"2023-10-09T05:52:59.587278+0000","last_fullsized":"2023-10-09T05:52:59.587278+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:30.741318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":438,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.572276+0000","last_change":"2023-10-09T05:52:14.275995+0000","last_active":"2023-10-09T05:52:59.572276+0000","last_peered":"2023-10-09T05:52:59.572276+0000","last_clean":"2023-10-09T05:52:59.572276+0000","last_became_active":"2023-10-09T05:52:14.275584+0000","last_became_peered":"2023-10-09T05:52:14.275584+0000","last_unstale":"2023-10-09T05:52:59.572276+0000","last_undegraded":"2023-10-09T05:52:59.572276+0000","last_fullsized":"2023-10-09T05:52:59.572276+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":171,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.575308+0000","last_change":"2023-10-09T05:52:11.209995+0000","last_active":"2023-10-09T05:52:59.575308+0000","last_peered":"2023-10-09T05:52:59.575308+0000","last_clean":"2023-10-09T05:52:59.575308+0000","last_became_active":"2023-10-09T05:52:11.209715+0000","last_became_peered":"2023-10-09T05:52:11.209715+0000","last_unstale":"2023-10-09T05:52:59.575308+0000","last_undegraded":"2023-10-09T05:52:59.575308+0000","last_fullsized":"2023-10-09T05:52:59.575308+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:13.217575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":92,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.603675+0000","last_change":"2023-10-09T05:52:23.643245+0000","last_active":"2023-10-09T05:52:59.603675+0000","last_peered":"2023-10-09T05:52:59.603675+0000","last_clean":"2023-10-09T05:52:59.603675+0000","last_became_active":"2023-10-09T05:52:23.642947+0000","last_became_peered":"2023-10-09T05:52:23.642947+0000","last_unstale":"2023-10-09T05:52:59.603675+0000","last_undegraded":"2023-10-09T05:52:59.603675+0000","last_fullsized":"2023-10-09T05:52:59.603675+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:52.260774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":150,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.604013+0000","last_change":"2023-10-09T05:52:29.734403+0000","last_active":"2023-10-09T05:52:59.604013+0000","last_peered":"2023-10-09T05:52:59.604013+0000","last_clean":"2023-10-09T05:52:59.604013+0000","last_became_active":"2023-10-09T05:52:29.734046+0000","last_became_peered":"2023-10-09T05:52:29.734046+0000","last_unstale":"2023-10-09T05:52:59.604013+0000","last_undegraded":"2023-10-09T05:52:59.604013+0000","last_fullsized":"2023-10-09T05:52:59.604013+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:53.346403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"153'6352","reported_seq":6432,"reported_epoch":153,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:53:09.687718+0000","last_change":"2023-10-09T05:52:51.733590+0000","last_active":"2023-10-09T05:53:09.687718+0000","last_peered":"2023-10-09T05:53:09.687718+0000","last_clean":"2023-10-09T05:51:24.557317+0000","last_became_active":"2023-10-09T05:52:51.422504+0000","last_became_peered":"2023-10-09T05:52:51.422504+0000","last_unstale":"2023-10-09T05:53:09.687718+0000","last_undegraded":"2023-10-09T05:52:51.194765+0000","last_fullsized":"2023-10-09T05:52:24.632398+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6352,"log_dups_size":0,"ondisk_log_size":6352,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:07.573528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377225216,"num_objects":5756,"num_object_clones":0,"num_object_copies":17268,"num_objects_missing_on_primary":2109,"num_objects_missing":2109,"num_objects_degraded":6054,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5756,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6352,"num_write_kb":387456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"1,3,5","objects":3647},{"shards":"1,5","objects":-148},{"shards":"3,5,6","objects":2109}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":503,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.587644+0000","last_change":"2023-10-09T05:52:26.679997+0000","last_active":"2023-10-09T05:52:59.587644+0000","last_peered":"2023-10-09T05:52:59.587644+0000","last_clean":"2023-10-09T05:52:59.587644+0000","last_became_active":"2023-10-09T05:52:26.679641+0000","last_became_peered":"2023-10-09T05:52:26.679641+0000","last_unstale":"2023-10-09T05:52:59.587644+0000","last_undegraded":"2023-10-09T05:52:59.587644+0000","last_fullsized":"2023-10-09T05:52:59.587644+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":134,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.575523+0000","last_change":"2023-10-09T05:51:57.112837+0000","last_active":"2023-10-09T05:52:59.575523+0000","last_peered":"2023-10-09T05:52:59.575523+0000","last_clean":"2023-10-09T05:52:59.575523+0000","last_became_active":"2023-10-09T05:51:57.112133+0000","last_became_peered":"2023-10-09T05:51:57.112133+0000","last_unstale":"2023-10-09T05:52:59.575523+0000","last_undegraded":"2023-10-09T05:52:59.575523+0000","last_fullsized":"2023-10-09T05:52:59.575523+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:27:43.538434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":30,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.699628+0000","last_change":"2023-10-09T05:52:53.923884+0000","last_active":"2023-10-09T05:52:59.699628+0000","last_peered":"2023-10-09T05:52:59.699628+0000","last_clean":"2023-10-09T05:52:59.699628+0000","last_became_active":"2023-10-09T05:52:53.923324+0000","last_became_peered":"2023-10-09T05:52:53.923324+0000","last_unstale":"2023-10-09T05:52:59.699628+0000","last_undegraded":"2023-10-09T05:52:59.699628+0000","last_fullsized":"2023-10-09T05:52:59.699628+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:28.764537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":58,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.603494+0000","last_change":"2023-10-09T05:51:57.210829+0000","last_active":"2023-10-09T05:52:59.603494+0000","last_peered":"2023-10-09T05:52:59.603494+0000","last_clean":"2023-10-09T05:52:59.603494+0000","last_became_active":"2023-10-09T05:51:57.209398+0000","last_became_peered":"2023-10-09T05:51:57.209398+0000","last_unstale":"2023-10-09T05:52:59.603494+0000","last_undegraded":"2023-10-09T05:52:59.603494+0000","last_fullsized":"2023-10-09T05:52:59.603494+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:44.351726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"153'6376","reported_seq":6458,"reported_epoch":153,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:53:09.499160+0000","last_change":"2023-10-09T05:52:53.957304+0000","last_active":"2023-10-09T05:53:09.499160+0000","last_peered":"2023-10-09T05:53:09.499160+0000","last_clean":"2023-10-09T05:51:24.563560+0000","last_became_active":"2023-10-09T05:52:53.702631+0000","last_became_peered":"2023-10-09T05:52:53.702631+0000","last_unstale":"2023-10-09T05:53:09.499160+0000","last_undegraded":"2023-10-09T05:52:46.592132+0000","last_fullsized":"2023-10-09T05:52:24.632845+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6376,"log_dups_size":0,"ondisk_log_size":6376,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:15.931450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382468096,"num_objects":5836,"num_object_clones":0,"num_object_copies":17508,"num_objects_missing_on_primary":2121,"num_objects_missing":2121,"num_objects_degraded":6104,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6376,"num_write_kb":390784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"1,2,6","objects":3715},{"shards":"1,6","objects":-126},{"shards":"2,6,7","objects":2121}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":477,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.584351+0000","last_change":"2023-10-09T05:52:26.676938+0000","last_active":"2023-10-09T05:52:59.584351+0000","last_peered":"2023-10-09T05:52:59.584351+0000","last_clean":"2023-10-09T05:52:59.584351+0000","last_became_active":"2023-10-09T05:52:26.676588+0000","last_became_peered":"2023-10-09T05:52:26.676588+0000","last_unstale":"2023-10-09T05:52:59.584351+0000","last_undegraded":"2023-10-09T05:52:59.584351+0000","last_fullsized":"2023-10-09T05:52:59.584351+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":130,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.567073+0000","last_change":"2023-10-09T05:51:57.112493+0000","last_active":"2023-10-09T05:52:59.567073+0000","last_peered":"2023-10-09T05:52:59.567073+0000","last_clean":"2023-10-09T05:52:59.567073+0000","last_became_active":"2023-10-09T05:51:57.112153+0000","last_became_peered":"2023-10-09T05:51:57.112153+0000","last_unstale":"2023-10-09T05:52:59.567073+0000","last_undegraded":"2023-10-09T05:52:59.567073+0000","last_fullsized":"2023-10-09T05:52:59.567073+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:29.925231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":30,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.572755+0000","last_change":"2023-10-09T05:52:11.415624+0000","last_active":"2023-10-09T05:52:59.572755+0000","last_peered":"2023-10-09T05:52:59.572755+0000","last_clean":"2023-10-09T05:52:59.572755+0000","last_became_active":"2023-10-09T05:52:11.415174+0000","last_became_peered":"2023-10-09T05:52:11.415174+0000","last_unstale":"2023-10-09T05:52:59.572755+0000","last_undegraded":"2023-10-09T05:52:59.572755+0000","last_fullsized":"2023-10-09T05:52:59.572755+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:13.657987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":122,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.575520+0000","last_change":"2023-10-09T05:52:23.639628+0000","last_active":"2023-10-09T05:52:59.575520+0000","last_peered":"2023-10-09T05:52:59.575520+0000","last_clean":"2023-10-09T05:52:59.575520+0000","last_became_active":"2023-10-09T05:52:23.639286+0000","last_became_peered":"2023-10-09T05:52:23.639286+0000","last_unstale":"2023-10-09T05:52:59.575520+0000","last_undegraded":"2023-10-09T05:52:59.575520+0000","last_fullsized":"2023-10-09T05:52:59.575520+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:21.450509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.2","version":"153'6358","reported_seq":13385,"reported_epoch":153,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:53:09.464811+0000","last_change":"2023-10-09T05:52:24.898238+0000","last_active":"2023-10-09T05:53:09.464811+0000","last_peered":"2023-10-09T05:53:09.464811+0000","last_clean":"2023-10-09T05:52:22.629746+0000","last_became_active":"2023-10-09T05:52:24.861945+0000","last_became_peered":"2023-10-09T05:52:24.861945+0000","last_unstale":"2023-10-09T05:53:09.464811+0000","last_undegraded":"2023-10-09T05:52:24.671738+0000","last_fullsized":"2023-10-09T05:52:24.632223+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6358,"log_dups_size":0,"ondisk_log_size":6358,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:37.391764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372244480,"num_objects":5680,"num_object_clones":0,"num_object_copies":17040,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":2358,"num_objects_unfound":0,"num_objects_dirty":5680,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6358,"num_write_kb":385216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3479,"num_bytes_recovered":227934208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"2,3,4","objects":2358},{"shards":"2,4","objects":3319}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":456,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.566552+0000","last_change":"2023-10-09T05:51:19.193833+0000","last_active":"2023-10-09T05:52:59.566552+0000","last_peered":"2023-10-09T05:52:59.566552+0000","last_clean":"2023-10-09T05:52:59.566552+0000","last_became_active":"2023-10-09T05:51:19.192996+0000","last_became_peered":"2023-10-09T05:51:19.192996+0000","last_unstale":"2023-10-09T05:52:59.566552+0000","last_undegraded":"2023-10-09T05:52:59.566552+0000","last_fullsized":"2023-10-09T05:52:59.566552+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":488690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":117,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.588037+0000","last_change":"2023-10-09T05:48:49.137565+0000","last_active":"2023-10-09T05:52:59.588037+0000","last_peered":"2023-10-09T05:52:59.588037+0000","last_clean":"2023-10-09T05:52:59.588037+0000","last_became_active":"2023-10-09T05:48:49.137099+0000","last_became_peered":"2023-10-09T05:48:49.137099+0000","last_unstale":"2023-10-09T05:52:59.588037+0000","last_undegraded":"2023-10-09T05:52:59.588037+0000","last_fullsized":"2023-10-09T05:52:59.588037+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:28.103821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":138,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.603869+0000","last_change":"2023-10-09T05:52:11.411584+0000","last_active":"2023-10-09T05:52:59.603869+0000","last_peered":"2023-10-09T05:52:59.603869+0000","last_clean":"2023-10-09T05:52:59.603869+0000","last_became_active":"2023-10-09T05:52:11.402696+0000","last_became_peered":"2023-10-09T05:52:11.402696+0000","last_unstale":"2023-10-09T05:52:59.603869+0000","last_undegraded":"2023-10-09T05:52:59.603869+0000","last_fullsized":"2023-10-09T05:52:59.603869+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:22.747118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":95,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.583856+0000","last_change":"2023-10-09T05:40:12.676587+0000","last_active":"2023-10-09T05:52:59.583856+0000","last_peered":"2023-10-09T05:52:59.583856+0000","last_clean":"2023-10-09T05:52:59.583856+0000","last_became_active":"2023-10-09T05:40:12.676330+0000","last_became_peered":"2023-10-09T05:40:12.676330+0000","last_unstale":"2023-10-09T05:52:59.583856+0000","last_undegraded":"2023-10-09T05:52:59.583856+0000","last_fullsized":"2023-10-09T05:52:59.583856+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:46.659556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"153'6344","reported_seq":6403,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-09T05:53:09.254955+0000","last_change":"2023-10-09T05:50:40.123172+0000","last_active":"2023-10-09T05:53:09.254955+0000","last_peered":"2023-10-09T05:53:09.254955+0000","last_clean":"2023-10-09T05:53:09.254955+0000","last_became_active":"2023-10-09T05:50:40.122867+0000","last_became_peered":"2023-10-09T05:50:40.122867+0000","last_unstale":"2023-10-09T05:53:09.254955+0000","last_undegraded":"2023-10-09T05:53:09.254955+0000","last_fullsized":"2023-10-09T05:53:09.254955+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6344,"log_dups_size":0,"ondisk_log_size":6344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:16.038279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381419520,"num_objects":5820,"num_object_clones":0,"num_object_copies":17460,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5820,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6344,"num_write_kb":389248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":62,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.603209+0000","last_change":"2023-10-09T05:51:57.209458+0000","last_active":"2023-10-09T05:52:59.603209+0000","last_peered":"2023-10-09T05:52:59.603209+0000","last_clean":"2023-10-09T05:52:59.603209+0000","last_became_active":"2023-10-09T05:51:57.208070+0000","last_became_peered":"2023-10-09T05:51:57.208070+0000","last_unstale":"2023-10-09T05:52:59.603209+0000","last_undegraded":"2023-10-09T05:52:59.603209+0000","last_fullsized":"2023-10-09T05:52:59.603209+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:30.631238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":166,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.699537+0000","last_change":"2023-10-09T05:52:53.923635+0000","last_active":"2023-10-09T05:52:59.699537+0000","last_peered":"2023-10-09T05:52:59.699537+0000","last_clean":"2023-10-09T05:52:59.699537+0000","last_became_active":"2023-10-09T05:52:53.922868+0000","last_became_peered":"2023-10-09T05:52:53.922868+0000","last_unstale":"2023-10-09T05:52:59.699537+0000","last_undegraded":"2023-10-09T05:52:59.699537+0000","last_fullsized":"2023-10-09T05:52:59.699537+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:32.418426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":143,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.601133+0000","last_change":"2023-10-09T05:52:11.422464+0000","last_active":"2023-10-09T05:52:59.601133+0000","last_peered":"2023-10-09T05:52:59.601133+0000","last_clean":"2023-10-09T05:52:59.601133+0000","last_became_active":"2023-10-09T05:52:11.341825+0000","last_became_peered":"2023-10-09T05:52:11.341825+0000","last_unstale":"2023-10-09T05:52:59.601133+0000","last_undegraded":"2023-10-09T05:52:59.601133+0000","last_fullsized":"2023-10-09T05:52:59.601133+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:48.933107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":149,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.584173+0000","last_change":"2023-10-09T05:52:11.425354+0000","last_active":"2023-10-09T05:52:59.584173+0000","last_peered":"2023-10-09T05:52:59.584173+0000","last_clean":"2023-10-09T05:52:59.584173+0000","last_became_active":"2023-10-09T05:52:11.425006+0000","last_became_peered":"2023-10-09T05:52:11.425006+0000","last_unstale":"2023-10-09T05:52:59.584173+0000","last_undegraded":"2023-10-09T05:52:59.584173+0000","last_fullsized":"2023-10-09T05:52:59.584173+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:07.273992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":98,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.603364+0000","last_change":"2023-10-09T05:40:12.633195+0000","last_active":"2023-10-09T05:52:59.603364+0000","last_peered":"2023-10-09T05:52:59.603364+0000","last_clean":"2023-10-09T05:52:59.603364+0000","last_became_active":"2023-10-09T05:40:12.632942+0000","last_became_peered":"2023-10-09T05:40:12.632942+0000","last_unstale":"2023-10-09T05:52:59.603364+0000","last_undegraded":"2023-10-09T05:52:59.603364+0000","last_fullsized":"2023-10-09T05:52:59.603364+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:04.470685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":146,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.583985+0000","last_change":"2023-10-09T05:51:57.093104+0000","last_active":"2023-10-09T05:52:59.583985+0000","last_peered":"2023-10-09T05:52:59.583985+0000","last_clean":"2023-10-09T05:52:59.583985+0000","last_became_active":"2023-10-09T05:51:57.092810+0000","last_became_peered":"2023-10-09T05:51:57.092810+0000","last_unstale":"2023-10-09T05:52:59.583985+0000","last_undegraded":"2023-10-09T05:52:59.583985+0000","last_fullsized":"2023-10-09T05:52:59.583985+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:51.192227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":31,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.587813+0000","last_change":"2023-10-09T05:52:29.736624+0000","last_active":"2023-10-09T05:52:59.587813+0000","last_peered":"2023-10-09T05:52:59.587813+0000","last_clean":"2023-10-09T05:52:59.587813+0000","last_became_active":"2023-10-09T05:52:29.736263+0000","last_became_peered":"2023-10-09T05:52:29.736263+0000","last_unstale":"2023-10-09T05:52:59.587813+0000","last_undegraded":"2023-10-09T05:52:59.587813+0000","last_fullsized":"2023-10-09T05:52:59.587813+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:23:56.032785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":141,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.623697+0000","last_change":"2023-10-09T05:52:42.261890+0000","last_active":"2023-10-09T05:52:59.623697+0000","last_peered":"2023-10-09T05:52:59.623697+0000","last_clean":"2023-10-09T05:52:59.623697+0000","last_became_active":"2023-10-09T05:52:42.261521+0000","last_became_peered":"2023-10-09T05:52:42.261521+0000","last_unstale":"2023-10-09T05:52:59.623697+0000","last_undegraded":"2023-10-09T05:52:59.623697+0000","last_fullsized":"2023-10-09T05:52:59.623697+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:59.208145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":102,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.604331+0000","last_change":"2023-10-09T05:52:23.654749+0000","last_active":"2023-10-09T05:52:59.604331+0000","last_peered":"2023-10-09T05:52:59.604331+0000","last_clean":"2023-10-09T05:52:59.604331+0000","last_became_active":"2023-10-09T05:52:23.643044+0000","last_became_peered":"2023-10-09T05:52:23.643044+0000","last_unstale":"2023-10-09T05:52:59.604331+0000","last_undegraded":"2023-10-09T05:52:59.604331+0000","last_fullsized":"2023-10-09T05:52:59.604331+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:23:08.898078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":132,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-09T05:52:59.583827+0000","last_change":"2023-10-09T05:52:11.411838+0000","last_active":"2023-10-09T05:52:59.583827+0000","last_peered":"2023-10-09T05:52:59.583827+0000","last_clean":"2023-10-09T05:52:59.583827+0000","last_became_active":"2023-10-09T05:52:11.411506+0000","last_became_peered":"2023-10-09T05:52:11.411506+0000","last_unstale":"2023-10-09T05:52:59.583827+0000","last_undegraded":"2023-10-09T05:52:59.583827+0000","last_fullsized":"2023-10-09T05:52:59.583827+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:55.135008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":6021644312,"num_objects":91884,"num_object_clones":0,"num_object_copies":275652,"num_objects_missing_on_primary":5403,"num_objects_missing":5403,"num_objects_degraded":41148,"num_objects_misplaced":4288,"num_objects_unfound":0,"num_objects_dirty":91884,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":101198,"num_write_kb":6178561,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13087,"num_bytes_recovered":857341952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18742325248,"data_stored":18742313032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":101198,"ondisk_log_size":101198,"up":48,"acting":37,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":536699,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":444,"num_read_kb":832,"num_write":218,"num_write_kb":2197,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2200463,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234163,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2877700,"kb_used_data":2580412,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":90867452,"statfs":{"total":95995035648,"available":93048270848,"internally_reserved":0,"allocated":2642341888,"data_stored":2641757926,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:52:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1400000000000001,"5min":2.1400000000000001,"15min":2.1400000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":20.247,"5min":20.247,"15min":20.247},"last":0.373},{"interface":"front","average":{"1min":1.974,"5min":1.974,"15min":1.974},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":20.236000000000001,"5min":20.236000000000001,"15min":20.236000000000001},"last":0.44400000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:52:18 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":0.77900000000000003,"15min":0.60899999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.9859999999999998,"5min":7.9859999999999998,"15min":7.9859999999999998},"last":6.0419999999999998},{"interface":"front","average":{"1min":1.534,"5min":0.78700000000000003,"15min":0.63},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":5.8129999999999997,"5min":5.8129999999999997,"15min":5.8129999999999997},"last":0.29699999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:52:57 2023","interfaces":[{"interface":"back","average":{"1min":4.6509999999999998,"5min":2.2349999999999999,"15min":1.095},"min":{"1min":0.309,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":37.130000000000003,"5min":50.57,"15min":50.57},"last":0.47799999999999998},{"interface":"front","average":{"1min":4.4870000000000001,"5min":2.3799999999999999,"15min":1.2030000000000001},"min":{"1min":0.36399999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":37.052,"5min":50.536999999999999,"15min":50.536999999999999},"last":0.58899999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:52:24 2023","interfaces":[{"interface":"back","average":{"1min":15.843,"5min":5.6440000000000001,"15min":2.2919999999999998},"min":{"1min":0.41599999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":147.54900000000001,"5min":147.54900000000001,"15min":147.54900000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":15.071,"5min":5.4699999999999998,"15min":2.214},"min":{"1min":0.40400000000000003,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":144.065,"5min":144.065,"15min":144.065},"last":0.42699999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.70199999999999996,"15min":0.55000000000000004},"min":{"1min":0.248,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.77700000000000002,"5min":9.5329999999999995,"15min":9.5329999999999995},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.90600000000000003,"15min":0.63},"min":{"1min":0.31900000000000001,"5min":0.27400000000000002,"15min":0.27100000000000002},"max":{"1min":3.0019999999999998,"5min":7.6120000000000001,"15min":7.6120000000000001},"last":0.40500000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.876,"15min":0.623},"min":{"1min":0.31900000000000001,"5min":0.29499999999999998,"15min":0.248},"max":{"1min":1.657,"5min":10.795999999999999,"15min":10.795999999999999},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.70999999999999996,"15min":0.59099999999999997},"min":{"1min":0.309,"5min":0.307,"15min":0.28100000000000003},"max":{"1min":3.9380000000000002,"5min":3.9380000000000002,"15min":3.9380000000000002},"last":0.53400000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":2.4430000000000001,"15min":1.173},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.6879999999999999,"5min":131.94399999999999,"15min":131.94399999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.61399999999999999,"5min":2.54,"15min":1.2110000000000001},"min":{"1min":0.36399999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.6779999999999999,"5min":135.23400000000001,"15min":135.23400000000001},"last":0.57699999999999996}]}]},{"osd":6,"up_from":46,"seq":197568495799,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2757688,"kb_used_data":2456304,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":90987464,"statfs":{"total":95995035648,"available":93171163136,"internally_reserved":0,"allocated":2515255296,"data_stored":2514683765,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:52:57 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":1.1619999999999999,"15min":1.1619999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":7.0599999999999996,"5min":7.0599999999999996,"15min":7.0599999999999996},"last":0.66700000000000004},{"interface":"front","average":{"1min":1.228,"5min":1.228,"15min":1.228},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":6.6459999999999999,"5min":6.6459999999999999,"15min":6.6459999999999999},"last":0.496}]},{"osd":1,"last update":"Mon Oct 9 05:52:30 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":0.75800000000000001,"15min":0.64800000000000002},"min":{"1min":0.25800000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.7229999999999999,"5min":3.7949999999999999,"15min":3.7949999999999999},"last":3.637},{"interface":"front","average":{"1min":1.863,"5min":0.88800000000000001,"15min":0.68899999999999995},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":9.9670000000000005,"5min":9.9670000000000005,"15min":9.9670000000000005},"last":0.69999999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:52:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":0.93000000000000005,"15min":0.67200000000000004},"min":{"1min":0.32900000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":6.6859999999999999,"5min":6.6859999999999999,"15min":6.6859999999999999},"last":5.8360000000000003},{"interface":"front","average":{"1min":1.4550000000000001,"5min":0.90400000000000003,"15min":0.67300000000000004},"min":{"1min":0.34300000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.1360000000000001,"5min":9.9390000000000001,"15min":9.9390000000000001},"last":0.628}]},{"osd":3,"last update":"Mon Oct 9 05:52:30 2023","interfaces":[{"interface":"back","average":{"1min":2.6779999999999999,"5min":1.0760000000000001,"15min":0.72799999999999998},"min":{"1min":0.33200000000000002,"5min":0.311,"15min":0.311},"max":{"1min":10.19,"5min":10.19,"15min":10.19},"last":0.84199999999999997},{"interface":"front","average":{"1min":3.0270000000000001,"5min":1.087,"15min":0.747},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":11.914,"5min":11.914,"15min":11.914},"last":6.3940000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:52:40 2023","interfaces":[{"interface":"back","average":{"1min":2.1139999999999999,"5min":0.93799999999999994,"15min":0.61499999999999999},"min":{"1min":0.28000000000000003,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.9100000000000001,"5min":7.9100000000000001,"15min":7.9100000000000001},"last":0.58799999999999997},{"interface":"front","average":{"1min":1.8440000000000001,"5min":0.83799999999999997,"15min":0.60799999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.3300000000000001,"5min":7.3300000000000001,"15min":7.3300000000000001},"last":0.81299999999999994}]},{"osd":5,"last update":"Mon Oct 9 05:52:40 2023","interfaces":[{"interface":"back","average":{"1min":2.3959999999999999,"5min":0.91300000000000003,"15min":0.62},"min":{"1min":0.30199999999999999,"5min":0.29799999999999999,"15min":0.27700000000000002},"max":{"1min":13.879,"5min":13.879,"15min":13.879},"last":0.72999999999999998},{"interface":"front","average":{"1min":2.0129999999999999,"5min":0.97899999999999998,"15min":0.67300000000000004},"min":{"1min":0.32100000000000001,"5min":0.307,"15min":0.307},"max":{"1min":7.29,"5min":7.29,"15min":7.29},"last":0.78100000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:53:04 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":1.252,"15min":0.77200000000000002},"min":{"1min":0.39100000000000001,"5min":0.316,"15min":0.316},"max":{"1min":5.4969999999999999,"5min":19.914999999999999,"15min":19.914999999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.1040000000000001,"15min":0.76600000000000001},"min":{"1min":0.434,"5min":0.39800000000000002,"15min":0.35799999999999998},"max":{"1min":5.5099999999999998,"5min":7.0309999999999997,"15min":7.0309999999999997},"last":0.86599999999999999}]}]},{"osd":1,"up_from":89,"seq":382252089399,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1386228,"kb_used_data":1310380,"kb_used_omap":9,"kb_used_meta":75830,"kb_avail":92358924,"statfs":{"total":95995035648,"available":94575538176,"internally_reserved":0,"allocated":1341829120,"data_stored":1341029358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":77650921},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,0,1,3,3,2,4,6],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:52:57 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":2.0299999999999998,"5min":2.0299999999999998,"15min":2.0299999999999998},"last":0.56999999999999995},{"interface":"front","average":{"1min":1.044,"5min":1.044,"15min":1.044},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.3879999999999999,"5min":6.3879999999999999,"15min":6.3879999999999999},"last":0.60999999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:53:01 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":0.73099999999999998,"15min":0.66300000000000003},"min":{"1min":0.29299999999999998,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":3.3330000000000002,"5min":5.0789999999999997,"15min":5.0789999999999997},"last":0.71099999999999997},{"interface":"front","average":{"1min":2.6539999999999999,"5min":1.0049999999999999,"15min":0.68899999999999995},"min":{"1min":0.28000000000000003,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":34.671999999999997,"5min":34.671999999999997,"15min":34.671999999999997},"last":10.875}]},{"osd":3,"last update":"Mon Oct 9 05:52:25 2023","interfaces":[{"interface":"back","average":{"1min":8.0670000000000002,"5min":4.3959999999999999,"15min":1.734},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":100.70699999999999,"5min":284.06099999999998,"15min":284.06099999999998},"last":0.73699999999999999},{"interface":"front","average":{"1min":8.8279999999999994,"5min":4.5510000000000002,"15min":1.788},"min":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":100.754,"5min":284.97300000000001,"15min":284.97300000000001},"last":7.5650000000000004}]},{"osd":4,"last update":"Mon Oct 9 05:52:21 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":0.79100000000000004,"15min":0.65800000000000003},"min":{"1min":0.32500000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":5.6390000000000002,"5min":5.6390000000000002,"15min":5.6390000000000002},"last":7.5410000000000004},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.622},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.8899999999999999,"5min":3.4220000000000002,"15min":3.4220000000000002},"last":0.68400000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:52:21 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":0.87,"15min":0.68799999999999994},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.4560000000000004,"5min":5.4560000000000004,"15min":5.4560000000000004},"last":7.4729999999999999},{"interface":"front","average":{"1min":1.4590000000000001,"5min":0.85199999999999998,"15min":0.68999999999999995},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":3.847,"5min":4.2859999999999996,"15min":4.2859999999999996},"last":7.4900000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:52:21 2023","interfaces":[{"interface":"back","average":{"1min":14.429,"5min":3.4060000000000001,"15min":1.55},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":204.62700000000001,"5min":204.62700000000001,"15min":204.62700000000001},"last":7.5220000000000002},{"interface":"front","average":{"1min":14.054,"5min":3.387,"15min":1.5529999999999999},"min":{"1min":0.38100000000000001,"5min":0.371,"15min":0.371},"max":{"1min":204.285,"5min":204.285,"15min":204.285},"last":7.5090000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:52:21 2023","interfaces":[{"interface":"back","average":{"1min":2.4020000000000001,"5min":1.0780000000000001,"15min":0.80400000000000005},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":10.669,"5min":10.669,"15min":10.669},"last":7.4349999999999996},{"interface":"front","average":{"1min":1.6899999999999999,"5min":0.88700000000000001,"15min":0.72099999999999997},"min":{"1min":0.42299999999999999,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":10.619999999999999,"5min":10.619999999999999,"15min":10.619999999999999},"last":0.64800000000000002}]}]},{"osd":0,"up_from":129,"seq":554050781196,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1320600,"kb_used_data":1277008,"kb_used_omap":29,"kb_used_meta":43490,"kb_avail":92424552,"statfs":{"total":95995035648,"available":94642741248,"internally_reserved":0,"allocated":1307656192,"data_stored":1306884419,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30322,"internal_metadata":44534158},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:52:58 2023","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":3.1299999999999999,"5min":3.1299999999999999,"15min":3.1299999999999999},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.1479999999999999,"15min":1.1479999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":4.0049999999999999,"5min":4.0049999999999999,"15min":4.0049999999999999},"last":0.42099999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:52:58 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.0389999999999999,"15min":1.0389999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.9199999999999999,"5min":3.9199999999999999,"15min":3.9199999999999999},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.0089999999999999,"15min":1.0089999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.7730000000000001,"5min":3.7730000000000001,"15min":3.7730000000000001},"last":0.44600000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:52:58 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.1930000000000001,"5min":3.1930000000000001,"15min":3.1930000000000001},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":3.8069999999999999,"5min":3.8069999999999999,"15min":3.8069999999999999},"last":0.56499999999999995}]},{"osd":4,"last update":"Mon Oct 9 05:52:58 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":1.2230000000000001,"15min":1.2230000000000001},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":5.1529999999999996,"5min":5.1529999999999996,"15min":5.1529999999999996},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.1549999999999998,"5min":3.1549999999999998,"15min":3.1549999999999998},"last":0.48799999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:52:58 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.149,"15min":1.149},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":3.851,"5min":3.851,"15min":3.851},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":4.9109999999999996,"5min":4.9109999999999996,"15min":4.9109999999999996},"last":2.319}]},{"osd":6,"last update":"Mon Oct 9 05:52:58 2023","interfaces":[{"interface":"back","average":{"1min":1.343,"5min":1.343,"15min":1.343},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":6.6849999999999996,"5min":6.6849999999999996,"15min":6.6849999999999996},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.226,"5min":1.226,"15min":1.226},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":4.8689999999999998,"5min":4.8689999999999998,"15min":4.8689999999999998},"last":0.61199999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:52:58 2023","interfaces":[{"interface":"back","average":{"1min":1.4199999999999999,"5min":1.4199999999999999,"15min":1.4199999999999999},"min":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"max":{"1min":5.859,"5min":5.859,"15min":5.859},"last":0.58299999999999996},{"interface":"front","average":{"1min":1.4059999999999999,"5min":1.4059999999999999,"15min":1.4059999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":5.8730000000000002,"5min":5.8730000000000002,"15min":5.8730000000000002},"last":0.59299999999999997}]}]},{"osd":2,"up_from":94,"seq":403726925878,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3268348,"kb_used_data":3156276,"kb_used_omap":81,"kb_used_meta":111982,"kb_avail":90476804,"statfs":{"total":95995035648,"available":92648247296,"internally_reserved":0,"allocated":3232026624,"data_stored":3231277219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":114669786},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:52:56 2023","interfaces":[{"interface":"back","average":{"1min":1.698,"5min":1.698,"15min":1.698},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":11.444000000000001,"5min":11.444000000000001,"15min":11.444000000000001},"last":0.433},{"interface":"front","average":{"1min":1.5609999999999999,"5min":1.5609999999999999,"15min":1.5609999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.883,"5min":6.883,"15min":6.883},"last":0.50900000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:52:53 2023","interfaces":[{"interface":"back","average":{"1min":2.2629999999999999,"5min":2.6349999999999998,"15min":1.165},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":11.492000000000001,"5min":163.24199999999999,"15min":163.24199999999999},"last":0.96499999999999997},{"interface":"front","average":{"1min":1.954,"5min":2.5550000000000002,"15min":1.1240000000000001},"min":{"1min":0.27600000000000002,"5min":0.214,"15min":0.214},"max":{"1min":6.9829999999999997,"5min":163.27000000000001,"15min":163.27000000000001},"last":0.55200000000000005}]},{"osd":3,"last update":"Mon Oct 9 05:52:24 2023","interfaces":[{"interface":"back","average":{"1min":2.1640000000000001,"5min":2.855,"15min":1.22},"min":{"1min":0.249,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":8.8000000000000007,"5min":170.07599999999999,"15min":170.07599999999999},"last":0.77100000000000002},{"interface":"front","average":{"1min":2.302,"5min":1.5429999999999999,"15min":0.80900000000000005},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":11.596,"5min":63.600000000000001,"15min":63.600000000000001},"last":0.70799999999999996}]},{"osd":4,"last update":"Mon Oct 9 05:52:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0459999999999998,"5min":1.079,"15min":0.81000000000000005},"min":{"1min":0.29999999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":11.372999999999999,"5min":11.372999999999999,"15min":11.372999999999999},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.6930000000000001,"5min":1.054,"15min":0.82599999999999996},"min":{"1min":0.33300000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.9390000000000001,"5min":6.5709999999999997,"15min":6.5709999999999997},"last":0.58599999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:52:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8460000000000001,"5min":0.96799999999999997,"15min":0.77500000000000002},"min":{"1min":0.40400000000000003,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":10.782,"5min":10.782,"15min":10.782},"last":0.73799999999999999},{"interface":"front","average":{"1min":1.708,"5min":1.02,"15min":0.78700000000000003},"min":{"1min":0.33900000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":11.416,"5min":11.416,"15min":11.416},"last":0.66000000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:52:53 2023","interfaces":[{"interface":"back","average":{"1min":2.234,"5min":1.105,"15min":0.81299999999999994},"min":{"1min":0.40400000000000003,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":11.59,"5min":11.59,"15min":11.59},"last":0.72099999999999997},{"interface":"front","average":{"1min":1.524,"5min":0.92400000000000004,"15min":0.76800000000000002},"min":{"1min":0.441,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":6.4710000000000001,"5min":6.4710000000000001,"15min":6.4710000000000001},"last":0.79800000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:52:53 2023","interfaces":[{"interface":"back","average":{"1min":3.3439999999999999,"5min":1.2869999999999999,"15min":0.877},"min":{"1min":0.45900000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":40.404000000000003,"5min":40.404000000000003,"15min":40.404000000000003},"last":0.82699999999999996},{"interface":"front","average":{"1min":3.097,"5min":1.3200000000000001,"15min":0.91200000000000003},"min":{"1min":0.38500000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":40.113,"5min":40.113,"15min":40.113},"last":0.68200000000000005}]}]},{"osd":3,"up_from":99,"seq":425201762352,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3808540,"kb_used_data":3704788,"kb_used_omap":7,"kb_used_meta":103736,"kb_avail":89936612,"statfs":{"total":95995035648,"available":92095090688,"internally_reserved":0,"allocated":3793702912,"data_stored":3792922538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":106226280},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:52:57 2023","interfaces":[{"interface":"back","average":{"1min":1.548,"5min":1.548,"15min":1.548},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":15.536,"5min":15.536,"15min":15.536},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.593,"5min":1.593,"15min":1.593},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":11.210000000000001,"5min":11.210000000000001,"15min":11.210000000000001},"last":0.27500000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:52:23 2023","interfaces":[{"interface":"back","average":{"1min":2.2109999999999999,"5min":1.9510000000000001,"15min":0.89700000000000002},"min":{"1min":0.253,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":9.2579999999999991,"5min":127.226,"15min":127.226},"last":0.33900000000000002},{"interface":"front","average":{"1min":2.2959999999999998,"5min":2.0470000000000002,"15min":0.94699999999999995},"min":{"1min":0.26300000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":15.680999999999999,"5min":127.468,"15min":127.468},"last":0.42199999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:52:23 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":1.282,"15min":0.67800000000000005},"min":{"1min":0.26600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.0350000000000001,"5min":62.938000000000002,"15min":62.938000000000002},"last":0.36199999999999999},{"interface":"front","average":{"1min":2.0369999999999999,"5min":1.5629999999999999,"15min":0.80900000000000005},"min":{"1min":0.32500000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":15.821999999999999,"5min":62.43,"15min":62.43},"last":0.97599999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:52:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8080000000000001,"5min":0.93400000000000005,"15min":0.71199999999999997},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.7439999999999998,"5min":7.7439999999999998,"15min":7.7439999999999998},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.5389999999999999,"5min":0.80700000000000005,"15min":0.68000000000000005},"min":{"1min":0.32100000000000001,"5min":0.311,"15min":0.311},"max":{"1min":6.2000000000000002,"5min":6.2000000000000002,"15min":6.2000000000000002},"last":0.45600000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:52:23 2023","interfaces":[{"interface":"back","average":{"1min":1.623,"5min":0.83199999999999996,"15min":0.67900000000000005},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":5.5860000000000003,"5min":5.5860000000000003,"15min":5.5860000000000003},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.3779999999999999,"5min":0.81100000000000005,"15min":0.69499999999999995},"min":{"1min":0.34300000000000003,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.2839999999999998,"5min":4.2839999999999998,"15min":4.2839999999999998},"last":0.52300000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:52:23 2023","interfaces":[{"interface":"back","average":{"1min":2.1339999999999999,"5min":0.94599999999999995,"15min":0.73699999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":12.798999999999999,"5min":12.798999999999999,"15min":12.798999999999999},"last":0.374},{"interface":"front","average":{"1min":2.0609999999999999,"5min":0.98099999999999998,"15min":0.78000000000000003},"min":{"1min":0.39900000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":12.773,"5min":12.773,"15min":12.773},"last":0.47299999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:52:23 2023","interfaces":[{"interface":"back","average":{"1min":1.954,"5min":0.95799999999999996,"15min":0.73899999999999999},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":7.4580000000000002,"5min":7.4580000000000002,"15min":7.4580000000000002},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.841,"5min":0.96199999999999997,"15min":0.75800000000000001},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":7.8399999999999999,"5min":7.8399999999999999,"15min":7.8399999999999999},"last":0.504}]}]},{"osd":4,"up_from":30,"seq":128849019072,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2139204,"kb_used_data":1837820,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":91605948,"statfs":{"total":95995035648,"available":93804490752,"internally_reserved":0,"allocated":1881927680,"data_stored":1881343971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:52:56 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.1699999999999999,"5min":3.1699999999999999,"15min":3.1699999999999999},"last":0.73999999999999999},{"interface":"front","average":{"1min":1.534,"5min":1.534,"15min":1.534},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":12.782,"5min":12.782,"15min":12.782},"last":0.56499999999999995}]},{"osd":1,"last update":"Mon Oct 9 05:52:27 2023","interfaces":[{"interface":"back","average":{"1min":2.4769999999999999,"5min":0.99199999999999999,"15min":0.70799999999999996},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":9.2390000000000008,"5min":9.2390000000000008,"15min":9.2390000000000008},"last":0.68700000000000006},{"interface":"front","average":{"1min":1.837,"5min":0.85299999999999998,"15min":0.629},"min":{"1min":0.29699999999999999,"5min":0.255,"15min":0.255},"max":{"1min":8.2629999999999999,"5min":8.2629999999999999,"15min":8.2629999999999999},"last":0.53000000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:52:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":1.2689999999999999,"15min":0.80700000000000005},"min":{"1min":0.31900000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.4939999999999998,"5min":33.262,"15min":33.262},"last":0.65300000000000002},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.3049999999999999,"15min":0.89000000000000001},"min":{"1min":0.36599999999999999,"5min":0.313,"15min":0.313},"max":{"1min":6.5510000000000002,"5min":33.240000000000002,"15min":33.240000000000002},"last":0.71599999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:52:27 2023","interfaces":[{"interface":"back","average":{"1min":1.883,"5min":1.3859999999999999,"15min":0.85399999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.3490000000000002,"5min":40.531999999999996,"15min":40.531999999999996},"last":0.59599999999999997},{"interface":"front","average":{"1min":2.2170000000000001,"5min":1.506,"15min":0.90200000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":9.9779999999999998,"5min":39.905999999999999,"15min":39.905999999999999},"last":0.47099999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:52:16 2023","interfaces":[{"interface":"back","average":{"1min":2.0289999999999999,"5min":0.80500000000000005,"15min":0.57699999999999996},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.012,"5min":10.012,"15min":10.012},"last":0.39200000000000002},{"interface":"front","average":{"1min":2.7069999999999999,"5min":0.99099999999999999,"15min":0.66000000000000003},"min":{"1min":0.30499999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":12.489000000000001,"5min":12.489000000000001,"15min":12.489000000000001},"last":0.77500000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:52:34 2023","interfaces":[{"interface":"back","average":{"1min":2.3450000000000002,"5min":0.95599999999999996,"15min":0.65500000000000003},"min":{"1min":0.35299999999999998,"5min":0.29799999999999999,"15min":0.27400000000000002},"max":{"1min":11.234999999999999,"5min":11.234999999999999,"15min":11.234999999999999},"last":0.95999999999999996},{"interface":"front","average":{"1min":1.4570000000000001,"5min":0.73999999999999999,"15min":0.60599999999999998},"min":{"1min":0.314,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":9.923,"5min":9.923,"15min":9.923},"last":0.97499999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:53:04 2023","interfaces":[{"interface":"back","average":{"1min":1.492,"5min":1.1799999999999999,"15min":0.77300000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.606,"5min":10.606,"15min":10.606},"last":0.82099999999999995},{"interface":"front","average":{"1min":1.0649999999999999,"5min":0.91600000000000004,"15min":0.69699999999999995},"min":{"1min":0.36199999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":5.665,"5min":8.3819999999999997,"15min":8.3819999999999997},"last":0.75900000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757435,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2295100,"kb_used_data":1997812,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":91450052,"statfs":{"total":95995035648,"available":93644853248,"internally_reserved":0,"allocated":2045759488,"data_stored":2045180283,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:53:01 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.7869999999999999,"5min":2.7869999999999999,"15min":2.7869999999999999},"last":0.375},{"interface":"front","average":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":3.4140000000000001,"5min":3.4140000000000001,"15min":3.4140000000000001},"last":1.3200000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:52:24 2023","interfaces":[{"interface":"back","average":{"1min":2.0670000000000002,"5min":1.913,"15min":0.995},"min":{"1min":0.27000000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.069,"5min":114.247,"15min":114.247},"last":1.355},{"interface":"front","average":{"1min":1.4950000000000001,"5min":1.8600000000000001,"15min":0.995},"min":{"1min":0.30399999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.9800000000000004,"5min":114.678,"15min":114.678},"last":0.34300000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:52:50 2023","interfaces":[{"interface":"back","average":{"1min":7.8920000000000003,"5min":2.2450000000000001,"15min":1.139},"min":{"1min":0.32900000000000001,"5min":0.25,"15min":0.25},"max":{"1min":106.66500000000001,"5min":106.66500000000001,"15min":106.66500000000001},"last":1.472},{"interface":"front","average":{"1min":7.9429999999999996,"5min":2.2389999999999999,"15min":1.1379999999999999},"min":{"1min":0.36299999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":106.779,"5min":106.779,"15min":106.779},"last":4.077}]},{"osd":3,"last update":"Mon Oct 9 05:52:24 2023","interfaces":[{"interface":"back","average":{"1min":19.015000000000001,"5min":6.7089999999999996,"15min":2.597},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":174.22200000000001,"5min":272.81299999999999,"15min":272.81299999999999},"last":0.443},{"interface":"front","average":{"1min":19.501999999999999,"5min":6.944,"15min":2.7429999999999999},"min":{"1min":0.36799999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":174.339,"5min":273.00999999999999,"15min":273.00999999999999},"last":4.9809999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:52:12 2023","interfaces":[{"interface":"back","average":{"1min":1.6220000000000001,"5min":0.72699999999999998,"15min":0.54900000000000004},"min":{"1min":0.27500000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.2050000000000001,"5min":6.2050000000000001,"15min":6.2050000000000001},"last":0.50600000000000001},{"interface":"front","average":{"1min":1.5149999999999999,"5min":0.71299999999999997,"15min":0.56100000000000005},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":13.050000000000001,"5min":13.050000000000001,"15min":13.050000000000001},"last":0.47499999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:52:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":0.73299999999999998,"15min":0.57099999999999995},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":10.606999999999999,"5min":10.606999999999999,"15min":10.606999999999999},"last":0.498},{"interface":"front","average":{"1min":1.891,"5min":0.878,"15min":0.64600000000000002},"min":{"1min":0.30299999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.9770000000000003,"5min":5.9770000000000003,"15min":5.9770000000000003},"last":1.379}]},{"osd":7,"last update":"Mon Oct 9 05:52:07 2023","interfaces":[{"interface":"back","average":{"1min":1.954,"5min":0.874,"15min":0.66400000000000003},"min":{"1min":0.36899999999999999,"5min":0.32700000000000001,"15min":0.315},"max":{"1min":6.1600000000000001,"5min":6.1600000000000001,"15min":6.1600000000000001},"last":2.8650000000000002},{"interface":"front","average":{"1min":1.218,"5min":0.73199999999999998,"15min":0.63500000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":5.032,"5min":5.032,"15min":5.032},"last":0.54200000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1474,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1305743360,"data_stored":1305739288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1337528320,"data_stored":1337524248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3228762112,"data_stored":3228762112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3791126528,"data_stored":3791126528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1878654976,"data_stored":1878654976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2042826752,"data_stored":2042822680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2516516864,"data_stored":2516516864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2641166336,"data_stored":2641166336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:53:12.256 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:53:12.256 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 21 2023-10-09T05:53:12.506 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:12.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:12.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:12.610 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:12.640 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:12.702 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:12.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:12.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:12.773 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:12 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 41148/275724 objects degraded (14.924%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:12.776 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:12 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 41148/275724 objects degraded (14.924%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:12.807 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:12.835 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:12.839 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:12 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 41148/275724 objects degraded (14.924%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:12.842 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:12.896 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:12.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:12.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:13.002 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:13.052 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:13.270 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:13.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:13.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:13.366 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:53:13.366 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:53:13.376 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:13.422 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:13.425 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:14.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:14 smithi064 ceph-mon[212750]: pgmap v353: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 143 op/s; 41148/275445 objects degraded (14.939%); 4288/275445 objects misplaced (1.557%); 12 MiB/s, 196 objects/s recovering 2023-10-09T05:53:14.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:14.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:53:14.285 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:14 smithi064 ceph-mon[220044]: pgmap v353: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 143 op/s; 41148/275445 objects degraded (14.939%); 4288/275445 objects misplaced (1.557%); 12 MiB/s, 196 objects/s recovering 2023-10-09T05:53:14.285 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:14.285 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:53:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:14 smithi156 ceph-mon[195101]: pgmap v353: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 143 op/s; 41148/275445 objects degraded (14.939%); 4288/275445 objects misplaced (1.557%); 12 MiB/s, 196 objects/s recovering 2023-10-09T05:53:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:53:14.654 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:14.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:14.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:14.760 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:14.789 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:14.799 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:53:14.846 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:14.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:14.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:14.953 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:14.989 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:15.055 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:53:15.305 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:53:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:53:15] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.43.0" 2023-10-09T05:53:15.471 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:15.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:15.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:15.577 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:15.621 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:15.914 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:15.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:15.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:16.024 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:16.057 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:16.147 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:16.175 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:16.185 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:16 smithi064 ceph-mon[212750]: pgmap v354: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 41148/274281 objects degraded (15.002%); 2982/274281 objects misplaced (1.087%); 14 MiB/s, 231 objects/s recovering 2023-10-09T05:53:16.185 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:16 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:16.185 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:16 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:16.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:16.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:16.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:16 smithi064 ceph-mon[220044]: pgmap v354: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 41148/274281 objects degraded (15.002%); 2982/274281 objects misplaced (1.087%); 14 MiB/s, 231 objects/s recovering 2023-10-09T05:53:16.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:16 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:16.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:16 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:16.287 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:16.322 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:16.324 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:16.376 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:16.380 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:16 smithi156 ceph-mon[195101]: pgmap v354: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 41148/274281 objects degraded (15.002%); 2982/274281 objects misplaced (1.087%); 14 MiB/s, 231 objects/s recovering 2023-10-09T05:53:16.380 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:16 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:16.380 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:16 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:16.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:16.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:16.483 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:16.587 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:16.704 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:16.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:16.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:16.809 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:16.861 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:17.129 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:17 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1205378369' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:53:17.129 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:17 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:53:17.130 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:17 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:17.130 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:17 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:17.130 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:17 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:17.130 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:17 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3827718505' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T05:53:17.130 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:17 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 41148/274281 objects degraded (15.002%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:17.137 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 1 pg_num to 21 2023-10-09T05:53:17.137 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T05:53:17.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:17 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1205378369' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:53:17.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:17 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:53:17.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:17 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:17.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:17 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:17.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:17 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:17.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:17 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3827718505' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T05:53:17.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:17 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 41148/274281 objects degraded (15.002%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:17.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:17 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1205378369' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:53:17.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:17 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:53:17.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:17 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:17.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:17 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:17.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:17 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:17.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:17 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3827718505' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T05:53:17.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:17 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 41148/274281 objects degraded (15.002%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:18.446 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:18 smithi064 ceph-mon[212750]: pgmap v355: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 41148/273786 objects degraded (15.029%); 2982/273786 objects misplaced (1.089%); 14 MiB/s, 223 objects/s recovering 2023-10-09T05:53:18.447 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:18 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:18.447 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:18 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:53:18.447 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:18 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3827718505' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-10-09T05:53:18.447 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:18 smithi064 ceph-mon[212750]: osdmap e154: 8 total, 8 up, 8 in 2023-10-09T05:53:18.447 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:18 smithi064 ceph-mon[212750]: failed to encode map e154 with expected crc 2023-10-09T05:53:18.447 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:18 smithi064 ceph-mon[212750]: failed to encode map e154 with expected crc 2023-10-09T05:53:18.447 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:18 smithi064 ceph-mon[212750]: failed to encode map e154 with expected crc 2023-10-09T05:53:18.448 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:18 smithi064 ceph-mon[220044]: pgmap v355: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 41148/273786 objects degraded (15.029%); 2982/273786 objects misplaced (1.089%); 14 MiB/s, 223 objects/s recovering 2023-10-09T05:53:18.448 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:18 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:18.448 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:18 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:53:18.448 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:18 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3827718505' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-10-09T05:53:18.448 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:18 smithi064 ceph-mon[220044]: osdmap e154: 8 total, 8 up, 8 in 2023-10-09T05:53:18.448 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:18 smithi064 ceph-mon[220044]: failed to encode map e154 with expected crc 2023-10-09T05:53:18.448 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:18 smithi064 ceph-mon[220044]: failed to encode map e154 with expected crc 2023-10-09T05:53:18.448 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:18 smithi064 ceph-mon[220044]: failed to encode map e154 with expected crc 2023-10-09T05:53:18.454 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:18.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:18.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:18.558 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:18.586 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:18 smithi156 ceph-mon[195101]: pgmap v355: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 41148/273786 objects degraded (15.029%); 2982/273786 objects misplaced (1.089%); 14 MiB/s, 223 objects/s recovering 2023-10-09T05:53:18.586 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:18 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:18.586 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:18 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:53:18.586 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:18 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3827718505' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-10-09T05:53:18.586 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:18 smithi156 ceph-mon[195101]: osdmap e154: 8 total, 8 up, 8 in 2023-10-09T05:53:18.586 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:18 smithi156 ceph-mon[195101]: failed to encode map e154 with expected crc 2023-10-09T05:53:18.586 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:18 smithi156 ceph-mon[195101]: failed to encode map e154 with expected crc 2023-10-09T05:53:18.586 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:18 smithi156 ceph-mon[195101]: failed to encode map e154 with expected crc 2023-10-09T05:53:18.589 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:18.652 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:18.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:18.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:18.763 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:18.796 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:18.850 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:18.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:18.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:18.956 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:18.985 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:19.036 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:19.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:19.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:19.134 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:19 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-10-09T05:53:19.134 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:19 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-10-09T05:53:19.144 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:19.189 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:19.296 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:19.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:19.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:19.402 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:19.435 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:19 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-10-09T05:53:19.435 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:19 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-10-09T05:53:19.442 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:19.610 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:19.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:19.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:19.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:19 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-10-09T05:53:19.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:19 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-10-09T05:53:19.630 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:53:19.631 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:53:19.720 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:19.777 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:19.893 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:53:19.913 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:19.915 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:53:19.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:19.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:19.948 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:53:19.973 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:53:20.000 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:53:20.018 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:20.021 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:53:20.053 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:20.055 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:53:20.081 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:53:20.082 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:53:20.082 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:53:20.083 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:53:20.083 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:53:20.083 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:53:20.083 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:53:20.083 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:53:20.083 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:53:20.111 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:20.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:20.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:20.218 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:20.256 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:20.324 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:20.326 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:20 smithi064 ceph-mon[212750]: pgmap v357: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 126 op/s; 41148/273591 objects degraded (15.040%); 2982/273591 objects misplaced (1.090%); 14 MiB/s, 231 objects/s recovering 2023-10-09T05:53:20.326 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:20 smithi064 ceph-mon[212750]: failed to encode map e155 with expected crc 2023-10-09T05:53:20.326 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:20 smithi064 ceph-mon[212750]: failed to encode map e155 with expected crc 2023-10-09T05:53:20.326 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:20 smithi064 ceph-mon[212750]: failed to encode map e155 with expected crc 2023-10-09T05:53:20.327 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:20 smithi064 ceph-mon[212750]: failed to encode map e155 with expected crc 2023-10-09T05:53:20.327 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:20 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-10-09T05:53:20.327 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:20 smithi064 ceph-mon[212750]: osdmap e155: 8 total, 8 up, 8 in 2023-10-09T05:53:20.327 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:20 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1273634603' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:20.327 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:20 smithi064 ceph-mon[220044]: pgmap v357: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 126 op/s; 41148/273591 objects degraded (15.040%); 2982/273591 objects misplaced (1.090%); 14 MiB/s, 231 objects/s recovering 2023-10-09T05:53:20.327 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:20 smithi064 ceph-mon[220044]: failed to encode map e155 with expected crc 2023-10-09T05:53:20.327 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:20 smithi064 ceph-mon[220044]: failed to encode map e155 with expected crc 2023-10-09T05:53:20.328 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:20 smithi064 ceph-mon[220044]: failed to encode map e155 with expected crc 2023-10-09T05:53:20.328 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:20 smithi064 ceph-mon[220044]: failed to encode map e155 with expected crc 2023-10-09T05:53:20.328 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:20 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-10-09T05:53:20.328 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:20 smithi064 ceph-mon[220044]: osdmap e155: 8 total, 8 up, 8 in 2023-10-09T05:53:20.328 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:20 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1273634603' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:20.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:20.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:20 smithi156 ceph-mon[195101]: pgmap v357: 99 pgs: 88 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 126 op/s; 41148/273591 objects degraded (15.040%); 2982/273591 objects misplaced (1.090%); 14 MiB/s, 231 objects/s recovering 2023-10-09T05:53:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:20 smithi156 ceph-mon[195101]: failed to encode map e155 with expected crc 2023-10-09T05:53:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:20 smithi156 ceph-mon[195101]: failed to encode map e155 with expected crc 2023-10-09T05:53:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:20 smithi156 ceph-mon[195101]: failed to encode map e155 with expected crc 2023-10-09T05:53:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:20 smithi156 ceph-mon[195101]: failed to encode map e155 with expected crc 2023-10-09T05:53:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:20 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-10-09T05:53:20.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:20 smithi156 ceph-mon[195101]: osdmap e155: 8 total, 8 up, 8 in 2023-10-09T05:53:20.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:20 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1273634603' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:20.429 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:20.450 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:53:20.458 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:20.507 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:20.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:20.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:20.612 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:20.649 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:21.257 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:53:21.263 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:21.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:21.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:21.373 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:21.400 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:21 smithi156 ceph-mon[195101]: failed to encode map e156 with expected crc 2023-10-09T05:53:21.400 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:21 smithi156 ceph-mon[195101]: osdmap e156: 8 total, 8 up, 8 in 2023-10-09T05:53:21.400 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:21 smithi156 ceph-mon[195101]: failed to encode map e156 with expected crc 2023-10-09T05:53:21.401 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:21 smithi156 ceph-mon[195101]: failed to encode map e156 with expected crc 2023-10-09T05:53:21.401 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:21 smithi156 ceph-mon[195101]: failed to encode map e156 with expected crc 2023-10-09T05:53:21.401 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:21 smithi156 ceph-mon[195101]: failed to encode map e156 with expected crc 2023-10-09T05:53:21.401 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:21 smithi156 ceph-mon[195101]: failed to encode map e156 with expected crc 2023-10-09T05:53:21.401 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:21 smithi156 ceph-mon[195101]: failed to encode map e156 with expected crc 2023-10-09T05:53:21.401 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:21.452 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:21.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:21.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:21.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:21 smithi064 ceph-mon[212750]: failed to encode map e156 with expected crc 2023-10-09T05:53:21.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:21 smithi064 ceph-mon[212750]: osdmap e156: 8 total, 8 up, 8 in 2023-10-09T05:53:21.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:21 smithi064 ceph-mon[212750]: failed to encode map e156 with expected crc 2023-10-09T05:53:21.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:21 smithi064 ceph-mon[212750]: failed to encode map e156 with expected crc 2023-10-09T05:53:21.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:21 smithi064 ceph-mon[212750]: failed to encode map e156 with expected crc 2023-10-09T05:53:21.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:21 smithi064 ceph-mon[212750]: failed to encode map e156 with expected crc 2023-10-09T05:53:21.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:21 smithi064 ceph-mon[212750]: failed to encode map e156 with expected crc 2023-10-09T05:53:21.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:21 smithi064 ceph-mon[212750]: failed to encode map e156 with expected crc 2023-10-09T05:53:21.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:21 smithi064 ceph-mon[220044]: failed to encode map e156 with expected crc 2023-10-09T05:53:21.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:21 smithi064 ceph-mon[220044]: osdmap e156: 8 total, 8 up, 8 in 2023-10-09T05:53:21.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:21 smithi064 ceph-mon[220044]: failed to encode map e156 with expected crc 2023-10-09T05:53:21.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:21 smithi064 ceph-mon[220044]: failed to encode map e156 with expected crc 2023-10-09T05:53:21.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:21 smithi064 ceph-mon[220044]: failed to encode map e156 with expected crc 2023-10-09T05:53:21.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:21 smithi064 ceph-mon[220044]: failed to encode map e156 with expected crc 2023-10-09T05:53:21.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:21 smithi064 ceph-mon[220044]: failed to encode map e156 with expected crc 2023-10-09T05:53:21.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:21 smithi064 ceph-mon[220044]: failed to encode map e156 with expected crc 2023-10-09T05:53:21.479 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:21.557 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:21.616 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:21.762 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:21.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:21.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:21.868 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:21.937 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:22.150 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:22.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:22.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:22.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:22 smithi064 ceph-mon[212750]: pgmap v360: 109 pgs: 7 peering, 10 unknown, 81 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 95 op/s; 41148/272382 objects degraded (15.107%); 1147/272382 objects misplaced (0.421%); 19 MiB/s, 305 objects/s recovering 2023-10-09T05:53:22.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:22 smithi064 ceph-mon[212750]: osdmap e157: 8 total, 8 up, 8 in 2023-10-09T05:53:22.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:22 smithi064 ceph-mon[212750]: failed to encode map e157 with expected crc 2023-10-09T05:53:22.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:22 smithi064 ceph-mon[212750]: failed to encode map e157 with expected crc 2023-10-09T05:53:22.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:22 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 41148/272382 objects degraded (15.107%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:22.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:22 smithi064 ceph-mon[220044]: pgmap v360: 109 pgs: 7 peering, 10 unknown, 81 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 95 op/s; 41148/272382 objects degraded (15.107%); 1147/272382 objects misplaced (0.421%); 19 MiB/s, 305 objects/s recovering 2023-10-09T05:53:22.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:22 smithi064 ceph-mon[220044]: osdmap e157: 8 total, 8 up, 8 in 2023-10-09T05:53:22.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:22 smithi064 ceph-mon[220044]: failed to encode map e157 with expected crc 2023-10-09T05:53:22.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:22 smithi064 ceph-mon[220044]: failed to encode map e157 with expected crc 2023-10-09T05:53:22.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:22 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 41148/272382 objects degraded (15.107%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:22.254 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:22.291 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:22.355 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:22.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:22.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:22.460 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:22.485 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:22 smithi156 ceph-mon[195101]: pgmap v360: 109 pgs: 7 peering, 10 unknown, 81 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 95 op/s; 41148/272382 objects degraded (15.107%); 1147/272382 objects misplaced (0.421%); 19 MiB/s, 305 objects/s recovering 2023-10-09T05:53:22.485 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:22 smithi156 ceph-mon[195101]: osdmap e157: 8 total, 8 up, 8 in 2023-10-09T05:53:22.485 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:22 smithi156 ceph-mon[195101]: failed to encode map e157 with expected crc 2023-10-09T05:53:22.485 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:22 smithi156 ceph-mon[195101]: failed to encode map e157 with expected crc 2023-10-09T05:53:22.485 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:22 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 41148/272382 objects degraded (15.107%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:22.487 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:22.538 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:22.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:22.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:22.645 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:22.688 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:22.900 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:22.918 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:22.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:22.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:23.025 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:23.080 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:23.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:23 smithi064 ceph-mon[212750]: failed to encode map e158 with expected crc 2023-10-09T05:53:23.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:23 smithi064 ceph-mon[212750]: osdmap e158: 8 total, 8 up, 8 in 2023-10-09T05:53:23.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:23 smithi064 ceph-mon[212750]: failed to encode map e158 with expected crc 2023-10-09T05:53:23.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:23 smithi064 ceph-mon[212750]: failed to encode map e158 with expected crc 2023-10-09T05:53:23.210 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:23 smithi064 ceph-mon[220044]: failed to encode map e158 with expected crc 2023-10-09T05:53:23.210 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:23 smithi064 ceph-mon[220044]: osdmap e158: 8 total, 8 up, 8 in 2023-10-09T05:53:23.210 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:23 smithi064 ceph-mon[220044]: failed to encode map e158 with expected crc 2023-10-09T05:53:23.210 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:23 smithi064 ceph-mon[220044]: failed to encode map e158 with expected crc 2023-10-09T05:53:23.225 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:23.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:23.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:23.264 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:53:23.337 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:23.474 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:23.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:23 smithi156 ceph-mon[195101]: failed to encode map e158 with expected crc 2023-10-09T05:53:23.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:23 smithi156 ceph-mon[195101]: osdmap e158: 8 total, 8 up, 8 in 2023-10-09T05:53:23.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:23 smithi156 ceph-mon[195101]: failed to encode map e158 with expected crc 2023-10-09T05:53:23.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:23 smithi156 ceph-mon[195101]: failed to encode map e158 with expected crc 2023-10-09T05:53:24.193 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T05:53:24.261 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T05:53:24.261 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:53:24.261 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 2, 3, 4, 5, 6, 7, 1] 2023-10-09T05:53:24.262 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd out 7 2023-10-09T05:53:24.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:24 smithi064 ceph-mon[220044]: pgmap v363: 109 pgs: 7 peering, 8 unknown, 83 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 41148/271944 objects degraded (15.131%); 1147/271944 objects misplaced (0.422%); 29 MiB/s, 457 objects/s recovering 2023-10-09T05:53:24.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:24 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2705221224' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:24.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:24 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:24.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:24 smithi064 ceph-mon[212750]: pgmap v363: 109 pgs: 7 peering, 8 unknown, 83 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 41148/271944 objects degraded (15.131%); 1147/271944 objects misplaced (0.422%); 29 MiB/s, 457 objects/s recovering 2023-10-09T05:53:24.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:24 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2705221224' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:24.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:24 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:24.598 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:24.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:24.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:24.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:24 smithi156 ceph-mon[195101]: pgmap v363: 109 pgs: 7 peering, 8 unknown, 83 active+clean, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 41148/271944 objects degraded (15.131%); 1147/271944 objects misplaced (0.422%); 29 MiB/s, 457 objects/s recovering 2023-10-09T05:53:24.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:24 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2705221224' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:24.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:24 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:24.703 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:24.729 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:24.774 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:24.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:24.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:24.878 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:24.922 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:25.179 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:25 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:53:25.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:25 smithi064 ceph-mon[212750]: failed to encode map e159 with expected crc 2023-10-09T05:53:25.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:25 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:53:25.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:25 smithi064 ceph-mon[212750]: osdmap e159: 8 total, 8 up, 8 in 2023-10-09T05:53:25.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:25 smithi064 ceph-mon[212750]: failed to encode map e159 with expected crc 2023-10-09T05:53:25.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:25 smithi064 ceph-mon[212750]: failed to encode map e159 with expected crc 2023-10-09T05:53:25.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:25 smithi064 ceph-mon[212750]: failed to encode map e159 with expected crc 2023-10-09T05:53:25.374 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:25.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:25.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:25.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:53:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:53:25] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.43.0" 2023-10-09T05:53:25.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:25 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:53:25.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:25 smithi064 ceph-mon[220044]: failed to encode map e159 with expected crc 2023-10-09T05:53:25.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:25 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:53:25.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:25 smithi064 ceph-mon[220044]: osdmap e159: 8 total, 8 up, 8 in 2023-10-09T05:53:25.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:25 smithi064 ceph-mon[220044]: failed to encode map e159 with expected crc 2023-10-09T05:53:25.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:25 smithi064 ceph-mon[220044]: failed to encode map e159 with expected crc 2023-10-09T05:53:25.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:25 smithi064 ceph-mon[220044]: failed to encode map e159 with expected crc 2023-10-09T05:53:25.480 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:25.507 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:25 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:53:25.507 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:25 smithi156 ceph-mon[195101]: failed to encode map e159 with expected crc 2023-10-09T05:53:25.507 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:25 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:53:25.507 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:25 smithi156 ceph-mon[195101]: osdmap e159: 8 total, 8 up, 8 in 2023-10-09T05:53:25.508 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:25 smithi156 ceph-mon[195101]: failed to encode map e159 with expected crc 2023-10-09T05:53:25.508 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:25 smithi156 ceph-mon[195101]: failed to encode map e159 with expected crc 2023-10-09T05:53:25.508 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:25 smithi156 ceph-mon[195101]: failed to encode map e159 with expected crc 2023-10-09T05:53:25.510 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:25.529 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:25.562 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:25.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:25.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:25.585 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:25.668 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:25.695 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:25.744 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:25.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:25.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:25.850 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:25.919 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:25.999 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:53:25.999 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T05:53:25.999 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T05:53:25.999 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T05:53:25.999 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:53:26.074 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:26.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:26.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:26.187 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:26.214 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:26.261 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:26.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:26.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:26.366 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:26.405 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:26.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:26 smithi064 ceph-mon[220044]: pgmap v365: 109 pgs: 2 peering, 2 unknown, 96 active+clean, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 48 KiB/s rd, 126 KiB/s wr, 153 op/s; 40963/270885 objects degraded (15.122%); 19 MiB/s, 381 objects/s recovering 2023-10-09T05:53:26.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:26 smithi064 ceph-mon[220044]: failed to encode map e160 with expected crc 2023-10-09T05:53:26.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:26 smithi064 ceph-mon[220044]: osdmap e160: 8 total, 8 up, 8 in 2023-10-09T05:53:26.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:26 smithi064 ceph-mon[220044]: failed to encode map e160 with expected crc 2023-10-09T05:53:26.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:26 smithi064 ceph-mon[220044]: failed to encode map e160 with expected crc 2023-10-09T05:53:26.466 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:26 smithi064 ceph-mon[212750]: pgmap v365: 109 pgs: 2 peering, 2 unknown, 96 active+clean, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 48 KiB/s rd, 126 KiB/s wr, 153 op/s; 40963/270885 objects degraded (15.122%); 19 MiB/s, 381 objects/s recovering 2023-10-09T05:53:26.466 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:26 smithi064 ceph-mon[212750]: failed to encode map e160 with expected crc 2023-10-09T05:53:26.466 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:26 smithi064 ceph-mon[212750]: osdmap e160: 8 total, 8 up, 8 in 2023-10-09T05:53:26.466 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:26 smithi064 ceph-mon[212750]: failed to encode map e160 with expected crc 2023-10-09T05:53:26.467 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:26 smithi064 ceph-mon[212750]: failed to encode map e160 with expected crc 2023-10-09T05:53:26.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:26 smithi156 ceph-mon[195101]: pgmap v365: 109 pgs: 2 peering, 2 unknown, 96 active+clean, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 48 KiB/s rd, 126 KiB/s wr, 153 op/s; 40963/270885 objects degraded (15.122%); 19 MiB/s, 381 objects/s recovering 2023-10-09T05:53:26.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:26 smithi156 ceph-mon[195101]: failed to encode map e160 with expected crc 2023-10-09T05:53:26.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:26 smithi156 ceph-mon[195101]: osdmap e160: 8 total, 8 up, 8 in 2023-10-09T05:53:26.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:26 smithi156 ceph-mon[195101]: failed to encode map e160 with expected crc 2023-10-09T05:53:26.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:26 smithi156 ceph-mon[195101]: failed to encode map e160 with expected crc 2023-10-09T05:53:27.354 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:53:27.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:27 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 40963/270885 objects degraded (15.122%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:27.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:27 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 40963/270885 objects degraded (15.122%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:27.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:27 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 40963/270885 objects degraded (15.122%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:28.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:28 smithi064 ceph-mon[212750]: pgmap v367: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 99 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 39 KiB/s rd, 103 KiB/s wr, 139 op/s; 38922/270414 objects degraded (14.393%); 1693/270414 objects misplaced (0.626%); 20 MiB/s, 376 objects/s recovering 2023-10-09T05:53:28.198 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:53:28.286 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:28.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:28.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:28.412 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:28.439 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:28.474 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:28 smithi064 ceph-mon[220044]: pgmap v367: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 99 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 39 KiB/s rd, 103 KiB/s wr, 139 op/s; 38922/270414 objects degraded (14.393%); 1693/270414 objects misplaced (0.626%); 20 MiB/s, 376 objects/s recovering 2023-10-09T05:53:28.486 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:28.488 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:28 smithi156 ceph-mon[195101]: pgmap v367: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 99 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 39 KiB/s rd, 103 KiB/s wr, 139 op/s; 38922/270414 objects degraded (14.393%); 1693/270414 objects misplaced (0.626%); 20 MiB/s, 376 objects/s recovering 2023-10-09T05:53:28.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:28.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:28.584 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:28.591 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:28.647 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:28.761 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:28.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:28.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:28.865 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:28.933 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:29.058 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:29.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:29.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:29.104 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:53:29.137 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:53:29.163 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:29.175 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:53:29.224 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:29.227 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:53:29.276 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:53:29.323 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:53:29.330 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:29.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:29.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:29.359 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:53:29.384 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:53:29.389 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:53:29.389 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:53:29.389 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:53:29.389 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:53:29.389 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:53:29.389 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:53:29.390 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:53:29.390 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:53:29.446 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:29.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:29 smithi064 ceph-mon[212750]: osdmap e161: 8 total, 8 up, 8 in 2023-10-09T05:53:29.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:29 smithi064 ceph-mon[212750]: failed to encode map e161 with expected crc 2023-10-09T05:53:29.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:29 smithi064 ceph-mon[212750]: failed to encode map e161 with expected crc 2023-10-09T05:53:29.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:29 smithi064 ceph-mon[212750]: failed to encode map e161 with expected crc 2023-10-09T05:53:29.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:29 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:29.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:53:29.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:29 smithi064 ceph-mon[212750]: pgmap v369: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 99 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 35 KiB/s rd, 92 KiB/s wr, 124 op/s; 38922/270210 objects degraded (14.404%); 1693/270210 objects misplaced (0.627%); 18 MiB/s, 335 objects/s recovering 2023-10-09T05:53:29.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:29 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2934033693' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:29.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:29 smithi064 ceph-mon[220044]: osdmap e161: 8 total, 8 up, 8 in 2023-10-09T05:53:29.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:29 smithi064 ceph-mon[220044]: failed to encode map e161 with expected crc 2023-10-09T05:53:29.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:29 smithi064 ceph-mon[220044]: failed to encode map e161 with expected crc 2023-10-09T05:53:29.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:29 smithi064 ceph-mon[220044]: failed to encode map e161 with expected crc 2023-10-09T05:53:29.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:29 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:29.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:53:29.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:29 smithi064 ceph-mon[220044]: pgmap v369: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 99 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 35 KiB/s rd, 92 KiB/s wr, 124 op/s; 38922/270210 objects degraded (14.404%); 1693/270210 objects misplaced (0.627%); 18 MiB/s, 335 objects/s recovering 2023-10-09T05:53:29.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:29 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2934033693' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:29.474 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:29.515 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:29.517 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:29 smithi156 ceph-mon[195101]: osdmap e161: 8 total, 8 up, 8 in 2023-10-09T05:53:29.517 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:29 smithi156 ceph-mon[195101]: failed to encode map e161 with expected crc 2023-10-09T05:53:29.518 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:29 smithi156 ceph-mon[195101]: failed to encode map e161 with expected crc 2023-10-09T05:53:29.518 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:29 smithi156 ceph-mon[195101]: failed to encode map e161 with expected crc 2023-10-09T05:53:29.518 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:29 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:29.518 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:53:29.518 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:29 smithi156 ceph-mon[195101]: pgmap v369: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 99 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 35 KiB/s rd, 92 KiB/s wr, 124 op/s; 38922/270210 objects degraded (14.404%); 1693/270210 objects misplaced (0.627%); 18 MiB/s, 335 objects/s recovering 2023-10-09T05:53:29.518 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:29 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2934033693' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:29.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:29.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:29.545 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:29.619 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:29.648 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:29.700 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:29.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:29.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:29.804 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:29 smithi064 conmon[212726]: 2023-10-09T05:53:29.559+0000 7f8823f23700 -1 mon.a@0(leader).osd e162 definitely_dead 0 2023-10-09T05:53:29.806 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:29.840 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:30.150 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:30.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:30.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:30.253 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:30.280 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:30.322 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:30.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:30.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:30.429 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:30.465 INFO:teuthology.orchestra.run.smithi064.stderr:marked out osd.7. 2023-10-09T05:53:30.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:30 smithi064 ceph-mon[212750]: osdmap e162: 8 total, 8 up, 8 in 2023-10-09T05:53:30.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:30 smithi064 ceph-mon[212750]: failed to encode map e162 with expected crc 2023-10-09T05:53:30.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:30 smithi064 ceph-mon[212750]: failed to encode map e162 with expected crc 2023-10-09T05:53:30.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:30 smithi064 ceph-mon[212750]: failed to encode map e162 with expected crc 2023-10-09T05:53:30.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:30 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1130799027' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:53:30.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:30 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:53:30.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:30 smithi064 ceph-mon[212750]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T05:53:30.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:30 smithi064 ceph-mon[220044]: osdmap e162: 8 total, 8 up, 8 in 2023-10-09T05:53:30.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:30 smithi064 ceph-mon[220044]: failed to encode map e162 with expected crc 2023-10-09T05:53:30.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:30 smithi064 ceph-mon[220044]: failed to encode map e162 with expected crc 2023-10-09T05:53:30.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:30 smithi064 ceph-mon[220044]: failed to encode map e162 with expected crc 2023-10-09T05:53:30.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:30 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1130799027' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:53:30.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:30 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:53:30.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:30 smithi064 ceph-mon[220044]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T05:53:30.476 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:30.520 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:30.524 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:30 smithi156 ceph-mon[195101]: osdmap e162: 8 total, 8 up, 8 in 2023-10-09T05:53:30.525 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:30 smithi156 ceph-mon[195101]: failed to encode map e162 with expected crc 2023-10-09T05:53:30.525 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:30 smithi156 ceph-mon[195101]: failed to encode map e162 with expected crc 2023-10-09T05:53:30.525 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:30 smithi156 ceph-mon[195101]: failed to encode map e162 with expected crc 2023-10-09T05:53:30.525 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:30 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1130799027' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:53:30.525 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:30 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:53:30.525 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:30 smithi156 ceph-mon[195101]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T05:53:30.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:30.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:30.627 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:30.653 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:30.695 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:30.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:30.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:30.800 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:30.834 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:31.498 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:31.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:31.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:31.615 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:31.657 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:31.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:31 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T05:53:31.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:31 smithi064 ceph-mon[220044]: osdmap e163: 8 total, 8 up, 7 in 2023-10-09T05:53:31.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:31 smithi064 ceph-mon[220044]: failed to encode map e163 with expected crc 2023-10-09T05:53:31.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:31 smithi064 ceph-mon[220044]: failed to encode map e163 with expected crc 2023-10-09T05:53:31.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:31 smithi064 ceph-mon[220044]: pgmap v372: 109 pgs: 1 active+recovering+undersized+remapped, 101 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 95 op/s; 38294/269118 objects degraded (14.229%); 1186/269118 objects misplaced (0.441%); 13 MiB/s, 213 objects/s recovering 2023-10-09T05:53:31.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:31 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T05:53:31.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:31 smithi064 ceph-mon[212750]: osdmap e163: 8 total, 8 up, 7 in 2023-10-09T05:53:31.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:31 smithi064 ceph-mon[212750]: failed to encode map e163 with expected crc 2023-10-09T05:53:31.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:31 smithi064 ceph-mon[212750]: failed to encode map e163 with expected crc 2023-10-09T05:53:31.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:31 smithi064 ceph-mon[212750]: pgmap v372: 109 pgs: 1 active+recovering+undersized+remapped, 101 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 95 op/s; 38294/269118 objects degraded (14.229%); 1186/269118 objects misplaced (0.441%); 13 MiB/s, 213 objects/s recovering 2023-10-09T05:53:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:31 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T05:53:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:31 smithi156 ceph-mon[195101]: osdmap e163: 8 total, 8 up, 7 in 2023-10-09T05:53:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:31 smithi156 ceph-mon[195101]: failed to encode map e163 with expected crc 2023-10-09T05:53:31.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:31 smithi156 ceph-mon[195101]: failed to encode map e163 with expected crc 2023-10-09T05:53:31.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:31 smithi156 ceph-mon[195101]: pgmap v372: 109 pgs: 1 active+recovering+undersized+remapped, 101 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 95 op/s; 38294/269118 objects degraded (14.229%); 1186/269118 objects misplaced (0.441%); 13 MiB/s, 213 objects/s recovering 2023-10-09T05:53:32.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[220044]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[220044]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[220044]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[220044]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[220044]: osdmap e164: 8 total, 8 up, 7 in 2023-10-09T05:53:32.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[220044]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[220044]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[220044]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[220044]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[220044]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[220044]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[220044]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[220044]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 38294/269118 objects degraded (14.229%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:32.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[212750]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[212750]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[212750]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[212750]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[212750]: osdmap e164: 8 total, 8 up, 7 in 2023-10-09T05:53:32.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[212750]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[212750]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[212750]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[212750]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[212750]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[212750]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[212750]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[212750]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:32 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 38294/269118 objects degraded (14.229%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:32.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:32 smithi156 ceph-mon[195101]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:32 smithi156 ceph-mon[195101]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:32 smithi156 ceph-mon[195101]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:32 smithi156 ceph-mon[195101]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:32 smithi156 ceph-mon[195101]: osdmap e164: 8 total, 8 up, 7 in 2023-10-09T05:53:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:32 smithi156 ceph-mon[195101]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:32 smithi156 ceph-mon[195101]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:32 smithi156 ceph-mon[195101]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:32 smithi156 ceph-mon[195101]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:32 smithi156 ceph-mon[195101]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:32 smithi156 ceph-mon[195101]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:32 smithi156 ceph-mon[195101]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:32 smithi156 ceph-mon[195101]: failed to encode map e164 with expected crc 2023-10-09T05:53:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:32 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 38294/269118 objects degraded (14.229%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:32.977 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:32.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:32.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:33.083 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:33.126 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:33.309 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:33.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:33.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:33.414 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:33.459 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:33.645 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:53:33.683 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:33 smithi064 ceph-mon[220044]: failed to encode map e165 with expected crc 2023-10-09T05:53:33.683 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:33 smithi064 ceph-mon[220044]: failed to encode map e165 with expected crc 2023-10-09T05:53:33.683 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:33 smithi064 ceph-mon[220044]: 6.1 starting backfill to osd.0 from (0'0,0'0] MAX to 162'6531 2023-10-09T05:53:33.683 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:33 smithi064 ceph-mon[220044]: osdmap e165: 8 total, 8 up, 7 in 2023-10-09T05:53:33.684 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:33 smithi064 ceph-mon[220044]: failed to encode map e165 with expected crc 2023-10-09T05:53:33.684 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:33 smithi064 ceph-mon[220044]: failed to encode map e165 with expected crc 2023-10-09T05:53:33.684 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:33 smithi064 ceph-mon[220044]: failed to encode map e165 with expected crc 2023-10-09T05:53:33.684 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:33 smithi064 ceph-mon[220044]: 6.c starting backfill to osd.1 from (0'0,0'0] MAX to 162'6490 2023-10-09T05:53:33.684 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:33 smithi064 ceph-mon[220044]: 6.3 starting backfill to osd.1 from (0'0,0'0] MAX to 162'6488 2023-10-09T05:53:33.684 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:33 smithi064 ceph-mon[220044]: 6.6 starting backfill to osd.5 from (0'0,0'0] MAX to 162'6495 2023-10-09T05:53:33.684 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:33 smithi064 ceph-mon[220044]: 6.8 starting backfill to osd.6 from (0'0,0'0] MAX to 162'6449 2023-10-09T05:53:33.684 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:33 smithi064 ceph-mon[220044]: pgmap v375: 109 pgs: 6 remapped+peering, 4 peering, 1 active+recovering+undersized+remapped, 92 active+clean, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 20 KiB/s rd, 55 KiB/s wr, 129 op/s; 26128/268851 objects degraded (9.718%); 6778/268851 objects misplaced (2.521%); 23 MiB/s, 363 objects/s recovering 2023-10-09T05:53:33.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:33 smithi064 ceph-mon[212750]: failed to encode map e165 with expected crc 2023-10-09T05:53:33.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:33 smithi064 ceph-mon[212750]: failed to encode map e165 with expected crc 2023-10-09T05:53:33.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:33 smithi064 ceph-mon[212750]: 6.1 starting backfill to osd.0 from (0'0,0'0] MAX to 162'6531 2023-10-09T05:53:33.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:33 smithi064 ceph-mon[212750]: osdmap e165: 8 total, 8 up, 7 in 2023-10-09T05:53:33.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:33 smithi064 ceph-mon[212750]: failed to encode map e165 with expected crc 2023-10-09T05:53:33.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:33 smithi064 ceph-mon[212750]: failed to encode map e165 with expected crc 2023-10-09T05:53:33.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:33 smithi064 ceph-mon[212750]: failed to encode map e165 with expected crc 2023-10-09T05:53:33.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:33 smithi064 ceph-mon[212750]: 6.c starting backfill to osd.1 from (0'0,0'0] MAX to 162'6490 2023-10-09T05:53:33.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:33 smithi064 ceph-mon[212750]: 6.3 starting backfill to osd.1 from (0'0,0'0] MAX to 162'6488 2023-10-09T05:53:33.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:33 smithi064 ceph-mon[212750]: 6.6 starting backfill to osd.5 from (0'0,0'0] MAX to 162'6495 2023-10-09T05:53:33.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:33 smithi064 ceph-mon[212750]: 6.8 starting backfill to osd.6 from (0'0,0'0] MAX to 162'6449 2023-10-09T05:53:33.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:33 smithi064 ceph-mon[212750]: pgmap v375: 109 pgs: 6 remapped+peering, 4 peering, 1 active+recovering+undersized+remapped, 92 active+clean, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 20 KiB/s rd, 55 KiB/s wr, 129 op/s; 26128/268851 objects degraded (9.718%); 6778/268851 objects misplaced (2.521%); 23 MiB/s, 363 objects/s recovering 2023-10-09T05:53:33.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:33 smithi156 ceph-mon[195101]: failed to encode map e165 with expected crc 2023-10-09T05:53:33.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:33 smithi156 ceph-mon[195101]: failed to encode map e165 with expected crc 2023-10-09T05:53:33.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:33 smithi156 ceph-mon[195101]: 6.1 starting backfill to osd.0 from (0'0,0'0] MAX to 162'6531 2023-10-09T05:53:33.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:33 smithi156 ceph-mon[195101]: osdmap e165: 8 total, 8 up, 7 in 2023-10-09T05:53:33.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:33 smithi156 ceph-mon[195101]: failed to encode map e165 with expected crc 2023-10-09T05:53:33.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:33 smithi156 ceph-mon[195101]: failed to encode map e165 with expected crc 2023-10-09T05:53:33.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:33 smithi156 ceph-mon[195101]: failed to encode map e165 with expected crc 2023-10-09T05:53:33.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:33 smithi156 ceph-mon[195101]: 6.c starting backfill to osd.1 from (0'0,0'0] MAX to 162'6490 2023-10-09T05:53:33.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:33 smithi156 ceph-mon[195101]: 6.3 starting backfill to osd.1 from (0'0,0'0] MAX to 162'6488 2023-10-09T05:53:33.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:33 smithi156 ceph-mon[195101]: 6.6 starting backfill to osd.5 from (0'0,0'0] MAX to 162'6495 2023-10-09T05:53:33.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:33 smithi156 ceph-mon[195101]: 6.8 starting backfill to osd.6 from (0'0,0'0] MAX to 162'6449 2023-10-09T05:53:33.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:33 smithi156 ceph-mon[195101]: pgmap v375: 109 pgs: 6 remapped+peering, 4 peering, 1 active+recovering+undersized+remapped, 92 active+clean, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 20 KiB/s rd, 55 KiB/s wr, 129 op/s; 26128/268851 objects degraded (9.718%); 6778/268851 objects misplaced (2.521%); 23 MiB/s, 363 objects/s recovering 2023-10-09T05:53:34.542 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:34.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:34.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:34.647 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:34.699 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:34 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 2 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-10-09T05:53:34.699 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:34 smithi064 ceph-mon[212750]: osdmap e166: 8 total, 8 up, 7 in 2023-10-09T05:53:34.699 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:34 smithi064 ceph-mon[212750]: failed to encode map e166 with expected crc 2023-10-09T05:53:34.699 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:34 smithi064 ceph-mon[212750]: failed to encode map e166 with expected crc 2023-10-09T05:53:34.699 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:34 smithi064 ceph-mon[212750]: failed to encode map e166 with expected crc 2023-10-09T05:53:34.699 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:34 smithi064 ceph-mon[212750]: failed to encode map e166 with expected crc 2023-10-09T05:53:34.700 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:34 smithi064 ceph-mon[212750]: failed to encode map e166 with expected crc 2023-10-09T05:53:34.700 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:34 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 2 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-10-09T05:53:34.700 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:34 smithi064 ceph-mon[220044]: osdmap e166: 8 total, 8 up, 7 in 2023-10-09T05:53:34.700 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:34 smithi064 ceph-mon[220044]: failed to encode map e166 with expected crc 2023-10-09T05:53:34.700 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:34 smithi064 ceph-mon[220044]: failed to encode map e166 with expected crc 2023-10-09T05:53:34.700 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:34 smithi064 ceph-mon[220044]: failed to encode map e166 with expected crc 2023-10-09T05:53:34.700 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:34 smithi064 ceph-mon[220044]: failed to encode map e166 with expected crc 2023-10-09T05:53:34.700 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:34 smithi064 ceph-mon[220044]: failed to encode map e166 with expected crc 2023-10-09T05:53:34.701 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:34.805 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:34.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:34.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:34.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:34 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 2 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-10-09T05:53:34.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:34 smithi156 ceph-mon[195101]: osdmap e166: 8 total, 8 up, 7 in 2023-10-09T05:53:34.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:34 smithi156 ceph-mon[195101]: failed to encode map e166 with expected crc 2023-10-09T05:53:34.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:34 smithi156 ceph-mon[195101]: failed to encode map e166 with expected crc 2023-10-09T05:53:34.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:34 smithi156 ceph-mon[195101]: failed to encode map e166 with expected crc 2023-10-09T05:53:34.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:34 smithi156 ceph-mon[195101]: failed to encode map e166 with expected crc 2023-10-09T05:53:34.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:34 smithi156 ceph-mon[195101]: failed to encode map e166 with expected crc 2023-10-09T05:53:34.871 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:34.909 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:34.935 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:34.979 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:34.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:34.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:35.084 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:35.111 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:35.156 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:35.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:35.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:35.262 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:35.289 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:35.331 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:35.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:35.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:35.436 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:35.464 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:35.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:53:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:53:35] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.43.0" 2023-10-09T05:53:35.504 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:53:35.504 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:53:35.507 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T05:53:35.517 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:35.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:35.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:35.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:35 smithi156 ceph-mon[195101]: failed to encode map e167 with expected crc 2023-10-09T05:53:35.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:35 smithi156 ceph-mon[195101]: osdmap e167: 8 total, 8 up, 7 in 2023-10-09T05:53:35.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:35 smithi156 ceph-mon[195101]: failed to encode map e167 with expected crc 2023-10-09T05:53:35.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:35 smithi156 ceph-mon[195101]: failed to encode map e167 with expected crc 2023-10-09T05:53:35.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:35 smithi156 ceph-mon[195101]: failed to encode map e167 with expected crc 2023-10-09T05:53:35.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:35 smithi156 ceph-mon[195101]: pgmap v378: 109 pgs: 6 remapped+peering, 6 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 87 active+clean, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 20 KiB/s rd, 55 KiB/s wr, 91 op/s; 25993/268215 objects degraded (9.691%); 35312/268215 objects misplaced (13.166%); 7.9 MiB/s, 2 keys/s, 126 objects/s recovering 2023-10-09T05:53:35.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:35 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2146062123' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:53:35.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:35 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:53:35.624 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:35.672 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:35.786 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:35.793 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:35 smithi064 ceph-mon[212750]: failed to encode map e167 with expected crc 2023-10-09T05:53:35.793 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:35 smithi064 ceph-mon[212750]: osdmap e167: 8 total, 8 up, 7 in 2023-10-09T05:53:35.793 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:35 smithi064 ceph-mon[212750]: failed to encode map e167 with expected crc 2023-10-09T05:53:35.793 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:35 smithi064 ceph-mon[212750]: failed to encode map e167 with expected crc 2023-10-09T05:53:35.793 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:35 smithi064 ceph-mon[212750]: failed to encode map e167 with expected crc 2023-10-09T05:53:35.793 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:35 smithi064 ceph-mon[212750]: pgmap v378: 109 pgs: 6 remapped+peering, 6 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 87 active+clean, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 20 KiB/s rd, 55 KiB/s wr, 91 op/s; 25993/268215 objects degraded (9.691%); 35312/268215 objects misplaced (13.166%); 7.9 MiB/s, 2 keys/s, 126 objects/s recovering 2023-10-09T05:53:35.793 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:35 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2146062123' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:53:35.793 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:35 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:53:35.794 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:35 smithi064 ceph-mon[220044]: failed to encode map e167 with expected crc 2023-10-09T05:53:35.794 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:35 smithi064 ceph-mon[220044]: osdmap e167: 8 total, 8 up, 7 in 2023-10-09T05:53:35.794 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:35 smithi064 ceph-mon[220044]: failed to encode map e167 with expected crc 2023-10-09T05:53:35.794 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:35 smithi064 ceph-mon[220044]: failed to encode map e167 with expected crc 2023-10-09T05:53:35.794 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:35 smithi064 ceph-mon[220044]: failed to encode map e167 with expected crc 2023-10-09T05:53:35.794 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:35 smithi064 ceph-mon[220044]: pgmap v378: 109 pgs: 6 remapped+peering, 6 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 87 active+clean, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 20 KiB/s rd, 55 KiB/s wr, 91 op/s; 25993/268215 objects degraded (9.691%); 35312/268215 objects misplaced (13.166%); 7.9 MiB/s, 2 keys/s, 126 objects/s recovering 2023-10-09T05:53:35.794 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:35 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2146062123' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:53:35.794 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:35 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:53:35.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:35.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:35.896 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:35.924 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:35.970 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:35.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:35.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:36.077 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:36.104 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:36.148 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:36.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:36.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:36.253 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:36.292 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:36.462 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:36.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:36.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:36.572 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:36.606 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:36.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:36 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:36.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:36 smithi156 ceph-mon[195101]: failed to encode map e168 with expected crc 2023-10-09T05:53:36.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:36 smithi156 ceph-mon[195101]: failed to encode map e168 with expected crc 2023-10-09T05:53:36.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:36 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:53:36.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:36 smithi156 ceph-mon[195101]: osdmap e168: 8 total, 8 up, 7 in 2023-10-09T05:53:36.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:36 smithi156 ceph-mon[195101]: failed to encode map e168 with expected crc 2023-10-09T05:53:36.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:36 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:36.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:36 smithi064 ceph-mon[212750]: failed to encode map e168 with expected crc 2023-10-09T05:53:36.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:36 smithi064 ceph-mon[212750]: failed to encode map e168 with expected crc 2023-10-09T05:53:36.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:36 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:53:36.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:36 smithi064 ceph-mon[212750]: osdmap e168: 8 total, 8 up, 7 in 2023-10-09T05:53:36.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:36 smithi064 ceph-mon[212750]: failed to encode map e168 with expected crc 2023-10-09T05:53:36.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:36 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:36.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:36 smithi064 ceph-mon[220044]: failed to encode map e168 with expected crc 2023-10-09T05:53:36.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:36 smithi064 ceph-mon[220044]: failed to encode map e168 with expected crc 2023-10-09T05:53:36.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:36 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:53:36.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:36 smithi064 ceph-mon[220044]: osdmap e168: 8 total, 8 up, 7 in 2023-10-09T05:53:36.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:36 smithi064 ceph-mon[220044]: failed to encode map e168 with expected crc 2023-10-09T05:53:37.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:37 smithi064 ceph-mon[220044]: pgmap v380: 109 pgs: 6 remapped+peering, 6 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 86 active+clean, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 62 op/s; 23520/268086 objects degraded (8.773%); 39703/268086 objects misplaced (14.810%); 6.3 MiB/s, 2 keys/s, 103 objects/s recovering 2023-10-09T05:53:37.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:37 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 25993/268215 objects degraded (9.691%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:37.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:37 smithi064 ceph-mon[220044]: osdmap e169: 8 total, 8 up, 7 in 2023-10-09T05:53:37.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:37 smithi064 ceph-mon[212750]: pgmap v380: 109 pgs: 6 remapped+peering, 6 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 86 active+clean, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 62 op/s; 23520/268086 objects degraded (8.773%); 39703/268086 objects misplaced (14.810%); 6.3 MiB/s, 2 keys/s, 103 objects/s recovering 2023-10-09T05:53:37.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:37 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 25993/268215 objects degraded (9.691%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:37.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:37 smithi064 ceph-mon[212750]: osdmap e169: 8 total, 8 up, 7 in 2023-10-09T05:53:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:37 smithi156 ceph-mon[195101]: pgmap v380: 109 pgs: 6 remapped+peering, 6 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 86 active+clean, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 62 op/s; 23520/268086 objects degraded (8.773%); 39703/268086 objects misplaced (14.810%); 6.3 MiB/s, 2 keys/s, 103 objects/s recovering 2023-10-09T05:53:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:37 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 25993/268215 objects degraded (9.691%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:37 smithi156 ceph-mon[195101]: osdmap e169: 8 total, 8 up, 7 in 2023-10-09T05:53:38.759 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:38 smithi064 ceph-mon[212750]: failed to encode map e169 with expected crc 2023-10-09T05:53:38.759 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:38 smithi064 ceph-mon[212750]: failed to encode map e169 with expected crc 2023-10-09T05:53:38.760 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:38 smithi064 ceph-mon[212750]: failed to encode map e170 with expected crc 2023-10-09T05:53:38.760 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:38 smithi064 ceph-mon[212750]: osdmap e170: 8 total, 8 up, 7 in 2023-10-09T05:53:38.760 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:38 smithi064 ceph-mon[220044]: failed to encode map e169 with expected crc 2023-10-09T05:53:38.760 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:38 smithi064 ceph-mon[220044]: failed to encode map e169 with expected crc 2023-10-09T05:53:38.760 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:38 smithi064 ceph-mon[220044]: failed to encode map e170 with expected crc 2023-10-09T05:53:38.761 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:38 smithi064 ceph-mon[220044]: osdmap e170: 8 total, 8 up, 7 in 2023-10-09T05:53:38.772 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:38.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:38.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:38.806 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T05:53:38.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:38 smithi156 ceph-mon[195101]: failed to encode map e169 with expected crc 2023-10-09T05:53:38.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:38 smithi156 ceph-mon[195101]: failed to encode map e169 with expected crc 2023-10-09T05:53:38.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:38 smithi156 ceph-mon[195101]: failed to encode map e170 with expected crc 2023-10-09T05:53:38.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:38 smithi156 ceph-mon[195101]: osdmap e170: 8 total, 8 up, 7 in 2023-10-09T05:53:38.882 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:38.921 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:39.038 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:39.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:39.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:39.142 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:39.169 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:39.217 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:39.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:39.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:39.241 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:53:39.301 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:39.326 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:39.357 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:39.405 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:39.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:39.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:39.452 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 1] out_osds: [7] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T05:53:39.452 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:53:39.452 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T05:53:39.452 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:53:39.452 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:53:39.453 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T05:53:39.511 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:39.540 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:39.588 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:39.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:39.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:39.696 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:39.738 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:39.835 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:39 smithi064 ceph-mon[220044]: failed to encode map e169 with expected crc 2023-10-09T05:53:39.835 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:39 smithi064 ceph-mon[220044]: failed to encode map e170 with expected crc 2023-10-09T05:53:39.835 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:39 smithi064 ceph-mon[220044]: failed to encode map e170 with expected crc 2023-10-09T05:53:39.835 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:39 smithi064 ceph-mon[220044]: pgmap v383: 109 pgs: 1 remapped+peering, 7 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 90 active+clean, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 32 op/s; 22662/268005 objects degraded (8.456%); 45171/268005 objects misplaced (16.855%); 16 MiB/s, 2 keys/s, 265 objects/s recovering 2023-10-09T05:53:39.835 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:39 smithi064 ceph-mon[220044]: failed to encode map e170 with expected crc 2023-10-09T05:53:39.835 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:39 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1557466155' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:39.835 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:39 smithi064 ceph-mon[212750]: failed to encode map e169 with expected crc 2023-10-09T05:53:39.836 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:39 smithi064 ceph-mon[212750]: failed to encode map e170 with expected crc 2023-10-09T05:53:39.836 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:39 smithi064 ceph-mon[212750]: failed to encode map e170 with expected crc 2023-10-09T05:53:39.836 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:39 smithi064 ceph-mon[212750]: pgmap v383: 109 pgs: 1 remapped+peering, 7 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 90 active+clean, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 32 op/s; 22662/268005 objects degraded (8.456%); 45171/268005 objects misplaced (16.855%); 16 MiB/s, 2 keys/s, 265 objects/s recovering 2023-10-09T05:53:39.836 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:39 smithi064 ceph-mon[212750]: failed to encode map e170 with expected crc 2023-10-09T05:53:39.836 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:39 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1557466155' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:39.857 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:53:39.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:39 smithi156 ceph-mon[195101]: failed to encode map e169 with expected crc 2023-10-09T05:53:39.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:39 smithi156 ceph-mon[195101]: failed to encode map e170 with expected crc 2023-10-09T05:53:39.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:39 smithi156 ceph-mon[195101]: failed to encode map e170 with expected crc 2023-10-09T05:53:39.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:39 smithi156 ceph-mon[195101]: pgmap v383: 109 pgs: 1 remapped+peering, 7 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 90 active+clean, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 32 op/s; 22662/268005 objects degraded (8.456%); 45171/268005 objects misplaced (16.855%); 16 MiB/s, 2 keys/s, 265 objects/s recovering 2023-10-09T05:53:39.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:39 smithi156 ceph-mon[195101]: failed to encode map e170 with expected crc 2023-10-09T05:53:39.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:39 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1557466155' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:53:39.893 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:39.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:39.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:39.997 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:40.025 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:40.078 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:40.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:40.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:40.184 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:40.215 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:40.265 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:40.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:40.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:40.370 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:40.401 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:40.450 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:40.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:40.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:40.554 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:40.583 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:40.634 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:40.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:40.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:40.738 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:40.762 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:53:40.766 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:40 smithi156 ceph-mon[195101]: Health check update: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T05:53:40.769 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:40.789 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:40 smithi064 ceph-mon[212750]: Health check update: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T05:53:40.789 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:40 smithi064 ceph-mon[220044]: Health check update: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T05:53:40.816 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:40.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:40.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:40.920 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:40.958 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:41.022 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:41.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:41.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:41.133 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:41.165 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:41.274 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:41.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:41.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:41.379 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:41.415 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:41.449 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:41.475 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:41.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:41.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:41.577 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:53:41.577 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:53:41.580 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:41.615 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:41.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:41 smithi064 ceph-mon[220044]: pgmap v384: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 5 peering, 1 active+recovery_wait+degraded+remapped, 92 active+clean, 3 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 22 op/s; 16769/267810 objects degraded (6.262%); 49864/267810 objects misplaced (18.619%); 17 MiB/s, 273 objects/s recovering 2023-10-09T05:53:41.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:41 smithi064 ceph-mon[212750]: pgmap v384: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 5 peering, 1 active+recovery_wait+degraded+remapped, 92 active+clean, 3 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 22 op/s; 16769/267810 objects degraded (6.262%); 49864/267810 objects misplaced (18.619%); 17 MiB/s, 273 objects/s recovering 2023-10-09T05:53:41.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:41 smithi156 ceph-mon[195101]: pgmap v384: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 5 peering, 1 active+recovery_wait+degraded+remapped, 92 active+clean, 3 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 22 op/s; 16769/267810 objects degraded (6.262%); 49864/267810 objects misplaced (18.619%); 17 MiB/s, 273 objects/s recovering 2023-10-09T05:53:41.877 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:41.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:41.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:41.985 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:42.017 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:42.428 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:42.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:42.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:42.534 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:42.566 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:42.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:42 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 16769/267810 objects degraded (6.262%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:42.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:42 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 16769/267810 objects degraded (6.262%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:42.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:42 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 16769/267810 objects degraded (6.262%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:43.019 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:43.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:43.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:43.123 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:43.153 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:43.203 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:43.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:43.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:43.308 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:43.338 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:43.802 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T05:53:43.845 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:43.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:43.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:43.949 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:43.978 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:44.033 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:44.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:44.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:44.128 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:44 smithi064 ceph-mon[212750]: pgmap v385: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 peering, 1 active+recovery_wait+degraded+remapped, 93 active+clean, 3 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 13 op/s; 16769/267810 objects degraded (6.262%); 49864/267810 objects misplaced (18.619%); 14 MiB/s, 230 objects/s recovering 2023-10-09T05:53:44.129 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:44.129 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:53:44.140 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:44.170 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:44.217 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:44 smithi156 ceph-mon[195101]: pgmap v385: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 peering, 1 active+recovery_wait+degraded+remapped, 93 active+clean, 3 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 13 op/s; 16769/267810 objects degraded (6.262%); 49864/267810 objects misplaced (18.619%); 14 MiB/s, 230 objects/s recovering 2023-10-09T05:53:44.217 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:44.217 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:53:44.221 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:44.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:44.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:44.328 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:44.358 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:44.394 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:44.410 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:44.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:44.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:44.435 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:53:44.435 INFO:teuthology.orchestra.run.smithi156.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T05:53:44.435 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:53:44.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:44 smithi064 ceph-mon[220044]: pgmap v385: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 peering, 1 active+recovery_wait+degraded+remapped, 93 active+clean, 3 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 13 op/s; 16769/267810 objects degraded (6.262%); 49864/267810 objects misplaced (18.619%); 14 MiB/s, 230 objects/s recovering 2023-10-09T05:53:44.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:44.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:53:44.514 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:44.548 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:44.659 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:44.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:44.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:44.763 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:44.793 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:44.843 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:44.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:44.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:44.946 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:44.980 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:45.035 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:45.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:45.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:45.139 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:45.151 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T05:53:45.170 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:45.217 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:45.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:45.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:45.321 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:45.355 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:45.404 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:45.406 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:45 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1448693100' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:45.406 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:45 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:45.406 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:45 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1448693100' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:45.406 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:45 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:45.407 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:53:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:53:45] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.43.0" 2023-10-09T05:53:45.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:45.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:45.508 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:45.533 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:45 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1448693100' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:45.533 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:45 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:53:45.537 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:45.583 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:45.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:45.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:45.688 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:45.716 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:45.765 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:45.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:45.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:45.869 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:45.896 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:45.947 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:45.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:45.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:46.050 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:46.077 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:46.122 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:46.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:46.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:46.226 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:46.251 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:46 smithi156 ceph-mon[195101]: pgmap v386: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 peering, 1 active+recovery_wait+degraded+remapped, 93 active+clean, 3 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 11 op/s; 16769/267810 objects degraded (6.262%); 49203/267810 objects misplaced (18.372%); 18 MiB/s, 284 objects/s recovering 2023-10-09T05:53:46.252 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:46 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:46.252 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:46 smithi156 ceph-mon[195101]: failed to encode map e171 with expected crc 2023-10-09T05:53:46.252 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:46 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:53:46.252 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:46 smithi156 ceph-mon[195101]: osdmap e171: 8 total, 8 up, 7 in 2023-10-09T05:53:46.252 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:46 smithi156 ceph-mon[195101]: failed to encode map e171 with expected crc 2023-10-09T05:53:46.254 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:46.302 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:46.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:46.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:46.405 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:46.433 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:46.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:46 smithi064 ceph-mon[212750]: pgmap v386: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 peering, 1 active+recovery_wait+degraded+remapped, 93 active+clean, 3 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 11 op/s; 16769/267810 objects degraded (6.262%); 49203/267810 objects misplaced (18.372%); 18 MiB/s, 284 objects/s recovering 2023-10-09T05:53:46.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:46 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:46.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:46 smithi064 ceph-mon[212750]: failed to encode map e171 with expected crc 2023-10-09T05:53:46.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:46 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:53:46.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:46 smithi064 ceph-mon[212750]: osdmap e171: 8 total, 8 up, 7 in 2023-10-09T05:53:46.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:46 smithi064 ceph-mon[212750]: failed to encode map e171 with expected crc 2023-10-09T05:53:46.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:46 smithi064 ceph-mon[220044]: pgmap v386: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 peering, 1 active+recovery_wait+degraded+remapped, 93 active+clean, 3 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 11 op/s; 16769/267810 objects degraded (6.262%); 49203/267810 objects misplaced (18.372%); 18 MiB/s, 284 objects/s recovering 2023-10-09T05:53:46.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:46 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:46.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:46 smithi064 ceph-mon[220044]: failed to encode map e171 with expected crc 2023-10-09T05:53:46.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:46 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:53:46.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:46 smithi064 ceph-mon[220044]: osdmap e171: 8 total, 8 up, 7 in 2023-10-09T05:53:46.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:46 smithi064 ceph-mon[220044]: failed to encode map e171 with expected crc 2023-10-09T05:53:46.480 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:46.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:46.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:46.584 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:46.613 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:47.090 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:47.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:47.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:47.194 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:47.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:47 smithi064 ceph-mon[212750]: failed to encode map e171 with expected crc 2023-10-09T05:53:47.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:47 smithi064 ceph-mon[212750]: failed to encode map e171 with expected crc 2023-10-09T05:53:47.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:47 smithi064 ceph-mon[220044]: failed to encode map e171 with expected crc 2023-10-09T05:53:47.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:47 smithi064 ceph-mon[220044]: failed to encode map e171 with expected crc 2023-10-09T05:53:47.227 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:47.285 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:47.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:47.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:47 smithi156 ceph-mon[195101]: failed to encode map e171 with expected crc 2023-10-09T05:53:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:47 smithi156 ceph-mon[195101]: failed to encode map e171 with expected crc 2023-10-09T05:53:47.389 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:47.428 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:47.536 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:47.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:47.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:47.617 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:53:47.617 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:53:47.641 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:47.670 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:47.716 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:47.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:47.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:47.823 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:47.850 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:47.898 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:47.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:47.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:48.005 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:48.032 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:48.077 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:48.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:48.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:48.182 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:48.211 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:48.262 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:48.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:48.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:48.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:48 smithi156 ceph-mon[195101]: pgmap v388: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 peering, 1 active+recovery_wait+degraded+remapped, 93 active+clean, 3 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 16769/267810 objects degraded (6.262%); 48931/267810 objects misplaced (18.271%); 14 MiB/s, 226 objects/s recovering 2023-10-09T05:53:48.371 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:48.399 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:48 smithi064 ceph-mon[212750]: pgmap v388: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 peering, 1 active+recovery_wait+degraded+remapped, 93 active+clean, 3 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 16769/267810 objects degraded (6.262%); 48931/267810 objects misplaced (18.271%); 14 MiB/s, 226 objects/s recovering 2023-10-09T05:53:48.399 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:48 smithi064 ceph-mon[220044]: pgmap v388: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 peering, 1 active+recovery_wait+degraded+remapped, 93 active+clean, 3 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 16769/267810 objects degraded (6.262%); 48931/267810 objects misplaced (18.271%); 14 MiB/s, 226 objects/s recovering 2023-10-09T05:53:48.402 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:48.458 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:48.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:48.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:48.562 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:48.590 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:48.639 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:48.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:48.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:48.743 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:48.776 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:48.807 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T05:53:48.827 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:48.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:48.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:48.932 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:48.963 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:49.021 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:49.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:49.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:49.125 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:49.153 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:49.165 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:53:49.198 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:49.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:49.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:49.261 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:49.303 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:49.339 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:49.410 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:49.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:49.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:49.515 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:49.542 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:49.590 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:49.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:49.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:49.699 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:49.733 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:49.946 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:49.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:49.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:50.050 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:50.081 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:50.134 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:50.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:50.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:50.241 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:50.268 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:50.315 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:50.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:50.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:50.421 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:50.448 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:50 smithi064 ceph-mon[212750]: pgmap v389: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 peering, 1 active+recovery_wait+degraded+remapped, 93 active+clean, 3 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 6 op/s; 16769/267810 objects degraded (6.262%); 48931/267810 objects misplaced (18.271%); 12 MiB/s, 190 objects/s recovering 2023-10-09T05:53:50.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:50 smithi064 ceph-mon[220044]: pgmap v389: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 peering, 1 active+recovery_wait+degraded+remapped, 93 active+clean, 3 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 6 op/s; 16769/267810 objects degraded (6.262%); 48931/267810 objects misplaced (18.271%); 12 MiB/s, 190 objects/s recovering 2023-10-09T05:53:50.451 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:50.508 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:50.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:50.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:50.615 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:50.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:50 smithi156 ceph-mon[195101]: pgmap v389: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 peering, 1 active+recovery_wait+degraded+remapped, 93 active+clean, 3 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 6 op/s; 16769/267810 objects degraded (6.262%); 48931/267810 objects misplaced (18.271%); 12 MiB/s, 190 objects/s recovering 2023-10-09T05:53:50.643 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:50.694 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:50.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:50.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:50.799 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:50.828 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:50.875 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:50.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:50.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:50.983 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:51.014 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:51.027 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:51.085 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:51.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:51.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:51.192 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:51.230 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:51.288 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:51.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:51.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:51.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:51.425 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:51.782 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:51.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:51.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:51.887 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:51.919 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:51.968 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 1] out_osds: [7] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T05:53:51.968 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:53:51.968 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-10-09T05:53:51.969 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd in 7 2023-10-09T05:53:52.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:52 smithi064 ceph-mon[212750]: pgmap v390: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 peering, 1 active+recovery_wait+degraded+remapped, 93 active+clean, 3 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 16769/267810 objects degraded (6.262%); 47938/267810 objects misplaced (17.900%); 13 MiB/s, 213 objects/s recovering 2023-10-09T05:53:52.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:52 smithi064 ceph-mon[220044]: pgmap v390: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 peering, 1 active+recovery_wait+degraded+remapped, 93 active+clean, 3 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 16769/267810 objects degraded (6.262%); 47938/267810 objects misplaced (17.900%); 13 MiB/s, 213 objects/s recovering 2023-10-09T05:53:52.470 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:53:52.470 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:53:52.522 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:52.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:52.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:52.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:52 smithi156 ceph-mon[195101]: pgmap v390: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 peering, 1 active+recovery_wait+degraded+remapped, 93 active+clean, 3 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 16769/267810 objects degraded (6.262%); 47938/267810 objects misplaced (17.900%); 13 MiB/s, 213 objects/s recovering 2023-10-09T05:53:52.626 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:53:52.662 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:53.517 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:53:53.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:53.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:53:53.621 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:53.649 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:53.688 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:53:53.700 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:53.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:53.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:53.736 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:53:53.759 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:53:53.782 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:53:53.792 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:53.802 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:53:53.805 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:53.833 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:53:53.850 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:53.864 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:53:53.896 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:53:53.904 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:53:53.904 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:53:53.904 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:53:53.904 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:53:53.904 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:53:53.905 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:53:53.905 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:53:53.905 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:53:53.951 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:53.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:53.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:54.056 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:54.072 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:53:54.090 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:54.149 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:54 smithi064 ceph-mon[212750]: pgmap v391: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 peering, 1 active+recovery_wait+degraded+remapped, 93 active+clean, 3 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 16769/267810 objects degraded (6.262%); 47938/267810 objects misplaced (17.900%); 12 MiB/s, 192 objects/s recovering 2023-10-09T05:53:54.149 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:54 smithi064 ceph-mon[220044]: pgmap v391: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 peering, 1 active+recovery_wait+degraded+remapped, 93 active+clean, 3 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 16769/267810 objects degraded (6.262%); 47938/267810 objects misplaced (17.900%); 12 MiB/s, 192 objects/s recovering 2023-10-09T05:53:54.152 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:54.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:54.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:54.256 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:54.298 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:54.489 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:54 smithi156 ceph-mon[195101]: pgmap v391: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 peering, 1 active+recovery_wait+degraded+remapped, 93 active+clean, 3 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 16769/267810 objects degraded (6.262%); 47938/267810 objects misplaced (17.900%); 12 MiB/s, 192 objects/s recovering 2023-10-09T05:53:55.242 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T05:53:55.412 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:55.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:55 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3221664338' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:53:55.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:55 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:53:55.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:55 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3221664338' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:53:55.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:55 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:53:55.415 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:53:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:53:55] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.43.0" 2023-10-09T05:53:55.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:55.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:55.517 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:55.544 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:55 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3221664338' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:53:55.545 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:55 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:53:55.548 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:55.598 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:55.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:55.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:55.702 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:55.730 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:55.780 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:55.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:55.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:55.884 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:55.912 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:55.957 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:55.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:55.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:56.060 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:56.094 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:56.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:56 smithi064 ceph-mon[220044]: pgmap v392: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 97 active+clean, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 28612/267741 objects degraded (10.686%); 43291/267741 objects misplaced (16.169%); 13 MiB/s, 202 objects/s recovering 2023-10-09T05:53:56.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:56 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 28612/267741 objects degraded (10.686%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:56.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:56 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T05:53:56.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:56 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:56.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:56 smithi064 ceph-mon[220044]: failed to encode map e172 with expected crc 2023-10-09T05:53:56.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:56 smithi064 ceph-mon[220044]: failed to encode map e172 with expected crc 2023-10-09T05:53:56.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:56 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:53:56.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:56 smithi064 ceph-mon[220044]: osdmap e172: 8 total, 8 up, 7 in 2023-10-09T05:53:56.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:56 smithi064 ceph-mon[212750]: pgmap v392: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 97 active+clean, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 28612/267741 objects degraded (10.686%); 43291/267741 objects misplaced (16.169%); 13 MiB/s, 202 objects/s recovering 2023-10-09T05:53:56.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:56 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 28612/267741 objects degraded (10.686%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:56.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:56 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T05:53:56.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:56 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:56.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:56 smithi064 ceph-mon[212750]: failed to encode map e172 with expected crc 2023-10-09T05:53:56.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:56 smithi064 ceph-mon[212750]: failed to encode map e172 with expected crc 2023-10-09T05:53:56.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:56 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:53:56.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:56 smithi064 ceph-mon[212750]: osdmap e172: 8 total, 8 up, 7 in 2023-10-09T05:53:56.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:56 smithi156 ceph-mon[195101]: pgmap v392: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 97 active+clean, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 28612/267741 objects degraded (10.686%); 43291/267741 objects misplaced (16.169%); 13 MiB/s, 202 objects/s recovering 2023-10-09T05:53:56.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:56 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 28612/267741 objects degraded (10.686%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:53:56.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:56 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T05:53:56.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:56 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:53:56.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:56 smithi156 ceph-mon[195101]: failed to encode map e172 with expected crc 2023-10-09T05:53:56.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:56 smithi156 ceph-mon[195101]: failed to encode map e172 with expected crc 2023-10-09T05:53:56.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:56 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:53:56.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:56 smithi156 ceph-mon[195101]: osdmap e172: 8 total, 8 up, 7 in 2023-10-09T05:53:56.654 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:56.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:56.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:56.761 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:56.788 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:56.835 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:56.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:56.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:56.940 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:56.968 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:57.016 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:57.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:57.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:57.123 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:57.151 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:57.198 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:57.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:57.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:57.302 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:57.341 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:57.397 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:57.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:57.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:57.500 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:57.529 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:57.577 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:57.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:57.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:57.680 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:53:57.708 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:57.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:57 smithi064 conmon[212726]: 2023-10-09T05:53:57.364+0000 7f8823f23700 -1 mon.a@0(leader).osd e172 definitely_dead 0 2023-10-09T05:53:57.754 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:53:57.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:57.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:53:57.859 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:53:57.892 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:57.944 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:53:57.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:57.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:53:58.048 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:58.078 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:58.128 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:58.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:58.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:58.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:58 smithi064 ceph-mon[212750]: pgmap v394: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 97 active+clean, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 28612/267741 objects degraded (10.686%); 42980/267741 objects misplaced (16.053%); 13 MiB/s, 206 objects/s recovering 2023-10-09T05:53:58.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:58 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2542928778' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:53:58.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:58 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:53:58.178 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:58 smithi064 ceph-mon[220044]: pgmap v394: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 97 active+clean, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 28612/267741 objects degraded (10.686%); 42980/267741 objects misplaced (16.053%); 13 MiB/s, 206 objects/s recovering 2023-10-09T05:53:58.178 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:58 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2542928778' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:53:58.178 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:58 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:53:58.187 INFO:teuthology.orchestra.run.smithi064.stderr:marked in osd.7. 2023-10-09T05:53:58.232 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:53:58.268 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:58.422 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:53:58.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:58.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:53:58.526 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:58.550 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:58 smithi156 ceph-mon[195101]: pgmap v394: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 97 active+clean, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 28612/267741 objects degraded (10.686%); 42980/267741 objects misplaced (16.053%); 13 MiB/s, 206 objects/s recovering 2023-10-09T05:53:58.550 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:58 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2542928778' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:53:58.550 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:58 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:53:58.553 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:58.572 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:53:58.596 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:53:58.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:58.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:53:58.602 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:53:58.700 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:58.726 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:58.774 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:58.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:58.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:58.877 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:53:58.905 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:58.949 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:53:58.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:58.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:53:59.052 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:53:59.081 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:59.124 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:53:59.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:59.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:53:59.184 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:59 smithi156 ceph-mon[195101]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.184 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:59 smithi156 ceph-mon[195101]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.184 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:59 smithi156 ceph-mon[195101]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.184 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:59 smithi156 ceph-mon[195101]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.184 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:59 smithi156 ceph-mon[195101]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.185 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:59 smithi156 ceph-mon[195101]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.185 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:59 smithi156 ceph-mon[195101]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.185 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:59 smithi156 ceph-mon[195101]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.185 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:59 smithi156 ceph-mon[195101]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.185 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:59 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T05:53:59.185 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:59 smithi156 ceph-mon[195101]: osdmap e173: 8 total, 8 up, 8 in 2023-10-09T05:53:59.185 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:59 smithi156 ceph-mon[195101]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.185 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:59 smithi156 ceph-mon[195101]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.185 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:59 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:59.185 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:53:59.185 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:53:59 smithi156 ceph-mon[195101]: osdmap e174: 8 total, 8 up, 8 in 2023-10-09T05:53:59.227 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:53:59.265 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:53:59.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[212750]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[212750]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[212750]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[212750]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[212750]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[212750]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[212750]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[212750]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[212750]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T05:53:59.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[212750]: osdmap e173: 8 total, 8 up, 8 in 2023-10-09T05:53:59.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[212750]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[212750]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:59.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:53:59.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[212750]: osdmap e174: 8 total, 8 up, 8 in 2023-10-09T05:53:59.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[220044]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[220044]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[220044]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[220044]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[220044]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[220044]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[220044]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[220044]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[220044]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T05:53:59.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[220044]: osdmap e173: 8 total, 8 up, 8 in 2023-10-09T05:53:59.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[220044]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[220044]: failed to encode map e173 with expected crc 2023-10-09T05:53:59.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:53:59.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:53:59.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:53:59 smithi064 ceph-mon[220044]: osdmap e174: 8 total, 8 up, 8 in 2023-10-09T05:53:59.834 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:53:59.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:53:59.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:53:59.937 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:53:59.964 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:00.009 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:00.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:00.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:00.047 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-10-09T05:54:00.088 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:00.112 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:00.148 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:00.225 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:00.253 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:00.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:00.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:00.359 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:00.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:00 smithi156 ceph-mon[195101]: pgmap v396: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 97 active+clean, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 28612/267741 objects degraded (10.686%); 42980/267741 objects misplaced (16.053%); 8.3 MiB/s, 133 objects/s recovering 2023-10-09T05:54:00.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:00 smithi156 ceph-mon[195101]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:00 smithi156 ceph-mon[195101]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:00 smithi156 ceph-mon[195101]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:00 smithi156 ceph-mon[195101]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:00 smithi156 ceph-mon[195101]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:00 smithi156 ceph-mon[195101]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:00 smithi156 ceph-mon[195101]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:00 smithi156 ceph-mon[195101]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:00 smithi156 ceph-mon[195101]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:00 smithi156 ceph-mon[195101]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:00 smithi156 ceph-mon[195101]: osdmap e175: 8 total, 8 up, 8 in 2023-10-09T05:54:00.388 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:00.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:00 smithi064 ceph-mon[212750]: pgmap v396: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 97 active+clean, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 28612/267741 objects degraded (10.686%); 42980/267741 objects misplaced (16.053%); 8.3 MiB/s, 133 objects/s recovering 2023-10-09T05:54:00.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:00 smithi064 ceph-mon[212750]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:00 smithi064 ceph-mon[212750]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:00 smithi064 ceph-mon[212750]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:00 smithi064 ceph-mon[212750]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:00 smithi064 ceph-mon[212750]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:00 smithi064 ceph-mon[212750]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:00 smithi064 ceph-mon[212750]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:00 smithi064 ceph-mon[212750]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:00 smithi064 ceph-mon[212750]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:00 smithi064 ceph-mon[212750]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:00 smithi064 ceph-mon[212750]: osdmap e175: 8 total, 8 up, 8 in 2023-10-09T05:54:00.403 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:00 smithi064 ceph-mon[220044]: pgmap v396: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 97 active+clean, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 28612/267741 objects degraded (10.686%); 42980/267741 objects misplaced (16.053%); 8.3 MiB/s, 133 objects/s recovering 2023-10-09T05:54:00.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:00 smithi064 ceph-mon[220044]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:00 smithi064 ceph-mon[220044]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:00 smithi064 ceph-mon[220044]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:00 smithi064 ceph-mon[220044]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:00 smithi064 ceph-mon[220044]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:00 smithi064 ceph-mon[220044]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:00 smithi064 ceph-mon[220044]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:00 smithi064 ceph-mon[220044]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:00 smithi064 ceph-mon[220044]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:00 smithi064 ceph-mon[220044]: failed to encode map e174 with expected crc 2023-10-09T05:54:00.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:00 smithi064 ceph-mon[220044]: osdmap e175: 8 total, 8 up, 8 in 2023-10-09T05:54:00.435 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:00.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:00.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:00.539 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:00.573 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:00.895 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:00.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:00.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:00.999 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:01.027 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:01.070 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:01.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:01.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:01.173 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:01.202 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:01 smithi064 ceph-mon[220044]: failed to encode map e175 with expected crc 2023-10-09T05:54:01.202 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:01 smithi064 ceph-mon[220044]: failed to encode map e175 with expected crc 2023-10-09T05:54:01.202 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:01 smithi064 ceph-mon[220044]: failed to encode map e175 with expected crc 2023-10-09T05:54:01.202 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:01 smithi064 ceph-mon[220044]: failed to encode map e175 with expected crc 2023-10-09T05:54:01.202 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:01 smithi064 ceph-mon[220044]: failed to encode map e175 with expected crc 2023-10-09T05:54:01.202 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:01 smithi064 ceph-mon[220044]: failed to encode map e175 with expected crc 2023-10-09T05:54:01.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:01 smithi064 ceph-mon[212750]: failed to encode map e175 with expected crc 2023-10-09T05:54:01.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:01 smithi064 ceph-mon[212750]: failed to encode map e175 with expected crc 2023-10-09T05:54:01.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:01 smithi064 ceph-mon[212750]: failed to encode map e175 with expected crc 2023-10-09T05:54:01.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:01 smithi064 ceph-mon[212750]: failed to encode map e175 with expected crc 2023-10-09T05:54:01.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:01 smithi064 ceph-mon[212750]: failed to encode map e175 with expected crc 2023-10-09T05:54:01.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:01 smithi064 ceph-mon[212750]: failed to encode map e175 with expected crc 2023-10-09T05:54:01.212 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:01.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:01 smithi156 ceph-mon[195101]: failed to encode map e175 with expected crc 2023-10-09T05:54:01.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:01 smithi156 ceph-mon[195101]: failed to encode map e175 with expected crc 2023-10-09T05:54:01.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:01 smithi156 ceph-mon[195101]: failed to encode map e175 with expected crc 2023-10-09T05:54:01.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:01 smithi156 ceph-mon[195101]: failed to encode map e175 with expected crc 2023-10-09T05:54:01.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:01 smithi156 ceph-mon[195101]: failed to encode map e175 with expected crc 2023-10-09T05:54:01.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:01 smithi156 ceph-mon[195101]: failed to encode map e175 with expected crc 2023-10-09T05:54:01.816 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:01.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:01.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:01.919 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:01.946 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:01.991 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:01.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:01.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:02.094 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:02.121 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:02.165 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:02.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:02.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:02.268 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:02.296 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:02.340 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:02.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:02.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:02.372 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:02 smithi064 ceph-mon[220044]: pgmap v399: 109 pgs: 1 active+recovering+undersized+remapped, 7 peering, 1 activating+undersized+degraded+remapped, 97 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 23957/267783 objects degraded (8.946%); 11048/267783 objects misplaced (4.126%) 2023-10-09T05:54:02.372 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:02 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 23957/267783 objects degraded (8.946%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:02.373 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:02 smithi064 ceph-mon[220044]: failed to encode map e176 with expected crc 2023-10-09T05:54:02.373 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:02 smithi064 ceph-mon[220044]: failed to encode map e176 with expected crc 2023-10-09T05:54:02.373 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:02 smithi064 ceph-mon[220044]: failed to encode map e176 with expected crc 2023-10-09T05:54:02.373 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:02 smithi064 ceph-mon[220044]: failed to encode map e176 with expected crc 2023-10-09T05:54:02.373 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:02 smithi064 ceph-mon[220044]: osdmap e176: 8 total, 8 up, 8 in 2023-10-09T05:54:02.373 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:02 smithi064 ceph-mon[212750]: pgmap v399: 109 pgs: 1 active+recovering+undersized+remapped, 7 peering, 1 activating+undersized+degraded+remapped, 97 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 23957/267783 objects degraded (8.946%); 11048/267783 objects misplaced (4.126%) 2023-10-09T05:54:02.373 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:02 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 23957/267783 objects degraded (8.946%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:02.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:02 smithi064 ceph-mon[212750]: failed to encode map e176 with expected crc 2023-10-09T05:54:02.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:02 smithi064 ceph-mon[212750]: failed to encode map e176 with expected crc 2023-10-09T05:54:02.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:02 smithi064 ceph-mon[212750]: failed to encode map e176 with expected crc 2023-10-09T05:54:02.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:02 smithi064 ceph-mon[212750]: failed to encode map e176 with expected crc 2023-10-09T05:54:02.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:02 smithi064 ceph-mon[212750]: osdmap e176: 8 total, 8 up, 8 in 2023-10-09T05:54:02.443 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:02.483 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:02.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:02 smithi156 ceph-mon[195101]: pgmap v399: 109 pgs: 1 active+recovering+undersized+remapped, 7 peering, 1 activating+undersized+degraded+remapped, 97 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 23957/267783 objects degraded (8.946%); 11048/267783 objects misplaced (4.126%) 2023-10-09T05:54:02.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:02 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 23957/267783 objects degraded (8.946%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:02.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:02 smithi156 ceph-mon[195101]: failed to encode map e176 with expected crc 2023-10-09T05:54:02.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:02 smithi156 ceph-mon[195101]: failed to encode map e176 with expected crc 2023-10-09T05:54:02.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:02 smithi156 ceph-mon[195101]: failed to encode map e176 with expected crc 2023-10-09T05:54:02.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:02 smithi156 ceph-mon[195101]: failed to encode map e176 with expected crc 2023-10-09T05:54:02.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:02 smithi156 ceph-mon[195101]: osdmap e176: 8 total, 8 up, 8 in 2023-10-09T05:54:02.809 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:02.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:02.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:02.913 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:02.958 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:03.024 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:03.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:03.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:03.128 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:03.161 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:03.211 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:03.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:03.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:03.239 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T05:54:03.273 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:03 smithi156 ceph-mon[195101]: osdmap e177: 8 total, 8 up, 8 in 2023-10-09T05:54:03.273 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:03 smithi156 ceph-mon[195101]: failed to encode map e177 with expected crc 2023-10-09T05:54:03.273 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:03 smithi156 ceph-mon[195101]: failed to encode map e177 with expected crc 2023-10-09T05:54:03.273 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:03 smithi156 ceph-mon[195101]: pgmap v402: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 7 peering, 97 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 12137/267783 objects degraded (4.532%); 14897/267783 objects misplaced (5.563%) 2023-10-09T05:54:03.274 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:03 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1604667939' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:54:03.274 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:03 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1214413469' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:03.274 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:03 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:03.315 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:03.345 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:03.432 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:03 smithi064 ceph-mon[212750]: osdmap e177: 8 total, 8 up, 8 in 2023-10-09T05:54:03.432 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:03 smithi064 ceph-mon[212750]: failed to encode map e177 with expected crc 2023-10-09T05:54:03.432 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:03 smithi064 ceph-mon[212750]: failed to encode map e177 with expected crc 2023-10-09T05:54:03.432 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:03 smithi064 ceph-mon[212750]: pgmap v402: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 7 peering, 97 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 12137/267783 objects degraded (4.532%); 14897/267783 objects misplaced (5.563%) 2023-10-09T05:54:03.432 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:03 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1604667939' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:54:03.432 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:03 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1214413469' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:03.432 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:03 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:03.433 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:03 smithi064 ceph-mon[220044]: osdmap e177: 8 total, 8 up, 8 in 2023-10-09T05:54:03.433 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:03 smithi064 ceph-mon[220044]: failed to encode map e177 with expected crc 2023-10-09T05:54:03.433 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:03 smithi064 ceph-mon[220044]: failed to encode map e177 with expected crc 2023-10-09T05:54:03.433 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:03 smithi064 ceph-mon[220044]: pgmap v402: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 7 peering, 97 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 12137/267783 objects degraded (4.532%); 14897/267783 objects misplaced (5.563%) 2023-10-09T05:54:03.433 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:03 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1604667939' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:54:03.433 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:03 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1214413469' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:03.433 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:03 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:03.444 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:03.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:03.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:03.487 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:54:03.488 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T05:54:03.488 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:03.488 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "MOSDPGPushReply(6.e 178/139 [PushReplyOp(6:717c01f0:::benchmark_data_smithi064_284193_object17509:head)])", 2023-10-09T05:54:03.488 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:54:03.487419+0000", 2023-10-09T05:54:03.488 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.000115956, 2023-10-09T05:54:03.488 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013646900000000001, 2023-10-09T05:54:03.488 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:03.488 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T05:54:03.488 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:03.488 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:03.489 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:03.489 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:03.487419+0000", 2023-10-09T05:54:03.489 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:03.489 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:03.489 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:03.489 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:03.489 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:03.487419+0000", 2023-10-09T05:54:03.489 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:03.489 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:03.489 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:03.489 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:03.489 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:03.487418+0000", 2023-10-09T05:54:03.490 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:54:03.490 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:03.490 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:03.490 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:03.490 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:03.487424+0000", 2023-10-09T05:54:03.490 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6169999999999998e-06 2023-10-09T05:54:03.490 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:03.490 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:03.490 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:03.490 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:03.487435+0000", 2023-10-09T05:54:03.490 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.151e-05 2023-10-09T05:54:03.491 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:03.491 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:03.491 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:03.491 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:03.487441+0000", 2023-10-09T05:54:03.491 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8490000000000002e-06 2023-10-09T05:54:03.491 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:03.491 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:03.491 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:03.491 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:03.487490+0000", 2023-10-09T05:54:03.491 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8569999999999997e-05 2023-10-09T05:54:03.491 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:03.491 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:03.491 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:03.492 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:03.492 INFO:teuthology.orchestra.run.smithi156.stdout: ], 2023-10-09T05:54:03.492 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 1 2023-10-09T05:54:03.492 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:54:03.549 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:03.581 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:03.638 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:03.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:03.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:03.742 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:03.772 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:04.032 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:04.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:04.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:04.136 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:04.164 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:04.209 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:04.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:04.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:04.313 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:04.339 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:04.384 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:04.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:04.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:04.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:04 smithi064 ceph-mon[212750]: failed to encode map e177 with expected crc 2023-10-09T05:54:04.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:04 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:54:04.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:04 smithi064 ceph-mon[212750]: failed to encode map e178 with expected crc 2023-10-09T05:54:04.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:04 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:54:04.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:04 smithi064 ceph-mon[212750]: osdmap e178: 8 total, 8 up, 8 in 2023-10-09T05:54:04.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:04 smithi064 ceph-mon[212750]: failed to encode map e178 with expected crc 2023-10-09T05:54:04.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:04 smithi064 ceph-mon[212750]: failed to encode map e178 with expected crc 2023-10-09T05:54:04.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:04 smithi064 ceph-mon[212750]: failed to encode map e178 with expected crc 2023-10-09T05:54:04.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:04 smithi064 ceph-mon[220044]: failed to encode map e177 with expected crc 2023-10-09T05:54:04.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:04 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:54:04.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:04 smithi064 ceph-mon[220044]: failed to encode map e178 with expected crc 2023-10-09T05:54:04.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:04 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:54:04.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:04 smithi064 ceph-mon[220044]: osdmap e178: 8 total, 8 up, 8 in 2023-10-09T05:54:04.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:04 smithi064 ceph-mon[220044]: failed to encode map e178 with expected crc 2023-10-09T05:54:04.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:04 smithi064 ceph-mon[220044]: failed to encode map e178 with expected crc 2023-10-09T05:54:04.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:04 smithi064 ceph-mon[220044]: failed to encode map e178 with expected crc 2023-10-09T05:54:04.487 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:04.511 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:04 smithi156 ceph-mon[195101]: failed to encode map e177 with expected crc 2023-10-09T05:54:04.511 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:04 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:54:04.511 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:04 smithi156 ceph-mon[195101]: failed to encode map e178 with expected crc 2023-10-09T05:54:04.511 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:04 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:54:04.511 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:04 smithi156 ceph-mon[195101]: osdmap e178: 8 total, 8 up, 8 in 2023-10-09T05:54:04.511 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:04 smithi156 ceph-mon[195101]: failed to encode map e178 with expected crc 2023-10-09T05:54:04.511 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:04 smithi156 ceph-mon[195101]: failed to encode map e178 with expected crc 2023-10-09T05:54:04.511 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:04 smithi156 ceph-mon[195101]: failed to encode map e178 with expected crc 2023-10-09T05:54:04.513 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:04.556 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:04.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:04.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:04.660 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:04.687 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:04.732 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:04.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:04.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:04.837 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:04.863 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:04.908 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:04.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:04.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:05.012 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:05.039 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:05.048 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 1, 7] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T05:54:05.048 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:54:05.048 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T05:54:05.049 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-10-09T05:54:05.224 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:05.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:05.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:05.277 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T05:54:05.328 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:05.354 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:05.402 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:05.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:05.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:05.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:05 smithi064 ceph-mon[212750]: pgmap v404: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 remapped+peering, 2 peering, 100 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 3 op/s; 18160/267741 objects degraded (6.783%); 14311/267741 objects misplaced (5.345%); 8.7 MiB/s, 139 objects/s recovering 2023-10-09T05:54:05.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:05 smithi064 ceph-mon[220044]: pgmap v404: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 remapped+peering, 2 peering, 100 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 3 op/s; 18160/267741 objects degraded (6.783%); 14311/267741 objects misplaced (5.345%); 8.7 MiB/s, 139 objects/s recovering 2023-10-09T05:54:05.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:54:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:54:05] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-10-09T05:54:05.504 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:05.507 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:05.533 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:05 smithi156 ceph-mon[195101]: pgmap v404: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 remapped+peering, 2 peering, 100 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 3 op/s; 18160/267741 objects degraded (6.783%); 14311/267741 objects misplaced (5.345%); 8.7 MiB/s, 139 objects/s recovering 2023-10-09T05:54:05.536 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:05.633 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:05.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:05.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:05.657 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:54:05.740 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:05.768 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:05.917 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:05.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:05.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:06.019 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:06.048 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:06.098 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:54:06.198 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:06.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:06.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:06.302 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:06.333 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:06.395 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:06.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:06.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:06.500 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:06.529 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:06.574 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:06.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:06.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:06.678 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:06.705 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:06.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:06 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 19272/267741 objects degraded (7.198%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:06.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:06 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 19272/267741 objects degraded (7.198%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:06.750 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:06.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:06.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:06.853 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:06.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:06 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 19272/267741 objects degraded (7.198%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:06.898 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:06.898 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:07.055 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:07.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:07.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:07.159 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:07.220 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:07.435 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:07.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:07.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:07.541 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:07.569 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:07.667 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:07.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:07.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:07.743 INFO:teuthology.orchestra.run.smithi064.stdout:moved 0 / 327 (0%) 2023-10-09T05:54:07.743 INFO:teuthology.orchestra.run.smithi064.stdout:avg 40.875 2023-10-09T05:54:07.743 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 9.71388 -> 9.71388 (expected baseline 5.98044) 2023-10-09T05:54:07.743 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.2 with 23 -> 23 pgs (0.562691 -> 0.562691 * mean) 2023-10-09T05:54:07.744 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.6 with 51 -> 51 pgs (1.24771 -> 1.24771 * mean) 2023-10-09T05:54:07.744 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:54:07.744 INFO:teuthology.orchestra.run.smithi064.stdout:oload 1000 2023-10-09T05:54:07.744 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 0.05 2023-10-09T05:54:07.744 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-10-09T05:54:07.744 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0268 2023-10-09T05:54:07.744 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.2684 2023-10-09T05:54:07.750 INFO:teuthology.orchestra.run.smithi064.stderr:no change 2023-10-09T05:54:07.771 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:07.808 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:07.862 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:54:07.862 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T05:54:07.862 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T05:54:07.862 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T05:54:07.862 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:54:07.863 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:07.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:07.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:07.967 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:08.004 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:08.287 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:08.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:08.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:08.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:08 smithi156 ceph-mon[195101]: pgmap v405: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 remapped+peering, 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 2 op/s; 19272/267741 objects degraded (7.198%); 14311/267741 objects misplaced (5.345%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T05:54:08.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:08.420 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:08 smithi064 ceph-mon[212750]: pgmap v405: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 remapped+peering, 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 2 op/s; 19272/267741 objects degraded (7.198%); 14311/267741 objects misplaced (5.345%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T05:54:08.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:08 smithi064 ceph-mon[220044]: pgmap v405: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 remapped+peering, 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 2 op/s; 19272/267741 objects degraded (7.198%); 14311/267741 objects misplaced (5.345%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T05:54:08.423 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:08.658 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:08.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:08.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:08.762 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:08.798 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:08.834 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T05:54:08.834 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-10-09T05:54:08.910 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:08.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:08.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:09.014 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:09.050 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:09.135 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T05:54:09.158 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:09.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:09.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:09.264 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:09.292 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:09.345 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:09.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:09.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:09.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:09 smithi156 ceph-mon[195101]: from='client.34947 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:09.391 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:09.457 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:09.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:09 smithi064 ceph-mon[212750]: from='client.34947 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:09.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:09 smithi064 ceph-mon[220044]: from='client.34947 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:09.488 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:09.527 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:54:09.537 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:09.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:09.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:09.647 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:09.675 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:09.725 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:09.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:09.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:09.829 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:09.858 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:10.108 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:10.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:10.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:10.148 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T05:54:10.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:10.246 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:10 smithi064 ceph-mon[212750]: pgmap v406: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 remapped+peering, 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 19272/267741 objects degraded (7.198%); 14311/267741 objects misplaced (5.345%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T05:54:10.246 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:10 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2766018198' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:54:10.246 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:10 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:54:10.246 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:10 smithi064 ceph-mon[220044]: pgmap v406: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 remapped+peering, 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 19272/267741 objects degraded (7.198%); 14311/267741 objects misplaced (5.345%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T05:54:10.246 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:10 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2766018198' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:54:10.246 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:10 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:54:10.249 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:10.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:10 smithi156 ceph-mon[195101]: pgmap v406: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 remapped+peering, 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 19272/267741 objects degraded (7.198%); 14311/267741 objects misplaced (5.345%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T05:54:10.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:10 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2766018198' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:54:10.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:10 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:54:10.461 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:10.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:10.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:10.565 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:10.602 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:10.715 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:10.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:10.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:10.822 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:10.857 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:11.109 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:11.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:11.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:11.214 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:11.245 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:11.343 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:11.344 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:11 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:11.344 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:11 smithi156 ceph-mon[195101]: failed to encode map e179 with expected crc 2023-10-09T05:54:11.344 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:11 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:54:11.344 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:11 smithi156 ceph-mon[195101]: osdmap e179: 8 total, 8 up, 8 in 2023-10-09T05:54:11.345 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:11 smithi156 ceph-mon[195101]: failed to encode map e179 with expected crc 2023-10-09T05:54:11.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:11.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:11.447 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:11.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:11 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:11.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:11 smithi064 ceph-mon[212750]: failed to encode map e179 with expected crc 2023-10-09T05:54:11.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:11 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:54:11.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:11 smithi064 ceph-mon[212750]: osdmap e179: 8 total, 8 up, 8 in 2023-10-09T05:54:11.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:11 smithi064 ceph-mon[212750]: failed to encode map e179 with expected crc 2023-10-09T05:54:11.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:11 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:11.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:11 smithi064 ceph-mon[220044]: failed to encode map e179 with expected crc 2023-10-09T05:54:11.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:11 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:54:11.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:11 smithi064 ceph-mon[220044]: osdmap e179: 8 total, 8 up, 8 in 2023-10-09T05:54:11.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:11 smithi064 ceph-mon[220044]: failed to encode map e179 with expected crc 2023-10-09T05:54:11.487 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:11.541 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:11.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:11.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:11.589 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:54:11.589 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T05:54:11.589 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T05:54:11.590 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T05:54:11.590 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.590 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:45182 6.2 6:44deee03:::benchmark_data_smithi064_284193_object45181:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T05:54:11.590 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:13.801651+0000", 2023-10-09T05:54:11.590 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 177.785267258, 2023-10-09T05:54:11.590 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.33387342800000003, 2023-10-09T05:54:11.590 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:11.590 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:11.590 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:11.590 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:11.590 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:11.591 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 45182 2023-10-09T05:54:11.591 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.591 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:11.591 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.591 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:11.591 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801651+0000", 2023-10-09T05:54:11.593 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.593 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.593 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.594 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:11.594 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801651+0000", 2023-10-09T05:54:11.594 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.594 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.594 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.595 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:11.595 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801649+0000", 2023-10-09T05:54:11.595 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999976 2023-10-09T05:54:11.595 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.595 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.595 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:11.596 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801765+0000", 2023-10-09T05:54:11.596 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000115567 2023-10-09T05:54:11.596 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.596 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.596 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:11.596 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801784+0000", 2023-10-09T05:54:11.596 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9372e-05 2023-10-09T05:54:11.597 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.597 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.597 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:11.597 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801791+0000", 2023-10-09T05:54:11.597 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.99e-06 2023-10-09T05:54:11.597 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.597 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.597 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.598 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801841+0000", 2023-10-09T05:54:11.598 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0146999999999999e-05 2023-10-09T05:54:11.598 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.598 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.598 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:11.598 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801965+0000", 2023-10-09T05:54:11.598 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012344799999999999 2023-10-09T05:54:11.599 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.599 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.599 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:54:11.599 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.802107+0000", 2023-10-09T05:54:11.599 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014182800000000001 2023-10-09T05:54:11.599 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.599 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.599 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:11.599 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.804072+0000", 2023-10-09T05:54:11.599 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0019656370000000001 2023-10-09T05:54:11.599 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.599 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.599 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.600 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.806044+0000", 2023-10-09T05:54:11.600 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001971619 2023-10-09T05:54:11.600 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.600 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.600 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.600 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:14.135426+0000", 2023-10-09T05:54:11.600 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.32938189000000001 2023-10-09T05:54:11.600 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.600 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.600 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:11.600 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:14.135482+0000", 2023-10-09T05:54:11.601 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6468999999999998e-05 2023-10-09T05:54:11.601 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.601 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.601 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:11.601 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:14.135525+0000", 2023-10-09T05:54:11.601 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2639999999999998e-05 2023-10-09T05:54:11.601 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.601 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:11.601 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.601 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.601 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.601 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:45195 6.2 6:4b43d35c:::benchmark_data_smithi064_284193_object45194:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T05:54:11.602 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:13.809108+0000", 2023-10-09T05:54:11.602 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 177.77781057300001, 2023-10-09T05:54:11.602 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.327171871, 2023-10-09T05:54:11.602 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:11.602 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:11.602 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:11.602 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:11.602 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:11.602 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 45195 2023-10-09T05:54:11.602 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.602 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:11.603 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.603 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:11.603 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.809108+0000", 2023-10-09T05:54:11.603 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.603 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.603 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.603 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:11.603 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.809108+0000", 2023-10-09T05:54:11.603 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.603 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.603 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.603 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:11.604 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.809106+0000", 2023-10-09T05:54:11.604 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T05:54:11.604 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.604 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.604 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:11.604 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.809166+0000", 2023-10-09T05:54:11.604 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9568000000000003e-05 2023-10-09T05:54:11.604 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.604 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.604 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:11.604 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.809183+0000", 2023-10-09T05:54:11.604 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7079999999999999e-05 2023-10-09T05:54:11.605 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.605 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.605 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:11.605 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.809190+0000", 2023-10-09T05:54:11.605 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7750000000000004e-06 2023-10-09T05:54:11.605 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.605 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.605 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.605 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.809230+0000", 2023-10-09T05:54:11.605 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0442999999999999e-05 2023-10-09T05:54:11.605 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.606 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.606 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:11.606 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.809339+0000", 2023-10-09T05:54:11.606 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010882099999999999 2023-10-09T05:54:11.606 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.606 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.606 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:54:11.606 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.809433+0000", 2023-10-09T05:54:11.606 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3995999999999999e-05 2023-10-09T05:54:11.606 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.606 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.607 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:11.607 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.810847+0000", 2023-10-09T05:54:11.607 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001413865 2023-10-09T05:54:11.607 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.607 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.607 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.607 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.811421+0000", 2023-10-09T05:54:11.607 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00057435099999999996 2023-10-09T05:54:11.607 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.607 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.607 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.607 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:14.136190+0000", 2023-10-09T05:54:11.608 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.32476842299999997 2023-10-09T05:54:11.608 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.608 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.608 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:11.608 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:14.136241+0000", 2023-10-09T05:54:11.608 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1186000000000001e-05 2023-10-09T05:54:11.608 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.608 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.608 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:11.608 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:14.136280+0000", 2023-10-09T05:54:11.608 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9150999999999998e-05 2023-10-09T05:54:11.609 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.609 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:11.609 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.609 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.609 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.609 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:79609 6.2 6:48d281f4:::benchmark_data_smithi064_284193_object79608:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:54:11.610 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:52.389004+0000", 2023-10-09T05:54:11.610 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 139.19791461299999, 2023-10-09T05:54:11.610 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24667813099999999, 2023-10-09T05:54:11.610 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:11.610 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:11.610 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:11.610 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:11.610 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:11.610 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 79609 2023-10-09T05:54:11.610 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.611 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:11.611 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.611 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:11.611 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.389004+0000", 2023-10-09T05:54:11.611 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.611 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.611 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.611 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:11.611 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.389004+0000", 2023-10-09T05:54:11.611 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.611 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.611 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.612 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:11.612 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.389002+0000", 2023-10-09T05:54:11.612 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T05:54:11.612 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.612 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.612 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:11.612 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.389075+0000", 2023-10-09T05:54:11.612 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3022000000000007e-05 2023-10-09T05:54:11.612 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.612 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.612 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:11.612 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.389097+0000", 2023-10-09T05:54:11.613 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1464e-05 2023-10-09T05:54:11.613 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.613 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.613 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:11.613 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.389105+0000", 2023-10-09T05:54:11.613 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1759999999999993e-06 2023-10-09T05:54:11.613 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.613 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.613 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.613 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.389159+0000", 2023-10-09T05:54:11.613 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3625999999999998e-05 2023-10-09T05:54:11.613 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.614 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.614 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:11.614 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.389300+0000", 2023-10-09T05:54:11.614 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014157899999999999 2023-10-09T05:54:11.614 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.614 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.614 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:54:11.614 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.389446+0000", 2023-10-09T05:54:11.614 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014543800000000001 2023-10-09T05:54:11.614 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.614 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.615 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.615 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.391982+0000", 2023-10-09T05:54:11.615 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0025359620000000001 2023-10-09T05:54:11.615 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.615 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.615 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:11.615 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.392146+0000", 2023-10-09T05:54:11.615 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00016487899999999999 2023-10-09T05:54:11.615 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.615 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.615 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.615 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.635586+0000", 2023-10-09T05:54:11.616 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24343909699999999 2023-10-09T05:54:11.616 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.616 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.616 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:11.616 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.635644+0000", 2023-10-09T05:54:11.616 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8319000000000002e-05 2023-10-09T05:54:11.616 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.616 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.616 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:11.616 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.635682+0000", 2023-10-09T05:54:11.616 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8379e-05 2023-10-09T05:54:11.616 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.617 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:11.617 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.617 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.617 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.617 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:81819 6.e 6:7a828088:::benchmark_data_smithi064_284193_object81818:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T05:54:11.617 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:55.444417+0000", 2023-10-09T05:54:11.617 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 136.14250216400001, 2023-10-09T05:54:11.617 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.228842936, 2023-10-09T05:54:11.617 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:11.618 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:11.618 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:11.618 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:11.618 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:11.618 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 81819 2023-10-09T05:54:11.618 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.618 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:11.619 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.619 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:11.619 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.444417+0000", 2023-10-09T05:54:11.619 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.619 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.619 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.619 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:11.619 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.444417+0000", 2023-10-09T05:54:11.620 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.620 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.620 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.620 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:11.620 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.444415+0000", 2023-10-09T05:54:11.620 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T05:54:11.620 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.620 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.621 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:11.621 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.444499+0000", 2023-10-09T05:54:11.621 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.3634999999999995e-05 2023-10-09T05:54:11.621 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.621 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.621 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:11.622 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.444521+0000", 2023-10-09T05:54:11.622 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2002999999999999e-05 2023-10-09T05:54:11.622 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.622 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.622 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:11.622 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.444529+0000", 2023-10-09T05:54:11.623 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.7320000000000004e-06 2023-10-09T05:54:11.623 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.623 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.623 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.623 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.447092+0000", 2023-10-09T05:54:11.623 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0025628360000000002 2023-10-09T05:54:11.623 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.624 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.624 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:11.624 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.447231+0000", 2023-10-09T05:54:11.624 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000139288 2023-10-09T05:54:11.624 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.624 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.624 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T05:54:11.625 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.447386+0000", 2023-10-09T05:54:11.625 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015484099999999999 2023-10-09T05:54:11.625 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.625 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.625 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:11.625 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.449674+0000", 2023-10-09T05:54:11.625 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0022881160000000002 2023-10-09T05:54:11.625 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.625 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.625 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.626 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.450374+0000", 2023-10-09T05:54:11.626 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00069923900000000005 2023-10-09T05:54:11.626 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.626 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.626 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.626 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.671234+0000", 2023-10-09T05:54:11.626 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.22086087400000001 2023-10-09T05:54:11.626 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.626 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.626 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:11.626 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.671294+0000", 2023-10-09T05:54:11.626 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9494999999999999e-05 2023-10-09T05:54:11.627 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.627 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.627 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:11.627 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.673260+0000", 2023-10-09T05:54:11.627 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0019655580000000001 2023-10-09T05:54:11.627 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.627 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:11.627 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.627 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.627 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.627 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:91015 6.e 6:7dc6a610:::benchmark_data_smithi064_284193_object91014:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-10-09T05:54:11.627 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:10.210536+0000", 2023-10-09T05:54:11.628 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 121.376382913, 2023-10-09T05:54:11.628 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3031288990000001, 2023-10-09T05:54:11.628 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:11.628 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:11.628 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:11.628 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:11.628 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:11.628 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 91015 2023-10-09T05:54:11.628 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.628 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:11.628 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.628 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:11.629 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210536+0000", 2023-10-09T05:54:11.629 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.629 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.629 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.629 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:11.629 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210536+0000", 2023-10-09T05:54:11.629 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.629 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.629 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.629 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:11.629 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210534+0000", 2023-10-09T05:54:11.629 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:11.629 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.630 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.630 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:11.630 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210591+0000", 2023-10-09T05:54:11.630 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6641999999999997e-05 2023-10-09T05:54:11.630 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.630 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.630 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:11.630 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210601+0000", 2023-10-09T05:54:11.630 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0485999999999999e-05 2023-10-09T05:54:11.630 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.630 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.631 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:11.631 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210606+0000", 2023-10-09T05:54:11.631 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0320000000000003e-06 2023-10-09T05:54:11.631 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.631 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.631 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.631 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210705+0000", 2023-10-09T05:54:11.631 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.8412000000000006e-05 2023-10-09T05:54:11.631 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.631 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.631 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:11.631 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210714+0000", 2023-10-09T05:54:11.632 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.6160000000000008e-06 2023-10-09T05:54:11.632 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.632 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.632 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.632 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.497372+0000", 2023-10-09T05:54:11.632 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2866577770000001 2023-10-09T05:54:11.632 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.632 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.632 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:11.632 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.497531+0000", 2023-10-09T05:54:11.632 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015918599999999999 2023-10-09T05:54:11.632 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:54:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.497704+0000", 2023-10-09T05:54:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000172103 2023-10-09T05:54:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.502321+0000", 2023-10-09T05:54:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0046177689999999999 2023-10-09T05:54:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:11.634 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.502336+0000", 2023-10-09T05:54:11.634 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4392999999999999e-05 2023-10-09T05:54:11.634 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.634 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.634 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.634 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.511914+0000", 2023-10-09T05:54:11.634 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0095783229999999997 2023-10-09T05:54:11.634 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.634 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.634 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:11.635 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.511973+0000", 2023-10-09T05:54:11.635 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8718999999999998e-05 2023-10-09T05:54:11.635 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.635 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.635 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:11.635 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.513665+0000", 2023-10-09T05:54:11.635 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001691992 2023-10-09T05:54:11.635 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.636 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:11.636 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.636 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.636 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.636 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:92873 6.e 6:7106f1be:::benchmark_data_smithi064_284193_object92872:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-10-09T05:54:11.636 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:13.825845+0000", 2023-10-09T05:54:11.636 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 117.761073271, 2023-10-09T05:54:11.636 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.21518323, 2023-10-09T05:54:11.636 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:11.636 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:11.636 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:11.636 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:11.637 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:11.637 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 92873 2023-10-09T05:54:11.637 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.637 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:11.637 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.637 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:11.637 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:13.825845+0000", 2023-10-09T05:54:11.637 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.637 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.637 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.637 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:11.637 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:13.825845+0000", 2023-10-09T05:54:11.638 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.638 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.638 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.638 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:11.638 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:13.825844+0000", 2023-10-09T05:54:11.638 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T05:54:11.638 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.638 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.638 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:11.638 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:13.825890+0000", 2023-10-09T05:54:11.638 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6431999999999999e-05 2023-10-09T05:54:11.639 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.639 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.639 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:11.639 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:13.825905+0000", 2023-10-09T05:54:11.639 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5037e-05 2023-10-09T05:54:11.639 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.639 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.639 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:11.639 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:13.825912+0000", 2023-10-09T05:54:11.639 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6220000000000003e-06 2023-10-09T05:54:11.639 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.639 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.640 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.640 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:13.827277+0000", 2023-10-09T05:54:11.640 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0013649020000000001 2023-10-09T05:54:11.640 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.640 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.640 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:11.640 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:13.827437+0000", 2023-10-09T05:54:11.640 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00016064000000000001 2023-10-09T05:54:11.640 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.640 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.640 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:54:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:13.827616+0000", 2023-10-09T05:54:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00017840799999999999 2023-10-09T05:54:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:13.830382+0000", 2023-10-09T05:54:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0027665260000000001 2023-10-09T05:54:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:13.831362+0000", 2023-10-09T05:54:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00097923999999999993 2023-10-09T05:54:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.642 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.642 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.642 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.039395+0000", 2023-10-09T05:54:11.642 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.20803323100000001 2023-10-09T05:54:11.642 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.642 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.642 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:11.642 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.039454+0000", 2023-10-09T05:54:11.642 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9593999999999999e-05 2023-10-09T05:54:11.642 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.642 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.041029+0000", 2023-10-09T05:54:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0015742729999999999 2023-10-09T05:54:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:92874 6.2 6:4d432c3e:::benchmark_data_smithi064_284193_object92873:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-10-09T05:54:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:13.830527+0000", 2023-10-09T05:54:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 117.756391631, 2023-10-09T05:54:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.221765819, 2023-10-09T05:54:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:11.644 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:11.644 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:11.644 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:11.644 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:11.644 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 92874 2023-10-09T05:54:11.644 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.644 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:11.644 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.644 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:11.644 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:13.830527+0000", 2023-10-09T05:54:11.644 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.644 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.645 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.645 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:11.645 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:13.830527+0000", 2023-10-09T05:54:11.645 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.645 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.645 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.645 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:11.645 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:13.830526+0000", 2023-10-09T05:54:11.645 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:11.645 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.645 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.645 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:11.646 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:13.830575+0000", 2023-10-09T05:54:11.646 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9648000000000003e-05 2023-10-09T05:54:11.646 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.646 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.646 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:11.646 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:13.830589+0000", 2023-10-09T05:54:11.646 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3427e-05 2023-10-09T05:54:11.646 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.646 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.646 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:11.646 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:13.830596+0000", 2023-10-09T05:54:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6969999999999993e-06 2023-10-09T05:54:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:13.830640+0000", 2023-10-09T05:54:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3121999999999998e-05 2023-10-09T05:54:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:13.830746+0000", 2023-10-09T05:54:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010618 2023-10-09T05:54:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:54:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:13.830837+0000", 2023-10-09T05:54:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1572999999999999e-05 2023-10-09T05:54:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:13.833178+0000", 2023-10-09T05:54:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0023406669999999998 2023-10-09T05:54:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.026406+0000", 2023-10-09T05:54:11.649 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.19322778700000001 2023-10-09T05:54:11.649 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.649 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.649 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.649 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.052189+0000", 2023-10-09T05:54:11.649 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.025783493000000001 2023-10-09T05:54:11.649 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.649 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.649 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:11.649 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.052254+0000", 2023-10-09T05:54:11.649 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4752000000000006e-05 2023-10-09T05:54:11.650 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.650 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.650 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:11.650 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.052293+0000", 2023-10-09T05:54:11.650 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8909000000000003e-05 2023-10-09T05:54:11.650 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.650 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:11.650 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.650 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.650 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.650 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:93658 6.e 6:7acdb428:::benchmark_data_smithi064_284193_object93657:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2023-10-09T05:54:11.651 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:14.966172+0000", 2023-10-09T05:54:11.651 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 116.620747065, 2023-10-09T05:54:11.651 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.193895597, 2023-10-09T05:54:11.651 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:11.651 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:11.651 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:11.651 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:11.651 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:11.651 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 93658 2023-10-09T05:54:11.651 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.651 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:11.651 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.652 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:11.652 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.966172+0000", 2023-10-09T05:54:11.652 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.652 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.652 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.652 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:11.652 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.966172+0000", 2023-10-09T05:54:11.652 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.652 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.652 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.652 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:11.652 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.966171+0000", 2023-10-09T05:54:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:54:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.966234+0000", 2023-10-09T05:54:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3096999999999996e-05 2023-10-09T05:54:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.966245+0000", 2023-10-09T05:54:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0862e-05 2023-10-09T05:54:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.654 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:11.654 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.966250+0000", 2023-10-09T05:54:11.654 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1449999999999999e-06 2023-10-09T05:54:11.654 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.654 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.654 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.654 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.966294+0000", 2023-10-09T05:54:11.654 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4301999999999997e-05 2023-10-09T05:54:11.654 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.654 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.654 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:11.655 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.966428+0000", 2023-10-09T05:54:11.655 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013359099999999999 2023-10-09T05:54:11.655 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.655 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.655 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:54:11.655 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.966579+0000", 2023-10-09T05:54:11.655 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015183500000000001 2023-10-09T05:54:11.655 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.655 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.655 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:11.655 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.971077+0000", 2023-10-09T05:54:11.655 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0044978020000000004 2023-10-09T05:54:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.971725+0000", 2023-10-09T05:54:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00064747800000000005 2023-10-09T05:54:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.158264+0000", 2023-10-09T05:54:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.18653883800000001 2023-10-09T05:54:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.657 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:11.657 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.158319+0000", 2023-10-09T05:54:11.657 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5213000000000001e-05 2023-10-09T05:54:11.657 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.657 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.657 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:11.657 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.160067+0000", 2023-10-09T05:54:11.657 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0017485650000000001 2023-10-09T05:54:11.657 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.657 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:11.657 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.658 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.658 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.658 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:93659 6.2 6:4eb3b787:::benchmark_data_smithi064_284193_object93658:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2023-10-09T05:54:11.658 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:14.966259+0000", 2023-10-09T05:54:11.658 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 116.620659632, 2023-10-09T05:54:11.658 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.191577948, 2023-10-09T05:54:11.658 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:11.658 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:11.658 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:11.658 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:11.658 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 93659 2023-10-09T05:54:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.966259+0000", 2023-10-09T05:54:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.966259+0000", 2023-10-09T05:54:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.966259+0000", 2023-10-09T05:54:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:54:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.966285+0000", 2023-10-09T05:54:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.6041e-05 2023-10-09T05:54:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.661 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:11.661 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.966289+0000", 2023-10-09T05:54:11.661 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5909999999999998e-06 2023-10-09T05:54:11.661 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.661 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.661 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:11.661 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.966293+0000", 2023-10-09T05:54:11.661 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.3909999999999998e-06 2023-10-09T05:54:11.661 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.661 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.661 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.662 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.966334+0000", 2023-10-09T05:54:11.662 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0781999999999997e-05 2023-10-09T05:54:11.662 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.662 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.662 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:11.662 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.966412+0000", 2023-10-09T05:54:11.662 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7978000000000005e-05 2023-10-09T05:54:11.662 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.662 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.662 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:54:11.662 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.966502+0000", 2023-10-09T05:54:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0045999999999998e-05 2023-10-09T05:54:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.969330+0000", 2023-10-09T05:54:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002828522 2023-10-09T05:54:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.157698+0000", 2023-10-09T05:54:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.18836755499999999 2023-10-09T05:54:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.664 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.664 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.664 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.157757+0000", 2023-10-09T05:54:11.664 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9688999999999997e-05 2023-10-09T05:54:11.664 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.664 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.664 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:11.664 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.157797+0000", 2023-10-09T05:54:11.664 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9637e-05 2023-10-09T05:54:11.664 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.664 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.664 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:11.665 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.157837+0000", 2023-10-09T05:54:11.665 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0046999999999998e-05 2023-10-09T05:54:11.665 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.665 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:11.665 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.665 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.665 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.665 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:93662 6.2 6:4af33192:::benchmark_data_smithi064_284193_object93661:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2023-10-09T05:54:11.665 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:14.970942+0000", 2023-10-09T05:54:11.665 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 116.615976366, 2023-10-09T05:54:11.665 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.18985310399999999, 2023-10-09T05:54:11.665 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:11.666 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:11.666 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:11.666 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:11.666 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:11.666 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 93662 2023-10-09T05:54:11.666 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.666 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:11.666 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.666 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:11.666 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.970942+0000", 2023-10-09T05:54:11.666 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.667 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.667 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.667 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:11.667 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.970942+0000", 2023-10-09T05:54:11.667 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.667 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.667 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.667 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:11.667 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.970941+0000", 2023-10-09T05:54:11.667 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:11.667 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.668 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.668 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:11.668 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.970991+0000", 2023-10-09T05:54:11.668 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0374999999999998e-05 2023-10-09T05:54:11.668 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.668 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.668 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:11.668 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.971002+0000", 2023-10-09T05:54:11.668 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0101e-05 2023-10-09T05:54:11.668 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.668 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.668 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:11.669 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.971007+0000", 2023-10-09T05:54:11.669 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1449999999999999e-06 2023-10-09T05:54:11.669 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.669 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.669 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.669 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.971043+0000", 2023-10-09T05:54:11.669 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6232000000000001e-05 2023-10-09T05:54:11.669 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.669 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.669 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:11.669 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.971135+0000", 2023-10-09T05:54:11.669 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2448999999999995e-05 2023-10-09T05:54:11.670 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.670 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.670 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:54:11.670 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.971224+0000", 2023-10-09T05:54:11.670 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8339999999999998e-05 2023-10-09T05:54:11.670 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.670 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.670 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.670 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.157716+0000", 2023-10-09T05:54:11.670 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.18649191200000001 2023-10-09T05:54:11.670 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.671 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.671 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.671 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.157858+0000", 2023-10-09T05:54:11.671 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014225800000000001 2023-10-09T05:54:11.671 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.671 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.671 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:11.671 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.160697+0000", 2023-10-09T05:54:11.671 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0028392109999999999 2023-10-09T05:54:11.671 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.671 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.672 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:11.672 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.160753+0000", 2023-10-09T05:54:11.672 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6331e-05 2023-10-09T05:54:11.672 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.672 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.672 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:11.672 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.160795+0000", 2023-10-09T05:54:11.672 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2045999999999997e-05 2023-10-09T05:54:11.672 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.672 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:11.672 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.672 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:93665 6.2 6:486acb3e:::benchmark_data_smithi064_284193_object93664:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2023-10-09T05:54:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:14.972759+0000", 2023-10-09T05:54:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 116.61415976799999, 2023-10-09T05:54:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.188128131, 2023-10-09T05:54:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 93665 2023-10-09T05:54:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.674 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:11.674 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.674 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:11.674 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.972759+0000", 2023-10-09T05:54:11.674 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.674 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.674 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.674 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:11.674 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.972759+0000", 2023-10-09T05:54:11.674 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.674 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.675 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.675 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:11.675 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.972758+0000", 2023-10-09T05:54:11.675 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:54:11.675 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.675 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.675 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:11.675 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.972803+0000", 2023-10-09T05:54:11.675 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5210000000000003e-05 2023-10-09T05:54:11.675 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.675 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.675 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:11.676 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.972815+0000", 2023-10-09T05:54:11.676 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1447000000000001e-05 2023-10-09T05:54:11.676 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.676 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.676 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:11.676 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.972821+0000", 2023-10-09T05:54:11.676 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9889999999999997e-06 2023-10-09T05:54:11.676 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.676 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.676 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.676 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.973792+0000", 2023-10-09T05:54:11.676 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00097197199999999998 2023-10-09T05:54:11.677 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.677 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.677 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:11.677 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.973874+0000", 2023-10-09T05:54:11.677 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1377000000000004e-05 2023-10-09T05:54:11.677 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.677 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.677 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:54:11.677 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.973958+0000", 2023-10-09T05:54:11.677 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.3689999999999993e-05 2023-10-09T05:54:11.677 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.678 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.678 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.678 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.157730+0000", 2023-10-09T05:54:11.678 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.18377232099999999 2023-10-09T05:54:11.678 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.678 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.678 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.678 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.157872+0000", 2023-10-09T05:54:11.678 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014213100000000001 2023-10-09T05:54:11.678 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.679 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.679 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:11.679 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.160808+0000", 2023-10-09T05:54:11.679 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0029357110000000001 2023-10-09T05:54:11.679 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.679 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.679 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:11.679 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.160853+0000", 2023-10-09T05:54:11.679 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5633000000000002e-05 2023-10-09T05:54:11.679 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.679 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.679 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:11.680 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.160887+0000", 2023-10-09T05:54:11.680 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.3781000000000001e-05 2023-10-09T05:54:11.680 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.680 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:11.680 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.680 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.680 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.680 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:93671 6.2 6:43078def:::benchmark_data_smithi064_284193_object93670:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2023-10-09T05:54:11.680 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:14.975248+0000", 2023-10-09T05:54:11.680 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 116.611670762, 2023-10-09T05:54:11.680 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.18572555099999999, 2023-10-09T05:54:11.680 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:11.681 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:11.681 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:11.681 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:11.681 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:11.681 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 93671 2023-10-09T05:54:11.681 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.681 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:11.681 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.681 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:11.681 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.975248+0000", 2023-10-09T05:54:11.681 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.681 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.682 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.682 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:11.682 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.975248+0000", 2023-10-09T05:54:11.682 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.682 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.682 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.682 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:11.682 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.975247+0000", 2023-10-09T05:54:11.682 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:11.682 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.682 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.683 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:11.683 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.975292+0000", 2023-10-09T05:54:11.683 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5701999999999997e-05 2023-10-09T05:54:11.683 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.683 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.683 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:11.683 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.975305+0000", 2023-10-09T05:54:11.683 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2798e-05 2023-10-09T05:54:11.683 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.683 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.683 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:11.684 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.975312+0000", 2023-10-09T05:54:11.684 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3709999999999996e-06 2023-10-09T05:54:11.684 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.684 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.684 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.684 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.976086+0000", 2023-10-09T05:54:11.684 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00077468299999999997 2023-10-09T05:54:11.684 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.684 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.684 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:11.684 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.976172+0000", 2023-10-09T05:54:11.684 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6199000000000006e-05 2023-10-09T05:54:11.685 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.685 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.685 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:54:11.685 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.976259+0000", 2023-10-09T05:54:11.685 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6996999999999994e-05 2023-10-09T05:54:11.685 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.685 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.685 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.685 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.157743+0000", 2023-10-09T05:54:11.685 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.18148362800000001 2023-10-09T05:54:11.685 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.685 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.686 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.686 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.157886+0000", 2023-10-09T05:54:11.686 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014285799999999999 2023-10-09T05:54:11.686 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.686 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.686 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:11.686 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.160897+0000", 2023-10-09T05:54:11.686 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0030106270000000001 2023-10-09T05:54:11.686 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.686 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.686 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.160943+0000", 2023-10-09T05:54:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6805000000000003e-05 2023-10-09T05:54:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.160974+0000", 2023-10-09T05:54:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0147000000000001e-05 2023-10-09T05:54:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.688 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop_reply(client.44588.0:93659 6.2 e137/102 ondisk, result = 0)", 2023-10-09T05:54:11.688 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:14.978579+0000", 2023-10-09T05:54:11.688 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 116.60833959, 2023-10-09T05:54:11.688 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.17911981099999999, 2023-10-09T05:54:11.688 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:11.688 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T05:54:11.688 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:11.688 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.688 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:11.688 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.978579+0000", 2023-10-09T05:54:11.688 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.688 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.689 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.689 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:11.689 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.978579+0000", 2023-10-09T05:54:11.689 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.689 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.689 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.689 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:11.689 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.978578+0000", 2023-10-09T05:54:11.689 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:54:11.689 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.689 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.690 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:11.690 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.978583+0000", 2023-10-09T05:54:11.690 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5120000000000002e-06 2023-10-09T05:54:11.690 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.690 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.690 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:11.690 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.154721+0000", 2023-10-09T05:54:11.690 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.176137655 2023-10-09T05:54:11.690 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.690 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.690 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:11.690 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.154735+0000", 2023-10-09T05:54:11.691 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4885999999999999e-05 2023-10-09T05:54:11.691 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.691 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.691 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.691 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.157686+0000", 2023-10-09T05:54:11.691 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0029509079999999999 2023-10-09T05:54:11.691 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.691 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.691 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:11.691 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.157693+0000", 2023-10-09T05:54:11.691 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9450000000000004e-06 2023-10-09T05:54:11.692 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.692 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.692 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:11.692 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.157699+0000", 2023-10-09T05:54:11.692 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6540000000000004e-06 2023-10-09T05:54:11.692 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.692 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:11.692 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.692 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.692 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.692 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:93677 6.e 6:73ccb0a0:::benchmark_data_smithi064_284193_object93676:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2023-10-09T05:54:11.692 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:14.988963+0000", 2023-10-09T05:54:11.693 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 116.597955665, 2023-10-09T05:54:11.693 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.179299283, 2023-10-09T05:54:11.693 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:11.693 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:11.693 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:11.693 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:11.693 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:11.693 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 93677 2023-10-09T05:54:11.693 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.693 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:11.693 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.693 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:11.694 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.988963+0000", 2023-10-09T05:54:11.694 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.694 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.694 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.694 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:11.694 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.988963+0000", 2023-10-09T05:54:11.694 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.694 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.694 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.694 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:11.695 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.988961+0000", 2023-10-09T05:54:11.695 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T05:54:11.695 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.695 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.695 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:11.695 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:14.989010+0000", 2023-10-09T05:54:11.695 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9225999999999999e-05 2023-10-09T05:54:11.695 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.695 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.695 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:11.695 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.154723+0000", 2023-10-09T05:54:11.695 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.16571256700000001 2023-10-09T05:54:11.696 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.696 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.696 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:11.696 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.154737+0000", 2023-10-09T05:54:11.696 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3864e-05 2023-10-09T05:54:11.696 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.696 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.696 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.696 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.154791+0000", 2023-10-09T05:54:11.696 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3841999999999997e-05 2023-10-09T05:54:11.696 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.696 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.154982+0000", 2023-10-09T05:54:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000190953 2023-10-09T05:54:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:54:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.155181+0000", 2023-10-09T05:54:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00019952899999999999 2023-10-09T05:54:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.160153+0000", 2023-10-09T05:54:11.698 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0049715419999999998 2023-10-09T05:54:11.698 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.698 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.698 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.698 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.163783+0000", 2023-10-09T05:54:11.698 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0036301010000000002 2023-10-09T05:54:11.698 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.698 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.698 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:11.698 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.166504+0000", 2023-10-09T05:54:11.698 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00272113 2023-10-09T05:54:11.699 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.699 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.699 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:11.699 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.166556+0000", 2023-10-09T05:54:11.699 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2395999999999998e-05 2023-10-09T05:54:11.699 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.699 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.699 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:11.699 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:15.168262+0000", 2023-10-09T05:54:11.699 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001706029 2023-10-09T05:54:11.699 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:102991 6.2 6:4bf2f80f:::benchmark_data_smithi064_284193_object6448:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-10-09T05:54:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:53:24.290181+0000", 2023-10-09T05:54:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 47.296738034999997, 2023-10-09T05:54:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.937272463, 2023-10-09T05:54:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:11.701 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:11.701 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 102991 2023-10-09T05:54:11.701 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.701 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:11.701 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.701 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:11.701 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.290181+0000", 2023-10-09T05:54:11.701 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.701 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.701 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.701 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:11.701 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.290181+0000", 2023-10-09T05:54:11.702 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.702 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.702 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.702 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:11.702 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.290179+0000", 2023-10-09T05:54:11.702 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:11.702 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.702 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.702 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:11.702 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.290186+0000", 2023-10-09T05:54:11.702 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.024e-06 2023-10-09T05:54:11.703 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.703 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.703 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:11.703 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.290199+0000", 2023-10-09T05:54:11.703 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2965e-05 2023-10-09T05:54:11.703 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.703 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.703 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:11.703 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.290206+0000", 2023-10-09T05:54:11.703 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0690000000000004e-06 2023-10-09T05:54:11.703 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.703 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.704 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.704 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.290278+0000", 2023-10-09T05:54:11.704 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1543000000000003e-05 2023-10-09T05:54:11.704 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.704 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.704 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:11.704 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.290291+0000", 2023-10-09T05:54:11.704 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2626000000000001e-05 2023-10-09T05:54:11.704 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.704 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.704 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.704 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.211370+0000", 2023-10-09T05:54:11.705 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.92107952299999996 2023-10-09T05:54:11.705 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.705 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.705 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:11.705 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.211505+0000", 2023-10-09T05:54:11.705 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013526 2023-10-09T05:54:11.705 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.705 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.705 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:54:11.705 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.211651+0000", 2023-10-09T05:54:11.705 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014595499999999999 2023-10-09T05:54:11.706 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.706 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.706 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:11.706 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.220677+0000", 2023-10-09T05:54:11.706 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0090260459999999994 2023-10-09T05:54:11.706 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.706 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.706 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.706 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.224907+0000", 2023-10-09T05:54:11.706 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0042298359999999998 2023-10-09T05:54:11.706 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.707 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.707 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.707 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.227398+0000", 2023-10-09T05:54:11.707 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002490889 2023-10-09T05:54:11.707 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.707 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.707 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:11.707 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.227430+0000", 2023-10-09T05:54:11.707 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1681999999999999e-05 2023-10-09T05:54:11.707 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.707 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.707 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.227453+0000", 2023-10-09T05:54:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3339000000000001e-05 2023-10-09T05:54:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:102998 6.2 6:494ff968:::benchmark_data_smithi064_284193_object6455:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-10-09T05:54:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:53:24.383762+0000", 2023-10-09T05:54:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 47.203156561, 2023-10-09T05:54:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.84377951799999995, 2023-10-09T05:54:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:11.709 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:11.709 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:11.709 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:11.709 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:11.709 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 102998 2023-10-09T05:54:11.709 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.709 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:11.709 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.709 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:11.710 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.383762+0000", 2023-10-09T05:54:11.710 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.710 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.710 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.710 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:11.710 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.383762+0000", 2023-10-09T05:54:11.710 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.710 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.710 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.710 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:11.710 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.383761+0000", 2023-10-09T05:54:11.711 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:54:11.711 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.711 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.711 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:11.711 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.383768+0000", 2023-10-09T05:54:11.711 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2849999999999997e-06 2023-10-09T05:54:11.711 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.711 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.711 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:11.711 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.383787+0000", 2023-10-09T05:54:11.711 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9096000000000001e-05 2023-10-09T05:54:11.711 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.712 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.712 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:11.712 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.383796+0000", 2023-10-09T05:54:11.712 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.7800000000000006e-06 2023-10-09T05:54:11.712 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.712 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.712 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.712 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.383859+0000", 2023-10-09T05:54:11.712 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3124000000000001e-05 2023-10-09T05:54:11.712 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.712 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.712 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:11.713 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.383867+0000", 2023-10-09T05:54:11.713 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8199999999999997e-06 2023-10-09T05:54:11.713 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.713 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.713 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.713 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.214007+0000", 2023-10-09T05:54:11.713 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.830139446 2023-10-09T05:54:11.713 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.713 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.713 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:11.713 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.214113+0000", 2023-10-09T05:54:11.713 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010590900000000001 2023-10-09T05:54:11.714 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.714 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.714 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:54:11.714 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.214191+0000", 2023-10-09T05:54:11.714 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8597000000000007e-05 2023-10-09T05:54:11.714 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.714 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.714 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:11.714 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.220683+0000", 2023-10-09T05:54:11.714 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0064916330000000001 2023-10-09T05:54:11.714 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.715 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.715 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.715 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.224959+0000", 2023-10-09T05:54:11.715 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0042762679999999997 2023-10-09T05:54:11.715 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.715 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.715 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.715 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.227487+0000", 2023-10-09T05:54:11.715 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0025277699999999999 2023-10-09T05:54:11.715 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.715 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.716 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:11.716 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.227518+0000", 2023-10-09T05:54:11.716 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1337999999999997e-05 2023-10-09T05:54:11.716 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.716 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.716 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:11.716 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.227542+0000", 2023-10-09T05:54:11.716 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3601e-05 2023-10-09T05:54:11.716 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.716 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:11.717 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.717 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.717 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.717 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:102999 6.2 6:428b727a:::benchmark_data_smithi064_284193_object6456:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-10-09T05:54:11.717 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:53:24.393899+0000", 2023-10-09T05:54:11.717 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 47.193019454999998, 2023-10-09T05:54:11.717 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.83371729999999999, 2023-10-09T05:54:11.717 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:11.717 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:11.717 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:11.717 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:11.717 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:11.718 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 102999 2023-10-09T05:54:11.718 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.718 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:11.718 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.718 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:11.718 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.393899+0000", 2023-10-09T05:54:11.718 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.718 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.718 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.718 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:11.718 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.393899+0000", 2023-10-09T05:54:11.719 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.719 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.719 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.719 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:11.719 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.393899+0000", 2023-10-09T05:54:11.719 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:54:11.719 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.719 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.719 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:11.719 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.393904+0000", 2023-10-09T05:54:11.719 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5010000000000004e-06 2023-10-09T05:54:11.719 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.720 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.720 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:11.720 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.393916+0000", 2023-10-09T05:54:11.720 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2123000000000001e-05 2023-10-09T05:54:11.720 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.720 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.720 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:11.720 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.393922+0000", 2023-10-09T05:54:11.720 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.178e-06 2023-10-09T05:54:11.720 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.720 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.720 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.721 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.394009+0000", 2023-10-09T05:54:11.721 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6229000000000004e-05 2023-10-09T05:54:11.721 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.721 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.721 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:11.721 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.394017+0000", 2023-10-09T05:54:11.721 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.3939999999999996e-06 2023-10-09T05:54:11.721 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.721 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.721 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.721 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.216646+0000", 2023-10-09T05:54:11.722 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.82262949600000002 2023-10-09T05:54:11.722 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.722 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.722 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:11.722 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.216774+0000", 2023-10-09T05:54:11.722 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012741200000000001 2023-10-09T05:54:11.722 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.722 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.722 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:54:11.722 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.216905+0000", 2023-10-09T05:54:11.722 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013101599999999999 2023-10-09T05:54:11.722 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.723 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.723 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:11.723 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.224571+0000", 2023-10-09T05:54:11.723 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0076660130000000002 2023-10-09T05:54:11.723 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.723 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.723 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.723 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.224985+0000", 2023-10-09T05:54:11.723 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00041443300000000002 2023-10-09T05:54:11.723 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.723 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.227566+0000", 2023-10-09T05:54:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0025811100000000002 2023-10-09T05:54:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.227596+0000", 2023-10-09T05:54:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9694000000000001e-05 2023-10-09T05:54:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.227617+0000", 2023-10-09T05:54:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0432999999999999e-05 2023-10-09T05:54:11.725 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.725 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:11.725 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.725 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.725 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.725 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:103012 6.2 6:44c531e7:::benchmark_data_smithi064_284193_object6469:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-10-09T05:54:11.725 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:53:24.688034+0000", 2023-10-09T05:54:11.725 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 46.898884633999998, 2023-10-09T05:54:11.725 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.542482621, 2023-10-09T05:54:11.725 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:11.725 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:11.725 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:11.726 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:11.726 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:11.726 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 103012 2023-10-09T05:54:11.726 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.726 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:11.726 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.726 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:11.726 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.688034+0000", 2023-10-09T05:54:11.726 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.726 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.726 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.688034+0000", 2023-10-09T05:54:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.688032+0000", 2023-10-09T05:54:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T05:54:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.688039+0000", 2023-10-09T05:54:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1690000000000003e-06 2023-10-09T05:54:11.728 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.728 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.728 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:11.728 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.688056+0000", 2023-10-09T05:54:11.728 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6341000000000002e-05 2023-10-09T05:54:11.728 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.728 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.728 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:11.728 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.688063+0000", 2023-10-09T05:54:11.728 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.401e-06 2023-10-09T05:54:11.728 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.728 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.729 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.729 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.688112+0000", 2023-10-09T05:54:11.729 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8986999999999997e-05 2023-10-09T05:54:11.729 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.729 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.729 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:11.729 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.688122+0000", 2023-10-09T05:54:11.729 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.4390000000000006e-06 2023-10-09T05:54:11.729 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.729 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.729 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.729 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.218772+0000", 2023-10-09T05:54:11.730 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.53065072000000002 2023-10-09T05:54:11.730 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.730 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.730 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:11.730 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.218888+0000", 2023-10-09T05:54:11.730 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000115769 2023-10-09T05:54:11.730 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.730 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.730 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:54:11.730 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.219018+0000", 2023-10-09T05:54:11.730 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012995199999999999 2023-10-09T05:54:11.730 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.731 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.731 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:11.731 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.224674+0000", 2023-10-09T05:54:11.731 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0056554830000000002 2023-10-09T05:54:11.731 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.731 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.731 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.731 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.225029+0000", 2023-10-09T05:54:11.731 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00035591900000000001 2023-10-09T05:54:11.731 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.731 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.731 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.230433+0000", 2023-10-09T05:54:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0054031790000000001 2023-10-09T05:54:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.230476+0000", 2023-10-09T05:54:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3677999999999998e-05 2023-10-09T05:54:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.230517+0000", 2023-10-09T05:54:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0413000000000001e-05 2023-10-09T05:54:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.733 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:11.733 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.733 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.733 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.733 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:103013 6.2 6:4a757a30:::benchmark_data_smithi064_284193_object6470:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-10-09T05:54:11.733 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:53:24.805205+0000", 2023-10-09T05:54:11.733 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 46.781713322000002, 2023-10-09T05:54:11.733 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.42539479200000002, 2023-10-09T05:54:11.733 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:11.733 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:11.733 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:11.734 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:11.734 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:11.734 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 103013 2023-10-09T05:54:11.734 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.734 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:11.734 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.734 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:11.734 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.805205+0000", 2023-10-09T05:54:11.734 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.734 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.734 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.805205+0000", 2023-10-09T05:54:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.805204+0000", 2023-10-09T05:54:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.805213+0000", 2023-10-09T05:54:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1649999999999995e-06 2023-10-09T05:54:11.736 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.736 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.736 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:11.736 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.805232+0000", 2023-10-09T05:54:11.736 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8817999999999998e-05 2023-10-09T05:54:11.736 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.736 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.736 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:11.736 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.805241+0000", 2023-10-09T05:54:11.736 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.7709999999999995e-06 2023-10-09T05:54:11.736 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.737 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.737 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.737 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.805303+0000", 2023-10-09T05:54:11.737 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1922999999999996e-05 2023-10-09T05:54:11.737 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.737 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.737 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:11.737 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.805317+0000", 2023-10-09T05:54:11.737 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4214e-05 2023-10-09T05:54:11.737 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.737 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.738 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.738 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.220716+0000", 2023-10-09T05:54:11.738 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.41539918100000001 2023-10-09T05:54:11.738 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.738 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.738 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:11.738 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.220798+0000", 2023-10-09T05:54:11.738 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.2056000000000002e-05 2023-10-09T05:54:11.738 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.738 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.738 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:54:11.739 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.220873+0000", 2023-10-09T05:54:11.739 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4758999999999994e-05 2023-10-09T05:54:11.739 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.739 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.739 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:11.739 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.224678+0000", 2023-10-09T05:54:11.739 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0038057479999999999 2023-10-09T05:54:11.739 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.739 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.739 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.739 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.225058+0000", 2023-10-09T05:54:11.739 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00037911599999999998 2023-10-09T05:54:11.739 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.740 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.740 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.740 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.230552+0000", 2023-10-09T05:54:11.740 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0054947110000000002 2023-10-09T05:54:11.740 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.740 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.740 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:11.740 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.230580+0000", 2023-10-09T05:54:11.740 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7974999999999999e-05 2023-10-09T05:54:11.740 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.740 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.741 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:11.741 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.230600+0000", 2023-10-09T05:54:11.741 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9944000000000001e-05 2023-10-09T05:54:11.741 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.741 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:11.741 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.741 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.741 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.741 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:103026 6.2 6:42739c5a:::benchmark_data_smithi064_284193_object6483:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-10-09T05:54:11.741 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:53:25.001893+0000", 2023-10-09T05:54:11.741 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 46.585025350999999, 2023-10-09T05:54:11.741 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.228789785, 2023-10-09T05:54:11.742 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:11.742 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:11.742 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:11.742 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:11.742 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:11.742 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 103026 2023-10-09T05:54:11.742 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.742 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:11.742 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.742 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:11.742 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.001893+0000", 2023-10-09T05:54:11.742 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.743 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.743 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.743 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:11.743 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.001893+0000", 2023-10-09T05:54:11.743 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:11.743 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.743 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.743 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:11.743 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.001892+0000", 2023-10-09T05:54:11.743 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:11.743 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.744 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.744 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:11.744 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.001898+0000", 2023-10-09T05:54:11.744 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0950000000000001e-06 2023-10-09T05:54:11.744 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.744 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.744 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:11.744 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.001911+0000", 2023-10-09T05:54:11.744 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2656999999999999e-05 2023-10-09T05:54:11.744 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.744 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.744 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:11.745 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.001917+0000", 2023-10-09T05:54:11.745 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6390000000000003e-06 2023-10-09T05:54:11.745 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.745 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.745 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.745 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.001980+0000", 2023-10-09T05:54:11.745 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2922999999999994e-05 2023-10-09T05:54:11.745 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.745 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.745 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:11.745 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.001992+0000", 2023-10-09T05:54:11.746 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1527e-05 2023-10-09T05:54:11.746 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.746 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.746 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:11.746 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.222777+0000", 2023-10-09T05:54:11.746 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.220785224 2023-10-09T05:54:11.746 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.746 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.746 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:11.746 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.222868+0000", 2023-10-09T05:54:11.746 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0660000000000003e-05 2023-10-09T05:54:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:54:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.222943+0000", 2023-10-09T05:54:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5191999999999994e-05 2023-10-09T05:54:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.224920+0000", 2023-10-09T05:54:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001976921 2023-10-09T05:54:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.748 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.225135+0000", 2023-10-09T05:54:11.748 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00021478400000000001 2023-10-09T05:54:11.748 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.748 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.748 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:11.748 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.230627+0000", 2023-10-09T05:54:11.748 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0054921340000000001 2023-10-09T05:54:11.748 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.748 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.748 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:11.748 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.230663+0000", 2023-10-09T05:54:11.749 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5982000000000002e-05 2023-10-09T05:54:11.749 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:11.749 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:11.749 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:11.749 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.230683+0000", 2023-10-09T05:54:11.749 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0420000000000001e-05 2023-10-09T05:54:11.749 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.749 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:11.749 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.749 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:11.749 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:11.750 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:54:11.750 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:11.752 INFO:teuthology.orchestra.run.smithi064.stdout:moved 0 / 327 (0%) 2023-10-09T05:54:11.752 INFO:teuthology.orchestra.run.smithi064.stdout:avg 40.875 2023-10-09T05:54:11.752 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 9.71388 -> 9.71388 (expected baseline 5.98044) 2023-10-09T05:54:11.752 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.2 with 23 -> 23 pgs (0.562691 -> 0.562691 * mean) 2023-10-09T05:54:11.752 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.6 with 51 -> 51 pgs (1.24771 -> 1.24771 * mean) 2023-10-09T05:54:11.753 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:54:11.753 INFO:teuthology.orchestra.run.smithi064.stdout:oload 1000 2023-10-09T05:54:11.753 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 1 2023-10-09T05:54:11.753 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-10-09T05:54:11.753 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0269 2023-10-09T05:54:11.753 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.2685 2023-10-09T05:54:11.754 INFO:teuthology.orchestra.run.smithi064.stderr:no change 2023-10-09T05:54:11.782 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:11.829 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:11.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:11.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:11.932 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:11.960 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:12.005 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:12.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:12.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:12.108 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:12.134 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:12.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:12 smithi064 ceph-mon[220044]: pgmap v408: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 remapped+peering, 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 19272/267741 objects degraded (7.198%); 13653/267741 objects misplaced (5.099%); 9.7 MiB/s, 155 objects/s recovering 2023-10-09T05:54:12.181 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:12.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:12.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:12.285 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:12.320 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:12.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:12 smithi156 ceph-mon[195101]: pgmap v408: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 remapped+peering, 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 19272/267741 objects degraded (7.198%); 13653/267741 objects misplaced (5.099%); 9.7 MiB/s, 155 objects/s recovering 2023-10-09T05:54:12.402 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:12.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:12.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:12.430 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:12 smithi064 ceph-mon[212750]: pgmap v408: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 remapped+peering, 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 19272/267741 objects degraded (7.198%); 13653/267741 objects misplaced (5.099%); 9.7 MiB/s, 155 objects/s recovering 2023-10-09T05:54:12.456 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T05:54:12.456 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-10-09T05:54:12.506 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:12.532 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:12.745 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:12.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:12.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:12.801 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:54:12.849 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:12.891 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:12.932 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:12.950 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:12.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:12.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:13.058 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:13.092 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:13.144 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:54:13.154 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:13.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:13.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:13.171 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:54:13.258 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:13.285 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:13.387 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:13.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:13.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:13.420 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:13 smithi156 ceph-mon[195101]: from='client.34956 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:13.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:13 smithi064 ceph-mon[212750]: from='client.34956 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:13.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:13 smithi064 ceph-mon[220044]: from='client.34956 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:13.492 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:13.528 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:13.585 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:13.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:13.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:13.689 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:13.718 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:13.765 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:13.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:13.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:13.868 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:13.896 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:13.940 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:13.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:13.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:14.045 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:14.072 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:14.120 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:14.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:14.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:14.223 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:14.236 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:14.267 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:14.326 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:14.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:14.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:14.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:14 smithi064 ceph-mon[212750]: pgmap v409: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 remapped+peering, 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 19272/267741 objects degraded (7.198%); 13653/267741 objects misplaced (5.099%); 8.4 MiB/s, 133 objects/s recovering 2023-10-09T05:54:14.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:14.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:14 smithi064 ceph-mon[220044]: pgmap v409: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 remapped+peering, 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 19272/267741 objects degraded (7.198%); 13653/267741 objects misplaced (5.099%); 8.4 MiB/s, 133 objects/s recovering 2023-10-09T05:54:14.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:14.431 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:14.455 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:14 smithi156 ceph-mon[195101]: pgmap v409: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 remapped+peering, 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 19272/267741 objects degraded (7.198%); 13653/267741 objects misplaced (5.099%); 8.4 MiB/s, 133 objects/s recovering 2023-10-09T05:54:14.455 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:14.458 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:14.503 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:14.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:14.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:14.607 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:14.653 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:14.720 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:14.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:14.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:14.825 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:14.866 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:15.035 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:15.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:15.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:15.142 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:15.182 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:15.193 INFO:teuthology.orchestra.run.smithi064.stdout:moved 31 / 327 (9.48012%) 2023-10-09T05:54:15.193 INFO:teuthology.orchestra.run.smithi064.stdout:avg 40.875 2023-10-09T05:54:15.193 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 9.71388 -> 13.5963 (expected baseline 5.98044) 2023-10-09T05:54:15.193 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.2 with 23 -> 21 pgs (0.562691 -> 0.513761 * mean) 2023-10-09T05:54:15.194 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.6 with 51 -> 56 pgs (1.24771 -> 1.37003 * mean) 2023-10-09T05:54:15.194 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:54:15.194 INFO:teuthology.orchestra.run.smithi064.stdout:oload 110 2023-10-09T05:54:15.194 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 3 2023-10-09T05:54:15.194 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-10-09T05:54:15.194 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0269 2023-10-09T05:54:15.194 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.0296 2023-10-09T05:54:15.194 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 weight 0.5832 -> 0.4023 2023-10-09T05:54:15.194 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 weight 0.7087 -> 0.5522 2023-10-09T05:54:15.194 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 weight 1.0000 -> 0.8530 2023-10-09T05:54:15.195 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 weight 1.0000 -> 0.8609 2023-10-09T05:54:15.250 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:15.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:15.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:15.277 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:54:15.277 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T05:54:15.277 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:15.277 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "MOSDPGPush(6.e 180/139 [PushOp(6:7d8a5a50:::benchmark_data_smithi064_284193_object46804:head, version: 112'2848, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:7d8a5a50:::benchmark_data_smithi064_284193_object46804:head@112'2848, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:54:15.277 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:54:15.276477+0000", 2023-10-09T05:54:15.278 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.00053805300000000003, 2023-10-09T05:54:15.278 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00058508500000000005, 2023-10-09T05:54:15.278 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:15.278 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T05:54:15.278 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:15.278 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:15.278 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:15.281 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:15.276477+0000", 2023-10-09T05:54:15.281 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:15.281 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:15.281 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:15.281 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:15.282 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:15.276477+0000", 2023-10-09T05:54:15.282 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:15.282 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:15.282 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:15.282 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:15.282 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:15.276477+0000", 2023-10-09T05:54:15.282 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:54:15.283 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:15.283 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:15.283 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:15.283 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:15.276502+0000", 2023-10-09T05:54:15.283 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.5179999999999999e-05 2023-10-09T05:54:15.284 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:15.284 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:15.284 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:15.284 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:15.276516+0000", 2023-10-09T05:54:15.284 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4246999999999999e-05 2023-10-09T05:54:15.284 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:15.284 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:15.285 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:15.285 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:15.276523+0000", 2023-10-09T05:54:15.285 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3509999999999996e-06 2023-10-09T05:54:15.285 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:15.285 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:15.285 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:15.285 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:15.276578+0000", 2023-10-09T05:54:15.286 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4688000000000002e-05 2023-10-09T05:54:15.286 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:15.286 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:15.286 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:15.286 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:15.276583+0000", 2023-10-09T05:54:15.286 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8500000000000002e-06 2023-10-09T05:54:15.286 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:15.286 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:15.287 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:15.287 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:15.287 INFO:teuthology.orchestra.run.smithi156.stdout: ], 2023-10-09T05:54:15.287 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 1 2023-10-09T05:54:15.287 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:54:15.300 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:15 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'36192','3':'26362','6':'55905','7':'56417'}"}]: dispatch 2023-10-09T05:54:15.301 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:15 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'36192','3':'26362','6':'55905','7':'56417'}"}]: dispatch 2023-10-09T05:54:15.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:15 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'36192','3':'26362','6':'55905','7':'56417'}"}]: dispatch 2023-10-09T05:54:15.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:15 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'36192','3':'26362','6':'55905','7':'56417'}"}]: dispatch 2023-10-09T05:54:15.303 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:15 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'36192','3':'26362','6':'55905','7':'56417'}"}]: dispatch 2023-10-09T05:54:15.304 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:15 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'36192','3':'26362','6':'55905','7':'56417'}"}]: dispatch 2023-10-09T05:54:15.356 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:15.389 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:15.490 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:15.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:15.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:15.594 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:15.624 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:54:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:54:15] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-10-09T05:54:15.627 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:15.934 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:15.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:15.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:16.038 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:16.066 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:16.105 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T05:54:16.106 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-10-09T05:54:16.195 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:16 smithi064 ceph-mon[220044]: pgmap v410: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 remapped+peering, 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 19272/267741 objects degraded (7.198%); 12821/267741 objects misplaced (4.789%); 9.3 MiB/s, 148 objects/s recovering 2023-10-09T05:54:16.196 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:16 smithi064 ceph-mon[220044]: from='client.44885 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:16.196 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:16 smithi064 ceph-mon[220044]: failed to encode map e180 with expected crc 2023-10-09T05:54:16.196 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:16 smithi064 ceph-mon[220044]: failed to encode map e180 with expected crc 2023-10-09T05:54:16.196 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:16 smithi064 ceph-mon[220044]: failed to encode map e180 with expected crc 2023-10-09T05:54:16.196 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:16 smithi064 ceph-mon[220044]: failed to encode map e180 with expected crc 2023-10-09T05:54:16.196 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:16 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'36192','3':'26362','6':'55905','7':'56417'}"}]': finished 2023-10-09T05:54:16.196 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:16 smithi064 ceph-mon[220044]: osdmap e180: 8 total, 8 up, 8 in 2023-10-09T05:54:16.196 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:16 smithi064 ceph-mon[220044]: failed to encode map e180 with expected crc 2023-10-09T05:54:16.196 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:16 smithi064 ceph-mon[220044]: failed to encode map e180 with expected crc 2023-10-09T05:54:16.263 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:16.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:16.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:16.324 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:16 smithi156 ceph-mon[195101]: pgmap v410: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 remapped+peering, 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 19272/267741 objects degraded (7.198%); 12821/267741 objects misplaced (4.789%); 9.3 MiB/s, 148 objects/s recovering 2023-10-09T05:54:16.324 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:16 smithi156 ceph-mon[195101]: from='client.44885 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:16.324 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:16 smithi156 ceph-mon[195101]: failed to encode map e180 with expected crc 2023-10-09T05:54:16.324 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:16 smithi156 ceph-mon[195101]: failed to encode map e180 with expected crc 2023-10-09T05:54:16.325 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:16 smithi156 ceph-mon[195101]: failed to encode map e180 with expected crc 2023-10-09T05:54:16.325 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:16 smithi156 ceph-mon[195101]: failed to encode map e180 with expected crc 2023-10-09T05:54:16.325 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:16 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'36192','3':'26362','6':'55905','7':'56417'}"}]': finished 2023-10-09T05:54:16.325 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:16 smithi156 ceph-mon[195101]: osdmap e180: 8 total, 8 up, 8 in 2023-10-09T05:54:16.325 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:16 smithi156 ceph-mon[195101]: failed to encode map e180 with expected crc 2023-10-09T05:54:16.325 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:16 smithi156 ceph-mon[195101]: failed to encode map e180 with expected crc 2023-10-09T05:54:16.342 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:54:16.367 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:16.438 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:16.466 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:16 smithi064 ceph-mon[212750]: pgmap v410: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 remapped+peering, 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 19272/267741 objects degraded (7.198%); 12821/267741 objects misplaced (4.789%); 9.3 MiB/s, 148 objects/s recovering 2023-10-09T05:54:16.466 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:16 smithi064 ceph-mon[212750]: from='client.44885 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:16.466 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:16 smithi064 ceph-mon[212750]: failed to encode map e180 with expected crc 2023-10-09T05:54:16.466 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:16 smithi064 ceph-mon[212750]: failed to encode map e180 with expected crc 2023-10-09T05:54:16.466 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:16 smithi064 ceph-mon[212750]: failed to encode map e180 with expected crc 2023-10-09T05:54:16.467 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:16 smithi064 ceph-mon[212750]: failed to encode map e180 with expected crc 2023-10-09T05:54:16.467 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:16 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'36192','3':'26362','6':'55905','7':'56417'}"}]': finished 2023-10-09T05:54:16.467 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:16 smithi064 ceph-mon[212750]: osdmap e180: 8 total, 8 up, 8 in 2023-10-09T05:54:16.467 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:16 smithi064 ceph-mon[212750]: failed to encode map e180 with expected crc 2023-10-09T05:54:16.467 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:16 smithi064 ceph-mon[212750]: failed to encode map e180 with expected crc 2023-10-09T05:54:16.545 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:54:16.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:16.595 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:16.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:16.661 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:16.696 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:16.726 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:16.728 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:54:16.776 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:16.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:16.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:16.880 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:16.934 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:17.000 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:17.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:17.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:17.105 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:17.152 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:17.226 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T05:54:17.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[212750]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[212750]: osdmap e181: 8 total, 8 up, 8 in 2023-10-09T05:54:17.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[212750]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[212750]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[212750]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[212750]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[212750]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[212750]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[212750]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[212750]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:54:17.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/255999880' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:17.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:17.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:17.276 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:17 smithi156 ceph-mon[195101]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.778 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:17 smithi156 ceph-mon[195101]: osdmap e181: 8 total, 8 up, 8 in 2023-10-09T05:54:17.778 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:17 smithi156 ceph-mon[195101]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.778 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:17 smithi156 ceph-mon[195101]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.778 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:17 smithi156 ceph-mon[195101]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.778 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:17 smithi156 ceph-mon[195101]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.778 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:17 smithi156 ceph-mon[195101]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.778 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:17 smithi156 ceph-mon[195101]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.778 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:17 smithi156 ceph-mon[195101]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.778 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:17 smithi156 ceph-mon[195101]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.779 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:17 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:54:17.779 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:17 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/255999880' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:17.780 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:17 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:17.783 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:17 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:17.783 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[220044]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.783 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[220044]: osdmap e181: 8 total, 8 up, 8 in 2023-10-09T05:54:17.784 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[220044]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.784 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[220044]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.784 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[220044]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.784 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[220044]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.784 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[220044]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.784 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[220044]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.784 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[220044]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.784 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[220044]: failed to encode map e181 with expected crc 2023-10-09T05:54:17.784 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:54:17.784 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/255999880' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:17.784 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:17.785 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:17 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:18.112 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:18.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:18.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:18.376 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:18.405 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[220044]: pgmap v413: 109 pgs: 1 activating+degraded, 1 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 4 remapped+peering, 97 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 19272/267741 objects degraded (7.198%); 6749/267741 objects misplaced (2.521%); 8.2 MiB/s, 130 objects/s recovering 2023-10-09T05:54:18.406 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[220044]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.406 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[220044]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.406 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[220044]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.406 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:54:18.406 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[220044]: osdmap e182: 8 total, 8 up, 8 in 2023-10-09T05:54:18.406 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[220044]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.406 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[220044]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.406 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[220044]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[220044]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[220044]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:54:18.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[220044]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[220044]: 6.6 continuing backfill to osd.5 from (106'613,168'6508] MIN to 168'6508 2023-10-09T05:54:18.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[220044]: 6.f starting backfill to osd.1 from (0'0,0'0] MAX to 168'6579 2023-10-09T05:54:18.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[220044]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[220044]: osdmap e183: 8 total, 8 up, 8 in 2023-10-09T05:54:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[212750]: pgmap v413: 109 pgs: 1 activating+degraded, 1 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 4 remapped+peering, 97 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 19272/267741 objects degraded (7.198%); 6749/267741 objects misplaced (2.521%); 8.2 MiB/s, 130 objects/s recovering 2023-10-09T05:54:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[212750]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[212750]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[212750]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:54:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[212750]: osdmap e182: 8 total, 8 up, 8 in 2023-10-09T05:54:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[212750]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[212750]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[212750]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[212750]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[212750]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:54:18.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[212750]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[212750]: 6.6 continuing backfill to osd.5 from (106'613,168'6508] MIN to 168'6508 2023-10-09T05:54:18.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[212750]: 6.f starting backfill to osd.1 from (0'0,0'0] MAX to 168'6579 2023-10-09T05:54:18.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[212750]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:18 smithi064 ceph-mon[212750]: osdmap e183: 8 total, 8 up, 8 in 2023-10-09T05:54:18.419 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:18.618 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:18 smithi156 ceph-mon[195101]: pgmap v413: 109 pgs: 1 activating+degraded, 1 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 4 remapped+peering, 97 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 19272/267741 objects degraded (7.198%); 6749/267741 objects misplaced (2.521%); 8.2 MiB/s, 130 objects/s recovering 2023-10-09T05:54:18.618 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:18 smithi156 ceph-mon[195101]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.619 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:18 smithi156 ceph-mon[195101]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.619 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:18 smithi156 ceph-mon[195101]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.619 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:18 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:54:18.619 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:18 smithi156 ceph-mon[195101]: osdmap e182: 8 total, 8 up, 8 in 2023-10-09T05:54:18.619 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:18 smithi156 ceph-mon[195101]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.619 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:18 smithi156 ceph-mon[195101]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.619 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:18 smithi156 ceph-mon[195101]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.619 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:18 smithi156 ceph-mon[195101]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.619 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:18 smithi156 ceph-mon[195101]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:18 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:54:18.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:18 smithi156 ceph-mon[195101]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:18 smithi156 ceph-mon[195101]: 6.6 continuing backfill to osd.5 from (106'613,168'6508] MIN to 168'6508 2023-10-09T05:54:18.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:18 smithi156 ceph-mon[195101]: 6.f starting backfill to osd.1 from (0'0,0'0] MAX to 168'6579 2023-10-09T05:54:18.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:18 smithi156 ceph-mon[195101]: failed to encode map e182 with expected crc 2023-10-09T05:54:18.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:18 smithi156 ceph-mon[195101]: osdmap e183: 8 total, 8 up, 8 in 2023-10-09T05:54:18.820 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:54:18.870 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T05:54:18.870 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T05:54:18.870 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T05:54:18.870 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:54:19.040 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:19.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:19.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:19.145 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:19.183 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:19.281 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:19.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:19.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:19.526 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:19 smithi064 ceph-mon[212750]: failed to encode map e183 with expected crc 2023-10-09T05:54:19.527 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:19 smithi064 ceph-mon[212750]: failed to encode map e183 with expected crc 2023-10-09T05:54:19.527 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:19 smithi064 ceph-mon[212750]: failed to encode map e183 with expected crc 2023-10-09T05:54:19.527 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:19 smithi064 ceph-mon[212750]: failed to encode map e183 with expected crc 2023-10-09T05:54:19.527 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:19 smithi064 ceph-mon[212750]: failed to encode map e183 with expected crc 2023-10-09T05:54:19.527 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:19 smithi064 ceph-mon[212750]: pgmap v416: 109 pgs: 1 activating+degraded, 1 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 4 remapped+peering, 97 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 19272/267741 objects degraded (7.198%); 6749/267741 objects misplaced (2.521%); 0 B/s, 0 objects/s recovering 2023-10-09T05:54:19.528 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:19 smithi064 ceph-mon[212750]: osdmap e184: 8 total, 8 up, 8 in 2023-10-09T05:54:19.528 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:19 smithi064 ceph-mon[220044]: failed to encode map e183 with expected crc 2023-10-09T05:54:19.528 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:19 smithi064 ceph-mon[220044]: failed to encode map e183 with expected crc 2023-10-09T05:54:19.528 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:19 smithi064 ceph-mon[220044]: failed to encode map e183 with expected crc 2023-10-09T05:54:19.528 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:19 smithi064 ceph-mon[220044]: failed to encode map e183 with expected crc 2023-10-09T05:54:19.528 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:19 smithi064 ceph-mon[220044]: failed to encode map e183 with expected crc 2023-10-09T05:54:19.528 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:19 smithi064 ceph-mon[220044]: pgmap v416: 109 pgs: 1 activating+degraded, 1 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 4 remapped+peering, 97 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 19272/267741 objects degraded (7.198%); 6749/267741 objects misplaced (2.521%); 0 B/s, 0 objects/s recovering 2023-10-09T05:54:19.529 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:19 smithi064 ceph-mon[220044]: osdmap e184: 8 total, 8 up, 8 in 2023-10-09T05:54:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:19 smithi156 ceph-mon[195101]: failed to encode map e183 with expected crc 2023-10-09T05:54:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:19 smithi156 ceph-mon[195101]: failed to encode map e183 with expected crc 2023-10-09T05:54:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:19 smithi156 ceph-mon[195101]: failed to encode map e183 with expected crc 2023-10-09T05:54:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:19 smithi156 ceph-mon[195101]: failed to encode map e183 with expected crc 2023-10-09T05:54:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:19 smithi156 ceph-mon[195101]: failed to encode map e183 with expected crc 2023-10-09T05:54:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:19 smithi156 ceph-mon[195101]: pgmap v416: 109 pgs: 1 activating+degraded, 1 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 4 remapped+peering, 97 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 19272/267741 objects degraded (7.198%); 6749/267741 objects misplaced (2.521%); 0 B/s, 0 objects/s recovering 2023-10-09T05:54:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:19 smithi156 ceph-mon[195101]: osdmap e184: 8 total, 8 up, 8 in 2023-10-09T05:54:19.627 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:19.650 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:54:19.664 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:19.719 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:19.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:19.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:19.759 INFO:teuthology.orchestra.run.smithi064.stdout:moved 11 / 327 (3.36391%) 2023-10-09T05:54:19.759 INFO:teuthology.orchestra.run.smithi064.stdout:avg 40.875 2023-10-09T05:54:19.759 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 13.5963 -> 13.4112 (expected baseline 5.98044) 2023-10-09T05:54:19.759 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.3 with 17 -> 17 pgs (0.415902 -> 0.415902 * mean) 2023-10-09T05:54:19.759 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.6 with 56 -> 49 pgs (1.37003 -> 1.19878 * mean) 2023-10-09T05:54:19.760 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:54:19.760 INFO:teuthology.orchestra.run.smithi064.stdout:oload 110 2023-10-09T05:54:19.760 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 0.05 2023-10-09T05:54:19.760 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-10-09T05:54:19.760 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0269 2023-10-09T05:54:19.760 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.0296 2023-10-09T05:54:19.760 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 weight 0.4023 -> 0.3523 2023-10-09T05:54:19.760 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 weight 0.5522 -> 0.5023 2023-10-09T05:54:19.760 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 weight 0.8530 -> 0.8031 2023-10-09T05:54:19.760 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 weight 0.8609 -> 0.8109 2023-10-09T05:54:19.766 INFO:teuthology.orchestra.run.smithi064.stderr:no change 2023-10-09T05:54:19.825 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:19.855 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:19.974 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:19.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:19.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:20.021 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:54:19 smithi064 conmon[198859]: 2023-10-09T05:54:19.757+0000 7fcb872b4700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-10-09T05:54:20.079 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:20.112 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:20.165 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:20.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:20.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:20.269 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:20.271 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:54:20.301 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:20 smithi064 ceph-mon[212750]: failed to encode map e184 with expected crc 2023-10-09T05:54:20.301 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:20 smithi064 ceph-mon[212750]: failed to encode map e184 with expected crc 2023-10-09T05:54:20.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:20 smithi064 ceph-mon[212750]: failed to encode map e184 with expected crc 2023-10-09T05:54:20.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:20 smithi064 ceph-mon[212750]: failed to encode map e184 with expected crc 2023-10-09T05:54:20.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:20 smithi064 ceph-mon[212750]: failed to encode map e184 with expected crc 2023-10-09T05:54:20.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:20 smithi064 ceph-mon[212750]: failed to encode map e184 with expected crc 2023-10-09T05:54:20.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:20 smithi064 ceph-mon[212750]: from='client.34971 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:20.302 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:20 smithi064 ceph-mon[220044]: failed to encode map e184 with expected crc 2023-10-09T05:54:20.302 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:20 smithi064 ceph-mon[220044]: failed to encode map e184 with expected crc 2023-10-09T05:54:20.303 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:20 smithi064 ceph-mon[220044]: failed to encode map e184 with expected crc 2023-10-09T05:54:20.303 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:20 smithi064 ceph-mon[220044]: failed to encode map e184 with expected crc 2023-10-09T05:54:20.303 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:20 smithi064 ceph-mon[220044]: failed to encode map e184 with expected crc 2023-10-09T05:54:20.303 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:20 smithi064 ceph-mon[220044]: failed to encode map e184 with expected crc 2023-10-09T05:54:20.303 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:20 smithi064 ceph-mon[220044]: from='client.34971 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:20.306 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:20.506 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:20 smithi156 ceph-mon[195101]: failed to encode map e184 with expected crc 2023-10-09T05:54:20.506 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:20 smithi156 ceph-mon[195101]: failed to encode map e184 with expected crc 2023-10-09T05:54:20.506 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:20 smithi156 ceph-mon[195101]: failed to encode map e184 with expected crc 2023-10-09T05:54:20.506 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:20 smithi156 ceph-mon[195101]: failed to encode map e184 with expected crc 2023-10-09T05:54:20.507 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:20 smithi156 ceph-mon[195101]: failed to encode map e184 with expected crc 2023-10-09T05:54:20.507 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:20 smithi156 ceph-mon[195101]: failed to encode map e184 with expected crc 2023-10-09T05:54:20.507 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:20 smithi156 ceph-mon[195101]: from='client.34971 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:20.859 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:20.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:20.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:20.963 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T05:54:20.964 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 3.0 2023-10-09T05:54:20.964 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:20.995 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:21.045 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:21.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:21.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:21.058 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:54:21.149 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:21.176 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:21.458 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:21.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:21.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:21.503 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:21.551 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:21 smithi156 ceph-mon[195101]: pgmap v418: 109 pgs: 1 activating+degraded, 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 18459/267741 objects degraded (6.894%); 14019/267741 objects misplaced (5.236%) 2023-10-09T05:54:21.562 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:21.602 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:21 smithi064 ceph-mon[212750]: pgmap v418: 109 pgs: 1 activating+degraded, 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 18459/267741 objects degraded (6.894%); 14019/267741 objects misplaced (5.236%) 2023-10-09T05:54:21.602 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:21 smithi064 ceph-mon[220044]: pgmap v418: 109 pgs: 1 activating+degraded, 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 18459/267741 objects degraded (6.894%); 14019/267741 objects misplaced (5.236%) 2023-10-09T05:54:21.605 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:21.699 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:21.713 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:21.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:21.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:21.823 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:21.866 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:22.032 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:22.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:22.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:22.153 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:22.189 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:22.232 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:54:22.232 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T05:54:22.232 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T05:54:22.232 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T05:54:22.232 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.232 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64028 6.d 6:b2a5011c:::benchmark_data_smithi064_284193_object64027:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:54:22.232 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.138056+0000", 2023-10-09T05:54:22.233 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 169.08940729400001, 2023-10-09T05:54:22.233 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0914284259999998, 2023-10-09T05:54:22.233 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:22.233 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:22.233 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:22.233 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:22.233 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:22.233 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64028 2023-10-09T05:54:22.233 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.233 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:22.233 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.234 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:22.234 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.138056+0000", 2023-10-09T05:54:22.234 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.234 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.234 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.234 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:22.234 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.138056+0000", 2023-10-09T05:54:22.234 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.234 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.234 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.234 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:22.235 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.138055+0000", 2023-10-09T05:54:22.235 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:22.235 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.235 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.235 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:22.235 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.138098+0000", 2023-10-09T05:54:22.235 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3702000000000003e-05 2023-10-09T05:54:22.235 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.235 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.235 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:22.235 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.138112+0000", 2023-10-09T05:54:22.236 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3393999999999999e-05 2023-10-09T05:54:22.236 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.236 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.236 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:22.236 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.138119+0000", 2023-10-09T05:54:22.236 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.905e-06 2023-10-09T05:54:22.236 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.236 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.236 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.236 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.138179+0000", 2023-10-09T05:54:22.237 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0426e-05 2023-10-09T05:54:22.237 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.237 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.237 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.237 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.138187+0000", 2023-10-09T05:54:22.237 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.3680000000000002e-06 2023-10-09T05:54:22.237 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.237 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.237 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.237 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111163+0000", 2023-10-09T05:54:22.237 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.97297587900000004 2023-10-09T05:54:22.238 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.238 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.238 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.238 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111172+0000", 2023-10-09T05:54:22.238 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.4999999999999999e-06 2023-10-09T05:54:22.238 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.238 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.238 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.238 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.204580+0000", 2023-10-09T05:54:22.238 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0934087079999999 2023-10-09T05:54:22.238 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.239 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.239 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:22.239 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.204699+0000", 2023-10-09T05:54:22.239 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000118097 2023-10-09T05:54:22.239 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.239 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.239 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:54:22.239 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.204810+0000", 2023-10-09T05:54:22.239 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011183099999999999 2023-10-09T05:54:22.239 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.239 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.240 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:22.240 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.211455+0000", 2023-10-09T05:54:22.240 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0066445150000000001 2023-10-09T05:54:22.240 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.240 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.240 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.240 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.227513+0000", 2023-10-09T05:54:22.240 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.016057795999999999 2023-10-09T05:54:22.240 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.240 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.240 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.240 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.227594+0000", 2023-10-09T05:54:22.241 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.174e-05 2023-10-09T05:54:22.241 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.241 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.241 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:22.241 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.227636+0000", 2023-10-09T05:54:22.241 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2018999999999999e-05 2023-10-09T05:54:22.241 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.241 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.241 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:22.241 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.229485+0000", 2023-10-09T05:54:22.241 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0018480969999999999 2023-10-09T05:54:22.241 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.242 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:22.242 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.242 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.242 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.242 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64042 6.d 6:b4a627c6:::benchmark_data_smithi064_284193_object64041:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:54:22.242 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.148881+0000", 2023-10-09T05:54:22.242 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 169.078582703, 2023-10-09T05:54:22.242 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.083266171, 2023-10-09T05:54:22.242 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:22.242 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:22.242 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:22.243 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:22.243 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:22.243 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64042 2023-10-09T05:54:22.243 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.243 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:22.243 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.243 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:22.243 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.148881+0000", 2023-10-09T05:54:22.243 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.243 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.243 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.243 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:22.244 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.148881+0000", 2023-10-09T05:54:22.244 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.244 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.244 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.244 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:22.244 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.148879+0000", 2023-10-09T05:54:22.244 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:22.244 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.244 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.244 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:22.244 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.148915+0000", 2023-10-09T05:54:22.244 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5583999999999997e-05 2023-10-09T05:54:22.245 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.245 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.245 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:22.245 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.148925+0000", 2023-10-09T05:54:22.245 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.9150000000000008e-06 2023-10-09T05:54:22.245 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.245 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.245 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:22.245 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.148930+0000", 2023-10-09T05:54:22.245 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2100000000000001e-06 2023-10-09T05:54:22.245 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.246 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.246 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.246 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.149088+0000", 2023-10-09T05:54:22.246 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015745000000000001 2023-10-09T05:54:22.246 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.246 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.246 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.246 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.149096+0000", 2023-10-09T05:54:22.246 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.5560000000000008e-06 2023-10-09T05:54:22.246 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.246 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.246 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.247 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111207+0000", 2023-10-09T05:54:22.247 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96211053099999999 2023-10-09T05:54:22.247 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.247 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.249 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.250 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111214+0000", 2023-10-09T05:54:22.250 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3189999999999997e-06 2023-10-09T05:54:22.250 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.250 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.250 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.250 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.206599+0000", 2023-10-09T05:54:22.250 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.095385359 2023-10-09T05:54:22.250 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.251 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.251 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:22.251 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.206696+0000", 2023-10-09T05:54:22.251 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.6280000000000007e-05 2023-10-09T05:54:22.251 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.251 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.251 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:54:22.251 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.206803+0000", 2023-10-09T05:54:22.251 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000107219 2023-10-09T05:54:22.251 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.252 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.252 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:22.252 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.211465+0000", 2023-10-09T05:54:22.252 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0046617569999999999 2023-10-09T05:54:22.252 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.252 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.252 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.252 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.227565+0000", 2023-10-09T05:54:22.252 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.016100023000000001 2023-10-09T05:54:22.252 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.252 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.253 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.253 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.229614+0000", 2023-10-09T05:54:22.253 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0020497720000000001 2023-10-09T05:54:22.253 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.253 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.253 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:22.253 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.229681+0000", 2023-10-09T05:54:22.253 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6727e-05 2023-10-09T05:54:22.253 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.253 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.253 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:22.253 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.232147+0000", 2023-10-09T05:54:22.254 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0024657419999999999 2023-10-09T05:54:22.254 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.254 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:22.254 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.254 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.254 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.254 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64051 6.d 6:b4ee6d36:::benchmark_data_smithi064_284193_object64050:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:54:22.254 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.160656+0000", 2023-10-09T05:54:22.254 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 169.06680697900001, 2023-10-09T05:54:22.254 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0740513780000001, 2023-10-09T05:54:22.254 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:22.255 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:22.255 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:22.255 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:22.255 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:22.255 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64051 2023-10-09T05:54:22.255 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.255 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:22.255 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.255 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:22.255 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.160656+0000", 2023-10-09T05:54:22.255 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.255 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.256 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.256 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:22.256 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.160656+0000", 2023-10-09T05:54:22.256 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.256 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.256 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.256 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:22.256 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.160656+0000", 2023-10-09T05:54:22.256 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:54:22.256 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.256 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.257 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:22.257 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.160686+0000", 2023-10-09T05:54:22.257 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0577000000000002e-05 2023-10-09T05:54:22.257 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.257 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.257 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:22.257 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.160694+0000", 2023-10-09T05:54:22.257 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4800000000000004e-06 2023-10-09T05:54:22.257 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.257 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.257 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:22.258 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.160698+0000", 2023-10-09T05:54:22.258 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2119999999999997e-06 2023-10-09T05:54:22.258 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.258 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.258 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.258 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.160752+0000", 2023-10-09T05:54:22.258 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4283999999999997e-05 2023-10-09T05:54:22.258 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.258 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.258 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.258 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.160762+0000", 2023-10-09T05:54:22.258 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.4539999999999996e-06 2023-10-09T05:54:22.259 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.259 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.259 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.259 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111247+0000", 2023-10-09T05:54:22.259 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.95048511800000002 2023-10-09T05:54:22.259 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.259 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.259 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.259 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111254+0000", 2023-10-09T05:54:22.259 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4229999999999999e-06 2023-10-09T05:54:22.260 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.260 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.260 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.260 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.207847+0000", 2023-10-09T05:54:22.260 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.096592206 2023-10-09T05:54:22.260 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.260 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.260 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:22.260 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.207986+0000", 2023-10-09T05:54:22.260 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000138961 2023-10-09T05:54:22.260 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.261 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.261 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:54:22.261 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.208089+0000", 2023-10-09T05:54:22.261 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010323900000000001 2023-10-09T05:54:22.261 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.261 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.261 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:22.261 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.213812+0000", 2023-10-09T05:54:22.261 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0057226880000000001 2023-10-09T05:54:22.261 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.261 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.261 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.262 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.229558+0000", 2023-10-09T05:54:22.262 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015746514999999999 2023-10-09T05:54:22.262 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.262 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.262 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.262 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.232222+0000", 2023-10-09T05:54:22.262 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0026640000000000001 2023-10-09T05:54:22.262 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.262 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.262 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:22.263 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.232292+0000", 2023-10-09T05:54:22.263 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9648999999999997e-05 2023-10-09T05:54:22.263 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.263 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.263 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:22.263 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.234708+0000", 2023-10-09T05:54:22.263 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0024161170000000002 2023-10-09T05:54:22.263 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.263 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:22.264 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.264 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.264 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.264 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64056 6.d 6:bb5661cc:::benchmark_data_smithi064_284193_object64055:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:54:22.264 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.165123+0000", 2023-10-09T05:54:22.264 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 169.06234079999999, 2023-10-09T05:54:22.264 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0726746980000001, 2023-10-09T05:54:22.264 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:22.264 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:22.264 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:22.264 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:22.264 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:22.265 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64056 2023-10-09T05:54:22.265 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.265 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:22.265 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.265 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:22.265 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.165123+0000", 2023-10-09T05:54:22.265 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.265 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.265 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.265 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:22.265 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.165123+0000", 2023-10-09T05:54:22.265 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.266 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.266 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.266 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:22.266 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.165122+0000", 2023-10-09T05:54:22.266 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:54:22.266 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.266 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.266 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:22.266 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.165150+0000", 2023-10-09T05:54:22.266 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7784999999999999e-05 2023-10-09T05:54:22.266 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.267 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.267 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:22.267 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.165158+0000", 2023-10-09T05:54:22.267 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1100000000000003e-06 2023-10-09T05:54:22.267 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.267 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.267 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:22.267 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.165163+0000", 2023-10-09T05:54:22.267 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5889999999999996e-06 2023-10-09T05:54:22.267 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.267 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.267 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.268 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.165204+0000", 2023-10-09T05:54:22.268 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0961000000000003e-05 2023-10-09T05:54:22.268 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.268 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.268 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.268 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.165211+0000", 2023-10-09T05:54:22.268 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1849999999999998e-06 2023-10-09T05:54:22.268 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.268 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.268 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.268 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111284+0000", 2023-10-09T05:54:22.269 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.94607339800000001 2023-10-09T05:54:22.269 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.269 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.269 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.269 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111289+0000", 2023-10-09T05:54:22.269 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1630000000000004e-06 2023-10-09T05:54:22.269 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.269 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.269 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.269 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.209079+0000", 2023-10-09T05:54:22.269 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.097789296 2023-10-09T05:54:22.269 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.270 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.270 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:22.270 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.209170+0000", 2023-10-09T05:54:22.270 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1420000000000002e-05 2023-10-09T05:54:22.270 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.270 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.270 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:54:22.270 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.209274+0000", 2023-10-09T05:54:22.270 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000103553 2023-10-09T05:54:22.270 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.270 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.271 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:22.271 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.213841+0000", 2023-10-09T05:54:22.271 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0045676650000000003 2023-10-09T05:54:22.271 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.271 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.271 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.271 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.234777+0000", 2023-10-09T05:54:22.271 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.020935958000000001 2023-10-09T05:54:22.271 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.271 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.271 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.271 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.234979+0000", 2023-10-09T05:54:22.272 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00020143200000000001 2023-10-09T05:54:22.272 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.272 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.272 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:22.272 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.235052+0000", 2023-10-09T05:54:22.272 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3369999999999997e-05 2023-10-09T05:54:22.272 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.272 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.272 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:22.272 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.237797+0000", 2023-10-09T05:54:22.273 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00274537 2023-10-09T05:54:22.273 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.273 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:22.273 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.273 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.273 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.273 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64062 6.d 6:bfc7fb96:::benchmark_data_smithi064_284193_object64061:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:54:22.273 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.176528+0000", 2023-10-09T05:54:22.273 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 169.05093507000001, 2023-10-09T05:54:22.273 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.063752225, 2023-10-09T05:54:22.273 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:22.273 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:22.274 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:22.274 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:22.274 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:22.274 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64062 2023-10-09T05:54:22.274 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.274 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:22.274 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.274 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:22.274 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.176528+0000", 2023-10-09T05:54:22.274 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.274 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.275 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.275 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:22.275 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.176528+0000", 2023-10-09T05:54:22.275 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.275 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.275 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.275 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:22.275 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.176527+0000", 2023-10-09T05:54:22.275 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:54:22.275 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.275 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.276 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:22.276 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.176580+0000", 2023-10-09T05:54:22.276 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2935000000000001e-05 2023-10-09T05:54:22.276 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.276 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.276 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:22.276 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.176592+0000", 2023-10-09T05:54:22.276 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1481000000000001e-05 2023-10-09T05:54:22.276 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.276 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.276 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:22.276 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.176598+0000", 2023-10-09T05:54:22.277 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9540000000000001e-06 2023-10-09T05:54:22.277 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.277 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.277 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.277 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.176646+0000", 2023-10-09T05:54:22.277 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7753999999999997e-05 2023-10-09T05:54:22.277 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.277 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.277 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.277 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.176653+0000", 2023-10-09T05:54:22.277 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8259999999999993e-06 2023-10-09T05:54:22.277 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.278 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.278 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.278 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111323+0000", 2023-10-09T05:54:22.278 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.93466992699999996 2023-10-09T05:54:22.278 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.278 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.278 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.278 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111329+0000", 2023-10-09T05:54:22.278 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7699999999999998e-06 2023-10-09T05:54:22.278 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.278 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.279 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.279 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.210262+0000", 2023-10-09T05:54:22.279 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.098933036 2023-10-09T05:54:22.279 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.279 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.279 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:22.279 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.210386+0000", 2023-10-09T05:54:22.279 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012342000000000001 2023-10-09T05:54:22.279 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.279 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.279 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:54:22.279 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.210493+0000", 2023-10-09T05:54:22.280 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000106988 2023-10-09T05:54:22.280 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.280 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.280 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:22.280 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.216295+0000", 2023-10-09T05:54:22.280 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0058020320000000004 2023-10-09T05:54:22.280 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.280 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.280 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.280 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.234811+0000", 2023-10-09T05:54:22.280 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018516469000000001 2023-10-09T05:54:22.280 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.281 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.281 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.281 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.237872+0000", 2023-10-09T05:54:22.281 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.003061032 2023-10-09T05:54:22.281 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.281 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.281 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:22.281 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.237919+0000", 2023-10-09T05:54:22.281 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7448999999999999e-05 2023-10-09T05:54:22.281 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.281 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.281 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:22.282 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.240281+0000", 2023-10-09T05:54:22.282 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0023610879999999999 2023-10-09T05:54:22.282 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.282 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:22.282 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.282 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.282 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.282 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64070 6.d 6:bb0f4d69:::benchmark_data_smithi064_284193_object64069:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:54:22.282 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.203536+0000", 2023-10-09T05:54:22.282 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 169.023927732, 2023-10-09T05:54:22.282 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.039812205, 2023-10-09T05:54:22.283 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:22.283 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:22.283 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:22.283 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:22.283 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:22.283 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64070 2023-10-09T05:54:22.283 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.283 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:22.283 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.283 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:22.283 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.203536+0000", 2023-10-09T05:54:22.284 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.284 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.284 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.284 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:22.284 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.203536+0000", 2023-10-09T05:54:22.284 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.284 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.284 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.284 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:22.284 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.203534+0000", 2023-10-09T05:54:22.284 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:22.285 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.285 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.285 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:22.285 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.203598+0000", 2023-10-09T05:54:22.285 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4401999999999998e-05 2023-10-09T05:54:22.285 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.285 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.285 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:22.285 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.203617+0000", 2023-10-09T05:54:22.285 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8428999999999999e-05 2023-10-09T05:54:22.285 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.286 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.286 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:22.286 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.203624+0000", 2023-10-09T05:54:22.286 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5000000000000002e-06 2023-10-09T05:54:22.286 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.286 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.286 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.286 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.203672+0000", 2023-10-09T05:54:22.286 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7793000000000001e-05 2023-10-09T05:54:22.286 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.286 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.287 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.287 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.203679+0000", 2023-10-09T05:54:22.287 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7750000000000004e-06 2023-10-09T05:54:22.287 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.287 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.287 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.287 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111355+0000", 2023-10-09T05:54:22.287 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.90767584199999995 2023-10-09T05:54:22.287 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.287 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.287 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.288 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111360+0000", 2023-10-09T05:54:22.288 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1970000000000004e-06 2023-10-09T05:54:22.288 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.288 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.288 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.288 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.211500+0000", 2023-10-09T05:54:22.288 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1001402220000001 2023-10-09T05:54:22.288 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.288 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.288 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:22.288 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.211590+0000", 2023-10-09T05:54:22.289 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9363000000000005e-05 2023-10-09T05:54:22.289 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.289 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.289 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:54:22.289 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.211698+0000", 2023-10-09T05:54:22.289 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000108789 2023-10-09T05:54:22.289 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.289 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.289 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:22.289 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.216302+0000", 2023-10-09T05:54:22.289 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0046034869999999999 2023-10-09T05:54:22.290 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.290 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.290 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.290 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.240331+0000", 2023-10-09T05:54:22.290 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.024028898 2023-10-09T05:54:22.290 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.290 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.290 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.290 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.240426+0000", 2023-10-09T05:54:22.290 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.5624000000000005e-05 2023-10-09T05:54:22.290 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.291 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.291 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:22.291 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.240483+0000", 2023-10-09T05:54:22.291 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6329999999999998e-05 2023-10-09T05:54:22.291 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.291 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.291 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:22.291 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.243348+0000", 2023-10-09T05:54:22.291 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002865214 2023-10-09T05:54:22.291 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.291 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:22.292 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.292 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.292 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.292 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64074 6.d 6:b61b5635:::benchmark_data_smithi064_284193_object64073:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-10-09T05:54:22.292 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:34.202267+0000", 2023-10-09T05:54:22.292 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 168.02519610499999, 2023-10-09T05:54:22.292 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.044023162, 2023-10-09T05:54:22.292 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:22.292 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:22.292 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:22.293 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:22.293 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:22.293 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64074 2023-10-09T05:54:22.293 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.293 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:22.293 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.293 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:22.293 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.202267+0000", 2023-10-09T05:54:22.293 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.293 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.293 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.294 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:22.294 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.202267+0000", 2023-10-09T05:54:22.294 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.294 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.294 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.294 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:22.294 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.202266+0000", 2023-10-09T05:54:22.294 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:22.294 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.294 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.294 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:22.295 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.203900+0000", 2023-10-09T05:54:22.295 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0016342100000000001 2023-10-09T05:54:22.295 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.295 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.295 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:22.295 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.203913+0000", 2023-10-09T05:54:22.295 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2914999999999999e-05 2023-10-09T05:54:22.295 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.295 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.295 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:22.295 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.203920+0000", 2023-10-09T05:54:22.296 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.443e-06 2023-10-09T05:54:22.296 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.296 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.296 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.296 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220752+0000", 2023-10-09T05:54:22.296 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.016832783 2023-10-09T05:54:22.296 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.296 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.296 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.296 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220761+0000", 2023-10-09T05:54:22.296 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6410000000000008e-06 2023-10-09T05:54:22.297 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.297 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.297 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.297 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.212668+0000", 2023-10-09T05:54:22.297 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99190736599999996 2023-10-09T05:54:22.297 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.297 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.297 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:22.297 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.212794+0000", 2023-10-09T05:54:22.297 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012561199999999999 2023-10-09T05:54:22.297 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.297 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.298 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:54:22.298 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.212886+0000", 2023-10-09T05:54:22.298 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2281000000000006e-05 2023-10-09T05:54:22.298 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.298 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.298 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:22.298 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.220148+0000", 2023-10-09T05:54:22.298 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0072622019999999997 2023-10-09T05:54:22.298 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.298 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.298 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.298 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.240385+0000", 2023-10-09T05:54:22.299 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.020236849000000001 2023-10-09T05:54:22.299 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.299 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.299 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.299 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.243402+0000", 2023-10-09T05:54:22.299 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0030166049999999999 2023-10-09T05:54:22.299 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.299 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.299 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:22.299 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.243449+0000", 2023-10-09T05:54:22.299 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7592000000000001e-05 2023-10-09T05:54:22.299 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.300 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.300 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:22.300 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.246290+0000", 2023-10-09T05:54:22.300 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0028410290000000001 2023-10-09T05:54:22.300 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.300 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:22.300 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.300 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.300 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.300 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64081 6.d 6:be2ec322:::benchmark_data_smithi064_284193_object64080:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-10-09T05:54:22.300 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:34.204234+0000", 2023-10-09T05:54:22.301 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 168.02322928300001, 2023-10-09T05:54:22.301 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0455772409999999, 2023-10-09T05:54:22.301 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:22.301 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:22.301 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:22.301 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:22.301 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:22.301 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64081 2023-10-09T05:54:22.301 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.301 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:22.301 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.301 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:22.302 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204234+0000", 2023-10-09T05:54:22.302 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.302 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.302 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.302 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:22.302 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204234+0000", 2023-10-09T05:54:22.302 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.302 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.302 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.302 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:22.302 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204233+0000", 2023-10-09T05:54:22.303 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:54:22.303 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.303 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.303 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:22.303 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204263+0000", 2023-10-09T05:54:22.303 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9601e-05 2023-10-09T05:54:22.303 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.303 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.303 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:22.303 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204271+0000", 2023-10-09T05:54:22.303 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8860000000000004e-06 2023-10-09T05:54:22.304 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.304 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.304 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:22.304 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204277+0000", 2023-10-09T05:54:22.304 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0140000000000004e-06 2023-10-09T05:54:22.304 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.304 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.304 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.304 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220790+0000", 2023-10-09T05:54:22.304 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.016513109000000002 2023-10-09T05:54:22.304 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.304 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.305 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.305 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220796+0000", 2023-10-09T05:54:22.305 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0090000000000004e-06 2023-10-09T05:54:22.305 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.305 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.305 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.305 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.213877+0000", 2023-10-09T05:54:22.305 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99308116199999996 2023-10-09T05:54:22.305 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.305 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.305 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:22.305 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.213975+0000", 2023-10-09T05:54:22.306 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7417999999999995e-05 2023-10-09T05:54:22.306 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.306 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.306 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:54:22.306 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.214074+0000", 2023-10-09T05:54:22.306 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.9256e-05 2023-10-09T05:54:22.306 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.306 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.306 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:22.306 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.220163+0000", 2023-10-09T05:54:22.306 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0060896270000000002 2023-10-09T05:54:22.307 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.307 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.307 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.307 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.246362+0000", 2023-10-09T05:54:22.307 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.026198430000000002 2023-10-09T05:54:22.307 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.307 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.307 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.307 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.246630+0000", 2023-10-09T05:54:22.307 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00026832000000000001 2023-10-09T05:54:22.307 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.307 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.308 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:22.308 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.246689+0000", 2023-10-09T05:54:22.308 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8675000000000003e-05 2023-10-09T05:54:22.308 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.308 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.308 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:22.308 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.249811+0000", 2023-10-09T05:54:22.308 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.003122465 2023-10-09T05:54:22.308 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.308 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:22.308 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.309 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.309 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.309 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64089 6.d 6:b58e5e7c:::benchmark_data_smithi064_284193_object64088:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-10-09T05:54:22.309 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:34.213048+0000", 2023-10-09T05:54:22.309 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 168.01441564000001, 2023-10-09T05:54:22.309 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0395069539999999, 2023-10-09T05:54:22.309 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:22.309 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:22.309 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:22.309 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:22.309 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:22.309 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64089 2023-10-09T05:54:22.310 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.310 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:22.310 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.310 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:22.310 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.213048+0000", 2023-10-09T05:54:22.310 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.310 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.310 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.310 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:22.310 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.213048+0000", 2023-10-09T05:54:22.310 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.311 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.311 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.311 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:22.311 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.213047+0000", 2023-10-09T05:54:22.311 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:54:22.311 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.311 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.311 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:22.311 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.213081+0000", 2023-10-09T05:54:22.311 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.4516e-05 2023-10-09T05:54:22.311 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.311 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.312 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:22.312 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.213094+0000", 2023-10-09T05:54:22.312 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2349e-05 2023-10-09T05:54:22.312 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.312 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.312 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:22.312 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.213099+0000", 2023-10-09T05:54:22.312 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7429999999999999e-06 2023-10-09T05:54:22.312 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.312 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.312 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.312 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220817+0000", 2023-10-09T05:54:22.313 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0077176060000000001 2023-10-09T05:54:22.313 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.313 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.313 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.313 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220823+0000", 2023-10-09T05:54:22.313 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6699999999999999e-06 2023-10-09T05:54:22.313 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.313 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.313 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.313 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.215043+0000", 2023-10-09T05:54:22.313 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99422076999999998 2023-10-09T05:54:22.314 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.314 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.314 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:22.314 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.215180+0000", 2023-10-09T05:54:22.314 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000136131 2023-10-09T05:54:22.314 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.314 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.314 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:54:22.314 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.215271+0000", 2023-10-09T05:54:22.314 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1936999999999997e-05 2023-10-09T05:54:22.314 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.315 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.315 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:22.315 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.222469+0000", 2023-10-09T05:54:22.315 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0071980409999999996 2023-10-09T05:54:22.315 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.315 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.315 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.315 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.246413+0000", 2023-10-09T05:54:22.315 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.023943939000000001 2023-10-09T05:54:22.315 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.315 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.316 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.316 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.249860+0000", 2023-10-09T05:54:22.316 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0034468910000000001 2023-10-09T05:54:22.316 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.316 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.316 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:22.316 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.249897+0000", 2023-10-09T05:54:22.316 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7141999999999998e-05 2023-10-09T05:54:22.316 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.316 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.316 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:22.316 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.252555+0000", 2023-10-09T05:54:22.317 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002657263 2023-10-09T05:54:22.317 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.317 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:22.317 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.317 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.317 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.317 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64093 6.d 6:b073101d:::benchmark_data_smithi064_284193_object64092:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-10-09T05:54:22.317 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:34.220415+0000", 2023-10-09T05:54:22.317 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 168.00704807100001, 2023-10-09T05:54:22.317 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.035115665, 2023-10-09T05:54:22.317 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:22.317 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:22.318 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:22.318 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:22.318 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:22.318 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64093 2023-10-09T05:54:22.318 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.318 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:22.318 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.318 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:22.318 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220415+0000", 2023-10-09T05:54:22.318 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.319 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.319 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:22.319 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220415+0000", 2023-10-09T05:54:22.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.319 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.319 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:22.319 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220409+0000", 2023-10-09T05:54:22.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999938 2023-10-09T05:54:22.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.319 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.319 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:22.320 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220472+0000", 2023-10-09T05:54:22.320 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3229000000000001e-05 2023-10-09T05:54:22.320 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.320 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.320 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:22.320 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220484+0000", 2023-10-09T05:54:22.320 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1467999999999999e-05 2023-10-09T05:54:22.320 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.320 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.320 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:22.320 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220491+0000", 2023-10-09T05:54:22.320 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9809999999999997e-06 2023-10-09T05:54:22.321 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.321 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.321 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.321 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220841+0000", 2023-10-09T05:54:22.321 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00035016700000000002 2023-10-09T05:54:22.321 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.321 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.321 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.321 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220846+0000", 2023-10-09T05:54:22.321 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2079999999999999e-06 2023-10-09T05:54:22.321 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.322 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.322 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.322 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.216340+0000", 2023-10-09T05:54:22.322 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99549343000000001 2023-10-09T05:54:22.322 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.322 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.322 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:22.322 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.216439+0000", 2023-10-09T05:54:22.322 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.8918000000000004e-05 2023-10-09T05:54:22.322 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.322 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.323 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:54:22.323 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.216552+0000", 2023-10-09T05:54:22.323 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000113495 2023-10-09T05:54:22.323 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.323 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.323 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:22.323 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.222481+0000", 2023-10-09T05:54:22.323 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0059289099999999999 2023-10-09T05:54:22.323 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.323 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.323 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.323 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.246451+0000", 2023-10-09T05:54:22.324 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.02396968 2023-10-09T05:54:22.324 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.324 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.324 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.324 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.252814+0000", 2023-10-09T05:54:22.324 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0063631360000000001 2023-10-09T05:54:22.324 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.324 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.324 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:22.324 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.252851+0000", 2023-10-09T05:54:22.324 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7106e-05 2023-10-09T05:54:22.325 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.325 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.325 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:22.325 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.255531+0000", 2023-10-09T05:54:22.325 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002680095 2023-10-09T05:54:22.325 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.325 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:22.325 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.325 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.325 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.325 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:82074 6.a 6:58f4f3d0:::benchmark_data_smithi064_284193_object82073:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T05:54:22.325 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:56.097757+0000", 2023-10-09T05:54:22.326 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 146.12970615699999, 2023-10-09T05:54:22.326 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.158105742, 2023-10-09T05:54:22.326 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:22.326 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T05:54:22.326 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:22.326 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:22.326 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:22.326 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 82074 2023-10-09T05:54:22.326 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.326 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:22.326 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.327 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:22.327 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.097757+0000", 2023-10-09T05:54:22.327 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.327 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.327 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.327 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:22.327 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.097757+0000", 2023-10-09T05:54:22.327 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.327 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.327 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.327 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:22.327 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.097756+0000", 2023-10-09T05:54:22.328 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:22.328 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.328 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.328 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:22.328 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.097799+0000", 2023-10-09T05:54:22.328 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3735000000000001e-05 2023-10-09T05:54:22.328 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.328 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.328 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:22.328 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.097817+0000", 2023-10-09T05:54:22.328 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7289e-05 2023-10-09T05:54:22.329 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.329 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.329 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:22.329 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.097824+0000", 2023-10-09T05:54:22.329 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1319999999999996e-06 2023-10-09T05:54:22.329 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.329 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.329 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.329 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.100302+0000", 2023-10-09T05:54:22.329 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0024780340000000001 2023-10-09T05:54:22.329 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.329 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.330 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:22.330 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.100452+0000", 2023-10-09T05:54:22.330 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000149863 2023-10-09T05:54:22.330 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.330 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.330 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 6", 2023-10-09T05:54:22.330 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.100616+0000", 2023-10-09T05:54:22.330 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000164673 2023-10-09T05:54:22.330 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.330 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.330 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:22.330 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.106430+0000", 2023-10-09T05:54:22.331 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0058139039999999999 2023-10-09T05:54:22.331 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.331 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.331 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.331 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.144922+0000", 2023-10-09T05:54:22.331 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.038491535 2023-10-09T05:54:22.331 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.331 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.331 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.331 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.144937+0000", 2023-10-09T05:54:22.331 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5568000000000001e-05 2023-10-09T05:54:22.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.332 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.332 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.332 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088545+0000", 2023-10-09T05:54:22.332 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.94360759900000002 2023-10-09T05:54:22.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.332 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.332 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.332 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088550+0000", 2023-10-09T05:54:22.332 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2449999999999998e-06 2023-10-09T05:54:22.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.332 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.333 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.333 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.255794+0000", 2023-10-09T05:54:22.333 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1672438190000001 2023-10-09T05:54:22.333 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.333 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.333 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:22.333 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.255863+0000", 2023-10-09T05:54:22.333 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8866000000000001e-05 2023-10-09T05:54:22.333 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.333 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:22.333 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.333 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.334 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.334 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:82079 6.a 6:59a1f462:::benchmark_data_smithi064_284193_object82078:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T05:54:22.334 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:56.101890+0000", 2023-10-09T05:54:22.334 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 146.12557355800001, 2023-10-09T05:54:22.334 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.154915967, 2023-10-09T05:54:22.334 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:22.334 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T05:54:22.334 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:22.334 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:22.334 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:22.335 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 82079 2023-10-09T05:54:22.335 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.335 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:22.335 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.335 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:22.335 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.101890+0000", 2023-10-09T05:54:22.335 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.335 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.335 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.335 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:22.335 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.101890+0000", 2023-10-09T05:54:22.336 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.336 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.336 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.336 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:22.336 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.101889+0000", 2023-10-09T05:54:22.336 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:54:22.336 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.336 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.336 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:22.336 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.101925+0000", 2023-10-09T05:54:22.336 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5763999999999998e-05 2023-10-09T05:54:22.336 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.337 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.337 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:22.337 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.101932+0000", 2023-10-09T05:54:22.337 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7290000000000003e-06 2023-10-09T05:54:22.337 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.337 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.337 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:22.337 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.101937+0000", 2023-10-09T05:54:22.337 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.972e-06 2023-10-09T05:54:22.337 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.337 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.338 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.338 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.103569+0000", 2023-10-09T05:54:22.338 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0016325479999999999 2023-10-09T05:54:22.338 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.338 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.338 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:22.338 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.103679+0000", 2023-10-09T05:54:22.338 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000109225 2023-10-09T05:54:22.338 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.338 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.338 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 6", 2023-10-09T05:54:22.339 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.103810+0000", 2023-10-09T05:54:22.339 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013134799999999999 2023-10-09T05:54:22.339 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.339 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.339 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:22.339 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.113355+0000", 2023-10-09T05:54:22.339 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.009545118 2023-10-09T05:54:22.339 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.339 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.339 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.339 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.147616+0000", 2023-10-09T05:54:22.339 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.034261319999999998 2023-10-09T05:54:22.340 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.340 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.340 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.340 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.147626+0000", 2023-10-09T05:54:22.340 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7899999999999994e-06 2023-10-09T05:54:22.340 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.340 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.340 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.340 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088568+0000", 2023-10-09T05:54:22.340 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.94094197000000002 2023-10-09T05:54:22.340 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.340 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.341 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.341 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088574+0000", 2023-10-09T05:54:22.341 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3439999999999997e-06 2023-10-09T05:54:22.341 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.341 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.341 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.341 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.256728+0000", 2023-10-09T05:54:22.341 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.168153088 2023-10-09T05:54:22.341 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.341 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.341 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:22.342 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.256806+0000", 2023-10-09T05:54:22.342 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8255000000000002e-05 2023-10-09T05:54:22.342 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.342 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:22.342 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.342 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.342 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.342 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:82095 6.d 6:b2648f7f:::benchmark_data_smithi064_284193_object82094:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:54:22.342 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:56.144382+0000", 2023-10-09T05:54:22.342 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 146.08308124600001, 2023-10-09T05:54:22.342 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1392636949999999, 2023-10-09T05:54:22.343 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:22.343 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:22.343 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:22.343 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:22.343 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:22.343 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 82095 2023-10-09T05:54:22.343 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.343 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:22.343 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.343 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:22.343 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.144382+0000", 2023-10-09T05:54:22.343 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.344 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.344 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.344 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:22.344 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.144382+0000", 2023-10-09T05:54:22.344 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.344 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.344 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.344 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:22.344 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.144382+0000", 2023-10-09T05:54:22.344 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:54:22.344 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.344 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.345 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:22.345 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.144417+0000", 2023-10-09T05:54:22.345 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5509999999999997e-05 2023-10-09T05:54:22.345 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.345 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.345 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:22.345 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.144426+0000", 2023-10-09T05:54:22.345 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6719999999999993e-06 2023-10-09T05:54:22.345 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.345 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.345 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:22.346 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.144431+0000", 2023-10-09T05:54:22.346 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2750000000000003e-06 2023-10-09T05:54:22.346 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.346 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.346 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.346 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.189174+0000", 2023-10-09T05:54:22.346 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.044742523999999999 2023-10-09T05:54:22.346 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.346 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.346 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.346 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.189184+0000", 2023-10-09T05:54:22.346 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0798999999999999e-05 2023-10-09T05:54:22.347 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.347 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.347 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.347 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088004+0000", 2023-10-09T05:54:22.347 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.89882000699999998 2023-10-09T05:54:22.347 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.347 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.347 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.347 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088016+0000", 2023-10-09T05:54:22.347 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1809e-05 2023-10-09T05:54:22.347 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.347 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.347 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.348 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.266944+0000", 2023-10-09T05:54:22.348 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.178928057 2023-10-09T05:54:22.348 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.348 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.348 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:22.348 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.267103+0000", 2023-10-09T05:54:22.348 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015863100000000001 2023-10-09T05:54:22.348 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.348 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.348 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:54:22.348 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.267262+0000", 2023-10-09T05:54:22.348 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000159388 2023-10-09T05:54:22.349 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.349 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.349 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:22.349 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.275961+0000", 2023-10-09T05:54:22.349 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0086985629999999994 2023-10-09T05:54:22.349 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.349 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.349 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.349 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.280428+0000", 2023-10-09T05:54:22.349 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0044668620000000003 2023-10-09T05:54:22.349 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.349 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.350 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.350 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.281192+0000", 2023-10-09T05:54:22.350 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00076383899999999999 2023-10-09T05:54:22.350 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.350 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.350 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:22.350 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.281240+0000", 2023-10-09T05:54:22.350 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8152999999999998e-05 2023-10-09T05:54:22.350 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.350 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.350 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:22.351 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.283646+0000", 2023-10-09T05:54:22.351 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0024061540000000002 2023-10-09T05:54:22.351 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.351 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:22.351 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.351 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.351 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.351 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:82106 6.d 6:b7051b26:::benchmark_data_smithi064_284193_object82105:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:54:22.351 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:56.173869+0000", 2023-10-09T05:54:22.351 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 146.05359464, 2023-10-09T05:54:22.351 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1196443020000002, 2023-10-09T05:54:22.352 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:22.352 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:22.352 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:22.352 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:22.352 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:22.352 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 82106 2023-10-09T05:54:22.352 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.352 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:22.352 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.352 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:22.352 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.173869+0000", 2023-10-09T05:54:22.352 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.353 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.353 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.353 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:22.353 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.173869+0000", 2023-10-09T05:54:22.353 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.353 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.353 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.353 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:22.353 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.173867+0000", 2023-10-09T05:54:22.353 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T05:54:22.353 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.353 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.354 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:22.354 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.173923+0000", 2023-10-09T05:54:22.354 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.647e-05 2023-10-09T05:54:22.354 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.354 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.354 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:22.354 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.173949+0000", 2023-10-09T05:54:22.354 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.5079000000000002e-05 2023-10-09T05:54:22.354 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.354 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.355 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:22.355 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.173955+0000", 2023-10-09T05:54:22.355 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4300000000000003e-06 2023-10-09T05:54:22.355 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.355 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.355 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.355 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.189208+0000", 2023-10-09T05:54:22.355 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015253040000000001 2023-10-09T05:54:22.355 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.355 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.355 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.355 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.189215+0000", 2023-10-09T05:54:22.356 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0999999999999998e-06 2023-10-09T05:54:22.356 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.356 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.356 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.356 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088046+0000", 2023-10-09T05:54:22.356 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.89883056900000002 2023-10-09T05:54:22.356 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.356 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.356 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.356 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088053+0000", 2023-10-09T05:54:22.356 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5909999999999998e-06 2023-10-09T05:54:22.356 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.356 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.357 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.357 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.269849+0000", 2023-10-09T05:54:22.357 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1817954580000001 2023-10-09T05:54:22.357 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.357 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.357 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:22.357 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.269978+0000", 2023-10-09T05:54:22.357 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012922500000000001 2023-10-09T05:54:22.357 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.357 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.357 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:54:22.357 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.270106+0000", 2023-10-09T05:54:22.358 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012807000000000001 2023-10-09T05:54:22.358 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.358 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.358 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:22.358 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.275969+0000", 2023-10-09T05:54:22.358 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0058634300000000002 2023-10-09T05:54:22.358 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.358 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.358 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.358 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.288964+0000", 2023-10-09T05:54:22.358 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012994916 2023-10-09T05:54:22.359 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.359 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.359 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.359 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.290167+0000", 2023-10-09T05:54:22.359 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0012025320000000001 2023-10-09T05:54:22.359 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.359 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.359 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:22.359 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.290229+0000", 2023-10-09T05:54:22.359 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2274999999999996e-05 2023-10-09T05:54:22.359 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.359 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.359 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:22.360 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.293513+0000", 2023-10-09T05:54:22.360 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0032839380000000001 2023-10-09T05:54:22.360 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.360 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:22.360 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.360 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.360 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.360 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:82129 6.a 6:5e1e98b2:::benchmark_data_smithi064_284193_object82128:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:54:22.360 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:56.214518+0000", 2023-10-09T05:54:22.360 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 146.01294575099999, 2023-10-09T05:54:22.360 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.051940192, 2023-10-09T05:54:22.360 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:22.361 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:22.361 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:22.361 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:22.361 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:22.361 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 82129 2023-10-09T05:54:22.361 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.361 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:22.361 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.361 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:22.361 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.214518+0000", 2023-10-09T05:54:22.361 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.362 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.362 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.362 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:22.362 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.214518+0000", 2023-10-09T05:54:22.362 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.362 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.362 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.362 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:22.362 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.214513+0000", 2023-10-09T05:54:22.362 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999948 2023-10-09T05:54:22.362 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.362 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.363 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:22.363 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.214557+0000", 2023-10-09T05:54:22.363 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4127e-05 2023-10-09T05:54:22.363 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.363 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.363 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:22.363 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.214580+0000", 2023-10-09T05:54:22.363 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3390000000000001e-05 2023-10-09T05:54:22.363 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.363 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.363 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:22.364 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.214589+0000", 2023-10-09T05:54:22.364 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9719999999999998e-06 2023-10-09T05:54:22.364 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.364 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.364 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.364 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.216231+0000", 2023-10-09T05:54:22.364 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001641637 2023-10-09T05:54:22.364 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.364 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.364 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.364 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.216242+0000", 2023-10-09T05:54:22.364 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0929e-05 2023-10-09T05:54:22.365 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.365 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.365 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.365 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088597+0000", 2023-10-09T05:54:22.365 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.87235592100000003 2023-10-09T05:54:22.365 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.365 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.365 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.365 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088604+0000", 2023-10-09T05:54:22.365 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7649999999999996e-06 2023-10-09T05:54:22.365 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.365 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.366 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.366 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.256832+0000", 2023-10-09T05:54:22.366 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1682279229999999 2023-10-09T05:54:22.366 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.366 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.366 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:22.366 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.256991+0000", 2023-10-09T05:54:22.366 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015927700000000001 2023-10-09T05:54:22.366 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.366 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.366 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:54:22.367 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.257137+0000", 2023-10-09T05:54:22.367 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014602500000000001 2023-10-09T05:54:22.367 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.367 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.367 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:22.367 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.263290+0000", 2023-10-09T05:54:22.367 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0061524459999999998 2023-10-09T05:54:22.367 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.367 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.367 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.367 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.265238+0000", 2023-10-09T05:54:22.367 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001948537 2023-10-09T05:54:22.368 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.368 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.368 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.368 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.265424+0000", 2023-10-09T05:54:22.368 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000185164 2023-10-09T05:54:22.368 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.368 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.368 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:22.368 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.265488+0000", 2023-10-09T05:54:22.368 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4632000000000001e-05 2023-10-09T05:54:22.368 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.369 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.369 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:22.369 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.266458+0000", 2023-10-09T05:54:22.369 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00096960600000000003 2023-10-09T05:54:22.369 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.369 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:22.369 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.369 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.369 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.369 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:82138 6.a 6:5fd97b9e:::benchmark_data_smithi064_284193_object82137:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:54:22.369 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:56.236652+0000", 2023-10-09T05:54:22.369 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 145.99081114800001, 2023-10-09T05:54:22.370 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0342095279999999, 2023-10-09T05:54:22.370 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:22.370 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:22.370 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:22.370 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:22.370 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:22.370 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 82138 2023-10-09T05:54:22.370 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.370 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:22.370 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.370 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:22.371 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.236652+0000", 2023-10-09T05:54:22.371 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.371 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.371 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.371 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:22.371 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.236652+0000", 2023-10-09T05:54:22.371 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.371 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.371 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.371 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:22.371 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.236650+0000", 2023-10-09T05:54:22.371 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T05:54:22.372 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.372 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.372 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:22.372 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.236699+0000", 2023-10-09T05:54:22.372 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8606000000000002e-05 2023-10-09T05:54:22.372 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.372 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.372 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:22.372 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.236723+0000", 2023-10-09T05:54:22.372 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4097999999999999e-05 2023-10-09T05:54:22.372 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.372 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.373 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:22.373 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.236732+0000", 2023-10-09T05:54:22.373 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.5709999999999998e-06 2023-10-09T05:54:22.373 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.373 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.373 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.373 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.236803+0000", 2023-10-09T05:54:22.373 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1162999999999996e-05 2023-10-09T05:54:22.373 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.373 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.373 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.374 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.236815+0000", 2023-10-09T05:54:22.374 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2274e-05 2023-10-09T05:54:22.374 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.374 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.374 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.374 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.089210+0000", 2023-10-09T05:54:22.374 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.852395291 2023-10-09T05:54:22.374 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.374 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.374 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.374 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.089216+0000", 2023-10-09T05:54:22.375 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6060000000000002e-06 2023-10-09T05:54:22.375 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.375 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.375 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.375 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.260084+0000", 2023-10-09T05:54:22.375 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.170867879 2023-10-09T05:54:22.375 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.375 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.375 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:22.375 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.260256+0000", 2023-10-09T05:54:22.375 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00017194700000000001 2023-10-09T05:54:22.375 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.376 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.376 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:54:22.376 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.260449+0000", 2023-10-09T05:54:22.376 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00019292300000000001 2023-10-09T05:54:22.376 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.376 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.376 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:22.376 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.265252+0000", 2023-10-09T05:54:22.376 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0048034929999999998 2023-10-09T05:54:22.376 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.376 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.376 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.377 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.265322+0000", 2023-10-09T05:54:22.377 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9429000000000002e-05 2023-10-09T05:54:22.377 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.377 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.377 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.377 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.269814+0000", 2023-10-09T05:54:22.377 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0044918750000000002 2023-10-09T05:54:22.377 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.377 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.377 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:22.377 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.269876+0000", 2023-10-09T05:54:22.378 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2621999999999998e-05 2023-10-09T05:54:22.378 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.378 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.378 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:22.378 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.270862+0000", 2023-10-09T05:54:22.378 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00098564500000000005 2023-10-09T05:54:22.378 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.378 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:22.378 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.378 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.378 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.379 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:91002 6.4 6:2cef0a59:::benchmark_data_smithi064_284193_object91001:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-10-09T05:54:22.379 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:10.173814+0000", 2023-10-09T05:54:22.379 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 132.053649568, 2023-10-09T05:54:22.379 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.336934939, 2023-10-09T05:54:22.379 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:22.379 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T05:54:22.379 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:22.379 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:22.379 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:22.379 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 91002 2023-10-09T05:54:22.379 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.380 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:22.380 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.380 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:22.380 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.173814+0000", 2023-10-09T05:54:22.380 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.380 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.380 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.380 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:22.380 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.173814+0000", 2023-10-09T05:54:22.380 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.380 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.380 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.381 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:22.381 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.173813+0000", 2023-10-09T05:54:22.381 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:54:22.381 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.381 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.381 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:22.381 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.173843+0000", 2023-10-09T05:54:22.381 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9298999999999998e-05 2023-10-09T05:54:22.381 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.381 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.381 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:22.381 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.173850+0000", 2023-10-09T05:54:22.382 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.413e-06 2023-10-09T05:54:22.382 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.382 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.382 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:22.382 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.173854+0000", 2023-10-09T05:54:22.382 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4159999999999997e-06 2023-10-09T05:54:22.382 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.382 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.382 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.382 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.177830+0000", 2023-10-09T05:54:22.382 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0039752570000000003 2023-10-09T05:54:22.383 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.383 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.383 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:22.383 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.178005+0000", 2023-10-09T05:54:22.383 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00017501100000000001 2023-10-09T05:54:22.383 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.383 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.383 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:54:22.383 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.178180+0000", 2023-10-09T05:54:22.383 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00017503899999999999 2023-10-09T05:54:22.383 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.384 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.384 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.384 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.212537+0000", 2023-10-09T05:54:22.384 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.034357681000000001 2023-10-09T05:54:22.384 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.384 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.384 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.384 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.212546+0000", 2023-10-09T05:54:22.384 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.3909999999999998e-06 2023-10-09T05:54:22.384 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.384 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.384 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.385 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.488951+0000", 2023-10-09T05:54:22.385 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.276405051 2023-10-09T05:54:22.385 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.385 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.385 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for degraded object", 2023-10-09T05:54:22.385 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.489540+0000", 2023-10-09T05:54:22.385 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00058895699999999998 2023-10-09T05:54:22.385 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.385 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.385 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.385 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.510637+0000", 2023-10-09T05:54:22.385 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.021097202999999998 2023-10-09T05:54:22.386 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.386 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.386 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:22.386 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.510749+0000", 2023-10-09T05:54:22.386 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011176299999999999 2023-10-09T05:54:22.386 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.386 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:22.386 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.386 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.386 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.386 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:91017 6.d 6:b3504a53:::benchmark_data_smithi064_284193_object91016:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-10-09T05:54:22.387 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:10.210870+0000", 2023-10-09T05:54:22.387 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 132.016593559, 2023-10-09T05:54:22.387 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2371660760000001, 2023-10-09T05:54:22.387 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:22.387 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:22.387 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:22.387 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:22.387 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:22.387 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 91017 2023-10-09T05:54:22.387 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.387 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:22.387 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.388 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:22.388 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210870+0000", 2023-10-09T05:54:22.388 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.388 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.388 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.388 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:22.388 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210870+0000", 2023-10-09T05:54:22.388 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.388 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.388 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.388 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:22.388 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210869+0000", 2023-10-09T05:54:22.389 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:54:22.389 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.389 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.389 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:22.389 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210904+0000", 2023-10-09T05:54:22.389 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.4656999999999997e-05 2023-10-09T05:54:22.389 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.389 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.389 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:22.389 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210914+0000", 2023-10-09T05:54:22.389 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0464e-05 2023-10-09T05:54:22.390 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.390 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.390 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:22.390 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210920+0000", 2023-10-09T05:54:22.390 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6380000000000001e-06 2023-10-09T05:54:22.390 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.390 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.390 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.390 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210970+0000", 2023-10-09T05:54:22.390 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0062999999999998e-05 2023-10-09T05:54:22.391 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.391 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.391 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.391 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210979+0000", 2023-10-09T05:54:22.391 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.7299999999999994e-06 2023-10-09T05:54:22.391 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.391 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.391 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.391 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.441103+0000", 2023-10-09T05:54:22.391 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2301243880000001 2023-10-09T05:54:22.391 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.391 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.392 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:22.392 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.441224+0000", 2023-10-09T05:54:22.392 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012059 2023-10-09T05:54:22.392 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.392 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.392 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:54:22.392 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.441351+0000", 2023-10-09T05:54:22.392 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012714700000000001 2023-10-09T05:54:22.392 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.392 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.392 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:22.392 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.445558+0000", 2023-10-09T05:54:22.393 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0042071510000000001 2023-10-09T05:54:22.393 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.393 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.393 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.393 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.446131+0000", 2023-10-09T05:54:22.393 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00057308200000000004 2023-10-09T05:54:22.393 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.393 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.393 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.393 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.447453+0000", 2023-10-09T05:54:22.393 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0013219460000000001 2023-10-09T05:54:22.393 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.394 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.394 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:22.394 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.447493+0000", 2023-10-09T05:54:22.394 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9920999999999999e-05 2023-10-09T05:54:22.394 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.394 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.394 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:22.394 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.448036+0000", 2023-10-09T05:54:22.394 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00054314100000000002 2023-10-09T05:54:22.394 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.394 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:22.395 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.395 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.395 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.395 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:91019 6.4 6:25997699:::benchmark_data_smithi064_284193_object91018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-10-09T05:54:22.395 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:10.215587+0000", 2023-10-09T05:54:22.395 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 132.011876677, 2023-10-09T05:54:22.395 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2962565879999999, 2023-10-09T05:54:22.395 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:22.395 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:22.395 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:22.395 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:22.395 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:22.396 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 91019 2023-10-09T05:54:22.396 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.396 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:22.396 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.396 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:22.396 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215587+0000", 2023-10-09T05:54:22.396 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.396 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.396 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.396 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:22.396 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215587+0000", 2023-10-09T05:54:22.396 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.397 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.397 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.397 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:22.397 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215585+0000", 2023-10-09T05:54:22.397 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:22.397 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.397 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.397 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:22.397 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215691+0000", 2023-10-09T05:54:22.397 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000105311 2023-10-09T05:54:22.397 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.398 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.398 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:22.398 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215710+0000", 2023-10-09T05:54:22.398 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.893e-05 2023-10-09T05:54:22.398 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.398 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.398 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:22.398 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215717+0000", 2023-10-09T05:54:22.398 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8950000000000001e-06 2023-10-09T05:54:22.398 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.398 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.399 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.399 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.216473+0000", 2023-10-09T05:54:22.399 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00075626899999999995 2023-10-09T05:54:22.399 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.399 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.399 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.399 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.216492+0000", 2023-10-09T05:54:22.399 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9072999999999998e-05 2023-10-09T05:54:22.399 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.399 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.399 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.399 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.489588+0000", 2023-10-09T05:54:22.400 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.273096518 2023-10-09T05:54:22.400 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.400 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.400 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:22.400 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.489698+0000", 2023-10-09T05:54:22.400 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000109848 2023-10-09T05:54:22.400 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.400 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.400 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:54:22.400 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.489836+0000", 2023-10-09T05:54:22.400 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013729700000000001 2023-10-09T05:54:22.401 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.401 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.401 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:22.401 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.496824+0000", 2023-10-09T05:54:22.401 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0069883929999999999 2023-10-09T05:54:22.401 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.401 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.401 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.401 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.504295+0000", 2023-10-09T05:54:22.401 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0074707799999999998 2023-10-09T05:54:22.401 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.402 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.402 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.402 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.510792+0000", 2023-10-09T05:54:22.402 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00649718 2023-10-09T05:54:22.402 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.402 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.402 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:22.402 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.510853+0000", 2023-10-09T05:54:22.402 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0868000000000001e-05 2023-10-09T05:54:22.402 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.402 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.402 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:22.403 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.511843+0000", 2023-10-09T05:54:22.403 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00099058400000000008 2023-10-09T05:54:22.403 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.403 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:22.403 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.403 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.403 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.403 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:91052 6.d 6:b9656c98:::benchmark_data_smithi064_284193_object91051:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-10-09T05:54:22.403 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:10.282349+0000", 2023-10-09T05:54:22.403 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 131.945114641, 2023-10-09T05:54:22.403 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.174734331, 2023-10-09T05:54:22.404 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:22.404 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:22.404 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:22.404 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:22.404 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:22.404 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 91052 2023-10-09T05:54:22.404 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.404 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:22.404 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.404 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:22.404 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.282349+0000", 2023-10-09T05:54:22.404 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.405 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.405 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.405 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:22.405 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.282349+0000", 2023-10-09T05:54:22.405 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:22.405 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.405 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.405 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:22.405 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.282347+0000", 2023-10-09T05:54:22.405 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T05:54:22.405 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.405 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.406 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:22.406 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.282415+0000", 2023-10-09T05:54:22.406 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8465000000000003e-05 2023-10-09T05:54:22.406 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.406 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.406 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:22.406 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.282431+0000", 2023-10-09T05:54:22.406 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5739e-05 2023-10-09T05:54:22.406 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.406 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.406 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:22.407 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.282439+0000", 2023-10-09T05:54:22.407 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5900000000000002e-06 2023-10-09T05:54:22.407 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.407 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.407 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.407 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.282507+0000", 2023-10-09T05:54:22.407 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7891000000000004e-05 2023-10-09T05:54:22.407 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.407 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.407 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:22.407 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.282519+0000", 2023-10-09T05:54:22.407 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2887e-05 2023-10-09T05:54:22.408 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.408 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.408 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:22.408 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.443395+0000", 2023-10-09T05:54:22.408 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.160875715 2023-10-09T05:54:22.408 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.408 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.408 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:22.408 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.443472+0000", 2023-10-09T05:54:22.408 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7158999999999998e-05 2023-10-09T05:54:22.408 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.408 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.409 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:54:22.409 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.443574+0000", 2023-10-09T05:54:22.409 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000102141 2023-10-09T05:54:22.409 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.409 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.409 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:22.409 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.445569+0000", 2023-10-09T05:54:22.409 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001994928 2023-10-09T05:54:22.409 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.409 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.409 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.409 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.448070+0000", 2023-10-09T05:54:22.410 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0025002280000000002 2023-10-09T05:54:22.410 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.410 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.410 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:22.410 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.456279+0000", 2023-10-09T05:54:22.410 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0082089169999999996 2023-10-09T05:54:22.410 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.410 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.410 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:22.410 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.456318+0000", 2023-10-09T05:54:22.410 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9218000000000003e-05 2023-10-09T05:54:22.411 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:22.411 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:22.411 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:22.411 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.457083+0000", 2023-10-09T05:54:22.411 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00076537899999999995 2023-10-09T05:54:22.411 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.411 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:22.411 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.411 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:22.411 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:22.411 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:54:22.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:22.413 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:22.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:22.461 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:22 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 18459/267741 objects degraded (6.894%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:22.461 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:22 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 18459/267741 objects degraded (6.894%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:22.513 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:22.539 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:22 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 18459/267741 objects degraded (6.894%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:22.543 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:22.801 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:22.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:22.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:22.905 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:22.940 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:23.265 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:54:23.548 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:23.550 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:23 smithi064 ceph-mon[212750]: pgmap v419: 109 pgs: 1 activating+degraded, 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 18459/267741 objects degraded (6.894%); 14019/267741 objects misplaced (5.236%) 2023-10-09T05:54:23.550 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:23 smithi064 ceph-mon[220044]: pgmap v419: 109 pgs: 1 activating+degraded, 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 18459/267741 objects degraded (6.894%); 14019/267741 objects misplaced (5.236%) 2023-10-09T05:54:23.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:23.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:23.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:23 smithi156 ceph-mon[195101]: pgmap v419: 109 pgs: 1 activating+degraded, 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 18459/267741 objects degraded (6.894%); 14019/267741 objects misplaced (5.236%) 2023-10-09T05:54:23.651 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:54:23.651 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:23.686 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:23.713 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:54:23.735 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:54:23.761 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:54:23.786 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:54:23.793 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:23.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:23.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:23.806 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:54:23.826 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:54:23.897 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:23.926 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:23.972 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:23.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:23.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:24.075 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:24.109 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:24.410 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:24.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:24 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:54:24.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:24 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:54:24.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:24.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:24.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:24 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:54:24.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:24 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:54:24.518 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:24.563 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:24.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:24 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:54:24.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:24 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:54:24.872 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:24.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:24.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:24.976 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:25.020 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:25.118 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:25.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:25.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:25.208 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T05:54:25.208 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:54:25.223 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:25.265 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:25.327 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T05:54:25.410 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:54:25.414 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:54:25.415 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:54:25.415 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:54:25.415 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:54:25.415 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:54:25.415 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:54:25.415 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:54:25.415 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:54:25.423 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:25.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:25.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:25.535 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:25.564 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:25 smithi156 ceph-mon[195101]: pgmap v420: 109 pgs: 1 activating+undersized+degraded+remapped, 1 activating+degraded, 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 30104/267741 objects degraded (11.244%); 8472/267741 objects misplaced (3.164%) 2023-10-09T05:54:25.565 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:25 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1069126674' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:54:25.565 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:25 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:54:25.568 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:25.618 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:25.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:25.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:25.646 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:25 smithi064 ceph-mon[212750]: pgmap v420: 109 pgs: 1 activating+undersized+degraded+remapped, 1 activating+degraded, 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 30104/267741 objects degraded (11.244%); 8472/267741 objects misplaced (3.164%) 2023-10-09T05:54:25.646 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:25 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1069126674' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:54:25.646 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:25 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:54:25.647 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:25 smithi064 ceph-mon[220044]: pgmap v420: 109 pgs: 1 activating+undersized+degraded+remapped, 1 activating+degraded, 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 30104/267741 objects degraded (11.244%); 8472/267741 objects misplaced (3.164%) 2023-10-09T05:54:25.647 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:25 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1069126674' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:54:25.647 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:25 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:54:25.647 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:54:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:54:25] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-10-09T05:54:25.723 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:25.759 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:25.898 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:54:25.926 INFO:teuthology.orchestra.run.smithi064.stdout:moved 0 / 327 (0%) 2023-10-09T05:54:25.927 INFO:teuthology.orchestra.run.smithi064.stdout:avg 40.875 2023-10-09T05:54:25.927 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 13.5963 -> 13.5963 (expected baseline 5.98044) 2023-10-09T05:54:25.927 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.3 with 17 -> 17 pgs (0.415902 -> 0.415902 * mean) 2023-10-09T05:54:25.927 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.6 with 56 -> 56 pgs (1.37003 -> 1.37003 * mean) 2023-10-09T05:54:25.927 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:54:25.927 INFO:teuthology.orchestra.run.smithi064.stdout:oload 1000 2023-10-09T05:54:25.927 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 3 2023-10-09T05:54:25.927 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-10-09T05:54:25.927 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0269 2023-10-09T05:54:25.927 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.2686 2023-10-09T05:54:25.931 INFO:teuthology.orchestra.run.smithi064.stderr:no change 2023-10-09T05:54:26.254 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:54:26.297 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:26 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:26.298 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:26 smithi156 ceph-mon[195101]: failed to encode map e185 with expected crc 2023-10-09T05:54:26.298 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:26 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:54:26.298 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:26 smithi156 ceph-mon[195101]: osdmap e185: 8 total, 8 up, 8 in 2023-10-09T05:54:26.298 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:26 smithi156 ceph-mon[195101]: failed to encode map e185 with expected crc 2023-10-09T05:54:26.298 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:26 smithi156 ceph-mon[195101]: from='client.44909 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:26.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:26 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:26.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:26 smithi064 ceph-mon[212750]: failed to encode map e185 with expected crc 2023-10-09T05:54:26.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:26 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:54:26.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:26 smithi064 ceph-mon[212750]: osdmap e185: 8 total, 8 up, 8 in 2023-10-09T05:54:26.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:26 smithi064 ceph-mon[212750]: failed to encode map e185 with expected crc 2023-10-09T05:54:26.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:26 smithi064 ceph-mon[212750]: from='client.44909 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:26 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:26 smithi064 ceph-mon[220044]: failed to encode map e185 with expected crc 2023-10-09T05:54:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:26 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:54:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:26 smithi064 ceph-mon[220044]: osdmap e185: 8 total, 8 up, 8 in 2023-10-09T05:54:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:26 smithi064 ceph-mon[220044]: failed to encode map e185 with expected crc 2023-10-09T05:54:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:26 smithi064 ceph-mon[220044]: from='client.44909 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:27.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:27 smithi064 ceph-mon[212750]: failed to encode map e185 with expected crc 2023-10-09T05:54:27.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:27 smithi064 ceph-mon[212750]: pgmap v422: 109 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 1 activating+degraded, 1 peering, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34633/267741 objects degraded (12.935%); 8081/267741 objects misplaced (3.018%) 2023-10-09T05:54:27.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:27 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 30104/267741 objects degraded (11.244%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:27.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:27 smithi064 ceph-mon[220044]: failed to encode map e185 with expected crc 2023-10-09T05:54:27.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:27 smithi064 ceph-mon[220044]: pgmap v422: 109 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 1 activating+degraded, 1 peering, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34633/267741 objects degraded (12.935%); 8081/267741 objects misplaced (3.018%) 2023-10-09T05:54:27.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:27 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 30104/267741 objects degraded (11.244%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:27.554 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:27 smithi156 ceph-mon[195101]: failed to encode map e185 with expected crc 2023-10-09T05:54:27.554 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:27 smithi156 ceph-mon[195101]: pgmap v422: 109 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 1 activating+degraded, 1 peering, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34633/267741 objects degraded (12.935%); 8081/267741 objects misplaced (3.018%) 2023-10-09T05:54:27.554 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:27 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 30104/267741 objects degraded (11.244%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:27.747 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T05:54:27.747 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:54:28.328 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:28.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:28.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:28.431 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:54:28.434 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:28.479 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:28.567 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:28 smithi064 ceph-mon[212750]: 1.10 scrub starts 2023-10-09T05:54:28.567 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:28 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:54:28.569 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:28 smithi064 ceph-mon[220044]: 1.10 scrub starts 2023-10-09T05:54:28.570 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:28 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:54:28.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:28 smithi156 ceph-mon[195101]: 1.10 scrub starts 2023-10-09T05:54:28.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:28 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:54:28.642 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:28.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:28.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:28.749 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:28.778 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:28.812 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:54:28.826 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:28.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:28.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:28.934 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:28.971 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:29.031 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:29.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:29.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:29.139 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:29.168 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:29.365 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:29.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:29.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:29.468 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:29.503 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:29.558 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:29.563 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:29 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:54:29.563 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:29 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:54:29.563 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:29.563 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:29 smithi064 ceph-mon[220044]: pgmap v423: 109 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 1 activating+degraded, 1 peering, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34633/267741 objects degraded (12.935%); 8081/267741 objects misplaced (3.018%) 2023-10-09T05:54:29.564 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:29.564 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:54:29.564 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:29 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:54:29.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:29 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:54:29.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:29 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:54:29.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:29.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:29 smithi064 ceph-mon[212750]: pgmap v423: 109 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 1 activating+degraded, 1 peering, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34633/267741 objects degraded (12.935%); 8081/267741 objects misplaced (3.018%) 2023-10-09T05:54:29.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:29.565 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:54:29.565 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:29 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:54:29.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:29.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:29.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:29 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:54:29.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:29 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:54:29.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:29.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:29 smithi156 ceph-mon[195101]: pgmap v423: 109 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 1 activating+degraded, 1 peering, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34633/267741 objects degraded (12.935%); 8081/267741 objects misplaced (3.018%) 2023-10-09T05:54:29.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:29.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:54:29.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:29 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:54:29.666 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:29.694 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:29.940 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:29.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:29.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:30.045 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:30.077 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:30.131 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:30.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:30.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:30.239 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:30.265 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:54:30.272 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:30.340 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:30.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:30.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:30.420 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T05:54:30.420 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:54:30.445 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:30.484 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:30.550 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:30.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:30.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:30.636 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:54:30.655 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:30.687 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:30.738 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:30.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:30.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:30.746 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:30.842 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:30.877 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:30.935 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:30.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:30.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:31.041 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:31.083 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:31.251 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:31.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:31.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:31.356 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:31.388 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:31.533 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:54:31.546 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:31.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:31.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:31.650 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:31.677 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:31.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:31 smithi064 ceph-mon[212750]: pgmap v424: 109 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 activating+degraded, 1 peering, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 30090/267741 objects degraded (11.238%); 11616/267741 objects misplaced (4.339%); 6.9 MiB/s, 112 objects/s recovering 2023-10-09T05:54:31.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:31 smithi064 ceph-mon[220044]: pgmap v424: 109 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 activating+degraded, 1 peering, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 30090/267741 objects degraded (11.238%); 11616/267741 objects misplaced (4.339%); 6.9 MiB/s, 112 objects/s recovering 2023-10-09T05:54:31.725 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:31.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:31.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:31.831 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:31.858 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:31 smithi156 ceph-mon[195101]: pgmap v424: 109 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 activating+degraded, 1 peering, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 30090/267741 objects degraded (11.238%); 11616/267741 objects misplaced (4.339%); 6.9 MiB/s, 112 objects/s recovering 2023-10-09T05:54:31.861 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:31.909 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:31.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:31.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:31.974 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:54:32.018 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:32.050 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:32.104 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:32.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:32.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:32.208 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:32.240 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:32.294 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:32.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:32.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:32.398 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:32.429 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:32.483 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:32.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:32.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:32.588 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:32.617 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:32.662 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:32.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:32.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:32.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:32 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 30090/267741 objects degraded (11.238%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:32.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:32 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 30090/267741 objects degraded (11.238%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:32.766 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:32.798 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:32.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:32 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 30090/267741 objects degraded (11.238%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:32.889 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:32.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:32.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:32.932 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:32.993 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:33.033 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:33.104 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:33.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:33.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:33.208 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:33.248 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:33.414 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:33.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:33.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:33.519 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:33.549 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:33.569 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 1, 7] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T05:54:33.569 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:54:33.570 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_0 2023-10-09T05:54:33.570 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:54:33.596 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:33.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:33.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:33.700 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:33.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:33 smithi064 ceph-mon[212750]: pgmap v425: 109 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 activating+degraded, 1 peering, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 30090/267741 objects degraded (11.238%); 11616/267741 objects misplaced (4.339%); 6.9 MiB/s, 112 objects/s recovering 2023-10-09T05:54:33.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:33 smithi064 ceph-mon[212750]: Health check update: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:54:33.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:33 smithi064 ceph-mon[220044]: pgmap v425: 109 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 activating+degraded, 1 peering, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 30090/267741 objects degraded (11.238%); 11616/267741 objects misplaced (4.339%); 6.9 MiB/s, 112 objects/s recovering 2023-10-09T05:54:33.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:33 smithi064 ceph-mon[220044]: Health check update: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:54:33.728 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:33.822 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:33 smithi156 ceph-mon[195101]: pgmap v425: 109 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 activating+degraded, 1 peering, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 30090/267741 objects degraded (11.238%); 11616/267741 objects misplaced (4.339%); 6.9 MiB/s, 112 objects/s recovering 2023-10-09T05:54:33.822 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:33 smithi156 ceph-mon[195101]: Health check update: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:54:33.827 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:33.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:33.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:33.932 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:33.965 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:34.036 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T05:54:34.036 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:54:34.572 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:34.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:34.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:34.677 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:34.721 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:34.791 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:34.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:34.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:34.860 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:34.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:34 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/128262377' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:54:34.898 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:34.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:34 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/128262377' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:54:34.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:34 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/128262377' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:54:34.948 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:35.129 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:35.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:35.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:35.215 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:54:35.235 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:35.272 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:35.352 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:35.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:35.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:35.457 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:35.492 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:35.550 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:35.552 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:54:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:54:35] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-10-09T05:54:35.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:35.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:35.601 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:54:35.655 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:35.684 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:35.734 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:35.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:35.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:35.838 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:35.869 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:35 smithi064 ceph-mon[212750]: pgmap v426: 109 pgs: 1 active+undersized+degraded+remapped+laggy, 2 active+recovering+undersized+remapped, 1 activating+degraded, 1 peering, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 30090/267741 objects degraded (11.238%); 11616/267741 objects misplaced (4.339%); 5.9 MiB/s, 96 objects/s recovering 2023-10-09T05:54:35.869 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:35 smithi064 ceph-mon[220044]: pgmap v426: 109 pgs: 1 active+undersized+degraded+remapped+laggy, 2 active+recovering+undersized+remapped, 1 activating+degraded, 1 peering, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 30090/267741 objects degraded (11.238%); 11616/267741 objects misplaced (4.339%); 5.9 MiB/s, 96 objects/s recovering 2023-10-09T05:54:35.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:35 smithi156 ceph-mon[195101]: pgmap v426: 109 pgs: 1 active+undersized+degraded+remapped+laggy, 2 active+recovering+undersized+remapped, 1 activating+degraded, 1 peering, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 30090/267741 objects degraded (11.238%); 11616/267741 objects misplaced (4.339%); 5.9 MiB/s, 96 objects/s recovering 2023-10-09T05:54:35.872 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:35.929 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:35.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:35.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:36.033 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:36.073 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:36.123 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:36.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:36.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:36.228 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:36.265 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:36.327 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:36.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:36.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:36.431 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:36.459 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:36.508 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:36.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:36.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:36.612 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:36.629 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T05:54:36.675 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:36.739 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:36.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:36.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:36.843 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:36.854 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:36 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/711571510' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:36.854 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:36 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:36.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:36 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/711571510' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:36.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:36 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:36.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:36 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/711571510' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:36.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:36 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:36.872 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:37.006 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:37.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:37.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:37.110 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:37.142 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:37.243 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:37.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:37.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:37.346 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:37.389 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:37.565 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:37.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:37.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:37.582 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T05:54:37.582 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:54:37.670 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:37.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:37 smithi064 ceph-mon[212750]: pgmap v427: 109 pgs: 1 active+undersized+degraded+remapped+laggy, 2 active+recovering+undersized+remapped, 1 activating+degraded, 1 peering, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 30090/267720 objects degraded (11.239%); 10687/267720 objects misplaced (3.992%); 8.3 MiB/s, 134 objects/s recovering 2023-10-09T05:54:37.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:37 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:54:37.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:37 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:54:37.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:37 smithi064 ceph-mon[212750]: osdmap e186: 8 total, 8 up, 8 in 2023-10-09T05:54:37.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:37 smithi064 ceph-mon[212750]: failed to encode map e186 with expected crc 2023-10-09T05:54:37.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:37 smithi064 ceph-mon[212750]: failed to encode map e186 with expected crc 2023-10-09T05:54:37.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:37 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 30090/267720 objects degraded (11.239%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:37.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:37 smithi064 ceph-mon[220044]: pgmap v427: 109 pgs: 1 active+undersized+degraded+remapped+laggy, 2 active+recovering+undersized+remapped, 1 activating+degraded, 1 peering, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 30090/267720 objects degraded (11.239%); 10687/267720 objects misplaced (3.992%); 8.3 MiB/s, 134 objects/s recovering 2023-10-09T05:54:37.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:37 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:54:37.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:37 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:54:37.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:37 smithi064 ceph-mon[220044]: osdmap e186: 8 total, 8 up, 8 in 2023-10-09T05:54:37.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:37 smithi064 ceph-mon[220044]: failed to encode map e186 with expected crc 2023-10-09T05:54:37.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:37 smithi064 ceph-mon[220044]: failed to encode map e186 with expected crc 2023-10-09T05:54:37.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:37 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 30090/267720 objects degraded (11.239%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:37.702 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:37.804 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:37.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:37.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:37 smithi156 ceph-mon[195101]: pgmap v427: 109 pgs: 1 active+undersized+degraded+remapped+laggy, 2 active+recovering+undersized+remapped, 1 activating+degraded, 1 peering, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 30090/267720 objects degraded (11.239%); 10687/267720 objects misplaced (3.992%); 8.3 MiB/s, 134 objects/s recovering 2023-10-09T05:54:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:37 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:54:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:37 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:54:37.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:37 smithi156 ceph-mon[195101]: osdmap e186: 8 total, 8 up, 8 in 2023-10-09T05:54:37.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:37 smithi156 ceph-mon[195101]: failed to encode map e186 with expected crc 2023-10-09T05:54:37.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:37 smithi156 ceph-mon[195101]: failed to encode map e186 with expected crc 2023-10-09T05:54:37.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:37 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 30090/267720 objects degraded (11.239%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:37.908 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:37.938 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:37.994 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:37.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:37.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:38.098 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:38.155 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:38.202 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:38.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:38.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:38.236 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:54:38.245 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T05:54:38.305 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:38.333 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:38.378 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:38.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:38.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:38.481 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:38.509 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:38.704 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:38.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:38.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:38.757 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:54:38.782 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:38 smithi156 ceph-mon[195101]: failed to encode map e186 with expected crc 2023-10-09T05:54:38.807 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:38.836 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:38.883 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:38.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:38.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:38.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:38 smithi064 ceph-mon[212750]: failed to encode map e186 with expected crc 2023-10-09T05:54:38.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:38 smithi064 ceph-mon[220044]: failed to encode map e186 with expected crc 2023-10-09T05:54:38.991 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:39.023 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:39.075 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:39.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:39.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:39.159 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:54:39.181 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:39.220 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:39.308 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":428,"stamp":"2023-10-09T05:54:36.600688+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5847390136,"num_objects":89240,"num_object_clones":0,"num_object_copies":267720,"num_objects_missing_on_primary":5218,"num_objects_missing":5218,"num_objects_degraded":30090,"num_objects_misplaced":10687,"num_objects_unfound":0,"num_objects_dirty":89240,"num_whiteouts":0,"num_read":488,"num_read_kb":914,"num_write":104166,"num_write_kb":6181006,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27912,"num_bytes_recovered":1806184574,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":98332,"ondisk_log_size":98332,"up":327,"acting":316,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":349,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20218156,"kb_used_data":18631468,"kb_used_omap":127,"kb_used_meta":1586496,"kb_avail":729743060,"statfs":{"total":767960285184,"available":747256893440,"internally_reserved":0,"allocated":19078623232,"data_stored":19072082803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":130887,"internal_metadata":1624572089},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-458752,"num_objects":-7,"num_object_clones":0,"num_object_copies":-21,"num_objects_missing_on_primary":-14,"num_objects_missing":-14,"num_objects_degraded":-4543,"num_objects_misplaced":2606,"num_objects_unfound":0,"num_objects_dirty":-7,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1504,"num_bytes_recovered":97648640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.077537"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":200,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.199354+0000","last_change":"2023-10-09T05:52:11.412762+0000","last_active":"2023-10-09T05:54:19.199354+0000","last_peered":"2023-10-09T05:54:19.199354+0000","last_clean":"2023-10-09T05:54:19.199354+0000","last_became_active":"2023-10-09T05:52:11.412360+0000","last_became_peered":"2023-10-09T05:52:11.412360+0000","last_unstale":"2023-10-09T05:54:19.199354+0000","last_undegraded":"2023-10-09T05:54:19.199354+0000","last_fullsized":"2023-10-09T05:54:19.199354+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:28.459463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":123,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:23.681361+0000","last_change":"2023-10-09T05:54:16.213101+0000","last_active":"2023-10-09T05:54:23.681361+0000","last_peered":"2023-10-09T05:54:23.681361+0000","last_clean":"2023-10-09T05:54:23.681361+0000","last_became_active":"2023-10-09T05:54:16.209047+0000","last_became_peered":"2023-10-09T05:54:16.209047+0000","last_unstale":"2023-10-09T05:54:23.681361+0000","last_undegraded":"2023-10-09T05:54:23.681361+0000","last_fullsized":"2023-10-09T05:54:23.681361+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:38.146494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":41,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:23.675140+0000","last_change":"2023-10-09T05:52:11.410566+0000","last_active":"2023-10-09T05:54:23.675140+0000","last_peered":"2023-10-09T05:54:23.675140+0000","last_clean":"2023-10-09T05:54:23.675140+0000","last_became_active":"2023-10-09T05:52:11.410252+0000","last_became_peered":"2023-10-09T05:52:11.410252+0000","last_unstale":"2023-10-09T05:54:23.675140+0000","last_undegraded":"2023-10-09T05:54:23.675140+0000","last_fullsized":"2023-10-09T05:54:23.675140+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:06.878658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":154,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T05:54:16.225697+0000","last_change":"2023-10-09T05:54:16.225697+0000","last_active":"2023-10-09T05:54:16.225697+0000","last_peered":"2023-10-09T05:54:16.225697+0000","last_clean":"2023-10-09T05:54:16.225697+0000","last_became_active":"2023-10-09T05:54:16.225383+0000","last_became_peered":"2023-10-09T05:54:16.225383+0000","last_unstale":"2023-10-09T05:54:16.225697+0000","last_undegraded":"2023-10-09T05:54:16.225697+0000","last_fullsized":"2023-10-09T05:54:16.225697+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:38:53.629954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":152,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T05:54:16.226278+0000","last_change":"2023-10-09T05:54:16.217288+0000","last_active":"2023-10-09T05:54:16.226278+0000","last_peered":"2023-10-09T05:54:16.226278+0000","last_clean":"2023-10-09T05:54:16.226278+0000","last_became_active":"2023-10-09T05:54:16.216955+0000","last_became_peered":"2023-10-09T05:54:16.216955+0000","last_unstale":"2023-10-09T05:54:16.226278+0000","last_undegraded":"2023-10-09T05:54:16.226278+0000","last_fullsized":"2023-10-09T05:54:16.226278+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:46:26.642710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":181,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.203193+0000","last_change":"2023-10-09T05:52:11.218711+0000","last_active":"2023-10-09T05:54:19.203193+0000","last_peered":"2023-10-09T05:54:19.203193+0000","last_clean":"2023-10-09T05:54:19.203193+0000","last_became_active":"2023-10-09T05:52:11.218513+0000","last_became_peered":"2023-10-09T05:52:11.218513+0000","last_unstale":"2023-10-09T05:54:19.203193+0000","last_undegraded":"2023-10-09T05:54:19.203193+0000","last_fullsized":"2023-10-09T05:54:19.203193+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:02.240130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":125,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T05:54:16.209688+0000","last_change":"2023-10-09T05:54:16.209688+0000","last_active":"2023-10-09T05:54:16.209688+0000","last_peered":"2023-10-09T05:54:16.209688+0000","last_clean":"2023-10-09T05:54:16.209688+0000","last_became_active":"2023-10-09T05:54:16.209338+0000","last_became_peered":"2023-10-09T05:54:16.209338+0000","last_unstale":"2023-10-09T05:54:16.209688+0000","last_undegraded":"2023-10-09T05:54:16.209688+0000","last_fullsized":"2023-10-09T05:54:16.209688+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:28.211691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":123,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:23.670943+0000","last_change":"2023-10-09T05:54:16.198945+0000","last_active":"2023-10-09T05:54:23.670943+0000","last_peered":"2023-10-09T05:54:23.670943+0000","last_clean":"2023-10-09T05:54:23.670943+0000","last_became_active":"2023-10-09T05:54:16.198625+0000","last_became_peered":"2023-10-09T05:54:16.198625+0000","last_unstale":"2023-10-09T05:54:23.670943+0000","last_undegraded":"2023-10-09T05:54:23.670943+0000","last_fullsized":"2023-10-09T05:54:23.670943+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:40.758090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":433,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.205546+0000","last_change":"2023-10-09T05:54:19.204752+0000","last_active":"2023-10-09T05:54:19.205546+0000","last_peered":"2023-10-09T05:54:19.205546+0000","last_clean":"2023-10-09T05:54:19.205546+0000","last_became_active":"2023-10-09T05:54:19.204405+0000","last_became_peered":"2023-10-09T05:54:19.204405+0000","last_unstale":"2023-10-09T05:54:19.205546+0000","last_undegraded":"2023-10-09T05:54:19.205546+0000","last_fullsized":"2023-10-09T05:54:19.205546+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":185,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.209213+0000","last_change":"2023-10-09T05:53:59.184396+0000","last_active":"2023-10-09T05:54:19.209213+0000","last_peered":"2023-10-09T05:54:19.209213+0000","last_clean":"2023-10-09T05:54:19.209213+0000","last_became_active":"2023-10-09T05:53:59.184083+0000","last_became_peered":"2023-10-09T05:53:59.184083+0000","last_unstale":"2023-10-09T05:54:19.209213+0000","last_undegraded":"2023-10-09T05:54:19.209213+0000","last_fullsized":"2023-10-09T05:54:19.209213+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:27.160599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":127,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.193692+0000","last_change":"2023-10-09T05:53:59.185702+0000","last_active":"2023-10-09T05:54:19.193692+0000","last_peered":"2023-10-09T05:54:19.193692+0000","last_clean":"2023-10-09T05:54:19.193692+0000","last_became_active":"2023-10-09T05:53:59.185373+0000","last_became_peered":"2023-10-09T05:53:59.185373+0000","last_unstale":"2023-10-09T05:54:19.193692+0000","last_undegraded":"2023-10-09T05:54:19.193692+0000","last_fullsized":"2023-10-09T05:54:19.193692+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:50.756967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":25,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:25.357727+0000","last_change":"2023-10-09T05:53:59.219168+0000","last_active":"2023-10-09T05:54:25.357727+0000","last_peered":"2023-10-09T05:54:25.357727+0000","last_clean":"2023-10-09T05:54:25.357727+0000","last_became_active":"2023-10-09T05:53:59.200886+0000","last_became_peered":"2023-10-09T05:53:59.200886+0000","last_unstale":"2023-10-09T05:54:25.357727+0000","last_undegraded":"2023-10-09T05:54:25.357727+0000","last_fullsized":"2023-10-09T05:54:25.357727+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:45.515036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":63,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T05:54:25.306734+0000","last_change":"2023-10-09T05:52:11.224890+0000","last_active":"2023-10-09T05:54:25.306734+0000","last_peered":"2023-10-09T05:54:25.306734+0000","last_clean":"2023-10-09T05:54:25.306734+0000","last_became_active":"2023-10-09T05:52:11.224591+0000","last_became_peered":"2023-10-09T05:52:11.224591+0000","last_unstale":"2023-10-09T05:54:25.306734+0000","last_undegraded":"2023-10-09T05:54:25.306734+0000","last_fullsized":"2023-10-09T05:54:25.306734+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:53.788492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":190,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.200846+0000","last_change":"2023-10-09T05:53:59.219141+0000","last_active":"2023-10-09T05:54:19.200846+0000","last_peered":"2023-10-09T05:54:19.200846+0000","last_clean":"2023-10-09T05:54:19.200846+0000","last_became_active":"2023-10-09T05:53:59.197318+0000","last_became_peered":"2023-10-09T05:53:59.197318+0000","last_unstale":"2023-10-09T05:54:19.200846+0000","last_undegraded":"2023-10-09T05:54:19.200846+0000","last_fullsized":"2023-10-09T05:54:19.200846+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:39.910794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":164,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.199463+0000","last_change":"2023-10-09T05:52:11.412779+0000","last_active":"2023-10-09T05:54:19.199463+0000","last_peered":"2023-10-09T05:54:19.199463+0000","last_clean":"2023-10-09T05:54:19.199463+0000","last_became_active":"2023-10-09T05:52:11.412489+0000","last_became_peered":"2023-10-09T05:52:11.412489+0000","last_unstale":"2023-10-09T05:54:19.199463+0000","last_undegraded":"2023-10-09T05:54:19.199463+0000","last_fullsized":"2023-10-09T05:54:19.199463+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:39:37.025949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":515,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.196905+0000","last_change":"2023-10-09T05:53:20.237354+0000","last_active":"2023-10-09T05:54:19.196905+0000","last_peered":"2023-10-09T05:54:19.196905+0000","last_clean":"2023-10-09T05:54:19.196905+0000","last_became_active":"2023-10-09T05:53:20.236993+0000","last_became_peered":"2023-10-09T05:53:20.236993+0000","last_unstale":"2023-10-09T05:54:19.196905+0000","last_undegraded":"2023-10-09T05:54:19.196905+0000","last_fullsized":"2023-10-09T05:54:19.196905+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":27,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.201131+0000","last_change":"2023-10-09T05:53:59.233898+0000","last_active":"2023-10-09T05:54:19.201131+0000","last_peered":"2023-10-09T05:54:19.201131+0000","last_clean":"2023-10-09T05:54:19.201131+0000","last_became_active":"2023-10-09T05:53:59.204175+0000","last_became_peered":"2023-10-09T05:53:59.204175+0000","last_unstale":"2023-10-09T05:54:19.201131+0000","last_undegraded":"2023-10-09T05:54:19.201131+0000","last_fullsized":"2023-10-09T05:54:19.201131+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:03.139318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":186,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.194391+0000","last_change":"2023-10-09T05:53:59.187417+0000","last_active":"2023-10-09T05:54:19.194391+0000","last_peered":"2023-10-09T05:54:19.194391+0000","last_clean":"2023-10-09T05:54:19.194391+0000","last_became_active":"2023-10-09T05:53:59.186834+0000","last_became_peered":"2023-10-09T05:53:59.186834+0000","last_unstale":"2023-10-09T05:54:19.194391+0000","last_undegraded":"2023-10-09T05:54:19.194391+0000","last_fullsized":"2023-10-09T05:54:19.194391+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:33.418594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":578,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T05:54:27.330211+0000","last_change":"2023-10-09T05:54:27.330211+0000","last_active":"2023-10-09T05:54:27.330211+0000","last_peered":"2023-10-09T05:54:27.330211+0000","last_clean":"2023-10-09T05:54:27.330211+0000","last_became_active":"2023-10-09T05:53:59.202399+0000","last_became_peered":"2023-10-09T05:53:59.202399+0000","last_unstale":"2023-10-09T05:54:27.330211+0000","last_undegraded":"2023-10-09T05:54:27.330211+0000","last_fullsized":"2023-10-09T05:54:27.330211+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:51:12.346243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:51:12.346243+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":145,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T05:54:16.205838+0000","last_change":"2023-10-09T05:53:59.185567+0000","last_active":"2023-10-09T05:54:16.205838+0000","last_peered":"2023-10-09T05:54:16.205838+0000","last_clean":"2023-10-09T05:54:16.205838+0000","last_became_active":"2023-10-09T05:53:59.185290+0000","last_became_peered":"2023-10-09T05:53:59.185290+0000","last_unstale":"2023-10-09T05:54:16.205838+0000","last_undegraded":"2023-10-09T05:54:16.205838+0000","last_fullsized":"2023-10-09T05:54:16.205838+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:10.289297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":164,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.198994+0000","last_change":"2023-10-09T05:53:59.182073+0000","last_active":"2023-10-09T05:54:19.198994+0000","last_peered":"2023-10-09T05:54:19.198994+0000","last_clean":"2023-10-09T05:54:19.198994+0000","last_became_active":"2023-10-09T05:53:59.181715+0000","last_became_peered":"2023-10-09T05:53:59.181715+0000","last_unstale":"2023-10-09T05:54:19.198994+0000","last_undegraded":"2023-10-09T05:54:19.198994+0000","last_fullsized":"2023-10-09T05:54:19.198994+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:58:31.918499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":496,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.223541+0000","last_change":"2023-10-09T05:54:19.221202+0000","last_active":"2023-10-09T05:54:19.223541+0000","last_peered":"2023-10-09T05:54:19.223541+0000","last_clean":"2023-10-09T05:54:19.223541+0000","last_became_active":"2023-10-09T05:54:19.220864+0000","last_became_peered":"2023-10-09T05:54:19.220864+0000","last_unstale":"2023-10-09T05:54:19.223541+0000","last_undegraded":"2023-10-09T05:54:19.223541+0000","last_fullsized":"2023-10-09T05:54:19.223541+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":21,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.200271+0000","last_change":"2023-10-09T05:53:59.206887+0000","last_active":"2023-10-09T05:54:19.200271+0000","last_peered":"2023-10-09T05:54:19.200271+0000","last_clean":"2023-10-09T05:54:19.200271+0000","last_became_active":"2023-10-09T05:53:59.206608+0000","last_became_peered":"2023-10-09T05:53:59.206608+0000","last_unstale":"2023-10-09T05:54:19.200271+0000","last_undegraded":"2023-10-09T05:54:19.200271+0000","last_fullsized":"2023-10-09T05:54:19.200271+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:48.975792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":143,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T05:54:16.225672+0000","last_change":"2023-10-09T05:54:16.225672+0000","last_active":"2023-10-09T05:54:16.225672+0000","last_peered":"2023-10-09T05:54:16.225672+0000","last_clean":"2023-10-09T05:54:16.225672+0000","last_became_active":"2023-10-09T05:54:16.225312+0000","last_became_peered":"2023-10-09T05:54:16.225312+0000","last_unstale":"2023-10-09T05:54:16.225672+0000","last_undegraded":"2023-10-09T05:54:16.225672+0000","last_fullsized":"2023-10-09T05:54:16.225672+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:52.172719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":434,"reported_epoch":181,"state":"remapped+peering","last_fresh":"2023-10-09T05:54:16.197764+0000","last_change":"2023-10-09T05:54:16.197915+0000","last_active":"2023-10-09T05:54:15.184962+0000","last_peered":"2023-10-09T05:53:55.682103+0000","last_clean":"2023-10-09T05:53:55.682103+0000","last_became_active":"2023-10-09T05:53:34.485883+0000","last_became_peered":"2023-10-09T05:53:34.485883+0000","last_unstale":"2023-10-09T05:54:16.197764+0000","last_undegraded":"2023-10-09T05:54:16.197764+0000","last_fullsized":"2023-10-09T05:54:16.197764+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,5,7],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":170,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.198194+0000","last_change":"2023-10-09T05:51:57.091639+0000","last_active":"2023-10-09T05:54:19.198194+0000","last_peered":"2023-10-09T05:54:19.198194+0000","last_clean":"2023-10-09T05:54:19.198194+0000","last_became_active":"2023-10-09T05:51:57.091284+0000","last_became_peered":"2023-10-09T05:51:57.091284+0000","last_unstale":"2023-10-09T05:54:19.198194+0000","last_undegraded":"2023-10-09T05:54:19.198194+0000","last_fullsized":"2023-10-09T05:54:19.198194+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:10.553453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":145,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.211113+0000","last_change":"2023-10-09T05:54:16.220064+0000","last_active":"2023-10-09T05:54:19.211113+0000","last_peered":"2023-10-09T05:54:19.211113+0000","last_clean":"2023-10-09T05:54:19.211113+0000","last_became_active":"2023-10-09T05:54:16.219716+0000","last_became_peered":"2023-10-09T05:54:16.219716+0000","last_unstale":"2023-10-09T05:54:19.211113+0000","last_undegraded":"2023-10-09T05:54:19.211113+0000","last_fullsized":"2023-10-09T05:54:19.211113+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:36.049124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":518,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.200195+0000","last_change":"2023-10-09T05:53:20.160853+0000","last_active":"2023-10-09T05:54:19.200195+0000","last_peered":"2023-10-09T05:54:19.200195+0000","last_clean":"2023-10-09T05:54:19.200195+0000","last_became_active":"2023-10-09T05:53:20.160520+0000","last_became_peered":"2023-10-09T05:53:20.160520+0000","last_unstale":"2023-10-09T05:54:19.200195+0000","last_undegraded":"2023-10-09T05:54:19.200195+0000","last_fullsized":"2023-10-09T05:54:19.200195+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.a","version":"168'6464","reported_seq":8754,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.198689+0000","last_change":"2023-10-09T05:53:29.244424+0000","last_active":"2023-10-09T05:54:19.198689+0000","last_peered":"2023-10-09T05:54:19.198689+0000","last_clean":"2023-10-09T05:54:19.198689+0000","last_became_active":"2023-10-09T05:53:29.236060+0000","last_became_peered":"2023-10-09T05:53:29.236060+0000","last_unstale":"2023-10-09T05:54:19.198689+0000","last_undegraded":"2023-10-09T05:54:19.198689+0000","last_fullsized":"2023-10-09T05:54:19.198689+0000","mapping_epoch":161,"log_start":"106'449","ondisk_log_start":"106'449","created":102,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6015,"log_dups_size":0,"ondisk_log_size":6015,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:10.331676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364511232,"num_objects":5562,"num_object_clones":0,"num_object_copies":16686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6464,"num_write_kb":384832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1085,"num_bytes_recovered":71106560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":25,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.200482+0000","last_change":"2023-10-09T05:53:59.222136+0000","last_active":"2023-10-09T05:54:19.200482+0000","last_peered":"2023-10-09T05:54:19.200482+0000","last_clean":"2023-10-09T05:54:19.200482+0000","last_became_active":"2023-10-09T05:53:59.201853+0000","last_became_peered":"2023-10-09T05:53:59.201853+0000","last_unstale":"2023-10-09T05:54:19.200482+0000","last_undegraded":"2023-10-09T05:54:19.200482+0000","last_fullsized":"2023-10-09T05:54:19.200482+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:26.662031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":147,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.196997+0000","last_change":"2023-10-09T05:53:59.187102+0000","last_active":"2023-10-09T05:54:19.196997+0000","last_peered":"2023-10-09T05:54:19.196997+0000","last_clean":"2023-10-09T05:54:19.196997+0000","last_became_active":"2023-10-09T05:53:59.186758+0000","last_became_peered":"2023-10-09T05:53:59.186758+0000","last_unstale":"2023-10-09T05:54:19.196997+0000","last_undegraded":"2023-10-09T05:54:19.196997+0000","last_fullsized":"2023-10-09T05:54:19.196997+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:35.806325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":423,"reported_epoch":184,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:54:19.199451+0000","last_change":"2023-10-09T05:54:17.325990+0000","last_active":"2023-10-09T05:54:19.199451+0000","last_peered":"2023-10-09T05:54:19.199451+0000","last_clean":"2023-10-09T05:43:58.449564+0000","last_became_active":"2023-10-09T05:54:17.325267+0000","last_became_peered":"2023-10-09T05:54:17.325267+0000","last_unstale":"2023-10-09T05:54:19.199451+0000","last_undegraded":"2023-10-09T05:54:17.200969+0000","last_fullsized":"2023-10-09T05:54:17.200582+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1},{"shards":"5,7","objects":-1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"185'6570","reported_seq":16125,"reported_epoch":185,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:54:34.600157+0000","last_change":"2023-10-09T05:54:17.412522+0000","last_active":"2023-10-09T05:54:34.600157+0000","last_peered":"2023-10-09T05:54:34.600157+0000","last_clean":"2023-10-09T05:51:24.569821+0000","last_became_active":"2023-10-09T05:54:17.327957+0000","last_became_peered":"2023-10-09T05:54:17.327957+0000","last_unstale":"2023-10-09T05:54:34.600157+0000","last_undegraded":"2023-10-09T05:54:34.600157+0000","last_fullsized":"2023-10-09T05:54:17.186934+0000","mapping_epoch":181,"log_start":"106'587","ondisk_log_start":"106'587","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5983,"log_dups_size":0,"ondisk_log_size":5983,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:30:14.955069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369360896,"num_objects":5636,"num_object_clones":0,"num_object_copies":16908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1394,"num_objects_unfound":0,"num_objects_dirty":5636,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6570,"num_write_kb":390592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4709,"num_bytes_recovered":308609024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":4242},{"shards":"1,3,7","objects":1394}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":25,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.200256+0000","last_change":"2023-10-09T05:53:59.228416+0000","last_active":"2023-10-09T05:54:19.200256+0000","last_peered":"2023-10-09T05:54:19.200256+0000","last_clean":"2023-10-09T05:54:19.200256+0000","last_became_active":"2023-10-09T05:53:59.203493+0000","last_became_peered":"2023-10-09T05:53:59.203493+0000","last_unstale":"2023-10-09T05:54:19.200256+0000","last_undegraded":"2023-10-09T05:54:19.200256+0000","last_fullsized":"2023-10-09T05:54:19.200256+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:46.033123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":191,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.198851+0000","last_change":"2023-10-09T05:52:23.642420+0000","last_active":"2023-10-09T05:54:19.198851+0000","last_peered":"2023-10-09T05:54:19.198851+0000","last_clean":"2023-10-09T05:54:19.198851+0000","last_became_active":"2023-10-09T05:52:23.641015+0000","last_became_peered":"2023-10-09T05:52:23.641015+0000","last_unstale":"2023-10-09T05:54:19.198851+0000","last_undegraded":"2023-10-09T05:54:19.198851+0000","last_fullsized":"2023-10-09T05:54:19.198851+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:14.092876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":557,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.203072+0000","last_change":"2023-10-09T05:53:59.187768+0000","last_active":"2023-10-09T05:54:19.203072+0000","last_peered":"2023-10-09T05:54:19.203072+0000","last_clean":"2023-10-09T05:54:19.203072+0000","last_became_active":"2023-10-09T05:53:59.187329+0000","last_became_peered":"2023-10-09T05:53:59.187329+0000","last_unstale":"2023-10-09T05:54:19.203072+0000","last_undegraded":"2023-10-09T05:54:19.203072+0000","last_fullsized":"2023-10-09T05:54:19.203072+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.8","version":"185'6466","reported_seq":6569,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T05:54:35.097015+0000","last_change":"2023-10-09T05:54:00.188782+0000","last_active":"2023-10-09T05:54:35.097015+0000","last_peered":"2023-10-09T05:54:35.097015+0000","last_clean":"2023-10-09T05:54:35.097015+0000","last_became_active":"2023-10-09T05:54:00.179758+0000","last_became_peered":"2023-10-09T05:54:00.179758+0000","last_unstale":"2023-10-09T05:54:35.097015+0000","last_undegraded":"2023-10-09T05:54:35.097015+0000","last_fullsized":"2023-10-09T05:54:35.097015+0000","mapping_epoch":174,"log_start":"106'122","ondisk_log_start":"106'122","created":102,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6344,"log_dups_size":0,"ondisk_log_size":6344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:18:05.031854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358219776,"num_objects":5466,"num_object_clones":0,"num_object_copies":16398,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5466,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6466,"num_write_kb":381824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":23,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:25.357885+0000","last_change":"2023-10-09T05:53:59.212161+0000","last_active":"2023-10-09T05:54:25.357885+0000","last_peered":"2023-10-09T05:54:25.357885+0000","last_clean":"2023-10-09T05:54:25.357885+0000","last_became_active":"2023-10-09T05:53:59.193814+0000","last_became_peered":"2023-10-09T05:53:59.193814+0000","last_unstale":"2023-10-09T05:54:25.357885+0000","last_undegraded":"2023-10-09T05:54:25.357885+0000","last_fullsized":"2023-10-09T05:54:25.357885+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:10.502408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":194,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.198492+0000","last_change":"2023-10-09T05:53:59.191188+0000","last_active":"2023-10-09T05:54:19.198492+0000","last_peered":"2023-10-09T05:54:19.198492+0000","last_clean":"2023-10-09T05:54:19.198492+0000","last_became_active":"2023-10-09T05:53:59.190847+0000","last_became_peered":"2023-10-09T05:53:59.190847+0000","last_unstale":"2023-10-09T05:54:19.198492+0000","last_undegraded":"2023-10-09T05:54:19.198492+0000","last_fullsized":"2023-10-09T05:54:19.198492+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:51.394603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":544,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.202703+0000","last_change":"2023-10-09T05:53:20.167463+0000","last_active":"2023-10-09T05:54:19.202703+0000","last_peered":"2023-10-09T05:54:19.202703+0000","last_clean":"2023-10-09T05:54:19.202703+0000","last_became_active":"2023-10-09T05:53:20.167014+0000","last_became_peered":"2023-10-09T05:53:20.167014+0000","last_unstale":"2023-10-09T05:54:19.202703+0000","last_undegraded":"2023-10-09T05:54:19.202703+0000","last_fullsized":"2023-10-09T05:54:19.202703+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"168'6502","reported_seq":6616,"reported_epoch":181,"state":"activating+degraded","last_fresh":"2023-10-09T05:54:16.446213+0000","last_change":"2023-10-09T05:54:16.446213+0000","last_active":"2023-10-09T05:53:58.201897+0000","last_peered":"2023-10-09T05:53:55.771031+0000","last_clean":"2023-10-09T05:51:33.020184+0000","last_became_active":"2023-10-09T05:53:32.717867+0000","last_became_peered":"2023-10-09T05:53:32.717867+0000","last_unstale":"2023-10-09T05:54:16.446213+0000","last_undegraded":"2023-10-09T05:54:16.335911+0000","last_fullsized":"2023-10-09T05:54:16.446213+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6502,"log_dups_size":0,"ondisk_log_size":6502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:33:28.815082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369754112,"num_objects":5642,"num_object_clones":0,"num_object_copies":16926,"num_objects_missing_on_primary":1173,"num_objects_missing":1173,"num_objects_degraded":7245,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5642,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6502,"num_write_kb":388608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"0,7","objects":4469},{"shards":"7","objects":1173}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":155,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.199785+0000","last_change":"2023-10-09T05:54:16.370039+0000","last_active":"2023-10-09T05:54:19.199785+0000","last_peered":"2023-10-09T05:54:19.199785+0000","last_clean":"2023-10-09T05:54:19.199785+0000","last_became_active":"2023-10-09T05:54:16.369668+0000","last_became_peered":"2023-10-09T05:54:16.369668+0000","last_unstale":"2023-10-09T05:54:19.199785+0000","last_undegraded":"2023-10-09T05:54:19.199785+0000","last_fullsized":"2023-10-09T05:54:19.199785+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:50.691727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":154,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T05:54:16.202282+0000","last_change":"2023-10-09T05:52:11.345261+0000","last_active":"2023-10-09T05:54:16.202282+0000","last_peered":"2023-10-09T05:54:16.202282+0000","last_clean":"2023-10-09T05:54:16.202282+0000","last_became_active":"2023-10-09T05:52:11.345031+0000","last_became_peered":"2023-10-09T05:52:11.345031+0000","last_unstale":"2023-10-09T05:54:16.202282+0000","last_undegraded":"2023-10-09T05:54:16.202282+0000","last_fullsized":"2023-10-09T05:54:16.202282+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:07.748821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":578,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.204967+0000","last_change":"2023-10-09T05:54:19.204406+0000","last_active":"2023-10-09T05:54:19.204967+0000","last_peered":"2023-10-09T05:54:19.204967+0000","last_clean":"2023-10-09T05:54:19.204967+0000","last_became_active":"2023-10-09T05:54:19.204082+0000","last_became_peered":"2023-10-09T05:54:19.204082+0000","last_unstale":"2023-10-09T05:54:19.204967+0000","last_undegraded":"2023-10-09T05:54:19.204967+0000","last_fullsized":"2023-10-09T05:54:19.204967+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.e","version":"168'6430","reported_seq":17323,"reported_epoch":184,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:54:19.204683+0000","last_change":"2023-10-09T05:54:17.398152+0000","last_active":"2023-10-09T05:54:19.204683+0000","last_peered":"2023-10-09T05:54:19.204683+0000","last_clean":"2023-10-09T05:51:24.573410+0000","last_became_active":"2023-10-09T05:52:24.814156+0000","last_became_peered":"2023-10-09T05:52:24.814156+0000","last_unstale":"2023-10-09T05:54:19.204683+0000","last_undegraded":"2023-10-09T05:54:17.393932+0000","last_fullsized":"2023-10-09T05:52:24.635559+0000","mapping_epoch":139,"log_start":"106'531","ondisk_log_start":"106'531","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5899,"log_dups_size":0,"ondisk_log_size":5899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:35.696825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363724800,"num_objects":5550,"num_object_clones":0,"num_object_copies":16650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":770,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6430,"num_write_kb":383360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5377,"num_bytes_recovered":352387072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":613},{"shards":"1,4","objects":4937}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":36,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T05:54:16.229175+0000","last_change":"2023-10-09T05:54:16.228877+0000","last_active":"2023-10-09T05:54:16.229175+0000","last_peered":"2023-10-09T05:54:16.229175+0000","last_clean":"2023-10-09T05:54:16.229175+0000","last_became_active":"2023-10-09T05:54:16.221615+0000","last_became_peered":"2023-10-09T05:54:16.221615+0000","last_unstale":"2023-10-09T05:54:16.229175+0000","last_undegraded":"2023-10-09T05:54:16.229175+0000","last_fullsized":"2023-10-09T05:54:16.229175+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:20.211314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":144,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.199578+0000","last_change":"2023-10-09T05:49:21.430742+0000","last_active":"2023-10-09T05:54:19.199578+0000","last_peered":"2023-10-09T05:54:19.199578+0000","last_clean":"2023-10-09T05:54:19.199578+0000","last_became_active":"2023-10-09T05:49:21.430233+0000","last_became_peered":"2023-10-09T05:49:21.430233+0000","last_unstale":"2023-10-09T05:54:19.199578+0000","last_undegraded":"2023-10-09T05:54:19.199578+0000","last_fullsized":"2023-10-09T05:54:19.199578+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:10.541302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":499,"reported_epoch":184,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:54:19.196993+0000","last_change":"2023-10-09T05:54:17.432010+0000","last_active":"2023-10-09T05:54:19.196993+0000","last_peered":"2023-10-09T05:54:19.196993+0000","last_clean":"2023-10-09T05:54:10.413690+0000","last_became_active":"2023-10-09T05:54:17.431670+0000","last_became_peered":"2023-10-09T05:54:17.431670+0000","last_unstale":"2023-10-09T05:54:19.196993+0000","last_undegraded":"2023-10-09T05:54:17.200549+0000","last_fullsized":"2023-10-09T05:54:17.200139+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1},{"shards":"2,5","objects":-1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.f","version":"168'6579","reported_seq":7074,"reported_epoch":184,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:54:19.207117+0000","last_change":"2023-10-09T05:54:17.429058+0000","last_active":"2023-10-09T05:54:19.207117+0000","last_peered":"2023-10-09T05:54:19.207117+0000","last_clean":"2023-10-09T05:52:22.632757+0000","last_became_active":"2023-10-09T05:54:17.365483+0000","last_became_peered":"2023-10-09T05:54:17.365483+0000","last_unstale":"2023-10-09T05:54:19.207117+0000","last_undegraded":"2023-10-09T05:54:17.347077+0000","last_fullsized":"2023-10-09T05:54:17.206490+0000","mapping_epoch":181,"log_start":"107'607","ondisk_log_start":"107'607","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5972,"log_dups_size":0,"ondisk_log_size":5972,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:01.150801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373751808,"num_objects":5703,"num_object_clones":0,"num_object_copies":17109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4563,"num_objects_misplaced":5703,"num_objects_unfound":0,"num_objects_dirty":5703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6579,"num_write_kb":393024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":200,"num_bytes_recovered":13107200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,3],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"2,3,4","objects":4135},{"shards":"3,4","objects":1568}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":165,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.198365+0000","last_change":"2023-10-09T05:52:11.344035+0000","last_active":"2023-10-09T05:54:19.198365+0000","last_peered":"2023-10-09T05:54:19.198365+0000","last_clean":"2023-10-09T05:54:19.198365+0000","last_became_active":"2023-10-09T05:52:11.343800+0000","last_became_peered":"2023-10-09T05:52:11.343800+0000","last_unstale":"2023-10-09T05:54:19.198365+0000","last_undegraded":"2023-10-09T05:54:19.198365+0000","last_fullsized":"2023-10-09T05:54:19.198365+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:07.588214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":191,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.200405+0000","last_change":"2023-10-09T05:54:16.213056+0000","last_active":"2023-10-09T05:54:19.200405+0000","last_peered":"2023-10-09T05:54:19.200405+0000","last_clean":"2023-10-09T05:54:19.200405+0000","last_became_active":"2023-10-09T05:54:16.212635+0000","last_became_peered":"2023-10-09T05:54:16.212635+0000","last_unstale":"2023-10-09T05:54:19.200405+0000","last_undegraded":"2023-10-09T05:54:19.200405+0000","last_fullsized":"2023-10-09T05:54:19.200405+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:52.393799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":496,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.226791+0000","last_change":"2023-10-09T05:54:19.225654+0000","last_active":"2023-10-09T05:54:19.226791+0000","last_peered":"2023-10-09T05:54:19.226791+0000","last_clean":"2023-10-09T05:54:19.226791+0000","last_became_active":"2023-10-09T05:54:19.225232+0000","last_became_peered":"2023-10-09T05:54:19.225232+0000","last_unstale":"2023-10-09T05:54:19.226791+0000","last_undegraded":"2023-10-09T05:54:19.226791+0000","last_fullsized":"2023-10-09T05:54:19.226791+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.c","version":"168'6518","reported_seq":18717,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.202491+0000","last_change":"2023-10-09T05:53:59.196684+0000","last_active":"2023-10-09T05:54:19.202491+0000","last_peered":"2023-10-09T05:54:19.202491+0000","last_clean":"2023-10-09T05:54:19.202491+0000","last_became_active":"2023-10-09T05:53:59.187954+0000","last_became_peered":"2023-10-09T05:53:59.187954+0000","last_unstale":"2023-10-09T05:54:19.202491+0000","last_undegraded":"2023-10-09T05:54:19.202491+0000","last_fullsized":"2023-10-09T05:54:19.202491+0000","mapping_epoch":173,"log_start":"107'1490","ondisk_log_start":"107'1490","created":102,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5028,"log_dups_size":0,"ondisk_log_size":5028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:28.562782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363593728,"num_objects":5548,"num_object_clones":0,"num_object_copies":16644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6518,"num_write_kb":386112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6033,"num_bytes_recovered":385286144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":165,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.198926+0000","last_change":"2023-10-09T05:54:00.619165+0000","last_active":"2023-10-09T05:54:19.198926+0000","last_peered":"2023-10-09T05:54:19.198926+0000","last_clean":"2023-10-09T05:54:19.198926+0000","last_became_active":"2023-10-09T05:54:00.618888+0000","last_became_peered":"2023-10-09T05:54:00.618888+0000","last_unstale":"2023-10-09T05:54:19.198926+0000","last_undegraded":"2023-10-09T05:54:19.198926+0000","last_fullsized":"2023-10-09T05:54:19.198926+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:07.175996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":164,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.199586+0000","last_change":"2023-10-09T05:52:11.412851+0000","last_active":"2023-10-09T05:54:19.199586+0000","last_peered":"2023-10-09T05:54:19.199586+0000","last_clean":"2023-10-09T05:54:19.199586+0000","last_became_active":"2023-10-09T05:52:11.412489+0000","last_became_peered":"2023-10-09T05:52:11.412489+0000","last_unstale":"2023-10-09T05:54:19.199586+0000","last_undegraded":"2023-10-09T05:54:19.199586+0000","last_fullsized":"2023-10-09T05:54:19.199586+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:30:34.286409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":489,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.201249+0000","last_change":"2023-10-09T05:51:19.195984+0000","last_active":"2023-10-09T05:54:19.201249+0000","last_peered":"2023-10-09T05:54:19.201249+0000","last_clean":"2023-10-09T05:54:19.201249+0000","last_became_active":"2023-10-09T05:51:19.195632+0000","last_became_peered":"2023-10-09T05:51:19.195632+0000","last_unstale":"2023-10-09T05:54:19.201249+0000","last_undegraded":"2023-10-09T05:54:19.201249+0000","last_fullsized":"2023-10-09T05:54:19.201249+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.d","version":"168'6465","reported_seq":11729,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.199156+0000","last_change":"2023-10-09T05:52:27.708167+0000","last_active":"2023-10-09T05:54:19.199156+0000","last_peered":"2023-10-09T05:54:19.199156+0000","last_clean":"2023-10-09T05:54:19.199156+0000","last_became_active":"2023-10-09T05:52:27.701168+0000","last_became_peered":"2023-10-09T05:52:27.701168+0000","last_unstale":"2023-10-09T05:54:19.199156+0000","last_undegraded":"2023-10-09T05:54:19.199156+0000","last_fullsized":"2023-10-09T05:54:19.199156+0000","mapping_epoch":142,"log_start":"106'451","ondisk_log_start":"106'451","created":102,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6014,"log_dups_size":0,"ondisk_log_size":6014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:17.406250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362938392,"num_objects":5539,"num_object_clones":0,"num_object_copies":16617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5539,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6465,"num_write_kb":384065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2559,"num_bytes_recovered":167706624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":184,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.197642+0000","last_change":"2023-10-09T05:53:59.204384+0000","last_active":"2023-10-09T05:54:19.197642+0000","last_peered":"2023-10-09T05:54:19.197642+0000","last_clean":"2023-10-09T05:54:19.197642+0000","last_became_active":"2023-10-09T05:53:59.203856+0000","last_became_peered":"2023-10-09T05:53:59.203856+0000","last_unstale":"2023-10-09T05:54:19.197642+0000","last_undegraded":"2023-10-09T05:54:19.197642+0000","last_fullsized":"2023-10-09T05:54:19.197642+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:28.169432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":63,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T05:54:25.316206+0000","last_change":"2023-10-09T05:52:11.409470+0000","last_active":"2023-10-09T05:54:25.316206+0000","last_peered":"2023-10-09T05:54:25.316206+0000","last_clean":"2023-10-09T05:54:25.316206+0000","last_became_active":"2023-10-09T05:52:11.409125+0000","last_became_peered":"2023-10-09T05:52:11.409125+0000","last_unstale":"2023-10-09T05:54:25.316206+0000","last_undegraded":"2023-10-09T05:54:25.316206+0000","last_fullsized":"2023-10-09T05:54:25.316206+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:39:35.220346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":558,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.202849+0000","last_change":"2023-10-09T05:53:59.202618+0000","last_active":"2023-10-09T05:54:19.202849+0000","last_peered":"2023-10-09T05:54:19.202849+0000","last_clean":"2023-10-09T05:54:19.202849+0000","last_became_active":"2023-10-09T05:53:59.202265+0000","last_became_peered":"2023-10-09T05:53:59.202265+0000","last_unstale":"2023-10-09T05:54:19.202849+0000","last_undegraded":"2023-10-09T05:54:19.202849+0000","last_fullsized":"2023-10-09T05:54:19.202849+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":72,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:23.681330+0000","last_change":"2023-10-09T05:53:59.186966+0000","last_active":"2023-10-09T05:54:23.681330+0000","last_peered":"2023-10-09T05:54:23.681330+0000","last_clean":"2023-10-09T05:54:23.681330+0000","last_became_active":"2023-10-09T05:53:59.186685+0000","last_became_peered":"2023-10-09T05:53:59.186685+0000","last_unstale":"2023-10-09T05:54:23.681330+0000","last_undegraded":"2023-10-09T05:54:23.681330+0000","last_fullsized":"2023-10-09T05:54:23.681330+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:33.708347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":172,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.198917+0000","last_change":"2023-10-09T05:52:29.735372+0000","last_active":"2023-10-09T05:54:19.198917+0000","last_peered":"2023-10-09T05:54:19.198917+0000","last_clean":"2023-10-09T05:54:19.198917+0000","last_became_active":"2023-10-09T05:52:29.735026+0000","last_became_peered":"2023-10-09T05:52:29.735026+0000","last_unstale":"2023-10-09T05:54:19.198917+0000","last_undegraded":"2023-10-09T05:54:19.198917+0000","last_fullsized":"2023-10-09T05:54:19.198917+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:53.849406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":63,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:23.679215+0000","last_change":"2023-10-09T05:52:11.407985+0000","last_active":"2023-10-09T05:54:23.679215+0000","last_peered":"2023-10-09T05:54:23.679215+0000","last_clean":"2023-10-09T05:54:23.679215+0000","last_became_active":"2023-10-09T05:52:11.407681+0000","last_became_peered":"2023-10-09T05:52:11.407681+0000","last_unstale":"2023-10-09T05:54:23.679215+0000","last_undegraded":"2023-10-09T05:54:23.679215+0000","last_fullsized":"2023-10-09T05:54:23.679215+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:54.384669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"168'6536","reported_seq":8579,"reported_epoch":181,"state":"remapped+peering","last_fresh":"2023-10-09T05:54:16.197834+0000","last_change":"2023-10-09T05:54:16.198009+0000","last_active":"2023-10-09T05:54:15.216853+0000","last_peered":"2023-10-09T05:54:10.146196+0000","last_clean":"2023-10-09T05:54:10.146196+0000","last_became_active":"2023-10-09T05:53:38.168314+0000","last_became_peered":"2023-10-09T05:53:38.168314+0000","last_unstale":"2023-10-09T05:54:16.197834+0000","last_undegraded":"2023-10-09T05:54:16.197834+0000","last_fullsized":"2023-10-09T05:54:16.197834+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6536,"log_dups_size":0,"ondisk_log_size":6536,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:17:36.667349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368050176,"num_objects":5616,"num_object_clones":0,"num_object_copies":16848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6536,"num_write_kb":388864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1172,"num_bytes_recovered":76808192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,3,5],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":516,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.196633+0000","last_change":"2023-10-09T05:53:20.234074+0000","last_active":"2023-10-09T05:54:19.196633+0000","last_peered":"2023-10-09T05:54:19.196633+0000","last_clean":"2023-10-09T05:54:19.196633+0000","last_became_active":"2023-10-09T05:53:20.233803+0000","last_became_peered":"2023-10-09T05:53:20.233803+0000","last_unstale":"2023-10-09T05:54:19.196633+0000","last_undegraded":"2023-10-09T05:54:19.196633+0000","last_fullsized":"2023-10-09T05:54:19.196633+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.6","version":"168'6508","reported_seq":6667,"reported_epoch":184,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T05:54:19.204848+0000","last_change":"2023-10-09T05:54:17.483219+0000","last_active":"2023-10-09T05:54:19.204848+0000","last_peered":"2023-10-09T05:54:19.204848+0000","last_clean":"2023-10-09T05:52:24.946135+0000","last_became_active":"2023-10-09T05:54:17.410473+0000","last_became_peered":"2023-10-09T05:54:17.410473+0000","last_unstale":"2023-10-09T05:54:19.204848+0000","last_undegraded":"2023-10-09T05:54:17.387963+0000","last_fullsized":"2023-10-09T05:54:19.204848+0000","mapping_epoch":181,"log_start":"106'613","ondisk_log_start":"106'613","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5895,"log_dups_size":0,"ondisk_log_size":5895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:57.274303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362807296,"num_objects":5536,"num_object_clones":0,"num_object_copies":16608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5536,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6508,"num_write_kb":385408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":851968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,6],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":5536}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":186,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.196908+0000","last_change":"2023-10-09T05:52:11.217079+0000","last_active":"2023-10-09T05:54:19.196908+0000","last_peered":"2023-10-09T05:54:19.196908+0000","last_clean":"2023-10-09T05:54:19.196908+0000","last_became_active":"2023-10-09T05:52:11.216737+0000","last_became_peered":"2023-10-09T05:52:11.216737+0000","last_unstale":"2023-10-09T05:54:19.196908+0000","last_undegraded":"2023-10-09T05:54:19.196908+0000","last_fullsized":"2023-10-09T05:54:19.196908+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:04.493525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":162,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.202262+0000","last_change":"2023-10-09T05:51:57.203142+0000","last_active":"2023-10-09T05:54:19.202262+0000","last_peered":"2023-10-09T05:54:19.202262+0000","last_clean":"2023-10-09T05:54:19.202262+0000","last_became_active":"2023-10-09T05:51:57.202850+0000","last_became_peered":"2023-10-09T05:51:57.202850+0000","last_unstale":"2023-10-09T05:54:19.202262+0000","last_undegraded":"2023-10-09T05:54:19.202262+0000","last_fullsized":"2023-10-09T05:54:19.202262+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:51.090737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":72,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:23.681310+0000","last_change":"2023-10-09T05:54:16.369153+0000","last_active":"2023-10-09T05:54:23.681310+0000","last_peered":"2023-10-09T05:54:23.681310+0000","last_clean":"2023-10-09T05:54:23.681310+0000","last_became_active":"2023-10-09T05:54:16.368789+0000","last_became_peered":"2023-10-09T05:54:16.368789+0000","last_unstale":"2023-10-09T05:54:23.681310+0000","last_undegraded":"2023-10-09T05:54:23.681310+0000","last_fullsized":"2023-10-09T05:54:23.681310+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:18.014883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":434,"reported_epoch":181,"state":"remapped+peering","last_fresh":"2023-10-09T05:54:16.205222+0000","last_change":"2023-10-09T05:54:16.205406+0000","last_active":"2023-10-09T05:54:15.185751+0000","last_peered":"2023-10-09T05:53:55.682311+0000","last_clean":"2023-10-09T05:53:55.682311+0000","last_became_active":"2023-10-09T05:53:34.486325+0000","last_became_peered":"2023-10-09T05:53:34.486325+0000","last_unstale":"2023-10-09T05:54:16.205222+0000","last_undegraded":"2023-10-09T05:54:16.205222+0000","last_fullsized":"2023-10-09T05:54:16.205222+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,5,7],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"185'6337","reported_seq":7477,"reported_epoch":185,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:54:35.372562+0000","last_change":"2023-10-09T05:54:28.494062+0000","last_active":"2023-10-09T05:54:35.372562+0000","last_peered":"2023-10-09T05:54:35.372562+0000","last_clean":"2023-10-09T05:53:30.128691+0000","last_became_active":"2023-10-09T05:54:28.169061+0000","last_became_peered":"2023-10-09T05:54:28.169061+0000","last_unstale":"2023-10-09T05:54:35.372562+0000","last_undegraded":"2023-10-09T05:54:35.372562+0000","last_fullsized":"2023-10-09T05:54:17.202774+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6337,"log_dups_size":0,"ondisk_log_size":6337,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:17.916186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358285312,"num_objects":5467,"num_object_clones":0,"num_object_copies":16401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3590,"num_objects_unfound":0,"num_objects_dirty":5467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6337,"num_write_kb":377728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":520,"num_bytes_recovered":33161216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3],"avail_no_missing":["7","3","6"],"object_location_counts":[{"shards":"3,6,7","objects":3590},{"shards":"3,7","objects":1877}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":140,"reported_epoch":181,"state":"peering","last_fresh":"2023-10-09T05:54:16.485593+0000","last_change":"2023-10-09T05:52:22.628140+0000","last_active":"2023-10-09T05:52:22.628051+0000","last_peered":"2023-10-09T05:52:14.259511+0000","last_clean":"2023-10-09T05:52:14.259511+0000","last_became_active":"2023-10-09T05:51:57.090995+0000","last_became_peered":"2023-10-09T05:51:57.090995+0000","last_unstale":"2023-10-09T05:54:16.485593+0000","last_undegraded":"2023-10-09T05:54:16.485593+0000","last_fullsized":"2023-10-09T05:54:16.485593+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:17.345258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,6],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":63,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:23.670981+0000","last_change":"2023-10-09T05:52:11.209763+0000","last_active":"2023-10-09T05:54:23.670981+0000","last_peered":"2023-10-09T05:54:23.670981+0000","last_clean":"2023-10-09T05:54:23.670981+0000","last_became_active":"2023-10-09T05:52:11.209472+0000","last_became_peered":"2023-10-09T05:52:11.209472+0000","last_unstale":"2023-10-09T05:54:23.670981+0000","last_undegraded":"2023-10-09T05:54:23.670981+0000","last_fullsized":"2023-10-09T05:54:23.670981+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:05.160815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":63,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:23.681382+0000","last_change":"2023-10-09T05:52:11.234634+0000","last_active":"2023-10-09T05:54:23.681382+0000","last_peered":"2023-10-09T05:54:23.681382+0000","last_clean":"2023-10-09T05:54:23.681382+0000","last_became_active":"2023-10-09T05:52:11.234307+0000","last_became_peered":"2023-10-09T05:52:11.234307+0000","last_unstale":"2023-10-09T05:54:23.681382+0000","last_undegraded":"2023-10-09T05:54:23.681382+0000","last_fullsized":"2023-10-09T05:54:23.681382+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:37.064308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"168'6390","reported_seq":6481,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.201359+0000","last_change":"2023-10-09T05:50:39.778086+0000","last_active":"2023-10-09T05:54:19.201359+0000","last_peered":"2023-10-09T05:54:19.201359+0000","last_clean":"2023-10-09T05:54:19.201359+0000","last_became_active":"2023-10-09T05:50:39.777777+0000","last_became_peered":"2023-10-09T05:50:39.777777+0000","last_unstale":"2023-10-09T05:54:19.201359+0000","last_undegraded":"2023-10-09T05:54:19.201359+0000","last_fullsized":"2023-10-09T05:54:19.201359+0000","mapping_epoch":102,"log_start":"106'370","ondisk_log_start":"106'370","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6020,"log_dups_size":0,"ondisk_log_size":6020,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:02.350198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362414080,"num_objects":5530,"num_object_clones":0,"num_object_copies":16590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6390,"num_write_kb":381440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":169,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.199170+0000","last_change":"2023-10-09T05:53:59.182813+0000","last_active":"2023-10-09T05:54:19.199170+0000","last_peered":"2023-10-09T05:54:19.199170+0000","last_clean":"2023-10-09T05:54:19.199170+0000","last_became_active":"2023-10-09T05:53:59.182441+0000","last_became_peered":"2023-10-09T05:53:59.182441+0000","last_unstale":"2023-10-09T05:54:19.199170+0000","last_undegraded":"2023-10-09T05:54:19.199170+0000","last_fullsized":"2023-10-09T05:54:19.199170+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:59.004246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":576,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.200777+0000","last_change":"2023-10-09T05:53:59.202723+0000","last_active":"2023-10-09T05:54:19.200777+0000","last_peered":"2023-10-09T05:54:19.200777+0000","last_clean":"2023-10-09T05:54:19.200777+0000","last_became_active":"2023-10-09T05:53:59.202004+0000","last_became_peered":"2023-10-09T05:53:59.202004+0000","last_unstale":"2023-10-09T05:54:19.200777+0000","last_undegraded":"2023-10-09T05:54:19.200777+0000","last_fullsized":"2023-10-09T05:54:19.200777+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:51:12.346243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:51:12.346243+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":72,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.196896+0000","last_change":"2023-10-09T05:52:23.648626+0000","last_active":"2023-10-09T05:54:19.196896+0000","last_peered":"2023-10-09T05:54:19.196896+0000","last_clean":"2023-10-09T05:54:19.196896+0000","last_became_active":"2023-10-09T05:52:23.646143+0000","last_became_peered":"2023-10-09T05:52:23.646143+0000","last_unstale":"2023-10-09T05:54:19.196896+0000","last_undegraded":"2023-10-09T05:54:19.196896+0000","last_fullsized":"2023-10-09T05:54:19.196896+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:51:04.733282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":147,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.202264+0000","last_change":"2023-10-09T05:53:59.195325+0000","last_active":"2023-10-09T05:54:19.202264+0000","last_peered":"2023-10-09T05:54:19.202264+0000","last_clean":"2023-10-09T05:54:19.202264+0000","last_became_active":"2023-10-09T05:53:59.194954+0000","last_became_peered":"2023-10-09T05:53:59.194954+0000","last_unstale":"2023-10-09T05:54:19.202264+0000","last_undegraded":"2023-10-09T05:54:19.202264+0000","last_fullsized":"2023-10-09T05:54:19.202264+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:23.228416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":497,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:23.679265+0000","last_change":"2023-10-09T05:54:19.225612+0000","last_active":"2023-10-09T05:54:23.679265+0000","last_peered":"2023-10-09T05:54:23.679265+0000","last_clean":"2023-10-09T05:54:23.679265+0000","last_became_active":"2023-10-09T05:54:19.225199+0000","last_became_peered":"2023-10-09T05:54:19.225199+0000","last_unstale":"2023-10-09T05:54:23.679265+0000","last_undegraded":"2023-10-09T05:54:23.679265+0000","last_fullsized":"2023-10-09T05:54:23.679265+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":21,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.197819+0000","last_change":"2023-10-09T05:53:59.201198+0000","last_active":"2023-10-09T05:54:19.197819+0000","last_peered":"2023-10-09T05:54:19.197819+0000","last_clean":"2023-10-09T05:54:19.197819+0000","last_became_active":"2023-10-09T05:53:59.200549+0000","last_became_peered":"2023-10-09T05:53:59.200549+0000","last_unstale":"2023-10-09T05:54:19.197819+0000","last_undegraded":"2023-10-09T05:54:19.197819+0000","last_fullsized":"2023-10-09T05:54:19.197819+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:37.316300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":115,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T05:53:59.163108+0000","last_change":"2023-10-09T05:52:23.643245+0000","last_active":"2023-10-09T05:53:59.163108+0000","last_peered":"2023-10-09T05:53:59.163108+0000","last_clean":"2023-10-09T05:53:59.163108+0000","last_became_active":"2023-10-09T05:52:23.642947+0000","last_became_peered":"2023-10-09T05:52:23.642947+0000","last_unstale":"2023-10-09T05:53:59.163108+0000","last_undegraded":"2023-10-09T05:53:59.163108+0000","last_fullsized":"2023-10-09T05:53:59.163108+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:52.260774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":197,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T05:54:16.204587+0000","last_change":"2023-10-09T05:53:59.189777+0000","last_active":"2023-10-09T05:54:16.204587+0000","last_peered":"2023-10-09T05:54:16.204587+0000","last_clean":"2023-10-09T05:54:16.204587+0000","last_became_active":"2023-10-09T05:53:59.189125+0000","last_became_peered":"2023-10-09T05:53:59.189125+0000","last_unstale":"2023-10-09T05:54:16.204587+0000","last_undegraded":"2023-10-09T05:54:16.204587+0000","last_fullsized":"2023-10-09T05:54:16.204587+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:58.739641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"168'6533","reported_seq":7021,"reported_epoch":184,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:54:19.228353+0000","last_change":"2023-10-09T05:54:17.419442+0000","last_active":"2023-10-09T05:54:19.228353+0000","last_peered":"2023-10-09T05:54:19.228353+0000","last_clean":"2023-10-09T05:51:24.557317+0000","last_became_active":"2023-10-09T05:52:51.422504+0000","last_became_peered":"2023-10-09T05:52:51.422504+0000","last_unstale":"2023-10-09T05:54:19.228353+0000","last_undegraded":"2023-10-09T05:52:51.194765+0000","last_fullsized":"2023-10-09T05:52:24.632398+0000","mapping_epoch":139,"log_start":"107'1426","ondisk_log_start":"107'1426","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5107,"log_dups_size":0,"ondisk_log_size":5107,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:07.573528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365363200,"num_objects":5575,"num_object_clones":0,"num_object_copies":16725,"num_objects_missing_on_primary":1924,"num_objects_missing":1924,"num_objects_degraded":5869,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6533,"num_write_kb":387456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":186,"num_bytes_recovered":12189696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"1,3,5","objects":3466},{"shards":"1,5","objects":-144},{"shards":"3,5,6","objects":1924}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":545,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.202925+0000","last_change":"2023-10-09T05:53:20.157691+0000","last_active":"2023-10-09T05:54:19.202925+0000","last_peered":"2023-10-09T05:54:19.202925+0000","last_clean":"2023-10-09T05:54:19.202925+0000","last_became_active":"2023-10-09T05:53:20.157392+0000","last_became_peered":"2023-10-09T05:53:20.157392+0000","last_unstale":"2023-10-09T05:54:19.202925+0000","last_undegraded":"2023-10-09T05:54:19.202925+0000","last_fullsized":"2023-10-09T05:54:19.202925+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:25.357547+0000","last_change":"2023-10-09T05:53:59.201576+0000","last_active":"2023-10-09T05:54:25.357547+0000","last_peered":"2023-10-09T05:54:25.357547+0000","last_clean":"2023-10-09T05:54:25.357547+0000","last_became_active":"2023-10-09T05:53:59.201309+0000","last_became_peered":"2023-10-09T05:53:59.201309+0000","last_unstale":"2023-10-09T05:54:25.357547+0000","last_undegraded":"2023-10-09T05:54:25.357547+0000","last_fullsized":"2023-10-09T05:54:25.357547+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:54.956371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":63,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:23.681351+0000","last_change":"2023-10-09T05:52:53.923884+0000","last_active":"2023-10-09T05:54:23.681351+0000","last_peered":"2023-10-09T05:54:23.681351+0000","last_clean":"2023-10-09T05:54:23.681351+0000","last_became_active":"2023-10-09T05:52:53.923324+0000","last_became_peered":"2023-10-09T05:52:53.923324+0000","last_unstale":"2023-10-09T05:54:23.681351+0000","last_undegraded":"2023-10-09T05:54:23.681351+0000","last_fullsized":"2023-10-09T05:54:23.681351+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:28.764537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":105,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T05:54:16.202427+0000","last_change":"2023-10-09T05:53:59.188462+0000","last_active":"2023-10-09T05:54:16.202427+0000","last_peered":"2023-10-09T05:54:16.202427+0000","last_clean":"2023-10-09T05:54:16.202427+0000","last_became_active":"2023-10-09T05:53:59.188061+0000","last_became_peered":"2023-10-09T05:53:59.188061+0000","last_unstale":"2023-10-09T05:54:16.202427+0000","last_undegraded":"2023-10-09T05:54:16.202427+0000","last_fullsized":"2023-10-09T05:54:16.202427+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:19.167114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"162'6531","reported_seq":6646,"reported_epoch":185,"state":"active+undersized+degraded+remapped+laggy","last_fresh":"2023-10-09T05:54:34.203688+0000","last_change":"2023-10-09T05:54:34.233589+0000","last_active":"2023-10-09T05:54:34.203688+0000","last_peered":"2023-10-09T05:54:34.203688+0000","last_clean":"2023-10-09T05:51:24.563560+0000","last_became_active":"2023-10-09T05:54:33.501402+0000","last_became_peered":"2023-10-09T05:54:33.501402+0000","last_unstale":"2023-10-09T05:54:34.203688+0000","last_undegraded":"2023-10-09T05:54:23.531518+0000","last_fullsized":"2023-10-09T05:54:02.208176+0000","mapping_epoch":176,"log_start":"106'204","ondisk_log_start":"106'204","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6327,"log_dups_size":0,"ondisk_log_size":6327,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:22.630111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372310016,"num_objects":5681,"num_object_clones":0,"num_object_copies":17043,"num_objects_missing_on_primary":2121,"num_objects_missing":2121,"num_objects_degraded":6105,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6531,"num_write_kb":390784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6],"avail_no_missing":["2"],"object_location_counts":[{"shards":"1,2,6","objects":3560},{"shards":"2,6,7","objects":2121}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":519,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.200281+0000","last_change":"2023-10-09T05:53:20.152495+0000","last_active":"2023-10-09T05:54:19.200281+0000","last_peered":"2023-10-09T05:54:19.200281+0000","last_clean":"2023-10-09T05:54:19.200281+0000","last_became_active":"2023-10-09T05:53:20.152109+0000","last_became_peered":"2023-10-09T05:53:20.152109+0000","last_unstale":"2023-10-09T05:54:19.200281+0000","last_undegraded":"2023-10-09T05:54:19.200281+0000","last_fullsized":"2023-10-09T05:54:19.200281+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":189,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.195007+0000","last_change":"2023-10-09T05:53:59.228948+0000","last_active":"2023-10-09T05:54:19.195007+0000","last_peered":"2023-10-09T05:54:19.195007+0000","last_clean":"2023-10-09T05:54:19.195007+0000","last_became_active":"2023-10-09T05:53:59.186359+0000","last_became_peered":"2023-10-09T05:53:59.186359+0000","last_unstale":"2023-10-09T05:54:19.195007+0000","last_undegraded":"2023-10-09T05:54:19.195007+0000","last_fullsized":"2023-10-09T05:54:19.195007+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:39.990576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":63,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:23.681373+0000","last_change":"2023-10-09T05:52:11.415624+0000","last_active":"2023-10-09T05:54:23.681373+0000","last_peered":"2023-10-09T05:54:23.681373+0000","last_clean":"2023-10-09T05:54:23.681373+0000","last_became_active":"2023-10-09T05:52:11.415174+0000","last_became_peered":"2023-10-09T05:52:11.415174+0000","last_unstale":"2023-10-09T05:54:23.681373+0000","last_undegraded":"2023-10-09T05:54:23.681373+0000","last_fullsized":"2023-10-09T05:54:23.681373+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:13.657987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":21,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.197760+0000","last_change":"2023-10-09T05:53:59.201348+0000","last_active":"2023-10-09T05:54:19.197760+0000","last_peered":"2023-10-09T05:54:19.197760+0000","last_clean":"2023-10-09T05:54:19.197760+0000","last_became_active":"2023-10-09T05:53:59.200899+0000","last_became_peered":"2023-10-09T05:53:59.200899+0000","last_unstale":"2023-10-09T05:54:19.197760+0000","last_undegraded":"2023-10-09T05:54:19.197760+0000","last_fullsized":"2023-10-09T05:54:19.197760+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:28.015706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.2","version":"168'6530","reported_seq":18680,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.199524+0000","last_change":"2023-10-09T05:53:25.224813+0000","last_active":"2023-10-09T05:54:19.199524+0000","last_peered":"2023-10-09T05:54:19.199524+0000","last_clean":"2023-10-09T05:54:19.199524+0000","last_became_active":"2023-10-09T05:53:25.205241+0000","last_became_peered":"2023-10-09T05:53:25.205241+0000","last_unstale":"2023-10-09T05:54:19.199524+0000","last_undegraded":"2023-10-09T05:54:19.199524+0000","last_fullsized":"2023-10-09T05:54:19.199524+0000","mapping_epoch":159,"log_start":"106'632","ondisk_log_start":"106'632","created":102,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5898,"log_dups_size":0,"ondisk_log_size":5898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:40.067208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360972288,"num_objects":5508,"num_object_clones":0,"num_object_copies":16524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6530,"num_write_kb":385216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6019,"num_bytes_recovered":382468096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":433,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.203128+0000","last_change":"2023-10-09T05:54:19.202509+0000","last_active":"2023-10-09T05:54:19.203128+0000","last_peered":"2023-10-09T05:54:19.203128+0000","last_clean":"2023-10-09T05:54:19.203128+0000","last_became_active":"2023-10-09T05:54:19.202144+0000","last_became_peered":"2023-10-09T05:54:19.202144+0000","last_unstale":"2023-10-09T05:54:19.203128+0000","last_undegraded":"2023-10-09T05:54:19.203128+0000","last_fullsized":"2023-10-09T05:54:19.203128+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":179,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.202922+0000","last_change":"2023-10-09T05:54:16.215036+0000","last_active":"2023-10-09T05:54:19.202922+0000","last_peered":"2023-10-09T05:54:19.202922+0000","last_clean":"2023-10-09T05:54:19.202922+0000","last_became_active":"2023-10-09T05:54:16.212771+0000","last_became_peered":"2023-10-09T05:54:16.212771+0000","last_unstale":"2023-10-09T05:54:19.202922+0000","last_undegraded":"2023-10-09T05:54:19.202922+0000","last_fullsized":"2023-10-09T05:54:19.202922+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:29.278423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":167,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T05:54:16.204702+0000","last_change":"2023-10-09T05:52:11.411584+0000","last_active":"2023-10-09T05:54:16.204702+0000","last_peered":"2023-10-09T05:54:16.204702+0000","last_clean":"2023-10-09T05:54:16.204702+0000","last_became_active":"2023-10-09T05:52:11.402696+0000","last_became_peered":"2023-10-09T05:52:11.402696+0000","last_unstale":"2023-10-09T05:54:16.204702+0000","last_undegraded":"2023-10-09T05:54:16.204702+0000","last_fullsized":"2023-10-09T05:54:16.204702+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:22.747118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":127,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.198726+0000","last_change":"2023-10-09T05:40:12.676587+0000","last_active":"2023-10-09T05:54:19.198726+0000","last_peered":"2023-10-09T05:54:19.198726+0000","last_clean":"2023-10-09T05:54:19.198726+0000","last_became_active":"2023-10-09T05:40:12.676330+0000","last_became_peered":"2023-10-09T05:40:12.676330+0000","last_unstale":"2023-10-09T05:54:19.198726+0000","last_undegraded":"2023-10-09T05:54:19.198726+0000","last_fullsized":"2023-10-09T05:54:19.198726+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:46.659556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"185'6507","reported_seq":6613,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T05:54:35.162526+0000","last_change":"2023-10-09T05:54:00.191578+0000","last_active":"2023-10-09T05:54:35.162526+0000","last_peered":"2023-10-09T05:54:35.162526+0000","last_clean":"2023-10-09T05:54:35.162526+0000","last_became_active":"2023-10-09T05:54:00.184272+0000","last_became_peered":"2023-10-09T05:54:00.184272+0000","last_unstale":"2023-10-09T05:54:35.162526+0000","last_undegraded":"2023-10-09T05:54:35.162526+0000","last_fullsized":"2023-10-09T05:54:35.162526+0000","mapping_epoch":174,"log_start":"107'1488","ondisk_log_start":"107'1488","created":102,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5019,"log_dups_size":0,"ondisk_log_size":5019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:26.489513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370737152,"num_objects":5657,"num_object_clones":0,"num_object_copies":16971,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5657,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6507,"num_write_kb":389248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":108,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T05:54:15.211245+0000","last_change":"2023-10-09T05:53:59.191302+0000","last_active":"2023-10-09T05:54:15.211245+0000","last_peered":"2023-10-09T05:54:15.211245+0000","last_clean":"2023-10-09T05:54:15.211245+0000","last_became_active":"2023-10-09T05:53:59.190911+0000","last_became_peered":"2023-10-09T05:53:59.190911+0000","last_unstale":"2023-10-09T05:54:15.211245+0000","last_undegraded":"2023-10-09T05:54:15.211245+0000","last_fullsized":"2023-10-09T05:54:15.211245+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:12.801201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":198,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.209976+0000","last_change":"2023-10-09T05:52:53.923635+0000","last_active":"2023-10-09T05:54:19.209976+0000","last_peered":"2023-10-09T05:54:19.209976+0000","last_clean":"2023-10-09T05:54:19.209976+0000","last_became_active":"2023-10-09T05:52:53.922868+0000","last_became_peered":"2023-10-09T05:52:53.922868+0000","last_unstale":"2023-10-09T05:54:19.209976+0000","last_undegraded":"2023-10-09T05:54:19.209976+0000","last_fullsized":"2023-10-09T05:54:19.209976+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:32.418426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":175,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.199638+0000","last_change":"2023-10-09T05:52:11.422464+0000","last_active":"2023-10-09T05:54:19.199638+0000","last_peered":"2023-10-09T05:54:19.199638+0000","last_clean":"2023-10-09T05:54:19.199638+0000","last_became_active":"2023-10-09T05:52:11.341825+0000","last_became_peered":"2023-10-09T05:52:11.341825+0000","last_unstale":"2023-10-09T05:54:19.199638+0000","last_undegraded":"2023-10-09T05:54:19.199638+0000","last_fullsized":"2023-10-09T05:54:19.199638+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:48.933107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":181,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.202337+0000","last_change":"2023-10-09T05:52:11.425354+0000","last_active":"2023-10-09T05:54:19.202337+0000","last_peered":"2023-10-09T05:54:19.202337+0000","last_clean":"2023-10-09T05:54:19.202337+0000","last_became_active":"2023-10-09T05:52:11.425006+0000","last_became_peered":"2023-10-09T05:52:11.425006+0000","last_unstale":"2023-10-09T05:54:19.202337+0000","last_undegraded":"2023-10-09T05:54:19.202337+0000","last_fullsized":"2023-10-09T05:54:19.202337+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:07.273992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":130,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.198827+0000","last_change":"2023-10-09T05:40:12.633195+0000","last_active":"2023-10-09T05:54:19.198827+0000","last_peered":"2023-10-09T05:54:19.198827+0000","last_clean":"2023-10-09T05:54:19.198827+0000","last_became_active":"2023-10-09T05:40:12.632942+0000","last_became_peered":"2023-10-09T05:40:12.632942+0000","last_unstale":"2023-10-09T05:54:19.198827+0000","last_undegraded":"2023-10-09T05:54:19.198827+0000","last_fullsized":"2023-10-09T05:54:19.198827+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:04.470685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":178,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.199203+0000","last_change":"2023-10-09T05:51:57.093104+0000","last_active":"2023-10-09T05:54:19.199203+0000","last_peered":"2023-10-09T05:54:19.199203+0000","last_clean":"2023-10-09T05:54:19.199203+0000","last_became_active":"2023-10-09T05:51:57.092810+0000","last_became_peered":"2023-10-09T05:51:57.092810+0000","last_unstale":"2023-10-09T05:54:19.199203+0000","last_undegraded":"2023-10-09T05:54:19.199203+0000","last_fullsized":"2023-10-09T05:54:19.199203+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:51.192227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":73,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.202627+0000","last_change":"2023-10-09T05:54:16.212256+0000","last_active":"2023-10-09T05:54:19.202627+0000","last_peered":"2023-10-09T05:54:19.202627+0000","last_clean":"2023-10-09T05:54:19.202627+0000","last_became_active":"2023-10-09T05:54:16.211851+0000","last_became_peered":"2023-10-09T05:54:16.211851+0000","last_unstale":"2023-10-09T05:54:19.202627+0000","last_undegraded":"2023-10-09T05:54:19.202627+0000","last_fullsized":"2023-10-09T05:54:19.202627+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:32:51.186363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":173,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.192167+0000","last_change":"2023-10-09T05:52:42.261890+0000","last_active":"2023-10-09T05:54:19.192167+0000","last_peered":"2023-10-09T05:54:19.192167+0000","last_clean":"2023-10-09T05:54:19.192167+0000","last_became_active":"2023-10-09T05:52:42.261521+0000","last_became_peered":"2023-10-09T05:52:42.261521+0000","last_unstale":"2023-10-09T05:54:19.192167+0000","last_undegraded":"2023-10-09T05:54:19.192167+0000","last_fullsized":"2023-10-09T05:54:19.192167+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:59.208145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":131,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T05:54:15.212449+0000","last_change":"2023-10-09T05:52:23.654749+0000","last_active":"2023-10-09T05:54:15.212449+0000","last_peered":"2023-10-09T05:54:15.212449+0000","last_clean":"2023-10-09T05:54:15.212449+0000","last_became_active":"2023-10-09T05:52:23.643044+0000","last_became_peered":"2023-10-09T05:52:23.643044+0000","last_unstale":"2023-10-09T05:54:15.212449+0000","last_undegraded":"2023-10-09T05:54:15.212449+0000","last_fullsized":"2023-10-09T05:54:15.212449+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:23:08.898078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":164,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T05:54:19.198703+0000","last_change":"2023-10-09T05:52:11.411838+0000","last_active":"2023-10-09T05:54:19.198703+0000","last_peered":"2023-10-09T05:54:19.198703+0000","last_clean":"2023-10-09T05:54:19.198703+0000","last_became_active":"2023-10-09T05:52:11.411506+0000","last_became_peered":"2023-10-09T05:52:11.411506+0000","last_unstale":"2023-10-09T05:54:19.198703+0000","last_undegraded":"2023-10-09T05:54:19.198703+0000","last_fullsized":"2023-10-09T05:54:19.198703+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:55.135008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":5846794264,"num_objects":89216,"num_object_clones":0,"num_object_copies":267648,"num_objects_missing_on_primary":5218,"num_objects_missing":5218,"num_objects_degraded":30088,"num_objects_misplaced":10687,"num_objects_unfound":0,"num_objects_dirty":89216,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":103866,"num_write_kb":6178561,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27882,"num_bytes_recovered":1804140544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19075837952,"data_stored":19075825736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94896,"ondisk_log_size":94896,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":10238,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2033792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":59,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234179,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2929896,"kb_used_data":2632608,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":90815256,"statfs":{"total":95995035648,"available":92994822144,"internally_reserved":0,"allocated":2695790592,"data_stored":2695108221,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:53:57 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":1.827,"15min":2.0350000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.5960000000000001,"5min":20.247,"15min":20.247},"last":0.37},{"interface":"front","average":{"1min":0.56299999999999994,"5min":1.6910000000000001,"15min":1.879},"min":{"1min":0.26800000000000002,"5min":0.252,"15min":0.252},"max":{"1min":1.5780000000000001,"5min":20.236000000000001,"15min":20.236000000000001},"last":0.26900000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:54:24 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.88100000000000001,"15min":0.64300000000000002},"min":{"1min":0.27200000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.747,"5min":7.9859999999999998,"15min":7.9859999999999998},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.83299999999999996,"15min":0.64500000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.5339999999999998,"5min":5.8129999999999997,"15min":5.8129999999999997},"last":0.35499999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:53:57 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":2.3039999999999998,"15min":1.1180000000000001},"min":{"1min":0.30199999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.1150000000000002,"5min":50.57,"15min":50.57},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.49299999999999999,"5min":2.3559999999999999,"15min":1.1950000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.226,"5min":50.536999999999999,"15min":50.536999999999999},"last":0.439}]},{"osd":3,"last update":"Mon Oct 9 05:54:30 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":5.6689999999999996,"15min":2.2999999999999998},"min":{"1min":0.40300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.4410000000000001,"5min":147.54900000000001,"15min":147.54900000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.53500000000000003,"5min":5.5519999999999996,"15min":2.242},"min":{"1min":0.35799999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.95599999999999996,"5min":144.065,"15min":144.065},"last":0.40500000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:54:04 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.72899999999999998,"15min":0.55500000000000005},"min":{"1min":0.33400000000000002,"5min":0.248,"15min":0.24399999999999999},"max":{"1min":3.7080000000000002,"5min":9.5329999999999995,"15min":9.5329999999999995},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.92200000000000004,"15min":0.63800000000000001},"min":{"1min":0.36699999999999999,"5min":0.29099999999999998,"15min":0.27100000000000002},"max":{"1min":2.5489999999999999,"5min":7.6120000000000001,"15min":7.6120000000000001},"last":0.45200000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:54:04 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.89100000000000001,"15min":0.628},"min":{"1min":0.31900000000000001,"5min":0.31,"15min":0.248},"max":{"1min":1.79,"5min":10.795999999999999,"15min":10.795999999999999},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.60499999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.28100000000000003},"max":{"1min":3.1869999999999998,"5min":3.9380000000000002,"15min":3.9380000000000002},"last":0.46200000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:54:04 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":2.4500000000000002,"15min":1.179},"min":{"1min":0.39100000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.661,"5min":131.94399999999999,"15min":131.94399999999999},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.68400000000000005,"5min":2.5569999999999999,"15min":1.222},"min":{"1min":0.439,"5min":0.35999999999999999,"15min":0.29999999999999999},"max":{"1min":3.2120000000000002,"5min":135.23400000000001,"15min":135.23400000000001},"last":0.54600000000000004}]}]},{"osd":6,"up_from":46,"seq":197568495816,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2979292,"kb_used_data":2669716,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":90765860,"statfs":{"total":95995035648,"available":92944240640,"internally_reserved":0,"allocated":2733789184,"data_stored":2733119244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:53:58 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":1.0049999999999999,"15min":1.109},"min":{"1min":0.25900000000000001,"5min":0.254,"15min":0.254},"max":{"1min":0.66700000000000004,"5min":7.0599999999999996,"15min":7.0599999999999996},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.47199999999999998,"5min":1.0760000000000001,"15min":1.177},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.294,"5min":6.6459999999999999,"15min":6.6459999999999999},"last":0.42899999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:53:31 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":0.81499999999999995,"15min":0.66700000000000004},"min":{"1min":0.29299999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.637,"5min":3.7949999999999999,"15min":3.7949999999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.877,"15min":0.68500000000000005},"min":{"1min":0.30199999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.0649999999999999,"5min":9.9670000000000005,"15min":9.9670000000000005},"last":0.48999999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:53:58 2023","interfaces":[{"interface":"back","average":{"1min":1.258,"5min":1.073,"15min":0.71899999999999997},"min":{"1min":0.33300000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":5.8360000000000003,"5min":6.6859999999999999,"15min":6.6859999999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":1.181,"5min":1.028,"15min":0.71399999999999997},"min":{"1min":0.31,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.3339999999999996,"5min":9.9390000000000001,"15min":9.9390000000000001},"last":0.44900000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:53:31 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":1.1120000000000001,"15min":0.73999999999999999},"min":{"1min":0.32400000000000001,"5min":0.311,"15min":0.311},"max":{"1min":3.915,"5min":10.19,"15min":10.19},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.84699999999999998,"5min":1.141,"15min":0.76500000000000001},"min":{"1min":0.33800000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.3940000000000001,"5min":11.914,"15min":11.914},"last":0.52400000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:53:41 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.93000000000000005,"15min":0.61299999999999999},"min":{"1min":0.28899999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.58799999999999997,"5min":7.9100000000000001,"15min":7.9100000000000001},"last":0.504},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.876,"15min":0.62},"min":{"1min":0.34300000000000003,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.696,"5min":7.3300000000000001,"15min":7.3300000000000001},"last":0.38700000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:53:41 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.91100000000000003,"15min":0.623},"min":{"1min":0.34100000000000003,"5min":0.29799999999999999,"15min":0.27700000000000002},"max":{"1min":0.72999999999999998,"5min":13.879,"15min":13.879},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.97999999999999998,"15min":0.67500000000000004},"min":{"1min":0.35199999999999998,"5min":0.307,"15min":0.307},"max":{"1min":0.78100000000000003,"5min":7.29,"15min":7.29},"last":0.36599999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:54:08 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":1.2450000000000001,"15min":0.77000000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.316},"max":{"1min":0.70299999999999996,"5min":19.914999999999999,"15min":19.914999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.60099999999999998,"5min":1.099,"15min":0.76800000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.35799999999999998},"max":{"1min":1.653,"5min":7.0309999999999997,"15min":7.0309999999999997},"last":0.55900000000000005}]}]},{"osd":1,"up_from":89,"seq":382252089407,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1628108,"kb_used_data":1524356,"kb_used_omap":9,"kb_used_meta":103734,"kb_avail":92117044,"statfs":{"total":95995035648,"available":94327853056,"internally_reserved":0,"allocated":1560940544,"data_stored":1559936019,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":106224617},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:54:00 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.47399999999999998,"15min":0.46999999999999997},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":1.5920000000000001,"5min":2.0299999999999998,"15min":2.0299999999999998},"last":0.437},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.94899999999999995,"15min":1.012},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":1.657,"5min":6.3879999999999999,"15min":6.3879999999999999},"last":0.308}]},{"osd":2,"last update":"Mon Oct 9 05:54:04 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.76100000000000001,"15min":0.67300000000000004},"min":{"1min":0.38800000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":2.9540000000000002,"5min":5.0789999999999997,"15min":5.0789999999999997},"last":0.34200000000000003},{"interface":"front","average":{"1min":1.3919999999999999,"5min":1.177,"15min":0.746},"min":{"1min":0.309,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":10.875,"5min":34.671999999999997,"15min":34.671999999999997},"last":0.45700000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:54:32 2023","interfaces":[{"interface":"back","average":{"1min":11.045999999999999,"5min":6.6100000000000003,"15min":2.472},"min":{"1min":0.35399999999999998,"5min":0.214,"15min":0.214},"max":{"1min":187.928,"5min":284.06099999999998,"15min":284.06099999999998},"last":0.40300000000000002},{"interface":"front","average":{"1min":11.101000000000001,"5min":6.851,"15min":2.5550000000000002},"min":{"1min":0.33500000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":187.899,"5min":284.97300000000001,"15min":284.97300000000001},"last":0.52500000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:54:22 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":1.0109999999999999,"15min":0.73099999999999998},"min":{"1min":0.38600000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.847,"5min":7.5410000000000004,"15min":7.5410000000000004},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.68400000000000005,"15min":0.63600000000000001},"min":{"1min":0.35299999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.284,"5min":3.4220000000000002,"15min":3.4220000000000002},"last":0.50600000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:54:22 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.98299999999999998,"15min":0.72499999999999998},"min":{"1min":0.39200000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.657,"5min":7.4729999999999999,"15min":7.4729999999999999},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.72799999999999998,"5min":1.0609999999999999,"15min":0.76000000000000001},"min":{"1min":0.435,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":2.8180000000000001,"5min":7.4900000000000002,"15min":7.4900000000000002},"last":0.65000000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:54:22 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":3.516,"15min":1.5860000000000001},"min":{"1min":0.36199999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":2.6200000000000001,"5min":204.62700000000001,"15min":204.62700000000001},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.71499999999999997,"5min":3.5950000000000002,"15min":1.6220000000000001},"min":{"1min":0.437,"5min":0.371,"15min":0.371},"max":{"1min":2.7599999999999998,"5min":204.285,"15min":204.285},"last":0.69199999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:54:22 2023","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":1.272,"15min":0.86799999999999999},"min":{"1min":0.40600000000000003,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":2.8730000000000002,"5min":10.669,"15min":10.669},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.93899999999999995,"15min":0.73899999999999999},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.323,"5min":10.619999999999999,"15min":10.619999999999999},"last":0.53900000000000003}]}]},{"osd":0,"up_from":129,"seq":554050781207,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1415408,"kb_used_data":1357928,"kb_used_omap":29,"kb_used_meta":57442,"kb_avail":92329744,"statfs":{"total":95995035648,"available":94545657856,"internally_reserved":0,"allocated":1390518272,"data_stored":1389601915,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30322,"internal_metadata":58821006},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:54:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":0.77300000000000002,"15min":0.72499999999999998},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":9.1720000000000006,"5min":9.1720000000000006,"15min":9.1720000000000006},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.84899999999999998,"5min":1.0880000000000001,"15min":1.1279999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":3.2200000000000002,"5min":4.0049999999999999,"15min":4.0049999999999999},"last":0.38400000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:54:00 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.96599999999999997,"15min":1.014},"min":{"1min":0.29999999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.25,"5min":3.9199999999999999,"15min":3.9199999999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.71299999999999997,"5min":0.94899999999999995,"15min":0.98899999999999999},"min":{"1min":0.309,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.3860000000000001,"5min":4.3860000000000001,"15min":4.3860000000000001},"last":0.51400000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:54:00 2023","interfaces":[{"interface":"back","average":{"1min":0.71799999999999997,"5min":0.76300000000000001,"15min":0.77100000000000002},"min":{"1min":0.315,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.1739999999999999,"5min":3.1930000000000001,"15min":3.1930000000000001},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.82099999999999995,"5min":0.88500000000000001,"15min":0.89500000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.5279999999999996,"5min":4.5279999999999996,"15min":4.5279999999999996},"last":0.48399999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:54:00 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.196,"15min":1.214},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":9.0050000000000008,"5min":9.0050000000000008,"15min":9.0050000000000008},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.72899999999999998,"5min":0.86899999999999999,"15min":0.89200000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":3.0950000000000002,"5min":3.1549999999999998,"15min":3.1549999999999998},"last":0.502}]},{"osd":5,"last update":"Mon Oct 9 05:54:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1479999999999999,"5min":1.1479999999999999,"15min":1.1479999999999999},"min":{"1min":0.44500000000000001,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":7.3959999999999999,"5min":7.3959999999999999,"15min":7.3959999999999999},"last":0.432},{"interface":"front","average":{"1min":1.2210000000000001,"5min":0.97599999999999998,"15min":0.93500000000000005},"min":{"1min":0.45700000000000002,"5min":0.439,"15min":0.439},"max":{"1min":9.1080000000000005,"5min":9.1080000000000005,"15min":9.1080000000000005},"last":0.55000000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:54:00 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":1.2290000000000001,"15min":1.3049999999999999},"min":{"1min":0.40600000000000003,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":3.3239999999999998,"5min":6.6849999999999996,"15min":6.6849999999999996},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.109,"5min":1.202,"15min":1.218},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":9.141,"5min":9.141,"15min":9.141},"last":0.60199999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:54:00 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":1.3899999999999999,"15min":1.4099999999999999},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":9.0700000000000003,"5min":9.0700000000000003,"15min":9.0700000000000003},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.91100000000000003,"5min":1.3069999999999999,"15min":1.373},"min":{"1min":0.41799999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":3.137,"5min":5.8730000000000002,"15min":5.8730000000000002},"last":0.61299999999999999}]}]},{"osd":2,"up_from":94,"seq":403726925895,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3213476,"kb_used_data":3105500,"kb_used_omap":81,"kb_used_meta":107886,"kb_avail":90531676,"statfs":{"total":95995035648,"available":92704436224,"internally_reserved":0,"allocated":3180032000,"data_stored":3179121157,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":110475482},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:53:59 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":1.4410000000000001,"15min":1.6120000000000001},"min":{"1min":0.254,"5min":0.245,"15min":0.245},"max":{"1min":1.7629999999999999,"5min":11.444000000000001,"15min":11.444000000000001},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.54900000000000004,"5min":1.3580000000000001,"15min":1.4930000000000001},"min":{"1min":0.26400000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":3.726,"5min":6.883,"15min":6.883},"last":0.28599999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:53:54 2023","interfaces":[{"interface":"back","average":{"1min":1.3640000000000001,"5min":2.8210000000000002,"15min":1.2270000000000001},"min":{"1min":0.246,"5min":0.218,"15min":0.218},"max":{"1min":7.7119999999999997,"5min":163.24199999999999,"15min":163.24199999999999},"last":0.27700000000000002},{"interface":"front","average":{"1min":0.755,"5min":2.6240000000000001,"15min":1.147},"min":{"1min":0.27600000000000002,"5min":0.214,"15min":0.214},"max":{"1min":3.7749999999999999,"5min":163.27000000000001,"15min":163.27000000000001},"last":0.42999999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:54:30 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":2.9910000000000001,"15min":1.2649999999999999},"min":{"1min":0.27200000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.7080000000000002,"5min":170.07599999999999,"15min":170.07599999999999},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.63100000000000001,"5min":1.6579999999999999,"15min":0.84699999999999998},"min":{"1min":0.32300000000000001,"5min":0.25,"15min":0.25},"max":{"1min":2.0950000000000002,"5min":63.600000000000001,"15min":63.600000000000001},"last":0.61299999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:53:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.153,"15min":0.83499999999999996},"min":{"1min":0.35999999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.9400000000000004,"5min":11.372999999999999,"15min":11.372999999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.79100000000000004,"5min":1.0700000000000001,"15min":0.83099999999999996},"min":{"1min":0.40400000000000003,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.8570000000000002,"5min":6.5709999999999997,"15min":6.5709999999999997},"last":0.47099999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:53:54 2023","interfaces":[{"interface":"back","average":{"1min":1.304,"5min":1.093,"15min":0.81699999999999995},"min":{"1min":0.36099999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":13.414,"5min":13.414,"15min":13.414},"last":0.629},{"interface":"front","average":{"1min":1.4890000000000001,"5min":1.1839999999999999,"15min":0.84199999999999997},"min":{"1min":0.40300000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":13.438000000000001,"5min":13.438000000000001,"15min":13.438000000000001},"last":0.58299999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:53:54 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.1519999999999999,"15min":0.82799999999999996},"min":{"1min":0.39200000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.823,"5min":11.59,"15min":11.59},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.93700000000000006,"15min":0.77300000000000002},"min":{"1min":0.374,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":5.0990000000000002,"5min":6.4710000000000001,"15min":6.4710000000000001},"last":0.65400000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:53:54 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":1.3300000000000001,"15min":0.89200000000000002},"min":{"1min":0.42199999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.1710000000000003,"5min":40.404000000000003,"15min":40.404000000000003},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.96599999999999997,"5min":1.371,"15min":0.92900000000000005},"min":{"1min":0.46700000000000003,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":7.0309999999999997,"5min":40.113,"15min":40.113},"last":0.64100000000000001}]}]},{"osd":3,"up_from":99,"seq":425201762369,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3561728,"kb_used_data":3453880,"kb_used_omap":7,"kb_used_meta":107832,"kb_avail":90183424,"statfs":{"total":95995035648,"available":92347826176,"internally_reserved":0,"allocated":3536773120,"data_stored":3535776459,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":110420584},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":1.4630000000000001,"15min":1.5189999999999999},"min":{"1min":0.23699999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":12.548,"5min":15.536,"15min":15.536},"last":0.20399999999999999},{"interface":"front","average":{"1min":0.66400000000000003,"5min":1.407,"15min":1.5309999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.109,"5min":11.210000000000001,"15min":11.210000000000001},"last":0.92900000000000005}]},{"osd":1,"last update":"Mon Oct 9 05:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":2.0760000000000001,"15min":0.93799999999999994},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":2.4060000000000001,"5min":127.226,"15min":127.226},"last":0.31},{"interface":"front","average":{"1min":0.56699999999999995,"5min":2.1859999999999999,"15min":0.99399999999999999},"min":{"1min":0.28199999999999997,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":1.3500000000000001,"5min":127.468,"15min":127.468},"last":0.24399999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":1.347,"15min":0.69899999999999995},"min":{"1min":0.252,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":2.5150000000000001,"5min":62.938000000000002,"15min":62.938000000000002},"last":0.372},{"interface":"front","average":{"1min":0.48199999999999998,"5min":1.677,"15min":0.84699999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.208,"5min":62.43,"15min":62.43},"last":0.432}]},{"osd":4,"last update":"Mon Oct 9 05:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.93400000000000005,"15min":0.71199999999999997},"min":{"1min":0.35599999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.3029999999999999,"5min":7.7439999999999998,"15min":7.7439999999999998},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.629,"5min":0.81499999999999995,"15min":0.68300000000000005},"min":{"1min":0.312,"5min":0.311,"15min":0.311},"max":{"1min":2.4420000000000002,"5min":6.2000000000000002,"15min":6.2000000000000002},"last":0.41199999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.85399999999999998,"15min":0.68600000000000005},"min":{"1min":0.39900000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":2.4740000000000002,"5min":5.5860000000000003,"15min":5.5860000000000003},"last":0.442},{"interface":"front","average":{"1min":0.626,"5min":0.82899999999999996,"15min":0.70099999999999996},"min":{"1min":0.34799999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.5680000000000001,"5min":4.2839999999999998,"15min":4.2839999999999998},"last":2.29}]},{"osd":6,"last update":"Mon Oct 9 05:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.97199999999999998,"15min":0.746},"min":{"1min":0.42299999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.5059999999999998,"5min":12.798999999999999,"15min":12.798999999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.68700000000000006,"5min":1.006,"15min":0.78800000000000003},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":2.6059999999999999,"5min":12.773,"15min":12.773},"last":0.58499999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.98899999999999999,"15min":0.749},"min":{"1min":0.442,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":2.6419999999999999,"5min":7.4580000000000002,"15min":7.4580000000000002},"last":2.4510000000000001},{"interface":"front","average":{"1min":0.62,"5min":0.98999999999999999,"15min":0.76800000000000002},"min":{"1min":0.42699999999999999,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.276,"5min":7.8399999999999999,"15min":7.8399999999999999},"last":2.4089999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019089,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2088296,"kb_used_data":1782816,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":91656856,"statfs":{"total":95995035648,"available":93856620544,"internally_reserved":0,"allocated":1825603584,"data_stored":1824921466,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.581,"5min":0.94099999999999995,"15min":0.83499999999999996},"min":{"1min":0.27400000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":15.173999999999999,"5min":15.173999999999999,"15min":15.173999999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.319,"5min":1.4910000000000001,"15min":1.5189999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":15.125999999999999,"5min":15.125999999999999,"15min":15.125999999999999},"last":0.40500000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:54:31 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":1.042,"15min":0.72499999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.512,"5min":9.2390000000000008,"15min":9.2390000000000008},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.91400000000000003,"15min":0.65000000000000002},"min":{"1min":0.27400000000000002,"5min":0.255,"15min":0.255},"max":{"1min":3.556,"5min":8.2629999999999999,"15min":8.2629999999999999},"last":0.626}]},{"osd":2,"last update":"Mon Oct 9 05:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.3819999999999999,"15min":0.84499999999999997},"min":{"1min":0.33500000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.5710000000000002,"5min":33.262,"15min":33.262},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.411,"15min":0.92500000000000004},"min":{"1min":0.35599999999999998,"5min":0.313,"15min":0.313},"max":{"1min":3.2850000000000001,"5min":33.240000000000002,"15min":33.240000000000002},"last":0.46200000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:54:31 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":1.4430000000000001,"15min":0.873},"min":{"1min":0.34799999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.60299999999999998,"5min":40.531999999999996,"15min":40.531999999999996},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.605,"15min":0.93500000000000005},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":4.827,"5min":39.905999999999999,"15min":39.905999999999999},"last":0.441}]},{"osd":5,"last update":"Mon Oct 9 05:54:21 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.81100000000000005,"15min":0.58599999999999997},"min":{"1min":0.27500000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.503,"5min":10.012,"15min":10.012},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.42899999999999999,"5min":1.0009999999999999,"15min":0.67900000000000005},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.61399999999999999,"5min":12.489000000000001,"15min":12.489000000000001},"last":0.53000000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:53:36 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.99299999999999999,"15min":0.66900000000000004},"min":{"1min":0.30499999999999999,"5min":0.29799999999999999,"15min":0.27400000000000002},"max":{"1min":3.1429999999999998,"5min":11.234999999999999,"15min":11.234999999999999},"last":0.503},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.749,"15min":0.61399999999999999},"min":{"1min":0.33300000000000002,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":1.8140000000000001,"5min":9.923,"15min":9.923},"last":0.51300000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:54:07 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":1.179,"15min":0.77700000000000002},"min":{"1min":0.33200000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.0209999999999999,"5min":10.606,"15min":10.606},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.90000000000000002,"15min":0.68999999999999995},"min":{"1min":0.35999999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.75900000000000001,"5min":8.3819999999999997,"15min":8.3819999999999997},"last":0.59299999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757452,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2401952,"kb_used_data":2104664,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":91343200,"statfs":{"total":95995035648,"available":93535436800,"internally_reserved":0,"allocated":2155175936,"data_stored":2154498322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:54:01 2023","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.64800000000000002,"15min":0.68700000000000006},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.877,"5min":2.7869999999999999,"15min":2.7869999999999999},"last":0.441},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.73999999999999999,"15min":0.77800000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":1.3200000000000001,"5min":3.4140000000000001,"15min":3.4140000000000001},"last":0.54300000000000004}]},{"osd":1,"last update":"Mon Oct 9 05:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":3.5019999999999998,"15min":1.5249999999999999},"min":{"1min":0.308,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.1269999999999998,"5min":114.247,"15min":114.247},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.49399999999999999,"5min":3.3610000000000002,"15min":1.4950000000000001},"min":{"1min":0.30099999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.85099999999999998,"5min":114.678,"15min":114.678},"last":0.40400000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:53:53 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":2.2959999999999998,"15min":1.1559999999999999},"min":{"1min":0.33000000000000002,"5min":0.25,"15min":0.25},"max":{"1min":4.6920000000000002,"5min":106.66500000000001,"15min":106.66500000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.76100000000000001,"5min":2.274,"15min":1.1499999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.077,"5min":106.779,"15min":106.779},"last":0.48699999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:54:25 2023","interfaces":[{"interface":"back","average":{"1min":5.7779999999999996,"5min":8.9489999999999998,"15min":3.3439999999999999},"min":{"1min":0.32600000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":89.256,"5min":272.81299999999999,"15min":272.81299999999999},"last":0.45900000000000002},{"interface":"front","average":{"1min":6.1929999999999996,"5min":9.3379999999999992,"15min":3.5409999999999999},"min":{"1min":0.33200000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":89.427000000000007,"5min":273.00999999999999,"15min":273.00999999999999},"last":0.46899999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:54:17 2023","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.70899999999999996,"15min":0.55700000000000005},"min":{"1min":0.26400000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.68200000000000005,"5min":6.2050000000000001,"15min":6.2050000000000001},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.68600000000000005,"15min":0.56699999999999995},"min":{"1min":0.31,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.51600000000000001,"5min":13.050000000000001,"15min":13.050000000000001},"last":0.39000000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:54:32 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.69799999999999995,"15min":0.57299999999999995},"min":{"1min":0.309,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.85099999999999998,"5min":10.606999999999999,"15min":10.606999999999999},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.624,"5min":0.874,"15min":0.66200000000000003},"min":{"1min":0.307,"5min":0.30299999999999999,"15min":0.28499999999999998},"max":{"1min":3.173,"5min":5.9770000000000003,"15min":5.9770000000000003},"last":0.307}]},{"osd":7,"last update":"Mon Oct 9 05:54:11 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":1.02,"15min":0.72199999999999998},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.315},"max":{"1min":0.66200000000000003,"5min":9.2739999999999991,"15min":9.2739999999999991},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.5,"5min":0.72299999999999998,"15min":0.61899999999999999},"min":{"1min":0.32100000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.76400000000000001,"5min":5.032,"15min":5.032},"last":0.70599999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1474,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1388384256,"data_stored":1388380184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1568673792,"data_stored":1568669720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3177840640,"data_stored":3177840640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3533832192,"data_stored":3533832192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1823670272,"data_stored":1823670272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2152665088,"data_stored":2152661016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2737963008,"data_stored":2737963008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2692808704,"data_stored":2692808704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:54:39.315 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:54:39.316 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_0 pg_num 26 2023-10-09T05:54:39.326 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:39.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:39.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:39.430 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:39.462 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:39.520 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:39.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:39.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:39.578 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:54:39.624 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:39.629 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:54:39.671 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:54:39.685 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:39.704 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:39 smithi156 ceph-mon[195101]: from='client.34989 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:54:39.704 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:39 smithi156 ceph-mon[195101]: pgmap v429: 109 pgs: 1 active+undersized+degraded+remapped+laggy, 2 active+recovering+undersized+remapped, 1 activating+degraded, 1 peering, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 30090/267720 objects degraded (11.239%); 10687/267720 objects misplaced (3.992%); 9.3 MiB/s, 150 objects/s recovering 2023-10-09T05:54:39.705 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:54:39.749 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:54:39.783 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:54:39.797 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:39.799 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:39 smithi064 ceph-mon[212750]: from='client.34989 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:54:39.799 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:39 smithi064 ceph-mon[212750]: pgmap v429: 109 pgs: 1 active+undersized+degraded+remapped+laggy, 2 active+recovering+undersized+remapped, 1 activating+degraded, 1 peering, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 30090/267720 objects degraded (11.239%); 10687/267720 objects misplaced (3.992%); 9.3 MiB/s, 150 objects/s recovering 2023-10-09T05:54:39.799 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:39 smithi064 ceph-mon[220044]: from='client.34989 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:54:39.799 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:39 smithi064 ceph-mon[220044]: pgmap v429: 109 pgs: 1 active+undersized+degraded+remapped+laggy, 2 active+recovering+undersized+remapped, 1 activating+degraded, 1 peering, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 95 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 30090/267720 objects degraded (11.239%); 10687/267720 objects misplaced (3.992%); 9.3 MiB/s, 150 objects/s recovering 2023-10-09T05:54:39.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:39.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:39.818 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:54:39.842 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:39.844 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:54:39.872 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:54:39.874 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:54:39.874 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:54:39.874 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:54:39.874 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:54:39.874 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:54:39.875 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:54:39.875 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:54:39.875 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:54:39.901 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:39.945 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:40.260 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:40.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:40.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:40.365 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:40.398 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:40.450 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:40.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:40.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:40.553 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:40.587 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:40.646 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:40.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:40.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:40.751 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:40.792 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:41.074 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:41.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:41.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:41.181 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:41.213 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:41.261 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:54:41.261 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T05:54:41.261 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T05:54:41.261 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T05:54:41.261 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.261 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:45183 6.5 6:aac00114:::benchmark_data_smithi064_284193_object45182:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T05:54:41.262 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:13.801557+0000", 2023-10-09T05:54:41.262 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 207.45531818399999, 2023-10-09T05:54:41.262 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.33244016100000001, 2023-10-09T05:54:41.262 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:41.262 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:41.262 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:41.262 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:41.262 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:41.262 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 45183 2023-10-09T05:54:41.262 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.262 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:41.263 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.263 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:41.263 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801557+0000", 2023-10-09T05:54:41.263 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.263 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.263 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.263 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:41.263 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801557+0000", 2023-10-09T05:54:41.263 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.263 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.263 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.263 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:41.264 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801555+0000", 2023-10-09T05:54:41.264 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:41.264 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.264 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.264 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:41.264 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801742+0000", 2023-10-09T05:54:41.264 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000186954 2023-10-09T05:54:41.264 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.264 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.264 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:41.265 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801755+0000", 2023-10-09T05:54:41.265 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2575000000000001e-05 2023-10-09T05:54:41.265 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.265 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.265 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:41.265 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801762+0000", 2023-10-09T05:54:41.265 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.827e-06 2023-10-09T05:54:41.265 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.265 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.265 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.265 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801828+0000", 2023-10-09T05:54:41.266 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6237999999999998e-05 2023-10-09T05:54:41.266 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.266 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.266 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:41.266 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801958+0000", 2023-10-09T05:54:41.266 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013046299999999999 2023-10-09T05:54:41.266 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.266 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.266 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:54:41.266 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.802104+0000", 2023-10-09T05:54:41.266 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000145465 2023-10-09T05:54:41.267 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.267 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.267 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.267 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.804037+0000", 2023-10-09T05:54:41.267 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001933113 2023-10-09T05:54:41.267 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.267 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.267 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:41.267 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.805890+0000", 2023-10-09T05:54:41.267 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00185311 2023-10-09T05:54:41.267 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.267 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.268 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.268 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:14.133916+0000", 2023-10-09T05:54:41.268 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.32802647899999998 2023-10-09T05:54:41.268 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.268 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.268 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:41.268 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:14.133958+0000", 2023-10-09T05:54:41.268 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1396999999999997e-05 2023-10-09T05:54:41.268 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.268 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.268 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:41.268 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:14.133997+0000", 2023-10-09T05:54:41.269 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9158e-05 2023-10-09T05:54:41.269 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.269 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:41.269 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.269 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.269 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.269 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:45190 6.8 6:1669d52f:::benchmark_data_smithi064_284193_object45189:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T05:54:41.269 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:13.804266+0000", 2023-10-09T05:54:41.269 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 207.45260907100001, 2023-10-09T05:54:41.269 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.32912121900000002, 2023-10-09T05:54:41.269 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:41.269 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:41.270 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:41.270 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:41.270 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:41.270 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 45190 2023-10-09T05:54:41.270 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.270 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:41.270 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.270 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:41.270 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.804266+0000", 2023-10-09T05:54:41.270 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.270 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.271 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.271 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:41.271 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.804266+0000", 2023-10-09T05:54:41.271 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.271 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.271 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.271 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:41.271 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.804264+0000", 2023-10-09T05:54:41.271 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:41.271 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.271 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.272 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:41.272 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.804311+0000", 2023-10-09T05:54:41.272 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6177999999999997e-05 2023-10-09T05:54:41.272 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.272 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.272 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:41.272 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.804328+0000", 2023-10-09T05:54:41.272 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6969e-05 2023-10-09T05:54:41.272 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.272 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.272 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:41.272 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.804334+0000", 2023-10-09T05:54:41.273 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1580000000000002e-06 2023-10-09T05:54:41.273 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.273 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.273 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.273 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.805072+0000", 2023-10-09T05:54:41.273 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00073821900000000003 2023-10-09T05:54:41.273 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.273 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.273 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:41.273 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.805186+0000", 2023-10-09T05:54:41.273 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011408099999999999 2023-10-09T05:54:41.273 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.274 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.274 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:54:41.274 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.805312+0000", 2023-10-09T05:54:41.274 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0001259 2023-10-09T05:54:41.274 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.274 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.274 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:41.274 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.806929+0000", 2023-10-09T05:54:41.275 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0016169839999999999 2023-10-09T05:54:41.275 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.275 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.275 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.275 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.808641+0000", 2023-10-09T05:54:41.275 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0017123469999999999 2023-10-09T05:54:41.275 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.275 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.276 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.276 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:14.133307+0000", 2023-10-09T05:54:41.276 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.32466545299999999 2023-10-09T05:54:41.276 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.276 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.276 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:41.276 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:14.133342+0000", 2023-10-09T05:54:41.276 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5429999999999998e-05 2023-10-09T05:54:41.276 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.276 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.276 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:41.276 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:14.133387+0000", 2023-10-09T05:54:41.277 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5037000000000003e-05 2023-10-09T05:54:41.277 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.277 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:41.277 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.277 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.277 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.277 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64016 6.9 6:964728b5:::benchmark_data_smithi064_284193_object64015:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:54:41.277 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.109619+0000", 2023-10-09T05:54:41.277 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 188.147255978, 2023-10-09T05:54:41.277 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0912698940000001, 2023-10-09T05:54:41.277 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:41.277 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:41.278 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:41.278 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:41.278 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:41.278 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64016 2023-10-09T05:54:41.278 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.278 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:41.278 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.278 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:41.278 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.109619+0000", 2023-10-09T05:54:41.278 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.278 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.279 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.279 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:41.279 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.109619+0000", 2023-10-09T05:54:41.279 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.279 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.279 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.279 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:41.279 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.109619+0000", 2023-10-09T05:54:41.279 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:54:41.279 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.279 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.279 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:41.280 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.109650+0000", 2023-10-09T05:54:41.280 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1087999999999998e-05 2023-10-09T05:54:41.280 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.280 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.280 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:41.280 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.109657+0000", 2023-10-09T05:54:41.280 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4290000000000004e-06 2023-10-09T05:54:41.280 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.280 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.280 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:41.280 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.109662+0000", 2023-10-09T05:54:41.280 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4510000000000002e-06 2023-10-09T05:54:41.281 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.281 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.281 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.281 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.115072+0000", 2023-10-09T05:54:41.281 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0054107180000000001 2023-10-09T05:54:41.281 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.281 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.281 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:41.281 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.115080+0000", 2023-10-09T05:54:41.281 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3710000000000002e-06 2023-10-09T05:54:41.281 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.281 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.282 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.282 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.189454+0000", 2023-10-09T05:54:41.282 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.074374384 2023-10-09T05:54:41.282 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.282 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.282 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:41.282 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.190617+0000", 2023-10-09T05:54:41.282 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0011633150000000001 2023-10-09T05:54:41.282 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.282 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.282 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2", 2023-10-09T05:54:41.283 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.190779+0000", 2023-10-09T05:54:41.283 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00016118799999999999 2023-10-09T05:54:41.283 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.283 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.283 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:41.283 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.198531+0000", 2023-10-09T05:54:41.283 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0077525600000000004 2023-10-09T05:54:41.283 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.283 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.283 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.283 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200740+0000", 2023-10-09T05:54:41.283 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002208785 2023-10-09T05:54:41.284 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.284 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.284 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:41.284 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200795+0000", 2023-10-09T05:54:41.284 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5225000000000001e-05 2023-10-09T05:54:41.284 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.284 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.284 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:41.284 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200889+0000", 2023-10-09T05:54:41.284 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3837000000000003e-05 2023-10-09T05:54:41.284 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.284 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:41.284 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.285 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.285 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.285 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64021 6.9 6:9b197d79:::benchmark_data_smithi064_284193_object64020:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:54:41.285 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.123178+0000", 2023-10-09T05:54:41.285 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 188.133697195, 2023-10-09T05:54:41.285 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0778894430000001, 2023-10-09T05:54:41.285 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:41.285 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:41.285 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:41.285 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:41.285 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:41.286 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64021 2023-10-09T05:54:41.286 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.286 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:41.286 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.286 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:41.286 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.123178+0000", 2023-10-09T05:54:41.286 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.286 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.286 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.287 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:41.287 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.123178+0000", 2023-10-09T05:54:41.287 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.287 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.287 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.287 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:41.287 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.123177+0000", 2023-10-09T05:54:41.287 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:41.287 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.287 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.287 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:41.287 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.123212+0000", 2023-10-09T05:54:41.288 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5641e-05 2023-10-09T05:54:41.288 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.288 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.288 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:41.288 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.123221+0000", 2023-10-09T05:54:41.288 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0909999999999999e-06 2023-10-09T05:54:41.288 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.288 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.288 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:41.288 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.123227+0000", 2023-10-09T05:54:41.288 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4469999999999997e-06 2023-10-09T05:54:41.288 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.289 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.289 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.289 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.124899+0000", 2023-10-09T05:54:41.289 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001671797 2023-10-09T05:54:41.289 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.289 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.289 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:41.289 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.124905+0000", 2023-10-09T05:54:41.289 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8650000000000003e-06 2023-10-09T05:54:41.289 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.289 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.289 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.290 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.192517+0000", 2023-10-09T05:54:41.290 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0676111340000001 2023-10-09T05:54:41.290 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.290 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.290 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:41.290 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.192607+0000", 2023-10-09T05:54:41.290 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0493000000000002e-05 2023-10-09T05:54:41.290 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.291 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.291 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2", 2023-10-09T05:54:41.291 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.192721+0000", 2023-10-09T05:54:41.291 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000114003 2023-10-09T05:54:41.291 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.291 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.291 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:41.291 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200480+0000", 2023-10-09T05:54:41.291 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0077586089999999996 2023-10-09T05:54:41.291 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.292 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.292 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.292 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200926+0000", 2023-10-09T05:54:41.292 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00044662899999999998 2023-10-09T05:54:41.292 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.292 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.292 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:41.292 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200976+0000", 2023-10-09T05:54:41.293 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9607999999999997e-05 2023-10-09T05:54:41.293 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.293 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.293 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:41.293 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.201067+0000", 2023-10-09T05:54:41.293 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1414000000000003e-05 2023-10-09T05:54:41.294 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.294 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:41.294 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.294 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.294 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.295 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64026 6.9 6:95eb16a1:::benchmark_data_smithi064_284193_object64025:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:54:41.295 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.131052+0000", 2023-10-09T05:54:41.295 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 188.12582273300001, 2023-10-09T05:54:41.295 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0701834699999999, 2023-10-09T05:54:41.295 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:41.295 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:41.295 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:41.295 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:41.296 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:41.296 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64026 2023-10-09T05:54:41.296 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.296 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:41.296 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.296 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:41.296 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.131052+0000", 2023-10-09T05:54:41.296 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.297 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.297 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.297 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:41.297 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.131052+0000", 2023-10-09T05:54:41.297 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.297 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.297 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.298 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:41.298 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.131051+0000", 2023-10-09T05:54:41.298 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:41.298 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.298 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.298 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:41.298 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.131088+0000", 2023-10-09T05:54:41.299 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6916999999999999e-05 2023-10-09T05:54:41.299 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.299 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.299 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:41.299 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.131101+0000", 2023-10-09T05:54:41.299 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2877999999999999e-05 2023-10-09T05:54:41.299 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.300 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.300 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:41.300 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.131107+0000", 2023-10-09T05:54:41.300 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5899999999999996e-06 2023-10-09T05:54:41.300 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.300 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.300 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.300 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.131156+0000", 2023-10-09T05:54:41.300 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8696000000000003e-05 2023-10-09T05:54:41.300 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.300 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.301 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:41.301 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.131164+0000", 2023-10-09T05:54:41.301 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9340000000000006e-06 2023-10-09T05:54:41.301 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.301 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.301 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.301 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.194260+0000", 2023-10-09T05:54:41.301 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.063095543 2023-10-09T05:54:41.302 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.302 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.302 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:41.302 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.194404+0000", 2023-10-09T05:54:41.302 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000144058 2023-10-09T05:54:41.302 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.302 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.302 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2", 2023-10-09T05:54:41.302 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.194588+0000", 2023-10-09T05:54:41.302 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00018420399999999999 2023-10-09T05:54:41.302 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.303 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.303 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:41.303 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200488+0000", 2023-10-09T05:54:41.303 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0059001640000000003 2023-10-09T05:54:41.303 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.303 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.303 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.303 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.201102+0000", 2023-10-09T05:54:41.303 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00061416900000000002 2023-10-09T05:54:41.303 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.304 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.304 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:41.304 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.201150+0000", 2023-10-09T05:54:41.304 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7922e-05 2023-10-09T05:54:41.304 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.304 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.304 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:41.304 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.201236+0000", 2023-10-09T05:54:41.304 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.5679000000000004e-05 2023-10-09T05:54:41.304 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.304 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:41.304 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.304 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.305 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.305 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64027 6.9 6:98f248d6:::benchmark_data_smithi064_284193_object64026:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:54:41.305 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.133112+0000", 2023-10-09T05:54:41.305 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 188.123763427, 2023-10-09T05:54:41.305 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0717281329999999, 2023-10-09T05:54:41.305 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:41.305 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:41.305 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:41.305 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:41.306 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:41.306 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64027 2023-10-09T05:54:41.306 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.306 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:41.306 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.306 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:41.306 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133112+0000", 2023-10-09T05:54:41.306 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.306 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.306 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.307 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:41.307 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133112+0000", 2023-10-09T05:54:41.307 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.307 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.307 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.307 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:41.307 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133110+0000", 2023-10-09T05:54:41.307 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:54:41.307 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.307 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.307 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:41.307 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133165+0000", 2023-10-09T05:54:41.308 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4601999999999997e-05 2023-10-09T05:54:41.308 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.308 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.308 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:41.308 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133174+0000", 2023-10-09T05:54:41.308 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6899999999999998e-06 2023-10-09T05:54:41.308 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.308 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.308 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:41.308 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133179+0000", 2023-10-09T05:54:41.308 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9509999999999997e-06 2023-10-09T05:54:41.309 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.309 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.309 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.309 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133222+0000", 2023-10-09T05:54:41.309 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3266000000000002e-05 2023-10-09T05:54:41.309 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.309 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.309 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:41.309 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133229+0000", 2023-10-09T05:54:41.309 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.486e-06 2023-10-09T05:54:41.309 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.309 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.309 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.310 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.196460+0000", 2023-10-09T05:54:41.310 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0632303270000001 2023-10-09T05:54:41.310 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.310 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.310 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:41.310 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.196575+0000", 2023-10-09T05:54:41.310 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000114918 2023-10-09T05:54:41.311 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.311 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.311 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2", 2023-10-09T05:54:41.311 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.196709+0000", 2023-10-09T05:54:41.311 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013413100000000001 2023-10-09T05:54:41.311 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.311 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.311 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.311 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.203829+0000", 2023-10-09T05:54:41.311 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0071203179999999996 2023-10-09T05:54:41.311 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.312 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.312 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:41.312 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204716+0000", 2023-10-09T05:54:41.312 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00088727100000000002 2023-10-09T05:54:41.312 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.312 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.312 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:41.312 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204766+0000", 2023-10-09T05:54:41.312 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9855999999999998e-05 2023-10-09T05:54:41.313 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.313 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.313 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:41.313 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204840+0000", 2023-10-09T05:54:41.313 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3491999999999994e-05 2023-10-09T05:54:41.313 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.313 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:41.313 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.313 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.313 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.314 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64047 6.9 6:98194c7b:::benchmark_data_smithi064_284193_object64046:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:54:41.314 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.155174+0000", 2023-10-09T05:54:41.314 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 188.101701327, 2023-10-09T05:54:41.314 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0497767680000001, 2023-10-09T05:54:41.314 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:41.314 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:41.314 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:41.314 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:41.314 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:41.314 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64047 2023-10-09T05:54:41.314 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.315 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:41.315 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.315 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:41.315 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.155174+0000", 2023-10-09T05:54:41.315 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.315 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.315 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.315 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:41.315 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.155174+0000", 2023-10-09T05:54:41.315 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.315 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.316 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.316 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:41.316 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.155172+0000", 2023-10-09T05:54:41.316 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:41.316 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.316 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.316 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:41.316 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.155288+0000", 2023-10-09T05:54:41.316 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011592299999999999 2023-10-09T05:54:41.316 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.316 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.316 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:41.317 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.155304+0000", 2023-10-09T05:54:41.317 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6515999999999999e-05 2023-10-09T05:54:41.317 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.317 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.317 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:41.317 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.155311+0000", 2023-10-09T05:54:41.317 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5470000000000002e-06 2023-10-09T05:54:41.317 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.317 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.318 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.318 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.155364+0000", 2023-10-09T05:54:41.318 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3075000000000003e-05 2023-10-09T05:54:41.318 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.318 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.318 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:54:41.318 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.155375+0000", 2023-10-09T05:54:41.318 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0631e-05 2023-10-09T05:54:41.318 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.318 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.318 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.318 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.198594+0000", 2023-10-09T05:54:41.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0432188309999999 2023-10-09T05:54:41.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.319 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.319 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:41.319 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.198721+0000", 2023-10-09T05:54:41.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000127398 2023-10-09T05:54:41.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.319 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.319 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2", 2023-10-09T05:54:41.319 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.198812+0000", 2023-10-09T05:54:41.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1410999999999996e-05 2023-10-09T05:54:41.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.319 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.320 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.321 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.203873+0000", 2023-10-09T05:54:41.321 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0050606719999999996 2023-10-09T05:54:41.321 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.321 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.322 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:41.322 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204849+0000", 2023-10-09T05:54:41.322 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00097625800000000003 2023-10-09T05:54:41.322 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.322 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.322 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:41.322 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204887+0000", 2023-10-09T05:54:41.322 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7339999999999998e-05 2023-10-09T05:54:41.323 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.323 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.323 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:41.323 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204950+0000", 2023-10-09T05:54:41.323 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3805000000000003e-05 2023-10-09T05:54:41.323 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.323 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:41.323 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.323 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.323 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.323 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:77504 6.5 6:af5f1d5a:::benchmark_data_smithi064_284193_object77503:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:54:41.323 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:49.602465+0000", 2023-10-09T05:54:41.324 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 171.65441021399999, 2023-10-09T05:54:41.324 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.36206133400000001, 2023-10-09T05:54:41.324 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:41.324 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:41.324 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:41.324 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:41.324 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:41.324 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 77504 2023-10-09T05:54:41.324 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.325 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:41.325 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.325 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:41.325 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602465+0000", 2023-10-09T05:54:41.325 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.325 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.325 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.325 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:41.325 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602465+0000", 2023-10-09T05:54:41.325 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.325 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.326 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.326 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:41.326 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602463+0000", 2023-10-09T05:54:41.326 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:41.326 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.326 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.326 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:41.326 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602503+0000", 2023-10-09T05:54:41.326 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0176000000000003e-05 2023-10-09T05:54:41.326 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.326 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.326 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:41.327 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602519+0000", 2023-10-09T05:54:41.327 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5979999999999999e-05 2023-10-09T05:54:41.327 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.327 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.327 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:41.327 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602534+0000", 2023-10-09T05:54:41.327 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4303e-05 2023-10-09T05:54:41.327 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.327 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.327 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.327 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602595+0000", 2023-10-09T05:54:41.327 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1381000000000001e-05 2023-10-09T05:54:41.328 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.328 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.328 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:41.328 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602714+0000", 2023-10-09T05:54:41.328 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000119098 2023-10-09T05:54:41.328 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.328 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.328 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:54:41.328 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602851+0000", 2023-10-09T05:54:41.328 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000137093 2023-10-09T05:54:41.329 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.329 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.329 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:41.329 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.605415+0000", 2023-10-09T05:54:41.329 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002564055 2023-10-09T05:54:41.329 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.329 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.329 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.329 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.607058+0000", 2023-10-09T05:54:41.329 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0016429680000000001 2023-10-09T05:54:41.329 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.329 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.330 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.330 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964454+0000", 2023-10-09T05:54:41.330 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.35739558999999999 2023-10-09T05:54:41.330 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.330 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.330 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:41.330 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964494+0000", 2023-10-09T05:54:41.330 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0250000000000003e-05 2023-10-09T05:54:41.330 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.330 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.330 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:41.330 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964526+0000", 2023-10-09T05:54:41.331 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1959999999999999e-05 2023-10-09T05:54:41.331 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.331 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:41.331 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.331 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.331 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.331 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:77509 6.5 6:ae2c471a:::benchmark_data_smithi064_284193_object77508:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:54:41.331 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:49.610882+0000", 2023-10-09T05:54:41.331 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 171.64599297500001, 2023-10-09T05:54:41.331 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.353839925, 2023-10-09T05:54:41.331 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:41.331 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:41.332 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:41.332 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:41.332 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:41.332 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 77509 2023-10-09T05:54:41.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.332 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:41.332 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.332 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:41.332 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.610882+0000", 2023-10-09T05:54:41.332 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.333 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.333 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:41.333 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.610882+0000", 2023-10-09T05:54:41.333 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.333 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.333 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.333 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:41.334 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.610881+0000", 2023-10-09T05:54:41.334 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:41.334 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.334 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.334 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:41.334 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.610960+0000", 2023-10-09T05:54:41.334 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9631999999999999e-05 2023-10-09T05:54:41.334 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.334 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.334 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:41.335 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.610979+0000", 2023-10-09T05:54:41.335 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9011000000000001e-05 2023-10-09T05:54:41.335 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.335 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.335 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:41.335 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.610987+0000", 2023-10-09T05:54:41.335 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3289999999999996e-06 2023-10-09T05:54:41.335 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.335 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.335 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.336 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.611048+0000", 2023-10-09T05:54:41.336 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0708999999999998e-05 2023-10-09T05:54:41.336 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.336 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.336 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:41.336 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.611203+0000", 2023-10-09T05:54:41.336 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015526900000000001 2023-10-09T05:54:41.336 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.336 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.336 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:54:41.337 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.611355+0000", 2023-10-09T05:54:41.337 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015201200000000001 2023-10-09T05:54:41.337 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.337 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.337 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:41.337 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.613876+0000", 2023-10-09T05:54:41.337 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0025209579999999998 2023-10-09T05:54:41.337 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.337 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.337 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.337 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.615859+0000", 2023-10-09T05:54:41.337 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001983744 2023-10-09T05:54:41.338 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.338 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.338 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.338 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964635+0000", 2023-10-09T05:54:41.338 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.34877526800000003 2023-10-09T05:54:41.338 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.338 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.338 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:41.338 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964685+0000", 2023-10-09T05:54:41.338 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0586e-05 2023-10-09T05:54:41.338 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.339 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.339 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:41.339 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964722+0000", 2023-10-09T05:54:41.339 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6636e-05 2023-10-09T05:54:41.339 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.339 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:41.339 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.339 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.339 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.339 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:77517 6.5 6:a52e5624:::benchmark_data_smithi064_284193_object77516:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:54:41.339 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:49.620459+0000", 2023-10-09T05:54:41.339 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 171.636416177, 2023-10-09T05:54:41.339 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.34437174100000001, 2023-10-09T05:54:41.340 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:41.340 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:41.340 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:41.340 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:41.340 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:41.340 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 77517 2023-10-09T05:54:41.340 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.340 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:41.340 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.340 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:41.341 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620459+0000", 2023-10-09T05:54:41.341 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.341 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.341 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.341 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:41.341 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620459+0000", 2023-10-09T05:54:41.341 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.341 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.341 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.341 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:41.341 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620458+0000", 2023-10-09T05:54:41.341 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:54:41.342 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.342 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.342 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:41.342 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620525+0000", 2023-10-09T05:54:41.342 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7404999999999996e-05 2023-10-09T05:54:41.342 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.342 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.342 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:41.342 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620539+0000", 2023-10-09T05:54:41.342 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3997e-05 2023-10-09T05:54:41.342 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.343 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.343 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:41.343 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620545+0000", 2023-10-09T05:54:41.343 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7189999999999998e-06 2023-10-09T05:54:41.343 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.343 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.344 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.344 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620612+0000", 2023-10-09T05:54:41.344 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7325999999999999e-05 2023-10-09T05:54:41.344 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.344 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.344 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:41.344 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620750+0000", 2023-10-09T05:54:41.345 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000137059 2023-10-09T05:54:41.345 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.345 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.345 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:54:41.345 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620901+0000", 2023-10-09T05:54:41.345 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015133200000000001 2023-10-09T05:54:41.346 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.346 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.346 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:41.346 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.623123+0000", 2023-10-09T05:54:41.346 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0022225970000000002 2023-10-09T05:54:41.346 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.347 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.347 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.347 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.623990+0000", 2023-10-09T05:54:41.347 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00086624800000000002 2023-10-09T05:54:41.347 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.347 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.347 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.348 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964757+0000", 2023-10-09T05:54:41.348 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.34076741799999999 2023-10-09T05:54:41.348 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.348 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.348 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:41.348 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964800+0000", 2023-10-09T05:54:41.348 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2741000000000003e-05 2023-10-09T05:54:41.349 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.349 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.349 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:41.349 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964831+0000", 2023-10-09T05:54:41.349 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0738999999999998e-05 2023-10-09T05:54:41.349 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.350 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:41.350 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.350 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.350 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.350 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:77534 6.3 6:c39d213f:::benchmark_data_smithi064_284193_object77533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:54:41.350 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:49.644389+0000", 2023-10-09T05:54:41.351 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 171.61248634699999, 2023-10-09T05:54:41.351 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.33628388399999998, 2023-10-09T05:54:41.351 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:41.351 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:41.351 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:41.351 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:41.352 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:41.352 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 77534 2023-10-09T05:54:41.352 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.352 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:41.352 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.352 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:41.352 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644389+0000", 2023-10-09T05:54:41.352 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.352 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.353 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.353 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:41.353 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644389+0000", 2023-10-09T05:54:41.353 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.353 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.353 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.353 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:41.353 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644388+0000", 2023-10-09T05:54:41.353 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:54:41.353 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.353 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.353 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:41.354 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644424+0000", 2023-10-09T05:54:41.354 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5824999999999997e-05 2023-10-09T05:54:41.354 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.354 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.354 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:41.354 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644434+0000", 2023-10-09T05:54:41.354 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0461e-05 2023-10-09T05:54:41.354 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.354 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.354 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:41.354 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644440+0000", 2023-10-09T05:54:41.354 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1100000000000002e-06 2023-10-09T05:54:41.355 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.355 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.355 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.355 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644481+0000", 2023-10-09T05:54:41.355 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1902e-05 2023-10-09T05:54:41.355 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.355 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.355 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:41.355 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644613+0000", 2023-10-09T05:54:41.355 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000131815 2023-10-09T05:54:41.355 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.356 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.356 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:54:41.356 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644762+0000", 2023-10-09T05:54:41.356 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000148777 2023-10-09T05:54:41.356 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.356 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.356 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:41.356 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.649170+0000", 2023-10-09T05:54:41.356 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0044076310000000004 2023-10-09T05:54:41.356 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.356 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.356 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.356 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.650165+0000", 2023-10-09T05:54:41.357 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00099568399999999994 2023-10-09T05:54:41.357 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.357 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.357 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.357 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.980566+0000", 2023-10-09T05:54:41.357 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.33040050599999998 2023-10-09T05:54:41.357 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.357 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.357 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:41.357 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.980621+0000", 2023-10-09T05:54:41.357 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5392000000000001e-05 2023-10-09T05:54:41.357 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.358 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.358 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:41.358 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.980673+0000", 2023-10-09T05:54:41.358 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1316999999999997e-05 2023-10-09T05:54:41.358 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.358 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:41.358 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.358 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.358 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.358 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:77535 6.3 6:c58d58f2:::benchmark_data_smithi064_284193_object77534:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:54:41.359 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:49.644455+0000", 2023-10-09T05:54:41.359 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 171.61241968300001, 2023-10-09T05:54:41.359 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.33631617699999999, 2023-10-09T05:54:41.359 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:41.359 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:41.359 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:41.359 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:41.359 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:41.359 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 77535 2023-10-09T05:54:41.359 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.359 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:41.360 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.360 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:41.360 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644455+0000", 2023-10-09T05:54:41.360 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.360 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.360 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.360 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:41.360 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644455+0000", 2023-10-09T05:54:41.360 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.360 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.360 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.360 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:41.361 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644455+0000", 2023-10-09T05:54:41.361 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:54:41.361 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.361 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.361 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:41.361 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644491+0000", 2023-10-09T05:54:41.361 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6570999999999999e-05 2023-10-09T05:54:41.361 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.361 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.361 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:41.361 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644507+0000", 2023-10-09T05:54:41.361 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5880999999999999e-05 2023-10-09T05:54:41.361 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.362 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.362 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:41.362 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644512+0000", 2023-10-09T05:54:41.362 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0880000000000002e-06 2023-10-09T05:54:41.362 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.362 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.362 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.362 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.647155+0000", 2023-10-09T05:54:41.362 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0026430009999999999 2023-10-09T05:54:41.362 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.362 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.362 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:41.363 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.647283+0000", 2023-10-09T05:54:41.363 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012812300000000001 2023-10-09T05:54:41.363 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.363 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.363 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:54:41.363 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.647407+0000", 2023-10-09T05:54:41.363 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012368899999999999 2023-10-09T05:54:41.363 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.363 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.363 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:41.363 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.649330+0000", 2023-10-09T05:54:41.363 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0019226779999999999 2023-10-09T05:54:41.364 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.364 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.364 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.364 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.650266+0000", 2023-10-09T05:54:41.364 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00093634999999999996 2023-10-09T05:54:41.364 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.364 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.364 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.364 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.980707+0000", 2023-10-09T05:54:41.364 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.33044109100000002 2023-10-09T05:54:41.364 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.364 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.365 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:41.365 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.980743+0000", 2023-10-09T05:54:41.365 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5800000000000003e-05 2023-10-09T05:54:41.365 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.365 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.365 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:41.365 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.980772+0000", 2023-10-09T05:54:41.365 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.8580000000000001e-05 2023-10-09T05:54:41.365 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.365 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:41.365 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.365 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.366 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.366 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:79589 6.5 6:a0fa1d08:::benchmark_data_smithi064_284193_object79588:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:54:41.366 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:52.363694+0000", 2023-10-09T05:54:41.366 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 168.893181476, 2023-10-09T05:54:41.366 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.26210419000000001, 2023-10-09T05:54:41.366 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:41.366 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:41.366 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:41.366 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:41.366 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:41.366 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 79589 2023-10-09T05:54:41.366 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.367 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:41.367 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.367 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:41.367 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.363694+0000", 2023-10-09T05:54:41.367 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.367 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.367 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.368 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:41.368 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.363694+0000", 2023-10-09T05:54:41.368 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.368 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.368 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.368 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:41.368 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.363693+0000", 2023-10-09T05:54:41.369 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:54:41.369 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.369 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.369 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:41.369 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.363741+0000", 2023-10-09T05:54:41.370 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8436999999999997e-05 2023-10-09T05:54:41.370 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.370 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.370 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:41.370 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.363750+0000", 2023-10-09T05:54:41.371 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1820000000000006e-06 2023-10-09T05:54:41.371 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.371 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.371 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:41.371 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.363754+0000", 2023-10-09T05:54:41.371 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6120000000000001e-06 2023-10-09T05:54:41.372 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.372 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.372 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.372 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.366906+0000", 2023-10-09T05:54:41.372 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0031518340000000001 2023-10-09T05:54:41.372 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.373 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.373 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:41.373 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.367025+0000", 2023-10-09T05:54:41.373 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000118854 2023-10-09T05:54:41.373 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.374 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.374 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:54:41.374 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.367157+0000", 2023-10-09T05:54:41.374 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013233 2023-10-09T05:54:41.374 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.374 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.375 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:41.375 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.372713+0000", 2023-10-09T05:54:41.375 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0055557940000000002 2023-10-09T05:54:41.375 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.375 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.375 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.376 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.376095+0000", 2023-10-09T05:54:41.376 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0033821039999999999 2023-10-09T05:54:41.376 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.376 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.376 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.376 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.625686+0000", 2023-10-09T05:54:41.377 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24959123 2023-10-09T05:54:41.377 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.377 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.377 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:41.377 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.625752+0000", 2023-10-09T05:54:41.377 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6228000000000004e-05 2023-10-09T05:54:41.378 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.378 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.378 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:41.378 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.625798+0000", 2023-10-09T05:54:41.378 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5246e-05 2023-10-09T05:54:41.379 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.379 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:41.379 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.379 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.379 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.379 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:79591 6.5 6:a2ce5c7b:::benchmark_data_smithi064_284193_object79590:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:54:41.380 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:52.366733+0000", 2023-10-09T05:54:41.380 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 168.89014249799999, 2023-10-09T05:54:41.380 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.25915231599999999, 2023-10-09T05:54:41.380 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:41.380 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:41.380 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:41.381 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:41.381 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:41.381 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 79591 2023-10-09T05:54:41.381 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.381 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:41.382 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.382 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:41.382 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.366733+0000", 2023-10-09T05:54:41.382 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.382 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.382 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.383 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:41.383 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.366733+0000", 2023-10-09T05:54:41.383 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.383 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.383 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.383 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:41.384 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.366731+0000", 2023-10-09T05:54:41.384 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:54:41.384 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.384 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.384 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:41.385 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.366762+0000", 2023-10-09T05:54:41.385 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0255000000000001e-05 2023-10-09T05:54:41.385 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.385 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.385 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:41.385 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.366770+0000", 2023-10-09T05:54:41.385 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6680000000000007e-06 2023-10-09T05:54:41.386 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.386 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.386 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:41.386 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.366775+0000", 2023-10-09T05:54:41.386 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4710000000000001e-06 2023-10-09T05:54:41.387 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.387 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.387 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.387 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.369642+0000", 2023-10-09T05:54:41.387 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0028674389999999998 2023-10-09T05:54:41.388 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.388 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.388 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:41.388 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.369755+0000", 2023-10-09T05:54:41.388 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011286900000000001 2023-10-09T05:54:41.388 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.389 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.389 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:54:41.389 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.369850+0000", 2023-10-09T05:54:41.389 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.4812e-05 2023-10-09T05:54:41.389 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.389 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.390 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:41.390 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.375836+0000", 2023-10-09T05:54:41.390 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0059865750000000001 2023-10-09T05:54:41.390 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.390 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.390 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.391 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.377143+0000", 2023-10-09T05:54:41.391 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001306967 2023-10-09T05:54:41.391 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.391 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.391 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.391 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.625835+0000", 2023-10-09T05:54:41.392 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24869169799999999 2023-10-09T05:54:41.392 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.392 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.392 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:41.392 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.625865+0000", 2023-10-09T05:54:41.393 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9937000000000002e-05 2023-10-09T05:54:41.393 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.393 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.393 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:41.393 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.625885+0000", 2023-10-09T05:54:41.393 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9755e-05 2023-10-09T05:54:41.394 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.394 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:41.394 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.394 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.394 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.395 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:79595 6.8 6:1929f855:::benchmark_data_smithi064_284193_object79594:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:54:41.395 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:52.372938+0000", 2023-10-09T05:54:41.395 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 168.88393710400001, 2023-10-09T05:54:41.395 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.25287009199999999, 2023-10-09T05:54:41.395 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:41.395 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:41.395 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:41.396 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:41.396 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:41.396 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 79595 2023-10-09T05:54:41.396 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.396 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:41.397 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.397 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:41.397 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.372938+0000", 2023-10-09T05:54:41.397 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.397 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.397 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.397 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:41.398 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.372938+0000", 2023-10-09T05:54:41.398 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.398 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.398 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.398 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:41.399 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.372937+0000", 2023-10-09T05:54:41.399 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:54:41.399 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.399 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.399 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:41.399 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.372969+0000", 2023-10-09T05:54:41.400 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.2190999999999997e-05 2023-10-09T05:54:41.400 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.400 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.400 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:41.400 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.372976+0000", 2023-10-09T05:54:41.400 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6680000000000004e-06 2023-10-09T05:54:41.401 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.401 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.401 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:41.401 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.372979+0000", 2023-10-09T05:54:41.401 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7369999999999998e-06 2023-10-09T05:54:41.402 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.402 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.402 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.402 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.373034+0000", 2023-10-09T05:54:41.402 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5637999999999998e-05 2023-10-09T05:54:41.402 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.403 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.403 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:41.403 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.373151+0000", 2023-10-09T05:54:41.403 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000116282 2023-10-09T05:54:41.403 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.404 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.404 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:54:41.404 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.373279+0000", 2023-10-09T05:54:41.404 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012823000000000001 2023-10-09T05:54:41.404 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.405 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.405 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:41.405 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.376056+0000", 2023-10-09T05:54:41.405 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0027769019999999999 2023-10-09T05:54:41.405 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.405 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.405 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.406 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.377172+0000", 2023-10-09T05:54:41.406 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0011158730000000001 2023-10-09T05:54:41.406 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.406 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.406 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.406 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.625736+0000", 2023-10-09T05:54:41.407 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24856469 2023-10-09T05:54:41.407 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.407 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.407 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:41.407 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.625772+0000", 2023-10-09T05:54:41.407 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5989999999999999e-05 2023-10-09T05:54:41.408 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.408 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.408 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:41.408 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.625808+0000", 2023-10-09T05:54:41.408 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5621999999999999e-05 2023-10-09T05:54:41.408 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.409 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:41.409 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.409 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.409 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.409 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.44588.0:81811 6.1 e128/118 6:880e97e1:::benchmark_data_smithi064_284193_object81810:head v 128'5158, mlcod=0'0)", 2023-10-09T05:54:41.409 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:55.436474+0000", 2023-10-09T05:54:41.410 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 165.820401277, 2023-10-09T05:54:41.410 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24507322600000001, 2023-10-09T05:54:41.410 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:41.410 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:41.410 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:41.411 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.411 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:41.411 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.436474+0000", 2023-10-09T05:54:41.411 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.411 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.411 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.411 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:41.412 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.436474+0000", 2023-10-09T05:54:41.412 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.412 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.412 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.412 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:41.412 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.436473+0000", 2023-10-09T05:54:41.413 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:54:41.413 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.413 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.413 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:41.413 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.436497+0000", 2023-10-09T05:54:41.414 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3646e-05 2023-10-09T05:54:41.414 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.414 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.414 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:41.414 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.436507+0000", 2023-10-09T05:54:41.414 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0539e-05 2023-10-09T05:54:41.414 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.415 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.415 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:41.415 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.436513+0000", 2023-10-09T05:54:41.415 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3689999999999998e-06 2023-10-09T05:54:41.415 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.416 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.416 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.416 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.440741+0000", 2023-10-09T05:54:41.416 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0042279179999999998 2023-10-09T05:54:41.416 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.416 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.417 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:41.417 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.440763+0000", 2023-10-09T05:54:41.417 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2685e-05 2023-10-09T05:54:41.417 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.417 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.417 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:41.417 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.681509+0000", 2023-10-09T05:54:41.418 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24074568599999999 2023-10-09T05:54:41.418 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.418 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.418 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:41.418 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.681547+0000", 2023-10-09T05:54:41.418 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7879000000000001e-05 2023-10-09T05:54:41.419 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.419 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:41.419 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.419 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.419 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.419 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:81814 6.9 6:90e965d6:::benchmark_data_smithi064_284193_object81813:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T05:54:41.420 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:55.439384+0000", 2023-10-09T05:54:41.420 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 165.81749084, 2023-10-09T05:54:41.420 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24229829899999999, 2023-10-09T05:54:41.420 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:41.420 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:41.420 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:41.421 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:41.421 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:41.421 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 81814 2023-10-09T05:54:41.421 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.421 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:41.421 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.422 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:41.422 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.439384+0000", 2023-10-09T05:54:41.422 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.422 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.422 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.423 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:41.423 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.439384+0000", 2023-10-09T05:54:41.423 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.423 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.423 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.423 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:41.424 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.439383+0000", 2023-10-09T05:54:41.424 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:54:41.424 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.424 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.424 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:41.424 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.439436+0000", 2023-10-09T05:54:41.425 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3418999999999998e-05 2023-10-09T05:54:41.425 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.425 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.425 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:41.425 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.439448+0000", 2023-10-09T05:54:41.425 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1861999999999999e-05 2023-10-09T05:54:41.426 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.426 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.426 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:41.426 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.439454+0000", 2023-10-09T05:54:41.426 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9869999999999996e-06 2023-10-09T05:54:41.427 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.427 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.427 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.427 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.440728+0000", 2023-10-09T05:54:41.427 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001273864 2023-10-09T05:54:41.427 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.428 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.428 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:41.428 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.440849+0000", 2023-10-09T05:54:41.428 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000120986 2023-10-09T05:54:41.428 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.428 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.429 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2", 2023-10-09T05:54:41.429 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.440983+0000", 2023-10-09T05:54:41.429 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013372199999999999 2023-10-09T05:54:41.429 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.429 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.429 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:41.429 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.449915+0000", 2023-10-09T05:54:41.430 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0089326160000000009 2023-10-09T05:54:41.430 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.430 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.430 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.430 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.681544+0000", 2023-10-09T05:54:41.431 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.231628475 2023-10-09T05:54:41.431 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.431 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.431 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:41.431 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.681595+0000", 2023-10-09T05:54:41.431 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1721999999999997e-05 2023-10-09T05:54:41.431 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.432 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.432 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:41.432 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.681682+0000", 2023-10-09T05:54:41.432 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.7125000000000004e-05 2023-10-09T05:54:41.432 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.432 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:41.432 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.432 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.432 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.432 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.44588.0:81818 6.1 e128/118 6:853e1ef7:::benchmark_data_smithi064_284193_object81817:head v 128'5159, mlcod=0'0)", 2023-10-09T05:54:41.432 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:55.443887+0000", 2023-10-09T05:54:41.432 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 165.81298771100001, 2023-10-09T05:54:41.433 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24230628000000001, 2023-10-09T05:54:41.433 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:41.433 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:41.433 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:41.433 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.433 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:41.433 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.443887+0000", 2023-10-09T05:54:41.433 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.433 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.433 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.433 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:41.434 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.443887+0000", 2023-10-09T05:54:41.434 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.434 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.434 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.434 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:41.434 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.443887+0000", 2023-10-09T05:54:41.434 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:54:41.434 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.434 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.434 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:41.434 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.443910+0000", 2023-10-09T05:54:41.435 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3011e-05 2023-10-09T05:54:41.435 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.435 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.435 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:41.435 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.443920+0000", 2023-10-09T05:54:41.435 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0465999999999999e-05 2023-10-09T05:54:41.435 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.435 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.435 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:41.435 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.443926+0000", 2023-10-09T05:54:41.435 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7429999999999999e-06 2023-10-09T05:54:41.435 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.436 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.436 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.436 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.446417+0000", 2023-10-09T05:54:41.436 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0024909649999999999 2023-10-09T05:54:41.436 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.436 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.436 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:41.436 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.446436+0000", 2023-10-09T05:54:41.436 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8998999999999999e-05 2023-10-09T05:54:41.436 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.436 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.436 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:41.436 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.686156+0000", 2023-10-09T05:54:41.437 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.23971962699999999 2023-10-09T05:54:41.437 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.437 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.437 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:41.437 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.686194+0000", 2023-10-09T05:54:41.437 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8049000000000001e-05 2023-10-09T05:54:41.437 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.437 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:41.437 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.437 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.437 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.437 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.44588.0:81822 6.b e128/118 6:d664fd33:::benchmark_data_smithi064_284193_object81821:head v 128'5197, mlcod=107'1276)", 2023-10-09T05:54:41.438 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:55.451714+0000", 2023-10-09T05:54:41.438 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 165.805161019, 2023-10-09T05:54:41.438 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.239798436, 2023-10-09T05:54:41.438 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:41.438 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:41.438 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:41.438 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.438 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:41.438 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.451714+0000", 2023-10-09T05:54:41.438 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.438 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.438 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.439 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:41.439 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.451714+0000", 2023-10-09T05:54:41.439 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.439 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.439 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.439 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:41.439 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.451713+0000", 2023-10-09T05:54:41.439 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:54:41.439 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.439 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.439 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:41.439 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.451740+0000", 2023-10-09T05:54:41.439 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.6672000000000001e-05 2023-10-09T05:54:41.440 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.440 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.440 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:41.440 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.678418+0000", 2023-10-09T05:54:41.440 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.226678458 2023-10-09T05:54:41.440 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.440 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.440 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:41.440 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.678430+0000", 2023-10-09T05:54:41.440 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.136e-05 2023-10-09T05:54:41.440 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.440 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.441 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.441 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.682221+0000", 2023-10-09T05:54:41.441 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0037916410000000001 2023-10-09T05:54:41.441 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.441 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.441 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:41.441 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.682249+0000", 2023-10-09T05:54:41.441 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7319999999999999e-05 2023-10-09T05:54:41.441 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.441 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.441 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:41.441 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.691482+0000", 2023-10-09T05:54:41.442 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0092338920000000005 2023-10-09T05:54:41.442 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.442 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.442 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:41.442 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.691512+0000", 2023-10-09T05:54:41.442 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9944999999999999e-05 2023-10-09T05:54:41.442 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.442 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:41.442 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.442 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.442 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.442 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:81827 6.9 6:9293b997:::benchmark_data_smithi064_284193_object81826:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T05:54:41.442 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:55.452859+0000", 2023-10-09T05:54:41.443 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 165.80401563300001, 2023-10-09T05:54:41.443 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24494559699999999, 2023-10-09T05:54:41.443 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:54:41.443 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:54:41.443 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:54:41.443 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:54:41.443 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:54:41.443 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 81827 2023-10-09T05:54:41.443 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.443 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:54:41.443 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.443 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:54:41.444 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.452859+0000", 2023-10-09T05:54:41.444 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.444 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.444 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.444 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:54:41.444 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.452859+0000", 2023-10-09T05:54:41.444 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:54:41.444 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.444 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.444 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:54:41.444 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.452858+0000", 2023-10-09T05:54:41.444 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:54:41.445 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.445 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.445 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:54:41.445 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.452889+0000", 2023-10-09T05:54:41.445 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1037000000000001e-05 2023-10-09T05:54:41.445 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.445 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.445 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:54:41.445 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.680046+0000", 2023-10-09T05:54:41.445 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.227156793 2023-10-09T05:54:41.445 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.445 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.445 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:54:41.446 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.680058+0000", 2023-10-09T05:54:41.446 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.182e-05 2023-10-09T05:54:41.446 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.446 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.446 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:54:41.446 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.686253+0000", 2023-10-09T05:54:41.446 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0061949459999999998 2023-10-09T05:54:41.446 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.446 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.446 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:54:41.446 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.686375+0000", 2023-10-09T05:54:41.446 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012215199999999999 2023-10-09T05:54:41.447 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.447 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.447 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2", 2023-10-09T05:54:41.447 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.686521+0000", 2023-10-09T05:54:41.447 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000146203 2023-10-09T05:54:41.447 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.447 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.447 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:54:41.447 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.697632+0000", 2023-10-09T05:54:41.447 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011110372 2023-10-09T05:54:41.447 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.447 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.447 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:54:41.448 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.697673+0000", 2023-10-09T05:54:41.448 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1384999999999998e-05 2023-10-09T05:54:41.448 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.448 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.448 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:54:41.448 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.697715+0000", 2023-10-09T05:54:41.448 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1755000000000003e-05 2023-10-09T05:54:41.448 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:54:41.448 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:54:41.448 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:54:41.448 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:55.697805+0000", 2023-10-09T05:54:41.448 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0086000000000005e-05 2023-10-09T05:54:41.449 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.449 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:41.449 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.449 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:54:41.449 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:54:41.449 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:54:41.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:41.450 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:41.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:41.551 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:41.581 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:41.625 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:41.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:41.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:41.733 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:41.769 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:41.837 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:41.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:41 smithi156 ceph-mon[195101]: pgmap v430: 109 pgs: 1 peering, 2 active+recovering+undersized+remapped, 1 active+degraded, 1 active+recovery_wait+degraded+remapped, 97 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 26 op/s; 42204/266937 objects degraded (15.810%); 9671/266937 objects misplaced (3.623%); 12 MiB/s, 195 objects/s recovering 2023-10-09T05:54:41.887 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:41.895 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:41 smithi064 ceph-mon[212750]: pgmap v430: 109 pgs: 1 peering, 2 active+recovering+undersized+remapped, 1 active+degraded, 1 active+recovery_wait+degraded+remapped, 97 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 26 op/s; 42204/266937 objects degraded (15.810%); 9671/266937 objects misplaced (3.623%); 12 MiB/s, 195 objects/s recovering 2023-10-09T05:54:41.895 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:41 smithi064 ceph-mon[220044]: pgmap v430: 109 pgs: 1 peering, 2 active+recovering+undersized+remapped, 1 active+degraded, 1 active+recovery_wait+degraded+remapped, 97 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 26 op/s; 42204/266937 objects degraded (15.810%); 9671/266937 objects misplaced (3.623%); 12 MiB/s, 195 objects/s recovering 2023-10-09T05:54:41.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:41.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:41.997 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:42.048 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:42.152 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:42.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:42.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:42.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:42.293 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:42.338 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:54:42.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:42 smithi156 ceph-mon[195101]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:54:42.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:42 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 42204/266937 objects degraded (15.810%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:42.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:42 smithi064 ceph-mon[212750]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:54:42.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:42 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 42204/266937 objects degraded (15.810%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:42.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:42 smithi064 ceph-mon[220044]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:54:42.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:42 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 42204/266937 objects degraded (15.810%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:43.020 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:54:43.315 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:43.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:43.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:43.419 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:43.449 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:43.488 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:43.495 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:43.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:43.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:43.602 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:43.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:43 smithi064 ceph-mon[220044]: pgmap v431: 109 pgs: 1 peering, 2 active+recovering+undersized+remapped, 1 active+degraded, 1 active+recovery_wait+degraded+remapped, 97 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 26 op/s; 42204/266937 objects degraded (15.810%); 9671/266937 objects misplaced (3.623%); 12 MiB/s, 195 objects/s recovering 2023-10-09T05:54:43.615 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:43 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2635338151' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:54:43.615 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:43 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:54:43.615 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:43 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:43.630 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:43.645 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 6 pg_num to 26 2023-10-09T05:54:43.677 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:43.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:43.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:43.787 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:43.828 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:43.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:43 smithi156 ceph-mon[195101]: pgmap v431: 109 pgs: 1 peering, 2 active+recovering+undersized+remapped, 1 active+degraded, 1 active+recovery_wait+degraded+remapped, 97 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 26 op/s; 42204/266937 objects degraded (15.810%); 9671/266937 objects misplaced (3.623%); 12 MiB/s, 195 objects/s recovering 2023-10-09T05:54:43.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:43 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2635338151' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:54:43.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:43 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:54:43.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:43 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:43.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:43 smithi064 ceph-mon[212750]: pgmap v431: 109 pgs: 1 peering, 2 active+recovering+undersized+remapped, 1 active+degraded, 1 active+recovery_wait+degraded+remapped, 97 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 26 op/s; 42204/266937 objects degraded (15.810%); 9671/266937 objects misplaced (3.623%); 12 MiB/s, 195 objects/s recovering 2023-10-09T05:54:43.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:43 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2635338151' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T05:54:43.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:43 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:54:43.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:43 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:44.667 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T05:54:44.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:44 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2635338151' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2023-10-09T05:54:44.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:44 smithi064 ceph-mon[212750]: osdmap e187: 8 total, 8 up, 8 in 2023-10-09T05:54:44.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:44 smithi064 ceph-mon[212750]: failed to encode map e187 with expected crc 2023-10-09T05:54:44.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:44 smithi064 ceph-mon[212750]: failed to encode map e187 with expected crc 2023-10-09T05:54:44.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:44 smithi064 ceph-mon[212750]: failed to encode map e187 with expected crc 2023-10-09T05:54:44.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:44 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2155122989' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:54:44.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:44 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:54:44.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T05:54:44.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T05:54:44.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:44 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2635338151' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2023-10-09T05:54:44.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:44 smithi064 ceph-mon[220044]: osdmap e187: 8 total, 8 up, 8 in 2023-10-09T05:54:44.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:44 smithi064 ceph-mon[220044]: failed to encode map e187 with expected crc 2023-10-09T05:54:44.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:44 smithi064 ceph-mon[220044]: failed to encode map e187 with expected crc 2023-10-09T05:54:44.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:44 smithi064 ceph-mon[220044]: failed to encode map e187 with expected crc 2023-10-09T05:54:44.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:44 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2155122989' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:54:44.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:44 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:54:44.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T05:54:44.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T05:54:45.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:44 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2635338151' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2023-10-09T05:54:45.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:44 smithi156 ceph-mon[195101]: osdmap e187: 8 total, 8 up, 8 in 2023-10-09T05:54:45.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:44 smithi156 ceph-mon[195101]: failed to encode map e187 with expected crc 2023-10-09T05:54:45.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:44 smithi156 ceph-mon[195101]: failed to encode map e187 with expected crc 2023-10-09T05:54:45.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:44 smithi156 ceph-mon[195101]: failed to encode map e187 with expected crc 2023-10-09T05:54:45.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:44 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2155122989' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:54:45.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:44 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:54:45.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T05:54:45.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T05:54:45.152 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:45.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:45.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:45.257 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:45.284 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:45.327 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:45.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:45.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:45.434 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:45.461 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:45.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:54:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:54:45] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2023-10-09T05:54:45.505 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:45.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:45.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:45.589 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:45.612 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:45.688 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:45.843 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:45 smithi064 ceph-mon[212750]: failed to encode map e187 with expected crc 2023-10-09T05:54:45.843 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:45 smithi064 ceph-mon[212750]: pgmap v433: 109 pgs: 1 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 97 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 44 op/s; 42198/266652 objects degraded (15.825%); 9659/266652 objects misplaced (3.622%); 8.1 MiB/s, 128 objects/s recovering 2023-10-09T05:54:45.843 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:45 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:45.843 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:45 smithi064 ceph-mon[212750]: failed to encode map e188 with expected crc 2023-10-09T05:54:45.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:45 smithi064 ceph-mon[212750]: failed to encode map e188 with expected crc 2023-10-09T05:54:45.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:45 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:54:45.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:45 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T05:54:45.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:45 smithi064 ceph-mon[212750]: osdmap e188: 8 total, 8 up, 8 in 2023-10-09T05:54:45.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:45 smithi064 ceph-mon[220044]: failed to encode map e187 with expected crc 2023-10-09T05:54:45.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:45 smithi064 ceph-mon[220044]: pgmap v433: 109 pgs: 1 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 97 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 44 op/s; 42198/266652 objects degraded (15.825%); 9659/266652 objects misplaced (3.622%); 8.1 MiB/s, 128 objects/s recovering 2023-10-09T05:54:45.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:45 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:45.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:45 smithi064 ceph-mon[220044]: failed to encode map e188 with expected crc 2023-10-09T05:54:45.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:45 smithi064 ceph-mon[220044]: failed to encode map e188 with expected crc 2023-10-09T05:54:45.846 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:45 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:54:45.846 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:45 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T05:54:45.846 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:45 smithi064 ceph-mon[220044]: osdmap e188: 8 total, 8 up, 8 in 2023-10-09T05:54:45.863 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:45.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:45.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:45.967 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:45.992 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:45 smithi156 ceph-mon[195101]: failed to encode map e187 with expected crc 2023-10-09T05:54:45.992 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:45 smithi156 ceph-mon[195101]: pgmap v433: 109 pgs: 1 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 97 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 44 op/s; 42198/266652 objects degraded (15.825%); 9659/266652 objects misplaced (3.622%); 8.1 MiB/s, 128 objects/s recovering 2023-10-09T05:54:45.993 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:45 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:45.993 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:45 smithi156 ceph-mon[195101]: failed to encode map e188 with expected crc 2023-10-09T05:54:45.993 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:45 smithi156 ceph-mon[195101]: failed to encode map e188 with expected crc 2023-10-09T05:54:45.993 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:45 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:54:45.993 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:45 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T05:54:45.993 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:45 smithi156 ceph-mon[195101]: osdmap e188: 8 total, 8 up, 8 in 2023-10-09T05:54:45.995 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:46.042 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:46.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:46.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:46.147 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:46.176 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:46.226 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:46.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:46.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:46.331 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:46.359 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:46.407 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:46.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:46.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:46.515 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:46.546 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:46.672 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:46.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:46.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:46.786 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:46.815 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:46.864 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:46.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:46.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:46.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:46 smithi064 ceph-mon[212750]: failed to encode map e188 with expected crc 2023-10-09T05:54:46.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:46 smithi064 ceph-mon[212750]: failed to encode map e189 with expected crc 2023-10-09T05:54:46.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:46 smithi064 ceph-mon[212750]: osdmap e189: 8 total, 8 up, 8 in 2023-10-09T05:54:46.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:46 smithi064 ceph-mon[212750]: failed to encode map e189 with expected crc 2023-10-09T05:54:46.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:46 smithi064 ceph-mon[212750]: failed to encode map e189 with expected crc 2023-10-09T05:54:46.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:46 smithi064 ceph-mon[212750]: failed to encode map e189 with expected crc 2023-10-09T05:54:46.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:46 smithi064 ceph-mon[212750]: failed to encode map e189 with expected crc 2023-10-09T05:54:46.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:46 smithi064 ceph-mon[212750]: failed to encode map e189 with expected crc 2023-10-09T05:54:46.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:46 smithi064 ceph-mon[212750]: failed to encode map e189 with expected crc 2023-10-09T05:54:46.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:46 smithi064 ceph-mon[212750]: 6.6 continuing backfill to osd.5 from (106'613,187'6529] MIN to 187'6529 2023-10-09T05:54:46.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:46 smithi064 ceph-mon[220044]: failed to encode map e188 with expected crc 2023-10-09T05:54:46.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:46 smithi064 ceph-mon[220044]: failed to encode map e189 with expected crc 2023-10-09T05:54:46.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:46 smithi064 ceph-mon[220044]: osdmap e189: 8 total, 8 up, 8 in 2023-10-09T05:54:46.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:46 smithi064 ceph-mon[220044]: failed to encode map e189 with expected crc 2023-10-09T05:54:46.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:46 smithi064 ceph-mon[220044]: failed to encode map e189 with expected crc 2023-10-09T05:54:46.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:46 smithi064 ceph-mon[220044]: failed to encode map e189 with expected crc 2023-10-09T05:54:46.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:46 smithi064 ceph-mon[220044]: failed to encode map e189 with expected crc 2023-10-09T05:54:46.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:46 smithi064 ceph-mon[220044]: failed to encode map e189 with expected crc 2023-10-09T05:54:46.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:46 smithi064 ceph-mon[220044]: failed to encode map e189 with expected crc 2023-10-09T05:54:46.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:46 smithi064 ceph-mon[220044]: 6.6 continuing backfill to osd.5 from (106'613,187'6529] MIN to 187'6529 2023-10-09T05:54:46.973 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:47.023 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:47.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:46 smithi156 ceph-mon[195101]: failed to encode map e188 with expected crc 2023-10-09T05:54:47.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:46 smithi156 ceph-mon[195101]: failed to encode map e189 with expected crc 2023-10-09T05:54:47.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:46 smithi156 ceph-mon[195101]: osdmap e189: 8 total, 8 up, 8 in 2023-10-09T05:54:47.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:46 smithi156 ceph-mon[195101]: failed to encode map e189 with expected crc 2023-10-09T05:54:47.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:46 smithi156 ceph-mon[195101]: failed to encode map e189 with expected crc 2023-10-09T05:54:47.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:46 smithi156 ceph-mon[195101]: failed to encode map e189 with expected crc 2023-10-09T05:54:47.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:46 smithi156 ceph-mon[195101]: failed to encode map e189 with expected crc 2023-10-09T05:54:47.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:46 smithi156 ceph-mon[195101]: failed to encode map e189 with expected crc 2023-10-09T05:54:47.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:46 smithi156 ceph-mon[195101]: failed to encode map e189 with expected crc 2023-10-09T05:54:47.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:46 smithi156 ceph-mon[195101]: 6.6 continuing backfill to osd.5 from (106'613,187'6529] MIN to 187'6529 2023-10-09T05:54:47.199 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:47.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:47.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:47.299 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:54:47.299 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T05:54:47.299 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2023-10-09T05:54:47.299 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:54:47.305 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:47.334 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:47.381 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:47.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:47.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:47.486 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:47.515 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:47.563 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:47.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:47.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:47.643 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:47 smithi064 ceph-mon[220044]: pgmap v436: 119 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 2 peering, 10 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 96 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42139/241605 objects degraded (17.441%); 5675/241605 objects misplaced (2.349%); 6.1 MiB/s, 102 objects/s recovering 2023-10-09T05:54:47.644 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:47 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:54:47.644 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:47 smithi064 ceph-mon[220044]: failed to encode map e190 with expected crc 2023-10-09T05:54:47.644 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:47 smithi064 ceph-mon[220044]: failed to encode map e190 with expected crc 2023-10-09T05:54:47.644 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:47 smithi064 ceph-mon[220044]: failed to encode map e190 with expected crc 2023-10-09T05:54:47.644 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:47 smithi064 ceph-mon[220044]: osdmap e190: 8 total, 8 up, 8 in 2023-10-09T05:54:47.644 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:47 smithi064 ceph-mon[220044]: failed to encode map e190 with expected crc 2023-10-09T05:54:47.644 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:47 smithi064 ceph-mon[220044]: failed to encode map e190 with expected crc 2023-10-09T05:54:47.644 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:47 smithi064 ceph-mon[220044]: failed to encode map e190 with expected crc 2023-10-09T05:54:47.644 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:47 smithi064 ceph-mon[220044]: failed to encode map e190 with expected crc 2023-10-09T05:54:47.644 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:47 smithi064 ceph-mon[220044]: failed to encode map e190 with expected crc 2023-10-09T05:54:47.645 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:47 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 42139/241605 objects degraded (17.441%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:47.668 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:47.696 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:47.749 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:47.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:47.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:47.855 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:47.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:47 smithi064 ceph-mon[212750]: pgmap v436: 119 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 2 peering, 10 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 96 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42139/241605 objects degraded (17.441%); 5675/241605 objects misplaced (2.349%); 6.1 MiB/s, 102 objects/s recovering 2023-10-09T05:54:47.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:47 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:54:47.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:47 smithi064 ceph-mon[212750]: failed to encode map e190 with expected crc 2023-10-09T05:54:47.892 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:47 smithi064 ceph-mon[212750]: failed to encode map e190 with expected crc 2023-10-09T05:54:47.892 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:47 smithi064 ceph-mon[212750]: failed to encode map e190 with expected crc 2023-10-09T05:54:47.892 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:47 smithi064 ceph-mon[212750]: osdmap e190: 8 total, 8 up, 8 in 2023-10-09T05:54:47.892 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:47 smithi064 ceph-mon[212750]: failed to encode map e190 with expected crc 2023-10-09T05:54:47.892 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:47 smithi064 ceph-mon[212750]: failed to encode map e190 with expected crc 2023-10-09T05:54:47.892 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:47 smithi064 ceph-mon[212750]: failed to encode map e190 with expected crc 2023-10-09T05:54:47.892 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:47 smithi064 ceph-mon[212750]: failed to encode map e190 with expected crc 2023-10-09T05:54:47.892 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:47 smithi064 ceph-mon[212750]: failed to encode map e190 with expected crc 2023-10-09T05:54:47.892 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:47 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 42139/241605 objects degraded (17.441%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:47.896 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:48.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:47 smithi156 ceph-mon[195101]: pgmap v436: 119 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 2 peering, 10 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 96 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42139/241605 objects degraded (17.441%); 5675/241605 objects misplaced (2.349%); 6.1 MiB/s, 102 objects/s recovering 2023-10-09T05:54:48.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:47 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:54:48.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:47 smithi156 ceph-mon[195101]: failed to encode map e190 with expected crc 2023-10-09T05:54:48.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:47 smithi156 ceph-mon[195101]: failed to encode map e190 with expected crc 2023-10-09T05:54:48.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:47 smithi156 ceph-mon[195101]: failed to encode map e190 with expected crc 2023-10-09T05:54:48.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:47 smithi156 ceph-mon[195101]: osdmap e190: 8 total, 8 up, 8 in 2023-10-09T05:54:48.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:47 smithi156 ceph-mon[195101]: failed to encode map e190 with expected crc 2023-10-09T05:54:48.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:47 smithi156 ceph-mon[195101]: failed to encode map e190 with expected crc 2023-10-09T05:54:48.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:47 smithi156 ceph-mon[195101]: failed to encode map e190 with expected crc 2023-10-09T05:54:48.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:47 smithi156 ceph-mon[195101]: failed to encode map e190 with expected crc 2023-10-09T05:54:48.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:47 smithi156 ceph-mon[195101]: failed to encode map e190 with expected crc 2023-10-09T05:54:48.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:47 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 42139/241605 objects degraded (17.441%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:54:48.483 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:54:48.661 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:48.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:48.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:48.731 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:54:48.773 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:54:48.798 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:48.816 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:48 smithi064 ceph-mon[212750]: failed to encode map e191 with expected crc 2023-10-09T05:54:48.816 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:48 smithi064 ceph-mon[212750]: osdmap e191: 8 total, 8 up, 8 in 2023-10-09T05:54:48.816 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:48 smithi064 ceph-mon[212750]: failed to encode map e191 with expected crc 2023-10-09T05:54:48.816 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:48 smithi064 ceph-mon[212750]: failed to encode map e191 with expected crc 2023-10-09T05:54:48.816 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:48 smithi064 ceph-mon[212750]: failed to encode map e191 with expected crc 2023-10-09T05:54:48.816 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:48 smithi064 ceph-mon[212750]: failed to encode map e191 with expected crc 2023-10-09T05:54:48.816 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:48 smithi064 ceph-mon[212750]: failed to encode map e191 with expected crc 2023-10-09T05:54:48.816 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:48 smithi064 ceph-mon[212750]: failed to encode map e191 with expected crc 2023-10-09T05:54:48.816 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:48 smithi064 ceph-mon[212750]: failed to encode map e191 with expected crc 2023-10-09T05:54:48.816 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:48 smithi064 ceph-mon[212750]: 6.16 continuing backfill to osd.5 from (106'613,187'6529] MIN to 187'6529 2023-10-09T05:54:48.817 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:48 smithi064 ceph-mon[220044]: failed to encode map e191 with expected crc 2023-10-09T05:54:48.817 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:48 smithi064 ceph-mon[220044]: osdmap e191: 8 total, 8 up, 8 in 2023-10-09T05:54:48.817 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:48 smithi064 ceph-mon[220044]: failed to encode map e191 with expected crc 2023-10-09T05:54:48.817 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:48 smithi064 ceph-mon[220044]: failed to encode map e191 with expected crc 2023-10-09T05:54:48.817 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:48 smithi064 ceph-mon[220044]: failed to encode map e191 with expected crc 2023-10-09T05:54:48.817 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:48 smithi064 ceph-mon[220044]: failed to encode map e191 with expected crc 2023-10-09T05:54:48.817 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:48 smithi064 ceph-mon[220044]: failed to encode map e191 with expected crc 2023-10-09T05:54:48.817 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:48 smithi064 ceph-mon[220044]: failed to encode map e191 with expected crc 2023-10-09T05:54:48.817 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:48 smithi064 ceph-mon[220044]: failed to encode map e191 with expected crc 2023-10-09T05:54:48.818 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:48 smithi064 ceph-mon[220044]: 6.16 continuing backfill to osd.5 from (106'613,187'6529] MIN to 187'6529 2023-10-09T05:54:48.843 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:54:48.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:48.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:54:48.948 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:48.972 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:48 smithi156 ceph-mon[195101]: failed to encode map e191 with expected crc 2023-10-09T05:54:48.972 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:48 smithi156 ceph-mon[195101]: osdmap e191: 8 total, 8 up, 8 in 2023-10-09T05:54:48.972 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:48 smithi156 ceph-mon[195101]: failed to encode map e191 with expected crc 2023-10-09T05:54:48.972 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:48 smithi156 ceph-mon[195101]: failed to encode map e191 with expected crc 2023-10-09T05:54:48.972 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:48 smithi156 ceph-mon[195101]: failed to encode map e191 with expected crc 2023-10-09T05:54:48.972 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:48 smithi156 ceph-mon[195101]: failed to encode map e191 with expected crc 2023-10-09T05:54:48.973 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:48 smithi156 ceph-mon[195101]: failed to encode map e191 with expected crc 2023-10-09T05:54:48.973 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:48 smithi156 ceph-mon[195101]: failed to encode map e191 with expected crc 2023-10-09T05:54:48.973 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:48 smithi156 ceph-mon[195101]: failed to encode map e191 with expected crc 2023-10-09T05:54:48.973 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:48 smithi156 ceph-mon[195101]: 6.16 continuing backfill to osd.5 from (106'613,187'6529] MIN to 187'6529 2023-10-09T05:54:48.974 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:49.019 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:49.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:49.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:49.124 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:49.193 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:49.386 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:49.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:49.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:49.427 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:49.463 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:54:49.489 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:49.499 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:49.584 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:49.693 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:49 smithi064 ceph-mon[212750]: pgmap v439: 119 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 2 peering, 10 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 96 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42139/241605 objects degraded (17.441%); 5675/241605 objects misplaced (2.349%); 9.1 MiB/s, 152 objects/s recovering 2023-10-09T05:54:49.693 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:49 smithi064 ceph-mon[212750]: osdmap e192: 8 total, 8 up, 8 in 2023-10-09T05:54:49.693 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:49 smithi064 ceph-mon[212750]: failed to encode map e192 with expected crc 2023-10-09T05:54:49.694 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:49 smithi064 ceph-mon[212750]: failed to encode map e192 with expected crc 2023-10-09T05:54:49.694 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:49 smithi064 ceph-mon[220044]: pgmap v439: 119 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 2 peering, 10 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 96 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42139/241605 objects degraded (17.441%); 5675/241605 objects misplaced (2.349%); 9.1 MiB/s, 152 objects/s recovering 2023-10-09T05:54:49.694 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:49 smithi064 ceph-mon[220044]: osdmap e192: 8 total, 8 up, 8 in 2023-10-09T05:54:49.694 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:49 smithi064 ceph-mon[220044]: failed to encode map e192 with expected crc 2023-10-09T05:54:49.694 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:49 smithi064 ceph-mon[220044]: failed to encode map e192 with expected crc 2023-10-09T05:54:49.695 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:49 smithi064 ceph-mon[220044]: failed to encode map e192 with expected crc 2023-10-09T05:54:49.695 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:49 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/672055695' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:54:49.808 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:49.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:49.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:49.926 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:49.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:49 smithi064 ceph-mon[212750]: failed to encode map e192 with expected crc 2023-10-09T05:54:49.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:49 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/672055695' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:54:49.977 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:50.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:49 smithi156 ceph-mon[195101]: pgmap v439: 119 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 2 peering, 10 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 96 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42139/241605 objects degraded (17.441%); 5675/241605 objects misplaced (2.349%); 9.1 MiB/s, 152 objects/s recovering 2023-10-09T05:54:50.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:49 smithi156 ceph-mon[195101]: osdmap e192: 8 total, 8 up, 8 in 2023-10-09T05:54:50.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:49 smithi156 ceph-mon[195101]: failed to encode map e192 with expected crc 2023-10-09T05:54:50.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:49 smithi156 ceph-mon[195101]: failed to encode map e192 with expected crc 2023-10-09T05:54:50.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:49 smithi156 ceph-mon[195101]: failed to encode map e192 with expected crc 2023-10-09T05:54:50.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:49 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/672055695' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:54:50.152 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:50.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:50.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:50.263 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:50.333 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:50.621 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 1, 7] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T05:54:50.621 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:54:50.621 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 2, 3, 4, 5, 6, 1, 7] 2023-10-09T05:54:50.621 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd out 1 2023-10-09T05:54:50.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:50 smithi064 ceph-mon[212750]: failed to encode map e193 with expected crc 2023-10-09T05:54:50.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:50 smithi064 ceph-mon[212750]: osdmap e193: 8 total, 8 up, 8 in 2023-10-09T05:54:50.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:50 smithi064 ceph-mon[212750]: failed to encode map e193 with expected crc 2023-10-09T05:54:50.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:50 smithi064 ceph-mon[220044]: failed to encode map e193 with expected crc 2023-10-09T05:54:50.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:50 smithi064 ceph-mon[220044]: osdmap e193: 8 total, 8 up, 8 in 2023-10-09T05:54:50.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:50 smithi064 ceph-mon[220044]: failed to encode map e193 with expected crc 2023-10-09T05:54:51.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:50 smithi156 ceph-mon[195101]: failed to encode map e193 with expected crc 2023-10-09T05:54:51.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:50 smithi156 ceph-mon[195101]: osdmap e193: 8 total, 8 up, 8 in 2023-10-09T05:54:51.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:50 smithi156 ceph-mon[195101]: failed to encode map e193 with expected crc 2023-10-09T05:54:51.198 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:51.208 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:54:51.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:51.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:51.312 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:54:51.339 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:51.373 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:54:51.373 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:54:51.373 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:54:51.373 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:54:51.373 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:54:51.373 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:54:51.384 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:54:51.384 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:54:51.384 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:54:51.384 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:54:51.385 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:54:51.385 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:54:51.385 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:54:51.385 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:54:51.385 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:54:51.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:51.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:54:51.492 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:54:51.535 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:51.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:51 smithi064 ceph-mon[212750]: pgmap v442: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 peering, 4 unknown, 2 active+recovery_wait+degraded+remapped, 104 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 140 MiB/s wr, 2.43k op/s; 42951/266328 objects degraded (16.127%); 6222/266328 objects misplaced (2.336%); 33 MiB/s, 677 objects/s recovering 2023-10-09T05:54:51.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:51 smithi064 ceph-mon[220044]: pgmap v442: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 peering, 4 unknown, 2 active+recovery_wait+degraded+remapped, 104 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 140 MiB/s wr, 2.43k op/s; 42951/266328 objects degraded (16.127%); 6222/266328 objects misplaced (2.336%); 33 MiB/s, 677 objects/s recovering 2023-10-09T05:54:52.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:51 smithi156 ceph-mon[195101]: pgmap v442: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 peering, 4 unknown, 2 active+recovery_wait+degraded+remapped, 104 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 140 MiB/s wr, 2.43k op/s; 42951/266328 objects degraded (16.127%); 6222/266328 objects misplaced (2.336%); 33 MiB/s, 677 objects/s recovering 2023-10-09T05:54:53.028 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:52 smithi064 ceph-mon[220044]: osdmap e194: 8 total, 8 up, 8 in 2023-10-09T05:54:53.028 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:52 smithi064 ceph-mon[220044]: failed to encode map e194 with expected crc 2023-10-09T05:54:53.028 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:52 smithi064 ceph-mon[220044]: failed to encode map e194 with expected crc 2023-10-09T05:54:53.028 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:52 smithi064 ceph-mon[220044]: failed to encode map e194 with expected crc 2023-10-09T05:54:53.028 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:52 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 42951/266328 objects degraded (16.127%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:54:53.029 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:52 smithi064 ceph-mon[212750]: osdmap e194: 8 total, 8 up, 8 in 2023-10-09T05:54:53.029 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:52 smithi064 ceph-mon[212750]: failed to encode map e194 with expected crc 2023-10-09T05:54:53.029 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:52 smithi064 ceph-mon[212750]: failed to encode map e194 with expected crc 2023-10-09T05:54:53.029 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:52 smithi064 ceph-mon[212750]: failed to encode map e194 with expected crc 2023-10-09T05:54:53.029 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:52 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 42951/266328 objects degraded (16.127%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:54:53.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:52 smithi156 ceph-mon[195101]: osdmap e194: 8 total, 8 up, 8 in 2023-10-09T05:54:53.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:52 smithi156 ceph-mon[195101]: failed to encode map e194 with expected crc 2023-10-09T05:54:53.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:52 smithi156 ceph-mon[195101]: failed to encode map e194 with expected crc 2023-10-09T05:54:53.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:52 smithi156 ceph-mon[195101]: failed to encode map e194 with expected crc 2023-10-09T05:54:53.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:52 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 42951/266328 objects degraded (16.127%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:54:53.542 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:54:53.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:53.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:54:53.647 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:53.674 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:53.721 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:53.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:53.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:53.826 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:53.866 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:53 smithi064 ceph-mon[212750]: failed to encode map e194 with expected crc 2023-10-09T05:54:53.866 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:53 smithi064 ceph-mon[212750]: pgmap v444: 119 pgs: 1 activating+undersized+degraded+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 peering, 4 unknown, 2 active+recovery_wait+degraded+remapped, 104 active+clean, 4 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 76 MiB/s wr, 1.32k op/s; 33933/257829 objects degraded (13.161%); 6222/257829 objects misplaced (2.413%); 27 MiB/s, 554 objects/s recovering 2023-10-09T05:54:53.866 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:53 smithi064 ceph-mon[220044]: failed to encode map e194 with expected crc 2023-10-09T05:54:53.866 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:53 smithi064 ceph-mon[220044]: pgmap v444: 119 pgs: 1 activating+undersized+degraded+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 peering, 4 unknown, 2 active+recovery_wait+degraded+remapped, 104 active+clean, 4 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 76 MiB/s wr, 1.32k op/s; 33933/257829 objects degraded (13.161%); 6222/257829 objects misplaced (2.413%); 27 MiB/s, 554 objects/s recovering 2023-10-09T05:54:53.870 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:54.050 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:54.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:54.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:54.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:53 smithi156 ceph-mon[195101]: failed to encode map e194 with expected crc 2023-10-09T05:54:54.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:53 smithi156 ceph-mon[195101]: pgmap v444: 119 pgs: 1 activating+undersized+degraded+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 peering, 4 unknown, 2 active+recovery_wait+degraded+remapped, 104 active+clean, 4 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 76 MiB/s wr, 1.32k op/s; 33933/257829 objects degraded (13.161%); 6222/257829 objects misplaced (2.413%); 27 MiB/s, 554 objects/s recovering 2023-10-09T05:54:54.156 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:54:54.204 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:54.210 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:54.416 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:54:54.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:54.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:54:54.525 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:54.581 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:54.941 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:54 smithi064 ceph-mon[220044]: osdmap e195: 8 total, 8 up, 8 in 2023-10-09T05:54:54.941 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:54 smithi064 ceph-mon[220044]: failed to encode map e195 with expected crc 2023-10-09T05:54:54.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:54 smithi064 ceph-mon[220044]: failed to encode map e195 with expected crc 2023-10-09T05:54:54.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:54 smithi064 ceph-mon[220044]: failed to encode map e195 with expected crc 2023-10-09T05:54:54.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:54 smithi064 ceph-mon[220044]: failed to encode map e195 with expected crc 2023-10-09T05:54:54.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:54 smithi064 ceph-mon[220044]: failed to encode map e195 with expected crc 2023-10-09T05:54:54.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:54 smithi064 ceph-mon[212750]: osdmap e195: 8 total, 8 up, 8 in 2023-10-09T05:54:54.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:54 smithi064 ceph-mon[212750]: failed to encode map e195 with expected crc 2023-10-09T05:54:54.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:54 smithi064 ceph-mon[212750]: failed to encode map e195 with expected crc 2023-10-09T05:54:54.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:54 smithi064 ceph-mon[212750]: failed to encode map e195 with expected crc 2023-10-09T05:54:54.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:54 smithi064 ceph-mon[212750]: failed to encode map e195 with expected crc 2023-10-09T05:54:54.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:54 smithi064 ceph-mon[212750]: failed to encode map e195 with expected crc 2023-10-09T05:54:54.945 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:54.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:54.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:54.982 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:54:54.982 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:54:55.053 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:55.077 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:54 smithi156 ceph-mon[195101]: osdmap e195: 8 total, 8 up, 8 in 2023-10-09T05:54:55.077 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:54 smithi156 ceph-mon[195101]: failed to encode map e195 with expected crc 2023-10-09T05:54:55.077 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:54 smithi156 ceph-mon[195101]: failed to encode map e195 with expected crc 2023-10-09T05:54:55.078 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:54 smithi156 ceph-mon[195101]: failed to encode map e195 with expected crc 2023-10-09T05:54:55.078 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:54 smithi156 ceph-mon[195101]: failed to encode map e195 with expected crc 2023-10-09T05:54:55.078 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:54 smithi156 ceph-mon[195101]: failed to encode map e195 with expected crc 2023-10-09T05:54:55.080 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:55.123 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:55.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:55.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:55.229 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:55.282 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:55.560 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:55.562 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:54:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:54:55] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2023-10-09T05:54:55.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:55.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:55.664 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:55.690 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:55.735 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:55.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:55.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:55.778 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T05:54:55.814 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:55 smithi064 ceph-mon[212750]: pgmap v446: 119 pgs: 2 activating+undersized+degraded+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 peering, 2 unknown, 2 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.10k op/s; 27912/257826 objects degraded (10.826%); 11145/257826 objects misplaced (4.323%); 26 MiB/s, 516 objects/s recovering 2023-10-09T05:54:55.814 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:55 smithi064 ceph-mon[212750]: osdmap e196: 8 total, 8 up, 8 in 2023-10-09T05:54:55.814 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:55 smithi064 ceph-mon[212750]: failed to encode map e196 with expected crc 2023-10-09T05:54:55.814 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:55 smithi064 ceph-mon[212750]: failed to encode map e196 with expected crc 2023-10-09T05:54:55.814 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:55 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3937361400' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:55.815 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:55 smithi064 ceph-mon[220044]: pgmap v446: 119 pgs: 2 activating+undersized+degraded+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 peering, 2 unknown, 2 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.10k op/s; 27912/257826 objects degraded (10.826%); 11145/257826 objects misplaced (4.323%); 26 MiB/s, 516 objects/s recovering 2023-10-09T05:54:55.815 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:55 smithi064 ceph-mon[220044]: osdmap e196: 8 total, 8 up, 8 in 2023-10-09T05:54:55.815 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:55 smithi064 ceph-mon[220044]: failed to encode map e196 with expected crc 2023-10-09T05:54:55.815 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:55 smithi064 ceph-mon[220044]: failed to encode map e196 with expected crc 2023-10-09T05:54:55.815 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:55 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3937361400' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:55.843 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:55.889 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:56.017 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:54:56.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:55 smithi156 ceph-mon[195101]: pgmap v446: 119 pgs: 2 activating+undersized+degraded+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 peering, 2 unknown, 2 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.10k op/s; 27912/257826 objects degraded (10.826%); 11145/257826 objects misplaced (4.323%); 26 MiB/s, 516 objects/s recovering 2023-10-09T05:54:56.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:55 smithi156 ceph-mon[195101]: osdmap e196: 8 total, 8 up, 8 in 2023-10-09T05:54:56.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:55 smithi156 ceph-mon[195101]: failed to encode map e196 with expected crc 2023-10-09T05:54:56.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:55 smithi156 ceph-mon[195101]: failed to encode map e196 with expected crc 2023-10-09T05:54:56.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:55 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3937361400' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:54:57.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:56 smithi156 ceph-mon[195101]: failed to encode map e196 with expected crc 2023-10-09T05:54:57.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:56 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:57.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:56 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3937361400' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:54:57.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:56 smithi156 ceph-mon[195101]: osdmap e197: 8 total, 8 up, 8 in 2023-10-09T05:54:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:56 smithi156 ceph-mon[195101]: failed to encode map e197 with expected crc 2023-10-09T05:54:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:56 smithi156 ceph-mon[195101]: failed to encode map e197 with expected crc 2023-10-09T05:54:57.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:56 smithi064 ceph-mon[212750]: failed to encode map e196 with expected crc 2023-10-09T05:54:57.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:56 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:57.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:56 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3937361400' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:54:57.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:56 smithi064 ceph-mon[212750]: osdmap e197: 8 total, 8 up, 8 in 2023-10-09T05:54:57.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:56 smithi064 ceph-mon[212750]: failed to encode map e197 with expected crc 2023-10-09T05:54:57.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:56 smithi064 ceph-mon[212750]: failed to encode map e197 with expected crc 2023-10-09T05:54:57.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:56 smithi064 ceph-mon[220044]: failed to encode map e196 with expected crc 2023-10-09T05:54:57.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:56 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:54:57.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:56 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3937361400' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:54:57.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:56 smithi064 ceph-mon[220044]: osdmap e197: 8 total, 8 up, 8 in 2023-10-09T05:54:57.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:56 smithi064 ceph-mon[220044]: failed to encode map e197 with expected crc 2023-10-09T05:54:57.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:56 smithi064 ceph-mon[220044]: failed to encode map e197 with expected crc 2023-10-09T05:54:58.057 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:58.060 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:57 smithi064 ceph-mon[212750]: failed to encode map e197 with expected crc 2023-10-09T05:54:58.061 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:57 smithi064 ceph-mon[212750]: pgmap v449: 119 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 peering, 2 unknown, 2 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 26326/257826 objects degraded (10.211%); 11544/257826 objects misplaced (4.477%); 17 MiB/s, 312 objects/s recovering 2023-10-09T05:54:58.061 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:57 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 26326/257826 objects degraded (10.211%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:58.061 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:57 smithi064 ceph-mon[220044]: failed to encode map e197 with expected crc 2023-10-09T05:54:58.061 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:57 smithi064 ceph-mon[220044]: pgmap v449: 119 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 peering, 2 unknown, 2 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 26326/257826 objects degraded (10.211%); 11544/257826 objects misplaced (4.477%); 17 MiB/s, 312 objects/s recovering 2023-10-09T05:54:58.061 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:57 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 26326/257826 objects degraded (10.211%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:58.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:58.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:57 smithi156 ceph-mon[195101]: failed to encode map e197 with expected crc 2023-10-09T05:54:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:57 smithi156 ceph-mon[195101]: pgmap v449: 119 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 peering, 2 unknown, 2 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 26326/257826 objects degraded (10.211%); 11544/257826 objects misplaced (4.477%); 17 MiB/s, 312 objects/s recovering 2023-10-09T05:54:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:57 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 26326/257826 objects degraded (10.211%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:54:58.163 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:54:58.167 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:54:58.202 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:58.377 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:54:58.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:58.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:54:58.482 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:54:58.508 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:58.551 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:54:58.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:58.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:54:58.745 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:58.775 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:59.260 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:59.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:59.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:59.364 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:59.390 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:59.434 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:59.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:59.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:59.514 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:59 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:54:59.514 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:59.515 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:54:59 smithi064 ceph-mon[212750]: pgmap v450: 119 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 peering, 2 unknown, 2 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 26326/257826 objects degraded (10.211%); 11544/257826 objects misplaced (4.477%); 14 MiB/s, 251 objects/s recovering 2023-10-09T05:54:59.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:59 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:54:59.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:59.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:54:59 smithi064 ceph-mon[220044]: pgmap v450: 119 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 peering, 2 unknown, 2 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 26326/257826 objects degraded (10.211%); 11544/257826 objects misplaced (4.477%); 14 MiB/s, 251 objects/s recovering 2023-10-09T05:54:59.540 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:54:59.579 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:59.664 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:59.691 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:54:59.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:59.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:54:59.716 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:59.796 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:54:59.820 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:59 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:54:59.820 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:54:59.820 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:54:59 smithi156 ceph-mon[195101]: pgmap v450: 119 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 peering, 2 unknown, 2 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 26326/257826 objects degraded (10.211%); 11544/257826 objects misplaced (4.477%); 14 MiB/s, 251 objects/s recovering 2023-10-09T05:54:59.822 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:54:59.864 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:54:59.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:54:59.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:54:59.969 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:00.029 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:00.099 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:00.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:00.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:00.204 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:00.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:00 smithi064 conmon[212726]: 2023-10-09T05:55:00.104+0000 7f8823f23700 -1 mon.a@0(leader).osd e198 definitely_dead 0 2023-10-09T05:55:00.247 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:00.365 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:00.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:00.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:00.470 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:00.485 INFO:teuthology.orchestra.run.smithi064.stderr:marked out osd.1. 2023-10-09T05:55:00.501 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:00.507 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:00 smithi064 ceph-mon[212750]: osdmap e198: 8 total, 8 up, 8 in 2023-10-09T05:55:00.508 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:00 smithi064 ceph-mon[212750]: failed to encode map e198 with expected crc 2023-10-09T05:55:00.508 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:00 smithi064 ceph-mon[212750]: failed to encode map e198 with expected crc 2023-10-09T05:55:00.508 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:00 smithi064 ceph-mon[212750]: failed to encode map e198 with expected crc 2023-10-09T05:55:00.508 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:00 smithi064 ceph-mon[212750]: failed to encode map e198 with expected crc 2023-10-09T05:55:00.508 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:00 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3511005686' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:55:00.508 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:00 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:55:00.508 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:00 smithi064 ceph-mon[212750]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T05:55:00.508 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:00 smithi064 ceph-mon[220044]: osdmap e198: 8 total, 8 up, 8 in 2023-10-09T05:55:00.508 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:00 smithi064 ceph-mon[220044]: failed to encode map e198 with expected crc 2023-10-09T05:55:00.508 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:00 smithi064 ceph-mon[220044]: failed to encode map e198 with expected crc 2023-10-09T05:55:00.509 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:00 smithi064 ceph-mon[220044]: failed to encode map e198 with expected crc 2023-10-09T05:55:00.509 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:00 smithi064 ceph-mon[220044]: failed to encode map e198 with expected crc 2023-10-09T05:55:00.509 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:00 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3511005686' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:55:00.509 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:00 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:55:00.509 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:00 smithi064 ceph-mon[220044]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T05:55:00.545 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:00.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:00.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:00.650 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:00.678 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:00.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:00 smithi156 ceph-mon[195101]: osdmap e198: 8 total, 8 up, 8 in 2023-10-09T05:55:00.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:00 smithi156 ceph-mon[195101]: failed to encode map e198 with expected crc 2023-10-09T05:55:00.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:00 smithi156 ceph-mon[195101]: failed to encode map e198 with expected crc 2023-10-09T05:55:00.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:00 smithi156 ceph-mon[195101]: failed to encode map e198 with expected crc 2023-10-09T05:55:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:00 smithi156 ceph-mon[195101]: failed to encode map e198 with expected crc 2023-10-09T05:55:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:00 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3511005686' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:55:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:00 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:55:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:00 smithi156 ceph-mon[195101]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T05:55:00.878 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:00.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:00.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:00.982 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:01.009 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:01.206 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:01.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:01.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:01.310 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:01.336 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:01.382 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:01.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:01.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:01.397 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:55:01.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:01 smithi064 ceph-mon[212750]: failed to encode map e199 with expected crc 2023-10-09T05:55:01.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:01 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T05:55:01.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:01 smithi064 ceph-mon[212750]: osdmap e199: 8 total, 8 up, 7 in 2023-10-09T05:55:01.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:01 smithi064 ceph-mon[212750]: failed to encode map e199 with expected crc 2023-10-09T05:55:01.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:01 smithi064 ceph-mon[212750]: failed to encode map e199 with expected crc 2023-10-09T05:55:01.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:01 smithi064 ceph-mon[212750]: failed to encode map e199 with expected crc 2023-10-09T05:55:01.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:01 smithi064 ceph-mon[212750]: failed to encode map e199 with expected crc 2023-10-09T05:55:01.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:01 smithi064 ceph-mon[212750]: failed to encode map e199 with expected crc 2023-10-09T05:55:01.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:01 smithi064 ceph-mon[212750]: pgmap v453: 119 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 107 active+clean, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 33 MiB/s wr, 568 op/s; 35345/266292 objects degraded (13.273%); 9831/266292 objects misplaced (3.692%); 25 MiB/s, 472 objects/s recovering 2023-10-09T05:55:01.477 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:01 smithi064 ceph-mon[220044]: failed to encode map e199 with expected crc 2023-10-09T05:55:01.477 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:01 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T05:55:01.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:01 smithi064 ceph-mon[220044]: osdmap e199: 8 total, 8 up, 7 in 2023-10-09T05:55:01.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:01 smithi064 ceph-mon[220044]: failed to encode map e199 with expected crc 2023-10-09T05:55:01.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:01 smithi064 ceph-mon[220044]: failed to encode map e199 with expected crc 2023-10-09T05:55:01.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:01 smithi064 ceph-mon[220044]: failed to encode map e199 with expected crc 2023-10-09T05:55:01.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:01 smithi064 ceph-mon[220044]: failed to encode map e199 with expected crc 2023-10-09T05:55:01.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:01 smithi064 ceph-mon[220044]: failed to encode map e199 with expected crc 2023-10-09T05:55:01.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:01 smithi064 ceph-mon[220044]: pgmap v453: 119 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 107 active+clean, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 33 MiB/s wr, 568 op/s; 35345/266292 objects degraded (13.273%); 9831/266292 objects misplaced (3.692%); 25 MiB/s, 472 objects/s recovering 2023-10-09T05:55:01.490 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:01.544 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:01.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:01 smithi156 ceph-mon[195101]: failed to encode map e199 with expected crc 2023-10-09T05:55:01.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:01 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T05:55:01.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:01 smithi156 ceph-mon[195101]: osdmap e199: 8 total, 8 up, 7 in 2023-10-09T05:55:01.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:01 smithi156 ceph-mon[195101]: failed to encode map e199 with expected crc 2023-10-09T05:55:01.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:01 smithi156 ceph-mon[195101]: failed to encode map e199 with expected crc 2023-10-09T05:55:01.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:01 smithi156 ceph-mon[195101]: failed to encode map e199 with expected crc 2023-10-09T05:55:01.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:01 smithi156 ceph-mon[195101]: failed to encode map e199 with expected crc 2023-10-09T05:55:01.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:01 smithi156 ceph-mon[195101]: failed to encode map e199 with expected crc 2023-10-09T05:55:01.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:01 smithi156 ceph-mon[195101]: pgmap v453: 119 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 107 active+clean, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 33 MiB/s wr, 568 op/s; 35345/266292 objects degraded (13.273%); 9831/266292 objects misplaced (3.692%); 25 MiB/s, 472 objects/s recovering 2023-10-09T05:55:01.942 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:01.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:01.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:02.046 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:02.073 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:02.525 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:02.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:02.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:02.629 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:02.682 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:02.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[212750]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[212750]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[212750]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[212750]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[212750]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[212750]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[212750]: osdmap e200: 8 total, 8 up, 7 in 2023-10-09T05:55:02.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[212750]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[212750]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[212750]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[212750]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[212750]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[212750]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[212750]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[212750]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[212750]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 35345/266292 objects degraded (13.273%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:02.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[220044]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[220044]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[220044]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[220044]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[220044]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[220044]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[220044]: osdmap e200: 8 total, 8 up, 7 in 2023-10-09T05:55:02.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[220044]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[220044]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[220044]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[220044]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[220044]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[220044]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[220044]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[220044]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[220044]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:02 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 35345/266292 objects degraded (13.273%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:02.758 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:02.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:02.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:02.773 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:02 smithi156 ceph-mon[195101]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.773 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:02 smithi156 ceph-mon[195101]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.773 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:02 smithi156 ceph-mon[195101]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.773 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:02 smithi156 ceph-mon[195101]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.774 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:02 smithi156 ceph-mon[195101]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.774 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:02 smithi156 ceph-mon[195101]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.774 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:02 smithi156 ceph-mon[195101]: osdmap e200: 8 total, 8 up, 7 in 2023-10-09T05:55:02.774 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:02 smithi156 ceph-mon[195101]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.774 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:02 smithi156 ceph-mon[195101]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.774 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:02 smithi156 ceph-mon[195101]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.774 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:02 smithi156 ceph-mon[195101]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.774 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:02 smithi156 ceph-mon[195101]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.774 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:02 smithi156 ceph-mon[195101]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.774 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:02 smithi156 ceph-mon[195101]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.774 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:02 smithi156 ceph-mon[195101]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.774 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:02 smithi156 ceph-mon[195101]: failed to encode map e200 with expected crc 2023-10-09T05:55:02.775 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:02 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 35345/266292 objects degraded (13.273%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:02.870 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:02.900 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:03.453 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:03.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:03.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:03.556 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:03.583 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:03.584 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:03.634 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:03.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:03.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:03.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[212750]: osdmap e201: 8 total, 8 up, 7 in 2023-10-09T05:55:03.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[212750]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[212750]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[212750]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[212750]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[212750]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[212750]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[212750]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[212750]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[212750]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[212750]: 6.e starting backfill to osd.7 from (0'0,0'0] MAX to 191'6469 2023-10-09T05:55:03.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[212750]: 6.d starting backfill to osd.2 from (0'0,132'5669] MIN to 191'6490 2023-10-09T05:55:03.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[212750]: 6.1 starting backfill to osd.0 from (106'204,162'6531] MIN to 186'6568 2023-10-09T05:55:03.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[212750]: 6.4 starting backfill to osd.4 from (0'0,0'0] MAX to 186'6562 2023-10-09T05:55:03.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[212750]: 6.11 starting backfill to osd.0 from (106'204,162'6531] MIN to 186'6568 2023-10-09T05:55:03.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[212750]: 6.f starting backfill to osd.5 from (0'0,0'0] MAX to 189'6609 2023-10-09T05:55:03.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[212750]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[212750]: pgmap v456: 119 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 107 active+clean, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 48 MiB/s wr, 820 op/s; 35345/266292 objects degraded (13.273%); 9831/266292 objects misplaced (3.692%); 21 MiB/s, 385 objects/s recovering 2023-10-09T05:55:03.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[212750]: 6.14 starting backfill to osd.4 from (0'0,0'0] MAX to 186'6562 2023-10-09T05:55:03.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[220044]: osdmap e201: 8 total, 8 up, 7 in 2023-10-09T05:55:03.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[220044]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[220044]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[220044]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[220044]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[220044]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[220044]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[220044]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[220044]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[220044]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[220044]: 6.e starting backfill to osd.7 from (0'0,0'0] MAX to 191'6469 2023-10-09T05:55:03.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[220044]: 6.d starting backfill to osd.2 from (0'0,132'5669] MIN to 191'6490 2023-10-09T05:55:03.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[220044]: 6.1 starting backfill to osd.0 from (106'204,162'6531] MIN to 186'6568 2023-10-09T05:55:03.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[220044]: 6.4 starting backfill to osd.4 from (0'0,0'0] MAX to 186'6562 2023-10-09T05:55:03.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[220044]: 6.11 starting backfill to osd.0 from (106'204,162'6531] MIN to 186'6568 2023-10-09T05:55:03.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[220044]: 6.f starting backfill to osd.5 from (0'0,0'0] MAX to 189'6609 2023-10-09T05:55:03.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[220044]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[220044]: pgmap v456: 119 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 107 active+clean, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 48 MiB/s wr, 820 op/s; 35345/266292 objects degraded (13.273%); 9831/266292 objects misplaced (3.692%); 21 MiB/s, 385 objects/s recovering 2023-10-09T05:55:03.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:03 smithi064 ceph-mon[220044]: 6.14 starting backfill to osd.4 from (0'0,0'0] MAX to 186'6562 2023-10-09T05:55:03.739 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:03.776 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:03.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:03 smithi156 ceph-mon[195101]: osdmap e201: 8 total, 8 up, 7 in 2023-10-09T05:55:03.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:03 smithi156 ceph-mon[195101]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:03 smithi156 ceph-mon[195101]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:03 smithi156 ceph-mon[195101]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:03 smithi156 ceph-mon[195101]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:03 smithi156 ceph-mon[195101]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:03 smithi156 ceph-mon[195101]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:03 smithi156 ceph-mon[195101]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:03 smithi156 ceph-mon[195101]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:03 smithi156 ceph-mon[195101]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:03 smithi156 ceph-mon[195101]: 6.e starting backfill to osd.7 from (0'0,0'0] MAX to 191'6469 2023-10-09T05:55:03.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:03 smithi156 ceph-mon[195101]: 6.d starting backfill to osd.2 from (0'0,132'5669] MIN to 191'6490 2023-10-09T05:55:03.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:03 smithi156 ceph-mon[195101]: 6.1 starting backfill to osd.0 from (106'204,162'6531] MIN to 186'6568 2023-10-09T05:55:03.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:03 smithi156 ceph-mon[195101]: 6.4 starting backfill to osd.4 from (0'0,0'0] MAX to 186'6562 2023-10-09T05:55:03.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:03 smithi156 ceph-mon[195101]: 6.11 starting backfill to osd.0 from (106'204,162'6531] MIN to 186'6568 2023-10-09T05:55:03.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:03 smithi156 ceph-mon[195101]: 6.f starting backfill to osd.5 from (0'0,0'0] MAX to 189'6609 2023-10-09T05:55:03.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:03 smithi156 ceph-mon[195101]: failed to encode map e201 with expected crc 2023-10-09T05:55:03.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:03 smithi156 ceph-mon[195101]: pgmap v456: 119 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 107 active+clean, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 48 MiB/s wr, 820 op/s; 35345/266292 objects degraded (13.273%); 9831/266292 objects misplaced (3.692%); 21 MiB/s, 385 objects/s recovering 2023-10-09T05:55:03.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:03 smithi156 ceph-mon[195101]: 6.14 starting backfill to osd.4 from (0'0,0'0] MAX to 186'6562 2023-10-09T05:55:03.881 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:03.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:03.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:03.918 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:55:03.940 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:55:03.968 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:55:03.985 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:03.990 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:55:03.990 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T05:55:03.990 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T05:55:03.990 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T05:55:03.990 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:03.990 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82093 6.0 6.a3a5430 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:55:03.990 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.146555+0000", 2023-10-09T05:55:03.990 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 187.84117455200001, 2023-10-09T05:55:03.991 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93485150500000003, 2023-10-09T05:55:03.991 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:03.991 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:03.991 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:03.991 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:03.991 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:03.991 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82093 2023-10-09T05:55:03.992 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:03.992 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:03.992 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:03.992 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:03.992 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.146555+0000", 2023-10-09T05:55:03.992 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:03.992 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:03.992 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:03.992 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:03.993 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.146555+0000", 2023-10-09T05:55:03.993 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:03.993 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:03.993 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:03.993 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:03.993 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.146553+0000", 2023-10-09T05:55:03.993 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:03.993 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:03.993 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:03.994 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:03.994 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.146607+0000", 2023-10-09T05:55:03.994 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4317000000000002e-05 2023-10-09T05:55:03.994 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:03.994 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:03.994 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:03.994 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.146636+0000", 2023-10-09T05:55:03.994 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8294999999999999e-05 2023-10-09T05:55:03.995 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:03.995 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:03.995 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:03.995 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.146647+0000", 2023-10-09T05:55:03.995 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1513e-05 2023-10-09T05:55:03.995 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:03.995 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:03.995 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:03.995 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.150385+0000", 2023-10-09T05:55:03.995 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037372640000000001 2023-10-09T05:55:03.995 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:03.996 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:03.996 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:55:03.996 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.150398+0000", 2023-10-09T05:55:03.996 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.34e-05 2023-10-09T05:55:03.996 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:03.996 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:03.996 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:03.996 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081393+0000", 2023-10-09T05:55:03.996 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93099492699999997 2023-10-09T05:55:03.996 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:03.996 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:03.997 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:03.997 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081406+0000", 2023-10-09T05:55:03.997 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.359e-05 2023-10-09T05:55:03.997 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:03.997 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:03.997 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:03.997 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:03.997 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:03.997 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82107 6.0 6.432cde0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:55:03.997 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.173816+0000", 2023-10-09T05:55:03.997 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 187.81391317999999, 2023-10-09T05:55:03.997 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90762603100000006, 2023-10-09T05:55:03.998 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:03.998 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:03.998 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:03.998 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:03.998 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:03.998 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82107 2023-10-09T05:55:03.998 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:03.998 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:03.998 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:03.998 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:03.998 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173816+0000", 2023-10-09T05:55:03.999 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:03.999 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:03.999 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:03.999 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:03.999 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173816+0000", 2023-10-09T05:55:03.999 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:03.999 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:03.999 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:03.999 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:03.999 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173814+0000", 2023-10-09T05:55:03.999 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:55:03.999 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.000 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.000 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:04.000 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173867+0000", 2023-10-09T05:55:04.000 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2862999999999999e-05 2023-10-09T05:55:04.000 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.000 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.000 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:04.000 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173887+0000", 2023-10-09T05:55:04.000 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0347e-05 2023-10-09T05:55:04.000 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.000 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.000 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:04.001 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173907+0000", 2023-10-09T05:55:04.001 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9782000000000001e-05 2023-10-09T05:55:04.001 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.001 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.001 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.001 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173983+0000", 2023-10-09T05:55:04.001 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6056000000000005e-05 2023-10-09T05:55:04.001 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.001 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.001 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:55:04.001 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173996+0000", 2023-10-09T05:55:04.001 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3237999999999999e-05 2023-10-09T05:55:04.002 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.002 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.002 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.002 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081433+0000", 2023-10-09T05:55:04.002 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90743656100000003 2023-10-09T05:55:04.002 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.002 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.002 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:04.002 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081442+0000", 2023-10-09T05:55:04.002 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5070000000000006e-06 2023-10-09T05:55:04.002 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.003 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:04.003 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.003 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.003 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.003 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82109 6.0 6.cee0ea60 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:55:04.003 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.183890+0000", 2023-10-09T05:55:04.003 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 187.803839047, 2023-10-09T05:55:04.003 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.89758420800000005, 2023-10-09T05:55:04.003 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:04.003 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:04.003 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:04.003 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:04.004 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:04.004 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82109 2023-10-09T05:55:04.004 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.004 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:04.004 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.004 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:04.004 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.183890+0000", 2023-10-09T05:55:04.004 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.004 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.004 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.004 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:04.005 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.183890+0000", 2023-10-09T05:55:04.005 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.005 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.005 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.005 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:04.005 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.183889+0000", 2023-10-09T05:55:04.005 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:55:04.005 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.005 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.006 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:04.006 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.184035+0000", 2023-10-09T05:55:04.006 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000145375 2023-10-09T05:55:04.006 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.006 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.006 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:04.006 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.184048+0000", 2023-10-09T05:55:04.006 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3730999999999999e-05 2023-10-09T05:55:04.006 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.006 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.006 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:04.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.184060+0000", 2023-10-09T05:55:04.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1338000000000001e-05 2023-10-09T05:55:04.007 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.007 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.007 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.184099+0000", 2023-10-09T05:55:04.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9183000000000001e-05 2023-10-09T05:55:04.007 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.007 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.007 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:55:04.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.184113+0000", 2023-10-09T05:55:04.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3750999999999999e-05 2023-10-09T05:55:04.008 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.008 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.008 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.008 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081465+0000", 2023-10-09T05:55:04.008 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.89735238100000003 2023-10-09T05:55:04.008 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.008 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.008 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:04.008 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081475+0000", 2023-10-09T05:55:04.008 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7950000000000002e-06 2023-10-09T05:55:04.008 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.009 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:04.009 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.009 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.009 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.009 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82116 6.9 6.318f1fa9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:55:04.009 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.194331+0000", 2023-10-09T05:55:04.009 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 187.793398344, 2023-10-09T05:55:04.009 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.888951253, 2023-10-09T05:55:04.009 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:04.009 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:04.009 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:04.009 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:04.010 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:04.010 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82116 2023-10-09T05:55:04.010 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.010 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:04.010 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.010 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:04.010 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194331+0000", 2023-10-09T05:55:04.010 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.010 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.010 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.010 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:04.010 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194331+0000", 2023-10-09T05:55:04.011 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.011 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.011 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.011 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:04.011 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194329+0000", 2023-10-09T05:55:04.011 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:04.011 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.011 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.011 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:04.011 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194387+0000", 2023-10-09T05:55:04.011 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7734e-05 2023-10-09T05:55:04.011 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.012 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.012 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:04.012 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194406+0000", 2023-10-09T05:55:04.012 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9304e-05 2023-10-09T05:55:04.012 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.012 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.012 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:04.012 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194417+0000", 2023-10-09T05:55:04.012 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0803e-05 2023-10-09T05:55:04.013 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.013 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.013 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.013 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194490+0000", 2023-10-09T05:55:04.013 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2939000000000001e-05 2023-10-09T05:55:04.013 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.013 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.013 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:55:04.013 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194504+0000", 2023-10-09T05:55:04.013 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4358e-05 2023-10-09T05:55:04.013 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.013 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.014 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.014 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083265+0000", 2023-10-09T05:55:04.014 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88876104300000003 2023-10-09T05:55:04.014 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.014 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.014 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:04.014 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083282+0000", 2023-10-09T05:55:04.014 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6945000000000001e-05 2023-10-09T05:55:04.014 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.014 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:04.014 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.015 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.015 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.015 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82117 6.9 6.b6ae8c09 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:55:04.015 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.200001+0000", 2023-10-09T05:55:04.015 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 187.787728238, 2023-10-09T05:55:04.015 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88338774900000006, 2023-10-09T05:55:04.015 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:04.015 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:04.015 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:04.015 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:04.015 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:04.016 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82117 2023-10-09T05:55:04.016 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.016 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:04.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.016 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:04.016 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200001+0000", 2023-10-09T05:55:04.016 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.016 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.016 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:04.016 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200001+0000", 2023-10-09T05:55:04.016 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.017 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.017 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:04.017 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.199999+0000", 2023-10-09T05:55:04.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:04.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.017 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.017 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:04.017 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200060+0000", 2023-10-09T05:55:04.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0708999999999998e-05 2023-10-09T05:55:04.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.018 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:04.018 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200090+0000", 2023-10-09T05:55:04.018 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0085000000000001e-05 2023-10-09T05:55:04.018 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.018 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:04.018 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200101+0000", 2023-10-09T05:55:04.018 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.06e-05 2023-10-09T05:55:04.018 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.018 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.019 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200179+0000", 2023-10-09T05:55:04.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7848000000000004e-05 2023-10-09T05:55:04.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.019 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:55:04.019 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200192+0000", 2023-10-09T05:55:04.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3943e-05 2023-10-09T05:55:04.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.019 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.019 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083361+0000", 2023-10-09T05:55:04.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88316805099999995 2023-10-09T05:55:04.020 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:04.020 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083389+0000", 2023-10-09T05:55:04.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8532000000000001e-05 2023-10-09T05:55:04.020 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.020 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:04.020 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.020 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.020 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82118 6.9 6.aad68289 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:55:04.021 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.200117+0000", 2023-10-09T05:55:04.021 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 187.78761273699999, 2023-10-09T05:55:04.021 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88331848099999999, 2023-10-09T05:55:04.021 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:04.021 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:04.021 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:04.021 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:04.021 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:04.021 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82118 2023-10-09T05:55:04.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.021 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:04.021 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.022 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:04.022 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200117+0000", 2023-10-09T05:55:04.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.022 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.022 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.022 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:04.022 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200117+0000", 2023-10-09T05:55:04.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.022 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.022 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.022 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:04.022 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200116+0000", 2023-10-09T05:55:04.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:04.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:04.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200162+0000", 2023-10-09T05:55:04.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5788999999999998e-05 2023-10-09T05:55:04.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:04.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200174+0000", 2023-10-09T05:55:04.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1905999999999999e-05 2023-10-09T05:55:04.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.024 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.024 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:04.024 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200181+0000", 2023-10-09T05:55:04.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2690000000000002e-06 2023-10-09T05:55:04.024 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.024 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.024 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.024 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200224+0000", 2023-10-09T05:55:04.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2839000000000001e-05 2023-10-09T05:55:04.024 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.024 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.024 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:55:04.024 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200233+0000", 2023-10-09T05:55:04.025 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2490000000000007e-06 2023-10-09T05:55:04.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.025 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083420+0000", 2023-10-09T05:55:04.025 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88318651100000001 2023-10-09T05:55:04.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:04.025 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083435+0000", 2023-10-09T05:55:04.025 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5474000000000001e-05 2023-10-09T05:55:04.026 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.026 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:04.026 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.026 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82125 6.0 6.dade3890 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:55:04.026 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.207895+0000", 2023-10-09T05:55:04.026 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 187.779834534, 2023-10-09T05:55:04.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.873606989, 2023-10-09T05:55:04.026 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:04.026 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:04.026 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:04.027 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:04.027 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:04.027 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82125 2023-10-09T05:55:04.027 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.027 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:04.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:04.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.207895+0000", 2023-10-09T05:55:04.027 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.027 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:04.028 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.207895+0000", 2023-10-09T05:55:04.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:04.028 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.207893+0000", 2023-10-09T05:55:04.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:55:04.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:04.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.207955+0000", 2023-10-09T05:55:04.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2361999999999997e-05 2023-10-09T05:55:04.030 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.030 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:04.031 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.207982+0000", 2023-10-09T05:55:04.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6838e-05 2023-10-09T05:55:04.031 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.031 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.031 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:04.031 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.207995+0000", 2023-10-09T05:55:04.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2992e-05 2023-10-09T05:55:04.032 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.032 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.032 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.032 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.208064+0000", 2023-10-09T05:55:04.032 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9366e-05 2023-10-09T05:55:04.032 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.032 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.032 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:55:04.032 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.208080+0000", 2023-10-09T05:55:04.032 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5690000000000001e-05 2023-10-09T05:55:04.032 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.033 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.033 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.033 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081493+0000", 2023-10-09T05:55:04.033 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.87341295699999999 2023-10-09T05:55:04.033 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.033 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.033 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:04.033 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081502+0000", 2023-10-09T05:55:04.033 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1030000000000008e-06 2023-10-09T05:55:04.033 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.033 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:04.034 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.034 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.034 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.034 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82137 6.0 6.90177890 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:55:04.034 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.236426+0000", 2023-10-09T05:55:04.034 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 187.751303361, 2023-10-09T05:55:04.034 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.84511718000000002, 2023-10-09T05:55:04.034 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:04.034 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:04.034 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:04.034 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:04.034 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:04.035 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82137 2023-10-09T05:55:04.035 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.035 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:04.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.035 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:04.035 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236426+0000", 2023-10-09T05:55:04.035 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.035 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.035 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:04.035 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236426+0000", 2023-10-09T05:55:04.035 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.036 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.036 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.036 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:04.036 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236424+0000", 2023-10-09T05:55:04.036 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:04.036 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.036 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.036 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:04.036 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236475+0000", 2023-10-09T05:55:04.036 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0819999999999998e-05 2023-10-09T05:55:04.036 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.037 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.037 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:04.037 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236499+0000", 2023-10-09T05:55:04.037 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3669e-05 2023-10-09T05:55:04.037 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.037 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.037 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:04.037 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236510+0000", 2023-10-09T05:55:04.037 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1258e-05 2023-10-09T05:55:04.037 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.037 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.037 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.038 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236585+0000", 2023-10-09T05:55:04.038 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4828999999999998e-05 2023-10-09T05:55:04.038 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.038 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.038 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:55:04.038 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236601+0000", 2023-10-09T05:55:04.038 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6308999999999999e-05 2023-10-09T05:55:04.038 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.038 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.038 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.038 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081531+0000", 2023-10-09T05:55:04.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.84493010099999999 2023-10-09T05:55:04.039 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.039 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.039 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:04.039 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081543+0000", 2023-10-09T05:55:04.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2048000000000001e-05 2023-10-09T05:55:04.039 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.039 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:04.039 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.039 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.041 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.041 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82139 6.0 6.c02e1490 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:55:04.042 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.237890+0000", 2023-10-09T05:55:04.042 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 187.74983999, 2023-10-09T05:55:04.042 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.84368548799999998, 2023-10-09T05:55:04.042 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:04.042 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:04.042 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:04.042 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:04.042 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:04.042 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82139 2023-10-09T05:55:04.042 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.043 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:04.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.043 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:04.043 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.237890+0000", 2023-10-09T05:55:04.043 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.043 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.043 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:04.043 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.237890+0000", 2023-10-09T05:55:04.043 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.043 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.044 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.044 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:04.044 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.237889+0000", 2023-10-09T05:55:04.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T05:55:04.044 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.044 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.044 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:04.044 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.237927+0000", 2023-10-09T05:55:04.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8469e-05 2023-10-09T05:55:04.044 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.044 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.044 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:04.045 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.237940+0000", 2023-10-09T05:55:04.045 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2994e-05 2023-10-09T05:55:04.045 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.045 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.045 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:04.045 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.237947+0000", 2023-10-09T05:55:04.045 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.652e-06 2023-10-09T05:55:04.045 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.045 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.045 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.046 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.237996+0000", 2023-10-09T05:55:04.046 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8798000000000003e-05 2023-10-09T05:55:04.046 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.046 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.046 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:55:04.046 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.238008+0000", 2023-10-09T05:55:04.046 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2527000000000001e-05 2023-10-09T05:55:04.046 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.046 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.046 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.046 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081565+0000", 2023-10-09T05:55:04.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.84355716700000005 2023-10-09T05:55:04.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:04.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081575+0000", 2023-10-09T05:55:04.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7329999999999998e-06 2023-10-09T05:55:04.047 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.047 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:04.047 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.048 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82142 6.9 6.3a5bcf49 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:55:04.048 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.245410+0000", 2023-10-09T05:55:04.048 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 187.742319227, 2023-10-09T05:55:04.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.83812356099999996, 2023-10-09T05:55:04.048 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:04.048 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:04.048 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:04.048 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:04.048 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:04.048 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82142 2023-10-09T05:55:04.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.049 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:04.049 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.049 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:04.049 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.245410+0000", 2023-10-09T05:55:04.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.049 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.049 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.049 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:04.049 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.245410+0000", 2023-10-09T05:55:04.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.049 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.050 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.050 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:04.050 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.245409+0000", 2023-10-09T05:55:04.050 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:04.050 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.050 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.050 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:04.050 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.245459+0000", 2023-10-09T05:55:04.050 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0399000000000003e-05 2023-10-09T05:55:04.050 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.050 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.051 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:04.051 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.245479+0000", 2023-10-09T05:55:04.051 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9783999999999999e-05 2023-10-09T05:55:04.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.051 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:04.051 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.245488+0000", 2023-10-09T05:55:04.051 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3899999999999999e-06 2023-10-09T05:55:04.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.051 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.052 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.245546+0000", 2023-10-09T05:55:04.052 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7602999999999997e-05 2023-10-09T05:55:04.052 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.052 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.052 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:55:04.052 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.245555+0000", 2023-10-09T05:55:04.052 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2439999999999999e-06 2023-10-09T05:55:04.052 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.052 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.052 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.052 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083520+0000", 2023-10-09T05:55:04.053 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.83796473599999999 2023-10-09T05:55:04.053 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.053 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.053 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:04.053 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083534+0000", 2023-10-09T05:55:04.053 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4087000000000001e-05 2023-10-09T05:55:04.053 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.053 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:04.053 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.053 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.053 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.053 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:102624 6.9 6:9974b320:::benchmark_data_smithi064_284193_object6081:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-10-09T05:55:04.054 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:53:21.186314+0000", 2023-10-09T05:55:04.054 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 102.801415793, 2023-10-09T05:55:04.054 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.39315317799999999, 2023-10-09T05:55:04.054 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:04.054 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:04.054 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:04.054 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:04.054 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:04.054 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 102624 2023-10-09T05:55:04.054 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.054 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:04.054 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.055 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:04.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:21.186314+0000", 2023-10-09T05:55:04.055 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.055 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.055 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.055 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:04.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:21.186314+0000", 2023-10-09T05:55:04.055 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.055 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.055 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.055 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:04.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:21.186311+0000", 2023-10-09T05:55:04.056 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:55:04.056 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.056 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.056 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:04.056 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:21.186327+0000", 2023-10-09T05:55:04.056 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5648e-05 2023-10-09T05:55:04.056 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.056 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.056 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:04.056 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:21.186353+0000", 2023-10-09T05:55:04.057 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6367999999999999e-05 2023-10-09T05:55:04.057 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.057 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.057 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:04.057 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:21.186367+0000", 2023-10-09T05:55:04.057 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3491e-05 2023-10-09T05:55:04.057 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.057 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.057 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.057 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:21.214944+0000", 2023-10-09T05:55:04.057 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.02857703 2023-10-09T05:55:04.058 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.058 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.058 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting_for_map not empty", 2023-10-09T05:55:04.058 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:21.216061+0000", 2023-10-09T05:55:04.058 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00111696 2023-10-09T05:55:04.058 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.058 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.058 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.058 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:21.402805+0000", 2023-10-09T05:55:04.058 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18674426 2023-10-09T05:55:04.059 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.059 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.059 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:04.059 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:21.424706+0000", 2023-10-09T05:55:04.059 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021900890999999999 2023-10-09T05:55:04.059 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.059 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.059 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:55:04.059 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:21.449290+0000", 2023-10-09T05:55:04.059 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.024584511 2023-10-09T05:55:04.059 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.060 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.060 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:04.060 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:21.475792+0000", 2023-10-09T05:55:04.060 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.026501857 2023-10-09T05:55:04.060 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.060 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.060 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:04.060 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:21.480285+0000", 2023-10-09T05:55:04.060 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044932729999999999 2023-10-09T05:55:04.060 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.060 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.061 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:04.061 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:21.548219+0000", 2023-10-09T05:55:04.061 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.067933761999999995 2023-10-09T05:55:04.061 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.061 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.061 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:04.061 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:21.554051+0000", 2023-10-09T05:55:04.061 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058314719999999999 2023-10-09T05:55:04.061 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.061 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.061 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:04.062 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:21.579467+0000", 2023-10-09T05:55:04.062 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.025416180999999999 2023-10-09T05:55:04.062 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.062 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:04.062 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.062 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.062 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.062 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104164 6.9 6:9795f466:::benchmark_data_smithi064_284193_object7621:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:55:04.062 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:40.474997+0000", 2023-10-09T05:55:04.063 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 23.512732252999999, 2023-10-09T05:55:04.063 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6425455449999999, 2023-10-09T05:55:04.063 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:04.063 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:04.063 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:04.063 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:04.063 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:04.063 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104164 2023-10-09T05:55:04.063 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.063 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:04.063 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.065 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:04.065 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.474997+0000", 2023-10-09T05:55:04.065 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.065 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.065 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:04.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.474997+0000", 2023-10-09T05:55:04.066 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:04.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.474995+0000", 2023-10-09T05:55:04.066 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:55:04.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:04.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.475006+0000", 2023-10-09T05:55:04.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1127e-05 2023-10-09T05:55:04.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:04.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.475022+0000", 2023-10-09T05:55:04.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6073999999999999e-05 2023-10-09T05:55:04.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:04.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.475034+0000", 2023-10-09T05:55:04.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.183e-05 2023-10-09T05:55:04.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.068 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.701067+0000", 2023-10-09T05:55:04.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2260326539999999 2023-10-09T05:55:04.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.068 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:55:04.069 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.733298+0000", 2023-10-09T05:55:04.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.032231152999999998 2023-10-09T05:55:04.069 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.069 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.069 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.911525+0000", 2023-10-09T05:55:04.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.178227464 2023-10-09T05:55:04.069 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.069 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:04.069 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.921036+0000", 2023-10-09T05:55:04.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.009510447 2023-10-09T05:55:04.070 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.070 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:55:04.071 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.934794+0000", 2023-10-09T05:55:04.071 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013758014000000001 2023-10-09T05:55:04.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.071 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:04.071 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.041761+0000", 2023-10-09T05:55:04.071 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.10696707699999999 2023-10-09T05:55:04.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.071 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:04.071 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.078719+0000", 2023-10-09T05:55:04.071 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.036957894999999998 2023-10-09T05:55:04.072 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.072 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.072 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:04.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.081584+0000", 2023-10-09T05:55:04.072 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002865233 2023-10-09T05:55:04.072 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.072 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.072 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:04.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.085706+0000", 2023-10-09T05:55:04.072 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041217950000000001 2023-10-09T05:55:04.072 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.073 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:04.073 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.117543+0000", 2023-10-09T05:55:04.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.031837034 2023-10-09T05:55:04.073 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.073 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:04.073 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.073 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104221 6.9 6:992da745:::benchmark_data_smithi064_284193_object7678:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:55:04.073 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:41.522885+0000", 2023-10-09T05:55:04.073 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 22.464844970000001, 2023-10-09T05:55:04.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.645910236, 2023-10-09T05:55:04.074 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:04.074 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:04.074 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:04.074 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:04.074 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:04.074 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104221 2023-10-09T05:55:04.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.074 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:04.074 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:04.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.522885+0000", 2023-10-09T05:55:04.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.075 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.075 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:04.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.522885+0000", 2023-10-09T05:55:04.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.075 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.075 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:04.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.522883+0000", 2023-10-09T05:55:04.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:04.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.076 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.076 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:04.076 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.522890+0000", 2023-10-09T05:55:04.076 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1389999999999998e-06 2023-10-09T05:55:04.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.076 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.076 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:04.076 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.522906+0000", 2023-10-09T05:55:04.076 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6056e-05 2023-10-09T05:55:04.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.076 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.077 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:04.077 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.522916+0000", 2023-10-09T05:55:04.077 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0098999999999999e-05 2023-10-09T05:55:04.077 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.077 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.077 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.077 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.734737+0000", 2023-10-09T05:55:04.077 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.211820541 2023-10-09T05:55:04.077 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.077 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.077 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:55:04.077 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.756962+0000", 2023-10-09T05:55:04.078 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.022225556 2023-10-09T05:55:04.078 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.078 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.078 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.078 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.001519+0000", 2023-10-09T05:55:04.078 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24455652899999999 2023-10-09T05:55:04.078 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.078 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.078 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:04.078 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.008693+0000", 2023-10-09T05:55:04.078 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0071741890000000001 2023-10-09T05:55:04.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:55:04.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.024456+0000", 2023-10-09T05:55:04.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015762548000000001 2023-10-09T05:55:04.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:04.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.075878+0000", 2023-10-09T05:55:04.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.051422622000000001 2023-10-09T05:55:04.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:04.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.122918+0000", 2023-10-09T05:55:04.080 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.047039865 2023-10-09T05:55:04.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:04.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.127710+0000", 2023-10-09T05:55:04.080 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047916829999999997 2023-10-09T05:55:04.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:04.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.136431+0000", 2023-10-09T05:55:04.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0087210410000000006 2023-10-09T05:55:04.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:04.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.168795+0000", 2023-10-09T05:55:04.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.032364048999999999 2023-10-09T05:55:04.081 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.081 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:04.081 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.082 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.082 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.082 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104231 6.9 6:9a9d673b:::benchmark_data_smithi064_284193_object7688:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:55:04.082 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:41.584372+0000", 2023-10-09T05:55:04.082 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 22.403357912000001, 2023-10-09T05:55:04.082 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1037428409999999, 2023-10-09T05:55:04.082 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:04.082 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:04.082 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:04.083 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:04.083 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:04.083 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104231 2023-10-09T05:55:04.083 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.083 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:04.083 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.083 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:04.083 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.584372+0000", 2023-10-09T05:55:04.083 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.083 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.084 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.084 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:04.084 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.584372+0000", 2023-10-09T05:55:04.084 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.084 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.084 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.084 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:04.084 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.584370+0000", 2023-10-09T05:55:04.084 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:55:04.084 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.085 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.085 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:04.085 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.584377+0000", 2023-10-09T05:55:04.085 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6560000000000003e-06 2023-10-09T05:55:04.085 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.085 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.085 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:04.085 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.584391+0000", 2023-10-09T05:55:04.085 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3842000000000001e-05 2023-10-09T05:55:04.085 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.086 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:04.086 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.584398+0000", 2023-10-09T05:55:04.086 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.34e-06 2023-10-09T05:55:04.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.086 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.086 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.170598+0000", 2023-10-09T05:55:04.086 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5861996390000002 2023-10-09T05:55:04.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.086 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:55:04.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.208581+0000", 2023-10-09T05:55:04.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.037983153999999998 2023-10-09T05:55:04.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.354464+0000", 2023-10-09T05:55:04.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14588357699999999 2023-10-09T05:55:04.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:04.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.362830+0000", 2023-10-09T05:55:04.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0083655650000000002 2023-10-09T05:55:04.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:55:04.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.387897+0000", 2023-10-09T05:55:04.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.025067605999999999 2023-10-09T05:55:04.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:04.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.541824+0000", 2023-10-09T05:55:04.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15392655499999999 2023-10-09T05:55:04.089 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.089 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.089 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:04.089 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.545759+0000", 2023-10-09T05:55:04.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039354539999999997 2023-10-09T05:55:04.089 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.089 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.089 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:04.089 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.640753+0000", 2023-10-09T05:55:04.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.094993853000000003 2023-10-09T05:55:04.089 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.090 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.090 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:04.090 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.650699+0000", 2023-10-09T05:55:04.090 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0099456360000000008 2023-10-09T05:55:04.090 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.090 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.090 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:04.090 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.688114+0000", 2023-10-09T05:55:04.090 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.037415551999999998 2023-10-09T05:55:04.090 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.091 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:04.091 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.091 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.091 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.091 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104288 6.9 6:97b881c9:::benchmark_data_smithi064_284193_object7745:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:55:04.091 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:42.618333+0000", 2023-10-09T05:55:04.091 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 21.369396046999999, 2023-10-09T05:55:04.091 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1678057799999999, 2023-10-09T05:55:04.091 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:04.091 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:04.092 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:04.092 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:04.092 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:04.092 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104288 2023-10-09T05:55:04.092 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.092 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:04.092 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.092 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:04.092 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.618333+0000", 2023-10-09T05:55:04.092 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.092 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.093 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.093 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:04.093 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.618333+0000", 2023-10-09T05:55:04.093 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.093 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.093 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.093 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:04.093 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.618331+0000", 2023-10-09T05:55:04.093 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:04.093 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.093 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.093 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:04.094 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.618341+0000", 2023-10-09T05:55:04.094 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3049999999999998e-06 2023-10-09T05:55:04.094 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.094 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.094 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:04.094 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.618366+0000", 2023-10-09T05:55:04.094 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5198000000000002e-05 2023-10-09T05:55:04.094 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.094 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.094 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:04.095 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.618379+0000", 2023-10-09T05:55:04.095 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2683e-05 2023-10-09T05:55:04.095 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.095 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.095 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.095 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.210649+0000", 2023-10-09T05:55:04.095 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.59226995500000001 2023-10-09T05:55:04.095 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.095 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.095 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:55:04.095 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.258221+0000", 2023-10-09T05:55:04.095 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.047572956 2023-10-09T05:55:04.096 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.096 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.096 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.096 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.480425+0000", 2023-10-09T05:55:04.096 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.22220395700000001 2023-10-09T05:55:04.096 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.096 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.096 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:04.096 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.495694+0000", 2023-10-09T05:55:04.096 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015268805999999999 2023-10-09T05:55:04.096 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.096 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.097 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:55:04.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.524222+0000", 2023-10-09T05:55:04.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.028527800999999998 2023-10-09T05:55:04.097 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.097 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.097 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:04.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.593453+0000", 2023-10-09T05:55:04.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.069230594000000006 2023-10-09T05:55:04.097 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.097 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.097 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:04.098 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.732762+0000", 2023-10-09T05:55:04.098 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.139309778 2023-10-09T05:55:04.098 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.098 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.098 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:04.098 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.738937+0000", 2023-10-09T05:55:04.098 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061743409999999999 2023-10-09T05:55:04.098 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.098 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.098 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:04.098 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.749997+0000", 2023-10-09T05:55:04.098 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011059897000000001 2023-10-09T05:55:04.099 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.099 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.099 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:04.099 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.786139+0000", 2023-10-09T05:55:04.099 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.036142625999999997 2023-10-09T05:55:04.099 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.099 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:04.099 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.099 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.099 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.099 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104292 6.9 6:907bb29e:::benchmark_data_smithi064_284193_object7749:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:55:04.100 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:43.057966+0000", 2023-10-09T05:55:04.100 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 20.929763824999998, 2023-10-09T05:55:04.100 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.78572469, 2023-10-09T05:55:04.100 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:04.100 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:04.100 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:04.100 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:04.100 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:04.100 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104292 2023-10-09T05:55:04.100 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.100 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:04.100 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.100 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:04.101 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.057966+0000", 2023-10-09T05:55:04.101 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.101 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.101 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.101 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:04.101 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.057966+0000", 2023-10-09T05:55:04.101 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.101 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.101 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.101 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:04.101 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.057963+0000", 2023-10-09T05:55:04.102 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999971 2023-10-09T05:55:04.102 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.102 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.102 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:04.102 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.057975+0000", 2023-10-09T05:55:04.102 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1848000000000001e-05 2023-10-09T05:55:04.102 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.102 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.102 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:04.102 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.057999+0000", 2023-10-09T05:55:04.102 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4148999999999999e-05 2023-10-09T05:55:04.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.103 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:04.103 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.058013+0000", 2023-10-09T05:55:04.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4307999999999999e-05 2023-10-09T05:55:04.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.103 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.103 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.259939+0000", 2023-10-09T05:55:04.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20192557699999999 2023-10-09T05:55:04.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.104 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:55:04.104 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.291642+0000", 2023-10-09T05:55:04.104 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.031703535999999997 2023-10-09T05:55:04.104 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.104 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.104 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.104 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.594231+0000", 2023-10-09T05:55:04.104 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.302588156 2023-10-09T05:55:04.104 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.104 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.104 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:04.105 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.601468+0000", 2023-10-09T05:55:04.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0072377839999999997 2023-10-09T05:55:04.105 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.105 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.105 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:55:04.105 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.613837+0000", 2023-10-09T05:55:04.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012368183 2023-10-09T05:55:04.105 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.105 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.105 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:04.105 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.635010+0000", 2023-10-09T05:55:04.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021173714 2023-10-09T05:55:04.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:04.106 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.789547+0000", 2023-10-09T05:55:04.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15453652500000001 2023-10-09T05:55:04.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:04.106 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.795226+0000", 2023-10-09T05:55:04.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056790080000000001 2023-10-09T05:55:04.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.107 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:04.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.804824+0000", 2023-10-09T05:55:04.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0095979610000000003 2023-10-09T05:55:04.107 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.107 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:04.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.843690+0000", 2023-10-09T05:55:04.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.038866671999999998 2023-10-09T05:55:04.107 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.107 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:04.108 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.108 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.108 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.108 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104301 6.9 6:922882d7:::benchmark_data_smithi064_284193_object7758:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:55:04.108 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:43.650961+0000", 2023-10-09T05:55:04.108 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 20.336768503999998, 2023-10-09T05:55:04.108 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.61580340700000002, 2023-10-09T05:55:04.108 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:04.108 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:04.108 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:04.108 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:04.109 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:04.109 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104301 2023-10-09T05:55:04.109 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.109 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:04.109 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.109 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:04.109 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.650961+0000", 2023-10-09T05:55:04.109 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.109 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.109 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.109 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:04.110 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.650961+0000", 2023-10-09T05:55:04.110 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.110 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:04.110 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.650959+0000", 2023-10-09T05:55:04.110 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:04.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.110 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:04.110 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.650969+0000", 2023-10-09T05:55:04.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8460000000000003e-06 2023-10-09T05:55:04.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.111 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:04.111 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.650986+0000", 2023-10-09T05:55:04.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6823000000000001e-05 2023-10-09T05:55:04.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.111 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:04.111 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.650997+0000", 2023-10-09T05:55:04.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1372000000000001e-05 2023-10-09T05:55:04.112 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.112 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.112 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.112 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.845356+0000", 2023-10-09T05:55:04.112 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.194358586 2023-10-09T05:55:04.112 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.112 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.112 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:55:04.112 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.873088+0000", 2023-10-09T05:55:04.112 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.027732275000000001 2023-10-09T05:55:04.112 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.113 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.113 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.984384+0000", 2023-10-09T05:55:04.113 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.11129644399999999 2023-10-09T05:55:04.113 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.113 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:04.113 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.996530+0000", 2023-10-09T05:55:04.113 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01214578 2023-10-09T05:55:04.113 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.114 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.114 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:55:04.114 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.016821+0000", 2023-10-09T05:55:04.114 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020290618 2023-10-09T05:55:04.114 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.114 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.114 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:04.114 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.115869+0000", 2023-10-09T05:55:04.114 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.099047882000000004 2023-10-09T05:55:04.114 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.115 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.115 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:04.115 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.116896+0000", 2023-10-09T05:55:04.115 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010277179999999999 2023-10-09T05:55:04.115 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.115 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.115 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:04.115 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.235914+0000", 2023-10-09T05:55:04.115 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.119017107 2023-10-09T05:55:04.116 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.116 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.116 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:04.116 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.240966+0000", 2023-10-09T05:55:04.116 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.005052511 2023-10-09T05:55:04.116 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.116 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.116 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:04.116 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.266764+0000", 2023-10-09T05:55:04.116 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.025798384000000001 2023-10-09T05:55:04.117 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.117 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:04.117 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.117 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.117 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.117 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104305 6.9 6:9bd64923:::benchmark_data_smithi064_284193_object7762:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T05:55:04.117 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:43.773851+0000", 2023-10-09T05:55:04.117 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 20.213879012, 2023-10-09T05:55:04.117 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.52990133800000006, 2023-10-09T05:55:04.117 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:04.118 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:04.118 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:04.118 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:04.118 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:04.118 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104305 2023-10-09T05:55:04.118 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.118 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:04.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:04.118 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.773851+0000", 2023-10-09T05:55:04.119 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.119 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.119 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.119 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:04.119 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.773851+0000", 2023-10-09T05:55:04.119 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.119 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.119 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.119 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:04.119 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.773849+0000", 2023-10-09T05:55:04.119 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:04.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.120 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:04.120 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.773857+0000", 2023-10-09T05:55:04.120 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5380000000000002e-06 2023-10-09T05:55:04.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.120 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:04.120 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.773876+0000", 2023-10-09T05:55:04.120 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9100999999999999e-05 2023-10-09T05:55:04.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:04.121 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.773888+0000", 2023-10-09T05:55:04.121 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1603000000000001e-05 2023-10-09T05:55:04.121 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.121 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.121 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.875006+0000", 2023-10-09T05:55:04.121 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.10111836 2023-10-09T05:55:04.121 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.121 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:55:04.122 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.903882+0000", 2023-10-09T05:55:04.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.028875774999999999 2023-10-09T05:55:04.122 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.122 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.122 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.122 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.077070+0000", 2023-10-09T05:55:04.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.17318757200000001 2023-10-09T05:55:04.122 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.122 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.122 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:04.122 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.083728+0000", 2023-10-09T05:55:04.123 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0066586839999999998 2023-10-09T05:55:04.123 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.123 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.123 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:55:04.123 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.097067+0000", 2023-10-09T05:55:04.123 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013338164 2023-10-09T05:55:04.123 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.123 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.123 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:04.123 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.231479+0000", 2023-10-09T05:55:04.123 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.13441233499999999 2023-10-09T05:55:04.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:04.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.271687+0000", 2023-10-09T05:55:04.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.040208470000000003 2023-10-09T05:55:04.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:04.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.276739+0000", 2023-10-09T05:55:04.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050511369999999998 2023-10-09T05:55:04.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:04.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.282799+0000", 2023-10-09T05:55:04.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060605399999999997 2023-10-09T05:55:04.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:04.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.303752+0000", 2023-10-09T05:55:04.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020952765000000002 2023-10-09T05:55:04.125 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.126 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:04.126 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.126 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.126 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.126 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104316 6.9 6:90001cfe:::benchmark_data_smithi064_284193_object7773:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T05:55:04.126 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:43.865269+0000", 2023-10-09T05:55:04.126 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 20.122460037, 2023-10-09T05:55:04.126 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.47637473899999999, 2023-10-09T05:55:04.126 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:04.126 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:04.126 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:04.127 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:04.127 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:04.127 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104316 2023-10-09T05:55:04.127 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.127 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:04.127 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.127 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:04.127 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.865269+0000", 2023-10-09T05:55:04.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.127 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.127 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.128 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:04.128 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.865269+0000", 2023-10-09T05:55:04.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.128 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:04.128 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.865267+0000", 2023-10-09T05:55:04.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:04.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.128 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:04.128 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.865276+0000", 2023-10-09T05:55:04.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4710000000000007e-06 2023-10-09T05:55:04.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:04.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.865292+0000", 2023-10-09T05:55:04.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6337999999999998e-05 2023-10-09T05:55:04.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:04.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.865301+0000", 2023-10-09T05:55:04.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0000000000000002e-06 2023-10-09T05:55:04.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.904993+0000", 2023-10-09T05:55:04.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.039691883999999997 2023-10-09T05:55:04.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:55:04.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.925775+0000", 2023-10-09T05:55:04.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020781500000000001 2023-10-09T05:55:04.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.177722+0000", 2023-10-09T05:55:04.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25194717700000002 2023-10-09T05:55:04.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:04.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.187550+0000", 2023-10-09T05:55:04.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0098285830000000001 2023-10-09T05:55:04.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:55:04.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.211437+0000", 2023-10-09T05:55:04.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023886775999999998 2023-10-09T05:55:04.132 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:04.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.273480+0000", 2023-10-09T05:55:04.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.062042468000000003 2023-10-09T05:55:04.132 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:04.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.307335+0000", 2023-10-09T05:55:04.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.033855341999999997 2023-10-09T05:55:04.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.133 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:04.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.312165+0000", 2023-10-09T05:55:04.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048296270000000004 2023-10-09T05:55:04.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.133 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:04.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.319080+0000", 2023-10-09T05:55:04.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0069150829999999998 2023-10-09T05:55:04.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:04.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.341644+0000", 2023-10-09T05:55:04.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.022564489 2023-10-09T05:55:04.134 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.134 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:04.134 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.134 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104339 6.9 6:9dc5b700:::benchmark_data_smithi064_284193_object7796:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T05:55:04.135 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:44.329869+0000", 2023-10-09T05:55:04.135 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 19.657860868, 2023-10-09T05:55:04.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.43524301100000001, 2023-10-09T05:55:04.135 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:04.135 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:04.135 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:04.135 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:04.135 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:04.135 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104339 2023-10-09T05:55:04.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.135 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:04.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:04.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.329869+0000", 2023-10-09T05:55:04.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:04.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.329869+0000", 2023-10-09T05:55:04.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:04.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:04.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.329867+0000", 2023-10-09T05:55:04.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:04.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:04.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.329876+0000", 2023-10-09T05:55:04.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1209999999999996e-06 2023-10-09T05:55:04.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:04.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.329893+0000", 2023-10-09T05:55:04.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6911000000000001e-05 2023-10-09T05:55:04.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:04.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.329903+0000", 2023-10-09T05:55:04.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0258000000000001e-05 2023-10-09T05:55:04.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.385852+0000", 2023-10-09T05:55:04.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.055948848000000002 2023-10-09T05:55:04.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:55:04.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.405483+0000", 2023-10-09T05:55:04.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.019630901999999999 2023-10-09T05:55:04.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:04.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.567683+0000", 2023-10-09T05:55:04.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16220021700000001 2023-10-09T05:55:04.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:04.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.574706+0000", 2023-10-09T05:55:04.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0070234130000000001 2023-10-09T05:55:04.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:55:04.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.586926+0000", 2023-10-09T05:55:04.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012219747 2023-10-09T05:55:04.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:04.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.625334+0000", 2023-10-09T05:55:04.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.038407579999999997 2023-10-09T05:55:04.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:04.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.699852+0000", 2023-10-09T05:55:04.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.074518049000000003 2023-10-09T05:55:04.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:04.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.706499+0000", 2023-10-09T05:55:04.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0066477669999999997 2023-10-09T05:55:04.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.142 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:04.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.717321+0000", 2023-10-09T05:55:04.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010821651 2023-10-09T05:55:04.142 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:04.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:04.142 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:04.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.765112+0000", 2023-10-09T05:55:04.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.047790559000000003 2023-10-09T05:55:04.142 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.142 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:04.142 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.142 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:04.143 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:04.143 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:55:04.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:04.144 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:55:04.145 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:55:04.145 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:55:04.145 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:55:04.145 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:55:04.145 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:55:04.145 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:55:04.145 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:55:04.146 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:55:04.146 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:55:04.146 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:55:04.146 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:55:04.146 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:55:04.146 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:04.146 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:04.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:04.247 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:04.273 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:04.321 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:04.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:04.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:04.428 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:04.454 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:04.510 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:04.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:04.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:04.616 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:04.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:04 smithi156 ceph-mon[195101]: failed to encode map e202 with expected crc 2023-10-09T05:55:04.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:04 smithi156 ceph-mon[195101]: failed to encode map e202 with expected crc 2023-10-09T05:55:04.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:04 smithi156 ceph-mon[195101]: osdmap e202: 8 total, 8 up, 7 in 2023-10-09T05:55:04.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:04 smithi156 ceph-mon[195101]: failed to encode map e202 with expected crc 2023-10-09T05:55:04.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:04 smithi156 ceph-mon[195101]: failed to encode map e202 with expected crc 2023-10-09T05:55:04.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:04 smithi156 ceph-mon[195101]: failed to encode map e202 with expected crc 2023-10-09T05:55:04.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:04 smithi156 ceph-mon[195101]: failed to encode map e202 with expected crc 2023-10-09T05:55:04.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:04 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2190469795' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:04.643 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:04.694 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:04.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:04.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:04.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:04 smithi064 ceph-mon[212750]: failed to encode map e202 with expected crc 2023-10-09T05:55:04.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:04 smithi064 ceph-mon[212750]: failed to encode map e202 with expected crc 2023-10-09T05:55:04.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:04 smithi064 ceph-mon[212750]: osdmap e202: 8 total, 8 up, 7 in 2023-10-09T05:55:04.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:04 smithi064 ceph-mon[212750]: failed to encode map e202 with expected crc 2023-10-09T05:55:04.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:04 smithi064 ceph-mon[212750]: failed to encode map e202 with expected crc 2023-10-09T05:55:04.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:04 smithi064 ceph-mon[212750]: failed to encode map e202 with expected crc 2023-10-09T05:55:04.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:04 smithi064 ceph-mon[212750]: failed to encode map e202 with expected crc 2023-10-09T05:55:04.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:04 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2190469795' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:04.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:04 smithi064 ceph-mon[220044]: failed to encode map e202 with expected crc 2023-10-09T05:55:04.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:04 smithi064 ceph-mon[220044]: failed to encode map e202 with expected crc 2023-10-09T05:55:04.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:04 smithi064 ceph-mon[220044]: osdmap e202: 8 total, 8 up, 7 in 2023-10-09T05:55:04.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:04 smithi064 ceph-mon[220044]: failed to encode map e202 with expected crc 2023-10-09T05:55:04.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:04 smithi064 ceph-mon[220044]: failed to encode map e202 with expected crc 2023-10-09T05:55:04.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:04 smithi064 ceph-mon[220044]: failed to encode map e202 with expected crc 2023-10-09T05:55:04.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:04 smithi064 ceph-mon[220044]: failed to encode map e202 with expected crc 2023-10-09T05:55:04.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:04 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2190469795' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:04.800 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:04.831 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:05.333 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:05.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:05.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:05.436 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:05.461 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:05.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:55:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:55:05] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.43.0" 2023-10-09T05:55:05.507 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:05.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:05.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:05.612 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:05.645 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:05.664 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:55:05.730 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:05 smithi064 ceph-mon[212750]: failed to encode map e203 with expected crc 2023-10-09T05:55:05.730 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:05 smithi064 ceph-mon[212750]: failed to encode map e203 with expected crc 2023-10-09T05:55:05.730 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:05 smithi064 ceph-mon[212750]: failed to encode map e203 with expected crc 2023-10-09T05:55:05.730 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:05 smithi064 ceph-mon[212750]: osdmap e203: 8 total, 8 up, 7 in 2023-10-09T05:55:05.730 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:05 smithi064 ceph-mon[212750]: failed to encode map e203 with expected crc 2023-10-09T05:55:05.731 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:05 smithi064 ceph-mon[212750]: failed to encode map e203 with expected crc 2023-10-09T05:55:05.731 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:05 smithi064 ceph-mon[212750]: pgmap v459: 119 pgs: 1 peering, 3 activating, 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 102 active+clean, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 37454/266292 objects degraded (14.065%); 23036/266292 objects misplaced (8.651%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T05:55:05.731 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:05 smithi064 ceph-mon[220044]: failed to encode map e203 with expected crc 2023-10-09T05:55:05.731 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:05 smithi064 ceph-mon[220044]: failed to encode map e203 with expected crc 2023-10-09T05:55:05.731 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:05 smithi064 ceph-mon[220044]: failed to encode map e203 with expected crc 2023-10-09T05:55:05.731 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:05 smithi064 ceph-mon[220044]: osdmap e203: 8 total, 8 up, 7 in 2023-10-09T05:55:05.731 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:05 smithi064 ceph-mon[220044]: failed to encode map e203 with expected crc 2023-10-09T05:55:05.731 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:05 smithi064 ceph-mon[220044]: failed to encode map e203 with expected crc 2023-10-09T05:55:05.731 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:05 smithi064 ceph-mon[220044]: pgmap v459: 119 pgs: 1 peering, 3 activating, 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 102 active+clean, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 37454/266292 objects degraded (14.065%); 23036/266292 objects misplaced (8.651%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T05:55:05.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:05 smithi156 ceph-mon[195101]: failed to encode map e203 with expected crc 2023-10-09T05:55:05.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:05 smithi156 ceph-mon[195101]: failed to encode map e203 with expected crc 2023-10-09T05:55:05.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:05 smithi156 ceph-mon[195101]: failed to encode map e203 with expected crc 2023-10-09T05:55:05.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:05 smithi156 ceph-mon[195101]: osdmap e203: 8 total, 8 up, 7 in 2023-10-09T05:55:05.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:05 smithi156 ceph-mon[195101]: failed to encode map e203 with expected crc 2023-10-09T05:55:05.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:05 smithi156 ceph-mon[195101]: failed to encode map e203 with expected crc 2023-10-09T05:55:05.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:05 smithi156 ceph-mon[195101]: pgmap v459: 119 pgs: 1 peering, 3 activating, 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 102 active+clean, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 37454/266292 objects degraded (14.065%); 23036/266292 objects misplaced (8.651%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T05:55:06.177 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:06.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:06.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:06.284 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:06.304 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:06.310 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:06.353 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:06.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:06.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:06.458 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:06.491 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:06.693 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:06.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:06.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:06.796 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:06.822 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:06.866 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:06.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:06.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:06.973 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:07.007 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:07.061 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:07.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:07.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:07.165 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:07.191 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:07.239 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:07.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:07.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:07.344 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:07.373 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:07.510 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:07.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:07.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:07.610 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T05:55:07.613 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:07.640 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:07.643 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:07 smithi064 ceph-mon[212750]: pgmap v460: 119 pgs: 3 activating, 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 102 active+clean, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 43557/266292 objects degraded (16.357%); 23036/266292 objects misplaced (8.651%); 4.7 MiB/s, 76 objects/s recovering 2023-10-09T05:55:07.643 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:07 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2767097609' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:55:07.643 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:07 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:55:07.643 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:07 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 43557/266292 objects degraded (16.357%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:07.643 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:07 smithi064 ceph-mon[220044]: pgmap v460: 119 pgs: 3 activating, 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 102 active+clean, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 43557/266292 objects degraded (16.357%); 23036/266292 objects misplaced (8.651%); 4.7 MiB/s, 76 objects/s recovering 2023-10-09T05:55:07.643 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:07 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2767097609' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:55:07.643 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:07 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:55:07.643 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:07 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 43557/266292 objects degraded (16.357%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:07.685 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:07.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:07.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:07.791 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:07.826 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:07.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:07 smithi156 ceph-mon[195101]: pgmap v460: 119 pgs: 3 activating, 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 102 active+clean, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 43557/266292 objects degraded (16.357%); 23036/266292 objects misplaced (8.651%); 4.7 MiB/s, 76 objects/s recovering 2023-10-09T05:55:07.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:07 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2767097609' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:55:07.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:07 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:55:07.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:07 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 43557/266292 objects degraded (16.357%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:07.929 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:07.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:07.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:08.033 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:08.063 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:08.184 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:55:08.536 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T05:55:08.536 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:55:08.536 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_0 2023-10-09T05:55:08.537 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:55:08.572 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:55:08.572 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T05:55:08.572 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2023-10-09T05:55:08.572 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:55:08.651 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:08.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:08.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:08.769 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:08.790 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:08 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:08.791 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:08 smithi064 ceph-mon[220044]: failed to encode map e204 with expected crc 2023-10-09T05:55:08.791 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:08 smithi064 ceph-mon[220044]: failed to encode map e204 with expected crc 2023-10-09T05:55:08.791 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:08 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:55:08.791 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:08 smithi064 ceph-mon[220044]: osdmap e204: 8 total, 8 up, 7 in 2023-10-09T05:55:08.791 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:08 smithi064 ceph-mon[220044]: failed to encode map e204 with expected crc 2023-10-09T05:55:08.791 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:08 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:08.791 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:08 smithi064 ceph-mon[212750]: failed to encode map e204 with expected crc 2023-10-09T05:55:08.791 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:08 smithi064 ceph-mon[212750]: failed to encode map e204 with expected crc 2023-10-09T05:55:08.791 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:08 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:55:08.792 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:08 smithi064 ceph-mon[212750]: osdmap e204: 8 total, 8 up, 7 in 2023-10-09T05:55:08.792 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:08 smithi064 ceph-mon[212750]: failed to encode map e204 with expected crc 2023-10-09T05:55:08.813 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:08.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:08 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:08.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:08 smithi156 ceph-mon[195101]: failed to encode map e204 with expected crc 2023-10-09T05:55:08.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:08 smithi156 ceph-mon[195101]: failed to encode map e204 with expected crc 2023-10-09T05:55:08.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:08 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:55:08.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:08 smithi156 ceph-mon[195101]: osdmap e204: 8 total, 8 up, 7 in 2023-10-09T05:55:08.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:08 smithi156 ceph-mon[195101]: failed to encode map e204 with expected crc 2023-10-09T05:55:08.882 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:08.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:08.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:08.987 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:09.023 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:09.093 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:09.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:09.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:09.202 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:09.231 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:09.283 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:09.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:09.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:09.335 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:55:09.388 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:09.431 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:09.438 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:09.517 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:09.527 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:09.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:09.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:09.604 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:09 smithi064 ceph-mon[220044]: failed to encode map e204 with expected crc 2023-10-09T05:55:09.604 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:09 smithi064 ceph-mon[220044]: pgmap v462: 119 pgs: 3 activating, 2 activating+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 102 active+clean, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 43557/266292 objects degraded (16.357%); 26559/266292 objects misplaced (9.974%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T05:55:09.604 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:09 smithi064 ceph-mon[212750]: failed to encode map e204 with expected crc 2023-10-09T05:55:09.604 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:09 smithi064 ceph-mon[212750]: pgmap v462: 119 pgs: 3 activating, 2 activating+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 102 active+clean, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 43557/266292 objects degraded (16.357%); 26559/266292 objects misplaced (9.974%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T05:55:09.606 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:09 smithi156 ceph-mon[195101]: failed to encode map e204 with expected crc 2023-10-09T05:55:09.606 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:09 smithi156 ceph-mon[195101]: pgmap v462: 119 pgs: 3 activating, 2 activating+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 102 active+clean, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 43557/266292 objects degraded (16.357%); 26559/266292 objects misplaced (9.974%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T05:55:09.648 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:09.678 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:09.724 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:09.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:09.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:09.829 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:09.877 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:09.988 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:09.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:09.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:10.094 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:10.147 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:10.671 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:55:10.908 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:10.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:10.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:11.012 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:11.039 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:11.087 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:11.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:11.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:11.193 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:11.226 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:11.281 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:11.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:11.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:11.388 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:11.421 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:11.468 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:11.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:11.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:11.575 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:11.604 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:11.662 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:11.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:11.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:11.768 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:11.815 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:11 smithi064 ceph-mon[212750]: pgmap v463: 119 pgs: 2 activating+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 78 op/s; 43363/264636 objects degraded (16.386%); 26280/264636 objects misplaced (9.931%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T05:55:11.816 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:11 smithi064 ceph-mon[220044]: pgmap v463: 119 pgs: 2 activating+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 78 op/s; 43363/264636 objects degraded (16.386%); 26280/264636 objects misplaced (9.931%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T05:55:11.820 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:11.863 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:11.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:11 smithi156 ceph-mon[195101]: pgmap v463: 119 pgs: 2 activating+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 78 op/s; 43363/264636 objects degraded (16.386%); 26280/264636 objects misplaced (9.931%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T05:55:12.011 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:12.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:12.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:12.117 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:12.160 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:12.164 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:12.273 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:55:12.308 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:55:12.311 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T05:55:12.311 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:55:12.340 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:55:12.370 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:55:12.401 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:55:12.425 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:55:12.453 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:55:12.516 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:55:12.517 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:55:12.517 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:55:12.517 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:55:12.517 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:55:12.517 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:55:12.517 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:55:12.517 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:55:12.517 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:55:12.834 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:12.837 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:12 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 43363/264636 objects degraded (16.386%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:12.837 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:12 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2884275905' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:12.837 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:12 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 43363/264636 objects degraded (16.386%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:12.837 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:12 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2884275905' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:12.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:12.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:12.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:12 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 43363/264636 objects degraded (16.386%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:12.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:12 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2884275905' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:12.940 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:12.966 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:13.007 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:13.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:13.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:13.115 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:13.154 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:13.997 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":463,"stamp":"2023-10-09T05:55:10.536214+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5780019128,"num_objects":88212,"num_object_clones":0,"num_object_copies":264636,"num_objects_missing_on_primary":982,"num_objects_missing":982,"num_objects_degraded":43363,"num_objects_misplaced":26280,"num_objects_unfound":0,"num_objects_dirty":88212,"num_whiteouts":0,"num_read":488,"num_read_kb":914,"num_write":105194,"num_write_kb":6181006,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35871,"num_bytes_recovered":2260939422,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":153394,"ondisk_log_size":153394,"up":357,"acting":348,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":382,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":18764528,"kb_used_data":17152824,"kb_used_omap":118,"kb_used_meta":1611529,"kb_avail":637451536,"statfs":{"total":671965249536,"available":652750372864,"internally_reserved":0,"allocated":17564491776,"data_stored":17558221146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":121648,"internal_metadata":1650205904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-36175872,"num_objects":-552,"num_object_clones":0,"num_object_copies":-1656,"num_objects_missing_on_primary":-4236,"num_objects_missing":-4236,"num_objects_degraded":8018,"num_objects_misplaced":16449,"num_objects_unfound":0,"num_objects_dirty":-552,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":552,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":557,"num_bytes_recovered":36241408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.069739"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":229,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.481739+0000","last_change":"2023-10-09T05:55:01.487478+0000","last_active":"2023-10-09T05:55:04.481739+0000","last_peered":"2023-10-09T05:55:04.481739+0000","last_clean":"2023-10-09T05:55:04.481739+0000","last_became_active":"2023-10-09T05:55:01.487132+0000","last_became_peered":"2023-10-09T05:55:01.487132+0000","last_unstale":"2023-10-09T05:55:04.481739+0000","last_undegraded":"2023-10-09T05:55:04.481739+0000","last_fullsized":"2023-10-09T05:55:04.481739+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:37.882363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":126,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.482778+0000","last_change":"2023-10-09T05:55:01.500390+0000","last_active":"2023-10-09T05:55:04.482778+0000","last_peered":"2023-10-09T05:55:04.482778+0000","last_clean":"2023-10-09T05:55:04.482778+0000","last_became_active":"2023-10-09T05:55:01.500087+0000","last_became_peered":"2023-10-09T05:55:01.500087+0000","last_unstale":"2023-10-09T05:55:04.482778+0000","last_undegraded":"2023-10-09T05:55:04.482778+0000","last_fullsized":"2023-10-09T05:55:04.482778+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:43.149876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":79,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.609900+0000","last_change":"2023-10-09T05:55:01.475912+0000","last_active":"2023-10-09T05:55:04.609900+0000","last_peered":"2023-10-09T05:55:04.609900+0000","last_clean":"2023-10-09T05:55:04.609900+0000","last_became_active":"2023-10-09T05:55:01.475554+0000","last_became_peered":"2023-10-09T05:55:01.475554+0000","last_unstale":"2023-10-09T05:55:04.609900+0000","last_undegraded":"2023-10-09T05:55:04.609900+0000","last_fullsized":"2023-10-09T05:55:04.609900+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:32.758931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":186,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.571334+0000","last_change":"2023-10-09T05:55:01.503032+0000","last_active":"2023-10-09T05:55:04.571334+0000","last_peered":"2023-10-09T05:55:04.571334+0000","last_clean":"2023-10-09T05:55:04.571334+0000","last_became_active":"2023-10-09T05:55:01.502593+0000","last_became_peered":"2023-10-09T05:55:01.502593+0000","last_unstale":"2023-10-09T05:55:04.571334+0000","last_undegraded":"2023-10-09T05:55:04.571334+0000","last_fullsized":"2023-10-09T05:55:04.571334+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:20.126017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":184,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.578045+0000","last_change":"2023-10-09T05:55:01.479269+0000","last_active":"2023-10-09T05:55:04.578045+0000","last_peered":"2023-10-09T05:55:04.578045+0000","last_clean":"2023-10-09T05:55:04.578045+0000","last_became_active":"2023-10-09T05:55:01.478992+0000","last_became_peered":"2023-10-09T05:55:01.478992+0000","last_unstale":"2023-10-09T05:55:04.578045+0000","last_undegraded":"2023-10-09T05:55:04.578045+0000","last_fullsized":"2023-10-09T05:55:04.578045+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:10.726724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":209,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T05:55:03.472292+0000","last_change":"2023-10-09T05:55:01.508900+0000","last_active":"2023-10-09T05:55:03.472292+0000","last_peered":"2023-10-09T05:55:03.472292+0000","last_clean":"2023-10-09T05:55:03.472292+0000","last_became_active":"2023-10-09T05:55:01.508557+0000","last_became_peered":"2023-10-09T05:55:01.508557+0000","last_unstale":"2023-10-09T05:55:03.472292+0000","last_undegraded":"2023-10-09T05:55:03.472292+0000","last_fullsized":"2023-10-09T05:55:03.472292+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:03.275413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.12","version":"204'6576","reported_seq":18755,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:55:09.332534+0000","last_change":"2023-10-09T05:54:45.695681+0000","last_active":"2023-10-09T05:55:09.332534+0000","last_peered":"2023-10-09T05:55:09.332534+0000","last_clean":"2023-10-09T05:55:09.332534+0000","last_became_active":"2023-10-09T05:54:45.691675+0000","last_became_peered":"2023-10-09T05:54:45.691675+0000","last_unstale":"2023-10-09T05:55:09.332534+0000","last_undegraded":"2023-10-09T05:55:09.332534+0000","last_fullsized":"2023-10-09T05:55:09.332534+0000","mapping_epoch":188,"log_start":"106'632","ondisk_log_start":"106'632","created":188,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5944,"log_dups_size":0,"ondisk_log_size":5944,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178421760,"num_objects":2723,"num_object_clones":0,"num_object_copies":8169,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2723,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3297,"num_write_kb":192608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3010,"num_bytes_recovered":191234048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":149,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.571075+0000","last_change":"2023-10-09T05:54:16.209688+0000","last_active":"2023-10-09T05:55:04.571075+0000","last_peered":"2023-10-09T05:55:04.571075+0000","last_clean":"2023-10-09T05:55:04.571075+0000","last_became_active":"2023-10-09T05:54:16.209338+0000","last_became_peered":"2023-10-09T05:54:16.209338+0000","last_unstale":"2023-10-09T05:55:04.571075+0000","last_undegraded":"2023-10-09T05:55:04.571075+0000","last_fullsized":"2023-10-09T05:55:04.571075+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:28.211691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":125,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T05:55:03.471564+0000","last_change":"2023-10-09T05:55:01.503293+0000","last_active":"2023-10-09T05:55:03.471564+0000","last_peered":"2023-10-09T05:55:03.471564+0000","last_clean":"2023-10-09T05:55:03.471564+0000","last_became_active":"2023-10-09T05:55:01.502917+0000","last_became_peered":"2023-10-09T05:55:01.502917+0000","last_unstale":"2023-10-09T05:55:03.471564+0000","last_undegraded":"2023-10-09T05:55:03.471564+0000","last_fullsized":"2023-10-09T05:55:03.471564+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:53.373821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.13","version":"204'6574","reported_seq":6709,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:55:10.283843+0000","last_change":"2023-10-09T05:54:45.686838+0000","last_active":"2023-10-09T05:55:10.283843+0000","last_peered":"2023-10-09T05:55:10.283843+0000","last_clean":"2023-10-09T05:55:10.283843+0000","last_became_active":"2023-10-09T05:54:45.682331+0000","last_became_peered":"2023-10-09T05:54:45.682331+0000","last_unstale":"2023-10-09T05:55:10.283843+0000","last_undegraded":"2023-10-09T05:55:10.283843+0000","last_fullsized":"2023-10-09T05:55:10.283843+0000","mapping_epoch":188,"log_start":"107'1488","ondisk_log_start":"107'1488","created":188,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5086,"log_dups_size":0,"ondisk_log_size":5086,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181895168,"num_objects":2776,"num_object_clones":0,"num_object_copies":8328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2776,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3307,"num_write_kb":194624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":452,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.484492+0000","last_change":"2023-10-09T05:54:19.204752+0000","last_active":"2023-10-09T05:55:04.484492+0000","last_peered":"2023-10-09T05:55:04.484492+0000","last_clean":"2023-10-09T05:55:04.484492+0000","last_became_active":"2023-10-09T05:54:19.204405+0000","last_became_peered":"2023-10-09T05:54:19.204405+0000","last_unstale":"2023-10-09T05:55:04.484492+0000","last_undegraded":"2023-10-09T05:55:04.484492+0000","last_fullsized":"2023-10-09T05:55:04.484492+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":214,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.488315+0000","last_change":"2023-10-09T05:55:01.493997+0000","last_active":"2023-10-09T05:55:04.488315+0000","last_peered":"2023-10-09T05:55:04.488315+0000","last_clean":"2023-10-09T05:55:04.488315+0000","last_became_active":"2023-10-09T05:55:01.493743+0000","last_became_peered":"2023-10-09T05:55:01.493743+0000","last_unstale":"2023-10-09T05:55:04.488315+0000","last_undegraded":"2023-10-09T05:55:04.488315+0000","last_fullsized":"2023-10-09T05:55:04.488315+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:54.234376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":146,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.474198+0000","last_change":"2023-10-09T05:53:59.185702+0000","last_active":"2023-10-09T05:55:04.474198+0000","last_peered":"2023-10-09T05:55:04.474198+0000","last_clean":"2023-10-09T05:55:04.474198+0000","last_became_active":"2023-10-09T05:53:59.185373+0000","last_became_peered":"2023-10-09T05:53:59.185373+0000","last_unstale":"2023-10-09T05:55:04.474198+0000","last_undegraded":"2023-10-09T05:55:04.474198+0000","last_fullsized":"2023-10-09T05:55:04.474198+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:50.756967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"204'6577","reported_seq":11262,"reported_epoch":204,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:55:08.935691+0000","last_change":"2023-10-09T05:55:02.837612+0000","last_active":"2023-10-09T05:55:08.935691+0000","last_peered":"2023-10-09T05:55:08.935691+0000","last_clean":"2023-10-09T05:54:10.146196+0000","last_became_active":"2023-10-09T05:55:02.831677+0000","last_became_peered":"2023-10-09T05:55:02.831677+0000","last_unstale":"2023-10-09T05:55:08.935691+0000","last_undegraded":"2023-10-09T05:55:02.658196+0000","last_fullsized":"2023-10-09T05:55:02.548259+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":188,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6577,"log_dups_size":0,"ondisk_log_size":6577,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182222848,"num_objects":2781,"num_object_clones":0,"num_object_copies":8343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5193,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3296,"num_write_kb":194432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1438,"num_bytes_recovered":94208000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,1,2,3,5","objects":824},{"shards":"0,2,3","objects":1944},{"shards":"0,5","objects":-1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":44,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.477257+0000","last_change":"2023-10-09T05:53:59.219168+0000","last_active":"2023-10-09T05:55:04.477257+0000","last_peered":"2023-10-09T05:55:04.477257+0000","last_clean":"2023-10-09T05:55:04.477257+0000","last_became_active":"2023-10-09T05:53:59.200886+0000","last_became_peered":"2023-10-09T05:53:59.200886+0000","last_unstale":"2023-10-09T05:55:04.477257+0000","last_undegraded":"2023-10-09T05:55:04.477257+0000","last_fullsized":"2023-10-09T05:55:04.477257+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:45.515036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":92,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T05:55:03.471739+0000","last_change":"2023-10-09T05:55:01.482469+0000","last_active":"2023-10-09T05:55:03.471739+0000","last_peered":"2023-10-09T05:55:03.471739+0000","last_clean":"2023-10-09T05:55:03.471739+0000","last_became_active":"2023-10-09T05:55:01.482236+0000","last_became_peered":"2023-10-09T05:55:01.482236+0000","last_unstale":"2023-10-09T05:55:03.471739+0000","last_undegraded":"2023-10-09T05:55:03.471739+0000","last_fullsized":"2023-10-09T05:55:03.471739+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:44.567964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.11","version":"186'6568","reported_seq":6701,"reported_epoch":201,"state":"activating+undersized+degraded+remapped","last_fresh":"2023-10-09T05:55:06.947122+0000","last_change":"2023-10-09T05:55:06.961551+0000","last_active":"2023-10-09T05:54:48.755496+0000","last_peered":"2023-10-09T05:54:39.413056+0000","last_clean":"2023-10-09T05:51:24.563560+0000","last_became_active":"2023-10-09T05:54:33.501402+0000","last_became_peered":"2023-10-09T05:54:33.501402+0000","last_unstale":"2023-10-09T05:55:06.947122+0000","last_undegraded":"2023-10-09T05:55:06.947122+0000","last_fullsized":"2023-10-09T05:55:02.468181+0000","mapping_epoch":200,"log_start":"107'871","ondisk_log_start":"107'871","created":188,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5697,"log_dups_size":0,"ondisk_log_size":5697,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184942592,"num_objects":2822,"num_object_clones":0,"num_object_copies":8466,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":3102,"num_objects_misplaced":1704,"num_objects_unfound":0,"num_objects_dirty":2822,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3284,"num_write_kb":195392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"1,6","objects":-28},{"shards":"6,7","objects":1118}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":209,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.482722+0000","last_change":"2023-10-09T05:53:59.219141+0000","last_active":"2023-10-09T05:55:04.482722+0000","last_peered":"2023-10-09T05:55:04.482722+0000","last_clean":"2023-10-09T05:55:04.482722+0000","last_became_active":"2023-10-09T05:53:59.197318+0000","last_became_peered":"2023-10-09T05:53:59.197318+0000","last_unstale":"2023-10-09T05:55:04.482722+0000","last_undegraded":"2023-10-09T05:55:04.482722+0000","last_fullsized":"2023-10-09T05:55:04.482722+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:39.910794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":193,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.482380+0000","last_change":"2023-10-09T05:55:01.520088+0000","last_active":"2023-10-09T05:55:04.482380+0000","last_peered":"2023-10-09T05:55:04.482380+0000","last_clean":"2023-10-09T05:55:04.482380+0000","last_became_active":"2023-10-09T05:55:01.519726+0000","last_became_peered":"2023-10-09T05:55:01.519726+0000","last_unstale":"2023-10-09T05:55:04.482380+0000","last_undegraded":"2023-10-09T05:55:04.482380+0000","last_fullsized":"2023-10-09T05:55:04.482380+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:00.318135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.16","version":"204'6552","reported_seq":6829,"reported_epoch":204,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T05:55:09.450374+0000","last_change":"2023-10-09T05:54:47.773999+0000","last_active":"2023-10-09T05:55:09.450374+0000","last_peered":"2023-10-09T05:55:09.450374+0000","last_clean":"2023-10-09T05:52:24.946135+0000","last_became_active":"2023-10-09T05:54:47.740044+0000","last_became_peered":"2023-10-09T05:54:47.740044+0000","last_unstale":"2023-10-09T05:55:09.450374+0000","last_undegraded":"2023-10-09T05:54:47.730219+0000","last_fullsized":"2023-10-09T05:55:09.450374+0000","mapping_epoch":190,"log_start":"106'613","ondisk_log_start":"106'613","created":188,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5939,"log_dups_size":0,"ondisk_log_size":5939,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179208192,"num_objects":2735,"num_object_clones":0,"num_object_copies":8205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2735,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2735,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3288,"num_write_kb":192704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":2621440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,6],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":2753},{"shards":"4,6,7","objects":-18}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":534,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.484280+0000","last_change":"2023-10-09T05:53:20.237354+0000","last_active":"2023-10-09T05:55:04.484280+0000","last_peered":"2023-10-09T05:55:04.484280+0000","last_clean":"2023-10-09T05:55:04.484280+0000","last_became_active":"2023-10-09T05:53:20.236993+0000","last_became_peered":"2023-10-09T05:53:20.236993+0000","last_unstale":"2023-10-09T05:55:04.484280+0000","last_undegraded":"2023-10-09T05:55:04.484280+0000","last_fullsized":"2023-10-09T05:55:04.484280+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":46,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.477794+0000","last_change":"2023-10-09T05:53:59.233898+0000","last_active":"2023-10-09T05:55:04.477794+0000","last_peered":"2023-10-09T05:55:04.477794+0000","last_clean":"2023-10-09T05:55:04.477794+0000","last_became_active":"2023-10-09T05:53:59.204175+0000","last_became_peered":"2023-10-09T05:53:59.204175+0000","last_unstale":"2023-10-09T05:55:04.477794+0000","last_undegraded":"2023-10-09T05:55:04.477794+0000","last_fullsized":"2023-10-09T05:55:04.477794+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:03.139318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":214,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.474867+0000","last_change":"2023-10-09T05:55:01.474741+0000","last_active":"2023-10-09T05:55:04.474867+0000","last_peered":"2023-10-09T05:55:04.474867+0000","last_clean":"2023-10-09T05:55:04.474867+0000","last_became_active":"2023-10-09T05:55:01.474433+0000","last_became_peered":"2023-10-09T05:55:01.474433+0000","last_unstale":"2023-10-09T05:55:04.474867+0000","last_undegraded":"2023-10-09T05:55:04.474867+0000","last_fullsized":"2023-10-09T05:55:04.474867+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:25.614070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"204'6434","reported_seq":6554,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:55:09.263176+0000","last_change":"2023-10-09T05:54:45.686122+0000","last_active":"2023-10-09T05:55:09.263176+0000","last_peered":"2023-10-09T05:55:09.263176+0000","last_clean":"2023-10-09T05:55:09.263176+0000","last_became_active":"2023-10-09T05:54:45.681370+0000","last_became_peered":"2023-10-09T05:54:45.681370+0000","last_unstale":"2023-10-09T05:55:09.263176+0000","last_undegraded":"2023-10-09T05:55:09.263176+0000","last_fullsized":"2023-10-09T05:55:09.263176+0000","mapping_epoch":188,"log_start":"106'370","ondisk_log_start":"106'370","created":188,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6064,"log_dups_size":0,"ondisk_log_size":6064,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179077120,"num_objects":2733,"num_object_clones":0,"num_object_copies":8199,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2733,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3228,"num_write_kb":190720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":597,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.476722+0000","last_change":"2023-10-09T05:54:27.330211+0000","last_active":"2023-10-09T05:55:04.476722+0000","last_peered":"2023-10-09T05:55:04.476722+0000","last_clean":"2023-10-09T05:55:04.476722+0000","last_became_active":"2023-10-09T05:53:59.202399+0000","last_became_peered":"2023-10-09T05:53:59.202399+0000","last_unstale":"2023-10-09T05:55:04.476722+0000","last_undegraded":"2023-10-09T05:55:04.476722+0000","last_fullsized":"2023-10-09T05:55:04.476722+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:51:12.346243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:51:12.346243+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":177,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.475745+0000","last_change":"2023-10-09T05:55:01.479822+0000","last_active":"2023-10-09T05:55:04.475745+0000","last_peered":"2023-10-09T05:55:04.475745+0000","last_clean":"2023-10-09T05:55:04.475745+0000","last_became_active":"2023-10-09T05:55:01.479024+0000","last_became_peered":"2023-10-09T05:55:01.479024+0000","last_unstale":"2023-10-09T05:55:04.475745+0000","last_undegraded":"2023-10-09T05:55:04.475745+0000","last_fullsized":"2023-10-09T05:55:04.475745+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:49.655069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":183,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.484720+0000","last_change":"2023-10-09T05:53:59.182073+0000","last_active":"2023-10-09T05:55:04.484720+0000","last_peered":"2023-10-09T05:55:04.484720+0000","last_clean":"2023-10-09T05:55:04.484720+0000","last_became_active":"2023-10-09T05:53:59.181715+0000","last_became_peered":"2023-10-09T05:53:59.181715+0000","last_unstale":"2023-10-09T05:55:04.484720+0000","last_undegraded":"2023-10-09T05:55:04.484720+0000","last_fullsized":"2023-10-09T05:55:04.484720+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:58:31.918499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.14","version":"204'6570","reported_seq":7073,"reported_epoch":204,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:55:09.140108+0000","last_change":"2023-10-09T05:55:02.810203+0000","last_active":"2023-10-09T05:55:09.140108+0000","last_peered":"2023-10-09T05:55:09.140108+0000","last_clean":"2023-10-09T05:51:24.557317+0000","last_became_active":"2023-10-09T05:55:02.802544+0000","last_became_peered":"2023-10-09T05:55:02.802544+0000","last_unstale":"2023-10-09T05:55:09.140108+0000","last_undegraded":"2023-10-09T05:55:02.710006+0000","last_fullsized":"2023-10-09T05:55:02.600439+0000","mapping_epoch":200,"log_start":"107'1426","ondisk_log_start":"107'1426","created":188,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5144,"log_dups_size":0,"ondisk_log_size":5144,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181207040,"num_objects":2765,"num_object_clones":0,"num_object_copies":8295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2915,"num_objects_misplaced":1803,"num_objects_unfound":0,"num_objects_dirty":2765,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3289,"num_write_kb":193728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":93,"num_bytes_recovered":6094848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,1],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"1,3,5","objects":1696},{"shards":"1,5","objects":99},{"shards":"3,5,6","objects":962}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":443,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.573652+0000","last_change":"2023-10-09T05:55:04.573347+0000","last_active":"2023-10-09T05:55:04.573652+0000","last_peered":"2023-10-09T05:55:04.573652+0000","last_clean":"2023-10-09T05:55:04.573652+0000","last_became_active":"2023-10-09T05:55:04.572967+0000","last_became_peered":"2023-10-09T05:55:04.572967+0000","last_unstale":"2023-10-09T05:55:04.573652+0000","last_undegraded":"2023-10-09T05:55:04.573652+0000","last_fullsized":"2023-10-09T05:55:04.573652+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":40,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.476900+0000","last_change":"2023-10-09T05:53:59.206887+0000","last_active":"2023-10-09T05:55:04.476900+0000","last_peered":"2023-10-09T05:55:04.476900+0000","last_clean":"2023-10-09T05:55:04.476900+0000","last_became_active":"2023-10-09T05:53:59.206608+0000","last_became_peered":"2023-10-09T05:53:59.206608+0000","last_unstale":"2023-10-09T05:55:04.476900+0000","last_undegraded":"2023-10-09T05:55:04.476900+0000","last_fullsized":"2023-10-09T05:55:04.476900+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:48.975792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":167,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.571444+0000","last_change":"2023-10-09T05:54:16.225672+0000","last_active":"2023-10-09T05:55:04.571444+0000","last_peered":"2023-10-09T05:55:04.571444+0000","last_clean":"2023-10-09T05:55:04.571444+0000","last_became_active":"2023-10-09T05:54:16.225312+0000","last_became_peered":"2023-10-09T05:54:16.225312+0000","last_unstale":"2023-10-09T05:55:04.571444+0000","last_undegraded":"2023-10-09T05:55:04.571444+0000","last_fullsized":"2023-10-09T05:55:04.571444+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:52.172719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"204'6388","reported_seq":12333,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:55:09.982959+0000","last_change":"2023-10-09T05:55:00.481291+0000","last_active":"2023-10-09T05:55:09.982959+0000","last_peered":"2023-10-09T05:55:09.982959+0000","last_clean":"2023-10-09T05:55:09.982959+0000","last_became_active":"2023-10-09T05:55:00.476488+0000","last_became_peered":"2023-10-09T05:55:00.476488+0000","last_unstale":"2023-10-09T05:55:09.982959+0000","last_undegraded":"2023-10-09T05:55:09.982959+0000","last_fullsized":"2023-10-09T05:55:09.982959+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":188,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6388,"log_dups_size":0,"ondisk_log_size":6388,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176586752,"num_objects":2695,"num_object_clones":0,"num_object_copies":8085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2695,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3208,"num_write_kb":188864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2244,"num_bytes_recovered":132907008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":472,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.475518+0000","last_change":"2023-10-09T05:54:19.202891+0000","last_active":"2023-10-09T05:55:04.475518+0000","last_peered":"2023-10-09T05:55:04.475518+0000","last_clean":"2023-10-09T05:55:04.475518+0000","last_became_active":"2023-10-09T05:54:19.202303+0000","last_became_peered":"2023-10-09T05:54:19.202303+0000","last_unstale":"2023-10-09T05:55:04.475518+0000","last_undegraded":"2023-10-09T05:55:04.475518+0000","last_fullsized":"2023-10-09T05:55:04.475518+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":188,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T05:55:03.471384+0000","last_change":"2023-10-09T05:51:57.091639+0000","last_active":"2023-10-09T05:55:03.471384+0000","last_peered":"2023-10-09T05:55:03.471384+0000","last_clean":"2023-10-09T05:55:03.471384+0000","last_became_active":"2023-10-09T05:51:57.091284+0000","last_became_peered":"2023-10-09T05:51:57.091284+0000","last_unstale":"2023-10-09T05:55:03.471384+0000","last_undegraded":"2023-10-09T05:55:03.471384+0000","last_fullsized":"2023-10-09T05:55:03.471384+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:10.553453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":147,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T05:55:03.469727+0000","last_change":"2023-10-09T05:55:01.486690+0000","last_active":"2023-10-09T05:55:03.469727+0000","last_peered":"2023-10-09T05:55:03.469727+0000","last_clean":"2023-10-09T05:55:03.469727+0000","last_became_active":"2023-10-09T05:55:01.486327+0000","last_became_peered":"2023-10-09T05:55:01.486327+0000","last_unstale":"2023-10-09T05:55:03.469727+0000","last_undegraded":"2023-10-09T05:55:03.469727+0000","last_fullsized":"2023-10-09T05:55:03.469727+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:58.026777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":537,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.482385+0000","last_change":"2023-10-09T05:53:20.160853+0000","last_active":"2023-10-09T05:55:04.482385+0000","last_peered":"2023-10-09T05:55:04.482385+0000","last_clean":"2023-10-09T05:55:04.482385+0000","last_became_active":"2023-10-09T05:53:20.160520+0000","last_became_peered":"2023-10-09T05:53:20.160520+0000","last_unstale":"2023-10-09T05:55:04.482385+0000","last_undegraded":"2023-10-09T05:55:04.482385+0000","last_fullsized":"2023-10-09T05:55:04.482385+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.a","version":"204'6535","reported_seq":8844,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:55:09.308111+0000","last_change":"2023-10-09T05:53:29.244424+0000","last_active":"2023-10-09T05:55:09.308111+0000","last_peered":"2023-10-09T05:55:09.308111+0000","last_clean":"2023-10-09T05:55:09.308111+0000","last_became_active":"2023-10-09T05:53:29.236060+0000","last_became_peered":"2023-10-09T05:53:29.236060+0000","last_unstale":"2023-10-09T05:55:09.308111+0000","last_undegraded":"2023-10-09T05:55:09.308111+0000","last_fullsized":"2023-10-09T05:55:09.308111+0000","mapping_epoch":161,"log_start":"107'1884","ondisk_log_start":"107'1884","created":102,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":4651,"log_dups_size":0,"ondisk_log_size":4651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:10.331676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359858176,"num_objects":5491,"num_object_clones":0,"num_object_copies":16473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6535,"num_write_kb":384832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1085,"num_bytes_recovered":71106560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":44,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.477163+0000","last_change":"2023-10-09T05:53:59.222136+0000","last_active":"2023-10-09T05:55:04.477163+0000","last_peered":"2023-10-09T05:55:04.477163+0000","last_clean":"2023-10-09T05:55:04.477163+0000","last_became_active":"2023-10-09T05:53:59.201853+0000","last_became_peered":"2023-10-09T05:53:59.201853+0000","last_unstale":"2023-10-09T05:55:04.477163+0000","last_undegraded":"2023-10-09T05:55:04.477163+0000","last_fullsized":"2023-10-09T05:55:04.477163+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:26.662031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":166,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.484154+0000","last_change":"2023-10-09T05:53:59.187102+0000","last_active":"2023-10-09T05:55:04.484154+0000","last_peered":"2023-10-09T05:55:04.484154+0000","last_clean":"2023-10-09T05:55:04.484154+0000","last_became_active":"2023-10-09T05:53:59.186758+0000","last_became_peered":"2023-10-09T05:53:59.186758+0000","last_unstale":"2023-10-09T05:55:04.484154+0000","last_undegraded":"2023-10-09T05:55:04.484154+0000","last_fullsized":"2023-10-09T05:55:04.484154+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:35.806325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":456,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.484574+0000","last_change":"2023-10-09T05:54:47.664930+0000","last_active":"2023-10-09T05:55:04.484574+0000","last_peered":"2023-10-09T05:55:04.484574+0000","last_clean":"2023-10-09T05:55:04.484574+0000","last_became_active":"2023-10-09T05:54:47.664533+0000","last_became_peered":"2023-10-09T05:54:47.664533+0000","last_unstale":"2023-10-09T05:55:04.484574+0000","last_undegraded":"2023-10-09T05:55:04.484574+0000","last_fullsized":"2023-10-09T05:55:04.484574+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":381350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"204'6649","reported_seq":19922,"reported_epoch":204,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:55:09.613401+0000","last_change":"2023-10-09T05:55:02.799714+0000","last_active":"2023-10-09T05:55:09.613401+0000","last_peered":"2023-10-09T05:55:09.613401+0000","last_clean":"2023-10-09T05:54:47.668795+0000","last_became_active":"2023-10-09T05:55:02.783605+0000","last_became_peered":"2023-10-09T05:55:02.783605+0000","last_unstale":"2023-10-09T05:55:09.613401+0000","last_undegraded":"2023-10-09T05:55:02.499708+0000","last_fullsized":"2023-10-09T05:55:02.457483+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6649,"log_dups_size":0,"ondisk_log_size":6649,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:59:00.887074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364183552,"num_objects":5557,"num_object_clones":0,"num_object_copies":16671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6103,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5557,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6649,"num_write_kb":390592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6549,"num_bytes_recovered":399966208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6],"avail_no_missing":["3","1","6","7"],"object_location_counts":[{"shards":"1,3,6","objects":5557},{"shards":"3,6","objects":-44}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":58,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.477508+0000","last_change":"2023-10-09T05:55:01.540434+0000","last_active":"2023-10-09T05:55:04.477508+0000","last_peered":"2023-10-09T05:55:04.477508+0000","last_clean":"2023-10-09T05:55:04.477508+0000","last_became_active":"2023-10-09T05:55:01.499983+0000","last_became_peered":"2023-10-09T05:55:01.499983+0000","last_unstale":"2023-10-09T05:55:04.477508+0000","last_undegraded":"2023-10-09T05:55:04.477508+0000","last_fullsized":"2023-10-09T05:55:04.477508+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:44:14.425439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":220,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.481326+0000","last_change":"2023-10-09T05:55:01.499557+0000","last_active":"2023-10-09T05:55:04.481326+0000","last_peered":"2023-10-09T05:55:04.481326+0000","last_clean":"2023-10-09T05:55:04.481326+0000","last_became_active":"2023-10-09T05:55:01.499278+0000","last_became_peered":"2023-10-09T05:55:01.499278+0000","last_unstale":"2023-10-09T05:55:04.481326+0000","last_undegraded":"2023-10-09T05:55:04.481326+0000","last_fullsized":"2023-10-09T05:55:04.481326+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:14:48.475853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":575,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T05:55:03.472517+0000","last_change":"2023-10-09T05:53:59.187768+0000","last_active":"2023-10-09T05:55:03.472517+0000","last_peered":"2023-10-09T05:55:03.472517+0000","last_clean":"2023-10-09T05:55:03.472517+0000","last_became_active":"2023-10-09T05:53:59.187329+0000","last_became_peered":"2023-10-09T05:53:59.187329+0000","last_unstale":"2023-10-09T05:55:03.472517+0000","last_undegraded":"2023-10-09T05:55:03.472517+0000","last_fullsized":"2023-10-09T05:55:03.472517+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.8","version":"204'6523","reported_seq":6656,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:55:10.267942+0000","last_change":"2023-10-09T05:54:45.661206+0000","last_active":"2023-10-09T05:55:10.267942+0000","last_peered":"2023-10-09T05:55:10.267942+0000","last_clean":"2023-10-09T05:55:10.267942+0000","last_became_active":"2023-10-09T05:54:45.655631+0000","last_became_peered":"2023-10-09T05:54:45.655631+0000","last_unstale":"2023-10-09T05:55:10.267942+0000","last_undegraded":"2023-10-09T05:55:10.267942+0000","last_fullsized":"2023-10-09T05:55:10.267942+0000","mapping_epoch":174,"log_start":"106'122","ondisk_log_start":"106'122","created":102,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6401,"log_dups_size":0,"ondisk_log_size":6401,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176291840,"num_objects":2690,"num_object_clones":0,"num_object_copies":8070,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2690,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3276,"num_write_kb":190912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":42,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.477369+0000","last_change":"2023-10-09T05:53:59.212161+0000","last_active":"2023-10-09T05:55:04.477369+0000","last_peered":"2023-10-09T05:55:04.477369+0000","last_clean":"2023-10-09T05:55:04.477369+0000","last_became_active":"2023-10-09T05:53:59.193814+0000","last_became_peered":"2023-10-09T05:53:59.193814+0000","last_unstale":"2023-10-09T05:55:04.477369+0000","last_undegraded":"2023-10-09T05:55:04.477369+0000","last_fullsized":"2023-10-09T05:55:04.477369+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:10.502408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":222,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T05:55:03.471650+0000","last_change":"2023-10-09T05:55:01.515191+0000","last_active":"2023-10-09T05:55:03.471650+0000","last_peered":"2023-10-09T05:55:03.471650+0000","last_clean":"2023-10-09T05:55:03.471650+0000","last_became_active":"2023-10-09T05:55:01.508614+0000","last_became_peered":"2023-10-09T05:55:01.508614+0000","last_unstale":"2023-10-09T05:55:03.471650+0000","last_undegraded":"2023-10-09T05:55:03.471650+0000","last_fullsized":"2023-10-09T05:55:03.471650+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:51.590362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":562,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T05:55:03.472105+0000","last_change":"2023-10-09T05:53:20.167463+0000","last_active":"2023-10-09T05:55:03.472105+0000","last_peered":"2023-10-09T05:55:03.472105+0000","last_clean":"2023-10-09T05:55:03.472105+0000","last_became_active":"2023-10-09T05:53:20.167014+0000","last_became_peered":"2023-10-09T05:53:20.167014+0000","last_unstale":"2023-10-09T05:55:03.472105+0000","last_undegraded":"2023-10-09T05:55:03.472105+0000","last_fullsized":"2023-10-09T05:55:03.472105+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"204'6536","reported_seq":7329,"reported_epoch":204,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:55:08.896804+0000","last_change":"2023-10-09T05:55:01.513240+0000","last_active":"2023-10-09T05:55:08.896804+0000","last_peered":"2023-10-09T05:55:08.896804+0000","last_clean":"2023-10-09T05:51:33.020184+0000","last_became_active":"2023-10-09T05:54:53.759742+0000","last_became_peered":"2023-10-09T05:54:53.759742+0000","last_unstale":"2023-10-09T05:55:08.896804+0000","last_undegraded":"2023-10-09T05:54:49.556279+0000","last_fullsized":"2023-10-09T05:54:46.645749+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6536,"log_dups_size":0,"ondisk_log_size":6536,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183533568,"num_objects":2800,"num_object_clones":0,"num_object_copies":8400,"num_objects_missing_on_primary":400,"num_objects_missing":400,"num_objects_degraded":400,"num_objects_misplaced":2612,"num_objects_unfound":0,"num_objects_dirty":2800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3271,"num_write_kb":194304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":217,"num_bytes_recovered":14221312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,3],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,7","objects":2212},{"shards":"0,3","objects":188},{"shards":"2,7","objects":400}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":184,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.484862+0000","last_change":"2023-10-09T05:55:01.506628+0000","last_active":"2023-10-09T05:55:04.484862+0000","last_peered":"2023-10-09T05:55:04.484862+0000","last_clean":"2023-10-09T05:55:04.484862+0000","last_became_active":"2023-10-09T05:55:01.500813+0000","last_became_peered":"2023-10-09T05:55:01.500813+0000","last_unstale":"2023-10-09T05:55:04.484862+0000","last_undegraded":"2023-10-09T05:55:04.484862+0000","last_fullsized":"2023-10-09T05:55:04.484862+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:51.258390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":186,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.570925+0000","last_change":"2023-10-09T05:55:01.502927+0000","last_active":"2023-10-09T05:55:04.570925+0000","last_peered":"2023-10-09T05:55:04.570925+0000","last_clean":"2023-10-09T05:55:04.570925+0000","last_became_active":"2023-10-09T05:55:01.502575+0000","last_became_peered":"2023-10-09T05:55:01.502575+0000","last_unstale":"2023-10-09T05:55:04.570925+0000","last_undegraded":"2023-10-09T05:55:04.570925+0000","last_fullsized":"2023-10-09T05:55:04.570925+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:03.191407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":597,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.477119+0000","last_change":"2023-10-09T05:54:19.204406+0000","last_active":"2023-10-09T05:55:04.477119+0000","last_peered":"2023-10-09T05:55:04.477119+0000","last_clean":"2023-10-09T05:55:04.477119+0000","last_became_active":"2023-10-09T05:54:19.204082+0000","last_became_peered":"2023-10-09T05:54:19.204082+0000","last_unstale":"2023-10-09T05:55:04.477119+0000","last_undegraded":"2023-10-09T05:55:04.477119+0000","last_fullsized":"2023-10-09T05:55:04.477119+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.e","version":"204'6499","reported_seq":18974,"reported_epoch":204,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:55:09.385921+0000","last_change":"2023-10-09T05:55:02.613660+0000","last_active":"2023-10-09T05:55:09.385921+0000","last_peered":"2023-10-09T05:55:09.385921+0000","last_clean":"2023-10-09T05:54:59.500528+0000","last_became_active":"2023-10-09T05:55:02.600031+0000","last_became_peered":"2023-10-09T05:55:02.600031+0000","last_unstale":"2023-10-09T05:55:09.385921+0000","last_undegraded":"2023-10-09T05:55:09.385921+0000","last_fullsized":"2023-10-09T05:55:09.385921+0000","mapping_epoch":200,"log_start":"107'1551","ondisk_log_start":"107'1551","created":102,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":4948,"log_dups_size":0,"ondisk_log_size":4948,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:51.949070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359202816,"num_objects":5481,"num_object_clones":0,"num_object_copies":16443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5481,"num_objects_unfound":0,"num_objects_dirty":5481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6499,"num_write_kb":383360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6147,"num_bytes_recovered":392560640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,1],"avail_no_missing":["4","1","2","5"],"object_location_counts":[{"shards":"1,4,5","objects":5481}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":72,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.571125+0000","last_change":"2023-10-09T05:55:01.498515+0000","last_active":"2023-10-09T05:55:04.571125+0000","last_peered":"2023-10-09T05:55:04.571125+0000","last_clean":"2023-10-09T05:55:04.571125+0000","last_became_active":"2023-10-09T05:55:01.482823+0000","last_became_peered":"2023-10-09T05:55:01.482823+0000","last_unstale":"2023-10-09T05:55:04.571125+0000","last_undegraded":"2023-10-09T05:55:04.571125+0000","last_fullsized":"2023-10-09T05:55:04.571125+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:24.437433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":163,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.482346+0000","last_change":"2023-10-09T05:49:21.430742+0000","last_active":"2023-10-09T05:55:04.482346+0000","last_peered":"2023-10-09T05:55:04.482346+0000","last_clean":"2023-10-09T05:55:04.482346+0000","last_became_active":"2023-10-09T05:49:21.430233+0000","last_became_peered":"2023-10-09T05:49:21.430233+0000","last_unstale":"2023-10-09T05:55:04.482346+0000","last_undegraded":"2023-10-09T05:55:04.482346+0000","last_fullsized":"2023-10-09T05:55:04.482346+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:10.541302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":532,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.484253+0000","last_change":"2023-10-09T05:54:54.779276+0000","last_active":"2023-10-09T05:55:04.484253+0000","last_peered":"2023-10-09T05:55:04.484253+0000","last_clean":"2023-10-09T05:55:04.484253+0000","last_became_active":"2023-10-09T05:54:54.778929+0000","last_became_peered":"2023-10-09T05:54:54.778929+0000","last_unstale":"2023-10-09T05:55:04.484253+0000","last_undegraded":"2023-10-09T05:55:04.484253+0000","last_fullsized":"2023-10-09T05:55:04.484253+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":686385,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.f","version":"204'6648","reported_seq":7174,"reported_epoch":204,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T05:55:09.462912+0000","last_change":"2023-10-09T05:55:02.813319+0000","last_active":"2023-10-09T05:55:09.462912+0000","last_peered":"2023-10-09T05:55:09.462912+0000","last_clean":"2023-10-09T05:52:22.632757+0000","last_became_active":"2023-10-09T05:55:02.721682+0000","last_became_peered":"2023-10-09T05:55:02.721682+0000","last_unstale":"2023-10-09T05:55:09.462912+0000","last_undegraded":"2023-10-09T05:55:02.700643+0000","last_fullsized":"2023-10-09T05:55:09.462912+0000","mapping_epoch":200,"log_start":"107'1691","ondisk_log_start":"107'1691","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":4957,"log_dups_size":0,"ondisk_log_size":4957,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:35.585692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369229824,"num_objects":5634,"num_object_clones":0,"num_object_copies":16902,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4563,"num_objects_misplaced":5634,"num_objects_unfound":0,"num_objects_dirty":5634,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6648,"num_write_kb":393024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":200,"num_bytes_recovered":13107200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,3],"avail_no_missing":["4","1","2","3"],"object_location_counts":[{"shards":"2,3,4","objects":4135},{"shards":"3,4,6","objects":1499}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":193,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T05:55:03.471599+0000","last_change":"2023-10-09T05:55:01.485334+0000","last_active":"2023-10-09T05:55:03.471599+0000","last_peered":"2023-10-09T05:55:03.471599+0000","last_clean":"2023-10-09T05:55:03.471599+0000","last_became_active":"2023-10-09T05:55:01.484584+0000","last_became_peered":"2023-10-09T05:55:01.484584+0000","last_unstale":"2023-10-09T05:55:03.471599+0000","last_undegraded":"2023-10-09T05:55:03.471599+0000","last_fullsized":"2023-10-09T05:55:03.471599+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:02.524256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":220,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.482615+0000","last_change":"2023-10-09T05:55:01.488260+0000","last_active":"2023-10-09T05:55:04.482615+0000","last_peered":"2023-10-09T05:55:04.482615+0000","last_clean":"2023-10-09T05:55:04.482615+0000","last_became_active":"2023-10-09T05:55:01.487158+0000","last_became_peered":"2023-10-09T05:55:01.487158+0000","last_unstale":"2023-10-09T05:55:04.482615+0000","last_undegraded":"2023-10-09T05:55:04.482615+0000","last_fullsized":"2023-10-09T05:55:04.482615+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:50:56.901286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":443,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.573531+0000","last_change":"2023-10-09T05:55:04.573104+0000","last_active":"2023-10-09T05:55:04.573531+0000","last_peered":"2023-10-09T05:55:04.573531+0000","last_clean":"2023-10-09T05:55:04.573531+0000","last_became_active":"2023-10-09T05:55:04.572460+0000","last_became_peered":"2023-10-09T05:55:04.572460+0000","last_unstale":"2023-10-09T05:55:04.573531+0000","last_undegraded":"2023-10-09T05:55:04.573531+0000","last_fullsized":"2023-10-09T05:55:04.573531+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"191'6550","reported_seq":18767,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T05:55:03.472568+0000","last_change":"2023-10-09T05:53:59.196684+0000","last_active":"2023-10-09T05:55:03.472568+0000","last_peered":"2023-10-09T05:55:03.472568+0000","last_clean":"2023-10-09T05:55:03.472568+0000","last_became_active":"2023-10-09T05:53:59.187954+0000","last_became_peered":"2023-10-09T05:53:59.187954+0000","last_unstale":"2023-10-09T05:55:03.472568+0000","last_undegraded":"2023-10-09T05:55:03.472568+0000","last_fullsized":"2023-10-09T05:55:03.472568+0000","mapping_epoch":173,"log_start":"107'2004","ondisk_log_start":"107'2004","created":102,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":4546,"log_dups_size":0,"ondisk_log_size":4546,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:28.562782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361496576,"num_objects":5516,"num_object_clones":0,"num_object_copies":16548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6550,"num_write_kb":386112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6033,"num_bytes_recovered":385286144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":184,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.481108+0000","last_change":"2023-10-09T05:54:00.619165+0000","last_active":"2023-10-09T05:55:04.481108+0000","last_peered":"2023-10-09T05:55:04.481108+0000","last_clean":"2023-10-09T05:55:04.481108+0000","last_became_active":"2023-10-09T05:54:00.618888+0000","last_became_peered":"2023-10-09T05:54:00.618888+0000","last_unstale":"2023-10-09T05:55:04.481108+0000","last_undegraded":"2023-10-09T05:55:04.481108+0000","last_fullsized":"2023-10-09T05:55:04.481108+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:07.175996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":193,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.487688+0000","last_change":"2023-10-09T05:55:01.495270+0000","last_active":"2023-10-09T05:55:04.487688+0000","last_peered":"2023-10-09T05:55:04.487688+0000","last_clean":"2023-10-09T05:55:04.487688+0000","last_became_active":"2023-10-09T05:55:01.494985+0000","last_became_peered":"2023-10-09T05:55:01.494985+0000","last_unstale":"2023-10-09T05:55:04.487688+0000","last_undegraded":"2023-10-09T05:55:04.487688+0000","last_fullsized":"2023-10-09T05:55:04.487688+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:03.998885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":508,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.484198+0000","last_change":"2023-10-09T05:51:19.195984+0000","last_active":"2023-10-09T05:55:04.484198+0000","last_peered":"2023-10-09T05:55:04.484198+0000","last_clean":"2023-10-09T05:55:04.484198+0000","last_became_active":"2023-10-09T05:51:19.195632+0000","last_became_peered":"2023-10-09T05:51:19.195632+0000","last_unstale":"2023-10-09T05:55:04.484198+0000","last_undegraded":"2023-10-09T05:55:04.484198+0000","last_fullsized":"2023-10-09T05:55:04.484198+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.d","version":"204'6530","reported_seq":11894,"reported_epoch":204,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:55:09.371479+0000","last_change":"2023-10-09T05:55:02.800639+0000","last_active":"2023-10-09T05:55:09.371479+0000","last_peered":"2023-10-09T05:55:09.371479+0000","last_clean":"2023-10-09T05:54:59.460612+0000","last_became_active":"2023-10-09T05:55:02.674251+0000","last_became_peered":"2023-10-09T05:55:02.674251+0000","last_unstale":"2023-10-09T05:55:09.371479+0000","last_undegraded":"2023-10-09T05:55:09.371479+0000","last_fullsized":"2023-10-09T05:55:09.371479+0000","mapping_epoch":200,"log_start":"107'1875","ondisk_log_start":"107'1875","created":102,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":4655,"log_dups_size":0,"ondisk_log_size":4655,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:23.567813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358678552,"num_objects":5474,"num_object_clones":0,"num_object_copies":16422,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5443,"num_objects_unfound":0,"num_objects_dirty":5474,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6530,"num_write_kb":384065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2592,"num_bytes_recovered":169869312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,1],"avail_no_missing":["5","0","1"],"object_location_counts":[{"shards":"0,1,5","objects":5474}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":181,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.476552+0000","last_change":"2023-10-09T05:55:01.487852+0000","last_active":"2023-10-09T05:55:04.476552+0000","last_peered":"2023-10-09T05:55:04.476552+0000","last_clean":"2023-10-09T05:55:04.476552+0000","last_became_active":"2023-10-09T05:55:01.486322+0000","last_became_peered":"2023-10-09T05:55:01.486322+0000","last_unstale":"2023-10-09T05:55:04.476552+0000","last_undegraded":"2023-10-09T05:55:04.476552+0000","last_fullsized":"2023-10-09T05:55:04.476552+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:43.099331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":79,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.609960+0000","last_change":"2023-10-09T05:55:01.487140+0000","last_active":"2023-10-09T05:55:04.609960+0000","last_peered":"2023-10-09T05:55:04.609960+0000","last_clean":"2023-10-09T05:55:04.609960+0000","last_became_active":"2023-10-09T05:55:01.486482+0000","last_became_peered":"2023-10-09T05:55:01.486482+0000","last_unstale":"2023-10-09T05:55:04.609960+0000","last_undegraded":"2023-10-09T05:55:04.609960+0000","last_fullsized":"2023-10-09T05:55:04.609960+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:22.544209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":576,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T05:55:03.472359+0000","last_change":"2023-10-09T05:53:59.202618+0000","last_active":"2023-10-09T05:55:03.472359+0000","last_peered":"2023-10-09T05:55:03.472359+0000","last_clean":"2023-10-09T05:55:03.472359+0000","last_became_active":"2023-10-09T05:53:59.202265+0000","last_became_peered":"2023-10-09T05:53:59.202265+0000","last_unstale":"2023-10-09T05:55:03.472359+0000","last_undegraded":"2023-10-09T05:55:03.472359+0000","last_fullsized":"2023-10-09T05:55:03.472359+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":94,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.475858+0000","last_change":"2023-10-09T05:55:01.484825+0000","last_active":"2023-10-09T05:55:04.475858+0000","last_peered":"2023-10-09T05:55:04.475858+0000","last_clean":"2023-10-09T05:55:04.475858+0000","last_became_active":"2023-10-09T05:55:01.483842+0000","last_became_peered":"2023-10-09T05:55:01.483842+0000","last_unstale":"2023-10-09T05:55:04.475858+0000","last_undegraded":"2023-10-09T05:55:04.475858+0000","last_fullsized":"2023-10-09T05:55:04.475858+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:32.211180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":191,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.484652+0000","last_change":"2023-10-09T05:52:29.735372+0000","last_active":"2023-10-09T05:55:04.484652+0000","last_peered":"2023-10-09T05:55:04.484652+0000","last_clean":"2023-10-09T05:55:04.484652+0000","last_became_active":"2023-10-09T05:52:29.735026+0000","last_became_peered":"2023-10-09T05:52:29.735026+0000","last_unstale":"2023-10-09T05:55:04.484652+0000","last_undegraded":"2023-10-09T05:55:04.484652+0000","last_fullsized":"2023-10-09T05:55:04.484652+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:53.849406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":73,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.484044+0000","last_change":"2023-10-09T05:55:01.490018+0000","last_active":"2023-10-09T05:55:04.484044+0000","last_peered":"2023-10-09T05:55:04.484044+0000","last_clean":"2023-10-09T05:55:04.484044+0000","last_became_active":"2023-10-09T05:55:01.489497+0000","last_became_peered":"2023-10-09T05:55:01.489497+0000","last_unstale":"2023-10-09T05:55:04.484044+0000","last_undegraded":"2023-10-09T05:55:04.484044+0000","last_fullsized":"2023-10-09T05:55:04.484044+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.796235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"204'6572","reported_seq":8770,"reported_epoch":204,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:55:08.828970+0000","last_change":"2023-10-09T05:55:02.837610+0000","last_active":"2023-10-09T05:55:08.828970+0000","last_peered":"2023-10-09T05:55:08.828970+0000","last_clean":"2023-10-09T05:54:10.146196+0000","last_became_active":"2023-10-09T05:55:02.831671+0000","last_became_peered":"2023-10-09T05:55:02.831671+0000","last_unstale":"2023-10-09T05:55:08.828970+0000","last_undegraded":"2023-10-09T05:55:02.546632+0000","last_fullsized":"2023-10-09T05:55:02.459265+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6572,"log_dups_size":0,"ondisk_log_size":6572,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182550528,"num_objects":2785,"num_object_clones":0,"num_object_copies":8355,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6077,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2785,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3290,"num_write_kb":194432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":608,"num_bytes_recovered":39878656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":2798},{"shards":"0,5","objects":-13}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":535,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.482601+0000","last_change":"2023-10-09T05:53:20.234074+0000","last_active":"2023-10-09T05:55:04.482601+0000","last_peered":"2023-10-09T05:55:04.482601+0000","last_clean":"2023-10-09T05:55:04.482601+0000","last_became_active":"2023-10-09T05:53:20.233803+0000","last_became_peered":"2023-10-09T05:53:20.233803+0000","last_unstale":"2023-10-09T05:55:04.482601+0000","last_undegraded":"2023-10-09T05:55:04.482601+0000","last_fullsized":"2023-10-09T05:55:04.482601+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.6","version":"204'6554","reported_seq":7089,"reported_epoch":204,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T05:55:09.351226+0000","last_change":"2023-10-09T05:55:01.518574+0000","last_active":"2023-10-09T05:55:09.351226+0000","last_peered":"2023-10-09T05:55:09.351226+0000","last_clean":"2023-10-09T05:52:24.946135+0000","last_became_active":"2023-10-09T05:54:45.741289+0000","last_became_peered":"2023-10-09T05:54:45.741289+0000","last_unstale":"2023-10-09T05:55:09.351226+0000","last_undegraded":"2023-10-09T05:55:01.516066+0000","last_fullsized":"2023-10-09T05:55:09.351226+0000","mapping_epoch":181,"log_start":"106'613","ondisk_log_start":"106'613","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5941,"log_dups_size":0,"ondisk_log_size":5941,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179077120,"num_objects":2732,"num_object_clones":0,"num_object_copies":8196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2732,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3289,"num_write_kb":192704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":168,"num_bytes_recovered":11010048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,6],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":2588},{"shards":"4,6,7","objects":144}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":215,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.483965+0000","last_change":"2023-10-09T05:55:01.492934+0000","last_active":"2023-10-09T05:55:04.483965+0000","last_peered":"2023-10-09T05:55:04.483965+0000","last_clean":"2023-10-09T05:55:04.483965+0000","last_became_active":"2023-10-09T05:55:01.489897+0000","last_became_peered":"2023-10-09T05:55:01.489897+0000","last_unstale":"2023-10-09T05:55:04.483965+0000","last_undegraded":"2023-10-09T05:55:04.483965+0000","last_fullsized":"2023-10-09T05:55:04.483965+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:17.947941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":181,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.482520+0000","last_change":"2023-10-09T05:51:57.203142+0000","last_active":"2023-10-09T05:55:04.482520+0000","last_peered":"2023-10-09T05:55:04.482520+0000","last_clean":"2023-10-09T05:55:04.482520+0000","last_became_active":"2023-10-09T05:51:57.202850+0000","last_became_peered":"2023-10-09T05:51:57.202850+0000","last_unstale":"2023-10-09T05:55:04.482520+0000","last_undegraded":"2023-10-09T05:55:04.482520+0000","last_fullsized":"2023-10-09T05:55:04.482520+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:51.090737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":104,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T05:55:03.470589+0000","last_change":"2023-10-09T05:55:01.494232+0000","last_active":"2023-10-09T05:55:03.470589+0000","last_peered":"2023-10-09T05:55:03.470589+0000","last_clean":"2023-10-09T05:55:03.470589+0000","last_became_active":"2023-10-09T05:55:01.493942+0000","last_became_peered":"2023-10-09T05:55:01.493942+0000","last_unstale":"2023-10-09T05:55:03.470589+0000","last_undegraded":"2023-10-09T05:55:03.470589+0000","last_fullsized":"2023-10-09T05:55:03.470589+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:46:21.615615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":472,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.475606+0000","last_change":"2023-10-09T05:54:19.203419+0000","last_active":"2023-10-09T05:55:04.475606+0000","last_peered":"2023-10-09T05:55:04.475606+0000","last_clean":"2023-10-09T05:55:04.475606+0000","last_became_active":"2023-10-09T05:54:19.203105+0000","last_became_peered":"2023-10-09T05:54:19.203105+0000","last_unstale":"2023-10-09T05:55:04.475606+0000","last_undegraded":"2023-10-09T05:55:04.475606+0000","last_fullsized":"2023-10-09T05:55:04.475606+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"204'6390","reported_seq":12414,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:55:10.181870+0000","last_change":"2023-10-09T05:54:54.773920+0000","last_active":"2023-10-09T05:55:10.181870+0000","last_peered":"2023-10-09T05:55:10.181870+0000","last_clean":"2023-10-09T05:55:10.181870+0000","last_became_active":"2023-10-09T05:54:54.770744+0000","last_became_peered":"2023-10-09T05:54:54.770744+0000","last_unstale":"2023-10-09T05:55:10.181870+0000","last_undegraded":"2023-10-09T05:55:10.181870+0000","last_fullsized":"2023-10-09T05:55:10.181870+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6390,"log_dups_size":0,"ondisk_log_size":6390,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176455680,"num_objects":2692,"num_object_clones":0,"num_object_copies":8076,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2692,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3209,"num_write_kb":188864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2284,"num_bytes_recovered":135462912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":173,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.581148+0000","last_change":"2023-10-09T05:55:01.488052+0000","last_active":"2023-10-09T05:55:04.581148+0000","last_peered":"2023-10-09T05:55:04.581148+0000","last_clean":"2023-10-09T05:55:04.581148+0000","last_became_active":"2023-10-09T05:55:01.487729+0000","last_became_peered":"2023-10-09T05:55:01.487729+0000","last_unstale":"2023-10-09T05:55:04.581148+0000","last_undegraded":"2023-10-09T05:55:04.581148+0000","last_fullsized":"2023-10-09T05:55:04.581148+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:09:39.869203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":88,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T05:55:03.469191+0000","last_change":"2023-10-09T05:55:01.473644+0000","last_active":"2023-10-09T05:55:03.469191+0000","last_peered":"2023-10-09T05:55:03.469191+0000","last_clean":"2023-10-09T05:55:03.469191+0000","last_became_active":"2023-10-09T05:55:01.473094+0000","last_became_peered":"2023-10-09T05:55:01.473094+0000","last_unstale":"2023-10-09T05:55:03.469191+0000","last_undegraded":"2023-10-09T05:55:03.469191+0000","last_fullsized":"2023-10-09T05:55:03.469191+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:36.129826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":105,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.483134+0000","last_change":"2023-10-09T05:55:01.492693+0000","last_active":"2023-10-09T05:55:04.483134+0000","last_peered":"2023-10-09T05:55:04.483134+0000","last_clean":"2023-10-09T05:55:04.483134+0000","last_became_active":"2023-10-09T05:55:01.492406+0000","last_became_peered":"2023-10-09T05:55:01.492406+0000","last_unstale":"2023-10-09T05:55:04.483134+0000","last_undegraded":"2023-10-09T05:55:04.483134+0000","last_fullsized":"2023-10-09T05:55:04.483134+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:31.166518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.7","version":"204'6430","reported_seq":6551,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:55:09.116490+0000","last_change":"2023-10-09T05:54:45.655400+0000","last_active":"2023-10-09T05:55:09.116490+0000","last_peered":"2023-10-09T05:55:09.116490+0000","last_clean":"2023-10-09T05:55:09.116490+0000","last_became_active":"2023-10-09T05:54:45.647040+0000","last_became_peered":"2023-10-09T05:54:45.647040+0000","last_unstale":"2023-10-09T05:55:09.116490+0000","last_undegraded":"2023-10-09T05:55:09.116490+0000","last_fullsized":"2023-10-09T05:55:09.116490+0000","mapping_epoch":102,"log_start":"106'370","ondisk_log_start":"106'370","created":102,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6060,"log_dups_size":0,"ondisk_log_size":6060,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179339264,"num_objects":2736,"num_object_clones":0,"num_object_copies":8208,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2736,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3223,"num_write_kb":190720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":188,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.483475+0000","last_change":"2023-10-09T05:53:59.182813+0000","last_active":"2023-10-09T05:55:04.483475+0000","last_peered":"2023-10-09T05:55:04.483475+0000","last_clean":"2023-10-09T05:55:04.483475+0000","last_became_active":"2023-10-09T05:53:59.182441+0000","last_became_peered":"2023-10-09T05:53:59.182441+0000","last_unstale":"2023-10-09T05:55:04.483475+0000","last_undegraded":"2023-10-09T05:55:04.483475+0000","last_fullsized":"2023-10-09T05:55:04.483475+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:59.004246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":595,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.476806+0000","last_change":"2023-10-09T05:53:59.202723+0000","last_active":"2023-10-09T05:55:04.476806+0000","last_peered":"2023-10-09T05:55:04.476806+0000","last_clean":"2023-10-09T05:55:04.476806+0000","last_became_active":"2023-10-09T05:53:59.202004+0000","last_became_peered":"2023-10-09T05:53:59.202004+0000","last_unstale":"2023-10-09T05:55:04.476806+0000","last_undegraded":"2023-10-09T05:55:04.476806+0000","last_fullsized":"2023-10-09T05:55:04.476806+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:51:12.346243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:51:12.346243+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":91,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.483881+0000","last_change":"2023-10-09T05:52:23.648626+0000","last_active":"2023-10-09T05:55:04.483881+0000","last_peered":"2023-10-09T05:55:04.483881+0000","last_clean":"2023-10-09T05:55:04.483881+0000","last_became_active":"2023-10-09T05:52:23.646143+0000","last_became_peered":"2023-10-09T05:52:23.646143+0000","last_unstale":"2023-10-09T05:55:04.483881+0000","last_undegraded":"2023-10-09T05:55:04.483881+0000","last_fullsized":"2023-10-09T05:55:04.483881+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:51:04.733282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":165,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T05:55:03.471900+0000","last_change":"2023-10-09T05:53:59.195325+0000","last_active":"2023-10-09T05:55:03.471900+0000","last_peered":"2023-10-09T05:55:03.471900+0000","last_clean":"2023-10-09T05:55:03.471900+0000","last_became_active":"2023-10-09T05:53:59.194954+0000","last_became_peered":"2023-10-09T05:53:59.194954+0000","last_unstale":"2023-10-09T05:55:03.471900+0000","last_undegraded":"2023-10-09T05:55:03.471900+0000","last_fullsized":"2023-10-09T05:55:03.471900+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:23.228416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":443,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.573277+0000","last_change":"2023-10-09T05:55:04.572885+0000","last_active":"2023-10-09T05:55:04.573277+0000","last_peered":"2023-10-09T05:55:04.573277+0000","last_clean":"2023-10-09T05:55:04.573277+0000","last_became_active":"2023-10-09T05:55:04.572464+0000","last_became_peered":"2023-10-09T05:55:04.572464+0000","last_unstale":"2023-10-09T05:55:04.573277+0000","last_undegraded":"2023-10-09T05:55:04.573277+0000","last_fullsized":"2023-10-09T05:55:04.573277+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":50,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.476690+0000","last_change":"2023-10-09T05:55:01.482471+0000","last_active":"2023-10-09T05:55:04.476690+0000","last_peered":"2023-10-09T05:55:04.476690+0000","last_clean":"2023-10-09T05:55:04.476690+0000","last_became_active":"2023-10-09T05:55:01.482140+0000","last_became_peered":"2023-10-09T05:55:01.482140+0000","last_unstale":"2023-10-09T05:55:04.476690+0000","last_undegraded":"2023-10-09T05:55:04.476690+0000","last_fullsized":"2023-10-09T05:55:04.476690+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:23:36.353139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":130,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.581060+0000","last_change":"2023-10-09T05:52:23.643245+0000","last_active":"2023-10-09T05:55:04.581060+0000","last_peered":"2023-10-09T05:55:04.581060+0000","last_clean":"2023-10-09T05:55:04.581060+0000","last_became_active":"2023-10-09T05:52:23.642947+0000","last_became_peered":"2023-10-09T05:52:23.642947+0000","last_unstale":"2023-10-09T05:55:04.581060+0000","last_undegraded":"2023-10-09T05:55:04.581060+0000","last_fullsized":"2023-10-09T05:55:04.581060+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:52.260774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":229,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.571256+0000","last_change":"2023-10-09T05:55:01.487796+0000","last_active":"2023-10-09T05:55:04.571256+0000","last_peered":"2023-10-09T05:55:04.571256+0000","last_clean":"2023-10-09T05:55:04.571256+0000","last_became_active":"2023-10-09T05:55:01.487037+0000","last_became_peered":"2023-10-09T05:55:01.487037+0000","last_unstale":"2023-10-09T05:55:04.571256+0000","last_undegraded":"2023-10-09T05:55:04.571256+0000","last_fullsized":"2023-10-09T05:55:04.571256+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:35.138646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"204'6589","reported_seq":7092,"reported_epoch":204,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:55:09.304642+0000","last_change":"2023-10-09T05:55:02.800180+0000","last_active":"2023-10-09T05:55:09.304642+0000","last_peered":"2023-10-09T05:55:09.304642+0000","last_clean":"2023-10-09T05:51:24.557317+0000","last_became_active":"2023-10-09T05:55:02.759224+0000","last_became_peered":"2023-10-09T05:55:02.759224+0000","last_unstale":"2023-10-09T05:55:09.304642+0000","last_undegraded":"2023-10-09T05:55:02.598542+0000","last_fullsized":"2023-10-09T05:55:02.468173+0000","mapping_epoch":200,"log_start":"107'1426","ondisk_log_start":"107'1426","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5163,"log_dups_size":0,"ondisk_log_size":5163,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179961856,"num_objects":2746,"num_object_clones":0,"num_object_copies":8238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2954,"num_objects_misplaced":1784,"num_objects_unfound":0,"num_objects_dirty":2746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3308,"num_write_kb":193728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":93,"num_bytes_recovered":6094848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,1],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"1,3,5","objects":1706},{"shards":"1,5","objects":51},{"shards":"3,5,6","objects":962}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":563,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T05:55:03.472201+0000","last_change":"2023-10-09T05:53:20.157691+0000","last_active":"2023-10-09T05:55:03.472201+0000","last_peered":"2023-10-09T05:55:03.472201+0000","last_clean":"2023-10-09T05:55:03.472201+0000","last_became_active":"2023-10-09T05:53:20.157392+0000","last_became_peered":"2023-10-09T05:53:20.157392+0000","last_unstale":"2023-10-09T05:55:03.472201+0000","last_undegraded":"2023-10-09T05:55:03.472201+0000","last_fullsized":"2023-10-09T05:55:03.472201+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":38,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.477167+0000","last_change":"2023-10-09T05:53:59.201576+0000","last_active":"2023-10-09T05:55:04.477167+0000","last_peered":"2023-10-09T05:55:04.477167+0000","last_clean":"2023-10-09T05:55:04.477167+0000","last_became_active":"2023-10-09T05:53:59.201309+0000","last_became_peered":"2023-10-09T05:53:59.201309+0000","last_unstale":"2023-10-09T05:55:04.477167+0000","last_undegraded":"2023-10-09T05:55:04.477167+0000","last_fullsized":"2023-10-09T05:55:04.477167+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:54.956371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":80,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.484997+0000","last_change":"2023-10-09T05:55:01.493636+0000","last_active":"2023-10-09T05:55:04.484997+0000","last_peered":"2023-10-09T05:55:04.484997+0000","last_clean":"2023-10-09T05:55:04.484997+0000","last_became_active":"2023-10-09T05:55:01.493357+0000","last_became_peered":"2023-10-09T05:55:01.493357+0000","last_unstale":"2023-10-09T05:55:04.484997+0000","last_undegraded":"2023-10-09T05:55:04.484997+0000","last_fullsized":"2023-10-09T05:55:04.484997+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:47.088692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":128,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.571202+0000","last_change":"2023-10-09T05:53:59.188462+0000","last_active":"2023-10-09T05:55:04.571202+0000","last_peered":"2023-10-09T05:55:04.571202+0000","last_clean":"2023-10-09T05:55:04.571202+0000","last_became_active":"2023-10-09T05:53:59.188061+0000","last_became_peered":"2023-10-09T05:53:59.188061+0000","last_unstale":"2023-10-09T05:55:04.571202+0000","last_undegraded":"2023-10-09T05:55:04.571202+0000","last_fullsized":"2023-10-09T05:55:04.571202+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:19.167114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"186'6568","reported_seq":6700,"reported_epoch":203,"state":"activating+undersized+degraded+remapped","last_fresh":"2023-10-09T05:55:06.970721+0000","last_change":"2023-10-09T05:55:06.970721+0000","last_active":"2023-10-09T05:54:48.755496+0000","last_peered":"2023-10-09T05:54:39.413056+0000","last_clean":"2023-10-09T05:51:24.563560+0000","last_became_active":"2023-10-09T05:54:33.501402+0000","last_became_peered":"2023-10-09T05:54:33.501402+0000","last_unstale":"2023-10-09T05:55:06.970721+0000","last_undegraded":"2023-10-09T05:55:02.605258+0000","last_fullsized":"2023-10-09T05:55:02.465162+0000","mapping_epoch":200,"log_start":"107'871","ondisk_log_start":"107'871","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5697,"log_dups_size":0,"ondisk_log_size":5697,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184942592,"num_objects":2822,"num_object_clones":0,"num_object_copies":8466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3002,"num_objects_misplaced":1819,"num_objects_unfound":0,"num_objects_dirty":2822,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3284,"num_write_kb":195392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,1],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"1,6","objects":28},{"shards":"6,7","objects":1003}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":538,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.482480+0000","last_change":"2023-10-09T05:53:20.152495+0000","last_active":"2023-10-09T05:55:04.482480+0000","last_peered":"2023-10-09T05:55:04.482480+0000","last_clean":"2023-10-09T05:55:04.482480+0000","last_became_active":"2023-10-09T05:53:20.152109+0000","last_became_peered":"2023-10-09T05:53:20.152109+0000","last_unstale":"2023-10-09T05:55:04.482480+0000","last_undegraded":"2023-10-09T05:55:04.482480+0000","last_fullsized":"2023-10-09T05:55:04.482480+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":208,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.474794+0000","last_change":"2023-10-09T05:53:59.228948+0000","last_active":"2023-10-09T05:55:04.474794+0000","last_peered":"2023-10-09T05:55:04.474794+0000","last_clean":"2023-10-09T05:55:04.474794+0000","last_became_active":"2023-10-09T05:53:59.186359+0000","last_became_peered":"2023-10-09T05:53:59.186359+0000","last_unstale":"2023-10-09T05:55:04.474794+0000","last_undegraded":"2023-10-09T05:55:04.474794+0000","last_fullsized":"2023-10-09T05:55:04.474794+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:39.990576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":55,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T05:55:03.472107+0000","last_change":"2023-10-09T05:55:01.504321+0000","last_active":"2023-10-09T05:55:03.472107+0000","last_peered":"2023-10-09T05:55:03.472107+0000","last_clean":"2023-10-09T05:55:03.472107+0000","last_became_active":"2023-10-09T05:55:01.503815+0000","last_became_peered":"2023-10-09T05:55:01.503815+0000","last_unstale":"2023-10-09T05:55:03.472107+0000","last_undegraded":"2023-10-09T05:55:03.472107+0000","last_fullsized":"2023-10-09T05:55:03.472107+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:00:42.980267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":50,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.476637+0000","last_change":"2023-10-09T05:55:01.488042+0000","last_active":"2023-10-09T05:55:04.476637+0000","last_peered":"2023-10-09T05:55:04.476637+0000","last_clean":"2023-10-09T05:55:04.476637+0000","last_became_active":"2023-10-09T05:55:01.487505+0000","last_became_peered":"2023-10-09T05:55:01.487505+0000","last_unstale":"2023-10-09T05:55:04.476637+0000","last_undegraded":"2023-10-09T05:55:04.476637+0000","last_fullsized":"2023-10-09T05:55:04.476637+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:06.660981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.2","version":"204'6585","reported_seq":18765,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:55:09.447666+0000","last_change":"2023-10-09T05:54:45.678323+0000","last_active":"2023-10-09T05:55:09.447666+0000","last_peered":"2023-10-09T05:55:09.447666+0000","last_clean":"2023-10-09T05:55:09.447666+0000","last_became_active":"2023-10-09T05:54:45.673809+0000","last_became_peered":"2023-10-09T05:54:45.673809+0000","last_unstale":"2023-10-09T05:55:09.447666+0000","last_undegraded":"2023-10-09T05:55:09.447666+0000","last_fullsized":"2023-10-09T05:55:09.447666+0000","mapping_epoch":159,"log_start":"106'632","ondisk_log_start":"106'632","created":102,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5953,"log_dups_size":0,"ondisk_log_size":5953,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177831936,"num_objects":2713,"num_object_clones":0,"num_object_copies":8139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2713,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3305,"num_write_kb":192608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3009,"num_bytes_recovered":191234048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":452,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.484654+0000","last_change":"2023-10-09T05:54:19.202509+0000","last_active":"2023-10-09T05:55:04.484654+0000","last_peered":"2023-10-09T05:55:04.484654+0000","last_clean":"2023-10-09T05:55:04.484654+0000","last_became_active":"2023-10-09T05:54:19.202144+0000","last_became_peered":"2023-10-09T05:54:19.202144+0000","last_unstale":"2023-10-09T05:55:04.484654+0000","last_undegraded":"2023-10-09T05:55:04.484654+0000","last_fullsized":"2023-10-09T05:55:04.484654+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":197,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T05:55:03.472403+0000","last_change":"2023-10-09T05:54:16.215036+0000","last_active":"2023-10-09T05:55:03.472403+0000","last_peered":"2023-10-09T05:55:03.472403+0000","last_clean":"2023-10-09T05:55:03.472403+0000","last_became_active":"2023-10-09T05:54:16.212771+0000","last_became_peered":"2023-10-09T05:54:16.212771+0000","last_unstale":"2023-10-09T05:55:03.472403+0000","last_undegraded":"2023-10-09T05:55:03.472403+0000","last_fullsized":"2023-10-09T05:55:03.472403+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:29.278423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":203,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.571352+0000","last_change":"2023-10-09T05:55:01.542309+0000","last_active":"2023-10-09T05:55:04.571352+0000","last_peered":"2023-10-09T05:55:04.571352+0000","last_clean":"2023-10-09T05:55:04.571352+0000","last_became_active":"2023-10-09T05:55:01.522225+0000","last_became_peered":"2023-10-09T05:55:01.522225+0000","last_unstale":"2023-10-09T05:55:04.571352+0000","last_undegraded":"2023-10-09T05:55:04.571352+0000","last_fullsized":"2023-10-09T05:55:04.571352+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:44.824868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":146,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.481196+0000","last_change":"2023-10-09T05:40:12.676587+0000","last_active":"2023-10-09T05:55:04.481196+0000","last_peered":"2023-10-09T05:55:04.481196+0000","last_clean":"2023-10-09T05:55:04.481196+0000","last_became_active":"2023-10-09T05:40:12.676330+0000","last_became_peered":"2023-10-09T05:40:12.676330+0000","last_unstale":"2023-10-09T05:55:04.481196+0000","last_undegraded":"2023-10-09T05:55:04.481196+0000","last_fullsized":"2023-10-09T05:55:04.481196+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:46.659556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"204'6557","reported_seq":6693,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:55:10.285894+0000","last_change":"2023-10-09T05:54:45.661663+0000","last_active":"2023-10-09T05:55:10.285894+0000","last_peered":"2023-10-09T05:55:10.285894+0000","last_clean":"2023-10-09T05:55:10.285894+0000","last_became_active":"2023-10-09T05:54:45.657155+0000","last_became_peered":"2023-10-09T05:54:45.657155+0000","last_unstale":"2023-10-09T05:55:10.285894+0000","last_undegraded":"2023-10-09T05:55:10.285894+0000","last_fullsized":"2023-10-09T05:55:10.285894+0000","mapping_epoch":174,"log_start":"107'1488","ondisk_log_start":"107'1488","created":102,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5069,"log_dups_size":0,"ondisk_log_size":5069,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183009280,"num_objects":2792,"num_object_clones":0,"num_object_copies":8376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3289,"num_write_kb":194624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":132,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.571431+0000","last_change":"2023-10-09T05:53:59.191302+0000","last_active":"2023-10-09T05:55:04.571431+0000","last_peered":"2023-10-09T05:55:04.571431+0000","last_clean":"2023-10-09T05:55:04.571431+0000","last_became_active":"2023-10-09T05:53:59.190911+0000","last_became_peered":"2023-10-09T05:53:59.190911+0000","last_unstale":"2023-10-09T05:55:04.571431+0000","last_undegraded":"2023-10-09T05:55:04.571431+0000","last_fullsized":"2023-10-09T05:55:04.571431+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:12.801201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":170,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.475772+0000","last_change":"2023-10-09T05:55:01.487715+0000","last_active":"2023-10-09T05:55:04.475772+0000","last_peered":"2023-10-09T05:55:04.475772+0000","last_clean":"2023-10-09T05:55:04.475772+0000","last_became_active":"2023-10-09T05:55:01.487465+0000","last_became_peered":"2023-10-09T05:55:01.487465+0000","last_unstale":"2023-10-09T05:55:04.475772+0000","last_undegraded":"2023-10-09T05:55:04.475772+0000","last_fullsized":"2023-10-09T05:55:04.475772+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:41.242604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.19","version":"204'6533","reported_seq":6743,"reported_epoch":204,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:55:08.887999+0000","last_change":"2023-10-09T05:54:57.558699+0000","last_active":"2023-10-09T05:55:08.887999+0000","last_peered":"2023-10-09T05:55:08.887999+0000","last_clean":"2023-10-09T05:51:33.020184+0000","last_became_active":"2023-10-09T05:54:57.470115+0000","last_became_peered":"2023-10-09T05:54:57.470115+0000","last_unstale":"2023-10-09T05:55:08.887999+0000","last_undegraded":"2023-10-09T05:54:52.846685+0000","last_fullsized":"2023-10-09T05:54:49.570105+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":188,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6533,"log_dups_size":0,"ondisk_log_size":6533,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183730176,"num_objects":2804,"num_object_clones":0,"num_object_copies":8412,"num_objects_missing_on_primary":581,"num_objects_missing":581,"num_objects_degraded":3587,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2804,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3269,"num_write_kb":194304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":1441792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,3],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,7","objects":2217},{"shards":"0,3","objects":6},{"shards":"2,7","objects":581}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":208,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.484731+0000","last_change":"2023-10-09T05:55:01.537690+0000","last_active":"2023-10-09T05:55:04.484731+0000","last_peered":"2023-10-09T05:55:04.484731+0000","last_clean":"2023-10-09T05:55:04.484731+0000","last_became_active":"2023-10-09T05:55:01.492416+0000","last_became_peered":"2023-10-09T05:55:01.492416+0000","last_unstale":"2023-10-09T05:55:04.484731+0000","last_undegraded":"2023-10-09T05:55:04.484731+0000","last_fullsized":"2023-10-09T05:55:04.484731+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:51.948190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":210,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.482586+0000","last_change":"2023-10-09T05:55:01.484005+0000","last_active":"2023-10-09T05:55:04.482586+0000","last_peered":"2023-10-09T05:55:04.482586+0000","last_clean":"2023-10-09T05:55:04.482586+0000","last_became_active":"2023-10-09T05:55:01.483679+0000","last_became_peered":"2023-10-09T05:55:01.483679+0000","last_unstale":"2023-10-09T05:55:04.482586+0000","last_undegraded":"2023-10-09T05:55:04.482586+0000","last_fullsized":"2023-10-09T05:55:04.482586+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:36:35.351926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"204'6520","reported_seq":6652,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T05:55:10.334688+0000","last_change":"2023-10-09T05:54:45.671348+0000","last_active":"2023-10-09T05:55:10.334688+0000","last_peered":"2023-10-09T05:55:10.334688+0000","last_clean":"2023-10-09T05:55:10.334688+0000","last_became_active":"2023-10-09T05:54:45.664954+0000","last_became_peered":"2023-10-09T05:54:45.664954+0000","last_unstale":"2023-10-09T05:55:10.334688+0000","last_undegraded":"2023-10-09T05:55:10.334688+0000","last_fullsized":"2023-10-09T05:55:10.334688+0000","mapping_epoch":188,"log_start":"106'122","ondisk_log_start":"106'122","created":188,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6398,"log_dups_size":0,"ondisk_log_size":6398,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176488448,"num_objects":2693,"num_object_clones":0,"num_object_copies":8079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3273,"num_write_kb":190912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":149,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.484578+0000","last_change":"2023-10-09T05:40:12.633195+0000","last_active":"2023-10-09T05:55:04.484578+0000","last_peered":"2023-10-09T05:55:04.484578+0000","last_clean":"2023-10-09T05:55:04.484578+0000","last_became_active":"2023-10-09T05:40:12.632942+0000","last_became_peered":"2023-10-09T05:40:12.632942+0000","last_unstale":"2023-10-09T05:55:04.484578+0000","last_undegraded":"2023-10-09T05:55:04.484578+0000","last_fullsized":"2023-10-09T05:55:04.484578+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:04.470685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":197,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.481649+0000","last_change":"2023-10-09T05:51:57.093104+0000","last_active":"2023-10-09T05:55:04.481649+0000","last_peered":"2023-10-09T05:55:04.481649+0000","last_clean":"2023-10-09T05:55:04.481649+0000","last_became_active":"2023-10-09T05:51:57.092810+0000","last_became_peered":"2023-10-09T05:51:57.092810+0000","last_unstale":"2023-10-09T05:55:04.481649+0000","last_undegraded":"2023-10-09T05:55:04.481649+0000","last_fullsized":"2023-10-09T05:55:04.481649+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:51.192227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":101,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T05:55:03.472062+0000","last_change":"2023-10-09T05:55:01.509174+0000","last_active":"2023-10-09T05:55:03.472062+0000","last_peered":"2023-10-09T05:55:03.472062+0000","last_clean":"2023-10-09T05:55:03.472062+0000","last_became_active":"2023-10-09T05:55:01.508856+0000","last_became_peered":"2023-10-09T05:55:01.508856+0000","last_unstale":"2023-10-09T05:55:03.472062+0000","last_undegraded":"2023-10-09T05:55:03.472062+0000","last_fullsized":"2023-10-09T05:55:03.472062+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:14.904462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":192,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T05:55:03.470633+0000","last_change":"2023-10-09T05:52:42.261890+0000","last_active":"2023-10-09T05:55:03.470633+0000","last_peered":"2023-10-09T05:55:03.470633+0000","last_clean":"2023-10-09T05:55:03.470633+0000","last_became_active":"2023-10-09T05:52:42.261521+0000","last_became_peered":"2023-10-09T05:52:42.261521+0000","last_unstale":"2023-10-09T05:55:03.470633+0000","last_undegraded":"2023-10-09T05:55:03.470633+0000","last_fullsized":"2023-10-09T05:55:03.470633+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:59.208145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":154,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.571116+0000","last_change":"2023-10-09T05:52:23.654749+0000","last_active":"2023-10-09T05:55:04.571116+0000","last_peered":"2023-10-09T05:55:04.571116+0000","last_clean":"2023-10-09T05:55:04.571116+0000","last_became_active":"2023-10-09T05:52:23.643044+0000","last_became_peered":"2023-10-09T05:52:23.643044+0000","last_unstale":"2023-10-09T05:55:04.571116+0000","last_undegraded":"2023-10-09T05:55:04.571116+0000","last_fullsized":"2023-10-09T05:55:04.571116+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:23:08.898078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":193,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T05:55:04.481001+0000","last_change":"2023-10-09T05:55:01.489827+0000","last_active":"2023-10-09T05:55:04.481001+0000","last_peered":"2023-10-09T05:55:04.481001+0000","last_clean":"2023-10-09T05:55:04.481001+0000","last_became_active":"2023-10-09T05:55:01.489523+0000","last_became_peered":"2023-10-09T05:55:01.489523+0000","last_unstale":"2023-10-09T05:55:04.481001+0000","last_undegraded":"2023-10-09T05:55:04.481001+0000","last_fullsized":"2023-10-09T05:55:04.481001+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:46:26.120419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":5779423256,"num_objects":88188,"num_object_clones":0,"num_object_copies":264564,"num_objects_missing_on_primary":982,"num_objects_missing":982,"num_objects_degraded":43363,"num_objects_misplaced":26280,"num_objects_unfound":0,"num_objects_dirty":88188,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":104894,"num_write_kb":6178561,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35835,"num_bytes_recovered":2258305024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19109851136,"data_stored":19109838920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149958,"ondisk_log_size":149958,"up":78,"acting":69,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":10238,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":2624160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234186,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2904844,"kb_used_data":2603460,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":90840308,"statfs":{"total":95995035648,"available":93020475392,"internally_reserved":0,"allocated":2665943040,"data_stored":2665185990,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:54:58 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":1.472,"15min":1.917},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.54700000000000004,"5min":20.247,"15min":20.247},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.42899999999999999,"5min":1.3819999999999999,"15min":1.776},"min":{"1min":0.26900000000000002,"5min":0.252,"15min":0.252},"max":{"1min":0.627,"5min":20.236000000000001,"15min":20.236000000000001},"last":0.32400000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:54:24 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.88100000000000001,"15min":0.64300000000000002},"min":{"1min":0.27200000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.747,"5min":7.9859999999999998,"15min":7.9859999999999998},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.83299999999999996,"15min":0.64500000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.5339999999999998,"5min":5.8129999999999997,"15min":5.8129999999999997},"last":0.54600000000000004}]},{"osd":2,"last update":"Mon Oct 9 05:54:58 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":2.3300000000000001,"15min":1.127},"min":{"1min":0.33100000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.7349999999999999,"5min":50.57,"15min":50.57},"last":0.376},{"interface":"front","average":{"1min":0.51000000000000001,"5min":2.335,"15min":1.1879999999999999},"min":{"1min":0.38200000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.25,"5min":50.536999999999999,"15min":50.536999999999999},"last":0.38800000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:54:30 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":5.6689999999999996,"15min":2.2999999999999998},"min":{"1min":0.40300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.4410000000000001,"5min":147.54900000000001,"15min":147.54900000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.53500000000000003,"5min":5.5519999999999996,"15min":2.242},"min":{"1min":0.35799999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.95599999999999996,"5min":144.065,"15min":144.065},"last":0.42599999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:55:06 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.69699999999999995,"15min":0.54800000000000004},"min":{"1min":0.28599999999999998,"5min":0.248,"15min":0.24399999999999999},"max":{"1min":0.64700000000000002,"5min":9.5329999999999995,"15min":9.5329999999999995},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.88900000000000001,"15min":0.63500000000000001},"min":{"1min":0.29899999999999999,"5min":0.29099999999999998,"15min":0.27100000000000002},"max":{"1min":0.622,"5min":7.6120000000000001,"15min":7.6120000000000001},"last":0.45100000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:55:06 2023","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.86399999999999999,"15min":0.623},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.248},"max":{"1min":0.58199999999999996,"5min":10.795999999999999,"15min":10.795999999999999},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.71199999999999997,"15min":0.59999999999999998},"min":{"1min":0.314,"5min":0.30499999999999999,"15min":0.28100000000000003},"max":{"1min":0.57699999999999996,"5min":3.9380000000000002,"15min":3.9380000000000002},"last":0.438}]},{"osd":6,"last update":"Mon Oct 9 05:55:06 2023","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":2.4249999999999998,"15min":1.175},"min":{"1min":0.33900000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.63500000000000001,"5min":131.94399999999999,"15min":131.94399999999999},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.49199999999999999,"5min":2.5270000000000001,"15min":1.22},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.29999999999999999},"max":{"1min":0.65900000000000003,"5min":135.23400000000001,"15min":135.23400000000001},"last":0.59399999999999997}]}]},{"osd":6,"up_from":46,"seq":197568495822,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3009848,"kb_used_data":2704368,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":90735304,"statfs":{"total":95995035648,"available":92912951296,"internally_reserved":0,"allocated":2769272832,"data_stored":2768528512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:54:58 2023","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":0.93300000000000005,"15min":1.085},"min":{"1min":0.25800000000000001,"5min":0.254,"15min":0.254},"max":{"1min":5.7999999999999998,"5min":7.0599999999999996,"15min":7.0599999999999996},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.629,"5min":0.95699999999999996,"15min":1.137},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":3.8410000000000002,"5min":6.6459999999999999,"15min":6.6459999999999999},"last":0.55000000000000004}]},{"osd":1,"last update":"Mon Oct 9 05:54:34 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.78800000000000003,"15min":0.65800000000000003},"min":{"1min":0.29699999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.141,"5min":3.7949999999999999,"15min":3.7949999999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.872,"15min":0.68400000000000005},"min":{"1min":0.30499999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.734,"5min":9.9670000000000005,"15min":9.9670000000000005},"last":0.56899999999999995}]},{"osd":2,"last update":"Mon Oct 9 05:54:58 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":1.0589999999999999,"15min":0.71499999999999997},"min":{"1min":0.28999999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.67400000000000004,"5min":6.6859999999999999,"15min":6.6859999999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.59999999999999998,"5min":1.0369999999999999,"15min":0.71699999999999997},"min":{"1min":0.32600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.121,"5min":9.9390000000000001,"15min":9.9390000000000001},"last":0.59299999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:54:34 2023","interfaces":[{"interface":"back","average":{"1min":1.232,"5min":1.248,"15min":0.78500000000000003},"min":{"1min":0.33400000000000002,"5min":0.311,"15min":0.311},"max":{"1min":6.0629999999999997,"5min":10.19,"15min":10.19},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.59699999999999998,"5min":1.145,"15min":0.76700000000000002},"min":{"1min":0.42799999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.24,"5min":11.914,"15min":11.914},"last":0.628}]},{"osd":4,"last update":"Mon Oct 9 05:54:44 2023","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.91600000000000004,"15min":0.61299999999999999},"min":{"1min":0.28699999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.66200000000000003,"5min":7.9100000000000001,"15min":7.9100000000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.86699999999999999,"15min":0.621},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.79500000000000004,"5min":7.3300000000000001,"15min":7.3300000000000001},"last":0.48399999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:54:44 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.89700000000000002,"15min":0.623},"min":{"1min":0.309,"5min":0.29799999999999999,"15min":0.27700000000000002},"max":{"1min":0.621,"5min":13.879,"15min":13.879},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.95899999999999996,"15min":0.67200000000000004},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.72399999999999998,"5min":7.29,"15min":7.29},"last":0.52400000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:54:08 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":1.2450000000000001,"15min":0.77000000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.316},"max":{"1min":0.70299999999999996,"5min":19.914999999999999,"15min":19.914999999999999},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.60099999999999998,"5min":1.099,"15min":0.76800000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.35799999999999998},"max":{"1min":1.653,"5min":7.0309999999999997,"15min":7.0309999999999997},"last":0.58299999999999996}]}]},{"osd":1,"up_from":0,"seq":382252089412,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":129,"seq":554050781213,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1438532,"kb_used_data":1358908,"kb_used_omap":29,"kb_used_meta":79586,"kb_avail":92306620,"statfs":{"total":95995035648,"available":94521978880,"internally_reserved":0,"allocated":1391521792,"data_stored":1390426380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30322,"internal_metadata":81496462},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":0.77900000000000003,"15min":0.72699999999999998},"min":{"1min":0.221,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":3.4910000000000001,"5min":9.1720000000000006,"15min":9.1720000000000006},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.69599999999999995,"5min":0.997,"15min":1.097},"min":{"1min":0.26500000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":3.2749999999999999,"5min":4.0049999999999999,"15min":4.0049999999999999},"last":0.36099999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.89100000000000001,"15min":0.98899999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":2.3719999999999999,"5min":3.9199999999999999,"15min":3.9199999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.91600000000000004,"15min":0.97799999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.4160000000000004,"5min":5.4160000000000004,"15min":5.4160000000000004},"last":0.30299999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.71299999999999997,"15min":0.754},"min":{"1min":0.28299999999999997,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.94199999999999995,"5min":3.1930000000000001,"15min":3.1930000000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.82199999999999995,"15min":0.874},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.5329999999999999,"5min":4.5279999999999996,"15min":4.5279999999999996},"last":0.38300000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":1.1080000000000001,"15min":1.1839999999999999},"min":{"1min":0.40500000000000003,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":3.7090000000000001,"5min":9.0050000000000008,"15min":9.0050000000000008},"last":1.089},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.80400000000000005,"15min":0.87},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.04,"5min":3.1549999999999998,"15min":3.1549999999999998},"last":1.105}]},{"osd":5,"last update":"Mon Oct 9 05:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":1.0389999999999999,"15min":1.1120000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":1.5720000000000001,"5min":7.3959999999999999,"15min":7.3959999999999999},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.94999999999999996,"15min":0.92600000000000005},"min":{"1min":0.443,"5min":0.439,"15min":0.439},"max":{"1min":3.6579999999999999,"5min":9.1080000000000005,"15min":9.1080000000000005},"last":1.1279999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":1.083,"15min":1.256},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.6040000000000001,"5min":6.6849999999999996,"15min":6.6849999999999996},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.77200000000000002,"5min":1.111,"15min":1.1870000000000001},"min":{"1min":0.47099999999999997,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":3.734,"5min":9.141,"15min":9.141},"last":0.46700000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":1.274,"15min":1.371},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":3.7559999999999998,"5min":9.0700000000000003,"15min":9.0700000000000003},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.64500000000000002,"5min":1.1539999999999999,"15min":1.3220000000000001},"min":{"1min":0.45700000000000002,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.01,"5min":5.8730000000000002,"15min":5.8730000000000002},"last":0.44900000000000001}]}]},{"osd":2,"up_from":94,"seq":403726925902,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3439852,"kb_used_data":3273636,"kb_used_omap":81,"kb_used_meta":166126,"kb_avail":90305300,"statfs":{"total":95995035648,"available":92472627200,"internally_reserved":0,"allocated":3352203264,"data_stored":3351176597,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":170113242},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:55:02 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":1.2010000000000001,"15min":1.532},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":1.607,"5min":11.444000000000001,"15min":11.444000000000001},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.58699999999999997,"5min":1.163,"15min":1.4279999999999999},"min":{"1min":0.249,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":2.371,"5min":6.883,"15min":6.883},"last":0.63600000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:54:57 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":2.851,"15min":1.2370000000000001},"min":{"1min":0.27700000000000002,"5min":0.218,"15min":0.218},"max":{"1min":1.732,"5min":163.24199999999999,"15min":163.24199999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":2.661,"15min":1.159},"min":{"1min":0.27100000000000002,"5min":0.214,"15min":0.214},"max":{"1min":2.5070000000000001,"5min":163.27000000000001,"15min":163.27000000000001},"last":0.34599999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:54:30 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":2.9910000000000001,"15min":1.2649999999999999},"min":{"1min":0.27200000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.7080000000000002,"5min":170.07599999999999,"15min":170.07599999999999},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.63100000000000001,"5min":1.6579999999999999,"15min":0.84699999999999998},"min":{"1min":0.32300000000000001,"5min":0.25,"15min":0.25},"max":{"1min":2.0950000000000002,"5min":63.600000000000001,"15min":63.600000000000001},"last":0.36699999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:54:57 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":1.1759999999999999,"15min":0.84199999999999997},"min":{"1min":0.38500000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.9700000000000002,"5min":11.372999999999999,"15min":11.372999999999999},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.78400000000000003,"5min":1.0840000000000001,"15min":0.83599999999999997},"min":{"1min":0.36799999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.9489999999999998,"5min":6.5709999999999997,"15min":6.5709999999999997},"last":0.52300000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:54:57 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":1.0960000000000001,"15min":0.81799999999999995},"min":{"1min":0.38600000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.9390000000000001,"5min":13.414,"15min":13.414},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.83099999999999996,"5min":1.216,"15min":0.85199999999999998},"min":{"1min":0.47299999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.8879999999999999,"5min":13.438000000000001,"15min":13.438000000000001},"last":0.53500000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:54:57 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":1.1839999999999999,"15min":0.83899999999999997},"min":{"1min":0.40100000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.8999999999999999,"5min":11.59,"15min":11.59},"last":0.495},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.94299999999999995,"15min":0.77500000000000002},"min":{"1min":0.39600000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.9140000000000001,"5min":6.4710000000000001,"15min":6.4710000000000001},"last":0.877}]},{"osd":7,"last update":"Mon Oct 9 05:54:57 2023","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":1.339,"15min":0.89500000000000002},"min":{"1min":0.36599999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":3.8769999999999998,"5min":40.404000000000003,"15min":40.404000000000003},"last":0.84199999999999997},{"interface":"front","average":{"1min":0.86499999999999999,"5min":1.403,"15min":0.93899999999999995},"min":{"1min":0.49099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.9279999999999999,"5min":40.113,"15min":40.113},"last":0.56499999999999995}]}]},{"osd":3,"up_from":99,"seq":425201762376,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3419308,"kb_used_data":3271268,"kb_used_omap":7,"kb_used_meta":148024,"kb_avail":90325844,"statfs":{"total":95995035648,"available":92493664256,"internally_reserved":0,"allocated":3349778432,"data_stored":3348640341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":151577192},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:55:04 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":1.25,"15min":1.448},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":1.7969999999999999,"5min":15.536,"15min":15.536},"last":0.316},{"interface":"front","average":{"1min":0.67400000000000004,"5min":1.2230000000000001,"15min":1.4690000000000001},"min":{"1min":0.255,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":2.6749999999999998,"5min":11.210000000000001,"15min":11.210000000000001},"last":0.30099999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":2.0760000000000001,"15min":0.93799999999999994},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":2.4060000000000001,"5min":127.226,"15min":127.226},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.56699999999999995,"5min":2.1859999999999999,"15min":0.99399999999999999},"min":{"1min":0.28199999999999997,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":1.3500000000000001,"5min":127.468,"15min":127.468},"last":0.40600000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":1.347,"15min":0.69899999999999995},"min":{"1min":0.252,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":2.5150000000000001,"5min":62.938000000000002,"15min":62.938000000000002},"last":0.379},{"interface":"front","average":{"1min":0.48199999999999998,"5min":1.677,"15min":0.84699999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.208,"5min":62.43,"15min":62.43},"last":0.44500000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.93400000000000005,"15min":0.71199999999999997},"min":{"1min":0.35599999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.3029999999999999,"5min":7.7439999999999998,"15min":7.7439999999999998},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.629,"5min":0.81499999999999995,"15min":0.68300000000000005},"min":{"1min":0.312,"5min":0.311,"15min":0.311},"max":{"1min":2.4420000000000002,"5min":6.2000000000000002,"15min":6.2000000000000002},"last":0.39300000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.85399999999999998,"15min":0.68600000000000005},"min":{"1min":0.39900000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":2.4740000000000002,"5min":5.5860000000000003,"15min":5.5860000000000003},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.626,"5min":0.82899999999999996,"15min":0.70099999999999996},"min":{"1min":0.34799999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.5680000000000001,"5min":4.2839999999999998,"15min":4.2839999999999998},"last":0.52700000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.97199999999999998,"15min":0.746},"min":{"1min":0.42299999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.5059999999999998,"5min":12.798999999999999,"15min":12.798999999999999},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.68700000000000006,"5min":1.006,"15min":0.78800000000000003},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":2.6059999999999999,"5min":12.773,"15min":12.773},"last":0.56299999999999994}]},{"osd":7,"last update":"Mon Oct 9 05:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.98899999999999999,"15min":0.749},"min":{"1min":0.442,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":2.6419999999999999,"5min":7.4580000000000002,"15min":7.4580000000000002},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.62,"5min":0.98999999999999999,"15min":0.76800000000000002},"min":{"1min":0.42699999999999999,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.276,"5min":7.8399999999999999,"15min":7.8399999999999999},"last":0.54100000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019096,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2076428,"kb_used_data":1770948,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":91668724,"statfs":{"total":95995035648,"available":93868773376,"internally_reserved":0,"allocated":1813450752,"data_stored":1812693955,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:55:02 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.88200000000000001,"15min":0.81499999999999995},"min":{"1min":0.28000000000000003,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.4730000000000001,"5min":15.173999999999999,"15min":15.173999999999999},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.51800000000000002,"5min":1.2869999999999999,"15min":1.4510000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.7010000000000001,"5min":15.125999999999999,"15min":15.125999999999999},"last":0.31900000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:54:31 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":1.042,"15min":0.72499999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.512,"5min":9.2390000000000008,"15min":9.2390000000000008},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.91400000000000003,"15min":0.65000000000000002},"min":{"1min":0.27400000000000002,"5min":0.255,"15min":0.255},"max":{"1min":3.556,"5min":8.2629999999999999,"15min":8.2629999999999999},"last":0.35799999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:55:02 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.4279999999999999,"15min":0.85999999999999999},"min":{"1min":0.32600000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":4.1980000000000004,"5min":33.262,"15min":33.262},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.98399999999999999,"5min":1.472,"15min":0.94599999999999995},"min":{"1min":0.35599999999999998,"5min":0.313,"15min":0.313},"max":{"1min":5.4720000000000004,"5min":33.240000000000002,"15min":33.240000000000002},"last":0.54000000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:54:31 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":1.4430000000000001,"15min":0.873},"min":{"1min":0.34799999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.60299999999999998,"5min":40.531999999999996,"15min":40.531999999999996},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.605,"15min":0.93500000000000005},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":4.827,"5min":39.905999999999999,"15min":39.905999999999999},"last":0.42999999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:54:21 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.81100000000000005,"15min":0.58599999999999997},"min":{"1min":0.27500000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.503,"5min":10.012,"15min":10.012},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.42899999999999999,"5min":1.0009999999999999,"15min":0.67900000000000005},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.61399999999999999,"5min":12.489000000000001,"15min":12.489000000000001},"last":0.60399999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:54:37 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.98199999999999998,"15min":0.66800000000000004},"min":{"1min":0.34899999999999998,"5min":0.29799999999999999,"15min":0.27400000000000002},"max":{"1min":0.56899999999999995,"5min":11.234999999999999,"15min":11.234999999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.73599999999999999,"15min":0.61099999999999999},"min":{"1min":0.312,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":0.60199999999999998,"5min":9.923,"15min":9.923},"last":0.45100000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:55:08 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":1.1599999999999999,"15min":0.77500000000000002},"min":{"1min":0.33500000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.63200000000000001,"5min":10.606,"15min":10.606},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.879,"15min":0.68500000000000005},"min":{"1min":0.39700000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.59299999999999997,"5min":8.3819999999999997,"15min":8.3819999999999997},"last":0.51700000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757459,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2475716,"kb_used_data":2170236,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":91269436,"statfs":{"total":95995035648,"available":93459902464,"internally_reserved":0,"allocated":2222321664,"data_stored":2221569371,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:55:05 2023","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.58499999999999996,"15min":0.66600000000000004},"min":{"1min":0.25700000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.59099999999999997,"5min":2.7869999999999999,"15min":2.7869999999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.70099999999999996,"15min":0.76500000000000001},"min":{"1min":0.33700000000000002,"5min":0.314,"15min":0.314},"max":{"1min":3.2010000000000001,"5min":3.4140000000000001,"15min":3.4140000000000001},"last":0.38700000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":3.5019999999999998,"15min":1.5249999999999999},"min":{"1min":0.308,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.1269999999999998,"5min":114.247,"15min":114.247},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.49399999999999999,"5min":3.3610000000000002,"15min":1.4950000000000001},"min":{"1min":0.30099999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.85099999999999998,"5min":114.678,"15min":114.678},"last":0.26400000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:54:57 2023","interfaces":[{"interface":"back","average":{"1min":0.83599999999999997,"5min":2.3460000000000001,"15min":1.173},"min":{"1min":0.28399999999999997,"5min":0.25,"15min":0.25},"max":{"1min":3.843,"5min":106.66500000000001,"15min":106.66500000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.55700000000000005,"5min":2.2679999999999998,"15min":1.1479999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.3239999999999998,"5min":106.779,"15min":106.779},"last":0.29799999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:54:25 2023","interfaces":[{"interface":"back","average":{"1min":5.7779999999999996,"5min":8.9489999999999998,"15min":3.3439999999999999},"min":{"1min":0.32600000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":89.256,"5min":272.81299999999999,"15min":272.81299999999999},"last":2.0289999999999999},{"interface":"front","average":{"1min":6.1929999999999996,"5min":9.3379999999999992,"15min":3.5409999999999999},"min":{"1min":0.33200000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":89.427000000000007,"5min":273.00999999999999,"15min":273.00999999999999},"last":0.317}]},{"osd":4,"last update":"Mon Oct 9 05:54:17 2023","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.70899999999999996,"15min":0.55700000000000005},"min":{"1min":0.26400000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.68200000000000005,"5min":6.2050000000000001,"15min":6.2050000000000001},"last":0.501},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.68600000000000005,"15min":0.56699999999999995},"min":{"1min":0.31,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.51600000000000001,"5min":13.050000000000001,"15min":13.050000000000001},"last":0.47199999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:54:32 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.69799999999999995,"15min":0.57299999999999995},"min":{"1min":0.309,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.85099999999999998,"5min":10.606999999999999,"15min":10.606999999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.624,"5min":0.874,"15min":0.66200000000000003},"min":{"1min":0.307,"5min":0.30299999999999999,"15min":0.28499999999999998},"max":{"1min":3.173,"5min":5.9770000000000003,"15min":5.9770000000000003},"last":0.35799999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:54:11 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":1.02,"15min":0.72199999999999998},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.315},"max":{"1min":0.66200000000000003,"5min":9.2739999999999991,"15min":9.2739999999999991},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.5,"5min":0.72299999999999998,"15min":0.61899999999999999},"min":{"1min":0.32100000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.76400000000000001,"5min":5.032,"15min":5.032},"last":0.52700000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1474,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1388056576,"data_stored":1388052504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1604325376,"data_stored":1604321304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3349479424,"data_stored":3349479424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3344891904,"data_stored":3344891904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1801256960,"data_stored":1801256960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2209353728,"data_stored":2209349656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2756706304,"data_stored":2756706304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2655780864,"data_stored":2655780864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:55:14.005 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:55:14.006 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_0 pg_num 36 2023-10-09T05:55:14.068 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:14.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:14.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:14.174 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:14.225 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:14.338 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:14.350 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:14 smithi064 ceph-mon[212750]: from='client.44960 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:55:14.350 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:14 smithi064 ceph-mon[212750]: pgmap v464: 119 pgs: 3 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 96 op/s; 43361/263967 objects degraded (16.427%); 26152/263967 objects misplaced (9.907%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T05:55:14.350 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:55:14.350 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:14 smithi064 ceph-mon[220044]: from='client.44960 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:55:14.350 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:14 smithi064 ceph-mon[220044]: pgmap v464: 119 pgs: 3 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 96 op/s; 43361/263967 objects degraded (16.427%); 26152/263967 objects misplaced (9.907%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T05:55:14.350 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:55:14.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:14.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:14.452 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:14.475 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:14 smithi156 ceph-mon[195101]: from='client.44960 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:55:14.476 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:14 smithi156 ceph-mon[195101]: pgmap v464: 119 pgs: 3 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 96 op/s; 43361/263967 objects degraded (16.427%); 26152/263967 objects misplaced (9.907%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T05:55:14.476 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:55:14.477 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:14.531 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:14.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:14.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:14.638 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:14.669 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:15.247 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:15.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:15.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:15.353 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:15.391 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:15.441 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:15.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:15.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:15.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:55:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:55:15] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.43.0" 2023-10-09T05:55:15.521 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:15.547 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:15.580 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:15.632 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:15.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:15.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:15.740 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:15.767 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:15.779 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:55:15.779 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:55:15.833 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:15.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:15.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:15.938 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:15.988 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:16.184 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:55:16.364 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:16.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:16.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:16.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:16 smithi064 ceph-mon[212750]: pgmap v465: 119 pgs: 3 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 205 op/s; 43062/260139 objects degraded (16.553%); 25587/260139 objects misplaced (9.836%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:55:16.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:16 smithi064 ceph-mon[220044]: pgmap v465: 119 pgs: 3 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 205 op/s; 43062/260139 objects degraded (16.553%); 25587/260139 objects misplaced (9.836%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:55:16.471 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:16.514 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:16.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:16 smithi156 ceph-mon[195101]: pgmap v465: 119 pgs: 3 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 205 op/s; 43062/260139 objects degraded (16.553%); 25587/260139 objects misplaced (9.836%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:55:16.730 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:16.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:16.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:16.834 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:16.874 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:16.936 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:16.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:16.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:17.042 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:17.080 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:17.194 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T05:55:17.387 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:17 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/853914218' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:17.388 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:17 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:17.388 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:17 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 43062/260139 objects degraded (16.553%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:17.389 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:17 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/853914218' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:17.389 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:17 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:17.389 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:17 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 43062/260139 objects degraded (16.553%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:17.424 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:55:17.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:17 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/853914218' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:17.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:17 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:17.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:17 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 43062/260139 objects degraded (16.553%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:17.946 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:17.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:17.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:18.051 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:18.087 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:18.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:18 smithi064 ceph-mon[212750]: pgmap v466: 119 pgs: 3 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 257 op/s; 43062/258561 objects degraded (16.654%); 25587/258561 objects misplaced (9.896%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:55:18.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:18 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:55:18.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:18 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:55:18.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:18 smithi064 ceph-mon[212750]: osdmap e205: 8 total, 8 up, 7 in 2023-10-09T05:55:18.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:18 smithi064 ceph-mon[212750]: failed to encode map e205 with expected crc 2023-10-09T05:55:18.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:18 smithi064 ceph-mon[212750]: failed to encode map e205 with expected crc 2023-10-09T05:55:18.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:18 smithi064 ceph-mon[212750]: failed to encode map e205 with expected crc 2023-10-09T05:55:18.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:18 smithi064 ceph-mon[212750]: failed to encode map e205 with expected crc 2023-10-09T05:55:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:18 smithi064 ceph-mon[220044]: pgmap v466: 119 pgs: 3 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 257 op/s; 43062/258561 objects degraded (16.654%); 25587/258561 objects misplaced (9.896%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:55:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:18 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:55:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:18 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:55:18.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:18 smithi064 ceph-mon[220044]: osdmap e205: 8 total, 8 up, 7 in 2023-10-09T05:55:18.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:18 smithi064 ceph-mon[220044]: failed to encode map e205 with expected crc 2023-10-09T05:55:18.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:18 smithi064 ceph-mon[220044]: failed to encode map e205 with expected crc 2023-10-09T05:55:18.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:18 smithi064 ceph-mon[220044]: failed to encode map e205 with expected crc 2023-10-09T05:55:18.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:18 smithi064 ceph-mon[220044]: failed to encode map e205 with expected crc 2023-10-09T05:55:18.268 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:18.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:18.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:18.352 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:18.375 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:18.411 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:18.463 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:18.467 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:18 smithi156 ceph-mon[195101]: pgmap v466: 119 pgs: 3 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 257 op/s; 43062/258561 objects degraded (16.654%); 25587/258561 objects misplaced (9.896%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:55:18.467 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:18 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:55:18.467 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:18 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:55:18.467 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:18 smithi156 ceph-mon[195101]: osdmap e205: 8 total, 8 up, 7 in 2023-10-09T05:55:18.467 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:18 smithi156 ceph-mon[195101]: failed to encode map e205 with expected crc 2023-10-09T05:55:18.467 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:18 smithi156 ceph-mon[195101]: failed to encode map e205 with expected crc 2023-10-09T05:55:18.467 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:18 smithi156 ceph-mon[195101]: failed to encode map e205 with expected crc 2023-10-09T05:55:18.467 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:18 smithi156 ceph-mon[195101]: failed to encode map e205 with expected crc 2023-10-09T05:55:18.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:18.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:18.569 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:18.618 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:18.775 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:18.786 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:18.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:18.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:18.896 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:18.931 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:18.988 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:18.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:18.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:19.095 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:19.267 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 6 pg_num to 36 2023-10-09T05:55:19.309 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:19.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:19 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2454385460' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:55:19.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:19 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2454385460' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:55:19.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:19 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2454385460' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:55:20.265 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:55:20.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:20 smithi064 ceph-mon[220044]: pgmap v468: 119 pgs: 3 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 281 op/s; 43062/257853 objects degraded (16.700%); 25456/257853 objects misplaced (9.872%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:55:20.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:20 smithi064 ceph-mon[220044]: failed to encode map e206 with expected crc 2023-10-09T05:55:20.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:20 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2454385460' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]': finished 2023-10-09T05:55:20.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:20 smithi064 ceph-mon[220044]: osdmap e206: 8 total, 8 up, 7 in 2023-10-09T05:55:20.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:20 smithi064 ceph-mon[220044]: failed to encode map e206 with expected crc 2023-10-09T05:55:20.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:20 smithi064 ceph-mon[220044]: failed to encode map e206 with expected crc 2023-10-09T05:55:20.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:20 smithi064 ceph-mon[220044]: failed to encode map e206 with expected crc 2023-10-09T05:55:20.467 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:20 smithi064 ceph-mon[212750]: pgmap v468: 119 pgs: 3 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 281 op/s; 43062/257853 objects degraded (16.700%); 25456/257853 objects misplaced (9.872%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:55:20.467 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:20 smithi064 ceph-mon[212750]: failed to encode map e206 with expected crc 2023-10-09T05:55:20.467 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:20 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2454385460' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]': finished 2023-10-09T05:55:20.467 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:20 smithi064 ceph-mon[212750]: osdmap e206: 8 total, 8 up, 7 in 2023-10-09T05:55:20.467 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:20 smithi064 ceph-mon[212750]: failed to encode map e206 with expected crc 2023-10-09T05:55:20.467 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:20 smithi064 ceph-mon[212750]: failed to encode map e206 with expected crc 2023-10-09T05:55:20.467 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:20 smithi064 ceph-mon[212750]: failed to encode map e206 with expected crc 2023-10-09T05:55:20.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:20 smithi156 ceph-mon[195101]: pgmap v468: 119 pgs: 3 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 281 op/s; 43062/257853 objects degraded (16.700%); 25456/257853 objects misplaced (9.872%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:55:20.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:20 smithi156 ceph-mon[195101]: failed to encode map e206 with expected crc 2023-10-09T05:55:20.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:20 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2454385460' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]': finished 2023-10-09T05:55:20.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:20 smithi156 ceph-mon[195101]: osdmap e206: 8 total, 8 up, 7 in 2023-10-09T05:55:20.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:20 smithi156 ceph-mon[195101]: failed to encode map e206 with expected crc 2023-10-09T05:55:20.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:20 smithi156 ceph-mon[195101]: failed to encode map e206 with expected crc 2023-10-09T05:55:20.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:20 smithi156 ceph-mon[195101]: failed to encode map e206 with expected crc 2023-10-09T05:55:20.774 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:20.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:20.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:20.878 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:20.908 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:20.975 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:20.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:20.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:21.086 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:21.124 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:21.194 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:21 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T05:55:21.194 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:21 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T05:55:21.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:21 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T05:55:21.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:21 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T05:55:21.199 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:21.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:21.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:21.307 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:21.474 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:21 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T05:55:21.474 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:21 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T05:55:21.475 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:21.530 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:21.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:21.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:21.627 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:21.638 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:21.679 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:21.748 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:55:21.752 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:21.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:21.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:21.770 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:55:21.791 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:55:21.820 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:55:21.842 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:55:21.856 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:21.870 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:55:21.896 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:55:21.896 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:21.919 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:55:21.921 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:55:21.921 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:55:21.921 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:55:21.921 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:55:21.921 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:55:21.922 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:55:21.922 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:55:21.922 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:55:22.138 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:55:22.138 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T05:55:22.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T05:55:22.138 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T05:55:22.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.139 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:99073 6.4 6:27b1f89e:::benchmark_data_smithi064_284193_object2530:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-10-09T05:55:22.139 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:52:42.783709+0000", 2023-10-09T05:55:22.139 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 159.353076399, 2023-10-09T05:55:22.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4128961629999992, 2023-10-09T05:55:22.139 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:22.139 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:22.139 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:22.139 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:22.139 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:22.139 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 99073 2023-10-09T05:55:22.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.140 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:22.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:22.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:42.783709+0000", 2023-10-09T05:55:22.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:22.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:42.783709+0000", 2023-10-09T05:55:22.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:22.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:42.783708+0000", 2023-10-09T05:55:22.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:55:22.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:22.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:42.783714+0000", 2023-10-09T05:55:22.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.686e-06 2023-10-09T05:55:22.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:22.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:42.783728+0000", 2023-10-09T05:55:22.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3182e-05 2023-10-09T05:55:22.142 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.142 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:22.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:42.783735+0000", 2023-10-09T05:55:22.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3300000000000001e-06 2023-10-09T05:55:22.142 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.142 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:22.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:52.071234+0000", 2023-10-09T05:55:22.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2874990569999998 2023-10-09T05:55:22.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.143 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:22.143 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:52.083105+0000", 2023-10-09T05:55:22.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011871532000000001 2023-10-09T05:55:22.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.143 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-10-09T05:55:22.143 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:52.097849+0000", 2023-10-09T05:55:22.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014743898 2023-10-09T05:55:22.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:22.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:52.148584+0000", 2023-10-09T05:55:22.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.050735114999999997 2023-10-09T05:55:22.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:22.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:52.154161+0000", 2023-10-09T05:55:22.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.005577009 2023-10-09T05:55:22.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:22.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:52.160285+0000", 2023-10-09T05:55:22.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061234569999999997 2023-10-09T05:55:22.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:22.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:52.168785+0000", 2023-10-09T05:55:22.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085001279999999992 2023-10-09T05:55:22.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.146 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:22.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:52.196605+0000", 2023-10-09T05:55:22.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.027820107 2023-10-09T05:55:22.146 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.146 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:22.146 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.146 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.146 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.146 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:102951 6.1 6:89d59d23:::benchmark_data_smithi064_284193_object6408:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T05:55:22.146 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:53:23.929846+0000", 2023-10-09T05:55:22.146 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 118.206939085, 2023-10-09T05:55:22.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8206131490000002, 2023-10-09T05:55:22.147 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:22.147 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:22.147 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:22.147 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:22.147 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:22.147 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 102951 2023-10-09T05:55:22.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.147 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:22.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:22.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:23.929846+0000", 2023-10-09T05:55:22.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:22.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:23.929846+0000", 2023-10-09T05:55:22.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:22.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:23.929844+0000", 2023-10-09T05:55:22.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:22.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.149 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:22.149 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:23.929854+0000", 2023-10-09T05:55:22.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7399999999999999e-06 2023-10-09T05:55:22.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.149 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:22.149 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:23.929874+0000", 2023-10-09T05:55:22.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0010999999999999e-05 2023-10-09T05:55:22.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:22.150 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:23.929884+0000", 2023-10-09T05:55:22.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9350000000000006e-06 2023-10-09T05:55:22.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:22.150 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:25.492104+0000", 2023-10-09T05:55:22.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.562219748 2023-10-09T05:55:22.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:22.150 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:25.522159+0000", 2023-10-09T05:55:22.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.030054887999999998 2023-10-09T05:55:22.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:55:22.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:25.562754+0000", 2023-10-09T05:55:22.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.040595332999999997 2023-10-09T05:55:22.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:22.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:25.688811+0000", 2023-10-09T05:55:22.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.12605655199999999 2023-10-09T05:55:22.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:22.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:25.695108+0000", 2023-10-09T05:55:22.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062976580000000002 2023-10-09T05:55:22.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:22.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:25.708279+0000", 2023-10-09T05:55:22.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013170538000000001 2023-10-09T05:55:22.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.153 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:22.153 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:25.718337+0000", 2023-10-09T05:55:22.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010058064 2023-10-09T05:55:22.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.153 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.153 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:22.153 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:25.750459+0000", 2023-10-09T05:55:22.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.032122713999999997 2023-10-09T05:55:22.153 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.153 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:22.153 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.154 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:102957 6.1 6:8c8d5cfd:::benchmark_data_smithi064_284193_object6414:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T05:55:22.154 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:53:23.954991+0000", 2023-10-09T05:55:22.154 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 118.181794477, 2023-10-09T05:55:22.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.043506812, 2023-10-09T05:55:22.154 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:22.154 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:22.154 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:22.154 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:22.154 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:22.154 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 102957 2023-10-09T05:55:22.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.155 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:22.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:22.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:23.954991+0000", 2023-10-09T05:55:22.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:22.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:23.954991+0000", 2023-10-09T05:55:22.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:22.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:23.954989+0000", 2023-10-09T05:55:22.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:22.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:22.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:23.954999+0000", 2023-10-09T05:55:22.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0419e-05 2023-10-09T05:55:22.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.157 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.157 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:22.157 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:23.955019+0000", 2023-10-09T05:55:22.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9361999999999999e-05 2023-10-09T05:55:22.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.157 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.157 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:22.157 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:23.955030+0000", 2023-10-09T05:55:22.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1573000000000001e-05 2023-10-09T05:55:22.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:22.158 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:25.586302+0000", 2023-10-09T05:55:22.158 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.63127139 2023-10-09T05:55:22.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:22.158 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:25.617831+0000", 2023-10-09T05:55:22.158 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.031529308999999998 2023-10-09T05:55:22.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.159 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:55:22.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:25.656808+0000", 2023-10-09T05:55:22.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.038976643999999998 2023-10-09T05:55:22.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.159 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.159 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:22.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:25.700595+0000", 2023-10-09T05:55:22.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.043787482000000003 2023-10-09T05:55:22.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.159 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.159 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:22.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:25.706232+0000", 2023-10-09T05:55:22.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056365080000000001 2023-10-09T05:55:22.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:22.160 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:25.948694+0000", 2023-10-09T05:55:22.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24246211500000001 2023-10-09T05:55:22.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:22.160 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:25.959350+0000", 2023-10-09T05:55:22.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010656412000000001 2023-10-09T05:55:22.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.161 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:22.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:25.998498+0000", 2023-10-09T05:55:22.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.039147696000000003 2023-10-09T05:55:22.161 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.161 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:22.161 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.161 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:102979 6.1 6:85b64555:::benchmark_data_smithi064_284193_object6436:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T05:55:22.161 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:53:24.125777+0000", 2023-10-09T05:55:22.162 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 118.01100836099999, 2023-10-09T05:55:22.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.928281286, 2023-10-09T05:55:22.162 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:22.162 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:22.162 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:22.162 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:22.162 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:22.162 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 102979 2023-10-09T05:55:22.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.162 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:22.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:22.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:24.125777+0000", 2023-10-09T05:55:22.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:22.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:24.125777+0000", 2023-10-09T05:55:22.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:22.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:24.125775+0000", 2023-10-09T05:55:22.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:55:22.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:22.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:24.125784+0000", 2023-10-09T05:55:22.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5999999999999996e-06 2023-10-09T05:55:22.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:22.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:24.125800+0000", 2023-10-09T05:55:22.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5441e-05 2023-10-09T05:55:22.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:22.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:24.125810+0000", 2023-10-09T05:55:22.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0594000000000001e-05 2023-10-09T05:55:22.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:22.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:25.752036+0000", 2023-10-09T05:55:22.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.626225869 2023-10-09T05:55:22.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:22.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:25.778725+0000", 2023-10-09T05:55:22.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.026689065000000001 2023-10-09T05:55:22.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:55:22.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:25.813796+0000", 2023-10-09T05:55:22.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.035070388000000001 2023-10-09T05:55:22.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:22.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:25.940608+0000", 2023-10-09T05:55:22.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.12681218499999999 2023-10-09T05:55:22.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.167 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:22.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:25.946655+0000", 2023-10-09T05:55:22.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060467200000000002 2023-10-09T05:55:22.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.167 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:22.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:26.007086+0000", 2023-10-09T05:55:22.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.060431536000000001 2023-10-09T05:55:22.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:22.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:26.017514+0000", 2023-10-09T05:55:22.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010427677999999999 2023-10-09T05:55:22.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:22.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:26.054058+0000", 2023-10-09T05:55:22.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.036544475999999999 2023-10-09T05:55:22.168 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.168 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:22.168 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.169 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:103008 6.1 6:815bb87a:::benchmark_data_smithi064_284193_object6465:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-10-09T05:55:22.169 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:53:24.545789+0000", 2023-10-09T05:55:22.169 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 117.59099680600001, 2023-10-09T05:55:22.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.820309728, 2023-10-09T05:55:22.169 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:22.169 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:22.169 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:22.169 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:22.169 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:22.169 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 103008 2023-10-09T05:55:22.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.170 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:22.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:22.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:24.545789+0000", 2023-10-09T05:55:22.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:22.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:24.545789+0000", 2023-10-09T05:55:22.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:22.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:24.545787+0000", 2023-10-09T05:55:22.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:55:22.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:22.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:24.545794+0000", 2023-10-09T05:55:22.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0679999999999999e-06 2023-10-09T05:55:22.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:22.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:24.545808+0000", 2023-10-09T05:55:22.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4323e-05 2023-10-09T05:55:22.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:22.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:24.545819+0000", 2023-10-09T05:55:22.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0421000000000001e-05 2023-10-09T05:55:22.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:22.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:26.125315+0000", 2023-10-09T05:55:22.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5794960489999998 2023-10-09T05:55:22.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:22.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:26.153850+0000", 2023-10-09T05:55:22.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.028535130999999998 2023-10-09T05:55:22.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:55:22.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:26.191585+0000", 2023-10-09T05:55:22.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.037735261999999999 2023-10-09T05:55:22.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:22.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:26.302277+0000", 2023-10-09T05:55:22.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.11069159000000001 2023-10-09T05:55:22.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:22.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:26.307638+0000", 2023-10-09T05:55:22.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053616209999999996 2023-10-09T05:55:22.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:22.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:26.316093+0000", 2023-10-09T05:55:22.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0084546110000000008 2023-10-09T05:55:22.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:22.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:26.326467+0000", 2023-10-09T05:55:22.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010374242000000001 2023-10-09T05:55:22.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:22.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:53:26.366098+0000", 2023-10-09T05:55:22.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.039631022000000002 2023-10-09T05:55:22.176 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.176 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:22.176 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.176 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104142 6.1 6:8dcfcc8f:::benchmark_data_smithi064_284193_object7599:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:55:22.176 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:40.204920+0000", 2023-10-09T05:55:22.176 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 41.931865461000001, 2023-10-09T05:55:22.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6444858290000006, 2023-10-09T05:55:22.176 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:22.176 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:22.177 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:22.177 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:22.177 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:22.177 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104142 2023-10-09T05:55:22.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.177 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:22.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:22.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.204920+0000", 2023-10-09T05:55:22.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:22.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.204920+0000", 2023-10-09T05:55:22.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:22.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.204917+0000", 2023-10-09T05:55:22.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:55:22.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:22.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.204929+0000", 2023-10-09T05:55:22.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.163e-05 2023-10-09T05:55:22.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:22.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.204945+0000", 2023-10-09T05:55:22.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5988e-05 2023-10-09T05:55:22.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:22.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.204955+0000", 2023-10-09T05:55:22.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9219999999999992e-06 2023-10-09T05:55:22.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:22.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.554927+0000", 2023-10-09T05:55:22.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.349971901 2023-10-09T05:55:22.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:22.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.582332+0000", 2023-10-09T05:55:22.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.027405307 2023-10-09T05:55:22.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:55:22.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.615389+0000", 2023-10-09T05:55:22.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.033057214000000001 2023-10-09T05:55:22.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:22.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849384+0000", 2023-10-09T05:55:22.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.23399434 2023-10-09T05:55:22.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:22.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849406+0000", 2023-10-09T05:55:22.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2062999999999999e-05 2023-10-09T05:55:22.182 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.182 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:22.182 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.182 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104152 6.1 6.22f756c1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:55:22.182 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:40.286265+0000", 2023-10-09T05:55:22.182 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 41.850520392999996, 2023-10-09T05:55:22.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5631822680000003, 2023-10-09T05:55:22.182 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:22.182 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:22.183 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:22.183 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:22.183 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:22.183 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104152 2023-10-09T05:55:22.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.183 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:22.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:22.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.286265+0000", 2023-10-09T05:55:22.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:22.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.286265+0000", 2023-10-09T05:55:22.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:22.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.286263+0000", 2023-10-09T05:55:22.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:55:22.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:22.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.286271+0000", 2023-10-09T05:55:22.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9289999999999998e-06 2023-10-09T05:55:22.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:22.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.286285+0000", 2023-10-09T05:55:22.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3209e-05 2023-10-09T05:55:22.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:22.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.286293+0000", 2023-10-09T05:55:22.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4379999999999995e-06 2023-10-09T05:55:22.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:22.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849433+0000", 2023-10-09T05:55:22.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5631398769999993 2023-10-09T05:55:22.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:22.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849447+0000", 2023-10-09T05:55:22.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4353e-05 2023-10-09T05:55:22.186 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.186 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:22.186 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.187 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104178 6.1 6.53d24ae1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:55:22.187 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:40.851048+0000", 2023-10-09T05:55:22.187 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 41.285737584000003, 2023-10-09T05:55:22.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9984412760000003, 2023-10-09T05:55:22.187 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:22.187 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:22.187 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:22.187 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:22.187 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:22.187 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104178 2023-10-09T05:55:22.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.188 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:22.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:22.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.851048+0000", 2023-10-09T05:55:22.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:22.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.851048+0000", 2023-10-09T05:55:22.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:22.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.851046+0000", 2023-10-09T05:55:22.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:22.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:22.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.851055+0000", 2023-10-09T05:55:22.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9400000000000008e-06 2023-10-09T05:55:22.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:22.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.851070+0000", 2023-10-09T05:55:22.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5096000000000001e-05 2023-10-09T05:55:22.190 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:22.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.851082+0000", 2023-10-09T05:55:22.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2004000000000001e-05 2023-10-09T05:55:22.190 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:22.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849474+0000", 2023-10-09T05:55:22.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9983919610000003 2023-10-09T05:55:22.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:22.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849489+0000", 2023-10-09T05:55:22.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5037e-05 2023-10-09T05:55:22.191 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.191 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:22.191 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.191 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104195 6.1 6.f5ea83b1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:55:22.191 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:41.193752+0000", 2023-10-09T05:55:22.192 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 40.943033092999997, 2023-10-09T05:55:22.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.655772367, 2023-10-09T05:55:22.192 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:22.192 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:22.192 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:22.192 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:22.192 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:22.192 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104195 2023-10-09T05:55:22.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.192 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:22.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:22.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.193752+0000", 2023-10-09T05:55:22.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:22.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.193752+0000", 2023-10-09T05:55:22.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:22.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.193750+0000", 2023-10-09T05:55:22.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999971 2023-10-09T05:55:22.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:22.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.193760+0000", 2023-10-09T05:55:22.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0652999999999999e-05 2023-10-09T05:55:22.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:22.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.193778+0000", 2023-10-09T05:55:22.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7849000000000001e-05 2023-10-09T05:55:22.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:22.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.193788+0000", 2023-10-09T05:55:22.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8609999999999993e-06 2023-10-09T05:55:22.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:22.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849512+0000", 2023-10-09T05:55:22.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6557239150000003 2023-10-09T05:55:22.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:22.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849525+0000", 2023-10-09T05:55:22.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2850999999999999e-05 2023-10-09T05:55:22.196 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.196 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:22.196 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.196 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104206 6.1 6.fe55d9d1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:55:22.196 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:41.359887+0000", 2023-10-09T05:55:22.196 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 40.776898396999997, 2023-10-09T05:55:22.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4896904470000001, 2023-10-09T05:55:22.196 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:22.196 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:22.196 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:22.197 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:22.197 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:22.197 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104206 2023-10-09T05:55:22.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.197 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:22.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:22.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.359887+0000", 2023-10-09T05:55:22.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:22.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.359887+0000", 2023-10-09T05:55:22.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:22.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.359885+0000", 2023-10-09T05:55:22.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:22.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:22.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.359894+0000", 2023-10-09T05:55:22.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9309999999999997e-06 2023-10-09T05:55:22.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:22.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.359908+0000", 2023-10-09T05:55:22.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4185999999999999e-05 2023-10-09T05:55:22.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:22.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.359919+0000", 2023-10-09T05:55:22.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0611000000000001e-05 2023-10-09T05:55:22.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:22.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849563+0000", 2023-10-09T05:55:22.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4896442780000001 2023-10-09T05:55:22.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:22.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849577+0000", 2023-10-09T05:55:22.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4161999999999999e-05 2023-10-09T05:55:22.200 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.200 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:22.200 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.201 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104234 6.1 6.d0896e1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:55:22.201 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:41.671864+0000", 2023-10-09T05:55:22.201 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 40.464921623999999, 2023-10-09T05:55:22.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1777485309999998, 2023-10-09T05:55:22.201 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:22.201 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:22.201 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:22.201 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:22.201 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:22.201 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104234 2023-10-09T05:55:22.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.202 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:22.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:22.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.671864+0000", 2023-10-09T05:55:22.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:22.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.671864+0000", 2023-10-09T05:55:22.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:22.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.671862+0000", 2023-10-09T05:55:22.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:22.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:22.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.671872+0000", 2023-10-09T05:55:22.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0444e-05 2023-10-09T05:55:22.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:22.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.671891+0000", 2023-10-09T05:55:22.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8637000000000001e-05 2023-10-09T05:55:22.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:22.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.671901+0000", 2023-10-09T05:55:22.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7079999999999992e-06 2023-10-09T05:55:22.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:22.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849599+0000", 2023-10-09T05:55:22.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1776986379999999 2023-10-09T05:55:22.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.205 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:22.205 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849612+0000", 2023-10-09T05:55:22.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2996999999999999e-05 2023-10-09T05:55:22.205 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.205 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:22.205 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.205 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104244 6.1 6.700f7241 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:55:22.205 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:41.971632+0000", 2023-10-09T05:55:22.206 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 40.165153044, 2023-10-09T05:55:22.206 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8780479510000001, 2023-10-09T05:55:22.206 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:22.206 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:22.206 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:22.206 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:22.206 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:22.206 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104244 2023-10-09T05:55:22.206 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.206 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:22.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.207 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:22.207 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.971632+0000", 2023-10-09T05:55:22.207 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.207 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:22.207 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.971632+0000", 2023-10-09T05:55:22.207 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.207 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:22.207 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.971630+0000", 2023-10-09T05:55:22.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:22.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:22.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.971640+0000", 2023-10-09T05:55:22.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0138e-05 2023-10-09T05:55:22.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:22.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.971663+0000", 2023-10-09T05:55:22.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2561999999999998e-05 2023-10-09T05:55:22.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:22.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.971675+0000", 2023-10-09T05:55:22.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1970000000000001e-05 2023-10-09T05:55:22.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:22.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849666+0000", 2023-10-09T05:55:22.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8779913940000004 2023-10-09T05:55:22.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:22.210 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849680+0000", 2023-10-09T05:55:22.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3917000000000001e-05 2023-10-09T05:55:22.210 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.210 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:22.210 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.210 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104255 6.1 6.f28f24e1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:55:22.210 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:42.044751+0000", 2023-10-09T05:55:22.210 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 40.092034435000002, 2023-10-09T05:55:22.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8049857540000005, 2023-10-09T05:55:22.211 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:22.211 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:22.211 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:22.211 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:22.211 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:22.211 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104255 2023-10-09T05:55:22.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.211 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:22.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.211 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:22.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.044751+0000", 2023-10-09T05:55:22.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:22.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.044751+0000", 2023-10-09T05:55:22.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:22.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.044749+0000", 2023-10-09T05:55:22.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:55:22.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:22.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.044759+0000", 2023-10-09T05:55:22.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0732e-05 2023-10-09T05:55:22.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:22.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.044779+0000", 2023-10-09T05:55:22.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9806e-05 2023-10-09T05:55:22.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:22.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.044791+0000", 2023-10-09T05:55:22.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1562999999999999e-05 2023-10-09T05:55:22.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.214 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:22.214 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849721+0000", 2023-10-09T05:55:22.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8049299540000003 2023-10-09T05:55:22.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.214 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:22.214 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849737+0000", 2023-10-09T05:55:22.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6058000000000001e-05 2023-10-09T05:55:22.214 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.215 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:22.215 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.215 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104262 6.1 6.9203e6d1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:55:22.215 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:42.111284+0000", 2023-10-09T05:55:22.215 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 40.025501624999997, 2023-10-09T05:55:22.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.738488383, 2023-10-09T05:55:22.215 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:22.215 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:22.215 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:22.215 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:22.216 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:22.216 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104262 2023-10-09T05:55:22.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.216 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:22.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:22.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.111284+0000", 2023-10-09T05:55:22.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:22.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.111284+0000", 2023-10-09T05:55:22.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:22.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.111282+0000", 2023-10-09T05:55:22.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:55:22.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:22.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.111289+0000", 2023-10-09T05:55:22.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7549999999999997e-06 2023-10-09T05:55:22.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.218 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:22.218 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.111303+0000", 2023-10-09T05:55:22.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4024e-05 2023-10-09T05:55:22.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.218 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:22.218 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.111311+0000", 2023-10-09T05:55:22.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2930000000000001e-06 2023-10-09T05:55:22.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.218 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:22.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849760+0000", 2023-10-09T05:55:22.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7384484999999996 2023-10-09T05:55:22.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:22.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849772+0000", 2023-10-09T05:55:22.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2429000000000001e-05 2023-10-09T05:55:22.219 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.219 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:22.219 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.219 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104285 6.1 6.2bf38801 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:55:22.220 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:42.596255+0000", 2023-10-09T05:55:22.220 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 39.540530484000001, 2023-10-09T05:55:22.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2535693370000001, 2023-10-09T05:55:22.220 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:22.220 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:22.220 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:22.220 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:22.220 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:22.220 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104285 2023-10-09T05:55:22.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.220 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:22.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:22.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.596255+0000", 2023-10-09T05:55:22.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:22.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.596255+0000", 2023-10-09T05:55:22.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:22.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.596253+0000", 2023-10-09T05:55:22.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:55:22.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:22.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.596264+0000", 2023-10-09T05:55:22.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1596e-05 2023-10-09T05:55:22.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:22.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.596292+0000", 2023-10-09T05:55:22.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7569e-05 2023-10-09T05:55:22.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:22.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.596305+0000", 2023-10-09T05:55:22.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2629999999999999e-05 2023-10-09T05:55:22.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:22.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849810+0000", 2023-10-09T05:55:22.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2535054219999999 2023-10-09T05:55:22.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:22.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849824+0000", 2023-10-09T05:55:22.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4324999999999999e-05 2023-10-09T05:55:22.224 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.224 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:22.224 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.224 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.224 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104290 6.1 6.415302b1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:55:22.224 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:42.695406+0000", 2023-10-09T05:55:22.224 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 39.441379519000002, 2023-10-09T05:55:22.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1544539709999997, 2023-10-09T05:55:22.224 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:22.225 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:22.225 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:22.225 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:22.225 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:22.225 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104290 2023-10-09T05:55:22.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.225 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:22.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.225 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:22.225 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.695406+0000", 2023-10-09T05:55:22.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:22.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.695406+0000", 2023-10-09T05:55:22.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:22.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.695404+0000", 2023-10-09T05:55:22.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:22.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:22.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.695413+0000", 2023-10-09T05:55:22.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5669999999999995e-06 2023-10-09T05:55:22.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.227 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:22.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.695428+0000", 2023-10-09T05:55:22.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5549999999999999e-05 2023-10-09T05:55:22.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.227 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:22.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.695438+0000", 2023-10-09T05:55:22.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6479999999999998e-06 2023-10-09T05:55:22.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:22.228 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849847+0000", 2023-10-09T05:55:22.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1544087489999999 2023-10-09T05:55:22.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:22.228 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849860+0000", 2023-10-09T05:55:22.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3277e-05 2023-10-09T05:55:22.228 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.228 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:22.229 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.229 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.229 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.229 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104294 6.1 6.b5b3b3c1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:55:22.229 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:43.090700+0000", 2023-10-09T05:55:22.229 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 39.046085472000001, 2023-10-09T05:55:22.229 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.759744338, 2023-10-09T05:55:22.229 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:22.229 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:22.229 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:22.229 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:22.229 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:22.229 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104294 2023-10-09T05:55:22.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.230 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:22.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:22.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.090700+0000", 2023-10-09T05:55:22.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:22.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.090700+0000", 2023-10-09T05:55:22.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:22.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.090698+0000", 2023-10-09T05:55:22.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:55:22.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:22.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.090709+0000", 2023-10-09T05:55:22.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1293e-05 2023-10-09T05:55:22.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:22.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.090731+0000", 2023-10-09T05:55:22.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1994999999999999e-05 2023-10-09T05:55:22.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.232 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.232 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:22.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.090742+0000", 2023-10-09T05:55:22.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1344e-05 2023-10-09T05:55:22.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.232 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.232 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:22.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.850429+0000", 2023-10-09T05:55:22.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7596869939999999 2023-10-09T05:55:22.233 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.233 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:22.233 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.850444+0000", 2023-10-09T05:55:22.233 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4970000000000001e-05 2023-10-09T05:55:22.233 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.233 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:22.233 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.233 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.233 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104300 6.1 6.3350edd1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:55:22.233 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:43.258508+0000", 2023-10-09T05:55:22.234 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 38.878277113999999, 2023-10-09T05:55:22.234 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5919754709999996, 2023-10-09T05:55:22.234 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:22.234 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:22.234 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:22.234 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:22.234 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:22.234 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104300 2023-10-09T05:55:22.234 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.234 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:22.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.234 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:22.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.258508+0000", 2023-10-09T05:55:22.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.235 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:22.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.258508+0000", 2023-10-09T05:55:22.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.235 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:22.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.258506+0000", 2023-10-09T05:55:22.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:55:22.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:22.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.258516+0000", 2023-10-09T05:55:22.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7499999999999998e-06 2023-10-09T05:55:22.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:22.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.258534+0000", 2023-10-09T05:55:22.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8093000000000001e-05 2023-10-09T05:55:22.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.237 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:22.237 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.258543+0000", 2023-10-09T05:55:22.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2299999999999997e-06 2023-10-09T05:55:22.237 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.237 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.237 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:22.237 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.850473+0000", 2023-10-09T05:55:22.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5919303070000002 2023-10-09T05:55:22.237 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.237 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.237 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:22.237 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.850484+0000", 2023-10-09T05:55:22.238 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0562e-05 2023-10-09T05:55:22.238 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.238 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:22.238 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.238 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.238 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.238 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104341 6.1 6.f4610811 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T05:55:22.238 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:44.621089+0000", 2023-10-09T05:55:22.238 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 37.515696564999999, 2023-10-09T05:55:22.238 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2294316959999998, 2023-10-09T05:55:22.238 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:22.238 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:22.239 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:22.239 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:22.239 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:22.239 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104341 2023-10-09T05:55:22.239 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.239 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:22.239 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.239 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:22.239 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.621089+0000", 2023-10-09T05:55:22.239 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.239 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.239 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.240 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:22.240 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.621089+0000", 2023-10-09T05:55:22.240 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.240 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.240 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.240 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:22.240 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.621087+0000", 2023-10-09T05:55:22.240 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:22.240 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.240 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.240 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:22.240 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.621096+0000", 2023-10-09T05:55:22.241 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5129999999999996e-06 2023-10-09T05:55:22.241 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.241 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.241 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:22.241 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.621114+0000", 2023-10-09T05:55:22.241 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8530999999999999e-05 2023-10-09T05:55:22.241 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.241 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.241 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:22.241 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.621122+0000", 2023-10-09T05:55:22.241 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0350000000000001e-06 2023-10-09T05:55:22.241 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.242 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.242 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:22.242 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.850510+0000", 2023-10-09T05:55:22.242 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2293879170000004 2023-10-09T05:55:22.242 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.242 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.242 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:22.242 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.850521+0000", 2023-10-09T05:55:22.242 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0424e-05 2023-10-09T05:55:22.242 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.242 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:22.242 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.243 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.243 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.243 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104342 6.1 6.841a5a71 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T05:55:22.243 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:44.645679+0000", 2023-10-09T05:55:22.243 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 37.491106268999999, 2023-10-09T05:55:22.243 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2049086400000002, 2023-10-09T05:55:22.243 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:22.243 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:55:22.243 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:22.243 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:22.243 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:22.243 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104342 2023-10-09T05:55:22.244 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.244 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:22.244 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.244 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:22.244 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.645679+0000", 2023-10-09T05:55:22.244 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.244 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.244 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.244 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:22.244 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.645679+0000", 2023-10-09T05:55:22.244 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:22.244 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.245 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.245 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:22.245 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.645676+0000", 2023-10-09T05:55:22.245 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999971 2023-10-09T05:55:22.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.245 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.245 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:22.245 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.645693+0000", 2023-10-09T05:55:22.245 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6878e-05 2023-10-09T05:55:22.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.245 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.245 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:22.245 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.645719+0000", 2023-10-09T05:55:22.246 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5686000000000001e-05 2023-10-09T05:55:22.246 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.246 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:22.246 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:44.645735+0000", 2023-10-09T05:55:22.246 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6005999999999999e-05 2023-10-09T05:55:22.246 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.246 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:22.246 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.850577+0000", 2023-10-09T05:55:22.246 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2048425490000003 2023-10-09T05:55:22.246 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:22.247 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:22.247 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:22.247 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.850588+0000", 2023-10-09T05:55:22.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0485e-05 2023-10-09T05:55:22.247 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.247 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:22.247 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.247 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:22.247 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:22.247 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:55:22.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:22 smithi064 ceph-mon[220044]: pgmap v470: 119 pgs: 3 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 499 op/s; 42712/251976 objects degraded (16.951%); 24820/251976 objects misplaced (9.850%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T05:55:22.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:22 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T05:55:22.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:22 smithi064 ceph-mon[220044]: osdmap e207: 8 total, 8 up, 7 in 2023-10-09T05:55:22.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:22 smithi064 ceph-mon[220044]: failed to encode map e207 with expected crc 2023-10-09T05:55:22.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:22 smithi064 ceph-mon[220044]: failed to encode map e207 with expected crc 2023-10-09T05:55:22.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:22 smithi064 ceph-mon[220044]: failed to encode map e207 with expected crc 2023-10-09T05:55:22.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:22 smithi064 ceph-mon[220044]: failed to encode map e207 with expected crc 2023-10-09T05:55:22.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:22 smithi064 ceph-mon[220044]: failed to encode map e207 with expected crc 2023-10-09T05:55:22.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:22 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4080416024' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:22.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:22 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 42712/251976 objects degraded (16.951%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:22.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:22 smithi064 ceph-mon[220044]: osdmap e208: 8 total, 8 up, 7 in 2023-10-09T05:55:22.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:22 smithi064 ceph-mon[212750]: pgmap v470: 119 pgs: 3 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 499 op/s; 42712/251976 objects degraded (16.951%); 24820/251976 objects misplaced (9.850%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T05:55:22.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:22 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T05:55:22.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:22 smithi064 ceph-mon[212750]: osdmap e207: 8 total, 8 up, 7 in 2023-10-09T05:55:22.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:22 smithi064 ceph-mon[212750]: failed to encode map e207 with expected crc 2023-10-09T05:55:22.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:22 smithi064 ceph-mon[212750]: failed to encode map e207 with expected crc 2023-10-09T05:55:22.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:22 smithi064 ceph-mon[212750]: failed to encode map e207 with expected crc 2023-10-09T05:55:22.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:22 smithi064 ceph-mon[212750]: failed to encode map e207 with expected crc 2023-10-09T05:55:22.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:22 smithi064 ceph-mon[212750]: failed to encode map e207 with expected crc 2023-10-09T05:55:22.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:22 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4080416024' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:22.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:22 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 42712/251976 objects degraded (16.951%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:22.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:22 smithi064 ceph-mon[212750]: osdmap e208: 8 total, 8 up, 7 in 2023-10-09T05:55:22.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:22 smithi156 ceph-mon[195101]: pgmap v470: 119 pgs: 3 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 499 op/s; 42712/251976 objects degraded (16.951%); 24820/251976 objects misplaced (9.850%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T05:55:22.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:22 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T05:55:22.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:22 smithi156 ceph-mon[195101]: osdmap e207: 8 total, 8 up, 7 in 2023-10-09T05:55:22.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:22 smithi156 ceph-mon[195101]: failed to encode map e207 with expected crc 2023-10-09T05:55:22.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:22 smithi156 ceph-mon[195101]: failed to encode map e207 with expected crc 2023-10-09T05:55:22.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:22 smithi156 ceph-mon[195101]: failed to encode map e207 with expected crc 2023-10-09T05:55:22.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:22 smithi156 ceph-mon[195101]: failed to encode map e207 with expected crc 2023-10-09T05:55:22.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:22 smithi156 ceph-mon[195101]: failed to encode map e207 with expected crc 2023-10-09T05:55:22.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:22 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4080416024' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:22.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:22 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 42712/251976 objects degraded (16.951%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:22.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:22 smithi156 ceph-mon[195101]: osdmap e208: 8 total, 8 up, 7 in 2023-10-09T05:55:23.012 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:23.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:23.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:23.116 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:23.142 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:23.209 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:23.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:23.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:23.315 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:23.342 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:23.388 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:23.392 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:23 smithi156 ceph-mon[195101]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.392 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:23 smithi156 ceph-mon[195101]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.392 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:23 smithi156 ceph-mon[195101]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.392 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:23 smithi156 ceph-mon[195101]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.392 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:23 smithi156 ceph-mon[195101]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.392 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:23 smithi156 ceph-mon[195101]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.392 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:23 smithi156 ceph-mon[195101]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.392 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:23 smithi156 ceph-mon[195101]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.392 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:23 smithi156 ceph-mon[195101]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.392 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:23 smithi156 ceph-mon[195101]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.392 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:23 smithi156 ceph-mon[195101]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.393 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:23 smithi156 ceph-mon[195101]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.393 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:23 smithi156 ceph-mon[195101]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.393 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:23 smithi156 ceph-mon[195101]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.393 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:23 smithi156 ceph-mon[195101]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.393 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:23 smithi156 ceph-mon[195101]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.393 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:23 smithi156 ceph-mon[195101]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.393 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:23 smithi156 ceph-mon[195101]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.393 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:23 smithi156 ceph-mon[195101]: 6.d continuing backfill to osd.2 from (108'2175,206'6859] MIN to 206'6859 2023-10-09T05:55:23.393 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:23 smithi156 ceph-mon[195101]: 6.e continuing backfill to osd.7 from (107'1851,206'6823] MIN to 206'6823 2023-10-09T05:55:23.393 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:23 smithi156 ceph-mon[195101]: 6.f continuing backfill to osd.5 from (107'1991,206'6935] MIN to 206'6935 2023-10-09T05:55:23.393 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:23 smithi156 ceph-mon[195101]: osdmap e209: 8 total, 8 up, 7 in 2023-10-09T05:55:23.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:23.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:23.424 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[220044]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[220044]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[220044]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[220044]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[220044]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[220044]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[220044]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[220044]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[220044]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[220044]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.426 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[220044]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.426 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[220044]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.426 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[220044]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.426 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[220044]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.426 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[220044]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.426 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[220044]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.427 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[220044]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.427 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[220044]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.427 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[220044]: 6.d continuing backfill to osd.2 from (108'2175,206'6859] MIN to 206'6859 2023-10-09T05:55:23.428 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[220044]: 6.e continuing backfill to osd.7 from (107'1851,206'6823] MIN to 206'6823 2023-10-09T05:55:23.428 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[220044]: 6.f continuing backfill to osd.5 from (107'1991,206'6935] MIN to 206'6935 2023-10-09T05:55:23.428 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[220044]: osdmap e209: 8 total, 8 up, 7 in 2023-10-09T05:55:23.428 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.428 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.428 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.428 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.428 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.429 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.429 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.429 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.429 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.429 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.429 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.429 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.429 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.429 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.429 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.430 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.430 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.430 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T05:55:23.430 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[212750]: 6.d continuing backfill to osd.2 from (108'2175,206'6859] MIN to 206'6859 2023-10-09T05:55:23.430 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[212750]: 6.e continuing backfill to osd.7 from (107'1851,206'6823] MIN to 206'6823 2023-10-09T05:55:23.430 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[212750]: 6.f continuing backfill to osd.5 from (107'1991,206'6935] MIN to 206'6935 2023-10-09T05:55:23.430 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:23 smithi064 ceph-mon[212750]: osdmap e209: 8 total, 8 up, 7 in 2023-10-09T05:55:23.495 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:23.525 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:55:23.530 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:23.637 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:23.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:23.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:23.742 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:23.774 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:23.977 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:23.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:23.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:24.082 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:24.109 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:24.150 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:24.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:24.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:24.255 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:24.292 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:24.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:24 smithi156 ceph-mon[195101]: pgmap v473: 129 pgs: 1 peering, 1 remapped+peering, 10 unknown, 3 active+recovery_wait+degraded+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 39710/239502 objects degraded (16.580%); 23071/239502 objects misplaced (9.633%) 2023-10-09T05:55:24.374 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:24 smithi156 ceph-mon[195101]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.374 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:24 smithi156 ceph-mon[195101]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.374 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:24 smithi156 ceph-mon[195101]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.374 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:24 smithi156 ceph-mon[195101]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.374 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:24 smithi156 ceph-mon[195101]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.374 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:24 smithi156 ceph-mon[195101]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.374 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:24 smithi156 ceph-mon[195101]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.374 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:24 smithi156 ceph-mon[195101]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.374 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:24 smithi156 ceph-mon[195101]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.375 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:24 smithi156 ceph-mon[195101]: 6.1 continuing backfill to osd.0 from (107'871,206'6765] MIN to 206'6765 2023-10-09T05:55:24.375 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:24 smithi156 ceph-mon[195101]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.375 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:24 smithi156 ceph-mon[195101]: osdmap e210: 8 total, 8 up, 7 in 2023-10-09T05:55:24.380 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:24.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:24.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:24.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[212750]: pgmap v473: 129 pgs: 1 peering, 1 remapped+peering, 10 unknown, 3 active+recovery_wait+degraded+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 39710/239502 objects degraded (16.580%); 23071/239502 objects misplaced (9.633%) 2023-10-09T05:55:24.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[212750]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[212750]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[212750]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[212750]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[212750]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[212750]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[212750]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[212750]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[212750]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[212750]: 6.1 continuing backfill to osd.0 from (107'871,206'6765] MIN to 206'6765 2023-10-09T05:55:24.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[212750]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[212750]: osdmap e210: 8 total, 8 up, 7 in 2023-10-09T05:55:24.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[220044]: pgmap v473: 129 pgs: 1 peering, 1 remapped+peering, 10 unknown, 3 active+recovery_wait+degraded+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 39710/239502 objects degraded (16.580%); 23071/239502 objects misplaced (9.633%) 2023-10-09T05:55:24.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[220044]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[220044]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[220044]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[220044]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.474 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[220044]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.474 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[220044]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.474 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[220044]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.474 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[220044]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.474 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[220044]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.474 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[220044]: 6.1 continuing backfill to osd.0 from (107'871,206'6765] MIN to 206'6765 2023-10-09T05:55:24.474 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[220044]: failed to encode map e209 with expected crc 2023-10-09T05:55:24.474 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:24 smithi064 ceph-mon[220044]: osdmap e210: 8 total, 8 up, 7 in 2023-10-09T05:55:24.490 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:24.546 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:24.616 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:24.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:24.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:24.699 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:24.722 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:24.767 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:24.855 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:24.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:24.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:24.959 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:24.992 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:25.055 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:25.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:25.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:25.160 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:25.186 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:25.235 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:25.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:25.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:25.340 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:25.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:25 smithi156 ceph-mon[195101]: failed to encode map e210 with expected crc 2023-10-09T05:55:25.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:25 smithi156 ceph-mon[195101]: failed to encode map e210 with expected crc 2023-10-09T05:55:25.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:25 smithi156 ceph-mon[195101]: failed to encode map e210 with expected crc 2023-10-09T05:55:25.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:25 smithi156 ceph-mon[195101]: failed to encode map e210 with expected crc 2023-10-09T05:55:25.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:25 smithi156 ceph-mon[195101]: failed to encode map e210 with expected crc 2023-10-09T05:55:25.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:25 smithi156 ceph-mon[195101]: failed to encode map e210 with expected crc 2023-10-09T05:55:25.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:25 smithi156 ceph-mon[195101]: failed to encode map e210 with expected crc 2023-10-09T05:55:25.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:25 smithi156 ceph-mon[195101]: 6.21 continuing backfill to osd.0 from (107'871,206'6765] MIN to 206'6765 2023-10-09T05:55:25.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:25 smithi156 ceph-mon[195101]: 6.1f continuing backfill to osd.5 from (107'1991,206'6935] MIN to 206'6935 2023-10-09T05:55:25.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:25 smithi156 ceph-mon[195101]: 6.1e continuing backfill to osd.7 from (107'1851,206'6823] MIN to 206'6823 2023-10-09T05:55:25.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:25 smithi156 ceph-mon[195101]: 6.1d continuing backfill to osd.2 from (108'2175,206'6859] MIN to 206'6859 2023-10-09T05:55:25.382 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:25 smithi064 ceph-mon[212750]: failed to encode map e210 with expected crc 2023-10-09T05:55:25.382 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:25 smithi064 ceph-mon[212750]: failed to encode map e210 with expected crc 2023-10-09T05:55:25.382 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:25 smithi064 ceph-mon[212750]: failed to encode map e210 with expected crc 2023-10-09T05:55:25.383 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:25 smithi064 ceph-mon[212750]: failed to encode map e210 with expected crc 2023-10-09T05:55:25.383 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:25 smithi064 ceph-mon[212750]: failed to encode map e210 with expected crc 2023-10-09T05:55:25.383 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:25 smithi064 ceph-mon[212750]: failed to encode map e210 with expected crc 2023-10-09T05:55:25.383 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:25 smithi064 ceph-mon[212750]: failed to encode map e210 with expected crc 2023-10-09T05:55:25.383 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:25 smithi064 ceph-mon[212750]: 6.21 continuing backfill to osd.0 from (107'871,206'6765] MIN to 206'6765 2023-10-09T05:55:25.383 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:25 smithi064 ceph-mon[212750]: 6.1f continuing backfill to osd.5 from (107'1991,206'6935] MIN to 206'6935 2023-10-09T05:55:25.383 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:25 smithi064 ceph-mon[212750]: 6.1e continuing backfill to osd.7 from (107'1851,206'6823] MIN to 206'6823 2023-10-09T05:55:25.383 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:25 smithi064 ceph-mon[212750]: 6.1d continuing backfill to osd.2 from (108'2175,206'6859] MIN to 206'6859 2023-10-09T05:55:25.383 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:25 smithi064 ceph-mon[220044]: failed to encode map e210 with expected crc 2023-10-09T05:55:25.384 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:25 smithi064 ceph-mon[220044]: failed to encode map e210 with expected crc 2023-10-09T05:55:25.384 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:25 smithi064 ceph-mon[220044]: failed to encode map e210 with expected crc 2023-10-09T05:55:25.384 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:25 smithi064 ceph-mon[220044]: failed to encode map e210 with expected crc 2023-10-09T05:55:25.384 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:25 smithi064 ceph-mon[220044]: failed to encode map e210 with expected crc 2023-10-09T05:55:25.384 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:25 smithi064 ceph-mon[220044]: failed to encode map e210 with expected crc 2023-10-09T05:55:25.384 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:25 smithi064 ceph-mon[220044]: failed to encode map e210 with expected crc 2023-10-09T05:55:25.384 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:25 smithi064 ceph-mon[220044]: 6.21 continuing backfill to osd.0 from (107'871,206'6765] MIN to 206'6765 2023-10-09T05:55:25.384 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:25 smithi064 ceph-mon[220044]: 6.1f continuing backfill to osd.5 from (107'1991,206'6935] MIN to 206'6935 2023-10-09T05:55:25.384 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:25 smithi064 ceph-mon[220044]: 6.1e continuing backfill to osd.7 from (107'1851,206'6823] MIN to 206'6823 2023-10-09T05:55:25.385 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:25 smithi064 ceph-mon[220044]: 6.1d continuing backfill to osd.2 from (108'2175,206'6859] MIN to 206'6859 2023-10-09T05:55:25.385 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:55:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:55:25] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.43.0" 2023-10-09T05:55:25.387 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:25.450 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:55:25.554 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T05:55:25.554 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:55:25.554 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-09T05:55:25.554 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:55:25.555 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:55:25.555 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-09T05:55:25.556 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:25.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:25.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:25.667 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:25.696 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:25.750 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:25.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:25.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:25.854 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:25.891 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:25.951 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:25.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:25.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:26.055 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:26.098 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:26.160 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:26.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:26.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:26.228 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:26.261 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T05:55:26.268 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:26.287 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:26.300 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:26.348 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:26.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:26.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:26.454 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:26.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:26 smithi064 ceph-mon[212750]: pgmap v476: 129 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 peering, 1 remapped+peering, 4 unknown, 4 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 39487/237516 objects degraded (16.625%); 22714/237516 objects misplaced (9.563%) 2023-10-09T05:55:26.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:26 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/153480121' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:55:26.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:26 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:55:26.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:26 smithi064 ceph-mon[220044]: pgmap v476: 129 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 peering, 1 remapped+peering, 4 unknown, 4 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 39487/237516 objects degraded (16.625%); 22714/237516 objects misplaced (9.563%) 2023-10-09T05:55:26.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:26 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/153480121' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:55:26.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:26 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:55:26.482 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:26 smithi156 ceph-mon[195101]: pgmap v476: 129 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 peering, 1 remapped+peering, 4 unknown, 4 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 39487/237516 objects degraded (16.625%); 22714/237516 objects misplaced (9.563%) 2023-10-09T05:55:26.482 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:26 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/153480121' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:55:26.482 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:26 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:55:26.485 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:26.546 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:26.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:26.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:26.651 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:26.683 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:26.739 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:26.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:26.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:26.845 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:26.878 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:27.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:27 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:27.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:27 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:55:27.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:27 smithi064 ceph-mon[212750]: osdmap e211: 8 total, 8 up, 7 in 2023-10-09T05:55:27.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:27 smithi064 ceph-mon[212750]: failed to encode map e211 with expected crc 2023-10-09T05:55:27.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:27 smithi064 ceph-mon[212750]: failed to encode map e211 with expected crc 2023-10-09T05:55:27.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:27 smithi064 ceph-mon[212750]: failed to encode map e211 with expected crc 2023-10-09T05:55:27.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:27 smithi064 ceph-mon[212750]: failed to encode map e211 with expected crc 2023-10-09T05:55:27.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:27 smithi064 ceph-mon[212750]: pgmap v478: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 peering, 1 remapped+peering, 2 unknown, 4 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 223 op/s; 39452/236565 objects degraded (16.677%); 22714/236565 objects misplaced (9.602%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T05:55:27.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:27 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 39487/237516 objects degraded (16.625%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:27.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:27 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:27.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:27 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:55:27.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:27 smithi064 ceph-mon[220044]: osdmap e211: 8 total, 8 up, 7 in 2023-10-09T05:55:27.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:27 smithi064 ceph-mon[220044]: failed to encode map e211 with expected crc 2023-10-09T05:55:27.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:27 smithi064 ceph-mon[220044]: failed to encode map e211 with expected crc 2023-10-09T05:55:27.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:27 smithi064 ceph-mon[220044]: failed to encode map e211 with expected crc 2023-10-09T05:55:27.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:27 smithi064 ceph-mon[220044]: failed to encode map e211 with expected crc 2023-10-09T05:55:27.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:27 smithi064 ceph-mon[220044]: pgmap v478: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 peering, 1 remapped+peering, 2 unknown, 4 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 223 op/s; 39452/236565 objects degraded (16.677%); 22714/236565 objects misplaced (9.602%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T05:55:27.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:27 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 39487/237516 objects degraded (16.625%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:27.495 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:27.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:27.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:27.600 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:27.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:27 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:27.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:27 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:55:27.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:27 smithi156 ceph-mon[195101]: osdmap e211: 8 total, 8 up, 7 in 2023-10-09T05:55:27.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:27 smithi156 ceph-mon[195101]: failed to encode map e211 with expected crc 2023-10-09T05:55:27.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:27 smithi156 ceph-mon[195101]: failed to encode map e211 with expected crc 2023-10-09T05:55:27.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:27 smithi156 ceph-mon[195101]: failed to encode map e211 with expected crc 2023-10-09T05:55:27.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:27 smithi156 ceph-mon[195101]: failed to encode map e211 with expected crc 2023-10-09T05:55:27.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:27 smithi156 ceph-mon[195101]: pgmap v478: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 peering, 1 remapped+peering, 2 unknown, 4 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 223 op/s; 39452/236565 objects degraded (16.677%); 22714/236565 objects misplaced (9.602%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T05:55:27.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:27 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 39487/237516 objects degraded (16.625%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:27.649 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:27.929 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:27.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:27.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:28.033 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:28.073 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:28.145 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:28.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:28.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:28.217 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:55:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T05:55:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2023-10-09T05:55:28.218 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:55:28.251 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:28.288 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:28.577 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:28 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:55:28.578 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:28 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:55:28.594 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:28.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:28.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:28.699 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:28.740 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:28.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:28 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:55:29.746 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:55:29.775 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:29 smithi064 ceph-mon[220044]: pgmap v479: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47 MiB/s wr, 1.05k op/s; 42408/247758 objects degraded (17.117%); 24245/247758 objects misplaced (9.786%); 34 MiB/s, 555 objects/s recovering 2023-10-09T05:55:29.775 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:29.776 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:29 smithi064 ceph-mon[212750]: pgmap v479: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47 MiB/s wr, 1.05k op/s; 42408/247758 objects degraded (17.117%); 24245/247758 objects misplaced (9.786%); 34 MiB/s, 555 objects/s recovering 2023-10-09T05:55:29.776 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:29 smithi156 ceph-mon[195101]: pgmap v479: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47 MiB/s wr, 1.05k op/s; 42408/247758 objects degraded (17.117%); 24245/247758 objects misplaced (9.786%); 34 MiB/s, 555 objects/s recovering 2023-10-09T05:55:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:29.882 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:55:29.952 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:29.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:29.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:30.057 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:30.141 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:30.363 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:30.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:30.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:30.472 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:30.534 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:30.619 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:30.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:30.627 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:30 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:30.627 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:30 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:30.627 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:30 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:30.627 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:30 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:30.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:30 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:30.628 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:30 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:30.628 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:30 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:30.628 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:30 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:30.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:30.633 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:55:30.669 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:55:30.702 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:55:30.729 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:30.734 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:55:30.773 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:30 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:30.773 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:30 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:30.773 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:30 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:30.773 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:30 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:30.774 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:55:30.775 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:30.796 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:55:30.819 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:55:30.844 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:55:30.846 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:55:30.846 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:55:30.846 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:55:30.846 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:55:30.846 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:55:30.846 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:55:30.846 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:55:30.846 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:55:30.881 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:30.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:30.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:30.986 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:31.025 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:31.038 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:55:31.038 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T05:55:31.038 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:55:31.066 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:31.133 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:31.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:31.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:31.239 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:31.267 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:31.314 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:31.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:31.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:31.420 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:31.456 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:31.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:31 smithi064 ceph-mon[212750]: pgmap v480: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 38 MiB/s wr, 1.11k op/s; 41945/242103 objects degraded (17.325%); 23593/242103 objects misplaced (9.745%); 31 MiB/s, 514 objects/s recovering 2023-10-09T05:55:31.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:31 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2562715448' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:31.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:31 smithi064 ceph-mon[220044]: pgmap v480: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 38 MiB/s wr, 1.11k op/s; 41945/242103 objects degraded (17.325%); 23593/242103 objects misplaced (9.745%); 31 MiB/s, 514 objects/s recovering 2023-10-09T05:55:31.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:31 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2562715448' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:31.763 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:31.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:31.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:31 smithi156 ceph-mon[195101]: pgmap v480: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 38 MiB/s wr, 1.11k op/s; 41945/242103 objects degraded (17.325%); 23593/242103 objects misplaced (9.745%); 31 MiB/s, 514 objects/s recovering 2023-10-09T05:55:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:31 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2562715448' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:31.871 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:31.909 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:32.578 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:32.654 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:32.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:32.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:32.760 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:32.812 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:32 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 41945/242103 objects degraded (17.325%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:32.812 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:32.832 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:32 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 41945/242103 objects degraded (17.325%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:32.833 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:32 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 41945/242103 objects degraded (17.325%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:32.882 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:32.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:32.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:32.991 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:33.044 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:33.110 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:33.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:33.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:33.217 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:33.250 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:33.309 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:33.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:33.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:33.419 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:33.449 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:33.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:33 smithi064 ceph-mon[212750]: pgmap v481: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 34 MiB/s wr, 1.00k op/s; 41863/241152 objects degraded (17.360%); 23434/241152 objects misplaced (9.718%); 27 MiB/s, 443 objects/s recovering 2023-10-09T05:55:33.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:33 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:33.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:33 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:33.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:33 smithi064 ceph-mon[220044]: pgmap v481: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 34 MiB/s wr, 1.00k op/s; 41863/241152 objects degraded (17.360%); 23434/241152 objects misplaced (9.718%); 27 MiB/s, 443 objects/s recovering 2023-10-09T05:55:33.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:33 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:33.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:33 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:33.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:33 smithi156 ceph-mon[195101]: pgmap v481: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 34 MiB/s wr, 1.00k op/s; 41863/241152 objects degraded (17.360%); 23434/241152 objects misplaced (9.718%); 27 MiB/s, 443 objects/s recovering 2023-10-09T05:55:33.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:33 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:33.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:33 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:34.287 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:34.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:34.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:34.391 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:34.452 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:34.520 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:55:34.652 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:34.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:34.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:34.758 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:34.800 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:35.280 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:35.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:35.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:35.385 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:35.446 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:55:35.446 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:55:35.464 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:35.478 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:35.616 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:35 smithi064 ceph-mon[212750]: pgmap v482: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 28 MiB/s wr, 943 op/s; 41464/236460 objects degraded (17.535%); 22735/236460 objects misplaced (9.615%); 25 MiB/s, 401 objects/s recovering 2023-10-09T05:55:35.616 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:55:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:55:35] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.43.0" 2023-10-09T05:55:35.668 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:35.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:35.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:35.776 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:35.806 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:35.866 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:35.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:35 smithi156 ceph-mon[195101]: pgmap v482: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 28 MiB/s wr, 943 op/s; 41464/236460 objects degraded (17.535%); 22735/236460 objects misplaced (9.615%); 25 MiB/s, 401 objects/s recovering 2023-10-09T05:55:35.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:35.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:35.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:35 smithi064 ceph-mon[220044]: pgmap v482: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 28 MiB/s wr, 943 op/s; 41464/236460 objects degraded (17.535%); 22735/236460 objects misplaced (9.615%); 25 MiB/s, 401 objects/s recovering 2023-10-09T05:55:35.972 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:36.002 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:36.050 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:36.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:36.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:36.158 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:36.204 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:36.389 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:36.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:36.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:36.494 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:36.498 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:55:36.526 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:36.581 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:36.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:36.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:36.653 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T05:55:36.690 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:36.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:36 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1940910654' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:36.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:36 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:36.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:36 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1940910654' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:36.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:36 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:36.756 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:36.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:36 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1940910654' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:36.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:36 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:37.603 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T05:55:37.604 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:55:37.604 INFO:tasks.thrashosds.thrasher:Adding osd 1 2023-10-09T05:55:37.604 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd in 1 2023-10-09T05:55:37.666 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:37.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:37.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:37.781 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:37.813 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:37 smithi156 ceph-mon[195101]: pgmap v483: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 28 MiB/s wr, 945 op/s; 41302/234618 objects degraded (17.604%); 22735/234618 objects misplaced (9.690%); 25 MiB/s, 402 objects/s recovering 2023-10-09T05:55:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:37 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:37 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:55:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:37 smithi156 ceph-mon[195101]: osdmap e212: 8 total, 8 up, 7 in 2023-10-09T05:55:37.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:37 smithi156 ceph-mon[195101]: failed to encode map e212 with expected crc 2023-10-09T05:55:37.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:37 smithi156 ceph-mon[195101]: failed to encode map e212 with expected crc 2023-10-09T05:55:37.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:37 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 41302/234618 objects degraded (17.604%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:37.878 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:37.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:37.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:37.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:37 smithi064 ceph-mon[212750]: pgmap v483: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 28 MiB/s wr, 945 op/s; 41302/234618 objects degraded (17.604%); 22735/234618 objects misplaced (9.690%); 25 MiB/s, 402 objects/s recovering 2023-10-09T05:55:37.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:37 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:37.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:37 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:55:37.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:37 smithi064 ceph-mon[212750]: osdmap e212: 8 total, 8 up, 7 in 2023-10-09T05:55:37.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:37 smithi064 ceph-mon[212750]: failed to encode map e212 with expected crc 2023-10-09T05:55:37.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:37 smithi064 ceph-mon[212750]: failed to encode map e212 with expected crc 2023-10-09T05:55:37.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:37 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 41302/234618 objects degraded (17.604%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:37.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:37 smithi064 ceph-mon[220044]: pgmap v483: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 28 MiB/s wr, 945 op/s; 41302/234618 objects degraded (17.604%); 22735/234618 objects misplaced (9.690%); 25 MiB/s, 402 objects/s recovering 2023-10-09T05:55:37.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:37 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:37.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:37 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:55:37.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:37 smithi064 ceph-mon[220044]: osdmap e212: 8 total, 8 up, 7 in 2023-10-09T05:55:37.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:37 smithi064 ceph-mon[220044]: failed to encode map e212 with expected crc 2023-10-09T05:55:37.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:37 smithi064 ceph-mon[220044]: failed to encode map e212 with expected crc 2023-10-09T05:55:37.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:37 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 41302/234618 objects degraded (17.604%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:37.983 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:38.021 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:38.212 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:38.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:38.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:38.262 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:38.322 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:38.359 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:38.421 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:38.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:38.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:38.532 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:38.559 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:38.613 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:38.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:38.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:38.632 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:38.719 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:38.778 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:38.814 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:55:38.839 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:55:38.866 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:55:38.893 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:55:38.951 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:55:38.951 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:55:38.971 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:55:39.003 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:55:39.003 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:55:39.003 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:55:39.003 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:55:39.003 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:55:39.003 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:55:39.004 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:55:39.004 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:55:39.004 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:55:39.148 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:39.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:39.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:39.254 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:39.306 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:39.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:39 smithi064 ceph-mon[212750]: pgmap v485: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 469 op/s; 41215/233655 objects degraded (17.639%); 22571/233655 objects misplaced (9.660%); 7.3 MiB/s, 116 objects/s recovering 2023-10-09T05:55:39.621 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:39 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1781606234' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:39.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:39 smithi156 ceph-mon[195101]: pgmap v485: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 469 op/s; 41215/233655 objects degraded (17.639%); 22571/233655 objects misplaced (9.660%); 7.3 MiB/s, 116 objects/s recovering 2023-10-09T05:55:39.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:39 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1781606234' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:39.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:39 smithi064 ceph-mon[220044]: pgmap v485: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 469 op/s; 41215/233655 objects degraded (17.639%); 22571/233655 objects misplaced (9.660%); 7.3 MiB/s, 116 objects/s recovering 2023-10-09T05:55:39.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:39 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1781606234' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:39.978 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:39.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:39.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:40.083 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:40.119 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:40.163 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:55:40.755 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:40.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:40.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:40.859 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:40.901 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:40.962 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:40.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:40.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:41.069 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:41.099 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:41.146 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:41.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:41.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:41.252 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:41.295 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:41.700 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:41.703 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:41 smithi064 ceph-mon[212750]: pgmap v486: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 40702/227451 objects degraded (17.895%); 21889/227451 objects misplaced (9.624%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T05:55:41.703 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:41 smithi064 ceph-mon[220044]: pgmap v486: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 40702/227451 objects degraded (17.895%); 21889/227451 objects misplaced (9.624%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T05:55:41.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:41.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:41.805 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:41.839 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:41.893 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:41.897 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:41 smithi156 ceph-mon[195101]: pgmap v486: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 40702/227451 objects degraded (17.895%); 21889/227451 objects misplaced (9.624%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T05:55:41.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:41.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:41.999 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:42.045 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:42.113 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:42.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:42.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:42.127 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:55:42.127 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T05:55:42.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T05:55:42.127 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T05:55:42.127 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.128 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:79598 6.f 6:f7f62a28:::benchmark_data_smithi064_284193_object79597:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:55:42.128 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:52.373147+0000", 2023-10-09T05:55:42.128 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 229.74953973300001, 2023-10-09T05:55:42.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.261622515, 2023-10-09T05:55:42.128 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:42.128 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:42.128 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:42.128 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:42.128 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:42.128 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 79598 2023-10-09T05:55:42.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.129 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:42.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:42.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:52.373147+0000", 2023-10-09T05:55:42.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:42.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:52.373147+0000", 2023-10-09T05:55:42.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:42.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:52.373146+0000", 2023-10-09T05:55:42.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:55:42.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:42.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:52.373168+0000", 2023-10-09T05:55:42.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1824999999999999e-05 2023-10-09T05:55:42.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:42.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:52.373180+0000", 2023-10-09T05:55:42.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2105e-05 2023-10-09T05:55:42.132 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:42.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:52.373189+0000", 2023-10-09T05:55:42.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7900000000000005e-06 2023-10-09T05:55:42.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.133 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:52.373245+0000", 2023-10-09T05:55:42.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6440000000000002e-05 2023-10-09T05:55:42.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:42.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:52.373334+0000", 2023-10-09T05:55:42.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8945999999999998e-05 2023-10-09T05:55:42.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:55:42.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:52.373453+0000", 2023-10-09T05:55:42.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118633 2023-10-09T05:55:42.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:42.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:52.376427+0000", 2023-10-09T05:55:42.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029739430000000002 2023-10-09T05:55:42.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:52.380957+0000", 2023-10-09T05:55:42.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0045307589999999997 2023-10-09T05:55:42.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:52.634660+0000", 2023-10-09T05:55:42.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25370290699999998 2023-10-09T05:55:42.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:42.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:52.634721+0000", 2023-10-09T05:55:42.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0650999999999999e-05 2023-10-09T05:55:42.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:42.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:52.634770+0000", 2023-10-09T05:55:42.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8767000000000003e-05 2023-10-09T05:55:42.138 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.138 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:42.138 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.138 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82093 6.0 6:0c2a5c50:::benchmark_data_smithi064_284193_object82092:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T05:55:42.138 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:57.082946+0000", 2023-10-09T05:55:42.138 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 225.03974105399999, 2023-10-09T05:55:42.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.198079436, 2023-10-09T05:55:42.139 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:42.139 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:42.139 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:42.139 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:42.139 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:42.139 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82093 2023-10-09T05:55:42.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.139 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:42.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:42.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.082946+0000", 2023-10-09T05:55:42.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:42.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.082946+0000", 2023-10-09T05:55:42.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:42.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.082944+0000", 2023-10-09T05:55:42.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:42.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:42.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.082981+0000", 2023-10-09T05:55:42.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.684e-05 2023-10-09T05:55:42.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:42.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.082999+0000", 2023-10-09T05:55:42.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7934000000000001e-05 2023-10-09T05:55:42.142 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.142 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:42.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083010+0000", 2023-10-09T05:55:42.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.079e-05 2023-10-09T05:55:42.142 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.143 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.143 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.084587+0000", 2023-10-09T05:55:42.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001577216 2023-10-09T05:55:42.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.143 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:55:42.143 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.084598+0000", 2023-10-09T05:55:42.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0812000000000001e-05 2023-10-09T05:55:42.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.258113+0000", 2023-10-09T05:55:42.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1735149090000001 2023-10-09T05:55:42.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:42.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.259837+0000", 2023-10-09T05:55:42.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001724377 2023-10-09T05:55:42.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:55:42.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.259999+0000", 2023-10-09T05:55:42.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016160799999999999 2023-10-09T05:55:42.146 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.146 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.146 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.269960+0000", 2023-10-09T05:55:42.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00996177 2023-10-09T05:55:42.146 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.146 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:42.147 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.269967+0000", 2023-10-09T05:55:42.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.241e-06 2023-10-09T05:55:42.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.147 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.280428+0000", 2023-10-09T05:55:42.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010461869 2023-10-09T05:55:42.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:42.147 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.280484+0000", 2023-10-09T05:55:42.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5316999999999999e-05 2023-10-09T05:55:42.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:42.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.281025+0000", 2023-10-09T05:55:42.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00054158299999999999 2023-10-09T05:55:42.148 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.148 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:42.148 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.148 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82107 6.0 6:07b34c20:::benchmark_data_smithi064_284193_object82106:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T05:55:42.149 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:57.083024+0000", 2023-10-09T05:55:42.149 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 225.03966324999999, 2023-10-09T05:55:42.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.198456921, 2023-10-09T05:55:42.149 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:42.149 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:42.149 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:42.149 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:42.149 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:42.149 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82107 2023-10-09T05:55:42.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.149 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:42.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:42.150 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083024+0000", 2023-10-09T05:55:42.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:42.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083024+0000", 2023-10-09T05:55:42.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:42.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083023+0000", 2023-10-09T05:55:42.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:42.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:42.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083051+0000", 2023-10-09T05:55:42.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7501999999999999e-05 2023-10-09T05:55:42.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:42.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083063+0000", 2023-10-09T05:55:42.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2048999999999999e-05 2023-10-09T05:55:42.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.153 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.153 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:42.153 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083069+0000", 2023-10-09T05:55:42.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.319e-06 2023-10-09T05:55:42.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.153 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.084628+0000", 2023-10-09T05:55:42.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00155887 2023-10-09T05:55:42.154 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:55:42.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.084636+0000", 2023-10-09T05:55:42.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2830000000000001e-06 2023-10-09T05:55:42.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.263669+0000", 2023-10-09T05:55:42.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1790324080000001 2023-10-09T05:55:42.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:42.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.263844+0000", 2023-10-09T05:55:42.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000175141 2023-10-09T05:55:42.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:55:42.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.264042+0000", 2023-10-09T05:55:42.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019832300000000001 2023-10-09T05:55:42.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.157 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.157 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:42.157 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.269970+0000", 2023-10-09T05:55:42.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059283410000000002 2023-10-09T05:55:42.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.157 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.157 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.157 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.279927+0000", 2023-10-09T05:55:42.158 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0099563829999999992 2023-10-09T05:55:42.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.158 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.281062+0000", 2023-10-09T05:55:42.158 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001135472 2023-10-09T05:55:42.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.159 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.159 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:42.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.281111+0000", 2023-10-09T05:55:42.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9181999999999998e-05 2023-10-09T05:55:42.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.159 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.159 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:42.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.281481+0000", 2023-10-09T05:55:42.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000369219 2023-10-09T05:55:42.159 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.159 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:42.160 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.160 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82109 6.0 6:06570773:::benchmark_data_smithi064_284193_object82108:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T05:55:42.160 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:57.083077+0000", 2023-10-09T05:55:42.160 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 225.03960989500001, 2023-10-09T05:55:42.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1988545770000001, 2023-10-09T05:55:42.160 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:42.160 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:42.160 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:42.160 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:42.160 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:42.160 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82109 2023-10-09T05:55:42.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.161 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:42.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.161 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:42.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083077+0000", 2023-10-09T05:55:42.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:42.162 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083077+0000", 2023-10-09T05:55:42.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:42.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083076+0000", 2023-10-09T05:55:42.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:42.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:42.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083104+0000", 2023-10-09T05:55:42.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7718000000000001e-05 2023-10-09T05:55:42.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:42.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083124+0000", 2023-10-09T05:55:42.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9729999999999999e-05 2023-10-09T05:55:42.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:42.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083131+0000", 2023-10-09T05:55:42.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5749999999999997e-06 2023-10-09T05:55:42.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.084656+0000", 2023-10-09T05:55:42.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001525607 2023-10-09T05:55:42.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:55:42.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.084663+0000", 2023-10-09T05:55:42.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8809999999999998e-06 2023-10-09T05:55:42.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.266388+0000", 2023-10-09T05:55:42.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1817245459999999 2023-10-09T05:55:42.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:42.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.266542+0000", 2023-10-09T05:55:42.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000154074 2023-10-09T05:55:42.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:55:42.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.266721+0000", 2023-10-09T05:55:42.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000179588 2023-10-09T05:55:42.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.167 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:42.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.276095+0000", 2023-10-09T05:55:42.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0093742140000000005 2023-10-09T05:55:42.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.167 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.279969+0000", 2023-10-09T05:55:42.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038732670000000001 2023-10-09T05:55:42.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.281523+0000", 2023-10-09T05:55:42.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001554452 2023-10-09T05:55:42.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:42.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.281577+0000", 2023-10-09T05:55:42.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4104999999999998e-05 2023-10-09T05:55:42.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:42.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.281932+0000", 2023-10-09T05:55:42.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00035437499999999998 2023-10-09T05:55:42.169 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.169 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:42.169 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.169 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82116 6.9 6:95f8f18c:::benchmark_data_smithi064_284193_object82115:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T05:55:42.169 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:57.083135+0000", 2023-10-09T05:55:42.170 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 225.039551445, 2023-10-09T05:55:42.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1900297900000001, 2023-10-09T05:55:42.170 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:42.170 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:42.170 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:42.170 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:42.170 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:42.171 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82116 2023-10-09T05:55:42.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.171 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:42.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:42.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083135+0000", 2023-10-09T05:55:42.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:42.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083135+0000", 2023-10-09T05:55:42.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:42.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083135+0000", 2023-10-09T05:55:42.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:42.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:42.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083164+0000", 2023-10-09T05:55:42.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8952e-05 2023-10-09T05:55:42.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:42.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083175+0000", 2023-10-09T05:55:42.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0849999999999999e-05 2023-10-09T05:55:42.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:42.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083181+0000", 2023-10-09T05:55:42.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9290000000000003e-06 2023-10-09T05:55:42.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083487+0000", 2023-10-09T05:55:42.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00030613999999999999 2023-10-09T05:55:42.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:55:42.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083499+0000", 2023-10-09T05:55:42.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2415000000000001e-05 2023-10-09T05:55:42.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.256999+0000", 2023-10-09T05:55:42.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.173500196 2023-10-09T05:55:42.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:42.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.258968+0000", 2023-10-09T05:55:42.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001968923 2023-10-09T05:55:42.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:55:42.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.259175+0000", 2023-10-09T05:55:42.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020691000000000001 2023-10-09T05:55:42.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:42.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.272115+0000", 2023-10-09T05:55:42.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012939625 2023-10-09T05:55:42.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.272249+0000", 2023-10-09T05:55:42.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013408800000000001 2023-10-09T05:55:42.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.272479+0000", 2023-10-09T05:55:42.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000230349 2023-10-09T05:55:42.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:42.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.272538+0000", 2023-10-09T05:55:42.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8999999999999998e-05 2023-10-09T05:55:42.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:42.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.273165+0000", 2023-10-09T05:55:42.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00062689199999999999 2023-10-09T05:55:42.178 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.178 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:42.178 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.178 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82117 6.9 6:9031756d:::benchmark_data_smithi064_284193_object82116:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T05:55:42.178 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:57.083192+0000", 2023-10-09T05:55:42.178 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 225.03949460499999, 2023-10-09T05:55:42.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1976068609999999, 2023-10-09T05:55:42.179 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:42.179 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:42.179 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:42.179 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:42.179 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:42.179 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82117 2023-10-09T05:55:42.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.179 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:42.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:42.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083192+0000", 2023-10-09T05:55:42.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:42.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083192+0000", 2023-10-09T05:55:42.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:42.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083192+0000", 2023-10-09T05:55:42.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:42.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:42.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083213+0000", 2023-10-09T05:55:42.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0933000000000001e-05 2023-10-09T05:55:42.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:42.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083223+0000", 2023-10-09T05:55:42.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0433e-05 2023-10-09T05:55:42.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:42.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083229+0000", 2023-10-09T05:55:42.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6910000000000002e-06 2023-10-09T05:55:42.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083526+0000", 2023-10-09T05:55:42.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029717799999999999 2023-10-09T05:55:42.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:55:42.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083535+0000", 2023-10-09T05:55:42.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5150000000000006e-06 2023-10-09T05:55:42.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.262719+0000", 2023-10-09T05:55:42.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.179183981 2023-10-09T05:55:42.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:42.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.262876+0000", 2023-10-09T05:55:42.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000157629 2023-10-09T05:55:42.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:55:42.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.263044+0000", 2023-10-09T05:55:42.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000167622 2023-10-09T05:55:42.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:42.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.272122+0000", 2023-10-09T05:55:42.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0090786059999999995 2023-10-09T05:55:42.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.279609+0000", 2023-10-09T05:55:42.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0074866949999999998 2023-10-09T05:55:42.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.280065+0000", 2023-10-09T05:55:42.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00045626899999999998 2023-10-09T05:55:42.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:42.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.280133+0000", 2023-10-09T05:55:42.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7339e-05 2023-10-09T05:55:42.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:42.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.280799+0000", 2023-10-09T05:55:42.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00066644000000000002 2023-10-09T05:55:42.187 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.188 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:42.188 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.188 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82118 6.9 6:91416b55:::benchmark_data_smithi064_284193_object82117:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T05:55:42.188 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:57.083233+0000", 2023-10-09T05:55:42.188 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 225.03945345, 2023-10-09T05:55:42.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.198061628, 2023-10-09T05:55:42.188 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:42.189 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:42.189 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:42.189 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:42.189 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:42.189 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82118 2023-10-09T05:55:42.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.189 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:42.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:42.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083233+0000", 2023-10-09T05:55:42.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.190 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:42.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083233+0000", 2023-10-09T05:55:42.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.190 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:42.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083233+0000", 2023-10-09T05:55:42.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:42.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:42.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083254+0000", 2023-10-09T05:55:42.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1355000000000001e-05 2023-10-09T05:55:42.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:42.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083264+0000", 2023-10-09T05:55:42.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5349999999999993e-06 2023-10-09T05:55:42.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:42.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083269+0000", 2023-10-09T05:55:42.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4770000000000002e-06 2023-10-09T05:55:42.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083556+0000", 2023-10-09T05:55:42.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00028701399999999999 2023-10-09T05:55:42.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:55:42.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083565+0000", 2023-10-09T05:55:42.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2840000000000006e-06 2023-10-09T05:55:42.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.265497+0000", 2023-10-09T05:55:42.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.181932202 2023-10-09T05:55:42.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:42.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.265684+0000", 2023-10-09T05:55:42.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000187033 2023-10-09T05:55:42.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:55:42.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.265875+0000", 2023-10-09T05:55:42.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000191557 2023-10-09T05:55:42.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:42.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.272209+0000", 2023-10-09T05:55:42.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063339520000000003 2023-10-09T05:55:42.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.279746+0000", 2023-10-09T05:55:42.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0075368450000000003 2023-10-09T05:55:42.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.280854+0000", 2023-10-09T05:55:42.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001107292 2023-10-09T05:55:42.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:42.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.280908+0000", 2023-10-09T05:55:42.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4079999999999997e-05 2023-10-09T05:55:42.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:42.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.281295+0000", 2023-10-09T05:55:42.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00038745199999999999 2023-10-09T05:55:42.196 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.196 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:42.196 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.197 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82125 6.0 6:091c7b5b:::benchmark_data_smithi064_284193_object82124:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T05:55:42.197 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:57.083274+0000", 2023-10-09T05:55:42.197 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 225.03941272899999, 2023-10-09T05:55:42.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1991231920000001, 2023-10-09T05:55:42.197 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:42.197 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:42.197 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:42.197 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:42.197 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:42.197 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82125 2023-10-09T05:55:42.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.198 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:42.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:42.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083274+0000", 2023-10-09T05:55:42.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:42.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083274+0000", 2023-10-09T05:55:42.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:42.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083274+0000", 2023-10-09T05:55:42.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:42.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:42.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083294+0000", 2023-10-09T05:55:42.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0389000000000001e-05 2023-10-09T05:55:42.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:42.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083304+0000", 2023-10-09T05:55:42.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0093e-05 2023-10-09T05:55:42.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:42.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083311+0000", 2023-10-09T05:55:42.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3409999999999999e-06 2023-10-09T05:55:42.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.084682+0000", 2023-10-09T05:55:42.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013715000000000001 2023-10-09T05:55:42.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:55:42.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.084689+0000", 2023-10-09T05:55:42.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7959999999999998e-06 2023-10-09T05:55:42.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.269986+0000", 2023-10-09T05:55:42.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.185297439 2023-10-09T05:55:42.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:42.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.270081+0000", 2023-10-09T05:55:42.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4499000000000004e-05 2023-10-09T05:55:42.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:55:42.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.270180+0000", 2023-10-09T05:55:42.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9643999999999997e-05 2023-10-09T05:55:42.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:42.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.276106+0000", 2023-10-09T05:55:42.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059251010000000003 2023-10-09T05:55:42.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.280388+0000", 2023-10-09T05:55:42.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042823109999999996 2023-10-09T05:55:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.282002+0000", 2023-10-09T05:55:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016145580000000001 2023-10-09T05:55:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.282053+0000", 2023-10-09T05:55:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0643000000000003e-05 2023-10-09T05:55:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.282397+0000", 2023-10-09T05:55:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00034435800000000001 2023-10-09T05:55:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82137 6.0 6:091ee809:::benchmark_data_smithi064_284193_object82136:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T05:55:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:57.083316+0000", 2023-10-09T05:55:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 225.03937093900001, 2023-10-09T05:55:42.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2159187149999999, 2023-10-09T05:55:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82137 2023-10-09T05:55:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083316+0000", 2023-10-09T05:55:42.206 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:42.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083316+0000", 2023-10-09T05:55:42.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:42.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083315+0000", 2023-10-09T05:55:42.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:42.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.210 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:42.210 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083339+0000", 2023-10-09T05:55:42.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3604999999999998e-05 2023-10-09T05:55:42.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.210 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083351+0000", 2023-10-09T05:55:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1593999999999999e-05 2023-10-09T05:55:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083357+0000", 2023-10-09T05:55:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1249999999999998e-06 2023-10-09T05:55:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.084707+0000", 2023-10-09T05:55:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001350724 2023-10-09T05:55:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:55:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.084714+0000", 2023-10-09T05:55:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.652e-06 2023-10-09T05:55:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.272703+0000", 2023-10-09T05:55:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1879891309999999 2023-10-09T05:55:42.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.272854+0000", 2023-10-09T05:55:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015069899999999999 2023-10-09T05:55:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:55:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.273034+0000", 2023-10-09T05:55:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000179865 2023-10-09T05:55:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:42.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.276191+0000", 2023-10-09T05:55:42.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031575280000000002 2023-10-09T05:55:42.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.214 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.214 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.281969+0000", 2023-10-09T05:55:42.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0057776440000000002 2023-10-09T05:55:42.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.214 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.298462+0000", 2023-10-09T05:55:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016493206999999999 2023-10-09T05:55:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.298522+0000", 2023-10-09T05:55:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9956e-05 2023-10-09T05:55:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.299235+0000", 2023-10-09T05:55:42.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00071259700000000004 2023-10-09T05:55:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82139 6.0 6:09287403:::benchmark_data_smithi064_284193_object82138:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T05:55:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:57.083362+0000", 2023-10-09T05:55:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 225.03932509699999, 2023-10-09T05:55:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2163568469999999, 2023-10-09T05:55:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:42.216 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82139 2023-10-09T05:55:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083362+0000", 2023-10-09T05:55:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:42.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083362+0000", 2023-10-09T05:55:42.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.218 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:42.218 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083361+0000", 2023-10-09T05:55:42.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:42.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.218 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:42.218 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083383+0000", 2023-10-09T05:55:42.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1542000000000001e-05 2023-10-09T05:55:42.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:42.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083394+0000", 2023-10-09T05:55:42.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0767e-05 2023-10-09T05:55:42.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:42.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083399+0000", 2023-10-09T05:55:42.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7570000000000001e-06 2023-10-09T05:55:42.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.220 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.084733+0000", 2023-10-09T05:55:42.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001333309 2023-10-09T05:55:42.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.220 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:55:42.220 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.084740+0000", 2023-10-09T05:55:42.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8029999999999999e-06 2023-10-09T05:55:42.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.220 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.220 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.276687+0000", 2023-10-09T05:55:42.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1919472209999999 2023-10-09T05:55:42.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:42.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.276860+0000", 2023-10-09T05:55:42.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017371800000000001 2023-10-09T05:55:42.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:55:42.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.277059+0000", 2023-10-09T05:55:42.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000198104 2023-10-09T05:55:42.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.282435+0000", 2023-10-09T05:55:42.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053760370000000002 2023-10-09T05:55:42.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:42.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.282677+0000", 2023-10-09T05:55:42.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000242854 2023-10-09T05:55:42.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.299289+0000", 2023-10-09T05:55:42.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016611679000000001 2023-10-09T05:55:42.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:42.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.299347+0000", 2023-10-09T05:55:42.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7896999999999998e-05 2023-10-09T05:55:42.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:42.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.299719+0000", 2023-10-09T05:55:42.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00037165400000000002 2023-10-09T05:55:42.224 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.224 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:42.224 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.225 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82142 6.9 6:92f3da5c:::benchmark_data_smithi064_284193_object82141:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T05:55:42.225 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:57.083404+0000", 2023-10-09T05:55:42.225 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 225.039282996, 2023-10-09T05:55:42.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2055263089999999, 2023-10-09T05:55:42.225 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:42.225 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:42.225 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:42.225 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:42.225 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:42.225 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82142 2023-10-09T05:55:42.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.226 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:42.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:42.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083404+0000", 2023-10-09T05:55:42.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:42.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083404+0000", 2023-10-09T05:55:42.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.227 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:42.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083403+0000", 2023-10-09T05:55:42.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:42.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.227 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:42.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083424+0000", 2023-10-09T05:55:42.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0803999999999999e-05 2023-10-09T05:55:42.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:42.228 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083431+0000", 2023-10-09T05:55:42.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5780000000000004e-06 2023-10-09T05:55:42.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:42.228 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083436+0000", 2023-10-09T05:55:42.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.181e-06 2023-10-09T05:55:42.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.229 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083586+0000", 2023-10-09T05:55:42.229 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015032199999999999 2023-10-09T05:55:42.229 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.229 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.229 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:55:42.229 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083595+0000", 2023-10-09T05:55:42.229 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3999999999999992e-06 2023-10-09T05:55:42.229 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.229 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.229 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.229 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.268821+0000", 2023-10-09T05:55:42.229 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.185226458 2023-10-09T05:55:42.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:42.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.268987+0000", 2023-10-09T05:55:42.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016546599999999999 2023-10-09T05:55:42.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:55:42.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.269196+0000", 2023-10-09T05:55:42.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020894199999999999 2023-10-09T05:55:42.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:42.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.276199+0000", 2023-10-09T05:55:42.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0070038699999999997 2023-10-09T05:55:42.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.279785+0000", 2023-10-09T05:55:42.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035854609999999999 2023-10-09T05:55:42.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.232 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.232 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.288179+0000", 2023-10-09T05:55:42.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0083937950000000008 2023-10-09T05:55:42.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.232 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.232 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:42.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.288258+0000", 2023-10-09T05:55:42.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9121000000000004e-05 2023-10-09T05:55:42.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.233 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:42.233 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.288930+0000", 2023-10-09T05:55:42.233 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00067237600000000003 2023-10-09T05:55:42.233 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.233 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:42.233 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.233 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.233 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:92846 6.0 6:0a55c0e9:::benchmark_data_smithi064_284193_object92845:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-10-09T05:55:42.233 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:52:13.790510+0000", 2023-10-09T05:55:42.233 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 208.332177121, 2023-10-09T05:55:42.234 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24779770400000001, 2023-10-09T05:55:42.234 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:42.234 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:42.234 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:42.234 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:42.234 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:42.234 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 92846 2023-10-09T05:55:42.234 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.234 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:42.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.234 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:42.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.790510+0000", 2023-10-09T05:55:42.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.235 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:42.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.790510+0000", 2023-10-09T05:55:42.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:42.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.790509+0000", 2023-10-09T05:55:42.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:42.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:42.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.790534+0000", 2023-10-09T05:55:42.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4510000000000001e-05 2023-10-09T05:55:42.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:42.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.790545+0000", 2023-10-09T05:55:42.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1652e-05 2023-10-09T05:55:42.237 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.237 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.237 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:42.237 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.790553+0000", 2023-10-09T05:55:42.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8450000000000003e-06 2023-10-09T05:55:42.237 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.237 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.237 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.237 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.797131+0000", 2023-10-09T05:55:42.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065773660000000003 2023-10-09T05:55:42.237 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.238 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.238 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:42.238 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.797282+0000", 2023-10-09T05:55:42.238 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000151037 2023-10-09T05:55:42.238 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.238 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.238 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:55:42.238 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.797440+0000", 2023-10-09T05:55:42.238 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015840599999999999 2023-10-09T05:55:42.238 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.239 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.239 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:42.239 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.804443+0000", 2023-10-09T05:55:42.239 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0070029369999999999 2023-10-09T05:55:42.239 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.239 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.239 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.239 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.805145+0000", 2023-10-09T05:55:42.239 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00070198999999999999 2023-10-09T05:55:42.239 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.239 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.239 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.240 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.037739+0000", 2023-10-09T05:55:42.240 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.232594096 2023-10-09T05:55:42.240 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.240 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.240 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:42.240 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.037800+0000", 2023-10-09T05:55:42.240 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0766000000000001e-05 2023-10-09T05:55:42.240 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.240 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.240 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:42.240 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.038308+0000", 2023-10-09T05:55:42.240 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000507732 2023-10-09T05:55:42.241 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.241 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:42.241 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.241 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.241 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.241 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:92858 6.f 6:f2068a8c:::benchmark_data_smithi064_284193_object92857:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-10-09T05:55:42.241 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:52:13.812695+0000", 2023-10-09T05:55:42.241 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 208.30999213800001, 2023-10-09T05:55:42.241 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.22242227000000001, 2023-10-09T05:55:42.241 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:42.241 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:42.241 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:42.242 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:42.242 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:42.242 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 92858 2023-10-09T05:55:42.242 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.242 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:42.242 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.242 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:42.242 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.812695+0000", 2023-10-09T05:55:42.242 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.243 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.243 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.243 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:42.243 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.812695+0000", 2023-10-09T05:55:42.243 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.243 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.243 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.243 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:42.243 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.812693+0000", 2023-10-09T05:55:42.243 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:42.243 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.243 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.243 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:42.244 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.812728+0000", 2023-10-09T05:55:42.244 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5358999999999998e-05 2023-10-09T05:55:42.244 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.244 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.244 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:42.244 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.812744+0000", 2023-10-09T05:55:42.244 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6096e-05 2023-10-09T05:55:42.244 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.244 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.245 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:42.245 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.812754+0000", 2023-10-09T05:55:42.245 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2820000000000002e-06 2023-10-09T05:55:42.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.245 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.245 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.245 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.812805+0000", 2023-10-09T05:55:42.245 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0903999999999999e-05 2023-10-09T05:55:42.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.245 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:42.246 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.812974+0000", 2023-10-09T05:55:42.246 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016909100000000001 2023-10-09T05:55:42.246 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.246 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:55:42.246 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.813142+0000", 2023-10-09T05:55:42.246 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016829799999999999 2023-10-09T05:55:42.246 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.246 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.247 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.818671+0000", 2023-10-09T05:55:42.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.005528682 2023-10-09T05:55:42.247 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.247 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.247 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:42.247 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.818685+0000", 2023-10-09T05:55:42.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4324e-05 2023-10-09T05:55:42.247 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.247 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.247 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.247 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.035024+0000", 2023-10-09T05:55:42.248 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21633867400000001 2023-10-09T05:55:42.248 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.248 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.248 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:42.248 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.035075+0000", 2023-10-09T05:55:42.248 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1199000000000002e-05 2023-10-09T05:55:42.248 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.248 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.248 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:42.248 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.035117+0000", 2023-10-09T05:55:42.248 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2231000000000003e-05 2023-10-09T05:55:42.249 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.249 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:42.249 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.249 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.249 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.249 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:92864 6.0 6:090f574b:::benchmark_data_smithi064_284193_object92863:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-10-09T05:55:42.249 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:52:13.818198+0000", 2023-10-09T05:55:42.249 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 208.30448855099999, 2023-10-09T05:55:42.249 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.22045798599999999, 2023-10-09T05:55:42.249 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:42.249 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:42.249 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:42.250 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:42.250 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:42.250 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 92864 2023-10-09T05:55:42.250 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.250 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:42.250 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.250 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:42.250 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.818198+0000", 2023-10-09T05:55:42.250 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.250 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.250 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.250 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:42.251 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.818198+0000", 2023-10-09T05:55:42.251 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.251 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.251 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.251 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:42.251 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.818197+0000", 2023-10-09T05:55:42.251 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:42.251 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.251 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.251 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:42.251 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.818223+0000", 2023-10-09T05:55:42.251 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6021e-05 2023-10-09T05:55:42.252 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.252 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.252 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:42.252 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.818233+0000", 2023-10-09T05:55:42.252 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0747000000000001e-05 2023-10-09T05:55:42.252 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.252 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.252 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:42.252 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.818240+0000", 2023-10-09T05:55:42.253 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.888e-06 2023-10-09T05:55:42.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.253 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.253 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.818298+0000", 2023-10-09T05:55:42.253 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7982999999999997e-05 2023-10-09T05:55:42.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.253 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:42.253 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.818489+0000", 2023-10-09T05:55:42.253 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000190323 2023-10-09T05:55:42.254 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.254 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:55:42.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.818704+0000", 2023-10-09T05:55:42.254 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00021522699999999999 2023-10-09T05:55:42.254 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.254 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:42.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.822173+0000", 2023-10-09T05:55:42.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034689619999999999 2023-10-09T05:55:42.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.255 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.255 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.038335+0000", 2023-10-09T05:55:42.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.216162774 2023-10-09T05:55:42.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.255 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.255 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.038355+0000", 2023-10-09T05:55:42.256 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9476000000000001e-05 2023-10-09T05:55:42.256 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.256 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.256 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:42.256 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.038389+0000", 2023-10-09T05:55:42.256 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3569000000000003e-05 2023-10-09T05:55:42.256 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.256 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.256 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:42.256 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.038656+0000", 2023-10-09T05:55:42.256 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00026784399999999999 2023-10-09T05:55:42.256 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.257 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:42.257 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.257 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.257 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.257 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:92868 6.f 6:f3dc4334:::benchmark_data_smithi064_284193_object92867:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-10-09T05:55:42.257 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:52:13.822373+0000", 2023-10-09T05:55:42.257 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 208.30031439499999, 2023-10-09T05:55:42.257 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.212853073, 2023-10-09T05:55:42.257 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:42.257 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:42.257 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:42.258 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:42.258 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:42.258 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 92868 2023-10-09T05:55:42.258 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.258 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:42.258 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.258 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:42.258 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.822373+0000", 2023-10-09T05:55:42.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.258 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.258 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.258 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:42.259 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.822373+0000", 2023-10-09T05:55:42.259 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.259 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.259 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.259 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:42.259 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.822371+0000", 2023-10-09T05:55:42.259 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:55:42.259 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.259 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.259 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:42.259 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.822396+0000", 2023-10-09T05:55:42.259 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5304e-05 2023-10-09T05:55:42.259 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.260 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.260 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:42.260 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.822410+0000", 2023-10-09T05:55:42.260 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3709999999999999e-05 2023-10-09T05:55:42.260 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.260 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.260 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:42.260 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.822418+0000", 2023-10-09T05:55:42.260 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.388e-06 2023-10-09T05:55:42.260 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.260 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.260 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.260 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.822467+0000", 2023-10-09T05:55:42.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8300000000000002e-05 2023-10-09T05:55:42.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.261 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.261 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:42.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.822589+0000", 2023-10-09T05:55:42.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000122688 2023-10-09T05:55:42.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.261 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.261 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:55:42.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.822743+0000", 2023-10-09T05:55:42.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000153697 2023-10-09T05:55:42.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:42.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.825609+0000", 2023-10-09T05:55:42.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028660500000000002 2023-10-09T05:55:42.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.034881+0000", 2023-10-09T05:55:42.263 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.209272074 2023-10-09T05:55:42.263 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.263 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.263 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.263 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.035156+0000", 2023-10-09T05:55:42.263 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00027449100000000003 2023-10-09T05:55:42.263 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.263 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.263 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:42.263 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.035195+0000", 2023-10-09T05:55:42.264 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9848000000000002e-05 2023-10-09T05:55:42.264 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.264 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.264 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:42.264 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.035226+0000", 2023-10-09T05:55:42.264 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.021e-05 2023-10-09T05:55:42.264 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.264 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:42.264 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.264 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.264 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.264 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:92871 6.0 6:00e90e93:::benchmark_data_smithi064_284193_object92870:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-10-09T05:55:42.265 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:52:13.825789+0000", 2023-10-09T05:55:42.265 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 208.29689776399999, 2023-10-09T05:55:42.265 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.216791293, 2023-10-09T05:55:42.265 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:42.265 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:42.265 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:42.265 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:42.265 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:42.265 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 92871 2023-10-09T05:55:42.266 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.266 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:42.266 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.266 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:42.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.825789+0000", 2023-10-09T05:55:42.266 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.266 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.266 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.266 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:42.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.825789+0000", 2023-10-09T05:55:42.266 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:42.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.825788+0000", 2023-10-09T05:55:42.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:55:42.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:42.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.825814+0000", 2023-10-09T05:55:42.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5989000000000001e-05 2023-10-09T05:55:42.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.268 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:42.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.825827+0000", 2023-10-09T05:55:42.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3259e-05 2023-10-09T05:55:42.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.268 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.268 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:42.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.825837+0000", 2023-10-09T05:55:42.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0231000000000001e-05 2023-10-09T05:55:42.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.268 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.268 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.825895+0000", 2023-10-09T05:55:42.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8322000000000002e-05 2023-10-09T05:55:42.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:42.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.826012+0000", 2023-10-09T05:55:42.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116735 2023-10-09T05:55:42.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:55:42.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.826126+0000", 2023-10-09T05:55:42.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000113699 2023-10-09T05:55:42.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.038705+0000", 2023-10-09T05:55:42.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21257870700000001 2023-10-09T05:55:42.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.038728+0000", 2023-10-09T05:55:42.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3606000000000001e-05 2023-10-09T05:55:42.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:42.271 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.041918+0000", 2023-10-09T05:55:42.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003189777 2023-10-09T05:55:42.271 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.271 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:42.271 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.041974+0000", 2023-10-09T05:55:42.272 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6189e-05 2023-10-09T05:55:42.272 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.272 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.272 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:42.272 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.042580+0000", 2023-10-09T05:55:42.272 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00060633400000000004 2023-10-09T05:55:42.272 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.272 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:42.273 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.273 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.273 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.273 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.44588.0:92862 6.3 e136/102 6:cc340b44:::benchmark_data_smithi064_284193_object92861:head v 136'5853, mlcod=136'5851)", 2023-10-09T05:55:42.273 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:52:13.827248+0000", 2023-10-09T05:55:42.273 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 208.29543900799999, 2023-10-09T05:55:42.273 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.213220359, 2023-10-09T05:55:42.273 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:42.273 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:42.273 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:42.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.274 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:42.274 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.827248+0000", 2023-10-09T05:55:42.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.274 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:42.274 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.827248+0000", 2023-10-09T05:55:42.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.274 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:42.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.827247+0000", 2023-10-09T05:55:42.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T05:55:42.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:42.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.827300+0000", 2023-10-09T05:55:42.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2859999999999999e-05 2023-10-09T05:55:42.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:42.276 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.034665+0000", 2023-10-09T05:55:42.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.207365668 2023-10-09T05:55:42.276 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.277 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:42.277 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.034677+0000", 2023-10-09T05:55:42.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1578e-05 2023-10-09T05:55:42.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.277 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.278 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.034723+0000", 2023-10-09T05:55:42.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5677000000000003e-05 2023-10-09T05:55:42.278 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.278 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:42.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.034751+0000", 2023-10-09T05:55:42.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8439e-05 2023-10-09T05:55:42.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:42.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.040426+0000", 2023-10-09T05:55:42.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056753949999999997 2023-10-09T05:55:42.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:42.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.040468+0000", 2023-10-09T05:55:42.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1789999999999998e-05 2023-10-09T05:55:42.280 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.280 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:42.280 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.280 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:92875 6.0 6:073275cc:::benchmark_data_smithi064_284193_object92874:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-10-09T05:55:42.280 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:52:13.830483+0000", 2023-10-09T05:55:42.280 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 208.292203774, 2023-10-09T05:55:42.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.217911365, 2023-10-09T05:55:42.280 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:42.280 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:42.281 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:42.281 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:42.281 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:42.281 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 92875 2023-10-09T05:55:42.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.281 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:42.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.281 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:42.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.830483+0000", 2023-10-09T05:55:42.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:42.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.830483+0000", 2023-10-09T05:55:42.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:42.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.830481+0000", 2023-10-09T05:55:42.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:55:42.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:42.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.830516+0000", 2023-10-09T05:55:42.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5855000000000001e-05 2023-10-09T05:55:42.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:42.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.034662+0000", 2023-10-09T05:55:42.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20414560200000001 2023-10-09T05:55:42.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:42.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.034673+0000", 2023-10-09T05:55:42.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0862999999999999e-05 2023-10-09T05:55:42.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.038742+0000", 2023-10-09T05:55:42.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040691449999999997 2023-10-09T05:55:42.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:42.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.038853+0000", 2023-10-09T05:55:42.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110759 2023-10-09T05:55:42.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:55:42.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.039012+0000", 2023-10-09T05:55:42.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015909799999999999 2023-10-09T05:55:42.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:42.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.045902+0000", 2023-10-09T05:55:42.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0068905169999999996 2023-10-09T05:55:42.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.047691+0000", 2023-10-09T05:55:42.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001788801 2023-10-09T05:55:42.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.047726+0000", 2023-10-09T05:55:42.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5125e-05 2023-10-09T05:55:42.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.287 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:42.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.047789+0000", 2023-10-09T05:55:42.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2605000000000001e-05 2023-10-09T05:55:42.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.287 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:42.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.048395+0000", 2023-10-09T05:55:42.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00060560200000000001 2023-10-09T05:55:42.287 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.288 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:42.288 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.288 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.288 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.288 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.44588.0:92846 6.0 e136/130 ondisk, result = 0)", 2023-10-09T05:55:42.288 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:52:13.832659+0000", 2023-10-09T05:55:42.288 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 208.29002830100001, 2023-10-09T05:55:42.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20565418599999999, 2023-10-09T05:55:42.288 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:42.288 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T05:55:42.289 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:42.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:42.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.832659+0000", 2023-10-09T05:55:42.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:42.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.832659+0000", 2023-10-09T05:55:42.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:42.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.832657+0000", 2023-10-09T05:55:42.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:55:42.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:42.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:13.832665+0000", 2023-10-09T05:55:42.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3680000000000002e-06 2023-10-09T05:55:42.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:42.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.034657+0000", 2023-10-09T05:55:42.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20199172300000001 2023-10-09T05:55:42.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.291 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:42.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.034672+0000", 2023-10-09T05:55:42.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4443999999999999e-05 2023-10-09T05:55:42.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.291 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.037723+0000", 2023-10-09T05:55:42.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003051471 2023-10-09T05:55:42.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:42.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.037732+0000", 2023-10-09T05:55:42.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6710000000000005e-06 2023-10-09T05:55:42.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:42.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.038313+0000", 2023-10-09T05:55:42.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00058114199999999999 2023-10-09T05:55:42.292 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.292 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:42.292 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.293 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:93663 6.f 6:f5a792ed:::benchmark_data_smithi064_284193_object93662:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2023-10-09T05:55:42.293 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:52:14.972107+0000", 2023-10-09T05:55:42.293 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 207.150580268, 2023-10-09T05:55:42.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.19443017800000001, 2023-10-09T05:55:42.293 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:55:42.293 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:55:42.293 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:55:42.293 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:55:42.293 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:55:42.293 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 93663 2023-10-09T05:55:42.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.294 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:55:42.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:55:42.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.972107+0000", 2023-10-09T05:55:42.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:55:42.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.972107+0000", 2023-10-09T05:55:42.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:55:42.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:55:42.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.972106+0000", 2023-10-09T05:55:42.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:55:42.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:55:42.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.972130+0000", 2023-10-09T05:55:42.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3538000000000001e-05 2023-10-09T05:55:42.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:55:42.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.972138+0000", 2023-10-09T05:55:42.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2260000000000005e-06 2023-10-09T05:55:42.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:55:42.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.972144+0000", 2023-10-09T05:55:42.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.699e-06 2023-10-09T05:55:42.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:55:42.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.975549+0000", 2023-10-09T05:55:42.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003405745 2023-10-09T05:55:42.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:55:42.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.975730+0000", 2023-10-09T05:55:42.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000181139 2023-10-09T05:55:42.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:55:42.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.975894+0000", 2023-10-09T05:55:42.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016320199999999999 2023-10-09T05:55:42.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:55:42.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.979005+0000", 2023-10-09T05:55:42.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031112829999999998 2023-10-09T05:55:42.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:14.986412+0000", 2023-10-09T05:55:42.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0074073109999999998 2023-10-09T05:55:42.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:55:42.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:15.166444+0000", 2023-10-09T05:55:42.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18003228499999999 2023-10-09T05:55:42.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:55:42.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:15.166489+0000", 2023-10-09T05:55:42.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4166999999999999e-05 2023-10-09T05:55:42.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:55:42.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:55:42.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:55:42.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:15.166537+0000", 2023-10-09T05:55:42.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8195999999999997e-05 2023-10-09T05:55:42.300 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.300 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:42.300 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.300 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:55:42.300 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:55:42.300 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:55:42.301 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:42.302 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:42.327 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:42.370 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:42.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:42.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:42.475 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:42.513 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:42.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:42 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 40702/227451 objects degraded (17.895%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:42.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:42 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 40702/227451 objects degraded (17.895%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:42.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:42 smithi064 conmon[212726]: 2023-10-09T05:55:42.730+0000 7f8823f23700 -1 mon.a@0(leader).osd e212 definitely_dead 0 2023-10-09T05:55:43.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:42 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 40702/227451 objects degraded (17.895%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:43.155 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:55:43.633 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:43.636 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T05:55:43.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:43.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:43.686 INFO:teuthology.orchestra.run.smithi064.stderr:marked in osd.1. 2023-10-09T05:55:43.743 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:43.788 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:43.853 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:43.862 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:43 smithi064 ceph-mon[212750]: pgmap v487: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 40632/226572 objects degraded (17.933%); 21753/226572 objects misplaced (9.601%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T05:55:43.862 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:43 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3316868988' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T05:55:43.862 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:43 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:55:43.862 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:43 smithi064 ceph-mon[220044]: pgmap v487: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 40632/226572 objects degraded (17.933%); 21753/226572 objects misplaced (9.601%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T05:55:43.863 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:43 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3316868988' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T05:55:43.863 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:43 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:55:43.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:43.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:43.964 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:44.001 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:44.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:43 smithi156 ceph-mon[195101]: pgmap v487: 129 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 40632/226572 objects degraded (17.933%); 21753/226572 objects misplaced (9.601%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T05:55:44.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:43 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3316868988' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T05:55:44.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:43 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:55:44.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:44 smithi064 ceph-mon[220044]: failed to encode map e213 with expected crc 2023-10-09T05:55:44.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:44 smithi064 ceph-mon[220044]: failed to encode map e213 with expected crc 2023-10-09T05:55:44.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:44 smithi064 ceph-mon[220044]: failed to encode map e213 with expected crc 2023-10-09T05:55:44.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:44 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3316868988' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T05:55:44.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:44 smithi064 ceph-mon[220044]: osdmap e213: 8 total, 8 up, 8 in 2023-10-09T05:55:44.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:44 smithi064 ceph-mon[220044]: failed to encode map e213 with expected crc 2023-10-09T05:55:44.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:44 smithi064 ceph-mon[220044]: failed to encode map e213 with expected crc 2023-10-09T05:55:44.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:44.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:44.872 INFO:tasks.thrashosds.thrasher:Added osd 1 2023-10-09T05:55:44.903 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:44.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:44.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:44.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:44 smithi064 ceph-mon[212750]: failed to encode map e213 with expected crc 2023-10-09T05:55:44.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:44 smithi064 ceph-mon[212750]: failed to encode map e213 with expected crc 2023-10-09T05:55:44.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:44 smithi064 ceph-mon[212750]: failed to encode map e213 with expected crc 2023-10-09T05:55:44.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:44 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3316868988' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T05:55:44.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:44 smithi064 ceph-mon[212750]: osdmap e213: 8 total, 8 up, 8 in 2023-10-09T05:55:44.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:44 smithi064 ceph-mon[212750]: failed to encode map e213 with expected crc 2023-10-09T05:55:44.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:44 smithi064 ceph-mon[212750]: failed to encode map e213 with expected crc 2023-10-09T05:55:44.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:44.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:45.011 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:45.041 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:45.095 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:45.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:45.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:45.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:44 smithi156 ceph-mon[195101]: failed to encode map e213 with expected crc 2023-10-09T05:55:45.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:44 smithi156 ceph-mon[195101]: failed to encode map e213 with expected crc 2023-10-09T05:55:45.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:44 smithi156 ceph-mon[195101]: failed to encode map e213 with expected crc 2023-10-09T05:55:45.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:44 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3316868988' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T05:55:45.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:44 smithi156 ceph-mon[195101]: osdmap e213: 8 total, 8 up, 8 in 2023-10-09T05:55:45.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:44 smithi156 ceph-mon[195101]: failed to encode map e213 with expected crc 2023-10-09T05:55:45.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:44 smithi156 ceph-mon[195101]: failed to encode map e213 with expected crc 2023-10-09T05:55:45.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:45.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:45.198 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:45.225 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:45.267 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:45.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:45.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:45.373 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:45.404 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:45.555 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:45.559 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:55:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:55:45] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.43.0" 2023-10-09T05:55:45.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:45.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:45.581 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:45.620 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:45.661 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:55:45.694 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:45.743 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:55:45.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:45.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:55:45.849 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:45.922 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[212750]: pgmap v489: 129 pgs: 9 remapped+peering, 36 peering, 3 unknown, 2 active+recovery_wait+degraded, 4 active+recovery_wait+degraded+remapped, 71 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 19482/223047 objects degraded (8.734%); 4694/223047 objects misplaced (2.104%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T05:55:45.922 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[212750]: osdmap e214: 8 total, 8 up, 8 in 2023-10-09T05:55:45.922 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[212750]: failed to encode map e214 with expected crc 2023-10-09T05:55:45.922 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[212750]: failed to encode map e214 with expected crc 2023-10-09T05:55:45.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[212750]: failed to encode map e214 with expected crc 2023-10-09T05:55:45.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[212750]: failed to encode map e214 with expected crc 2023-10-09T05:55:45.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[212750]: failed to encode map e214 with expected crc 2023-10-09T05:55:45.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[212750]: failed to encode map e214 with expected crc 2023-10-09T05:55:45.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[212750]: failed to encode map e214 with expected crc 2023-10-09T05:55:45.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[212750]: failed to encode map e214 with expected crc 2023-10-09T05:55:45.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[212750]: failed to encode map e214 with expected crc 2023-10-09T05:55:45.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[212750]: failed to encode map e214 with expected crc 2023-10-09T05:55:45.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[212750]: 6.1f starting backfill to osd.1 from (107'1052,189'6609] MIN to 212'7217 2023-10-09T05:55:45.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[212750]: 6.f starting backfill to osd.1 from (107'1052,189'6609] MIN to 212'7258 2023-10-09T05:55:45.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 9 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2023-10-09T05:55:45.924 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:45.924 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:45.924 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:45.924 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[212750]: osdmap e215: 8 total, 8 up, 8 in 2023-10-09T05:55:45.924 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[220044]: pgmap v489: 129 pgs: 9 remapped+peering, 36 peering, 3 unknown, 2 active+recovery_wait+degraded, 4 active+recovery_wait+degraded+remapped, 71 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 19482/223047 objects degraded (8.734%); 4694/223047 objects misplaced (2.104%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T05:55:45.924 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[220044]: osdmap e214: 8 total, 8 up, 8 in 2023-10-09T05:55:45.924 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[220044]: failed to encode map e214 with expected crc 2023-10-09T05:55:45.925 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[220044]: failed to encode map e214 with expected crc 2023-10-09T05:55:45.925 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[220044]: failed to encode map e214 with expected crc 2023-10-09T05:55:45.925 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[220044]: failed to encode map e214 with expected crc 2023-10-09T05:55:45.925 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[220044]: failed to encode map e214 with expected crc 2023-10-09T05:55:45.925 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[220044]: failed to encode map e214 with expected crc 2023-10-09T05:55:45.925 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[220044]: failed to encode map e214 with expected crc 2023-10-09T05:55:45.925 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[220044]: failed to encode map e214 with expected crc 2023-10-09T05:55:45.925 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[220044]: failed to encode map e214 with expected crc 2023-10-09T05:55:45.925 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[220044]: failed to encode map e214 with expected crc 2023-10-09T05:55:45.925 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[220044]: 6.1f starting backfill to osd.1 from (107'1052,189'6609] MIN to 212'7217 2023-10-09T05:55:45.926 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[220044]: 6.f starting backfill to osd.1 from (107'1052,189'6609] MIN to 212'7258 2023-10-09T05:55:45.926 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 9 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2023-10-09T05:55:45.926 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:45.926 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:45.926 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:45.926 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:45 smithi064 ceph-mon[220044]: osdmap e215: 8 total, 8 up, 8 in 2023-10-09T05:55:45.970 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:46.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:45 smithi156 ceph-mon[195101]: pgmap v489: 129 pgs: 9 remapped+peering, 36 peering, 3 unknown, 2 active+recovery_wait+degraded, 4 active+recovery_wait+degraded+remapped, 71 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 19482/223047 objects degraded (8.734%); 4694/223047 objects misplaced (2.104%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T05:55:46.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:45 smithi156 ceph-mon[195101]: osdmap e214: 8 total, 8 up, 8 in 2023-10-09T05:55:46.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:45 smithi156 ceph-mon[195101]: failed to encode map e214 with expected crc 2023-10-09T05:55:46.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:45 smithi156 ceph-mon[195101]: failed to encode map e214 with expected crc 2023-10-09T05:55:46.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:45 smithi156 ceph-mon[195101]: failed to encode map e214 with expected crc 2023-10-09T05:55:46.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:45 smithi156 ceph-mon[195101]: failed to encode map e214 with expected crc 2023-10-09T05:55:46.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:45 smithi156 ceph-mon[195101]: failed to encode map e214 with expected crc 2023-10-09T05:55:46.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:45 smithi156 ceph-mon[195101]: failed to encode map e214 with expected crc 2023-10-09T05:55:46.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:45 smithi156 ceph-mon[195101]: failed to encode map e214 with expected crc 2023-10-09T05:55:46.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:45 smithi156 ceph-mon[195101]: failed to encode map e214 with expected crc 2023-10-09T05:55:46.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:45 smithi156 ceph-mon[195101]: failed to encode map e214 with expected crc 2023-10-09T05:55:46.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:45 smithi156 ceph-mon[195101]: failed to encode map e214 with expected crc 2023-10-09T05:55:46.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:45 smithi156 ceph-mon[195101]: 6.1f starting backfill to osd.1 from (107'1052,189'6609] MIN to 212'7217 2023-10-09T05:55:46.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:45 smithi156 ceph-mon[195101]: 6.f starting backfill to osd.1 from (107'1052,189'6609] MIN to 212'7258 2023-10-09T05:55:46.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:45 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 9 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2023-10-09T05:55:46.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:45 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:46.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:45 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:46.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:45 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:46.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:45 smithi156 ceph-mon[195101]: osdmap e215: 8 total, 8 up, 8 in 2023-10-09T05:55:46.187 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:46.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:46.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:46.295 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:46.321 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:46.363 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:46.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:46.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:46.467 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:46.597 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:46.724 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:46 smithi064 ceph-mon[212750]: failed to encode map e215 with expected crc 2023-10-09T05:55:46.724 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:46 smithi064 ceph-mon[212750]: failed to encode map e215 with expected crc 2023-10-09T05:55:46.724 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:46 smithi064 ceph-mon[212750]: failed to encode map e215 with expected crc 2023-10-09T05:55:46.724 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:46 smithi064 ceph-mon[212750]: failed to encode map e215 with expected crc 2023-10-09T05:55:46.725 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:46 smithi064 ceph-mon[212750]: failed to encode map e215 with expected crc 2023-10-09T05:55:46.725 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:46 smithi064 ceph-mon[212750]: failed to encode map e215 with expected crc 2023-10-09T05:55:46.725 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:46 smithi064 ceph-mon[220044]: failed to encode map e215 with expected crc 2023-10-09T05:55:46.725 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:46 smithi064 ceph-mon[220044]: failed to encode map e215 with expected crc 2023-10-09T05:55:46.725 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:46 smithi064 ceph-mon[220044]: failed to encode map e215 with expected crc 2023-10-09T05:55:46.725 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:46 smithi064 ceph-mon[220044]: failed to encode map e215 with expected crc 2023-10-09T05:55:46.726 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:46 smithi064 ceph-mon[220044]: failed to encode map e215 with expected crc 2023-10-09T05:55:46.726 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:46 smithi064 ceph-mon[220044]: failed to encode map e215 with expected crc 2023-10-09T05:55:47.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:46 smithi156 ceph-mon[195101]: failed to encode map e215 with expected crc 2023-10-09T05:55:47.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:46 smithi156 ceph-mon[195101]: failed to encode map e215 with expected crc 2023-10-09T05:55:47.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:46 smithi156 ceph-mon[195101]: failed to encode map e215 with expected crc 2023-10-09T05:55:47.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:46 smithi156 ceph-mon[195101]: failed to encode map e215 with expected crc 2023-10-09T05:55:47.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:46 smithi156 ceph-mon[195101]: failed to encode map e215 with expected crc 2023-10-09T05:55:47.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:46 smithi156 ceph-mon[195101]: failed to encode map e215 with expected crc 2023-10-09T05:55:47.239 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:47.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:47.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:47.349 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:47.375 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:47.419 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:47.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:47.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:47.525 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:47.577 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:47.657 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:47.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:47.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:47.730 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T05:55:47.763 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:47.808 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:47.886 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:47.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:47 smithi064 ceph-mon[212750]: pgmap v492: 129 pgs: 11 remapped+peering, 36 peering, 3 unknown, 4 active+recovery_wait+degraded+remapped, 71 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 301 op/s; 13852/221949 objects degraded (6.241%); 4694/221949 objects misplaced (2.115%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T05:55:47.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:47 smithi064 ceph-mon[212750]: failed to encode map e216 with expected crc 2023-10-09T05:55:47.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:47 smithi064 ceph-mon[212750]: osdmap e216: 8 total, 8 up, 8 in 2023-10-09T05:55:47.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:47 smithi064 ceph-mon[212750]: failed to encode map e216 with expected crc 2023-10-09T05:55:47.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:47 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 13852/221949 objects degraded (6.241%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:47.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:47 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3902270263' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:55:47.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:47 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:55:47.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:47 smithi064 ceph-mon[212750]: failed to encode map e216 with expected crc 2023-10-09T05:55:47.891 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:47 smithi064 ceph-mon[220044]: pgmap v492: 129 pgs: 11 remapped+peering, 36 peering, 3 unknown, 4 active+recovery_wait+degraded+remapped, 71 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 301 op/s; 13852/221949 objects degraded (6.241%); 4694/221949 objects misplaced (2.115%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T05:55:47.892 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:47 smithi064 ceph-mon[220044]: failed to encode map e216 with expected crc 2023-10-09T05:55:47.892 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:47 smithi064 ceph-mon[220044]: osdmap e216: 8 total, 8 up, 8 in 2023-10-09T05:55:47.892 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:47 smithi064 ceph-mon[220044]: failed to encode map e216 with expected crc 2023-10-09T05:55:47.892 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:47 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 13852/221949 objects degraded (6.241%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:47.892 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:47 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3902270263' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:55:47.892 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:47 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:55:47.892 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:47 smithi064 ceph-mon[220044]: failed to encode map e216 with expected crc 2023-10-09T05:55:47.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:47.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:47.994 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:48.040 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:48.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:47 smithi156 ceph-mon[195101]: pgmap v492: 129 pgs: 11 remapped+peering, 36 peering, 3 unknown, 4 active+recovery_wait+degraded+remapped, 71 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 301 op/s; 13852/221949 objects degraded (6.241%); 4694/221949 objects misplaced (2.115%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T05:55:48.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:47 smithi156 ceph-mon[195101]: failed to encode map e216 with expected crc 2023-10-09T05:55:48.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:47 smithi156 ceph-mon[195101]: osdmap e216: 8 total, 8 up, 8 in 2023-10-09T05:55:48.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:47 smithi156 ceph-mon[195101]: failed to encode map e216 with expected crc 2023-10-09T05:55:48.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:47 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 13852/221949 objects degraded (6.241%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T05:55:48.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:47 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3902270263' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:55:48.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:47 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:55:48.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:47 smithi156 ceph-mon[195101]: failed to encode map e216 with expected crc 2023-10-09T05:55:48.138 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:48.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:48.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:48.248 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:48.290 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:49.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:48 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:49.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:48 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:55:49.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:48 smithi156 ceph-mon[195101]: osdmap e217: 8 total, 8 up, 8 in 2023-10-09T05:55:49.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:48 smithi156 ceph-mon[195101]: failed to encode map e217 with expected crc 2023-10-09T05:55:49.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:48 smithi156 ceph-mon[195101]: failed to encode map e217 with expected crc 2023-10-09T05:55:49.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:48 smithi156 ceph-mon[195101]: failed to encode map e217 with expected crc 2023-10-09T05:55:49.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:48 smithi156 ceph-mon[195101]: failed to encode map e217 with expected crc 2023-10-09T05:55:49.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:48 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:49.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:48 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:55:49.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:48 smithi064 ceph-mon[212750]: osdmap e217: 8 total, 8 up, 8 in 2023-10-09T05:55:49.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:48 smithi064 ceph-mon[212750]: failed to encode map e217 with expected crc 2023-10-09T05:55:49.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:48 smithi064 ceph-mon[212750]: failed to encode map e217 with expected crc 2023-10-09T05:55:49.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:48 smithi064 ceph-mon[212750]: failed to encode map e217 with expected crc 2023-10-09T05:55:49.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:48 smithi064 ceph-mon[212750]: failed to encode map e217 with expected crc 2023-10-09T05:55:49.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:48 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:55:49.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:48 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:55:49.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:48 smithi064 ceph-mon[220044]: osdmap e217: 8 total, 8 up, 8 in 2023-10-09T05:55:49.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:48 smithi064 ceph-mon[220044]: failed to encode map e217 with expected crc 2023-10-09T05:55:49.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:48 smithi064 ceph-mon[220044]: failed to encode map e217 with expected crc 2023-10-09T05:55:49.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:48 smithi064 ceph-mon[220044]: failed to encode map e217 with expected crc 2023-10-09T05:55:49.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:48 smithi064 ceph-mon[220044]: failed to encode map e217 with expected crc 2023-10-09T05:55:49.693 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:49.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:49.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:49.798 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:49.826 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:49.873 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:49.873 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T05:55:49.873 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:55:49.874 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 2, 3, 4, 5, 6, 7, 1] 2023-10-09T05:55:49.874 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd out 1 2023-10-09T05:55:49.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:49.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:49.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:49 smithi064 ceph-mon[212750]: pgmap v495: 129 pgs: 11 remapped+peering, 36 peering, 3 unknown, 4 active+recovery_wait+degraded+remapped, 71 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 101 op/s; 13852/221829 objects degraded (6.244%); 4694/221829 objects misplaced (2.116%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T05:55:49.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:49 smithi064 ceph-mon[220044]: pgmap v495: 129 pgs: 11 remapped+peering, 36 peering, 3 unknown, 4 active+recovery_wait+degraded+remapped, 71 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 101 op/s; 13852/221829 objects degraded (6.244%); 4694/221829 objects misplaced (2.116%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T05:55:49.977 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:50.025 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:50.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:49 smithi156 ceph-mon[195101]: pgmap v495: 129 pgs: 11 remapped+peering, 36 peering, 3 unknown, 4 active+recovery_wait+degraded+remapped, 71 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 101 op/s; 13852/221829 objects degraded (6.244%); 4694/221829 objects misplaced (2.116%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T05:55:50.392 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:50.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:50.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:50.498 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:50.509 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:55:50.509 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T05:55:50.509 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2023-10-09T05:55:50.510 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:55:50.584 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:51.348 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:55:51.374 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:51.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:51.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:51.480 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:51.506 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:51.549 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:51.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:51.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:51.656 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:51.682 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:51.725 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:51.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:51.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:51.830 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:51.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:51 smithi156 ceph-mon[195101]: pgmap v496: 129 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 15 peering, 3 unknown, 6 active+recovery_wait+degraded+remapped, 94 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 85 op/s; 24845/221526 objects degraded (11.215%); 12671/221526 objects misplaced (5.720%); 5.4 MiB/s, 87 objects/s recovering 2023-10-09T05:55:51.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:51 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/355879409' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:51.871 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:51.881 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:55:51.976 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:51.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:51.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:52.080 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:52.107 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:52.149 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:52.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:52.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:52.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:51 smithi064 ceph-mon[212750]: pgmap v496: 129 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 15 peering, 3 unknown, 6 active+recovery_wait+degraded+remapped, 94 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 85 op/s; 24845/221526 objects degraded (11.215%); 12671/221526 objects misplaced (5.720%); 5.4 MiB/s, 87 objects/s recovering 2023-10-09T05:55:52.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:51 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/355879409' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:52.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:51 smithi064 ceph-mon[220044]: pgmap v496: 129 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 15 peering, 3 unknown, 6 active+recovery_wait+degraded+remapped, 94 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 85 op/s; 24845/221526 objects degraded (11.215%); 12671/221526 objects misplaced (5.720%); 5.4 MiB/s, 87 objects/s recovering 2023-10-09T05:55:52.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:51 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/355879409' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:55:52.254 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:52.280 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:52.323 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:52.323 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:52.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:52.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:52.362 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T05:55:52.429 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:52.455 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:52.498 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:52.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:52.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:52.603 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:52.629 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:52.673 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:52.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:52.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:52.777 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:52.804 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:52.825 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:52.960 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:52 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 24845/221526 objects degraded (11.215%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:55:52.962 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:52 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 24845/221526 objects degraded (11.215%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:55:53.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:52 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 24845/221526 objects degraded (11.215%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:55:53.941 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:53.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:53.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:54.058 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:54.101 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:54.201 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:55:54.229 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:54 smithi064 ceph-mon[212750]: pgmap v497: 129 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 15 peering, 3 unknown, 6 active+recovery_wait+degraded+remapped, 94 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 20 op/s; 24845/221526 objects degraded (11.215%); 12671/221526 objects misplaced (5.720%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:55:54.229 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:54 smithi064 ceph-mon[220044]: pgmap v497: 129 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 15 peering, 3 unknown, 6 active+recovery_wait+degraded+remapped, 94 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 20 op/s; 24845/221526 objects degraded (11.215%); 12671/221526 objects misplaced (5.720%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:55:54.250 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:55:54.286 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:55:54.300 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:54.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:54.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:54.324 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:55:54.363 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:55:54.397 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:55:54.421 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:54.436 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:55:54.444 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:55:54.444 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:55:54.444 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:55:54.444 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:55:54.444 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:55:54.444 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:55:54.444 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:55:54.444 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:55:54.490 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:54.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:54 smithi156 ceph-mon[195101]: pgmap v497: 129 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 15 peering, 3 unknown, 6 active+recovery_wait+degraded+remapped, 94 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 20 op/s; 24845/221526 objects degraded (11.215%); 12671/221526 objects misplaced (5.720%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T05:55:54.745 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:55:54.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:54.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:55:54.858 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:54.887 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:54.934 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:54.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:54.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:55.038 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:55.106 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:55.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:55 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 9 pgs peering) 2023-10-09T05:55:55.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:55 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 9 pgs peering) 2023-10-09T05:55:55.470 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:55:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:55:55] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.43.0" 2023-10-09T05:55:55.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:55 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 9 pgs peering) 2023-10-09T05:55:55.821 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:55:55.996 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:56.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:56.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:56.109 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:56.141 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:56.192 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:56.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:56.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:56.247 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:56 smithi064 ceph-mon[220044]: pgmap v498: 129 pgs: 2 activating+degraded, 1 active+recovering+undersized+remapped, 3 active+degraded, 4 active+recovery_wait+degraded+remapped, 112 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 42 op/s; 30533/221136 objects degraded (13.807%); 9121/221136 objects misplaced (4.125%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T05:55:56.298 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:56.328 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:56.379 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:56.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:56.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:56.485 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:56.513 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:56 smithi156 ceph-mon[195101]: pgmap v498: 129 pgs: 2 activating+degraded, 1 active+recovering+undersized+remapped, 3 active+degraded, 4 active+recovery_wait+degraded+remapped, 112 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 42 op/s; 30533/221136 objects degraded (13.807%); 9121/221136 objects misplaced (4.125%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T05:55:56.516 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:56.570 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:56.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:56.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:56.677 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:56.726 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:56 smithi064 ceph-mon[212750]: pgmap v498: 129 pgs: 2 activating+degraded, 1 active+recovering+undersized+remapped, 3 active+degraded, 4 active+recovery_wait+degraded+remapped, 112 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 42 op/s; 30533/221136 objects degraded (13.807%); 9121/221136 objects misplaced (4.125%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T05:55:56.746 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:57.072 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:57.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:57.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:57.181 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:55:57.208 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:57.252 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:55:57.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:57.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:55:57.347 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:57 smithi064 ceph-mon[212750]: pgmap v499: 129 pgs: 2 activating+degraded, 1 active+recovering+undersized+remapped, 3 active+degraded, 4 active+recovery_wait+degraded+remapped, 112 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 8.4 KiB/s rd, 22 KiB/s wr, 44 op/s; 30533/220824 objects degraded (13.827%); 9121/220824 objects misplaced (4.130%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T05:55:57.348 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:57 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 30533/221136 objects degraded (13.807%), 15 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:55:57.348 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:57 smithi064 ceph-mon[220044]: pgmap v499: 129 pgs: 2 activating+degraded, 1 active+recovering+undersized+remapped, 3 active+degraded, 4 active+recovery_wait+degraded+remapped, 112 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 8.4 KiB/s rd, 22 KiB/s wr, 44 op/s; 30533/220824 objects degraded (13.827%); 9121/220824 objects misplaced (4.130%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T05:55:57.348 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:57 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 30533/221136 objects degraded (13.807%), 15 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:55:57.358 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:55:57.386 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:57.431 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:55:57.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:57.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:55:57.536 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:55:57.609 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:57 smithi064 conmon[212726]: 2023-10-09T05:55:57.456+0000 7f8823f23700 -1 mon.a@0(leader).osd e217 definitely_dead 0 2023-10-09T05:55:57.613 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:57.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:57 smithi156 ceph-mon[195101]: pgmap v499: 129 pgs: 2 activating+degraded, 1 active+recovering+undersized+remapped, 3 active+degraded, 4 active+recovery_wait+degraded+remapped, 112 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 8.4 KiB/s rd, 22 KiB/s wr, 44 op/s; 30533/220824 objects degraded (13.827%); 9121/220824 objects misplaced (4.130%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T05:55:57.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:57 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 30533/221136 objects degraded (13.807%), 15 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:55:57.771 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:55:57.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:57.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:55:57.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:55:57.946 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:58.095 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:55:58.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:58.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:55:58.210 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:58.295 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:58.309 INFO:teuthology.orchestra.run.smithi064.stderr:marked out osd.1. 2023-10-09T05:55:58.322 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T05:55:58.375 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:58.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:58.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:58.479 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:55:58.507 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:58.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:58 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3755222816' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:55:58.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:58 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:55:58.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:58 smithi156 ceph-mon[195101]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T05:55:58.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:58 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1470822819' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:58.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:58 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:58.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:58 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3755222816' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:55:58.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:58 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:55:58.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:58 smithi064 ceph-mon[220044]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T05:55:58.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:58 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1470822819' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:58.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:58 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:58.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:58 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3755222816' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:55:58.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:58 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:55:58.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:58 smithi064 ceph-mon[212750]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T05:55:58.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:58 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1470822819' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:58.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:58 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:55:58.857 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:55:59.010 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:55:59.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:59.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:55:59.114 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:55:59.141 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:59.183 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:55:59.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:55:59.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:55:59.288 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:55:59.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:55:59.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[212750]: failed to encode map e218 with expected crc 2023-10-09T05:55:59.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T05:55:59.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:55:59.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[212750]: osdmap e218: 8 total, 8 up, 7 in 2023-10-09T05:55:59.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[212750]: failed to encode map e218 with expected crc 2023-10-09T05:55:59.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[212750]: failed to encode map e218 with expected crc 2023-10-09T05:55:59.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[212750]: failed to encode map e218 with expected crc 2023-10-09T05:55:59.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[212750]: failed to encode map e218 with expected crc 2023-10-09T05:55:59.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:59.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:55:59.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[212750]: pgmap v501: 129 pgs: 2 activating+degraded, 1 active+recovering+undersized+remapped, 3 active+degraded, 4 active+recovery_wait+degraded+remapped, 112 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 8.3 KiB/s rd, 22 KiB/s wr, 48 op/s; 30533/220578 objects degraded (13.842%); 9121/220578 objects misplaced (4.135%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T05:55:59.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[212750]: 1.a scrub starts 2023-10-09T05:55:59.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[212750]: osdmap e219: 8 total, 8 up, 7 in 2023-10-09T05:55:59.359 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:55:59.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:59 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:55:59.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:59 smithi156 ceph-mon[195101]: failed to encode map e218 with expected crc 2023-10-09T05:55:59.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:59 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T05:55:59.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:59 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:55:59.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:59 smithi156 ceph-mon[195101]: osdmap e218: 8 total, 8 up, 7 in 2023-10-09T05:55:59.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:59 smithi156 ceph-mon[195101]: failed to encode map e218 with expected crc 2023-10-09T05:55:59.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:59 smithi156 ceph-mon[195101]: failed to encode map e218 with expected crc 2023-10-09T05:55:59.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:59 smithi156 ceph-mon[195101]: failed to encode map e218 with expected crc 2023-10-09T05:55:59.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:59 smithi156 ceph-mon[195101]: failed to encode map e218 with expected crc 2023-10-09T05:55:59.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:59 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:59.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:55:59.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:59 smithi156 ceph-mon[195101]: pgmap v501: 129 pgs: 2 activating+degraded, 1 active+recovering+undersized+remapped, 3 active+degraded, 4 active+recovery_wait+degraded+remapped, 112 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 8.3 KiB/s rd, 22 KiB/s wr, 48 op/s; 30533/220578 objects degraded (13.842%); 9121/220578 objects misplaced (4.135%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T05:55:59.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:59 smithi156 ceph-mon[195101]: 1.a scrub starts 2023-10-09T05:55:59.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:55:59 smithi156 ceph-mon[195101]: osdmap e219: 8 total, 8 up, 7 in 2023-10-09T05:55:59.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:55:59.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[220044]: failed to encode map e218 with expected crc 2023-10-09T05:55:59.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T05:55:59.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:55:59.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[220044]: osdmap e218: 8 total, 8 up, 7 in 2023-10-09T05:55:59.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[220044]: failed to encode map e218 with expected crc 2023-10-09T05:55:59.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[220044]: failed to encode map e218 with expected crc 2023-10-09T05:55:59.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[220044]: failed to encode map e218 with expected crc 2023-10-09T05:55:59.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[220044]: failed to encode map e218 with expected crc 2023-10-09T05:55:59.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:55:59.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:55:59.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[220044]: pgmap v501: 129 pgs: 2 activating+degraded, 1 active+recovering+undersized+remapped, 3 active+degraded, 4 active+recovery_wait+degraded+remapped, 112 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 8.3 KiB/s rd, 22 KiB/s wr, 48 op/s; 30533/220578 objects degraded (13.842%); 9121/220578 objects misplaced (4.135%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T05:55:59.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[220044]: 1.a scrub starts 2023-10-09T05:55:59.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:55:59 smithi064 ceph-mon[220044]: osdmap e219: 8 total, 8 up, 7 in 2023-10-09T05:56:00.045 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:00.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:00.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:00.148 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:00.175 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:00.178 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:00.220 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:56:00.221 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:56:00.223 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:00.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:00.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:00.330 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:00.406 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:00.469 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:00.476 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:00 smithi156 ceph-mon[195101]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.476 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:00 smithi156 ceph-mon[195101]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.476 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:00 smithi156 ceph-mon[195101]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.476 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:00 smithi156 ceph-mon[195101]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.477 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:00 smithi156 ceph-mon[195101]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.477 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:00 smithi156 ceph-mon[195101]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.477 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:00 smithi156 ceph-mon[195101]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.477 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:00 smithi156 ceph-mon[195101]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.477 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:00 smithi156 ceph-mon[195101]: 6.f starting backfill to osd.5 from (107'1991,212'7258] MIN to 217'7299 2023-10-09T05:56:00.477 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:00 smithi156 ceph-mon[195101]: 6.1f starting backfill to osd.5 from (107'1991,212'7217] MIN to 217'7264 2023-10-09T05:56:00.477 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:00 smithi156 ceph-mon[195101]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.477 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:00 smithi156 ceph-mon[195101]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.477 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:00 smithi156 ceph-mon[195101]: osdmap e220: 8 total, 8 up, 7 in 2023-10-09T05:56:00.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:00.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:00.502 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[212750]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.503 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[212750]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.503 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[212750]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.503 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[212750]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.503 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[212750]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.503 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[212750]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.503 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[212750]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.504 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[212750]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.504 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[212750]: 6.f starting backfill to osd.5 from (107'1991,212'7258] MIN to 217'7299 2023-10-09T05:56:00.504 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[212750]: 6.1f starting backfill to osd.5 from (107'1991,212'7217] MIN to 217'7264 2023-10-09T05:56:00.504 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[212750]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.504 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[212750]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.504 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[212750]: osdmap e220: 8 total, 8 up, 7 in 2023-10-09T05:56:00.505 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[220044]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.505 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[220044]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.505 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[220044]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.505 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[220044]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.505 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[220044]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.506 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[220044]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.506 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[220044]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.506 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[220044]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.506 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[220044]: 6.f starting backfill to osd.5 from (107'1991,212'7258] MIN to 217'7299 2023-10-09T05:56:00.506 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[220044]: 6.1f starting backfill to osd.5 from (107'1991,212'7217] MIN to 217'7264 2023-10-09T05:56:00.506 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[220044]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[220044]: failed to encode map e219 with expected crc 2023-10-09T05:56:00.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:00 smithi064 ceph-mon[220044]: osdmap e220: 8 total, 8 up, 7 in 2023-10-09T05:56:00.579 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:00.628 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:00.691 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:00.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:00.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:00.797 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:00.837 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:00.892 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:00.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:00.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:00.997 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:01.034 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:01.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[212750]: 1.6 scrub starts 2023-10-09T05:56:01.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[212750]: failed to encode map e220 with expected crc 2023-10-09T05:56:01.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[212750]: failed to encode map e220 with expected crc 2023-10-09T05:56:01.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[212750]: failed to encode map e220 with expected crc 2023-10-09T05:56:01.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[212750]: failed to encode map e220 with expected crc 2023-10-09T05:56:01.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[212750]: failed to encode map e220 with expected crc 2023-10-09T05:56:01.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[212750]: 6.4 starting backfill to osd.4 from (107'1426,212'7050] MIN to 217'7080 2023-10-09T05:56:01.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[212750]: failed to encode map e220 with expected crc 2023-10-09T05:56:01.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[212750]: 6.1e starting backfill to osd.7 from (0'0,0'0] MAX to 217'7113 2023-10-09T05:56:01.594 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[212750]: 6.21 starting backfill to osd.0 from (107'871,212'6927] MIN to 217'6954 2023-10-09T05:56:01.594 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[212750]: 6.d starting backfill to osd.2 from (0'0,0'0] MAX to 217'7184 2023-10-09T05:56:01.594 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[212750]: 6.14 starting backfill to osd.4 from (107'1426,212'7027] MIN to 217'7060 2023-10-09T05:56:01.594 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[212750]: failed to encode map e220 with expected crc 2023-10-09T05:56:01.594 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[212750]: 6.e starting backfill to osd.7 from (0'0,0'0] MAX to 217'7162 2023-10-09T05:56:01.594 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[212750]: 6.1d starting backfill to osd.2 from (0'0,0'0] MAX to 217'7202 2023-10-09T05:56:01.594 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[212750]: 6.1 starting backfill to osd.0 from (107'871,212'6911] MIN to 217'6934 2023-10-09T05:56:01.594 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[212750]: 6.11 starting backfill to osd.0 from (107'871,212'7048] MIN to 217'7094 2023-10-09T05:56:01.594 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[212750]: pgmap v504: 129 pgs: 6 remapped+peering, 4 peering, 1 activating, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 activating+degraded, 2 active+degraded, 3 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 188 op/s; 18628/217746 objects degraded (8.555%); 8650/217746 objects misplaced (3.973%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T05:56:01.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[220044]: 1.6 scrub starts 2023-10-09T05:56:01.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[220044]: failed to encode map e220 with expected crc 2023-10-09T05:56:01.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[220044]: failed to encode map e220 with expected crc 2023-10-09T05:56:01.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[220044]: failed to encode map e220 with expected crc 2023-10-09T05:56:01.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[220044]: failed to encode map e220 with expected crc 2023-10-09T05:56:01.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[220044]: failed to encode map e220 with expected crc 2023-10-09T05:56:01.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[220044]: 6.4 starting backfill to osd.4 from (107'1426,212'7050] MIN to 217'7080 2023-10-09T05:56:01.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[220044]: failed to encode map e220 with expected crc 2023-10-09T05:56:01.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[220044]: 6.1e starting backfill to osd.7 from (0'0,0'0] MAX to 217'7113 2023-10-09T05:56:01.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[220044]: 6.21 starting backfill to osd.0 from (107'871,212'6927] MIN to 217'6954 2023-10-09T05:56:01.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[220044]: 6.d starting backfill to osd.2 from (0'0,0'0] MAX to 217'7184 2023-10-09T05:56:01.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[220044]: 6.14 starting backfill to osd.4 from (107'1426,212'7027] MIN to 217'7060 2023-10-09T05:56:01.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[220044]: failed to encode map e220 with expected crc 2023-10-09T05:56:01.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[220044]: 6.e starting backfill to osd.7 from (0'0,0'0] MAX to 217'7162 2023-10-09T05:56:01.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[220044]: 6.1d starting backfill to osd.2 from (0'0,0'0] MAX to 217'7202 2023-10-09T05:56:01.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[220044]: 6.1 starting backfill to osd.0 from (107'871,212'6911] MIN to 217'6934 2023-10-09T05:56:01.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[220044]: 6.11 starting backfill to osd.0 from (107'871,212'7048] MIN to 217'7094 2023-10-09T05:56:01.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:01 smithi064 ceph-mon[220044]: pgmap v504: 129 pgs: 6 remapped+peering, 4 peering, 1 activating, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 activating+degraded, 2 active+degraded, 3 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 188 op/s; 18628/217746 objects degraded (8.555%); 8650/217746 objects misplaced (3.973%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T05:56:01.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:01 smithi156 ceph-mon[195101]: 1.6 scrub starts 2023-10-09T05:56:01.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:01 smithi156 ceph-mon[195101]: failed to encode map e220 with expected crc 2023-10-09T05:56:01.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:01 smithi156 ceph-mon[195101]: failed to encode map e220 with expected crc 2023-10-09T05:56:01.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:01 smithi156 ceph-mon[195101]: failed to encode map e220 with expected crc 2023-10-09T05:56:01.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:01 smithi156 ceph-mon[195101]: failed to encode map e220 with expected crc 2023-10-09T05:56:01.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:01 smithi156 ceph-mon[195101]: failed to encode map e220 with expected crc 2023-10-09T05:56:01.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:01 smithi156 ceph-mon[195101]: 6.4 starting backfill to osd.4 from (107'1426,212'7050] MIN to 217'7080 2023-10-09T05:56:01.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:01 smithi156 ceph-mon[195101]: failed to encode map e220 with expected crc 2023-10-09T05:56:01.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:01 smithi156 ceph-mon[195101]: 6.1e starting backfill to osd.7 from (0'0,0'0] MAX to 217'7113 2023-10-09T05:56:01.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:01 smithi156 ceph-mon[195101]: 6.21 starting backfill to osd.0 from (107'871,212'6927] MIN to 217'6954 2023-10-09T05:56:01.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:01 smithi156 ceph-mon[195101]: 6.d starting backfill to osd.2 from (0'0,0'0] MAX to 217'7184 2023-10-09T05:56:01.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:01 smithi156 ceph-mon[195101]: 6.14 starting backfill to osd.4 from (107'1426,212'7027] MIN to 217'7060 2023-10-09T05:56:01.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:01 smithi156 ceph-mon[195101]: failed to encode map e220 with expected crc 2023-10-09T05:56:01.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:01 smithi156 ceph-mon[195101]: 6.e starting backfill to osd.7 from (0'0,0'0] MAX to 217'7162 2023-10-09T05:56:01.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:01 smithi156 ceph-mon[195101]: 6.1d starting backfill to osd.2 from (0'0,0'0] MAX to 217'7202 2023-10-09T05:56:01.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:01 smithi156 ceph-mon[195101]: 6.1 starting backfill to osd.0 from (107'871,212'6911] MIN to 217'6934 2023-10-09T05:56:01.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:01 smithi156 ceph-mon[195101]: 6.11 starting backfill to osd.0 from (107'871,212'7048] MIN to 217'7094 2023-10-09T05:56:01.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:01 smithi156 ceph-mon[195101]: pgmap v504: 129 pgs: 6 remapped+peering, 4 peering, 1 activating, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 3 activating+degraded, 2 active+degraded, 3 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 188 op/s; 18628/217746 objects degraded (8.555%); 8650/217746 objects misplaced (3.973%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T05:56:01.709 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T05:56:01.951 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:56:01.967 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:01.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:01.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:02.071 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:56:02.106 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:02.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:02 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 18628/217746 objects degraded (8.555%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:02.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:02 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 18628/217746 objects degraded (8.555%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:02.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:02 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 18628/217746 objects degraded (8.555%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:02.893 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:02.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:02.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:02.997 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:03.038 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:03.108 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:03.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:03.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:03.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:03.215 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:03.256 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:03.365 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:03.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:03.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:03.471 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:03.500 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:03.528 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:03 smithi064 ceph-mon[212750]: pgmap v505: 129 pgs: 6 remapped+peering, 4 peering, 1 activating, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 activating+degraded, 1 active+degraded, 3 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 197 op/s; 18530/217263 objects degraded (8.529%); 11070/217263 objects misplaced (5.095%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T05:56:03.528 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:03 smithi064 ceph-mon[220044]: pgmap v505: 129 pgs: 6 remapped+peering, 4 peering, 1 activating, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 activating+degraded, 1 active+degraded, 3 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 197 op/s; 18530/217263 objects degraded (8.529%); 11070/217263 objects misplaced (5.095%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T05:56:03.550 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:03.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:03.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:03.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:03 smithi156 ceph-mon[195101]: pgmap v505: 129 pgs: 6 remapped+peering, 4 peering, 1 activating, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 activating+degraded, 1 active+degraded, 3 active+recovery_wait+degraded+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 197 op/s; 18530/217263 objects degraded (8.529%); 11070/217263 objects misplaced (5.095%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T05:56:03.629 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:03.656 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:56:03.702 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:03.772 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:56:03.798 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:56:03.825 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:56:03.851 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:56:03.871 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:03.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:03.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:03.874 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:56:03.897 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:56:03.920 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:56:03.945 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:56:03.949 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:56:03.949 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:56:03.949 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:56:03.949 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:56:03.949 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:56:03.949 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:56:03.949 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:56:03.949 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:56:03.975 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:04.007 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:04.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:04 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1669730004' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:04.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:04 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1669730004' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:04.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:04 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1669730004' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:04.958 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T05:56:04.958 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:56:04.958 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_0 2023-10-09T05:56:04.959 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:56:05.218 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:05.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:05.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:05.322 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:05.378 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:05.470 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:05.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:05.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:05.577 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:05.616 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:05 smithi064 ceph-mon[212750]: osdmap e221: 8 total, 8 up, 7 in 2023-10-09T05:56:05.617 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:05 smithi064 ceph-mon[212750]: failed to encode map e221 with expected crc 2023-10-09T05:56:05.617 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:05 smithi064 ceph-mon[212750]: failed to encode map e221 with expected crc 2023-10-09T05:56:05.617 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:05 smithi064 ceph-mon[212750]: failed to encode map e221 with expected crc 2023-10-09T05:56:05.617 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:05 smithi064 ceph-mon[212750]: failed to encode map e221 with expected crc 2023-10-09T05:56:05.617 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:05 smithi064 ceph-mon[212750]: pgmap v507: 129 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 3 peering, 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+degraded, 3 active+recovery_wait+degraded+remapped, 108 active+clean, 2 active+remapped+backfill_wait, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 201 op/s; 32759/216951 objects degraded (15.100%); 16471/216951 objects misplaced (7.592%); 40 MiB/s, 637 objects/s recovering 2023-10-09T05:56:05.618 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:05 smithi064 ceph-mon[220044]: osdmap e221: 8 total, 8 up, 7 in 2023-10-09T05:56:05.618 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:05 smithi064 ceph-mon[220044]: failed to encode map e221 with expected crc 2023-10-09T05:56:05.618 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:05 smithi064 ceph-mon[220044]: failed to encode map e221 with expected crc 2023-10-09T05:56:05.618 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:05 smithi064 ceph-mon[220044]: failed to encode map e221 with expected crc 2023-10-09T05:56:05.618 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:05 smithi064 ceph-mon[220044]: failed to encode map e221 with expected crc 2023-10-09T05:56:05.618 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:05 smithi064 ceph-mon[220044]: pgmap v507: 129 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 3 peering, 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+degraded, 3 active+recovery_wait+degraded+remapped, 108 active+clean, 2 active+remapped+backfill_wait, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 201 op/s; 32759/216951 objects degraded (15.100%); 16471/216951 objects misplaced (7.592%); 40 MiB/s, 637 objects/s recovering 2023-10-09T05:56:05.619 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:56:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:56:05] "GET /metrics HTTP/1.1" 200 36488 "" "Prometheus/2.43.0" 2023-10-09T05:56:05.619 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:05.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:05 smithi156 ceph-mon[195101]: osdmap e221: 8 total, 8 up, 7 in 2023-10-09T05:56:05.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:05 smithi156 ceph-mon[195101]: failed to encode map e221 with expected crc 2023-10-09T05:56:05.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:05 smithi156 ceph-mon[195101]: failed to encode map e221 with expected crc 2023-10-09T05:56:05.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:05 smithi156 ceph-mon[195101]: failed to encode map e221 with expected crc 2023-10-09T05:56:05.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:05 smithi156 ceph-mon[195101]: failed to encode map e221 with expected crc 2023-10-09T05:56:05.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:05 smithi156 ceph-mon[195101]: pgmap v507: 129 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 3 peering, 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+degraded, 3 active+recovery_wait+degraded+remapped, 108 active+clean, 2 active+remapped+backfill_wait, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 201 op/s; 32759/216951 objects degraded (15.100%); 16471/216951 objects misplaced (7.592%); 40 MiB/s, 637 objects/s recovering 2023-10-09T05:56:05.994 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:06.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:06.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:06.103 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:06.133 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:06.189 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:06.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:06.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:06.295 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:06.327 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:06.401 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:06.405 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:06 smithi156 ceph-mon[195101]: failed to encode map e222 with expected crc 2023-10-09T05:56:06.405 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:06 smithi156 ceph-mon[195101]: osdmap e222: 8 total, 8 up, 7 in 2023-10-09T05:56:06.405 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:06 smithi156 ceph-mon[195101]: failed to encode map e222 with expected crc 2023-10-09T05:56:06.405 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:06 smithi156 ceph-mon[195101]: failed to encode map e222 with expected crc 2023-10-09T05:56:06.405 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:06 smithi156 ceph-mon[195101]: failed to encode map e222 with expected crc 2023-10-09T05:56:06.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:06.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:06.507 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:06.529 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:06 smithi064 ceph-mon[212750]: failed to encode map e222 with expected crc 2023-10-09T05:56:06.529 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:06 smithi064 ceph-mon[212750]: osdmap e222: 8 total, 8 up, 7 in 2023-10-09T05:56:06.529 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:06 smithi064 ceph-mon[212750]: failed to encode map e222 with expected crc 2023-10-09T05:56:06.529 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:06 smithi064 ceph-mon[212750]: failed to encode map e222 with expected crc 2023-10-09T05:56:06.529 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:06 smithi064 ceph-mon[212750]: failed to encode map e222 with expected crc 2023-10-09T05:56:06.529 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:06 smithi064 ceph-mon[220044]: failed to encode map e222 with expected crc 2023-10-09T05:56:06.530 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:06 smithi064 ceph-mon[220044]: osdmap e222: 8 total, 8 up, 7 in 2023-10-09T05:56:06.530 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:06 smithi064 ceph-mon[220044]: failed to encode map e222 with expected crc 2023-10-09T05:56:06.530 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:06 smithi064 ceph-mon[220044]: failed to encode map e222 with expected crc 2023-10-09T05:56:06.530 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:06 smithi064 ceph-mon[220044]: failed to encode map e222 with expected crc 2023-10-09T05:56:06.539 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:06.616 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:06.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:06.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:06.723 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:06.759 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:06.827 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:06.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:06.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:06.939 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:06.983 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:07.054 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:07.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:07.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:07.162 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:07.194 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:07.241 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:07.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:07.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:07.347 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:07.351 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:07 smithi156 ceph-mon[195101]: pgmap v509: 129 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 peering, 1 active+undersized+degraded+remapped, 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering, 2 active+remapped+backfill_wait, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 91 op/s; 35559/216570 objects degraded (16.419%); 19501/216570 objects misplaced (9.004%); 38 MiB/s, 601 objects/s recovering 2023-10-09T05:56:07.351 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:07 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/280888967' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:56:07.351 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:07 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:56:07.351 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:07 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 32759/216951 objects degraded (15.100%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:07.382 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T05:56:07.398 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:07.482 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:07.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:07.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:07.592 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:56:07.654 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:07 smithi064 ceph-mon[212750]: pgmap v509: 129 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 peering, 1 active+undersized+degraded+remapped, 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering, 2 active+remapped+backfill_wait, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 91 op/s; 35559/216570 objects degraded (16.419%); 19501/216570 objects misplaced (9.004%); 38 MiB/s, 601 objects/s recovering 2023-10-09T05:56:07.654 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:07 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/280888967' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:56:07.654 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:07 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:56:07.654 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:07 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 32759/216951 objects degraded (15.100%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:07.655 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:07 smithi064 ceph-mon[220044]: pgmap v509: 129 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 peering, 1 active+undersized+degraded+remapped, 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering, 2 active+remapped+backfill_wait, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 91 op/s; 35559/216570 objects degraded (16.419%); 19501/216570 objects misplaced (9.004%); 38 MiB/s, 601 objects/s recovering 2023-10-09T05:56:07.655 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:07 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/280888967' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:56:07.655 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:07 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:56:07.655 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:07 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 32759/216951 objects degraded (15.100%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:07.660 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:08.151 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:56:08.462 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:08.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:08 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:08.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:08 smithi064 ceph-mon[212750]: failed to encode map e223 with expected crc 2023-10-09T05:56:08.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:08 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:56:08.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:08 smithi064 ceph-mon[212750]: osdmap e223: 8 total, 8 up, 7 in 2023-10-09T05:56:08.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:08 smithi064 ceph-mon[212750]: failed to encode map e223 with expected crc 2023-10-09T05:56:08.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:08 smithi064 ceph-mon[212750]: failed to encode map e223 with expected crc 2023-10-09T05:56:08.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:08 smithi064 ceph-mon[212750]: failed to encode map e223 with expected crc 2023-10-09T05:56:08.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:08 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:08.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:08 smithi064 ceph-mon[220044]: failed to encode map e223 with expected crc 2023-10-09T05:56:08.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:08 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:56:08.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:08 smithi064 ceph-mon[220044]: osdmap e223: 8 total, 8 up, 7 in 2023-10-09T05:56:08.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:08 smithi064 ceph-mon[220044]: failed to encode map e223 with expected crc 2023-10-09T05:56:08.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:08 smithi064 ceph-mon[220044]: failed to encode map e223 with expected crc 2023-10-09T05:56:08.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:08 smithi064 ceph-mon[220044]: failed to encode map e223 with expected crc 2023-10-09T05:56:08.485 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:08.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:08.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:08.599 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:08.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:08 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:08.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:08 smithi156 ceph-mon[195101]: failed to encode map e223 with expected crc 2023-10-09T05:56:08.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:08 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:56:08.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:08 smithi156 ceph-mon[195101]: osdmap e223: 8 total, 8 up, 7 in 2023-10-09T05:56:08.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:08 smithi156 ceph-mon[195101]: failed to encode map e223 with expected crc 2023-10-09T05:56:08.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:08 smithi156 ceph-mon[195101]: failed to encode map e223 with expected crc 2023-10-09T05:56:08.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:08 smithi156 ceph-mon[195101]: failed to encode map e223 with expected crc 2023-10-09T05:56:08.632 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:08.693 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:08.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:08.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:08.799 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:08.838 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:08.886 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:08.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:08.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:08.992 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:09.029 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:09.086 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:09.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:09.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:09.151 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:56:09.151 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T05:56:09.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T05:56:09.152 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T05:56:09.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.152 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:45172 6.c 6:37ba6ed9:::benchmark_data_smithi064_284193_object45171:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T05:56:09.152 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:13.795507+0000", 2023-10-09T05:56:09.152 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 295.35209068500001, 2023-10-09T05:56:09.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.335230044, 2023-10-09T05:56:09.152 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:56:09.152 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:09.153 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:56:09.153 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:56:09.153 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:09.153 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 45172 2023-10-09T05:56:09.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.153 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:56:09.153 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.153 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:56:09.153 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.795507+0000", 2023-10-09T05:56:09.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:56:09.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.795507+0000", 2023-10-09T05:56:09.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.154 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:56:09.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.795506+0000", 2023-10-09T05:56:09.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:09.154 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:56:09.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.795528+0000", 2023-10-09T05:56:09.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2045999999999999e-05 2023-10-09T05:56:09.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:56:09.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.795540+0000", 2023-10-09T05:56:09.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2024e-05 2023-10-09T05:56:09.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:56:09.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.795548+0000", 2023-10-09T05:56:09.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0490000000000004e-06 2023-10-09T05:56:09.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.795596+0000", 2023-10-09T05:56:09.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8189000000000002e-05 2023-10-09T05:56:09.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:56:09.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.795754+0000", 2023-10-09T05:56:09.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000157493 2023-10-09T05:56:09.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.157 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.157 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-10-09T05:56:09.157 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.795873+0000", 2023-10-09T05:56:09.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118888 2023-10-09T05:56:09.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.157 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.157 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:56:09.157 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.797665+0000", 2023-10-09T05:56:09.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001792089 2023-10-09T05:56:09.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.158 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:13.801374+0000", 2023-10-09T05:56:09.158 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037091530000000002 2023-10-09T05:56:09.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.158 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:14.130645+0000", 2023-10-09T05:56:09.158 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32927110999999998 2023-10-09T05:56:09.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.159 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:56:09.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:14.130692+0000", 2023-10-09T05:56:09.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7392000000000003e-05 2023-10-09T05:56:09.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.159 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.159 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:56:09.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:14.130737+0000", 2023-10-09T05:56:09.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4988999999999999e-05 2023-10-09T05:56:09.159 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.159 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:56:09.159 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.160 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:64030 6.0 6:0d22a79d:::benchmark_data_smithi064_284193_object64029:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:56:09.160 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:33.135892+0000", 2023-10-09T05:56:09.160 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 276.011706221, 2023-10-09T05:56:09.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0587179769999999, 2023-10-09T05:56:09.160 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:56:09.160 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:09.160 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:56:09.160 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:56:09.160 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:09.161 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 64030 2023-10-09T05:56:09.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.161 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:56:09.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.161 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:56:09.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.135892+0000", 2023-10-09T05:56:09.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.161 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:56:09.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.135892+0000", 2023-10-09T05:56:09.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:56:09.162 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.135890+0000", 2023-10-09T05:56:09.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:09.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:56:09.162 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.135916+0000", 2023-10-09T05:56:09.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5043999999999999e-05 2023-10-09T05:56:09.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:56:09.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.135927+0000", 2023-10-09T05:56:09.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1265e-05 2023-10-09T05:56:09.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:56:09.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.135935+0000", 2023-10-09T05:56:09.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9160000000000001e-06 2023-10-09T05:56:09.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.135981+0000", 2023-10-09T05:56:09.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6699000000000001e-05 2023-10-09T05:56:09.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:56:09.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.135992+0000", 2023-10-09T05:56:09.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0831e-05 2023-10-09T05:56:09.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.186197+0000", 2023-10-09T05:56:09.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0502045719999999 2023-10-09T05:56:09.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:56:09.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.187610+0000", 2023-10-09T05:56:09.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014131759999999999 2023-10-09T05:56:09.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2", 2023-10-09T05:56:09.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.187814+0000", 2023-10-09T05:56:09.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020371500000000001 2023-10-09T05:56:09.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:56:09.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.194387+0000", 2023-10-09T05:56:09.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065733900000000001 2023-10-09T05:56:09.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.194445+0000", 2023-10-09T05:56:09.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7760000000000003e-05 2023-10-09T05:56:09.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:56:09.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.194507+0000", 2023-10-09T05:56:09.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2569000000000004e-05 2023-10-09T05:56:09.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.167 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:56:09.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.194610+0000", 2023-10-09T05:56:09.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000102435 2023-10-09T05:56:09.167 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.167 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:56:09.167 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.168 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:64031 6.0 6:07291387:::benchmark_data_smithi064_284193_object64030:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:56:09.168 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:33.136819+0000", 2023-10-09T05:56:09.168 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 276.01077932700002, 2023-10-09T05:56:09.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0605182040000001, 2023-10-09T05:56:09.168 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:56:09.168 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:09.168 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:56:09.168 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:56:09.168 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:09.168 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 64031 2023-10-09T05:56:09.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.168 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:56:09.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:56:09.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.136819+0000", 2023-10-09T05:56:09.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:56:09.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.136819+0000", 2023-10-09T05:56:09.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:56:09.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.136818+0000", 2023-10-09T05:56:09.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T05:56:09.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:56:09.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.136841+0000", 2023-10-09T05:56:09.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2728999999999999e-05 2023-10-09T05:56:09.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:56:09.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.136851+0000", 2023-10-09T05:56:09.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0661e-05 2023-10-09T05:56:09.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:56:09.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.136858+0000", 2023-10-09T05:56:09.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9190000000000002e-06 2023-10-09T05:56:09.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.136892+0000", 2023-10-09T05:56:09.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3714999999999998e-05 2023-10-09T05:56:09.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:56:09.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.136897+0000", 2023-10-09T05:56:09.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2510000000000002e-06 2023-10-09T05:56:09.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.190049+0000", 2023-10-09T05:56:09.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.053151634 2023-10-09T05:56:09.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:56:09.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.190179+0000", 2023-10-09T05:56:09.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012988799999999999 2023-10-09T05:56:09.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2", 2023-10-09T05:56:09.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.190339+0000", 2023-10-09T05:56:09.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016041800000000001 2023-10-09T05:56:09.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:56:09.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.194632+0000", 2023-10-09T05:56:09.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042929709999999996 2023-10-09T05:56:09.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.197207+0000", 2023-10-09T05:56:09.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025747869999999998 2023-10-09T05:56:09.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:56:09.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.197260+0000", 2023-10-09T05:56:09.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2716999999999997e-05 2023-10-09T05:56:09.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:56:09.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.197337+0000", 2023-10-09T05:56:09.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7325999999999998e-05 2023-10-09T05:56:09.175 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.175 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:56:09.175 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.175 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:64054 6.0 6:0df51b64:::benchmark_data_smithi064_284193_object64053:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:56:09.175 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:33.161336+0000", 2023-10-09T05:56:09.175 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 275.98626212599999, 2023-10-09T05:56:09.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0361608, 2023-10-09T05:56:09.176 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:56:09.176 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:09.176 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:56:09.176 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:56:09.176 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:09.176 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 64054 2023-10-09T05:56:09.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.176 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:56:09.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:56:09.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.161336+0000", 2023-10-09T05:56:09.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:56:09.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.161336+0000", 2023-10-09T05:56:09.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:56:09.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.161335+0000", 2023-10-09T05:56:09.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:09.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:56:09.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.161361+0000", 2023-10-09T05:56:09.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6407999999999999e-05 2023-10-09T05:56:09.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:56:09.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.161374+0000", 2023-10-09T05:56:09.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.29e-05 2023-10-09T05:56:09.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:56:09.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.161386+0000", 2023-10-09T05:56:09.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1554e-05 2023-10-09T05:56:09.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.161445+0000", 2023-10-09T05:56:09.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9639000000000003e-05 2023-10-09T05:56:09.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:56:09.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.161452+0000", 2023-10-09T05:56:09.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3559999999999998e-06 2023-10-09T05:56:09.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.192449+0000", 2023-10-09T05:56:09.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0309973189999999 2023-10-09T05:56:09.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:56:09.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.192604+0000", 2023-10-09T05:56:09.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015488500000000001 2023-10-09T05:56:09.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2", 2023-10-09T05:56:09.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.192762+0000", 2023-10-09T05:56:09.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015782599999999999 2023-10-09T05:56:09.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.197371+0000", 2023-10-09T05:56:09.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046097730000000002 2023-10-09T05:56:09.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:56:09.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.197383+0000", 2023-10-09T05:56:09.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1313e-05 2023-10-09T05:56:09.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:56:09.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.197420+0000", 2023-10-09T05:56:09.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6835999999999998e-05 2023-10-09T05:56:09.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:56:09.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.197497+0000", 2023-10-09T05:56:09.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7219999999999996e-05 2023-10-09T05:56:09.182 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.183 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:56:09.183 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.183 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:64063 6.0 6:09fc8313:::benchmark_data_smithi064_284193_object64062:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:56:09.183 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:33.177206+0000", 2023-10-09T05:56:09.183 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 275.97039253999998, 2023-10-09T05:56:09.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.020686282, 2023-10-09T05:56:09.183 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:56:09.183 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:09.183 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:56:09.183 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:56:09.184 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:09.184 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 64063 2023-10-09T05:56:09.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.184 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:56:09.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:56:09.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.177206+0000", 2023-10-09T05:56:09.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:56:09.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.177206+0000", 2023-10-09T05:56:09.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:56:09.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.177204+0000", 2023-10-09T05:56:09.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:09.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:56:09.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.177226+0000", 2023-10-09T05:56:09.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2039e-05 2023-10-09T05:56:09.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:56:09.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.177238+0000", 2023-10-09T05:56:09.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1878e-05 2023-10-09T05:56:09.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:56:09.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.177246+0000", 2023-10-09T05:56:09.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2870000000000004e-06 2023-10-09T05:56:09.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.177300+0000", 2023-10-09T05:56:09.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4024999999999999e-05 2023-10-09T05:56:09.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:56:09.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:33.177312+0000", 2023-10-09T05:56:09.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1642e-05 2023-10-09T05:56:09.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.194663+0000", 2023-10-09T05:56:09.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.017350899 2023-10-09T05:56:09.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:56:09.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.194827+0000", 2023-10-09T05:56:09.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016443799999999999 2023-10-09T05:56:09.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2", 2023-10-09T05:56:09.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.194996+0000", 2023-10-09T05:56:09.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016896500000000001 2023-10-09T05:56:09.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:56:09.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.197736+0000", 2023-10-09T05:56:09.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027397659999999998 2023-10-09T05:56:09.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.197759+0000", 2023-10-09T05:56:09.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3705000000000001e-05 2023-10-09T05:56:09.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:56:09.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.197814+0000", 2023-10-09T05:56:09.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4209999999999998e-05 2023-10-09T05:56:09.190 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:56:09.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:34.197892+0000", 2023-10-09T05:56:09.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8141999999999998e-05 2023-10-09T05:56:09.190 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.190 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:56:09.190 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.190 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.191 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:77499 6.c 6:3ecfb377:::benchmark_data_smithi064_284193_object77498:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:56:09.191 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:49.594240+0000", 2023-10-09T05:56:09.191 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 259.55335820300002, 2023-10-09T05:56:09.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.37754989799999999, 2023-10-09T05:56:09.191 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:56:09.191 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:09.191 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:56:09.191 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:56:09.191 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:09.191 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 77499 2023-10-09T05:56:09.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.192 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:56:09.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:56:09.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.594240+0000", 2023-10-09T05:56:09.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:56:09.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.594240+0000", 2023-10-09T05:56:09.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:56:09.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.594238+0000", 2023-10-09T05:56:09.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:09.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:56:09.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.594269+0000", 2023-10-09T05:56:09.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1443999999999999e-05 2023-10-09T05:56:09.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:56:09.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.594289+0000", 2023-10-09T05:56:09.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9349000000000001e-05 2023-10-09T05:56:09.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:56:09.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.594298+0000", 2023-10-09T05:56:09.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1260000000000004e-06 2023-10-09T05:56:09.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.601431+0000", 2023-10-09T05:56:09.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0071332009999999996 2023-10-09T05:56:09.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:56:09.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.601597+0000", 2023-10-09T05:56:09.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016616499999999999 2023-10-09T05:56:09.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-10-09T05:56:09.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.601767+0000", 2023-10-09T05:56:09.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000169669 2023-10-09T05:56:09.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.604916+0000", 2023-10-09T05:56:09.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003148788 2023-10-09T05:56:09.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:56:09.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.605659+0000", 2023-10-09T05:56:09.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00074344600000000004 2023-10-09T05:56:09.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.971732+0000", 2023-10-09T05:56:09.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.36607279399999998 2023-10-09T05:56:09.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:56:09.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.971764+0000", 2023-10-09T05:56:09.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2280000000000003e-05 2023-10-09T05:56:09.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:56:09.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.971790+0000", 2023-10-09T05:56:09.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5548e-05 2023-10-09T05:56:09.197 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.197 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:56:09.198 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.198 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:77501 6.b 6:d90af487:::benchmark_data_smithi064_284193_object77500:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:56:09.198 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:49.598040+0000", 2023-10-09T05:56:09.198 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 259.54955814300001, 2023-10-09T05:56:09.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.379657244, 2023-10-09T05:56:09.198 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:56:09.198 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:09.198 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:56:09.198 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:56:09.198 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:09.199 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 77501 2023-10-09T05:56:09.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.199 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:56:09.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:56:09.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.598040+0000", 2023-10-09T05:56:09.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:56:09.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.598040+0000", 2023-10-09T05:56:09.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:56:09.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.598039+0000", 2023-10-09T05:56:09.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:09.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:56:09.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.598071+0000", 2023-10-09T05:56:09.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2203000000000003e-05 2023-10-09T05:56:09.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:56:09.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.598414+0000", 2023-10-09T05:56:09.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00034331599999999998 2023-10-09T05:56:09.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:56:09.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.598426+0000", 2023-10-09T05:56:09.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1833e-05 2023-10-09T05:56:09.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.598979+0000", 2023-10-09T05:56:09.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00055277400000000002 2023-10-09T05:56:09.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:56:09.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.599160+0000", 2023-10-09T05:56:09.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018157000000000001 2023-10-09T05:56:09.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:56:09.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.599360+0000", 2023-10-09T05:56:09.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000199354 2023-10-09T05:56:09.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.603722+0000", 2023-10-09T05:56:09.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043628169999999997 2023-10-09T05:56:09.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:56:09.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.605652+0000", 2023-10-09T05:56:09.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019296160000000001 2023-10-09T05:56:09.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.976141+0000", 2023-10-09T05:56:09.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.370489295 2023-10-09T05:56:09.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:56:09.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.976195+0000", 2023-10-09T05:56:09.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3205999999999998e-05 2023-10-09T05:56:09.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:56:09.205 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.977697+0000", 2023-10-09T05:56:09.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0015026110000000001 2023-10-09T05:56:09.205 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.205 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:56:09.205 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.205 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:77508 6.c 6:3b622b4a:::benchmark_data_smithi064_284193_object77507:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:56:09.205 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:49.607174+0000", 2023-10-09T05:56:09.205 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 259.54042370799999, 2023-10-09T05:56:09.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.36476230199999998, 2023-10-09T05:56:09.205 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:56:09.206 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:09.206 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:56:09.206 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:56:09.206 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:09.206 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 77508 2023-10-09T05:56:09.206 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.206 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:56:09.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.206 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:56:09.206 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.607174+0000", 2023-10-09T05:56:09.206 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.207 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:56:09.207 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.607174+0000", 2023-10-09T05:56:09.207 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.207 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:56:09.207 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.607173+0000", 2023-10-09T05:56:09.207 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:09.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:56:09.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.607197+0000", 2023-10-09T05:56:09.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4363999999999999e-05 2023-10-09T05:56:09.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:56:09.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.607208+0000", 2023-10-09T05:56:09.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0719e-05 2023-10-09T05:56:09.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:56:09.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.607218+0000", 2023-10-09T05:56:09.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7499999999999998e-06 2023-10-09T05:56:09.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.607262+0000", 2023-10-09T05:56:09.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4322e-05 2023-10-09T05:56:09.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:56:09.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.607422+0000", 2023-10-09T05:56:09.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015961 2023-10-09T05:56:09.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.210 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-10-09T05:56:09.210 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.607580+0000", 2023-10-09T05:56:09.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000157854 2023-10-09T05:56:09.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.210 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.210 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.610953+0000", 2023-10-09T05:56:09.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033735169999999999 2023-10-09T05:56:09.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.211 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:56:09.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.618419+0000", 2023-10-09T05:56:09.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0074658119999999996 2023-10-09T05:56:09.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.211 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.971846+0000", 2023-10-09T05:56:09.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.353427292 2023-10-09T05:56:09.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.211 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:56:09.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.971899+0000", 2023-10-09T05:56:09.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3050000000000002e-05 2023-10-09T05:56:09.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:56:09.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.971937+0000", 2023-10-09T05:56:09.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7245e-05 2023-10-09T05:56:09.212 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.212 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:56:09.212 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.212 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:77522 6.b 6:d92534d2:::benchmark_data_smithi064_284193_object77521:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:56:09.213 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:49.627690+0000", 2023-10-09T05:56:09.213 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 259.51990792399999, 2023-10-09T05:56:09.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.35095207299999998, 2023-10-09T05:56:09.213 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:56:09.213 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:09.213 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:56:09.213 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:56:09.213 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:09.213 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 77522 2023-10-09T05:56:09.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.213 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:56:09.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.214 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:56:09.214 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.627690+0000", 2023-10-09T05:56:09.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.214 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:56:09.214 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.627690+0000", 2023-10-09T05:56:09.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:56:09.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.627688+0000", 2023-10-09T05:56:09.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:56:09.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:56:09.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.627721+0000", 2023-10-09T05:56:09.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2582000000000001e-05 2023-10-09T05:56:09.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:56:09.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.627736+0000", 2023-10-09T05:56:09.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5034e-05 2023-10-09T05:56:09.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:56:09.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.627747+0000", 2023-10-09T05:56:09.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1041e-05 2023-10-09T05:56:09.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.627812+0000", 2023-10-09T05:56:09.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4999999999999994e-05 2023-10-09T05:56:09.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:56:09.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.627971+0000", 2023-10-09T05:56:09.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000158875 2023-10-09T05:56:09.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:56:09.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.628147+0000", 2023-10-09T05:56:09.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017625799999999999 2023-10-09T05:56:09.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:56:09.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.631178+0000", 2023-10-09T05:56:09.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030313580000000001 2023-10-09T05:56:09.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.218 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.218 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.631414+0000", 2023-10-09T05:56:09.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00023603400000000001 2023-10-09T05:56:09.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.218 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.218 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.977749+0000", 2023-10-09T05:56:09.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34633501900000002 2023-10-09T05:56:09.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:56:09.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.977784+0000", 2023-10-09T05:56:09.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5160000000000002e-05 2023-10-09T05:56:09.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:56:09.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.978642+0000", 2023-10-09T05:56:09.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00085788900000000003 2023-10-09T05:56:09.219 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.219 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:56:09.220 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.220 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:77527 6.c 6:3117254c:::benchmark_data_smithi064_284193_object77526:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:56:09.220 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:49.633414+0000", 2023-10-09T05:56:09.220 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 259.51418391599998, 2023-10-09T05:56:09.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.338660931, 2023-10-09T05:56:09.220 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:56:09.220 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:09.220 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:56:09.220 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:56:09.220 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:09.221 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 77527 2023-10-09T05:56:09.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.221 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:56:09.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:56:09.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.633414+0000", 2023-10-09T05:56:09.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:56:09.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.633414+0000", 2023-10-09T05:56:09.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:56:09.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.633413+0000", 2023-10-09T05:56:09.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:09.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:56:09.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.633438+0000", 2023-10-09T05:56:09.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5062000000000002e-05 2023-10-09T05:56:09.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:56:09.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.633453+0000", 2023-10-09T05:56:09.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.571e-05 2023-10-09T05:56:09.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:56:09.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.633462+0000", 2023-10-09T05:56:09.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1750000000000005e-06 2023-10-09T05:56:09.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.635860+0000", 2023-10-09T05:56:09.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002398865 2023-10-09T05:56:09.224 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:56:09.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.636013+0000", 2023-10-09T05:56:09.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015303699999999999 2023-10-09T05:56:09.224 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-10-09T05:56:09.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.636166+0000", 2023-10-09T05:56:09.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015298800000000001 2023-10-09T05:56:09.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.225 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.225 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.638801+0000", 2023-10-09T05:56:09.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026347440000000001 2023-10-09T05:56:09.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.225 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:56:09.225 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.640787+0000", 2023-10-09T05:56:09.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019853940000000001 2023-10-09T05:56:09.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.971985+0000", 2023-10-09T05:56:09.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33119807099999998 2023-10-09T05:56:09.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:56:09.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.972032+0000", 2023-10-09T05:56:09.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7049999999999998e-05 2023-10-09T05:56:09.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:56:09.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:49.972075+0000", 2023-10-09T05:56:09.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3424999999999998e-05 2023-10-09T05:56:09.227 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.227 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:56:09.227 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.227 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:90988 6.b 6:dbb9d063:::benchmark_data_smithi064_284193_object90987:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-10-09T05:56:09.227 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:52:10.161611+0000", 2023-10-09T05:56:09.227 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 238.98598683, 2023-10-09T05:56:09.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.332104728, 2023-10-09T05:56:09.227 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:56:09.227 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:56:09.228 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:56:09.228 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:56:09.228 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:09.228 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 90988 2023-10-09T05:56:09.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.228 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:56:09.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:56:09.228 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.161611+0000", 2023-10-09T05:56:09.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.229 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:56:09.229 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.161611+0000", 2023-10-09T05:56:09.229 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.229 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.229 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.229 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:56:09.229 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.161601+0000", 2023-10-09T05:56:09.229 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.99999 2023-10-09T05:56:09.229 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.229 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.229 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:56:09.229 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.161649+0000", 2023-10-09T05:56:09.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7815999999999998e-05 2023-10-09T05:56:09.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:56:09.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.161666+0000", 2023-10-09T05:56:09.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7184999999999999e-05 2023-10-09T05:56:09.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:56:09.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.161675+0000", 2023-10-09T05:56:09.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0929999999999992e-06 2023-10-09T05:56:09.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.161767+0000", 2023-10-09T05:56:09.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1494000000000002e-05 2023-10-09T05:56:09.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:56:09.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.161925+0000", 2023-10-09T05:56:09.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000157765 2023-10-09T05:56:09.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:56:09.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.162131+0000", 2023-10-09T05:56:09.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020610600000000001 2023-10-09T05:56:09.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.232 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.232 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.167481+0000", 2023-10-09T05:56:09.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053501709999999999 2023-10-09T05:56:09.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.232 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.232 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.176656+0000", 2023-10-09T05:56:09.233 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.009174771 2023-10-09T05:56:09.233 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.233 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.233 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.196258+0000", 2023-10-09T05:56:09.233 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.019602624999999999 2023-10-09T05:56:09.233 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.233 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:56:09.233 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.196272+0000", 2023-10-09T05:56:09.233 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3740999999999999e-05 2023-10-09T05:56:09.233 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.234 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.234 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:11.493586+0000", 2023-10-09T05:56:09.234 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.297313919 2023-10-09T05:56:09.234 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.234 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:56:09.234 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:11.493716+0000", 2023-10-09T05:56:09.234 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000130092 2023-10-09T05:56:09.234 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.234 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:56:09.234 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.235 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:90996 6.b 6:de46b7c1:::benchmark_data_smithi064_284193_object90995:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-10-09T05:56:09.235 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:52:10.172459+0000", 2023-10-09T05:56:09.235 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 238.97513894599999, 2023-10-09T05:56:09.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3230595890000001, 2023-10-09T05:56:09.235 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:56:09.235 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:56:09.235 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:56:09.235 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:56:09.235 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:09.235 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 90996 2023-10-09T05:56:09.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.236 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:56:09.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:56:09.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.172459+0000", 2023-10-09T05:56:09.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:56:09.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.172459+0000", 2023-10-09T05:56:09.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.237 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:56:09.237 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.172457+0000", 2023-10-09T05:56:09.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:09.237 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.237 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.237 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:56:09.237 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.172493+0000", 2023-10-09T05:56:09.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5252000000000001e-05 2023-10-09T05:56:09.237 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.237 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.237 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:56:09.237 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.172514+0000", 2023-10-09T05:56:09.238 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1152e-05 2023-10-09T05:56:09.238 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.238 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.238 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:56:09.238 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.172524+0000", 2023-10-09T05:56:09.238 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0081000000000001e-05 2023-10-09T05:56:09.238 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.238 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.238 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.238 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.176684+0000", 2023-10-09T05:56:09.238 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.004160558 2023-10-09T05:56:09.238 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.239 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.239 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:56:09.239 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.176833+0000", 2023-10-09T05:56:09.239 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014817300000000001 2023-10-09T05:56:09.239 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.239 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.239 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:56:09.239 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.177013+0000", 2023-10-09T05:56:09.239 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018032500000000001 2023-10-09T05:56:09.239 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.239 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.239 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.239 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.196315+0000", 2023-10-09T05:56:09.240 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.019301670999999999 2023-10-09T05:56:09.240 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.240 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.240 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:56:09.240 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.196340+0000", 2023-10-09T05:56:09.240 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4972000000000001e-05 2023-10-09T05:56:09.240 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.240 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.240 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.240 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:11.495420+0000", 2023-10-09T05:56:09.240 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2990805029999999 2023-10-09T05:56:09.240 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.241 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.241 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:56:09.241 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:11.495519+0000", 2023-10-09T05:56:09.241 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8661999999999998e-05 2023-10-09T05:56:09.241 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.241 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:56:09.241 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.241 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.241 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.241 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:91003 6.b 6:d92b5247:::benchmark_data_smithi064_284193_object91002:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-10-09T05:56:09.241 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:52:10.176388+0000", 2023-10-09T05:56:09.241 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 238.97120996000001, 2023-10-09T05:56:09.242 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.31928738, 2023-10-09T05:56:09.242 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:56:09.242 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:56:09.242 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:56:09.242 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:56:09.242 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:09.242 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 91003 2023-10-09T05:56:09.242 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.242 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:56:09.242 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.242 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:56:09.242 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.176388+0000", 2023-10-09T05:56:09.243 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.243 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.243 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.243 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:56:09.243 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.176388+0000", 2023-10-09T05:56:09.243 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.243 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.243 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.243 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:56:09.243 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.176386+0000", 2023-10-09T05:56:09.243 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:09.243 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.244 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.244 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:56:09.244 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.176415+0000", 2023-10-09T05:56:09.244 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9098000000000002e-05 2023-10-09T05:56:09.244 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.244 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.244 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:56:09.244 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.176427+0000", 2023-10-09T05:56:09.244 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1411e-05 2023-10-09T05:56:09.244 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.244 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.244 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:56:09.244 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.176437+0000", 2023-10-09T05:56:09.245 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.006e-05 2023-10-09T05:56:09.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.245 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.245 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.245 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.179976+0000", 2023-10-09T05:56:09.245 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035393439999999998 2023-10-09T05:56:09.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.245 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.245 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:56:09.245 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.180171+0000", 2023-10-09T05:56:09.245 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019471000000000001 2023-10-09T05:56:09.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.246 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:56:09.246 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.180349+0000", 2023-10-09T05:56:09.246 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017780199999999999 2023-10-09T05:56:09.246 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.246 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.246 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.196384+0000", 2023-10-09T05:56:09.246 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016035038000000001 2023-10-09T05:56:09.246 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.246 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:56:09.247 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.196395+0000", 2023-10-09T05:56:09.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1659e-05 2023-10-09T05:56:09.247 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.247 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.247 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.247 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:11.495569+0000", 2023-10-09T05:56:09.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.299173242 2023-10-09T05:56:09.247 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.247 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.247 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:56:09.247 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:11.495675+0000", 2023-10-09T05:56:09.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010683600000000001 2023-10-09T05:56:09.248 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.248 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:56:09.248 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.248 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.248 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.248 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:91009 6.b 6:d09c3914:::benchmark_data_smithi064_284193_object91008:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-10-09T05:56:09.248 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:52:10.196442+0000", 2023-10-09T05:56:09.248 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 238.951155911, 2023-10-09T05:56:09.248 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3118305109999999, 2023-10-09T05:56:09.248 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:56:09.248 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:09.248 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:56:09.248 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:56:09.249 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:09.249 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 91009 2023-10-09T05:56:09.249 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.249 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:56:09.249 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.249 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:56:09.249 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.196442+0000", 2023-10-09T05:56:09.249 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.249 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.249 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.249 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:56:09.249 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.196442+0000", 2023-10-09T05:56:09.250 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.250 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.250 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.250 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:56:09.250 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.196440+0000", 2023-10-09T05:56:09.250 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:09.250 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.250 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.250 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:56:09.250 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.196465+0000", 2023-10-09T05:56:09.250 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4545999999999998e-05 2023-10-09T05:56:09.250 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.251 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.251 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:56:09.251 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.196475+0000", 2023-10-09T05:56:09.251 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0681e-05 2023-10-09T05:56:09.251 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.251 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.251 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:56:09.251 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.196485+0000", 2023-10-09T05:56:09.251 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.132e-06 2023-10-09T05:56:09.251 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.251 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.251 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.252 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.196563+0000", 2023-10-09T05:56:09.252 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8715999999999996e-05 2023-10-09T05:56:09.252 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.252 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.252 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:56:09.252 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.196573+0000", 2023-10-09T05:56:09.252 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8090000000000004e-06 2023-10-09T05:56:09.252 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.252 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.252 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.252 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:11.495708+0000", 2023-10-09T05:56:09.252 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.29913441 2023-10-09T05:56:09.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.253 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:56:09.253 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:11.495901+0000", 2023-10-09T05:56:09.253 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000193599 2023-10-09T05:56:09.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.253 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:56:09.253 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:11.496122+0000", 2023-10-09T05:56:09.253 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022035699999999999 2023-10-09T05:56:09.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:11.506098+0000", 2023-10-09T05:56:09.254 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0099768240000000005 2023-10-09T05:56:09.254 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.254 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:56:09.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:11.506109+0000", 2023-10-09T05:56:09.254 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0454e-05 2023-10-09T05:56:09.254 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.254 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:11.506159+0000", 2023-10-09T05:56:09.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9823000000000001e-05 2023-10-09T05:56:09.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.255 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:56:09.255 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:11.506229+0000", 2023-10-09T05:56:09.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9968000000000005e-05 2023-10-09T05:56:09.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.255 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:56:09.255 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:11.508273+0000", 2023-10-09T05:56:09.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020440860000000001 2023-10-09T05:56:09.255 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.256 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:56:09.256 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.256 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.256 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.256 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:91011 6.b 6:d611b469:::benchmark_data_smithi064_284193_object91010:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-10-09T05:56:09.256 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:52:10.201113+0000", 2023-10-09T05:56:09.256 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 238.94648461400001, 2023-10-09T05:56:09.256 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.312609795, 2023-10-09T05:56:09.256 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:56:09.256 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:09.256 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:56:09.256 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:56:09.257 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:09.257 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 91011 2023-10-09T05:56:09.257 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.257 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:56:09.257 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.257 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:56:09.257 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.201113+0000", 2023-10-09T05:56:09.257 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.257 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.257 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.257 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:56:09.257 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.201113+0000", 2023-10-09T05:56:09.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.258 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.258 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.258 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:56:09.258 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.201112+0000", 2023-10-09T05:56:09.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:09.258 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.258 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.258 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:56:09.258 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.201138+0000", 2023-10-09T05:56:09.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6325999999999998e-05 2023-10-09T05:56:09.258 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.259 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.259 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:56:09.259 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.201149+0000", 2023-10-09T05:56:09.259 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1249e-05 2023-10-09T05:56:09.259 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.259 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.259 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:56:09.259 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.201157+0000", 2023-10-09T05:56:09.259 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5750000000000003e-06 2023-10-09T05:56:09.259 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.259 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.259 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.260 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.201579+0000", 2023-10-09T05:56:09.260 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00042194800000000001 2023-10-09T05:56:09.260 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.260 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.260 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:56:09.260 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:10.201591+0000", 2023-10-09T05:56:09.260 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2513e-05 2023-10-09T05:56:09.260 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.260 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.260 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.260 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:11.499344+0000", 2023-10-09T05:56:09.260 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2977521519999999 2023-10-09T05:56:09.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.261 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.261 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:56:09.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:11.499529+0000", 2023-10-09T05:56:09.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018526 2023-10-09T05:56:09.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.261 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.261 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:56:09.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:11.499734+0000", 2023-10-09T05:56:09.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000205062 2023-10-09T05:56:09.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.261 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.261 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:56:09.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:11.506115+0000", 2023-10-09T05:56:09.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063815180000000001 2023-10-09T05:56:09.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:11.511838+0000", 2023-10-09T05:56:09.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0057223049999999996 2023-10-09T05:56:09.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:11.511883+0000", 2023-10-09T05:56:09.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5178e-05 2023-10-09T05:56:09.263 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.263 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.263 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:56:09.263 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:11.511948+0000", 2023-10-09T05:56:09.263 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4758000000000006e-05 2023-10-09T05:56:09.263 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.263 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.263 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:56:09.263 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:11.513723+0000", 2023-10-09T05:56:09.263 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001775609 2023-10-09T05:56:09.263 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.264 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:56:09.264 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.264 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.264 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.264 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:110004 6.b 6:d75a3c6f:::benchmark_data_smithi064_284193_object13461:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2023-10-09T05:56:09.264 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:22.320520+0000", 2023-10-09T05:56:09.264 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 46.827078434999997, 2023-10-09T05:56:09.264 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96501756000000005, 2023-10-09T05:56:09.264 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:56:09.264 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:09.264 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:56:09.264 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:56:09.265 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:09.265 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 110004 2023-10-09T05:56:09.265 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.265 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:56:09.265 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.265 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:56:09.265 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:22.320520+0000", 2023-10-09T05:56:09.265 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.265 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.265 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.265 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:56:09.265 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:22.320520+0000", 2023-10-09T05:56:09.265 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.266 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.266 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.266 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:56:09.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:22.320518+0000", 2023-10-09T05:56:09.266 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:09.266 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.266 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.266 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:56:09.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:22.320526+0000", 2023-10-09T05:56:09.266 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9310000000000008e-06 2023-10-09T05:56:09.266 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.266 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:56:09.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:22.320540+0000", 2023-10-09T05:56:09.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4163e-05 2023-10-09T05:56:09.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:56:09.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:22.320550+0000", 2023-10-09T05:56:09.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0257e-05 2023-10-09T05:56:09.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:22.328661+0000", 2023-10-09T05:56:09.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0081107620000000005 2023-10-09T05:56:09.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.268 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.268 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:56:09.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:22.328673+0000", 2023-10-09T05:56:09.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1865000000000001e-05 2023-10-09T05:56:09.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.268 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.268 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.259676+0000", 2023-10-09T05:56:09.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93100316100000002 2023-10-09T05:56:09.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:56:09.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.260280+0000", 2023-10-09T05:56:09.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00060370699999999998 2023-10-09T05:56:09.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.270474+0000", 2023-10-09T05:56:09.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010194679 2023-10-09T05:56:09.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:56:09.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.270573+0000", 2023-10-09T05:56:09.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8684999999999995e-05 2023-10-09T05:56:09.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:56:09.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.270662+0000", 2023-10-09T05:56:09.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8628000000000006e-05 2023-10-09T05:56:09.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.278896+0000", 2023-10-09T05:56:09.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0082344270000000008 2023-10-09T05:56:09.271 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.271 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:56:09.271 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.278905+0000", 2023-10-09T05:56:09.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2539999999999998e-06 2023-10-09T05:56:09.271 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.271 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.271 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.283970+0000", 2023-10-09T05:56:09.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.005064916 2023-10-09T05:56:09.271 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.272 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:56:09.272 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.284030+0000", 2023-10-09T05:56:09.272 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9633999999999999e-05 2023-10-09T05:56:09.272 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.272 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.272 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:56:09.272 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.285537+0000", 2023-10-09T05:56:09.272 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00150719 2023-10-09T05:56:09.272 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.272 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:56:09.272 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.272 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.273 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.273 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:110016 6.b 6:d2b93dfe:::benchmark_data_smithi064_284193_object13473:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2023-10-09T05:56:09.273 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:22.364603+0000", 2023-10-09T05:56:09.273 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 46.782994604000002, 2023-10-09T05:56:09.273 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.92251474099999997, 2023-10-09T05:56:09.273 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:56:09.273 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:09.273 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:56:09.273 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:56:09.273 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:09.273 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 110016 2023-10-09T05:56:09.273 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.274 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:56:09.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.274 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:56:09.274 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:22.364603+0000", 2023-10-09T05:56:09.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.274 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:56:09.274 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:22.364603+0000", 2023-10-09T05:56:09.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:56:09.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:22.364601+0000", 2023-10-09T05:56:09.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:09.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:56:09.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:22.364610+0000", 2023-10-09T05:56:09.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3240000000000003e-06 2023-10-09T05:56:09.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:56:09.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:22.364633+0000", 2023-10-09T05:56:09.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2901e-05 2023-10-09T05:56:09.276 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:56:09.276 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:22.364642+0000", 2023-10-09T05:56:09.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6490000000000003e-06 2023-10-09T05:56:09.276 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.276 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:22.364700+0000", 2023-10-09T05:56:09.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7719000000000001e-05 2023-10-09T05:56:09.276 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.277 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:56:09.277 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:22.364710+0000", 2023-10-09T05:56:09.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0063e-05 2023-10-09T05:56:09.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.277 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.277 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.260328+0000", 2023-10-09T05:56:09.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.89561813199999996 2023-10-09T05:56:09.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.277 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:56:09.278 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.260879+0000", 2023-10-09T05:56:09.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00055087599999999995 2023-10-09T05:56:09.278 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.278 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.278 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.276280+0000", 2023-10-09T05:56:09.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015400545999999999 2023-10-09T05:56:09.278 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.278 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:56:09.278 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.276405+0000", 2023-10-09T05:56:09.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012533800000000001 2023-10-09T05:56:09.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.281 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:56:09.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.276575+0000", 2023-10-09T05:56:09.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016984699999999999 2023-10-09T05:56:09.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.283919+0000", 2023-10-09T05:56:09.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0073436839999999996 2023-10-09T05:56:09.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.285607+0000", 2023-10-09T05:56:09.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001688925 2023-10-09T05:56:09.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:56:09.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.285635+0000", 2023-10-09T05:56:09.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7542000000000002e-05 2023-10-09T05:56:09.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:56:09.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.285694+0000", 2023-10-09T05:56:09.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9333999999999998e-05 2023-10-09T05:56:09.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:56:09.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.287118+0000", 2023-10-09T05:56:09.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001423835 2023-10-09T05:56:09.284 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.284 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:56:09.284 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.284 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:110020 6.b 6:d74e637e:::benchmark_data_smithi064_284193_object13477:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2023-10-09T05:56:09.284 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:22.423948+0000", 2023-10-09T05:56:09.284 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 46.723650556000003, 2023-10-09T05:56:09.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.86494267899999999, 2023-10-09T05:56:09.284 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:56:09.284 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:09.285 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:56:09.285 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:56:09.285 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:09.285 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 110020 2023-10-09T05:56:09.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.285 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:56:09.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:56:09.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:22.423948+0000", 2023-10-09T05:56:09.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:56:09.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:22.423948+0000", 2023-10-09T05:56:09.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:56:09.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:22.423946+0000", 2023-10-09T05:56:09.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:09.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:56:09.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:22.423952+0000", 2023-10-09T05:56:09.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1009999999999997e-06 2023-10-09T05:56:09.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.287 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:56:09.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:22.423971+0000", 2023-10-09T05:56:09.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9130000000000001e-05 2023-10-09T05:56:09.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.287 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:56:09.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:22.423982+0000", 2023-10-09T05:56:09.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0261999999999999e-05 2023-10-09T05:56:09.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.288 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.288 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.288 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:22.424042+0000", 2023-10-09T05:56:09.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0446999999999999e-05 2023-10-09T05:56:09.288 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.288 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.288 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:56:09.288 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:22.424052+0000", 2023-10-09T05:56:09.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7610000000000002e-06 2023-10-09T05:56:09.288 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.288 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.288 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.260990+0000", 2023-10-09T05:56:09.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.83693806500000001 2023-10-09T05:56:09.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:56:09.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.261541+0000", 2023-10-09T05:56:09.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00055057999999999995 2023-10-09T05:56:09.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.282005+0000", 2023-10-09T05:56:09.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020464110000000001 2023-10-09T05:56:09.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:56:09.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.282122+0000", 2023-10-09T05:56:09.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000117145 2023-10-09T05:56:09.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:56:09.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.282287+0000", 2023-10-09T05:56:09.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016477899999999999 2023-10-09T05:56:09.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.291 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.287176+0000", 2023-10-09T05:56:09.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048898689999999998 2023-10-09T05:56:09.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.291 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.287244+0000", 2023-10-09T05:56:09.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7080000000000001e-05 2023-10-09T05:56:09.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.291 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:56:09.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.287253+0000", 2023-10-09T05:56:09.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1689999999999998e-06 2023-10-09T05:56:09.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:56:09.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.287308+0000", 2023-10-09T05:56:09.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5420000000000001e-05 2023-10-09T05:56:09.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:56:09.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:23.288890+0000", 2023-10-09T05:56:09.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001582062 2023-10-09T05:56:09.292 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.292 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:56:09.293 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.293 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119191 6.b 6:d6f68c17:::benchmark_data_smithi064_284193_object22648:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-10-09T05:56:09.293 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:43.834077+0000", 2023-10-09T05:56:09.293 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 25.313520782000001, 2023-10-09T05:56:09.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9884594920000001, 2023-10-09T05:56:09.293 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:56:09.293 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:09.293 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:56:09.293 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:56:09.293 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:09.294 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119191 2023-10-09T05:56:09.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.294 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:56:09.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:56:09.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:43.834077+0000", 2023-10-09T05:56:09.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:56:09.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:43.834077+0000", 2023-10-09T05:56:09.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:56:09.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:43.834075+0000", 2023-10-09T05:56:09.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:09.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:56:09.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:43.834089+0000", 2023-10-09T05:56:09.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3407999999999999e-05 2023-10-09T05:56:09.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:56:09.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:43.834113+0000", 2023-10-09T05:56:09.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4422000000000001e-05 2023-10-09T05:56:09.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:56:09.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:43.834126+0000", 2023-10-09T05:56:09.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3008e-05 2023-10-09T05:56:09.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:43.834204+0000", 2023-10-09T05:56:09.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8036999999999998e-05 2023-10-09T05:56:09.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:56:09.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:43.834220+0000", 2023-10-09T05:56:09.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6059e-05 2023-10-09T05:56:09.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.649940+0000", 2023-10-09T05:56:09.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.81571972000000004 2023-10-09T05:56:09.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:56:09.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.649949+0000", 2023-10-09T05:56:09.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1789999999999997e-06 2023-10-09T05:56:09.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.807583+0000", 2023-10-09T05:56:09.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1576343410000001 2023-10-09T05:56:09.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:56:09.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.807729+0000", 2023-10-09T05:56:09.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014557700000000001 2023-10-09T05:56:09.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:56:09.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.807856+0000", 2023-10-09T05:56:09.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012673200000000001 2023-10-09T05:56:09.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:56:09.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.811901+0000", 2023-10-09T05:56:09.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040453240000000003 2023-10-09T05:56:09.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.812688+0000", 2023-10-09T05:56:09.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000786577 2023-10-09T05:56:09.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.822291+0000", 2023-10-09T05:56:09.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0096031109999999992 2023-10-09T05:56:09.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:56:09.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.822357+0000", 2023-10-09T05:56:09.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6701999999999999e-05 2023-10-09T05:56:09.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:56:09.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.822537+0000", 2023-10-09T05:56:09.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017941699999999999 2023-10-09T05:56:09.301 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.301 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:56:09.301 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.302 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119218 6.1b 6:dee627dc:::benchmark_data_smithi064_284193_object22675:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-10-09T05:56:09.302 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:44.160416+0000", 2023-10-09T05:56:09.302 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 24.987181865, 2023-10-09T05:56:09.302 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6504987519999998, 2023-10-09T05:56:09.302 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:56:09.302 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:09.302 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:56:09.302 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:56:09.302 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:09.302 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119218 2023-10-09T05:56:09.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.303 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:56:09.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.303 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:56:09.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.160416+0000", 2023-10-09T05:56:09.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.303 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:56:09.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.160416+0000", 2023-10-09T05:56:09.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:56:09.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:56:09.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.160415+0000", 2023-10-09T05:56:09.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:09.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:56:09.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.160421+0000", 2023-10-09T05:56:09.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5880000000000003e-06 2023-10-09T05:56:09.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:56:09.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.160431+0000", 2023-10-09T05:56:09.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2939999999999994e-06 2023-10-09T05:56:09.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:56:09.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.160437+0000", 2023-10-09T05:56:09.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0610000000000001e-06 2023-10-09T05:56:09.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.160493+0000", 2023-10-09T05:56:09.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6221999999999998e-05 2023-10-09T05:56:09.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:56:09.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.160504+0000", 2023-10-09T05:56:09.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1413000000000001e-05 2023-10-09T05:56:09.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.649964+0000", 2023-10-09T05:56:09.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.48945948700000003 2023-10-09T05:56:09.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:56:09.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.649968+0000", 2023-10-09T05:56:09.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4669999999999998e-06 2023-10-09T05:56:09.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:56:09.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.795143+0000", 2023-10-09T05:56:09.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.145175058 2023-10-09T05:56:09.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:56:09.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.795289+0000", 2023-10-09T05:56:09.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000145933 2023-10-09T05:56:09.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:56:09.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.795423+0000", 2023-10-09T05:56:09.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000133263 2023-10-09T05:56:09.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.807558+0000", 2023-10-09T05:56:09.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012135826000000001 2023-10-09T05:56:09.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:09.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.810681+0000", 2023-10-09T05:56:09.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003122793 2023-10-09T05:56:09.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:56:09.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.810696+0000", 2023-10-09T05:56:09.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4643e-05 2023-10-09T05:56:09.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:56:09.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.810742+0000", 2023-10-09T05:56:09.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6051999999999999e-05 2023-10-09T05:56:09.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:56:09.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:56:09.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:56:09.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.810915+0000", 2023-10-09T05:56:09.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000173031 2023-10-09T05:56:09.310 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.310 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:56:09.310 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.310 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:56:09.311 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:56:09.311 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:56:09.311 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:09.358 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:09.366 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:09 smithi156 ceph-mon[195101]: pgmap v511: 129 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 peering, 1 active+undersized+degraded+remapped, 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering, 2 active+remapped+backfill_wait, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 57 op/s; 35559/216219 objects degraded (16.446%); 19452/216219 objects misplaced (8.996%); 38 MiB/s, 612 objects/s recovering 2023-10-09T05:56:09.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:09 smithi064 ceph-mon[212750]: pgmap v511: 129 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 peering, 1 active+undersized+degraded+remapped, 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering, 2 active+remapped+backfill_wait, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 57 op/s; 35559/216219 objects degraded (16.446%); 19452/216219 objects misplaced (8.996%); 38 MiB/s, 612 objects/s recovering 2023-10-09T05:56:09.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:09 smithi064 ceph-mon[220044]: pgmap v511: 129 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 peering, 1 active+undersized+degraded+remapped, 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering, 2 active+remapped+backfill_wait, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 57 op/s; 35559/216219 objects degraded (16.446%); 19452/216219 objects misplaced (8.996%); 38 MiB/s, 612 objects/s recovering 2023-10-09T05:56:10.505 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:56:10.530 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:10.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:10.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:10.634 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:10.673 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:10.738 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:10.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:10.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:10.826 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:10.843 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:10.869 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:56:10.916 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:11.320 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:11.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:11.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:11.397 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:56:11.425 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:11.464 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:11.473 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:11.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:11 smithi064 ceph-mon[212750]: pgmap v512: 129 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 109 active+clean, 1 active+recovering, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 392 op/s; 34975/209880 objects degraded (16.664%); 17630/209880 objects misplaced (8.400%); 55 MiB/s, 873 objects/s recovering 2023-10-09T05:56:11.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:11 smithi064 ceph-mon[220044]: pgmap v512: 129 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 109 active+clean, 1 active+recovering, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 392 op/s; 34975/209880 objects degraded (16.664%); 17630/209880 objects misplaced (8.400%); 55 MiB/s, 873 objects/s recovering 2023-10-09T05:56:11.786 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:11.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:11.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:11.905 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:11.958 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:12.023 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:11 smithi156 ceph-mon[195101]: pgmap v512: 129 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 109 active+clean, 1 active+recovering, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 392 op/s; 34975/209880 objects degraded (16.664%); 17630/209880 objects misplaced (8.400%); 55 MiB/s, 873 objects/s recovering 2023-10-09T05:56:12.042 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:56:12.045 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:12.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:12.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:12.051 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T05:56:12.151 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:12.199 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:12.260 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:12.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:12.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:12.363 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:12.411 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:12.488 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:12.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:12.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:12.596 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:12.626 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:12.692 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:12.694 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:12 smithi156 ceph-mon[195101]: from='client.45026 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:56:12.694 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:12 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 34975/209880 objects degraded (16.664%), 14 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:12.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:12.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:12.796 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:12.826 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:12.881 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:12.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:12.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:12.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:12 smithi064 ceph-mon[220044]: from='client.45026 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:56:12.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:12 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 34975/209880 objects degraded (16.664%), 14 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:12.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:12 smithi064 ceph-mon[212750]: from='client.45026 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:56:12.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:12 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 34975/209880 objects degraded (16.664%), 14 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:12.985 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:13.011 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:13.061 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:13.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:13.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:13.164 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:13.203 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:13.494 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":512,"stamp":"2023-10-09T05:56:10.557528+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4583856056,"num_objects":69960,"num_object_clones":0,"num_object_copies":209880,"num_objects_missing_on_primary":1549,"num_objects_missing":1549,"num_objects_degraded":34975,"num_objects_misplaced":17630,"num_objects_unfound":0,"num_objects_dirty":69960,"num_whiteouts":0,"num_read":488,"num_read_kb":914,"num_write":123446,"num_write_kb":6181006,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":46013,"num_bytes_recovered":2924753590,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":224471,"ondisk_log_size":224471,"up":387,"acting":379,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":418,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":15930100,"kb_used_data":14264188,"kb_used_omap":118,"kb_used_meta":1665673,"kb_avail":640285964,"statfs":{"total":671965249536,"available":655652827136,"internally_reserved":0,"allocated":14606528512,"data_stored":14599567316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":121648,"internal_metadata":1705649360},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,2,2,2,3,4,4,2],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-159973376,"num_objects":-2441,"num_object_clones":0,"num_object_copies":-7323,"num_objects_missing_on_primary":-962,"num_objects_missing":-962,"num_objects_degraded":5094,"num_objects_misplaced":8296,"num_objects_unfound":0,"num_objects_dirty":-2441,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2441,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5433,"num_bytes_recovered":355926040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.217950"},"pg_stats":[{"pgid":"6.23","version":"223'6925","reported_seq":7090,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:56:10.292963+0000","last_change":"2023-10-09T05:55:22.225278+0000","last_active":"2023-10-09T05:56:10.292963+0000","last_peered":"2023-10-09T05:56:10.292963+0000","last_clean":"2023-10-09T05:56:10.292963+0000","last_became_active":"2023-10-09T05:55:22.222511+0000","last_became_peered":"2023-10-09T05:55:22.222511+0000","last_unstale":"2023-10-09T05:56:10.292963+0000","last_undegraded":"2023-10-09T05:56:10.292963+0000","last_fullsized":"2023-10-09T05:56:10.292963+0000","mapping_epoch":207,"log_start":"107'1488","ondisk_log_start":"107'1488","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5437,"log_dups_size":0,"ondisk_log_size":5437,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72761344,"num_objects":1110,"num_object_clones":0,"num_object_copies":3330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1931,"num_write_kb":97312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.22","version":"223'6917","reported_seq":19125,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:56:09.195248+0000","last_change":"2023-10-09T05:55:22.229226+0000","last_active":"2023-10-09T05:56:09.195248+0000","last_peered":"2023-10-09T05:56:09.195248+0000","last_clean":"2023-10-09T05:56:09.195248+0000","last_became_active":"2023-10-09T05:55:22.223863+0000","last_became_peered":"2023-10-09T05:55:22.223863+0000","last_unstale":"2023-10-09T05:56:09.195248+0000","last_undegraded":"2023-10-09T05:56:09.195248+0000","last_fullsized":"2023-10-09T05:56:09.195248+0000","mapping_epoch":207,"log_start":"106'632","ondisk_log_start":"106'632","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6285,"log_dups_size":0,"ondisk_log_size":6285,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71581696,"num_objects":1092,"num_object_clones":0,"num_object_copies":3276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1092,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1917,"num_write_kb":96304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1505,"num_bytes_recovered":95617024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.21","version":"222'6973","reported_seq":7536,"reported_epoch":222,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:06.809571+0000","last_change":"2023-10-09T05:56:00.525133+0000","last_active":"2023-10-09T05:56:06.809571+0000","last_peered":"2023-10-09T05:56:06.809571+0000","last_clean":"2023-10-09T05:51:24.563560+0000","last_became_active":"2023-10-09T05:56:00.506782+0000","last_became_peered":"2023-10-09T05:56:00.506782+0000","last_unstale":"2023-10-09T05:56:06.809571+0000","last_undegraded":"2023-10-09T05:56:00.339085+0000","last_fullsized":"2023-10-09T05:56:00.284538+0000","mapping_epoch":219,"log_start":"107'871","ondisk_log_start":"107'871","created":207,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6102,"log_dups_size":0,"ondisk_log_size":6102,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72384512,"num_objects":1105,"num_object_clones":0,"num_object_copies":3315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1314,"num_objects_misplaced":578,"num_objects_unfound":0,"num_objects_dirty":1105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1949,"num_write_kb":97696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":190,"num_bytes_recovered":12386304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,1],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"1,2,6","objects":580},{"shards":"1,6","objects":-21},{"shards":"2,6,7","objects":527}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.20","version":"223'6909","reported_seq":10163,"reported_epoch":223,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:56:09.270134+0000","last_change":"2023-10-09T05:56:00.444495+0000","last_active":"2023-10-09T05:56:09.270134+0000","last_peered":"2023-10-09T05:56:09.270134+0000","last_clean":"2023-10-09T05:54:10.146196+0000","last_became_active":"2023-10-09T05:56:00.423961+0000","last_became_peered":"2023-10-09T05:56:00.423961+0000","last_unstale":"2023-10-09T05:56:09.270134+0000","last_undegraded":"2023-10-09T05:56:00.318106+0000","last_fullsized":"2023-10-09T05:56:09.270134+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6909,"log_dups_size":0,"ondisk_log_size":6909,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73809920,"num_objects":1126,"num_object_clones":0,"num_object_copies":3378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2502,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1126,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1912,"num_write_kb":97216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":571,"num_bytes_recovered":37404672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":1125},{"shards":"0,4,5","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1f","version":"223'7350","reported_seq":7926,"reported_epoch":223,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T05:56:09.380035+0000","last_change":"2023-10-09T05:55:59.464124+0000","last_active":"2023-10-09T05:56:09.380035+0000","last_peered":"2023-10-09T05:56:09.380035+0000","last_clean":"2023-10-09T05:52:22.632757+0000","last_became_active":"2023-10-09T05:55:59.409285+0000","last_became_peered":"2023-10-09T05:55:59.409285+0000","last_unstale":"2023-10-09T05:56:09.380035+0000","last_undegraded":"2023-10-09T05:55:59.398316+0000","last_fullsized":"2023-10-09T05:56:09.380035+0000","mapping_epoch":218,"log_start":"107'1991","ondisk_log_start":"107'1991","created":207,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5359,"log_dups_size":0,"ondisk_log_size":5359,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148013056,"num_objects":2259,"num_object_clones":0,"num_object_copies":6777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2259,"num_objects_misplaced":2311,"num_objects_unfound":0,"num_objects_dirty":2259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3883,"num_write_kb":196512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":6553600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,3],"avail_no_missing":["4","1","2","3"],"object_location_counts":[{"shards":"2,3,4","objects":2113},{"shards":"3,4,6","objects":146}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":268,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.363852+0000","last_change":"2023-10-09T05:55:59.341215+0000","last_active":"2023-10-09T05:56:05.363852+0000","last_peered":"2023-10-09T05:56:05.363852+0000","last_clean":"2023-10-09T05:56:05.363852+0000","last_became_active":"2023-10-09T05:55:59.340856+0000","last_became_peered":"2023-10-09T05:55:59.340856+0000","last_unstale":"2023-10-09T05:56:05.363852+0000","last_undegraded":"2023-10-09T05:56:05.363852+0000","last_fullsized":"2023-10-09T05:56:05.363852+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:33.686454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":150,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.368263+0000","last_change":"2023-10-09T05:55:59.328988+0000","last_active":"2023-10-09T05:56:05.368263+0000","last_peered":"2023-10-09T05:56:05.368263+0000","last_clean":"2023-10-09T05:56:05.368263+0000","last_became_active":"2023-10-09T05:55:59.323104+0000","last_became_peered":"2023-10-09T05:55:59.323104+0000","last_unstale":"2023-10-09T05:56:05.368263+0000","last_undegraded":"2023-10-09T05:56:05.368263+0000","last_fullsized":"2023-10-09T05:56:05.368263+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:09:14.587595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.1c","version":"222'7251","reported_seq":19497,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:06.771635+0000","last_change":"2023-10-09T05:55:22.232121+0000","last_active":"2023-10-09T05:56:06.771635+0000","last_peered":"2023-10-09T05:56:06.771635+0000","last_clean":"2023-10-09T05:56:06.771635+0000","last_became_active":"2023-10-09T05:55:22.226434+0000","last_became_peered":"2023-10-09T05:55:22.226434+0000","last_unstale":"2023-10-09T05:56:06.771635+0000","last_undegraded":"2023-10-09T05:56:06.771635+0000","last_fullsized":"2023-10-09T05:56:06.771635+0000","mapping_epoch":207,"log_start":"108'2503","ondisk_log_start":"108'2503","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":4748,"log_dups_size":0,"ondisk_log_size":4748,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145293312,"num_objects":2217,"num_object_clones":0,"num_object_copies":6651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3816,"num_write_kb":193056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3017,"num_bytes_recovered":192643072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":101,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.354821+0000","last_change":"2023-10-09T05:55:59.304893+0000","last_active":"2023-10-09T05:56:05.354821+0000","last_peered":"2023-10-09T05:56:05.354821+0000","last_clean":"2023-10-09T05:56:05.354821+0000","last_became_active":"2023-10-09T05:55:59.304449+0000","last_became_peered":"2023-10-09T05:55:59.304449+0000","last_unstale":"2023-10-09T05:56:05.354821+0000","last_undegraded":"2023-10-09T05:56:05.354821+0000","last_fullsized":"2023-10-09T05:56:05.354821+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:44:39.453982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":224,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.357895+0000","last_change":"2023-10-09T05:55:59.306657+0000","last_active":"2023-10-09T05:56:05.357895+0000","last_peered":"2023-10-09T05:56:05.357895+0000","last_clean":"2023-10-09T05:56:05.357895+0000","last_became_active":"2023-10-09T05:55:59.306185+0000","last_became_peered":"2023-10-09T05:55:59.306185+0000","last_unstale":"2023-10-09T05:56:05.357895+0000","last_undegraded":"2023-10-09T05:56:05.357895+0000","last_fullsized":"2023-10-09T05:56:05.357895+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:26.767566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1d","version":"223'7281","reported_seq":17156,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:56:09.435564+0000","last_change":"2023-10-09T05:56:05.381643+0000","last_active":"2023-10-09T05:56:09.435564+0000","last_peered":"2023-10-09T05:56:09.435564+0000","last_clean":"2023-10-09T05:56:09.435564+0000","last_became_active":"2023-10-09T05:56:05.368691+0000","last_became_peered":"2023-10-09T05:56:05.368691+0000","last_unstale":"2023-10-09T05:56:09.435564+0000","last_undegraded":"2023-10-09T05:56:09.435564+0000","last_fullsized":"2023-10-09T05:56:09.435564+0000","mapping_epoch":221,"log_start":"108'2175","ondisk_log_start":"108'2175","created":207,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5106,"log_dups_size":0,"ondisk_log_size":5106,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140902412,"num_objects":2151,"num_object_clones":0,"num_object_copies":6453,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2151,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":3852,"num_write_kb":192033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3518,"num_bytes_recovered":230424600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":221,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.360020+0000","last_change":"2023-10-09T05:55:59.306975+0000","last_active":"2023-10-09T05:56:05.360020+0000","last_peered":"2023-10-09T05:56:05.360020+0000","last_clean":"2023-10-09T05:56:05.360020+0000","last_became_active":"2023-10-09T05:55:59.305932+0000","last_became_peered":"2023-10-09T05:55:59.305932+0000","last_unstale":"2023-10-09T05:56:05.360020+0000","last_undegraded":"2023-10-09T05:56:05.360020+0000","last_fullsized":"2023-10-09T05:56:05.360020+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:41.191008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":248,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:56:04.357999+0000","last_change":"2023-10-09T05:55:59.335464+0000","last_active":"2023-10-09T05:56:04.357999+0000","last_peered":"2023-10-09T05:56:04.357999+0000","last_clean":"2023-10-09T05:56:04.357999+0000","last_became_active":"2023-10-09T05:55:59.335259+0000","last_became_peered":"2023-10-09T05:55:59.335259+0000","last_unstale":"2023-10-09T05:56:04.357999+0000","last_undegraded":"2023-10-09T05:56:04.357999+0000","last_fullsized":"2023-10-09T05:56:04.357999+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:28.536682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.12","version":"223'7163","reported_seq":19361,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:56:09.418215+0000","last_change":"2023-10-09T05:54:45.695681+0000","last_active":"2023-10-09T05:56:09.418215+0000","last_peered":"2023-10-09T05:56:09.418215+0000","last_clean":"2023-10-09T05:56:09.418215+0000","last_became_active":"2023-10-09T05:54:45.691675+0000","last_became_peered":"2023-10-09T05:54:45.691675+0000","last_unstale":"2023-10-09T05:56:09.418215+0000","last_undegraded":"2023-10-09T05:56:09.418215+0000","last_fullsized":"2023-10-09T05:56:09.418215+0000","mapping_epoch":188,"log_start":"106'632","ondisk_log_start":"106'632","created":188,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6531,"log_dups_size":0,"ondisk_log_size":6531,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139952128,"num_objects":2136,"num_object_clones":0,"num_object_copies":6408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2136,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3884,"num_write_kb":192608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3010,"num_bytes_recovered":191234048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":168,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.359191+0000","last_change":"2023-10-09T05:54:16.209688+0000","last_active":"2023-10-09T05:56:05.359191+0000","last_peered":"2023-10-09T05:56:05.359191+0000","last_clean":"2023-10-09T05:56:05.359191+0000","last_became_active":"2023-10-09T05:54:16.209338+0000","last_became_peered":"2023-10-09T05:54:16.209338+0000","last_unstale":"2023-10-09T05:56:05.359191+0000","last_undegraded":"2023-10-09T05:56:05.359191+0000","last_fullsized":"2023-10-09T05:56:05.359191+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:28.211691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":149,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:56:04.352963+0000","last_change":"2023-10-09T05:55:59.322664+0000","last_active":"2023-10-09T05:56:04.352963+0000","last_peered":"2023-10-09T05:56:04.352963+0000","last_clean":"2023-10-09T05:56:04.352963+0000","last_became_active":"2023-10-09T05:55:59.322242+0000","last_became_peered":"2023-10-09T05:55:59.322242+0000","last_unstale":"2023-10-09T05:56:04.352963+0000","last_undegraded":"2023-10-09T05:56:04.352963+0000","last_fullsized":"2023-10-09T05:56:04.352963+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:40:15.452836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.13","version":"223'7189","reported_seq":7343,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:56:10.337580+0000","last_change":"2023-10-09T05:54:45.686838+0000","last_active":"2023-10-09T05:56:10.337580+0000","last_peered":"2023-10-09T05:56:10.337580+0000","last_clean":"2023-10-09T05:56:10.337580+0000","last_became_active":"2023-10-09T05:54:45.682331+0000","last_became_peered":"2023-10-09T05:54:45.682331+0000","last_unstale":"2023-10-09T05:56:10.337580+0000","last_undegraded":"2023-10-09T05:56:10.337580+0000","last_fullsized":"2023-10-09T05:56:10.337580+0000","mapping_epoch":188,"log_start":"107'1488","ondisk_log_start":"107'1488","created":188,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5701,"log_dups_size":0,"ondisk_log_size":5701,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141590528,"num_objects":2161,"num_object_clones":0,"num_object_copies":6483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3922,"num_write_kb":194624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":471,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.361910+0000","last_change":"2023-10-09T05:54:19.204752+0000","last_active":"2023-10-09T05:56:05.361910+0000","last_peered":"2023-10-09T05:56:05.361910+0000","last_clean":"2023-10-09T05:56:05.361910+0000","last_became_active":"2023-10-09T05:54:19.204405+0000","last_became_peered":"2023-10-09T05:54:19.204405+0000","last_unstale":"2023-10-09T05:56:05.361910+0000","last_undegraded":"2023-10-09T05:56:05.361910+0000","last_fullsized":"2023-10-09T05:56:05.361910+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":253,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.361596+0000","last_change":"2023-10-09T05:55:59.305767+0000","last_active":"2023-10-09T05:56:05.361596+0000","last_peered":"2023-10-09T05:56:05.361596+0000","last_clean":"2023-10-09T05:56:05.361596+0000","last_became_active":"2023-10-09T05:55:59.305447+0000","last_became_peered":"2023-10-09T05:55:59.305447+0000","last_unstale":"2023-10-09T05:56:05.361596+0000","last_undegraded":"2023-10-09T05:56:05.361596+0000","last_fullsized":"2023-10-09T05:56:05.361596+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:43.228681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":165,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.354746+0000","last_change":"2023-10-09T05:53:59.185702+0000","last_active":"2023-10-09T05:56:05.354746+0000","last_peered":"2023-10-09T05:56:05.354746+0000","last_clean":"2023-10-09T05:56:05.354746+0000","last_became_active":"2023-10-09T05:53:59.185373+0000","last_became_peered":"2023-10-09T05:53:59.185373+0000","last_unstale":"2023-10-09T05:56:05.354746+0000","last_undegraded":"2023-10-09T05:56:05.354746+0000","last_fullsized":"2023-10-09T05:56:05.354746+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:50.756967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"223'7167","reported_seq":14767,"reported_epoch":223,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:09.062951+0000","last_change":"2023-10-09T05:56:00.609171+0000","last_active":"2023-10-09T05:56:09.062951+0000","last_peered":"2023-10-09T05:56:09.062951+0000","last_clean":"2023-10-09T05:54:10.146196+0000","last_became_active":"2023-10-09T05:56:00.562362+0000","last_became_peered":"2023-10-09T05:56:00.562362+0000","last_unstale":"2023-10-09T05:56:09.062951+0000","last_undegraded":"2023-10-09T05:56:00.391169+0000","last_fullsized":"2023-10-09T05:56:00.277979+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":188,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7167,"log_dups_size":0,"ondisk_log_size":7167,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143556608,"num_objects":2191,"num_object_clones":0,"num_object_copies":6573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4214,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3886,"num_write_kb":194432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2416,"num_bytes_recovered":158302208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,4],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,1,2,3,5","objects":1156},{"shards":"0,2,3","objects":1021},{"shards":"0,2,3,5","objects":1},{"shards":"0,4","objects":13}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":63,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.358386+0000","last_change":"2023-10-09T05:53:59.219168+0000","last_active":"2023-10-09T05:56:05.358386+0000","last_peered":"2023-10-09T05:56:05.358386+0000","last_clean":"2023-10-09T05:56:05.358386+0000","last_became_active":"2023-10-09T05:53:59.200886+0000","last_became_peered":"2023-10-09T05:53:59.200886+0000","last_unstale":"2023-10-09T05:56:05.358386+0000","last_undegraded":"2023-10-09T05:56:05.358386+0000","last_fullsized":"2023-10-09T05:56:05.358386+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:45.515036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":116,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:56:04.354733+0000","last_change":"2023-10-09T05:55:59.305100+0000","last_active":"2023-10-09T05:56:04.354733+0000","last_peered":"2023-10-09T05:56:04.354733+0000","last_clean":"2023-10-09T05:56:04.354733+0000","last_became_active":"2023-10-09T05:55:59.304758+0000","last_became_peered":"2023-10-09T05:55:59.304758+0000","last_unstale":"2023-10-09T05:56:04.354733+0000","last_undegraded":"2023-10-09T05:56:04.354733+0000","last_fullsized":"2023-10-09T05:56:04.354733+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:54:32.510945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.11","version":"222'7122","reported_seq":7390,"reported_epoch":222,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:06.878132+0000","last_change":"2023-10-09T05:56:00.563506+0000","last_active":"2023-10-09T05:56:06.878132+0000","last_peered":"2023-10-09T05:56:06.878132+0000","last_clean":"2023-10-09T05:51:24.563560+0000","last_became_active":"2023-10-09T05:56:00.515673+0000","last_became_peered":"2023-10-09T05:56:00.515673+0000","last_unstale":"2023-10-09T05:56:06.878132+0000","last_undegraded":"2023-10-09T05:56:00.486623+0000","last_fullsized":"2023-10-09T05:56:00.384416+0000","mapping_epoch":219,"log_start":"107'871","ondisk_log_start":"107'871","created":188,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6251,"log_dups_size":0,"ondisk_log_size":6251,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148635648,"num_objects":2268,"num_object_clones":0,"num_object_copies":6804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3053,"num_objects_misplaced":1150,"num_objects_unfound":0,"num_objects_dirty":2268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3838,"num_write_kb":195392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":3014656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,1],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"1,2,6","objects":1178},{"shards":"1,6","objects":-55},{"shards":"2,6,7","objects":1118}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":228,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.371506+0000","last_change":"2023-10-09T05:53:59.219141+0000","last_active":"2023-10-09T05:56:05.371506+0000","last_peered":"2023-10-09T05:56:05.371506+0000","last_clean":"2023-10-09T05:56:05.371506+0000","last_became_active":"2023-10-09T05:53:59.197318+0000","last_became_peered":"2023-10-09T05:53:59.197318+0000","last_unstale":"2023-10-09T05:56:05.371506+0000","last_undegraded":"2023-10-09T05:56:05.371506+0000","last_fullsized":"2023-10-09T05:56:05.371506+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:39.910794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":232,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.377530+0000","last_change":"2023-10-09T05:55:59.335824+0000","last_active":"2023-10-09T05:56:05.377530+0000","last_peered":"2023-10-09T05:56:05.377530+0000","last_clean":"2023-10-09T05:56:05.377530+0000","last_became_active":"2023-10-09T05:55:59.335478+0000","last_became_peered":"2023-10-09T05:55:59.335478+0000","last_unstale":"2023-10-09T05:56:05.377530+0000","last_undegraded":"2023-10-09T05:56:05.377530+0000","last_fullsized":"2023-10-09T05:56:05.377530+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:26.014501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.16","version":"223'7118","reported_seq":11667,"reported_epoch":223,"state":"active+recovering+remapped","last_fresh":"2023-10-09T05:56:09.515757+0000","last_change":"2023-10-09T05:55:59.398328+0000","last_active":"2023-10-09T05:56:09.515757+0000","last_peered":"2023-10-09T05:56:09.515757+0000","last_clean":"2023-10-09T05:52:24.946135+0000","last_became_active":"2023-10-09T05:54:47.740044+0000","last_became_peered":"2023-10-09T05:54:47.740044+0000","last_unstale":"2023-10-09T05:56:09.515757+0000","last_undegraded":"2023-10-09T05:56:09.515757+0000","last_fullsized":"2023-10-09T05:56:09.515757+0000","mapping_epoch":190,"log_start":"106'613","ondisk_log_start":"106'613","created":188,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6505,"log_dups_size":0,"ondisk_log_size":6505,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142114816,"num_objects":2169,"num_object_clones":0,"num_object_copies":6507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":627,"num_objects_unfound":0,"num_objects_dirty":2169,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3854,"num_write_kb":192704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2166,"num_bytes_recovered":141950976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,6],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":627},{"shards":"4,6,7","objects":1542}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":553,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.360871+0000","last_change":"2023-10-09T05:53:20.237354+0000","last_active":"2023-10-09T05:56:05.360871+0000","last_peered":"2023-10-09T05:56:05.360871+0000","last_clean":"2023-10-09T05:56:05.360871+0000","last_became_active":"2023-10-09T05:53:20.236993+0000","last_became_peered":"2023-10-09T05:53:20.236993+0000","last_unstale":"2023-10-09T05:56:05.360871+0000","last_undegraded":"2023-10-09T05:56:05.360871+0000","last_fullsized":"2023-10-09T05:56:05.360871+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":65,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.359410+0000","last_change":"2023-10-09T05:53:59.233898+0000","last_active":"2023-10-09T05:56:05.359410+0000","last_peered":"2023-10-09T05:56:05.359410+0000","last_clean":"2023-10-09T05:56:05.359410+0000","last_became_active":"2023-10-09T05:53:59.204175+0000","last_became_peered":"2023-10-09T05:53:59.204175+0000","last_unstale":"2023-10-09T05:56:05.359410+0000","last_undegraded":"2023-10-09T05:56:05.359410+0000","last_fullsized":"2023-10-09T05:56:05.359410+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:03.139318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":251,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.355054+0000","last_change":"2023-10-09T05:55:59.297087+0000","last_active":"2023-10-09T05:56:05.355054+0000","last_peered":"2023-10-09T05:56:05.355054+0000","last_clean":"2023-10-09T05:56:05.355054+0000","last_became_active":"2023-10-09T05:55:59.296768+0000","last_became_peered":"2023-10-09T05:55:59.296768+0000","last_unstale":"2023-10-09T05:56:05.355054+0000","last_undegraded":"2023-10-09T05:56:05.355054+0000","last_fullsized":"2023-10-09T05:56:05.355054+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:35.067705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"223'7020","reported_seq":7159,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:56:09.406619+0000","last_change":"2023-10-09T05:54:45.686122+0000","last_active":"2023-10-09T05:56:09.406619+0000","last_peered":"2023-10-09T05:56:09.406619+0000","last_clean":"2023-10-09T05:56:09.406619+0000","last_became_active":"2023-10-09T05:54:45.681370+0000","last_became_peered":"2023-10-09T05:54:45.681370+0000","last_unstale":"2023-10-09T05:56:09.406619+0000","last_undegraded":"2023-10-09T05:56:09.406619+0000","last_fullsized":"2023-10-09T05:56:09.406619+0000","mapping_epoch":188,"log_start":"106'370","ondisk_log_start":"106'370","created":188,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6650,"log_dups_size":0,"ondisk_log_size":6650,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140673024,"num_objects":2147,"num_object_clones":0,"num_object_copies":6441,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2147,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3814,"num_write_kb":190720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":616,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.358931+0000","last_change":"2023-10-09T05:54:27.330211+0000","last_active":"2023-10-09T05:56:05.358931+0000","last_peered":"2023-10-09T05:56:05.358931+0000","last_clean":"2023-10-09T05:56:05.358931+0000","last_became_active":"2023-10-09T05:53:59.202399+0000","last_became_peered":"2023-10-09T05:53:59.202399+0000","last_unstale":"2023-10-09T05:56:05.358931+0000","last_undegraded":"2023-10-09T05:56:05.358931+0000","last_fullsized":"2023-10-09T05:56:05.358931+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:51:12.346243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:51:12.346243+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":214,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.357382+0000","last_change":"2023-10-09T05:55:59.297178+0000","last_active":"2023-10-09T05:56:05.357382+0000","last_peered":"2023-10-09T05:56:05.357382+0000","last_clean":"2023-10-09T05:56:05.357382+0000","last_became_active":"2023-10-09T05:55:59.296874+0000","last_became_peered":"2023-10-09T05:55:59.296874+0000","last_unstale":"2023-10-09T05:56:05.357382+0000","last_undegraded":"2023-10-09T05:56:05.357382+0000","last_fullsized":"2023-10-09T05:56:05.357382+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:32.817774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":202,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.361514+0000","last_change":"2023-10-09T05:53:59.182073+0000","last_active":"2023-10-09T05:56:05.361514+0000","last_peered":"2023-10-09T05:56:05.361514+0000","last_clean":"2023-10-09T05:56:05.361514+0000","last_became_active":"2023-10-09T05:53:59.181715+0000","last_became_peered":"2023-10-09T05:53:59.181715+0000","last_unstale":"2023-10-09T05:56:05.361514+0000","last_undegraded":"2023-10-09T05:56:05.361514+0000","last_fullsized":"2023-10-09T05:56:05.361514+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:58:31.918499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.14","version":"223'7138","reported_seq":7742,"reported_epoch":223,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:09.190894+0000","last_change":"2023-10-09T05:56:04.581075+0000","last_active":"2023-10-09T05:56:09.190894+0000","last_peered":"2023-10-09T05:56:09.190894+0000","last_clean":"2023-10-09T05:51:24.557317+0000","last_became_active":"2023-10-09T05:56:04.557931+0000","last_became_peered":"2023-10-09T05:56:04.557931+0000","last_unstale":"2023-10-09T05:56:09.190894+0000","last_undegraded":"2023-10-09T05:56:00.398978+0000","last_fullsized":"2023-10-09T05:56:00.286535+0000","mapping_epoch":219,"log_start":"107'1426","ondisk_log_start":"107'1426","created":188,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5712,"log_dups_size":0,"ondisk_log_size":5712,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143982592,"num_objects":2197,"num_object_clones":0,"num_object_copies":6591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2882,"num_objects_misplaced":1235,"num_objects_unfound":0,"num_objects_dirty":2197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3857,"num_write_kb":193728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":126,"num_bytes_recovered":8257536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,1],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"1,3,5","objects":1128},{"shards":"1,5","objects":29},{"shards":"3,5,6","objects":962}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":467,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.377304+0000","last_change":"2023-10-09T05:55:59.344087+0000","last_active":"2023-10-09T05:56:05.377304+0000","last_peered":"2023-10-09T05:56:05.377304+0000","last_clean":"2023-10-09T05:56:05.377304+0000","last_became_active":"2023-10-09T05:55:59.343288+0000","last_became_peered":"2023-10-09T05:55:59.343288+0000","last_unstale":"2023-10-09T05:56:05.377304+0000","last_undegraded":"2023-10-09T05:56:05.377304+0000","last_fullsized":"2023-10-09T05:56:05.377304+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":59,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.359169+0000","last_change":"2023-10-09T05:53:59.206887+0000","last_active":"2023-10-09T05:56:05.359169+0000","last_peered":"2023-10-09T05:56:05.359169+0000","last_clean":"2023-10-09T05:56:05.359169+0000","last_became_active":"2023-10-09T05:53:59.206608+0000","last_became_peered":"2023-10-09T05:53:59.206608+0000","last_unstale":"2023-10-09T05:56:05.359169+0000","last_undegraded":"2023-10-09T05:56:05.359169+0000","last_fullsized":"2023-10-09T05:56:05.359169+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:48.975792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":186,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.358040+0000","last_change":"2023-10-09T05:54:16.225672+0000","last_active":"2023-10-09T05:56:05.358040+0000","last_peered":"2023-10-09T05:56:05.358040+0000","last_clean":"2023-10-09T05:56:05.358040+0000","last_became_active":"2023-10-09T05:54:16.225312+0000","last_became_peered":"2023-10-09T05:54:16.225312+0000","last_unstale":"2023-10-09T05:56:05.358040+0000","last_undegraded":"2023-10-09T05:56:05.358040+0000","last_fullsized":"2023-10-09T05:56:05.358040+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:52.172719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"223'6905","reported_seq":12869,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:56:10.208616+0000","last_change":"2023-10-09T05:55:00.481291+0000","last_active":"2023-10-09T05:56:10.208616+0000","last_peered":"2023-10-09T05:56:10.208616+0000","last_clean":"2023-10-09T05:56:10.208616+0000","last_became_active":"2023-10-09T05:55:00.476488+0000","last_became_peered":"2023-10-09T05:55:00.476488+0000","last_unstale":"2023-10-09T05:56:10.208616+0000","last_undegraded":"2023-10-09T05:56:10.208616+0000","last_fullsized":"2023-10-09T05:56:10.208616+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":188,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6905,"log_dups_size":0,"ondisk_log_size":6905,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142704640,"num_objects":2178,"num_object_clones":0,"num_object_copies":6534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3725,"num_write_kb":188864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2244,"num_bytes_recovered":132907008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":492,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.356737+0000","last_change":"2023-10-09T05:54:19.202891+0000","last_active":"2023-10-09T05:56:05.356737+0000","last_peered":"2023-10-09T05:56:05.356737+0000","last_clean":"2023-10-09T05:56:05.356737+0000","last_became_active":"2023-10-09T05:54:19.202303+0000","last_became_peered":"2023-10-09T05:54:19.202303+0000","last_unstale":"2023-10-09T05:56:05.356737+0000","last_undegraded":"2023-10-09T05:56:05.356737+0000","last_fullsized":"2023-10-09T05:56:05.356737+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":207,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:56:04.352792+0000","last_change":"2023-10-09T05:51:57.091639+0000","last_active":"2023-10-09T05:56:04.352792+0000","last_peered":"2023-10-09T05:56:04.352792+0000","last_clean":"2023-10-09T05:56:04.352792+0000","last_became_active":"2023-10-09T05:51:57.091284+0000","last_became_peered":"2023-10-09T05:51:57.091284+0000","last_unstale":"2023-10-09T05:56:04.352792+0000","last_undegraded":"2023-10-09T05:56:04.352792+0000","last_fullsized":"2023-10-09T05:56:04.352792+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:10.553453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":169,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:56:04.352814+0000","last_change":"2023-10-09T05:55:59.308136+0000","last_active":"2023-10-09T05:56:04.352814+0000","last_peered":"2023-10-09T05:56:04.352814+0000","last_clean":"2023-10-09T05:56:04.352814+0000","last_became_active":"2023-10-09T05:55:59.307803+0000","last_became_peered":"2023-10-09T05:55:59.307803+0000","last_unstale":"2023-10-09T05:56:04.352814+0000","last_undegraded":"2023-10-09T05:56:04.352814+0000","last_fullsized":"2023-10-09T05:56:04.352814+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:42.016632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":556,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.367827+0000","last_change":"2023-10-09T05:53:20.160853+0000","last_active":"2023-10-09T05:56:05.367827+0000","last_peered":"2023-10-09T05:56:05.367827+0000","last_clean":"2023-10-09T05:56:05.367827+0000","last_became_active":"2023-10-09T05:53:20.160520+0000","last_became_peered":"2023-10-09T05:53:20.160520+0000","last_unstale":"2023-10-09T05:56:05.367827+0000","last_undegraded":"2023-10-09T05:56:05.367827+0000","last_fullsized":"2023-10-09T05:56:05.367827+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.a","version":"223'7284","reported_seq":9623,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:56:09.268764+0000","last_change":"2023-10-09T05:55:22.206257+0000","last_active":"2023-10-09T05:56:09.268764+0000","last_peered":"2023-10-09T05:56:09.268764+0000","last_clean":"2023-10-09T05:56:09.268764+0000","last_became_active":"2023-10-09T05:55:22.197933+0000","last_became_peered":"2023-10-09T05:55:22.197933+0000","last_unstale":"2023-10-09T05:56:09.268764+0000","last_undegraded":"2023-10-09T05:56:09.268764+0000","last_fullsized":"2023-10-09T05:56:09.268764+0000","mapping_epoch":161,"log_start":"108'2184","ondisk_log_start":"108'2184","created":102,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5100,"log_dups_size":0,"ondisk_log_size":5100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141099008,"num_objects":2153,"num_object_clones":0,"num_object_copies":6459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3860,"num_write_kb":192416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":542,"num_bytes_recovered":35553280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":63,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.358268+0000","last_change":"2023-10-09T05:53:59.222136+0000","last_active":"2023-10-09T05:56:05.358268+0000","last_peered":"2023-10-09T05:56:05.358268+0000","last_clean":"2023-10-09T05:56:05.358268+0000","last_became_active":"2023-10-09T05:53:59.201853+0000","last_became_peered":"2023-10-09T05:53:59.201853+0000","last_unstale":"2023-10-09T05:56:05.358268+0000","last_undegraded":"2023-10-09T05:56:05.358268+0000","last_fullsized":"2023-10-09T05:56:05.358268+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:26.662031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":185,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.360620+0000","last_change":"2023-10-09T05:53:59.187102+0000","last_active":"2023-10-09T05:56:05.360620+0000","last_peered":"2023-10-09T05:56:05.360620+0000","last_clean":"2023-10-09T05:56:05.360620+0000","last_became_active":"2023-10-09T05:53:59.186758+0000","last_became_peered":"2023-10-09T05:53:59.186758+0000","last_unstale":"2023-10-09T05:56:05.360620+0000","last_undegraded":"2023-10-09T05:56:05.360620+0000","last_fullsized":"2023-10-09T05:56:05.360620+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:35.806325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":475,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.361980+0000","last_change":"2023-10-09T05:54:47.664930+0000","last_active":"2023-10-09T05:56:05.361980+0000","last_peered":"2023-10-09T05:56:05.361980+0000","last_clean":"2023-10-09T05:56:05.361980+0000","last_became_active":"2023-10-09T05:54:47.664533+0000","last_became_peered":"2023-10-09T05:54:47.664533+0000","last_unstale":"2023-10-09T05:56:05.361980+0000","last_undegraded":"2023-10-09T05:56:05.361980+0000","last_fullsized":"2023-10-09T05:56:05.361980+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":381350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"223'7376","reported_seq":21426,"reported_epoch":223,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:56:09.578958+0000","last_change":"2023-10-09T05:56:00.454478+0000","last_active":"2023-10-09T05:56:09.578958+0000","last_peered":"2023-10-09T05:56:09.578958+0000","last_clean":"2023-10-09T05:54:47.668795+0000","last_became_active":"2023-10-09T05:56:00.438148+0000","last_became_peered":"2023-10-09T05:56:00.438148+0000","last_unstale":"2023-10-09T05:56:09.578958+0000","last_undegraded":"2023-10-09T05:56:00.295210+0000","last_fullsized":"2023-10-09T05:56:09.578958+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7376,"log_dups_size":0,"ondisk_log_size":7376,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144801792,"num_objects":2209,"num_object_clones":0,"num_object_copies":6627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2749,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3893,"num_write_kb":195296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3639,"num_bytes_recovered":223903744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":["3","6","7"],"object_location_counts":[{"shards":"1,3,6","objects":2273},{"shards":"3,4,6","objects":-64}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":105,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.359500+0000","last_change":"2023-10-09T05:55:59.379174+0000","last_active":"2023-10-09T05:56:05.359500+0000","last_peered":"2023-10-09T05:56:05.359500+0000","last_clean":"2023-10-09T05:56:05.359500+0000","last_became_active":"2023-10-09T05:55:59.341932+0000","last_became_peered":"2023-10-09T05:55:59.341932+0000","last_unstale":"2023-10-09T05:56:05.359500+0000","last_undegraded":"2023-10-09T05:56:05.359500+0000","last_fullsized":"2023-10-09T05:56:05.359500+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.636171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":259,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.363540+0000","last_change":"2023-10-09T05:55:59.341253+0000","last_active":"2023-10-09T05:56:05.363540+0000","last_peered":"2023-10-09T05:56:05.363540+0000","last_clean":"2023-10-09T05:56:05.363540+0000","last_became_active":"2023-10-09T05:55:59.340936+0000","last_became_peered":"2023-10-09T05:55:59.340936+0000","last_unstale":"2023-10-09T05:56:05.363540+0000","last_undegraded":"2023-10-09T05:56:05.363540+0000","last_fullsized":"2023-10-09T05:56:05.363540+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:43.120104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":594,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:56:04.358196+0000","last_change":"2023-10-09T05:53:59.187768+0000","last_active":"2023-10-09T05:56:04.358196+0000","last_peered":"2023-10-09T05:56:04.358196+0000","last_clean":"2023-10-09T05:56:04.358196+0000","last_became_active":"2023-10-09T05:53:59.187329+0000","last_became_peered":"2023-10-09T05:53:59.187329+0000","last_unstale":"2023-10-09T05:56:04.358196+0000","last_undegraded":"2023-10-09T05:56:04.358196+0000","last_fullsized":"2023-10-09T05:56:04.358196+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.8","version":"223'7085","reported_seq":7237,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:56:10.256306+0000","last_change":"2023-10-09T05:54:45.661206+0000","last_active":"2023-10-09T05:56:10.256306+0000","last_peered":"2023-10-09T05:56:10.256306+0000","last_clean":"2023-10-09T05:56:10.256306+0000","last_became_active":"2023-10-09T05:54:45.655631+0000","last_became_peered":"2023-10-09T05:54:45.655631+0000","last_unstale":"2023-10-09T05:56:10.256306+0000","last_undegraded":"2023-10-09T05:56:10.256306+0000","last_fullsized":"2023-10-09T05:56:10.256306+0000","mapping_epoch":174,"log_start":"106'122","ondisk_log_start":"106'122","created":102,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6963,"log_dups_size":0,"ondisk_log_size":6963,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139460608,"num_objects":2128,"num_object_clones":0,"num_object_copies":6384,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2128,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3838,"num_write_kb":190912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":61,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.358476+0000","last_change":"2023-10-09T05:53:59.212161+0000","last_active":"2023-10-09T05:56:05.358476+0000","last_peered":"2023-10-09T05:56:05.358476+0000","last_clean":"2023-10-09T05:56:05.358476+0000","last_became_active":"2023-10-09T05:53:59.193814+0000","last_became_peered":"2023-10-09T05:53:59.193814+0000","last_unstale":"2023-10-09T05:56:05.358476+0000","last_undegraded":"2023-10-09T05:56:05.358476+0000","last_fullsized":"2023-10-09T05:56:05.358476+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:10.502408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":261,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:56:04.354523+0000","last_change":"2023-10-09T05:55:59.303352+0000","last_active":"2023-10-09T05:56:04.354523+0000","last_peered":"2023-10-09T05:56:04.354523+0000","last_clean":"2023-10-09T05:56:04.354523+0000","last_became_active":"2023-10-09T05:55:59.302923+0000","last_became_peered":"2023-10-09T05:55:59.302923+0000","last_unstale":"2023-10-09T05:56:04.354523+0000","last_undegraded":"2023-10-09T05:56:04.354523+0000","last_fullsized":"2023-10-09T05:56:04.354523+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:53.013376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":581,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:56:04.357510+0000","last_change":"2023-10-09T05:53:20.167463+0000","last_active":"2023-10-09T05:56:04.357510+0000","last_peered":"2023-10-09T05:56:04.357510+0000","last_clean":"2023-10-09T05:56:04.357510+0000","last_became_active":"2023-10-09T05:53:20.167014+0000","last_became_peered":"2023-10-09T05:53:20.167014+0000","last_unstale":"2023-10-09T05:56:04.357510+0000","last_undegraded":"2023-10-09T05:56:04.357510+0000","last_fullsized":"2023-10-09T05:56:04.357510+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"223'7100","reported_seq":10224,"reported_epoch":223,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:09.291271+0000","last_change":"2023-10-09T05:55:59.403890+0000","last_active":"2023-10-09T05:56:09.291271+0000","last_peered":"2023-10-09T05:56:09.291271+0000","last_clean":"2023-10-09T05:51:33.020184+0000","last_became_active":"2023-10-09T05:54:53.759742+0000","last_became_peered":"2023-10-09T05:54:53.759742+0000","last_unstale":"2023-10-09T05:56:09.291271+0000","last_undegraded":"2023-10-09T05:54:49.556279+0000","last_fullsized":"2023-10-09T05:54:46.645749+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7100,"log_dups_size":0,"ondisk_log_size":7100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146571264,"num_objects":2236,"num_object_clones":0,"num_object_copies":6708,"num_objects_missing_on_primary":6,"num_objects_missing":6,"num_objects_degraded":1882,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3835,"num_write_kb":194304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1175,"num_bytes_recovered":77004800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,3],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,7","objects":1648},{"shards":"0,3","objects":582},{"shards":"2,7","objects":6}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":223,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.362246+0000","last_change":"2023-10-09T05:55:59.356434+0000","last_active":"2023-10-09T05:56:05.362246+0000","last_peered":"2023-10-09T05:56:05.362246+0000","last_clean":"2023-10-09T05:56:05.362246+0000","last_became_active":"2023-10-09T05:55:59.355451+0000","last_became_peered":"2023-10-09T05:55:59.355451+0000","last_unstale":"2023-10-09T05:56:05.362246+0000","last_undegraded":"2023-10-09T05:56:05.362246+0000","last_fullsized":"2023-10-09T05:56:05.362246+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:46.692106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":223,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.355985+0000","last_change":"2023-10-09T05:55:59.341649+0000","last_active":"2023-10-09T05:56:05.355985+0000","last_peered":"2023-10-09T05:56:05.355985+0000","last_clean":"2023-10-09T05:56:05.355985+0000","last_became_active":"2023-10-09T05:55:59.341325+0000","last_became_peered":"2023-10-09T05:55:59.341325+0000","last_unstale":"2023-10-09T05:56:05.355985+0000","last_undegraded":"2023-10-09T05:56:05.355985+0000","last_fullsized":"2023-10-09T05:56:05.355985+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:38.135295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":616,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.359358+0000","last_change":"2023-10-09T05:54:19.204406+0000","last_active":"2023-10-09T05:56:05.359358+0000","last_peered":"2023-10-09T05:56:05.359358+0000","last_clean":"2023-10-09T05:56:05.359358+0000","last_became_active":"2023-10-09T05:54:19.204082+0000","last_became_peered":"2023-10-09T05:54:19.204082+0000","last_unstale":"2023-10-09T05:56:05.359358+0000","last_undegraded":"2023-10-09T05:56:05.359358+0000","last_fullsized":"2023-10-09T05:56:05.359358+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.e","version":"223'7246","reported_seq":19773,"reported_epoch":223,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:56:09.499170+0000","last_change":"2023-10-09T05:56:00.422102+0000","last_active":"2023-10-09T05:56:09.499170+0000","last_peered":"2023-10-09T05:56:09.499170+0000","last_clean":"2023-10-09T05:55:58.237229+0000","last_became_active":"2023-10-09T05:56:00.411796+0000","last_became_peered":"2023-10-09T05:56:00.411796+0000","last_unstale":"2023-10-09T05:56:09.499170+0000","last_undegraded":"2023-10-09T05:56:09.499170+0000","last_fullsized":"2023-10-09T05:56:09.499170+0000","mapping_epoch":219,"log_start":"107'1851","ondisk_log_start":"107'1851","created":102,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5395,"log_dups_size":0,"ondisk_log_size":5395,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141262848,"num_objects":2155,"num_object_clones":0,"num_object_copies":6465,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2155,"num_objects_unfound":0,"num_objects_dirty":2155,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3834,"num_write_kb":191680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3073,"num_bytes_recovered":196280320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,1],"avail_no_missing":["4","1","2","5"],"object_location_counts":[{"shards":"1,4,5","objects":2155}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":117,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.356075+0000","last_change":"2023-10-09T05:55:59.388969+0000","last_active":"2023-10-09T05:56:05.356075+0000","last_peered":"2023-10-09T05:56:05.356075+0000","last_clean":"2023-10-09T05:56:05.356075+0000","last_became_active":"2023-10-09T05:55:59.358004+0000","last_became_peered":"2023-10-09T05:55:59.358004+0000","last_unstale":"2023-10-09T05:56:05.356075+0000","last_undegraded":"2023-10-09T05:56:05.356075+0000","last_fullsized":"2023-10-09T05:56:05.356075+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:39.051275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":182,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.371314+0000","last_change":"2023-10-09T05:49:21.430742+0000","last_active":"2023-10-09T05:56:05.371314+0000","last_peered":"2023-10-09T05:56:05.371314+0000","last_clean":"2023-10-09T05:56:05.371314+0000","last_became_active":"2023-10-09T05:49:21.430233+0000","last_became_peered":"2023-10-09T05:49:21.430233+0000","last_unstale":"2023-10-09T05:56:05.371314+0000","last_undegraded":"2023-10-09T05:56:05.371314+0000","last_fullsized":"2023-10-09T05:56:05.371314+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:10.541302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":551,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.360708+0000","last_change":"2023-10-09T05:54:54.779276+0000","last_active":"2023-10-09T05:56:05.360708+0000","last_peered":"2023-10-09T05:56:05.360708+0000","last_clean":"2023-10-09T05:56:05.360708+0000","last_became_active":"2023-10-09T05:54:54.778929+0000","last_became_peered":"2023-10-09T05:54:54.778929+0000","last_unstale":"2023-10-09T05:56:05.360708+0000","last_undegraded":"2023-10-09T05:56:05.360708+0000","last_fullsized":"2023-10-09T05:56:05.360708+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":686385,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.f","version":"223'7379","reported_seq":7954,"reported_epoch":223,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T05:56:09.439359+0000","last_change":"2023-10-09T05:55:59.491890+0000","last_active":"2023-10-09T05:56:09.439359+0000","last_peered":"2023-10-09T05:56:09.439359+0000","last_clean":"2023-10-09T05:52:22.632757+0000","last_became_active":"2023-10-09T05:55:59.467190+0000","last_became_peered":"2023-10-09T05:55:59.467190+0000","last_unstale":"2023-10-09T05:56:09.439359+0000","last_undegraded":"2023-10-09T05:55:59.403431+0000","last_fullsized":"2023-10-09T05:56:09.439359+0000","mapping_epoch":218,"log_start":"107'1991","ondisk_log_start":"107'1991","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5388,"log_dups_size":0,"ondisk_log_size":5388,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146112512,"num_objects":2229,"num_object_clones":0,"num_object_copies":6687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2229,"num_objects_misplaced":2252,"num_objects_unfound":0,"num_objects_dirty":2229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3911,"num_write_kb":196512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":6553600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,3],"avail_no_missing":["4","1","2","3"],"object_location_counts":[{"shards":"2,3,4","objects":2022},{"shards":"3,4,6","objects":207}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":232,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:56:04.354406+0000","last_change":"2023-10-09T05:55:59.320669+0000","last_active":"2023-10-09T05:56:04.354406+0000","last_peered":"2023-10-09T05:56:04.354406+0000","last_clean":"2023-10-09T05:56:04.354406+0000","last_became_active":"2023-10-09T05:55:59.320352+0000","last_became_peered":"2023-10-09T05:55:59.320352+0000","last_unstale":"2023-10-09T05:56:04.354406+0000","last_undegraded":"2023-10-09T05:56:04.354406+0000","last_fullsized":"2023-10-09T05:56:04.354406+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:09:14.227028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":259,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.368035+0000","last_change":"2023-10-09T05:55:59.346466+0000","last_active":"2023-10-09T05:56:05.368035+0000","last_peered":"2023-10-09T05:56:05.368035+0000","last_clean":"2023-10-09T05:56:05.368035+0000","last_became_active":"2023-10-09T05:55:59.345389+0000","last_became_peered":"2023-10-09T05:55:59.345389+0000","last_unstale":"2023-10-09T05:56:05.368035+0000","last_undegraded":"2023-10-09T05:56:05.368035+0000","last_fullsized":"2023-10-09T05:56:05.368035+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:17.657937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":467,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.377437+0000","last_change":"2023-10-09T05:55:59.344203+0000","last_active":"2023-10-09T05:56:05.377437+0000","last_peered":"2023-10-09T05:56:05.377437+0000","last_clean":"2023-10-09T05:56:05.377437+0000","last_became_active":"2023-10-09T05:55:59.343631+0000","last_became_peered":"2023-10-09T05:55:59.343631+0000","last_unstale":"2023-10-09T05:56:05.377437+0000","last_undegraded":"2023-10-09T05:56:05.377437+0000","last_fullsized":"2023-10-09T05:56:05.377437+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"222'7283","reported_seq":19530,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:06.924269+0000","last_change":"2023-10-09T05:55:22.215428+0000","last_active":"2023-10-09T05:56:06.924269+0000","last_peered":"2023-10-09T05:56:06.924269+0000","last_clean":"2023-10-09T05:56:06.924269+0000","last_became_active":"2023-10-09T05:55:22.207092+0000","last_became_peered":"2023-10-09T05:55:22.207092+0000","last_unstale":"2023-10-09T05:56:06.924269+0000","last_undegraded":"2023-10-09T05:56:06.924269+0000","last_fullsized":"2023-10-09T05:56:06.924269+0000","mapping_epoch":173,"log_start":"108'2503","ondisk_log_start":"108'2503","created":102,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":4780,"log_dups_size":0,"ondisk_log_size":4780,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143196160,"num_objects":2185,"num_object_clones":0,"num_object_copies":6555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3848,"num_write_kb":193056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3016,"num_bytes_recovered":192643072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":203,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.363727+0000","last_change":"2023-10-09T05:54:00.619165+0000","last_active":"2023-10-09T05:56:05.363727+0000","last_peered":"2023-10-09T05:56:05.363727+0000","last_clean":"2023-10-09T05:56:05.363727+0000","last_became_active":"2023-10-09T05:54:00.618888+0000","last_became_peered":"2023-10-09T05:54:00.618888+0000","last_unstale":"2023-10-09T05:56:05.363727+0000","last_undegraded":"2023-10-09T05:56:05.363727+0000","last_fullsized":"2023-10-09T05:56:05.363727+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:07.175996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":232,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.371538+0000","last_change":"2023-10-09T05:55:59.336539+0000","last_active":"2023-10-09T05:56:05.371538+0000","last_peered":"2023-10-09T05:56:05.371538+0000","last_clean":"2023-10-09T05:56:05.371538+0000","last_became_active":"2023-10-09T05:55:59.334624+0000","last_became_peered":"2023-10-09T05:55:59.334624+0000","last_unstale":"2023-10-09T05:56:05.371538+0000","last_undegraded":"2023-10-09T05:56:05.371538+0000","last_fullsized":"2023-10-09T05:56:05.371538+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:33.120802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":530,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.361307+0000","last_change":"2023-10-09T05:55:58.750810+0000","last_active":"2023-10-09T05:56:05.361307+0000","last_peered":"2023-10-09T05:56:05.361307+0000","last_clean":"2023-10-09T05:56:05.361307+0000","last_became_active":"2023-10-09T05:51:19.195632+0000","last_became_peered":"2023-10-09T05:51:19.195632+0000","last_unstale":"2023-10-09T05:56:05.361307+0000","last_undegraded":"2023-10-09T05:56:05.361307+0000","last_fullsized":"2023-10-09T05:56:05.361307+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.d","version":"223'7272","reported_seq":12943,"reported_epoch":223,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:56:09.413930+0000","last_change":"2023-10-09T05:56:04.582001+0000","last_active":"2023-10-09T05:56:09.413930+0000","last_peered":"2023-10-09T05:56:09.413930+0000","last_clean":"2023-10-09T05:55:58.102294+0000","last_became_active":"2023-10-09T05:56:00.395752+0000","last_became_peered":"2023-10-09T05:56:00.395752+0000","last_unstale":"2023-10-09T05:56:09.413930+0000","last_undegraded":"2023-10-09T05:56:09.413930+0000","last_fullsized":"2023-10-09T05:56:09.413930+0000","mapping_epoch":219,"log_start":"108'2175","ondisk_log_start":"108'2175","created":102,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5097,"log_dups_size":0,"ondisk_log_size":5097,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141492236,"num_objects":2159,"num_object_clones":0,"num_object_copies":6477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2040,"num_objects_unfound":0,"num_objects_dirty":2159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3842,"num_write_kb":192032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1422,"num_bytes_recovered":93192192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,1],"avail_no_missing":["5","0","1"],"object_location_counts":[{"shards":"0,1,5","objects":2159}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":205,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.358573+0000","last_change":"2023-10-09T05:55:59.320634+0000","last_active":"2023-10-09T05:56:05.358573+0000","last_peered":"2023-10-09T05:56:05.358573+0000","last_clean":"2023-10-09T05:56:05.358573+0000","last_became_active":"2023-10-09T05:55:59.320341+0000","last_became_peered":"2023-10-09T05:55:59.320341+0000","last_unstale":"2023-10-09T05:56:05.358573+0000","last_undegraded":"2023-10-09T05:56:05.358573+0000","last_fullsized":"2023-10-09T05:56:05.358573+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:15:13.249107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":101,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.361335+0000","last_change":"2023-10-09T05:55:59.409563+0000","last_active":"2023-10-09T05:56:05.361335+0000","last_peered":"2023-10-09T05:56:05.361335+0000","last_clean":"2023-10-09T05:56:05.361335+0000","last_became_active":"2023-10-09T05:55:59.409184+0000","last_became_peered":"2023-10-09T05:55:59.409184+0000","last_unstale":"2023-10-09T05:56:05.361335+0000","last_undegraded":"2023-10-09T05:56:05.361335+0000","last_fullsized":"2023-10-09T05:56:05.361335+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:39.513591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":595,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:56:04.358096+0000","last_change":"2023-10-09T05:53:59.202618+0000","last_active":"2023-10-09T05:56:04.358096+0000","last_peered":"2023-10-09T05:56:04.358096+0000","last_clean":"2023-10-09T05:56:04.358096+0000","last_became_active":"2023-10-09T05:53:59.202265+0000","last_became_peered":"2023-10-09T05:53:59.202265+0000","last_unstale":"2023-10-09T05:56:04.358096+0000","last_undegraded":"2023-10-09T05:56:04.358096+0000","last_fullsized":"2023-10-09T05:56:04.358096+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":116,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.356538+0000","last_change":"2023-10-09T05:55:59.298953+0000","last_active":"2023-10-09T05:56:05.356538+0000","last_peered":"2023-10-09T05:56:05.356538+0000","last_clean":"2023-10-09T05:56:05.356538+0000","last_became_active":"2023-10-09T05:55:59.298624+0000","last_became_peered":"2023-10-09T05:55:59.298624+0000","last_unstale":"2023-10-09T05:56:05.356538+0000","last_undegraded":"2023-10-09T05:56:05.356538+0000","last_fullsized":"2023-10-09T05:56:05.356538+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:00.513870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":210,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.361470+0000","last_change":"2023-10-09T05:52:29.735372+0000","last_active":"2023-10-09T05:56:05.361470+0000","last_peered":"2023-10-09T05:56:05.361470+0000","last_clean":"2023-10-09T05:56:05.361470+0000","last_became_active":"2023-10-09T05:52:29.735026+0000","last_became_peered":"2023-10-09T05:52:29.735026+0000","last_unstale":"2023-10-09T05:56:05.361470+0000","last_undegraded":"2023-10-09T05:56:05.361470+0000","last_fullsized":"2023-10-09T05:56:05.361470+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:53.849406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":97,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.361215+0000","last_change":"2023-10-09T05:55:59.307598+0000","last_active":"2023-10-09T05:56:05.361215+0000","last_peered":"2023-10-09T05:56:05.361215+0000","last_clean":"2023-10-09T05:56:05.361215+0000","last_became_active":"2023-10-09T05:55:59.306968+0000","last_became_peered":"2023-10-09T05:55:59.306968+0000","last_unstale":"2023-10-09T05:56:05.361215+0000","last_undegraded":"2023-10-09T05:56:05.361215+0000","last_fullsized":"2023-10-09T05:56:05.361215+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:43.253091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"223'6945","reported_seq":10309,"reported_epoch":223,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:56:09.319492+0000","last_change":"2023-10-09T05:56:00.610946+0000","last_active":"2023-10-09T05:56:09.319492+0000","last_peered":"2023-10-09T05:56:09.319492+0000","last_clean":"2023-10-09T05:54:10.146196+0000","last_became_active":"2023-10-09T05:56:00.532512+0000","last_became_peered":"2023-10-09T05:56:00.532512+0000","last_unstale":"2023-10-09T05:56:09.319492+0000","last_undegraded":"2023-10-09T05:56:00.361695+0000","last_fullsized":"2023-10-09T05:56:09.319492+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6945,"log_dups_size":0,"ondisk_log_size":6945,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71450624,"num_objects":1090,"num_object_clones":0,"num_object_copies":3270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2480,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1090,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1947,"num_write_kb":97216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":606,"num_bytes_recovered":39763968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":1104},{"shards":"0,4,5","objects":-14}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":554,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.360666+0000","last_change":"2023-10-09T05:53:20.234074+0000","last_active":"2023-10-09T05:56:05.360666+0000","last_peered":"2023-10-09T05:56:05.360666+0000","last_clean":"2023-10-09T05:56:05.360666+0000","last_became_active":"2023-10-09T05:53:20.233803+0000","last_became_peered":"2023-10-09T05:53:20.233803+0000","last_unstale":"2023-10-09T05:56:05.360666+0000","last_undegraded":"2023-10-09T05:56:05.360666+0000","last_fullsized":"2023-10-09T05:56:05.360666+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.6","version":"223'7141","reported_seq":8617,"reported_epoch":223,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T05:56:09.503888+0000","last_change":"2023-10-09T05:55:01.518574+0000","last_active":"2023-10-09T05:56:09.503888+0000","last_peered":"2023-10-09T05:56:09.503888+0000","last_clean":"2023-10-09T05:52:24.946135+0000","last_became_active":"2023-10-09T05:54:45.741289+0000","last_became_peered":"2023-10-09T05:54:45.741289+0000","last_unstale":"2023-10-09T05:56:09.503888+0000","last_undegraded":"2023-10-09T05:55:01.516066+0000","last_fullsized":"2023-10-09T05:56:09.503888+0000","mapping_epoch":181,"log_start":"106'613","ondisk_log_start":"106'613","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6528,"log_dups_size":0,"ondisk_log_size":6528,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140607488,"num_objects":2145,"num_object_clones":0,"num_object_copies":6435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2145,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2145,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3876,"num_write_kb":192704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":629,"num_bytes_recovered":41222144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,6],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":2127},{"shards":"4,6,7","objects":18}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":254,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.361144+0000","last_change":"2023-10-09T05:55:59.307771+0000","last_active":"2023-10-09T05:56:05.361144+0000","last_peered":"2023-10-09T05:56:05.361144+0000","last_clean":"2023-10-09T05:56:05.361144+0000","last_became_active":"2023-10-09T05:55:59.307299+0000","last_became_peered":"2023-10-09T05:55:59.307299+0000","last_unstale":"2023-10-09T05:56:05.361144+0000","last_undegraded":"2023-10-09T05:56:05.361144+0000","last_fullsized":"2023-10-09T05:56:05.361144+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:53.874666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":200,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.377674+0000","last_change":"2023-10-09T05:51:57.203142+0000","last_active":"2023-10-09T05:56:05.377674+0000","last_peered":"2023-10-09T05:56:05.377674+0000","last_clean":"2023-10-09T05:56:05.377674+0000","last_became_active":"2023-10-09T05:51:57.202850+0000","last_became_peered":"2023-10-09T05:51:57.202850+0000","last_unstale":"2023-10-09T05:56:05.377674+0000","last_undegraded":"2023-10-09T05:56:05.377674+0000","last_fullsized":"2023-10-09T05:56:05.377674+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:51.090737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":126,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:56:04.352397+0000","last_change":"2023-10-09T05:55:59.298261+0000","last_active":"2023-10-09T05:56:04.352397+0000","last_peered":"2023-10-09T05:56:04.352397+0000","last_clean":"2023-10-09T05:56:04.352397+0000","last_became_active":"2023-10-09T05:55:59.297549+0000","last_became_peered":"2023-10-09T05:55:59.297549+0000","last_unstale":"2023-10-09T05:56:04.352397+0000","last_undegraded":"2023-10-09T05:56:04.352397+0000","last_fullsized":"2023-10-09T05:56:04.352397+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:05:53.922360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":492,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.356946+0000","last_change":"2023-10-09T05:54:19.203419+0000","last_active":"2023-10-09T05:56:05.356946+0000","last_peered":"2023-10-09T05:56:05.356946+0000","last_clean":"2023-10-09T05:56:05.356946+0000","last_became_active":"2023-10-09T05:54:19.203105+0000","last_became_peered":"2023-10-09T05:54:19.203105+0000","last_unstale":"2023-10-09T05:56:05.356946+0000","last_undegraded":"2023-10-09T05:56:05.356946+0000","last_fullsized":"2023-10-09T05:56:05.356946+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"223'6970","reported_seq":13013,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:56:10.377299+0000","last_change":"2023-10-09T05:54:54.773920+0000","last_active":"2023-10-09T05:56:10.377299+0000","last_peered":"2023-10-09T05:56:10.377299+0000","last_clean":"2023-10-09T05:56:10.377299+0000","last_became_active":"2023-10-09T05:54:54.770744+0000","last_became_peered":"2023-10-09T05:54:54.770744+0000","last_unstale":"2023-10-09T05:56:10.377299+0000","last_undegraded":"2023-10-09T05:56:10.377299+0000","last_fullsized":"2023-10-09T05:56:10.377299+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6970,"log_dups_size":0,"ondisk_log_size":6970,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138444800,"num_objects":2112,"num_object_clones":0,"num_object_copies":6336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3789,"num_write_kb":188864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2284,"num_bytes_recovered":135462912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":210,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.378950+0000","last_change":"2023-10-09T05:55:59.314996+0000","last_active":"2023-10-09T05:56:05.378950+0000","last_peered":"2023-10-09T05:56:05.378950+0000","last_clean":"2023-10-09T05:56:05.378950+0000","last_became_active":"2023-10-09T05:55:59.314633+0000","last_became_peered":"2023-10-09T05:55:59.314633+0000","last_unstale":"2023-10-09T05:56:05.378950+0000","last_undegraded":"2023-10-09T05:56:05.378950+0000","last_fullsized":"2023-10-09T05:56:05.378950+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:08.653191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":110,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:56:04.351727+0000","last_change":"2023-10-09T05:55:59.308743+0000","last_active":"2023-10-09T05:56:04.351727+0000","last_peered":"2023-10-09T05:56:04.351727+0000","last_clean":"2023-10-09T05:56:04.351727+0000","last_became_active":"2023-10-09T05:55:59.308220+0000","last_became_peered":"2023-10-09T05:55:59.308220+0000","last_unstale":"2023-10-09T05:56:04.351727+0000","last_undegraded":"2023-10-09T05:56:04.351727+0000","last_fullsized":"2023-10-09T05:56:04.351727+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:44.246081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":129,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.362975+0000","last_change":"2023-10-09T05:55:59.305883+0000","last_active":"2023-10-09T05:56:05.362975+0000","last_peered":"2023-10-09T05:56:05.362975+0000","last_clean":"2023-10-09T05:56:05.362975+0000","last_became_active":"2023-10-09T05:55:59.305588+0000","last_became_peered":"2023-10-09T05:55:59.305588+0000","last_unstale":"2023-10-09T05:56:05.362975+0000","last_undegraded":"2023-10-09T05:56:05.362975+0000","last_fullsized":"2023-10-09T05:56:05.362975+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:06:21.115125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.7","version":"223'6988","reported_seq":7128,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:56:08.724488+0000","last_change":"2023-10-09T05:54:45.655400+0000","last_active":"2023-10-09T05:56:08.724488+0000","last_peered":"2023-10-09T05:56:08.724488+0000","last_clean":"2023-10-09T05:56:08.724488+0000","last_became_active":"2023-10-09T05:54:45.647040+0000","last_became_peered":"2023-10-09T05:54:45.647040+0000","last_unstale":"2023-10-09T05:56:08.724488+0000","last_undegraded":"2023-10-09T05:56:08.724488+0000","last_fullsized":"2023-10-09T05:56:08.724488+0000","mapping_epoch":102,"log_start":"106'370","ondisk_log_start":"106'370","created":102,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6618,"log_dups_size":0,"ondisk_log_size":6618,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142770176,"num_objects":2178,"num_object_clones":0,"num_object_copies":6534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3781,"num_write_kb":190720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":207,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.360938+0000","last_change":"2023-10-09T05:53:59.182813+0000","last_active":"2023-10-09T05:56:05.360938+0000","last_peered":"2023-10-09T05:56:05.360938+0000","last_clean":"2023-10-09T05:56:05.360938+0000","last_became_active":"2023-10-09T05:53:59.182441+0000","last_became_peered":"2023-10-09T05:53:59.182441+0000","last_unstale":"2023-10-09T05:56:05.360938+0000","last_undegraded":"2023-10-09T05:56:05.360938+0000","last_fullsized":"2023-10-09T05:56:05.360938+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:59.004246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":614,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.359011+0000","last_change":"2023-10-09T05:53:59.202723+0000","last_active":"2023-10-09T05:56:05.359011+0000","last_peered":"2023-10-09T05:56:05.359011+0000","last_clean":"2023-10-09T05:56:05.359011+0000","last_became_active":"2023-10-09T05:53:59.202004+0000","last_became_peered":"2023-10-09T05:53:59.202004+0000","last_unstale":"2023-10-09T05:56:05.359011+0000","last_undegraded":"2023-10-09T05:56:05.359011+0000","last_fullsized":"2023-10-09T05:56:05.359011+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:51:12.346243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:51:12.346243+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":110,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.361127+0000","last_change":"2023-10-09T05:52:23.648626+0000","last_active":"2023-10-09T05:56:05.361127+0000","last_peered":"2023-10-09T05:56:05.361127+0000","last_clean":"2023-10-09T05:56:05.361127+0000","last_became_active":"2023-10-09T05:52:23.646143+0000","last_became_peered":"2023-10-09T05:52:23.646143+0000","last_unstale":"2023-10-09T05:56:05.361127+0000","last_undegraded":"2023-10-09T05:56:05.361127+0000","last_fullsized":"2023-10-09T05:56:05.361127+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:51:04.733282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":184,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:56:04.356541+0000","last_change":"2023-10-09T05:53:59.195325+0000","last_active":"2023-10-09T05:56:04.356541+0000","last_peered":"2023-10-09T05:56:04.356541+0000","last_clean":"2023-10-09T05:56:04.356541+0000","last_became_active":"2023-10-09T05:53:59.194954+0000","last_became_peered":"2023-10-09T05:53:59.194954+0000","last_unstale":"2023-10-09T05:56:04.356541+0000","last_undegraded":"2023-10-09T05:56:04.356541+0000","last_fullsized":"2023-10-09T05:56:04.356541+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:23.228416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":467,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.377450+0000","last_change":"2023-10-09T05:55:59.344330+0000","last_active":"2023-10-09T05:56:05.377450+0000","last_peered":"2023-10-09T05:56:05.377450+0000","last_clean":"2023-10-09T05:56:05.377450+0000","last_became_active":"2023-10-09T05:55:59.343887+0000","last_became_peered":"2023-10-09T05:55:59.343887+0000","last_unstale":"2023-10-09T05:56:05.377450+0000","last_undegraded":"2023-10-09T05:56:05.377450+0000","last_fullsized":"2023-10-09T05:56:05.377450+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":89,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.358848+0000","last_change":"2023-10-09T05:55:59.300015+0000","last_active":"2023-10-09T05:56:05.358848+0000","last_peered":"2023-10-09T05:56:05.358848+0000","last_clean":"2023-10-09T05:56:05.358848+0000","last_became_active":"2023-10-09T05:55:59.299281+0000","last_became_peered":"2023-10-09T05:55:59.299281+0000","last_unstale":"2023-10-09T05:56:05.358848+0000","last_undegraded":"2023-10-09T05:56:05.358848+0000","last_fullsized":"2023-10-09T05:56:05.358848+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:50:32.465904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":150,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.378821+0000","last_change":"2023-10-09T05:52:23.643245+0000","last_active":"2023-10-09T05:56:05.378821+0000","last_peered":"2023-10-09T05:56:05.378821+0000","last_clean":"2023-10-09T05:56:05.378821+0000","last_became_active":"2023-10-09T05:52:23.642947+0000","last_became_peered":"2023-10-09T05:52:23.642947+0000","last_unstale":"2023-10-09T05:56:05.378821+0000","last_undegraded":"2023-10-09T05:56:05.378821+0000","last_fullsized":"2023-10-09T05:56:05.378821+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:52.260774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":266,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.357827+0000","last_change":"2023-10-09T05:55:59.301184+0000","last_active":"2023-10-09T05:56:05.357827+0000","last_peered":"2023-10-09T05:56:05.357827+0000","last_clean":"2023-10-09T05:56:05.357827+0000","last_became_active":"2023-10-09T05:55:59.299733+0000","last_became_peered":"2023-10-09T05:55:59.299733+0000","last_unstale":"2023-10-09T05:56:05.357827+0000","last_undegraded":"2023-10-09T05:56:05.357827+0000","last_fullsized":"2023-10-09T05:56:05.357827+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:36.929150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"223'7159","reported_seq":7755,"reported_epoch":223,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:09.530873+0000","last_change":"2023-10-09T05:56:04.839436+0000","last_active":"2023-10-09T05:56:09.530873+0000","last_peered":"2023-10-09T05:56:09.530873+0000","last_clean":"2023-10-09T05:51:24.557317+0000","last_became_active":"2023-10-09T05:56:04.570363+0000","last_became_peered":"2023-10-09T05:56:04.570363+0000","last_unstale":"2023-10-09T05:56:09.530873+0000","last_undegraded":"2023-10-09T05:56:03.552905+0000","last_fullsized":"2023-10-09T05:56:00.280042+0000","mapping_epoch":219,"log_start":"107'1426","ondisk_log_start":"107'1426","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5733,"log_dups_size":0,"ondisk_log_size":5733,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142606336,"num_objects":2176,"num_object_clones":0,"num_object_copies":6528,"num_objects_missing_on_primary":962,"num_objects_missing":962,"num_objects_degraded":2924,"num_objects_misplaced":1214,"num_objects_unfound":0,"num_objects_dirty":2176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3878,"num_write_kb":193728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8060928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[1,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"1,3,5","objects":1136},{"shards":"1,5","objects":-1},{"shards":"3,5,6","objects":962}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":585,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:56:04.357627+0000","last_change":"2023-10-09T05:55:59.752131+0000","last_active":"2023-10-09T05:56:04.357627+0000","last_peered":"2023-10-09T05:56:04.357627+0000","last_clean":"2023-10-09T05:56:04.357627+0000","last_became_active":"2023-10-09T05:53:20.157392+0000","last_became_peered":"2023-10-09T05:53:20.157392+0000","last_unstale":"2023-10-09T05:56:04.357627+0000","last_undegraded":"2023-10-09T05:56:04.357627+0000","last_fullsized":"2023-10-09T05:56:04.357627+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":57,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.358300+0000","last_change":"2023-10-09T05:53:59.201576+0000","last_active":"2023-10-09T05:56:05.358300+0000","last_peered":"2023-10-09T05:56:05.358300+0000","last_clean":"2023-10-09T05:56:05.358300+0000","last_became_active":"2023-10-09T05:53:59.201309+0000","last_became_peered":"2023-10-09T05:53:59.201309+0000","last_unstale":"2023-10-09T05:56:05.358300+0000","last_undegraded":"2023-10-09T05:56:05.358300+0000","last_fullsized":"2023-10-09T05:56:05.358300+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:54.956371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":104,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.362362+0000","last_change":"2023-10-09T05:55:59.356259+0000","last_active":"2023-10-09T05:56:05.362362+0000","last_peered":"2023-10-09T05:56:05.362362+0000","last_clean":"2023-10-09T05:56:05.362362+0000","last_became_active":"2023-10-09T05:55:59.354136+0000","last_became_peered":"2023-10-09T05:55:59.354136+0000","last_unstale":"2023-10-09T05:56:05.362362+0000","last_undegraded":"2023-10-09T05:56:05.362362+0000","last_fullsized":"2023-10-09T05:56:05.362362+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:44:11.833451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":147,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.356149+0000","last_change":"2023-10-09T05:53:59.188462+0000","last_active":"2023-10-09T05:56:05.356149+0000","last_peered":"2023-10-09T05:56:05.356149+0000","last_clean":"2023-10-09T05:56:05.356149+0000","last_became_active":"2023-10-09T05:53:59.188061+0000","last_became_peered":"2023-10-09T05:53:59.188061+0000","last_unstale":"2023-10-09T05:56:05.356149+0000","last_undegraded":"2023-10-09T05:56:05.356149+0000","last_fullsized":"2023-10-09T05:56:05.356149+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:19.167114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"222'6958","reported_seq":7479,"reported_epoch":222,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:06.828245+0000","last_change":"2023-10-09T05:56:00.526011+0000","last_active":"2023-10-09T05:56:06.828245+0000","last_peered":"2023-10-09T05:56:06.828245+0000","last_clean":"2023-10-09T05:51:24.563560+0000","last_became_active":"2023-10-09T05:56:00.507313+0000","last_became_peered":"2023-10-09T05:56:00.507313+0000","last_unstale":"2023-10-09T05:56:06.828245+0000","last_undegraded":"2023-10-09T05:56:00.383644+0000","last_fullsized":"2023-10-09T05:56:00.340221+0000","mapping_epoch":219,"log_start":"107'871","ondisk_log_start":"107'871","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6087,"log_dups_size":0,"ondisk_log_size":6087,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73367552,"num_objects":1119,"num_object_clones":0,"num_object_copies":3357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1329,"num_objects_misplaced":643,"num_objects_unfound":0,"num_objects_dirty":1119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1933,"num_write_kb":97696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":169,"num_bytes_recovered":11075584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,1],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"1,2,6","objects":613},{"shards":"1,6","objects":7},{"shards":"2,6,7","objects":476}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":557,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.367907+0000","last_change":"2023-10-09T05:53:20.152495+0000","last_active":"2023-10-09T05:56:05.367907+0000","last_peered":"2023-10-09T05:56:05.367907+0000","last_clean":"2023-10-09T05:56:05.367907+0000","last_became_active":"2023-10-09T05:53:20.152109+0000","last_became_peered":"2023-10-09T05:53:20.152109+0000","last_unstale":"2023-10-09T05:56:05.367907+0000","last_undegraded":"2023-10-09T05:56:05.367907+0000","last_fullsized":"2023-10-09T05:56:05.367907+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":228,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.361433+0000","last_change":"2023-10-09T05:53:59.228948+0000","last_active":"2023-10-09T05:56:05.361433+0000","last_peered":"2023-10-09T05:56:05.361433+0000","last_clean":"2023-10-09T05:56:05.361433+0000","last_became_active":"2023-10-09T05:53:59.186359+0000","last_became_peered":"2023-10-09T05:53:59.186359+0000","last_unstale":"2023-10-09T05:56:05.361433+0000","last_undegraded":"2023-10-09T05:56:05.361433+0000","last_fullsized":"2023-10-09T05:56:05.361433+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:39.990576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":79,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:56:04.357552+0000","last_change":"2023-10-09T05:55:59.339957+0000","last_active":"2023-10-09T05:56:04.357552+0000","last_peered":"2023-10-09T05:56:04.357552+0000","last_clean":"2023-10-09T05:56:04.357552+0000","last_became_active":"2023-10-09T05:55:59.339614+0000","last_became_peered":"2023-10-09T05:55:59.339614+0000","last_unstale":"2023-10-09T05:56:04.357552+0000","last_undegraded":"2023-10-09T05:56:04.357552+0000","last_fullsized":"2023-10-09T05:56:04.357552+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:01.613113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":89,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.358759+0000","last_change":"2023-10-09T05:55:59.300201+0000","last_active":"2023-10-09T05:56:05.358759+0000","last_peered":"2023-10-09T05:56:05.358759+0000","last_clean":"2023-10-09T05:56:05.358759+0000","last_became_active":"2023-10-09T05:55:59.299709+0000","last_became_peered":"2023-10-09T05:55:59.299709+0000","last_unstale":"2023-10-09T05:56:05.358759+0000","last_undegraded":"2023-10-09T05:56:05.358759+0000","last_fullsized":"2023-10-09T05:56:05.358759+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:32:55.608383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.2","version":"223'6933","reported_seq":19142,"reported_epoch":223,"state":"active+recovery_wait","last_fresh":"2023-10-09T05:56:09.183073+0000","last_change":"2023-10-09T05:55:22.328640+0000","last_active":"2023-10-09T05:56:09.183073+0000","last_peered":"2023-10-09T05:56:09.183073+0000","last_clean":"2023-10-09T05:55:20.973604+0000","last_became_active":"2023-10-09T05:55:22.208547+0000","last_became_peered":"2023-10-09T05:55:22.208547+0000","last_unstale":"2023-10-09T05:56:09.183073+0000","last_undegraded":"2023-10-09T05:56:09.183073+0000","last_fullsized":"2023-10-09T05:56:09.183073+0000","mapping_epoch":159,"log_start":"106'632","ondisk_log_start":"106'632","created":102,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6301,"log_dups_size":0,"ondisk_log_size":6301,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70533120,"num_objects":1076,"num_object_clones":0,"num_object_copies":3228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1933,"num_write_kb":96304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1505,"num_bytes_recovered":95617024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":1076}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":471,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.362054+0000","last_change":"2023-10-09T05:54:19.202509+0000","last_active":"2023-10-09T05:56:05.362054+0000","last_peered":"2023-10-09T05:56:05.362054+0000","last_clean":"2023-10-09T05:56:05.362054+0000","last_became_active":"2023-10-09T05:54:19.202144+0000","last_became_peered":"2023-10-09T05:54:19.202144+0000","last_unstale":"2023-10-09T05:56:05.362054+0000","last_undegraded":"2023-10-09T05:56:05.362054+0000","last_fullsized":"2023-10-09T05:56:05.362054+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":216,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:56:04.358098+0000","last_change":"2023-10-09T05:54:16.215036+0000","last_active":"2023-10-09T05:56:04.358098+0000","last_peered":"2023-10-09T05:56:04.358098+0000","last_clean":"2023-10-09T05:56:04.358098+0000","last_became_active":"2023-10-09T05:54:16.212771+0000","last_became_peered":"2023-10-09T05:54:16.212771+0000","last_unstale":"2023-10-09T05:56:04.358098+0000","last_undegraded":"2023-10-09T05:56:04.358098+0000","last_fullsized":"2023-10-09T05:56:04.358098+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:29.278423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":248,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.357970+0000","last_change":"2023-10-09T05:55:59.374662+0000","last_active":"2023-10-09T05:56:05.357970+0000","last_peered":"2023-10-09T05:56:05.357970+0000","last_clean":"2023-10-09T05:56:05.357970+0000","last_became_active":"2023-10-09T05:55:59.305777+0000","last_became_peered":"2023-10-09T05:55:59.305777+0000","last_unstale":"2023-10-09T05:56:05.357970+0000","last_undegraded":"2023-10-09T05:56:05.357970+0000","last_fullsized":"2023-10-09T05:56:05.357970+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:05.569760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":165,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.363402+0000","last_change":"2023-10-09T05:40:12.676587+0000","last_active":"2023-10-09T05:56:05.363402+0000","last_peered":"2023-10-09T05:56:05.363402+0000","last_clean":"2023-10-09T05:56:05.363402+0000","last_became_active":"2023-10-09T05:40:12.676330+0000","last_became_peered":"2023-10-09T05:40:12.676330+0000","last_unstale":"2023-10-09T05:56:05.363402+0000","last_undegraded":"2023-10-09T05:56:05.363402+0000","last_fullsized":"2023-10-09T05:56:05.363402+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:46.659556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"223'6932","reported_seq":7098,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:56:10.146873+0000","last_change":"2023-10-09T05:55:22.211279+0000","last_active":"2023-10-09T05:56:10.146873+0000","last_peered":"2023-10-09T05:56:10.146873+0000","last_clean":"2023-10-09T05:56:10.146873+0000","last_became_active":"2023-10-09T05:55:22.208477+0000","last_became_peered":"2023-10-09T05:55:22.208477+0000","last_unstale":"2023-10-09T05:56:10.146873+0000","last_undegraded":"2023-10-09T05:56:10.146873+0000","last_fullsized":"2023-10-09T05:56:10.146873+0000","mapping_epoch":174,"log_start":"107'1488","ondisk_log_start":"107'1488","created":102,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5444,"log_dups_size":0,"ondisk_log_size":5444,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72302592,"num_objects":1103,"num_object_clones":0,"num_object_copies":3309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1937,"num_write_kb":97312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.1e","version":"223'7203","reported_seq":19731,"reported_epoch":223,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:56:09.488274+0000","last_change":"2023-10-09T05:56:00.413231+0000","last_active":"2023-10-09T05:56:09.488274+0000","last_peered":"2023-10-09T05:56:09.488274+0000","last_clean":"2023-10-09T05:55:58.105865+0000","last_became_active":"2023-10-09T05:56:00.403893+0000","last_became_peered":"2023-10-09T05:56:00.403893+0000","last_unstale":"2023-10-09T05:56:09.488274+0000","last_undegraded":"2023-10-09T05:56:09.488274+0000","last_fullsized":"2023-10-09T05:56:09.488274+0000","mapping_epoch":219,"log_start":"107'1851","ondisk_log_start":"107'1851","created":207,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5352,"log_dups_size":0,"ondisk_log_size":5352,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144080896,"num_objects":2199,"num_object_clones":0,"num_object_copies":6597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2199,"num_objects_unfound":0,"num_objects_dirty":2199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3792,"num_write_kb":191680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3074,"num_bytes_recovered":196280320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,1],"avail_no_missing":["4","1","2","5"],"object_location_counts":[{"shards":"1,4,5","objects":2199}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":151,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.356237+0000","last_change":"2023-10-09T05:53:59.191302+0000","last_active":"2023-10-09T05:56:05.356237+0000","last_peered":"2023-10-09T05:56:05.356237+0000","last_clean":"2023-10-09T05:56:05.356237+0000","last_became_active":"2023-10-09T05:53:59.190911+0000","last_became_peered":"2023-10-09T05:53:59.190911+0000","last_unstale":"2023-10-09T05:56:05.356237+0000","last_undegraded":"2023-10-09T05:56:05.356237+0000","last_fullsized":"2023-10-09T05:56:05.356237+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:12.801201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":192,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.356451+0000","last_change":"2023-10-09T05:55:59.305449+0000","last_active":"2023-10-09T05:56:05.356451+0000","last_peered":"2023-10-09T05:56:05.356451+0000","last_clean":"2023-10-09T05:56:05.356451+0000","last_became_active":"2023-10-09T05:55:59.305109+0000","last_became_peered":"2023-10-09T05:55:59.305109+0000","last_unstale":"2023-10-09T05:56:05.356451+0000","last_undegraded":"2023-10-09T05:56:05.356451+0000","last_fullsized":"2023-10-09T05:56:05.356451+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:31.830141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.19","version":"223'7107","reported_seq":8482,"reported_epoch":223,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:09.351902+0000","last_change":"2023-10-09T05:54:57.558699+0000","last_active":"2023-10-09T05:56:09.351902+0000","last_peered":"2023-10-09T05:56:09.351902+0000","last_clean":"2023-10-09T05:51:33.020184+0000","last_became_active":"2023-10-09T05:54:57.470115+0000","last_became_peered":"2023-10-09T05:54:57.470115+0000","last_unstale":"2023-10-09T05:56:09.351902+0000","last_undegraded":"2023-10-09T05:54:52.846685+0000","last_fullsized":"2023-10-09T05:54:49.570105+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":188,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7107,"log_dups_size":0,"ondisk_log_size":7107,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146112512,"num_objects":2230,"num_object_clones":0,"num_object_copies":6690,"num_objects_missing_on_primary":581,"num_objects_missing":581,"num_objects_degraded":3013,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3843,"num_write_kb":194304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":596,"num_bytes_recovered":39059456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,3],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,7","objects":1643},{"shards":"0,3","objects":6},{"shards":"2,7","objects":581}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":255,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.362129+0000","last_change":"2023-10-09T05:55:59.376804+0000","last_active":"2023-10-09T05:56:05.362129+0000","last_peered":"2023-10-09T05:56:05.362129+0000","last_clean":"2023-10-09T05:56:05.362129+0000","last_became_active":"2023-10-09T05:55:59.355912+0000","last_became_peered":"2023-10-09T05:55:59.355912+0000","last_unstale":"2023-10-09T05:56:05.362129+0000","last_undegraded":"2023-10-09T05:56:05.362129+0000","last_fullsized":"2023-10-09T05:56:05.362129+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:15.551718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":249,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.377750+0000","last_change":"2023-10-09T05:55:59.336237+0000","last_active":"2023-10-09T05:56:05.377750+0000","last_peered":"2023-10-09T05:56:05.377750+0000","last_clean":"2023-10-09T05:56:05.377750+0000","last_became_active":"2023-10-09T05:55:59.335912+0000","last_became_peered":"2023-10-09T05:55:59.335912+0000","last_unstale":"2023-10-09T05:56:05.377750+0000","last_undegraded":"2023-10-09T05:56:05.377750+0000","last_fullsized":"2023-10-09T05:56:05.377750+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:07.757931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"223'7066","reported_seq":7217,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:56:09.779187+0000","last_change":"2023-10-09T05:54:45.671348+0000","last_active":"2023-10-09T05:56:09.779187+0000","last_peered":"2023-10-09T05:56:09.779187+0000","last_clean":"2023-10-09T05:56:09.779187+0000","last_became_active":"2023-10-09T05:54:45.664954+0000","last_became_peered":"2023-10-09T05:54:45.664954+0000","last_unstale":"2023-10-09T05:56:09.779187+0000","last_undegraded":"2023-10-09T05:56:09.779187+0000","last_fullsized":"2023-10-09T05:56:09.779187+0000","mapping_epoch":188,"log_start":"106'122","ondisk_log_start":"106'122","created":188,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6944,"log_dups_size":0,"ondisk_log_size":6944,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140705792,"num_objects":2147,"num_object_clones":0,"num_object_copies":6441,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2147,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3819,"num_write_kb":190912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":168,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.361412+0000","last_change":"2023-10-09T05:40:12.633195+0000","last_active":"2023-10-09T05:56:05.361412+0000","last_peered":"2023-10-09T05:56:05.361412+0000","last_clean":"2023-10-09T05:56:05.361412+0000","last_became_active":"2023-10-09T05:40:12.632942+0000","last_became_peered":"2023-10-09T05:40:12.632942+0000","last_unstale":"2023-10-09T05:56:05.361412+0000","last_undegraded":"2023-10-09T05:56:05.361412+0000","last_fullsized":"2023-10-09T05:56:05.361412+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:04.470685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":216,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.363734+0000","last_change":"2023-10-09T05:51:57.093104+0000","last_active":"2023-10-09T05:56:05.363734+0000","last_peered":"2023-10-09T05:56:05.363734+0000","last_clean":"2023-10-09T05:56:05.363734+0000","last_became_active":"2023-10-09T05:51:57.092810+0000","last_became_peered":"2023-10-09T05:51:57.092810+0000","last_unstale":"2023-10-09T05:56:05.363734+0000","last_undegraded":"2023-10-09T05:56:05.363734+0000","last_fullsized":"2023-10-09T05:56:05.363734+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:51.192227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.1b","version":"223'7388","reported_seq":23278,"reported_epoch":223,"state":"active+recovering","last_fresh":"2023-10-09T05:56:09.680186+0000","last_change":"2023-10-09T05:56:00.465977+0000","last_active":"2023-10-09T05:56:09.680186+0000","last_peered":"2023-10-09T05:56:09.680186+0000","last_clean":"2023-10-09T05:54:47.668795+0000","last_became_active":"2023-10-09T05:56:00.422122+0000","last_became_peered":"2023-10-09T05:56:00.422122+0000","last_unstale":"2023-10-09T05:56:09.680186+0000","last_undegraded":"2023-10-09T05:56:09.680186+0000","last_fullsized":"2023-10-09T05:56:09.680186+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7388,"log_dups_size":0,"ondisk_log_size":7388,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144015360,"num_objects":2198,"num_object_clones":0,"num_object_copies":6594,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1226,"num_objects_unfound":0,"num_objects_dirty":2198,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3906,"num_write_kb":195296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4560,"num_bytes_recovered":284196864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":["3","6","7"],"object_location_counts":[{"shards":"1,3,6","objects":1226},{"shards":"3,4,6","objects":972}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":140,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:56:04.357448+0000","last_change":"2023-10-09T05:55:59.328124+0000","last_active":"2023-10-09T05:56:04.357448+0000","last_peered":"2023-10-09T05:56:04.357448+0000","last_clean":"2023-10-09T05:56:04.357448+0000","last_became_active":"2023-10-09T05:55:59.327740+0000","last_became_peered":"2023-10-09T05:55:59.327740+0000","last_unstale":"2023-10-09T05:56:04.357448+0000","last_undegraded":"2023-10-09T05:56:04.357448+0000","last_fullsized":"2023-10-09T05:56:04.357448+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:35.523860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":211,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T05:56:04.351939+0000","last_change":"2023-10-09T05:52:42.261890+0000","last_active":"2023-10-09T05:56:04.351939+0000","last_peered":"2023-10-09T05:56:04.351939+0000","last_clean":"2023-10-09T05:56:04.351939+0000","last_became_active":"2023-10-09T05:52:42.261521+0000","last_became_peered":"2023-10-09T05:52:42.261521+0000","last_unstale":"2023-10-09T05:56:04.351939+0000","last_undegraded":"2023-10-09T05:56:04.351939+0000","last_fullsized":"2023-10-09T05:56:04.351939+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:59.208145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.1a","version":"223'7235","reported_seq":9573,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T05:56:09.229055+0000","last_change":"2023-10-09T05:55:22.219965+0000","last_active":"2023-10-09T05:56:09.229055+0000","last_peered":"2023-10-09T05:56:09.229055+0000","last_clean":"2023-10-09T05:56:09.229055+0000","last_became_active":"2023-10-09T05:55:22.214658+0000","last_became_peered":"2023-10-09T05:55:22.214658+0000","last_unstale":"2023-10-09T05:56:09.229055+0000","last_undegraded":"2023-10-09T05:56:09.229055+0000","last_fullsized":"2023-10-09T05:56:09.229055+0000","mapping_epoch":207,"log_start":"108'2184","ondisk_log_start":"108'2184","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5051,"log_dups_size":0,"ondisk_log_size":5051,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144310272,"num_objects":2202,"num_object_clones":0,"num_object_copies":6606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3811,"num_write_kb":192416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":543,"num_bytes_recovered":35553280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":174,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.359240+0000","last_change":"2023-10-09T05:52:23.654749+0000","last_active":"2023-10-09T05:56:05.359240+0000","last_peered":"2023-10-09T05:56:05.359240+0000","last_clean":"2023-10-09T05:56:05.359240+0000","last_became_active":"2023-10-09T05:52:23.643044+0000","last_became_peered":"2023-10-09T05:52:23.643044+0000","last_unstale":"2023-10-09T05:56:05.359240+0000","last_undegraded":"2023-10-09T05:56:05.359240+0000","last_fullsized":"2023-10-09T05:56:05.359240+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:23:08.898078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":232,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T05:56:05.363403+0000","last_change":"2023-10-09T05:55:59.300656+0000","last_active":"2023-10-09T05:56:05.363403+0000","last_peered":"2023-10-09T05:56:05.363403+0000","last_clean":"2023-10-09T05:56:05.363403+0000","last_became_active":"2023-10-09T05:55:59.300240+0000","last_became_peered":"2023-10-09T05:55:59.300240+0000","last_unstale":"2023-10-09T05:56:05.363403+0000","last_undegraded":"2023-10-09T05:56:05.363403+0000","last_fullsized":"2023-10-09T05:56:05.363403+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:53.634777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":36,"stat_sum":{"num_bytes":4583260184,"num_objects":69936,"num_object_clones":0,"num_object_copies":209808,"num_objects_missing_on_primary":1549,"num_objects_missing":1549,"num_objects_degraded":34975,"num_objects_misplaced":17630,"num_objects_unfound":0,"num_objects_dirty":69936,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":123146,"num_write_kb":6178561,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45969,"num_bytes_recovered":2922119192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15907962880,"data_stored":15907946592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":221035,"ondisk_log_size":221035,"up":108,"acting":100,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":10238,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":2624160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234198,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2470896,"kb_used_data":2165416,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":91274256,"statfs":{"total":95995035648,"available":93464838144,"internally_reserved":0,"allocated":2217385984,"data_stored":2216556103,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":1.1599999999999999,"15min":1.8129999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":1.98,"5min":20.247,"15min":20.247},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.88700000000000001,"5min":1.165,"15min":1.704},"min":{"1min":0.313,"5min":0.252,"15min":0.252},"max":{"1min":3.9740000000000002,"5min":20.236000000000001,"15min":20.236000000000001},"last":0.438}]},{"osd":1,"last update":"Mon Oct 9 05:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.875,"15min":0.64100000000000001},"min":{"1min":0.32800000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.76,"5min":7.9859999999999998,"15min":7.9859999999999998},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.82099999999999995,"15min":0.64200000000000002},"min":{"1min":0.33300000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.9990000000000001,"5min":5.8129999999999997,"15min":5.8129999999999997},"last":0.34999999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":2.2229999999999999,"15min":1.1499999999999999},"min":{"1min":0.28799999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.8860000000000001,"5min":50.57,"15min":50.57},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.998,"5min":2.2440000000000002,"15min":1.2130000000000001},"min":{"1min":0.35999999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":4.2220000000000004,"5min":50.536999999999999,"15min":50.536999999999999},"last":0.37}]},{"osd":3,"last update":"Mon Oct 9 05:55:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":5.6600000000000001,"15min":2.2970000000000002},"min":{"1min":0.38,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.458,"5min":147.54900000000001,"15min":147.54900000000001},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.61599999999999999,"5min":5.5579999999999998,"15min":2.2440000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":1.9590000000000001,"5min":144.065,"15min":144.065},"last":0.36499999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:56:07 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.68600000000000005,"15min":0.54800000000000004},"min":{"1min":0.28599999999999998,"5min":0.248,"15min":0.24399999999999999},"max":{"1min":1.589,"5min":9.5329999999999995,"15min":9.5329999999999995},"last":0.311},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.89800000000000002,"15min":0.63200000000000001},"min":{"1min":0.33100000000000002,"5min":0.29099999999999998,"15min":0.27400000000000002},"max":{"1min":1.877,"5min":7.6120000000000001,"15min":7.6120000000000001},"last":0.38100000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:56:07 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.94399999999999995,"15min":0.64200000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.7970000000000002,"5min":10.795999999999999,"15min":10.795999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.69799999999999995,"15min":0.59799999999999998},"min":{"1min":0.315,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.365,"5min":3.9380000000000002,"15min":3.9380000000000002},"last":0.49299999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:56:07 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":2.4889999999999999,"15min":1.1890000000000001},"min":{"1min":0.33100000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":2.46,"5min":131.94399999999999,"15min":131.94399999999999},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.63200000000000001,"5min":2.5369999999999999,"15min":1.218},"min":{"1min":0.35899999999999999,"5min":0.33100000000000002,"15min":0.29999999999999999},"max":{"1min":1.931,"5min":135.23400000000001,"15min":135.23400000000001},"last":0.40500000000000003}]}]},{"osd":6,"up_from":46,"seq":197568495834,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2338908,"kb_used_data":2033428,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":91406244,"statfs":{"total":95995035648,"available":93599993856,"internally_reserved":0,"allocated":2082230272,"data_stored":2081413084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:56:01 2023","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":0.85499999999999998,"15min":1.0589999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":3.1339999999999999,"5min":7.0599999999999996,"15min":7.0599999999999996},"last":0.254},{"interface":"front","average":{"1min":0.877,"5min":0.88600000000000001,"15min":1.1140000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":3.0259999999999998,"5min":6.6459999999999999,"15min":6.6459999999999999},"last":0.31}]},{"osd":1,"last update":"Mon Oct 9 05:55:40 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":0.84599999999999997,"15min":0.67400000000000004},"min":{"1min":0.29499999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.1339999999999999,"5min":3.7949999999999999,"15min":3.7949999999999999},"last":0.373},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.95299999999999996,"15min":0.70699999999999996},"min":{"1min":0.29899999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.2130000000000001,"5min":9.9670000000000005,"15min":9.9670000000000005},"last":0.28999999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:56:01 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.1639999999999999,"15min":0.747},"min":{"1min":0.29299999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.3520000000000003,"5min":6.6859999999999999,"15min":6.6859999999999999},"last":0.27500000000000002},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.1639999999999999,"15min":0.75900000000000001},"min":{"1min":0.32000000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.4900000000000002,"5min":9.9390000000000001,"15min":9.9390000000000001},"last":0.35499999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:55:40 2023","interfaces":[{"interface":"back","average":{"1min":9.6150000000000002,"5min":3.0600000000000001,"15min":1.389},"min":{"1min":0.33300000000000002,"5min":0.311,"15min":0.311},"max":{"1min":93.626000000000005,"5min":93.626000000000005,"15min":93.626000000000005},"last":0.41499999999999998},{"interface":"front","average":{"1min":9.2959999999999994,"5min":2.8879999999999999,"15min":1.3480000000000001},"min":{"1min":0.32500000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":93.643000000000001,"5min":93.643000000000001,"15min":93.643000000000001},"last":0.43099999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:55:45 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.93999999999999995,"15min":0.624},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":2.9159999999999999,"5min":7.9100000000000001,"15min":7.9100000000000001},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.755,"5min":0.90600000000000003,"15min":0.63900000000000001},"min":{"1min":0.32600000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":2.3450000000000002,"5min":7.3300000000000001,"15min":7.3300000000000001},"last":0.53600000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:55:45 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.91700000000000004,"15min":0.63600000000000001},"min":{"1min":0.316,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.875,"5min":13.879,"15min":13.879},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.93799999999999994,"5min":1.0249999999999999,"15min":0.69999999999999996},"min":{"1min":0.35099999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.9429999999999996,"5min":7.29,"15min":7.29},"last":0.56200000000000006}]},{"osd":7,"last update":"Mon Oct 9 05:55:10 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":1.2370000000000001,"15min":0.76700000000000002},"min":{"1min":0.33800000000000002,"5min":0.33400000000000002,"15min":0.316},"max":{"1min":0.72799999999999998,"5min":19.914999999999999,"15min":19.914999999999999},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.50600000000000001,"5min":1.075,"15min":0.76200000000000001},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.35799999999999998},"max":{"1min":0.71299999999999997,"5min":7.0309999999999997,"15min":7.0309999999999997},"last":0.61499999999999999}]}]},{"osd":1,"up_from":0,"seq":382252089423,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":129,"seq":554050781225,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1195184,"kb_used_data":1089256,"kb_used_omap":29,"kb_used_meta":105826,"kb_avail":92549968,"statfs":{"total":95995035648,"available":94771167232,"internally_reserved":0,"allocated":1115398144,"data_stored":1114164507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30322,"internal_metadata":108366222},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2,2,2,4,4,2],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:56:02 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.81699999999999995,"15min":0.73999999999999999},"min":{"1min":0.249,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.6429999999999998,"5min":9.1720000000000006,"15min":9.1720000000000006},"last":0.44},{"interface":"front","average":{"1min":0.78100000000000003,"5min":0.92400000000000004,"15min":1.073},"min":{"1min":0.29599999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.411,"5min":4.0049999999999999,"15min":4.0049999999999999},"last":0.45400000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:56:02 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.81899999999999995,"15min":0.96499999999999997},"min":{"1min":0.23599999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":2.456,"5min":3.9199999999999999,"15min":3.9199999999999999},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.755,"5min":0.86499999999999999,"15min":0.96099999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.5599999999999996,"5min":5.4160000000000004,"15min":5.4160000000000004},"last":2.956}]},{"osd":3,"last update":"Mon Oct 9 05:56:02 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":0.80000000000000004,"15min":0.78300000000000003},"min":{"1min":0.30399999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.6609999999999996,"5min":9.6609999999999996,"15min":9.6609999999999996},"last":0.81799999999999995},{"interface":"front","average":{"1min":1.01,"5min":0.84299999999999997,"15min":0.88100000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":7.1749999999999998,"5min":7.1749999999999998,"15min":7.1749999999999998},"last":3.0030000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:56:02 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.97899999999999998,"15min":1.141},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.089,"5min":9.0050000000000008,"15min":9.0050000000000008},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.79900000000000004,"5min":0.78300000000000003,"15min":0.86299999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.645,"5min":3.1549999999999998,"15min":3.1549999999999998},"last":0.51200000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:56:02 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.97099999999999997,"15min":1.089},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.4850000000000003,"5min":7.3959999999999999,"15min":7.3959999999999999},"last":3.056},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.89600000000000002,"15min":0.90800000000000003},"min":{"1min":0.47099999999999997,"5min":0.439,"15min":0.439},"max":{"1min":1.1279999999999999,"5min":9.1080000000000005,"15min":9.1080000000000005},"last":0.49099999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:56:02 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":0.99199999999999999,"15min":1.226},"min":{"1min":0.378,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":4.5279999999999996,"5min":6.6849999999999996,"15min":6.6849999999999996},"last":3.036},{"interface":"front","average":{"1min":0.81799999999999995,"5min":1.03,"15min":1.1599999999999999},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":3.7890000000000001,"5min":9.141,"15min":9.141},"last":0.79400000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:56:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.1950000000000001,"15min":1.345},"min":{"1min":0.438,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":5.548,"5min":9.0700000000000003,"15min":9.0700000000000003},"last":0.74099999999999999},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.0920000000000001,"15min":1.3009999999999999},"min":{"1min":0.44900000000000001,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":4.431,"5min":5.8730000000000002,"15min":5.8730000000000002},"last":0.623}]}]},{"osd":2,"up_from":94,"seq":403726925914,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3264772,"kb_used_data":3088700,"kb_used_omap":81,"kb_used_meta":175982,"kb_avail":90480380,"statfs":{"total":95995035648,"available":92651909120,"internally_reserved":0,"allocated":3162828800,"data_stored":3161682784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":180205786},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:56:07 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":1.1020000000000001,"15min":1.4990000000000001},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":7.9889999999999999,"5min":11.444000000000001,"15min":11.444000000000001},"last":4.2800000000000002},{"interface":"front","average":{"1min":0.78500000000000003,"5min":1.008,"15min":1.3759999999999999},"min":{"1min":0.254,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":2.6840000000000002,"5min":6.883,"15min":6.883},"last":2.5310000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":2.9399999999999999,"15min":1.262},"min":{"1min":0.255,"5min":0.218,"15min":0.218},"max":{"1min":2.7789999999999999,"5min":163.24199999999999,"15min":163.24199999999999},"last":3.859},{"interface":"front","average":{"1min":0.65000000000000002,"5min":2.7210000000000001,"15min":1.1759999999999999},"min":{"1min":0.28399999999999997,"5min":0.214,"15min":0.214},"max":{"1min":4.1299999999999999,"5min":163.27000000000001,"15min":163.27000000000001},"last":3.9569999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:55:31 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":3.0129999999999999,"15min":1.2729999999999999},"min":{"1min":0.25900000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.85199999999999998,"5min":170.07599999999999,"15min":170.07599999999999},"last":3.0529999999999999},{"interface":"front","average":{"1min":0.80700000000000005,"5min":1.7310000000000001,"15min":0.871},"min":{"1min":0.33300000000000002,"5min":0.25,"15min":0.25},"max":{"1min":4.218,"5min":63.600000000000001,"15min":63.600000000000001},"last":1.075}]},{"osd":4,"last update":"Mon Oct 9 05:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":1.206,"15min":0.84799999999999998},"min":{"1min":0.39400000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.552,"5min":11.372999999999999,"15min":11.372999999999999},"last":1.276},{"interface":"front","average":{"1min":0.873,"5min":1.1319999999999999,"15min":0.84599999999999997},"min":{"1min":0.33200000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.5519999999999996,"5min":6.5709999999999997,"15min":6.5709999999999997},"last":2.581}]},{"osd":5,"last update":"Mon Oct 9 05:55:59 2023","interfaces":[{"interface":"back","average":{"1min":1.256,"5min":1.2210000000000001,"15min":0.85599999999999998},"min":{"1min":0.371,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":8.8699999999999992,"5min":13.414,"15min":13.414},"last":2.9729999999999999},{"interface":"front","average":{"1min":0.753,"5min":1.2470000000000001,"15min":0.85799999999999998},"min":{"1min":0.42399999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.52,"5min":13.438000000000001,"15min":13.438000000000001},"last":0.313}]},{"osd":6,"last update":"Mon Oct 9 05:55:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":1.2809999999999999,"15min":0.86599999999999999},"min":{"1min":0.40300000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":8.6920000000000002,"5min":11.59,"15min":11.59},"last":2.625},{"interface":"front","average":{"1min":0.80400000000000005,"5min":0.97499999999999998,"15min":0.78200000000000003},"min":{"1min":0.37,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":2.0190000000000001,"5min":6.4710000000000001,"15min":6.4710000000000001},"last":3.0169999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:55:59 2023","interfaces":[{"interface":"back","average":{"1min":1.788,"5min":1.5720000000000001,"15min":0.96899999999999997},"min":{"1min":0.46999999999999997,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":10.715999999999999,"5min":40.404000000000003,"15min":40.404000000000003},"last":3.085},{"interface":"front","average":{"1min":0.80500000000000005,"5min":1.4350000000000001,"15min":0.94599999999999995},"min":{"1min":0.49199999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":2.6240000000000001,"5min":40.113,"15min":40.113},"last":0.35599999999999998}]}]},{"osd":3,"up_from":99,"seq":425201762388,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2846812,"kb_used_data":2680724,"kb_used_omap":7,"kb_used_meta":166072,"kb_avail":90898340,"statfs":{"total":95995035648,"available":93079900160,"internally_reserved":0,"allocated":2745061376,"data_stored":2743781654,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":170058344},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:56:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.2350000000000001,"15min":1.4430000000000001},"min":{"1min":0.22900000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":9.4510000000000005,"5min":15.536,"15min":15.536},"last":0.39600000000000002},{"interface":"front","average":{"1min":2.4660000000000002,"5min":1.3979999999999999,"15min":1.528},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":14.707000000000001,"5min":14.707000000000001,"15min":14.707000000000001},"last":0.28499999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:55:29 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":2.2010000000000001,"15min":0.97999999999999998},"min":{"1min":0.25,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":6.2329999999999997,"5min":127.226,"15min":127.226},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.83599999999999997,"5min":2.274,"15min":1.0229999999999999},"min":{"1min":0.24399999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":5.1559999999999997,"5min":127.468,"15min":127.468},"last":0.438}]},{"osd":2,"last update":"Mon Oct 9 05:55:29 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":1.3700000000000001,"15min":0.70699999999999996},"min":{"1min":0.29999999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.1160000000000001,"5min":62.938000000000002,"15min":62.938000000000002},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.94799999999999995,"5min":1.78,"15min":0.88100000000000001},"min":{"1min":0.317,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":5.5220000000000002,"5min":62.43,"15min":62.43},"last":0.53800000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:55:29 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.91900000000000004,"15min":0.70699999999999996},"min":{"1min":0.34899999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.95999999999999996,"5min":7.7439999999999998,"15min":7.7439999999999998},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.109,"5min":0.91300000000000003,"15min":0.71499999999999997},"min":{"1min":0.371,"5min":0.311,"15min":0.311},"max":{"1min":6.3250000000000002,"5min":6.3250000000000002,"15min":6.3250000000000002},"last":0.505}]},{"osd":5,"last update":"Mon Oct 9 05:55:29 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":0.95399999999999996,"15min":0.71999999999999997},"min":{"1min":0.40999999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":6.274,"5min":6.274,"15min":6.274},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.83599999999999997,"15min":0.70299999999999996},"min":{"1min":0.41399999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.29,"5min":4.2839999999999998,"15min":4.2839999999999998},"last":0.36199999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:55:29 2023","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":1.069,"15min":0.77800000000000002},"min":{"1min":0.42999999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.298,"5min":12.798999999999999,"15min":12.798999999999999},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.99099999999999999,"15min":0.78300000000000003},"min":{"1min":0.44800000000000001,"5min":0.379,"15min":0.379},"max":{"1min":1.2110000000000001,"5min":12.773,"15min":12.773},"last":0.42099999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:55:29 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":1.0009999999999999,"15min":0.753},"min":{"1min":0.42899999999999999,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":2.4510000000000001,"5min":7.4580000000000002,"15min":7.4580000000000002},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.85599999999999998,"5min":1.03,"15min":0.78100000000000003},"min":{"1min":0.41399999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":2.9900000000000002,"5min":7.8399999999999999,"15min":7.8399999999999999},"last":0.68899999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019108,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1755440,"kb_used_data":1454056,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":91989712,"statfs":{"total":95995035648,"available":94197465088,"internally_reserved":0,"allocated":1488953344,"data_stored":1488123716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:56:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1950000000000001,"5min":0.96499999999999997,"15min":0.84299999999999997},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":13.503,"5min":15.173999999999999,"15min":15.173999999999999},"last":0.249},{"interface":"front","average":{"1min":1.4410000000000001,"5min":1.2689999999999999,"15min":1.4450000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":13.561999999999999,"5min":15.125999999999999,"15min":15.125999999999999},"last":0.29299999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:55:31 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":1,"15min":0.71499999999999997},"min":{"1min":0.28499999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.64500000000000002,"5min":9.2390000000000008,"15min":9.2390000000000008},"last":0.31},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.95999999999999996,"15min":0.66800000000000004},"min":{"1min":0.30099999999999999,"5min":0.255,"15min":0.255},"max":{"1min":6.7779999999999996,"5min":8.2629999999999999,"15min":8.2629999999999999},"last":0.45600000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:56:04 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":1.1279999999999999,"15min":0.85899999999999999},"min":{"1min":0.32500000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.6799999999999999,"5min":6.9240000000000004,"15min":33.262},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.79200000000000004,"5min":1.21,"15min":0.95299999999999996},"min":{"1min":0.36499999999999999,"5min":0.313,"15min":0.313},"max":{"1min":6.1760000000000002,"5min":9.0030000000000001,"15min":33.240000000000002},"last":5.117}]},{"osd":3,"last update":"Mon Oct 9 05:55:31 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.524,"15min":0.90000000000000002},"min":{"1min":0.30599999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.6440000000000001,"5min":40.531999999999996,"15min":40.531999999999996},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.56599999999999995,"5min":1.5980000000000001,"15min":0.93300000000000005},"min":{"1min":0.36099999999999999,"5min":0.31,"15min":0.31},"max":{"1min":1.341,"5min":39.905999999999999,"15min":39.905999999999999},"last":2.7810000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:55:22 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.79100000000000004,"15min":0.57899999999999996},"min":{"1min":0.29599999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.70099999999999996,"5min":10.012,"15min":10.012},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.435,"5min":0.97599999999999998,"15min":0.67400000000000004},"min":{"1min":0.30099999999999999,"5min":0.254,"15min":0.254},"max":{"1min":0.60399999999999998,"5min":12.489000000000001,"15min":12.489000000000001},"last":0.41199999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:55:39 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.96599999999999997,"15min":0.66300000000000003},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.27400000000000002},"max":{"1min":1.327,"5min":11.234999999999999,"15min":11.234999999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.72199999999999998,"15min":0.60599999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30299999999999999},"max":{"1min":1.0289999999999999,"5min":9.923,"15min":9.923},"last":0.39600000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:56:08 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":1.147,"15min":0.77700000000000002},"min":{"1min":0.32500000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.796,"5min":10.606,"15min":10.606},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.878,"15min":0.68500000000000005},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.363,"5min":8.3819999999999997,"15min":8.3819999999999997},"last":0.47699999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757471,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2058088,"kb_used_data":1752608,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":91687064,"statfs":{"total":95995035648,"available":93887553536,"internally_reserved":0,"allocated":1794670592,"data_stored":1793845468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:56:09 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":0.59199999999999997,"15min":0.66800000000000004},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":6.9530000000000003,"5min":6.9530000000000003,"15min":6.9530000000000003},"last":1.1879999999999999},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.73799999999999999,"15min":0.77800000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.9020000000000001,"5min":6.9020000000000001,"15min":6.9020000000000001},"last":1.198}]},{"osd":1,"last update":"Mon Oct 9 05:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":3.5350000000000001,"15min":1.536},"min":{"1min":0.314,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.4660000000000002,"5min":114.247,"15min":114.247},"last":2.1179999999999999},{"interface":"front","average":{"1min":0.46200000000000002,"5min":3.3399999999999999,"15min":1.4890000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":1.091,"5min":114.678,"15min":114.678},"last":1.216}]},{"osd":2,"last update":"Mon Oct 9 05:55:57 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":2.4380000000000002,"15min":1.2},"min":{"1min":0.314,"5min":0.25,"15min":0.25},"max":{"1min":3.915,"5min":106.66500000000001,"15min":106.66500000000001},"last":1.2330000000000001},{"interface":"front","average":{"1min":0.59999999999999998,"5min":2.2719999999999998,"15min":1.1479999999999999},"min":{"1min":0.29799999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.4910000000000001,"5min":106.779,"15min":106.779},"last":4.194}]},{"osd":3,"last update":"Mon Oct 9 05:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":8.9670000000000005,"15min":3.3500000000000001},"min":{"1min":0.33600000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.0289999999999999,"5min":272.81299999999999,"15min":272.81299999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.61099999999999999,"5min":9.3320000000000007,"15min":3.5390000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":2.3220000000000001,"5min":273.00999999999999,"15min":273.00999999999999},"last":0.88500000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:55:18 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.70299999999999996,"15min":0.54800000000000004},"min":{"1min":0.28499999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":2.093,"5min":6.2050000000000001,"15min":6.2050000000000001},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.69099999999999995,"15min":0.56799999999999995},"min":{"1min":0.32300000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.7250000000000001,"5min":13.050000000000001,"15min":13.050000000000001},"last":0.33800000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:55:33 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.71799999999999997,"15min":0.58399999999999996},"min":{"1min":0.30199999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.0030000000000001,"5min":10.606999999999999,"15min":10.606999999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.92400000000000004,"15min":0.68300000000000005},"min":{"1min":0.34399999999999997,"5min":0.30299999999999999,"15min":0.28499999999999998},"max":{"1min":5.0609999999999999,"5min":5.9770000000000003,"15min":5.9770000000000003},"last":0.35299999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:55:13 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":1.056,"15min":0.73999999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.5419999999999998,"5min":9.2739999999999991,"15min":9.2739999999999991},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.70599999999999996,"15min":0.60999999999999999},"min":{"1min":0.35899999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.0349999999999999,"5min":5.032,"15min":5.032},"last":0.42399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1474,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1111232512,"data_stored":1111228440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1355419648,"data_stored":1355415576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3156480000,"data_stored":3156475928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2731737088,"data_stored":2731737088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1485701120,"data_stored":1485701120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1783566336,"data_stored":1783562264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2075262976,"data_stored":2075262976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2208563200,"data_stored":2208563200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:56:13.503 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_0 pgp_num 36 2023-10-09T05:56:13.594 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:13.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:13.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:13.704 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:13.736 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:13.796 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:13.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:13.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:13.901 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:13.929 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:13.990 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:13.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:13.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:14.095 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:56:14.144 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:14.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:14 smithi156 ceph-mon[195101]: pgmap v513: 129 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 109 active+clean, 1 active+recovering, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 333 op/s; 34975/208935 objects degraded (16.740%); 17467/208935 objects misplaced (8.360%); 18 MiB/s, 286 objects/s recovering 2023-10-09T05:56:14.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:56:14.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:56:14.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:14 smithi064 ceph-mon[212750]: pgmap v513: 129 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 109 active+clean, 1 active+recovering, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 333 op/s; 34975/208935 objects degraded (16.740%); 17467/208935 objects misplaced (8.360%); 18 MiB/s, 286 objects/s recovering 2023-10-09T05:56:14.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:56:14.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:56:14.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:14 smithi064 ceph-mon[220044]: pgmap v513: 129 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 109 active+clean, 1 active+recovering, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 333 op/s; 34975/208935 objects degraded (16.740%); 17467/208935 objects misplaced (8.360%); 18 MiB/s, 286 objects/s recovering 2023-10-09T05:56:14.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:56:14.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:56:14.734 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:56:14.735 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:56:14.735 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:56:14.735 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:56:14.735 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:56:14.735 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:56:14.735 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:56:14.735 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:56:14.739 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:56:14.739 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:56:14.739 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:56:14.739 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:56:14.739 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:56:14.739 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:56:14.740 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:56:14.740 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:56:15.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:15 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/508578884' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:15.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:56:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:56:15] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.43.0" 2023-10-09T05:56:15.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:15 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/508578884' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:15.576 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:15.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:15.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:15.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:15 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/508578884' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:15.681 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:15.708 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:15.752 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:15.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:15.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:15.856 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:15.884 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:15.934 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:15.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:15.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:16.040 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:16.117 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:16.289 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:16.319 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:16.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:16.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:16.423 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:16.458 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:16 smithi156 ceph-mon[195101]: pgmap v514: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 109 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 32033/204798 objects degraded (15.641%); 20813/204798 objects misplaced (10.163%); 19 MiB/s, 329 objects/s recovering 2023-10-09T05:56:16.461 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:16.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:16 smithi064 ceph-mon[220044]: pgmap v514: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 109 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 32033/204798 objects degraded (15.641%); 20813/204798 objects misplaced (10.163%); 19 MiB/s, 329 objects/s recovering 2023-10-09T05:56:16.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:16 smithi064 ceph-mon[212750]: pgmap v514: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 109 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 32033/204798 objects degraded (15.641%); 20813/204798 objects misplaced (10.163%); 19 MiB/s, 329 objects/s recovering 2023-10-09T05:56:16.519 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:16.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:16.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:16.623 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:16.653 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:16.712 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:16.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:16.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:16.822 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:16.875 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:17.001 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:17.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:17.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:17.106 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:17.145 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:17.205 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:17.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:17.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:17.228 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T05:56:17.308 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:17.349 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:17.416 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:17.417 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:17 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/626574879' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:17.417 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:17 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 32033/204798 objects degraded (15.641%), 14 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:17.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:17.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:17.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:17 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/626574879' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:17.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:17 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 32033/204798 objects degraded (15.641%), 14 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:17.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:17 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/626574879' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:17.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:17 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 32033/204798 objects degraded (15.641%), 14 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:17.519 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:17.560 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:17.621 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:17.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:17.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:17.727 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:17.773 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:17.921 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:56:18.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:18 smithi064 ceph-mon[212750]: pgmap v515: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 109 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 452 op/s; 31966/203004 objects degraded (15.746%); 20115/203004 objects misplaced (9.909%); 22 MiB/s, 373 objects/s recovering 2023-10-09T05:56:18.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:18 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:18.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:18 smithi064 ceph-mon[212750]: failed to encode map e224 with expected crc 2023-10-09T05:56:18.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:18 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/626574879' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:56:18.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:18 smithi064 ceph-mon[212750]: osdmap e224: 8 total, 8 up, 7 in 2023-10-09T05:56:18.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:18 smithi064 ceph-mon[212750]: failed to encode map e224 with expected crc 2023-10-09T05:56:18.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:18 smithi064 ceph-mon[212750]: failed to encode map e224 with expected crc 2023-10-09T05:56:18.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:18 smithi064 ceph-mon[220044]: pgmap v515: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 109 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 452 op/s; 31966/203004 objects degraded (15.746%); 20115/203004 objects misplaced (9.909%); 22 MiB/s, 373 objects/s recovering 2023-10-09T05:56:18.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:18 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:18.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:18 smithi064 ceph-mon[220044]: failed to encode map e224 with expected crc 2023-10-09T05:56:18.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:18 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/626574879' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:56:18.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:18 smithi064 ceph-mon[220044]: osdmap e224: 8 total, 8 up, 7 in 2023-10-09T05:56:18.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:18 smithi064 ceph-mon[220044]: failed to encode map e224 with expected crc 2023-10-09T05:56:18.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:18 smithi064 ceph-mon[220044]: failed to encode map e224 with expected crc 2023-10-09T05:56:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:18 smithi156 ceph-mon[195101]: pgmap v515: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 109 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 452 op/s; 31966/203004 objects degraded (15.746%); 20115/203004 objects misplaced (9.909%); 22 MiB/s, 373 objects/s recovering 2023-10-09T05:56:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:18 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:18 smithi156 ceph-mon[195101]: failed to encode map e224 with expected crc 2023-10-09T05:56:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:18 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/626574879' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:56:18.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:18 smithi156 ceph-mon[195101]: osdmap e224: 8 total, 8 up, 7 in 2023-10-09T05:56:18.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:18 smithi156 ceph-mon[195101]: failed to encode map e224 with expected crc 2023-10-09T05:56:18.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:18 smithi156 ceph-mon[195101]: failed to encode map e224 with expected crc 2023-10-09T05:56:18.696 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:18.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:18.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:18.805 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:18.858 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:18.926 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:18.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:18.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:19.028 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:19.076 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:19.163 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:19.354 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:19.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:19.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:19.464 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:19.523 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:19.677 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:19.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:19.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:19.788 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:19.859 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:19.955 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:19.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:19.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:19.982 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T05:56:19.982 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:56:20.062 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:20.106 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:20.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:20 smithi064 ceph-mon[212750]: pgmap v517: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 109 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 466 op/s; 31966/202233 objects degraded (15.807%); 19998/202233 objects misplaced (9.889%); 22 MiB/s, 373 objects/s recovering 2023-10-09T05:56:20.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:20 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2909476303' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "36"}]: dispatch 2023-10-09T05:56:20.224 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:20 smithi064 ceph-mon[220044]: pgmap v517: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 109 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 466 op/s; 31966/202233 objects degraded (15.807%); 19998/202233 objects misplaced (9.889%); 22 MiB/s, 373 objects/s recovering 2023-10-09T05:56:20.224 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:20 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2909476303' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "36"}]: dispatch 2023-10-09T05:56:20.225 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:20 smithi156 ceph-mon[195101]: pgmap v517: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 109 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 466 op/s; 31966/202233 objects degraded (15.807%); 19998/202233 objects misplaced (9.889%); 22 MiB/s, 373 objects/s recovering 2023-10-09T05:56:20.225 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:20 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2909476303' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "36"}]: dispatch 2023-10-09T05:56:20.226 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 6 pgp_num to 36 2023-10-09T05:56:20.230 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:20.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:20.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:20.334 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:56:20.384 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:20.569 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:20.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:20.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:20.673 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:20.727 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:20.907 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:56:21.075 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:21.080 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T05:56:21.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:21.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:21.183 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:21.216 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:21.277 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:21.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:21.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:21.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:21 smithi156 ceph-mon[195101]: failed to encode map e225 with expected crc 2023-10-09T05:56:21.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:21 smithi156 ceph-mon[195101]: failed to encode map e225 with expected crc 2023-10-09T05:56:21.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:21 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2909476303' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "36"}]': finished 2023-10-09T05:56:21.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:21 smithi156 ceph-mon[195101]: osdmap e225: 8 total, 8 up, 7 in 2023-10-09T05:56:21.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:21 smithi156 ceph-mon[195101]: failed to encode map e225 with expected crc 2023-10-09T05:56:21.396 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:21.443 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:21.508 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:21.509 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:21 smithi064 ceph-mon[220044]: failed to encode map e225 with expected crc 2023-10-09T05:56:21.509 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:21 smithi064 ceph-mon[220044]: failed to encode map e225 with expected crc 2023-10-09T05:56:21.509 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:21 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2909476303' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "36"}]': finished 2023-10-09T05:56:21.509 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:21 smithi064 ceph-mon[220044]: osdmap e225: 8 total, 8 up, 7 in 2023-10-09T05:56:21.509 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:21 smithi064 ceph-mon[220044]: failed to encode map e225 with expected crc 2023-10-09T05:56:21.510 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:21 smithi064 ceph-mon[212750]: failed to encode map e225 with expected crc 2023-10-09T05:56:21.510 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:21 smithi064 ceph-mon[212750]: failed to encode map e225 with expected crc 2023-10-09T05:56:21.510 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:21 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2909476303' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "36"}]': finished 2023-10-09T05:56:21.510 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:21 smithi064 ceph-mon[212750]: osdmap e225: 8 total, 8 up, 7 in 2023-10-09T05:56:21.510 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:21 smithi064 ceph-mon[212750]: failed to encode map e225 with expected crc 2023-10-09T05:56:21.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:21.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:21.575 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:56:21.618 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:56:21.658 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:21.662 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:21.754 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:21.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:21.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:21.825 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:56:21.848 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:56:21.860 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:21.870 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:56:21.898 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:56:21.903 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:21.928 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:56:21.960 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:56:22.004 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:56:22.030 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:22.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:22.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:22.040 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:56:22.046 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:56:22.046 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:56:22.046 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:56:22.046 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:56:22.046 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:56:22.047 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:56:22.047 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:56:22.047 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:56:22.138 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:22.184 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:22.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:22 smithi064 ceph-mon[212750]: pgmap v519: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 109 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 31273/195498 objects degraded (15.997%); 18882/195498 objects misplaced (9.658%); 22 MiB/s, 420 objects/s recovering 2023-10-09T05:56:22.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:22 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1905148455' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:22.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:22 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 31273/195498 objects degraded (15.997%), 14 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:22.242 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:22 smithi064 ceph-mon[220044]: pgmap v519: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 109 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 31273/195498 objects degraded (15.997%); 18882/195498 objects misplaced (9.658%); 22 MiB/s, 420 objects/s recovering 2023-10-09T05:56:22.242 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:22 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1905148455' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:22.242 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:22 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 31273/195498 objects degraded (15.997%), 14 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:22.244 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:22.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:22.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:22.348 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:22.392 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:22.461 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:22.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:22.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:22.466 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:22 smithi156 ceph-mon[195101]: pgmap v519: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 109 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 31273/195498 objects degraded (15.997%); 18882/195498 objects misplaced (9.658%); 22 MiB/s, 420 objects/s recovering 2023-10-09T05:56:22.466 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:22 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1905148455' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:22.466 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:22 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 31273/195498 objects degraded (15.997%), 14 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:22.566 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:22.597 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:23.290 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:23.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:23.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:23.395 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:23.456 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:23.596 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:56:23.596 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T05:56:23.597 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T05:56:23.597 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T05:56:23.597 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:56:23.716 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:23.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:23.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:23.821 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:23.863 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:23.947 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:23.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:23.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:24.054 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:24.093 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:24.262 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:24.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:24.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:24.372 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:24.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:24 smithi064 ceph-mon[212750]: pgmap v520: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 109 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 425 op/s; 31273/194589 objects degraded (16.071%); 18721/194589 objects misplaced (9.621%); 14 MiB/s, 263 objects/s recovering 2023-10-09T05:56:24.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:24 smithi064 ceph-mon[220044]: pgmap v520: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 109 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 425 op/s; 31273/194589 objects degraded (16.071%); 18721/194589 objects misplaced (9.621%); 14 MiB/s, 263 objects/s recovering 2023-10-09T05:56:24.413 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:24.492 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:24.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:24.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:24.515 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:24 smithi156 ceph-mon[195101]: pgmap v520: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 109 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 425 op/s; 31273/194589 objects degraded (16.071%); 18721/194589 objects misplaced (9.621%); 14 MiB/s, 263 objects/s recovering 2023-10-09T05:56:24.542 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T05:56:24.597 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:24.640 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:24.719 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:24.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:24.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:24.825 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:24.859 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:24.935 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:24.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:24.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:25.042 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:25.086 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:25.094 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:56:25.146 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:25.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:25.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:25.251 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:25.302 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:25.306 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T05:56:25.465 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:56:25.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:25 smithi064 ceph-mon[212750]: pgmap v521: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 109 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 30682/190017 objects degraded (16.147%); 18094/190017 objects misplaced (9.522%); 13 MiB/s, 241 objects/s recovering 2023-10-09T05:56:25.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:25 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1774792664' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:56:25.470 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:56:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:56:25] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.43.0" 2023-10-09T05:56:25.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:25 smithi064 ceph-mon[220044]: pgmap v521: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 109 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 30682/190017 objects degraded (16.147%); 18094/190017 objects misplaced (9.522%); 13 MiB/s, 241 objects/s recovering 2023-10-09T05:56:25.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:25 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1774792664' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:56:25.598 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:25.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:25.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:25.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:25 smithi156 ceph-mon[195101]: pgmap v521: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 109 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 30682/190017 objects degraded (16.147%); 18094/190017 objects misplaced (9.522%); 13 MiB/s, 241 objects/s recovering 2023-10-09T05:56:25.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:25 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1774792664' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:56:25.703 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:25.740 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:25.824 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:25.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:25.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:25.929 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:25.968 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:26.237 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:26.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:26.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:26.328 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T05:56:26.328 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:56:26.328 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_0 2023-10-09T05:56:26.329 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:56:26.347 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:26.376 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:26.438 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:26.478 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:26.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:26.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:26.591 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:26.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:26 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:26.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:26 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1774792664' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:56:26.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:26 smithi156 ceph-mon[195101]: osdmap e226: 8 total, 8 up, 7 in 2023-10-09T05:56:26.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:26 smithi156 ceph-mon[195101]: failed to encode map e226 with expected crc 2023-10-09T05:56:26.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:26 smithi156 ceph-mon[195101]: failed to encode map e226 with expected crc 2023-10-09T05:56:26.637 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:26.707 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:26 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:26.708 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:26 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1774792664' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:56:26.708 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:26 smithi064 ceph-mon[212750]: osdmap e226: 8 total, 8 up, 7 in 2023-10-09T05:56:26.708 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:26 smithi064 ceph-mon[212750]: failed to encode map e226 with expected crc 2023-10-09T05:56:26.708 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:26 smithi064 ceph-mon[212750]: failed to encode map e226 with expected crc 2023-10-09T05:56:26.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:26 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:26.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:26 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1774792664' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:56:26.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:26 smithi064 ceph-mon[220044]: osdmap e226: 8 total, 8 up, 7 in 2023-10-09T05:56:26.709 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:26 smithi064 ceph-mon[220044]: failed to encode map e226 with expected crc 2023-10-09T05:56:26.709 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:26 smithi064 ceph-mon[220044]: failed to encode map e226 with expected crc 2023-10-09T05:56:26.760 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:26.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:26.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:26.868 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:26.915 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:27.004 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:27.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:27.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:27.052 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:56:27.052 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T05:56:27.052 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T05:56:27.052 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T05:56:27.052 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.052 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:91015 6.e 6:7dc6a610:::benchmark_data_smithi064_284193_object91014:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-10-09T05:56:27.052 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:10.210536+0000", 2023-10-09T05:56:27.053 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 256.83782444500002, 2023-10-09T05:56:27.053 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3031288990000001, 2023-10-09T05:56:27.053 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:27.053 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:27.053 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:27.053 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:27.053 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:27.053 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 91015 2023-10-09T05:56:27.053 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.053 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:27.053 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.053 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:27.054 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210536+0000", 2023-10-09T05:56:27.054 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.054 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.054 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.054 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:27.054 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210536+0000", 2023-10-09T05:56:27.054 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.054 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.054 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.054 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:27.054 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210534+0000", 2023-10-09T05:56:27.055 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:27.055 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.055 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.055 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:27.055 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210591+0000", 2023-10-09T05:56:27.055 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6641999999999997e-05 2023-10-09T05:56:27.055 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.055 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.055 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:27.055 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210601+0000", 2023-10-09T05:56:27.055 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0485999999999999e-05 2023-10-09T05:56:27.055 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.056 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.056 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:27.056 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210606+0000", 2023-10-09T05:56:27.056 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0320000000000003e-06 2023-10-09T05:56:27.056 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.056 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.056 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.056 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210705+0000", 2023-10-09T05:56:27.056 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.8412000000000006e-05 2023-10-09T05:56:27.056 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.056 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.056 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:27.056 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210714+0000", 2023-10-09T05:56:27.057 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.6160000000000008e-06 2023-10-09T05:56:27.057 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.057 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.057 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.057 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.497372+0000", 2023-10-09T05:56:27.057 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2866577770000001 2023-10-09T05:56:27.057 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.057 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.057 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:27.057 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.497531+0000", 2023-10-09T05:56:27.057 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015918599999999999 2023-10-09T05:56:27.057 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.058 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.058 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:56:27.058 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.497704+0000", 2023-10-09T05:56:27.058 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000172103 2023-10-09T05:56:27.058 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.058 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.058 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.058 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.502321+0000", 2023-10-09T05:56:27.058 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0046177689999999999 2023-10-09T05:56:27.058 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.058 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.058 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:27.059 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.502336+0000", 2023-10-09T05:56:27.059 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4392999999999999e-05 2023-10-09T05:56:27.059 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.059 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.059 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.059 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.511914+0000", 2023-10-09T05:56:27.059 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0095783229999999997 2023-10-09T05:56:27.059 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.059 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.059 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:27.059 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.511973+0000", 2023-10-09T05:56:27.059 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8718999999999998e-05 2023-10-09T05:56:27.060 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.060 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.060 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:27.060 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.513665+0000", 2023-10-09T05:56:27.060 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001691992 2023-10-09T05:56:27.060 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.060 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:27.060 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.060 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.060 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.060 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:102991 6.2 6:4bf2f80f:::benchmark_data_smithi064_284193_object6448:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-10-09T05:56:27.060 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:53:24.290181+0000", 2023-10-09T05:56:27.061 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 182.75817956700001, 2023-10-09T05:56:27.061 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.937272463, 2023-10-09T05:56:27.061 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:27.061 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:27.061 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:27.061 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:27.061 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:27.061 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 102991 2023-10-09T05:56:27.061 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.061 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:27.061 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.061 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:27.062 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.290181+0000", 2023-10-09T05:56:27.062 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.062 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.062 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.062 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:27.062 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.290181+0000", 2023-10-09T05:56:27.062 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.062 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.062 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.062 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:27.062 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.290179+0000", 2023-10-09T05:56:27.063 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:27.063 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.063 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.063 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:27.063 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.290186+0000", 2023-10-09T05:56:27.063 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.024e-06 2023-10-09T05:56:27.063 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.063 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.063 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:27.063 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.290199+0000", 2023-10-09T05:56:27.063 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2965e-05 2023-10-09T05:56:27.063 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.064 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.064 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:27.064 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.290206+0000", 2023-10-09T05:56:27.064 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0690000000000004e-06 2023-10-09T05:56:27.064 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.064 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.064 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.064 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.290278+0000", 2023-10-09T05:56:27.064 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1543000000000003e-05 2023-10-09T05:56:27.064 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.064 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.064 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:27.065 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.290291+0000", 2023-10-09T05:56:27.065 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2626000000000001e-05 2023-10-09T05:56:27.065 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.065 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.065 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.065 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.211370+0000", 2023-10-09T05:56:27.065 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.92107952299999996 2023-10-09T05:56:27.065 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.065 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.065 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:27.065 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.211505+0000", 2023-10-09T05:56:27.065 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013526 2023-10-09T05:56:27.066 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.066 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.066 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:56:27.066 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.211651+0000", 2023-10-09T05:56:27.066 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014595499999999999 2023-10-09T05:56:27.066 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.066 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.066 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:27.066 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.220677+0000", 2023-10-09T05:56:27.066 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0090260459999999994 2023-10-09T05:56:27.066 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.067 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.067 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.067 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.224907+0000", 2023-10-09T05:56:27.067 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0042298359999999998 2023-10-09T05:56:27.067 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.067 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.067 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.067 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.227398+0000", 2023-10-09T05:56:27.067 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002490889 2023-10-09T05:56:27.067 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.067 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.067 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:27.068 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.227430+0000", 2023-10-09T05:56:27.068 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1681999999999999e-05 2023-10-09T05:56:27.068 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.068 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.068 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:27.068 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.227453+0000", 2023-10-09T05:56:27.068 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3339000000000001e-05 2023-10-09T05:56:27.068 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.068 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:27.068 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.068 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.069 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.069 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:102998 6.2 6:494ff968:::benchmark_data_smithi064_284193_object6455:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-10-09T05:56:27.069 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:53:24.383762+0000", 2023-10-09T05:56:27.069 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 182.664598093, 2023-10-09T05:56:27.069 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.84377951799999995, 2023-10-09T05:56:27.069 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:27.069 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:27.069 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:27.069 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:27.069 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:27.069 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 102998 2023-10-09T05:56:27.070 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.070 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:27.070 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.070 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:27.070 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.383762+0000", 2023-10-09T05:56:27.070 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.070 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.070 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.070 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:27.070 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.383762+0000", 2023-10-09T05:56:27.070 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.071 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.071 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.071 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:27.071 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.383761+0000", 2023-10-09T05:56:27.071 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:56:27.071 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.071 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.071 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:27.072 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.383768+0000", 2023-10-09T05:56:27.072 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2849999999999997e-06 2023-10-09T05:56:27.072 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.072 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.072 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:27.072 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.383787+0000", 2023-10-09T05:56:27.072 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9096000000000001e-05 2023-10-09T05:56:27.072 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.072 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.072 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:27.072 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.383796+0000", 2023-10-09T05:56:27.073 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.7800000000000006e-06 2023-10-09T05:56:27.073 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.073 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.073 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.073 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.383859+0000", 2023-10-09T05:56:27.073 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3124000000000001e-05 2023-10-09T05:56:27.073 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.073 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.073 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:27.073 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.383867+0000", 2023-10-09T05:56:27.073 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8199999999999997e-06 2023-10-09T05:56:27.073 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.074 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.074 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.074 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.214007+0000", 2023-10-09T05:56:27.074 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.830139446 2023-10-09T05:56:27.074 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.074 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.074 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:27.074 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.214113+0000", 2023-10-09T05:56:27.074 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010590900000000001 2023-10-09T05:56:27.074 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.074 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.074 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:56:27.075 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.214191+0000", 2023-10-09T05:56:27.075 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8597000000000007e-05 2023-10-09T05:56:27.075 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.075 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.075 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:27.075 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.220683+0000", 2023-10-09T05:56:27.075 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0064916330000000001 2023-10-09T05:56:27.075 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.075 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.075 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.075 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.224959+0000", 2023-10-09T05:56:27.075 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0042762679999999997 2023-10-09T05:56:27.076 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.076 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.076 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.076 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.227487+0000", 2023-10-09T05:56:27.076 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0025277699999999999 2023-10-09T05:56:27.076 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.076 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.076 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:27.076 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.227518+0000", 2023-10-09T05:56:27.076 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1337999999999997e-05 2023-10-09T05:56:27.076 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.076 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.077 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:27.077 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.227542+0000", 2023-10-09T05:56:27.077 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3601e-05 2023-10-09T05:56:27.077 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.077 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:27.077 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.077 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.077 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.077 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:102999 6.2 6:428b727a:::benchmark_data_smithi064_284193_object6456:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-10-09T05:56:27.077 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:53:24.393899+0000", 2023-10-09T05:56:27.077 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 182.65446098699999, 2023-10-09T05:56:27.077 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.83371729999999999, 2023-10-09T05:56:27.078 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:27.078 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:27.078 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:27.078 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:27.078 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:27.078 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 102999 2023-10-09T05:56:27.078 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.078 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:27.078 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.078 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:27.078 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.393899+0000", 2023-10-09T05:56:27.079 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.079 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.079 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.079 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:27.079 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.393899+0000", 2023-10-09T05:56:27.079 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.079 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.079 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.079 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:27.079 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.393899+0000", 2023-10-09T05:56:27.079 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:56:27.079 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.080 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.080 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:27.080 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.393904+0000", 2023-10-09T05:56:27.080 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5010000000000004e-06 2023-10-09T05:56:27.080 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.080 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.080 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:27.080 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.393916+0000", 2023-10-09T05:56:27.080 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2123000000000001e-05 2023-10-09T05:56:27.080 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.080 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.081 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:27.081 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.393922+0000", 2023-10-09T05:56:27.081 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.178e-06 2023-10-09T05:56:27.081 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.081 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.081 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.081 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.394009+0000", 2023-10-09T05:56:27.081 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6229000000000004e-05 2023-10-09T05:56:27.081 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.081 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.081 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:27.081 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:24.394017+0000", 2023-10-09T05:56:27.082 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.3939999999999996e-06 2023-10-09T05:56:27.082 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.082 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.082 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.082 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.216646+0000", 2023-10-09T05:56:27.082 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.82262949600000002 2023-10-09T05:56:27.082 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.082 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.082 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:27.082 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.216774+0000", 2023-10-09T05:56:27.082 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012741200000000001 2023-10-09T05:56:27.082 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.083 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.083 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:56:27.083 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.216905+0000", 2023-10-09T05:56:27.083 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013101599999999999 2023-10-09T05:56:27.083 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.083 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.083 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:27.083 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.224571+0000", 2023-10-09T05:56:27.083 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0076660130000000002 2023-10-09T05:56:27.083 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.083 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.083 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.084 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.224985+0000", 2023-10-09T05:56:27.084 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00041443300000000002 2023-10-09T05:56:27.084 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.084 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.084 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.084 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.227566+0000", 2023-10-09T05:56:27.084 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0025811100000000002 2023-10-09T05:56:27.084 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.084 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.084 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:27.084 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.227596+0000", 2023-10-09T05:56:27.085 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9694000000000001e-05 2023-10-09T05:56:27.085 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.085 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.085 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:27.085 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:25.227617+0000", 2023-10-09T05:56:27.085 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0432999999999999e-05 2023-10-09T05:56:27.085 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.085 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:27.085 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.085 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.085 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.085 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:110019 6.1e 6:7b30049e:::benchmark_data_smithi064_284193_object13476:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2023-10-09T05:56:27.086 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:22.411566+0000", 2023-10-09T05:56:27.086 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 64.636794320000007, 2023-10-09T05:56:27.086 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7746888279999999, 2023-10-09T05:56:27.086 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:27.086 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:27.086 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:27.086 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:27.086 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:27.086 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 110019 2023-10-09T05:56:27.086 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.086 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:27.086 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.087 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:27.087 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.411566+0000", 2023-10-09T05:56:27.087 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.087 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.087 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.087 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:27.087 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.411566+0000", 2023-10-09T05:56:27.087 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.087 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.087 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.087 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:27.087 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.411565+0000", 2023-10-09T05:56:27.088 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:56:27.088 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.088 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.088 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:27.088 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.411571+0000", 2023-10-09T05:56:27.088 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0850000000000002e-06 2023-10-09T05:56:27.088 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.088 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.088 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:27.088 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.411585+0000", 2023-10-09T05:56:27.088 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4361e-05 2023-10-09T05:56:27.088 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.089 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.089 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:27.089 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.411593+0000", 2023-10-09T05:56:27.089 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2180000000000002e-06 2023-10-09T05:56:27.089 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.089 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.089 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.089 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.411639+0000", 2023-10-09T05:56:27.089 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6915000000000001e-05 2023-10-09T05:56:27.089 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.089 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.090 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:27.090 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.411647+0000", 2023-10-09T05:56:27.090 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8590000000000005e-06 2023-10-09T05:56:27.090 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.090 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.090 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.090 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.170887+0000", 2023-10-09T05:56:27.090 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.75923964700000002 2023-10-09T05:56:27.090 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.090 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.090 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:27.090 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.170919+0000", 2023-10-09T05:56:27.091 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.2492e-05 2023-10-09T05:56:27.091 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.091 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.091 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.091 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.182074+0000", 2023-10-09T05:56:27.091 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.011155026 2023-10-09T05:56:27.091 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.091 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.091 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:27.091 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.182219+0000", 2023-10-09T05:56:27.091 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014427600000000001 2023-10-09T05:56:27.091 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.092 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.092 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T05:56:27.092 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.182389+0000", 2023-10-09T05:56:27.092 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000170509 2023-10-09T05:56:27.092 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.092 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.092 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:27.092 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.184044+0000", 2023-10-09T05:56:27.092 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0016549080000000001 2023-10-09T05:56:27.092 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.092 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.093 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.093 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.184912+0000", 2023-10-09T05:56:27.093 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00086812100000000002 2023-10-09T05:56:27.093 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.093 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.093 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.093 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.185028+0000", 2023-10-09T05:56:27.093 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000115294 2023-10-09T05:56:27.093 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.093 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.093 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.093 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.186080+0000", 2023-10-09T05:56:27.094 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0010523290000000001 2023-10-09T05:56:27.094 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.094 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.094 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:27.094 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.186132+0000", 2023-10-09T05:56:27.094 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2432999999999998e-05 2023-10-09T05:56:27.094 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.094 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.094 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:27.094 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.186255+0000", 2023-10-09T05:56:27.094 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012244299999999999 2023-10-09T05:56:27.095 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.095 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:27.095 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.095 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.095 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.095 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:110021 6.1e 6:7cb142d9:::benchmark_data_smithi064_284193_object13478:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2023-10-09T05:56:27.095 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:22.456698+0000", 2023-10-09T05:56:27.095 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 64.591662305, 2023-10-09T05:56:27.095 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.72968491, 2023-10-09T05:56:27.095 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:27.095 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:27.095 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:27.096 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:27.096 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:27.096 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 110021 2023-10-09T05:56:27.096 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.096 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:27.096 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.096 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:27.096 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.456698+0000", 2023-10-09T05:56:27.096 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.096 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.096 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.096 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.456698+0000", 2023-10-09T05:56:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.456696+0000", 2023-10-09T05:56:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.456708+0000", 2023-10-09T05:56:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1503e-05 2023-10-09T05:56:27.098 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.098 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.098 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:27.098 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.456741+0000", 2023-10-09T05:56:27.098 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.2963000000000003e-05 2023-10-09T05:56:27.098 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.098 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.098 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:27.098 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.456758+0000", 2023-10-09T05:56:27.098 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.753e-05 2023-10-09T05:56:27.098 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.099 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.099 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.099 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.456875+0000", 2023-10-09T05:56:27.099 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011678799999999999 2023-10-09T05:56:27.099 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.099 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.099 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:27.099 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.456892+0000", 2023-10-09T05:56:27.099 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6663e-05 2023-10-09T05:56:27.099 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.100 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.100 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.100 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.170942+0000", 2023-10-09T05:56:27.100 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.71405068599999999 2023-10-09T05:56:27.100 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.100 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.100 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:27.100 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.170949+0000", 2023-10-09T05:56:27.100 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5050000000000004e-06 2023-10-09T05:56:27.100 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.100 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.100 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.101 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.183286+0000", 2023-10-09T05:56:27.101 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0123373369999999 2023-10-09T05:56:27.101 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.101 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.101 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:27.101 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.183352+0000", 2023-10-09T05:56:27.101 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6153000000000002e-05 2023-10-09T05:56:27.101 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.101 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.101 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T05:56:27.101 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.183455+0000", 2023-10-09T05:56:27.101 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000102785 2023-10-09T05:56:27.102 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.102 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.102 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:27.102 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.184924+0000", 2023-10-09T05:56:27.102 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001468802 2023-10-09T05:56:27.102 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.102 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.102 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.102 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.185074+0000", 2023-10-09T05:56:27.102 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014996399999999999 2023-10-09T05:56:27.102 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.103 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.103 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.103 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.185109+0000", 2023-10-09T05:56:27.103 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5689999999999999e-05 2023-10-09T05:56:27.103 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.103 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.103 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.103 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.186288+0000", 2023-10-09T05:56:27.103 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0011783119999999999 2023-10-09T05:56:27.103 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.103 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.104 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:27.104 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.186318+0000", 2023-10-09T05:56:27.104 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0375999999999999e-05 2023-10-09T05:56:27.104 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.104 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.104 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:27.104 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.186383+0000", 2023-10-09T05:56:27.104 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4735999999999998e-05 2023-10-09T05:56:27.104 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.104 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:27.104 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.105 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.105 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.105 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:110047 6.1f 6:fd8aac31:::benchmark_data_smithi064_284193_object13504:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2023-10-09T05:56:27.105 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:23.278484+0000", 2023-10-09T05:56:27.105 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 63.769875915999997, 2023-10-09T05:56:27.105 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.023028037, 2023-10-09T05:56:27.105 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:27.105 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:27.105 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:27.105 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:27.106 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:27.106 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 110047 2023-10-09T05:56:27.106 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.106 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:27.106 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.106 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:27.106 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.278484+0000", 2023-10-09T05:56:27.106 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.106 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.106 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.106 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:27.106 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.278484+0000", 2023-10-09T05:56:27.107 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.107 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.107 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.107 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:27.107 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.278483+0000", 2023-10-09T05:56:27.107 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:56:27.107 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.107 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.107 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:27.107 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.278488+0000", 2023-10-09T05:56:27.107 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8860000000000003e-06 2023-10-09T05:56:27.108 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.108 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.108 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:27.108 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.278496+0000", 2023-10-09T05:56:27.108 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1529999999999997e-06 2023-10-09T05:56:27.108 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.108 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.108 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:27.108 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.278501+0000", 2023-10-09T05:56:27.108 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7400000000000004e-06 2023-10-09T05:56:27.108 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.108 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.109 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.109 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.278538+0000", 2023-10-09T05:56:27.109 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7184999999999997e-05 2023-10-09T05:56:27.109 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.109 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.109 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:27.109 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.278546+0000", 2023-10-09T05:56:27.109 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9980000000000003e-06 2023-10-09T05:56:27.109 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.109 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.109 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.110 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.291647+0000", 2023-10-09T05:56:27.110 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.013100771 2023-10-09T05:56:27.110 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.110 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.110 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:27.110 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.291798+0000", 2023-10-09T05:56:27.110 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015106200000000001 2023-10-09T05:56:27.110 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.110 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.110 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,5,6", 2023-10-09T05:56:27.110 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.292129+0000", 2023-10-09T05:56:27.111 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00033066399999999999 2023-10-09T05:56:27.111 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.111 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.111 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.111 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.297460+0000", 2023-10-09T05:56:27.111 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0053315569999999998 2023-10-09T05:56:27.111 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.111 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.111 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.111 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.297497+0000", 2023-10-09T05:56:27.111 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.625e-05 2023-10-09T05:56:27.111 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.112 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.112 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.112 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.297534+0000", 2023-10-09T05:56:27.112 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6977000000000002e-05 2023-10-09T05:56:27.112 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.112 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.112 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:27.112 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.299568+0000", 2023-10-09T05:56:27.112 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0020343230000000002 2023-10-09T05:56:27.112 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.112 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.112 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:27.113 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.299618+0000", 2023-10-09T05:56:27.113 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0380999999999998e-05 2023-10-09T05:56:27.113 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.113 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.113 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:27.113 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.301512+0000", 2023-10-09T05:56:27.113 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0018940680000000001 2023-10-09T05:56:27.113 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.113 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:27.113 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.113 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.113 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.114 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:110055 6.1e 6:7e204b52:::benchmark_data_smithi064_284193_object13512:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2023-10-09T05:56:27.114 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:23.290179+0000", 2023-10-09T05:56:27.114 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 63.758181741999998, 2023-10-09T05:56:27.114 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.89634445600000001, 2023-10-09T05:56:27.114 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:27.114 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:27.114 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:27.114 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:27.114 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:27.114 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 110055 2023-10-09T05:56:27.114 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.114 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:27.115 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.115 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:27.115 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.290179+0000", 2023-10-09T05:56:27.115 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.115 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.115 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.115 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:27.115 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.290179+0000", 2023-10-09T05:56:27.115 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.115 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.115 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.115 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:27.116 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.290177+0000", 2023-10-09T05:56:27.116 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:27.116 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.116 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.116 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:27.116 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.290189+0000", 2023-10-09T05:56:27.116 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1785e-05 2023-10-09T05:56:27.116 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.116 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.116 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:27.116 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.290205+0000", 2023-10-09T05:56:27.116 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5899000000000001e-05 2023-10-09T05:56:27.116 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.117 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.117 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:27.117 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.290211+0000", 2023-10-09T05:56:27.117 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6059999999999999e-06 2023-10-09T05:56:27.117 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.117 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.117 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.117 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.290265+0000", 2023-10-09T05:56:27.117 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3848999999999999e-05 2023-10-09T05:56:27.117 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.117 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.118 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:27.118 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.290276+0000", 2023-10-09T05:56:27.118 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.084e-05 2023-10-09T05:56:27.118 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.118 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.118 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.118 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.184072+0000", 2023-10-09T05:56:27.118 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.89379556599999999 2023-10-09T05:56:27.118 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.118 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.118 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:27.118 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.184163+0000", 2023-10-09T05:56:27.119 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1681000000000005e-05 2023-10-09T05:56:27.119 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.119 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.119 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T05:56:27.119 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.184266+0000", 2023-10-09T05:56:27.119 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000102252 2023-10-09T05:56:27.119 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.119 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.119 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:27.119 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.185035+0000", 2023-10-09T05:56:27.119 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00076948000000000001 2023-10-09T05:56:27.119 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.120 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.120 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.120 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.185192+0000", 2023-10-09T05:56:27.120 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015741400000000001 2023-10-09T05:56:27.120 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.120 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.120 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.120 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.186417+0000", 2023-10-09T05:56:27.120 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001224473 2023-10-09T05:56:27.120 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.120 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.120 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.121 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.186441+0000", 2023-10-09T05:56:27.121 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3907999999999999e-05 2023-10-09T05:56:27.121 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.121 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.121 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:27.121 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.186465+0000", 2023-10-09T05:56:27.121 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4530999999999999e-05 2023-10-09T05:56:27.121 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.121 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.121 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:27.121 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.186523+0000", 2023-10-09T05:56:27.121 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7671999999999999e-05 2023-10-09T05:56:27.122 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.122 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:27.122 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.122 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.122 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.122 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:110067 6.1f 6:f8d8a82d:::benchmark_data_smithi064_284193_object13524:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2023-10-09T05:56:27.122 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:23.316013+0000", 2023-10-09T05:56:27.122 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 63.732347390000001, 2023-10-09T05:56:27.122 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99153563, 2023-10-09T05:56:27.122 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:27.122 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:27.122 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:27.123 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:27.123 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:27.123 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 110067 2023-10-09T05:56:27.123 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.123 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:27.123 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.123 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:27.123 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.316013+0000", 2023-10-09T05:56:27.123 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.123 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.123 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.123 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:27.124 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.316013+0000", 2023-10-09T05:56:27.124 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.124 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.124 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.124 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:27.124 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.316012+0000", 2023-10-09T05:56:27.124 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:56:27.124 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.124 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.124 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:27.124 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.316016+0000", 2023-10-09T05:56:27.124 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0419999999999996e-06 2023-10-09T05:56:27.125 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.125 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.125 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:27.125 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.316037+0000", 2023-10-09T05:56:27.125 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0730999999999999e-05 2023-10-09T05:56:27.125 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.125 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.125 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:27.125 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.316042+0000", 2023-10-09T05:56:27.125 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8879999999999996e-06 2023-10-09T05:56:27.125 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.125 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.126 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.126 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.316079+0000", 2023-10-09T05:56:27.126 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7410999999999998e-05 2023-10-09T05:56:27.126 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.126 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.126 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:27.126 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.316087+0000", 2023-10-09T05:56:27.126 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5020000000000003e-06 2023-10-09T05:56:27.126 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.126 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.126 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.126 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.295918+0000", 2023-10-09T05:56:27.127 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.97983070100000003 2023-10-09T05:56:27.127 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.127 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.127 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:27.127 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.296067+0000", 2023-10-09T05:56:27.127 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014895500000000001 2023-10-09T05:56:27.127 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.127 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.127 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,5,6", 2023-10-09T05:56:27.127 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.296317+0000", 2023-10-09T05:56:27.127 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00025033100000000001 2023-10-09T05:56:27.127 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.128 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.128 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.128 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.299559+0000", 2023-10-09T05:56:27.128 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0032425230000000002 2023-10-09T05:56:27.128 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.128 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.128 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.128 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.301585+0000", 2023-10-09T05:56:27.128 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0020257320000000001 2023-10-09T05:56:27.128 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.128 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.128 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:27.128 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.301593+0000", 2023-10-09T05:56:27.129 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6179999999999997e-06 2023-10-09T05:56:27.129 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.129 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.129 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.129 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.305356+0000", 2023-10-09T05:56:27.129 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0037634410000000002 2023-10-09T05:56:27.129 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.129 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.129 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:27.129 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.305399+0000", 2023-10-09T05:56:27.129 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3151000000000001e-05 2023-10-09T05:56:27.130 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.130 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.130 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:27.130 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.307549+0000", 2023-10-09T05:56:27.130 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002149186 2023-10-09T05:56:27.130 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.130 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:27.130 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.130 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.130 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.130 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:110151 6.1f 6:fcafb54e:::benchmark_data_smithi064_284193_object13608:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2023-10-09T05:56:27.130 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:23.712475+0000", 2023-10-09T05:56:27.131 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 63.335885785999999, 2023-10-09T05:56:27.131 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.59716937800000003, 2023-10-09T05:56:27.131 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:27.131 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:27.131 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:27.131 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:27.131 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:27.131 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 110151 2023-10-09T05:56:27.131 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.131 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:27.131 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.131 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:27.132 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.712475+0000", 2023-10-09T05:56:27.132 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.132 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.132 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.132 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:27.132 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.712475+0000", 2023-10-09T05:56:27.132 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.132 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.132 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.132 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:27.132 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.712473+0000", 2023-10-09T05:56:27.132 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:56:27.132 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.133 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.133 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:27.133 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.712480+0000", 2023-10-09T05:56:27.133 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.443e-06 2023-10-09T05:56:27.133 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.133 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.133 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:27.133 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.712493+0000", 2023-10-09T05:56:27.133 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2876e-05 2023-10-09T05:56:27.133 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.133 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.134 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:27.134 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.712498+0000", 2023-10-09T05:56:27.134 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6110000000000001e-06 2023-10-09T05:56:27.134 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.134 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.134 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.134 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.712539+0000", 2023-10-09T05:56:27.134 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0524e-05 2023-10-09T05:56:27.134 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.134 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.134 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:27.134 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.712546+0000", 2023-10-09T05:56:27.135 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8950000000000001e-06 2023-10-09T05:56:27.135 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.135 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.135 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.135 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.297975+0000", 2023-10-09T05:56:27.135 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.58542938499999997 2023-10-09T05:56:27.135 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.135 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.135 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:27.135 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.298147+0000", 2023-10-09T05:56:27.135 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000171449 2023-10-09T05:56:27.135 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.136 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.136 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,5,6", 2023-10-09T05:56:27.136 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.298436+0000", 2023-10-09T05:56:27.136 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00028933499999999999 2023-10-09T05:56:27.136 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.136 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.136 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.136 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.301556+0000", 2023-10-09T05:56:27.136 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0031203229999999999 2023-10-09T05:56:27.136 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.136 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.136 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:27.137 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.301598+0000", 2023-10-09T05:56:27.137 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1519e-05 2023-10-09T05:56:27.137 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.137 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.137 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.137 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.303743+0000", 2023-10-09T05:56:27.137 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0021452369999999999 2023-10-09T05:56:27.137 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.137 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.137 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.137 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.307636+0000", 2023-10-09T05:56:27.137 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0038927219999999999 2023-10-09T05:56:27.137 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.138 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.138 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:27.138 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.307692+0000", 2023-10-09T05:56:27.138 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6091000000000002e-05 2023-10-09T05:56:27.138 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.138 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.138 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:27.138 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.309644+0000", 2023-10-09T05:56:27.138 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001952042 2023-10-09T05:56:27.138 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.138 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:27.139 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.139 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.139 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.139 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:118735 6.6 6:648f9503:::benchmark_data_smithi064_284193_object22192:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e212)", 2023-10-09T05:56:27.139 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:42.286777+0000", 2023-10-09T05:56:27.139 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 44.761583434999999, 2023-10-09T05:56:27.139 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.56895495500000004, 2023-10-09T05:56:27.139 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:27.139 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:27.139 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:27.139 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:27.140 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:27.140 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 118735 2023-10-09T05:56:27.140 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.140 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:27.140 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.140 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:27.140 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.286777+0000", 2023-10-09T05:56:27.140 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.140 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.141 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.141 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:27.141 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.286777+0000", 2023-10-09T05:56:27.141 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.141 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.141 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.141 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:27.141 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.286775+0000", 2023-10-09T05:56:27.141 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:27.141 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.141 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.141 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:27.142 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.286785+0000", 2023-10-09T05:56:27.142 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.8330000000000006e-06 2023-10-09T05:56:27.142 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.142 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.142 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:27.142 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.286811+0000", 2023-10-09T05:56:27.142 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.616e-05 2023-10-09T05:56:27.142 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.142 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.142 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:27.142 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.286822+0000", 2023-10-09T05:56:27.143 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1454000000000001e-05 2023-10-09T05:56:27.143 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.143 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.143 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.143 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.286928+0000", 2023-10-09T05:56:27.143 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000105518 2023-10-09T05:56:27.143 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.143 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.143 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for degraded object", 2023-10-09T05:56:27.143 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.287336+0000", 2023-10-09T05:56:27.143 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00040847600000000001 2023-10-09T05:56:27.144 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.144 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.144 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.144 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.292474+0000", 2023-10-09T05:56:27.144 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0051374089999999999 2023-10-09T05:56:27.144 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.144 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.144 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:27.144 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.292549+0000", 2023-10-09T05:56:27.144 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4849000000000001e-05 2023-10-09T05:56:27.144 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.144 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.144 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6,7", 2023-10-09T05:56:27.145 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.292849+0000", 2023-10-09T05:56:27.145 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00030025800000000002 2023-10-09T05:56:27.145 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.145 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.145 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:27.145 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.295086+0000", 2023-10-09T05:56:27.145 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0022369590000000002 2023-10-09T05:56:27.145 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.145 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.145 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.145 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.295133+0000", 2023-10-09T05:56:27.146 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7256000000000003e-05 2023-10-09T05:56:27.146 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.146 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.146 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.146 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.296118+0000", 2023-10-09T05:56:27.146 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00098473599999999994 2023-10-09T05:56:27.146 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.146 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.146 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.146 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.853614+0000", 2023-10-09T05:56:27.146 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.55749634199999998 2023-10-09T05:56:27.147 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.147 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.147 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:27.147 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.853663+0000", 2023-10-09T05:56:27.147 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8331999999999997e-05 2023-10-09T05:56:27.147 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.147 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.147 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:27.147 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.855732+0000", 2023-10-09T05:56:27.147 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0020692200000000001 2023-10-09T05:56:27.147 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.148 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:27.148 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.148 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.148 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.148 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:118736 6.f 6:f600b7d9:::benchmark_data_smithi064_284193_object22193:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e212)", 2023-10-09T05:56:27.148 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:42.286841+0000", 2023-10-09T05:56:27.148 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 44.761518985999999, 2023-10-09T05:56:27.148 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.56470399699999996, 2023-10-09T05:56:27.148 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:27.148 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:27.148 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:27.148 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:27.149 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:27.149 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 118736 2023-10-09T05:56:27.149 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.149 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:27.149 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.149 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:27.149 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.286841+0000", 2023-10-09T05:56:27.149 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.149 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.149 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.149 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:27.149 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.286841+0000", 2023-10-09T05:56:27.150 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.150 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.150 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.150 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:27.150 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.286840+0000", 2023-10-09T05:56:27.150 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:56:27.150 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.150 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.150 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:27.150 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.286847+0000", 2023-10-09T05:56:27.150 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.426e-06 2023-10-09T05:56:27.151 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.151 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.151 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:27.151 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.286865+0000", 2023-10-09T05:56:27.151 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7796999999999999e-05 2023-10-09T05:56:27.151 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.151 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.151 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:27.151 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.286875+0000", 2023-10-09T05:56:27.151 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0776e-05 2023-10-09T05:56:27.151 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.151 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.152 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.152 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.287030+0000", 2023-10-09T05:56:27.152 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015471699999999999 2023-10-09T05:56:27.152 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.152 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.152 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:27.152 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.287177+0000", 2023-10-09T05:56:27.152 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014684600000000001 2023-10-09T05:56:27.152 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.152 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.152 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,5,6", 2023-10-09T05:56:27.152 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.287517+0000", 2023-10-09T05:56:27.153 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00034051099999999999 2023-10-09T05:56:27.153 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.153 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.153 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:27.153 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.288755+0000", 2023-10-09T05:56:27.153 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0012379540000000001 2023-10-09T05:56:27.153 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.153 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.153 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.153 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.289532+0000", 2023-10-09T05:56:27.153 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000776365 2023-10-09T05:56:27.153 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.154 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.154 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.154 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.290684+0000", 2023-10-09T05:56:27.154 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0011524319999999999 2023-10-09T05:56:27.154 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.154 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.154 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.154 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.849609+0000", 2023-10-09T05:56:27.154 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.55892491 2023-10-09T05:56:27.154 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.154 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.155 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:27.155 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.849649+0000", 2023-10-09T05:56:27.155 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0289e-05 2023-10-09T05:56:27.155 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.155 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.155 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:27.155 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.851545+0000", 2023-10-09T05:56:27.155 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0018959770000000001 2023-10-09T05:56:27.155 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.155 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:27.155 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.155 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.156 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.156 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:118739 6.16 6:6ad9c422:::benchmark_data_smithi064_284193_object22196:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e212)", 2023-10-09T05:56:27.156 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:42.311798+0000", 2023-10-09T05:56:27.156 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 44.736562630000002, 2023-10-09T05:56:27.156 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.543727187, 2023-10-09T05:56:27.156 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:27.156 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:27.156 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:27.156 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:27.156 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:27.156 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 118739 2023-10-09T05:56:27.156 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.156 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:27.157 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.157 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:27.157 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.311798+0000", 2023-10-09T05:56:27.157 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.157 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.157 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.157 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:27.157 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.311798+0000", 2023-10-09T05:56:27.157 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.157 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.157 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.157 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:27.158 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.311796+0000", 2023-10-09T05:56:27.158 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:27.158 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.158 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.158 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:27.158 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.311803+0000", 2023-10-09T05:56:27.158 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0099999999999998e-06 2023-10-09T05:56:27.158 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.158 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.158 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:27.158 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.311820+0000", 2023-10-09T05:56:27.158 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6379e-05 2023-10-09T05:56:27.159 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.159 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.159 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:27.159 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.311826+0000", 2023-10-09T05:56:27.159 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4779999999999996e-06 2023-10-09T05:56:27.159 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.159 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.159 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.159 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.311876+0000", 2023-10-09T05:56:27.159 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9648999999999998e-05 2023-10-09T05:56:27.159 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.159 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.160 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for degraded object", 2023-10-09T05:56:27.160 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.312229+0000", 2023-10-09T05:56:27.160 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00035294500000000002 2023-10-09T05:56:27.160 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.160 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.160 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.160 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.316446+0000", 2023-10-09T05:56:27.160 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0042165800000000002 2023-10-09T05:56:27.160 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.160 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.160 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:27.160 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.316517+0000", 2023-10-09T05:56:27.161 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1451000000000004e-05 2023-10-09T05:56:27.161 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.161 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.161 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6,7", 2023-10-09T05:56:27.161 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.316783+0000", 2023-10-09T05:56:27.161 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00026648699999999998 2023-10-09T05:56:27.161 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.161 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.161 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:27.161 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.318845+0000", 2023-10-09T05:56:27.161 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0020614940000000001 2023-10-09T05:56:27.161 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.161 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.162 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.162 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.320307+0000", 2023-10-09T05:56:27.162 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0014617180000000001 2023-10-09T05:56:27.162 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.162 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.162 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.162 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.320347+0000", 2023-10-09T05:56:27.162 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0311000000000001e-05 2023-10-09T05:56:27.162 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.162 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.162 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.162 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.853374+0000", 2023-10-09T05:56:27.163 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.533026993 2023-10-09T05:56:27.163 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.163 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.163 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:27.163 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.853430+0000", 2023-10-09T05:56:27.163 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.558e-05 2023-10-09T05:56:27.163 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.163 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.163 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:27.163 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:42.855525+0000", 2023-10-09T05:56:27.163 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0020953109999999999 2023-10-09T05:56:27.163 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.164 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:27.164 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.164 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.164 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.164 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:119187 6.1e 6:7c546603:::benchmark_data_smithi064_284193_object22644:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-10-09T05:56:27.164 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:43.711700+0000", 2023-10-09T05:56:27.164 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 43.336660047999999, 2023-10-09T05:56:27.164 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.010368224, 2023-10-09T05:56:27.164 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:27.164 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:27.164 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:27.164 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:27.165 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:27.165 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 119187 2023-10-09T05:56:27.165 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.165 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:27.165 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.165 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:27.165 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.711700+0000", 2023-10-09T05:56:27.165 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.165 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.165 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.165 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:27.165 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.711700+0000", 2023-10-09T05:56:27.166 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.166 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.166 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.166 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:27.166 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.711699+0000", 2023-10-09T05:56:27.166 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:56:27.166 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.166 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.166 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:27.166 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.711705+0000", 2023-10-09T05:56:27.166 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7089999999999999e-06 2023-10-09T05:56:27.167 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.167 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.167 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:27.167 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.711715+0000", 2023-10-09T05:56:27.167 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0485e-05 2023-10-09T05:56:27.167 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.167 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.167 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:27.167 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.711720+0000", 2023-10-09T05:56:27.167 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.403e-06 2023-10-09T05:56:27.167 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.167 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.167 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.168 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.711796+0000", 2023-10-09T05:56:27.168 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5733000000000001e-05 2023-10-09T05:56:27.168 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.168 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.168 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:27.168 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.711802+0000", 2023-10-09T05:56:27.168 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7089999999999996e-06 2023-10-09T05:56:27.168 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.168 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.168 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.168 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.674452+0000", 2023-10-09T05:56:27.168 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96265013799999999 2023-10-09T05:56:27.169 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.169 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.169 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:27.169 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.674461+0000", 2023-10-09T05:56:27.169 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8769999999999999e-06 2023-10-09T05:56:27.169 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.169 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.169 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.169 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.707191+0000", 2023-10-09T05:56:27.169 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0327298499999999 2023-10-09T05:56:27.169 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.169 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.169 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:27.170 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.707308+0000", 2023-10-09T05:56:27.170 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000116965 2023-10-09T05:56:27.170 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.170 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.170 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:56:27.170 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.707543+0000", 2023-10-09T05:56:27.170 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00023478600000000001 2023-10-09T05:56:27.170 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.170 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.170 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:27.170 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.712510+0000", 2023-10-09T05:56:27.170 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0049671439999999997 2023-10-09T05:56:27.171 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.171 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.171 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.171 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.716010+0000", 2023-10-09T05:56:27.171 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0034996469999999998 2023-10-09T05:56:27.171 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.171 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.171 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.171 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.721949+0000", 2023-10-09T05:56:27.171 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0059391629999999999 2023-10-09T05:56:27.171 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.171 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.172 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:27.172 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.722008+0000", 2023-10-09T05:56:27.172 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8845000000000003e-05 2023-10-09T05:56:27.172 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.172 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.172 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:27.172 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.722068+0000", 2023-10-09T05:56:27.172 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0822999999999997e-05 2023-10-09T05:56:27.172 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.172 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:27.172 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.172 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.173 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.173 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:119189 6.f 6:f56825aa:::benchmark_data_smithi064_284193_object22646:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-10-09T05:56:27.173 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:43.768575+0000", 2023-10-09T05:56:27.173 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 43.279785054999998, 2023-10-09T05:56:27.173 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1074308939999999, 2023-10-09T05:56:27.173 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:27.173 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:27.173 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:27.173 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:27.173 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:27.173 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 119189 2023-10-09T05:56:27.173 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.174 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:27.174 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.174 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:27.174 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.768575+0000", 2023-10-09T05:56:27.174 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.174 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.174 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.174 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:27.174 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.768575+0000", 2023-10-09T05:56:27.174 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.174 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.175 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.175 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:27.175 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.768575+0000", 2023-10-09T05:56:27.175 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:56:27.175 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.175 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.175 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:27.175 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.768578+0000", 2023-10-09T05:56:27.175 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.596e-06 2023-10-09T05:56:27.175 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.175 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.175 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:27.176 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.768589+0000", 2023-10-09T05:56:27.176 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0692e-05 2023-10-09T05:56:27.176 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.176 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.176 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:27.176 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.768593+0000", 2023-10-09T05:56:27.176 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3379999999999998e-06 2023-10-09T05:56:27.176 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.176 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.176 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.176 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.768648+0000", 2023-10-09T05:56:27.176 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4255000000000001e-05 2023-10-09T05:56:27.176 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.177 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.177 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:27.177 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.768653+0000", 2023-10-09T05:56:27.177 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6570000000000002e-06 2023-10-09T05:56:27.177 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.177 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.177 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.177 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.868318+0000", 2023-10-09T05:56:27.177 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0996647669999999 2023-10-09T05:56:27.177 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.177 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.177 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:27.178 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.868426+0000", 2023-10-09T05:56:27.178 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000108053 2023-10-09T05:56:27.178 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.178 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.178 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T05:56:27.178 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.868605+0000", 2023-10-09T05:56:27.178 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00017891700000000001 2023-10-09T05:56:27.178 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.178 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.178 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:27.179 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.870617+0000", 2023-10-09T05:56:27.179 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002012401 2023-10-09T05:56:27.179 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.179 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.179 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.179 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.871235+0000", 2023-10-09T05:56:27.179 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00061724299999999996 2023-10-09T05:56:27.179 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.179 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.179 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.179 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.871577+0000", 2023-10-09T05:56:27.179 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00034259999999999998 2023-10-09T05:56:27.180 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.180 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.180 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.180 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.874425+0000", 2023-10-09T05:56:27.180 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0028476040000000001 2023-10-09T05:56:27.180 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.180 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.180 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:27.180 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.874484+0000", 2023-10-09T05:56:27.180 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9565999999999999e-05 2023-10-09T05:56:27.180 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.180 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.181 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:27.181 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.876006+0000", 2023-10-09T05:56:27.181 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0015218110000000001 2023-10-09T05:56:27.181 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.181 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:27.181 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.181 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.181 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.181 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:119207 6.e 6:72b0e584:::benchmark_data_smithi064_284193_object22664:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-10-09T05:56:27.181 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:44.039853+0000", 2023-10-09T05:56:27.181 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 43.008507109999996, 2023-10-09T05:56:27.182 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.710139471, 2023-10-09T05:56:27.182 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:27.182 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:27.182 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:27.182 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:27.182 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:27.182 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 119207 2023-10-09T05:56:27.182 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.182 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:27.182 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.182 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:27.182 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.039853+0000", 2023-10-09T05:56:27.183 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.183 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.183 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.183 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:27.183 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.039853+0000", 2023-10-09T05:56:27.183 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.183 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.183 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.183 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:27.183 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.039852+0000", 2023-10-09T05:56:27.183 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:56:27.183 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.184 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.184 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:27.184 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.039858+0000", 2023-10-09T05:56:27.184 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4859999999999997e-06 2023-10-09T05:56:27.184 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.184 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.184 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:27.184 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.039871+0000", 2023-10-09T05:56:27.184 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3217999999999999e-05 2023-10-09T05:56:27.184 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.184 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.184 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:27.185 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.039877+0000", 2023-10-09T05:56:27.185 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7950000000000004e-06 2023-10-09T05:56:27.185 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.185 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.185 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.185 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.039923+0000", 2023-10-09T05:56:27.185 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5973999999999997e-05 2023-10-09T05:56:27.185 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.185 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.185 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:27.185 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.039932+0000", 2023-10-09T05:56:27.185 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9910000000000008e-06 2023-10-09T05:56:27.185 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.186 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.186 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.186 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.674566+0000", 2023-10-09T05:56:27.186 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.63463462199999998 2023-10-09T05:56:27.186 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.186 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.186 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:27.186 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.674573+0000", 2023-10-09T05:56:27.186 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7800000000000003e-06 2023-10-09T05:56:27.186 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.186 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.187 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.187 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.711205+0000", 2023-10-09T05:56:27.187 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0366318240000001 2023-10-09T05:56:27.187 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.187 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.187 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:27.187 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.711325+0000", 2023-10-09T05:56:27.187 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012053900000000001 2023-10-09T05:56:27.187 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.187 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.187 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:56:27.187 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.711613+0000", 2023-10-09T05:56:27.188 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00028770999999999999 2023-10-09T05:56:27.188 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.188 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.188 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:27.188 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.712912+0000", 2023-10-09T05:56:27.188 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0012985970000000001 2023-10-09T05:56:27.188 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.188 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.188 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.188 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.716093+0000", 2023-10-09T05:56:27.188 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0031810470000000002 2023-10-09T05:56:27.188 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.188 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.189 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.189 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.749899+0000", 2023-10-09T05:56:27.189 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.033806514000000003 2023-10-09T05:56:27.189 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.189 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.189 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:27.189 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.749953+0000", 2023-10-09T05:56:27.189 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3845999999999999e-05 2023-10-09T05:56:27.189 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.189 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.189 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:27.189 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.749993+0000", 2023-10-09T05:56:27.190 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9647000000000002e-05 2023-10-09T05:56:27.190 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.190 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:27.190 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.190 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.190 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.190 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:119214 6.1e 6:7d4d0cb9:::benchmark_data_smithi064_284193_object22671:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-10-09T05:56:27.190 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:44.133647+0000", 2023-10-09T05:56:27.190 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 42.914713018999997, 2023-10-09T05:56:27.190 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5885200099999999, 2023-10-09T05:56:27.190 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:27.190 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:27.191 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:27.191 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:27.191 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:27.191 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 119214 2023-10-09T05:56:27.191 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.191 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:27.191 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.191 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:27.191 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.133647+0000", 2023-10-09T05:56:27.191 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.191 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.192 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.192 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:27.192 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.133647+0000", 2023-10-09T05:56:27.192 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.192 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.192 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.192 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:27.192 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.133646+0000", 2023-10-09T05:56:27.192 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:27.192 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.192 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.192 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:27.193 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.133656+0000", 2023-10-09T05:56:27.193 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0013999999999999e-05 2023-10-09T05:56:27.193 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.193 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.193 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:27.193 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.133673+0000", 2023-10-09T05:56:27.193 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7379000000000001e-05 2023-10-09T05:56:27.193 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.193 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.193 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:27.193 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.133680+0000", 2023-10-09T05:56:27.194 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6699999999999997e-06 2023-10-09T05:56:27.194 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.194 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.194 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.194 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.133725+0000", 2023-10-09T05:56:27.194 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5188999999999997e-05 2023-10-09T05:56:27.194 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.194 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.194 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:27.194 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.133734+0000", 2023-10-09T05:56:27.194 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9379999999999998e-06 2023-10-09T05:56:27.194 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.195 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.195 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.195 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.674484+0000", 2023-10-09T05:56:27.195 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.54074985399999997 2023-10-09T05:56:27.195 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.195 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.195 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:27.195 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.674491+0000", 2023-10-09T05:56:27.195 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5529999999999998e-06 2023-10-09T05:56:27.195 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.195 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.195 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.196 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.708538+0000", 2023-10-09T05:56:27.196 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0340476890000001 2023-10-09T05:56:27.196 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.196 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.196 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:27.196 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.708662+0000", 2023-10-09T05:56:27.196 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012336700000000001 2023-10-09T05:56:27.196 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.196 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.196 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:56:27.196 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.708855+0000", 2023-10-09T05:56:27.197 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00019366300000000001 2023-10-09T05:56:27.197 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.197 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.197 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:27.197 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.712515+0000", 2023-10-09T05:56:27.197 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0036598580000000002 2023-10-09T05:56:27.197 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.197 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.197 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.197 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.716056+0000", 2023-10-09T05:56:27.197 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0035411169999999999 2023-10-09T05:56:27.197 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.198 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.198 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.198 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.722107+0000", 2023-10-09T05:56:27.198 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0060505769999999997 2023-10-09T05:56:27.198 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.198 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.198 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:27.198 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.722142+0000", 2023-10-09T05:56:27.198 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.4631000000000001e-05 2023-10-09T05:56:27.198 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.198 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.199 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:27.199 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.722167+0000", 2023-10-09T05:56:27.199 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.5778e-05 2023-10-09T05:56:27.199 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.199 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:27.199 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.199 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.199 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.199 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:119217 6.1e 6:7ea05517:::benchmark_data_smithi064_284193_object22674:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-10-09T05:56:27.199 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:44.159123+0000", 2023-10-09T05:56:27.199 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 42.889237368000003, 2023-10-09T05:56:27.199 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.563185697, 2023-10-09T05:56:27.200 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:27.200 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:27.200 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:27.200 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:27.200 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:27.200 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 119217 2023-10-09T05:56:27.200 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.200 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:27.200 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.200 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:27.200 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.159123+0000", 2023-10-09T05:56:27.201 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.201 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.201 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.201 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:27.201 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.159123+0000", 2023-10-09T05:56:27.201 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.201 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.201 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.201 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:27.201 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.159122+0000", 2023-10-09T05:56:27.201 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:27.201 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.202 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.202 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:27.202 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.159127+0000", 2023-10-09T05:56:27.202 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5890000000000002e-06 2023-10-09T05:56:27.202 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.202 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.202 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:27.202 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.159141+0000", 2023-10-09T05:56:27.202 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3817e-05 2023-10-09T05:56:27.202 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.202 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.203 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:27.203 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.159147+0000", 2023-10-09T05:56:27.203 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3559999999999998e-06 2023-10-09T05:56:27.203 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.203 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.203 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.203 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.159199+0000", 2023-10-09T05:56:27.203 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1994999999999999e-05 2023-10-09T05:56:27.203 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.203 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.203 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:27.203 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.159212+0000", 2023-10-09T05:56:27.204 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.274e-05 2023-10-09T05:56:27.204 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.204 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.204 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.204 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.674511+0000", 2023-10-09T05:56:27.204 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.51529928599999997 2023-10-09T05:56:27.204 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.204 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.204 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:27.204 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.674517+0000", 2023-10-09T05:56:27.204 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0000000000000002e-06 2023-10-09T05:56:27.204 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.205 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.205 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.205 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.709948+0000", 2023-10-09T05:56:27.205 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.035430275 2023-10-09T05:56:27.205 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.205 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.205 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:27.205 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.710076+0000", 2023-10-09T05:56:27.205 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000128572 2023-10-09T05:56:27.205 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.205 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.206 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:56:27.206 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.710257+0000", 2023-10-09T05:56:27.206 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000180853 2023-10-09T05:56:27.206 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.206 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.206 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:27.206 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.712521+0000", 2023-10-09T05:56:27.206 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0022643009999999998 2023-10-09T05:56:27.206 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.206 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.206 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.207 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.716103+0000", 2023-10-09T05:56:27.207 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0035815819999999998 2023-10-09T05:56:27.207 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.207 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.207 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.207 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.722210+0000", 2023-10-09T05:56:27.207 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0061075399999999998 2023-10-09T05:56:27.207 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.207 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.207 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:27.207 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.722264+0000", 2023-10-09T05:56:27.208 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3260000000000002e-05 2023-10-09T05:56:27.208 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.208 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.208 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:27.208 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.722309+0000", 2023-10-09T05:56:27.208 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4950999999999997e-05 2023-10-09T05:56:27.208 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.208 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:27.208 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.208 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.208 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.208 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:119223 6.1f 6:ff85ea67:::benchmark_data_smithi064_284193_object22680:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-10-09T05:56:27.209 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:44.240943+0000", 2023-10-09T05:56:27.209 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 42.807416953000001, 2023-10-09T05:56:27.209 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.63528576199999998, 2023-10-09T05:56:27.209 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:27.209 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:27.209 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:27.209 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:27.209 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:27.209 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 119223 2023-10-09T05:56:27.209 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.209 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:27.209 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.209 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:27.210 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.240943+0000", 2023-10-09T05:56:27.210 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.210 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.210 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.210 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:27.210 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.240943+0000", 2023-10-09T05:56:27.210 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.210 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.210 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.210 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:27.210 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.240942+0000", 2023-10-09T05:56:27.211 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:27.211 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.211 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.211 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:27.211 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.240950+0000", 2023-10-09T05:56:27.211 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.622e-06 2023-10-09T05:56:27.211 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.211 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.211 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:27.211 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.240971+0000", 2023-10-09T05:56:27.211 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0917e-05 2023-10-09T05:56:27.212 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.212 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.212 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:27.212 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.240977+0000", 2023-10-09T05:56:27.212 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8639999999999998e-06 2023-10-09T05:56:27.212 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.212 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.212 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.212 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.241041+0000", 2023-10-09T05:56:27.212 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3283999999999999e-05 2023-10-09T05:56:27.212 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.212 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.213 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:27.213 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.241052+0000", 2023-10-09T05:56:27.213 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1531e-05 2023-10-09T05:56:27.213 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.213 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.213 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.213 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.870636+0000", 2023-10-09T05:56:27.213 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.62958364099999997 2023-10-09T05:56:27.213 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.213 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.213 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:27.213 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.870728+0000", 2023-10-09T05:56:27.213 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1800999999999997e-05 2023-10-09T05:56:27.214 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.214 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.214 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T05:56:27.214 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.870954+0000", 2023-10-09T05:56:27.214 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00022582900000000001 2023-10-09T05:56:27.214 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.214 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.214 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:27.214 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.871996+0000", 2023-10-09T05:56:27.214 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001042659 2023-10-09T05:56:27.214 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.215 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.215 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.215 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.873826+0000", 2023-10-09T05:56:27.215 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0018302049999999999 2023-10-09T05:56:27.215 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.215 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.215 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.215 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.874054+0000", 2023-10-09T05:56:27.215 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000227211 2023-10-09T05:56:27.215 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.215 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.215 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.215 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.874516+0000", 2023-10-09T05:56:27.216 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00046207300000000002 2023-10-09T05:56:27.216 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.216 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.216 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:27.216 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.874550+0000", 2023-10-09T05:56:27.216 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.4430000000000001e-05 2023-10-09T05:56:27.216 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.216 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.216 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:27.216 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.876229+0000", 2023-10-09T05:56:27.216 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0016789680000000001 2023-10-09T05:56:27.216 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.217 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:27.217 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.217 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.217 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.217 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:119228 6.1e 6:7e232574:::benchmark_data_smithi064_284193_object22685:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-10-09T05:56:27.217 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:44.298324+0000", 2023-10-09T05:56:27.217 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 42.750036233000003, 2023-10-09T05:56:27.217 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4516483600000001, 2023-10-09T05:56:27.217 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:27.217 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:27.218 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:27.218 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:27.218 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:27.218 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 119228 2023-10-09T05:56:27.218 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.218 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:27.218 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.218 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:27.218 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.298324+0000", 2023-10-09T05:56:27.218 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.218 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.218 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.219 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:27.219 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.298324+0000", 2023-10-09T05:56:27.219 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:27.219 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.219 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.219 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:27.219 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.298323+0000", 2023-10-09T05:56:27.219 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:56:27.219 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.219 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.219 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:27.220 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.298331+0000", 2023-10-09T05:56:27.220 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.3340000000000002e-06 2023-10-09T05:56:27.220 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.220 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.220 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:27.220 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.298346+0000", 2023-10-09T05:56:27.220 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4276e-05 2023-10-09T05:56:27.220 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.220 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.220 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:27.221 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.298352+0000", 2023-10-09T05:56:27.221 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1850000000000001e-06 2023-10-09T05:56:27.221 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.221 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.221 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.221 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.298398+0000", 2023-10-09T05:56:27.221 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5918000000000003e-05 2023-10-09T05:56:27.221 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.221 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.221 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:27.222 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.298406+0000", 2023-10-09T05:56:27.222 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.4370000000000007e-06 2023-10-09T05:56:27.222 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.222 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.222 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.222 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.674538+0000", 2023-10-09T05:56:27.222 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.37613181699999998 2023-10-09T05:56:27.222 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.222 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.222 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:27.222 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.674544+0000", 2023-10-09T05:56:27.222 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0889999999999996e-06 2023-10-09T05:56:27.223 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.223 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.223 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:27.223 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.711440+0000", 2023-10-09T05:56:27.223 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0368962559999999 2023-10-09T05:56:27.223 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.223 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.223 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:27.223 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.711559+0000", 2023-10-09T05:56:27.223 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011837600000000001 2023-10-09T05:56:27.223 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.223 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.223 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:56:27.224 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.711718+0000", 2023-10-09T05:56:27.224 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015891200000000001 2023-10-09T05:56:27.224 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.224 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.224 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:27.224 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.712613+0000", 2023-10-09T05:56:27.224 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00089554800000000003 2023-10-09T05:56:27.224 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.224 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.224 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.224 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.716132+0000", 2023-10-09T05:56:27.225 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0035189499999999999 2023-10-09T05:56:27.225 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.225 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.225 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:27.225 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.749904+0000", 2023-10-09T05:56:27.225 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.033771382000000003 2023-10-09T05:56:27.225 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.225 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.225 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:27.225 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.749944+0000", 2023-10-09T05:56:27.225 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0908999999999997e-05 2023-10-09T05:56:27.226 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:27.226 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:27.226 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:27.226 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.749972+0000", 2023-10-09T05:56:27.226 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7999000000000001e-05 2023-10-09T05:56:27.226 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.226 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:27.226 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.226 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:27.226 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:27.226 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:56:27.227 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:27.290 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:27.380 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:27.383 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:27 smithi064 ceph-mon[212750]: pgmap v523: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 30590/187875 objects degraded (16.282%); 18023/187875 objects misplaced (9.593%); 14 MiB/s, 276 objects/s recovering 2023-10-09T05:56:27.383 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:27 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 30682/190017 objects degraded (16.147%), 14 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:27.383 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:27 smithi064 ceph-mon[220044]: pgmap v523: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 30590/187875 objects degraded (16.282%); 18023/187875 objects misplaced (9.593%); 14 MiB/s, 276 objects/s recovering 2023-10-09T05:56:27.383 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:27 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 30682/190017 objects degraded (16.147%), 14 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:27.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:27.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:27.485 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:27.514 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:27 smithi156 ceph-mon[195101]: pgmap v523: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 30590/187875 objects degraded (16.282%); 18023/187875 objects misplaced (9.593%); 14 MiB/s, 276 objects/s recovering 2023-10-09T05:56:27.514 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:27 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 30682/190017 objects degraded (16.147%), 14 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:27.516 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:27.559 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:27.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:27.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:27.662 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:27.708 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:27.798 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:27.819 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:27.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:27.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:27.925 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:27.958 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:28.036 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:28.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:28.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:28.071 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:56:28.143 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:28.206 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:28.301 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:28.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:28.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:28.331 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:56:28.409 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:28.441 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:28.508 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:28.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:28.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:28.586 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:56:28.603 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:56:28.613 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:28.630 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:56:28.672 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:56:28.675 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:28 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:56:28.675 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:28 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:56:28.679 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:28.705 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:56:28.738 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:56:28.763 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:56:28.787 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:56:28.813 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:56:28.816 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:56:28.816 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:56:28.816 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:56:28.816 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:56:28.816 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:56:28.816 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:56:28.816 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:56:28.816 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:56:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:28 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:56:29.609 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:29.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:29.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:29.712 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:29.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:29 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/566007553' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:29.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:29 smithi064 ceph-mon[220044]: pgmap v524: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 405 op/s; 30590/186867 objects degraded (16.370%); 17841/186867 objects misplaced (9.547%); 4.6 MiB/s, 96 objects/s recovering 2023-10-09T05:56:29.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:29 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/566007553' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:29.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:29 smithi064 ceph-mon[212750]: pgmap v524: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 405 op/s; 30590/186867 objects degraded (16.370%); 17841/186867 objects misplaced (9.547%); 4.6 MiB/s, 96 objects/s recovering 2023-10-09T05:56:29.759 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:29.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:29 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/566007553' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:29.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:29 smithi156 ceph-mon[195101]: pgmap v524: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 405 op/s; 30590/186867 objects degraded (16.370%); 17841/186867 objects misplaced (9.547%); 4.6 MiB/s, 96 objects/s recovering 2023-10-09T05:56:29.919 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:29.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:29.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:30.026 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:56:30.108 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:30.226 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:30.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:30.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:30.342 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:30.343 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:30.387 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:30.468 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:30.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:30.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:30.573 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:30.644 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:30.700 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T05:56:30.700 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:56:30.825 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:56:30.827 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T05:56:30.840 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:30.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:30.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:30.952 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:31.007 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:31.073 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:31.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:31.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:31.176 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:56:31.217 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:31.376 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:31.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:31.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:31.480 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:31.517 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:31.629 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:31 smithi064 ceph-mon[212750]: pgmap v525: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 29746/179100 objects degraded (16.609%); 17090/179100 objects misplaced (9.542%); 8.5 MiB/s, 156 objects/s recovering 2023-10-09T05:56:31.630 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:31 smithi064 ceph-mon[212750]: from='client.45056 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:56:31.745 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:56:31.834 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:31.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:31.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:31.942 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:56:31.977 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:31 smithi064 ceph-mon[220044]: pgmap v525: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 29746/179100 objects degraded (16.609%); 17090/179100 objects misplaced (9.542%); 8.5 MiB/s, 156 objects/s recovering 2023-10-09T05:56:31.977 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:31 smithi064 ceph-mon[220044]: from='client.45056 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:56:31.985 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:32.120 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:56:32.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:31 smithi156 ceph-mon[195101]: pgmap v525: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 29746/179100 objects degraded (16.609%); 17090/179100 objects misplaced (9.542%); 8.5 MiB/s, 156 objects/s recovering 2023-10-09T05:56:32.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:31 smithi156 ceph-mon[195101]: from='client.45056 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:56:32.285 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:32.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:32.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:32.353 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:56:32.399 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:32.415 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":525,"stamp":"2023-10-09T05:56:30.564713+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3911456696,"num_objects":59700,"num_object_clones":0,"num_object_copies":179100,"num_objects_missing_on_primary":1539,"num_objects_missing":1539,"num_objects_degraded":29746,"num_objects_misplaced":17090,"num_objects_unfound":0,"num_objects_dirty":59700,"num_whiteouts":0,"num_read":488,"num_read_kb":914,"num_write":133706,"num_write_kb":6181006,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50937,"num_bytes_recovered":3200594614,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":234731,"ondisk_log_size":234731,"up":387,"acting":379,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":417,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14361692,"kb_used_data":12673572,"kb_used_omap":118,"kb_used_meta":1687817,"kb_avail":641854372,"statfs":{"total":671965249536,"available":657258876928,"internally_reserved":0,"allocated":12977737728,"data_stored":12970670230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":121648,"internal_metadata":1728324816},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,2,1,2,1,1,3,5,1,1],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-358219776,"num_objects":-5466,"num_object_clones":0,"num_object_copies":-16398,"num_objects_missing_on_primary":-9,"num_objects_missing":-9,"num_objects_degraded":-1527,"num_objects_misplaced":-1792,"num_objects_unfound":0,"num_objects_dirty":-5466,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5466,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1562,"num_bytes_recovered":89456640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003400"},"pg_stats":[{"pgid":"6.23","version":"226'7126","reported_seq":7294,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:56:30.267282+0000","last_change":"2023-10-09T05:55:22.225278+0000","last_active":"2023-10-09T05:56:30.267282+0000","last_peered":"2023-10-09T05:56:30.267282+0000","last_clean":"2023-10-09T05:56:30.267282+0000","last_became_active":"2023-10-09T05:55:22.222511+0000","last_became_peered":"2023-10-09T05:55:22.222511+0000","last_unstale":"2023-10-09T05:56:30.267282+0000","last_undegraded":"2023-10-09T05:56:30.267282+0000","last_fullsized":"2023-10-09T05:56:30.267282+0000","mapping_epoch":207,"log_start":"107'1488","ondisk_log_start":"107'1488","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5638,"log_dups_size":0,"ondisk_log_size":5638,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59588608,"num_objects":909,"num_object_clones":0,"num_object_copies":2727,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":909,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2132,"num_write_kb":97312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.22","version":"226'7084","reported_seq":19295,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:56:29.338317+0000","last_change":"2023-10-09T05:55:22.229226+0000","last_active":"2023-10-09T05:56:29.338317+0000","last_peered":"2023-10-09T05:56:29.338317+0000","last_clean":"2023-10-09T05:56:29.338317+0000","last_became_active":"2023-10-09T05:55:22.223863+0000","last_became_peered":"2023-10-09T05:55:22.223863+0000","last_unstale":"2023-10-09T05:56:29.338317+0000","last_undegraded":"2023-10-09T05:56:29.338317+0000","last_fullsized":"2023-10-09T05:56:29.338317+0000","mapping_epoch":207,"log_start":"106'632","ondisk_log_start":"106'632","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6452,"log_dups_size":0,"ondisk_log_size":6452,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60637184,"num_objects":925,"num_object_clones":0,"num_object_copies":2775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2084,"num_write_kb":96304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1505,"num_bytes_recovered":95617024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.21","version":"226'7129","reported_seq":7696,"reported_epoch":226,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:26.966760+0000","last_change":"2023-10-09T05:56:00.525133+0000","last_active":"2023-10-09T05:56:26.966760+0000","last_peered":"2023-10-09T05:56:26.966760+0000","last_clean":"2023-10-09T05:51:24.563560+0000","last_became_active":"2023-10-09T05:56:00.506782+0000","last_became_peered":"2023-10-09T05:56:00.506782+0000","last_unstale":"2023-10-09T05:56:26.966760+0000","last_undegraded":"2023-10-09T05:56:00.339085+0000","last_fullsized":"2023-10-09T05:56:00.284538+0000","mapping_epoch":219,"log_start":"107'871","ondisk_log_start":"107'871","created":207,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6258,"log_dups_size":0,"ondisk_log_size":6258,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62160896,"num_objects":949,"num_object_clones":0,"num_object_copies":2847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1314,"num_objects_misplaced":422,"num_objects_unfound":0,"num_objects_dirty":949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2105,"num_write_kb":97696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":190,"num_bytes_recovered":12386304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,1],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"1,2,6","objects":424},{"shards":"1,6","objects":-177},{"shards":"2,6,7","objects":527}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.20","version":"226'7087","reported_seq":10878,"reported_epoch":226,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:56:29.318946+0000","last_change":"2023-10-09T05:56:00.444495+0000","last_active":"2023-10-09T05:56:29.318946+0000","last_peered":"2023-10-09T05:56:29.318946+0000","last_clean":"2023-10-09T05:54:10.146196+0000","last_became_active":"2023-10-09T05:56:00.423961+0000","last_became_peered":"2023-10-09T05:56:00.423961+0000","last_unstale":"2023-10-09T05:56:29.318946+0000","last_undegraded":"2023-10-09T05:56:00.318106+0000","last_fullsized":"2023-10-09T05:56:29.318946+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7087,"log_dups_size":0,"ondisk_log_size":7087,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62144512,"num_objects":948,"num_object_clones":0,"num_object_copies":2844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2146,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2090,"num_write_kb":97216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":749,"num_bytes_recovered":49070080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":947},{"shards":"0,4,5","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1f","version":"226'7718","reported_seq":8653,"reported_epoch":226,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T05:56:29.471945+0000","last_change":"2023-10-09T05:55:59.464124+0000","last_active":"2023-10-09T05:56:29.471945+0000","last_peered":"2023-10-09T05:56:29.471945+0000","last_clean":"2023-10-09T05:52:22.632757+0000","last_became_active":"2023-10-09T05:55:59.409285+0000","last_became_peered":"2023-10-09T05:55:59.409285+0000","last_unstale":"2023-10-09T05:56:29.471945+0000","last_undegraded":"2023-10-09T05:55:59.398316+0000","last_fullsized":"2023-10-09T05:56:29.471945+0000","mapping_epoch":218,"log_start":"107'1991","ondisk_log_start":"107'1991","created":207,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5727,"log_dups_size":0,"ondisk_log_size":5727,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123895808,"num_objects":1891,"num_object_clones":0,"num_object_copies":5673,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1891,"num_objects_misplaced":2133,"num_objects_unfound":0,"num_objects_dirty":1891,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4251,"num_write_kb":196512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":278,"num_bytes_recovered":18219008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,3],"avail_no_missing":["4","1","2","3"],"object_location_counts":[{"shards":"2,3,4","objects":1935},{"shards":"3,4,6","objects":-44}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":271,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.221544+0000","last_change":"2023-10-09T05:55:59.341215+0000","last_active":"2023-10-09T05:56:20.221544+0000","last_peered":"2023-10-09T05:56:20.221544+0000","last_clean":"2023-10-09T05:56:20.221544+0000","last_became_active":"2023-10-09T05:55:59.340856+0000","last_became_peered":"2023-10-09T05:55:59.340856+0000","last_unstale":"2023-10-09T05:56:20.221544+0000","last_undegraded":"2023-10-09T05:56:20.221544+0000","last_fullsized":"2023-10-09T05:56:20.221544+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:33.686454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":153,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.221898+0000","last_change":"2023-10-09T05:55:59.328988+0000","last_active":"2023-10-09T05:56:20.221898+0000","last_peered":"2023-10-09T05:56:20.221898+0000","last_clean":"2023-10-09T05:56:20.221898+0000","last_became_active":"2023-10-09T05:55:59.323104+0000","last_became_peered":"2023-10-09T05:55:59.323104+0000","last_unstale":"2023-10-09T05:56:20.221898+0000","last_undegraded":"2023-10-09T05:56:20.221898+0000","last_fullsized":"2023-10-09T05:56:20.221898+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:09:14.587595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.1c","version":"226'7532","reported_seq":19782,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:56:26.890744+0000","last_change":"2023-10-09T05:55:22.232121+0000","last_active":"2023-10-09T05:56:26.890744+0000","last_peered":"2023-10-09T05:56:26.890744+0000","last_clean":"2023-10-09T05:56:26.890744+0000","last_became_active":"2023-10-09T05:55:22.226434+0000","last_became_peered":"2023-10-09T05:55:22.226434+0000","last_unstale":"2023-10-09T05:56:26.890744+0000","last_undegraded":"2023-10-09T05:56:26.890744+0000","last_fullsized":"2023-10-09T05:56:26.890744+0000","mapping_epoch":207,"log_start":"108'2503","ondisk_log_start":"108'2503","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5029,"log_dups_size":0,"ondisk_log_size":5029,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126877696,"num_objects":1936,"num_object_clones":0,"num_object_copies":5808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1936,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4097,"num_write_kb":193056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3017,"num_bytes_recovered":192643072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":104,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.331969+0000","last_change":"2023-10-09T05:55:59.304893+0000","last_active":"2023-10-09T05:56:20.331969+0000","last_peered":"2023-10-09T05:56:20.331969+0000","last_clean":"2023-10-09T05:56:20.331969+0000","last_became_active":"2023-10-09T05:55:59.304449+0000","last_became_peered":"2023-10-09T05:55:59.304449+0000","last_unstale":"2023-10-09T05:56:20.331969+0000","last_undegraded":"2023-10-09T05:56:20.331969+0000","last_fullsized":"2023-10-09T05:56:20.331969+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:44:39.453982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":227,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.221033+0000","last_change":"2023-10-09T05:55:59.306657+0000","last_active":"2023-10-09T05:56:20.221033+0000","last_peered":"2023-10-09T05:56:20.221033+0000","last_clean":"2023-10-09T05:56:20.221033+0000","last_became_active":"2023-10-09T05:55:59.306185+0000","last_became_peered":"2023-10-09T05:55:59.306185+0000","last_unstale":"2023-10-09T05:56:20.221033+0000","last_undegraded":"2023-10-09T05:56:20.221033+0000","last_fullsized":"2023-10-09T05:56:20.221033+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:26.767566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1d","version":"226'7581","reported_seq":17459,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:56:29.313920+0000","last_change":"2023-10-09T05:56:05.381643+0000","last_active":"2023-10-09T05:56:29.313920+0000","last_peered":"2023-10-09T05:56:29.313920+0000","last_clean":"2023-10-09T05:56:29.313920+0000","last_became_active":"2023-10-09T05:56:05.368691+0000","last_became_peered":"2023-10-09T05:56:05.368691+0000","last_unstale":"2023-10-09T05:56:29.313920+0000","last_undegraded":"2023-10-09T05:56:29.313920+0000","last_fullsized":"2023-10-09T05:56:29.313920+0000","mapping_epoch":221,"log_start":"108'2175","ondisk_log_start":"108'2175","created":207,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5406,"log_dups_size":0,"ondisk_log_size":5406,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121241612,"num_objects":1851,"num_object_clones":0,"num_object_copies":5553,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1851,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4152,"num_write_kb":192033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3518,"num_bytes_recovered":230424600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":224,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.220179+0000","last_change":"2023-10-09T05:55:59.306975+0000","last_active":"2023-10-09T05:56:20.220179+0000","last_peered":"2023-10-09T05:56:20.220179+0000","last_clean":"2023-10-09T05:56:20.220179+0000","last_became_active":"2023-10-09T05:55:59.305932+0000","last_became_peered":"2023-10-09T05:55:59.305932+0000","last_unstale":"2023-10-09T05:56:20.220179+0000","last_undegraded":"2023-10-09T05:56:20.220179+0000","last_fullsized":"2023-10-09T05:56:20.220179+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:41.191008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":252,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.346038+0000","last_change":"2023-10-09T05:55:59.335464+0000","last_active":"2023-10-09T05:56:20.346038+0000","last_peered":"2023-10-09T05:56:20.346038+0000","last_clean":"2023-10-09T05:56:20.346038+0000","last_became_active":"2023-10-09T05:55:59.335259+0000","last_became_peered":"2023-10-09T05:55:59.335259+0000","last_unstale":"2023-10-09T05:56:20.346038+0000","last_undegraded":"2023-10-09T05:56:20.346038+0000","last_fullsized":"2023-10-09T05:56:20.346038+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:28.536682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.12","version":"226'7498","reported_seq":19699,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:56:29.381407+0000","last_change":"2023-10-09T05:54:45.695681+0000","last_active":"2023-10-09T05:56:29.381407+0000","last_peered":"2023-10-09T05:56:29.381407+0000","last_clean":"2023-10-09T05:56:29.381407+0000","last_became_active":"2023-10-09T05:54:45.691675+0000","last_became_peered":"2023-10-09T05:54:45.691675+0000","last_unstale":"2023-10-09T05:56:29.381407+0000","last_undegraded":"2023-10-09T05:56:29.381407+0000","last_fullsized":"2023-10-09T05:56:29.381407+0000","mapping_epoch":188,"log_start":"106'632","ondisk_log_start":"106'632","created":188,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6866,"log_dups_size":0,"ondisk_log_size":6866,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117997568,"num_objects":1801,"num_object_clones":0,"num_object_copies":5403,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1801,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4219,"num_write_kb":192608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3010,"num_bytes_recovered":191234048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":171,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.221470+0000","last_change":"2023-10-09T05:54:16.209688+0000","last_active":"2023-10-09T05:56:20.221470+0000","last_peered":"2023-10-09T05:56:20.221470+0000","last_clean":"2023-10-09T05:56:20.221470+0000","last_became_active":"2023-10-09T05:54:16.209338+0000","last_became_peered":"2023-10-09T05:54:16.209338+0000","last_unstale":"2023-10-09T05:56:20.221470+0000","last_undegraded":"2023-10-09T05:56:20.221470+0000","last_fullsized":"2023-10-09T05:56:20.221470+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:28.211691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":153,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.336080+0000","last_change":"2023-10-09T05:55:59.322664+0000","last_active":"2023-10-09T05:56:20.336080+0000","last_peered":"2023-10-09T05:56:20.336080+0000","last_clean":"2023-10-09T05:56:20.336080+0000","last_became_active":"2023-10-09T05:55:59.322242+0000","last_became_peered":"2023-10-09T05:55:59.322242+0000","last_unstale":"2023-10-09T05:56:20.336080+0000","last_undegraded":"2023-10-09T05:56:20.336080+0000","last_fullsized":"2023-10-09T05:56:20.336080+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:40:15.452836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.13","version":"226'7524","reported_seq":7681,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:56:30.393933+0000","last_change":"2023-10-09T05:54:45.686838+0000","last_active":"2023-10-09T05:56:30.393933+0000","last_peered":"2023-10-09T05:56:30.393933+0000","last_clean":"2023-10-09T05:56:30.393933+0000","last_became_active":"2023-10-09T05:54:45.682331+0000","last_became_peered":"2023-10-09T05:54:45.682331+0000","last_unstale":"2023-10-09T05:56:30.393933+0000","last_undegraded":"2023-10-09T05:56:30.393933+0000","last_fullsized":"2023-10-09T05:56:30.393933+0000","mapping_epoch":188,"log_start":"107'1488","ondisk_log_start":"107'1488","created":188,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6036,"log_dups_size":0,"ondisk_log_size":6036,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119635968,"num_objects":1826,"num_object_clones":0,"num_object_copies":5478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4257,"num_write_kb":194624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":474,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.339297+0000","last_change":"2023-10-09T05:54:19.204752+0000","last_active":"2023-10-09T05:56:20.339297+0000","last_peered":"2023-10-09T05:56:20.339297+0000","last_clean":"2023-10-09T05:56:20.339297+0000","last_became_active":"2023-10-09T05:54:19.204405+0000","last_became_peered":"2023-10-09T05:54:19.204405+0000","last_unstale":"2023-10-09T05:56:20.339297+0000","last_undegraded":"2023-10-09T05:56:20.339297+0000","last_fullsized":"2023-10-09T05:56:20.339297+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":256,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.336426+0000","last_change":"2023-10-09T05:55:59.305767+0000","last_active":"2023-10-09T05:56:20.336426+0000","last_peered":"2023-10-09T05:56:20.336426+0000","last_clean":"2023-10-09T05:56:20.336426+0000","last_became_active":"2023-10-09T05:55:59.305447+0000","last_became_peered":"2023-10-09T05:55:59.305447+0000","last_unstale":"2023-10-09T05:56:20.336426+0000","last_undegraded":"2023-10-09T05:56:20.336426+0000","last_fullsized":"2023-10-09T05:56:20.336426+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:43.228681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":169,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.331856+0000","last_change":"2023-10-09T05:53:59.185702+0000","last_active":"2023-10-09T05:56:20.331856+0000","last_peered":"2023-10-09T05:56:20.331856+0000","last_clean":"2023-10-09T05:56:20.331856+0000","last_became_active":"2023-10-09T05:53:59.185373+0000","last_became_peered":"2023-10-09T05:53:59.185373+0000","last_unstale":"2023-10-09T05:56:20.331856+0000","last_undegraded":"2023-10-09T05:56:20.331856+0000","last_fullsized":"2023-10-09T05:56:20.331856+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:50.756967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"226'7481","reported_seq":15712,"reported_epoch":226,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:29.414572+0000","last_change":"2023-10-09T05:56:00.609171+0000","last_active":"2023-10-09T05:56:29.414572+0000","last_peered":"2023-10-09T05:56:29.414572+0000","last_clean":"2023-10-09T05:54:10.146196+0000","last_became_active":"2023-10-09T05:56:00.562362+0000","last_became_peered":"2023-10-09T05:56:00.562362+0000","last_unstale":"2023-10-09T05:56:29.414572+0000","last_undegraded":"2023-10-09T05:56:00.391169+0000","last_fullsized":"2023-10-09T05:56:00.277979+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":188,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7481,"log_dups_size":0,"ondisk_log_size":7481,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122978304,"num_objects":1877,"num_object_clones":0,"num_object_copies":5631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3900,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4200,"num_write_kb":194432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2730,"num_bytes_recovered":178880512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,4],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,1,2,3,5","objects":842},{"shards":"0,2,3","objects":1021},{"shards":"0,2,3,5","objects":1},{"shards":"0,4","objects":13}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":66,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.220602+0000","last_change":"2023-10-09T05:53:59.219168+0000","last_active":"2023-10-09T05:56:20.220602+0000","last_peered":"2023-10-09T05:56:20.220602+0000","last_clean":"2023-10-09T05:56:20.220602+0000","last_became_active":"2023-10-09T05:53:59.200886+0000","last_became_peered":"2023-10-09T05:53:59.200886+0000","last_unstale":"2023-10-09T05:56:20.220602+0000","last_undegraded":"2023-10-09T05:56:20.220602+0000","last_fullsized":"2023-10-09T05:56:20.220602+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:45.515036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":120,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.346352+0000","last_change":"2023-10-09T05:55:59.305100+0000","last_active":"2023-10-09T05:56:20.346352+0000","last_peered":"2023-10-09T05:56:20.346352+0000","last_clean":"2023-10-09T05:56:20.346352+0000","last_became_active":"2023-10-09T05:55:59.304758+0000","last_became_peered":"2023-10-09T05:55:59.304758+0000","last_unstale":"2023-10-09T05:56:20.346352+0000","last_undegraded":"2023-10-09T05:56:20.346352+0000","last_fullsized":"2023-10-09T05:56:20.346352+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:54:32.510945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.11","version":"226'7434","reported_seq":7706,"reported_epoch":226,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:26.836892+0000","last_change":"2023-10-09T05:56:00.563506+0000","last_active":"2023-10-09T05:56:26.836892+0000","last_peered":"2023-10-09T05:56:26.836892+0000","last_clean":"2023-10-09T05:51:24.563560+0000","last_became_active":"2023-10-09T05:56:00.515673+0000","last_became_peered":"2023-10-09T05:56:00.515673+0000","last_unstale":"2023-10-09T05:56:26.836892+0000","last_undegraded":"2023-10-09T05:56:00.486623+0000","last_fullsized":"2023-10-09T05:56:00.384416+0000","mapping_epoch":219,"log_start":"107'871","ondisk_log_start":"107'871","created":188,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6563,"log_dups_size":0,"ondisk_log_size":6563,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128188416,"num_objects":1956,"num_object_clones":0,"num_object_copies":5868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3053,"num_objects_misplaced":838,"num_objects_unfound":0,"num_objects_dirty":1956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4150,"num_write_kb":195392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":3014656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,1],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"1,2,6","objects":866},{"shards":"1,6","objects":-367},{"shards":"2,6,7","objects":1118}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":231,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.221407+0000","last_change":"2023-10-09T05:53:59.219141+0000","last_active":"2023-10-09T05:56:20.221407+0000","last_peered":"2023-10-09T05:56:20.221407+0000","last_clean":"2023-10-09T05:56:20.221407+0000","last_became_active":"2023-10-09T05:53:59.197318+0000","last_became_peered":"2023-10-09T05:53:59.197318+0000","last_unstale":"2023-10-09T05:56:20.221407+0000","last_undegraded":"2023-10-09T05:56:20.221407+0000","last_fullsized":"2023-10-09T05:56:20.221407+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:39.910794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":235,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.222440+0000","last_change":"2023-10-09T05:55:59.335824+0000","last_active":"2023-10-09T05:56:20.222440+0000","last_peered":"2023-10-09T05:56:20.222440+0000","last_clean":"2023-10-09T05:56:20.222440+0000","last_became_active":"2023-10-09T05:55:59.335478+0000","last_became_peered":"2023-10-09T05:55:59.335478+0000","last_unstale":"2023-10-09T05:56:20.222440+0000","last_undegraded":"2023-10-09T05:56:20.222440+0000","last_fullsized":"2023-10-09T05:56:20.222440+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:26.014501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.16","version":"226'7432","reported_seq":13714,"reported_epoch":226,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:56:29.375054+0000","last_change":"2023-10-09T05:56:12.831696+0000","last_active":"2023-10-09T05:56:29.375054+0000","last_peered":"2023-10-09T05:56:29.375054+0000","last_clean":"2023-10-09T05:52:24.946135+0000","last_became_active":"2023-10-09T05:54:47.740044+0000","last_became_peered":"2023-10-09T05:54:47.740044+0000","last_unstale":"2023-10-09T05:56:29.375054+0000","last_undegraded":"2023-10-09T05:56:29.375054+0000","last_fullsized":"2023-10-09T05:56:29.375054+0000","mapping_epoch":190,"log_start":"106'613","ondisk_log_start":"106'613","created":188,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6819,"log_dups_size":0,"ondisk_log_size":6819,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121536512,"num_objects":1855,"num_object_clones":0,"num_object_copies":5565,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1855,"num_objects_unfound":0,"num_objects_dirty":1855,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4168,"num_write_kb":192704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3030,"num_bytes_recovered":183042048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,6],"avail_no_missing":["4","3","6","7"],"object_location_counts":[{"shards":"4,6,7","objects":1855}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":556,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.336412+0000","last_change":"2023-10-09T05:53:20.237354+0000","last_active":"2023-10-09T05:56:20.336412+0000","last_peered":"2023-10-09T05:56:20.336412+0000","last_clean":"2023-10-09T05:56:20.336412+0000","last_became_active":"2023-10-09T05:53:20.236993+0000","last_became_peered":"2023-10-09T05:53:20.236993+0000","last_unstale":"2023-10-09T05:56:20.336412+0000","last_undegraded":"2023-10-09T05:56:20.336412+0000","last_fullsized":"2023-10-09T05:56:20.336412+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":68,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.220791+0000","last_change":"2023-10-09T05:53:59.233898+0000","last_active":"2023-10-09T05:56:20.220791+0000","last_peered":"2023-10-09T05:56:20.220791+0000","last_clean":"2023-10-09T05:56:20.220791+0000","last_became_active":"2023-10-09T05:53:59.204175+0000","last_became_peered":"2023-10-09T05:53:59.204175+0000","last_unstale":"2023-10-09T05:56:20.220791+0000","last_undegraded":"2023-10-09T05:56:20.220791+0000","last_fullsized":"2023-10-09T05:56:20.220791+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:03.139318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":255,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.331222+0000","last_change":"2023-10-09T05:55:59.297087+0000","last_active":"2023-10-09T05:56:20.331222+0000","last_peered":"2023-10-09T05:56:20.331222+0000","last_clean":"2023-10-09T05:56:20.331222+0000","last_became_active":"2023-10-09T05:55:59.296768+0000","last_became_peered":"2023-10-09T05:55:59.296768+0000","last_unstale":"2023-10-09T05:56:20.331222+0000","last_undegraded":"2023-10-09T05:56:20.331222+0000","last_fullsized":"2023-10-09T05:56:20.331222+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:35.067705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"226'7320","reported_seq":7462,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:56:29.442346+0000","last_change":"2023-10-09T05:54:45.686122+0000","last_active":"2023-10-09T05:56:29.442346+0000","last_peered":"2023-10-09T05:56:29.442346+0000","last_clean":"2023-10-09T05:56:29.442346+0000","last_became_active":"2023-10-09T05:54:45.681370+0000","last_became_peered":"2023-10-09T05:54:45.681370+0000","last_unstale":"2023-10-09T05:56:29.442346+0000","last_undegraded":"2023-10-09T05:56:29.442346+0000","last_fullsized":"2023-10-09T05:56:29.442346+0000","mapping_epoch":188,"log_start":"106'370","ondisk_log_start":"106'370","created":188,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6950,"log_dups_size":0,"ondisk_log_size":6950,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121012224,"num_objects":1847,"num_object_clones":0,"num_object_copies":5541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4114,"num_write_kb":190720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":619,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.220342+0000","last_change":"2023-10-09T05:54:27.330211+0000","last_active":"2023-10-09T05:56:20.220342+0000","last_peered":"2023-10-09T05:56:20.220342+0000","last_clean":"2023-10-09T05:56:20.220342+0000","last_became_active":"2023-10-09T05:53:59.202399+0000","last_became_peered":"2023-10-09T05:53:59.202399+0000","last_unstale":"2023-10-09T05:56:20.220342+0000","last_undegraded":"2023-10-09T05:56:20.220342+0000","last_fullsized":"2023-10-09T05:56:20.220342+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:51:12.346243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:51:12.346243+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":217,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.219549+0000","last_change":"2023-10-09T05:55:59.297178+0000","last_active":"2023-10-09T05:56:20.219549+0000","last_peered":"2023-10-09T05:56:20.219549+0000","last_clean":"2023-10-09T05:56:20.219549+0000","last_became_active":"2023-10-09T05:55:59.296874+0000","last_became_peered":"2023-10-09T05:55:59.296874+0000","last_unstale":"2023-10-09T05:56:20.219549+0000","last_undegraded":"2023-10-09T05:56:20.219549+0000","last_fullsized":"2023-10-09T05:56:20.219549+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:32.817774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":205,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.336939+0000","last_change":"2023-10-09T05:53:59.182073+0000","last_active":"2023-10-09T05:56:20.336939+0000","last_peered":"2023-10-09T05:56:20.336939+0000","last_clean":"2023-10-09T05:56:20.336939+0000","last_became_active":"2023-10-09T05:53:59.181715+0000","last_became_peered":"2023-10-09T05:53:59.181715+0000","last_unstale":"2023-10-09T05:56:20.336939+0000","last_undegraded":"2023-10-09T05:56:20.336939+0000","last_fullsized":"2023-10-09T05:56:20.336939+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:58:31.918499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.14","version":"226'7413","reported_seq":8020,"reported_epoch":226,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:29.413535+0000","last_change":"2023-10-09T05:56:04.581075+0000","last_active":"2023-10-09T05:56:29.413535+0000","last_peered":"2023-10-09T05:56:29.413535+0000","last_clean":"2023-10-09T05:51:24.557317+0000","last_became_active":"2023-10-09T05:56:04.557931+0000","last_became_peered":"2023-10-09T05:56:04.557931+0000","last_unstale":"2023-10-09T05:56:29.413535+0000","last_undegraded":"2023-10-09T05:56:00.398978+0000","last_fullsized":"2023-10-09T05:56:00.286535+0000","mapping_epoch":219,"log_start":"107'1426","ondisk_log_start":"107'1426","created":188,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5987,"log_dups_size":0,"ondisk_log_size":5987,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125960192,"num_objects":1922,"num_object_clones":0,"num_object_copies":5766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2882,"num_objects_misplaced":960,"num_objects_unfound":0,"num_objects_dirty":1922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4132,"num_write_kb":193728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":126,"num_bytes_recovered":8257536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,1],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"1,3,5","objects":853},{"shards":"1,5","objects":-246},{"shards":"3,5,6","objects":962}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":470,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.222160+0000","last_change":"2023-10-09T05:55:59.344087+0000","last_active":"2023-10-09T05:56:20.222160+0000","last_peered":"2023-10-09T05:56:20.222160+0000","last_clean":"2023-10-09T05:56:20.222160+0000","last_became_active":"2023-10-09T05:55:59.343288+0000","last_became_peered":"2023-10-09T05:55:59.343288+0000","last_unstale":"2023-10-09T05:56:20.222160+0000","last_undegraded":"2023-10-09T05:56:20.222160+0000","last_fullsized":"2023-10-09T05:56:20.222160+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":62,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.220280+0000","last_change":"2023-10-09T05:53:59.206887+0000","last_active":"2023-10-09T05:56:20.220280+0000","last_peered":"2023-10-09T05:56:20.220280+0000","last_clean":"2023-10-09T05:56:20.220280+0000","last_became_active":"2023-10-09T05:53:59.206608+0000","last_became_peered":"2023-10-09T05:53:59.206608+0000","last_unstale":"2023-10-09T05:56:20.220280+0000","last_undegraded":"2023-10-09T05:56:20.220280+0000","last_fullsized":"2023-10-09T05:56:20.220280+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:48.975792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":189,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.221181+0000","last_change":"2023-10-09T05:54:16.225672+0000","last_active":"2023-10-09T05:56:20.221181+0000","last_peered":"2023-10-09T05:56:20.221181+0000","last_clean":"2023-10-09T05:56:20.221181+0000","last_became_active":"2023-10-09T05:54:16.225312+0000","last_became_peered":"2023-10-09T05:54:16.225312+0000","last_unstale":"2023-10-09T05:56:20.221181+0000","last_undegraded":"2023-10-09T05:56:20.221181+0000","last_fullsized":"2023-10-09T05:56:20.221181+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:52.172719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"226'7245","reported_seq":13212,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:56:30.287334+0000","last_change":"2023-10-09T05:55:00.481291+0000","last_active":"2023-10-09T05:56:30.287334+0000","last_peered":"2023-10-09T05:56:30.287334+0000","last_clean":"2023-10-09T05:56:30.287334+0000","last_became_active":"2023-10-09T05:55:00.476488+0000","last_became_peered":"2023-10-09T05:55:00.476488+0000","last_unstale":"2023-10-09T05:56:30.287334+0000","last_undegraded":"2023-10-09T05:56:30.287334+0000","last_fullsized":"2023-10-09T05:56:30.287334+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":188,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7245,"log_dups_size":0,"ondisk_log_size":7245,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120422400,"num_objects":1838,"num_object_clones":0,"num_object_copies":5514,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1838,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4065,"num_write_kb":188864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2244,"num_bytes_recovered":132907008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":495,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.219154+0000","last_change":"2023-10-09T05:54:19.202891+0000","last_active":"2023-10-09T05:56:20.219154+0000","last_peered":"2023-10-09T05:56:20.219154+0000","last_clean":"2023-10-09T05:56:20.219154+0000","last_became_active":"2023-10-09T05:54:19.202303+0000","last_became_peered":"2023-10-09T05:54:19.202303+0000","last_unstale":"2023-10-09T05:56:20.219154+0000","last_undegraded":"2023-10-09T05:56:20.219154+0000","last_fullsized":"2023-10-09T05:56:20.219154+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":211,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.335838+0000","last_change":"2023-10-09T05:51:57.091639+0000","last_active":"2023-10-09T05:56:20.335838+0000","last_peered":"2023-10-09T05:56:20.335838+0000","last_clean":"2023-10-09T05:56:20.335838+0000","last_became_active":"2023-10-09T05:51:57.091284+0000","last_became_peered":"2023-10-09T05:51:57.091284+0000","last_unstale":"2023-10-09T05:56:20.335838+0000","last_undegraded":"2023-10-09T05:56:20.335838+0000","last_fullsized":"2023-10-09T05:56:20.335838+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:10.553453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":173,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.335884+0000","last_change":"2023-10-09T05:55:59.308136+0000","last_active":"2023-10-09T05:56:20.335884+0000","last_peered":"2023-10-09T05:56:20.335884+0000","last_clean":"2023-10-09T05:56:20.335884+0000","last_became_active":"2023-10-09T05:55:59.307803+0000","last_became_peered":"2023-10-09T05:55:59.307803+0000","last_unstale":"2023-10-09T05:56:20.335884+0000","last_undegraded":"2023-10-09T05:56:20.335884+0000","last_fullsized":"2023-10-09T05:56:20.335884+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:42.016632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":559,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.221492+0000","last_change":"2023-10-09T05:53:20.160853+0000","last_active":"2023-10-09T05:56:20.221492+0000","last_peered":"2023-10-09T05:56:20.221492+0000","last_clean":"2023-10-09T05:56:20.221492+0000","last_became_active":"2023-10-09T05:53:20.160520+0000","last_became_peered":"2023-10-09T05:53:20.160520+0000","last_unstale":"2023-10-09T05:56:20.221492+0000","last_undegraded":"2023-10-09T05:56:20.221492+0000","last_fullsized":"2023-10-09T05:56:20.221492+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.a","version":"226'7603","reported_seq":9945,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:56:29.367543+0000","last_change":"2023-10-09T05:55:22.206257+0000","last_active":"2023-10-09T05:56:29.367543+0000","last_peered":"2023-10-09T05:56:29.367543+0000","last_clean":"2023-10-09T05:56:29.367543+0000","last_became_active":"2023-10-09T05:55:22.197933+0000","last_became_peered":"2023-10-09T05:55:22.197933+0000","last_unstale":"2023-10-09T05:56:29.367543+0000","last_undegraded":"2023-10-09T05:56:29.367543+0000","last_fullsized":"2023-10-09T05:56:29.367543+0000","mapping_epoch":161,"log_start":"108'2184","ondisk_log_start":"108'2184","created":102,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5419,"log_dups_size":0,"ondisk_log_size":5419,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120193024,"num_objects":1834,"num_object_clones":0,"num_object_copies":5502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4179,"num_write_kb":192416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":542,"num_bytes_recovered":35553280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":66,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.221178+0000","last_change":"2023-10-09T05:53:59.222136+0000","last_active":"2023-10-09T05:56:20.221178+0000","last_peered":"2023-10-09T05:56:20.221178+0000","last_clean":"2023-10-09T05:56:20.221178+0000","last_became_active":"2023-10-09T05:53:59.201853+0000","last_became_peered":"2023-10-09T05:53:59.201853+0000","last_unstale":"2023-10-09T05:56:20.221178+0000","last_undegraded":"2023-10-09T05:56:20.221178+0000","last_fullsized":"2023-10-09T05:56:20.221178+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:26.662031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":188,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.336120+0000","last_change":"2023-10-09T05:53:59.187102+0000","last_active":"2023-10-09T05:56:20.336120+0000","last_peered":"2023-10-09T05:56:20.336120+0000","last_clean":"2023-10-09T05:56:20.336120+0000","last_became_active":"2023-10-09T05:53:59.186758+0000","last_became_peered":"2023-10-09T05:53:59.186758+0000","last_unstale":"2023-10-09T05:56:20.336120+0000","last_undegraded":"2023-10-09T05:56:20.336120+0000","last_fullsized":"2023-10-09T05:56:20.336120+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:35.806325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":478,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.339398+0000","last_change":"2023-10-09T05:54:47.664930+0000","last_active":"2023-10-09T05:56:20.339398+0000","last_peered":"2023-10-09T05:56:20.339398+0000","last_clean":"2023-10-09T05:56:20.339398+0000","last_became_active":"2023-10-09T05:54:47.664533+0000","last_became_peered":"2023-10-09T05:54:47.664533+0000","last_unstale":"2023-10-09T05:56:20.339398+0000","last_undegraded":"2023-10-09T05:56:20.339398+0000","last_fullsized":"2023-10-09T05:56:20.339398+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":381350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"226'7718","reported_seq":22455,"reported_epoch":226,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:56:29.418020+0000","last_change":"2023-10-09T05:56:00.454478+0000","last_active":"2023-10-09T05:56:29.418020+0000","last_peered":"2023-10-09T05:56:29.418020+0000","last_clean":"2023-10-09T05:54:47.668795+0000","last_became_active":"2023-10-09T05:56:00.438148+0000","last_became_peered":"2023-10-09T05:56:00.438148+0000","last_unstale":"2023-10-09T05:56:29.418020+0000","last_undegraded":"2023-10-09T05:56:00.295210+0000","last_fullsized":"2023-10-09T05:56:29.418020+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7718,"log_dups_size":0,"ondisk_log_size":7718,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122388480,"num_objects":1867,"num_object_clones":0,"num_object_copies":5601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2407,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4235,"num_write_kb":195296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3981,"num_bytes_recovered":246317056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":["3","6","7"],"object_location_counts":[{"shards":"1,3,6","objects":1931},{"shards":"3,4,6","objects":-64}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":108,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.222589+0000","last_change":"2023-10-09T05:55:59.379174+0000","last_active":"2023-10-09T05:56:20.222589+0000","last_peered":"2023-10-09T05:56:20.222589+0000","last_clean":"2023-10-09T05:56:20.222589+0000","last_became_active":"2023-10-09T05:55:59.341932+0000","last_became_peered":"2023-10-09T05:55:59.341932+0000","last_unstale":"2023-10-09T05:56:20.222589+0000","last_undegraded":"2023-10-09T05:56:20.222589+0000","last_fullsized":"2023-10-09T05:56:20.222589+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.636171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":262,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.220877+0000","last_change":"2023-10-09T05:55:59.341253+0000","last_active":"2023-10-09T05:56:20.220877+0000","last_peered":"2023-10-09T05:56:20.220877+0000","last_clean":"2023-10-09T05:56:20.220877+0000","last_became_active":"2023-10-09T05:55:59.340936+0000","last_became_peered":"2023-10-09T05:55:59.340936+0000","last_unstale":"2023-10-09T05:56:20.220877+0000","last_undegraded":"2023-10-09T05:56:20.220877+0000","last_fullsized":"2023-10-09T05:56:20.220877+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:43.120104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":598,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.345784+0000","last_change":"2023-10-09T05:53:59.187768+0000","last_active":"2023-10-09T05:56:20.345784+0000","last_peered":"2023-10-09T05:56:20.345784+0000","last_clean":"2023-10-09T05:56:20.345784+0000","last_became_active":"2023-10-09T05:53:59.187329+0000","last_became_peered":"2023-10-09T05:53:59.187329+0000","last_unstale":"2023-10-09T05:56:20.345784+0000","last_undegraded":"2023-10-09T05:56:20.345784+0000","last_fullsized":"2023-10-09T05:56:20.345784+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.8","version":"226'7389","reported_seq":7544,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:56:30.271870+0000","last_change":"2023-10-09T05:54:45.661206+0000","last_active":"2023-10-09T05:56:30.271870+0000","last_peered":"2023-10-09T05:56:30.271870+0000","last_clean":"2023-10-09T05:56:30.271870+0000","last_became_active":"2023-10-09T05:54:45.655631+0000","last_became_peered":"2023-10-09T05:54:45.655631+0000","last_unstale":"2023-10-09T05:56:30.271870+0000","last_undegraded":"2023-10-09T05:56:30.271870+0000","last_fullsized":"2023-10-09T05:56:30.271870+0000","mapping_epoch":174,"log_start":"106'122","ondisk_log_start":"106'122","created":102,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7267,"log_dups_size":0,"ondisk_log_size":7267,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119537664,"num_objects":1824,"num_object_clones":0,"num_object_copies":5472,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1824,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4142,"num_write_kb":190912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":64,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.221650+0000","last_change":"2023-10-09T05:53:59.212161+0000","last_active":"2023-10-09T05:56:20.221650+0000","last_peered":"2023-10-09T05:56:20.221650+0000","last_clean":"2023-10-09T05:56:20.221650+0000","last_became_active":"2023-10-09T05:53:59.193814+0000","last_became_peered":"2023-10-09T05:53:59.193814+0000","last_unstale":"2023-10-09T05:56:20.221650+0000","last_undegraded":"2023-10-09T05:56:20.221650+0000","last_fullsized":"2023-10-09T05:56:20.221650+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:10.502408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":265,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.346232+0000","last_change":"2023-10-09T05:55:59.303352+0000","last_active":"2023-10-09T05:56:20.346232+0000","last_peered":"2023-10-09T05:56:20.346232+0000","last_clean":"2023-10-09T05:56:20.346232+0000","last_became_active":"2023-10-09T05:55:59.302923+0000","last_became_peered":"2023-10-09T05:55:59.302923+0000","last_unstale":"2023-10-09T05:56:20.346232+0000","last_undegraded":"2023-10-09T05:56:20.346232+0000","last_fullsized":"2023-10-09T05:56:20.346232+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:53.013376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":585,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.346003+0000","last_change":"2023-10-09T05:53:20.167463+0000","last_active":"2023-10-09T05:56:20.346003+0000","last_peered":"2023-10-09T05:56:20.346003+0000","last_clean":"2023-10-09T05:56:20.346003+0000","last_became_active":"2023-10-09T05:53:20.167014+0000","last_became_peered":"2023-10-09T05:53:20.167014+0000","last_unstale":"2023-10-09T05:56:20.346003+0000","last_undegraded":"2023-10-09T05:56:20.346003+0000","last_fullsized":"2023-10-09T05:56:20.346003+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"226'7417","reported_seq":11178,"reported_epoch":226,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:29.475039+0000","last_change":"2023-10-09T05:55:59.403890+0000","last_active":"2023-10-09T05:56:29.475039+0000","last_peered":"2023-10-09T05:56:29.475039+0000","last_clean":"2023-10-09T05:51:33.020184+0000","last_became_active":"2023-10-09T05:54:53.759742+0000","last_became_peered":"2023-10-09T05:54:53.759742+0000","last_unstale":"2023-10-09T05:56:29.475039+0000","last_undegraded":"2023-10-09T05:54:49.556279+0000","last_fullsized":"2023-10-09T05:54:46.645749+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7417,"log_dups_size":0,"ondisk_log_size":7417,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125796352,"num_objects":1919,"num_object_clones":0,"num_object_copies":5757,"num_objects_missing_on_primary":6,"num_objects_missing":6,"num_objects_degraded":1565,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1919,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4152,"num_write_kb":194304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1492,"num_bytes_recovered":97779712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,3],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,7","objects":1331},{"shards":"0,3","objects":582},{"shards":"2,7","objects":6}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":226,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.339725+0000","last_change":"2023-10-09T05:55:59.356434+0000","last_active":"2023-10-09T05:56:20.339725+0000","last_peered":"2023-10-09T05:56:20.339725+0000","last_clean":"2023-10-09T05:56:20.339725+0000","last_became_active":"2023-10-09T05:55:59.355451+0000","last_became_peered":"2023-10-09T05:55:59.355451+0000","last_unstale":"2023-10-09T05:56:20.339725+0000","last_undegraded":"2023-10-09T05:56:20.339725+0000","last_fullsized":"2023-10-09T05:56:20.339725+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:46.692106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":227,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:56:25.274642+0000","last_change":"2023-10-09T05:55:59.341649+0000","last_active":"2023-10-09T05:56:25.274642+0000","last_peered":"2023-10-09T05:56:25.274642+0000","last_clean":"2023-10-09T05:56:25.274642+0000","last_became_active":"2023-10-09T05:55:59.341325+0000","last_became_peered":"2023-10-09T05:55:59.341325+0000","last_unstale":"2023-10-09T05:56:25.274642+0000","last_undegraded":"2023-10-09T05:56:25.274642+0000","last_fullsized":"2023-10-09T05:56:25.274642+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:38.135295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":619,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.221383+0000","last_change":"2023-10-09T05:54:19.204406+0000","last_active":"2023-10-09T05:56:20.221383+0000","last_peered":"2023-10-09T05:56:20.221383+0000","last_clean":"2023-10-09T05:56:20.221383+0000","last_became_active":"2023-10-09T05:54:19.204082+0000","last_became_peered":"2023-10-09T05:54:19.204082+0000","last_unstale":"2023-10-09T05:56:20.221383+0000","last_undegraded":"2023-10-09T05:56:20.221383+0000","last_fullsized":"2023-10-09T05:56:20.221383+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.e","version":"226'7574","reported_seq":20104,"reported_epoch":226,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:56:29.384083+0000","last_change":"2023-10-09T05:56:00.422102+0000","last_active":"2023-10-09T05:56:29.384083+0000","last_peered":"2023-10-09T05:56:29.384083+0000","last_clean":"2023-10-09T05:55:58.237229+0000","last_became_active":"2023-10-09T05:56:00.411796+0000","last_became_peered":"2023-10-09T05:56:00.411796+0000","last_unstale":"2023-10-09T05:56:29.384083+0000","last_undegraded":"2023-10-09T05:56:29.384083+0000","last_fullsized":"2023-10-09T05:56:29.384083+0000","mapping_epoch":219,"log_start":"107'1851","ondisk_log_start":"107'1851","created":102,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5723,"log_dups_size":0,"ondisk_log_size":5723,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119767040,"num_objects":1827,"num_object_clones":0,"num_object_copies":5481,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1827,"num_objects_unfound":0,"num_objects_dirty":1827,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4162,"num_write_kb":191680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3073,"num_bytes_recovered":196280320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,1],"avail_no_missing":["4","1","2","5"],"object_location_counts":[{"shards":"1,4,5","objects":1827}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":121,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:56:25.274737+0000","last_change":"2023-10-09T05:55:59.388969+0000","last_active":"2023-10-09T05:56:25.274737+0000","last_peered":"2023-10-09T05:56:25.274737+0000","last_clean":"2023-10-09T05:56:25.274737+0000","last_became_active":"2023-10-09T05:55:59.358004+0000","last_became_peered":"2023-10-09T05:55:59.358004+0000","last_unstale":"2023-10-09T05:56:25.274737+0000","last_undegraded":"2023-10-09T05:56:25.274737+0000","last_fullsized":"2023-10-09T05:56:25.274737+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:39.051275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":185,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.221253+0000","last_change":"2023-10-09T05:49:21.430742+0000","last_active":"2023-10-09T05:56:20.221253+0000","last_peered":"2023-10-09T05:56:20.221253+0000","last_clean":"2023-10-09T05:56:20.221253+0000","last_became_active":"2023-10-09T05:49:21.430233+0000","last_became_peered":"2023-10-09T05:49:21.430233+0000","last_unstale":"2023-10-09T05:56:20.221253+0000","last_undegraded":"2023-10-09T05:56:20.221253+0000","last_fullsized":"2023-10-09T05:56:20.221253+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:10.541302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":554,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.336217+0000","last_change":"2023-10-09T05:54:54.779276+0000","last_active":"2023-10-09T05:56:20.336217+0000","last_peered":"2023-10-09T05:56:20.336217+0000","last_clean":"2023-10-09T05:56:20.336217+0000","last_became_active":"2023-10-09T05:54:54.778929+0000","last_became_peered":"2023-10-09T05:54:54.778929+0000","last_unstale":"2023-10-09T05:56:20.336217+0000","last_undegraded":"2023-10-09T05:56:20.336217+0000","last_fullsized":"2023-10-09T05:56:20.336217+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":686385,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.f","version":"226'7712","reported_seq":8630,"reported_epoch":226,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T05:56:29.471031+0000","last_change":"2023-10-09T05:55:59.491890+0000","last_active":"2023-10-09T05:56:29.471031+0000","last_peered":"2023-10-09T05:56:29.471031+0000","last_clean":"2023-10-09T05:52:22.632757+0000","last_became_active":"2023-10-09T05:55:59.467190+0000","last_became_peered":"2023-10-09T05:55:59.467190+0000","last_unstale":"2023-10-09T05:56:29.471031+0000","last_undegraded":"2023-10-09T05:55:59.403431+0000","last_fullsized":"2023-10-09T05:56:29.471031+0000","mapping_epoch":218,"log_start":"107'1991","ondisk_log_start":"107'1991","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5721,"log_dups_size":0,"ondisk_log_size":5721,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124289024,"num_objects":1896,"num_object_clones":0,"num_object_copies":5688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1896,"num_objects_misplaced":2082,"num_objects_unfound":0,"num_objects_dirty":1896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4244,"num_write_kb":196512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":270,"num_bytes_recovered":17694720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,3],"avail_no_missing":["4","1","2","3"],"object_location_counts":[{"shards":"2,3,4","objects":1852},{"shards":"3,4,6","objects":44}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":236,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.346150+0000","last_change":"2023-10-09T05:55:59.320669+0000","last_active":"2023-10-09T05:56:20.346150+0000","last_peered":"2023-10-09T05:56:20.346150+0000","last_clean":"2023-10-09T05:56:20.346150+0000","last_became_active":"2023-10-09T05:55:59.320352+0000","last_became_peered":"2023-10-09T05:55:59.320352+0000","last_unstale":"2023-10-09T05:56:20.346150+0000","last_undegraded":"2023-10-09T05:56:20.346150+0000","last_fullsized":"2023-10-09T05:56:20.346150+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:09:14.227028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":262,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.221671+0000","last_change":"2023-10-09T05:55:59.346466+0000","last_active":"2023-10-09T05:56:20.221671+0000","last_peered":"2023-10-09T05:56:20.221671+0000","last_clean":"2023-10-09T05:56:20.221671+0000","last_became_active":"2023-10-09T05:55:59.345389+0000","last_became_peered":"2023-10-09T05:55:59.345389+0000","last_unstale":"2023-10-09T05:56:20.221671+0000","last_undegraded":"2023-10-09T05:56:20.221671+0000","last_fullsized":"2023-10-09T05:56:20.221671+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:17.657937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":470,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.222237+0000","last_change":"2023-10-09T05:55:59.344203+0000","last_active":"2023-10-09T05:56:20.222237+0000","last_peered":"2023-10-09T05:56:20.222237+0000","last_clean":"2023-10-09T05:56:20.222237+0000","last_became_active":"2023-10-09T05:55:59.343631+0000","last_became_peered":"2023-10-09T05:55:59.343631+0000","last_unstale":"2023-10-09T05:56:20.222237+0000","last_undegraded":"2023-10-09T05:56:20.222237+0000","last_fullsized":"2023-10-09T05:56:20.222237+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"226'7590","reported_seq":19841,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:56:26.829542+0000","last_change":"2023-10-09T05:55:22.215428+0000","last_active":"2023-10-09T05:56:26.829542+0000","last_peered":"2023-10-09T05:56:26.829542+0000","last_clean":"2023-10-09T05:56:26.829542+0000","last_became_active":"2023-10-09T05:55:22.207092+0000","last_became_peered":"2023-10-09T05:55:22.207092+0000","last_unstale":"2023-10-09T05:56:26.829542+0000","last_undegraded":"2023-10-09T05:56:26.829542+0000","last_fullsized":"2023-10-09T05:56:26.829542+0000","mapping_epoch":173,"log_start":"108'2503","ondisk_log_start":"108'2503","created":102,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5087,"log_dups_size":0,"ondisk_log_size":5087,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123076608,"num_objects":1878,"num_object_clones":0,"num_object_copies":5634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4155,"num_write_kb":193056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3016,"num_bytes_recovered":192643072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":206,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.220664+0000","last_change":"2023-10-09T05:54:00.619165+0000","last_active":"2023-10-09T05:56:20.220664+0000","last_peered":"2023-10-09T05:56:20.220664+0000","last_clean":"2023-10-09T05:56:20.220664+0000","last_became_active":"2023-10-09T05:54:00.618888+0000","last_became_peered":"2023-10-09T05:54:00.618888+0000","last_unstale":"2023-10-09T05:56:20.220664+0000","last_undegraded":"2023-10-09T05:56:20.220664+0000","last_fullsized":"2023-10-09T05:56:20.220664+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:07.175996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":235,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.222281+0000","last_change":"2023-10-09T05:55:59.336539+0000","last_active":"2023-10-09T05:56:20.222281+0000","last_peered":"2023-10-09T05:56:20.222281+0000","last_clean":"2023-10-09T05:56:20.222281+0000","last_became_active":"2023-10-09T05:55:59.334624+0000","last_became_peered":"2023-10-09T05:55:59.334624+0000","last_unstale":"2023-10-09T05:56:20.222281+0000","last_undegraded":"2023-10-09T05:56:20.222281+0000","last_fullsized":"2023-10-09T05:56:20.222281+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:33.120802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":533,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.339891+0000","last_change":"2023-10-09T05:55:58.750810+0000","last_active":"2023-10-09T05:56:20.339891+0000","last_peered":"2023-10-09T05:56:20.339891+0000","last_clean":"2023-10-09T05:56:20.339891+0000","last_became_active":"2023-10-09T05:51:19.195632+0000","last_became_peered":"2023-10-09T05:51:19.195632+0000","last_unstale":"2023-10-09T05:56:20.339891+0000","last_undegraded":"2023-10-09T05:56:20.339891+0000","last_fullsized":"2023-10-09T05:56:20.339891+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.d","version":"226'7597","reported_seq":13271,"reported_epoch":226,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:56:29.316012+0000","last_change":"2023-10-09T05:56:04.582001+0000","last_active":"2023-10-09T05:56:29.316012+0000","last_peered":"2023-10-09T05:56:29.316012+0000","last_clean":"2023-10-09T05:55:58.102294+0000","last_became_active":"2023-10-09T05:56:00.395752+0000","last_became_peered":"2023-10-09T05:56:00.395752+0000","last_unstale":"2023-10-09T05:56:29.316012+0000","last_undegraded":"2023-10-09T05:56:29.316012+0000","last_fullsized":"2023-10-09T05:56:29.316012+0000","mapping_epoch":219,"log_start":"108'2175","ondisk_log_start":"108'2175","created":102,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5422,"log_dups_size":0,"ondisk_log_size":5422,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120193036,"num_objects":1834,"num_object_clones":0,"num_object_copies":5502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1729,"num_objects_unfound":0,"num_objects_dirty":1834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4167,"num_write_kb":192032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1422,"num_bytes_recovered":93192192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,1],"avail_no_missing":["5","0","1"],"object_location_counts":[{"shards":"0,1,5","objects":1834}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":208,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.221612+0000","last_change":"2023-10-09T05:55:59.320634+0000","last_active":"2023-10-09T05:56:20.221612+0000","last_peered":"2023-10-09T05:56:20.221612+0000","last_clean":"2023-10-09T05:56:20.221612+0000","last_became_active":"2023-10-09T05:55:59.320341+0000","last_became_peered":"2023-10-09T05:55:59.320341+0000","last_unstale":"2023-10-09T05:56:20.221612+0000","last_undegraded":"2023-10-09T05:56:20.221612+0000","last_fullsized":"2023-10-09T05:56:20.221612+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:15:13.249107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":104,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.331076+0000","last_change":"2023-10-09T05:55:59.409563+0000","last_active":"2023-10-09T05:56:20.331076+0000","last_peered":"2023-10-09T05:56:20.331076+0000","last_clean":"2023-10-09T05:56:20.331076+0000","last_became_active":"2023-10-09T05:55:59.409184+0000","last_became_peered":"2023-10-09T05:55:59.409184+0000","last_unstale":"2023-10-09T05:56:20.331076+0000","last_undegraded":"2023-10-09T05:56:20.331076+0000","last_fullsized":"2023-10-09T05:56:20.331076+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:39.513591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":599,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.345361+0000","last_change":"2023-10-09T05:53:59.202618+0000","last_active":"2023-10-09T05:56:20.345361+0000","last_peered":"2023-10-09T05:56:20.345361+0000","last_clean":"2023-10-09T05:56:20.345361+0000","last_became_active":"2023-10-09T05:53:59.202265+0000","last_became_peered":"2023-10-09T05:53:59.202265+0000","last_unstale":"2023-10-09T05:56:20.345361+0000","last_undegraded":"2023-10-09T05:56:20.345361+0000","last_fullsized":"2023-10-09T05:56:20.345361+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":119,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.220413+0000","last_change":"2023-10-09T05:55:59.298953+0000","last_active":"2023-10-09T05:56:20.220413+0000","last_peered":"2023-10-09T05:56:20.220413+0000","last_clean":"2023-10-09T05:56:20.220413+0000","last_became_active":"2023-10-09T05:55:59.298624+0000","last_became_peered":"2023-10-09T05:55:59.298624+0000","last_unstale":"2023-10-09T05:56:20.220413+0000","last_undegraded":"2023-10-09T05:56:20.220413+0000","last_fullsized":"2023-10-09T05:56:20.220413+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:00.513870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":213,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.336855+0000","last_change":"2023-10-09T05:52:29.735372+0000","last_active":"2023-10-09T05:56:20.336855+0000","last_peered":"2023-10-09T05:56:20.336855+0000","last_clean":"2023-10-09T05:56:20.336855+0000","last_became_active":"2023-10-09T05:52:29.735026+0000","last_became_peered":"2023-10-09T05:52:29.735026+0000","last_unstale":"2023-10-09T05:56:20.336855+0000","last_undegraded":"2023-10-09T05:56:20.336855+0000","last_fullsized":"2023-10-09T05:56:20.336855+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:53.849406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":100,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.339579+0000","last_change":"2023-10-09T05:55:59.307598+0000","last_active":"2023-10-09T05:56:20.339579+0000","last_peered":"2023-10-09T05:56:20.339579+0000","last_clean":"2023-10-09T05:56:20.339579+0000","last_became_active":"2023-10-09T05:55:59.306968+0000","last_became_peered":"2023-10-09T05:55:59.306968+0000","last_unstale":"2023-10-09T05:56:20.339579+0000","last_undegraded":"2023-10-09T05:56:20.339579+0000","last_fullsized":"2023-10-09T05:56:20.339579+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:43.253091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"226'7110","reported_seq":10972,"reported_epoch":226,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:56:29.428218+0000","last_change":"2023-10-09T05:56:00.610946+0000","last_active":"2023-10-09T05:56:29.428218+0000","last_peered":"2023-10-09T05:56:29.428218+0000","last_clean":"2023-10-09T05:54:10.146196+0000","last_became_active":"2023-10-09T05:56:00.532512+0000","last_became_peered":"2023-10-09T05:56:00.532512+0000","last_unstale":"2023-10-09T05:56:29.428218+0000","last_undegraded":"2023-10-09T05:56:00.361695+0000","last_fullsized":"2023-10-09T05:56:29.428218+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7110,"log_dups_size":0,"ondisk_log_size":7110,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60637184,"num_objects":925,"num_object_clones":0,"num_object_copies":2775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2150,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2112,"num_write_kb":97216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":771,"num_bytes_recovered":50577408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":939},{"shards":"0,4,5","objects":-14}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":557,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.336232+0000","last_change":"2023-10-09T05:53:20.234074+0000","last_active":"2023-10-09T05:56:20.336232+0000","last_peered":"2023-10-09T05:56:20.336232+0000","last_clean":"2023-10-09T05:56:20.336232+0000","last_became_active":"2023-10-09T05:53:20.233803+0000","last_became_peered":"2023-10-09T05:53:20.233803+0000","last_unstale":"2023-10-09T05:56:20.336232+0000","last_undegraded":"2023-10-09T05:56:20.336232+0000","last_fullsized":"2023-10-09T05:56:20.336232+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.6","version":"226'7471","reported_seq":9610,"reported_epoch":226,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T05:56:29.472720+0000","last_change":"2023-10-09T05:55:01.518574+0000","last_active":"2023-10-09T05:56:29.472720+0000","last_peered":"2023-10-09T05:56:29.472720+0000","last_clean":"2023-10-09T05:52:24.946135+0000","last_became_active":"2023-10-09T05:54:45.741289+0000","last_became_peered":"2023-10-09T05:54:45.741289+0000","last_unstale":"2023-10-09T05:56:29.472720+0000","last_undegraded":"2023-10-09T05:55:01.516066+0000","last_fullsized":"2023-10-09T05:56:29.472720+0000","mapping_epoch":181,"log_start":"106'613","ondisk_log_start":"106'613","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6858,"log_dups_size":0,"ondisk_log_size":6858,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118980608,"num_objects":1815,"num_object_clones":0,"num_object_copies":5445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1815,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4206,"num_write_kb":192704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":959,"num_bytes_recovered":62849024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,6],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":1797},{"shards":"4,6,7","objects":18}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":257,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.339475+0000","last_change":"2023-10-09T05:55:59.307771+0000","last_active":"2023-10-09T05:56:20.339475+0000","last_peered":"2023-10-09T05:56:20.339475+0000","last_clean":"2023-10-09T05:56:20.339475+0000","last_became_active":"2023-10-09T05:55:59.307299+0000","last_became_peered":"2023-10-09T05:55:59.307299+0000","last_unstale":"2023-10-09T05:56:20.339475+0000","last_undegraded":"2023-10-09T05:56:20.339475+0000","last_fullsized":"2023-10-09T05:56:20.339475+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:53.874666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":203,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.222562+0000","last_change":"2023-10-09T05:51:57.203142+0000","last_active":"2023-10-09T05:56:20.222562+0000","last_peered":"2023-10-09T05:56:20.222562+0000","last_clean":"2023-10-09T05:56:20.222562+0000","last_became_active":"2023-10-09T05:51:57.202850+0000","last_became_peered":"2023-10-09T05:51:57.202850+0000","last_unstale":"2023-10-09T05:56:20.222562+0000","last_undegraded":"2023-10-09T05:56:20.222562+0000","last_fullsized":"2023-10-09T05:56:20.222562+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:51.090737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":130,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.335178+0000","last_change":"2023-10-09T05:55:59.298261+0000","last_active":"2023-10-09T05:56:20.335178+0000","last_peered":"2023-10-09T05:56:20.335178+0000","last_clean":"2023-10-09T05:56:20.335178+0000","last_became_active":"2023-10-09T05:55:59.297549+0000","last_became_peered":"2023-10-09T05:55:59.297549+0000","last_unstale":"2023-10-09T05:56:20.335178+0000","last_undegraded":"2023-10-09T05:56:20.335178+0000","last_fullsized":"2023-10-09T05:56:20.335178+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:05:53.922360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":495,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.219400+0000","last_change":"2023-10-09T05:54:19.203419+0000","last_active":"2023-10-09T05:56:20.219400+0000","last_peered":"2023-10-09T05:56:20.219400+0000","last_clean":"2023-10-09T05:56:20.219400+0000","last_became_active":"2023-10-09T05:54:19.203105+0000","last_became_peered":"2023-10-09T05:54:19.203105+0000","last_unstale":"2023-10-09T05:56:20.219400+0000","last_undegraded":"2023-10-09T05:56:20.219400+0000","last_fullsized":"2023-10-09T05:56:20.219400+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"226'7286","reported_seq":13332,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:56:30.363348+0000","last_change":"2023-10-09T05:54:54.773920+0000","last_active":"2023-10-09T05:56:30.363348+0000","last_peered":"2023-10-09T05:56:30.363348+0000","last_clean":"2023-10-09T05:56:30.363348+0000","last_became_active":"2023-10-09T05:54:54.770744+0000","last_became_peered":"2023-10-09T05:54:54.770744+0000","last_unstale":"2023-10-09T05:56:30.363348+0000","last_undegraded":"2023-10-09T05:56:30.363348+0000","last_fullsized":"2023-10-09T05:56:30.363348+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7286,"log_dups_size":0,"ondisk_log_size":7286,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117735424,"num_objects":1796,"num_object_clones":0,"num_object_copies":5388,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1796,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4105,"num_write_kb":188864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2284,"num_bytes_recovered":135462912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":214,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.227427+0000","last_change":"2023-10-09T05:55:59.314996+0000","last_active":"2023-10-09T05:56:20.227427+0000","last_peered":"2023-10-09T05:56:20.227427+0000","last_clean":"2023-10-09T05:56:20.227427+0000","last_became_active":"2023-10-09T05:55:59.314633+0000","last_became_peered":"2023-10-09T05:55:59.314633+0000","last_unstale":"2023-10-09T05:56:20.227427+0000","last_undegraded":"2023-10-09T05:56:20.227427+0000","last_fullsized":"2023-10-09T05:56:20.227427+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:08.653191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":114,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.334227+0000","last_change":"2023-10-09T05:55:59.308743+0000","last_active":"2023-10-09T05:56:20.334227+0000","last_peered":"2023-10-09T05:56:20.334227+0000","last_clean":"2023-10-09T05:56:20.334227+0000","last_became_active":"2023-10-09T05:55:59.308220+0000","last_became_peered":"2023-10-09T05:55:59.308220+0000","last_unstale":"2023-10-09T05:56:20.334227+0000","last_undegraded":"2023-10-09T05:56:20.334227+0000","last_fullsized":"2023-10-09T05:56:20.334227+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:44.246081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":132,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.337884+0000","last_change":"2023-10-09T05:55:59.305883+0000","last_active":"2023-10-09T05:56:20.337884+0000","last_peered":"2023-10-09T05:56:20.337884+0000","last_clean":"2023-10-09T05:56:20.337884+0000","last_became_active":"2023-10-09T05:55:59.305588+0000","last_became_peered":"2023-10-09T05:55:59.305588+0000","last_unstale":"2023-10-09T05:56:20.337884+0000","last_undegraded":"2023-10-09T05:56:20.337884+0000","last_fullsized":"2023-10-09T05:56:20.337884+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:06:21.115125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.7","version":"226'7338","reported_seq":7481,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:56:29.372825+0000","last_change":"2023-10-09T05:54:45.655400+0000","last_active":"2023-10-09T05:56:29.372825+0000","last_peered":"2023-10-09T05:56:29.372825+0000","last_clean":"2023-10-09T05:56:29.372825+0000","last_became_active":"2023-10-09T05:54:45.647040+0000","last_became_peered":"2023-10-09T05:54:45.647040+0000","last_unstale":"2023-10-09T05:56:29.372825+0000","last_undegraded":"2023-10-09T05:56:29.372825+0000","last_fullsized":"2023-10-09T05:56:29.372825+0000","mapping_epoch":102,"log_start":"106'370","ondisk_log_start":"106'370","created":102,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6968,"log_dups_size":0,"ondisk_log_size":6968,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119832576,"num_objects":1828,"num_object_clones":0,"num_object_copies":5484,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1828,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4131,"num_write_kb":190720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":210,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.336814+0000","last_change":"2023-10-09T05:53:59.182813+0000","last_active":"2023-10-09T05:56:20.336814+0000","last_peered":"2023-10-09T05:56:20.336814+0000","last_clean":"2023-10-09T05:56:20.336814+0000","last_became_active":"2023-10-09T05:53:59.182441+0000","last_became_peered":"2023-10-09T05:53:59.182441+0000","last_unstale":"2023-10-09T05:56:20.336814+0000","last_undegraded":"2023-10-09T05:56:20.336814+0000","last_fullsized":"2023-10-09T05:56:20.336814+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:59.004246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":617,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.220430+0000","last_change":"2023-10-09T05:53:59.202723+0000","last_active":"2023-10-09T05:56:20.220430+0000","last_peered":"2023-10-09T05:56:20.220430+0000","last_clean":"2023-10-09T05:56:20.220430+0000","last_became_active":"2023-10-09T05:53:59.202004+0000","last_became_peered":"2023-10-09T05:53:59.202004+0000","last_unstale":"2023-10-09T05:56:20.220430+0000","last_undegraded":"2023-10-09T05:56:20.220430+0000","last_fullsized":"2023-10-09T05:56:20.220430+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:51:12.346243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:51:12.346243+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":113,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.339411+0000","last_change":"2023-10-09T05:52:23.648626+0000","last_active":"2023-10-09T05:56:20.339411+0000","last_peered":"2023-10-09T05:56:20.339411+0000","last_clean":"2023-10-09T05:56:20.339411+0000","last_became_active":"2023-10-09T05:52:23.646143+0000","last_became_peered":"2023-10-09T05:52:23.646143+0000","last_unstale":"2023-10-09T05:56:20.339411+0000","last_undegraded":"2023-10-09T05:56:20.339411+0000","last_fullsized":"2023-10-09T05:56:20.339411+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:51:04.733282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":188,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.336631+0000","last_change":"2023-10-09T05:53:59.195325+0000","last_active":"2023-10-09T05:56:20.336631+0000","last_peered":"2023-10-09T05:56:20.336631+0000","last_clean":"2023-10-09T05:56:20.336631+0000","last_became_active":"2023-10-09T05:53:59.194954+0000","last_became_peered":"2023-10-09T05:53:59.194954+0000","last_unstale":"2023-10-09T05:56:20.336631+0000","last_undegraded":"2023-10-09T05:56:20.336631+0000","last_fullsized":"2023-10-09T05:56:20.336631+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:23.228416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":470,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.222316+0000","last_change":"2023-10-09T05:55:59.344330+0000","last_active":"2023-10-09T05:56:20.222316+0000","last_peered":"2023-10-09T05:56:20.222316+0000","last_clean":"2023-10-09T05:56:20.222316+0000","last_became_active":"2023-10-09T05:55:59.343887+0000","last_became_peered":"2023-10-09T05:55:59.343887+0000","last_unstale":"2023-10-09T05:56:20.222316+0000","last_undegraded":"2023-10-09T05:56:20.222316+0000","last_fullsized":"2023-10-09T05:56:20.222316+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":92,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.220253+0000","last_change":"2023-10-09T05:55:59.300015+0000","last_active":"2023-10-09T05:56:20.220253+0000","last_peered":"2023-10-09T05:56:20.220253+0000","last_clean":"2023-10-09T05:56:20.220253+0000","last_became_active":"2023-10-09T05:55:59.299281+0000","last_became_peered":"2023-10-09T05:55:59.299281+0000","last_unstale":"2023-10-09T05:56:20.220253+0000","last_undegraded":"2023-10-09T05:56:20.220253+0000","last_fullsized":"2023-10-09T05:56:20.220253+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:50:32.465904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":153,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.227257+0000","last_change":"2023-10-09T05:52:23.643245+0000","last_active":"2023-10-09T05:56:20.227257+0000","last_peered":"2023-10-09T05:56:20.227257+0000","last_clean":"2023-10-09T05:56:20.227257+0000","last_became_active":"2023-10-09T05:52:23.642947+0000","last_became_peered":"2023-10-09T05:52:23.642947+0000","last_unstale":"2023-10-09T05:56:20.227257+0000","last_undegraded":"2023-10-09T05:56:20.227257+0000","last_fullsized":"2023-10-09T05:56:20.227257+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:52.260774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":270,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:24.478974+0000","last_change":"2023-10-09T05:55:59.301184+0000","last_active":"2023-10-09T05:56:24.478974+0000","last_peered":"2023-10-09T05:56:24.478974+0000","last_clean":"2023-10-09T05:56:24.478974+0000","last_became_active":"2023-10-09T05:55:59.299733+0000","last_became_peered":"2023-10-09T05:55:59.299733+0000","last_unstale":"2023-10-09T05:56:24.478974+0000","last_undegraded":"2023-10-09T05:56:24.478974+0000","last_fullsized":"2023-10-09T05:56:24.478974+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:36.929150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"226'7470","reported_seq":8069,"reported_epoch":226,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:29.339251+0000","last_change":"2023-10-09T05:56:04.839436+0000","last_active":"2023-10-09T05:56:29.339251+0000","last_peered":"2023-10-09T05:56:29.339251+0000","last_clean":"2023-10-09T05:51:24.557317+0000","last_became_active":"2023-10-09T05:56:04.570363+0000","last_became_peered":"2023-10-09T05:56:04.570363+0000","last_unstale":"2023-10-09T05:56:29.339251+0000","last_undegraded":"2023-10-09T05:56:03.552905+0000","last_fullsized":"2023-10-09T05:56:00.280042+0000","mapping_epoch":219,"log_start":"107'1426","ondisk_log_start":"107'1426","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6044,"log_dups_size":0,"ondisk_log_size":6044,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122224640,"num_objects":1865,"num_object_clones":0,"num_object_copies":5595,"num_objects_missing_on_primary":962,"num_objects_missing":962,"num_objects_degraded":2827,"num_objects_misplaced":1000,"num_objects_unfound":0,"num_objects_dirty":1865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4189,"num_write_kb":193728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8060928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[1,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"1,3,5","objects":825},{"shards":"1,5","objects":-312},{"shards":"3,5,6","objects":962}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":589,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.346198+0000","last_change":"2023-10-09T05:55:59.752131+0000","last_active":"2023-10-09T05:56:20.346198+0000","last_peered":"2023-10-09T05:56:20.346198+0000","last_clean":"2023-10-09T05:56:20.346198+0000","last_became_active":"2023-10-09T05:53:20.157392+0000","last_became_peered":"2023-10-09T05:53:20.157392+0000","last_unstale":"2023-10-09T05:56:20.346198+0000","last_undegraded":"2023-10-09T05:56:20.346198+0000","last_fullsized":"2023-10-09T05:56:20.346198+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":60,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.220507+0000","last_change":"2023-10-09T05:53:59.201576+0000","last_active":"2023-10-09T05:56:20.220507+0000","last_peered":"2023-10-09T05:56:20.220507+0000","last_clean":"2023-10-09T05:56:20.220507+0000","last_became_active":"2023-10-09T05:53:59.201309+0000","last_became_peered":"2023-10-09T05:53:59.201309+0000","last_unstale":"2023-10-09T05:56:20.220507+0000","last_undegraded":"2023-10-09T05:56:20.220507+0000","last_fullsized":"2023-10-09T05:56:20.220507+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:54.956371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":107,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.339854+0000","last_change":"2023-10-09T05:55:59.356259+0000","last_active":"2023-10-09T05:56:20.339854+0000","last_peered":"2023-10-09T05:56:20.339854+0000","last_clean":"2023-10-09T05:56:20.339854+0000","last_became_active":"2023-10-09T05:55:59.354136+0000","last_became_peered":"2023-10-09T05:55:59.354136+0000","last_unstale":"2023-10-09T05:56:20.339854+0000","last_undegraded":"2023-10-09T05:56:20.339854+0000","last_fullsized":"2023-10-09T05:56:20.339854+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:44:11.833451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":151,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:24.478941+0000","last_change":"2023-10-09T05:53:59.188462+0000","last_active":"2023-10-09T05:56:24.478941+0000","last_peered":"2023-10-09T05:56:24.478941+0000","last_clean":"2023-10-09T05:56:24.478941+0000","last_became_active":"2023-10-09T05:53:59.188061+0000","last_became_peered":"2023-10-09T05:53:59.188061+0000","last_unstale":"2023-10-09T05:56:24.478941+0000","last_undegraded":"2023-10-09T05:56:24.478941+0000","last_fullsized":"2023-10-09T05:56:24.478941+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:19.167114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"226'7113","reported_seq":7638,"reported_epoch":226,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:26.835127+0000","last_change":"2023-10-09T05:56:00.526011+0000","last_active":"2023-10-09T05:56:26.835127+0000","last_peered":"2023-10-09T05:56:26.835127+0000","last_clean":"2023-10-09T05:51:24.563560+0000","last_became_active":"2023-10-09T05:56:00.507313+0000","last_became_peered":"2023-10-09T05:56:00.507313+0000","last_unstale":"2023-10-09T05:56:26.835127+0000","last_undegraded":"2023-10-09T05:56:00.383644+0000","last_fullsized":"2023-10-09T05:56:00.340221+0000","mapping_epoch":219,"log_start":"107'871","ondisk_log_start":"107'871","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6242,"log_dups_size":0,"ondisk_log_size":6242,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63209472,"num_objects":964,"num_object_clones":0,"num_object_copies":2892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1329,"num_objects_misplaced":488,"num_objects_unfound":0,"num_objects_dirty":964,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2088,"num_write_kb":97696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":169,"num_bytes_recovered":11075584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,1],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"1,2,6","objects":458},{"shards":"1,6","objects":-148},{"shards":"2,6,7","objects":476}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":560,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.221563+0000","last_change":"2023-10-09T05:53:20.152495+0000","last_active":"2023-10-09T05:56:20.221563+0000","last_peered":"2023-10-09T05:56:20.221563+0000","last_clean":"2023-10-09T05:56:20.221563+0000","last_became_active":"2023-10-09T05:53:20.152109+0000","last_became_peered":"2023-10-09T05:53:20.152109+0000","last_unstale":"2023-10-09T05:56:20.221563+0000","last_undegraded":"2023-10-09T05:56:20.221563+0000","last_fullsized":"2023-10-09T05:56:20.221563+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":231,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.331163+0000","last_change":"2023-10-09T05:53:59.228948+0000","last_active":"2023-10-09T05:56:20.331163+0000","last_peered":"2023-10-09T05:56:20.331163+0000","last_clean":"2023-10-09T05:56:20.331163+0000","last_became_active":"2023-10-09T05:53:59.186359+0000","last_became_peered":"2023-10-09T05:53:59.186359+0000","last_unstale":"2023-10-09T05:56:20.331163+0000","last_undegraded":"2023-10-09T05:56:20.331163+0000","last_fullsized":"2023-10-09T05:56:20.331163+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:39.990576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":83,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.346098+0000","last_change":"2023-10-09T05:55:59.339957+0000","last_active":"2023-10-09T05:56:20.346098+0000","last_peered":"2023-10-09T05:56:20.346098+0000","last_clean":"2023-10-09T05:56:20.346098+0000","last_became_active":"2023-10-09T05:55:59.339614+0000","last_became_peered":"2023-10-09T05:55:59.339614+0000","last_unstale":"2023-10-09T05:56:20.346098+0000","last_undegraded":"2023-10-09T05:56:20.346098+0000","last_fullsized":"2023-10-09T05:56:20.346098+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:01.613113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":92,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.220171+0000","last_change":"2023-10-09T05:55:59.300201+0000","last_active":"2023-10-09T05:56:20.220171+0000","last_peered":"2023-10-09T05:56:20.220171+0000","last_clean":"2023-10-09T05:56:20.220171+0000","last_became_active":"2023-10-09T05:55:59.299709+0000","last_became_peered":"2023-10-09T05:55:59.299709+0000","last_unstale":"2023-10-09T05:56:20.220171+0000","last_undegraded":"2023-10-09T05:56:20.220171+0000","last_fullsized":"2023-10-09T05:56:20.220171+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:32:55.608383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.2","version":"226'7084","reported_seq":19296,"reported_epoch":226,"state":"active+recovery_wait","last_fresh":"2023-10-09T05:56:29.211498+0000","last_change":"2023-10-09T05:55:22.328640+0000","last_active":"2023-10-09T05:56:29.211498+0000","last_peered":"2023-10-09T05:56:29.211498+0000","last_clean":"2023-10-09T05:55:20.973604+0000","last_became_active":"2023-10-09T05:55:22.208547+0000","last_became_peered":"2023-10-09T05:55:22.208547+0000","last_unstale":"2023-10-09T05:56:29.211498+0000","last_undegraded":"2023-10-09T05:56:29.211498+0000","last_fullsized":"2023-10-09T05:56:29.211498+0000","mapping_epoch":159,"log_start":"106'632","ondisk_log_start":"106'632","created":102,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6452,"log_dups_size":0,"ondisk_log_size":6452,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60637184,"num_objects":925,"num_object_clones":0,"num_object_copies":2775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2084,"num_write_kb":96304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1505,"num_bytes_recovered":95617024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":925}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":474,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.339467+0000","last_change":"2023-10-09T05:54:19.202509+0000","last_active":"2023-10-09T05:56:20.339467+0000","last_peered":"2023-10-09T05:56:20.339467+0000","last_clean":"2023-10-09T05:56:20.339467+0000","last_became_active":"2023-10-09T05:54:19.202144+0000","last_became_peered":"2023-10-09T05:54:19.202144+0000","last_unstale":"2023-10-09T05:56:20.339467+0000","last_undegraded":"2023-10-09T05:56:20.339467+0000","last_fullsized":"2023-10-09T05:56:20.339467+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":220,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.345636+0000","last_change":"2023-10-09T05:54:16.215036+0000","last_active":"2023-10-09T05:56:20.345636+0000","last_peered":"2023-10-09T05:56:20.345636+0000","last_clean":"2023-10-09T05:56:20.345636+0000","last_became_active":"2023-10-09T05:54:16.212771+0000","last_became_peered":"2023-10-09T05:54:16.212771+0000","last_unstale":"2023-10-09T05:56:20.345636+0000","last_undegraded":"2023-10-09T05:56:20.345636+0000","last_fullsized":"2023-10-09T05:56:20.345636+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:29.278423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":252,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:24.478990+0000","last_change":"2023-10-09T05:55:59.374662+0000","last_active":"2023-10-09T05:56:24.478990+0000","last_peered":"2023-10-09T05:56:24.478990+0000","last_clean":"2023-10-09T05:56:24.478990+0000","last_became_active":"2023-10-09T05:55:59.305777+0000","last_became_peered":"2023-10-09T05:55:59.305777+0000","last_unstale":"2023-10-09T05:56:24.478990+0000","last_undegraded":"2023-10-09T05:56:24.478990+0000","last_fullsized":"2023-10-09T05:56:24.478990+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:05.569760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":168,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.220746+0000","last_change":"2023-10-09T05:40:12.676587+0000","last_active":"2023-10-09T05:56:20.220746+0000","last_peered":"2023-10-09T05:56:20.220746+0000","last_clean":"2023-10-09T05:56:20.220746+0000","last_became_active":"2023-10-09T05:40:12.676330+0000","last_became_peered":"2023-10-09T05:40:12.676330+0000","last_unstale":"2023-10-09T05:56:20.220746+0000","last_undegraded":"2023-10-09T05:56:20.220746+0000","last_fullsized":"2023-10-09T05:56:20.220746+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:46.659556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"226'7082","reported_seq":7251,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:56:30.359380+0000","last_change":"2023-10-09T05:55:22.211279+0000","last_active":"2023-10-09T05:56:30.359380+0000","last_peered":"2023-10-09T05:56:30.359380+0000","last_clean":"2023-10-09T05:56:30.359380+0000","last_became_active":"2023-10-09T05:55:22.208477+0000","last_became_peered":"2023-10-09T05:55:22.208477+0000","last_unstale":"2023-10-09T05:56:30.359380+0000","last_undegraded":"2023-10-09T05:56:30.359380+0000","last_fullsized":"2023-10-09T05:56:30.359380+0000","mapping_epoch":174,"log_start":"107'1488","ondisk_log_start":"107'1488","created":102,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5594,"log_dups_size":0,"ondisk_log_size":5594,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62472192,"num_objects":953,"num_object_clones":0,"num_object_copies":2859,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":953,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2087,"num_write_kb":97312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.1e","version":"226'7508","reported_seq":20039,"reported_epoch":226,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:56:29.421892+0000","last_change":"2023-10-09T05:56:00.413231+0000","last_active":"2023-10-09T05:56:29.421892+0000","last_peered":"2023-10-09T05:56:29.421892+0000","last_clean":"2023-10-09T05:55:58.105865+0000","last_became_active":"2023-10-09T05:56:00.403893+0000","last_became_peered":"2023-10-09T05:56:00.403893+0000","last_unstale":"2023-10-09T05:56:29.421892+0000","last_undegraded":"2023-10-09T05:56:29.421892+0000","last_fullsized":"2023-10-09T05:56:29.421892+0000","mapping_epoch":219,"log_start":"107'1851","ondisk_log_start":"107'1851","created":207,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5657,"log_dups_size":0,"ondisk_log_size":5657,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124092416,"num_objects":1894,"num_object_clones":0,"num_object_copies":5682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1894,"num_objects_unfound":0,"num_objects_dirty":1894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4097,"num_write_kb":191680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3074,"num_bytes_recovered":196280320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,1],"avail_no_missing":["4","1","2","5"],"object_location_counts":[{"shards":"1,4,5","objects":1894}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":154,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.220141+0000","last_change":"2023-10-09T05:53:59.191302+0000","last_active":"2023-10-09T05:56:20.220141+0000","last_peered":"2023-10-09T05:56:20.220141+0000","last_clean":"2023-10-09T05:56:20.220141+0000","last_became_active":"2023-10-09T05:53:59.190911+0000","last_became_peered":"2023-10-09T05:53:59.190911+0000","last_unstale":"2023-10-09T05:56:20.220141+0000","last_undegraded":"2023-10-09T05:56:20.220141+0000","last_fullsized":"2023-10-09T05:56:20.220141+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:12.801201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":195,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.220347+0000","last_change":"2023-10-09T05:55:59.305449+0000","last_active":"2023-10-09T05:56:20.220347+0000","last_peered":"2023-10-09T05:56:20.220347+0000","last_clean":"2023-10-09T05:56:20.220347+0000","last_became_active":"2023-10-09T05:55:59.305109+0000","last_became_peered":"2023-10-09T05:55:59.305109+0000","last_unstale":"2023-10-09T05:56:20.220347+0000","last_undegraded":"2023-10-09T05:56:20.220347+0000","last_fullsized":"2023-10-09T05:56:20.220347+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:31.830141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.19","version":"226'7459","reported_seq":9574,"reported_epoch":226,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:56:29.476730+0000","last_change":"2023-10-09T05:56:12.857185+0000","last_active":"2023-10-09T05:56:29.476730+0000","last_peered":"2023-10-09T05:56:29.476730+0000","last_clean":"2023-10-09T05:51:33.020184+0000","last_became_active":"2023-10-09T05:54:57.470115+0000","last_became_peered":"2023-10-09T05:54:57.470115+0000","last_unstale":"2023-10-09T05:56:29.476730+0000","last_undegraded":"2023-10-09T05:54:52.846685+0000","last_fullsized":"2023-10-09T05:54:49.570105+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":188,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7459,"log_dups_size":0,"ondisk_log_size":7459,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123043840,"num_objects":1878,"num_object_clones":0,"num_object_copies":5634,"num_objects_missing_on_primary":571,"num_objects_missing":571,"num_objects_degraded":571,"num_objects_misplaced":1862,"num_objects_unfound":0,"num_objects_dirty":1878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4195,"num_write_kb":194304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":958,"num_bytes_recovered":62783488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,3],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,7","objects":1291},{"shards":"0,3","objects":16},{"shards":"2,7","objects":571}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":258,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.339577+0000","last_change":"2023-10-09T05:55:59.376804+0000","last_active":"2023-10-09T05:56:20.339577+0000","last_peered":"2023-10-09T05:56:20.339577+0000","last_clean":"2023-10-09T05:56:20.339577+0000","last_became_active":"2023-10-09T05:55:59.355912+0000","last_became_peered":"2023-10-09T05:55:59.355912+0000","last_unstale":"2023-10-09T05:56:20.339577+0000","last_undegraded":"2023-10-09T05:56:20.339577+0000","last_fullsized":"2023-10-09T05:56:20.339577+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:15.551718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":252,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.222626+0000","last_change":"2023-10-09T05:55:59.336237+0000","last_active":"2023-10-09T05:56:20.222626+0000","last_peered":"2023-10-09T05:56:20.222626+0000","last_clean":"2023-10-09T05:56:20.222626+0000","last_became_active":"2023-10-09T05:55:59.335912+0000","last_became_peered":"2023-10-09T05:55:59.335912+0000","last_unstale":"2023-10-09T05:56:20.222626+0000","last_undegraded":"2023-10-09T05:56:20.222626+0000","last_fullsized":"2023-10-09T05:56:20.222626+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:07.757931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"226'7356","reported_seq":7510,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:56:30.389231+0000","last_change":"2023-10-09T05:54:45.671348+0000","last_active":"2023-10-09T05:56:30.389231+0000","last_peered":"2023-10-09T05:56:30.389231+0000","last_clean":"2023-10-09T05:56:30.389231+0000","last_became_active":"2023-10-09T05:54:45.664954+0000","last_became_peered":"2023-10-09T05:54:45.664954+0000","last_unstale":"2023-10-09T05:56:30.389231+0000","last_undegraded":"2023-10-09T05:56:30.389231+0000","last_fullsized":"2023-10-09T05:56:30.389231+0000","mapping_epoch":188,"log_start":"106'122","ondisk_log_start":"106'122","created":188,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7234,"log_dups_size":0,"ondisk_log_size":7234,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121700352,"num_objects":1857,"num_object_clones":0,"num_object_copies":5571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4109,"num_write_kb":190912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":171,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.336762+0000","last_change":"2023-10-09T05:40:12.633195+0000","last_active":"2023-10-09T05:56:20.336762+0000","last_peered":"2023-10-09T05:56:20.336762+0000","last_clean":"2023-10-09T05:56:20.336762+0000","last_became_active":"2023-10-09T05:40:12.632942+0000","last_became_peered":"2023-10-09T05:40:12.632942+0000","last_unstale":"2023-10-09T05:56:20.336762+0000","last_undegraded":"2023-10-09T05:56:20.336762+0000","last_fullsized":"2023-10-09T05:56:20.336762+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:04.470685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":219,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.221398+0000","last_change":"2023-10-09T05:51:57.093104+0000","last_active":"2023-10-09T05:56:20.221398+0000","last_peered":"2023-10-09T05:56:20.221398+0000","last_clean":"2023-10-09T05:56:20.221398+0000","last_became_active":"2023-10-09T05:51:57.092810+0000","last_became_peered":"2023-10-09T05:51:57.092810+0000","last_unstale":"2023-10-09T05:56:20.221398+0000","last_undegraded":"2023-10-09T05:56:20.221398+0000","last_fullsized":"2023-10-09T05:56:20.221398+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:51.192227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.1b","version":"226'7691","reported_seq":26995,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:56:29.382355+0000","last_change":"2023-10-09T05:56:21.202420+0000","last_active":"2023-10-09T05:56:29.382355+0000","last_peered":"2023-10-09T05:56:29.382355+0000","last_clean":"2023-10-09T05:56:29.382355+0000","last_became_active":"2023-10-09T05:56:00.422122+0000","last_became_peered":"2023-10-09T05:56:00.422122+0000","last_unstale":"2023-10-09T05:56:29.382355+0000","last_undegraded":"2023-10-09T05:56:29.382355+0000","last_fullsized":"2023-10-09T05:56:29.382355+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7691,"log_dups_size":0,"ondisk_log_size":7691,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124157952,"num_objects":1895,"num_object_clones":0,"num_object_copies":5685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4209,"num_write_kb":195296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6264,"num_bytes_recovered":364544000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":144,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.345966+0000","last_change":"2023-10-09T05:55:59.328124+0000","last_active":"2023-10-09T05:56:20.345966+0000","last_peered":"2023-10-09T05:56:20.345966+0000","last_clean":"2023-10-09T05:56:20.345966+0000","last_became_active":"2023-10-09T05:55:59.327740+0000","last_became_peered":"2023-10-09T05:55:59.327740+0000","last_unstale":"2023-10-09T05:56:20.345966+0000","last_undegraded":"2023-10-09T05:56:20.345966+0000","last_fullsized":"2023-10-09T05:56:20.345966+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:35.523860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":215,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.334815+0000","last_change":"2023-10-09T05:52:42.261890+0000","last_active":"2023-10-09T05:56:20.334815+0000","last_peered":"2023-10-09T05:56:20.334815+0000","last_clean":"2023-10-09T05:56:20.334815+0000","last_became_active":"2023-10-09T05:52:42.261521+0000","last_became_peered":"2023-10-09T05:52:42.261521+0000","last_unstale":"2023-10-09T05:56:20.334815+0000","last_undegraded":"2023-10-09T05:56:20.334815+0000","last_fullsized":"2023-10-09T05:56:20.334815+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:59.208145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.1a","version":"226'7566","reported_seq":9907,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-10-09T05:56:29.344804+0000","last_change":"2023-10-09T05:55:22.219965+0000","last_active":"2023-10-09T05:56:29.344804+0000","last_peered":"2023-10-09T05:56:29.344804+0000","last_clean":"2023-10-09T05:56:29.344804+0000","last_became_active":"2023-10-09T05:55:22.214658+0000","last_became_peered":"2023-10-09T05:55:22.214658+0000","last_unstale":"2023-10-09T05:56:29.344804+0000","last_undegraded":"2023-10-09T05:56:29.344804+0000","last_fullsized":"2023-10-09T05:56:29.344804+0000","mapping_epoch":207,"log_start":"108'2184","ondisk_log_start":"108'2184","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5382,"log_dups_size":0,"ondisk_log_size":5382,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122617856,"num_objects":1871,"num_object_clones":0,"num_object_copies":5613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4142,"num_write_kb":192416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":543,"num_bytes_recovered":35553280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":177,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.221548+0000","last_change":"2023-10-09T05:52:23.654749+0000","last_active":"2023-10-09T05:56:20.221548+0000","last_peered":"2023-10-09T05:56:20.221548+0000","last_clean":"2023-10-09T05:56:20.221548+0000","last_became_active":"2023-10-09T05:52:23.643044+0000","last_became_peered":"2023-10-09T05:52:23.643044+0000","last_unstale":"2023-10-09T05:56:20.221548+0000","last_undegraded":"2023-10-09T05:56:20.221548+0000","last_fullsized":"2023-10-09T05:56:20.221548+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":139,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:23:08.898078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":235,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-10-09T05:56:20.221124+0000","last_change":"2023-10-09T05:55:59.300656+0000","last_active":"2023-10-09T05:56:20.221124+0000","last_peered":"2023-10-09T05:56:20.221124+0000","last_clean":"2023-10-09T05:56:20.221124+0000","last_became_active":"2023-10-09T05:55:59.300240+0000","last_became_peered":"2023-10-09T05:55:59.300240+0000","last_unstale":"2023-10-09T05:56:20.221124+0000","last_undegraded":"2023-10-09T05:56:20.221124+0000","last_fullsized":"2023-10-09T05:56:20.221124+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:53.634777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":36,"stat_sum":{"num_bytes":3910860824,"num_objects":59676,"num_object_clones":0,"num_object_copies":179028,"num_objects_missing_on_primary":1539,"num_objects_missing":1539,"num_objects_degraded":29746,"num_objects_misplaced":17090,"num_objects_unfound":0,"num_objects_dirty":59676,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":133406,"num_write_kb":6178561,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50893,"num_bytes_recovered":3197960216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14127087616,"data_stored":14127071328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":231295,"ondisk_log_size":231295,"up":108,"acting":100,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":10238,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":2624160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234202,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2207764,"kb_used_data":1898188,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":91537388,"statfs":{"total":95995035648,"available":93734285312,"internally_reserved":0,"allocated":1943744512,"data_stored":1942903189,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":1.1599999999999999,"15min":1.8129999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":1.98,"5min":20.247,"15min":20.247},"last":2.6669999999999998},{"interface":"front","average":{"1min":0.88700000000000001,"5min":1.165,"15min":1.704},"min":{"1min":0.313,"5min":0.252,"15min":0.252},"max":{"1min":3.9740000000000002,"5min":20.236000000000001,"15min":20.236000000000001},"last":0.44700000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:56:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3340000000000001,"5min":1.0029999999999999,"15min":0.69499999999999995},"min":{"1min":0.308,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.048,"5min":7.9859999999999998,"15min":7.9859999999999998},"last":2.3540000000000001},{"interface":"front","average":{"1min":0.91200000000000003,"5min":0.85699999999999998,"15min":0.66600000000000004},"min":{"1min":0.32400000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.4480000000000004,"5min":5.8129999999999997,"15min":5.8129999999999997},"last":0.63600000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":2.2229999999999999,"15min":1.1499999999999999},"min":{"1min":0.28799999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.8860000000000001,"5min":50.57,"15min":50.57},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.998,"5min":2.2440000000000002,"15min":1.2130000000000001},"min":{"1min":0.35999999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":4.2220000000000004,"5min":50.536999999999999,"15min":50.536999999999999},"last":2.0609999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:55:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":5.6600000000000001,"15min":2.2970000000000002},"min":{"1min":0.38,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.458,"5min":147.54900000000001,"15min":147.54900000000001},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.61599999999999999,"5min":5.5579999999999998,"15min":2.2440000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":1.9590000000000001,"5min":144.065,"15min":144.065},"last":2.2709999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:56:07 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.68600000000000005,"15min":0.54800000000000004},"min":{"1min":0.28599999999999998,"5min":0.248,"15min":0.24399999999999999},"max":{"1min":1.589,"5min":9.5329999999999995,"15min":9.5329999999999995},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.89800000000000002,"15min":0.63200000000000001},"min":{"1min":0.33100000000000002,"5min":0.29099999999999998,"15min":0.27400000000000002},"max":{"1min":1.877,"5min":7.6120000000000001,"15min":7.6120000000000001},"last":0.34499999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:56:07 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.94399999999999995,"15min":0.64200000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.7970000000000002,"5min":10.795999999999999,"15min":10.795999999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.69799999999999995,"15min":0.59799999999999998},"min":{"1min":0.315,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.365,"5min":3.9380000000000002,"15min":3.9380000000000002},"last":0.33100000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:56:07 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":2.4889999999999999,"15min":1.1890000000000001},"min":{"1min":0.33100000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":2.46,"5min":131.94399999999999,"15min":131.94399999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.63200000000000001,"5min":2.5369999999999999,"15min":1.218},"min":{"1min":0.35899999999999999,"5min":0.33100000000000002,"15min":0.29999999999999999},"max":{"1min":1.931,"5min":135.23400000000001,"15min":135.23400000000001},"last":0.38500000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495838,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2042060,"kb_used_data":1736580,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":91703092,"statfs":{"total":95995035648,"available":93903966208,"internally_reserved":0,"allocated":1778257920,"data_stored":1777425476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,1,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:56:01 2023","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":0.85499999999999998,"15min":1.0589999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":3.1339999999999999,"5min":7.0599999999999996,"15min":7.0599999999999996},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.877,"5min":0.88600000000000001,"15min":1.1140000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":3.0259999999999998,"5min":6.6459999999999999,"15min":6.6459999999999999},"last":0.47299999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:55:40 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":0.84599999999999997,"15min":0.67400000000000004},"min":{"1min":0.29499999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.1339999999999999,"5min":3.7949999999999999,"15min":3.7949999999999999},"last":0.436},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.95299999999999996,"15min":0.70699999999999996},"min":{"1min":0.29899999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.2130000000000001,"5min":9.9670000000000005,"15min":9.9670000000000005},"last":0.40899999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:56:01 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.1639999999999999,"15min":0.747},"min":{"1min":0.29299999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.3520000000000003,"5min":6.6859999999999999,"15min":6.6859999999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.1639999999999999,"15min":0.75900000000000001},"min":{"1min":0.32000000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.4900000000000002,"5min":9.9390000000000001,"15min":9.9390000000000001},"last":0.45800000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:55:40 2023","interfaces":[{"interface":"back","average":{"1min":9.6150000000000002,"5min":3.0600000000000001,"15min":1.389},"min":{"1min":0.33300000000000002,"5min":0.311,"15min":0.311},"max":{"1min":93.626000000000005,"5min":93.626000000000005,"15min":93.626000000000005},"last":0.39900000000000002},{"interface":"front","average":{"1min":9.2959999999999994,"5min":2.8879999999999999,"15min":1.3480000000000001},"min":{"1min":0.32500000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":93.643000000000001,"5min":93.643000000000001,"15min":93.643000000000001},"last":0.48199999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:55:45 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.93999999999999995,"15min":0.624},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":2.9159999999999999,"5min":7.9100000000000001,"15min":7.9100000000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.755,"5min":0.90600000000000003,"15min":0.63900000000000001},"min":{"1min":0.32600000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":2.3450000000000002,"5min":7.3300000000000001,"15min":7.3300000000000001},"last":0.37}]},{"osd":5,"last update":"Mon Oct 9 05:55:45 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.91700000000000004,"15min":0.63600000000000001},"min":{"1min":0.316,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.875,"5min":13.879,"15min":13.879},"last":45.933999999999997},{"interface":"front","average":{"1min":0.93799999999999994,"5min":1.0249999999999999,"15min":0.69999999999999996},"min":{"1min":0.35099999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.9429999999999996,"5min":7.29,"15min":7.29},"last":45.917999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:56:10 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":1.2669999999999999,"15min":0.78200000000000003},"min":{"1min":0.36599999999999999,"5min":0.33400000000000002,"15min":0.316},"max":{"1min":2.956,"5min":19.914999999999999,"15min":19.914999999999999},"last":52.042999999999999},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.125,"15min":0.79200000000000004},"min":{"1min":0.38800000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.7869999999999999,"5min":7.0309999999999997,"15min":7.0309999999999997},"last":51.259}]}]},{"osd":1,"up_from":0,"seq":382252089427,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":129,"seq":554050781229,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1025756,"kb_used_data":923988,"kb_used_omap":29,"kb_used_meta":101730,"kb_avail":92719396,"statfs":{"total":95995035648,"available":94944661504,"internally_reserved":0,"allocated":946163712,"data_stored":944911269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30322,"internal_metadata":104171918},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,0,1,1,2,4,1,1],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:56:02 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.81699999999999995,"15min":0.73999999999999999},"min":{"1min":0.249,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.6429999999999998,"5min":9.1720000000000006,"15min":9.1720000000000006},"last":0.22},{"interface":"front","average":{"1min":0.78100000000000003,"5min":0.92400000000000004,"15min":1.073},"min":{"1min":0.29599999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.411,"5min":4.0049999999999999,"15min":4.0049999999999999},"last":6.6420000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:56:02 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.81899999999999995,"15min":0.96499999999999997},"min":{"1min":0.23599999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":2.456,"5min":3.9199999999999999,"15min":3.9199999999999999},"last":6.6040000000000001},{"interface":"front","average":{"1min":0.755,"5min":0.86499999999999999,"15min":0.96099999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.5599999999999996,"5min":5.4160000000000004,"15min":5.4160000000000004},"last":1.5069999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:56:02 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":0.80000000000000004,"15min":0.78300000000000003},"min":{"1min":0.30399999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.6609999999999996,"5min":9.6609999999999996,"15min":9.6609999999999996},"last":6.6790000000000003},{"interface":"front","average":{"1min":1.01,"5min":0.84299999999999997,"15min":0.88100000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":7.1749999999999998,"5min":7.1749999999999998,"15min":7.1749999999999998},"last":1.5289999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:56:02 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.97899999999999998,"15min":1.141},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.089,"5min":9.0050000000000008,"15min":9.0050000000000008},"last":1.893},{"interface":"front","average":{"1min":0.79900000000000004,"5min":0.78300000000000003,"15min":0.86299999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.645,"5min":3.1549999999999998,"15min":3.1549999999999998},"last":0.77000000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:56:02 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.97099999999999997,"15min":1.089},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.4850000000000003,"5min":7.3959999999999999,"15min":7.3959999999999999},"last":1.4410000000000001},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.89600000000000002,"15min":0.90800000000000003},"min":{"1min":0.47099999999999997,"5min":0.439,"15min":0.439},"max":{"1min":1.1279999999999999,"5min":9.1080000000000005,"15min":9.1080000000000005},"last":1.9630000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:56:02 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":0.99199999999999999,"15min":1.226},"min":{"1min":0.378,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":4.5279999999999996,"5min":6.6849999999999996,"15min":6.6849999999999996},"last":1.4810000000000001},{"interface":"front","average":{"1min":0.81799999999999995,"5min":1.03,"15min":1.1599999999999999},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":3.7890000000000001,"5min":9.141,"15min":9.141},"last":1.9339999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:56:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.1950000000000001,"15min":1.345},"min":{"1min":0.438,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":5.548,"5min":9.0700000000000003,"15min":9.0700000000000003},"last":1.994},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.0920000000000001,"15min":1.3009999999999999},"min":{"1min":0.44900000000000001,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":4.431,"5min":5.8730000000000002,"15min":5.8730000000000002},"last":6.5529999999999999}]}]},{"osd":2,"up_from":94,"seq":403726925918,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3078644,"kb_used_data":2906668,"kb_used_omap":81,"kb_used_meta":171886,"kb_avail":90666508,"statfs":{"total":95995035648,"available":92842504192,"internally_reserved":0,"allocated":2976428032,"data_stored":2975266760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":176011482},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:56:07 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":1.1020000000000001,"15min":1.4990000000000001},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":7.9889999999999999,"5min":11.444000000000001,"15min":11.444000000000001},"last":1.851},{"interface":"front","average":{"1min":0.78500000000000003,"5min":1.008,"15min":1.3759999999999999},"min":{"1min":0.254,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":2.6840000000000002,"5min":6.883,"15min":6.883},"last":13.093999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":2.9399999999999999,"15min":1.262},"min":{"1min":0.255,"5min":0.218,"15min":0.218},"max":{"1min":2.7789999999999999,"5min":163.24199999999999,"15min":163.24199999999999},"last":1.002},{"interface":"front","average":{"1min":0.65000000000000002,"5min":2.7210000000000001,"15min":1.1759999999999999},"min":{"1min":0.28399999999999997,"5min":0.214,"15min":0.214},"max":{"1min":4.1299999999999999,"5min":163.27000000000001,"15min":163.27000000000001},"last":0.58199999999999996}]},{"osd":3,"last update":"Mon Oct 9 05:55:31 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":3.0129999999999999,"15min":1.2729999999999999},"min":{"1min":0.25900000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.85199999999999998,"5min":170.07599999999999,"15min":170.07599999999999},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.80700000000000005,"5min":1.7310000000000001,"15min":0.871},"min":{"1min":0.33300000000000002,"5min":0.25,"15min":0.25},"max":{"1min":4.218,"5min":63.600000000000001,"15min":63.600000000000001},"last":0.35899999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":1.206,"15min":0.84799999999999998},"min":{"1min":0.39400000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.552,"5min":11.372999999999999,"15min":11.372999999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.873,"5min":1.1319999999999999,"15min":0.84599999999999997},"min":{"1min":0.33200000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.5519999999999996,"5min":6.5709999999999997,"15min":6.5709999999999997},"last":0.54600000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:55:59 2023","interfaces":[{"interface":"back","average":{"1min":1.256,"5min":1.2210000000000001,"15min":0.85599999999999998},"min":{"1min":0.371,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":8.8699999999999992,"5min":13.414,"15min":13.414},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.753,"5min":1.2470000000000001,"15min":0.85799999999999998},"min":{"1min":0.42399999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.52,"5min":13.438000000000001,"15min":13.438000000000001},"last":0.41999999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:55:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":1.2809999999999999,"15min":0.86599999999999999},"min":{"1min":0.40300000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":8.6920000000000002,"5min":11.59,"15min":11.59},"last":0.746},{"interface":"front","average":{"1min":0.80400000000000005,"5min":0.97499999999999998,"15min":0.78200000000000003},"min":{"1min":0.37,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":2.0190000000000001,"5min":6.4710000000000001,"15min":6.4710000000000001},"last":0.77600000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:55:59 2023","interfaces":[{"interface":"back","average":{"1min":1.788,"5min":1.5720000000000001,"15min":0.96899999999999997},"min":{"1min":0.46999999999999997,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":10.715999999999999,"5min":40.404000000000003,"15min":40.404000000000003},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.80500000000000005,"5min":1.4350000000000001,"15min":0.94599999999999995},"min":{"1min":0.49199999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":2.6240000000000001,"5min":40.113,"15min":40.113},"last":0.56499999999999995}]}]},{"osd":3,"up_from":99,"seq":425201762392,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2591820,"kb_used_data":2403588,"kb_used_omap":7,"kb_used_meta":188216,"kb_avail":91153332,"statfs":{"total":95995035648,"available":93341011968,"internally_reserved":0,"allocated":2461274112,"data_stored":2459971732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":192733800},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:56:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.2350000000000001,"15min":1.4430000000000001},"min":{"1min":0.22900000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":9.4510000000000005,"5min":15.536,"15min":15.536},"last":0.219},{"interface":"front","average":{"1min":2.4660000000000002,"5min":1.3979999999999999,"15min":1.528},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":14.707000000000001,"5min":14.707000000000001,"15min":14.707000000000001},"last":0.35699999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:55:29 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":2.2010000000000001,"15min":0.97999999999999998},"min":{"1min":0.25,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":6.2329999999999997,"5min":127.226,"15min":127.226},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.83599999999999997,"5min":2.274,"15min":1.0229999999999999},"min":{"1min":0.24399999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":5.1559999999999997,"5min":127.468,"15min":127.468},"last":0.36599999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:55:29 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":1.3700000000000001,"15min":0.70699999999999996},"min":{"1min":0.29999999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.1160000000000001,"5min":62.938000000000002,"15min":62.938000000000002},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.94799999999999995,"5min":1.78,"15min":0.88100000000000001},"min":{"1min":0.317,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":5.5220000000000002,"5min":62.43,"15min":62.43},"last":0.27400000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:55:29 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.91900000000000004,"15min":0.70699999999999996},"min":{"1min":0.34899999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.95999999999999996,"5min":7.7439999999999998,"15min":7.7439999999999998},"last":0.433},{"interface":"front","average":{"1min":1.109,"5min":0.91300000000000003,"15min":0.71499999999999997},"min":{"1min":0.371,"5min":0.311,"15min":0.311},"max":{"1min":6.3250000000000002,"5min":6.3250000000000002,"15min":6.3250000000000002},"last":0.90700000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:55:29 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":0.95399999999999996,"15min":0.71999999999999997},"min":{"1min":0.40999999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":6.274,"5min":6.274,"15min":6.274},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.83599999999999997,"15min":0.70299999999999996},"min":{"1min":0.41399999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.29,"5min":4.2839999999999998,"15min":4.2839999999999998},"last":0.56399999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:55:29 2023","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":1.069,"15min":0.77800000000000002},"min":{"1min":0.42999999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.298,"5min":12.798999999999999,"15min":12.798999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.99099999999999999,"15min":0.78300000000000003},"min":{"1min":0.44800000000000001,"5min":0.379,"15min":0.379},"max":{"1min":1.2110000000000001,"5min":12.773,"15min":12.773},"last":0.46200000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:55:29 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":1.0009999999999999,"15min":0.753},"min":{"1min":0.42899999999999999,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":2.4510000000000001,"5min":7.4580000000000002,"15min":7.4580000000000002},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.85599999999999998,"5min":1.03,"15min":0.78100000000000003},"min":{"1min":0.41399999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":2.9900000000000002,"5min":7.8399999999999999,"15min":7.8399999999999999},"last":0.49399999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019112,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1614036,"kb_used_data":1312524,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":92131116,"statfs":{"total":95995035648,"available":94342262784,"internally_reserved":0,"allocated":1344024576,"data_stored":1343183506,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:56:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1950000000000001,"5min":0.96499999999999997,"15min":0.84299999999999997},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":13.503,"5min":15.173999999999999,"15min":15.173999999999999},"last":6.5590000000000002},{"interface":"front","average":{"1min":1.4410000000000001,"5min":1.2689999999999999,"15min":1.4450000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":13.561999999999999,"5min":15.125999999999999,"15min":15.125999999999999},"last":6.5350000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:55:31 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":1,"15min":0.71499999999999997},"min":{"1min":0.28499999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.64500000000000002,"5min":9.2390000000000008,"15min":9.2390000000000008},"last":1.8979999999999999},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.95999999999999996,"15min":0.66800000000000004},"min":{"1min":0.30099999999999999,"5min":0.255,"15min":0.255},"max":{"1min":6.7779999999999996,"5min":8.2629999999999999,"15min":8.2629999999999999},"last":0.56899999999999995}]},{"osd":2,"last update":"Mon Oct 9 05:56:04 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":1.1279999999999999,"15min":0.85899999999999999},"min":{"1min":0.32500000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.6799999999999999,"5min":6.9240000000000004,"15min":33.262},"last":1.9530000000000001},{"interface":"front","average":{"1min":0.79200000000000004,"5min":1.21,"15min":0.95299999999999996},"min":{"1min":0.36499999999999999,"5min":0.313,"15min":0.313},"max":{"1min":6.1760000000000002,"5min":9.0030000000000001,"15min":33.240000000000002},"last":1.994}]},{"osd":3,"last update":"Mon Oct 9 05:55:31 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.524,"15min":0.90000000000000002},"min":{"1min":0.30599999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.6440000000000001,"5min":40.531999999999996,"15min":40.531999999999996},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.56599999999999995,"5min":1.5980000000000001,"15min":0.93300000000000005},"min":{"1min":0.36099999999999999,"5min":0.31,"15min":0.31},"max":{"1min":1.341,"5min":39.905999999999999,"15min":39.905999999999999},"last":0.42499999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:56:25 2023","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.76100000000000001,"15min":0.57499999999999996},"min":{"1min":0.30099999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.77400000000000002,"5min":10.012,"15min":10.012},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.502,"5min":0.95699999999999996,"15min":0.67100000000000004},"min":{"1min":0.26100000000000001,"5min":0.254,"15min":0.254},"max":{"1min":1.764,"5min":12.489000000000001,"15min":12.489000000000001},"last":1.929}]},{"osd":6,"last update":"Mon Oct 9 05:55:39 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.96599999999999997,"15min":0.66300000000000003},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.27400000000000002},"max":{"1min":1.327,"5min":11.234999999999999,"15min":11.234999999999999},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.72199999999999998,"15min":0.60599999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30299999999999999},"max":{"1min":1.0289999999999999,"5min":9.923,"15min":9.923},"last":0.45300000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:56:08 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":1.147,"15min":0.77700000000000002},"min":{"1min":0.32500000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.796,"5min":10.606,"15min":10.606},"last":1.972},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.878,"15min":0.68500000000000005},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.363,"5min":8.3819999999999997,"15min":8.3819999999999997},"last":0.63600000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757475,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1801612,"kb_used_data":1492036,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":91943540,"statfs":{"total":95995035648,"available":94150184960,"internally_reserved":0,"allocated":1527844864,"data_stored":1527008298,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:56:09 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":0.59199999999999997,"15min":0.66800000000000004},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":6.9530000000000003,"5min":6.9530000000000003,"15min":6.9530000000000003},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.73799999999999999,"15min":0.77800000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.9020000000000001,"5min":6.9020000000000001,"15min":6.9020000000000001},"last":1.155}]},{"osd":1,"last update":"Mon Oct 9 05:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":3.5350000000000001,"15min":1.536},"min":{"1min":0.314,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.4660000000000002,"5min":114.247,"15min":114.247},"last":1.179},{"interface":"front","average":{"1min":0.46200000000000002,"5min":3.3399999999999999,"15min":1.4890000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":1.091,"5min":114.678,"15min":114.678},"last":0.34000000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:55:57 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":2.4380000000000002,"15min":1.2},"min":{"1min":0.314,"5min":0.25,"15min":0.25},"max":{"1min":3.915,"5min":106.66500000000001,"15min":106.66500000000001},"last":1.196},{"interface":"front","average":{"1min":0.59999999999999998,"5min":2.2719999999999998,"15min":1.1479999999999999},"min":{"1min":0.29799999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.4910000000000001,"5min":106.779,"15min":106.779},"last":0.36699999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":8.9670000000000005,"15min":3.3500000000000001},"min":{"1min":0.33600000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.0289999999999999,"5min":272.81299999999999,"15min":272.81299999999999},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.61099999999999999,"5min":9.3320000000000007,"15min":3.5390000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":2.3220000000000001,"5min":273.00999999999999,"15min":273.00999999999999},"last":1.2110000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:56:19 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.70199999999999996,"15min":0.55100000000000005},"min":{"1min":0.29999999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":2.577,"5min":6.2050000000000001,"15min":6.2050000000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.72399999999999998,"15min":0.58099999999999996},"min":{"1min":0.308,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.484,"5min":13.050000000000001,"15min":13.050000000000001},"last":0.67700000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:55:33 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.71799999999999997,"15min":0.58399999999999996},"min":{"1min":0.30199999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.0030000000000001,"5min":10.606999999999999,"15min":10.606999999999999},"last":2.2450000000000001},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.92400000000000004,"15min":0.68300000000000005},"min":{"1min":0.34399999999999997,"5min":0.30299999999999999,"15min":0.28499999999999998},"max":{"1min":5.0609999999999999,"5min":5.9770000000000003,"15min":5.9770000000000003},"last":1.2250000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:56:13 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":1.077,"15min":0.749},"min":{"1min":0.33100000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.8860000000000001,"5min":9.2739999999999991,"15min":9.2739999999999991},"last":1.2390000000000001},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.68999999999999995,"15min":0.61399999999999999},"min":{"1min":0.371,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":2.1989999999999998,"5min":5.032,"15min":5.032},"last":1.615}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1474,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":940838912,"data_stored":940834840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1212223488,"data_stored":1212219416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2969702400,"data_stored":2969698328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2451570688,"data_stored":2451570688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1332412416,"data_stored":1332412416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1515720704,"data_stored":1515716632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1771765760,"data_stored":1771765760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1932853248,"data_stored":1932853248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:56:32.424 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:56:32.424 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_0 pg_num 46 2023-10-09T05:56:32.461 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:32.532 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:32.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:32.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:32.636 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:32.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:32 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 29746/179100 objects degraded (16.609%), 14 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:32.641 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:32 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 29746/179100 objects degraded (16.609%), 14 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:32.706 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:32.788 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:32.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:32.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:32.807 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:32 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 29746/179100 objects degraded (16.609%), 14 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:32.898 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:32.943 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:33.054 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:33.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:33.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:33.079 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:33.158 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:33.175 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:33.195 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:33.250 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:33.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:33.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:33.354 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:33.390 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:33.497 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:33.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:33.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:33.601 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:33.650 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:33.873 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:33.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:33.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:33.986 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:34.040 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:34.121 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:34.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:34.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:34.199 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:34 smithi064 ceph-mon[220044]: pgmap v526: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 29682/177930 objects degraded (16.682%); 16961/177930 objects misplaced (9.532%); 8.5 MiB/s, 156 objects/s recovering 2023-10-09T05:56:34.199 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:34 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/826861697' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:34.199 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:34 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:34.222 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T05:56:34.225 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:34.283 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:34.306 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:34 smithi156 ceph-mon[195101]: pgmap v526: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 29682/177930 objects degraded (16.682%); 16961/177930 objects misplaced (9.532%); 8.5 MiB/s, 156 objects/s recovering 2023-10-09T05:56:34.307 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:34 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/826861697' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:34.307 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:34 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:34.468 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:34.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:34 smithi064 ceph-mon[212750]: pgmap v526: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 29682/177930 objects degraded (16.682%); 16961/177930 objects misplaced (9.532%); 8.5 MiB/s, 156 objects/s recovering 2023-10-09T05:56:34.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:34 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/826861697' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:34.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:34 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:34.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:34.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:34.538 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T05:56:34.538 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:56:34.573 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:34.647 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:34.818 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:34.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:34.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:34.923 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:34.977 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:35.036 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:35.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:35.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:35.142 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:35.176 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:35.201 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:35 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:56:35.201 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:35 smithi064 ceph-mon[220044]: failed to encode map e227 with expected crc 2023-10-09T05:56:35.201 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:35 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:56:35.201 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:35 smithi064 ceph-mon[220044]: osdmap e227: 8 total, 8 up, 7 in 2023-10-09T05:56:35.201 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:35 smithi064 ceph-mon[220044]: failed to encode map e227 with expected crc 2023-10-09T05:56:35.201 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:35 smithi064 ceph-mon[220044]: failed to encode map e227 with expected crc 2023-10-09T05:56:35.287 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:35.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:35.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:35.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:35 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:56:35.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:35 smithi156 ceph-mon[195101]: failed to encode map e227 with expected crc 2023-10-09T05:56:35.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:35 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:56:35.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:35 smithi156 ceph-mon[195101]: osdmap e227: 8 total, 8 up, 7 in 2023-10-09T05:56:35.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:35 smithi156 ceph-mon[195101]: failed to encode map e227 with expected crc 2023-10-09T05:56:35.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:35 smithi156 ceph-mon[195101]: failed to encode map e227 with expected crc 2023-10-09T05:56:35.394 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:35.449 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:35.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:35 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:56:35.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:35 smithi064 ceph-mon[212750]: failed to encode map e227 with expected crc 2023-10-09T05:56:35.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:35 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:56:35.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:35 smithi064 ceph-mon[212750]: osdmap e227: 8 total, 8 up, 7 in 2023-10-09T05:56:35.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:35 smithi064 ceph-mon[212750]: failed to encode map e227 with expected crc 2023-10-09T05:56:35.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:35 smithi064 ceph-mon[212750]: failed to encode map e227 with expected crc 2023-10-09T05:56:35.470 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:56:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:56:35] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.43.0" 2023-10-09T05:56:35.619 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:56:35.624 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:35.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:35.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:35.733 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:35.798 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:35.908 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:35.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:35.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:36.016 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:36.070 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:36.133 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:56:36.211 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:36 smithi064 ceph-mon[212750]: pgmap v528: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 29185/174957 objects degraded (16.681%); 16797/174957 objects misplaced (9.601%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T05:56:36.265 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:36.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:36.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:36.344 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:56:36.371 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:36.385 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:56:36.427 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:56:36.440 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:36.460 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:56:36.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:36 smithi064 ceph-mon[220044]: pgmap v528: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 29185/174957 objects degraded (16.681%); 16797/174957 objects misplaced (9.601%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T05:56:36.495 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:56:36.520 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:36 smithi156 ceph-mon[195101]: pgmap v528: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 29185/174957 objects degraded (16.681%); 16797/174957 objects misplaced (9.601%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T05:56:36.546 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:56:36.558 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:36.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:36.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:36.598 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:56:36.630 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:56:36.631 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:56:36.631 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:56:36.632 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:56:36.632 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:56:36.632 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:56:36.632 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:56:36.632 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:56:36.632 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:56:36.666 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:36.703 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:36.778 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:36.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:36.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:36.885 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:56:36.946 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:37.000 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:37.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:37.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:37.104 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:37.130 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:37.182 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:37.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:37.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:37.218 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:37 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2497516755' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:37.234 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 6 pg_num to 46 2023-10-09T05:56:37.292 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:37.350 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:37.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:37 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/121775391' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "46"}]: dispatch 2023-10-09T05:56:37.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:37 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "46"}]: dispatch 2023-10-09T05:56:37.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:37 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 29185/174957 objects degraded (16.681%), 14 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:37.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:37 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2497516755' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:37.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:37 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/121775391' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "46"}]: dispatch 2023-10-09T05:56:37.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:37 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "46"}]: dispatch 2023-10-09T05:56:37.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:37 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 29185/174957 objects degraded (16.681%), 14 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:37.510 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:37 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2497516755' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:37.510 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:37 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/121775391' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "46"}]: dispatch 2023-10-09T05:56:37.510 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:37 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "46"}]: dispatch 2023-10-09T05:56:37.510 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:37 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 29185/174957 objects degraded (16.681%), 14 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:56:37.542 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:37.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:37.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:37.648 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:37.698 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:37.835 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:37.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:37.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:37.895 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:56:37.944 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:37.973 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:38.020 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:38.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:38.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:38.124 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:38.151 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:56:38.151 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T05:56:38.152 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T05:56:38.152 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T05:56:38.152 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.152 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64028 6.d 6:b2a5011c:::benchmark_data_smithi064_284193_object64027:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:56:38.152 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.138056+0000", 2023-10-09T05:56:38.153 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 305.00364922300002, 2023-10-09T05:56:38.153 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0914284259999998, 2023-10-09T05:56:38.153 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:38.153 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:38.153 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:38.154 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:38.154 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:38.154 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64028 2023-10-09T05:56:38.154 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.154 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:38.154 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.154 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:38.154 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.138056+0000", 2023-10-09T05:56:38.154 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.154 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.155 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.155 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:38.155 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.138056+0000", 2023-10-09T05:56:38.155 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.155 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.155 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.155 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:38.155 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.138055+0000", 2023-10-09T05:56:38.155 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:38.155 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.155 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.156 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:38.156 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.138098+0000", 2023-10-09T05:56:38.156 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3702000000000003e-05 2023-10-09T05:56:38.156 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.156 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.156 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:38.156 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.138112+0000", 2023-10-09T05:56:38.156 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3393999999999999e-05 2023-10-09T05:56:38.156 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.156 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.156 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:38.156 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.138119+0000", 2023-10-09T05:56:38.157 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.905e-06 2023-10-09T05:56:38.157 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.157 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.157 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.157 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.138179+0000", 2023-10-09T05:56:38.157 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0426e-05 2023-10-09T05:56:38.157 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.157 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.157 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.157 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.138187+0000", 2023-10-09T05:56:38.157 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.3680000000000002e-06 2023-10-09T05:56:38.158 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.158 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.158 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.158 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111163+0000", 2023-10-09T05:56:38.158 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.97297587900000004 2023-10-09T05:56:38.158 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.158 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.158 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.158 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111172+0000", 2023-10-09T05:56:38.158 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.4999999999999999e-06 2023-10-09T05:56:38.158 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.159 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.159 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.159 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.204580+0000", 2023-10-09T05:56:38.159 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0934087079999999 2023-10-09T05:56:38.159 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.159 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.159 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:38.159 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.204699+0000", 2023-10-09T05:56:38.159 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000118097 2023-10-09T05:56:38.159 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.159 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.159 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:56:38.160 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.204810+0000", 2023-10-09T05:56:38.160 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011183099999999999 2023-10-09T05:56:38.160 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.160 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.160 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:38.160 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.211455+0000", 2023-10-09T05:56:38.160 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0066445150000000001 2023-10-09T05:56:38.160 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.160 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.160 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.161 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.227513+0000", 2023-10-09T05:56:38.161 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.016057795999999999 2023-10-09T05:56:38.161 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.161 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.161 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.161 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.227594+0000", 2023-10-09T05:56:38.161 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.174e-05 2023-10-09T05:56:38.161 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.161 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.161 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:38.161 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.227636+0000", 2023-10-09T05:56:38.162 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2018999999999999e-05 2023-10-09T05:56:38.162 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.162 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.162 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:38.162 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.229485+0000", 2023-10-09T05:56:38.162 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0018480969999999999 2023-10-09T05:56:38.162 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.162 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:38.162 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.162 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.162 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.162 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64042 6.d 6:b4a627c6:::benchmark_data_smithi064_284193_object64041:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:56:38.163 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.148881+0000", 2023-10-09T05:56:38.163 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 304.99282463200001, 2023-10-09T05:56:38.163 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.083266171, 2023-10-09T05:56:38.163 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:38.163 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:38.163 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:38.163 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:38.163 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:38.163 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64042 2023-10-09T05:56:38.163 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.163 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:38.164 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.164 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:38.164 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.148881+0000", 2023-10-09T05:56:38.164 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.164 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.164 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.164 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:38.164 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.148881+0000", 2023-10-09T05:56:38.164 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.164 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.164 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.164 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:38.165 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.148879+0000", 2023-10-09T05:56:38.165 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:38.165 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.165 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.165 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:38.165 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.148915+0000", 2023-10-09T05:56:38.165 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5583999999999997e-05 2023-10-09T05:56:38.165 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.165 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.165 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:38.165 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.148925+0000", 2023-10-09T05:56:38.166 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.9150000000000008e-06 2023-10-09T05:56:38.166 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.166 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.166 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:38.166 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.148930+0000", 2023-10-09T05:56:38.166 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2100000000000001e-06 2023-10-09T05:56:38.166 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.166 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.166 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.166 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.149088+0000", 2023-10-09T05:56:38.166 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015745000000000001 2023-10-09T05:56:38.167 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.167 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.167 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.167 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.149096+0000", 2023-10-09T05:56:38.167 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.5560000000000008e-06 2023-10-09T05:56:38.167 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.167 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.167 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.167 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111207+0000", 2023-10-09T05:56:38.167 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96211053099999999 2023-10-09T05:56:38.167 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.168 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.168 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.168 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111214+0000", 2023-10-09T05:56:38.168 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3189999999999997e-06 2023-10-09T05:56:38.168 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.168 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.168 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.168 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.206599+0000", 2023-10-09T05:56:38.169 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.095385359 2023-10-09T05:56:38.169 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.169 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.169 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:38.169 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.206696+0000", 2023-10-09T05:56:38.170 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.6280000000000007e-05 2023-10-09T05:56:38.170 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.170 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.170 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:56:38.170 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.206803+0000", 2023-10-09T05:56:38.170 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000107219 2023-10-09T05:56:38.170 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.170 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.170 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:38.170 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.211465+0000", 2023-10-09T05:56:38.171 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0046617569999999999 2023-10-09T05:56:38.171 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.171 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.171 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.171 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.227565+0000", 2023-10-09T05:56:38.171 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.016100023000000001 2023-10-09T05:56:38.171 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.171 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.171 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.171 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.229614+0000", 2023-10-09T05:56:38.171 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0020497720000000001 2023-10-09T05:56:38.172 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.172 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.172 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:38.172 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.229681+0000", 2023-10-09T05:56:38.172 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6727e-05 2023-10-09T05:56:38.172 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.172 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.172 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:38.172 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.232147+0000", 2023-10-09T05:56:38.172 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0024657419999999999 2023-10-09T05:56:38.172 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.172 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:38.173 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.173 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.173 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.173 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64051 6.d 6:b4ee6d36:::benchmark_data_smithi064_284193_object64050:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:56:38.173 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.160656+0000", 2023-10-09T05:56:38.173 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 304.98104890799999, 2023-10-09T05:56:38.173 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0740513780000001, 2023-10-09T05:56:38.173 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:38.173 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:38.173 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:38.173 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:38.174 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:38.174 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64051 2023-10-09T05:56:38.174 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.174 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:38.174 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.174 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:38.174 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.160656+0000", 2023-10-09T05:56:38.174 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.174 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.174 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.174 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:38.175 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.160656+0000", 2023-10-09T05:56:38.175 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.175 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.175 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.175 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:38.175 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.160656+0000", 2023-10-09T05:56:38.175 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:56:38.175 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.175 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.175 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:38.175 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.160686+0000", 2023-10-09T05:56:38.176 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0577000000000002e-05 2023-10-09T05:56:38.176 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.176 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.176 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:38.176 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.160694+0000", 2023-10-09T05:56:38.176 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4800000000000004e-06 2023-10-09T05:56:38.176 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.176 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.176 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:38.176 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.160698+0000", 2023-10-09T05:56:38.176 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2119999999999997e-06 2023-10-09T05:56:38.177 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.177 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.177 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.177 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.160752+0000", 2023-10-09T05:56:38.177 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4283999999999997e-05 2023-10-09T05:56:38.177 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.177 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.177 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.177 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.160762+0000", 2023-10-09T05:56:38.177 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.4539999999999996e-06 2023-10-09T05:56:38.177 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.178 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.178 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.178 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111247+0000", 2023-10-09T05:56:38.178 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.95048511800000002 2023-10-09T05:56:38.178 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.178 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.178 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.178 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111254+0000", 2023-10-09T05:56:38.178 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4229999999999999e-06 2023-10-09T05:56:38.178 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.178 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.178 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.179 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.207847+0000", 2023-10-09T05:56:38.179 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.096592206 2023-10-09T05:56:38.179 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.179 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.179 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:38.179 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.207986+0000", 2023-10-09T05:56:38.179 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000138961 2023-10-09T05:56:38.179 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.179 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.179 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:56:38.179 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.208089+0000", 2023-10-09T05:56:38.179 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010323900000000001 2023-10-09T05:56:38.179 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.180 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.180 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:38.180 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.213812+0000", 2023-10-09T05:56:38.180 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0057226880000000001 2023-10-09T05:56:38.180 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.180 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.180 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.180 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.229558+0000", 2023-10-09T05:56:38.180 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015746514999999999 2023-10-09T05:56:38.180 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.180 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.180 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.181 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.232222+0000", 2023-10-09T05:56:38.181 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0026640000000000001 2023-10-09T05:56:38.181 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.181 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.181 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:38.181 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.232292+0000", 2023-10-09T05:56:38.181 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9648999999999997e-05 2023-10-09T05:56:38.181 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.181 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.181 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:38.181 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.234708+0000", 2023-10-09T05:56:38.182 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0024161170000000002 2023-10-09T05:56:38.182 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.182 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:38.182 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.182 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.182 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.182 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64056 6.d 6:bb5661cc:::benchmark_data_smithi064_284193_object64055:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:56:38.182 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.165123+0000", 2023-10-09T05:56:38.182 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 304.97658272899997, 2023-10-09T05:56:38.182 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0726746980000001, 2023-10-09T05:56:38.182 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:38.183 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:38.183 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:38.183 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:38.183 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:38.183 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64056 2023-10-09T05:56:38.183 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.183 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:38.183 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.183 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:38.183 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.165123+0000", 2023-10-09T05:56:38.183 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.183 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.184 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.184 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:38.184 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.165123+0000", 2023-10-09T05:56:38.184 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.184 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.184 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.184 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:38.184 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.165122+0000", 2023-10-09T05:56:38.184 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:56:38.184 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.184 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.185 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:38.185 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.165150+0000", 2023-10-09T05:56:38.185 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7784999999999999e-05 2023-10-09T05:56:38.185 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.185 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.185 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:38.185 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.165158+0000", 2023-10-09T05:56:38.185 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1100000000000003e-06 2023-10-09T05:56:38.185 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.185 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.185 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:38.185 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.165163+0000", 2023-10-09T05:56:38.186 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5889999999999996e-06 2023-10-09T05:56:38.186 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.186 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.186 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.186 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.165204+0000", 2023-10-09T05:56:38.186 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0961000000000003e-05 2023-10-09T05:56:38.186 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.186 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.186 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.186 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.165211+0000", 2023-10-09T05:56:38.186 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1849999999999998e-06 2023-10-09T05:56:38.187 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.187 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.187 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.187 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111284+0000", 2023-10-09T05:56:38.187 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.94607339800000001 2023-10-09T05:56:38.187 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.187 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.187 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.187 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111289+0000", 2023-10-09T05:56:38.187 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1630000000000004e-06 2023-10-09T05:56:38.187 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.187 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.188 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.188 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.209079+0000", 2023-10-09T05:56:38.188 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.097789296 2023-10-09T05:56:38.188 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.188 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.188 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:38.188 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.209170+0000", 2023-10-09T05:56:38.188 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1420000000000002e-05 2023-10-09T05:56:38.188 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.188 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.188 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:56:38.188 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.209274+0000", 2023-10-09T05:56:38.188 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000103553 2023-10-09T05:56:38.189 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.189 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.189 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:38.189 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.213841+0000", 2023-10-09T05:56:38.189 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0045676650000000003 2023-10-09T05:56:38.189 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.189 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.189 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.189 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.234777+0000", 2023-10-09T05:56:38.189 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.020935958000000001 2023-10-09T05:56:38.189 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.189 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.190 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.190 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.234979+0000", 2023-10-09T05:56:38.190 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00020143200000000001 2023-10-09T05:56:38.190 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.190 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.190 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:38.190 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.235052+0000", 2023-10-09T05:56:38.190 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3369999999999997e-05 2023-10-09T05:56:38.190 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.190 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.190 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:38.191 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.237797+0000", 2023-10-09T05:56:38.191 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00274537 2023-10-09T05:56:38.191 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.191 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:38.191 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.191 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.191 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.191 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64062 6.d 6:bfc7fb96:::benchmark_data_smithi064_284193_object64061:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:56:38.191 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.176528+0000", 2023-10-09T05:56:38.191 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 304.96517699899999, 2023-10-09T05:56:38.191 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.063752225, 2023-10-09T05:56:38.192 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:38.192 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:38.192 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:38.192 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:38.192 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:38.192 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64062 2023-10-09T05:56:38.192 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.192 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:38.192 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.192 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:38.193 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.176528+0000", 2023-10-09T05:56:38.193 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.193 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.193 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.193 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:38.193 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.176528+0000", 2023-10-09T05:56:38.193 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.193 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.193 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.193 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:38.193 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.176527+0000", 2023-10-09T05:56:38.194 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:56:38.194 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.194 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.194 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:38.194 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.176580+0000", 2023-10-09T05:56:38.194 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2935000000000001e-05 2023-10-09T05:56:38.194 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.194 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.194 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:38.194 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.176592+0000", 2023-10-09T05:56:38.194 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1481000000000001e-05 2023-10-09T05:56:38.195 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.195 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.195 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:38.195 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.176598+0000", 2023-10-09T05:56:38.195 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9540000000000001e-06 2023-10-09T05:56:38.195 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.195 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.195 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.195 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.176646+0000", 2023-10-09T05:56:38.195 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7753999999999997e-05 2023-10-09T05:56:38.195 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.196 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.196 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.196 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.176653+0000", 2023-10-09T05:56:38.196 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8259999999999993e-06 2023-10-09T05:56:38.196 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.196 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.196 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.196 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111323+0000", 2023-10-09T05:56:38.196 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.93466992699999996 2023-10-09T05:56:38.196 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.196 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.197 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.197 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111329+0000", 2023-10-09T05:56:38.197 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7699999999999998e-06 2023-10-09T05:56:38.197 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.197 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.197 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.197 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.210262+0000", 2023-10-09T05:56:38.197 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.098933036 2023-10-09T05:56:38.197 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.197 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.197 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:38.198 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.210386+0000", 2023-10-09T05:56:38.198 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012342000000000001 2023-10-09T05:56:38.198 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.198 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.198 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:56:38.198 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.210493+0000", 2023-10-09T05:56:38.198 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000106988 2023-10-09T05:56:38.198 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.199 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.199 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:38.199 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.216295+0000", 2023-10-09T05:56:38.199 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0058020320000000004 2023-10-09T05:56:38.199 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.199 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.199 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.199 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.234811+0000", 2023-10-09T05:56:38.199 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018516469000000001 2023-10-09T05:56:38.199 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.200 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.200 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.200 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.237872+0000", 2023-10-09T05:56:38.200 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.003061032 2023-10-09T05:56:38.200 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.200 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.200 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:38.200 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.237919+0000", 2023-10-09T05:56:38.200 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7448999999999999e-05 2023-10-09T05:56:38.200 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.200 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.201 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:38.201 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.240281+0000", 2023-10-09T05:56:38.201 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0023610879999999999 2023-10-09T05:56:38.201 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.201 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:38.201 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.201 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.201 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.201 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64070 6.d 6:bb0f4d69:::benchmark_data_smithi064_284193_object64069:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:56:38.201 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.203536+0000", 2023-10-09T05:56:38.201 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 304.93816966100002, 2023-10-09T05:56:38.202 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.039812205, 2023-10-09T05:56:38.202 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:38.202 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:38.202 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:38.202 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:38.202 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:38.202 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64070 2023-10-09T05:56:38.202 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.202 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:38.202 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.202 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:38.203 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.203536+0000", 2023-10-09T05:56:38.203 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.203 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.203 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.203 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:38.203 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.203536+0000", 2023-10-09T05:56:38.203 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.203 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.203 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.203 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:38.203 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.203534+0000", 2023-10-09T05:56:38.204 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:38.204 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.204 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.204 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:38.204 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.203598+0000", 2023-10-09T05:56:38.204 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4401999999999998e-05 2023-10-09T05:56:38.204 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.204 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.204 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:38.204 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.203617+0000", 2023-10-09T05:56:38.204 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8428999999999999e-05 2023-10-09T05:56:38.205 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.205 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.205 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:38.205 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.203624+0000", 2023-10-09T05:56:38.205 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5000000000000002e-06 2023-10-09T05:56:38.205 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.205 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.205 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.205 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.203672+0000", 2023-10-09T05:56:38.205 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7793000000000001e-05 2023-10-09T05:56:38.205 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.206 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.206 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.206 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.203679+0000", 2023-10-09T05:56:38.206 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7750000000000004e-06 2023-10-09T05:56:38.206 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.206 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.206 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.207 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111355+0000", 2023-10-09T05:56:38.207 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.90767584199999995 2023-10-09T05:56:38.207 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.207 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.207 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.207 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.111360+0000", 2023-10-09T05:56:38.207 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1970000000000004e-06 2023-10-09T05:56:38.207 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.207 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.207 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.207 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.211500+0000", 2023-10-09T05:56:38.208 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1001402220000001 2023-10-09T05:56:38.208 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.208 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.208 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:38.208 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.211590+0000", 2023-10-09T05:56:38.208 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9363000000000005e-05 2023-10-09T05:56:38.208 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.208 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.208 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:56:38.208 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.211698+0000", 2023-10-09T05:56:38.208 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000108789 2023-10-09T05:56:38.208 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.209 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.209 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:38.209 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.216302+0000", 2023-10-09T05:56:38.209 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0046034869999999999 2023-10-09T05:56:38.209 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.209 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.209 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.209 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.240331+0000", 2023-10-09T05:56:38.209 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.024028898 2023-10-09T05:56:38.209 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.209 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.210 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.210 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.240426+0000", 2023-10-09T05:56:38.210 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.5624000000000005e-05 2023-10-09T05:56:38.210 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.210 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.210 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:38.210 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.240483+0000", 2023-10-09T05:56:38.210 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6329999999999998e-05 2023-10-09T05:56:38.210 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.210 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.210 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:38.211 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.243348+0000", 2023-10-09T05:56:38.211 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002865214 2023-10-09T05:56:38.211 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.211 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:38.211 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.211 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.211 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.211 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64081 6.d 6:be2ec322:::benchmark_data_smithi064_284193_object64080:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-10-09T05:56:38.211 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:34.204234+0000", 2023-10-09T05:56:38.211 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 303.93747121199999, 2023-10-09T05:56:38.211 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0455772409999999, 2023-10-09T05:56:38.211 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:38.212 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:38.212 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:38.212 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:38.212 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:38.212 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64081 2023-10-09T05:56:38.212 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.212 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:38.212 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.212 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:38.212 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204234+0000", 2023-10-09T05:56:38.212 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.212 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.213 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.213 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:38.213 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204234+0000", 2023-10-09T05:56:38.213 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.213 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.213 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.213 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:38.213 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204233+0000", 2023-10-09T05:56:38.213 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:56:38.213 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.213 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.214 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:38.214 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204263+0000", 2023-10-09T05:56:38.214 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9601e-05 2023-10-09T05:56:38.214 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.214 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.214 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:38.214 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204271+0000", 2023-10-09T05:56:38.214 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8860000000000004e-06 2023-10-09T05:56:38.214 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.214 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.215 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:38.215 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204277+0000", 2023-10-09T05:56:38.215 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0140000000000004e-06 2023-10-09T05:56:38.215 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.215 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.215 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.215 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220790+0000", 2023-10-09T05:56:38.215 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.016513109000000002 2023-10-09T05:56:38.215 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.215 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.215 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.215 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.220796+0000", 2023-10-09T05:56:38.216 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0090000000000004e-06 2023-10-09T05:56:38.216 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.216 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.216 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.216 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.213877+0000", 2023-10-09T05:56:38.216 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99308116199999996 2023-10-09T05:56:38.216 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.216 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.216 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:38.216 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.213975+0000", 2023-10-09T05:56:38.216 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7417999999999995e-05 2023-10-09T05:56:38.217 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.217 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.217 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:56:38.217 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.214074+0000", 2023-10-09T05:56:38.217 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.9256e-05 2023-10-09T05:56:38.217 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.217 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.217 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:38.217 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.220163+0000", 2023-10-09T05:56:38.217 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0060896270000000002 2023-10-09T05:56:38.217 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.217 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.217 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.218 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.246362+0000", 2023-10-09T05:56:38.218 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.026198430000000002 2023-10-09T05:56:38.218 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.218 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.218 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.218 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.246630+0000", 2023-10-09T05:56:38.218 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00026832000000000001 2023-10-09T05:56:38.218 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.218 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.218 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:38.218 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.246689+0000", 2023-10-09T05:56:38.219 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8675000000000003e-05 2023-10-09T05:56:38.219 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.219 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.219 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:38.219 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:35.249811+0000", 2023-10-09T05:56:38.219 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.003122465 2023-10-09T05:56:38.219 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.219 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:38.219 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.219 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.219 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.219 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:82074 6.a 6:58f4f3d0:::benchmark_data_smithi064_284193_object82073:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T05:56:38.219 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:56.097757+0000", 2023-10-09T05:56:38.220 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 282.043948086, 2023-10-09T05:56:38.220 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.158105742, 2023-10-09T05:56:38.220 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:38.220 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T05:56:38.220 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:38.220 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:38.220 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:38.220 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 82074 2023-10-09T05:56:38.220 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.220 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:38.220 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.221 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:38.221 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.097757+0000", 2023-10-09T05:56:38.221 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.221 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.221 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.221 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:38.221 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.097757+0000", 2023-10-09T05:56:38.221 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.221 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.221 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.221 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:38.221 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.097756+0000", 2023-10-09T05:56:38.222 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:38.222 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.222 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.222 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:38.222 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.097799+0000", 2023-10-09T05:56:38.222 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3735000000000001e-05 2023-10-09T05:56:38.222 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.222 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.222 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:38.222 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.097817+0000", 2023-10-09T05:56:38.222 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7289e-05 2023-10-09T05:56:38.223 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.223 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.223 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:38.223 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.097824+0000", 2023-10-09T05:56:38.223 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1319999999999996e-06 2023-10-09T05:56:38.223 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.223 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.223 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.223 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.100302+0000", 2023-10-09T05:56:38.223 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0024780340000000001 2023-10-09T05:56:38.223 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.224 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.224 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:38.224 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.100452+0000", 2023-10-09T05:56:38.224 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000149863 2023-10-09T05:56:38.224 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.224 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.224 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 6", 2023-10-09T05:56:38.224 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.100616+0000", 2023-10-09T05:56:38.224 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000164673 2023-10-09T05:56:38.224 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.224 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.225 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:38.225 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.106430+0000", 2023-10-09T05:56:38.225 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0058139039999999999 2023-10-09T05:56:38.225 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.225 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.225 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.225 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.144922+0000", 2023-10-09T05:56:38.225 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.038491535 2023-10-09T05:56:38.225 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.225 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.225 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.225 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.144937+0000", 2023-10-09T05:56:38.226 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5568000000000001e-05 2023-10-09T05:56:38.226 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.226 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.226 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.226 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088545+0000", 2023-10-09T05:56:38.226 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.94360759900000002 2023-10-09T05:56:38.226 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.226 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.226 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.226 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088550+0000", 2023-10-09T05:56:38.226 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2449999999999998e-06 2023-10-09T05:56:38.227 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.227 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.227 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.227 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.255794+0000", 2023-10-09T05:56:38.227 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1672438190000001 2023-10-09T05:56:38.227 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.227 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.227 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:38.227 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.255863+0000", 2023-10-09T05:56:38.227 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8866000000000001e-05 2023-10-09T05:56:38.228 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.228 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:38.228 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.228 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.228 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.228 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:82079 6.a 6:59a1f462:::benchmark_data_smithi064_284193_object82078:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T05:56:38.228 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:56.101890+0000", 2023-10-09T05:56:38.228 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 282.039815487, 2023-10-09T05:56:38.228 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.154915967, 2023-10-09T05:56:38.228 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:38.228 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T05:56:38.228 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:38.229 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:38.229 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:38.229 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 82079 2023-10-09T05:56:38.229 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.229 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:38.229 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.229 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:38.229 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.101890+0000", 2023-10-09T05:56:38.229 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.229 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.229 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.230 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:38.230 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.101890+0000", 2023-10-09T05:56:38.230 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.230 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.230 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.230 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:38.230 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.101889+0000", 2023-10-09T05:56:38.230 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:56:38.230 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.230 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.230 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:38.230 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.101925+0000", 2023-10-09T05:56:38.231 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5763999999999998e-05 2023-10-09T05:56:38.231 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.231 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.231 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:38.231 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.101932+0000", 2023-10-09T05:56:38.231 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7290000000000003e-06 2023-10-09T05:56:38.231 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.231 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.231 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:38.231 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.101937+0000", 2023-10-09T05:56:38.231 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.972e-06 2023-10-09T05:56:38.231 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.231 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.232 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.232 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.103569+0000", 2023-10-09T05:56:38.232 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0016325479999999999 2023-10-09T05:56:38.232 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.232 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.232 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:38.232 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.103679+0000", 2023-10-09T05:56:38.232 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000109225 2023-10-09T05:56:38.232 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.232 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.232 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 6", 2023-10-09T05:56:38.232 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.103810+0000", 2023-10-09T05:56:38.233 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013134799999999999 2023-10-09T05:56:38.233 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.233 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.233 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:38.233 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.113355+0000", 2023-10-09T05:56:38.233 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.009545118 2023-10-09T05:56:38.233 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.233 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.233 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.233 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.147616+0000", 2023-10-09T05:56:38.233 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.034261319999999998 2023-10-09T05:56:38.233 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.234 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.234 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.234 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.147626+0000", 2023-10-09T05:56:38.234 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7899999999999994e-06 2023-10-09T05:56:38.234 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.234 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.234 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.234 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088568+0000", 2023-10-09T05:56:38.234 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.94094197000000002 2023-10-09T05:56:38.234 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.234 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.235 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.235 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088574+0000", 2023-10-09T05:56:38.235 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3439999999999997e-06 2023-10-09T05:56:38.235 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.235 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.235 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.235 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.256728+0000", 2023-10-09T05:56:38.235 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.168153088 2023-10-09T05:56:38.235 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.235 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.235 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:38.235 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.256806+0000", 2023-10-09T05:56:38.236 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8255000000000002e-05 2023-10-09T05:56:38.236 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.236 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:38.236 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.236 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.236 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.236 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:82095 6.d 6:b2648f7f:::benchmark_data_smithi064_284193_object82094:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:56:38.236 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:56.144382+0000", 2023-10-09T05:56:38.236 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 281.99732317500002, 2023-10-09T05:56:38.236 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1392636949999999, 2023-10-09T05:56:38.236 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:38.236 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:38.237 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:38.237 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:38.237 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:38.237 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 82095 2023-10-09T05:56:38.237 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.237 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:38.237 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.237 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:38.237 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.144382+0000", 2023-10-09T05:56:38.237 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.237 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.237 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.238 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:38.238 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.144382+0000", 2023-10-09T05:56:38.238 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.238 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.238 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.238 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:38.238 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.144382+0000", 2023-10-09T05:56:38.238 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:56:38.238 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.238 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.238 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:38.238 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.144417+0000", 2023-10-09T05:56:38.239 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5509999999999997e-05 2023-10-09T05:56:38.239 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.239 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.239 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:38.239 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.144426+0000", 2023-10-09T05:56:38.239 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6719999999999993e-06 2023-10-09T05:56:38.239 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.239 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.239 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:38.239 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.144431+0000", 2023-10-09T05:56:38.239 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2750000000000003e-06 2023-10-09T05:56:38.239 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.240 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.240 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.240 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.189174+0000", 2023-10-09T05:56:38.240 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.044742523999999999 2023-10-09T05:56:38.240 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.240 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.240 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.240 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.189184+0000", 2023-10-09T05:56:38.240 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0798999999999999e-05 2023-10-09T05:56:38.240 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.240 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.240 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.240 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088004+0000", 2023-10-09T05:56:38.241 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.89882000699999998 2023-10-09T05:56:38.241 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.241 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.241 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.241 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088016+0000", 2023-10-09T05:56:38.241 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1809e-05 2023-10-09T05:56:38.241 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.241 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.241 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.241 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.266944+0000", 2023-10-09T05:56:38.241 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.178928057 2023-10-09T05:56:38.241 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.242 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.242 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:38.242 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.267103+0000", 2023-10-09T05:56:38.242 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015863100000000001 2023-10-09T05:56:38.242 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.242 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.242 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:56:38.242 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.267262+0000", 2023-10-09T05:56:38.242 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000159388 2023-10-09T05:56:38.242 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.242 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.242 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:38.243 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.275961+0000", 2023-10-09T05:56:38.243 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0086985629999999994 2023-10-09T05:56:38.243 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.243 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.243 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.243 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.280428+0000", 2023-10-09T05:56:38.243 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0044668620000000003 2023-10-09T05:56:38.243 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.243 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.243 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.243 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.281192+0000", 2023-10-09T05:56:38.244 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00076383899999999999 2023-10-09T05:56:38.244 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.244 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.244 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:38.244 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.281240+0000", 2023-10-09T05:56:38.244 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8152999999999998e-05 2023-10-09T05:56:38.244 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.244 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.244 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:38.244 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.283646+0000", 2023-10-09T05:56:38.244 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0024061540000000002 2023-10-09T05:56:38.244 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.245 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:38.245 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.245 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.245 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.245 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:82106 6.d 6:b7051b26:::benchmark_data_smithi064_284193_object82105:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:56:38.245 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:56.173869+0000", 2023-10-09T05:56:38.245 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 281.96783656899999, 2023-10-09T05:56:38.245 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1196443020000002, 2023-10-09T05:56:38.245 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:38.245 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:38.245 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:38.246 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:38.246 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:38.246 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 82106 2023-10-09T05:56:38.246 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.246 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:38.246 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.246 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:38.246 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.173869+0000", 2023-10-09T05:56:38.246 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.246 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.246 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.247 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:38.247 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.173869+0000", 2023-10-09T05:56:38.247 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.247 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.247 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.247 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:38.247 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.173867+0000", 2023-10-09T05:56:38.247 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:38.247 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.247 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.247 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:38.248 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.173923+0000", 2023-10-09T05:56:38.248 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.647e-05 2023-10-09T05:56:38.248 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.248 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.248 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:38.248 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.173949+0000", 2023-10-09T05:56:38.248 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.5079000000000002e-05 2023-10-09T05:56:38.248 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.248 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.248 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:38.248 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.173955+0000", 2023-10-09T05:56:38.248 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4300000000000003e-06 2023-10-09T05:56:38.248 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.249 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.249 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.249 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.189208+0000", 2023-10-09T05:56:38.249 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015253040000000001 2023-10-09T05:56:38.249 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.249 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.249 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.249 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.189215+0000", 2023-10-09T05:56:38.249 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0999999999999998e-06 2023-10-09T05:56:38.249 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.249 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.249 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.250 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088046+0000", 2023-10-09T05:56:38.250 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.89883056900000002 2023-10-09T05:56:38.250 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.250 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.250 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.250 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088053+0000", 2023-10-09T05:56:38.250 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5909999999999998e-06 2023-10-09T05:56:38.250 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.250 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.250 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.250 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.269849+0000", 2023-10-09T05:56:38.250 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1817954580000001 2023-10-09T05:56:38.251 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.251 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.251 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:38.251 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.269978+0000", 2023-10-09T05:56:38.251 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012922500000000001 2023-10-09T05:56:38.251 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.251 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.251 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:56:38.251 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.270106+0000", 2023-10-09T05:56:38.251 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012807000000000001 2023-10-09T05:56:38.251 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.252 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.252 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:38.252 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.275969+0000", 2023-10-09T05:56:38.252 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0058634300000000002 2023-10-09T05:56:38.252 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.252 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.252 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.252 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.288964+0000", 2023-10-09T05:56:38.252 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012994916 2023-10-09T05:56:38.252 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.252 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.252 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.253 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.290167+0000", 2023-10-09T05:56:38.253 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0012025320000000001 2023-10-09T05:56:38.253 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.253 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.253 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:38.253 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.290229+0000", 2023-10-09T05:56:38.253 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2274999999999996e-05 2023-10-09T05:56:38.253 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.253 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.253 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:38.253 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.293513+0000", 2023-10-09T05:56:38.254 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0032839380000000001 2023-10-09T05:56:38.254 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.254 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:38.254 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.254 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.254 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.254 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:82129 6.a 6:5e1e98b2:::benchmark_data_smithi064_284193_object82128:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:56:38.254 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:56.214518+0000", 2023-10-09T05:56:38.254 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 281.92718767999997, 2023-10-09T05:56:38.254 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.051940192, 2023-10-09T05:56:38.254 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:38.254 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:38.255 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:38.255 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:38.255 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:38.255 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 82129 2023-10-09T05:56:38.255 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.255 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:38.255 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.255 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:38.255 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.214518+0000", 2023-10-09T05:56:38.255 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.255 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.255 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.256 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:38.256 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.214518+0000", 2023-10-09T05:56:38.256 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.256 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.256 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.256 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:38.256 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.214513+0000", 2023-10-09T05:56:38.256 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999948 2023-10-09T05:56:38.256 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.256 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.256 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:38.256 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.214557+0000", 2023-10-09T05:56:38.257 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4127e-05 2023-10-09T05:56:38.257 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.257 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.257 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:38.257 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.214580+0000", 2023-10-09T05:56:38.257 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3390000000000001e-05 2023-10-09T05:56:38.257 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.257 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.257 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:38.257 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.214589+0000", 2023-10-09T05:56:38.257 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9719999999999998e-06 2023-10-09T05:56:38.257 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.258 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.258 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.258 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.216231+0000", 2023-10-09T05:56:38.258 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001641637 2023-10-09T05:56:38.258 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.258 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.258 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.258 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.216242+0000", 2023-10-09T05:56:38.258 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0929e-05 2023-10-09T05:56:38.258 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.258 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.258 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.259 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088597+0000", 2023-10-09T05:56:38.259 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.87235592100000003 2023-10-09T05:56:38.259 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.259 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.259 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.259 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.088604+0000", 2023-10-09T05:56:38.259 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7649999999999996e-06 2023-10-09T05:56:38.259 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.259 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.259 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.259 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.256832+0000", 2023-10-09T05:56:38.260 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1682279229999999 2023-10-09T05:56:38.260 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.260 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.260 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:38.260 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.256991+0000", 2023-10-09T05:56:38.260 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015927700000000001 2023-10-09T05:56:38.260 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.260 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.260 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:56:38.260 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.257137+0000", 2023-10-09T05:56:38.260 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014602500000000001 2023-10-09T05:56:38.260 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.260 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.261 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:38.261 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.263290+0000", 2023-10-09T05:56:38.261 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0061524459999999998 2023-10-09T05:56:38.261 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.261 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.261 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.261 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.265238+0000", 2023-10-09T05:56:38.261 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001948537 2023-10-09T05:56:38.261 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.261 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.261 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.262 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.265424+0000", 2023-10-09T05:56:38.262 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000185164 2023-10-09T05:56:38.262 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.262 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.262 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:38.262 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.265488+0000", 2023-10-09T05:56:38.262 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4632000000000001e-05 2023-10-09T05:56:38.262 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.262 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.262 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:38.262 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.266458+0000", 2023-10-09T05:56:38.262 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00096960600000000003 2023-10-09T05:56:38.263 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.263 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:38.263 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.263 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.263 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.263 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:82138 6.a 6:5fd97b9e:::benchmark_data_smithi064_284193_object82137:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:56:38.263 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:56.236652+0000", 2023-10-09T05:56:38.263 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 281.90505307699999, 2023-10-09T05:56:38.263 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0342095279999999, 2023-10-09T05:56:38.263 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:38.263 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:38.264 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:38.264 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:38.264 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:38.264 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 82138 2023-10-09T05:56:38.264 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.264 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:38.264 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.264 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:38.264 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.236652+0000", 2023-10-09T05:56:38.264 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.264 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.264 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.265 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:38.265 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.236652+0000", 2023-10-09T05:56:38.265 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.265 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.265 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.265 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:38.265 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.236650+0000", 2023-10-09T05:56:38.265 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:38.265 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.265 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.265 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:38.265 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.236699+0000", 2023-10-09T05:56:38.266 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8606000000000002e-05 2023-10-09T05:56:38.266 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.266 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.266 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:38.266 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.236723+0000", 2023-10-09T05:56:38.266 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4097999999999999e-05 2023-10-09T05:56:38.266 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.266 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.266 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:38.266 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.236732+0000", 2023-10-09T05:56:38.266 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.5709999999999998e-06 2023-10-09T05:56:38.266 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.267 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.267 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.267 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.236803+0000", 2023-10-09T05:56:38.267 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1162999999999996e-05 2023-10-09T05:56:38.267 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.267 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.267 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.267 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:56.236815+0000", 2023-10-09T05:56:38.267 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2274e-05 2023-10-09T05:56:38.267 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.267 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.267 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.268 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.089210+0000", 2023-10-09T05:56:38.268 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.852395291 2023-10-09T05:56:38.268 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.268 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.268 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.268 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:57.089216+0000", 2023-10-09T05:56:38.268 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6060000000000002e-06 2023-10-09T05:56:38.268 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.268 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.268 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.268 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.260084+0000", 2023-10-09T05:56:38.268 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.170867879 2023-10-09T05:56:38.269 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.269 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.269 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:38.269 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.260256+0000", 2023-10-09T05:56:38.269 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00017194700000000001 2023-10-09T05:56:38.269 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.269 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.269 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:56:38.269 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.260449+0000", 2023-10-09T05:56:38.269 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00019292300000000001 2023-10-09T05:56:38.269 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.269 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.270 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:38.270 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.265252+0000", 2023-10-09T05:56:38.270 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0048034929999999998 2023-10-09T05:56:38.270 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.270 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.270 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.270 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.265322+0000", 2023-10-09T05:56:38.270 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9429000000000002e-05 2023-10-09T05:56:38.270 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.270 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.270 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.271 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.269814+0000", 2023-10-09T05:56:38.271 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0044918750000000002 2023-10-09T05:56:38.271 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.271 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.271 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:38.271 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.269876+0000", 2023-10-09T05:56:38.271 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2621999999999998e-05 2023-10-09T05:56:38.271 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.271 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.271 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:38.271 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:58.270862+0000", 2023-10-09T05:56:38.272 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00098564500000000005 2023-10-09T05:56:38.272 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.272 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:38.272 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.272 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.272 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.272 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:91002 6.4 6:2cef0a59:::benchmark_data_smithi064_284193_object91001:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-10-09T05:56:38.272 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:10.173814+0000", 2023-10-09T05:56:38.272 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 267.96789149699998, 2023-10-09T05:56:38.272 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.336934939, 2023-10-09T05:56:38.272 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:38.272 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T05:56:38.273 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:38.273 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:38.273 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:38.273 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 91002 2023-10-09T05:56:38.273 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.273 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:38.273 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.273 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:38.273 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.173814+0000", 2023-10-09T05:56:38.273 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.273 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.274 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.274 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:38.274 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.173814+0000", 2023-10-09T05:56:38.274 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.274 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.274 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.274 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:38.274 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.173813+0000", 2023-10-09T05:56:38.274 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:56:38.274 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.274 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.275 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:38.275 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.173843+0000", 2023-10-09T05:56:38.275 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9298999999999998e-05 2023-10-09T05:56:38.275 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.275 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.275 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:38.275 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.173850+0000", 2023-10-09T05:56:38.275 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.413e-06 2023-10-09T05:56:38.275 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.275 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.275 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:38.275 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.173854+0000", 2023-10-09T05:56:38.275 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4159999999999997e-06 2023-10-09T05:56:38.276 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.276 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.276 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.276 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.177830+0000", 2023-10-09T05:56:38.276 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0039752570000000003 2023-10-09T05:56:38.276 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.276 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.276 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:38.276 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.178005+0000", 2023-10-09T05:56:38.276 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00017501100000000001 2023-10-09T05:56:38.276 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.276 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.277 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:56:38.277 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.178180+0000", 2023-10-09T05:56:38.277 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00017503899999999999 2023-10-09T05:56:38.277 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.277 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.277 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.277 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.212537+0000", 2023-10-09T05:56:38.277 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.034357681000000001 2023-10-09T05:56:38.277 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.277 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.277 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.278 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.212546+0000", 2023-10-09T05:56:38.278 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.3909999999999998e-06 2023-10-09T05:56:38.278 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.278 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.278 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.278 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.488951+0000", 2023-10-09T05:56:38.278 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.276405051 2023-10-09T05:56:38.278 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.278 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.278 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for degraded object", 2023-10-09T05:56:38.278 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.489540+0000", 2023-10-09T05:56:38.279 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00058895699999999998 2023-10-09T05:56:38.279 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.279 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.279 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.279 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.510637+0000", 2023-10-09T05:56:38.279 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.021097202999999998 2023-10-09T05:56:38.279 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.279 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.279 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:38.279 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.510749+0000", 2023-10-09T05:56:38.279 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011176299999999999 2023-10-09T05:56:38.279 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.280 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:38.280 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.280 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.280 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.280 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:91017 6.d 6:b3504a53:::benchmark_data_smithi064_284193_object91016:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-10-09T05:56:38.280 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:10.210870+0000", 2023-10-09T05:56:38.280 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 267.93083548800001, 2023-10-09T05:56:38.280 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2371660760000001, 2023-10-09T05:56:38.280 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:38.280 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:38.280 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:38.281 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:38.281 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:38.281 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 91017 2023-10-09T05:56:38.281 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.281 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:38.281 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.281 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:38.281 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210870+0000", 2023-10-09T05:56:38.281 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.281 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.281 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.281 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:38.282 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210870+0000", 2023-10-09T05:56:38.282 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.282 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.282 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.282 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:38.282 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210869+0000", 2023-10-09T05:56:38.282 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:56:38.282 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.282 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.282 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:38.282 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210904+0000", 2023-10-09T05:56:38.283 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.4656999999999997e-05 2023-10-09T05:56:38.283 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.283 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.283 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:38.283 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210914+0000", 2023-10-09T05:56:38.283 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0464e-05 2023-10-09T05:56:38.283 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.283 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.283 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:38.283 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210920+0000", 2023-10-09T05:56:38.283 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6380000000000001e-06 2023-10-09T05:56:38.283 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.284 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.284 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.284 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210970+0000", 2023-10-09T05:56:38.284 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0062999999999998e-05 2023-10-09T05:56:38.284 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.284 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.284 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.284 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210979+0000", 2023-10-09T05:56:38.284 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.7299999999999994e-06 2023-10-09T05:56:38.284 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.284 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.285 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.285 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.441103+0000", 2023-10-09T05:56:38.285 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2301243880000001 2023-10-09T05:56:38.285 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.285 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.285 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:38.285 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.441224+0000", 2023-10-09T05:56:38.285 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012059 2023-10-09T05:56:38.285 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.285 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.285 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:56:38.286 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.441351+0000", 2023-10-09T05:56:38.286 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012714700000000001 2023-10-09T05:56:38.286 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.286 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.286 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:38.286 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.445558+0000", 2023-10-09T05:56:38.286 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0042071510000000001 2023-10-09T05:56:38.286 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.286 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.286 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.286 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.446131+0000", 2023-10-09T05:56:38.287 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00057308200000000004 2023-10-09T05:56:38.287 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.287 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.287 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.287 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.447453+0000", 2023-10-09T05:56:38.287 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0013219460000000001 2023-10-09T05:56:38.287 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.287 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.287 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:38.287 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.447493+0000", 2023-10-09T05:56:38.287 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9920999999999999e-05 2023-10-09T05:56:38.287 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.288 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.288 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:38.288 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.448036+0000", 2023-10-09T05:56:38.288 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00054314100000000002 2023-10-09T05:56:38.288 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.288 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:38.288 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.288 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.288 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.288 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:91019 6.4 6:25997699:::benchmark_data_smithi064_284193_object91018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-10-09T05:56:38.288 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:10.215587+0000", 2023-10-09T05:56:38.289 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 267.92611860599999, 2023-10-09T05:56:38.289 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2962565879999999, 2023-10-09T05:56:38.289 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:38.289 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:38.289 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:38.289 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:38.289 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:38.289 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 91019 2023-10-09T05:56:38.289 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.289 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:38.289 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.289 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:38.290 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215587+0000", 2023-10-09T05:56:38.290 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.290 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.290 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.290 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:38.290 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215587+0000", 2023-10-09T05:56:38.290 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.290 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.290 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.290 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:38.290 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215585+0000", 2023-10-09T05:56:38.291 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:38.291 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.291 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.291 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:38.291 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215691+0000", 2023-10-09T05:56:38.291 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000105311 2023-10-09T05:56:38.291 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.291 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.291 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:38.291 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215710+0000", 2023-10-09T05:56:38.291 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.893e-05 2023-10-09T05:56:38.291 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.291 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.292 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:38.292 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215717+0000", 2023-10-09T05:56:38.292 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8950000000000001e-06 2023-10-09T05:56:38.292 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.292 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.292 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.292 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.216473+0000", 2023-10-09T05:56:38.292 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00075626899999999995 2023-10-09T05:56:38.292 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.292 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.293 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.293 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.216492+0000", 2023-10-09T05:56:38.293 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9072999999999998e-05 2023-10-09T05:56:38.293 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.293 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.293 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.293 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.489588+0000", 2023-10-09T05:56:38.293 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.273096518 2023-10-09T05:56:38.293 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.293 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.293 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:38.294 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.489698+0000", 2023-10-09T05:56:38.294 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000109848 2023-10-09T05:56:38.294 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.294 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.294 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:56:38.294 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.489836+0000", 2023-10-09T05:56:38.294 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013729700000000001 2023-10-09T05:56:38.294 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.294 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.294 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:38.294 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.496824+0000", 2023-10-09T05:56:38.294 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0069883929999999999 2023-10-09T05:56:38.295 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.295 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.295 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.295 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.504295+0000", 2023-10-09T05:56:38.295 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0074707799999999998 2023-10-09T05:56:38.295 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.295 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.295 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.295 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.510792+0000", 2023-10-09T05:56:38.295 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00649718 2023-10-09T05:56:38.295 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.295 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.296 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:38.296 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.510853+0000", 2023-10-09T05:56:38.296 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0868000000000001e-05 2023-10-09T05:56:38.296 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.296 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.296 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:38.296 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.511843+0000", 2023-10-09T05:56:38.296 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00099058400000000008 2023-10-09T05:56:38.296 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.296 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:38.296 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.296 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.297 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.297 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:91052 6.d 6:b9656c98:::benchmark_data_smithi064_284193_object91051:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-10-09T05:56:38.297 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:10.282349+0000", 2023-10-09T05:56:38.297 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 267.85935656999999, 2023-10-09T05:56:38.297 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.174734331, 2023-10-09T05:56:38.297 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:38.297 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:38.297 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:38.297 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:38.297 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:38.297 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 91052 2023-10-09T05:56:38.298 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.298 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:38.298 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.298 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:38.298 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.282349+0000", 2023-10-09T05:56:38.298 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.298 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.298 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.298 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:38.298 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.282349+0000", 2023-10-09T05:56:38.298 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.299 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.299 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.299 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:38.299 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.282347+0000", 2023-10-09T05:56:38.299 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:38.299 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.299 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.299 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:38.299 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.282415+0000", 2023-10-09T05:56:38.299 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8465000000000003e-05 2023-10-09T05:56:38.299 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.300 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.300 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:38.300 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.282431+0000", 2023-10-09T05:56:38.300 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5739e-05 2023-10-09T05:56:38.300 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.300 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.300 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:38.300 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.282439+0000", 2023-10-09T05:56:38.300 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5900000000000002e-06 2023-10-09T05:56:38.300 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.300 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.301 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.301 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.282507+0000", 2023-10-09T05:56:38.301 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7891000000000004e-05 2023-10-09T05:56:38.301 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.301 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.301 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.301 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.282519+0000", 2023-10-09T05:56:38.301 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2887e-05 2023-10-09T05:56:38.301 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.301 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.301 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.302 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.443395+0000", 2023-10-09T05:56:38.302 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.160875715 2023-10-09T05:56:38.302 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.302 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.302 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:38.302 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.443472+0000", 2023-10-09T05:56:38.302 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7158999999999998e-05 2023-10-09T05:56:38.302 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.302 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.302 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:56:38.302 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.443574+0000", 2023-10-09T05:56:38.303 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000102141 2023-10-09T05:56:38.303 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.303 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.303 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:38.303 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.445569+0000", 2023-10-09T05:56:38.303 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001994928 2023-10-09T05:56:38.303 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.303 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.303 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.303 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.448070+0000", 2023-10-09T05:56:38.303 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0025002280000000002 2023-10-09T05:56:38.304 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.304 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.304 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.304 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.456279+0000", 2023-10-09T05:56:38.304 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0082089169999999996 2023-10-09T05:56:38.304 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.304 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.304 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:38.304 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.456318+0000", 2023-10-09T05:56:38.304 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9218000000000003e-05 2023-10-09T05:56:38.304 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.304 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.304 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:38.305 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.457083+0000", 2023-10-09T05:56:38.305 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00076537899999999995 2023-10-09T05:56:38.305 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.305 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:38.305 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.305 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.305 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.305 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:110007 6.1d 6:b8aa09df:::benchmark_data_smithi064_284193_object13464:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2023-10-09T05:56:38.305 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:22.341065+0000", 2023-10-09T05:56:38.305 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 75.800640271999995, 2023-10-09T05:56:38.305 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9528985840000002, 2023-10-09T05:56:38.305 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:38.306 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:38.306 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:38.306 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:38.306 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:38.306 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 110007 2023-10-09T05:56:38.306 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.306 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:38.306 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.306 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:38.306 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.341065+0000", 2023-10-09T05:56:38.306 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.306 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.307 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.307 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:38.307 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.341065+0000", 2023-10-09T05:56:38.307 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.307 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.307 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.307 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:38.307 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.341064+0000", 2023-10-09T05:56:38.307 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:38.307 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.307 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.308 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:38.308 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.341070+0000", 2023-10-09T05:56:38.308 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6549999999999998e-06 2023-10-09T05:56:38.308 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.308 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.308 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:38.308 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.341087+0000", 2023-10-09T05:56:38.308 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6824999999999999e-05 2023-10-09T05:56:38.308 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.308 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.308 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:38.309 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.341093+0000", 2023-10-09T05:56:38.309 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8510000000000004e-06 2023-10-09T05:56:38.309 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.309 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.309 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.309 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.341175+0000", 2023-10-09T05:56:38.309 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.2087000000000002e-05 2023-10-09T05:56:38.309 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.309 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.309 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.309 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.341186+0000", 2023-10-09T05:56:38.309 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0701e-05 2023-10-09T05:56:38.310 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.310 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.310 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.310 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.172841+0000", 2023-10-09T05:56:38.310 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.83165533400000002 2023-10-09T05:56:38.310 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.310 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.310 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.310 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.172847+0000", 2023-10-09T05:56:38.310 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.891e-06 2023-10-09T05:56:38.310 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.311 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.311 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.311 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.286688+0000", 2023-10-09T05:56:38.311 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1138407260000001 2023-10-09T05:56:38.311 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.311 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.311 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:38.311 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.286821+0000", 2023-10-09T05:56:38.311 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013355999999999999 2023-10-09T05:56:38.311 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.312 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.312 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1,2", 2023-10-09T05:56:38.312 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.287004+0000", 2023-10-09T05:56:38.312 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00018287199999999999 2023-10-09T05:56:38.312 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.312 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.312 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:38.312 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.291811+0000", 2023-10-09T05:56:38.312 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0048072410000000003 2023-10-09T05:56:38.312 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.312 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.313 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.313 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.293518+0000", 2023-10-09T05:56:38.313 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0017064420000000001 2023-10-09T05:56:38.313 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.313 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.313 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.313 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.293622+0000", 2023-10-09T05:56:38.313 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000104602 2023-10-09T05:56:38.313 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.313 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.314 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.314 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.293839+0000", 2023-10-09T05:56:38.314 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000217054 2023-10-09T05:56:38.314 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.314 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.314 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:38.314 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.293884+0000", 2023-10-09T05:56:38.314 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4354000000000003e-05 2023-10-09T05:56:38.314 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.314 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.314 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:38.314 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.293964+0000", 2023-10-09T05:56:38.315 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9850000000000003e-05 2023-10-09T05:56:38.315 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.315 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:38.315 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.315 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.315 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.315 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:119182 6.d 6:b79c5f24:::benchmark_data_smithi064_284193_object22639:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e212)", 2023-10-09T05:56:38.315 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:43.662278+0000", 2023-10-09T05:56:38.315 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 54.479427194000003, 2023-10-09T05:56:38.315 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0380490080000002, 2023-10-09T05:56:38.315 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:38.316 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:38.316 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:38.316 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:38.316 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:38.316 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 119182 2023-10-09T05:56:38.316 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.316 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:38.316 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.316 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:38.316 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.662278+0000", 2023-10-09T05:56:38.316 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.316 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.317 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.317 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:38.317 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.662278+0000", 2023-10-09T05:56:38.317 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.317 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.317 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.317 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:38.317 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.662277+0000", 2023-10-09T05:56:38.317 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:38.317 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.317 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.318 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:38.318 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.662281+0000", 2023-10-09T05:56:38.318 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3100000000000002e-06 2023-10-09T05:56:38.318 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.318 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.318 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:38.318 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.662292+0000", 2023-10-09T05:56:38.318 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0444e-05 2023-10-09T05:56:38.318 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.318 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.318 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:38.319 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.662299+0000", 2023-10-09T05:56:38.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6140000000000003e-06 2023-10-09T05:56:38.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.319 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.319 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.319 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.680967+0000", 2023-10-09T05:56:38.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018667429999999999 2023-10-09T05:56:38.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.319 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.319 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.319 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.680979+0000", 2023-10-09T05:56:38.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2771e-05 2023-10-09T05:56:38.320 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.320 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.320 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.320 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.668837+0000", 2023-10-09T05:56:38.320 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.98785785100000001 2023-10-09T05:56:38.320 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.320 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.320 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.320 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.668849+0000", 2023-10-09T05:56:38.320 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1681e-05 2023-10-09T05:56:38.320 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.320 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.321 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.321 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.689311+0000", 2023-10-09T05:56:38.321 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0204623779999999 2023-10-09T05:56:38.321 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.321 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.321 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:38.321 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.689472+0000", 2023-10-09T05:56:38.321 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00016038200000000001 2023-10-09T05:56:38.321 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.321 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.321 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:56:38.321 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.689627+0000", 2023-10-09T05:56:38.322 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000155284 2023-10-09T05:56:38.322 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.322 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.322 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:38.322 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.691124+0000", 2023-10-09T05:56:38.322 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0014973219999999999 2023-10-09T05:56:38.322 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.322 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.322 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.322 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.700147+0000", 2023-10-09T05:56:38.322 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0090226330000000004 2023-10-09T05:56:38.323 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.323 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.323 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.323 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.700223+0000", 2023-10-09T05:56:38.323 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.648e-05 2023-10-09T05:56:38.323 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.323 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.323 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:38.323 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.700284+0000", 2023-10-09T05:56:38.323 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0338999999999999e-05 2023-10-09T05:56:38.323 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.324 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.324 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:38.324 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.700327+0000", 2023-10-09T05:56:38.324 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3349000000000001e-05 2023-10-09T05:56:38.324 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.324 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:38.324 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.324 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.324 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.324 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:119212 6.1d 6:bf532f68:::benchmark_data_smithi064_284193_object22669:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-10-09T05:56:38.324 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:44.125794+0000", 2023-10-09T05:56:38.324 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 54.015911668999998, 2023-10-09T05:56:38.325 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5860949519999998, 2023-10-09T05:56:38.325 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:38.325 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:38.325 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:38.325 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:38.325 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:38.325 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 119212 2023-10-09T05:56:38.325 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.325 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:38.325 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.325 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:38.326 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.125794+0000", 2023-10-09T05:56:38.326 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.326 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.326 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.326 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:38.326 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.125794+0000", 2023-10-09T05:56:38.326 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:38.326 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.326 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.326 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:38.326 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.125792+0000", 2023-10-09T05:56:38.327 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:38.327 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.327 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.327 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:38.327 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.125799+0000", 2023-10-09T05:56:38.327 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7340000000000002e-06 2023-10-09T05:56:38.327 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.327 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.327 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:38.327 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.125818+0000", 2023-10-09T05:56:38.327 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9021e-05 2023-10-09T05:56:38.327 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.328 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.328 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:38.328 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.125824+0000", 2023-10-09T05:56:38.328 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7479999999999999e-06 2023-10-09T05:56:38.328 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.328 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.328 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.328 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.125878+0000", 2023-10-09T05:56:38.328 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4055999999999999e-05 2023-10-09T05:56:38.328 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.328 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.328 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.329 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.125887+0000", 2023-10-09T05:56:38.329 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.4939999999999992e-06 2023-10-09T05:56:38.329 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.329 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.329 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.329 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.672545+0000", 2023-10-09T05:56:38.329 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.54665808100000002 2023-10-09T05:56:38.329 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.329 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.329 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:38.329 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.672554+0000", 2023-10-09T05:56:38.329 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1819999999999995e-06 2023-10-09T05:56:38.330 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.330 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.330 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:38.330 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.698457+0000", 2023-10-09T05:56:38.330 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0259025020000001 2023-10-09T05:56:38.330 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.330 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.330 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:38.330 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.698603+0000", 2023-10-09T05:56:38.330 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014633800000000001 2023-10-09T05:56:38.330 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.330 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.331 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:56:38.331 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.698704+0000", 2023-10-09T05:56:38.331 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010069900000000001 2023-10-09T05:56:38.331 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.331 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.331 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:38.331 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.700342+0000", 2023-10-09T05:56:38.331 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0016376730000000001 2023-10-09T05:56:38.331 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.331 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.331 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.331 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.708516+0000", 2023-10-09T05:56:38.332 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0081747599999999997 2023-10-09T05:56:38.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.332 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.332 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:38.332 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.711810+0000", 2023-10-09T05:56:38.332 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.003293921 2023-10-09T05:56:38.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.332 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.332 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:38.332 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.711852+0000", 2023-10-09T05:56:38.332 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1709000000000003e-05 2023-10-09T05:56:38.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:38.332 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:38.333 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:38.333 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.711889+0000", 2023-10-09T05:56:38.333 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6625999999999998e-05 2023-10-09T05:56:38.333 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.333 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:38.333 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.333 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:38.333 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:38.333 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:56:38.334 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:38.440 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:38.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:38 smithi064 ceph-mon[212750]: pgmap v529: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 28680/170130 objects degraded (16.858%); 16320/170130 objects misplaced (9.593%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T05:56:38.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:38 smithi064 ceph-mon[212750]: failed to encode map e228 with expected crc 2023-10-09T05:56:38.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:38 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "46"}]': finished 2023-10-09T05:56:38.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:38 smithi064 ceph-mon[212750]: osdmap e228: 8 total, 8 up, 7 in 2023-10-09T05:56:38.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:38 smithi064 ceph-mon[212750]: failed to encode map e228 with expected crc 2023-10-09T05:56:38.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:38 smithi064 ceph-mon[212750]: failed to encode map e228 with expected crc 2023-10-09T05:56:38.451 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:38 smithi064 ceph-mon[220044]: pgmap v529: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 28680/170130 objects degraded (16.858%); 16320/170130 objects misplaced (9.593%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T05:56:38.451 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:38 smithi064 ceph-mon[220044]: failed to encode map e228 with expected crc 2023-10-09T05:56:38.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:38 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "46"}]': finished 2023-10-09T05:56:38.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:38 smithi064 ceph-mon[220044]: osdmap e228: 8 total, 8 up, 7 in 2023-10-09T05:56:38.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:38 smithi064 ceph-mon[220044]: failed to encode map e228 with expected crc 2023-10-09T05:56:38.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:38 smithi064 ceph-mon[220044]: failed to encode map e228 with expected crc 2023-10-09T05:56:38.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:38.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:38.558 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:38.592 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:38.609 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:38 smithi156 ceph-mon[195101]: pgmap v529: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 28680/170130 objects degraded (16.858%); 16320/170130 objects misplaced (9.593%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T05:56:38.609 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:38 smithi156 ceph-mon[195101]: failed to encode map e228 with expected crc 2023-10-09T05:56:38.609 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:38 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "46"}]': finished 2023-10-09T05:56:38.609 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:38 smithi156 ceph-mon[195101]: osdmap e228: 8 total, 8 up, 7 in 2023-10-09T05:56:38.609 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:38 smithi156 ceph-mon[195101]: failed to encode map e228 with expected crc 2023-10-09T05:56:38.609 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:38 smithi156 ceph-mon[195101]: failed to encode map e228 with expected crc 2023-10-09T05:56:38.615 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:38.668 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:38.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:38.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:38.772 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:38.802 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:38.868 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:38.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:38.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:38.973 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:39.032 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:39.226 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:56:39.232 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:39.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:39.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:39.339 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:39.353 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:39 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "46"}]: dispatch 2023-10-09T05:56:39.353 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:39 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "46"}]: dispatch 2023-10-09T05:56:39.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:39 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "46"}]: dispatch 2023-10-09T05:56:39.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:39 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "46"}]: dispatch 2023-10-09T05:56:39.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:39 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "46"}]: dispatch 2023-10-09T05:56:39.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:39 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "46"}]: dispatch 2023-10-09T05:56:39.496 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:39.569 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:39.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:39.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:39.678 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:39.707 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:39.756 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:39.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:39.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:39.779 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:56:39.810 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:56:39.863 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:39.890 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:39.986 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:39.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:39.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:40.091 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:40.133 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:40.194 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:40.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:40.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:40.259 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:40 smithi156 ceph-mon[195101]: pgmap v531: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 28574/168924 objects degraded (16.915%); 16231/168924 objects misplaced (9.608%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T05:56:40.259 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:40 smithi156 ceph-mon[195101]: failed to encode map e229 with expected crc 2023-10-09T05:56:40.261 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:40 smithi156 ceph-mon[195101]: failed to encode map e229 with expected crc 2023-10-09T05:56:40.261 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:40 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "46"}]': finished 2023-10-09T05:56:40.261 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:40 smithi156 ceph-mon[195101]: osdmap e229: 8 total, 8 up, 7 in 2023-10-09T05:56:40.261 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:40 smithi156 ceph-mon[195101]: failed to encode map e229 with expected crc 2023-10-09T05:56:40.261 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:40 smithi156 ceph-mon[195101]: failed to encode map e229 with expected crc 2023-10-09T05:56:40.262 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:40 smithi156 ceph-mon[195101]: failed to encode map e229 with expected crc 2023-10-09T05:56:40.298 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:40.356 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:40.418 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:40.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:40.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:40.437 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:40.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:40 smithi064 ceph-mon[212750]: pgmap v531: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 28574/168924 objects degraded (16.915%); 16231/168924 objects misplaced (9.608%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T05:56:40.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:40 smithi064 ceph-mon[212750]: failed to encode map e229 with expected crc 2023-10-09T05:56:40.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:40 smithi064 ceph-mon[212750]: failed to encode map e229 with expected crc 2023-10-09T05:56:40.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:40 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "46"}]': finished 2023-10-09T05:56:40.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:40 smithi064 ceph-mon[212750]: osdmap e229: 8 total, 8 up, 7 in 2023-10-09T05:56:40.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:40 smithi064 ceph-mon[212750]: failed to encode map e229 with expected crc 2023-10-09T05:56:40.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:40 smithi064 ceph-mon[212750]: failed to encode map e229 with expected crc 2023-10-09T05:56:40.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:40 smithi064 ceph-mon[212750]: failed to encode map e229 with expected crc 2023-10-09T05:56:40.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:40 smithi064 ceph-mon[220044]: pgmap v531: 129 pgs: 3 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 28574/168924 objects degraded (16.915%); 16231/168924 objects misplaced (9.608%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T05:56:40.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:40 smithi064 ceph-mon[220044]: failed to encode map e229 with expected crc 2023-10-09T05:56:40.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:40 smithi064 ceph-mon[220044]: failed to encode map e229 with expected crc 2023-10-09T05:56:40.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:40 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "46"}]': finished 2023-10-09T05:56:40.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:40 smithi064 ceph-mon[220044]: osdmap e229: 8 total, 8 up, 7 in 2023-10-09T05:56:40.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:40 smithi064 ceph-mon[220044]: failed to encode map e229 with expected crc 2023-10-09T05:56:40.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:40 smithi064 ceph-mon[220044]: failed to encode map e229 with expected crc 2023-10-09T05:56:40.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:40 smithi064 ceph-mon[220044]: failed to encode map e229 with expected crc 2023-10-09T05:56:40.521 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:40.557 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:40.674 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:40.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:40.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:40.780 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:40.808 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:40.854 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:40.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:40.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:40.958 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:40.987 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:41.160 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:41.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:41.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:41.252 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T05:56:41.265 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:41.310 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:41.366 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:41.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:41.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:41.395 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:41 smithi156 ceph-mon[195101]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.396 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:41 smithi156 ceph-mon[195101]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.396 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:41 smithi156 ceph-mon[195101]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.396 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:41 smithi156 ceph-mon[195101]: osdmap e230: 8 total, 8 up, 7 in 2023-10-09T05:56:41.396 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:41 smithi156 ceph-mon[195101]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.396 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:41 smithi156 ceph-mon[195101]: 6.4 starting backfill to osd.4 from (107'1426,228'7660] MIN to 228'7661 2023-10-09T05:56:41.396 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:41 smithi156 ceph-mon[195101]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.396 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:41 smithi156 ceph-mon[195101]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.396 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:41 smithi156 ceph-mon[195101]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.396 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:41 smithi156 ceph-mon[195101]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.396 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:41 smithi156 ceph-mon[195101]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.397 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:41 smithi156 ceph-mon[195101]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.397 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:41 smithi156 ceph-mon[195101]: 6.d continuing backfill to osd.2 from (108'2175,228'7750] MIN to 228'7750 2023-10-09T05:56:41.397 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:41 smithi156 ceph-mon[195101]: 6.6 continuing backfill to osd.5 from (106'613,228'7649] MIN to 228'7649 2023-10-09T05:56:41.397 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:41 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3678653389' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:56:41.397 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:41 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:56:41.397 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:41 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:41.397 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:41 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:56:41.397 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:41 smithi156 ceph-mon[195101]: osdmap e231: 8 total, 8 up, 7 in 2023-10-09T05:56:41.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[212750]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[212750]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[212750]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[212750]: osdmap e230: 8 total, 8 up, 7 in 2023-10-09T05:56:41.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[212750]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[212750]: 6.4 starting backfill to osd.4 from (107'1426,228'7660] MIN to 228'7661 2023-10-09T05:56:41.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[212750]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[212750]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[212750]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[212750]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[212750]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[212750]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[212750]: 6.d continuing backfill to osd.2 from (108'2175,228'7750] MIN to 228'7750 2023-10-09T05:56:41.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[212750]: 6.6 continuing backfill to osd.5 from (106'613,228'7649] MIN to 228'7649 2023-10-09T05:56:41.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3678653389' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:56:41.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:56:41.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:41.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:56:41.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[212750]: osdmap e231: 8 total, 8 up, 7 in 2023-10-09T05:56:41.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[220044]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[220044]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[220044]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[220044]: osdmap e230: 8 total, 8 up, 7 in 2023-10-09T05:56:41.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[220044]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[220044]: 6.4 starting backfill to osd.4 from (107'1426,228'7660] MIN to 228'7661 2023-10-09T05:56:41.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[220044]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[220044]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[220044]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[220044]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[220044]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[220044]: failed to encode map e230 with expected crc 2023-10-09T05:56:41.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[220044]: 6.d continuing backfill to osd.2 from (108'2175,228'7750] MIN to 228'7750 2023-10-09T05:56:41.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[220044]: 6.6 continuing backfill to osd.5 from (106'613,228'7649] MIN to 228'7649 2023-10-09T05:56:41.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3678653389' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:56:41.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:56:41.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:41.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:56:41.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:41 smithi064 ceph-mon[220044]: osdmap e231: 8 total, 8 up, 7 in 2023-10-09T05:56:41.473 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:41.502 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:41.552 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:41.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:41.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:41.634 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:56:41.634 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T05:56:41.634 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.634 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:139495 6.1 6:838b46fe:::benchmark_data_smithi064_284193_object42952:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e231)", 2023-10-09T05:56:41.634 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:56:41.300296+0000", 2023-10-09T05:56:41.634 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.33335957100000002, 2023-10-09T05:56:41.634 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.33338275499999998, 2023-10-09T05:56:41.634 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:41.634 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:56:41.635 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:41.635 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:41.635 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:41.635 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 139495 2023-10-09T05:56:41.635 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.635 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:41.635 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.635 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:41.635 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.300296+0000", 2023-10-09T05:56:41.635 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:41.636 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.636 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.636 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:41.636 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.300296+0000", 2023-10-09T05:56:41.636 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:41.636 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.636 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.636 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:41.637 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.300295+0000", 2023-10-09T05:56:41.637 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:56:41.637 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.637 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.637 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:41.637 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.300302+0000", 2023-10-09T05:56:41.637 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9800000000000001e-06 2023-10-09T05:56:41.637 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.638 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.638 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:41.638 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.300314+0000", 2023-10-09T05:56:41.638 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1477999999999999e-05 2023-10-09T05:56:41.638 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.638 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.638 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:41.639 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.300318+0000", 2023-10-09T05:56:41.639 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5480000000000003e-06 2023-10-09T05:56:41.639 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.639 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.639 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:41.639 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.300361+0000", 2023-10-09T05:56:41.639 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2327999999999999e-05 2023-10-09T05:56:41.639 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.640 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.640 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:41.640 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.300496+0000", 2023-10-09T05:56:41.640 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013526099999999999 2023-10-09T05:56:41.640 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.640 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.640 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1,7", 2023-10-09T05:56:41.640 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.300739+0000", 2023-10-09T05:56:41.641 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00024274499999999999 2023-10-09T05:56:41.641 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.641 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.641 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:41.641 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.301824+0000", 2023-10-09T05:56:41.641 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001084813 2023-10-09T05:56:41.641 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.641 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.642 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:41.642 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.302621+0000", 2023-10-09T05:56:41.642 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00079788500000000002 2023-10-09T05:56:41.642 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.642 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.642 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:41.642 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.302651+0000", 2023-10-09T05:56:41.642 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9686e-05 2023-10-09T05:56:41.642 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:41.643 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:41.643 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:41.643 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.643 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.643 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:139498 6.21 6:85a496ef:::benchmark_data_smithi064_284193_object42955:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e231)", 2023-10-09T05:56:41.643 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:56:41.307417+0000", 2023-10-09T05:56:41.643 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.32623941699999998, 2023-10-09T05:56:41.643 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.32632501800000002, 2023-10-09T05:56:41.643 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:41.643 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:56:41.643 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:41.644 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:41.644 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:41.644 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 139498 2023-10-09T05:56:41.644 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.644 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:41.644 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.644 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:41.644 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.307417+0000", 2023-10-09T05:56:41.644 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:41.644 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.645 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.645 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:41.645 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.307417+0000", 2023-10-09T05:56:41.645 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:41.645 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.645 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.645 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:41.645 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.307415+0000", 2023-10-09T05:56:41.645 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:41.645 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.646 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.646 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:41.646 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.307422+0000", 2023-10-09T05:56:41.646 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8340000000000001e-06 2023-10-09T05:56:41.646 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.646 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.646 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:41.647 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.307444+0000", 2023-10-09T05:56:41.647 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1951e-05 2023-10-09T05:56:41.647 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.647 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.647 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:41.647 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.307450+0000", 2023-10-09T05:56:41.647 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.959e-06 2023-10-09T05:56:41.647 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.647 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.647 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.307495+0000", 2023-10-09T05:56:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4999000000000001e-05 2023-10-09T05:56:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.307642+0000", 2023-10-09T05:56:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014772 2023-10-09T05:56:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1,7", 2023-10-09T05:56:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.307864+0000", 2023-10-09T05:56:41.648 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00022108800000000001 2023-10-09T05:56:41.649 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.649 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.649 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:41.649 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.308808+0000", 2023-10-09T05:56:41.649 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00094405900000000002 2023-10-09T05:56:41.649 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.649 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.649 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:41.649 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.309562+0000", 2023-10-09T05:56:41.649 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00075475900000000003 2023-10-09T05:56:41.649 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.649 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.650 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:41.650 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.309722+0000", 2023-10-09T05:56:41.650 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015941499999999999 2023-10-09T05:56:41.650 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:41.650 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:41.650 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:41.650 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.650 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.650 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:139518 6.21 6:84d2e324:::benchmark_data_smithi064_284193_object42975:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e231)", 2023-10-09T05:56:41.650 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:56:41.379821+0000", 2023-10-09T05:56:41.650 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.25383507799999999, 2023-10-09T05:56:41.650 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.25397521699999998, 2023-10-09T05:56:41.651 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:41.651 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:56:41.651 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:41.651 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:41.651 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:41.651 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 139518 2023-10-09T05:56:41.651 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.651 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:41.651 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.651 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:41.651 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.379821+0000", 2023-10-09T05:56:41.651 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:41.652 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.652 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.652 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:41.652 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.379821+0000", 2023-10-09T05:56:41.652 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:41.652 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.652 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.652 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:41.652 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.379819+0000", 2023-10-09T05:56:41.652 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T05:56:41.652 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.652 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.653 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:41.653 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.379827+0000", 2023-10-09T05:56:41.653 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7409999999999992e-06 2023-10-09T05:56:41.653 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.653 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.653 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:41.653 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.379845+0000", 2023-10-09T05:56:41.653 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8749e-05 2023-10-09T05:56:41.653 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.653 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.653 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:41.654 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.379851+0000", 2023-10-09T05:56:41.654 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9070000000000004e-06 2023-10-09T05:56:41.654 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.654 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.654 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:41.654 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.379906+0000", 2023-10-09T05:56:41.654 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5099000000000002e-05 2023-10-09T05:56:41.654 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.654 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.654 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:41.654 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.380057+0000", 2023-10-09T05:56:41.654 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015089899999999999 2023-10-09T05:56:41.655 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.655 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.655 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1,7", 2023-10-09T05:56:41.655 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.380394+0000", 2023-10-09T05:56:41.655 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00033689400000000002 2023-10-09T05:56:41.655 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.655 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.655 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:41.655 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.381523+0000", 2023-10-09T05:56:41.655 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001129012 2023-10-09T05:56:41.655 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.655 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.656 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:41.656 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.381928+0000", 2023-10-09T05:56:41.656 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00040511900000000001 2023-10-09T05:56:41.656 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:41.656 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:41.656 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:41.656 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.383094+0000", 2023-10-09T05:56:41.656 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0011651599999999999 2023-10-09T05:56:41.656 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:41.656 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:41.656 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:41.656 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:41.657 INFO:teuthology.orchestra.run.smithi156.stdout: ], 2023-10-09T05:56:41.657 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 3 2023-10-09T05:56:41.657 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:56:41.657 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:41.691 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:42.533 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:56:42.533 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:42 smithi156 ceph-mon[195101]: pgmap v534: 139 pgs: 3 peering, 4 remapped+peering, 8 unknown, 1 active+recovery_wait+degraded, 2 active+recovery_wait+degraded+remapped, 111 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 17849/149436 objects degraded (11.944%); 14811/149436 objects misplaced (9.911%) 2023-10-09T05:56:42.533 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:42 smithi156 ceph-mon[195101]: failed to encode map e231 with expected crc 2023-10-09T05:56:42.534 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:42 smithi156 ceph-mon[195101]: failed to encode map e231 with expected crc 2023-10-09T05:56:42.534 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:42 smithi156 ceph-mon[195101]: failed to encode map e231 with expected crc 2023-10-09T05:56:42.534 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:42 smithi156 ceph-mon[195101]: failed to encode map e231 with expected crc 2023-10-09T05:56:42.534 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:42 smithi156 ceph-mon[195101]: failed to encode map e231 with expected crc 2023-10-09T05:56:42.534 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:42 smithi156 ceph-mon[195101]: failed to encode map e231 with expected crc 2023-10-09T05:56:42.534 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:42 smithi156 ceph-mon[195101]: failed to encode map e231 with expected crc 2023-10-09T05:56:42.534 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:42 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 17849/149436 objects degraded (11.944%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:56:42.535 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:42 smithi156 ceph-mon[195101]: osdmap e232: 8 total, 8 up, 7 in 2023-10-09T05:56:42.594 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:42 smithi064 ceph-mon[212750]: pgmap v534: 139 pgs: 3 peering, 4 remapped+peering, 8 unknown, 1 active+recovery_wait+degraded, 2 active+recovery_wait+degraded+remapped, 111 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 17849/149436 objects degraded (11.944%); 14811/149436 objects misplaced (9.911%) 2023-10-09T05:56:42.594 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:42 smithi064 ceph-mon[212750]: failed to encode map e231 with expected crc 2023-10-09T05:56:42.594 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:42 smithi064 ceph-mon[212750]: failed to encode map e231 with expected crc 2023-10-09T05:56:42.594 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:42 smithi064 ceph-mon[212750]: failed to encode map e231 with expected crc 2023-10-09T05:56:42.594 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:42 smithi064 ceph-mon[212750]: failed to encode map e231 with expected crc 2023-10-09T05:56:42.594 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:42 smithi064 ceph-mon[212750]: failed to encode map e231 with expected crc 2023-10-09T05:56:42.594 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:42 smithi064 ceph-mon[212750]: failed to encode map e231 with expected crc 2023-10-09T05:56:42.595 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:42 smithi064 ceph-mon[212750]: failed to encode map e231 with expected crc 2023-10-09T05:56:42.595 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:42 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 17849/149436 objects degraded (11.944%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:56:42.595 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:42 smithi064 ceph-mon[212750]: osdmap e232: 8 total, 8 up, 7 in 2023-10-09T05:56:42.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:42 smithi064 ceph-mon[220044]: pgmap v534: 139 pgs: 3 peering, 4 remapped+peering, 8 unknown, 1 active+recovery_wait+degraded, 2 active+recovery_wait+degraded+remapped, 111 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 17849/149436 objects degraded (11.944%); 14811/149436 objects misplaced (9.911%) 2023-10-09T05:56:42.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:42 smithi064 ceph-mon[220044]: failed to encode map e231 with expected crc 2023-10-09T05:56:42.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:42 smithi064 ceph-mon[220044]: failed to encode map e231 with expected crc 2023-10-09T05:56:42.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:42 smithi064 ceph-mon[220044]: failed to encode map e231 with expected crc 2023-10-09T05:56:42.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:42 smithi064 ceph-mon[220044]: failed to encode map e231 with expected crc 2023-10-09T05:56:42.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:42 smithi064 ceph-mon[220044]: failed to encode map e231 with expected crc 2023-10-09T05:56:42.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:42 smithi064 ceph-mon[220044]: failed to encode map e231 with expected crc 2023-10-09T05:56:42.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:42 smithi064 ceph-mon[220044]: failed to encode map e231 with expected crc 2023-10-09T05:56:42.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:42 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 17849/149436 objects degraded (11.944%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:56:42.597 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:42 smithi064 ceph-mon[220044]: osdmap e232: 8 total, 8 up, 7 in 2023-10-09T05:56:42.599 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:42.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:42.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:42.707 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:56:42.745 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:43.075 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:56:43.388 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:43.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:43.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:43.451 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T05:56:43.451 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:56:43.451 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T05:56:43.452 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.000000 2023-10-09T05:56:43.452 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 0 2023-10-09T05:56:43.463 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[212750]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.463 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[212750]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.463 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[212750]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.463 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[212750]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.463 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[212750]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.463 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[212750]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.463 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[212750]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.464 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[212750]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.464 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[212750]: 6.24 starting backfill to osd.4 from (107'1426,228'7660] MIN to 228'7661 2023-10-09T05:56:43.464 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[212750]: 6.2d continuing backfill to osd.2 from (108'2175,228'7750] MIN to 228'7750 2023-10-09T05:56:43.464 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[212750]: 6.26 continuing backfill to osd.5 from (106'613,228'7649] MIN to 228'7649 2023-10-09T05:56:43.464 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[212750]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.464 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[212750]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.464 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[212750]: pgmap v537: 139 pgs: 3 peering, 4 remapped+peering, 7 unknown, 1 active+recovery_wait+degraded, 2 active+recovery_wait+degraded+remapped, 112 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 17779/148791 objects degraded (11.949%); 14778/148791 objects misplaced (9.932%) 2023-10-09T05:56:43.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[220044]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[220044]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[220044]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[220044]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[220044]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[220044]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[220044]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[220044]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[220044]: 6.24 starting backfill to osd.4 from (107'1426,228'7660] MIN to 228'7661 2023-10-09T05:56:43.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[220044]: 6.2d continuing backfill to osd.2 from (108'2175,228'7750] MIN to 228'7750 2023-10-09T05:56:43.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[220044]: 6.26 continuing backfill to osd.5 from (106'613,228'7649] MIN to 228'7649 2023-10-09T05:56:43.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[220044]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[220044]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.467 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:43 smithi064 ceph-mon[220044]: pgmap v537: 139 pgs: 3 peering, 4 remapped+peering, 7 unknown, 1 active+recovery_wait+degraded, 2 active+recovery_wait+degraded+remapped, 112 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 17779/148791 objects degraded (11.949%); 14778/148791 objects misplaced (9.932%) 2023-10-09T05:56:43.494 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:43.545 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:43 smithi156 ceph-mon[195101]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.545 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:43 smithi156 ceph-mon[195101]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.546 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:43 smithi156 ceph-mon[195101]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.546 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:43 smithi156 ceph-mon[195101]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.546 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:43 smithi156 ceph-mon[195101]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.546 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:43 smithi156 ceph-mon[195101]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.546 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:43 smithi156 ceph-mon[195101]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.546 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:43 smithi156 ceph-mon[195101]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.546 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:43 smithi156 ceph-mon[195101]: 6.24 starting backfill to osd.4 from (107'1426,228'7660] MIN to 228'7661 2023-10-09T05:56:43.546 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:43 smithi156 ceph-mon[195101]: 6.2d continuing backfill to osd.2 from (108'2175,228'7750] MIN to 228'7750 2023-10-09T05:56:43.546 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:43 smithi156 ceph-mon[195101]: 6.26 continuing backfill to osd.5 from (106'613,228'7649] MIN to 228'7649 2023-10-09T05:56:43.546 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:43 smithi156 ceph-mon[195101]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.547 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:43 smithi156 ceph-mon[195101]: failed to encode map e232 with expected crc 2023-10-09T05:56:43.547 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:43 smithi156 ceph-mon[195101]: pgmap v537: 139 pgs: 3 peering, 4 remapped+peering, 7 unknown, 1 active+recovery_wait+degraded, 2 active+recovery_wait+degraded+remapped, 112 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 17779/148791 objects degraded (11.949%); 14778/148791 objects misplaced (9.932%) 2023-10-09T05:56:43.565 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:43.695 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:43.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:43.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:43.806 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:43.871 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:43.948 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:43.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:43.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:43.981 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:56:44.025 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:56:44.048 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:44.054 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:44.062 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:56:44.100 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:56:44.107 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:44.199 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:44.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:44.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:44.302 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:44.329 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:56:44.338 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:44.361 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:56:44.407 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:56:44.436 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:56:44.443 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:56:44.444 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:56:44.444 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:56:44.444 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:56:44.444 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:56:44.444 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:56:44.444 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:56:44.444 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:56:44.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:56:44.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:44 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3551264387' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:44.472 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:44.474 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:56:44.474 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:44 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3551264387' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:44.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:44.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:44.578 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:44.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:56:44.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:44 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3551264387' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:44.621 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:45.278 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T05:56:45.279 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:56:45.280 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:45 smithi156 ceph-mon[195101]: pgmap v538: 139 pgs: 1 peering, 3 remapped+peering, 3 unknown, 1 active+recovery_wait+degraded, 2 active+recovery_wait+degraded+remapped, 116 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 142 op/s; 19014/157608 objects degraded (12.064%); 16808/157608 objects misplaced (10.664%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:56:45.280 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:45 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:45.486 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:56:45.530 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:45 smithi064 ceph-mon[212750]: pgmap v538: 139 pgs: 1 peering, 3 remapped+peering, 3 unknown, 1 active+recovery_wait+degraded, 2 active+recovery_wait+degraded+remapped, 116 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 142 op/s; 19014/157608 objects degraded (12.064%); 16808/157608 objects misplaced (10.664%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:56:45.530 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:45 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:45.531 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:45 smithi064 ceph-mon[220044]: pgmap v538: 139 pgs: 1 peering, 3 remapped+peering, 3 unknown, 1 active+recovery_wait+degraded, 2 active+recovery_wait+degraded+remapped, 116 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 142 op/s; 19014/157608 objects degraded (12.064%); 16808/157608 objects misplaced (10.664%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T05:56:45.531 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:45 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:45.533 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:56:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:56:45] "GET /metrics HTTP/1.1" 200 36467 "" "Prometheus/2.43.0" 2023-10-09T05:56:46.258 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:46.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:46.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:46.347 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:56:46.369 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:46.404 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:46.431 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:46.530 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:46.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:46.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:46.636 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:46.672 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:46.797 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:46.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:46.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:46.904 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:56:46.967 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:47.002 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:56:47.143 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:56:47.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:47.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:56:47.259 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:47.353 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:47.463 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:47.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:47.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:47.567 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:47.608 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:47.833 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:47 smithi156 ceph-mon[195101]: pgmap v539: 139 pgs: 5 active+recovery_wait+degraded, 4 active+recovery_wait+degraded+remapped, 116 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 74 MiB/s wr, 2.09k op/s; 25103/156219 objects degraded (16.069%); 15293/156219 objects misplaced (9.789%); 28 MiB/s, 455 objects/s recovering 2023-10-09T05:56:47.834 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:47 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 25103/156219 objects degraded (16.069%), 17 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:56:47.834 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:47 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3685183192' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-10-09T05:56:47.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:47 smithi064 ceph-mon[212750]: pgmap v539: 139 pgs: 5 active+recovery_wait+degraded, 4 active+recovery_wait+degraded+remapped, 116 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 74 MiB/s wr, 2.09k op/s; 25103/156219 objects degraded (16.069%); 15293/156219 objects misplaced (9.789%); 28 MiB/s, 455 objects/s recovering 2023-10-09T05:56:47.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:47 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 25103/156219 objects degraded (16.069%), 17 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:56:47.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:47 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3685183192' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-10-09T05:56:47.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:47 smithi064 ceph-mon[220044]: pgmap v539: 139 pgs: 5 active+recovery_wait+degraded, 4 active+recovery_wait+degraded+remapped, 116 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 74 MiB/s wr, 2.09k op/s; 25103/156219 objects degraded (16.069%); 15293/156219 objects misplaced (9.789%); 28 MiB/s, 455 objects/s recovering 2023-10-09T05:56:47.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:47 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 25103/156219 objects degraded (16.069%), 17 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:56:47.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:47 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3685183192' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-10-09T05:56:48.180 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:56:48.245 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.5 primary-affinity to 0 (802) 2023-10-09T05:56:48.353 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:48.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:48.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:48.461 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:48.505 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:48.670 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:48.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:48.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:48.775 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:48.806 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:49.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:49 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3685183192' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-10-09T05:56:49.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:49 smithi064 ceph-mon[220044]: osdmap e233: 8 total, 8 up, 7 in 2023-10-09T05:56:49.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:49 smithi064 ceph-mon[220044]: failed to encode map e233 with expected crc 2023-10-09T05:56:49.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:49 smithi064 ceph-mon[220044]: failed to encode map e233 with expected crc 2023-10-09T05:56:49.363 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:49.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:49.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:49.469 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:49.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:49 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3685183192' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-10-09T05:56:49.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:49 smithi064 ceph-mon[212750]: osdmap e233: 8 total, 8 up, 7 in 2023-10-09T05:56:49.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:49 smithi064 ceph-mon[212750]: failed to encode map e233 with expected crc 2023-10-09T05:56:49.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:49 smithi064 ceph-mon[212750]: failed to encode map e233 with expected crc 2023-10-09T05:56:49.471 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:49 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3685183192' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-10-09T05:56:49.471 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:49 smithi156 ceph-mon[195101]: osdmap e233: 8 total, 8 up, 7 in 2023-10-09T05:56:49.471 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:49 smithi156 ceph-mon[195101]: failed to encode map e233 with expected crc 2023-10-09T05:56:49.471 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:49 smithi156 ceph-mon[195101]: failed to encode map e233 with expected crc 2023-10-09T05:56:49.513 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:49.685 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:49.711 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T05:56:49.711 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:56:49.769 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:49.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:49.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:49.874 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:49.909 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:50.017 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:50.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:50.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:50.121 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:50.149 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:50.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[220044]: pgmap v541: 139 pgs: 5 active+recovery_wait+degraded, 4 active+recovery_wait+degraded+remapped, 116 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 64 MiB/s wr, 1.80k op/s; 24946/154989 objects degraded (16.095%); 15250/154989 objects misplaced (9.839%); 25 MiB/s, 394 objects/s recovering 2023-10-09T05:56:50.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[220044]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[220044]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[220044]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.223 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T05:56:50.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[212750]: pgmap v541: 139 pgs: 5 active+recovery_wait+degraded, 4 active+recovery_wait+degraded+remapped, 116 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 64 MiB/s wr, 1.80k op/s; 24946/154989 objects degraded (16.095%); 15250/154989 objects misplaced (9.839%); 25 MiB/s, 394 objects/s recovering 2023-10-09T05:56:50.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[212750]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[212750]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[212750]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[212750]: osdmap e234: 8 total, 8 up, 7 in 2023-10-09T05:56:50.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[212750]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[212750]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[212750]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[212750]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[212750]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[212750]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[212750]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[212750]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/424379441' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:50.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:50.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[220044]: osdmap e234: 8 total, 8 up, 7 in 2023-10-09T05:56:50.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[220044]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[220044]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[220044]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[220044]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[220044]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[220044]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[220044]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[220044]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/424379441' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:50.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:50 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:50.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:50 smithi156 ceph-mon[195101]: pgmap v541: 139 pgs: 5 active+recovery_wait+degraded, 4 active+recovery_wait+degraded+remapped, 116 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 64 MiB/s wr, 1.80k op/s; 24946/154989 objects degraded (16.095%); 15250/154989 objects misplaced (9.839%); 25 MiB/s, 394 objects/s recovering 2023-10-09T05:56:50.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:50 smithi156 ceph-mon[195101]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:50 smithi156 ceph-mon[195101]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:50 smithi156 ceph-mon[195101]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:50 smithi156 ceph-mon[195101]: osdmap e234: 8 total, 8 up, 7 in 2023-10-09T05:56:50.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:50 smithi156 ceph-mon[195101]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:50 smithi156 ceph-mon[195101]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:50 smithi156 ceph-mon[195101]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:50 smithi156 ceph-mon[195101]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:50 smithi156 ceph-mon[195101]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:50 smithi156 ceph-mon[195101]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:50 smithi156 ceph-mon[195101]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:50 smithi156 ceph-mon[195101]: failed to encode map e234 with expected crc 2023-10-09T05:56:50.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:50 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/424379441' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:50.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:50 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:56:50.649 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:50.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:50.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:50.754 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:50.788 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:50.859 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:56:51.368 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:56:51.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:51 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:51.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:51 smithi156 ceph-mon[195101]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:51 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:56:51.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:51 smithi156 ceph-mon[195101]: osdmap e235: 8 total, 8 up, 7 in 2023-10-09T05:56:51.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:51 smithi156 ceph-mon[195101]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:51 smithi156 ceph-mon[195101]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:51 smithi156 ceph-mon[195101]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:51 smithi156 ceph-mon[195101]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:51 smithi156 ceph-mon[195101]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:51 smithi156 ceph-mon[195101]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:51 smithi156 ceph-mon[195101]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:51 smithi156 ceph-mon[195101]: 6.d continuing backfill to osd.2 from (108'2175,232'7807] MIN to 232'7807 2023-10-09T05:56:51.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:51 smithi156 ceph-mon[195101]: 6.2d continuing backfill to osd.2 from (108'2175,232'7828] MIN to 232'7828 2023-10-09T05:56:51.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:51 smithi156 ceph-mon[195101]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:51 smithi156 ceph-mon[195101]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:51 smithi156 ceph-mon[195101]: 6.14 continuing backfill to osd.4 from (107'1426,232'7761] MIN to 232'7761 2023-10-09T05:56:51.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:51 smithi156 ceph-mon[195101]: 6.24 continuing backfill to osd.4 from (107'1426,232'7717] MIN to 232'7717 2023-10-09T05:56:51.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:51 smithi156 ceph-mon[195101]: 6.4 continuing backfill to osd.4 from (107'1426,232'7728] MIN to 232'7728 2023-10-09T05:56:51.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:51.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[220044]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:56:51.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[220044]: osdmap e235: 8 total, 8 up, 7 in 2023-10-09T05:56:51.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[220044]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[220044]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[220044]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[220044]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[220044]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[220044]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[220044]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[220044]: 6.d continuing backfill to osd.2 from (108'2175,232'7807] MIN to 232'7807 2023-10-09T05:56:51.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[220044]: 6.2d continuing backfill to osd.2 from (108'2175,232'7828] MIN to 232'7828 2023-10-09T05:56:51.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[220044]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[220044]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[220044]: 6.14 continuing backfill to osd.4 from (107'1426,232'7761] MIN to 232'7761 2023-10-09T05:56:51.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[220044]: 6.24 continuing backfill to osd.4 from (107'1426,232'7717] MIN to 232'7717 2023-10-09T05:56:51.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[220044]: 6.4 continuing backfill to osd.4 from (107'1426,232'7728] MIN to 232'7728 2023-10-09T05:56:51.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:56:51.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[212750]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:56:51.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[212750]: osdmap e235: 8 total, 8 up, 7 in 2023-10-09T05:56:51.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[212750]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[212750]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[212750]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[212750]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[212750]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[212750]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[212750]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[212750]: 6.d continuing backfill to osd.2 from (108'2175,232'7807] MIN to 232'7807 2023-10-09T05:56:51.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[212750]: 6.2d continuing backfill to osd.2 from (108'2175,232'7828] MIN to 232'7828 2023-10-09T05:56:51.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[212750]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[212750]: failed to encode map e235 with expected crc 2023-10-09T05:56:51.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[212750]: 6.14 continuing backfill to osd.4 from (107'1426,232'7761] MIN to 232'7761 2023-10-09T05:56:51.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[212750]: 6.24 continuing backfill to osd.4 from (107'1426,232'7717] MIN to 232'7717 2023-10-09T05:56:51.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:51 smithi064 ceph-mon[212750]: 6.4 continuing backfill to osd.4 from (107'1426,232'7728] MIN to 232'7728 2023-10-09T05:56:52.228 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:52.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:52.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:52.368 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:52.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:52 smithi156 ceph-mon[195101]: pgmap v544: 139 pgs: 3 remapped+peering, 4 active+recovery_wait+degraded, 4 active+recovery_wait+degraded+remapped, 117 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 170 B/s rd, 16 MiB/s wr, 980 op/s; 21956/148998 objects degraded (14.736%); 12741/148998 objects misplaced (8.551%); 33 MiB/s, 552 objects/s recovering 2023-10-09T05:56:52.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:52 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:56:52.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:52 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:56:52.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:52 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 21956/148998 objects degraded (14.736%), 15 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:56:52.396 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:52.539 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:52 smithi064 ceph-mon[212750]: pgmap v544: 139 pgs: 3 remapped+peering, 4 active+recovery_wait+degraded, 4 active+recovery_wait+degraded+remapped, 117 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 170 B/s rd, 16 MiB/s wr, 980 op/s; 21956/148998 objects degraded (14.736%); 12741/148998 objects misplaced (8.551%); 33 MiB/s, 552 objects/s recovering 2023-10-09T05:56:52.539 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:52 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:56:52.539 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:52 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:56:52.539 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:52 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 21956/148998 objects degraded (14.736%), 15 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:56:52.539 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:52 smithi064 ceph-mon[220044]: pgmap v544: 139 pgs: 3 remapped+peering, 4 active+recovery_wait+degraded, 4 active+recovery_wait+degraded+remapped, 117 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 170 B/s rd, 16 MiB/s wr, 980 op/s; 21956/148998 objects degraded (14.736%); 12741/148998 objects misplaced (8.551%); 33 MiB/s, 552 objects/s recovering 2023-10-09T05:56:52.539 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:52 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:56:52.539 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:52 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:56:52.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:52 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 21956/148998 objects degraded (14.736%), 15 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:56:52.627 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:52.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:52.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:52.730 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:52.760 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:52.816 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:56:52.841 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:56:52.854 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:52.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:52.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:52.868 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:56:52.890 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:56:52.913 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:56:52.933 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:56:52.959 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:52.990 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:53.088 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:53.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:53.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:53.176 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T05:56:53.177 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:56:53.191 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:53.219 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:53.338 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:56:53.360 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:56:53.361 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:56:53.361 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:56:53.361 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:56:53.362 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:56:53.362 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:56:53.362 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:56:53.362 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:56:53.362 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:56:53.423 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:53.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:53.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:53.528 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:53.554 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:53 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/249530474' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:53.557 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:53.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:53 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/249530474' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:53.566 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:53 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/249530474' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:56:53.604 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:53.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:53.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:53.708 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:53.735 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:53.817 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:53.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:53.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:53.875 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:56:53.921 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:53.948 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:53.995 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:54.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:54.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:54.102 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:54.130 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:54.180 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:54.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:54.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:54.262 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:56:54.268 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:56:54.284 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:54.345 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:54.413 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T05:56:54.413 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T05:56:54.474 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:54 smithi156 ceph-mon[195101]: pgmap v545: 139 pgs: 3 remapped+peering, 4 active+recovery_wait+degraded, 4 active+recovery_wait+degraded+remapped, 117 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 170 B/s rd, 0 B/s wr, 401 op/s; 21956/148998 objects degraded (14.736%); 12741/148998 objects misplaced (8.551%); 14 MiB/s, 244 objects/s recovering 2023-10-09T05:56:54.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:54 smithi064 ceph-mon[212750]: pgmap v545: 139 pgs: 3 remapped+peering, 4 active+recovery_wait+degraded, 4 active+recovery_wait+degraded+remapped, 117 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 170 B/s rd, 0 B/s wr, 401 op/s; 21956/148998 objects degraded (14.736%); 12741/148998 objects misplaced (8.551%); 14 MiB/s, 244 objects/s recovering 2023-10-09T05:56:54.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:54 smithi064 ceph-mon[220044]: pgmap v545: 139 pgs: 3 remapped+peering, 4 active+recovery_wait+degraded, 4 active+recovery_wait+degraded+remapped, 117 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 170 B/s rd, 0 B/s wr, 401 op/s; 21956/148998 objects degraded (14.736%); 12741/148998 objects misplaced (8.551%); 14 MiB/s, 244 objects/s recovering 2023-10-09T05:56:54.833 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:54.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:54.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:54.937 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:56:54.995 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:55.064 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:56:55.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:55.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:56:55.080 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:55.107 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:55.169 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:55.215 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:55.425 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:55.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:55.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:55.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:56:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:56:55] "GET /metrics HTTP/1.1" 200 36467 "" "Prometheus/2.43.0" 2023-10-09T05:56:55.533 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:56:55.613 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:55.823 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:56:55.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:55.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:56:55.927 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:55.969 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:56.061 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:56.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:56.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:56.141 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:56:56.141 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T05:56:56.142 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T05:56:56.142 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T05:56:56.142 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:56:56.180 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:56:56.224 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:56.343 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:56 smithi156 ceph-mon[195101]: pgmap v546: 139 pgs: 2 activating+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 4 active+recovery_wait+degraded+remapped, 117 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 160 B/s rd, 0 B/s wr, 385 op/s; 23030/147639 objects degraded (15.599%); 13236/147639 objects misplaced (8.965%); 13 MiB/s, 230 objects/s recovering 2023-10-09T05:56:56.343 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:56 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:56:56.343 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:56 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:56:56.368 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:56:56.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:56.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:56:56.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:56 smithi064 ceph-mon[212750]: pgmap v546: 139 pgs: 2 activating+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 4 active+recovery_wait+degraded+remapped, 117 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 160 B/s rd, 0 B/s wr, 385 op/s; 23030/147639 objects degraded (15.599%); 13236/147639 objects misplaced (8.965%); 13 MiB/s, 230 objects/s recovering 2023-10-09T05:56:56.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:56 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:56:56.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:56 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:56:56.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:56 smithi064 ceph-mon[220044]: pgmap v546: 139 pgs: 2 activating+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 4 active+recovery_wait+degraded+remapped, 117 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 160 B/s rd, 0 B/s wr, 385 op/s; 23030/147639 objects degraded (15.599%); 13236/147639 objects misplaced (8.965%); 13 MiB/s, 230 objects/s recovering 2023-10-09T05:56:56.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:56 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:56:56.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:56 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:56:56.472 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:56:56.524 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:56.974 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:56:57.332 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:56:57.457 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:56:57.459 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:57 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:57.460 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:57 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:57.460 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:57 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:56:57.460 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:57 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 23030/147639 objects degraded (15.599%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:56:57.460 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:57 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:57.460 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:57 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:57.460 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:57 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:56:57.460 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:57 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 23030/147639 objects degraded (15.599%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:56:57.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:57.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:56:57.562 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:57.595 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:57 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:57.595 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:57 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:57.596 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:57 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:56:57.596 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:57 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 23030/147639 objects degraded (15.599%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:56:57.599 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:57.664 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:57.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:57.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:57.770 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:57.809 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:57.849 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:56:57.871 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:57.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:57.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:57.977 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:58.018 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:58.331 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:58.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:58.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:58.436 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:56:58.468 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:58.475 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:58 smithi156 ceph-mon[195101]: pgmap v547: 139 pgs: 1 active+recovering+undersized+remapped, 2 activating+remapped, 4 active+recovery_wait+degraded, 4 active+recovery_wait+degraded+remapped, 118 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 127 B/s rd, 0 B/s wr, 329 op/s; 19196/147075 objects degraded (13.052%); 16352/147075 objects misplaced (11.118%); 17 MiB/s, 326 objects/s recovering 2023-10-09T05:56:58.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:58 smithi064 ceph-mon[212750]: pgmap v547: 139 pgs: 1 active+recovering+undersized+remapped, 2 activating+remapped, 4 active+recovery_wait+degraded, 4 active+recovery_wait+degraded+remapped, 118 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 127 B/s rd, 0 B/s wr, 329 op/s; 19196/147075 objects degraded (13.052%); 16352/147075 objects misplaced (11.118%); 17 MiB/s, 326 objects/s recovering 2023-10-09T05:56:58.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:58 smithi064 ceph-mon[220044]: pgmap v547: 139 pgs: 1 active+recovering+undersized+remapped, 2 activating+remapped, 4 active+recovery_wait+degraded, 4 active+recovery_wait+degraded+remapped, 118 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 127 B/s rd, 0 B/s wr, 329 op/s; 19196/147075 objects degraded (13.052%); 16352/147075 objects misplaced (11.118%); 17 MiB/s, 326 objects/s recovering 2023-10-09T05:56:58.878 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:56:58.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:58.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:56:58.958 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:58.985 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:59.018 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:59.224 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:56:59.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:59.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:56:59.333 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:56:59.394 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:59.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:56:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:56:59.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:56:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:56:59.487 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:56:59.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:56:59.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:56:59.600 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:56:59.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:56:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:56:59.652 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:56:59.963 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:56:59.964 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T05:56:59.964 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T05:56:59.964 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T05:56:59.964 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.964 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:45183 6.5 6:aac00114:::benchmark_data_smithi064_284193_object45182:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T05:56:59.964 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:13.801557+0000", 2023-10-09T05:56:59.964 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 346.15716637200001, 2023-10-09T05:56:59.964 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.33244016100000001, 2023-10-09T05:56:59.964 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:59.964 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:59.964 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:59.965 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:59.965 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:59.965 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 45183 2023-10-09T05:56:59.965 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.965 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:59.965 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.965 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:59.965 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801557+0000", 2023-10-09T05:56:59.965 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:59.965 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.965 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.965 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:59.966 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801557+0000", 2023-10-09T05:56:59.966 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:59.966 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.966 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.966 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:59.966 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801555+0000", 2023-10-09T05:56:59.966 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:59.966 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.966 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.966 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:59.967 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801742+0000", 2023-10-09T05:56:59.967 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000186954 2023-10-09T05:56:59.967 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.967 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.967 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:59.967 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801755+0000", 2023-10-09T05:56:59.967 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2575000000000001e-05 2023-10-09T05:56:59.967 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.967 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.967 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:59.967 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801762+0000", 2023-10-09T05:56:59.967 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.827e-06 2023-10-09T05:56:59.968 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.968 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.968 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:59.968 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801828+0000", 2023-10-09T05:56:59.968 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6237999999999998e-05 2023-10-09T05:56:59.968 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.968 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.968 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:59.968 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.801958+0000", 2023-10-09T05:56:59.968 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013046299999999999 2023-10-09T05:56:59.968 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.969 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.969 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:56:59.969 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.802104+0000", 2023-10-09T05:56:59.969 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000145465 2023-10-09T05:56:59.969 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.969 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.969 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:59.969 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.804037+0000", 2023-10-09T05:56:59.969 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001933113 2023-10-09T05:56:59.969 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.969 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.970 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:59.970 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.805890+0000", 2023-10-09T05:56:59.970 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00185311 2023-10-09T05:56:59.970 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.970 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.970 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:59.970 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:14.133916+0000", 2023-10-09T05:56:59.970 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.32802647899999998 2023-10-09T05:56:59.970 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.970 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.970 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:59.971 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:14.133958+0000", 2023-10-09T05:56:59.971 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1396999999999997e-05 2023-10-09T05:56:59.971 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.971 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.971 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:59.971 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:14.133997+0000", 2023-10-09T05:56:59.971 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9158e-05 2023-10-09T05:56:59.971 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:59.971 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:59.971 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:59.971 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.971 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.972 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:45190 6.8 6:1669d52f:::benchmark_data_smithi064_284193_object45189:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T05:56:59.972 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:13.804266+0000", 2023-10-09T05:56:59.972 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 346.15445725900003, 2023-10-09T05:56:59.972 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.32912121900000002, 2023-10-09T05:56:59.972 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:59.972 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:59.972 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:59.972 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:59.972 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:59.972 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 45190 2023-10-09T05:56:59.972 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.973 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:59.973 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.973 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:59.973 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.804266+0000", 2023-10-09T05:56:59.973 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:59.973 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.973 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.973 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:59.973 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.804266+0000", 2023-10-09T05:56:59.973 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:59.973 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.973 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.974 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:59.974 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.804264+0000", 2023-10-09T05:56:59.974 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:59.974 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.974 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.974 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:59.974 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.804311+0000", 2023-10-09T05:56:59.974 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6177999999999997e-05 2023-10-09T05:56:59.974 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.974 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.974 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:59.975 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.804328+0000", 2023-10-09T05:56:59.975 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6969e-05 2023-10-09T05:56:59.975 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.975 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.975 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:59.975 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.804334+0000", 2023-10-09T05:56:59.975 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1580000000000002e-06 2023-10-09T05:56:59.975 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.975 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.975 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:59.975 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.805072+0000", 2023-10-09T05:56:59.976 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00073821900000000003 2023-10-09T05:56:59.976 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.976 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.976 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:59.976 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.805186+0000", 2023-10-09T05:56:59.976 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011408099999999999 2023-10-09T05:56:59.976 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.976 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.976 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:56:59.976 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.805312+0000", 2023-10-09T05:56:59.976 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0001259 2023-10-09T05:56:59.976 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.977 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.977 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:59.977 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.806929+0000", 2023-10-09T05:56:59.977 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0016169839999999999 2023-10-09T05:56:59.977 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.977 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.977 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:59.977 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:13.808641+0000", 2023-10-09T05:56:59.977 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0017123469999999999 2023-10-09T05:56:59.977 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.977 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.978 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:59.978 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:14.133307+0000", 2023-10-09T05:56:59.978 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.32466545299999999 2023-10-09T05:56:59.978 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.978 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.978 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:59.978 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:14.133342+0000", 2023-10-09T05:56:59.978 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5429999999999998e-05 2023-10-09T05:56:59.978 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.978 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.978 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:59.979 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:14.133387+0000", 2023-10-09T05:56:59.979 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5037000000000003e-05 2023-10-09T05:56:59.979 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:59.979 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:59.979 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:59.979 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.979 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.979 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64016 6.9 6:964728b5:::benchmark_data_smithi064_284193_object64015:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:56:59.979 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.109619+0000", 2023-10-09T05:56:59.979 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 326.84910416600002, 2023-10-09T05:56:59.979 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0912698940000001, 2023-10-09T05:56:59.979 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:59.980 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:59.980 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:59.980 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:59.980 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:59.980 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64016 2023-10-09T05:56:59.980 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.980 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:59.980 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.980 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:59.980 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.109619+0000", 2023-10-09T05:56:59.980 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:59.981 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.981 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.981 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:59.981 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.109619+0000", 2023-10-09T05:56:59.981 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:59.981 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.981 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.981 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:59.981 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.109619+0000", 2023-10-09T05:56:59.981 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:56:59.981 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.982 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.982 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:59.982 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.109650+0000", 2023-10-09T05:56:59.982 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1087999999999998e-05 2023-10-09T05:56:59.982 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.982 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.982 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:59.982 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.109657+0000", 2023-10-09T05:56:59.982 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4290000000000004e-06 2023-10-09T05:56:59.982 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.982 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.983 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:59.983 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.109662+0000", 2023-10-09T05:56:59.983 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4510000000000002e-06 2023-10-09T05:56:59.983 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.983 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.983 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:59.983 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.115072+0000", 2023-10-09T05:56:59.983 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0054107180000000001 2023-10-09T05:56:59.983 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.983 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.983 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:59.983 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.115080+0000", 2023-10-09T05:56:59.984 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3710000000000002e-06 2023-10-09T05:56:59.984 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.984 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.984 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:59.984 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.189454+0000", 2023-10-09T05:56:59.984 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.074374384 2023-10-09T05:56:59.984 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.984 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.984 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:59.984 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.190617+0000", 2023-10-09T05:56:59.984 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0011633150000000001 2023-10-09T05:56:59.985 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.985 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.985 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2", 2023-10-09T05:56:59.985 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.190779+0000", 2023-10-09T05:56:59.985 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00016118799999999999 2023-10-09T05:56:59.985 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.985 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.985 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:59.985 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.198531+0000", 2023-10-09T05:56:59.985 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0077525600000000004 2023-10-09T05:56:59.985 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.985 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.986 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:59.986 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200740+0000", 2023-10-09T05:56:59.986 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002208785 2023-10-09T05:56:59.986 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.986 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.986 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:59.986 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200795+0000", 2023-10-09T05:56:59.986 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5225000000000001e-05 2023-10-09T05:56:59.986 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.986 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.986 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:59.987 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200889+0000", 2023-10-09T05:56:59.987 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3837000000000003e-05 2023-10-09T05:56:59.987 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:59.987 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:59.987 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:59.987 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.987 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.987 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64021 6.9 6:9b197d79:::benchmark_data_smithi064_284193_object64020:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:56:59.987 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.123178+0000", 2023-10-09T05:56:59.987 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 326.83554538300001, 2023-10-09T05:56:59.987 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0778894430000001, 2023-10-09T05:56:59.988 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:59.988 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:59.988 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:59.988 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:59.988 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:59.988 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64021 2023-10-09T05:56:59.988 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.988 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:59.988 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.988 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:59.988 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.123178+0000", 2023-10-09T05:56:59.988 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:59.989 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.989 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.989 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:59.989 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.123178+0000", 2023-10-09T05:56:59.989 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:59.989 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.989 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.989 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:59.989 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.123177+0000", 2023-10-09T05:56:59.989 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:59.989 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.989 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.990 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:59.990 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.123212+0000", 2023-10-09T05:56:59.990 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5641e-05 2023-10-09T05:56:59.990 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.990 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.990 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:59.990 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.123221+0000", 2023-10-09T05:56:59.990 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0909999999999999e-06 2023-10-09T05:56:59.990 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.990 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.990 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:59.991 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.123227+0000", 2023-10-09T05:56:59.991 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4469999999999997e-06 2023-10-09T05:56:59.991 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.991 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.991 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:59.991 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.124899+0000", 2023-10-09T05:56:59.991 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001671797 2023-10-09T05:56:59.991 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.991 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.991 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:59.991 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.124905+0000", 2023-10-09T05:56:59.992 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8650000000000003e-06 2023-10-09T05:56:59.992 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.992 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.992 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:59.992 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.192517+0000", 2023-10-09T05:56:59.992 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0676111340000001 2023-10-09T05:56:59.992 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.992 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.992 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:56:59.992 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.192607+0000", 2023-10-09T05:56:59.992 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0493000000000002e-05 2023-10-09T05:56:59.992 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.993 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.993 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2", 2023-10-09T05:56:59.993 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.192721+0000", 2023-10-09T05:56:59.993 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000114003 2023-10-09T05:56:59.993 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.993 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.993 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:56:59.993 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200480+0000", 2023-10-09T05:56:59.993 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0077586089999999996 2023-10-09T05:56:59.993 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.993 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.994 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:56:59.994 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200926+0000", 2023-10-09T05:56:59.994 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00044662899999999998 2023-10-09T05:56:59.994 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.994 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.994 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:56:59.994 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200976+0000", 2023-10-09T05:56:59.994 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9607999999999997e-05 2023-10-09T05:56:59.994 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.994 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.994 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:56:59.995 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.201067+0000", 2023-10-09T05:56:59.995 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1414000000000003e-05 2023-10-09T05:56:59.995 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:59.995 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:56:59.995 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:56:59.995 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.995 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.995 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64026 6.9 6:95eb16a1:::benchmark_data_smithi064_284193_object64025:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:56:59.995 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.131052+0000", 2023-10-09T05:56:59.995 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 326.82767092099999, 2023-10-09T05:56:59.995 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0701834699999999, 2023-10-09T05:56:59.995 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:56:59.996 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:56:59.996 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:56:59.996 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:56:59.996 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:56:59.996 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64026 2023-10-09T05:56:59.996 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.996 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:56:59.996 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.996 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:56:59.996 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.131052+0000", 2023-10-09T05:56:59.996 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:59.997 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.997 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.997 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:56:59.997 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.131052+0000", 2023-10-09T05:56:59.997 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:56:59.997 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.997 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.997 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:56:59.997 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.131051+0000", 2023-10-09T05:56:59.997 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:56:59.997 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.997 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.998 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:56:59.998 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.131088+0000", 2023-10-09T05:56:59.998 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6916999999999999e-05 2023-10-09T05:56:59.998 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.998 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.998 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:56:59.998 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.131101+0000", 2023-10-09T05:56:59.998 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2877999999999999e-05 2023-10-09T05:56:59.998 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.998 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.998 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:56:59.999 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.131107+0000", 2023-10-09T05:56:59.999 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5899999999999996e-06 2023-10-09T05:56:59.999 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.999 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.999 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:56:59.999 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.131156+0000", 2023-10-09T05:56:59.999 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8696000000000003e-05 2023-10-09T05:56:59.999 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:56:59.999 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:56:59.999 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:56:59.999 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.131164+0000", 2023-10-09T05:56:59.999 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9340000000000006e-06 2023-10-09T05:57:00.000 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.000 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.000 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.000 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.194260+0000", 2023-10-09T05:57:00.000 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.063095543 2023-10-09T05:57:00.000 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.000 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.000 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:57:00.000 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.194404+0000", 2023-10-09T05:57:00.000 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000144058 2023-10-09T05:57:00.000 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.001 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.001 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2", 2023-10-09T05:57:00.001 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.194588+0000", 2023-10-09T05:57:00.001 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00018420399999999999 2023-10-09T05:57:00.001 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.001 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.001 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:57:00.001 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200488+0000", 2023-10-09T05:57:00.001 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0059001640000000003 2023-10-09T05:57:00.001 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.001 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.001 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.002 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.201102+0000", 2023-10-09T05:57:00.002 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00061416900000000002 2023-10-09T05:57:00.002 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.002 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.002 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:57:00.002 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.201150+0000", 2023-10-09T05:57:00.002 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7922e-05 2023-10-09T05:57:00.002 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.002 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.002 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:57:00.002 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.201236+0000", 2023-10-09T05:57:00.002 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.5679000000000004e-05 2023-10-09T05:57:00.003 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.003 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:57:00.003 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.003 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.003 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.003 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64027 6.9 6:98f248d6:::benchmark_data_smithi064_284193_object64026:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:57:00.003 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.133112+0000", 2023-10-09T05:57:00.003 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 326.82561161500001, 2023-10-09T05:57:00.003 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0717281329999999, 2023-10-09T05:57:00.003 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:57:00.003 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:00.004 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:57:00.004 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:57:00.004 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:00.004 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64027 2023-10-09T05:57:00.004 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.004 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:57:00.004 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.004 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:57:00.004 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133112+0000", 2023-10-09T05:57:00.004 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.004 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.004 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.005 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:57:00.005 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133112+0000", 2023-10-09T05:57:00.005 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.005 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.005 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.005 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:57:00.005 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133110+0000", 2023-10-09T05:57:00.005 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:57:00.005 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.005 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.005 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:57:00.006 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133165+0000", 2023-10-09T05:57:00.006 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4601999999999997e-05 2023-10-09T05:57:00.006 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.006 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.006 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:57:00.006 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133174+0000", 2023-10-09T05:57:00.006 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6899999999999998e-06 2023-10-09T05:57:00.006 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.006 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.006 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:57:00.006 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133179+0000", 2023-10-09T05:57:00.006 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9509999999999997e-06 2023-10-09T05:57:00.007 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.007 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.007 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.007 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133222+0000", 2023-10-09T05:57:00.007 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3266000000000002e-05 2023-10-09T05:57:00.007 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.007 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.007 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:57:00.007 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133229+0000", 2023-10-09T05:57:00.007 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.486e-06 2023-10-09T05:57:00.007 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.008 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.008 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.008 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.196460+0000", 2023-10-09T05:57:00.008 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0632303270000001 2023-10-09T05:57:00.008 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.008 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.008 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:57:00.008 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.196575+0000", 2023-10-09T05:57:00.008 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000114918 2023-10-09T05:57:00.008 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.008 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.008 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2", 2023-10-09T05:57:00.009 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.196709+0000", 2023-10-09T05:57:00.009 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013413100000000001 2023-10-09T05:57:00.009 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.009 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.009 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.009 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.203829+0000", 2023-10-09T05:57:00.009 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0071203179999999996 2023-10-09T05:57:00.009 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.009 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.009 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:57:00.009 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204716+0000", 2023-10-09T05:57:00.009 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00088727100000000002 2023-10-09T05:57:00.010 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.010 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.010 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:57:00.010 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204766+0000", 2023-10-09T05:57:00.010 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9855999999999998e-05 2023-10-09T05:57:00.010 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.010 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.010 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:57:00.010 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204840+0000", 2023-10-09T05:57:00.010 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3491999999999994e-05 2023-10-09T05:57:00.010 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.011 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:57:00.011 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.011 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.011 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.011 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64047 6.9 6:98194c7b:::benchmark_data_smithi064_284193_object64046:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:57:00.011 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.155174+0000", 2023-10-09T05:57:00.011 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 326.80354951499999, 2023-10-09T05:57:00.011 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0497767680000001, 2023-10-09T05:57:00.011 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:57:00.011 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:00.011 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:57:00.011 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:57:00.012 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:00.012 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64047 2023-10-09T05:57:00.012 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.012 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:57:00.012 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.012 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:57:00.012 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.155174+0000", 2023-10-09T05:57:00.012 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.012 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.012 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.012 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:57:00.012 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.155174+0000", 2023-10-09T05:57:00.013 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.013 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.013 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.013 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:57:00.013 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.155172+0000", 2023-10-09T05:57:00.013 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:57:00.013 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.013 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.013 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:57:00.013 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.155288+0000", 2023-10-09T05:57:00.013 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011592299999999999 2023-10-09T05:57:00.014 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.014 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.014 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:57:00.014 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.155304+0000", 2023-10-09T05:57:00.014 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6515999999999999e-05 2023-10-09T05:57:00.014 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.014 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.014 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:57:00.014 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.155311+0000", 2023-10-09T05:57:00.014 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5470000000000002e-06 2023-10-09T05:57:00.014 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.015 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.015 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.015 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.155364+0000", 2023-10-09T05:57:00.015 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3075000000000003e-05 2023-10-09T05:57:00.015 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.015 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.015 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:57:00.015 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.155375+0000", 2023-10-09T05:57:00.015 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0631e-05 2023-10-09T05:57:00.015 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.015 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.015 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.016 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.198594+0000", 2023-10-09T05:57:00.016 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0432188309999999 2023-10-09T05:57:00.016 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.016 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.016 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:57:00.016 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.198721+0000", 2023-10-09T05:57:00.016 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000127398 2023-10-09T05:57:00.016 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.016 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.016 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2", 2023-10-09T05:57:00.016 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.198812+0000", 2023-10-09T05:57:00.016 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1410999999999996e-05 2023-10-09T05:57:00.017 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.017 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.017 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.017 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.203873+0000", 2023-10-09T05:57:00.017 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0050606719999999996 2023-10-09T05:57:00.017 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.017 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.017 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:57:00.017 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204849+0000", 2023-10-09T05:57:00.017 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00097625800000000003 2023-10-09T05:57:00.017 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.017 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.018 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:57:00.018 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204887+0000", 2023-10-09T05:57:00.018 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7339999999999998e-05 2023-10-09T05:57:00.018 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.018 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.018 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:57:00.018 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204950+0000", 2023-10-09T05:57:00.018 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3805000000000003e-05 2023-10-09T05:57:00.018 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.018 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:57:00.018 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.018 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.019 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.019 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:77504 6.5 6:af5f1d5a:::benchmark_data_smithi064_284193_object77503:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:57:00.019 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:49.602465+0000", 2023-10-09T05:57:00.019 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 310.35625840199998, 2023-10-09T05:57:00.019 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.36206133400000001, 2023-10-09T05:57:00.019 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:57:00.019 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:00.019 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:57:00.019 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:57:00.019 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:00.019 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 77504 2023-10-09T05:57:00.019 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.020 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:57:00.020 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.020 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:57:00.020 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602465+0000", 2023-10-09T05:57:00.020 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.020 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.020 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.020 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:57:00.020 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602465+0000", 2023-10-09T05:57:00.020 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.020 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.020 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.021 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:57:00.021 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602463+0000", 2023-10-09T05:57:00.021 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:57:00.021 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.021 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.021 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:57:00.021 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602503+0000", 2023-10-09T05:57:00.021 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0176000000000003e-05 2023-10-09T05:57:00.021 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.021 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.021 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:57:00.022 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602519+0000", 2023-10-09T05:57:00.022 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5979999999999999e-05 2023-10-09T05:57:00.022 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.022 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.022 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:57:00.022 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602534+0000", 2023-10-09T05:57:00.022 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4303e-05 2023-10-09T05:57:00.022 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.022 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.022 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.022 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602595+0000", 2023-10-09T05:57:00.023 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1381000000000001e-05 2023-10-09T05:57:00.023 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.023 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.023 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:57:00.023 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602714+0000", 2023-10-09T05:57:00.023 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000119098 2023-10-09T05:57:00.023 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.023 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.023 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:57:00.023 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.602851+0000", 2023-10-09T05:57:00.023 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000137093 2023-10-09T05:57:00.023 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.024 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.024 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:57:00.024 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.605415+0000", 2023-10-09T05:57:00.024 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002564055 2023-10-09T05:57:00.024 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.024 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.024 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.024 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.607058+0000", 2023-10-09T05:57:00.024 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0016429680000000001 2023-10-09T05:57:00.024 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.024 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.024 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.025 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964454+0000", 2023-10-09T05:57:00.025 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.35739558999999999 2023-10-09T05:57:00.025 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.025 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.025 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:57:00.025 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964494+0000", 2023-10-09T05:57:00.025 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0250000000000003e-05 2023-10-09T05:57:00.025 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.025 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.025 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:57:00.025 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964526+0000", 2023-10-09T05:57:00.026 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1959999999999999e-05 2023-10-09T05:57:00.026 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.026 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:57:00.026 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.026 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.026 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.026 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:77509 6.5 6:ae2c471a:::benchmark_data_smithi064_284193_object77508:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:57:00.026 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:49.610882+0000", 2023-10-09T05:57:00.026 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 310.347841163, 2023-10-09T05:57:00.026 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.353839925, 2023-10-09T05:57:00.026 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:57:00.027 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:00.027 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:57:00.027 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:57:00.027 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:00.027 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 77509 2023-10-09T05:57:00.027 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.027 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:57:00.027 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.027 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:57:00.027 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.610882+0000", 2023-10-09T05:57:00.027 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.027 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.028 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.028 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:57:00.028 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.610882+0000", 2023-10-09T05:57:00.028 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.028 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.028 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.028 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:57:00.028 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.610881+0000", 2023-10-09T05:57:00.028 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:57:00.028 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.028 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.029 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:57:00.029 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.610960+0000", 2023-10-09T05:57:00.029 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9631999999999999e-05 2023-10-09T05:57:00.029 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.029 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.029 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:57:00.029 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.610979+0000", 2023-10-09T05:57:00.029 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9011000000000001e-05 2023-10-09T05:57:00.029 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.029 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.029 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:57:00.030 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.610987+0000", 2023-10-09T05:57:00.030 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3289999999999996e-06 2023-10-09T05:57:00.030 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.030 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.030 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.030 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.611048+0000", 2023-10-09T05:57:00.030 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0708999999999998e-05 2023-10-09T05:57:00.030 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.030 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.030 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:57:00.030 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.611203+0000", 2023-10-09T05:57:00.031 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015526900000000001 2023-10-09T05:57:00.031 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.031 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.031 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:57:00.031 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.611355+0000", 2023-10-09T05:57:00.031 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015201200000000001 2023-10-09T05:57:00.031 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.031 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.031 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:57:00.031 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.613876+0000", 2023-10-09T05:57:00.031 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0025209579999999998 2023-10-09T05:57:00.032 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.032 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.032 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.032 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.615859+0000", 2023-10-09T05:57:00.032 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001983744 2023-10-09T05:57:00.032 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.032 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.032 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.032 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964635+0000", 2023-10-09T05:57:00.032 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.34877526800000003 2023-10-09T05:57:00.032 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.033 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.033 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:57:00.033 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964685+0000", 2023-10-09T05:57:00.033 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0586e-05 2023-10-09T05:57:00.033 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.033 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.033 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:57:00.033 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964722+0000", 2023-10-09T05:57:00.033 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6636e-05 2023-10-09T05:57:00.033 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.033 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:57:00.034 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.034 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.034 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.034 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:77517 6.5 6:a52e5624:::benchmark_data_smithi064_284193_object77516:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:57:00.034 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:49.620459+0000", 2023-10-09T05:57:00.034 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 310.33826436499999, 2023-10-09T05:57:00.034 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.34437174100000001, 2023-10-09T05:57:00.034 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:57:00.034 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:00.034 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:57:00.034 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:57:00.035 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:00.035 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 77517 2023-10-09T05:57:00.035 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.035 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:57:00.035 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.035 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:57:00.035 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620459+0000", 2023-10-09T05:57:00.035 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.035 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.035 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.035 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:57:00.035 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620459+0000", 2023-10-09T05:57:00.036 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.036 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.036 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.036 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:57:00.036 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620458+0000", 2023-10-09T05:57:00.036 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:57:00.036 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.036 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.036 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:57:00.036 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620525+0000", 2023-10-09T05:57:00.036 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7404999999999996e-05 2023-10-09T05:57:00.036 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.036 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.037 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:57:00.037 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620539+0000", 2023-10-09T05:57:00.037 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3997e-05 2023-10-09T05:57:00.037 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.037 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.037 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:57:00.037 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620545+0000", 2023-10-09T05:57:00.037 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7189999999999998e-06 2023-10-09T05:57:00.037 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.037 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.037 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.038 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620612+0000", 2023-10-09T05:57:00.038 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7325999999999999e-05 2023-10-09T05:57:00.038 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.038 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.038 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:57:00.038 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620750+0000", 2023-10-09T05:57:00.038 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000137059 2023-10-09T05:57:00.038 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.038 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.038 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:57:00.038 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.620901+0000", 2023-10-09T05:57:00.039 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015133200000000001 2023-10-09T05:57:00.039 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.039 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.039 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:57:00.039 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.623123+0000", 2023-10-09T05:57:00.039 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0022225970000000002 2023-10-09T05:57:00.039 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.039 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.039 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.039 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.623990+0000", 2023-10-09T05:57:00.039 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00086624800000000002 2023-10-09T05:57:00.040 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.040 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.040 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.040 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964757+0000", 2023-10-09T05:57:00.040 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.34076741799999999 2023-10-09T05:57:00.040 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.040 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.040 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:57:00.040 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964800+0000", 2023-10-09T05:57:00.040 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2741000000000003e-05 2023-10-09T05:57:00.040 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.040 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.041 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:57:00.041 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.964831+0000", 2023-10-09T05:57:00.041 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0738999999999998e-05 2023-10-09T05:57:00.041 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.041 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:57:00.041 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.041 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.041 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.041 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:77534 6.3 6:c39d213f:::benchmark_data_smithi064_284193_object77533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:57:00.041 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:49.644389+0000", 2023-10-09T05:57:00.041 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 310.314334535, 2023-10-09T05:57:00.041 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.33628388399999998, 2023-10-09T05:57:00.042 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:57:00.042 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:00.042 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:57:00.042 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:57:00.042 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:00.042 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 77534 2023-10-09T05:57:00.042 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.042 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:57:00.042 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.042 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:57:00.042 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644389+0000", 2023-10-09T05:57:00.043 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.043 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.043 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.043 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:57:00.043 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644389+0000", 2023-10-09T05:57:00.043 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.043 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.043 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.043 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:57:00.043 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644388+0000", 2023-10-09T05:57:00.043 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:00.044 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.044 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.044 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:57:00.044 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644424+0000", 2023-10-09T05:57:00.044 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5824999999999997e-05 2023-10-09T05:57:00.044 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.044 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.044 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:57:00.044 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644434+0000", 2023-10-09T05:57:00.044 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0461e-05 2023-10-09T05:57:00.044 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.044 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.045 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:57:00.045 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644440+0000", 2023-10-09T05:57:00.045 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1100000000000002e-06 2023-10-09T05:57:00.045 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.045 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.045 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.045 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644481+0000", 2023-10-09T05:57:00.045 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1902e-05 2023-10-09T05:57:00.045 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.045 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.045 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:57:00.046 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644613+0000", 2023-10-09T05:57:00.046 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000131815 2023-10-09T05:57:00.046 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.046 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.046 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:57:00.046 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644762+0000", 2023-10-09T05:57:00.046 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000148777 2023-10-09T05:57:00.046 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.046 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.046 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:57:00.047 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.649170+0000", 2023-10-09T05:57:00.047 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0044076310000000004 2023-10-09T05:57:00.047 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.047 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.047 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.047 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.650165+0000", 2023-10-09T05:57:00.047 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00099568399999999994 2023-10-09T05:57:00.047 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.047 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.047 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.047 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.980566+0000", 2023-10-09T05:57:00.047 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.33040050599999998 2023-10-09T05:57:00.048 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.048 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.048 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:57:00.048 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.980621+0000", 2023-10-09T05:57:00.048 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5392000000000001e-05 2023-10-09T05:57:00.048 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.048 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.048 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:57:00.048 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.980673+0000", 2023-10-09T05:57:00.048 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1316999999999997e-05 2023-10-09T05:57:00.048 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.049 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:57:00.049 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.049 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.049 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.049 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:77535 6.3 6:c58d58f2:::benchmark_data_smithi064_284193_object77534:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:57:00.049 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:49.644455+0000", 2023-10-09T05:57:00.049 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 310.31426787100003, 2023-10-09T05:57:00.049 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.33631617699999999, 2023-10-09T05:57:00.049 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:57:00.049 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:00.049 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:57:00.050 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:57:00.050 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:00.050 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 77535 2023-10-09T05:57:00.050 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.050 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:57:00.050 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.050 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:57:00.050 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644455+0000", 2023-10-09T05:57:00.050 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.050 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.050 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.051 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:57:00.051 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644455+0000", 2023-10-09T05:57:00.051 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.051 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.051 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.051 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:57:00.051 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644455+0000", 2023-10-09T05:57:00.051 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:00.051 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.051 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.051 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:57:00.051 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644491+0000", 2023-10-09T05:57:00.052 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6570999999999999e-05 2023-10-09T05:57:00.052 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.052 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.052 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:57:00.052 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644507+0000", 2023-10-09T05:57:00.052 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5880999999999999e-05 2023-10-09T05:57:00.052 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.052 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.052 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:57:00.052 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.644512+0000", 2023-10-09T05:57:00.052 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0880000000000002e-06 2023-10-09T05:57:00.052 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.053 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.053 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.053 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.647155+0000", 2023-10-09T05:57:00.053 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0026430009999999999 2023-10-09T05:57:00.053 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.053 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.053 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:57:00.053 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.647283+0000", 2023-10-09T05:57:00.053 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012812300000000001 2023-10-09T05:57:00.053 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.053 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.053 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:57:00.053 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.647407+0000", 2023-10-09T05:57:00.054 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012368899999999999 2023-10-09T05:57:00.054 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.054 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.054 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:57:00.054 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.649330+0000", 2023-10-09T05:57:00.054 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0019226779999999999 2023-10-09T05:57:00.054 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.054 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.054 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.054 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.650266+0000", 2023-10-09T05:57:00.054 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00093634999999999996 2023-10-09T05:57:00.055 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.055 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.055 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.055 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.980707+0000", 2023-10-09T05:57:00.055 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.33044109100000002 2023-10-09T05:57:00.055 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.055 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.055 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:57:00.055 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.980743+0000", 2023-10-09T05:57:00.056 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5800000000000003e-05 2023-10-09T05:57:00.056 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.056 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.056 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:57:00.056 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:49.980772+0000", 2023-10-09T05:57:00.056 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.8580000000000001e-05 2023-10-09T05:57:00.056 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.056 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:57:00.056 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.056 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.056 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.057 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:79589 6.5 6:a0fa1d08:::benchmark_data_smithi064_284193_object79588:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:57:00.057 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:52.363694+0000", 2023-10-09T05:57:00.057 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 307.59502966399998, 2023-10-09T05:57:00.057 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.26210419000000001, 2023-10-09T05:57:00.057 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:57:00.057 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:00.057 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:57:00.057 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:57:00.057 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:00.057 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 79589 2023-10-09T05:57:00.057 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.057 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:57:00.058 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.058 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:57:00.058 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.363694+0000", 2023-10-09T05:57:00.058 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.058 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.058 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.058 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:57:00.058 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.363694+0000", 2023-10-09T05:57:00.058 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.058 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.058 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.059 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:57:00.059 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.363693+0000", 2023-10-09T05:57:00.059 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:00.059 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.059 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.059 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:57:00.059 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.363741+0000", 2023-10-09T05:57:00.059 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8436999999999997e-05 2023-10-09T05:57:00.059 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.059 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.059 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:57:00.059 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.363750+0000", 2023-10-09T05:57:00.060 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1820000000000006e-06 2023-10-09T05:57:00.060 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.060 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.060 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:57:00.060 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.363754+0000", 2023-10-09T05:57:00.060 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6120000000000001e-06 2023-10-09T05:57:00.060 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.060 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.060 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.060 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.366906+0000", 2023-10-09T05:57:00.060 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0031518340000000001 2023-10-09T05:57:00.060 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.061 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.061 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:57:00.061 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.367025+0000", 2023-10-09T05:57:00.061 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000118854 2023-10-09T05:57:00.061 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.061 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.061 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:57:00.061 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.367157+0000", 2023-10-09T05:57:00.061 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013233 2023-10-09T05:57:00.061 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.061 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.061 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:57:00.062 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.372713+0000", 2023-10-09T05:57:00.062 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0055557940000000002 2023-10-09T05:57:00.062 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.062 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.062 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.062 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.376095+0000", 2023-10-09T05:57:00.062 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0033821039999999999 2023-10-09T05:57:00.062 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.062 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.062 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.063 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.625686+0000", 2023-10-09T05:57:00.063 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24959123 2023-10-09T05:57:00.063 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.063 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.063 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:57:00.063 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.625752+0000", 2023-10-09T05:57:00.063 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6228000000000004e-05 2023-10-09T05:57:00.063 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.063 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.063 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:57:00.063 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.625798+0000", 2023-10-09T05:57:00.063 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5246e-05 2023-10-09T05:57:00.064 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.064 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:57:00.064 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.064 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.064 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.064 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:79591 6.5 6:a2ce5c7b:::benchmark_data_smithi064_284193_object79590:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T05:57:00.064 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:52.366733+0000", 2023-10-09T05:57:00.064 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 307.59199068599997, 2023-10-09T05:57:00.064 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.25915231599999999, 2023-10-09T05:57:00.064 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:57:00.064 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:00.064 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:57:00.065 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:57:00.065 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:00.065 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 79591 2023-10-09T05:57:00.065 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.065 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:57:00.065 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.065 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:57:00.065 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.366733+0000", 2023-10-09T05:57:00.065 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.065 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.065 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.065 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:57:00.066 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.366733+0000", 2023-10-09T05:57:00.066 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.066 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.066 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.066 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:57:00.066 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.366731+0000", 2023-10-09T05:57:00.066 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:57:00.066 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.066 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.066 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:57:00.066 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.366762+0000", 2023-10-09T05:57:00.067 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0255000000000001e-05 2023-10-09T05:57:00.067 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.067 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.067 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:57:00.067 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.366770+0000", 2023-10-09T05:57:00.067 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6680000000000007e-06 2023-10-09T05:57:00.067 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.067 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.067 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:57:00.067 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.366775+0000", 2023-10-09T05:57:00.067 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4710000000000001e-06 2023-10-09T05:57:00.067 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.068 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.068 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.068 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.369642+0000", 2023-10-09T05:57:00.068 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0028674389999999998 2023-10-09T05:57:00.068 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.068 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.068 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:57:00.068 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.369755+0000", 2023-10-09T05:57:00.068 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011286900000000001 2023-10-09T05:57:00.068 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.068 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.068 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:57:00.069 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.369850+0000", 2023-10-09T05:57:00.069 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.4812e-05 2023-10-09T05:57:00.069 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.069 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.069 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:57:00.069 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.375836+0000", 2023-10-09T05:57:00.069 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0059865750000000001 2023-10-09T05:57:00.069 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.069 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.069 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.069 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.377143+0000", 2023-10-09T05:57:00.069 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001306967 2023-10-09T05:57:00.070 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.070 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.070 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.070 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.625835+0000", 2023-10-09T05:57:00.070 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24869169799999999 2023-10-09T05:57:00.070 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.070 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.070 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:57:00.070 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.625865+0000", 2023-10-09T05:57:00.070 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9937000000000002e-05 2023-10-09T05:57:00.070 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.070 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.071 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:57:00.071 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:52.625885+0000", 2023-10-09T05:57:00.071 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9755e-05 2023-10-09T05:57:00.071 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.071 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:57:00.071 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.071 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.071 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.071 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:104343 6.18 6:1b5b6ba0:::benchmark_data_smithi064_284193_object7800:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e188)", 2023-10-09T05:57:00.071 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:54:44.947302+0000", 2023-10-09T05:57:00.071 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 135.01142162299999, 2023-10-09T05:57:00.072 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.74489010499999997, 2023-10-09T05:57:00.072 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:57:00.072 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:00.072 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:57:00.072 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:57:00.072 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:00.072 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 104343 2023-10-09T05:57:00.072 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.072 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:57:00.072 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.072 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:57:00.072 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:44.947302+0000", 2023-10-09T05:57:00.073 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.073 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.073 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.073 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:57:00.073 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:44.947302+0000", 2023-10-09T05:57:00.073 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.073 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.073 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.073 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:57:00.073 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:44.947300+0000", 2023-10-09T05:57:00.073 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:57:00.074 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.074 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.074 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:57:00.074 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:44.947308+0000", 2023-10-09T05:57:00.074 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8399999999999995e-06 2023-10-09T05:57:00.074 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.074 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.074 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:57:00.074 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:44.947325+0000", 2023-10-09T05:57:00.074 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7082999999999999e-05 2023-10-09T05:57:00.074 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.075 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.075 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:57:00.075 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:44.947333+0000", 2023-10-09T05:57:00.075 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6599999999999995e-06 2023-10-09T05:57:00.075 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.075 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.075 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.075 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:44.947375+0000", 2023-10-09T05:57:00.075 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2401999999999998e-05 2023-10-09T05:57:00.075 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.075 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.075 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:57:00.076 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:44.947383+0000", 2023-10-09T05:57:00.076 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.926e-06 2023-10-09T05:57:00.076 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.076 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.076 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.076 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.669246+0000", 2023-10-09T05:57:00.076 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.72186271000000002 2023-10-09T05:57:00.076 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.076 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.076 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:57:00.076 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.669397+0000", 2023-10-09T05:57:00.076 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000151516 2023-10-09T05:57:00.077 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.077 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.077 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:57:00.077 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.669671+0000", 2023-10-09T05:57:00.077 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000273615 2023-10-09T05:57:00.077 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.077 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.077 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:57:00.077 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.671422+0000", 2023-10-09T05:57:00.077 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001751671 2023-10-09T05:57:00.077 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.078 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.078 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.078 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.679093+0000", 2023-10-09T05:57:00.078 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0076703379999999996 2023-10-09T05:57:00.078 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.078 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.078 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.078 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.692096+0000", 2023-10-09T05:57:00.078 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.01300318 2023-10-09T05:57:00.078 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.078 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.079 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:57:00.079 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.692151+0000", 2023-10-09T05:57:00.079 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5368000000000003e-05 2023-10-09T05:57:00.079 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.079 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.079 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:57:00.079 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.692192+0000", 2023-10-09T05:57:00.079 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0425e-05 2023-10-09T05:57:00.079 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.079 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:57:00.079 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.079 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.080 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.080 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:104350 6.15 6:af513ac9:::benchmark_data_smithi064_284193_object7807:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2023-10-09T05:57:00.080 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:54:45.800000+0000", 2023-10-09T05:57:00.080 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 134.158723159, 2023-10-09T05:57:00.080 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.898542277, 2023-10-09T05:57:00.080 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:57:00.080 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:00.080 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:57:00.080 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:57:00.080 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:00.080 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 104350 2023-10-09T05:57:00.081 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.081 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:57:00.081 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.081 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:57:00.081 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.800000+0000", 2023-10-09T05:57:00.081 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.081 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.081 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.081 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:57:00.081 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.800000+0000", 2023-10-09T05:57:00.081 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.081 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.082 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.082 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:57:00.082 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.799999+0000", 2023-10-09T05:57:00.082 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:57:00.082 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.082 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.082 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:57:00.082 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.800004+0000", 2023-10-09T05:57:00.082 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4840000000000003e-06 2023-10-09T05:57:00.082 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.082 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.082 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:57:00.083 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.800015+0000", 2023-10-09T05:57:00.083 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0581000000000001e-05 2023-10-09T05:57:00.083 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.083 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.083 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:57:00.083 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.800030+0000", 2023-10-09T05:57:00.083 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4489e-05 2023-10-09T05:57:00.083 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.083 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.083 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.083 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.800068+0000", 2023-10-09T05:57:00.083 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8318999999999997e-05 2023-10-09T05:57:00.084 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.084 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.084 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:57:00.084 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.800076+0000", 2023-10-09T05:57:00.084 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6690000000000006e-06 2023-10-09T05:57:00.084 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.084 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.084 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.084 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:46.648630+0000", 2023-10-09T05:57:00.084 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.84855464599999997 2023-10-09T05:57:00.084 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.084 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.085 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:57:00.085 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:46.648639+0000", 2023-10-09T05:57:00.085 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.5259999999999993e-06 2023-10-09T05:57:00.085 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.085 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.085 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.085 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:47.674779+0000", 2023-10-09T05:57:00.085 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.026139881 2023-10-09T05:57:00.085 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.085 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.085 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:57:00.086 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:47.674949+0000", 2023-10-09T05:57:00.086 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000170765 2023-10-09T05:57:00.086 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.086 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.086 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:57:00.086 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:47.675251+0000", 2023-10-09T05:57:00.086 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000301682 2023-10-09T05:57:00.086 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.086 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.088 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.088 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:47.697358+0000", 2023-10-09T05:57:00.088 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.022106995000000001 2023-10-09T05:57:00.088 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.088 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.088 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:57:00.088 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:47.697368+0000", 2023-10-09T05:57:00.088 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0220999999999999e-05 2023-10-09T05:57:00.088 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.088 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.088 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.089 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:47.697409+0000", 2023-10-09T05:57:00.089 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0881999999999999e-05 2023-10-09T05:57:00.089 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.089 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.089 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:57:00.089 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:47.697470+0000", 2023-10-09T05:57:00.089 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0804000000000003e-05 2023-10-09T05:57:00.089 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.089 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.089 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:57:00.089 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:47.698542+0000", 2023-10-09T05:57:00.090 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001072386 2023-10-09T05:57:00.090 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.090 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:57:00.090 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.090 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.090 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.090 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:139292 6.25 6:a794a077:::benchmark_data_smithi064_284193_object42749:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T05:57:00.090 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:56:39.261633+0000", 2023-10-09T05:57:00.090 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 20.697090574000001, 2023-10-09T05:57:00.090 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0412176790000001, 2023-10-09T05:57:00.090 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:57:00.090 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:00.091 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:57:00.091 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:57:00.091 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:00.091 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 139292 2023-10-09T05:57:00.091 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.091 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:57:00.091 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.091 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:57:00.091 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.261633+0000", 2023-10-09T05:57:00.091 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.091 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.092 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.092 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:57:00.092 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.261633+0000", 2023-10-09T05:57:00.092 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.092 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.092 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.092 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:57:00.092 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.261631+0000", 2023-10-09T05:57:00.092 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T05:57:00.092 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.092 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.092 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:57:00.093 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.261644+0000", 2023-10-09T05:57:00.093 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3363999999999999e-05 2023-10-09T05:57:00.093 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.093 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.093 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:57:00.093 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.261668+0000", 2023-10-09T05:57:00.093 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4235000000000001e-05 2023-10-09T05:57:00.093 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.093 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.093 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:57:00.093 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.261678+0000", 2023-10-09T05:57:00.093 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3049999999999998e-06 2023-10-09T05:57:00.094 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.094 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.094 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.094 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.508760+0000", 2023-10-09T05:57:00.094 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.247081986 2023-10-09T05:57:00.094 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.094 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.094 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:57:00.094 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.508768+0000", 2023-10-09T05:57:00.094 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.2430000000000005e-06 2023-10-09T05:57:00.095 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.095 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.095 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.095 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.288423+0000", 2023-10-09T05:57:00.095 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.77965499400000005 2023-10-09T05:57:00.095 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.095 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.095 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:57:00.095 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.288530+0000", 2023-10-09T05:57:00.095 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000107154 2023-10-09T05:57:00.095 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.095 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.095 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:57:00.096 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.288718+0000", 2023-10-09T05:57:00.096 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00018789800000000001 2023-10-09T05:57:00.096 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.096 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.096 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:57:00.096 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.292600+0000", 2023-10-09T05:57:00.096 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0038821020000000001 2023-10-09T05:57:00.096 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.096 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.096 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.096 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.295012+0000", 2023-10-09T05:57:00.097 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002412408 2023-10-09T05:57:00.097 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.097 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.097 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.097 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.302745+0000", 2023-10-09T05:57:00.097 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0077325529999999996 2023-10-09T05:57:00.097 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.097 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.097 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:57:00.097 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.302813+0000", 2023-10-09T05:57:00.097 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7644999999999994e-05 2023-10-09T05:57:00.097 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.098 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.098 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:57:00.098 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.302850+0000", 2023-10-09T05:57:00.098 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7735999999999999e-05 2023-10-09T05:57:00.098 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.098 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:57:00.098 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.098 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.098 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.098 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:139304 6.25 6:a525254f:::benchmark_data_smithi064_284193_object42761:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T05:57:00.098 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:56:39.544134+0000", 2023-10-09T05:57:00.099 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 20.414589487000001, 2023-10-09T05:57:00.099 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.75882438500000005, 2023-10-09T05:57:00.099 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:57:00.099 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:00.099 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:57:00.099 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:57:00.099 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:00.099 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 139304 2023-10-09T05:57:00.099 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.099 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:57:00.099 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.100 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:57:00.100 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.544134+0000", 2023-10-09T05:57:00.100 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.100 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.100 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.100 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:57:00.100 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.544134+0000", 2023-10-09T05:57:00.100 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.100 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.100 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.100 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:57:00.100 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.544132+0000", 2023-10-09T05:57:00.101 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:57:00.101 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.101 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.101 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:57:00.101 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.544142+0000", 2023-10-09T05:57:00.101 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.4199999999999996e-06 2023-10-09T05:57:00.101 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.101 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.101 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:57:00.101 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.544160+0000", 2023-10-09T05:57:00.101 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8728000000000001e-05 2023-10-09T05:57:00.101 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.102 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.102 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:57:00.102 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.544169+0000", 2023-10-09T05:57:00.102 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.5809999999999997e-06 2023-10-09T05:57:00.102 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.102 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.102 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.102 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.544230+0000", 2023-10-09T05:57:00.102 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0988e-05 2023-10-09T05:57:00.102 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.102 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.103 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:57:00.103 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.544242+0000", 2023-10-09T05:57:00.103 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1705999999999999e-05 2023-10-09T05:57:00.103 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.103 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.103 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.103 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.289623+0000", 2023-10-09T05:57:00.103 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.74538180799999998 2023-10-09T05:57:00.103 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.103 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.103 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:57:00.104 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.289723+0000", 2023-10-09T05:57:00.104 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000100059 2023-10-09T05:57:00.104 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.104 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.104 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:57:00.104 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.289862+0000", 2023-10-09T05:57:00.104 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013901899999999999 2023-10-09T05:57:00.104 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.104 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.104 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:57:00.104 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.292605+0000", 2023-10-09T05:57:00.104 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0027423550000000001 2023-10-09T05:57:00.105 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.105 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.105 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.105 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.296515+0000", 2023-10-09T05:57:00.105 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0039106449999999999 2023-10-09T05:57:00.105 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.105 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.105 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.105 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.302881+0000", 2023-10-09T05:57:00.105 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.006365959 2023-10-09T05:57:00.105 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.105 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.106 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:57:00.106 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.302928+0000", 2023-10-09T05:57:00.106 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6638000000000003e-05 2023-10-09T05:57:00.106 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.106 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.106 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:57:00.106 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.302958+0000", 2023-10-09T05:57:00.106 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0090000000000002e-05 2023-10-09T05:57:00.106 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.106 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:57:00.106 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.107 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.107 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.107 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:139305 6.8 6:13d25f3c:::benchmark_data_smithi064_284193_object42762:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T05:57:00.107 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:56:39.544744+0000", 2023-10-09T05:57:00.107 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 20.413978982, 2023-10-09T05:57:00.107 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.75808377400000004, 2023-10-09T05:57:00.107 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:57:00.107 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:00.107 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:57:00.107 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:57:00.107 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:00.107 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 139305 2023-10-09T05:57:00.108 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.108 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:57:00.108 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.108 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:57:00.108 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.544744+0000", 2023-10-09T05:57:00.108 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.108 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.108 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.108 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:57:00.108 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.544744+0000", 2023-10-09T05:57:00.108 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.109 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.109 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.109 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:57:00.109 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.544743+0000", 2023-10-09T05:57:00.109 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:57:00.109 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.109 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.109 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:57:00.109 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.544749+0000", 2023-10-09T05:57:00.109 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8429999999999998e-06 2023-10-09T05:57:00.109 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.109 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.110 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:57:00.110 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.544763+0000", 2023-10-09T05:57:00.110 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3886999999999999e-05 2023-10-09T05:57:00.110 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.110 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.110 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:57:00.110 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.544769+0000", 2023-10-09T05:57:00.110 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8019999999999997e-06 2023-10-09T05:57:00.110 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.110 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.110 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.111 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.544837+0000", 2023-10-09T05:57:00.111 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7607999999999994e-05 2023-10-09T05:57:00.111 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.111 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.111 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:57:00.111 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.544847+0000", 2023-10-09T05:57:00.111 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0797e-05 2023-10-09T05:57:00.111 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.111 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.111 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.111 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.286110+0000", 2023-10-09T05:57:00.112 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.74126308399999996 2023-10-09T05:57:00.112 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.112 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.112 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:57:00.112 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.290699+0000", 2023-10-09T05:57:00.112 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0045887080000000004 2023-10-09T05:57:00.112 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.112 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.112 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:57:00.112 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.290961+0000", 2023-10-09T05:57:00.112 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00026228599999999998 2023-10-09T05:57:00.112 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.113 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.113 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:57:00.113 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.292483+0000", 2023-10-09T05:57:00.113 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001521765 2023-10-09T05:57:00.113 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.113 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.113 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.113 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.298522+0000", 2023-10-09T05:57:00.113 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0060383859999999998 2023-10-09T05:57:00.113 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.113 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.113 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.114 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.302737+0000", 2023-10-09T05:57:00.114 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0042157480000000001 2023-10-09T05:57:00.114 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.114 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.114 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:57:00.114 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.302789+0000", 2023-10-09T05:57:00.114 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2039000000000001e-05 2023-10-09T05:57:00.114 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.114 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.114 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:57:00.114 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.302828+0000", 2023-10-09T05:57:00.115 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8665999999999999e-05 2023-10-09T05:57:00.115 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.115 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:57:00.115 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.115 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.115 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.115 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:139328 6.28 6:17f64c24:::benchmark_data_smithi064_284193_object42785:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T05:57:00.115 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:56:39.598753+0000", 2023-10-09T05:57:00.115 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 20.359970608000001, 2023-10-09T05:57:00.115 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.70973481199999999, 2023-10-09T05:57:00.115 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:57:00.115 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:00.116 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:57:00.116 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:57:00.116 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:00.116 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 139328 2023-10-09T05:57:00.116 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.116 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:57:00.116 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.116 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:57:00.116 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.598753+0000", 2023-10-09T05:57:00.116 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.116 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.117 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.117 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:57:00.117 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.598753+0000", 2023-10-09T05:57:00.117 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:57:00.117 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.117 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.117 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:57:00.117 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.598751+0000", 2023-10-09T05:57:00.117 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:57:00.117 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.117 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.117 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:57:00.118 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.598759+0000", 2023-10-09T05:57:00.118 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4590000000000001e-06 2023-10-09T05:57:00.118 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.118 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.118 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:57:00.118 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.598776+0000", 2023-10-09T05:57:00.118 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7139e-05 2023-10-09T05:57:00.118 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.118 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.118 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:57:00.118 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.598785+0000", 2023-10-09T05:57:00.119 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.4530000000000008e-06 2023-10-09T05:57:00.119 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.119 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.119 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.119 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.598840+0000", 2023-10-09T05:57:00.119 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5254000000000003e-05 2023-10-09T05:57:00.119 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.119 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.119 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:57:00.119 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.598851+0000", 2023-10-09T05:57:00.119 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0791999999999999e-05 2023-10-09T05:57:00.120 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.120 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.120 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:57:00.120 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.297388+0000", 2023-10-09T05:57:00.120 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.69853710099999999 2023-10-09T05:57:00.120 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.120 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.120 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:57:00.120 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.297502+0000", 2023-10-09T05:57:00.120 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000114027 2023-10-09T05:57:00.120 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.120 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.120 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:57:00.121 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.297674+0000", 2023-10-09T05:57:00.121 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000171829 2023-10-09T05:57:00.121 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.121 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.121 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:57:00.121 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.299665+0000", 2023-10-09T05:57:00.121 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0019912350000000001 2023-10-09T05:57:00.121 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.121 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.121 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.121 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.305923+0000", 2023-10-09T05:57:00.121 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0062575199999999999 2023-10-09T05:57:00.122 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.122 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.122 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:00.122 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.308389+0000", 2023-10-09T05:57:00.122 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0024665339999999998 2023-10-09T05:57:00.122 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.122 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.122 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:57:00.122 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.308445+0000", 2023-10-09T05:57:00.122 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5374999999999998e-05 2023-10-09T05:57:00.122 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:57:00.123 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:57:00.123 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:57:00.123 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.308487+0000", 2023-10-09T05:57:00.123 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2555999999999997e-05 2023-10-09T05:57:00.123 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.123 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:57:00.123 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.123 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:57:00.123 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:57:00.123 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:57:00.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:00.125 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:00.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:00.226 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:00.270 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:00.271 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:00 smithi156 ceph-mon[195101]: pgmap v548: 139 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 118 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 109 B/s rd, 0 B/s wr, 334 op/s; 18966/145596 objects degraded (13.026%); 16249/145596 objects misplaced (11.160%); 16 MiB/s, 289 objects/s recovering 2023-10-09T05:57:00.271 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:00 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3547973092' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:57:00.271 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:00 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:57:00.272 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:00 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3462854240' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:57:00.278 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T05:57:00.280 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T05:57:00.345 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:00.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:00.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:00.449 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:00.482 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:00.534 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:00.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:00.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:00.638 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:00.691 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:00.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:00 smithi064 ceph-mon[212750]: pgmap v548: 139 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 118 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 109 B/s rd, 0 B/s wr, 334 op/s; 18966/145596 objects degraded (13.026%); 16249/145596 objects misplaced (11.160%); 16 MiB/s, 289 objects/s recovering 2023-10-09T05:57:00.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:00 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3547973092' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:57:00.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:00 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:57:00.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:00 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3462854240' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:57:00.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:00 smithi064 ceph-mon[220044]: pgmap v548: 139 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 118 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 109 B/s rd, 0 B/s wr, 334 op/s; 18966/145596 objects degraded (13.026%); 16249/145596 objects misplaced (11.160%); 16 MiB/s, 289 objects/s recovering 2023-10-09T05:57:00.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:00 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3547973092' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:57:00.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:00 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:57:00.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:00 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3462854240' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:57:00.773 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:00.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:00.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:00.876 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:00.927 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:00.984 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:57:01.275 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:01 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:01.276 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:01 smithi156 ceph-mon[195101]: failed to encode map e236 with expected crc 2023-10-09T05:57:01.276 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:01 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:57:01.276 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:01 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3462854240' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:57:01.276 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:01 smithi156 ceph-mon[195101]: osdmap e236: 8 total, 8 up, 7 in 2023-10-09T05:57:01.276 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:01 smithi156 ceph-mon[195101]: failed to encode map e236 with expected crc 2023-10-09T05:57:01.276 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:01 smithi156 ceph-mon[195101]: failed to encode map e236 with expected crc 2023-10-09T05:57:01.276 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:01 smithi156 ceph-mon[195101]: pgmap v550: 139 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 118 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 315 op/s; 18331/139530 objects degraded (13.138%); 14814/139530 objects misplaced (10.617%); 13 MiB/s, 226 objects/s recovering 2023-10-09T05:57:01.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:01 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:01.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:01 smithi064 ceph-mon[220044]: failed to encode map e236 with expected crc 2023-10-09T05:57:01.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:01 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:57:01.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:01 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3462854240' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:57:01.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:01 smithi064 ceph-mon[220044]: osdmap e236: 8 total, 8 up, 7 in 2023-10-09T05:57:01.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:01 smithi064 ceph-mon[220044]: failed to encode map e236 with expected crc 2023-10-09T05:57:01.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:01 smithi064 ceph-mon[220044]: failed to encode map e236 with expected crc 2023-10-09T05:57:01.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:01 smithi064 ceph-mon[220044]: pgmap v550: 139 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 118 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 315 op/s; 18331/139530 objects degraded (13.138%); 14814/139530 objects misplaced (10.617%); 13 MiB/s, 226 objects/s recovering 2023-10-09T05:57:01.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:01 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:01.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:01 smithi064 ceph-mon[212750]: failed to encode map e236 with expected crc 2023-10-09T05:57:01.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:01 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:57:01.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:01 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3462854240' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:57:01.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:01 smithi064 ceph-mon[212750]: osdmap e236: 8 total, 8 up, 7 in 2023-10-09T05:57:01.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:01 smithi064 ceph-mon[212750]: failed to encode map e236 with expected crc 2023-10-09T05:57:01.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:01 smithi064 ceph-mon[212750]: failed to encode map e236 with expected crc 2023-10-09T05:57:01.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:01 smithi064 ceph-mon[212750]: pgmap v550: 139 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 118 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 315 op/s; 18331/139530 objects degraded (13.138%); 14814/139530 objects misplaced (10.617%); 13 MiB/s, 226 objects/s recovering 2023-10-09T05:57:02.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:02 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 18331/139530 objects degraded (13.138%), 15 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:02.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:02 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 18331/139530 objects degraded (13.138%), 15 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:02.542 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:02.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:02.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:02.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:02 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 18331/139530 objects degraded (13.138%), 15 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:02.645 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:02.680 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:02.740 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T05:57:03.030 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:03.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:03.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:03.147 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:03.193 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:03.284 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:03.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:03.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:03.365 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:03.389 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:03.428 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:03.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:03 smithi064 ceph-mon[212750]: pgmap v551: 139 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 118 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 315 op/s; 18331/139530 objects degraded (13.138%); 14814/139530 objects misplaced (10.617%); 13 MiB/s, 226 objects/s recovering 2023-10-09T05:57:03.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:03 smithi064 ceph-mon[220044]: pgmap v551: 139 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 118 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 315 op/s; 18331/139530 objects degraded (13.138%); 14814/139530 objects misplaced (10.617%); 13 MiB/s, 226 objects/s recovering 2023-10-09T05:57:03.497 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:03.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:03.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:03.531 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:03.533 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:57:03.572 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:57:03.605 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:57:03.607 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:03.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:03 smithi156 ceph-mon[195101]: pgmap v551: 139 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 118 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 315 op/s; 18331/139530 objects degraded (13.138%); 14814/139530 objects misplaced (10.617%); 13 MiB/s, 226 objects/s recovering 2023-10-09T05:57:03.637 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:57:03.649 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:03.669 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:57:03.701 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:57:03.736 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:57:03.740 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:03.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:03.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:03.761 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:57:03.763 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:57:03.763 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:57:03.763 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:57:03.764 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:57:03.764 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:57:03.764 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:57:03.764 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:57:03.764 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:57:03.844 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:03.893 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:03.964 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:03.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:03.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:04.072 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:04.103 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:04.164 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:04.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:04.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:04.274 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:57:04.309 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:04.383 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:57:04.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:04.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:57:04.420 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:04 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1625176379' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:04.421 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:04 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1625176379' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:04.488 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:04.516 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:04 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1625176379' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:04.521 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:04.578 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:04.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:04.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:04.685 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:04.725 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:05.057 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:57:05.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:05 smithi064 ceph-mon[212750]: pgmap v552: 139 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 118 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 336 op/s; 18010/137538 objects degraded (13.095%); 14685/137538 objects misplaced (10.677%); 13 MiB/s, 239 objects/s recovering 2023-10-09T05:57:05.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:05 smithi064 ceph-mon[220044]: pgmap v552: 139 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 118 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 336 op/s; 18010/137538 objects degraded (13.095%); 14685/137538 objects misplaced (10.677%); 13 MiB/s, 239 objects/s recovering 2023-10-09T05:57:05.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:57:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:57:05] "GET /metrics HTTP/1.1" 200 36488 "" "Prometheus/2.43.0" 2023-10-09T05:57:05.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:05 smithi156 ceph-mon[195101]: pgmap v552: 139 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 118 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 336 op/s; 18010/137538 objects degraded (13.095%); 14685/137538 objects misplaced (10.677%); 13 MiB/s, 239 objects/s recovering 2023-10-09T05:57:06.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:06 smithi064 ceph-mon[212750]: 1.8 scrub starts 2023-10-09T05:57:06.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:06 smithi064 ceph-mon[220044]: 1.8 scrub starts 2023-10-09T05:57:06.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:06 smithi156 ceph-mon[195101]: 1.8 scrub starts 2023-10-09T05:57:06.651 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:06.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:06.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:06.757 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:06.798 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:07.365 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:07.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:07.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:07.469 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:07.503 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:07.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:07 smithi156 ceph-mon[195101]: pgmap v553: 139 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 118 active+clean, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 17466/131526 objects degraded (13.280%); 12630/131526 objects misplaced (9.603%); 17 MiB/s, 269 objects/s recovering 2023-10-09T05:57:07.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:07 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 18010/137538 objects degraded (13.095%), 15 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:07.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:07 smithi064 ceph-mon[212750]: pgmap v553: 139 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 118 active+clean, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 17466/131526 objects degraded (13.280%); 12630/131526 objects misplaced (9.603%); 17 MiB/s, 269 objects/s recovering 2023-10-09T05:57:07.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:07 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 18010/137538 objects degraded (13.095%), 15 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:07.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:07 smithi064 ceph-mon[220044]: pgmap v553: 139 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 118 active+clean, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 17466/131526 objects degraded (13.280%); 12630/131526 objects misplaced (9.603%); 17 MiB/s, 269 objects/s recovering 2023-10-09T05:57:07.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:07 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 18010/137538 objects degraded (13.095%), 15 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:08.080 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:08.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:08.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:08.183 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:08.216 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:08.288 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:08.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:08.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:08.399 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:08.434 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:08.599 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:08.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:08.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:08.704 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:57:08.756 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:08.791 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:08.833 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:57:08.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:08.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:57:08.858 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:57:08.858 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:57:08.940 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:08.991 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:09.157 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:09.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:09.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:09.261 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:09.293 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:09.341 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T05:57:09.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:09 smithi064 ceph-mon[220044]: failed to encode map e237 with expected crc 2023-10-09T05:57:09.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:09 smithi064 ceph-mon[220044]: failed to encode map e237 with expected crc 2023-10-09T05:57:09.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:09 smithi064 ceph-mon[220044]: osdmap e237: 8 total, 8 up, 7 in 2023-10-09T05:57:09.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:09 smithi064 ceph-mon[220044]: failed to encode map e237 with expected crc 2023-10-09T05:57:09.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:09 smithi064 ceph-mon[220044]: failed to encode map e237 with expected crc 2023-10-09T05:57:09.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:09 smithi064 ceph-mon[220044]: failed to encode map e237 with expected crc 2023-10-09T05:57:09.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:09 smithi064 ceph-mon[220044]: failed to encode map e237 with expected crc 2023-10-09T05:57:09.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:09 smithi064 ceph-mon[220044]: pgmap v555: 139 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 117 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 458 op/s; 16479/129645 objects degraded (12.711%); 13377/129645 objects misplaced (10.318%); 14 MiB/s, 216 objects/s recovering 2023-10-09T05:57:09.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:09 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2159010177' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:57:09.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:09 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:57:09.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:09 smithi064 ceph-mon[212750]: failed to encode map e237 with expected crc 2023-10-09T05:57:09.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:09 smithi064 ceph-mon[212750]: failed to encode map e237 with expected crc 2023-10-09T05:57:09.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:09 smithi064 ceph-mon[212750]: osdmap e237: 8 total, 8 up, 7 in 2023-10-09T05:57:09.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:09 smithi064 ceph-mon[212750]: failed to encode map e237 with expected crc 2023-10-09T05:57:09.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:09 smithi064 ceph-mon[212750]: failed to encode map e237 with expected crc 2023-10-09T05:57:09.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:09 smithi064 ceph-mon[212750]: failed to encode map e237 with expected crc 2023-10-09T05:57:09.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:09 smithi064 ceph-mon[212750]: failed to encode map e237 with expected crc 2023-10-09T05:57:09.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:09 smithi064 ceph-mon[212750]: pgmap v555: 139 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 117 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 458 op/s; 16479/129645 objects degraded (12.711%); 13377/129645 objects misplaced (10.318%); 14 MiB/s, 216 objects/s recovering 2023-10-09T05:57:09.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:09 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2159010177' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:57:09.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:09 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:57:09.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:09 smithi156 ceph-mon[195101]: failed to encode map e237 with expected crc 2023-10-09T05:57:09.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:09 smithi156 ceph-mon[195101]: failed to encode map e237 with expected crc 2023-10-09T05:57:09.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:09 smithi156 ceph-mon[195101]: osdmap e237: 8 total, 8 up, 7 in 2023-10-09T05:57:09.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:09 smithi156 ceph-mon[195101]: failed to encode map e237 with expected crc 2023-10-09T05:57:09.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:09 smithi156 ceph-mon[195101]: failed to encode map e237 with expected crc 2023-10-09T05:57:09.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:09 smithi156 ceph-mon[195101]: failed to encode map e237 with expected crc 2023-10-09T05:57:09.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:09 smithi156 ceph-mon[195101]: failed to encode map e237 with expected crc 2023-10-09T05:57:09.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:09 smithi156 ceph-mon[195101]: pgmap v555: 139 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 6 active+recovery_wait+degraded+remapped, 117 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 458 op/s; 16479/129645 objects degraded (12.711%); 13377/129645 objects misplaced (10.318%); 14 MiB/s, 216 objects/s recovering 2023-10-09T05:57:09.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:09 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2159010177' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:57:09.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:09 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:57:10.180 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:57:10.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:10 smithi064 ceph-mon[220044]: 1.a scrub starts 2023-10-09T05:57:10.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:10 smithi064 ceph-mon[220044]: failed to encode map e238 with expected crc 2023-10-09T05:57:10.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:10 smithi064 ceph-mon[220044]: failed to encode map e238 with expected crc 2023-10-09T05:57:10.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:10 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:57:10.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:10 smithi064 ceph-mon[220044]: osdmap e238: 8 total, 8 up, 7 in 2023-10-09T05:57:10.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:10 smithi064 ceph-mon[220044]: failed to encode map e238 with expected crc 2023-10-09T05:57:10.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:10 smithi064 ceph-mon[220044]: failed to encode map e238 with expected crc 2023-10-09T05:57:10.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:10 smithi064 ceph-mon[220044]: failed to encode map e238 with expected crc 2023-10-09T05:57:10.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:10 smithi064 ceph-mon[212750]: 1.a scrub starts 2023-10-09T05:57:10.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:10 smithi064 ceph-mon[212750]: failed to encode map e238 with expected crc 2023-10-09T05:57:10.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:10 smithi064 ceph-mon[212750]: failed to encode map e238 with expected crc 2023-10-09T05:57:10.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:10 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:57:10.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:10 smithi064 ceph-mon[212750]: osdmap e238: 8 total, 8 up, 7 in 2023-10-09T05:57:10.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:10 smithi064 ceph-mon[212750]: failed to encode map e238 with expected crc 2023-10-09T05:57:10.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:10 smithi064 ceph-mon[212750]: failed to encode map e238 with expected crc 2023-10-09T05:57:10.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:10 smithi064 ceph-mon[212750]: failed to encode map e238 with expected crc 2023-10-09T05:57:10.472 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:10.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:10.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:10.579 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:10.613 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:10.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:10 smithi156 ceph-mon[195101]: 1.a scrub starts 2023-10-09T05:57:10.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:10 smithi156 ceph-mon[195101]: failed to encode map e238 with expected crc 2023-10-09T05:57:10.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:10 smithi156 ceph-mon[195101]: failed to encode map e238 with expected crc 2023-10-09T05:57:10.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:10 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:57:10.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:10 smithi156 ceph-mon[195101]: osdmap e238: 8 total, 8 up, 7 in 2023-10-09T05:57:10.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:10 smithi156 ceph-mon[195101]: failed to encode map e238 with expected crc 2023-10-09T05:57:10.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:10 smithi156 ceph-mon[195101]: failed to encode map e238 with expected crc 2023-10-09T05:57:10.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:10 smithi156 ceph-mon[195101]: failed to encode map e238 with expected crc 2023-10-09T05:57:11.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:11 smithi064 ceph-mon[220044]: osdmap e239: 8 total, 8 up, 7 in 2023-10-09T05:57:11.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:11 smithi064 ceph-mon[220044]: failed to encode map e239 with expected crc 2023-10-09T05:57:11.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:11 smithi064 ceph-mon[220044]: failed to encode map e239 with expected crc 2023-10-09T05:57:11.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:11 smithi064 ceph-mon[220044]: failed to encode map e239 with expected crc 2023-10-09T05:57:11.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:11 smithi064 ceph-mon[220044]: failed to encode map e239 with expected crc 2023-10-09T05:57:11.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:11 smithi064 ceph-mon[220044]: 6.1e starting backfill to osd.2 from (0'0,137'6147] MIN to 237'8078 2023-10-09T05:57:11.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:11 smithi064 ceph-mon[220044]: 6.e starting backfill to osd.2 from (0'0,137'6147] MIN to 237'8109 2023-10-09T05:57:11.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:11 smithi064 ceph-mon[220044]: pgmap v558: 139 pgs: 6 remapped+peering, 1 active+remapped+backfilling, 13 peering, 1 active+clean+scrubbing, 4 active+recovery_wait+degraded+remapped, 107 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 732 op/s; 9912/124344 objects degraded (7.971%); 6751/124344 objects misplaced (5.429%); 31 MiB/s, 650 objects/s recovering 2023-10-09T05:57:11.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:11 smithi064 ceph-mon[212750]: osdmap e239: 8 total, 8 up, 7 in 2023-10-09T05:57:11.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:11 smithi064 ceph-mon[212750]: failed to encode map e239 with expected crc 2023-10-09T05:57:11.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:11 smithi064 ceph-mon[212750]: failed to encode map e239 with expected crc 2023-10-09T05:57:11.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:11 smithi064 ceph-mon[212750]: failed to encode map e239 with expected crc 2023-10-09T05:57:11.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:11 smithi064 ceph-mon[212750]: failed to encode map e239 with expected crc 2023-10-09T05:57:11.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:11 smithi064 ceph-mon[212750]: 6.1e starting backfill to osd.2 from (0'0,137'6147] MIN to 237'8078 2023-10-09T05:57:11.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:11 smithi064 ceph-mon[212750]: 6.e starting backfill to osd.2 from (0'0,137'6147] MIN to 237'8109 2023-10-09T05:57:11.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:11 smithi064 ceph-mon[212750]: pgmap v558: 139 pgs: 6 remapped+peering, 1 active+remapped+backfilling, 13 peering, 1 active+clean+scrubbing, 4 active+recovery_wait+degraded+remapped, 107 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 732 op/s; 9912/124344 objects degraded (7.971%); 6751/124344 objects misplaced (5.429%); 31 MiB/s, 650 objects/s recovering 2023-10-09T05:57:11.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:11 smithi156 ceph-mon[195101]: osdmap e239: 8 total, 8 up, 7 in 2023-10-09T05:57:11.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:11 smithi156 ceph-mon[195101]: failed to encode map e239 with expected crc 2023-10-09T05:57:11.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:11 smithi156 ceph-mon[195101]: failed to encode map e239 with expected crc 2023-10-09T05:57:11.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:11 smithi156 ceph-mon[195101]: failed to encode map e239 with expected crc 2023-10-09T05:57:11.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:11 smithi156 ceph-mon[195101]: failed to encode map e239 with expected crc 2023-10-09T05:57:11.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:11 smithi156 ceph-mon[195101]: 6.1e starting backfill to osd.2 from (0'0,137'6147] MIN to 237'8078 2023-10-09T05:57:11.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:11 smithi156 ceph-mon[195101]: 6.e starting backfill to osd.2 from (0'0,137'6147] MIN to 237'8109 2023-10-09T05:57:11.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:11 smithi156 ceph-mon[195101]: pgmap v558: 139 pgs: 6 remapped+peering, 1 active+remapped+backfilling, 13 peering, 1 active+clean+scrubbing, 4 active+recovery_wait+degraded+remapped, 107 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 732 op/s; 9912/124344 objects degraded (7.971%); 6751/124344 objects misplaced (5.429%); 31 MiB/s, 650 objects/s recovering 2023-10-09T05:57:11.768 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:57:12.582 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:12.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:12.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:12.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:12 smithi156 ceph-mon[195101]: 1.0 scrub starts 2023-10-09T05:57:12.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:12 smithi156 ceph-mon[195101]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:12 smithi156 ceph-mon[195101]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:12 smithi156 ceph-mon[195101]: osdmap e240: 8 total, 8 up, 7 in 2023-10-09T05:57:12.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:12 smithi156 ceph-mon[195101]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:12 smithi156 ceph-mon[195101]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:12 smithi156 ceph-mon[195101]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:12 smithi156 ceph-mon[195101]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:12 smithi156 ceph-mon[195101]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:12 smithi156 ceph-mon[195101]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:12 smithi156 ceph-mon[195101]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:12 smithi156 ceph-mon[195101]: 6.20 starting backfill to osd.2 from (0'0,165'6531] MIN to 237'7388 2023-10-09T05:57:12.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:12 smithi156 ceph-mon[195101]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:12 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 9912/124344 objects degraded (7.971%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:12.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:12 smithi156 ceph-mon[195101]: osdmap e241: 8 total, 8 up, 7 in 2023-10-09T05:57:12.688 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:12.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[220044]: 1.0 scrub starts 2023-10-09T05:57:12.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[220044]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[220044]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[220044]: osdmap e240: 8 total, 8 up, 7 in 2023-10-09T05:57:12.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[220044]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[220044]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[220044]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[220044]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[220044]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[220044]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[220044]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[220044]: 6.20 starting backfill to osd.2 from (0'0,165'6531] MIN to 237'7388 2023-10-09T05:57:12.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[220044]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 9912/124344 objects degraded (7.971%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:12.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[220044]: osdmap e241: 8 total, 8 up, 7 in 2023-10-09T05:57:12.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[212750]: 1.0 scrub starts 2023-10-09T05:57:12.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[212750]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[212750]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[212750]: osdmap e240: 8 total, 8 up, 7 in 2023-10-09T05:57:12.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[212750]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[212750]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[212750]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[212750]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[212750]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[212750]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[212750]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[212750]: 6.20 starting backfill to osd.2 from (0'0,165'6531] MIN to 237'7388 2023-10-09T05:57:12.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[212750]: failed to encode map e240 with expected crc 2023-10-09T05:57:12.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 9912/124344 objects degraded (7.971%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:57:12.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:12 smithi064 ceph-mon[212750]: osdmap e241: 8 total, 8 up, 7 in 2023-10-09T05:57:12.723 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:13.171 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T05:57:13.194 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:13.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:13.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:13.299 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:13.339 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:13.345 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:13.362 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:13.398 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:13.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:13.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:13.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:13 smithi064 ceph-mon[220044]: failed to encode map e241 with expected crc 2023-10-09T05:57:13.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:13 smithi064 ceph-mon[220044]: failed to encode map e241 with expected crc 2023-10-09T05:57:13.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:13 smithi064 ceph-mon[220044]: failed to encode map e241 with expected crc 2023-10-09T05:57:13.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:13 smithi064 ceph-mon[220044]: failed to encode map e241 with expected crc 2023-10-09T05:57:13.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:13 smithi064 ceph-mon[220044]: failed to encode map e241 with expected crc 2023-10-09T05:57:13.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:13 smithi064 ceph-mon[220044]: failed to encode map e241 with expected crc 2023-10-09T05:57:13.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:13 smithi064 ceph-mon[220044]: failed to encode map e241 with expected crc 2023-10-09T05:57:13.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:13 smithi064 ceph-mon[220044]: pgmap v561: 139 pgs: 6 remapped+peering, 1 active+remapped+backfilling, 13 peering, 4 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 441 op/s; 9912/124344 objects degraded (7.971%); 6751/124344 objects misplaced (5.429%); 20 MiB/s, 555 objects/s recovering 2023-10-09T05:57:13.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:13 smithi064 ceph-mon[220044]: osdmap e242: 8 total, 8 up, 7 in 2023-10-09T05:57:13.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:13 smithi064 ceph-mon[212750]: failed to encode map e241 with expected crc 2023-10-09T05:57:13.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:13 smithi064 ceph-mon[212750]: failed to encode map e241 with expected crc 2023-10-09T05:57:13.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:13 smithi064 ceph-mon[212750]: failed to encode map e241 with expected crc 2023-10-09T05:57:13.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:13 smithi064 ceph-mon[212750]: failed to encode map e241 with expected crc 2023-10-09T05:57:13.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:13 smithi064 ceph-mon[212750]: failed to encode map e241 with expected crc 2023-10-09T05:57:13.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:13 smithi064 ceph-mon[212750]: failed to encode map e241 with expected crc 2023-10-09T05:57:13.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:13 smithi064 ceph-mon[212750]: failed to encode map e241 with expected crc 2023-10-09T05:57:13.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:13 smithi064 ceph-mon[212750]: pgmap v561: 139 pgs: 6 remapped+peering, 1 active+remapped+backfilling, 13 peering, 4 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 441 op/s; 9912/124344 objects degraded (7.971%); 6751/124344 objects misplaced (5.429%); 20 MiB/s, 555 objects/s recovering 2023-10-09T05:57:13.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:13 smithi064 ceph-mon[212750]: osdmap e242: 8 total, 8 up, 7 in 2023-10-09T05:57:13.507 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:13.562 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:13.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:13 smithi156 ceph-mon[195101]: failed to encode map e241 with expected crc 2023-10-09T05:57:13.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:13 smithi156 ceph-mon[195101]: failed to encode map e241 with expected crc 2023-10-09T05:57:13.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:13 smithi156 ceph-mon[195101]: failed to encode map e241 with expected crc 2023-10-09T05:57:13.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:13 smithi156 ceph-mon[195101]: failed to encode map e241 with expected crc 2023-10-09T05:57:13.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:13 smithi156 ceph-mon[195101]: failed to encode map e241 with expected crc 2023-10-09T05:57:13.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:13 smithi156 ceph-mon[195101]: failed to encode map e241 with expected crc 2023-10-09T05:57:13.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:13 smithi156 ceph-mon[195101]: failed to encode map e241 with expected crc 2023-10-09T05:57:13.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:13 smithi156 ceph-mon[195101]: pgmap v561: 139 pgs: 6 remapped+peering, 1 active+remapped+backfilling, 13 peering, 4 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 441 op/s; 9912/124344 objects degraded (7.971%); 6751/124344 objects misplaced (5.429%); 20 MiB/s, 555 objects/s recovering 2023-10-09T05:57:13.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:13 smithi156 ceph-mon[195101]: osdmap e242: 8 total, 8 up, 7 in 2023-10-09T05:57:13.740 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:13.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:13.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:13.844 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:13.891 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:13.963 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:13.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:13.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:14.072 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:14.110 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:14.193 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:14.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:14.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:14.302 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:14.341 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:14.359 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T05:57:14.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:14 smithi156 ceph-mon[195101]: failed to encode map e242 with expected crc 2023-10-09T05:57:14.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:14 smithi156 ceph-mon[195101]: failed to encode map e242 with expected crc 2023-10-09T05:57:14.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:14 smithi156 ceph-mon[195101]: failed to encode map e242 with expected crc 2023-10-09T05:57:14.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:14 smithi156 ceph-mon[195101]: failed to encode map e242 with expected crc 2023-10-09T05:57:14.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:14 smithi156 ceph-mon[195101]: failed to encode map e242 with expected crc 2023-10-09T05:57:14.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:14.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:14 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2080677074' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:57:14.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:14 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:57:14.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:14 smithi064 ceph-mon[212750]: failed to encode map e242 with expected crc 2023-10-09T05:57:14.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:14 smithi064 ceph-mon[212750]: failed to encode map e242 with expected crc 2023-10-09T05:57:14.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:14 smithi064 ceph-mon[212750]: failed to encode map e242 with expected crc 2023-10-09T05:57:14.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:14 smithi064 ceph-mon[212750]: failed to encode map e242 with expected crc 2023-10-09T05:57:14.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:14 smithi064 ceph-mon[212750]: failed to encode map e242 with expected crc 2023-10-09T05:57:14.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:14.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:14 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2080677074' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:57:14.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:14 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:57:14.634 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:14 smithi064 ceph-mon[220044]: failed to encode map e242 with expected crc 2023-10-09T05:57:14.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:14 smithi064 ceph-mon[220044]: failed to encode map e242 with expected crc 2023-10-09T05:57:14.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:14 smithi064 ceph-mon[220044]: failed to encode map e242 with expected crc 2023-10-09T05:57:14.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:14 smithi064 ceph-mon[220044]: failed to encode map e242 with expected crc 2023-10-09T05:57:14.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:14 smithi064 ceph-mon[220044]: failed to encode map e242 with expected crc 2023-10-09T05:57:14.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:14.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:14 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2080677074' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:57:14.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:14 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:57:15.274 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:15.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:15.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:15.378 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:15.418 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:15.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:15 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:57:15.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:15 smithi156 ceph-mon[195101]: failed to encode map e243 with expected crc 2023-10-09T05:57:15.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:15 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:57:15.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:15 smithi156 ceph-mon[195101]: osdmap e243: 8 total, 8 up, 7 in 2023-10-09T05:57:15.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:15 smithi156 ceph-mon[195101]: failed to encode map e243 with expected crc 2023-10-09T05:57:15.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:15 smithi156 ceph-mon[195101]: failed to encode map e243 with expected crc 2023-10-09T05:57:15.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:15 smithi156 ceph-mon[195101]: pgmap v564: 139 pgs: 6 remapped+peering, 1 active+remapped+backfilling, 13 peering, 4 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 88 op/s; 12578/123279 objects degraded (10.203%); 4343/123279 objects misplaced (3.523%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T05:57:15.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:15 smithi156 ceph-mon[195101]: 6.1b scrub starts 2023-10-09T05:57:15.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:57:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:57:15] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.43.0" 2023-10-09T05:57:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:15 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:57:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:15 smithi064 ceph-mon[212750]: failed to encode map e243 with expected crc 2023-10-09T05:57:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:15 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:57:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:15 smithi064 ceph-mon[212750]: osdmap e243: 8 total, 8 up, 7 in 2023-10-09T05:57:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:15 smithi064 ceph-mon[212750]: failed to encode map e243 with expected crc 2023-10-09T05:57:15.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:15 smithi064 ceph-mon[212750]: failed to encode map e243 with expected crc 2023-10-09T05:57:15.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:15 smithi064 ceph-mon[212750]: pgmap v564: 139 pgs: 6 remapped+peering, 1 active+remapped+backfilling, 13 peering, 4 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 88 op/s; 12578/123279 objects degraded (10.203%); 4343/123279 objects misplaced (3.523%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T05:57:15.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:15 smithi064 ceph-mon[212750]: 6.1b scrub starts 2023-10-09T05:57:15.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:15 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:57:15.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:15 smithi064 ceph-mon[220044]: failed to encode map e243 with expected crc 2023-10-09T05:57:15.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:15 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:57:15.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:15 smithi064 ceph-mon[220044]: osdmap e243: 8 total, 8 up, 7 in 2023-10-09T05:57:15.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:15 smithi064 ceph-mon[220044]: failed to encode map e243 with expected crc 2023-10-09T05:57:15.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:15 smithi064 ceph-mon[220044]: failed to encode map e243 with expected crc 2023-10-09T05:57:15.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:15 smithi064 ceph-mon[220044]: pgmap v564: 139 pgs: 6 remapped+peering, 1 active+remapped+backfilling, 13 peering, 4 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 88 op/s; 12578/123279 objects degraded (10.203%); 4343/123279 objects misplaced (3.523%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T05:57:15.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:15 smithi064 ceph-mon[220044]: 6.1b scrub starts 2023-10-09T05:57:16.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:16 smithi156 ceph-mon[195101]: failed to encode map e244 with expected crc 2023-10-09T05:57:16.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:16 smithi156 ceph-mon[195101]: osdmap e244: 8 total, 8 up, 7 in 2023-10-09T05:57:16.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:16 smithi156 ceph-mon[195101]: failed to encode map e244 with expected crc 2023-10-09T05:57:16.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:16 smithi156 ceph-mon[195101]: failed to encode map e244 with expected crc 2023-10-09T05:57:16.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:16 smithi156 ceph-mon[195101]: failed to encode map e244 with expected crc 2023-10-09T05:57:16.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:16 smithi064 ceph-mon[220044]: failed to encode map e244 with expected crc 2023-10-09T05:57:16.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:16 smithi064 ceph-mon[220044]: osdmap e244: 8 total, 8 up, 7 in 2023-10-09T05:57:16.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:16 smithi064 ceph-mon[220044]: failed to encode map e244 with expected crc 2023-10-09T05:57:16.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:16 smithi064 ceph-mon[220044]: failed to encode map e244 with expected crc 2023-10-09T05:57:16.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:16 smithi064 ceph-mon[220044]: failed to encode map e244 with expected crc 2023-10-09T05:57:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:16 smithi064 ceph-mon[212750]: failed to encode map e244 with expected crc 2023-10-09T05:57:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:16 smithi064 ceph-mon[212750]: osdmap e244: 8 total, 8 up, 7 in 2023-10-09T05:57:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:16 smithi064 ceph-mon[212750]: failed to encode map e244 with expected crc 2023-10-09T05:57:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:16 smithi064 ceph-mon[212750]: failed to encode map e244 with expected crc 2023-10-09T05:57:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:16 smithi064 ceph-mon[212750]: failed to encode map e244 with expected crc 2023-10-09T05:57:16.788 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:16.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:16.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:16.892 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:16.935 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:17.042 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:17.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:17.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:17.151 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:17.191 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:17.248 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:17.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:17.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:17.358 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:17.420 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:17.479 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:17.484 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:17 smithi156 ceph-mon[195101]: osdmap e245: 8 total, 8 up, 7 in 2023-10-09T05:57:17.484 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:17 smithi156 ceph-mon[195101]: failed to encode map e245 with expected crc 2023-10-09T05:57:17.484 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:17 smithi156 ceph-mon[195101]: failed to encode map e245 with expected crc 2023-10-09T05:57:17.484 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:17 smithi156 ceph-mon[195101]: failed to encode map e245 with expected crc 2023-10-09T05:57:17.484 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:17 smithi156 ceph-mon[195101]: failed to encode map e245 with expected crc 2023-10-09T05:57:17.485 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:17 smithi156 ceph-mon[195101]: failed to encode map e245 with expected crc 2023-10-09T05:57:17.485 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:17 smithi156 ceph-mon[195101]: pgmap v567: 139 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded+remapped, 118 active+clean, 1 active+undersized+remapped, 6 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 484 op/s; 23460/118533 objects degraded (19.792%); 5981/118533 objects misplaced (5.046%); 29 MiB/s, 739 objects/s recovering 2023-10-09T05:57:17.485 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:17 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 12578/123279 objects degraded (10.203%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:17.485 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:17 smithi156 ceph-mon[195101]: osdmap e246: 8 total, 8 up, 7 in 2023-10-09T05:57:17.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:17.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:17.588 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:17.625 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:17.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:17 smithi064 ceph-mon[212750]: osdmap e245: 8 total, 8 up, 7 in 2023-10-09T05:57:17.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:17 smithi064 ceph-mon[212750]: failed to encode map e245 with expected crc 2023-10-09T05:57:17.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:17 smithi064 ceph-mon[212750]: failed to encode map e245 with expected crc 2023-10-09T05:57:17.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:17 smithi064 ceph-mon[212750]: failed to encode map e245 with expected crc 2023-10-09T05:57:17.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:17 smithi064 ceph-mon[212750]: failed to encode map e245 with expected crc 2023-10-09T05:57:17.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:17 smithi064 ceph-mon[212750]: failed to encode map e245 with expected crc 2023-10-09T05:57:17.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:17 smithi064 ceph-mon[212750]: pgmap v567: 139 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded+remapped, 118 active+clean, 1 active+undersized+remapped, 6 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 484 op/s; 23460/118533 objects degraded (19.792%); 5981/118533 objects misplaced (5.046%); 29 MiB/s, 739 objects/s recovering 2023-10-09T05:57:17.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:17 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 12578/123279 objects degraded (10.203%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:17.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:17 smithi064 ceph-mon[212750]: osdmap e246: 8 total, 8 up, 7 in 2023-10-09T05:57:17.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:17 smithi064 ceph-mon[220044]: osdmap e245: 8 total, 8 up, 7 in 2023-10-09T05:57:17.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:17 smithi064 ceph-mon[220044]: failed to encode map e245 with expected crc 2023-10-09T05:57:17.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:17 smithi064 ceph-mon[220044]: failed to encode map e245 with expected crc 2023-10-09T05:57:17.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:17 smithi064 ceph-mon[220044]: failed to encode map e245 with expected crc 2023-10-09T05:57:17.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:17 smithi064 ceph-mon[220044]: failed to encode map e245 with expected crc 2023-10-09T05:57:17.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:17 smithi064 ceph-mon[220044]: failed to encode map e245 with expected crc 2023-10-09T05:57:17.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:17 smithi064 ceph-mon[220044]: pgmap v567: 139 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded+remapped, 118 active+clean, 1 active+undersized+remapped, 6 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 484 op/s; 23460/118533 objects degraded (19.792%); 5981/118533 objects misplaced (5.046%); 29 MiB/s, 739 objects/s recovering 2023-10-09T05:57:17.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:17 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 12578/123279 objects degraded (10.203%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:17.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:17 smithi064 ceph-mon[220044]: osdmap e246: 8 total, 8 up, 7 in 2023-10-09T05:57:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:17.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:17.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:17.867 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:17.944 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:17.966 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:57:17.966 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:57:17.975 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:18.043 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:18.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:18.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:18.148 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:18.196 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:18.279 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:57:18.316 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:57:18.349 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:57:18.383 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:57:18.424 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:57:18.447 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:57:18.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:18 smithi064 ceph-mon[212750]: failed to encode map e246 with expected crc 2023-10-09T05:57:18.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:18 smithi064 ceph-mon[212750]: osdmap e247: 8 total, 8 up, 7 in 2023-10-09T05:57:18.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:18 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1293517258' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:18.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:18 smithi064 ceph-mon[220044]: failed to encode map e246 with expected crc 2023-10-09T05:57:18.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:18 smithi064 ceph-mon[220044]: osdmap e247: 8 total, 8 up, 7 in 2023-10-09T05:57:18.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:18 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1293517258' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:18.480 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:57:18.507 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:57:18.509 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:57:18.509 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:57:18.509 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:57:18.509 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:57:18.509 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:57:18.510 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:57:18.510 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:57:18.510 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:57:18.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:18 smithi156 ceph-mon[195101]: failed to encode map e246 with expected crc 2023-10-09T05:57:18.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:18 smithi156 ceph-mon[195101]: osdmap e247: 8 total, 8 up, 7 in 2023-10-09T05:57:18.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:18 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1293517258' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:19.571 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:57:19.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:19 smithi064 ceph-mon[212750]: 6.18 deep-scrub starts 2023-10-09T05:57:19.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:19 smithi064 ceph-mon[212750]: 6.b scrub starts 2023-10-09T05:57:19.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:19 smithi064 ceph-mon[212750]: failed to encode map e247 with expected crc 2023-10-09T05:57:19.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:19 smithi064 ceph-mon[212750]: failed to encode map e247 with expected crc 2023-10-09T05:57:19.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:19 smithi064 ceph-mon[212750]: failed to encode map e247 with expected crc 2023-10-09T05:57:19.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:19 smithi064 ceph-mon[212750]: failed to encode map e247 with expected crc 2023-10-09T05:57:19.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:19 smithi064 ceph-mon[212750]: pgmap v570: 139 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded+remapped, 118 active+clean, 1 active+undersized+remapped, 6 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 23203/116349 objects degraded (19.943%); 5891/116349 objects misplaced (5.063%); 28 MiB/s, 721 objects/s recovering 2023-10-09T05:57:19.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:19 smithi064 ceph-mon[220044]: 6.18 deep-scrub starts 2023-10-09T05:57:19.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:19 smithi064 ceph-mon[220044]: 6.b scrub starts 2023-10-09T05:57:19.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:19 smithi064 ceph-mon[220044]: failed to encode map e247 with expected crc 2023-10-09T05:57:19.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:19 smithi064 ceph-mon[220044]: failed to encode map e247 with expected crc 2023-10-09T05:57:19.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:19 smithi064 ceph-mon[220044]: failed to encode map e247 with expected crc 2023-10-09T05:57:19.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:19 smithi064 ceph-mon[220044]: failed to encode map e247 with expected crc 2023-10-09T05:57:19.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:19 smithi064 ceph-mon[220044]: pgmap v570: 139 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded+remapped, 118 active+clean, 1 active+undersized+remapped, 6 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 23203/116349 objects degraded (19.943%); 5891/116349 objects misplaced (5.063%); 28 MiB/s, 721 objects/s recovering 2023-10-09T05:57:19.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:19 smithi156 ceph-mon[195101]: 6.18 deep-scrub starts 2023-10-09T05:57:19.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:19 smithi156 ceph-mon[195101]: 6.b scrub starts 2023-10-09T05:57:19.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:19 smithi156 ceph-mon[195101]: failed to encode map e247 with expected crc 2023-10-09T05:57:19.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:19 smithi156 ceph-mon[195101]: failed to encode map e247 with expected crc 2023-10-09T05:57:19.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:19 smithi156 ceph-mon[195101]: failed to encode map e247 with expected crc 2023-10-09T05:57:19.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:19 smithi156 ceph-mon[195101]: failed to encode map e247 with expected crc 2023-10-09T05:57:19.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:19 smithi156 ceph-mon[195101]: pgmap v570: 139 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded+remapped, 118 active+clean, 1 active+undersized+remapped, 6 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 23203/116349 objects degraded (19.943%); 5891/116349 objects misplaced (5.063%); 28 MiB/s, 721 objects/s recovering 2023-10-09T05:57:20.058 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:20.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:20.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:20.163 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:20.191 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:20.343 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:57:20.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:20 smithi064 ceph-mon[212750]: 6.13 scrub starts 2023-10-09T05:57:20.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:20 smithi064 ceph-mon[220044]: 6.13 scrub starts 2023-10-09T05:57:20.732 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:20.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:20.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:20.844 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:20.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:20 smithi156 ceph-mon[195101]: 6.13 scrub starts 2023-10-09T05:57:20.888 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:20.944 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:21.210 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:21.220 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:21.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:21.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:21.325 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:21.375 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:21.462 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:21.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:21.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:21.573 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:21.612 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:21.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:21 smithi064 ceph-mon[212750]: pgmap v571: 139 pgs: 4 active+recovery_wait+degraded+remapped, 120 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 775 op/s; 21833/107739 objects degraded (20.265%); 5104/107739 objects misplaced (4.737%); 14 MiB/s, 395 objects/s recovering 2023-10-09T05:57:21.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:21 smithi064 ceph-mon[220044]: pgmap v571: 139 pgs: 4 active+recovery_wait+degraded+remapped, 120 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 775 op/s; 21833/107739 objects degraded (20.265%); 5104/107739 objects misplaced (4.737%); 14 MiB/s, 395 objects/s recovering 2023-10-09T05:57:21.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:21 smithi156 ceph-mon[195101]: pgmap v571: 139 pgs: 4 active+recovery_wait+degraded+remapped, 120 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 775 op/s; 21833/107739 objects degraded (20.265%); 5104/107739 objects misplaced (4.737%); 14 MiB/s, 395 objects/s recovering 2023-10-09T05:57:22.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:22 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3740884573' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:57:22.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:22 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:57:22.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:22 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 21833/107739 objects degraded (20.265%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:22.462 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T05:57:22.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:22 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3740884573' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:57:22.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:22 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:57:22.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:22 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 21833/107739 objects degraded (20.265%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:22.726 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:57:22.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:22 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3740884573' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:57:22.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:22 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:57:22.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:22 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 21833/107739 objects degraded (20.265%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:23.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:23 smithi064 ceph-mon[212750]: 6.2b scrub starts 2023-10-09T05:57:23.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:23 smithi064 ceph-mon[212750]: failed to encode map e248 with expected crc 2023-10-09T05:57:23.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:23 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:57:23.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:23 smithi064 ceph-mon[212750]: osdmap e248: 8 total, 8 up, 7 in 2023-10-09T05:57:23.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:23 smithi064 ceph-mon[212750]: failed to encode map e248 with expected crc 2023-10-09T05:57:23.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:23 smithi064 ceph-mon[212750]: failed to encode map e248 with expected crc 2023-10-09T05:57:23.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:23 smithi064 ceph-mon[212750]: failed to encode map e248 with expected crc 2023-10-09T05:57:23.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:23 smithi064 ceph-mon[212750]: failed to encode map e248 with expected crc 2023-10-09T05:57:23.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:23 smithi064 ceph-mon[212750]: failed to encode map e248 with expected crc 2023-10-09T05:57:23.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:23 smithi064 ceph-mon[212750]: failed to encode map e248 with expected crc 2023-10-09T05:57:23.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:23 smithi064 ceph-mon[212750]: pgmap v573: 139 pgs: 4 active+recovery_wait+degraded+remapped, 120 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 21833/107739 objects degraded (20.265%); 5104/107739 objects misplaced (4.737%); 12 MiB/s, 344 objects/s recovering 2023-10-09T05:57:23.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:23 smithi064 ceph-mon[220044]: 6.2b scrub starts 2023-10-09T05:57:23.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:23 smithi064 ceph-mon[220044]: failed to encode map e248 with expected crc 2023-10-09T05:57:23.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:23 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:57:23.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:23 smithi064 ceph-mon[220044]: osdmap e248: 8 total, 8 up, 7 in 2023-10-09T05:57:23.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:23 smithi064 ceph-mon[220044]: failed to encode map e248 with expected crc 2023-10-09T05:57:23.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:23 smithi064 ceph-mon[220044]: failed to encode map e248 with expected crc 2023-10-09T05:57:23.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:23 smithi064 ceph-mon[220044]: failed to encode map e248 with expected crc 2023-10-09T05:57:23.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:23 smithi064 ceph-mon[220044]: failed to encode map e248 with expected crc 2023-10-09T05:57:23.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:23 smithi064 ceph-mon[220044]: failed to encode map e248 with expected crc 2023-10-09T05:57:23.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:23 smithi064 ceph-mon[220044]: failed to encode map e248 with expected crc 2023-10-09T05:57:23.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:23 smithi064 ceph-mon[220044]: pgmap v573: 139 pgs: 4 active+recovery_wait+degraded+remapped, 120 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 21833/107739 objects degraded (20.265%); 5104/107739 objects misplaced (4.737%); 12 MiB/s, 344 objects/s recovering 2023-10-09T05:57:23.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:23 smithi156 ceph-mon[195101]: 6.2b scrub starts 2023-10-09T05:57:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:23 smithi156 ceph-mon[195101]: failed to encode map e248 with expected crc 2023-10-09T05:57:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:23 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:57:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:23 smithi156 ceph-mon[195101]: osdmap e248: 8 total, 8 up, 7 in 2023-10-09T05:57:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:23 smithi156 ceph-mon[195101]: failed to encode map e248 with expected crc 2023-10-09T05:57:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:23 smithi156 ceph-mon[195101]: failed to encode map e248 with expected crc 2023-10-09T05:57:23.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:23 smithi156 ceph-mon[195101]: failed to encode map e248 with expected crc 2023-10-09T05:57:23.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:23 smithi156 ceph-mon[195101]: failed to encode map e248 with expected crc 2023-10-09T05:57:23.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:23 smithi156 ceph-mon[195101]: failed to encode map e248 with expected crc 2023-10-09T05:57:23.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:23 smithi156 ceph-mon[195101]: failed to encode map e248 with expected crc 2023-10-09T05:57:23.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:23 smithi156 ceph-mon[195101]: pgmap v573: 139 pgs: 4 active+recovery_wait+degraded+remapped, 120 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 21833/107739 objects degraded (20.265%); 5104/107739 objects misplaced (4.737%); 12 MiB/s, 344 objects/s recovering 2023-10-09T05:57:23.967 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:23.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:23.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:24.073 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:24.103 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:24.719 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T05:57:24.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:24 smithi064 ceph-mon[212750]: failed to encode map e249 with expected crc 2023-10-09T05:57:24.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:24 smithi064 ceph-mon[212750]: failed to encode map e249 with expected crc 2023-10-09T05:57:24.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:24 smithi064 ceph-mon[212750]: failed to encode map e249 with expected crc 2023-10-09T05:57:24.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:24 smithi064 ceph-mon[212750]: osdmap e249: 8 total, 8 up, 7 in 2023-10-09T05:57:24.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:24 smithi064 ceph-mon[212750]: failed to encode map e249 with expected crc 2023-10-09T05:57:24.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:24 smithi064 ceph-mon[212750]: failed to encode map e249 with expected crc 2023-10-09T05:57:24.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:24 smithi064 ceph-mon[212750]: failed to encode map e249 with expected crc 2023-10-09T05:57:24.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:24 smithi064 ceph-mon[212750]: failed to encode map e249 with expected crc 2023-10-09T05:57:24.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:24 smithi064 ceph-mon[212750]: 1.13 scrub starts 2023-10-09T05:57:24.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:24 smithi064 ceph-mon[220044]: failed to encode map e249 with expected crc 2023-10-09T05:57:24.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:24 smithi064 ceph-mon[220044]: failed to encode map e249 with expected crc 2023-10-09T05:57:24.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:24 smithi064 ceph-mon[220044]: failed to encode map e249 with expected crc 2023-10-09T05:57:24.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:24 smithi064 ceph-mon[220044]: osdmap e249: 8 total, 8 up, 7 in 2023-10-09T05:57:24.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:24 smithi064 ceph-mon[220044]: failed to encode map e249 with expected crc 2023-10-09T05:57:24.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:24 smithi064 ceph-mon[220044]: failed to encode map e249 with expected crc 2023-10-09T05:57:24.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:24 smithi064 ceph-mon[220044]: failed to encode map e249 with expected crc 2023-10-09T05:57:24.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:24 smithi064 ceph-mon[220044]: failed to encode map e249 with expected crc 2023-10-09T05:57:24.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:24 smithi064 ceph-mon[220044]: 1.13 scrub starts 2023-10-09T05:57:24.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:24 smithi156 ceph-mon[195101]: failed to encode map e249 with expected crc 2023-10-09T05:57:24.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:24 smithi156 ceph-mon[195101]: failed to encode map e249 with expected crc 2023-10-09T05:57:24.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:24 smithi156 ceph-mon[195101]: failed to encode map e249 with expected crc 2023-10-09T05:57:24.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:24 smithi156 ceph-mon[195101]: osdmap e249: 8 total, 8 up, 7 in 2023-10-09T05:57:24.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:24 smithi156 ceph-mon[195101]: failed to encode map e249 with expected crc 2023-10-09T05:57:24.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:24 smithi156 ceph-mon[195101]: failed to encode map e249 with expected crc 2023-10-09T05:57:24.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:24 smithi156 ceph-mon[195101]: failed to encode map e249 with expected crc 2023-10-09T05:57:24.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:24 smithi156 ceph-mon[195101]: failed to encode map e249 with expected crc 2023-10-09T05:57:24.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:24 smithi156 ceph-mon[195101]: 1.13 scrub starts 2023-10-09T05:57:24.937 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:24.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:24.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:25.042 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:25.088 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:25.285 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:25.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:25.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:25.397 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:57:25.411 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:25.424 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:25.442 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:25.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:57:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:57:25] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.43.0" 2023-10-09T05:57:25.469 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:57:25.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:25.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:57:25.578 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:25.615 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:25.697 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:25.705 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:25 smithi156 ceph-mon[195101]: 6.7 scrub starts 2023-10-09T05:57:25.705 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:25 smithi156 ceph-mon[195101]: 6.7 scrub ok 2023-10-09T05:57:25.705 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:25 smithi156 ceph-mon[195101]: failed to encode map e250 with expected crc 2023-10-09T05:57:25.705 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:25 smithi156 ceph-mon[195101]: osdmap e250: 8 total, 8 up, 7 in 2023-10-09T05:57:25.705 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:25 smithi156 ceph-mon[195101]: failed to encode map e250 with expected crc 2023-10-09T05:57:25.706 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:25 smithi156 ceph-mon[195101]: failed to encode map e250 with expected crc 2023-10-09T05:57:25.706 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:25 smithi156 ceph-mon[195101]: failed to encode map e250 with expected crc 2023-10-09T05:57:25.706 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:25 smithi156 ceph-mon[195101]: 6.22 starting backfill to osd.3 from (0'0,0'0] MAX to 247'7494 2023-10-09T05:57:25.706 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:25 smithi156 ceph-mon[195101]: 6.12 starting backfill to osd.3 from (0'0,0'0] MAX to 247'8298 2023-10-09T05:57:25.706 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:25 smithi156 ceph-mon[195101]: 6.2 starting backfill to osd.3 from (0'0,0'0] MAX to 247'7493 2023-10-09T05:57:25.706 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:25 smithi156 ceph-mon[195101]: 6.1c starting backfill to osd.3 from (0'0,0'0] MAX to 247'8355 2023-10-09T05:57:25.706 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:25 smithi156 ceph-mon[195101]: 6.20 continuing backfill to osd.2 from (0'0,247'7519] MIN to 247'7519 2023-10-09T05:57:25.706 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:25 smithi156 ceph-mon[195101]: 6.2c starting backfill to osd.3 from (0'0,0'0] MAX to 247'8179 2023-10-09T05:57:25.706 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:25 smithi156 ceph-mon[195101]: 6.c starting backfill to osd.3 from (0'0,0'0] MAX to 247'8183 2023-10-09T05:57:25.707 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:25 smithi156 ceph-mon[195101]: pgmap v576: 139 pgs: 5 activating+undersized+remapped, 4 remapped+peering, 4 peering, 2 active+recovery_wait+degraded+remapped, 110 active+clean, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 23505/105183 objects degraded (22.347%); 4021/105183 objects misplaced (3.823%); 11 MiB/s, 344 objects/s recovering 2023-10-09T05:57:25.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:25.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:25.808 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:25.814 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[212750]: 6.7 scrub starts 2023-10-09T05:57:25.815 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[212750]: 6.7 scrub ok 2023-10-09T05:57:25.815 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[212750]: failed to encode map e250 with expected crc 2023-10-09T05:57:25.815 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[212750]: osdmap e250: 8 total, 8 up, 7 in 2023-10-09T05:57:25.815 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[212750]: failed to encode map e250 with expected crc 2023-10-09T05:57:25.815 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[212750]: failed to encode map e250 with expected crc 2023-10-09T05:57:25.818 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[212750]: failed to encode map e250 with expected crc 2023-10-09T05:57:25.818 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[212750]: 6.22 starting backfill to osd.3 from (0'0,0'0] MAX to 247'7494 2023-10-09T05:57:25.818 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[212750]: 6.12 starting backfill to osd.3 from (0'0,0'0] MAX to 247'8298 2023-10-09T05:57:25.818 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[212750]: 6.2 starting backfill to osd.3 from (0'0,0'0] MAX to 247'7493 2023-10-09T05:57:25.818 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[212750]: 6.1c starting backfill to osd.3 from (0'0,0'0] MAX to 247'8355 2023-10-09T05:57:25.818 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[212750]: 6.20 continuing backfill to osd.2 from (0'0,247'7519] MIN to 247'7519 2023-10-09T05:57:25.818 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[212750]: 6.2c starting backfill to osd.3 from (0'0,0'0] MAX to 247'8179 2023-10-09T05:57:25.818 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[212750]: 6.c starting backfill to osd.3 from (0'0,0'0] MAX to 247'8183 2023-10-09T05:57:25.818 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[212750]: pgmap v576: 139 pgs: 5 activating+undersized+remapped, 4 remapped+peering, 4 peering, 2 active+recovery_wait+degraded+remapped, 110 active+clean, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 23505/105183 objects degraded (22.347%); 4021/105183 objects misplaced (3.823%); 11 MiB/s, 344 objects/s recovering 2023-10-09T05:57:25.819 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[220044]: 6.7 scrub starts 2023-10-09T05:57:25.819 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[220044]: 6.7 scrub ok 2023-10-09T05:57:25.819 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[220044]: failed to encode map e250 with expected crc 2023-10-09T05:57:25.819 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[220044]: osdmap e250: 8 total, 8 up, 7 in 2023-10-09T05:57:25.819 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[220044]: failed to encode map e250 with expected crc 2023-10-09T05:57:25.819 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[220044]: failed to encode map e250 with expected crc 2023-10-09T05:57:25.819 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[220044]: failed to encode map e250 with expected crc 2023-10-09T05:57:25.819 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[220044]: 6.22 starting backfill to osd.3 from (0'0,0'0] MAX to 247'7494 2023-10-09T05:57:25.819 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[220044]: 6.12 starting backfill to osd.3 from (0'0,0'0] MAX to 247'8298 2023-10-09T05:57:25.819 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[220044]: 6.2 starting backfill to osd.3 from (0'0,0'0] MAX to 247'7493 2023-10-09T05:57:25.820 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[220044]: 6.1c starting backfill to osd.3 from (0'0,0'0] MAX to 247'8355 2023-10-09T05:57:25.820 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[220044]: 6.20 continuing backfill to osd.2 from (0'0,247'7519] MIN to 247'7519 2023-10-09T05:57:25.820 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[220044]: 6.2c starting backfill to osd.3 from (0'0,0'0] MAX to 247'8179 2023-10-09T05:57:25.820 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[220044]: 6.c starting backfill to osd.3 from (0'0,0'0] MAX to 247'8183 2023-10-09T05:57:25.820 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:25 smithi064 ceph-mon[220044]: pgmap v576: 139 pgs: 5 activating+undersized+remapped, 4 remapped+peering, 4 peering, 2 active+recovery_wait+degraded+remapped, 110 active+clean, 6 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 23505/105183 objects degraded (22.347%); 4021/105183 objects misplaced (3.823%); 11 MiB/s, 344 objects/s recovering 2023-10-09T05:57:25.897 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:25.952 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:57:25.952 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T05:57:25.952 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T05:57:25.953 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T05:57:25.953 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.953 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82093 6.0 6.a3a5430 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:57:25.953 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.146555+0000", 2023-10-09T05:57:25.953 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 329.80216002100002, 2023-10-09T05:57:25.953 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93485150500000003, 2023-10-09T05:57:25.953 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:25.953 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:57:25.953 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:25.953 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:25.953 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:25.953 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82093 2023-10-09T05:57:25.954 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.954 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:25.954 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.954 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:25.954 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.146555+0000", 2023-10-09T05:57:25.954 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:25.954 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.954 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.954 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:25.954 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.146555+0000", 2023-10-09T05:57:25.954 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:25.955 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.955 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.955 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:25.955 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.146553+0000", 2023-10-09T05:57:25.955 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:57:25.955 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.955 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.955 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:25.955 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.146607+0000", 2023-10-09T05:57:25.955 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4317000000000002e-05 2023-10-09T05:57:25.955 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.955 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.956 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:25.956 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.146636+0000", 2023-10-09T05:57:25.956 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8294999999999999e-05 2023-10-09T05:57:25.956 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.956 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.956 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:25.956 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.146647+0000", 2023-10-09T05:57:25.956 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1513e-05 2023-10-09T05:57:25.956 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.956 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.956 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:25.956 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.150385+0000", 2023-10-09T05:57:25.957 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037372640000000001 2023-10-09T05:57:25.957 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.957 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.957 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:57:25.957 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.150398+0000", 2023-10-09T05:57:25.957 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.34e-05 2023-10-09T05:57:25.957 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.957 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.957 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:25.957 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081393+0000", 2023-10-09T05:57:25.957 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93099492699999997 2023-10-09T05:57:25.957 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.957 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.958 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:25.958 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081406+0000", 2023-10-09T05:57:25.958 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.359e-05 2023-10-09T05:57:25.958 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:25.958 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:25.958 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:25.958 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.958 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.958 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82107 6.0 6.432cde0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:57:25.958 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.173816+0000", 2023-10-09T05:57:25.958 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 329.77489864900002, 2023-10-09T05:57:25.958 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90762603100000006, 2023-10-09T05:57:25.959 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:25.959 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:57:25.959 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:25.959 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:25.959 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:25.959 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82107 2023-10-09T05:57:25.959 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.959 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:25.959 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.959 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:25.959 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173816+0000", 2023-10-09T05:57:25.959 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:25.959 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.960 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.960 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:25.960 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173816+0000", 2023-10-09T05:57:25.960 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:25.960 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.960 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.960 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:25.960 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173814+0000", 2023-10-09T05:57:25.960 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:57:25.960 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.960 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.960 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:25.961 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173867+0000", 2023-10-09T05:57:25.961 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2862999999999999e-05 2023-10-09T05:57:25.961 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.961 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.961 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:25.961 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173887+0000", 2023-10-09T05:57:25.961 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0347e-05 2023-10-09T05:57:25.961 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.961 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.961 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:25.961 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173907+0000", 2023-10-09T05:57:25.961 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9782000000000001e-05 2023-10-09T05:57:25.962 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.962 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.962 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:25.962 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173983+0000", 2023-10-09T05:57:25.962 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6056000000000005e-05 2023-10-09T05:57:25.962 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.962 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.962 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:57:25.962 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173996+0000", 2023-10-09T05:57:25.962 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3237999999999999e-05 2023-10-09T05:57:25.962 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.962 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.963 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:25.963 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081433+0000", 2023-10-09T05:57:25.963 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90743656100000003 2023-10-09T05:57:25.963 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.963 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.963 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:25.963 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081442+0000", 2023-10-09T05:57:25.963 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5070000000000006e-06 2023-10-09T05:57:25.963 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:25.963 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:25.963 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:25.963 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.963 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.964 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82109 6.0 6.cee0ea60 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:57:25.964 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.183890+0000", 2023-10-09T05:57:25.964 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 329.76482451599998, 2023-10-09T05:57:25.964 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.89758420800000005, 2023-10-09T05:57:25.964 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:25.964 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:57:25.964 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:25.964 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:25.964 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:25.964 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82109 2023-10-09T05:57:25.964 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.964 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:25.965 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.965 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:25.965 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.183890+0000", 2023-10-09T05:57:25.965 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:25.965 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.965 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.965 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:25.965 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.183890+0000", 2023-10-09T05:57:25.965 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:25.965 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.965 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.965 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:25.965 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.183889+0000", 2023-10-09T05:57:25.966 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:57:25.966 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.966 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.966 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:25.966 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.184035+0000", 2023-10-09T05:57:25.966 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000145375 2023-10-09T05:57:25.966 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.966 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.966 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:25.966 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.184048+0000", 2023-10-09T05:57:25.966 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3730999999999999e-05 2023-10-09T05:57:25.966 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.967 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.967 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:25.967 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.184060+0000", 2023-10-09T05:57:25.967 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1338000000000001e-05 2023-10-09T05:57:25.967 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.967 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.967 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:25.967 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.184099+0000", 2023-10-09T05:57:25.967 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9183000000000001e-05 2023-10-09T05:57:25.967 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.967 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.967 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:57:25.967 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.184113+0000", 2023-10-09T05:57:25.968 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3750999999999999e-05 2023-10-09T05:57:25.968 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.968 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.968 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:25.968 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081465+0000", 2023-10-09T05:57:25.968 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.89735238100000003 2023-10-09T05:57:25.968 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.968 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.968 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:25.968 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081475+0000", 2023-10-09T05:57:25.968 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7950000000000002e-06 2023-10-09T05:57:25.968 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:25.969 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:25.969 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:25.969 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.969 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.969 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82116 6.9 6.318f1fa9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:57:25.969 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.194331+0000", 2023-10-09T05:57:25.969 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 329.754383813, 2023-10-09T05:57:25.969 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.888951253, 2023-10-09T05:57:25.969 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:25.969 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:57:25.969 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:25.969 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:25.969 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:25.970 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82116 2023-10-09T05:57:25.970 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.970 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:25.970 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.970 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:25.970 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194331+0000", 2023-10-09T05:57:25.970 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:25.970 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.970 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.970 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:25.970 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194331+0000", 2023-10-09T05:57:25.970 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:25.971 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.971 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.971 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:25.971 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194329+0000", 2023-10-09T05:57:25.971 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:57:25.971 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.971 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.971 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:25.971 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194387+0000", 2023-10-09T05:57:25.971 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7734e-05 2023-10-09T05:57:25.971 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.971 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.971 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:25.972 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194406+0000", 2023-10-09T05:57:25.972 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9304e-05 2023-10-09T05:57:25.972 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.972 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.972 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:25.972 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194417+0000", 2023-10-09T05:57:25.972 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0803e-05 2023-10-09T05:57:25.972 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.972 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.972 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:25.972 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194490+0000", 2023-10-09T05:57:25.972 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2939000000000001e-05 2023-10-09T05:57:25.973 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.973 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.973 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:57:25.973 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194504+0000", 2023-10-09T05:57:25.973 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4358e-05 2023-10-09T05:57:25.973 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.973 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.973 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:25.973 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083265+0000", 2023-10-09T05:57:25.973 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88876104300000003 2023-10-09T05:57:25.973 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.973 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.973 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:25.974 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083282+0000", 2023-10-09T05:57:25.974 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6945000000000001e-05 2023-10-09T05:57:25.974 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:25.974 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:25.974 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:25.974 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.974 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.974 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82117 6.9 6.b6ae8c09 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:57:25.974 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.200001+0000", 2023-10-09T05:57:25.974 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 329.74871370699998, 2023-10-09T05:57:25.974 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88338774900000006, 2023-10-09T05:57:25.974 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:25.974 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:57:25.975 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:25.977 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:25.977 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:25.977 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82117 2023-10-09T05:57:25.977 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.977 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:25.977 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.977 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:25.977 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200001+0000", 2023-10-09T05:57:25.977 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:25.978 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.978 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.978 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:25.978 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200001+0000", 2023-10-09T05:57:25.978 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:25.978 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.978 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.978 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:25.978 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.199999+0000", 2023-10-09T05:57:25.978 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:57:25.978 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.978 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.979 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:25.979 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200060+0000", 2023-10-09T05:57:25.979 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0708999999999998e-05 2023-10-09T05:57:25.979 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.979 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.979 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:25.979 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200090+0000", 2023-10-09T05:57:25.979 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0085000000000001e-05 2023-10-09T05:57:25.979 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.979 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.979 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:25.979 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200101+0000", 2023-10-09T05:57:25.980 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.06e-05 2023-10-09T05:57:25.980 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.980 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.980 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:25.980 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200179+0000", 2023-10-09T05:57:25.980 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7848000000000004e-05 2023-10-09T05:57:25.980 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.980 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.980 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:57:25.980 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200192+0000", 2023-10-09T05:57:25.980 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3943e-05 2023-10-09T05:57:25.980 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.980 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.981 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:25.981 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083361+0000", 2023-10-09T05:57:25.981 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88316805099999995 2023-10-09T05:57:25.981 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.981 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.981 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:25.981 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083389+0000", 2023-10-09T05:57:25.981 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8532000000000001e-05 2023-10-09T05:57:25.981 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:25.981 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:25.981 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:25.981 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.982 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.982 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82118 6.9 6.aad68289 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:57:25.982 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.200117+0000", 2023-10-09T05:57:25.982 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 329.748598206, 2023-10-09T05:57:25.982 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88331848099999999, 2023-10-09T05:57:25.982 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:25.982 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:57:25.982 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:25.982 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:25.982 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:25.982 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82118 2023-10-09T05:57:25.982 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.982 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:25.983 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.983 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:25.983 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200117+0000", 2023-10-09T05:57:25.983 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:25.983 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.983 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.983 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:25.983 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200117+0000", 2023-10-09T05:57:25.983 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:25.983 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.983 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.983 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:25.984 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200116+0000", 2023-10-09T05:57:25.984 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:25.984 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.984 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.984 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:25.984 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200162+0000", 2023-10-09T05:57:25.984 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5788999999999998e-05 2023-10-09T05:57:25.984 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.984 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.984 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:25.984 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200174+0000", 2023-10-09T05:57:25.984 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1905999999999999e-05 2023-10-09T05:57:25.984 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.985 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.985 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:25.985 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200181+0000", 2023-10-09T05:57:25.985 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2690000000000002e-06 2023-10-09T05:57:25.985 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.985 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.985 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:25.985 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200224+0000", 2023-10-09T05:57:25.985 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2839000000000001e-05 2023-10-09T05:57:25.985 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.985 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.985 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:57:25.985 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200233+0000", 2023-10-09T05:57:25.986 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2490000000000007e-06 2023-10-09T05:57:25.986 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.986 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.986 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:25.986 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083420+0000", 2023-10-09T05:57:25.986 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88318651100000001 2023-10-09T05:57:25.986 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.986 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.986 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:25.986 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083435+0000", 2023-10-09T05:57:25.986 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5474000000000001e-05 2023-10-09T05:57:25.986 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:25.987 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:25.987 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:25.987 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.987 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.987 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82125 6.0 6.dade3890 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:57:25.987 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.207895+0000", 2023-10-09T05:57:25.987 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 329.74082000300001, 2023-10-09T05:57:25.987 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.873606989, 2023-10-09T05:57:25.987 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:25.987 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:57:25.987 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:25.987 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:25.988 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:25.988 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82125 2023-10-09T05:57:25.988 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.988 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:25.988 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.988 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:25.988 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.207895+0000", 2023-10-09T05:57:25.988 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:25.988 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.988 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.988 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:25.988 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.207895+0000", 2023-10-09T05:57:25.989 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:25.989 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.989 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.989 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:25.989 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.207893+0000", 2023-10-09T05:57:25.989 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:57:25.989 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.989 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.989 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:25.989 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.207955+0000", 2023-10-09T05:57:25.989 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2361999999999997e-05 2023-10-09T05:57:25.989 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.989 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.990 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:25.990 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.207982+0000", 2023-10-09T05:57:25.990 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6838e-05 2023-10-09T05:57:25.990 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.990 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.990 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:25.990 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.207995+0000", 2023-10-09T05:57:25.990 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2992e-05 2023-10-09T05:57:25.990 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.990 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.990 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:25.991 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.208064+0000", 2023-10-09T05:57:25.991 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9366e-05 2023-10-09T05:57:25.991 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.991 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.991 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:57:25.991 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.208080+0000", 2023-10-09T05:57:25.991 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5690000000000001e-05 2023-10-09T05:57:25.991 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.991 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.991 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:25.991 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081493+0000", 2023-10-09T05:57:25.991 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.87341295699999999 2023-10-09T05:57:25.991 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.992 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.992 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:25.992 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081502+0000", 2023-10-09T05:57:25.992 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1030000000000008e-06 2023-10-09T05:57:25.992 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:25.992 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:25.992 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:25.992 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.992 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.992 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82137 6.0 6.90177890 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:57:25.992 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.236426+0000", 2023-10-09T05:57:25.992 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 329.71228882999998, 2023-10-09T05:57:25.992 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.84511718000000002, 2023-10-09T05:57:25.993 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:25.993 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:57:25.993 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:25.993 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:25.993 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:25.993 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82137 2023-10-09T05:57:25.993 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.993 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:25.993 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.993 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:25.993 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236426+0000", 2023-10-09T05:57:25.993 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:25.994 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.994 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.994 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:25.994 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236426+0000", 2023-10-09T05:57:25.994 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:25.994 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.994 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.994 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:25.994 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236424+0000", 2023-10-09T05:57:25.994 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:57:25.994 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.994 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.995 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:25.995 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236475+0000", 2023-10-09T05:57:25.995 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0819999999999998e-05 2023-10-09T05:57:25.995 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.995 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.995 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:25.995 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236499+0000", 2023-10-09T05:57:25.995 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3669e-05 2023-10-09T05:57:25.995 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.995 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.995 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:25.995 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236510+0000", 2023-10-09T05:57:25.996 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1258e-05 2023-10-09T05:57:25.996 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.996 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.996 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:25.996 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236585+0000", 2023-10-09T05:57:25.996 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4828999999999998e-05 2023-10-09T05:57:25.996 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.996 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.996 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:57:25.996 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236601+0000", 2023-10-09T05:57:25.996 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6308999999999999e-05 2023-10-09T05:57:25.996 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.996 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.997 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:25.997 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081531+0000", 2023-10-09T05:57:25.997 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.84493010099999999 2023-10-09T05:57:25.997 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.997 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.997 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:25.997 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081543+0000", 2023-10-09T05:57:25.997 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2048000000000001e-05 2023-10-09T05:57:25.997 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:25.997 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:25.997 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:25.997 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.998 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.998 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104164 6.9 6:9795f466:::benchmark_data_smithi064_284193_object7621:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:57:25.998 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:40.474997+0000", 2023-10-09T05:57:25.998 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 165.473717722, 2023-10-09T05:57:25.998 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6425455449999999, 2023-10-09T05:57:25.998 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:25.998 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:25.998 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:25.998 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:25.998 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:25.998 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104164 2023-10-09T05:57:25.998 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.998 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:25.999 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.999 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:25.999 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.474997+0000", 2023-10-09T05:57:25.999 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:25.999 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.999 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.999 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:25.999 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.474997+0000", 2023-10-09T05:57:25.999 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:25.999 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:25.999 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:25.999 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:26.000 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.474995+0000", 2023-10-09T05:57:26.000 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:57:26.000 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.000 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.000 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:26.000 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.475006+0000", 2023-10-09T05:57:26.000 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1127e-05 2023-10-09T05:57:26.000 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.000 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.000 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:26.000 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.475022+0000", 2023-10-09T05:57:26.000 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6073999999999999e-05 2023-10-09T05:57:26.001 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.001 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.001 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:26.001 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.475034+0000", 2023-10-09T05:57:26.001 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.183e-05 2023-10-09T05:57:26.001 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.001 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.001 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.001 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.701067+0000", 2023-10-09T05:57:26.001 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2260326539999999 2023-10-09T05:57:26.001 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.001 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.001 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:57:26.002 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.733298+0000", 2023-10-09T05:57:26.002 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.032231152999999998 2023-10-09T05:57:26.002 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.002 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.002 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.002 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.911525+0000", 2023-10-09T05:57:26.002 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.178227464 2023-10-09T05:57:26.002 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.002 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.002 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:57:26.002 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.921036+0000", 2023-10-09T05:57:26.003 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.009510447 2023-10-09T05:57:26.003 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.003 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.003 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:57:26.003 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.934794+0000", 2023-10-09T05:57:26.003 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013758014000000001 2023-10-09T05:57:26.003 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.003 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.003 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:57:26.003 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.041761+0000", 2023-10-09T05:57:26.003 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.10696707699999999 2023-10-09T05:57:26.003 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.004 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.004 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:26.004 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.078719+0000", 2023-10-09T05:57:26.004 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.036957894999999998 2023-10-09T05:57:26.004 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.004 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.004 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:26.004 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.081584+0000", 2023-10-09T05:57:26.004 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002865233 2023-10-09T05:57:26.004 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.004 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.004 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:57:26.005 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.085706+0000", 2023-10-09T05:57:26.005 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041217950000000001 2023-10-09T05:57:26.005 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.005 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.005 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:26.005 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.117543+0000", 2023-10-09T05:57:26.005 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.031837034 2023-10-09T05:57:26.005 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.005 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:26.005 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.005 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.005 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.005 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104221 6.9 6:992da745:::benchmark_data_smithi064_284193_object7678:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:57:26.006 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:41.522885+0000", 2023-10-09T05:57:26.006 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 164.42583043900001, 2023-10-09T05:57:26.006 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.645910236, 2023-10-09T05:57:26.006 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:26.006 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:26.006 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:26.006 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:26.006 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:26.006 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104221 2023-10-09T05:57:26.006 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.006 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:26.006 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.007 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:26.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.522885+0000", 2023-10-09T05:57:26.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:26.007 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.007 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.007 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:26.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.522885+0000", 2023-10-09T05:57:26.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:26.007 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.007 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.007 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:26.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.522883+0000", 2023-10-09T05:57:26.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:57:26.008 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.008 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.008 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:26.008 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.522890+0000", 2023-10-09T05:57:26.008 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1389999999999998e-06 2023-10-09T05:57:26.008 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.008 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.008 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:26.008 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.522906+0000", 2023-10-09T05:57:26.008 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6056e-05 2023-10-09T05:57:26.008 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.008 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.009 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:26.009 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.522916+0000", 2023-10-09T05:57:26.009 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0098999999999999e-05 2023-10-09T05:57:26.009 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.009 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.009 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.009 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.734737+0000", 2023-10-09T05:57:26.009 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.211820541 2023-10-09T05:57:26.009 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.009 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.009 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:57:26.009 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.756962+0000", 2023-10-09T05:57:26.009 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.022225556 2023-10-09T05:57:26.010 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.010 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.010 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.010 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.001519+0000", 2023-10-09T05:57:26.010 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24455652899999999 2023-10-09T05:57:26.010 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.010 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.010 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:57:26.010 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.008693+0000", 2023-10-09T05:57:26.010 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0071741890000000001 2023-10-09T05:57:26.010 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.010 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.011 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:57:26.011 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.024456+0000", 2023-10-09T05:57:26.011 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015762548000000001 2023-10-09T05:57:26.011 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.011 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.011 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:57:26.011 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.075878+0000", 2023-10-09T05:57:26.011 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.051422622000000001 2023-10-09T05:57:26.011 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.011 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.011 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:26.011 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.122918+0000", 2023-10-09T05:57:26.012 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.047039865 2023-10-09T05:57:26.012 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.012 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.012 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:26.012 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.127710+0000", 2023-10-09T05:57:26.012 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047916829999999997 2023-10-09T05:57:26.012 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.012 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.012 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:57:26.012 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.136431+0000", 2023-10-09T05:57:26.012 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0087210410000000006 2023-10-09T05:57:26.012 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.013 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.013 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:26.013 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.168795+0000", 2023-10-09T05:57:26.013 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.032364048999999999 2023-10-09T05:57:26.013 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.013 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:26.013 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.013 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.013 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.013 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104231 6.9 6:9a9d673b:::benchmark_data_smithi064_284193_object7688:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:57:26.013 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:41.584372+0000", 2023-10-09T05:57:26.013 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 164.364343381, 2023-10-09T05:57:26.014 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1037428409999999, 2023-10-09T05:57:26.014 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:26.014 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:26.014 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:26.014 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:26.014 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:26.014 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104231 2023-10-09T05:57:26.014 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.014 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:26.014 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.014 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:26.014 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.584372+0000", 2023-10-09T05:57:26.014 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:26.015 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.015 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.015 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:26.015 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.584372+0000", 2023-10-09T05:57:26.015 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:26.015 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.015 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.015 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:26.015 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.584370+0000", 2023-10-09T05:57:26.015 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:57:26.015 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.015 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.016 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:26.016 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.584377+0000", 2023-10-09T05:57:26.016 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6560000000000003e-06 2023-10-09T05:57:26.016 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.016 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:26.016 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.584391+0000", 2023-10-09T05:57:26.016 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3842000000000001e-05 2023-10-09T05:57:26.016 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.016 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:26.016 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.584398+0000", 2023-10-09T05:57:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.34e-06 2023-10-09T05:57:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.170598+0000", 2023-10-09T05:57:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5861996390000002 2023-10-09T05:57:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:57:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.208581+0000", 2023-10-09T05:57:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.037983153999999998 2023-10-09T05:57:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.017 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.018 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.018 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.354464+0000", 2023-10-09T05:57:26.018 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14588357699999999 2023-10-09T05:57:26.018 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.018 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:57:26.018 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.362830+0000", 2023-10-09T05:57:26.018 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0083655650000000002 2023-10-09T05:57:26.018 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.018 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:57:26.018 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.387897+0000", 2023-10-09T05:57:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.025067605999999999 2023-10-09T05:57:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:57:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.541824+0000", 2023-10-09T05:57:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15392655499999999 2023-10-09T05:57:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.545759+0000", 2023-10-09T05:57:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039354539999999997 2023-10-09T05:57:26.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.640753+0000", 2023-10-09T05:57:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.094993853000000003 2023-10-09T05:57:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:57:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.650699+0000", 2023-10-09T05:57:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0099456360000000008 2023-10-09T05:57:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:26.020 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.688114+0000", 2023-10-09T05:57:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.037415551999999998 2023-10-09T05:57:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104288 6.9 6:97b881c9:::benchmark_data_smithi064_284193_object7745:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:57:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:42.618333+0000", 2023-10-09T05:57:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 163.33038151599999, 2023-10-09T05:57:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1678057799999999, 2023-10-09T05:57:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:26.021 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:26.022 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:26.022 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:26.022 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:26.022 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104288 2023-10-09T05:57:26.022 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.022 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:26.022 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.022 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:26.022 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.618333+0000", 2023-10-09T05:57:26.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:26.022 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.022 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:26.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.618333+0000", 2023-10-09T05:57:26.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:26.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:26.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.618331+0000", 2023-10-09T05:57:26.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:57:26.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:26.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.618341+0000", 2023-10-09T05:57:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3049999999999998e-06 2023-10-09T05:57:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.618366+0000", 2023-10-09T05:57:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5198000000000002e-05 2023-10-09T05:57:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.618379+0000", 2023-10-09T05:57:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2683e-05 2023-10-09T05:57:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.024 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.210649+0000", 2023-10-09T05:57:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.59226995500000001 2023-10-09T05:57:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:57:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.258221+0000", 2023-10-09T05:57:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.047572956 2023-10-09T05:57:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.025 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.480425+0000", 2023-10-09T05:57:26.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.22220395700000001 2023-10-09T05:57:26.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.026 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:57:26.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.495694+0000", 2023-10-09T05:57:26.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015268805999999999 2023-10-09T05:57:26.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.026 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:57:26.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.524222+0000", 2023-10-09T05:57:26.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.028527800999999998 2023-10-09T05:57:26.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:57:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.593453+0000", 2023-10-09T05:57:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.069230594000000006 2023-10-09T05:57:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.732762+0000", 2023-10-09T05:57:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.139309778 2023-10-09T05:57:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:26.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.738937+0000", 2023-10-09T05:57:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061743409999999999 2023-10-09T05:57:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:57:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.749997+0000", 2023-10-09T05:57:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011059897000000001 2023-10-09T05:57:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.786139+0000", 2023-10-09T05:57:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.036142625999999997 2023-10-09T05:57:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.028 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPushReply(6.9 204/189 [PushReplyOp(6:94d4d87b:::benchmark_data_smithi064_284193_object70172:head)])", 2023-10-09T05:57:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:08.385201+0000", 2023-10-09T05:57:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 137.56351385100001, 2023-10-09T05:57:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4252824470000001, 2023-10-09T05:57:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:57:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.029 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:26.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:08.385201+0000", 2023-10-09T05:57:26.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:26.030 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.030 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:26.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:08.385201+0000", 2023-10-09T05:57:26.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:26.030 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.030 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:26.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:08.385200+0000", 2023-10-09T05:57:26.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:57:26.030 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:08.385210+0000", 2023-10-09T05:57:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8349999999999999e-06 2023-10-09T05:57:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:08.385224+0000", 2023-10-09T05:57:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4494e-05 2023-10-09T05:57:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.031 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:08.385236+0000", 2023-10-09T05:57:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1654000000000001e-05 2023-10-09T05:57:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:09.796497+0000", 2023-10-09T05:57:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4112608579999999 2023-10-09T05:57:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:09.810484+0000", 2023-10-09T05:57:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013987006 2023-10-09T05:57:26.032 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119183 6.10 6:0977e43c:::benchmark_data_smithi064_284193_object22640:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e212)", 2023-10-09T05:57:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:43.676425+0000", 2023-10-09T05:57:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 102.272289939, 2023-10-09T05:57:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.201728417, 2023-10-09T05:57:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:26.033 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:26.034 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:26.034 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119183 2023-10-09T05:57:26.034 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.034 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:26.034 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.034 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:26.034 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:43.676425+0000", 2023-10-09T05:57:26.034 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:26.034 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.034 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.034 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:26.034 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:43.676425+0000", 2023-10-09T05:57:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:43.676424+0000", 2023-10-09T05:57:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:57:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:43.676433+0000", 2023-10-09T05:57:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6989999999999998e-06 2023-10-09T05:57:26.035 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.036 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.036 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:26.036 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:43.676445+0000", 2023-10-09T05:57:26.036 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1764e-05 2023-10-09T05:57:26.036 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.036 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.036 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:26.036 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:43.676459+0000", 2023-10-09T05:57:26.036 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3677e-05 2023-10-09T05:57:26.036 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.036 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.036 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:43.695833+0000", 2023-10-09T05:57:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.019374216999999999 2023-10-09T05:57:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:57:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:43.695851+0000", 2023-10-09T05:57:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7960000000000001e-05 2023-10-09T05:57:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.867602+0000", 2023-10-09T05:57:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1717509829999999 2023-10-09T05:57:26.037 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.038 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.038 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:57:26.038 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.867785+0000", 2023-10-09T05:57:26.038 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000182685 2023-10-09T05:57:26.038 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.038 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.038 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:57:26.038 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.868119+0000", 2023-10-09T05:57:26.038 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00033449400000000001 2023-10-09T05:57:26.038 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.038 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.038 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:57:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.871702+0000", 2023-10-09T05:57:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035825039999999998 2023-10-09T05:57:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.873074+0000", 2023-10-09T05:57:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013719579999999999 2023-10-09T05:57:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.876541+0000", 2023-10-09T05:57:26.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034674469999999998 2023-10-09T05:57:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:57:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.876608+0000", 2023-10-09T05:57:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7090999999999999e-05 2023-10-09T05:57:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.878153+0000", 2023-10-09T05:57:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001545329 2023-10-09T05:57:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:26.040 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPushReply(6.19 225/190 [PushReplyOp(6:9ac767a4:::benchmark_data_smithi064_284193_object64254:head)])", 2023-10-09T05:57:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:56:22.641093+0000", 2023-10-09T05:57:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 63.307621537000003, 2023-10-09T05:57:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0279770349999999, 2023-10-09T05:57:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:57:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:26.041 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:22.641093+0000", 2023-10-09T05:57:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:22.641093+0000", 2023-10-09T05:57:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:22.641092+0000", 2023-10-09T05:57:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T05:57:26.042 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:22.641110+0000", 2023-10-09T05:57:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7961e-05 2023-10-09T05:57:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:22.641123+0000", 2023-10-09T05:57:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3114e-05 2023-10-09T05:57:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.043 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:22.641133+0000", 2023-10-09T05:57:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4700000000000008e-06 2023-10-09T05:57:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:23.657800+0000", 2023-10-09T05:57:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0166667439999999 2023-10-09T05:57:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:23.669070+0000", 2023-10-09T05:57:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011270913 2023-10-09T05:57:26.044 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPushReply(6.19 225/190 [PushReplyOp(6:9df2b95e:::benchmark_data_smithi064_284193_object64255:head)])", 2023-10-09T05:57:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:56:22.710736+0000", 2023-10-09T05:57:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 63.237978693000002, 2023-10-09T05:57:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96840995299999999, 2023-10-09T05:57:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:57:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:26.045 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:22.710736+0000", 2023-10-09T05:57:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:22.710736+0000", 2023-10-09T05:57:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:26.046 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:22.710735+0000", 2023-10-09T05:57:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T05:57:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:22.710743+0000", 2023-10-09T05:57:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5530000000000004e-06 2023-10-09T05:57:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:22.710755+0000", 2023-10-09T05:57:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2262000000000001e-05 2023-10-09T05:57:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:22.710765+0000", 2023-10-09T05:57:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3959999999999995e-06 2023-10-09T05:57:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:23.669291+0000", 2023-10-09T05:57:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95852639900000003 2023-10-09T05:57:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:23.679146+0000", 2023-10-09T05:57:26.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0098552329999999997 2023-10-09T05:57:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPush(6.19 226/190 [PushOp(6:9a476af7:::benchmark_data_smithi064_284193_object64315:head, version: 124'3985, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:9a476af7:::benchmark_data_smithi064_284193_object64315:head@124'3985, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:57:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:56:26.154513+0000", 2023-10-09T05:57:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 59.794201708000003, 2023-10-09T05:57:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.06635038, 2023-10-09T05:57:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T05:57:26.049 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:26.050 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.050 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:26.050 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:26.154513+0000", 2023-10-09T05:57:26.050 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:26.050 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.050 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.050 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:26.050 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:26.154513+0000", 2023-10-09T05:57:26.050 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:26.050 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.050 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.050 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:26.051 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:26.154513+0000", 2023-10-09T05:57:26.051 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:26.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.051 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:26.051 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:26.154545+0000", 2023-10-09T05:57:26.051 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2435999999999999e-05 2023-10-09T05:57:26.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.051 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:26.051 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:26.154556+0000", 2023-10-09T05:57:26.051 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0723999999999999e-05 2023-10-09T05:57:26.052 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.052 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.052 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:26.052 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:26.154564+0000", 2023-10-09T05:57:26.052 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7149999999999998e-06 2023-10-09T05:57:26.052 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.052 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.052 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.052 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:27.209883+0000", 2023-10-09T05:57:26.052 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.05531872 2023-10-09T05:57:26.052 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.052 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.052 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:57:26.053 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:27.210207+0000", 2023-10-09T05:57:26.053 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00032434 2023-10-09T05:57:26.053 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.053 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.053 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:26.053 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:27.220864+0000", 2023-10-09T05:57:26.053 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01065669 2023-10-09T05:57:26.053 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.053 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:26.053 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.053 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.053 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.054 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:139223 6.29 6:94b85dbd:::benchmark_data_smithi064_284193_object42680:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T05:57:26.054 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:56:39.263746+0000", 2023-10-09T05:57:26.054 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 46.684969111999997, 2023-10-09T05:57:26.054 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.612463668, 2023-10-09T05:57:26.054 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:26.054 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:26.054 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:26.054 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:26.054 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:26.054 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 139223 2023-10-09T05:57:26.054 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.054 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:26.055 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.055 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:26.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:39.263746+0000", 2023-10-09T05:57:26.055 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:26.055 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.055 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.055 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:26.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:39.263746+0000", 2023-10-09T05:57:26.055 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:26.055 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.055 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.055 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:26.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:39.263744+0000", 2023-10-09T05:57:26.056 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:57:26.056 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.056 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.056 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:26.056 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:39.263754+0000", 2023-10-09T05:57:26.056 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8779999999999993e-06 2023-10-09T05:57:26.056 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.056 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.056 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:26.056 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:39.284922+0000", 2023-10-09T05:57:26.056 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021167807 2023-10-09T05:57:26.056 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.057 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.057 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:26.057 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:39.284938+0000", 2023-10-09T05:57:26.057 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6745999999999999e-05 2023-10-09T05:57:26.057 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.057 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.057 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.057 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:39.396958+0000", 2023-10-09T05:57:26.057 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.11202013800000001 2023-10-09T05:57:26.057 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.057 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.057 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:57:26.058 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:39.396970+0000", 2023-10-09T05:57:26.058 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1328000000000001e-05 2023-10-09T05:57:26.058 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.058 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.058 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.058 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:40.855893+0000", 2023-10-09T05:57:26.058 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4589233369999999 2023-10-09T05:57:26.058 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.058 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.058 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:57:26.058 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:40.858875+0000", 2023-10-09T05:57:26.058 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029821449999999998 2023-10-09T05:57:26.059 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.059 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.059 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.059 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:40.866916+0000", 2023-10-09T05:57:26.059 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0080405540000000001 2023-10-09T05:57:26.059 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.059 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.059 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:57:26.059 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:40.867946+0000", 2023-10-09T05:57:26.059 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010304240000000001 2023-10-09T05:57:26.059 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.059 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.059 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:57:26.060 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:40.870193+0000", 2023-10-09T05:57:26.060 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0022469429999999999 2023-10-09T05:57:26.060 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.060 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.060 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:26.060 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:40.872753+0000", 2023-10-09T05:57:26.060 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025593970000000001 2023-10-09T05:57:26.060 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.060 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.060 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:57:26.060 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:40.872901+0000", 2023-10-09T05:57:26.060 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014856099999999999 2023-10-09T05:57:26.061 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.061 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.061 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:26.061 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:40.873263+0000", 2023-10-09T05:57:26.061 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00036204500000000002 2023-10-09T05:57:26.061 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.061 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.061 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:57:26.061 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:40.873868+0000", 2023-10-09T05:57:26.061 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00060503100000000001 2023-10-09T05:57:26.061 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.061 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.062 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:26.062 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:40.876210+0000", 2023-10-09T05:57:26.062 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0023413230000000002 2023-10-09T05:57:26.062 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.062 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:26.062 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.062 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.062 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.062 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:159583 6.10 6:0dc577d6:::benchmark_data_smithi064_284193_object63040:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2023-10-09T05:57:26.062 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:57:22.430471+0000", 2023-10-09T05:57:26.062 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 3.5182435000000001, 2023-10-09T05:57:26.062 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.364807377, 2023-10-09T05:57:26.063 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:26.063 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:57:26.063 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:26.063 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:26.063 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:26.063 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 159583 2023-10-09T05:57:26.063 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.063 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:26.063 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.063 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:26.063 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.430471+0000", 2023-10-09T05:57:26.063 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:26.064 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.064 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.064 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:26.064 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.430471+0000", 2023-10-09T05:57:26.064 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:26.064 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.064 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.064 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:26.064 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.430471+0000", 2023-10-09T05:57:26.064 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:26.064 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.064 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.065 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:26.065 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.430475+0000", 2023-10-09T05:57:26.065 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5000000000000001e-06 2023-10-09T05:57:26.065 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.065 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.065 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:26.065 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.430488+0000", 2023-10-09T05:57:26.065 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2247e-05 2023-10-09T05:57:26.065 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.065 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.065 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:26.065 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.430493+0000", 2023-10-09T05:57:26.065 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.631e-06 2023-10-09T05:57:26.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.430543+0000", 2023-10-09T05:57:26.066 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9444000000000003e-05 2023-10-09T05:57:26.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:57:26.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.430666+0000", 2023-10-09T05:57:26.066 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012383399999999999 2023-10-09T05:57:26.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,5", 2023-10-09T05:57:26.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.431023+0000", 2023-10-09T05:57:26.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00035648400000000001 2023-10-09T05:57:26.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:57:26.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.433954+0000", 2023-10-09T05:57:26.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029310740000000001 2023-10-09T05:57:26.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:26.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.437343+0000", 2023-10-09T05:57:26.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033885790000000001 2023-10-09T05:57:26.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.068 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.451162+0000", 2023-10-09T05:57:26.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01381925 2023-10-09T05:57:26.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.068 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:57:26.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.451173+0000", 2023-10-09T05:57:26.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1229999999999999e-05 2023-10-09T05:57:26.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.069 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.069 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.469718+0000", 2023-10-09T05:57:26.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.018544736 2023-10-09T05:57:26.069 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.069 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:57:26.069 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.469731+0000", 2023-10-09T05:57:26.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3584000000000001e-05 2023-10-09T05:57:26.069 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.069 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.784199+0000", 2023-10-09T05:57:26.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.314468078 2023-10-09T05:57:26.070 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.070 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-10-09T05:57:26.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.784351+0000", 2023-10-09T05:57:26.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015171100000000001 2023-10-09T05:57:26.070 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.070 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.795179+0000", 2023-10-09T05:57:26.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010827577 2023-10-09T05:57:26.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.071 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:26.071 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.795279+0000", 2023-10-09T05:57:26.071 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000100115 2023-10-09T05:57:26.071 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.071 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:26.071 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.071 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:159606 6.20 6:0672fcbe:::benchmark_data_smithi064_284193_object63063:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-10-09T05:57:26.071 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:57:22.478808+0000", 2023-10-09T05:57:26.071 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 3.4699071159999999, 2023-10-09T05:57:26.072 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3242037419999999, 2023-10-09T05:57:26.072 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:26.072 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:26.072 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:26.072 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:26.072 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:26.072 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 159606 2023-10-09T05:57:26.072 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.072 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:26.072 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.072 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:26.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.478808+0000", 2023-10-09T05:57:26.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:26.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.073 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:26.073 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.478808+0000", 2023-10-09T05:57:26.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:26.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.073 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:26.073 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.478806+0000", 2023-10-09T05:57:26.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:57:26.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:26.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.478813+0000", 2023-10-09T05:57:26.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.511e-06 2023-10-09T05:57:26.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.074 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:26.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.478829+0000", 2023-10-09T05:57:26.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5574999999999999e-05 2023-10-09T05:57:26.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.074 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:26.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.478837+0000", 2023-10-09T05:57:26.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2160000000000006e-06 2023-10-09T05:57:26.075 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.075 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.478888+0000", 2023-10-09T05:57:26.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1242000000000002e-05 2023-10-09T05:57:26.075 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.075 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:57:26.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.478897+0000", 2023-10-09T05:57:26.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4220000000000006e-06 2023-10-09T05:57:26.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.076 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.076 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.076 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.469800+0000", 2023-10-09T05:57:26.076 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99090245600000004 2023-10-09T05:57:26.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.076 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.076 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:57:26.076 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.469808+0000", 2023-10-09T05:57:26.076 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3110000000000006e-06 2023-10-09T05:57:26.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.076 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.076 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:26.077 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.784786+0000", 2023-10-09T05:57:26.077 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.314978027 2023-10-09T05:57:26.077 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.077 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.077 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:57:26.077 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.784932+0000", 2023-10-09T05:57:26.077 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014589599999999999 2023-10-09T05:57:26.077 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.077 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.077 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,3,5", 2023-10-09T05:57:26.077 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.785236+0000", 2023-10-09T05:57:26.077 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00030418699999999998 2023-10-09T05:57:26.078 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.078 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.078 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:26.078 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.786818+0000", 2023-10-09T05:57:26.078 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0015812479999999999 2023-10-09T05:57:26.078 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.078 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.078 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:57:26.078 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.790322+0000", 2023-10-09T05:57:26.078 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035042110000000001 2023-10-09T05:57:26.078 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.078 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:26.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.794740+0000", 2023-10-09T05:57:26.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044178969999999996 2023-10-09T05:57:26.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:26.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.802768+0000", 2023-10-09T05:57:26.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0080280739999999996 2023-10-09T05:57:26.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:57:26.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.802806+0000", 2023-10-09T05:57:26.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8460000000000001e-05 2023-10-09T05:57:26.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:26.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:26.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:26.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.803012+0000", 2023-10-09T05:57:26.080 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020540000000000001 2023-10-09T05:57:26.080 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.080 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:26.080 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.080 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:26.080 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:26.080 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:57:26.184 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:26.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:26.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:26.294 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:26.370 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:26.500 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T05:57:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:26 smithi064 ceph-mon[220044]: failed to encode map e251 with expected crc 2023-10-09T05:57:26.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:26 smithi064 ceph-mon[220044]: failed to encode map e251 with expected crc 2023-10-09T05:57:26.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:26 smithi064 ceph-mon[220044]: failed to encode map e251 with expected crc 2023-10-09T05:57:26.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:26 smithi064 ceph-mon[220044]: failed to encode map e251 with expected crc 2023-10-09T05:57:26.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:26 smithi064 ceph-mon[220044]: osdmap e251: 8 total, 8 up, 7 in 2023-10-09T05:57:26.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:26 smithi064 ceph-mon[220044]: failed to encode map e251 with expected crc 2023-10-09T05:57:26.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:26 smithi064 ceph-mon[220044]: failed to encode map e251 with expected crc 2023-10-09T05:57:26.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:26 smithi064 ceph-mon[220044]: failed to encode map e251 with expected crc 2023-10-09T05:57:26.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:26 smithi064 ceph-mon[220044]: failed to encode map e251 with expected crc 2023-10-09T05:57:26.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:26 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1455960015' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:57:26.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:26 smithi064 ceph-mon[212750]: failed to encode map e251 with expected crc 2023-10-09T05:57:26.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:26 smithi064 ceph-mon[212750]: failed to encode map e251 with expected crc 2023-10-09T05:57:26.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:26 smithi064 ceph-mon[212750]: failed to encode map e251 with expected crc 2023-10-09T05:57:26.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:26 smithi064 ceph-mon[212750]: failed to encode map e251 with expected crc 2023-10-09T05:57:26.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:26 smithi064 ceph-mon[212750]: osdmap e251: 8 total, 8 up, 7 in 2023-10-09T05:57:26.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:26 smithi064 ceph-mon[212750]: failed to encode map e251 with expected crc 2023-10-09T05:57:26.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:26 smithi064 ceph-mon[212750]: failed to encode map e251 with expected crc 2023-10-09T05:57:26.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:26 smithi064 ceph-mon[212750]: failed to encode map e251 with expected crc 2023-10-09T05:57:26.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:26 smithi064 ceph-mon[212750]: failed to encode map e251 with expected crc 2023-10-09T05:57:26.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:26 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1455960015' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:57:26.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:26 smithi156 ceph-mon[195101]: failed to encode map e251 with expected crc 2023-10-09T05:57:26.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:26 smithi156 ceph-mon[195101]: failed to encode map e251 with expected crc 2023-10-09T05:57:26.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:26 smithi156 ceph-mon[195101]: failed to encode map e251 with expected crc 2023-10-09T05:57:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:26 smithi156 ceph-mon[195101]: failed to encode map e251 with expected crc 2023-10-09T05:57:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:26 smithi156 ceph-mon[195101]: osdmap e251: 8 total, 8 up, 7 in 2023-10-09T05:57:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:26 smithi156 ceph-mon[195101]: failed to encode map e251 with expected crc 2023-10-09T05:57:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:26 smithi156 ceph-mon[195101]: failed to encode map e251 with expected crc 2023-10-09T05:57:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:26 smithi156 ceph-mon[195101]: failed to encode map e251 with expected crc 2023-10-09T05:57:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:26 smithi156 ceph-mon[195101]: failed to encode map e251 with expected crc 2023-10-09T05:57:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:26 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1455960015' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:57:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:27 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:27 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1455960015' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:57:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:27 smithi156 ceph-mon[195101]: osdmap e252: 8 total, 8 up, 7 in 2023-10-09T05:57:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:27 smithi156 ceph-mon[195101]: failed to encode map e252 with expected crc 2023-10-09T05:57:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:27 smithi156 ceph-mon[195101]: failed to encode map e252 with expected crc 2023-10-09T05:57:27.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:27 smithi156 ceph-mon[195101]: failed to encode map e252 with expected crc 2023-10-09T05:57:27.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:27 smithi156 ceph-mon[195101]: pgmap v579: 139 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 6 activating+undersized+remapped, 4 remapped+peering, 4 peering, 107 active+clean, 7 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 27035/100821 objects degraded (26.815%); 5109/100821 objects misplaced (5.067%); 2.9 MiB/s, 238 objects/s recovering 2023-10-09T05:57:27.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:27 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 23505/105183 objects degraded (22.347%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:27.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:27 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:27.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:27 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1455960015' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:57:27.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:27 smithi064 ceph-mon[212750]: osdmap e252: 8 total, 8 up, 7 in 2023-10-09T05:57:27.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:27 smithi064 ceph-mon[212750]: failed to encode map e252 with expected crc 2023-10-09T05:57:27.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:27 smithi064 ceph-mon[212750]: failed to encode map e252 with expected crc 2023-10-09T05:57:27.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:27 smithi064 ceph-mon[212750]: failed to encode map e252 with expected crc 2023-10-09T05:57:27.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:27 smithi064 ceph-mon[212750]: pgmap v579: 139 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 6 activating+undersized+remapped, 4 remapped+peering, 4 peering, 107 active+clean, 7 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 27035/100821 objects degraded (26.815%); 5109/100821 objects misplaced (5.067%); 2.9 MiB/s, 238 objects/s recovering 2023-10-09T05:57:27.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:27 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 23505/105183 objects degraded (22.347%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:27.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:27 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:27.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:27 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1455960015' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:57:27.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:27 smithi064 ceph-mon[220044]: osdmap e252: 8 total, 8 up, 7 in 2023-10-09T05:57:27.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:27 smithi064 ceph-mon[220044]: failed to encode map e252 with expected crc 2023-10-09T05:57:27.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:27 smithi064 ceph-mon[220044]: failed to encode map e252 with expected crc 2023-10-09T05:57:27.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:27 smithi064 ceph-mon[220044]: failed to encode map e252 with expected crc 2023-10-09T05:57:27.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:27 smithi064 ceph-mon[220044]: pgmap v579: 139 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 6 activating+undersized+remapped, 4 remapped+peering, 4 peering, 107 active+clean, 7 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 27035/100821 objects degraded (26.815%); 5109/100821 objects misplaced (5.067%); 2.9 MiB/s, 238 objects/s recovering 2023-10-09T05:57:27.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:27 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 23505/105183 objects degraded (22.347%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:28.166 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:28.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:28.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:28.272 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:28.302 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:28.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:28 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:28.833 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:28 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:28.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:28 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:29.164 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:29.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:29.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:29.268 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:57:29.315 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:29.376 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:57:29.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:29.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:57:29.481 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:29.509 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:29.559 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:29.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:29.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:29.568 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:29 smithi156 ceph-mon[195101]: pgmap v580: 139 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 6 activating+undersized+remapped, 4 remapped+peering, 111 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 23171/98559 objects degraded (23.510%); 5109/98559 objects misplaced (5.184%); 4.2 MiB/s, 281 objects/s recovering 2023-10-09T05:57:29.669 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:29.718 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:29.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:29 smithi064 ceph-mon[212750]: pgmap v580: 139 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 6 activating+undersized+remapped, 4 remapped+peering, 111 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 23171/98559 objects degraded (23.510%); 5109/98559 objects misplaced (5.184%); 4.2 MiB/s, 281 objects/s recovering 2023-10-09T05:57:29.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:29 smithi064 ceph-mon[220044]: pgmap v580: 139 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 6 activating+undersized+remapped, 4 remapped+peering, 111 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 23171/98559 objects degraded (23.510%); 5109/98559 objects misplaced (5.184%); 4.2 MiB/s, 281 objects/s recovering 2023-10-09T05:57:30.494 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:57:30.529 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:30.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:30.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:30.655 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:30.728 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:30.847 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:30.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:30.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:30.952 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:31.000 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:31.071 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:31.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:31.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:31.177 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:31.270 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:31.436 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:31.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:31.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:31.500 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:57:31.510 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:31.547 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:31.554 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:57:31.591 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:57:31.636 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:57:31.645 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:31.673 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:57:31.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:31 smithi064 ceph-mon[212750]: pgmap v581: 139 pgs: 1 active+recovering+undersized+remapped, 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 24133/87255 objects degraded (27.658%); 6314/87255 objects misplaced (7.236%); 3.5 MiB/s, 292 objects/s recovering 2023-10-09T05:57:31.724 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:31 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3234649816' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:31.724 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:57:31.727 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:31 smithi064 ceph-mon[220044]: pgmap v581: 139 pgs: 1 active+recovering+undersized+remapped, 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 24133/87255 objects degraded (27.658%); 6314/87255 objects misplaced (7.236%); 3.5 MiB/s, 292 objects/s recovering 2023-10-09T05:57:31.727 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:31 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3234649816' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:31.761 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:57:31.794 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:57:31.796 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:57:31.797 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:57:31.797 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:57:31.797 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:57:31.797 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:57:31.797 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:57:31.797 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:57:31.797 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:57:31.928 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:31.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:31.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:32.040 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:32.111 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:32.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:31 smithi156 ceph-mon[195101]: pgmap v581: 139 pgs: 1 active+recovering+undersized+remapped, 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 24133/87255 objects degraded (27.658%); 6314/87255 objects misplaced (7.236%); 3.5 MiB/s, 292 objects/s recovering 2023-10-09T05:57:32.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:31 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3234649816' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:32.332 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:57:32.702 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T05:57:32.758 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:32.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:32.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:32.869 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:32.921 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:32.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:32 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1734315089' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:57:32.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:32 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:57:32.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:32 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 24133/87255 objects degraded (27.658%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:32.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:32 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1734315089' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:57:32.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:32 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:57:32.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:32 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 24133/87255 objects degraded (27.658%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:33.008 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:33.014 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:32 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1734315089' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:57:33.014 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:32 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:57:33.014 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:32 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 24133/87255 objects degraded (27.658%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:33.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:33.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:33.117 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:33.158 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:33.239 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:33.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:33.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:33.349 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:33.396 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:33.484 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:33.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:33.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:33.593 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:33.672 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:33.709 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:33 smithi156 ceph-mon[195101]: pgmap v582: 139 pgs: 1 active+recovering+undersized+remapped, 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 475 op/s; 24133/86379 objects degraded (27.939%); 6314/86379 objects misplaced (7.310%); 2.6 MiB/s, 210 objects/s recovering 2023-10-09T05:57:33.709 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:33 smithi156 ceph-mon[195101]: failed to encode map e253 with expected crc 2023-10-09T05:57:33.709 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:33 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:57:33.709 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:33 smithi156 ceph-mon[195101]: osdmap e253: 8 total, 8 up, 7 in 2023-10-09T05:57:33.709 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:33 smithi156 ceph-mon[195101]: failed to encode map e253 with expected crc 2023-10-09T05:57:33.709 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:33 smithi156 ceph-mon[195101]: failed to encode map e253 with expected crc 2023-10-09T05:57:33.709 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:33 smithi156 ceph-mon[195101]: failed to encode map e253 with expected crc 2023-10-09T05:57:33.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:33 smithi064 ceph-mon[220044]: pgmap v582: 139 pgs: 1 active+recovering+undersized+remapped, 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 475 op/s; 24133/86379 objects degraded (27.939%); 6314/86379 objects misplaced (7.310%); 2.6 MiB/s, 210 objects/s recovering 2023-10-09T05:57:33.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:33 smithi064 ceph-mon[220044]: failed to encode map e253 with expected crc 2023-10-09T05:57:33.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:33 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:57:33.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:33 smithi064 ceph-mon[220044]: osdmap e253: 8 total, 8 up, 7 in 2023-10-09T05:57:33.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:33 smithi064 ceph-mon[220044]: failed to encode map e253 with expected crc 2023-10-09T05:57:33.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:33 smithi064 ceph-mon[220044]: failed to encode map e253 with expected crc 2023-10-09T05:57:33.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:33 smithi064 ceph-mon[220044]: failed to encode map e253 with expected crc 2023-10-09T05:57:33.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:33 smithi064 ceph-mon[212750]: pgmap v582: 139 pgs: 1 active+recovering+undersized+remapped, 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 475 op/s; 24133/86379 objects degraded (27.939%); 6314/86379 objects misplaced (7.310%); 2.6 MiB/s, 210 objects/s recovering 2023-10-09T05:57:33.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:33 smithi064 ceph-mon[212750]: failed to encode map e253 with expected crc 2023-10-09T05:57:33.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:33 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:57:33.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:33 smithi064 ceph-mon[212750]: osdmap e253: 8 total, 8 up, 7 in 2023-10-09T05:57:33.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:33 smithi064 ceph-mon[212750]: failed to encode map e253 with expected crc 2023-10-09T05:57:33.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:33 smithi064 ceph-mon[212750]: failed to encode map e253 with expected crc 2023-10-09T05:57:33.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:33 smithi064 ceph-mon[212750]: failed to encode map e253 with expected crc 2023-10-09T05:57:34.897 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:34.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:34.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:35.002 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:35.034 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:35.047 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T05:57:35.105 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:35.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:35.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:35.216 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:35.246 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:35.322 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:35.331 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:35 smithi156 ceph-mon[195101]: failed to encode map e254 with expected crc 2023-10-09T05:57:35.331 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:35 smithi156 ceph-mon[195101]: failed to encode map e254 with expected crc 2023-10-09T05:57:35.331 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:35 smithi156 ceph-mon[195101]: osdmap e254: 8 total, 8 up, 7 in 2023-10-09T05:57:35.331 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:35 smithi156 ceph-mon[195101]: failed to encode map e254 with expected crc 2023-10-09T05:57:35.331 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:35 smithi156 ceph-mon[195101]: failed to encode map e254 with expected crc 2023-10-09T05:57:35.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:35.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:35.379 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:35.433 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:35.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:35 smithi064 ceph-mon[220044]: failed to encode map e254 with expected crc 2023-10-09T05:57:35.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:35 smithi064 ceph-mon[220044]: failed to encode map e254 with expected crc 2023-10-09T05:57:35.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:35 smithi064 ceph-mon[220044]: osdmap e254: 8 total, 8 up, 7 in 2023-10-09T05:57:35.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:35 smithi064 ceph-mon[220044]: failed to encode map e254 with expected crc 2023-10-09T05:57:35.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:35 smithi064 ceph-mon[220044]: failed to encode map e254 with expected crc 2023-10-09T05:57:35.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:35 smithi064 ceph-mon[212750]: failed to encode map e254 with expected crc 2023-10-09T05:57:35.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:35 smithi064 ceph-mon[212750]: failed to encode map e254 with expected crc 2023-10-09T05:57:35.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:35 smithi064 ceph-mon[212750]: osdmap e254: 8 total, 8 up, 7 in 2023-10-09T05:57:35.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:35 smithi064 ceph-mon[212750]: failed to encode map e254 with expected crc 2023-10-09T05:57:35.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:35 smithi064 ceph-mon[212750]: failed to encode map e254 with expected crc 2023-10-09T05:57:35.473 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:57:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:57:35] "GET /metrics HTTP/1.1" 200 36467 "" "Prometheus/2.43.0" 2023-10-09T05:57:35.482 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:35.563 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:35.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:35.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:35.670 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:35.715 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:35.794 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:35.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:35.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:35.903 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:35.942 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:35.988 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:57:35.988 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:57:36.012 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:36.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:36.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:36.041 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:36.121 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:36.202 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:36.249 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:36 smithi064 ceph-mon[220044]: pgmap v585: 139 pgs: 1 active+recovering+undersized+remapped, 113 active+clean, 1 active+undersized+remapped, 10 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 761 op/s; 24133/82539 objects degraded (29.238%); 5980/82539 objects misplaced (7.245%); 1.9 MiB/s, 183 objects/s recovering 2023-10-09T05:57:36.249 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:36 smithi064 ceph-mon[220044]: failed to encode map e255 with expected crc 2023-10-09T05:57:36.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:36 smithi156 ceph-mon[195101]: pgmap v585: 139 pgs: 1 active+recovering+undersized+remapped, 113 active+clean, 1 active+undersized+remapped, 10 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 761 op/s; 24133/82539 objects degraded (29.238%); 5980/82539 objects misplaced (7.245%); 1.9 MiB/s, 183 objects/s recovering 2023-10-09T05:57:36.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:36 smithi156 ceph-mon[195101]: failed to encode map e255 with expected crc 2023-10-09T05:57:36.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:36 smithi156 ceph-mon[195101]: failed to encode map e255 with expected crc 2023-10-09T05:57:36.374 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:36 smithi156 ceph-mon[195101]: osdmap e255: 8 total, 8 up, 7 in 2023-10-09T05:57:36.374 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:36 smithi156 ceph-mon[195101]: failed to encode map e255 with expected crc 2023-10-09T05:57:36.374 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:36 smithi156 ceph-mon[195101]: failed to encode map e255 with expected crc 2023-10-09T05:57:36.544 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:57:36.569 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:36 smithi064 ceph-mon[220044]: failed to encode map e255 with expected crc 2023-10-09T05:57:36.569 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:36 smithi064 ceph-mon[220044]: osdmap e255: 8 total, 8 up, 7 in 2023-10-09T05:57:36.569 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:36 smithi064 ceph-mon[220044]: failed to encode map e255 with expected crc 2023-10-09T05:57:36.569 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:36 smithi064 ceph-mon[220044]: failed to encode map e255 with expected crc 2023-10-09T05:57:36.570 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:36 smithi064 ceph-mon[212750]: pgmap v585: 139 pgs: 1 active+recovering+undersized+remapped, 113 active+clean, 1 active+undersized+remapped, 10 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 761 op/s; 24133/82539 objects degraded (29.238%); 5980/82539 objects misplaced (7.245%); 1.9 MiB/s, 183 objects/s recovering 2023-10-09T05:57:36.570 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:36 smithi064 ceph-mon[212750]: failed to encode map e255 with expected crc 2023-10-09T05:57:36.570 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:36 smithi064 ceph-mon[212750]: failed to encode map e255 with expected crc 2023-10-09T05:57:36.570 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:36 smithi064 ceph-mon[212750]: osdmap e255: 8 total, 8 up, 7 in 2023-10-09T05:57:36.570 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:36 smithi064 ceph-mon[212750]: failed to encode map e255 with expected crc 2023-10-09T05:57:36.570 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:36 smithi064 ceph-mon[212750]: failed to encode map e255 with expected crc 2023-10-09T05:57:36.914 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:36.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:36.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:37.023 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:37.096 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:37.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:37 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 24133/82539 objects degraded (29.238%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:37.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:37 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 24133/82539 objects degraded (29.238%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:37.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:37 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 24133/82539 objects degraded (29.238%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:37.891 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:57:38.048 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:38.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:38.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:38.159 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:38.258 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:38.385 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:38.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:38.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:38.501 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:38.561 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:38 smithi064 ceph-mon[212750]: pgmap v587: 139 pgs: 1 active+recovering+undersized+remapped, 113 active+clean, 1 active+undersized+remapped, 10 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 851 op/s; 24052/71913 objects degraded (33.446%); 5321/71913 objects misplaced (7.399%); 0 B/s, 144 objects/s recovering 2023-10-09T05:57:38.562 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:38 smithi064 ceph-mon[220044]: pgmap v587: 139 pgs: 1 active+recovering+undersized+remapped, 113 active+clean, 1 active+undersized+remapped, 10 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 851 op/s; 24052/71913 objects degraded (33.446%); 5321/71913 objects misplaced (7.399%); 0 B/s, 144 objects/s recovering 2023-10-09T05:57:38.569 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:38.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:38 smithi156 ceph-mon[195101]: pgmap v587: 139 pgs: 1 active+recovering+undersized+remapped, 113 active+clean, 1 active+undersized+remapped, 10 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 851 op/s; 24052/71913 objects degraded (33.446%); 5321/71913 objects misplaced (7.399%); 0 B/s, 144 objects/s recovering 2023-10-09T05:57:38.668 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:38.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:38.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:38.773 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:38.821 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:39.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:39 smithi156 ceph-mon[195101]: osdmap e256: 8 total, 8 up, 7 in 2023-10-09T05:57:39.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:39 smithi156 ceph-mon[195101]: failed to encode map e256 with expected crc 2023-10-09T05:57:39.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:39 smithi156 ceph-mon[195101]: failed to encode map e256 with expected crc 2023-10-09T05:57:39.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:39 smithi156 ceph-mon[195101]: pgmap v589: 139 pgs: 2 active+recovering+undersized+remapped, 114 active+clean, 10 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 970 op/s; 20554/69183 objects degraded (29.710%); 5400/69183 objects misplaced (7.805%); 1.1 MiB/s, 165 objects/s recovering 2023-10-09T05:57:39.668 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:39.672 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:39 smithi064 ceph-mon[212750]: osdmap e256: 8 total, 8 up, 7 in 2023-10-09T05:57:39.672 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:39 smithi064 ceph-mon[212750]: failed to encode map e256 with expected crc 2023-10-09T05:57:39.673 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:39 smithi064 ceph-mon[212750]: failed to encode map e256 with expected crc 2023-10-09T05:57:39.673 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:39 smithi064 ceph-mon[212750]: pgmap v589: 139 pgs: 2 active+recovering+undersized+remapped, 114 active+clean, 10 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 970 op/s; 20554/69183 objects degraded (29.710%); 5400/69183 objects misplaced (7.805%); 1.1 MiB/s, 165 objects/s recovering 2023-10-09T05:57:39.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:39.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:39.676 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:39 smithi064 ceph-mon[220044]: osdmap e256: 8 total, 8 up, 7 in 2023-10-09T05:57:39.676 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:39 smithi064 ceph-mon[220044]: failed to encode map e256 with expected crc 2023-10-09T05:57:39.676 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:39 smithi064 ceph-mon[220044]: failed to encode map e256 with expected crc 2023-10-09T05:57:39.676 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:39 smithi064 ceph-mon[220044]: pgmap v589: 139 pgs: 2 active+recovering+undersized+remapped, 114 active+clean, 10 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 970 op/s; 20554/69183 objects degraded (29.710%); 5400/69183 objects misplaced (7.805%); 1.1 MiB/s, 165 objects/s recovering 2023-10-09T05:57:39.729 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:39.761 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:39.777 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:39.905 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:40.188 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:40.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:40.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:40.300 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:40.338 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:40 smithi064 ceph-mon[212750]: failed to encode map e257 with expected crc 2023-10-09T05:57:40.338 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:40 smithi064 ceph-mon[212750]: osdmap e257: 8 total, 8 up, 7 in 2023-10-09T05:57:40.338 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:40 smithi064 ceph-mon[212750]: failed to encode map e257 with expected crc 2023-10-09T05:57:40.338 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:40 smithi064 ceph-mon[220044]: failed to encode map e257 with expected crc 2023-10-09T05:57:40.338 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:40 smithi064 ceph-mon[220044]: osdmap e257: 8 total, 8 up, 7 in 2023-10-09T05:57:40.338 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:40 smithi064 ceph-mon[220044]: failed to encode map e257 with expected crc 2023-10-09T05:57:40.384 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:40.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:40 smithi156 ceph-mon[195101]: failed to encode map e257 with expected crc 2023-10-09T05:57:40.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:40 smithi156 ceph-mon[195101]: osdmap e257: 8 total, 8 up, 7 in 2023-10-09T05:57:40.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:40 smithi156 ceph-mon[195101]: failed to encode map e257 with expected crc 2023-10-09T05:57:40.895 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:40.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:40.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:41.009 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:41.057 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:41.141 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:41.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:41.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:41.249 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:57:41.301 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:41.333 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T05:57:41.335 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T05:57:41.391 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:57:41.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:41.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:57:41.501 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:41.526 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:41.611 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:41.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:41.619 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:41 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/457109152' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:57:41.619 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:41 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:57:41.619 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:41 smithi156 ceph-mon[195101]: pgmap v591: 139 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 10 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.39k op/s; 20481/57450 objects degraded (35.650%); 4454/57450 objects misplaced (7.753%); 1.2 MiB/s, 111 objects/s recovering 2023-10-09T05:57:41.619 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:41 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3260976443' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:57:41.619 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:41 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:57:41.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:41.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:41 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/457109152' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:57:41.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:41 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:57:41.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:41 smithi064 ceph-mon[212750]: pgmap v591: 139 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 10 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.39k op/s; 20481/57450 objects degraded (35.650%); 4454/57450 objects misplaced (7.753%); 1.2 MiB/s, 111 objects/s recovering 2023-10-09T05:57:41.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:41 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3260976443' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:57:41.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:41 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:57:41.721 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:41.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:41 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/457109152' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:57:41.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:41 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:57:41.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:41 smithi064 ceph-mon[220044]: pgmap v591: 139 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 10 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.39k op/s; 20481/57450 objects degraded (35.650%); 4454/57450 objects misplaced (7.753%); 1.2 MiB/s, 111 objects/s recovering 2023-10-09T05:57:41.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:41 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3260976443' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:57:41.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:41 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:57:41.769 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:41.849 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:41.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:41.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:41.956 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:41.986 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:42.105 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:42.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:42.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:42.214 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:42.290 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:42.407 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:42.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:42.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:42.513 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:57:42.562 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:42 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:42.562 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:42 smithi156 ceph-mon[195101]: failed to encode map e258 with expected crc 2023-10-09T05:57:42.562 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:42 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:57:42.562 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:42 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:57:42.562 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:42 smithi156 ceph-mon[195101]: osdmap e258: 8 total, 8 up, 7 in 2023-10-09T05:57:42.562 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:42 smithi156 ceph-mon[195101]: failed to encode map e258 with expected crc 2023-10-09T05:57:42.562 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:42 smithi156 ceph-mon[195101]: failed to encode map e258 with expected crc 2023-10-09T05:57:42.562 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:42 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 20481/57450 objects degraded (35.650%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:42.564 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:42.614 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:57:42.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:42.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:57:42.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:42 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:42.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:42 smithi064 ceph-mon[212750]: failed to encode map e258 with expected crc 2023-10-09T05:57:42.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:42 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:57:42.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:42 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:57:42.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:42 smithi064 ceph-mon[212750]: osdmap e258: 8 total, 8 up, 7 in 2023-10-09T05:57:42.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:42 smithi064 ceph-mon[212750]: failed to encode map e258 with expected crc 2023-10-09T05:57:42.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:42 smithi064 ceph-mon[212750]: failed to encode map e258 with expected crc 2023-10-09T05:57:42.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:42 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 20481/57450 objects degraded (35.650%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:42.721 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:42.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:42 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:42.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:42 smithi064 ceph-mon[220044]: failed to encode map e258 with expected crc 2023-10-09T05:57:42.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:42 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:57:42.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:42 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:57:42.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:42 smithi064 ceph-mon[220044]: osdmap e258: 8 total, 8 up, 7 in 2023-10-09T05:57:42.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:42 smithi064 ceph-mon[220044]: failed to encode map e258 with expected crc 2023-10-09T05:57:42.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:42 smithi064 ceph-mon[220044]: failed to encode map e258 with expected crc 2023-10-09T05:57:42.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:42 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 20481/57450 objects degraded (35.650%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:42.795 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:42.935 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:42.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:42.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:43.040 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:43.068 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:43.343 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:43 smithi064 ceph-mon[220044]: pgmap v593: 139 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 10 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 861 op/s; 20481/56397 objects degraded (36.316%); 4454/56397 objects misplaced (7.898%); 1.2 MiB/s, 58 objects/s recovering 2023-10-09T05:57:43.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:43 smithi156 ceph-mon[195101]: pgmap v593: 139 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 10 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 861 op/s; 20481/56397 objects degraded (36.316%); 4454/56397 objects misplaced (7.898%); 1.2 MiB/s, 58 objects/s recovering 2023-10-09T05:57:43.658 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:43.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:43.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:43.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:43 smithi064 ceph-mon[212750]: pgmap v593: 139 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 10 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 861 op/s; 20481/56397 objects degraded (36.316%); 4454/56397 objects misplaced (7.898%); 1.2 MiB/s, 58 objects/s recovering 2023-10-09T05:57:43.764 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:43.811 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:43.814 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T05:57:43.889 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:43.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:43.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:44.000 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:44.108 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:44.414 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:57:44.456 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:57:44.514 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:57:44.525 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:44.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:44.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:44.562 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:57:44.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:44.620 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:57:44.636 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:44.645 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:57:44.645 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T05:57:44.645 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T05:57:44.645 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T05:57:44.645 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:57:44.665 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:57:44.679 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:44.680 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:44.686 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:44.707 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:57:44.751 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:57:44.764 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:44.766 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:57:44.766 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:57:44.766 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:57:44.766 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:57:44.766 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:57:44.766 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:57:44.766 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:57:44.767 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:57:44.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:44.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:44.879 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:57:44.934 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:45.006 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:57:45.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:45.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:57:45.108 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:45.162 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:45.271 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:45.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:45.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:45.303 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:57:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:57:45] "GET /metrics HTTP/1.1" 200 36463 "" "Prometheus/2.43.0" 2023-10-09T05:57:45.383 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:45.447 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:45.545 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:45.552 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:45 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1164592172' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:45.552 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:45 smithi156 ceph-mon[195101]: pgmap v594: 139 pgs: 116 active+clean, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 979 op/s; 20481/52473 objects degraded (39.032%); 4047/52473 objects misplaced (7.713%); 1.9 MiB/s, 111 objects/s recovering 2023-10-09T05:57:45.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:45.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:45.653 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:45.686 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:45 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1164592172' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:45.686 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:45 smithi064 ceph-mon[212750]: pgmap v594: 139 pgs: 116 active+clean, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 979 op/s; 20481/52473 objects degraded (39.032%); 4047/52473 objects misplaced (7.713%); 1.9 MiB/s, 111 objects/s recovering 2023-10-09T05:57:45.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:45 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1164592172' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:45.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:45 smithi064 ceph-mon[220044]: pgmap v594: 139 pgs: 116 active+clean, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 979 op/s; 20481/52473 objects degraded (39.032%); 4047/52473 objects misplaced (7.713%); 1.9 MiB/s, 111 objects/s recovering 2023-10-09T05:57:45.690 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:45.702 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:57:46.409 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:46.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:46.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:46.518 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:46.582 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:46.671 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:57:46.714 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:46.718 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:46.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:46.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:46.828 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:46.955 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:47.078 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:47.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:47.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:47.182 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:47.231 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:47.322 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:47.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:47.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:47.362 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:47.427 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:47.518 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:47.611 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:47.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:47.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:47.718 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:47.772 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:47.814 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:47.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:47.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:47.918 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:47.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:47 smithi064 ceph-mon[212750]: pgmap v595: 139 pgs: 116 active+clean, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped; 863 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 1.15k op/s; 20409/41475 objects degraded (49.208%); 3340/41475 objects misplaced (8.053%); 671 KiB/s, 74 objects/s recovering 2023-10-09T05:57:47.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:47 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 20409/41475 objects degraded (49.208%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:47.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:47 smithi064 ceph-mon[220044]: pgmap v595: 139 pgs: 116 active+clean, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped; 863 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 1.15k op/s; 20409/41475 objects degraded (49.208%); 3340/41475 objects misplaced (8.053%); 671 KiB/s, 74 objects/s recovering 2023-10-09T05:57:47.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:47 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 20409/41475 objects degraded (49.208%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:47.946 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:48.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:47 smithi156 ceph-mon[195101]: pgmap v595: 139 pgs: 116 active+clean, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped; 863 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 1.15k op/s; 20409/41475 objects degraded (49.208%); 3340/41475 objects misplaced (8.053%); 671 KiB/s, 74 objects/s recovering 2023-10-09T05:57:48.124 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:47 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 20409/41475 objects degraded (49.208%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:48.267 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:48.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:48.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:48.375 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:48.480 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:48.653 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:48.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:48.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:48.697 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:57:48.761 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:48.809 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:48.895 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:48.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:48.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:49.000 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:49.115 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:49.881 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:49.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:49 smithi064 ceph-mon[212750]: pgmap v596: 139 pgs: 116 active+clean, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped; 801 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s; 20409/38508 objects degraded (52.999%); 3340/38508 objects misplaced (8.674%); 578 KiB/s, 96 objects/s recovering 2023-10-09T05:57:49.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:49 smithi064 ceph-mon[220044]: pgmap v596: 139 pgs: 116 active+clean, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped; 801 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s; 20409/38508 objects degraded (52.999%); 3340/38508 objects misplaced (8.674%); 578 KiB/s, 96 objects/s recovering 2023-10-09T05:57:50.057 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:50.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:50.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:50.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:49 smithi156 ceph-mon[195101]: pgmap v596: 139 pgs: 116 active+clean, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped; 801 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s; 20409/38508 objects degraded (52.999%); 3340/38508 objects misplaced (8.674%); 578 KiB/s, 96 objects/s recovering 2023-10-09T05:57:50.170 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:50.229 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:50.372 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:50.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:50.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:50.480 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:50.725 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T05:57:50.924 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:50.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:50 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1489705645' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:57:50.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:50 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:57:50.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:50 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1489705645' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:57:50.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:50 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:57:51.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:50 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1489705645' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:57:51.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:50 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:57:51.378 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:51.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:51.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:51.484 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:51.513 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:51.560 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:51.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:51.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:51.664 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:57:51.691 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:51.763 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:51 smithi064 ceph-mon[212750]: pgmap v597: 139 pgs: 116 active+clean, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped; 534 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 20327/25701 objects degraded (79.090%); 2359/25701 objects misplaced (9.179%); 505 KiB/s, 65 objects/s recovering 2023-10-09T05:57:51.763 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:51 smithi064 ceph-mon[212750]: failed to encode map e259 with expected crc 2023-10-09T05:57:51.763 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:51 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:57:51.763 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:51 smithi064 ceph-mon[212750]: osdmap e259: 8 total, 8 up, 7 in 2023-10-09T05:57:51.763 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:51 smithi064 ceph-mon[212750]: failed to encode map e259 with expected crc 2023-10-09T05:57:51.763 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:51 smithi064 ceph-mon[212750]: failed to encode map e259 with expected crc 2023-10-09T05:57:51.763 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:51 smithi064 ceph-mon[212750]: failed to encode map e259 with expected crc 2023-10-09T05:57:51.763 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:51 smithi064 ceph-mon[212750]: failed to encode map e259 with expected crc 2023-10-09T05:57:51.763 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:51 smithi064 ceph-mon[212750]: failed to encode map e259 with expected crc 2023-10-09T05:57:51.764 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:51 smithi064 ceph-mon[212750]: failed to encode map e259 with expected crc 2023-10-09T05:57:51.764 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:51 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3882489622' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:57:51.764 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:51 smithi064 ceph-mon[220044]: pgmap v597: 139 pgs: 116 active+clean, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped; 534 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 20327/25701 objects degraded (79.090%); 2359/25701 objects misplaced (9.179%); 505 KiB/s, 65 objects/s recovering 2023-10-09T05:57:51.764 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:51 smithi064 ceph-mon[220044]: failed to encode map e259 with expected crc 2023-10-09T05:57:51.764 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:51 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:57:51.764 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:51 smithi064 ceph-mon[220044]: osdmap e259: 8 total, 8 up, 7 in 2023-10-09T05:57:51.764 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:51 smithi064 ceph-mon[220044]: failed to encode map e259 with expected crc 2023-10-09T05:57:51.765 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:51 smithi064 ceph-mon[220044]: failed to encode map e259 with expected crc 2023-10-09T05:57:51.765 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:51 smithi064 ceph-mon[220044]: failed to encode map e259 with expected crc 2023-10-09T05:57:51.765 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:51 smithi064 ceph-mon[220044]: failed to encode map e259 with expected crc 2023-10-09T05:57:51.765 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:51 smithi064 ceph-mon[220044]: failed to encode map e259 with expected crc 2023-10-09T05:57:51.765 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:51 smithi064 ceph-mon[220044]: failed to encode map e259 with expected crc 2023-10-09T05:57:51.765 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:51 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3882489622' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:57:51.773 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:57:51.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:51.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:57:51.782 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T05:57:51.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:57:51.906 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:52.251 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:51 smithi156 ceph-mon[195101]: pgmap v597: 139 pgs: 116 active+clean, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped; 534 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 20327/25701 objects degraded (79.090%); 2359/25701 objects misplaced (9.179%); 505 KiB/s, 65 objects/s recovering 2023-10-09T05:57:52.251 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:51 smithi156 ceph-mon[195101]: failed to encode map e259 with expected crc 2023-10-09T05:57:52.252 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:51 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:57:52.252 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:51 smithi156 ceph-mon[195101]: osdmap e259: 8 total, 8 up, 7 in 2023-10-09T05:57:52.252 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:51 smithi156 ceph-mon[195101]: failed to encode map e259 with expected crc 2023-10-09T05:57:52.252 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:51 smithi156 ceph-mon[195101]: failed to encode map e259 with expected crc 2023-10-09T05:57:52.252 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:51 smithi156 ceph-mon[195101]: failed to encode map e259 with expected crc 2023-10-09T05:57:52.252 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:51 smithi156 ceph-mon[195101]: failed to encode map e259 with expected crc 2023-10-09T05:57:52.252 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:51 smithi156 ceph-mon[195101]: failed to encode map e259 with expected crc 2023-10-09T05:57:52.252 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:51 smithi156 ceph-mon[195101]: failed to encode map e259 with expected crc 2023-10-09T05:57:52.252 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:51 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3882489622' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:57:52.336 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:57:52.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:52.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:57:52.359 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T05:57:52.442 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:52.477 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:52.676 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:52.841 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:52.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:52.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:52.944 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:57:52.991 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:52 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:52.991 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:52 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3882489622' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:57:52.991 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:52 smithi064 ceph-mon[212750]: osdmap e260: 8 total, 8 up, 7 in 2023-10-09T05:57:52.991 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:52 smithi064 ceph-mon[212750]: failed to encode map e260 with expected crc 2023-10-09T05:57:52.992 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:52 smithi064 ceph-mon[212750]: failed to encode map e260 with expected crc 2023-10-09T05:57:52.992 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:52 smithi064 ceph-mon[212750]: failed to encode map e260 with expected crc 2023-10-09T05:57:52.992 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:52 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 20327/25701 objects degraded (79.090%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:52.992 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:52 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:52.992 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:52 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3882489622' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:57:52.992 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:52 smithi064 ceph-mon[220044]: osdmap e260: 8 total, 8 up, 7 in 2023-10-09T05:57:52.992 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:52 smithi064 ceph-mon[220044]: failed to encode map e260 with expected crc 2023-10-09T05:57:52.992 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:52 smithi064 ceph-mon[220044]: failed to encode map e260 with expected crc 2023-10-09T05:57:52.993 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:52 smithi064 ceph-mon[220044]: failed to encode map e260 with expected crc 2023-10-09T05:57:52.993 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:52 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 20327/25701 objects degraded (79.090%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:53.032 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:53.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:52 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:57:53.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:52 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3882489622' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:57:53.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:52 smithi156 ceph-mon[195101]: osdmap e260: 8 total, 8 up, 7 in 2023-10-09T05:57:53.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:52 smithi156 ceph-mon[195101]: failed to encode map e260 with expected crc 2023-10-09T05:57:53.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:52 smithi156 ceph-mon[195101]: failed to encode map e260 with expected crc 2023-10-09T05:57:53.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:52 smithi156 ceph-mon[195101]: failed to encode map e260 with expected crc 2023-10-09T05:57:53.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:52 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 20327/25701 objects degraded (79.090%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:53.327 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:53.492 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:57:53.492 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T05:57:53.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T05:57:53.493 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T05:57:53.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.493 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:99073 6.4 6:27b1f89e:::benchmark_data_smithi064_284193_object2530:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-10-09T05:57:53.493 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:52:42.783709+0000", 2023-10-09T05:57:53.493 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 310.70044721400001, 2023-10-09T05:57:53.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4128961629999992, 2023-10-09T05:57:53.493 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:53.493 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:53.493 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:53.494 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:53.494 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:53.494 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 99073 2023-10-09T05:57:53.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.494 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:53.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:53.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:42.783709+0000", 2023-10-09T05:57:53.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.495 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:53.495 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:42.783709+0000", 2023-10-09T05:57:53.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.495 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:53.495 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:42.783708+0000", 2023-10-09T05:57:53.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:57:53.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:53.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:42.783714+0000", 2023-10-09T05:57:53.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.686e-06 2023-10-09T05:57:53.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:53.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:42.783728+0000", 2023-10-09T05:57:53.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3182e-05 2023-10-09T05:57:53.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:53.497 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:42.783735+0000", 2023-10-09T05:57:53.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3300000000000001e-06 2023-10-09T05:57:53.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.497 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:52.071234+0000", 2023-10-09T05:57:53.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2874990569999998 2023-10-09T05:57:53.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.498 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:57:53.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:52.083105+0000", 2023-10-09T05:57:53.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011871532000000001 2023-10-09T05:57:53.498 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.498 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-10-09T05:57:53.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:52.097849+0000", 2023-10-09T05:57:53.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014743898 2023-10-09T05:57:53.498 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.499 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:57:53.499 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:52.148584+0000", 2023-10-09T05:57:53.499 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.050735114999999997 2023-10-09T05:57:53.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.499 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:53.499 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:52.154161+0000", 2023-10-09T05:57:53.499 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.005577009 2023-10-09T05:57:53.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.499 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:53.499 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:52.160285+0000", 2023-10-09T05:57:53.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061234569999999997 2023-10-09T05:57:53.500 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.500 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.500 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:57:53.500 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:52.168785+0000", 2023-10-09T05:57:53.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085001279999999992 2023-10-09T05:57:53.500 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.500 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.500 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:53.500 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:52.196605+0000", 2023-10-09T05:57:53.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.027820107 2023-10-09T05:57:53.501 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.501 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:53.501 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.501 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104142 6.1 6:8dcfcc8f:::benchmark_data_smithi064_284193_object7599:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:57:53.501 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:40.204920+0000", 2023-10-09T05:57:53.501 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 193.27923627600001, 2023-10-09T05:57:53.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6444858290000006, 2023-10-09T05:57:53.501 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:53.501 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:57:53.501 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:53.502 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:53.502 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:53.502 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104142 2023-10-09T05:57:53.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.502 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:53.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:53.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.204920+0000", 2023-10-09T05:57:53.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:53.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.204920+0000", 2023-10-09T05:57:53.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:53.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.204917+0000", 2023-10-09T05:57:53.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:57:53.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:53.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.204929+0000", 2023-10-09T05:57:53.504 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.163e-05 2023-10-09T05:57:53.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:53.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.204945+0000", 2023-10-09T05:57:53.504 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5988e-05 2023-10-09T05:57:53.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:53.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.204955+0000", 2023-10-09T05:57:53.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9219999999999992e-06 2023-10-09T05:57:53.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.554927+0000", 2023-10-09T05:57:53.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.349971901 2023-10-09T05:57:53.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:57:53.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.582332+0000", 2023-10-09T05:57:53.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.027405307 2023-10-09T05:57:53.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:57:53.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.615389+0000", 2023-10-09T05:57:53.506 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.033057214000000001 2023-10-09T05:57:53.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849384+0000", 2023-10-09T05:57:53.506 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.23399434 2023-10-09T05:57:53.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.507 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:53.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849406+0000", 2023-10-09T05:57:53.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2062999999999999e-05 2023-10-09T05:57:53.507 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.507 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:53.507 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.507 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.507 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104152 6.1 6.22f756c1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:57:53.507 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:40.286265+0000", 2023-10-09T05:57:53.507 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 193.19789120799999, 2023-10-09T05:57:53.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5631822680000003, 2023-10-09T05:57:53.508 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:53.508 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:57:53.508 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:53.508 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:53.508 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:53.508 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104152 2023-10-09T05:57:53.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.508 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:53.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:53.508 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.286265+0000", 2023-10-09T05:57:53.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:53.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.286265+0000", 2023-10-09T05:57:53.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:53.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.286263+0000", 2023-10-09T05:57:53.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:57:53.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:53.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.286271+0000", 2023-10-09T05:57:53.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9289999999999998e-06 2023-10-09T05:57:53.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:53.511 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.286285+0000", 2023-10-09T05:57:53.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3209e-05 2023-10-09T05:57:53.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:53.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.286293+0000", 2023-10-09T05:57:53.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4379999999999995e-06 2023-10-09T05:57:53.512 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849433+0000", 2023-10-09T05:57:53.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5631398769999993 2023-10-09T05:57:53.512 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:53.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849447+0000", 2023-10-09T05:57:53.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4353e-05 2023-10-09T05:57:53.513 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.513 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:53.513 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.513 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.513 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104178 6.1 6.53d24ae1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:57:53.513 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:40.851048+0000", 2023-10-09T05:57:53.513 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 192.63310839900001, 2023-10-09T05:57:53.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9984412760000003, 2023-10-09T05:57:53.513 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:53.514 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:57:53.514 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:53.514 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:53.514 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:53.514 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104178 2023-10-09T05:57:53.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.514 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:53.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:53.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.851048+0000", 2023-10-09T05:57:53.514 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:53.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.851048+0000", 2023-10-09T05:57:53.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:53.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.851046+0000", 2023-10-09T05:57:53.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:57:53.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.516 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:53.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.851055+0000", 2023-10-09T05:57:53.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9400000000000008e-06 2023-10-09T05:57:53.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.516 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:53.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.851070+0000", 2023-10-09T05:57:53.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5096000000000001e-05 2023-10-09T05:57:53.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.516 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:53.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.851082+0000", 2023-10-09T05:57:53.517 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2004000000000001e-05 2023-10-09T05:57:53.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.517 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.517 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849474+0000", 2023-10-09T05:57:53.517 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9983919610000003 2023-10-09T05:57:53.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.517 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:53.517 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849489+0000", 2023-10-09T05:57:53.517 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5037e-05 2023-10-09T05:57:53.518 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.518 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:53.518 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.518 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.518 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104195 6.1 6.f5ea83b1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:57:53.518 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:41.193752+0000", 2023-10-09T05:57:53.518 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 192.290403908, 2023-10-09T05:57:53.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.655772367, 2023-10-09T05:57:53.518 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:53.518 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:57:53.519 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:53.519 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:53.519 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:53.519 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104195 2023-10-09T05:57:53.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.519 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:53.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:53.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.193752+0000", 2023-10-09T05:57:53.519 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.520 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:53.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.193752+0000", 2023-10-09T05:57:53.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.520 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:53.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.193750+0000", 2023-10-09T05:57:53.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999971 2023-10-09T05:57:53.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:53.521 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.193760+0000", 2023-10-09T05:57:53.521 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0652999999999999e-05 2023-10-09T05:57:53.521 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:53.521 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.193778+0000", 2023-10-09T05:57:53.521 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7849000000000001e-05 2023-10-09T05:57:53.521 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:53.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.193788+0000", 2023-10-09T05:57:53.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8609999999999993e-06 2023-10-09T05:57:53.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849512+0000", 2023-10-09T05:57:53.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6557239150000003 2023-10-09T05:57:53.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:53.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849525+0000", 2023-10-09T05:57:53.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2850999999999999e-05 2023-10-09T05:57:53.523 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.523 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:53.523 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.523 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104206 6.1 6.fe55d9d1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:57:53.523 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:41.359887+0000", 2023-10-09T05:57:53.523 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 192.124269212, 2023-10-09T05:57:53.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4896904470000001, 2023-10-09T05:57:53.523 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:53.524 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:57:53.524 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:53.524 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:53.524 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:53.524 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104206 2023-10-09T05:57:53.524 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.524 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:53.524 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.524 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:53.524 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.359887+0000", 2023-10-09T05:57:53.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.525 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:53.525 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.359887+0000", 2023-10-09T05:57:53.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.525 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:53.525 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.359885+0000", 2023-10-09T05:57:53.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:57:53.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:53.526 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.359894+0000", 2023-10-09T05:57:53.526 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9309999999999997e-06 2023-10-09T05:57:53.526 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:53.526 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.359908+0000", 2023-10-09T05:57:53.526 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4185999999999999e-05 2023-10-09T05:57:53.526 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.527 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:53.527 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.359919+0000", 2023-10-09T05:57:53.527 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0611000000000001e-05 2023-10-09T05:57:53.527 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.527 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.527 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.527 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849563+0000", 2023-10-09T05:57:53.527 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4896442780000001 2023-10-09T05:57:53.527 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.527 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.527 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:53.528 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849577+0000", 2023-10-09T05:57:53.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4161999999999999e-05 2023-10-09T05:57:53.528 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.528 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:53.528 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.528 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104234 6.1 6.d0896e1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:57:53.528 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:41.671864+0000", 2023-10-09T05:57:53.528 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 191.812292439, 2023-10-09T05:57:53.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1777485309999998, 2023-10-09T05:57:53.528 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:53.528 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:57:53.529 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:53.529 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:53.529 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:53.529 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104234 2023-10-09T05:57:53.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.529 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:53.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:53.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.671864+0000", 2023-10-09T05:57:53.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:53.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.671864+0000", 2023-10-09T05:57:53.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:53.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.671862+0000", 2023-10-09T05:57:53.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:57:53.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:53.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.671872+0000", 2023-10-09T05:57:53.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0444e-05 2023-10-09T05:57:53.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:53.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.671891+0000", 2023-10-09T05:57:53.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8637000000000001e-05 2023-10-09T05:57:53.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.532 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:53.532 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.671901+0000", 2023-10-09T05:57:53.532 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7079999999999992e-06 2023-10-09T05:57:53.532 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.532 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.532 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849599+0000", 2023-10-09T05:57:53.532 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1776986379999999 2023-10-09T05:57:53.532 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.532 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:53.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849612+0000", 2023-10-09T05:57:53.533 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2996999999999999e-05 2023-10-09T05:57:53.533 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.533 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:53.533 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.533 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.533 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.533 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104244 6.1 6.700f7241 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:57:53.533 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:41.971632+0000", 2023-10-09T05:57:53.533 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 191.512523859, 2023-10-09T05:57:53.533 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8780479510000001, 2023-10-09T05:57:53.533 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:53.534 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:57:53.534 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:53.534 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:53.534 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:53.534 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104244 2023-10-09T05:57:53.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.534 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:53.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.534 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:53.534 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.971632+0000", 2023-10-09T05:57:53.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.535 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:53.535 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.971632+0000", 2023-10-09T05:57:53.535 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.535 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:53.535 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.971630+0000", 2023-10-09T05:57:53.535 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:57:53.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.536 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:53.536 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.971640+0000", 2023-10-09T05:57:53.536 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0138e-05 2023-10-09T05:57:53.536 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.536 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.536 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:53.536 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.971663+0000", 2023-10-09T05:57:53.536 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2561999999999998e-05 2023-10-09T05:57:53.536 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.536 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.536 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:53.536 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.971675+0000", 2023-10-09T05:57:53.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1970000000000001e-05 2023-10-09T05:57:53.537 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.537 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.537 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.537 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849666+0000", 2023-10-09T05:57:53.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8779913940000004 2023-10-09T05:57:53.537 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.537 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.537 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:53.537 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849680+0000", 2023-10-09T05:57:53.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3917000000000001e-05 2023-10-09T05:57:53.537 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.538 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:53.538 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.538 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.538 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104255 6.1 6.f28f24e1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:57:53.538 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:42.044751+0000", 2023-10-09T05:57:53.538 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 191.43940524999999, 2023-10-09T05:57:53.538 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8049857540000005, 2023-10-09T05:57:53.538 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:53.538 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:57:53.538 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:53.539 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:53.539 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:53.539 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104255 2023-10-09T05:57:53.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.539 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:53.539 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.539 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:53.539 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.044751+0000", 2023-10-09T05:57:53.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.539 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.539 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:53.540 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.044751+0000", 2023-10-09T05:57:53.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.540 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.540 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.540 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:53.540 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.044749+0000", 2023-10-09T05:57:53.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:57:53.540 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.540 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.540 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:53.540 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.044759+0000", 2023-10-09T05:57:53.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0732e-05 2023-10-09T05:57:53.540 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.541 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.541 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:53.541 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.044779+0000", 2023-10-09T05:57:53.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9806e-05 2023-10-09T05:57:53.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.541 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.541 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:53.541 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.044791+0000", 2023-10-09T05:57:53.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1562999999999999e-05 2023-10-09T05:57:53.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.541 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.541 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.542 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849721+0000", 2023-10-09T05:57:53.542 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8049299540000003 2023-10-09T05:57:53.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.542 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:53.542 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849737+0000", 2023-10-09T05:57:53.542 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6058000000000001e-05 2023-10-09T05:57:53.542 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.542 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:53.542 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.543 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104262 6.1 6.9203e6d1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:57:53.543 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:42.111284+0000", 2023-10-09T05:57:53.543 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 191.37287244000001, 2023-10-09T05:57:53.543 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.738488383, 2023-10-09T05:57:53.543 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:53.543 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:57:53.543 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:53.543 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:53.543 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:53.543 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104262 2023-10-09T05:57:53.543 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.543 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:53.544 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.544 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:53.544 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.111284+0000", 2023-10-09T05:57:53.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.544 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.544 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.544 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:53.544 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.111284+0000", 2023-10-09T05:57:53.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.544 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.544 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.544 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:53.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.111282+0000", 2023-10-09T05:57:53.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:57:53.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.545 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:53.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.111289+0000", 2023-10-09T05:57:53.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7549999999999997e-06 2023-10-09T05:57:53.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.545 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:53.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.111303+0000", 2023-10-09T05:57:53.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4024e-05 2023-10-09T05:57:53.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.546 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:53.546 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.111311+0000", 2023-10-09T05:57:53.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2930000000000001e-06 2023-10-09T05:57:53.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.546 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.546 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849760+0000", 2023-10-09T05:57:53.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7384484999999996 2023-10-09T05:57:53.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.547 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:53.547 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849772+0000", 2023-10-09T05:57:53.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2429000000000001e-05 2023-10-09T05:57:53.547 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.547 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:53.547 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.547 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.547 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.547 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104285 6.1 6.2bf38801 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:57:53.547 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:42.596255+0000", 2023-10-09T05:57:53.547 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 190.88790129899999, 2023-10-09T05:57:53.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2535693370000001, 2023-10-09T05:57:53.548 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:53.548 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:57:53.548 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:53.548 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:53.548 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:53.548 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104285 2023-10-09T05:57:53.548 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.548 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:53.548 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.548 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:53.548 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.596255+0000", 2023-10-09T05:57:53.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.549 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:53.549 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.596255+0000", 2023-10-09T05:57:53.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.549 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:53.549 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.596253+0000", 2023-10-09T05:57:53.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:57:53.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:53.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.596264+0000", 2023-10-09T05:57:53.550 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1596e-05 2023-10-09T05:57:53.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:53.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.596292+0000", 2023-10-09T05:57:53.550 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7569e-05 2023-10-09T05:57:53.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:53.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.596305+0000", 2023-10-09T05:57:53.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2629999999999999e-05 2023-10-09T05:57:53.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849810+0000", 2023-10-09T05:57:53.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2535054219999999 2023-10-09T05:57:53.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:53.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849824+0000", 2023-10-09T05:57:53.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4324999999999999e-05 2023-10-09T05:57:53.552 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.552 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:53.552 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.552 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104290 6.1 6.415302b1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:57:53.552 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:42.695406+0000", 2023-10-09T05:57:53.552 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 190.78875033400001, 2023-10-09T05:57:53.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1544539709999997, 2023-10-09T05:57:53.552 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:53.552 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:57:53.552 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:53.552 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:53.553 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:53.553 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104290 2023-10-09T05:57:53.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.553 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:53.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:53.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.695406+0000", 2023-10-09T05:57:53.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:53.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.695406+0000", 2023-10-09T05:57:53.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:53.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.695404+0000", 2023-10-09T05:57:53.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:57:53.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:53.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.695413+0000", 2023-10-09T05:57:53.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5669999999999995e-06 2023-10-09T05:57:53.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:53.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.695428+0000", 2023-10-09T05:57:53.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5549999999999999e-05 2023-10-09T05:57:53.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:53.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.695438+0000", 2023-10-09T05:57:53.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6479999999999998e-06 2023-10-09T05:57:53.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.556 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849847+0000", 2023-10-09T05:57:53.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1544087489999999 2023-10-09T05:57:53.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:53.556 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849860+0000", 2023-10-09T05:57:53.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3277e-05 2023-10-09T05:57:53.556 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.556 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:53.556 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.557 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119186 6.21 6:86b6e4ca:::benchmark_data_smithi064_284193_object22643:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e214)", 2023-10-09T05:57:53.557 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:44.656755+0000", 2023-10-09T05:57:53.557 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 128.827401487, 2023-10-09T05:57:53.557 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1584478259999997, 2023-10-09T05:57:53.557 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:53.557 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:53.557 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:53.557 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:53.557 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:53.557 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119186 2023-10-09T05:57:53.557 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.557 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:53.558 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.558 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:53.558 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656755+0000", 2023-10-09T05:57:53.558 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.558 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.558 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.558 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:53.558 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656755+0000", 2023-10-09T05:57:53.558 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.558 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.558 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.558 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:53.559 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656753+0000", 2023-10-09T05:57:53.559 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:57:53.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.559 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:53.559 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656760+0000", 2023-10-09T05:57:53.559 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8290000000000002e-06 2023-10-09T05:57:53.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.559 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:53.559 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656773+0000", 2023-10-09T05:57:53.559 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3166000000000001e-05 2023-10-09T05:57:53.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:53.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656783+0000", 2023-10-09T05:57:53.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0210000000000001e-05 2023-10-09T05:57:53.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.658579+0000", 2023-10-09T05:57:53.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001795488 2023-10-09T05:57:53.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.561 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:57:53.561 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.658592+0000", 2023-10-09T05:57:53.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.278e-05 2023-10-09T05:57:53.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.561 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.561 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.561 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.743680+0000", 2023-10-09T05:57:53.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0850882960000003 2023-10-09T05:57:53.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.561 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.561 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:57:53.561 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.749035+0000", 2023-10-09T05:57:53.562 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053553639999999996 2023-10-09T05:57:53.562 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.562 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.562 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.562 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.776409+0000", 2023-10-09T05:57:53.562 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.027373859 2023-10-09T05:57:53.562 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.562 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.562 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:57:53.562 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.778175+0000", 2023-10-09T05:57:53.562 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001765953 2023-10-09T05:57:53.562 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:57:53.563 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.781558+0000", 2023-10-09T05:57:53.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033831220000000001 2023-10-09T05:57:53.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:53.563 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.784638+0000", 2023-10-09T05:57:53.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030801969999999998 2023-10-09T05:57:53.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:53.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.785224+0000", 2023-10-09T05:57:53.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00058571999999999997 2023-10-09T05:57:53.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:57:53.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.811281+0000", 2023-10-09T05:57:53.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.026056551000000001 2023-10-09T05:57:53.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:57:53.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.811998+0000", 2023-10-09T05:57:53.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00071781999999999998 2023-10-09T05:57:53.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:53.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.815203+0000", 2023-10-09T05:57:53.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032041819999999999 2023-10-09T05:57:53.565 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.565 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:53.565 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.565 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119190 6.11 6:89fd5615:::benchmark_data_smithi064_284193_object22647:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e214)", 2023-10-09T05:57:53.565 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:44.656790+0000", 2023-10-09T05:57:53.565 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 128.82736650300001, 2023-10-09T05:57:53.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0863524140000003, 2023-10-09T05:57:53.566 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:53.566 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:53.566 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:53.566 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:53.566 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:53.566 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119190 2023-10-09T05:57:53.566 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.566 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:53.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.566 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:53.567 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656790+0000", 2023-10-09T05:57:53.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.567 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.567 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:53.567 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656790+0000", 2023-10-09T05:57:53.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.567 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.567 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:53.567 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656789+0000", 2023-10-09T05:57:53.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:57:53.567 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.568 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.568 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:53.568 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656792+0000", 2023-10-09T05:57:53.568 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6599999999999999e-06 2023-10-09T05:57:53.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.568 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.568 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:53.568 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656798+0000", 2023-10-09T05:57:53.568 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9449999999999998e-06 2023-10-09T05:57:53.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.568 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.568 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:53.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656803+0000", 2023-10-09T05:57:53.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6060000000000002e-06 2023-10-09T05:57:53.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.658616+0000", 2023-10-09T05:57:53.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018122049999999999 2023-10-09T05:57:53.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:57:53.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.658631+0000", 2023-10-09T05:57:53.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4922999999999999e-05 2023-10-09T05:57:53.570 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.570 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.570 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.663857+0000", 2023-10-09T05:57:53.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0052261009999999 2023-10-09T05:57:53.570 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.570 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.570 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:57:53.571 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.673837+0000", 2023-10-09T05:57:53.571 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0099803089999999997 2023-10-09T05:57:53.571 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.571 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.571 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.571 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.705811+0000", 2023-10-09T05:57:53.571 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.031974225000000002 2023-10-09T05:57:53.571 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.571 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.571 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:57:53.571 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.708838+0000", 2023-10-09T05:57:53.571 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030267570000000001 2023-10-09T05:57:53.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.572 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.572 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:57:53.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.715396+0000", 2023-10-09T05:57:53.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065577710000000004 2023-10-09T05:57:53.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.572 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.572 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:53.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.731615+0000", 2023-10-09T05:57:53.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016219506000000002 2023-10-09T05:57:53.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.572 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:53.573 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.732772+0000", 2023-10-09T05:57:53.573 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011562300000000001 2023-10-09T05:57:53.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:57:53.573 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.733208+0000", 2023-10-09T05:57:53.573 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000436728 2023-10-09T05:57:53.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:57:53.573 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.734975+0000", 2023-10-09T05:57:53.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001766739 2023-10-09T05:57:53.574 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.574 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.574 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:53.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.743142+0000", 2023-10-09T05:57:53.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0081671590000000002 2023-10-09T05:57:53.574 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.574 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:53.574 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.574 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.574 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.575 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119234 6.11 6:8be56ac0:::benchmark_data_smithi064_284193_object22691:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-10-09T05:57:53.575 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:44.944674+0000", 2023-10-09T05:57:53.575 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 128.53948231999999, 2023-10-09T05:57:53.575 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9036751780000003, 2023-10-09T05:57:53.575 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:53.575 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:53.575 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:53.575 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:53.575 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:53.575 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119234 2023-10-09T05:57:53.575 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.575 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:53.575 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.576 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:53.576 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.944674+0000", 2023-10-09T05:57:53.576 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.576 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.576 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.576 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:53.576 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.944674+0000", 2023-10-09T05:57:53.576 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.576 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.576 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.576 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:53.576 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.944672+0000", 2023-10-09T05:57:53.577 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:57:53.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:53.577 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.944681+0000", 2023-10-09T05:57:53.577 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1509999999999993e-06 2023-10-09T05:57:53.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:53.577 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.944699+0000", 2023-10-09T05:57:53.577 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7589e-05 2023-10-09T05:57:53.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.578 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.578 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:53.578 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.944708+0000", 2023-10-09T05:57:53.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9090000000000006e-06 2023-10-09T05:57:53.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.578 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.578 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.578 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.944762+0000", 2023-10-09T05:57:53.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4095000000000003e-05 2023-10-09T05:57:53.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.578 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.578 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:57:53.579 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.944774+0000", 2023-10-09T05:57:53.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2248e-05 2023-10-09T05:57:53.579 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.579 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.579 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.720858+0000", 2023-10-09T05:57:53.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7760837649999992 2023-10-09T05:57:53.579 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.579 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:57:53.579 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.730024+0000", 2023-10-09T05:57:53.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0091661759999999998 2023-10-09T05:57:53.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.580 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.580 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.580 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.775720+0000", 2023-10-09T05:57:53.580 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.045695982000000003 2023-10-09T05:57:53.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.580 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.580 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:57:53.580 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.782885+0000", 2023-10-09T05:57:53.580 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0071655690000000001 2023-10-09T05:57:53.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.580 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:57:53.581 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.792563+0000", 2023-10-09T05:57:53.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0096778569999999998 2023-10-09T05:57:53.581 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.581 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:53.581 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.810903+0000", 2023-10-09T05:57:53.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018339273999999999 2023-10-09T05:57:53.581 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.581 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:53.581 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.812158+0000", 2023-10-09T05:57:53.582 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001255775 2023-10-09T05:57:53.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.582 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:57:53.582 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.836393+0000", 2023-10-09T05:57:53.582 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.024234970000000002 2023-10-09T05:57:53.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.582 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:57:53.582 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.839508+0000", 2023-10-09T05:57:53.582 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031143970000000001 2023-10-09T05:57:53.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.583 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.583 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:53.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.848349+0000", 2023-10-09T05:57:53.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0088413299999999997 2023-10-09T05:57:53.583 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.583 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:53.583 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.583 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.583 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.583 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119237 6.11 6:8ee80634:::benchmark_data_smithi064_284193_object22694:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-10-09T05:57:53.583 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:45.702303+0000", 2023-10-09T05:57:53.583 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 127.78185354599999, 2023-10-09T05:57:53.584 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1973806820000004, 2023-10-09T05:57:53.584 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:53.584 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:53.584 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:53.584 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:53.584 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:53.584 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119237 2023-10-09T05:57:53.584 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.584 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:53.584 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.584 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:53.585 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.702303+0000", 2023-10-09T05:57:53.585 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.585 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.585 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:53.585 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.702303+0000", 2023-10-09T05:57:53.585 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.585 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.585 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:53.585 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.702301+0000", 2023-10-09T05:57:53.585 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:57:53.586 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.586 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.586 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:53.586 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.702308+0000", 2023-10-09T05:57:53.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2830000000000004e-06 2023-10-09T05:57:53.586 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.586 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.586 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:53.586 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.716185+0000", 2023-10-09T05:57:53.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013876292 2023-10-09T05:57:53.586 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.586 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:53.587 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.716200+0000", 2023-10-09T05:57:53.587 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5145e-05 2023-10-09T05:57:53.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.587 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.798524+0000", 2023-10-09T05:57:53.587 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0823238269999997 2023-10-09T05:57:53.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:57:53.587 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.809274+0000", 2023-10-09T05:57:53.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010750116000000001 2023-10-09T05:57:53.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.848945+0000", 2023-10-09T05:57:53.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.039671062999999999 2023-10-09T05:57:53.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:57:53.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.854842+0000", 2023-10-09T05:57:53.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058972850000000004 2023-10-09T05:57:53.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:57:53.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.862009+0000", 2023-10-09T05:57:53.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0071667279999999998 2023-10-09T05:57:53.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:53.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.885199+0000", 2023-10-09T05:57:53.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023190399 2023-10-09T05:57:53.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:57:53.590 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.885882+0000", 2023-10-09T05:57:53.590 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00068252700000000005 2023-10-09T05:57:53.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.590 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:53.590 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.888407+0000", 2023-10-09T05:57:53.590 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025256649999999999 2023-10-09T05:57:53.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.590 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:57:53.590 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.890797+0000", 2023-10-09T05:57:53.590 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0023894400000000001 2023-10-09T05:57:53.591 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.591 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.591 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:53.591 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.899683+0000", 2023-10-09T05:57:53.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0088864919999999993 2023-10-09T05:57:53.591 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.591 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:53.591 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.591 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.591 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.591 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119262 6.21 6:845e8c29:::benchmark_data_smithi064_284193_object22719:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-10-09T05:57:53.591 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:45.846736+0000", 2023-10-09T05:57:53.592 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 127.637420105, 2023-10-09T05:57:53.592 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0664286440000001, 2023-10-09T05:57:53.592 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:53.592 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:53.592 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:53.592 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:53.592 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:53.592 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119262 2023-10-09T05:57:53.592 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.592 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:53.592 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.592 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:53.593 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.846736+0000", 2023-10-09T05:57:53.593 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.593 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.593 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:53.593 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.846736+0000", 2023-10-09T05:57:53.593 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.593 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.593 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:53.593 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.846734+0000", 2023-10-09T05:57:53.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:57:53.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.594 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.594 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:53.594 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.846743+0000", 2023-10-09T05:57:53.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0820000000000005e-06 2023-10-09T05:57:53.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.594 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.594 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:53.594 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.846757+0000", 2023-10-09T05:57:53.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4083999999999999e-05 2023-10-09T05:57:53.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.595 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.595 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:53.595 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.846765+0000", 2023-10-09T05:57:53.595 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3339999999999996e-06 2023-10-09T05:57:53.595 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.595 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.595 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.595 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.848937+0000", 2023-10-09T05:57:53.595 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0021717930000005 2023-10-09T05:57:53.595 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.595 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.595 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:57:53.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.854164+0000", 2023-10-09T05:57:53.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0052279120000000004 2023-10-09T05:57:53.596 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.596 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.596 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.900206+0000", 2023-10-09T05:57:53.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.046042012 2023-10-09T05:57:53.596 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.596 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.596 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:57:53.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.901864+0000", 2023-10-09T05:57:53.597 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001657258 2023-10-09T05:57:53.597 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.597 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.597 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:57:53.597 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.905026+0000", 2023-10-09T05:57:53.597 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031627719999999999 2023-10-09T05:57:53.597 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.597 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.597 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:53.597 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.908161+0000", 2023-10-09T05:57:53.597 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031345349999999999 2023-10-09T05:57:53.597 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.598 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.598 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:57:53.598 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.908349+0000", 2023-10-09T05:57:53.598 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000187911 2023-10-09T05:57:53.598 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.598 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.598 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:53.598 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.908935+0000", 2023-10-09T05:57:53.598 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00058620600000000003 2023-10-09T05:57:53.598 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.598 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.598 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:57:53.599 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.909877+0000", 2023-10-09T05:57:53.599 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00094206200000000002 2023-10-09T05:57:53.599 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.599 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:53.599 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.913165+0000", 2023-10-09T05:57:53.599 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003287594 2023-10-09T05:57:53.599 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.599 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:53.599 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.599 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.600 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119308 6.11 6:8cb42c43:::benchmark_data_smithi064_284193_object22765:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-10-09T05:57:53.600 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:46.087787+0000", 2023-10-09T05:57:53.600 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 127.396369595, 2023-10-09T05:57:53.600 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8988220839999999, 2023-10-09T05:57:53.600 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:53.600 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:53.600 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:53.600 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:53.600 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:53.600 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119308 2023-10-09T05:57:53.600 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.600 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:53.601 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.601 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:53.601 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.087787+0000", 2023-10-09T05:57:53.601 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.601 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.601 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.601 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:53.601 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.087787+0000", 2023-10-09T05:57:53.601 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.601 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.601 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.601 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:53.602 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.087785+0000", 2023-10-09T05:57:53.602 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:57:53.602 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.602 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.602 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:53.602 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.087796+0000", 2023-10-09T05:57:53.602 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1157e-05 2023-10-09T05:57:53.602 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.602 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.602 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:53.602 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.087811+0000", 2023-10-09T05:57:53.602 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5238999999999999e-05 2023-10-09T05:57:53.603 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.603 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:53.603 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.087822+0000", 2023-10-09T05:57:53.603 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1399e-05 2023-10-09T05:57:53.603 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.603 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.603 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.867168+0000", 2023-10-09T05:57:53.603 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7793453689999996 2023-10-09T05:57:53.603 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.604 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:57:53.604 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.883209+0000", 2023-10-09T05:57:53.604 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016041239999999998 2023-10-09T05:57:53.604 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.604 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.604 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.604 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.922185+0000", 2023-10-09T05:57:53.604 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.038975888 2023-10-09T05:57:53.604 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.604 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.604 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:57:53.604 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.928697+0000", 2023-10-09T05:57:53.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065118830000000004 2023-10-09T05:57:53.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:57:53.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.941529+0000", 2023-10-09T05:57:53.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012832487 2023-10-09T05:57:53.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:57:53.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.966726+0000", 2023-10-09T05:57:53.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.025196347000000001 2023-10-09T05:57:53.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:53.606 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.968542+0000", 2023-10-09T05:57:53.606 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001816736 2023-10-09T05:57:53.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:53.606 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.970863+0000", 2023-10-09T05:57:53.606 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00232102 2023-10-09T05:57:53.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:57:53.607 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.974139+0000", 2023-10-09T05:57:53.607 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003276143 2023-10-09T05:57:53.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.607 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:53.607 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.986609+0000", 2023-10-09T05:57:53.607 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012469236 2023-10-09T05:57:53.607 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.607 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:53.607 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.608 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119315 6.21 6:844be45b:::benchmark_data_smithi064_284193_object22772:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-10-09T05:57:53.608 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:46.187262+0000", 2023-10-09T05:57:53.608 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 127.29689454299999, 2023-10-09T05:57:53.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7650340010000001, 2023-10-09T05:57:53.608 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:53.608 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:53.608 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:53.608 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:53.608 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:53.608 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119315 2023-10-09T05:57:53.608 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.608 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:53.609 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.609 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:53.609 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.187262+0000", 2023-10-09T05:57:53.609 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.609 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.609 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.609 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:53.609 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.187262+0000", 2023-10-09T05:57:53.609 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.609 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.609 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.609 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:53.610 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.187260+0000", 2023-10-09T05:57:53.610 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:57:53.610 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.610 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.610 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:53.610 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.187270+0000", 2023-10-09T05:57:53.610 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0227000000000001e-05 2023-10-09T05:57:53.610 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.610 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.610 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:53.610 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.187287+0000", 2023-10-09T05:57:53.610 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6762e-05 2023-10-09T05:57:53.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.611 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:53.611 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.187298+0000", 2023-10-09T05:57:53.611 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1574e-05 2023-10-09T05:57:53.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.611 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.611 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.913351+0000", 2023-10-09T05:57:53.611 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7260523179999998 2023-10-09T05:57:53.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:57:53.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.917641+0000", 2023-10-09T05:57:53.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042900730000000002 2023-10-09T05:57:53.612 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.930458+0000", 2023-10-09T05:57:53.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012817034999999999 2023-10-09T05:57:53.612 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:57:53.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.932203+0000", 2023-10-09T05:57:53.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001745459 2023-10-09T05:57:53.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:57:53.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.935768+0000", 2023-10-09T05:57:53.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00356522 2023-10-09T05:57:53.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:53.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.938933+0000", 2023-10-09T05:57:53.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031648140000000002 2023-10-09T05:57:53.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:53.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.947606+0000", 2023-10-09T05:57:53.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086732140000000003 2023-10-09T05:57:53.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:57:53.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.947767+0000", 2023-10-09T05:57:53.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016054900000000001 2023-10-09T05:57:53.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:57:53.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.948714+0000", 2023-10-09T05:57:53.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00094673999999999995 2023-10-09T05:57:53.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.615 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:53.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.952296+0000", 2023-10-09T05:57:53.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035820740000000002 2023-10-09T05:57:53.615 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.615 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:53.615 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.615 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119330 6.11 6:8f58021f:::benchmark_data_smithi064_284193_object22787:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-10-09T05:57:53.616 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:46.350194+0000", 2023-10-09T05:57:53.616 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 127.133962481, 2023-10-09T05:57:53.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7595639490000003, 2023-10-09T05:57:53.616 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:57:53.616 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:57:53.616 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:57:53.616 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:57:53.616 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:57:53.616 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119330 2023-10-09T05:57:53.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.616 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:57:53.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.617 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:57:53.617 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.350194+0000", 2023-10-09T05:57:53.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.617 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.617 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.617 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:57:53.617 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.350194+0000", 2023-10-09T05:57:53.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:57:53.617 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.617 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.617 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:57:53.617 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.350192+0000", 2023-10-09T05:57:53.618 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:57:53.618 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.618 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.618 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:57:53.618 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.350201+0000", 2023-10-09T05:57:53.618 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0410000000000003e-06 2023-10-09T05:57:53.618 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.618 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.618 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:57:53.618 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.350222+0000", 2023-10-09T05:57:53.618 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0968e-05 2023-10-09T05:57:53.619 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.619 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:57:53.619 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.350232+0000", 2023-10-09T05:57:53.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8330000000000006e-06 2023-10-09T05:57:53.619 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.619 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.619 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.947802+0000", 2023-10-09T05:57:53.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5975698659999997 2023-10-09T05:57:53.619 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.619 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:57:53.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.966003+0000", 2023-10-09T05:57:53.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018200917 2023-10-09T05:57:53.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.620 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:57:53.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:54.029395+0000", 2023-10-09T05:57:53.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.063392047000000007 2023-10-09T05:57:53.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.620 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:57:53.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:54.038664+0000", 2023-10-09T05:57:53.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0092691679999999995 2023-10-09T05:57:53.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:57:53.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:54.047296+0000", 2023-10-09T05:57:53.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086318609999999994 2023-10-09T05:57:53.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:53.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:54.067975+0000", 2023-10-09T05:57:53.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020679792999999998 2023-10-09T05:57:53.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:57:53.622 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:54.070461+0000", 2023-10-09T05:57:53.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024859119999999998 2023-10-09T05:57:53.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:57:53.622 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:54.094248+0000", 2023-10-09T05:57:53.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023787081000000002 2023-10-09T05:57:53.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:57:53.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:54.098008+0000", 2023-10-09T05:57:53.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037596750000000001 2023-10-09T05:57:53.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:57:53.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:57:53.623 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:57:53.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:54.109758+0000", 2023-10-09T05:57:53.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011749625 2023-10-09T05:57:53.623 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.623 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:53.623 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.623 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:57:53.623 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:57:53.624 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:57:53.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:53.625 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:57:53.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:57:53.725 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:53.786 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:53 smithi064 ceph-mon[212750]: pgmap v600: 139 pgs: 116 active+clean, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped; 519 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.15k op/s; 20327/24954 objects degraded (81.458%); 2359/24954 objects misplaced (9.453%); 0 B/s, 37 objects/s recovering 2023-10-09T05:57:53.786 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:53 smithi064 ceph-mon[212750]: osdmap e261: 8 total, 8 up, 7 in 2023-10-09T05:57:53.786 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:53 smithi064 ceph-mon[212750]: failed to encode map e261 with expected crc 2023-10-09T05:57:53.786 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:53 smithi064 ceph-mon[212750]: failed to encode map e261 with expected crc 2023-10-09T05:57:53.786 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:53 smithi064 ceph-mon[212750]: failed to encode map e261 with expected crc 2023-10-09T05:57:53.786 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:53 smithi064 ceph-mon[212750]: failed to encode map e261 with expected crc 2023-10-09T05:57:53.787 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:53 smithi064 ceph-mon[212750]: failed to encode map e261 with expected crc 2023-10-09T05:57:53.787 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:53 smithi064 ceph-mon[212750]: 6.15 starting backfill to osd.6 from (0'0,0'0] MAX to 258'8819 2023-10-09T05:57:53.789 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:53 smithi064 ceph-mon[220044]: pgmap v600: 139 pgs: 116 active+clean, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped; 519 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.15k op/s; 20327/24954 objects degraded (81.458%); 2359/24954 objects misplaced (9.453%); 0 B/s, 37 objects/s recovering 2023-10-09T05:57:53.789 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:53 smithi064 ceph-mon[220044]: osdmap e261: 8 total, 8 up, 7 in 2023-10-09T05:57:53.789 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:53 smithi064 ceph-mon[220044]: failed to encode map e261 with expected crc 2023-10-09T05:57:53.789 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:53 smithi064 ceph-mon[220044]: failed to encode map e261 with expected crc 2023-10-09T05:57:53.789 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:53 smithi064 ceph-mon[220044]: failed to encode map e261 with expected crc 2023-10-09T05:57:53.789 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:53 smithi064 ceph-mon[220044]: failed to encode map e261 with expected crc 2023-10-09T05:57:53.789 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:53 smithi064 ceph-mon[220044]: failed to encode map e261 with expected crc 2023-10-09T05:57:53.790 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:53 smithi064 ceph-mon[220044]: 6.15 starting backfill to osd.6 from (0'0,0'0] MAX to 258'8819 2023-10-09T05:57:53.794 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:54.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:53 smithi156 ceph-mon[195101]: pgmap v600: 139 pgs: 116 active+clean, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped; 519 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.15k op/s; 20327/24954 objects degraded (81.458%); 2359/24954 objects misplaced (9.453%); 0 B/s, 37 objects/s recovering 2023-10-09T05:57:54.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:53 smithi156 ceph-mon[195101]: osdmap e261: 8 total, 8 up, 7 in 2023-10-09T05:57:54.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:53 smithi156 ceph-mon[195101]: failed to encode map e261 with expected crc 2023-10-09T05:57:54.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:53 smithi156 ceph-mon[195101]: failed to encode map e261 with expected crc 2023-10-09T05:57:54.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:53 smithi156 ceph-mon[195101]: failed to encode map e261 with expected crc 2023-10-09T05:57:54.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:53 smithi156 ceph-mon[195101]: failed to encode map e261 with expected crc 2023-10-09T05:57:54.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:53 smithi156 ceph-mon[195101]: failed to encode map e261 with expected crc 2023-10-09T05:57:54.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:53 smithi156 ceph-mon[195101]: 6.15 starting backfill to osd.6 from (0'0,0'0] MAX to 258'8819 2023-10-09T05:57:54.567 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:57:54.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:54.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:57:54.673 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:57:54.773 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:57:54.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:54 smithi064 ceph-mon[220044]: failed to encode map e262 with expected crc 2023-10-09T05:57:54.887 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:54 smithi064 ceph-mon[220044]: osdmap e262: 8 total, 8 up, 7 in 2023-10-09T05:57:54.887 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:54 smithi064 ceph-mon[220044]: failed to encode map e262 with expected crc 2023-10-09T05:57:54.887 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:54 smithi064 ceph-mon[220044]: failed to encode map e262 with expected crc 2023-10-09T05:57:54.887 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:54 smithi064 ceph-mon[220044]: failed to encode map e262 with expected crc 2023-10-09T05:57:54.887 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:54 smithi064 ceph-mon[220044]: failed to encode map e262 with expected crc 2023-10-09T05:57:54.888 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:54 smithi064 ceph-mon[220044]: failed to encode map e262 with expected crc 2023-10-09T05:57:54.888 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:54 smithi064 ceph-mon[220044]: failed to encode map e262 with expected crc 2023-10-09T05:57:54.888 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:54 smithi064 ceph-mon[212750]: failed to encode map e262 with expected crc 2023-10-09T05:57:54.889 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:54 smithi064 ceph-mon[212750]: osdmap e262: 8 total, 8 up, 7 in 2023-10-09T05:57:54.889 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:54 smithi064 ceph-mon[212750]: failed to encode map e262 with expected crc 2023-10-09T05:57:54.889 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:54 smithi064 ceph-mon[212750]: failed to encode map e262 with expected crc 2023-10-09T05:57:54.889 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:54 smithi064 ceph-mon[212750]: failed to encode map e262 with expected crc 2023-10-09T05:57:54.889 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:54 smithi064 ceph-mon[212750]: failed to encode map e262 with expected crc 2023-10-09T05:57:54.889 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:54 smithi064 ceph-mon[212750]: failed to encode map e262 with expected crc 2023-10-09T05:57:54.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:54 smithi064 ceph-mon[212750]: failed to encode map e262 with expected crc 2023-10-09T05:57:54.994 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:57:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:54 smithi156 ceph-mon[195101]: failed to encode map e262 with expected crc 2023-10-09T05:57:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:54 smithi156 ceph-mon[195101]: osdmap e262: 8 total, 8 up, 7 in 2023-10-09T05:57:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:54 smithi156 ceph-mon[195101]: failed to encode map e262 with expected crc 2023-10-09T05:57:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:54 smithi156 ceph-mon[195101]: failed to encode map e262 with expected crc 2023-10-09T05:57:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:54 smithi156 ceph-mon[195101]: failed to encode map e262 with expected crc 2023-10-09T05:57:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:54 smithi156 ceph-mon[195101]: failed to encode map e262 with expected crc 2023-10-09T05:57:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:54 smithi156 ceph-mon[195101]: failed to encode map e262 with expected crc 2023-10-09T05:57:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:54 smithi156 ceph-mon[195101]: failed to encode map e262 with expected crc 2023-10-09T05:57:55.711 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:57:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:57:55] "GET /metrics HTTP/1.1" 200 36463 "" "Prometheus/2.43.0" 2023-10-09T05:57:55.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:55 smithi064 ceph-mon[212750]: pgmap v603: 139 pgs: 116 active+clean, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped; 469 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 260 op/s; 20314/22575 objects degraded (89.984%); 2173/22575 objects misplaced (9.626%); 208 KiB/s, 204 objects/s recovering 2023-10-09T05:57:55.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:55 smithi064 ceph-mon[212750]: failed to encode map e263 with expected crc 2023-10-09T05:57:55.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:55 smithi064 ceph-mon[212750]: failed to encode map e263 with expected crc 2023-10-09T05:57:55.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:55 smithi064 ceph-mon[212750]: osdmap e263: 8 total, 8 up, 7 in 2023-10-09T05:57:55.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:55 smithi064 ceph-mon[212750]: failed to encode map e263 with expected crc 2023-10-09T05:57:55.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:55 smithi064 ceph-mon[212750]: failed to encode map e263 with expected crc 2023-10-09T05:57:55.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:55 smithi064 ceph-mon[212750]: failed to encode map e263 with expected crc 2023-10-09T05:57:55.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:55 smithi064 ceph-mon[212750]: failed to encode map e263 with expected crc 2023-10-09T05:57:55.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:55 smithi064 ceph-mon[212750]: failed to encode map e263 with expected crc 2023-10-09T05:57:55.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:55 smithi064 ceph-mon[220044]: pgmap v603: 139 pgs: 116 active+clean, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped; 469 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 260 op/s; 20314/22575 objects degraded (89.984%); 2173/22575 objects misplaced (9.626%); 208 KiB/s, 204 objects/s recovering 2023-10-09T05:57:55.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:55 smithi064 ceph-mon[220044]: failed to encode map e263 with expected crc 2023-10-09T05:57:55.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:55 smithi064 ceph-mon[220044]: failed to encode map e263 with expected crc 2023-10-09T05:57:55.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:55 smithi064 ceph-mon[220044]: osdmap e263: 8 total, 8 up, 7 in 2023-10-09T05:57:55.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:55 smithi064 ceph-mon[220044]: failed to encode map e263 with expected crc 2023-10-09T05:57:55.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:55 smithi064 ceph-mon[220044]: failed to encode map e263 with expected crc 2023-10-09T05:57:55.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:55 smithi064 ceph-mon[220044]: failed to encode map e263 with expected crc 2023-10-09T05:57:55.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:55 smithi064 ceph-mon[220044]: failed to encode map e263 with expected crc 2023-10-09T05:57:55.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:55 smithi064 ceph-mon[220044]: failed to encode map e263 with expected crc 2023-10-09T05:57:56.045 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:55 smithi156 ceph-mon[195101]: pgmap v603: 139 pgs: 116 active+clean, 1 active+recovering+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped; 469 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 260 op/s; 20314/22575 objects degraded (89.984%); 2173/22575 objects misplaced (9.626%); 208 KiB/s, 204 objects/s recovering 2023-10-09T05:57:56.045 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:55 smithi156 ceph-mon[195101]: failed to encode map e263 with expected crc 2023-10-09T05:57:56.045 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:55 smithi156 ceph-mon[195101]: failed to encode map e263 with expected crc 2023-10-09T05:57:56.045 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:55 smithi156 ceph-mon[195101]: osdmap e263: 8 total, 8 up, 7 in 2023-10-09T05:57:56.046 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:55 smithi156 ceph-mon[195101]: failed to encode map e263 with expected crc 2023-10-09T05:57:56.046 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:55 smithi156 ceph-mon[195101]: failed to encode map e263 with expected crc 2023-10-09T05:57:56.046 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:55 smithi156 ceph-mon[195101]: failed to encode map e263 with expected crc 2023-10-09T05:57:56.046 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:55 smithi156 ceph-mon[195101]: failed to encode map e263 with expected crc 2023-10-09T05:57:56.046 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:55 smithi156 ceph-mon[195101]: failed to encode map e263 with expected crc 2023-10-09T05:57:56.189 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:56.779 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:56.971 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:57:57.009 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:57:57.047 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:57.048 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:57:57.084 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:57:57.124 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:57:57.153 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:57:57.179 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:56 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:57:57.186 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:57:57.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:56 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:57:57.221 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:57:57.228 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:56 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:57:57.234 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:57:57.234 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:57:57.234 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:57:57.235 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:57:57.235 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:57:57.235 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:57:57.235 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:57:57.235 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:57:57.459 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:57:57.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:57.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:57:57.807 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:57:57.816 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T05:57:57.917 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:57 smithi156 ceph-mon[195101]: pgmap v605: 139 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped; 330 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 23245/15873 objects degraded (146.444%); 1953/15873 objects misplaced (12.304%); 170 KiB/s, 167 objects/s recovering 2023-10-09T05:57:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:57 smithi156 ceph-mon[195101]: failed to encode map e264 with expected crc 2023-10-09T05:57:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:57 smithi156 ceph-mon[195101]: osdmap e264: 8 total, 8 up, 7 in 2023-10-09T05:57:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:57 smithi156 ceph-mon[195101]: failed to encode map e264 with expected crc 2023-10-09T05:57:58.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:57 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3845410219' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:58.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:57 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 23245/15873 objects degraded (146.444%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:58.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:57 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3506067604' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:57:58.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:57 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:57:58.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:57 smithi064 ceph-mon[212750]: pgmap v605: 139 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped; 330 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 23245/15873 objects degraded (146.444%); 1953/15873 objects misplaced (12.304%); 170 KiB/s, 167 objects/s recovering 2023-10-09T05:57:58.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:57 smithi064 ceph-mon[212750]: failed to encode map e264 with expected crc 2023-10-09T05:57:58.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:57 smithi064 ceph-mon[212750]: osdmap e264: 8 total, 8 up, 7 in 2023-10-09T05:57:58.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:57 smithi064 ceph-mon[212750]: failed to encode map e264 with expected crc 2023-10-09T05:57:58.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:57 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3845410219' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:58.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:57 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 23245/15873 objects degraded (146.444%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:58.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:57 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3506067604' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:57:58.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:57 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:57:58.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:57 smithi064 ceph-mon[220044]: pgmap v605: 139 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped; 330 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 23245/15873 objects degraded (146.444%); 1953/15873 objects misplaced (12.304%); 170 KiB/s, 167 objects/s recovering 2023-10-09T05:57:58.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:57 smithi064 ceph-mon[220044]: failed to encode map e264 with expected crc 2023-10-09T05:57:58.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:57 smithi064 ceph-mon[220044]: osdmap e264: 8 total, 8 up, 7 in 2023-10-09T05:57:58.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:57 smithi064 ceph-mon[220044]: failed to encode map e264 with expected crc 2023-10-09T05:57:58.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:57 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3845410219' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:57:58.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:57 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 23245/15873 objects degraded (146.444%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T05:57:58.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:57 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3506067604' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:57:58.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:57 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:57:58.602 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:57:58.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:58.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:57:58.708 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:57:58.743 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:58.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:58 smithi156 ceph-mon[195101]: failed to encode map e265 with expected crc 2023-10-09T05:57:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:58 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:57:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:58 smithi156 ceph-mon[195101]: osdmap e265: 8 total, 8 up, 7 in 2023-10-09T05:57:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:58 smithi156 ceph-mon[195101]: failed to encode map e265 with expected crc 2023-10-09T05:57:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:58 smithi156 ceph-mon[195101]: failed to encode map e265 with expected crc 2023-10-09T05:57:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:58 smithi156 ceph-mon[195101]: failed to encode map e265 with expected crc 2023-10-09T05:57:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:58 smithi156 ceph-mon[195101]: failed to encode map e265 with expected crc 2023-10-09T05:57:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:58 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:59.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:58 smithi064 ceph-mon[212750]: failed to encode map e265 with expected crc 2023-10-09T05:57:59.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:58 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:57:59.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:58 smithi064 ceph-mon[212750]: osdmap e265: 8 total, 8 up, 7 in 2023-10-09T05:57:59.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:58 smithi064 ceph-mon[212750]: failed to encode map e265 with expected crc 2023-10-09T05:57:59.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:58 smithi064 ceph-mon[212750]: failed to encode map e265 with expected crc 2023-10-09T05:57:59.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:58 smithi064 ceph-mon[212750]: failed to encode map e265 with expected crc 2023-10-09T05:57:59.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:58 smithi064 ceph-mon[212750]: failed to encode map e265 with expected crc 2023-10-09T05:57:59.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:59.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:58 smithi064 ceph-mon[220044]: failed to encode map e265 with expected crc 2023-10-09T05:57:59.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:58 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:57:59.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:58 smithi064 ceph-mon[220044]: osdmap e265: 8 total, 8 up, 7 in 2023-10-09T05:57:59.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:58 smithi064 ceph-mon[220044]: failed to encode map e265 with expected crc 2023-10-09T05:57:59.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:58 smithi064 ceph-mon[220044]: failed to encode map e265 with expected crc 2023-10-09T05:57:59.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:58 smithi064 ceph-mon[220044]: failed to encode map e265 with expected crc 2023-10-09T05:57:59.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:58 smithi064 ceph-mon[220044]: failed to encode map e265 with expected crc 2023-10-09T05:57:59.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:58 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:59.546 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:57:59.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:57:59.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:57:59.650 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:57:59.680 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:57:59.829 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:59 smithi064 ceph-mon[220044]: pgmap v608: 139 pgs: 1 peering, 113 active+clean, 11 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped; 279 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 23199/13458 objects degraded (172.381%); 1953/13458 objects misplaced (14.512%); 778 KiB/s, 81 objects/s recovering 2023-10-09T05:57:59.829 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:59 smithi064 ceph-mon[220044]: failed to encode map e266 with expected crc 2023-10-09T05:57:59.829 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:59 smithi064 ceph-mon[220044]: failed to encode map e266 with expected crc 2023-10-09T05:57:59.830 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:59 smithi064 ceph-mon[220044]: osdmap e266: 8 total, 8 up, 7 in 2023-10-09T05:57:59.830 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:59 smithi064 ceph-mon[220044]: failed to encode map e266 with expected crc 2023-10-09T05:57:59.830 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:59 smithi064 ceph-mon[220044]: failed to encode map e266 with expected crc 2023-10-09T05:57:59.830 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:59 smithi064 ceph-mon[220044]: failed to encode map e266 with expected crc 2023-10-09T05:57:59.830 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:59 smithi064 ceph-mon[220044]: failed to encode map e266 with expected crc 2023-10-09T05:57:59.830 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:59 smithi064 ceph-mon[220044]: failed to encode map e266 with expected crc 2023-10-09T05:57:59.830 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:59 smithi064 ceph-mon[220044]: failed to encode map e266 with expected crc 2023-10-09T05:58:00.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:59 smithi156 ceph-mon[195101]: pgmap v608: 139 pgs: 1 peering, 113 active+clean, 11 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped; 279 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 23199/13458 objects degraded (172.381%); 1953/13458 objects misplaced (14.512%); 778 KiB/s, 81 objects/s recovering 2023-10-09T05:58:00.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:59 smithi156 ceph-mon[195101]: failed to encode map e266 with expected crc 2023-10-09T05:58:00.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:59 smithi156 ceph-mon[195101]: failed to encode map e266 with expected crc 2023-10-09T05:58:00.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:59 smithi156 ceph-mon[195101]: osdmap e266: 8 total, 8 up, 7 in 2023-10-09T05:58:00.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:59 smithi156 ceph-mon[195101]: failed to encode map e266 with expected crc 2023-10-09T05:58:00.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:59 smithi156 ceph-mon[195101]: failed to encode map e266 with expected crc 2023-10-09T05:58:00.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:59 smithi156 ceph-mon[195101]: failed to encode map e266 with expected crc 2023-10-09T05:58:00.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:59 smithi156 ceph-mon[195101]: failed to encode map e266 with expected crc 2023-10-09T05:58:00.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:59 smithi156 ceph-mon[195101]: failed to encode map e266 with expected crc 2023-10-09T05:58:00.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:59 smithi156 ceph-mon[195101]: failed to encode map e266 with expected crc 2023-10-09T05:58:00.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:59 smithi156 ceph-mon[195101]: failed to encode map e266 with expected crc 2023-10-09T05:58:00.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:57:59 smithi156 ceph-mon[195101]: failed to encode map e266 with expected crc 2023-10-09T05:58:00.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:59 smithi064 ceph-mon[212750]: pgmap v608: 139 pgs: 1 peering, 113 active+clean, 11 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped; 279 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 23199/13458 objects degraded (172.381%); 1953/13458 objects misplaced (14.512%); 778 KiB/s, 81 objects/s recovering 2023-10-09T05:58:00.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:59 smithi064 ceph-mon[212750]: failed to encode map e266 with expected crc 2023-10-09T05:58:00.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:59 smithi064 ceph-mon[212750]: failed to encode map e266 with expected crc 2023-10-09T05:58:00.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:59 smithi064 ceph-mon[212750]: osdmap e266: 8 total, 8 up, 7 in 2023-10-09T05:58:00.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:59 smithi064 ceph-mon[212750]: failed to encode map e266 with expected crc 2023-10-09T05:58:00.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:59 smithi064 ceph-mon[212750]: failed to encode map e266 with expected crc 2023-10-09T05:58:00.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:59 smithi064 ceph-mon[212750]: failed to encode map e266 with expected crc 2023-10-09T05:58:00.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:59 smithi064 ceph-mon[212750]: failed to encode map e266 with expected crc 2023-10-09T05:58:00.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:59 smithi064 ceph-mon[212750]: failed to encode map e266 with expected crc 2023-10-09T05:58:00.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:59 smithi064 ceph-mon[212750]: failed to encode map e266 with expected crc 2023-10-09T05:58:00.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:59 smithi064 ceph-mon[212750]: failed to encode map e266 with expected crc 2023-10-09T05:58:00.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:57:59 smithi064 ceph-mon[212750]: failed to encode map e266 with expected crc 2023-10-09T05:58:00.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:59 smithi064 ceph-mon[220044]: failed to encode map e266 with expected crc 2023-10-09T05:58:00.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:57:59 smithi064 ceph-mon[220044]: failed to encode map e266 with expected crc 2023-10-09T05:58:00.572 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T05:58:00.572 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:58:00.575 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:00.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:00.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:00.687 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:00.781 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:00.846 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:00.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:00.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:00.954 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:00.999 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:01.077 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:00 smithi064 ceph-mon[212750]: 6.1b starting backfill to osd.7 from (0'0,0'0] MAX to 264'9442 2023-10-09T05:58:01.077 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:00 smithi064 ceph-mon[212750]: failed to encode map e267 with expected crc 2023-10-09T05:58:01.077 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:00 smithi064 ceph-mon[212750]: osdmap e267: 8 total, 8 up, 7 in 2023-10-09T05:58:01.077 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:00 smithi064 ceph-mon[212750]: failed to encode map e267 with expected crc 2023-10-09T05:58:01.077 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:00 smithi064 ceph-mon[212750]: failed to encode map e267 with expected crc 2023-10-09T05:58:01.077 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:00 smithi064 ceph-mon[212750]: 6.2b starting backfill to osd.7 from (106'587,168'6569] MIN to 264'8720 2023-10-09T05:58:01.078 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:00 smithi064 ceph-mon[220044]: 6.1b starting backfill to osd.7 from (0'0,0'0] MAX to 264'9442 2023-10-09T05:58:01.078 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:00 smithi064 ceph-mon[220044]: failed to encode map e267 with expected crc 2023-10-09T05:58:01.078 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:00 smithi064 ceph-mon[220044]: osdmap e267: 8 total, 8 up, 7 in 2023-10-09T05:58:01.078 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:00 smithi064 ceph-mon[220044]: failed to encode map e267 with expected crc 2023-10-09T05:58:01.078 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:00 smithi064 ceph-mon[220044]: failed to encode map e267 with expected crc 2023-10-09T05:58:01.079 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:00 smithi064 ceph-mon[220044]: 6.2b starting backfill to osd.7 from (106'587,168'6569] MIN to 264'8720 2023-10-09T05:58:01.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:00 smithi156 ceph-mon[195101]: 6.1b starting backfill to osd.7 from (0'0,0'0] MAX to 264'9442 2023-10-09T05:58:01.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:00 smithi156 ceph-mon[195101]: failed to encode map e267 with expected crc 2023-10-09T05:58:01.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:00 smithi156 ceph-mon[195101]: osdmap e267: 8 total, 8 up, 7 in 2023-10-09T05:58:01.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:00 smithi156 ceph-mon[195101]: failed to encode map e267 with expected crc 2023-10-09T05:58:01.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:00 smithi156 ceph-mon[195101]: failed to encode map e267 with expected crc 2023-10-09T05:58:01.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:00 smithi156 ceph-mon[195101]: 6.2b starting backfill to osd.7 from (106'587,168'6569] MIN to 264'8720 2023-10-09T05:58:01.133 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:01.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:01.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:01.238 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:58:01.238 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:58:01.238 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:01.275 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:01.357 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:01.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:01.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:01.471 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:01.537 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:01.637 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:01.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:01.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:01.743 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:01.809 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:01.890 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T05:58:01.950 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:01.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:01.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:02.060 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:02.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:01 smithi156 ceph-mon[195101]: pgmap v611: 139 pgs: 5 remapped+peering, 1 peering, 108 active+clean, 11 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped; 152 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 714 op/s; 22362/7356 objects degraded (303.997%); 1389/7356 objects misplaced (18.883%); 2.7 MiB/s, 296 objects/s recovering 2023-10-09T05:58:02.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:01 smithi156 ceph-mon[195101]: failed to encode map e268 with expected crc 2023-10-09T05:58:02.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:01 smithi156 ceph-mon[195101]: failed to encode map e268 with expected crc 2023-10-09T05:58:02.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:01 smithi156 ceph-mon[195101]: osdmap e268: 8 total, 8 up, 7 in 2023-10-09T05:58:02.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:01 smithi156 ceph-mon[195101]: failed to encode map e268 with expected crc 2023-10-09T05:58:02.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:01 smithi156 ceph-mon[195101]: failed to encode map e268 with expected crc 2023-10-09T05:58:02.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:01 smithi156 ceph-mon[195101]: failed to encode map e268 with expected crc 2023-10-09T05:58:02.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:01 smithi156 ceph-mon[195101]: failed to encode map e268 with expected crc 2023-10-09T05:58:02.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:01 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1483749588' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:02.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:01 smithi064 ceph-mon[212750]: pgmap v611: 139 pgs: 5 remapped+peering, 1 peering, 108 active+clean, 11 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped; 152 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 714 op/s; 22362/7356 objects degraded (303.997%); 1389/7356 objects misplaced (18.883%); 2.7 MiB/s, 296 objects/s recovering 2023-10-09T05:58:02.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:01 smithi064 ceph-mon[212750]: failed to encode map e268 with expected crc 2023-10-09T05:58:02.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:01 smithi064 ceph-mon[212750]: failed to encode map e268 with expected crc 2023-10-09T05:58:02.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:01 smithi064 ceph-mon[212750]: osdmap e268: 8 total, 8 up, 7 in 2023-10-09T05:58:02.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:01 smithi064 ceph-mon[212750]: failed to encode map e268 with expected crc 2023-10-09T05:58:02.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:01 smithi064 ceph-mon[212750]: failed to encode map e268 with expected crc 2023-10-09T05:58:02.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:01 smithi064 ceph-mon[212750]: failed to encode map e268 with expected crc 2023-10-09T05:58:02.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:01 smithi064 ceph-mon[212750]: failed to encode map e268 with expected crc 2023-10-09T05:58:02.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:01 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1483749588' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:02.123 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:01 smithi064 ceph-mon[220044]: pgmap v611: 139 pgs: 5 remapped+peering, 1 peering, 108 active+clean, 11 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped; 152 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 714 op/s; 22362/7356 objects degraded (303.997%); 1389/7356 objects misplaced (18.883%); 2.7 MiB/s, 296 objects/s recovering 2023-10-09T05:58:02.123 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:01 smithi064 ceph-mon[220044]: failed to encode map e268 with expected crc 2023-10-09T05:58:02.123 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:01 smithi064 ceph-mon[220044]: failed to encode map e268 with expected crc 2023-10-09T05:58:02.123 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:01 smithi064 ceph-mon[220044]: osdmap e268: 8 total, 8 up, 7 in 2023-10-09T05:58:02.123 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:01 smithi064 ceph-mon[220044]: failed to encode map e268 with expected crc 2023-10-09T05:58:02.123 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:01 smithi064 ceph-mon[220044]: failed to encode map e268 with expected crc 2023-10-09T05:58:02.123 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:01 smithi064 ceph-mon[220044]: failed to encode map e268 with expected crc 2023-10-09T05:58:02.123 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:01 smithi064 ceph-mon[220044]: failed to encode map e268 with expected crc 2023-10-09T05:58:02.123 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:01 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1483749588' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:02.135 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:02.546 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:58:02.800 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Removed 96540 objects 2023-10-09T05:58:02.801 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Clean up completed and total clean up time :343.4 2023-10-09T05:58:02.815 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:02.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:02.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:02.921 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:02.953 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:03.020 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:03.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:03.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:03.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:02 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:58:03.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:02 smithi156 ceph-mon[195101]: failed to encode map e269 with expected crc 2023-10-09T05:58:03.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:02 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1483749588' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:58:03.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:02 smithi156 ceph-mon[195101]: osdmap e269: 8 total, 8 up, 7 in 2023-10-09T05:58:03.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:02 smithi156 ceph-mon[195101]: failed to encode map e269 with expected crc 2023-10-09T05:58:03.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:02 smithi156 ceph-mon[195101]: failed to encode map e269 with expected crc 2023-10-09T05:58:03.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:02 smithi156 ceph-mon[195101]: failed to encode map e269 with expected crc 2023-10-09T05:58:03.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:02 smithi156 ceph-mon[195101]: failed to encode map e269 with expected crc 2023-10-09T05:58:03.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:02 smithi156 ceph-mon[195101]: failed to encode map e269 with expected crc 2023-10-09T05:58:03.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:02 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 22362/7356 objects degraded (303.997%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T05:58:03.126 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:03.152 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:02 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:58:03.153 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:02 smithi064 ceph-mon[220044]: failed to encode map e269 with expected crc 2023-10-09T05:58:03.153 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:02 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1483749588' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:58:03.153 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:02 smithi064 ceph-mon[220044]: osdmap e269: 8 total, 8 up, 7 in 2023-10-09T05:58:03.153 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:02 smithi064 ceph-mon[220044]: failed to encode map e269 with expected crc 2023-10-09T05:58:03.153 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:02 smithi064 ceph-mon[220044]: failed to encode map e269 with expected crc 2023-10-09T05:58:03.153 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:02 smithi064 ceph-mon[220044]: failed to encode map e269 with expected crc 2023-10-09T05:58:03.153 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:02 smithi064 ceph-mon[220044]: failed to encode map e269 with expected crc 2023-10-09T05:58:03.154 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:02 smithi064 ceph-mon[220044]: failed to encode map e269 with expected crc 2023-10-09T05:58:03.154 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:02 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 22362/7356 objects degraded (303.997%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T05:58:03.154 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:02 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:58:03.154 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:02 smithi064 ceph-mon[212750]: failed to encode map e269 with expected crc 2023-10-09T05:58:03.154 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:02 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1483749588' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:58:03.155 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:02 smithi064 ceph-mon[212750]: osdmap e269: 8 total, 8 up, 7 in 2023-10-09T05:58:03.155 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:02 smithi064 ceph-mon[212750]: failed to encode map e269 with expected crc 2023-10-09T05:58:03.155 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:02 smithi064 ceph-mon[212750]: failed to encode map e269 with expected crc 2023-10-09T05:58:03.155 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:02 smithi064 ceph-mon[212750]: failed to encode map e269 with expected crc 2023-10-09T05:58:03.155 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:02 smithi064 ceph-mon[212750]: failed to encode map e269 with expected crc 2023-10-09T05:58:03.155 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:02 smithi064 ceph-mon[212750]: failed to encode map e269 with expected crc 2023-10-09T05:58:03.155 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:02 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 22362/7356 objects degraded (303.997%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T05:58:03.156 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:03.935 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:58:03.966 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:03.968 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:03.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:03.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:04.075 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:04.104 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:04.151 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:04.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:04.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:04.215 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:04.256 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:04.294 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:04.368 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:04.370 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 05:58:04 smithi156 conmon[175419]: logger=sqlstore.transactions t=2023-10-09T05:58:04.002411124Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-10-09T05:58:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:04 smithi156 ceph-mon[195101]: pgmap v614: 139 pgs: 1 active+recovering+undersized+remapped, 5 remapped+peering, 1 peering, 108 active+clean, 11 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped; 146 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 20411/7050 objects degraded (289.518%); 1389/7050 objects misplaced (19.702%); 2.0 MiB/s, 364 objects/s recovering 2023-10-09T05:58:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:04 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:58:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:04 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:58:04.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:04.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:04.473 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:04.499 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:04.543 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:04.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:04.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:04.649 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:04.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:04 smithi064 ceph-mon[212750]: pgmap v614: 139 pgs: 1 active+recovering+undersized+remapped, 5 remapped+peering, 1 peering, 108 active+clean, 11 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped; 146 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 20411/7050 objects degraded (289.518%); 1389/7050 objects misplaced (19.702%); 2.0 MiB/s, 364 objects/s recovering 2023-10-09T05:58:04.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:04 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:58:04.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:04 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:58:04.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:04 smithi064 ceph-mon[220044]: pgmap v614: 139 pgs: 1 active+recovering+undersized+remapped, 5 remapped+peering, 1 peering, 108 active+clean, 11 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped; 146 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 20411/7050 objects degraded (289.518%); 1389/7050 objects misplaced (19.702%); 2.0 MiB/s, 364 objects/s recovering 2023-10-09T05:58:04.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:04 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:58:04.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:04 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:58:04.676 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:04.720 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:04.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:04.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:04.825 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:04.863 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:05.721 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:58:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:58:05] "GET /metrics HTTP/1.1" 200 36444 "" "Prometheus/2.43.0" 2023-10-09T05:58:05.822 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:05.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:05.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:05.927 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:05.954 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:05.988 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:58:06.001 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:06.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:06.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:06.111 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:06.138 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:06.184 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:06.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:06.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:06.289 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:06.318 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:06.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:06 smithi156 ceph-mon[195101]: pgmap v615: 139 pgs: 1 active+recovering+undersized+remapped, 5 remapped+peering, 109 active+clean, 11 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped; 116 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 20266/5592 objects degraded (362.411%); 1299/5592 objects misplaced (23.230%); 3.0 MiB/s, 276 objects/s recovering 2023-10-09T05:58:06.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:06 smithi064 ceph-mon[212750]: pgmap v615: 139 pgs: 1 active+recovering+undersized+remapped, 5 remapped+peering, 109 active+clean, 11 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped; 116 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 20266/5592 objects degraded (362.411%); 1299/5592 objects misplaced (23.230%); 3.0 MiB/s, 276 objects/s recovering 2023-10-09T05:58:06.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:06 smithi064 ceph-mon[220044]: pgmap v615: 139 pgs: 1 active+recovering+undersized+remapped, 5 remapped+peering, 109 active+clean, 11 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped; 116 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 20266/5592 objects degraded (362.411%); 1299/5592 objects misplaced (23.230%); 3.0 MiB/s, 276 objects/s recovering 2023-10-09T05:58:06.818 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:06.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:06.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:06.921 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:06.938 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:06.967 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:07.050 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:07.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:07.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:07.143 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:07.159 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:07.207 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:07.371 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:07.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:07.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:07 smithi064 ceph-mon[212750]: failed to encode map e270 with expected crc 2023-10-09T05:58:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:07 smithi064 ceph-mon[212750]: failed to encode map e270 with expected crc 2023-10-09T05:58:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:07 smithi064 ceph-mon[212750]: osdmap e270: 8 total, 8 up, 7 in 2023-10-09T05:58:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:07 smithi064 ceph-mon[212750]: failed to encode map e270 with expected crc 2023-10-09T05:58:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:07 smithi064 ceph-mon[212750]: failed to encode map e270 with expected crc 2023-10-09T05:58:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:07 smithi064 ceph-mon[212750]: failed to encode map e270 with expected crc 2023-10-09T05:58:07.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:07 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 20266/5592 objects degraded (362.411%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T05:58:07.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:07 smithi064 ceph-mon[212750]: osdmap e271: 8 total, 8 up, 7 in 2023-10-09T05:58:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:07 smithi064 ceph-mon[220044]: failed to encode map e270 with expected crc 2023-10-09T05:58:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:07 smithi064 ceph-mon[220044]: failed to encode map e270 with expected crc 2023-10-09T05:58:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:07 smithi064 ceph-mon[220044]: osdmap e270: 8 total, 8 up, 7 in 2023-10-09T05:58:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:07 smithi064 ceph-mon[220044]: failed to encode map e270 with expected crc 2023-10-09T05:58:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:07 smithi064 ceph-mon[220044]: failed to encode map e270 with expected crc 2023-10-09T05:58:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:07 smithi064 ceph-mon[220044]: failed to encode map e270 with expected crc 2023-10-09T05:58:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:07 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 20266/5592 objects degraded (362.411%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T05:58:07.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:07 smithi064 ceph-mon[220044]: osdmap e271: 8 total, 8 up, 7 in 2023-10-09T05:58:07.477 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:07.509 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:07.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:07 smithi156 ceph-mon[195101]: failed to encode map e270 with expected crc 2023-10-09T05:58:07.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:07 smithi156 ceph-mon[195101]: failed to encode map e270 with expected crc 2023-10-09T05:58:07.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:07 smithi156 ceph-mon[195101]: osdmap e270: 8 total, 8 up, 7 in 2023-10-09T05:58:07.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:07 smithi156 ceph-mon[195101]: failed to encode map e270 with expected crc 2023-10-09T05:58:07.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:07 smithi156 ceph-mon[195101]: failed to encode map e270 with expected crc 2023-10-09T05:58:07.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:07 smithi156 ceph-mon[195101]: failed to encode map e270 with expected crc 2023-10-09T05:58:07.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:07 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 20266/5592 objects degraded (362.411%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T05:58:07.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:07 smithi156 ceph-mon[195101]: osdmap e271: 8 total, 8 up, 7 in 2023-10-09T05:58:07.914 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:07.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:07.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:08.018 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:08.045 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:08.195 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:08.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:08.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:08.301 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:08.326 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:08 smithi156 ceph-mon[195101]: pgmap v617: 139 pgs: 1 active+recovering+undersized+remapped, 111 active+clean, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped; 33 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 395 op/s; 20227/1644 objects degraded (1230.353%); 1092/1644 objects misplaced (66.423%); 2.0 MiB/s, 290 objects/s recovering 2023-10-09T05:58:08.327 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:08 smithi156 ceph-mon[195101]: failed to encode map e271 with expected crc 2023-10-09T05:58:08.327 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:08 smithi156 ceph-mon[195101]: failed to encode map e271 with expected crc 2023-10-09T05:58:08.327 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:08 smithi156 ceph-mon[195101]: failed to encode map e271 with expected crc 2023-10-09T05:58:08.327 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:08 smithi156 ceph-mon[195101]: failed to encode map e271 with expected crc 2023-10-09T05:58:08.327 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:08 smithi156 ceph-mon[195101]: failed to encode map e271 with expected crc 2023-10-09T05:58:08.327 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:08 smithi156 ceph-mon[195101]: failed to encode map e271 with expected crc 2023-10-09T05:58:08.328 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:08 smithi156 ceph-mon[195101]: failed to encode map e271 with expected crc 2023-10-09T05:58:08.328 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:08 smithi156 ceph-mon[195101]: osdmap e272: 8 total, 8 up, 7 in 2023-10-09T05:58:08.329 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:08.380 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:08.386 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:58:08.389 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T05:58:08.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:08.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:08.490 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:08.519 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:08.569 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:08.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:08.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:08.673 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:08.684 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:08 smithi064 ceph-mon[212750]: pgmap v617: 139 pgs: 1 active+recovering+undersized+remapped, 111 active+clean, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped; 33 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 395 op/s; 20227/1644 objects degraded (1230.353%); 1092/1644 objects misplaced (66.423%); 2.0 MiB/s, 290 objects/s recovering 2023-10-09T05:58:08.684 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:08 smithi064 ceph-mon[212750]: failed to encode map e271 with expected crc 2023-10-09T05:58:08.684 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:08 smithi064 ceph-mon[212750]: failed to encode map e271 with expected crc 2023-10-09T05:58:08.684 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:08 smithi064 ceph-mon[212750]: failed to encode map e271 with expected crc 2023-10-09T05:58:08.684 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:08 smithi064 ceph-mon[212750]: failed to encode map e271 with expected crc 2023-10-09T05:58:08.684 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:08 smithi064 ceph-mon[212750]: failed to encode map e271 with expected crc 2023-10-09T05:58:08.684 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:08 smithi064 ceph-mon[212750]: failed to encode map e271 with expected crc 2023-10-09T05:58:08.684 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:08 smithi064 ceph-mon[212750]: failed to encode map e271 with expected crc 2023-10-09T05:58:08.684 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:08 smithi064 ceph-mon[212750]: osdmap e272: 8 total, 8 up, 7 in 2023-10-09T05:58:08.685 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:08 smithi064 ceph-mon[220044]: pgmap v617: 139 pgs: 1 active+recovering+undersized+remapped, 111 active+clean, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped; 33 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 395 op/s; 20227/1644 objects degraded (1230.353%); 1092/1644 objects misplaced (66.423%); 2.0 MiB/s, 290 objects/s recovering 2023-10-09T05:58:08.685 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:08 smithi064 ceph-mon[220044]: failed to encode map e271 with expected crc 2023-10-09T05:58:08.685 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:08 smithi064 ceph-mon[220044]: failed to encode map e271 with expected crc 2023-10-09T05:58:08.685 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:08 smithi064 ceph-mon[220044]: failed to encode map e271 with expected crc 2023-10-09T05:58:08.685 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:08 smithi064 ceph-mon[220044]: failed to encode map e271 with expected crc 2023-10-09T05:58:08.685 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:08 smithi064 ceph-mon[220044]: failed to encode map e271 with expected crc 2023-10-09T05:58:08.685 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:08 smithi064 ceph-mon[220044]: failed to encode map e271 with expected crc 2023-10-09T05:58:08.685 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:08 smithi064 ceph-mon[220044]: failed to encode map e271 with expected crc 2023-10-09T05:58:08.685 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:08 smithi064 ceph-mon[220044]: osdmap e272: 8 total, 8 up, 7 in 2023-10-09T05:58:08.689 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-10-09T05:58:08.689 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-10-09T05:58:08.702 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:08.754 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:08.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:08.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:08.860 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:08.889 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:09.430 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:58:09.452 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:58:09.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:09 smithi064 ceph-mon[212750]: failed to encode map e272 with expected crc 2023-10-09T05:58:09.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:09 smithi064 ceph-mon[212750]: failed to encode map e272 with expected crc 2023-10-09T05:58:09.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:09 smithi064 ceph-mon[212750]: failed to encode map e272 with expected crc 2023-10-09T05:58:09.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:09 smithi064 ceph-mon[212750]: failed to encode map e272 with expected crc 2023-10-09T05:58:09.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:09 smithi064 ceph-mon[212750]: failed to encode map e272 with expected crc 2023-10-09T05:58:09.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:09 smithi064 ceph-mon[212750]: failed to encode map e272 with expected crc 2023-10-09T05:58:09.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:09 smithi064 ceph-mon[212750]: osdmap e273: 8 total, 8 up, 7 in 2023-10-09T05:58:09.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:09 smithi064 ceph-mon[220044]: failed to encode map e272 with expected crc 2023-10-09T05:58:09.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:09 smithi064 ceph-mon[220044]: failed to encode map e272 with expected crc 2023-10-09T05:58:09.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:09 smithi064 ceph-mon[220044]: failed to encode map e272 with expected crc 2023-10-09T05:58:09.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:09 smithi064 ceph-mon[220044]: failed to encode map e272 with expected crc 2023-10-09T05:58:09.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:09 smithi064 ceph-mon[220044]: failed to encode map e272 with expected crc 2023-10-09T05:58:09.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:09 smithi064 ceph-mon[220044]: failed to encode map e272 with expected crc 2023-10-09T05:58:09.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:09 smithi064 ceph-mon[220044]: osdmap e273: 8 total, 8 up, 7 in 2023-10-09T05:58:09.478 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:58:09.498 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:58:09.523 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:58:09.545 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:58:09.564 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:58:09.584 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:58:09.585 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:58:09.585 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:58:09.585 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:58:09.585 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:58:09.585 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:58:09.585 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:58:09.585 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:58:09.585 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:58:09.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:09 smithi156 ceph-mon[195101]: failed to encode map e272 with expected crc 2023-10-09T05:58:09.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:09 smithi156 ceph-mon[195101]: failed to encode map e272 with expected crc 2023-10-09T05:58:09.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:09 smithi156 ceph-mon[195101]: failed to encode map e272 with expected crc 2023-10-09T05:58:09.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:09 smithi156 ceph-mon[195101]: failed to encode map e272 with expected crc 2023-10-09T05:58:09.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:09 smithi156 ceph-mon[195101]: failed to encode map e272 with expected crc 2023-10-09T05:58:09.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:09 smithi156 ceph-mon[195101]: failed to encode map e272 with expected crc 2023-10-09T05:58:09.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:09 smithi156 ceph-mon[195101]: osdmap e273: 8 total, 8 up, 7 in 2023-10-09T05:58:10.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:10 smithi156 ceph-mon[195101]: from='client.45179 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:58:10.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:10 smithi156 ceph-mon[195101]: pgmap v620: 139 pgs: 1 remapped+peering, 111 active+clean, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped; 33 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 375 op/s; 20227/1614 objects degraded (1253.222%); 1092/1614 objects misplaced (67.658%); 2.0 MiB/s, 396 objects/s recovering 2023-10-09T05:58:10.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:10 smithi156 ceph-mon[195101]: failed to encode map e273 with expected crc 2023-10-09T05:58:10.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:10 smithi156 ceph-mon[195101]: failed to encode map e273 with expected crc 2023-10-09T05:58:10.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:10 smithi156 ceph-mon[195101]: failed to encode map e273 with expected crc 2023-10-09T05:58:10.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:10 smithi156 ceph-mon[195101]: failed to encode map e273 with expected crc 2023-10-09T05:58:10.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:10 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2239290908' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:10.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:10 smithi156 ceph-mon[195101]: osdmap e274: 8 total, 8 up, 7 in 2023-10-09T05:58:10.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:10 smithi064 ceph-mon[212750]: from='client.45179 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:58:10.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:10 smithi064 ceph-mon[212750]: pgmap v620: 139 pgs: 1 remapped+peering, 111 active+clean, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped; 33 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 375 op/s; 20227/1614 objects degraded (1253.222%); 1092/1614 objects misplaced (67.658%); 2.0 MiB/s, 396 objects/s recovering 2023-10-09T05:58:10.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:10 smithi064 ceph-mon[212750]: failed to encode map e273 with expected crc 2023-10-09T05:58:10.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:10 smithi064 ceph-mon[212750]: failed to encode map e273 with expected crc 2023-10-09T05:58:10.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:10 smithi064 ceph-mon[212750]: failed to encode map e273 with expected crc 2023-10-09T05:58:10.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:10 smithi064 ceph-mon[212750]: failed to encode map e273 with expected crc 2023-10-09T05:58:10.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:10 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2239290908' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:10.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:10 smithi064 ceph-mon[212750]: osdmap e274: 8 total, 8 up, 7 in 2023-10-09T05:58:10.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:10 smithi064 ceph-mon[220044]: from='client.45179 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:58:10.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:10 smithi064 ceph-mon[220044]: pgmap v620: 139 pgs: 1 remapped+peering, 111 active+clean, 1 active+recovering+undersized+degraded+remapped, 13 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped; 33 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 375 op/s; 20227/1614 objects degraded (1253.222%); 1092/1614 objects misplaced (67.658%); 2.0 MiB/s, 396 objects/s recovering 2023-10-09T05:58:10.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:10 smithi064 ceph-mon[220044]: failed to encode map e273 with expected crc 2023-10-09T05:58:10.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:10 smithi064 ceph-mon[220044]: failed to encode map e273 with expected crc 2023-10-09T05:58:10.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:10 smithi064 ceph-mon[220044]: failed to encode map e273 with expected crc 2023-10-09T05:58:10.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:10 smithi064 ceph-mon[220044]: failed to encode map e273 with expected crc 2023-10-09T05:58:10.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:10 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2239290908' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:10.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:10 smithi064 ceph-mon[220044]: osdmap e274: 8 total, 8 up, 7 in 2023-10-09T05:58:11.177 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":619,"stamp":"2023-10-09T05:58:08.237342+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":34346906,"num_objects":538,"num_object_clones":0,"num_object_copies":1614,"num_objects_missing_on_primary":1682,"num_objects_missing":1682,"num_objects_degraded":20227,"num_objects_misplaced":1092,"num_objects_unfound":0,"num_objects_dirty":538,"num_whiteouts":0,"num_read":488,"num_read_kb":914,"num_write":190907,"num_write_kb":6085646,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":75264,"num_bytes_recovered":4066587328,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":324201,"ondisk_log_size":324201,"up":417,"acting":402,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":483,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":3574228,"kb_used_data":1689564,"kb_used_omap":118,"kb_used_meta":1884489,"kb_avail":652641836,"statfs":{"total":671965249536,"available":668305240064,"internally_reserved":0,"allocated":1730113536,"data_stored":1721540849,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":121648,"internal_metadata":1929716944},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-125272094,"num_objects":-1914,"num_object_clones":0,"num_object_copies":-5742,"num_objects_missing_on_primary":-1691,"num_objects_missing":-1691,"num_objects_degraded":-2135,"num_objects_misplaced":-297,"num_objects_unfound":0,"num_objects_dirty":-1914,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2392,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3038,"num_bytes_recovered":12648448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.368293"},"pg_stats":[{"pgid":"6.2d","version":"269'8615","reported_seq":15250,"reported_epoch":269,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:58:02.774282+0000","last_change":"2023-10-09T05:57:11.666825+0000","last_active":"2023-10-09T05:58:02.774282+0000","last_peered":"2023-10-09T05:58:02.774282+0000","last_clean":"2023-10-09T05:55:58.102294+0000","last_became_active":"2023-10-09T05:56:50.227444+0000","last_became_peered":"2023-10-09T05:56:50.227444+0000","last_unstale":"2023-10-09T05:58:02.774282+0000","last_undegraded":"2023-10-09T05:58:02.774282+0000","last_fullsized":"2023-10-09T05:58:02.774282+0000","mapping_epoch":234,"log_start":"108'2175","ondisk_log_start":"108'2175","created":229,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6440,"log_dups_size":0,"ondisk_log_size":6440,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3025,"num_write_kb":96016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1160,"num_bytes_recovered":76021760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,1],"avail_no_missing":["0","1","2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2c","version":"269'8668","reported_seq":20979,"reported_epoch":269,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:58:02.545250+0000","last_change":"2023-10-09T05:57:24.712437+0000","last_active":"2023-10-09T05:58:02.545250+0000","last_peered":"2023-10-09T05:58:02.545250+0000","last_clean":"2023-10-09T05:57:22.296318+0000","last_became_active":"2023-10-09T05:57:24.704597+0000","last_became_peered":"2023-10-09T05:57:24.704597+0000","last_unstale":"2023-10-09T05:58:02.545250+0000","last_undegraded":"2023-10-09T05:58:02.545250+0000","last_fullsized":"2023-10-09T05:58:02.545250+0000","mapping_epoch":249,"log_start":"108'2503","ondisk_log_start":"108'2503","created":229,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6165,"log_dups_size":0,"ondisk_log_size":6165,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3021,"num_write_kb":96528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1508,"num_bytes_recovered":96321536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[5,6,7],"avail_no_missing":["5","3","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.2b","version":"269'8765","reported_seq":26142,"reported_epoch":269,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:58:02.613866+0000","last_change":"2023-10-09T05:58:00.213279+0000","last_active":"2023-10-09T05:58:02.613866+0000","last_peered":"2023-10-09T05:58:02.613866+0000","last_clean":"2023-10-09T05:57:57.193384+0000","last_became_active":"2023-10-09T05:58:00.182912+0000","last_became_peered":"2023-10-09T05:58:00.182912+0000","last_unstale":"2023-10-09T05:58:02.613866+0000","last_undegraded":"2023-10-09T05:58:02.613866+0000","last_fullsized":"2023-10-09T05:58:02.613866+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":229,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":8765,"log_dups_size":0,"ondisk_log_size":8765,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3063,"num_write_kb":97648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3175,"num_bytes_recovered":185122816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,6,4],"avail_no_missing":["3","1","4","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2a","version":"269'8598","reported_seq":11002,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T05:58:02.599437+0000","last_change":"2023-10-09T05:56:49.254174+0000","last_active":"2023-10-09T05:58:02.599437+0000","last_peered":"2023-10-09T05:58:02.599437+0000","last_clean":"2023-10-09T05:58:02.599437+0000","last_became_active":"2023-10-09T05:56:49.238119+0000","last_became_peered":"2023-10-09T05:56:49.238119+0000","last_unstale":"2023-10-09T05:58:02.599437+0000","last_undegraded":"2023-10-09T05:58:02.599437+0000","last_fullsized":"2023-10-09T05:58:02.599437+0000","mapping_epoch":233,"log_start":"108'2184","ondisk_log_start":"108'2184","created":229,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6414,"log_dups_size":0,"ondisk_log_size":6414,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":688128,"num_objects":11,"num_object_clones":0,"num_object_copies":33,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2996,"num_write_kb":96208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17776640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.29","version":"269'8440","reported_seq":14478,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T05:58:02.714285+0000","last_change":"2023-10-09T05:57:18.237407+0000","last_active":"2023-10-09T05:58:02.714285+0000","last_peered":"2023-10-09T05:58:02.714285+0000","last_clean":"2023-10-09T05:58:02.714285+0000","last_became_active":"2023-10-09T05:57:18.229501+0000","last_became_peered":"2023-10-09T05:57:18.229501+0000","last_unstale":"2023-10-09T05:58:02.714285+0000","last_undegraded":"2023-10-09T05:58:02.714285+0000","last_fullsized":"2023-10-09T05:58:02.714285+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":229,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":8440,"log_dups_size":0,"ondisk_log_size":8440,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1851392,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3008,"num_write_kb":97152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1757,"num_bytes_recovered":65863680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.28","version":"269'8409","reported_seq":8617,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T05:58:02.729179+0000","last_change":"2023-10-09T05:56:40.298679+0000","last_active":"2023-10-09T05:58:02.729179+0000","last_peered":"2023-10-09T05:58:02.729179+0000","last_clean":"2023-10-09T05:58:02.729179+0000","last_became_active":"2023-10-09T05:56:40.292466+0000","last_became_peered":"2023-10-09T05:56:40.292466+0000","last_unstale":"2023-10-09T05:58:02.729179+0000","last_undegraded":"2023-10-09T05:58:02.729179+0000","last_fullsized":"2023-10-09T05:58:02.729179+0000","mapping_epoch":229,"log_start":"106'122","ondisk_log_start":"106'122","created":229,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":8287,"log_dups_size":0,"ondisk_log_size":8287,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3009,"num_write_kb":95456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.27","version":"269'8331","reported_seq":8535,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T05:58:02.623774+0000","last_change":"2023-10-09T05:56:49.257404+0000","last_active":"2023-10-09T05:58:02.623774+0000","last_peered":"2023-10-09T05:58:02.623774+0000","last_clean":"2023-10-09T05:58:02.623774+0000","last_became_active":"2023-10-09T05:56:49.243016+0000","last_became_peered":"2023-10-09T05:56:49.243016+0000","last_unstale":"2023-10-09T05:58:02.623774+0000","last_undegraded":"2023-10-09T05:58:02.623774+0000","last_fullsized":"2023-10-09T05:58:02.623774+0000","mapping_epoch":233,"log_start":"106'370","ondisk_log_start":"106'370","created":229,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7961,"log_dups_size":0,"ondisk_log_size":7961,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":475136,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2973,"num_write_kb":95360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.26","version":"269'8463","reported_seq":11636,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:02.375210+0000","last_change":"2023-10-09T05:57:24.807644+0000","last_active":"2023-10-09T05:58:02.375210+0000","last_peered":"2023-10-09T05:58:02.375210+0000","last_clean":"2023-10-09T05:52:24.946135+0000","last_became_active":"2023-10-09T05:57:24.608711+0000","last_became_peered":"2023-10-09T05:57:24.608711+0000","last_unstale":"2023-10-09T05:58:02.375210+0000","last_undegraded":"2023-10-09T05:57:24.805634+0000","last_fullsized":"2023-10-09T05:57:24.466532+0000","mapping_epoch":249,"log_start":"106'613","ondisk_log_start":"106'613","created":229,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7850,"log_dups_size":0,"ondisk_log_size":7850,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311296,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":942,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3006,"num_write_kb":96352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":874,"num_bytes_recovered":57049088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[4,7],"avail_no_missing":["4","5","7"],"object_location_counts":[{"shards":"4,7","objects":5}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.25","version":"269'8277","reported_seq":14386,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:02.728790+0000","last_change":"2023-10-09T05:57:52.900513+0000","last_active":"2023-10-09T05:58:02.728790+0000","last_peered":"2023-10-09T05:58:02.728790+0000","last_clean":"2023-10-09T05:57:50.665311+0000","last_became_active":"2023-10-09T05:57:52.862206+0000","last_became_peered":"2023-10-09T05:57:52.862206+0000","last_unstale":"2023-10-09T05:58:02.728790+0000","last_undegraded":"2023-10-09T05:57:52.749643+0000","last_fullsized":"2023-10-09T05:57:52.733001+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":229,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":8277,"log_dups_size":0,"ondisk_log_size":8277,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1501,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2961,"num_write_kb":94432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1142,"num_bytes_recovered":67731456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"3,7","objects":-126}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.24","version":"269'8519","reported_seq":9789,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:02.766464+0000","last_change":"2023-10-09T05:57:24.710999+0000","last_active":"2023-10-09T05:58:02.766464+0000","last_peered":"2023-10-09T05:58:02.766464+0000","last_clean":"2023-10-09T05:51:24.557317+0000","last_became_active":"2023-10-09T05:57:24.682889+0000","last_became_peered":"2023-10-09T05:57:24.682889+0000","last_unstale":"2023-10-09T05:58:02.766464+0000","last_undegraded":"2023-10-09T05:57:24.558381+0000","last_fullsized":"2023-10-09T05:57:24.537309+0000","mapping_epoch":249,"log_start":"107'1426","ondisk_log_start":"107'1426","created":229,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7093,"log_dups_size":0,"ondisk_log_size":7093,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1655,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3048,"num_write_kb":96864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":364,"num_bytes_recovered":23822336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[5,6],"avail_no_missing":["5","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.23","version":"269'8024","reported_seq":8235,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T05:58:02.696462+0000","last_change":"2023-10-09T05:55:22.225278+0000","last_active":"2023-10-09T05:58:02.696462+0000","last_peered":"2023-10-09T05:58:02.696462+0000","last_clean":"2023-10-09T05:58:02.696462+0000","last_became_active":"2023-10-09T05:55:22.222511+0000","last_became_peered":"2023-10-09T05:55:22.222511+0000","last_unstale":"2023-10-09T05:58:02.696462+0000","last_undegraded":"2023-10-09T05:58:02.696462+0000","last_fullsized":"2023-10-09T05:58:02.696462+0000","mapping_epoch":207,"log_start":"107'1488","ondisk_log_start":"107'1488","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6536,"log_dups_size":0,"ondisk_log_size":6536,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":737280,"num_objects":11,"num_object_clones":0,"num_object_copies":33,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3030,"num_write_kb":97312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.22","version":"269'7999","reported_seq":20264,"reported_epoch":269,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:58:02.613556+0000","last_change":"2023-10-09T05:57:24.625724+0000","last_active":"2023-10-09T05:58:02.613556+0000","last_peered":"2023-10-09T05:58:02.613556+0000","last_clean":"2023-10-09T05:57:22.430161+0000","last_became_active":"2023-10-09T05:57:24.616066+0000","last_became_peered":"2023-10-09T05:57:24.616066+0000","last_unstale":"2023-10-09T05:58:02.613556+0000","last_undegraded":"2023-10-09T05:58:02.613556+0000","last_fullsized":"2023-10-09T05:58:02.613556+0000","mapping_epoch":249,"log_start":"106'632","ondisk_log_start":"106'632","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7367,"log_dups_size":0,"ondisk_log_size":7367,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":671744,"num_objects":10,"num_object_clones":0,"num_object_copies":30,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":10,"num_objects_unfound":0,"num_objects_dirty":10,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2999,"num_write_kb":96304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1505,"num_bytes_recovered":95617024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,2,6],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":10}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.21","version":"269'8080","reported_seq":9386,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:02.628858+0000","last_change":"2023-10-09T05:57:11.708222+0000","last_active":"2023-10-09T05:58:02.628858+0000","last_peered":"2023-10-09T05:58:02.628858+0000","last_clean":"2023-10-09T05:51:24.563560+0000","last_became_active":"2023-10-09T05:57:11.700582+0000","last_became_peered":"2023-10-09T05:57:11.700582+0000","last_unstale":"2023-10-09T05:58:02.628858+0000","last_undegraded":"2023-10-09T05:57:11.372488+0000","last_fullsized":"2023-10-09T05:57:11.342231+0000","mapping_epoch":239,"log_start":"107'871","ondisk_log_start":"107'871","created":207,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7209,"log_dups_size":0,"ondisk_log_size":7209,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1507,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3056,"num_write_kb":97696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":532,"num_bytes_recovered":34799616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,7],"avail_no_missing":["6","0"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.20","version":"269'8034","reported_seq":15125,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:02.715652+0000","last_change":"2023-10-09T05:57:24.769717+0000","last_active":"2023-10-09T05:58:02.715652+0000","last_peered":"2023-10-09T05:58:02.715652+0000","last_clean":"2023-10-09T05:57:08.850071+0000","last_became_active":"2023-10-09T05:57:24.739384+0000","last_became_peered":"2023-10-09T05:57:24.739384+0000","last_unstale":"2023-10-09T05:58:02.715652+0000","last_undegraded":"2023-10-09T05:57:24.589067+0000","last_fullsized":"2023-10-09T05:57:24.538377+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":8034,"log_dups_size":0,"ondisk_log_size":8034,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81920,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":876,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3037,"num_write_kb":97216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1836,"num_bytes_recovered":111132672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,5],"avail_no_missing":["0","4","5"],"object_location_counts":[{"shards":"0,5","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1f","version":"269'9653","reported_seq":12325,"reported_epoch":270,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:06.253372+0000","last_change":"2023-10-09T05:57:25.462531+0000","last_active":"2023-10-09T05:58:06.253372+0000","last_peered":"2023-10-09T05:58:06.253372+0000","last_clean":"2023-10-09T05:52:22.632757+0000","last_became_active":"2023-10-09T05:57:25.116371+0000","last_became_peered":"2023-10-09T05:57:25.116371+0000","last_unstale":"2023-10-09T05:58:06.253372+0000","last_undegraded":"2023-10-09T05:57:24.776710+0000","last_fullsized":"2023-10-09T05:57:24.474570+0000","mapping_epoch":249,"log_start":"108'2407","ondisk_log_start":"108'2407","created":207,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7246,"log_dups_size":0,"ondisk_log_size":7246,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":1682,"num_objects_missing":1682,"num_objects_degraded":1928,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6186,"num_write_kb":196512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1116,"num_bytes_recovered":73072640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,4],"avail_no_missing":["1","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":314,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.864811+0000","last_change":"2023-10-09T05:55:59.341215+0000","last_active":"2023-10-09T05:58:00.864811+0000","last_peered":"2023-10-09T05:58:00.864811+0000","last_clean":"2023-10-09T05:58:00.864811+0000","last_became_active":"2023-10-09T05:55:59.340856+0000","last_became_peered":"2023-10-09T05:55:59.340856+0000","last_unstale":"2023-10-09T05:58:00.864811+0000","last_undegraded":"2023-10-09T05:58:00.864811+0000","last_fullsized":"2023-10-09T05:58:00.864811+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:33.686454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":31,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.860802+0000","last_change":"2023-10-09T05:57:23.474348+0000","last_active":"2023-10-09T05:58:00.860802+0000","last_peered":"2023-10-09T05:58:00.860802+0000","last_clean":"2023-10-09T05:58:00.860802+0000","last_became_active":"2023-10-09T05:57:23.474010+0000","last_became_peered":"2023-10-09T05:57:23.474010+0000","last_unstale":"2023-10-09T05:58:00.860802+0000","last_undegraded":"2023-10-09T05:58:00.860802+0000","last_fullsized":"2023-10-09T05:58:00.860802+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:45.039777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.1c","version":"269'9418","reported_seq":21717,"reported_epoch":269,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:58:02.667730+0000","last_change":"2023-10-09T05:57:24.711863+0000","last_active":"2023-10-09T05:58:02.667730+0000","last_peered":"2023-10-09T05:58:02.667730+0000","last_clean":"2023-10-09T05:57:22.403801+0000","last_became_active":"2023-10-09T05:57:24.689040+0000","last_became_peered":"2023-10-09T05:57:24.689040+0000","last_unstale":"2023-10-09T05:58:02.667730+0000","last_undegraded":"2023-10-09T05:58:02.667730+0000","last_fullsized":"2023-10-09T05:58:02.667730+0000","mapping_epoch":249,"log_start":"108'2503","ondisk_log_start":"108'2503","created":207,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6915,"log_dups_size":0,"ondisk_log_size":6915,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3276800,"num_objects":50,"num_object_clones":0,"num_object_copies":150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":50,"num_objects_unfound":0,"num_objects_dirty":50,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5983,"num_write_kb":193056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3017,"num_bytes_recovered":192643072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[5,6,7],"avail_no_missing":["5","6","7"],"object_location_counts":[{"shards":"5,6,7","objects":50}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":148,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.859042+0000","last_change":"2023-10-09T05:55:59.304893+0000","last_active":"2023-10-09T05:58:00.859042+0000","last_peered":"2023-10-09T05:58:00.859042+0000","last_clean":"2023-10-09T05:58:00.859042+0000","last_became_active":"2023-10-09T05:55:59.304449+0000","last_became_peered":"2023-10-09T05:55:59.304449+0000","last_unstale":"2023-10-09T05:58:00.859042+0000","last_undegraded":"2023-10-09T05:58:00.859042+0000","last_fullsized":"2023-10-09T05:58:00.859042+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:44:39.453982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":289,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.860776+0000","last_change":"2023-10-09T05:57:58.834014+0000","last_active":"2023-10-09T05:58:00.860776+0000","last_peered":"2023-10-09T05:58:00.860776+0000","last_clean":"2023-10-09T05:58:00.860776+0000","last_became_active":"2023-10-09T05:57:58.833718+0000","last_became_peered":"2023-10-09T05:57:58.833718+0000","last_unstale":"2023-10-09T05:58:00.860776+0000","last_undegraded":"2023-10-09T05:58:00.860776+0000","last_fullsized":"2023-10-09T05:58:00.860776+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:23.752353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1d","version":"269'9424","reported_seq":19353,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T05:58:02.773101+0000","last_change":"2023-10-09T05:56:49.250996+0000","last_active":"2023-10-09T05:58:02.773101+0000","last_peered":"2023-10-09T05:58:02.773101+0000","last_clean":"2023-10-09T05:58:02.773101+0000","last_became_active":"2023-10-09T05:56:49.237858+0000","last_became_peered":"2023-10-09T05:56:49.237858+0000","last_unstale":"2023-10-09T05:58:02.773101+0000","last_undegraded":"2023-10-09T05:58:02.773101+0000","last_fullsized":"2023-10-09T05:58:02.773101+0000","mapping_epoch":233,"log_start":"108'2440","ondisk_log_start":"108'2440","created":207,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6984,"log_dups_size":0,"ondisk_log_size":6984,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":524276,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5995,"num_write_kb":192033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3518,"num_bytes_recovered":230424600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":31,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.859394+0000","last_change":"2023-10-09T05:57:23.476832+0000","last_active":"2023-10-09T05:58:00.859394+0000","last_peered":"2023-10-09T05:58:00.859394+0000","last_clean":"2023-10-09T05:58:00.859394+0000","last_became_active":"2023-10-09T05:57:23.475216+0000","last_became_peered":"2023-10-09T05:57:23.475216+0000","last_unstale":"2023-10-09T05:58:00.859394+0000","last_undegraded":"2023-10-09T05:58:00.859394+0000","last_fullsized":"2023-10-09T05:58:00.859394+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:48.279660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":295,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.875131+0000","last_change":"2023-10-09T05:55:59.335464+0000","last_active":"2023-10-09T05:58:00.875131+0000","last_peered":"2023-10-09T05:58:00.875131+0000","last_clean":"2023-10-09T05:58:00.875131+0000","last_became_active":"2023-10-09T05:55:59.335259+0000","last_became_peered":"2023-10-09T05:55:59.335259+0000","last_unstale":"2023-10-09T05:58:00.875131+0000","last_undegraded":"2023-10-09T05:58:00.875131+0000","last_fullsized":"2023-10-09T05:58:00.875131+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:28.536682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.12","version":"269'9359","reported_seq":21614,"reported_epoch":269,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:58:02.787842+0000","last_change":"2023-10-09T05:57:24.645788+0000","last_active":"2023-10-09T05:58:02.787842+0000","last_peered":"2023-10-09T05:58:02.787842+0000","last_clean":"2023-10-09T05:57:22.372216+0000","last_became_active":"2023-10-09T05:57:24.625707+0000","last_became_peered":"2023-10-09T05:57:24.625707+0000","last_unstale":"2023-10-09T05:58:02.787842+0000","last_undegraded":"2023-10-09T05:58:02.787842+0000","last_fullsized":"2023-10-09T05:58:02.787842+0000","mapping_epoch":249,"log_start":"115'3187","ondisk_log_start":"115'3187","created":188,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6172,"log_dups_size":0,"ondisk_log_size":6172,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6080,"num_write_kb":192608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3010,"num_bytes_recovered":191234048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,2,6],"avail_no_missing":["4","2","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":224,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.861286+0000","last_change":"2023-10-09T05:57:23.477980+0000","last_active":"2023-10-09T05:58:00.861286+0000","last_peered":"2023-10-09T05:58:00.861286+0000","last_clean":"2023-10-09T05:58:00.861286+0000","last_became_active":"2023-10-09T05:57:23.477654+0000","last_became_peered":"2023-10-09T05:57:23.477654+0000","last_unstale":"2023-10-09T05:58:00.861286+0000","last_undegraded":"2023-10-09T05:58:00.861286+0000","last_fullsized":"2023-10-09T05:58:00.861286+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:17.524683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":31,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.859484+0000","last_change":"2023-10-09T05:57:23.473644+0000","last_active":"2023-10-09T05:58:00.859484+0000","last_peered":"2023-10-09T05:58:00.859484+0000","last_clean":"2023-10-09T05:58:00.859484+0000","last_became_active":"2023-10-09T05:57:23.473391+0000","last_became_peered":"2023-10-09T05:57:23.473391+0000","last_unstale":"2023-10-09T05:58:00.859484+0000","last_undegraded":"2023-10-09T05:58:00.859484+0000","last_fullsized":"2023-10-09T05:58:00.859484+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:17.138034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"269'9310","reported_seq":9513,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T05:58:02.773120+0000","last_change":"2023-10-09T05:57:18.770940+0000","last_active":"2023-10-09T05:58:02.773120+0000","last_peered":"2023-10-09T05:58:02.773120+0000","last_clean":"2023-10-09T05:58:02.773120+0000","last_became_active":"2023-10-09T05:54:45.682331+0000","last_became_peered":"2023-10-09T05:54:45.682331+0000","last_unstale":"2023-10-09T05:58:02.773120+0000","last_undegraded":"2023-10-09T05:58:02.773120+0000","last_fullsized":"2023-10-09T05:58:02.773120+0000","mapping_epoch":188,"log_start":"115'3241","ondisk_log_start":"115'3241","created":188,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6069,"log_dups_size":0,"ondisk_log_size":6069,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2588672,"num_objects":40,"num_object_clones":0,"num_object_copies":120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":40,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6043,"num_write_kb":194624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":513,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.861019+0000","last_change":"2023-10-09T05:57:26.503936+0000","last_active":"2023-10-09T05:58:00.861019+0000","last_peered":"2023-10-09T05:58:00.861019+0000","last_clean":"2023-10-09T05:58:00.861019+0000","last_became_active":"2023-10-09T05:57:26.503383+0000","last_became_peered":"2023-10-09T05:57:26.503383+0000","last_unstale":"2023-10-09T05:58:00.861019+0000","last_undegraded":"2023-10-09T05:58:00.861019+0000","last_fullsized":"2023-10-09T05:58:00.861019+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":308,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.862652+0000","last_change":"2023-10-09T05:57:23.478483+0000","last_active":"2023-10-09T05:58:00.862652+0000","last_peered":"2023-10-09T05:58:00.862652+0000","last_clean":"2023-10-09T05:58:00.862652+0000","last_became_active":"2023-10-09T05:57:23.478122+0000","last_became_peered":"2023-10-09T05:57:23.478122+0000","last_unstale":"2023-10-09T05:58:00.862652+0000","last_undegraded":"2023-10-09T05:58:00.862652+0000","last_fullsized":"2023-10-09T05:58:00.862652+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:37.768717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":212,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.858954+0000","last_change":"2023-10-09T05:53:59.185702+0000","last_active":"2023-10-09T05:58:00.858954+0000","last_peered":"2023-10-09T05:58:00.858954+0000","last_clean":"2023-10-09T05:58:00.858954+0000","last_became_active":"2023-10-09T05:53:59.185373+0000","last_became_peered":"2023-10-09T05:53:59.185373+0000","last_unstale":"2023-10-09T05:58:00.858954+0000","last_undegraded":"2023-10-09T05:58:00.858954+0000","last_fullsized":"2023-10-09T05:58:00.858954+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:50.756967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"269'9345","reported_seq":24056,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:02.796241+0000","last_change":"2023-10-09T05:57:24.784795+0000","last_active":"2023-10-09T05:58:02.796241+0000","last_peered":"2023-10-09T05:58:02.796241+0000","last_clean":"2023-10-09T05:54:10.146196+0000","last_became_active":"2023-10-09T05:57:24.766527+0000","last_became_peered":"2023-10-09T05:57:24.766527+0000","last_unstale":"2023-10-09T05:58:02.796241+0000","last_undegraded":"2023-10-09T05:57:24.620015+0000","last_fullsized":"2023-10-09T05:57:24.557771+0000","mapping_epoch":249,"log_start":"109'2595","ondisk_log_start":"109'2595","created":188,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6750,"log_dups_size":0,"ondisk_log_size":6750,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":819200,"num_objects":13,"num_object_clones":0,"num_object_copies":39,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3195,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6064,"num_write_kb":194432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5315,"num_bytes_recovered":301039616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,2],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,2","objects":13}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":109,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.863445+0000","last_change":"2023-10-09T05:53:59.219168+0000","last_active":"2023-10-09T05:58:00.863445+0000","last_peered":"2023-10-09T05:58:00.863445+0000","last_clean":"2023-10-09T05:58:00.863445+0000","last_became_active":"2023-10-09T05:53:59.200886+0000","last_became_peered":"2023-10-09T05:53:59.200886+0000","last_unstale":"2023-10-09T05:58:00.863445+0000","last_undegraded":"2023-10-09T05:58:00.863445+0000","last_fullsized":"2023-10-09T05:58:00.863445+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:45.515036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":163,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.874523+0000","last_change":"2023-10-09T05:55:59.305100+0000","last_active":"2023-10-09T05:58:00.874523+0000","last_peered":"2023-10-09T05:58:00.874523+0000","last_clean":"2023-10-09T05:58:00.874523+0000","last_became_active":"2023-10-09T05:55:59.304758+0000","last_became_peered":"2023-10-09T05:55:59.304758+0000","last_unstale":"2023-10-09T05:58:00.874523+0000","last_undegraded":"2023-10-09T05:58:00.874523+0000","last_fullsized":"2023-10-09T05:58:00.874523+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:54:32.510945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.11","version":"269'9418","reported_seq":15317,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T05:58:02.691769+0000","last_change":"2023-10-09T05:57:57.834050+0000","last_active":"2023-10-09T05:58:02.691769+0000","last_peered":"2023-10-09T05:58:02.691769+0000","last_clean":"2023-10-09T05:58:02.691769+0000","last_became_active":"2023-10-09T05:57:57.816053+0000","last_became_peered":"2023-10-09T05:57:57.816053+0000","last_unstale":"2023-10-09T05:58:02.691769+0000","last_undegraded":"2023-10-09T05:58:02.691769+0000","last_fullsized":"2023-10-09T05:58:02.691769+0000","mapping_epoch":264,"log_start":"131'5436","ondisk_log_start":"131'5436","created":188,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":3982,"log_dups_size":0,"ondisk_log_size":3982,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6134,"num_write_kb":195392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2034,"num_bytes_recovered":20316160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":274,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.862676+0000","last_change":"2023-10-09T05:53:59.219141+0000","last_active":"2023-10-09T05:58:00.862676+0000","last_peered":"2023-10-09T05:58:00.862676+0000","last_clean":"2023-10-09T05:58:00.862676+0000","last_became_active":"2023-10-09T05:53:59.197318+0000","last_became_peered":"2023-10-09T05:53:59.197318+0000","last_unstale":"2023-10-09T05:58:00.862676+0000","last_undegraded":"2023-10-09T05:58:00.862676+0000","last_fullsized":"2023-10-09T05:58:00.862676+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:39.910794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":278,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.863574+0000","last_change":"2023-10-09T05:55:59.335824+0000","last_active":"2023-10-09T05:58:00.863574+0000","last_peered":"2023-10-09T05:58:00.863574+0000","last_clean":"2023-10-09T05:58:00.863574+0000","last_became_active":"2023-10-09T05:55:59.335478+0000","last_became_peered":"2023-10-09T05:55:59.335478+0000","last_unstale":"2023-10-09T05:58:00.863574+0000","last_undegraded":"2023-10-09T05:58:00.863574+0000","last_fullsized":"2023-10-09T05:58:00.863574+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:26.014501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.16","version":"269'9305","reported_seq":15641,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:02.765127+0000","last_change":"2023-10-09T05:57:24.649008+0000","last_active":"2023-10-09T05:58:02.765127+0000","last_peered":"2023-10-09T05:58:02.765127+0000","last_clean":"2023-10-09T05:52:24.946135+0000","last_became_active":"2023-10-09T05:57:24.636478+0000","last_became_peered":"2023-10-09T05:57:24.636478+0000","last_unstale":"2023-10-09T05:58:02.765127+0000","last_undegraded":"2023-10-09T05:57:24.488807+0000","last_fullsized":"2023-10-09T05:57:24.471078+0000","mapping_epoch":249,"log_start":"112'3039","ondisk_log_start":"112'3039","created":188,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6266,"log_dups_size":0,"ondisk_log_size":6266,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1915,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6041,"num_write_kb":192704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3030,"num_bytes_recovered":183042048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[4,7],"avail_no_missing":["4","5","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":459,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.862452+0000","last_change":"2023-10-09T05:56:49.247703+0000","last_active":"2023-10-09T05:58:00.862452+0000","last_peered":"2023-10-09T05:58:00.862452+0000","last_clean":"2023-10-09T05:58:00.862452+0000","last_became_active":"2023-10-09T05:56:49.247308+0000","last_became_peered":"2023-10-09T05:56:49.247308+0000","last_unstale":"2023-10-09T05:58:00.862452+0000","last_undegraded":"2023-10-09T05:58:00.862452+0000","last_fullsized":"2023-10-09T05:58:00.862452+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":111,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.864133+0000","last_change":"2023-10-09T05:53:59.233898+0000","last_active":"2023-10-09T05:58:00.864133+0000","last_peered":"2023-10-09T05:58:00.864133+0000","last_clean":"2023-10-09T05:58:00.864133+0000","last_became_active":"2023-10-09T05:53:59.204175+0000","last_became_peered":"2023-10-09T05:53:59.204175+0000","last_unstale":"2023-10-09T05:58:00.864133+0000","last_undegraded":"2023-10-09T05:58:00.864133+0000","last_fullsized":"2023-10-09T05:58:00.864133+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:03.139318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":298,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.872794+0000","last_change":"2023-10-09T05:55:59.297087+0000","last_active":"2023-10-09T05:58:00.872794+0000","last_peered":"2023-10-09T05:58:00.872794+0000","last_clean":"2023-10-09T05:58:00.872794+0000","last_became_active":"2023-10-09T05:55:59.296768+0000","last_became_peered":"2023-10-09T05:55:59.296768+0000","last_unstale":"2023-10-09T05:58:00.872794+0000","last_undegraded":"2023-10-09T05:58:00.872794+0000","last_fullsized":"2023-10-09T05:58:00.872794+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:35.067705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"269'9159","reported_seq":9352,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T05:58:02.756131+0000","last_change":"2023-10-09T05:56:49.257400+0000","last_active":"2023-10-09T05:58:02.756131+0000","last_peered":"2023-10-09T05:58:02.756131+0000","last_clean":"2023-10-09T05:58:02.756131+0000","last_became_active":"2023-10-09T05:56:49.238939+0000","last_became_peered":"2023-10-09T05:56:49.238939+0000","last_unstale":"2023-10-09T05:58:02.756131+0000","last_undegraded":"2023-10-09T05:58:02.756131+0000","last_fullsized":"2023-10-09T05:58:02.756131+0000","mapping_epoch":233,"log_start":"125'4150","ondisk_log_start":"125'4150","created":188,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5009,"log_dups_size":0,"ondisk_log_size":5009,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":491520,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5953,"num_write_kb":190720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":662,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.863248+0000","last_change":"2023-10-09T05:54:27.330211+0000","last_active":"2023-10-09T05:58:00.863248+0000","last_peered":"2023-10-09T05:58:00.863248+0000","last_clean":"2023-10-09T05:58:00.863248+0000","last_became_active":"2023-10-09T05:53:59.202399+0000","last_became_peered":"2023-10-09T05:53:59.202399+0000","last_unstale":"2023-10-09T05:58:00.863248+0000","last_undegraded":"2023-10-09T05:58:00.863248+0000","last_fullsized":"2023-10-09T05:58:00.863248+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:51:12.346243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:51:12.346243+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":270,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.861807+0000","last_change":"2023-10-09T05:57:23.473445+0000","last_active":"2023-10-09T05:58:00.861807+0000","last_peered":"2023-10-09T05:58:00.861807+0000","last_clean":"2023-10-09T05:58:00.861807+0000","last_became_active":"2023-10-09T05:57:23.473103+0000","last_became_peered":"2023-10-09T05:57:23.473103+0000","last_unstale":"2023-10-09T05:58:00.861807+0000","last_undegraded":"2023-10-09T05:58:00.861807+0000","last_fullsized":"2023-10-09T05:58:00.861807+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:39:53.825734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":192,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.863267+0000","last_change":"2023-10-09T05:56:49.237784+0000","last_active":"2023-10-09T05:58:00.863267+0000","last_peered":"2023-10-09T05:58:00.863267+0000","last_clean":"2023-10-09T05:58:00.863267+0000","last_became_active":"2023-10-09T05:56:49.237417+0000","last_became_peered":"2023-10-09T05:56:49.237417+0000","last_unstale":"2023-10-09T05:58:00.863267+0000","last_undegraded":"2023-10-09T05:58:00.863267+0000","last_fullsized":"2023-10-09T05:58:00.863267+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:33.129268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.14","version":"269'9315","reported_seq":11107,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:02.747871+0000","last_change":"2023-10-09T05:57:24.711984+0000","last_active":"2023-10-09T05:58:02.747871+0000","last_peered":"2023-10-09T05:58:02.747871+0000","last_clean":"2023-10-09T05:51:24.557317+0000","last_became_active":"2023-10-09T05:57:24.689515+0000","last_became_peered":"2023-10-09T05:57:24.689515+0000","last_unstale":"2023-10-09T05:58:02.747871+0000","last_undegraded":"2023-10-09T05:57:24.536734+0000","last_fullsized":"2023-10-09T05:57:24.496360+0000","mapping_epoch":249,"log_start":"108'2433","ondisk_log_start":"108'2433","created":188,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6882,"log_dups_size":0,"ondisk_log_size":6882,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1310720,"num_objects":20,"num_object_clones":0,"num_object_copies":60,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3270,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6034,"num_write_kb":193728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":689,"num_bytes_recovered":45088768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[5,6],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"5,6","objects":20}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":454,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.873842+0000","last_change":"2023-10-09T05:57:54.797244+0000","last_active":"2023-10-09T05:58:00.873842+0000","last_peered":"2023-10-09T05:58:00.873842+0000","last_clean":"2023-10-09T05:58:00.873842+0000","last_became_active":"2023-10-09T05:57:54.796149+0000","last_became_peered":"2023-10-09T05:57:54.796149+0000","last_unstale":"2023-10-09T05:58:00.873842+0000","last_undegraded":"2023-10-09T05:58:00.873842+0000","last_fullsized":"2023-10-09T05:58:00.873842+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":105,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.862039+0000","last_change":"2023-10-09T05:53:59.206887+0000","last_active":"2023-10-09T05:58:00.862039+0000","last_peered":"2023-10-09T05:58:00.862039+0000","last_clean":"2023-10-09T05:58:00.862039+0000","last_became_active":"2023-10-09T05:53:59.206608+0000","last_became_peered":"2023-10-09T05:53:59.206608+0000","last_unstale":"2023-10-09T05:58:00.862039+0000","last_undegraded":"2023-10-09T05:58:00.862039+0000","last_fullsized":"2023-10-09T05:58:00.862039+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:48.975792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":251,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.860946+0000","last_change":"2023-10-09T05:57:58.828837+0000","last_active":"2023-10-09T05:58:00.860946+0000","last_peered":"2023-10-09T05:58:00.860946+0000","last_clean":"2023-10-09T05:58:00.860946+0000","last_became_active":"2023-10-09T05:57:58.828503+0000","last_became_peered":"2023-10-09T05:57:58.828503+0000","last_unstale":"2023-10-09T05:58:00.860946+0000","last_undegraded":"2023-10-09T05:58:00.860946+0000","last_fullsized":"2023-10-09T05:58:00.860946+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:08.332644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"269'9053","reported_seq":15073,"reported_epoch":269,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:58:02.747084+0000","last_change":"2023-10-09T05:57:52.902164+0000","last_active":"2023-10-09T05:58:02.747084+0000","last_peered":"2023-10-09T05:58:02.747084+0000","last_clean":"2023-10-09T05:57:50.722184+0000","last_became_active":"2023-10-09T05:57:52.862205+0000","last_became_peered":"2023-10-09T05:57:52.862205+0000","last_unstale":"2023-10-09T05:58:02.747084+0000","last_undegraded":"2023-10-09T05:58:02.747084+0000","last_fullsized":"2023-10-09T05:58:02.747084+0000","mapping_epoch":260,"log_start":"113'3012","ondisk_log_start":"113'3012","created":188,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6041,"log_dups_size":0,"ondisk_log_size":6041,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1933312,"num_objects":30,"num_object_clones":0,"num_object_copies":90,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":30,"num_objects_unfound":0,"num_objects_dirty":30,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5873,"num_write_kb":188864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2244,"num_bytes_recovered":132907008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,3,2],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":30}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":565,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T05:58:01.899971+0000","last_change":"2023-10-09T05:58:01.899971+0000","last_active":"2023-10-09T05:58:01.899971+0000","last_peered":"2023-10-09T05:58:01.899971+0000","last_clean":"2023-10-09T05:58:01.899971+0000","last_became_active":"2023-10-09T05:58:01.899521+0000","last_became_peered":"2023-10-09T05:58:01.899521+0000","last_unstale":"2023-10-09T05:58:01.899971+0000","last_undegraded":"2023-10-09T05:58:01.899971+0000","last_fullsized":"2023-10-09T05:58:01.899971+0000","mapping_epoch":268,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":269,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":254,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.873613+0000","last_change":"2023-10-09T05:51:57.091639+0000","last_active":"2023-10-09T05:58:00.873613+0000","last_peered":"2023-10-09T05:58:00.873613+0000","last_clean":"2023-10-09T05:58:00.873613+0000","last_became_active":"2023-10-09T05:51:57.091284+0000","last_became_peered":"2023-10-09T05:51:57.091284+0000","last_unstale":"2023-10-09T05:58:00.873613+0000","last_undegraded":"2023-10-09T05:58:00.873613+0000","last_fullsized":"2023-10-09T05:58:00.873613+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:10.553453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":40,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.861847+0000","last_change":"2023-10-09T05:57:58.833939+0000","last_active":"2023-10-09T05:58:00.861847+0000","last_peered":"2023-10-09T05:58:00.861847+0000","last_clean":"2023-10-09T05:58:00.861847+0000","last_became_active":"2023-10-09T05:57:58.833567+0000","last_became_peered":"2023-10-09T05:57:58.833567+0000","last_unstale":"2023-10-09T05:58:00.861847+0000","last_undegraded":"2023-10-09T05:58:00.861847+0000","last_fullsized":"2023-10-09T05:58:00.861847+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:58.347572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":622,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.862864+0000","last_change":"2023-10-09T05:57:13.235857+0000","last_active":"2023-10-09T05:58:00.862864+0000","last_peered":"2023-10-09T05:58:00.862864+0000","last_clean":"2023-10-09T05:58:00.862864+0000","last_became_active":"2023-10-09T05:57:13.235379+0000","last_became_peered":"2023-10-09T05:57:13.235379+0000","last_unstale":"2023-10-09T05:58:00.862864+0000","last_undegraded":"2023-10-09T05:58:00.862864+0000","last_fullsized":"2023-10-09T05:58:00.862864+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.a","version":"269'8625","reported_seq":11030,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T05:58:02.561234+0000","last_change":"2023-10-09T05:56:49.265060+0000","last_active":"2023-10-09T05:58:02.561234+0000","last_peered":"2023-10-09T05:58:02.561234+0000","last_clean":"2023-10-09T05:58:02.561234+0000","last_became_active":"2023-10-09T05:56:49.248743+0000","last_became_peered":"2023-10-09T05:56:49.248743+0000","last_unstale":"2023-10-09T05:58:02.561234+0000","last_undegraded":"2023-10-09T05:58:02.561234+0000","last_fullsized":"2023-10-09T05:58:02.561234+0000","mapping_epoch":233,"log_start":"108'2184","ondisk_log_start":"108'2184","created":102,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6441,"log_dups_size":0,"ondisk_log_size":6441,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3023,"num_write_kb":96208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17776640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":109,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.864205+0000","last_change":"2023-10-09T05:53:59.222136+0000","last_active":"2023-10-09T05:58:00.864205+0000","last_peered":"2023-10-09T05:58:00.864205+0000","last_clean":"2023-10-09T05:58:00.864205+0000","last_became_active":"2023-10-09T05:53:59.201853+0000","last_became_peered":"2023-10-09T05:53:59.201853+0000","last_unstale":"2023-10-09T05:58:00.864205+0000","last_undegraded":"2023-10-09T05:58:00.864205+0000","last_fullsized":"2023-10-09T05:58:00.864205+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:26.662031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":175,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.863043+0000","last_change":"2023-10-09T05:56:49.234185+0000","last_active":"2023-10-09T05:58:00.863043+0000","last_peered":"2023-10-09T05:58:00.863043+0000","last_clean":"2023-10-09T05:58:00.863043+0000","last_became_active":"2023-10-09T05:56:49.233884+0000","last_became_peered":"2023-10-09T05:56:49.233884+0000","last_unstale":"2023-10-09T05:58:00.863043+0000","last_undegraded":"2023-10-09T05:58:00.863043+0000","last_fullsized":"2023-10-09T05:58:00.863043+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:12.142436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":512,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.860932+0000","last_change":"2023-10-09T05:57:39.317706+0000","last_active":"2023-10-09T05:58:00.860932+0000","last_peered":"2023-10-09T05:58:00.860932+0000","last_clean":"2023-10-09T05:58:00.860932+0000","last_became_active":"2023-10-09T05:57:39.317408+0000","last_became_peered":"2023-10-09T05:57:39.317408+0000","last_unstale":"2023-10-09T05:58:00.860932+0000","last_undegraded":"2023-10-09T05:58:00.860932+0000","last_fullsized":"2023-10-09T05:58:00.860932+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":709030,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.b","version":"269'8805","reported_seq":28140,"reported_epoch":269,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:04.717266+0000","last_change":"2023-10-09T05:58:00.236870+0000","last_active":"2023-10-09T05:58:04.717266+0000","last_peered":"2023-10-09T05:58:04.717266+0000","last_clean":"2023-10-09T05:57:57.189281+0000","last_became_active":"2023-10-09T05:58:00.183114+0000","last_became_peered":"2023-10-09T05:58:00.183114+0000","last_unstale":"2023-10-09T05:58:04.717266+0000","last_undegraded":"2023-10-09T05:58:00.236448+0000","last_fullsized":"2023-10-09T05:57:59.818062+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":8805,"log_dups_size":0,"ondisk_log_size":8805,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3103,"num_write_kb":97648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4155,"num_bytes_recovered":190889984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,6","objects":-5}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":151,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.862716+0000","last_change":"2023-10-09T05:55:59.379174+0000","last_active":"2023-10-09T05:58:00.862716+0000","last_peered":"2023-10-09T05:58:00.862716+0000","last_clean":"2023-10-09T05:58:00.862716+0000","last_became_active":"2023-10-09T05:55:59.341932+0000","last_became_peered":"2023-10-09T05:55:59.341932+0000","last_unstale":"2023-10-09T05:58:00.862716+0000","last_undegraded":"2023-10-09T05:58:00.862716+0000","last_fullsized":"2023-10-09T05:58:00.862716+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.636171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":315,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.862664+0000","last_change":"2023-10-09T05:57:23.475146+0000","last_active":"2023-10-09T05:58:00.862664+0000","last_peered":"2023-10-09T05:58:00.862664+0000","last_clean":"2023-10-09T05:58:00.862664+0000","last_became_active":"2023-10-09T05:57:23.474776+0000","last_became_peered":"2023-10-09T05:57:23.474776+0000","last_unstale":"2023-10-09T05:58:00.862664+0000","last_undegraded":"2023-10-09T05:58:00.862664+0000","last_fullsized":"2023-10-09T05:58:00.862664+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:47.775743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":661,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.875441+0000","last_change":"2023-10-09T05:57:26.501017+0000","last_active":"2023-10-09T05:58:00.875441+0000","last_peered":"2023-10-09T05:58:00.875441+0000","last_clean":"2023-10-09T05:58:00.875441+0000","last_became_active":"2023-10-09T05:57:26.500079+0000","last_became_peered":"2023-10-09T05:57:26.500079+0000","last_unstale":"2023-10-09T05:58:00.875441+0000","last_undegraded":"2023-10-09T05:58:00.875441+0000","last_fullsized":"2023-10-09T05:58:00.875441+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.8","version":"269'8352","reported_seq":8561,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T05:58:02.728793+0000","last_change":"2023-10-09T05:56:40.292166+0000","last_active":"2023-10-09T05:58:02.728793+0000","last_peered":"2023-10-09T05:58:02.728793+0000","last_clean":"2023-10-09T05:58:02.728793+0000","last_became_active":"2023-10-09T05:56:40.280945+0000","last_became_peered":"2023-10-09T05:56:40.280945+0000","last_unstale":"2023-10-09T05:58:02.728793+0000","last_undegraded":"2023-10-09T05:58:02.728793+0000","last_fullsized":"2023-10-09T05:58:02.728793+0000","mapping_epoch":174,"log_start":"106'122","ondisk_log_start":"106'122","created":102,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":8230,"log_dups_size":0,"ondisk_log_size":8230,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2064384,"num_objects":31,"num_object_clones":0,"num_object_copies":93,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":31,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2951,"num_write_kb":95456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":107,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.864043+0000","last_change":"2023-10-09T05:53:59.212161+0000","last_active":"2023-10-09T05:58:00.864043+0000","last_peered":"2023-10-09T05:58:00.864043+0000","last_clean":"2023-10-09T05:58:00.864043+0000","last_became_active":"2023-10-09T05:53:59.193814+0000","last_became_peered":"2023-10-09T05:53:59.193814+0000","last_unstale":"2023-10-09T05:58:00.864043+0000","last_undegraded":"2023-10-09T05:58:00.864043+0000","last_fullsized":"2023-10-09T05:58:00.864043+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:10.502408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":308,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.874649+0000","last_change":"2023-10-09T05:55:59.303352+0000","last_active":"2023-10-09T05:58:00.874649+0000","last_peered":"2023-10-09T05:58:00.874649+0000","last_clean":"2023-10-09T05:58:00.874649+0000","last_became_active":"2023-10-09T05:55:59.302923+0000","last_became_peered":"2023-10-09T05:55:59.302923+0000","last_unstale":"2023-10-09T05:58:00.874649+0000","last_undegraded":"2023-10-09T05:58:00.874649+0000","last_fullsized":"2023-10-09T05:58:00.874649+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:53.013376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":648,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.875575+0000","last_change":"2023-10-09T05:57:26.518272+0000","last_active":"2023-10-09T05:58:00.875575+0000","last_peered":"2023-10-09T05:58:00.875575+0000","last_clean":"2023-10-09T05:58:00.875575+0000","last_became_active":"2023-10-09T05:57:26.517614+0000","last_became_peered":"2023-10-09T05:57:26.517614+0000","last_unstale":"2023-10-09T05:58:00.875575+0000","last_undegraded":"2023-10-09T05:58:00.875575+0000","last_fullsized":"2023-10-09T05:58:00.875575+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"269'8501","reported_seq":14844,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T05:58:02.653836+0000","last_change":"2023-10-09T05:57:16.404611+0000","last_active":"2023-10-09T05:58:02.653836+0000","last_peered":"2023-10-09T05:58:02.653836+0000","last_clean":"2023-10-09T05:58:02.653836+0000","last_became_active":"2023-10-09T05:57:16.394685+0000","last_became_peered":"2023-10-09T05:57:16.394685+0000","last_unstale":"2023-10-09T05:58:02.653836+0000","last_undegraded":"2023-10-09T05:58:02.653836+0000","last_fullsized":"2023-10-09T05:58:02.653836+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":8501,"log_dups_size":0,"ondisk_log_size":8501,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3068,"num_write_kb":97152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1909,"num_bytes_recovered":69074944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":31,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.861249+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T05:58:00.861249+0000","last_peered":"2023-10-09T05:58:00.861249+0000","last_clean":"2023-10-09T05:58:00.861249+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T05:58:00.861249+0000","last_undegraded":"2023-10-09T05:58:00.861249+0000","last_fullsized":"2023-10-09T05:58:00.861249+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":279,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.863301+0000","last_change":"2023-10-09T05:57:23.473860+0000","last_active":"2023-10-09T05:58:00.863301+0000","last_peered":"2023-10-09T05:58:00.863301+0000","last_clean":"2023-10-09T05:58:00.863301+0000","last_became_active":"2023-10-09T05:57:23.473101+0000","last_became_peered":"2023-10-09T05:57:23.473101+0000","last_unstale":"2023-10-09T05:58:00.863301+0000","last_undegraded":"2023-10-09T05:58:00.863301+0000","last_fullsized":"2023-10-09T05:58:00.863301+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:05.682865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":682,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.862234+0000","last_change":"2023-10-09T05:57:13.234252+0000","last_active":"2023-10-09T05:58:00.862234+0000","last_peered":"2023-10-09T05:58:00.862234+0000","last_clean":"2023-10-09T05:58:00.862234+0000","last_became_active":"2023-10-09T05:57:13.233849+0000","last_became_peered":"2023-10-09T05:57:13.233849+0000","last_unstale":"2023-10-09T05:58:00.862234+0000","last_undegraded":"2023-10-09T05:58:00.862234+0000","last_fullsized":"2023-10-09T05:58:00.862234+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.e","version":"269'9389","reported_seq":21972,"reported_epoch":269,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:58:02.773019+0000","last_change":"2023-10-09T05:57:10.458251+0000","last_active":"2023-10-09T05:58:02.773019+0000","last_peered":"2023-10-09T05:58:02.773019+0000","last_clean":"2023-10-09T05:55:58.237229+0000","last_became_active":"2023-10-09T05:57:10.441400+0000","last_became_peered":"2023-10-09T05:57:10.441400+0000","last_unstale":"2023-10-09T05:58:02.773019+0000","last_undegraded":"2023-10-09T05:58:02.773019+0000","last_fullsized":"2023-10-09T05:58:02.773019+0000","mapping_epoch":238,"log_start":"112'2854","ondisk_log_start":"112'2854","created":102,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6535,"log_dups_size":0,"ondisk_log_size":6535,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":819200,"num_objects":12,"num_object_clones":0,"num_object_copies":36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":12,"num_objects_unfound":0,"num_objects_dirty":12,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5977,"num_write_kb":191680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3073,"num_bytes_recovered":196280320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5,1],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4,5","objects":12}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":35,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.863045+0000","last_change":"2023-10-09T05:57:23.504733+0000","last_active":"2023-10-09T05:58:00.863045+0000","last_peered":"2023-10-09T05:58:00.863045+0000","last_clean":"2023-10-09T05:58:00.863045+0000","last_became_active":"2023-10-09T05:57:23.468709+0000","last_became_peered":"2023-10-09T05:57:23.468709+0000","last_unstale":"2023-10-09T05:58:00.863045+0000","last_undegraded":"2023-10-09T05:58:00.863045+0000","last_fullsized":"2023-10-09T05:58:00.863045+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:00.459980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":228,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.862871+0000","last_change":"2023-10-09T05:49:21.430742+0000","last_active":"2023-10-09T05:58:00.862871+0000","last_peered":"2023-10-09T05:58:00.862871+0000","last_clean":"2023-10-09T05:58:00.862871+0000","last_became_active":"2023-10-09T05:49:21.430233+0000","last_became_peered":"2023-10-09T05:49:21.430233+0000","last_unstale":"2023-10-09T05:58:00.862871+0000","last_undegraded":"2023-10-09T05:58:00.862871+0000","last_fullsized":"2023-10-09T05:58:00.862871+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:10.541302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":477,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T05:58:06.252838+0000","last_change":"2023-10-09T05:57:26.517046+0000","last_active":"2023-10-09T05:58:06.252838+0000","last_peered":"2023-10-09T05:58:06.252838+0000","last_clean":"2023-10-09T05:58:06.252838+0000","last_became_active":"2023-10-09T05:57:26.516674+0000","last_became_peered":"2023-10-09T05:57:26.516674+0000","last_unstale":"2023-10-09T05:58:06.252838+0000","last_undegraded":"2023-10-09T05:58:06.252838+0000","last_fullsized":"2023-10-09T05:58:06.252838+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.f","version":"269'9564","reported_seq":16107,"reported_epoch":271,"state":"remapped+peering","last_fresh":"2023-10-09T05:58:07.237455+0000","last_change":"2023-10-09T05:58:07.237667+0000","last_active":"2023-10-09T05:58:06.254342+0000","last_peered":"2023-10-09T05:58:05.928164+0000","last_clean":"2023-10-09T05:52:22.632757+0000","last_became_active":"2023-10-09T05:57:25.119934+0000","last_became_peered":"2023-10-09T05:57:25.119934+0000","last_unstale":"2023-10-09T05:58:07.237455+0000","last_undegraded":"2023-10-09T05:58:07.237455+0000","last_fullsized":"2023-10-09T05:58:07.237455+0000","mapping_epoch":271,"log_start":"108'2170","ondisk_log_start":"108'2170","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7394,"log_dups_size":0,"ondisk_log_size":7394,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2916352,"num_objects":44,"num_object_clones":0,"num_object_copies":132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":44,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6096,"num_write_kb":196512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3049,"num_bytes_recovered":71958528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,4,6],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":279,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.874359+0000","last_change":"2023-10-09T05:55:59.320669+0000","last_active":"2023-10-09T05:58:00.874359+0000","last_peered":"2023-10-09T05:58:00.874359+0000","last_clean":"2023-10-09T05:58:00.874359+0000","last_became_active":"2023-10-09T05:55:59.320352+0000","last_became_peered":"2023-10-09T05:55:59.320352+0000","last_unstale":"2023-10-09T05:58:00.874359+0000","last_undegraded":"2023-10-09T05:58:00.874359+0000","last_fullsized":"2023-10-09T05:58:00.874359+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:09:14.227028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":315,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.863073+0000","last_change":"2023-10-09T05:57:10.362295+0000","last_active":"2023-10-09T05:58:00.863073+0000","last_peered":"2023-10-09T05:58:00.863073+0000","last_clean":"2023-10-09T05:58:00.863073+0000","last_became_active":"2023-10-09T05:57:10.361249+0000","last_became_peered":"2023-10-09T05:57:10.361249+0000","last_unstale":"2023-10-09T05:58:00.863073+0000","last_undegraded":"2023-10-09T05:58:00.863073+0000","last_fullsized":"2023-10-09T05:58:00.863073+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:22.088261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":454,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.873763+0000","last_change":"2023-10-09T05:57:54.797410+0000","last_active":"2023-10-09T05:58:00.873763+0000","last_peered":"2023-10-09T05:58:00.873763+0000","last_clean":"2023-10-09T05:58:00.873763+0000","last_became_active":"2023-10-09T05:57:54.796543+0000","last_became_peered":"2023-10-09T05:57:54.796543+0000","last_unstale":"2023-10-09T05:58:00.873763+0000","last_undegraded":"2023-10-09T05:58:00.873763+0000","last_fullsized":"2023-10-09T05:58:00.873763+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.c","version":"269'8710","reported_seq":21021,"reported_epoch":269,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:58:02.639209+0000","last_change":"2023-10-09T05:57:24.712386+0000","last_active":"2023-10-09T05:58:02.639209+0000","last_peered":"2023-10-09T05:58:02.639209+0000","last_clean":"2023-10-09T05:57:22.413612+0000","last_became_active":"2023-10-09T05:57:24.703534+0000","last_became_peered":"2023-10-09T05:57:24.703534+0000","last_unstale":"2023-10-09T05:58:02.639209+0000","last_undegraded":"2023-10-09T05:58:02.639209+0000","last_fullsized":"2023-10-09T05:58:02.639209+0000","mapping_epoch":249,"log_start":"108'2503","ondisk_log_start":"108'2503","created":102,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6207,"log_dups_size":0,"ondisk_log_size":6207,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3062,"num_write_kb":96528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1508,"num_bytes_recovered":96321536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[5,6,7],"avail_no_missing":["5","3","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":249,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.862519+0000","last_change":"2023-10-09T05:54:00.619165+0000","last_active":"2023-10-09T05:58:00.862519+0000","last_peered":"2023-10-09T05:58:00.862519+0000","last_clean":"2023-10-09T05:58:00.862519+0000","last_became_active":"2023-10-09T05:54:00.618888+0000","last_became_peered":"2023-10-09T05:54:00.618888+0000","last_unstale":"2023-10-09T05:58:00.862519+0000","last_undegraded":"2023-10-09T05:58:00.862519+0000","last_fullsized":"2023-10-09T05:58:00.862519+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:07.175996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":288,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.862974+0000","last_change":"2023-10-09T05:57:23.483158+0000","last_active":"2023-10-09T05:58:00.862974+0000","last_peered":"2023-10-09T05:58:00.862974+0000","last_clean":"2023-10-09T05:58:00.862974+0000","last_became_active":"2023-10-09T05:57:23.482701+0000","last_became_peered":"2023-10-09T05:57:23.482701+0000","last_unstale":"2023-10-09T05:58:00.862974+0000","last_undegraded":"2023-10-09T05:58:00.862974+0000","last_fullsized":"2023-10-09T05:58:00.862974+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:51:08.624310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":456,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T05:58:06.252573+0000","last_change":"2023-10-09T05:57:09.347095+0000","last_active":"2023-10-09T05:58:06.252573+0000","last_peered":"2023-10-09T05:58:06.252573+0000","last_clean":"2023-10-09T05:58:06.252573+0000","last_became_active":"2023-10-09T05:56:49.229606+0000","last_became_peered":"2023-10-09T05:56:49.229606+0000","last_unstale":"2023-10-09T05:58:06.252573+0000","last_undegraded":"2023-10-09T05:58:06.252573+0000","last_fullsized":"2023-10-09T05:58:06.252573+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.d","version":"269'8574","reported_seq":15344,"reported_epoch":269,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:58:02.680970+0000","last_change":"2023-10-09T05:57:11.742947+0000","last_active":"2023-10-09T05:58:02.680970+0000","last_peered":"2023-10-09T05:58:02.680970+0000","last_clean":"2023-10-09T05:55:58.102294+0000","last_became_active":"2023-10-09T05:56:50.227382+0000","last_became_peered":"2023-10-09T05:56:50.227382+0000","last_unstale":"2023-10-09T05:58:02.680970+0000","last_undegraded":"2023-10-09T05:58:02.680970+0000","last_fullsized":"2023-10-09T05:58:02.680970+0000","mapping_epoch":234,"log_start":"108'2175","ondisk_log_start":"108'2175","created":102,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6399,"log_dups_size":0,"ondisk_log_size":6399,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1081350,"num_objects":16,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2984,"num_write_kb":96016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1226,"num_bytes_recovered":80347136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,1],"avail_no_missing":["0","1","2","5"],"object_location_counts":[{"shards":"0,1,5","objects":16}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":31,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.859299+0000","last_change":"2023-10-09T05:57:23.469928+0000","last_active":"2023-10-09T05:58:00.859299+0000","last_peered":"2023-10-09T05:58:00.859299+0000","last_clean":"2023-10-09T05:58:00.859299+0000","last_became_active":"2023-10-09T05:57:23.469697+0000","last_became_peered":"2023-10-09T05:57:23.469697+0000","last_unstale":"2023-10-09T05:58:00.859299+0000","last_undegraded":"2023-10-09T05:58:00.859299+0000","last_fullsized":"2023-10-09T05:58:00.859299+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:21:07.179434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":148,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.862188+0000","last_change":"2023-10-09T05:55:59.409563+0000","last_active":"2023-10-09T05:58:00.862188+0000","last_peered":"2023-10-09T05:58:00.862188+0000","last_clean":"2023-10-09T05:58:00.862188+0000","last_became_active":"2023-10-09T05:55:59.409184+0000","last_became_peered":"2023-10-09T05:55:59.409184+0000","last_unstale":"2023-10-09T05:58:00.862188+0000","last_undegraded":"2023-10-09T05:58:00.862188+0000","last_fullsized":"2023-10-09T05:58:00.862188+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:39.513591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":662,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.875522+0000","last_change":"2023-10-09T05:57:26.501248+0000","last_active":"2023-10-09T05:58:00.875522+0000","last_peered":"2023-10-09T05:58:00.875522+0000","last_clean":"2023-10-09T05:58:00.875522+0000","last_became_active":"2023-10-09T05:57:26.500602+0000","last_became_peered":"2023-10-09T05:57:26.500602+0000","last_unstale":"2023-10-09T05:58:00.875522+0000","last_undegraded":"2023-10-09T05:58:00.875522+0000","last_fullsized":"2023-10-09T05:58:00.875522+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":163,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.860463+0000","last_change":"2023-10-09T05:55:59.298953+0000","last_active":"2023-10-09T05:58:00.860463+0000","last_peered":"2023-10-09T05:58:00.860463+0000","last_clean":"2023-10-09T05:58:00.860463+0000","last_became_active":"2023-10-09T05:55:59.298624+0000","last_became_peered":"2023-10-09T05:55:59.298624+0000","last_unstale":"2023-10-09T05:58:00.860463+0000","last_undegraded":"2023-10-09T05:58:00.860463+0000","last_fullsized":"2023-10-09T05:58:00.860463+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:00.513870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":31,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.860361+0000","last_change":"2023-10-09T05:57:23.468480+0000","last_active":"2023-10-09T05:58:00.860361+0000","last_peered":"2023-10-09T05:58:00.860361+0000","last_clean":"2023-10-09T05:58:00.860361+0000","last_became_active":"2023-10-09T05:57:23.468179+0000","last_became_peered":"2023-10-09T05:57:23.468179+0000","last_unstale":"2023-10-09T05:58:00.860361+0000","last_undegraded":"2023-10-09T05:58:00.860361+0000","last_fullsized":"2023-10-09T05:58:00.860361+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:18.157188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":132,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.865100+0000","last_change":"2023-10-09T05:56:49.249711+0000","last_active":"2023-10-09T05:58:00.865100+0000","last_peered":"2023-10-09T05:58:00.865100+0000","last_clean":"2023-10-09T05:58:00.865100+0000","last_became_active":"2023-10-09T05:56:49.249272+0000","last_became_peered":"2023-10-09T05:56:49.249272+0000","last_unstale":"2023-10-09T05:58:00.865100+0000","last_undegraded":"2023-10-09T05:58:00.865100+0000","last_fullsized":"2023-10-09T05:58:00.865100+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:42.596986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.0","version":"269'8049","reported_seq":15628,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T05:58:02.714449+0000","last_change":"2023-10-09T05:57:39.325450+0000","last_active":"2023-10-09T05:58:02.714449+0000","last_peered":"2023-10-09T05:58:02.714449+0000","last_clean":"2023-10-09T05:58:02.714449+0000","last_became_active":"2023-10-09T05:57:39.315314+0000","last_became_peered":"2023-10-09T05:57:39.315314+0000","last_unstale":"2023-10-09T05:58:02.714449+0000","last_undegraded":"2023-10-09T05:58:02.714449+0000","last_fullsized":"2023-10-09T05:58:02.714449+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":8049,"log_dups_size":0,"ondisk_log_size":8049,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3051,"num_write_kb":97216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2066,"num_bytes_recovered":76922880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":459,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.862151+0000","last_change":"2023-10-09T05:56:49.237597+0000","last_active":"2023-10-09T05:58:00.862151+0000","last_peered":"2023-10-09T05:58:00.862151+0000","last_clean":"2023-10-09T05:58:00.862151+0000","last_became_active":"2023-10-09T05:56:49.237269+0000","last_became_peered":"2023-10-09T05:56:49.237269+0000","last_unstale":"2023-10-09T05:58:00.862151+0000","last_undegraded":"2023-10-09T05:58:00.862151+0000","last_fullsized":"2023-10-09T05:58:00.862151+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"269'8454","reported_seq":12983,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:02.723394+0000","last_change":"2023-10-09T05:57:58.835004+0000","last_active":"2023-10-09T05:58:02.723394+0000","last_peered":"2023-10-09T05:58:02.723394+0000","last_clean":"2023-10-09T05:52:24.946135+0000","last_became_active":"2023-10-09T05:57:24.618856+0000","last_became_peered":"2023-10-09T05:57:24.618856+0000","last_unstale":"2023-10-09T05:58:02.723394+0000","last_undegraded":"2023-10-09T05:57:58.834878+0000","last_fullsized":"2023-10-09T05:57:24.489261+0000","mapping_epoch":249,"log_start":"106'613","ondisk_log_start":"106'613","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7841,"log_dups_size":0,"ondisk_log_size":7841,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":901120,"num_objects":13,"num_object_clones":0,"num_object_copies":39,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":282,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2997,"num_write_kb":96352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1550,"num_bytes_recovered":57638912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[4,7],"avail_no_missing":["4","5","7"],"object_location_counts":[{"shards":"4,7","objects":13}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":289,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.874384+0000","last_change":"2023-10-09T05:56:49.254337+0000","last_active":"2023-10-09T05:58:00.874384+0000","last_peered":"2023-10-09T05:58:00.874384+0000","last_clean":"2023-10-09T05:58:00.874384+0000","last_became_active":"2023-10-09T05:56:49.247892+0000","last_became_peered":"2023-10-09T05:56:49.247892+0000","last_unstale":"2023-10-09T05:58:00.874384+0000","last_undegraded":"2023-10-09T05:58:00.874384+0000","last_fullsized":"2023-10-09T05:58:00.874384+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:44:20.157638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":246,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.863689+0000","last_change":"2023-10-09T05:51:57.203142+0000","last_active":"2023-10-09T05:58:00.863689+0000","last_peered":"2023-10-09T05:58:00.863689+0000","last_clean":"2023-10-09T05:58:00.863689+0000","last_became_active":"2023-10-09T05:51:57.202850+0000","last_became_peered":"2023-10-09T05:51:57.202850+0000","last_unstale":"2023-10-09T05:58:00.863689+0000","last_undegraded":"2023-10-09T05:58:00.863689+0000","last_fullsized":"2023-10-09T05:58:00.863689+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:51.090737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":194,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T05:58:06.251858+0000","last_change":"2023-10-09T05:57:51.789746+0000","last_active":"2023-10-09T05:58:06.251858+0000","last_peered":"2023-10-09T05:58:06.251858+0000","last_clean":"2023-10-09T05:58:06.251858+0000","last_became_active":"2023-10-09T05:57:51.789353+0000","last_became_peered":"2023-10-09T05:57:51.789353+0000","last_unstale":"2023-10-09T05:58:06.251858+0000","last_undegraded":"2023-10-09T05:58:06.251858+0000","last_fullsized":"2023-10-09T05:58:06.251858+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:28.241819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":566,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T05:58:01.899855+0000","last_change":"2023-10-09T05:58:01.899855+0000","last_active":"2023-10-09T05:58:01.899855+0000","last_peered":"2023-10-09T05:58:01.899855+0000","last_clean":"2023-10-09T05:58:01.899855+0000","last_became_active":"2023-10-09T05:58:01.899103+0000","last_became_peered":"2023-10-09T05:58:01.899103+0000","last_unstale":"2023-10-09T05:58:01.899855+0000","last_undegraded":"2023-10-09T05:58:01.899855+0000","last_fullsized":"2023-10-09T05:58:01.899855+0000","mapping_epoch":268,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":269,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.5","version":"269'8286","reported_seq":14396,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:02.641002+0000","last_change":"2023-10-09T05:57:52.875635+0000","last_active":"2023-10-09T05:58:02.641002+0000","last_peered":"2023-10-09T05:58:02.641002+0000","last_clean":"2023-10-09T05:57:50.707773+0000","last_became_active":"2023-10-09T05:57:52.850870+0000","last_became_peered":"2023-10-09T05:57:52.850870+0000","last_unstale":"2023-10-09T05:58:02.641002+0000","last_undegraded":"2023-10-09T05:57:52.746995+0000","last_fullsized":"2023-10-09T05:57:52.733005+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":8286,"log_dups_size":0,"ondisk_log_size":8286,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1482,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2970,"num_write_kb":94432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1142,"num_bytes_recovered":67731456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"3,7","objects":-96}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":52,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T05:58:06.251798+0000","last_change":"2023-10-09T05:57:23.465501+0000","last_active":"2023-10-09T05:58:06.251798+0000","last_peered":"2023-10-09T05:58:06.251798+0000","last_clean":"2023-10-09T05:58:06.251798+0000","last_became_active":"2023-10-09T05:57:23.465103+0000","last_became_peered":"2023-10-09T05:57:23.465103+0000","last_unstale":"2023-10-09T05:58:06.251798+0000","last_undegraded":"2023-10-09T05:58:06.251798+0000","last_fullsized":"2023-10-09T05:58:06.251798+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:02:45.891751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":160,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T05:58:06.252136+0000","last_change":"2023-10-09T05:55:59.308743+0000","last_active":"2023-10-09T05:58:06.252136+0000","last_peered":"2023-10-09T05:58:06.252136+0000","last_clean":"2023-10-09T05:58:06.252136+0000","last_became_active":"2023-10-09T05:55:59.308220+0000","last_became_peered":"2023-10-09T05:55:59.308220+0000","last_unstale":"2023-10-09T05:58:06.252136+0000","last_undegraded":"2023-10-09T05:58:06.252136+0000","last_fullsized":"2023-10-09T05:58:06.252136+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:44.246081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":163,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.861206+0000","last_change":"2023-10-09T05:56:49.238219+0000","last_active":"2023-10-09T05:58:00.861206+0000","last_peered":"2023-10-09T05:58:00.861206+0000","last_clean":"2023-10-09T05:58:00.861206+0000","last_became_active":"2023-10-09T05:56:49.237919+0000","last_became_peered":"2023-10-09T05:56:49.237919+0000","last_unstale":"2023-10-09T05:58:00.861206+0000","last_undegraded":"2023-10-09T05:58:00.861206+0000","last_fullsized":"2023-10-09T05:58:00.861206+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:44.599032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.7","version":"269'8353","reported_seq":8580,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T05:58:02.365610+0000","last_change":"2023-10-09T05:57:23.909942+0000","last_active":"2023-10-09T05:58:02.365610+0000","last_peered":"2023-10-09T05:58:02.365610+0000","last_clean":"2023-10-09T05:58:02.365610+0000","last_became_active":"2023-10-09T05:56:49.237277+0000","last_became_peered":"2023-10-09T05:56:49.237277+0000","last_unstale":"2023-10-09T05:58:02.365610+0000","last_undegraded":"2023-10-09T05:58:02.365610+0000","last_fullsized":"2023-10-09T05:58:02.365610+0000","mapping_epoch":233,"log_start":"106'370","ondisk_log_start":"106'370","created":102,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"247'7834","last_scrub_stamp":"2023-10-09T05:57:23.909751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:57:23.909751+0000","objects_scrubbed":519,"log_size":7983,"log_dups_size":0,"ondisk_log_size":7983,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:47:33.411071+0000","scrub_duration":0.066643623999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":519,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":198,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T05:58:06.252784+0000","last_change":"2023-10-09T05:56:49.233224+0000","last_active":"2023-10-09T05:58:06.252784+0000","last_peered":"2023-10-09T05:58:06.252784+0000","last_clean":"2023-10-09T05:58:06.252784+0000","last_became_active":"2023-10-09T05:56:49.232971+0000","last_became_peered":"2023-10-09T05:56:49.232971+0000","last_unstale":"2023-10-09T05:58:06.252784+0000","last_undegraded":"2023-10-09T05:58:06.252784+0000","last_fullsized":"2023-10-09T05:58:06.252784+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:55.265488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":663,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.863332+0000","last_change":"2023-10-09T05:57:10.809854+0000","last_active":"2023-10-09T05:58:00.863332+0000","last_peered":"2023-10-09T05:58:00.863332+0000","last_clean":"2023-10-09T05:58:00.863332+0000","last_became_active":"2023-10-09T05:53:59.202004+0000","last_became_peered":"2023-10-09T05:53:59.202004+0000","last_unstale":"2023-10-09T05:58:00.863332+0000","last_undegraded":"2023-10-09T05:58:00.863332+0000","last_fullsized":"2023-10-09T05:58:00.863332+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:51:12.346243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:51:12.346243+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":31,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.859461+0000","last_change":"2023-10-09T05:57:23.479174+0000","last_active":"2023-10-09T05:58:00.859461+0000","last_peered":"2023-10-09T05:58:00.859461+0000","last_clean":"2023-10-09T05:58:00.859461+0000","last_became_active":"2023-10-09T05:57:23.478897+0000","last_became_peered":"2023-10-09T05:57:23.478897+0000","last_unstale":"2023-10-09T05:58:00.859461+0000","last_undegraded":"2023-10-09T05:58:00.859461+0000","last_fullsized":"2023-10-09T05:58:00.859461+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:00.160042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":231,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.874632+0000","last_change":"2023-10-09T05:53:59.195325+0000","last_active":"2023-10-09T05:58:00.874632+0000","last_peered":"2023-10-09T05:58:00.874632+0000","last_clean":"2023-10-09T05:58:00.874632+0000","last_became_active":"2023-10-09T05:53:59.194954+0000","last_became_peered":"2023-10-09T05:53:59.194954+0000","last_unstale":"2023-10-09T05:58:00.874632+0000","last_undegraded":"2023-10-09T05:58:00.874632+0000","last_fullsized":"2023-10-09T05:58:00.874632+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:23.228416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":454,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.873746+0000","last_change":"2023-10-09T05:57:54.797571+0000","last_active":"2023-10-09T05:58:00.873746+0000","last_peered":"2023-10-09T05:58:00.873746+0000","last_clean":"2023-10-09T05:58:00.873746+0000","last_became_active":"2023-10-09T05:57:54.796937+0000","last_became_peered":"2023-10-09T05:57:54.796937+0000","last_unstale":"2023-10-09T05:58:00.873746+0000","last_undegraded":"2023-10-09T05:58:00.873746+0000","last_fullsized":"2023-10-09T05:58:00.873746+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":135,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.863169+0000","last_change":"2023-10-09T05:55:59.300015+0000","last_active":"2023-10-09T05:58:00.863169+0000","last_peered":"2023-10-09T05:58:00.863169+0000","last_clean":"2023-10-09T05:58:00.863169+0000","last_became_active":"2023-10-09T05:55:59.299281+0000","last_became_peered":"2023-10-09T05:55:59.299281+0000","last_unstale":"2023-10-09T05:58:00.863169+0000","last_undegraded":"2023-10-09T05:58:00.863169+0000","last_fullsized":"2023-10-09T05:58:00.863169+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:50:32.465904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":206,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T05:58:01.876526+0000","last_change":"2023-10-09T05:57:23.471369+0000","last_active":"2023-10-09T05:58:01.876526+0000","last_peered":"2023-10-09T05:58:01.876526+0000","last_clean":"2023-10-09T05:58:01.876526+0000","last_became_active":"2023-10-09T05:57:23.471049+0000","last_became_peered":"2023-10-09T05:57:23.471049+0000","last_unstale":"2023-10-09T05:58:01.876526+0000","last_undegraded":"2023-10-09T05:58:01.876526+0000","last_fullsized":"2023-10-09T05:58:01.876526+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:21.208825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":322,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.860700+0000","last_change":"2023-10-09T05:57:10.348399+0000","last_active":"2023-10-09T05:58:00.860700+0000","last_peered":"2023-10-09T05:58:00.860700+0000","last_clean":"2023-10-09T05:58:00.860700+0000","last_became_active":"2023-10-09T05:57:10.348061+0000","last_became_peered":"2023-10-09T05:57:10.348061+0000","last_unstale":"2023-10-09T05:58:00.860700+0000","last_undegraded":"2023-10-09T05:58:00.860700+0000","last_fullsized":"2023-10-09T05:58:00.860700+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:12.453118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"269'8497","reported_seq":10835,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:02.580015+0000","last_change":"2023-10-09T05:57:24.710787+0000","last_active":"2023-10-09T05:58:02.580015+0000","last_peered":"2023-10-09T05:58:02.580015+0000","last_clean":"2023-10-09T05:51:24.557317+0000","last_became_active":"2023-10-09T05:57:24.681937+0000","last_became_peered":"2023-10-09T05:57:24.681937+0000","last_unstale":"2023-10-09T05:58:02.580015+0000","last_undegraded":"2023-10-09T05:57:24.494361+0000","last_fullsized":"2023-10-09T05:57:24.468975+0000","mapping_epoch":249,"log_start":"107'1426","ondisk_log_start":"107'1426","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7071,"log_dups_size":0,"ondisk_log_size":7071,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65536,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1585,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3026,"num_write_kb":96864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":896,"num_bytes_recovered":41320448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[5,6],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"5,6","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":652,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.875333+0000","last_change":"2023-10-09T05:57:26.518453+0000","last_active":"2023-10-09T05:58:00.875333+0000","last_peered":"2023-10-09T05:58:00.875333+0000","last_clean":"2023-10-09T05:58:00.875333+0000","last_became_active":"2023-10-09T05:57:26.517990+0000","last_became_peered":"2023-10-09T05:57:26.517990+0000","last_unstale":"2023-10-09T05:58:00.875333+0000","last_undegraded":"2023-10-09T05:58:00.875333+0000","last_fullsized":"2023-10-09T05:58:00.875333+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":103,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.863351+0000","last_change":"2023-10-09T05:53:59.201576+0000","last_active":"2023-10-09T05:58:00.863351+0000","last_peered":"2023-10-09T05:58:00.863351+0000","last_clean":"2023-10-09T05:58:00.863351+0000","last_became_active":"2023-10-09T05:53:59.201309+0000","last_became_peered":"2023-10-09T05:53:59.201309+0000","last_unstale":"2023-10-09T05:58:00.863351+0000","last_undegraded":"2023-10-09T05:58:00.863351+0000","last_fullsized":"2023-10-09T05:58:00.863351+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:54.956371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":43,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T05:58:06.251777+0000","last_change":"2023-10-09T05:57:10.368477+0000","last_active":"2023-10-09T05:58:06.251777+0000","last_peered":"2023-10-09T05:58:06.251777+0000","last_clean":"2023-10-09T05:58:06.251777+0000","last_became_active":"2023-10-09T05:57:10.368178+0000","last_became_peered":"2023-10-09T05:57:10.368178+0000","last_unstale":"2023-10-09T05:58:06.251777+0000","last_undegraded":"2023-10-09T05:58:06.251777+0000","last_fullsized":"2023-10-09T05:58:06.251777+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:38.053000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":194,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.863453+0000","last_change":"2023-10-09T05:53:59.188462+0000","last_active":"2023-10-09T05:58:00.863453+0000","last_peered":"2023-10-09T05:58:00.863453+0000","last_clean":"2023-10-09T05:58:00.863453+0000","last_became_active":"2023-10-09T05:53:59.188061+0000","last_became_peered":"2023-10-09T05:53:59.188061+0000","last_unstale":"2023-10-09T05:58:00.863453+0000","last_undegraded":"2023-10-09T05:58:00.863453+0000","last_fullsized":"2023-10-09T05:58:00.863453+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:19.167114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"269'8047","reported_seq":11377,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T05:58:02.734831+0000","last_change":"2023-10-09T05:57:35.278805+0000","last_active":"2023-10-09T05:58:02.734831+0000","last_peered":"2023-10-09T05:58:02.734831+0000","last_clean":"2023-10-09T05:58:02.734831+0000","last_became_active":"2023-10-09T05:57:35.270507+0000","last_became_peered":"2023-10-09T05:57:35.270507+0000","last_unstale":"2023-10-09T05:58:02.734831+0000","last_undegraded":"2023-10-09T05:58:02.734831+0000","last_fullsized":"2023-10-09T05:58:02.734831+0000","mapping_epoch":254,"log_start":"107'871","ondisk_log_start":"107'871","created":102,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7176,"log_dups_size":0,"ondisk_log_size":7176,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1998848,"num_objects":30,"num_object_clones":0,"num_object_copies":90,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":30,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3022,"num_write_kb":97696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1192,"num_bytes_recovered":19791872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":623,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.862942+0000","last_change":"2023-10-09T05:57:13.235727+0000","last_active":"2023-10-09T05:58:00.862942+0000","last_peered":"2023-10-09T05:58:00.862942+0000","last_clean":"2023-10-09T05:58:00.862942+0000","last_became_active":"2023-10-09T05:57:13.235353+0000","last_became_peered":"2023-10-09T05:57:13.235353+0000","last_unstale":"2023-10-09T05:58:00.862942+0000","last_undegraded":"2023-10-09T05:58:00.862942+0000","last_fullsized":"2023-10-09T05:58:00.862942+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":275,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.862283+0000","last_change":"2023-10-09T05:53:59.228948+0000","last_active":"2023-10-09T05:58:00.862283+0000","last_peered":"2023-10-09T05:58:00.862283+0000","last_clean":"2023-10-09T05:58:00.862283+0000","last_became_active":"2023-10-09T05:53:59.186359+0000","last_became_peered":"2023-10-09T05:53:59.186359+0000","last_unstale":"2023-10-09T05:58:00.862283+0000","last_undegraded":"2023-10-09T05:58:00.862283+0000","last_fullsized":"2023-10-09T05:58:00.862283+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:39.990576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":126,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.875208+0000","last_change":"2023-10-09T05:55:59.339957+0000","last_active":"2023-10-09T05:58:00.875208+0000","last_peered":"2023-10-09T05:58:00.875208+0000","last_clean":"2023-10-09T05:58:00.875208+0000","last_became_active":"2023-10-09T05:55:59.339614+0000","last_became_peered":"2023-10-09T05:55:59.339614+0000","last_unstale":"2023-10-09T05:58:00.875208+0000","last_undegraded":"2023-10-09T05:58:00.875208+0000","last_fullsized":"2023-10-09T05:58:00.875208+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:01.613113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":31,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.871095+0000","last_change":"2023-10-09T05:57:23.477143+0000","last_active":"2023-10-09T05:58:00.871095+0000","last_peered":"2023-10-09T05:58:00.871095+0000","last_clean":"2023-10-09T05:58:00.871095+0000","last_became_active":"2023-10-09T05:57:23.475655+0000","last_became_peered":"2023-10-09T05:57:23.475655+0000","last_unstale":"2023-10-09T05:58:00.871095+0000","last_undegraded":"2023-10-09T05:58:00.871095+0000","last_fullsized":"2023-10-09T05:58:00.871095+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:29.528023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"269'7959","reported_seq":20229,"reported_epoch":269,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:58:02.593597+0000","last_change":"2023-10-09T05:57:24.625602+0000","last_active":"2023-10-09T05:58:02.593597+0000","last_peered":"2023-10-09T05:58:02.593597+0000","last_clean":"2023-10-09T05:57:22.341427+0000","last_became_active":"2023-10-09T05:57:24.616071+0000","last_became_peered":"2023-10-09T05:57:24.616071+0000","last_unstale":"2023-10-09T05:58:02.593597+0000","last_undegraded":"2023-10-09T05:58:02.593597+0000","last_fullsized":"2023-10-09T05:58:02.593597+0000","mapping_epoch":249,"log_start":"106'632","ondisk_log_start":"106'632","created":102,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":7327,"log_dups_size":0,"ondisk_log_size":7327,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3293184,"num_objects":50,"num_object_clones":0,"num_object_copies":150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":50,"num_objects_unfound":0,"num_objects_dirty":50,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2959,"num_write_kb":96304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1505,"num_bytes_recovered":95617024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,2,6],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":50}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":513,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.860843+0000","last_change":"2023-10-09T05:57:26.503736+0000","last_active":"2023-10-09T05:58:00.860843+0000","last_peered":"2023-10-09T05:58:00.860843+0000","last_clean":"2023-10-09T05:58:00.860843+0000","last_became_active":"2023-10-09T05:57:26.502911+0000","last_became_peered":"2023-10-09T05:57:26.502911+0000","last_unstale":"2023-10-09T05:58:00.860843+0000","last_undegraded":"2023-10-09T05:58:00.860843+0000","last_fullsized":"2023-10-09T05:58:00.860843+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":273,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.875274+0000","last_change":"2023-10-09T05:57:58.832790+0000","last_active":"2023-10-09T05:58:00.875274+0000","last_peered":"2023-10-09T05:58:00.875274+0000","last_clean":"2023-10-09T05:58:00.875274+0000","last_became_active":"2023-10-09T05:57:58.832471+0000","last_became_peered":"2023-10-09T05:57:58.832471+0000","last_unstale":"2023-10-09T05:58:00.875274+0000","last_undegraded":"2023-10-09T05:58:00.875274+0000","last_fullsized":"2023-10-09T05:58:00.875274+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:39.358174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":308,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.860867+0000","last_change":"2023-10-09T05:57:58.836050+0000","last_active":"2023-10-09T05:58:00.860867+0000","last_peered":"2023-10-09T05:58:00.860867+0000","last_clean":"2023-10-09T05:58:00.860867+0000","last_became_active":"2023-10-09T05:57:58.833691+0000","last_became_peered":"2023-10-09T05:57:58.833691+0000","last_unstale":"2023-10-09T05:58:00.860867+0000","last_undegraded":"2023-10-09T05:58:00.860867+0000","last_fullsized":"2023-10-09T05:58:00.860867+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:32.257849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":211,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.862586+0000","last_change":"2023-10-09T05:40:12.676587+0000","last_active":"2023-10-09T05:58:00.862586+0000","last_peered":"2023-10-09T05:58:00.862586+0000","last_clean":"2023-10-09T05:58:00.862586+0000","last_became_active":"2023-10-09T05:40:12.676330+0000","last_became_peered":"2023-10-09T05:40:12.676330+0000","last_unstale":"2023-10-09T05:58:00.862586+0000","last_undegraded":"2023-10-09T05:58:00.862586+0000","last_fullsized":"2023-10-09T05:58:00.862586+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:46.659556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"269'8086","reported_seq":8298,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T05:58:02.391803+0000","last_change":"2023-10-09T05:55:22.211279+0000","last_active":"2023-10-09T05:58:02.391803+0000","last_peered":"2023-10-09T05:58:02.391803+0000","last_clean":"2023-10-09T05:58:02.391803+0000","last_became_active":"2023-10-09T05:55:22.208477+0000","last_became_peered":"2023-10-09T05:55:22.208477+0000","last_unstale":"2023-10-09T05:58:02.391803+0000","last_undegraded":"2023-10-09T05:58:02.391803+0000","last_fullsized":"2023-10-09T05:58:02.391803+0000","mapping_epoch":174,"log_start":"107'1488","ondisk_log_start":"107'1488","created":102,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6598,"log_dups_size":0,"ondisk_log_size":6598,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3091,"num_write_kb":97312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.1e","version":"269'9414","reported_seq":21998,"reported_epoch":269,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:58:02.751424+0000","last_change":"2023-10-09T05:57:10.441127+0000","last_active":"2023-10-09T05:58:02.751424+0000","last_peered":"2023-10-09T05:58:02.751424+0000","last_clean":"2023-10-09T05:55:58.105865+0000","last_became_active":"2023-10-09T05:57:10.429390+0000","last_became_peered":"2023-10-09T05:57:10.429390+0000","last_unstale":"2023-10-09T05:58:02.751424+0000","last_undegraded":"2023-10-09T05:58:02.751424+0000","last_fullsized":"2023-10-09T05:58:02.751424+0000","mapping_epoch":238,"log_start":"113'2920","ondisk_log_start":"113'2920","created":207,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6494,"log_dups_size":0,"ondisk_log_size":6494,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6003,"num_write_kb":191680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3074,"num_bytes_recovered":196280320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,5,1],"avail_no_missing":["4","1","2","5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":198,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.863545+0000","last_change":"2023-10-09T05:53:59.191302+0000","last_active":"2023-10-09T05:58:00.863545+0000","last_peered":"2023-10-09T05:58:00.863545+0000","last_clean":"2023-10-09T05:58:00.863545+0000","last_became_active":"2023-10-09T05:53:59.190911+0000","last_became_peered":"2023-10-09T05:53:59.190911+0000","last_unstale":"2023-10-09T05:58:00.863545+0000","last_undegraded":"2023-10-09T05:58:00.863545+0000","last_fullsized":"2023-10-09T05:58:00.863545+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:12.801201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":52,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T05:58:06.251600+0000","last_change":"2023-10-09T05:57:23.469986+0000","last_active":"2023-10-09T05:58:06.251600+0000","last_peered":"2023-10-09T05:58:06.251600+0000","last_clean":"2023-10-09T05:58:06.251600+0000","last_became_active":"2023-10-09T05:57:23.469546+0000","last_became_peered":"2023-10-09T05:57:23.469546+0000","last_unstale":"2023-10-09T05:58:06.251600+0000","last_undegraded":"2023-10-09T05:58:06.251600+0000","last_fullsized":"2023-10-09T05:58:06.251600+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:53.762345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.19","version":"269'9332","reported_seq":17561,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:58:02.751414+0000","last_change":"2023-10-09T05:57:24.790448+0000","last_active":"2023-10-09T05:58:02.751414+0000","last_peered":"2023-10-09T05:58:02.751414+0000","last_clean":"2023-10-09T05:51:33.020184+0000","last_became_active":"2023-10-09T05:57:11.693953+0000","last_became_peered":"2023-10-09T05:57:11.693953+0000","last_unstale":"2023-10-09T05:58:02.751414+0000","last_undegraded":"2023-10-09T05:57:24.784586+0000","last_fullsized":"2023-10-09T05:57:11.339841+0000","mapping_epoch":239,"log_start":"110'2565","ondisk_log_start":"110'2565","created":188,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":6767,"log_dups_size":0,"ondisk_log_size":6767,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294912,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":959,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6068,"num_write_kb":194304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3701,"num_bytes_recovered":184811520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,7","objects":5}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":59,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.862957+0000","last_change":"2023-10-09T05:56:49.244931+0000","last_active":"2023-10-09T05:58:00.862957+0000","last_peered":"2023-10-09T05:58:00.862957+0000","last_clean":"2023-10-09T05:58:00.862957+0000","last_became_active":"2023-10-09T05:56:49.244651+0000","last_became_peered":"2023-10-09T05:56:49.244651+0000","last_unstale":"2023-10-09T05:58:00.862957+0000","last_undegraded":"2023-10-09T05:58:00.862957+0000","last_fullsized":"2023-10-09T05:58:00.862957+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:53.906578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":305,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.863731+0000","last_change":"2023-10-09T05:57:10.356351+0000","last_active":"2023-10-09T05:58:00.863731+0000","last_peered":"2023-10-09T05:58:00.863731+0000","last_clean":"2023-10-09T05:58:00.863731+0000","last_became_active":"2023-10-09T05:57:10.356037+0000","last_became_peered":"2023-10-09T05:57:10.356037+0000","last_unstale":"2023-10-09T05:58:00.863731+0000","last_undegraded":"2023-10-09T05:58:00.863731+0000","last_fullsized":"2023-10-09T05:58:00.863731+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:15.139912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"269'9219","reported_seq":9419,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T05:58:02.754896+0000","last_change":"2023-10-09T05:57:17.757452+0000","last_active":"2023-10-09T05:58:02.754896+0000","last_peered":"2023-10-09T05:58:02.754896+0000","last_clean":"2023-10-09T05:58:02.754896+0000","last_became_active":"2023-10-09T05:54:45.664954+0000","last_became_peered":"2023-10-09T05:54:45.664954+0000","last_unstale":"2023-10-09T05:58:02.754896+0000","last_undegraded":"2023-10-09T05:58:02.754896+0000","last_fullsized":"2023-10-09T05:58:02.754896+0000","mapping_epoch":188,"log_start":"116'3301","ondisk_log_start":"116'3301","created":188,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5918,"log_dups_size":0,"ondisk_log_size":5918,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5972,"num_write_kb":190912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":59,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.863345+0000","last_change":"2023-10-09T05:56:49.245612+0000","last_active":"2023-10-09T05:58:00.863345+0000","last_peered":"2023-10-09T05:58:00.863345+0000","last_clean":"2023-10-09T05:58:00.863345+0000","last_became_active":"2023-10-09T05:56:49.245235+0000","last_became_peered":"2023-10-09T05:56:49.245235+0000","last_unstale":"2023-10-09T05:58:00.863345+0000","last_undegraded":"2023-10-09T05:58:00.863345+0000","last_fullsized":"2023-10-09T05:58:00.863345+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:29.016794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":262,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.864651+0000","last_change":"2023-10-09T05:51:57.093104+0000","last_active":"2023-10-09T05:58:00.864651+0000","last_peered":"2023-10-09T05:58:00.864651+0000","last_clean":"2023-10-09T05:58:00.864651+0000","last_became_active":"2023-10-09T05:51:57.092810+0000","last_became_peered":"2023-10-09T05:51:57.092810+0000","last_unstale":"2023-10-09T05:58:00.864651+0000","last_undegraded":"2023-10-09T05:58:00.864651+0000","last_fullsized":"2023-10-09T05:58:00.864651+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:51.192227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.1b","version":"269'9522","reported_seq":28880,"reported_epoch":269,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:58:02.766284+0000","last_change":"2023-10-09T05:58:00.209920+0000","last_active":"2023-10-09T05:58:02.766284+0000","last_peered":"2023-10-09T05:58:02.766284+0000","last_clean":"2023-10-09T05:57:57.295804+0000","last_became_active":"2023-10-09T05:58:00.191244+0000","last_became_peered":"2023-10-09T05:58:00.191244+0000","last_unstale":"2023-10-09T05:58:02.766284+0000","last_undegraded":"2023-10-09T05:58:02.766284+0000","last_fullsized":"2023-10-09T05:58:02.766284+0000","mapping_epoch":266,"log_start":"125'4255","ondisk_log_start":"125'4255","created":207,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":5267,"log_dups_size":0,"ondisk_log_size":5267,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4161536,"num_objects":64,"num_object_clones":0,"num_object_copies":192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":64,"num_objects_unfound":0,"num_objects_dirty":64,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6040,"num_write_kb":195296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6264,"num_bytes_recovered":364544000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,6,4],"avail_no_missing":["3","1","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":64}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":31,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.860142+0000","last_change":"2023-10-09T05:57:23.479560+0000","last_active":"2023-10-09T05:58:00.860142+0000","last_peered":"2023-10-09T05:58:00.860142+0000","last_clean":"2023-10-09T05:58:00.860142+0000","last_became_active":"2023-10-09T05:57:23.479177+0000","last_became_peered":"2023-10-09T05:57:23.479177+0000","last_unstale":"2023-10-09T05:58:00.860142+0000","last_undegraded":"2023-10-09T05:58:00.860142+0000","last_fullsized":"2023-10-09T05:58:00.860142+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:41.925161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":31,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.858603+0000","last_change":"2023-10-09T05:57:23.475179+0000","last_active":"2023-10-09T05:58:00.858603+0000","last_peered":"2023-10-09T05:58:00.858603+0000","last_clean":"2023-10-09T05:58:00.858603+0000","last_became_active":"2023-10-09T05:57:23.474904+0000","last_became_peered":"2023-10-09T05:57:23.474904+0000","last_unstale":"2023-10-09T05:58:00.858603+0000","last_undegraded":"2023-10-09T05:58:00.858603+0000","last_fullsized":"2023-10-09T05:58:00.858603+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:54.587601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.1a","version":"269'9431","reported_seq":11824,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T05:58:02.766031+0000","last_change":"2023-10-09T05:56:49.264672+0000","last_active":"2023-10-09T05:58:02.766031+0000","last_peered":"2023-10-09T05:58:02.766031+0000","last_clean":"2023-10-09T05:58:02.766031+0000","last_became_active":"2023-10-09T05:56:49.239350+0000","last_became_peered":"2023-10-09T05:56:49.239350+0000","last_unstale":"2023-10-09T05:58:02.766031+0000","last_undegraded":"2023-10-09T05:58:02.766031+0000","last_fullsized":"2023-10-09T05:58:02.766031+0000","mapping_epoch":233,"log_start":"131'5109","ondisk_log_start":"131'5109","created":207,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:50:38.764338+0000","last_clean_scrub_stamp":"2023-10-09T05:50:38.764338+0000","objects_scrubbed":0,"log_size":4322,"log_dups_size":0,"ondisk_log_size":4322,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393216,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6007,"num_write_kb":192416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":543,"num_bytes_recovered":35553280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":236,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T05:58:01.875360+0000","last_change":"2023-10-09T05:57:10.379258+0000","last_active":"2023-10-09T05:58:01.875360+0000","last_peered":"2023-10-09T05:58:01.875360+0000","last_clean":"2023-10-09T05:58:01.875360+0000","last_became_active":"2023-10-09T05:57:10.359745+0000","last_became_peered":"2023-10-09T05:57:10.359745+0000","last_unstale":"2023-10-09T05:58:01.875360+0000","last_undegraded":"2023-10-09T05:58:01.875360+0000","last_fullsized":"2023-10-09T05:58:01.875360+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:14.750256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":288,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T05:58:00.862419+0000","last_change":"2023-10-09T05:57:23.481065+0000","last_active":"2023-10-09T05:58:00.862419+0000","last_peered":"2023-10-09T05:58:00.862419+0000","last_clean":"2023-10-09T05:58:00.862419+0000","last_became_active":"2023-10-09T05:57:23.480203+0000","last_became_peered":"2023-10-09T05:57:23.480203+0000","last_unstale":"2023-10-09T05:58:00.862419+0000","last_undegraded":"2023-10-09T05:58:00.862419+0000","last_fullsized":"2023-10-09T05:58:00.862419+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:15:53.196457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":46,"stat_sum":{"num_bytes":33751034,"num_objects":514,"num_object_clones":0,"num_object_copies":1542,"num_objects_missing_on_primary":1682,"num_objects_missing":1682,"num_objects_degraded":20227,"num_objects_misplaced":1092,"num_objects_unfound":0,"num_objects_dirty":514,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":190607,"num_write_kb":6083201,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":75221,"num_bytes_recovered":4063887384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1914372096,"data_stored":1914372096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":320765,"ondisk_log_size":320765,"up":138,"acting":123,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":10792,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":2689152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234221,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311064,"kb_used_data":5584,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93434088,"statfs":{"total":95995035648,"available":95676506112,"internally_reserved":0,"allocated":5718016,"data_stored":4723793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:02 2023","interfaces":[{"interface":"back","average":{"1min":1.9319999999999999,"5min":1.105,"15min":1.7949999999999999},"min":{"1min":0.246,"5min":0.22,"15min":0.22},"max":{"1min":21.038,"5min":21.038,"15min":21.038},"last":0.32200000000000001},{"interface":"front","average":{"1min":2.0129999999999999,"5min":1.1100000000000001,"15min":1.6859999999999999},"min":{"1min":0.245,"5min":0.224,"15min":0.224},"max":{"1min":21.152000000000001,"5min":21.152000000000001,"15min":21.152000000000001},"last":0.48699999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:57:26 2023","interfaces":[{"interface":"back","average":{"1min":4.3209999999999997,"5min":1.54,"15min":0.94799999999999995},"min":{"1min":0.30299999999999999,"5min":0.27200000000000002,"15min":0.22900000000000001},"max":{"1min":35.113999999999997,"5min":35.113999999999997,"15min":35.113999999999997},"last":1.28},{"interface":"front","average":{"1min":3.383,"5min":1.2270000000000001,"15min":0.85499999999999998},"min":{"1min":0.317,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":34.768999999999998,"5min":34.768999999999998,"15min":34.768999999999998},"last":0.628}]},{"osd":2,"last update":"Mon Oct 9 05:58:02 2023","interfaces":[{"interface":"back","average":{"1min":12.797000000000001,"5min":3.3069999999999999,"15min":2.0230000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28199999999999997},"max":{"1min":119.91200000000001,"5min":119.91200000000001,"15min":119.91200000000001},"last":0.45200000000000001},{"interface":"front","average":{"1min":12.898999999999999,"5min":3.3580000000000001,"15min":2.117},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":119.95099999999999,"5min":119.95099999999999,"15min":119.95099999999999},"last":0.439}]},{"osd":3,"last update":"Mon Oct 9 05:57:32 2023","interfaces":[{"interface":"back","average":{"1min":7.0090000000000003,"5min":2.5270000000000001,"15min":2.9289999999999998},"min":{"1min":0.38700000000000001,"5min":0.35999999999999999,"15min":0.33300000000000002},"max":{"1min":70.840999999999994,"5min":70.840999999999994,"15min":147.54900000000001},"last":0.60499999999999998},{"interface":"front","average":{"1min":6.6109999999999998,"5min":2.4199999999999999,"15min":2.8250000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":71.212000000000003,"5min":71.212000000000003,"15min":144.065},"last":0.54800000000000004}]},{"osd":4,"last update":"Mon Oct 9 05:57:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":0.58999999999999997,"15min":0.58499999999999996},"min":{"1min":0.27000000000000002,"5min":0.248,"15min":0.24399999999999999},"max":{"1min":3.7120000000000002,"5min":3.7120000000000002,"15min":9.5329999999999995},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.55100000000000005,"15min":0.63700000000000001},"min":{"1min":0.34499999999999997,"5min":0.29899999999999999,"15min":0.27400000000000002},"max":{"1min":1.381,"5min":3.0019999999999998,"15min":7.6120000000000001},"last":0.35999999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:57:11 2023","interfaces":[{"interface":"back","average":{"1min":1.8160000000000001,"5min":0.86399999999999999,"15min":0.72799999999999998},"min":{"1min":0.30399999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":15.622,"5min":15.622,"15min":15.622},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.69599999999999995,"5min":0.61699999999999999,"15min":0.60499999999999998},"min":{"1min":0.32200000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.982,"5min":3.9380000000000002,"15min":3.9380000000000002},"last":0.47099999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:57:11 2023","interfaces":[{"interface":"back","average":{"1min":1.389,"5min":0.77700000000000002,"15min":1.244},"min":{"1min":0.28799999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":12.727,"5min":12.727,"15min":131.94399999999999},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.879,"5min":0.66000000000000003,"15min":1.238},"min":{"1min":0.36699999999999999,"5min":0.33100000000000002,"15min":0.29999999999999999},"max":{"1min":2.4580000000000002,"5min":3.2120000000000002,"15min":135.23400000000001},"last":0.38700000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495857,"num_pgs":87,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":372688,"kb_used_data":71304,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93372464,"statfs":{"total":95995035648,"available":95613403136,"internally_reserved":0,"allocated":73015296,"data_stored":72029952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2200000000000002,"5min":1.534,"15min":1.286},"min":{"1min":0.23499999999999999,"5min":0.224,"15min":0.224},"max":{"1min":13.026999999999999,"5min":49.389000000000003,"15min":49.389000000000003},"last":0.33500000000000002},{"interface":"front","average":{"1min":1.8280000000000001,"5min":1.4710000000000001,"15min":1.3089999999999999},"min":{"1min":0.26300000000000001,"5min":0.253,"15min":0.253},"max":{"1min":12.629,"5min":49.338999999999999,"15min":49.338999999999999},"last":0.45200000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:57:40 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":0.96399999999999997,"15min":0.77200000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.23899999999999999},"max":{"1min":17.341000000000001,"5min":17.341000000000001,"15min":17.341000000000001},"last":0.432},{"interface":"front","average":{"1min":1.867,"5min":0.96799999999999997,"15min":0.81499999999999995},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":12.474,"5min":12.474,"15min":12.474},"last":0.38800000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:58:02 2023","interfaces":[{"interface":"back","average":{"1min":6.8380000000000001,"5min":2.097,"15min":1.1899999999999999},"min":{"1min":0.311,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":61.088999999999999,"5min":61.088999999999999,"15min":61.088999999999999},"last":0.311},{"interface":"front","average":{"1min":6.3579999999999997,"5min":2.1040000000000001,"15min":1.1879999999999999},"min":{"1min":0.38800000000000001,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":61.082000000000001,"5min":61.082000000000001,"15min":61.082000000000001},"last":0.41699999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:57:40 2023","interfaces":[{"interface":"back","average":{"1min":2.6619999999999999,"5min":3.714,"15min":1.7809999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":20.478000000000002,"5min":93.626000000000005,"15min":93.626000000000005},"last":1.5940000000000001},{"interface":"front","average":{"1min":2.7450000000000001,"5min":3.444,"15min":1.7030000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":20.5,"5min":93.643000000000001,"15min":93.643000000000001},"last":1.653}]},{"osd":4,"last update":"Mon Oct 9 05:57:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":0.72599999999999998,"15min":0.71099999999999997},"min":{"1min":0.252,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.5709999999999997,"5min":4.5709999999999997,"15min":7.9100000000000001},"last":0.35599999999999998},{"interface":"front","average":{"1min":1.2629999999999999,"5min":0.84999999999999998,"15min":0.73099999999999998},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.2069999999999999,"5min":8.734,"15min":8.734},"last":0.54300000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:57:48 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":1.284,"15min":0.89500000000000002},"min":{"1min":0.316,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.5910000000000002,"5min":45.933999999999997,"15min":45.933999999999997},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.2669999999999999,"15min":0.92400000000000004},"min":{"1min":0.35899999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.786,"5min":45.917999999999999,"15min":45.917999999999999},"last":0.51100000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:57:12 2023","interfaces":[{"interface":"back","average":{"1min":3.5019999999999998,"5min":1.222,"15min":0.97999999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":52.042999999999999,"5min":52.042999999999999,"15min":52.042999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":3.306,"5min":1.266,"15min":0.97199999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":51.259,"5min":51.259,"15min":51.259},"last":0.47799999999999998}]}]},{"osd":1,"up_from":0,"seq":382252089446,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":129,"seq":554050781248,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":141384,"kb_used_data":3520,"kb_used_omap":29,"kb_used_meta":137826,"kb_avail":93603768,"statfs":{"total":95995035648,"available":95850258432,"internally_reserved":0,"allocated":3604480,"data_stored":2042913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30322,"internal_metadata":141134222},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:57:04 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.81200000000000006,"15min":0.73799999999999999},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":8.3300000000000001,"5min":9.1720000000000006,"15min":9.1720000000000006},"last":0.309},{"interface":"front","average":{"1min":0.93100000000000005,"5min":0.88100000000000001,"15min":1.0589999999999999},"min":{"1min":0.245,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":6.6420000000000003,"5min":6.6420000000000003,"15min":6.6420000000000003},"last":0.26800000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:57:04 2023","interfaces":[{"interface":"back","average":{"1min":1.208,"5min":0.85199999999999998,"15min":0.97599999999999998},"min":{"1min":0.32300000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":6.6040000000000001,"5min":6.6040000000000001,"15min":6.6040000000000001},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.84299999999999997,"5min":0.83199999999999996,"15min":0.94999999999999996},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.1389999999999998,"5min":5.4160000000000004,"15min":5.4160000000000004},"last":0.246}]},{"osd":3,"last update":"Mon Oct 9 05:57:04 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":0.85499999999999998,"15min":0.80100000000000005},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.6790000000000003,"5min":9.6609999999999996,"15min":9.6609999999999996},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.83599999999999997,"15min":0.879},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.0030000000000001,"5min":7.1749999999999998,"15min":7.1749999999999998},"last":0.33600000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:57:04 2023","interfaces":[{"interface":"back","average":{"1min":16.059000000000001,"5min":3.9460000000000002,"15min":2.1299999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":149.07400000000001,"5min":149.07400000000001,"15min":149.07400000000001},"last":0.51600000000000001},{"interface":"front","average":{"1min":15.882,"5min":3.778,"15min":1.8620000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":149.12700000000001,"5min":149.12700000000001,"15min":149.12700000000001},"last":0.76900000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:57:04 2023","interfaces":[{"interface":"back","average":{"1min":7.8250000000000002,"5min":2.306,"15min":1.534},"min":{"1min":0.373,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":100.25,"5min":100.25,"15min":100.25},"last":0.57399999999999995},{"interface":"front","average":{"1min":7.9779999999999998,"5min":2.3090000000000002,"15min":1.379},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":102.40300000000001,"5min":102.40300000000001,"15min":102.40300000000001},"last":0.64500000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:57:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":0.95899999999999996,"15min":1.2150000000000001},"min":{"1min":0.378,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":8.1470000000000002,"5min":8.1470000000000002,"15min":8.1470000000000002},"last":0.59599999999999997},{"interface":"front","average":{"1min":1.6539999999999999,"5min":1.115,"15min":1.1890000000000001},"min":{"1min":0.38100000000000001,"5min":0.378,"15min":0.378},"max":{"1min":8.0510000000000002,"5min":9.141,"15min":9.141},"last":0.67600000000000005}]},{"osd":7,"last update":"Mon Oct 9 05:57:04 2023","interfaces":[{"interface":"back","average":{"1min":5.5890000000000004,"5min":2.0289999999999999,"15min":1.623},"min":{"1min":0.443,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":99.489000000000004,"5min":99.489000000000004,"15min":99.489000000000004},"last":0.69599999999999995},{"interface":"front","average":{"1min":5.2709999999999999,"5min":1.865,"15min":1.5589999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":99.510000000000005,"5min":99.510000000000005,"15min":99.510000000000005},"last":0.57899999999999996}]}]},{"osd":2,"up_from":94,"seq":403726925938,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":953028,"kb_used_data":683772,"kb_used_omap":81,"kb_used_meta":269166,"kb_avail":92792124,"statfs":{"total":95995035648,"available":95019134976,"internally_reserved":0,"allocated":700182528,"data_stored":698745631,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":275626202},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:57:10 2023","interfaces":[{"interface":"back","average":{"1min":3.2509999999999999,"5min":1.413,"15min":1.603},"min":{"1min":0.23300000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":50.271000000000001,"5min":50.271000000000001,"15min":50.271000000000001},"last":0.23599999999999999},{"interface":"front","average":{"1min":3.6600000000000001,"5min":1.4279999999999999,"15min":1.516},"min":{"1min":0.23899999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":48.045000000000002,"5min":48.045000000000002,"15min":48.045000000000002},"last":0.28199999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:58:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.0069999999999999,"15min":1.357},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":3.8959999999999999,"5min":7.7119999999999997,"15min":163.24199999999999},"last":0.33800000000000002},{"interface":"front","average":{"1min":1.4550000000000001,"5min":0.85299999999999998,"15min":1.272},"min":{"1min":0.223,"5min":0.223,"15min":0.214},"max":{"1min":6.9790000000000001,"5min":6.9790000000000001,"15min":163.27000000000001},"last":0.35899999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:57:32 2023","interfaces":[{"interface":"back","average":{"1min":7.1760000000000002,"5min":2.0089999999999999,"15min":1.7549999999999999},"min":{"1min":0.27500000000000002,"5min":0.25900000000000001,"15min":0.23899999999999999},"max":{"1min":55.118000000000002,"5min":55.118000000000002,"15min":170.07599999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":6.8540000000000001,"5min":2.0499999999999998,"15min":1.345},"min":{"1min":0.33400000000000002,"5min":0.29799999999999999,"15min":0.25},"max":{"1min":55.365000000000002,"5min":55.365000000000002,"15min":63.600000000000001},"last":0.41899999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:58:05 2023","interfaces":[{"interface":"back","average":{"1min":1.7529999999999999,"5min":1.1060000000000001,"15min":0.95299999999999996},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28199999999999997},"max":{"1min":8.6280000000000001,"5min":8.6280000000000001,"15min":11.372999999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":1.391,"5min":0.93700000000000006,"15min":0.90100000000000002},"min":{"1min":0.373,"5min":0.33200000000000002,"15min":0.26800000000000002},"max":{"1min":6.1710000000000003,"5min":6.1710000000000003,"15min":6.5709999999999997},"last":0.52400000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:58:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5860000000000001,"5min":1.3380000000000001,"15min":0.995},"min":{"1min":0.35099999999999998,"5min":0.30599999999999999,"15min":0.28999999999999998},"max":{"1min":5.3380000000000001,"5min":13.414,"15min":13.414},"last":0.48799999999999999},{"interface":"front","average":{"1min":2.101,"5min":1.242,"15min":0.97799999999999998},"min":{"1min":0.33000000000000002,"5min":0.313,"15min":0.29799999999999999},"max":{"1min":8.5570000000000004,"5min":13.438000000000001,"15min":13.438000000000001},"last":0.53700000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:58:05 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":0.96199999999999997,"15min":0.91100000000000003},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.29399999999999998},"max":{"1min":3.2999999999999998,"5min":8.6920000000000002,"15min":11.59},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.48,"5min":1.113,"15min":0.90900000000000003},"min":{"1min":0.39800000000000002,"5min":0.37,"15min":0.34000000000000002},"max":{"1min":4.4850000000000003,"5min":8.6140000000000008,"15min":8.6140000000000008},"last":0.46899999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:58:05 2023","interfaces":[{"interface":"back","average":{"1min":4.4020000000000001,"5min":1.8620000000000001,"15min":1.274},"min":{"1min":0.44700000000000001,"5min":0.36599999999999999,"15min":0.35899999999999999},"max":{"1min":55.052,"5min":55.052,"15min":55.052},"last":0.505},{"interface":"front","average":{"1min":2.0110000000000001,"5min":1.252,"15min":1.093},"min":{"1min":0.438,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.6010000000000009,"5min":8.6010000000000009,"15min":40.113},"last":0.62}]}]},{"osd":3,"up_from":99,"seq":425201762411,"num_pgs":77,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":980988,"kb_used_data":729460,"kb_used_omap":7,"kb_used_meta":251512,"kb_avail":92764164,"statfs":{"total":95995035648,"available":94990503936,"internally_reserved":0,"allocated":746967040,"data_stored":745351748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":257548904},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:57:06 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":1.0329999999999999,"15min":1.3759999999999999},"min":{"1min":0.186,"5min":0.186,"15min":0.186},"max":{"1min":2.3959999999999999,"5min":15.536,"15min":15.536},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.95699999999999996,"5min":1.27,"15min":1.4850000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":7.891,"5min":14.707000000000001,"15min":14.707000000000001},"last":0.251}]},{"osd":1,"last update":"Mon Oct 9 05:57:38 2023","interfaces":[{"interface":"back","average":{"1min":1.4019999999999999,"5min":0.99399999999999999,"15min":1.105},"min":{"1min":0.217,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":14.568,"5min":14.568,"15min":127.226},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.82099999999999995,"15min":1.089},"min":{"1min":0.28599999999999998,"5min":0.24399999999999999,"15min":0.21099999999999999},"max":{"1min":2.5230000000000001,"5min":9.4239999999999995,"15min":127.468},"last":0.32600000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:57:38 2023","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":0.61499999999999999,"15min":0.75800000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":2.677,"5min":3.177,"15min":62.938000000000002},"last":69.421999999999997},{"interface":"front","average":{"1min":1.268,"5min":1.0169999999999999,"15min":1.004},"min":{"1min":0.30499999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":9.1039999999999992,"5min":9.8369999999999997,"15min":62.43},"last":69.355000000000004}]},{"osd":4,"last update":"Mon Oct 9 05:57:38 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":2.1829999999999998,"15min":1.2390000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.28199999999999997},"max":{"1min":13.154999999999999,"5min":142.994,"15min":142.994},"last":0.50900000000000001},{"interface":"front","average":{"1min":2.2570000000000001,"5min":2.4380000000000002,"15min":1.2869999999999999},"min":{"1min":0.33600000000000002,"5min":0.312,"15min":0.311},"max":{"1min":14.455,"5min":143.411,"15min":143.411},"last":0.40300000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:57:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6519999999999999,"5min":1.0309999999999999,"15min":0.82199999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.33800000000000002},"max":{"1min":14.404,"5min":14.404,"15min":14.404},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.8859999999999999,"5min":0.93000000000000005,"15min":0.79200000000000004},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":18.013999999999999,"5min":18.013999999999999,"15min":18.013999999999999},"last":0.52900000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:57:38 2023","interfaces":[{"interface":"back","average":{"1min":3.8929999999999998,"5min":1.544,"15min":1.0409999999999999},"min":{"1min":0.44,"5min":0.374,"15min":0.30499999999999999},"max":{"1min":44.079000000000001,"5min":44.079000000000001,"15min":44.079000000000001},"last":0.47199999999999998},{"interface":"front","average":{"1min":3.5619999999999998,"5min":1.3140000000000001,"15min":0.99199999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":44.094000000000001,"5min":44.094000000000001,"15min":44.094000000000001},"last":0.432}]},{"osd":7,"last update":"Mon Oct 9 05:57:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0539999999999998,"5min":0.98599999999999999,"15min":0.85799999999999998},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":10.295,"5min":10.295,"15min":10.295},"last":0.49099999999999999},{"interface":"front","average":{"1min":1.0620000000000001,"5min":0.91900000000000004,"15min":0.84599999999999997},"min":{"1min":0.436,"5min":0.41399999999999998,"15min":0.40600000000000003},"max":{"1min":3.9689999999999999,"5min":9.3879999999999999,"15min":9.3879999999999999},"last":0.63600000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019131,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":433240,"kb_used_data":123664,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93311912,"statfs":{"total":95995035648,"available":95551397888,"internally_reserved":0,"allocated":126631936,"data_stored":125637966,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:57:04 2023","interfaces":[{"interface":"back","average":{"1min":3.1280000000000001,"5min":1.4339999999999999,"15min":0.999},"min":{"1min":0.249,"5min":0.222,"15min":0.222},"max":{"1min":19.489000000000001,"5min":19.489000000000001,"15min":19.489000000000001},"last":0.29299999999999998},{"interface":"front","average":{"1min":3.2010000000000001,"5min":1.6020000000000001,"15min":1.556},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":19.757000000000001,"5min":19.757000000000001,"15min":19.757000000000001},"last":0.443}]},{"osd":1,"last update":"Mon Oct 9 05:57:33 2023","interfaces":[{"interface":"back","average":{"1min":4.7229999999999999,"5min":1.5309999999999999,"15min":1.03},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":77.951999999999998,"5min":77.951999999999998,"15min":77.951999999999998},"last":0.35899999999999999},{"interface":"front","average":{"1min":5.4809999999999999,"5min":1.7090000000000001,"15min":1.0269999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":77.968000000000004,"5min":77.968000000000004,"15min":77.968000000000004},"last":0.52300000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:57:04 2023","interfaces":[{"interface":"back","average":{"1min":1.3680000000000001,"5min":1.0509999999999999,"15min":0.91200000000000003},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.27400000000000002},"max":{"1min":6.2240000000000002,"5min":6.4939999999999998,"15min":33.262},"last":0.33500000000000002},{"interface":"front","average":{"1min":1.573,"5min":1.117,"15min":1.012},"min":{"1min":0.38900000000000001,"5min":0.35599999999999998,"15min":0.313},"max":{"1min":6.2469999999999999,"5min":6.5510000000000002,"15min":33.240000000000002},"last":0.39600000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:57:33 2023","interfaces":[{"interface":"back","average":{"1min":2.847,"5min":1.353,"15min":1.109},"min":{"1min":0.311,"5min":0.30599999999999999,"15min":0.29299999999999998},"max":{"1min":12.554,"5min":12.554,"15min":40.531999999999996},"last":0.53700000000000003},{"interface":"front","average":{"1min":2.8610000000000002,"5min":1.2749999999999999,"15min":1.127},"min":{"1min":0.377,"5min":0.31,"15min":0.31},"max":{"1min":12.403,"5min":12.403,"15min":39.905999999999999},"last":0.41399999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:57:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":0.61899999999999999,"15min":0.629},"min":{"1min":0.30299999999999999,"5min":0.27500000000000002,"15min":0.25900000000000001},"max":{"1min":4.9349999999999996,"5min":4.9349999999999996,"15min":10.012},"last":0.31},{"interface":"front","average":{"1min":1.5489999999999999,"5min":0.72599999999999998,"15min":0.73399999999999999},"min":{"1min":0.32000000000000001,"5min":0.254,"15min":0.254},"max":{"1min":12.121,"5min":12.121,"15min":12.489000000000001},"last":0.377}]},{"osd":6,"last update":"Mon Oct 9 05:57:45 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":0.69999999999999996,"15min":0.71999999999999997},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.4379999999999997,"5min":5.4379999999999997,"15min":11.234999999999999},"last":0.505},{"interface":"front","average":{"1min":2.1230000000000002,"5min":0.85799999999999998,"15min":0.71499999999999997},"min":{"1min":0.39200000000000002,"5min":0.30599999999999999,"15min":0.30299999999999999},"max":{"1min":8.8610000000000007,"5min":8.8610000000000007,"15min":9.923},"last":0.34799999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:57:08 2023","interfaces":[{"interface":"back","average":{"1min":1.53,"5min":0.94999999999999996,"15min":0.83499999999999996},"min":{"1min":0.38200000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.0570000000000004,"5min":10.606,"15min":10.606},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.167,"5min":0.76000000000000001,"15min":0.71799999999999997},"min":{"1min":0.36899999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.524,"5min":5.665,"15min":8.3819999999999997},"last":0.47499999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757494,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":381836,"kb_used_data":72260,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93363316,"statfs":{"total":95995035648,"available":95604035584,"internally_reserved":0,"allocated":73994240,"data_stored":73008846,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:57:10 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":0.65300000000000002,"15min":0.68899999999999995},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":7.0359999999999996,"5min":7.0359999999999996,"15min":7.0359999999999996},"last":0.28999999999999998},{"interface":"front","average":{"1min":1.242,"5min":0.82699999999999996,"15min":0.80700000000000005},"min":{"1min":0.28599999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.0540000000000003,"5min":7.0540000000000003,"15min":7.0540000000000003},"last":0.39800000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:57:33 2023","interfaces":[{"interface":"back","average":{"1min":1.3200000000000001,"5min":2.4900000000000002,"15min":1.6459999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.8410000000000002,"5min":99.605999999999995,"15min":114.247},"last":1.47},{"interface":"front","average":{"1min":1.0089999999999999,"5min":2.1579999999999999,"15min":1.5269999999999999},"min":{"1min":0.28399999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.7130000000000001,"5min":99.671999999999997,"15min":114.678},"last":0.755}]},{"osd":2,"last update":"Mon Oct 9 05:57:59 2023","interfaces":[{"interface":"back","average":{"1min":12.375999999999999,"5min":3.2130000000000001,"15min":2.0150000000000001},"min":{"1min":0.28499999999999998,"5min":0.28399999999999997,"15min":0.25},"max":{"1min":217.06399999999999,"5min":217.06399999999999,"15min":217.06399999999999},"last":0.435},{"interface":"front","average":{"1min":11.965999999999999,"5min":3.0870000000000002,"15min":1.9710000000000001},"min":{"1min":0.308,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":217.018,"5min":217.018,"15min":217.018},"last":0.51500000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:57:33 2023","interfaces":[{"interface":"back","average":{"1min":5.4450000000000003,"5min":3.8919999999999999,"15min":3.714},"min":{"1min":0.35799999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":60,"5min":100.774,"15min":272.81299999999999},"last":1.6100000000000001},{"interface":"front","average":{"1min":5.9500000000000002,"5min":4.274,"15min":3.9529999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":59.817,"5min":100.83499999999999,"15min":273.00999999999999},"last":0.36099999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:57:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":0.58599999999999997,"15min":0.58999999999999997},"min":{"1min":0.27000000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":3.1080000000000001,"5min":3.1080000000000001,"15min":6.2050000000000001},"last":0.375},{"interface":"front","average":{"1min":0.76400000000000001,"5min":0.57399999999999995,"15min":0.59699999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.26000000000000001},"max":{"1min":2.444,"5min":2.484,"15min":13.050000000000001},"last":0.41599999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:57:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":0.72599999999999998,"15min":0.65500000000000003},"min":{"1min":0.33600000000000002,"5min":0.30199999999999999,"15min":0.26600000000000001},"max":{"1min":10.94,"5min":10.94,"15min":10.94},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.72399999999999998,"15min":0.71799999999999997},"min":{"1min":0.33100000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.5069999999999997,"5min":5.5069999999999997,"15min":5.9770000000000003},"last":0.62}]},{"osd":7,"last update":"Mon Oct 9 05:57:15 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":0.91600000000000004,"15min":0.78300000000000003},"min":{"1min":0.33100000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.5800000000000001,"5min":9.2739999999999991,"15min":9.2739999999999991},"last":0.58499999999999996},{"interface":"front","average":{"1min":1.1679999999999999,"5min":0.68000000000000005,"15min":0.65200000000000002},"min":{"1min":0.40100000000000002,"5min":0.32100000000000001,"15min":0.32000000000000001},"max":{"1min":2.7530000000000001,"5min":2.7530000000000001,"15min":5.032},"last":0.34499999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1474,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":211353600,"data_stored":211353600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":696451072,"data_stored":696451072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":742719488,"data_stored":742719488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":123666432,"data_stored":123666432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":70451200,"data_stored":70451200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":67305472,"data_stored":67305472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2424832,"data_stored":2424832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:58:11.187 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:58:11.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:11 smithi064 ceph-mon[212750]: failed to encode map e274 with expected crc 2023-10-09T05:58:11.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:11 smithi064 ceph-mon[212750]: failed to encode map e274 with expected crc 2023-10-09T05:58:11.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:11 smithi064 ceph-mon[212750]: failed to encode map e274 with expected crc 2023-10-09T05:58:11.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:11 smithi064 ceph-mon[212750]: failed to encode map e274 with expected crc 2023-10-09T05:58:11.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:11 smithi064 ceph-mon[212750]: osdmap e275: 8 total, 8 up, 7 in 2023-10-09T05:58:11.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:11 smithi064 ceph-mon[220044]: failed to encode map e274 with expected crc 2023-10-09T05:58:11.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:11 smithi064 ceph-mon[220044]: failed to encode map e274 with expected crc 2023-10-09T05:58:11.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:11 smithi064 ceph-mon[220044]: failed to encode map e274 with expected crc 2023-10-09T05:58:11.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:11 smithi064 ceph-mon[220044]: failed to encode map e274 with expected crc 2023-10-09T05:58:11.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:11 smithi064 ceph-mon[220044]: osdmap e275: 8 total, 8 up, 7 in 2023-10-09T05:58:11.548 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:11.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:11.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:11.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:11 smithi156 ceph-mon[195101]: failed to encode map e274 with expected crc 2023-10-09T05:58:11.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:11 smithi156 ceph-mon[195101]: failed to encode map e274 with expected crc 2023-10-09T05:58:11.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:11 smithi156 ceph-mon[195101]: failed to encode map e274 with expected crc 2023-10-09T05:58:11.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:11 smithi156 ceph-mon[195101]: failed to encode map e274 with expected crc 2023-10-09T05:58:11.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:11 smithi156 ceph-mon[195101]: osdmap e275: 8 total, 8 up, 7 in 2023-10-09T05:58:11.653 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:11.697 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:11.742 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:11.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:11.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:11.782 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:11.825 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:58:11.825 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T05:58:11.825 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T05:58:11.825 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T05:58:11.826 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:58:11.849 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:11.875 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:11.919 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:11.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:11.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:12.024 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:12.056 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:12.156 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:12.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:12.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:12.261 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:12.293 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:12.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:12 smithi064 ceph-mon[212750]: pgmap v623: 139 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 remapped+peering, 115 active+clean, 11 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 33 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 3 op/s; 17154/1614 objects degraded (1062.825%); 152/1614 objects misplaced (9.418%); 0 B/s, 1.10k objects/s recovering 2023-10-09T05:58:12.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:12 smithi064 ceph-mon[212750]: failed to encode map e275 with expected crc 2023-10-09T05:58:12.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:12 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 17154/1614 objects degraded (1062.825%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:58:12.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:12 smithi064 ceph-mon[212750]: osdmap e276: 8 total, 8 up, 7 in 2023-10-09T05:58:12.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:12 smithi064 ceph-mon[220044]: pgmap v623: 139 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 remapped+peering, 115 active+clean, 11 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 33 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 3 op/s; 17154/1614 objects degraded (1062.825%); 152/1614 objects misplaced (9.418%); 0 B/s, 1.10k objects/s recovering 2023-10-09T05:58:12.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:12 smithi064 ceph-mon[220044]: failed to encode map e275 with expected crc 2023-10-09T05:58:12.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:12 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 17154/1614 objects degraded (1062.825%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:58:12.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:12 smithi064 ceph-mon[220044]: osdmap e276: 8 total, 8 up, 7 in 2023-10-09T05:58:12.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:12 smithi156 ceph-mon[195101]: pgmap v623: 139 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 remapped+peering, 115 active+clean, 11 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 33 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 3 op/s; 17154/1614 objects degraded (1062.825%); 152/1614 objects misplaced (9.418%); 0 B/s, 1.10k objects/s recovering 2023-10-09T05:58:12.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:12 smithi156 ceph-mon[195101]: failed to encode map e275 with expected crc 2023-10-09T05:58:12.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:12 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 17154/1614 objects degraded (1062.825%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T05:58:12.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:12 smithi156 ceph-mon[195101]: osdmap e276: 8 total, 8 up, 7 in 2023-10-09T05:58:13.249 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T05:58:13.532 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:13 smithi064 ceph-mon[220044]: failed to encode map e276 with expected crc 2023-10-09T05:58:13.532 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:13 smithi064 ceph-mon[220044]: failed to encode map e276 with expected crc 2023-10-09T05:58:13.532 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:13 smithi064 ceph-mon[220044]: failed to encode map e276 with expected crc 2023-10-09T05:58:13.532 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:13 smithi064 ceph-mon[220044]: failed to encode map e276 with expected crc 2023-10-09T05:58:13.532 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:13 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2312221958' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:58:13.533 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:13 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:58:13.533 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:13 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:58:13.533 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:13 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:58:13.533 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:13 smithi064 ceph-mon[220044]: osdmap e277: 8 total, 8 up, 7 in 2023-10-09T05:58:13.533 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:13 smithi064 ceph-mon[212750]: failed to encode map e276 with expected crc 2023-10-09T05:58:13.533 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:13 smithi064 ceph-mon[212750]: failed to encode map e276 with expected crc 2023-10-09T05:58:13.533 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:13 smithi064 ceph-mon[212750]: failed to encode map e276 with expected crc 2023-10-09T05:58:13.533 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:13 smithi064 ceph-mon[212750]: failed to encode map e276 with expected crc 2023-10-09T05:58:13.533 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:13 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2312221958' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:58:13.534 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:13 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:58:13.534 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:13 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:58:13.534 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:13 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:58:13.534 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:13 smithi064 ceph-mon[212750]: osdmap e277: 8 total, 8 up, 7 in 2023-10-09T05:58:13.601 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:13.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:13.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:13.613 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:58:13.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:13 smithi156 ceph-mon[195101]: failed to encode map e276 with expected crc 2023-10-09T05:58:13.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:13 smithi156 ceph-mon[195101]: failed to encode map e276 with expected crc 2023-10-09T05:58:13.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:13 smithi156 ceph-mon[195101]: failed to encode map e276 with expected crc 2023-10-09T05:58:13.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:13 smithi156 ceph-mon[195101]: failed to encode map e276 with expected crc 2023-10-09T05:58:13.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:13 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2312221958' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:58:13.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:13 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:58:13.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:13 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:58:13.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:13 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:58:13.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:13 smithi156 ceph-mon[195101]: osdmap e277: 8 total, 8 up, 7 in 2023-10-09T05:58:13.702 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:58:13.704 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:13.752 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:14.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:14 smithi064 ceph-mon[212750]: pgmap v626: 139 pgs: 1 peering, 1 active+recovering+undersized+remapped, 116 active+clean, 11 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 33 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 17154/1614 objects degraded (1062.825%); 152/1614 objects misplaced (9.418%); 0 B/s, 781 objects/s recovering 2023-10-09T05:58:14.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:14 smithi064 ceph-mon[212750]: failed to encode map e277 with expected crc 2023-10-09T05:58:14.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:14 smithi064 ceph-mon[212750]: 6.20 continuing backfill to osd.2 from (0'0,269'8034] MIN to 269'8034 2023-10-09T05:58:14.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:14 smithi064 ceph-mon[212750]: failed to encode map e277 with expected crc 2023-10-09T05:58:14.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:14 smithi064 ceph-mon[212750]: failed to encode map e277 with expected crc 2023-10-09T05:58:14.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:58:14.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:58:14.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:14.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:14.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:14.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:58:14.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:14 smithi064 ceph-mon[220044]: pgmap v626: 139 pgs: 1 peering, 1 active+recovering+undersized+remapped, 116 active+clean, 11 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 33 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 17154/1614 objects degraded (1062.825%); 152/1614 objects misplaced (9.418%); 0 B/s, 781 objects/s recovering 2023-10-09T05:58:14.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:14 smithi064 ceph-mon[220044]: failed to encode map e277 with expected crc 2023-10-09T05:58:14.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:14 smithi064 ceph-mon[220044]: 6.20 continuing backfill to osd.2 from (0'0,269'8034] MIN to 269'8034 2023-10-09T05:58:14.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:14 smithi064 ceph-mon[220044]: failed to encode map e277 with expected crc 2023-10-09T05:58:14.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:14 smithi064 ceph-mon[220044]: failed to encode map e277 with expected crc 2023-10-09T05:58:14.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:58:14.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:58:14.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:14.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:14.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:14.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:58:14.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:14 smithi156 ceph-mon[195101]: pgmap v626: 139 pgs: 1 peering, 1 active+recovering+undersized+remapped, 116 active+clean, 11 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 33 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 17154/1614 objects degraded (1062.825%); 152/1614 objects misplaced (9.418%); 0 B/s, 781 objects/s recovering 2023-10-09T05:58:14.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:14 smithi156 ceph-mon[195101]: failed to encode map e277 with expected crc 2023-10-09T05:58:14.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:14 smithi156 ceph-mon[195101]: 6.20 continuing backfill to osd.2 from (0'0,269'8034] MIN to 269'8034 2023-10-09T05:58:14.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:14 smithi156 ceph-mon[195101]: failed to encode map e277 with expected crc 2023-10-09T05:58:14.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:14 smithi156 ceph-mon[195101]: failed to encode map e277 with expected crc 2023-10-09T05:58:14.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:58:14.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:58:14.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:14.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:14.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:14.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:58:14.669 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:14.769 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:14.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:14.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:14.874 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:14.909 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:15.167 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:15.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:15.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:15.190 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:15.210 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:15.273 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:15.313 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:15.369 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:15.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:15.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:15.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:58:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:58:15] "GET /metrics HTTP/1.1" 200 36423 "" "Prometheus/2.43.0" 2023-10-09T05:58:15.471 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:15.497 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:15.555 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:15.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:15.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:15.663 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:15.689 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:15.735 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:15.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:15.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:15.842 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:15.869 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:15.914 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:15.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:15.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:16.019 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:16.048 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:16.347 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_0' removed 2023-10-09T05:58:16.427 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:16 smithi064 ceph-mon[212750]: pgmap v628: 139 pgs: 1 peering, 1 active+recovering+undersized+remapped, 116 active+clean, 11 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 33 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 17154/1614 objects degraded (1062.825%); 152/1614 objects misplaced (9.418%); 0 B/s, 338 objects/s recovering 2023-10-09T05:58:16.427 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:16 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3449394980' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:58:16.427 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:16 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:58:16.427 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:16 smithi064 ceph-mon[220044]: pgmap v628: 139 pgs: 1 peering, 1 active+recovering+undersized+remapped, 116 active+clean, 11 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 33 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 17154/1614 objects degraded (1062.825%); 152/1614 objects misplaced (9.418%); 0 B/s, 338 objects/s recovering 2023-10-09T05:58:16.427 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:16 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3449394980' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:58:16.427 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:16 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:58:16.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:16 smithi156 ceph-mon[195101]: pgmap v628: 139 pgs: 1 peering, 1 active+recovering+undersized+remapped, 116 active+clean, 11 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 33 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 17154/1614 objects degraded (1062.825%); 152/1614 objects misplaced (9.418%); 0 B/s, 338 objects/s recovering 2023-10-09T05:58:16.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:16 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3449394980' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:58:16.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:16 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:58:16.666 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:58:16.808 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:16.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:16.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:16.912 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:16.938 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:16.980 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:16.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:16.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:17.087 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:17.113 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:17.155 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:17.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:17.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:17.261 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:17.291 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:17.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:17 smithi156 ceph-mon[195101]: failed to encode map e278 with expected crc 2023-10-09T05:58:17.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:17 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:58:17.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:17 smithi156 ceph-mon[195101]: osdmap e278: 8 total, 8 up, 7 in 2023-10-09T05:58:17.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:17 smithi156 ceph-mon[195101]: failed to encode map e278 with expected crc 2023-10-09T05:58:17.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:17 smithi156 ceph-mon[195101]: failed to encode map e278 with expected crc 2023-10-09T05:58:17.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:17 smithi156 ceph-mon[195101]: failed to encode map e278 with expected crc 2023-10-09T05:58:17.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:17 smithi156 ceph-mon[195101]: pgmap v630: 93 pgs: 93 active+clean; 582 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:17.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:17 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T05:58:17.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:17 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T05:58:17.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:17 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 17154/1614 objects degraded (1062.825%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:17.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:17 smithi064 ceph-mon[220044]: failed to encode map e278 with expected crc 2023-10-09T05:58:17.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:17 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:58:17.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:17 smithi064 ceph-mon[220044]: osdmap e278: 8 total, 8 up, 7 in 2023-10-09T05:58:17.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:17 smithi064 ceph-mon[220044]: failed to encode map e278 with expected crc 2023-10-09T05:58:17.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:17 smithi064 ceph-mon[220044]: failed to encode map e278 with expected crc 2023-10-09T05:58:17.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:17 smithi064 ceph-mon[220044]: failed to encode map e278 with expected crc 2023-10-09T05:58:17.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:17 smithi064 ceph-mon[220044]: pgmap v630: 93 pgs: 93 active+clean; 582 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:17.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:17 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T05:58:17.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:17 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T05:58:17.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:17 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 17154/1614 objects degraded (1062.825%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:17.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:17 smithi064 ceph-mon[212750]: failed to encode map e278 with expected crc 2023-10-09T05:58:17.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:17 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:58:17.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:17 smithi064 ceph-mon[212750]: osdmap e278: 8 total, 8 up, 7 in 2023-10-09T05:58:17.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:17 smithi064 ceph-mon[212750]: failed to encode map e278 with expected crc 2023-10-09T05:58:17.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:17 smithi064 ceph-mon[212750]: failed to encode map e278 with expected crc 2023-10-09T05:58:17.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:17 smithi064 ceph-mon[212750]: failed to encode map e278 with expected crc 2023-10-09T05:58:17.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:17 smithi064 ceph-mon[212750]: pgmap v630: 93 pgs: 93 active+clean; 582 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:17.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:17 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T05:58:17.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:17 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T05:58:17.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:17 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 17154/1614 objects degraded (1062.825%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:58:18.215 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T05:58:18.216 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T05:58:18.216 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-10-09T05:58:18.216 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-10-09T05:58:18.269 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:18.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:18.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:18.359 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:18.373 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:18.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[212750]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17154/1614 objects degraded (1062.825%), 10 pgs degraded, 2 pgs undersized) 2023-10-09T05:58:18.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[212750]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[212750]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[212750]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[212750]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[212750]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[212750]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[212750]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[212750]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T05:58:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[212750]: osdmap e279: 8 total, 8 up, 7 in 2023-10-09T05:58:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[212750]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[212750]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[212750]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[212750]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[220044]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17154/1614 objects degraded (1062.825%), 10 pgs degraded, 2 pgs undersized) 2023-10-09T05:58:18.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[220044]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[220044]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[220044]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[220044]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[220044]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[220044]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[220044]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[220044]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T05:58:18.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[220044]: osdmap e279: 8 total, 8 up, 7 in 2023-10-09T05:58:18.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[220044]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[220044]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[220044]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:18 smithi064 ceph-mon[220044]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.411 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:18.469 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:18.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:18.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:18.576 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:18.606 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:18.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:18 smithi156 ceph-mon[195101]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17154/1614 objects degraded (1062.825%), 10 pgs degraded, 2 pgs undersized) 2023-10-09T05:58:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:18 smithi156 ceph-mon[195101]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:18 smithi156 ceph-mon[195101]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:18 smithi156 ceph-mon[195101]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:18 smithi156 ceph-mon[195101]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:18 smithi156 ceph-mon[195101]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:18 smithi156 ceph-mon[195101]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:18 smithi156 ceph-mon[195101]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:18 smithi156 ceph-mon[195101]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:18 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T05:58:18.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:18 smithi156 ceph-mon[195101]: osdmap e279: 8 total, 8 up, 7 in 2023-10-09T05:58:18.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:18 smithi156 ceph-mon[195101]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:18 smithi156 ceph-mon[195101]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:18 smithi156 ceph-mon[195101]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:18 smithi156 ceph-mon[195101]: failed to encode map e279 with expected crc 2023-10-09T05:58:18.790 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:58:18.791 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T05:58:18.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T05:58:18.791 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T05:58:18.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.791 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82093 6.0 6:0c2a5c50:::benchmark_data_smithi064_284193_object82092:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T05:58:18.791 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:57.082946+0000", 2023-10-09T05:58:18.791 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 381.70499883500003, 2023-10-09T05:58:18.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.198079436, 2023-10-09T05:58:18.791 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:58:18.791 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:18.792 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:58:18.792 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:58:18.792 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:18.792 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82093 2023-10-09T05:58:18.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.792 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:58:18.792 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.792 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:58:18.792 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.082946+0000", 2023-10-09T05:58:18.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:58:18.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.082946+0000", 2023-10-09T05:58:18.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:58:18.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.082944+0000", 2023-10-09T05:58:18.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:58:18.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:58:18.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.082981+0000", 2023-10-09T05:58:18.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.684e-05 2023-10-09T05:58:18.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:58:18.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.082999+0000", 2023-10-09T05:58:18.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7934000000000001e-05 2023-10-09T05:58:18.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:58:18.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083010+0000", 2023-10-09T05:58:18.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.079e-05 2023-10-09T05:58:18.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.084587+0000", 2023-10-09T05:58:18.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001577216 2023-10-09T05:58:18.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:58:18.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.084598+0000", 2023-10-09T05:58:18.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0812000000000001e-05 2023-10-09T05:58:18.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.258113+0000", 2023-10-09T05:58:18.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1735149090000001 2023-10-09T05:58:18.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:58:18.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.259837+0000", 2023-10-09T05:58:18.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001724377 2023-10-09T05:58:18.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:58:18.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.259999+0000", 2023-10-09T05:58:18.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016160799999999999 2023-10-09T05:58:18.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.269960+0000", 2023-10-09T05:58:18.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00996177 2023-10-09T05:58:18.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:58:18.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.269967+0000", 2023-10-09T05:58:18.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.241e-06 2023-10-09T05:58:18.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.280428+0000", 2023-10-09T05:58:18.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010461869 2023-10-09T05:58:18.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:58:18.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.280484+0000", 2023-10-09T05:58:18.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5316999999999999e-05 2023-10-09T05:58:18.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:58:18.799 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.281025+0000", 2023-10-09T05:58:18.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00054158299999999999 2023-10-09T05:58:18.799 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.799 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:58:18.799 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.799 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.799 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82107 6.0 6:07b34c20:::benchmark_data_smithi064_284193_object82106:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T05:58:18.799 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:57.083024+0000", 2023-10-09T05:58:18.799 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 381.70492103100003, 2023-10-09T05:58:18.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.198456921, 2023-10-09T05:58:18.799 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:58:18.800 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:18.800 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:58:18.800 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:58:18.800 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:18.800 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82107 2023-10-09T05:58:18.800 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.800 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:58:18.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:58:18.800 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083024+0000", 2023-10-09T05:58:18.800 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.800 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:58:18.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083024+0000", 2023-10-09T05:58:18.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:58:18.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083023+0000", 2023-10-09T05:58:18.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:18.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:58:18.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083051+0000", 2023-10-09T05:58:18.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7501999999999999e-05 2023-10-09T05:58:18.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:58:18.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083063+0000", 2023-10-09T05:58:18.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2048999999999999e-05 2023-10-09T05:58:18.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:58:18.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083069+0000", 2023-10-09T05:58:18.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.319e-06 2023-10-09T05:58:18.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.803 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.803 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.084628+0000", 2023-10-09T05:58:18.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00155887 2023-10-09T05:58:18.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.803 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:58:18.803 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.084636+0000", 2023-10-09T05:58:18.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2830000000000001e-06 2023-10-09T05:58:18.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.263669+0000", 2023-10-09T05:58:18.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1790324080000001 2023-10-09T05:58:18.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:58:18.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.263844+0000", 2023-10-09T05:58:18.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000175141 2023-10-09T05:58:18.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:58:18.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.264042+0000", 2023-10-09T05:58:18.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019832300000000001 2023-10-09T05:58:18.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:58:18.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.269970+0000", 2023-10-09T05:58:18.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059283410000000002 2023-10-09T05:58:18.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.279927+0000", 2023-10-09T05:58:18.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0099563829999999992 2023-10-09T05:58:18.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.281062+0000", 2023-10-09T05:58:18.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001135472 2023-10-09T05:58:18.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:58:18.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.281111+0000", 2023-10-09T05:58:18.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9181999999999998e-05 2023-10-09T05:58:18.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:58:18.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.281481+0000", 2023-10-09T05:58:18.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000369219 2023-10-09T05:58:18.807 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.807 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:58:18.807 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.807 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82109 6.0 6:06570773:::benchmark_data_smithi064_284193_object82108:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T05:58:18.807 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:57.083077+0000", 2023-10-09T05:58:18.807 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 381.70486767599999, 2023-10-09T05:58:18.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1988545770000001, 2023-10-09T05:58:18.807 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:58:18.808 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:18.808 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:58:18.808 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:58:18.808 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:18.808 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82109 2023-10-09T05:58:18.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.808 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:58:18.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:58:18.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083077+0000", 2023-10-09T05:58:18.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:58:18.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083077+0000", 2023-10-09T05:58:18.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:58:18.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083076+0000", 2023-10-09T05:58:18.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:18.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:58:18.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083104+0000", 2023-10-09T05:58:18.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7718000000000001e-05 2023-10-09T05:58:18.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:58:18.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083124+0000", 2023-10-09T05:58:18.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9729999999999999e-05 2023-10-09T05:58:18.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:58:18.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083131+0000", 2023-10-09T05:58:18.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5749999999999997e-06 2023-10-09T05:58:18.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.084656+0000", 2023-10-09T05:58:18.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001525607 2023-10-09T05:58:18.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:58:18.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.084663+0000", 2023-10-09T05:58:18.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8809999999999998e-06 2023-10-09T05:58:18.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.266388+0000", 2023-10-09T05:58:18.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1817245459999999 2023-10-09T05:58:18.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:58:18.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.266542+0000", 2023-10-09T05:58:18.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000154074 2023-10-09T05:58:18.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:58:18.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.266721+0000", 2023-10-09T05:58:18.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000179588 2023-10-09T05:58:18.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:58:18.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.276095+0000", 2023-10-09T05:58:18.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0093742140000000005 2023-10-09T05:58:18.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.279969+0000", 2023-10-09T05:58:18.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038732670000000001 2023-10-09T05:58:18.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.281523+0000", 2023-10-09T05:58:18.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001554452 2023-10-09T05:58:18.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:58:18.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.281577+0000", 2023-10-09T05:58:18.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4104999999999998e-05 2023-10-09T05:58:18.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:58:18.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.281932+0000", 2023-10-09T05:58:18.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00035437499999999998 2023-10-09T05:58:18.815 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.815 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:58:18.815 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.815 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82116 6.9 6:95f8f18c:::benchmark_data_smithi064_284193_object82115:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T05:58:18.815 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:57.083135+0000", 2023-10-09T05:58:18.815 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 381.70480922600001, 2023-10-09T05:58:18.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1900297900000001, 2023-10-09T05:58:18.815 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:58:18.815 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:18.815 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:58:18.816 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:58:18.816 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:18.816 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82116 2023-10-09T05:58:18.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.816 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:58:18.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:58:18.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083135+0000", 2023-10-09T05:58:18.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:58:18.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083135+0000", 2023-10-09T05:58:18.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:58:18.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083135+0000", 2023-10-09T05:58:18.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:18.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:58:18.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083164+0000", 2023-10-09T05:58:18.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8952e-05 2023-10-09T05:58:18.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:58:18.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083175+0000", 2023-10-09T05:58:18.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0849999999999999e-05 2023-10-09T05:58:18.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:58:18.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083181+0000", 2023-10-09T05:58:18.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9290000000000003e-06 2023-10-09T05:58:18.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083487+0000", 2023-10-09T05:58:18.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00030613999999999999 2023-10-09T05:58:18.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:58:18.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083499+0000", 2023-10-09T05:58:18.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2415000000000001e-05 2023-10-09T05:58:18.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.256999+0000", 2023-10-09T05:58:18.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.173500196 2023-10-09T05:58:18.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:58:18.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.258968+0000", 2023-10-09T05:58:18.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001968923 2023-10-09T05:58:18.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:58:18.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.259175+0000", 2023-10-09T05:58:18.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020691000000000001 2023-10-09T05:58:18.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:58:18.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.272115+0000", 2023-10-09T05:58:18.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012939625 2023-10-09T05:58:18.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.272249+0000", 2023-10-09T05:58:18.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013408800000000001 2023-10-09T05:58:18.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.272479+0000", 2023-10-09T05:58:18.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000230349 2023-10-09T05:58:18.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:58:18.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.272538+0000", 2023-10-09T05:58:18.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8999999999999998e-05 2023-10-09T05:58:18.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:58:18.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.273165+0000", 2023-10-09T05:58:18.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00062689199999999999 2023-10-09T05:58:18.822 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.822 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:58:18.823 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.823 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82117 6.9 6:9031756d:::benchmark_data_smithi064_284193_object82116:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T05:58:18.823 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:57.083192+0000", 2023-10-09T05:58:18.823 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 381.704752386, 2023-10-09T05:58:18.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1976068609999999, 2023-10-09T05:58:18.823 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:58:18.823 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:18.823 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:58:18.823 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:58:18.823 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:18.824 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82117 2023-10-09T05:58:18.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.824 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:58:18.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:58:18.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083192+0000", 2023-10-09T05:58:18.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:58:18.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083192+0000", 2023-10-09T05:58:18.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:58:18.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083192+0000", 2023-10-09T05:58:18.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:18.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:58:18.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083213+0000", 2023-10-09T05:58:18.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0933000000000001e-05 2023-10-09T05:58:18.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:58:18.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083223+0000", 2023-10-09T05:58:18.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0433e-05 2023-10-09T05:58:18.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:58:18.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083229+0000", 2023-10-09T05:58:18.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6910000000000002e-06 2023-10-09T05:58:18.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083526+0000", 2023-10-09T05:58:18.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029717799999999999 2023-10-09T05:58:18.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:58:18.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083535+0000", 2023-10-09T05:58:18.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5150000000000006e-06 2023-10-09T05:58:18.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.262719+0000", 2023-10-09T05:58:18.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.179183981 2023-10-09T05:58:18.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:58:18.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.262876+0000", 2023-10-09T05:58:18.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000157629 2023-10-09T05:58:18.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:58:18.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.263044+0000", 2023-10-09T05:58:18.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000167622 2023-10-09T05:58:18.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:58:18.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.272122+0000", 2023-10-09T05:58:18.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0090786059999999995 2023-10-09T05:58:18.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.279609+0000", 2023-10-09T05:58:18.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0074866949999999998 2023-10-09T05:58:18.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.280065+0000", 2023-10-09T05:58:18.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00045626899999999998 2023-10-09T05:58:18.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:58:18.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.280133+0000", 2023-10-09T05:58:18.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7339e-05 2023-10-09T05:58:18.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:58:18.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.280799+0000", 2023-10-09T05:58:18.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00066644000000000002 2023-10-09T05:58:18.830 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.830 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:58:18.831 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.831 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82118 6.9 6:91416b55:::benchmark_data_smithi064_284193_object82117:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T05:58:18.831 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:57.083233+0000", 2023-10-09T05:58:18.831 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 381.70471123099998, 2023-10-09T05:58:18.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.198061628, 2023-10-09T05:58:18.831 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:58:18.831 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:18.831 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:58:18.831 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:58:18.831 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:18.832 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82118 2023-10-09T05:58:18.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.832 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:58:18.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:58:18.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083233+0000", 2023-10-09T05:58:18.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:58:18.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083233+0000", 2023-10-09T05:58:18.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:58:18.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083233+0000", 2023-10-09T05:58:18.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:18.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:58:18.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083254+0000", 2023-10-09T05:58:18.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1355000000000001e-05 2023-10-09T05:58:18.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:58:18.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083264+0000", 2023-10-09T05:58:18.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5349999999999993e-06 2023-10-09T05:58:18.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:58:18.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083269+0000", 2023-10-09T05:58:18.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4770000000000002e-06 2023-10-09T05:58:18.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083556+0000", 2023-10-09T05:58:18.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00028701399999999999 2023-10-09T05:58:18.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:58:18.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083565+0000", 2023-10-09T05:58:18.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2840000000000006e-06 2023-10-09T05:58:18.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.265497+0000", 2023-10-09T05:58:18.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.181932202 2023-10-09T05:58:18.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:58:18.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.265684+0000", 2023-10-09T05:58:18.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000187033 2023-10-09T05:58:18.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:58:18.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.265875+0000", 2023-10-09T05:58:18.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000191557 2023-10-09T05:58:18.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:58:18.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.272209+0000", 2023-10-09T05:58:18.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063339520000000003 2023-10-09T05:58:18.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.279746+0000", 2023-10-09T05:58:18.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0075368450000000003 2023-10-09T05:58:18.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.280854+0000", 2023-10-09T05:58:18.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001107292 2023-10-09T05:58:18.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:58:18.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.280908+0000", 2023-10-09T05:58:18.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4079999999999997e-05 2023-10-09T05:58:18.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:58:18.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.281295+0000", 2023-10-09T05:58:18.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00038745199999999999 2023-10-09T05:58:18.838 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.838 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:58:18.839 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.839 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82125 6.0 6:091c7b5b:::benchmark_data_smithi064_284193_object82124:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T05:58:18.839 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:57.083274+0000", 2023-10-09T05:58:18.839 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 381.70467051000003, 2023-10-09T05:58:18.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1991231920000001, 2023-10-09T05:58:18.839 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:58:18.839 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:18.839 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:58:18.839 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:58:18.839 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:18.840 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82125 2023-10-09T05:58:18.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.840 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:58:18.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:58:18.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083274+0000", 2023-10-09T05:58:18.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:58:18.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083274+0000", 2023-10-09T05:58:18.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:58:18.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083274+0000", 2023-10-09T05:58:18.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:18.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:58:18.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083294+0000", 2023-10-09T05:58:18.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0389000000000001e-05 2023-10-09T05:58:18.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:58:18.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083304+0000", 2023-10-09T05:58:18.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0093e-05 2023-10-09T05:58:18.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:58:18.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083311+0000", 2023-10-09T05:58:18.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3409999999999999e-06 2023-10-09T05:58:18.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.084682+0000", 2023-10-09T05:58:18.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013715000000000001 2023-10-09T05:58:18.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:58:18.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.084689+0000", 2023-10-09T05:58:18.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7959999999999998e-06 2023-10-09T05:58:18.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.269986+0000", 2023-10-09T05:58:18.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.185297439 2023-10-09T05:58:18.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:58:18.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.270081+0000", 2023-10-09T05:58:18.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4499000000000004e-05 2023-10-09T05:58:18.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:58:18.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.270180+0000", 2023-10-09T05:58:18.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9643999999999997e-05 2023-10-09T05:58:18.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:58:18.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.276106+0000", 2023-10-09T05:58:18.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059251010000000003 2023-10-09T05:58:18.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.280388+0000", 2023-10-09T05:58:18.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042823109999999996 2023-10-09T05:58:18.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.282002+0000", 2023-10-09T05:58:18.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016145580000000001 2023-10-09T05:58:18.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:58:18.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.282053+0000", 2023-10-09T05:58:18.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0643000000000003e-05 2023-10-09T05:58:18.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:58:18.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.282397+0000", 2023-10-09T05:58:18.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00034435800000000001 2023-10-09T05:58:18.846 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.846 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:58:18.847 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.847 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82137 6.0 6:091ee809:::benchmark_data_smithi064_284193_object82136:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T05:58:18.847 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:57.083316+0000", 2023-10-09T05:58:18.847 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 381.70462872000002, 2023-10-09T05:58:18.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2159187149999999, 2023-10-09T05:58:18.847 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:58:18.847 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:18.847 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:58:18.847 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:58:18.847 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:18.848 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82137 2023-10-09T05:58:18.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.848 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:58:18.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:58:18.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083316+0000", 2023-10-09T05:58:18.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:58:18.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083316+0000", 2023-10-09T05:58:18.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:58:18.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083315+0000", 2023-10-09T05:58:18.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:18.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:58:18.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083339+0000", 2023-10-09T05:58:18.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3604999999999998e-05 2023-10-09T05:58:18.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:58:18.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083351+0000", 2023-10-09T05:58:18.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1593999999999999e-05 2023-10-09T05:58:18.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:58:18.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083357+0000", 2023-10-09T05:58:18.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1249999999999998e-06 2023-10-09T05:58:18.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.084707+0000", 2023-10-09T05:58:18.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001350724 2023-10-09T05:58:18.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:58:18.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.084714+0000", 2023-10-09T05:58:18.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.652e-06 2023-10-09T05:58:18.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.272703+0000", 2023-10-09T05:58:18.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1879891309999999 2023-10-09T05:58:18.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:58:18.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.272854+0000", 2023-10-09T05:58:18.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015069899999999999 2023-10-09T05:58:18.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:58:18.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.273034+0000", 2023-10-09T05:58:18.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000179865 2023-10-09T05:58:18.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:58:18.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.276191+0000", 2023-10-09T05:58:18.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031575280000000002 2023-10-09T05:58:18.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.281969+0000", 2023-10-09T05:58:18.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0057776440000000002 2023-10-09T05:58:18.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.298462+0000", 2023-10-09T05:58:18.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016493206999999999 2023-10-09T05:58:18.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:58:18.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.298522+0000", 2023-10-09T05:58:18.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9956e-05 2023-10-09T05:58:18.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:58:18.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.299235+0000", 2023-10-09T05:58:18.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00071259700000000004 2023-10-09T05:58:18.854 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.854 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:58:18.855 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.855 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82139 6.0 6:09287403:::benchmark_data_smithi064_284193_object82138:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T05:58:18.855 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:57.083362+0000", 2023-10-09T05:58:18.855 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 381.704582878, 2023-10-09T05:58:18.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2163568469999999, 2023-10-09T05:58:18.855 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:58:18.855 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:18.855 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:58:18.855 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:58:18.855 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:18.856 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82139 2023-10-09T05:58:18.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.856 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:58:18.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:58:18.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083362+0000", 2023-10-09T05:58:18.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:58:18.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083362+0000", 2023-10-09T05:58:18.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:58:18.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083361+0000", 2023-10-09T05:58:18.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:18.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:58:18.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083383+0000", 2023-10-09T05:58:18.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1542000000000001e-05 2023-10-09T05:58:18.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.858 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:58:18.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083394+0000", 2023-10-09T05:58:18.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0767e-05 2023-10-09T05:58:18.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.858 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:58:18.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083399+0000", 2023-10-09T05:58:18.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7570000000000001e-06 2023-10-09T05:58:18.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.858 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.084733+0000", 2023-10-09T05:58:18.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001333309 2023-10-09T05:58:18.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.859 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:58:18.859 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.084740+0000", 2023-10-09T05:58:18.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8029999999999999e-06 2023-10-09T05:58:18.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.859 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.859 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.276687+0000", 2023-10-09T05:58:18.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1919472209999999 2023-10-09T05:58:18.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:58:18.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.276860+0000", 2023-10-09T05:58:18.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017371800000000001 2023-10-09T05:58:18.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:58:18.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.277059+0000", 2023-10-09T05:58:18.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000198104 2023-10-09T05:58:18.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.282435+0000", 2023-10-09T05:58:18.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053760370000000002 2023-10-09T05:58:18.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:58:18.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.282677+0000", 2023-10-09T05:58:18.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000242854 2023-10-09T05:58:18.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.299289+0000", 2023-10-09T05:58:18.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016611679000000001 2023-10-09T05:58:18.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:58:18.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.299347+0000", 2023-10-09T05:58:18.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7896999999999998e-05 2023-10-09T05:58:18.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.864 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:58:18.864 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.299719+0000", 2023-10-09T05:58:18.864 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00037165400000000002 2023-10-09T05:58:18.864 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.864 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:58:18.864 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.864 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82142 6.9 6:92f3da5c:::benchmark_data_smithi064_284193_object82141:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T05:58:18.864 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:57.083404+0000", 2023-10-09T05:58:18.864 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 381.70454077699998, 2023-10-09T05:58:18.864 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2055263089999999, 2023-10-09T05:58:18.865 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:58:18.865 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:18.865 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:58:18.865 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:58:18.865 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:18.865 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82142 2023-10-09T05:58:18.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.865 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:58:18.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:58:18.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083404+0000", 2023-10-09T05:58:18.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:58:18.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083404+0000", 2023-10-09T05:58:18.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:58:18.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083403+0000", 2023-10-09T05:58:18.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:18.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:58:18.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083424+0000", 2023-10-09T05:58:18.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0803999999999999e-05 2023-10-09T05:58:18.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:58:18.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083431+0000", 2023-10-09T05:58:18.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5780000000000004e-06 2023-10-09T05:58:18.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:58:18.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083436+0000", 2023-10-09T05:58:18.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.181e-06 2023-10-09T05:58:18.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083586+0000", 2023-10-09T05:58:18.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015032199999999999 2023-10-09T05:58:18.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:58:18.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083595+0000", 2023-10-09T05:58:18.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3999999999999992e-06 2023-10-09T05:58:18.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.869 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.869 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.268821+0000", 2023-10-09T05:58:18.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.185226458 2023-10-09T05:58:18.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.869 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:58:18.869 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.268987+0000", 2023-10-09T05:58:18.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016546599999999999 2023-10-09T05:58:18.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:58:18.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.269196+0000", 2023-10-09T05:58:18.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020894199999999999 2023-10-09T05:58:18.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:58:18.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.276199+0000", 2023-10-09T05:58:18.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0070038699999999997 2023-10-09T05:58:18.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.279785+0000", 2023-10-09T05:58:18.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035854609999999999 2023-10-09T05:58:18.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.288179+0000", 2023-10-09T05:58:18.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0083937950000000008 2023-10-09T05:58:18.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:58:18.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.288258+0000", 2023-10-09T05:58:18.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9121000000000004e-05 2023-10-09T05:58:18.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:58:18.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:58.288930+0000", 2023-10-09T05:58:18.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00067237600000000003 2023-10-09T05:58:18.872 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.872 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:58:18.872 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.872 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:159588 6.f 6:f2345e47:::benchmark_data_smithi064_284193_object63045:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e249)", 2023-10-09T05:58:18.872 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:57:23.458648+0000", 2023-10-09T05:58:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 55.329297115999999, 2023-10-09T05:58:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.017528263, 2023-10-09T05:58:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:58:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:58:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:58:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 159588 2023-10-09T05:58:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:58:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:58:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458648+0000", 2023-10-09T05:58:18.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.874 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.874 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:58:18.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458648+0000", 2023-10-09T05:58:18.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.874 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.874 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:58:18.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458637+0000", 2023-10-09T05:58:18.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999989 2023-10-09T05:58:18.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:58:18.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458653+0000", 2023-10-09T05:58:18.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5855999999999999e-05 2023-10-09T05:58:18.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:58:18.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458665+0000", 2023-10-09T05:58:18.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2064e-05 2023-10-09T05:58:18.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:58:18.876 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458674+0000", 2023-10-09T05:58:18.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3950000000000007e-06 2023-10-09T05:58:18.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.876 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.460075+0000", 2023-10-09T05:58:18.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001401086 2023-10-09T05:58:18.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:58:18.876 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.460087+0000", 2023-10-09T05:58:18.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1527e-05 2023-10-09T05:58:18.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.461560+0000", 2023-10-09T05:58:18.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0014731719999999 2023-10-09T05:58:18.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:58:18.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.462110+0000", 2023-10-09T05:58:18.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00054960100000000004 2023-10-09T05:58:18.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:58:18.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.463093+0000", 2023-10-09T05:58:18.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00098321699999999999 2023-10-09T05:58:18.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.467039+0000", 2023-10-09T05:58:18.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039463830000000004 2023-10-09T05:58:18.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.470896+0000", 2023-10-09T05:58:18.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003856805 2023-10-09T05:58:18.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:58:18.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.475276+0000", 2023-10-09T05:58:18.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043798400000000003 2023-10-09T05:58:18.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:58:18.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.475478+0000", 2023-10-09T05:58:18.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020193899999999999 2023-10-09T05:58:18.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:58:18.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.476176+0000", 2023-10-09T05:58:18.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00069817800000000004 2023-10-09T05:58:18.880 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.880 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:58:18.880 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.880 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:159600 6.1f 6:f9ce68c5:::benchmark_data_smithi064_284193_object63057:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e249)", 2023-10-09T05:58:18.880 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:57:23.458682+0000", 2023-10-09T05:58:18.880 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 55.329262800999999, 2023-10-09T05:58:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0352311799999998, 2023-10-09T05:58:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:58:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:58:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:58:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 159600 2023-10-09T05:58:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:58:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:58:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458682+0000", 2023-10-09T05:58:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:58:18.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458682+0000", 2023-10-09T05:58:18.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:58:18.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458681+0000", 2023-10-09T05:58:18.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:18.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:58:18.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458684+0000", 2023-10-09T05:58:18.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4700000000000001e-06 2023-10-09T05:58:18.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:58:18.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458690+0000", 2023-10-09T05:58:18.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.993e-06 2023-10-09T05:58:18.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:58:18.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458695+0000", 2023-10-09T05:58:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8300000000000003e-06 2023-10-09T05:58:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.460091+0000", 2023-10-09T05:58:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013964229999999999 2023-10-09T05:58:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:58:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.460101+0000", 2023-10-09T05:58:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0040999999999999e-05 2023-10-09T05:58:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.467208+0000", 2023-10-09T05:58:18.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.007106442 2023-10-09T05:58:18.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:58:18.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.467772+0000", 2023-10-09T05:58:18.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00056436500000000005 2023-10-09T05:58:18.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:58:18.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.468789+0000", 2023-10-09T05:58:18.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001016711 2023-10-09T05:58:18.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.886 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.475179+0000", 2023-10-09T05:58:18.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063902730000000001 2023-10-09T05:58:18.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.886 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.475320+0000", 2023-10-09T05:58:18.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000141423 2023-10-09T05:58:18.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.887 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:58:18.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.492859+0000", 2023-10-09T05:58:18.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017538893 2023-10-09T05:58:18.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.887 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:58:18.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.493094+0000", 2023-10-09T05:58:18.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00023475199999999999 2023-10-09T05:58:18.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.887 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:58:18.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.493913+0000", 2023-10-09T05:58:18.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00081901499999999998 2023-10-09T05:58:18.888 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.888 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:58:18.888 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.888 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:159626 6.f 6:f162f1a2:::benchmark_data_smithi064_284193_object63083:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2023-10-09T05:58:18.888 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:57:24.716483+0000", 2023-10-09T05:58:18.888 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 54.071461339999999, 2023-10-09T05:58:18.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.78104533300000001, 2023-10-09T05:58:18.888 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:58:18.888 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:18.889 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:58:18.889 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:58:18.889 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:18.889 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 159626 2023-10-09T05:58:18.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.889 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:58:18.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:58:18.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.716483+0000", 2023-10-09T05:58:18.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:58:18.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.716483+0000", 2023-10-09T05:58:18.890 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:58:18.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.716482+0000", 2023-10-09T05:58:18.890 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:58:18.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:58:18.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.716489+0000", 2023-10-09T05:58:18.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3339999999999996e-06 2023-10-09T05:58:18.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:58:18.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.716502+0000", 2023-10-09T05:58:18.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.225e-05 2023-10-09T05:58:18.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:58:18.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.716509+0000", 2023-10-09T05:58:18.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4100000000000002e-06 2023-10-09T05:58:18.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.470955+0000", 2023-10-09T05:58:18.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.75444619099999999 2023-10-09T05:58:18.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:58:18.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.471459+0000", 2023-10-09T05:58:18.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00050348000000000005 2023-10-09T05:58:18.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:58:18.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.472490+0000", 2023-10-09T05:58:18.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001031052 2023-10-09T05:58:18.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.484089+0000", 2023-10-09T05:58:18.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011599174 2023-10-09T05:58:18.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.491423+0000", 2023-10-09T05:58:18.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0073340339999999997 2023-10-09T05:58:18.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:58:18.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.496443+0000", 2023-10-09T05:58:18.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050197100000000001 2023-10-09T05:58:18.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:58:18.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.496712+0000", 2023-10-09T05:58:18.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000269545 2023-10-09T05:58:18.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:58:18.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.497529+0000", 2023-10-09T05:58:18.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00081657400000000003 2023-10-09T05:58:18.895 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.895 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:58:18.895 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.895 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:159643 6.1f 6:ff3464fe:::benchmark_data_smithi064_284193_object63100:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2023-10-09T05:58:18.895 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:57:24.734794+0000", 2023-10-09T05:58:18.895 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 54.053150354000003, 2023-10-09T05:58:18.896 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.76032684500000003, 2023-10-09T05:58:18.896 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:58:18.896 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:18.896 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:58:18.896 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:58:18.896 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:18.896 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 159643 2023-10-09T05:58:18.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.896 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:58:18.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.896 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:58:18.896 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.734794+0000", 2023-10-09T05:58:18.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:58:18.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.734794+0000", 2023-10-09T05:58:18.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:58:18.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.734793+0000", 2023-10-09T05:58:18.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:58:18.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.898 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:58:18.898 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.734801+0000", 2023-10-09T05:58:18.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4689999999999997e-06 2023-10-09T05:58:18.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.898 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:58:18.898 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.734813+0000", 2023-10-09T05:58:18.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1992e-05 2023-10-09T05:58:18.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.898 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:58:18.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.734825+0000", 2023-10-09T05:58:18.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2136e-05 2023-10-09T05:58:18.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.471299+0000", 2023-10-09T05:58:18.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.73647401999999995 2023-10-09T05:58:18.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:58:18.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.471852+0000", 2023-10-09T05:58:18.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00055305399999999998 2023-10-09T05:58:18.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:58:18.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.472827+0000", 2023-10-09T05:58:18.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00097435000000000002 2023-10-09T05:58:18.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.484261+0000", 2023-10-09T05:58:18.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011434659999999999 2023-10-09T05:58:18.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.901 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.901 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.492812+0000", 2023-10-09T05:58:18.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085508909999999997 2023-10-09T05:58:18.901 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.901 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:58:18.901 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.494107+0000", 2023-10-09T05:58:18.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001294672 2023-10-09T05:58:18.901 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.901 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:58:18.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.494329+0000", 2023-10-09T05:58:18.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022168599999999999 2023-10-09T05:58:18.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:58:18.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.495121+0000", 2023-10-09T05:58:18.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00079260400000000003 2023-10-09T05:58:18.902 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.902 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:58:18.902 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.903 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:159645 6.1f 6:fc3f4f0b:::benchmark_data_smithi064_284193_object63102:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2023-10-09T05:58:18.903 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:57:24.746020+0000", 2023-10-09T05:58:18.903 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 54.041924786999999, 2023-10-09T05:58:18.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.75281700900000004, 2023-10-09T05:58:18.903 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:58:18.903 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:18.903 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:58:18.903 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:58:18.903 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:18.903 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 159645 2023-10-09T05:58:18.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.904 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:58:18.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.904 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:58:18.904 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.746020+0000", 2023-10-09T05:58:18.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.904 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:58:18.904 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.746020+0000", 2023-10-09T05:58:18.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.905 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.905 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:58:18.905 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.746018+0000", 2023-10-09T05:58:18.905 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:58:18.905 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.905 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.905 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:58:18.905 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.746027+0000", 2023-10-09T05:58:18.905 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1730000000000001e-06 2023-10-09T05:58:18.905 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.905 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.905 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:58:18.906 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.746042+0000", 2023-10-09T05:58:18.906 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5104e-05 2023-10-09T05:58:18.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.906 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:58:18.906 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.746052+0000", 2023-10-09T05:58:18.906 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0276e-05 2023-10-09T05:58:18.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.906 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.906 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.475381+0000", 2023-10-09T05:58:18.906 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.72932878199999995 2023-10-09T05:58:18.907 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.907 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:58:18.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.475964+0000", 2023-10-09T05:58:18.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00058292399999999996 2023-10-09T05:58:18.907 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.907 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:58:18.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.477002+0000", 2023-10-09T05:58:18.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010378729999999999 2023-10-09T05:58:18.907 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.908 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.908 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.491255+0000", 2023-10-09T05:58:18.908 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014253159 2023-10-09T05:58:18.908 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.908 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.908 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.908 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.494063+0000", 2023-10-09T05:58:18.908 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028082229999999999 2023-10-09T05:58:18.908 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.908 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.908 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:58:18.908 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.497725+0000", 2023-10-09T05:58:18.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0036619040000000001 2023-10-09T05:58:18.909 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.909 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.909 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:58:18.909 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.497974+0000", 2023-10-09T05:58:18.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00024911099999999999 2023-10-09T05:58:18.909 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.909 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.909 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:58:18.909 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.498837+0000", 2023-10-09T05:58:18.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00086290100000000001 2023-10-09T05:58:18.909 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.910 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:58:18.910 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.910 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.910 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:159658 6.1f 6:fcf68710:::benchmark_data_smithi064_284193_object63115:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2023-10-09T05:58:18.910 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:57:24.769758+0000", 2023-10-09T05:58:18.910 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 54.018187202, 2023-10-09T05:58:18.910 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.73009417799999998, 2023-10-09T05:58:18.910 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:58:18.910 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:18.910 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:58:18.910 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:58:18.911 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:18.911 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 159658 2023-10-09T05:58:18.911 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.911 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:58:18.911 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.911 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:58:18.911 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.769758+0000", 2023-10-09T05:58:18.911 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.911 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.911 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.911 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:58:18.911 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.769758+0000", 2023-10-09T05:58:18.912 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.912 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.912 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.912 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:58:18.912 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.769756+0000", 2023-10-09T05:58:18.912 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:58:18.912 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.912 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.912 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:58:18.912 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.769764+0000", 2023-10-09T05:58:18.912 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9950000000000005e-06 2023-10-09T05:58:18.912 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.913 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.913 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:58:18.913 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.769779+0000", 2023-10-09T05:58:18.913 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5109999999999999e-05 2023-10-09T05:58:18.913 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.913 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.913 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:58:18.913 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.769787+0000", 2023-10-09T05:58:18.913 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0229999999999992e-06 2023-10-09T05:58:18.913 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.913 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.913 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.913 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.479571+0000", 2023-10-09T05:58:18.914 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.70978438099999996 2023-10-09T05:58:18.914 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.914 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:58:18.914 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.480164+0000", 2023-10-09T05:58:18.914 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00059256399999999996 2023-10-09T05:58:18.914 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.914 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:58:18.914 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.481306+0000", 2023-10-09T05:58:18.914 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011416270000000001 2023-10-09T05:58:18.914 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.915 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.915 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.915 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.491611+0000", 2023-10-09T05:58:18.915 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010305322 2023-10-09T05:58:18.915 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.915 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.915 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.915 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.496545+0000", 2023-10-09T05:58:18.915 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049339700000000002 2023-10-09T05:58:18.915 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.915 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.915 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:58:18.915 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.498876+0000", 2023-10-09T05:58:18.916 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0023313679999999999 2023-10-09T05:58:18.916 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.916 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.916 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:58:18.916 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.499074+0000", 2023-10-09T05:58:18.916 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019782400000000001 2023-10-09T05:58:18.916 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.916 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.916 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:58:18.916 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.499852+0000", 2023-10-09T05:58:18.916 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00077764900000000005 2023-10-09T05:58:18.916 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.917 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:58:18.917 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.917 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.917 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.917 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:159663 6.f 6:f2ece728:::benchmark_data_smithi064_284193_object63120:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2023-10-09T05:58:18.917 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:57:24.775303+0000", 2023-10-09T05:58:18.917 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 54.012642215, 2023-10-09T05:58:18.917 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.73579451699999998, 2023-10-09T05:58:18.917 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:58:18.917 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:18.917 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:58:18.917 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:58:18.918 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:18.918 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 159663 2023-10-09T05:58:18.918 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.918 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:58:18.918 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.918 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:58:18.918 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.775303+0000", 2023-10-09T05:58:18.918 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.918 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.918 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.918 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:58:18.918 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.775303+0000", 2023-10-09T05:58:18.919 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.919 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:58:18.919 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.775301+0000", 2023-10-09T05:58:18.919 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:58:18.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.919 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:58:18.919 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.775309+0000", 2023-10-09T05:58:18.919 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6109999999999994e-06 2023-10-09T05:58:18.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.920 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.920 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:58:18.920 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.775323+0000", 2023-10-09T05:58:18.920 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3444999999999999e-05 2023-10-09T05:58:18.920 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.920 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.920 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:58:18.920 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.775332+0000", 2023-10-09T05:58:18.920 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1099999999999992e-06 2023-10-09T05:58:18.920 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.920 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.920 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.484327+0000", 2023-10-09T05:58:18.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.70899562999999999 2023-10-09T05:58:18.921 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.921 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.921 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:58:18.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.484906+0000", 2023-10-09T05:58:18.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00057899099999999995 2023-10-09T05:58:18.921 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.921 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.921 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:58:18.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.487858+0000", 2023-10-09T05:58:18.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002951122 2023-10-09T05:58:18.921 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.922 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:58:18.922 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.499920+0000", 2023-10-09T05:58:18.922 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012062241 2023-10-09T05:58:18.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.922 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.922 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.509720+0000", 2023-10-09T05:58:18.922 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0098000269999999993 2023-10-09T05:58:18.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.922 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.923 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.509824+0000", 2023-10-09T05:58:18.923 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000104054 2023-10-09T05:58:18.923 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.923 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.923 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:58:18.923 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.509998+0000", 2023-10-09T05:58:18.923 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017401 2023-10-09T05:58:18.923 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.923 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.923 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:58:18.923 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.511097+0000", 2023-10-09T05:58:18.923 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001099091 2023-10-09T05:58:18.924 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.924 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:58:18.924 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.924 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.924 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.924 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:159676 6.f 6:f1e6b117:::benchmark_data_smithi064_284193_object63133:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2023-10-09T05:58:18.924 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:57:24.804817+0000", 2023-10-09T05:58:18.924 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 53.983127613000001, 2023-10-09T05:58:18.924 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.70768902600000005, 2023-10-09T05:58:18.924 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:58:18.924 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:18.924 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:58:18.924 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:58:18.925 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:18.925 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 159676 2023-10-09T05:58:18.925 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.925 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:58:18.925 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.925 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:58:18.925 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.804817+0000", 2023-10-09T05:58:18.925 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.925 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.925 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.925 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:58:18.926 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.804817+0000", 2023-10-09T05:58:18.926 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.926 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.926 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.926 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:58:18.926 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.804816+0000", 2023-10-09T05:58:18.926 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:58:18.926 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.926 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.926 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:58:18.926 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.804823+0000", 2023-10-09T05:58:18.926 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4499999999999998e-06 2023-10-09T05:58:18.927 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.927 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.927 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:58:18.927 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.804835+0000", 2023-10-09T05:58:18.927 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1863e-05 2023-10-09T05:58:18.927 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.927 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.927 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:58:18.927 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.804843+0000", 2023-10-09T05:58:18.927 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1440000000000003e-06 2023-10-09T05:58:18.927 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.927 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.928 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.928 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.491683+0000", 2023-10-09T05:58:18.928 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.68683944399999997 2023-10-09T05:58:18.928 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.928 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.928 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:58:18.928 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.493003+0000", 2023-10-09T05:58:18.928 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013208810000000001 2023-10-09T05:58:18.928 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.928 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.928 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:58:18.928 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.494031+0000", 2023-10-09T05:58:18.929 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010271880000000001 2023-10-09T05:58:18.929 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.929 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.929 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.929 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.511295+0000", 2023-10-09T05:58:18.929 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017264623999999999 2023-10-09T05:58:18.929 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.929 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.929 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:58:18.929 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.511338+0000", 2023-10-09T05:58:18.929 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3163e-05 2023-10-09T05:58:18.929 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.929 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.930 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.930 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.511537+0000", 2023-10-09T05:58:18.930 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019832500000000001 2023-10-09T05:58:18.930 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.930 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.930 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:58:18.930 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.511748+0000", 2023-10-09T05:58:18.930 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00021075099999999999 2023-10-09T05:58:18.930 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.930 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.930 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:58:18.930 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.512506+0000", 2023-10-09T05:58:18.931 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00075862 2023-10-09T05:58:18.931 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.931 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:58:18.931 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.931 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.931 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.931 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:159708 6.1f 6:fb43fd9f:::benchmark_data_smithi064_284193_object63165:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2023-10-09T05:58:18.931 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:57:24.870366+0000", 2023-10-09T05:58:18.931 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 53.917578333000002, 2023-10-09T05:58:18.931 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.66420900599999999, 2023-10-09T05:58:18.931 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:58:18.931 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:18.932 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:58:18.932 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:58:18.932 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:18.932 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 159708 2023-10-09T05:58:18.932 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.932 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:58:18.932 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.932 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:58:18.932 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.870366+0000", 2023-10-09T05:58:18.932 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.932 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.932 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.932 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:58:18.933 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.870366+0000", 2023-10-09T05:58:18.933 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.933 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.933 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.933 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:58:18.933 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.870365+0000", 2023-10-09T05:58:18.933 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:58:18.933 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.933 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.933 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:58:18.933 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.870372+0000", 2023-10-09T05:58:18.933 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8050000000000001e-06 2023-10-09T05:58:18.934 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.934 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.934 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:58:18.934 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.870383+0000", 2023-10-09T05:58:18.934 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1178e-05 2023-10-09T05:58:18.934 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.934 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.934 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:58:18.934 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.870389+0000", 2023-10-09T05:58:18.934 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0979999999999999e-06 2023-10-09T05:58:18.934 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.934 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.935 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.511474+0000", 2023-10-09T05:58:18.935 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.64108517600000003 2023-10-09T05:58:18.935 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.935 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:58:18.935 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.512073+0000", 2023-10-09T05:58:18.935 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00059926300000000005 2023-10-09T05:58:18.935 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.935 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:58:18.935 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.513124+0000", 2023-10-09T05:58:18.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010503590000000001 2023-10-09T05:58:18.936 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.936 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.936 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.936 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.523526+0000", 2023-10-09T05:58:18.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010402514999999999 2023-10-09T05:58:18.936 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.936 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.936 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.936 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.523834+0000", 2023-10-09T05:58:18.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00030743799999999999 2023-10-09T05:58:18.936 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.937 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:58:18.937 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.533503+0000", 2023-10-09T05:58:18.937 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0096693739999999997 2023-10-09T05:58:18.937 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.937 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:58:18.937 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.533817+0000", 2023-10-09T05:58:18.937 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000314431 2023-10-09T05:58:18.937 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.937 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:58:18.937 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.534575+0000", 2023-10-09T05:58:18.938 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00075793500000000005 2023-10-09T05:58:18.938 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.938 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:58:18.938 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.938 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.938 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.938 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:159752 6.1f 6:f8ab3624:::benchmark_data_smithi064_284193_object63209:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2023-10-09T05:58:18.938 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:57:24.916697+0000", 2023-10-09T05:58:18.938 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 53.871248018999999, 2023-10-09T05:58:18.938 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.61881897200000002, 2023-10-09T05:58:18.938 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:58:18.939 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:18.939 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:58:18.939 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:58:18.939 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:18.939 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 159752 2023-10-09T05:58:18.939 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.939 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:58:18.939 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.939 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:58:18.939 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.916697+0000", 2023-10-09T05:58:18.939 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.939 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.940 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:58:18.940 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.916697+0000", 2023-10-09T05:58:18.940 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:58:18.940 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.940 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:58:18.940 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.916695+0000", 2023-10-09T05:58:18.940 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:58:18.940 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.940 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:58:18.940 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.916702+0000", 2023-10-09T05:58:18.941 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.888e-06 2023-10-09T05:58:18.941 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.941 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.941 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:58:18.941 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.916714+0000", 2023-10-09T05:58:18.941 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2279000000000001e-05 2023-10-09T05:58:18.941 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.941 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.941 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:58:18.941 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.916722+0000", 2023-10-09T05:58:18.941 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.469e-06 2023-10-09T05:58:18.941 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.942 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.942 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:58:18.942 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.515429+0000", 2023-10-09T05:58:18.942 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.59870694400000002 2023-10-09T05:58:18.942 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.942 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.942 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:58:18.942 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.516259+0000", 2023-10-09T05:58:18.942 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00082991900000000003 2023-10-09T05:58:18.942 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.942 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.942 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2023-10-09T05:58:18.943 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.517589+0000", 2023-10-09T05:58:18.943 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001330538 2023-10-09T05:58:18.943 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.943 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.943 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.943 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.523681+0000", 2023-10-09T05:58:18.943 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060920169999999999 2023-10-09T05:58:18.943 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.943 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.943 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:18.943 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.523989+0000", 2023-10-09T05:58:18.943 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00030792699999999999 2023-10-09T05:58:18.943 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.944 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.944 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:58:18.944 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.534615+0000", 2023-10-09T05:58:18.944 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010625882999999999 2023-10-09T05:58:18.944 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.944 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.944 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:58:18.944 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.534817+0000", 2023-10-09T05:58:18.944 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020159600000000001 2023-10-09T05:58:18.944 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:58:18.944 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:58:18.944 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:58:18.944 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.535516+0000", 2023-10-09T05:58:18.945 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00069890999999999996 2023-10-09T05:58:18.945 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.945 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:58:18.945 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.945 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:58:18.945 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:58:18.945 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:58:19.012 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:19.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:19.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:19.086 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:58:19.106 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:58:19.118 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:19.125 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:58:19.147 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:58:19.149 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:19.169 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:58:19.196 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:58:19.217 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:58:19.237 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:58:19.239 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:58:19.239 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:58:19.239 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:58:19.239 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:58:19.239 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:58:19.239 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:58:19.239 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:58:19.239 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:58:19.443 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:58:19.444 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T05:58:19.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[220044]: osdmap e280: 8 total, 8 up, 7 in 2023-10-09T05:58:19.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[220044]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[220044]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[220044]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[220044]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[220044]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[220044]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[220044]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[220044]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[220044]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[220044]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[220044]: pgmap v633: 93 pgs: 1 remapped+peering, 1 unknown, 91 active+clean; 564 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:19.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/215007134' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:19.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[212750]: osdmap e280: 8 total, 8 up, 7 in 2023-10-09T05:58:19.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[212750]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[212750]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[212750]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[212750]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[212750]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[212750]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[212750]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[212750]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.458 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[212750]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.458 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[212750]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.458 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[212750]: pgmap v633: 93 pgs: 1 remapped+peering, 1 unknown, 91 active+clean; 564 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:19.458 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:19 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/215007134' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:19.558 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:19.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:19.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:19.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:19 smithi156 ceph-mon[195101]: osdmap e280: 8 total, 8 up, 7 in 2023-10-09T05:58:19.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:19 smithi156 ceph-mon[195101]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:19 smithi156 ceph-mon[195101]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:19 smithi156 ceph-mon[195101]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:19 smithi156 ceph-mon[195101]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:19 smithi156 ceph-mon[195101]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:19 smithi156 ceph-mon[195101]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:19 smithi156 ceph-mon[195101]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:19 smithi156 ceph-mon[195101]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:19 smithi156 ceph-mon[195101]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:19 smithi156 ceph-mon[195101]: failed to encode map e280 with expected crc 2023-10-09T05:58:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:19 smithi156 ceph-mon[195101]: pgmap v633: 93 pgs: 1 remapped+peering, 1 unknown, 91 active+clean; 564 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:19.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:19 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/215007134' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:19.662 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:19.696 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:19.747 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:19.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:19.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:19.854 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:19.886 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:20.387 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T05:58:20.387 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:20.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:20.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:20.490 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:20.521 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:20.570 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:20.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:20.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:20.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:20 smithi156 ceph-mon[195101]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:20 smithi156 ceph-mon[195101]: osdmap e281: 8 total, 8 up, 7 in 2023-10-09T05:58:20.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:20 smithi156 ceph-mon[195101]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:20 smithi156 ceph-mon[195101]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:20 smithi156 ceph-mon[195101]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:20 smithi156 ceph-mon[195101]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:20 smithi156 ceph-mon[195101]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:20 smithi156 ceph-mon[195101]: 1.c continuing backfill to osd.6 from (0'0,80'161] MIN to 80'161 2023-10-09T05:58:20.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:20 smithi156 ceph-mon[195101]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:20 smithi156 ceph-mon[195101]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:20 smithi156 ceph-mon[195101]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:20 smithi156 ceph-mon[195101]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:20 smithi156 ceph-mon[195101]: from='client.45209 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:58:20.678 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:20.705 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[212750]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.705 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[212750]: osdmap e281: 8 total, 8 up, 7 in 2023-10-09T05:58:20.705 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[212750]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[212750]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[212750]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[212750]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[212750]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[212750]: 1.c continuing backfill to osd.6 from (0'0,80'161] MIN to 80'161 2023-10-09T05:58:20.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[212750]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[212750]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[212750]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.707 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[212750]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.707 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[212750]: from='client.45209 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:58:20.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[220044]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[220044]: osdmap e281: 8 total, 8 up, 7 in 2023-10-09T05:58:20.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[220044]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[220044]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[220044]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[220044]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[220044]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[220044]: 1.c continuing backfill to osd.6 from (0'0,80'161] MIN to 80'161 2023-10-09T05:58:20.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[220044]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[220044]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[220044]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[220044]: failed to encode map e281 with expected crc 2023-10-09T05:58:20.709 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:20 smithi064 ceph-mon[220044]: from='client.45209 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:58:20.709 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:21.006 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:21.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:21.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:21.109 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:21.142 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:21.192 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:21.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:21.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:21.297 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:21.335 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:21.335 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[220044]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.335 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[220044]: osdmap e282: 8 total, 8 up, 7 in 2023-10-09T05:58:21.336 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[220044]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.336 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[220044]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.336 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[220044]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.336 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[220044]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.336 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[220044]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.336 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[220044]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.336 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[220044]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.336 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[220044]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.336 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[220044]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.336 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[220044]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.336 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[220044]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.336 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[220044]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.337 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[220044]: pgmap v636: 93 pgs: 2 peering, 7 active+undersized+remapped, 1 remapped+peering, 83 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 160 KiB/s, 0 objects/s recovering 2023-10-09T05:58:21.337 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[220044]: osdmap e283: 8 total, 8 up, 7 in 2023-10-09T05:58:21.337 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[212750]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.337 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[212750]: osdmap e282: 8 total, 8 up, 7 in 2023-10-09T05:58:21.337 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[212750]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.337 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[212750]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.337 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[212750]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.337 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[212750]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.337 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[212750]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.337 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[212750]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.337 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[212750]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.338 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[212750]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.338 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[212750]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.338 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[212750]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.338 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[212750]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.338 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[212750]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.338 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[212750]: pgmap v636: 93 pgs: 2 peering, 7 active+undersized+remapped, 1 remapped+peering, 83 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 160 KiB/s, 0 objects/s recovering 2023-10-09T05:58:21.338 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:21 smithi064 ceph-mon[212750]: osdmap e283: 8 total, 8 up, 7 in 2023-10-09T05:58:21.378 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:21.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:21.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:21.484 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:21.518 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:21.564 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:21.598 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":634,"stamp":"2023-10-09T05:58:19.314944+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":577982,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":472,"num_read_kb":885,"num_write":293,"num_write_kb":2372,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":2646274,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3275,"ondisk_log_size":3275,"up":279,"acting":278,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":483,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":3368992,"kb_used_data":1467944,"kb_used_omap":118,"kb_used_meta":1900873,"kb_avail":652847072,"statfs":{"total":671965249536,"available":668515401728,"internally_reserved":0,"allocated":1503174656,"data_stored":1494314577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":121648,"internal_metadata":1946494160},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":26,"apply_latency_ms":26,"commit_latency_ns":26000000,"apply_latency_ns":26000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-33768924,"num_objects":-514,"num_object_clones":0,"num_object_copies":-1542,"num_objects_missing_on_primary":-1682,"num_objects_missing":-1682,"num_objects_degraded":-17154,"num_objects_misplaced":-152,"num_objects_unfound":0,"num_objects_dirty":-514,"num_whiteouts":0,"num_read":-16,"num_read_kb":-29,"num_write":-190614,"num_write_kb":-6083274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-78349,"num_bytes_recovered":-4063941054,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.695482"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":322,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.246437+0000","last_change":"2023-10-09T05:55:59.341215+0000","last_active":"2023-10-09T05:58:12.246437+0000","last_peered":"2023-10-09T05:58:12.246437+0000","last_clean":"2023-10-09T05:58:12.246437+0000","last_became_active":"2023-10-09T05:55:59.340856+0000","last_became_peered":"2023-10-09T05:55:59.340856+0000","last_unstale":"2023-10-09T05:58:12.246437+0000","last_undegraded":"2023-10-09T05:58:12.246437+0000","last_fullsized":"2023-10-09T05:58:12.246437+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:33.686454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":39,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.239006+0000","last_change":"2023-10-09T05:57:23.474348+0000","last_active":"2023-10-09T05:58:12.239006+0000","last_peered":"2023-10-09T05:58:12.239006+0000","last_clean":"2023-10-09T05:58:12.239006+0000","last_became_active":"2023-10-09T05:57:23.474010+0000","last_became_peered":"2023-10-09T05:57:23.474010+0000","last_unstale":"2023-10-09T05:58:12.239006+0000","last_undegraded":"2023-10-09T05:58:12.239006+0000","last_fullsized":"2023-10-09T05:58:12.239006+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:45.039777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":156,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.238219+0000","last_change":"2023-10-09T05:55:59.304893+0000","last_active":"2023-10-09T05:58:12.238219+0000","last_peered":"2023-10-09T05:58:12.238219+0000","last_clean":"2023-10-09T05:58:12.238219+0000","last_became_active":"2023-10-09T05:55:59.304449+0000","last_became_peered":"2023-10-09T05:55:59.304449+0000","last_unstale":"2023-10-09T05:58:12.238219+0000","last_undegraded":"2023-10-09T05:58:12.238219+0000","last_fullsized":"2023-10-09T05:58:12.238219+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:44:39.453982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":297,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.246664+0000","last_change":"2023-10-09T05:57:58.834014+0000","last_active":"2023-10-09T05:58:12.246664+0000","last_peered":"2023-10-09T05:58:12.246664+0000","last_clean":"2023-10-09T05:58:12.246664+0000","last_became_active":"2023-10-09T05:57:58.833718+0000","last_became_peered":"2023-10-09T05:57:58.833718+0000","last_unstale":"2023-10-09T05:58:12.246664+0000","last_undegraded":"2023-10-09T05:58:12.246664+0000","last_fullsized":"2023-10-09T05:58:12.246664+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:23.752353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":39,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.237874+0000","last_change":"2023-10-09T05:57:23.476832+0000","last_active":"2023-10-09T05:58:12.237874+0000","last_peered":"2023-10-09T05:58:12.237874+0000","last_clean":"2023-10-09T05:58:12.237874+0000","last_became_active":"2023-10-09T05:57:23.475216+0000","last_became_peered":"2023-10-09T05:57:23.475216+0000","last_unstale":"2023-10-09T05:58:12.237874+0000","last_undegraded":"2023-10-09T05:58:12.237874+0000","last_fullsized":"2023-10-09T05:58:12.237874+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:48.279660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":306,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.321517+0000","last_change":"2023-10-09T05:55:59.335464+0000","last_active":"2023-10-09T05:58:17.321517+0000","last_peered":"2023-10-09T05:58:17.321517+0000","last_clean":"2023-10-09T05:58:17.321517+0000","last_became_active":"2023-10-09T05:55:59.335259+0000","last_became_peered":"2023-10-09T05:55:59.335259+0000","last_unstale":"2023-10-09T05:58:17.321517+0000","last_undegraded":"2023-10-09T05:58:17.321517+0000","last_fullsized":"2023-10-09T05:58:17.321517+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:28.536682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":232,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.247059+0000","last_change":"2023-10-09T05:57:23.477980+0000","last_active":"2023-10-09T05:58:12.247059+0000","last_peered":"2023-10-09T05:58:12.247059+0000","last_clean":"2023-10-09T05:58:12.247059+0000","last_became_active":"2023-10-09T05:57:23.477654+0000","last_became_peered":"2023-10-09T05:57:23.477654+0000","last_unstale":"2023-10-09T05:58:12.247059+0000","last_undegraded":"2023-10-09T05:58:12.247059+0000","last_fullsized":"2023-10-09T05:58:12.247059+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:17.524683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":39,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.237892+0000","last_change":"2023-10-09T05:57:23.473644+0000","last_active":"2023-10-09T05:58:12.237892+0000","last_peered":"2023-10-09T05:58:12.237892+0000","last_clean":"2023-10-09T05:58:12.237892+0000","last_became_active":"2023-10-09T05:57:23.473391+0000","last_became_peered":"2023-10-09T05:57:23.473391+0000","last_unstale":"2023-10-09T05:58:12.237892+0000","last_undegraded":"2023-10-09T05:58:12.237892+0000","last_fullsized":"2023-10-09T05:58:12.237892+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:17.138034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"0'0","reported_seq":1,"reported_epoch":279,"state":"unknown","last_fresh":"2023-10-09T05:58:17.310476+0000","last_change":"2023-10-09T05:58:17.427190+0000","last_active":"0.000000","last_peered":"0.000000","last_clean":"0.000000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T05:58:17.310476+0000","last_undegraded":"2023-10-09T05:58:17.310476+0000","last_fullsized":"2023-10-09T05:58:17.310476+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:23.252969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":316,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.277898+0000","last_change":"2023-10-09T05:57:23.478483+0000","last_active":"2023-10-09T05:58:12.277898+0000","last_peered":"2023-10-09T05:58:12.277898+0000","last_clean":"2023-10-09T05:58:12.277898+0000","last_became_active":"2023-10-09T05:57:23.478122+0000","last_became_peered":"2023-10-09T05:57:23.478122+0000","last_unstale":"2023-10-09T05:58:12.277898+0000","last_undegraded":"2023-10-09T05:58:12.277898+0000","last_fullsized":"2023-10-09T05:58:12.277898+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:37.768717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":220,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.238133+0000","last_change":"2023-10-09T05:53:59.185702+0000","last_active":"2023-10-09T05:58:12.238133+0000","last_peered":"2023-10-09T05:58:12.238133+0000","last_clean":"2023-10-09T05:58:12.238133+0000","last_became_active":"2023-10-09T05:53:59.185373+0000","last_became_peered":"2023-10-09T05:53:59.185373+0000","last_unstale":"2023-10-09T05:58:12.238133+0000","last_undegraded":"2023-10-09T05:58:12.238133+0000","last_fullsized":"2023-10-09T05:58:12.238133+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:50.756967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":117,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.277445+0000","last_change":"2023-10-09T05:53:59.219168+0000","last_active":"2023-10-09T05:58:12.277445+0000","last_peered":"2023-10-09T05:58:12.277445+0000","last_clean":"2023-10-09T05:58:12.277445+0000","last_became_active":"2023-10-09T05:53:59.200886+0000","last_became_peered":"2023-10-09T05:53:59.200886+0000","last_unstale":"2023-10-09T05:58:12.277445+0000","last_undegraded":"2023-10-09T05:58:12.277445+0000","last_fullsized":"2023-10-09T05:58:12.277445+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:45.515036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":174,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.316088+0000","last_change":"2023-10-09T05:55:59.305100+0000","last_active":"2023-10-09T05:58:17.316088+0000","last_peered":"2023-10-09T05:58:17.316088+0000","last_clean":"2023-10-09T05:58:17.316088+0000","last_became_active":"2023-10-09T05:55:59.304758+0000","last_became_peered":"2023-10-09T05:55:59.304758+0000","last_unstale":"2023-10-09T05:58:17.316088+0000","last_undegraded":"2023-10-09T05:58:17.316088+0000","last_fullsized":"2023-10-09T05:58:17.316088+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:54:32.510945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":282,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.245997+0000","last_change":"2023-10-09T05:53:59.219141+0000","last_active":"2023-10-09T05:58:12.245997+0000","last_peered":"2023-10-09T05:58:12.245997+0000","last_clean":"2023-10-09T05:58:12.245997+0000","last_became_active":"2023-10-09T05:53:59.197318+0000","last_became_peered":"2023-10-09T05:53:59.197318+0000","last_unstale":"2023-10-09T05:58:12.245997+0000","last_undegraded":"2023-10-09T05:58:12.245997+0000","last_fullsized":"2023-10-09T05:58:12.245997+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:39.910794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":286,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.246126+0000","last_change":"2023-10-09T05:55:59.335824+0000","last_active":"2023-10-09T05:58:12.246126+0000","last_peered":"2023-10-09T05:58:12.246126+0000","last_clean":"2023-10-09T05:58:12.246126+0000","last_became_active":"2023-10-09T05:55:59.335478+0000","last_became_peered":"2023-10-09T05:55:59.335478+0000","last_unstale":"2023-10-09T05:58:12.246126+0000","last_undegraded":"2023-10-09T05:58:12.246126+0000","last_fullsized":"2023-10-09T05:58:12.246126+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:26.014501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":467,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.245192+0000","last_change":"2023-10-09T05:56:49.247703+0000","last_active":"2023-10-09T05:58:12.245192+0000","last_peered":"2023-10-09T05:58:12.245192+0000","last_clean":"2023-10-09T05:58:12.245192+0000","last_became_active":"2023-10-09T05:56:49.247308+0000","last_became_peered":"2023-10-09T05:56:49.247308+0000","last_unstale":"2023-10-09T05:58:12.245192+0000","last_undegraded":"2023-10-09T05:58:12.245192+0000","last_fullsized":"2023-10-09T05:58:12.245192+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":119,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.277185+0000","last_change":"2023-10-09T05:53:59.233898+0000","last_active":"2023-10-09T05:58:12.277185+0000","last_peered":"2023-10-09T05:58:12.277185+0000","last_clean":"2023-10-09T05:58:12.277185+0000","last_became_active":"2023-10-09T05:53:59.204175+0000","last_became_peered":"2023-10-09T05:53:59.204175+0000","last_unstale":"2023-10-09T05:58:12.277185+0000","last_undegraded":"2023-10-09T05:58:12.277185+0000","last_fullsized":"2023-10-09T05:58:12.277185+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:03.139318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":306,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.238905+0000","last_change":"2023-10-09T05:55:59.297087+0000","last_active":"2023-10-09T05:58:12.238905+0000","last_peered":"2023-10-09T05:58:12.238905+0000","last_clean":"2023-10-09T05:58:12.238905+0000","last_became_active":"2023-10-09T05:55:59.296768+0000","last_became_peered":"2023-10-09T05:55:59.296768+0000","last_unstale":"2023-10-09T05:58:12.238905+0000","last_undegraded":"2023-10-09T05:58:12.238905+0000","last_fullsized":"2023-10-09T05:58:12.238905+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:35.067705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":670,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.276646+0000","last_change":"2023-10-09T05:54:27.330211+0000","last_active":"2023-10-09T05:58:12.276646+0000","last_peered":"2023-10-09T05:58:12.276646+0000","last_clean":"2023-10-09T05:58:12.276646+0000","last_became_active":"2023-10-09T05:53:59.202399+0000","last_became_peered":"2023-10-09T05:53:59.202399+0000","last_unstale":"2023-10-09T05:58:12.276646+0000","last_undegraded":"2023-10-09T05:58:12.276646+0000","last_fullsized":"2023-10-09T05:58:12.276646+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:51:12.346243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:51:12.346243+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":278,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.245405+0000","last_change":"2023-10-09T05:57:23.473445+0000","last_active":"2023-10-09T05:58:12.245405+0000","last_peered":"2023-10-09T05:58:12.245405+0000","last_clean":"2023-10-09T05:58:12.245405+0000","last_became_active":"2023-10-09T05:57:23.473103+0000","last_became_peered":"2023-10-09T05:57:23.473103+0000","last_unstale":"2023-10-09T05:58:12.245405+0000","last_undegraded":"2023-10-09T05:58:12.245405+0000","last_fullsized":"2023-10-09T05:58:12.245405+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:39:53.825734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":200,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.277316+0000","last_change":"2023-10-09T05:56:49.237784+0000","last_active":"2023-10-09T05:58:12.277316+0000","last_peered":"2023-10-09T05:58:12.277316+0000","last_clean":"2023-10-09T05:58:12.277316+0000","last_became_active":"2023-10-09T05:56:49.237417+0000","last_became_peered":"2023-10-09T05:56:49.237417+0000","last_unstale":"2023-10-09T05:58:12.277316+0000","last_undegraded":"2023-10-09T05:58:12.277316+0000","last_fullsized":"2023-10-09T05:58:12.277316+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:33.129268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":462,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.249615+0000","last_change":"2023-10-09T05:57:54.797244+0000","last_active":"2023-10-09T05:58:12.249615+0000","last_peered":"2023-10-09T05:58:12.249615+0000","last_clean":"2023-10-09T05:58:12.249615+0000","last_became_active":"2023-10-09T05:57:54.796149+0000","last_became_peered":"2023-10-09T05:57:54.796149+0000","last_unstale":"2023-10-09T05:58:12.249615+0000","last_undegraded":"2023-10-09T05:58:12.249615+0000","last_fullsized":"2023-10-09T05:58:12.249615+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":113,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.276765+0000","last_change":"2023-10-09T05:53:59.206887+0000","last_active":"2023-10-09T05:58:12.276765+0000","last_peered":"2023-10-09T05:58:12.276765+0000","last_clean":"2023-10-09T05:58:12.276765+0000","last_became_active":"2023-10-09T05:53:59.206608+0000","last_became_peered":"2023-10-09T05:53:59.206608+0000","last_unstale":"2023-10-09T05:58:12.276765+0000","last_undegraded":"2023-10-09T05:58:12.276765+0000","last_fullsized":"2023-10-09T05:58:12.276765+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:48.975792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":259,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.246798+0000","last_change":"2023-10-09T05:57:58.828837+0000","last_active":"2023-10-09T05:58:12.246798+0000","last_peered":"2023-10-09T05:58:12.246798+0000","last_clean":"2023-10-09T05:58:12.246798+0000","last_became_active":"2023-10-09T05:57:58.828503+0000","last_became_peered":"2023-10-09T05:57:58.828503+0000","last_unstale":"2023-10-09T05:58:12.246798+0000","last_undegraded":"2023-10-09T05:58:12.246798+0000","last_fullsized":"2023-10-09T05:58:12.246798+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:08.332644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":573,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.276654+0000","last_change":"2023-10-09T05:58:01.899971+0000","last_active":"2023-10-09T05:58:12.276654+0000","last_peered":"2023-10-09T05:58:12.276654+0000","last_clean":"2023-10-09T05:58:12.276654+0000","last_became_active":"2023-10-09T05:58:01.899521+0000","last_became_peered":"2023-10-09T05:58:01.899521+0000","last_unstale":"2023-10-09T05:58:12.276654+0000","last_undegraded":"2023-10-09T05:58:12.276654+0000","last_fullsized":"2023-10-09T05:58:12.276654+0000","mapping_epoch":268,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":269,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":265,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.314845+0000","last_change":"2023-10-09T05:51:57.091639+0000","last_active":"2023-10-09T05:58:17.314845+0000","last_peered":"2023-10-09T05:58:17.314845+0000","last_clean":"2023-10-09T05:58:17.314845+0000","last_became_active":"2023-10-09T05:51:57.091284+0000","last_became_peered":"2023-10-09T05:51:57.091284+0000","last_unstale":"2023-10-09T05:58:17.314845+0000","last_undegraded":"2023-10-09T05:58:17.314845+0000","last_fullsized":"2023-10-09T05:58:17.314845+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:10.553453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":48,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.239246+0000","last_change":"2023-10-09T05:57:58.833939+0000","last_active":"2023-10-09T05:58:12.239246+0000","last_peered":"2023-10-09T05:58:12.239246+0000","last_clean":"2023-10-09T05:58:12.239246+0000","last_became_active":"2023-10-09T05:57:58.833567+0000","last_became_peered":"2023-10-09T05:57:58.833567+0000","last_unstale":"2023-10-09T05:58:12.239246+0000","last_undegraded":"2023-10-09T05:58:12.239246+0000","last_fullsized":"2023-10-09T05:58:12.239246+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:58.347572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":630,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.245704+0000","last_change":"2023-10-09T05:57:13.235857+0000","last_active":"2023-10-09T05:58:12.245704+0000","last_peered":"2023-10-09T05:58:12.245704+0000","last_clean":"2023-10-09T05:58:12.245704+0000","last_became_active":"2023-10-09T05:57:13.235379+0000","last_became_peered":"2023-10-09T05:57:13.235379+0000","last_unstale":"2023-10-09T05:58:12.245704+0000","last_undegraded":"2023-10-09T05:58:12.245704+0000","last_fullsized":"2023-10-09T05:58:12.245704+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":117,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.277838+0000","last_change":"2023-10-09T05:53:59.222136+0000","last_active":"2023-10-09T05:58:12.277838+0000","last_peered":"2023-10-09T05:58:12.277838+0000","last_clean":"2023-10-09T05:58:12.277838+0000","last_became_active":"2023-10-09T05:53:59.201853+0000","last_became_peered":"2023-10-09T05:53:59.201853+0000","last_unstale":"2023-10-09T05:58:12.277838+0000","last_undegraded":"2023-10-09T05:58:12.277838+0000","last_fullsized":"2023-10-09T05:58:12.277838+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:26.662031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":183,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.276565+0000","last_change":"2023-10-09T05:56:49.234185+0000","last_active":"2023-10-09T05:58:12.276565+0000","last_peered":"2023-10-09T05:58:12.276565+0000","last_clean":"2023-10-09T05:58:12.276565+0000","last_became_active":"2023-10-09T05:56:49.233884+0000","last_became_peered":"2023-10-09T05:56:49.233884+0000","last_unstale":"2023-10-09T05:58:12.276565+0000","last_undegraded":"2023-10-09T05:58:12.276565+0000","last_fullsized":"2023-10-09T05:58:12.276565+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:12.142436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":520,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.238259+0000","last_change":"2023-10-09T05:57:39.317706+0000","last_active":"2023-10-09T05:58:12.238259+0000","last_peered":"2023-10-09T05:58:12.238259+0000","last_clean":"2023-10-09T05:58:12.238259+0000","last_became_active":"2023-10-09T05:57:39.317408+0000","last_became_peered":"2023-10-09T05:57:39.317408+0000","last_unstale":"2023-10-09T05:58:12.238259+0000","last_undegraded":"2023-10-09T05:58:12.238259+0000","last_fullsized":"2023-10-09T05:58:12.238259+0000","mapping_epoch":256,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":709030,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":159,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.278035+0000","last_change":"2023-10-09T05:55:59.379174+0000","last_active":"2023-10-09T05:58:12.278035+0000","last_peered":"2023-10-09T05:58:12.278035+0000","last_clean":"2023-10-09T05:58:12.278035+0000","last_became_active":"2023-10-09T05:55:59.341932+0000","last_became_peered":"2023-10-09T05:55:59.341932+0000","last_unstale":"2023-10-09T05:58:12.278035+0000","last_undegraded":"2023-10-09T05:58:12.278035+0000","last_fullsized":"2023-10-09T05:58:12.278035+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.636171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":323,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.245433+0000","last_change":"2023-10-09T05:57:23.475146+0000","last_active":"2023-10-09T05:58:12.245433+0000","last_peered":"2023-10-09T05:58:12.245433+0000","last_clean":"2023-10-09T05:58:12.245433+0000","last_became_active":"2023-10-09T05:57:23.474776+0000","last_became_peered":"2023-10-09T05:57:23.474776+0000","last_unstale":"2023-10-09T05:58:12.245433+0000","last_undegraded":"2023-10-09T05:58:12.245433+0000","last_fullsized":"2023-10-09T05:58:12.245433+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:47.775743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":676,"reported_epoch":280,"state":"remapped+peering","last_fresh":"2023-10-09T05:58:18.319801+0000","last_change":"2023-10-09T05:58:18.320113+0000","last_active":"2023-10-09T05:58:17.313251+0000","last_peered":"2023-10-09T05:58:16.318923+0000","last_clean":"2023-10-09T05:58:16.318923+0000","last_became_active":"2023-10-09T05:57:26.500079+0000","last_became_peered":"2023-10-09T05:57:26.500079+0000","last_unstale":"2023-10-09T05:58:18.319801+0000","last_undegraded":"2023-10-09T05:58:18.319801+0000","last_fullsized":"2023-10-09T05:58:18.319801+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,4,5,7],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":115,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.277483+0000","last_change":"2023-10-09T05:53:59.212161+0000","last_active":"2023-10-09T05:58:12.277483+0000","last_peered":"2023-10-09T05:58:12.277483+0000","last_clean":"2023-10-09T05:58:12.277483+0000","last_became_active":"2023-10-09T05:53:59.193814+0000","last_became_peered":"2023-10-09T05:53:59.193814+0000","last_unstale":"2023-10-09T05:58:12.277483+0000","last_undegraded":"2023-10-09T05:58:12.277483+0000","last_fullsized":"2023-10-09T05:58:12.277483+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:10.502408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":319,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.315955+0000","last_change":"2023-10-09T05:55:59.303352+0000","last_active":"2023-10-09T05:58:17.315955+0000","last_peered":"2023-10-09T05:58:17.315955+0000","last_clean":"2023-10-09T05:58:17.315955+0000","last_became_active":"2023-10-09T05:55:59.302923+0000","last_became_peered":"2023-10-09T05:55:59.302923+0000","last_unstale":"2023-10-09T05:58:17.315955+0000","last_undegraded":"2023-10-09T05:58:17.315955+0000","last_fullsized":"2023-10-09T05:58:17.315955+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:53.013376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":656,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.249885+0000","last_change":"2023-10-09T05:57:26.518272+0000","last_active":"2023-10-09T05:58:12.249885+0000","last_peered":"2023-10-09T05:58:12.249885+0000","last_clean":"2023-10-09T05:58:12.249885+0000","last_became_active":"2023-10-09T05:57:26.517614+0000","last_became_peered":"2023-10-09T05:57:26.517614+0000","last_unstale":"2023-10-09T05:58:12.249885+0000","last_undegraded":"2023-10-09T05:58:12.249885+0000","last_fullsized":"2023-10-09T05:58:12.249885+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":39,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.238725+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T05:58:12.238725+0000","last_peered":"2023-10-09T05:58:12.238725+0000","last_clean":"2023-10-09T05:58:12.238725+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T05:58:12.238725+0000","last_undegraded":"2023-10-09T05:58:12.238725+0000","last_fullsized":"2023-10-09T05:58:12.238725+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":287,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.245503+0000","last_change":"2023-10-09T05:57:23.473860+0000","last_active":"2023-10-09T05:58:12.245503+0000","last_peered":"2023-10-09T05:58:12.245503+0000","last_clean":"2023-10-09T05:58:12.245503+0000","last_became_active":"2023-10-09T05:57:23.473101+0000","last_became_peered":"2023-10-09T05:57:23.473101+0000","last_unstale":"2023-10-09T05:58:12.245503+0000","last_undegraded":"2023-10-09T05:58:12.245503+0000","last_fullsized":"2023-10-09T05:58:12.245503+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:05.682865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":690,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.276890+0000","last_change":"2023-10-09T05:57:13.234252+0000","last_active":"2023-10-09T05:58:12.276890+0000","last_peered":"2023-10-09T05:58:12.276890+0000","last_clean":"2023-10-09T05:58:12.276890+0000","last_became_active":"2023-10-09T05:57:13.233849+0000","last_became_peered":"2023-10-09T05:57:13.233849+0000","last_unstale":"2023-10-09T05:58:12.276890+0000","last_undegraded":"2023-10-09T05:58:12.276890+0000","last_fullsized":"2023-10-09T05:58:12.276890+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":43,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.238155+0000","last_change":"2023-10-09T05:57:23.504733+0000","last_active":"2023-10-09T05:58:12.238155+0000","last_peered":"2023-10-09T05:58:12.238155+0000","last_clean":"2023-10-09T05:58:12.238155+0000","last_became_active":"2023-10-09T05:57:23.468709+0000","last_became_peered":"2023-10-09T05:57:23.468709+0000","last_unstale":"2023-10-09T05:58:12.238155+0000","last_undegraded":"2023-10-09T05:58:12.238155+0000","last_fullsized":"2023-10-09T05:58:12.238155+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:00.459980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":236,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.246581+0000","last_change":"2023-10-09T05:49:21.430742+0000","last_active":"2023-10-09T05:58:12.246581+0000","last_peered":"2023-10-09T05:58:12.246581+0000","last_clean":"2023-10-09T05:58:12.246581+0000","last_became_active":"2023-10-09T05:49:21.430233+0000","last_became_peered":"2023-10-09T05:49:21.430233+0000","last_unstale":"2023-10-09T05:58:12.246581+0000","last_undegraded":"2023-10-09T05:58:12.246581+0000","last_fullsized":"2023-10-09T05:58:12.246581+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:10.541302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":485,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T05:58:16.296762+0000","last_change":"2023-10-09T05:57:26.517046+0000","last_active":"2023-10-09T05:58:16.296762+0000","last_peered":"2023-10-09T05:58:16.296762+0000","last_clean":"2023-10-09T05:58:16.296762+0000","last_became_active":"2023-10-09T05:57:26.516674+0000","last_became_peered":"2023-10-09T05:57:26.516674+0000","last_unstale":"2023-10-09T05:58:16.296762+0000","last_undegraded":"2023-10-09T05:58:16.296762+0000","last_fullsized":"2023-10-09T05:58:16.296762+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":290,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.315869+0000","last_change":"2023-10-09T05:55:59.320669+0000","last_active":"2023-10-09T05:58:17.315869+0000","last_peered":"2023-10-09T05:58:17.315869+0000","last_clean":"2023-10-09T05:58:17.315869+0000","last_became_active":"2023-10-09T05:55:59.320352+0000","last_became_peered":"2023-10-09T05:55:59.320352+0000","last_unstale":"2023-10-09T05:58:17.315869+0000","last_undegraded":"2023-10-09T05:58:17.315869+0000","last_fullsized":"2023-10-09T05:58:17.315869+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:09:14.227028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":323,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.245915+0000","last_change":"2023-10-09T05:57:10.362295+0000","last_active":"2023-10-09T05:58:12.245915+0000","last_peered":"2023-10-09T05:58:12.245915+0000","last_clean":"2023-10-09T05:58:12.245915+0000","last_became_active":"2023-10-09T05:57:10.361249+0000","last_became_peered":"2023-10-09T05:57:10.361249+0000","last_unstale":"2023-10-09T05:58:12.245915+0000","last_undegraded":"2023-10-09T05:58:12.245915+0000","last_fullsized":"2023-10-09T05:58:12.245915+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:22.088261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":462,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.249535+0000","last_change":"2023-10-09T05:57:54.797410+0000","last_active":"2023-10-09T05:58:12.249535+0000","last_peered":"2023-10-09T05:58:12.249535+0000","last_clean":"2023-10-09T05:58:12.249535+0000","last_became_active":"2023-10-09T05:57:54.796543+0000","last_became_peered":"2023-10-09T05:57:54.796543+0000","last_unstale":"2023-10-09T05:58:12.249535+0000","last_undegraded":"2023-10-09T05:58:12.249535+0000","last_fullsized":"2023-10-09T05:58:12.249535+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":257,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.245326+0000","last_change":"2023-10-09T05:54:00.619165+0000","last_active":"2023-10-09T05:58:12.245326+0000","last_peered":"2023-10-09T05:58:12.245326+0000","last_clean":"2023-10-09T05:58:12.245326+0000","last_became_active":"2023-10-09T05:54:00.618888+0000","last_became_peered":"2023-10-09T05:54:00.618888+0000","last_unstale":"2023-10-09T05:58:12.245326+0000","last_undegraded":"2023-10-09T05:58:12.245326+0000","last_fullsized":"2023-10-09T05:58:12.245326+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:07.175996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":296,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.246200+0000","last_change":"2023-10-09T05:57:23.483158+0000","last_active":"2023-10-09T05:58:12.246200+0000","last_peered":"2023-10-09T05:58:12.246200+0000","last_clean":"2023-10-09T05:58:12.246200+0000","last_became_active":"2023-10-09T05:57:23.482701+0000","last_became_peered":"2023-10-09T05:57:23.482701+0000","last_unstale":"2023-10-09T05:58:12.246200+0000","last_undegraded":"2023-10-09T05:58:12.246200+0000","last_fullsized":"2023-10-09T05:58:12.246200+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:51:08.624310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":464,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T05:58:16.297137+0000","last_change":"2023-10-09T05:57:09.347095+0000","last_active":"2023-10-09T05:58:16.297137+0000","last_peered":"2023-10-09T05:58:16.297137+0000","last_clean":"2023-10-09T05:58:16.297137+0000","last_became_active":"2023-10-09T05:56:49.229606+0000","last_became_peered":"2023-10-09T05:56:49.229606+0000","last_unstale":"2023-10-09T05:58:16.297137+0000","last_undegraded":"2023-10-09T05:58:16.297137+0000","last_fullsized":"2023-10-09T05:58:16.297137+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":39,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.237831+0000","last_change":"2023-10-09T05:57:23.469928+0000","last_active":"2023-10-09T05:58:12.237831+0000","last_peered":"2023-10-09T05:58:12.237831+0000","last_clean":"2023-10-09T05:58:12.237831+0000","last_became_active":"2023-10-09T05:57:23.469697+0000","last_became_peered":"2023-10-09T05:57:23.469697+0000","last_unstale":"2023-10-09T05:58:12.237831+0000","last_undegraded":"2023-10-09T05:58:12.237831+0000","last_fullsized":"2023-10-09T05:58:12.237831+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:21:07.179434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":156,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.239469+0000","last_change":"2023-10-09T05:55:59.409563+0000","last_active":"2023-10-09T05:58:12.239469+0000","last_peered":"2023-10-09T05:58:12.239469+0000","last_clean":"2023-10-09T05:58:12.239469+0000","last_became_active":"2023-10-09T05:55:59.409184+0000","last_became_peered":"2023-10-09T05:55:59.409184+0000","last_unstale":"2023-10-09T05:58:12.239469+0000","last_undegraded":"2023-10-09T05:58:12.239469+0000","last_fullsized":"2023-10-09T05:58:12.239469+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:39.513591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":673,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.321547+0000","last_change":"2023-10-09T05:57:26.501248+0000","last_active":"2023-10-09T05:58:17.321547+0000","last_peered":"2023-10-09T05:58:17.321547+0000","last_clean":"2023-10-09T05:58:17.321547+0000","last_became_active":"2023-10-09T05:57:26.500602+0000","last_became_peered":"2023-10-09T05:57:26.500602+0000","last_unstale":"2023-10-09T05:58:17.321547+0000","last_undegraded":"2023-10-09T05:58:17.321547+0000","last_fullsized":"2023-10-09T05:58:17.321547+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":171,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.246463+0000","last_change":"2023-10-09T05:55:59.298953+0000","last_active":"2023-10-09T05:58:12.246463+0000","last_peered":"2023-10-09T05:58:12.246463+0000","last_clean":"2023-10-09T05:58:12.246463+0000","last_became_active":"2023-10-09T05:55:59.298624+0000","last_became_peered":"2023-10-09T05:55:59.298624+0000","last_unstale":"2023-10-09T05:58:12.246463+0000","last_undegraded":"2023-10-09T05:58:12.246463+0000","last_fullsized":"2023-10-09T05:58:12.246463+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:00.513870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":39,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.238733+0000","last_change":"2023-10-09T05:57:23.468480+0000","last_active":"2023-10-09T05:58:12.238733+0000","last_peered":"2023-10-09T05:58:12.238733+0000","last_clean":"2023-10-09T05:58:12.238733+0000","last_became_active":"2023-10-09T05:57:23.468179+0000","last_became_peered":"2023-10-09T05:57:23.468179+0000","last_unstale":"2023-10-09T05:58:12.238733+0000","last_undegraded":"2023-10-09T05:58:12.238733+0000","last_fullsized":"2023-10-09T05:58:12.238733+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:18.157188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":140,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.246709+0000","last_change":"2023-10-09T05:56:49.249711+0000","last_active":"2023-10-09T05:58:12.246709+0000","last_peered":"2023-10-09T05:58:12.246709+0000","last_clean":"2023-10-09T05:58:12.246709+0000","last_became_active":"2023-10-09T05:56:49.249272+0000","last_became_peered":"2023-10-09T05:56:49.249272+0000","last_unstale":"2023-10-09T05:58:12.246709+0000","last_undegraded":"2023-10-09T05:58:12.246709+0000","last_fullsized":"2023-10-09T05:58:12.246709+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:42.596986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":467,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.244924+0000","last_change":"2023-10-09T05:56:49.237597+0000","last_active":"2023-10-09T05:58:12.244924+0000","last_peered":"2023-10-09T05:58:12.244924+0000","last_clean":"2023-10-09T05:58:12.244924+0000","last_became_active":"2023-10-09T05:56:49.237269+0000","last_became_peered":"2023-10-09T05:56:49.237269+0000","last_unstale":"2023-10-09T05:58:12.244924+0000","last_undegraded":"2023-10-09T05:58:12.244924+0000","last_fullsized":"2023-10-09T05:58:12.244924+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":300,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.315913+0000","last_change":"2023-10-09T05:56:49.254337+0000","last_active":"2023-10-09T05:58:17.315913+0000","last_peered":"2023-10-09T05:58:17.315913+0000","last_clean":"2023-10-09T05:58:17.315913+0000","last_became_active":"2023-10-09T05:56:49.247892+0000","last_became_peered":"2023-10-09T05:56:49.247892+0000","last_unstale":"2023-10-09T05:58:17.315913+0000","last_undegraded":"2023-10-09T05:58:17.315913+0000","last_fullsized":"2023-10-09T05:58:17.315913+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:44:20.157638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":254,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.246269+0000","last_change":"2023-10-09T05:51:57.203142+0000","last_active":"2023-10-09T05:58:12.246269+0000","last_peered":"2023-10-09T05:58:12.246269+0000","last_clean":"2023-10-09T05:58:12.246269+0000","last_became_active":"2023-10-09T05:51:57.202850+0000","last_became_peered":"2023-10-09T05:51:57.202850+0000","last_unstale":"2023-10-09T05:58:12.246269+0000","last_undegraded":"2023-10-09T05:58:12.246269+0000","last_fullsized":"2023-10-09T05:58:12.246269+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:51.090737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":202,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T05:58:16.302436+0000","last_change":"2023-10-09T05:57:51.789746+0000","last_active":"2023-10-09T05:58:16.302436+0000","last_peered":"2023-10-09T05:58:16.302436+0000","last_clean":"2023-10-09T05:58:16.302436+0000","last_became_active":"2023-10-09T05:57:51.789353+0000","last_became_peered":"2023-10-09T05:57:51.789353+0000","last_unstale":"2023-10-09T05:58:16.302436+0000","last_undegraded":"2023-10-09T05:58:16.302436+0000","last_fullsized":"2023-10-09T05:58:16.302436+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:28.241819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":574,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.276758+0000","last_change":"2023-10-09T05:58:01.899855+0000","last_active":"2023-10-09T05:58:12.276758+0000","last_peered":"2023-10-09T05:58:12.276758+0000","last_clean":"2023-10-09T05:58:12.276758+0000","last_became_active":"2023-10-09T05:58:01.899103+0000","last_became_peered":"2023-10-09T05:58:01.899103+0000","last_unstale":"2023-10-09T05:58:12.276758+0000","last_undegraded":"2023-10-09T05:58:12.276758+0000","last_fullsized":"2023-10-09T05:58:12.276758+0000","mapping_epoch":268,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":269,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":60,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T05:58:16.296389+0000","last_change":"2023-10-09T05:57:23.465501+0000","last_active":"2023-10-09T05:58:16.296389+0000","last_peered":"2023-10-09T05:58:16.296389+0000","last_clean":"2023-10-09T05:58:16.296389+0000","last_became_active":"2023-10-09T05:57:23.465103+0000","last_became_peered":"2023-10-09T05:57:23.465103+0000","last_unstale":"2023-10-09T05:58:16.296389+0000","last_undegraded":"2023-10-09T05:58:16.296389+0000","last_fullsized":"2023-10-09T05:58:16.296389+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:02:45.891751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":168,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T05:58:16.296114+0000","last_change":"2023-10-09T05:55:59.308743+0000","last_active":"2023-10-09T05:58:16.296114+0000","last_peered":"2023-10-09T05:58:16.296114+0000","last_clean":"2023-10-09T05:58:16.296114+0000","last_became_active":"2023-10-09T05:55:59.308220+0000","last_became_peered":"2023-10-09T05:55:59.308220+0000","last_unstale":"2023-10-09T05:58:16.296114+0000","last_undegraded":"2023-10-09T05:58:16.296114+0000","last_fullsized":"2023-10-09T05:58:16.296114+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:44.246081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":171,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.247004+0000","last_change":"2023-10-09T05:56:49.238219+0000","last_active":"2023-10-09T05:58:12.247004+0000","last_peered":"2023-10-09T05:58:12.247004+0000","last_clean":"2023-10-09T05:58:12.247004+0000","last_became_active":"2023-10-09T05:56:49.237919+0000","last_became_peered":"2023-10-09T05:56:49.237919+0000","last_unstale":"2023-10-09T05:58:12.247004+0000","last_undegraded":"2023-10-09T05:58:12.247004+0000","last_fullsized":"2023-10-09T05:58:12.247004+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:44.599032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":206,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T05:58:16.303007+0000","last_change":"2023-10-09T05:56:49.233224+0000","last_active":"2023-10-09T05:58:16.303007+0000","last_peered":"2023-10-09T05:58:16.303007+0000","last_clean":"2023-10-09T05:58:16.303007+0000","last_became_active":"2023-10-09T05:56:49.232971+0000","last_became_peered":"2023-10-09T05:56:49.232971+0000","last_unstale":"2023-10-09T05:58:16.303007+0000","last_undegraded":"2023-10-09T05:58:16.303007+0000","last_fullsized":"2023-10-09T05:58:16.303007+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:55.265488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":671,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.277532+0000","last_change":"2023-10-09T05:57:10.809854+0000","last_active":"2023-10-09T05:58:12.277532+0000","last_peered":"2023-10-09T05:58:12.277532+0000","last_clean":"2023-10-09T05:58:12.277532+0000","last_became_active":"2023-10-09T05:53:59.202004+0000","last_became_peered":"2023-10-09T05:53:59.202004+0000","last_unstale":"2023-10-09T05:58:12.277532+0000","last_undegraded":"2023-10-09T05:58:12.277532+0000","last_fullsized":"2023-10-09T05:58:12.277532+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:51:12.346243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:51:12.346243+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":39,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.238262+0000","last_change":"2023-10-09T05:57:23.479174+0000","last_active":"2023-10-09T05:58:12.238262+0000","last_peered":"2023-10-09T05:58:12.238262+0000","last_clean":"2023-10-09T05:58:12.238262+0000","last_became_active":"2023-10-09T05:57:23.478897+0000","last_became_peered":"2023-10-09T05:57:23.478897+0000","last_unstale":"2023-10-09T05:58:12.238262+0000","last_undegraded":"2023-10-09T05:58:12.238262+0000","last_fullsized":"2023-10-09T05:58:12.238262+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:00.160042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":242,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.321507+0000","last_change":"2023-10-09T05:53:59.195325+0000","last_active":"2023-10-09T05:58:17.321507+0000","last_peered":"2023-10-09T05:58:17.321507+0000","last_clean":"2023-10-09T05:58:17.321507+0000","last_became_active":"2023-10-09T05:53:59.194954+0000","last_became_peered":"2023-10-09T05:53:59.194954+0000","last_unstale":"2023-10-09T05:58:17.321507+0000","last_undegraded":"2023-10-09T05:58:17.321507+0000","last_fullsized":"2023-10-09T05:58:17.321507+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:23.228416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":465,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.318927+0000","last_change":"2023-10-09T05:57:54.797571+0000","last_active":"2023-10-09T05:58:17.318927+0000","last_peered":"2023-10-09T05:58:17.318927+0000","last_clean":"2023-10-09T05:58:17.318927+0000","last_became_active":"2023-10-09T05:57:54.796937+0000","last_became_peered":"2023-10-09T05:57:54.796937+0000","last_unstale":"2023-10-09T05:58:17.318927+0000","last_undegraded":"2023-10-09T05:58:17.318927+0000","last_fullsized":"2023-10-09T05:58:17.318927+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":143,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.276637+0000","last_change":"2023-10-09T05:55:59.300015+0000","last_active":"2023-10-09T05:58:12.276637+0000","last_peered":"2023-10-09T05:58:12.276637+0000","last_clean":"2023-10-09T05:58:12.276637+0000","last_became_active":"2023-10-09T05:55:59.299281+0000","last_became_peered":"2023-10-09T05:55:59.299281+0000","last_unstale":"2023-10-09T05:58:12.276637+0000","last_undegraded":"2023-10-09T05:58:12.276637+0000","last_fullsized":"2023-10-09T05:58:12.276637+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:50:32.465904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":214,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.245142+0000","last_change":"2023-10-09T05:57:23.471369+0000","last_active":"2023-10-09T05:58:12.245142+0000","last_peered":"2023-10-09T05:58:12.245142+0000","last_clean":"2023-10-09T05:58:12.245142+0000","last_became_active":"2023-10-09T05:57:23.471049+0000","last_became_peered":"2023-10-09T05:57:23.471049+0000","last_unstale":"2023-10-09T05:58:12.245142+0000","last_undegraded":"2023-10-09T05:58:12.245142+0000","last_fullsized":"2023-10-09T05:58:12.245142+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:21.208825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":330,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.246670+0000","last_change":"2023-10-09T05:57:10.348399+0000","last_active":"2023-10-09T05:58:12.246670+0000","last_peered":"2023-10-09T05:58:12.246670+0000","last_clean":"2023-10-09T05:58:12.246670+0000","last_became_active":"2023-10-09T05:57:10.348061+0000","last_became_peered":"2023-10-09T05:57:10.348061+0000","last_unstale":"2023-10-09T05:58:12.246670+0000","last_undegraded":"2023-10-09T05:58:12.246670+0000","last_fullsized":"2023-10-09T05:58:12.246670+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:12.453118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":663,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.321613+0000","last_change":"2023-10-09T05:57:26.518453+0000","last_active":"2023-10-09T05:58:17.321613+0000","last_peered":"2023-10-09T05:58:17.321613+0000","last_clean":"2023-10-09T05:58:17.321613+0000","last_became_active":"2023-10-09T05:57:26.517990+0000","last_became_peered":"2023-10-09T05:57:26.517990+0000","last_unstale":"2023-10-09T05:58:17.321613+0000","last_undegraded":"2023-10-09T05:58:17.321613+0000","last_fullsized":"2023-10-09T05:58:17.321613+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":111,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.277385+0000","last_change":"2023-10-09T05:53:59.201576+0000","last_active":"2023-10-09T05:58:12.277385+0000","last_peered":"2023-10-09T05:58:12.277385+0000","last_clean":"2023-10-09T05:58:12.277385+0000","last_became_active":"2023-10-09T05:53:59.201309+0000","last_became_peered":"2023-10-09T05:53:59.201309+0000","last_unstale":"2023-10-09T05:58:12.277385+0000","last_undegraded":"2023-10-09T05:58:12.277385+0000","last_fullsized":"2023-10-09T05:58:12.277385+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:54.956371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":51,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T05:58:16.302394+0000","last_change":"2023-10-09T05:57:10.368477+0000","last_active":"2023-10-09T05:58:16.302394+0000","last_peered":"2023-10-09T05:58:16.302394+0000","last_clean":"2023-10-09T05:58:16.302394+0000","last_became_active":"2023-10-09T05:57:10.368178+0000","last_became_peered":"2023-10-09T05:57:10.368178+0000","last_unstale":"2023-10-09T05:58:16.302394+0000","last_undegraded":"2023-10-09T05:58:16.302394+0000","last_fullsized":"2023-10-09T05:58:16.302394+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:38.053000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":202,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.245676+0000","last_change":"2023-10-09T05:53:59.188462+0000","last_active":"2023-10-09T05:58:12.245676+0000","last_peered":"2023-10-09T05:58:12.245676+0000","last_clean":"2023-10-09T05:58:12.245676+0000","last_became_active":"2023-10-09T05:53:59.188061+0000","last_became_peered":"2023-10-09T05:53:59.188061+0000","last_unstale":"2023-10-09T05:58:12.245676+0000","last_undegraded":"2023-10-09T05:58:12.245676+0000","last_fullsized":"2023-10-09T05:58:12.245676+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:19.167114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":631,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.245795+0000","last_change":"2023-10-09T05:57:13.235727+0000","last_active":"2023-10-09T05:58:12.245795+0000","last_peered":"2023-10-09T05:58:12.245795+0000","last_clean":"2023-10-09T05:58:12.245795+0000","last_became_active":"2023-10-09T05:57:13.235353+0000","last_became_peered":"2023-10-09T05:57:13.235353+0000","last_unstale":"2023-10-09T05:58:12.245795+0000","last_undegraded":"2023-10-09T05:58:12.245795+0000","last_fullsized":"2023-10-09T05:58:12.245795+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":283,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.239515+0000","last_change":"2023-10-09T05:53:59.228948+0000","last_active":"2023-10-09T05:58:12.239515+0000","last_peered":"2023-10-09T05:58:12.239515+0000","last_clean":"2023-10-09T05:58:12.239515+0000","last_became_active":"2023-10-09T05:53:59.186359+0000","last_became_peered":"2023-10-09T05:53:59.186359+0000","last_unstale":"2023-10-09T05:58:12.239515+0000","last_undegraded":"2023-10-09T05:58:12.239515+0000","last_fullsized":"2023-10-09T05:58:12.239515+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:39.990576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":137,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.321532+0000","last_change":"2023-10-09T05:55:59.339957+0000","last_active":"2023-10-09T05:58:17.321532+0000","last_peered":"2023-10-09T05:58:17.321532+0000","last_clean":"2023-10-09T05:58:17.321532+0000","last_became_active":"2023-10-09T05:55:59.339614+0000","last_became_peered":"2023-10-09T05:55:59.339614+0000","last_unstale":"2023-10-09T05:58:17.321532+0000","last_undegraded":"2023-10-09T05:58:17.321532+0000","last_fullsized":"2023-10-09T05:58:17.321532+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:01.613113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":39,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.238100+0000","last_change":"2023-10-09T05:57:23.477143+0000","last_active":"2023-10-09T05:58:12.238100+0000","last_peered":"2023-10-09T05:58:12.238100+0000","last_clean":"2023-10-09T05:58:12.238100+0000","last_became_active":"2023-10-09T05:57:23.475655+0000","last_became_peered":"2023-10-09T05:57:23.475655+0000","last_unstale":"2023-10-09T05:58:12.238100+0000","last_undegraded":"2023-10-09T05:58:12.238100+0000","last_fullsized":"2023-10-09T05:58:12.238100+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:29.528023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":521,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.238164+0000","last_change":"2023-10-09T05:57:26.503736+0000","last_active":"2023-10-09T05:58:12.238164+0000","last_peered":"2023-10-09T05:58:12.238164+0000","last_clean":"2023-10-09T05:58:12.238164+0000","last_became_active":"2023-10-09T05:57:26.502911+0000","last_became_peered":"2023-10-09T05:57:26.502911+0000","last_unstale":"2023-10-09T05:58:12.238164+0000","last_undegraded":"2023-10-09T05:58:12.238164+0000","last_fullsized":"2023-10-09T05:58:12.238164+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":284,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T05:58:17.321596+0000","last_change":"2023-10-09T05:57:58.832790+0000","last_active":"2023-10-09T05:58:17.321596+0000","last_peered":"2023-10-09T05:58:17.321596+0000","last_clean":"2023-10-09T05:58:17.321596+0000","last_became_active":"2023-10-09T05:57:58.832471+0000","last_became_peered":"2023-10-09T05:57:58.832471+0000","last_unstale":"2023-10-09T05:58:17.321596+0000","last_undegraded":"2023-10-09T05:58:17.321596+0000","last_fullsized":"2023-10-09T05:58:17.321596+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:39.358174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":316,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.246800+0000","last_change":"2023-10-09T05:57:58.836050+0000","last_active":"2023-10-09T05:58:12.246800+0000","last_peered":"2023-10-09T05:58:12.246800+0000","last_clean":"2023-10-09T05:58:12.246800+0000","last_became_active":"2023-10-09T05:57:58.833691+0000","last_became_peered":"2023-10-09T05:57:58.833691+0000","last_unstale":"2023-10-09T05:58:12.246800+0000","last_undegraded":"2023-10-09T05:58:12.246800+0000","last_fullsized":"2023-10-09T05:58:12.246800+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:32.257849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":219,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.245338+0000","last_change":"2023-10-09T05:40:12.676587+0000","last_active":"2023-10-09T05:58:12.245338+0000","last_peered":"2023-10-09T05:58:12.245338+0000","last_clean":"2023-10-09T05:58:12.245338+0000","last_became_active":"2023-10-09T05:40:12.676330+0000","last_became_peered":"2023-10-09T05:40:12.676330+0000","last_unstale":"2023-10-09T05:58:12.245338+0000","last_undegraded":"2023-10-09T05:58:12.245338+0000","last_fullsized":"2023-10-09T05:58:12.245338+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:46.659556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":206,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.245708+0000","last_change":"2023-10-09T05:53:59.191302+0000","last_active":"2023-10-09T05:58:12.245708+0000","last_peered":"2023-10-09T05:58:12.245708+0000","last_clean":"2023-10-09T05:58:12.245708+0000","last_became_active":"2023-10-09T05:53:59.190911+0000","last_became_peered":"2023-10-09T05:53:59.190911+0000","last_unstale":"2023-10-09T05:58:12.245708+0000","last_undegraded":"2023-10-09T05:58:12.245708+0000","last_fullsized":"2023-10-09T05:58:12.245708+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:12.801201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":60,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T05:58:16.302279+0000","last_change":"2023-10-09T05:57:23.469986+0000","last_active":"2023-10-09T05:58:16.302279+0000","last_peered":"2023-10-09T05:58:16.302279+0000","last_clean":"2023-10-09T05:58:16.302279+0000","last_became_active":"2023-10-09T05:57:23.469546+0000","last_became_peered":"2023-10-09T05:57:23.469546+0000","last_unstale":"2023-10-09T05:58:16.302279+0000","last_undegraded":"2023-10-09T05:58:16.302279+0000","last_fullsized":"2023-10-09T05:58:16.302279+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:53.762345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":67,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.246679+0000","last_change":"2023-10-09T05:56:49.244931+0000","last_active":"2023-10-09T05:58:12.246679+0000","last_peered":"2023-10-09T05:58:12.246679+0000","last_clean":"2023-10-09T05:58:12.246679+0000","last_became_active":"2023-10-09T05:56:49.244651+0000","last_became_peered":"2023-10-09T05:56:49.244651+0000","last_unstale":"2023-10-09T05:58:12.246679+0000","last_undegraded":"2023-10-09T05:58:12.246679+0000","last_fullsized":"2023-10-09T05:58:12.246679+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:53.906578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":313,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.246343+0000","last_change":"2023-10-09T05:57:10.356351+0000","last_active":"2023-10-09T05:58:12.246343+0000","last_peered":"2023-10-09T05:58:12.246343+0000","last_clean":"2023-10-09T05:58:12.246343+0000","last_became_active":"2023-10-09T05:57:10.356037+0000","last_became_peered":"2023-10-09T05:57:10.356037+0000","last_unstale":"2023-10-09T05:58:12.246343+0000","last_undegraded":"2023-10-09T05:58:12.246343+0000","last_fullsized":"2023-10-09T05:58:12.246343+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:15.139912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":67,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.246004+0000","last_change":"2023-10-09T05:56:49.245612+0000","last_active":"2023-10-09T05:58:12.246004+0000","last_peered":"2023-10-09T05:58:12.246004+0000","last_clean":"2023-10-09T05:58:12.246004+0000","last_became_active":"2023-10-09T05:56:49.245235+0000","last_became_peered":"2023-10-09T05:56:49.245235+0000","last_unstale":"2023-10-09T05:58:12.246004+0000","last_undegraded":"2023-10-09T05:58:12.246004+0000","last_fullsized":"2023-10-09T05:58:12.246004+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:29.016794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":270,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.246290+0000","last_change":"2023-10-09T05:51:57.093104+0000","last_active":"2023-10-09T05:58:12.246290+0000","last_peered":"2023-10-09T05:58:12.246290+0000","last_clean":"2023-10-09T05:58:12.246290+0000","last_became_active":"2023-10-09T05:51:57.092810+0000","last_became_peered":"2023-10-09T05:51:57.092810+0000","last_unstale":"2023-10-09T05:58:12.246290+0000","last_undegraded":"2023-10-09T05:58:12.246290+0000","last_fullsized":"2023-10-09T05:58:12.246290+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:51.192227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":39,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.239077+0000","last_change":"2023-10-09T05:57:23.479560+0000","last_active":"2023-10-09T05:58:12.239077+0000","last_peered":"2023-10-09T05:58:12.239077+0000","last_clean":"2023-10-09T05:58:12.239077+0000","last_became_active":"2023-10-09T05:57:23.479177+0000","last_became_peered":"2023-10-09T05:57:23.479177+0000","last_unstale":"2023-10-09T05:58:12.239077+0000","last_undegraded":"2023-10-09T05:58:12.239077+0000","last_fullsized":"2023-10-09T05:58:12.239077+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:41.925161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":39,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.237961+0000","last_change":"2023-10-09T05:57:23.475179+0000","last_active":"2023-10-09T05:58:12.237961+0000","last_peered":"2023-10-09T05:58:12.237961+0000","last_clean":"2023-10-09T05:58:12.237961+0000","last_became_active":"2023-10-09T05:57:23.474904+0000","last_became_peered":"2023-10-09T05:57:23.474904+0000","last_unstale":"2023-10-09T05:58:12.237961+0000","last_undegraded":"2023-10-09T05:58:12.237961+0000","last_fullsized":"2023-10-09T05:58:12.237961+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:54.587601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":244,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.247115+0000","last_change":"2023-10-09T05:57:10.379258+0000","last_active":"2023-10-09T05:58:12.247115+0000","last_peered":"2023-10-09T05:58:12.247115+0000","last_clean":"2023-10-09T05:58:12.247115+0000","last_became_active":"2023-10-09T05:57:10.359745+0000","last_became_peered":"2023-10-09T05:57:10.359745+0000","last_unstale":"2023-10-09T05:58:12.247115+0000","last_undegraded":"2023-10-09T05:58:12.247115+0000","last_fullsized":"2023-10-09T05:58:12.247115+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:14.750256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":296,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T05:58:12.245626+0000","last_change":"2023-10-09T05:57:23.481065+0000","last_active":"2023-10-09T05:58:12.245626+0000","last_peered":"2023-10-09T05:58:12.245626+0000","last_clean":"2023-10-09T05:58:12.245626+0000","last_became_active":"2023-10-09T05:57:23.480203+0000","last_became_peered":"2023-10-09T05:57:23.480203+0000","last_unstale":"2023-10-09T05:58:12.245626+0000","last_undegraded":"2023-10-09T05:58:12.245626+0000","last_fullsized":"2023-10-09T05:58:12.245626+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:15:53.196457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":10792,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":572478,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":472,"num_read_kb":885,"num_write":232,"num_write_kb":2343,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":2635482,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3220,"ondisk_log_size":3220,"up":63,"acting":62,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234223,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":316984,"kb_used_data":3312,"kb_used_omap":0,"kb_used_meta":313664,"kb_avail":93428168,"statfs":{"total":95995035648,"available":95670444032,"internally_reserved":0,"allocated":3391488,"data_stored":2369001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321191936},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:02 2023","interfaces":[{"interface":"back","average":{"1min":1.9319999999999999,"5min":1.105,"15min":1.7949999999999999},"min":{"1min":0.246,"5min":0.22,"15min":0.22},"max":{"1min":21.038,"5min":21.038,"15min":21.038},"last":0.52700000000000002},{"interface":"front","average":{"1min":2.0129999999999999,"5min":1.1100000000000001,"15min":1.6859999999999999},"min":{"1min":0.245,"5min":0.224,"15min":0.224},"max":{"1min":21.152000000000001,"5min":21.152000000000001,"15min":21.152000000000001},"last":0.308}]},{"osd":1,"last update":"Mon Oct 9 05:57:26 2023","interfaces":[{"interface":"back","average":{"1min":4.3209999999999997,"5min":1.54,"15min":0.94799999999999995},"min":{"1min":0.30299999999999999,"5min":0.27200000000000002,"15min":0.22900000000000001},"max":{"1min":35.113999999999997,"5min":35.113999999999997,"15min":35.113999999999997},"last":0.46400000000000002},{"interface":"front","average":{"1min":3.383,"5min":1.2270000000000001,"15min":0.85499999999999998},"min":{"1min":0.317,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":34.768999999999998,"5min":34.768999999999998,"15min":34.768999999999998},"last":0.39600000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:58:02 2023","interfaces":[{"interface":"back","average":{"1min":12.797000000000001,"5min":3.3069999999999999,"15min":2.0230000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28199999999999997},"max":{"1min":119.91200000000001,"5min":119.91200000000001,"15min":119.91200000000001},"last":0.42199999999999999},{"interface":"front","average":{"1min":12.898999999999999,"5min":3.3580000000000001,"15min":2.117},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":119.95099999999999,"5min":119.95099999999999,"15min":119.95099999999999},"last":0.47299999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:57:32 2023","interfaces":[{"interface":"back","average":{"1min":7.0090000000000003,"5min":2.5270000000000001,"15min":2.9289999999999998},"min":{"1min":0.38700000000000001,"5min":0.35999999999999999,"15min":0.33300000000000002},"max":{"1min":70.840999999999994,"5min":70.840999999999994,"15min":147.54900000000001},"last":0.499},{"interface":"front","average":{"1min":6.6109999999999998,"5min":2.4199999999999999,"15min":2.8250000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":71.212000000000003,"5min":71.212000000000003,"15min":144.065},"last":0.48499999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":0.66500000000000004,"15min":0.60899999999999999},"min":{"1min":0.27500000000000002,"5min":0.27000000000000002,"15min":0.24399999999999999},"max":{"1min":2.9990000000000001,"5min":3.7120000000000002,"15min":9.5329999999999995},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.60599999999999998,"15min":0.66100000000000003},"min":{"1min":0.33500000000000002,"5min":0.29899999999999999,"15min":0.27400000000000002},"max":{"1min":3.835,"5min":3.835,"15min":7.6120000000000001},"last":0.56100000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.91900000000000004,"15min":0.75800000000000001},"min":{"1min":0.29799999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.8090000000000002,"5min":15.622,"15min":15.622},"last":0.38},{"interface":"front","average":{"1min":0.76400000000000001,"5min":0.64000000000000001,"15min":0.624},"min":{"1min":0.33300000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.3969999999999998,"5min":3.1869999999999998,"15min":3.9380000000000002},"last":0.51800000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":0.86599999999999999,"15min":1.2769999999999999},"min":{"1min":0.29499999999999998,"5min":0.28799999999999998,"15min":0.28299999999999997},"max":{"1min":3.2229999999999999,"5min":12.727,"15min":131.94399999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.82099999999999995,"5min":0.70099999999999996,"15min":1.258},"min":{"1min":0.378,"5min":0.33100000000000002,"15min":0.29999999999999999},"max":{"1min":3.5859999999999999,"5min":3.5859999999999999,"15min":135.23400000000001},"last":0.58899999999999997}]}]},{"osd":6,"up_from":46,"seq":197568495860,"num_pgs":87,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":366268,"kb_used_data":64884,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93378884,"statfs":{"total":95995035648,"available":95619977216,"internally_reserved":0,"allocated":66441216,"data_stored":65429943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":14,"apply_latency_ms":14,"commit_latency_ns":14000000,"apply_latency_ns":14000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2200000000000002,"5min":1.534,"15min":1.286},"min":{"1min":0.23499999999999999,"5min":0.224,"15min":0.224},"max":{"1min":13.026999999999999,"5min":49.389000000000003,"15min":49.389000000000003},"last":0.27100000000000002},{"interface":"front","average":{"1min":1.8280000000000001,"5min":1.4710000000000001,"15min":1.3089999999999999},"min":{"1min":0.26300000000000001,"5min":0.253,"15min":0.253},"max":{"1min":12.629,"5min":49.338999999999999,"15min":49.338999999999999},"last":0.39500000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:57:40 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":0.96399999999999997,"15min":0.77200000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.23899999999999999},"max":{"1min":17.341000000000001,"5min":17.341000000000001,"15min":17.341000000000001},"last":0.32200000000000001},{"interface":"front","average":{"1min":1.867,"5min":0.96799999999999997,"15min":0.81499999999999995},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":12.474,"5min":12.474,"15min":12.474},"last":0.38300000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:58:02 2023","interfaces":[{"interface":"back","average":{"1min":6.8380000000000001,"5min":2.097,"15min":1.1899999999999999},"min":{"1min":0.311,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":61.088999999999999,"5min":61.088999999999999,"15min":61.088999999999999},"last":0.42499999999999999},{"interface":"front","average":{"1min":6.3579999999999997,"5min":2.1040000000000001,"15min":1.1879999999999999},"min":{"1min":0.38800000000000001,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":61.082000000000001,"5min":61.082000000000001,"15min":61.082000000000001},"last":0.34200000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:57:40 2023","interfaces":[{"interface":"back","average":{"1min":2.6619999999999999,"5min":3.714,"15min":1.7809999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":20.478000000000002,"5min":93.626000000000005,"15min":93.626000000000005},"last":0.40999999999999998},{"interface":"front","average":{"1min":2.7450000000000001,"5min":3.444,"15min":1.7030000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":20.5,"5min":93.643000000000001,"15min":93.643000000000001},"last":0.47499999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:57:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":0.72599999999999998,"15min":0.71099999999999997},"min":{"1min":0.252,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.5709999999999997,"5min":4.5709999999999997,"15min":7.9100000000000001},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.2629999999999999,"5min":0.84999999999999998,"15min":0.73099999999999998},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.2069999999999999,"5min":8.734,"15min":8.734},"last":0.501}]},{"osd":5,"last update":"Mon Oct 9 05:57:48 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":1.284,"15min":0.89500000000000002},"min":{"1min":0.316,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.5910000000000002,"5min":45.933999999999997,"15min":45.933999999999997},"last":0.437},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.2669999999999999,"15min":0.92400000000000004},"min":{"1min":0.35899999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.786,"5min":45.917999999999999,"15min":45.917999999999999},"last":0.46300000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":1.337,"15min":1.04},"min":{"1min":0.33500000000000002,"5min":0.314,"15min":0.314},"max":{"1min":4.4820000000000002,"5min":52.042999999999999,"15min":52.042999999999999},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.093,"5min":1.3080000000000001,"15min":1.008},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.9539999999999997,"5min":51.259,"15min":51.259},"last":0.48399999999999999}]}]},{"osd":1,"up_from":0,"seq":382252089448,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":129,"seq":554050781250,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":141512,"kb_used_data":3648,"kb_used_omap":29,"kb_used_meta":137826,"kb_avail":93603640,"statfs":{"total":95995035648,"available":95850127360,"internally_reserved":0,"allocated":3735552,"data_stored":2115218,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30322,"internal_metadata":141134222},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:58:04 2023","interfaces":[{"interface":"back","average":{"1min":1.387,"5min":0.94899999999999995,"15min":0.78400000000000003},"min":{"1min":0.21299999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":6.6120000000000001,"5min":9.1720000000000006,"15min":9.1720000000000006},"last":0.22600000000000001},{"interface":"front","average":{"1min":0.99399999999999999,"5min":0.84999999999999998,"15min":1.048},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":3.4449999999999998,"5min":6.6420000000000003,"15min":6.6420000000000003},"last":0.249}]},{"osd":2,"last update":"Mon Oct 9 05:58:04 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":1.204,"15min":1.0940000000000001},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":13.471,"5min":13.471,"15min":13.471},"last":0.28499999999999998},{"interface":"front","average":{"1min":2.4500000000000002,"5min":1.1200000000000001,"15min":1.046},"min":{"1min":0.245,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":13.561,"5min":13.561,"15min":13.561},"last":0.38500000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:58:04 2023","interfaces":[{"interface":"back","average":{"1min":4.6760000000000002,"5min":1.635,"15min":1.0609999999999999},"min":{"1min":0.29499999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":27.954000000000001,"5min":27.954000000000001,"15min":27.954000000000001},"last":0.31900000000000001},{"interface":"front","average":{"1min":3.851,"5min":1.4259999999999999,"15min":1.0760000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":20.027000000000001,"5min":20.027000000000001,"15min":20.027000000000001},"last":0.436}]},{"osd":4,"last update":"Mon Oct 9 05:58:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":3.9910000000000001,"15min":2.145},"min":{"1min":0.316,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.3639999999999999,"5min":149.07400000000001,"15min":149.07400000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":1.2110000000000001,"5min":3.8399999999999999,"15min":1.8819999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.4509999999999996,"5min":149.12700000000001,"15min":149.12700000000001},"last":0.34699999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:58:04 2023","interfaces":[{"interface":"back","average":{"1min":1.873,"5min":2.4510000000000001,"15min":1.583},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":11.09,"5min":100.25,"15min":100.25},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.6459999999999999,"5min":2.4550000000000001,"15min":1.4279999999999999},"min":{"1min":0.39800000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":5.077,"5min":102.40300000000001,"15min":102.40300000000001},"last":0.35899999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:58:04 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":0.94599999999999995,"15min":1.21},"min":{"1min":0.35199999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":6.5439999999999996,"5min":8.1470000000000002,"15min":8.1470000000000002},"last":0.63500000000000001},{"interface":"front","average":{"1min":1.825,"5min":1.2350000000000001,"15min":1.2290000000000001},"min":{"1min":0.44500000000000001,"5min":0.378,"15min":0.378},"max":{"1min":4.4000000000000004,"5min":9.141,"15min":9.141},"last":0.44700000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:58:04 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":2.0470000000000002,"15min":1.629},"min":{"1min":0.42399999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":4.6139999999999999,"5min":99.489000000000004,"15min":99.489000000000004},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.032,"5min":1.7909999999999999,"15min":1.534},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":3.581,"5min":99.510000000000005,"15min":99.510000000000005},"last":0.42399999999999999}]}]},{"osd":2,"up_from":94,"seq":403726925940,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":936452,"kb_used_data":667196,"kb_used_omap":81,"kb_used_meta":269166,"kb_avail":92808700,"statfs":{"total":95995035648,"available":95036108800,"internally_reserved":0,"allocated":683208704,"data_stored":681712670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":275626202},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:11 2023","interfaces":[{"interface":"back","average":{"1min":4.9169999999999998,"5min":2.056,"15min":1.8169999999999999},"min":{"1min":0.23300000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":55.409999999999997,"5min":55.409999999999997,"15min":55.409999999999997},"last":0.745},{"interface":"front","average":{"1min":4.7009999999999996,"5min":2.056,"15min":1.726},"min":{"1min":0.28199999999999997,"5min":0.23899999999999999,"15min":0.23100000000000001},"max":{"1min":55.326000000000001,"5min":55.326000000000001,"15min":55.326000000000001},"last":0.72899999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:58:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.0069999999999999,"15min":1.357},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":3.8959999999999999,"5min":7.7119999999999997,"15min":163.24199999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":1.4550000000000001,"5min":0.85299999999999998,"15min":1.272},"min":{"1min":0.223,"5min":0.223,"15min":0.214},"max":{"1min":6.9790000000000001,"5min":6.9790000000000001,"15min":163.27000000000001},"last":0.81599999999999995}]},{"osd":3,"last update":"Mon Oct 9 05:57:32 2023","interfaces":[{"interface":"back","average":{"1min":7.1760000000000002,"5min":2.0089999999999999,"15min":1.7549999999999999},"min":{"1min":0.27500000000000002,"5min":0.25900000000000001,"15min":0.23899999999999999},"max":{"1min":55.118000000000002,"5min":55.118000000000002,"15min":170.07599999999999},"last":1.464},{"interface":"front","average":{"1min":6.8540000000000001,"5min":2.0499999999999998,"15min":1.345},"min":{"1min":0.33400000000000002,"5min":0.29799999999999999,"15min":0.25},"max":{"1min":55.365000000000002,"5min":55.365000000000002,"15min":63.600000000000001},"last":0.39200000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:58:05 2023","interfaces":[{"interface":"back","average":{"1min":1.7529999999999999,"5min":1.1060000000000001,"15min":0.95299999999999996},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28199999999999997},"max":{"1min":8.6280000000000001,"5min":8.6280000000000001,"15min":11.372999999999999},"last":0.48099999999999998},{"interface":"front","average":{"1min":1.391,"5min":0.93700000000000006,"15min":0.90100000000000002},"min":{"1min":0.373,"5min":0.33200000000000002,"15min":0.26800000000000002},"max":{"1min":6.1710000000000003,"5min":6.1710000000000003,"15min":6.5709999999999997},"last":0.76300000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:58:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5860000000000001,"5min":1.3380000000000001,"15min":0.995},"min":{"1min":0.35099999999999998,"5min":0.30599999999999999,"15min":0.28999999999999998},"max":{"1min":5.3380000000000001,"5min":13.414,"15min":13.414},"last":8.8819999999999997},{"interface":"front","average":{"1min":2.101,"5min":1.242,"15min":0.97799999999999998},"min":{"1min":0.33000000000000002,"5min":0.313,"15min":0.29799999999999999},"max":{"1min":8.5570000000000004,"5min":13.438000000000001,"15min":13.438000000000001},"last":1.3600000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:58:05 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":0.96199999999999997,"15min":0.91100000000000003},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.29399999999999998},"max":{"1min":3.2999999999999998,"5min":8.6920000000000002,"15min":11.59},"last":1.718},{"interface":"front","average":{"1min":1.48,"5min":1.113,"15min":0.90900000000000003},"min":{"1min":0.39800000000000002,"5min":0.37,"15min":0.34000000000000002},"max":{"1min":4.4850000000000003,"5min":8.6140000000000008,"15min":8.6140000000000008},"last":8.9399999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:58:05 2023","interfaces":[{"interface":"back","average":{"1min":4.4020000000000001,"5min":1.8620000000000001,"15min":1.274},"min":{"1min":0.44700000000000001,"5min":0.36599999999999999,"15min":0.35899999999999999},"max":{"1min":55.052,"5min":55.052,"15min":55.052},"last":0.92400000000000004},{"interface":"front","average":{"1min":2.0110000000000001,"5min":1.252,"15min":1.093},"min":{"1min":0.438,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.6010000000000009,"5min":8.6010000000000009,"15min":40.113},"last":2.0779999999999998}]}]},{"osd":3,"up_from":99,"seq":425201762413,"num_pgs":77,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":792508,"kb_used_data":532788,"kb_used_omap":7,"kb_used_meta":259704,"kb_avail":92952644,"statfs":{"total":95995035648,"available":95183507456,"internally_reserved":0,"allocated":545574912,"data_stored":543900853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":265937512},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":0.93100000000000005,"15min":1.3420000000000001},"min":{"1min":0.224,"5min":0.186,"15min":0.186},"max":{"1min":4.7380000000000004,"5min":12.548,"15min":15.536},"last":0.27200000000000002},{"interface":"front","average":{"1min":2.0550000000000002,"5min":1.363,"15min":1.516},"min":{"1min":0.22800000000000001,"5min":0.214,"15min":0.214},"max":{"1min":14.247,"5min":14.707000000000001,"15min":14.707000000000001},"last":0.46300000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:57:38 2023","interfaces":[{"interface":"back","average":{"1min":1.4019999999999999,"5min":0.99399999999999999,"15min":1.105},"min":{"1min":0.217,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":14.568,"5min":14.568,"15min":127.226},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.82099999999999995,"15min":1.089},"min":{"1min":0.28599999999999998,"5min":0.24399999999999999,"15min":0.21099999999999999},"max":{"1min":2.5230000000000001,"5min":9.4239999999999995,"15min":127.468},"last":0.45100000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:57:38 2023","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":0.61499999999999999,"15min":0.75800000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":2.677,"5min":3.177,"15min":62.938000000000002},"last":0.375},{"interface":"front","average":{"1min":1.268,"5min":1.0169999999999999,"15min":1.004},"min":{"1min":0.30499999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":9.1039999999999992,"5min":9.8369999999999997,"15min":62.43},"last":0.33900000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:57:38 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":2.1829999999999998,"15min":1.2390000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.28199999999999997},"max":{"1min":13.154999999999999,"5min":142.994,"15min":142.994},"last":0.40999999999999998},{"interface":"front","average":{"1min":2.2570000000000001,"5min":2.4380000000000002,"15min":1.2869999999999999},"min":{"1min":0.33600000000000002,"5min":0.312,"15min":0.311},"max":{"1min":14.455,"5min":143.411,"15min":143.411},"last":0.61499999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:57:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6519999999999999,"5min":1.0309999999999999,"15min":0.82199999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.33800000000000002},"max":{"1min":14.404,"5min":14.404,"15min":14.404},"last":0.67600000000000005},{"interface":"front","average":{"1min":1.8859999999999999,"5min":0.93000000000000005,"15min":0.79200000000000004},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":18.013999999999999,"5min":18.013999999999999,"15min":18.013999999999999},"last":0.56599999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:57:38 2023","interfaces":[{"interface":"back","average":{"1min":3.8929999999999998,"5min":1.544,"15min":1.0409999999999999},"min":{"1min":0.44,"5min":0.374,"15min":0.30499999999999999},"max":{"1min":44.079000000000001,"5min":44.079000000000001,"15min":44.079000000000001},"last":0.64800000000000002},{"interface":"front","average":{"1min":3.5619999999999998,"5min":1.3140000000000001,"15min":0.99199999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":44.094000000000001,"5min":44.094000000000001,"15min":44.094000000000001},"last":0.69199999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:57:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0539999999999998,"5min":0.98599999999999999,"15min":0.85799999999999998},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":10.295,"5min":10.295,"15min":10.295},"last":0.59999999999999998},{"interface":"front","average":{"1min":1.0620000000000001,"5min":0.91900000000000004,"15min":0.84599999999999997},"min":{"1min":0.436,"5min":0.41399999999999998,"15min":0.40600000000000003},"max":{"1min":3.9689999999999999,"5min":9.3879999999999999,"15min":9.3879999999999999},"last":0.48499999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019133,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":433336,"kb_used_data":123760,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93311816,"statfs":{"total":95995035648,"available":95551299584,"internally_reserved":0,"allocated":126730240,"data_stored":125708006,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:08 2023","interfaces":[{"interface":"back","average":{"1min":1.3240000000000001,"5min":1.5429999999999999,"15min":1.0349999999999999},"min":{"1min":0.23300000000000001,"5min":0.222,"15min":0.222},"max":{"1min":7.6200000000000001,"5min":19.489000000000001,"15min":19.489000000000001},"last":0.316},{"interface":"front","average":{"1min":2.2909999999999999,"5min":1.754,"15min":1.607},"min":{"1min":0.248,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":8.9800000000000004,"5min":19.757000000000001,"15min":19.757000000000001},"last":0.47299999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:57:33 2023","interfaces":[{"interface":"back","average":{"1min":4.7229999999999999,"5min":1.5309999999999999,"15min":1.03},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":77.951999999999998,"5min":77.951999999999998,"15min":77.951999999999998},"last":0.372},{"interface":"front","average":{"1min":5.4809999999999999,"5min":1.7090000000000001,"15min":1.0269999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":77.968000000000004,"5min":77.968000000000004,"15min":77.968000000000004},"last":0.42199999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:58:08 2023","interfaces":[{"interface":"back","average":{"1min":7.7610000000000001,"5min":2.3260000000000001,"15min":1.391},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":136.15700000000001,"5min":136.15700000000001,"15min":136.15700000000001},"last":0.434},{"interface":"front","average":{"1min":7.968,"5min":2.5059999999999998,"15min":1.498},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":136.30500000000001,"5min":136.30500000000001,"15min":136.30500000000001},"last":1.2629999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:57:33 2023","interfaces":[{"interface":"back","average":{"1min":2.847,"5min":1.353,"15min":1.109},"min":{"1min":0.311,"5min":0.30599999999999999,"15min":0.29299999999999998},"max":{"1min":12.554,"5min":12.554,"15min":40.531999999999996},"last":0.44900000000000001},{"interface":"front","average":{"1min":2.8610000000000002,"5min":1.2749999999999999,"15min":1.127},"min":{"1min":0.377,"5min":0.31,"15min":0.31},"max":{"1min":12.403,"5min":12.403,"15min":39.905999999999999},"last":0.751}]},{"osd":5,"last update":"Mon Oct 9 05:57:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":0.61899999999999999,"15min":0.629},"min":{"1min":0.30299999999999999,"5min":0.27500000000000002,"15min":0.25900000000000001},"max":{"1min":4.9349999999999996,"5min":4.9349999999999996,"15min":10.012},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.5489999999999999,"5min":0.72599999999999998,"15min":0.73399999999999999},"min":{"1min":0.32000000000000001,"5min":0.254,"15min":0.254},"max":{"1min":12.121,"5min":12.121,"15min":12.489000000000001},"last":0.45900000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:57:45 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":0.69999999999999996,"15min":0.71999999999999997},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.4379999999999997,"5min":5.4379999999999997,"15min":11.234999999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":2.1230000000000002,"5min":0.85799999999999998,"15min":0.71499999999999997},"min":{"1min":0.39200000000000002,"5min":0.30599999999999999,"15min":0.30299999999999999},"max":{"1min":8.8610000000000007,"5min":8.8610000000000007,"15min":9.923},"last":0.40500000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:58:11 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.82299999999999995,"15min":0.85499999999999998},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.28599999999999998},"max":{"1min":2.5209999999999999,"5min":6.0570000000000004,"15min":10.606},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.98399999999999999,"5min":0.74399999999999999,"15min":0.747},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.145,"5min":5.524,"15min":8.3819999999999997},"last":0.39100000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757496,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":381932,"kb_used_data":72356,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93363220,"statfs":{"total":95995035648,"available":95603937280,"internally_reserved":0,"allocated":74092544,"data_stored":73078886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:10 2023","interfaces":[{"interface":"back","average":{"1min":2.8690000000000002,"5min":1.085,"15min":0.83299999999999996},"min":{"1min":0.26000000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":24.956,"5min":24.956,"15min":24.956},"last":0.47999999999999998},{"interface":"front","average":{"1min":3.2389999999999999,"5min":1.3149999999999999,"15min":0.96999999999999997},"min":{"1min":0.32100000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":24.975000000000001,"5min":24.975000000000001,"15min":24.975000000000001},"last":0.48999999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:57:33 2023","interfaces":[{"interface":"back","average":{"1min":1.3200000000000001,"5min":2.4900000000000002,"15min":1.6459999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.8410000000000002,"5min":99.605999999999995,"15min":114.247},"last":0.35099999999999998},{"interface":"front","average":{"1min":1.0089999999999999,"5min":2.1579999999999999,"15min":1.5269999999999999},"min":{"1min":0.28399999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.7130000000000001,"5min":99.671999999999997,"15min":114.678},"last":0.40000000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:57:59 2023","interfaces":[{"interface":"back","average":{"1min":12.375999999999999,"5min":3.2130000000000001,"15min":2.0150000000000001},"min":{"1min":0.28499999999999998,"5min":0.28399999999999997,"15min":0.25},"max":{"1min":217.06399999999999,"5min":217.06399999999999,"15min":217.06399999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":11.965999999999999,"5min":3.0870000000000002,"15min":1.9710000000000001},"min":{"1min":0.308,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":217.018,"5min":217.018,"15min":217.018},"last":0.36899999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:57:33 2023","interfaces":[{"interface":"back","average":{"1min":5.4450000000000003,"5min":3.8919999999999999,"15min":3.714},"min":{"1min":0.35799999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":60,"5min":100.774,"15min":272.81299999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":5.9500000000000002,"5min":4.274,"15min":3.9529999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":59.817,"5min":100.83499999999999,"15min":273.00999999999999},"last":0.42399999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:57:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":0.58599999999999997,"15min":0.58999999999999997},"min":{"1min":0.27000000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":3.1080000000000001,"5min":3.1080000000000001,"15min":6.2050000000000001},"last":0.318},{"interface":"front","average":{"1min":0.76400000000000001,"5min":0.57399999999999995,"15min":0.59699999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.26000000000000001},"max":{"1min":2.444,"5min":2.484,"15min":13.050000000000001},"last":0.442}]},{"osd":6,"last update":"Mon Oct 9 05:57:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":0.72599999999999998,"15min":0.65500000000000003},"min":{"1min":0.33600000000000002,"5min":0.30199999999999999,"15min":0.26600000000000001},"max":{"1min":10.94,"5min":10.94,"15min":10.94},"last":0.499},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.72399999999999998,"15min":0.71799999999999997},"min":{"1min":0.33100000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.5069999999999997,"5min":5.5069999999999997,"15min":5.9770000000000003},"last":0.46999999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:57:15 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":0.91600000000000004,"15min":0.78300000000000003},"min":{"1min":0.33100000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.5800000000000001,"5min":9.2739999999999991,"15min":9.2739999999999991},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.1679999999999999,"5min":0.68000000000000005,"15min":0.65200000000000002},"min":{"1min":0.40100000000000002,"5min":0.32100000000000001,"15min":0.32000000000000001},"max":{"1min":2.7530000000000001,"5min":2.7530000000000001,"15min":5.032},"last":0.45900000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1474,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:58:21.604 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T05:58:21.617 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:21.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:21 smithi156 ceph-mon[195101]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:21 smithi156 ceph-mon[195101]: osdmap e282: 8 total, 8 up, 7 in 2023-10-09T05:58:21.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:21 smithi156 ceph-mon[195101]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:21 smithi156 ceph-mon[195101]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:21 smithi156 ceph-mon[195101]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:21 smithi156 ceph-mon[195101]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:21 smithi156 ceph-mon[195101]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:21 smithi156 ceph-mon[195101]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:21 smithi156 ceph-mon[195101]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:21 smithi156 ceph-mon[195101]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:21 smithi156 ceph-mon[195101]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:21 smithi156 ceph-mon[195101]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:21 smithi156 ceph-mon[195101]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:21 smithi156 ceph-mon[195101]: failed to encode map e282 with expected crc 2023-10-09T05:58:21.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:21 smithi156 ceph-mon[195101]: pgmap v636: 93 pgs: 2 peering, 7 active+undersized+remapped, 1 remapped+peering, 83 active+clean; 582 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 160 KiB/s, 0 objects/s recovering 2023-10-09T05:58:21.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:21 smithi156 ceph-mon[195101]: osdmap e283: 8 total, 8 up, 7 in 2023-10-09T05:58:21.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:21.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:21.723 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:21.750 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:21.970 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:21.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:21.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:22.074 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:22.082 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:22.106 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:22.154 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:22.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:22.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:22.259 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:22.291 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:22.341 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:22.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:22.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:22.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:22 smithi156 ceph-mon[195101]: failed to encode map e283 with expected crc 2023-10-09T05:58:22.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:22 smithi156 ceph-mon[195101]: failed to encode map e283 with expected crc 2023-10-09T05:58:22.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:22 smithi156 ceph-mon[195101]: failed to encode map e283 with expected crc 2023-10-09T05:58:22.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:22 smithi156 ceph-mon[195101]: failed to encode map e283 with expected crc 2023-10-09T05:58:22.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:22 smithi156 ceph-mon[195101]: failed to encode map e283 with expected crc 2023-10-09T05:58:22.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:22 smithi156 ceph-mon[195101]: failed to encode map e283 with expected crc 2023-10-09T05:58:22.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:22 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:58:22.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:22 smithi156 ceph-mon[195101]: failed to encode map e283 with expected crc 2023-10-09T05:58:22.446 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:22.474 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:22.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:22 smithi064 ceph-mon[212750]: failed to encode map e283 with expected crc 2023-10-09T05:58:22.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:22 smithi064 ceph-mon[212750]: failed to encode map e283 with expected crc 2023-10-09T05:58:22.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:22 smithi064 ceph-mon[212750]: failed to encode map e283 with expected crc 2023-10-09T05:58:22.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:22 smithi064 ceph-mon[212750]: failed to encode map e283 with expected crc 2023-10-09T05:58:22.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:22 smithi064 ceph-mon[212750]: failed to encode map e283 with expected crc 2023-10-09T05:58:22.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:22 smithi064 ceph-mon[212750]: failed to encode map e283 with expected crc 2023-10-09T05:58:22.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:22 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:58:22.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:22 smithi064 ceph-mon[212750]: failed to encode map e283 with expected crc 2023-10-09T05:58:22.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:22 smithi064 ceph-mon[220044]: failed to encode map e283 with expected crc 2023-10-09T05:58:22.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:22 smithi064 ceph-mon[220044]: failed to encode map e283 with expected crc 2023-10-09T05:58:22.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:22 smithi064 ceph-mon[220044]: failed to encode map e283 with expected crc 2023-10-09T05:58:22.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:22 smithi064 ceph-mon[220044]: failed to encode map e283 with expected crc 2023-10-09T05:58:22.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:22 smithi064 ceph-mon[220044]: failed to encode map e283 with expected crc 2023-10-09T05:58:22.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:22 smithi064 ceph-mon[220044]: failed to encode map e283 with expected crc 2023-10-09T05:58:22.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:22 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:58:22.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:22 smithi064 ceph-mon[220044]: failed to encode map e283 with expected crc 2023-10-09T05:58:22.871 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:58:22.943 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:22.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:22.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:23.048 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:23.080 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:23.129 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:23.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:23.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:23.205 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:23.236 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:23.273 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:23.340 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:23 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4176078865' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:23.340 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:23 smithi064 ceph-mon[220044]: pgmap v638: 93 pgs: 2 peering, 6 active+undersized+remapped, 1 remapped+peering, 84 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 6.5 KiB/s rd, 17 KiB/s wr, 5 op/s; 161 KiB/s, 0 objects/s recovering 2023-10-09T05:58:23.347 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T05:58:23.383 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:23.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:23.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:23.486 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:23.514 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:23.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:23 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4176078865' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:23.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:23 smithi156 ceph-mon[195101]: pgmap v638: 93 pgs: 2 peering, 6 active+undersized+remapped, 1 remapped+peering, 84 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 6.5 KiB/s rd, 17 KiB/s wr, 5 op/s; 161 KiB/s, 0 objects/s recovering 2023-10-09T05:58:23.627 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:23 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4176078865' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:23.627 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:23 smithi064 ceph-mon[212750]: pgmap v638: 93 pgs: 2 peering, 6 active+undersized+remapped, 1 remapped+peering, 84 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 6.5 KiB/s rd, 17 KiB/s wr, 5 op/s; 161 KiB/s, 0 objects/s recovering 2023-10-09T05:58:24.607 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:24 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:58:24.607 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:24 smithi064 ceph-mon[212750]: failed to encode map e284 with expected crc 2023-10-09T05:58:24.607 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:24 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4176078865' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:58:24.607 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:24 smithi064 ceph-mon[212750]: osdmap e284: 8 total, 8 up, 7 in 2023-10-09T05:58:24.608 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:24 smithi064 ceph-mon[212750]: failed to encode map e284 with expected crc 2023-10-09T05:58:24.608 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:24 smithi064 ceph-mon[212750]: failed to encode map e284 with expected crc 2023-10-09T05:58:24.608 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:24 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:58:24.608 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:24 smithi064 ceph-mon[220044]: failed to encode map e284 with expected crc 2023-10-09T05:58:24.608 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:24 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4176078865' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:58:24.608 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:24 smithi064 ceph-mon[220044]: osdmap e284: 8 total, 8 up, 7 in 2023-10-09T05:58:24.608 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:24 smithi064 ceph-mon[220044]: failed to encode map e284 with expected crc 2023-10-09T05:58:24.609 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:24 smithi064 ceph-mon[220044]: failed to encode map e284 with expected crc 2023-10-09T05:58:24.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:24 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:58:24.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:24 smithi156 ceph-mon[195101]: failed to encode map e284 with expected crc 2023-10-09T05:58:24.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:24 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4176078865' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:58:24.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:24 smithi156 ceph-mon[195101]: osdmap e284: 8 total, 8 up, 7 in 2023-10-09T05:58:24.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:24 smithi156 ceph-mon[195101]: failed to encode map e284 with expected crc 2023-10-09T05:58:24.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:24 smithi156 ceph-mon[195101]: failed to encode map e284 with expected crc 2023-10-09T05:58:24.943 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:24.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:24.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:25.047 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:25.080 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:25.437 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:25.601 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:58:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:58:25] "GET /metrics HTTP/1.1" 200 36423 "" "Prometheus/2.43.0" 2023-10-09T05:58:25.602 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:25 smithi064 ceph-mon[212750]: failed to encode map e284 with expected crc 2023-10-09T05:58:25.602 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:25 smithi064 ceph-mon[212750]: pgmap v640: 93 pgs: 2 peering, 5 active+undersized+remapped, 86 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 5.3 KiB/s rd, 14 KiB/s wr, 4 op/s; 130 KiB/s, 0 objects/s recovering 2023-10-09T05:58:25.602 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:25 smithi064 ceph-mon[220044]: failed to encode map e284 with expected crc 2023-10-09T05:58:25.602 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:25 smithi064 ceph-mon[220044]: pgmap v640: 93 pgs: 2 peering, 5 active+undersized+remapped, 86 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 5.3 KiB/s rd, 14 KiB/s wr, 4 op/s; 130 KiB/s, 0 objects/s recovering 2023-10-09T05:58:25.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:25 smithi156 ceph-mon[195101]: failed to encode map e284 with expected crc 2023-10-09T05:58:25.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:25 smithi156 ceph-mon[195101]: pgmap v640: 93 pgs: 2 peering, 5 active+undersized+remapped, 86 active+clean; 582 KiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 5.3 KiB/s rd, 14 KiB/s wr, 4 op/s; 130 KiB/s, 0 objects/s recovering 2023-10-09T05:58:25.632 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:25.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:25.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:25.737 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:25.772 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:25.921 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:26.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:26.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:26.014 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:58:26.015 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:58:26.115 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:26.148 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:26.364 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_1' created 2023-10-09T05:58:26.462 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:26 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3026298253' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T05:58:26.463 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:26 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T05:58:26.463 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:26 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3026298253' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T05:58:26.463 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:26 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T05:58:26.499 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:26.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:26.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:26.604 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:26.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:26 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3026298253' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T05:58:26.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:26 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T05:58:26.637 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:26.686 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:26.687 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:58:26.687 INFO:teuthology.orchestra.run.smithi064.stdout:{"fsid":"44c77acc-6665-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_AVAILABILITY":{"severity":"HEALTH_WARN","summary":{"message":"Reduced data availability: 1 pg inactive, 1 pg peering","count":2},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":768,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":285,"num_osds":8,"num_up_osds":8,"osd_up_since":1696830716,"num_in_osds":7,"osd_in_since":1696830958,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":86},{"state_name":"active+undersized+remapped","count":5},{"state_name":"peering","count":2}],"num_pgs":93,"num_pools":4,"num_objects":24,"data_bytes":595872,"bytes_used":3241570304,"bytes_avail":668723679232,"bytes_total":671965249536,"inactive_pgs_ratio":0.021505376324057579,"recovering_objects_per_sec":0,"recovering_bytes_per_sec":133601,"recovering_keys_per_sec":0,"num_objects_recovered":2,"num_bytes_recovered":709030,"num_keys_recovered":0,"read_bytes_sec":5402,"write_bytes_sec":14085,"read_op_per_sec":2,"write_op_per_sec":1},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi064.gmrwsy","status":"up:active","gid":24551}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"}},"servicemap":{"epoch":22,"modified":"2023-10-09T05:52:56.493933+0000","services":{}},"progress_events":{"28a7925c-af7b-4940-a0f9-d3ed10a1f964":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"b79f15ab-da7e-4e62-ba02-3864805e1b86":{"message":"Global Recovery Event (6m)\n [=========================...] (remaining: 43s)","progress":0.90322577953338623,"add_to_ceph_s":true}}} 2023-10-09T05:58:26.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:26.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:26.792 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:26.820 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:26.865 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:26.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:26.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:26.969 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:27.003 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:27.277 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:58:27.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:27 smithi156 ceph-mon[195101]: failed to encode map e285 with expected crc 2023-10-09T05:58:27.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:27 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T05:58:27.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:27 smithi156 ceph-mon[195101]: osdmap e285: 8 total, 8 up, 7 in 2023-10-09T05:58:27.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:27 smithi156 ceph-mon[195101]: failed to encode map e285 with expected crc 2023-10-09T05:58:27.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:27 smithi156 ceph-mon[195101]: failed to encode map e285 with expected crc 2023-10-09T05:58:27.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:27 smithi156 ceph-mon[195101]: failed to encode map e285 with expected crc 2023-10-09T05:58:27.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:27 smithi156 ceph-mon[195101]: failed to encode map e285 with expected crc 2023-10-09T05:58:27.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:27 smithi156 ceph-mon[195101]: failed to encode map e285 with expected crc 2023-10-09T05:58:27.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:27 smithi156 ceph-mon[195101]: pgmap v642: 109 pgs: 16 unknown, 93 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:27.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:27 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1686512164' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:58:27.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:27 smithi064 ceph-mon[212750]: failed to encode map e285 with expected crc 2023-10-09T05:58:27.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:27 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T05:58:27.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:27 smithi064 ceph-mon[212750]: osdmap e285: 8 total, 8 up, 7 in 2023-10-09T05:58:27.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:27 smithi064 ceph-mon[212750]: failed to encode map e285 with expected crc 2023-10-09T05:58:27.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:27 smithi064 ceph-mon[212750]: failed to encode map e285 with expected crc 2023-10-09T05:58:27.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:27 smithi064 ceph-mon[212750]: failed to encode map e285 with expected crc 2023-10-09T05:58:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:27 smithi064 ceph-mon[212750]: failed to encode map e285 with expected crc 2023-10-09T05:58:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:27 smithi064 ceph-mon[212750]: failed to encode map e285 with expected crc 2023-10-09T05:58:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:27 smithi064 ceph-mon[212750]: pgmap v642: 109 pgs: 16 unknown, 93 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:27 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1686512164' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:58:27.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:27 smithi064 ceph-mon[220044]: failed to encode map e285 with expected crc 2023-10-09T05:58:27.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:27 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T05:58:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:27 smithi064 ceph-mon[220044]: osdmap e285: 8 total, 8 up, 7 in 2023-10-09T05:58:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:27 smithi064 ceph-mon[220044]: failed to encode map e285 with expected crc 2023-10-09T05:58:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:27 smithi064 ceph-mon[220044]: failed to encode map e285 with expected crc 2023-10-09T05:58:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:27 smithi064 ceph-mon[220044]: failed to encode map e285 with expected crc 2023-10-09T05:58:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:27 smithi064 ceph-mon[220044]: failed to encode map e285 with expected crc 2023-10-09T05:58:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:27 smithi064 ceph-mon[220044]: failed to encode map e285 with expected crc 2023-10-09T05:58:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:27 smithi064 ceph-mon[220044]: pgmap v642: 109 pgs: 16 unknown, 93 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:27.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:27 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1686512164' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:58:27.741 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:27.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:27.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:27.846 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:27.873 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:28.056 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T05:58:28.288 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:28.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:28.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:28.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:28.418 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:28.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:28 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:58:28.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:28 smithi156 ceph-mon[195101]: osdmap e286: 8 total, 8 up, 7 in 2023-10-09T05:58:28.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:28 smithi156 ceph-mon[195101]: failed to encode map e286 with expected crc 2023-10-09T05:58:28.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:28 smithi156 ceph-mon[195101]: failed to encode map e286 with expected crc 2023-10-09T05:58:28.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:28 smithi156 ceph-mon[195101]: failed to encode map e286 with expected crc 2023-10-09T05:58:28.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:28 smithi156 ceph-mon[195101]: failed to encode map e286 with expected crc 2023-10-09T05:58:28.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:28 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:58:28.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:28 smithi064 ceph-mon[220044]: osdmap e286: 8 total, 8 up, 7 in 2023-10-09T05:58:28.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:28 smithi064 ceph-mon[220044]: failed to encode map e286 with expected crc 2023-10-09T05:58:28.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:28 smithi064 ceph-mon[220044]: failed to encode map e286 with expected crc 2023-10-09T05:58:28.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:28 smithi064 ceph-mon[220044]: failed to encode map e286 with expected crc 2023-10-09T05:58:28.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:28 smithi064 ceph-mon[220044]: failed to encode map e286 with expected crc 2023-10-09T05:58:28.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:28 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:58:28.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:28 smithi064 ceph-mon[212750]: osdmap e286: 8 total, 8 up, 7 in 2023-10-09T05:58:28.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:28 smithi064 ceph-mon[212750]: failed to encode map e286 with expected crc 2023-10-09T05:58:28.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:28 smithi064 ceph-mon[212750]: failed to encode map e286 with expected crc 2023-10-09T05:58:28.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:28 smithi064 ceph-mon[212750]: failed to encode map e286 with expected crc 2023-10-09T05:58:28.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:28 smithi064 ceph-mon[212750]: failed to encode map e286 with expected crc 2023-10-09T05:58:28.781 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T05:58:28.781 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:58:28.915 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:28.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:28.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:28.982 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-10-09T05:58:29.019 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:29.051 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:29.092 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:29.101 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:29.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:29.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:29.127 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:29.188 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:58:29.208 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:29.209 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:58:29.232 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:58:29.238 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:29.260 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:58:29.282 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:58:29.294 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:29.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:29.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:29.308 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:58:29.338 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:58:29.363 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:58:29.365 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:58:29.365 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:58:29.365 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:58:29.366 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:58:29.366 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:58:29.366 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:58:29.366 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:58:29.366 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:58:29.399 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:29.429 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:29.594 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:29 smithi064 ceph-mon[212750]: failed to encode map e287 with expected crc 2023-10-09T05:58:29.595 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:29 smithi064 ceph-mon[212750]: osdmap e287: 8 total, 8 up, 7 in 2023-10-09T05:58:29.595 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:29 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:58:29.595 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:29.595 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:29 smithi064 ceph-mon[212750]: failed to encode map e287 with expected crc 2023-10-09T05:58:29.595 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:29 smithi064 ceph-mon[212750]: pgmap v645: 109 pgs: 10 unknown, 99 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:29.595 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:29 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/704635769' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:29.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:29 smithi064 ceph-mon[220044]: failed to encode map e287 with expected crc 2023-10-09T05:58:29.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:29 smithi064 ceph-mon[220044]: osdmap e287: 8 total, 8 up, 7 in 2023-10-09T05:58:29.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:29 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:58:29.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:29.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:29 smithi064 ceph-mon[220044]: failed to encode map e287 with expected crc 2023-10-09T05:58:29.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:29 smithi064 ceph-mon[220044]: pgmap v645: 109 pgs: 10 unknown, 99 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:29.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:29 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/704635769' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:29.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:29 smithi156 ceph-mon[195101]: failed to encode map e287 with expected crc 2023-10-09T05:58:29.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:29 smithi156 ceph-mon[195101]: osdmap e287: 8 total, 8 up, 7 in 2023-10-09T05:58:29.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:29 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:58:29.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:29.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:29 smithi156 ceph-mon[195101]: failed to encode map e287 with expected crc 2023-10-09T05:58:29.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:29 smithi156 ceph-mon[195101]: pgmap v645: 109 pgs: 10 unknown, 99 active+clean; 582 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:29.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:29 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/704635769' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:30.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:30 smithi064 ceph-mon[212750]: failed to encode map e287 with expected crc 2023-10-09T05:58:30.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:30 smithi064 ceph-mon[212750]: failed to encode map e287 with expected crc 2023-10-09T05:58:30.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:30 smithi064 ceph-mon[220044]: failed to encode map e287 with expected crc 2023-10-09T05:58:30.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:30 smithi064 ceph-mon[220044]: failed to encode map e287 with expected crc 2023-10-09T05:58:30.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:30 smithi156 ceph-mon[195101]: failed to encode map e287 with expected crc 2023-10-09T05:58:30.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:30 smithi156 ceph-mon[195101]: failed to encode map e287 with expected crc 2023-10-09T05:58:30.987 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:30.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:30.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:31.091 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:31.123 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:31.173 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:31.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:31.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:31.277 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:31.306 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:31.352 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:31.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:31.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:31.460 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:31.489 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:31.541 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:31.635 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:31.645 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:31.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:31 smithi064 ceph-mon[212750]: pgmap v646: 109 pgs: 109 active+clean; 582 KiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:31.648 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:31 smithi064 ceph-mon[220044]: pgmap v646: 109 pgs: 109 active+clean; 582 KiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:31.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:31.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:31.751 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:31.780 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:31 smithi156 ceph-mon[195101]: pgmap v646: 109 pgs: 109 active+clean; 582 KiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:31.783 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:31.834 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:31.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:31.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:31.941 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:31.970 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:32.006 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:58:32.006 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T05:58:32.007 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T05:58:32.007 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T05:58:32.007 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:58:32.015 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:32.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:32.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:32.123 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:32.150 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:32.192 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:32.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:32.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:32.297 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:32.323 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:32.341 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:58:32.366 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:32.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:32.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:32.471 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:32.501 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:33.174 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:33.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:33.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:33.194 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T05:58:33.252 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T05:58:33.279 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:33.307 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:33.365 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:33.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:33.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:33.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:33 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2484056354' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:58:33.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:33 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:58:33.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:33 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2484056354' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:58:33.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:33 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:58:33.470 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:33.499 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:33 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2484056354' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:58:33.499 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:33 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:58:33.502 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:33.552 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:33.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:33.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:33.660 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:33.688 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:34.048 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:34.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:34.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:34.152 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:34.180 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:34.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:34 smithi064 ceph-mon[212750]: pgmap v647: 109 pgs: 109 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:34.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:34 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:58:34.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:34 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:58:34.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:34 smithi064 ceph-mon[212750]: osdmap e288: 8 total, 8 up, 7 in 2023-10-09T05:58:34.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:34 smithi064 ceph-mon[212750]: failed to encode map e288 with expected crc 2023-10-09T05:58:34.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:34 smithi064 ceph-mon[212750]: failed to encode map e288 with expected crc 2023-10-09T05:58:34.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:34 smithi064 ceph-mon[212750]: failed to encode map e288 with expected crc 2023-10-09T05:58:34.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:34 smithi064 ceph-mon[212750]: failed to encode map e288 with expected crc 2023-10-09T05:58:34.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:34 smithi064 ceph-mon[212750]: failed to encode map e288 with expected crc 2023-10-09T05:58:34.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:34 smithi064 ceph-mon[220044]: pgmap v647: 109 pgs: 109 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:34.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:34 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:58:34.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:34 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:58:34.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:34 smithi064 ceph-mon[220044]: osdmap e288: 8 total, 8 up, 7 in 2023-10-09T05:58:34.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:34 smithi064 ceph-mon[220044]: failed to encode map e288 with expected crc 2023-10-09T05:58:34.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:34 smithi064 ceph-mon[220044]: failed to encode map e288 with expected crc 2023-10-09T05:58:34.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:34 smithi064 ceph-mon[220044]: failed to encode map e288 with expected crc 2023-10-09T05:58:34.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:34 smithi064 ceph-mon[220044]: failed to encode map e288 with expected crc 2023-10-09T05:58:34.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:34 smithi064 ceph-mon[220044]: failed to encode map e288 with expected crc 2023-10-09T05:58:34.528 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:34.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:34.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:34.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:34 smithi156 ceph-mon[195101]: pgmap v647: 109 pgs: 109 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:34.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:34 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:58:34.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:34 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:58:34.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:34 smithi156 ceph-mon[195101]: osdmap e288: 8 total, 8 up, 7 in 2023-10-09T05:58:34.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:34 smithi156 ceph-mon[195101]: failed to encode map e288 with expected crc 2023-10-09T05:58:34.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:34 smithi156 ceph-mon[195101]: failed to encode map e288 with expected crc 2023-10-09T05:58:34.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:34 smithi156 ceph-mon[195101]: failed to encode map e288 with expected crc 2023-10-09T05:58:34.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:34 smithi156 ceph-mon[195101]: failed to encode map e288 with expected crc 2023-10-09T05:58:34.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:34 smithi156 ceph-mon[195101]: failed to encode map e288 with expected crc 2023-10-09T05:58:34.631 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:34.660 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:34.706 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:34.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:34.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:34.810 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:34.842 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:34.892 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:34.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:34.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:35.000 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:35.027 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:35.075 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:35.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:35.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:35.178 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:35.209 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:35.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:35 smithi064 ceph-mon[212750]: failed to encode map e288 with expected crc 2023-10-09T05:58:35.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:35 smithi064 ceph-mon[212750]: 1.4 scrub starts 2023-10-09T05:58:35.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:35 smithi064 ceph-mon[212750]: 1.4 scrub ok 2023-10-09T05:58:35.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:35 smithi064 ceph-mon[212750]: 1.7 scrub starts 2023-10-09T05:58:35.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:35 smithi064 ceph-mon[212750]: 1.7 scrub ok 2023-10-09T05:58:35.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:35 smithi064 ceph-mon[212750]: 1.a scrub starts 2023-10-09T05:58:35.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:35 smithi064 ceph-mon[212750]: 1.a scrub ok 2023-10-09T05:58:35.304 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:35 smithi064 ceph-mon[220044]: failed to encode map e288 with expected crc 2023-10-09T05:58:35.304 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:35 smithi064 ceph-mon[220044]: 1.4 scrub starts 2023-10-09T05:58:35.304 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:35 smithi064 ceph-mon[220044]: 1.4 scrub ok 2023-10-09T05:58:35.304 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:35 smithi064 ceph-mon[220044]: 1.7 scrub starts 2023-10-09T05:58:35.304 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:35 smithi064 ceph-mon[220044]: 1.7 scrub ok 2023-10-09T05:58:35.304 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:35 smithi064 ceph-mon[220044]: 1.a scrub starts 2023-10-09T05:58:35.304 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:35 smithi064 ceph-mon[220044]: 1.a scrub ok 2023-10-09T05:58:35.304 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:58:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:58:35] "GET /metrics HTTP/1.1" 200 36329 "" "Prometheus/2.43.0" 2023-10-09T05:58:35.364 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:35.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:35.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:35.374 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:35.455 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:35.470 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:35.499 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:35 smithi156 ceph-mon[195101]: failed to encode map e288 with expected crc 2023-10-09T05:58:35.499 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:35 smithi156 ceph-mon[195101]: 1.4 scrub starts 2023-10-09T05:58:35.500 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:35 smithi156 ceph-mon[195101]: 1.4 scrub ok 2023-10-09T05:58:35.500 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:35 smithi156 ceph-mon[195101]: 1.7 scrub starts 2023-10-09T05:58:35.500 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:35 smithi156 ceph-mon[195101]: 1.7 scrub ok 2023-10-09T05:58:35.500 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:35 smithi156 ceph-mon[195101]: 1.a scrub starts 2023-10-09T05:58:35.500 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:35 smithi156 ceph-mon[195101]: 1.a scrub ok 2023-10-09T05:58:35.502 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:35.551 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:35.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:35.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:35.658 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:35.667 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:58:35.669 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T05:58:35.686 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:35.730 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:35.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:35.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:35.837 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:35.869 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:36.230 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:36.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:36.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:36.333 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:36.363 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:36.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:36 smithi064 ceph-mon[220044]: pgmap v649: 109 pgs: 109 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:36.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:36 smithi064 ceph-mon[220044]: 1.5 scrub starts 2023-10-09T05:58:36.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:36 smithi064 ceph-mon[220044]: 1.5 scrub ok 2023-10-09T05:58:36.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:36 smithi064 ceph-mon[220044]: 1.6 scrub starts 2023-10-09T05:58:36.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:36 smithi064 ceph-mon[220044]: 1.6 scrub ok 2023-10-09T05:58:36.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:36 smithi064 ceph-mon[220044]: 1.8 scrub starts 2023-10-09T05:58:36.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:36 smithi064 ceph-mon[220044]: 1.8 scrub ok 2023-10-09T05:58:36.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:36 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2620152194' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:58:36.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:36 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:58:36.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:36 smithi064 ceph-mon[212750]: pgmap v649: 109 pgs: 109 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:36.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:36 smithi064 ceph-mon[212750]: 1.5 scrub starts 2023-10-09T05:58:36.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:36 smithi064 ceph-mon[212750]: 1.5 scrub ok 2023-10-09T05:58:36.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:36 smithi064 ceph-mon[212750]: 1.6 scrub starts 2023-10-09T05:58:36.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:36 smithi064 ceph-mon[212750]: 1.6 scrub ok 2023-10-09T05:58:36.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:36 smithi064 ceph-mon[212750]: 1.8 scrub starts 2023-10-09T05:58:36.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:36 smithi064 ceph-mon[212750]: 1.8 scrub ok 2023-10-09T05:58:36.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:36 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2620152194' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:58:36.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:36 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:58:36.595 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:58:36.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:36 smithi156 ceph-mon[195101]: pgmap v649: 109 pgs: 109 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:36.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:36 smithi156 ceph-mon[195101]: 1.5 scrub starts 2023-10-09T05:58:36.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:36 smithi156 ceph-mon[195101]: 1.5 scrub ok 2023-10-09T05:58:36.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:36 smithi156 ceph-mon[195101]: 1.6 scrub starts 2023-10-09T05:58:36.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:36 smithi156 ceph-mon[195101]: 1.6 scrub ok 2023-10-09T05:58:36.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:36 smithi156 ceph-mon[195101]: 1.8 scrub starts 2023-10-09T05:58:36.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:36 smithi156 ceph-mon[195101]: 1.8 scrub ok 2023-10-09T05:58:36.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:36 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2620152194' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:58:36.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:36 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:58:36.659 INFO:teuthology.orchestra.run.smithi064.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-10-09T05:58:37.212 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:37.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:37.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:37.316 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:37.348 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:37.394 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:37.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:37.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:37.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:37 smithi064 ceph-mon[212750]: from='client.45242 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:58:37.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:37 smithi064 ceph-mon[212750]: 1.1 scrub starts 2023-10-09T05:58:37.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:37 smithi064 ceph-mon[212750]: 1.1 scrub ok 2023-10-09T05:58:37.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:37 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:58:37.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:37 smithi064 ceph-mon[212750]: osdmap e289: 8 total, 8 up, 7 in 2023-10-09T05:58:37.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:37 smithi064 ceph-mon[212750]: failed to encode map e289 with expected crc 2023-10-09T05:58:37.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:37 smithi064 ceph-mon[212750]: failed to encode map e289 with expected crc 2023-10-09T05:58:37.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:37 smithi064 ceph-mon[212750]: 1.14 scrub starts 2023-10-09T05:58:37.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:37 smithi064 ceph-mon[212750]: 1.14 scrub ok 2023-10-09T05:58:37.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:37 smithi064 ceph-mon[220044]: from='client.45242 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:58:37.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:37 smithi064 ceph-mon[220044]: 1.1 scrub starts 2023-10-09T05:58:37.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:37 smithi064 ceph-mon[220044]: 1.1 scrub ok 2023-10-09T05:58:37.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:37 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:58:37.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:37 smithi064 ceph-mon[220044]: osdmap e289: 8 total, 8 up, 7 in 2023-10-09T05:58:37.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:37 smithi064 ceph-mon[220044]: failed to encode map e289 with expected crc 2023-10-09T05:58:37.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:37 smithi064 ceph-mon[220044]: failed to encode map e289 with expected crc 2023-10-09T05:58:37.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:37 smithi064 ceph-mon[220044]: 1.14 scrub starts 2023-10-09T05:58:37.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:37 smithi064 ceph-mon[220044]: 1.14 scrub ok 2023-10-09T05:58:37.500 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:37.525 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:37 smithi156 ceph-mon[195101]: from='client.45242 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:58:37.525 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:37 smithi156 ceph-mon[195101]: 1.1 scrub starts 2023-10-09T05:58:37.525 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:37 smithi156 ceph-mon[195101]: 1.1 scrub ok 2023-10-09T05:58:37.525 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:37 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:58:37.525 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:37 smithi156 ceph-mon[195101]: osdmap e289: 8 total, 8 up, 7 in 2023-10-09T05:58:37.526 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:37 smithi156 ceph-mon[195101]: failed to encode map e289 with expected crc 2023-10-09T05:58:37.526 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:37 smithi156 ceph-mon[195101]: failed to encode map e289 with expected crc 2023-10-09T05:58:37.526 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:37 smithi156 ceph-mon[195101]: 1.14 scrub starts 2023-10-09T05:58:37.526 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:37 smithi156 ceph-mon[195101]: 1.14 scrub ok 2023-10-09T05:58:37.527 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:37.571 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:37.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:37.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:37.676 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:37.703 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:37.707 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":649,"stamp":"2023-10-09T05:58:34.624103+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":3355304,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3436,"ondisk_log_size":3436,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":328,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":2827464,"kb_used_data":916560,"kb_used_omap":118,"kb_used_meta":1910729,"kb_avail":653388600,"statfs":{"total":671965249536,"available":669069926400,"internally_reserved":0,"allocated":938557440,"data_stored":929550297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":121648,"internal_metadata":1956586704},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.260002"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":332,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.371773+0000","last_change":"2023-10-09T05:55:59.341215+0000","last_active":"2023-10-09T05:58:27.371773+0000","last_peered":"2023-10-09T05:58:27.371773+0000","last_clean":"2023-10-09T05:58:27.371773+0000","last_became_active":"2023-10-09T05:55:59.340856+0000","last_became_peered":"2023-10-09T05:55:59.340856+0000","last_unstale":"2023-10-09T05:58:27.371773+0000","last_undegraded":"2023-10-09T05:58:27.371773+0000","last_fullsized":"2023-10-09T05:58:27.371773+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:33.686454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":49,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.361495+0000","last_change":"2023-10-09T05:57:23.474348+0000","last_active":"2023-10-09T05:58:27.361495+0000","last_peered":"2023-10-09T05:58:27.361495+0000","last_clean":"2023-10-09T05:58:27.361495+0000","last_became_active":"2023-10-09T05:57:23.474010+0000","last_became_peered":"2023-10-09T05:57:23.474010+0000","last_unstale":"2023-10-09T05:58:27.361495+0000","last_undegraded":"2023-10-09T05:58:27.361495+0000","last_fullsized":"2023-10-09T05:58:27.361495+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:45.039777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":166,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.361212+0000","last_change":"2023-10-09T05:55:59.304893+0000","last_active":"2023-10-09T05:58:27.361212+0000","last_peered":"2023-10-09T05:58:27.361212+0000","last_clean":"2023-10-09T05:58:27.361212+0000","last_became_active":"2023-10-09T05:55:59.304449+0000","last_became_peered":"2023-10-09T05:55:59.304449+0000","last_unstale":"2023-10-09T05:58:27.361212+0000","last_undegraded":"2023-10-09T05:58:27.361212+0000","last_fullsized":"2023-10-09T05:58:27.361212+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:44:39.453982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":307,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.371615+0000","last_change":"2023-10-09T05:57:58.834014+0000","last_active":"2023-10-09T05:58:27.371615+0000","last_peered":"2023-10-09T05:58:27.371615+0000","last_clean":"2023-10-09T05:58:27.371615+0000","last_became_active":"2023-10-09T05:57:58.833718+0000","last_became_peered":"2023-10-09T05:57:58.833718+0000","last_unstale":"2023-10-09T05:58:27.371615+0000","last_undegraded":"2023-10-09T05:58:27.371615+0000","last_fullsized":"2023-10-09T05:58:27.371615+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:23.752353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":49,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.361217+0000","last_change":"2023-10-09T05:57:23.476832+0000","last_active":"2023-10-09T05:58:27.361217+0000","last_peered":"2023-10-09T05:58:27.361217+0000","last_clean":"2023-10-09T05:58:27.361217+0000","last_became_active":"2023-10-09T05:57:23.475216+0000","last_became_peered":"2023-10-09T05:57:23.475216+0000","last_unstale":"2023-10-09T05:58:27.361217+0000","last_undegraded":"2023-10-09T05:58:27.361217+0000","last_fullsized":"2023-10-09T05:58:27.361217+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:48.279660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":313,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.373001+0000","last_change":"2023-10-09T05:55:59.335464+0000","last_active":"2023-10-09T05:58:27.373001+0000","last_peered":"2023-10-09T05:58:27.373001+0000","last_clean":"2023-10-09T05:58:27.373001+0000","last_became_active":"2023-10-09T05:55:59.335259+0000","last_became_peered":"2023-10-09T05:55:59.335259+0000","last_unstale":"2023-10-09T05:58:27.373001+0000","last_undegraded":"2023-10-09T05:58:27.373001+0000","last_fullsized":"2023-10-09T05:58:27.373001+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:28.536682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":242,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.371596+0000","last_change":"2023-10-09T05:57:23.477980+0000","last_active":"2023-10-09T05:58:27.371596+0000","last_peered":"2023-10-09T05:58:27.371596+0000","last_clean":"2023-10-09T05:58:27.371596+0000","last_became_active":"2023-10-09T05:57:23.477654+0000","last_became_peered":"2023-10-09T05:57:23.477654+0000","last_unstale":"2023-10-09T05:58:27.371596+0000","last_undegraded":"2023-10-09T05:58:27.371596+0000","last_fullsized":"2023-10-09T05:58:27.371596+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:17.524683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":49,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.361283+0000","last_change":"2023-10-09T05:57:23.473644+0000","last_active":"2023-10-09T05:58:27.361283+0000","last_peered":"2023-10-09T05:58:27.361283+0000","last_clean":"2023-10-09T05:58:27.361283+0000","last_became_active":"2023-10-09T05:57:23.473391+0000","last_became_peered":"2023-10-09T05:57:23.473391+0000","last_unstale":"2023-10-09T05:58:27.361283+0000","last_undegraded":"2023-10-09T05:58:27.361283+0000","last_fullsized":"2023-10-09T05:58:27.361283+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:17.138034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":539,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.364858+0000","last_change":"2023-10-09T05:58:21.330352+0000","last_active":"2023-10-09T05:58:27.364858+0000","last_peered":"2023-10-09T05:58:27.364858+0000","last_clean":"2023-10-09T05:58:27.364858+0000","last_became_active":"2023-10-09T05:58:21.329980+0000","last_became_peered":"2023-10-09T05:58:21.329980+0000","last_unstale":"2023-10-09T05:58:27.364858+0000","last_undegraded":"2023-10-09T05:58:27.364858+0000","last_fullsized":"2023-10-09T05:58:27.364858+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":326,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.375122+0000","last_change":"2023-10-09T05:57:23.478483+0000","last_active":"2023-10-09T05:58:27.375122+0000","last_peered":"2023-10-09T05:58:27.375122+0000","last_clean":"2023-10-09T05:58:27.375122+0000","last_became_active":"2023-10-09T05:57:23.478122+0000","last_became_peered":"2023-10-09T05:57:23.478122+0000","last_unstale":"2023-10-09T05:58:27.375122+0000","last_undegraded":"2023-10-09T05:58:27.375122+0000","last_fullsized":"2023-10-09T05:58:27.375122+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:37.768717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":230,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.361105+0000","last_change":"2023-10-09T05:53:59.185702+0000","last_active":"2023-10-09T05:58:27.361105+0000","last_peered":"2023-10-09T05:58:27.361105+0000","last_clean":"2023-10-09T05:58:27.361105+0000","last_became_active":"2023-10-09T05:53:59.185373+0000","last_became_peered":"2023-10-09T05:53:59.185373+0000","last_unstale":"2023-10-09T05:58:27.361105+0000","last_undegraded":"2023-10-09T05:58:27.361105+0000","last_fullsized":"2023-10-09T05:58:27.361105+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:50.756967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":127,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.372092+0000","last_change":"2023-10-09T05:53:59.219168+0000","last_active":"2023-10-09T05:58:27.372092+0000","last_peered":"2023-10-09T05:58:27.372092+0000","last_clean":"2023-10-09T05:58:27.372092+0000","last_became_active":"2023-10-09T05:53:59.200886+0000","last_became_peered":"2023-10-09T05:53:59.200886+0000","last_unstale":"2023-10-09T05:58:27.372092+0000","last_undegraded":"2023-10-09T05:58:27.372092+0000","last_fullsized":"2023-10-09T05:58:27.372092+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:45.515036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":181,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.373867+0000","last_change":"2023-10-09T05:55:59.305100+0000","last_active":"2023-10-09T05:58:27.373867+0000","last_peered":"2023-10-09T05:58:27.373867+0000","last_clean":"2023-10-09T05:58:27.373867+0000","last_became_active":"2023-10-09T05:55:59.304758+0000","last_became_peered":"2023-10-09T05:55:59.304758+0000","last_unstale":"2023-10-09T05:58:27.373867+0000","last_undegraded":"2023-10-09T05:58:27.373867+0000","last_fullsized":"2023-10-09T05:58:27.373867+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:54:32.510945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":292,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.373311+0000","last_change":"2023-10-09T05:53:59.219141+0000","last_active":"2023-10-09T05:58:27.373311+0000","last_peered":"2023-10-09T05:58:27.373311+0000","last_clean":"2023-10-09T05:58:27.373311+0000","last_became_active":"2023-10-09T05:53:59.197318+0000","last_became_peered":"2023-10-09T05:53:59.197318+0000","last_unstale":"2023-10-09T05:58:27.373311+0000","last_undegraded":"2023-10-09T05:58:27.373311+0000","last_fullsized":"2023-10-09T05:58:27.373311+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:39.910794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":296,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.373988+0000","last_change":"2023-10-09T05:55:59.335824+0000","last_active":"2023-10-09T05:58:27.373988+0000","last_peered":"2023-10-09T05:58:27.373988+0000","last_clean":"2023-10-09T05:58:27.373988+0000","last_became_active":"2023-10-09T05:55:59.335478+0000","last_became_peered":"2023-10-09T05:55:59.335478+0000","last_unstale":"2023-10-09T05:58:27.373988+0000","last_undegraded":"2023-10-09T05:58:27.373988+0000","last_fullsized":"2023-10-09T05:58:27.373988+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:26.014501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":494,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.370795+0000","last_change":"2023-10-09T05:58:21.333118+0000","last_active":"2023-10-09T05:58:27.370795+0000","last_peered":"2023-10-09T05:58:27.370795+0000","last_clean":"2023-10-09T05:58:27.370795+0000","last_became_active":"2023-10-09T05:58:21.332729+0000","last_became_peered":"2023-10-09T05:58:21.332729+0000","last_unstale":"2023-10-09T05:58:27.370795+0000","last_undegraded":"2023-10-09T05:58:27.370795+0000","last_fullsized":"2023-10-09T05:58:27.370795+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":129,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.374165+0000","last_change":"2023-10-09T05:53:59.233898+0000","last_active":"2023-10-09T05:58:27.374165+0000","last_peered":"2023-10-09T05:58:27.374165+0000","last_clean":"2023-10-09T05:58:27.374165+0000","last_became_active":"2023-10-09T05:53:59.204175+0000","last_became_peered":"2023-10-09T05:53:59.204175+0000","last_unstale":"2023-10-09T05:58:27.374165+0000","last_undegraded":"2023-10-09T05:58:27.374165+0000","last_fullsized":"2023-10-09T05:58:27.374165+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:03.139318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":316,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.361998+0000","last_change":"2023-10-09T05:55:59.297087+0000","last_active":"2023-10-09T05:58:27.361998+0000","last_peered":"2023-10-09T05:58:27.361998+0000","last_clean":"2023-10-09T05:58:27.361998+0000","last_became_active":"2023-10-09T05:55:59.296768+0000","last_became_peered":"2023-10-09T05:55:59.296768+0000","last_unstale":"2023-10-09T05:58:27.361998+0000","last_undegraded":"2023-10-09T05:58:27.361998+0000","last_fullsized":"2023-10-09T05:58:27.361998+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:35.067705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":700,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.373781+0000","last_change":"2023-10-09T05:58:21.332888+0000","last_active":"2023-10-09T05:58:27.373781+0000","last_peered":"2023-10-09T05:58:27.373781+0000","last_clean":"2023-10-09T05:58:27.373781+0000","last_became_active":"2023-10-09T05:58:21.332533+0000","last_became_peered":"2023-10-09T05:58:21.332533+0000","last_unstale":"2023-10-09T05:58:27.373781+0000","last_undegraded":"2023-10-09T05:58:27.373781+0000","last_fullsized":"2023-10-09T05:58:27.373781+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:51:12.346243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:51:12.346243+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":288,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.371414+0000","last_change":"2023-10-09T05:57:23.473445+0000","last_active":"2023-10-09T05:58:27.371414+0000","last_peered":"2023-10-09T05:58:27.371414+0000","last_clean":"2023-10-09T05:58:27.371414+0000","last_became_active":"2023-10-09T05:57:23.473103+0000","last_became_peered":"2023-10-09T05:57:23.473103+0000","last_unstale":"2023-10-09T05:58:27.371414+0000","last_undegraded":"2023-10-09T05:58:27.371414+0000","last_fullsized":"2023-10-09T05:58:27.371414+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:39:53.825734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":210,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.371913+0000","last_change":"2023-10-09T05:56:49.237784+0000","last_active":"2023-10-09T05:58:27.371913+0000","last_peered":"2023-10-09T05:58:27.371913+0000","last_clean":"2023-10-09T05:58:27.371913+0000","last_became_active":"2023-10-09T05:56:49.237417+0000","last_became_peered":"2023-10-09T05:56:49.237417+0000","last_unstale":"2023-10-09T05:58:27.371913+0000","last_undegraded":"2023-10-09T05:58:27.371913+0000","last_fullsized":"2023-10-09T05:58:27.371913+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:33.129268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":525,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.360944+0000","last_change":"2023-10-09T05:58:21.328344+0000","last_active":"2023-10-09T05:58:27.360944+0000","last_peered":"2023-10-09T05:58:27.360944+0000","last_clean":"2023-10-09T05:58:27.360944+0000","last_became_active":"2023-10-09T05:58:21.327911+0000","last_became_peered":"2023-10-09T05:58:21.327911+0000","last_unstale":"2023-10-09T05:58:27.360944+0000","last_undegraded":"2023-10-09T05:58:27.360944+0000","last_fullsized":"2023-10-09T05:58:27.360944+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":123,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.375396+0000","last_change":"2023-10-09T05:53:59.206887+0000","last_active":"2023-10-09T05:58:27.375396+0000","last_peered":"2023-10-09T05:58:27.375396+0000","last_clean":"2023-10-09T05:58:27.375396+0000","last_became_active":"2023-10-09T05:53:59.206608+0000","last_became_peered":"2023-10-09T05:53:59.206608+0000","last_unstale":"2023-10-09T05:58:27.375396+0000","last_undegraded":"2023-10-09T05:58:27.375396+0000","last_fullsized":"2023-10-09T05:58:27.375396+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:48.975792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":269,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.371800+0000","last_change":"2023-10-09T05:57:58.828837+0000","last_active":"2023-10-09T05:58:27.371800+0000","last_peered":"2023-10-09T05:58:27.371800+0000","last_clean":"2023-10-09T05:58:27.371800+0000","last_became_active":"2023-10-09T05:57:58.828503+0000","last_became_peered":"2023-10-09T05:57:58.828503+0000","last_unstale":"2023-10-09T05:58:27.371800+0000","last_undegraded":"2023-10-09T05:58:27.371800+0000","last_fullsized":"2023-10-09T05:58:27.371800+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:08.332644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":577,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.373412+0000","last_change":"2023-10-09T05:58:21.333260+0000","last_active":"2023-10-09T05:58:27.373412+0000","last_peered":"2023-10-09T05:58:27.373412+0000","last_clean":"2023-10-09T05:58:27.373412+0000","last_became_active":"2023-10-09T05:58:21.332713+0000","last_became_peered":"2023-10-09T05:58:21.332713+0000","last_unstale":"2023-10-09T05:58:27.373412+0000","last_undegraded":"2023-10-09T05:58:27.373412+0000","last_fullsized":"2023-10-09T05:58:27.373412+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":272,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.371767+0000","last_change":"2023-10-09T05:51:57.091639+0000","last_active":"2023-10-09T05:58:27.371767+0000","last_peered":"2023-10-09T05:58:27.371767+0000","last_clean":"2023-10-09T05:58:27.371767+0000","last_became_active":"2023-10-09T05:51:57.091284+0000","last_became_peered":"2023-10-09T05:51:57.091284+0000","last_unstale":"2023-10-09T05:58:27.371767+0000","last_undegraded":"2023-10-09T05:58:27.371767+0000","last_fullsized":"2023-10-09T05:58:27.371767+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:10.553453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":58,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.361723+0000","last_change":"2023-10-09T05:57:58.833939+0000","last_active":"2023-10-09T05:58:27.361723+0000","last_peered":"2023-10-09T05:58:27.361723+0000","last_clean":"2023-10-09T05:58:27.361723+0000","last_became_active":"2023-10-09T05:57:58.833567+0000","last_became_peered":"2023-10-09T05:57:58.833567+0000","last_unstale":"2023-10-09T05:58:27.361723+0000","last_undegraded":"2023-10-09T05:58:27.361723+0000","last_fullsized":"2023-10-09T05:58:27.361723+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:58.347572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"0'0","reported_seq":10,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.406075+0000","last_change":"2023-10-09T05:58:27.405856+0000","last_active":"2023-10-09T05:58:27.406075+0000","last_peered":"2023-10-09T05:58:27.406075+0000","last_clean":"2023-10-09T05:58:27.406075+0000","last_became_active":"2023-10-09T05:58:27.405560+0000","last_became_peered":"2023-10-09T05:58:27.405560+0000","last_unstale":"2023-10-09T05:58:27.406075+0000","last_undegraded":"2023-10-09T05:58:27.406075+0000","last_fullsized":"2023-10-09T05:58:27.406075+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:24.937739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":596,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.371769+0000","last_change":"2023-10-09T05:58:21.333719+0000","last_active":"2023-10-09T05:58:27.371769+0000","last_peered":"2023-10-09T05:58:27.371769+0000","last_clean":"2023-10-09T05:58:27.371769+0000","last_became_active":"2023-10-09T05:58:21.333365+0000","last_became_peered":"2023-10-09T05:58:21.333365+0000","last_unstale":"2023-10-09T05:58:27.371769+0000","last_undegraded":"2023-10-09T05:58:27.371769+0000","last_fullsized":"2023-10-09T05:58:27.371769+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":127,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.372118+0000","last_change":"2023-10-09T05:53:59.222136+0000","last_active":"2023-10-09T05:58:27.372118+0000","last_peered":"2023-10-09T05:58:27.372118+0000","last_clean":"2023-10-09T05:58:27.372118+0000","last_became_active":"2023-10-09T05:53:59.201853+0000","last_became_peered":"2023-10-09T05:53:59.201853+0000","last_unstale":"2023-10-09T05:58:27.372118+0000","last_undegraded":"2023-10-09T05:58:27.372118+0000","last_fullsized":"2023-10-09T05:58:27.372118+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:26.662031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":193,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.373534+0000","last_change":"2023-10-09T05:56:49.234185+0000","last_active":"2023-10-09T05:58:27.373534+0000","last_peered":"2023-10-09T05:58:27.373534+0000","last_clean":"2023-10-09T05:58:27.373534+0000","last_became_active":"2023-10-09T05:56:49.233884+0000","last_became_peered":"2023-10-09T05:56:49.233884+0000","last_unstale":"2023-10-09T05:58:27.373534+0000","last_undegraded":"2023-10-09T05:58:27.373534+0000","last_fullsized":"2023-10-09T05:58:27.373534+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:12.142436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.a","version":"0'0","reported_seq":11,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T05:58:28.369078+0000","last_change":"2023-10-09T05:58:27.406918+0000","last_active":"2023-10-09T05:58:28.369078+0000","last_peered":"2023-10-09T05:58:28.369078+0000","last_clean":"2023-10-09T05:58:28.369078+0000","last_became_active":"2023-10-09T05:58:27.406600+0000","last_became_peered":"2023-10-09T05:58:27.406600+0000","last_unstale":"2023-10-09T05:58:28.369078+0000","last_undegraded":"2023-10-09T05:58:28.369078+0000","last_fullsized":"2023-10-09T05:58:28.369078+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:00.484611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":558,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.361598+0000","last_change":"2023-10-09T05:58:21.327906+0000","last_active":"2023-10-09T05:58:27.361598+0000","last_peered":"2023-10-09T05:58:27.361598+0000","last_clean":"2023-10-09T05:58:27.361598+0000","last_became_active":"2023-10-09T05:58:21.327449+0000","last_became_peered":"2023-10-09T05:58:21.327449+0000","last_unstale":"2023-10-09T05:58:27.361598+0000","last_undegraded":"2023-10-09T05:58:27.361598+0000","last_fullsized":"2023-10-09T05:58:27.361598+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1364390,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":169,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.375225+0000","last_change":"2023-10-09T05:55:59.379174+0000","last_active":"2023-10-09T05:58:27.375225+0000","last_peered":"2023-10-09T05:58:27.375225+0000","last_clean":"2023-10-09T05:58:27.375225+0000","last_became_active":"2023-10-09T05:55:59.341932+0000","last_became_peered":"2023-10-09T05:55:59.341932+0000","last_unstale":"2023-10-09T05:58:27.375225+0000","last_undegraded":"2023-10-09T05:58:27.375225+0000","last_fullsized":"2023-10-09T05:58:27.375225+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.636171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":333,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.372742+0000","last_change":"2023-10-09T05:57:23.475146+0000","last_active":"2023-10-09T05:58:27.372742+0000","last_peered":"2023-10-09T05:58:27.372742+0000","last_clean":"2023-10-09T05:58:27.372742+0000","last_became_active":"2023-10-09T05:57:23.474776+0000","last_became_peered":"2023-10-09T05:57:23.474776+0000","last_unstale":"2023-10-09T05:58:27.372742+0000","last_undegraded":"2023-10-09T05:58:27.372742+0000","last_fullsized":"2023-10-09T05:58:27.372742+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:47.775743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"0'0","reported_seq":11,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T05:58:29.080545+0000","last_change":"2023-10-09T05:58:27.377562+0000","last_active":"2023-10-09T05:58:29.080545+0000","last_peered":"2023-10-09T05:58:29.080545+0000","last_clean":"2023-10-09T05:58:29.080545+0000","last_became_active":"2023-10-09T05:58:27.377253+0000","last_became_peered":"2023-10-09T05:58:27.377253+0000","last_unstale":"2023-10-09T05:58:29.080545+0000","last_undegraded":"2023-10-09T05:58:29.080545+0000","last_fullsized":"2023-10-09T05:58:29.080545+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:34.871962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":699,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.373110+0000","last_change":"2023-10-09T05:58:21.333204+0000","last_active":"2023-10-09T05:58:27.373110+0000","last_peered":"2023-10-09T05:58:27.373110+0000","last_clean":"2023-10-09T05:58:27.373110+0000","last_became_active":"2023-10-09T05:58:21.332726+0000","last_became_peered":"2023-10-09T05:58:21.332726+0000","last_unstale":"2023-10-09T05:58:27.373110+0000","last_undegraded":"2023-10-09T05:58:27.373110+0000","last_fullsized":"2023-10-09T05:58:27.373110+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":125,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.372175+0000","last_change":"2023-10-09T05:53:59.212161+0000","last_active":"2023-10-09T05:58:27.372175+0000","last_peered":"2023-10-09T05:58:27.372175+0000","last_clean":"2023-10-09T05:58:27.372175+0000","last_became_active":"2023-10-09T05:53:59.193814+0000","last_became_peered":"2023-10-09T05:53:59.193814+0000","last_unstale":"2023-10-09T05:58:27.372175+0000","last_undegraded":"2023-10-09T05:58:27.372175+0000","last_fullsized":"2023-10-09T05:58:27.372175+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:10.502408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":326,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.373732+0000","last_change":"2023-10-09T05:55:59.303352+0000","last_active":"2023-10-09T05:58:27.373732+0000","last_peered":"2023-10-09T05:58:27.373732+0000","last_clean":"2023-10-09T05:58:27.373732+0000","last_became_active":"2023-10-09T05:55:59.302923+0000","last_became_peered":"2023-10-09T05:55:59.302923+0000","last_unstale":"2023-10-09T05:58:27.373732+0000","last_undegraded":"2023-10-09T05:58:27.373732+0000","last_fullsized":"2023-10-09T05:58:27.373732+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:53.013376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"0'0","reported_seq":11,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.427991+0000","last_change":"2023-10-09T05:58:27.376696+0000","last_active":"2023-10-09T05:58:27.427991+0000","last_peered":"2023-10-09T05:58:27.427991+0000","last_clean":"2023-10-09T05:58:27.427991+0000","last_became_active":"2023-10-09T05:58:27.376391+0000","last_became_peered":"2023-10-09T05:58:27.376391+0000","last_unstale":"2023-10-09T05:58:27.427991+0000","last_undegraded":"2023-10-09T05:58:27.427991+0000","last_fullsized":"2023-10-09T05:58:27.427991+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:36.863116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":423,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.371292+0000","last_change":"2023-10-09T05:58:21.331166+0000","last_active":"2023-10-09T05:58:27.371292+0000","last_peered":"2023-10-09T05:58:27.371292+0000","last_clean":"2023-10-09T05:58:27.371292+0000","last_became_active":"2023-10-09T05:58:21.330754+0000","last_became_peered":"2023-10-09T05:58:21.330754+0000","last_unstale":"2023-10-09T05:58:27.371292+0000","last_undegraded":"2023-10-09T05:58:27.371292+0000","last_fullsized":"2023-10-09T05:58:27.371292+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":49,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.361724+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T05:58:27.361724+0000","last_peered":"2023-10-09T05:58:27.361724+0000","last_clean":"2023-10-09T05:58:27.361724+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T05:58:27.361724+0000","last_undegraded":"2023-10-09T05:58:27.361724+0000","last_fullsized":"2023-10-09T05:58:27.361724+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":297,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.371911+0000","last_change":"2023-10-09T05:57:23.473860+0000","last_active":"2023-10-09T05:58:27.371911+0000","last_peered":"2023-10-09T05:58:27.371911+0000","last_clean":"2023-10-09T05:58:27.371911+0000","last_became_active":"2023-10-09T05:57:23.473101+0000","last_became_peered":"2023-10-09T05:57:23.473101+0000","last_unstale":"2023-10-09T05:58:27.371911+0000","last_undegraded":"2023-10-09T05:58:27.371911+0000","last_fullsized":"2023-10-09T05:58:27.371911+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:05.682865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"0'0","reported_seq":11,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T05:58:29.081996+0000","last_change":"2023-10-09T05:58:27.376809+0000","last_active":"2023-10-09T05:58:29.081996+0000","last_peered":"2023-10-09T05:58:29.081996+0000","last_clean":"2023-10-09T05:58:29.081996+0000","last_became_active":"2023-10-09T05:58:27.376476+0000","last_became_peered":"2023-10-09T05:58:27.376476+0000","last_unstale":"2023-10-09T05:58:29.081996+0000","last_undegraded":"2023-10-09T05:58:29.081996+0000","last_fullsized":"2023-10-09T05:58:29.081996+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:53:15.515971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":700,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.375541+0000","last_change":"2023-10-09T05:57:13.234252+0000","last_active":"2023-10-09T05:58:27.375541+0000","last_peered":"2023-10-09T05:58:27.375541+0000","last_clean":"2023-10-09T05:58:27.375541+0000","last_became_active":"2023-10-09T05:57:13.233849+0000","last_became_peered":"2023-10-09T05:57:13.233849+0000","last_unstale":"2023-10-09T05:58:27.375541+0000","last_undegraded":"2023-10-09T05:58:27.375541+0000","last_fullsized":"2023-10-09T05:58:27.375541+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":53,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.360808+0000","last_change":"2023-10-09T05:57:23.504733+0000","last_active":"2023-10-09T05:58:27.360808+0000","last_peered":"2023-10-09T05:58:27.360808+0000","last_clean":"2023-10-09T05:58:27.360808+0000","last_became_active":"2023-10-09T05:57:23.468709+0000","last_became_peered":"2023-10-09T05:57:23.468709+0000","last_unstale":"2023-10-09T05:58:27.360808+0000","last_undegraded":"2023-10-09T05:58:27.360808+0000","last_fullsized":"2023-10-09T05:58:27.360808+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:00.459980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":246,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.373381+0000","last_change":"2023-10-09T05:49:21.430742+0000","last_active":"2023-10-09T05:58:27.373381+0000","last_peered":"2023-10-09T05:58:27.373381+0000","last_clean":"2023-10-09T05:58:27.373381+0000","last_became_active":"2023-10-09T05:49:21.430233+0000","last_became_peered":"2023-10-09T05:49:21.430233+0000","last_unstale":"2023-10-09T05:58:27.373381+0000","last_undegraded":"2023-10-09T05:58:27.373381+0000","last_fullsized":"2023-10-09T05:58:27.373381+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:10.541302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"0'0","reported_seq":11,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.428098+0000","last_change":"2023-10-09T05:58:27.405765+0000","last_active":"2023-10-09T05:58:27.428098+0000","last_peered":"2023-10-09T05:58:27.428098+0000","last_clean":"2023-10-09T05:58:27.428098+0000","last_became_active":"2023-10-09T05:58:27.405447+0000","last_became_peered":"2023-10-09T05:58:27.405447+0000","last_unstale":"2023-10-09T05:58:27.428098+0000","last_undegraded":"2023-10-09T05:58:27.428098+0000","last_fullsized":"2023-10-09T05:58:27.428098+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:52.191564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":493,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.364374+0000","last_change":"2023-10-09T05:57:26.517046+0000","last_active":"2023-10-09T05:58:27.364374+0000","last_peered":"2023-10-09T05:58:27.364374+0000","last_clean":"2023-10-09T05:58:27.364374+0000","last_became_active":"2023-10-09T05:57:26.516674+0000","last_became_peered":"2023-10-09T05:57:26.516674+0000","last_unstale":"2023-10-09T05:58:27.364374+0000","last_undegraded":"2023-10-09T05:58:27.364374+0000","last_fullsized":"2023-10-09T05:58:27.364374+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":297,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.373580+0000","last_change":"2023-10-09T05:55:59.320669+0000","last_active":"2023-10-09T05:58:27.373580+0000","last_peered":"2023-10-09T05:58:27.373580+0000","last_clean":"2023-10-09T05:58:27.373580+0000","last_became_active":"2023-10-09T05:55:59.320352+0000","last_became_peered":"2023-10-09T05:55:59.320352+0000","last_unstale":"2023-10-09T05:58:27.373580+0000","last_undegraded":"2023-10-09T05:58:27.373580+0000","last_fullsized":"2023-10-09T05:58:27.373580+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:09:14.227028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":333,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.373611+0000","last_change":"2023-10-09T05:57:10.362295+0000","last_active":"2023-10-09T05:58:27.373611+0000","last_peered":"2023-10-09T05:58:27.373611+0000","last_clean":"2023-10-09T05:58:27.373611+0000","last_became_active":"2023-10-09T05:57:10.361249+0000","last_became_peered":"2023-10-09T05:57:10.361249+0000","last_unstale":"2023-10-09T05:58:27.373611+0000","last_undegraded":"2023-10-09T05:58:27.373611+0000","last_fullsized":"2023-10-09T05:58:27.373611+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:22.088261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"0'0","reported_seq":10,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.379052+0000","last_change":"2023-10-09T05:58:27.378739+0000","last_active":"2023-10-09T05:58:27.379052+0000","last_peered":"2023-10-09T05:58:27.379052+0000","last_clean":"2023-10-09T05:58:27.379052+0000","last_became_active":"2023-10-09T05:58:27.378459+0000","last_became_peered":"2023-10-09T05:58:27.378459+0000","last_unstale":"2023-10-09T05:58:27.379052+0000","last_undegraded":"2023-10-09T05:58:27.379052+0000","last_fullsized":"2023-10-09T05:58:27.379052+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:32.861544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":522,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.371770+0000","last_change":"2023-10-09T05:58:21.333074+0000","last_active":"2023-10-09T05:58:27.371770+0000","last_peered":"2023-10-09T05:58:27.371770+0000","last_clean":"2023-10-09T05:58:27.371770+0000","last_became_active":"2023-10-09T05:58:21.332677+0000","last_became_peered":"2023-10-09T05:58:21.332677+0000","last_unstale":"2023-10-09T05:58:27.371770+0000","last_undegraded":"2023-10-09T05:58:27.371770+0000","last_fullsized":"2023-10-09T05:58:27.371770+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":267,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.372616+0000","last_change":"2023-10-09T05:54:00.619165+0000","last_active":"2023-10-09T05:58:27.372616+0000","last_peered":"2023-10-09T05:58:27.372616+0000","last_clean":"2023-10-09T05:58:27.372616+0000","last_became_active":"2023-10-09T05:54:00.618888+0000","last_became_peered":"2023-10-09T05:54:00.618888+0000","last_unstale":"2023-10-09T05:58:27.372616+0000","last_undegraded":"2023-10-09T05:58:27.372616+0000","last_fullsized":"2023-10-09T05:58:27.372616+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:07.175996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":306,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.371429+0000","last_change":"2023-10-09T05:57:23.483158+0000","last_active":"2023-10-09T05:58:27.371429+0000","last_peered":"2023-10-09T05:58:27.371429+0000","last_clean":"2023-10-09T05:58:27.371429+0000","last_became_active":"2023-10-09T05:57:23.482701+0000","last_became_peered":"2023-10-09T05:57:23.482701+0000","last_unstale":"2023-10-09T05:58:27.371429+0000","last_undegraded":"2023-10-09T05:58:27.371429+0000","last_fullsized":"2023-10-09T05:58:27.371429+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:51:08.624310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"0'0","reported_seq":10,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.376624+0000","last_change":"2023-10-09T05:58:27.376357+0000","last_active":"2023-10-09T05:58:27.376624+0000","last_peered":"2023-10-09T05:58:27.376624+0000","last_clean":"2023-10-09T05:58:27.376624+0000","last_became_active":"2023-10-09T05:58:27.376017+0000","last_became_peered":"2023-10-09T05:58:27.376017+0000","last_unstale":"2023-10-09T05:58:27.376624+0000","last_undegraded":"2023-10-09T05:58:27.376624+0000","last_fullsized":"2023-10-09T05:58:27.376624+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:30.067479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":472,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.364763+0000","last_change":"2023-10-09T05:57:09.347095+0000","last_active":"2023-10-09T05:58:27.364763+0000","last_peered":"2023-10-09T05:58:27.364763+0000","last_clean":"2023-10-09T05:58:27.364763+0000","last_became_active":"2023-10-09T05:56:49.229606+0000","last_became_peered":"2023-10-09T05:56:49.229606+0000","last_unstale":"2023-10-09T05:58:27.364763+0000","last_undegraded":"2023-10-09T05:58:27.364763+0000","last_fullsized":"2023-10-09T05:58:27.364763+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":49,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.361149+0000","last_change":"2023-10-09T05:57:23.469928+0000","last_active":"2023-10-09T05:58:27.361149+0000","last_peered":"2023-10-09T05:58:27.361149+0000","last_clean":"2023-10-09T05:58:27.361149+0000","last_became_active":"2023-10-09T05:57:23.469697+0000","last_became_peered":"2023-10-09T05:57:23.469697+0000","last_unstale":"2023-10-09T05:58:27.361149+0000","last_undegraded":"2023-10-09T05:58:27.361149+0000","last_fullsized":"2023-10-09T05:58:27.361149+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:21:07.179434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":166,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.361812+0000","last_change":"2023-10-09T05:55:59.409563+0000","last_active":"2023-10-09T05:58:27.361812+0000","last_peered":"2023-10-09T05:58:27.361812+0000","last_clean":"2023-10-09T05:58:27.361812+0000","last_became_active":"2023-10-09T05:55:59.409184+0000","last_became_peered":"2023-10-09T05:55:59.409184+0000","last_unstale":"2023-10-09T05:58:27.361812+0000","last_undegraded":"2023-10-09T05:58:27.361812+0000","last_fullsized":"2023-10-09T05:58:27.361812+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:39.513591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1","version":"0'0","reported_seq":10,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.377725+0000","last_change":"2023-10-09T05:58:27.377316+0000","last_active":"2023-10-09T05:58:27.377725+0000","last_peered":"2023-10-09T05:58:27.377725+0000","last_clean":"2023-10-09T05:58:27.377725+0000","last_became_active":"2023-10-09T05:58:27.377007+0000","last_became_peered":"2023-10-09T05:58:27.377007+0000","last_unstale":"2023-10-09T05:58:27.377725+0000","last_undegraded":"2023-10-09T05:58:27.377725+0000","last_fullsized":"2023-10-09T05:58:27.377725+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:10.982167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":680,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.373003+0000","last_change":"2023-10-09T05:57:26.501248+0000","last_active":"2023-10-09T05:58:27.373003+0000","last_peered":"2023-10-09T05:58:27.373003+0000","last_clean":"2023-10-09T05:58:27.373003+0000","last_became_active":"2023-10-09T05:57:26.500602+0000","last_became_peered":"2023-10-09T05:57:26.500602+0000","last_unstale":"2023-10-09T05:58:27.373003+0000","last_undegraded":"2023-10-09T05:58:27.373003+0000","last_fullsized":"2023-10-09T05:58:27.373003+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":181,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.371116+0000","last_change":"2023-10-09T05:55:59.298953+0000","last_active":"2023-10-09T05:58:27.371116+0000","last_peered":"2023-10-09T05:58:27.371116+0000","last_clean":"2023-10-09T05:58:27.371116+0000","last_became_active":"2023-10-09T05:55:59.298624+0000","last_became_peered":"2023-10-09T05:55:59.298624+0000","last_unstale":"2023-10-09T05:58:27.371116+0000","last_undegraded":"2023-10-09T05:58:27.371116+0000","last_fullsized":"2023-10-09T05:58:27.371116+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:00.513870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":49,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.361365+0000","last_change":"2023-10-09T05:57:23.468480+0000","last_active":"2023-10-09T05:58:27.361365+0000","last_peered":"2023-10-09T05:58:27.361365+0000","last_clean":"2023-10-09T05:58:27.361365+0000","last_became_active":"2023-10-09T05:57:23.468179+0000","last_became_peered":"2023-10-09T05:57:23.468179+0000","last_unstale":"2023-10-09T05:58:27.361365+0000","last_undegraded":"2023-10-09T05:58:27.361365+0000","last_fullsized":"2023-10-09T05:58:27.361365+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:18.157188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":150,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.371845+0000","last_change":"2023-10-09T05:56:49.249711+0000","last_active":"2023-10-09T05:58:27.371845+0000","last_peered":"2023-10-09T05:58:27.371845+0000","last_clean":"2023-10-09T05:58:27.371845+0000","last_became_active":"2023-10-09T05:56:49.249272+0000","last_became_peered":"2023-10-09T05:56:49.249272+0000","last_unstale":"2023-10-09T05:58:27.371845+0000","last_undegraded":"2023-10-09T05:58:27.371845+0000","last_fullsized":"2023-10-09T05:58:27.371845+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:42.596986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.7","version":"0'0","reported_seq":10,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.379060+0000","last_change":"2023-10-09T05:58:27.378776+0000","last_active":"2023-10-09T05:58:27.379060+0000","last_peered":"2023-10-09T05:58:27.379060+0000","last_clean":"2023-10-09T05:58:27.379060+0000","last_became_active":"2023-10-09T05:58:27.378504+0000","last_became_peered":"2023-10-09T05:58:27.378504+0000","last_unstale":"2023-10-09T05:58:27.379060+0000","last_undegraded":"2023-10-09T05:58:27.379060+0000","last_fullsized":"2023-10-09T05:58:27.379060+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:27.090596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":477,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.370536+0000","last_change":"2023-10-09T05:56:49.237597+0000","last_active":"2023-10-09T05:58:27.370536+0000","last_peered":"2023-10-09T05:58:27.370536+0000","last_clean":"2023-10-09T05:58:27.370536+0000","last_became_active":"2023-10-09T05:56:49.237269+0000","last_became_peered":"2023-10-09T05:56:49.237269+0000","last_unstale":"2023-10-09T05:58:27.370536+0000","last_undegraded":"2023-10-09T05:58:27.370536+0000","last_fullsized":"2023-10-09T05:58:27.370536+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":307,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.373657+0000","last_change":"2023-10-09T05:56:49.254337+0000","last_active":"2023-10-09T05:58:27.373657+0000","last_peered":"2023-10-09T05:58:27.373657+0000","last_clean":"2023-10-09T05:58:27.373657+0000","last_became_active":"2023-10-09T05:56:49.247892+0000","last_became_peered":"2023-10-09T05:56:49.247892+0000","last_unstale":"2023-10-09T05:58:27.373657+0000","last_undegraded":"2023-10-09T05:58:27.373657+0000","last_fullsized":"2023-10-09T05:58:27.373657+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:44:20.157638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":264,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.374094+0000","last_change":"2023-10-09T05:51:57.203142+0000","last_active":"2023-10-09T05:58:27.374094+0000","last_peered":"2023-10-09T05:58:27.374094+0000","last_clean":"2023-10-09T05:58:27.374094+0000","last_became_active":"2023-10-09T05:51:57.202850+0000","last_became_peered":"2023-10-09T05:51:57.202850+0000","last_unstale":"2023-10-09T05:58:27.374094+0000","last_undegraded":"2023-10-09T05:58:27.374094+0000","last_fullsized":"2023-10-09T05:58:27.374094+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:51.090737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":210,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.364944+0000","last_change":"2023-10-09T05:57:51.789746+0000","last_active":"2023-10-09T05:58:27.364944+0000","last_peered":"2023-10-09T05:58:27.364944+0000","last_clean":"2023-10-09T05:58:27.364944+0000","last_became_active":"2023-10-09T05:57:51.789353+0000","last_became_peered":"2023-10-09T05:57:51.789353+0000","last_unstale":"2023-10-09T05:58:27.364944+0000","last_undegraded":"2023-10-09T05:58:27.364944+0000","last_fullsized":"2023-10-09T05:58:27.364944+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:28.241819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.4","version":"0'0","reported_seq":11,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T05:58:29.081006+0000","last_change":"2023-10-09T05:58:27.375343+0000","last_active":"2023-10-09T05:58:29.081006+0000","last_peered":"2023-10-09T05:58:29.081006+0000","last_clean":"2023-10-09T05:58:29.081006+0000","last_became_active":"2023-10-09T05:58:27.375094+0000","last_became_peered":"2023-10-09T05:58:27.375094+0000","last_unstale":"2023-10-09T05:58:29.081006+0000","last_undegraded":"2023-10-09T05:58:29.081006+0000","last_fullsized":"2023-10-09T05:58:29.081006+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:48:08.192351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":584,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.372844+0000","last_change":"2023-10-09T05:58:01.899855+0000","last_active":"2023-10-09T05:58:27.372844+0000","last_peered":"2023-10-09T05:58:27.372844+0000","last_clean":"2023-10-09T05:58:27.372844+0000","last_became_active":"2023-10-09T05:58:01.899103+0000","last_became_peered":"2023-10-09T05:58:01.899103+0000","last_unstale":"2023-10-09T05:58:27.372844+0000","last_undegraded":"2023-10-09T05:58:27.372844+0000","last_fullsized":"2023-10-09T05:58:27.372844+0000","mapping_epoch":268,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":269,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":68,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.364194+0000","last_change":"2023-10-09T05:57:23.465501+0000","last_active":"2023-10-09T05:58:27.364194+0000","last_peered":"2023-10-09T05:58:27.364194+0000","last_clean":"2023-10-09T05:58:27.364194+0000","last_became_active":"2023-10-09T05:57:23.465103+0000","last_became_peered":"2023-10-09T05:57:23.465103+0000","last_unstale":"2023-10-09T05:58:27.364194+0000","last_undegraded":"2023-10-09T05:58:27.364194+0000","last_fullsized":"2023-10-09T05:58:27.364194+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:02:45.891751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":176,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.364205+0000","last_change":"2023-10-09T05:55:59.308743+0000","last_active":"2023-10-09T05:58:27.364205+0000","last_peered":"2023-10-09T05:58:27.364205+0000","last_clean":"2023-10-09T05:58:27.364205+0000","last_became_active":"2023-10-09T05:55:59.308220+0000","last_became_peered":"2023-10-09T05:55:59.308220+0000","last_unstale":"2023-10-09T05:58:27.364205+0000","last_undegraded":"2023-10-09T05:58:27.364205+0000","last_fullsized":"2023-10-09T05:58:27.364205+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:44.246081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":181,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.371530+0000","last_change":"2023-10-09T05:56:49.238219+0000","last_active":"2023-10-09T05:58:27.371530+0000","last_peered":"2023-10-09T05:58:27.371530+0000","last_clean":"2023-10-09T05:58:27.371530+0000","last_became_active":"2023-10-09T05:56:49.237919+0000","last_became_peered":"2023-10-09T05:56:49.237919+0000","last_unstale":"2023-10-09T05:58:27.371530+0000","last_undegraded":"2023-10-09T05:58:27.371530+0000","last_fullsized":"2023-10-09T05:58:27.371530+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:44.599032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.6","version":"0'0","reported_seq":11,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T05:58:29.082998+0000","last_change":"2023-10-09T05:58:27.377502+0000","last_active":"2023-10-09T05:58:29.082998+0000","last_peered":"2023-10-09T05:58:29.082998+0000","last_clean":"2023-10-09T05:58:29.082998+0000","last_became_active":"2023-10-09T05:58:27.377195+0000","last_became_peered":"2023-10-09T05:58:27.377195+0000","last_unstale":"2023-10-09T05:58:29.082998+0000","last_undegraded":"2023-10-09T05:58:29.082998+0000","last_fullsized":"2023-10-09T05:58:29.082998+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:14.980773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":214,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.364455+0000","last_change":"2023-10-09T05:56:49.233224+0000","last_active":"2023-10-09T05:58:27.364455+0000","last_peered":"2023-10-09T05:58:27.364455+0000","last_clean":"2023-10-09T05:58:27.364455+0000","last_became_active":"2023-10-09T05:56:49.232971+0000","last_became_peered":"2023-10-09T05:56:49.232971+0000","last_unstale":"2023-10-09T05:58:27.364455+0000","last_undegraded":"2023-10-09T05:58:27.364455+0000","last_fullsized":"2023-10-09T05:58:27.364455+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:55.265488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":681,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.373869+0000","last_change":"2023-10-09T05:57:10.809854+0000","last_active":"2023-10-09T05:58:27.373869+0000","last_peered":"2023-10-09T05:58:27.373869+0000","last_clean":"2023-10-09T05:58:27.373869+0000","last_became_active":"2023-10-09T05:53:59.202004+0000","last_became_peered":"2023-10-09T05:53:59.202004+0000","last_unstale":"2023-10-09T05:58:27.373869+0000","last_undegraded":"2023-10-09T05:58:27.373869+0000","last_fullsized":"2023-10-09T05:58:27.373869+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:51:12.346243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:51:12.346243+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":49,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.361916+0000","last_change":"2023-10-09T05:57:23.479174+0000","last_active":"2023-10-09T05:58:27.361916+0000","last_peered":"2023-10-09T05:58:27.361916+0000","last_clean":"2023-10-09T05:58:27.361916+0000","last_became_active":"2023-10-09T05:57:23.478897+0000","last_became_peered":"2023-10-09T05:57:23.478897+0000","last_unstale":"2023-10-09T05:58:27.361916+0000","last_undegraded":"2023-10-09T05:58:27.361916+0000","last_fullsized":"2023-10-09T05:58:27.361916+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:00.160042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":249,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.372187+0000","last_change":"2023-10-09T05:53:59.195325+0000","last_active":"2023-10-09T05:58:27.372187+0000","last_peered":"2023-10-09T05:58:27.372187+0000","last_clean":"2023-10-09T05:58:27.372187+0000","last_became_active":"2023-10-09T05:53:59.194954+0000","last_became_peered":"2023-10-09T05:53:59.194954+0000","last_unstale":"2023-10-09T05:58:27.372187+0000","last_undegraded":"2023-10-09T05:58:27.372187+0000","last_fullsized":"2023-10-09T05:58:27.372187+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:23.228416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"0'0","reported_seq":10,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.375675+0000","last_change":"2023-10-09T05:58:27.375153+0000","last_active":"2023-10-09T05:58:27.375675+0000","last_peered":"2023-10-09T05:58:27.375675+0000","last_clean":"2023-10-09T05:58:27.375675+0000","last_became_active":"2023-10-09T05:58:27.374820+0000","last_became_peered":"2023-10-09T05:58:27.374820+0000","last_unstale":"2023-10-09T05:58:27.375675+0000","last_undegraded":"2023-10-09T05:58:27.375675+0000","last_fullsized":"2023-10-09T05:58:27.375675+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:08.834399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":472,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.371377+0000","last_change":"2023-10-09T05:57:54.797571+0000","last_active":"2023-10-09T05:58:27.371377+0000","last_peered":"2023-10-09T05:58:27.371377+0000","last_clean":"2023-10-09T05:58:27.371377+0000","last_became_active":"2023-10-09T05:57:54.796937+0000","last_became_peered":"2023-10-09T05:57:54.796937+0000","last_unstale":"2023-10-09T05:58:27.371377+0000","last_undegraded":"2023-10-09T05:58:27.371377+0000","last_fullsized":"2023-10-09T05:58:27.371377+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":153,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.373690+0000","last_change":"2023-10-09T05:55:59.300015+0000","last_active":"2023-10-09T05:58:27.373690+0000","last_peered":"2023-10-09T05:58:27.373690+0000","last_clean":"2023-10-09T05:58:27.373690+0000","last_became_active":"2023-10-09T05:55:59.299281+0000","last_became_peered":"2023-10-09T05:55:59.299281+0000","last_unstale":"2023-10-09T05:58:27.373690+0000","last_undegraded":"2023-10-09T05:58:27.373690+0000","last_fullsized":"2023-10-09T05:58:27.373690+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:50:32.465904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":224,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.370704+0000","last_change":"2023-10-09T05:57:23.471369+0000","last_active":"2023-10-09T05:58:27.370704+0000","last_peered":"2023-10-09T05:58:27.370704+0000","last_clean":"2023-10-09T05:58:27.370704+0000","last_became_active":"2023-10-09T05:57:23.471049+0000","last_became_peered":"2023-10-09T05:57:23.471049+0000","last_unstale":"2023-10-09T05:58:27.370704+0000","last_undegraded":"2023-10-09T05:58:27.370704+0000","last_fullsized":"2023-10-09T05:58:27.370704+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:21.208825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":340,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.371573+0000","last_change":"2023-10-09T05:57:10.348399+0000","last_active":"2023-10-09T05:58:27.371573+0000","last_peered":"2023-10-09T05:58:27.371573+0000","last_clean":"2023-10-09T05:58:27.371573+0000","last_became_active":"2023-10-09T05:57:10.348061+0000","last_became_peered":"2023-10-09T05:57:10.348061+0000","last_unstale":"2023-10-09T05:58:27.371573+0000","last_undegraded":"2023-10-09T05:58:27.371573+0000","last_fullsized":"2023-10-09T05:58:27.371573+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:12.453118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"0'0","reported_seq":11,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T05:58:28.368598+0000","last_change":"2023-10-09T05:58:27.405929+0000","last_active":"2023-10-09T05:58:28.368598+0000","last_peered":"2023-10-09T05:58:28.368598+0000","last_clean":"2023-10-09T05:58:28.368598+0000","last_became_active":"2023-10-09T05:58:27.405604+0000","last_became_peered":"2023-10-09T05:58:27.405604+0000","last_unstale":"2023-10-09T05:58:28.368598+0000","last_undegraded":"2023-10-09T05:58:28.368598+0000","last_fullsized":"2023-10-09T05:58:28.368598+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:19.748373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":670,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.372656+0000","last_change":"2023-10-09T05:57:26.518453+0000","last_active":"2023-10-09T05:58:27.372656+0000","last_peered":"2023-10-09T05:58:27.372656+0000","last_clean":"2023-10-09T05:58:27.372656+0000","last_became_active":"2023-10-09T05:57:26.517990+0000","last_became_peered":"2023-10-09T05:57:26.517990+0000","last_unstale":"2023-10-09T05:58:27.372656+0000","last_undegraded":"2023-10-09T05:58:27.372656+0000","last_fullsized":"2023-10-09T05:58:27.372656+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":121,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.371994+0000","last_change":"2023-10-09T05:53:59.201576+0000","last_active":"2023-10-09T05:58:27.371994+0000","last_peered":"2023-10-09T05:58:27.371994+0000","last_clean":"2023-10-09T05:58:27.371994+0000","last_became_active":"2023-10-09T05:53:59.201309+0000","last_became_peered":"2023-10-09T05:53:59.201309+0000","last_unstale":"2023-10-09T05:58:27.371994+0000","last_undegraded":"2023-10-09T05:58:27.371994+0000","last_fullsized":"2023-10-09T05:58:27.371994+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:54.956371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":59,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.364878+0000","last_change":"2023-10-09T05:57:10.368477+0000","last_active":"2023-10-09T05:58:27.364878+0000","last_peered":"2023-10-09T05:58:27.364878+0000","last_clean":"2023-10-09T05:58:27.364878+0000","last_became_active":"2023-10-09T05:57:10.368178+0000","last_became_peered":"2023-10-09T05:57:10.368178+0000","last_unstale":"2023-10-09T05:58:27.364878+0000","last_undegraded":"2023-10-09T05:58:27.364878+0000","last_fullsized":"2023-10-09T05:58:27.364878+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:38.053000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":213,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T05:58:29.082097+0000","last_change":"2023-10-09T05:53:59.188462+0000","last_active":"2023-10-09T05:58:29.082097+0000","last_peered":"2023-10-09T05:58:29.082097+0000","last_clean":"2023-10-09T05:58:29.082097+0000","last_became_active":"2023-10-09T05:53:59.188061+0000","last_became_peered":"2023-10-09T05:53:59.188061+0000","last_unstale":"2023-10-09T05:58:29.082097+0000","last_undegraded":"2023-10-09T05:58:29.082097+0000","last_fullsized":"2023-10-09T05:58:29.082097+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:19.167114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"0'0","reported_seq":11,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T05:58:28.368887+0000","last_change":"2023-10-09T05:58:27.377750+0000","last_active":"2023-10-09T05:58:28.368887+0000","last_peered":"2023-10-09T05:58:28.368887+0000","last_clean":"2023-10-09T05:58:28.368887+0000","last_became_active":"2023-10-09T05:58:27.377457+0000","last_became_peered":"2023-10-09T05:58:27.377457+0000","last_unstale":"2023-10-09T05:58:28.368887+0000","last_undegraded":"2023-10-09T05:58:28.368887+0000","last_fullsized":"2023-10-09T05:58:28.368887+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:52:04.242389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":641,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.373471+0000","last_change":"2023-10-09T05:57:13.235727+0000","last_active":"2023-10-09T05:58:27.373471+0000","last_peered":"2023-10-09T05:58:27.373471+0000","last_clean":"2023-10-09T05:58:27.373471+0000","last_became_active":"2023-10-09T05:57:13.235353+0000","last_became_peered":"2023-10-09T05:57:13.235353+0000","last_unstale":"2023-10-09T05:58:27.373471+0000","last_undegraded":"2023-10-09T05:58:27.373471+0000","last_fullsized":"2023-10-09T05:58:27.373471+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":293,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.361914+0000","last_change":"2023-10-09T05:53:59.228948+0000","last_active":"2023-10-09T05:58:27.361914+0000","last_peered":"2023-10-09T05:58:27.361914+0000","last_clean":"2023-10-09T05:58:27.361914+0000","last_became_active":"2023-10-09T05:53:59.186359+0000","last_became_peered":"2023-10-09T05:53:59.186359+0000","last_unstale":"2023-10-09T05:58:27.361914+0000","last_undegraded":"2023-10-09T05:58:27.361914+0000","last_fullsized":"2023-10-09T05:58:27.361914+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:39.990576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":144,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.372531+0000","last_change":"2023-10-09T05:55:59.339957+0000","last_active":"2023-10-09T05:58:27.372531+0000","last_peered":"2023-10-09T05:58:27.372531+0000","last_clean":"2023-10-09T05:58:27.372531+0000","last_became_active":"2023-10-09T05:55:59.339614+0000","last_became_peered":"2023-10-09T05:55:59.339614+0000","last_unstale":"2023-10-09T05:58:27.372531+0000","last_undegraded":"2023-10-09T05:58:27.372531+0000","last_fullsized":"2023-10-09T05:58:27.372531+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:01.613113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":49,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.361358+0000","last_change":"2023-10-09T05:57:23.477143+0000","last_active":"2023-10-09T05:58:27.361358+0000","last_peered":"2023-10-09T05:58:27.361358+0000","last_clean":"2023-10-09T05:58:27.361358+0000","last_became_active":"2023-10-09T05:57:23.475655+0000","last_became_peered":"2023-10-09T05:57:23.475655+0000","last_unstale":"2023-10-09T05:58:27.361358+0000","last_undegraded":"2023-10-09T05:58:27.361358+0000","last_fullsized":"2023-10-09T05:58:27.361358+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:29.528023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"0'0","reported_seq":11,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T05:58:28.368966+0000","last_change":"2023-10-09T05:58:27.376100+0000","last_active":"2023-10-09T05:58:28.368966+0000","last_peered":"2023-10-09T05:58:28.368966+0000","last_clean":"2023-10-09T05:58:28.368966+0000","last_became_active":"2023-10-09T05:58:27.375806+0000","last_became_peered":"2023-10-09T05:58:27.375806+0000","last_unstale":"2023-10-09T05:58:28.368966+0000","last_undegraded":"2023-10-09T05:58:28.368966+0000","last_fullsized":"2023-10-09T05:58:28.368966+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:56.466666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":531,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.361477+0000","last_change":"2023-10-09T05:57:26.503736+0000","last_active":"2023-10-09T05:58:27.361477+0000","last_peered":"2023-10-09T05:58:27.361477+0000","last_clean":"2023-10-09T05:58:27.361477+0000","last_became_active":"2023-10-09T05:57:26.502911+0000","last_became_peered":"2023-10-09T05:57:26.502911+0000","last_unstale":"2023-10-09T05:58:27.361477+0000","last_undegraded":"2023-10-09T05:58:27.361477+0000","last_fullsized":"2023-10-09T05:58:27.361477+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":291,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.373114+0000","last_change":"2023-10-09T05:57:58.832790+0000","last_active":"2023-10-09T05:58:27.373114+0000","last_peered":"2023-10-09T05:58:27.373114+0000","last_clean":"2023-10-09T05:58:27.373114+0000","last_became_active":"2023-10-09T05:57:58.832471+0000","last_became_peered":"2023-10-09T05:57:58.832471+0000","last_unstale":"2023-10-09T05:58:27.373114+0000","last_undegraded":"2023-10-09T05:58:27.373114+0000","last_fullsized":"2023-10-09T05:58:27.373114+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:39.358174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":326,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.371709+0000","last_change":"2023-10-09T05:57:58.836050+0000","last_active":"2023-10-09T05:58:27.371709+0000","last_peered":"2023-10-09T05:58:27.371709+0000","last_clean":"2023-10-09T05:58:27.371709+0000","last_became_active":"2023-10-09T05:57:58.833691+0000","last_became_peered":"2023-10-09T05:57:58.833691+0000","last_unstale":"2023-10-09T05:58:27.371709+0000","last_undegraded":"2023-10-09T05:58:27.371709+0000","last_fullsized":"2023-10-09T05:58:27.371709+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:32.257849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":229,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.372665+0000","last_change":"2023-10-09T05:40:12.676587+0000","last_active":"2023-10-09T05:58:27.372665+0000","last_peered":"2023-10-09T05:58:27.372665+0000","last_clean":"2023-10-09T05:58:27.372665+0000","last_became_active":"2023-10-09T05:40:12.676330+0000","last_became_peered":"2023-10-09T05:40:12.676330+0000","last_unstale":"2023-10-09T05:58:27.372665+0000","last_undegraded":"2023-10-09T05:58:27.372665+0000","last_fullsized":"2023-10-09T05:58:27.372665+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:46.659556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":216,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.372082+0000","last_change":"2023-10-09T05:53:59.191302+0000","last_active":"2023-10-09T05:58:27.372082+0000","last_peered":"2023-10-09T05:58:27.372082+0000","last_clean":"2023-10-09T05:58:27.372082+0000","last_became_active":"2023-10-09T05:53:59.190911+0000","last_became_peered":"2023-10-09T05:53:59.190911+0000","last_unstale":"2023-10-09T05:58:27.372082+0000","last_undegraded":"2023-10-09T05:58:27.372082+0000","last_fullsized":"2023-10-09T05:58:27.372082+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:12.801201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":68,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.364361+0000","last_change":"2023-10-09T05:57:23.469986+0000","last_active":"2023-10-09T05:58:27.364361+0000","last_peered":"2023-10-09T05:58:27.364361+0000","last_clean":"2023-10-09T05:58:27.364361+0000","last_became_active":"2023-10-09T05:57:23.469546+0000","last_became_peered":"2023-10-09T05:57:23.469546+0000","last_unstale":"2023-10-09T05:58:27.364361+0000","last_undegraded":"2023-10-09T05:58:27.364361+0000","last_fullsized":"2023-10-09T05:58:27.364361+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:53.762345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":77,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.373422+0000","last_change":"2023-10-09T05:56:49.244931+0000","last_active":"2023-10-09T05:58:27.373422+0000","last_peered":"2023-10-09T05:58:27.373422+0000","last_clean":"2023-10-09T05:58:27.373422+0000","last_became_active":"2023-10-09T05:56:49.244651+0000","last_became_peered":"2023-10-09T05:56:49.244651+0000","last_unstale":"2023-10-09T05:58:27.373422+0000","last_undegraded":"2023-10-09T05:58:27.373422+0000","last_fullsized":"2023-10-09T05:58:27.373422+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:53.906578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":323,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.374115+0000","last_change":"2023-10-09T05:57:10.356351+0000","last_active":"2023-10-09T05:58:27.374115+0000","last_peered":"2023-10-09T05:58:27.374115+0000","last_clean":"2023-10-09T05:58:27.374115+0000","last_became_active":"2023-10-09T05:57:10.356037+0000","last_became_peered":"2023-10-09T05:57:10.356037+0000","last_unstale":"2023-10-09T05:58:27.374115+0000","last_undegraded":"2023-10-09T05:58:27.374115+0000","last_fullsized":"2023-10-09T05:58:27.374115+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:15.139912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":77,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.373694+0000","last_change":"2023-10-09T05:56:49.245612+0000","last_active":"2023-10-09T05:58:27.373694+0000","last_peered":"2023-10-09T05:58:27.373694+0000","last_clean":"2023-10-09T05:58:27.373694+0000","last_became_active":"2023-10-09T05:56:49.245235+0000","last_became_peered":"2023-10-09T05:56:49.245235+0000","last_unstale":"2023-10-09T05:58:27.373694+0000","last_undegraded":"2023-10-09T05:58:27.373694+0000","last_fullsized":"2023-10-09T05:58:27.373694+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:29.016794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":280,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.371630+0000","last_change":"2023-10-09T05:51:57.093104+0000","last_active":"2023-10-09T05:58:27.371630+0000","last_peered":"2023-10-09T05:58:27.371630+0000","last_clean":"2023-10-09T05:58:27.371630+0000","last_became_active":"2023-10-09T05:51:57.092810+0000","last_became_peered":"2023-10-09T05:51:57.092810+0000","last_unstale":"2023-10-09T05:58:27.371630+0000","last_undegraded":"2023-10-09T05:58:27.371630+0000","last_fullsized":"2023-10-09T05:58:27.371630+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:51.192227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":49,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.362043+0000","last_change":"2023-10-09T05:57:23.479560+0000","last_active":"2023-10-09T05:58:27.362043+0000","last_peered":"2023-10-09T05:58:27.362043+0000","last_clean":"2023-10-09T05:58:27.362043+0000","last_became_active":"2023-10-09T05:57:23.479177+0000","last_became_peered":"2023-10-09T05:57:23.479177+0000","last_unstale":"2023-10-09T05:58:27.362043+0000","last_undegraded":"2023-10-09T05:58:27.362043+0000","last_fullsized":"2023-10-09T05:58:27.362043+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:41.925161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":49,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.360789+0000","last_change":"2023-10-09T05:57:23.475179+0000","last_active":"2023-10-09T05:58:27.360789+0000","last_peered":"2023-10-09T05:58:27.360789+0000","last_clean":"2023-10-09T05:58:27.360789+0000","last_became_active":"2023-10-09T05:57:23.474904+0000","last_became_peered":"2023-10-09T05:57:23.474904+0000","last_unstale":"2023-10-09T05:58:27.360789+0000","last_undegraded":"2023-10-09T05:58:27.360789+0000","last_fullsized":"2023-10-09T05:58:27.360789+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:54.587601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":254,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.371653+0000","last_change":"2023-10-09T05:57:10.379258+0000","last_active":"2023-10-09T05:58:27.371653+0000","last_peered":"2023-10-09T05:58:27.371653+0000","last_clean":"2023-10-09T05:58:27.371653+0000","last_became_active":"2023-10-09T05:57:10.359745+0000","last_became_peered":"2023-10-09T05:57:10.359745+0000","last_unstale":"2023-10-09T05:58:27.371653+0000","last_undegraded":"2023-10-09T05:58:27.371653+0000","last_fullsized":"2023-10-09T05:58:27.371653+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:14.750256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":306,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T05:58:27.373724+0000","last_change":"2023-10-09T05:57:23.481065+0000","last_active":"2023-10-09T05:58:27.373724+0000","last_peered":"2023-10-09T05:58:27.373724+0000","last_clean":"2023-10-09T05:58:27.373724+0000","last_became_active":"2023-10-09T05:57:23.480203+0000","last_became_peered":"2023-10-09T05:57:23.480203+0000","last_unstale":"2023-10-09T05:58:27.373724+0000","last_undegraded":"2023-10-09T05:58:27.373724+0000","last_fullsized":"2023-10-09T05:58:27.373724+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:15:53.196457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":10792,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":3344512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234226,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":316748,"kb_used_data":3076,"kb_used_omap":0,"kb_used_meta":313664,"kb_avail":93428404,"statfs":{"total":95995035648,"available":95670685696,"internally_reserved":0,"allocated":3149824,"data_stored":2120122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321191936},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:02 2023","interfaces":[{"interface":"back","average":{"1min":1.9319999999999999,"5min":1.105,"15min":1.7949999999999999},"min":{"1min":0.246,"5min":0.22,"15min":0.22},"max":{"1min":21.038,"5min":21.038,"15min":21.038},"last":0.54900000000000004},{"interface":"front","average":{"1min":2.0129999999999999,"5min":1.1100000000000001,"15min":1.6859999999999999},"min":{"1min":0.245,"5min":0.224,"15min":0.224},"max":{"1min":21.152000000000001,"5min":21.152000000000001,"15min":21.152000000000001},"last":0.57199999999999995}]},{"osd":1,"last update":"Mon Oct 9 05:57:26 2023","interfaces":[{"interface":"back","average":{"1min":4.3209999999999997,"5min":1.54,"15min":0.94799999999999995},"min":{"1min":0.30299999999999999,"5min":0.27200000000000002,"15min":0.22900000000000001},"max":{"1min":35.113999999999997,"5min":35.113999999999997,"15min":35.113999999999997},"last":0.49299999999999999},{"interface":"front","average":{"1min":3.383,"5min":1.2270000000000001,"15min":0.85499999999999998},"min":{"1min":0.317,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":34.768999999999998,"5min":34.768999999999998,"15min":34.768999999999998},"last":0.37}]},{"osd":2,"last update":"Mon Oct 9 05:58:02 2023","interfaces":[{"interface":"back","average":{"1min":12.797000000000001,"5min":3.3069999999999999,"15min":2.0230000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28199999999999997},"max":{"1min":119.91200000000001,"5min":119.91200000000001,"15min":119.91200000000001},"last":0.39900000000000002},{"interface":"front","average":{"1min":12.898999999999999,"5min":3.3580000000000001,"15min":2.117},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":119.95099999999999,"5min":119.95099999999999,"15min":119.95099999999999},"last":0.442}]},{"osd":3,"last update":"Mon Oct 9 05:57:32 2023","interfaces":[{"interface":"back","average":{"1min":7.0090000000000003,"5min":2.5270000000000001,"15min":2.9289999999999998},"min":{"1min":0.38700000000000001,"5min":0.35999999999999999,"15min":0.33300000000000002},"max":{"1min":70.840999999999994,"5min":70.840999999999994,"15min":147.54900000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":6.6109999999999998,"5min":2.4199999999999999,"15min":2.8250000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":71.212000000000003,"5min":71.212000000000003,"15min":144.065},"last":0.65200000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":0.66500000000000004,"15min":0.60899999999999999},"min":{"1min":0.27500000000000002,"5min":0.27000000000000002,"15min":0.24399999999999999},"max":{"1min":2.9990000000000001,"5min":3.7120000000000002,"15min":9.5329999999999995},"last":0.624},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.60599999999999998,"15min":0.66100000000000003},"min":{"1min":0.33500000000000002,"5min":0.29899999999999999,"15min":0.27400000000000002},"max":{"1min":3.835,"5min":3.835,"15min":7.6120000000000001},"last":0.60699999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.91900000000000004,"15min":0.75800000000000001},"min":{"1min":0.29799999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.8090000000000002,"5min":15.622,"15min":15.622},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.76400000000000001,"5min":0.64000000000000001,"15min":0.624},"min":{"1min":0.33300000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.3969999999999998,"5min":3.1869999999999998,"15min":3.9380000000000002},"last":0.59099999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":0.86599999999999999,"15min":1.2769999999999999},"min":{"1min":0.29499999999999998,"5min":0.28799999999999998,"15min":0.28299999999999997},"max":{"1min":3.2229999999999999,"5min":12.727,"15min":131.94399999999999},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.82099999999999995,"5min":0.70099999999999996,"15min":1.258},"min":{"1min":0.378,"5min":0.33100000000000002,"15min":0.29999999999999999},"max":{"1min":3.5859999999999999,"5min":3.5859999999999999,"15min":135.23400000000001},"last":0.70499999999999996}]}]},{"osd":6,"up_from":46,"seq":197568495863,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":337852,"kb_used_data":36468,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93407300,"statfs":{"total":95995035648,"available":95649075200,"internally_reserved":0,"allocated":37343232,"data_stored":36329503,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2200000000000002,"5min":1.534,"15min":1.286},"min":{"1min":0.23499999999999999,"5min":0.224,"15min":0.224},"max":{"1min":13.026999999999999,"5min":49.389000000000003,"15min":49.389000000000003},"last":0.58399999999999996},{"interface":"front","average":{"1min":1.8280000000000001,"5min":1.4710000000000001,"15min":1.3089999999999999},"min":{"1min":0.26300000000000001,"5min":0.253,"15min":0.253},"max":{"1min":12.629,"5min":49.338999999999999,"15min":49.338999999999999},"last":0.64800000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:57:40 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":0.96399999999999997,"15min":0.77200000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.23899999999999999},"max":{"1min":17.341000000000001,"5min":17.341000000000001,"15min":17.341000000000001},"last":0.66700000000000004},{"interface":"front","average":{"1min":1.867,"5min":0.96799999999999997,"15min":0.81499999999999995},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":12.474,"5min":12.474,"15min":12.474},"last":0.60799999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:58:02 2023","interfaces":[{"interface":"back","average":{"1min":6.8380000000000001,"5min":2.097,"15min":1.1899999999999999},"min":{"1min":0.311,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":61.088999999999999,"5min":61.088999999999999,"15min":61.088999999999999},"last":0.69299999999999995},{"interface":"front","average":{"1min":6.3579999999999997,"5min":2.1040000000000001,"15min":1.1879999999999999},"min":{"1min":0.38800000000000001,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":61.082000000000001,"5min":61.082000000000001,"15min":61.082000000000001},"last":0.54900000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:57:40 2023","interfaces":[{"interface":"back","average":{"1min":2.6619999999999999,"5min":3.714,"15min":1.7809999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":20.478000000000002,"5min":93.626000000000005,"15min":93.626000000000005},"last":0.56000000000000005},{"interface":"front","average":{"1min":2.7450000000000001,"5min":3.444,"15min":1.7030000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":20.5,"5min":93.643000000000001,"15min":93.643000000000001},"last":0.63}]},{"osd":4,"last update":"Mon Oct 9 05:57:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":0.72599999999999998,"15min":0.71099999999999997},"min":{"1min":0.252,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.5709999999999997,"5min":4.5709999999999997,"15min":7.9100000000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.2629999999999999,"5min":0.84999999999999998,"15min":0.73099999999999998},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.2069999999999999,"5min":8.734,"15min":8.734},"last":0.374}]},{"osd":5,"last update":"Mon Oct 9 05:57:48 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":1.284,"15min":0.89500000000000002},"min":{"1min":0.316,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.5910000000000002,"5min":45.933999999999997,"15min":45.933999999999997},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.2669999999999999,"15min":0.92400000000000004},"min":{"1min":0.35899999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.786,"5min":45.917999999999999,"15min":45.917999999999999},"last":0.40699999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":1.337,"15min":1.04},"min":{"1min":0.33500000000000002,"5min":0.314,"15min":0.314},"max":{"1min":4.4820000000000002,"5min":52.042999999999999,"15min":52.042999999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.093,"5min":1.3080000000000001,"15min":1.008},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.9539999999999997,"5min":51.259,"15min":51.259},"last":0.44}]}]},{"osd":1,"up_from":0,"seq":382252089451,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":129,"seq":554050781253,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":141956,"kb_used_data":4092,"kb_used_omap":29,"kb_used_meta":137826,"kb_avail":93603196,"statfs":{"total":95995035648,"available":95849672704,"internally_reserved":0,"allocated":4190208,"data_stored":2526398,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30322,"internal_metadata":141134222},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:58:04 2023","interfaces":[{"interface":"back","average":{"1min":1.387,"5min":0.94899999999999995,"15min":0.78400000000000003},"min":{"1min":0.21299999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":6.6120000000000001,"5min":9.1720000000000006,"15min":9.1720000000000006},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.99399999999999999,"5min":0.84999999999999998,"15min":1.048},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":3.4449999999999998,"5min":6.6420000000000003,"15min":6.6420000000000003},"last":0.435}]},{"osd":2,"last update":"Mon Oct 9 05:58:04 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":1.204,"15min":1.0940000000000001},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":13.471,"5min":13.471,"15min":13.471},"last":0.48299999999999998},{"interface":"front","average":{"1min":2.4500000000000002,"5min":1.1200000000000001,"15min":1.046},"min":{"1min":0.245,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":13.561,"5min":13.561,"15min":13.561},"last":0.39100000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:58:04 2023","interfaces":[{"interface":"back","average":{"1min":4.6760000000000002,"5min":1.635,"15min":1.0609999999999999},"min":{"1min":0.29499999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":27.954000000000001,"5min":27.954000000000001,"15min":27.954000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":3.851,"5min":1.4259999999999999,"15min":1.0760000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":20.027000000000001,"5min":20.027000000000001,"15min":20.027000000000001},"last":0.317}]},{"osd":4,"last update":"Mon Oct 9 05:58:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":3.9910000000000001,"15min":2.145},"min":{"1min":0.316,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.3639999999999999,"5min":149.07400000000001,"15min":149.07400000000001},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.2110000000000001,"5min":3.8399999999999999,"15min":1.8819999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.4509999999999996,"5min":149.12700000000001,"15min":149.12700000000001},"last":0.56699999999999995}]},{"osd":5,"last update":"Mon Oct 9 05:58:04 2023","interfaces":[{"interface":"back","average":{"1min":1.873,"5min":2.4510000000000001,"15min":1.583},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":11.09,"5min":100.25,"15min":100.25},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.6459999999999999,"5min":2.4550000000000001,"15min":1.4279999999999999},"min":{"1min":0.39800000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":5.077,"5min":102.40300000000001,"15min":102.40300000000001},"last":0.53300000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:58:04 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":0.94599999999999995,"15min":1.21},"min":{"1min":0.35199999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":6.5439999999999996,"5min":8.1470000000000002,"15min":8.1470000000000002},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.825,"5min":1.2350000000000001,"15min":1.2290000000000001},"min":{"1min":0.44500000000000001,"5min":0.378,"15min":0.378},"max":{"1min":4.4000000000000004,"5min":9.141,"15min":9.141},"last":0.499}]},{"osd":7,"last update":"Mon Oct 9 05:58:04 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":2.0470000000000002,"15min":1.629},"min":{"1min":0.42399999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":4.6139999999999999,"5min":99.489000000000004,"15min":99.489000000000004},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.032,"5min":1.7909999999999999,"15min":1.534},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":3.581,"5min":99.510000000000005,"15min":99.510000000000005},"last":0.59899999999999998}]}]},{"osd":2,"up_from":94,"seq":403726925943,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":724464,"kb_used_data":455208,"kb_used_omap":81,"kb_used_meta":269166,"kb_avail":93020688,"statfs":{"total":95995035648,"available":95253184512,"internally_reserved":0,"allocated":466132992,"data_stored":464601109,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":275626202},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:11 2023","interfaces":[{"interface":"back","average":{"1min":4.9169999999999998,"5min":2.056,"15min":1.8169999999999999},"min":{"1min":0.23300000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":55.409999999999997,"5min":55.409999999999997,"15min":55.409999999999997},"last":0.29299999999999998},{"interface":"front","average":{"1min":4.7009999999999996,"5min":2.056,"15min":1.726},"min":{"1min":0.28199999999999997,"5min":0.23899999999999999,"15min":0.23100000000000001},"max":{"1min":55.326000000000001,"5min":55.326000000000001,"15min":55.326000000000001},"last":0.318}]},{"osd":1,"last update":"Mon Oct 9 05:58:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.0069999999999999,"15min":1.357},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":3.8959999999999999,"5min":7.7119999999999997,"15min":163.24199999999999},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.4550000000000001,"5min":0.85299999999999998,"15min":1.272},"min":{"1min":0.223,"5min":0.223,"15min":0.214},"max":{"1min":6.9790000000000001,"5min":6.9790000000000001,"15min":163.27000000000001},"last":0.39000000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:57:32 2023","interfaces":[{"interface":"back","average":{"1min":7.1760000000000002,"5min":2.0089999999999999,"15min":1.7549999999999999},"min":{"1min":0.27500000000000002,"5min":0.25900000000000001,"15min":0.23899999999999999},"max":{"1min":55.118000000000002,"5min":55.118000000000002,"15min":170.07599999999999},"last":0.375},{"interface":"front","average":{"1min":6.8540000000000001,"5min":2.0499999999999998,"15min":1.345},"min":{"1min":0.33400000000000002,"5min":0.29799999999999999,"15min":0.25},"max":{"1min":55.365000000000002,"5min":55.365000000000002,"15min":63.600000000000001},"last":0.44700000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:58:05 2023","interfaces":[{"interface":"back","average":{"1min":1.7529999999999999,"5min":1.1060000000000001,"15min":0.95299999999999996},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28199999999999997},"max":{"1min":8.6280000000000001,"5min":8.6280000000000001,"15min":11.372999999999999},"last":0.84499999999999997},{"interface":"front","average":{"1min":1.391,"5min":0.93700000000000006,"15min":0.90100000000000002},"min":{"1min":0.373,"5min":0.33200000000000002,"15min":0.26800000000000002},"max":{"1min":6.1710000000000003,"5min":6.1710000000000003,"15min":6.5709999999999997},"last":0.82899999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:58:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5860000000000001,"5min":1.3380000000000001,"15min":0.995},"min":{"1min":0.35099999999999998,"5min":0.30599999999999999,"15min":0.28999999999999998},"max":{"1min":5.3380000000000001,"5min":13.414,"15min":13.414},"last":0.79500000000000004},{"interface":"front","average":{"1min":2.101,"5min":1.242,"15min":0.97799999999999998},"min":{"1min":0.33000000000000002,"5min":0.313,"15min":0.29799999999999999},"max":{"1min":8.5570000000000004,"5min":13.438000000000001,"15min":13.438000000000001},"last":0.625}]},{"osd":6,"last update":"Mon Oct 9 05:58:05 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":0.96199999999999997,"15min":0.91100000000000003},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.29399999999999998},"max":{"1min":3.2999999999999998,"5min":8.6920000000000002,"15min":11.59},"last":0.85599999999999998},{"interface":"front","average":{"1min":1.48,"5min":1.113,"15min":0.90900000000000003},"min":{"1min":0.39800000000000002,"5min":0.37,"15min":0.34000000000000002},"max":{"1min":4.4850000000000003,"5min":8.6140000000000008,"15min":8.6140000000000008},"last":0.60499999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:58:05 2023","interfaces":[{"interface":"back","average":{"1min":4.4020000000000001,"5min":1.8620000000000001,"15min":1.274},"min":{"1min":0.44700000000000001,"5min":0.36599999999999999,"15min":0.35899999999999999},"max":{"1min":55.052,"5min":55.052,"15min":55.052},"last":0.65800000000000003},{"interface":"front","average":{"1min":2.0110000000000001,"5min":1.252,"15min":1.093},"min":{"1min":0.438,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.6010000000000009,"5min":8.6010000000000009,"15min":40.113},"last":0.72699999999999998}]}]},{"osd":3,"up_from":99,"seq":425201762416,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":582840,"kb_used_data":313264,"kb_used_omap":7,"kb_used_meta":269560,"kb_avail":93162312,"statfs":{"total":95995035648,"available":95398207488,"internally_reserved":0,"allocated":320782336,"data_stored":319064801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":276030056},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":0.93100000000000005,"15min":1.3420000000000001},"min":{"1min":0.224,"5min":0.186,"15min":0.186},"max":{"1min":4.7380000000000004,"5min":12.548,"15min":15.536},"last":0.53100000000000003},{"interface":"front","average":{"1min":2.0550000000000002,"5min":1.363,"15min":1.516},"min":{"1min":0.22800000000000001,"5min":0.214,"15min":0.214},"max":{"1min":14.247,"5min":14.707000000000001,"15min":14.707000000000001},"last":1.0009999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:57:38 2023","interfaces":[{"interface":"back","average":{"1min":1.4019999999999999,"5min":0.99399999999999999,"15min":1.105},"min":{"1min":0.217,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":14.568,"5min":14.568,"15min":127.226},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.82099999999999995,"15min":1.089},"min":{"1min":0.28599999999999998,"5min":0.24399999999999999,"15min":0.21099999999999999},"max":{"1min":2.5230000000000001,"5min":9.4239999999999995,"15min":127.468},"last":1.431}]},{"osd":2,"last update":"Mon Oct 9 05:57:38 2023","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":0.61499999999999999,"15min":0.75800000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":2.677,"5min":3.177,"15min":62.938000000000002},"last":1.26},{"interface":"front","average":{"1min":1.268,"5min":1.0169999999999999,"15min":1.004},"min":{"1min":0.30499999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":9.1039999999999992,"5min":9.8369999999999997,"15min":62.43},"last":1.3580000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:57:38 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":2.1829999999999998,"15min":1.2390000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.28199999999999997},"max":{"1min":13.154999999999999,"5min":142.994,"15min":142.994},"last":1.327},{"interface":"front","average":{"1min":2.2570000000000001,"5min":2.4380000000000002,"15min":1.2869999999999999},"min":{"1min":0.33600000000000002,"5min":0.312,"15min":0.311},"max":{"1min":14.455,"5min":143.411,"15min":143.411},"last":0.879}]},{"osd":5,"last update":"Mon Oct 9 05:57:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6519999999999999,"5min":1.0309999999999999,"15min":0.82199999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.33800000000000002},"max":{"1min":14.404,"5min":14.404,"15min":14.404},"last":0.82499999999999996},{"interface":"front","average":{"1min":1.8859999999999999,"5min":0.93000000000000005,"15min":0.79200000000000004},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":18.013999999999999,"5min":18.013999999999999,"15min":18.013999999999999},"last":1.1459999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:57:38 2023","interfaces":[{"interface":"back","average":{"1min":3.8929999999999998,"5min":1.544,"15min":1.0409999999999999},"min":{"1min":0.44,"5min":0.374,"15min":0.30499999999999999},"max":{"1min":44.079000000000001,"5min":44.079000000000001,"15min":44.079000000000001},"last":0.91300000000000003},{"interface":"front","average":{"1min":3.5619999999999998,"5min":1.3140000000000001,"15min":0.99199999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":44.094000000000001,"5min":44.094000000000001,"15min":44.094000000000001},"last":1.2330000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:57:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0539999999999998,"5min":0.98599999999999999,"15min":0.85799999999999998},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":10.295,"5min":10.295,"15min":10.295},"last":1.1830000000000001},{"interface":"front","average":{"1min":1.0620000000000001,"5min":0.91900000000000004,"15min":0.84599999999999997},"min":{"1min":0.436,"5min":0.41399999999999998,"15min":0.40600000000000003},"max":{"1min":3.9689999999999999,"5min":9.3879999999999999,"15min":9.3879999999999999},"last":1.3919999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019136,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":376396,"kb_used_data":66820,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93368756,"statfs":{"total":95995035648,"available":95609606144,"internally_reserved":0,"allocated":68423680,"data_stored":67394231,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:08 2023","interfaces":[{"interface":"back","average":{"1min":1.3240000000000001,"5min":1.5429999999999999,"15min":1.0349999999999999},"min":{"1min":0.23300000000000001,"5min":0.222,"15min":0.222},"max":{"1min":7.6200000000000001,"5min":19.489000000000001,"15min":19.489000000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":2.2909999999999999,"5min":1.754,"15min":1.607},"min":{"1min":0.248,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":8.9800000000000004,"5min":19.757000000000001,"15min":19.757000000000001},"last":0.42799999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:57:33 2023","interfaces":[{"interface":"back","average":{"1min":4.7229999999999999,"5min":1.5309999999999999,"15min":1.03},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":77.951999999999998,"5min":77.951999999999998,"15min":77.951999999999998},"last":0.628},{"interface":"front","average":{"1min":5.4809999999999999,"5min":1.7090000000000001,"15min":1.0269999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":77.968000000000004,"5min":77.968000000000004,"15min":77.968000000000004},"last":0.52000000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:58:08 2023","interfaces":[{"interface":"back","average":{"1min":7.7610000000000001,"5min":2.3260000000000001,"15min":1.391},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":136.15700000000001,"5min":136.15700000000001,"15min":136.15700000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":7.968,"5min":2.5059999999999998,"15min":1.498},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":136.30500000000001,"5min":136.30500000000001,"15min":136.30500000000001},"last":0.58499999999999996}]},{"osd":3,"last update":"Mon Oct 9 05:57:33 2023","interfaces":[{"interface":"back","average":{"1min":2.847,"5min":1.353,"15min":1.109},"min":{"1min":0.311,"5min":0.30599999999999999,"15min":0.29299999999999998},"max":{"1min":12.554,"5min":12.554,"15min":40.531999999999996},"last":0.60199999999999998},{"interface":"front","average":{"1min":2.8610000000000002,"5min":1.2749999999999999,"15min":1.127},"min":{"1min":0.377,"5min":0.31,"15min":0.31},"max":{"1min":12.403,"5min":12.403,"15min":39.905999999999999},"last":0.55600000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:58:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":0.77200000000000002,"15min":0.69199999999999995},"min":{"1min":0.28299999999999997,"5min":0.27500000000000002,"15min":0.25900000000000001},"max":{"1min":4.0780000000000003,"5min":4.9349999999999996,"15min":10.012},"last":0.498},{"interface":"front","average":{"1min":0.94799999999999995,"5min":0.77200000000000002,"15min":0.76800000000000002},"min":{"1min":0.29599999999999999,"5min":0.254,"15min":0.254},"max":{"1min":5.3319999999999999,"5min":12.121,"15min":12.489000000000001},"last":0.39200000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:57:45 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":0.69999999999999996,"15min":0.71999999999999997},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.4379999999999997,"5min":5.4379999999999997,"15min":11.234999999999999},"last":0.69599999999999995},{"interface":"front","average":{"1min":2.1230000000000002,"5min":0.85799999999999998,"15min":0.71499999999999997},"min":{"1min":0.39200000000000002,"5min":0.30599999999999999,"15min":0.30299999999999999},"max":{"1min":8.8610000000000007,"5min":8.8610000000000007,"15min":9.923},"last":0.70999999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:58:11 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.82299999999999995,"15min":0.85499999999999998},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.28599999999999998},"max":{"1min":2.5209999999999999,"5min":6.0570000000000004,"15min":10.606},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.98399999999999999,"5min":0.74399999999999999,"15min":0.747},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.145,"5min":5.524,"15min":8.3819999999999997},"last":0.73399999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757500,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":347208,"kb_used_data":37632,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93397944,"statfs":{"total":95995035648,"available":95639494656,"internally_reserved":0,"allocated":38535168,"data_stored":37514133,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:10 2023","interfaces":[{"interface":"back","average":{"1min":2.8690000000000002,"5min":1.085,"15min":0.83299999999999996},"min":{"1min":0.26000000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":24.956,"5min":24.956,"15min":24.956},"last":0.75},{"interface":"front","average":{"1min":3.2389999999999999,"5min":1.3149999999999999,"15min":0.96999999999999997},"min":{"1min":0.32100000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":24.975000000000001,"5min":24.975000000000001,"15min":24.975000000000001},"last":0.56899999999999995}]},{"osd":1,"last update":"Mon Oct 9 05:57:33 2023","interfaces":[{"interface":"back","average":{"1min":1.3200000000000001,"5min":2.4900000000000002,"15min":1.6459999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.8410000000000002,"5min":99.605999999999995,"15min":114.247},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.0089999999999999,"5min":2.1579999999999999,"15min":1.5269999999999999},"min":{"1min":0.28399999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.7130000000000001,"5min":99.671999999999997,"15min":114.678},"last":0.78200000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:57:59 2023","interfaces":[{"interface":"back","average":{"1min":12.375999999999999,"5min":3.2130000000000001,"15min":2.0150000000000001},"min":{"1min":0.28499999999999998,"5min":0.28399999999999997,"15min":0.25},"max":{"1min":217.06399999999999,"5min":217.06399999999999,"15min":217.06399999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":11.965999999999999,"5min":3.0870000000000002,"15min":1.9710000000000001},"min":{"1min":0.308,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":217.018,"5min":217.018,"15min":217.018},"last":0.51400000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:57:33 2023","interfaces":[{"interface":"back","average":{"1min":5.4450000000000003,"5min":3.8919999999999999,"15min":3.714},"min":{"1min":0.35799999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":60,"5min":100.774,"15min":272.81299999999999},"last":0.58699999999999997},{"interface":"front","average":{"1min":5.9500000000000002,"5min":4.274,"15min":3.9529999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":59.817,"5min":100.83499999999999,"15min":273.00999999999999},"last":0.72099999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:58:25 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.27400000000000002,"5min":0.26400000000000001,"15min":0.24199999999999999},"max":{"1min":2.8620000000000001,"5min":3.1080000000000001,"15min":6.2050000000000001},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.73699999999999999,"5min":0.63,"15min":0.61599999999999999},"min":{"1min":0.313,"5min":0.27600000000000002,"15min":0.26000000000000001},"max":{"1min":3.879,"5min":3.879,"15min":13.050000000000001},"last":0.64000000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:57:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":0.72599999999999998,"15min":0.65500000000000003},"min":{"1min":0.33600000000000002,"5min":0.30199999999999999,"15min":0.26600000000000001},"max":{"1min":10.94,"5min":10.94,"15min":10.94},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.72399999999999998,"15min":0.71799999999999997},"min":{"1min":0.33100000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.5069999999999997,"5min":5.5069999999999997,"15min":5.9770000000000003},"last":0.66300000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:58:16 2023","interfaces":[{"interface":"back","average":{"1min":1.204,"5min":0.871,"15min":0.82999999999999996},"min":{"1min":0.39500000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":4.1970000000000001,"5min":5.5800000000000001,"15min":9.2739999999999991},"last":0.76600000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.68000000000000005,"15min":0.65900000000000003},"min":{"1min":0.34499999999999997,"5min":0.32100000000000001,"15min":0.32000000000000001},"max":{"1min":1.762,"5min":2.7530000000000001,"15min":5.032},"last":0.81299999999999994}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1474,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:58:37.745 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:37.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:37.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:37.854 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:37.885 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:38.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:38 smithi064 ceph-mon[220044]: pgmap v651: 109 pgs: 109 active+clean; 564 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:38.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:38 smithi064 ceph-mon[220044]: failed to encode map e289 with expected crc 2023-10-09T05:58:38.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:38 smithi064 ceph-mon[220044]: failed to encode map e289 with expected crc 2023-10-09T05:58:38.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:38 smithi064 ceph-mon[220044]: 1.13 scrub starts 2023-10-09T05:58:38.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:38 smithi064 ceph-mon[220044]: 1.13 scrub ok 2023-10-09T05:58:38.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:38 smithi064 ceph-mon[220044]: 1.e scrub starts 2023-10-09T05:58:38.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:38 smithi064 ceph-mon[220044]: 1.e scrub ok 2023-10-09T05:58:38.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:38 smithi064 ceph-mon[212750]: pgmap v651: 109 pgs: 109 active+clean; 564 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:38.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:38 smithi064 ceph-mon[212750]: failed to encode map e289 with expected crc 2023-10-09T05:58:38.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:38 smithi064 ceph-mon[212750]: failed to encode map e289 with expected crc 2023-10-09T05:58:38.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:38 smithi064 ceph-mon[212750]: 1.13 scrub starts 2023-10-09T05:58:38.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:38 smithi064 ceph-mon[212750]: 1.13 scrub ok 2023-10-09T05:58:38.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:38 smithi064 ceph-mon[212750]: 1.e scrub starts 2023-10-09T05:58:38.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:38 smithi064 ceph-mon[212750]: 1.e scrub ok 2023-10-09T05:58:38.535 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:38.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:38.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:38.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:38 smithi156 ceph-mon[195101]: pgmap v651: 109 pgs: 109 active+clean; 564 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:38.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:38 smithi156 ceph-mon[195101]: failed to encode map e289 with expected crc 2023-10-09T05:58:38.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:38 smithi156 ceph-mon[195101]: failed to encode map e289 with expected crc 2023-10-09T05:58:38.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:38 smithi156 ceph-mon[195101]: 1.13 scrub starts 2023-10-09T05:58:38.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:38 smithi156 ceph-mon[195101]: 1.13 scrub ok 2023-10-09T05:58:38.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:38 smithi156 ceph-mon[195101]: 1.e scrub starts 2023-10-09T05:58:38.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:38 smithi156 ceph-mon[195101]: 1.e scrub ok 2023-10-09T05:58:38.639 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:38.671 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:38.823 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:38.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:38.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:38.884 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:58:38.904 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:58:38.906 DEBUG:teuthology.orchestra.run.smithi064:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-10-09T05:58:38.908 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T05:58:38.908 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T05:58:38.926 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:38.928 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:38.933 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:58:38.958 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:58:38.960 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:38.964 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:hints = 1 2023-10-09T05:58:38.965 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T05:58:38.965 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object prefix: benchmark_data_smithi064_398483 2023-10-09T05:58:38.965 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:58:38.965 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 0 0 0 0 0 0 - 0 2023-10-09T05:58:39.000 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:58:39.035 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:39.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:39.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:39.043 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:58:39.087 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:58:39.141 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:39.161 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:58:39.161 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:58:39.161 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:58:39.162 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:58:39.162 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:58:39.162 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:58:39.162 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:58:39.162 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:58:39.162 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:58:39.205 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:39.278 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:39 smithi156 ceph-mon[195101]: 1.0 scrub starts 2023-10-09T05:58:39.278 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:39 smithi156 ceph-mon[195101]: 1.0 scrub ok 2023-10-09T05:58:39.278 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:39 smithi156 ceph-mon[195101]: 1.11 scrub starts 2023-10-09T05:58:39.278 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:39 smithi156 ceph-mon[195101]: 1.11 scrub ok 2023-10-09T05:58:39.278 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:39 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:58:39.279 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:39 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2164761637' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:39.421 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:58:39.421 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:58:39.453 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:39 smithi064 ceph-mon[212750]: 1.0 scrub starts 2023-10-09T05:58:39.453 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:39 smithi064 ceph-mon[212750]: 1.0 scrub ok 2023-10-09T05:58:39.454 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:39 smithi064 ceph-mon[212750]: 1.11 scrub starts 2023-10-09T05:58:39.454 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:39 smithi064 ceph-mon[212750]: 1.11 scrub ok 2023-10-09T05:58:39.454 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:39 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:58:39.454 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:39 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2164761637' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:39.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:39 smithi064 ceph-mon[220044]: 1.0 scrub starts 2023-10-09T05:58:39.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:39 smithi064 ceph-mon[220044]: 1.0 scrub ok 2023-10-09T05:58:39.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:39 smithi064 ceph-mon[220044]: 1.11 scrub starts 2023-10-09T05:58:39.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:39 smithi064 ceph-mon[220044]: 1.11 scrub ok 2023-10-09T05:58:39.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:39 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:58:39.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:39 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2164761637' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:39.965 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 1 16 3068 3052 190.719 190.75 0.0025163 0.00522331 2023-10-09T05:58:40.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:40 smithi156 ceph-mon[195101]: pgmap v652: 109 pgs: 109 active+clean; 646 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:40.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:40 smithi156 ceph-mon[195101]: 1.b scrub starts 2023-10-09T05:58:40.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:40 smithi156 ceph-mon[195101]: 1.b scrub ok 2023-10-09T05:58:40.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:40 smithi156 ceph-mon[195101]: 1.c scrub starts 2023-10-09T05:58:40.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:40 smithi156 ceph-mon[195101]: 1.c scrub ok 2023-10-09T05:58:40.714 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:58:40.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:40 smithi064 ceph-mon[212750]: pgmap v652: 109 pgs: 109 active+clean; 646 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:40.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:40 smithi064 ceph-mon[212750]: 1.b scrub starts 2023-10-09T05:58:40.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:40 smithi064 ceph-mon[212750]: 1.b scrub ok 2023-10-09T05:58:40.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:40 smithi064 ceph-mon[212750]: 1.c scrub starts 2023-10-09T05:58:40.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:40 smithi064 ceph-mon[212750]: 1.c scrub ok 2023-10-09T05:58:40.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:40 smithi064 ceph-mon[220044]: pgmap v652: 109 pgs: 109 active+clean; 646 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-10-09T05:58:40.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:40 smithi064 ceph-mon[220044]: 1.b scrub starts 2023-10-09T05:58:40.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:40 smithi064 ceph-mon[220044]: 1.b scrub ok 2023-10-09T05:58:40.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:40 smithi064 ceph-mon[220044]: 1.c scrub starts 2023-10-09T05:58:40.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:40 smithi064 ceph-mon[220044]: 1.c scrub ok 2023-10-09T05:58:40.926 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:40.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:40.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:40.965 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 2 16 6480 6464 201.974 213.25 0.00491944 0.00494471 2023-10-09T05:58:41.043 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:41.089 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:41.170 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:41.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:41.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:41.275 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:41.341 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:41.965 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 3 16 9454 9438 196.603 185.875 0.00522207 0.00507819 2023-10-09T05:58:42.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:42 smithi156 ceph-mon[195101]: pgmap v653: 109 pgs: 109 active+clean; 111 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 14 MiB/s wr, 201 op/s 2023-10-09T05:58:42.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:42 smithi156 ceph-mon[195101]: 1.3 scrub starts 2023-10-09T05:58:42.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:42 smithi156 ceph-mon[195101]: 1.3 scrub ok 2023-10-09T05:58:42.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:42 smithi064 ceph-mon[212750]: pgmap v653: 109 pgs: 109 active+clean; 111 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 14 MiB/s wr, 201 op/s 2023-10-09T05:58:42.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:42 smithi064 ceph-mon[212750]: 1.3 scrub starts 2023-10-09T05:58:42.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:42 smithi064 ceph-mon[212750]: 1.3 scrub ok 2023-10-09T05:58:42.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:42 smithi064 ceph-mon[220044]: pgmap v653: 109 pgs: 109 active+clean; 111 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 14 MiB/s wr, 201 op/s 2023-10-09T05:58:42.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:42 smithi064 ceph-mon[220044]: 1.3 scrub starts 2023-10-09T05:58:42.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:42 smithi064 ceph-mon[220044]: 1.3 scrub ok 2023-10-09T05:58:42.965 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 4 16 12465 12449 194.494 188.188 0.00591646 0.00513628 2023-10-09T05:58:43.064 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:43.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:43.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:43.170 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:43.212 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:43.292 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:43.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:43.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:43.398 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:43.440 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:43.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:43 smithi064 ceph-mon[220044]: 1.f scrub starts 2023-10-09T05:58:43.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:43 smithi064 ceph-mon[220044]: 1.f scrub ok 2023-10-09T05:58:43.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:43 smithi064 ceph-mon[220044]: pgmap v654: 109 pgs: 109 active+clean; 197 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 21 MiB/s wr, 317 op/s 2023-10-09T05:58:43.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:43 smithi064 ceph-mon[212750]: 1.f scrub starts 2023-10-09T05:58:43.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:43 smithi064 ceph-mon[212750]: 1.f scrub ok 2023-10-09T05:58:43.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:43 smithi064 ceph-mon[212750]: pgmap v654: 109 pgs: 109 active+clean; 197 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 21 MiB/s wr, 317 op/s 2023-10-09T05:58:43.509 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:43.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:43.516 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:43 smithi156 ceph-mon[195101]: 1.f scrub starts 2023-10-09T05:58:43.516 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:43 smithi156 ceph-mon[195101]: 1.f scrub ok 2023-10-09T05:58:43.516 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:43 smithi156 ceph-mon[195101]: pgmap v654: 109 pgs: 109 active+clean; 197 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 21 MiB/s wr, 317 op/s 2023-10-09T05:58:43.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:43.617 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:43.682 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:43.769 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:58:43.910 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:43.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:43.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:43.966 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 5 16 15234 15218 190.206 173.062 0.0073429 0.00525119 2023-10-09T05:58:44.016 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:44.060 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:44.137 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:44.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:44.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:44.186 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:44.243 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:44.295 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:44.305 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:44.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:44.378 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:44.384 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:58:44.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:44.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:44.492 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:44.509 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:58:44.529 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:44.559 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:44.654 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:44.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:44.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:44.760 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:44.887 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:44.966 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 6 16 17323 17307 180.263 130.562 0.00676622 0.00554011 2023-10-09T05:58:45.316 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:45 smithi156 ceph-mon[195101]: pgmap v655: 109 pgs: 109 active+clean; 409 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 41 MiB/s wr, 635 op/s 2023-10-09T05:58:45.316 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:45 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/157243021' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:45.316 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:45 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:45.334 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T05:58:45.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:45 smithi064 ceph-mon[212750]: pgmap v655: 109 pgs: 109 active+clean; 409 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 41 MiB/s wr, 635 op/s 2023-10-09T05:58:45.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:45 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/157243021' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:45.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:45 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:45.720 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:58:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:58:45] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.43.0" 2023-10-09T05:58:45.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:45 smithi064 ceph-mon[220044]: pgmap v655: 109 pgs: 109 active+clean; 409 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 41 MiB/s wr, 635 op/s 2023-10-09T05:58:45.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:45 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/157243021' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:45.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:45 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:58:45.966 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 7 16 19766 19750 176.322 152.688 0.00605094 0.00566284 2023-10-09T05:58:46.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:46 smithi156 ceph-mon[195101]: 1.9 scrub starts 2023-10-09T05:58:46.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:46 smithi156 ceph-mon[195101]: 1.9 scrub ok 2023-10-09T05:58:46.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:46 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:58:46.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:46 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T05:58:46.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:46 smithi156 ceph-mon[195101]: failed to encode map e290 with expected crc 2023-10-09T05:58:46.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:46 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:58:46.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:46 smithi156 ceph-mon[195101]: osdmap e290: 8 total, 8 up, 7 in 2023-10-09T05:58:46.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:46 smithi156 ceph-mon[195101]: failed to encode map e290 with expected crc 2023-10-09T05:58:46.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:46 smithi064 ceph-mon[212750]: 1.9 scrub starts 2023-10-09T05:58:46.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:46 smithi064 ceph-mon[212750]: 1.9 scrub ok 2023-10-09T05:58:46.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:46 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:58:46.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:46 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T05:58:46.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:46 smithi064 ceph-mon[212750]: failed to encode map e290 with expected crc 2023-10-09T05:58:46.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:46 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:58:46.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:46 smithi064 ceph-mon[212750]: osdmap e290: 8 total, 8 up, 7 in 2023-10-09T05:58:46.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:46 smithi064 ceph-mon[212750]: failed to encode map e290 with expected crc 2023-10-09T05:58:46.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:46 smithi064 ceph-mon[220044]: 1.9 scrub starts 2023-10-09T05:58:46.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:46 smithi064 ceph-mon[220044]: 1.9 scrub ok 2023-10-09T05:58:46.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:46 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:58:46.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:46 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T05:58:46.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:46 smithi064 ceph-mon[220044]: failed to encode map e290 with expected crc 2023-10-09T05:58:46.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:46 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:58:46.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:46 smithi064 ceph-mon[220044]: osdmap e290: 8 total, 8 up, 7 in 2023-10-09T05:58:46.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:46 smithi064 ceph-mon[220044]: failed to encode map e290 with expected crc 2023-10-09T05:58:46.966 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 8 16 21916 21900 171.062 134.375 0.00731776 0.00583967 2023-10-09T05:58:47.031 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:47.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:47.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:47.140 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:47.199 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:47.379 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:47.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:47.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:47.483 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:47.533 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:47.621 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:47.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:47.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:47.726 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:47.767 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:47.837 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:47.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:47.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:47.940 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:47.967 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 9 16 23785 23769 165.034 116.812 0.00622043 0.00605306 2023-10-09T05:58:47.978 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:48.011 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:48.137 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:48.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:48.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:48.241 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:48.310 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:48.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:48 smithi064 ceph-mon[220044]: pgmap v657: 109 pgs: 109 active+clean; 966 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 96 MiB/s wr, 1.52k op/s 2023-10-09T05:58:48.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:48 smithi064 ceph-mon[220044]: 1.12 scrub starts 2023-10-09T05:58:48.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:48 smithi064 ceph-mon[220044]: 1.12 scrub ok 2023-10-09T05:58:48.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:48 smithi064 ceph-mon[212750]: pgmap v657: 109 pgs: 109 active+clean; 966 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 96 MiB/s wr, 1.52k op/s 2023-10-09T05:58:48.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:48 smithi064 ceph-mon[212750]: 1.12 scrub starts 2023-10-09T05:58:48.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:48 smithi064 ceph-mon[212750]: 1.12 scrub ok 2023-10-09T05:58:48.516 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:48 smithi156 ceph-mon[195101]: pgmap v657: 109 pgs: 109 active+clean; 966 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 96 MiB/s wr, 1.52k op/s 2023-10-09T05:58:48.516 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:48 smithi156 ceph-mon[195101]: 1.12 scrub starts 2023-10-09T05:58:48.516 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:48 smithi156 ceph-mon[195101]: 1.12 scrub ok 2023-10-09T05:58:48.572 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:48.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:48.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:48.685 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:48.756 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:48.826 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T05:58:48.826 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:58:48.967 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 10 16 25604 25588 159.898 113.688 0.00468512 0.00624727 2023-10-09T05:58:48.980 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:48.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:48.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:49.033 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:58:49.050 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T05:58:49.090 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:49.142 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:49.251 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:49.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:49.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:49.359 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:49.600 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:49.652 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:49.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:49.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:49.758 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:49.817 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:49.880 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:58:49.967 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 11 16 27487 27471 156.06 117.688 0.00363109 0.00640219 2023-10-09T05:58:50.080 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T05:58:50.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:50 smithi156 ceph-mon[195101]: pgmap v658: 109 pgs: 109 active+clean; 1.2 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 123 MiB/s wr, 1.96k op/s 2023-10-09T05:58:50.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:50 smithi156 ceph-mon[195101]: from='client.35334 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:58:50.412 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:50.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:50.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:50.519 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:50.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:50 smithi064 ceph-mon[220044]: pgmap v658: 109 pgs: 109 active+clean; 1.2 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 123 MiB/s wr, 1.96k op/s 2023-10-09T05:58:50.548 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:50 smithi064 ceph-mon[220044]: from='client.35334 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:58:50.548 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:50 smithi064 ceph-mon[212750]: pgmap v658: 109 pgs: 109 active+clean; 1.2 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 123 MiB/s wr, 1.96k op/s 2023-10-09T05:58:50.548 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:50 smithi064 ceph-mon[212750]: from='client.35334 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:58:50.567 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:50.620 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:50.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:50.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:50.652 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:58:50.726 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:58:50.752 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:50.795 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:58:50.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:50.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:58:50.898 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:50.932 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:50.967 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 12 16 28818 28802 149.987 83.1875 0.00501847 0.00665229 2023-10-09T05:58:51.252 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:51.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:51.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:51.364 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:51.397 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":658,"stamp":"2023-10-09T05:58:48.629299+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1291670041,"num_objects":19723,"num_object_clones":0,"num_object_copies":59169,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19723,"num_whiteouts":0,"num_read":66,"num_read_kb":124,"num_write":19793,"num_write_kb":1261095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":252307,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":23135,"ondisk_log_size":23135,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":340,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5898460,"kb_used_data":3987492,"kb_used_omap":118,"kb_used_meta":1910729,"kb_avail":650317604,"statfs":{"total":671965249536,"available":665925226496,"internally_reserved":0,"allocated":4083191808,"data_stored":4074156408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":121648,"internal_metadata":1956586704},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1291008270,"num_objects":19698,"num_object_clones":0,"num_object_copies":59094,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19698,"num_whiteouts":0,"num_read":-260,"num_read_kb":-486,"num_write":19572,"num_write_kb":1259454,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-9,"num_bytes_recovered":-2249943,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004396"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":334,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.274542+0000","last_change":"2023-10-09T05:55:59.341215+0000","last_active":"2023-10-09T05:58:33.274542+0000","last_peered":"2023-10-09T05:58:33.274542+0000","last_clean":"2023-10-09T05:58:33.274542+0000","last_became_active":"2023-10-09T05:55:59.340856+0000","last_became_peered":"2023-10-09T05:55:59.340856+0000","last_unstale":"2023-10-09T05:58:33.274542+0000","last_undegraded":"2023-10-09T05:58:33.274542+0000","last_fullsized":"2023-10-09T05:58:33.274542+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:33.686454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":51,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.248535+0000","last_change":"2023-10-09T05:57:23.474348+0000","last_active":"2023-10-09T05:58:33.248535+0000","last_peered":"2023-10-09T05:58:33.248535+0000","last_clean":"2023-10-09T05:58:33.248535+0000","last_became_active":"2023-10-09T05:57:23.474010+0000","last_became_peered":"2023-10-09T05:57:23.474010+0000","last_unstale":"2023-10-09T05:58:33.248535+0000","last_undegraded":"2023-10-09T05:58:33.248535+0000","last_fullsized":"2023-10-09T05:58:33.248535+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:45.039777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":168,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.249154+0000","last_change":"2023-10-09T05:55:59.304893+0000","last_active":"2023-10-09T05:58:33.249154+0000","last_peered":"2023-10-09T05:58:33.249154+0000","last_clean":"2023-10-09T05:58:33.249154+0000","last_became_active":"2023-10-09T05:55:59.304449+0000","last_became_peered":"2023-10-09T05:55:59.304449+0000","last_unstale":"2023-10-09T05:58:33.249154+0000","last_undegraded":"2023-10-09T05:58:33.249154+0000","last_fullsized":"2023-10-09T05:58:33.249154+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:44:39.453982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":309,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.247137+0000","last_change":"2023-10-09T05:57:58.834014+0000","last_active":"2023-10-09T05:58:33.247137+0000","last_peered":"2023-10-09T05:58:33.247137+0000","last_clean":"2023-10-09T05:58:33.247137+0000","last_became_active":"2023-10-09T05:57:58.833718+0000","last_became_peered":"2023-10-09T05:57:58.833718+0000","last_unstale":"2023-10-09T05:58:33.247137+0000","last_undegraded":"2023-10-09T05:58:33.247137+0000","last_fullsized":"2023-10-09T05:58:33.247137+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:23.752353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":51,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.247866+0000","last_change":"2023-10-09T05:57:23.476832+0000","last_active":"2023-10-09T05:58:33.247866+0000","last_peered":"2023-10-09T05:58:33.247866+0000","last_clean":"2023-10-09T05:58:33.247866+0000","last_became_active":"2023-10-09T05:57:23.475216+0000","last_became_peered":"2023-10-09T05:57:23.475216+0000","last_unstale":"2023-10-09T05:58:33.247866+0000","last_undegraded":"2023-10-09T05:58:33.247866+0000","last_fullsized":"2023-10-09T05:58:33.247866+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:48.279660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":316,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.829475+0000","last_change":"2023-10-09T05:55:59.335464+0000","last_active":"2023-10-09T05:58:36.829475+0000","last_peered":"2023-10-09T05:58:36.829475+0000","last_clean":"2023-10-09T05:58:36.829475+0000","last_became_active":"2023-10-09T05:55:59.335259+0000","last_became_peered":"2023-10-09T05:55:59.335259+0000","last_unstale":"2023-10-09T05:58:36.829475+0000","last_undegraded":"2023-10-09T05:58:36.829475+0000","last_fullsized":"2023-10-09T05:58:36.829475+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:28.536682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":244,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.247179+0000","last_change":"2023-10-09T05:57:23.477980+0000","last_active":"2023-10-09T05:58:33.247179+0000","last_peered":"2023-10-09T05:58:33.247179+0000","last_clean":"2023-10-09T05:58:33.247179+0000","last_became_active":"2023-10-09T05:57:23.477654+0000","last_became_peered":"2023-10-09T05:57:23.477654+0000","last_unstale":"2023-10-09T05:58:33.247179+0000","last_undegraded":"2023-10-09T05:58:33.247179+0000","last_fullsized":"2023-10-09T05:58:33.247179+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:17.524683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":51,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.247939+0000","last_change":"2023-10-09T05:57:23.473644+0000","last_active":"2023-10-09T05:58:33.247939+0000","last_peered":"2023-10-09T05:58:33.247939+0000","last_clean":"2023-10-09T05:58:33.247939+0000","last_became_active":"2023-10-09T05:57:23.473391+0000","last_became_peered":"2023-10-09T05:57:23.473391+0000","last_unstale":"2023-10-09T05:58:33.247939+0000","last_undegraded":"2023-10-09T05:58:33.247939+0000","last_fullsized":"2023-10-09T05:58:33.247939+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:17.138034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":554,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.913116+0000","last_change":"2023-10-09T05:58:36.913116+0000","last_active":"2023-10-09T05:58:36.913116+0000","last_peered":"2023-10-09T05:58:36.913116+0000","last_clean":"2023-10-09T05:58:36.913116+0000","last_became_active":"2023-10-09T05:58:21.329980+0000","last_became_peered":"2023-10-09T05:58:21.329980+0000","last_unstale":"2023-10-09T05:58:36.913116+0000","last_undegraded":"2023-10-09T05:58:36.913116+0000","last_fullsized":"2023-10-09T05:58:36.913116+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.913004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.913004+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:32:18.561414+0000","scrub_duration":0.0017791980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":329,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.832759+0000","last_change":"2023-10-09T05:57:23.478483+0000","last_active":"2023-10-09T05:58:36.832759+0000","last_peered":"2023-10-09T05:58:36.832759+0000","last_clean":"2023-10-09T05:58:36.832759+0000","last_became_active":"2023-10-09T05:57:23.478122+0000","last_became_peered":"2023-10-09T05:57:23.478122+0000","last_unstale":"2023-10-09T05:58:36.832759+0000","last_undegraded":"2023-10-09T05:58:36.832759+0000","last_fullsized":"2023-10-09T05:58:36.832759+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:37.768717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":233,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.310884+0000","last_change":"2023-10-09T05:53:59.185702+0000","last_active":"2023-10-09T05:58:36.310884+0000","last_peered":"2023-10-09T05:58:36.310884+0000","last_clean":"2023-10-09T05:58:36.310884+0000","last_became_active":"2023-10-09T05:53:59.185373+0000","last_became_peered":"2023-10-09T05:53:59.185373+0000","last_unstale":"2023-10-09T05:58:36.310884+0000","last_undegraded":"2023-10-09T05:58:36.310884+0000","last_fullsized":"2023-10-09T05:58:36.310884+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:50.756967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":130,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.832364+0000","last_change":"2023-10-09T05:53:59.219168+0000","last_active":"2023-10-09T05:58:36.832364+0000","last_peered":"2023-10-09T05:58:36.832364+0000","last_clean":"2023-10-09T05:58:36.832364+0000","last_became_active":"2023-10-09T05:53:59.200886+0000","last_became_peered":"2023-10-09T05:53:59.200886+0000","last_unstale":"2023-10-09T05:58:36.832364+0000","last_undegraded":"2023-10-09T05:58:36.832364+0000","last_fullsized":"2023-10-09T05:58:36.832364+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:45.515036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":184,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.828884+0000","last_change":"2023-10-09T05:55:59.305100+0000","last_active":"2023-10-09T05:58:36.828884+0000","last_peered":"2023-10-09T05:58:36.828884+0000","last_clean":"2023-10-09T05:58:36.828884+0000","last_became_active":"2023-10-09T05:55:59.304758+0000","last_became_peered":"2023-10-09T05:55:59.304758+0000","last_unstale":"2023-10-09T05:58:36.828884+0000","last_undegraded":"2023-10-09T05:58:36.828884+0000","last_fullsized":"2023-10-09T05:58:36.828884+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:54:32.510945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":294,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.274762+0000","last_change":"2023-10-09T05:53:59.219141+0000","last_active":"2023-10-09T05:58:33.274762+0000","last_peered":"2023-10-09T05:58:33.274762+0000","last_clean":"2023-10-09T05:58:33.274762+0000","last_became_active":"2023-10-09T05:53:59.197318+0000","last_became_peered":"2023-10-09T05:53:59.197318+0000","last_unstale":"2023-10-09T05:58:33.274762+0000","last_undegraded":"2023-10-09T05:58:33.274762+0000","last_fullsized":"2023-10-09T05:58:33.274762+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:39.910794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":298,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.274714+0000","last_change":"2023-10-09T05:55:59.335824+0000","last_active":"2023-10-09T05:58:33.274714+0000","last_peered":"2023-10-09T05:58:33.274714+0000","last_clean":"2023-10-09T05:58:33.274714+0000","last_became_active":"2023-10-09T05:55:59.335478+0000","last_became_peered":"2023-10-09T05:55:59.335478+0000","last_unstale":"2023-10-09T05:58:33.274714+0000","last_undegraded":"2023-10-09T05:58:33.274714+0000","last_fullsized":"2023-10-09T05:58:33.274714+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:26.014501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":508,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:38.230098+0000","last_change":"2023-10-09T05:58:38.230201+0000","last_active":"2023-10-09T05:58:38.230098+0000","last_peered":"2023-10-09T05:58:38.230098+0000","last_clean":"2023-10-09T05:58:38.230098+0000","last_became_active":"2023-10-09T05:58:21.332729+0000","last_became_peered":"2023-10-09T05:58:21.332729+0000","last_unstale":"2023-10-09T05:58:38.230098+0000","last_undegraded":"2023-10-09T05:58:38.230098+0000","last_fullsized":"2023-10-09T05:58:38.230098+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:38.230079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:38.230079+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:33.800036+0000","scrub_duration":0.001618488,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":132,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.831413+0000","last_change":"2023-10-09T05:53:59.233898+0000","last_active":"2023-10-09T05:58:36.831413+0000","last_peered":"2023-10-09T05:58:36.831413+0000","last_clean":"2023-10-09T05:58:36.831413+0000","last_became_active":"2023-10-09T05:53:59.204175+0000","last_became_peered":"2023-10-09T05:53:59.204175+0000","last_unstale":"2023-10-09T05:58:36.831413+0000","last_undegraded":"2023-10-09T05:58:36.831413+0000","last_fullsized":"2023-10-09T05:58:36.831413+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:03.139318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":319,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.310877+0000","last_change":"2023-10-09T05:55:59.297087+0000","last_active":"2023-10-09T05:58:36.310877+0000","last_peered":"2023-10-09T05:58:36.310877+0000","last_clean":"2023-10-09T05:58:36.310877+0000","last_became_active":"2023-10-09T05:55:59.296768+0000","last_became_peered":"2023-10-09T05:55:59.296768+0000","last_unstale":"2023-10-09T05:58:36.310877+0000","last_undegraded":"2023-10-09T05:58:36.310877+0000","last_fullsized":"2023-10-09T05:58:36.310877+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:35.067705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":703,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.830903+0000","last_change":"2023-10-09T05:58:21.332888+0000","last_active":"2023-10-09T05:58:36.830903+0000","last_peered":"2023-10-09T05:58:36.830903+0000","last_clean":"2023-10-09T05:58:36.830903+0000","last_became_active":"2023-10-09T05:58:21.332533+0000","last_became_peered":"2023-10-09T05:58:21.332533+0000","last_unstale":"2023-10-09T05:58:36.830903+0000","last_undegraded":"2023-10-09T05:58:36.830903+0000","last_fullsized":"2023-10-09T05:58:36.830903+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:51:12.346243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:51:12.346243+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":290,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.247542+0000","last_change":"2023-10-09T05:57:23.473445+0000","last_active":"2023-10-09T05:58:33.247542+0000","last_peered":"2023-10-09T05:58:33.247542+0000","last_clean":"2023-10-09T05:58:33.247542+0000","last_became_active":"2023-10-09T05:57:23.473103+0000","last_became_peered":"2023-10-09T05:57:23.473103+0000","last_unstale":"2023-10-09T05:58:33.247542+0000","last_undegraded":"2023-10-09T05:58:33.247542+0000","last_fullsized":"2023-10-09T05:58:33.247542+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:39:53.825734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":213,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.832260+0000","last_change":"2023-10-09T05:56:49.237784+0000","last_active":"2023-10-09T05:58:36.832260+0000","last_peered":"2023-10-09T05:58:36.832260+0000","last_clean":"2023-10-09T05:58:36.832260+0000","last_became_active":"2023-10-09T05:56:49.237417+0000","last_became_peered":"2023-10-09T05:56:49.237417+0000","last_unstale":"2023-10-09T05:58:36.832260+0000","last_undegraded":"2023-10-09T05:58:36.832260+0000","last_fullsized":"2023-10-09T05:58:36.832260+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:33.129268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":539,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:37.295110+0000","last_change":"2023-10-09T05:58:37.295185+0000","last_active":"2023-10-09T05:58:37.295110+0000","last_peered":"2023-10-09T05:58:37.295110+0000","last_clean":"2023-10-09T05:58:37.295110+0000","last_became_active":"2023-10-09T05:58:21.327911+0000","last_became_peered":"2023-10-09T05:58:21.327911+0000","last_unstale":"2023-10-09T05:58:37.295110+0000","last_undegraded":"2023-10-09T05:58:37.295110+0000","last_fullsized":"2023-10-09T05:58:37.295110+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:37.295094+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:37.295094+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:10.882014+0000","scrub_duration":0.0017309510000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":126,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.831924+0000","last_change":"2023-10-09T05:53:59.206887+0000","last_active":"2023-10-09T05:58:36.831924+0000","last_peered":"2023-10-09T05:58:36.831924+0000","last_clean":"2023-10-09T05:58:36.831924+0000","last_became_active":"2023-10-09T05:53:59.206608+0000","last_became_peered":"2023-10-09T05:53:59.206608+0000","last_unstale":"2023-10-09T05:58:36.831924+0000","last_undegraded":"2023-10-09T05:58:36.831924+0000","last_fullsized":"2023-10-09T05:58:36.831924+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:48.975792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":271,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.247301+0000","last_change":"2023-10-09T05:57:58.828837+0000","last_active":"2023-10-09T05:58:33.247301+0000","last_peered":"2023-10-09T05:58:33.247301+0000","last_clean":"2023-10-09T05:58:33.247301+0000","last_became_active":"2023-10-09T05:57:58.828503+0000","last_became_peered":"2023-10-09T05:57:58.828503+0000","last_unstale":"2023-10-09T05:58:33.247301+0000","last_undegraded":"2023-10-09T05:58:33.247301+0000","last_fullsized":"2023-10-09T05:58:33.247301+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:08.332644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":590,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T05:58:46.725523+0000","last_change":"2023-10-09T05:58:46.725484+0000","last_active":"2023-10-09T05:58:46.725523+0000","last_peered":"2023-10-09T05:58:46.725523+0000","last_clean":"2023-10-09T05:58:46.725523+0000","last_became_active":"2023-10-09T05:58:21.332713+0000","last_became_peered":"2023-10-09T05:58:21.332713+0000","last_unstale":"2023-10-09T05:58:46.725523+0000","last_undegraded":"2023-10-09T05:58:46.725523+0000","last_fullsized":"2023-10-09T05:58:46.725523+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:46.725374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:46.725374+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:34.070739+0000","scrub_duration":0.002074771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":275,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.828200+0000","last_change":"2023-10-09T05:51:57.091639+0000","last_active":"2023-10-09T05:58:36.828200+0000","last_peered":"2023-10-09T05:58:36.828200+0000","last_clean":"2023-10-09T05:58:36.828200+0000","last_became_active":"2023-10-09T05:51:57.091284+0000","last_became_peered":"2023-10-09T05:51:57.091284+0000","last_unstale":"2023-10-09T05:58:36.828200+0000","last_undegraded":"2023-10-09T05:58:36.828200+0000","last_fullsized":"2023-10-09T05:58:36.828200+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:10.553453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":60,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.249709+0000","last_change":"2023-10-09T05:57:58.833939+0000","last_active":"2023-10-09T05:58:33.249709+0000","last_peered":"2023-10-09T05:58:33.249709+0000","last_clean":"2023-10-09T05:58:33.249709+0000","last_became_active":"2023-10-09T05:57:58.833567+0000","last_became_peered":"2023-10-09T05:57:58.833567+0000","last_unstale":"2023-10-09T05:58:33.249709+0000","last_undegraded":"2023-10-09T05:58:33.249709+0000","last_fullsized":"2023-10-09T05:58:33.249709+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:58.347572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"290'1530","reported_seq":1541,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T05:58:48.383891+0000","last_change":"2023-10-09T05:58:27.405856+0000","last_active":"2023-10-09T05:58:48.383891+0000","last_peered":"2023-10-09T05:58:48.383891+0000","last_clean":"2023-10-09T05:58:48.383891+0000","last_became_active":"2023-10-09T05:58:27.405560+0000","last_became_peered":"2023-10-09T05:58:27.405560+0000","last_unstale":"2023-10-09T05:58:48.383891+0000","last_undegraded":"2023-10-09T05:58:48.383891+0000","last_fullsized":"2023-10-09T05:58:48.383891+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":1530,"log_dups_size":0,"ondisk_log_size":1530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:24.937739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100270080,"num_objects":1530,"num_object_clones":0,"num_object_copies":4590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1530,"num_write_kb":97920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":599,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.832178+0000","last_change":"2023-10-09T05:58:21.333719+0000","last_active":"2023-10-09T05:58:36.832178+0000","last_peered":"2023-10-09T05:58:36.832178+0000","last_clean":"2023-10-09T05:58:36.832178+0000","last_became_active":"2023-10-09T05:58:21.333365+0000","last_became_peered":"2023-10-09T05:58:21.333365+0000","last_unstale":"2023-10-09T05:58:36.832178+0000","last_undegraded":"2023-10-09T05:58:36.832178+0000","last_fullsized":"2023-10-09T05:58:36.832178+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":130,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.832711+0000","last_change":"2023-10-09T05:53:59.222136+0000","last_active":"2023-10-09T05:58:36.832711+0000","last_peered":"2023-10-09T05:58:36.832711+0000","last_clean":"2023-10-09T05:58:36.832711+0000","last_became_active":"2023-10-09T05:53:59.201853+0000","last_became_peered":"2023-10-09T05:53:59.201853+0000","last_unstale":"2023-10-09T05:58:36.832711+0000","last_undegraded":"2023-10-09T05:58:36.832711+0000","last_fullsized":"2023-10-09T05:58:36.832711+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:26.662031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":196,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.830688+0000","last_change":"2023-10-09T05:56:49.234185+0000","last_active":"2023-10-09T05:58:36.830688+0000","last_peered":"2023-10-09T05:58:36.830688+0000","last_clean":"2023-10-09T05:58:36.830688+0000","last_became_active":"2023-10-09T05:56:49.233884+0000","last_became_peered":"2023-10-09T05:56:49.233884+0000","last_unstale":"2023-10-09T05:58:36.830688+0000","last_undegraded":"2023-10-09T05:58:36.830688+0000","last_fullsized":"2023-10-09T05:58:36.830688+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:12.142436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.a","version":"289'1110","reported_seq":1123,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:44.727398+0000","last_change":"2023-10-09T05:58:27.406918+0000","last_active":"2023-10-09T05:58:44.727398+0000","last_peered":"2023-10-09T05:58:44.727398+0000","last_clean":"2023-10-09T05:58:44.727398+0000","last_became_active":"2023-10-09T05:58:27.406600+0000","last_became_peered":"2023-10-09T05:58:27.406600+0000","last_unstale":"2023-10-09T05:58:44.727398+0000","last_undegraded":"2023-10-09T05:58:44.727398+0000","last_fullsized":"2023-10-09T05:58:44.727398+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":1110,"log_dups_size":0,"ondisk_log_size":1110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:00.484611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72744960,"num_objects":1110,"num_object_clones":0,"num_object_copies":3330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1110,"num_write_kb":71040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":572,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:39.265062+0000","last_change":"2023-10-09T05:58:39.265193+0000","last_active":"2023-10-09T05:58:39.265062+0000","last_peered":"2023-10-09T05:58:39.265062+0000","last_clean":"2023-10-09T05:58:39.265062+0000","last_became_active":"2023-10-09T05:58:21.327449+0000","last_became_peered":"2023-10-09T05:58:21.327449+0000","last_unstale":"2023-10-09T05:58:39.265062+0000","last_undegraded":"2023-10-09T05:58:39.265062+0000","last_fullsized":"2023-10-09T05:58:39.265062+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:39.265029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:39.265029+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:30:46.849393+0000","scrub_duration":0.011791197999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":172,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.832847+0000","last_change":"2023-10-09T05:55:59.379174+0000","last_active":"2023-10-09T05:58:36.832847+0000","last_peered":"2023-10-09T05:58:36.832847+0000","last_clean":"2023-10-09T05:58:36.832847+0000","last_became_active":"2023-10-09T05:55:59.341932+0000","last_became_peered":"2023-10-09T05:55:59.341932+0000","last_unstale":"2023-10-09T05:58:36.832847+0000","last_undegraded":"2023-10-09T05:58:36.832847+0000","last_fullsized":"2023-10-09T05:58:36.832847+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.636171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":335,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.274574+0000","last_change":"2023-10-09T05:57:23.475146+0000","last_active":"2023-10-09T05:58:33.274574+0000","last_peered":"2023-10-09T05:58:33.274574+0000","last_clean":"2023-10-09T05:58:33.274574+0000","last_became_active":"2023-10-09T05:57:23.474776+0000","last_became_peered":"2023-10-09T05:57:23.474776+0000","last_unstale":"2023-10-09T05:58:33.274574+0000","last_undegraded":"2023-10-09T05:58:33.274574+0000","last_fullsized":"2023-10-09T05:58:33.274574+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:47.775743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"289'1017","reported_seq":1030,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:44.745451+0000","last_change":"2023-10-09T05:58:27.377562+0000","last_active":"2023-10-09T05:58:44.745451+0000","last_peered":"2023-10-09T05:58:44.745451+0000","last_clean":"2023-10-09T05:58:44.745451+0000","last_became_active":"2023-10-09T05:58:27.377253+0000","last_became_peered":"2023-10-09T05:58:27.377253+0000","last_unstale":"2023-10-09T05:58:44.745451+0000","last_undegraded":"2023-10-09T05:58:44.745451+0000","last_fullsized":"2023-10-09T05:58:44.745451+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":1017,"log_dups_size":0,"ondisk_log_size":1017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:34.871962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66650112,"num_objects":1017,"num_object_clones":0,"num_object_copies":3051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1017,"num_write_kb":65088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":712,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:41.753671+0000","last_change":"2023-10-09T05:58:41.753606+0000","last_active":"2023-10-09T05:58:41.753671+0000","last_peered":"2023-10-09T05:58:41.753671+0000","last_clean":"2023-10-09T05:58:41.753671+0000","last_became_active":"2023-10-09T05:58:21.332726+0000","last_became_peered":"2023-10-09T05:58:21.332726+0000","last_unstale":"2023-10-09T05:58:41.753671+0000","last_undegraded":"2023-10-09T05:58:41.753671+0000","last_fullsized":"2023-10-09T05:58:41.753671+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:41.753506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:41.753506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:15.947090+0000","scrub_duration":0.0022121929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":128,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.832421+0000","last_change":"2023-10-09T05:53:59.212161+0000","last_active":"2023-10-09T05:58:36.832421+0000","last_peered":"2023-10-09T05:58:36.832421+0000","last_clean":"2023-10-09T05:58:36.832421+0000","last_became_active":"2023-10-09T05:53:59.193814+0000","last_became_peered":"2023-10-09T05:53:59.193814+0000","last_unstale":"2023-10-09T05:58:36.832421+0000","last_undegraded":"2023-10-09T05:58:36.832421+0000","last_fullsized":"2023-10-09T05:58:36.832421+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:10.502408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":329,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.828779+0000","last_change":"2023-10-09T05:55:59.303352+0000","last_active":"2023-10-09T05:58:36.828779+0000","last_peered":"2023-10-09T05:58:36.828779+0000","last_clean":"2023-10-09T05:58:36.828779+0000","last_became_active":"2023-10-09T05:55:59.302923+0000","last_became_peered":"2023-10-09T05:55:59.302923+0000","last_unstale":"2023-10-09T05:58:36.828779+0000","last_undegraded":"2023-10-09T05:58:36.828779+0000","last_fullsized":"2023-10-09T05:58:36.828779+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:53.013376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"290'1428","reported_seq":1442,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T05:58:47.308915+0000","last_change":"2023-10-09T05:58:27.376696+0000","last_active":"2023-10-09T05:58:47.308915+0000","last_peered":"2023-10-09T05:58:47.308915+0000","last_clean":"2023-10-09T05:58:47.308915+0000","last_became_active":"2023-10-09T05:58:27.376391+0000","last_became_peered":"2023-10-09T05:58:27.376391+0000","last_unstale":"2023-10-09T05:58:47.308915+0000","last_undegraded":"2023-10-09T05:58:47.308915+0000","last_fullsized":"2023-10-09T05:58:47.308915+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":1428,"log_dups_size":0,"ondisk_log_size":1428,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:36.863116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93585408,"num_objects":1428,"num_object_clones":0,"num_object_copies":4284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1428,"num_write_kb":91392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":435,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:37.499664+0000","last_change":"2023-10-09T05:58:37.499614+0000","last_active":"2023-10-09T05:58:37.499664+0000","last_peered":"2023-10-09T05:58:37.499664+0000","last_clean":"2023-10-09T05:58:37.499664+0000","last_became_active":"2023-10-09T05:58:21.330754+0000","last_became_peered":"2023-10-09T05:58:21.330754+0000","last_unstale":"2023-10-09T05:58:37.499664+0000","last_undegraded":"2023-10-09T05:58:37.499664+0000","last_fullsized":"2023-10-09T05:58:37.499664+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:37.499499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:37.499499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:01.909666+0000","scrub_duration":0.0018066499999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":51,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.249188+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T05:58:33.249188+0000","last_peered":"2023-10-09T05:58:33.249188+0000","last_clean":"2023-10-09T05:58:33.249188+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T05:58:33.249188+0000","last_undegraded":"2023-10-09T05:58:33.249188+0000","last_fullsized":"2023-10-09T05:58:33.249188+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":300,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.311087+0000","last_change":"2023-10-09T05:57:23.473860+0000","last_active":"2023-10-09T05:58:36.311087+0000","last_peered":"2023-10-09T05:58:36.311087+0000","last_clean":"2023-10-09T05:58:36.311087+0000","last_became_active":"2023-10-09T05:57:23.473101+0000","last_became_peered":"2023-10-09T05:57:23.473101+0000","last_unstale":"2023-10-09T05:58:36.311087+0000","last_undegraded":"2023-10-09T05:58:36.311087+0000","last_fullsized":"2023-10-09T05:58:36.311087+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:05.682865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"289'1011","reported_seq":1024,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:44.727715+0000","last_change":"2023-10-09T05:58:27.376809+0000","last_active":"2023-10-09T05:58:44.727715+0000","last_peered":"2023-10-09T05:58:44.727715+0000","last_clean":"2023-10-09T05:58:44.727715+0000","last_became_active":"2023-10-09T05:58:27.376476+0000","last_became_peered":"2023-10-09T05:58:27.376476+0000","last_unstale":"2023-10-09T05:58:44.727715+0000","last_undegraded":"2023-10-09T05:58:44.727715+0000","last_fullsized":"2023-10-09T05:58:44.727715+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":1011,"log_dups_size":0,"ondisk_log_size":1011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:53:15.515971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66256896,"num_objects":1011,"num_object_clones":0,"num_object_copies":3033,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1011,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1011,"num_write_kb":64704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":713,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:44.765275+0000","last_change":"2023-10-09T05:58:44.765224+0000","last_active":"2023-10-09T05:58:44.765275+0000","last_peered":"2023-10-09T05:58:44.765275+0000","last_clean":"2023-10-09T05:58:44.765275+0000","last_became_active":"2023-10-09T05:57:13.233849+0000","last_became_peered":"2023-10-09T05:57:13.233849+0000","last_unstale":"2023-10-09T05:58:44.765275+0000","last_undegraded":"2023-10-09T05:58:44.765275+0000","last_fullsized":"2023-10-09T05:58:44.765275+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:44.765164+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:44.765164+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:24.336340+0000","scrub_duration":0.0050417350000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":55,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.248039+0000","last_change":"2023-10-09T05:57:23.504733+0000","last_active":"2023-10-09T05:58:33.248039+0000","last_peered":"2023-10-09T05:58:33.248039+0000","last_clean":"2023-10-09T05:58:33.248039+0000","last_became_active":"2023-10-09T05:57:23.468709+0000","last_became_peered":"2023-10-09T05:57:23.468709+0000","last_unstale":"2023-10-09T05:58:33.248039+0000","last_undegraded":"2023-10-09T05:58:33.248039+0000","last_fullsized":"2023-10-09T05:58:33.248039+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:00.459980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":248,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.274877+0000","last_change":"2023-10-09T05:49:21.430742+0000","last_active":"2023-10-09T05:58:33.274877+0000","last_peered":"2023-10-09T05:58:33.274877+0000","last_clean":"2023-10-09T05:58:33.274877+0000","last_became_active":"2023-10-09T05:49:21.430233+0000","last_became_peered":"2023-10-09T05:49:21.430233+0000","last_unstale":"2023-10-09T05:58:33.274877+0000","last_undegraded":"2023-10-09T05:58:33.274877+0000","last_fullsized":"2023-10-09T05:58:33.274877+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:10.541302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"290'1444","reported_seq":1456,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T05:58:47.472601+0000","last_change":"2023-10-09T05:58:27.405765+0000","last_active":"2023-10-09T05:58:47.472601+0000","last_peered":"2023-10-09T05:58:47.472601+0000","last_clean":"2023-10-09T05:58:47.472601+0000","last_became_active":"2023-10-09T05:58:27.405447+0000","last_became_peered":"2023-10-09T05:58:27.405447+0000","last_unstale":"2023-10-09T05:58:47.472601+0000","last_undegraded":"2023-10-09T05:58:47.472601+0000","last_fullsized":"2023-10-09T05:58:47.472601+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":1444,"log_dups_size":0,"ondisk_log_size":1444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:52.191564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94633984,"num_objects":1444,"num_object_clones":0,"num_object_copies":4332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1444,"num_write_kb":92416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":508,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.826795+0000","last_change":"2023-10-09T05:58:35.945941+0000","last_active":"2023-10-09T05:58:36.826795+0000","last_peered":"2023-10-09T05:58:36.826795+0000","last_clean":"2023-10-09T05:58:36.826795+0000","last_became_active":"2023-10-09T05:57:26.516674+0000","last_became_peered":"2023-10-09T05:57:26.516674+0000","last_unstale":"2023-10-09T05:58:36.826795+0000","last_undegraded":"2023-10-09T05:58:36.826795+0000","last_fullsized":"2023-10-09T05:58:36.826795+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:35.945829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:35.945829+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:43.121459+0000","scrub_duration":0.0021243680000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":300,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.828645+0000","last_change":"2023-10-09T05:55:59.320669+0000","last_active":"2023-10-09T05:58:36.828645+0000","last_peered":"2023-10-09T05:58:36.828645+0000","last_clean":"2023-10-09T05:58:36.828645+0000","last_became_active":"2023-10-09T05:55:59.320352+0000","last_became_peered":"2023-10-09T05:55:59.320352+0000","last_unstale":"2023-10-09T05:58:36.828645+0000","last_undegraded":"2023-10-09T05:58:36.828645+0000","last_fullsized":"2023-10-09T05:58:36.828645+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:09:14.227028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":335,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.275016+0000","last_change":"2023-10-09T05:57:10.362295+0000","last_active":"2023-10-09T05:58:33.275016+0000","last_peered":"2023-10-09T05:58:33.275016+0000","last_clean":"2023-10-09T05:58:33.275016+0000","last_became_active":"2023-10-09T05:57:10.361249+0000","last_became_peered":"2023-10-09T05:57:10.361249+0000","last_unstale":"2023-10-09T05:58:33.275016+0000","last_undegraded":"2023-10-09T05:58:33.275016+0000","last_fullsized":"2023-10-09T05:58:33.275016+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:22.088261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"290'1567","reported_seq":1580,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T05:58:48.389868+0000","last_change":"2023-10-09T05:58:27.378739+0000","last_active":"2023-10-09T05:58:48.389868+0000","last_peered":"2023-10-09T05:58:48.389868+0000","last_clean":"2023-10-09T05:58:48.389868+0000","last_became_active":"2023-10-09T05:58:27.378459+0000","last_became_peered":"2023-10-09T05:58:27.378459+0000","last_unstale":"2023-10-09T05:58:48.389868+0000","last_undegraded":"2023-10-09T05:58:48.389868+0000","last_fullsized":"2023-10-09T05:58:48.389868+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":1567,"log_dups_size":0,"ondisk_log_size":1567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:32.861544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102694912,"num_objects":1567,"num_object_clones":0,"num_object_copies":4701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1567,"num_write_kb":100288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":536,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:39.230409+0000","last_change":"2023-10-09T05:58:39.230541+0000","last_active":"2023-10-09T05:58:39.230409+0000","last_peered":"2023-10-09T05:58:39.230409+0000","last_clean":"2023-10-09T05:58:39.230409+0000","last_became_active":"2023-10-09T05:58:21.332677+0000","last_became_peered":"2023-10-09T05:58:21.332677+0000","last_unstale":"2023-10-09T05:58:39.230409+0000","last_undegraded":"2023-10-09T05:58:39.230409+0000","last_fullsized":"2023-10-09T05:58:39.230409+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:39.230347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:39.230347+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:14:06.774208+0000","scrub_duration":0.0041260619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":269,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.274412+0000","last_change":"2023-10-09T05:54:00.619165+0000","last_active":"2023-10-09T05:58:33.274412+0000","last_peered":"2023-10-09T05:58:33.274412+0000","last_clean":"2023-10-09T05:58:33.274412+0000","last_became_active":"2023-10-09T05:54:00.618888+0000","last_became_peered":"2023-10-09T05:54:00.618888+0000","last_unstale":"2023-10-09T05:58:33.274412+0000","last_undegraded":"2023-10-09T05:58:33.274412+0000","last_fullsized":"2023-10-09T05:58:33.274412+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:07.175996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":308,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.274942+0000","last_change":"2023-10-09T05:57:23.483158+0000","last_active":"2023-10-09T05:58:33.274942+0000","last_peered":"2023-10-09T05:58:33.274942+0000","last_clean":"2023-10-09T05:58:33.274942+0000","last_became_active":"2023-10-09T05:57:23.482701+0000","last_became_peered":"2023-10-09T05:57:23.482701+0000","last_unstale":"2023-10-09T05:58:33.274942+0000","last_undegraded":"2023-10-09T05:58:33.274942+0000","last_fullsized":"2023-10-09T05:58:33.274942+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:51:08.624310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"290'1555","reported_seq":1568,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T05:58:48.386058+0000","last_change":"2023-10-09T05:58:27.376357+0000","last_active":"2023-10-09T05:58:48.386058+0000","last_peered":"2023-10-09T05:58:48.386058+0000","last_clean":"2023-10-09T05:58:48.386058+0000","last_became_active":"2023-10-09T05:58:27.376017+0000","last_became_peered":"2023-10-09T05:58:27.376017+0000","last_unstale":"2023-10-09T05:58:48.386058+0000","last_undegraded":"2023-10-09T05:58:48.386058+0000","last_fullsized":"2023-10-09T05:58:48.386058+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":1555,"log_dups_size":0,"ondisk_log_size":1555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:30.067479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101908480,"num_objects":1555,"num_object_clones":0,"num_object_copies":4665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1555,"num_write_kb":99520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":487,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.826847+0000","last_change":"2023-10-09T05:58:34.952053+0000","last_active":"2023-10-09T05:58:36.826847+0000","last_peered":"2023-10-09T05:58:36.826847+0000","last_clean":"2023-10-09T05:58:36.826847+0000","last_became_active":"2023-10-09T05:56:49.229606+0000","last_became_peered":"2023-10-09T05:56:49.229606+0000","last_unstale":"2023-10-09T05:58:36.826847+0000","last_undegraded":"2023-10-09T05:58:36.826847+0000","last_fullsized":"2023-10-09T05:58:36.826847+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.951922+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.951922+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:20.811103+0000","scrub_duration":0.0023402980000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":51,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.247788+0000","last_change":"2023-10-09T05:57:23.469928+0000","last_active":"2023-10-09T05:58:33.247788+0000","last_peered":"2023-10-09T05:58:33.247788+0000","last_clean":"2023-10-09T05:58:33.247788+0000","last_became_active":"2023-10-09T05:57:23.469697+0000","last_became_peered":"2023-10-09T05:57:23.469697+0000","last_unstale":"2023-10-09T05:58:33.247788+0000","last_undegraded":"2023-10-09T05:58:33.247788+0000","last_fullsized":"2023-10-09T05:58:33.247788+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:21:07.179434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":168,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.249785+0000","last_change":"2023-10-09T05:55:59.409563+0000","last_active":"2023-10-09T05:58:33.249785+0000","last_peered":"2023-10-09T05:58:33.249785+0000","last_clean":"2023-10-09T05:58:33.249785+0000","last_became_active":"2023-10-09T05:55:59.409184+0000","last_became_peered":"2023-10-09T05:55:59.409184+0000","last_unstale":"2023-10-09T05:58:33.249785+0000","last_undegraded":"2023-10-09T05:58:33.249785+0000","last_fullsized":"2023-10-09T05:58:33.249785+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:39.513591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1","version":"289'1058","reported_seq":1069,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:44.756696+0000","last_change":"2023-10-09T05:58:27.377316+0000","last_active":"2023-10-09T05:58:44.756696+0000","last_peered":"2023-10-09T05:58:44.756696+0000","last_clean":"2023-10-09T05:58:44.756696+0000","last_became_active":"2023-10-09T05:58:27.377007+0000","last_became_peered":"2023-10-09T05:58:27.377007+0000","last_unstale":"2023-10-09T05:58:44.756696+0000","last_undegraded":"2023-10-09T05:58:44.756696+0000","last_fullsized":"2023-10-09T05:58:44.756696+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":1058,"log_dups_size":0,"ondisk_log_size":1058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:10.982167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69337088,"num_objects":1058,"num_object_clones":0,"num_object_copies":3174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1058,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1058,"num_write_kb":67712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":693,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.829571+0000","last_change":"2023-10-09T05:58:34.586268+0000","last_active":"2023-10-09T05:58:36.829571+0000","last_peered":"2023-10-09T05:58:36.829571+0000","last_clean":"2023-10-09T05:58:36.829571+0000","last_became_active":"2023-10-09T05:57:26.500602+0000","last_became_peered":"2023-10-09T05:57:26.500602+0000","last_unstale":"2023-10-09T05:58:36.829571+0000","last_undegraded":"2023-10-09T05:58:36.829571+0000","last_fullsized":"2023-10-09T05:58:36.829571+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.586180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.586180+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:22.464022+0000","scrub_duration":0.002003992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":183,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.247117+0000","last_change":"2023-10-09T05:55:59.298953+0000","last_active":"2023-10-09T05:58:33.247117+0000","last_peered":"2023-10-09T05:58:33.247117+0000","last_clean":"2023-10-09T05:58:33.247117+0000","last_became_active":"2023-10-09T05:55:59.298624+0000","last_became_peered":"2023-10-09T05:55:59.298624+0000","last_unstale":"2023-10-09T05:58:33.247117+0000","last_undegraded":"2023-10-09T05:58:33.247117+0000","last_fullsized":"2023-10-09T05:58:33.247117+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:00.513870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":51,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.248341+0000","last_change":"2023-10-09T05:57:23.468480+0000","last_active":"2023-10-09T05:58:33.248341+0000","last_peered":"2023-10-09T05:58:33.248341+0000","last_clean":"2023-10-09T05:58:33.248341+0000","last_became_active":"2023-10-09T05:57:23.468179+0000","last_became_peered":"2023-10-09T05:57:23.468179+0000","last_unstale":"2023-10-09T05:58:33.248341+0000","last_undegraded":"2023-10-09T05:58:33.248341+0000","last_fullsized":"2023-10-09T05:58:33.248341+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:18.157188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":152,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.274209+0000","last_change":"2023-10-09T05:56:49.249711+0000","last_active":"2023-10-09T05:58:33.274209+0000","last_peered":"2023-10-09T05:58:33.274209+0000","last_clean":"2023-10-09T05:58:33.274209+0000","last_became_active":"2023-10-09T05:56:49.249272+0000","last_became_peered":"2023-10-09T05:56:49.249272+0000","last_unstale":"2023-10-09T05:58:33.274209+0000","last_undegraded":"2023-10-09T05:58:33.274209+0000","last_fullsized":"2023-10-09T05:58:33.274209+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:42.596986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.7","version":"290'1486","reported_seq":1498,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T05:58:48.374312+0000","last_change":"2023-10-09T05:58:27.378776+0000","last_active":"2023-10-09T05:58:48.374312+0000","last_peered":"2023-10-09T05:58:48.374312+0000","last_clean":"2023-10-09T05:58:48.374312+0000","last_became_active":"2023-10-09T05:58:27.378504+0000","last_became_peered":"2023-10-09T05:58:27.378504+0000","last_unstale":"2023-10-09T05:58:48.374312+0000","last_undegraded":"2023-10-09T05:58:48.374312+0000","last_fullsized":"2023-10-09T05:58:48.374312+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":1486,"log_dups_size":0,"ondisk_log_size":1486,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:27.090596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97386496,"num_objects":1486,"num_object_clones":0,"num_object_copies":4458,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1486,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1486,"num_write_kb":95104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":491,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.225081+0000","last_change":"2023-10-09T05:58:36.225081+0000","last_active":"2023-10-09T05:58:36.225081+0000","last_peered":"2023-10-09T05:58:36.225081+0000","last_clean":"2023-10-09T05:58:36.225081+0000","last_became_active":"2023-10-09T05:56:49.237269+0000","last_became_peered":"2023-10-09T05:56:49.237269+0000","last_unstale":"2023-10-09T05:58:36.225081+0000","last_undegraded":"2023-10-09T05:58:36.225081+0000","last_fullsized":"2023-10-09T05:58:36.225081+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:02.908583+0000","scrub_duration":0.0016660799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":310,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.828720+0000","last_change":"2023-10-09T05:56:49.254337+0000","last_active":"2023-10-09T05:58:36.828720+0000","last_peered":"2023-10-09T05:58:36.828720+0000","last_clean":"2023-10-09T05:58:36.828720+0000","last_became_active":"2023-10-09T05:56:49.247892+0000","last_became_peered":"2023-10-09T05:56:49.247892+0000","last_unstale":"2023-10-09T05:58:36.828720+0000","last_undegraded":"2023-10-09T05:58:36.828720+0000","last_fullsized":"2023-10-09T05:58:36.828720+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:44:20.157638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":266,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.274845+0000","last_change":"2023-10-09T05:51:57.203142+0000","last_active":"2023-10-09T05:58:33.274845+0000","last_peered":"2023-10-09T05:58:33.274845+0000","last_clean":"2023-10-09T05:58:33.274845+0000","last_became_active":"2023-10-09T05:51:57.202850+0000","last_became_peered":"2023-10-09T05:51:57.202850+0000","last_unstale":"2023-10-09T05:58:33.274845+0000","last_undegraded":"2023-10-09T05:58:33.274845+0000","last_fullsized":"2023-10-09T05:58:33.274845+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:51.090737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":213,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.836163+0000","last_change":"2023-10-09T05:57:51.789746+0000","last_active":"2023-10-09T05:58:36.836163+0000","last_peered":"2023-10-09T05:58:36.836163+0000","last_clean":"2023-10-09T05:58:36.836163+0000","last_became_active":"2023-10-09T05:57:51.789353+0000","last_became_peered":"2023-10-09T05:57:51.789353+0000","last_unstale":"2023-10-09T05:58:36.836163+0000","last_undegraded":"2023-10-09T05:58:36.836163+0000","last_fullsized":"2023-10-09T05:58:36.836163+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:28.241819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.4","version":"289'1103","reported_seq":1116,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:44.732844+0000","last_change":"2023-10-09T05:58:27.375343+0000","last_active":"2023-10-09T05:58:44.732844+0000","last_peered":"2023-10-09T05:58:44.732844+0000","last_clean":"2023-10-09T05:58:44.732844+0000","last_became_active":"2023-10-09T05:58:27.375094+0000","last_became_peered":"2023-10-09T05:58:27.375094+0000","last_unstale":"2023-10-09T05:58:44.732844+0000","last_undegraded":"2023-10-09T05:58:44.732844+0000","last_fullsized":"2023-10-09T05:58:44.732844+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":1103,"log_dups_size":0,"ondisk_log_size":1103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:48:08.192351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72286208,"num_objects":1103,"num_object_clones":0,"num_object_copies":3309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1103,"num_write_kb":70592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":587,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.831177+0000","last_change":"2023-10-09T05:58:01.899855+0000","last_active":"2023-10-09T05:58:36.831177+0000","last_peered":"2023-10-09T05:58:36.831177+0000","last_clean":"2023-10-09T05:58:36.831177+0000","last_became_active":"2023-10-09T05:58:01.899103+0000","last_became_peered":"2023-10-09T05:58:01.899103+0000","last_unstale":"2023-10-09T05:58:36.831177+0000","last_undegraded":"2023-10-09T05:58:36.831177+0000","last_fullsized":"2023-10-09T05:58:36.831177+0000","mapping_epoch":268,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":269,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:36:42.354759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":71,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.826588+0000","last_change":"2023-10-09T05:57:23.465501+0000","last_active":"2023-10-09T05:58:36.826588+0000","last_peered":"2023-10-09T05:58:36.826588+0000","last_clean":"2023-10-09T05:58:36.826588+0000","last_became_active":"2023-10-09T05:57:23.465103+0000","last_became_peered":"2023-10-09T05:57:23.465103+0000","last_unstale":"2023-10-09T05:58:36.826588+0000","last_undegraded":"2023-10-09T05:58:36.826588+0000","last_fullsized":"2023-10-09T05:58:36.826588+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:02:45.891751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":179,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.826679+0000","last_change":"2023-10-09T05:55:59.308743+0000","last_active":"2023-10-09T05:58:36.826679+0000","last_peered":"2023-10-09T05:58:36.826679+0000","last_clean":"2023-10-09T05:58:36.826679+0000","last_became_active":"2023-10-09T05:55:59.308220+0000","last_became_peered":"2023-10-09T05:55:59.308220+0000","last_unstale":"2023-10-09T05:58:36.826679+0000","last_undegraded":"2023-10-09T05:58:36.826679+0000","last_fullsized":"2023-10-09T05:58:36.826679+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:44.246081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":183,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.247096+0000","last_change":"2023-10-09T05:56:49.238219+0000","last_active":"2023-10-09T05:58:33.247096+0000","last_peered":"2023-10-09T05:58:33.247096+0000","last_clean":"2023-10-09T05:58:33.247096+0000","last_became_active":"2023-10-09T05:56:49.237919+0000","last_became_peered":"2023-10-09T05:56:49.237919+0000","last_unstale":"2023-10-09T05:58:33.247096+0000","last_undegraded":"2023-10-09T05:58:33.247096+0000","last_fullsized":"2023-10-09T05:58:33.247096+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:44.599032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.6","version":"289'1091","reported_seq":1103,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:44.742738+0000","last_change":"2023-10-09T05:58:27.377502+0000","last_active":"2023-10-09T05:58:44.742738+0000","last_peered":"2023-10-09T05:58:44.742738+0000","last_clean":"2023-10-09T05:58:44.742738+0000","last_became_active":"2023-10-09T05:58:27.377195+0000","last_became_peered":"2023-10-09T05:58:27.377195+0000","last_unstale":"2023-10-09T05:58:44.742738+0000","last_undegraded":"2023-10-09T05:58:44.742738+0000","last_fullsized":"2023-10-09T05:58:44.742738+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":1091,"log_dups_size":0,"ondisk_log_size":1091,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:14.980773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71499776,"num_objects":1091,"num_object_clones":0,"num_object_copies":3273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1091,"num_write_kb":69824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":217,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.836114+0000","last_change":"2023-10-09T05:56:49.233224+0000","last_active":"2023-10-09T05:58:36.836114+0000","last_peered":"2023-10-09T05:58:36.836114+0000","last_clean":"2023-10-09T05:58:36.836114+0000","last_became_active":"2023-10-09T05:56:49.232971+0000","last_became_peered":"2023-10-09T05:56:49.232971+0000","last_unstale":"2023-10-09T05:58:36.836114+0000","last_undegraded":"2023-10-09T05:58:36.836114+0000","last_fullsized":"2023-10-09T05:58:36.836114+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:55.265488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":694,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:37.782281+0000","last_change":"2023-10-09T05:58:37.782237+0000","last_active":"2023-10-09T05:58:37.782281+0000","last_peered":"2023-10-09T05:58:37.782281+0000","last_clean":"2023-10-09T05:58:37.782281+0000","last_became_active":"2023-10-09T05:53:59.202004+0000","last_became_peered":"2023-10-09T05:53:59.202004+0000","last_unstale":"2023-10-09T05:58:37.782281+0000","last_undegraded":"2023-10-09T05:58:37.782281+0000","last_fullsized":"2023-10-09T05:58:37.782281+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:37.782173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:37.782173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:20.787963+0000","scrub_duration":0.001325908,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":51,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.248801+0000","last_change":"2023-10-09T05:57:23.479174+0000","last_active":"2023-10-09T05:58:33.248801+0000","last_peered":"2023-10-09T05:58:33.248801+0000","last_clean":"2023-10-09T05:58:33.248801+0000","last_became_active":"2023-10-09T05:57:23.478897+0000","last_became_peered":"2023-10-09T05:57:23.478897+0000","last_unstale":"2023-10-09T05:58:33.248801+0000","last_undegraded":"2023-10-09T05:58:33.248801+0000","last_fullsized":"2023-10-09T05:58:33.248801+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:00.160042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":252,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.829188+0000","last_change":"2023-10-09T05:53:59.195325+0000","last_active":"2023-10-09T05:58:36.829188+0000","last_peered":"2023-10-09T05:58:36.829188+0000","last_clean":"2023-10-09T05:58:36.829188+0000","last_became_active":"2023-10-09T05:53:59.194954+0000","last_became_peered":"2023-10-09T05:53:59.194954+0000","last_unstale":"2023-10-09T05:58:36.829188+0000","last_undegraded":"2023-10-09T05:58:36.829188+0000","last_fullsized":"2023-10-09T05:58:36.829188+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:23.228416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"290'1164","reported_seq":1177,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T05:58:45.425514+0000","last_change":"2023-10-09T05:58:27.375153+0000","last_active":"2023-10-09T05:58:45.425514+0000","last_peered":"2023-10-09T05:58:45.425514+0000","last_clean":"2023-10-09T05:58:45.425514+0000","last_became_active":"2023-10-09T05:58:27.374820+0000","last_became_peered":"2023-10-09T05:58:27.374820+0000","last_unstale":"2023-10-09T05:58:45.425514+0000","last_undegraded":"2023-10-09T05:58:45.425514+0000","last_fullsized":"2023-10-09T05:58:45.425514+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":1164,"log_dups_size":0,"ondisk_log_size":1164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:08.834399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76283904,"num_objects":1164,"num_object_clones":0,"num_object_copies":3492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1164,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1164,"num_write_kb":74496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":485,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:40.724500+0000","last_change":"2023-10-09T05:58:40.724430+0000","last_active":"2023-10-09T05:58:40.724500+0000","last_peered":"2023-10-09T05:58:40.724500+0000","last_clean":"2023-10-09T05:58:40.724500+0000","last_became_active":"2023-10-09T05:57:54.796937+0000","last_became_peered":"2023-10-09T05:57:54.796937+0000","last_unstale":"2023-10-09T05:58:40.724500+0000","last_undegraded":"2023-10-09T05:58:40.724500+0000","last_fullsized":"2023-10-09T05:58:40.724500+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:40.724331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:40.724331+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:10.562541+0000","scrub_duration":0.0016947469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":156,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.830797+0000","last_change":"2023-10-09T05:55:59.300015+0000","last_active":"2023-10-09T05:58:36.830797+0000","last_peered":"2023-10-09T05:58:36.830797+0000","last_clean":"2023-10-09T05:58:36.830797+0000","last_became_active":"2023-10-09T05:55:59.299281+0000","last_became_peered":"2023-10-09T05:55:59.299281+0000","last_unstale":"2023-10-09T05:58:36.830797+0000","last_undegraded":"2023-10-09T05:58:36.830797+0000","last_fullsized":"2023-10-09T05:58:36.830797+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:50:32.465904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":226,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.248540+0000","last_change":"2023-10-09T05:57:23.471369+0000","last_active":"2023-10-09T05:58:33.248540+0000","last_peered":"2023-10-09T05:58:33.248540+0000","last_clean":"2023-10-09T05:58:33.248540+0000","last_became_active":"2023-10-09T05:57:23.471049+0000","last_became_peered":"2023-10-09T05:57:23.471049+0000","last_unstale":"2023-10-09T05:58:33.248540+0000","last_undegraded":"2023-10-09T05:58:33.248540+0000","last_fullsized":"2023-10-09T05:58:33.248540+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:21.208825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":343,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:34.750928+0000","last_change":"2023-10-09T05:57:10.348399+0000","last_active":"2023-10-09T05:58:34.750928+0000","last_peered":"2023-10-09T05:58:34.750928+0000","last_clean":"2023-10-09T05:58:34.750928+0000","last_became_active":"2023-10-09T05:57:10.348061+0000","last_became_peered":"2023-10-09T05:57:10.348061+0000","last_unstale":"2023-10-09T05:58:34.750928+0000","last_undegraded":"2023-10-09T05:58:34.750928+0000","last_fullsized":"2023-10-09T05:58:34.750928+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:12.453118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"289'1045","reported_seq":1058,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:44.681439+0000","last_change":"2023-10-09T05:58:27.405929+0000","last_active":"2023-10-09T05:58:44.681439+0000","last_peered":"2023-10-09T05:58:44.681439+0000","last_clean":"2023-10-09T05:58:44.681439+0000","last_became_active":"2023-10-09T05:58:27.405604+0000","last_became_peered":"2023-10-09T05:58:27.405604+0000","last_unstale":"2023-10-09T05:58:44.681439+0000","last_undegraded":"2023-10-09T05:58:44.681439+0000","last_fullsized":"2023-10-09T05:58:44.681439+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":1045,"log_dups_size":0,"ondisk_log_size":1045,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:19.748373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68485120,"num_objects":1045,"num_object_clones":0,"num_object_copies":3135,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1045,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1045,"num_write_kb":66880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":683,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.829819+0000","last_change":"2023-10-09T05:58:35.630429+0000","last_active":"2023-10-09T05:58:36.829819+0000","last_peered":"2023-10-09T05:58:36.829819+0000","last_clean":"2023-10-09T05:58:36.829819+0000","last_became_active":"2023-10-09T05:57:26.517990+0000","last_became_peered":"2023-10-09T05:57:26.517990+0000","last_unstale":"2023-10-09T05:58:36.829819+0000","last_undegraded":"2023-10-09T05:58:36.829819+0000","last_fullsized":"2023-10-09T05:58:36.829819+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:35.630348+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:35.630348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:14:01.075118+0000","scrub_duration":0.0020631920000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":124,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.832307+0000","last_change":"2023-10-09T05:53:59.201576+0000","last_active":"2023-10-09T05:58:36.832307+0000","last_peered":"2023-10-09T05:58:36.832307+0000","last_clean":"2023-10-09T05:58:36.832307+0000","last_became_active":"2023-10-09T05:53:59.201309+0000","last_became_peered":"2023-10-09T05:53:59.201309+0000","last_unstale":"2023-10-09T05:58:36.832307+0000","last_undegraded":"2023-10-09T05:58:36.832307+0000","last_fullsized":"2023-10-09T05:58:36.832307+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:54.956371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":62,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.827003+0000","last_change":"2023-10-09T05:57:10.368477+0000","last_active":"2023-10-09T05:58:36.827003+0000","last_peered":"2023-10-09T05:58:36.827003+0000","last_clean":"2023-10-09T05:58:36.827003+0000","last_became_active":"2023-10-09T05:57:10.368178+0000","last_became_peered":"2023-10-09T05:57:10.368178+0000","last_unstale":"2023-10-09T05:58:36.827003+0000","last_undegraded":"2023-10-09T05:58:36.827003+0000","last_fullsized":"2023-10-09T05:58:36.827003+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:38.053000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":215,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.248127+0000","last_change":"2023-10-09T05:53:59.188462+0000","last_active":"2023-10-09T05:58:33.248127+0000","last_peered":"2023-10-09T05:58:33.248127+0000","last_clean":"2023-10-09T05:58:33.248127+0000","last_became_active":"2023-10-09T05:53:59.188061+0000","last_became_peered":"2023-10-09T05:53:59.188061+0000","last_unstale":"2023-10-09T05:58:33.248127+0000","last_undegraded":"2023-10-09T05:58:33.248127+0000","last_fullsized":"2023-10-09T05:58:33.248127+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:19.167114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"289'1020","reported_seq":1033,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:44.718327+0000","last_change":"2023-10-09T05:58:27.377750+0000","last_active":"2023-10-09T05:58:44.718327+0000","last_peered":"2023-10-09T05:58:44.718327+0000","last_clean":"2023-10-09T05:58:44.718327+0000","last_became_active":"2023-10-09T05:58:27.377457+0000","last_became_peered":"2023-10-09T05:58:27.377457+0000","last_unstale":"2023-10-09T05:58:44.718327+0000","last_undegraded":"2023-10-09T05:58:44.718327+0000","last_fullsized":"2023-10-09T05:58:44.718327+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":1020,"log_dups_size":0,"ondisk_log_size":1020,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:52:04.242389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66846720,"num_objects":1020,"num_object_clones":0,"num_object_copies":3060,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1020,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1020,"num_write_kb":65280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":653,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:35.543061+0000","last_change":"2023-10-09T05:58:35.542976+0000","last_active":"2023-10-09T05:58:35.543061+0000","last_peered":"2023-10-09T05:58:35.543061+0000","last_clean":"2023-10-09T05:58:35.543061+0000","last_became_active":"2023-10-09T05:57:13.235353+0000","last_became_peered":"2023-10-09T05:57:13.235353+0000","last_unstale":"2023-10-09T05:58:35.543061+0000","last_undegraded":"2023-10-09T05:58:35.543061+0000","last_fullsized":"2023-10-09T05:58:35.543061+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:35.542853+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:35.542853+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:30.605580+0000","scrub_duration":0.001732169,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":295,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.249870+0000","last_change":"2023-10-09T05:53:59.228948+0000","last_active":"2023-10-09T05:58:33.249870+0000","last_peered":"2023-10-09T05:58:33.249870+0000","last_clean":"2023-10-09T05:58:33.249870+0000","last_became_active":"2023-10-09T05:53:59.186359+0000","last_became_peered":"2023-10-09T05:53:59.186359+0000","last_unstale":"2023-10-09T05:58:33.249870+0000","last_undegraded":"2023-10-09T05:58:33.249870+0000","last_fullsized":"2023-10-09T05:58:33.249870+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:39.990576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":147,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.829660+0000","last_change":"2023-10-09T05:55:59.339957+0000","last_active":"2023-10-09T05:58:36.829660+0000","last_peered":"2023-10-09T05:58:36.829660+0000","last_clean":"2023-10-09T05:58:36.829660+0000","last_became_active":"2023-10-09T05:55:59.339614+0000","last_became_peered":"2023-10-09T05:55:59.339614+0000","last_unstale":"2023-10-09T05:58:36.829660+0000","last_undegraded":"2023-10-09T05:58:36.829660+0000","last_fullsized":"2023-10-09T05:58:36.829660+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:01.613113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":51,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.248036+0000","last_change":"2023-10-09T05:57:23.477143+0000","last_active":"2023-10-09T05:58:33.248036+0000","last_peered":"2023-10-09T05:58:33.248036+0000","last_clean":"2023-10-09T05:58:33.248036+0000","last_became_active":"2023-10-09T05:57:23.475655+0000","last_became_peered":"2023-10-09T05:57:23.475655+0000","last_unstale":"2023-10-09T05:58:33.248036+0000","last_undegraded":"2023-10-09T05:58:33.248036+0000","last_fullsized":"2023-10-09T05:58:33.248036+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:29.528023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"289'1070","reported_seq":1083,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:44.713747+0000","last_change":"2023-10-09T05:58:27.376100+0000","last_active":"2023-10-09T05:58:44.713747+0000","last_peered":"2023-10-09T05:58:44.713747+0000","last_clean":"2023-10-09T05:58:44.713747+0000","last_became_active":"2023-10-09T05:58:27.375806+0000","last_became_peered":"2023-10-09T05:58:27.375806+0000","last_unstale":"2023-10-09T05:58:44.713747+0000","last_undegraded":"2023-10-09T05:58:44.713747+0000","last_fullsized":"2023-10-09T05:58:44.713747+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":1070,"log_dups_size":0,"ondisk_log_size":1070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:56.466666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70123520,"num_objects":1070,"num_object_clones":0,"num_object_copies":3210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1070,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1070,"num_write_kb":68480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":545,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:34.333129+0000","last_change":"2023-10-09T05:58:34.333129+0000","last_active":"2023-10-09T05:58:34.333129+0000","last_peered":"2023-10-09T05:58:34.333129+0000","last_clean":"2023-10-09T05:58:34.333129+0000","last_became_active":"2023-10-09T05:57:26.502911+0000","last_became_peered":"2023-10-09T05:57:26.502911+0000","last_unstale":"2023-10-09T05:58:34.333129+0000","last_undegraded":"2023-10-09T05:58:34.333129+0000","last_fullsized":"2023-10-09T05:58:34.333129+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:25.230712+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":294,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.829667+0000","last_change":"2023-10-09T05:57:58.832790+0000","last_active":"2023-10-09T05:58:36.829667+0000","last_peered":"2023-10-09T05:58:36.829667+0000","last_clean":"2023-10-09T05:58:36.829667+0000","last_became_active":"2023-10-09T05:57:58.832471+0000","last_became_peered":"2023-10-09T05:57:58.832471+0000","last_unstale":"2023-10-09T05:58:36.829667+0000","last_undegraded":"2023-10-09T05:58:36.829667+0000","last_fullsized":"2023-10-09T05:58:36.829667+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:39.358174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":329,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:34.750982+0000","last_change":"2023-10-09T05:57:58.836050+0000","last_active":"2023-10-09T05:58:34.750982+0000","last_peered":"2023-10-09T05:58:34.750982+0000","last_clean":"2023-10-09T05:58:34.750982+0000","last_became_active":"2023-10-09T05:57:58.833691+0000","last_became_peered":"2023-10-09T05:57:58.833691+0000","last_unstale":"2023-10-09T05:58:34.750982+0000","last_undegraded":"2023-10-09T05:58:34.750982+0000","last_fullsized":"2023-10-09T05:58:34.750982+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:32.257849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":231,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.274474+0000","last_change":"2023-10-09T05:40:12.676587+0000","last_active":"2023-10-09T05:58:33.274474+0000","last_peered":"2023-10-09T05:58:33.274474+0000","last_clean":"2023-10-09T05:58:33.274474+0000","last_became_active":"2023-10-09T05:40:12.676330+0000","last_became_peered":"2023-10-09T05:40:12.676330+0000","last_unstale":"2023-10-09T05:58:33.274474+0000","last_undegraded":"2023-10-09T05:58:33.274474+0000","last_fullsized":"2023-10-09T05:58:33.274474+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:46.659556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":218,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.248216+0000","last_change":"2023-10-09T05:53:59.191302+0000","last_active":"2023-10-09T05:58:33.248216+0000","last_peered":"2023-10-09T05:58:33.248216+0000","last_clean":"2023-10-09T05:58:33.248216+0000","last_became_active":"2023-10-09T05:53:59.190911+0000","last_became_peered":"2023-10-09T05:53:59.190911+0000","last_unstale":"2023-10-09T05:58:33.248216+0000","last_undegraded":"2023-10-09T05:58:33.248216+0000","last_fullsized":"2023-10-09T05:58:33.248216+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:12.801201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":71,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T05:58:36.826796+0000","last_change":"2023-10-09T05:57:23.469986+0000","last_active":"2023-10-09T05:58:36.826796+0000","last_peered":"2023-10-09T05:58:36.826796+0000","last_clean":"2023-10-09T05:58:36.826796+0000","last_became_active":"2023-10-09T05:57:23.469546+0000","last_became_peered":"2023-10-09T05:57:23.469546+0000","last_unstale":"2023-10-09T05:58:36.826796+0000","last_undegraded":"2023-10-09T05:58:36.826796+0000","last_fullsized":"2023-10-09T05:58:36.826796+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:53.762345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":79,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.275068+0000","last_change":"2023-10-09T05:56:49.244931+0000","last_active":"2023-10-09T05:58:33.275068+0000","last_peered":"2023-10-09T05:58:33.275068+0000","last_clean":"2023-10-09T05:58:33.275068+0000","last_became_active":"2023-10-09T05:56:49.244651+0000","last_became_peered":"2023-10-09T05:56:49.244651+0000","last_unstale":"2023-10-09T05:58:33.275068+0000","last_undegraded":"2023-10-09T05:58:33.275068+0000","last_fullsized":"2023-10-09T05:58:33.275068+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:53.906578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":325,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.274849+0000","last_change":"2023-10-09T05:57:10.356351+0000","last_active":"2023-10-09T05:58:33.274849+0000","last_peered":"2023-10-09T05:58:33.274849+0000","last_clean":"2023-10-09T05:58:33.274849+0000","last_became_active":"2023-10-09T05:57:10.356037+0000","last_became_peered":"2023-10-09T05:57:10.356037+0000","last_unstale":"2023-10-09T05:58:33.274849+0000","last_undegraded":"2023-10-09T05:58:33.274849+0000","last_fullsized":"2023-10-09T05:58:33.274849+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:15.139912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":79,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.275031+0000","last_change":"2023-10-09T05:56:49.245612+0000","last_active":"2023-10-09T05:58:33.275031+0000","last_peered":"2023-10-09T05:58:33.275031+0000","last_clean":"2023-10-09T05:58:33.275031+0000","last_became_active":"2023-10-09T05:56:49.245235+0000","last_became_peered":"2023-10-09T05:56:49.245235+0000","last_unstale":"2023-10-09T05:58:33.275031+0000","last_undegraded":"2023-10-09T05:58:33.275031+0000","last_fullsized":"2023-10-09T05:58:33.275031+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:29.016794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":282,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.274452+0000","last_change":"2023-10-09T05:51:57.093104+0000","last_active":"2023-10-09T05:58:33.274452+0000","last_peered":"2023-10-09T05:58:33.274452+0000","last_clean":"2023-10-09T05:58:33.274452+0000","last_became_active":"2023-10-09T05:51:57.092810+0000","last_became_peered":"2023-10-09T05:51:57.092810+0000","last_unstale":"2023-10-09T05:58:33.274452+0000","last_undegraded":"2023-10-09T05:58:33.274452+0000","last_fullsized":"2023-10-09T05:58:33.274452+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:51.192227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":51,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.249426+0000","last_change":"2023-10-09T05:57:23.479560+0000","last_active":"2023-10-09T05:58:33.249426+0000","last_peered":"2023-10-09T05:58:33.249426+0000","last_clean":"2023-10-09T05:58:33.249426+0000","last_became_active":"2023-10-09T05:57:23.479177+0000","last_became_peered":"2023-10-09T05:57:23.479177+0000","last_unstale":"2023-10-09T05:58:33.249426+0000","last_undegraded":"2023-10-09T05:58:33.249426+0000","last_fullsized":"2023-10-09T05:58:33.249426+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:41.925161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":51,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.248895+0000","last_change":"2023-10-09T05:57:23.475179+0000","last_active":"2023-10-09T05:58:33.248895+0000","last_peered":"2023-10-09T05:58:33.248895+0000","last_clean":"2023-10-09T05:58:33.248895+0000","last_became_active":"2023-10-09T05:57:23.474904+0000","last_became_peered":"2023-10-09T05:57:23.474904+0000","last_unstale":"2023-10-09T05:58:33.248895+0000","last_undegraded":"2023-10-09T05:58:33.248895+0000","last_fullsized":"2023-10-09T05:58:33.248895+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:54.587601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":256,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.247268+0000","last_change":"2023-10-09T05:57:10.379258+0000","last_active":"2023-10-09T05:58:33.247268+0000","last_peered":"2023-10-09T05:58:33.247268+0000","last_clean":"2023-10-09T05:58:33.247268+0000","last_became_active":"2023-10-09T05:57:10.359745+0000","last_became_peered":"2023-10-09T05:57:10.359745+0000","last_unstale":"2023-10-09T05:58:33.247268+0000","last_undegraded":"2023-10-09T05:58:33.247268+0000","last_fullsized":"2023-10-09T05:58:33.247268+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:14.750256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":308,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T05:58:33.274450+0000","last_change":"2023-10-09T05:57:23.481065+0000","last_active":"2023-10-09T05:58:33.274450+0000","last_peered":"2023-10-09T05:58:33.274450+0000","last_clean":"2023-10-09T05:58:33.274450+0000","last_became_active":"2023-10-09T05:57:23.480203+0000","last_became_peered":"2023-10-09T05:57:23.480203+0000","last_unstale":"2023-10-09T05:58:33.274450+0000","last_undegraded":"2023-10-09T05:58:33.274450+0000","last_fullsized":"2023-10-09T05:58:33.274450+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:15:53.196457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":1290993664,"num_objects":19699,"num_object_clones":0,"num_object_copies":59097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":19699,"num_write_kb":1260736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3744268288,"data_stored":3744268288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":19699,"ondisk_log_size":19699,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":10792,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":670873,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":124,"num_write":33,"num_write_kb":330,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":241515,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234229,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":782620,"kb_used_data":477140,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92962532,"statfs":{"total":95995035648,"available":95193632768,"internally_reserved":0,"allocated":488591360,"data_stored":487561347,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:02 2023","interfaces":[{"interface":"back","average":{"1min":1.9319999999999999,"5min":1.105,"15min":1.7949999999999999},"min":{"1min":0.246,"5min":0.22,"15min":0.22},"max":{"1min":21.038,"5min":21.038,"15min":21.038},"last":1.4039999999999999},{"interface":"front","average":{"1min":2.0129999999999999,"5min":1.1100000000000001,"15min":1.6859999999999999},"min":{"1min":0.245,"5min":0.224,"15min":0.224},"max":{"1min":21.152000000000001,"5min":21.152000000000001,"15min":21.152000000000001},"last":0.379}]},{"osd":1,"last update":"Mon Oct 9 05:58:31 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":1.4590000000000001,"15min":0.95899999999999996},"min":{"1min":0.32500000000000001,"5min":0.27200000000000002,"15min":0.22900000000000001},"max":{"1min":2.1699999999999999,"5min":35.113999999999997,"15min":35.113999999999997},"last":0.76800000000000002},{"interface":"front","average":{"1min":0.68100000000000005,"5min":1.2130000000000001,"15min":0.86399999999999999},"min":{"1min":0.30099999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.4449999999999998,"5min":34.768999999999998,"15min":34.768999999999998},"last":1.522}]},{"osd":2,"last update":"Mon Oct 9 05:58:02 2023","interfaces":[{"interface":"back","average":{"1min":12.797000000000001,"5min":3.3069999999999999,"15min":2.0230000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28199999999999997},"max":{"1min":119.91200000000001,"5min":119.91200000000001,"15min":119.91200000000001},"last":1.3680000000000001},{"interface":"front","average":{"1min":12.898999999999999,"5min":3.3580000000000001,"15min":2.117},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":119.95099999999999,"5min":119.95099999999999,"15min":119.95099999999999},"last":0.40200000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":2.552,"15min":2.9470000000000001},"min":{"1min":0.36599999999999999,"5min":0.35999999999999999,"15min":0.33300000000000002},"max":{"1min":2.7069999999999999,"5min":70.840999999999994,"15min":147.54900000000001},"last":2.0139999999999998},{"interface":"front","average":{"1min":0.90700000000000003,"5min":2.391,"15min":2.847},"min":{"1min":0.35299999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.4590000000000001,"5min":71.212000000000003,"15min":144.065},"last":3.0739999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":0.66500000000000004,"15min":0.60899999999999999},"min":{"1min":0.27500000000000002,"5min":0.27000000000000002,"15min":0.24399999999999999},"max":{"1min":2.9990000000000001,"5min":3.7120000000000002,"15min":9.5329999999999995},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.60599999999999998,"15min":0.66100000000000003},"min":{"1min":0.33500000000000002,"5min":0.29899999999999999,"15min":0.27400000000000002},"max":{"1min":3.835,"5min":3.835,"15min":7.6120000000000001},"last":0.45400000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.91900000000000004,"15min":0.75800000000000001},"min":{"1min":0.29799999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.8090000000000002,"5min":15.622,"15min":15.622},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.76400000000000001,"5min":0.64000000000000001,"15min":0.624},"min":{"1min":0.33300000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.3969999999999998,"5min":3.1869999999999998,"15min":3.9380000000000002},"last":0.36299999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":0.86599999999999999,"15min":1.2769999999999999},"min":{"1min":0.29499999999999998,"5min":0.28799999999999998,"15min":0.28299999999999997},"max":{"1min":3.2229999999999999,"5min":12.727,"15min":131.94399999999999},"last":1.085},{"interface":"front","average":{"1min":0.82099999999999995,"5min":0.70099999999999996,"15min":1.258},"min":{"1min":0.378,"5min":0.33100000000000002,"15min":0.29999999999999999},"max":{"1min":3.5859999999999999,"5min":3.5859999999999999,"15min":135.23400000000001},"last":0.48899999999999999}]}]},{"osd":6,"up_from":46,"seq":197568495866,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":866324,"kb_used_data":569036,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":92878828,"statfs":{"total":95995035648,"available":95107919872,"internally_reserved":0,"allocated":582692864,"data_stored":581678675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2200000000000002,"5min":1.534,"15min":1.286},"min":{"1min":0.23499999999999999,"5min":0.224,"15min":0.224},"max":{"1min":13.026999999999999,"5min":49.389000000000003,"15min":49.389000000000003},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.8280000000000001,"5min":1.4710000000000001,"15min":1.3089999999999999},"min":{"1min":0.26300000000000001,"5min":0.253,"15min":0.253},"max":{"1min":12.629,"5min":49.338999999999999,"15min":49.338999999999999},"last":0.441}]},{"osd":1,"last update":"Mon Oct 9 05:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0580000000000001,"5min":1,"15min":0.80300000000000005},"min":{"1min":0.32200000000000001,"5min":0.246,"15min":0.23899999999999999},"max":{"1min":8.7720000000000002,"5min":17.341000000000001,"15min":17.341000000000001},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.99199999999999999,"15min":0.81999999999999995},"min":{"1min":0.29199999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":2.9980000000000002,"5min":12.474,"15min":12.474},"last":0.40899999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:58:02 2023","interfaces":[{"interface":"back","average":{"1min":6.8380000000000001,"5min":2.097,"15min":1.1899999999999999},"min":{"1min":0.311,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":61.088999999999999,"5min":61.088999999999999,"15min":61.088999999999999},"last":27.224},{"interface":"front","average":{"1min":6.3579999999999997,"5min":2.1040000000000001,"15min":1.1879999999999999},"min":{"1min":0.38800000000000001,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":61.082000000000001,"5min":61.082000000000001,"15min":61.082000000000001},"last":27.177}]},{"osd":3,"last update":"Mon Oct 9 05:58:43 2023","interfaces":[{"interface":"back","average":{"1min":4.609,"5min":4.4900000000000002,"15min":2.052},"min":{"1min":0.32000000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":62.423999999999999,"5min":93.626000000000005,"15min":93.626000000000005},"last":9.3140000000000001},{"interface":"front","average":{"1min":4.6669999999999998,"5min":4.2080000000000002,"15min":1.9750000000000001},"min":{"1min":0.47499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":62.518999999999998,"5min":93.643000000000001,"15min":93.643000000000001},"last":9.2080000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:57:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":0.72599999999999998,"15min":0.71099999999999997},"min":{"1min":0.252,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.5709999999999997,"5min":4.5709999999999997,"15min":7.9100000000000001},"last":0.36199999999999999},{"interface":"front","average":{"1min":1.2629999999999999,"5min":0.84999999999999998,"15min":0.73099999999999998},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.2069999999999999,"5min":8.734,"15min":8.734},"last":0.32500000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:57:48 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":1.284,"15min":0.89500000000000002},"min":{"1min":0.316,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.5910000000000002,"5min":45.933999999999997,"15min":45.933999999999997},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.2669999999999999,"15min":0.92400000000000004},"min":{"1min":0.35899999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.786,"5min":45.917999999999999,"15min":45.917999999999999},"last":0.39500000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":1.337,"15min":1.04},"min":{"1min":0.33500000000000002,"5min":0.314,"15min":0.314},"max":{"1min":4.4820000000000002,"5min":52.042999999999999,"15min":52.042999999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.093,"5min":1.3080000000000001,"15min":1.008},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.9539999999999997,"5min":51.259,"15min":51.259},"last":0.45300000000000001}]}]},{"osd":1,"up_from":0,"seq":382252089454,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":129,"seq":554050781256,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":646748,"kb_used_data":504788,"kb_used_omap":29,"kb_used_meta":141922,"kb_avail":93098404,"statfs":{"total":95995035648,"available":95332765696,"internally_reserved":0,"allocated":516902912,"data_stored":515231427,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30322,"internal_metadata":145328526},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:58:04 2023","interfaces":[{"interface":"back","average":{"1min":1.387,"5min":0.94899999999999995,"15min":0.78400000000000003},"min":{"1min":0.21299999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":6.6120000000000001,"5min":9.1720000000000006,"15min":9.1720000000000006},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.99399999999999999,"5min":0.84999999999999998,"15min":1.048},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":3.4449999999999998,"5min":6.6420000000000003,"15min":6.6420000000000003},"last":0.61799999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:58:04 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":1.204,"15min":1.0940000000000001},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":13.471,"5min":13.471,"15min":13.471},"last":0.73199999999999998},{"interface":"front","average":{"1min":2.4500000000000002,"5min":1.1200000000000001,"15min":1.046},"min":{"1min":0.245,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":13.561,"5min":13.561,"15min":13.561},"last":0.31900000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:58:04 2023","interfaces":[{"interface":"back","average":{"1min":4.6760000000000002,"5min":1.635,"15min":1.0609999999999999},"min":{"1min":0.29499999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":27.954000000000001,"5min":27.954000000000001,"15min":27.954000000000001},"last":1.4199999999999999},{"interface":"front","average":{"1min":3.851,"5min":1.4259999999999999,"15min":1.0760000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":20.027000000000001,"5min":20.027000000000001,"15min":20.027000000000001},"last":0.71899999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:58:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":3.9910000000000001,"15min":2.145},"min":{"1min":0.316,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.3639999999999999,"5min":149.07400000000001,"15min":149.07400000000001},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.2110000000000001,"5min":3.8399999999999999,"15min":1.8819999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.4509999999999996,"5min":149.12700000000001,"15min":149.12700000000001},"last":0.76500000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:58:04 2023","interfaces":[{"interface":"back","average":{"1min":1.873,"5min":2.4510000000000001,"15min":1.583},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":11.09,"5min":100.25,"15min":100.25},"last":0.64000000000000001},{"interface":"front","average":{"1min":1.6459999999999999,"5min":2.4550000000000001,"15min":1.4279999999999999},"min":{"1min":0.39800000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":5.077,"5min":102.40300000000001,"15min":102.40300000000001},"last":0.48999999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:58:04 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":0.94599999999999995,"15min":1.21},"min":{"1min":0.35199999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":6.5439999999999996,"5min":8.1470000000000002,"15min":8.1470000000000002},"last":0.68400000000000005},{"interface":"front","average":{"1min":1.825,"5min":1.2350000000000001,"15min":1.2290000000000001},"min":{"1min":0.44500000000000001,"5min":0.378,"15min":0.378},"max":{"1min":4.4000000000000004,"5min":9.141,"15min":9.141},"last":0.54600000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:58:04 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":2.0470000000000002,"15min":1.629},"min":{"1min":0.42399999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":4.6139999999999999,"5min":99.489000000000004,"15min":99.489000000000004},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.032,"5min":1.7909999999999999,"15min":1.534},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":3.581,"5min":99.510000000000005,"15min":99.510000000000005},"last":0.78500000000000003}]}]},{"osd":2,"up_from":94,"seq":403726925946,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1251796,"kb_used_data":982540,"kb_used_omap":81,"kb_used_meta":269166,"kb_avail":92493356,"statfs":{"total":95995035648,"available":94713196544,"internally_reserved":0,"allocated":1006120960,"data_stored":1004577401,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":275626202},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:11 2023","interfaces":[{"interface":"back","average":{"1min":4.9169999999999998,"5min":2.056,"15min":1.8169999999999999},"min":{"1min":0.23300000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":55.409999999999997,"5min":55.409999999999997,"15min":55.409999999999997},"last":1.891},{"interface":"front","average":{"1min":4.7009999999999996,"5min":2.056,"15min":1.726},"min":{"1min":0.28199999999999997,"5min":0.23899999999999999,"15min":0.23100000000000001},"max":{"1min":55.326000000000001,"5min":55.326000000000001,"15min":55.326000000000001},"last":1.8620000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:58:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.0069999999999999,"15min":1.357},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":3.8959999999999999,"5min":7.7119999999999997,"15min":163.24199999999999},"last":0.60799999999999998},{"interface":"front","average":{"1min":1.4550000000000001,"5min":0.85299999999999998,"15min":1.272},"min":{"1min":0.223,"5min":0.223,"15min":0.214},"max":{"1min":6.9790000000000001,"5min":6.9790000000000001,"15min":163.27000000000001},"last":1.661}]},{"osd":3,"last update":"Mon Oct 9 05:58:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1559999999999999,"5min":2.0880000000000001,"15min":1.806},"min":{"1min":0.30299999999999999,"5min":0.25900000000000001,"15min":0.23899999999999999},"max":{"1min":9.6620000000000008,"5min":55.118000000000002,"15min":170.07599999999999},"last":1.5900000000000001},{"interface":"front","average":{"1min":0.95599999999999996,"5min":2.0760000000000001,"15min":1.379},"min":{"1min":0.39200000000000002,"5min":0.31900000000000001,"15min":0.25},"max":{"1min":3.952,"5min":55.365000000000002,"15min":63.600000000000001},"last":2.9540000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:58:05 2023","interfaces":[{"interface":"back","average":{"1min":1.7529999999999999,"5min":1.1060000000000001,"15min":0.95299999999999996},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28199999999999997},"max":{"1min":8.6280000000000001,"5min":8.6280000000000001,"15min":11.372999999999999},"last":0.81699999999999995},{"interface":"front","average":{"1min":1.391,"5min":0.93700000000000006,"15min":0.90100000000000002},"min":{"1min":0.373,"5min":0.33200000000000002,"15min":0.26800000000000002},"max":{"1min":6.1710000000000003,"5min":6.1710000000000003,"15min":6.5709999999999997},"last":1.6259999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:58:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5860000000000001,"5min":1.3380000000000001,"15min":0.995},"min":{"1min":0.35099999999999998,"5min":0.30599999999999999,"15min":0.28999999999999998},"max":{"1min":5.3380000000000001,"5min":13.414,"15min":13.414},"last":1.5109999999999999},{"interface":"front","average":{"1min":2.101,"5min":1.242,"15min":0.97799999999999998},"min":{"1min":0.33000000000000002,"5min":0.313,"15min":0.29799999999999999},"max":{"1min":8.5570000000000004,"5min":13.438000000000001,"15min":13.438000000000001},"last":0.76600000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:58:05 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":0.96199999999999997,"15min":0.91100000000000003},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.29399999999999998},"max":{"1min":3.2999999999999998,"5min":8.6920000000000002,"15min":11.59},"last":1.6879999999999999},{"interface":"front","average":{"1min":1.48,"5min":1.113,"15min":0.90900000000000003},"min":{"1min":0.39800000000000002,"5min":0.37,"15min":0.34000000000000002},"max":{"1min":4.4850000000000003,"5min":8.6140000000000008,"15min":8.6140000000000008},"last":1.54}]},{"osd":7,"last update":"Mon Oct 9 05:58:05 2023","interfaces":[{"interface":"back","average":{"1min":4.4020000000000001,"5min":1.8620000000000001,"15min":1.274},"min":{"1min":0.44700000000000001,"5min":0.36599999999999999,"15min":0.35899999999999999},"max":{"1min":55.052,"5min":55.052,"15min":55.052},"last":1.5669999999999999},{"interface":"front","average":{"1min":2.0110000000000001,"5min":1.252,"15min":1.093},"min":{"1min":0.438,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.6010000000000009,"5min":8.6010000000000009,"15min":40.113},"last":0.65200000000000002}]}]},{"osd":3,"up_from":99,"seq":425201762419,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1091024,"kb_used_data":813192,"kb_used_omap":7,"kb_used_meta":277752,"kb_avail":92654128,"statfs":{"total":95995035648,"available":94877827072,"internally_reserved":0,"allocated":832708608,"data_stored":830983398,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":284418664},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":0.93100000000000005,"15min":1.3420000000000001},"min":{"1min":0.224,"5min":0.186,"15min":0.186},"max":{"1min":4.7380000000000004,"5min":12.548,"15min":15.536},"last":0.73399999999999999},{"interface":"front","average":{"1min":2.0550000000000002,"5min":1.363,"15min":1.516},"min":{"1min":0.22800000000000001,"5min":0.214,"15min":0.214},"max":{"1min":14.247,"5min":14.707000000000001,"15min":14.707000000000001},"last":0.93700000000000006}]},{"osd":1,"last update":"Mon Oct 9 05:58:42 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":1.0700000000000001,"15min":1.1579999999999999},"min":{"1min":0.28299999999999997,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":8.2379999999999995,"5min":14.568,"15min":127.226},"last":1.012},{"interface":"front","average":{"1min":0.97099999999999997,"5min":0.83099999999999996,"15min":1.127},"min":{"1min":0.32600000000000001,"5min":0.24399999999999999,"15min":0.21099999999999999},"max":{"1min":3.8159999999999998,"5min":9.4239999999999995,"15min":127.468},"last":0.58999999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:58:42 2023","interfaces":[{"interface":"back","average":{"1min":4.4450000000000003,"5min":1.3959999999999999,"15min":1.0289999999999999},"min":{"1min":0.28100000000000003,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":69.421999999999997,"5min":69.421999999999997,"15min":69.421999999999997},"last":0.44900000000000001},{"interface":"front","average":{"1min":4.431,"5min":1.712,"15min":1.27},"min":{"1min":0.33900000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":69.355000000000004,"5min":69.355000000000004,"15min":69.355000000000004},"last":0.61799999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:58:42 2023","interfaces":[{"interface":"back","average":{"1min":2.0409999999999999,"5min":2.4510000000000001,"15min":1.335},"min":{"1min":0.40999999999999998,"5min":0.33000000000000002,"15min":0.28199999999999997},"max":{"1min":19.338000000000001,"5min":142.994,"15min":142.994},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.262,"5min":2.5619999999999998,"15min":1.3300000000000001},"min":{"1min":0.32300000000000001,"5min":0.312,"15min":0.311},"max":{"1min":4.9109999999999996,"5min":143.411,"15min":143.411},"last":0.97999999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:58:42 2023","interfaces":[{"interface":"back","average":{"1min":1.9019999999999999,"5min":1.2789999999999999,"15min":0.90800000000000003},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.33800000000000002},"max":{"1min":11.488,"5min":14.404,"15min":14.404},"last":1.0409999999999999},{"interface":"front","average":{"1min":1.359,"5min":1.0529999999999999,"15min":0.83999999999999997},"min":{"1min":0.441,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.375,"5min":18.013999999999999,"15min":18.013999999999999},"last":0.53100000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:58:42 2023","interfaces":[{"interface":"back","average":{"1min":1.1739999999999999,"5min":1.635,"15min":1.077},"min":{"1min":0.376,"5min":0.376,"15min":0.30499999999999999},"max":{"1min":5.2039999999999997,"5min":44.079000000000001,"15min":44.079000000000001},"last":1.0589999999999999},{"interface":"front","average":{"1min":1.4650000000000001,"5min":1.4470000000000001,"15min":1.044},"min":{"1min":0.42399999999999999,"5min":0.318,"15min":0.318},"max":{"1min":5.4450000000000003,"5min":44.094000000000001,"15min":44.094000000000001},"last":0.56699999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:58:42 2023","interfaces":[{"interface":"back","average":{"1min":1.508,"5min":1.1419999999999999,"15min":0.91600000000000004},"min":{"1min":0.47399999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.4199999999999999,"5min":10.295,"15min":10.295},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.661,"5min":1.0840000000000001,"15min":0.91300000000000003},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":16.306999999999999,"5min":16.306999999999999,"15min":16.306999999999999},"last":0.65400000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019139,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":671644,"kb_used_data":362068,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93073508,"statfs":{"total":95995035648,"available":95307272192,"internally_reserved":0,"allocated":370757632,"data_stored":369727872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:08 2023","interfaces":[{"interface":"back","average":{"1min":1.3240000000000001,"5min":1.5429999999999999,"15min":1.0349999999999999},"min":{"1min":0.23300000000000001,"5min":0.222,"15min":0.222},"max":{"1min":7.6200000000000001,"5min":19.489000000000001,"15min":19.489000000000001},"last":0.96599999999999997},{"interface":"front","average":{"1min":2.2909999999999999,"5min":1.754,"15min":1.607},"min":{"1min":0.248,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":8.9800000000000004,"5min":19.757000000000001,"15min":19.757000000000001},"last":0.23400000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:58:33 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":1.512,"15min":1.0349999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.6120000000000001,"5min":77.951999999999998,"15min":77.951999999999998},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.61199999999999999,"5min":1.6970000000000001,"15min":1.0329999999999999},"min":{"1min":0.34499999999999997,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":1.6870000000000001,"5min":77.968000000000004,"15min":77.968000000000004},"last":0.51100000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:58:08 2023","interfaces":[{"interface":"back","average":{"1min":7.7610000000000001,"5min":2.3260000000000001,"15min":1.391},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":136.15700000000001,"5min":136.15700000000001,"15min":136.15700000000001},"last":1.014},{"interface":"front","average":{"1min":7.968,"5min":2.5059999999999998,"15min":1.498},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":136.30500000000001,"5min":136.30500000000001,"15min":136.30500000000001},"last":0.59999999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:58:33 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":1.2789999999999999,"15min":1.1120000000000001},"min":{"1min":0.35999999999999999,"5min":0.30599999999999999,"15min":0.29299999999999998},"max":{"1min":1.72,"5min":12.554,"15min":40.531999999999996},"last":0.33200000000000002},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.3160000000000001,"15min":1.1579999999999999},"min":{"1min":0.36599999999999999,"5min":0.31,"15min":0.31},"max":{"1min":3.4729999999999999,"5min":12.403,"15min":39.905999999999999},"last":0.76000000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:58:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":0.77200000000000002,"15min":0.69199999999999995},"min":{"1min":0.28299999999999997,"5min":0.27500000000000002,"15min":0.25900000000000001},"max":{"1min":4.0780000000000003,"5min":4.9349999999999996,"15min":10.012},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.94799999999999995,"5min":0.77200000000000002,"15min":0.76800000000000002},"min":{"1min":0.29599999999999999,"5min":0.254,"15min":0.254},"max":{"1min":5.3319999999999999,"5min":12.121,"15min":12.489000000000001},"last":0.47399999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:57:45 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":0.69999999999999996,"15min":0.71999999999999997},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.4379999999999997,"5min":5.4379999999999997,"15min":11.234999999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":2.1230000000000002,"5min":0.85799999999999998,"15min":0.71499999999999997},"min":{"1min":0.39200000000000002,"5min":0.30599999999999999,"15min":0.30299999999999999},"max":{"1min":8.8610000000000007,"5min":8.8610000000000007,"15min":9.923},"last":0.41999999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:58:11 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.82299999999999995,"15min":0.85499999999999998},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.28599999999999998},"max":{"1min":2.5209999999999999,"5min":6.0570000000000004,"15min":10.606},"last":1.069},{"interface":"front","average":{"1min":0.98399999999999999,"5min":0.74399999999999999,"15min":0.747},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.145,"5min":5.524,"15min":8.3819999999999997},"last":0.48999999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757502,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":588304,"kb_used_data":278728,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93156848,"statfs":{"total":95995035648,"available":95392612352,"internally_reserved":0,"allocated":285417472,"data_stored":284396288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:10 2023","interfaces":[{"interface":"back","average":{"1min":2.8690000000000002,"5min":1.085,"15min":0.83299999999999996},"min":{"1min":0.26000000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":24.956,"5min":24.956,"15min":24.956},"last":0.39400000000000002},{"interface":"front","average":{"1min":3.2389999999999999,"5min":1.3149999999999999,"15min":0.96999999999999997},"min":{"1min":0.32100000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":24.975000000000001,"5min":24.975000000000001,"15min":24.975000000000001},"last":0.80000000000000004}]},{"osd":1,"last update":"Mon Oct 9 05:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":0.97599999999999998,"15min":1.6679999999999999},"min":{"1min":0.32000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.0220000000000002,"5min":6.8410000000000002,"15min":114.247},"last":0.88},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.64800000000000002,"15min":1.528},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.9140000000000001,"5min":2.9140000000000001,"15min":114.678},"last":0.66900000000000004}]},{"osd":2,"last update":"Mon Oct 9 05:57:59 2023","interfaces":[{"interface":"back","average":{"1min":12.375999999999999,"5min":3.2130000000000001,"15min":2.0150000000000001},"min":{"1min":0.28499999999999998,"5min":0.28399999999999997,"15min":0.25},"max":{"1min":217.06399999999999,"5min":217.06399999999999,"15min":217.06399999999999},"last":0.93999999999999995},{"interface":"front","average":{"1min":11.965999999999999,"5min":3.0870000000000002,"15min":1.9710000000000001},"min":{"1min":0.308,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":217.018,"5min":217.018,"15min":217.018},"last":0.73299999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:58:35 2023","interfaces":[{"interface":"back","average":{"1min":10.228,"5min":4.6369999999999996,"15min":4.3600000000000003},"min":{"1min":0.29799999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":211.40199999999999,"5min":211.40199999999999,"15min":272.81299999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":10.225,"5min":4.9059999999999997,"15min":4.5919999999999996},"min":{"1min":0.36099999999999999,"5min":0.317,"15min":0.317},"max":{"1min":211.578,"5min":211.578,"15min":273.00999999999999},"last":0.91100000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:58:25 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.27400000000000002,"5min":0.26400000000000001,"15min":0.24199999999999999},"max":{"1min":2.8620000000000001,"5min":3.1080000000000001,"15min":6.2050000000000001},"last":0.76400000000000001},{"interface":"front","average":{"1min":0.73699999999999999,"5min":0.63,"15min":0.61599999999999999},"min":{"1min":0.313,"5min":0.27600000000000002,"15min":0.26000000000000001},"max":{"1min":3.879,"5min":3.879,"15min":13.050000000000001},"last":0.68300000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:58:38 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":0.83799999999999997,"15min":0.68999999999999995},"min":{"1min":0.34000000000000002,"5min":0.30199999999999999,"15min":0.26600000000000001},"max":{"1min":5.5750000000000002,"5min":10.94,"15min":10.94},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.79200000000000004,"15min":0.74099999999999999},"min":{"1min":0.36599999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.3399999999999999,"5min":5.5069999999999997,"15min":5.9770000000000003},"last":0.84899999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:58:16 2023","interfaces":[{"interface":"back","average":{"1min":1.204,"5min":0.871,"15min":0.82999999999999996},"min":{"1min":0.39500000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":4.1970000000000001,"5min":5.5800000000000001,"15min":9.2739999999999991},"last":1.2450000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.68000000000000005,"15min":0.65900000000000003},"min":{"1min":0.34499999999999997,"5min":0.32100000000000001,"15min":0.32000000000000001},"max":{"1min":1.762,"5min":2.7530000000000001,"15min":5.032},"last":0.83999999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1474,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":555220992,"data_stored":555220992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":744620032,"data_stored":744620032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":692649984,"data_stored":692649984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":345374720,"data_stored":345374720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":273416192,"data_stored":273416192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":606732288,"data_stored":606732288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":526254080,"data_stored":526254080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:58:51.405 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T05:58:51.405 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:58:51.406 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T05:58:51.406 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.170685 2023-10-09T05:58:51.406 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 0.1706845453979685 2023-10-09T05:58:51.448 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:51.521 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:51.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:51.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:51.536 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:51.625 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:51.654 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:51.756 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:51.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:51.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:51.862 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:51.907 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:51.970 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 13 16 30237 30221 145.272 88.6875 0.00766417 0.00687819 2023-10-09T05:58:52.060 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:52.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:52.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:52.171 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:52.197 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:58:52.238 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:58:52.258 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:52.266 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:52 smithi156 ceph-mon[195101]: pgmap v659: 109 pgs: 109 active+clean; 1.6 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 151 MiB/s wr, 2.42k op/s 2023-10-09T05:58:52.289 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:58:52.331 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:58:52.368 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:58:52.405 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:58:52.409 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:52.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:52.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:52 smithi064 ceph-mon[212750]: pgmap v659: 109 pgs: 109 active+clean; 1.6 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 151 MiB/s wr, 2.42k op/s 2023-10-09T05:58:52.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:52 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1686111896' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:52.418 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:52 smithi064 ceph-mon[220044]: pgmap v659: 109 pgs: 109 active+clean; 1.6 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 151 MiB/s wr, 2.42k op/s 2023-10-09T05:58:52.418 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:52 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1686111896' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:52.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:52.441 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:58:52.486 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:58:52.492 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:58:52.492 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:58:52.492 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:58:52.492 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:58:52.492 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:58:52.492 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:58:52.492 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:58:52.492 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:58:52.520 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:58:52.577 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:52.615 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:58:52.615 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T05:58:52.615 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T05:58:52.615 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T05:58:52.615 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:58:52.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:52 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1686111896' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:58:52.676 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:58:52.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:52.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:58:52.785 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:52.847 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:52.967 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 14 16 31914 31898 142.381 104.812 0.0138821 0.00701832 2023-10-09T05:58:53.656 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T05:58:53.967 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 15 16 33801 33785 140.751 117.938 0.00736243 0.00709947 2023-10-09T05:58:54.163 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:54.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:54.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:54.269 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:54.297 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:58:54.329 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:54.435 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:54.435 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:54 smithi064 ceph-mon[220044]: pgmap v660: 109 pgs: 109 active+clean; 1.7 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 149 MiB/s wr, 2.38k op/s 2023-10-09T05:58:54.435 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:54 smithi064 ceph-mon[212750]: pgmap v660: 109 pgs: 109 active+clean; 1.7 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 149 MiB/s wr, 2.38k op/s 2023-10-09T05:58:54.550 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:54 smithi156 ceph-mon[195101]: pgmap v660: 109 pgs: 109 active+clean; 1.7 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 149 MiB/s wr, 2.38k op/s 2023-10-09T05:58:54.617 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:54.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:54.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:54.726 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:58:54.784 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:54.967 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 16 16 35504 35488 138.607 106.438 0.00628155 0.00720983 2023-10-09T05:58:55.007 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:58:55.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:55.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:58:55.113 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:55.164 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:55.234 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:55.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:55.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:55.337 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:55.376 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:55.424 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:55.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:55.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:55.527 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:55.567 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:55.721 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:58:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:58:55] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.43.0" 2023-10-09T05:58:55.791 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:55.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:55.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:55.895 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:55.933 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:55.969 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 17 16 36483 36467 134.038 61.1875 0.0123034 0.0074551 2023-10-09T05:58:56.221 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:56.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:56.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:56.316 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T05:58:56.328 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:56.377 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:56.457 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:58:56.457 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T05:58:56.457 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T05:58:56.457 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T05:58:56.457 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.457 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:91015 6.e 6:7dc6a610:::benchmark_data_smithi064_284193_object91014:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-10-09T05:58:56.457 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:10.210536+0000", 2023-10-09T05:58:56.458 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 406.240541646, 2023-10-09T05:58:56.458 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3031288990000001, 2023-10-09T05:58:56.458 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:58:56.458 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:56.458 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:58:56.458 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:58:56.458 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:56.458 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 91015 2023-10-09T05:58:56.458 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.458 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:58:56.458 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.459 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:58:56.459 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210536+0000", 2023-10-09T05:58:56.459 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.459 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.459 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.459 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:58:56.459 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210536+0000", 2023-10-09T05:58:56.459 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.459 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.459 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.459 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:58:56.459 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210534+0000", 2023-10-09T05:58:56.460 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:58:56.460 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.460 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.460 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:58:56.460 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210591+0000", 2023-10-09T05:58:56.460 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6641999999999997e-05 2023-10-09T05:58:56.460 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.460 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.460 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:58:56.460 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210601+0000", 2023-10-09T05:58:56.460 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0485999999999999e-05 2023-10-09T05:58:56.460 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.461 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.461 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:58:56.461 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210606+0000", 2023-10-09T05:58:56.461 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0320000000000003e-06 2023-10-09T05:58:56.461 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.461 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.461 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.461 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210705+0000", 2023-10-09T05:58:56.461 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.8412000000000006e-05 2023-10-09T05:58:56.461 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.461 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.461 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.462 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210714+0000", 2023-10-09T05:58:56.462 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.6160000000000008e-06 2023-10-09T05:58:56.462 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.462 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.462 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.462 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.497372+0000", 2023-10-09T05:58:56.462 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2866577770000001 2023-10-09T05:58:56.462 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.462 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.462 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:58:56.462 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.497531+0000", 2023-10-09T05:58:56.463 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015918599999999999 2023-10-09T05:58:56.463 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.463 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.463 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T05:58:56.463 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.497704+0000", 2023-10-09T05:58:56.463 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000172103 2023-10-09T05:58:56.463 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.463 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.463 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.463 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.502321+0000", 2023-10-09T05:58:56.463 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0046177689999999999 2023-10-09T05:58:56.463 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.464 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.464 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:58:56.464 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.502336+0000", 2023-10-09T05:58:56.464 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4392999999999999e-05 2023-10-09T05:58:56.464 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.464 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.464 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.464 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.511914+0000", 2023-10-09T05:58:56.464 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0095783229999999997 2023-10-09T05:58:56.464 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.464 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.464 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:58:56.465 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.511973+0000", 2023-10-09T05:58:56.465 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8718999999999998e-05 2023-10-09T05:58:56.465 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.465 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.465 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:58:56.465 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.513665+0000", 2023-10-09T05:58:56.465 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001691992 2023-10-09T05:58:56.465 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.465 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:58:56.465 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.465 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.465 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.465 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:110019 6.1e 6:7b30049e:::benchmark_data_smithi064_284193_object13476:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2023-10-09T05:58:56.466 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:22.411566+0000", 2023-10-09T05:58:56.466 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 214.03951152100001, 2023-10-09T05:58:56.466 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7746888279999999, 2023-10-09T05:58:56.466 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:58:56.466 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:56.466 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:58:56.466 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:58:56.466 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:56.466 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 110019 2023-10-09T05:58:56.466 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.466 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:58:56.467 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.467 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:58:56.467 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.411566+0000", 2023-10-09T05:58:56.467 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.467 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.467 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.467 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:58:56.467 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.411566+0000", 2023-10-09T05:58:56.467 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.467 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.467 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.467 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:58:56.468 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.411565+0000", 2023-10-09T05:58:56.468 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:58:56.468 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.468 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.468 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:58:56.468 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.411571+0000", 2023-10-09T05:58:56.468 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0850000000000002e-06 2023-10-09T05:58:56.468 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.468 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.468 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:58:56.468 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.411585+0000", 2023-10-09T05:58:56.468 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4361e-05 2023-10-09T05:58:56.469 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.469 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.469 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:58:56.469 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.411593+0000", 2023-10-09T05:58:56.469 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2180000000000002e-06 2023-10-09T05:58:56.469 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.469 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.469 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.469 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.411639+0000", 2023-10-09T05:58:56.469 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6915000000000001e-05 2023-10-09T05:58:56.469 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.469 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.469 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.470 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.411647+0000", 2023-10-09T05:58:56.470 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8590000000000005e-06 2023-10-09T05:58:56.470 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.470 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.470 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.470 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.170887+0000", 2023-10-09T05:58:56.470 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.75923964700000002 2023-10-09T05:58:56.470 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.470 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.470 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.470 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.170919+0000", 2023-10-09T05:58:56.471 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.2492e-05 2023-10-09T05:58:56.471 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.471 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.471 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.471 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.182074+0000", 2023-10-09T05:58:56.471 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.011155026 2023-10-09T05:58:56.471 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.471 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.471 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:58:56.471 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.182219+0000", 2023-10-09T05:58:56.471 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014427600000000001 2023-10-09T05:58:56.472 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.472 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.472 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T05:58:56.472 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.182389+0000", 2023-10-09T05:58:56.472 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000170509 2023-10-09T05:58:56.472 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.472 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.472 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:58:56.472 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.184044+0000", 2023-10-09T05:58:56.472 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0016549080000000001 2023-10-09T05:58:56.472 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.473 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.473 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.473 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.184912+0000", 2023-10-09T05:58:56.473 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00086812100000000002 2023-10-09T05:58:56.473 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.473 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.473 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.473 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.185028+0000", 2023-10-09T05:58:56.473 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000115294 2023-10-09T05:58:56.473 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.473 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.474 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.474 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.186080+0000", 2023-10-09T05:58:56.474 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0010523290000000001 2023-10-09T05:58:56.474 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.474 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.474 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:58:56.474 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.186132+0000", 2023-10-09T05:58:56.474 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2432999999999998e-05 2023-10-09T05:58:56.474 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.474 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.474 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:58:56.474 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.186255+0000", 2023-10-09T05:58:56.475 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012244299999999999 2023-10-09T05:58:56.475 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.475 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:58:56.475 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.475 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.475 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.475 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:110021 6.1e 6:7cb142d9:::benchmark_data_smithi064_284193_object13478:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2023-10-09T05:58:56.475 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:22.456698+0000", 2023-10-09T05:58:56.475 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 213.994379506, 2023-10-09T05:58:56.475 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.72968491, 2023-10-09T05:58:56.475 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:58:56.475 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:56.476 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:58:56.476 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:58:56.476 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:56.476 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 110021 2023-10-09T05:58:56.476 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.476 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:58:56.476 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.476 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:58:56.476 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.456698+0000", 2023-10-09T05:58:56.476 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.476 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.476 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.477 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:58:56.477 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.456698+0000", 2023-10-09T05:58:56.477 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.477 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.477 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.477 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:58:56.477 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.456696+0000", 2023-10-09T05:58:56.477 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T05:58:56.477 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.477 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.477 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:58:56.477 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.456708+0000", 2023-10-09T05:58:56.478 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1503e-05 2023-10-09T05:58:56.478 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.478 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.478 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:58:56.478 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.456741+0000", 2023-10-09T05:58:56.478 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.2963000000000003e-05 2023-10-09T05:58:56.478 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.478 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.478 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:58:56.478 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.456758+0000", 2023-10-09T05:58:56.478 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.753e-05 2023-10-09T05:58:56.479 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.479 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.479 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.479 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.456875+0000", 2023-10-09T05:58:56.479 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011678799999999999 2023-10-09T05:58:56.479 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.479 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.479 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.479 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.456892+0000", 2023-10-09T05:58:56.479 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6663e-05 2023-10-09T05:58:56.479 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.479 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.479 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.480 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.170942+0000", 2023-10-09T05:58:56.480 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.71405068599999999 2023-10-09T05:58:56.480 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.480 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.480 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.480 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.170949+0000", 2023-10-09T05:58:56.480 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5050000000000004e-06 2023-10-09T05:58:56.480 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.480 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.480 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.480 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.183286+0000", 2023-10-09T05:58:56.480 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0123373369999999 2023-10-09T05:58:56.481 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.481 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.481 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:58:56.481 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.183352+0000", 2023-10-09T05:58:56.481 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6153000000000002e-05 2023-10-09T05:58:56.481 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.481 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.481 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T05:58:56.481 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.183455+0000", 2023-10-09T05:58:56.481 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000102785 2023-10-09T05:58:56.481 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.481 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.482 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:58:56.482 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.184924+0000", 2023-10-09T05:58:56.482 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001468802 2023-10-09T05:58:56.482 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.482 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.482 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.482 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.185074+0000", 2023-10-09T05:58:56.482 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014996399999999999 2023-10-09T05:58:56.482 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.482 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.482 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.483 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.185109+0000", 2023-10-09T05:58:56.483 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5689999999999999e-05 2023-10-09T05:58:56.483 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.483 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.483 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.483 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.186288+0000", 2023-10-09T05:58:56.483 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0011783119999999999 2023-10-09T05:58:56.483 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.483 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.483 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:58:56.483 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.186318+0000", 2023-10-09T05:58:56.483 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0375999999999999e-05 2023-10-09T05:58:56.483 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.484 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.484 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:58:56.484 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.186383+0000", 2023-10-09T05:58:56.484 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4735999999999998e-05 2023-10-09T05:58:56.484 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.484 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:58:56.484 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.484 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.484 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.484 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:110047 6.1f 6:fd8aac31:::benchmark_data_smithi064_284193_object13504:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2023-10-09T05:58:56.484 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:23.278484+0000", 2023-10-09T05:58:56.484 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 213.17259311699999, 2023-10-09T05:58:56.485 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.023028037, 2023-10-09T05:58:56.485 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:58:56.485 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:56.485 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:58:56.485 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:58:56.485 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:56.485 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 110047 2023-10-09T05:58:56.485 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.485 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:58:56.485 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.485 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:58:56.485 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.278484+0000", 2023-10-09T05:58:56.486 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.486 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.486 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.486 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:58:56.486 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.278484+0000", 2023-10-09T05:58:56.486 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.486 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.486 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.486 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:58:56.486 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.278483+0000", 2023-10-09T05:58:56.486 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:58:56.487 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.487 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.487 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:58:56.487 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.278488+0000", 2023-10-09T05:58:56.487 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8860000000000003e-06 2023-10-09T05:58:56.487 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.487 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.487 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:58:56.487 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.278496+0000", 2023-10-09T05:58:56.487 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1529999999999997e-06 2023-10-09T05:58:56.487 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.487 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.488 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:58:56.488 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.278501+0000", 2023-10-09T05:58:56.488 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7400000000000004e-06 2023-10-09T05:58:56.488 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.488 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.488 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.488 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.278538+0000", 2023-10-09T05:58:56.488 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7184999999999997e-05 2023-10-09T05:58:56.488 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.488 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.488 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.488 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.278546+0000", 2023-10-09T05:58:56.489 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9980000000000003e-06 2023-10-09T05:58:56.489 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.489 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.489 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.489 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.291647+0000", 2023-10-09T05:58:56.489 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.013100771 2023-10-09T05:58:56.489 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.489 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.489 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:58:56.489 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.291798+0000", 2023-10-09T05:58:56.489 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015106200000000001 2023-10-09T05:58:56.489 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.490 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.490 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,5,6", 2023-10-09T05:58:56.490 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.292129+0000", 2023-10-09T05:58:56.490 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00033066399999999999 2023-10-09T05:58:56.490 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.490 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.490 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.490 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.297460+0000", 2023-10-09T05:58:56.490 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0053315569999999998 2023-10-09T05:58:56.490 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.490 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.491 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.491 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.297497+0000", 2023-10-09T05:58:56.491 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.625e-05 2023-10-09T05:58:56.491 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.491 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.491 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.491 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.297534+0000", 2023-10-09T05:58:56.491 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6977000000000002e-05 2023-10-09T05:58:56.491 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.491 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.491 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:58:56.491 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.299568+0000", 2023-10-09T05:58:56.492 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0020343230000000002 2023-10-09T05:58:56.492 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.492 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.492 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:58:56.492 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.299618+0000", 2023-10-09T05:58:56.492 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0380999999999998e-05 2023-10-09T05:58:56.492 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.492 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.492 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:58:56.492 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.301512+0000", 2023-10-09T05:58:56.492 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0018940680000000001 2023-10-09T05:58:56.492 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.493 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:58:56.493 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.493 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.493 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.493 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:110067 6.1f 6:f8d8a82d:::benchmark_data_smithi064_284193_object13524:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2023-10-09T05:58:56.493 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:23.316013+0000", 2023-10-09T05:58:56.493 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 213.135064591, 2023-10-09T05:58:56.493 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99153563, 2023-10-09T05:58:56.493 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:58:56.493 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:56.493 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:58:56.493 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:58:56.494 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:56.494 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 110067 2023-10-09T05:58:56.494 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.494 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:58:56.494 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.494 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:58:56.494 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.316013+0000", 2023-10-09T05:58:56.494 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.494 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.494 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.494 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:58:56.494 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.316013+0000", 2023-10-09T05:58:56.495 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.495 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.495 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.495 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:58:56.495 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.316012+0000", 2023-10-09T05:58:56.495 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:56.495 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.495 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.495 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:58:56.495 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.316016+0000", 2023-10-09T05:58:56.495 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0419999999999996e-06 2023-10-09T05:58:56.495 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.496 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.496 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:58:56.496 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.316037+0000", 2023-10-09T05:58:56.496 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0730999999999999e-05 2023-10-09T05:58:56.496 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.496 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.496 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:58:56.496 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.316042+0000", 2023-10-09T05:58:56.496 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8879999999999996e-06 2023-10-09T05:58:56.496 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.496 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.496 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.497 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.316079+0000", 2023-10-09T05:58:56.497 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7410999999999998e-05 2023-10-09T05:58:56.497 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.497 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.497 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.497 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.316087+0000", 2023-10-09T05:58:56.497 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5020000000000003e-06 2023-10-09T05:58:56.497 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.497 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.497 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.497 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.295918+0000", 2023-10-09T05:58:56.497 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.97983070100000003 2023-10-09T05:58:56.498 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.498 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.498 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:58:56.498 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.296067+0000", 2023-10-09T05:58:56.498 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014895500000000001 2023-10-09T05:58:56.498 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.498 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.498 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,5,6", 2023-10-09T05:58:56.498 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.296317+0000", 2023-10-09T05:58:56.498 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00025033100000000001 2023-10-09T05:58:56.498 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.498 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.499 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.499 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.299559+0000", 2023-10-09T05:58:56.499 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0032425230000000002 2023-10-09T05:58:56.499 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.499 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.499 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.499 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.301585+0000", 2023-10-09T05:58:56.499 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0020257320000000001 2023-10-09T05:58:56.499 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.499 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.499 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:58:56.499 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.301593+0000", 2023-10-09T05:58:56.500 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6179999999999997e-06 2023-10-09T05:58:56.500 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.500 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.500 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.500 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.305356+0000", 2023-10-09T05:58:56.500 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0037634410000000002 2023-10-09T05:58:56.500 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.500 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.500 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:58:56.500 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.305399+0000", 2023-10-09T05:58:56.500 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3151000000000001e-05 2023-10-09T05:58:56.500 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.501 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.501 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:58:56.501 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.307549+0000", 2023-10-09T05:58:56.501 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002149186 2023-10-09T05:58:56.501 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.501 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:58:56.501 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.501 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.501 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.501 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:119187 6.1e 6:7c546603:::benchmark_data_smithi064_284193_object22644:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-10-09T05:58:56.501 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:43.711700+0000", 2023-10-09T05:58:56.501 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 192.739377249, 2023-10-09T05:58:56.502 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.010368224, 2023-10-09T05:58:56.502 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:58:56.502 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:56.502 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:58:56.502 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:58:56.502 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:56.502 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 119187 2023-10-09T05:58:56.502 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.502 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:58:56.502 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.502 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:58:56.502 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.711700+0000", 2023-10-09T05:58:56.503 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.503 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.503 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.503 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:58:56.503 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.711700+0000", 2023-10-09T05:58:56.503 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.503 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.503 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.503 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:58:56.503 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.711699+0000", 2023-10-09T05:58:56.503 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:58:56.503 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.504 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.504 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:58:56.504 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.711705+0000", 2023-10-09T05:58:56.504 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7089999999999999e-06 2023-10-09T05:58:56.504 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.504 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.504 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:58:56.504 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.711715+0000", 2023-10-09T05:58:56.504 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0485e-05 2023-10-09T05:58:56.504 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.504 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.504 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:58:56.504 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.711720+0000", 2023-10-09T05:58:56.505 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.403e-06 2023-10-09T05:58:56.505 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.505 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.505 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.505 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.711796+0000", 2023-10-09T05:58:56.505 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5733000000000001e-05 2023-10-09T05:58:56.505 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.505 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.505 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.505 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.711802+0000", 2023-10-09T05:58:56.505 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7089999999999996e-06 2023-10-09T05:58:56.506 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.506 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.506 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.506 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.674452+0000", 2023-10-09T05:58:56.506 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96265013799999999 2023-10-09T05:58:56.506 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.506 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.506 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.506 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.674461+0000", 2023-10-09T05:58:56.506 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8769999999999999e-06 2023-10-09T05:58:56.506 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.506 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.507 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.507 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.707191+0000", 2023-10-09T05:58:56.507 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0327298499999999 2023-10-09T05:58:56.507 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.507 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.507 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:58:56.507 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.707308+0000", 2023-10-09T05:58:56.507 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000116965 2023-10-09T05:58:56.507 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.507 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.507 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:58:56.507 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.707543+0000", 2023-10-09T05:58:56.508 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00023478600000000001 2023-10-09T05:58:56.508 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.508 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.508 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:58:56.508 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.712510+0000", 2023-10-09T05:58:56.508 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0049671439999999997 2023-10-09T05:58:56.508 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.508 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.508 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.508 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.716010+0000", 2023-10-09T05:58:56.508 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0034996469999999998 2023-10-09T05:58:56.508 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.508 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.509 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.509 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.721949+0000", 2023-10-09T05:58:56.509 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0059391629999999999 2023-10-09T05:58:56.509 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.509 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.509 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:58:56.509 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.722008+0000", 2023-10-09T05:58:56.509 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8845000000000003e-05 2023-10-09T05:58:56.509 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.509 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.509 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:58:56.509 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.722068+0000", 2023-10-09T05:58:56.510 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0822999999999997e-05 2023-10-09T05:58:56.510 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.510 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:58:56.510 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.510 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.510 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.510 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:119189 6.f 6:f56825aa:::benchmark_data_smithi064_284193_object22646:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-10-09T05:58:56.510 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:43.768575+0000", 2023-10-09T05:58:56.510 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 192.68250225599999, 2023-10-09T05:58:56.510 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1074308939999999, 2023-10-09T05:58:56.510 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:58:56.511 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:56.511 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:58:56.511 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:58:56.511 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:56.511 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 119189 2023-10-09T05:58:56.511 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.511 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:58:56.511 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.511 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:58:56.511 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.768575+0000", 2023-10-09T05:58:56.511 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.511 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.512 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.512 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:58:56.512 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.768575+0000", 2023-10-09T05:58:56.512 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.512 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.512 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.512 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:58:56.512 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.768575+0000", 2023-10-09T05:58:56.512 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:56.512 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.512 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.512 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:58:56.513 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.768578+0000", 2023-10-09T05:58:56.513 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.596e-06 2023-10-09T05:58:56.513 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.513 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.513 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:58:56.513 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.768589+0000", 2023-10-09T05:58:56.513 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0692e-05 2023-10-09T05:58:56.513 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.513 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.513 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:58:56.513 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.768593+0000", 2023-10-09T05:58:56.513 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3379999999999998e-06 2023-10-09T05:58:56.514 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.514 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.514 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.514 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.768648+0000", 2023-10-09T05:58:56.514 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4255000000000001e-05 2023-10-09T05:58:56.514 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.514 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.514 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.514 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.768653+0000", 2023-10-09T05:58:56.514 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6570000000000002e-06 2023-10-09T05:58:56.514 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.514 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.515 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.515 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.868318+0000", 2023-10-09T05:58:56.515 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0996647669999999 2023-10-09T05:58:56.515 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.515 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.515 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:58:56.515 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.868426+0000", 2023-10-09T05:58:56.515 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000108053 2023-10-09T05:58:56.515 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.515 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.515 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T05:58:56.516 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.868605+0000", 2023-10-09T05:58:56.516 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00017891700000000001 2023-10-09T05:58:56.516 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.516 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.516 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:58:56.516 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.870617+0000", 2023-10-09T05:58:56.516 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002012401 2023-10-09T05:58:56.516 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.516 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.516 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.516 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.871235+0000", 2023-10-09T05:58:56.516 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00061724299999999996 2023-10-09T05:58:56.516 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.517 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.517 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.517 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.871577+0000", 2023-10-09T05:58:56.517 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00034259999999999998 2023-10-09T05:58:56.517 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.517 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.517 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.517 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.874425+0000", 2023-10-09T05:58:56.517 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0028476040000000001 2023-10-09T05:58:56.517 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.517 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.517 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:58:56.518 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.874484+0000", 2023-10-09T05:58:56.518 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9565999999999999e-05 2023-10-09T05:58:56.518 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.518 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.518 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:58:56.518 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.876006+0000", 2023-10-09T05:58:56.518 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0015218110000000001 2023-10-09T05:58:56.518 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.518 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:58:56.518 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.518 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.519 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.519 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:119207 6.e 6:72b0e584:::benchmark_data_smithi064_284193_object22664:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-10-09T05:58:56.519 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:44.039853+0000", 2023-10-09T05:58:56.519 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 192.41122431100001, 2023-10-09T05:58:56.519 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.710139471, 2023-10-09T05:58:56.519 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:58:56.519 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:56.519 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:58:56.519 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:58:56.519 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:56.519 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 119207 2023-10-09T05:58:56.519 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.520 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:58:56.520 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.520 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:58:56.520 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.039853+0000", 2023-10-09T05:58:56.520 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.520 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.520 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.520 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:58:56.520 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.039853+0000", 2023-10-09T05:58:56.520 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.520 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.520 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.520 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:58:56.521 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.039852+0000", 2023-10-09T05:58:56.521 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:58:56.521 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.521 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.521 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:58:56.521 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.039858+0000", 2023-10-09T05:58:56.521 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4859999999999997e-06 2023-10-09T05:58:56.521 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.521 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.521 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:58:56.521 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.039871+0000", 2023-10-09T05:58:56.522 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3217999999999999e-05 2023-10-09T05:58:56.522 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.522 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.522 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:58:56.522 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.039877+0000", 2023-10-09T05:58:56.522 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7950000000000004e-06 2023-10-09T05:58:56.522 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.522 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.522 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.522 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.039923+0000", 2023-10-09T05:58:56.522 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5973999999999997e-05 2023-10-09T05:58:56.523 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.523 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.523 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.523 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.039932+0000", 2023-10-09T05:58:56.523 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9910000000000008e-06 2023-10-09T05:58:56.523 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.523 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.523 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.523 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.674566+0000", 2023-10-09T05:58:56.523 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.63463462199999998 2023-10-09T05:58:56.523 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.523 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.524 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.524 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.674573+0000", 2023-10-09T05:58:56.524 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7800000000000003e-06 2023-10-09T05:58:56.524 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.524 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.524 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.524 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.711205+0000", 2023-10-09T05:58:56.524 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0366318240000001 2023-10-09T05:58:56.524 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.524 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.524 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:58:56.524 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.711325+0000", 2023-10-09T05:58:56.524 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012053900000000001 2023-10-09T05:58:56.525 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.525 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.525 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:58:56.525 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.711613+0000", 2023-10-09T05:58:56.525 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00028770999999999999 2023-10-09T05:58:56.525 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.525 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.525 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:58:56.525 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.712912+0000", 2023-10-09T05:58:56.525 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0012985970000000001 2023-10-09T05:58:56.525 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.525 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.526 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.526 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.716093+0000", 2023-10-09T05:58:56.526 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0031810470000000002 2023-10-09T05:58:56.526 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.526 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.526 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.526 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.749899+0000", 2023-10-09T05:58:56.526 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.033806514000000003 2023-10-09T05:58:56.526 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.526 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.526 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:58:56.527 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.749953+0000", 2023-10-09T05:58:56.527 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3845999999999999e-05 2023-10-09T05:58:56.527 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.527 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.527 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:58:56.527 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.749993+0000", 2023-10-09T05:58:56.527 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9647000000000002e-05 2023-10-09T05:58:56.527 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.527 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:58:56.527 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.527 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.527 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.528 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:119214 6.1e 6:7d4d0cb9:::benchmark_data_smithi064_284193_object22671:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-10-09T05:58:56.528 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:44.133647+0000", 2023-10-09T05:58:56.528 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 192.31743022000001, 2023-10-09T05:58:56.528 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5885200099999999, 2023-10-09T05:58:56.528 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:58:56.528 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:56.528 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:58:56.528 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:58:56.528 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:56.528 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 119214 2023-10-09T05:58:56.528 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.528 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:58:56.529 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.529 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:58:56.529 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.133647+0000", 2023-10-09T05:58:56.529 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.529 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.529 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.529 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:58:56.529 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.133647+0000", 2023-10-09T05:58:56.529 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.529 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.529 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.529 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:58:56.529 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.133646+0000", 2023-10-09T05:58:56.530 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:58:56.530 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.530 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.530 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:58:56.530 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.133656+0000", 2023-10-09T05:58:56.530 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0013999999999999e-05 2023-10-09T05:58:56.530 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.530 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.530 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:58:56.530 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.133673+0000", 2023-10-09T05:58:56.530 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7379000000000001e-05 2023-10-09T05:58:56.531 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.531 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.531 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:58:56.531 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.133680+0000", 2023-10-09T05:58:56.531 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6699999999999997e-06 2023-10-09T05:58:56.531 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.531 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.531 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.531 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.133725+0000", 2023-10-09T05:58:56.531 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5188999999999997e-05 2023-10-09T05:58:56.531 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.531 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.532 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.532 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.133734+0000", 2023-10-09T05:58:56.532 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9379999999999998e-06 2023-10-09T05:58:56.532 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.532 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.532 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.532 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.674484+0000", 2023-10-09T05:58:56.532 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.54074985399999997 2023-10-09T05:58:56.532 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.532 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.532 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.532 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.674491+0000", 2023-10-09T05:58:56.533 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5529999999999998e-06 2023-10-09T05:58:56.533 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.533 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.533 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.533 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.708538+0000", 2023-10-09T05:58:56.533 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0340476890000001 2023-10-09T05:58:56.533 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.533 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.533 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:58:56.533 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.708662+0000", 2023-10-09T05:58:56.533 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012336700000000001 2023-10-09T05:58:56.533 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.534 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.534 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:58:56.534 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.708855+0000", 2023-10-09T05:58:56.534 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00019366300000000001 2023-10-09T05:58:56.534 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.534 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.534 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:58:56.534 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.712515+0000", 2023-10-09T05:58:56.534 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0036598580000000002 2023-10-09T05:58:56.534 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.534 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.534 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.535 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.716056+0000", 2023-10-09T05:58:56.535 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0035411169999999999 2023-10-09T05:58:56.535 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.535 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.535 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.535 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.722107+0000", 2023-10-09T05:58:56.535 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0060505769999999997 2023-10-09T05:58:56.535 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.535 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.535 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:58:56.535 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.722142+0000", 2023-10-09T05:58:56.536 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.4631000000000001e-05 2023-10-09T05:58:56.536 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.536 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.536 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:58:56.536 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.722167+0000", 2023-10-09T05:58:56.536 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.5778e-05 2023-10-09T05:58:56.536 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.536 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:58:56.536 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.536 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.536 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.536 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:119217 6.1e 6:7ea05517:::benchmark_data_smithi064_284193_object22674:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-10-09T05:58:56.537 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:44.159123+0000", 2023-10-09T05:58:56.537 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 192.29195456900001, 2023-10-09T05:58:56.537 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.563185697, 2023-10-09T05:58:56.537 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:58:56.537 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:56.537 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:58:56.537 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:58:56.537 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:56.537 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 119217 2023-10-09T05:58:56.537 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.537 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:58:56.537 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.538 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:58:56.538 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.159123+0000", 2023-10-09T05:58:56.538 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.538 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.538 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.538 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:58:56.538 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.159123+0000", 2023-10-09T05:58:56.538 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.538 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.538 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.538 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:58:56.539 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.159122+0000", 2023-10-09T05:58:56.539 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:58:56.539 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.539 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.539 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:58:56.539 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.159127+0000", 2023-10-09T05:58:56.539 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5890000000000002e-06 2023-10-09T05:58:56.539 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.539 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.539 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:58:56.539 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.159141+0000", 2023-10-09T05:58:56.539 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3817e-05 2023-10-09T05:58:56.540 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.540 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.540 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:58:56.540 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.159147+0000", 2023-10-09T05:58:56.540 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3559999999999998e-06 2023-10-09T05:58:56.540 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.540 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.540 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.540 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.159199+0000", 2023-10-09T05:58:56.540 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1994999999999999e-05 2023-10-09T05:58:56.540 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.540 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.541 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.541 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.159212+0000", 2023-10-09T05:58:56.541 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.274e-05 2023-10-09T05:58:56.541 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.541 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.541 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.541 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.674511+0000", 2023-10-09T05:58:56.541 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.51529928599999997 2023-10-09T05:58:56.541 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.541 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.541 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.541 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.674517+0000", 2023-10-09T05:58:56.542 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0000000000000002e-06 2023-10-09T05:58:56.542 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.542 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.542 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.542 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.709948+0000", 2023-10-09T05:58:56.542 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.035430275 2023-10-09T05:58:56.542 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.542 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.542 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:58:56.542 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.710076+0000", 2023-10-09T05:58:56.542 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000128572 2023-10-09T05:58:56.542 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.543 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.543 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:58:56.543 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.710257+0000", 2023-10-09T05:58:56.543 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000180853 2023-10-09T05:58:56.543 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.543 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.543 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:58:56.543 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.712521+0000", 2023-10-09T05:58:56.543 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0022643009999999998 2023-10-09T05:58:56.543 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.543 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.543 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.544 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.716103+0000", 2023-10-09T05:58:56.544 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0035815819999999998 2023-10-09T05:58:56.544 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.544 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.544 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.544 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.722210+0000", 2023-10-09T05:58:56.544 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0061075399999999998 2023-10-09T05:58:56.544 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.544 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.544 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:58:56.544 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.722264+0000", 2023-10-09T05:58:56.544 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3260000000000002e-05 2023-10-09T05:58:56.545 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.545 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.545 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:58:56.545 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.722309+0000", 2023-10-09T05:58:56.545 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4950999999999997e-05 2023-10-09T05:58:56.545 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.545 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:58:56.545 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.545 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.545 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.545 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:119228 6.1e 6:7e232574:::benchmark_data_smithi064_284193_object22685:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-10-09T05:58:56.545 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:44.298324+0000", 2023-10-09T05:58:56.546 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 192.152753434, 2023-10-09T05:58:56.546 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4516483600000001, 2023-10-09T05:58:56.546 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:58:56.546 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:56.546 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:58:56.546 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:58:56.546 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:56.546 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 119228 2023-10-09T05:58:56.546 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.546 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:58:56.546 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.546 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:58:56.547 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.298324+0000", 2023-10-09T05:58:56.547 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.547 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.547 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.547 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:58:56.547 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.298324+0000", 2023-10-09T05:58:56.547 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.547 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.547 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.547 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:58:56.547 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.298323+0000", 2023-10-09T05:58:56.547 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:58:56.548 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.548 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.548 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:58:56.548 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.298331+0000", 2023-10-09T05:58:56.548 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.3340000000000002e-06 2023-10-09T05:58:56.548 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.548 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.548 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:58:56.548 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.298346+0000", 2023-10-09T05:58:56.548 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4276e-05 2023-10-09T05:58:56.548 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.548 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.549 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:58:56.549 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.298352+0000", 2023-10-09T05:58:56.549 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1850000000000001e-06 2023-10-09T05:58:56.549 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.549 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.549 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.549 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.298398+0000", 2023-10-09T05:58:56.549 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5918000000000003e-05 2023-10-09T05:58:56.549 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.549 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.549 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.549 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.298406+0000", 2023-10-09T05:58:56.550 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.4370000000000007e-06 2023-10-09T05:58:56.550 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.550 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.550 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.550 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.674538+0000", 2023-10-09T05:58:56.550 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.37613181699999998 2023-10-09T05:58:56.550 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.550 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.550 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.550 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.674544+0000", 2023-10-09T05:58:56.550 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0889999999999996e-06 2023-10-09T05:58:56.550 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.551 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.551 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.551 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.711440+0000", 2023-10-09T05:58:56.551 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0368962559999999 2023-10-09T05:58:56.551 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.551 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.551 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:58:56.551 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.711559+0000", 2023-10-09T05:58:56.551 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011837600000000001 2023-10-09T05:58:56.551 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.551 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.551 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:58:56.552 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.711718+0000", 2023-10-09T05:58:56.552 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015891200000000001 2023-10-09T05:58:56.552 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.552 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.552 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:58:56.552 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.712613+0000", 2023-10-09T05:58:56.552 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00089554800000000003 2023-10-09T05:58:56.552 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.552 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.552 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.552 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.716132+0000", 2023-10-09T05:58:56.552 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0035189499999999999 2023-10-09T05:58:56.553 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.553 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.553 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.553 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.749904+0000", 2023-10-09T05:58:56.553 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.033771382000000003 2023-10-09T05:58:56.553 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.553 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.553 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:58:56.553 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.749944+0000", 2023-10-09T05:58:56.553 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0908999999999997e-05 2023-10-09T05:58:56.553 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.553 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.554 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:58:56.554 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.749972+0000", 2023-10-09T05:58:56.554 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7999000000000001e-05 2023-10-09T05:58:56.554 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.554 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:58:56.554 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.554 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.554 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.554 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:139321 6.26 6:65881619:::benchmark_data_smithi064_284193_object42778:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T05:58:56.554 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:56:39.575549+0000", 2023-10-09T05:58:56.554 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 136.87552812199999, 2023-10-09T05:58:56.554 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7395880610000001, 2023-10-09T05:58:56.555 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:58:56.555 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:56.555 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:58:56.555 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:58:56.555 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:56.555 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 139321 2023-10-09T05:58:56.555 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.555 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:58:56.555 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.555 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:58:56.555 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.575549+0000", 2023-10-09T05:58:56.555 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.555 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.556 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.556 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:58:56.556 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.575549+0000", 2023-10-09T05:58:56.556 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.556 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.556 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.556 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:58:56.556 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.575548+0000", 2023-10-09T05:58:56.556 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:58:56.556 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.556 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.556 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:58:56.556 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.575554+0000", 2023-10-09T05:58:56.557 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6529999999999999e-06 2023-10-09T05:58:56.557 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.557 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.557 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:58:56.557 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.575565+0000", 2023-10-09T05:58:56.557 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1021e-05 2023-10-09T05:58:56.557 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.557 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.557 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:58:56.557 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.575571+0000", 2023-10-09T05:58:56.557 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8159999999999999e-06 2023-10-09T05:58:56.557 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.558 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.558 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.558 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.575621+0000", 2023-10-09T05:58:56.558 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0800000000000002e-05 2023-10-09T05:58:56.558 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.558 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.558 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.558 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.575628+0000", 2023-10-09T05:58:56.558 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6989999999999997e-06 2023-10-09T05:58:56.558 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.558 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.558 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.559 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.248765+0000", 2023-10-09T05:58:56.559 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6731371259999999 2023-10-09T05:58:56.559 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.559 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.559 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.559 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.248773+0000", 2023-10-09T05:58:56.559 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.357e-06 2023-10-09T05:58:56.559 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.559 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.559 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.559 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.304612+0000", 2023-10-09T05:58:56.559 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.05583939 2023-10-09T05:58:56.560 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.560 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.560 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for degraded object", 2023-10-09T05:58:56.560 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.305091+0000", 2023-10-09T05:58:56.560 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000479156 2023-10-09T05:58:56.560 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.560 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.560 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.560 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.307893+0000", 2023-10-09T05:58:56.560 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0028018219999999998 2023-10-09T05:58:56.560 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.560 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.560 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:58:56.561 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.307991+0000", 2023-10-09T05:58:56.561 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.8207000000000003e-05 2023-10-09T05:58:56.561 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.561 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.561 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6,7", 2023-10-09T05:58:56.561 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.308192+0000", 2023-10-09T05:58:56.561 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00020113800000000001 2023-10-09T05:58:56.561 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.561 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.561 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.561 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.312175+0000", 2023-10-09T05:58:56.561 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0039830229999999996 2023-10-09T05:58:56.562 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.562 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.562 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:58:56.562 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.314384+0000", 2023-10-09T05:58:56.562 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0022086089999999998 2023-10-09T05:58:56.562 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.562 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.562 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.562 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.314426+0000", 2023-10-09T05:58:56.562 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2395999999999999e-05 2023-10-09T05:58:56.562 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.562 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.563 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.563 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.314509+0000", 2023-10-09T05:58:56.563 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.2572999999999997e-05 2023-10-09T05:58:56.563 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.563 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.563 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:58:56.563 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.314565+0000", 2023-10-09T05:58:56.563 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6342999999999999e-05 2023-10-09T05:58:56.563 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.563 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.563 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:58:56.563 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.315137+0000", 2023-10-09T05:58:56.564 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000572191 2023-10-09T05:58:56.564 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.564 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:58:56.564 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.564 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.564 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.564 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:139330 6.26 6:6496ae97:::benchmark_data_smithi064_284193_object42787:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T05:58:56.564 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:56:39.644429+0000", 2023-10-09T05:58:56.564 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 136.806648981, 2023-10-09T05:58:56.564 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.6734255889999998, 2023-10-09T05:58:56.564 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:58:56.564 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:56.564 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:58:56.565 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:58:56.565 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:56.565 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 139330 2023-10-09T05:58:56.565 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.565 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:58:56.565 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.565 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:58:56.565 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.644429+0000", 2023-10-09T05:58:56.565 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.565 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.565 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.566 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:58:56.566 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.644429+0000", 2023-10-09T05:58:56.566 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.566 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.566 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.566 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:58:56.566 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.644427+0000", 2023-10-09T05:58:56.566 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:58:56.566 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.566 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.566 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:58:56.566 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.644433+0000", 2023-10-09T05:58:56.567 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8819999999999998e-06 2023-10-09T05:58:56.567 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.567 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.567 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:58:56.567 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.644445+0000", 2023-10-09T05:58:56.567 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2211e-05 2023-10-09T05:58:56.567 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.567 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.567 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:58:56.567 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.644451+0000", 2023-10-09T05:58:56.567 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.054e-06 2023-10-09T05:58:56.567 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.568 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.568 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.568 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.644491+0000", 2023-10-09T05:58:56.568 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9509999999999999e-05 2023-10-09T05:58:56.568 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.568 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.568 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.568 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:39.644499+0000", 2023-10-09T05:58:56.568 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.4330000000000004e-06 2023-10-09T05:58:56.568 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.568 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.568 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.569 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.249648+0000", 2023-10-09T05:58:56.569 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6051490909999999 2023-10-09T05:58:56.569 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.569 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.569 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.569 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.249655+0000", 2023-10-09T05:58:56.569 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5860000000000001e-06 2023-10-09T05:58:56.569 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.569 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.569 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.569 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.306112+0000", 2023-10-09T05:58:56.569 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0564567119999999 2023-10-09T05:58:56.570 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.570 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.570 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for degraded object", 2023-10-09T05:58:56.570 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.306381+0000", 2023-10-09T05:58:56.570 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00026925400000000002 2023-10-09T05:58:56.570 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.570 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.570 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.570 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.310997+0000", 2023-10-09T05:58:56.570 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0046156319999999997 2023-10-09T05:58:56.570 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.570 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.571 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:58:56.571 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.311119+0000", 2023-10-09T05:58:56.571 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000121971 2023-10-09T05:58:56.571 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.571 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.571 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6,7", 2023-10-09T05:58:56.571 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.311368+0000", 2023-10-09T05:58:56.571 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000249666 2023-10-09T05:58:56.571 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.571 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.571 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:58:56.571 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.314434+0000", 2023-10-09T05:58:56.572 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0030660599999999998 2023-10-09T05:58:56.572 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.572 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.572 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.572 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.314470+0000", 2023-10-09T05:58:56.572 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6071000000000001e-05 2023-10-09T05:58:56.572 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.572 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.572 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.572 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.317184+0000", 2023-10-09T05:58:56.572 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002713961 2023-10-09T05:58:56.572 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.573 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.573 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.573 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.317220+0000", 2023-10-09T05:58:56.573 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5871999999999998e-05 2023-10-09T05:58:56.573 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.573 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.573 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:58:56.573 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.317274+0000", 2023-10-09T05:58:56.573 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3498000000000002e-05 2023-10-09T05:58:56.573 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.573 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.573 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:58:56.574 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.317854+0000", 2023-10-09T05:58:56.574 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00058033599999999998 2023-10-09T05:58:56.574 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.574 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:58:56.574 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.574 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.574 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.574 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:139445 6.26 6:675e1bb4:::benchmark_data_smithi064_284193_object42902:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-10-09T05:58:56.574 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:56:40.964836+0000", 2023-10-09T05:58:56.574 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 135.48624155100001, 2023-10-09T05:58:56.574 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3535769659999999, 2023-10-09T05:58:56.574 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:58:56.575 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:56.575 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:58:56.575 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:58:56.575 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:56.575 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 139445 2023-10-09T05:58:56.575 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.575 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:58:56.575 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.575 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:58:56.575 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.964836+0000", 2023-10-09T05:58:56.575 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.575 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.576 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.576 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:58:56.576 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.964836+0000", 2023-10-09T05:58:56.576 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.576 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.576 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.576 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:58:56.576 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.964835+0000", 2023-10-09T05:58:56.576 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:58:56.576 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.576 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.576 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:58:56.577 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.964839+0000", 2023-10-09T05:58:56.577 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9389999999999996e-06 2023-10-09T05:58:56.577 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.577 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.577 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:58:56.577 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.964848+0000", 2023-10-09T05:58:56.577 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.3420000000000008e-06 2023-10-09T05:58:56.577 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.577 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.577 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:58:56.577 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.964852+0000", 2023-10-09T05:58:56.577 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3830000000000002e-06 2023-10-09T05:58:56.578 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.578 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.578 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.578 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.964880+0000", 2023-10-09T05:58:56.578 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7385e-05 2023-10-09T05:58:56.578 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.578 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.578 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.578 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.964885+0000", 2023-10-09T05:58:56.578 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3109999999999996e-06 2023-10-09T05:58:56.578 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.578 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.578 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.579 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.249674+0000", 2023-10-09T05:58:56.579 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.28478929200000003 2023-10-09T05:58:56.579 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.579 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.579 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.579 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.249679+0000", 2023-10-09T05:58:56.579 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2079999999999999e-06 2023-10-09T05:58:56.579 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.579 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.579 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.579 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.306423+0000", 2023-10-09T05:58:56.579 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.056743507 2023-10-09T05:58:56.580 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.580 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.580 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for degraded object", 2023-10-09T05:58:56.580 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.306777+0000", 2023-10-09T05:58:56.580 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00035445099999999998 2023-10-09T05:58:56.580 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.580 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.580 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.580 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.313197+0000", 2023-10-09T05:58:56.580 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0064191880000000002 2023-10-09T05:58:56.580 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.580 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.581 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:58:56.581 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.313288+0000", 2023-10-09T05:58:56.581 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1255999999999995e-05 2023-10-09T05:58:56.581 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.581 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.581 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6,7", 2023-10-09T05:58:56.581 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.313434+0000", 2023-10-09T05:58:56.581 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000146038 2023-10-09T05:58:56.581 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.581 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.581 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.581 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.317901+0000", 2023-10-09T05:58:56.581 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0044669389999999996 2023-10-09T05:58:56.582 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.582 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.582 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.582 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.317930+0000", 2023-10-09T05:58:56.582 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9113000000000001e-05 2023-10-09T05:58:56.582 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.582 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.582 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.582 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.317960+0000", 2023-10-09T05:58:56.582 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9869000000000002e-05 2023-10-09T05:58:56.582 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.582 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.583 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:58:56.583 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.317969+0000", 2023-10-09T05:58:56.583 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9970000000000004e-06 2023-10-09T05:58:56.583 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.583 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.583 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:58:56.583 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.318001+0000", 2023-10-09T05:58:56.583 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.2580000000000003e-05 2023-10-09T05:58:56.583 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.583 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.583 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:58:56.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:56 smithi064 ceph-mon[212750]: pgmap v661: 109 pgs: 109 active+clean; 1.8 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 141 MiB/s wr, 2.25k op/s 2023-10-09T05:58:56.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:56 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/655356762' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:58:56.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:56 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:58:56.585 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:56.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:56 smithi064 ceph-mon[220044]: pgmap v661: 109 pgs: 109 active+clean; 1.8 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 141 MiB/s wr, 2.25k op/s 2023-10-09T05:58:56.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:56 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/655356762' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:58:56.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:56 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:58:56.585 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.318413+0000", 2023-10-09T05:58:56.585 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00041155800000000002 2023-10-09T05:58:56.585 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.585 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:58:56.586 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.586 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.586 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.586 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:139496 6.26 6:66d793e9:::benchmark_data_smithi064_284193_object42953:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e231)", 2023-10-09T05:58:56.586 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:56:41.303770+0000", 2023-10-09T05:58:56.586 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 135.14730797600001, 2023-10-09T05:58:56.586 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.016947491, 2023-10-09T05:58:56.586 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:58:56.586 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:56.586 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:58:56.586 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:58:56.587 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:56.587 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 139496 2023-10-09T05:58:56.587 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.587 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:58:56.587 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.587 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:58:56.587 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.303770+0000", 2023-10-09T05:58:56.587 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.587 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.587 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.587 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:58:56.587 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.303770+0000", 2023-10-09T05:58:56.588 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.588 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.588 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.588 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:58:56.588 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.303769+0000", 2023-10-09T05:58:56.588 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:58:56.588 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.588 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.588 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:58:56.588 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.303773+0000", 2023-10-09T05:58:56.588 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9130000000000002e-06 2023-10-09T05:58:56.588 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.589 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.589 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:58:56.589 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.303785+0000", 2023-10-09T05:58:56.589 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.153e-05 2023-10-09T05:58:56.589 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.589 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.589 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:58:56.589 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.303792+0000", 2023-10-09T05:58:56.589 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7190000000000004e-06 2023-10-09T05:58:56.589 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.589 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.589 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.590 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.303836+0000", 2023-10-09T05:58:56.590 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4391999999999998e-05 2023-10-09T05:58:56.590 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.590 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.590 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.590 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.303844+0000", 2023-10-09T05:58:56.590 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7670000000000003e-06 2023-10-09T05:58:56.590 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.590 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.590 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.590 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.309367+0000", 2023-10-09T05:58:56.590 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0055227419999999 2023-10-09T05:58:56.591 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.591 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.591 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for degraded object", 2023-10-09T05:58:56.591 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.309787+0000", 2023-10-09T05:58:56.591 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00042061000000000003 2023-10-09T05:58:56.591 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.591 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.591 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.591 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.316125+0000", 2023-10-09T05:58:56.591 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0063375990000000002 2023-10-09T05:58:56.591 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.592 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.592 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:58:56.592 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.316216+0000", 2023-10-09T05:58:56.592 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1362999999999999e-05 2023-10-09T05:58:56.592 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.592 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.592 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6,7", 2023-10-09T05:58:56.592 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.316414+0000", 2023-10-09T05:58:56.592 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000198066 2023-10-09T05:58:56.592 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.592 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.592 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.593 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.318466+0000", 2023-10-09T05:58:56.593 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0020518820000000001 2023-10-09T05:58:56.593 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.593 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.593 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.593 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.319381+0000", 2023-10-09T05:58:56.593 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00091480800000000003 2023-10-09T05:58:56.593 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.593 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.593 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.593 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.319696+0000", 2023-10-09T05:58:56.593 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00031472599999999998 2023-10-09T05:58:56.594 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.594 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.594 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:58:56.594 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.320052+0000", 2023-10-09T05:58:56.594 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00035611399999999999 2023-10-09T05:58:56.594 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.594 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.594 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:58:56.594 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.320098+0000", 2023-10-09T05:58:56.594 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5905999999999997e-05 2023-10-09T05:58:56.594 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.595 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.595 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:58:56.595 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.320717+0000", 2023-10-09T05:58:56.595 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000619325 2023-10-09T05:58:56.595 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.595 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:58:56.595 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.595 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.595 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.595 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:159588 6.f 6:f2345e47:::benchmark_data_smithi064_284193_object63045:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2023-10-09T05:58:56.595 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:22.436723+0000", 2023-10-09T05:58:56.595 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 94.014354924000003, 2023-10-09T05:58:56.596 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0262967700000001, 2023-10-09T05:58:56.596 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:58:56.596 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T05:58:56.596 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:58:56.596 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:58:56.596 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:56.596 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 159588 2023-10-09T05:58:56.596 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.596 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:58:56.596 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.597 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:58:56.597 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.436723+0000", 2023-10-09T05:58:56.597 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.597 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.597 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.597 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:58:56.597 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.436723+0000", 2023-10-09T05:58:56.597 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.597 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.597 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.597 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:58:56.597 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.436722+0000", 2023-10-09T05:58:56.597 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:58:56.598 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.598 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.598 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:58:56.598 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.436725+0000", 2023-10-09T05:58:56.598 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.3560000000000001e-06 2023-10-09T05:58:56.598 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.598 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.598 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:58:56.598 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.436732+0000", 2023-10-09T05:58:56.598 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2629999999999997e-06 2023-10-09T05:58:56.598 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.599 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.599 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:58:56.599 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.436737+0000", 2023-10-09T05:58:56.599 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3819999999999997e-06 2023-10-09T05:58:56.599 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.599 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.599 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.599 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.436770+0000", 2023-10-09T05:58:56.599 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.2956000000000001e-05 2023-10-09T05:58:56.599 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.599 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.599 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for degraded object", 2023-10-09T05:58:56.600 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.437182+0000", 2023-10-09T05:58:56.600 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00041272200000000002 2023-10-09T05:58:56.600 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.600 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.600 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.600 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.447894+0000", 2023-10-09T05:58:56.600 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010711461 2023-10-09T05:58:56.600 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.600 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.600 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.600 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.447904+0000", 2023-10-09T05:58:56.600 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0423e-05 2023-10-09T05:58:56.601 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.601 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.601 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.601 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:23.463007+0000", 2023-10-09T05:58:56.601 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.015102561 2023-10-09T05:58:56.601 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.601 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.601 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:58:56.601 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:23.463019+0000", 2023-10-09T05:58:56.601 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2544000000000001e-05 2023-10-09T05:58:56.601 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.601 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:58:56.601 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:56.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:56.603 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:56 smithi156 ceph-mon[195101]: pgmap v661: 109 pgs: 109 active+clean; 1.8 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 141 MiB/s wr, 2.25k op/s 2023-10-09T05:58:56.603 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:56 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/655356762' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:58:56.603 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:56 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:58:56.603 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.604 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.604 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:159594 6.26 6:67500365:::benchmark_data_smithi064_284193_object63051:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2023-10-09T05:58:56.604 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:22.437976+0000", 2023-10-09T05:58:56.604 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 94.013101214000002, 2023-10-09T05:58:56.604 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.180718208, 2023-10-09T05:58:56.604 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:58:56.604 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:56.604 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:58:56.604 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:58:56.604 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:56.604 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 159594 2023-10-09T05:58:56.605 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.605 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:58:56.605 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.605 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:58:56.605 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.437976+0000", 2023-10-09T05:58:56.605 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.605 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.605 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.605 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:58:56.605 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.437976+0000", 2023-10-09T05:58:56.605 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.605 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.606 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.606 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:58:56.606 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.437976+0000", 2023-10-09T05:58:56.606 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:56.606 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.606 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.606 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:58:56.606 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.437978+0000", 2023-10-09T05:58:56.606 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.368e-06 2023-10-09T05:58:56.606 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.606 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.606 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:58:56.607 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.437984+0000", 2023-10-09T05:58:56.607 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2170000000000002e-06 2023-10-09T05:58:56.607 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.607 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.607 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:58:56.607 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.437987+0000", 2023-10-09T05:58:56.607 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.2940000000000001e-06 2023-10-09T05:58:56.607 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.607 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.607 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.607 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.440386+0000", 2023-10-09T05:58:56.607 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00239909 2023-10-09T05:58:56.608 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.608 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.608 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for degraded object", 2023-10-09T05:58:56.608 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.446633+0000", 2023-10-09T05:58:56.608 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0062471810000000001 2023-10-09T05:58:56.608 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.608 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.608 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.608 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.451866+0000", 2023-10-09T05:58:56.608 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.005232441 2023-10-09T05:58:56.609 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.609 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.609 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.609 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.451875+0000", 2023-10-09T05:58:56.609 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7920000000000004e-06 2023-10-09T05:58:56.609 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.609 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.609 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.609 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:23.457201+0000", 2023-10-09T05:58:56.609 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0053255270000001 2023-10-09T05:58:56.609 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.609 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.610 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.610 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:23.457217+0000", 2023-10-09T05:58:56.610 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6141999999999999e-05 2023-10-09T05:58:56.610 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.610 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.610 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.610 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.614861+0000", 2023-10-09T05:58:56.610 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.157644117 2023-10-09T05:58:56.610 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.610 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.610 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:58:56.611 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.615002+0000", 2023-10-09T05:58:56.611 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014138200000000001 2023-10-09T05:58:56.611 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.611 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.611 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:58:56.611 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.615225+0000", 2023-10-09T05:58:56.611 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00022210200000000001 2023-10-09T05:58:56.611 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.611 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.611 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.611 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.616961+0000", 2023-10-09T05:58:56.611 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0017364710000000001 2023-10-09T05:58:56.611 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.612 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.612 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:58:56.612 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.617287+0000", 2023-10-09T05:58:56.612 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00032552599999999997 2023-10-09T05:58:56.612 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.612 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.612 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.612 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.618555+0000", 2023-10-09T05:58:56.612 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0012683010000000001 2023-10-09T05:58:56.612 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.612 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.612 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:58:56.613 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.618590+0000", 2023-10-09T05:58:56.613 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5231000000000002e-05 2023-10-09T05:58:56.613 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.613 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.613 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:58:56.613 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.618694+0000", 2023-10-09T05:58:56.613 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000104355 2023-10-09T05:58:56.613 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.613 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:58:56.613 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.613 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.613 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.614 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:159600 6.1f 6.a316739f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-10-09T05:58:56.614 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:22.465019+0000", 2023-10-09T05:58:56.614 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 93.986058846999995, 2023-10-09T05:58:56.614 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99762972400000005, 2023-10-09T05:58:56.614 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:58:56.614 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T05:58:56.614 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:58:56.614 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:58:56.614 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:56.614 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 159600 2023-10-09T05:58:56.614 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.615 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:58:56.615 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.615 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:58:56.615 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.465019+0000", 2023-10-09T05:58:56.615 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.615 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.615 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.615 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:58:56.615 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.465019+0000", 2023-10-09T05:58:56.615 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.615 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.615 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.615 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:58:56.616 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.465017+0000", 2023-10-09T05:58:56.616 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T05:58:56.616 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.616 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.616 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:58:56.616 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.465027+0000", 2023-10-09T05:58:56.616 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0339e-05 2023-10-09T05:58:56.616 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.616 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.616 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:58:56.616 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.465037+0000", 2023-10-09T05:58:56.616 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.5810000000000003e-06 2023-10-09T05:58:56.617 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.617 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.617 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:58:56.617 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.465044+0000", 2023-10-09T05:58:56.617 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.968e-06 2023-10-09T05:58:56.617 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.617 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.617 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.617 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.465409+0000", 2023-10-09T05:58:56.617 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00036546199999999999 2023-10-09T05:58:56.617 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.618 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.618 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.618 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.465418+0000", 2023-10-09T05:58:56.618 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9290000000000004e-06 2023-10-09T05:58:56.618 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.618 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.618 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.618 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:23.462637+0000", 2023-10-09T05:58:56.618 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.997218453 2023-10-09T05:58:56.618 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.618 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.619 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:58:56.619 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:23.462648+0000", 2023-10-09T05:58:56.619 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1687e-05 2023-10-09T05:58:56.619 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.619 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:58:56.619 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.619 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.619 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.619 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:159608 6.16 6:68970006:::benchmark_data_smithi064_284193_object63065:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-10-09T05:58:56.619 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:22.480587+0000", 2023-10-09T05:58:56.619 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 93.970490858999995, 2023-10-09T05:58:56.619 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1816533549999999, 2023-10-09T05:58:56.620 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:58:56.620 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:56.620 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:58:56.620 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:58:56.620 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:56.620 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 159608 2023-10-09T05:58:56.620 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.620 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:58:56.620 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.620 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:58:56.620 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.480587+0000", 2023-10-09T05:58:56.620 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.621 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.621 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.621 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:58:56.621 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.480587+0000", 2023-10-09T05:58:56.621 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.621 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.621 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.621 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:58:56.621 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.480586+0000", 2023-10-09T05:58:56.621 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:58:56.621 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.621 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.621 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:58:56.622 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.480589+0000", 2023-10-09T05:58:56.622 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0230000000000001e-06 2023-10-09T05:58:56.622 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.622 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.622 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:58:56.622 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.480596+0000", 2023-10-09T05:58:56.622 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7989999999999996e-06 2023-10-09T05:58:56.622 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.622 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.622 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:58:56.622 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.480601+0000", 2023-10-09T05:58:56.623 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.605e-06 2023-10-09T05:58:56.623 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.623 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.623 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.623 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.480646+0000", 2023-10-09T05:58:56.623 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5846000000000001e-05 2023-10-09T05:58:56.623 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.623 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.623 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.623 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.480652+0000", 2023-10-09T05:58:56.623 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.682e-06 2023-10-09T05:58:56.623 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.623 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.624 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.624 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:23.465671+0000", 2023-10-09T05:58:56.624 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.98501928999999999 2023-10-09T05:58:56.624 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.624 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.624 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.624 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:23.465683+0000", 2023-10-09T05:58:56.624 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1367e-05 2023-10-09T05:58:56.624 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.624 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.624 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.625 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.647260+0000", 2023-10-09T05:58:56.625 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.181576781 2023-10-09T05:58:56.625 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.625 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.625 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:58:56.625 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.647362+0000", 2023-10-09T05:58:56.625 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000102829 2023-10-09T05:58:56.625 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.625 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.625 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:58:56.625 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.647682+0000", 2023-10-09T05:58:56.625 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00031999900000000001 2023-10-09T05:58:56.626 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.626 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.626 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:58:56.626 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.649150+0000", 2023-10-09T05:58:56.626 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001467545 2023-10-09T05:58:56.626 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.626 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.626 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.626 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.649872+0000", 2023-10-09T05:58:56.626 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00072161899999999995 2023-10-09T05:58:56.626 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.626 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.627 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.627 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.662001+0000", 2023-10-09T05:58:56.627 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012129071999999999 2023-10-09T05:58:56.627 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.627 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.627 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:58:56.627 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.662050+0000", 2023-10-09T05:58:56.627 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9101999999999998e-05 2023-10-09T05:58:56.627 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.627 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.627 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:58:56.627 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.662240+0000", 2023-10-09T05:58:56.628 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000190261 2023-10-09T05:58:56.628 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.628 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:58:56.628 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.628 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.628 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.628 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:159612 6.2 6:43636cab:::benchmark_data_smithi064_284193_object63069:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-10-09T05:58:56.628 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:22.487091+0000", 2023-10-09T05:58:56.628 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 93.963986759999997, 2023-10-09T05:58:56.628 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1619505939999999, 2023-10-09T05:58:56.628 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:58:56.628 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:58:56.629 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:58:56.629 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:58:56.629 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:58:56.629 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 159612 2023-10-09T05:58:56.629 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.629 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:58:56.629 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.629 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:58:56.629 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.487091+0000", 2023-10-09T05:58:56.629 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.629 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.629 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.630 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:58:56.630 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.487091+0000", 2023-10-09T05:58:56.630 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:58:56.630 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.630 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.630 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:58:56.630 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.487089+0000", 2023-10-09T05:58:56.630 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:58:56.630 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.630 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.630 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:58:56.630 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.487097+0000", 2023-10-09T05:58:56.631 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4399999999999999e-06 2023-10-09T05:58:56.631 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.631 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.631 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:58:56.631 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.487115+0000", 2023-10-09T05:58:56.631 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8176e-05 2023-10-09T05:58:56.631 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.631 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.631 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:58:56.631 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.487122+0000", 2023-10-09T05:58:56.631 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5310000000000004e-06 2023-10-09T05:58:56.631 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.632 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.632 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.632 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.487192+0000", 2023-10-09T05:58:56.632 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9129999999999997e-05 2023-10-09T05:58:56.632 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.632 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.632 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.632 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.487217+0000", 2023-10-09T05:58:56.632 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4967999999999999e-05 2023-10-09T05:58:56.632 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.632 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.632 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.633 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:23.459418+0000", 2023-10-09T05:58:56.633 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.97220148299999998 2023-10-09T05:58:56.633 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.633 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.633 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:58:56.633 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:23.459430+0000", 2023-10-09T05:58:56.633 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.148e-05 2023-10-09T05:58:56.633 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.633 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.633 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:58:56.633 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.622306+0000", 2023-10-09T05:58:56.633 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1628761910000001 2023-10-09T05:58:56.634 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.634 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.634 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:58:56.634 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.622447+0000", 2023-10-09T05:58:56.634 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000141462 2023-10-09T05:58:56.634 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.634 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.634 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T05:58:56.634 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.622730+0000", 2023-10-09T05:58:56.634 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00028278599999999999 2023-10-09T05:58:56.634 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.634 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.635 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.635 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.645674+0000", 2023-10-09T05:58:56.635 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.022943631999999999 2023-10-09T05:58:56.635 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.635 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.635 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:58:56.635 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.645841+0000", 2023-10-09T05:58:56.635 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000167307 2023-10-09T05:58:56.635 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.635 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.635 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.635 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.646814+0000", 2023-10-09T05:58:56.635 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00097347899999999997 2023-10-09T05:58:56.636 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.636 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.636 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:58:56.636 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.648859+0000", 2023-10-09T05:58:56.636 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0020445820000000001 2023-10-09T05:58:56.636 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.636 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.636 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:58:56.636 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.648919+0000", 2023-10-09T05:58:56.636 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0476999999999997e-05 2023-10-09T05:58:56.636 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:58:56.636 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:58:56.637 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:58:56.637 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.649041+0000", 2023-10-09T05:58:56.637 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012188000000000001 2023-10-09T05:58:56.637 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.637 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:58:56.637 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.637 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:58:56.637 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:58:56.637 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:58:56.697 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:58:56.703 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:56.795 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:56.969 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 18 16 37860 37844 131.373 86.0625 0.0021309 0.00752831 2023-10-09T05:58:57.254 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:57.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:57.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:57.299 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:57 smithi064 ceph-mon[212750]: 1.2 scrub starts 2023-10-09T05:58:57.299 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:57 smithi064 ceph-mon[212750]: 1.2 scrub ok 2023-10-09T05:58:57.299 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:57 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:58:57.299 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:57 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:58:57.299 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:57 smithi064 ceph-mon[212750]: osdmap e291: 8 total, 8 up, 7 in 2023-10-09T05:58:57.299 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:57 smithi064 ceph-mon[212750]: failed to encode map e291 with expected crc 2023-10-09T05:58:57.299 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:57 smithi064 ceph-mon[212750]: failed to encode map e291 with expected crc 2023-10-09T05:58:57.299 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:57 smithi064 ceph-mon[212750]: failed to encode map e291 with expected crc 2023-10-09T05:58:57.299 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:57 smithi064 ceph-mon[212750]: failed to encode map e291 with expected crc 2023-10-09T05:58:57.300 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:57 smithi064 ceph-mon[220044]: 1.2 scrub starts 2023-10-09T05:58:57.300 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:57 smithi064 ceph-mon[220044]: 1.2 scrub ok 2023-10-09T05:58:57.300 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:57 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:58:57.300 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:57 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:58:57.300 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:57 smithi064 ceph-mon[220044]: osdmap e291: 8 total, 8 up, 7 in 2023-10-09T05:58:57.300 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:57 smithi064 ceph-mon[220044]: failed to encode map e291 with expected crc 2023-10-09T05:58:57.300 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:57 smithi064 ceph-mon[220044]: failed to encode map e291 with expected crc 2023-10-09T05:58:57.300 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:57 smithi064 ceph-mon[220044]: failed to encode map e291 with expected crc 2023-10-09T05:58:57.300 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:57 smithi064 ceph-mon[220044]: failed to encode map e291 with expected crc 2023-10-09T05:58:57.363 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:58:57.424 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:57.543 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:58:57.551 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:58:57.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:57.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:58:57.554 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:57 smithi156 ceph-mon[195101]: 1.2 scrub starts 2023-10-09T05:58:57.555 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:57 smithi156 ceph-mon[195101]: 1.2 scrub ok 2023-10-09T05:58:57.555 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:57 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:58:57.555 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:57 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:58:57.555 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:57 smithi156 ceph-mon[195101]: osdmap e291: 8 total, 8 up, 7 in 2023-10-09T05:58:57.555 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:57 smithi156 ceph-mon[195101]: failed to encode map e291 with expected crc 2023-10-09T05:58:57.555 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:57 smithi156 ceph-mon[195101]: failed to encode map e291 with expected crc 2023-10-09T05:58:57.555 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:57 smithi156 ceph-mon[195101]: failed to encode map e291 with expected crc 2023-10-09T05:58:57.555 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:57 smithi156 ceph-mon[195101]: failed to encode map e291 with expected crc 2023-10-09T05:58:57.653 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:57.661 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:57.722 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:57.947 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:57.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:57.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:57.969 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 19 16 39168 39152 128.762 81.75 0.0128799 0.0077608 2023-10-09T05:58:58.055 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:58:58.075 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:58:58.104 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:58.190 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:58:58.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:58.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:58:58.295 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:58:58.325 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.2 primary-affinity to 0.170685 (8111852) 2023-10-09T05:58:58.352 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:58 smithi156 ceph-mon[195101]: pgmap v663: 109 pgs: 109 active+clean; 2.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 117 MiB/s wr, 1.86k op/s 2023-10-09T05:58:58.352 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:58 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1882295038' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.1706845453979685}]: dispatch 2023-10-09T05:58:58.354 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:58.412 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:58:58.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:58.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:58:58.515 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:58.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:58 smithi064 ceph-mon[212750]: pgmap v663: 109 pgs: 109 active+clean; 2.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 117 MiB/s wr, 1.86k op/s 2023-10-09T05:58:58.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:58 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1882295038' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.1706845453979685}]: dispatch 2023-10-09T05:58:58.555 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:58 smithi064 ceph-mon[220044]: pgmap v663: 109 pgs: 109 active+clean; 2.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 117 MiB/s wr, 1.86k op/s 2023-10-09T05:58:58.555 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:58 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1882295038' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.1706845453979685}]: dispatch 2023-10-09T05:58:58.561 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:58.622 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:58.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:58.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:58.727 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:58:58.756 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:58.969 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-10-09T05:58:58.970172+0000 min lat: 0.00127391 max lat: 0.268173 avg lat: 0.00779813 2023-10-09T05:58:58.969 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:58:58.970 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 20 16 40026 40010 125.005 53.625 0.00272992 0.00779813 2023-10-09T05:58:59.424 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:58:59.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:59.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:58:59.483 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:58:59.529 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:58:59.586 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[220044]: failed to encode map e292 with expected crc 2023-10-09T05:58:59.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1882295038' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.1706845453979685}]': finished 2023-10-09T05:58:59.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[220044]: osdmap e292: 8 total, 8 up, 7 in 2023-10-09T05:58:59.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[220044]: failed to encode map e292 with expected crc 2023-10-09T05:58:59.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[220044]: failed to encode map e292 with expected crc 2023-10-09T05:58:59.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[220044]: failed to encode map e292 with expected crc 2023-10-09T05:58:59.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[220044]: failed to encode map e292 with expected crc 2023-10-09T05:58:59.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[220044]: failed to encode map e292 with expected crc 2023-10-09T05:58:59.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[220044]: failed to encode map e292 with expected crc 2023-10-09T05:58:59.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 5]}]: dispatch 2023-10-09T05:58:59.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 2]}]: dispatch 2023-10-09T05:58:59.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 5]}]: dispatch 2023-10-09T05:58:59.588 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 2]}]: dispatch 2023-10-09T05:58:59.588 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:59.588 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[220044]: pgmap v665: 109 pgs: 2 peering, 107 active+clean; 2.2 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 10 KiB/s rd, 84 MiB/s wr, 1.35k op/s; 20 KiB/s, 0 objects/s recovering 2023-10-09T05:58:59.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[212750]: failed to encode map e292 with expected crc 2023-10-09T05:58:59.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1882295038' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.1706845453979685}]': finished 2023-10-09T05:58:59.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[212750]: osdmap e292: 8 total, 8 up, 7 in 2023-10-09T05:58:59.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[212750]: failed to encode map e292 with expected crc 2023-10-09T05:58:59.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[212750]: failed to encode map e292 with expected crc 2023-10-09T05:58:59.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[212750]: failed to encode map e292 with expected crc 2023-10-09T05:58:59.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[212750]: failed to encode map e292 with expected crc 2023-10-09T05:58:59.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[212750]: failed to encode map e292 with expected crc 2023-10-09T05:58:59.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[212750]: failed to encode map e292 with expected crc 2023-10-09T05:58:59.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 5]}]: dispatch 2023-10-09T05:58:59.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 2]}]: dispatch 2023-10-09T05:58:59.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 5]}]: dispatch 2023-10-09T05:58:59.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 2]}]: dispatch 2023-10-09T05:58:59.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:59.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:58:59 smithi064 ceph-mon[212750]: pgmap v665: 109 pgs: 2 peering, 107 active+clean; 2.2 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 10 KiB/s rd, 84 MiB/s wr, 1.35k op/s; 20 KiB/s, 0 objects/s recovering 2023-10-09T05:58:59.593 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:58:59.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:59 smithi156 ceph-mon[195101]: failed to encode map e292 with expected crc 2023-10-09T05:58:59.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:59 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1882295038' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.1706845453979685}]': finished 2023-10-09T05:58:59.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:59 smithi156 ceph-mon[195101]: osdmap e292: 8 total, 8 up, 7 in 2023-10-09T05:58:59.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:59 smithi156 ceph-mon[195101]: failed to encode map e292 with expected crc 2023-10-09T05:58:59.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:59 smithi156 ceph-mon[195101]: failed to encode map e292 with expected crc 2023-10-09T05:58:59.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:59 smithi156 ceph-mon[195101]: failed to encode map e292 with expected crc 2023-10-09T05:58:59.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:59 smithi156 ceph-mon[195101]: failed to encode map e292 with expected crc 2023-10-09T05:58:59.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:59 smithi156 ceph-mon[195101]: failed to encode map e292 with expected crc 2023-10-09T05:58:59.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:59 smithi156 ceph-mon[195101]: failed to encode map e292 with expected crc 2023-10-09T05:58:59.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 5]}]: dispatch 2023-10-09T05:58:59.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 2]}]: dispatch 2023-10-09T05:58:59.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:59 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 5]}]: dispatch 2023-10-09T05:58:59.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:59 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 2]}]: dispatch 2023-10-09T05:58:59.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:59.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:58:59 smithi156 ceph-mon[195101]: pgmap v665: 109 pgs: 2 peering, 107 active+clean; 2.2 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 10 KiB/s rd, 84 MiB/s wr, 1.35k op/s; 20 KiB/s, 0 objects/s recovering 2023-10-09T05:58:59.927 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:58:59.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:58:59.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:58:59.969 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 21 16 40968 40952 121.856 58.875 0.00707554 0.00819986 2023-10-09T05:58:59.997 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:58:59.997 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T05:58:59.997 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 0 2023-10-09T05:58:59.997 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:59:00.040 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:00.102 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:00.247 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:00.605 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:00 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T05:59:00.605 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:00 smithi156 ceph-mon[195101]: failed to encode map e293 with expected crc 2023-10-09T05:59:00.606 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:00 smithi156 ceph-mon[195101]: failed to encode map e293 with expected crc 2023-10-09T05:59:00.606 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:00 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 5]}]': finished 2023-10-09T05:59:00.606 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:00 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 2]}]': finished 2023-10-09T05:59:00.606 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:00 smithi156 ceph-mon[195101]: osdmap e293: 8 total, 8 up, 7 in 2023-10-09T05:59:00.606 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:00 smithi156 ceph-mon[195101]: failed to encode map e293 with expected crc 2023-10-09T05:59:00.606 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:00 smithi156 ceph-mon[195101]: failed to encode map e293 with expected crc 2023-10-09T05:59:00.606 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:00 smithi156 ceph-mon[195101]: failed to encode map e293 with expected crc 2023-10-09T05:59:00.606 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:00 smithi156 ceph-mon[195101]: failed to encode map e293 with expected crc 2023-10-09T05:59:00.629 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:00.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:00.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:00.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:00 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T05:59:00.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:00 smithi064 ceph-mon[212750]: failed to encode map e293 with expected crc 2023-10-09T05:59:00.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:00 smithi064 ceph-mon[212750]: failed to encode map e293 with expected crc 2023-10-09T05:59:00.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:00 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 5]}]': finished 2023-10-09T05:59:00.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:00 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 2]}]': finished 2023-10-09T05:59:00.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:00 smithi064 ceph-mon[212750]: osdmap e293: 8 total, 8 up, 7 in 2023-10-09T05:59:00.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:00 smithi064 ceph-mon[212750]: failed to encode map e293 with expected crc 2023-10-09T05:59:00.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:00 smithi064 ceph-mon[212750]: failed to encode map e293 with expected crc 2023-10-09T05:59:00.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:00 smithi064 ceph-mon[212750]: failed to encode map e293 with expected crc 2023-10-09T05:59:00.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:00 smithi064 ceph-mon[212750]: failed to encode map e293 with expected crc 2023-10-09T05:59:00.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:00 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T05:59:00.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:00 smithi064 ceph-mon[220044]: failed to encode map e293 with expected crc 2023-10-09T05:59:00.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:00 smithi064 ceph-mon[220044]: failed to encode map e293 with expected crc 2023-10-09T05:59:00.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:00 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 5]}]': finished 2023-10-09T05:59:00.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:00 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 2]}]': finished 2023-10-09T05:59:00.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:00 smithi064 ceph-mon[220044]: osdmap e293: 8 total, 8 up, 7 in 2023-10-09T05:59:00.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:00 smithi064 ceph-mon[220044]: failed to encode map e293 with expected crc 2023-10-09T05:59:00.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:00 smithi064 ceph-mon[220044]: failed to encode map e293 with expected crc 2023-10-09T05:59:00.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:00 smithi064 ceph-mon[220044]: failed to encode map e293 with expected crc 2023-10-09T05:59:00.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:00 smithi064 ceph-mon[220044]: failed to encode map e293 with expected crc 2023-10-09T05:59:00.716 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:59:00.733 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:00.757 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:59:00.800 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:59:00.807 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:00.853 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:59:00.900 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:59:00.937 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:59:00.970 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 22 16 42217 42201 119.859 78.0625 0.013406 0.0083383 2023-10-09T05:59:00.976 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:59:01.009 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:59:01.025 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:59:01.026 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:59:01.026 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:59:01.026 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:59:01.026 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:59:01.026 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:59:01.026 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:59:01.026 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:59:01.090 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:59:01.341 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:01 smithi156 ceph-mon[195101]: osdmap e294: 8 total, 8 up, 7 in 2023-10-09T05:59:01.341 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:01 smithi156 ceph-mon[195101]: failed to encode map e294 with expected crc 2023-10-09T05:59:01.341 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:01 smithi156 ceph-mon[195101]: failed to encode map e294 with expected crc 2023-10-09T05:59:01.341 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:01 smithi156 ceph-mon[195101]: pgmap v668: 109 pgs: 4 peering, 105 active+clean; 2.5 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 29 KiB/s rd, 91 MiB/s wr, 1.47k op/s; 54 KiB/s, 0 objects/s recovering 2023-10-09T05:59:01.341 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:01 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1062446028' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:01.344 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:01 smithi064 ceph-mon[220044]: osdmap e294: 8 total, 8 up, 7 in 2023-10-09T05:59:01.344 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:01 smithi064 ceph-mon[220044]: failed to encode map e294 with expected crc 2023-10-09T05:59:01.344 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:01 smithi064 ceph-mon[220044]: failed to encode map e294 with expected crc 2023-10-09T05:59:01.344 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:01 smithi064 ceph-mon[220044]: pgmap v668: 109 pgs: 4 peering, 105 active+clean; 2.5 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 29 KiB/s rd, 91 MiB/s wr, 1.47k op/s; 54 KiB/s, 0 objects/s recovering 2023-10-09T05:59:01.344 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:01 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1062446028' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:01.712 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:59:01.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:01 smithi064 ceph-mon[212750]: osdmap e294: 8 total, 8 up, 7 in 2023-10-09T05:59:01.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:01 smithi064 ceph-mon[212750]: failed to encode map e294 with expected crc 2023-10-09T05:59:01.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:01 smithi064 ceph-mon[212750]: failed to encode map e294 with expected crc 2023-10-09T05:59:01.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:01 smithi064 ceph-mon[212750]: pgmap v668: 109 pgs: 4 peering, 105 active+clean; 2.5 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 29 KiB/s rd, 91 MiB/s wr, 1.47k op/s; 54 KiB/s, 0 objects/s recovering 2023-10-09T05:59:01.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:01 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1062446028' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:01.969 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:01.972 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 23 16 43597 43581 118.387 86.25 0.00979843 0.00844 2023-10-09T05:59:01.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:01.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:02.079 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:02.124 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:02.550 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:02.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:02.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:02.653 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:02.681 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:02.785 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:02.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:02.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:02.902 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:02.958 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:02.973 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 24 16 44571 44555 115.991 60.875 0.0123979 0.00861631 2023-10-09T05:59:03.110 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:03.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:03.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:03.215 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:03.274 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:03.396 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:03.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:03.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:03.503 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:03.554 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:03.644 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:03.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:03.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:03.687 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T05:59:03.687 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:59:03.754 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:03.826 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:03.932 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:03.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:03.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:03.974 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 25 16 46012 45996 114.953 90.0625 0.00734041 0.00869401 2023-10-09T05:59:04.037 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:04.086 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:04.136 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:04.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:04.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:04.239 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:04.266 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:04.279 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T05:59:04.310 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:04.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:04.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:04.413 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:04.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:04 smithi064 ceph-mon[220044]: pgmap v669: 109 pgs: 4 peering, 105 active+clean; 2.5 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 21 KiB/s rd, 69 MiB/s wr, 1.12k op/s; 39 KiB/s, 0 objects/s recovering 2023-10-09T05:59:04.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:04 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2730697580' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:04.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:04 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:04.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:04 smithi064 ceph-mon[212750]: pgmap v669: 109 pgs: 4 peering, 105 active+clean; 2.5 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 21 KiB/s rd, 69 MiB/s wr, 1.12k op/s; 39 KiB/s, 0 objects/s recovering 2023-10-09T05:59:04.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:04 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2730697580' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:04.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:04 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:04.491 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:04.594 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:04.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:04.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:04.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:04 smithi156 ceph-mon[195101]: pgmap v669: 109 pgs: 4 peering, 105 active+clean; 2.5 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 21 KiB/s rd, 69 MiB/s wr, 1.12k op/s; 39 KiB/s, 0 objects/s recovering 2023-10-09T05:59:04.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:04 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2730697580' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:04.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:04 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:04.652 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:59:04.694 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:59:04.698 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:04.786 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:04.895 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:04.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:04.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:04.971 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T05:59:04.971 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:59:04.971 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-10-09T05:59:04.972 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:59:04.972 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:59:04.972 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-10-09T05:59:04.973 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 26 15 47137 47122 113.238 70.375 0.0095088 0.00882683 2023-10-09T05:59:05.004 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:05.031 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:05.129 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:05.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:05.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:05.137 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:59:05.235 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:05.286 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:05.413 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:05.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:05.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:05.519 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:05.566 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:05 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:05.566 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:05 smithi156 ceph-mon[195101]: failed to encode map e295 with expected crc 2023-10-09T05:59:05.567 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:05 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:59:05.567 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:05 smithi156 ceph-mon[195101]: osdmap e295: 8 total, 8 up, 7 in 2023-10-09T05:59:05.567 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:05 smithi156 ceph-mon[195101]: failed to encode map e295 with expected crc 2023-10-09T05:59:05.571 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:05.718 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:05 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:05.718 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:05 smithi064 ceph-mon[220044]: failed to encode map e295 with expected crc 2023-10-09T05:59:05.718 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:05 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:59:05.718 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:05 smithi064 ceph-mon[220044]: osdmap e295: 8 total, 8 up, 7 in 2023-10-09T05:59:05.718 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:05 smithi064 ceph-mon[220044]: failed to encode map e295 with expected crc 2023-10-09T05:59:05.718 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:59:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:59:05] "GET /metrics HTTP/1.1" 200 36391 "" "Prometheus/2.43.0" 2023-10-09T05:59:05.726 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:05 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:05.726 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:05 smithi064 ceph-mon[212750]: failed to encode map e295 with expected crc 2023-10-09T05:59:05.726 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:05 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:59:05.726 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:05 smithi064 ceph-mon[212750]: osdmap e295: 8 total, 8 up, 7 in 2023-10-09T05:59:05.726 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:05 smithi064 ceph-mon[212750]: failed to encode map e295 with expected crc 2023-10-09T05:59:05.809 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:05.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:05.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:05.903 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:05.916 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:05.951 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:05.974 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 27 16 48227 48211 111.564 68.0625 0.00623724 0.0089573 2023-10-09T05:59:06.002 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:06.180 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:06.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:06.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:06.284 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:06.328 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:06.558 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:06 smithi064 ceph-mon[212750]: pgmap v671: 109 pgs: 2 peering, 107 active+clean; 2.6 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 7.0 KiB/s rd, 56 MiB/s wr, 907 op/s; 13 KiB/s, 0 objects/s recovering 2023-10-09T05:59:06.558 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:06 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T05:59:06.558 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:06 smithi064 ceph-mon[220044]: pgmap v671: 109 pgs: 2 peering, 107 active+clean; 2.6 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 7.0 KiB/s rd, 56 MiB/s wr, 907 op/s; 13 KiB/s, 0 objects/s recovering 2023-10-09T05:59:06.558 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:06 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T05:59:06.559 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:06.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:06.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:06.564 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:06 smithi156 ceph-mon[195101]: pgmap v671: 109 pgs: 2 peering, 107 active+clean; 2.6 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 7.0 KiB/s rd, 56 MiB/s wr, 907 op/s; 13 KiB/s, 0 objects/s recovering 2023-10-09T05:59:06.565 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:06 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T05:59:06.665 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:06.715 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:06.865 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:06.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:06.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:06.970 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:06.974 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 28 16 49633 49617 110.718 87.875 0.00843733 0.00902629 2023-10-09T05:59:07.042 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:07.123 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T05:59:07.123 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:59:07.754 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:59:07.974 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 29 16 50419 50403 108.594 49.125 0.00950827 0.00920462 2023-10-09T05:59:08.114 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:08.119 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:59:08.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:08.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:08.228 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:08.308 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:08 smithi064 ceph-mon[220044]: pgmap v672: 109 pgs: 109 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 5.7 KiB/s rd, 82 MiB/s wr, 1.31k op/s; 11 KiB/s, 0 objects/s recovering 2023-10-09T05:59:08.311 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:08 smithi064 ceph-mon[212750]: pgmap v672: 109 pgs: 109 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 5.7 KiB/s rd, 82 MiB/s wr, 1.31k op/s; 11 KiB/s, 0 objects/s recovering 2023-10-09T05:59:08.313 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:08.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:08 smithi156 ceph-mon[195101]: pgmap v672: 109 pgs: 109 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 5.7 KiB/s rd, 82 MiB/s wr, 1.31k op/s; 11 KiB/s, 0 objects/s recovering 2023-10-09T05:59:08.641 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:59:08.734 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:08.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:08.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:08.825 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:08.838 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:08.887 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:08.958 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:08.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:08.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:08.977 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 30 16 51643 51627 107.514 76.5 0.0116241 0.00929548 2023-10-09T05:59:08.996 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:59:08.996 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T05:59:08.996 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:59:09.064 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:09.092 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:59:09.112 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:09.133 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:59:09.176 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:59:09.226 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:59:09.263 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:59:09.284 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:09.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:09.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:09.303 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:59:09.332 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:59:09.353 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:59:09.360 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:59:09.360 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:59:09.360 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:59:09.360 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:59:09.360 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:59:09.360 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:59:09.360 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:59:09.360 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:59:09.389 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:09.439 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:09.492 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:09.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:09.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:09.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:09.626 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:09.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:09 smithi156 ceph-mon[195101]: pgmap v673: 109 pgs: 109 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 62 MiB/s wr, 996 op/s 2023-10-09T05:59:09.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:09 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:09.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:09 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1877557768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:09.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:09 smithi064 ceph-mon[212750]: pgmap v673: 109 pgs: 109 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 62 MiB/s wr, 996 op/s 2023-10-09T05:59:09.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:09 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:09.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:09 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1877557768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:09.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:09 smithi064 ceph-mon[220044]: pgmap v673: 109 pgs: 109 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 62 MiB/s wr, 996 op/s 2023-10-09T05:59:09.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:09 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:09.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:09 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1877557768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:09.977 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 31 16 52545 52529 105.865 56.375 0.152832 0.00944187 2023-10-09T05:59:10.636 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:59:10.636 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T05:59:10.636 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.636 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45263.0:53456 7.c 7:33ff9d26:::benchmark_data_smithi064_398483_object53455:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e295)", 2023-10-09T05:59:10.636 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:59:10.627976+0000", 2023-10-09T05:59:10.636 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.002819907, 2023-10-09T05:59:10.636 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0028540649999999998, 2023-10-09T05:59:10.637 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:10.637 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:59:10.637 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:59:10.637 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45263", 2023-10-09T05:59:10.637 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T05:59:10.637 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 53456 2023-10-09T05:59:10.637 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.637 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:10.637 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.637 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:10.637 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.627976+0000", 2023-10-09T05:59:10.637 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:10.638 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.638 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.638 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:10.638 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.627976+0000", 2023-10-09T05:59:10.638 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:10.638 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.638 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.638 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:10.638 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.627975+0000", 2023-10-09T05:59:10.638 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:59:10.638 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.639 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.639 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:10.639 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.628007+0000", 2023-10-09T05:59:10.639 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1633e-05 2023-10-09T05:59:10.639 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.639 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.639 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:10.639 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.628035+0000", 2023-10-09T05:59:10.639 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.8030000000000001e-05 2023-10-09T05:59:10.639 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.639 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.640 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:10.640 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.628041+0000", 2023-10-09T05:59:10.640 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4219999999999997e-06 2023-10-09T05:59:10.640 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.640 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.640 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:10.640 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.628091+0000", 2023-10-09T05:59:10.640 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9503999999999999e-05 2023-10-09T05:59:10.640 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.640 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.640 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:10.640 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.628247+0000", 2023-10-09T05:59:10.641 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015631900000000001 2023-10-09T05:59:10.641 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.641 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.641 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:59:10.641 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.628431+0000", 2023-10-09T05:59:10.641 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000183971 2023-10-09T05:59:10.641 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:10.641 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:10.641 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:10.641 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.641 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.641 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45263.0:53457 7.7 7:e7050a77:::benchmark_data_smithi064_398483_object53456:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e295)", 2023-10-09T05:59:10.642 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:59:10.628512+0000", 2023-10-09T05:59:10.642 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0022845640000000002, 2023-10-09T05:59:10.642 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0024012220000000002, 2023-10-09T05:59:10.642 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:10.642 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:59:10.642 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:59:10.642 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45263", 2023-10-09T05:59:10.642 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T05:59:10.642 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 53457 2023-10-09T05:59:10.642 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.642 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:10.643 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.643 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:10.643 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.628512+0000", 2023-10-09T05:59:10.643 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:10.643 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.643 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.643 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:10.643 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.628512+0000", 2023-10-09T05:59:10.643 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:10.643 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.643 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.643 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:10.644 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.628511+0000", 2023-10-09T05:59:10.644 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:59:10.644 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.644 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.644 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:10.644 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.628612+0000", 2023-10-09T05:59:10.644 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000101548 2023-10-09T05:59:10.644 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.644 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.644 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:10.644 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.628627+0000", 2023-10-09T05:59:10.644 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4567e-05 2023-10-09T05:59:10.645 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.645 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.645 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:10.645 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.628633+0000", 2023-10-09T05:59:10.645 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9429999999999997e-06 2023-10-09T05:59:10.645 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.645 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.645 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:10.645 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.628674+0000", 2023-10-09T05:59:10.645 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0649999999999999e-05 2023-10-09T05:59:10.645 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.645 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.646 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:10.646 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.628816+0000", 2023-10-09T05:59:10.646 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014217400000000001 2023-10-09T05:59:10.646 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.646 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.646 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:59:10.646 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.629006+0000", 2023-10-09T05:59:10.646 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00019049799999999999 2023-10-09T05:59:10.646 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:10.646 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:10.646 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:10.647 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.647 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.647 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45263.0:53460 7.4 e295/285 7:2d715e15:::benchmark_data_smithi064_398483_object53459:head v 295'3370, mlcod=295'3367)", 2023-10-09T05:59:10.647 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:59:10.630409+0000", 2023-10-09T05:59:10.647 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.00038695999999999999, 2023-10-09T05:59:10.647 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00057415600000000004, 2023-10-09T05:59:10.647 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:10.647 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T05:59:10.647 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:10.647 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.647 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:10.647 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.630409+0000", 2023-10-09T05:59:10.648 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:10.648 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.648 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.648 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:10.648 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.630409+0000", 2023-10-09T05:59:10.648 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:10.648 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.648 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.648 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:10.648 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.630408+0000", 2023-10-09T05:59:10.648 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:59:10.648 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.649 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.649 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:10.649 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.630449+0000", 2023-10-09T05:59:10.649 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0821000000000001e-05 2023-10-09T05:59:10.649 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.649 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.649 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:10.649 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.630460+0000", 2023-10-09T05:59:10.650 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1078e-05 2023-10-09T05:59:10.650 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.650 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.650 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:10.650 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.630466+0000", 2023-10-09T05:59:10.650 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0809999999999999e-06 2023-10-09T05:59:10.650 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.650 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.650 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:10.650 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.630499+0000", 2023-10-09T05:59:10.650 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.2725000000000003e-05 2023-10-09T05:59:10.651 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.651 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.651 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:10.651 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.630516+0000", 2023-10-09T05:59:10.651 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6951000000000001e-05 2023-10-09T05:59:10.651 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:10.651 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:10.651 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:10.651 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.651 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.651 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45263.0:53445 7.d 7:b565b69d:::benchmark_data_smithi064_398483_object53444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e295)", 2023-10-09T05:59:10.652 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:59:10.618705+0000", 2023-10-09T05:59:10.652 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.012090821999999999, 2023-10-09T05:59:10.652 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012347696999999999, 2023-10-09T05:59:10.652 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:10.652 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:59:10.652 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:59:10.652 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45263", 2023-10-09T05:59:10.652 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T05:59:10.652 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 53445 2023-10-09T05:59:10.652 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.653 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:10.653 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.653 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:10.653 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.618705+0000", 2023-10-09T05:59:10.653 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:10.653 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.653 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.653 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:10.653 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.618705+0000", 2023-10-09T05:59:10.653 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:10.653 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.653 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.654 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:10.654 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.618705+0000", 2023-10-09T05:59:10.654 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:59:10.654 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.654 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.654 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:10.654 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.618739+0000", 2023-10-09T05:59:10.654 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.3779999999999998e-05 2023-10-09T05:59:10.654 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.654 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.654 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:10.655 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.618745+0000", 2023-10-09T05:59:10.655 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8460000000000002e-06 2023-10-09T05:59:10.655 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.655 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.655 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:10.655 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.618750+0000", 2023-10-09T05:59:10.655 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3470000000000001e-06 2023-10-09T05:59:10.655 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.657 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.657 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:10.657 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.623354+0000", 2023-10-09T05:59:10.657 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0046043400000000002 2023-10-09T05:59:10.658 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.658 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.658 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:10.658 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.623505+0000", 2023-10-09T05:59:10.658 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015137199999999999 2023-10-09T05:59:10.658 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.658 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.658 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:59:10.658 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.623662+0000", 2023-10-09T05:59:10.658 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015658600000000001 2023-10-09T05:59:10.658 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.659 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.659 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:59:10.659 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.627356+0000", 2023-10-09T05:59:10.659 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.003693843 2023-10-09T05:59:10.659 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:10.659 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:10.659 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:10.659 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.659 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.659 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45263.0:53447 7.c 7:32740707:::benchmark_data_smithi064_398483_object53446:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e295)", 2023-10-09T05:59:10.659 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:59:10.621113+0000", 2023-10-09T05:59:10.660 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0096830240000000001, 2023-10-09T05:59:10.660 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010019271999999999, 2023-10-09T05:59:10.660 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:10.660 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:59:10.660 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:59:10.660 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45263", 2023-10-09T05:59:10.660 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T05:59:10.660 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 53447 2023-10-09T05:59:10.660 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.660 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:10.660 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.660 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:10.661 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.621113+0000", 2023-10-09T05:59:10.661 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:10.661 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.661 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.661 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:10.661 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.621113+0000", 2023-10-09T05:59:10.661 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:10.661 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.673 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.673 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:10.673 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.621112+0000", 2023-10-09T05:59:10.673 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:59:10.674 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.674 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.674 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:10.674 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.621145+0000", 2023-10-09T05:59:10.674 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.3117000000000002e-05 2023-10-09T05:59:10.674 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.674 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.674 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:10.674 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.621162+0000", 2023-10-09T05:59:10.674 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7311000000000001e-05 2023-10-09T05:59:10.674 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.675 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.675 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:10.675 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.621169+0000", 2023-10-09T05:59:10.675 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6900000000000003e-06 2023-10-09T05:59:10.675 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.675 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.675 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:10.675 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.625251+0000", 2023-10-09T05:59:10.675 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0040819619999999997 2023-10-09T05:59:10.675 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.675 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.675 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:10.676 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.625370+0000", 2023-10-09T05:59:10.676 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000119184 2023-10-09T05:59:10.676 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.676 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.676 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:59:10.676 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.625505+0000", 2023-10-09T05:59:10.676 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000134654 2023-10-09T05:59:10.676 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.676 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.676 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:59:10.676 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.630448+0000", 2023-10-09T05:59:10.676 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0049432770000000003 2023-10-09T05:59:10.676 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:10.677 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:10.677 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:10.677 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:59:10.630647+0000", 2023-10-09T05:59:10.677 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00019847800000000001 2023-10-09T05:59:10.677 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:10.677 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:10.677 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:10.677 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:10.677 INFO:teuthology.orchestra.run.smithi156.stdout: ], 2023-10-09T05:59:10.677 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 5 2023-10-09T05:59:10.677 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:59:10.977 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 32 16 53866 53850 105.137 82.5625 0.00583394 0.0095042 2023-10-09T05:59:11.568 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:59:11.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:11 smithi064 ceph-mon[212750]: pgmap v674: 109 pgs: 109 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T05:59:11.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:11 smithi064 ceph-mon[220044]: pgmap v674: 109 pgs: 109 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T05:59:11.784 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:11.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:11.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:11.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:11 smithi156 ceph-mon[195101]: pgmap v674: 109 pgs: 109 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T05:59:11.890 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:11.936 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:11.977 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 33 16 54800 54784 103.72 58.375 0.00920786 0.00963729 2023-10-09T05:59:12.016 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:12.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:12.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:12.089 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:59:12.124 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:12.191 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:12.291 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:12.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:12.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:12.398 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:12.469 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:12.698 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:12.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:12.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:12.808 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:12.842 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:12.893 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:12.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:12.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:12.977 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 34 16 55924 55908 102.736 70.25 0.00299682 0.00970629 2023-10-09T05:59:12.999 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:13.028 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:13.076 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:13.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:13.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:13.180 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:13.233 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:13.303 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T05:59:13.337 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:13 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3796194594' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:59:13.337 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:13 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:59:13.341 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:13 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3796194594' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:59:13.341 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:13 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:59:13.500 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:13.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:13.513 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:13 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3796194594' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:59:13.513 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:13 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:59:13.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:13.614 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:13.691 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:13.795 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:13.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:13.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:13.906 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:13.969 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:13.977 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 35 16 57100 57084 101.9 73.5 0.00728019 0.00980839 2023-10-09T05:59:14.067 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:14.071 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:59:14.071 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T05:59:14.072 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T05:59:14.072 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T05:59:14.072 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:59:14.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:14.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:14.142 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:59:14.174 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:14.217 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:14.308 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:14.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:14.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:14 smithi156 ceph-mon[195101]: pgmap v675: 109 pgs: 109 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T05:59:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:14 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:14 smithi156 ceph-mon[195101]: failed to encode map e296 with expected crc 2023-10-09T05:59:14.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:14 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:59:14.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:14 smithi156 ceph-mon[195101]: osdmap e296: 8 total, 8 up, 7 in 2023-10-09T05:59:14.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:14 smithi156 ceph-mon[195101]: failed to encode map e296 with expected crc 2023-10-09T05:59:14.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:14 smithi156 ceph-mon[195101]: failed to encode map e296 with expected crc 2023-10-09T05:59:14.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:14 smithi156 ceph-mon[195101]: failed to encode map e296 with expected crc 2023-10-09T05:59:14.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:14.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:14 smithi156 ceph-mon[195101]: 1.a scrub starts 2023-10-09T05:59:14.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:14 smithi156 ceph-mon[195101]: 1.a scrub ok 2023-10-09T05:59:14.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:59:14.412 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:14.507 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:14.574 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:14 smithi064 ceph-mon[212750]: pgmap v675: 109 pgs: 109 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T05:59:14.574 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:14 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:14.574 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:14 smithi064 ceph-mon[212750]: failed to encode map e296 with expected crc 2023-10-09T05:59:14.574 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:14 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:59:14.574 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:14 smithi064 ceph-mon[212750]: osdmap e296: 8 total, 8 up, 7 in 2023-10-09T05:59:14.574 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:14 smithi064 ceph-mon[212750]: failed to encode map e296 with expected crc 2023-10-09T05:59:14.574 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:14 smithi064 ceph-mon[212750]: failed to encode map e296 with expected crc 2023-10-09T05:59:14.575 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:14 smithi064 ceph-mon[212750]: failed to encode map e296 with expected crc 2023-10-09T05:59:14.575 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:14.575 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:14 smithi064 ceph-mon[212750]: 1.a scrub starts 2023-10-09T05:59:14.575 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:14 smithi064 ceph-mon[212750]: 1.a scrub ok 2023-10-09T05:59:14.575 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:59:14.575 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:14 smithi064 ceph-mon[220044]: pgmap v675: 109 pgs: 109 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T05:59:14.575 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:14 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:14.575 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:14 smithi064 ceph-mon[220044]: failed to encode map e296 with expected crc 2023-10-09T05:59:14.575 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:14 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:59:14.575 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:14 smithi064 ceph-mon[220044]: osdmap e296: 8 total, 8 up, 7 in 2023-10-09T05:59:14.575 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:14 smithi064 ceph-mon[220044]: failed to encode map e296 with expected crc 2023-10-09T05:59:14.576 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:14 smithi064 ceph-mon[220044]: failed to encode map e296 with expected crc 2023-10-09T05:59:14.576 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:14 smithi064 ceph-mon[220044]: failed to encode map e296 with expected crc 2023-10-09T05:59:14.576 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:14.576 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:14 smithi064 ceph-mon[220044]: 1.a scrub starts 2023-10-09T05:59:14.576 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:14 smithi064 ceph-mon[220044]: 1.a scrub ok 2023-10-09T05:59:14.576 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:59:14.612 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:14.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:14.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:14.717 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:14.755 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:14.799 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:14.977 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 36 15 58011 57996 100.653 57 0.0120291 0.0099303 2023-10-09T05:59:14.996 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:15.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:15.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:15.103 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:15.139 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:59:15.157 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:15.303 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:15.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:15.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:15.414 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:15.470 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:59:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:59:15] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.43.0" 2023-10-09T05:59:15.494 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:15.591 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T05:59:15.591 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:59:15.591 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T05:59:15.592 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:59:15.817 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:15.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:15.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:15.851 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:59:15.926 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:15.978 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 37 16 59171 59155 99.8883 72.4375 0.0153759 0.0100055 2023-10-09T05:59:16.013 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:16.595 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:59:16.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:16 smithi156 ceph-mon[195101]: pgmap v677: 109 pgs: 109 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T05:59:16.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:16 smithi156 ceph-mon[195101]: 1.10 scrub starts 2023-10-09T05:59:16.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:16 smithi156 ceph-mon[195101]: 1.10 scrub ok 2023-10-09T05:59:16.650 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:16 smithi064 ceph-mon[212750]: pgmap v677: 109 pgs: 109 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T05:59:16.650 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:16 smithi064 ceph-mon[212750]: 1.10 scrub starts 2023-10-09T05:59:16.650 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:16 smithi064 ceph-mon[212750]: 1.10 scrub ok 2023-10-09T05:59:16.650 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:16 smithi064 ceph-mon[220044]: pgmap v677: 109 pgs: 109 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T05:59:16.650 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:16 smithi064 ceph-mon[220044]: 1.10 scrub starts 2023-10-09T05:59:16.650 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:16 smithi064 ceph-mon[220044]: 1.10 scrub ok 2023-10-09T05:59:16.978 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 38 16 60144 60128 98.8601 60.8125 0.012351 0.0101096 2023-10-09T05:59:17.026 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:17.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:17.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:17.144 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:17.191 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:17.243 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:17.298 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:17 smithi064 ceph-mon[212750]: 1.d scrub starts 2023-10-09T05:59:17.298 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:17 smithi064 ceph-mon[212750]: 1.d scrub ok 2023-10-09T05:59:17.408 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:17.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:17.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:17.437 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:17.513 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:17.553 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:17 smithi156 ceph-mon[195101]: 1.d scrub starts 2023-10-09T05:59:17.553 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:17 smithi156 ceph-mon[195101]: 1.d scrub ok 2023-10-09T05:59:17.557 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:17.626 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:17 smithi064 ceph-mon[220044]: 1.d scrub starts 2023-10-09T05:59:17.626 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:17 smithi064 ceph-mon[220044]: 1.d scrub ok 2023-10-09T05:59:17.911 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:17.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:17.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:17.979 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 39 16 61085 61069 97.8333 58.8125 0.0146847 0.0102149 2023-10-09T05:59:18.019 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:18.021 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:59:18.074 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:59:18.140 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:18.141 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:59:18.203 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:59:18.237 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:59:18.276 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:59:18.312 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:59:18.358 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:59:18.365 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:59:18.365 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:59:18.365 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:59:18.365 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:59:18.365 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:59:18.366 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:59:18.366 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:59:18.366 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:59:18.568 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T05:59:18.568 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:59:18.601 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:18 smithi156 ceph-mon[195101]: pgmap v678: 109 pgs: 109 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T05:59:18.601 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:18 smithi156 ceph-mon[195101]: 1.8 scrub starts 2023-10-09T05:59:18.601 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:18 smithi156 ceph-mon[195101]: 1.8 scrub ok 2023-10-09T05:59:18.601 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:18 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3678413014' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:18.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:18 smithi064 ceph-mon[220044]: pgmap v678: 109 pgs: 109 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T05:59:18.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:18 smithi064 ceph-mon[220044]: 1.8 scrub starts 2023-10-09T05:59:18.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:18 smithi064 ceph-mon[220044]: 1.8 scrub ok 2023-10-09T05:59:18.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:18 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3678413014' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:18.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:18 smithi064 ceph-mon[212750]: pgmap v678: 109 pgs: 109 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T05:59:18.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:18 smithi064 ceph-mon[212750]: 1.8 scrub starts 2023-10-09T05:59:18.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:18 smithi064 ceph-mon[212750]: 1.8 scrub ok 2023-10-09T05:59:18.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:18 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3678413014' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:18.979 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-10-09T05:59:18.979516+0000 min lat: 0.00127391 max lat: 1.01073 avg lat: 0.0102813 2023-10-09T05:59:18.979 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:59:18.979 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 40 16 62256 62240 97.2171 73.1875 0.00607037 0.0102813 2023-10-09T05:59:19.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:19 smithi156 ceph-mon[195101]: pgmap v679: 109 pgs: 109 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T05:59:19.717 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:59:19.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:19 smithi064 ceph-mon[212750]: pgmap v679: 109 pgs: 109 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T05:59:19.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:19 smithi064 ceph-mon[220044]: pgmap v679: 109 pgs: 109 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T05:59:19.979 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 41 16 63101 63085 96.1341 52.8125 0.00338103 0.0103829 2023-10-09T05:59:20.221 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:20.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:20.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:20.328 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:20.371 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:59:20.407 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:20.499 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:20.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:20.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:20.604 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:20.684 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:20.867 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:20.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:20.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:20.975 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:20.979 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 42 16 64233 64217 95.5297 70.75 0.0141302 0.0104623 2023-10-09T05:59:21.065 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:21.232 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:21.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:21.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:21.346 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:21.444 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:21.705 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:21.708 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:21 smithi064 ceph-mon[212750]: pgmap v680: 109 pgs: 109 active+clean; 3.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T05:59:21.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:21.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:21.810 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:21.835 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T05:59:21.854 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T05:59:21.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:21 smithi156 ceph-mon[195101]: pgmap v680: 109 pgs: 109 active+clean; 3.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T05:59:21.906 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:21.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:21 smithi064 ceph-mon[220044]: pgmap v680: 109 pgs: 109 active+clean; 3.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T05:59:21.982 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 43 16 65070 65054 94.5188 52.3125 0.010129 0.0105733 2023-10-09T05:59:22.099 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:22.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:22.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:22.210 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:22.293 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:22.345 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:59:22.367 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:22.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:22.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:22.474 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:22.523 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:22.648 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:22.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:22.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:22.712 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:22 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2193809728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:22.712 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:22 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:22.712 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:22 smithi064 ceph-mon[212750]: from='client.45311 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:59:22.712 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:22 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:22.713 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:22 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:22.718 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:22 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2193809728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:22.718 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:22 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:22.718 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:22 smithi064 ceph-mon[220044]: from='client.45311 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:59:22.718 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:22 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:22.718 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:22 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:22.750 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T05:59:22.757 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:22.809 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:22.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:22 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2193809728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:22.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:22 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:22.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:22 smithi156 ceph-mon[195101]: from='client.45311 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:59:22.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:22 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:22.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:22 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:22.903 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T05:59:22.903 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:59:22.982 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 44 16 65985 65969 93.6704 57.1875 0.00311672 0.0106261 2023-10-09T05:59:23.824 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:23 smithi064 ceph-mon[220044]: pgmap v681: 109 pgs: 109 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T05:59:23.825 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:23 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:59:23.825 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:23 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T05:59:23.825 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:23 smithi064 ceph-mon[220044]: failed to encode map e297 with expected crc 2023-10-09T05:59:23.825 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:23 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:59:23.825 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:23 smithi064 ceph-mon[220044]: osdmap e297: 8 total, 8 up, 7 in 2023-10-09T05:59:23.825 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:23 smithi064 ceph-mon[220044]: failed to encode map e297 with expected crc 2023-10-09T05:59:23.825 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:23 smithi064 ceph-mon[220044]: failed to encode map e297 with expected crc 2023-10-09T05:59:23.825 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:23 smithi064 ceph-mon[220044]: failed to encode map e297 with expected crc 2023-10-09T05:59:23.826 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:23 smithi064 ceph-mon[212750]: pgmap v681: 109 pgs: 109 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T05:59:23.826 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:23 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:59:23.826 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:23 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T05:59:23.826 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:23 smithi064 ceph-mon[212750]: failed to encode map e297 with expected crc 2023-10-09T05:59:23.826 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:23 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:59:23.826 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:23 smithi064 ceph-mon[212750]: osdmap e297: 8 total, 8 up, 7 in 2023-10-09T05:59:23.826 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:23 smithi064 ceph-mon[212750]: failed to encode map e297 with expected crc 2023-10-09T05:59:23.826 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:23 smithi064 ceph-mon[212750]: failed to encode map e297 with expected crc 2023-10-09T05:59:23.827 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:23 smithi064 ceph-mon[212750]: failed to encode map e297 with expected crc 2023-10-09T05:59:23.857 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:59:23.903 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":680,"stamp":"2023-10-09T05:59:20.646500+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4101773216,"num_objects":62603,"num_object_clones":0,"num_object_copies":187809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":62603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62640,"num_write_kb":4005085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":10792,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":66015,"ondisk_log_size":66015,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":330,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14142548,"kb_used_data":12009308,"kb_used_omap":118,"kb_used_meta":2133065,"kb_avail":642073516,"statfs":{"total":671965249536,"available":657483280384,"internally_reserved":0,"allocated":12297531392,"data_stored":12288429639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":121648,"internal_metadata":2184258768},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,4,5,4,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":694390063,"num_objects":10595,"num_object_clones":0,"num_object_copies":31785,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10595,"num_whiteouts":0,"num_read":-132,"num_read_kb":-249,"num_write":10529,"num_write_kb":677358,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":-483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007206"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":341,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.321433+0000","last_change":"2023-10-09T05:55:59.341215+0000","last_active":"2023-10-09T05:59:04.321433+0000","last_peered":"2023-10-09T05:59:04.321433+0000","last_clean":"2023-10-09T05:59:04.321433+0000","last_became_active":"2023-10-09T05:55:59.340856+0000","last_became_peered":"2023-10-09T05:55:59.340856+0000","last_unstale":"2023-10-09T05:59:04.321433+0000","last_undegraded":"2023-10-09T05:59:04.321433+0000","last_fullsized":"2023-10-09T05:59:04.321433+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:33.686454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":58,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.274764+0000","last_change":"2023-10-09T05:57:23.474348+0000","last_active":"2023-10-09T05:59:04.274764+0000","last_peered":"2023-10-09T05:59:04.274764+0000","last_clean":"2023-10-09T05:59:04.274764+0000","last_became_active":"2023-10-09T05:57:23.474010+0000","last_became_peered":"2023-10-09T05:57:23.474010+0000","last_unstale":"2023-10-09T05:59:04.274764+0000","last_undegraded":"2023-10-09T05:59:04.274764+0000","last_fullsized":"2023-10-09T05:59:04.274764+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:45.039777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":175,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.275168+0000","last_change":"2023-10-09T05:55:59.304893+0000","last_active":"2023-10-09T05:59:04.275168+0000","last_peered":"2023-10-09T05:59:04.275168+0000","last_clean":"2023-10-09T05:59:04.275168+0000","last_became_active":"2023-10-09T05:55:59.304449+0000","last_became_peered":"2023-10-09T05:55:59.304449+0000","last_unstale":"2023-10-09T05:59:04.275168+0000","last_undegraded":"2023-10-09T05:59:04.275168+0000","last_fullsized":"2023-10-09T05:59:04.275168+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:44:39.453982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":316,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.320557+0000","last_change":"2023-10-09T05:57:58.834014+0000","last_active":"2023-10-09T05:59:04.320557+0000","last_peered":"2023-10-09T05:59:04.320557+0000","last_clean":"2023-10-09T05:59:04.320557+0000","last_became_active":"2023-10-09T05:57:58.833718+0000","last_became_peered":"2023-10-09T05:57:58.833718+0000","last_unstale":"2023-10-09T05:59:04.320557+0000","last_undegraded":"2023-10-09T05:59:04.320557+0000","last_fullsized":"2023-10-09T05:59:04.320557+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:23.752353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":58,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.274045+0000","last_change":"2023-10-09T05:57:23.476832+0000","last_active":"2023-10-09T05:59:04.274045+0000","last_peered":"2023-10-09T05:59:04.274045+0000","last_clean":"2023-10-09T05:59:04.274045+0000","last_became_active":"2023-10-09T05:57:23.475216+0000","last_became_peered":"2023-10-09T05:57:23.475216+0000","last_unstale":"2023-10-09T05:59:04.274045+0000","last_undegraded":"2023-10-09T05:59:04.274045+0000","last_fullsized":"2023-10-09T05:59:04.274045+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:48.279660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":322,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.335248+0000","last_change":"2023-10-09T05:55:59.335464+0000","last_active":"2023-10-09T05:59:04.335248+0000","last_peered":"2023-10-09T05:59:04.335248+0000","last_clean":"2023-10-09T05:59:04.335248+0000","last_became_active":"2023-10-09T05:55:59.335259+0000","last_became_peered":"2023-10-09T05:55:59.335259+0000","last_unstale":"2023-10-09T05:59:04.335248+0000","last_undegraded":"2023-10-09T05:59:04.335248+0000","last_fullsized":"2023-10-09T05:59:04.335248+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:28.536682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":251,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.320423+0000","last_change":"2023-10-09T05:57:23.477980+0000","last_active":"2023-10-09T05:59:04.320423+0000","last_peered":"2023-10-09T05:59:04.320423+0000","last_clean":"2023-10-09T05:59:04.320423+0000","last_became_active":"2023-10-09T05:57:23.477654+0000","last_became_peered":"2023-10-09T05:57:23.477654+0000","last_unstale":"2023-10-09T05:59:04.320423+0000","last_undegraded":"2023-10-09T05:59:04.320423+0000","last_fullsized":"2023-10-09T05:59:04.320423+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:17.524683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":58,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.274074+0000","last_change":"2023-10-09T05:57:23.473644+0000","last_active":"2023-10-09T05:59:04.274074+0000","last_peered":"2023-10-09T05:59:04.274074+0000","last_clean":"2023-10-09T05:59:04.274074+0000","last_became_active":"2023-10-09T05:57:23.473391+0000","last_became_peered":"2023-10-09T05:57:23.473391+0000","last_unstale":"2023-10-09T05:59:04.274074+0000","last_undegraded":"2023-10-09T05:59:04.274074+0000","last_fullsized":"2023-10-09T05:59:04.274074+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:17.138034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":560,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.322158+0000","last_change":"2023-10-09T05:58:36.913116+0000","last_active":"2023-10-09T05:59:04.322158+0000","last_peered":"2023-10-09T05:59:04.322158+0000","last_clean":"2023-10-09T05:59:04.322158+0000","last_became_active":"2023-10-09T05:58:21.329980+0000","last_became_peered":"2023-10-09T05:58:21.329980+0000","last_unstale":"2023-10-09T05:59:04.322158+0000","last_undegraded":"2023-10-09T05:59:04.322158+0000","last_fullsized":"2023-10-09T05:59:04.322158+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.913004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.913004+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:32:18.561414+0000","scrub_duration":0.0017791980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":335,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.319453+0000","last_change":"2023-10-09T05:57:23.478483+0000","last_active":"2023-10-09T05:59:04.319453+0000","last_peered":"2023-10-09T05:59:04.319453+0000","last_clean":"2023-10-09T05:59:04.319453+0000","last_became_active":"2023-10-09T05:57:23.478122+0000","last_became_peered":"2023-10-09T05:57:23.478122+0000","last_unstale":"2023-10-09T05:59:04.319453+0000","last_undegraded":"2023-10-09T05:59:04.319453+0000","last_fullsized":"2023-10-09T05:59:04.319453+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:37.768717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":240,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.275123+0000","last_change":"2023-10-09T05:53:59.185702+0000","last_active":"2023-10-09T05:59:04.275123+0000","last_peered":"2023-10-09T05:59:04.275123+0000","last_clean":"2023-10-09T05:59:04.275123+0000","last_became_active":"2023-10-09T05:53:59.185373+0000","last_became_peered":"2023-10-09T05:53:59.185373+0000","last_unstale":"2023-10-09T05:59:04.275123+0000","last_undegraded":"2023-10-09T05:59:04.275123+0000","last_fullsized":"2023-10-09T05:59:04.275123+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:50.756967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":136,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.320696+0000","last_change":"2023-10-09T05:53:59.219168+0000","last_active":"2023-10-09T05:59:04.320696+0000","last_peered":"2023-10-09T05:59:04.320696+0000","last_clean":"2023-10-09T05:59:04.320696+0000","last_became_active":"2023-10-09T05:53:59.200886+0000","last_became_peered":"2023-10-09T05:53:59.200886+0000","last_unstale":"2023-10-09T05:59:04.320696+0000","last_undegraded":"2023-10-09T05:59:04.320696+0000","last_fullsized":"2023-10-09T05:59:04.320696+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:45.515036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":190,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.335370+0000","last_change":"2023-10-09T05:55:59.305100+0000","last_active":"2023-10-09T05:59:04.335370+0000","last_peered":"2023-10-09T05:59:04.335370+0000","last_clean":"2023-10-09T05:59:04.335370+0000","last_became_active":"2023-10-09T05:55:59.304758+0000","last_became_peered":"2023-10-09T05:55:59.304758+0000","last_unstale":"2023-10-09T05:59:04.335370+0000","last_undegraded":"2023-10-09T05:59:04.335370+0000","last_fullsized":"2023-10-09T05:59:04.335370+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:54:32.510945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":301,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.321565+0000","last_change":"2023-10-09T05:53:59.219141+0000","last_active":"2023-10-09T05:59:04.321565+0000","last_peered":"2023-10-09T05:59:04.321565+0000","last_clean":"2023-10-09T05:59:04.321565+0000","last_became_active":"2023-10-09T05:53:59.197318+0000","last_became_peered":"2023-10-09T05:53:59.197318+0000","last_unstale":"2023-10-09T05:59:04.321565+0000","last_undegraded":"2023-10-09T05:59:04.321565+0000","last_fullsized":"2023-10-09T05:59:04.321565+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:39.910794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":305,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.321628+0000","last_change":"2023-10-09T05:55:59.335824+0000","last_active":"2023-10-09T05:59:04.321628+0000","last_peered":"2023-10-09T05:59:04.321628+0000","last_clean":"2023-10-09T05:59:04.321628+0000","last_became_active":"2023-10-09T05:55:59.335478+0000","last_became_peered":"2023-10-09T05:55:59.335478+0000","last_unstale":"2023-10-09T05:59:04.321628+0000","last_undegraded":"2023-10-09T05:59:04.321628+0000","last_fullsized":"2023-10-09T05:59:04.321628+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:26.014501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":516,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:09.765233+0000","last_change":"2023-10-09T05:58:38.230201+0000","last_active":"2023-10-09T05:59:09.765233+0000","last_peered":"2023-10-09T05:59:09.765233+0000","last_clean":"2023-10-09T05:59:09.765233+0000","last_became_active":"2023-10-09T05:58:21.332729+0000","last_became_peered":"2023-10-09T05:58:21.332729+0000","last_unstale":"2023-10-09T05:59:09.765233+0000","last_undegraded":"2023-10-09T05:59:09.765233+0000","last_fullsized":"2023-10-09T05:59:09.765233+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:38.230079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:38.230079+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:33.800036+0000","scrub_duration":0.001618488,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":138,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.319181+0000","last_change":"2023-10-09T05:53:59.233898+0000","last_active":"2023-10-09T05:59:04.319181+0000","last_peered":"2023-10-09T05:59:04.319181+0000","last_clean":"2023-10-09T05:59:04.319181+0000","last_became_active":"2023-10-09T05:53:59.204175+0000","last_became_peered":"2023-10-09T05:53:59.204175+0000","last_unstale":"2023-10-09T05:59:04.319181+0000","last_undegraded":"2023-10-09T05:59:04.319181+0000","last_fullsized":"2023-10-09T05:59:04.319181+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:03.139318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":326,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.275007+0000","last_change":"2023-10-09T05:55:59.297087+0000","last_active":"2023-10-09T05:59:04.275007+0000","last_peered":"2023-10-09T05:59:04.275007+0000","last_clean":"2023-10-09T05:59:04.275007+0000","last_became_active":"2023-10-09T05:55:59.296768+0000","last_became_peered":"2023-10-09T05:55:59.296768+0000","last_unstale":"2023-10-09T05:59:04.275007+0000","last_undegraded":"2023-10-09T05:59:04.275007+0000","last_fullsized":"2023-10-09T05:59:04.275007+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:35.067705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":719,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:59:15.170604+0000","last_change":"2023-10-09T05:59:15.170543+0000","last_active":"2023-10-09T05:59:15.170604+0000","last_peered":"2023-10-09T05:59:15.170604+0000","last_clean":"2023-10-09T05:59:15.170604+0000","last_became_active":"2023-10-09T05:58:21.332533+0000","last_became_peered":"2023-10-09T05:58:21.332533+0000","last_unstale":"2023-10-09T05:59:15.170604+0000","last_undegraded":"2023-10-09T05:59:15.170604+0000","last_fullsized":"2023-10-09T05:59:15.170604+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:33.995695+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":298,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.320208+0000","last_change":"2023-10-09T05:57:23.473445+0000","last_active":"2023-10-09T05:59:04.320208+0000","last_peered":"2023-10-09T05:59:04.320208+0000","last_clean":"2023-10-09T05:59:04.320208+0000","last_became_active":"2023-10-09T05:57:23.473103+0000","last_became_peered":"2023-10-09T05:57:23.473103+0000","last_unstale":"2023-10-09T05:59:04.320208+0000","last_undegraded":"2023-10-09T05:59:04.320208+0000","last_fullsized":"2023-10-09T05:59:04.320208+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:39:53.825734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":219,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.320627+0000","last_change":"2023-10-09T05:56:49.237784+0000","last_active":"2023-10-09T05:59:04.320627+0000","last_peered":"2023-10-09T05:59:04.320627+0000","last_clean":"2023-10-09T05:59:04.320627+0000","last_became_active":"2023-10-09T05:56:49.237417+0000","last_became_peered":"2023-10-09T05:56:49.237417+0000","last_unstale":"2023-10-09T05:59:04.320627+0000","last_undegraded":"2023-10-09T05:59:04.320627+0000","last_fullsized":"2023-10-09T05:59:04.320627+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:33.129268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":546,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.274518+0000","last_change":"2023-10-09T05:58:37.295185+0000","last_active":"2023-10-09T05:59:04.274518+0000","last_peered":"2023-10-09T05:59:04.274518+0000","last_clean":"2023-10-09T05:59:04.274518+0000","last_became_active":"2023-10-09T05:58:21.327911+0000","last_became_peered":"2023-10-09T05:58:21.327911+0000","last_unstale":"2023-10-09T05:59:04.274518+0000","last_undegraded":"2023-10-09T05:59:04.274518+0000","last_fullsized":"2023-10-09T05:59:04.274518+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:37.295094+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:37.295094+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:10.882014+0000","scrub_duration":0.0017309510000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":132,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.318919+0000","last_change":"2023-10-09T05:53:59.206887+0000","last_active":"2023-10-09T05:59:04.318919+0000","last_peered":"2023-10-09T05:59:04.318919+0000","last_clean":"2023-10-09T05:59:04.318919+0000","last_became_active":"2023-10-09T05:53:59.206608+0000","last_became_peered":"2023-10-09T05:53:59.206608+0000","last_unstale":"2023-10-09T05:59:04.318919+0000","last_undegraded":"2023-10-09T05:59:04.318919+0000","last_fullsized":"2023-10-09T05:59:04.318919+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:48.975792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":278,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.320760+0000","last_change":"2023-10-09T05:57:58.828837+0000","last_active":"2023-10-09T05:59:04.320760+0000","last_peered":"2023-10-09T05:59:04.320760+0000","last_clean":"2023-10-09T05:59:04.320760+0000","last_became_active":"2023-10-09T05:57:58.828503+0000","last_became_peered":"2023-10-09T05:57:58.828503+0000","last_unstale":"2023-10-09T05:59:04.320760+0000","last_undegraded":"2023-10-09T05:59:04.320760+0000","last_fullsized":"2023-10-09T05:59:04.320760+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:08.332644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":596,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.333293+0000","last_change":"2023-10-09T05:58:46.725484+0000","last_active":"2023-10-09T05:59:04.333293+0000","last_peered":"2023-10-09T05:59:04.333293+0000","last_clean":"2023-10-09T05:59:04.333293+0000","last_became_active":"2023-10-09T05:58:21.332713+0000","last_became_peered":"2023-10-09T05:58:21.332713+0000","last_unstale":"2023-10-09T05:59:04.333293+0000","last_undegraded":"2023-10-09T05:59:04.333293+0000","last_fullsized":"2023-10-09T05:59:04.333293+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:46.725374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:46.725374+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:34.070739+0000","scrub_duration":0.002074771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":281,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.333158+0000","last_change":"2023-10-09T05:51:57.091639+0000","last_active":"2023-10-09T05:59:04.333158+0000","last_peered":"2023-10-09T05:59:04.333158+0000","last_clean":"2023-10-09T05:59:04.333158+0000","last_became_active":"2023-10-09T05:51:57.091284+0000","last_became_peered":"2023-10-09T05:51:57.091284+0000","last_unstale":"2023-10-09T05:59:04.333158+0000","last_undegraded":"2023-10-09T05:59:04.333158+0000","last_fullsized":"2023-10-09T05:59:04.333158+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:10.553453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":67,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.274517+0000","last_change":"2023-10-09T05:57:58.833939+0000","last_active":"2023-10-09T05:59:04.274517+0000","last_peered":"2023-10-09T05:59:04.274517+0000","last_clean":"2023-10-09T05:59:04.274517+0000","last_became_active":"2023-10-09T05:57:58.833567+0000","last_became_peered":"2023-10-09T05:57:58.833567+0000","last_unstale":"2023-10-09T05:59:04.274517+0000","last_undegraded":"2023-10-09T05:59:04.274517+0000","last_fullsized":"2023-10-09T05:59:04.274517+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:58.347572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"296'3869","reported_seq":3888,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:59:18.393493+0000","last_change":"2023-10-09T05:58:27.405856+0000","last_active":"2023-10-09T05:59:18.393493+0000","last_peered":"2023-10-09T05:59:18.393493+0000","last_clean":"2023-10-09T05:59:18.393493+0000","last_became_active":"2023-10-09T05:58:27.405560+0000","last_became_peered":"2023-10-09T05:58:27.405560+0000","last_unstale":"2023-10-09T05:59:18.393493+0000","last_undegraded":"2023-10-09T05:59:18.393493+0000","last_fullsized":"2023-10-09T05:59:18.393493+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":3869,"log_dups_size":0,"ondisk_log_size":3869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:24.937739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253558784,"num_objects":3869,"num_object_clones":0,"num_object_copies":11607,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3869,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3869,"num_write_kb":247616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":615,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:59:16.165398+0000","last_change":"2023-10-09T05:59:16.165359+0000","last_active":"2023-10-09T05:59:16.165398+0000","last_peered":"2023-10-09T05:59:16.165398+0000","last_clean":"2023-10-09T05:59:16.165398+0000","last_became_active":"2023-10-09T05:58:21.333365+0000","last_became_peered":"2023-10-09T05:58:21.333365+0000","last_unstale":"2023-10-09T05:59:16.165398+0000","last_undegraded":"2023-10-09T05:59:16.165398+0000","last_fullsized":"2023-10-09T05:59:16.165398+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:16.165290+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:16.165290+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:13.429418+0000","scrub_duration":0.0081608050000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":136,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.319935+0000","last_change":"2023-10-09T05:53:59.222136+0000","last_active":"2023-10-09T05:59:04.319935+0000","last_peered":"2023-10-09T05:59:04.319935+0000","last_clean":"2023-10-09T05:59:04.319935+0000","last_became_active":"2023-10-09T05:53:59.201853+0000","last_became_peered":"2023-10-09T05:53:59.201853+0000","last_unstale":"2023-10-09T05:59:04.319935+0000","last_undegraded":"2023-10-09T05:59:04.319935+0000","last_fullsized":"2023-10-09T05:59:04.319935+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:26.662031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":202,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.320604+0000","last_change":"2023-10-09T05:56:49.234185+0000","last_active":"2023-10-09T05:59:04.320604+0000","last_peered":"2023-10-09T05:59:04.320604+0000","last_clean":"2023-10-09T05:59:04.320604+0000","last_became_active":"2023-10-09T05:56:49.233884+0000","last_became_peered":"2023-10-09T05:56:49.233884+0000","last_unstale":"2023-10-09T05:59:04.320604+0000","last_undegraded":"2023-10-09T05:59:04.320604+0000","last_fullsized":"2023-10-09T05:59:04.320604+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:12.142436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.a","version":"296'3961","reported_seq":3981,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:59:19.739170+0000","last_change":"2023-10-09T05:58:27.406918+0000","last_active":"2023-10-09T05:59:19.739170+0000","last_peered":"2023-10-09T05:59:19.739170+0000","last_clean":"2023-10-09T05:59:19.739170+0000","last_became_active":"2023-10-09T05:58:27.406600+0000","last_became_peered":"2023-10-09T05:58:27.406600+0000","last_unstale":"2023-10-09T05:59:19.739170+0000","last_undegraded":"2023-10-09T05:59:19.739170+0000","last_fullsized":"2023-10-09T05:59:19.739170+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":3961,"log_dups_size":0,"ondisk_log_size":3961,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:00.484611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259588096,"num_objects":3961,"num_object_clones":0,"num_object_copies":11883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3961,"num_write_kb":253504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":579,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.274844+0000","last_change":"2023-10-09T05:58:39.265193+0000","last_active":"2023-10-09T05:59:04.274844+0000","last_peered":"2023-10-09T05:59:04.274844+0000","last_clean":"2023-10-09T05:59:04.274844+0000","last_became_active":"2023-10-09T05:58:21.327449+0000","last_became_peered":"2023-10-09T05:58:21.327449+0000","last_unstale":"2023-10-09T05:59:04.274844+0000","last_undegraded":"2023-10-09T05:59:04.274844+0000","last_fullsized":"2023-10-09T05:59:04.274844+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:39.265029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:39.265029+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:30:46.849393+0000","scrub_duration":0.011791197999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":178,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.319574+0000","last_change":"2023-10-09T05:55:59.379174+0000","last_active":"2023-10-09T05:59:04.319574+0000","last_peered":"2023-10-09T05:59:04.319574+0000","last_clean":"2023-10-09T05:59:04.319574+0000","last_became_active":"2023-10-09T05:55:59.341932+0000","last_became_peered":"2023-10-09T05:55:59.341932+0000","last_unstale":"2023-10-09T05:59:04.319574+0000","last_undegraded":"2023-10-09T05:59:04.319574+0000","last_fullsized":"2023-10-09T05:59:04.319574+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.636171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":342,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.323041+0000","last_change":"2023-10-09T05:57:23.475146+0000","last_active":"2023-10-09T05:59:04.323041+0000","last_peered":"2023-10-09T05:59:04.323041+0000","last_clean":"2023-10-09T05:59:04.323041+0000","last_became_active":"2023-10-09T05:57:23.474776+0000","last_became_peered":"2023-10-09T05:57:23.474776+0000","last_unstale":"2023-10-09T05:59:04.323041+0000","last_undegraded":"2023-10-09T05:59:04.323041+0000","last_fullsized":"2023-10-09T05:59:04.323041+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:47.775743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"296'3941","reported_seq":3961,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:59:19.727363+0000","last_change":"2023-10-09T05:58:27.377562+0000","last_active":"2023-10-09T05:59:19.727363+0000","last_peered":"2023-10-09T05:59:19.727363+0000","last_clean":"2023-10-09T05:59:19.727363+0000","last_became_active":"2023-10-09T05:58:27.377253+0000","last_became_peered":"2023-10-09T05:58:27.377253+0000","last_unstale":"2023-10-09T05:59:19.727363+0000","last_undegraded":"2023-10-09T05:59:19.727363+0000","last_fullsized":"2023-10-09T05:59:19.727363+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":3941,"log_dups_size":0,"ondisk_log_size":3941,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:34.871962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258277376,"num_objects":3941,"num_object_clones":0,"num_object_copies":11823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3941,"num_write_kb":252224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":718,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.335491+0000","last_change":"2023-10-09T05:58:41.753606+0000","last_active":"2023-10-09T05:59:04.335491+0000","last_peered":"2023-10-09T05:59:04.335491+0000","last_clean":"2023-10-09T05:59:04.335491+0000","last_became_active":"2023-10-09T05:58:21.332726+0000","last_became_peered":"2023-10-09T05:58:21.332726+0000","last_unstale":"2023-10-09T05:59:04.335491+0000","last_undegraded":"2023-10-09T05:59:04.335491+0000","last_fullsized":"2023-10-09T05:59:04.335491+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:41.753506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:41.753506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:15.947090+0000","scrub_duration":0.0022121929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":134,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.320775+0000","last_change":"2023-10-09T05:53:59.212161+0000","last_active":"2023-10-09T05:59:04.320775+0000","last_peered":"2023-10-09T05:59:04.320775+0000","last_clean":"2023-10-09T05:59:04.320775+0000","last_became_active":"2023-10-09T05:53:59.193814+0000","last_became_peered":"2023-10-09T05:53:59.193814+0000","last_unstale":"2023-10-09T05:59:04.320775+0000","last_undegraded":"2023-10-09T05:59:04.320775+0000","last_fullsized":"2023-10-09T05:59:04.320775+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:10.502408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":335,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.335268+0000","last_change":"2023-10-09T05:55:59.303352+0000","last_active":"2023-10-09T05:59:04.335268+0000","last_peered":"2023-10-09T05:59:04.335268+0000","last_clean":"2023-10-09T05:59:04.335268+0000","last_became_active":"2023-10-09T05:55:59.302923+0000","last_became_peered":"2023-10-09T05:55:59.302923+0000","last_unstale":"2023-10-09T05:59:04.335268+0000","last_undegraded":"2023-10-09T05:59:04.335268+0000","last_fullsized":"2023-10-09T05:59:04.335268+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:53.013376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"296'3950","reported_seq":3978,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:59:20.523692+0000","last_change":"2023-10-09T05:58:59.347359+0000","last_active":"2023-10-09T05:59:20.523692+0000","last_peered":"2023-10-09T05:59:20.523692+0000","last_clean":"2023-10-09T05:59:20.523692+0000","last_became_active":"2023-10-09T05:58:59.328422+0000","last_became_peered":"2023-10-09T05:58:59.328422+0000","last_unstale":"2023-10-09T05:59:20.523692+0000","last_undegraded":"2023-10-09T05:59:20.523692+0000","last_fullsized":"2023-10-09T05:59:20.523692+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":3950,"log_dups_size":0,"ondisk_log_size":3950,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:31.541181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258867200,"num_objects":3950,"num_object_clones":0,"num_object_copies":11850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3950,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3950,"num_write_kb":252800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":442,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.321582+0000","last_change":"2023-10-09T05:58:37.499614+0000","last_active":"2023-10-09T05:59:04.321582+0000","last_peered":"2023-10-09T05:59:04.321582+0000","last_clean":"2023-10-09T05:59:04.321582+0000","last_became_active":"2023-10-09T05:58:21.330754+0000","last_became_peered":"2023-10-09T05:58:21.330754+0000","last_unstale":"2023-10-09T05:59:04.321582+0000","last_undegraded":"2023-10-09T05:59:04.321582+0000","last_fullsized":"2023-10-09T05:59:04.321582+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:37.499499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:37.499499+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:01.909666+0000","scrub_duration":0.0018066499999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":58,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.274889+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T05:59:04.274889+0000","last_peered":"2023-10-09T05:59:04.274889+0000","last_clean":"2023-10-09T05:59:04.274889+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T05:59:04.274889+0000","last_undegraded":"2023-10-09T05:59:04.274889+0000","last_fullsized":"2023-10-09T05:59:04.274889+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":307,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.320252+0000","last_change":"2023-10-09T05:57:23.473860+0000","last_active":"2023-10-09T05:59:04.320252+0000","last_peered":"2023-10-09T05:59:04.320252+0000","last_clean":"2023-10-09T05:59:04.320252+0000","last_became_active":"2023-10-09T05:57:23.473101+0000","last_became_peered":"2023-10-09T05:57:23.473101+0000","last_unstale":"2023-10-09T05:59:04.320252+0000","last_undegraded":"2023-10-09T05:59:04.320252+0000","last_fullsized":"2023-10-09T05:59:04.320252+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:05.682865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"296'3838","reported_seq":3858,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:59:19.754700+0000","last_change":"2023-10-09T05:58:27.376809+0000","last_active":"2023-10-09T05:59:19.754700+0000","last_peered":"2023-10-09T05:59:19.754700+0000","last_clean":"2023-10-09T05:59:19.754700+0000","last_became_active":"2023-10-09T05:58:27.376476+0000","last_became_peered":"2023-10-09T05:58:27.376476+0000","last_unstale":"2023-10-09T05:59:19.754700+0000","last_undegraded":"2023-10-09T05:59:19.754700+0000","last_fullsized":"2023-10-09T05:59:19.754700+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":3838,"log_dups_size":0,"ondisk_log_size":3838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:53:15.515971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251527168,"num_objects":3838,"num_object_clones":0,"num_object_copies":11514,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3838,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3838,"num_write_kb":245632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":719,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.319328+0000","last_change":"2023-10-09T05:58:44.765224+0000","last_active":"2023-10-09T05:59:04.319328+0000","last_peered":"2023-10-09T05:59:04.319328+0000","last_clean":"2023-10-09T05:59:04.319328+0000","last_became_active":"2023-10-09T05:57:13.233849+0000","last_became_peered":"2023-10-09T05:57:13.233849+0000","last_unstale":"2023-10-09T05:59:04.319328+0000","last_undegraded":"2023-10-09T05:59:04.319328+0000","last_fullsized":"2023-10-09T05:59:04.319328+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:44.765164+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:44.765164+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:24.336340+0000","scrub_duration":0.0050417350000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":62,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.274417+0000","last_change":"2023-10-09T05:57:23.504733+0000","last_active":"2023-10-09T05:59:04.274417+0000","last_peered":"2023-10-09T05:59:04.274417+0000","last_clean":"2023-10-09T05:59:04.274417+0000","last_became_active":"2023-10-09T05:57:23.468709+0000","last_became_peered":"2023-10-09T05:57:23.468709+0000","last_unstale":"2023-10-09T05:59:04.274417+0000","last_undegraded":"2023-10-09T05:59:04.274417+0000","last_fullsized":"2023-10-09T05:59:04.274417+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:00.459980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":265,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.321772+0000","last_change":"2023-10-09T05:59:00.402498+0000","last_active":"2023-10-09T05:59:04.321772+0000","last_peered":"2023-10-09T05:59:04.321772+0000","last_clean":"2023-10-09T05:59:04.321772+0000","last_became_active":"2023-10-09T05:59:00.402218+0000","last_became_peered":"2023-10-09T05:59:00.402218+0000","last_unstale":"2023-10-09T05:59:04.321772+0000","last_undegraded":"2023-10-09T05:59:04.321772+0000","last_fullsized":"2023-10-09T05:59:04.321772+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:53.653861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"296'3874","reported_seq":3894,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:59:17.446922+0000","last_change":"2023-10-09T05:58:27.405765+0000","last_active":"2023-10-09T05:59:17.446922+0000","last_peered":"2023-10-09T05:59:17.446922+0000","last_clean":"2023-10-09T05:59:17.446922+0000","last_became_active":"2023-10-09T05:58:27.405447+0000","last_became_peered":"2023-10-09T05:58:27.405447+0000","last_unstale":"2023-10-09T05:59:17.446922+0000","last_undegraded":"2023-10-09T05:59:17.446922+0000","last_fullsized":"2023-10-09T05:59:17.446922+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":3874,"log_dups_size":0,"ondisk_log_size":3874,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:52.191564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253886464,"num_objects":3874,"num_object_clones":0,"num_object_copies":11622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3874,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3874,"num_write_kb":247936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":463,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:59:17.309212+0000","last_change":"2023-10-09T05:59:17.309338+0000","last_active":"2023-10-09T05:59:17.309212+0000","last_peered":"2023-10-09T05:59:17.309212+0000","last_clean":"2023-10-09T05:59:17.309212+0000","last_became_active":"2023-10-09T05:58:59.324458+0000","last_became_peered":"2023-10-09T05:58:59.324458+0000","last_unstale":"2023-10-09T05:59:17.309212+0000","last_undegraded":"2023-10-09T05:59:17.309212+0000","last_fullsized":"2023-10-09T05:59:17.309212+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:17.309185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:17.309185+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:13.583428+0000","scrub_duration":0.020245381,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":306,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.333456+0000","last_change":"2023-10-09T05:55:59.320669+0000","last_active":"2023-10-09T05:59:04.333456+0000","last_peered":"2023-10-09T05:59:04.333456+0000","last_clean":"2023-10-09T05:59:04.333456+0000","last_became_active":"2023-10-09T05:55:59.320352+0000","last_became_peered":"2023-10-09T05:55:59.320352+0000","last_unstale":"2023-10-09T05:59:04.333456+0000","last_undegraded":"2023-10-09T05:59:04.333456+0000","last_fullsized":"2023-10-09T05:59:04.333456+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:09:14.227028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":342,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.322993+0000","last_change":"2023-10-09T05:57:10.362295+0000","last_active":"2023-10-09T05:59:04.322993+0000","last_peered":"2023-10-09T05:59:04.322993+0000","last_clean":"2023-10-09T05:59:04.322993+0000","last_became_active":"2023-10-09T05:57:10.361249+0000","last_became_peered":"2023-10-09T05:57:10.361249+0000","last_unstale":"2023-10-09T05:59:04.322993+0000","last_undegraded":"2023-10-09T05:59:04.322993+0000","last_fullsized":"2023-10-09T05:59:04.322993+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:22.088261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"296'3847","reported_seq":3866,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:59:18.383755+0000","last_change":"2023-10-09T05:58:27.378739+0000","last_active":"2023-10-09T05:59:18.383755+0000","last_peered":"2023-10-09T05:59:18.383755+0000","last_clean":"2023-10-09T05:59:18.383755+0000","last_became_active":"2023-10-09T05:58:27.378459+0000","last_became_peered":"2023-10-09T05:58:27.378459+0000","last_unstale":"2023-10-09T05:59:18.383755+0000","last_undegraded":"2023-10-09T05:59:18.383755+0000","last_fullsized":"2023-10-09T05:59:18.383755+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":3847,"log_dups_size":0,"ondisk_log_size":3847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:32.861544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252116992,"num_objects":3847,"num_object_clones":0,"num_object_copies":11541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3847,"num_write_kb":246208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":543,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.320591+0000","last_change":"2023-10-09T05:58:39.230541+0000","last_active":"2023-10-09T05:59:04.320591+0000","last_peered":"2023-10-09T05:59:04.320591+0000","last_clean":"2023-10-09T05:59:04.320591+0000","last_became_active":"2023-10-09T05:58:21.332677+0000","last_became_peered":"2023-10-09T05:58:21.332677+0000","last_unstale":"2023-10-09T05:59:04.320591+0000","last_undegraded":"2023-10-09T05:59:04.320591+0000","last_fullsized":"2023-10-09T05:59:04.320591+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:39.230347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:39.230347+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:14:06.774208+0000","scrub_duration":0.0041260619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":276,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.322864+0000","last_change":"2023-10-09T05:54:00.619165+0000","last_active":"2023-10-09T05:59:04.322864+0000","last_peered":"2023-10-09T05:59:04.322864+0000","last_clean":"2023-10-09T05:59:04.322864+0000","last_became_active":"2023-10-09T05:54:00.618888+0000","last_became_peered":"2023-10-09T05:54:00.618888+0000","last_unstale":"2023-10-09T05:59:04.322864+0000","last_undegraded":"2023-10-09T05:59:04.322864+0000","last_fullsized":"2023-10-09T05:59:04.322864+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:07.175996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":315,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.321683+0000","last_change":"2023-10-09T05:57:23.483158+0000","last_active":"2023-10-09T05:59:04.321683+0000","last_peered":"2023-10-09T05:59:04.321683+0000","last_clean":"2023-10-09T05:59:04.321683+0000","last_became_active":"2023-10-09T05:57:23.482701+0000","last_became_peered":"2023-10-09T05:57:23.482701+0000","last_unstale":"2023-10-09T05:59:04.321683+0000","last_undegraded":"2023-10-09T05:59:04.321683+0000","last_fullsized":"2023-10-09T05:59:04.321683+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:51:08.624310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"296'3805","reported_seq":3824,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:59:18.353966+0000","last_change":"2023-10-09T05:58:27.376357+0000","last_active":"2023-10-09T05:59:18.353966+0000","last_peered":"2023-10-09T05:59:18.353966+0000","last_clean":"2023-10-09T05:59:18.353966+0000","last_became_active":"2023-10-09T05:58:27.376017+0000","last_became_peered":"2023-10-09T05:58:27.376017+0000","last_unstale":"2023-10-09T05:59:18.353966+0000","last_undegraded":"2023-10-09T05:59:18.353966+0000","last_fullsized":"2023-10-09T05:59:18.353966+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":3805,"log_dups_size":0,"ondisk_log_size":3805,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:30.067479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249364480,"num_objects":3805,"num_object_clones":0,"num_object_copies":11415,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3805,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3805,"num_write_kb":243520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":429,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:59:13.703458+0000","last_change":"2023-10-09T05:59:13.703415+0000","last_active":"2023-10-09T05:59:13.703458+0000","last_peered":"2023-10-09T05:59:13.703458+0000","last_clean":"2023-10-09T05:59:13.703458+0000","last_became_active":"2023-10-09T05:58:59.328850+0000","last_became_peered":"2023-10-09T05:58:59.328850+0000","last_unstale":"2023-10-09T05:59:13.703458+0000","last_undegraded":"2023-10-09T05:59:13.703458+0000","last_fullsized":"2023-10-09T05:59:13.703458+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:13.703307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:13.703307+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:15.310384+0000","scrub_duration":0.0069768790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":58,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.273990+0000","last_change":"2023-10-09T05:57:23.469928+0000","last_active":"2023-10-09T05:59:04.273990+0000","last_peered":"2023-10-09T05:59:04.273990+0000","last_clean":"2023-10-09T05:59:04.273990+0000","last_became_active":"2023-10-09T05:57:23.469697+0000","last_became_peered":"2023-10-09T05:57:23.469697+0000","last_unstale":"2023-10-09T05:59:04.273990+0000","last_undegraded":"2023-10-09T05:59:04.273990+0000","last_fullsized":"2023-10-09T05:59:04.273990+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:21:07.179434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":175,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.274588+0000","last_change":"2023-10-09T05:55:59.409563+0000","last_active":"2023-10-09T05:59:04.274588+0000","last_peered":"2023-10-09T05:59:04.274588+0000","last_clean":"2023-10-09T05:59:04.274588+0000","last_became_active":"2023-10-09T05:55:59.409184+0000","last_became_peered":"2023-10-09T05:55:59.409184+0000","last_unstale":"2023-10-09T05:59:04.274588+0000","last_undegraded":"2023-10-09T05:59:04.274588+0000","last_fullsized":"2023-10-09T05:59:04.274588+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:39.513591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1","version":"296'3958","reported_seq":3977,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:59:19.743732+0000","last_change":"2023-10-09T05:58:27.377316+0000","last_active":"2023-10-09T05:59:19.743732+0000","last_peered":"2023-10-09T05:59:19.743732+0000","last_clean":"2023-10-09T05:59:19.743732+0000","last_became_active":"2023-10-09T05:58:27.377007+0000","last_became_peered":"2023-10-09T05:58:27.377007+0000","last_unstale":"2023-10-09T05:59:19.743732+0000","last_undegraded":"2023-10-09T05:59:19.743732+0000","last_fullsized":"2023-10-09T05:59:19.743732+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":3958,"log_dups_size":0,"ondisk_log_size":3958,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:10.982167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259391488,"num_objects":3958,"num_object_clones":0,"num_object_copies":11874,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3958,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3958,"num_write_kb":253312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":699,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.335339+0000","last_change":"2023-10-09T05:58:34.586268+0000","last_active":"2023-10-09T05:59:04.335339+0000","last_peered":"2023-10-09T05:59:04.335339+0000","last_clean":"2023-10-09T05:59:04.335339+0000","last_became_active":"2023-10-09T05:57:26.500602+0000","last_became_peered":"2023-10-09T05:57:26.500602+0000","last_unstale":"2023-10-09T05:59:04.335339+0000","last_undegraded":"2023-10-09T05:59:04.335339+0000","last_fullsized":"2023-10-09T05:59:04.335339+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.586180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.586180+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:22.464022+0000","scrub_duration":0.002003992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":191,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:59:13.289214+0000","last_change":"2023-10-09T05:55:59.298953+0000","last_active":"2023-10-09T05:59:13.289214+0000","last_peered":"2023-10-09T05:59:13.289214+0000","last_clean":"2023-10-09T05:59:13.289214+0000","last_became_active":"2023-10-09T05:55:59.298624+0000","last_became_peered":"2023-10-09T05:55:59.298624+0000","last_unstale":"2023-10-09T05:59:13.289214+0000","last_undegraded":"2023-10-09T05:59:13.289214+0000","last_fullsized":"2023-10-09T05:59:13.289214+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:00.513870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":58,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.274553+0000","last_change":"2023-10-09T05:57:23.468480+0000","last_active":"2023-10-09T05:59:04.274553+0000","last_peered":"2023-10-09T05:59:04.274553+0000","last_clean":"2023-10-09T05:59:04.274553+0000","last_became_active":"2023-10-09T05:57:23.468179+0000","last_became_peered":"2023-10-09T05:57:23.468179+0000","last_unstale":"2023-10-09T05:59:04.274553+0000","last_undegraded":"2023-10-09T05:59:04.274553+0000","last_fullsized":"2023-10-09T05:59:04.274553+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:18.157188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":159,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.321437+0000","last_change":"2023-10-09T05:56:49.249711+0000","last_active":"2023-10-09T05:59:04.321437+0000","last_peered":"2023-10-09T05:59:04.321437+0000","last_clean":"2023-10-09T05:59:04.321437+0000","last_became_active":"2023-10-09T05:56:49.249272+0000","last_became_peered":"2023-10-09T05:56:49.249272+0000","last_unstale":"2023-10-09T05:59:04.321437+0000","last_undegraded":"2023-10-09T05:59:04.321437+0000","last_fullsized":"2023-10-09T05:59:04.321437+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:42.596986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.7","version":"296'3754","reported_seq":3773,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:59:18.355177+0000","last_change":"2023-10-09T05:58:27.378776+0000","last_active":"2023-10-09T05:59:18.355177+0000","last_peered":"2023-10-09T05:59:18.355177+0000","last_clean":"2023-10-09T05:59:18.355177+0000","last_became_active":"2023-10-09T05:58:27.378504+0000","last_became_peered":"2023-10-09T05:58:27.378504+0000","last_unstale":"2023-10-09T05:59:18.355177+0000","last_undegraded":"2023-10-09T05:59:18.355177+0000","last_fullsized":"2023-10-09T05:59:18.355177+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":3754,"log_dups_size":0,"ondisk_log_size":3754,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:27.090596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246022144,"num_objects":3754,"num_object_clones":0,"num_object_copies":11262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3754,"num_write_kb":240256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":499,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.319960+0000","last_change":"2023-10-09T05:58:36.225081+0000","last_active":"2023-10-09T05:59:04.319960+0000","last_peered":"2023-10-09T05:59:04.319960+0000","last_clean":"2023-10-09T05:59:04.319960+0000","last_became_active":"2023-10-09T05:56:49.237269+0000","last_became_peered":"2023-10-09T05:56:49.237269+0000","last_unstale":"2023-10-09T05:59:04.319960+0000","last_undegraded":"2023-10-09T05:59:04.319960+0000","last_fullsized":"2023-10-09T05:59:04.319960+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:02.908583+0000","scrub_duration":0.0016660799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":316,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.335183+0000","last_change":"2023-10-09T05:56:49.254337+0000","last_active":"2023-10-09T05:59:04.335183+0000","last_peered":"2023-10-09T05:59:04.335183+0000","last_clean":"2023-10-09T05:59:04.335183+0000","last_became_active":"2023-10-09T05:56:49.247892+0000","last_became_peered":"2023-10-09T05:56:49.247892+0000","last_unstale":"2023-10-09T05:59:04.335183+0000","last_undegraded":"2023-10-09T05:59:04.335183+0000","last_fullsized":"2023-10-09T05:59:04.335183+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:44:20.157638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":273,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.321730+0000","last_change":"2023-10-09T05:51:57.203142+0000","last_active":"2023-10-09T05:59:04.321730+0000","last_peered":"2023-10-09T05:59:04.321730+0000","last_clean":"2023-10-09T05:59:04.321730+0000","last_became_active":"2023-10-09T05:51:57.202850+0000","last_became_peered":"2023-10-09T05:51:57.202850+0000","last_unstale":"2023-10-09T05:59:04.321730+0000","last_undegraded":"2023-10-09T05:59:04.321730+0000","last_fullsized":"2023-10-09T05:59:04.321730+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:51.090737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":189,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.274934+0000","last_change":"2023-10-09T05:58:59.330420+0000","last_active":"2023-10-09T05:59:04.274934+0000","last_peered":"2023-10-09T05:59:04.274934+0000","last_clean":"2023-10-09T05:59:04.274934+0000","last_became_active":"2023-10-09T05:58:59.330137+0000","last_became_peered":"2023-10-09T05:58:59.330137+0000","last_unstale":"2023-10-09T05:59:04.274934+0000","last_undegraded":"2023-10-09T05:59:04.274934+0000","last_fullsized":"2023-10-09T05:59:04.274934+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:33.801594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.4","version":"296'3942","reported_seq":3961,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:59:19.751700+0000","last_change":"2023-10-09T05:58:27.375343+0000","last_active":"2023-10-09T05:59:19.751700+0000","last_peered":"2023-10-09T05:59:19.751700+0000","last_clean":"2023-10-09T05:59:19.751700+0000","last_became_active":"2023-10-09T05:58:27.375094+0000","last_became_peered":"2023-10-09T05:58:27.375094+0000","last_unstale":"2023-10-09T05:59:19.751700+0000","last_undegraded":"2023-10-09T05:59:19.751700+0000","last_fullsized":"2023-10-09T05:59:19.751700+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":3942,"log_dups_size":0,"ondisk_log_size":3942,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:48:08.192351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258342912,"num_objects":3942,"num_object_clones":0,"num_object_copies":11826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3942,"num_write_kb":252288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":603,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.320414+0000","last_change":"2023-10-09T05:58:55.897880+0000","last_active":"2023-10-09T05:59:04.320414+0000","last_peered":"2023-10-09T05:59:04.320414+0000","last_clean":"2023-10-09T05:59:04.320414+0000","last_became_active":"2023-10-09T05:58:01.899103+0000","last_became_peered":"2023-10-09T05:58:01.899103+0000","last_unstale":"2023-10-09T05:59:04.320414+0000","last_undegraded":"2023-10-09T05:59:04.320414+0000","last_fullsized":"2023-10-09T05:59:04.320414+0000","mapping_epoch":268,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":269,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:55.897812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:55.897812+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:01.223576+0000","scrub_duration":0.0014511420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":36,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.274885+0000","last_change":"2023-10-09T05:58:59.325257+0000","last_active":"2023-10-09T05:59:04.274885+0000","last_peered":"2023-10-09T05:59:04.274885+0000","last_clean":"2023-10-09T05:59:04.274885+0000","last_became_active":"2023-10-09T05:58:59.324993+0000","last_became_peered":"2023-10-09T05:58:59.324993+0000","last_unstale":"2023-10-09T05:59:04.274885+0000","last_undegraded":"2023-10-09T05:59:04.274885+0000","last_fullsized":"2023-10-09T05:59:04.274885+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:20.825840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":114,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.333392+0000","last_change":"2023-10-09T05:58:59.328724+0000","last_active":"2023-10-09T05:59:04.333392+0000","last_peered":"2023-10-09T05:59:04.333392+0000","last_clean":"2023-10-09T05:59:04.333392+0000","last_became_active":"2023-10-09T05:58:59.328395+0000","last_became_peered":"2023-10-09T05:58:59.328395+0000","last_unstale":"2023-10-09T05:59:04.333392+0000","last_undegraded":"2023-10-09T05:59:04.333392+0000","last_fullsized":"2023-10-09T05:59:04.333392+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:37.528856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":191,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.320412+0000","last_change":"2023-10-09T05:56:49.238219+0000","last_active":"2023-10-09T05:59:04.320412+0000","last_peered":"2023-10-09T05:59:04.320412+0000","last_clean":"2023-10-09T05:59:04.320412+0000","last_became_active":"2023-10-09T05:56:49.237919+0000","last_became_peered":"2023-10-09T05:56:49.237919+0000","last_unstale":"2023-10-09T05:59:04.320412+0000","last_undegraded":"2023-10-09T05:59:04.320412+0000","last_fullsized":"2023-10-09T05:59:04.320412+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:44.599032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.6","version":"296'3918","reported_seq":3938,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:59:19.753430+0000","last_change":"2023-10-09T05:58:27.377502+0000","last_active":"2023-10-09T05:59:19.753430+0000","last_peered":"2023-10-09T05:59:19.753430+0000","last_clean":"2023-10-09T05:59:19.753430+0000","last_became_active":"2023-10-09T05:58:27.377195+0000","last_became_peered":"2023-10-09T05:58:27.377195+0000","last_unstale":"2023-10-09T05:59:19.753430+0000","last_undegraded":"2023-10-09T05:59:19.753430+0000","last_fullsized":"2023-10-09T05:59:19.753430+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":3918,"log_dups_size":0,"ondisk_log_size":3918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:14.980773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256770048,"num_objects":3918,"num_object_clones":0,"num_object_copies":11754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3918,"num_write_kb":250752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":167,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.319083+0000","last_change":"2023-10-09T05:58:59.330109+0000","last_active":"2023-10-09T05:59:04.319083+0000","last_peered":"2023-10-09T05:59:04.319083+0000","last_clean":"2023-10-09T05:59:04.319083+0000","last_became_active":"2023-10-09T05:58:59.329808+0000","last_became_peered":"2023-10-09T05:58:59.329808+0000","last_unstale":"2023-10-09T05:59:04.319083+0000","last_undegraded":"2023-10-09T05:59:04.319083+0000","last_fullsized":"2023-10-09T05:59:04.319083+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:30.680622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":700,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.320880+0000","last_change":"2023-10-09T05:58:37.782237+0000","last_active":"2023-10-09T05:59:04.320880+0000","last_peered":"2023-10-09T05:59:04.320880+0000","last_clean":"2023-10-09T05:59:04.320880+0000","last_became_active":"2023-10-09T05:53:59.202004+0000","last_became_peered":"2023-10-09T05:53:59.202004+0000","last_unstale":"2023-10-09T05:59:04.320880+0000","last_undegraded":"2023-10-09T05:59:04.320880+0000","last_fullsized":"2023-10-09T05:59:04.320880+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:37.782173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:37.782173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:20.787963+0000","scrub_duration":0.001325908,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":58,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.274392+0000","last_change":"2023-10-09T05:57:23.479174+0000","last_active":"2023-10-09T05:59:04.274392+0000","last_peered":"2023-10-09T05:59:04.274392+0000","last_clean":"2023-10-09T05:59:04.274392+0000","last_became_active":"2023-10-09T05:57:23.478897+0000","last_became_peered":"2023-10-09T05:57:23.478897+0000","last_unstale":"2023-10-09T05:59:04.274392+0000","last_undegraded":"2023-10-09T05:59:04.274392+0000","last_fullsized":"2023-10-09T05:59:04.274392+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:00.160042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":258,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.336641+0000","last_change":"2023-10-09T05:53:59.195325+0000","last_active":"2023-10-09T05:59:04.336641+0000","last_peered":"2023-10-09T05:59:04.336641+0000","last_clean":"2023-10-09T05:59:04.336641+0000","last_became_active":"2023-10-09T05:53:59.194954+0000","last_became_peered":"2023-10-09T05:53:59.194954+0000","last_unstale":"2023-10-09T05:59:04.336641+0000","last_undegraded":"2023-10-09T05:59:04.336641+0000","last_fullsized":"2023-10-09T05:59:04.336641+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:23.228416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"296'4072","reported_seq":4091,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:59:20.493113+0000","last_change":"2023-10-09T05:58:27.375153+0000","last_active":"2023-10-09T05:59:20.493113+0000","last_peered":"2023-10-09T05:59:20.493113+0000","last_clean":"2023-10-09T05:59:20.493113+0000","last_became_active":"2023-10-09T05:58:27.374820+0000","last_became_peered":"2023-10-09T05:58:27.374820+0000","last_unstale":"2023-10-09T05:59:20.493113+0000","last_undegraded":"2023-10-09T05:59:20.493113+0000","last_fullsized":"2023-10-09T05:59:20.493113+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":4072,"log_dups_size":0,"ondisk_log_size":4072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:08.834399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266862592,"num_objects":4072,"num_object_clones":0,"num_object_copies":12216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4072,"num_write_kb":260608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":491,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.335333+0000","last_change":"2023-10-09T05:58:40.724430+0000","last_active":"2023-10-09T05:59:04.335333+0000","last_peered":"2023-10-09T05:59:04.335333+0000","last_clean":"2023-10-09T05:59:04.335333+0000","last_became_active":"2023-10-09T05:57:54.796937+0000","last_became_peered":"2023-10-09T05:57:54.796937+0000","last_unstale":"2023-10-09T05:59:04.335333+0000","last_undegraded":"2023-10-09T05:59:04.335333+0000","last_fullsized":"2023-10-09T05:59:04.335333+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:40.724331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:40.724331+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:10.562541+0000","scrub_duration":0.0016947469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":162,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.320735+0000","last_change":"2023-10-09T05:55:59.300015+0000","last_active":"2023-10-09T05:59:04.320735+0000","last_peered":"2023-10-09T05:59:04.320735+0000","last_clean":"2023-10-09T05:59:04.320735+0000","last_became_active":"2023-10-09T05:55:59.299281+0000","last_became_peered":"2023-10-09T05:55:59.299281+0000","last_unstale":"2023-10-09T05:59:04.320735+0000","last_undegraded":"2023-10-09T05:59:04.320735+0000","last_fullsized":"2023-10-09T05:59:04.320735+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:50:32.465904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":233,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.320065+0000","last_change":"2023-10-09T05:57:23.471369+0000","last_active":"2023-10-09T05:59:04.320065+0000","last_peered":"2023-10-09T05:59:04.320065+0000","last_clean":"2023-10-09T05:59:04.320065+0000","last_became_active":"2023-10-09T05:57:23.471049+0000","last_became_peered":"2023-10-09T05:57:23.471049+0000","last_unstale":"2023-10-09T05:59:04.320065+0000","last_undegraded":"2023-10-09T05:59:04.320065+0000","last_fullsized":"2023-10-09T05:59:04.320065+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:21.208825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":350,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.320409+0000","last_change":"2023-10-09T05:57:10.348399+0000","last_active":"2023-10-09T05:59:04.320409+0000","last_peered":"2023-10-09T05:59:04.320409+0000","last_clean":"2023-10-09T05:59:04.320409+0000","last_became_active":"2023-10-09T05:57:10.348061+0000","last_became_peered":"2023-10-09T05:57:10.348061+0000","last_unstale":"2023-10-09T05:59:04.320409+0000","last_undegraded":"2023-10-09T05:59:04.320409+0000","last_fullsized":"2023-10-09T05:59:04.320409+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:12.453118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"296'4024","reported_seq":4044,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:59:19.750660+0000","last_change":"2023-10-09T05:58:27.405929+0000","last_active":"2023-10-09T05:59:19.750660+0000","last_peered":"2023-10-09T05:59:19.750660+0000","last_clean":"2023-10-09T05:59:19.750660+0000","last_became_active":"2023-10-09T05:58:27.405604+0000","last_became_peered":"2023-10-09T05:58:27.405604+0000","last_unstale":"2023-10-09T05:59:19.750660+0000","last_undegraded":"2023-10-09T05:59:19.750660+0000","last_fullsized":"2023-10-09T05:59:19.750660+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":4024,"log_dups_size":0,"ondisk_log_size":4024,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:19.748373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263716864,"num_objects":4024,"num_object_clones":0,"num_object_copies":12072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4024,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4024,"num_write_kb":257536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":689,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.336228+0000","last_change":"2023-10-09T05:58:35.630429+0000","last_active":"2023-10-09T05:59:04.336228+0000","last_peered":"2023-10-09T05:59:04.336228+0000","last_clean":"2023-10-09T05:59:04.336228+0000","last_became_active":"2023-10-09T05:57:26.517990+0000","last_became_peered":"2023-10-09T05:57:26.517990+0000","last_unstale":"2023-10-09T05:59:04.336228+0000","last_undegraded":"2023-10-09T05:59:04.336228+0000","last_fullsized":"2023-10-09T05:59:04.336228+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:35.630348+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:35.630348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:14:01.075118+0000","scrub_duration":0.0020631920000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":130,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.320671+0000","last_change":"2023-10-09T05:53:59.201576+0000","last_active":"2023-10-09T05:59:04.320671+0000","last_peered":"2023-10-09T05:59:04.320671+0000","last_clean":"2023-10-09T05:59:04.320671+0000","last_became_active":"2023-10-09T05:53:59.201309+0000","last_became_peered":"2023-10-09T05:53:59.201309+0000","last_unstale":"2023-10-09T05:59:04.320671+0000","last_undegraded":"2023-10-09T05:59:04.320671+0000","last_fullsized":"2023-10-09T05:59:04.320671+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:54.956371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":120,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.320147+0000","last_change":"2023-10-09T05:58:59.328614+0000","last_active":"2023-10-09T05:59:04.320147+0000","last_peered":"2023-10-09T05:59:04.320147+0000","last_clean":"2023-10-09T05:59:04.320147+0000","last_became_active":"2023-10-09T05:58:59.328203+0000","last_became_peered":"2023-10-09T05:58:59.328203+0000","last_unstale":"2023-10-09T05:59:04.320147+0000","last_undegraded":"2023-10-09T05:59:04.320147+0000","last_fullsized":"2023-10-09T05:59:04.320147+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:21.572990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":222,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.320420+0000","last_change":"2023-10-09T05:53:59.188462+0000","last_active":"2023-10-09T05:59:04.320420+0000","last_peered":"2023-10-09T05:59:04.320420+0000","last_clean":"2023-10-09T05:59:04.320420+0000","last_became_active":"2023-10-09T05:53:59.188061+0000","last_became_peered":"2023-10-09T05:53:59.188061+0000","last_unstale":"2023-10-09T05:59:04.320420+0000","last_undegraded":"2023-10-09T05:59:04.320420+0000","last_fullsized":"2023-10-09T05:59:04.320420+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:19.167114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"296'3895","reported_seq":3915,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:59:19.749096+0000","last_change":"2023-10-09T05:58:27.377750+0000","last_active":"2023-10-09T05:59:19.749096+0000","last_peered":"2023-10-09T05:59:19.749096+0000","last_clean":"2023-10-09T05:59:19.749096+0000","last_became_active":"2023-10-09T05:58:27.377457+0000","last_became_peered":"2023-10-09T05:58:27.377457+0000","last_unstale":"2023-10-09T05:59:19.749096+0000","last_undegraded":"2023-10-09T05:59:19.749096+0000","last_fullsized":"2023-10-09T05:59:19.749096+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":3895,"log_dups_size":0,"ondisk_log_size":3895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:52:04.242389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255262720,"num_objects":3895,"num_object_clones":0,"num_object_copies":11685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3895,"num_write_kb":249280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":660,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.322873+0000","last_change":"2023-10-09T05:58:35.542976+0000","last_active":"2023-10-09T05:59:04.322873+0000","last_peered":"2023-10-09T05:59:04.322873+0000","last_clean":"2023-10-09T05:59:04.322873+0000","last_became_active":"2023-10-09T05:57:13.235353+0000","last_became_peered":"2023-10-09T05:57:13.235353+0000","last_unstale":"2023-10-09T05:59:04.322873+0000","last_undegraded":"2023-10-09T05:59:04.322873+0000","last_fullsized":"2023-10-09T05:59:04.322873+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:35.542853+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:35.542853+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:30.605580+0000","scrub_duration":0.001732169,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":302,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.274653+0000","last_change":"2023-10-09T05:53:59.228948+0000","last_active":"2023-10-09T05:59:04.274653+0000","last_peered":"2023-10-09T05:59:04.274653+0000","last_clean":"2023-10-09T05:59:04.274653+0000","last_became_active":"2023-10-09T05:53:59.186359+0000","last_became_peered":"2023-10-09T05:53:59.186359+0000","last_unstale":"2023-10-09T05:59:04.274653+0000","last_undegraded":"2023-10-09T05:59:04.274653+0000","last_fullsized":"2023-10-09T05:59:04.274653+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:39.990576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":153,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.336084+0000","last_change":"2023-10-09T05:55:59.339957+0000","last_active":"2023-10-09T05:59:04.336084+0000","last_peered":"2023-10-09T05:59:04.336084+0000","last_clean":"2023-10-09T05:59:04.336084+0000","last_became_active":"2023-10-09T05:55:59.339614+0000","last_became_peered":"2023-10-09T05:55:59.339614+0000","last_unstale":"2023-10-09T05:59:04.336084+0000","last_undegraded":"2023-10-09T05:59:04.336084+0000","last_fullsized":"2023-10-09T05:59:04.336084+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:01.613113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":58,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.274148+0000","last_change":"2023-10-09T05:57:23.477143+0000","last_active":"2023-10-09T05:59:04.274148+0000","last_peered":"2023-10-09T05:59:04.274148+0000","last_clean":"2023-10-09T05:59:04.274148+0000","last_became_active":"2023-10-09T05:57:23.475655+0000","last_became_peered":"2023-10-09T05:57:23.475655+0000","last_unstale":"2023-10-09T05:59:04.274148+0000","last_undegraded":"2023-10-09T05:59:04.274148+0000","last_fullsized":"2023-10-09T05:59:04.274148+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:29.528023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"296'3931","reported_seq":3951,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T05:59:19.715776+0000","last_change":"2023-10-09T05:58:27.376100+0000","last_active":"2023-10-09T05:59:19.715776+0000","last_peered":"2023-10-09T05:59:19.715776+0000","last_clean":"2023-10-09T05:59:19.715776+0000","last_became_active":"2023-10-09T05:58:27.375806+0000","last_became_peered":"2023-10-09T05:58:27.375806+0000","last_unstale":"2023-10-09T05:59:19.715776+0000","last_undegraded":"2023-10-09T05:59:19.715776+0000","last_fullsized":"2023-10-09T05:59:19.715776+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":3931,"log_dups_size":0,"ondisk_log_size":3931,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:56.466666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257622016,"num_objects":3931,"num_object_clones":0,"num_object_copies":11793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3931,"num_write_kb":251584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":552,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.274796+0000","last_change":"2023-10-09T05:58:34.333129+0000","last_active":"2023-10-09T05:59:04.274796+0000","last_peered":"2023-10-09T05:59:04.274796+0000","last_clean":"2023-10-09T05:59:04.274796+0000","last_became_active":"2023-10-09T05:57:26.502911+0000","last_became_peered":"2023-10-09T05:57:26.502911+0000","last_unstale":"2023-10-09T05:59:04.274796+0000","last_undegraded":"2023-10-09T05:59:04.274796+0000","last_fullsized":"2023-10-09T05:59:04.274796+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:25.230712+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":300,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.335415+0000","last_change":"2023-10-09T05:57:58.832790+0000","last_active":"2023-10-09T05:59:04.335415+0000","last_peered":"2023-10-09T05:59:04.335415+0000","last_clean":"2023-10-09T05:59:04.335415+0000","last_became_active":"2023-10-09T05:57:58.832471+0000","last_became_peered":"2023-10-09T05:57:58.832471+0000","last_unstale":"2023-10-09T05:59:04.335415+0000","last_undegraded":"2023-10-09T05:59:04.335415+0000","last_fullsized":"2023-10-09T05:59:04.335415+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:39.358174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":336,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.320650+0000","last_change":"2023-10-09T05:57:58.836050+0000","last_active":"2023-10-09T05:59:04.320650+0000","last_peered":"2023-10-09T05:59:04.320650+0000","last_clean":"2023-10-09T05:59:04.320650+0000","last_became_active":"2023-10-09T05:57:58.833691+0000","last_became_peered":"2023-10-09T05:57:58.833691+0000","last_unstale":"2023-10-09T05:59:04.320650+0000","last_undegraded":"2023-10-09T05:59:04.320650+0000","last_fullsized":"2023-10-09T05:59:04.320650+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:32.257849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":238,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.323012+0000","last_change":"2023-10-09T05:40:12.676587+0000","last_active":"2023-10-09T05:59:04.323012+0000","last_peered":"2023-10-09T05:59:04.323012+0000","last_clean":"2023-10-09T05:59:04.323012+0000","last_became_active":"2023-10-09T05:40:12.676330+0000","last_became_peered":"2023-10-09T05:40:12.676330+0000","last_unstale":"2023-10-09T05:59:04.323012+0000","last_undegraded":"2023-10-09T05:59:04.323012+0000","last_fullsized":"2023-10-09T05:59:04.323012+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:46.659556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":225,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.320503+0000","last_change":"2023-10-09T05:53:59.191302+0000","last_active":"2023-10-09T05:59:04.320503+0000","last_peered":"2023-10-09T05:59:04.320503+0000","last_clean":"2023-10-09T05:59:04.320503+0000","last_became_active":"2023-10-09T05:53:59.190911+0000","last_became_peered":"2023-10-09T05:53:59.190911+0000","last_unstale":"2023-10-09T05:59:04.320503+0000","last_undegraded":"2023-10-09T05:59:04.320503+0000","last_fullsized":"2023-10-09T05:59:04.320503+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:12.801201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":36,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.274749+0000","last_change":"2023-10-09T05:58:59.326172+0000","last_active":"2023-10-09T05:59:04.274749+0000","last_peered":"2023-10-09T05:59:04.274749+0000","last_clean":"2023-10-09T05:59:04.274749+0000","last_became_active":"2023-10-09T05:58:59.325795+0000","last_became_peered":"2023-10-09T05:58:59.325795+0000","last_unstale":"2023-10-09T05:59:04.274749+0000","last_undegraded":"2023-10-09T05:59:04.274749+0000","last_fullsized":"2023-10-09T05:59:04.274749+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:34.308820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":86,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.321761+0000","last_change":"2023-10-09T05:56:49.244931+0000","last_active":"2023-10-09T05:59:04.321761+0000","last_peered":"2023-10-09T05:59:04.321761+0000","last_clean":"2023-10-09T05:59:04.321761+0000","last_became_active":"2023-10-09T05:56:49.244651+0000","last_became_peered":"2023-10-09T05:56:49.244651+0000","last_unstale":"2023-10-09T05:59:04.321761+0000","last_undegraded":"2023-10-09T05:59:04.321761+0000","last_fullsized":"2023-10-09T05:59:04.321761+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:53.906578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":332,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.321800+0000","last_change":"2023-10-09T05:57:10.356351+0000","last_active":"2023-10-09T05:59:04.321800+0000","last_peered":"2023-10-09T05:59:04.321800+0000","last_clean":"2023-10-09T05:59:04.321800+0000","last_became_active":"2023-10-09T05:57:10.356037+0000","last_became_peered":"2023-10-09T05:57:10.356037+0000","last_unstale":"2023-10-09T05:59:04.321800+0000","last_undegraded":"2023-10-09T05:59:04.321800+0000","last_fullsized":"2023-10-09T05:59:04.321800+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:15.139912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":86,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.323079+0000","last_change":"2023-10-09T05:56:49.245612+0000","last_active":"2023-10-09T05:59:04.323079+0000","last_peered":"2023-10-09T05:59:04.323079+0000","last_clean":"2023-10-09T05:59:04.323079+0000","last_became_active":"2023-10-09T05:56:49.245235+0000","last_became_peered":"2023-10-09T05:56:49.245235+0000","last_unstale":"2023-10-09T05:59:04.323079+0000","last_undegraded":"2023-10-09T05:59:04.323079+0000","last_fullsized":"2023-10-09T05:59:04.323079+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:29.016794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":299,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.321326+0000","last_change":"2023-10-09T05:59:00.400894+0000","last_active":"2023-10-09T05:59:04.321326+0000","last_peered":"2023-10-09T05:59:04.321326+0000","last_clean":"2023-10-09T05:59:04.321326+0000","last_became_active":"2023-10-09T05:59:00.399730+0000","last_became_peered":"2023-10-09T05:59:00.399730+0000","last_unstale":"2023-10-09T05:59:04.321326+0000","last_undegraded":"2023-10-09T05:59:04.321326+0000","last_fullsized":"2023-10-09T05:59:04.321326+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:46.821954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":58,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.274581+0000","last_change":"2023-10-09T05:57:23.479560+0000","last_active":"2023-10-09T05:59:04.274581+0000","last_peered":"2023-10-09T05:59:04.274581+0000","last_clean":"2023-10-09T05:59:04.274581+0000","last_became_active":"2023-10-09T05:57:23.479177+0000","last_became_peered":"2023-10-09T05:57:23.479177+0000","last_unstale":"2023-10-09T05:59:04.274581+0000","last_undegraded":"2023-10-09T05:59:04.274581+0000","last_fullsized":"2023-10-09T05:59:04.274581+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:41.925161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":58,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.274929+0000","last_change":"2023-10-09T05:57:23.475179+0000","last_active":"2023-10-09T05:59:04.274929+0000","last_peered":"2023-10-09T05:59:04.274929+0000","last_clean":"2023-10-09T05:59:04.274929+0000","last_became_active":"2023-10-09T05:57:23.474904+0000","last_became_peered":"2023-10-09T05:57:23.474904+0000","last_unstale":"2023-10-09T05:59:04.274929+0000","last_undegraded":"2023-10-09T05:59:04.274929+0000","last_fullsized":"2023-10-09T05:59:04.274929+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:54.587601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":263,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.320527+0000","last_change":"2023-10-09T05:57:10.379258+0000","last_active":"2023-10-09T05:59:04.320527+0000","last_peered":"2023-10-09T05:59:04.320527+0000","last_clean":"2023-10-09T05:59:04.320527+0000","last_became_active":"2023-10-09T05:57:10.359745+0000","last_became_peered":"2023-10-09T05:57:10.359745+0000","last_unstale":"2023-10-09T05:59:04.320527+0000","last_undegraded":"2023-10-09T05:59:04.320527+0000","last_fullsized":"2023-10-09T05:59:04.320527+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:14.750256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":315,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T05:59:04.321324+0000","last_change":"2023-10-09T05:57:23.481065+0000","last_active":"2023-10-09T05:59:04.321324+0000","last_peered":"2023-10-09T05:59:04.321324+0000","last_clean":"2023-10-09T05:59:04.321324+0000","last_became_active":"2023-10-09T05:57:23.480203+0000","last_became_peered":"2023-10-09T05:57:23.480203+0000","last_unstale":"2023-10-09T05:59:04.321324+0000","last_undegraded":"2023-10-09T05:59:04.321324+0000","last_fullsized":"2023-10-09T05:59:04.321324+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:15:53.196457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":4101177344,"num_objects":62579,"num_object_clones":0,"num_object_copies":187737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":62579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62579,"num_write_kb":4005056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12287082496,"data_stored":12287082496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":62579,"ondisk_log_size":62579,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":10792,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234236,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2084948,"kb_used_data":1771276,"kb_used_omap":0,"kb_used_meta":313664,"kb_avail":91660204,"statfs":{"total":95995035648,"available":93860048896,"internally_reserved":0,"allocated":1813786624,"data_stored":1812755800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321191936},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:59:03 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":1.179,"15min":1.7150000000000001},"min":{"1min":0.32900000000000001,"5min":0.22,"15min":0.22},"max":{"1min":2.46,"5min":21.038,"15min":21.038},"last":0.86599999999999999},{"interface":"front","average":{"1min":0.81799999999999995,"5min":1.161,"15min":1.609},"min":{"1min":0.308,"5min":0.224,"15min":0.224},"max":{"1min":2.6139999999999999,"5min":21.152000000000001,"15min":21.152000000000001},"last":1.629}]},{"osd":1,"last update":"Mon Oct 9 05:58:31 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":1.4590000000000001,"15min":0.95899999999999996},"min":{"1min":0.32500000000000001,"5min":0.27200000000000002,"15min":0.22900000000000001},"max":{"1min":2.1699999999999999,"5min":35.113999999999997,"15min":35.113999999999997},"last":2.3919999999999999},{"interface":"front","average":{"1min":0.68100000000000005,"5min":1.2130000000000001,"15min":0.86399999999999999},"min":{"1min":0.30099999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.4449999999999998,"5min":34.768999999999998,"15min":34.768999999999998},"last":3.113}]},{"osd":2,"last update":"Mon Oct 9 05:59:03 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":3.343,"15min":2.0569999999999999},"min":{"1min":0.372,"5min":0.28699999999999998,"15min":0.28199999999999997},"max":{"1min":4.6909999999999998,"5min":119.91200000000001,"15min":119.91200000000001},"last":3.8279999999999998},{"interface":"front","average":{"1min":1.0029999999999999,"5min":3.46,"15min":2.1429999999999998},"min":{"1min":0.38600000000000001,"5min":0.318,"15min":0.318},"max":{"1min":2.9500000000000002,"5min":119.95099999999999,"15min":119.95099999999999},"last":1.492}]},{"osd":3,"last update":"Mon Oct 9 05:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":2.552,"15min":2.9470000000000001},"min":{"1min":0.36599999999999999,"5min":0.35999999999999999,"15min":0.33300000000000002},"max":{"1min":2.7069999999999999,"5min":70.840999999999994,"15min":147.54900000000001},"last":0.84499999999999997},{"interface":"front","average":{"1min":0.90700000000000003,"5min":2.391,"15min":2.847},"min":{"1min":0.35299999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.4590000000000001,"5min":71.212000000000003,"15min":144.065},"last":3.867}]},{"osd":4,"last update":"Mon Oct 9 05:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":0.68300000000000005,"15min":0.626},"min":{"1min":0.29899999999999999,"5min":0.27000000000000002,"15min":0.24399999999999999},"max":{"1min":2.1230000000000002,"5min":3.7120000000000002,"15min":9.5329999999999995},"last":11.635},{"interface":"front","average":{"1min":0.77700000000000002,"5min":0.64000000000000001,"15min":0.68500000000000005},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.27400000000000002},"max":{"1min":2.1280000000000001,"5min":3.835,"15min":7.6120000000000001},"last":5.6760000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.92700000000000005,"15min":0.76800000000000002},"min":{"1min":0.28499999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.5900000000000001,"5min":15.622,"15min":15.622},"last":0.88800000000000001},{"interface":"front","average":{"1min":0.74099999999999999,"5min":0.63900000000000001,"15min":0.64100000000000001},"min":{"1min":0.34699999999999998,"5min":0.314,"15min":0.30499999999999999},"max":{"1min":2.0529999999999999,"5min":2.3969999999999998,"15min":3.9380000000000002},"last":0.35499999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":0.89800000000000002,"15min":1.296},"min":{"1min":0.31,"5min":0.28799999999999998,"15min":0.28299999999999997},"max":{"1min":2.5830000000000002,"5min":12.727,"15min":131.94399999999999},"last":1.0189999999999999},{"interface":"front","average":{"1min":0.77500000000000002,"5min":0.71899999999999997,"15min":1.278},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.29999999999999999},"max":{"1min":2.5779999999999998,"5min":3.5859999999999999,"15min":135.23400000000001},"last":5.3280000000000003}]}]},{"osd":6,"up_from":46,"seq":197568495872,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1754308,"kb_used_data":1457020,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":91990844,"statfs":{"total":95995035648,"available":94198624256,"internally_reserved":0,"allocated":1491988480,"data_stored":1490973629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:59:03 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":1.583,"15min":1.25},"min":{"1min":0.24199999999999999,"5min":0.224,"15min":0.224},"max":{"1min":2.5350000000000001,"5min":49.389000000000003,"15min":49.389000000000003},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.92900000000000005,"5min":1.5629999999999999,"15min":1.2889999999999999},"min":{"1min":0.35299999999999998,"5min":0.253,"15min":0.253},"max":{"1min":3.9660000000000002,"5min":49.338999999999999,"15min":49.338999999999999},"last":12.263999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0580000000000001,"5min":1,"15min":0.80300000000000005},"min":{"1min":0.32200000000000001,"5min":0.246,"15min":0.23899999999999999},"max":{"1min":8.7720000000000002,"5min":17.341000000000001,"15min":17.341000000000001},"last":1.0049999999999999},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.99199999999999999,"15min":0.81999999999999995},"min":{"1min":0.29199999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":2.9980000000000002,"5min":12.474,"15min":12.474},"last":1.079}]},{"osd":2,"last update":"Mon Oct 9 05:59:03 2023","interfaces":[{"interface":"back","average":{"1min":2.4769999999999999,"5min":2.3399999999999999,"15min":1.319},"min":{"1min":0.30199999999999999,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":27.224,"5min":61.088999999999999,"15min":61.088999999999999},"last":4.2560000000000002},{"interface":"front","average":{"1min":2.6019999999999999,"5min":2.3879999999999999,"15min":1.325},"min":{"1min":0.34200000000000003,"5min":0.32000000000000001,"15min":0.25600000000000001},"max":{"1min":27.177,"5min":61.082000000000001,"15min":61.082000000000001},"last":1.6000000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:58:43 2023","interfaces":[{"interface":"back","average":{"1min":4.609,"5min":4.4900000000000002,"15min":2.052},"min":{"1min":0.32000000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":62.423999999999999,"5min":93.626000000000005,"15min":93.626000000000005},"last":1.3069999999999999},{"interface":"front","average":{"1min":4.6669999999999998,"5min":4.2080000000000002,"15min":1.9750000000000001},"min":{"1min":0.47499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":62.518999999999998,"5min":93.643000000000001,"15min":93.643000000000001},"last":1.25}]},{"osd":4,"last update":"Mon Oct 9 05:58:49 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.747,"15min":0.71499999999999997},"min":{"1min":0.32200000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.403,"5min":4.5709999999999997,"15min":7.9100000000000001},"last":1.762},{"interface":"front","average":{"1min":0.496,"5min":0.81799999999999995,"15min":0.73399999999999999},"min":{"1min":0.29699999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.79200000000000004,"5min":8.734,"15min":8.734},"last":0.32300000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:58:49 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.2909999999999999,"15min":0.89900000000000002},"min":{"1min":0.317,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.92600000000000005,"5min":45.933999999999997,"15min":45.933999999999997},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.55000000000000004,"5min":1.274,"15min":0.92800000000000005},"min":{"1min":0.36199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.86499999999999999,"5min":45.917999999999999,"15min":45.917999999999999},"last":0.40300000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:59:15 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":1.365,"15min":1.0529999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.758,"5min":52.042999999999999,"15min":52.042999999999999},"last":3.5329999999999999},{"interface":"front","average":{"1min":0.73699999999999999,"5min":1.335,"15min":1.0209999999999999},"min":{"1min":0.34899999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.6190000000000002,"5min":51.259,"15min":51.259},"last":0.35799999999999998}]}]},{"osd":1,"up_from":0,"seq":382252089461,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":129,"seq":554050781263,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2164264,"kb_used_data":1998496,"kb_used_omap":29,"kb_used_meta":165730,"kb_avail":91580888,"statfs":{"total":95995035648,"available":93778829312,"internally_reserved":0,"allocated":2046459904,"data_stored":2044768570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30322,"internal_metadata":169707918},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:59:07 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.875,"15min":0.78300000000000003},"min":{"1min":0.219,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":3.9620000000000002,"5min":8.3300000000000001,"15min":9.1720000000000006},"last":0.251},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.83099999999999996,"15min":1.022},"min":{"1min":0.248,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":3.891,"5min":6.6420000000000003,"15min":6.6420000000000003},"last":0.27200000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:59:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.2709999999999999,"15min":1.0920000000000001},"min":{"1min":0.26700000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":6.8150000000000004,"5min":13.471,"15min":13.471},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.995,"5min":1.177,"15min":1.0449999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":6.2709999999999999,"5min":13.561,"15min":13.561},"last":0.29099999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:59:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.7070000000000001,"15min":1.0820000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":6.7569999999999997,"5min":27.954000000000001,"15min":27.954000000000001},"last":134.185},{"interface":"front","average":{"1min":1.0489999999999999,"5min":1.4710000000000001,"15min":1.085},"min":{"1min":0.27600000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.2240000000000002,"5min":20.027000000000001,"15min":20.027000000000001},"last":134.16800000000001}]},{"osd":4,"last update":"Mon Oct 9 05:59:07 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":3.9089999999999998,"15min":2.109},"min":{"1min":0.29799999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.837,"5min":149.07400000000001,"15min":149.07400000000001},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.92400000000000004,"5min":3.879,"15min":1.8839999999999999},"min":{"1min":0.34499999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.9780000000000002,"5min":149.12700000000001,"15min":149.12700000000001},"last":0.36199999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:59:07 2023","interfaces":[{"interface":"back","average":{"1min":2.5169999999999999,"5min":2.7250000000000001,"15min":1.6739999999999999},"min":{"1min":0.318,"5min":0.314,"15min":0.314},"max":{"1min":39.188000000000002,"5min":100.25,"15min":100.25},"last":0.46100000000000002},{"interface":"front","average":{"1min":2.4790000000000001,"5min":2.7069999999999999,"15min":1.532},"min":{"1min":0.35899999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":39.143000000000001,"5min":102.40300000000001,"15min":102.40300000000001},"last":0.56799999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:59:07 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":0.96899999999999997,"15min":1.1799999999999999},"min":{"1min":0.34399999999999997,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.1509999999999998,"5min":8.1470000000000002,"15min":8.1470000000000002},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.79200000000000004,"5min":1.1719999999999999,"15min":1.2},"min":{"1min":0.44700000000000001,"5min":0.378,"15min":0.378},"max":{"1min":1.859,"5min":8.0510000000000002,"15min":9.141},"last":0.501}]},{"osd":7,"last update":"Mon Oct 9 05:59:07 2023","interfaces":[{"interface":"back","average":{"1min":5.3239999999999998,"5min":2.8570000000000002,"15min":1.889},"min":{"1min":0.47299999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":103.866,"5min":103.866,"15min":103.866},"last":0.53900000000000003},{"interface":"front","average":{"1min":5.4870000000000001,"5min":2.706,"15min":1.806},"min":{"1min":0.38300000000000001,"5min":0.378,"15min":0.378},"max":{"1min":103.848,"5min":103.848,"15min":103.848},"last":0.51500000000000001}]}]},{"osd":2,"up_from":94,"seq":403726925952,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2381652,"kb_used_data":2013964,"kb_used_omap":81,"kb_used_meta":367598,"kb_avail":91363500,"statfs":{"total":95995035648,"available":93556224000,"internally_reserved":0,"allocated":2062299136,"data_stored":2060739663,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":376420570},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:59:14 2023","interfaces":[{"interface":"back","average":{"1min":1.1719999999999999,"5min":2.2080000000000002,"15min":1.782},"min":{"1min":0.22500000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":5.4669999999999996,"5min":55.409999999999997,"15min":55.409999999999997},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.1970000000000001,"5min":2.1859999999999999,"15min":1.7010000000000001},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23100000000000001},"max":{"1min":4.5979999999999999,"5min":55.326000000000001,"15min":55.326000000000001},"last":2.0350000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:59:05 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.88100000000000001,"15min":1.3779999999999999},"min":{"1min":0.25900000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.8599999999999999,"5min":3.8959999999999999,"15min":163.24199999999999},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.3049999999999999,"5min":0.96299999999999997,"15min":1.3320000000000001},"min":{"1min":0.29799999999999999,"5min":0.223,"15min":0.214},"max":{"1min":6.2030000000000003,"5min":6.9790000000000001,"15min":163.27000000000001},"last":2.0009999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:58:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1559999999999999,"5min":2.0880000000000001,"15min":1.806},"min":{"1min":0.30299999999999999,"5min":0.25900000000000001,"15min":0.23899999999999999},"max":{"1min":9.6620000000000008,"5min":55.118000000000002,"15min":170.07599999999999},"last":1.4239999999999999},{"interface":"front","average":{"1min":0.95599999999999996,"5min":2.0760000000000001,"15min":1.379},"min":{"1min":0.39200000000000002,"5min":0.31900000000000001,"15min":0.25},"max":{"1min":3.952,"5min":55.365000000000002,"15min":63.600000000000001},"last":0.45700000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:59:05 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":1.0720000000000001,"15min":0.96699999999999997},"min":{"1min":0.34499999999999997,"5min":0.30599999999999999,"15min":0.28199999999999997},"max":{"1min":2.407,"5min":8.6280000000000001,"15min":11.372999999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.0860000000000001,"5min":0.996,"15min":0.92600000000000005},"min":{"1min":0.40600000000000003,"5min":0.33200000000000002,"15min":0.26800000000000002},"max":{"1min":3.0979999999999999,"5min":6.1710000000000003,"15min":6.5709999999999997},"last":2.0819999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:59:05 2023","interfaces":[{"interface":"back","average":{"1min":1.7470000000000001,"5min":1.427,"15min":1.0660000000000001},"min":{"1min":0.44700000000000001,"5min":0.30599999999999999,"15min":0.28999999999999998},"max":{"1min":8.8819999999999997,"5min":8.8819999999999997,"15min":13.414},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.92600000000000005,"5min":1.1299999999999999,"15min":0.995},"min":{"1min":0.40500000000000003,"5min":0.313,"15min":0.29799999999999999},"max":{"1min":2.2080000000000002,"5min":8.5570000000000004,"15min":13.438000000000001},"last":0.41499999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:59:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2769999999999999,"5min":1.0369999999999999,"15min":0.95199999999999996},"min":{"1min":0.42299999999999999,"5min":0.31900000000000001,"15min":0.29399999999999998},"max":{"1min":4.3040000000000003,"5min":8.6920000000000002,"15min":11.59},"last":2.0579999999999998},{"interface":"front","average":{"1min":1.851,"5min":1.3320000000000001,"15min":0.98599999999999999},"min":{"1min":0.52100000000000002,"5min":0.37,"15min":0.34000000000000002},"max":{"1min":8.9399999999999995,"5min":8.9399999999999995,"15min":8.9399999999999995},"last":0.58399999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:59:05 2023","interfaces":[{"interface":"back","average":{"1min":1.331,"5min":1.9510000000000001,"15min":1.3180000000000001},"min":{"1min":0.56000000000000005,"5min":0.36599999999999999,"15min":0.35899999999999999},"max":{"1min":2.726,"5min":55.052,"15min":55.052},"last":0.60699999999999998},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.2709999999999999,"15min":1.1160000000000001},"min":{"1min":0.439,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":3.1989999999999998,"5min":8.6010000000000009,"15min":40.113},"last":0.59099999999999997}]}]},{"osd":3,"up_from":99,"seq":425201762426,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2879268,"kb_used_data":2493212,"kb_used_omap":7,"kb_used_meta":386040,"kb_avail":90865884,"statfs":{"total":95995035648,"available":93046665216,"internally_reserved":0,"allocated":2553049088,"data_stored":2551296383,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":395305576},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,4,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:59:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":0.92200000000000004,"15min":1.3109999999999999},"min":{"1min":0.26800000000000002,"5min":0.186,"15min":0.186},"max":{"1min":3.6299999999999999,"5min":9.4510000000000005,"15min":15.536},"last":2.9340000000000002},{"interface":"front","average":{"1min":0.84899999999999998,"5min":1.3999999999999999,"15min":1.466},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.9289999999999998,"5min":14.707000000000001,"15min":14.707000000000001},"last":3.4020000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:58:42 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":1.0700000000000001,"15min":1.1579999999999999},"min":{"1min":0.28299999999999997,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":8.2379999999999995,"5min":14.568,"15min":127.226},"last":3.8460000000000001},{"interface":"front","average":{"1min":0.97099999999999997,"5min":0.83099999999999996,"15min":1.127},"min":{"1min":0.32600000000000001,"5min":0.24399999999999999,"15min":0.21099999999999999},"max":{"1min":3.8159999999999998,"5min":9.4239999999999995,"15min":127.468},"last":3.3149999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:58:42 2023","interfaces":[{"interface":"back","average":{"1min":4.4450000000000003,"5min":1.3959999999999999,"15min":1.0289999999999999},"min":{"1min":0.28100000000000003,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":69.421999999999997,"5min":69.421999999999997,"15min":69.421999999999997},"last":5.9880000000000004},{"interface":"front","average":{"1min":4.431,"5min":1.712,"15min":1.27},"min":{"1min":0.33900000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":69.355000000000004,"5min":69.355000000000004,"15min":69.355000000000004},"last":3.141}]},{"osd":4,"last update":"Mon Oct 9 05:58:42 2023","interfaces":[{"interface":"back","average":{"1min":2.0409999999999999,"5min":2.4510000000000001,"15min":1.335},"min":{"1min":0.40999999999999998,"5min":0.33000000000000002,"15min":0.28199999999999997},"max":{"1min":19.338000000000001,"5min":142.994,"15min":142.994},"last":2.8969999999999998},{"interface":"front","average":{"1min":1.262,"5min":2.5619999999999998,"15min":1.3300000000000001},"min":{"1min":0.32300000000000001,"5min":0.312,"15min":0.311},"max":{"1min":4.9109999999999996,"5min":143.411,"15min":143.411},"last":3.4489999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:58:42 2023","interfaces":[{"interface":"back","average":{"1min":1.9019999999999999,"5min":1.2789999999999999,"15min":0.90800000000000003},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.33800000000000002},"max":{"1min":11.488,"5min":14.404,"15min":14.404},"last":3.4980000000000002},{"interface":"front","average":{"1min":1.359,"5min":1.0529999999999999,"15min":0.83999999999999997},"min":{"1min":0.441,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.375,"5min":18.013999999999999,"15min":18.013999999999999},"last":2.819}]},{"osd":6,"last update":"Mon Oct 9 05:58:42 2023","interfaces":[{"interface":"back","average":{"1min":1.1739999999999999,"5min":1.635,"15min":1.077},"min":{"1min":0.376,"5min":0.376,"15min":0.30499999999999999},"max":{"1min":5.2039999999999997,"5min":44.079000000000001,"15min":44.079000000000001},"last":3.5259999999999998},{"interface":"front","average":{"1min":1.4650000000000001,"5min":1.4470000000000001,"15min":1.044},"min":{"1min":0.42399999999999999,"5min":0.318,"15min":0.318},"max":{"1min":5.4450000000000003,"5min":44.094000000000001,"15min":44.094000000000001},"last":2.8540000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:58:42 2023","interfaces":[{"interface":"back","average":{"1min":1.508,"5min":1.1419999999999999,"15min":0.91600000000000004},"min":{"1min":0.47399999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.4199999999999999,"5min":10.295,"15min":10.295},"last":2.8809999999999998},{"interface":"front","average":{"1min":1.661,"5min":1.0840000000000001,"15min":0.91300000000000003},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":16.306999999999999,"5min":16.306999999999999,"15min":16.306999999999999},"last":3.5950000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019146,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1554900,"kb_used_data":1257612,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":92190252,"statfs":{"total":95995035648,"available":94402818048,"internally_reserved":0,"allocated":1287794688,"data_stored":1286764117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:59:11 2023","interfaces":[{"interface":"back","average":{"1min":10.833,"5min":3.3929999999999998,"15min":1.7050000000000001},"min":{"1min":0.316,"5min":0.222,"15min":0.222},"max":{"1min":128.511,"5min":128.511,"15min":128.511},"last":6.3849999999999998},{"interface":"front","average":{"1min":10.657,"5min":3.621,"15min":2.2149999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":129.91800000000001,"5min":129.91800000000001,"15min":129.91800000000001},"last":4.5540000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:58:33 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":1.512,"15min":1.0349999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.6120000000000001,"5min":77.951999999999998,"15min":77.951999999999998},"last":6.4710000000000001},{"interface":"front","average":{"1min":0.61199999999999999,"5min":1.6970000000000001,"15min":1.0329999999999999},"min":{"1min":0.34499999999999997,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":1.6870000000000001,"5min":77.968000000000004,"15min":77.968000000000004},"last":5.016}]},{"osd":2,"last update":"Mon Oct 9 05:59:10 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":2.23,"15min":1.3959999999999999},"min":{"1min":0.31,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.415,"5min":136.15700000000001,"15min":136.15700000000001},"last":6.3579999999999997},{"interface":"front","average":{"1min":1.01,"5min":2.4649999999999999,"15min":1.52},"min":{"1min":0.40500000000000003,"5min":0.313,"15min":0.313},"max":{"1min":6.1870000000000003,"5min":136.30500000000001,"15min":136.30500000000001},"last":8.718}]},{"osd":3,"last update":"Mon Oct 9 05:58:33 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":1.2789999999999999,"15min":1.1120000000000001},"min":{"1min":0.35999999999999999,"5min":0.30599999999999999,"15min":0.29299999999999998},"max":{"1min":1.72,"5min":12.554,"15min":40.531999999999996},"last":6.2690000000000001},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.3160000000000001,"15min":1.1579999999999999},"min":{"1min":0.36599999999999999,"5min":0.31,"15min":0.31},"max":{"1min":3.4729999999999999,"5min":12.403,"15min":39.905999999999999},"last":5.2859999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:58:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":0.77200000000000002,"15min":0.69199999999999995},"min":{"1min":0.28299999999999997,"5min":0.27500000000000002,"15min":0.25900000000000001},"max":{"1min":4.0780000000000003,"5min":4.9349999999999996,"15min":10.012},"last":4.7610000000000001},{"interface":"front","average":{"1min":0.94799999999999995,"5min":0.77200000000000002,"15min":0.76800000000000002},"min":{"1min":0.29599999999999999,"5min":0.254,"15min":0.254},"max":{"1min":5.3319999999999999,"5min":12.121,"15min":12.489000000000001},"last":6.4130000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:58:49 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.69799999999999995,"15min":0.73499999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.8680000000000001,"5min":5.4379999999999997,"15min":11.234999999999999},"last":4.8620000000000001},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.86199999999999999,"15min":0.72399999999999998},"min":{"1min":0.32200000000000001,"5min":0.30599999999999999,"15min":0.30299999999999999},"max":{"1min":3.6219999999999999,"5min":8.8610000000000007,"15min":9.923},"last":4.468}]},{"osd":7,"last update":"Mon Oct 9 05:59:12 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.83999999999999997,"15min":0.86599999999999999},"min":{"1min":0.34599999999999997,"5min":0.30099999999999999,"15min":0.28599999999999998},"max":{"1min":1.069,"5min":6.0570000000000004,"15min":10.606},"last":6.2050000000000001},{"interface":"front","average":{"1min":1.034,"5min":0.85499999999999998,"15min":0.78100000000000003},"min":{"1min":0.35499999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.726,"5min":5.524,"15min":8.3819999999999997},"last":4.9900000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757509,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1323208,"kb_used_data":1017728,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92421944,"statfs":{"total":95995035648,"available":94640070656,"internally_reserved":0,"allocated":1042153472,"data_stored":1041131477,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:59:12 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":1.1120000000000001,"15min":0.82199999999999995},"min":{"1min":0.23899999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":2.0209999999999999,"5min":24.956,"15min":24.956},"last":0.315},{"interface":"front","average":{"1min":0.96199999999999997,"5min":1.405,"15min":0.98099999999999998},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":7.0759999999999996,"5min":24.975000000000001,"15min":24.975000000000001},"last":0.435}]},{"osd":1,"last update":"Mon Oct 9 05:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":0.97599999999999998,"15min":1.6679999999999999},"min":{"1min":0.32000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.0220000000000002,"5min":6.8410000000000002,"15min":114.247},"last":3.1339999999999999},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.64800000000000002,"15min":1.528},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.9140000000000001,"5min":2.9140000000000001,"15min":114.678},"last":0.94299999999999995}]},{"osd":2,"last update":"Mon Oct 9 05:59:02 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":3.1440000000000001,"15min":2.0089999999999999},"min":{"1min":0.29299999999999998,"5min":0.28399999999999997,"15min":0.25},"max":{"1min":0.93999999999999995,"5min":217.06399999999999,"15min":217.06399999999999},"last":2.8050000000000002},{"interface":"front","average":{"1min":0.57899999999999996,"5min":3.0499999999999998,"15min":1.9710000000000001},"min":{"1min":0.32400000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.5509999999999999,"5min":217.018,"15min":217.018},"last":2.5129999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:58:35 2023","interfaces":[{"interface":"back","average":{"1min":10.228,"5min":4.6369999999999996,"15min":4.3600000000000003},"min":{"1min":0.29799999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":211.40199999999999,"5min":211.40199999999999,"15min":272.81299999999999},"last":3.4279999999999999},{"interface":"front","average":{"1min":10.225,"5min":4.9059999999999997,"15min":4.5919999999999996},"min":{"1min":0.36099999999999999,"5min":0.317,"15min":0.317},"max":{"1min":211.578,"5min":211.578,"15min":273.00999999999999},"last":3.3210000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:58:25 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.27400000000000002,"5min":0.26400000000000001,"15min":0.24199999999999999},"max":{"1min":2.8620000000000001,"5min":3.1080000000000001,"15min":6.2050000000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.73699999999999999,"5min":0.63,"15min":0.61599999999999999},"min":{"1min":0.313,"5min":0.27600000000000002,"15min":0.26000000000000001},"max":{"1min":3.879,"5min":3.879,"15min":13.050000000000001},"last":0.45000000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:58:38 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":0.83799999999999997,"15min":0.68999999999999995},"min":{"1min":0.34000000000000002,"5min":0.30199999999999999,"15min":0.26600000000000001},"max":{"1min":5.5750000000000002,"5min":10.94,"15min":10.94},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.79200000000000004,"15min":0.74099999999999999},"min":{"1min":0.36599999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.3399999999999999,"5min":5.5069999999999997,"15min":5.9770000000000003},"last":0.56799999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:59:18 2023","interfaces":[{"interface":"back","average":{"1min":6.5099999999999998,"5min":2.0739999999999998,"15min":1.228},"min":{"1min":0.34699999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":106.01000000000001,"5min":106.01000000000001,"15min":106.01000000000001},"last":0.63300000000000001},{"interface":"front","average":{"1min":6.524,"5min":1.885,"15min":1.0609999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.32000000000000001},"max":{"1min":106.69,"5min":106.69,"15min":106.69},"last":0.50900000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1474,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2062090240,"data_stored":2062090240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2000093184,"data_stored":2000093184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2569666560,"data_stored":2569666560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1287127040,"data_stored":1287127040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1042087936,"data_stored":1042087936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1504903168,"data_stored":1504903168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1821114368,"data_stored":1821114368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:59:23.944 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:59:23.944 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 31 2023-10-09T05:59:23.983 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 45 16 66735 66719 92.6289 46.875 0.0145229 0.0107902 2023-10-09T05:59:24.013 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:24.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:24.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:24.121 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:24.125 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:23 smithi156 ceph-mon[195101]: pgmap v681: 109 pgs: 109 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T05:59:24.125 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:23 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:59:24.125 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:23 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T05:59:24.125 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:23 smithi156 ceph-mon[195101]: failed to encode map e297 with expected crc 2023-10-09T05:59:24.126 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:23 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:59:24.126 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:23 smithi156 ceph-mon[195101]: osdmap e297: 8 total, 8 up, 7 in 2023-10-09T05:59:24.126 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:23 smithi156 ceph-mon[195101]: failed to encode map e297 with expected crc 2023-10-09T05:59:24.126 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:23 smithi156 ceph-mon[195101]: failed to encode map e297 with expected crc 2023-10-09T05:59:24.126 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:23 smithi156 ceph-mon[195101]: failed to encode map e297 with expected crc 2023-10-09T05:59:24.239 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:24.247 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:24.342 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:59:24.402 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:24.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:24.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:24.520 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:24.587 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:24.848 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:24.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:24.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:24.956 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:24.985 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 46 16 67730 67714 91.9621 62.1875 0.0133405 0.010869 2023-10-09T05:59:25.018 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:25.208 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:25.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:25.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:25.305 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:59:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:59:25] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.43.0" 2023-10-09T05:59:25.312 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:25.369 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:25.958 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:25.977 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:25.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:25.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:25.994 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 47 16 68647 68631 91.2073 57.3125 0.0156757 0.010958 2023-10-09T05:59:26.094 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:26.105 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:59:26.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:25 smithi064 ceph-mon[212750]: pgmap v683: 109 pgs: 109 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T05:59:26.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:25 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:26.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:25 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:26.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:25 smithi064 ceph-mon[220044]: pgmap v683: 109 pgs: 109 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T05:59:26.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:25 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:26.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:25 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:26.167 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:26.232 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T05:59:26.233 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:59:26.257 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:25 smithi156 ceph-mon[195101]: pgmap v683: 109 pgs: 109 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T05:59:26.257 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:25 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:26.258 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:25 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:26.567 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:26.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:26.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:26.672 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:26.702 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:26.800 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:59:26.809 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:26.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:26.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:26.843 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:59:26.892 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:59:26.892 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:26 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:59:26.893 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:26 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:59:26.916 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:26.923 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:59:26.943 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:59:26.948 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:26.974 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:59:26.999 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 48 16 69469 69453 90.3702 51.375 0.0104881 0.0110602 2023-10-09T05:59:27.015 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:59:27.055 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:59:27.063 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:59:27.063 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:59:27.063 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:59:27.063 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:59:27.063 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:59:27.063 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:59:27.063 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:59:27.063 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:59:27.063 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:27.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:27.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:27.100 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:26 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:59:27.100 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:26 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:59:27.100 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:26 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:27.101 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:26 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3735833497' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:27.115 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:59:27.166 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:27.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:26 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:27.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:26 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3735833497' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:27.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:26 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:59:27.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:26 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:59:27.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:26 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:27.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:26 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3735833497' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:27.195 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:27.239 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:27.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:27.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:27.343 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:27.413 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:27.483 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:27.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:27.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:27.593 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:27.647 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:59:27.685 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:27.999 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 49 16 70239 70223 89.5084 48.125 0.00296601 0.0111562 2023-10-09T05:59:28.306 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:28.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:28.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:28.410 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:28.437 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:28.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:28 smithi064 ceph-mon[212750]: pgmap v684: 109 pgs: 109 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 61 MiB/s wr, 974 op/s 2023-10-09T05:59:28.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:28 smithi064 ceph-mon[220044]: pgmap v684: 109 pgs: 109 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 61 MiB/s wr, 974 op/s 2023-10-09T05:59:28.469 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:28.592 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:28.595 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:28 smithi156 ceph-mon[195101]: pgmap v684: 109 pgs: 109 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 61 MiB/s wr, 974 op/s 2023-10-09T05:59:28.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:28.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:28.697 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:28.740 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:28.950 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:28.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:28.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:28.999 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 50 16 71014 70998 88.6874 48.4375 0.0135213 0.01127 2023-10-09T05:59:29.060 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:29.127 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:29.451 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:29.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:29.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:29.485 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 1 pg_num to 31 2023-10-09T05:59:29.510 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:29 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:29.511 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:29.511 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:29 smithi064 ceph-mon[220044]: pgmap v685: 109 pgs: 109 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 59 MiB/s wr, 948 op/s 2023-10-09T05:59:29.511 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:29 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1549727974' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T05:59:29.511 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:29 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T05:59:29.511 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:29 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:29.511 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:29.511 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:29 smithi064 ceph-mon[212750]: pgmap v685: 109 pgs: 109 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 59 MiB/s wr, 948 op/s 2023-10-09T05:59:29.511 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:29 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1549727974' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T05:59:29.512 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:29 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T05:59:29.557 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:29.576 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T05:59:29.577 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T05:59:29.577 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T05:59:29.577 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T05:59:29.577 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.577 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64016 6.9 6:964728b5:::benchmark_data_smithi064_284193_object64015:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:59:29.577 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.109619+0000", 2023-10-09T05:59:29.577 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 476.463795242, 2023-10-09T05:59:29.577 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0912698940000001, 2023-10-09T05:59:29.577 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:29.578 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:29.578 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:59:29.578 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:59:29.578 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:29.578 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64016 2023-10-09T05:59:29.578 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.578 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:29.578 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.578 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:29.578 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.109619+0000", 2023-10-09T05:59:29.578 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.579 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.579 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.579 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:29.579 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.109619+0000", 2023-10-09T05:59:29.579 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.579 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.579 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.579 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:29.579 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.109619+0000", 2023-10-09T05:59:29.579 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:59:29.579 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.579 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.580 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:29.580 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.109650+0000", 2023-10-09T05:59:29.580 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1087999999999998e-05 2023-10-09T05:59:29.580 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.580 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.580 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:29.580 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.109657+0000", 2023-10-09T05:59:29.580 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4290000000000004e-06 2023-10-09T05:59:29.580 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.580 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.580 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:29.580 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.109662+0000", 2023-10-09T05:59:29.581 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4510000000000002e-06 2023-10-09T05:59:29.581 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.581 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.581 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.581 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.115072+0000", 2023-10-09T05:59:29.581 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0054107180000000001 2023-10-09T05:59:29.581 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.581 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.581 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.581 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.115080+0000", 2023-10-09T05:59:29.581 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3710000000000002e-06 2023-10-09T05:59:29.582 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.582 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.582 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.582 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.189454+0000", 2023-10-09T05:59:29.582 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.074374384 2023-10-09T05:59:29.582 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.582 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.582 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:29.582 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.190617+0000", 2023-10-09T05:59:29.582 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0011633150000000001 2023-10-09T05:59:29.582 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.582 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.583 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2", 2023-10-09T05:59:29.583 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.190779+0000", 2023-10-09T05:59:29.583 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00016118799999999999 2023-10-09T05:59:29.583 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.583 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.583 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:59:29.583 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.198531+0000", 2023-10-09T05:59:29.583 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0077525600000000004 2023-10-09T05:59:29.583 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.583 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.583 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.583 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200740+0000", 2023-10-09T05:59:29.584 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002208785 2023-10-09T05:59:29.584 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.584 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.584 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:59:29.584 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200795+0000", 2023-10-09T05:59:29.584 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5225000000000001e-05 2023-10-09T05:59:29.584 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.584 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.584 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:59:29.584 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200889+0000", 2023-10-09T05:59:29.584 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3837000000000003e-05 2023-10-09T05:59:29.584 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.585 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:29.585 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.585 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.585 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.585 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64021 6.9 6:9b197d79:::benchmark_data_smithi064_284193_object64020:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:59:29.585 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.123178+0000", 2023-10-09T05:59:29.585 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 476.450236459, 2023-10-09T05:59:29.585 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0778894430000001, 2023-10-09T05:59:29.585 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:29.585 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:29.585 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:59:29.585 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:59:29.586 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:29.586 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64021 2023-10-09T05:59:29.586 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.586 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:29.586 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.586 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:29.586 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.123178+0000", 2023-10-09T05:59:29.586 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.586 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.586 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.586 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:29.587 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.123178+0000", 2023-10-09T05:59:29.587 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.587 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.587 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.587 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:29.587 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.123177+0000", 2023-10-09T05:59:29.587 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:59:29.587 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.587 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.587 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:29.587 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.123212+0000", 2023-10-09T05:59:29.587 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5641e-05 2023-10-09T05:59:29.588 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.588 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.588 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:29.588 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.123221+0000", 2023-10-09T05:59:29.588 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0909999999999999e-06 2023-10-09T05:59:29.588 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.588 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.588 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:29.588 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.123227+0000", 2023-10-09T05:59:29.588 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4469999999999997e-06 2023-10-09T05:59:29.588 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.588 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.589 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.589 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.124899+0000", 2023-10-09T05:59:29.589 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001671797 2023-10-09T05:59:29.589 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.589 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.589 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.589 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.124905+0000", 2023-10-09T05:59:29.589 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8650000000000003e-06 2023-10-09T05:59:29.589 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.589 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.589 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.589 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.192517+0000", 2023-10-09T05:59:29.590 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0676111340000001 2023-10-09T05:59:29.590 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.590 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.590 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:29.590 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.192607+0000", 2023-10-09T05:59:29.590 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0493000000000002e-05 2023-10-09T05:59:29.590 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.590 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.590 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2", 2023-10-09T05:59:29.590 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.192721+0000", 2023-10-09T05:59:29.590 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000114003 2023-10-09T05:59:29.591 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.591 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.591 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:59:29.591 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200480+0000", 2023-10-09T05:59:29.591 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0077586089999999996 2023-10-09T05:59:29.591 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.591 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.591 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.591 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200926+0000", 2023-10-09T05:59:29.591 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00044662899999999998 2023-10-09T05:59:29.591 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.591 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.592 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:59:29.592 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.200976+0000", 2023-10-09T05:59:29.592 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9607999999999997e-05 2023-10-09T05:59:29.592 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.592 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.592 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:59:29.592 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.201067+0000", 2023-10-09T05:59:29.592 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1414000000000003e-05 2023-10-09T05:59:29.592 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.592 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:29.592 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.592 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.593 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.593 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:64027 6.9 6:98f248d6:::benchmark_data_smithi064_284193_object64026:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T05:59:29.593 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:51:33.133112+0000", 2023-10-09T05:59:29.593 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 476.440302691, 2023-10-09T05:59:29.593 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0717281329999999, 2023-10-09T05:59:29.593 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:29.593 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:29.593 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:59:29.593 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:59:29.593 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:29.593 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 64027 2023-10-09T05:59:29.594 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.594 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:29.594 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.594 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:29.594 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133112+0000", 2023-10-09T05:59:29.594 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.594 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.594 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.594 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:29.594 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133112+0000", 2023-10-09T05:59:29.595 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.595 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.595 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.595 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:29.595 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133110+0000", 2023-10-09T05:59:29.595 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:59:29.595 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.595 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.595 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:29.595 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133165+0000", 2023-10-09T05:59:29.595 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4601999999999997e-05 2023-10-09T05:59:29.595 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.596 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.596 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:29.596 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133174+0000", 2023-10-09T05:59:29.596 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6899999999999998e-06 2023-10-09T05:59:29.596 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.596 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.596 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:29.596 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133179+0000", 2023-10-09T05:59:29.596 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9509999999999997e-06 2023-10-09T05:59:29.596 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.596 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.597 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.597 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133222+0000", 2023-10-09T05:59:29.597 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3266000000000002e-05 2023-10-09T05:59:29.597 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.597 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.597 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.597 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:33.133229+0000", 2023-10-09T05:59:29.597 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.486e-06 2023-10-09T05:59:29.597 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.597 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.597 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.597 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.196460+0000", 2023-10-09T05:59:29.598 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0632303270000001 2023-10-09T05:59:29.598 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.598 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.598 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:29.598 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.196575+0000", 2023-10-09T05:59:29.598 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000114918 2023-10-09T05:59:29.598 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.598 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.598 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2", 2023-10-09T05:59:29.598 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.196709+0000", 2023-10-09T05:59:29.598 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013413100000000001 2023-10-09T05:59:29.598 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.599 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.599 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.599 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.203829+0000", 2023-10-09T05:59:29.599 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0071203179999999996 2023-10-09T05:59:29.599 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.599 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.599 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:59:29.599 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204716+0000", 2023-10-09T05:59:29.599 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00088727100000000002 2023-10-09T05:59:29.599 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.599 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.599 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:59:29.600 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204766+0000", 2023-10-09T05:59:29.600 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9855999999999998e-05 2023-10-09T05:59:29.600 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.600 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.600 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:59:29.600 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:51:34.204840+0000", 2023-10-09T05:59:29.600 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3491999999999994e-05 2023-10-09T05:59:29.600 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.600 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:29.600 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.600 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.601 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.601 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:104350 6.15 6:af513ac9:::benchmark_data_smithi064_284193_object7807:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2023-10-09T05:59:29.601 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:54:45.800000+0000", 2023-10-09T05:59:29.601 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 283.77341423500002, 2023-10-09T05:59:29.601 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.898542277, 2023-10-09T05:59:29.601 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:29.601 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:29.601 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:59:29.601 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:59:29.601 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:29.601 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 104350 2023-10-09T05:59:29.601 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.602 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:29.602 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.602 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:29.602 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.800000+0000", 2023-10-09T05:59:29.602 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.602 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.602 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.602 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:29.602 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.800000+0000", 2023-10-09T05:59:29.602 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.602 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.603 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.603 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:29.603 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.799999+0000", 2023-10-09T05:59:29.603 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:59:29.603 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.603 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.603 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:29.603 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.800004+0000", 2023-10-09T05:59:29.603 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4840000000000003e-06 2023-10-09T05:59:29.603 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.603 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.603 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:29.604 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.800015+0000", 2023-10-09T05:59:29.604 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0581000000000001e-05 2023-10-09T05:59:29.604 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.604 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.604 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:29.604 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.800030+0000", 2023-10-09T05:59:29.604 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4489e-05 2023-10-09T05:59:29.604 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.604 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.604 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.604 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.800068+0000", 2023-10-09T05:59:29.604 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8318999999999997e-05 2023-10-09T05:59:29.605 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.605 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.605 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.605 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.800076+0000", 2023-10-09T05:59:29.605 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6690000000000006e-06 2023-10-09T05:59:29.605 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.605 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.605 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.605 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:46.648630+0000", 2023-10-09T05:59:29.605 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.84855464599999997 2023-10-09T05:59:29.605 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.605 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.606 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.606 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:46.648639+0000", 2023-10-09T05:59:29.606 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.5259999999999993e-06 2023-10-09T05:59:29.606 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.606 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.606 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.606 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:47.674779+0000", 2023-10-09T05:59:29.606 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.026139881 2023-10-09T05:59:29.606 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.606 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.606 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:29.606 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:47.674949+0000", 2023-10-09T05:59:29.607 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000170765 2023-10-09T05:59:29.607 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.607 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.607 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:59:29.607 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:47.675251+0000", 2023-10-09T05:59:29.607 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000301682 2023-10-09T05:59:29.607 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.607 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.607 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.607 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:47.697358+0000", 2023-10-09T05:59:29.607 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.022106995000000001 2023-10-09T05:59:29.607 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.608 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.608 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:59:29.608 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:47.697368+0000", 2023-10-09T05:59:29.608 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0220999999999999e-05 2023-10-09T05:59:29.608 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.608 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.608 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.608 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:47.697409+0000", 2023-10-09T05:59:29.608 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0881999999999999e-05 2023-10-09T05:59:29.608 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.608 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.608 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:59:29.609 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:47.697470+0000", 2023-10-09T05:59:29.609 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0804000000000003e-05 2023-10-09T05:59:29.609 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.609 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.609 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:59:29.610 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:47.698542+0000", 2023-10-09T05:59:29.610 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001072386 2023-10-09T05:59:29.610 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.611 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:29.611 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.611 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.611 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.611 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:185875 6.15 6:aa8ef49b:::benchmark_data_smithi064_284193_object89332:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-10-09T05:59:29.611 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.701818+0000", 2023-10-09T05:59:29.611 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 98.871596175999997, 2023-10-09T05:59:29.611 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.183023392, 2023-10-09T05:59:29.611 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:29.611 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T05:59:29.611 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:59:29.611 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:59:29.612 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:29.612 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 185875 2023-10-09T05:59:29.612 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.612 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:29.612 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.612 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:29.612 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.701818+0000", 2023-10-09T05:59:29.612 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.612 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.612 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.612 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:29.612 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.701818+0000", 2023-10-09T05:59:29.613 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.613 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.613 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.613 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:29.613 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.701818+0000", 2023-10-09T05:59:29.613 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:59:29.613 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.613 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.613 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:29.613 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.701821+0000", 2023-10-09T05:59:29.613 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1410000000000001e-06 2023-10-09T05:59:29.614 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.614 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.614 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:29.614 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.701831+0000", 2023-10-09T05:59:29.614 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0142e-05 2023-10-09T05:59:29.614 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.614 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.614 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:29.614 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.701835+0000", 2023-10-09T05:59:29.614 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2919999999999997e-06 2023-10-09T05:59:29.614 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.614 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.615 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.615 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.707726+0000", 2023-10-09T05:59:29.615 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0058907179999999996 2023-10-09T05:59:29.615 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.615 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.615 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:29.615 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.707842+0000", 2023-10-09T05:59:29.615 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000115937 2023-10-09T05:59:29.615 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.615 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.615 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:59:29.617 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.708169+0000", 2023-10-09T05:59:29.617 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00032739800000000001 2023-10-09T05:59:29.617 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.617 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.617 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:59:29.617 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.711326+0000", 2023-10-09T05:59:29.617 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.003157235 2023-10-09T05:59:29.618 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.618 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.618 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.618 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.732073+0000", 2023-10-09T05:59:29.618 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.020746568999999999 2023-10-09T05:59:29.618 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.618 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.618 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.618 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.732081+0000", 2023-10-09T05:59:29.618 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8420000000000005e-06 2023-10-09T05:59:29.619 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.619 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.619 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.619 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735784+0000", 2023-10-09T05:59:29.619 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.003702782 2023-10-09T05:59:29.619 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.619 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.619 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.619 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735796+0000", 2023-10-09T05:59:29.619 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2009e-05 2023-10-09T05:59:29.619 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.619 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.620 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.620 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.884749+0000", 2023-10-09T05:59:29.620 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.148953603 2023-10-09T05:59:29.620 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.620 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.620 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:59:29.620 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.884842+0000", 2023-10-09T05:59:29.620 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2200000000000005e-05 2023-10-09T05:59:29.620 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.620 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:29.620 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.620 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.621 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.621 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:185894 6.15 6:aea4a65e:::benchmark_data_smithi064_284193_object89351:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T05:59:29.621 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.718597+0000", 2023-10-09T05:59:29.621 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 98.854817351999998, 2023-10-09T05:59:29.621 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1840398429999999, 2023-10-09T05:59:29.621 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:29.621 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:29.621 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:59:29.621 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:59:29.621 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:29.621 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 185894 2023-10-09T05:59:29.621 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.622 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:29.622 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.622 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:29.622 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.718597+0000", 2023-10-09T05:59:29.622 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.622 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.622 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.622 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:29.622 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.718597+0000", 2023-10-09T05:59:29.622 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.623 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.623 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.623 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:29.623 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.718596+0000", 2023-10-09T05:59:29.623 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:59:29.623 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.623 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.623 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:29.623 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.718616+0000", 2023-10-09T05:59:29.623 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0449999999999999e-05 2023-10-09T05:59:29.623 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.623 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.624 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:29.624 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.718644+0000", 2023-10-09T05:59:29.624 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7945000000000001e-05 2023-10-09T05:59:29.624 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.624 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.624 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:29.624 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.718651+0000", 2023-10-09T05:59:29.624 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6490000000000002e-06 2023-10-09T05:59:29.624 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.624 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.624 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.625 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.718696+0000", 2023-10-09T05:59:29.625 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4870000000000002e-05 2023-10-09T05:59:29.625 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.625 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.625 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op must wait for map", 2023-10-09T05:59:29.625 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.718703+0000", 2023-10-09T05:59:29.625 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2409999999999998e-06 2023-10-09T05:59:29.625 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.625 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.625 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.625 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.732103+0000", 2023-10-09T05:59:29.626 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013399708999999999 2023-10-09T05:59:29.626 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.626 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.626 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.626 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.732109+0000", 2023-10-09T05:59:29.626 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4869999999999999e-06 2023-10-09T05:59:29.626 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.626 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.626 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.626 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735816+0000", 2023-10-09T05:59:29.627 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.003706183 2023-10-09T05:59:29.627 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.627 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.627 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.627 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735822+0000", 2023-10-09T05:59:29.627 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8580000000000002e-06 2023-10-09T05:59:29.627 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.627 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.627 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.627 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.884859+0000", 2023-10-09T05:59:29.627 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1490370620000001 2023-10-09T05:59:29.627 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.628 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.628 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:29.628 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.884937+0000", 2023-10-09T05:59:29.628 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7639000000000007e-05 2023-10-09T05:59:29.628 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.628 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.628 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T05:59:29.628 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.885342+0000", 2023-10-09T05:59:29.628 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00040508599999999998 2023-10-09T05:59:29.628 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.628 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.629 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:59:29.629 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.893721+0000", 2023-10-09T05:59:29.629 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0083789090000000004 2023-10-09T05:59:29.629 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.629 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.629 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.629 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902271+0000", 2023-10-09T05:59:29.629 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0085503039999999999 2023-10-09T05:59:29.629 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.629 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.630 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.630 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902383+0000", 2023-10-09T05:59:29.630 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000111969 2023-10-09T05:59:29.630 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.630 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.630 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.630 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902489+0000", 2023-10-09T05:59:29.630 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010552099999999999 2023-10-09T05:59:29.630 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.630 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.630 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:59:29.631 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902536+0000", 2023-10-09T05:59:29.631 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7466999999999998e-05 2023-10-09T05:59:29.631 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.631 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.631 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:59:29.631 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902637+0000", 2023-10-09T05:59:29.631 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010043999999999999 2023-10-09T05:59:29.631 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.631 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:29.631 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.631 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.631 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.632 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:185895 6.15 6:a9b7b9d9:::benchmark_data_smithi064_284193_object89352:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T05:59:29.632 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.726247+0000", 2023-10-09T05:59:29.632 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 98.847167678999995, 2023-10-09T05:59:29.632 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1767582440000002, 2023-10-09T05:59:29.632 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:29.632 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:29.632 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:59:29.632 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:59:29.632 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:29.632 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 185895 2023-10-09T05:59:29.632 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.632 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:29.633 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.633 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:29.633 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.726247+0000", 2023-10-09T05:59:29.633 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.633 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.633 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.633 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:29.633 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.726247+0000", 2023-10-09T05:59:29.633 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.633 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.633 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.634 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:29.634 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.726245+0000", 2023-10-09T05:59:29.634 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:59:29.634 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.634 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.634 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:29.634 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.726252+0000", 2023-10-09T05:59:29.634 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1350000000000003e-06 2023-10-09T05:59:29.634 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.634 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.634 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:29.634 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.726264+0000", 2023-10-09T05:59:29.635 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2021000000000001e-05 2023-10-09T05:59:29.635 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.635 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.635 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:29.635 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.726271+0000", 2023-10-09T05:59:29.635 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6869999999999997e-06 2023-10-09T05:59:29.635 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.635 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.635 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.635 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.726427+0000", 2023-10-09T05:59:29.635 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015600299999999999 2023-10-09T05:59:29.636 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.636 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.636 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting_for_map not empty", 2023-10-09T05:59:29.636 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.726433+0000", 2023-10-09T05:59:29.636 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9449999999999998e-06 2023-10-09T05:59:29.636 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.636 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.636 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.636 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.732127+0000", 2023-10-09T05:59:29.636 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0056935670000000001 2023-10-09T05:59:29.636 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.636 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.637 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.637 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.732133+0000", 2023-10-09T05:59:29.637 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1999999999999999e-06 2023-10-09T05:59:29.637 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.637 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.637 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.637 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735842+0000", 2023-10-09T05:59:29.637 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0037096569999999 2023-10-09T05:59:29.637 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.637 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.637 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.638 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735851+0000", 2023-10-09T05:59:29.638 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.3650000000000004e-06 2023-10-09T05:59:29.638 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.638 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.638 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.638 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887467+0000", 2023-10-09T05:59:29.638 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1516161149999999 2023-10-09T05:59:29.638 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.638 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.638 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:29.638 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887570+0000", 2023-10-09T05:59:29.638 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000103458 2023-10-09T05:59:29.639 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.639 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.639 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T05:59:29.639 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887826+0000", 2023-10-09T05:59:29.639 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000255212 2023-10-09T05:59:29.639 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.639 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.639 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:59:29.639 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.893729+0000", 2023-10-09T05:59:29.639 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0059037949999999999 2023-10-09T05:59:29.639 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.639 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.640 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.640 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902299+0000", 2023-10-09T05:59:29.640 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0085701249999999996 2023-10-09T05:59:29.640 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.640 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.640 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.640 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902678+0000", 2023-10-09T05:59:29.640 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000379069 2023-10-09T05:59:29.640 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.640 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.640 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.641 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902845+0000", 2023-10-09T05:59:29.641 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00016605400000000001 2023-10-09T05:59:29.641 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.641 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.641 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:59:29.641 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902896+0000", 2023-10-09T05:59:29.641 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1298999999999998e-05 2023-10-09T05:59:29.641 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.641 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.641 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:59:29.641 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.903005+0000", 2023-10-09T05:59:29.641 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000109002 2023-10-09T05:59:29.642 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.642 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:29.642 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.642 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.642 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.642 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:185920 6.15 6:aed0640a:::benchmark_data_smithi064_284193_object89377:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T05:59:29.642 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.758471+0000", 2023-10-09T05:59:29.642 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 98.814942904000006, 2023-10-09T05:59:29.642 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1447898919999999, 2023-10-09T05:59:29.642 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:29.642 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:29.643 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:59:29.643 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:59:29.643 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:29.643 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 185920 2023-10-09T05:59:29.643 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.643 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:29.643 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.643 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:29.643 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.758471+0000", 2023-10-09T05:59:29.643 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.643 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.643 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.644 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:29.644 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.758471+0000", 2023-10-09T05:59:29.644 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.644 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.644 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.644 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:29.644 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.758470+0000", 2023-10-09T05:59:29.644 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:59:29.644 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.644 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.644 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:29.644 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.758476+0000", 2023-10-09T05:59:29.645 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3690000000000003e-06 2023-10-09T05:59:29.645 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.645 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.645 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:29.645 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.758490+0000", 2023-10-09T05:59:29.645 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3767e-05 2023-10-09T05:59:29.645 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.645 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.645 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:29.645 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.758496+0000", 2023-10-09T05:59:29.645 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0610000000000001e-06 2023-10-09T05:59:29.646 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.646 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.646 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.646 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.758550+0000", 2023-10-09T05:59:29.646 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3894000000000003e-05 2023-10-09T05:59:29.646 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.646 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.646 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.646 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.758556+0000", 2023-10-09T05:59:29.647 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6799999999999998e-06 2023-10-09T05:59:29.647 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.647 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.647 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.647 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735869+0000", 2023-10-09T05:59:29.647 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.97731343199999998 2023-10-09T05:59:29.647 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.647 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.647 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.647 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735875+0000", 2023-10-09T05:59:29.647 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0639999999999999e-06 2023-10-09T05:59:29.648 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.648 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.648 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.648 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.889339+0000", 2023-10-09T05:59:29.648 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1534636359999999 2023-10-09T05:59:29.648 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.648 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.648 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:29.648 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.889423+0000", 2023-10-09T05:59:29.648 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.4303999999999998e-05 2023-10-09T05:59:29.648 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.648 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.649 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T05:59:29.649 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.889619+0000", 2023-10-09T05:59:29.649 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000195606 2023-10-09T05:59:29.649 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.649 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.649 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:59:29.649 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.896800+0000", 2023-10-09T05:59:29.649 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0071807390000000002 2023-10-09T05:59:29.649 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.649 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.649 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.649 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902329+0000", 2023-10-09T05:59:29.650 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0055295759999999996 2023-10-09T05:59:29.650 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.650 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.650 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.650 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902708+0000", 2023-10-09T05:59:29.650 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00037905300000000002 2023-10-09T05:59:29.650 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.650 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.650 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.650 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.903105+0000", 2023-10-09T05:59:29.650 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00039682200000000002 2023-10-09T05:59:29.651 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.651 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.651 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:59:29.651 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.903155+0000", 2023-10-09T05:59:29.651 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9982999999999999e-05 2023-10-09T05:59:29.651 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.651 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.651 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:59:29.651 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.903261+0000", 2023-10-09T05:59:29.651 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010619399999999999 2023-10-09T05:59:29.651 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.651 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:29.652 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.652 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.652 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.652 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:185935 6.25 6:a62e1056:::benchmark_data_smithi064_284193_object89392:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T05:59:29.652 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.768157+0000", 2023-10-09T05:59:29.652 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 98.805256845000002, 2023-10-09T05:59:29.652 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1328383999999998, 2023-10-09T05:59:29.652 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:29.652 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:29.652 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:59:29.652 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:59:29.653 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:29.653 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 185935 2023-10-09T05:59:29.653 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.653 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:29.653 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.653 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:29.653 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.768157+0000", 2023-10-09T05:59:29.653 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.653 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.653 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.653 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:29.654 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.768157+0000", 2023-10-09T05:59:29.654 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.654 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.654 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.654 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:29.654 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.768157+0000", 2023-10-09T05:59:29.654 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:59:29.654 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.654 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.654 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:29.654 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.768166+0000", 2023-10-09T05:59:29.655 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9919999999999996e-06 2023-10-09T05:59:29.655 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.655 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.655 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:29.655 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.768179+0000", 2023-10-09T05:59:29.655 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3552e-05 2023-10-09T05:59:29.655 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.655 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.655 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:29.655 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.768186+0000", 2023-10-09T05:59:29.655 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5459999999999997e-06 2023-10-09T05:59:29.656 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.656 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.656 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.656 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.768252+0000", 2023-10-09T05:59:29.656 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6629000000000002e-05 2023-10-09T05:59:29.656 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.656 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.656 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.656 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.768259+0000", 2023-10-09T05:59:29.656 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4389999999999997e-06 2023-10-09T05:59:29.656 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.656 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.657 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.657 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737077+0000", 2023-10-09T05:59:29.657 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96881812599999995 2023-10-09T05:59:29.657 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.657 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.657 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.657 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737085+0000", 2023-10-09T05:59:29.657 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.4970000000000001e-06 2023-10-09T05:59:29.657 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.657 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.657 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.658 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.870073+0000", 2023-10-09T05:59:29.658 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1329872590000001 2023-10-09T05:59:29.658 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.658 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.658 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:29.658 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.870168+0000", 2023-10-09T05:59:29.658 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.5539999999999997e-05 2023-10-09T05:59:29.658 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.658 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.658 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:59:29.658 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.870401+0000", 2023-10-09T05:59:29.659 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000232465 2023-10-09T05:59:29.659 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.659 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.659 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:59:29.659 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887473+0000", 2023-10-09T05:59:29.659 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.017071975999999999 2023-10-09T05:59:29.659 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.659 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.659 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.659 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900782+0000", 2023-10-09T05:59:29.659 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013308943 2023-10-09T05:59:29.659 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.660 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.660 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.660 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900835+0000", 2023-10-09T05:59:29.660 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3684999999999998e-05 2023-10-09T05:59:29.660 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.660 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.660 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:59:29.660 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900869+0000", 2023-10-09T05:59:29.660 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.3395000000000001e-05 2023-10-09T05:59:29.660 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.660 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.660 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:59:29.661 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900996+0000", 2023-10-09T05:59:29.661 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012716 2023-10-09T05:59:29.661 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.661 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:29.661 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.661 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.661 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.661 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:185975 6.15 6:ad237c98:::benchmark_data_smithi064_284193_object89432:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T05:59:29.661 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.808092+0000", 2023-10-09T05:59:29.661 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 98.765322488999999, 2023-10-09T05:59:29.661 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.100164849, 2023-10-09T05:59:29.662 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:29.662 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:29.662 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:59:29.662 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:59:29.662 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:29.662 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 185975 2023-10-09T05:59:29.662 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.662 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:29.662 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.662 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:29.662 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.808092+0000", 2023-10-09T05:59:29.663 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.663 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.663 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.663 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:29.663 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.808092+0000", 2023-10-09T05:59:29.663 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.663 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.663 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.663 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:29.663 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.808091+0000", 2023-10-09T05:59:29.663 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:59:29.664 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.664 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.664 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:29.664 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.808098+0000", 2023-10-09T05:59:29.664 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8979999999999999e-06 2023-10-09T05:59:29.664 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.664 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.664 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:29.664 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.808110+0000", 2023-10-09T05:59:29.664 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.269e-05 2023-10-09T05:59:29.664 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.664 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.665 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:29.665 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.808118+0000", 2023-10-09T05:59:29.665 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1369999999999996e-06 2023-10-09T05:59:29.665 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.665 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.665 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.665 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.808181+0000", 2023-10-09T05:59:29.665 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3409000000000002e-05 2023-10-09T05:59:29.665 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.665 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.665 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.666 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.808190+0000", 2023-10-09T05:59:29.666 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8319999999999995e-06 2023-10-09T05:59:29.666 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.666 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.666 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.666 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735892+0000", 2023-10-09T05:59:29.666 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.92770270899999996 2023-10-09T05:59:29.666 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.666 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.666 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.667 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735904+0000", 2023-10-09T05:59:29.667 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1919999999999999e-05 2023-10-09T05:59:29.667 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.667 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.667 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.667 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.890799+0000", 2023-10-09T05:59:29.667 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1548949850000001 2023-10-09T05:59:29.667 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.667 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.667 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:29.667 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.890900+0000", 2023-10-09T05:59:29.667 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000100787 2023-10-09T05:59:29.668 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.668 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.668 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T05:59:29.668 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.891095+0000", 2023-10-09T05:59:29.668 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00019488099999999999 2023-10-09T05:59:29.668 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.668 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.668 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:59:29.668 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.896810+0000", 2023-10-09T05:59:29.668 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0057148370000000004 2023-10-09T05:59:29.669 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.669 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.669 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.669 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902357+0000", 2023-10-09T05:59:29.669 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0055466739999999997 2023-10-09T05:59:29.669 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.669 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.669 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.669 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902737+0000", 2023-10-09T05:59:29.669 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00038066800000000002 2023-10-09T05:59:29.669 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.669 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.670 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.670 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908154+0000", 2023-10-09T05:59:29.670 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0054167620000000003 2023-10-09T05:59:29.670 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.670 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.670 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:59:29.670 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908191+0000", 2023-10-09T05:59:29.670 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7197000000000003e-05 2023-10-09T05:59:29.670 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.670 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.671 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:59:29.671 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908257+0000", 2023-10-09T05:59:29.671 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5439999999999997e-05 2023-10-09T05:59:29.671 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.671 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:29.671 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.671 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.671 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.671 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:185985 6.5 6:a34f1df4:::benchmark_data_smithi064_284193_object89442:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T05:59:29.671 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.815985+0000", 2023-10-09T05:59:29.671 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 98.757429238, 2023-10-09T05:59:29.672 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0848691029999999, 2023-10-09T05:59:29.672 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:29.672 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:29.672 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:59:29.672 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:59:29.672 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:29.672 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 185985 2023-10-09T05:59:29.672 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.672 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:29.672 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.672 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:29.672 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.815985+0000", 2023-10-09T05:59:29.673 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.673 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.673 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.673 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:29.673 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.815985+0000", 2023-10-09T05:59:29.673 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.673 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.673 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.673 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:29.673 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.815984+0000", 2023-10-09T05:59:29.673 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:59:29.674 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.674 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.674 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:29.674 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.815991+0000", 2023-10-09T05:59:29.674 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.973e-06 2023-10-09T05:59:29.674 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.674 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.674 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:29.674 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.816004+0000", 2023-10-09T05:59:29.674 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3162999999999999e-05 2023-10-09T05:59:29.674 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.675 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.675 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:29.675 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.816009+0000", 2023-10-09T05:59:29.675 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8049999999999997e-06 2023-10-09T05:59:29.675 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.675 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.675 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.675 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.816081+0000", 2023-10-09T05:59:29.675 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2614000000000006e-05 2023-10-09T05:59:29.675 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.675 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.676 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.676 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.816093+0000", 2023-10-09T05:59:29.676 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1919999999999999e-05 2023-10-09T05:59:29.676 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.676 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.676 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.676 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.738574+0000", 2023-10-09T05:59:29.676 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.922481047 2023-10-09T05:59:29.676 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.676 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.676 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.676 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.738585+0000", 2023-10-09T05:59:29.677 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0457e-05 2023-10-09T05:59:29.677 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.677 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.677 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.677 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.860265+0000", 2023-10-09T05:59:29.677 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1216806349999999 2023-10-09T05:59:29.677 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.677 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.677 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:29.677 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.860401+0000", 2023-10-09T05:59:29.678 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000135158 2023-10-09T05:59:29.678 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.678 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.678 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:59:29.678 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.860685+0000", 2023-10-09T05:59:29.678 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00028465499999999998 2023-10-09T05:59:29.678 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.678 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.678 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.678 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.875933+0000", 2023-10-09T05:59:29.678 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015247671000000001 2023-10-09T05:59:29.678 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.679 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.679 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:59:29.679 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887462+0000", 2023-10-09T05:59:29.679 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011528974000000001 2023-10-09T05:59:29.679 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.679 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.679 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.679 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900446+0000", 2023-10-09T05:59:29.679 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012984446 2023-10-09T05:59:29.679 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.679 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.680 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:59:29.680 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900505+0000", 2023-10-09T05:59:29.680 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8844000000000001e-05 2023-10-09T05:59:29.680 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.680 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.680 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:59:29.680 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900854+0000", 2023-10-09T05:59:29.680 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00034893000000000001 2023-10-09T05:59:29.680 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.680 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:29.680 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.681 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.681 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.681 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186010 6.25 6:a4768b4e:::benchmark_data_smithi064_284193_object89467:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T05:59:29.681 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.842412+0000", 2023-10-09T05:59:29.681 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 98.731001855000002, 2023-10-09T05:59:29.681 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.058999177, 2023-10-09T05:59:29.681 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:29.681 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:29.681 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:59:29.681 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:59:29.681 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:29.681 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186010 2023-10-09T05:59:29.682 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.682 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:29.682 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.682 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:29.682 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842412+0000", 2023-10-09T05:59:29.682 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.682 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.682 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.682 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:29.682 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842412+0000", 2023-10-09T05:59:29.682 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.683 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.683 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.683 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:29.683 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842411+0000", 2023-10-09T05:59:29.683 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:59:29.683 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.683 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.683 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:29.683 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842424+0000", 2023-10-09T05:59:29.683 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2785999999999999e-05 2023-10-09T05:59:29.683 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.684 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.684 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:29.684 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842440+0000", 2023-10-09T05:59:29.684 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5608e-05 2023-10-09T05:59:29.684 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.684 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.684 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:29.684 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842445+0000", 2023-10-09T05:59:29.684 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5300000000000004e-06 2023-10-09T05:59:29.684 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.684 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.685 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.685 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842503+0000", 2023-10-09T05:59:29.685 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7425e-05 2023-10-09T05:59:29.685 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.685 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.685 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.685 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842512+0000", 2023-10-09T05:59:29.685 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3090000000000001e-06 2023-10-09T05:59:29.685 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.685 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.685 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.685 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737132+0000", 2023-10-09T05:59:29.686 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.89461971600000001 2023-10-09T05:59:29.686 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.686 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.686 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.686 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737140+0000", 2023-10-09T05:59:29.686 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0819999999999999e-06 2023-10-09T05:59:29.686 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.686 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.686 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.686 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.871339+0000", 2023-10-09T05:59:29.686 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1341993539999999 2023-10-09T05:59:29.687 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.687 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.687 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:29.687 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.871455+0000", 2023-10-09T05:59:29.687 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000116223 2023-10-09T05:59:29.687 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.687 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.687 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:59:29.687 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.871601+0000", 2023-10-09T05:59:29.687 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000145215 2023-10-09T05:59:29.687 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.687 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.688 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:59:29.688 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887480+0000", 2023-10-09T05:59:29.688 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015879000000000001 2023-10-09T05:59:29.688 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.688 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.688 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.688 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900800+0000", 2023-10-09T05:59:29.688 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013320784 2023-10-09T05:59:29.688 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.688 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.688 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.688 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901152+0000", 2023-10-09T05:59:29.689 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00035196099999999997 2023-10-09T05:59:29.689 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.689 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.689 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:59:29.689 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901201+0000", 2023-10-09T05:59:29.689 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9029000000000001e-05 2023-10-09T05:59:29.689 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.689 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.689 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:59:29.689 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901412+0000", 2023-10-09T05:59:29.689 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00021017099999999999 2023-10-09T05:59:29.690 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.690 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:29.690 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.690 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.690 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.690 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186011 6.25 6:a4aaf67a:::benchmark_data_smithi064_284193_object89468:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T05:59:29.690 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.842461+0000", 2023-10-09T05:59:29.690 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 98.730953462000002, 2023-10-09T05:59:29.690 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.059291977, 2023-10-09T05:59:29.690 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:29.690 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:29.691 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:59:29.691 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:59:29.691 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:29.691 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186011 2023-10-09T05:59:29.691 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.691 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:29.691 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.691 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:29.691 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842461+0000", 2023-10-09T05:59:29.691 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.691 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.692 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.692 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:29.692 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842461+0000", 2023-10-09T05:59:29.692 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.692 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.692 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.692 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:29.692 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842460+0000", 2023-10-09T05:59:29.692 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:59:29.692 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.692 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.692 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:29.693 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842464+0000", 2023-10-09T05:59:29.693 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5949999999999999e-06 2023-10-09T05:59:29.693 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.693 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.693 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:29.693 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842470+0000", 2023-10-09T05:59:29.693 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4500000000000001e-06 2023-10-09T05:59:29.693 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.693 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.693 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:29.693 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842475+0000", 2023-10-09T05:59:29.693 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1629999999999998e-06 2023-10-09T05:59:29.694 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.694 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.694 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.694 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.844758+0000", 2023-10-09T05:59:29.694 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002283799 2023-10-09T05:59:29.694 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.694 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.694 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.694 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.844765+0000", 2023-10-09T05:59:29.694 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5250000000000002e-06 2023-10-09T05:59:29.694 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.695 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.695 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.695 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737182+0000", 2023-10-09T05:59:29.695 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.89241688699999999 2023-10-09T05:59:29.695 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.695 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.695 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.695 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737191+0000", 2023-10-09T05:59:29.695 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0219999999999993e-06 2023-10-09T05:59:29.695 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.695 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.696 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.696 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.872509+0000", 2023-10-09T05:59:29.696 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1353186339999999 2023-10-09T05:59:29.696 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.696 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.696 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:29.696 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.872596+0000", 2023-10-09T05:59:29.696 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6112000000000005e-05 2023-10-09T05:59:29.696 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.696 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.696 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:59:29.696 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.872731+0000", 2023-10-09T05:59:29.697 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000134964 2023-10-09T05:59:29.697 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.697 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.697 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:59:29.697 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887486+0000", 2023-10-09T05:59:29.697 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.014755367 2023-10-09T05:59:29.697 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.697 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.697 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.697 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900818+0000", 2023-10-09T05:59:29.697 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013332274 2023-10-09T05:59:29.698 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.698 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.698 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.698 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901502+0000", 2023-10-09T05:59:29.698 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00068389899999999997 2023-10-09T05:59:29.698 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.698 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.698 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:59:29.698 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901562+0000", 2023-10-09T05:59:29.698 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0087999999999998e-05 2023-10-09T05:59:29.698 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.699 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.699 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:59:29.699 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901753+0000", 2023-10-09T05:59:29.699 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00019062199999999999 2023-10-09T05:59:29.699 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.699 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:29.699 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.699 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.699 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.699 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186073 6.15 6:ac3db1f6:::benchmark_data_smithi064_284193_object89530:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T05:59:29.699 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.899281+0000", 2023-10-09T05:59:29.700 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 98.674133169000001, 2023-10-09T05:59:29.700 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0091446479999999, 2023-10-09T05:59:29.700 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:29.700 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:29.700 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:59:29.700 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:59:29.700 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:29.700 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186073 2023-10-09T05:59:29.700 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.700 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:29.700 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.700 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:29.701 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.899281+0000", 2023-10-09T05:59:29.701 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.701 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.701 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.701 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:29.701 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.899281+0000", 2023-10-09T05:59:29.701 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.701 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.701 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.701 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:29.701 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.899280+0000", 2023-10-09T05:59:29.701 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:59:29.702 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.702 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.702 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:29.702 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.899285+0000", 2023-10-09T05:59:29.702 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7600000000000002e-06 2023-10-09T05:59:29.702 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.702 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.702 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:29.702 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.899294+0000", 2023-10-09T05:59:29.702 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.397e-06 2023-10-09T05:59:29.702 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.703 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.703 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:29.703 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.899300+0000", 2023-10-09T05:59:29.703 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.412e-06 2023-10-09T05:59:29.703 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.703 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.703 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.703 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.899361+0000", 2023-10-09T05:59:29.703 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1532999999999995e-05 2023-10-09T05:59:29.703 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.703 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.704 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.704 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.899370+0000", 2023-10-09T05:59:29.704 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.4649999999999994e-06 2023-10-09T05:59:29.704 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.704 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.704 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.704 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735922+0000", 2023-10-09T05:59:29.704 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.83655174300000001 2023-10-09T05:59:29.704 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.704 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.704 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.704 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735928+0000", 2023-10-09T05:59:29.705 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4790000000000001e-06 2023-10-09T05:59:29.705 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.705 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.705 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.705 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.892186+0000", 2023-10-09T05:59:29.705 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.156257533 2023-10-09T05:59:29.705 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.705 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.705 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:29.705 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.892275+0000", 2023-10-09T05:59:29.705 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9951000000000007e-05 2023-10-09T05:59:29.705 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.706 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.706 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T05:59:29.706 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.892504+0000", 2023-10-09T05:59:29.706 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00022880200000000001 2023-10-09T05:59:29.706 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.706 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.706 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:59:29.706 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900294+0000", 2023-10-09T05:59:29.706 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.007790033 2023-10-09T05:59:29.707 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.707 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.707 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.707 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902411+0000", 2023-10-09T05:59:29.707 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0021167360000000001 2023-10-09T05:59:29.707 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.707 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.707 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.707 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.906259+0000", 2023-10-09T05:59:29.707 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0038482970000000001 2023-10-09T05:59:29.707 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.708 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.708 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.708 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908294+0000", 2023-10-09T05:59:29.708 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00203426 2023-10-09T05:59:29.708 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.708 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.708 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:59:29.708 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908329+0000", 2023-10-09T05:59:29.708 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5098999999999997e-05 2023-10-09T05:59:29.708 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.708 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.709 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:59:29.709 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908426+0000", 2023-10-09T05:59:29.709 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7058000000000005e-05 2023-10-09T05:59:29.709 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.709 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:29.709 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.709 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.709 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.709 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186096 6.25 6:a53a4882:::benchmark_data_smithi064_284193_object89553:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T05:59:29.709 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.926438+0000", 2023-10-09T05:59:29.709 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 98.646976605000006, 2023-10-09T05:59:29.709 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.975667222, 2023-10-09T05:59:29.710 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:29.710 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:29.710 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:59:29.710 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:59:29.710 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:29.710 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186096 2023-10-09T05:59:29.710 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.710 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:29.710 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.710 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:29.711 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.926438+0000", 2023-10-09T05:59:29.711 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.711 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.711 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.711 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:29.711 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.926438+0000", 2023-10-09T05:59:29.711 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.711 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.711 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.711 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:29.711 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.926436+0000", 2023-10-09T05:59:29.711 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:59:29.712 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.712 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.712 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:29.712 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.926443+0000", 2023-10-09T05:59:29.712 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3099999999999997e-06 2023-10-09T05:59:29.712 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.712 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.712 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:29.712 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.926455+0000", 2023-10-09T05:59:29.712 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2026e-05 2023-10-09T05:59:29.712 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.713 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.713 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:29.713 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.926460+0000", 2023-10-09T05:59:29.713 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8379999999999998e-06 2023-10-09T05:59:29.713 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.713 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.713 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.713 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.926522+0000", 2023-10-09T05:59:29.713 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1502999999999997e-05 2023-10-09T05:59:29.713 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.713 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.714 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.714 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.926531+0000", 2023-10-09T05:59:29.714 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6559999999999998e-06 2023-10-09T05:59:29.714 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.714 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.714 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.714 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737222+0000", 2023-10-09T05:59:29.714 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.81069113400000004 2023-10-09T05:59:29.714 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.714 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.715 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.715 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737229+0000", 2023-10-09T05:59:29.715 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6669999999999996e-06 2023-10-09T05:59:29.715 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.715 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.715 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.715 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.873463+0000", 2023-10-09T05:59:29.715 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1362332390000001 2023-10-09T05:59:29.715 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.715 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.715 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:29.715 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.873604+0000", 2023-10-09T05:59:29.716 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000141108 2023-10-09T05:59:29.716 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.716 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.716 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:59:29.716 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.873782+0000", 2023-10-09T05:59:29.716 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00017831699999999999 2023-10-09T05:59:29.716 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.716 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.716 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:59:29.716 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887492+0000", 2023-10-09T05:59:29.717 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013710119 2023-10-09T05:59:29.717 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.717 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.717 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.717 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901054+0000", 2023-10-09T05:59:29.717 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013561478 2023-10-09T05:59:29.717 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.717 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.717 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.718 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901854+0000", 2023-10-09T05:59:29.718 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00079998700000000005 2023-10-09T05:59:29.718 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.718 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.718 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:59:29.718 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901914+0000", 2023-10-09T05:59:29.718 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0770000000000003e-05 2023-10-09T05:59:29.718 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.718 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.718 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:59:29.718 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902105+0000", 2023-10-09T05:59:29.719 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000190497 2023-10-09T05:59:29.719 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.719 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:29.719 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.719 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.719 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.719 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186101 6.15 6:ac185365:::benchmark_data_smithi064_284193_object89558:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T05:59:29.719 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.933029+0000", 2023-10-09T05:59:29.719 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 98.640385773999995, 2023-10-09T05:59:29.719 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.97558872, 2023-10-09T05:59:29.719 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:29.720 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:29.720 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:59:29.720 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:59:29.720 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:29.720 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186101 2023-10-09T05:59:29.720 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.720 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:29.720 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.720 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:29.720 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.933029+0000", 2023-10-09T05:59:29.720 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.721 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.721 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.721 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:29.721 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.933029+0000", 2023-10-09T05:59:29.721 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.721 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.721 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.721 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:29.721 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.933027+0000", 2023-10-09T05:59:29.721 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:59:29.721 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.722 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.722 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:29.722 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.933033+0000", 2023-10-09T05:59:29.722 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1380000000000004e-06 2023-10-09T05:59:29.722 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.722 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.722 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:29.722 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.933048+0000", 2023-10-09T05:59:29.722 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4273e-05 2023-10-09T05:59:29.722 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.722 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.723 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:29.723 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.933053+0000", 2023-10-09T05:59:29.723 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.699e-06 2023-10-09T05:59:29.723 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.723 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.723 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.723 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.933117+0000", 2023-10-09T05:59:29.723 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3668000000000001e-05 2023-10-09T05:59:29.723 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.723 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.723 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.723 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.933125+0000", 2023-10-09T05:59:29.724 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8990000000000001e-06 2023-10-09T05:59:29.724 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.724 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.724 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.724 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735946+0000", 2023-10-09T05:59:29.724 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.802820796 2023-10-09T05:59:29.724 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.724 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.724 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.724 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735952+0000", 2023-10-09T05:59:29.724 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3230000000000003e-06 2023-10-09T05:59:29.725 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.725 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.725 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.725 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.893777+0000", 2023-10-09T05:59:29.725 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1578248010000001 2023-10-09T05:59:29.725 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.725 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.725 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:29.725 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.893902+0000", 2023-10-09T05:59:29.725 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012566899999999999 2023-10-09T05:59:29.726 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.726 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.726 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T05:59:29.726 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.894163+0000", 2023-10-09T05:59:29.726 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000260253 2023-10-09T05:59:29.726 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.726 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.726 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:59:29.726 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900304+0000", 2023-10-09T05:59:29.726 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0061414939999999999 2023-10-09T05:59:29.726 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.726 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.727 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.727 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902438+0000", 2023-10-09T05:59:29.727 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0021343109999999998 2023-10-09T05:59:29.727 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.727 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.727 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.727 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.906306+0000", 2023-10-09T05:59:29.727 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0038679690000000002 2023-10-09T05:59:29.727 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.727 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.727 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.728 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908502+0000", 2023-10-09T05:59:29.728 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00219549 2023-10-09T05:59:29.728 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.728 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.728 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:59:29.728 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908544+0000", 2023-10-09T05:59:29.728 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2407000000000002e-05 2023-10-09T05:59:29.728 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.728 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.728 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:59:29.728 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908617+0000", 2023-10-09T05:59:29.728 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2891000000000004e-05 2023-10-09T05:59:29.729 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.729 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:29.729 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.729 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.729 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.729 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186105 6.15 6:a80320df:::benchmark_data_smithi064_284193_object89562:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T05:59:29.729 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.936806+0000", 2023-10-09T05:59:29.729 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 98.636608338000002, 2023-10-09T05:59:29.729 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.971922425, 2023-10-09T05:59:29.729 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:29.730 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:29.730 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:59:29.730 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:59:29.730 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:29.730 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186105 2023-10-09T05:59:29.730 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.730 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:29.730 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.730 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:29.730 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.936806+0000", 2023-10-09T05:59:29.730 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.731 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.731 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.731 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:29.731 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.936806+0000", 2023-10-09T05:59:29.731 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.731 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.731 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.731 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:29.731 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.936805+0000", 2023-10-09T05:59:29.731 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T05:59:29.731 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.731 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.732 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:29.732 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.936812+0000", 2023-10-09T05:59:29.732 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0870000000000001e-06 2023-10-09T05:59:29.732 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.732 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.732 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:29.732 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.936826+0000", 2023-10-09T05:59:29.732 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4511999999999999e-05 2023-10-09T05:59:29.732 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.732 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.732 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:29.733 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.936833+0000", 2023-10-09T05:59:29.733 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9140000000000002e-06 2023-10-09T05:59:29.733 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.733 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.733 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.733 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.936903+0000", 2023-10-09T05:59:29.733 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0197999999999994e-05 2023-10-09T05:59:29.733 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.733 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.733 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.733 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.936913+0000", 2023-10-09T05:59:29.734 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3549999999999994e-06 2023-10-09T05:59:29.734 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.734 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.734 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.734 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735976+0000", 2023-10-09T05:59:29.734 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.79906334099999998 2023-10-09T05:59:29.734 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.734 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.734 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.734 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735983+0000", 2023-10-09T05:59:29.734 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8859999999999998e-06 2023-10-09T05:59:29.735 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.735 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.735 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.735 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.895376+0000", 2023-10-09T05:59:29.735 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.159393398 2023-10-09T05:59:29.735 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.735 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.735 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:29.735 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.895479+0000", 2023-10-09T05:59:29.735 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000103028 2023-10-09T05:59:29.735 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.736 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.736 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T05:59:29.736 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.895680+0000", 2023-10-09T05:59:29.736 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000200216 2023-10-09T05:59:29.736 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.736 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.736 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:59:29.736 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900382+0000", 2023-10-09T05:59:29.736 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0047027529999999996 2023-10-09T05:59:29.736 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.736 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.737 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.737 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902767+0000", 2023-10-09T05:59:29.737 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0023844389999999999 2023-10-09T05:59:29.737 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.737 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.737 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.737 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.906331+0000", 2023-10-09T05:59:29.737 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0035642719999999998 2023-10-09T05:59:29.737 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.737 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.737 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.737 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908649+0000", 2023-10-09T05:59:29.738 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0023178819999999998 2023-10-09T05:59:29.738 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.738 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.738 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:59:29.738 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908673+0000", 2023-10-09T05:59:29.738 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4329999999999999e-05 2023-10-09T05:59:29.738 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.738 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.738 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:59:29.738 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908728+0000", 2023-10-09T05:59:29.738 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5074999999999997e-05 2023-10-09T05:59:29.739 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.739 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:29.739 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.739 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.739 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.739 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186107 6.15 6:adc2ed22:::benchmark_data_smithi064_284193_object89564:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T05:59:29.739 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.937578+0000", 2023-10-09T05:59:29.739 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 98.635836327999996, 2023-10-09T05:59:29.739 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9730703780000001, 2023-10-09T05:59:29.739 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:29.739 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:29.739 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:59:29.740 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:59:29.740 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:29.740 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186107 2023-10-09T05:59:29.740 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.740 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:29.740 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.740 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:29.740 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.937578+0000", 2023-10-09T05:59:29.740 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.740 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.740 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.740 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:29.741 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.937578+0000", 2023-10-09T05:59:29.741 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.741 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.741 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.741 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:29.741 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.937577+0000", 2023-10-09T05:59:29.741 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:59:29.741 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.741 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.741 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:29.741 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.937586+0000", 2023-10-09T05:59:29.742 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0250000000000008e-06 2023-10-09T05:59:29.742 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.742 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.742 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:29.742 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.937601+0000", 2023-10-09T05:59:29.742 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5301000000000001e-05 2023-10-09T05:59:29.742 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.742 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.742 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:29.742 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.937608+0000", 2023-10-09T05:59:29.742 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1689999999999997e-06 2023-10-09T05:59:29.742 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.743 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.743 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.743 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.937654+0000", 2023-10-09T05:59:29.743 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6179e-05 2023-10-09T05:59:29.743 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.743 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.743 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.743 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.937660+0000", 2023-10-09T05:59:29.743 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3640000000000004e-06 2023-10-09T05:59:29.743 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.743 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.744 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.744 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.736002+0000", 2023-10-09T05:59:29.744 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.79834211899999996 2023-10-09T05:59:29.744 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.744 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.744 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.744 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.736009+0000", 2023-10-09T05:59:29.744 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4899999999999997e-06 2023-10-09T05:59:29.744 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.744 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.744 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.744 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.896857+0000", 2023-10-09T05:59:29.745 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1608479190000001 2023-10-09T05:59:29.745 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.745 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.745 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:29.745 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.896983+0000", 2023-10-09T05:59:29.745 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000125962 2023-10-09T05:59:29.745 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.745 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.745 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T05:59:29.745 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.897240+0000", 2023-10-09T05:59:29.745 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000257753 2023-10-09T05:59:29.746 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.746 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.746 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:59:29.746 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900398+0000", 2023-10-09T05:59:29.746 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0031575539999999999 2023-10-09T05:59:29.746 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.746 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.746 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.746 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902797+0000", 2023-10-09T05:59:29.746 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0023991149999999998 2023-10-09T05:59:29.746 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.747 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.747 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.747 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908771+0000", 2023-10-09T05:59:29.747 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0059742579999999997 2023-10-09T05:59:29.747 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.747 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.747 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.747 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.910483+0000", 2023-10-09T05:59:29.747 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0017115489999999999 2023-10-09T05:59:29.748 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.748 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.748 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:59:29.748 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.910530+0000", 2023-10-09T05:59:29.748 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7488999999999999e-05 2023-10-09T05:59:29.748 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.748 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.748 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:59:29.748 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.910648+0000", 2023-10-09T05:59:29.748 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000117941 2023-10-09T05:59:29.748 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.748 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:29.749 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.749 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.749 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.749 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186109 6.15 6:ac6fdfb9:::benchmark_data_smithi064_284193_object89566:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T05:59:29.749 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.940173+0000", 2023-10-09T05:59:29.749 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 98.633241157, 2023-10-09T05:59:29.749 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9706640229999999, 2023-10-09T05:59:29.749 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:29.749 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:29.749 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:59:29.749 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:59:29.750 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:29.750 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186109 2023-10-09T05:59:29.750 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.750 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:29.750 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.750 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:29.750 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.940173+0000", 2023-10-09T05:59:29.750 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.750 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.750 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.750 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:29.751 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.940173+0000", 2023-10-09T05:59:29.751 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.751 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.751 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.751 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:29.751 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.940172+0000", 2023-10-09T05:59:29.751 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T05:59:29.751 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.751 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.751 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:29.751 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.940177+0000", 2023-10-09T05:59:29.751 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4750000000000004e-06 2023-10-09T05:59:29.752 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.752 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.752 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:29.752 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.940188+0000", 2023-10-09T05:59:29.752 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1508999999999999e-05 2023-10-09T05:59:29.752 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.752 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.752 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:29.752 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.940193+0000", 2023-10-09T05:59:29.752 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.147e-06 2023-10-09T05:59:29.752 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.753 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.753 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.753 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.940244+0000", 2023-10-09T05:59:29.753 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0552e-05 2023-10-09T05:59:29.753 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.753 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.753 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.753 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.940254+0000", 2023-10-09T05:59:29.753 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0096e-05 2023-10-09T05:59:29.753 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.753 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.753 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.754 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.736034+0000", 2023-10-09T05:59:29.754 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.79577992900000005 2023-10-09T05:59:29.754 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.754 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.754 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.754 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.736041+0000", 2023-10-09T05:59:29.754 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.849e-06 2023-10-09T05:59:29.754 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.754 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.754 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.754 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.898427+0000", 2023-10-09T05:59:29.755 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.162385929 2023-10-09T05:59:29.755 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.755 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.755 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:29.755 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.898534+0000", 2023-10-09T05:59:29.755 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000107227 2023-10-09T05:59:29.755 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.755 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.755 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T05:59:29.755 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.898738+0000", 2023-10-09T05:59:29.755 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000203928 2023-10-09T05:59:29.756 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.756 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.756 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:59:29.756 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900865+0000", 2023-10-09T05:59:29.756 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0021271129999999999 2023-10-09T05:59:29.756 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.756 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.756 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.756 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.903326+0000", 2023-10-09T05:59:29.756 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002460991 2023-10-09T05:59:29.756 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.757 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.757 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.757 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908797+0000", 2023-10-09T05:59:29.757 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0054708930000000001 2023-10-09T05:59:29.757 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.757 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.757 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.757 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.910685+0000", 2023-10-09T05:59:29.757 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0018880940000000001 2023-10-09T05:59:29.757 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.757 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.757 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:59:29.758 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.910732+0000", 2023-10-09T05:59:29.758 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7129000000000002e-05 2023-10-09T05:59:29.758 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.758 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.758 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:59:29.758 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.910837+0000", 2023-10-09T05:59:29.758 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000104962 2023-10-09T05:59:29.758 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.758 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:29.758 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.758 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.759 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.759 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186112 6.25 6:a55bc7ba:::benchmark_data_smithi064_284193_object89569:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T05:59:29.759 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.946091+0000", 2023-10-09T05:59:29.759 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 98.627322868999997, 2023-10-09T05:59:29.759 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.956330801, 2023-10-09T05:59:29.759 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T05:59:29.759 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:29.759 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T05:59:29.759 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T05:59:29.759 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:29.759 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186112 2023-10-09T05:59:29.760 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.760 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T05:59:29.760 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.760 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T05:59:29.760 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.946091+0000", 2023-10-09T05:59:29.760 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.760 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.760 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.760 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T05:59:29.760 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.946091+0000", 2023-10-09T05:59:29.760 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T05:59:29.760 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.761 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.761 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T05:59:29.761 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.946091+0000", 2023-10-09T05:59:29.761 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T05:59:29.761 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.761 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.761 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T05:59:29.761 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.946095+0000", 2023-10-09T05:59:29.761 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1099999999999996e-06 2023-10-09T05:59:29.761 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.761 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.761 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T05:59:29.762 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.946104+0000", 2023-10-09T05:59:29.762 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.687e-06 2023-10-09T05:59:29.762 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.762 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.762 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T05:59:29.762 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.946109+0000", 2023-10-09T05:59:29.762 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2700000000000004e-06 2023-10-09T05:59:29.762 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.762 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.762 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.762 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.946170+0000", 2023-10-09T05:59:29.763 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1313000000000001e-05 2023-10-09T05:59:29.763 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.763 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.763 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.763 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.946181+0000", 2023-10-09T05:59:29.763 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0849000000000001e-05 2023-10-09T05:59:29.763 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.763 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.763 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.763 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737296+0000", 2023-10-09T05:59:29.763 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.79111525800000004 2023-10-09T05:59:29.764 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.764 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.764 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T05:59:29.764 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737305+0000", 2023-10-09T05:59:29.764 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0000000000000002e-06 2023-10-09T05:59:29.764 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.764 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.764 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T05:59:29.764 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.874487+0000", 2023-10-09T05:59:29.764 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.137181663 2023-10-09T05:59:29.764 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.764 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.765 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T05:59:29.765 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.874590+0000", 2023-10-09T05:59:29.765 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000102778 2023-10-09T05:59:29.765 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.765 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.765 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:59:29.765 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.874748+0000", 2023-10-09T05:59:29.765 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015821699999999999 2023-10-09T05:59:29.765 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.765 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.765 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T05:59:29.765 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887498+0000", 2023-10-09T05:59:29.766 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012750441 2023-10-09T05:59:29.766 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.766 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.766 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.766 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901113+0000", 2023-10-09T05:59:29.766 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013614316 2023-10-09T05:59:29.766 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.766 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.766 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:29.766 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902187+0000", 2023-10-09T05:59:29.766 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00107467 2023-10-09T05:59:29.767 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.767 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.767 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T05:59:29.767 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902236+0000", 2023-10-09T05:59:29.767 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8364e-05 2023-10-09T05:59:29.767 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T05:59:29.767 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T05:59:29.767 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T05:59:29.767 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902422+0000", 2023-10-09T05:59:29.767 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00018654000000000001 2023-10-09T05:59:29.767 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.767 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:29.768 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.768 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T05:59:29.768 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T05:59:29.768 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T05:59:29.768 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:29 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:29.768 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:29.768 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:29 smithi156 ceph-mon[195101]: pgmap v685: 109 pgs: 109 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 59 MiB/s wr, 948 op/s 2023-10-09T05:59:29.768 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:29 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1549727974' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T05:59:29.769 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:29 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T05:59:29.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:29.770 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:29.770 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:29.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:29.871 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:29.930 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:29.999 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 51 16 72043 72027 88.2097 64.3125 0.00329157 0.0113267 2023-10-09T05:59:30.182 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:30.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:30.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:30.288 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:30.333 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:30.411 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:59:30.594 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:59:30.595 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:30.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:30.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:30.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:30 smithi064 ceph-mon[220044]: failed to encode map e298 with expected crc 2023-10-09T05:59:30.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:30 smithi064 ceph-mon[220044]: failed to encode map e298 with expected crc 2023-10-09T05:59:30.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:30 smithi064 ceph-mon[220044]: failed to encode map e298 with expected crc 2023-10-09T05:59:30.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:30 smithi064 ceph-mon[220044]: failed to encode map e298 with expected crc 2023-10-09T05:59:30.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:30 smithi064 ceph-mon[220044]: failed to encode map e298 with expected crc 2023-10-09T05:59:30.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:30 smithi064 ceph-mon[220044]: failed to encode map e298 with expected crc 2023-10-09T05:59:30.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:30 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-10-09T05:59:30.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:30 smithi064 ceph-mon[220044]: osdmap e298: 8 total, 8 up, 7 in 2023-10-09T05:59:30.674 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:30 smithi064 ceph-mon[212750]: failed to encode map e298 with expected crc 2023-10-09T05:59:30.675 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:30 smithi064 ceph-mon[212750]: failed to encode map e298 with expected crc 2023-10-09T05:59:30.675 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:30 smithi064 ceph-mon[212750]: failed to encode map e298 with expected crc 2023-10-09T05:59:30.675 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:30 smithi064 ceph-mon[212750]: failed to encode map e298 with expected crc 2023-10-09T05:59:30.675 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:30 smithi064 ceph-mon[212750]: failed to encode map e298 with expected crc 2023-10-09T05:59:30.675 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:30 smithi064 ceph-mon[212750]: failed to encode map e298 with expected crc 2023-10-09T05:59:30.675 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:30 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-10-09T05:59:30.675 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:30 smithi064 ceph-mon[212750]: osdmap e298: 8 total, 8 up, 7 in 2023-10-09T05:59:30.704 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:30.816 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:30.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:30 smithi156 ceph-mon[195101]: failed to encode map e298 with expected crc 2023-10-09T05:59:30.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:30 smithi156 ceph-mon[195101]: failed to encode map e298 with expected crc 2023-10-09T05:59:30.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:30 smithi156 ceph-mon[195101]: failed to encode map e298 with expected crc 2023-10-09T05:59:30.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:30 smithi156 ceph-mon[195101]: failed to encode map e298 with expected crc 2023-10-09T05:59:30.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:30 smithi156 ceph-mon[195101]: failed to encode map e298 with expected crc 2023-10-09T05:59:30.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:30 smithi156 ceph-mon[195101]: failed to encode map e298 with expected crc 2023-10-09T05:59:30.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:30 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-10-09T05:59:30.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:30 smithi156 ceph-mon[195101]: osdmap e298: 8 total, 8 up, 7 in 2023-10-09T05:59:30.972 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:30.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:30.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:30.999 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 52 16 72807 72791 87.432 47.75 0.0189831 0.0114298 2023-10-09T05:59:31.086 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:31.128 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:31.218 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:31.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:31.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:31.324 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:31.361 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:31.447 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:31.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:31.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:31.556 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:31.619 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:31.652 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:31 smithi064 ceph-mon[212750]: pgmap v687: 109 pgs: 109 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T05:59:31.652 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:31 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-10-09T05:59:31.653 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:31 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-10-09T05:59:31.653 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:31 smithi064 ceph-mon[220044]: pgmap v687: 109 pgs: 109 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T05:59:31.653 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:31 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-10-09T05:59:31.653 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:31 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-10-09T05:59:31.658 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:31.664 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:31.800 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:31.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:31.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:31 smithi156 ceph-mon[195101]: pgmap v687: 109 pgs: 109 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T05:59:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:31 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-10-09T05:59:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:31 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-10-09T05:59:31.907 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:31.999 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 53 16 73748 73732 86.8922 58.8125 0.0101774 0.0115035 2023-10-09T05:59:32.007 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:32.273 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:32.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:32.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:32.379 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:32.452 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:32.524 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T05:59:32.560 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:32 smithi064 ceph-mon[212750]: failed to encode map e299 with expected crc 2023-10-09T05:59:32.560 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:32 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-10-09T05:59:32.560 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:32 smithi064 ceph-mon[212750]: osdmap e299: 8 total, 8 up, 7 in 2023-10-09T05:59:32.560 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:32 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2207980840' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:59:32.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:32 smithi064 ceph-mon[220044]: failed to encode map e299 with expected crc 2023-10-09T05:59:32.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:32 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-10-09T05:59:32.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:32 smithi064 ceph-mon[220044]: osdmap e299: 8 total, 8 up, 7 in 2023-10-09T05:59:32.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:32 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2207980840' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:59:32.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:32 smithi156 ceph-mon[195101]: failed to encode map e299 with expected crc 2023-10-09T05:59:32.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:32 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-10-09T05:59:32.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:32 smithi156 ceph-mon[195101]: osdmap e299: 8 total, 8 up, 7 in 2023-10-09T05:59:32.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:32 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2207980840' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:59:32.999 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 54 16 74809 74793 86.5111 66.3125 0.011064 0.0115539 2023-10-09T05:59:33.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:33 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:33.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:33 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2207980840' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:59:33.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:33 smithi156 ceph-mon[195101]: osdmap e300: 8 total, 8 up, 7 in 2023-10-09T05:59:33.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:33 smithi156 ceph-mon[195101]: failed to encode map e300 with expected crc 2023-10-09T05:59:33.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:33 smithi156 ceph-mon[195101]: failed to encode map e300 with expected crc 2023-10-09T05:59:33.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:33 smithi156 ceph-mon[195101]: failed to encode map e300 with expected crc 2023-10-09T05:59:33.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:33 smithi156 ceph-mon[195101]: failed to encode map e300 with expected crc 2023-10-09T05:59:33.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:33 smithi156 ceph-mon[195101]: failed to encode map e300 with expected crc 2023-10-09T05:59:33.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:33 smithi156 ceph-mon[195101]: failed to encode map e300 with expected crc 2023-10-09T05:59:33.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:33 smithi156 ceph-mon[195101]: pgmap v690: 119 pgs: 10 unknown, 109 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 48 MiB/s wr, 763 op/s 2023-10-09T05:59:33.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:33 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:33.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:33 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2207980840' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:59:33.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:33 smithi064 ceph-mon[220044]: osdmap e300: 8 total, 8 up, 7 in 2023-10-09T05:59:33.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:33 smithi064 ceph-mon[220044]: failed to encode map e300 with expected crc 2023-10-09T05:59:33.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:33 smithi064 ceph-mon[220044]: failed to encode map e300 with expected crc 2023-10-09T05:59:33.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:33 smithi064 ceph-mon[220044]: failed to encode map e300 with expected crc 2023-10-09T05:59:33.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:33 smithi064 ceph-mon[220044]: failed to encode map e300 with expected crc 2023-10-09T05:59:33.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:33 smithi064 ceph-mon[220044]: failed to encode map e300 with expected crc 2023-10-09T05:59:33.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:33 smithi064 ceph-mon[220044]: failed to encode map e300 with expected crc 2023-10-09T05:59:33.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:33 smithi064 ceph-mon[220044]: pgmap v690: 119 pgs: 10 unknown, 109 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 48 MiB/s wr, 763 op/s 2023-10-09T05:59:33.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:33 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:33.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:33 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2207980840' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:59:33.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:33 smithi064 ceph-mon[212750]: osdmap e300: 8 total, 8 up, 7 in 2023-10-09T05:59:33.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:33 smithi064 ceph-mon[212750]: failed to encode map e300 with expected crc 2023-10-09T05:59:33.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:33 smithi064 ceph-mon[212750]: failed to encode map e300 with expected crc 2023-10-09T05:59:33.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:33 smithi064 ceph-mon[212750]: failed to encode map e300 with expected crc 2023-10-09T05:59:33.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:33 smithi064 ceph-mon[212750]: failed to encode map e300 with expected crc 2023-10-09T05:59:33.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:33 smithi064 ceph-mon[212750]: failed to encode map e300 with expected crc 2023-10-09T05:59:33.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:33 smithi064 ceph-mon[212750]: failed to encode map e300 with expected crc 2023-10-09T05:59:33.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:33 smithi064 ceph-mon[212750]: pgmap v690: 119 pgs: 10 unknown, 109 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 48 MiB/s wr, 763 op/s 2023-10-09T05:59:33.999 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 55 16 75608 75592 85.8464 49.9375 0.0442454 0.0116431 2023-10-09T05:59:34.155 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:34.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:34.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:34.263 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:34.300 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:34.366 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:34.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:34.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:34.473 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:59:34.503 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:34.882 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:59:34.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:34.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:59:34.991 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:34.999 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 56 16 76586 76570 85.4051 61.125 0.014394 0.0117038 2023-10-09T05:59:35.070 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:35.217 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:35.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:35.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:35.324 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:35.386 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:35.470 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:59:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:59:35] "GET /metrics HTTP/1.1" 200 36382 "" "Prometheus/2.43.0" 2023-10-09T05:59:35.530 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:59:35.544 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:35.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:35.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:35.563 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:59:35.611 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:59:35.655 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:35.656 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:59:35.691 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:35.700 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:59:35.735 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:59:35.759 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:35.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:35.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:35.786 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:59:35.839 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:59:35.841 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:59:35.841 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:59:35.841 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:59:35.841 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:59:35.841 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:59:35.841 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:59:35.842 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:59:35.842 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:59:35.863 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:35.883 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:35 smithi064 ceph-mon[220044]: pgmap v691: 119 pgs: 7 unknown, 112 active+clean; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 46 MiB/s wr, 735 op/s 2023-10-09T05:59:35.883 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:35 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/501800078' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:35.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:35 smithi064 ceph-mon[212750]: pgmap v691: 119 pgs: 7 unknown, 112 active+clean; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 46 MiB/s wr, 735 op/s 2023-10-09T05:59:35.884 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:35 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/501800078' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:35.914 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:59:35.914 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:59:35.942 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:35.957 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T05:59:35.958 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T05:59:35.999 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 57 16 77525 77509 84.9365 58.6875 0.00411078 0.0117683 2023-10-09T05:59:36.118 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:36.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:35 smithi156 ceph-mon[195101]: pgmap v691: 119 pgs: 7 unknown, 112 active+clean; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 46 MiB/s wr, 735 op/s 2023-10-09T05:59:36.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:35 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/501800078' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:36.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:36.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:36.233 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:36.320 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:36.342 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:59:36.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:36 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-10-09T05:59:36.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:36 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-10-09T05:59:36.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:36 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-10-09T05:59:36.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:36 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-10-09T05:59:37.002 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 58 16 78354 78338 84.3618 51.8125 0.0176423 0.0118492 2023-10-09T05:59:37.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:36 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-10-09T05:59:37.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:36 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-10-09T05:59:37.486 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:37.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:37.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:37.593 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:37.637 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:37.719 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:37.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:37.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:37.748 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:37 smithi064 ceph-mon[212750]: pgmap v692: 119 pgs: 119 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T05:59:37.748 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:37 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-10-09T05:59:37.748 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:37 smithi064 ceph-mon[212750]: osdmap e301: 8 total, 8 up, 7 in 2023-10-09T05:59:37.748 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:37 smithi064 ceph-mon[212750]: failed to encode map e301 with expected crc 2023-10-09T05:59:37.748 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:37 smithi064 ceph-mon[212750]: failed to encode map e301 with expected crc 2023-10-09T05:59:37.748 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:37 smithi064 ceph-mon[212750]: osdmap e302: 8 total, 8 up, 7 in 2023-10-09T05:59:37.748 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:37 smithi064 ceph-mon[220044]: pgmap v692: 119 pgs: 119 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T05:59:37.749 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:37 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-10-09T05:59:37.749 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:37 smithi064 ceph-mon[220044]: osdmap e301: 8 total, 8 up, 7 in 2023-10-09T05:59:37.749 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:37 smithi064 ceph-mon[220044]: failed to encode map e301 with expected crc 2023-10-09T05:59:37.749 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:37 smithi064 ceph-mon[220044]: failed to encode map e301 with expected crc 2023-10-09T05:59:37.749 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:37 smithi064 ceph-mon[220044]: osdmap e302: 8 total, 8 up, 7 in 2023-10-09T05:59:37.750 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:59:37.824 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:37.921 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:37.954 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:38.002 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 59 16 79064 79048 83.6844 44.375 0.0213038 0.0119444 2023-10-09T05:59:38.021 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:38.112 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:38.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:37 smithi156 ceph-mon[195101]: pgmap v692: 119 pgs: 119 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T05:59:38.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:37 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-10-09T05:59:38.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:37 smithi156 ceph-mon[195101]: osdmap e301: 8 total, 8 up, 7 in 2023-10-09T05:59:38.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:37 smithi156 ceph-mon[195101]: failed to encode map e301 with expected crc 2023-10-09T05:59:38.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:37 smithi156 ceph-mon[195101]: failed to encode map e301 with expected crc 2023-10-09T05:59:38.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:37 smithi156 ceph-mon[195101]: osdmap e302: 8 total, 8 up, 7 in 2023-10-09T05:59:38.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:38.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:38.226 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:38.319 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:38.575 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:38.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:38.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:38.682 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:38.719 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:38.795 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:38.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:38.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:38.903 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:38.951 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:38.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:38 smithi064 ceph-mon[212750]: failed to encode map e302 with expected crc 2023-10-09T05:59:38.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:38 smithi064 ceph-mon[212750]: failed to encode map e302 with expected crc 2023-10-09T05:59:38.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:38 smithi064 ceph-mon[212750]: failed to encode map e302 with expected crc 2023-10-09T05:59:38.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:38 smithi064 ceph-mon[212750]: failed to encode map e302 with expected crc 2023-10-09T05:59:38.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:38 smithi064 ceph-mon[212750]: failed to encode map e302 with expected crc 2023-10-09T05:59:38.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:38 smithi064 ceph-mon[212750]: osdmap e303: 8 total, 8 up, 7 in 2023-10-09T05:59:38.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:38 smithi064 ceph-mon[220044]: failed to encode map e302 with expected crc 2023-10-09T05:59:38.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:38 smithi064 ceph-mon[220044]: failed to encode map e302 with expected crc 2023-10-09T05:59:38.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:38 smithi064 ceph-mon[220044]: failed to encode map e302 with expected crc 2023-10-09T05:59:38.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:38 smithi064 ceph-mon[220044]: failed to encode map e302 with expected crc 2023-10-09T05:59:38.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:38 smithi064 ceph-mon[220044]: failed to encode map e302 with expected crc 2023-10-09T05:59:38.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:38 smithi064 ceph-mon[220044]: osdmap e303: 8 total, 8 up, 7 in 2023-10-09T05:59:39.003 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-10-09T05:59:39.003768+0000 min lat: 0.00127391 max lat: 1.01073 avg lat: 0.0120012 2023-10-09T05:59:39.003 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:59:39.003 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 60 16 80028 80012 83.2934 60.25 0.00974068 0.0120012 2023-10-09T05:59:39.041 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:39.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:39.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:39.044 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:38 smithi156 ceph-mon[195101]: failed to encode map e302 with expected crc 2023-10-09T05:59:39.044 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:38 smithi156 ceph-mon[195101]: failed to encode map e302 with expected crc 2023-10-09T05:59:39.044 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:38 smithi156 ceph-mon[195101]: failed to encode map e302 with expected crc 2023-10-09T05:59:39.045 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:38 smithi156 ceph-mon[195101]: failed to encode map e302 with expected crc 2023-10-09T05:59:39.045 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:38 smithi156 ceph-mon[195101]: failed to encode map e302 with expected crc 2023-10-09T05:59:39.045 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:38 smithi156 ceph-mon[195101]: osdmap e303: 8 total, 8 up, 7 in 2023-10-09T05:59:39.144 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:39.233 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:39.559 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:59:39.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:39 smithi064 ceph-mon[212750]: failed to encode map e303 with expected crc 2023-10-09T05:59:39.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:39 smithi064 ceph-mon[212750]: failed to encode map e303 with expected crc 2023-10-09T05:59:39.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:39 smithi064 ceph-mon[212750]: failed to encode map e303 with expected crc 2023-10-09T05:59:39.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:39 smithi064 ceph-mon[212750]: pgmap v696: 119 pgs: 4 remapped+peering, 115 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 7.0 KiB/s rd, 59 MiB/s wr, 943 op/s; 13 KiB/s, 0 objects/s recovering 2023-10-09T05:59:39.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:39 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T05:59:39.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:39 smithi064 ceph-mon[212750]: osdmap e304: 8 total, 8 up, 7 in 2023-10-09T05:59:39.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:39 smithi064 ceph-mon[220044]: failed to encode map e303 with expected crc 2023-10-09T05:59:39.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:39 smithi064 ceph-mon[220044]: failed to encode map e303 with expected crc 2023-10-09T05:59:39.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:39 smithi064 ceph-mon[220044]: failed to encode map e303 with expected crc 2023-10-09T05:59:39.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:39 smithi064 ceph-mon[220044]: pgmap v696: 119 pgs: 4 remapped+peering, 115 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 7.0 KiB/s rd, 59 MiB/s wr, 943 op/s; 13 KiB/s, 0 objects/s recovering 2023-10-09T05:59:39.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:39 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T05:59:39.940 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:39 smithi064 ceph-mon[220044]: osdmap e304: 8 total, 8 up, 7 in 2023-10-09T05:59:40.001 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:40.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:40.007 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 61 16 80866 80850 82.7839 52.375 0.0218838 0.0120739 2023-10-09T05:59:40.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:40.109 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:40.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:39 smithi156 ceph-mon[195101]: failed to encode map e303 with expected crc 2023-10-09T05:59:40.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:39 smithi156 ceph-mon[195101]: failed to encode map e303 with expected crc 2023-10-09T05:59:40.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:39 smithi156 ceph-mon[195101]: failed to encode map e303 with expected crc 2023-10-09T05:59:40.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:39 smithi156 ceph-mon[195101]: pgmap v696: 119 pgs: 4 remapped+peering, 115 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 7.0 KiB/s rd, 59 MiB/s wr, 943 op/s; 13 KiB/s, 0 objects/s recovering 2023-10-09T05:59:40.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:39 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T05:59:40.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:39 smithi156 ceph-mon[195101]: osdmap e304: 8 total, 8 up, 7 in 2023-10-09T05:59:40.171 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:40.461 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:40.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:40.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:40.551 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:40.574 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:40.637 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:40.797 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:40.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:40.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:40.905 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:40.953 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:41.005 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 62 16 81851 81835 82.4417 61.5625 0.0170904 0.0121243 2023-10-09T05:59:41.031 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:41.039 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:40 smithi156 ceph-mon[195101]: failed to encode map e304 with expected crc 2023-10-09T05:59:41.039 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:40 smithi156 ceph-mon[195101]: failed to encode map e304 with expected crc 2023-10-09T05:59:41.039 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:40 smithi156 ceph-mon[195101]: failed to encode map e304 with expected crc 2023-10-09T05:59:41.039 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:40 smithi156 ceph-mon[195101]: osdmap e305: 8 total, 8 up, 7 in 2023-10-09T05:59:41.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:41.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:41.141 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:41.176 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:40 smithi064 ceph-mon[212750]: failed to encode map e304 with expected crc 2023-10-09T05:59:41.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:40 smithi064 ceph-mon[212750]: failed to encode map e304 with expected crc 2023-10-09T05:59:41.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:40 smithi064 ceph-mon[212750]: failed to encode map e304 with expected crc 2023-10-09T05:59:41.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:40 smithi064 ceph-mon[212750]: osdmap e305: 8 total, 8 up, 7 in 2023-10-09T05:59:41.178 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:40 smithi064 ceph-mon[220044]: failed to encode map e304 with expected crc 2023-10-09T05:59:41.178 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:40 smithi064 ceph-mon[220044]: failed to encode map e304 with expected crc 2023-10-09T05:59:41.178 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:40 smithi064 ceph-mon[220044]: failed to encode map e304 with expected crc 2023-10-09T05:59:41.178 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:40 smithi064 ceph-mon[220044]: osdmap e305: 8 total, 8 up, 7 in 2023-10-09T05:59:41.186 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:41.249 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:41.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:41.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:41.359 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:41.451 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:42.005 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 63 16 82756 82740 82.031 56.5625 0.00754832 0.0121859 2023-10-09T05:59:42.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:41 smithi156 ceph-mon[195101]: failed to encode map e305 with expected crc 2023-10-09T05:59:42.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:41 smithi156 ceph-mon[195101]: failed to encode map e305 with expected crc 2023-10-09T05:59:42.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:41 smithi156 ceph-mon[195101]: failed to encode map e305 with expected crc 2023-10-09T05:59:42.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:41 smithi156 ceph-mon[195101]: failed to encode map e305 with expected crc 2023-10-09T05:59:42.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:41 smithi156 ceph-mon[195101]: pgmap v699: 119 pgs: 7 peering, 112 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 41 KiB/s rd, 70 MiB/s wr, 1.14k op/s; 223 KiB/s, 1 objects/s recovering 2023-10-09T05:59:42.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:41 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T05:59:42.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:41 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2249642148' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:59:42.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:41 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:59:42.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:41 smithi064 ceph-mon[212750]: failed to encode map e305 with expected crc 2023-10-09T05:59:42.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:41 smithi064 ceph-mon[212750]: failed to encode map e305 with expected crc 2023-10-09T05:59:42.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:41 smithi064 ceph-mon[212750]: failed to encode map e305 with expected crc 2023-10-09T05:59:42.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:41 smithi064 ceph-mon[212750]: failed to encode map e305 with expected crc 2023-10-09T05:59:42.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:41 smithi064 ceph-mon[212750]: pgmap v699: 119 pgs: 7 peering, 112 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 41 KiB/s rd, 70 MiB/s wr, 1.14k op/s; 223 KiB/s, 1 objects/s recovering 2023-10-09T05:59:42.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:41 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T05:59:42.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:41 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2249642148' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:59:42.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:41 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:59:42.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:41 smithi064 ceph-mon[220044]: failed to encode map e305 with expected crc 2023-10-09T05:59:42.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:41 smithi064 ceph-mon[220044]: failed to encode map e305 with expected crc 2023-10-09T05:59:42.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:41 smithi064 ceph-mon[220044]: failed to encode map e305 with expected crc 2023-10-09T05:59:42.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:41 smithi064 ceph-mon[220044]: failed to encode map e305 with expected crc 2023-10-09T05:59:42.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:41 smithi064 ceph-mon[220044]: pgmap v699: 119 pgs: 7 peering, 112 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 41 KiB/s rd, 70 MiB/s wr, 1.14k op/s; 223 KiB/s, 1 objects/s recovering 2023-10-09T05:59:42.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:41 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T05:59:42.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:41 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2249642148' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:59:42.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:41 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:59:42.326 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T05:59:42.327 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T05:59:42.348 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:42.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:42 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/430569207' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:42.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:42 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:42.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:42 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:42.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:42 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:59:42.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:42 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:59:42.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:42 smithi064 ceph-mon[212750]: osdmap e306: 8 total, 8 up, 7 in 2023-10-09T05:59:42.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:42 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/430569207' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:42.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:42 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:42.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:42 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:42.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:42 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:59:42.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:42 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:59:42.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:42 smithi064 ceph-mon[220044]: osdmap e306: 8 total, 8 up, 7 in 2023-10-09T05:59:43.005 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 64 16 83425 83409 81.4029 41.8125 0.00246311 0.0122688 2023-10-09T05:59:43.075 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:43.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:43.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:43.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:42 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/430569207' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:43.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:42 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:59:43.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:42 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:43.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:42 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:59:43.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:42 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:59:43.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:42 smithi156 ceph-mon[195101]: osdmap e306: 8 total, 8 up, 7 in 2023-10-09T05:59:43.180 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:43.261 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:43.758 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:43.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:43.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:43.862 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:43.907 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:43.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:43 smithi064 ceph-mon[220044]: failed to encode map e306 with expected crc 2023-10-09T05:59:43.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:43 smithi064 ceph-mon[220044]: failed to encode map e306 with expected crc 2023-10-09T05:59:43.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:43 smithi064 ceph-mon[220044]: failed to encode map e306 with expected crc 2023-10-09T05:59:43.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:43 smithi064 ceph-mon[220044]: failed to encode map e306 with expected crc 2023-10-09T05:59:43.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:43 smithi064 ceph-mon[220044]: pgmap v701: 119 pgs: 7 peering, 112 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 27 KiB/s rd, 62 MiB/s wr, 1.01k op/s; 182 KiB/s, 0 objects/s recovering 2023-10-09T05:59:43.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:43 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:43.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:43 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:43.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:43 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:43.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:43 smithi064 ceph-mon[212750]: failed to encode map e306 with expected crc 2023-10-09T05:59:43.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:43 smithi064 ceph-mon[212750]: failed to encode map e306 with expected crc 2023-10-09T05:59:43.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:43 smithi064 ceph-mon[212750]: failed to encode map e306 with expected crc 2023-10-09T05:59:43.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:43 smithi064 ceph-mon[212750]: failed to encode map e306 with expected crc 2023-10-09T05:59:43.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:43 smithi064 ceph-mon[212750]: pgmap v701: 119 pgs: 7 peering, 112 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 27 KiB/s rd, 62 MiB/s wr, 1.01k op/s; 182 KiB/s, 0 objects/s recovering 2023-10-09T05:59:43.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:43 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:43.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:43 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:43.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:43 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:44.002 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:44.005 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 65 16 84344 84328 81.0343 57.4375 0.0118423 0.0123362 2023-10-09T05:59:44.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:44.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:44.111 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:44.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:43 smithi156 ceph-mon[195101]: failed to encode map e306 with expected crc 2023-10-09T05:59:44.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:43 smithi156 ceph-mon[195101]: failed to encode map e306 with expected crc 2023-10-09T05:59:44.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:43 smithi156 ceph-mon[195101]: failed to encode map e306 with expected crc 2023-10-09T05:59:44.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:43 smithi156 ceph-mon[195101]: failed to encode map e306 with expected crc 2023-10-09T05:59:44.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:43 smithi156 ceph-mon[195101]: pgmap v701: 119 pgs: 7 peering, 112 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 27 KiB/s rd, 62 MiB/s wr, 1.01k op/s; 182 KiB/s, 0 objects/s recovering 2023-10-09T05:59:44.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:43 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:44.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:43 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:44.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:43 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:44.157 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:44.620 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T05:59:44.815 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:44.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:44.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:44.925 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:44.972 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:45.006 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 66 16 85246 85230 80.6608 56.375 0.00800753 0.0123877 2023-10-09T05:59:45.240 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:45.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:45.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:45.347 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:45.441 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:45.576 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:45.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:45.581 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:59:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:59:45] "GET /metrics HTTP/1.1" 200 36407 "" "Prometheus/2.43.0" 2023-10-09T05:59:45.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:45.651 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T05:59:45.651 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:59:45.682 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:45.706 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:45.722 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:45.789 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:45.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:45.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:45.897 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:45.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:45 smithi064 ceph-mon[212750]: pgmap v702: 119 pgs: 7 peering, 112 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 20 KiB/s rd, 45 MiB/s wr, 737 op/s; 133 KiB/s, 0 objects/s recovering 2023-10-09T05:59:45.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:45 smithi064 ceph-mon[220044]: pgmap v702: 119 pgs: 7 peering, 112 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 20 KiB/s rd, 45 MiB/s wr, 737 op/s; 133 KiB/s, 0 objects/s recovering 2023-10-09T05:59:45.976 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:46.006 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 67 16 86059 86043 80.2154 50.8125 0.0631012 0.0124548 2023-10-09T05:59:46.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:45 smithi156 ceph-mon[195101]: pgmap v702: 119 pgs: 7 peering, 112 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 20 KiB/s rd, 45 MiB/s wr, 737 op/s; 133 KiB/s, 0 objects/s recovering 2023-10-09T05:59:46.305 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:59:46.948 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:59:47.006 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 68 16 87061 87045 79.9568 62.625 0.00762157 0.0125023 2023-10-09T05:59:47.107 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:47.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:47.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:47.194 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:59:47.229 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:47.252 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:59:47.271 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:47.288 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:59:47.325 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:47.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:47.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:47.333 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:59:47.379 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:59:47.415 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:59:47.431 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:47.457 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:59:47.485 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:47.490 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:59:47.505 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:59:47.505 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:59:47.505 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:59:47.505 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:59:47.505 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:59:47.505 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:59:47.505 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:59:47.505 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:59:47.573 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:47.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:47.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:47.599 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:47.682 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:47.723 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:47.817 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:47.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:47.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:47.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:47 smithi156 ceph-mon[195101]: pgmap v703: 119 pgs: 119 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 16 KiB/s rd, 63 MiB/s wr, 1.01k op/s; 109 KiB/s, 0 objects/s recovering 2023-10-09T05:59:47.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:47 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3179840195' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:47.924 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:47.954 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:48.006 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 69 16 87759 87743 79.4304 43.625 0.00812137 0.0125757 2023-10-09T05:59:48.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:47 smithi064 ceph-mon[212750]: pgmap v703: 119 pgs: 119 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 16 KiB/s rd, 63 MiB/s wr, 1.01k op/s; 109 KiB/s, 0 objects/s recovering 2023-10-09T05:59:48.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:47 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3179840195' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:48.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:47 smithi064 ceph-mon[220044]: pgmap v703: 119 pgs: 119 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 16 KiB/s rd, 63 MiB/s wr, 1.01k op/s; 109 KiB/s, 0 objects/s recovering 2023-10-09T05:59:48.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:47 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3179840195' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:49.008 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 70 16 88669 88653 79.1063 56.875 0.00891752 0.0126367 2023-10-09T05:59:49.094 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:49.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:49.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:49.201 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:49.234 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:49.314 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:49.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:49.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:49.422 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:49.457 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:49.552 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:49.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:49.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:49.663 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:49.749 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:49.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:49 smithi156 ceph-mon[195101]: pgmap v704: 119 pgs: 119 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 1.7 KiB/s rd, 47 MiB/s wr, 746 op/s; 72 KiB/s, 0 objects/s recovering 2023-10-09T05:59:49.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:49 smithi064 ceph-mon[212750]: pgmap v704: 119 pgs: 119 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 1.7 KiB/s rd, 47 MiB/s wr, 746 op/s; 72 KiB/s, 0 objects/s recovering 2023-10-09T05:59:49.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:49 smithi064 ceph-mon[220044]: pgmap v704: 119 pgs: 119 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 1.7 KiB/s rd, 47 MiB/s wr, 746 op/s; 72 KiB/s, 0 objects/s recovering 2023-10-09T05:59:50.008 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 71 16 89642 89626 78.8486 60.8125 0.00713696 0.0126774 2023-10-09T05:59:50.334 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:50.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:50.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:50.440 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:50.475 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:50.544 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:50.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:50.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:50.640 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:50.664 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:50.789 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:51.008 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 72 16 90567 90551 78.5566 57.8125 0.0288866 0.0127236 2023-10-09T05:59:51.101 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:51.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:51.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:51.220 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:51.246 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:59:51.306 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:51.489 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:51.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:51.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:51.613 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:51.653 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:51.709 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:51.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:51.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:51.819 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:51.846 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T05:59:51.846 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T05:59:51.870 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:52.008 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 73 16 91246 91230 78.062 42.4375 0.011235 0.0127776 2023-10-09T05:59:52.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:51 smithi156 ceph-mon[195101]: pgmap v705: 119 pgs: 119 active+clean; 5.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 55 MiB/s wr, 874 op/s 2023-10-09T05:59:52.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:51 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3906900824' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:59:52.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:51 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:59:52.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:51 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3642398531' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:59:52.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:51 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:59:52.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:51 smithi064 ceph-mon[212750]: pgmap v705: 119 pgs: 119 active+clean; 5.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 55 MiB/s wr, 874 op/s 2023-10-09T05:59:52.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:51 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3906900824' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:59:52.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:51 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:59:52.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:51 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3642398531' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:59:52.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:51 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:59:52.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:51 smithi064 ceph-mon[220044]: pgmap v705: 119 pgs: 119 active+clean; 5.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 55 MiB/s wr, 874 op/s 2023-10-09T05:59:52.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:51 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3906900824' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:59:52.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:51 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:59:52.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:51 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3642398531' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:59:52.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:51 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:59:52.670 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:52.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:52.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:52.785 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:52.811 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:52.862 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:52.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:52.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:52.968 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:52.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:52 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:52.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:52 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:59:52.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:52 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:59:52.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:52 smithi064 ceph-mon[220044]: osdmap e307: 8 total, 8 up, 7 in 2023-10-09T05:59:52.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:52 smithi064 ceph-mon[220044]: failed to encode map e307 with expected crc 2023-10-09T05:59:52.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:52 smithi064 ceph-mon[220044]: 1.a scrub starts 2023-10-09T05:59:52.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:52 smithi064 ceph-mon[220044]: 1.a scrub ok 2023-10-09T05:59:52.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:52 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:52.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:52 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:59:52.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:52 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:59:52.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:52 smithi064 ceph-mon[212750]: osdmap e307: 8 total, 8 up, 7 in 2023-10-09T05:59:52.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:52 smithi064 ceph-mon[212750]: failed to encode map e307 with expected crc 2023-10-09T05:59:52.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:52 smithi064 ceph-mon[212750]: 1.a scrub starts 2023-10-09T05:59:52.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:52 smithi064 ceph-mon[212750]: 1.a scrub ok 2023-10-09T05:59:53.008 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 74 16 92019 92003 77.66 48.3125 0.0104517 0.0128722 2023-10-09T05:59:53.043 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:53.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:52 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:59:53.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:52 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:59:53.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:52 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:59:53.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:52 smithi156 ceph-mon[195101]: osdmap e307: 8 total, 8 up, 7 in 2023-10-09T05:59:53.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:52 smithi156 ceph-mon[195101]: failed to encode map e307 with expected crc 2023-10-09T05:59:53.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:52 smithi156 ceph-mon[195101]: 1.a scrub starts 2023-10-09T05:59:53.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:52 smithi156 ceph-mon[195101]: 1.a scrub ok 2023-10-09T05:59:53.137 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:53.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:53.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:53.253 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:53.279 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:53.347 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:53.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:53.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:53.456 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:53.521 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:53.606 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:53.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:53.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:53.678 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T05:59:53.715 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:53.753 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:53.815 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:53.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:53.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:53.929 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:53.966 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:54.008 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 75 16 92863 92847 77.328 52.75 0.0280184 0.0129252 2023-10-09T05:59:54.017 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:54.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:54.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:54.125 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:54.158 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:54.226 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:54.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:54.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:54.233 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:54.288 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:54 smithi064 ceph-mon[212750]: pgmap v707: 119 pgs: 119 active+clean; 5.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 54 MiB/s wr, 870 op/s 2023-10-09T05:59:54.288 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:54 smithi064 ceph-mon[212750]: 1.d scrub starts 2023-10-09T05:59:54.288 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:54 smithi064 ceph-mon[212750]: 1.d scrub ok 2023-10-09T05:59:54.288 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:54 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:54.332 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:54.361 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T05:59:54.361 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T05:59:54.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T05:59:54.362 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T05:59:54.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.362 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82093 6.0 6.a3a5430 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:59:54.362 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.146555+0000", 2023-10-09T05:59:54.362 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 478.20600261700002, 2023-10-09T05:59:54.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93485150500000003, 2023-10-09T05:59:54.362 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:59:54.362 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:59:54.362 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:59:54.363 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:59:54.363 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:54.363 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82093 2023-10-09T05:59:54.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.363 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:59:54.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:59:54.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.146555+0000", 2023-10-09T05:59:54.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:59:54.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.146555+0000", 2023-10-09T05:59:54.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.364 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:59:54.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.146553+0000", 2023-10-09T05:59:54.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:59:54.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.364 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:59:54.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.146607+0000", 2023-10-09T05:59:54.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4317000000000002e-05 2023-10-09T05:59:54.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:59:54.365 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.146636+0000", 2023-10-09T05:59:54.365 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8294999999999999e-05 2023-10-09T05:59:54.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:59:54.365 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.146647+0000", 2023-10-09T05:59:54.365 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1513e-05 2023-10-09T05:59:54.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.150385+0000", 2023-10-09T05:59:54.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037372640000000001 2023-10-09T05:59:54.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:59:54.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.150398+0000", 2023-10-09T05:59:54.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.34e-05 2023-10-09T05:59:54.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081393+0000", 2023-10-09T05:59:54.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93099492699999997 2023-10-09T05:59:54.367 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:59:54.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081406+0000", 2023-10-09T05:59:54.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.359e-05 2023-10-09T05:59:54.367 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.367 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:59:54.367 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.367 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.367 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82107 6.0 6.432cde0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:59:54.367 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.173816+0000", 2023-10-09T05:59:54.368 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 478.17874124500003, 2023-10-09T05:59:54.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90762603100000006, 2023-10-09T05:59:54.368 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:59:54.368 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:59:54.368 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:59:54.368 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:59:54.368 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:54.368 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82107 2023-10-09T05:59:54.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.368 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:59:54.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.368 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:59:54.368 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173816+0000", 2023-10-09T05:59:54.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:59:54.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173816+0000", 2023-10-09T05:59:54.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:59:54.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173814+0000", 2023-10-09T05:59:54.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:59:54.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:59:54.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173867+0000", 2023-10-09T05:59:54.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2862999999999999e-05 2023-10-09T05:59:54.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:59:54.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173887+0000", 2023-10-09T05:59:54.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0347e-05 2023-10-09T05:59:54.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:59:54.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173907+0000", 2023-10-09T05:59:54.371 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9782000000000001e-05 2023-10-09T05:59:54.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173983+0000", 2023-10-09T05:59:54.371 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6056000000000005e-05 2023-10-09T05:59:54.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:59:54.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.173996+0000", 2023-10-09T05:59:54.371 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3237999999999999e-05 2023-10-09T05:59:54.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081433+0000", 2023-10-09T05:59:54.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90743656100000003 2023-10-09T05:59:54.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:59:54.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081442+0000", 2023-10-09T05:59:54.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5070000000000006e-06 2023-10-09T05:59:54.372 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.372 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:59:54.372 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.373 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82109 6.0 6.cee0ea60 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:59:54.373 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.183890+0000", 2023-10-09T05:59:54.373 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 478.16866711199998, 2023-10-09T05:59:54.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.89758420800000005, 2023-10-09T05:59:54.373 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:59:54.373 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:59:54.373 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:59:54.373 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:59:54.373 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:54.373 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82109 2023-10-09T05:59:54.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.374 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:59:54.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:59:54.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.183890+0000", 2023-10-09T05:59:54.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:59:54.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.183890+0000", 2023-10-09T05:59:54.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:59:54.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.183889+0000", 2023-10-09T05:59:54.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:59:54.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:59:54.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.184035+0000", 2023-10-09T05:59:54.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000145375 2023-10-09T05:59:54.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:59:54.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.184048+0000", 2023-10-09T05:59:54.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3730999999999999e-05 2023-10-09T05:59:54.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.376 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:59:54.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.184060+0000", 2023-10-09T05:59:54.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1338000000000001e-05 2023-10-09T05:59:54.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.376 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.184099+0000", 2023-10-09T05:59:54.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9183000000000001e-05 2023-10-09T05:59:54.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:59:54.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.184113+0000", 2023-10-09T05:59:54.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3750999999999999e-05 2023-10-09T05:59:54.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081465+0000", 2023-10-09T05:59:54.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.89735238100000003 2023-10-09T05:59:54.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:59:54.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081475+0000", 2023-10-09T05:59:54.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7950000000000002e-06 2023-10-09T05:59:54.378 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.378 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:59:54.378 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.378 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82116 6.9 6.318f1fa9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:59:54.378 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.194331+0000", 2023-10-09T05:59:54.378 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 478.15822640900001, 2023-10-09T05:59:54.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.888951253, 2023-10-09T05:59:54.378 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:59:54.379 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:59:54.379 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:59:54.379 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:59:54.379 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:54.379 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82116 2023-10-09T05:59:54.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.379 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:59:54.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:59:54.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194331+0000", 2023-10-09T05:59:54.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:59:54.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194331+0000", 2023-10-09T05:59:54.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:59:54.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194329+0000", 2023-10-09T05:59:54.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:59:54.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:59:54.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194387+0000", 2023-10-09T05:59:54.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7734e-05 2023-10-09T05:59:54.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:59:54.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194406+0000", 2023-10-09T05:59:54.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9304e-05 2023-10-09T05:59:54.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:59:54.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194417+0000", 2023-10-09T05:59:54.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0803e-05 2023-10-09T05:59:54.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194490+0000", 2023-10-09T05:59:54.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2939000000000001e-05 2023-10-09T05:59:54.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:59:54.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.194504+0000", 2023-10-09T05:59:54.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4358e-05 2023-10-09T05:59:54.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083265+0000", 2023-10-09T05:59:54.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88876104300000003 2023-10-09T05:59:54.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.383 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.383 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:59:54.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083282+0000", 2023-10-09T05:59:54.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6945000000000001e-05 2023-10-09T05:59:54.383 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.383 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:59:54.383 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.385 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82117 6.9 6.b6ae8c09 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:59:54.385 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.200001+0000", 2023-10-09T05:59:54.385 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 478.15255630299998, 2023-10-09T05:59:54.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88338774900000006, 2023-10-09T05:59:54.385 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:59:54.385 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:59:54.385 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:59:54.385 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:59:54.386 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:54.386 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82117 2023-10-09T05:59:54.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.386 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:59:54.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:59:54.386 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200001+0000", 2023-10-09T05:59:54.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:59:54.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200001+0000", 2023-10-09T05:59:54.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.387 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:59:54.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.199999+0000", 2023-10-09T05:59:54.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:59:54.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.387 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:59:54.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200060+0000", 2023-10-09T05:59:54.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0708999999999998e-05 2023-10-09T05:59:54.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:59:54.388 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200090+0000", 2023-10-09T05:59:54.388 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0085000000000001e-05 2023-10-09T05:59:54.388 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:59:54.388 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200101+0000", 2023-10-09T05:59:54.388 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.06e-05 2023-10-09T05:59:54.388 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200179+0000", 2023-10-09T05:59:54.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7848000000000004e-05 2023-10-09T05:59:54.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:59:54.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200192+0000", 2023-10-09T05:59:54.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3943e-05 2023-10-09T05:59:54.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083361+0000", 2023-10-09T05:59:54.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88316805099999995 2023-10-09T05:59:54.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:59:54.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083389+0000", 2023-10-09T05:59:54.390 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8532000000000001e-05 2023-10-09T05:59:54.390 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.390 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:59:54.390 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.390 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.390 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82118 6.9 6.aad68289 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:59:54.390 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.200117+0000", 2023-10-09T05:59:54.391 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 478.152440802, 2023-10-09T05:59:54.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88331848099999999, 2023-10-09T05:59:54.391 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:59:54.391 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:59:54.391 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:59:54.391 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:59:54.391 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:54.391 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82118 2023-10-09T05:59:54.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.391 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:59:54.391 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.391 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:59:54.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200117+0000", 2023-10-09T05:59:54.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:59:54.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200117+0000", 2023-10-09T05:59:54.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:59:54.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200116+0000", 2023-10-09T05:59:54.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:59:54.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:59:54.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200162+0000", 2023-10-09T05:59:54.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5788999999999998e-05 2023-10-09T05:59:54.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:59:54.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200174+0000", 2023-10-09T05:59:54.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1905999999999999e-05 2023-10-09T05:59:54.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:59:54.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200181+0000", 2023-10-09T05:59:54.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2690000000000002e-06 2023-10-09T05:59:54.394 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.394 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.394 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200224+0000", 2023-10-09T05:59:54.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2839000000000001e-05 2023-10-09T05:59:54.394 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.394 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.394 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:59:54.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.200233+0000", 2023-10-09T05:59:54.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2490000000000007e-06 2023-10-09T05:59:54.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.395 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083420+0000", 2023-10-09T05:59:54.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88318651100000001 2023-10-09T05:59:54.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.395 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:59:54.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.083435+0000", 2023-10-09T05:59:54.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5474000000000001e-05 2023-10-09T05:59:54.395 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.395 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:59:54.396 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.396 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.396 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.396 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82125 6.0 6.dade3890 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:59:54.396 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.207895+0000", 2023-10-09T05:59:54.396 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 478.14466259900001, 2023-10-09T05:59:54.396 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.873606989, 2023-10-09T05:59:54.396 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:59:54.396 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:59:54.396 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:59:54.396 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:59:54.397 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:54.397 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82125 2023-10-09T05:59:54.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.397 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:59:54.397 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.397 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:59:54.397 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.207895+0000", 2023-10-09T05:59:54.397 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.397 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.397 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:59:54.397 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.207895+0000", 2023-10-09T05:59:54.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.398 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:59:54.398 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.207893+0000", 2023-10-09T05:59:54.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:59:54.398 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:59:54.398 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.207955+0000", 2023-10-09T05:59:54.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2361999999999997e-05 2023-10-09T05:59:54.398 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:59:54.399 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.207982+0000", 2023-10-09T05:59:54.399 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6838e-05 2023-10-09T05:59:54.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:59:54.399 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.207995+0000", 2023-10-09T05:59:54.399 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2992e-05 2023-10-09T05:59:54.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.208064+0000", 2023-10-09T05:59:54.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9366e-05 2023-10-09T05:59:54.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.400 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:59:54.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.208080+0000", 2023-10-09T05:59:54.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5690000000000001e-05 2023-10-09T05:59:54.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.400 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081493+0000", 2023-10-09T05:59:54.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.87341295699999999 2023-10-09T05:59:54.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:59:54.401 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081502+0000", 2023-10-09T05:59:54.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1030000000000008e-06 2023-10-09T05:59:54.401 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.401 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:59:54.401 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.401 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:82137 6.0 6.90177890 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T05:59:54.401 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:51:56.236426+0000", 2023-10-09T05:59:54.402 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 478.11613142599998, 2023-10-09T05:59:54.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.84511718000000002, 2023-10-09T05:59:54.402 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:59:54.402 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:59:54.402 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:59:54.402 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:59:54.402 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:54.402 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 82137 2023-10-09T05:59:54.402 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.402 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:59:54.402 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.402 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:59:54.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236426+0000", 2023-10-09T05:59:54.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:59:54.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236426+0000", 2023-10-09T05:59:54.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.404 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.404 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:59:54.404 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236424+0000", 2023-10-09T05:59:54.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:59:54.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.404 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.404 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:59:54.404 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236475+0000", 2023-10-09T05:59:54.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0819999999999998e-05 2023-10-09T05:59:54.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.404 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.404 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:59:54.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236499+0000", 2023-10-09T05:59:54.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3669e-05 2023-10-09T05:59:54.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:59:54.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236510+0000", 2023-10-09T05:59:54.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1258e-05 2023-10-09T05:59:54.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236585+0000", 2023-10-09T05:59:54.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4828999999999998e-05 2023-10-09T05:59:54.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.406 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:59:54.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:56.236601+0000", 2023-10-09T05:59:54.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6308999999999999e-05 2023-10-09T05:59:54.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.406 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081531+0000", 2023-10-09T05:59:54.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.84493010099999999 2023-10-09T05:59:54.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:59:54.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:51:57.081543+0000", 2023-10-09T05:59:54.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2048000000000001e-05 2023-10-09T05:59:54.407 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.407 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:59:54.407 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.407 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104164 6.9 6:9795f466:::benchmark_data_smithi064_284193_object7621:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:59:54.407 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:40.474997+0000", 2023-10-09T05:59:54.407 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 313.87756031800001, 2023-10-09T05:59:54.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6425455449999999, 2023-10-09T05:59:54.408 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:59:54.408 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:54.408 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:59:54.408 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:59:54.408 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:54.408 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104164 2023-10-09T05:59:54.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.408 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:59:54.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:59:54.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.474997+0000", 2023-10-09T05:59:54.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:59:54.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.474997+0000", 2023-10-09T05:59:54.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:59:54.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.474995+0000", 2023-10-09T05:59:54.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T05:59:54.410 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.410 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:59:54.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.475006+0000", 2023-10-09T05:59:54.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1127e-05 2023-10-09T05:59:54.410 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.410 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:59:54.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.475022+0000", 2023-10-09T05:59:54.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6073999999999999e-05 2023-10-09T05:59:54.410 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.411 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:59:54.411 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.475034+0000", 2023-10-09T05:59:54.411 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.183e-05 2023-10-09T05:59:54.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.411 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.411 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.701067+0000", 2023-10-09T05:59:54.411 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2260326539999999 2023-10-09T05:59:54.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.411 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:59:54.411 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.733298+0000", 2023-10-09T05:59:54.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.032231152999999998 2023-10-09T05:59:54.412 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.412 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.412 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.412 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.911525+0000", 2023-10-09T05:59:54.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.178227464 2023-10-09T05:59:54.412 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.412 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.412 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:59:54.412 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.921036+0000", 2023-10-09T05:59:54.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.009510447 2023-10-09T05:59:54.412 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.413 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.413 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:59:54.413 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.934794+0000", 2023-10-09T05:59:54.413 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013758014000000001 2023-10-09T05:59:54.413 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.413 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.413 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:59:54.413 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.041761+0000", 2023-10-09T05:59:54.413 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.10696707699999999 2023-10-09T05:59:54.413 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.413 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.413 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:54.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.078719+0000", 2023-10-09T05:59:54.414 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.036957894999999998 2023-10-09T05:59:54.414 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.414 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.414 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:54.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.081584+0000", 2023-10-09T05:59:54.414 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002865233 2023-10-09T05:59:54.414 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.414 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.414 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:59:54.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.085706+0000", 2023-10-09T05:59:54.415 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041217950000000001 2023-10-09T05:59:54.415 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.415 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.415 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:59:54.415 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.117543+0000", 2023-10-09T05:59:54.415 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.031837034 2023-10-09T05:59:54.415 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.415 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:59:54.415 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.415 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.416 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.416 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104221 6.9 6:992da745:::benchmark_data_smithi064_284193_object7678:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:59:54.416 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:41.522885+0000", 2023-10-09T05:59:54.416 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 312.82967303499998, 2023-10-09T05:59:54.416 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.645910236, 2023-10-09T05:59:54.416 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:59:54.416 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:54.416 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:59:54.416 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:59:54.416 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:54.416 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104221 2023-10-09T05:59:54.416 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.416 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:59:54.417 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.417 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:59:54.417 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.522885+0000", 2023-10-09T05:59:54.417 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.417 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.417 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.417 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:59:54.417 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.522885+0000", 2023-10-09T05:59:54.417 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.417 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.417 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.417 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:59:54.418 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.522883+0000", 2023-10-09T05:59:54.418 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:59:54.418 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.418 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.418 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:59:54.418 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.522890+0000", 2023-10-09T05:59:54.418 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1389999999999998e-06 2023-10-09T05:59:54.418 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.418 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.418 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:59:54.418 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.522906+0000", 2023-10-09T05:59:54.418 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6056e-05 2023-10-09T05:59:54.419 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.419 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.419 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:59:54.419 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.522916+0000", 2023-10-09T05:59:54.419 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0098999999999999e-05 2023-10-09T05:59:54.419 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.419 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.419 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.419 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.734737+0000", 2023-10-09T05:59:54.419 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.211820541 2023-10-09T05:59:54.419 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.419 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.420 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:59:54.420 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.756962+0000", 2023-10-09T05:59:54.420 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.022225556 2023-10-09T05:59:54.420 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.420 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.420 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.420 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.001519+0000", 2023-10-09T05:59:54.420 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24455652899999999 2023-10-09T05:59:54.420 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.420 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.420 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:59:54.420 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.008693+0000", 2023-10-09T05:59:54.420 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0071741890000000001 2023-10-09T05:59:54.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:59:54.421 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.024456+0000", 2023-10-09T05:59:54.421 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015762548000000001 2023-10-09T05:59:54.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:59:54.421 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.075878+0000", 2023-10-09T05:59:54.421 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.051422622000000001 2023-10-09T05:59:54.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:54.422 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.122918+0000", 2023-10-09T05:59:54.422 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.047039865 2023-10-09T05:59:54.422 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.422 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.422 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:54.422 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.127710+0000", 2023-10-09T05:59:54.422 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047916829999999997 2023-10-09T05:59:54.422 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.422 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.422 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:59:54.422 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.136431+0000", 2023-10-09T05:59:54.422 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0087210410000000006 2023-10-09T05:59:54.423 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.423 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.423 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:59:54.423 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.168795+0000", 2023-10-09T05:59:54.423 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.032364048999999999 2023-10-09T05:59:54.423 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.423 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:59:54.423 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.423 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.423 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.423 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104231 6.9 6:9a9d673b:::benchmark_data_smithi064_284193_object7688:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:59:54.424 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:41.584372+0000", 2023-10-09T05:59:54.424 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 312.76818597699997, 2023-10-09T05:59:54.424 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1037428409999999, 2023-10-09T05:59:54.424 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:59:54.424 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:54.424 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:59:54.424 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:59:54.424 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:54.424 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104231 2023-10-09T05:59:54.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.424 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:59:54.424 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:59:54.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.584372+0000", 2023-10-09T05:59:54.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:59:54.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.584372+0000", 2023-10-09T05:59:54.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:59:54.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.584370+0000", 2023-10-09T05:59:54.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:59:54.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:59:54.426 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.584377+0000", 2023-10-09T05:59:54.426 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6560000000000003e-06 2023-10-09T05:59:54.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:59:54.426 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.584391+0000", 2023-10-09T05:59:54.426 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3842000000000001e-05 2023-10-09T05:59:54.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.427 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:59:54.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.584398+0000", 2023-10-09T05:59:54.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.34e-06 2023-10-09T05:59:54.427 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.427 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.427 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.170598+0000", 2023-10-09T05:59:54.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5861996390000002 2023-10-09T05:59:54.427 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.427 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.427 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:59:54.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.208581+0000", 2023-10-09T05:59:54.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.037983153999999998 2023-10-09T05:59:54.428 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.428 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.428 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.428 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.354464+0000", 2023-10-09T05:59:54.428 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14588357699999999 2023-10-09T05:59:54.428 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.428 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.428 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:59:54.428 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.362830+0000", 2023-10-09T05:59:54.428 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0083655650000000002 2023-10-09T05:59:54.428 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.429 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.429 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:59:54.429 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.387897+0000", 2023-10-09T05:59:54.429 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.025067605999999999 2023-10-09T05:59:54.429 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.429 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.429 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:59:54.429 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.541824+0000", 2023-10-09T05:59:54.429 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15392655499999999 2023-10-09T05:59:54.429 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.429 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.429 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:54.430 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.545759+0000", 2023-10-09T05:59:54.430 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039354539999999997 2023-10-09T05:59:54.430 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.430 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.430 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:54.430 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.640753+0000", 2023-10-09T05:59:54.430 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.094993853000000003 2023-10-09T05:59:54.430 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.430 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.430 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:59:54.430 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.650699+0000", 2023-10-09T05:59:54.430 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0099456360000000008 2023-10-09T05:59:54.431 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.431 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.431 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:59:54.431 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.688114+0000", 2023-10-09T05:59:54.431 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.037415551999999998 2023-10-09T05:59:54.431 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.431 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:59:54.431 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.431 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.431 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.431 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104288 6.9 6:97b881c9:::benchmark_data_smithi064_284193_object7745:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T05:59:54.431 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:42.618333+0000", 2023-10-09T05:59:54.431 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 311.73422411199999, 2023-10-09T05:59:54.432 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1678057799999999, 2023-10-09T05:59:54.432 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:59:54.432 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:54.432 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:59:54.432 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:59:54.432 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:54.432 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104288 2023-10-09T05:59:54.432 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.432 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:59:54.432 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.432 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:59:54.432 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.618333+0000", 2023-10-09T05:59:54.433 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.433 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.433 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.433 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:59:54.433 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.618333+0000", 2023-10-09T05:59:54.433 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.433 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.433 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.433 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:59:54.433 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.618331+0000", 2023-10-09T05:59:54.433 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:59:54.433 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.434 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.434 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:59:54.434 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.618341+0000", 2023-10-09T05:59:54.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3049999999999998e-06 2023-10-09T05:59:54.434 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.434 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.434 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:59:54.434 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.618366+0000", 2023-10-09T05:59:54.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5198000000000002e-05 2023-10-09T05:59:54.434 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.434 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.434 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:59:54.435 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.618379+0000", 2023-10-09T05:59:54.435 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2683e-05 2023-10-09T05:59:54.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.435 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.435 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.210649+0000", 2023-10-09T05:59:54.435 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.59226995500000001 2023-10-09T05:59:54.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.435 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:59:54.435 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.258221+0000", 2023-10-09T05:59:54.435 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.047572956 2023-10-09T05:59:54.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.436 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.480425+0000", 2023-10-09T05:59:54.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.22220395700000001 2023-10-09T05:59:54.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:59:54.436 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.495694+0000", 2023-10-09T05:59:54.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015268805999999999 2023-10-09T05:59:54.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:59:54.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.524222+0000", 2023-10-09T05:59:54.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.028527800999999998 2023-10-09T05:59:54.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:59:54.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.593453+0000", 2023-10-09T05:59:54.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.069230594000000006 2023-10-09T05:59:54.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:54.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.732762+0000", 2023-10-09T05:59:54.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.139309778 2023-10-09T05:59:54.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.438 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.438 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:54.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.738937+0000", 2023-10-09T05:59:54.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061743409999999999 2023-10-09T05:59:54.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.438 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.438 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:59:54.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.749997+0000", 2023-10-09T05:59:54.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011059897000000001 2023-10-09T05:59:54.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.438 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.438 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:59:54.439 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:43.786139+0000", 2023-10-09T05:59:54.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.036142625999999997 2023-10-09T05:59:54.439 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.439 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:59:54.439 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.439 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPushReply(6.9 204/189 [PushReplyOp(6:94d4d87b:::benchmark_data_smithi064_284193_object70172:head)])", 2023-10-09T05:59:54.439 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:08.385201+0000", 2023-10-09T05:59:54.439 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 285.96735644699999, 2023-10-09T05:59:54.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4252824470000001, 2023-10-09T05:59:54.439 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:59:54.439 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:59:54.440 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:59:54.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:59:54.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:08.385201+0000", 2023-10-09T05:59:54.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:59:54.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:08.385201+0000", 2023-10-09T05:59:54.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:59:54.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:08.385200+0000", 2023-10-09T05:59:54.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:59:54.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:59:54.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:08.385210+0000", 2023-10-09T05:59:54.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8349999999999999e-06 2023-10-09T05:59:54.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:59:54.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:08.385224+0000", 2023-10-09T05:59:54.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4494e-05 2023-10-09T05:59:54.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.442 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.442 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:59:54.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:08.385236+0000", 2023-10-09T05:59:54.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1654000000000001e-05 2023-10-09T05:59:54.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.442 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.442 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:09.796497+0000", 2023-10-09T05:59:54.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4112608579999999 2023-10-09T05:59:54.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.443 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.443 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:59:54.443 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:09.810484+0000", 2023-10-09T05:59:54.443 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013987006 2023-10-09T05:59:54.443 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.443 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:59:54.443 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.443 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.443 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.443 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119183 6.10 6:0977e43c:::benchmark_data_smithi064_284193_object22640:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e212)", 2023-10-09T05:59:54.443 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:43.676425+0000", 2023-10-09T05:59:54.443 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 250.67613253499999, 2023-10-09T05:59:54.444 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.201728417, 2023-10-09T05:59:54.444 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:59:54.444 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:54.444 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:59:54.444 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:59:54.444 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:54.444 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119183 2023-10-09T05:59:54.444 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.444 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:59:54.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.444 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:59:54.444 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:43.676425+0000", 2023-10-09T05:59:54.445 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:59:54.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:43.676425+0000", 2023-10-09T05:59:54.445 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:59:54.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:43.676424+0000", 2023-10-09T05:59:54.445 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:59:54.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.446 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:59:54.446 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:43.676433+0000", 2023-10-09T05:59:54.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6989999999999998e-06 2023-10-09T05:59:54.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.446 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:59:54.446 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:43.676445+0000", 2023-10-09T05:59:54.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1764e-05 2023-10-09T05:59:54.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:59:54.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:43.676459+0000", 2023-10-09T05:59:54.447 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3677e-05 2023-10-09T05:59:54.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:43.695833+0000", 2023-10-09T05:59:54.447 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.019374216999999999 2023-10-09T05:59:54.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:59:54.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:43.695851+0000", 2023-10-09T05:59:54.447 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7960000000000001e-05 2023-10-09T05:59:54.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.448 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.448 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.867602+0000", 2023-10-09T05:59:54.448 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1717509829999999 2023-10-09T05:59:54.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.448 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.448 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:59:54.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.867785+0000", 2023-10-09T05:59:54.448 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000182685 2023-10-09T05:59:54.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:59:54.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.868119+0000", 2023-10-09T05:59:54.449 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00033449400000000001 2023-10-09T05:59:54.449 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:59:54.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.871702+0000", 2023-10-09T05:59:54.449 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035825039999999998 2023-10-09T05:59:54.449 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:54.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.873074+0000", 2023-10-09T05:59:54.450 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013719579999999999 2023-10-09T05:59:54.450 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.450 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.450 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:54.450 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.876541+0000", 2023-10-09T05:59:54.450 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034674469999999998 2023-10-09T05:59:54.450 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.450 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.450 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:59:54.450 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.876608+0000", 2023-10-09T05:59:54.450 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7090999999999999e-05 2023-10-09T05:59:54.451 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.451 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.451 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:59:54.451 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.878153+0000", 2023-10-09T05:59:54.451 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001545329 2023-10-09T05:59:54.451 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.451 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:59:54.451 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.451 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.451 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.451 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPushReply(6.19 225/190 [PushReplyOp(6:9ac767a4:::benchmark_data_smithi064_284193_object64254:head)])", 2023-10-09T05:59:54.451 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:56:22.641093+0000", 2023-10-09T05:59:54.451 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 211.71146413299999, 2023-10-09T05:59:54.452 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0279770349999999, 2023-10-09T05:59:54.452 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:59:54.452 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:59:54.452 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:59:54.452 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.452 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:59:54.452 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:22.641093+0000", 2023-10-09T05:59:54.452 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.452 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.452 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.452 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:59:54.452 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:22.641093+0000", 2023-10-09T05:59:54.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.453 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.453 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.453 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:59:54.453 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:22.641092+0000", 2023-10-09T05:59:54.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T05:59:54.453 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.453 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.453 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:59:54.453 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:22.641110+0000", 2023-10-09T05:59:54.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7961e-05 2023-10-09T05:59:54.453 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.454 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:59:54.454 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:22.641123+0000", 2023-10-09T05:59:54.454 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3114e-05 2023-10-09T05:59:54.454 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.454 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:59:54.454 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:22.641133+0000", 2023-10-09T05:59:54.454 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4700000000000008e-06 2023-10-09T05:59:54.454 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.454 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.455 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:23.657800+0000", 2023-10-09T05:59:54.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0166667439999999 2023-10-09T05:59:54.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.455 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:59:54.455 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:23.669070+0000", 2023-10-09T05:59:54.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011270913 2023-10-09T05:59:54.455 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.455 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:59:54.455 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.456 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPushReply(6.19 225/190 [PushReplyOp(6:9df2b95e:::benchmark_data_smithi064_284193_object64255:head)])", 2023-10-09T05:59:54.456 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:56:22.710736+0000", 2023-10-09T05:59:54.456 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 211.64182128900001, 2023-10-09T05:59:54.456 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96840995299999999, 2023-10-09T05:59:54.456 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:59:54.456 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:59:54.456 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:59:54.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.456 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:59:54.456 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:22.710736+0000", 2023-10-09T05:59:54.456 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.456 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:59:54.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:22.710736+0000", 2023-10-09T05:59:54.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.457 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:59:54.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:22.710735+0000", 2023-10-09T05:59:54.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T05:59:54.457 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:59:54.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:22.710743+0000", 2023-10-09T05:59:54.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5530000000000004e-06 2023-10-09T05:59:54.458 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.458 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.458 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:59:54.458 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:22.710755+0000", 2023-10-09T05:59:54.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2262000000000001e-05 2023-10-09T05:59:54.458 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.458 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.458 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:59:54.458 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:22.710765+0000", 2023-10-09T05:59:54.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3959999999999995e-06 2023-10-09T05:59:54.458 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.459 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.459 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.459 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:23.669291+0000", 2023-10-09T05:59:54.459 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95852639900000003 2023-10-09T05:59:54.459 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.459 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.459 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:59:54.459 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:23.679146+0000", 2023-10-09T05:59:54.459 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0098552329999999997 2023-10-09T05:59:54.459 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.459 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:59:54.459 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.460 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPush(6.19 226/190 [PushOp(6:9a476af7:::benchmark_data_smithi064_284193_object64315:head, version: 124'3985, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:9a476af7:::benchmark_data_smithi064_284193_object64315:head@124'3985, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:59:54.460 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:56:26.154513+0000", 2023-10-09T05:59:54.460 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 208.19804430400001, 2023-10-09T05:59:54.460 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.06635038, 2023-10-09T05:59:54.460 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:59:54.460 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T05:59:54.460 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:59:54.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:59:54.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:26.154513+0000", 2023-10-09T05:59:54.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.461 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:59:54.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:26.154513+0000", 2023-10-09T05:59:54.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.461 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:59:54.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:26.154513+0000", 2023-10-09T05:59:54.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:59:54.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.462 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:59:54.462 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:26.154545+0000", 2023-10-09T05:59:54.462 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2435999999999999e-05 2023-10-09T05:59:54.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.462 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:59:54.462 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:26.154556+0000", 2023-10-09T05:59:54.462 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0723999999999999e-05 2023-10-09T05:59:54.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.463 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:59:54.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:26.154564+0000", 2023-10-09T05:59:54.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7149999999999998e-06 2023-10-09T05:59:54.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.463 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:27.209883+0000", 2023-10-09T05:59:54.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.05531872 2023-10-09T05:59:54.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.463 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:59:54.464 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:27.210207+0000", 2023-10-09T05:59:54.464 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00032434 2023-10-09T05:59:54.464 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.464 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:59:54.464 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:27.220864+0000", 2023-10-09T05:59:54.464 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01065669 2023-10-09T05:59:54.464 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.464 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:59:54.464 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.464 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.465 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:139223 6.29 6:94b85dbd:::benchmark_data_smithi064_284193_object42680:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T05:59:54.465 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:56:39.263746+0000", 2023-10-09T05:59:54.465 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 195.08881170800001, 2023-10-09T05:59:54.465 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.612463668, 2023-10-09T05:59:54.465 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:59:54.465 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:54.465 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:59:54.465 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:59:54.465 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:54.465 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 139223 2023-10-09T05:59:54.465 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.465 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:59:54.465 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:59:54.466 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:39.263746+0000", 2023-10-09T05:59:54.466 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.466 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.466 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:59:54.466 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:39.263746+0000", 2023-10-09T05:59:54.466 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.466 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.466 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:59:54.467 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:39.263744+0000", 2023-10-09T05:59:54.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T05:59:54.467 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.467 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.467 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:59:54.467 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:39.263754+0000", 2023-10-09T05:59:54.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8779999999999993e-06 2023-10-09T05:59:54.467 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.467 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.467 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:59:54.467 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:39.284922+0000", 2023-10-09T05:59:54.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021167807 2023-10-09T05:59:54.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:59:54.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:39.284938+0000", 2023-10-09T05:59:54.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6745999999999999e-05 2023-10-09T05:59:54.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:39.396958+0000", 2023-10-09T05:59:54.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.11202013800000001 2023-10-09T05:59:54.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:59:54.469 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:39.396970+0000", 2023-10-09T05:59:54.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1328000000000001e-05 2023-10-09T05:59:54.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.469 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:40.855893+0000", 2023-10-09T05:59:54.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4589233369999999 2023-10-09T05:59:54.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T05:59:54.469 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:40.858875+0000", 2023-10-09T05:59:54.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029821449999999998 2023-10-09T05:59:54.470 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.470 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.470 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:40.866916+0000", 2023-10-09T05:59:54.470 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0080405540000000001 2023-10-09T05:59:54.470 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.470 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:59:54.470 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:40.867946+0000", 2023-10-09T05:59:54.470 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010304240000000001 2023-10-09T05:59:54.470 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-10-09T05:59:54.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:40.870193+0000", 2023-10-09T05:59:54.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0022469429999999999 2023-10-09T05:59:54.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:54.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:40.872753+0000", 2023-10-09T05:59:54.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025593970000000001 2023-10-09T05:59:54.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:59:54.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:40.872901+0000", 2023-10-09T05:59:54.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014856099999999999 2023-10-09T05:59:54.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:54.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:40.873263+0000", 2023-10-09T05:59:54.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00036204500000000002 2023-10-09T05:59:54.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:59:54.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:40.873868+0000", 2023-10-09T05:59:54.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00060503100000000001 2023-10-09T05:59:54.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.473 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:59:54.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:56:40.876210+0000", 2023-10-09T05:59:54.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0023413230000000002 2023-10-09T05:59:54.473 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.473 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:59:54.473 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.473 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:159583 6.10 6:0dc577d6:::benchmark_data_smithi064_284193_object63040:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2023-10-09T05:59:54.473 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:57:22.430471+0000", 2023-10-09T05:59:54.473 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 151.92208609599999, 2023-10-09T05:59:54.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.364807377, 2023-10-09T05:59:54.474 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:59:54.474 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T05:59:54.474 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:59:54.474 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:59:54.474 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:54.474 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 159583 2023-10-09T05:59:54.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.474 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:59:54.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:59:54.474 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.430471+0000", 2023-10-09T05:59:54.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:59:54.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.430471+0000", 2023-10-09T05:59:54.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:59:54.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.430471+0000", 2023-10-09T05:59:54.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T05:59:54.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:59:54.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.430475+0000", 2023-10-09T05:59:54.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5000000000000001e-06 2023-10-09T05:59:54.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:59:54.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.430488+0000", 2023-10-09T05:59:54.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2247e-05 2023-10-09T05:59:54.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:59:54.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.430493+0000", 2023-10-09T05:59:54.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.631e-06 2023-10-09T05:59:54.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.430543+0000", 2023-10-09T05:59:54.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9444000000000003e-05 2023-10-09T05:59:54.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:59:54.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.430666+0000", 2023-10-09T05:59:54.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012383399999999999 2023-10-09T05:59:54.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,5", 2023-10-09T05:59:54.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.431023+0000", 2023-10-09T05:59:54.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00035648400000000001 2023-10-09T05:59:54.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:59:54.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.433954+0000", 2023-10-09T05:59:54.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029310740000000001 2023-10-09T05:59:54.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:54.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.437343+0000", 2023-10-09T05:59:54.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033885790000000001 2023-10-09T05:59:54.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.451162+0000", 2023-10-09T05:59:54.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01381925 2023-10-09T05:59:54.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:59:54.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.451173+0000", 2023-10-09T05:59:54.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1229999999999999e-05 2023-10-09T05:59:54.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.469718+0000", 2023-10-09T05:59:54.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.018544736 2023-10-09T05:59:54.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:59:54.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.469731+0000", 2023-10-09T05:59:54.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3584000000000001e-05 2023-10-09T05:59:54.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.784199+0000", 2023-10-09T05:59:54.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.314468078 2023-10-09T05:59:54.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-10-09T05:59:54.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.784351+0000", 2023-10-09T05:59:54.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015171100000000001 2023-10-09T05:59:54.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.795179+0000", 2023-10-09T05:59:54.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010827577 2023-10-09T05:59:54.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.482 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:59:54.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.795279+0000", 2023-10-09T05:59:54.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000100115 2023-10-09T05:59:54.482 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.482 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:59:54.482 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.482 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:159606 6.20 6:0672fcbe:::benchmark_data_smithi064_284193_object63063:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-10-09T05:59:54.483 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:57:22.478808+0000", 2023-10-09T05:59:54.483 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 151.87374971200001, 2023-10-09T05:59:54.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3242037419999999, 2023-10-09T05:59:54.483 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T05:59:54.483 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:59:54.483 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T05:59:54.483 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T05:59:54.483 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T05:59:54.483 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 159606 2023-10-09T05:59:54.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.483 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T05:59:54.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T05:59:54.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.478808+0000", 2023-10-09T05:59:54.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T05:59:54.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.478808+0000", 2023-10-09T05:59:54.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T05:59:54.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T05:59:54.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.478806+0000", 2023-10-09T05:59:54.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T05:59:54.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T05:59:54.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.478813+0000", 2023-10-09T05:59:54.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.511e-06 2023-10-09T05:59:54.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T05:59:54.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.478829+0000", 2023-10-09T05:59:54.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5574999999999999e-05 2023-10-09T05:59:54.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T05:59:54.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.478837+0000", 2023-10-09T05:59:54.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2160000000000006e-06 2023-10-09T05:59:54.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.478888+0000", 2023-10-09T05:59:54.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1242000000000002e-05 2023-10-09T05:59:54.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:59:54.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:22.478897+0000", 2023-10-09T05:59:54.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4220000000000006e-06 2023-10-09T05:59:54.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.469800+0000", 2023-10-09T05:59:54.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99090245600000004 2023-10-09T05:59:54.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T05:59:54.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.469808+0000", 2023-10-09T05:59:54.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3110000000000006e-06 2023-10-09T05:59:54.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T05:59:54.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.784786+0000", 2023-10-09T05:59:54.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.314978027 2023-10-09T05:59:54.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T05:59:54.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.784932+0000", 2023-10-09T05:59:54.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014589599999999999 2023-10-09T05:59:54.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,3,5", 2023-10-09T05:59:54.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.785236+0000", 2023-10-09T05:59:54.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00030418699999999998 2023-10-09T05:59:54.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:54.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.786818+0000", 2023-10-09T05:59:54.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0015812479999999999 2023-10-09T05:59:54.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T05:59:54.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.790322+0000", 2023-10-09T05:59:54.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035042110000000001 2023-10-09T05:59:54.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:54.490 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.794740+0000", 2023-10-09T05:59:54.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044178969999999996 2023-10-09T05:59:54.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:59:54.490 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.802768+0000", 2023-10-09T05:59:54.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0080280739999999996 2023-10-09T05:59:54.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T05:59:54.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.802806+0000", 2023-10-09T05:59:54.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8460000000000001e-05 2023-10-09T05:59:54.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T05:59:54.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T05:59:54.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T05:59:54.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:24.803012+0000", 2023-10-09T05:59:54.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020540000000000001 2023-10-09T05:59:54.491 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.491 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:59:54.491 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.491 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T05:59:54.491 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T05:59:54.492 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T05:59:54.492 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:54.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:54 smithi156 ceph-mon[195101]: pgmap v707: 119 pgs: 119 active+clean; 5.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 54 MiB/s wr, 870 op/s 2023-10-09T05:59:54.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:54 smithi156 ceph-mon[195101]: 1.d scrub starts 2023-10-09T05:59:54.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:54 smithi156 ceph-mon[195101]: 1.d scrub ok 2023-10-09T05:59:54.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:54 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:54.621 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:54.701 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:54 smithi064 ceph-mon[220044]: pgmap v707: 119 pgs: 119 active+clean; 5.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 54 MiB/s wr, 870 op/s 2023-10-09T05:59:54.701 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:54 smithi064 ceph-mon[220044]: 1.d scrub starts 2023-10-09T05:59:54.701 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:54 smithi064 ceph-mon[220044]: 1.d scrub ok 2023-10-09T05:59:54.701 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:54 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T05:59:54.811 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:54.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:54.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:54.917 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:54.976 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:55.008 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 76 16 93744 93728 77.0351 55.0625 0.016684 0.0129765 2023-10-09T05:59:55.063 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:55.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:55.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:55.175 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:59:55.213 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:55.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:55 smithi064 ceph-mon[220044]: 1.5 scrub starts 2023-10-09T05:59:55.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:55 smithi064 ceph-mon[220044]: 1.5 scrub ok 2023-10-09T05:59:55.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:55 smithi064 ceph-mon[220044]: 1.6 scrub starts 2023-10-09T05:59:55.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:55 smithi064 ceph-mon[220044]: 1.6 scrub ok 2023-10-09T05:59:55.470 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 05:59:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:05:59:55] "GET /metrics HTTP/1.1" 200 36407 "" "Prometheus/2.43.0" 2023-10-09T05:59:55.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:55 smithi064 ceph-mon[212750]: 1.5 scrub starts 2023-10-09T05:59:55.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:55 smithi064 ceph-mon[212750]: 1.5 scrub ok 2023-10-09T05:59:55.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:55 smithi064 ceph-mon[212750]: 1.6 scrub starts 2023-10-09T05:59:55.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:55 smithi064 ceph-mon[212750]: 1.6 scrub ok 2023-10-09T05:59:55.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:55 smithi156 ceph-mon[195101]: 1.5 scrub starts 2023-10-09T05:59:55.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:55 smithi156 ceph-mon[195101]: 1.5 scrub ok 2023-10-09T05:59:55.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:55 smithi156 ceph-mon[195101]: 1.6 scrub starts 2023-10-09T05:59:55.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:55 smithi156 ceph-mon[195101]: 1.6 scrub ok 2023-10-09T05:59:55.729 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:59:55.806 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:59:55.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:55.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:59:55.923 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:55.948 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:55.996 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:56.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:56.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:56.008 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 77 16 94571 94555 76.706 51.6875 0.00358408 0.0130258 2023-10-09T05:59:56.102 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:59:56.128 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:56.220 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:59:56.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:56.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:59:56.302 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:59:56.327 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:56.414 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:56.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:56 smithi156 ceph-mon[195101]: pgmap v708: 119 pgs: 119 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 54 MiB/s wr, 871 op/s 2023-10-09T05:59:56.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:56 smithi156 ceph-mon[195101]: 1.9 scrub starts 2023-10-09T05:59:56.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:56 smithi156 ceph-mon[195101]: 1.9 scrub ok 2023-10-09T05:59:56.635 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:56 smithi064 ceph-mon[220044]: pgmap v708: 119 pgs: 119 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 54 MiB/s wr, 871 op/s 2023-10-09T05:59:56.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:56 smithi064 ceph-mon[220044]: 1.9 scrub starts 2023-10-09T05:59:56.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:56 smithi064 ceph-mon[220044]: 1.9 scrub ok 2023-10-09T05:59:56.637 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:56 smithi064 ceph-mon[212750]: pgmap v708: 119 pgs: 119 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 54 MiB/s wr, 871 op/s 2023-10-09T05:59:56.637 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:56 smithi064 ceph-mon[212750]: 1.9 scrub starts 2023-10-09T05:59:56.637 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:56 smithi064 ceph-mon[212750]: 1.9 scrub ok 2023-10-09T05:59:56.756 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:56.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:56.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:56.865 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:56.907 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:56.971 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:56.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:56.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:57.009 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 78 16 95290 95274 76.2988 44.9375 0.0255986 0.0131013 2023-10-09T05:59:57.082 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:59:57.164 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:57.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:57 smithi064 ceph-mon[212750]: 1.7 scrub starts 2023-10-09T05:59:57.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:57 smithi064 ceph-mon[212750]: 1.7 scrub ok 2023-10-09T05:59:57.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:57 smithi064 ceph-mon[212750]: 1.b scrub starts 2023-10-09T05:59:57.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:57 smithi064 ceph-mon[212750]: 1.b scrub ok 2023-10-09T05:59:57.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:57 smithi064 ceph-mon[220044]: 1.7 scrub starts 2023-10-09T05:59:57.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:57 smithi064 ceph-mon[220044]: 1.7 scrub ok 2023-10-09T05:59:57.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:57 smithi064 ceph-mon[220044]: 1.b scrub starts 2023-10-09T05:59:57.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:57 smithi064 ceph-mon[220044]: 1.b scrub ok 2023-10-09T05:59:57.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:57 smithi156 ceph-mon[195101]: 1.7 scrub starts 2023-10-09T05:59:57.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:57 smithi156 ceph-mon[195101]: 1.7 scrub ok 2023-10-09T05:59:57.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:57 smithi156 ceph-mon[195101]: 1.b scrub starts 2023-10-09T05:59:57.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:57 smithi156 ceph-mon[195101]: 1.b scrub ok 2023-10-09T05:59:57.949 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:59:57.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:57.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:59:58.009 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 79 16 96132 96116 75.9992 52.625 0.0140981 0.0131539 2023-10-09T05:59:58.057 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:59:58.097 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:58.164 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:59:58.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:58.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:59:58.275 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:58.292 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:58.353 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:58.383 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:58 smithi156 ceph-mon[195101]: pgmap v709: 119 pgs: 119 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 53 MiB/s wr, 851 op/s 2023-10-09T05:59:58.383 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:58 smithi156 ceph-mon[195101]: 1.e scrub starts 2023-10-09T05:59:58.383 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:58 smithi156 ceph-mon[195101]: 1.e scrub ok 2023-10-09T05:59:58.383 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:58 smithi156 ceph-mon[195101]: 1.16 scrub starts 2023-10-09T05:59:58.383 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:58 smithi156 ceph-mon[195101]: 1.16 scrub ok 2023-10-09T05:59:58.383 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:58.449 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T05:59:58.493 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T05:59:58.531 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T05:59:58.569 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T05:59:58.584 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:59:58.589 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:58 smithi064 ceph-mon[220044]: pgmap v709: 119 pgs: 119 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 53 MiB/s wr, 851 op/s 2023-10-09T05:59:58.589 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:58 smithi064 ceph-mon[220044]: 1.e scrub starts 2023-10-09T05:59:58.589 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:58 smithi064 ceph-mon[220044]: 1.e scrub ok 2023-10-09T05:59:58.589 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:58 smithi064 ceph-mon[220044]: 1.16 scrub starts 2023-10-09T05:59:58.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:58 smithi064 ceph-mon[220044]: 1.16 scrub ok 2023-10-09T05:59:58.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:58 smithi064 ceph-mon[212750]: pgmap v709: 119 pgs: 119 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 53 MiB/s wr, 851 op/s 2023-10-09T05:59:58.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:58 smithi064 ceph-mon[212750]: 1.e scrub starts 2023-10-09T05:59:58.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:58 smithi064 ceph-mon[212750]: 1.e scrub ok 2023-10-09T05:59:58.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:58 smithi064 ceph-mon[212750]: 1.16 scrub starts 2023-10-09T05:59:58.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:58 smithi064 ceph-mon[212750]: 1.16 scrub ok 2023-10-09T05:59:58.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:58.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:59:58.606 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T05:59:58.658 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T05:59:58.693 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:59:58.696 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T05:59:58.737 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T05:59:58.745 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:58.748 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:59:58.748 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:59:58.749 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:59:58.749 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:59:58.749 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:59:58.749 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:59:58.749 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:59:58.749 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:59:58.806 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:59:58.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:58.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:59:58.917 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:58.975 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:59.010 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-10-09T05:59:59.010445+0000 min lat: 0.00127391 max lat: 1.01073 avg lat: 0.0132094 2023-10-09T05:59:59.010 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:59:59.010 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 80 16 96935 96919 75.6759 50.1875 0.0192826 0.0132094 2023-10-09T05:59:59.055 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:59.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:59.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:59.163 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:59:59.190 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:59.267 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:59:59.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:59:59.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:59:59.377 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:59:59.422 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:59:59.613 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:59 smithi064 ceph-mon[212750]: 1.1a scrub starts 2023-10-09T05:59:59.613 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:59 smithi064 ceph-mon[212750]: 1.1a scrub ok 2023-10-09T05:59:59.613 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:59 smithi064 ceph-mon[212750]: 1.8 scrub starts 2023-10-09T05:59:59.613 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:59 smithi064 ceph-mon[212750]: 1.8 scrub ok 2023-10-09T05:59:59.613 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:59 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2043357571' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:59.613 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:59.613 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 05:59:59 smithi064 ceph-mon[212750]: pgmap v710: 119 pgs: 119 active+clean; 5.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 53 MiB/s wr, 842 op/s 2023-10-09T05:59:59.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:59 smithi064 ceph-mon[220044]: 1.1a scrub starts 2023-10-09T05:59:59.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:59 smithi064 ceph-mon[220044]: 1.1a scrub ok 2023-10-09T05:59:59.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:59 smithi064 ceph-mon[220044]: 1.8 scrub starts 2023-10-09T05:59:59.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:59 smithi064 ceph-mon[220044]: 1.8 scrub ok 2023-10-09T05:59:59.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:59 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2043357571' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:59.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:59.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 05:59:59 smithi064 ceph-mon[220044]: pgmap v710: 119 pgs: 119 active+clean; 5.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 53 MiB/s wr, 842 op/s 2023-10-09T05:59:59.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:59 smithi156 ceph-mon[195101]: 1.1a scrub starts 2023-10-09T05:59:59.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:59 smithi156 ceph-mon[195101]: 1.1a scrub ok 2023-10-09T05:59:59.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:59 smithi156 ceph-mon[195101]: 1.8 scrub starts 2023-10-09T05:59:59.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:59 smithi156 ceph-mon[195101]: 1.8 scrub ok 2023-10-09T05:59:59.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:59 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2043357571' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:59:59.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:59.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 05:59:59 smithi156 ceph-mon[195101]: pgmap v710: 119 pgs: 119 active+clean; 5.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 53 MiB/s wr, 842 op/s 2023-10-09T06:00:00.010 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 81 16 97779 97763 75.3929 52.75 0.0149681 0.0132597 2023-10-09T06:00:00.222 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:00.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:00.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:00.330 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:00.365 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T06:00:00.428 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:00.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:00 smithi064 ceph-mon[212750]: 1.1b scrub starts 2023-10-09T06:00:00.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:00 smithi064 ceph-mon[212750]: 1.1b scrub ok 2023-10-09T06:00:00.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:00 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1612393719' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:00:00.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:00 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:00:00.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:00 smithi064 ceph-mon[212750]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-10-09T06:00:00.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:00 smithi064 ceph-mon[212750]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T06:00:00.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:00 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:00:00.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:00 smithi064 ceph-mon[212750]: osdmap e308: 8 total, 8 up, 7 in 2023-10-09T06:00:00.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:00 smithi064 ceph-mon[220044]: 1.1b scrub starts 2023-10-09T06:00:00.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:00 smithi064 ceph-mon[220044]: 1.1b scrub ok 2023-10-09T06:00:00.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:00 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1612393719' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:00:00.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:00 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:00:00.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:00 smithi064 ceph-mon[220044]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-10-09T06:00:00.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:00 smithi064 ceph-mon[220044]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T06:00:00.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:00 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:00:00.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:00 smithi064 ceph-mon[220044]: osdmap e308: 8 total, 8 up, 7 in 2023-10-09T06:00:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:00 smithi156 ceph-mon[195101]: 1.1b scrub starts 2023-10-09T06:00:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:00 smithi156 ceph-mon[195101]: 1.1b scrub ok 2023-10-09T06:00:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:00 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1612393719' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:00:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:00 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:00:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:00 smithi156 ceph-mon[195101]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-10-09T06:00:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:00 smithi156 ceph-mon[195101]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T06:00:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:00 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:00:00.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:00 smithi156 ceph-mon[195101]: osdmap e308: 8 total, 8 up, 7 in 2023-10-09T06:00:01.010 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 82 16 98581 98565 75.0849 50.125 0.0144688 0.0133129 2023-10-09T06:00:01.250 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:01.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:01.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:01.357 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:01.433 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:01.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:01 smithi064 ceph-mon[212750]: failed to encode map e308 with expected crc 2023-10-09T06:00:01.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:01 smithi064 ceph-mon[212750]: failed to encode map e308 with expected crc 2023-10-09T06:00:01.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:01 smithi064 ceph-mon[212750]: failed to encode map e308 with expected crc 2023-10-09T06:00:01.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:01 smithi064 ceph-mon[212750]: failed to encode map e308 with expected crc 2023-10-09T06:00:01.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:01 smithi064 ceph-mon[212750]: 1.1c scrub starts 2023-10-09T06:00:01.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:01 smithi064 ceph-mon[212750]: 1.1c scrub ok 2023-10-09T06:00:01.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:01 smithi064 ceph-mon[212750]: pgmap v712: 119 pgs: 119 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s 2023-10-09T06:00:01.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:01 smithi064 ceph-mon[220044]: failed to encode map e308 with expected crc 2023-10-09T06:00:01.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:01 smithi064 ceph-mon[220044]: failed to encode map e308 with expected crc 2023-10-09T06:00:01.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:01 smithi064 ceph-mon[220044]: failed to encode map e308 with expected crc 2023-10-09T06:00:01.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:01 smithi064 ceph-mon[220044]: failed to encode map e308 with expected crc 2023-10-09T06:00:01.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:01 smithi064 ceph-mon[220044]: 1.1c scrub starts 2023-10-09T06:00:01.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:01 smithi064 ceph-mon[220044]: 1.1c scrub ok 2023-10-09T06:00:01.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:01 smithi064 ceph-mon[220044]: pgmap v712: 119 pgs: 119 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s 2023-10-09T06:00:01.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:01 smithi156 ceph-mon[195101]: failed to encode map e308 with expected crc 2023-10-09T06:00:01.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:01 smithi156 ceph-mon[195101]: failed to encode map e308 with expected crc 2023-10-09T06:00:01.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:01 smithi156 ceph-mon[195101]: failed to encode map e308 with expected crc 2023-10-09T06:00:01.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:01 smithi156 ceph-mon[195101]: failed to encode map e308 with expected crc 2023-10-09T06:00:01.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:01 smithi156 ceph-mon[195101]: 1.1c scrub starts 2023-10-09T06:00:01.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:01 smithi156 ceph-mon[195101]: 1.1c scrub ok 2023-10-09T06:00:01.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:01 smithi156 ceph-mon[195101]: pgmap v712: 119 pgs: 119 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s 2023-10-09T06:00:02.010 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 83 16 99466 99450 74.8467 55.3125 0.00385099 0.0133547 2023-10-09T06:00:02.257 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:00:02.336 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:02.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:02.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:02.441 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:02.497 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T06:00:02.559 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:02.763 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:02.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:02.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:02.893 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:02.975 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:03.013 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 84 16 100205 100189 74.5033 46.1875 0.00893816 0.0134168 2023-10-09T06:00:03.158 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:03.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:03.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:03.167 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:00:03.167 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:00:03.264 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:03.359 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:03.526 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:03.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:03.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:03.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:03 smithi156 ceph-mon[195101]: 1.17 scrub starts 2023-10-09T06:00:03.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:03 smithi156 ceph-mon[195101]: 1.17 scrub ok 2023-10-09T06:00:03.636 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:03.697 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:03 smithi064 ceph-mon[220044]: 1.17 scrub starts 2023-10-09T06:00:03.697 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:03 smithi064 ceph-mon[220044]: 1.17 scrub ok 2023-10-09T06:00:03.700 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:03 smithi064 ceph-mon[212750]: 1.17 scrub starts 2023-10-09T06:00:03.700 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:03 smithi064 ceph-mon[212750]: 1.17 scrub ok 2023-10-09T06:00:03.733 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:03.871 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:03.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:03.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:03.977 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:04.012 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 85 16 101026 101010 74.2306 51.3125 0.00537906 0.0134665 2023-10-09T06:00:04.046 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:04.123 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:04.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:04.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:04.227 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:04.309 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:04.329 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:00:04.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:04 smithi156 ceph-mon[195101]: pgmap v713: 119 pgs: 119 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 51 MiB/s wr, 818 op/s 2023-10-09T06:00:04.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:04 smithi156 ceph-mon[195101]: 1.c scrub starts 2023-10-09T06:00:04.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:04 smithi156 ceph-mon[195101]: 1.c scrub ok 2023-10-09T06:00:04.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:04 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3608899091' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:04.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:04 smithi064 ceph-mon[220044]: pgmap v713: 119 pgs: 119 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 51 MiB/s wr, 818 op/s 2023-10-09T06:00:04.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:04 smithi064 ceph-mon[220044]: 1.c scrub starts 2023-10-09T06:00:04.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:04 smithi064 ceph-mon[220044]: 1.c scrub ok 2023-10-09T06:00:04.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:04 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3608899091' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:04.725 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:04 smithi064 ceph-mon[212750]: pgmap v713: 119 pgs: 119 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 51 MiB/s wr, 818 op/s 2023-10-09T06:00:04.725 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:04 smithi064 ceph-mon[212750]: 1.c scrub starts 2023-10-09T06:00:04.725 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:04 smithi064 ceph-mon[212750]: 1.c scrub ok 2023-10-09T06:00:04.725 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:04 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3608899091' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:04.777 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:04.811 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:04.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:04.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:04.921 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:04.968 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:05.013 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 86 16 101926 101910 74.0215 56.25 0.0082258 0.0135054 2023-10-09T06:00:05.059 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:05.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:05.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:05.171 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:05.234 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:05.267 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:05.288 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:00:05.355 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:05 smithi156 ceph-mon[195101]: 1.1e scrub starts 2023-10-09T06:00:05.355 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:05 smithi156 ceph-mon[195101]: 1.1e scrub ok 2023-10-09T06:00:05.355 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:05 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:00:05.355 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:05 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T06:00:05.355 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:05 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3608899091' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:00:05.355 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:05 smithi156 ceph-mon[195101]: osdmap e309: 8 total, 8 up, 7 in 2023-10-09T06:00:05.355 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:05 smithi156 ceph-mon[195101]: failed to encode map e309 with expected crc 2023-10-09T06:00:05.356 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:05 smithi156 ceph-mon[195101]: failed to encode map e309 with expected crc 2023-10-09T06:00:05.356 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:05 smithi156 ceph-mon[195101]: failed to encode map e309 with expected crc 2023-10-09T06:00:05.356 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:05 smithi156 ceph-mon[195101]: failed to encode map e309 with expected crc 2023-10-09T06:00:05.356 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:05 smithi156 ceph-mon[195101]: pgmap v715: 119 pgs: 119 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 42 MiB/s wr, 663 op/s 2023-10-09T06:00:05.510 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:05.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:05.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:05.524 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:05 smithi064 ceph-mon[220044]: 1.1e scrub starts 2023-10-09T06:00:05.524 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:05 smithi064 ceph-mon[220044]: 1.1e scrub ok 2023-10-09T06:00:05.525 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:05 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:00:05.525 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:05 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T06:00:05.525 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:05 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3608899091' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:00:05.525 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:05 smithi064 ceph-mon[220044]: osdmap e309: 8 total, 8 up, 7 in 2023-10-09T06:00:05.525 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:05 smithi064 ceph-mon[220044]: failed to encode map e309 with expected crc 2023-10-09T06:00:05.525 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:05 smithi064 ceph-mon[220044]: failed to encode map e309 with expected crc 2023-10-09T06:00:05.525 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:05 smithi064 ceph-mon[220044]: failed to encode map e309 with expected crc 2023-10-09T06:00:05.525 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:05 smithi064 ceph-mon[220044]: failed to encode map e309 with expected crc 2023-10-09T06:00:05.525 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:05 smithi064 ceph-mon[220044]: pgmap v715: 119 pgs: 119 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 42 MiB/s wr, 663 op/s 2023-10-09T06:00:05.526 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:05 smithi064 ceph-mon[212750]: 1.1e scrub starts 2023-10-09T06:00:05.526 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:05 smithi064 ceph-mon[212750]: 1.1e scrub ok 2023-10-09T06:00:05.526 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:05 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:00:05.526 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:05 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:00:05.526 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:05 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3608899091' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:00:05.526 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:05 smithi064 ceph-mon[212750]: osdmap e309: 8 total, 8 up, 7 in 2023-10-09T06:00:05.526 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:05 smithi064 ceph-mon[212750]: failed to encode map e309 with expected crc 2023-10-09T06:00:05.526 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:05 smithi064 ceph-mon[212750]: failed to encode map e309 with expected crc 2023-10-09T06:00:05.526 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:05 smithi064 ceph-mon[212750]: failed to encode map e309 with expected crc 2023-10-09T06:00:05.526 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:05 smithi064 ceph-mon[212750]: failed to encode map e309 with expected crc 2023-10-09T06:00:05.527 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:05 smithi064 ceph-mon[212750]: pgmap v715: 119 pgs: 119 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 42 MiB/s wr, 663 op/s 2023-10-09T06:00:05.527 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:00:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:00:05] "GET /metrics HTTP/1.1" 200 36392 "" "Prometheus/2.43.0" 2023-10-09T06:00:05.625 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:05.665 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:05.748 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:05.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:05.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:05.859 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:05.898 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:05.977 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:05.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:05.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:06.013 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 87 16 102823 102807 73.8152 56.0625 0.0146688 0.0135422 2023-10-09T06:00:06.086 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:06.133 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:06.229 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:06.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:06.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:06.336 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:06.359 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:06 smithi156 ceph-mon[195101]: 1.18 scrub starts 2023-10-09T06:00:06.359 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:06 smithi156 ceph-mon[195101]: 1.18 scrub ok 2023-10-09T06:00:06.398 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:06.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:06 smithi064 ceph-mon[220044]: 1.18 scrub starts 2023-10-09T06:00:06.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:06 smithi064 ceph-mon[220044]: 1.18 scrub ok 2023-10-09T06:00:06.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:06 smithi064 ceph-mon[212750]: 1.18 scrub starts 2023-10-09T06:00:06.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:06 smithi064 ceph-mon[212750]: 1.18 scrub ok 2023-10-09T06:00:07.013 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 88 16 103683 103667 73.5872 53.75 0.0157409 0.0135839 2023-10-09T06:00:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:07 smithi064 ceph-mon[212750]: 1.19 scrub starts 2023-10-09T06:00:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:07 smithi064 ceph-mon[212750]: 1.19 scrub ok 2023-10-09T06:00:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:07 smithi064 ceph-mon[212750]: 1.15 scrub starts 2023-10-09T06:00:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:07 smithi064 ceph-mon[212750]: 1.15 scrub ok 2023-10-09T06:00:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:07 smithi064 ceph-mon[212750]: pgmap v716: 119 pgs: 119 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T06:00:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:07 smithi064 ceph-mon[220044]: 1.19 scrub starts 2023-10-09T06:00:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:07 smithi064 ceph-mon[220044]: 1.19 scrub ok 2023-10-09T06:00:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:07 smithi064 ceph-mon[220044]: 1.15 scrub starts 2023-10-09T06:00:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:07 smithi064 ceph-mon[220044]: 1.15 scrub ok 2023-10-09T06:00:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:07 smithi064 ceph-mon[220044]: pgmap v716: 119 pgs: 119 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T06:00:07.551 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:00:07.652 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:07.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:07.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:07.762 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:07.763 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:07.859 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:07.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:07 smithi156 ceph-mon[195101]: 1.19 scrub starts 2023-10-09T06:00:07.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:07 smithi156 ceph-mon[195101]: 1.19 scrub ok 2023-10-09T06:00:07.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:07 smithi156 ceph-mon[195101]: 1.15 scrub starts 2023-10-09T06:00:07.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:07 smithi156 ceph-mon[195101]: 1.15 scrub ok 2023-10-09T06:00:07.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:07 smithi156 ceph-mon[195101]: pgmap v716: 119 pgs: 119 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T06:00:08.013 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 89 16 104537 104521 73.3601 53.375 0.0099951 0.0136269 2023-10-09T06:00:08.488 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:00:08.530 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:00:08.572 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:00:08.616 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:00:08.626 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:08.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:08.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:08.655 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:00:08.684 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:08 smithi064 ceph-mon[212750]: 1.1d scrub starts 2023-10-09T06:00:08.685 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:08 smithi064 ceph-mon[212750]: 1.1d scrub ok 2023-10-09T06:00:08.686 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:08 smithi064 ceph-mon[220044]: 1.1d scrub starts 2023-10-09T06:00:08.686 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:08 smithi064 ceph-mon[220044]: 1.1d scrub ok 2023-10-09T06:00:08.694 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:00:08.733 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:08.735 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:00:08.769 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:08 smithi156 ceph-mon[195101]: 1.1d scrub starts 2023-10-09T06:00:08.769 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:08 smithi156 ceph-mon[195101]: 1.1d scrub ok 2023-10-09T06:00:08.772 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:08.772 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:00:08.788 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:00:08.789 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:00:08.789 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:00:08.789 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:00:08.789 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:00:08.789 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:00:08.789 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:00:08.789 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:00:08.835 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:08.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:08.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:08.946 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:08.981 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:09.013 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total time run: 90.0317 2023-10-09T06:00:09.013 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total writes made: 105350 2023-10-09T06:00:09.013 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Write size: 65536 2023-10-09T06:00:09.014 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object size: 65536 2023-10-09T06:00:09.014 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Bandwidth (MB/sec): 73.1339 2023-10-09T06:00:09.014 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Bandwidth: 36.3693 2023-10-09T06:00:09.014 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max bandwidth (MB/sec): 213.25 2023-10-09T06:00:09.014 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min bandwidth (MB/sec): 41.8125 2023-10-09T06:00:09.014 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average IOPS: 1170 2023-10-09T06:00:09.014 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev IOPS: 581.909 2023-10-09T06:00:09.014 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max IOPS: 3412 2023-10-09T06:00:09.014 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min IOPS: 669 2023-10-09T06:00:09.014 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average Latency(s): 0.0136677 2023-10-09T06:00:09.014 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Latency(s): 0.0205915 2023-10-09T06:00:09.015 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max latency(s): 1.01073 2023-10-09T06:00:09.015 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min latency(s): 0.00127391 2023-10-09T06:00:09.017 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T06:00:09.042 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:09.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:09.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:09.154 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:09.194 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:09.267 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:09.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:09.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:09.375 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:09.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:09 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3635577263' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:09.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:09 smithi064 ceph-mon[212750]: pgmap v717: 119 pgs: 119 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 45 MiB/s wr, 724 op/s 2023-10-09T06:00:09.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:09 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1630690536' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:00:09.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:09 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:00:09.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:09 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3635577263' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:09.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:09 smithi064 ceph-mon[220044]: pgmap v717: 119 pgs: 119 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 45 MiB/s wr, 724 op/s 2023-10-09T06:00:09.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:09 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1630690536' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:00:09.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:09 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:00:09.419 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:09.467 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T06:00:09.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:09 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3635577263' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:09.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:09 smithi156 ceph-mon[195101]: pgmap v717: 119 pgs: 119 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 45 MiB/s wr, 724 op/s 2023-10-09T06:00:09.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:09 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1630690536' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:00:09.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:09 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:00:10.303 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:10.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:10.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:10.412 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:10.522 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:10.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:10 smithi064 ceph-mon[220044]: failed to encode map e310 with expected crc 2023-10-09T06:00:10.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:10 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:00:10.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:10 smithi064 ceph-mon[220044]: osdmap e310: 8 total, 8 up, 7 in 2023-10-09T06:00:10.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:10 smithi064 ceph-mon[220044]: failed to encode map e310 with expected crc 2023-10-09T06:00:10.694 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:10 smithi064 ceph-mon[212750]: failed to encode map e310 with expected crc 2023-10-09T06:00:10.694 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:10 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:00:10.694 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:10 smithi064 ceph-mon[212750]: osdmap e310: 8 total, 8 up, 7 in 2023-10-09T06:00:10.694 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:10 smithi064 ceph-mon[212750]: failed to encode map e310 with expected crc 2023-10-09T06:00:10.695 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:10.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:10.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:10.814 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:10.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:10 smithi156 ceph-mon[195101]: failed to encode map e310 with expected crc 2023-10-09T06:00:10.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:10 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:00:10.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:10 smithi156 ceph-mon[195101]: osdmap e310: 8 total, 8 up, 7 in 2023-10-09T06:00:10.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:10 smithi156 ceph-mon[195101]: failed to encode map e310 with expected crc 2023-10-09T06:00:10.882 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:11.334 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:11.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:11.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:11.444 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:11.457 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T06:00:11.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:11 smithi064 ceph-mon[212750]: pgmap v719: 119 pgs: 119 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 60 MiB/s wr, 1.02k op/s 2023-10-09T06:00:11.458 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:11 smithi064 ceph-mon[220044]: pgmap v719: 119 pgs: 119 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 60 MiB/s wr, 1.02k op/s 2023-10-09T06:00:11.458 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:11.538 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:11.675 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:11.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:11.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:11.799 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:11.838 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:11 smithi156 ceph-mon[195101]: pgmap v719: 119 pgs: 119 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 60 MiB/s wr, 1.02k op/s 2023-10-09T06:00:11.841 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:11.907 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:11.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:11.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:12.013 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:12.019 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:00:12.019 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T06:00:12.019 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T06:00:12.019 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T06:00:12.019 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:00:12.048 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:12.112 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:12.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:12.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:12.219 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:12.288 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:12.598 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:12.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:12.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:12.706 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:12.754 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:13.223 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:00:13.767 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:00:13.803 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:13.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:13.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:13.834 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:13.916 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:13.948 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:14.017 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:14.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:14.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:14.124 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:14.159 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:14.226 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:14.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:14.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:14.338 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:14.388 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:14.463 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:14 smithi064 ceph-mon[212750]: pgmap v720: 119 pgs: 119 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 50 MiB/s wr, 887 op/s 2023-10-09T06:00:14.463 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:00:14.463 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:14.464 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:14.464 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:14 smithi064 ceph-mon[220044]: pgmap v720: 119 pgs: 119 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 50 MiB/s wr, 887 op/s 2023-10-09T06:00:14.464 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:00:14.464 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:14.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:14.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:14.544 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:14.573 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:14.604 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:14 smithi156 ceph-mon[195101]: pgmap v720: 119 pgs: 119 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 50 MiB/s wr, 887 op/s 2023-10-09T06:00:14.605 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:00:14.605 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:14.607 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:14.676 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:14.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:14.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:14.792 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:14.828 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:14.901 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:14.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:14.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:15.008 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:15.084 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:15.313 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:15 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1239877339' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:00:15.313 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:15 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:00:15.340 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:00:15.374 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:15 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1239877339' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:00:15.374 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:15 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:00:15.375 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:00:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:00:15] "GET /metrics HTTP/1.1" 200 36380 "" "Prometheus/2.43.0" 2023-10-09T06:00:15.376 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:15 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1239877339' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:00:15.376 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:15 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:00:15.471 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:15.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:15.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:15.582 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:15.630 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:15.725 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:15.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:15.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:15.831 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:15.875 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:15.942 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:15.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:15.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:16.049 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:16.086 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:16.131 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:16.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:16.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:16.238 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:16.264 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:16.305 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:16.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:16.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:16.410 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:16.450 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:16.516 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:16.520 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:16 smithi156 ceph-mon[195101]: pgmap v721: 119 pgs: 119 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 42 MiB/s wr, 845 op/s 2023-10-09T06:00:16.521 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:16 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:00:16.521 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:16 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:00:16.521 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:16 smithi156 ceph-mon[195101]: osdmap e311: 8 total, 8 up, 7 in 2023-10-09T06:00:16.521 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:16 smithi156 ceph-mon[195101]: failed to encode map e311 with expected crc 2023-10-09T06:00:16.521 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:16 smithi156 ceph-mon[195101]: failed to encode map e311 with expected crc 2023-10-09T06:00:16.521 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:16 smithi156 ceph-mon[195101]: failed to encode map e311 with expected crc 2023-10-09T06:00:16.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:16.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:16.627 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:16.675 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:16 smithi064 ceph-mon[212750]: pgmap v721: 119 pgs: 119 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 42 MiB/s wr, 845 op/s 2023-10-09T06:00:16.675 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:16 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:00:16.675 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:16 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:00:16.675 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:16 smithi064 ceph-mon[212750]: osdmap e311: 8 total, 8 up, 7 in 2023-10-09T06:00:16.675 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:16 smithi064 ceph-mon[212750]: failed to encode map e311 with expected crc 2023-10-09T06:00:16.675 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:16 smithi064 ceph-mon[212750]: failed to encode map e311 with expected crc 2023-10-09T06:00:16.675 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:16 smithi064 ceph-mon[212750]: failed to encode map e311 with expected crc 2023-10-09T06:00:16.675 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:16 smithi064 ceph-mon[220044]: pgmap v721: 119 pgs: 119 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 42 MiB/s wr, 845 op/s 2023-10-09T06:00:16.676 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:16 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:00:16.676 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:16 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:00:16.676 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:16 smithi064 ceph-mon[220044]: osdmap e311: 8 total, 8 up, 7 in 2023-10-09T06:00:16.676 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:16 smithi064 ceph-mon[220044]: failed to encode map e311 with expected crc 2023-10-09T06:00:16.676 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:16 smithi064 ceph-mon[220044]: failed to encode map e311 with expected crc 2023-10-09T06:00:16.676 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:16 smithi064 ceph-mon[220044]: failed to encode map e311 with expected crc 2023-10-09T06:00:16.701 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:17.082 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:17.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:17.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:17.189 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:17.215 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:17.256 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:17.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:17.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:17.363 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:17.411 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:17.472 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:17.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:17.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:17.583 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:17.628 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:17.718 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:17.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:17.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:17.826 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:17.828 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:17.862 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:17.922 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:17.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:17.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:18.029 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:18.067 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:18.132 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:18.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:18.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:18.240 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:18.280 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:18.349 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T06:00:18.352 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:18.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:18.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:18.463 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:18.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:18 smithi064 ceph-mon[212750]: pgmap v723: 119 pgs: 119 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 20 MiB/s wr, 896 op/s 2023-10-09T06:00:18.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:18 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3614636589' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:00:18.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:18 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:00:18.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:18 smithi064 ceph-mon[220044]: pgmap v723: 119 pgs: 119 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 20 MiB/s wr, 896 op/s 2023-10-09T06:00:18.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:18 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3614636589' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:00:18.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:18 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:00:18.494 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:18 smithi156 ceph-mon[195101]: pgmap v723: 119 pgs: 119 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 20 MiB/s wr, 896 op/s 2023-10-09T06:00:18.495 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:18 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3614636589' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:00:18.495 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:18 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:00:18.496 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:18.559 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:18.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:18.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:18.568 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:00:18.610 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:00:18.651 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:00:18.668 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:18.708 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:00:18.740 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:18.760 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:00:18.794 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:00:18.825 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:00:18.847 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:00:18.855 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:00:18.855 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:00:18.855 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:00:18.855 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:00:18.856 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:00:18.856 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:00:18.856 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:00:18.856 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:00:18.879 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:00:19.557 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T06:00:19.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:19 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:00:19.615 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:19 smithi064 ceph-mon[220044]: osdmap e312: 8 total, 8 up, 7 in 2023-10-09T06:00:19.615 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:19 smithi064 ceph-mon[220044]: failed to encode map e312 with expected crc 2023-10-09T06:00:19.615 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:19 smithi064 ceph-mon[220044]: failed to encode map e312 with expected crc 2023-10-09T06:00:19.615 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:19 smithi064 ceph-mon[220044]: failed to encode map e312 with expected crc 2023-10-09T06:00:19.615 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:19 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4168999648' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:19.615 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:19 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:00:19.615 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:19 smithi064 ceph-mon[212750]: osdmap e312: 8 total, 8 up, 7 in 2023-10-09T06:00:19.616 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:19 smithi064 ceph-mon[212750]: failed to encode map e312 with expected crc 2023-10-09T06:00:19.616 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:19 smithi064 ceph-mon[212750]: failed to encode map e312 with expected crc 2023-10-09T06:00:19.616 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:19 smithi064 ceph-mon[212750]: failed to encode map e312 with expected crc 2023-10-09T06:00:19.616 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:19 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4168999648' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:19.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:19 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:00:19.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:19 smithi156 ceph-mon[195101]: osdmap e312: 8 total, 8 up, 7 in 2023-10-09T06:00:19.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:19 smithi156 ceph-mon[195101]: failed to encode map e312 with expected crc 2023-10-09T06:00:19.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:19 smithi156 ceph-mon[195101]: failed to encode map e312 with expected crc 2023-10-09T06:00:19.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:19 smithi156 ceph-mon[195101]: failed to encode map e312 with expected crc 2023-10-09T06:00:19.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:19 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4168999648' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:20.363 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:20 smithi064 ceph-mon[212750]: pgmap v725: 119 pgs: 119 active+clean; 6.1 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 127 B/s rd, 1.2 MiB/s wr, 700 op/s 2023-10-09T06:00:20.364 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:20 smithi064 ceph-mon[220044]: pgmap v725: 119 pgs: 119 active+clean; 6.1 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 127 B/s rd, 1.2 MiB/s wr, 700 op/s 2023-10-09T06:00:20.470 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:20.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:20.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:20.581 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:20.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:20 smithi156 ceph-mon[195101]: pgmap v725: 119 pgs: 119 active+clean; 6.1 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 127 B/s rd, 1.2 MiB/s wr, 700 op/s 2023-10-09T06:00:20.683 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:20.785 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:20.821 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:20.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:20.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:20.933 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:20.997 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:21.293 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:21.316 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:21.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:21.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:21.421 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:21.452 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:00:21.452 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:00:21.458 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:21.527 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:21.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:21.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:21.633 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:21.671 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:21 smithi156 ceph-mon[195101]: pgmap v726: 119 pgs: 119 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 127 B/s rd, 648 KiB/s wr, 1.04k op/s 2023-10-09T06:00:21.677 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:21.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:21 smithi064 ceph-mon[212750]: pgmap v726: 119 pgs: 119 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 127 B/s rd, 648 KiB/s wr, 1.04k op/s 2023-10-09T06:00:21.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:21 smithi064 ceph-mon[220044]: pgmap v726: 119 pgs: 119 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 127 B/s rd, 648 KiB/s wr, 1.04k op/s 2023-10-09T06:00:21.760 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:21.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:21.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:21.876 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:21.917 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:21.991 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:21.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:21.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:22.099 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:22.145 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:22.189 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:22.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:22.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:22.297 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:22.334 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:22.406 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:22.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:22.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:22.512 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:22.555 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:22.574 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:00:22.617 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:22.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:22.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:22.727 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:22.767 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:22.825 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:22.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:22.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:22.929 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:22.994 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:24.501 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:00:24.560 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:24 smithi064 ceph-mon[212750]: pgmap v727: 119 pgs: 119 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 939 op/s 2023-10-09T06:00:24.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:24 smithi064 ceph-mon[220044]: pgmap v727: 119 pgs: 119 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 939 op/s 2023-10-09T06:00:24.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:24 smithi156 ceph-mon[195101]: pgmap v727: 119 pgs: 119 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 939 op/s 2023-10-09T06:00:24.641 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:24.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:24.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:24.758 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:24.800 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:24.863 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:24.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:24.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:24.971 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:25.005 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:25.060 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:25.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:25.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:25.171 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:25.209 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:25.219 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:25.261 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:25.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:25.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:25.310 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:00:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:00:25] "GET /metrics HTTP/1.1" 200 36380 "" "Prometheus/2.43.0" 2023-10-09T06:00:25.369 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:25.404 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:25.469 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:25.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:25.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:25.502 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:25.580 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:25.669 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:25.940 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:25.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:25.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:26.058 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:26.144 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:26.334 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:00:26.338 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T06:00:26.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:26 smithi156 ceph-mon[195101]: pgmap v728: 119 pgs: 119 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 661 op/s 2023-10-09T06:00:26.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:26 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/894835902' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:26.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:26 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3598000867' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:00:26.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:26 smithi064 ceph-mon[212750]: pgmap v728: 119 pgs: 119 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 661 op/s 2023-10-09T06:00:26.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:26 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/894835902' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:26.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:26 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3598000867' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:00:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:26 smithi064 ceph-mon[220044]: pgmap v728: 119 pgs: 119 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 661 op/s 2023-10-09T06:00:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:26 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/894835902' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:26.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:26 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3598000867' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:00:27.267 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:27.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:27.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:27.374 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:27.404 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:27.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:27 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:00:27.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:27 smithi156 ceph-mon[195101]: failed to encode map e313 with expected crc 2023-10-09T06:00:27.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:27 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/894835902' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:00:27.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:27 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3598000867' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:00:27.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:27 smithi156 ceph-mon[195101]: osdmap e313: 8 total, 8 up, 7 in 2023-10-09T06:00:27.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:27 smithi156 ceph-mon[195101]: failed to encode map e313 with expected crc 2023-10-09T06:00:27.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:27 smithi156 ceph-mon[195101]: failed to encode map e313 with expected crc 2023-10-09T06:00:27.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:27 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:00:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:27 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:00:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:27 smithi064 ceph-mon[212750]: failed to encode map e313 with expected crc 2023-10-09T06:00:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:27 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/894835902' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:00:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:27 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3598000867' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:00:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:27 smithi064 ceph-mon[212750]: osdmap e313: 8 total, 8 up, 7 in 2023-10-09T06:00:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:27 smithi064 ceph-mon[212750]: failed to encode map e313 with expected crc 2023-10-09T06:00:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:27 smithi064 ceph-mon[212750]: failed to encode map e313 with expected crc 2023-10-09T06:00:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:27 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:00:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:27 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:00:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:27 smithi064 ceph-mon[220044]: failed to encode map e313 with expected crc 2023-10-09T06:00:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:27 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/894835902' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:00:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:27 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3598000867' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:00:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:27 smithi064 ceph-mon[220044]: osdmap e313: 8 total, 8 up, 7 in 2023-10-09T06:00:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:27 smithi064 ceph-mon[220044]: failed to encode map e313 with expected crc 2023-10-09T06:00:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:27 smithi064 ceph-mon[220044]: failed to encode map e313 with expected crc 2023-10-09T06:00:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:27 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:00:28.374 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:28.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:28.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:28.483 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:28.571 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:28 smithi064 ceph-mon[212750]: pgmap v730: 119 pgs: 119 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 990 op/s 2023-10-09T06:00:28.577 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:28.578 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:28 smithi064 ceph-mon[220044]: pgmap v730: 119 pgs: 119 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 990 op/s 2023-10-09T06:00:28.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:28 smithi156 ceph-mon[195101]: pgmap v730: 119 pgs: 119 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 990 op/s 2023-10-09T06:00:29.479 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:29.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:29.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:29.503 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T06:00:29.595 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:29.633 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:29.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:29 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:00:29.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:29.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:29 smithi064 ceph-mon[212750]: pgmap v731: 119 pgs: 119 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 843 op/s 2023-10-09T06:00:29.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:29 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:00:29.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:29.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:29 smithi064 ceph-mon[220044]: pgmap v731: 119 pgs: 119 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 843 op/s 2023-10-09T06:00:29.726 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:29.741 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:29 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:00:29.741 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:29.741 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:29 smithi156 ceph-mon[195101]: pgmap v731: 119 pgs: 119 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 843 op/s 2023-10-09T06:00:29.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:29.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:29.843 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:29.970 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:30.143 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:30.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:30.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:30.253 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:30.274 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:00:30.274 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:00:30.289 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:00:30.291 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:30.329 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:00:30.362 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:30.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:30.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:30.398 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:00:30.446 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:00:30.463 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:30.482 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:00:30.516 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:30.543 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:00:30.578 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:00:30.614 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:00:30.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:30 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1896738618' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:30.625 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:00:30.626 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:00:30.626 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:00:30.626 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:00:30.626 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:00:30.626 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:00:30.626 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:00:30.626 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:00:30.746 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:30 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1896738618' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:30.746 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:30 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1896738618' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:31.319 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:31.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:31.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:31.424 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:31.446 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:00:31.459 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:31.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:31 smithi156 ceph-mon[195101]: pgmap v732: 119 pgs: 119 active+clean; 5.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 834 op/s 2023-10-09T06:00:31.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:31 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:00:31.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:31 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:00:31.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:31 smithi064 ceph-mon[212750]: pgmap v732: 119 pgs: 119 active+clean; 5.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 834 op/s 2023-10-09T06:00:31.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:31 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:00:31.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:31 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:00:31.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:31 smithi064 ceph-mon[220044]: pgmap v732: 119 pgs: 119 active+clean; 5.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 834 op/s 2023-10-09T06:00:31.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:31 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:00:31.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:31 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:00:32.002 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:32.008 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:00:32.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:32.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:32.116 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:32.148 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:32.216 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:32.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:32.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:32.323 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:32.352 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:32.715 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:32.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:32.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:32.800 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:32.833 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:32.978 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:33.054 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:33.188 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:33.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:33.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:33.296 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:33.338 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:33.411 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:33.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:33.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:33.497 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:33.522 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:33.620 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:33.883 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:33.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:33.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:33.990 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:34.019 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:34.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:34 smithi064 ceph-mon[212750]: pgmap v733: 119 pgs: 119 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 834 op/s 2023-10-09T06:00:34.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:34 smithi064 ceph-mon[220044]: pgmap v733: 119 pgs: 119 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 834 op/s 2023-10-09T06:00:34.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:34 smithi156 ceph-mon[195101]: pgmap v733: 119 pgs: 119 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 834 op/s 2023-10-09T06:00:34.737 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:34.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:34.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:34.835 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:00:34.846 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:34.905 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:35.452 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:00:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:00:35] "GET /metrics HTTP/1.1" 200 36386 "" "Prometheus/2.43.0" 2023-10-09T06:00:35.802 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:35.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:35.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:35.908 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:35.949 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:36.381 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:36.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:36.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:36.495 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:36.539 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:36.617 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:36.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:36 smithi156 ceph-mon[195101]: pgmap v734: 119 pgs: 119 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 828 op/s 2023-10-09T06:00:36.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:36.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:36.629 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:36 smithi064 ceph-mon[212750]: pgmap v734: 119 pgs: 119 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 828 op/s 2023-10-09T06:00:36.630 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:36 smithi064 ceph-mon[220044]: pgmap v734: 119 pgs: 119 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 828 op/s 2023-10-09T06:00:36.723 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:36.817 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:37.099 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:37.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:37.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:37.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:37.304 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:37.338 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:00:37.338 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T06:00:37.338 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T06:00:37.338 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T06:00:37.338 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:00:37.643 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:38.332 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T06:00:38.332 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:00:38.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:38 smithi156 ceph-mon[195101]: pgmap v735: 119 pgs: 119 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 782 op/s 2023-10-09T06:00:38.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:38 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1278051578' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:00:38.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:38 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:00:38.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:38 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1051524806' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:00:38.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:38 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:00:38.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:38 smithi064 ceph-mon[220044]: pgmap v735: 119 pgs: 119 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 782 op/s 2023-10-09T06:00:38.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:38 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1278051578' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:00:38.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:38 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:00:38.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:38 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1051524806' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:00:38.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:38 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:00:38.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:38 smithi064 ceph-mon[212750]: pgmap v735: 119 pgs: 119 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 782 op/s 2023-10-09T06:00:38.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:38 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1278051578' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:00:38.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:38 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:00:38.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:38 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1051524806' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:00:38.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:38 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:00:39.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:39 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:00:39.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:39 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:00:39.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:39 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:00:39.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:39 smithi156 ceph-mon[195101]: osdmap e314: 8 total, 8 up, 7 in 2023-10-09T06:00:39.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:39 smithi156 ceph-mon[195101]: failed to encode map e314 with expected crc 2023-10-09T06:00:39.701 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:39 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:00:39.702 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:39 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:00:39.702 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:39 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:00:39.702 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:39 smithi064 ceph-mon[220044]: osdmap e314: 8 total, 8 up, 7 in 2023-10-09T06:00:39.702 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:39 smithi064 ceph-mon[220044]: failed to encode map e314 with expected crc 2023-10-09T06:00:39.704 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:39 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:00:39.704 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:39 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:00:39.705 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:39 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:00:39.705 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:39 smithi064 ceph-mon[212750]: osdmap e314: 8 total, 8 up, 7 in 2023-10-09T06:00:39.705 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:39 smithi064 ceph-mon[212750]: failed to encode map e314 with expected crc 2023-10-09T06:00:40.028 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:40.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:40.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:40.136 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:40.162 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T06:00:40.162 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:00:40.213 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:00:40.232 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:40.502 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:40.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:40.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:40.569 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:40 smithi064 ceph-mon[220044]: pgmap v737: 119 pgs: 119 active+clean; 5.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 818 op/s 2023-10-09T06:00:40.571 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:40 smithi064 ceph-mon[212750]: pgmap v737: 119 pgs: 119 active+clean; 5.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 818 op/s 2023-10-09T06:00:40.610 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:40.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:40 smithi156 ceph-mon[195101]: pgmap v737: 119 pgs: 119 active+clean; 5.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 818 op/s 2023-10-09T06:00:40.646 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:40.714 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:40.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:40.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:40.825 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:40.927 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:41.137 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:41.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:41.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:41.244 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:41.293 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:41.379 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:41.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:41.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:41.489 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:41.514 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:41.556 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:41.559 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:41 smithi156 ceph-mon[195101]: pgmap v738: 119 pgs: 119 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 815 op/s 2023-10-09T06:00:41.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:41.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:41.661 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:41.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:41 smithi064 ceph-mon[212750]: pgmap v738: 119 pgs: 119 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 815 op/s 2023-10-09T06:00:41.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:41 smithi064 ceph-mon[220044]: pgmap v738: 119 pgs: 119 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 815 op/s 2023-10-09T06:00:41.744 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:42.134 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:42.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:42.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:42.242 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:42.310 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:42.344 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:42.379 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:42.489 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:42.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:42.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:42.514 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:00:42.598 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:42.637 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:42.690 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:42.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:42.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:42.797 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:42.853 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:42.918 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:42.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:42.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:43.025 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:43.146 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:43.171 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:00:43.227 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:00:43.255 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:00:43.312 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:00:43.328 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:43.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:43.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:43.364 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:00:43.406 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:00:43.441 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:43.442 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:00:43.471 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:43.499 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:00:43.515 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:00:43.515 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:00:43.515 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:00:43.516 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:00:43.516 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:00:43.516 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:00:43.516 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:00:43.516 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:00:43.539 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:43.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:43.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:43.607 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:43 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3899708968' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:43.607 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:43 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3899708968' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:43.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:43 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3899708968' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:43.647 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:43.739 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:44.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:44 smithi156 ceph-mon[195101]: pgmap v739: 119 pgs: 119 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 814 op/s 2023-10-09T06:00:44.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:44.644 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:44 smithi064 ceph-mon[220044]: pgmap v739: 119 pgs: 119 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 814 op/s 2023-10-09T06:00:44.644 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:44.645 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:44 smithi064 ceph-mon[212750]: pgmap v739: 119 pgs: 119 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 814 op/s 2023-10-09T06:00:44.645 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:45.397 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:45.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:45.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:45.510 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:45.546 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:45.599 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:45.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:45.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:45.704 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:45.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:00:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:00:45] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.43.0" 2023-10-09T06:00:45.781 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:46.065 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:46.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:46.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:46.175 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:46.254 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:46.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:46 smithi156 ceph-mon[195101]: pgmap v740: 119 pgs: 119 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 816 op/s 2023-10-09T06:00:46.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:46 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:00:46.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:46 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:00:46.426 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:46 smithi064 ceph-mon[212750]: pgmap v740: 119 pgs: 119 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 816 op/s 2023-10-09T06:00:46.426 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:46 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:00:46.426 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:46 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:00:46.426 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:46 smithi064 ceph-mon[220044]: pgmap v740: 119 pgs: 119 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 816 op/s 2023-10-09T06:00:46.426 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:46 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:00:46.426 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:46 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:00:46.583 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:46.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:46.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:46.694 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:46.737 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:46.800 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:46.809 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:46.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:46.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:46.916 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:46.918 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:00:46.919 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:00:46.951 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:47.026 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:47.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:47.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:47.136 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:47.180 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:00:47.207 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T06:00:47.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:47 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:00:47.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:47 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:00:47.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:47 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:00:47.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:47 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:00:47.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:47 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:00:47.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:47 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:00:47.241 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:47 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:00:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:47 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:00:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:47 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:00:48.290 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:00:48.494 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:48.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:48.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:48.608 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:48.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:48 smithi156 ceph-mon[195101]: pgmap v741: 119 pgs: 119 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 837 op/s 2023-10-09T06:00:48.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:48 smithi156 ceph-mon[195101]: from='client.45410 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:00:48.654 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:48 smithi064 ceph-mon[212750]: pgmap v741: 119 pgs: 119 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 837 op/s 2023-10-09T06:00:48.654 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:48 smithi064 ceph-mon[212750]: from='client.45410 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:00:48.654 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:48 smithi064 ceph-mon[220044]: pgmap v741: 119 pgs: 119 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 837 op/s 2023-10-09T06:00:48.655 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:48 smithi064 ceph-mon[220044]: from='client.45410 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:00:48.705 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:49.072 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":741,"stamp":"2023-10-09T06:00:46.722750+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4954920888,"num_objects":75622,"num_object_clones":0,"num_object_copies":226866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":75622,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":135165,"num_write_kb":6742430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":10792,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95045,"ondisk_log_size":95045,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16927488,"kb_used_data":14635464,"kb_used_omap":118,"kb_used_meta":2290953,"kb_avail":639288576,"statfs":{"total":671965249536,"available":654631501824,"internally_reserved":0,"allocated":14986715136,"data_stored":14977365939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":121648,"internal_metadata":2345936080},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,6,2,0,0,3,13],"upper_bound":512},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-549650432,"num_objects":-8387,"num_object_clones":0,"num_object_copies":-25161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8387,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.010359"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":544,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.325268+0000","last_change":"2023-10-09T06:00:05.413041+0000","last_active":"2023-10-09T06:00:26.325268+0000","last_peered":"2023-10-09T06:00:26.325268+0000","last_clean":"2023-10-09T06:00:26.325268+0000","last_became_active":"2023-10-09T05:59:40.308608+0000","last_became_peered":"2023-10-09T05:59:40.308608+0000","last_unstale":"2023-10-09T06:00:26.325268+0000","last_undegraded":"2023-10-09T06:00:26.325268+0000","last_fullsized":"2023-10-09T06:00:26.325268+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:05.412941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:05.412941+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:53.518055+0000","scrub_duration":0.0061014329999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":359,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320534+0000","last_change":"2023-10-09T05:55:59.341215+0000","last_active":"2023-10-09T06:00:26.320534+0000","last_peered":"2023-10-09T06:00:26.320534+0000","last_clean":"2023-10-09T06:00:26.320534+0000","last_became_active":"2023-10-09T05:55:59.340856+0000","last_became_peered":"2023-10-09T05:55:59.340856+0000","last_unstale":"2023-10-09T06:00:26.320534+0000","last_undegraded":"2023-10-09T06:00:26.320534+0000","last_fullsized":"2023-10-09T06:00:26.320534+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:33.686454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":77,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.319377+0000","last_change":"2023-10-09T05:57:23.474348+0000","last_active":"2023-10-09T06:00:26.319377+0000","last_peered":"2023-10-09T06:00:26.319377+0000","last_clean":"2023-10-09T06:00:26.319377+0000","last_became_active":"2023-10-09T05:57:23.474010+0000","last_became_peered":"2023-10-09T05:57:23.474010+0000","last_unstale":"2023-10-09T06:00:26.319377+0000","last_undegraded":"2023-10-09T06:00:26.319377+0000","last_fullsized":"2023-10-09T06:00:26.319377+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:45.039777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":590,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320426+0000","last_change":"2023-10-09T05:59:59.067739+0000","last_active":"2023-10-09T06:00:26.320426+0000","last_peered":"2023-10-09T06:00:26.320426+0000","last_clean":"2023-10-09T06:00:26.320426+0000","last_became_active":"2023-10-09T05:59:40.315955+0000","last_became_peered":"2023-10-09T05:59:40.315955+0000","last_unstale":"2023-10-09T06:00:26.320426+0000","last_undegraded":"2023-10-09T06:00:26.320426+0000","last_fullsized":"2023-10-09T06:00:26.320426+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:59.067648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:59.067648+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:46:50.528412+0000","scrub_duration":0.0016747089999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":194,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.319310+0000","last_change":"2023-10-09T05:55:59.304893+0000","last_active":"2023-10-09T06:00:26.319310+0000","last_peered":"2023-10-09T06:00:26.319310+0000","last_clean":"2023-10-09T06:00:26.319310+0000","last_became_active":"2023-10-09T05:55:59.304449+0000","last_became_peered":"2023-10-09T05:55:59.304449+0000","last_unstale":"2023-10-09T06:00:26.319310+0000","last_undegraded":"2023-10-09T06:00:26.319310+0000","last_fullsized":"2023-10-09T06:00:26.319310+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:44:39.453982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":335,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.325546+0000","last_change":"2023-10-09T05:57:58.834014+0000","last_active":"2023-10-09T06:00:26.325546+0000","last_peered":"2023-10-09T06:00:26.325546+0000","last_clean":"2023-10-09T06:00:26.325546+0000","last_became_active":"2023-10-09T05:57:58.833718+0000","last_became_peered":"2023-10-09T05:57:58.833718+0000","last_unstale":"2023-10-09T06:00:26.325546+0000","last_undegraded":"2023-10-09T06:00:26.325546+0000","last_fullsized":"2023-10-09T06:00:26.325546+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:23.752353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":523,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320425+0000","last_change":"2023-10-09T05:59:58.065729+0000","last_active":"2023-10-09T06:00:26.320425+0000","last_peered":"2023-10-09T06:00:26.320425+0000","last_clean":"2023-10-09T06:00:26.320425+0000","last_became_active":"2023-10-09T05:59:40.313876+0000","last_became_peered":"2023-10-09T05:59:40.313876+0000","last_unstale":"2023-10-09T06:00:26.320425+0000","last_undegraded":"2023-10-09T06:00:26.320425+0000","last_fullsized":"2023-10-09T06:00:26.320425+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.065661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.065661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:23:10.873130+0000","scrub_duration":0.0056656750000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":77,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.315511+0000","last_change":"2023-10-09T05:57:23.476832+0000","last_active":"2023-10-09T06:00:26.315511+0000","last_peered":"2023-10-09T06:00:26.315511+0000","last_clean":"2023-10-09T06:00:26.315511+0000","last_became_active":"2023-10-09T05:57:23.475216+0000","last_became_peered":"2023-10-09T05:57:23.475216+0000","last_unstale":"2023-10-09T06:00:26.315511+0000","last_undegraded":"2023-10-09T06:00:26.315511+0000","last_fullsized":"2023-10-09T06:00:26.315511+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:48.279660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":340,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.318898+0000","last_change":"2023-10-09T05:55:59.335464+0000","last_active":"2023-10-09T06:00:26.318898+0000","last_peered":"2023-10-09T06:00:26.318898+0000","last_clean":"2023-10-09T06:00:26.318898+0000","last_became_active":"2023-10-09T05:55:59.335259+0000","last_became_peered":"2023-10-09T05:55:59.335259+0000","last_unstale":"2023-10-09T06:00:26.318898+0000","last_undegraded":"2023-10-09T06:00:26.318898+0000","last_fullsized":"2023-10-09T06:00:26.318898+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:28.536682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":613,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.328226+0000","last_change":"2023-10-09T06:00:06.384509+0000","last_active":"2023-10-09T06:00:26.328226+0000","last_peered":"2023-10-09T06:00:26.328226+0000","last_clean":"2023-10-09T06:00:26.328226+0000","last_became_active":"2023-10-09T05:59:40.322690+0000","last_became_peered":"2023-10-09T05:59:40.322690+0000","last_unstale":"2023-10-09T06:00:26.328226+0000","last_undegraded":"2023-10-09T06:00:26.328226+0000","last_fullsized":"2023-10-09T06:00:26.328226+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.384407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.384407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:37.280271+0000","scrub_duration":0.0040651960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":270,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.324874+0000","last_change":"2023-10-09T05:57:23.477980+0000","last_active":"2023-10-09T06:00:26.324874+0000","last_peered":"2023-10-09T06:00:26.324874+0000","last_clean":"2023-10-09T06:00:26.324874+0000","last_became_active":"2023-10-09T05:57:23.477654+0000","last_became_peered":"2023-10-09T05:57:23.477654+0000","last_unstale":"2023-10-09T06:00:26.324874+0000","last_undegraded":"2023-10-09T06:00:26.324874+0000","last_fullsized":"2023-10-09T06:00:26.324874+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:17.524683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":77,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.315586+0000","last_change":"2023-10-09T05:57:23.473644+0000","last_active":"2023-10-09T06:00:26.315586+0000","last_peered":"2023-10-09T06:00:26.315586+0000","last_clean":"2023-10-09T06:00:26.315586+0000","last_became_active":"2023-10-09T05:57:23.473391+0000","last_became_peered":"2023-10-09T05:57:23.473391+0000","last_unstale":"2023-10-09T06:00:26.315586+0000","last_undegraded":"2023-10-09T06:00:26.315586+0000","last_fullsized":"2023-10-09T06:00:26.315586+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:17.138034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":579,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:37.563140+0000","last_change":"2023-10-09T05:58:36.913116+0000","last_active":"2023-10-09T06:00:37.563140+0000","last_peered":"2023-10-09T06:00:37.563140+0000","last_clean":"2023-10-09T06:00:37.563140+0000","last_became_active":"2023-10-09T05:58:21.329980+0000","last_became_peered":"2023-10-09T05:58:21.329980+0000","last_unstale":"2023-10-09T06:00:37.563140+0000","last_undegraded":"2023-10-09T06:00:37.563140+0000","last_fullsized":"2023-10-09T06:00:37.563140+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.913004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.913004+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:32:18.561414+0000","scrub_duration":0.0017791980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":353,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.325958+0000","last_change":"2023-10-09T05:57:23.478483+0000","last_active":"2023-10-09T06:00:26.325958+0000","last_peered":"2023-10-09T06:00:26.325958+0000","last_clean":"2023-10-09T06:00:26.325958+0000","last_became_active":"2023-10-09T05:57:23.478122+0000","last_became_peered":"2023-10-09T05:57:23.478122+0000","last_unstale":"2023-10-09T06:00:26.325958+0000","last_undegraded":"2023-10-09T06:00:26.325958+0000","last_fullsized":"2023-10-09T06:00:26.325958+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:37.768717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":258,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.319224+0000","last_change":"2023-10-09T05:53:59.185702+0000","last_active":"2023-10-09T06:00:26.319224+0000","last_peered":"2023-10-09T06:00:26.319224+0000","last_clean":"2023-10-09T06:00:26.319224+0000","last_became_active":"2023-10-09T05:53:59.185373+0000","last_became_peered":"2023-10-09T05:53:59.185373+0000","last_unstale":"2023-10-09T06:00:26.319224+0000","last_undegraded":"2023-10-09T06:00:26.319224+0000","last_fullsized":"2023-10-09T06:00:26.319224+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:50.756967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":662,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.321921+0000","last_change":"2023-10-09T06:00:02.265614+0000","last_active":"2023-10-09T06:00:26.321921+0000","last_peered":"2023-10-09T06:00:26.321921+0000","last_clean":"2023-10-09T06:00:26.321921+0000","last_became_active":"2023-10-09T05:59:40.316490+0000","last_became_peered":"2023-10-09T05:59:40.316490+0000","last_unstale":"2023-10-09T06:00:26.321921+0000","last_undegraded":"2023-10-09T06:00:26.321921+0000","last_fullsized":"2023-10-09T06:00:26.321921+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:02.265460+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:02.265460+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:00:52.028473+0000","scrub_duration":0.010116385,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":154,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.328483+0000","last_change":"2023-10-09T05:53:59.219168+0000","last_active":"2023-10-09T06:00:26.328483+0000","last_peered":"2023-10-09T06:00:26.328483+0000","last_clean":"2023-10-09T06:00:26.328483+0000","last_became_active":"2023-10-09T05:53:59.200886+0000","last_became_peered":"2023-10-09T05:53:59.200886+0000","last_unstale":"2023-10-09T06:00:26.328483+0000","last_undegraded":"2023-10-09T06:00:26.328483+0000","last_fullsized":"2023-10-09T06:00:26.328483+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:45.515036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":208,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.318712+0000","last_change":"2023-10-09T05:55:59.305100+0000","last_active":"2023-10-09T06:00:26.318712+0000","last_peered":"2023-10-09T06:00:26.318712+0000","last_clean":"2023-10-09T06:00:26.318712+0000","last_became_active":"2023-10-09T05:55:59.304758+0000","last_became_peered":"2023-10-09T05:55:59.304758+0000","last_unstale":"2023-10-09T06:00:26.318712+0000","last_undegraded":"2023-10-09T06:00:26.318712+0000","last_fullsized":"2023-10-09T06:00:26.318712+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:54:32.510945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":447,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.325225+0000","last_change":"2023-10-09T05:59:57.339284+0000","last_active":"2023-10-09T06:00:26.325225+0000","last_peered":"2023-10-09T06:00:26.325225+0000","last_clean":"2023-10-09T06:00:26.325225+0000","last_became_active":"2023-10-09T05:59:40.334085+0000","last_became_peered":"2023-10-09T05:59:40.334085+0000","last_unstale":"2023-10-09T06:00:26.325225+0000","last_undegraded":"2023-10-09T06:00:26.325225+0000","last_fullsized":"2023-10-09T06:00:26.325225+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.339121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.339121+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:07.972002+0000","scrub_duration":0.010474344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":319,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320568+0000","last_change":"2023-10-09T05:53:59.219141+0000","last_active":"2023-10-09T06:00:26.320568+0000","last_peered":"2023-10-09T06:00:26.320568+0000","last_clean":"2023-10-09T06:00:26.320568+0000","last_became_active":"2023-10-09T05:53:59.197318+0000","last_became_peered":"2023-10-09T05:53:59.197318+0000","last_unstale":"2023-10-09T06:00:26.320568+0000","last_undegraded":"2023-10-09T06:00:26.320568+0000","last_fullsized":"2023-10-09T06:00:26.320568+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:39.910794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":323,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320746+0000","last_change":"2023-10-09T05:55:59.335824+0000","last_active":"2023-10-09T06:00:26.320746+0000","last_peered":"2023-10-09T06:00:26.320746+0000","last_clean":"2023-10-09T06:00:26.320746+0000","last_became_active":"2023-10-09T05:55:59.335478+0000","last_became_peered":"2023-10-09T05:55:59.335478+0000","last_unstale":"2023-10-09T06:00:26.320746+0000","last_undegraded":"2023-10-09T06:00:26.320746+0000","last_fullsized":"2023-10-09T06:00:26.320746+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:26.014501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":534,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.321589+0000","last_change":"2023-10-09T05:58:38.230201+0000","last_active":"2023-10-09T06:00:26.321589+0000","last_peered":"2023-10-09T06:00:26.321589+0000","last_clean":"2023-10-09T06:00:26.321589+0000","last_became_active":"2023-10-09T05:58:21.332729+0000","last_became_peered":"2023-10-09T05:58:21.332729+0000","last_unstale":"2023-10-09T06:00:26.321589+0000","last_undegraded":"2023-10-09T06:00:26.321589+0000","last_fullsized":"2023-10-09T06:00:26.321589+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:38.230079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:38.230079+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:33.800036+0000","scrub_duration":0.001618488,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":156,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.325717+0000","last_change":"2023-10-09T05:53:59.233898+0000","last_active":"2023-10-09T06:00:26.325717+0000","last_peered":"2023-10-09T06:00:26.325717+0000","last_clean":"2023-10-09T06:00:26.325717+0000","last_became_active":"2023-10-09T05:53:59.204175+0000","last_became_peered":"2023-10-09T05:53:59.204175+0000","last_unstale":"2023-10-09T06:00:26.325717+0000","last_undegraded":"2023-10-09T06:00:26.325717+0000","last_fullsized":"2023-10-09T06:00:26.325717+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:03.139318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":344,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.319470+0000","last_change":"2023-10-09T05:55:59.297087+0000","last_active":"2023-10-09T06:00:26.319470+0000","last_peered":"2023-10-09T06:00:26.319470+0000","last_clean":"2023-10-09T06:00:26.319470+0000","last_became_active":"2023-10-09T05:55:59.296768+0000","last_became_peered":"2023-10-09T05:55:59.296768+0000","last_unstale":"2023-10-09T06:00:26.319470+0000","last_undegraded":"2023-10-09T06:00:26.319470+0000","last_fullsized":"2023-10-09T06:00:26.319470+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:35.067705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":737,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.325706+0000","last_change":"2023-10-09T05:59:15.170543+0000","last_active":"2023-10-09T06:00:26.325706+0000","last_peered":"2023-10-09T06:00:26.325706+0000","last_clean":"2023-10-09T06:00:26.325706+0000","last_became_active":"2023-10-09T05:58:21.332533+0000","last_became_peered":"2023-10-09T05:58:21.332533+0000","last_unstale":"2023-10-09T06:00:26.325706+0000","last_undegraded":"2023-10-09T06:00:26.325706+0000","last_fullsized":"2023-10-09T06:00:26.325706+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:33.995695+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":316,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.321864+0000","last_change":"2023-10-09T05:57:23.473445+0000","last_active":"2023-10-09T06:00:26.321864+0000","last_peered":"2023-10-09T06:00:26.321864+0000","last_clean":"2023-10-09T06:00:26.321864+0000","last_became_active":"2023-10-09T05:57:23.473103+0000","last_became_peered":"2023-10-09T05:57:23.473103+0000","last_unstale":"2023-10-09T06:00:26.321864+0000","last_undegraded":"2023-10-09T06:00:26.321864+0000","last_fullsized":"2023-10-09T06:00:26.321864+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:39:53.825734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":237,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.328395+0000","last_change":"2023-10-09T05:56:49.237784+0000","last_active":"2023-10-09T06:00:26.328395+0000","last_peered":"2023-10-09T06:00:26.328395+0000","last_clean":"2023-10-09T06:00:26.328395+0000","last_became_active":"2023-10-09T05:56:49.237417+0000","last_became_peered":"2023-10-09T05:56:49.237417+0000","last_unstale":"2023-10-09T06:00:26.328395+0000","last_undegraded":"2023-10-09T06:00:26.328395+0000","last_fullsized":"2023-10-09T06:00:26.328395+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:33.129268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":565,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.338343+0000","last_change":"2023-10-09T05:58:37.295185+0000","last_active":"2023-10-09T06:00:38.338343+0000","last_peered":"2023-10-09T06:00:38.338343+0000","last_clean":"2023-10-09T06:00:38.338343+0000","last_became_active":"2023-10-09T05:58:21.327911+0000","last_became_peered":"2023-10-09T05:58:21.327911+0000","last_unstale":"2023-10-09T06:00:38.338343+0000","last_undegraded":"2023-10-09T06:00:38.338343+0000","last_fullsized":"2023-10-09T06:00:38.338343+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:37.295094+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:37.295094+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:10.882014+0000","scrub_duration":0.0017309510000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":150,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.327243+0000","last_change":"2023-10-09T05:53:59.206887+0000","last_active":"2023-10-09T06:00:26.327243+0000","last_peered":"2023-10-09T06:00:26.327243+0000","last_clean":"2023-10-09T06:00:26.327243+0000","last_became_active":"2023-10-09T05:53:59.206608+0000","last_became_peered":"2023-10-09T05:53:59.206608+0000","last_unstale":"2023-10-09T06:00:26.327243+0000","last_undegraded":"2023-10-09T06:00:26.327243+0000","last_fullsized":"2023-10-09T06:00:26.327243+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:48.975792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":297,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.325745+0000","last_change":"2023-10-09T05:57:58.828837+0000","last_active":"2023-10-09T06:00:26.325745+0000","last_peered":"2023-10-09T06:00:26.325745+0000","last_clean":"2023-10-09T06:00:26.325745+0000","last_became_active":"2023-10-09T05:57:58.828503+0000","last_became_peered":"2023-10-09T05:57:58.828503+0000","last_unstale":"2023-10-09T06:00:26.325745+0000","last_undegraded":"2023-10-09T06:00:26.325745+0000","last_fullsized":"2023-10-09T06:00:26.325745+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:08.332644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":614,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.318456+0000","last_change":"2023-10-09T05:58:46.725484+0000","last_active":"2023-10-09T06:00:26.318456+0000","last_peered":"2023-10-09T06:00:26.318456+0000","last_clean":"2023-10-09T06:00:26.318456+0000","last_became_active":"2023-10-09T05:58:21.332713+0000","last_became_peered":"2023-10-09T05:58:21.332713+0000","last_unstale":"2023-10-09T06:00:26.318456+0000","last_undegraded":"2023-10-09T06:00:26.318456+0000","last_fullsized":"2023-10-09T06:00:26.318456+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:46.725374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:46.725374+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:34.070739+0000","scrub_duration":0.002074771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":299,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.318353+0000","last_change":"2023-10-09T05:51:57.091639+0000","last_active":"2023-10-09T06:00:26.318353+0000","last_peered":"2023-10-09T06:00:26.318353+0000","last_clean":"2023-10-09T06:00:26.318353+0000","last_became_active":"2023-10-09T05:51:57.091284+0000","last_became_peered":"2023-10-09T05:51:57.091284+0000","last_unstale":"2023-10-09T06:00:26.318353+0000","last_undegraded":"2023-10-09T06:00:26.318353+0000","last_fullsized":"2023-10-09T06:00:26.318353+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:10.553453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":86,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.319481+0000","last_change":"2023-10-09T05:57:58.833939+0000","last_active":"2023-10-09T06:00:26.319481+0000","last_peered":"2023-10-09T06:00:26.319481+0000","last_clean":"2023-10-09T06:00:26.319481+0000","last_became_active":"2023-10-09T05:57:58.833567+0000","last_became_peered":"2023-10-09T05:57:58.833567+0000","last_unstale":"2023-10-09T06:00:26.319481+0000","last_undegraded":"2023-10-09T06:00:26.319481+0000","last_fullsized":"2023-10-09T06:00:26.319481+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:58.347572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"314'8445","reported_seq":8482,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:43.582208+0000","last_change":"2023-10-09T05:58:27.405856+0000","last_active":"2023-10-09T06:00:43.582208+0000","last_peered":"2023-10-09T06:00:43.582208+0000","last_clean":"2023-10-09T06:00:43.582208+0000","last_became_active":"2023-10-09T05:58:27.405560+0000","last_became_peered":"2023-10-09T05:58:27.405560+0000","last_unstale":"2023-10-09T06:00:43.582208+0000","last_undegraded":"2023-10-09T06:00:43.582208+0000","last_fullsized":"2023-10-09T06:00:43.582208+0000","mapping_epoch":285,"log_start":"294'2600","ondisk_log_start":"294'2600","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5845,"log_dups_size":0,"ondisk_log_size":5845,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:24.937739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316604416,"num_objects":4831,"num_object_clones":0,"num_object_copies":14493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8445,"num_write_kb":424832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":653,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.328306+0000","last_change":"2023-10-09T05:59:53.469494+0000","last_active":"2023-10-09T06:00:26.328306+0000","last_peered":"2023-10-09T06:00:26.328306+0000","last_clean":"2023-10-09T06:00:26.328306+0000","last_became_active":"2023-10-09T05:59:32.542653+0000","last_became_peered":"2023-10-09T05:59:32.542653+0000","last_unstale":"2023-10-09T06:00:26.328306+0000","last_undegraded":"2023-10-09T06:00:26.328306+0000","last_fullsized":"2023-10-09T06:00:26.328306+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:53.469394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:53.469394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:32.901454+0000","scrub_duration":0.0059716659999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":154,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.326131+0000","last_change":"2023-10-09T05:53:59.222136+0000","last_active":"2023-10-09T06:00:26.326131+0000","last_peered":"2023-10-09T06:00:26.326131+0000","last_clean":"2023-10-09T06:00:26.326131+0000","last_became_active":"2023-10-09T05:53:59.201853+0000","last_became_peered":"2023-10-09T05:53:59.201853+0000","last_unstale":"2023-10-09T06:00:26.326131+0000","last_undegraded":"2023-10-09T06:00:26.326131+0000","last_fullsized":"2023-10-09T06:00:26.326131+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:26.662031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":220,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.325502+0000","last_change":"2023-10-09T05:56:49.234185+0000","last_active":"2023-10-09T06:00:26.325502+0000","last_peered":"2023-10-09T06:00:26.325502+0000","last_clean":"2023-10-09T06:00:26.325502+0000","last_became_active":"2023-10-09T05:56:49.233884+0000","last_became_peered":"2023-10-09T05:56:49.233884+0000","last_unstale":"2023-10-09T06:00:26.325502+0000","last_undegraded":"2023-10-09T06:00:26.325502+0000","last_fullsized":"2023-10-09T06:00:26.325502+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:12.142436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.a","version":"314'8523","reported_seq":8562,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:44.934084+0000","last_change":"2023-10-09T05:58:27.406918+0000","last_active":"2023-10-09T06:00:44.934084+0000","last_peered":"2023-10-09T06:00:44.934084+0000","last_clean":"2023-10-09T06:00:44.934084+0000","last_became_active":"2023-10-09T05:58:27.406600+0000","last_became_peered":"2023-10-09T05:58:27.406600+0000","last_unstale":"2023-10-09T06:00:44.934084+0000","last_undegraded":"2023-10-09T06:00:44.934084+0000","last_fullsized":"2023-10-09T06:00:44.934084+0000","mapping_epoch":285,"log_start":"295'3500","ondisk_log_start":"295'3500","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5023,"log_dups_size":0,"ondisk_log_size":5023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:00.484611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305070080,"num_objects":4655,"num_object_clones":0,"num_object_copies":13965,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4655,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8523,"num_write_kb":421696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":620,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.316072+0000","last_change":"2023-10-09T06:00:03.241479+0000","last_active":"2023-10-09T06:00:26.316072+0000","last_peered":"2023-10-09T06:00:26.316072+0000","last_clean":"2023-10-09T06:00:26.316072+0000","last_became_active":"2023-10-09T05:59:32.530753+0000","last_became_peered":"2023-10-09T05:59:32.530753+0000","last_unstale":"2023-10-09T06:00:26.316072+0000","last_undegraded":"2023-10-09T06:00:26.316072+0000","last_fullsized":"2023-10-09T06:00:26.316072+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:03.241335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:03.241335+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:54.822114+0000","scrub_duration":0.017780547000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":196,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.326096+0000","last_change":"2023-10-09T05:55:59.379174+0000","last_active":"2023-10-09T06:00:26.326096+0000","last_peered":"2023-10-09T06:00:26.326096+0000","last_clean":"2023-10-09T06:00:26.326096+0000","last_became_active":"2023-10-09T05:55:59.341932+0000","last_became_peered":"2023-10-09T05:55:59.341932+0000","last_unstale":"2023-10-09T06:00:26.326096+0000","last_undegraded":"2023-10-09T06:00:26.326096+0000","last_fullsized":"2023-10-09T06:00:26.326096+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.636171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":360,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.323153+0000","last_change":"2023-10-09T05:57:23.475146+0000","last_active":"2023-10-09T06:00:26.323153+0000","last_peered":"2023-10-09T06:00:26.323153+0000","last_clean":"2023-10-09T06:00:26.323153+0000","last_became_active":"2023-10-09T05:57:23.474776+0000","last_became_peered":"2023-10-09T05:57:23.474776+0000","last_unstale":"2023-10-09T06:00:26.323153+0000","last_undegraded":"2023-10-09T06:00:26.323153+0000","last_fullsized":"2023-10-09T06:00:26.323153+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:47.775743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"314'8479","reported_seq":8518,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:44.919440+0000","last_change":"2023-10-09T05:58:27.377562+0000","last_active":"2023-10-09T06:00:44.919440+0000","last_peered":"2023-10-09T06:00:44.919440+0000","last_clean":"2023-10-09T06:00:44.919440+0000","last_became_active":"2023-10-09T05:58:27.377253+0000","last_became_peered":"2023-10-09T05:58:27.377253+0000","last_unstale":"2023-10-09T06:00:44.919440+0000","last_undegraded":"2023-10-09T06:00:44.919440+0000","last_fullsized":"2023-10-09T06:00:44.919440+0000","mapping_epoch":285,"log_start":"295'2900","ondisk_log_start":"295'2900","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5579,"log_dups_size":0,"ondisk_log_size":5579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:34.871962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316735488,"num_objects":4833,"num_object_clones":0,"num_object_copies":14499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4833,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8479,"num_write_kb":425984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":736,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.319140+0000","last_change":"2023-10-09T05:58:41.753606+0000","last_active":"2023-10-09T06:00:26.319140+0000","last_peered":"2023-10-09T06:00:26.319140+0000","last_clean":"2023-10-09T06:00:26.319140+0000","last_became_active":"2023-10-09T05:58:21.332726+0000","last_became_peered":"2023-10-09T05:58:21.332726+0000","last_unstale":"2023-10-09T06:00:26.319140+0000","last_undegraded":"2023-10-09T06:00:26.319140+0000","last_fullsized":"2023-10-09T06:00:26.319140+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:41.753506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:41.753506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:15.947090+0000","scrub_duration":0.0022121929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":152,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.328492+0000","last_change":"2023-10-09T05:53:59.212161+0000","last_active":"2023-10-09T06:00:26.328492+0000","last_peered":"2023-10-09T06:00:26.328492+0000","last_clean":"2023-10-09T06:00:26.328492+0000","last_became_active":"2023-10-09T05:53:59.193814+0000","last_became_peered":"2023-10-09T05:53:59.193814+0000","last_unstale":"2023-10-09T06:00:26.328492+0000","last_undegraded":"2023-10-09T06:00:26.328492+0000","last_fullsized":"2023-10-09T06:00:26.328492+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:10.502408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":353,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.318635+0000","last_change":"2023-10-09T05:55:59.303352+0000","last_active":"2023-10-09T06:00:26.318635+0000","last_peered":"2023-10-09T06:00:26.318635+0000","last_clean":"2023-10-09T06:00:26.318635+0000","last_became_active":"2023-10-09T05:55:59.302923+0000","last_became_peered":"2023-10-09T05:55:59.302923+0000","last_unstale":"2023-10-09T06:00:26.318635+0000","last_undegraded":"2023-10-09T06:00:26.318635+0000","last_fullsized":"2023-10-09T06:00:26.318635+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:53.013376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"314'8480","reported_seq":8526,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:45.564721+0000","last_change":"2023-10-09T05:58:59.347359+0000","last_active":"2023-10-09T06:00:45.564721+0000","last_peered":"2023-10-09T06:00:45.564721+0000","last_clean":"2023-10-09T06:00:45.564721+0000","last_became_active":"2023-10-09T05:58:59.328422+0000","last_became_peered":"2023-10-09T05:58:59.328422+0000","last_unstale":"2023-10-09T06:00:45.564721+0000","last_undegraded":"2023-10-09T06:00:45.564721+0000","last_fullsized":"2023-10-09T06:00:45.564721+0000","mapping_epoch":292,"log_start":"295'3000","ondisk_log_start":"295'3000","created":285,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5480,"log_dups_size":0,"ondisk_log_size":5480,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:31.541181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303955968,"num_objects":4638,"num_object_clones":0,"num_object_copies":13914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8480,"num_write_kb":419776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":480,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320933+0000","last_change":"2023-10-09T05:59:57.043556+0000","last_active":"2023-10-09T06:00:26.320933+0000","last_peered":"2023-10-09T06:00:26.320933+0000","last_clean":"2023-10-09T06:00:26.320933+0000","last_became_active":"2023-10-09T05:59:32.525447+0000","last_became_peered":"2023-10-09T05:59:32.525447+0000","last_unstale":"2023-10-09T06:00:26.320933+0000","last_undegraded":"2023-10-09T06:00:26.320933+0000","last_fullsized":"2023-10-09T06:00:26.320933+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.043489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.043489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:17.051206+0000","scrub_duration":0.012927141,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":77,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.316160+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T06:00:26.316160+0000","last_peered":"2023-10-09T06:00:26.316160+0000","last_clean":"2023-10-09T06:00:26.316160+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T06:00:26.316160+0000","last_undegraded":"2023-10-09T06:00:26.316160+0000","last_fullsized":"2023-10-09T06:00:26.316160+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":325,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.322739+0000","last_change":"2023-10-09T05:57:23.473860+0000","last_active":"2023-10-09T06:00:26.322739+0000","last_peered":"2023-10-09T06:00:26.322739+0000","last_clean":"2023-10-09T06:00:26.322739+0000","last_became_active":"2023-10-09T05:57:23.473101+0000","last_became_peered":"2023-10-09T05:57:23.473101+0000","last_unstale":"2023-10-09T06:00:26.322739+0000","last_undegraded":"2023-10-09T06:00:26.322739+0000","last_fullsized":"2023-10-09T06:00:26.322739+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:05.682865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"314'8331","reported_seq":8370,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:44.934697+0000","last_change":"2023-10-09T05:58:27.376809+0000","last_active":"2023-10-09T06:00:44.934697+0000","last_peered":"2023-10-09T06:00:44.934697+0000","last_clean":"2023-10-09T06:00:44.934697+0000","last_became_active":"2023-10-09T05:58:27.376476+0000","last_became_peered":"2023-10-09T05:58:27.376476+0000","last_unstale":"2023-10-09T06:00:44.934697+0000","last_undegraded":"2023-10-09T06:00:44.934697+0000","last_fullsized":"2023-10-09T06:00:44.934697+0000","mapping_epoch":285,"log_start":"294'2700","ondisk_log_start":"294'2700","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5631,"log_dups_size":0,"ondisk_log_size":5631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:53:15.515971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305594368,"num_objects":4663,"num_object_clones":0,"num_object_copies":13989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4663,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8331,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":757,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.327365+0000","last_change":"2023-10-09T05:59:55.438018+0000","last_active":"2023-10-09T06:00:26.327365+0000","last_peered":"2023-10-09T06:00:26.327365+0000","last_clean":"2023-10-09T06:00:26.327365+0000","last_became_active":"2023-10-09T05:59:32.538631+0000","last_became_peered":"2023-10-09T05:59:32.538631+0000","last_unstale":"2023-10-09T06:00:26.327365+0000","last_undegraded":"2023-10-09T06:00:26.327365+0000","last_fullsized":"2023-10-09T06:00:26.327365+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:55.437947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:55.437947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:37.566205+0000","scrub_duration":0.0091262979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":81,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.315021+0000","last_change":"2023-10-09T05:57:23.504733+0000","last_active":"2023-10-09T06:00:26.315021+0000","last_peered":"2023-10-09T06:00:26.315021+0000","last_clean":"2023-10-09T06:00:26.315021+0000","last_became_active":"2023-10-09T05:57:23.468709+0000","last_became_peered":"2023-10-09T05:57:23.468709+0000","last_unstale":"2023-10-09T06:00:26.315021+0000","last_undegraded":"2023-10-09T06:00:26.315021+0000","last_fullsized":"2023-10-09T06:00:26.315021+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:00.459980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":283,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320604+0000","last_change":"2023-10-09T05:59:00.402498+0000","last_active":"2023-10-09T06:00:26.320604+0000","last_peered":"2023-10-09T06:00:26.320604+0000","last_clean":"2023-10-09T06:00:26.320604+0000","last_became_active":"2023-10-09T05:59:00.402218+0000","last_became_peered":"2023-10-09T05:59:00.402218+0000","last_unstale":"2023-10-09T06:00:26.320604+0000","last_undegraded":"2023-10-09T06:00:26.320604+0000","last_fullsized":"2023-10-09T06:00:26.320604+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:53.653861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"314'8449","reported_seq":8487,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:42.542991+0000","last_change":"2023-10-09T05:58:27.405765+0000","last_active":"2023-10-09T06:00:42.542991+0000","last_peered":"2023-10-09T06:00:42.542991+0000","last_clean":"2023-10-09T06:00:42.542991+0000","last_became_active":"2023-10-09T05:58:27.405447+0000","last_became_peered":"2023-10-09T05:58:27.405447+0000","last_unstale":"2023-10-09T06:00:42.542991+0000","last_undegraded":"2023-10-09T06:00:42.542991+0000","last_fullsized":"2023-10-09T06:00:42.542991+0000","mapping_epoch":285,"log_start":"289'1100","ondisk_log_start":"289'1100","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":7349,"log_dups_size":0,"ondisk_log_size":7349,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:52.191564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319619072,"num_objects":4877,"num_object_clones":0,"num_object_copies":14631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8449,"num_write_kb":426432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":502,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.314840+0000","last_change":"2023-10-09T05:59:58.282543+0000","last_active":"2023-10-09T06:00:26.314840+0000","last_peered":"2023-10-09T06:00:26.314840+0000","last_clean":"2023-10-09T06:00:26.314840+0000","last_became_active":"2023-10-09T05:59:32.552960+0000","last_became_peered":"2023-10-09T05:59:32.552960+0000","last_unstale":"2023-10-09T06:00:26.314840+0000","last_undegraded":"2023-10-09T06:00:26.314840+0000","last_fullsized":"2023-10-09T06:00:26.314840+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.282393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.282393+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:10.201323+0000","scrub_duration":0.01402403,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":324,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.318553+0000","last_change":"2023-10-09T05:55:59.320669+0000","last_active":"2023-10-09T06:00:26.318553+0000","last_peered":"2023-10-09T06:00:26.318553+0000","last_clean":"2023-10-09T06:00:26.318553+0000","last_became_active":"2023-10-09T05:55:59.320352+0000","last_became_peered":"2023-10-09T05:55:59.320352+0000","last_unstale":"2023-10-09T06:00:26.318553+0000","last_undegraded":"2023-10-09T06:00:26.318553+0000","last_fullsized":"2023-10-09T06:00:26.318553+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:09:14.227028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":360,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320882+0000","last_change":"2023-10-09T05:57:10.362295+0000","last_active":"2023-10-09T06:00:26.320882+0000","last_peered":"2023-10-09T06:00:26.320882+0000","last_clean":"2023-10-09T06:00:26.320882+0000","last_became_active":"2023-10-09T05:57:10.361249+0000","last_became_peered":"2023-10-09T05:57:10.361249+0000","last_unstale":"2023-10-09T06:00:26.320882+0000","last_undegraded":"2023-10-09T06:00:26.320882+0000","last_fullsized":"2023-10-09T06:00:26.320882+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:22.088261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"314'8422","reported_seq":8460,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:43.569549+0000","last_change":"2023-10-09T05:58:27.378739+0000","last_active":"2023-10-09T06:00:43.569549+0000","last_peered":"2023-10-09T06:00:43.569549+0000","last_clean":"2023-10-09T06:00:43.569549+0000","last_became_active":"2023-10-09T05:58:27.378459+0000","last_became_peered":"2023-10-09T05:58:27.378459+0000","last_unstale":"2023-10-09T06:00:43.569549+0000","last_undegraded":"2023-10-09T06:00:43.569549+0000","last_fullsized":"2023-10-09T06:00:43.569549+0000","mapping_epoch":285,"log_start":"293'2600","ondisk_log_start":"293'2600","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5822,"log_dups_size":0,"ondisk_log_size":5822,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:32.861544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312147992,"num_objects":4764,"num_object_clones":0,"num_object_copies":14292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4764,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8422,"num_write_kb":421889,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":583,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.319243+0000","last_change":"2023-10-09T05:59:56.373898+0000","last_active":"2023-10-09T06:00:38.319243+0000","last_peered":"2023-10-09T06:00:38.319243+0000","last_clean":"2023-10-09T06:00:38.319243+0000","last_became_active":"2023-10-09T05:59:32.541068+0000","last_became_peered":"2023-10-09T05:59:32.541068+0000","last_unstale":"2023-10-09T06:00:38.319243+0000","last_undegraded":"2023-10-09T06:00:38.319243+0000","last_fullsized":"2023-10-09T06:00:38.319243+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.373748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.373748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:53:42.272936+0000","scrub_duration":0.0024576450000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":294,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.322943+0000","last_change":"2023-10-09T05:54:00.619165+0000","last_active":"2023-10-09T06:00:26.322943+0000","last_peered":"2023-10-09T06:00:26.322943+0000","last_clean":"2023-10-09T06:00:26.322943+0000","last_became_active":"2023-10-09T05:54:00.618888+0000","last_became_peered":"2023-10-09T05:54:00.618888+0000","last_unstale":"2023-10-09T06:00:26.322943+0000","last_undegraded":"2023-10-09T06:00:26.322943+0000","last_fullsized":"2023-10-09T06:00:26.322943+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:07.175996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":333,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.321083+0000","last_change":"2023-10-09T05:57:23.483158+0000","last_active":"2023-10-09T06:00:26.321083+0000","last_peered":"2023-10-09T06:00:26.321083+0000","last_clean":"2023-10-09T06:00:26.321083+0000","last_became_active":"2023-10-09T05:57:23.482701+0000","last_became_peered":"2023-10-09T05:57:23.482701+0000","last_unstale":"2023-10-09T06:00:26.321083+0000","last_undegraded":"2023-10-09T06:00:26.321083+0000","last_fullsized":"2023-10-09T06:00:26.321083+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:51:08.624310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"314'8302","reported_seq":8338,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:43.573785+0000","last_change":"2023-10-09T05:58:27.376357+0000","last_active":"2023-10-09T06:00:43.573785+0000","last_peered":"2023-10-09T06:00:43.573785+0000","last_clean":"2023-10-09T06:00:43.573785+0000","last_became_active":"2023-10-09T05:58:27.376017+0000","last_became_peered":"2023-10-09T05:58:27.376017+0000","last_unstale":"2023-10-09T06:00:43.573785+0000","last_undegraded":"2023-10-09T06:00:43.573785+0000","last_fullsized":"2023-10-09T06:00:43.573785+0000","mapping_epoch":285,"log_start":"293'2500","ondisk_log_start":"293'2500","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5802,"log_dups_size":0,"ondisk_log_size":5802,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:30.067479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309198848,"num_objects":4718,"num_object_clones":0,"num_object_copies":14154,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4718,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8302,"num_write_kb":416640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":467,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.318473+0000","last_change":"2023-10-09T05:59:52.129883+0000","last_active":"2023-10-09T06:00:26.318473+0000","last_peered":"2023-10-09T06:00:26.318473+0000","last_clean":"2023-10-09T06:00:26.318473+0000","last_became_active":"2023-10-09T05:59:32.529083+0000","last_became_peered":"2023-10-09T05:59:32.529083+0000","last_unstale":"2023-10-09T06:00:26.318473+0000","last_undegraded":"2023-10-09T06:00:26.318473+0000","last_fullsized":"2023-10-09T06:00:26.318473+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:52.129816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:52.129816+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:38.309831+0000","scrub_duration":0.0039938459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":77,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.323207+0000","last_change":"2023-10-09T05:57:23.469928+0000","last_active":"2023-10-09T06:00:26.323207+0000","last_peered":"2023-10-09T06:00:26.323207+0000","last_clean":"2023-10-09T06:00:26.323207+0000","last_became_active":"2023-10-09T05:57:23.469697+0000","last_became_peered":"2023-10-09T05:57:23.469697+0000","last_unstale":"2023-10-09T06:00:26.323207+0000","last_undegraded":"2023-10-09T06:00:26.323207+0000","last_fullsized":"2023-10-09T06:00:26.323207+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:21:07.179434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":194,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.322543+0000","last_change":"2023-10-09T05:55:59.409563+0000","last_active":"2023-10-09T06:00:26.322543+0000","last_peered":"2023-10-09T06:00:26.322543+0000","last_clean":"2023-10-09T06:00:26.322543+0000","last_became_active":"2023-10-09T05:55:59.409184+0000","last_became_peered":"2023-10-09T05:55:59.409184+0000","last_unstale":"2023-10-09T06:00:26.322543+0000","last_undegraded":"2023-10-09T06:00:26.322543+0000","last_fullsized":"2023-10-09T06:00:26.322543+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:39.513591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1","version":"314'8440","reported_seq":8476,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:44.652281+0000","last_change":"2023-10-09T05:58:27.377316+0000","last_active":"2023-10-09T06:00:44.652281+0000","last_peered":"2023-10-09T06:00:44.652281+0000","last_clean":"2023-10-09T06:00:44.652281+0000","last_became_active":"2023-10-09T05:58:27.377007+0000","last_became_peered":"2023-10-09T05:58:27.377007+0000","last_unstale":"2023-10-09T06:00:44.652281+0000","last_undegraded":"2023-10-09T06:00:44.652281+0000","last_fullsized":"2023-10-09T06:00:44.652281+0000","mapping_epoch":285,"log_start":"293'2600","ondisk_log_start":"293'2600","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5840,"log_dups_size":0,"ondisk_log_size":5840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:10.982167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307101696,"num_objects":4686,"num_object_clones":0,"num_object_copies":14058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8440,"num_write_kb":420032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":737,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.318980+0000","last_change":"2023-10-09T05:59:56.092654+0000","last_active":"2023-10-09T06:00:26.318980+0000","last_peered":"2023-10-09T06:00:26.318980+0000","last_clean":"2023-10-09T06:00:26.318980+0000","last_became_active":"2023-10-09T05:59:32.529949+0000","last_became_peered":"2023-10-09T05:59:32.529949+0000","last_unstale":"2023-10-09T06:00:26.318980+0000","last_undegraded":"2023-10-09T06:00:26.318980+0000","last_fullsized":"2023-10-09T06:00:26.318980+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.092586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.092586+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:15:13.234903+0000","scrub_duration":0.0013484219999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":209,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.324713+0000","last_change":"2023-10-09T05:55:59.298953+0000","last_active":"2023-10-09T06:00:26.324713+0000","last_peered":"2023-10-09T06:00:26.324713+0000","last_clean":"2023-10-09T06:00:26.324713+0000","last_became_active":"2023-10-09T05:55:59.298624+0000","last_became_peered":"2023-10-09T05:55:59.298624+0000","last_unstale":"2023-10-09T06:00:26.324713+0000","last_undegraded":"2023-10-09T06:00:26.324713+0000","last_fullsized":"2023-10-09T06:00:26.324713+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:00.513870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":77,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.319161+0000","last_change":"2023-10-09T05:57:23.468480+0000","last_active":"2023-10-09T06:00:26.319161+0000","last_peered":"2023-10-09T06:00:26.319161+0000","last_clean":"2023-10-09T06:00:26.319161+0000","last_became_active":"2023-10-09T05:57:23.468179+0000","last_became_peered":"2023-10-09T05:57:23.468179+0000","last_unstale":"2023-10-09T06:00:26.319161+0000","last_undegraded":"2023-10-09T06:00:26.319161+0000","last_fullsized":"2023-10-09T06:00:26.319161+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:18.157188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":177,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320548+0000","last_change":"2023-10-09T05:56:49.249711+0000","last_active":"2023-10-09T06:00:26.320548+0000","last_peered":"2023-10-09T06:00:26.320548+0000","last_clean":"2023-10-09T06:00:26.320548+0000","last_became_active":"2023-10-09T05:56:49.249272+0000","last_became_peered":"2023-10-09T05:56:49.249272+0000","last_unstale":"2023-10-09T06:00:26.320548+0000","last_undegraded":"2023-10-09T06:00:26.320548+0000","last_fullsized":"2023-10-09T06:00:26.320548+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:42.596986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.7","version":"314'8181","reported_seq":8218,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:43.586062+0000","last_change":"2023-10-09T05:58:27.378776+0000","last_active":"2023-10-09T06:00:43.586062+0000","last_peered":"2023-10-09T06:00:43.586062+0000","last_clean":"2023-10-09T06:00:43.586062+0000","last_became_active":"2023-10-09T05:58:27.378504+0000","last_became_peered":"2023-10-09T05:58:27.378504+0000","last_unstale":"2023-10-09T06:00:43.586062+0000","last_undegraded":"2023-10-09T06:00:43.586062+0000","last_fullsized":"2023-10-09T06:00:43.586062+0000","mapping_epoch":285,"log_start":"291'2400","ondisk_log_start":"291'2400","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5781,"log_dups_size":0,"ondisk_log_size":5781,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:27.090596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305856512,"num_objects":4667,"num_object_clones":0,"num_object_copies":14001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8181,"num_write_kb":411136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":517,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.321384+0000","last_change":"2023-10-09T05:58:36.225081+0000","last_active":"2023-10-09T06:00:26.321384+0000","last_peered":"2023-10-09T06:00:26.321384+0000","last_clean":"2023-10-09T06:00:26.321384+0000","last_became_active":"2023-10-09T05:56:49.237269+0000","last_became_peered":"2023-10-09T05:56:49.237269+0000","last_unstale":"2023-10-09T06:00:26.321384+0000","last_undegraded":"2023-10-09T06:00:26.321384+0000","last_fullsized":"2023-10-09T06:00:26.321384+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:02.908583+0000","scrub_duration":0.0016660799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":334,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.318579+0000","last_change":"2023-10-09T05:56:49.254337+0000","last_active":"2023-10-09T06:00:26.318579+0000","last_peered":"2023-10-09T06:00:26.318579+0000","last_clean":"2023-10-09T06:00:26.318579+0000","last_became_active":"2023-10-09T05:56:49.247892+0000","last_became_peered":"2023-10-09T05:56:49.247892+0000","last_unstale":"2023-10-09T06:00:26.318579+0000","last_undegraded":"2023-10-09T06:00:26.318579+0000","last_fullsized":"2023-10-09T06:00:26.318579+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:44:20.157638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":291,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320885+0000","last_change":"2023-10-09T05:51:57.203142+0000","last_active":"2023-10-09T06:00:26.320885+0000","last_peered":"2023-10-09T06:00:26.320885+0000","last_clean":"2023-10-09T06:00:26.320885+0000","last_became_active":"2023-10-09T05:51:57.202850+0000","last_became_peered":"2023-10-09T05:51:57.202850+0000","last_unstale":"2023-10-09T06:00:26.320885+0000","last_undegraded":"2023-10-09T06:00:26.320885+0000","last_fullsized":"2023-10-09T06:00:26.320885+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:51.090737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":207,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320451+0000","last_change":"2023-10-09T05:58:59.330420+0000","last_active":"2023-10-09T06:00:26.320451+0000","last_peered":"2023-10-09T06:00:26.320451+0000","last_clean":"2023-10-09T06:00:26.320451+0000","last_became_active":"2023-10-09T05:58:59.330137+0000","last_became_peered":"2023-10-09T05:58:59.330137+0000","last_unstale":"2023-10-09T06:00:26.320451+0000","last_undegraded":"2023-10-09T06:00:26.320451+0000","last_fullsized":"2023-10-09T06:00:26.320451+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:33.801594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.4","version":"314'8509","reported_seq":8548,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:44.905137+0000","last_change":"2023-10-09T05:58:27.375343+0000","last_active":"2023-10-09T06:00:44.905137+0000","last_peered":"2023-10-09T06:00:44.905137+0000","last_clean":"2023-10-09T06:00:44.905137+0000","last_became_active":"2023-10-09T05:58:27.375094+0000","last_became_peered":"2023-10-09T05:58:27.375094+0000","last_unstale":"2023-10-09T06:00:44.905137+0000","last_undegraded":"2023-10-09T06:00:44.905137+0000","last_fullsized":"2023-10-09T06:00:44.905137+0000","mapping_epoch":285,"log_start":"294'2900","ondisk_log_start":"294'2900","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5609,"log_dups_size":0,"ondisk_log_size":5609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:48:08.192351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305725440,"num_objects":4665,"num_object_clones":0,"num_object_copies":13995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8509,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":621,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.326136+0000","last_change":"2023-10-09T05:58:55.897880+0000","last_active":"2023-10-09T06:00:26.326136+0000","last_peered":"2023-10-09T06:00:26.326136+0000","last_clean":"2023-10-09T06:00:26.326136+0000","last_became_active":"2023-10-09T05:58:01.899103+0000","last_became_peered":"2023-10-09T05:58:01.899103+0000","last_unstale":"2023-10-09T06:00:26.326136+0000","last_undegraded":"2023-10-09T06:00:26.326136+0000","last_fullsized":"2023-10-09T06:00:26.326136+0000","mapping_epoch":268,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":269,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:55.897812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:55.897812+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:01.223576+0000","scrub_duration":0.0014511420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":54,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.323163+0000","last_change":"2023-10-09T05:58:59.325257+0000","last_active":"2023-10-09T06:00:26.323163+0000","last_peered":"2023-10-09T06:00:26.323163+0000","last_clean":"2023-10-09T06:00:26.323163+0000","last_became_active":"2023-10-09T05:58:59.324993+0000","last_became_peered":"2023-10-09T05:58:59.324993+0000","last_unstale":"2023-10-09T06:00:26.323163+0000","last_undegraded":"2023-10-09T06:00:26.323163+0000","last_fullsized":"2023-10-09T06:00:26.323163+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:20.825840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":132,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.318512+0000","last_change":"2023-10-09T05:58:59.328724+0000","last_active":"2023-10-09T06:00:26.318512+0000","last_peered":"2023-10-09T06:00:26.318512+0000","last_clean":"2023-10-09T06:00:26.318512+0000","last_became_active":"2023-10-09T05:58:59.328395+0000","last_became_peered":"2023-10-09T05:58:59.328395+0000","last_unstale":"2023-10-09T06:00:26.318512+0000","last_undegraded":"2023-10-09T06:00:26.318512+0000","last_fullsized":"2023-10-09T06:00:26.318512+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:37.528856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":209,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.322686+0000","last_change":"2023-10-09T05:56:49.238219+0000","last_active":"2023-10-09T06:00:26.322686+0000","last_peered":"2023-10-09T06:00:26.322686+0000","last_clean":"2023-10-09T06:00:26.322686+0000","last_became_active":"2023-10-09T05:56:49.237919+0000","last_became_peered":"2023-10-09T05:56:49.237919+0000","last_unstale":"2023-10-09T06:00:26.322686+0000","last_undegraded":"2023-10-09T06:00:26.322686+0000","last_fullsized":"2023-10-09T06:00:26.322686+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:44.599032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.6","version":"314'8515","reported_seq":8554,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:44.918004+0000","last_change":"2023-10-09T05:58:27.377502+0000","last_active":"2023-10-09T06:00:44.918004+0000","last_peered":"2023-10-09T06:00:44.918004+0000","last_clean":"2023-10-09T06:00:44.918004+0000","last_became_active":"2023-10-09T05:58:27.377195+0000","last_became_peered":"2023-10-09T05:58:27.377195+0000","last_unstale":"2023-10-09T06:00:44.918004+0000","last_undegraded":"2023-10-09T06:00:44.918004+0000","last_fullsized":"2023-10-09T06:00:44.918004+0000","mapping_epoch":285,"log_start":"295'2900","ondisk_log_start":"295'2900","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5615,"log_dups_size":0,"ondisk_log_size":5615,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:14.980773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308084736,"num_objects":4701,"num_object_clones":0,"num_object_copies":14103,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4701,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8515,"num_write_kb":422912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":185,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.325522+0000","last_change":"2023-10-09T05:58:59.330109+0000","last_active":"2023-10-09T06:00:26.325522+0000","last_peered":"2023-10-09T06:00:26.325522+0000","last_clean":"2023-10-09T06:00:26.325522+0000","last_became_active":"2023-10-09T05:58:59.329808+0000","last_became_peered":"2023-10-09T05:58:59.329808+0000","last_unstale":"2023-10-09T06:00:26.325522+0000","last_undegraded":"2023-10-09T06:00:26.325522+0000","last_fullsized":"2023-10-09T06:00:26.325522+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:30.680622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":718,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.325787+0000","last_change":"2023-10-09T05:58:37.782237+0000","last_active":"2023-10-09T06:00:26.325787+0000","last_peered":"2023-10-09T06:00:26.325787+0000","last_clean":"2023-10-09T06:00:26.325787+0000","last_became_active":"2023-10-09T05:53:59.202004+0000","last_became_peered":"2023-10-09T05:53:59.202004+0000","last_unstale":"2023-10-09T06:00:26.325787+0000","last_undegraded":"2023-10-09T06:00:26.325787+0000","last_fullsized":"2023-10-09T06:00:26.325787+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:37.782173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:37.782173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:20.787963+0000","scrub_duration":0.001325908,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":77,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.323674+0000","last_change":"2023-10-09T05:57:23.479174+0000","last_active":"2023-10-09T06:00:26.323674+0000","last_peered":"2023-10-09T06:00:26.323674+0000","last_clean":"2023-10-09T06:00:26.323674+0000","last_became_active":"2023-10-09T05:57:23.478897+0000","last_became_peered":"2023-10-09T05:57:23.478897+0000","last_unstale":"2023-10-09T06:00:26.323674+0000","last_undegraded":"2023-10-09T06:00:26.323674+0000","last_fullsized":"2023-10-09T06:00:26.323674+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:00.160042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":276,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.318963+0000","last_change":"2023-10-09T05:53:59.195325+0000","last_active":"2023-10-09T06:00:26.318963+0000","last_peered":"2023-10-09T06:00:26.318963+0000","last_clean":"2023-10-09T06:00:26.318963+0000","last_became_active":"2023-10-09T05:53:59.194954+0000","last_became_peered":"2023-10-09T05:53:59.194954+0000","last_unstale":"2023-10-09T06:00:26.318963+0000","last_undegraded":"2023-10-09T06:00:26.318963+0000","last_fullsized":"2023-10-09T06:00:26.318963+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:23.228416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"314'8661","reported_seq":8698,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:45.531170+0000","last_change":"2023-10-09T05:58:27.375153+0000","last_active":"2023-10-09T06:00:45.531170+0000","last_peered":"2023-10-09T06:00:45.531170+0000","last_clean":"2023-10-09T06:00:45.531170+0000","last_became_active":"2023-10-09T05:58:27.374820+0000","last_became_peered":"2023-10-09T05:58:27.374820+0000","last_unstale":"2023-10-09T06:00:45.531170+0000","last_undegraded":"2023-10-09T06:00:45.531170+0000","last_fullsized":"2023-10-09T06:00:45.531170+0000","mapping_epoch":285,"log_start":"295'3200","ondisk_log_start":"295'3200","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5461,"log_dups_size":0,"ondisk_log_size":5461,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:08.834399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313720832,"num_objects":4787,"num_object_clones":0,"num_object_copies":14361,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4787,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8661,"num_write_kb":430336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":509,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.318736+0000","last_change":"2023-10-09T05:58:40.724430+0000","last_active":"2023-10-09T06:00:26.318736+0000","last_peered":"2023-10-09T06:00:26.318736+0000","last_clean":"2023-10-09T06:00:26.318736+0000","last_became_active":"2023-10-09T05:57:54.796937+0000","last_became_peered":"2023-10-09T05:57:54.796937+0000","last_unstale":"2023-10-09T06:00:26.318736+0000","last_undegraded":"2023-10-09T06:00:26.318736+0000","last_fullsized":"2023-10-09T06:00:26.318736+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:40.724331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:40.724331+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:10.562541+0000","scrub_duration":0.0016947469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":180,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.325629+0000","last_change":"2023-10-09T05:55:59.300015+0000","last_active":"2023-10-09T06:00:26.325629+0000","last_peered":"2023-10-09T06:00:26.325629+0000","last_clean":"2023-10-09T06:00:26.325629+0000","last_became_active":"2023-10-09T05:55:59.299281+0000","last_became_peered":"2023-10-09T05:55:59.299281+0000","last_unstale":"2023-10-09T06:00:26.325629+0000","last_undegraded":"2023-10-09T06:00:26.325629+0000","last_fullsized":"2023-10-09T06:00:26.325629+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:50:32.465904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":252,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.321511+0000","last_change":"2023-10-09T05:57:23.471369+0000","last_active":"2023-10-09T06:00:26.321511+0000","last_peered":"2023-10-09T06:00:26.321511+0000","last_clean":"2023-10-09T06:00:26.321511+0000","last_became_active":"2023-10-09T05:57:23.471049+0000","last_became_peered":"2023-10-09T05:57:23.471049+0000","last_unstale":"2023-10-09T06:00:26.321511+0000","last_undegraded":"2023-10-09T06:00:26.321511+0000","last_fullsized":"2023-10-09T06:00:26.321511+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:21.208825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":369,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.323821+0000","last_change":"2023-10-09T05:57:10.348399+0000","last_active":"2023-10-09T06:00:38.323821+0000","last_peered":"2023-10-09T06:00:38.323821+0000","last_clean":"2023-10-09T06:00:38.323821+0000","last_became_active":"2023-10-09T05:57:10.348061+0000","last_became_peered":"2023-10-09T05:57:10.348061+0000","last_unstale":"2023-10-09T06:00:38.323821+0000","last_undegraded":"2023-10-09T06:00:38.323821+0000","last_fullsized":"2023-10-09T06:00:38.323821+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:12.453118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"314'8550","reported_seq":8588,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:44.923222+0000","last_change":"2023-10-09T05:58:27.405929+0000","last_active":"2023-10-09T06:00:44.923222+0000","last_peered":"2023-10-09T06:00:44.923222+0000","last_clean":"2023-10-09T06:00:44.923222+0000","last_became_active":"2023-10-09T05:58:27.405604+0000","last_became_peered":"2023-10-09T05:58:27.405604+0000","last_unstale":"2023-10-09T06:00:44.923222+0000","last_undegraded":"2023-10-09T06:00:44.923222+0000","last_fullsized":"2023-10-09T06:00:44.923222+0000","mapping_epoch":285,"log_start":"295'3505","ondisk_log_start":"295'3505","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5045,"log_dups_size":0,"ondisk_log_size":5045,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:19.748373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312475648,"num_objects":4768,"num_object_clones":0,"num_object_copies":14304,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4768,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8550,"num_write_kb":426176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":727,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.319007+0000","last_change":"2023-10-09T05:59:54.074413+0000","last_active":"2023-10-09T06:00:26.319007+0000","last_peered":"2023-10-09T06:00:26.319007+0000","last_clean":"2023-10-09T06:00:26.319007+0000","last_became_active":"2023-10-09T05:59:32.538959+0000","last_became_peered":"2023-10-09T05:59:32.538959+0000","last_unstale":"2023-10-09T06:00:26.319007+0000","last_undegraded":"2023-10-09T06:00:26.319007+0000","last_fullsized":"2023-10-09T06:00:26.319007+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:54.074310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:54.074310+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:10.811269+0000","scrub_duration":0.011040648,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":148,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.328396+0000","last_change":"2023-10-09T05:53:59.201576+0000","last_active":"2023-10-09T06:00:26.328396+0000","last_peered":"2023-10-09T06:00:26.328396+0000","last_clean":"2023-10-09T06:00:26.328396+0000","last_became_active":"2023-10-09T05:53:59.201309+0000","last_became_peered":"2023-10-09T05:53:59.201309+0000","last_unstale":"2023-10-09T06:00:26.328396+0000","last_undegraded":"2023-10-09T06:00:26.328396+0000","last_fullsized":"2023-10-09T06:00:26.328396+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:54.956371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":138,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.321963+0000","last_change":"2023-10-09T05:58:59.328614+0000","last_active":"2023-10-09T06:00:26.321963+0000","last_peered":"2023-10-09T06:00:26.321963+0000","last_clean":"2023-10-09T06:00:26.321963+0000","last_became_active":"2023-10-09T05:58:59.328203+0000","last_became_peered":"2023-10-09T05:58:59.328203+0000","last_unstale":"2023-10-09T06:00:26.321963+0000","last_undegraded":"2023-10-09T06:00:26.321963+0000","last_fullsized":"2023-10-09T06:00:26.321963+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:21.572990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":241,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:34.918693+0000","last_change":"2023-10-09T05:53:59.188462+0000","last_active":"2023-10-09T06:00:34.918693+0000","last_peered":"2023-10-09T06:00:34.918693+0000","last_clean":"2023-10-09T06:00:34.918693+0000","last_became_active":"2023-10-09T05:53:59.188061+0000","last_became_peered":"2023-10-09T05:53:59.188061+0000","last_unstale":"2023-10-09T06:00:34.918693+0000","last_undegraded":"2023-10-09T06:00:34.918693+0000","last_fullsized":"2023-10-09T06:00:34.918693+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:19.167114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"314'8438","reported_seq":8477,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:44.846537+0000","last_change":"2023-10-09T05:58:27.377750+0000","last_active":"2023-10-09T06:00:44.846537+0000","last_peered":"2023-10-09T06:00:44.846537+0000","last_clean":"2023-10-09T06:00:44.846537+0000","last_became_active":"2023-10-09T05:58:27.377457+0000","last_became_peered":"2023-10-09T05:58:27.377457+0000","last_unstale":"2023-10-09T06:00:44.846537+0000","last_undegraded":"2023-10-09T06:00:44.846537+0000","last_fullsized":"2023-10-09T06:00:44.846537+0000","mapping_epoch":285,"log_start":"295'3400","ondisk_log_start":"295'3400","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5038,"log_dups_size":0,"ondisk_log_size":5038,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:52:04.242389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308543488,"num_objects":4708,"num_object_clones":0,"num_object_copies":14124,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4708,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8438,"num_write_kb":420672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":698,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320794+0000","last_change":"2023-10-09T05:59:54.020810+0000","last_active":"2023-10-09T06:00:26.320794+0000","last_peered":"2023-10-09T06:00:26.320794+0000","last_clean":"2023-10-09T06:00:26.320794+0000","last_became_active":"2023-10-09T05:59:32.542578+0000","last_became_peered":"2023-10-09T05:59:32.542578+0000","last_unstale":"2023-10-09T06:00:26.320794+0000","last_undegraded":"2023-10-09T06:00:26.320794+0000","last_fullsized":"2023-10-09T06:00:26.320794+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:54.020712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:54.020712+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:21.885126+0000","scrub_duration":0.0025782679999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":321,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.322668+0000","last_change":"2023-10-09T05:53:59.228948+0000","last_active":"2023-10-09T06:00:26.322668+0000","last_peered":"2023-10-09T06:00:26.322668+0000","last_clean":"2023-10-09T06:00:26.322668+0000","last_became_active":"2023-10-09T05:53:59.186359+0000","last_became_peered":"2023-10-09T05:53:59.186359+0000","last_unstale":"2023-10-09T06:00:26.322668+0000","last_undegraded":"2023-10-09T06:00:26.322668+0000","last_fullsized":"2023-10-09T06:00:26.322668+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:39.990576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":171,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.318853+0000","last_change":"2023-10-09T05:55:59.339957+0000","last_active":"2023-10-09T06:00:26.318853+0000","last_peered":"2023-10-09T06:00:26.318853+0000","last_clean":"2023-10-09T06:00:26.318853+0000","last_became_active":"2023-10-09T05:55:59.339614+0000","last_became_peered":"2023-10-09T05:55:59.339614+0000","last_unstale":"2023-10-09T06:00:26.318853+0000","last_undegraded":"2023-10-09T06:00:26.318853+0000","last_fullsized":"2023-10-09T06:00:26.318853+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:01.613113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":77,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.315711+0000","last_change":"2023-10-09T05:57:23.477143+0000","last_active":"2023-10-09T06:00:26.315711+0000","last_peered":"2023-10-09T06:00:26.315711+0000","last_clean":"2023-10-09T06:00:26.315711+0000","last_became_active":"2023-10-09T05:57:23.475655+0000","last_became_peered":"2023-10-09T05:57:23.475655+0000","last_unstale":"2023-10-09T06:00:26.315711+0000","last_undegraded":"2023-10-09T06:00:26.315711+0000","last_fullsized":"2023-10-09T06:00:26.315711+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:29.528023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"314'8379","reported_seq":8418,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:44.906184+0000","last_change":"2023-10-09T05:58:27.376100+0000","last_active":"2023-10-09T06:00:44.906184+0000","last_peered":"2023-10-09T06:00:44.906184+0000","last_clean":"2023-10-09T06:00:44.906184+0000","last_became_active":"2023-10-09T05:58:27.375806+0000","last_became_peered":"2023-10-09T05:58:27.375806+0000","last_unstale":"2023-10-09T06:00:44.906184+0000","last_undegraded":"2023-10-09T06:00:44.906184+0000","last_fullsized":"2023-10-09T06:00:44.906184+0000","mapping_epoch":285,"log_start":"295'3300","ondisk_log_start":"295'3300","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5079,"log_dups_size":0,"ondisk_log_size":5079,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:56.466666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303890432,"num_objects":4637,"num_object_clones":0,"num_object_copies":13911,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4637,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8379,"num_write_kb":416512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":571,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.315984+0000","last_change":"2023-10-09T05:58:34.333129+0000","last_active":"2023-10-09T06:00:26.315984+0000","last_peered":"2023-10-09T06:00:26.315984+0000","last_clean":"2023-10-09T06:00:26.315984+0000","last_became_active":"2023-10-09T05:57:26.502911+0000","last_became_peered":"2023-10-09T05:57:26.502911+0000","last_unstale":"2023-10-09T06:00:26.315984+0000","last_undegraded":"2023-10-09T06:00:26.315984+0000","last_fullsized":"2023-10-09T06:00:26.315984+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:25.230712+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":318,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.319097+0000","last_change":"2023-10-09T05:57:58.832790+0000","last_active":"2023-10-09T06:00:26.319097+0000","last_peered":"2023-10-09T06:00:26.319097+0000","last_clean":"2023-10-09T06:00:26.319097+0000","last_became_active":"2023-10-09T05:57:58.832471+0000","last_became_peered":"2023-10-09T05:57:58.832471+0000","last_unstale":"2023-10-09T06:00:26.319097+0000","last_undegraded":"2023-10-09T06:00:26.319097+0000","last_fullsized":"2023-10-09T06:00:26.319097+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:39.358174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":355,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.328768+0000","last_change":"2023-10-09T05:57:58.836050+0000","last_active":"2023-10-09T06:00:38.328768+0000","last_peered":"2023-10-09T06:00:38.328768+0000","last_clean":"2023-10-09T06:00:38.328768+0000","last_became_active":"2023-10-09T05:57:58.833691+0000","last_became_peered":"2023-10-09T05:57:58.833691+0000","last_unstale":"2023-10-09T06:00:38.328768+0000","last_undegraded":"2023-10-09T06:00:38.328768+0000","last_fullsized":"2023-10-09T06:00:38.328768+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:32.257849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":256,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.323012+0000","last_change":"2023-10-09T05:40:12.676587+0000","last_active":"2023-10-09T06:00:26.323012+0000","last_peered":"2023-10-09T06:00:26.323012+0000","last_clean":"2023-10-09T06:00:26.323012+0000","last_became_active":"2023-10-09T05:40:12.676330+0000","last_became_peered":"2023-10-09T05:40:12.676330+0000","last_unstale":"2023-10-09T06:00:26.323012+0000","last_undegraded":"2023-10-09T06:00:26.323012+0000","last_fullsized":"2023-10-09T06:00:26.323012+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:46.659556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":539,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.314757+0000","last_change":"2023-10-09T06:00:06.181968+0000","last_active":"2023-10-09T06:00:26.314757+0000","last_peered":"2023-10-09T06:00:26.314757+0000","last_clean":"2023-10-09T06:00:26.314757+0000","last_became_active":"2023-10-09T05:59:40.324653+0000","last_became_peered":"2023-10-09T05:59:40.324653+0000","last_unstale":"2023-10-09T06:00:26.314757+0000","last_undegraded":"2023-10-09T06:00:26.314757+0000","last_fullsized":"2023-10-09T06:00:26.314757+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.181811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.181811+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:05:16.987523+0000","scrub_duration":0.011553313000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":244,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.322886+0000","last_change":"2023-10-09T05:53:59.191302+0000","last_active":"2023-10-09T06:00:26.322886+0000","last_peered":"2023-10-09T06:00:26.322886+0000","last_clean":"2023-10-09T06:00:26.322886+0000","last_became_active":"2023-10-09T05:53:59.190911+0000","last_became_peered":"2023-10-09T05:53:59.190911+0000","last_unstale":"2023-10-09T06:00:26.322886+0000","last_undegraded":"2023-10-09T06:00:26.322886+0000","last_fullsized":"2023-10-09T06:00:26.322886+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:12.801201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":54,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.315900+0000","last_change":"2023-10-09T05:58:59.326172+0000","last_active":"2023-10-09T06:00:26.315900+0000","last_peered":"2023-10-09T06:00:26.315900+0000","last_clean":"2023-10-09T06:00:26.315900+0000","last_became_active":"2023-10-09T05:58:59.325795+0000","last_became_peered":"2023-10-09T05:58:59.325795+0000","last_unstale":"2023-10-09T06:00:26.315900+0000","last_undegraded":"2023-10-09T06:00:26.315900+0000","last_fullsized":"2023-10-09T06:00:26.315900+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:34.308820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":656,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.318719+0000","last_change":"2023-10-09T06:00:04.215425+0000","last_active":"2023-10-09T06:00:26.318719+0000","last_peered":"2023-10-09T06:00:26.318719+0000","last_clean":"2023-10-09T06:00:26.318719+0000","last_became_active":"2023-10-09T05:59:40.326685+0000","last_became_peered":"2023-10-09T05:59:40.326685+0000","last_unstale":"2023-10-09T06:00:26.318719+0000","last_undegraded":"2023-10-09T06:00:26.318719+0000","last_fullsized":"2023-10-09T06:00:26.318719+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:04.215270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:04.215270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:03.963694+0000","scrub_duration":0.002850652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":104,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320705+0000","last_change":"2023-10-09T05:56:49.244931+0000","last_active":"2023-10-09T06:00:26.320705+0000","last_peered":"2023-10-09T06:00:26.320705+0000","last_clean":"2023-10-09T06:00:26.320705+0000","last_became_active":"2023-10-09T05:56:49.244651+0000","last_became_peered":"2023-10-09T05:56:49.244651+0000","last_unstale":"2023-10-09T06:00:26.320705+0000","last_undegraded":"2023-10-09T06:00:26.320705+0000","last_fullsized":"2023-10-09T06:00:26.320705+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:53.906578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":350,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320958+0000","last_change":"2023-10-09T05:57:10.356351+0000","last_active":"2023-10-09T06:00:26.320958+0000","last_peered":"2023-10-09T06:00:26.320958+0000","last_clean":"2023-10-09T06:00:26.320958+0000","last_became_active":"2023-10-09T05:57:10.356037+0000","last_became_peered":"2023-10-09T05:57:10.356037+0000","last_unstale":"2023-10-09T06:00:26.320958+0000","last_undegraded":"2023-10-09T06:00:26.320958+0000","last_fullsized":"2023-10-09T06:00:26.320958+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:15.139912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":104,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320902+0000","last_change":"2023-10-09T05:56:49.245612+0000","last_active":"2023-10-09T06:00:26.320902+0000","last_peered":"2023-10-09T06:00:26.320902+0000","last_clean":"2023-10-09T06:00:26.320902+0000","last_became_active":"2023-10-09T05:56:49.245235+0000","last_became_peered":"2023-10-09T05:56:49.245235+0000","last_unstale":"2023-10-09T06:00:26.320902+0000","last_undegraded":"2023-10-09T06:00:26.320902+0000","last_fullsized":"2023-10-09T06:00:26.320902+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:29.016794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":317,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320402+0000","last_change":"2023-10-09T05:59:00.400894+0000","last_active":"2023-10-09T06:00:26.320402+0000","last_peered":"2023-10-09T06:00:26.320402+0000","last_clean":"2023-10-09T06:00:26.320402+0000","last_became_active":"2023-10-09T05:59:00.399730+0000","last_became_peered":"2023-10-09T05:59:00.399730+0000","last_unstale":"2023-10-09T06:00:26.320402+0000","last_undegraded":"2023-10-09T06:00:26.320402+0000","last_fullsized":"2023-10-09T06:00:26.320402+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:46.821954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":560,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.327433+0000","last_change":"2023-10-09T06:00:00.456225+0000","last_active":"2023-10-09T06:00:26.327433+0000","last_peered":"2023-10-09T06:00:26.327433+0000","last_clean":"2023-10-09T06:00:26.327433+0000","last_became_active":"2023-10-09T05:59:40.311408+0000","last_became_peered":"2023-10-09T05:59:40.311408+0000","last_unstale":"2023-10-09T06:00:26.327433+0000","last_undegraded":"2023-10-09T06:00:26.327433+0000","last_fullsized":"2023-10-09T06:00:26.327433+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:00.456132+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:00.456132+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:29.222405+0000","scrub_duration":0.0014992199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":77,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.319165+0000","last_change":"2023-10-09T05:57:23.479560+0000","last_active":"2023-10-09T06:00:26.319165+0000","last_peered":"2023-10-09T06:00:26.319165+0000","last_clean":"2023-10-09T06:00:26.319165+0000","last_became_active":"2023-10-09T05:57:23.479177+0000","last_became_peered":"2023-10-09T05:57:23.479177+0000","last_unstale":"2023-10-09T06:00:26.319165+0000","last_undegraded":"2023-10-09T06:00:26.319165+0000","last_fullsized":"2023-10-09T06:00:26.319165+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:41.925161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":77,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.319032+0000","last_change":"2023-10-09T05:57:23.475179+0000","last_active":"2023-10-09T06:00:26.319032+0000","last_peered":"2023-10-09T06:00:26.319032+0000","last_clean":"2023-10-09T06:00:26.319032+0000","last_became_active":"2023-10-09T05:57:23.474904+0000","last_became_peered":"2023-10-09T05:57:23.474904+0000","last_unstale":"2023-10-09T06:00:26.319032+0000","last_undegraded":"2023-10-09T06:00:26.319032+0000","last_fullsized":"2023-10-09T06:00:26.319032+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:54.587601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":621,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.316149+0000","last_change":"2023-10-09T06:00:07.173005+0000","last_active":"2023-10-09T06:00:26.316149+0000","last_peered":"2023-10-09T06:00:26.316149+0000","last_clean":"2023-10-09T06:00:26.316149+0000","last_became_active":"2023-10-09T05:59:40.327732+0000","last_became_peered":"2023-10-09T05:59:40.327732+0000","last_unstale":"2023-10-09T06:00:26.316149+0000","last_undegraded":"2023-10-09T06:00:26.316149+0000","last_fullsized":"2023-10-09T06:00:26.316149+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:07.172878+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:07.172878+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:14.811803+0000","scrub_duration":0.013883573,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":282,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.324960+0000","last_change":"2023-10-09T05:57:10.379258+0000","last_active":"2023-10-09T06:00:26.324960+0000","last_peered":"2023-10-09T06:00:26.324960+0000","last_clean":"2023-10-09T06:00:26.324960+0000","last_became_active":"2023-10-09T05:57:10.359745+0000","last_became_peered":"2023-10-09T05:57:10.359745+0000","last_unstale":"2023-10-09T06:00:26.324960+0000","last_undegraded":"2023-10-09T06:00:26.324960+0000","last_fullsized":"2023-10-09T06:00:26.324960+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:14.750256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":333,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320343+0000","last_change":"2023-10-09T05:57:23.481065+0000","last_active":"2023-10-09T06:00:26.320343+0000","last_peered":"2023-10-09T06:00:26.320343+0000","last_clean":"2023-10-09T06:00:26.320343+0000","last_became_active":"2023-10-09T05:57:23.480203+0000","last_became_peered":"2023-10-09T05:57:23.480203+0000","last_unstale":"2023-10-09T06:00:26.320343+0000","last_undegraded":"2023-10-09T06:00:26.320343+0000","last_fullsized":"2023-10-09T06:00:26.320343+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:15:53.196457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":4954325016,"num_objects":75598,"num_object_clones":0,"num_object_copies":226794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":75598,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":135104,"num_write_kb":6742401,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14871179264,"data_stored":14871167048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89999,"ondisk_log_size":89999,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":10792,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234253,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2400176,"kb_used_data":2102888,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":91344976,"statfs":{"total":95995035648,"available":93537255424,"internally_reserved":0,"allocated":2153357312,"data_stored":2152313694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:04 2023","interfaces":[{"interface":"back","average":{"1min":13.327999999999999,"5min":3.7709999999999999,"15min":2.4609999999999999},"min":{"1min":0.22500000000000001,"5min":0.22,"15min":0.22},"max":{"1min":134.59800000000001,"5min":134.59800000000001,"15min":134.59800000000001},"last":0.27100000000000002},{"interface":"front","average":{"1min":13.724,"5min":3.8199999999999998,"15min":2.3919999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":135.04400000000001,"5min":135.04400000000001,"15min":135.04400000000001},"last":0.25600000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:00:36 2023","interfaces":[{"interface":"back","average":{"1min":1.542,"5min":1.776,"15min":1.0580000000000001},"min":{"1min":0.311,"5min":0.249,"15min":0.22900000000000001},"max":{"1min":7.8849999999999998,"5min":35.113999999999997,"15min":35.113999999999997},"last":0.34899999999999998},{"interface":"front","average":{"1min":1.093,"5min":1.419,"15min":0.93100000000000005},"min":{"1min":0.36599999999999999,"5min":0.30099999999999999,"15min":0.28499999999999998},"max":{"1min":3.1680000000000001,"5min":34.768999999999998,"15min":34.768999999999998},"last":0.28699999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:00:04 2023","interfaces":[{"interface":"back","average":{"1min":3.2410000000000001,"5min":3.8599999999999999,"15min":2.238},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":30.309999999999999,"5min":119.91200000000001,"15min":119.91200000000001},"last":0.39200000000000002},{"interface":"front","average":{"1min":1.7609999999999999,"5min":3.71,"15min":2.2200000000000002},"min":{"1min":0.32900000000000001,"5min":0.318,"15min":0.318},"max":{"1min":9.2850000000000001,"5min":119.95099999999999,"15min":119.95099999999999},"last":0.626}]},{"osd":3,"last update":"Mon Oct 9 06:00:36 2023","interfaces":[{"interface":"back","average":{"1min":17.663,"5min":8.4049999999999994,"15min":4.8940000000000001},"min":{"1min":0.40799999999999997,"5min":0.35999999999999999,"15min":0.33300000000000002},"max":{"1min":127.83799999999999,"5min":157.58699999999999,"15min":157.58699999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":16.949999999999999,"5min":8.1340000000000003,"15min":4.7599999999999998},"min":{"1min":0.58599999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":131.16,"5min":160.02799999999999,"15min":160.02799999999999},"last":4.5650000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:00:20 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":0.85299999999999998,"15min":0.67800000000000005},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.24399999999999999},"max":{"1min":11.635,"5min":11.635,"15min":11.635},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.5069999999999999,"5min":0.85699999999999998,"15min":0.753},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":6.085,"5min":6.085,"15min":7.6120000000000001},"last":0.33000000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:00:20 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":1.0249999999999999,"15min":0.79600000000000004},"min":{"1min":0.29599999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":3.5230000000000001,"5min":15.622,"15min":15.622},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.71899999999999997,"15min":0.66200000000000003},"min":{"1min":0.313,"5min":0.313,"15min":0.30499999999999999},"max":{"1min":2.7170000000000001,"5min":2.7170000000000001,"15min":3.9380000000000002},"last":1.0840000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:00:20 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":0.97199999999999998,"15min":1.319},"min":{"1min":0.29099999999999998,"5min":0.28799999999999998,"15min":0.28299999999999997},"max":{"1min":3.5379999999999998,"5min":12.727,"15min":131.94399999999999},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.234,"5min":0.86799999999999999,"15min":1.3220000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.29999999999999999},"max":{"1min":5.3280000000000003,"5min":5.3280000000000003,"15min":135.23400000000001},"last":0.41099999999999998}]}]},{"osd":6,"up_from":46,"seq":197568495889,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2130784,"kb_used_data":1833496,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":91614368,"statfs":{"total":95995035648,"available":93813112832,"internally_reserved":0,"allocated":1877499904,"data_stored":1876471739,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,3],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:04 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":1.762,"15min":1.286},"min":{"1min":0.23200000000000001,"5min":0.224,"15min":0.224},"max":{"1min":4.9710000000000001,"5min":49.389000000000003,"15min":49.389000000000003},"last":61.793999999999997},{"interface":"front","average":{"1min":2.8889999999999998,"5min":2.0150000000000001,"15min":1.3999999999999999},"min":{"1min":0.36199999999999999,"5min":0.253,"15min":0.253},"max":{"1min":12.263999999999999,"5min":49.338999999999999,"15min":49.338999999999999},"last":61.805}]},{"osd":1,"last update":"Mon Oct 9 05:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.113,"15min":0.83099999999999996},"min":{"1min":0.29599999999999999,"5min":0.246,"15min":0.23899999999999999},"max":{"1min":2.5950000000000002,"5min":17.341000000000001,"15min":17.341000000000001},"last":0.434},{"interface":"front","average":{"1min":1.6719999999999999,"5min":1.214,"15min":0.89200000000000002},"min":{"1min":0.29499999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.2709999999999999,"5min":12.474,"15min":12.474},"last":0.40300000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:00:04 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":2.6589999999999998,"15min":1.4199999999999999},"min":{"1min":0.32600000000000001,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":6.5730000000000004,"5min":61.088999999999999,"15min":61.088999999999999},"last":0.38},{"interface":"front","average":{"1min":1.9690000000000001,"5min":2.6619999999999999,"15min":1.419},"min":{"1min":0.42499999999999999,"5min":0.32000000000000001,"15min":0.25600000000000001},"max":{"1min":4.0880000000000001,"5min":61.082000000000001,"15min":61.082000000000001},"last":0.36199999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:59:45 2023","interfaces":[{"interface":"back","average":{"1min":1.9710000000000001,"5min":4.6369999999999996,"15min":2.1459999999999999},"min":{"1min":0.32400000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":9.3140000000000001,"5min":93.626000000000005,"15min":93.626000000000005},"last":68.504999999999995},{"interface":"front","average":{"1min":2.3450000000000002,"5min":4.5570000000000004,"15min":2.093},"min":{"1min":0.39900000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":9.4920000000000009,"5min":93.643000000000001,"15min":93.643000000000001},"last":62.662999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:59:52 2023","interfaces":[{"interface":"back","average":{"1min":1.375,"5min":0.93700000000000006,"15min":0.77900000000000003},"min":{"1min":0.28299999999999997,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.1070000000000002,"5min":5.1070000000000002,"15min":7.9100000000000001},"last":0.313},{"interface":"front","average":{"1min":1.29,"5min":0.98099999999999998,"15min":0.78800000000000003},"min":{"1min":0.28299999999999997,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.1559999999999997,"5min":8.734,"15min":8.734},"last":0.34100000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:59:52 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":1.3979999999999999,"15min":0.93500000000000005},"min":{"1min":0.32800000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":3.2810000000000001,"5min":45.933999999999997,"15min":45.933999999999997},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.367,"5min":1.4550000000000001,"15min":0.98599999999999999},"min":{"1min":0.378,"5min":0.30499999999999999,"15min":0.30199999999999999},"max":{"1min":4.976,"5min":45.917999999999999,"15min":45.917999999999999},"last":0.39200000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:00:17 2023","interfaces":[{"interface":"back","average":{"1min":1.673,"5min":1.603,"15min":1.129},"min":{"1min":0.35599999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.6769999999999996,"5min":52.042999999999999,"15min":52.042999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.1859999999999999,"5min":1.4710000000000001,"15min":1.0580000000000001},"min":{"1min":0.35799999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.0759999999999996,"5min":51.259,"15min":51.259},"last":0.41799999999999998}]}]},{"osd":1,"up_from":0,"seq":382252089478,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":129,"seq":554050781280,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2652020,"kb_used_data":2422700,"kb_used_omap":29,"kb_used_meta":228962,"kb_avail":91093132,"statfs":{"total":95995035648,"available":93279367168,"internally_reserved":0,"allocated":2480844800,"data_stored":2479095670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30322,"internal_metadata":234457486},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,5],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:00:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":0.95099999999999996,"15min":0.81000000000000005},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":4.9850000000000003,"5min":8.3300000000000001,"15min":9.1720000000000006},"last":0.27400000000000002},{"interface":"front","average":{"1min":1.4119999999999999,"5min":0.97499999999999998,"15min":1.04},"min":{"1min":0.27200000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":6.9470000000000001,"5min":6.9470000000000001,"15min":6.9470000000000001},"last":0.32000000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:00:10 2023","interfaces":[{"interface":"back","average":{"1min":1.5680000000000001,"5min":1.452,"15min":1.127},"min":{"1min":0.32600000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.165,"5min":13.471,"15min":13.471},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.958,"5min":1.3999999999999999,"15min":1.1080000000000001},"min":{"1min":0.29099999999999998,"5min":0.224,"15min":0.224},"max":{"1min":6.9240000000000004,"5min":13.561,"15min":13.561},"last":1.1120000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:00:10 2023","interfaces":[{"interface":"back","average":{"1min":21.715,"5min":5.9450000000000003,"15min":2.4780000000000002},"min":{"1min":0.34599999999999997,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":134.185,"5min":134.185,"15min":134.185},"last":0.37},{"interface":"front","average":{"1min":21.986000000000001,"5min":5.7510000000000003,"15min":2.4910000000000001},"min":{"1min":0.432,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":134.16800000000001,"5min":134.16800000000001,"15min":134.16800000000001},"last":1.165}]},{"osd":4,"last update":"Mon Oct 9 06:00:10 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":3.9740000000000002,"15min":2.101},"min":{"1min":0.29999999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.9929999999999999,"5min":149.07400000000001,"15min":149.07400000000001},"last":0.434},{"interface":"front","average":{"1min":1.5349999999999999,"5min":4.0700000000000003,"15min":1.9259999999999999},"min":{"1min":0.34000000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.8979999999999997,"5min":149.12700000000001,"15min":149.12700000000001},"last":0.39800000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:00:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2050000000000001,"5min":2.8450000000000002,"15min":1.6779999999999999},"min":{"1min":0.34999999999999998,"5min":0.314,"15min":0.314},"max":{"1min":4.0730000000000004,"5min":100.25,"15min":100.25},"last":1.1990000000000001},{"interface":"front","average":{"1min":1.3360000000000001,"5min":2.8170000000000002,"15min":1.5600000000000001},"min":{"1min":0.41999999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":5.7510000000000003,"5min":102.40300000000001,"15min":102.40300000000001},"last":0.49399999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:00:10 2023","interfaces":[{"interface":"back","average":{"1min":1.3620000000000001,"5min":1.1180000000000001,"15min":1.181},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.1369999999999996,"5min":8.1470000000000002,"15min":8.1470000000000002},"last":7.3330000000000002},{"interface":"front","average":{"1min":1.2949999999999999,"5min":1.276,"15min":1.204},"min":{"1min":0.39000000000000001,"5min":0.378,"15min":0.378},"max":{"1min":4.0250000000000004,"5min":8.0510000000000002,"15min":9.141},"last":0.46500000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:00:10 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":2.9790000000000001,"15min":1.891},"min":{"1min":0.46200000000000002,"5min":0.42399999999999999,"15min":0.41999999999999998},"max":{"1min":5.7030000000000003,"5min":103.866,"15min":103.866},"last":1.627},{"interface":"front","average":{"1min":1.6870000000000001,"5min":2.9140000000000001,"15min":1.825},"min":{"1min":0.40799999999999997,"5min":0.378,"15min":0.378},"max":{"1min":6.8529999999999998,"5min":103.848,"15min":103.848},"last":0.61399999999999999}]}]},{"osd":2,"up_from":94,"seq":403726925969,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2915628,"kb_used_data":2515748,"kb_used_omap":81,"kb_used_meta":399598,"kb_avail":90829524,"statfs":{"total":95995035648,"available":93009432576,"internally_reserved":0,"allocated":2576125952,"data_stored":2574497659,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":409188570},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,13],"upper_bound":512},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:18 2023","interfaces":[{"interface":"back","average":{"1min":5.5300000000000002,"5min":3.214,"15min":2.0379999999999998},"min":{"1min":0.32800000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":57.798999999999999,"5min":57.798999999999999,"15min":57.798999999999999},"last":4.8140000000000001},{"interface":"front","average":{"1min":6.5670000000000002,"5min":3.3820000000000001,"15min":2.0350000000000001},"min":{"1min":0.34499999999999997,"5min":0.23300000000000001,"15min":0.23100000000000001},"max":{"1min":57.210999999999999,"5min":57.210999999999999,"15min":57.210999999999999},"last":5.226}]},{"osd":1,"last update":"Mon Oct 9 06:00:07 2023","interfaces":[{"interface":"back","average":{"1min":1.964,"5min":1.1579999999999999,"15min":1.48},"min":{"1min":0.224,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.8949999999999996,"5min":9.8949999999999996,"15min":163.24199999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":2.6720000000000002,"5min":1.3779999999999999,"15min":1.4830000000000001},"min":{"1min":0.26300000000000001,"5min":0.223,"15min":0.214},"max":{"1min":12.278,"5min":12.278,"15min":163.27000000000001},"last":0.439}]},{"osd":3,"last update":"Mon Oct 9 06:00:39 2023","interfaces":[{"interface":"back","average":{"1min":2.0310000000000001,"5min":4.4779999999999998,"15min":2.6309999999999998},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.23899999999999999},"max":{"1min":7.6769999999999996,"5min":144.88200000000001,"15min":170.07599999999999},"last":0.63600000000000001},{"interface":"front","average":{"1min":2.524,"5min":4.4809999999999999,"15min":2.218},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.25},"max":{"1min":9.1790000000000003,"5min":140.79499999999999,"15min":140.79499999999999},"last":0.39200000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:00:07 2023","interfaces":[{"interface":"back","average":{"1min":1.4630000000000001,"5min":1.2070000000000001,"15min":1.0189999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28199999999999997},"max":{"1min":5.7690000000000001,"5min":8.6280000000000001,"15min":11.372999999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":2.7730000000000001,"5min":1.3939999999999999,"15min":1.0629999999999999},"min":{"1min":0.379,"5min":0.33200000000000002,"15min":0.26800000000000002},"max":{"1min":21.477,"5min":21.477,"15min":21.477},"last":0.52300000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:00:07 2023","interfaces":[{"interface":"back","average":{"1min":1.5069999999999999,"5min":1.589,"15min":1.121},"min":{"1min":0.34599999999999997,"5min":0.30599999999999999,"15min":0.28999999999999998},"max":{"1min":8.1869999999999994,"5min":8.8819999999999997,"15min":13.414},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.575,"5min":1.278,"15min":1.0549999999999999},"min":{"1min":0.32300000000000001,"5min":0.313,"15min":0.29799999999999999},"max":{"1min":5.8040000000000003,"5min":8.5570000000000004,"15min":13.438000000000001},"last":1.1060000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:00:07 2023","interfaces":[{"interface":"back","average":{"1min":2.7890000000000001,"5min":1.429,"15min":1.093},"min":{"1min":0.42199999999999999,"5min":0.31900000000000001,"15min":0.29399999999999998},"max":{"1min":21.524999999999999,"5min":21.524999999999999,"15min":21.524999999999999},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.4019999999999999,"5min":1.468,"15min":1.034},"min":{"1min":0.38100000000000001,"5min":0.37,"15min":0.34000000000000002},"max":{"1min":3.8279999999999998,"5min":8.9399999999999995,"15min":8.9399999999999995},"last":0.58299999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:00:07 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":2.2559999999999998,"15min":1.423},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":8.4149999999999991,"5min":55.052,"15min":55.052},"last":0.60899999999999999},{"interface":"front","average":{"1min":1.556,"5min":1.409,"15min":1.173},"min":{"1min":0.35899999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.8879999999999999,"5min":8.6010000000000009,"15min":40.113},"last":0.48299999999999998}]}]},{"osd":3,"up_from":99,"seq":425201762443,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3457412,"kb_used_data":3000444,"kb_used_omap":7,"kb_used_meta":456888,"kb_avail":90287740,"statfs":{"total":95995035648,"available":92454645760,"internally_reserved":0,"allocated":3072454656,"data_stored":3070625715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":467853928},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:11 2023","interfaces":[{"interface":"back","average":{"1min":2.532,"5min":1.3320000000000001,"15min":1.3759999999999999},"min":{"1min":0.25600000000000001,"5min":0.186,"15min":0.186},"max":{"1min":14.56,"5min":14.56,"15min":15.536},"last":9.218},{"interface":"front","average":{"1min":2.6110000000000002,"5min":1.7869999999999999,"15min":1.534},"min":{"1min":0.21299999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":14.747,"5min":14.747,"15min":14.747},"last":7.617}]},{"osd":1,"last update":"Mon Oct 9 06:00:44 2023","interfaces":[{"interface":"back","average":{"1min":2.165,"5min":1.4490000000000001,"15min":1.3400000000000001},"min":{"1min":0.29999999999999999,"5min":0.217,"15min":0.20999999999999999},"max":{"1min":5.9340000000000002,"5min":14.568,"15min":127.226},"last":4.8200000000000003},{"interface":"front","average":{"1min":7.9249999999999998,"5min":2.4319999999999999,"15min":1.7010000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.21099999999999999},"max":{"1min":106.95699999999999,"5min":106.95699999999999,"15min":127.468},"last":8.7400000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:00:44 2023","interfaces":[{"interface":"back","average":{"1min":8.4870000000000001,"5min":3.363,"15min":1.7030000000000001},"min":{"1min":0.23499999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":107.01300000000001,"5min":107.01300000000001,"15min":107.01300000000001},"last":0.38100000000000001},{"interface":"front","average":{"1min":8.9529999999999994,"5min":3.609,"15min":1.9399999999999999},"min":{"1min":0.27800000000000002,"5min":0.27400000000000002,"15min":0.25800000000000001},"max":{"1min":109.807,"5min":109.807,"15min":109.807},"last":10.541}]},{"osd":4,"last update":"Mon Oct 9 06:00:44 2023","interfaces":[{"interface":"back","average":{"1min":2.3820000000000001,"5min":3.0419999999999998,"15min":1.5209999999999999},"min":{"1min":0.33500000000000002,"5min":0.33000000000000002,"15min":0.28199999999999997},"max":{"1min":9.3070000000000004,"5min":142.994,"15min":142.994},"last":9.3070000000000004},{"interface":"front","average":{"1min":2.516,"5min":3.004,"15min":1.5109999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.0339999999999998,"5min":143.411,"15min":143.411},"last":6.0339999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:00:44 2023","interfaces":[{"interface":"back","average":{"1min":2.589,"5min":1.7130000000000001,"15min":1.0900000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":5.8810000000000002,"5min":14.404,"15min":14.404},"last":5.8810000000000002},{"interface":"front","average":{"1min":2.069,"5min":1.611,"15min":1.028},"min":{"1min":0.40500000000000003,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.8410000000000002,"5min":18.013999999999999,"15min":18.013999999999999},"last":3.2080000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:00:44 2023","interfaces":[{"interface":"back","average":{"1min":7.7919999999999998,"5min":4.5890000000000004,"15min":2.097},"min":{"1min":0.38800000000000001,"5min":0.376,"15min":0.30499999999999999},"max":{"1min":107.056,"5min":152.69,"15min":152.69},"last":4.8499999999999996},{"interface":"front","average":{"1min":8.2159999999999993,"5min":4.6589999999999998,"15min":2.1099999999999999},"min":{"1min":0.39700000000000002,"5min":0.318,"15min":0.318},"max":{"1min":107.08199999999999,"5min":152.70400000000001,"15min":152.70400000000001},"last":0.59799999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:00:44 2023","interfaces":[{"interface":"back","average":{"1min":7.7119999999999997,"5min":4.5510000000000002,"15min":2.0600000000000001},"min":{"1min":0.443,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":107.495,"5min":149.93600000000001,"15min":149.93600000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":8.3789999999999996,"5min":4.5469999999999997,"15min":2.0779999999999998},"min":{"1min":0.39000000000000001,"5min":0.379,"15min":0.379},"max":{"1min":109.858,"5min":150.12299999999999,"15min":150.12299999999999},"last":9.3399999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019163,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1827628,"kb_used_data":1525924,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":91917524,"statfs":{"total":95995035648,"available":94123544576,"internally_reserved":0,"allocated":1562546176,"data_stored":1561506363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:14 2023","interfaces":[{"interface":"back","average":{"1min":4.5640000000000001,"5min":4.2080000000000002,"15min":1.9570000000000001},"min":{"1min":0.254,"5min":0.222,"15min":0.222},"max":{"1min":64.703999999999994,"5min":128.511,"15min":128.511},"last":0.28999999999999998},{"interface":"front","average":{"1min":5.4320000000000004,"5min":4.6040000000000001,"15min":2.4750000000000001},"min":{"1min":0.34399999999999997,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":64.673000000000002,"5min":129.91800000000001,"15min":129.91800000000001},"last":0.33900000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:00:40 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":1.734,"15min":1.1040000000000001},"min":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.2549999999999999,"5min":77.951999999999998,"15min":77.951999999999998},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.24,"5min":2.0099999999999998,"15min":1.165},"min":{"1min":0.29299999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":5.4139999999999997,"5min":77.968000000000004,"15min":77.968000000000004},"last":0.91400000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:00:14 2023","interfaces":[{"interface":"back","average":{"1min":1.8220000000000001,"5min":2.4319999999999999,"15min":1.4790000000000001},"min":{"1min":0.28399999999999997,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.3579999999999997,"5min":136.15700000000001,"15min":136.15700000000001},"last":1.079},{"interface":"front","average":{"1min":1.8340000000000001,"5min":2.6349999999999998,"15min":1.5960000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":8.718,"5min":136.30500000000001,"15min":136.30500000000001},"last":0.39300000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:00:40 2023","interfaces":[{"interface":"back","average":{"1min":14.808999999999999,"5min":6.6879999999999997,"15min":2.9329999999999998},"min":{"1min":0.39300000000000002,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":144.62,"5min":173.73400000000001,"15min":173.73400000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":14.613,"5min":6.4089999999999998,"15min":2.8690000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.31},"max":{"1min":144.01900000000001,"5min":173.79599999999999,"15min":173.79599999999999},"last":0.41299999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:00:27 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.93300000000000005,"15min":0.73299999999999998},"min":{"1min":0.27200000000000002,"5min":0.26300000000000001,"15min":0.25900000000000001},"max":{"1min":2.8039999999999998,"5min":4.9349999999999996,"15min":10.012},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.91600000000000004,"15min":0.80700000000000005},"min":{"1min":0.27600000000000002,"5min":0.26100000000000001,"15min":0.254},"max":{"1min":3.0600000000000001,"5min":12.121,"15min":12.489000000000001},"last":0.32300000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:59:52 2023","interfaces":[{"interface":"back","average":{"1min":2.0339999999999998,"5min":1.0149999999999999,"15min":0.83499999999999996},"min":{"1min":0.28699999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":8.5429999999999993,"5min":8.5429999999999993,"15min":11.234999999999999},"last":0.432},{"interface":"front","average":{"1min":0.73599999999999999,"5min":0.91800000000000004,"15min":0.73699999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.468,"5min":8.8610000000000007,"15min":9.923},"last":0.30599999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:00:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":0.94199999999999995,"15min":0.89500000000000002},"min":{"1min":0.309,"5min":0.30099999999999999,"15min":0.28599999999999998},"max":{"1min":6.2050000000000001,"5min":6.2050000000000001,"15min":10.606},"last":0.371},{"interface":"front","average":{"1min":1.6819999999999999,"5min":1.093,"15min":0.85299999999999998},"min":{"1min":0.33300000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.4700000000000006,"5min":8.4700000000000006,"15min":8.4700000000000006},"last":0.39900000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757526,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1543840,"kb_used_data":1234264,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":92201312,"statfs":{"total":95995035648,"available":94414143488,"internally_reserved":0,"allocated":1263886336,"data_stored":1262855099,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2949999999999999,"5min":1.2929999999999999,"15min":0.86099999999999999},"min":{"1min":0.253,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.5129999999999999,"5min":24.956,"15min":24.956},"last":2.6760000000000002},{"interface":"front","average":{"1min":1.7709999999999999,"5min":1.639,"15min":1.046},"min":{"1min":0.32600000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":5.5510000000000002,"5min":24.975000000000001,"15min":24.975000000000001},"last":0.78500000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:00:37 2023","interfaces":[{"interface":"back","average":{"1min":1.4550000000000001,"5min":1.2589999999999999,"15min":1.7769999999999999},"min":{"1min":0.30499999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.6879999999999997,"5min":6.8410000000000002,"15min":114.247},"last":4.4390000000000001},{"interface":"front","average":{"1min":0.85099999999999998,"5min":0.77100000000000002,"15min":1.5580000000000001},"min":{"1min":0.30199999999999999,"5min":0.28100000000000003,"15min":0.26400000000000001},"max":{"1min":3.367,"5min":3.367,"15min":114.678},"last":0.56399999999999995}]},{"osd":2,"last update":"Mon Oct 9 06:00:05 2023","interfaces":[{"interface":"back","average":{"1min":4.0410000000000004,"5min":3.7850000000000001,"15min":2.2389999999999999},"min":{"1min":0.32900000000000001,"5min":0.28499999999999998,"15min":0.25},"max":{"1min":36.981999999999999,"5min":217.06399999999999,"15min":217.06399999999999},"last":5.6600000000000001},{"interface":"front","average":{"1min":2.899,"5min":3.5190000000000001,"15min":2.125},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.27600000000000002},"max":{"1min":37.104999999999997,"5min":217.018,"15min":217.018},"last":2.702}]},{"osd":3,"last update":"Mon Oct 9 06:00:37 2023","interfaces":[{"interface":"back","average":{"1min":11.707000000000001,"5min":5.9100000000000001,"15min":5.1390000000000002},"min":{"1min":0.34899999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":80.692999999999998,"5min":211.40199999999999,"15min":272.81299999999999},"last":0.36199999999999999},{"interface":"front","average":{"1min":11.638999999999999,"5min":6.117,"15min":5.3639999999999999},"min":{"1min":0.40899999999999997,"5min":0.317,"15min":0.317},"max":{"1min":80.766999999999996,"5min":211.578,"15min":273.00999999999999},"last":1.6319999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:00:33 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.79500000000000004,"15min":0.66200000000000003},"min":{"1min":0.308,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.2120000000000002,"5min":13.052,"15min":13.052},"last":0.24399999999999999},{"interface":"front","average":{"1min":0.79300000000000004,"5min":0.71599999999999997,"15min":0.63800000000000001},"min":{"1min":0.32100000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.069,"5min":3.879,"15min":13.050000000000001},"last":0.28399999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:00:42 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":2.165,"15min":1.1399999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.26600000000000001},"max":{"1min":4.1970000000000001,"5min":126.89400000000001,"15min":126.89400000000001},"last":0.32800000000000001},{"interface":"front","average":{"1min":1.2070000000000001,"5min":2.121,"15min":1.2070000000000001},"min":{"1min":0.36699999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.5949999999999998,"5min":126.616,"15min":126.616},"last":0.36699999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:00:22 2023","interfaces":[{"interface":"back","average":{"1min":8.6899999999999995,"5min":3.6509999999999998,"15min":1.772},"min":{"1min":0.47499999999999998,"5min":0.33100000000000002,"15min":0.29999999999999999},"max":{"1min":133.27099999999999,"5min":133.27099999999999,"15min":133.27099999999999},"last":0.34000000000000002},{"interface":"front","average":{"1min":8.3249999999999993,"5min":3.4430000000000001,"15min":1.579},"min":{"1min":0.378,"5min":0.34300000000000003,"15min":0.32000000000000001},"max":{"1min":135.09200000000001,"5min":135.09200000000001,"15min":135.09200000000001},"last":0.379}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1474,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2468810752,"data_stored":2468806680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2544500736,"data_stored":2544500736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3059879936,"data_stored":3059875864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1545863168,"data_stored":1545863168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1242431488,"data_stored":1242431488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1866338304,"data_stored":1866334232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2143354880,"data_stored":2143354880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:00:49.080 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:00:49.228 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:00:49.257 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:49.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:49.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:49.367 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:49.408 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:49.472 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:49.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:49.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:49.578 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:00:49.652 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:49.922 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:00:49.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:49.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:00:50.028 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:50.059 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:50.108 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:50.120 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:50.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:50.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:50.217 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:50.225 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:50.252 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:50.256 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:50.314 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:50.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:50.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:50.421 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:50.463 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:50.540 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:50.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:50.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:50.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:50 smithi064 ceph-mon[212750]: pgmap v742: 119 pgs: 119 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 924 op/s 2023-10-09T06:00:50.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:50 smithi064 ceph-mon[220044]: pgmap v742: 119 pgs: 119 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 924 op/s 2023-10-09T06:00:50.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:50 smithi156 ceph-mon[195101]: pgmap v742: 119 pgs: 119 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 924 op/s 2023-10-09T06:00:50.649 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:50.759 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:50.995 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:51.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:51.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:51.104 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:00:51.135 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:51.201 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:00:51.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:51.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:00:51.309 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:51.352 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:51.379 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:00:51.420 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:51.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:51.429 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:51 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3590321689' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:51.429 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:51 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:51.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:51.530 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:51.582 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:51.668 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:51.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:51.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:51.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:51 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3590321689' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:51.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:51 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:51.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:51 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3590321689' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:51.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:51 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:00:51.776 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:51.813 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:51.897 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:51.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:51.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:52.007 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:52.037 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:52.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:52 smithi156 ceph-mon[195101]: pgmap v743: 119 pgs: 119 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 971 op/s 2023-10-09T06:00:52.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:52 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:00:52.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:52 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T06:00:52.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:52 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:00:52.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:52 smithi156 ceph-mon[195101]: osdmap e315: 8 total, 8 up, 7 in 2023-10-09T06:00:52.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:52 smithi156 ceph-mon[195101]: failed to encode map e315 with expected crc 2023-10-09T06:00:52.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:52 smithi156 ceph-mon[195101]: failed to encode map e315 with expected crc 2023-10-09T06:00:52.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:52 smithi156 ceph-mon[195101]: failed to encode map e315 with expected crc 2023-10-09T06:00:52.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:52 smithi156 ceph-mon[195101]: failed to encode map e315 with expected crc 2023-10-09T06:00:52.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:52 smithi156 ceph-mon[195101]: failed to encode map e315 with expected crc 2023-10-09T06:00:52.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:52 smithi156 ceph-mon[195101]: failed to encode map e315 with expected crc 2023-10-09T06:00:52.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:52 smithi156 ceph-mon[195101]: failed to encode map e315 with expected crc 2023-10-09T06:00:52.697 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:52 smithi064 ceph-mon[212750]: pgmap v743: 119 pgs: 119 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 971 op/s 2023-10-09T06:00:52.697 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:52 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:00:52.697 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:52 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:00:52.697 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:52 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:00:52.697 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:52 smithi064 ceph-mon[212750]: osdmap e315: 8 total, 8 up, 7 in 2023-10-09T06:00:52.697 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:52 smithi064 ceph-mon[212750]: failed to encode map e315 with expected crc 2023-10-09T06:00:52.697 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:52 smithi064 ceph-mon[212750]: failed to encode map e315 with expected crc 2023-10-09T06:00:52.697 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:52 smithi064 ceph-mon[212750]: failed to encode map e315 with expected crc 2023-10-09T06:00:52.697 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:52 smithi064 ceph-mon[212750]: failed to encode map e315 with expected crc 2023-10-09T06:00:52.697 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:52 smithi064 ceph-mon[212750]: failed to encode map e315 with expected crc 2023-10-09T06:00:52.697 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:52 smithi064 ceph-mon[212750]: failed to encode map e315 with expected crc 2023-10-09T06:00:52.697 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:52 smithi064 ceph-mon[212750]: failed to encode map e315 with expected crc 2023-10-09T06:00:52.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:52 smithi064 ceph-mon[220044]: pgmap v743: 119 pgs: 119 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 971 op/s 2023-10-09T06:00:52.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:52 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:00:52.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:52 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T06:00:52.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:52 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:00:52.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:52 smithi064 ceph-mon[220044]: osdmap e315: 8 total, 8 up, 7 in 2023-10-09T06:00:52.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:52 smithi064 ceph-mon[220044]: failed to encode map e315 with expected crc 2023-10-09T06:00:52.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:52 smithi064 ceph-mon[220044]: failed to encode map e315 with expected crc 2023-10-09T06:00:52.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:52 smithi064 ceph-mon[220044]: failed to encode map e315 with expected crc 2023-10-09T06:00:52.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:52 smithi064 ceph-mon[220044]: failed to encode map e315 with expected crc 2023-10-09T06:00:52.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:52 smithi064 ceph-mon[220044]: failed to encode map e315 with expected crc 2023-10-09T06:00:52.699 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:52 smithi064 ceph-mon[220044]: failed to encode map e315 with expected crc 2023-10-09T06:00:52.699 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:52 smithi064 ceph-mon[220044]: failed to encode map e315 with expected crc 2023-10-09T06:00:53.207 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:53.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:53.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:53.320 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:53.369 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:54.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:54 smithi064 ceph-mon[212750]: pgmap v745: 119 pgs: 119 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 884 op/s 2023-10-09T06:00:54.594 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:54 smithi064 ceph-mon[220044]: pgmap v745: 119 pgs: 119 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 884 op/s 2023-10-09T06:00:54.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:54 smithi156 ceph-mon[195101]: pgmap v745: 119 pgs: 119 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 884 op/s 2023-10-09T06:00:54.778 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:54.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:54.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:54.898 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:54.979 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:55.150 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:55.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:55.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:55.266 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:55.298 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:55.364 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:55.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:55.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:55.457 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:00:55.473 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:55.500 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T06:00:55.556 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:55.608 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:00:55.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:55 smithi156 ceph-mon[195101]: pgmap v746: 119 pgs: 119 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 880 op/s 2023-10-09T06:00:55.652 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:00:55.699 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:00:55.716 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:00:55.716 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:00:55.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:55 smithi064 ceph-mon[212750]: pgmap v746: 119 pgs: 119 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 880 op/s 2023-10-09T06:00:55.720 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:00:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:00:55] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.43.0" 2023-10-09T06:00:55.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:55 smithi064 ceph-mon[220044]: pgmap v746: 119 pgs: 119 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 880 op/s 2023-10-09T06:00:55.726 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:55.729 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:00:55.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:55.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:55.747 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:00:55.785 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:00:55.826 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:00:55.838 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:55.869 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:00:55.904 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:00:55.914 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:55.923 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:00:55.923 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:00:55.923 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:00:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:00:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:00:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:00:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:00:55.924 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:00:56.078 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:56.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:56.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:56.192 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:00:56.238 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:56.319 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:00:56.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:56.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:00:56.422 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:56 smithi064 ceph-mon[212750]: from='client.35484 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:00:56.422 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:56 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3992195414' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:56.425 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:56.474 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:56.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:56 smithi156 ceph-mon[195101]: from='client.35484 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:00:56.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:56 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3992195414' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:56.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:56 smithi064 ceph-mon[220044]: from='client.35484 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:00:56.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:56 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3992195414' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:00:56.809 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:56.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:56.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:56.922 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:00:57.018 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:57.993 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:00:58.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:58.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:00:58.089 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:00:58.102 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:00:58.139 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:58.149 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":746,"stamp":"2023-10-09T06:00:54.725237+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4571994040,"num_objects":69779,"num_object_clones":0,"num_object_copies":209337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":69779,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":141008,"num_write_kb":6742430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":10792,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94988,"ondisk_log_size":94988,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":15760344,"kb_used_data":13450976,"kb_used_omap":118,"kb_used_meta":2309001,"kb_avail":640455720,"statfs":{"total":671965249536,"available":655826657280,"internally_reserved":0,"allocated":13773799424,"data_stored":13764449954,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":121648,"internal_metadata":2364417232},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-577437696,"num_objects":-8811,"num_object_clones":0,"num_object_copies":-26433,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8811,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8811,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003831"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":544,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.325268+0000","last_change":"2023-10-09T06:00:05.413041+0000","last_active":"2023-10-09T06:00:26.325268+0000","last_peered":"2023-10-09T06:00:26.325268+0000","last_clean":"2023-10-09T06:00:26.325268+0000","last_became_active":"2023-10-09T05:59:40.308608+0000","last_became_peered":"2023-10-09T05:59:40.308608+0000","last_unstale":"2023-10-09T06:00:26.325268+0000","last_undegraded":"2023-10-09T06:00:26.325268+0000","last_fullsized":"2023-10-09T06:00:26.325268+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:05.412941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:05.412941+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:53.518055+0000","scrub_duration":0.0061014329999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":359,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320534+0000","last_change":"2023-10-09T05:55:59.341215+0000","last_active":"2023-10-09T06:00:26.320534+0000","last_peered":"2023-10-09T06:00:26.320534+0000","last_clean":"2023-10-09T06:00:26.320534+0000","last_became_active":"2023-10-09T05:55:59.340856+0000","last_became_peered":"2023-10-09T05:55:59.340856+0000","last_unstale":"2023-10-09T06:00:26.320534+0000","last_undegraded":"2023-10-09T06:00:26.320534+0000","last_fullsized":"2023-10-09T06:00:26.320534+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:33.686454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":77,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.319377+0000","last_change":"2023-10-09T05:57:23.474348+0000","last_active":"2023-10-09T06:00:26.319377+0000","last_peered":"2023-10-09T06:00:26.319377+0000","last_clean":"2023-10-09T06:00:26.319377+0000","last_became_active":"2023-10-09T05:57:23.474010+0000","last_became_peered":"2023-10-09T05:57:23.474010+0000","last_unstale":"2023-10-09T06:00:26.319377+0000","last_undegraded":"2023-10-09T06:00:26.319377+0000","last_fullsized":"2023-10-09T06:00:26.319377+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:45.039777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":590,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320426+0000","last_change":"2023-10-09T05:59:59.067739+0000","last_active":"2023-10-09T06:00:26.320426+0000","last_peered":"2023-10-09T06:00:26.320426+0000","last_clean":"2023-10-09T06:00:26.320426+0000","last_became_active":"2023-10-09T05:59:40.315955+0000","last_became_peered":"2023-10-09T05:59:40.315955+0000","last_unstale":"2023-10-09T06:00:26.320426+0000","last_undegraded":"2023-10-09T06:00:26.320426+0000","last_fullsized":"2023-10-09T06:00:26.320426+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:59.067648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:59.067648+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:46:50.528412+0000","scrub_duration":0.0016747089999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":194,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.319310+0000","last_change":"2023-10-09T05:55:59.304893+0000","last_active":"2023-10-09T06:00:26.319310+0000","last_peered":"2023-10-09T06:00:26.319310+0000","last_clean":"2023-10-09T06:00:26.319310+0000","last_became_active":"2023-10-09T05:55:59.304449+0000","last_became_peered":"2023-10-09T05:55:59.304449+0000","last_unstale":"2023-10-09T06:00:26.319310+0000","last_undegraded":"2023-10-09T06:00:26.319310+0000","last_fullsized":"2023-10-09T06:00:26.319310+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:44:39.453982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":335,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.325546+0000","last_change":"2023-10-09T05:57:58.834014+0000","last_active":"2023-10-09T06:00:26.325546+0000","last_peered":"2023-10-09T06:00:26.325546+0000","last_clean":"2023-10-09T06:00:26.325546+0000","last_became_active":"2023-10-09T05:57:58.833718+0000","last_became_peered":"2023-10-09T05:57:58.833718+0000","last_unstale":"2023-10-09T06:00:26.325546+0000","last_undegraded":"2023-10-09T06:00:26.325546+0000","last_fullsized":"2023-10-09T06:00:26.325546+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:23.752353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":523,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320425+0000","last_change":"2023-10-09T05:59:58.065729+0000","last_active":"2023-10-09T06:00:26.320425+0000","last_peered":"2023-10-09T06:00:26.320425+0000","last_clean":"2023-10-09T06:00:26.320425+0000","last_became_active":"2023-10-09T05:59:40.313876+0000","last_became_peered":"2023-10-09T05:59:40.313876+0000","last_unstale":"2023-10-09T06:00:26.320425+0000","last_undegraded":"2023-10-09T06:00:26.320425+0000","last_fullsized":"2023-10-09T06:00:26.320425+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.065661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.065661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:23:10.873130+0000","scrub_duration":0.0056656750000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":77,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.315511+0000","last_change":"2023-10-09T05:57:23.476832+0000","last_active":"2023-10-09T06:00:26.315511+0000","last_peered":"2023-10-09T06:00:26.315511+0000","last_clean":"2023-10-09T06:00:26.315511+0000","last_became_active":"2023-10-09T05:57:23.475216+0000","last_became_peered":"2023-10-09T05:57:23.475216+0000","last_unstale":"2023-10-09T06:00:26.315511+0000","last_undegraded":"2023-10-09T06:00:26.315511+0000","last_fullsized":"2023-10-09T06:00:26.315511+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:48.279660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":341,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.327952+0000","last_change":"2023-10-09T05:55:59.335464+0000","last_active":"2023-10-09T06:00:38.327952+0000","last_peered":"2023-10-09T06:00:38.327952+0000","last_clean":"2023-10-09T06:00:38.327952+0000","last_became_active":"2023-10-09T05:55:59.335259+0000","last_became_peered":"2023-10-09T05:55:59.335259+0000","last_unstale":"2023-10-09T06:00:38.327952+0000","last_undegraded":"2023-10-09T06:00:38.327952+0000","last_fullsized":"2023-10-09T06:00:38.327952+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:28.536682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":613,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.328226+0000","last_change":"2023-10-09T06:00:06.384509+0000","last_active":"2023-10-09T06:00:26.328226+0000","last_peered":"2023-10-09T06:00:26.328226+0000","last_clean":"2023-10-09T06:00:26.328226+0000","last_became_active":"2023-10-09T05:59:40.322690+0000","last_became_peered":"2023-10-09T05:59:40.322690+0000","last_unstale":"2023-10-09T06:00:26.328226+0000","last_undegraded":"2023-10-09T06:00:26.328226+0000","last_fullsized":"2023-10-09T06:00:26.328226+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.384407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.384407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:37.280271+0000","scrub_duration":0.0040651960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":270,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.324874+0000","last_change":"2023-10-09T05:57:23.477980+0000","last_active":"2023-10-09T06:00:26.324874+0000","last_peered":"2023-10-09T06:00:26.324874+0000","last_clean":"2023-10-09T06:00:26.324874+0000","last_became_active":"2023-10-09T05:57:23.477654+0000","last_became_peered":"2023-10-09T05:57:23.477654+0000","last_unstale":"2023-10-09T06:00:26.324874+0000","last_undegraded":"2023-10-09T06:00:26.324874+0000","last_fullsized":"2023-10-09T06:00:26.324874+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:17.524683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":77,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.315586+0000","last_change":"2023-10-09T05:57:23.473644+0000","last_active":"2023-10-09T06:00:26.315586+0000","last_peered":"2023-10-09T06:00:26.315586+0000","last_clean":"2023-10-09T06:00:26.315586+0000","last_became_active":"2023-10-09T05:57:23.473391+0000","last_became_peered":"2023-10-09T05:57:23.473391+0000","last_unstale":"2023-10-09T06:00:26.315586+0000","last_undegraded":"2023-10-09T06:00:26.315586+0000","last_fullsized":"2023-10-09T06:00:26.315586+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:17.138034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":580,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.343164+0000","last_change":"2023-10-09T05:58:36.913116+0000","last_active":"2023-10-09T06:00:38.343164+0000","last_peered":"2023-10-09T06:00:38.343164+0000","last_clean":"2023-10-09T06:00:38.343164+0000","last_became_active":"2023-10-09T05:58:21.329980+0000","last_became_peered":"2023-10-09T05:58:21.329980+0000","last_unstale":"2023-10-09T06:00:38.343164+0000","last_undegraded":"2023-10-09T06:00:38.343164+0000","last_fullsized":"2023-10-09T06:00:38.343164+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.913004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.913004+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:32:18.561414+0000","scrub_duration":0.0017791980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":353,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.325958+0000","last_change":"2023-10-09T05:57:23.478483+0000","last_active":"2023-10-09T06:00:26.325958+0000","last_peered":"2023-10-09T06:00:26.325958+0000","last_clean":"2023-10-09T06:00:26.325958+0000","last_became_active":"2023-10-09T05:57:23.478122+0000","last_became_peered":"2023-10-09T05:57:23.478122+0000","last_unstale":"2023-10-09T06:00:26.325958+0000","last_undegraded":"2023-10-09T06:00:26.325958+0000","last_fullsized":"2023-10-09T06:00:26.325958+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:37.768717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":258,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.319224+0000","last_change":"2023-10-09T05:53:59.185702+0000","last_active":"2023-10-09T06:00:26.319224+0000","last_peered":"2023-10-09T06:00:26.319224+0000","last_clean":"2023-10-09T06:00:26.319224+0000","last_became_active":"2023-10-09T05:53:59.185373+0000","last_became_peered":"2023-10-09T05:53:59.185373+0000","last_unstale":"2023-10-09T06:00:26.319224+0000","last_undegraded":"2023-10-09T06:00:26.319224+0000","last_fullsized":"2023-10-09T06:00:26.319224+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:50.756967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":662,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.321921+0000","last_change":"2023-10-09T06:00:02.265614+0000","last_active":"2023-10-09T06:00:26.321921+0000","last_peered":"2023-10-09T06:00:26.321921+0000","last_clean":"2023-10-09T06:00:26.321921+0000","last_became_active":"2023-10-09T05:59:40.316490+0000","last_became_peered":"2023-10-09T05:59:40.316490+0000","last_unstale":"2023-10-09T06:00:26.321921+0000","last_undegraded":"2023-10-09T06:00:26.321921+0000","last_fullsized":"2023-10-09T06:00:26.321921+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:02.265460+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:02.265460+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:00:52.028473+0000","scrub_duration":0.010116385,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":154,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.328483+0000","last_change":"2023-10-09T05:53:59.219168+0000","last_active":"2023-10-09T06:00:26.328483+0000","last_peered":"2023-10-09T06:00:26.328483+0000","last_clean":"2023-10-09T06:00:26.328483+0000","last_became_active":"2023-10-09T05:53:59.200886+0000","last_became_peered":"2023-10-09T05:53:59.200886+0000","last_unstale":"2023-10-09T06:00:26.328483+0000","last_undegraded":"2023-10-09T06:00:26.328483+0000","last_fullsized":"2023-10-09T06:00:26.328483+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:45.515036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":209,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.328105+0000","last_change":"2023-10-09T05:55:59.305100+0000","last_active":"2023-10-09T06:00:38.328105+0000","last_peered":"2023-10-09T06:00:38.328105+0000","last_clean":"2023-10-09T06:00:38.328105+0000","last_became_active":"2023-10-09T05:55:59.304758+0000","last_became_peered":"2023-10-09T05:55:59.304758+0000","last_unstale":"2023-10-09T06:00:38.328105+0000","last_undegraded":"2023-10-09T06:00:38.328105+0000","last_fullsized":"2023-10-09T06:00:38.328105+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:54:32.510945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":447,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.325225+0000","last_change":"2023-10-09T05:59:57.339284+0000","last_active":"2023-10-09T06:00:26.325225+0000","last_peered":"2023-10-09T06:00:26.325225+0000","last_clean":"2023-10-09T06:00:26.325225+0000","last_became_active":"2023-10-09T05:59:40.334085+0000","last_became_peered":"2023-10-09T05:59:40.334085+0000","last_unstale":"2023-10-09T06:00:26.325225+0000","last_undegraded":"2023-10-09T06:00:26.325225+0000","last_fullsized":"2023-10-09T06:00:26.325225+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.339121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.339121+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:07.972002+0000","scrub_duration":0.010474344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":319,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320568+0000","last_change":"2023-10-09T05:53:59.219141+0000","last_active":"2023-10-09T06:00:26.320568+0000","last_peered":"2023-10-09T06:00:26.320568+0000","last_clean":"2023-10-09T06:00:26.320568+0000","last_became_active":"2023-10-09T05:53:59.197318+0000","last_became_peered":"2023-10-09T05:53:59.197318+0000","last_unstale":"2023-10-09T06:00:26.320568+0000","last_undegraded":"2023-10-09T06:00:26.320568+0000","last_fullsized":"2023-10-09T06:00:26.320568+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:39.910794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":323,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320746+0000","last_change":"2023-10-09T05:55:59.335824+0000","last_active":"2023-10-09T06:00:26.320746+0000","last_peered":"2023-10-09T06:00:26.320746+0000","last_clean":"2023-10-09T06:00:26.320746+0000","last_became_active":"2023-10-09T05:55:59.335478+0000","last_became_peered":"2023-10-09T05:55:59.335478+0000","last_unstale":"2023-10-09T06:00:26.320746+0000","last_undegraded":"2023-10-09T06:00:26.320746+0000","last_fullsized":"2023-10-09T06:00:26.320746+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:26.014501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":534,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.321589+0000","last_change":"2023-10-09T05:58:38.230201+0000","last_active":"2023-10-09T06:00:26.321589+0000","last_peered":"2023-10-09T06:00:26.321589+0000","last_clean":"2023-10-09T06:00:26.321589+0000","last_became_active":"2023-10-09T05:58:21.332729+0000","last_became_peered":"2023-10-09T05:58:21.332729+0000","last_unstale":"2023-10-09T06:00:26.321589+0000","last_undegraded":"2023-10-09T06:00:26.321589+0000","last_fullsized":"2023-10-09T06:00:26.321589+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:38.230079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:38.230079+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:33.800036+0000","scrub_duration":0.001618488,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":156,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.325717+0000","last_change":"2023-10-09T05:53:59.233898+0000","last_active":"2023-10-09T06:00:26.325717+0000","last_peered":"2023-10-09T06:00:26.325717+0000","last_clean":"2023-10-09T06:00:26.325717+0000","last_became_active":"2023-10-09T05:53:59.204175+0000","last_became_peered":"2023-10-09T05:53:59.204175+0000","last_unstale":"2023-10-09T06:00:26.325717+0000","last_undegraded":"2023-10-09T06:00:26.325717+0000","last_fullsized":"2023-10-09T06:00:26.325717+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:03.139318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":344,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.319470+0000","last_change":"2023-10-09T05:55:59.297087+0000","last_active":"2023-10-09T06:00:26.319470+0000","last_peered":"2023-10-09T06:00:26.319470+0000","last_clean":"2023-10-09T06:00:26.319470+0000","last_became_active":"2023-10-09T05:55:59.296768+0000","last_became_peered":"2023-10-09T05:55:59.296768+0000","last_unstale":"2023-10-09T06:00:26.319470+0000","last_undegraded":"2023-10-09T06:00:26.319470+0000","last_fullsized":"2023-10-09T06:00:26.319470+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:35.067705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":737,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.325706+0000","last_change":"2023-10-09T05:59:15.170543+0000","last_active":"2023-10-09T06:00:26.325706+0000","last_peered":"2023-10-09T06:00:26.325706+0000","last_clean":"2023-10-09T06:00:26.325706+0000","last_became_active":"2023-10-09T05:58:21.332533+0000","last_became_peered":"2023-10-09T05:58:21.332533+0000","last_unstale":"2023-10-09T06:00:26.325706+0000","last_undegraded":"2023-10-09T06:00:26.325706+0000","last_fullsized":"2023-10-09T06:00:26.325706+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:33.995695+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":316,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.321864+0000","last_change":"2023-10-09T05:57:23.473445+0000","last_active":"2023-10-09T06:00:26.321864+0000","last_peered":"2023-10-09T06:00:26.321864+0000","last_clean":"2023-10-09T06:00:26.321864+0000","last_became_active":"2023-10-09T05:57:23.473103+0000","last_became_peered":"2023-10-09T05:57:23.473103+0000","last_unstale":"2023-10-09T06:00:26.321864+0000","last_undegraded":"2023-10-09T06:00:26.321864+0000","last_fullsized":"2023-10-09T06:00:26.321864+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:39:53.825734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":237,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.328395+0000","last_change":"2023-10-09T05:56:49.237784+0000","last_active":"2023-10-09T06:00:26.328395+0000","last_peered":"2023-10-09T06:00:26.328395+0000","last_clean":"2023-10-09T06:00:26.328395+0000","last_became_active":"2023-10-09T05:56:49.237417+0000","last_became_peered":"2023-10-09T05:56:49.237417+0000","last_unstale":"2023-10-09T06:00:26.328395+0000","last_undegraded":"2023-10-09T06:00:26.328395+0000","last_fullsized":"2023-10-09T06:00:26.328395+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:33.129268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":565,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.338343+0000","last_change":"2023-10-09T05:58:37.295185+0000","last_active":"2023-10-09T06:00:38.338343+0000","last_peered":"2023-10-09T06:00:38.338343+0000","last_clean":"2023-10-09T06:00:38.338343+0000","last_became_active":"2023-10-09T05:58:21.327911+0000","last_became_peered":"2023-10-09T05:58:21.327911+0000","last_unstale":"2023-10-09T06:00:38.338343+0000","last_undegraded":"2023-10-09T06:00:38.338343+0000","last_fullsized":"2023-10-09T06:00:38.338343+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:37.295094+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:37.295094+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:10.882014+0000","scrub_duration":0.0017309510000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":150,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.327243+0000","last_change":"2023-10-09T05:53:59.206887+0000","last_active":"2023-10-09T06:00:26.327243+0000","last_peered":"2023-10-09T06:00:26.327243+0000","last_clean":"2023-10-09T06:00:26.327243+0000","last_became_active":"2023-10-09T05:53:59.206608+0000","last_became_peered":"2023-10-09T05:53:59.206608+0000","last_unstale":"2023-10-09T06:00:26.327243+0000","last_undegraded":"2023-10-09T06:00:26.327243+0000","last_fullsized":"2023-10-09T06:00:26.327243+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:48.975792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":297,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.325745+0000","last_change":"2023-10-09T05:57:58.828837+0000","last_active":"2023-10-09T06:00:26.325745+0000","last_peered":"2023-10-09T06:00:26.325745+0000","last_clean":"2023-10-09T06:00:26.325745+0000","last_became_active":"2023-10-09T05:57:58.828503+0000","last_became_peered":"2023-10-09T05:57:58.828503+0000","last_unstale":"2023-10-09T06:00:26.325745+0000","last_undegraded":"2023-10-09T06:00:26.325745+0000","last_fullsized":"2023-10-09T06:00:26.325745+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:08.332644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":615,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.327648+0000","last_change":"2023-10-09T05:58:46.725484+0000","last_active":"2023-10-09T06:00:38.327648+0000","last_peered":"2023-10-09T06:00:38.327648+0000","last_clean":"2023-10-09T06:00:38.327648+0000","last_became_active":"2023-10-09T05:58:21.332713+0000","last_became_peered":"2023-10-09T05:58:21.332713+0000","last_unstale":"2023-10-09T06:00:38.327648+0000","last_undegraded":"2023-10-09T06:00:38.327648+0000","last_fullsized":"2023-10-09T06:00:38.327648+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:46.725374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:46.725374+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:34.070739+0000","scrub_duration":0.002074771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":300,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.327479+0000","last_change":"2023-10-09T05:51:57.091639+0000","last_active":"2023-10-09T06:00:38.327479+0000","last_peered":"2023-10-09T06:00:38.327479+0000","last_clean":"2023-10-09T06:00:38.327479+0000","last_became_active":"2023-10-09T05:51:57.091284+0000","last_became_peered":"2023-10-09T05:51:57.091284+0000","last_unstale":"2023-10-09T06:00:38.327479+0000","last_undegraded":"2023-10-09T06:00:38.327479+0000","last_fullsized":"2023-10-09T06:00:38.327479+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:10.553453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":86,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.319481+0000","last_change":"2023-10-09T05:57:58.833939+0000","last_active":"2023-10-09T06:00:26.319481+0000","last_peered":"2023-10-09T06:00:26.319481+0000","last_clean":"2023-10-09T06:00:26.319481+0000","last_became_active":"2023-10-09T05:57:58.833567+0000","last_became_peered":"2023-10-09T05:57:58.833567+0000","last_unstale":"2023-10-09T06:00:26.319481+0000","last_undegraded":"2023-10-09T06:00:26.319481+0000","last_fullsized":"2023-10-09T06:00:26.319481+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:58.347572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"315'8984","reported_seq":9022,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T06:00:53.362540+0000","last_change":"2023-10-09T05:58:27.405856+0000","last_active":"2023-10-09T06:00:53.362540+0000","last_peered":"2023-10-09T06:00:53.362540+0000","last_clean":"2023-10-09T06:00:53.362540+0000","last_became_active":"2023-10-09T05:58:27.405560+0000","last_became_peered":"2023-10-09T05:58:27.405560+0000","last_unstale":"2023-10-09T06:00:53.362540+0000","last_undegraded":"2023-10-09T06:00:53.362540+0000","last_fullsized":"2023-10-09T06:00:53.362540+0000","mapping_epoch":285,"log_start":"295'3200","ondisk_log_start":"295'3200","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5784,"log_dups_size":0,"ondisk_log_size":5784,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:24.937739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281280512,"num_objects":4292,"num_object_clones":0,"num_object_copies":12876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4292,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8984,"num_write_kb":424832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":653,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.328306+0000","last_change":"2023-10-09T05:59:53.469494+0000","last_active":"2023-10-09T06:00:26.328306+0000","last_peered":"2023-10-09T06:00:26.328306+0000","last_clean":"2023-10-09T06:00:26.328306+0000","last_became_active":"2023-10-09T05:59:32.542653+0000","last_became_peered":"2023-10-09T05:59:32.542653+0000","last_unstale":"2023-10-09T06:00:26.328306+0000","last_undegraded":"2023-10-09T06:00:26.328306+0000","last_fullsized":"2023-10-09T06:00:26.328306+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:53.469394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:53.469394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:32.901454+0000","scrub_duration":0.0059716659999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":154,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.326131+0000","last_change":"2023-10-09T05:53:59.222136+0000","last_active":"2023-10-09T06:00:26.326131+0000","last_peered":"2023-10-09T06:00:26.326131+0000","last_clean":"2023-10-09T06:00:26.326131+0000","last_became_active":"2023-10-09T05:53:59.201853+0000","last_became_peered":"2023-10-09T05:53:59.201853+0000","last_unstale":"2023-10-09T06:00:26.326131+0000","last_undegraded":"2023-10-09T06:00:26.326131+0000","last_fullsized":"2023-10-09T06:00:26.326131+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:26.662031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":220,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.325502+0000","last_change":"2023-10-09T05:56:49.234185+0000","last_active":"2023-10-09T06:00:26.325502+0000","last_peered":"2023-10-09T06:00:26.325502+0000","last_clean":"2023-10-09T06:00:26.325502+0000","last_became_active":"2023-10-09T05:56:49.233884+0000","last_became_peered":"2023-10-09T05:56:49.233884+0000","last_unstale":"2023-10-09T06:00:26.325502+0000","last_undegraded":"2023-10-09T06:00:26.325502+0000","last_fullsized":"2023-10-09T06:00:26.325502+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:12.142436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.a","version":"314'8803","reported_seq":8842,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:49.889448+0000","last_change":"2023-10-09T05:58:27.406918+0000","last_active":"2023-10-09T06:00:49.889448+0000","last_peered":"2023-10-09T06:00:49.889448+0000","last_clean":"2023-10-09T06:00:49.889448+0000","last_became_active":"2023-10-09T05:58:27.406600+0000","last_became_peered":"2023-10-09T05:58:27.406600+0000","last_unstale":"2023-10-09T06:00:49.889448+0000","last_undegraded":"2023-10-09T06:00:49.889448+0000","last_fullsized":"2023-10-09T06:00:49.889448+0000","mapping_epoch":285,"log_start":"296'3800","ondisk_log_start":"296'3800","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5003,"log_dups_size":0,"ondisk_log_size":5003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:00.484611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286720000,"num_objects":4375,"num_object_clones":0,"num_object_copies":13125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8803,"num_write_kb":421696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":620,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.316072+0000","last_change":"2023-10-09T06:00:03.241479+0000","last_active":"2023-10-09T06:00:26.316072+0000","last_peered":"2023-10-09T06:00:26.316072+0000","last_clean":"2023-10-09T06:00:26.316072+0000","last_became_active":"2023-10-09T05:59:32.530753+0000","last_became_peered":"2023-10-09T05:59:32.530753+0000","last_unstale":"2023-10-09T06:00:26.316072+0000","last_undegraded":"2023-10-09T06:00:26.316072+0000","last_fullsized":"2023-10-09T06:00:26.316072+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:03.241335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:03.241335+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:54.822114+0000","scrub_duration":0.017780547000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":196,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.326096+0000","last_change":"2023-10-09T05:55:59.379174+0000","last_active":"2023-10-09T06:00:26.326096+0000","last_peered":"2023-10-09T06:00:26.326096+0000","last_clean":"2023-10-09T06:00:26.326096+0000","last_became_active":"2023-10-09T05:55:59.341932+0000","last_became_peered":"2023-10-09T05:55:59.341932+0000","last_unstale":"2023-10-09T06:00:26.326096+0000","last_undegraded":"2023-10-09T06:00:26.326096+0000","last_fullsized":"2023-10-09T06:00:26.326096+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.636171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":360,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.323153+0000","last_change":"2023-10-09T05:57:23.475146+0000","last_active":"2023-10-09T06:00:26.323153+0000","last_peered":"2023-10-09T06:00:26.323153+0000","last_clean":"2023-10-09T06:00:26.323153+0000","last_became_active":"2023-10-09T05:57:23.474776+0000","last_became_peered":"2023-10-09T05:57:23.474776+0000","last_unstale":"2023-10-09T06:00:26.323153+0000","last_undegraded":"2023-10-09T06:00:26.323153+0000","last_fullsized":"2023-10-09T06:00:26.323153+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:47.775743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"314'8749","reported_seq":8788,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:49.906390+0000","last_change":"2023-10-09T05:58:27.377562+0000","last_active":"2023-10-09T06:00:49.906390+0000","last_peered":"2023-10-09T06:00:49.906390+0000","last_clean":"2023-10-09T06:00:49.906390+0000","last_became_active":"2023-10-09T05:58:27.377253+0000","last_became_peered":"2023-10-09T05:58:27.377253+0000","last_unstale":"2023-10-09T06:00:49.906390+0000","last_undegraded":"2023-10-09T06:00:49.906390+0000","last_fullsized":"2023-10-09T06:00:49.906390+0000","mapping_epoch":285,"log_start":"295'3100","ondisk_log_start":"295'3100","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5649,"log_dups_size":0,"ondisk_log_size":5649,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:34.871962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299040768,"num_objects":4563,"num_object_clones":0,"num_object_copies":13689,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4563,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8749,"num_write_kb":425984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":737,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.328598+0000","last_change":"2023-10-09T05:58:41.753606+0000","last_active":"2023-10-09T06:00:38.328598+0000","last_peered":"2023-10-09T06:00:38.328598+0000","last_clean":"2023-10-09T06:00:38.328598+0000","last_became_active":"2023-10-09T05:58:21.332726+0000","last_became_peered":"2023-10-09T05:58:21.332726+0000","last_unstale":"2023-10-09T06:00:38.328598+0000","last_undegraded":"2023-10-09T06:00:38.328598+0000","last_fullsized":"2023-10-09T06:00:38.328598+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:41.753506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:41.753506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:15.947090+0000","scrub_duration":0.0022121929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":152,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.328492+0000","last_change":"2023-10-09T05:53:59.212161+0000","last_active":"2023-10-09T06:00:26.328492+0000","last_peered":"2023-10-09T06:00:26.328492+0000","last_clean":"2023-10-09T06:00:26.328492+0000","last_became_active":"2023-10-09T05:53:59.193814+0000","last_became_peered":"2023-10-09T05:53:59.193814+0000","last_unstale":"2023-10-09T06:00:26.328492+0000","last_undegraded":"2023-10-09T06:00:26.328492+0000","last_fullsized":"2023-10-09T06:00:26.328492+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:10.502408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":354,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.328060+0000","last_change":"2023-10-09T05:55:59.303352+0000","last_active":"2023-10-09T06:00:38.328060+0000","last_peered":"2023-10-09T06:00:38.328060+0000","last_clean":"2023-10-09T06:00:38.328060+0000","last_became_active":"2023-10-09T05:55:59.302923+0000","last_became_peered":"2023-10-09T05:55:59.302923+0000","last_unstale":"2023-10-09T06:00:38.328060+0000","last_undegraded":"2023-10-09T06:00:38.328060+0000","last_fullsized":"2023-10-09T06:00:38.328060+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:53.013376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"314'8721","reported_seq":8767,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:50.569642+0000","last_change":"2023-10-09T05:58:59.347359+0000","last_active":"2023-10-09T06:00:50.569642+0000","last_peered":"2023-10-09T06:00:50.569642+0000","last_clean":"2023-10-09T06:00:50.569642+0000","last_became_active":"2023-10-09T05:58:59.328422+0000","last_became_peered":"2023-10-09T05:58:59.328422+0000","last_unstale":"2023-10-09T06:00:50.569642+0000","last_undegraded":"2023-10-09T06:00:50.569642+0000","last_fullsized":"2023-10-09T06:00:50.569642+0000","mapping_epoch":292,"log_start":"295'3200","ondisk_log_start":"295'3200","created":285,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5521,"log_dups_size":0,"ondisk_log_size":5521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:31.541181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288161792,"num_objects":4397,"num_object_clones":0,"num_object_copies":13191,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4397,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8721,"num_write_kb":419776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":480,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320933+0000","last_change":"2023-10-09T05:59:57.043556+0000","last_active":"2023-10-09T06:00:26.320933+0000","last_peered":"2023-10-09T06:00:26.320933+0000","last_clean":"2023-10-09T06:00:26.320933+0000","last_became_active":"2023-10-09T05:59:32.525447+0000","last_became_peered":"2023-10-09T05:59:32.525447+0000","last_unstale":"2023-10-09T06:00:26.320933+0000","last_undegraded":"2023-10-09T06:00:26.320933+0000","last_fullsized":"2023-10-09T06:00:26.320933+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.043489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.043489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:17.051206+0000","scrub_duration":0.012927141,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":77,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.316160+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T06:00:26.316160+0000","last_peered":"2023-10-09T06:00:26.316160+0000","last_clean":"2023-10-09T06:00:26.316160+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T06:00:26.316160+0000","last_undegraded":"2023-10-09T06:00:26.316160+0000","last_fullsized":"2023-10-09T06:00:26.316160+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":325,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.322739+0000","last_change":"2023-10-09T05:57:23.473860+0000","last_active":"2023-10-09T06:00:26.322739+0000","last_peered":"2023-10-09T06:00:26.322739+0000","last_clean":"2023-10-09T06:00:26.322739+0000","last_became_active":"2023-10-09T05:57:23.473101+0000","last_became_peered":"2023-10-09T05:57:23.473101+0000","last_unstale":"2023-10-09T06:00:26.322739+0000","last_undegraded":"2023-10-09T06:00:26.322739+0000","last_fullsized":"2023-10-09T06:00:26.322739+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:05.682865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"314'8605","reported_seq":8644,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:49.930759+0000","last_change":"2023-10-09T05:58:27.376809+0000","last_active":"2023-10-09T06:00:49.930759+0000","last_peered":"2023-10-09T06:00:49.930759+0000","last_clean":"2023-10-09T06:00:49.930759+0000","last_became_active":"2023-10-09T05:58:27.376476+0000","last_became_peered":"2023-10-09T05:58:27.376476+0000","last_unstale":"2023-10-09T06:00:49.930759+0000","last_undegraded":"2023-10-09T06:00:49.930759+0000","last_fullsized":"2023-10-09T06:00:49.930759+0000","mapping_epoch":285,"log_start":"295'3000","ondisk_log_start":"295'3000","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5605,"log_dups_size":0,"ondisk_log_size":5605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:53:15.515971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287637504,"num_objects":4389,"num_object_clones":0,"num_object_copies":13167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8605,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":757,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.327365+0000","last_change":"2023-10-09T05:59:55.438018+0000","last_active":"2023-10-09T06:00:26.327365+0000","last_peered":"2023-10-09T06:00:26.327365+0000","last_clean":"2023-10-09T06:00:26.327365+0000","last_became_active":"2023-10-09T05:59:32.538631+0000","last_became_peered":"2023-10-09T05:59:32.538631+0000","last_unstale":"2023-10-09T06:00:26.327365+0000","last_undegraded":"2023-10-09T06:00:26.327365+0000","last_fullsized":"2023-10-09T06:00:26.327365+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:55.437947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:55.437947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:37.566205+0000","scrub_duration":0.0091262979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":81,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.315021+0000","last_change":"2023-10-09T05:57:23.504733+0000","last_active":"2023-10-09T06:00:26.315021+0000","last_peered":"2023-10-09T06:00:26.315021+0000","last_clean":"2023-10-09T06:00:26.315021+0000","last_became_active":"2023-10-09T05:57:23.468709+0000","last_became_peered":"2023-10-09T05:57:23.468709+0000","last_unstale":"2023-10-09T06:00:26.315021+0000","last_undegraded":"2023-10-09T06:00:26.315021+0000","last_fullsized":"2023-10-09T06:00:26.315021+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:00.459980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":283,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320604+0000","last_change":"2023-10-09T05:59:00.402498+0000","last_active":"2023-10-09T06:00:26.320604+0000","last_peered":"2023-10-09T06:00:26.320604+0000","last_clean":"2023-10-09T06:00:26.320604+0000","last_became_active":"2023-10-09T05:59:00.402218+0000","last_became_peered":"2023-10-09T05:59:00.402218+0000","last_unstale":"2023-10-09T06:00:26.320604+0000","last_undegraded":"2023-10-09T06:00:26.320604+0000","last_fullsized":"2023-10-09T06:00:26.320604+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:53.653861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"315'9021","reported_seq":9061,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T06:00:52.556217+0000","last_change":"2023-10-09T05:58:27.405765+0000","last_active":"2023-10-09T06:00:52.556217+0000","last_peered":"2023-10-09T06:00:52.556217+0000","last_clean":"2023-10-09T06:00:52.556217+0000","last_became_active":"2023-10-09T05:58:27.405447+0000","last_became_peered":"2023-10-09T05:58:27.405447+0000","last_unstale":"2023-10-09T06:00:52.556217+0000","last_undegraded":"2023-10-09T06:00:52.556217+0000","last_fullsized":"2023-10-09T06:00:52.556217+0000","mapping_epoch":285,"log_start":"290'1700","ondisk_log_start":"290'1700","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":7321,"log_dups_size":0,"ondisk_log_size":7321,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:52.191564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282132480,"num_objects":4305,"num_object_clones":0,"num_object_copies":12915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9021,"num_write_kb":426432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":502,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.314840+0000","last_change":"2023-10-09T05:59:58.282543+0000","last_active":"2023-10-09T06:00:26.314840+0000","last_peered":"2023-10-09T06:00:26.314840+0000","last_clean":"2023-10-09T06:00:26.314840+0000","last_became_active":"2023-10-09T05:59:32.552960+0000","last_became_peered":"2023-10-09T05:59:32.552960+0000","last_unstale":"2023-10-09T06:00:26.314840+0000","last_undegraded":"2023-10-09T06:00:26.314840+0000","last_fullsized":"2023-10-09T06:00:26.314840+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.282393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.282393+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:10.201323+0000","scrub_duration":0.01402403,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":325,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.327842+0000","last_change":"2023-10-09T05:55:59.320669+0000","last_active":"2023-10-09T06:00:38.327842+0000","last_peered":"2023-10-09T06:00:38.327842+0000","last_clean":"2023-10-09T06:00:38.327842+0000","last_became_active":"2023-10-09T05:55:59.320352+0000","last_became_peered":"2023-10-09T05:55:59.320352+0000","last_unstale":"2023-10-09T06:00:38.327842+0000","last_undegraded":"2023-10-09T06:00:38.327842+0000","last_fullsized":"2023-10-09T06:00:38.327842+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:09:14.227028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":360,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320882+0000","last_change":"2023-10-09T05:57:10.362295+0000","last_active":"2023-10-09T06:00:26.320882+0000","last_peered":"2023-10-09T06:00:26.320882+0000","last_clean":"2023-10-09T06:00:26.320882+0000","last_became_active":"2023-10-09T05:57:10.361249+0000","last_became_peered":"2023-10-09T05:57:10.361249+0000","last_unstale":"2023-10-09T06:00:26.320882+0000","last_undegraded":"2023-10-09T06:00:26.320882+0000","last_fullsized":"2023-10-09T06:00:26.320882+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:22.088261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"315'8975","reported_seq":9014,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T06:00:53.588364+0000","last_change":"2023-10-09T05:58:27.378739+0000","last_active":"2023-10-09T06:00:53.588364+0000","last_peered":"2023-10-09T06:00:53.588364+0000","last_clean":"2023-10-09T06:00:53.588364+0000","last_became_active":"2023-10-09T05:58:27.378459+0000","last_became_peered":"2023-10-09T05:58:27.378459+0000","last_unstale":"2023-10-09T06:00:53.588364+0000","last_undegraded":"2023-10-09T06:00:53.588364+0000","last_fullsized":"2023-10-09T06:00:53.588364+0000","mapping_epoch":285,"log_start":"295'3200","ondisk_log_start":"295'3200","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5775,"log_dups_size":0,"ondisk_log_size":5775,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:32.861544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275906584,"num_objects":4211,"num_object_clones":0,"num_object_copies":12633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4211,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8975,"num_write_kb":421889,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":583,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.319243+0000","last_change":"2023-10-09T05:59:56.373898+0000","last_active":"2023-10-09T06:00:38.319243+0000","last_peered":"2023-10-09T06:00:38.319243+0000","last_clean":"2023-10-09T06:00:38.319243+0000","last_became_active":"2023-10-09T05:59:32.541068+0000","last_became_peered":"2023-10-09T05:59:32.541068+0000","last_unstale":"2023-10-09T06:00:38.319243+0000","last_undegraded":"2023-10-09T06:00:38.319243+0000","last_fullsized":"2023-10-09T06:00:38.319243+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.373748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.373748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:53:42.272936+0000","scrub_duration":0.0024576450000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":294,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.322943+0000","last_change":"2023-10-09T05:54:00.619165+0000","last_active":"2023-10-09T06:00:26.322943+0000","last_peered":"2023-10-09T06:00:26.322943+0000","last_clean":"2023-10-09T06:00:26.322943+0000","last_became_active":"2023-10-09T05:54:00.618888+0000","last_became_peered":"2023-10-09T05:54:00.618888+0000","last_unstale":"2023-10-09T06:00:26.322943+0000","last_undegraded":"2023-10-09T06:00:26.322943+0000","last_fullsized":"2023-10-09T06:00:26.322943+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:07.175996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":333,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.321083+0000","last_change":"2023-10-09T05:57:23.483158+0000","last_active":"2023-10-09T06:00:26.321083+0000","last_peered":"2023-10-09T06:00:26.321083+0000","last_clean":"2023-10-09T06:00:26.321083+0000","last_became_active":"2023-10-09T05:57:23.482701+0000","last_became_peered":"2023-10-09T05:57:23.482701+0000","last_unstale":"2023-10-09T06:00:26.321083+0000","last_undegraded":"2023-10-09T06:00:26.321083+0000","last_fullsized":"2023-10-09T06:00:26.321083+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:51:08.624310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"315'8848","reported_seq":8885,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T06:00:53.602657+0000","last_change":"2023-10-09T05:58:27.376357+0000","last_active":"2023-10-09T06:00:53.602657+0000","last_peered":"2023-10-09T06:00:53.602657+0000","last_clean":"2023-10-09T06:00:53.602657+0000","last_became_active":"2023-10-09T05:58:27.376017+0000","last_became_peered":"2023-10-09T05:58:27.376017+0000","last_unstale":"2023-10-09T06:00:53.602657+0000","last_undegraded":"2023-10-09T06:00:53.602657+0000","last_fullsized":"2023-10-09T06:00:53.602657+0000","mapping_epoch":285,"log_start":"295'3000","ondisk_log_start":"295'3000","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5848,"log_dups_size":0,"ondisk_log_size":5848,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:30.067479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273416192,"num_objects":4172,"num_object_clones":0,"num_object_copies":12516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8848,"num_write_kb":416640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":468,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.327737+0000","last_change":"2023-10-09T05:59:52.129883+0000","last_active":"2023-10-09T06:00:38.327737+0000","last_peered":"2023-10-09T06:00:38.327737+0000","last_clean":"2023-10-09T06:00:38.327737+0000","last_became_active":"2023-10-09T05:59:32.529083+0000","last_became_peered":"2023-10-09T05:59:32.529083+0000","last_unstale":"2023-10-09T06:00:38.327737+0000","last_undegraded":"2023-10-09T06:00:38.327737+0000","last_fullsized":"2023-10-09T06:00:38.327737+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:52.129816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:52.129816+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:38.309831+0000","scrub_duration":0.0039938459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":77,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.323207+0000","last_change":"2023-10-09T05:57:23.469928+0000","last_active":"2023-10-09T06:00:26.323207+0000","last_peered":"2023-10-09T06:00:26.323207+0000","last_clean":"2023-10-09T06:00:26.323207+0000","last_became_active":"2023-10-09T05:57:23.469697+0000","last_became_peered":"2023-10-09T05:57:23.469697+0000","last_unstale":"2023-10-09T06:00:26.323207+0000","last_undegraded":"2023-10-09T06:00:26.323207+0000","last_fullsized":"2023-10-09T06:00:26.323207+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:21:07.179434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":194,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.322543+0000","last_change":"2023-10-09T05:55:59.409563+0000","last_active":"2023-10-09T06:00:26.322543+0000","last_peered":"2023-10-09T06:00:26.322543+0000","last_clean":"2023-10-09T06:00:26.322543+0000","last_became_active":"2023-10-09T05:55:59.409184+0000","last_became_peered":"2023-10-09T05:55:59.409184+0000","last_unstale":"2023-10-09T06:00:26.322543+0000","last_undegraded":"2023-10-09T06:00:26.322543+0000","last_fullsized":"2023-10-09T06:00:26.322543+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:39.513591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1","version":"314'8742","reported_seq":8779,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:49.761596+0000","last_change":"2023-10-09T05:58:27.377316+0000","last_active":"2023-10-09T06:00:49.761596+0000","last_peered":"2023-10-09T06:00:49.761596+0000","last_clean":"2023-10-09T06:00:49.761596+0000","last_became_active":"2023-10-09T05:58:27.377007+0000","last_became_peered":"2023-10-09T05:58:27.377007+0000","last_unstale":"2023-10-09T06:00:49.761596+0000","last_undegraded":"2023-10-09T06:00:49.761596+0000","last_fullsized":"2023-10-09T06:00:49.761596+0000","mapping_epoch":285,"log_start":"294'2900","ondisk_log_start":"294'2900","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5842,"log_dups_size":0,"ondisk_log_size":5842,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:10.982167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287309824,"num_objects":4384,"num_object_clones":0,"num_object_copies":13152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8742,"num_write_kb":420032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":738,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.328055+0000","last_change":"2023-10-09T05:59:56.092654+0000","last_active":"2023-10-09T06:00:38.328055+0000","last_peered":"2023-10-09T06:00:38.328055+0000","last_clean":"2023-10-09T06:00:38.328055+0000","last_became_active":"2023-10-09T05:59:32.529949+0000","last_became_peered":"2023-10-09T05:59:32.529949+0000","last_unstale":"2023-10-09T06:00:38.328055+0000","last_undegraded":"2023-10-09T06:00:38.328055+0000","last_fullsized":"2023-10-09T06:00:38.328055+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.092586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.092586+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:15:13.234903+0000","scrub_duration":0.0013484219999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":209,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.324713+0000","last_change":"2023-10-09T05:55:59.298953+0000","last_active":"2023-10-09T06:00:26.324713+0000","last_peered":"2023-10-09T06:00:26.324713+0000","last_clean":"2023-10-09T06:00:26.324713+0000","last_became_active":"2023-10-09T05:55:59.298624+0000","last_became_peered":"2023-10-09T05:55:59.298624+0000","last_unstale":"2023-10-09T06:00:26.324713+0000","last_undegraded":"2023-10-09T06:00:26.324713+0000","last_fullsized":"2023-10-09T06:00:26.324713+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:00.513870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":77,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.319161+0000","last_change":"2023-10-09T05:57:23.468480+0000","last_active":"2023-10-09T06:00:26.319161+0000","last_peered":"2023-10-09T06:00:26.319161+0000","last_clean":"2023-10-09T06:00:26.319161+0000","last_became_active":"2023-10-09T05:57:23.468179+0000","last_became_peered":"2023-10-09T05:57:23.468179+0000","last_unstale":"2023-10-09T06:00:26.319161+0000","last_undegraded":"2023-10-09T06:00:26.319161+0000","last_fullsized":"2023-10-09T06:00:26.319161+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:18.157188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":177,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320548+0000","last_change":"2023-10-09T05:56:49.249711+0000","last_active":"2023-10-09T06:00:26.320548+0000","last_peered":"2023-10-09T06:00:26.320548+0000","last_clean":"2023-10-09T06:00:26.320548+0000","last_became_active":"2023-10-09T05:56:49.249272+0000","last_became_peered":"2023-10-09T05:56:49.249272+0000","last_unstale":"2023-10-09T06:00:26.320548+0000","last_undegraded":"2023-10-09T06:00:26.320548+0000","last_fullsized":"2023-10-09T06:00:26.320548+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:42.596986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.7","version":"315'8713","reported_seq":8751,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T06:00:53.430680+0000","last_change":"2023-10-09T05:58:27.378776+0000","last_active":"2023-10-09T06:00:53.430680+0000","last_peered":"2023-10-09T06:00:53.430680+0000","last_clean":"2023-10-09T06:00:53.430680+0000","last_became_active":"2023-10-09T05:58:27.378504+0000","last_became_peered":"2023-10-09T05:58:27.378504+0000","last_unstale":"2023-10-09T06:00:53.430680+0000","last_undegraded":"2023-10-09T06:00:53.430680+0000","last_fullsized":"2023-10-09T06:00:53.430680+0000","mapping_epoch":285,"log_start":"295'2900","ondisk_log_start":"295'2900","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5813,"log_dups_size":0,"ondisk_log_size":5813,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:27.090596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270991360,"num_objects":4135,"num_object_clones":0,"num_object_copies":12405,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4135,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8713,"num_write_kb":411136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":517,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.321384+0000","last_change":"2023-10-09T05:58:36.225081+0000","last_active":"2023-10-09T06:00:26.321384+0000","last_peered":"2023-10-09T06:00:26.321384+0000","last_clean":"2023-10-09T06:00:26.321384+0000","last_became_active":"2023-10-09T05:56:49.237269+0000","last_became_peered":"2023-10-09T05:56:49.237269+0000","last_unstale":"2023-10-09T06:00:26.321384+0000","last_undegraded":"2023-10-09T06:00:26.321384+0000","last_fullsized":"2023-10-09T06:00:26.321384+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:02.908583+0000","scrub_duration":0.0016660799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":335,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.327916+0000","last_change":"2023-10-09T05:56:49.254337+0000","last_active":"2023-10-09T06:00:38.327916+0000","last_peered":"2023-10-09T06:00:38.327916+0000","last_clean":"2023-10-09T06:00:38.327916+0000","last_became_active":"2023-10-09T05:56:49.247892+0000","last_became_peered":"2023-10-09T05:56:49.247892+0000","last_unstale":"2023-10-09T06:00:38.327916+0000","last_undegraded":"2023-10-09T06:00:38.327916+0000","last_fullsized":"2023-10-09T06:00:38.327916+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:44:20.157638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":291,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320885+0000","last_change":"2023-10-09T05:51:57.203142+0000","last_active":"2023-10-09T06:00:26.320885+0000","last_peered":"2023-10-09T06:00:26.320885+0000","last_clean":"2023-10-09T06:00:26.320885+0000","last_became_active":"2023-10-09T05:51:57.202850+0000","last_became_peered":"2023-10-09T05:51:57.202850+0000","last_unstale":"2023-10-09T06:00:26.320885+0000","last_undegraded":"2023-10-09T06:00:26.320885+0000","last_fullsized":"2023-10-09T06:00:26.320885+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:51.090737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":207,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320451+0000","last_change":"2023-10-09T05:58:59.330420+0000","last_active":"2023-10-09T06:00:26.320451+0000","last_peered":"2023-10-09T06:00:26.320451+0000","last_clean":"2023-10-09T06:00:26.320451+0000","last_became_active":"2023-10-09T05:58:59.330137+0000","last_became_peered":"2023-10-09T05:58:59.330137+0000","last_unstale":"2023-10-09T06:00:26.320451+0000","last_undegraded":"2023-10-09T06:00:26.320451+0000","last_fullsized":"2023-10-09T06:00:26.320451+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:33.801594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.4","version":"314'8778","reported_seq":8817,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:49.935217+0000","last_change":"2023-10-09T05:58:27.375343+0000","last_active":"2023-10-09T06:00:49.935217+0000","last_peered":"2023-10-09T06:00:49.935217+0000","last_clean":"2023-10-09T06:00:49.935217+0000","last_became_active":"2023-10-09T05:58:27.375094+0000","last_became_peered":"2023-10-09T05:58:27.375094+0000","last_unstale":"2023-10-09T06:00:49.935217+0000","last_undegraded":"2023-10-09T06:00:49.935217+0000","last_fullsized":"2023-10-09T06:00:49.935217+0000","mapping_epoch":285,"log_start":"295'3200","ondisk_log_start":"295'3200","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5578,"log_dups_size":0,"ondisk_log_size":5578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:48:08.192351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288096256,"num_objects":4396,"num_object_clones":0,"num_object_copies":13188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8778,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":621,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.326136+0000","last_change":"2023-10-09T05:58:55.897880+0000","last_active":"2023-10-09T06:00:26.326136+0000","last_peered":"2023-10-09T06:00:26.326136+0000","last_clean":"2023-10-09T06:00:26.326136+0000","last_became_active":"2023-10-09T05:58:01.899103+0000","last_became_peered":"2023-10-09T05:58:01.899103+0000","last_unstale":"2023-10-09T06:00:26.326136+0000","last_undegraded":"2023-10-09T06:00:26.326136+0000","last_fullsized":"2023-10-09T06:00:26.326136+0000","mapping_epoch":268,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":269,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:55.897812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:55.897812+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:01.223576+0000","scrub_duration":0.0014511420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":54,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.323163+0000","last_change":"2023-10-09T05:58:59.325257+0000","last_active":"2023-10-09T06:00:26.323163+0000","last_peered":"2023-10-09T06:00:26.323163+0000","last_clean":"2023-10-09T06:00:26.323163+0000","last_became_active":"2023-10-09T05:58:59.324993+0000","last_became_peered":"2023-10-09T05:58:59.324993+0000","last_unstale":"2023-10-09T06:00:26.323163+0000","last_undegraded":"2023-10-09T06:00:26.323163+0000","last_fullsized":"2023-10-09T06:00:26.323163+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:20.825840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":133,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.327716+0000","last_change":"2023-10-09T05:58:59.328724+0000","last_active":"2023-10-09T06:00:38.327716+0000","last_peered":"2023-10-09T06:00:38.327716+0000","last_clean":"2023-10-09T06:00:38.327716+0000","last_became_active":"2023-10-09T05:58:59.328395+0000","last_became_peered":"2023-10-09T05:58:59.328395+0000","last_unstale":"2023-10-09T06:00:38.327716+0000","last_undegraded":"2023-10-09T06:00:38.327716+0000","last_fullsized":"2023-10-09T06:00:38.327716+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:37.528856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":209,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.322686+0000","last_change":"2023-10-09T05:56:49.238219+0000","last_active":"2023-10-09T06:00:26.322686+0000","last_peered":"2023-10-09T06:00:26.322686+0000","last_clean":"2023-10-09T06:00:26.322686+0000","last_became_active":"2023-10-09T05:56:49.237919+0000","last_became_peered":"2023-10-09T05:56:49.237919+0000","last_unstale":"2023-10-09T06:00:26.322686+0000","last_undegraded":"2023-10-09T06:00:26.322686+0000","last_fullsized":"2023-10-09T06:00:26.322686+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:44.599032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.6","version":"314'8792","reported_seq":8831,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:49.937131+0000","last_change":"2023-10-09T05:58:27.377502+0000","last_active":"2023-10-09T06:00:49.937131+0000","last_peered":"2023-10-09T06:00:49.937131+0000","last_clean":"2023-10-09T06:00:49.937131+0000","last_became_active":"2023-10-09T05:58:27.377195+0000","last_became_peered":"2023-10-09T05:58:27.377195+0000","last_unstale":"2023-10-09T06:00:49.937131+0000","last_undegraded":"2023-10-09T06:00:49.937131+0000","last_fullsized":"2023-10-09T06:00:49.937131+0000","mapping_epoch":285,"log_start":"295'3200","ondisk_log_start":"295'3200","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5592,"log_dups_size":0,"ondisk_log_size":5592,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:14.980773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289931264,"num_objects":4424,"num_object_clones":0,"num_object_copies":13272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8792,"num_write_kb":422912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":185,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.325522+0000","last_change":"2023-10-09T05:58:59.330109+0000","last_active":"2023-10-09T06:00:26.325522+0000","last_peered":"2023-10-09T06:00:26.325522+0000","last_clean":"2023-10-09T06:00:26.325522+0000","last_became_active":"2023-10-09T05:58:59.329808+0000","last_became_peered":"2023-10-09T05:58:59.329808+0000","last_unstale":"2023-10-09T06:00:26.325522+0000","last_undegraded":"2023-10-09T06:00:26.325522+0000","last_fullsized":"2023-10-09T06:00:26.325522+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:30.680622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":718,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.325787+0000","last_change":"2023-10-09T05:58:37.782237+0000","last_active":"2023-10-09T06:00:26.325787+0000","last_peered":"2023-10-09T06:00:26.325787+0000","last_clean":"2023-10-09T06:00:26.325787+0000","last_became_active":"2023-10-09T05:53:59.202004+0000","last_became_peered":"2023-10-09T05:53:59.202004+0000","last_unstale":"2023-10-09T06:00:26.325787+0000","last_undegraded":"2023-10-09T06:00:26.325787+0000","last_fullsized":"2023-10-09T06:00:26.325787+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:37.782173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:37.782173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:20.787963+0000","scrub_duration":0.001325908,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":77,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.323674+0000","last_change":"2023-10-09T05:57:23.479174+0000","last_active":"2023-10-09T06:00:26.323674+0000","last_peered":"2023-10-09T06:00:26.323674+0000","last_clean":"2023-10-09T06:00:26.323674+0000","last_became_active":"2023-10-09T05:57:23.478897+0000","last_became_peered":"2023-10-09T05:57:23.478897+0000","last_unstale":"2023-10-09T06:00:26.323674+0000","last_undegraded":"2023-10-09T06:00:26.323674+0000","last_fullsized":"2023-10-09T06:00:26.323674+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:00.160042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":277,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.328372+0000","last_change":"2023-10-09T05:53:59.195325+0000","last_active":"2023-10-09T06:00:38.328372+0000","last_peered":"2023-10-09T06:00:38.328372+0000","last_clean":"2023-10-09T06:00:38.328372+0000","last_became_active":"2023-10-09T05:53:59.194954+0000","last_became_peered":"2023-10-09T05:53:59.194954+0000","last_unstale":"2023-10-09T06:00:38.328372+0000","last_undegraded":"2023-10-09T06:00:38.328372+0000","last_fullsized":"2023-10-09T06:00:38.328372+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:23.228416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"314'8972","reported_seq":9009,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:50.544611+0000","last_change":"2023-10-09T05:58:27.375153+0000","last_active":"2023-10-09T06:00:50.544611+0000","last_peered":"2023-10-09T06:00:50.544611+0000","last_clean":"2023-10-09T06:00:50.544611+0000","last_became_active":"2023-10-09T05:58:27.374820+0000","last_became_peered":"2023-10-09T05:58:27.374820+0000","last_unstale":"2023-10-09T06:00:50.544611+0000","last_undegraded":"2023-10-09T06:00:50.544611+0000","last_fullsized":"2023-10-09T06:00:50.544611+0000","mapping_epoch":285,"log_start":"295'3500","ondisk_log_start":"295'3500","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5472,"log_dups_size":0,"ondisk_log_size":5472,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:08.834399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293339136,"num_objects":4476,"num_object_clones":0,"num_object_copies":13428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8972,"num_write_kb":430336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":510,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.328350+0000","last_change":"2023-10-09T05:58:40.724430+0000","last_active":"2023-10-09T06:00:38.328350+0000","last_peered":"2023-10-09T06:00:38.328350+0000","last_clean":"2023-10-09T06:00:38.328350+0000","last_became_active":"2023-10-09T05:57:54.796937+0000","last_became_peered":"2023-10-09T05:57:54.796937+0000","last_unstale":"2023-10-09T06:00:38.328350+0000","last_undegraded":"2023-10-09T06:00:38.328350+0000","last_fullsized":"2023-10-09T06:00:38.328350+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:40.724331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:40.724331+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:10.562541+0000","scrub_duration":0.0016947469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":180,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.325629+0000","last_change":"2023-10-09T05:55:59.300015+0000","last_active":"2023-10-09T06:00:26.325629+0000","last_peered":"2023-10-09T06:00:26.325629+0000","last_clean":"2023-10-09T06:00:26.325629+0000","last_became_active":"2023-10-09T05:55:59.299281+0000","last_became_peered":"2023-10-09T05:55:59.299281+0000","last_unstale":"2023-10-09T06:00:26.325629+0000","last_undegraded":"2023-10-09T06:00:26.325629+0000","last_fullsized":"2023-10-09T06:00:26.325629+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:50:32.465904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":252,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.321511+0000","last_change":"2023-10-09T05:57:23.471369+0000","last_active":"2023-10-09T06:00:26.321511+0000","last_peered":"2023-10-09T06:00:26.321511+0000","last_clean":"2023-10-09T06:00:26.321511+0000","last_became_active":"2023-10-09T05:57:23.471049+0000","last_became_peered":"2023-10-09T05:57:23.471049+0000","last_unstale":"2023-10-09T06:00:26.321511+0000","last_undegraded":"2023-10-09T06:00:26.321511+0000","last_fullsized":"2023-10-09T06:00:26.321511+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:21.208825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":369,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.323821+0000","last_change":"2023-10-09T05:57:10.348399+0000","last_active":"2023-10-09T06:00:38.323821+0000","last_peered":"2023-10-09T06:00:38.323821+0000","last_clean":"2023-10-09T06:00:38.323821+0000","last_became_active":"2023-10-09T05:57:10.348061+0000","last_became_peered":"2023-10-09T05:57:10.348061+0000","last_unstale":"2023-10-09T06:00:38.323821+0000","last_undegraded":"2023-10-09T06:00:38.323821+0000","last_fullsized":"2023-10-09T06:00:38.323821+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:12.453118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"314'8838","reported_seq":8877,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:49.922386+0000","last_change":"2023-10-09T05:58:27.405929+0000","last_active":"2023-10-09T06:00:49.922386+0000","last_peered":"2023-10-09T06:00:49.922386+0000","last_clean":"2023-10-09T06:00:49.922386+0000","last_became_active":"2023-10-09T05:58:27.405604+0000","last_became_peered":"2023-10-09T05:58:27.405604+0000","last_unstale":"2023-10-09T06:00:49.922386+0000","last_undegraded":"2023-10-09T06:00:49.922386+0000","last_fullsized":"2023-10-09T06:00:49.922386+0000","mapping_epoch":285,"log_start":"296'3805","ondisk_log_start":"296'3805","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5033,"log_dups_size":0,"ondisk_log_size":5033,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:19.748373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293601280,"num_objects":4480,"num_object_clones":0,"num_object_copies":13440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8838,"num_write_kb":426176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":728,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.328056+0000","last_change":"2023-10-09T05:59:54.074413+0000","last_active":"2023-10-09T06:00:38.328056+0000","last_peered":"2023-10-09T06:00:38.328056+0000","last_clean":"2023-10-09T06:00:38.328056+0000","last_became_active":"2023-10-09T05:59:32.538959+0000","last_became_peered":"2023-10-09T05:59:32.538959+0000","last_unstale":"2023-10-09T06:00:38.328056+0000","last_undegraded":"2023-10-09T06:00:38.328056+0000","last_fullsized":"2023-10-09T06:00:38.328056+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:54.074310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:54.074310+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:10.811269+0000","scrub_duration":0.011040648,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":148,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.328396+0000","last_change":"2023-10-09T05:53:59.201576+0000","last_active":"2023-10-09T06:00:26.328396+0000","last_peered":"2023-10-09T06:00:26.328396+0000","last_clean":"2023-10-09T06:00:26.328396+0000","last_became_active":"2023-10-09T05:53:59.201309+0000","last_became_peered":"2023-10-09T05:53:59.201309+0000","last_unstale":"2023-10-09T06:00:26.328396+0000","last_undegraded":"2023-10-09T06:00:26.328396+0000","last_fullsized":"2023-10-09T06:00:26.328396+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:54.956371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":138,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.321963+0000","last_change":"2023-10-09T05:58:59.328614+0000","last_active":"2023-10-09T06:00:26.321963+0000","last_peered":"2023-10-09T06:00:26.321963+0000","last_clean":"2023-10-09T06:00:26.321963+0000","last_became_active":"2023-10-09T05:58:59.328203+0000","last_became_peered":"2023-10-09T05:58:59.328203+0000","last_unstale":"2023-10-09T06:00:26.321963+0000","last_undegraded":"2023-10-09T06:00:26.321963+0000","last_fullsized":"2023-10-09T06:00:26.321963+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:21.572990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":241,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:34.918693+0000","last_change":"2023-10-09T05:53:59.188462+0000","last_active":"2023-10-09T06:00:34.918693+0000","last_peered":"2023-10-09T06:00:34.918693+0000","last_clean":"2023-10-09T06:00:34.918693+0000","last_became_active":"2023-10-09T05:53:59.188061+0000","last_became_peered":"2023-10-09T05:53:59.188061+0000","last_unstale":"2023-10-09T06:00:34.918693+0000","last_undegraded":"2023-10-09T06:00:34.918693+0000","last_fullsized":"2023-10-09T06:00:34.918693+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:19.167114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"314'8723","reported_seq":8761,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:49.923761+0000","last_change":"2023-10-09T05:58:27.377750+0000","last_active":"2023-10-09T06:00:49.923761+0000","last_peered":"2023-10-09T06:00:49.923761+0000","last_clean":"2023-10-09T06:00:49.923761+0000","last_became_active":"2023-10-09T05:58:27.377457+0000","last_became_peered":"2023-10-09T05:58:27.377457+0000","last_unstale":"2023-10-09T06:00:49.923761+0000","last_undegraded":"2023-10-09T06:00:49.923761+0000","last_fullsized":"2023-10-09T06:00:49.923761+0000","mapping_epoch":285,"log_start":"296'3700","ondisk_log_start":"296'3700","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5023,"log_dups_size":0,"ondisk_log_size":5023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:52:04.242389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289865728,"num_objects":4423,"num_object_clones":0,"num_object_copies":13269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8723,"num_write_kb":420672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":698,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320794+0000","last_change":"2023-10-09T05:59:54.020810+0000","last_active":"2023-10-09T06:00:26.320794+0000","last_peered":"2023-10-09T06:00:26.320794+0000","last_clean":"2023-10-09T06:00:26.320794+0000","last_became_active":"2023-10-09T05:59:32.542578+0000","last_became_peered":"2023-10-09T05:59:32.542578+0000","last_unstale":"2023-10-09T06:00:26.320794+0000","last_undegraded":"2023-10-09T06:00:26.320794+0000","last_fullsized":"2023-10-09T06:00:26.320794+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:54.020712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:54.020712+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:21.885126+0000","scrub_duration":0.0025782679999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":321,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.322668+0000","last_change":"2023-10-09T05:53:59.228948+0000","last_active":"2023-10-09T06:00:26.322668+0000","last_peered":"2023-10-09T06:00:26.322668+0000","last_clean":"2023-10-09T06:00:26.322668+0000","last_became_active":"2023-10-09T05:53:59.186359+0000","last_became_peered":"2023-10-09T05:53:59.186359+0000","last_unstale":"2023-10-09T06:00:26.322668+0000","last_undegraded":"2023-10-09T06:00:26.322668+0000","last_fullsized":"2023-10-09T06:00:26.322668+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:39.990576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":172,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.327909+0000","last_change":"2023-10-09T05:55:59.339957+0000","last_active":"2023-10-09T06:00:38.327909+0000","last_peered":"2023-10-09T06:00:38.327909+0000","last_clean":"2023-10-09T06:00:38.327909+0000","last_became_active":"2023-10-09T05:55:59.339614+0000","last_became_peered":"2023-10-09T05:55:59.339614+0000","last_unstale":"2023-10-09T06:00:38.327909+0000","last_undegraded":"2023-10-09T06:00:38.327909+0000","last_fullsized":"2023-10-09T06:00:38.327909+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:01.613113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":77,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.315711+0000","last_change":"2023-10-09T05:57:23.477143+0000","last_active":"2023-10-09T06:00:26.315711+0000","last_peered":"2023-10-09T06:00:26.315711+0000","last_clean":"2023-10-09T06:00:26.315711+0000","last_became_active":"2023-10-09T05:57:23.475655+0000","last_became_peered":"2023-10-09T05:57:23.475655+0000","last_unstale":"2023-10-09T06:00:26.315711+0000","last_undegraded":"2023-10-09T06:00:26.315711+0000","last_fullsized":"2023-10-09T06:00:26.315711+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:29.528023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"314'8683","reported_seq":8722,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:49.922300+0000","last_change":"2023-10-09T05:58:27.376100+0000","last_active":"2023-10-09T06:00:49.922300+0000","last_peered":"2023-10-09T06:00:49.922300+0000","last_clean":"2023-10-09T06:00:49.922300+0000","last_became_active":"2023-10-09T05:58:27.375806+0000","last_became_peered":"2023-10-09T05:58:27.375806+0000","last_unstale":"2023-10-09T06:00:49.922300+0000","last_undegraded":"2023-10-09T06:00:49.922300+0000","last_fullsized":"2023-10-09T06:00:49.922300+0000","mapping_epoch":285,"log_start":"296'3600","ondisk_log_start":"296'3600","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5083,"log_dups_size":0,"ondisk_log_size":5083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:56.466666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283967488,"num_objects":4333,"num_object_clones":0,"num_object_copies":12999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4333,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8683,"num_write_kb":416512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":571,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.315984+0000","last_change":"2023-10-09T05:58:34.333129+0000","last_active":"2023-10-09T06:00:26.315984+0000","last_peered":"2023-10-09T06:00:26.315984+0000","last_clean":"2023-10-09T06:00:26.315984+0000","last_became_active":"2023-10-09T05:57:26.502911+0000","last_became_peered":"2023-10-09T05:57:26.502911+0000","last_unstale":"2023-10-09T06:00:26.315984+0000","last_undegraded":"2023-10-09T06:00:26.315984+0000","last_fullsized":"2023-10-09T06:00:26.315984+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:25.230712+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":319,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.328561+0000","last_change":"2023-10-09T05:57:58.832790+0000","last_active":"2023-10-09T06:00:38.328561+0000","last_peered":"2023-10-09T06:00:38.328561+0000","last_clean":"2023-10-09T06:00:38.328561+0000","last_became_active":"2023-10-09T05:57:58.832471+0000","last_became_peered":"2023-10-09T05:57:58.832471+0000","last_unstale":"2023-10-09T06:00:38.328561+0000","last_undegraded":"2023-10-09T06:00:38.328561+0000","last_fullsized":"2023-10-09T06:00:38.328561+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:39.358174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":355,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T06:00:38.328768+0000","last_change":"2023-10-09T05:57:58.836050+0000","last_active":"2023-10-09T06:00:38.328768+0000","last_peered":"2023-10-09T06:00:38.328768+0000","last_clean":"2023-10-09T06:00:38.328768+0000","last_became_active":"2023-10-09T05:57:58.833691+0000","last_became_peered":"2023-10-09T05:57:58.833691+0000","last_unstale":"2023-10-09T06:00:38.328768+0000","last_undegraded":"2023-10-09T06:00:38.328768+0000","last_fullsized":"2023-10-09T06:00:38.328768+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:32.257849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":256,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.323012+0000","last_change":"2023-10-09T05:40:12.676587+0000","last_active":"2023-10-09T06:00:26.323012+0000","last_peered":"2023-10-09T06:00:26.323012+0000","last_clean":"2023-10-09T06:00:26.323012+0000","last_became_active":"2023-10-09T05:40:12.676330+0000","last_became_peered":"2023-10-09T05:40:12.676330+0000","last_unstale":"2023-10-09T06:00:26.323012+0000","last_undegraded":"2023-10-09T06:00:26.323012+0000","last_fullsized":"2023-10-09T06:00:26.323012+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:46.659556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":539,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.314757+0000","last_change":"2023-10-09T06:00:06.181968+0000","last_active":"2023-10-09T06:00:26.314757+0000","last_peered":"2023-10-09T06:00:26.314757+0000","last_clean":"2023-10-09T06:00:26.314757+0000","last_became_active":"2023-10-09T05:59:40.324653+0000","last_became_peered":"2023-10-09T05:59:40.324653+0000","last_unstale":"2023-10-09T06:00:26.314757+0000","last_undegraded":"2023-10-09T06:00:26.314757+0000","last_fullsized":"2023-10-09T06:00:26.314757+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.181811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.181811+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:05:16.987523+0000","scrub_duration":0.011553313000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":244,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.322886+0000","last_change":"2023-10-09T05:53:59.191302+0000","last_active":"2023-10-09T06:00:26.322886+0000","last_peered":"2023-10-09T06:00:26.322886+0000","last_clean":"2023-10-09T06:00:26.322886+0000","last_became_active":"2023-10-09T05:53:59.190911+0000","last_became_peered":"2023-10-09T05:53:59.190911+0000","last_unstale":"2023-10-09T06:00:26.322886+0000","last_undegraded":"2023-10-09T06:00:26.322886+0000","last_fullsized":"2023-10-09T06:00:26.322886+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:12.801201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":54,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.315900+0000","last_change":"2023-10-09T05:58:59.326172+0000","last_active":"2023-10-09T06:00:26.315900+0000","last_peered":"2023-10-09T06:00:26.315900+0000","last_clean":"2023-10-09T06:00:26.315900+0000","last_became_active":"2023-10-09T05:58:59.325795+0000","last_became_peered":"2023-10-09T05:58:59.325795+0000","last_unstale":"2023-10-09T06:00:26.315900+0000","last_undegraded":"2023-10-09T06:00:26.315900+0000","last_fullsized":"2023-10-09T06:00:26.315900+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:34.308820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":656,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.318719+0000","last_change":"2023-10-09T06:00:04.215425+0000","last_active":"2023-10-09T06:00:26.318719+0000","last_peered":"2023-10-09T06:00:26.318719+0000","last_clean":"2023-10-09T06:00:26.318719+0000","last_became_active":"2023-10-09T05:59:40.326685+0000","last_became_peered":"2023-10-09T05:59:40.326685+0000","last_unstale":"2023-10-09T06:00:26.318719+0000","last_undegraded":"2023-10-09T06:00:26.318719+0000","last_fullsized":"2023-10-09T06:00:26.318719+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:04.215270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:04.215270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:03.963694+0000","scrub_duration":0.002850652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":104,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320705+0000","last_change":"2023-10-09T05:56:49.244931+0000","last_active":"2023-10-09T06:00:26.320705+0000","last_peered":"2023-10-09T06:00:26.320705+0000","last_clean":"2023-10-09T06:00:26.320705+0000","last_became_active":"2023-10-09T05:56:49.244651+0000","last_became_peered":"2023-10-09T05:56:49.244651+0000","last_unstale":"2023-10-09T06:00:26.320705+0000","last_undegraded":"2023-10-09T06:00:26.320705+0000","last_fullsized":"2023-10-09T06:00:26.320705+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:53.906578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":350,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320958+0000","last_change":"2023-10-09T05:57:10.356351+0000","last_active":"2023-10-09T06:00:26.320958+0000","last_peered":"2023-10-09T06:00:26.320958+0000","last_clean":"2023-10-09T06:00:26.320958+0000","last_became_active":"2023-10-09T05:57:10.356037+0000","last_became_peered":"2023-10-09T05:57:10.356037+0000","last_unstale":"2023-10-09T06:00:26.320958+0000","last_undegraded":"2023-10-09T06:00:26.320958+0000","last_fullsized":"2023-10-09T06:00:26.320958+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:15.139912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":104,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320902+0000","last_change":"2023-10-09T05:56:49.245612+0000","last_active":"2023-10-09T06:00:26.320902+0000","last_peered":"2023-10-09T06:00:26.320902+0000","last_clean":"2023-10-09T06:00:26.320902+0000","last_became_active":"2023-10-09T05:56:49.245235+0000","last_became_peered":"2023-10-09T05:56:49.245235+0000","last_unstale":"2023-10-09T06:00:26.320902+0000","last_undegraded":"2023-10-09T06:00:26.320902+0000","last_fullsized":"2023-10-09T06:00:26.320902+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:29.016794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":317,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320402+0000","last_change":"2023-10-09T05:59:00.400894+0000","last_active":"2023-10-09T06:00:26.320402+0000","last_peered":"2023-10-09T06:00:26.320402+0000","last_clean":"2023-10-09T06:00:26.320402+0000","last_became_active":"2023-10-09T05:59:00.399730+0000","last_became_peered":"2023-10-09T05:59:00.399730+0000","last_unstale":"2023-10-09T06:00:26.320402+0000","last_undegraded":"2023-10-09T06:00:26.320402+0000","last_fullsized":"2023-10-09T06:00:26.320402+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:46.821954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":560,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.327433+0000","last_change":"2023-10-09T06:00:00.456225+0000","last_active":"2023-10-09T06:00:26.327433+0000","last_peered":"2023-10-09T06:00:26.327433+0000","last_clean":"2023-10-09T06:00:26.327433+0000","last_became_active":"2023-10-09T05:59:40.311408+0000","last_became_peered":"2023-10-09T05:59:40.311408+0000","last_unstale":"2023-10-09T06:00:26.327433+0000","last_undegraded":"2023-10-09T06:00:26.327433+0000","last_fullsized":"2023-10-09T06:00:26.327433+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:00.456132+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:00.456132+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:29.222405+0000","scrub_duration":0.0014992199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":77,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.319165+0000","last_change":"2023-10-09T05:57:23.479560+0000","last_active":"2023-10-09T06:00:26.319165+0000","last_peered":"2023-10-09T06:00:26.319165+0000","last_clean":"2023-10-09T06:00:26.319165+0000","last_became_active":"2023-10-09T05:57:23.479177+0000","last_became_peered":"2023-10-09T05:57:23.479177+0000","last_unstale":"2023-10-09T06:00:26.319165+0000","last_undegraded":"2023-10-09T06:00:26.319165+0000","last_fullsized":"2023-10-09T06:00:26.319165+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:41.925161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":77,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.319032+0000","last_change":"2023-10-09T05:57:23.475179+0000","last_active":"2023-10-09T06:00:26.319032+0000","last_peered":"2023-10-09T06:00:26.319032+0000","last_clean":"2023-10-09T06:00:26.319032+0000","last_became_active":"2023-10-09T05:57:23.474904+0000","last_became_peered":"2023-10-09T05:57:23.474904+0000","last_unstale":"2023-10-09T06:00:26.319032+0000","last_undegraded":"2023-10-09T06:00:26.319032+0000","last_fullsized":"2023-10-09T06:00:26.319032+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:54.587601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":621,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.316149+0000","last_change":"2023-10-09T06:00:07.173005+0000","last_active":"2023-10-09T06:00:26.316149+0000","last_peered":"2023-10-09T06:00:26.316149+0000","last_clean":"2023-10-09T06:00:26.316149+0000","last_became_active":"2023-10-09T05:59:40.327732+0000","last_became_peered":"2023-10-09T05:59:40.327732+0000","last_unstale":"2023-10-09T06:00:26.316149+0000","last_undegraded":"2023-10-09T06:00:26.316149+0000","last_fullsized":"2023-10-09T06:00:26.316149+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:07.172878+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:07.172878+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:14.811803+0000","scrub_duration":0.013883573,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":282,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.324960+0000","last_change":"2023-10-09T05:57:10.379258+0000","last_active":"2023-10-09T06:00:26.324960+0000","last_peered":"2023-10-09T06:00:26.324960+0000","last_clean":"2023-10-09T06:00:26.324960+0000","last_became_active":"2023-10-09T05:57:10.359745+0000","last_became_peered":"2023-10-09T05:57:10.359745+0000","last_unstale":"2023-10-09T06:00:26.324960+0000","last_undegraded":"2023-10-09T06:00:26.324960+0000","last_fullsized":"2023-10-09T06:00:26.324960+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:14.750256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":333,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T06:00:26.320343+0000","last_change":"2023-10-09T05:57:23.481065+0000","last_active":"2023-10-09T06:00:26.320343+0000","last_peered":"2023-10-09T06:00:26.320343+0000","last_clean":"2023-10-09T06:00:26.320343+0000","last_became_active":"2023-10-09T05:57:23.480203+0000","last_became_peered":"2023-10-09T05:57:23.480203+0000","last_unstale":"2023-10-09T06:00:26.320343+0000","last_undegraded":"2023-10-09T06:00:26.320343+0000","last_fullsized":"2023-10-09T06:00:26.320343+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:15:53.196457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":4571398168,"num_objects":69755,"num_object_clones":0,"num_object_copies":209265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":69755,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":140947,"num_write_kb":6742401,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13659549696,"data_stored":13659537480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89942,"ondisk_log_size":89942,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":10792,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234254,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2284272,"kb_used_data":1978728,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":91460880,"statfs":{"total":95995035648,"available":93655941120,"internally_reserved":0,"allocated":2026217472,"data_stored":2025173854,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:04 2023","interfaces":[{"interface":"back","average":{"1min":13.327999999999999,"5min":3.7709999999999999,"15min":2.4609999999999999},"min":{"1min":0.22500000000000001,"5min":0.22,"15min":0.22},"max":{"1min":134.59800000000001,"5min":134.59800000000001,"15min":134.59800000000001},"last":0.25700000000000001},{"interface":"front","average":{"1min":13.724,"5min":3.8199999999999998,"15min":2.3919999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":135.04400000000001,"5min":135.04400000000001,"15min":135.04400000000001},"last":2.3980000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:00:36 2023","interfaces":[{"interface":"back","average":{"1min":1.542,"5min":1.776,"15min":1.0580000000000001},"min":{"1min":0.311,"5min":0.249,"15min":0.22900000000000001},"max":{"1min":7.8849999999999998,"5min":35.113999999999997,"15min":35.113999999999997},"last":2.4489999999999998},{"interface":"front","average":{"1min":1.093,"5min":1.419,"15min":0.93100000000000005},"min":{"1min":0.36599999999999999,"5min":0.30099999999999999,"15min":0.28499999999999998},"max":{"1min":3.1680000000000001,"5min":34.768999999999998,"15min":34.768999999999998},"last":0.89700000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:00:04 2023","interfaces":[{"interface":"back","average":{"1min":3.2410000000000001,"5min":3.8599999999999999,"15min":2.238},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":30.309999999999999,"5min":119.91200000000001,"15min":119.91200000000001},"last":0.28899999999999998},{"interface":"front","average":{"1min":1.7609999999999999,"5min":3.71,"15min":2.2200000000000002},"min":{"1min":0.32900000000000001,"5min":0.318,"15min":0.318},"max":{"1min":9.2850000000000001,"5min":119.95099999999999,"15min":119.95099999999999},"last":0.873}]},{"osd":3,"last update":"Mon Oct 9 06:00:36 2023","interfaces":[{"interface":"back","average":{"1min":17.663,"5min":8.4049999999999994,"15min":4.8940000000000001},"min":{"1min":0.40799999999999997,"5min":0.35999999999999999,"15min":0.33300000000000002},"max":{"1min":127.83799999999999,"5min":157.58699999999999,"15min":157.58699999999999},"last":1.0720000000000001},{"interface":"front","average":{"1min":16.949999999999999,"5min":8.1340000000000003,"15min":4.7599999999999998},"min":{"1min":0.58599999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":131.16,"5min":160.02799999999999,"15min":160.02799999999999},"last":1.0129999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:00:20 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":0.85299999999999998,"15min":0.67800000000000005},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.24399999999999999},"max":{"1min":11.635,"5min":11.635,"15min":11.635},"last":0.83599999999999997},{"interface":"front","average":{"1min":1.5069999999999999,"5min":0.85699999999999998,"15min":0.753},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":6.085,"5min":6.085,"15min":7.6120000000000001},"last":0.48499999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:00:20 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":1.0249999999999999,"15min":0.79600000000000004},"min":{"1min":0.29599999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":3.5230000000000001,"5min":15.622,"15min":15.622},"last":2.3639999999999999},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.71899999999999997,"15min":0.66200000000000003},"min":{"1min":0.313,"5min":0.313,"15min":0.30499999999999999},"max":{"1min":2.7170000000000001,"5min":2.7170000000000001,"15min":3.9380000000000002},"last":2.1400000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:00:20 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":0.97199999999999998,"15min":1.319},"min":{"1min":0.29099999999999998,"5min":0.28799999999999998,"15min":0.28299999999999997},"max":{"1min":3.5379999999999998,"5min":12.727,"15min":131.94399999999999},"last":2.4220000000000002},{"interface":"front","average":{"1min":1.234,"5min":0.86799999999999999,"15min":1.3220000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.29999999999999999},"max":{"1min":5.3280000000000003,"5min":5.3280000000000003,"15min":135.23400000000001},"last":0.313}]}]},{"osd":6,"up_from":46,"seq":197568495891,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1929128,"kb_used_data":1619488,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":91816024,"statfs":{"total":95995035648,"available":94019608576,"internally_reserved":0,"allocated":1658355712,"data_stored":1657327456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:04 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":1.762,"15min":1.286},"min":{"1min":0.23200000000000001,"5min":0.224,"15min":0.224},"max":{"1min":4.9710000000000001,"5min":49.389000000000003,"15min":49.389000000000003},"last":165.124},{"interface":"front","average":{"1min":2.8889999999999998,"5min":2.0150000000000001,"15min":1.3999999999999999},"min":{"1min":0.36199999999999999,"5min":0.253,"15min":0.253},"max":{"1min":12.263999999999999,"5min":49.338999999999999,"15min":49.338999999999999},"last":165.221}]},{"osd":1,"last update":"Mon Oct 9 06:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.7310000000000001,"5min":1.29,"15min":0.90700000000000003},"min":{"1min":0.34599999999999997,"5min":0.246,"15min":0.23899999999999999},"max":{"1min":5.194,"5min":17.341000000000001,"15min":17.341000000000001},"last":0.434},{"interface":"front","average":{"1min":3.0430000000000001,"5min":1.6339999999999999,"15min":1.0549999999999999},"min":{"1min":0.315,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":22.445,"5min":22.445,"15min":22.445},"last":0.42199999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:00:04 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":2.6589999999999998,"15min":1.4199999999999999},"min":{"1min":0.32600000000000001,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":6.5730000000000004,"5min":61.088999999999999,"15min":61.088999999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.9690000000000001,"5min":2.6619999999999999,"15min":1.419},"min":{"1min":0.42499999999999999,"5min":0.32000000000000001,"15min":0.25600000000000001},"max":{"1min":4.0880000000000001,"5min":61.082000000000001,"15min":61.082000000000001},"last":0.50600000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:00:45 2023","interfaces":[{"interface":"back","average":{"1min":7.9550000000000001,"5min":4.3049999999999997,"15min":2.6400000000000001},"min":{"1min":0.49199999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":68.504999999999995,"5min":68.504999999999995,"15min":93.626000000000005},"last":0.45400000000000001},{"interface":"front","average":{"1min":7.1680000000000001,"5min":4.1319999999999997,"15min":2.532},"min":{"1min":0.314,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":62.662999999999997,"5min":62.662999999999997,"15min":93.643000000000001},"last":0.47299999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:00:52 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.99199999999999999,"15min":0.80600000000000005},"min":{"1min":0.29399999999999998,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":3.6429999999999998,"5min":5.1070000000000002,"15min":7.9100000000000001},"last":0.37},{"interface":"front","average":{"1min":1.079,"5min":1.0449999999999999,"15min":0.82499999999999996},"min":{"1min":0.311,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.085,"5min":8.734,"15min":8.734},"last":0.311}]},{"osd":5,"last update":"Mon Oct 9 06:00:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":1.4670000000000001,"15min":0.96699999999999997},"min":{"1min":0.29499999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.9699999999999998,"5min":45.933999999999997,"15min":45.933999999999997},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.68999999999999995,"5min":1.4059999999999999,"15min":0.997},"min":{"1min":0.33200000000000002,"5min":0.30499999999999999,"15min":0.30199999999999999},"max":{"1min":2.9860000000000002,"5min":45.917999999999999,"15min":45.917999999999999},"last":0.33800000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:00:17 2023","interfaces":[{"interface":"back","average":{"1min":1.673,"5min":1.603,"15min":1.129},"min":{"1min":0.35599999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.6769999999999996,"5min":52.042999999999999,"15min":52.042999999999999},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.1859999999999999,"5min":1.4710000000000001,"15min":1.0580000000000001},"min":{"1min":0.35799999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.0759999999999996,"5min":51.259,"15min":51.259},"last":0.35899999999999999}]}]},{"osd":1,"up_from":0,"seq":382252089479,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":129,"seq":554050781281,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2503092,"kb_used_data":2282284,"kb_used_omap":29,"kb_used_meta":220770,"kb_avail":91242060,"statfs":{"total":95995035648,"available":93431869440,"internally_reserved":0,"allocated":2337058816,"data_stored":2335309686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30322,"internal_metadata":226068878},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:00:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":0.95099999999999996,"15min":0.81000000000000005},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":4.9850000000000003,"5min":8.3300000000000001,"15min":9.1720000000000006},"last":2.7349999999999999},{"interface":"front","average":{"1min":1.4119999999999999,"5min":0.97499999999999998,"15min":1.04},"min":{"1min":0.27200000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":6.9470000000000001,"5min":6.9470000000000001,"15min":6.9470000000000001},"last":1.8740000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:00:10 2023","interfaces":[{"interface":"back","average":{"1min":1.5680000000000001,"5min":1.452,"15min":1.127},"min":{"1min":0.32600000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.165,"5min":13.471,"15min":13.471},"last":1.911},{"interface":"front","average":{"1min":1.958,"5min":1.3999999999999999,"15min":1.1080000000000001},"min":{"1min":0.29099999999999998,"5min":0.224,"15min":0.224},"max":{"1min":6.9240000000000004,"5min":13.561,"15min":13.561},"last":2.7789999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:00:10 2023","interfaces":[{"interface":"back","average":{"1min":21.715,"5min":5.9450000000000003,"15min":2.4780000000000002},"min":{"1min":0.34599999999999997,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":134.185,"5min":134.185,"15min":134.185},"last":1.9370000000000001},{"interface":"front","average":{"1min":21.986000000000001,"5min":5.7510000000000003,"15min":2.4910000000000001},"min":{"1min":0.432,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":134.16800000000001,"5min":134.16800000000001,"15min":134.16800000000001},"last":3.0419999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:00:10 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":3.9740000000000002,"15min":2.101},"min":{"1min":0.29999999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.9929999999999999,"5min":149.07400000000001,"15min":149.07400000000001},"last":2.6949999999999998},{"interface":"front","average":{"1min":1.5349999999999999,"5min":4.0700000000000003,"15min":1.9259999999999999},"min":{"1min":0.34000000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.8979999999999997,"5min":149.12700000000001,"15min":149.12700000000001},"last":1.952}]},{"osd":5,"last update":"Mon Oct 9 06:00:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2050000000000001,"5min":2.8450000000000002,"15min":1.6779999999999999},"min":{"1min":0.34999999999999998,"5min":0.314,"15min":0.314},"max":{"1min":4.0730000000000004,"5min":100.25,"15min":100.25},"last":2.7130000000000001},{"interface":"front","average":{"1min":1.3360000000000001,"5min":2.8170000000000002,"15min":1.5600000000000001},"min":{"1min":0.41999999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":5.7510000000000003,"5min":102.40300000000001,"15min":102.40300000000001},"last":2.7669999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:00:10 2023","interfaces":[{"interface":"back","average":{"1min":1.3620000000000001,"5min":1.1180000000000001,"15min":1.181},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.1369999999999996,"5min":8.1470000000000002,"15min":8.1470000000000002},"last":2.7509999999999999},{"interface":"front","average":{"1min":1.2949999999999999,"5min":1.276,"15min":1.204},"min":{"1min":0.39000000000000001,"5min":0.378,"15min":0.378},"max":{"1min":4.0250000000000004,"5min":8.0510000000000002,"15min":9.141},"last":2.8220000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:00:10 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":2.9790000000000001,"15min":1.891},"min":{"1min":0.46200000000000002,"5min":0.42399999999999999,"15min":0.41999999999999998},"max":{"1min":5.7030000000000003,"5min":103.866,"15min":103.866},"last":2.7970000000000002},{"interface":"front","average":{"1min":1.6870000000000001,"5min":2.9140000000000001,"15min":1.825},"min":{"1min":0.40799999999999997,"5min":0.378,"15min":0.378},"max":{"1min":6.8529999999999998,"5min":103.848,"15min":103.848},"last":1.9690000000000001}]}]},{"osd":2,"up_from":94,"seq":403726925971,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2636532,"kb_used_data":2218796,"kb_used_omap":81,"kb_used_meta":417646,"kb_avail":91108620,"statfs":{"total":95995035648,"available":93295226880,"internally_reserved":0,"allocated":2272047104,"data_stored":2270418720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":427669722},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:18 2023","interfaces":[{"interface":"back","average":{"1min":5.5300000000000002,"5min":3.214,"15min":2.0379999999999998},"min":{"1min":0.32800000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":57.798999999999999,"5min":57.798999999999999,"15min":57.798999999999999},"last":0.20799999999999999},{"interface":"front","average":{"1min":6.5670000000000002,"5min":3.3820000000000001,"15min":2.0350000000000001},"min":{"1min":0.34499999999999997,"5min":0.23300000000000001,"15min":0.23100000000000001},"max":{"1min":57.210999999999999,"5min":57.210999999999999,"15min":57.210999999999999},"last":1.726}]},{"osd":1,"last update":"Mon Oct 9 06:00:07 2023","interfaces":[{"interface":"back","average":{"1min":1.964,"5min":1.1579999999999999,"15min":1.48},"min":{"1min":0.224,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.8949999999999996,"5min":9.8949999999999996,"15min":163.24199999999999},"last":0.442},{"interface":"front","average":{"1min":2.6720000000000002,"5min":1.3779999999999999,"15min":1.4830000000000001},"min":{"1min":0.26300000000000001,"5min":0.223,"15min":0.214},"max":{"1min":12.278,"5min":12.278,"15min":163.27000000000001},"last":3.2690000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:00:39 2023","interfaces":[{"interface":"back","average":{"1min":2.0310000000000001,"5min":4.4779999999999998,"15min":2.6309999999999998},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.23899999999999999},"max":{"1min":7.6769999999999996,"5min":144.88200000000001,"15min":170.07599999999999},"last":9.1999999999999993},{"interface":"front","average":{"1min":2.524,"5min":4.4809999999999999,"15min":2.218},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.25},"max":{"1min":9.1790000000000003,"5min":140.79499999999999,"15min":140.79499999999999},"last":0.27600000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:00:07 2023","interfaces":[{"interface":"back","average":{"1min":1.4630000000000001,"5min":1.2070000000000001,"15min":1.0189999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28199999999999997},"max":{"1min":5.7690000000000001,"5min":8.6280000000000001,"15min":11.372999999999999},"last":0.32800000000000001},{"interface":"front","average":{"1min":2.7730000000000001,"5min":1.3939999999999999,"15min":1.0629999999999999},"min":{"1min":0.379,"5min":0.33200000000000002,"15min":0.26800000000000002},"max":{"1min":21.477,"5min":21.477,"15min":21.477},"last":1.7589999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:00:07 2023","interfaces":[{"interface":"back","average":{"1min":1.5069999999999999,"5min":1.589,"15min":1.121},"min":{"1min":0.34599999999999997,"5min":0.30599999999999999,"15min":0.28999999999999998},"max":{"1min":8.1869999999999994,"5min":8.8819999999999997,"15min":13.414},"last":1.036},{"interface":"front","average":{"1min":1.575,"5min":1.278,"15min":1.0549999999999999},"min":{"1min":0.32300000000000001,"5min":0.313,"15min":0.29799999999999999},"max":{"1min":5.8040000000000003,"5min":8.5570000000000004,"15min":13.438000000000001},"last":0.96999999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:00:07 2023","interfaces":[{"interface":"back","average":{"1min":2.7890000000000001,"5min":1.429,"15min":1.093},"min":{"1min":0.42199999999999999,"5min":0.31900000000000001,"15min":0.29399999999999998},"max":{"1min":21.524999999999999,"5min":21.524999999999999,"15min":21.524999999999999},"last":1.786},{"interface":"front","average":{"1min":1.4019999999999999,"5min":1.468,"15min":1.034},"min":{"1min":0.38100000000000001,"5min":0.37,"15min":0.34000000000000002},"max":{"1min":3.8279999999999998,"5min":8.9399999999999995,"15min":8.9399999999999995},"last":1.008}]},{"osd":7,"last update":"Mon Oct 9 06:00:07 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":2.2559999999999998,"15min":1.423},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":8.4149999999999991,"5min":55.052,"15min":55.052},"last":1.0669999999999999},{"interface":"front","average":{"1min":1.556,"5min":1.409,"15min":1.173},"min":{"1min":0.35899999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.8879999999999999,"5min":8.6010000000000009,"15min":40.113},"last":0.93700000000000006}]}]},{"osd":3,"up_from":99,"seq":425201762444,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3272196,"kb_used_data":2823484,"kb_used_omap":7,"kb_used_meta":448696,"kb_avail":90472956,"statfs":{"total":95995035648,"available":92644306944,"internally_reserved":0,"allocated":2891247616,"data_stored":2889418675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":459465320},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:11 2023","interfaces":[{"interface":"back","average":{"1min":2.532,"5min":1.3320000000000001,"15min":1.3759999999999999},"min":{"1min":0.25600000000000001,"5min":0.186,"15min":0.186},"max":{"1min":14.56,"5min":14.56,"15min":15.536},"last":0.34100000000000003},{"interface":"front","average":{"1min":2.6110000000000002,"5min":1.7869999999999999,"15min":1.534},"min":{"1min":0.21299999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":14.747,"5min":14.747,"15min":14.747},"last":0.99299999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:00:44 2023","interfaces":[{"interface":"back","average":{"1min":2.165,"5min":1.4490000000000001,"15min":1.3400000000000001},"min":{"1min":0.29999999999999999,"5min":0.217,"15min":0.20999999999999999},"max":{"1min":5.9340000000000002,"5min":14.568,"15min":127.226},"last":0.38200000000000001},{"interface":"front","average":{"1min":7.9249999999999998,"5min":2.4319999999999999,"15min":1.7010000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.21099999999999999},"max":{"1min":106.95699999999999,"5min":106.95699999999999,"15min":127.468},"last":2.1669999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:00:44 2023","interfaces":[{"interface":"back","average":{"1min":8.4870000000000001,"5min":3.363,"15min":1.7030000000000001},"min":{"1min":0.23499999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":107.01300000000001,"5min":107.01300000000001,"15min":107.01300000000001},"last":0.35499999999999998},{"interface":"front","average":{"1min":8.9529999999999994,"5min":3.609,"15min":1.9399999999999999},"min":{"1min":0.27800000000000002,"5min":0.27400000000000002,"15min":0.25800000000000001},"max":{"1min":109.807,"5min":109.807,"15min":109.807},"last":0.67000000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:00:44 2023","interfaces":[{"interface":"back","average":{"1min":2.3820000000000001,"5min":3.0419999999999998,"15min":1.5209999999999999},"min":{"1min":0.33500000000000002,"5min":0.33000000000000002,"15min":0.28199999999999997},"max":{"1min":9.3070000000000004,"5min":142.994,"15min":142.994},"last":0.63800000000000001},{"interface":"front","average":{"1min":2.516,"5min":3.004,"15min":1.5109999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.0339999999999998,"5min":143.411,"15min":143.411},"last":0.41799999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:00:44 2023","interfaces":[{"interface":"back","average":{"1min":2.589,"5min":1.7130000000000001,"15min":1.0900000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":5.8810000000000002,"5min":14.404,"15min":14.404},"last":0.46800000000000003},{"interface":"front","average":{"1min":2.069,"5min":1.611,"15min":1.028},"min":{"1min":0.40500000000000003,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.8410000000000002,"5min":18.013999999999999,"15min":18.013999999999999},"last":3.5750000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:00:44 2023","interfaces":[{"interface":"back","average":{"1min":7.7919999999999998,"5min":4.5890000000000004,"15min":2.097},"min":{"1min":0.38800000000000001,"5min":0.376,"15min":0.30499999999999999},"max":{"1min":107.056,"5min":152.69,"15min":152.69},"last":0.501},{"interface":"front","average":{"1min":8.2159999999999993,"5min":4.6589999999999998,"15min":2.1099999999999999},"min":{"1min":0.39700000000000002,"5min":0.318,"15min":0.318},"max":{"1min":107.08199999999999,"5min":152.70400000000001,"15min":152.70400000000001},"last":0.45400000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:00:44 2023","interfaces":[{"interface":"back","average":{"1min":7.7119999999999997,"5min":4.5510000000000002,"15min":2.0600000000000001},"min":{"1min":0.443,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":107.495,"5min":149.93600000000001,"15min":149.93600000000001},"last":0.39400000000000002},{"interface":"front","average":{"1min":8.3789999999999996,"5min":4.5469999999999997,"15min":2.0779999999999998},"min":{"1min":0.39000000000000001,"5min":0.379,"15min":0.379},"max":{"1min":109.858,"5min":150.12299999999999,"15min":150.12299999999999},"last":0.748}]}]},{"osd":4,"up_from":30,"seq":128849019164,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1740844,"kb_used_data":1435300,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92004308,"statfs":{"total":95995035648,"available":94212411392,"internally_reserved":0,"allocated":1469747200,"data_stored":1468707387,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:14 2023","interfaces":[{"interface":"back","average":{"1min":4.5640000000000001,"5min":4.2080000000000002,"15min":1.9570000000000001},"min":{"1min":0.254,"5min":0.222,"15min":0.222},"max":{"1min":64.703999999999994,"5min":128.511,"15min":128.511},"last":2.6779999999999999},{"interface":"front","average":{"1min":5.4320000000000004,"5min":4.6040000000000001,"15min":2.4750000000000001},"min":{"1min":0.34399999999999997,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":64.673000000000002,"5min":129.91800000000001,"15min":129.91800000000001},"last":0.71199999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:00:40 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":1.734,"15min":1.1040000000000001},"min":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.2549999999999999,"5min":77.951999999999998,"15min":77.951999999999998},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.24,"5min":2.0099999999999998,"15min":1.165},"min":{"1min":0.29299999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":5.4139999999999997,"5min":77.968000000000004,"15min":77.968000000000004},"last":0.61799999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:00:14 2023","interfaces":[{"interface":"back","average":{"1min":1.8220000000000001,"5min":2.4319999999999999,"15min":1.4790000000000001},"min":{"1min":0.28399999999999997,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.3579999999999997,"5min":136.15700000000001,"15min":136.15700000000001},"last":0.80000000000000004},{"interface":"front","average":{"1min":1.8340000000000001,"5min":2.6349999999999998,"15min":1.5960000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":8.718,"5min":136.30500000000001,"15min":136.30500000000001},"last":4.0030000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:00:40 2023","interfaces":[{"interface":"back","average":{"1min":14.808999999999999,"5min":6.6879999999999997,"15min":2.9329999999999998},"min":{"1min":0.39300000000000002,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":144.62,"5min":173.73400000000001,"15min":173.73400000000001},"last":0.85299999999999998},{"interface":"front","average":{"1min":14.613,"5min":6.4089999999999998,"15min":2.8690000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.31},"max":{"1min":144.01900000000001,"5min":173.79599999999999,"15min":173.79599999999999},"last":4.141}]},{"osd":5,"last update":"Mon Oct 9 06:00:27 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.93300000000000005,"15min":0.73299999999999998},"min":{"1min":0.27200000000000002,"5min":0.26300000000000001,"15min":0.25900000000000001},"max":{"1min":2.8039999999999998,"5min":4.9349999999999996,"15min":10.012},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.91600000000000004,"15min":0.80700000000000005},"min":{"1min":0.27600000000000002,"5min":0.26100000000000001,"15min":0.254},"max":{"1min":3.0600000000000001,"5min":12.121,"15min":12.489000000000001},"last":0.36799999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:59:52 2023","interfaces":[{"interface":"back","average":{"1min":2.0339999999999998,"5min":1.0149999999999999,"15min":0.83499999999999996},"min":{"1min":0.28699999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":8.5429999999999993,"5min":8.5429999999999993,"15min":11.234999999999999},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.73599999999999999,"5min":0.91800000000000004,"15min":0.73699999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.468,"5min":8.8610000000000007,"15min":9.923},"last":0.42999999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:00:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":0.94199999999999995,"15min":0.89500000000000002},"min":{"1min":0.309,"5min":0.30099999999999999,"15min":0.28599999999999998},"max":{"1min":6.2050000000000001,"5min":6.2050000000000001,"15min":10.606},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.6819999999999999,"5min":1.093,"15min":0.85299999999999998},"min":{"1min":0.33300000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.4700000000000006,"5min":8.4700000000000006,"15min":8.4700000000000006},"last":0.40500000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757528,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1394280,"kb_used_data":1092896,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":92350872,"statfs":{"total":95995035648,"available":94567292928,"internally_reserved":0,"allocated":1119125504,"data_stored":1118094176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2949999999999999,"5min":1.2929999999999999,"15min":0.86099999999999999},"min":{"1min":0.253,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.5129999999999999,"5min":24.956,"15min":24.956},"last":0.32700000000000001},{"interface":"front","average":{"1min":1.7709999999999999,"5min":1.639,"15min":1.046},"min":{"1min":0.32600000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":5.5510000000000002,"5min":24.975000000000001,"15min":24.975000000000001},"last":0.48799999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:00:37 2023","interfaces":[{"interface":"back","average":{"1min":1.4550000000000001,"5min":1.2589999999999999,"15min":1.7769999999999999},"min":{"1min":0.30499999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.6879999999999997,"5min":6.8410000000000002,"15min":114.247},"last":3.621},{"interface":"front","average":{"1min":0.85099999999999998,"5min":0.77100000000000002,"15min":1.5580000000000001},"min":{"1min":0.30199999999999999,"5min":0.28100000000000003,"15min":0.26400000000000001},"max":{"1min":3.367,"5min":3.367,"15min":114.678},"last":1.575}]},{"osd":2,"last update":"Mon Oct 9 06:00:05 2023","interfaces":[{"interface":"back","average":{"1min":4.0410000000000004,"5min":3.7850000000000001,"15min":2.2389999999999999},"min":{"1min":0.32900000000000001,"5min":0.28499999999999998,"15min":0.25},"max":{"1min":36.981999999999999,"5min":217.06399999999999,"15min":217.06399999999999},"last":0.54200000000000004},{"interface":"front","average":{"1min":2.899,"5min":3.5190000000000001,"15min":2.125},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.27600000000000002},"max":{"1min":37.104999999999997,"5min":217.018,"15min":217.018},"last":0.53100000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:00:37 2023","interfaces":[{"interface":"back","average":{"1min":11.707000000000001,"5min":5.9100000000000001,"15min":5.1390000000000002},"min":{"1min":0.34899999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":80.692999999999998,"5min":211.40199999999999,"15min":272.81299999999999},"last":1.7050000000000001},{"interface":"front","average":{"1min":11.638999999999999,"5min":6.117,"15min":5.3639999999999999},"min":{"1min":0.40899999999999997,"5min":0.317,"15min":0.317},"max":{"1min":80.766999999999996,"5min":211.578,"15min":273.00999999999999},"last":0.69499999999999995}]},{"osd":4,"last update":"Mon Oct 9 06:00:33 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.79500000000000004,"15min":0.66200000000000003},"min":{"1min":0.308,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.2120000000000002,"5min":13.052,"15min":13.052},"last":0.504},{"interface":"front","average":{"1min":0.79300000000000004,"5min":0.71599999999999997,"15min":0.63800000000000001},"min":{"1min":0.32100000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.069,"5min":3.879,"15min":13.050000000000001},"last":0.45100000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:00:42 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":2.165,"15min":1.1399999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.26600000000000001},"max":{"1min":4.1970000000000001,"5min":126.89400000000001,"15min":126.89400000000001},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.2070000000000001,"5min":2.121,"15min":1.2070000000000001},"min":{"1min":0.36699999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.5949999999999998,"5min":126.616,"15min":126.616},"last":0.51800000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:00:22 2023","interfaces":[{"interface":"back","average":{"1min":8.6899999999999995,"5min":3.6509999999999998,"15min":1.772},"min":{"1min":0.47499999999999998,"5min":0.33100000000000002,"15min":0.29999999999999999},"max":{"1min":133.27099999999999,"5min":133.27099999999999,"15min":133.27099999999999},"last":0.66600000000000004},{"interface":"front","average":{"1min":8.3249999999999993,"5min":3.4430000000000001,"15min":1.579},"min":{"1min":0.378,"5min":0.34300000000000003,"15min":0.32000000000000001},"max":{"1min":135.09200000000001,"5min":135.09200000000001,"15min":135.09200000000001},"last":0.46300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1474,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2320437248,"data_stored":2320433176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2245066752,"data_stored":2245066752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2872184832,"data_stored":2872180760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1456668672,"data_stored":1456668672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1098842112,"data_stored":1098842112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1651970048,"data_stored":1651965976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2014380032,"data_stored":2014380032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:00:58.157 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T06:00:58.158 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:00:58.158 INFO:tasks.thrashosds.thrasher:Killing osd 4, live_osds are [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T06:00:58.158 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-10-09T06:00:58.158 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.4 2023-10-09T06:00:58.207 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:00:58.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:58.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:00:58.316 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:58.340 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:58.365 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:58.437 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:58.440 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:58 smithi156 ceph-mon[195101]: pgmap v747: 119 pgs: 119 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 887 op/s 2023-10-09T06:00:58.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:58.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:58.442 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:00:58 smithi156 systemd[1]: Stopping Ceph osd.4 for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T06:00:58.542 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:58.603 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:58 smithi064 ceph-mon[212750]: pgmap v747: 119 pgs: 119 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 887 op/s 2023-10-09T06:00:58.604 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:58 smithi064 ceph-mon[220044]: pgmap v747: 119 pgs: 119 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 887 op/s 2023-10-09T06:00:58.612 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:58.717 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:58.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:58.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:58.831 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:58.870 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:00:58 smithi156 conmon[114036]: 2023-10-09T06:00:58.553+0000 7f120aef6700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:00:58.870 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:00:58 smithi156 conmon[114036]: 2023-10-09T06:00:58.553+0000 7f120aef6700 -1 osd.4 315 *** Got signal Terminated *** 2023-10-09T06:00:58.870 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:00:58 smithi156 conmon[114036]: 2023-10-09T06:00:58.553+0000 7f120aef6700 -1 osd.4 315 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:00:58.878 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:58.957 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:58.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:58.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:59.059 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:59.120 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:59.198 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:59.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:59.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:59.307 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:00:59.336 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:59.381 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:00:59.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:59.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:00:59.484 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:00:59.536 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:59.584 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:00:59.585 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:59 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:00:59.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:59.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:59 smithi064 ceph-mon[212750]: osd.4 marked itself down and dead 2023-10-09T06:00:59.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:00:59 smithi064 ceph-mon[212750]: pgmap v748: 119 pgs: 119 active+clean; 4.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 873 op/s 2023-10-09T06:00:59.586 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:59 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:00:59.586 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:59.586 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:59 smithi064 ceph-mon[220044]: osd.4 marked itself down and dead 2023-10-09T06:00:59.586 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:00:59 smithi064 ceph-mon[220044]: pgmap v748: 119 pgs: 119 active+clean; 4.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 873 op/s 2023-10-09T06:00:59.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:59.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:00:59.688 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:59.714 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:59.756 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:59.757 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:59 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:00:59.757 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:59.757 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:59 smithi156 ceph-mon[195101]: osd.4 marked itself down and dead 2023-10-09T06:00:59.757 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:00:59 smithi156 ceph-mon[195101]: pgmap v748: 119 pgs: 119 active+clean; 4.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 873 op/s 2023-10-09T06:00:59.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:59.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:00:59.859 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:00:59.885 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:59.886 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:00:59.931 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:00:59.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:00:59.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:00.034 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:00.043 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:01:00.063 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:00.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:00 smithi064 ceph-mon[220044]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:01:00.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:00 smithi064 ceph-mon[220044]: failed to encode map e316 with expected crc 2023-10-09T06:01:00.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:00 smithi064 ceph-mon[220044]: osdmap e316: 8 total, 7 up, 7 in 2023-10-09T06:01:00.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:00 smithi064 ceph-mon[220044]: failed to encode map e316 with expected crc 2023-10-09T06:01:00.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:00 smithi064 ceph-mon[212750]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:01:00.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:00 smithi064 ceph-mon[212750]: failed to encode map e316 with expected crc 2023-10-09T06:01:00.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:00 smithi064 ceph-mon[212750]: osdmap e316: 8 total, 7 up, 7 in 2023-10-09T06:01:00.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:00 smithi064 ceph-mon[212750]: failed to encode map e316 with expected crc 2023-10-09T06:01:00.870 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:01:00 smithi156 bash[278659]: 72313d594357a555cb89a8dac9de3a110b9ecf15e08784147a59c3fee5e90b3a 2023-10-09T06:01:00.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:00 smithi156 ceph-mon[195101]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:01:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:00 smithi156 ceph-mon[195101]: failed to encode map e316 with expected crc 2023-10-09T06:01:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:00 smithi156 ceph-mon[195101]: osdmap e316: 8 total, 7 up, 7 in 2023-10-09T06:01:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:00 smithi156 ceph-mon[195101]: failed to encode map e316 with expected crc 2023-10-09T06:01:01.263 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:01.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:01.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:01.369 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:01.403 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:01.547 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:01:01.563 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:01.570 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:01.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:01.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:01.575 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:01 smithi156 ceph-mon[195101]: failed to encode map e317 with expected crc 2023-10-09T06:01:01.575 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:01 smithi156 ceph-mon[195101]: osdmap e317: 8 total, 7 up, 7 in 2023-10-09T06:01:01.575 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:01 smithi156 ceph-mon[195101]: failed to encode map e317 with expected crc 2023-10-09T06:01:01.575 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:01 smithi156 ceph-mon[195101]: failed to encode map e317 with expected crc 2023-10-09T06:01:01.576 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:01 smithi156 ceph-mon[195101]: failed to encode map e317 with expected crc 2023-10-09T06:01:01.576 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:01 smithi156 ceph-mon[195101]: failed to encode map e317 with expected crc 2023-10-09T06:01:01.576 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:01 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1308115041' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:01:01.576 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:01 smithi156 ceph-mon[195101]: pgmap v751: 119 pgs: 6 active+undersized+degraded, 7 active+undersized, 23 peering, 10 stale+active+clean, 73 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 2.6 KiB/s rd, 6.9 KiB/s wr, 978 op/s; 3959/188823 objects degraded (2.097%); 4.3 KiB/s, 0 objects/s recovering 2023-10-09T06:01:01.675 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:01.719 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:01.761 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:01 smithi064 ceph-mon[212750]: failed to encode map e317 with expected crc 2023-10-09T06:01:01.762 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:01 smithi064 ceph-mon[212750]: osdmap e317: 8 total, 7 up, 7 in 2023-10-09T06:01:01.762 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:01 smithi064 ceph-mon[212750]: failed to encode map e317 with expected crc 2023-10-09T06:01:01.762 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:01 smithi064 ceph-mon[212750]: failed to encode map e317 with expected crc 2023-10-09T06:01:01.762 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:01 smithi064 ceph-mon[212750]: failed to encode map e317 with expected crc 2023-10-09T06:01:01.762 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:01 smithi064 ceph-mon[212750]: failed to encode map e317 with expected crc 2023-10-09T06:01:01.762 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:01 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1308115041' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:01:01.762 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:01 smithi064 ceph-mon[212750]: pgmap v751: 119 pgs: 6 active+undersized+degraded, 7 active+undersized, 23 peering, 10 stale+active+clean, 73 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 2.6 KiB/s rd, 6.9 KiB/s wr, 978 op/s; 3959/188823 objects degraded (2.097%); 4.3 KiB/s, 0 objects/s recovering 2023-10-09T06:01:01.762 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:01 smithi064 ceph-mon[220044]: failed to encode map e317 with expected crc 2023-10-09T06:01:01.763 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:01 smithi064 ceph-mon[220044]: osdmap e317: 8 total, 7 up, 7 in 2023-10-09T06:01:01.763 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:01 smithi064 ceph-mon[220044]: failed to encode map e317 with expected crc 2023-10-09T06:01:01.763 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:01 smithi064 ceph-mon[220044]: failed to encode map e317 with expected crc 2023-10-09T06:01:01.763 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:01 smithi064 ceph-mon[220044]: failed to encode map e317 with expected crc 2023-10-09T06:01:01.763 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:01 smithi064 ceph-mon[220044]: failed to encode map e317 with expected crc 2023-10-09T06:01:01.763 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:01 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1308115041' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:01:01.763 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:01 smithi064 ceph-mon[220044]: pgmap v751: 119 pgs: 6 active+undersized+degraded, 7 active+undersized, 23 peering, 10 stale+active+clean, 73 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 2.6 KiB/s rd, 6.9 KiB/s wr, 978 op/s; 3959/188823 objects degraded (2.097%); 4.3 KiB/s, 0 objects/s recovering 2023-10-09T06:01:01.781 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:01.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:01.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:01.886 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:01.950 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:02.390 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:02.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:02.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:02.496 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:02.576 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:02.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:02 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-10-09T06:01:02.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:02 smithi156 ceph-mon[195101]: Health check failed: Degraded data redundancy: 3959/188823 objects degraded (2.097%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:02.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:02 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:02.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:02 smithi156 ceph-mon[195101]: failed to encode map e318 with expected crc 2023-10-09T06:01:02.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:02 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1308115041' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:01:02.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:02 smithi156 ceph-mon[195101]: osdmap e318: 8 total, 7 up, 7 in 2023-10-09T06:01:02.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:02 smithi156 ceph-mon[195101]: failed to encode map e318 with expected crc 2023-10-09T06:01:02.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:02 smithi156 ceph-mon[195101]: failed to encode map e318 with expected crc 2023-10-09T06:01:02.704 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:02 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-10-09T06:01:02.794 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:02 smithi064 ceph-mon[212750]: Health check failed: Degraded data redundancy: 3959/188823 objects degraded (2.097%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:02.794 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:02 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:02.794 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:02 smithi064 ceph-mon[212750]: failed to encode map e318 with expected crc 2023-10-09T06:01:02.794 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:02 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1308115041' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:01:02.794 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:02 smithi064 ceph-mon[212750]: osdmap e318: 8 total, 7 up, 7 in 2023-10-09T06:01:02.794 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:02 smithi064 ceph-mon[212750]: failed to encode map e318 with expected crc 2023-10-09T06:01:02.794 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:02 smithi064 ceph-mon[212750]: failed to encode map e318 with expected crc 2023-10-09T06:01:02.795 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:02 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-10-09T06:01:02.795 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:02 smithi064 ceph-mon[220044]: Health check failed: Degraded data redundancy: 3959/188823 objects degraded (2.097%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:02.795 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:02 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:02.795 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:02 smithi064 ceph-mon[220044]: failed to encode map e318 with expected crc 2023-10-09T06:01:02.795 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:02 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1308115041' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:01:02.795 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:02 smithi064 ceph-mon[220044]: osdmap e318: 8 total, 7 up, 7 in 2023-10-09T06:01:02.796 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:02 smithi064 ceph-mon[220044]: failed to encode map e318 with expected crc 2023-10-09T06:01:02.796 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:02 smithi064 ceph-mon[220044]: failed to encode map e318 with expected crc 2023-10-09T06:01:02.889 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:02.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:02.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:03.052 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:03.087 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:03.146 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:03.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:03.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:03.250 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:03.261 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:01:03.261 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T06:01:03.261 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T06:01:03.261 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T06:01:03.261 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:01:03.329 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:03.436 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:03.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:03.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:03.539 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:03.607 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:03.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:03 smithi156 ceph-mon[195101]: pgmap v753: 119 pgs: 6 active+undersized+degraded, 8 active+undersized, 23 peering, 10 stale+active+clean, 72 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 3.5 KiB/s rd, 9.2 KiB/s wr, 672 op/s; 3959/188160 objects degraded (2.104%); 5.8 KiB/s, 0 objects/s recovering 2023-10-09T06:01:03.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:03 smithi064 ceph-mon[212750]: pgmap v753: 119 pgs: 6 active+undersized+degraded, 8 active+undersized, 23 peering, 10 stale+active+clean, 72 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 3.5 KiB/s rd, 9.2 KiB/s wr, 672 op/s; 3959/188160 objects degraded (2.104%); 5.8 KiB/s, 0 objects/s recovering 2023-10-09T06:01:03.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:03 smithi064 ceph-mon[220044]: pgmap v753: 119 pgs: 6 active+undersized+degraded, 8 active+undersized, 23 peering, 10 stale+active+clean, 72 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 3.5 KiB/s rd, 9.2 KiB/s wr, 672 op/s; 3959/188160 objects degraded (2.104%); 5.8 KiB/s, 0 objects/s recovering 2023-10-09T06:01:04.387 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:04.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:04.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:04.496 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:04.558 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:01:04.566 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:04.708 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:04.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:04.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:04.763 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:01:04.834 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:04.872 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:05.092 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:05.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:05.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:05.196 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:05.258 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:05.455 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:05.460 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:01:05.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:01:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:01:05] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.43.0" 2023-10-09T06:01:05.491 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:05.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:05.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:05.517 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:01:05.565 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:01:05.601 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:05.617 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:01:05.622 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:05.634 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:05.654 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:01:05.693 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:01:05.729 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:05.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:05.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:05.741 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:01:05.741 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:01:05.741 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:01:05.741 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:01:05.741 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:01:05.742 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-10-09T06:01:05.742 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:01:05.742 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:01:05.742 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:01:05.775 DEBUG:teuthology.orchestra.run.smithi156:> sudo pkill -f 'journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.4.service' 2023-10-09T06:01:05.835 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:05.885 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:05 smithi156 ceph-mon[195101]: pgmap v754: 119 pgs: 8 active+undersized+degraded, 20 active+undersized, 23 peering, 1 stale+active+clean, 67 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 6.8 KiB/s rd, 18 KiB/s wr, 603 op/s; 3961/185373 objects degraded (2.137%); 12 KiB/s, 0 objects/s recovering 2023-10-09T06:01:05.885 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:05 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2295468337' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:05.887 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:01:05.887 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-10-09T06:01:05.895 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:05.953 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:05.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:05.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:06.061 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:06.127 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:05 smithi064 ceph-mon[212750]: pgmap v754: 119 pgs: 8 active+undersized+degraded, 20 active+undersized, 23 peering, 1 stale+active+clean, 67 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 6.8 KiB/s rd, 18 KiB/s wr, 603 op/s; 3961/185373 objects degraded (2.137%); 12 KiB/s, 0 objects/s recovering 2023-10-09T06:01:06.128 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:05 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2295468337' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:06.128 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:05 smithi064 ceph-mon[220044]: pgmap v754: 119 pgs: 8 active+undersized+degraded, 20 active+undersized, 23 peering, 1 stale+active+clean, 67 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 6.8 KiB/s rd, 18 KiB/s wr, 603 op/s; 3961/185373 objects degraded (2.137%); 12 KiB/s, 0 objects/s recovering 2023-10-09T06:01:06.128 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:05 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2295468337' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:06.142 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:07.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:06 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 18405/176979 objects degraded (10.400%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:07.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:06 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2023-10-09T06:01:07.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:06 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 18405/176979 objects degraded (10.400%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:07.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:06 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2023-10-09T06:01:07.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:06 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 18405/176979 objects degraded (10.400%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:07.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:06 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2023-10-09T06:01:07.450 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:07.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:07.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:07.505 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T06:01:07.505 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T06:01:07.556 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:07.594 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:07.659 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:07.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:07.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:07.768 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:07.854 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:08.016 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:08.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:08.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:08.126 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:08.160 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:08.217 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:08.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:08.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:08.323 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:08.351 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:08.418 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:08.422 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:08 smithi156 ceph-mon[195101]: pgmap v755: 119 pgs: 14 active+undersized+degraded, 38 active+undersized, 67 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 5.7 KiB/s rd, 15 KiB/s wr, 885 op/s; 18405/176979 objects degraded (10.400%); 10 KiB/s, 0 objects/s recovering 2023-10-09T06:01:08.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:08.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:08.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:08 smithi064 ceph-mon[220044]: pgmap v755: 119 pgs: 14 active+undersized+degraded, 38 active+undersized, 67 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 5.7 KiB/s rd, 15 KiB/s wr, 885 op/s; 18405/176979 objects degraded (10.400%); 10 KiB/s, 0 objects/s recovering 2023-10-09T06:01:08.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:08 smithi064 ceph-mon[212750]: pgmap v755: 119 pgs: 14 active+undersized+degraded, 38 active+undersized, 67 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 5.7 KiB/s rd, 15 KiB/s wr, 885 op/s; 18405/176979 objects degraded (10.400%); 10 KiB/s, 0 objects/s recovering 2023-10-09T06:01:08.525 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:08.611 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:08.921 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:08.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:08.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:09.025 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:09.052 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:09.110 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:09.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:09.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:09.217 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:09.256 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:09.318 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:09.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:09.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:09.360 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:01:09.423 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:09.460 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:09.506 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:01:09.520 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:09.525 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:09 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1101341301' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:01:09.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:09.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:09.566 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:09 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1101341301' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:01:09.570 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:09 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1101341301' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:01:09.627 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:09.738 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:10.335 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:10.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:10.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:10.440 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:10.522 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:10.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:10 smithi064 ceph-mon[212750]: pgmap v756: 119 pgs: 14 active+undersized+degraded, 38 active+undersized, 67 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 5.0 KiB/s rd, 13 KiB/s wr, 698 op/s; 18405/172812 objects degraded (10.650%); 9.0 KiB/s, 0 objects/s recovering 2023-10-09T06:01:10.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:10 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:10.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:10 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1101341301' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:01:10.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:10 smithi064 ceph-mon[212750]: osdmap e319: 8 total, 7 up, 7 in 2023-10-09T06:01:10.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:10 smithi064 ceph-mon[212750]: failed to encode map e319 with expected crc 2023-10-09T06:01:10.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:10 smithi064 ceph-mon[212750]: failed to encode map e319 with expected crc 2023-10-09T06:01:10.594 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:10 smithi064 ceph-mon[220044]: pgmap v756: 119 pgs: 14 active+undersized+degraded, 38 active+undersized, 67 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 5.0 KiB/s rd, 13 KiB/s wr, 698 op/s; 18405/172812 objects degraded (10.650%); 9.0 KiB/s, 0 objects/s recovering 2023-10-09T06:01:10.594 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:10 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:10.594 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:10 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1101341301' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:01:10.594 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:10 smithi064 ceph-mon[220044]: osdmap e319: 8 total, 7 up, 7 in 2023-10-09T06:01:10.594 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:10 smithi064 ceph-mon[220044]: failed to encode map e319 with expected crc 2023-10-09T06:01:10.594 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:10 smithi064 ceph-mon[220044]: failed to encode map e319 with expected crc 2023-10-09T06:01:10.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:10 smithi156 ceph-mon[195101]: pgmap v756: 119 pgs: 14 active+undersized+degraded, 38 active+undersized, 67 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 5.0 KiB/s rd, 13 KiB/s wr, 698 op/s; 18405/172812 objects degraded (10.650%); 9.0 KiB/s, 0 objects/s recovering 2023-10-09T06:01:10.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:10 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:10.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:10 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1101341301' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:01:10.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:10 smithi156 ceph-mon[195101]: osdmap e319: 8 total, 7 up, 7 in 2023-10-09T06:01:10.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:10 smithi156 ceph-mon[195101]: failed to encode map e319 with expected crc 2023-10-09T06:01:10.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:10 smithi156 ceph-mon[195101]: failed to encode map e319 with expected crc 2023-10-09T06:01:10.660 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:10.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:10.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:10.774 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:10.787 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:10.805 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:10.861 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:10.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:10.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:10.888 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [4] live_osds: [1, 2, 3, 5, 6, 7, 0] 2023-10-09T06:01:10.888 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:01:10.888 INFO:tasks.thrashosds.thrasher:Reviving osd 4 2023-10-09T06:01:10.888 INFO:tasks.cephadm.osd.4:Restarting osd.4 (starting--it wasn't running)... 2023-10-09T06:01:10.888 DEBUG:teuthology.orchestra.run.smithi156:osd.4> sudo journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.4.service 2023-10-09T06:01:10.911 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl start ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.4 2023-10-09T06:01:10.937 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:01:10.937 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:01:10.943 INFO:journalctl@ceph.osd.4.smithi156.stdout:-- Logs begin at Mon 2023-10-09 05:16:12 UTC. -- 2023-10-09T06:01:10.970 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:11.030 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:11.099 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:11.112 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:01:10 smithi156 systemd[1]: Starting Ceph osd.4 for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T06:01:11.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:11.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:11.214 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:11.280 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:12.278 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:12.284 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:01:12.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:12.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:12.339 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:01:12 smithi156 bash[280323]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T06:01:12.340 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:01:12 smithi156 bash[280323]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-10-09T06:01:12.340 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:01:12 smithi156 bash[280323]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-4 --no-mon-config 2023-10-09T06:01:12.340 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:01:12 smithi156 bash[280323]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-4/block 2023-10-09T06:01:12.340 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:01:12 smithi156 bash[280323]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-4/block 2023-10-09T06:01:12.340 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:01:12 smithi156 bash[280323]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-10-09T06:01:12.340 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:01:12 smithi156 bash[280323]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-10-09T06:01:12.340 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:01:12 smithi156 bash[280323]: --> ceph-volume lvm activate successful for osd ID: 4 2023-10-09T06:01:12.390 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:12.428 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:12.461 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:01:12.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:12 smithi064 ceph-mon[220044]: pgmap v758: 119 pgs: 14 active+undersized+degraded, 38 active+undersized, 67 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 2.2 KiB/s rd, 6.0 KiB/s wr, 903 op/s; 17063/163860 objects degraded (10.413%); 4.3 KiB/s, 0 objects/s recovering 2023-10-09T06:01:12.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:12 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 17063/163860 objects degraded (10.413%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:12.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:12 smithi064 ceph-mon[212750]: pgmap v758: 119 pgs: 14 active+undersized+degraded, 38 active+undersized, 67 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 2.2 KiB/s rd, 6.0 KiB/s wr, 903 op/s; 17063/163860 objects degraded (10.413%); 4.3 KiB/s, 0 objects/s recovering 2023-10-09T06:01:12.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:12 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 17063/163860 objects degraded (10.413%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:12.514 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:12.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:12.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:12.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:12 smithi156 ceph-mon[195101]: pgmap v758: 119 pgs: 14 active+undersized+degraded, 38 active+undersized, 67 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 2.2 KiB/s rd, 6.0 KiB/s wr, 903 op/s; 17063/163860 objects degraded (10.413%); 4.3 KiB/s, 0 objects/s recovering 2023-10-09T06:01:12.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:12 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 17063/163860 objects degraded (10.413%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:12.622 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:12.720 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:12.828 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:12.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:12.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:12.932 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:12.986 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:13.217 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:13.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:13.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:13.321 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:13.355 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:13.493 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:13.701 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:13.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:13.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:13.785 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:01:13.812 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:13.873 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:14.052 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:14.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:14.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:14.140 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:01:14.161 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:14.186 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:01:14.209 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:14.241 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:01:14.271 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:14.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:14.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:14.291 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:01:14.334 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:01:14.376 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:14.381 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:01:14.419 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:01:14.423 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:14.432 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:01:14.432 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:01:14.432 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:01:14.432 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:01:14.432 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-10-09T06:01:14.432 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:01:14.432 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:01:14.433 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:01:14.491 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:14.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:14.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:14.534 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:14 smithi156 ceph-mon[195101]: pgmap v759: 119 pgs: 14 active+undersized+degraded, 38 active+undersized, 67 active+clean; 3.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 2.0 KiB/s rd, 5.5 KiB/s wr, 840 op/s; 17063/162972 objects degraded (10.470%); 3.9 KiB/s, 0 objects/s recovering 2023-10-09T06:01:14.534 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:01:14.534 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:14.534 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:01:14.534 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:14 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3796200811' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:14.541 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:01:14.597 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:14.624 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:14 smithi064 ceph-mon[212750]: pgmap v759: 119 pgs: 14 active+undersized+degraded, 38 active+undersized, 67 active+clean; 3.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 2.0 KiB/s rd, 5.5 KiB/s wr, 840 op/s; 17063/162972 objects degraded (10.470%); 3.9 KiB/s, 0 objects/s recovering 2023-10-09T06:01:14.625 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:01:14.625 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:14.625 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:01:14.625 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:14 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3796200811' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:14.625 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:14 smithi064 ceph-mon[220044]: pgmap v759: 119 pgs: 14 active+undersized+degraded, 38 active+undersized, 67 active+clean; 3.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 2.0 KiB/s rd, 5.5 KiB/s wr, 840 op/s; 17063/162972 objects degraded (10.470%); 3.9 KiB/s, 0 objects/s recovering 2023-10-09T06:01:14.625 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:01:14.625 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:14.626 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:01:14.626 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:14 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3796200811' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:14.627 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:14.870 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:01:14 smithi156 bash[281028]: 0d0094a29e696a0407d59c08e34f2ab3aa535312d1b2fc9c4626b221cc647b8f 2023-10-09T06:01:14.870 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:01:14 smithi156 systemd[1]: Started Ceph osd.4 for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T06:01:15.299 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:01:15 smithi156 conmon[281149]: 2023-10-09T06:01:15.008+0000 7fd2a2317540 -1 Falling back to public interface 2023-10-09T06:01:15.431 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:01:15.721 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:01:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:01:15] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.43.0" 2023-10-09T06:01:15.870 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:01:15.871 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:01:15.871 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 0 2023-10-09T06:01:15.871 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:01:15.975 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:15.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:15.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:16.024 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T06:01:16.024 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T06:01:16.096 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:16.150 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:16.194 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:16.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:16.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:16.298 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:16.339 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:16.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:16 smithi156 ceph-mon[195101]: pgmap v760: 119 pgs: 14 active+undersized+degraded, 38 active+undersized, 67 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 850 op/s; 17063/159825 objects degraded (10.676%) 2023-10-09T06:01:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:16 smithi064 ceph-mon[220044]: pgmap v760: 119 pgs: 14 active+undersized+degraded, 38 active+undersized, 67 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 850 op/s; 17063/159825 objects degraded (10.676%) 2023-10-09T06:01:16.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:16 smithi064 ceph-mon[212750]: pgmap v760: 119 pgs: 14 active+undersized+degraded, 38 active+undersized, 67 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 850 op/s; 17063/159825 objects degraded (10.676%) 2023-10-09T06:01:17.185 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:01:17.322 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:17.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:17.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:17.384 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:01:17.433 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:17.507 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:17.612 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:17 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/627038315' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:01:17.612 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:17 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 17063/159825 objects degraded (10.676%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:17.613 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:17 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/627038315' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:01:17.613 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:17 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 17063/159825 objects degraded (10.676%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:17.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:17 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/627038315' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:01:17.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:17 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 17063/159825 objects degraded (10.676%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:17.688 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:17.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:17.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:17.794 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:17.864 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:17.978 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:17.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:17.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:17.995 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:01:18.025 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:01:18.083 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:18.103 INFO:teuthology.orchestra.run.smithi156.stderr:no valid command found; 10 closest matches: 2023-10-09T06:01:18.103 INFO:teuthology.orchestra.run.smithi156.stderr:0 2023-10-09T06:01:18.103 INFO:teuthology.orchestra.run.smithi156.stderr:1 2023-10-09T06:01:18.103 INFO:teuthology.orchestra.run.smithi156.stderr:2 2023-10-09T06:01:18.104 INFO:teuthology.orchestra.run.smithi156.stderr:abort 2023-10-09T06:01:18.104 INFO:teuthology.orchestra.run.smithi156.stderr:assert 2023-10-09T06:01:18.104 INFO:teuthology.orchestra.run.smithi156.stderr:bluefs debug_inject_read_zeros 2023-10-09T06:01:18.104 INFO:teuthology.orchestra.run.smithi156.stderr:bluefs files list 2023-10-09T06:01:18.104 INFO:teuthology.orchestra.run.smithi156.stderr:bluefs stats 2023-10-09T06:01:18.104 INFO:teuthology.orchestra.run.smithi156.stderr:bluestore bluefs device info [] 2023-10-09T06:01:18.104 INFO:teuthology.orchestra.run.smithi156.stderr:config diff 2023-10-09T06:01:18.104 INFO:teuthology.orchestra.run.smithi156.stderr:admin_socket: invalid command 2023-10-09T06:01:18.119 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:18.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:18 smithi156 ceph-mon[195101]: pgmap v761: 119 pgs: 14 active+undersized+degraded, 38 active+undersized, 67 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 872 op/s; 15675/150765 objects degraded (10.397%) 2023-10-09T06:01:18.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:18 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:18 smithi156 ceph-mon[195101]: failed to encode map e320 with expected crc 2023-10-09T06:01:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:18 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/627038315' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:01:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:18 smithi156 ceph-mon[195101]: osdmap e320: 8 total, 7 up, 7 in 2023-10-09T06:01:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:18 smithi156 ceph-mon[195101]: failed to encode map e320 with expected crc 2023-10-09T06:01:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:18 smithi156 ceph-mon[195101]: failed to encode map e320 with expected crc 2023-10-09T06:01:18.698 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:18.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:18.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:18.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:18 smithi064 ceph-mon[220044]: pgmap v761: 119 pgs: 14 active+undersized+degraded, 38 active+undersized, 67 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 872 op/s; 15675/150765 objects degraded (10.397%) 2023-10-09T06:01:18.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:18 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:18.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:18 smithi064 ceph-mon[220044]: failed to encode map e320 with expected crc 2023-10-09T06:01:18.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:18 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/627038315' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:01:18.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:18 smithi064 ceph-mon[220044]: osdmap e320: 8 total, 7 up, 7 in 2023-10-09T06:01:18.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:18 smithi064 ceph-mon[220044]: failed to encode map e320 with expected crc 2023-10-09T06:01:18.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:18 smithi064 ceph-mon[220044]: failed to encode map e320 with expected crc 2023-10-09T06:01:18.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:18 smithi064 ceph-mon[212750]: pgmap v761: 119 pgs: 14 active+undersized+degraded, 38 active+undersized, 67 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 872 op/s; 15675/150765 objects degraded (10.397%) 2023-10-09T06:01:18.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:18 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:18.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:18 smithi064 ceph-mon[212750]: failed to encode map e320 with expected crc 2023-10-09T06:01:18.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:18 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/627038315' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:01:18.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:18 smithi064 ceph-mon[212750]: osdmap e320: 8 total, 7 up, 7 in 2023-10-09T06:01:18.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:18 smithi064 ceph-mon[212750]: failed to encode map e320 with expected crc 2023-10-09T06:01:18.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:18 smithi064 ceph-mon[212750]: failed to encode map e320 with expected crc 2023-10-09T06:01:18.787 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:18.804 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:18.833 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:18.894 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:18.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:18.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:19.002 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:19.042 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:19.251 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:19.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:19.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:19.258 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T06:01:19.259 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-4, ['dump_ops_in_flight'] 2023-10-09T06:01:19.357 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:19.435 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:19.524 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:19.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:19.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:19.628 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:19.689 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:19.942 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:19.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:19.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:20.053 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:20.126 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:20.223 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:01:20 smithi156 conmon[281149]: 2023-10-09T06:01:20.129+0000 7fd2a2317540 -1 osd.4 315 log_to_monitors true 2023-10-09T06:01:20.390 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:20 smithi064 ceph-mon[212750]: pgmap v763: 119 pgs: 14 active+undersized+degraded, 38 active+undersized, 67 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 916 op/s; 15675/146931 objects degraded (10.668%) 2023-10-09T06:01:20.391 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:20 smithi064 ceph-mon[212750]: from='osd.4 [v2:172.21.15.156:6800/3173423256,v1:172.21.15.156:6801/3173423256]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T06:01:20.391 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:20 smithi064 ceph-mon[212750]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T06:01:20.433 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:20.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:20.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:20.489 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:01:20.489 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:01:20.489 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T06:01:20.489 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T06:01:20.489 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:01:20.513 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:01:20 smithi156 conmon[281149]: 2023-10-09T06:01:20.407+0000 7fd2938f7700 -1 osd.4 315 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T06:01:20.513 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:20 smithi156 ceph-mon[195101]: pgmap v763: 119 pgs: 14 active+undersized+degraded, 38 active+undersized, 67 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 916 op/s; 15675/146931 objects degraded (10.668%) 2023-10-09T06:01:20.513 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:20 smithi156 ceph-mon[195101]: from='osd.4 [v2:172.21.15.156:6800/3173423256,v1:172.21.15.156:6801/3173423256]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T06:01:20.513 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:20 smithi156 ceph-mon[195101]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T06:01:20.549 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:20.595 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:20.601 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:01:20.681 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:20 smithi064 ceph-mon[220044]: pgmap v763: 119 pgs: 14 active+undersized+degraded, 38 active+undersized, 67 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 916 op/s; 15675/146931 objects degraded (10.668%) 2023-10-09T06:01:20.681 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:20 smithi064 ceph-mon[220044]: from='osd.4 [v2:172.21.15.156:6800/3173423256,v1:172.21.15.156:6801/3173423256]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T06:01:20.681 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:20 smithi064 ceph-mon[220044]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T06:01:20.711 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:20.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:20.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:20.817 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:20.845 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:20.887 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:20.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:20.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:20.991 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:21.025 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:21.082 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:21.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:21.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:21.159 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:01:21.197 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:21.210 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:01:21.247 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:01:21.274 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:21.298 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:01:21.340 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:01:21.340 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:21.382 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:01:21.391 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:21.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:21.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:21.435 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:01:21.439 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:01:21.440 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:01:21.440 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:01:21.440 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:01:21.440 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:01:21.440 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-10-09T06:01:21.440 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:01:21.440 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:01:21.440 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:01:21.497 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:21.524 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:21.705 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:21 smithi064 ceph-mon[212750]: failed to encode map e321 with expected crc 2023-10-09T06:01:21.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:21 smithi064 ceph-mon[212750]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T06:01:21.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:21 smithi064 ceph-mon[212750]: osdmap e321: 8 total, 7 up, 7 in 2023-10-09T06:01:21.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:21 smithi064 ceph-mon[212750]: from='osd.4 [v2:172.21.15.156:6800/3173423256,v1:172.21.15.156:6801/3173423256]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T06:01:21.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:21 smithi064 ceph-mon[212750]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T06:01:21.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:21 smithi064 ceph-mon[212750]: failed to encode map e321 with expected crc 2023-10-09T06:01:21.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:21 smithi064 ceph-mon[212750]: failed to encode map e321 with expected crc 2023-10-09T06:01:21.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:21 smithi064 ceph-mon[212750]: pgmap v765: 119 pgs: 14 active+undersized+degraded, 38 active+undersized, 67 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 14389/138357 objects degraded (10.400%) 2023-10-09T06:01:21.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:21 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2479254956' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:21.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:21 smithi064 ceph-mon[220044]: failed to encode map e321 with expected crc 2023-10-09T06:01:21.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:21 smithi064 ceph-mon[220044]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T06:01:21.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:21 smithi064 ceph-mon[220044]: osdmap e321: 8 total, 7 up, 7 in 2023-10-09T06:01:21.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:21 smithi064 ceph-mon[220044]: from='osd.4 [v2:172.21.15.156:6800/3173423256,v1:172.21.15.156:6801/3173423256]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T06:01:21.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:21 smithi064 ceph-mon[220044]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T06:01:21.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:21 smithi064 ceph-mon[220044]: failed to encode map e321 with expected crc 2023-10-09T06:01:21.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:21 smithi064 ceph-mon[220044]: failed to encode map e321 with expected crc 2023-10-09T06:01:21.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:21 smithi064 ceph-mon[220044]: pgmap v765: 119 pgs: 14 active+undersized+degraded, 38 active+undersized, 67 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 14389/138357 objects degraded (10.400%) 2023-10-09T06:01:21.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:21 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2479254956' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:21.725 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:21.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:21.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:21.798 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:01:21.829 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:21.857 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:21.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:21 smithi156 ceph-mon[195101]: failed to encode map e321 with expected crc 2023-10-09T06:01:21.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:21 smithi156 ceph-mon[195101]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T06:01:21.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:21 smithi156 ceph-mon[195101]: osdmap e321: 8 total, 7 up, 7 in 2023-10-09T06:01:21.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:21 smithi156 ceph-mon[195101]: from='osd.4 [v2:172.21.15.156:6800/3173423256,v1:172.21.15.156:6801/3173423256]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T06:01:21.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:21 smithi156 ceph-mon[195101]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T06:01:21.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:21 smithi156 ceph-mon[195101]: failed to encode map e321 with expected crc 2023-10-09T06:01:21.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:21 smithi156 ceph-mon[195101]: failed to encode map e321 with expected crc 2023-10-09T06:01:21.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:21 smithi156 ceph-mon[195101]: pgmap v765: 119 pgs: 14 active+undersized+degraded, 38 active+undersized, 67 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 14389/138357 objects degraded (10.400%) 2023-10-09T06:01:21.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:21 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2479254956' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:22.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:22 smithi064 ceph-mon[220044]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:01:22.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:22 smithi064 ceph-mon[220044]: failed to encode map e322 with expected crc 2023-10-09T06:01:22.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:22 smithi064 ceph-mon[220044]: osd.4 [v2:172.21.15.156:6800/3173423256,v1:172.21.15.156:6801/3173423256] boot 2023-10-09T06:01:22.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:22 smithi064 ceph-mon[220044]: osdmap e322: 8 total, 8 up, 7 in 2023-10-09T06:01:22.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:22 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T06:01:22.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:22 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 14389/138357 objects degraded (10.400%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:22.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:22 smithi064 ceph-mon[220044]: osdmap e323: 8 total, 8 up, 7 in 2023-10-09T06:01:22.666 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:22 smithi064 ceph-mon[212750]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:01:22.666 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:22 smithi064 ceph-mon[212750]: failed to encode map e322 with expected crc 2023-10-09T06:01:22.666 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:22 smithi064 ceph-mon[212750]: osd.4 [v2:172.21.15.156:6800/3173423256,v1:172.21.15.156:6801/3173423256] boot 2023-10-09T06:01:22.666 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:22 smithi064 ceph-mon[212750]: osdmap e322: 8 total, 8 up, 7 in 2023-10-09T06:01:22.666 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:22 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T06:01:22.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:22 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 14389/138357 objects degraded (10.400%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:22.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:22 smithi064 ceph-mon[212750]: osdmap e323: 8 total, 8 up, 7 in 2023-10-09T06:01:22.719 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T06:01:22.720 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T06:01:22.745 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:22.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:22.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:22.848 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:22.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:22 smithi156 ceph-mon[195101]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:01:22.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:22 smithi156 ceph-mon[195101]: failed to encode map e322 with expected crc 2023-10-09T06:01:22.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:22 smithi156 ceph-mon[195101]: osd.4 [v2:172.21.15.156:6800/3173423256,v1:172.21.15.156:6801/3173423256] boot 2023-10-09T06:01:22.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:22 smithi156 ceph-mon[195101]: osdmap e322: 8 total, 8 up, 7 in 2023-10-09T06:01:22.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:22 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T06:01:22.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:22 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 14389/138357 objects degraded (10.400%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:22.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:22 smithi156 ceph-mon[195101]: osdmap e323: 8 total, 8 up, 7 in 2023-10-09T06:01:22.878 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:22.960 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:22.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:22.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:23.063 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:23.095 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:23.141 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:23.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:23.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:23.244 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:23.274 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:23.427 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:23.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:23.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:23.483 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:01:23.483 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:01:23.510 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:23 smithi156 ceph-mon[195101]: failed to encode map e316 with expected crc 2023-10-09T06:01:23.510 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:23 smithi156 ceph-mon[195101]: failed to encode map e322 with expected crc 2023-10-09T06:01:23.510 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:23 smithi156 ceph-mon[195101]: failed to encode map e323 with expected crc 2023-10-09T06:01:23.511 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:23 smithi156 ceph-mon[195101]: failed to encode map e323 with expected crc 2023-10-09T06:01:23.511 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:23 smithi156 ceph-mon[195101]: failed to encode map e323 with expected crc 2023-10-09T06:01:23.511 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:23 smithi156 ceph-mon[195101]: failed to encode map e323 with expected crc 2023-10-09T06:01:23.511 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:23 smithi156 ceph-mon[195101]: failed to encode map e323 with expected crc 2023-10-09T06:01:23.511 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:23 smithi156 ceph-mon[195101]: failed to encode map e323 with expected crc 2023-10-09T06:01:23.511 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:23 smithi156 ceph-mon[195101]: failed to encode map e323 with expected crc 2023-10-09T06:01:23.511 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:23 smithi156 ceph-mon[195101]: pgmap v768: 119 pgs: 14 active+undersized+degraded, 37 active+undersized, 68 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 805 op/s; 14389/137781 objects degraded (10.443%) 2023-10-09T06:01:23.511 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:23 smithi156 ceph-mon[195101]: osdmap e324: 8 total, 8 up, 7 in 2023-10-09T06:01:23.532 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:23.561 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:23.608 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:23.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:23.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:23.717 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:23.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:23 smithi064 ceph-mon[212750]: failed to encode map e316 with expected crc 2023-10-09T06:01:23.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:23 smithi064 ceph-mon[212750]: failed to encode map e322 with expected crc 2023-10-09T06:01:23.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:23 smithi064 ceph-mon[212750]: failed to encode map e323 with expected crc 2023-10-09T06:01:23.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:23 smithi064 ceph-mon[212750]: failed to encode map e323 with expected crc 2023-10-09T06:01:23.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:23 smithi064 ceph-mon[212750]: failed to encode map e323 with expected crc 2023-10-09T06:01:23.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:23 smithi064 ceph-mon[212750]: failed to encode map e323 with expected crc 2023-10-09T06:01:23.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:23 smithi064 ceph-mon[212750]: failed to encode map e323 with expected crc 2023-10-09T06:01:23.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:23 smithi064 ceph-mon[212750]: failed to encode map e323 with expected crc 2023-10-09T06:01:23.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:23 smithi064 ceph-mon[212750]: failed to encode map e323 with expected crc 2023-10-09T06:01:23.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:23 smithi064 ceph-mon[212750]: pgmap v768: 119 pgs: 14 active+undersized+degraded, 37 active+undersized, 68 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 805 op/s; 14389/137781 objects degraded (10.443%) 2023-10-09T06:01:23.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:23 smithi064 ceph-mon[212750]: osdmap e324: 8 total, 8 up, 7 in 2023-10-09T06:01:23.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:23 smithi064 ceph-mon[220044]: failed to encode map e316 with expected crc 2023-10-09T06:01:23.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:23 smithi064 ceph-mon[220044]: failed to encode map e322 with expected crc 2023-10-09T06:01:23.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:23 smithi064 ceph-mon[220044]: failed to encode map e323 with expected crc 2023-10-09T06:01:23.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:23 smithi064 ceph-mon[220044]: failed to encode map e323 with expected crc 2023-10-09T06:01:23.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:23 smithi064 ceph-mon[220044]: failed to encode map e323 with expected crc 2023-10-09T06:01:23.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:23 smithi064 ceph-mon[220044]: failed to encode map e323 with expected crc 2023-10-09T06:01:23.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:23 smithi064 ceph-mon[220044]: failed to encode map e323 with expected crc 2023-10-09T06:01:23.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:23 smithi064 ceph-mon[220044]: failed to encode map e323 with expected crc 2023-10-09T06:01:23.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:23 smithi064 ceph-mon[220044]: failed to encode map e323 with expected crc 2023-10-09T06:01:23.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:23 smithi064 ceph-mon[220044]: pgmap v768: 119 pgs: 14 active+undersized+degraded, 37 active+undersized, 68 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 805 op/s; 14389/137781 objects degraded (10.443%) 2023-10-09T06:01:23.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:23 smithi064 ceph-mon[220044]: osdmap e324: 8 total, 8 up, 7 in 2023-10-09T06:01:23.770 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:23.921 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:23.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:23.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:24.031 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:24.096 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:24.147 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:01:24.211 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:24.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:24.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:24.260 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:01:24.318 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:24.420 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:24 smithi064 ceph-mon[212750]: failed to encode map e324 with expected crc 2023-10-09T06:01:24.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:24 smithi064 ceph-mon[212750]: failed to encode map e324 with expected crc 2023-10-09T06:01:24.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:24 smithi064 ceph-mon[212750]: failed to encode map e324 with expected crc 2023-10-09T06:01:24.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:24 smithi064 ceph-mon[212750]: failed to encode map e324 with expected crc 2023-10-09T06:01:24.458 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:24.614 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:24.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:24.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:24.720 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:01:24.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:24 smithi064 ceph-mon[220044]: failed to encode map e324 with expected crc 2023-10-09T06:01:24.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:24 smithi064 ceph-mon[220044]: failed to encode map e324 with expected crc 2023-10-09T06:01:24.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:24 smithi064 ceph-mon[220044]: failed to encode map e324 with expected crc 2023-10-09T06:01:24.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:24 smithi064 ceph-mon[220044]: failed to encode map e324 with expected crc 2023-10-09T06:01:24.725 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:24.799 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:24 smithi156 ceph-mon[195101]: failed to encode map e324 with expected crc 2023-10-09T06:01:24.799 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:24 smithi156 ceph-mon[195101]: failed to encode map e324 with expected crc 2023-10-09T06:01:24.799 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:24 smithi156 ceph-mon[195101]: failed to encode map e324 with expected crc 2023-10-09T06:01:24.799 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:24 smithi156 ceph-mon[195101]: failed to encode map e324 with expected crc 2023-10-09T06:01:24.801 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:24.897 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:01:24.945 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:24.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:24.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:25.031 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:01:25.061 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:25.117 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:25.220 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:25.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:25.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:25.325 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:25.395 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:01:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:01:25] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.43.0" 2023-10-09T06:01:25.401 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:25.402 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:25.474 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:01:25.536 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:25.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:25.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:25.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:25 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2061589183' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:01:25.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:25 smithi156 ceph-mon[195101]: pgmap v770: 119 pgs: 14 active+undersized+degraded, 32 active+undersized, 73 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 291 op/s; 14389/136116 objects degraded (10.571%) 2023-10-09T06:01:25.645 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:25.701 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:25 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2061589183' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:01:25.701 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:25 smithi064 ceph-mon[212750]: pgmap v770: 119 pgs: 14 active+undersized+degraded, 32 active+undersized, 73 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 291 op/s; 14389/136116 objects degraded (10.571%) 2023-10-09T06:01:25.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:25 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2061589183' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:01:25.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:25 smithi064 ceph-mon[220044]: pgmap v770: 119 pgs: 14 active+undersized+degraded, 32 active+undersized, 73 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 291 op/s; 14389/136116 objects degraded (10.571%) 2023-10-09T06:01:25.712 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:25.931 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:25.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:25.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:26.039 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:26.118 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:26.204 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:26.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:26.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:26.307 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:26.357 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:26.427 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:26 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:01:26.427 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:26 smithi156 ceph-mon[195101]: failed to encode map e325 with expected crc 2023-10-09T06:01:26.427 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:26 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2061589183' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:01:26.427 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:26 smithi156 ceph-mon[195101]: osdmap e325: 8 total, 8 up, 7 in 2023-10-09T06:01:26.427 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:26 smithi156 ceph-mon[195101]: failed to encode map e325 with expected crc 2023-10-09T06:01:26.428 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:26 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:01:26.428 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:26 smithi064 ceph-mon[220044]: failed to encode map e325 with expected crc 2023-10-09T06:01:26.428 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:26 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:01:26.428 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:26 smithi064 ceph-mon[212750]: failed to encode map e325 with expected crc 2023-10-09T06:01:26.428 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:26 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2061589183' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:01:26.428 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:26 smithi064 ceph-mon[212750]: osdmap e325: 8 total, 8 up, 7 in 2023-10-09T06:01:26.428 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:26 smithi064 ceph-mon[212750]: failed to encode map e325 with expected crc 2023-10-09T06:01:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:26 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2061589183' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:01:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:26 smithi064 ceph-mon[220044]: osdmap e325: 8 total, 8 up, 7 in 2023-10-09T06:01:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:26 smithi064 ceph-mon[220044]: failed to encode map e325 with expected crc 2023-10-09T06:01:27.185 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:27.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:27.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:27.291 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:27.325 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:27.452 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:27.452 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:27 smithi156 ceph-mon[195101]: pgmap v772: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 7.9 KiB/s rd, 21 KiB/s wr, 451 op/s; 4511/131232 objects degraded (3.437%); 14 KiB/s, 33 objects/s recovering 2023-10-09T06:01:27.453 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:27 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 14389/136116 objects degraded (10.571%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:27.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:27.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:27.561 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:27.628 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:27 smithi064 ceph-mon[212750]: pgmap v772: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 7.9 KiB/s rd, 21 KiB/s wr, 451 op/s; 4511/131232 objects degraded (3.437%); 14 KiB/s, 33 objects/s recovering 2023-10-09T06:01:27.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:27 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 14389/136116 objects degraded (10.571%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:27 smithi064 ceph-mon[220044]: pgmap v772: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 7.9 KiB/s rd, 21 KiB/s wr, 451 op/s; 4511/131232 objects degraded (3.437%); 14 KiB/s, 33 objects/s recovering 2023-10-09T06:01:27.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:27 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 14389/136116 objects degraded (10.571%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:27.762 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:27.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:27.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:27.879 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:27.925 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:01:27.926 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:01:27.942 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:28.117 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:28.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:28.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:28.186 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:01:28.186 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:01:28.186 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 0 2023-10-09T06:01:28.186 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:01:28.226 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:28.282 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:28.399 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:28.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:28.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:28.426 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:01:28.470 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:01:28.512 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:28.527 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:01:28.582 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:01:28.582 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:28.630 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:01:28.669 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:01:28.705 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:01:28.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:28 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4094211853' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:28.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:28 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4094211853' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:28.727 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:28.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:28.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:28.745 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:01:28.760 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:01:28.760 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:01:28.760 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:01:28.760 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:01:28.761 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:01:28.761 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:01:28.761 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:01:28.761 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:01:28.840 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:28.869 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:28.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:28 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4094211853' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:28.916 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:01:28.966 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:28.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:28.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:29.049 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:01:29.074 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:29.156 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:29.263 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:29.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:29.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:29.367 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:29.393 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:29.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:29 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:01:29.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:29.623 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:29 smithi156 ceph-mon[195101]: pgmap v773: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 6.5 KiB/s rd, 17 KiB/s wr, 563 op/s; 4511/127575 objects degraded (3.536%); 11 KiB/s, 27 objects/s recovering 2023-10-09T06:01:29.846 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:29.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:29 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:01:29.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:29.859 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:29 smithi064 ceph-mon[220044]: pgmap v773: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 6.5 KiB/s rd, 17 KiB/s wr, 563 op/s; 4511/127575 objects degraded (3.536%); 11 KiB/s, 27 objects/s recovering 2023-10-09T06:01:29.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:29.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:29.860 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:29 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:01:29.860 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:29.860 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:29 smithi064 ceph-mon[212750]: pgmap v773: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 6.5 KiB/s rd, 17 KiB/s wr, 563 op/s; 4511/127575 objects degraded (3.536%); 11 KiB/s, 27 objects/s recovering 2023-10-09T06:01:29.960 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:29.981 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:01:30.040 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:30.120 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:30.167 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:30.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:30.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:30.279 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:30.344 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:30.505 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:30.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:30.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:30.621 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:30.685 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:30.892 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:30.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:30.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:30.997 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:31.068 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:31.115 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:31.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:31.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:31.218 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:01:31.249 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:31.295 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:01:31.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:31.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:01:31.401 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:31.466 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:31.817 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:31.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:31.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:31.924 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:31 smithi156 ceph-mon[195101]: pgmap v774: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 777 op/s; 4511/119217 objects degraded (3.784%); 9.2 KiB/s, 117 objects/s recovering 2023-10-09T06:01:31.932 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:31.946 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:01:31.978 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:32.094 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:32.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:32.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:32.180 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:01:32.180 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:01:32.180 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T06:01:32.180 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T06:01:32.180 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:01:32.203 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:01:32.205 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:31 smithi064 ceph-mon[212750]: pgmap v774: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 777 op/s; 4511/119217 objects degraded (3.784%); 9.2 KiB/s, 117 objects/s recovering 2023-10-09T06:01:32.206 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:31 smithi064 ceph-mon[220044]: pgmap v774: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 777 op/s; 4511/119217 objects degraded (3.784%); 9.2 KiB/s, 117 objects/s recovering 2023-10-09T06:01:32.230 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:32.378 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:01:32.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:32.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:01:32.482 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:32.518 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:32.520 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:32.575 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:32.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:32.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:32.678 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:32.707 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:32.759 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:32.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:32.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:32.837 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:01:32.865 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:32.897 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:32.991 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:01:33.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:32 smithi156 ceph-mon[195101]: failed to encode map e326 with expected crc 2023-10-09T06:01:33.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:32 smithi156 ceph-mon[195101]: osdmap e326: 8 total, 8 up, 7 in 2023-10-09T06:01:33.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:32 smithi156 ceph-mon[195101]: failed to encode map e326 with expected crc 2023-10-09T06:01:33.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:32 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 4511/119217 objects degraded (3.784%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:33.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:32 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3631943690' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:01:33.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:32 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:01:33.138 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:33.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:32 smithi064 ceph-mon[220044]: failed to encode map e326 with expected crc 2023-10-09T06:01:33.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:32 smithi064 ceph-mon[220044]: osdmap e326: 8 total, 8 up, 7 in 2023-10-09T06:01:33.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:32 smithi064 ceph-mon[220044]: failed to encode map e326 with expected crc 2023-10-09T06:01:33.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:32 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 4511/119217 objects degraded (3.784%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:33.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:32 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3631943690' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:01:33.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:32 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:01:33.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:33.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:33.149 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:32 smithi064 ceph-mon[212750]: failed to encode map e326 with expected crc 2023-10-09T06:01:33.149 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:32 smithi064 ceph-mon[212750]: osdmap e326: 8 total, 8 up, 7 in 2023-10-09T06:01:33.150 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:32 smithi064 ceph-mon[212750]: failed to encode map e326 with expected crc 2023-10-09T06:01:33.150 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:32 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 4511/119217 objects degraded (3.784%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:33.150 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:32 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3631943690' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:01:33.150 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:32 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:01:33.250 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:33.311 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:33.426 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:33.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:33.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:33.533 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:33.601 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:33.616 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:01:33.715 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:33.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:33.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:33.823 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:33.848 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:33 smithi156 ceph-mon[195101]: pgmap v776: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.4 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 737 op/s; 4511/118497 objects degraded (3.807%); 9.2 KiB/s, 117 objects/s recovering 2023-10-09T06:01:33.849 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:33 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:33.849 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:33 smithi156 ceph-mon[195101]: failed to encode map e327 with expected crc 2023-10-09T06:01:33.849 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:33 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:01:33.849 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:33 smithi156 ceph-mon[195101]: osdmap e327: 8 total, 8 up, 7 in 2023-10-09T06:01:33.849 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:33 smithi156 ceph-mon[195101]: failed to encode map e327 with expected crc 2023-10-09T06:01:33.892 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:33.983 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:33.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:33.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:34.087 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:34.116 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:34.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:33 smithi064 ceph-mon[220044]: pgmap v776: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.4 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 737 op/s; 4511/118497 objects degraded (3.807%); 9.2 KiB/s, 117 objects/s recovering 2023-10-09T06:01:34.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:33 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:34.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:33 smithi064 ceph-mon[220044]: failed to encode map e327 with expected crc 2023-10-09T06:01:34.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:33 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:01:34.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:33 smithi064 ceph-mon[220044]: osdmap e327: 8 total, 8 up, 7 in 2023-10-09T06:01:34.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:33 smithi064 ceph-mon[220044]: failed to encode map e327 with expected crc 2023-10-09T06:01:34.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:33 smithi064 ceph-mon[212750]: pgmap v776: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.4 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 737 op/s; 4511/118497 objects degraded (3.807%); 9.2 KiB/s, 117 objects/s recovering 2023-10-09T06:01:34.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:33 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:34.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:33 smithi064 ceph-mon[212750]: failed to encode map e327 with expected crc 2023-10-09T06:01:34.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:33 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:01:34.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:33 smithi064 ceph-mon[212750]: osdmap e327: 8 total, 8 up, 7 in 2023-10-09T06:01:34.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:33 smithi064 ceph-mon[212750]: failed to encode map e327 with expected crc 2023-10-09T06:01:34.291 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:34.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:34.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:34.399 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:34.457 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:34.876 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 0, 4] 2023-10-09T06:01:34.877 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:01:34.877 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:01:34.877 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.529070 2023-10-09T06:01:34.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 0.5290699089564754 2023-10-09T06:01:35.113 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:35.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:35.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:35.228 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:35.281 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:35.420 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:35.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:35.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:35.526 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:01:35.550 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:01:35.550 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T06:01:35.550 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T06:01:35.550 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:01:35.551 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.551 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:91012 6.1 6:8339cfc9:::benchmark_data_smithi064_284193_object91011:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-10-09T06:01:35.551 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:10.210146+0000", 2023-10-09T06:01:35.551 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 565.33730157800005, 2023-10-09T06:01:35.551 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2909072420000001, 2023-10-09T06:01:35.551 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:01:35.551 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:01:35.551 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:01:35.551 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:01:35.551 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:01:35.552 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 91012 2023-10-09T06:01:35.552 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.552 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:01:35.552 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.552 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:01:35.552 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210146+0000", 2023-10-09T06:01:35.552 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.552 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.552 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.552 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:01:35.552 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210146+0000", 2023-10-09T06:01:35.553 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.553 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.553 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.553 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:01:35.553 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210145+0000", 2023-10-09T06:01:35.553 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:01:35.553 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.553 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.553 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:01:35.553 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210198+0000", 2023-10-09T06:01:35.553 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2784000000000001e-05 2023-10-09T06:01:35.554 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.554 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.554 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:01:35.554 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210209+0000", 2023-10-09T06:01:35.554 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0909e-05 2023-10-09T06:01:35.554 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.554 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.554 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:01:35.554 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.210216+0000", 2023-10-09T06:01:35.554 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6790000000000005e-06 2023-10-09T06:01:35.555 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.555 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.555 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.555 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.211501+0000", 2023-10-09T06:01:35.555 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001284348 2023-10-09T06:01:35.555 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.555 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.555 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:01:35.555 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.211509+0000", 2023-10-09T06:01:35.555 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6500000000000002e-06 2023-10-09T06:01:35.555 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.556 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.556 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.556 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.492952+0000", 2023-10-09T06:01:35.556 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2814423289999999 2023-10-09T06:01:35.556 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.556 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.556 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:01:35.556 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.493075+0000", 2023-10-09T06:01:35.556 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012328299999999999 2023-10-09T06:01:35.556 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.556 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.556 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:01:35.557 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.493189+0000", 2023-10-09T06:01:35.557 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000114299 2023-10-09T06:01:35.557 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.557 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.557 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:01:35.557 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.499919+0000", 2023-10-09T06:01:35.557 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0067300809999999997 2023-10-09T06:01:35.557 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.557 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.557 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.557 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.499987+0000", 2023-10-09T06:01:35.558 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8307999999999997e-05 2023-10-09T06:01:35.558 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.558 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.558 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.558 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.500057+0000", 2023-10-09T06:01:35.558 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9235999999999999e-05 2023-10-09T06:01:35.558 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.558 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.558 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:01:35.558 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.500122+0000", 2023-10-09T06:01:35.559 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5704999999999995e-05 2023-10-09T06:01:35.559 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.559 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.559 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:01:35.559 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.501053+0000", 2023-10-09T06:01:35.559 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00093106199999999997 2023-10-09T06:01:35.559 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.559 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:01:35.559 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.559 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.559 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.559 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:91018 6.1 6:8b58d611:::benchmark_data_smithi064_284193_object91017:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-10-09T06:01:35.560 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:10.215575+0000", 2023-10-09T06:01:35.560 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 565.33187277299999, 2023-10-09T06:01:35.560 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2973208469999999, 2023-10-09T06:01:35.560 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:01:35.560 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:01:35.560 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:01:35.560 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:01:35.560 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:01:35.560 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 91018 2023-10-09T06:01:35.560 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.560 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:01:35.561 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.561 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:01:35.561 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215575+0000", 2023-10-09T06:01:35.561 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.561 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.561 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.561 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:01:35.561 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215575+0000", 2023-10-09T06:01:35.561 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.561 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.561 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.562 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:01:35.562 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215574+0000", 2023-10-09T06:01:35.562 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:01:35.562 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.562 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.562 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:01:35.562 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215679+0000", 2023-10-09T06:01:35.562 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000104965 2023-10-09T06:01:35.562 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.562 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.562 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:01:35.563 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215692+0000", 2023-10-09T06:01:35.563 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3878e-05 2023-10-09T06:01:35.563 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.563 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.563 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:01:35.563 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215697+0000", 2023-10-09T06:01:35.563 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0259999999999998e-06 2023-10-09T06:01:35.563 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.563 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.563 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.563 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215746+0000", 2023-10-09T06:01:35.563 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8752999999999999e-05 2023-10-09T06:01:35.564 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.564 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.564 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:01:35.564 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.215755+0000", 2023-10-09T06:01:35.564 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0310000000000004e-06 2023-10-09T06:01:35.564 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.564 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.564 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.564 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.497327+0000", 2023-10-09T06:01:35.564 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.281571282 2023-10-09T06:01:35.565 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.565 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.565 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:01:35.565 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.497478+0000", 2023-10-09T06:01:35.565 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015099499999999999 2023-10-09T06:01:35.565 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.565 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.565 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:01:35.565 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.497652+0000", 2023-10-09T06:01:35.565 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00017489900000000001 2023-10-09T06:01:35.565 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.566 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.566 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:01:35.566 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.503871+0000", 2023-10-09T06:01:35.566 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0062182849999999996 2023-10-09T06:01:35.566 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.566 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.566 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.566 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.503926+0000", 2023-10-09T06:01:35.566 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5476999999999998e-05 2023-10-09T06:01:35.566 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.566 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.566 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.567 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.511760+0000", 2023-10-09T06:01:35.567 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0078334660000000007 2023-10-09T06:01:35.567 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.567 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.567 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:01:35.567 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.511816+0000", 2023-10-09T06:01:35.567 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5998000000000002e-05 2023-10-09T06:01:35.567 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.567 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.567 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:01:35.567 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.512896+0000", 2023-10-09T06:01:35.568 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001080225 2023-10-09T06:01:35.568 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.568 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:01:35.568 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.568 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.568 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.568 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:91030 6.1 6:82911adc:::benchmark_data_smithi064_284193_object91029:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-10-09T06:01:35.568 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:10.243578+0000", 2023-10-09T06:01:35.568 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 565.30386932600004, 2023-10-09T06:01:35.568 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2795882700000001, 2023-10-09T06:01:35.568 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:01:35.569 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:01:35.569 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:01:35.569 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:01:35.569 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:01:35.569 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 91030 2023-10-09T06:01:35.569 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.569 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:01:35.569 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.569 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:01:35.569 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.243578+0000", 2023-10-09T06:01:35.569 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.570 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.570 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.570 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:01:35.570 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.243578+0000", 2023-10-09T06:01:35.570 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.570 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.570 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.570 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:01:35.570 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.243577+0000", 2023-10-09T06:01:35.570 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:01:35.570 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.571 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.571 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:01:35.571 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.243622+0000", 2023-10-09T06:01:35.571 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5160000000000001e-05 2023-10-09T06:01:35.571 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.571 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.571 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:01:35.571 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.243635+0000", 2023-10-09T06:01:35.571 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2809000000000001e-05 2023-10-09T06:01:35.571 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.571 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.572 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:01:35.572 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.243642+0000", 2023-10-09T06:01:35.572 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4490000000000002e-06 2023-10-09T06:01:35.572 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.572 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.572 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.572 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.251460+0000", 2023-10-09T06:01:35.572 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0078177839999999995 2023-10-09T06:01:35.572 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.572 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.572 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:01:35.572 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.251471+0000", 2023-10-09T06:01:35.573 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1392e-05 2023-10-09T06:01:35.573 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.573 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.573 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.573 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.501114+0000", 2023-10-09T06:01:35.573 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.249642527 2023-10-09T06:01:35.573 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.573 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.573 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:01:35.573 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.501250+0000", 2023-10-09T06:01:35.573 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000135685 2023-10-09T06:01:35.574 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.574 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.574 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:01:35.574 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.501419+0000", 2023-10-09T06:01:35.574 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000169531 2023-10-09T06:01:35.574 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.574 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.574 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.574 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.511703+0000", 2023-10-09T06:01:35.574 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010283771000000001 2023-10-09T06:01:35.574 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.575 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.575 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:01:35.575 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.511717+0000", 2023-10-09T06:01:35.575 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3709999999999999e-05 2023-10-09T06:01:35.575 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.575 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.575 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.575 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.522279+0000", 2023-10-09T06:01:35.575 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010562826000000001 2023-10-09T06:01:35.575 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.575 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.576 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:01:35.576 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.522321+0000", 2023-10-09T06:01:35.576 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1269000000000001e-05 2023-10-09T06:01:35.576 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.576 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.576 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:01:35.576 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.523167+0000", 2023-10-09T06:01:35.576 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00084601999999999995 2023-10-09T06:01:35.576 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.576 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:01:35.576 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.577 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.577 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.577 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:91034 6.1 6:8c8f0c3f:::benchmark_data_smithi064_284193_object91033:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-10-09T06:01:35.577 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:10.252454+0000", 2023-10-09T06:01:35.577 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 565.29499374199997, 2023-10-09T06:01:35.577 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2936661039999999, 2023-10-09T06:01:35.577 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:01:35.577 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:01:35.577 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:01:35.577 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:01:35.577 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:01:35.578 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 91034 2023-10-09T06:01:35.578 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.578 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:01:35.578 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.578 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:01:35.578 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.252454+0000", 2023-10-09T06:01:35.578 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.578 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.578 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.578 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:01:35.578 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.252454+0000", 2023-10-09T06:01:35.579 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.579 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.579 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.579 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:01:35.579 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.252452+0000", 2023-10-09T06:01:35.579 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T06:01:35.579 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.579 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.579 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:01:35.579 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.252483+0000", 2023-10-09T06:01:35.579 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0713000000000002e-05 2023-10-09T06:01:35.580 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.580 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.580 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:01:35.580 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.252567+0000", 2023-10-09T06:01:35.580 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.4252999999999994e-05 2023-10-09T06:01:35.580 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.580 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.580 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:01:35.580 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.252576+0000", 2023-10-09T06:01:35.580 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8060000000000001e-06 2023-10-09T06:01:35.580 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.580 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.581 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.581 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.252630+0000", 2023-10-09T06:01:35.581 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4530000000000001e-05 2023-10-09T06:01:35.581 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.581 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.581 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:01:35.581 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.252641+0000", 2023-10-09T06:01:35.581 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0849999999999999e-05 2023-10-09T06:01:35.581 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.581 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.581 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.581 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.503991+0000", 2023-10-09T06:01:35.582 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.251350226 2023-10-09T06:01:35.582 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.582 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.582 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:01:35.582 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.504134+0000", 2023-10-09T06:01:35.582 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000142977 2023-10-09T06:01:35.582 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.582 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.582 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:01:35.582 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.504275+0000", 2023-10-09T06:01:35.582 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014111700000000001 2023-10-09T06:01:35.583 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.583 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.583 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:01:35.583 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.511724+0000", 2023-10-09T06:01:35.583 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0074489150000000004 2023-10-09T06:01:35.583 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.583 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.583 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.583 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.533876+0000", 2023-10-09T06:01:35.584 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.022151206 2023-10-09T06:01:35.584 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.584 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.584 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.584 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.544957+0000", 2023-10-09T06:01:35.584 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011081841 2023-10-09T06:01:35.584 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.584 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.585 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:01:35.585 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.545017+0000", 2023-10-09T06:01:35.585 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9849999999999998e-05 2023-10-09T06:01:35.585 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.586 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.586 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:01:35.586 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.546120+0000", 2023-10-09T06:01:35.586 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001102907 2023-10-09T06:01:35.586 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.586 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:01:35.586 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.586 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.586 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.586 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:91057 6.1 6:8e8f7f4c:::benchmark_data_smithi064_284193_object91056:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-10-09T06:01:35.586 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:10.295277+0000", 2023-10-09T06:01:35.587 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 565.25217085400004, 2023-10-09T06:01:35.587 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2516209790000001, 2023-10-09T06:01:35.587 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:01:35.587 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:01:35.587 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:01:35.587 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:01:35.587 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:01:35.587 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 91057 2023-10-09T06:01:35.587 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.587 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:01:35.587 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.588 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:01:35.588 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.295277+0000", 2023-10-09T06:01:35.588 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.588 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.588 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.588 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:01:35.588 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.295277+0000", 2023-10-09T06:01:35.588 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.588 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.588 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.588 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:01:35.588 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.295276+0000", 2023-10-09T06:01:35.589 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:01:35.589 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.589 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.589 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:01:35.589 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.295321+0000", 2023-10-09T06:01:35.589 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4869e-05 2023-10-09T06:01:35.589 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.589 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.589 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:01:35.589 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.295337+0000", 2023-10-09T06:01:35.589 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6314e-05 2023-10-09T06:01:35.589 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.590 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.590 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:01:35.590 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.295344+0000", 2023-10-09T06:01:35.590 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1269999999999997e-06 2023-10-09T06:01:35.590 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.590 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.590 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.590 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.295417+0000", 2023-10-09T06:01:35.590 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3091999999999997e-05 2023-10-09T06:01:35.590 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.591 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.591 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:01:35.591 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.295431+0000", 2023-10-09T06:01:35.591 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3939e-05 2023-10-09T06:01:35.591 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.591 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.591 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.591 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.506467+0000", 2023-10-09T06:01:35.591 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.211035485 2023-10-09T06:01:35.591 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.591 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.591 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:01:35.592 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.506593+0000", 2023-10-09T06:01:35.592 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012686699999999999 2023-10-09T06:01:35.592 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.592 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.592 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:01:35.592 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.506697+0000", 2023-10-09T06:01:35.592 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010326400000000001 2023-10-09T06:01:35.592 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.592 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.592 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:01:35.592 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.512920+0000", 2023-10-09T06:01:35.593 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.006223564 2023-10-09T06:01:35.593 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.593 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.593 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.593 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.533909+0000", 2023-10-09T06:01:35.593 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.020988494999999999 2023-10-09T06:01:35.593 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.593 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.593 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.593 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.546169+0000", 2023-10-09T06:01:35.593 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012259902 2023-10-09T06:01:35.593 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.594 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.594 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:01:35.594 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.546208+0000", 2023-10-09T06:01:35.594 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9106999999999997e-05 2023-10-09T06:01:35.594 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.594 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.594 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:01:35.594 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.546898+0000", 2023-10-09T06:01:35.594 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00069023700000000001 2023-10-09T06:01:35.594 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.594 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:01:35.595 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.595 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.595 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.595 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:91061 6.1 6:8cea4c39:::benchmark_data_smithi064_284193_object91060:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-10-09T06:01:35.595 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:52:10.313568+0000", 2023-10-09T06:01:35.595 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 565.23388019799995, 2023-10-09T06:01:35.595 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.233873561, 2023-10-09T06:01:35.595 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:01:35.595 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:01:35.595 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:01:35.595 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:01:35.596 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:01:35.596 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 91061 2023-10-09T06:01:35.596 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.596 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:01:35.596 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.596 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:01:35.596 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.313568+0000", 2023-10-09T06:01:35.596 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.596 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.596 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.597 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:01:35.597 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.313568+0000", 2023-10-09T06:01:35.597 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.597 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.597 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.597 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:01:35.597 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.313565+0000", 2023-10-09T06:01:35.597 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999971 2023-10-09T06:01:35.597 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.597 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.597 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:01:35.597 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.313611+0000", 2023-10-09T06:01:35.597 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6279999999999997e-05 2023-10-09T06:01:35.598 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.598 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.598 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:01:35.598 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.313636+0000", 2023-10-09T06:01:35.598 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.5298000000000001e-05 2023-10-09T06:01:35.598 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.598 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.598 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:01:35.598 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.313647+0000", 2023-10-09T06:01:35.598 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0319999999999999e-05 2023-10-09T06:01:35.598 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.598 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.599 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.599 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.313711+0000", 2023-10-09T06:01:35.599 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4789999999999995e-05 2023-10-09T06:01:35.599 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.599 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.599 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:01:35.599 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:10.313722+0000", 2023-10-09T06:01:35.599 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0789e-05 2023-10-09T06:01:35.599 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.599 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.599 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.599 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.508651+0000", 2023-10-09T06:01:35.600 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1949285670000001 2023-10-09T06:01:35.600 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.600 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.600 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:01:35.600 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.508783+0000", 2023-10-09T06:01:35.600 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013234499999999999 2023-10-09T06:01:35.600 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.600 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.600 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:01:35.600 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.508967+0000", 2023-10-09T06:01:35.600 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000183866 2023-10-09T06:01:35.600 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.601 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.601 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:01:35.601 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.513408+0000", 2023-10-09T06:01:35.601 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0044413930000000001 2023-10-09T06:01:35.601 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.601 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.601 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.601 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.533967+0000", 2023-10-09T06:01:35.601 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.020559194999999999 2023-10-09T06:01:35.601 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.601 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.601 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.601 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.546935+0000", 2023-10-09T06:01:35.602 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012967261000000001 2023-10-09T06:01:35.602 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.602 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.602 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:01:35.602 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.546968+0000", 2023-10-09T06:01:35.602 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.3235999999999998e-05 2023-10-09T06:01:35.602 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.602 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.602 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:01:35.602 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:52:11.547441+0000", 2023-10-09T06:01:35.602 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00047321899999999998 2023-10-09T06:01:35.602 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.603 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:01:35.603 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.603 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.603 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.603 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:102627 6.c 6:34d09788:::benchmark_data_smithi064_284193_object6084:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-10-09T06:01:35.603 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:53:21.277311+0000", 2023-10-09T06:01:35.603 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 494.27013654699999, 2023-10-09T06:01:35.603 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.93660849800000001, 2023-10-09T06:01:35.603 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:01:35.603 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:01:35.603 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:01:35.603 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:01:35.603 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:01:35.604 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 102627 2023-10-09T06:01:35.604 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.604 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:01:35.604 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.604 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:01:35.604 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:21.277311+0000", 2023-10-09T06:01:35.604 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.604 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.604 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.604 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:01:35.604 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:21.277311+0000", 2023-10-09T06:01:35.604 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.605 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.605 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.605 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:01:35.605 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:21.277310+0000", 2023-10-09T06:01:35.605 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:01:35.605 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.605 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.605 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:01:35.605 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:21.277316+0000", 2023-10-09T06:01:35.605 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5740000000000003e-06 2023-10-09T06:01:35.605 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.605 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.606 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:01:35.606 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:21.277329+0000", 2023-10-09T06:01:35.606 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2874000000000001e-05 2023-10-09T06:01:35.606 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.606 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.606 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:01:35.606 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:21.277334+0000", 2023-10-09T06:01:35.606 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0189999999999997e-06 2023-10-09T06:01:35.606 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.606 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.606 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.606 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:21.277394+0000", 2023-10-09T06:01:35.607 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0441e-05 2023-10-09T06:01:35.607 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.607 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.607 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:01:35.607 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:21.277403+0000", 2023-10-09T06:01:35.607 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1689999999999998e-06 2023-10-09T06:01:35.607 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.607 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.607 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.607 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:22.193437+0000", 2023-10-09T06:01:35.607 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.91603401600000001 2023-10-09T06:01:35.607 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.608 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.608 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:01:35.608 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:22.193515+0000", 2023-10-09T06:01:35.608 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7521999999999994e-05 2023-10-09T06:01:35.608 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.608 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.608 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,7", 2023-10-09T06:01:35.608 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:22.193593+0000", 2023-10-09T06:01:35.608 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7917000000000006e-05 2023-10-09T06:01:35.608 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.608 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.608 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:01:35.608 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:22.205663+0000", 2023-10-09T06:01:35.609 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.01207039 2023-10-09T06:01:35.609 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.609 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.609 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.609 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:22.213769+0000", 2023-10-09T06:01:35.609 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0081057839999999996 2023-10-09T06:01:35.609 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.609 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.609 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.609 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:22.213826+0000", 2023-10-09T06:01:35.610 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7033000000000001e-05 2023-10-09T06:01:35.610 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.610 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.610 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:01:35.610 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:22.213878+0000", 2023-10-09T06:01:35.610 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2361000000000002e-05 2023-10-09T06:01:35.610 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.610 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.610 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:01:35.610 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:53:22.213920+0000", 2023-10-09T06:01:35.610 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1567999999999999e-05 2023-10-09T06:01:35.611 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.611 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:01:35.611 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.611 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.611 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.611 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:109976 6.21 6:85084e13:::benchmark_data_smithi064_284193_object13433:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2023-10-09T06:01:35.611 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:22.220723+0000", 2023-10-09T06:01:35.611 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 373.32672477, 2023-10-09T06:01:35.611 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.134667216, 2023-10-09T06:01:35.611 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:01:35.612 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:01:35.612 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:01:35.612 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:01:35.612 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:01:35.612 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 109976 2023-10-09T06:01:35.612 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.612 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:01:35.612 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.612 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:01:35.612 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.220723+0000", 2023-10-09T06:01:35.612 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.613 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.613 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.613 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:01:35.613 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.220723+0000", 2023-10-09T06:01:35.613 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.613 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.613 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.613 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:01:35.613 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.220721+0000", 2023-10-09T06:01:35.613 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:01:35.613 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.614 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.614 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:01:35.614 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.220729+0000", 2023-10-09T06:01:35.614 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6699999999999994e-06 2023-10-09T06:01:35.614 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.614 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.614 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:01:35.614 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.220744+0000", 2023-10-09T06:01:35.614 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4598e-05 2023-10-09T06:01:35.614 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.615 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.615 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:01:35.615 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.220750+0000", 2023-10-09T06:01:35.615 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4760000000000003e-06 2023-10-09T06:01:35.615 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.615 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.615 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.615 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.220824+0000", 2023-10-09T06:01:35.615 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3264999999999997e-05 2023-10-09T06:01:35.615 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.615 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.616 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:01:35.616 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.220831+0000", 2023-10-09T06:01:35.616 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8140000000000001e-06 2023-10-09T06:01:35.616 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.616 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.616 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.616 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.219898+0000", 2023-10-09T06:01:35.616 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99906647000000004 2023-10-09T06:01:35.616 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.616 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.616 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:01:35.617 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.219909+0000", 2023-10-09T06:01:35.617 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1167e-05 2023-10-09T06:01:35.617 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.617 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.617 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.617 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.315716+0000", 2023-10-09T06:01:35.617 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.095807035 2023-10-09T06:01:35.617 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.617 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.617 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for degraded object", 2023-10-09T06:01:35.617 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.316210+0000", 2023-10-09T06:01:35.617 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00049411799999999999 2023-10-09T06:01:35.618 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.618 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.618 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.618 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.335877+0000", 2023-10-09T06:01:35.618 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.019666757 2023-10-09T06:01:35.618 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.618 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.618 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:01:35.618 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.335976+0000", 2023-10-09T06:01:35.618 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.8820999999999995e-05 2023-10-09T06:01:35.619 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.619 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.619 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1,7", 2023-10-09T06:01:35.619 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.336151+0000", 2023-10-09T06:01:35.619 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00017507699999999999 2023-10-09T06:01:35.619 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.619 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.619 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.619 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.338161+0000", 2023-10-09T06:01:35.619 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0020105320000000002 2023-10-09T06:01:35.619 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.620 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.620 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:01:35.620 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.340847+0000", 2023-10-09T06:01:35.620 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0026856860000000001 2023-10-09T06:01:35.620 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.620 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.620 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.620 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.342749+0000", 2023-10-09T06:01:35.620 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0019020090000000001 2023-10-09T06:01:35.620 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.620 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.621 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.621 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.354486+0000", 2023-10-09T06:01:35.621 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.01173688 2023-10-09T06:01:35.621 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.621 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.621 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:01:35.621 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.354531+0000", 2023-10-09T06:01:35.621 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4817000000000001e-05 2023-10-09T06:01:35.621 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.621 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.621 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:01:35.622 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.355390+0000", 2023-10-09T06:01:35.622 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00085957899999999996 2023-10-09T06:01:35.622 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.622 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:01:35.622 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.622 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.622 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.622 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:110006 6.21 6:85ed4bc8:::benchmark_data_smithi064_284193_object13463:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2023-10-09T06:01:35.622 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:22.328800+0000", 2023-10-09T06:01:35.622 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 373.21864818699999, 2023-10-09T06:01:35.622 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0219389460000001, 2023-10-09T06:01:35.622 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:01:35.623 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:01:35.623 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:01:35.623 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:01:35.623 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:01:35.623 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 110006 2023-10-09T06:01:35.623 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.623 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:01:35.623 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.623 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:01:35.623 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.328800+0000", 2023-10-09T06:01:35.624 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.624 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.624 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.624 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:01:35.624 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.328800+0000", 2023-10-09T06:01:35.624 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.624 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.624 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.624 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:01:35.624 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.328798+0000", 2023-10-09T06:01:35.624 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:01:35.624 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.624 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.625 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:01:35.625 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.328804+0000", 2023-10-09T06:01:35.625 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8039999999999998e-06 2023-10-09T06:01:35.625 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.625 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.625 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:01:35.625 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.328816+0000", 2023-10-09T06:01:35.625 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1617000000000001e-05 2023-10-09T06:01:35.625 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.625 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.625 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:01:35.625 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.328820+0000", 2023-10-09T06:01:35.626 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8829999999999997e-06 2023-10-09T06:01:35.626 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.626 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.626 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.626 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.328857+0000", 2023-10-09T06:01:35.626 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6310999999999998e-05 2023-10-09T06:01:35.626 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.626 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.626 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:01:35.626 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.328863+0000", 2023-10-09T06:01:35.626 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.263e-06 2023-10-09T06:01:35.626 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.627 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.627 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.627 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.219935+0000", 2023-10-09T06:01:35.627 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.891072015 2023-10-09T06:01:35.627 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.627 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.627 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:01:35.627 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.219942+0000", 2023-10-09T06:01:35.627 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3069999999999997e-06 2023-10-09T06:01:35.627 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.627 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.627 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.627 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.316343+0000", 2023-10-09T06:01:35.628 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0964007730000001 2023-10-09T06:01:35.628 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.628 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.628 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for degraded object", 2023-10-09T06:01:35.628 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.316719+0000", 2023-10-09T06:01:35.628 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00037633099999999999 2023-10-09T06:01:35.628 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.628 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.628 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.628 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.323220+0000", 2023-10-09T06:01:35.628 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.006500029 2023-10-09T06:01:35.628 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.629 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.629 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:01:35.629 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.323312+0000", 2023-10-09T06:01:35.629 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2373999999999993e-05 2023-10-09T06:01:35.629 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.629 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.629 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1,7", 2023-10-09T06:01:35.629 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.323535+0000", 2023-10-09T06:01:35.629 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00022307100000000001 2023-10-09T06:01:35.629 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.629 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.629 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:01:35.630 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.327521+0000", 2023-10-09T06:01:35.630 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0039856470000000001 2023-10-09T06:01:35.630 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.630 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.630 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.630 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.335667+0000", 2023-10-09T06:01:35.630 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0081462110000000004 2023-10-09T06:01:35.630 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.630 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.630 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.630 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.335706+0000", 2023-10-09T06:01:35.631 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9229e-05 2023-10-09T06:01:35.631 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.631 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.631 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.631 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.349356+0000", 2023-10-09T06:01:35.631 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013650367 2023-10-09T06:01:35.631 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.631 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.631 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:01:35.631 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.349412+0000", 2023-10-09T06:01:35.631 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5300000000000002e-05 2023-10-09T06:01:35.632 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.632 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.632 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:01:35.632 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.350739+0000", 2023-10-09T06:01:35.632 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0013268690000000001 2023-10-09T06:01:35.632 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.632 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:01:35.632 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.632 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.632 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.632 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:110053 6.21 6:876911f3:::benchmark_data_smithi064_284193_object13510:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2023-10-09T06:01:35.632 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:23.288180+0000", 2023-10-09T06:01:35.633 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 372.259267749, 2023-10-09T06:01:35.633 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.063805025, 2023-10-09T06:01:35.633 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:01:35.633 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:01:35.633 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:01:35.633 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:01:35.633 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:01:35.633 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 110053 2023-10-09T06:01:35.633 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.633 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:01:35.633 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.633 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:01:35.633 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.288180+0000", 2023-10-09T06:01:35.634 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.634 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.634 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.634 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:01:35.634 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.288180+0000", 2023-10-09T06:01:35.634 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.634 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.634 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.634 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:01:35.634 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.288179+0000", 2023-10-09T06:01:35.635 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:01:35.635 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.635 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.635 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:01:35.635 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.288185+0000", 2023-10-09T06:01:35.635 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4790000000000004e-06 2023-10-09T06:01:35.635 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.635 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.635 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:01:35.636 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.288199+0000", 2023-10-09T06:01:35.636 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4280999999999999e-05 2023-10-09T06:01:35.636 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.636 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.636 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:01:35.636 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.288205+0000", 2023-10-09T06:01:35.636 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8529999999999997e-06 2023-10-09T06:01:35.636 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.636 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.636 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.636 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.288251+0000", 2023-10-09T06:01:35.637 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5917000000000001e-05 2023-10-09T06:01:35.637 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.637 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.637 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:01:35.637 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.288257+0000", 2023-10-09T06:01:35.637 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0900000000000001e-06 2023-10-09T06:01:35.637 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.637 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.637 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.637 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.316765+0000", 2023-10-09T06:01:35.637 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.028508288 2023-10-09T06:01:35.637 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.638 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.638 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for degraded object", 2023-10-09T06:01:35.638 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.317151+0000", 2023-10-09T06:01:35.638 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000385635 2023-10-09T06:01:35.638 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.638 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.638 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.638 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.326489+0000", 2023-10-09T06:01:35.638 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0093383319999999995 2023-10-09T06:01:35.638 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.638 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.638 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:01:35.639 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.326588+0000", 2023-10-09T06:01:35.639 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.8735999999999998e-05 2023-10-09T06:01:35.639 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.639 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.639 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1,7", 2023-10-09T06:01:35.639 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.326814+0000", 2023-10-09T06:01:35.639 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00022614 2023-10-09T06:01:35.639 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.639 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.639 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:01:35.639 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.330828+0000", 2023-10-09T06:01:35.639 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0040141630000000003 2023-10-09T06:01:35.640 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.640 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.640 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.640 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.335741+0000", 2023-10-09T06:01:35.640 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0049126939999999996 2023-10-09T06:01:35.640 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.640 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.640 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.640 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.335776+0000", 2023-10-09T06:01:35.640 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5185999999999998e-05 2023-10-09T06:01:35.640 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.640 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.640 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.641 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.350800+0000", 2023-10-09T06:01:35.641 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015023610999999999 2023-10-09T06:01:35.641 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.641 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.641 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:01:35.641 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.350859+0000", 2023-10-09T06:01:35.641 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8959000000000003e-05 2023-10-09T06:01:35.641 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.641 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.641 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:01:35.641 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.351985+0000", 2023-10-09T06:01:35.641 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001126436 2023-10-09T06:01:35.642 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.642 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:01:35.642 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.642 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.642 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.642 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:110068 6.21 6:8410efa4:::benchmark_data_smithi064_284193_object13525:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2023-10-09T06:01:35.642 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:23.319596+0000", 2023-10-09T06:01:35.642 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 372.22785148499997, 2023-10-09T06:01:35.642 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0337012750000001, 2023-10-09T06:01:35.642 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:01:35.642 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:01:35.642 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:01:35.643 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:01:35.643 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:01:35.643 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 110068 2023-10-09T06:01:35.643 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.643 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:01:35.643 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.643 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:01:35.643 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.319596+0000", 2023-10-09T06:01:35.643 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.643 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.643 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.644 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:01:35.644 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.319596+0000", 2023-10-09T06:01:35.644 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.644 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.644 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.644 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:01:35.644 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.319595+0000", 2023-10-09T06:01:35.644 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:01:35.644 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.644 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.644 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:01:35.644 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.319604+0000", 2023-10-09T06:01:35.645 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6170000000000007e-06 2023-10-09T06:01:35.645 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.645 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.645 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:01:35.645 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.319621+0000", 2023-10-09T06:01:35.645 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6889e-05 2023-10-09T06:01:35.645 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.645 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.645 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:01:35.645 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.319627+0000", 2023-10-09T06:01:35.645 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3949999999999998e-06 2023-10-09T06:01:35.645 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.646 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.646 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.646 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.319667+0000", 2023-10-09T06:01:35.646 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9730000000000001e-05 2023-10-09T06:01:35.646 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.646 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.646 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:01:35.646 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.319675+0000", 2023-10-09T06:01:35.646 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.2700000000000004e-06 2023-10-09T06:01:35.646 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.646 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.646 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.646 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.317190+0000", 2023-10-09T06:01:35.647 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99751477600000005 2023-10-09T06:01:35.647 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.647 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.647 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for degraded object", 2023-10-09T06:01:35.647 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.317652+0000", 2023-10-09T06:01:35.647 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00046239000000000002 2023-10-09T06:01:35.647 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.647 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.647 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.647 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.329705+0000", 2023-10-09T06:01:35.647 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012052564999999999 2023-10-09T06:01:35.647 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.647 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.648 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:01:35.648 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.329802+0000", 2023-10-09T06:01:35.648 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7205000000000002e-05 2023-10-09T06:01:35.648 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.648 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.648 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1,7", 2023-10-09T06:01:35.648 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.330029+0000", 2023-10-09T06:01:35.648 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000226727 2023-10-09T06:01:35.648 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.648 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.648 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:01:35.648 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.335626+0000", 2023-10-09T06:01:35.649 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0055977759999999996 2023-10-09T06:01:35.649 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.649 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.649 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.649 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.335812+0000", 2023-10-09T06:01:35.649 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000185463 2023-10-09T06:01:35.649 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.649 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.649 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.649 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.335852+0000", 2023-10-09T06:01:35.649 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0194000000000002e-05 2023-10-09T06:01:35.649 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.649 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.650 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:01:35.650 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.352058+0000", 2023-10-09T06:01:35.650 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.016205780999999999 2023-10-09T06:01:35.650 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.650 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.650 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:01:35.650 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.352116+0000", 2023-10-09T06:01:35.650 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8004000000000002e-05 2023-10-09T06:01:35.650 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.650 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.650 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:01:35.651 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.353298+0000", 2023-10-09T06:01:35.651 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0011818250000000001 2023-10-09T06:01:35.651 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.651 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:01:35.651 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.651 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.651 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.651 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:119186 6.21 6.53276d61 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-10-09T06:01:35.651 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:43.708969+0000", 2023-10-09T06:01:35.651 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 351.83847839200001, 2023-10-09T06:01:35.652 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96142623199999999, 2023-10-09T06:01:35.652 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:01:35.652 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:01:35.652 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:01:35.652 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:01:35.652 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:01:35.652 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 119186 2023-10-09T06:01:35.652 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.652 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:01:35.652 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.652 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:01:35.652 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.708969+0000", 2023-10-09T06:01:35.652 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.653 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.653 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.653 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:01:35.653 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.708969+0000", 2023-10-09T06:01:35.653 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.653 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.653 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.653 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:01:35.653 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.708968+0000", 2023-10-09T06:01:35.653 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:01:35.653 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.653 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.654 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:01:35.654 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.708975+0000", 2023-10-09T06:01:35.654 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4830000000000002e-06 2023-10-09T06:01:35.654 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.654 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.654 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:01:35.654 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.708993+0000", 2023-10-09T06:01:35.654 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7846000000000002e-05 2023-10-09T06:01:35.654 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.654 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.654 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:01:35.654 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.709000+0000", 2023-10-09T06:01:35.655 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5050000000000004e-06 2023-10-09T06:01:35.655 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.655 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.655 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.655 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.709058+0000", 2023-10-09T06:01:35.655 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7797999999999998e-05 2023-10-09T06:01:35.655 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.655 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.655 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:01:35.655 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.709065+0000", 2023-10-09T06:01:35.655 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.464e-06 2023-10-09T06:01:35.655 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.656 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.656 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.656 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.670387+0000", 2023-10-09T06:01:35.656 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96132219799999996 2023-10-09T06:01:35.656 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.656 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.656 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:01:35.656 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.670396+0000", 2023-10-09T06:01:35.656 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.4409999999999993e-06 2023-10-09T06:01:35.656 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.656 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:01:35.656 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.657 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.657 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.657 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "MOSDPGPush(6.24 235/234 [PushOp(6:26cb90f2:::benchmark_data_smithi064_284193_object47300:head, version: 112'2935, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:26cb90f2:::benchmark_data_smithi064_284193_object47300:head@112'2935, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T06:01:35.657 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:56:53.542726+0000", 2023-10-09T06:01:35.657 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 282.00472133599999, 2023-10-09T06:01:35.657 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.85973538599999999, 2023-10-09T06:01:35.657 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:01:35.657 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:01:35.657 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:01:35.657 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.657 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:01:35.657 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.542726+0000", 2023-10-09T06:01:35.657 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.658 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.658 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.658 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:01:35.658 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.542726+0000", 2023-10-09T06:01:35.658 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.658 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.658 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.658 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:01:35.658 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.542726+0000", 2023-10-09T06:01:35.658 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:01:35.658 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.659 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.659 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:01:35.659 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.542855+0000", 2023-10-09T06:01:35.659 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0001292 2023-10-09T06:01:35.659 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.659 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.659 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:01:35.659 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.542870+0000", 2023-10-09T06:01:35.659 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5679e-05 2023-10-09T06:01:35.659 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.659 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.660 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:01:35.660 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.542883+0000", 2023-10-09T06:01:35.660 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2077e-05 2023-10-09T06:01:35.660 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.660 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.660 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.660 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:54.395627+0000", 2023-10-09T06:01:35.660 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.85274424599999998 2023-10-09T06:01:35.660 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.660 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.660 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:01:35.661 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:54.395768+0000", 2023-10-09T06:01:35.661 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014087600000000001 2023-10-09T06:01:35.661 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.661 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.661 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:01:35.661 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:54.402462+0000", 2023-10-09T06:01:35.661 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.006694233 2023-10-09T06:01:35.661 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.661 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:01:35.661 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.662 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.662 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.662 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.44588.0:143737 6.4 e235/234 6:21808b25:::benchmark_data_smithi064_284193_object47194:head v 235'7729, mlcod=0'0)", 2023-10-09T06:01:35.662 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:56:53.549226+0000", 2023-10-09T06:01:35.662 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 281.99822175499997, 2023-10-09T06:01:35.662 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.84857932199999997, 2023-10-09T06:01:35.662 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:01:35.662 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:01:35.662 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:01:35.662 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.662 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:01:35.663 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.549226+0000", 2023-10-09T06:01:35.663 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.663 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.663 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.663 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:01:35.663 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.549226+0000", 2023-10-09T06:01:35.663 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.663 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.663 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.663 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:01:35.663 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.549225+0000", 2023-10-09T06:01:35.664 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:01:35.664 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.664 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.664 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:01:35.664 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.549234+0000", 2023-10-09T06:01:35.664 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.7579999999999998e-06 2023-10-09T06:01:35.664 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.664 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.664 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:01:35.664 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.549248+0000", 2023-10-09T06:01:35.664 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4245e-05 2023-10-09T06:01:35.664 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.665 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.665 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:01:35.665 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.549255+0000", 2023-10-09T06:01:35.665 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3440000000000003e-06 2023-10-09T06:01:35.665 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.665 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.665 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.665 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:54.395527+0000", 2023-10-09T06:01:35.665 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.84627187999999998 2023-10-09T06:01:35.665 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.665 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.666 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:01:35.666 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:54.395553+0000", 2023-10-09T06:01:35.666 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.5734000000000001e-05 2023-10-09T06:01:35.666 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.666 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.666 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:01:35.666 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:54.397766+0000", 2023-10-09T06:01:35.666 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0022124129999999999 2023-10-09T06:01:35.666 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.666 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.667 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:01:35.667 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:54.397805+0000", 2023-10-09T06:01:35.667 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9872e-05 2023-10-09T06:01:35.667 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.667 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:01:35.667 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.667 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.667 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.667 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "MOSDPGPush(6.14 235/234 [PushOp(6:2de4afdc:::benchmark_data_smithi064_284193_object47175:head, version: 112'2931, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:2de4afdc:::benchmark_data_smithi064_284193_object47175:head@112'2931, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T06:01:35.667 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:56:53.568140+0000", 2023-10-09T06:01:35.667 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 281.97930807799997, 2023-10-09T06:01:35.667 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.85536617299999995, 2023-10-09T06:01:35.667 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:01:35.668 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:01:35.668 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:01:35.668 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.668 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:01:35.668 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.568140+0000", 2023-10-09T06:01:35.668 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.668 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.668 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.668 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:01:35.668 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.568140+0000", 2023-10-09T06:01:35.668 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.668 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.669 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.669 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:01:35.669 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.568139+0000", 2023-10-09T06:01:35.669 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:01:35.669 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.669 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.669 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:01:35.669 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.568171+0000", 2023-10-09T06:01:35.669 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.2484999999999998e-05 2023-10-09T06:01:35.669 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.669 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.670 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:01:35.670 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.568194+0000", 2023-10-09T06:01:35.670 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2340999999999998e-05 2023-10-09T06:01:35.670 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.670 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.670 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:01:35.670 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.568205+0000", 2023-10-09T06:01:35.670 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1069e-05 2023-10-09T06:01:35.670 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.670 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.670 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.671 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:54.413175+0000", 2023-10-09T06:01:35.671 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.84496972699999995 2023-10-09T06:01:35.671 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.671 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.671 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:01:35.671 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:54.413449+0000", 2023-10-09T06:01:35.671 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00027475200000000001 2023-10-09T06:01:35.671 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.671 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.671 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:01:35.671 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:54.423506+0000", 2023-10-09T06:01:35.671 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010056644999999999 2023-10-09T06:01:35.671 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.672 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:01:35.672 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.672 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.672 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.672 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "MOSDPGPush(6.14 235/234 [PushOp(6:280e042a:::benchmark_data_smithi064_284193_object47198:head, version: 112'2933, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:280e042a:::benchmark_data_smithi064_284193_object47198:head@112'2933, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T06:01:35.672 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:56:53.571690+0000", 2023-10-09T06:01:35.672 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 281.97575738299997, 2023-10-09T06:01:35.672 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.87127679300000005, 2023-10-09T06:01:35.672 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:01:35.672 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:01:35.672 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:01:35.672 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.672 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:01:35.673 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.571690+0000", 2023-10-09T06:01:35.673 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.673 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.673 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.673 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:01:35.673 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.571690+0000", 2023-10-09T06:01:35.673 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.673 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.673 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.673 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:01:35.673 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.571689+0000", 2023-10-09T06:01:35.673 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:01:35.673 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.674 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.674 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:01:35.674 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.571749+0000", 2023-10-09T06:01:35.674 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0055e-05 2023-10-09T06:01:35.674 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.674 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.674 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:01:35.674 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.571769+0000", 2023-10-09T06:01:35.674 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9701e-05 2023-10-09T06:01:35.674 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.674 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.675 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:01:35.675 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.571780+0000", 2023-10-09T06:01:35.675 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0492000000000001e-05 2023-10-09T06:01:35.675 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.675 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.675 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.675 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:54.427826+0000", 2023-10-09T06:01:35.675 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.85604601999999996 2023-10-09T06:01:35.675 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.675 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.675 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:01:35.675 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:54.428098+0000", 2023-10-09T06:01:35.675 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00027188700000000001 2023-10-09T06:01:35.676 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.676 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.676 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:01:35.676 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:54.442967+0000", 2023-10-09T06:01:35.676 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.014869701000000001 2023-10-09T06:01:35.676 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.676 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:01:35.676 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.676 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.676 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.676 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "MOSDPGPush(6.14 235/234 [PushOp(6:2a776da5:::benchmark_data_smithi064_284193_object47237:head, version: 112'2934, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:2a776da5:::benchmark_data_smithi064_284193_object47237:head@112'2934, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T06:01:35.676 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:56:53.575154+0000", 2023-10-09T06:01:35.676 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 281.97229333500002, 2023-10-09T06:01:35.677 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.88072734799999997, 2023-10-09T06:01:35.677 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:01:35.677 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:01:35.677 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:01:35.677 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.677 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:01:35.677 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.575154+0000", 2023-10-09T06:01:35.677 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.677 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.677 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.677 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:01:35.677 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.575154+0000", 2023-10-09T06:01:35.678 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.678 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.678 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.678 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:01:35.678 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.575154+0000", 2023-10-09T06:01:35.678 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:01:35.678 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.678 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.678 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:01:35.678 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.575198+0000", 2023-10-09T06:01:35.678 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3674999999999998e-05 2023-10-09T06:01:35.679 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.679 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.679 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:01:35.679 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.575213+0000", 2023-10-09T06:01:35.679 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5234e-05 2023-10-09T06:01:35.679 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.680 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:01:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:01:35] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.43.0" 2023-10-09T06:01:35.680 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.680 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:01:35.680 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.575220+0000", 2023-10-09T06:01:35.680 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0659999999999998e-06 2023-10-09T06:01:35.680 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.680 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.680 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.680 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:54.443267+0000", 2023-10-09T06:01:35.680 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.86804690200000001 2023-10-09T06:01:35.680 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.681 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.681 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:01:35.681 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:54.443563+0000", 2023-10-09T06:01:35.681 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00029646499999999999 2023-10-09T06:01:35.681 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.681 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.681 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:01:35.681 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:54.455882+0000", 2023-10-09T06:01:35.681 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012318607000000001 2023-10-09T06:01:35.681 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.681 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:01:35.682 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.682 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.682 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.682 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "MOSDPGPush(6.14 235/234 [PushOp(6:2c9a5287:::benchmark_data_smithi064_284193_object47302:head, version: 112'2936, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:2c9a5287:::benchmark_data_smithi064_284193_object47302:head@112'2936, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T06:01:35.682 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:56:53.578539+0000", 2023-10-09T06:01:35.682 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 281.96890913200002, 2023-10-09T06:01:35.682 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.89352408100000003, 2023-10-09T06:01:35.682 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:01:35.682 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:01:35.682 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:01:35.682 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.683 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:01:35.683 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.578539+0000", 2023-10-09T06:01:35.683 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.683 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.683 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.683 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:01:35.683 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.578539+0000", 2023-10-09T06:01:35.683 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.683 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.683 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.684 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:01:35.684 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.578538+0000", 2023-10-09T06:01:35.684 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:01:35.684 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.684 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.684 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:01:35.684 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.578595+0000", 2023-10-09T06:01:35.684 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7040999999999998e-05 2023-10-09T06:01:35.684 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.684 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.684 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:01:35.685 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.578614+0000", 2023-10-09T06:01:35.685 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8749e-05 2023-10-09T06:01:35.685 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.685 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.685 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:01:35.685 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:53.578624+0000", 2023-10-09T06:01:35.685 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0339999999999999e-05 2023-10-09T06:01:35.685 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.685 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.685 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.685 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:54.459873+0000", 2023-10-09T06:01:35.686 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.88124873400000003 2023-10-09T06:01:35.686 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.686 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.686 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:01:35.686 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:54.460159+0000", 2023-10-09T06:01:35.686 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00028563900000000002 2023-10-09T06:01:35.686 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.686 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.686 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:01:35.686 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:54.472063+0000", 2023-10-09T06:01:35.686 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011904253 2023-10-09T06:01:35.687 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.687 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:01:35.687 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.687 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.687 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.687 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:159559 6.14 6:2ab23c03:::benchmark_data_smithi064_284193_object63016:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2023-10-09T06:01:35.687 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:22.405499+0000", 2023-10-09T06:01:35.687 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 253.14194856099999, 2023-10-09T06:01:35.687 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0718210610000001, 2023-10-09T06:01:35.687 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:01:35.688 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:01:35.688 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:01:35.688 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:01:35.688 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:01:35.688 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 159559 2023-10-09T06:01:35.688 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.688 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:01:35.688 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.688 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:01:35.688 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.405499+0000", 2023-10-09T06:01:35.689 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.689 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.689 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.689 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:01:35.689 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.405499+0000", 2023-10-09T06:01:35.689 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.689 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.689 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.689 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:01:35.689 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.405498+0000", 2023-10-09T06:01:35.689 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:01:35.690 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.690 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.690 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:01:35.690 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.405504+0000", 2023-10-09T06:01:35.690 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4590000000000003e-06 2023-10-09T06:01:35.690 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.690 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.690 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:01:35.690 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.405536+0000", 2023-10-09T06:01:35.690 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1925999999999999e-05 2023-10-09T06:01:35.690 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.691 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.691 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:01:35.691 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.405543+0000", 2023-10-09T06:01:35.691 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9349999999999997e-06 2023-10-09T06:01:35.691 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.691 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.691 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.691 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.406423+0000", 2023-10-09T06:01:35.691 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00087985299999999999 2023-10-09T06:01:35.691 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.691 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.691 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for degraded object", 2023-10-09T06:01:35.692 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.413486+0000", 2023-10-09T06:01:35.692 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0070627880000000004 2023-10-09T06:01:35.692 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.692 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.692 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.692 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.431989+0000", 2023-10-09T06:01:35.692 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018503188 2023-10-09T06:01:35.692 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.692 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.692 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:01:35.692 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.434240+0000", 2023-10-09T06:01:35.692 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002251327 2023-10-09T06:01:35.692 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.693 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.693 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T06:01:35.693 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.439336+0000", 2023-10-09T06:01:35.693 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0050958389999999996 2023-10-09T06:01:35.693 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.693 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.693 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.693 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.482279+0000", 2023-10-09T06:01:35.693 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.042942532999999998 2023-10-09T06:01:35.693 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.693 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.693 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:01:35.694 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.482288+0000", 2023-10-09T06:01:35.694 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.6220000000000004e-06 2023-10-09T06:01:35.694 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.694 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.694 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.694 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:23.477304+0000", 2023-10-09T06:01:35.694 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99501541400000004 2023-10-09T06:01:35.694 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.694 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.694 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:01:35.694 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:23.477320+0000", 2023-10-09T06:01:35.694 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6549e-05 2023-10-09T06:01:35.695 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.695 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:01:35.695 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.695 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.695 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.695 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:159609 6.14 6.5ca57ad4 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-10-09T06:01:35.695 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:22.481099+0000", 2023-10-09T06:01:35.695 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 253.06634894300001, 2023-10-09T06:01:35.695 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99643836100000005, 2023-10-09T06:01:35.695 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:01:35.695 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:01:35.696 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:01:35.696 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:01:35.696 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:01:35.696 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 159609 2023-10-09T06:01:35.696 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.696 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:01:35.696 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.696 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:01:35.696 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.481099+0000", 2023-10-09T06:01:35.697 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.697 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.697 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.697 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:01:35.697 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.481099+0000", 2023-10-09T06:01:35.697 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:01:35.697 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.697 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.697 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:01:35.697 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.481098+0000", 2023-10-09T06:01:35.698 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:01:35.698 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.698 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.698 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:01:35.698 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.481103+0000", 2023-10-09T06:01:35.698 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8029999999999996e-06 2023-10-09T06:01:35.698 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.698 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.698 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:01:35.698 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.481114+0000", 2023-10-09T06:01:35.699 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1382e-05 2023-10-09T06:01:35.699 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.699 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.699 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:01:35.699 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.481119+0000", 2023-10-09T06:01:35.699 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7679999999999999e-06 2023-10-09T06:01:35.699 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.699 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.699 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.699 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.487792+0000", 2023-10-09T06:01:35.699 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0066734580000000002 2023-10-09T06:01:35.700 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.700 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.700 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:01:35.700 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.487798+0000", 2023-10-09T06:01:35.700 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4589999999999997e-06 2023-10-09T06:01:35.700 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.700 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.700 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:01:35.700 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:23.477526+0000", 2023-10-09T06:01:35.700 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.98972824299999995 2023-10-09T06:01:35.700 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:01:35.701 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:01:35.701 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:01:35.701 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:23.477537+0000", 2023-10-09T06:01:35.701 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1112e-05 2023-10-09T06:01:35.701 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.701 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:01:35.701 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.701 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:01:35.701 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:01:35.701 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:01:35.702 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:35.702 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:01:35.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:35.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:01:35.807 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:01:35.856 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:35.867 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:35.922 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:01:35.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:35.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:01:35.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:35 smithi064 ceph-mon[212750]: pgmap v778: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.4 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 4511/116007 objects degraded (3.889%); 0 B/s, 95 objects/s recovering 2023-10-09T06:01:35.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:35 smithi064 ceph-mon[220044]: pgmap v778: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.4 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 4511/116007 objects degraded (3.889%); 0 B/s, 95 objects/s recovering 2023-10-09T06:01:36.019 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:01:36.028 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:36.053 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:01:36.100 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:01:36.113 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:36.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:35 smithi156 ceph-mon[195101]: pgmap v778: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.4 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 4511/116007 objects degraded (3.889%); 0 B/s, 95 objects/s recovering 2023-10-09T06:01:36.127 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:01:36.168 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:01:36.199 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:36.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:36.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:36.213 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:01:36.259 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:01:36.293 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:01:36.295 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:01:36.295 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:01:36.295 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:01:36.295 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:01:36.295 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:01:36.296 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:01:36.296 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:01:36.296 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:01:36.313 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:36.385 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:36.533 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:01:36.637 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:36.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:36.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:36.749 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:01:36.812 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:36.888 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:01:36.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:36.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:01:36.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:36 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3032703473' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:36.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:36 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3032703473' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:36.997 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:37.070 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:37.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:36 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3032703473' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:37.150 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:01:37.284 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:01:37.307 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:37.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:37.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:37.414 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:37.469 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:37.594 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:37.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:37.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:37.698 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:37.745 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:38.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:37 smithi156 ceph-mon[195101]: pgmap v779: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.2 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 772 op/s; 3325/109035 objects degraded (3.049%); 0 B/s, 179 objects/s recovering 2023-10-09T06:01:38.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:37 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 3325/109035 objects degraded (3.049%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:38.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:37 smithi064 ceph-mon[212750]: pgmap v779: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.2 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 772 op/s; 3325/109035 objects degraded (3.049%); 0 B/s, 179 objects/s recovering 2023-10-09T06:01:38.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:37 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 3325/109035 objects degraded (3.049%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:38.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:37 smithi064 ceph-mon[220044]: pgmap v779: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.2 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 772 op/s; 3325/109035 objects degraded (3.049%); 0 B/s, 179 objects/s recovering 2023-10-09T06:01:38.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:37 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 3325/109035 objects degraded (3.049%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:38.748 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:38.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:38.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:38.873 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:38.894 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:38.971 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:39.080 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:01:39.080 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:01:39.246 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:39.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:39.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:39.356 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:01:39.400 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:39.445 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:01:39.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:39.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:01:39.549 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:39.583 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:39.872 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:01:39.883 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:39.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:39.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:39.989 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:40.070 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:40.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:39 smithi156 ceph-mon[195101]: pgmap v780: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.1 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 3325/105471 objects degraded (3.153%); 0 B/s, 83 objects/s recovering 2023-10-09T06:01:40.140 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:40.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:40.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:40.181 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:39 smithi064 ceph-mon[212750]: pgmap v780: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.1 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 3325/105471 objects degraded (3.153%); 0 B/s, 83 objects/s recovering 2023-10-09T06:01:40.181 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:39 smithi064 ceph-mon[220044]: pgmap v780: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.1 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 3325/105471 objects degraded (3.153%); 0 B/s, 83 objects/s recovering 2023-10-09T06:01:40.247 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:40.279 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:40.328 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:01:40.371 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:40.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:40.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:40.475 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:01:40.517 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:40.611 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:01:40.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:40.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:01:40.666 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:01:40.718 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:40.815 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:40.951 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.0 primary-affinity to 0.52907 (8346732) 2023-10-09T06:01:40.958 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:40.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:40 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1108099944' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.5290699089564754}]: dispatch 2023-10-09T06:01:40.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:40 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.5290699089564754}]: dispatch 2023-10-09T06:01:40.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:40 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1108099944' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.5290699089564754}]: dispatch 2023-10-09T06:01:40.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:40 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.5290699089564754}]: dispatch 2023-10-09T06:01:40.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:40.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:41.080 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:01:41.106 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:40 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1108099944' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.5290699089564754}]: dispatch 2023-10-09T06:01:41.106 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:40 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.5290699089564754}]: dispatch 2023-10-09T06:01:41.109 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:41.154 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:01:41.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:41.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:01:41.258 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:41.284 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:41.569 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:41.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:41.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:41.678 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:41.707 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:41.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:41 smithi064 ceph-mon[212750]: pgmap v781: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.0 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 817 op/s; 3325/97359 objects degraded (3.415%); 0 B/s, 142 objects/s recovering 2023-10-09T06:01:41.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:41 smithi064 ceph-mon[212750]: failed to encode map e328 with expected crc 2023-10-09T06:01:41.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:41 smithi064 ceph-mon[212750]: failed to encode map e328 with expected crc 2023-10-09T06:01:41.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:41 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.5290699089564754}]': finished 2023-10-09T06:01:41.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:41 smithi064 ceph-mon[212750]: osdmap e328: 8 total, 8 up, 7 in 2023-10-09T06:01:41.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:41 smithi064 ceph-mon[212750]: failed to encode map e328 with expected crc 2023-10-09T06:01:41.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:41 smithi064 ceph-mon[212750]: failed to encode map e328 with expected crc 2023-10-09T06:01:41.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:41 smithi064 ceph-mon[220044]: pgmap v781: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.0 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 817 op/s; 3325/97359 objects degraded (3.415%); 0 B/s, 142 objects/s recovering 2023-10-09T06:01:41.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:41 smithi064 ceph-mon[220044]: failed to encode map e328 with expected crc 2023-10-09T06:01:41.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:41 smithi064 ceph-mon[220044]: failed to encode map e328 with expected crc 2023-10-09T06:01:41.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:41 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.5290699089564754}]': finished 2023-10-09T06:01:41.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:41 smithi064 ceph-mon[220044]: osdmap e328: 8 total, 8 up, 7 in 2023-10-09T06:01:41.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:41 smithi064 ceph-mon[220044]: failed to encode map e328 with expected crc 2023-10-09T06:01:41.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:41 smithi064 ceph-mon[220044]: failed to encode map e328 with expected crc 2023-10-09T06:01:42.003 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:01:42.004 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:01:42.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:41 smithi156 ceph-mon[195101]: pgmap v781: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.0 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 817 op/s; 3325/97359 objects degraded (3.415%); 0 B/s, 142 objects/s recovering 2023-10-09T06:01:42.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:41 smithi156 ceph-mon[195101]: failed to encode map e328 with expected crc 2023-10-09T06:01:42.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:41 smithi156 ceph-mon[195101]: failed to encode map e328 with expected crc 2023-10-09T06:01:42.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:41 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.5290699089564754}]': finished 2023-10-09T06:01:42.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:41 smithi156 ceph-mon[195101]: osdmap e328: 8 total, 8 up, 7 in 2023-10-09T06:01:42.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:41 smithi156 ceph-mon[195101]: failed to encode map e328 with expected crc 2023-10-09T06:01:42.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:41 smithi156 ceph-mon[195101]: failed to encode map e328 with expected crc 2023-10-09T06:01:42.216 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:42.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:42.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:42.329 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:01:42.360 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:42.417 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:01:42.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:42.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:01:42.527 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:42.575 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:42.611 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:42.787 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:01:42.790 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:42.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:42.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:42.898 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:42.993 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:43.106 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:43.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:43.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:43.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:42 smithi156 ceph-mon[195101]: failed to encode map e329 with expected crc 2023-10-09T06:01:43.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:42 smithi156 ceph-mon[195101]: osdmap e329: 8 total, 8 up, 7 in 2023-10-09T06:01:43.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:42 smithi156 ceph-mon[195101]: failed to encode map e329 with expected crc 2023-10-09T06:01:43.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:42 smithi156 ceph-mon[195101]: failed to encode map e329 with expected crc 2023-10-09T06:01:43.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:42 smithi156 ceph-mon[195101]: failed to encode map e329 with expected crc 2023-10-09T06:01:43.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:42 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 3325/97359 objects degraded (3.415%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:43.214 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:43.216 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:42 smithi064 ceph-mon[212750]: failed to encode map e329 with expected crc 2023-10-09T06:01:43.216 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:42 smithi064 ceph-mon[212750]: osdmap e329: 8 total, 8 up, 7 in 2023-10-09T06:01:43.216 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:42 smithi064 ceph-mon[212750]: failed to encode map e329 with expected crc 2023-10-09T06:01:43.216 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:42 smithi064 ceph-mon[212750]: failed to encode map e329 with expected crc 2023-10-09T06:01:43.216 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:42 smithi064 ceph-mon[212750]: failed to encode map e329 with expected crc 2023-10-09T06:01:43.217 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:42 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 3325/97359 objects degraded (3.415%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:43.217 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:42 smithi064 ceph-mon[220044]: failed to encode map e329 with expected crc 2023-10-09T06:01:43.217 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:42 smithi064 ceph-mon[220044]: osdmap e329: 8 total, 8 up, 7 in 2023-10-09T06:01:43.217 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:42 smithi064 ceph-mon[220044]: failed to encode map e329 with expected crc 2023-10-09T06:01:43.217 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:42 smithi064 ceph-mon[220044]: failed to encode map e329 with expected crc 2023-10-09T06:01:43.217 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:42 smithi064 ceph-mon[220044]: failed to encode map e329 with expected crc 2023-10-09T06:01:43.217 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:42 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 3325/97359 objects degraded (3.415%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:43.279 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:43.351 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:01:43.399 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:43.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:43.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:43.506 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:43.551 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:43.684 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:43.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:43.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:43.790 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:43.860 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:43.973 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:01:43.985 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:43.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:43.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:44.090 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:44.135 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:44.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:43 smithi064 ceph-mon[220044]: pgmap v784: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.0 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 802 op/s; 3325/96747 objects degraded (3.437%); 0 B/s, 159 objects/s recovering 2023-10-09T06:01:44.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:43 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1803888210' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:01:44.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:43 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:01:44.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:43 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:01:44.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:43 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:44.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:43 smithi064 ceph-mon[212750]: pgmap v784: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.0 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 802 op/s; 3325/96747 objects degraded (3.437%); 0 B/s, 159 objects/s recovering 2023-10-09T06:01:44.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:43 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1803888210' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:01:44.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:43 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:01:44.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:43 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:01:44.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:43 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:44.228 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:44.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:44.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:44.235 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:43 smithi156 ceph-mon[195101]: pgmap v784: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.0 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 802 op/s; 3325/96747 objects degraded (3.437%); 0 B/s, 159 objects/s recovering 2023-10-09T06:01:44.236 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:43 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1803888210' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:01:44.236 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:43 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:01:44.236 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:43 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:01:44.236 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:43 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:44.337 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:44.400 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:45.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:44 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:45.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:44 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:01:45.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:44 smithi064 ceph-mon[212750]: osdmap e330: 8 total, 8 up, 7 in 2023-10-09T06:01:45.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:44 smithi064 ceph-mon[212750]: failed to encode map e330 with expected crc 2023-10-09T06:01:45.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:44 smithi064 ceph-mon[212750]: failed to encode map e330 with expected crc 2023-10-09T06:01:45.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:44 smithi064 ceph-mon[212750]: failed to encode map e330 with expected crc 2023-10-09T06:01:45.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:44 smithi064 ceph-mon[212750]: failed to encode map e330 with expected crc 2023-10-09T06:01:45.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:44 smithi064 ceph-mon[212750]: failed to encode map e330 with expected crc 2023-10-09T06:01:45.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:44 smithi064 ceph-mon[212750]: failed to encode map e330 with expected crc 2023-10-09T06:01:45.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:44 smithi064 ceph-mon[212750]: failed to encode map e330 with expected crc 2023-10-09T06:01:45.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:44 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:45.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:44 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:01:45.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:44 smithi064 ceph-mon[220044]: osdmap e330: 8 total, 8 up, 7 in 2023-10-09T06:01:45.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:44 smithi064 ceph-mon[220044]: failed to encode map e330 with expected crc 2023-10-09T06:01:45.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:44 smithi064 ceph-mon[220044]: failed to encode map e330 with expected crc 2023-10-09T06:01:45.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:44 smithi064 ceph-mon[220044]: failed to encode map e330 with expected crc 2023-10-09T06:01:45.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:44 smithi064 ceph-mon[220044]: failed to encode map e330 with expected crc 2023-10-09T06:01:45.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:44 smithi064 ceph-mon[220044]: failed to encode map e330 with expected crc 2023-10-09T06:01:45.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:44 smithi064 ceph-mon[220044]: failed to encode map e330 with expected crc 2023-10-09T06:01:45.224 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:44 smithi064 ceph-mon[220044]: failed to encode map e330 with expected crc 2023-10-09T06:01:45.243 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:01:45.243 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:01:45.270 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:44 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:01:45.270 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:44 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:01:45.270 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:44 smithi156 ceph-mon[195101]: osdmap e330: 8 total, 8 up, 7 in 2023-10-09T06:01:45.270 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:44 smithi156 ceph-mon[195101]: failed to encode map e330 with expected crc 2023-10-09T06:01:45.271 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:44 smithi156 ceph-mon[195101]: failed to encode map e330 with expected crc 2023-10-09T06:01:45.271 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:44 smithi156 ceph-mon[195101]: failed to encode map e330 with expected crc 2023-10-09T06:01:45.271 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:44 smithi156 ceph-mon[195101]: failed to encode map e330 with expected crc 2023-10-09T06:01:45.271 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:44 smithi156 ceph-mon[195101]: failed to encode map e330 with expected crc 2023-10-09T06:01:45.271 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:44 smithi156 ceph-mon[195101]: failed to encode map e330 with expected crc 2023-10-09T06:01:45.271 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:44 smithi156 ceph-mon[195101]: failed to encode map e330 with expected crc 2023-10-09T06:01:45.445 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:45.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:45.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:45.541 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:01:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:01:45] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.43.0" 2023-10-09T06:01:45.553 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:45.609 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:45.743 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:45.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:45.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:45.847 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:45.934 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:46.081 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:46.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:46.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:46.091 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:01:46.129 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:01:46.139 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:01:46.166 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:01:46.183 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:46.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:45 smithi064 ceph-mon[212750]: pgmap v786: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.9 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 3325/94410 objects degraded (3.522%); 0 B/s, 100 objects/s recovering 2023-10-09T06:01:46.209 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:45 smithi064 ceph-mon[220044]: pgmap v786: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.9 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 3325/94410 objects degraded (3.522%); 0 B/s, 100 objects/s recovering 2023-10-09T06:01:46.219 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:01:46.246 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:01:46.287 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:01:46.291 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:46.348 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:01:46.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:45 smithi156 ceph-mon[195101]: pgmap v786: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.9 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 3325/94410 objects degraded (3.522%); 0 B/s, 100 objects/s recovering 2023-10-09T06:01:46.400 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:01:46.414 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:01:46.415 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:01:46.415 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:01:46.415 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:01:46.415 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:01:46.415 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:01:46.415 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:01:46.415 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:01:46.479 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:46.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:46.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:46.587 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:46.653 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:46.859 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:46.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:46.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:46.973 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:47.037 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:47.177 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:47.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:47.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:47.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:46 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3745353695' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:47.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:46 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:01:47.224 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:46 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3745353695' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:47.224 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:46 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:01:47.292 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:47.293 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 0, 4] 2023-10-09T06:01:47.293 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:01:47.293 INFO:tasks.thrashosds.thrasher:Adding osd 1 2023-10-09T06:01:47.293 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd in 1 2023-10-09T06:01:47.355 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:47.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:46 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3745353695' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:47.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:46 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:01:47.568 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:01:47.705 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:47.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:47.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:47.790 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:47.813 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:47.844 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:47.898 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:47.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:47.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:48.004 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:48.036 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:48.096 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:48.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:48.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:48.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:47 smithi156 ceph-mon[195101]: pgmap v787: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 1.8 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 570 op/s; 2222/87246 objects degraded (2.547%); 20 KiB/s, 147 objects/s recovering 2023-10-09T06:01:48.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:47 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 2222/87246 objects degraded (2.547%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:48.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:47 smithi156 ceph-mon[195101]: osdmap e331: 8 total, 8 up, 7 in 2023-10-09T06:01:48.200 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:48.228 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:48.254 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:48.278 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:48.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:48.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:48.383 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:48.434 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:47 smithi064 ceph-mon[212750]: pgmap v787: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 1.8 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 570 op/s; 2222/87246 objects degraded (2.547%); 20 KiB/s, 147 objects/s recovering 2023-10-09T06:01:48.434 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:47 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 2222/87246 objects degraded (2.547%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:48.434 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:47 smithi064 ceph-mon[212750]: osdmap e331: 8 total, 8 up, 7 in 2023-10-09T06:01:48.434 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:47 smithi064 ceph-mon[220044]: pgmap v787: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 1.8 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 570 op/s; 2222/87246 objects degraded (2.547%); 20 KiB/s, 147 objects/s recovering 2023-10-09T06:01:48.434 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:47 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 2222/87246 objects degraded (2.547%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:01:48.435 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:47 smithi064 ceph-mon[220044]: osdmap e331: 8 total, 8 up, 7 in 2023-10-09T06:01:48.446 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:48.605 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:48.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:48.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:48.718 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:48.785 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:49.090 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:49.094 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:48 smithi064 ceph-mon[212750]: failed to encode map e331 with expected crc 2023-10-09T06:01:49.095 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:48 smithi064 ceph-mon[212750]: failed to encode map e331 with expected crc 2023-10-09T06:01:49.095 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:48 smithi064 ceph-mon[212750]: osdmap e332: 8 total, 8 up, 7 in 2023-10-09T06:01:49.095 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:48 smithi064 ceph-mon[212750]: failed to encode map e332 with expected crc 2023-10-09T06:01:49.095 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:48 smithi064 ceph-mon[220044]: failed to encode map e331 with expected crc 2023-10-09T06:01:49.095 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:48 smithi064 ceph-mon[220044]: failed to encode map e331 with expected crc 2023-10-09T06:01:49.095 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:48 smithi064 ceph-mon[220044]: osdmap e332: 8 total, 8 up, 7 in 2023-10-09T06:01:49.095 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:48 smithi064 ceph-mon[220044]: failed to encode map e332 with expected crc 2023-10-09T06:01:49.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:49.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:49.197 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:49.261 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:48 smithi156 ceph-mon[195101]: failed to encode map e331 with expected crc 2023-10-09T06:01:49.261 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:48 smithi156 ceph-mon[195101]: failed to encode map e331 with expected crc 2023-10-09T06:01:49.262 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:48 smithi156 ceph-mon[195101]: osdmap e332: 8 total, 8 up, 7 in 2023-10-09T06:01:49.262 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:48 smithi156 ceph-mon[195101]: failed to encode map e332 with expected crc 2023-10-09T06:01:49.263 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:49.347 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:49.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:49.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:49.454 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:49.513 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:49.614 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:49.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:49.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:49.724 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:49.727 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:01:49.751 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:49.797 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:49.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:49.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:49.905 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:49.934 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:50.003 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:50 smithi064 ceph-mon[212750]: failed to encode map e332 with expected crc 2023-10-09T06:01:50.003 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:50 smithi064 ceph-mon[212750]: failed to encode map e332 with expected crc 2023-10-09T06:01:50.003 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:50 smithi064 ceph-mon[212750]: failed to encode map e332 with expected crc 2023-10-09T06:01:50.006 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:50 smithi064 ceph-mon[220044]: failed to encode map e332 with expected crc 2023-10-09T06:01:50.006 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:50 smithi064 ceph-mon[220044]: failed to encode map e332 with expected crc 2023-10-09T06:01:50.006 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:50 smithi064 ceph-mon[220044]: failed to encode map e332 with expected crc 2023-10-09T06:01:50.006 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:50 smithi064 ceph-mon[220044]: pgmap v790: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 1.7 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 736 op/s; 2222/83634 objects degraded (2.657%); 20 KiB/s, 147 objects/s recovering 2023-10-09T06:01:50.038 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:50.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:50.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:50.141 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:01:50.193 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:50 smithi156 ceph-mon[195101]: failed to encode map e332 with expected crc 2023-10-09T06:01:50.193 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:50 smithi156 ceph-mon[195101]: failed to encode map e332 with expected crc 2023-10-09T06:01:50.193 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:50 smithi156 ceph-mon[195101]: failed to encode map e332 with expected crc 2023-10-09T06:01:50.194 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:50 smithi156 ceph-mon[195101]: pgmap v790: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 1.7 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 736 op/s; 2222/83634 objects degraded (2.657%); 20 KiB/s, 147 objects/s recovering 2023-10-09T06:01:50.199 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:50.282 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:01:50.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:50.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:01:50.391 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:50.476 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:50 smithi064 ceph-mon[212750]: pgmap v790: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 1.7 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 736 op/s; 2222/83634 objects degraded (2.657%); 20 KiB/s, 147 objects/s recovering 2023-10-09T06:01:50.480 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:51.231 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:51.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:51.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:51.338 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:51.434 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:51.456 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:51.582 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:51.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:51.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:51.701 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:51.742 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:51.813 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:51.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:51.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:51.835 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:51.927 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:51.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:51 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:01:51.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:51 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:01:51.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:51 smithi064 ceph-mon[220044]: pgmap v791: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.6 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 9.3 KiB/s rd, 24 KiB/s wr, 898 op/s; 1126/76236 objects degraded (1.477%); 17 KiB/s, 235 objects/s recovering 2023-10-09T06:01:51.940 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:51 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:01:51.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:51 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:01:51.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:51 smithi064 ceph-mon[212750]: pgmap v791: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.6 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 9.3 KiB/s rd, 24 KiB/s wr, 898 op/s; 1126/76236 objects degraded (1.477%); 17 KiB/s, 235 objects/s recovering 2023-10-09T06:01:51.946 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:01:51.946 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:01:51.947 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.947 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45263.0:187101 7.6 e332/328 7:67a25187:::benchmark_data_smithi064_398483_object81748:head v 332'11698, mlcod=332'11694)", 2023-10-09T06:01:51.947 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:01:51.939462+0000", 2023-10-09T06:01:51.947 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0068482530000000003, 2023-10-09T06:01:51.947 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0068803290000000001, 2023-10-09T06:01:51.947 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:01:51.947 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:01:51.947 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:01:51.947 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.947 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:01:51.947 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.939462+0000", 2023-10-09T06:01:51.948 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:01:51.948 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.948 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.948 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:01:51.948 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.939462+0000", 2023-10-09T06:01:51.948 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:01:51.948 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.948 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.948 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:01:51.948 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.939461+0000", 2023-10-09T06:01:51.948 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:01:51.949 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.949 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.949 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:01:51.949 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.939477+0000", 2023-10-09T06:01:51.949 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5940999999999999e-05 2023-10-09T06:01:51.949 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.949 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.949 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:01:51.949 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.939497+0000", 2023-10-09T06:01:51.949 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9805000000000001e-05 2023-10-09T06:01:51.949 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.950 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.950 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:01:51.950 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.939505+0000", 2023-10-09T06:01:51.950 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8359999999999998e-06 2023-10-09T06:01:51.950 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.950 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.950 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:01:51.950 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.939718+0000", 2023-10-09T06:01:51.950 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00021276199999999999 2023-10-09T06:01:51.950 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.950 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.951 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:01:51.951 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.939739+0000", 2023-10-09T06:01:51.951 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0710999999999999e-05 2023-10-09T06:01:51.951 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:01:51.951 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:01:51.951 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:01:51.951 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.951 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.951 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45263.0:187107 7.6 e332/328 7:6f3b0128:::benchmark_data_smithi064_398483_object81754:head v 332'11699, mlcod=332'11694)", 2023-10-09T06:01:51.951 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:01:51.939518+0000", 2023-10-09T06:01:51.951 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0067920979999999999, 2023-10-09T06:01:51.951 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0068976990000000002, 2023-10-09T06:01:51.952 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:01:51.952 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:01:51.952 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:01:51.952 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.952 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:01:51.952 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.939518+0000", 2023-10-09T06:01:51.952 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:01:51.952 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.952 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.952 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:01:51.952 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.939518+0000", 2023-10-09T06:01:51.953 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:01:51.953 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.953 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.953 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:01:51.953 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.939518+0000", 2023-10-09T06:01:51.953 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967296 2023-10-09T06:01:51.953 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.953 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.953 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:01:51.953 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.939522+0000", 2023-10-09T06:01:51.953 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6480000000000001e-06 2023-10-09T06:01:51.953 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.954 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.954 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:01:51.954 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.939532+0000", 2023-10-09T06:01:51.954 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0018e-05 2023-10-09T06:01:51.954 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.954 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.954 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:01:51.954 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.939545+0000", 2023-10-09T06:01:51.954 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3261999999999999e-05 2023-10-09T06:01:51.954 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.954 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.955 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:01:51.955 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.943207+0000", 2023-10-09T06:01:51.955 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0036620260000000001 2023-10-09T06:01:51.955 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.955 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.955 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:01:51.955 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.943228+0000", 2023-10-09T06:01:51.955 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0948e-05 2023-10-09T06:01:51.955 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:01:51.955 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:01:51.955 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:01:51.955 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.956 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.956 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45263.0:187109 7.6 e332/328)", 2023-10-09T06:01:51.956 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:01:51.939552+0000", 2023-10-09T06:01:51.956 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0067580160000000004, 2023-10-09T06:01:51.956 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0069257640000000001, 2023-10-09T06:01:51.956 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:01:51.956 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-10-09T06:01:51.956 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:01:51.956 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.956 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:01:51.956 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.939552+0000", 2023-10-09T06:01:51.956 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:01:51.957 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.957 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.957 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:01:51.957 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.939552+0000", 2023-10-09T06:01:51.957 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:01:51.957 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.957 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.957 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:01:51.957 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.939552+0000", 2023-10-09T06:01:51.957 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:01:51.957 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.958 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.958 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:01:51.958 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.939563+0000", 2023-10-09T06:01:51.958 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0628999999999999e-05 2023-10-09T06:01:51.958 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.958 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.958 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:01:51.958 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.939569+0000", 2023-10-09T06:01:51.958 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6279999999999999e-06 2023-10-09T06:01:51.958 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.959 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.959 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:01:51.959 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.939575+0000", 2023-10-09T06:01:51.959 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3090000000000003e-06 2023-10-09T06:01:51.959 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:01:51.959 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:01:51.959 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:01:51.959 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.959 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.959 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45263.0:187108 7.e e332/285 7:7bbae2f9:::benchmark_data_smithi064_398483_object81755:head v 332'11919, mlcod=332'11916)", 2023-10-09T06:01:51.959 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:01:51.933140+0000", 2023-10-09T06:01:51.959 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.013170863999999999, 2023-10-09T06:01:51.960 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013394082, 2023-10-09T06:01:51.960 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:01:51.960 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:01:51.960 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:01:51.960 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.960 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:01:51.960 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.933140+0000", 2023-10-09T06:01:51.960 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:01:51.960 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.960 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.960 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:01:51.960 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.933140+0000", 2023-10-09T06:01:51.961 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:01:51.961 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.961 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.961 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:01:51.961 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.933139+0000", 2023-10-09T06:01:51.961 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:01:51.961 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.961 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.961 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:01:51.961 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.933145+0000", 2023-10-09T06:01:51.961 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0730000000000001e-06 2023-10-09T06:01:51.962 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.962 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.962 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:01:51.962 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.933152+0000", 2023-10-09T06:01:51.962 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7750000000000004e-06 2023-10-09T06:01:51.962 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.962 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.962 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:01:51.962 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.933157+0000", 2023-10-09T06:01:51.962 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6460000000000001e-06 2023-10-09T06:01:51.962 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.963 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.963 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:01:51.963 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.933258+0000", 2023-10-09T06:01:51.963 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010162699999999999 2023-10-09T06:01:51.963 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:01:51.963 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:01:51.963 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:01:51.963 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:51.933277+0000", 2023-10-09T06:01:51.963 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8915000000000001e-05 2023-10-09T06:01:51.963 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:01:51.963 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:01:51.964 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:01:51.964 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:01:51.964 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-10-09T06:01:51.964 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 4 2023-10-09T06:01:51.964 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:01:51.999 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:52.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:51 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:01:52.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:51 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:01:52.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:51 smithi156 ceph-mon[195101]: pgmap v791: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.6 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 9.3 KiB/s rd, 24 KiB/s wr, 898 op/s; 1126/76236 objects degraded (1.477%); 17 KiB/s, 235 objects/s recovering 2023-10-09T06:01:52.354 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:52.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:52.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:52.468 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:52.501 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:52.561 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:52.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:52.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:52.668 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:52.699 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:52.769 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:52.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:52.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:52.777 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:52 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 1126/76236 objects degraded (1.477%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:01:52.878 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:52.928 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:52.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:52 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 1126/76236 objects degraded (1.477%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:01:52.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:52 smithi064 conmon[212726]: 2023-10-09T06:01:52.716+0000 7f8823f23700 -1 mon.a@0(leader).osd e332 definitely_dead 0 2023-10-09T06:01:52.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:52 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 1126/76236 objects degraded (1.477%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:01:53.009 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:53.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:53.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:53.118 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:01:53.175 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:53.757 INFO:teuthology.orchestra.run.smithi064.stderr:marked in osd.1. 2023-10-09T06:01:54.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:53 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1322420155' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T06:01:54.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:53 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T06:01:54.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:53 smithi156 ceph-mon[195101]: pgmap v792: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.5 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 7.9 KiB/s rd, 21 KiB/s wr, 789 op/s; 1126/75612 objects degraded (1.489%); 15 KiB/s, 200 objects/s recovering 2023-10-09T06:01:54.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:53 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1322420155' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T06:01:54.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:53 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T06:01:54.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:53 smithi064 ceph-mon[212750]: pgmap v792: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.5 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 7.9 KiB/s rd, 21 KiB/s wr, 789 op/s; 1126/75612 objects degraded (1.489%); 15 KiB/s, 200 objects/s recovering 2023-10-09T06:01:54.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:53 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1322420155' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T06:01:54.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:53 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T06:01:54.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:53 smithi064 ceph-mon[220044]: pgmap v792: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.5 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 7.9 KiB/s rd, 21 KiB/s wr, 789 op/s; 1126/75612 objects degraded (1.489%); 15 KiB/s, 200 objects/s recovering 2023-10-09T06:01:54.978 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:54 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T06:01:54.978 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:54 smithi064 ceph-mon[212750]: osdmap e333: 8 total, 8 up, 8 in 2023-10-09T06:01:54.979 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:54 smithi064 ceph-mon[212750]: failed to encode map e333 with expected crc 2023-10-09T06:01:54.979 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:54 smithi064 ceph-mon[212750]: failed to encode map e333 with expected crc 2023-10-09T06:01:54.979 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:54 smithi064 ceph-mon[212750]: failed to encode map e333 with expected crc 2023-10-09T06:01:54.979 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:54 smithi064 ceph-mon[212750]: failed to encode map e333 with expected crc 2023-10-09T06:01:54.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:54 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T06:01:54.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:54 smithi064 ceph-mon[220044]: osdmap e333: 8 total, 8 up, 8 in 2023-10-09T06:01:54.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:54 smithi064 ceph-mon[220044]: failed to encode map e333 with expected crc 2023-10-09T06:01:54.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:54 smithi064 ceph-mon[220044]: failed to encode map e333 with expected crc 2023-10-09T06:01:54.980 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:54 smithi064 ceph-mon[220044]: failed to encode map e333 with expected crc 2023-10-09T06:01:54.980 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:54 smithi064 ceph-mon[220044]: failed to encode map e333 with expected crc 2023-10-09T06:01:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:54 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T06:01:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:54 smithi156 ceph-mon[195101]: osdmap e333: 8 total, 8 up, 8 in 2023-10-09T06:01:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:54 smithi156 ceph-mon[195101]: failed to encode map e333 with expected crc 2023-10-09T06:01:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:54 smithi156 ceph-mon[195101]: failed to encode map e333 with expected crc 2023-10-09T06:01:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:54 smithi156 ceph-mon[195101]: failed to encode map e333 with expected crc 2023-10-09T06:01:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:54 smithi156 ceph-mon[195101]: failed to encode map e333 with expected crc 2023-10-09T06:01:55.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:01:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:01:55] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.43.0" 2023-10-09T06:01:55.754 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:01:55.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:55.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:01:55.821 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:01:55.859 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:01:55.885 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:55.935 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:01:55.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:55.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:01:56.041 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:01:56.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:55 smithi156 ceph-mon[195101]: failed to encode map e334 with expected crc 2023-10-09T06:01:56.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:55 smithi156 ceph-mon[195101]: osdmap e334: 8 total, 8 up, 8 in 2023-10-09T06:01:56.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:55 smithi156 ceph-mon[195101]: failed to encode map e334 with expected crc 2023-10-09T06:01:56.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:55 smithi156 ceph-mon[195101]: failed to encode map e334 with expected crc 2023-10-09T06:01:56.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:55 smithi156 ceph-mon[195101]: failed to encode map e334 with expected crc 2023-10-09T06:01:56.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:55 smithi156 ceph-mon[195101]: 7.4 starting backfill to osd.1 from (0'0,0'0] MAX to 332'11782 2023-10-09T06:01:56.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:55 smithi156 ceph-mon[195101]: failed to encode map e334 with expected crc 2023-10-09T06:01:56.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:55 smithi156 ceph-mon[195101]: failed to encode map e334 with expected crc 2023-10-09T06:01:56.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:55 smithi156 ceph-mon[195101]: failed to encode map e334 with expected crc 2023-10-09T06:01:56.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:55 smithi156 ceph-mon[195101]: failed to encode map e334 with expected crc 2023-10-09T06:01:56.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:55 smithi156 ceph-mon[195101]: pgmap v795: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.5 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 717 op/s; 1126/72573 objects degraded (1.552%); 0 B/s, 112 objects/s recovering 2023-10-09T06:01:56.141 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:55 smithi064 ceph-mon[212750]: failed to encode map e334 with expected crc 2023-10-09T06:01:56.141 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:55 smithi064 ceph-mon[212750]: osdmap e334: 8 total, 8 up, 8 in 2023-10-09T06:01:56.141 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:55 smithi064 ceph-mon[212750]: failed to encode map e334 with expected crc 2023-10-09T06:01:56.142 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:55 smithi064 ceph-mon[212750]: failed to encode map e334 with expected crc 2023-10-09T06:01:56.142 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:55 smithi064 ceph-mon[212750]: failed to encode map e334 with expected crc 2023-10-09T06:01:56.142 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:55 smithi064 ceph-mon[212750]: 7.4 starting backfill to osd.1 from (0'0,0'0] MAX to 332'11782 2023-10-09T06:01:56.142 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:55 smithi064 ceph-mon[212750]: failed to encode map e334 with expected crc 2023-10-09T06:01:56.142 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:55 smithi064 ceph-mon[212750]: failed to encode map e334 with expected crc 2023-10-09T06:01:56.142 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:55 smithi064 ceph-mon[212750]: failed to encode map e334 with expected crc 2023-10-09T06:01:56.142 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:55 smithi064 ceph-mon[212750]: failed to encode map e334 with expected crc 2023-10-09T06:01:56.142 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:55 smithi064 ceph-mon[212750]: pgmap v795: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.5 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 717 op/s; 1126/72573 objects degraded (1.552%); 0 B/s, 112 objects/s recovering 2023-10-09T06:01:56.149 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:56.150 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:55 smithi064 ceph-mon[220044]: failed to encode map e334 with expected crc 2023-10-09T06:01:56.150 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:55 smithi064 ceph-mon[220044]: osdmap e334: 8 total, 8 up, 8 in 2023-10-09T06:01:56.151 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:55 smithi064 ceph-mon[220044]: failed to encode map e334 with expected crc 2023-10-09T06:01:56.151 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:55 smithi064 ceph-mon[220044]: failed to encode map e334 with expected crc 2023-10-09T06:01:56.151 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:55 smithi064 ceph-mon[220044]: failed to encode map e334 with expected crc 2023-10-09T06:01:56.151 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:55 smithi064 ceph-mon[220044]: 7.4 starting backfill to osd.1 from (0'0,0'0] MAX to 332'11782 2023-10-09T06:01:56.151 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:55 smithi064 ceph-mon[220044]: failed to encode map e334 with expected crc 2023-10-09T06:01:56.151 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:55 smithi064 ceph-mon[220044]: failed to encode map e334 with expected crc 2023-10-09T06:01:56.151 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:55 smithi064 ceph-mon[220044]: failed to encode map e334 with expected crc 2023-10-09T06:01:56.151 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:55 smithi064 ceph-mon[220044]: failed to encode map e334 with expected crc 2023-10-09T06:01:56.151 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:55 smithi064 ceph-mon[220044]: pgmap v795: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.5 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 717 op/s; 1126/72573 objects degraded (1.552%); 0 B/s, 112 objects/s recovering 2023-10-09T06:01:56.232 INFO:tasks.thrashosds.thrasher:Added osd 1 2023-10-09T06:01:56.999 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[212750]: failed to encode map e335 with expected crc 2023-10-09T06:01:56.999 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[212750]: failed to encode map e335 with expected crc 2023-10-09T06:01:56.999 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[212750]: failed to encode map e335 with expected crc 2023-10-09T06:01:56.999 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[212750]: failed to encode map e335 with expected crc 2023-10-09T06:01:57.000 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[212750]: osdmap e335: 8 total, 8 up, 8 in 2023-10-09T06:01:57.000 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[212750]: failed to encode map e335 with expected crc 2023-10-09T06:01:57.000 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[212750]: failed to encode map e335 with expected crc 2023-10-09T06:01:57.000 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[212750]: 7.0 starting backfill to osd.1 from (0'0,0'0] MAX to 332'11957 2023-10-09T06:01:57.000 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[212750]: 7.b starting backfill to osd.1 from (0'0,0'0] MAX to 332'11865 2023-10-09T06:01:57.000 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[212750]: failed to encode map e335 with expected crc 2023-10-09T06:01:57.000 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[212750]: failed to encode map e335 with expected crc 2023-10-09T06:01:57.000 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[212750]: 7.2 starting backfill to osd.1 from (0'0,0'0] MAX to 332'11700 2023-10-09T06:01:57.000 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[212750]: 7.d starting backfill to osd.1 from (0'0,0'0] MAX to 332'11824 2023-10-09T06:01:57.000 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[212750]: 7.9 starting backfill to osd.1 from (0'0,0'0] MAX to 332'11922 2023-10-09T06:01:57.001 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[220044]: failed to encode map e335 with expected crc 2023-10-09T06:01:57.001 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[220044]: failed to encode map e335 with expected crc 2023-10-09T06:01:57.001 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[220044]: failed to encode map e335 with expected crc 2023-10-09T06:01:57.001 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[220044]: failed to encode map e335 with expected crc 2023-10-09T06:01:57.001 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[220044]: osdmap e335: 8 total, 8 up, 8 in 2023-10-09T06:01:57.001 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[220044]: failed to encode map e335 with expected crc 2023-10-09T06:01:57.001 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[220044]: failed to encode map e335 with expected crc 2023-10-09T06:01:57.001 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[220044]: 7.0 starting backfill to osd.1 from (0'0,0'0] MAX to 332'11957 2023-10-09T06:01:57.001 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[220044]: 7.b starting backfill to osd.1 from (0'0,0'0] MAX to 332'11865 2023-10-09T06:01:57.002 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[220044]: failed to encode map e335 with expected crc 2023-10-09T06:01:57.002 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[220044]: failed to encode map e335 with expected crc 2023-10-09T06:01:57.002 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[220044]: 7.2 starting backfill to osd.1 from (0'0,0'0] MAX to 332'11700 2023-10-09T06:01:57.002 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[220044]: 7.d starting backfill to osd.1 from (0'0,0'0] MAX to 332'11824 2023-10-09T06:01:57.002 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:56 smithi064 ceph-mon[220044]: 7.9 starting backfill to osd.1 from (0'0,0'0] MAX to 332'11922 2023-10-09T06:01:57.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:56 smithi156 ceph-mon[195101]: failed to encode map e335 with expected crc 2023-10-09T06:01:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:56 smithi156 ceph-mon[195101]: failed to encode map e335 with expected crc 2023-10-09T06:01:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:56 smithi156 ceph-mon[195101]: failed to encode map e335 with expected crc 2023-10-09T06:01:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:56 smithi156 ceph-mon[195101]: failed to encode map e335 with expected crc 2023-10-09T06:01:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:56 smithi156 ceph-mon[195101]: osdmap e335: 8 total, 8 up, 8 in 2023-10-09T06:01:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:56 smithi156 ceph-mon[195101]: failed to encode map e335 with expected crc 2023-10-09T06:01:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:56 smithi156 ceph-mon[195101]: failed to encode map e335 with expected crc 2023-10-09T06:01:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:56 smithi156 ceph-mon[195101]: 7.0 starting backfill to osd.1 from (0'0,0'0] MAX to 332'11957 2023-10-09T06:01:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:56 smithi156 ceph-mon[195101]: 7.b starting backfill to osd.1 from (0'0,0'0] MAX to 332'11865 2023-10-09T06:01:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:56 smithi156 ceph-mon[195101]: failed to encode map e335 with expected crc 2023-10-09T06:01:57.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:56 smithi156 ceph-mon[195101]: failed to encode map e335 with expected crc 2023-10-09T06:01:57.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:56 smithi156 ceph-mon[195101]: 7.2 starting backfill to osd.1 from (0'0,0'0] MAX to 332'11700 2023-10-09T06:01:57.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:56 smithi156 ceph-mon[195101]: 7.d starting backfill to osd.1 from (0'0,0'0] MAX to 332'11824 2023-10-09T06:01:57.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:56 smithi156 ceph-mon[195101]: 7.9 starting backfill to osd.1 from (0'0,0'0] MAX to 332'11922 2023-10-09T06:01:57.289 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:01:57.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:57.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:01:57.402 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:57.504 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:57.630 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:57.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:57.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:57.706 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:57.742 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:57.777 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:57.846 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:57.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:57.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:57.953 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:01:57.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:57 smithi064 ceph-mon[212750]: pgmap v797: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 9 remapped+peering, 1 active+recovering+undersized+remapped, 108 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 9.2 KiB/s rd, 24 KiB/s wr, 520 op/s; 251/66993 objects degraded (0.375%); 1391/66993 objects misplaced (2.076%); 17 KiB/s, 145 objects/s recovering 2023-10-09T06:01:57.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:57 smithi064 ceph-mon[212750]: osdmap e336: 8 total, 8 up, 8 in 2023-10-09T06:01:57.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:57 smithi064 ceph-mon[212750]: failed to encode map e336 with expected crc 2023-10-09T06:01:57.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:57 smithi064 ceph-mon[212750]: failed to encode map e336 with expected crc 2023-10-09T06:01:57.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:57 smithi064 ceph-mon[212750]: failed to encode map e336 with expected crc 2023-10-09T06:01:57.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:57 smithi064 ceph-mon[212750]: failed to encode map e336 with expected crc 2023-10-09T06:01:57.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:57 smithi064 ceph-mon[212750]: failed to encode map e336 with expected crc 2023-10-09T06:01:57.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:57 smithi064 ceph-mon[212750]: failed to encode map e336 with expected crc 2023-10-09T06:01:57.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:57 smithi064 ceph-mon[212750]: failed to encode map e336 with expected crc 2023-10-09T06:01:57.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:57 smithi064 ceph-mon[212750]: failed to encode map e336 with expected crc 2023-10-09T06:01:57.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:57 smithi064 ceph-mon[212750]: failed to encode map e336 with expected crc 2023-10-09T06:01:57.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:57 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 1126/72573 objects degraded (1.552%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:01:57.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:57 smithi064 ceph-mon[220044]: pgmap v797: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 9 remapped+peering, 1 active+recovering+undersized+remapped, 108 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 9.2 KiB/s rd, 24 KiB/s wr, 520 op/s; 251/66993 objects degraded (0.375%); 1391/66993 objects misplaced (2.076%); 17 KiB/s, 145 objects/s recovering 2023-10-09T06:01:57.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:57 smithi064 ceph-mon[220044]: osdmap e336: 8 total, 8 up, 8 in 2023-10-09T06:01:57.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:57 smithi064 ceph-mon[220044]: failed to encode map e336 with expected crc 2023-10-09T06:01:57.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:57 smithi064 ceph-mon[220044]: failed to encode map e336 with expected crc 2023-10-09T06:01:57.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:57 smithi064 ceph-mon[220044]: failed to encode map e336 with expected crc 2023-10-09T06:01:57.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:57 smithi064 ceph-mon[220044]: failed to encode map e336 with expected crc 2023-10-09T06:01:57.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:57 smithi064 ceph-mon[220044]: failed to encode map e336 with expected crc 2023-10-09T06:01:57.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:57 smithi064 ceph-mon[220044]: failed to encode map e336 with expected crc 2023-10-09T06:01:57.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:57 smithi064 ceph-mon[220044]: failed to encode map e336 with expected crc 2023-10-09T06:01:57.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:57 smithi064 ceph-mon[220044]: failed to encode map e336 with expected crc 2023-10-09T06:01:57.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:57 smithi064 ceph-mon[220044]: failed to encode map e336 with expected crc 2023-10-09T06:01:57.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:57 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 1126/72573 objects degraded (1.552%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:01:57.986 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:58.045 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:01:58.048 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:57 smithi156 ceph-mon[195101]: pgmap v797: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 9 remapped+peering, 1 active+recovering+undersized+remapped, 108 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 9.2 KiB/s rd, 24 KiB/s wr, 520 op/s; 251/66993 objects degraded (0.375%); 1391/66993 objects misplaced (2.076%); 17 KiB/s, 145 objects/s recovering 2023-10-09T06:01:58.048 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:57 smithi156 ceph-mon[195101]: osdmap e336: 8 total, 8 up, 8 in 2023-10-09T06:01:58.048 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:57 smithi156 ceph-mon[195101]: failed to encode map e336 with expected crc 2023-10-09T06:01:58.048 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:57 smithi156 ceph-mon[195101]: failed to encode map e336 with expected crc 2023-10-09T06:01:58.048 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:57 smithi156 ceph-mon[195101]: failed to encode map e336 with expected crc 2023-10-09T06:01:58.048 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:57 smithi156 ceph-mon[195101]: failed to encode map e336 with expected crc 2023-10-09T06:01:58.048 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:57 smithi156 ceph-mon[195101]: failed to encode map e336 with expected crc 2023-10-09T06:01:58.048 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:57 smithi156 ceph-mon[195101]: failed to encode map e336 with expected crc 2023-10-09T06:01:58.049 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:57 smithi156 ceph-mon[195101]: failed to encode map e336 with expected crc 2023-10-09T06:01:58.049 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:57 smithi156 ceph-mon[195101]: failed to encode map e336 with expected crc 2023-10-09T06:01:58.049 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:57 smithi156 ceph-mon[195101]: failed to encode map e336 with expected crc 2023-10-09T06:01:58.049 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:57 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 1126/72573 objects degraded (1.552%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:01:58.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:58.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:01:58.151 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:01:58.249 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:58.688 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:01:58.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:58.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:01:58.800 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:01:58.832 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:01:58.852 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:58 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:01:58.852 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:58 smithi064 ceph-mon[212750]: osdmap e337: 8 total, 8 up, 8 in 2023-10-09T06:01:58.852 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:58 smithi064 ceph-mon[212750]: failed to encode map e337 with expected crc 2023-10-09T06:01:58.852 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:58 smithi064 ceph-mon[212750]: failed to encode map e337 with expected crc 2023-10-09T06:01:58.852 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:58 smithi064 ceph-mon[212750]: failed to encode map e337 with expected crc 2023-10-09T06:01:58.852 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:58 smithi064 ceph-mon[212750]: failed to encode map e337 with expected crc 2023-10-09T06:01:58.852 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:58 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/160781498' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:01:58.852 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:58 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:01:58.852 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:58 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:01:58.853 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:58.853 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:58 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4028276200' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:58.853 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:58 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:01:58.853 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:58 smithi064 ceph-mon[220044]: osdmap e337: 8 total, 8 up, 8 in 2023-10-09T06:01:58.853 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:58 smithi064 ceph-mon[220044]: failed to encode map e337 with expected crc 2023-10-09T06:01:58.853 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:58 smithi064 ceph-mon[220044]: failed to encode map e337 with expected crc 2023-10-09T06:01:58.853 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:58 smithi064 ceph-mon[220044]: failed to encode map e337 with expected crc 2023-10-09T06:01:58.853 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:58 smithi064 ceph-mon[220044]: failed to encode map e337 with expected crc 2023-10-09T06:01:58.853 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:58 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/160781498' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:01:58.853 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:58 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:01:58.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:58 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:01:58.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:58 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:58.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:58 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4028276200' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:58.900 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:58.937 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:01:58.937 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:01:58.937 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:01:58.937 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:01:58.937 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:01:58.937 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:01:58.937 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:01:58.937 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:01:58.938 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:01:58.938 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:01:58.938 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:01:58.938 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:01:58.938 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:01:58.938 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:01:58.938 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:01:58.938 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:01:59.070 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:01:59.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:59.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:01:59.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:58 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:01:59.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:58 smithi156 ceph-mon[195101]: osdmap e337: 8 total, 8 up, 8 in 2023-10-09T06:01:59.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:58 smithi156 ceph-mon[195101]: failed to encode map e337 with expected crc 2023-10-09T06:01:59.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:58 smithi156 ceph-mon[195101]: failed to encode map e337 with expected crc 2023-10-09T06:01:59.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:58 smithi156 ceph-mon[195101]: failed to encode map e337 with expected crc 2023-10-09T06:01:59.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:58 smithi156 ceph-mon[195101]: failed to encode map e337 with expected crc 2023-10-09T06:01:59.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:58 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/160781498' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:01:59.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:58 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:01:59.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:58 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:01:59.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:58 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:59.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:58 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4028276200' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:01:59.177 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:59.209 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:59.272 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:59.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:59.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:59.381 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:59.408 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:59.468 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:59.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:59.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:59.574 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:01:59.600 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:59.644 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:01:59.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:59.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:01:59.751 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:01:59.777 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:01:59.830 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:01:59.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:01:59.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:01:59.940 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:00.003 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:00.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:59 smithi156 ceph-mon[195101]: pgmap v800: 119 pgs: 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 9 remapped+peering, 1 active+recovering+undersized+remapped, 105 active+clean; 1.3 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 692 op/s; 251/64338 objects degraded (0.390%); 5228/64338 objects misplaced (8.126%); 26 KiB/s, 217 objects/s recovering 2023-10-09T06:02:00.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:59 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:02:00.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:59 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:02:00.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:59 smithi156 ceph-mon[195101]: osdmap e338: 8 total, 8 up, 8 in 2023-10-09T06:02:00.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:59 smithi156 ceph-mon[195101]: failed to encode map e338 with expected crc 2023-10-09T06:02:00.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:59 smithi156 ceph-mon[195101]: failed to encode map e338 with expected crc 2023-10-09T06:02:00.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:59 smithi156 ceph-mon[195101]: failed to encode map e338 with expected crc 2023-10-09T06:02:00.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:59 smithi156 ceph-mon[195101]: failed to encode map e338 with expected crc 2023-10-09T06:02:00.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:59 smithi156 ceph-mon[195101]: failed to encode map e338 with expected crc 2023-10-09T06:02:00.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:01:59 smithi156 ceph-mon[195101]: failed to encode map e338 with expected crc 2023-10-09T06:02:00.130 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:59 smithi064 ceph-mon[212750]: pgmap v800: 119 pgs: 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 9 remapped+peering, 1 active+recovering+undersized+remapped, 105 active+clean; 1.3 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 692 op/s; 251/64338 objects degraded (0.390%); 5228/64338 objects misplaced (8.126%); 26 KiB/s, 217 objects/s recovering 2023-10-09T06:02:00.130 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:59 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:02:00.130 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:59 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:02:00.130 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:59 smithi064 ceph-mon[212750]: osdmap e338: 8 total, 8 up, 8 in 2023-10-09T06:02:00.130 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:59 smithi064 ceph-mon[212750]: failed to encode map e338 with expected crc 2023-10-09T06:02:00.131 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:59 smithi064 ceph-mon[212750]: failed to encode map e338 with expected crc 2023-10-09T06:02:00.131 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:59 smithi064 ceph-mon[212750]: failed to encode map e338 with expected crc 2023-10-09T06:02:00.131 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:59 smithi064 ceph-mon[212750]: failed to encode map e338 with expected crc 2023-10-09T06:02:00.131 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:59 smithi064 ceph-mon[212750]: failed to encode map e338 with expected crc 2023-10-09T06:02:00.131 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:01:59 smithi064 ceph-mon[212750]: failed to encode map e338 with expected crc 2023-10-09T06:02:00.131 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:59 smithi064 ceph-mon[220044]: pgmap v800: 119 pgs: 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 9 remapped+peering, 1 active+recovering+undersized+remapped, 105 active+clean; 1.3 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 692 op/s; 251/64338 objects degraded (0.390%); 5228/64338 objects misplaced (8.126%); 26 KiB/s, 217 objects/s recovering 2023-10-09T06:02:00.131 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:59 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:02:00.131 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:59 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:02:00.132 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:59 smithi064 ceph-mon[220044]: osdmap e338: 8 total, 8 up, 8 in 2023-10-09T06:02:00.132 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:59 smithi064 ceph-mon[220044]: failed to encode map e338 with expected crc 2023-10-09T06:02:00.132 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:59 smithi064 ceph-mon[220044]: failed to encode map e338 with expected crc 2023-10-09T06:02:00.132 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:59 smithi064 ceph-mon[220044]: failed to encode map e338 with expected crc 2023-10-09T06:02:00.132 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:59 smithi064 ceph-mon[220044]: failed to encode map e338 with expected crc 2023-10-09T06:02:00.132 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:59 smithi064 ceph-mon[220044]: failed to encode map e338 with expected crc 2023-10-09T06:02:00.132 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:01:59 smithi064 ceph-mon[220044]: failed to encode map e338 with expected crc 2023-10-09T06:02:00.722 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:00.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:00.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:00.826 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:00.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:00 smithi064 ceph-mon[212750]: failed to encode map e339 with expected crc 2023-10-09T06:02:00.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:00 smithi064 ceph-mon[212750]: osdmap e339: 8 total, 8 up, 8 in 2023-10-09T06:02:00.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:00 smithi064 ceph-mon[212750]: failed to encode map e339 with expected crc 2023-10-09T06:02:00.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:00 smithi064 ceph-mon[212750]: failed to encode map e339 with expected crc 2023-10-09T06:02:00.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:00 smithi064 ceph-mon[212750]: failed to encode map e339 with expected crc 2023-10-09T06:02:00.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:00 smithi064 ceph-mon[212750]: failed to encode map e339 with expected crc 2023-10-09T06:02:00.834 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:00 smithi064 ceph-mon[220044]: failed to encode map e339 with expected crc 2023-10-09T06:02:00.834 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:00 smithi064 ceph-mon[220044]: osdmap e339: 8 total, 8 up, 8 in 2023-10-09T06:02:00.834 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:00 smithi064 ceph-mon[220044]: failed to encode map e339 with expected crc 2023-10-09T06:02:00.834 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:00 smithi064 ceph-mon[220044]: failed to encode map e339 with expected crc 2023-10-09T06:02:00.834 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:00 smithi064 ceph-mon[220044]: failed to encode map e339 with expected crc 2023-10-09T06:02:00.834 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:00 smithi064 ceph-mon[220044]: failed to encode map e339 with expected crc 2023-10-09T06:02:00.857 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:01.089 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:01.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:01.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:01.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:00 smithi156 ceph-mon[195101]: failed to encode map e339 with expected crc 2023-10-09T06:02:01.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:00 smithi156 ceph-mon[195101]: osdmap e339: 8 total, 8 up, 8 in 2023-10-09T06:02:01.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:00 smithi156 ceph-mon[195101]: failed to encode map e339 with expected crc 2023-10-09T06:02:01.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:00 smithi156 ceph-mon[195101]: failed to encode map e339 with expected crc 2023-10-09T06:02:01.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:00 smithi156 ceph-mon[195101]: failed to encode map e339 with expected crc 2023-10-09T06:02:01.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:00 smithi156 ceph-mon[195101]: failed to encode map e339 with expected crc 2023-10-09T06:02:01.196 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:02:01.223 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:01.241 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 0, 4] 2023-10-09T06:02:01.241 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:02:01.241 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-10-09T06:02:01.241 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:02:01.268 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:02:01.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:01.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:02:01.373 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:01.405 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:02.085 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:02.094 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:01 smithi064 ceph-mon[220044]: pgmap v803: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 112 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 684 op/s; 6188/58884 objects misplaced (10.509%); 20 KiB/s, 211 objects/s recovering 2023-10-09T06:02:02.094 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:01 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T06:02:02.094 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:01 smithi064 ceph-mon[220044]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 251/64338 objects degraded (0.390%), 1 pg degraded) 2023-10-09T06:02:02.094 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:01 smithi064 ceph-mon[220044]: failed to encode map e340 with expected crc 2023-10-09T06:02:02.094 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:01 smithi064 ceph-mon[220044]: osdmap e340: 8 total, 8 up, 8 in 2023-10-09T06:02:02.094 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:01 smithi064 ceph-mon[220044]: failed to encode map e340 with expected crc 2023-10-09T06:02:02.094 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:01 smithi064 ceph-mon[220044]: failed to encode map e340 with expected crc 2023-10-09T06:02:02.094 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:01 smithi064 ceph-mon[220044]: failed to encode map e340 with expected crc 2023-10-09T06:02:02.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:02.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:02.096 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:01 smithi064 ceph-mon[212750]: pgmap v803: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 112 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 684 op/s; 6188/58884 objects misplaced (10.509%); 20 KiB/s, 211 objects/s recovering 2023-10-09T06:02:02.096 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:01 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T06:02:02.096 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:01 smithi064 ceph-mon[212750]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 251/64338 objects degraded (0.390%), 1 pg degraded) 2023-10-09T06:02:02.096 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:01 smithi064 ceph-mon[212750]: failed to encode map e340 with expected crc 2023-10-09T06:02:02.096 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:01 smithi064 ceph-mon[212750]: osdmap e340: 8 total, 8 up, 8 in 2023-10-09T06:02:02.096 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:01 smithi064 ceph-mon[212750]: failed to encode map e340 with expected crc 2023-10-09T06:02:02.096 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:01 smithi064 ceph-mon[212750]: failed to encode map e340 with expected crc 2023-10-09T06:02:02.097 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:01 smithi064 ceph-mon[212750]: failed to encode map e340 with expected crc 2023-10-09T06:02:02.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:01 smithi156 ceph-mon[195101]: pgmap v803: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 112 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 684 op/s; 6188/58884 objects misplaced (10.509%); 20 KiB/s, 211 objects/s recovering 2023-10-09T06:02:02.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:01 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T06:02:02.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:01 smithi156 ceph-mon[195101]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 251/64338 objects degraded (0.390%), 1 pg degraded) 2023-10-09T06:02:02.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:01 smithi156 ceph-mon[195101]: failed to encode map e340 with expected crc 2023-10-09T06:02:02.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:01 smithi156 ceph-mon[195101]: osdmap e340: 8 total, 8 up, 8 in 2023-10-09T06:02:02.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:01 smithi156 ceph-mon[195101]: failed to encode map e340 with expected crc 2023-10-09T06:02:02.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:01 smithi156 ceph-mon[195101]: failed to encode map e340 with expected crc 2023-10-09T06:02:02.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:01 smithi156 ceph-mon[195101]: failed to encode map e340 with expected crc 2023-10-09T06:02:02.196 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:02:02.223 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:02:02.223 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:02:02.232 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:02.302 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:02:02.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:02.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:02:02.407 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:02.483 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:02.836 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:02:03.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:02 smithi156 ceph-mon[195101]: 1.1b scrub starts 2023-10-09T06:02:03.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:02 smithi156 ceph-mon[195101]: 1.1b scrub ok 2023-10-09T06:02:03.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:02 smithi156 ceph-mon[195101]: osdmap e341: 8 total, 8 up, 8 in 2023-10-09T06:02:03.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:02 smithi156 ceph-mon[195101]: 7.4 continuing backfill to osd.1 from (308'6200,339'11970] 7:231f005e:::benchmark_data_smithi064_398483_object95525:head to 339'11970 2023-10-09T06:02:03.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:02 smithi156 ceph-mon[195101]: failed to encode map e341 with expected crc 2023-10-09T06:02:03.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:02 smithi156 ceph-mon[195101]: failed to encode map e341 with expected crc 2023-10-09T06:02:03.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:02 smithi156 ceph-mon[195101]: failed to encode map e341 with expected crc 2023-10-09T06:02:03.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:02 smithi156 ceph-mon[195101]: failed to encode map e341 with expected crc 2023-10-09T06:02:03.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:02 smithi156 ceph-mon[195101]: failed to encode map e341 with expected crc 2023-10-09T06:02:03.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:02 smithi064 ceph-mon[220044]: 1.1b scrub starts 2023-10-09T06:02:03.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:02 smithi064 ceph-mon[220044]: 1.1b scrub ok 2023-10-09T06:02:03.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:02 smithi064 ceph-mon[220044]: osdmap e341: 8 total, 8 up, 8 in 2023-10-09T06:02:03.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:02 smithi064 ceph-mon[220044]: 7.4 continuing backfill to osd.1 from (308'6200,339'11970] 7:231f005e:::benchmark_data_smithi064_398483_object95525:head to 339'11970 2023-10-09T06:02:03.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:02 smithi064 ceph-mon[220044]: failed to encode map e341 with expected crc 2023-10-09T06:02:03.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:02 smithi064 ceph-mon[220044]: failed to encode map e341 with expected crc 2023-10-09T06:02:03.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:02 smithi064 ceph-mon[220044]: failed to encode map e341 with expected crc 2023-10-09T06:02:03.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:02 smithi064 ceph-mon[220044]: failed to encode map e341 with expected crc 2023-10-09T06:02:03.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:02 smithi064 ceph-mon[220044]: failed to encode map e341 with expected crc 2023-10-09T06:02:03.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:02 smithi064 ceph-mon[212750]: 1.1b scrub starts 2023-10-09T06:02:03.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:02 smithi064 ceph-mon[212750]: 1.1b scrub ok 2023-10-09T06:02:03.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:02 smithi064 ceph-mon[212750]: osdmap e341: 8 total, 8 up, 8 in 2023-10-09T06:02:03.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:02 smithi064 ceph-mon[212750]: 7.4 continuing backfill to osd.1 from (308'6200,339'11970] 7:231f005e:::benchmark_data_smithi064_398483_object95525:head to 339'11970 2023-10-09T06:02:03.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:02 smithi064 ceph-mon[212750]: failed to encode map e341 with expected crc 2023-10-09T06:02:03.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:02 smithi064 ceph-mon[212750]: failed to encode map e341 with expected crc 2023-10-09T06:02:03.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:02 smithi064 ceph-mon[212750]: failed to encode map e341 with expected crc 2023-10-09T06:02:03.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:02 smithi064 ceph-mon[212750]: failed to encode map e341 with expected crc 2023-10-09T06:02:03.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:02 smithi064 ceph-mon[212750]: failed to encode map e341 with expected crc 2023-10-09T06:02:03.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:03.318 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:03.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:03.419 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:03.446 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:03.457 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:03.498 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:03.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:03.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:03.526 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:02:03.604 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:03.633 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:03.689 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:03.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:03.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:03.798 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:03.922 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:03.941 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:04.210 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:02:04.368 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:04.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:04.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:04.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:04 smithi064 ceph-mon[220044]: pgmap v806: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 112 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 568 op/s; 5891/57615 objects misplaced (10.225%); 20 KiB/s, 211 objects/s recovering 2023-10-09T06:02:04.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:04 smithi064 ceph-mon[220044]: 1.e scrub starts 2023-10-09T06:02:04.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:04 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:02:04.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:04 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:02:04.454 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:04 smithi064 ceph-mon[212750]: pgmap v806: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 112 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 568 op/s; 5891/57615 objects misplaced (10.225%); 20 KiB/s, 211 objects/s recovering 2023-10-09T06:02:04.454 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:04 smithi064 ceph-mon[212750]: 1.e scrub starts 2023-10-09T06:02:04.454 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:04 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:02:04.454 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:04 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:02:04.481 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:04.512 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:04.566 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:04.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:04.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:04.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:04 smithi156 ceph-mon[195101]: pgmap v806: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 112 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 568 op/s; 5891/57615 objects misplaced (10.225%); 20 KiB/s, 211 objects/s recovering 2023-10-09T06:02:04.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:04 smithi156 ceph-mon[195101]: 1.e scrub starts 2023-10-09T06:02:04.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:04 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:02:04.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:04 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:02:04.673 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:04.767 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:05.354 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:05.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:05.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:05.461 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:05.565 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:02:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:02:05] "GET /metrics HTTP/1.1" 200 36410 "" "Prometheus/2.43.0" 2023-10-09T06:02:05.572 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:05.936 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:05.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:05.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:06.046 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:06.115 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:06.625 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:06.628 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:06 smithi064 ceph-mon[220044]: pgmap v807: 119 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 peering, 4 active+remapped+backfill_wait, 112 active+clean; 1.1 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 7.1 KiB/s rd, 18 KiB/s wr, 458 op/s; 4899/56253 objects misplaced (8.709%); 9.3 MiB/s, 291 objects/s recovering 2023-10-09T06:02:06.628 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:06 smithi064 ceph-mon[220044]: 1.5 scrub starts 2023-10-09T06:02:06.628 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:06 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:02:06.628 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:06 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:02:06.628 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:06 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:02:06.629 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:06 smithi064 ceph-mon[212750]: pgmap v807: 119 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 peering, 4 active+remapped+backfill_wait, 112 active+clean; 1.1 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 7.1 KiB/s rd, 18 KiB/s wr, 458 op/s; 4899/56253 objects misplaced (8.709%); 9.3 MiB/s, 291 objects/s recovering 2023-10-09T06:02:06.629 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:06 smithi064 ceph-mon[212750]: 1.5 scrub starts 2023-10-09T06:02:06.629 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:06 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:02:06.629 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:06 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:02:06.629 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:06 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:02:06.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:06.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:06.731 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:06.755 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:06 smithi156 ceph-mon[195101]: pgmap v807: 119 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 peering, 4 active+remapped+backfill_wait, 112 active+clean; 1.1 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 7.1 KiB/s rd, 18 KiB/s wr, 458 op/s; 4899/56253 objects misplaced (8.709%); 9.3 MiB/s, 291 objects/s recovering 2023-10-09T06:02:06.755 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:06 smithi156 ceph-mon[195101]: 1.5 scrub starts 2023-10-09T06:02:06.755 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:06 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:02:06.755 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:06 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:02:06.755 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:06 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:02:06.757 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:06.806 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:06.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:06.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:06.911 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:06.919 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:06.964 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:07.131 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:07.239 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:07.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:07.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:07.344 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:07.382 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:07.453 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:02:07.466 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T06:02:07.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:07 smithi064 ceph-mon[212750]: osdmap e342: 8 total, 8 up, 8 in 2023-10-09T06:02:07.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:07 smithi064 ceph-mon[212750]: failed to encode map e342 with expected crc 2023-10-09T06:02:07.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:07 smithi064 ceph-mon[212750]: failed to encode map e342 with expected crc 2023-10-09T06:02:07.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:07 smithi064 ceph-mon[212750]: failed to encode map e342 with expected crc 2023-10-09T06:02:07.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:07 smithi064 ceph-mon[212750]: failed to encode map e342 with expected crc 2023-10-09T06:02:07.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:07 smithi064 ceph-mon[212750]: failed to encode map e342 with expected crc 2023-10-09T06:02:07.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:07 smithi064 ceph-mon[212750]: failed to encode map e342 with expected crc 2023-10-09T06:02:07.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:07 smithi064 ceph-mon[220044]: osdmap e342: 8 total, 8 up, 8 in 2023-10-09T06:02:07.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:07 smithi064 ceph-mon[220044]: failed to encode map e342 with expected crc 2023-10-09T06:02:07.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:07 smithi064 ceph-mon[220044]: failed to encode map e342 with expected crc 2023-10-09T06:02:07.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:07 smithi064 ceph-mon[220044]: failed to encode map e342 with expected crc 2023-10-09T06:02:07.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:07 smithi064 ceph-mon[220044]: failed to encode map e342 with expected crc 2023-10-09T06:02:07.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:07 smithi064 ceph-mon[220044]: failed to encode map e342 with expected crc 2023-10-09T06:02:07.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:07 smithi064 ceph-mon[220044]: failed to encode map e342 with expected crc 2023-10-09T06:02:07.480 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:07.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:07.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:07.586 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:07.686 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:07.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:07 smithi156 ceph-mon[195101]: osdmap e342: 8 total, 8 up, 8 in 2023-10-09T06:02:07.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:07 smithi156 ceph-mon[195101]: failed to encode map e342 with expected crc 2023-10-09T06:02:07.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:07 smithi156 ceph-mon[195101]: failed to encode map e342 with expected crc 2023-10-09T06:02:07.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:07 smithi156 ceph-mon[195101]: failed to encode map e342 with expected crc 2023-10-09T06:02:07.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:07 smithi156 ceph-mon[195101]: failed to encode map e342 with expected crc 2023-10-09T06:02:07.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:07 smithi156 ceph-mon[195101]: failed to encode map e342 with expected crc 2023-10-09T06:02:07.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:07 smithi156 ceph-mon[195101]: failed to encode map e342 with expected crc 2023-10-09T06:02:08.395 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:08 smithi064 ceph-mon[212750]: pgmap v809: 119 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 113 active+clean; 1.0 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 406 op/s; 5495/51558 objects misplaced (10.658%); 12 MiB/s, 203 objects/s recovering 2023-10-09T06:02:08.395 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:08 smithi064 ceph-mon[212750]: failed to encode map e343 with expected crc 2023-10-09T06:02:08.396 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:08 smithi064 ceph-mon[212750]: failed to encode map e343 with expected crc 2023-10-09T06:02:08.396 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:08 smithi064 ceph-mon[212750]: osdmap e343: 8 total, 8 up, 8 in 2023-10-09T06:02:08.396 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:08 smithi064 ceph-mon[212750]: failed to encode map e343 with expected crc 2023-10-09T06:02:08.396 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:08 smithi064 ceph-mon[212750]: failed to encode map e343 with expected crc 2023-10-09T06:02:08.396 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:08 smithi064 ceph-mon[212750]: failed to encode map e343 with expected crc 2023-10-09T06:02:08.396 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:08 smithi064 ceph-mon[212750]: failed to encode map e343 with expected crc 2023-10-09T06:02:08.396 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:08 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/41395274' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:02:08.396 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:08 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:02:08.418 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:02:08.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:08 smithi064 ceph-mon[220044]: pgmap v809: 119 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 113 active+clean; 1.0 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 406 op/s; 5495/51558 objects misplaced (10.658%); 12 MiB/s, 203 objects/s recovering 2023-10-09T06:02:08.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:08 smithi064 ceph-mon[220044]: failed to encode map e343 with expected crc 2023-10-09T06:02:08.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:08 smithi064 ceph-mon[220044]: failed to encode map e343 with expected crc 2023-10-09T06:02:08.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:08 smithi064 ceph-mon[220044]: osdmap e343: 8 total, 8 up, 8 in 2023-10-09T06:02:08.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:08 smithi064 ceph-mon[220044]: failed to encode map e343 with expected crc 2023-10-09T06:02:08.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:08 smithi064 ceph-mon[220044]: failed to encode map e343 with expected crc 2023-10-09T06:02:08.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:08 smithi064 ceph-mon[220044]: failed to encode map e343 with expected crc 2023-10-09T06:02:08.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:08 smithi064 ceph-mon[220044]: failed to encode map e343 with expected crc 2023-10-09T06:02:08.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:08 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/41395274' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:02:08.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:08 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:02:08.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:08 smithi156 ceph-mon[195101]: pgmap v809: 119 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 113 active+clean; 1.0 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 406 op/s; 5495/51558 objects misplaced (10.658%); 12 MiB/s, 203 objects/s recovering 2023-10-09T06:02:08.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:08 smithi156 ceph-mon[195101]: failed to encode map e343 with expected crc 2023-10-09T06:02:08.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:08 smithi156 ceph-mon[195101]: failed to encode map e343 with expected crc 2023-10-09T06:02:08.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:08 smithi156 ceph-mon[195101]: osdmap e343: 8 total, 8 up, 8 in 2023-10-09T06:02:08.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:08 smithi156 ceph-mon[195101]: failed to encode map e343 with expected crc 2023-10-09T06:02:08.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:08 smithi156 ceph-mon[195101]: failed to encode map e343 with expected crc 2023-10-09T06:02:08.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:08 smithi156 ceph-mon[195101]: failed to encode map e343 with expected crc 2023-10-09T06:02:08.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:08 smithi156 ceph-mon[195101]: failed to encode map e343 with expected crc 2023-10-09T06:02:08.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:08 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/41395274' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:02:08.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:08 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:02:09.397 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:09.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:09.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:09.506 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:09.568 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:09.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:09 smithi064 ceph-mon[212750]: from='client.35559 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:02:09.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:09 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:02:09.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:09 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:02:09.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:09 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:02:09.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:09 smithi064 ceph-mon[212750]: osdmap e344: 8 total, 8 up, 8 in 2023-10-09T06:02:09.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:09 smithi064 ceph-mon[212750]: failed to encode map e344 with expected crc 2023-10-09T06:02:09.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:09 smithi064 ceph-mon[212750]: failed to encode map e344 with expected crc 2023-10-09T06:02:09.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:09 smithi064 ceph-mon[212750]: failed to encode map e344 with expected crc 2023-10-09T06:02:09.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:09 smithi064 ceph-mon[220044]: from='client.35559 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:02:09.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:09 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:02:09.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:09 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T06:02:09.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:09 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:02:09.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:09 smithi064 ceph-mon[220044]: osdmap e344: 8 total, 8 up, 8 in 2023-10-09T06:02:09.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:09 smithi064 ceph-mon[220044]: failed to encode map e344 with expected crc 2023-10-09T06:02:09.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:09 smithi064 ceph-mon[220044]: failed to encode map e344 with expected crc 2023-10-09T06:02:09.725 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:09 smithi064 ceph-mon[220044]: failed to encode map e344 with expected crc 2023-10-09T06:02:09.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:09 smithi156 ceph-mon[195101]: from='client.35559 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:02:09.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:09 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:02:09.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:09 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T06:02:09.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:09 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:02:09.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:09 smithi156 ceph-mon[195101]: osdmap e344: 8 total, 8 up, 8 in 2023-10-09T06:02:09.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:09 smithi156 ceph-mon[195101]: failed to encode map e344 with expected crc 2023-10-09T06:02:09.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:09 smithi156 ceph-mon[195101]: failed to encode map e344 with expected crc 2023-10-09T06:02:09.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:09 smithi156 ceph-mon[195101]: failed to encode map e344 with expected crc 2023-10-09T06:02:10.150 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:10.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:10.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:10.254 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:10.279 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:10.348 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:10.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:10.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:10.405 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:10 smithi064 ceph-mon[212750]: failed to encode map e344 with expected crc 2023-10-09T06:02:10.405 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:10 smithi064 ceph-mon[212750]: failed to encode map e344 with expected crc 2023-10-09T06:02:10.405 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:10 smithi064 ceph-mon[212750]: failed to encode map e344 with expected crc 2023-10-09T06:02:10.405 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:10 smithi064 ceph-mon[212750]: pgmap v812: 119 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 114 active+clean; 1007 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 4431/48360 objects misplaced (9.163%); 19 MiB/s, 319 objects/s recovering 2023-10-09T06:02:10.406 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:10 smithi064 ceph-mon[212750]: 1.11 scrub starts 2023-10-09T06:02:10.455 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:10.491 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":810,"stamp":"2023-10-09T06:02:07.396363+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1125284657,"num_objects":17186,"num_object_clones":0,"num_object_copies":51558,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5495,"num_objects_unfound":0,"num_objects_dirty":17186,"num_whiteouts":0,"num_read":131,"num_read_kb":244,"num_write":193666,"num_write_kb":6743071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6797,"num_bytes_recovered":73150384,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":98767,"ondisk_log_size":98767,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":363,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6029404,"kb_used_data":3420252,"kb_used_omap":163,"kb_used_meta":2608540,"kb_avail":743931812,"statfs":{"total":767960285184,"available":761786175488,"internally_reserved":0,"allocated":3502338048,"data_stored":3490406766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":167439,"internal_metadata":2671145457},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-160038912,"num_objects":-2442,"num_object_clones":0,"num_object_copies":-7326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-693,"num_objects_unfound":0,"num_objects_dirty":-2442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2442,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1220,"num_bytes_recovered":72679448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.592790"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":571,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.871830+0000","last_change":"2023-10-09T06:00:05.413041+0000","last_active":"2023-10-09T06:02:00.871830+0000","last_peered":"2023-10-09T06:02:00.871830+0000","last_clean":"2023-10-09T06:02:00.871830+0000","last_became_active":"2023-10-09T05:59:40.308608+0000","last_became_peered":"2023-10-09T05:59:40.308608+0000","last_unstale":"2023-10-09T06:02:00.871830+0000","last_undegraded":"2023-10-09T06:02:00.871830+0000","last_fullsized":"2023-10-09T06:02:00.871830+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:05.412941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:05.412941+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:53.518055+0000","scrub_duration":0.0061014329999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":389,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.809627+0000","last_change":"2023-10-09T06:01:54.757141+0000","last_active":"2023-10-09T06:02:00.809627+0000","last_peered":"2023-10-09T06:02:00.809627+0000","last_clean":"2023-10-09T06:02:00.809627+0000","last_became_active":"2023-10-09T06:01:54.756855+0000","last_became_peered":"2023-10-09T06:01:54.756855+0000","last_unstale":"2023-10-09T06:02:00.809627+0000","last_undegraded":"2023-10-09T06:02:00.809627+0000","last_fullsized":"2023-10-09T06:02:00.809627+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:57.741319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":128,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.811793+0000","last_change":"2023-10-09T06:01:54.762232+0000","last_active":"2023-10-09T06:02:00.811793+0000","last_peered":"2023-10-09T06:02:00.811793+0000","last_clean":"2023-10-09T06:02:00.811793+0000","last_became_active":"2023-10-09T06:01:54.761935+0000","last_became_peered":"2023-10-09T06:01:54.761935+0000","last_unstale":"2023-10-09T06:02:00.811793+0000","last_undegraded":"2023-10-09T06:02:00.811793+0000","last_fullsized":"2023-10-09T06:02:00.811793+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:20.380704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":623,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:01.468516+0000","last_change":"2023-10-09T06:02:01.468449+0000","last_active":"2023-10-09T06:02:01.468516+0000","last_peered":"2023-10-09T06:02:01.468516+0000","last_clean":"2023-10-09T06:02:01.468516+0000","last_became_active":"2023-10-09T06:01:22.334923+0000","last_became_peered":"2023-10-09T06:01:22.334923+0000","last_unstale":"2023-10-09T06:02:01.468516+0000","last_undegraded":"2023-10-09T06:02:01.468516+0000","last_fullsized":"2023-10-09T06:02:01.468516+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:01.468346+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:01.468346+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:20:04.967048+0000","scrub_duration":0.0016258220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":18,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.804907+0000","last_change":"2023-10-09T06:01:54.762060+0000","last_active":"2023-10-09T06:02:00.804907+0000","last_peered":"2023-10-09T06:02:00.804907+0000","last_clean":"2023-10-09T06:02:00.804907+0000","last_became_active":"2023-10-09T06:01:54.761757+0000","last_became_peered":"2023-10-09T06:01:54.761757+0000","last_unstale":"2023-10-09T06:02:00.804907+0000","last_undegraded":"2023-10-09T06:02:00.804907+0000","last_fullsized":"2023-10-09T06:02:00.804907+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:40.294232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":301,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.872482+0000","last_change":"2023-10-09T06:01:41.942412+0000","last_active":"2023-10-09T06:02:00.872482+0000","last_peered":"2023-10-09T06:02:00.872482+0000","last_clean":"2023-10-09T06:02:00.872482+0000","last_became_active":"2023-10-09T06:01:41.941186+0000","last_became_peered":"2023-10-09T06:01:41.941186+0000","last_unstale":"2023-10-09T06:02:00.872482+0000","last_undegraded":"2023-10-09T06:02:00.872482+0000","last_fullsized":"2023-10-09T06:02:00.872482+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:31.680824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":566,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.809816+0000","last_change":"2023-10-09T06:01:57.804007+0000","last_active":"2023-10-09T06:02:00.809816+0000","last_peered":"2023-10-09T06:02:00.809816+0000","last_clean":"2023-10-09T06:02:00.809816+0000","last_became_active":"2023-10-09T06:01:57.803642+0000","last_became_peered":"2023-10-09T06:01:57.803642+0000","last_unstale":"2023-10-09T06:02:00.809816+0000","last_undegraded":"2023-10-09T06:02:00.809816+0000","last_fullsized":"2023-10-09T06:02:00.809816+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.065661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.065661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":113,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.811146+0000","last_change":"2023-10-09T06:01:54.760903+0000","last_active":"2023-10-09T06:02:00.811146+0000","last_peered":"2023-10-09T06:02:00.811146+0000","last_clean":"2023-10-09T06:02:00.811146+0000","last_became_active":"2023-10-09T06:01:54.760338+0000","last_became_peered":"2023-10-09T06:01:54.760338+0000","last_unstale":"2023-10-09T06:02:00.811146+0000","last_undegraded":"2023-10-09T06:02:00.811146+0000","last_fullsized":"2023-10-09T06:02:00.811146+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:53.904647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":377,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.813241+0000","last_change":"2023-10-09T06:01:54.757269+0000","last_active":"2023-10-09T06:02:00.813241+0000","last_peered":"2023-10-09T06:02:00.813241+0000","last_clean":"2023-10-09T06:02:00.813241+0000","last_became_active":"2023-10-09T06:01:54.756894+0000","last_became_peered":"2023-10-09T06:01:54.756894+0000","last_unstale":"2023-10-09T06:02:00.813241+0000","last_undegraded":"2023-10-09T06:02:00.813241+0000","last_fullsized":"2023-10-09T06:02:00.813241+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:45.212821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":660,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.872121+0000","last_change":"2023-10-09T06:01:57.804160+0000","last_active":"2023-10-09T06:02:00.872121+0000","last_peered":"2023-10-09T06:02:00.872121+0000","last_clean":"2023-10-09T06:02:00.872121+0000","last_became_active":"2023-10-09T06:01:57.803424+0000","last_became_peered":"2023-10-09T06:01:57.803424+0000","last_unstale":"2023-10-09T06:02:00.872121+0000","last_undegraded":"2023-10-09T06:02:00.872121+0000","last_fullsized":"2023-10-09T06:02:00.872121+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.384407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.384407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:42.857346+0000","scrub_duration":0.0040651960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":219,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.812187+0000","last_change":"2023-10-09T06:01:41.939852+0000","last_active":"2023-10-09T06:02:00.812187+0000","last_peered":"2023-10-09T06:02:00.812187+0000","last_clean":"2023-10-09T06:02:00.812187+0000","last_became_active":"2023-10-09T06:01:41.939556+0000","last_became_peered":"2023-10-09T06:01:41.939556+0000","last_unstale":"2023-10-09T06:02:00.812187+0000","last_undegraded":"2023-10-09T06:02:00.812187+0000","last_fullsized":"2023-10-09T06:02:00.812187+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:06.980511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":129,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-09T06:02:01.813499+0000","last_change":"2023-10-09T06:01:54.761101+0000","last_active":"2023-10-09T06:02:01.813499+0000","last_peered":"2023-10-09T06:02:01.813499+0000","last_clean":"2023-10-09T06:02:01.813499+0000","last_became_active":"2023-10-09T06:01:54.760571+0000","last_became_peered":"2023-10-09T06:01:54.760571+0000","last_unstale":"2023-10-09T06:02:01.813499+0000","last_undegraded":"2023-10-09T06:02:01.813499+0000","last_fullsized":"2023-10-09T06:02:01.813499+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:56.812358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":621,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.809097+0000","last_change":"2023-10-09T06:01:22.334061+0000","last_active":"2023-10-09T06:02:00.809097+0000","last_peered":"2023-10-09T06:02:00.809097+0000","last_clean":"2023-10-09T06:02:00.809097+0000","last_became_active":"2023-10-09T06:01:22.333698+0000","last_became_peered":"2023-10-09T06:01:22.333698+0000","last_unstale":"2023-10-09T06:02:00.809097+0000","last_undegraded":"2023-10-09T06:02:00.809097+0000","last_fullsized":"2023-10-09T06:02:00.809097+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.913004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.913004+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:57.020327+0000","scrub_duration":0.0017791980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":390,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.872755+0000","last_change":"2023-10-09T06:01:54.766344+0000","last_active":"2023-10-09T06:02:00.872755+0000","last_peered":"2023-10-09T06:02:00.872755+0000","last_clean":"2023-10-09T06:02:00.872755+0000","last_became_active":"2023-10-09T06:01:54.766086+0000","last_became_peered":"2023-10-09T06:01:54.766086+0000","last_unstale":"2023-10-09T06:02:00.872755+0000","last_undegraded":"2023-10-09T06:02:00.872755+0000","last_fullsized":"2023-10-09T06:02:00.872755+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:47.163700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":286,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.811937+0000","last_change":"2023-10-09T05:53:59.185702+0000","last_active":"2023-10-09T06:02:00.811937+0000","last_peered":"2023-10-09T06:02:00.811937+0000","last_clean":"2023-10-09T06:02:00.811937+0000","last_became_active":"2023-10-09T05:53:59.185373+0000","last_became_peered":"2023-10-09T05:53:59.185373+0000","last_unstale":"2023-10-09T06:02:00.811937+0000","last_undegraded":"2023-10-09T06:02:00.811937+0000","last_fullsized":"2023-10-09T06:02:00.811937+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:50.756967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":736,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.813372+0000","last_change":"2023-10-09T06:01:41.940053+0000","last_active":"2023-10-09T06:02:00.813372+0000","last_peered":"2023-10-09T06:02:00.813372+0000","last_clean":"2023-10-09T06:02:00.813372+0000","last_became_active":"2023-10-09T06:01:41.939691+0000","last_became_peered":"2023-10-09T06:01:41.939691+0000","last_unstale":"2023-10-09T06:02:00.813372+0000","last_undegraded":"2023-10-09T06:02:00.813372+0000","last_fullsized":"2023-10-09T06:02:00.813372+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:02.265460+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:02.265460+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002003992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":198,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.872569+0000","last_change":"2023-10-09T06:01:22.337444+0000","last_active":"2023-10-09T06:02:00.872569+0000","last_peered":"2023-10-09T06:02:00.872569+0000","last_clean":"2023-10-09T06:02:00.872569+0000","last_became_active":"2023-10-09T06:01:22.336348+0000","last_became_peered":"2023-10-09T06:01:22.336348+0000","last_unstale":"2023-10-09T06:02:00.872569+0000","last_undegraded":"2023-10-09T06:02:00.872569+0000","last_fullsized":"2023-10-09T06:02:00.872569+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:43:11.149649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":18,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.805119+0000","last_change":"2023-10-09T06:01:54.755723+0000","last_active":"2023-10-09T06:02:00.805119+0000","last_peered":"2023-10-09T06:02:00.805119+0000","last_clean":"2023-10-09T06:02:00.805119+0000","last_became_active":"2023-10-09T06:01:54.755334+0000","last_became_peered":"2023-10-09T06:01:54.755334+0000","last_unstale":"2023-10-09T06:02:00.805119+0000","last_undegraded":"2023-10-09T06:02:00.805119+0000","last_fullsized":"2023-10-09T06:02:00.805119+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:12.041652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":493,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.806155+0000","last_change":"2023-10-09T06:01:57.807839+0000","last_active":"2023-10-09T06:02:00.806155+0000","last_peered":"2023-10-09T06:02:00.806155+0000","last_clean":"2023-10-09T06:02:00.806155+0000","last_became_active":"2023-10-09T06:01:57.807451+0000","last_became_peered":"2023-10-09T06:01:57.807451+0000","last_unstale":"2023-10-09T06:02:00.806155+0000","last_undegraded":"2023-10-09T06:02:00.806155+0000","last_fullsized":"2023-10-09T06:02:00.806155+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.339121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.339121+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:47.740601+0000","scrub_duration":0.010474344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":335,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.810521+0000","last_change":"2023-10-09T06:01:22.335608+0000","last_active":"2023-10-09T06:02:00.810521+0000","last_peered":"2023-10-09T06:02:00.810521+0000","last_clean":"2023-10-09T06:02:00.810521+0000","last_became_active":"2023-10-09T06:01:22.335296+0000","last_became_peered":"2023-10-09T06:01:22.335296+0000","last_unstale":"2023-10-09T06:02:00.810521+0000","last_undegraded":"2023-10-09T06:02:00.810521+0000","last_fullsized":"2023-10-09T06:02:00.810521+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:21.843341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":353,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.810192+0000","last_change":"2023-10-09T06:01:54.764916+0000","last_active":"2023-10-09T06:02:00.810192+0000","last_peered":"2023-10-09T06:02:00.810192+0000","last_clean":"2023-10-09T06:02:00.810192+0000","last_became_active":"2023-10-09T06:01:54.764705+0000","last_became_peered":"2023-10-09T06:01:54.764705+0000","last_unstale":"2023-10-09T06:02:00.810192+0000","last_undegraded":"2023-10-09T06:02:00.810192+0000","last_fullsized":"2023-10-09T06:02:00.810192+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:12.527719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":496,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.811319+0000","last_change":"2023-10-09T06:01:41.939797+0000","last_active":"2023-10-09T06:02:00.811319+0000","last_peered":"2023-10-09T06:02:00.811319+0000","last_clean":"2023-10-09T06:02:00.811319+0000","last_became_active":"2023-10-09T06:01:41.939455+0000","last_became_peered":"2023-10-09T06:01:41.939455+0000","last_unstale":"2023-10-09T06:02:00.811319+0000","last_undegraded":"2023-10-09T06:02:00.811319+0000","last_fullsized":"2023-10-09T06:02:00.811319+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:38.230079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:38.230079+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":200,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.872332+0000","last_change":"2023-10-09T06:01:22.339830+0000","last_active":"2023-10-09T06:02:00.872332+0000","last_peered":"2023-10-09T06:02:00.872332+0000","last_clean":"2023-10-09T06:02:00.872332+0000","last_became_active":"2023-10-09T06:01:22.339525+0000","last_became_peered":"2023-10-09T06:01:22.339525+0000","last_unstale":"2023-10-09T06:02:00.872332+0000","last_undegraded":"2023-10-09T06:02:00.872332+0000","last_fullsized":"2023-10-09T06:02:00.872332+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:57.521163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":381,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.811730+0000","last_change":"2023-10-09T06:01:54.761558+0000","last_active":"2023-10-09T06:02:00.811730+0000","last_peered":"2023-10-09T06:02:00.811730+0000","last_clean":"2023-10-09T06:02:00.811730+0000","last_became_active":"2023-10-09T06:01:54.761251+0000","last_became_peered":"2023-10-09T06:01:54.761251+0000","last_unstale":"2023-10-09T06:02:00.811730+0000","last_undegraded":"2023-10-09T06:02:00.811730+0000","last_fullsized":"2023-10-09T06:02:00.811730+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:53.245267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":764,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.872061+0000","last_change":"2023-10-09T05:59:15.170543+0000","last_active":"2023-10-09T06:02:00.872061+0000","last_peered":"2023-10-09T06:02:00.872061+0000","last_clean":"2023-10-09T06:02:00.872061+0000","last_became_active":"2023-10-09T05:58:21.332533+0000","last_became_peered":"2023-10-09T05:58:21.332533+0000","last_unstale":"2023-10-09T06:02:00.872061+0000","last_undegraded":"2023-10-09T06:02:00.872061+0000","last_fullsized":"2023-10-09T06:02:00.872061+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:33.995695+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":266,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.872294+0000","last_change":"2023-10-09T06:01:41.942115+0000","last_active":"2023-10-09T06:02:00.872294+0000","last_peered":"2023-10-09T06:02:00.872294+0000","last_clean":"2023-10-09T06:02:00.872294+0000","last_became_active":"2023-10-09T06:01:41.941827+0000","last_became_peered":"2023-10-09T06:01:41.941827+0000","last_unstale":"2023-10-09T06:02:00.872294+0000","last_undegraded":"2023-10-09T06:02:00.872294+0000","last_fullsized":"2023-10-09T06:02:00.872294+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:51:15.226428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":264,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.872399+0000","last_change":"2023-10-09T05:56:49.237784+0000","last_active":"2023-10-09T06:02:00.872399+0000","last_peered":"2023-10-09T06:02:00.872399+0000","last_clean":"2023-10-09T06:02:00.872399+0000","last_became_active":"2023-10-09T05:56:49.237417+0000","last_became_peered":"2023-10-09T05:56:49.237417+0000","last_unstale":"2023-10-09T06:02:00.872399+0000","last_undegraded":"2023-10-09T06:02:00.872399+0000","last_fullsized":"2023-10-09T06:02:00.872399+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:33.129268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":528,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.805516+0000","last_change":"2023-10-09T06:01:57.807925+0000","last_active":"2023-10-09T06:02:00.805516+0000","last_peered":"2023-10-09T06:02:00.805516+0000","last_clean":"2023-10-09T06:02:00.805516+0000","last_became_active":"2023-10-09T06:01:57.807514+0000","last_became_peered":"2023-10-09T06:01:57.807514+0000","last_unstale":"2023-10-09T06:02:00.805516+0000","last_undegraded":"2023-10-09T06:02:00.805516+0000","last_fullsized":"2023-10-09T06:02:00.805516+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:37.295094+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:37.295094+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":194,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.871960+0000","last_change":"2023-10-09T06:01:22.337171+0000","last_active":"2023-10-09T06:02:00.871960+0000","last_peered":"2023-10-09T06:02:00.871960+0000","last_clean":"2023-10-09T06:02:00.871960+0000","last_became_active":"2023-10-09T06:01:22.335438+0000","last_became_peered":"2023-10-09T06:01:22.335438+0000","last_unstale":"2023-10-09T06:02:00.871960+0000","last_undegraded":"2023-10-09T06:02:00.871960+0000","last_fullsized":"2023-10-09T06:02:00.871960+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:25.036448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":263,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.811847+0000","last_change":"2023-10-09T06:01:41.941888+0000","last_active":"2023-10-09T06:02:00.811847+0000","last_peered":"2023-10-09T06:02:00.811847+0000","last_clean":"2023-10-09T06:02:00.811847+0000","last_became_active":"2023-10-09T06:01:41.941413+0000","last_became_peered":"2023-10-09T06:01:41.941413+0000","last_unstale":"2023-10-09T06:02:00.811847+0000","last_undegraded":"2023-10-09T06:02:00.811847+0000","last_fullsized":"2023-10-09T06:02:00.811847+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:32:41.773690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":580,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.805228+0000","last_change":"2023-10-09T06:01:57.804588+0000","last_active":"2023-10-09T06:02:00.805228+0000","last_peered":"2023-10-09T06:02:00.805228+0000","last_clean":"2023-10-09T06:02:00.805228+0000","last_became_active":"2023-10-09T06:01:57.803733+0000","last_became_peered":"2023-10-09T06:01:57.803733+0000","last_unstale":"2023-10-09T06:02:00.805228+0000","last_undegraded":"2023-10-09T06:02:00.805228+0000","last_fullsized":"2023-10-09T06:02:00.805228+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:46.725374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:46.725374+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":326,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.812890+0000","last_change":"2023-10-09T05:51:57.091639+0000","last_active":"2023-10-09T06:02:00.812890+0000","last_peered":"2023-10-09T06:02:00.812890+0000","last_clean":"2023-10-09T06:02:00.812890+0000","last_became_active":"2023-10-09T05:51:57.091284+0000","last_became_peered":"2023-10-09T05:51:57.091284+0000","last_unstale":"2023-10-09T06:02:00.812890+0000","last_undegraded":"2023-10-09T06:02:00.812890+0000","last_fullsized":"2023-10-09T06:02:00.812890+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:10.553453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":122,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.812220+0000","last_change":"2023-10-09T06:01:54.760790+0000","last_active":"2023-10-09T06:02:00.812220+0000","last_peered":"2023-10-09T06:02:00.812220+0000","last_clean":"2023-10-09T06:02:00.812220+0000","last_became_active":"2023-10-09T06:01:54.760439+0000","last_became_peered":"2023-10-09T06:01:54.760439+0000","last_unstale":"2023-10-09T06:02:00.812220+0000","last_undegraded":"2023-10-09T06:02:00.812220+0000","last_fullsized":"2023-10-09T06:02:00.812220+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:50.428356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"341'12094","reported_seq":12163,"reported_epoch":341,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:02:02.715718+0000","last_change":"2023-10-09T06:01:55.948202+0000","last_active":"2023-10-09T06:02:02.715718+0000","last_peered":"2023-10-09T06:02:02.715718+0000","last_clean":"2023-10-09T06:01:53.691284+0000","last_became_active":"2023-10-09T06:01:55.901526+0000","last_became_peered":"2023-10-09T06:01:55.901526+0000","last_unstale":"2023-10-09T06:02:02.715718+0000","last_undegraded":"2023-10-09T06:02:02.715718+0000","last_fullsized":"2023-10-09T06:02:02.715718+0000","mapping_epoch":334,"log_start":"307'6000","ondisk_log_start":"307'6000","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":6094,"log_dups_size":0,"ondisk_log_size":6094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:43.858721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77463552,"num_objects":1182,"num_object_clones":0,"num_object_copies":3546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1182,"num_objects_unfound":0,"num_objects_dirty":1182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12094,"num_write_kb":424832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[2,5,6],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5,6","objects":1182}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":717,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.872280+0000","last_change":"2023-10-09T06:01:57.804339+0000","last_active":"2023-10-09T06:02:00.872280+0000","last_peered":"2023-10-09T06:02:00.872280+0000","last_clean":"2023-10-09T06:02:00.872280+0000","last_became_active":"2023-10-09T06:01:57.803844+0000","last_became_peered":"2023-10-09T06:01:57.803844+0000","last_unstale":"2023-10-09T06:02:00.872280+0000","last_undegraded":"2023-10-09T06:02:00.872280+0000","last_fullsized":"2023-10-09T06:02:00.872280+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:53.469394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:53.469394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:51.125394+0000","scrub_duration":0.0059716659999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":198,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.872940+0000","last_change":"2023-10-09T06:01:22.336769+0000","last_active":"2023-10-09T06:02:00.872940+0000","last_peered":"2023-10-09T06:02:00.872940+0000","last_clean":"2023-10-09T06:02:00.872940+0000","last_became_active":"2023-10-09T06:01:22.336264+0000","last_became_peered":"2023-10-09T06:01:22.336264+0000","last_unstale":"2023-10-09T06:02:00.872940+0000","last_undegraded":"2023-10-09T06:02:00.872940+0000","last_fullsized":"2023-10-09T06:02:00.872940+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:17.065771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":247,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.871933+0000","last_change":"2023-10-09T05:56:49.234185+0000","last_active":"2023-10-09T06:02:00.871933+0000","last_peered":"2023-10-09T06:02:00.871933+0000","last_clean":"2023-10-09T06:02:00.871933+0000","last_became_active":"2023-10-09T05:56:49.233884+0000","last_became_peered":"2023-10-09T05:56:49.233884+0000","last_unstale":"2023-10-09T06:02:00.871933+0000","last_undegraded":"2023-10-09T06:02:00.871933+0000","last_fullsized":"2023-10-09T06:02:00.871933+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:12.142436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.a","version":"341'12158","reported_seq":12224,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-09T06:02:05.050725+0000","last_change":"2023-10-09T05:58:27.406918+0000","last_active":"2023-10-09T06:02:05.050725+0000","last_peered":"2023-10-09T06:02:05.050725+0000","last_clean":"2023-10-09T06:02:05.050725+0000","last_became_active":"2023-10-09T05:58:27.406600+0000","last_became_peered":"2023-10-09T05:58:27.406600+0000","last_unstale":"2023-10-09T06:02:05.050725+0000","last_undegraded":"2023-10-09T06:02:05.050725+0000","last_fullsized":"2023-10-09T06:02:05.050725+0000","mapping_epoch":285,"log_start":"310'6800","ondisk_log_start":"310'6800","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5358,"log_dups_size":0,"ondisk_log_size":5358,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:00.484611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66846720,"num_objects":1020,"num_object_clones":0,"num_object_copies":3060,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1020,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12158,"num_write_kb":421696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":647,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.811447+0000","last_change":"2023-10-09T06:00:03.241479+0000","last_active":"2023-10-09T06:02:00.811447+0000","last_peered":"2023-10-09T06:02:00.811447+0000","last_clean":"2023-10-09T06:02:00.811447+0000","last_became_active":"2023-10-09T05:59:32.530753+0000","last_became_peered":"2023-10-09T05:59:32.530753+0000","last_unstale":"2023-10-09T06:02:00.811447+0000","last_undegraded":"2023-10-09T06:02:00.811447+0000","last_fullsized":"2023-10-09T06:02:00.811447+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:03.241335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:03.241335+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:54.822114+0000","scrub_duration":0.017780547000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":254,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.872840+0000","last_change":"2023-10-09T06:01:54.826540+0000","last_active":"2023-10-09T06:02:00.872840+0000","last_peered":"2023-10-09T06:02:00.872840+0000","last_clean":"2023-10-09T06:02:00.872840+0000","last_became_active":"2023-10-09T06:01:54.766205+0000","last_became_peered":"2023-10-09T06:01:54.766205+0000","last_unstale":"2023-10-09T06:02:00.872840+0000","last_undegraded":"2023-10-09T06:02:00.872840+0000","last_fullsized":"2023-10-09T06:02:00.872840+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:45.220336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":379,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.809537+0000","last_change":"2023-10-09T06:01:54.760367+0000","last_active":"2023-10-09T06:02:00.809537+0000","last_peered":"2023-10-09T06:02:00.809537+0000","last_clean":"2023-10-09T06:02:00.809537+0000","last_became_active":"2023-10-09T06:01:54.760087+0000","last_became_peered":"2023-10-09T06:01:54.760087+0000","last_unstale":"2023-10-09T06:02:00.809537+0000","last_undegraded":"2023-10-09T06:02:00.809537+0000","last_fullsized":"2023-10-09T06:02:00.809537+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:26.000104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"341'12243","reported_seq":14716,"reported_epoch":341,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:02:05.004675+0000","last_change":"2023-10-09T06:01:55.953277+0000","last_active":"2023-10-09T06:02:05.004675+0000","last_peered":"2023-10-09T06:02:05.004675+0000","last_clean":"2023-10-09T06:01:53.735768+0000","last_became_active":"2023-10-09T06:01:55.904478+0000","last_became_peered":"2023-10-09T06:01:55.904478+0000","last_unstale":"2023-10-09T06:02:05.004675+0000","last_undegraded":"2023-10-09T06:02:05.004675+0000","last_fullsized":"2023-10-09T06:02:05.004675+0000","mapping_epoch":334,"log_start":"308'6300","ondisk_log_start":"308'6300","created":285,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5943,"log_dups_size":0,"ondisk_log_size":5943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:45:17.072152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70057984,"num_objects":1069,"num_object_clones":0,"num_object_copies":3207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1069,"num_objects_unfound":0,"num_objects_dirty":1069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12243,"num_write_kb":425984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1186,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,4,7],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4,7","objects":1069}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":780,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.813471+0000","last_change":"2023-10-09T06:01:22.326209+0000","last_active":"2023-10-09T06:02:00.813471+0000","last_peered":"2023-10-09T06:02:00.813471+0000","last_clean":"2023-10-09T06:02:00.813471+0000","last_became_active":"2023-10-09T06:01:22.325866+0000","last_became_peered":"2023-10-09T06:01:22.325866+0000","last_unstale":"2023-10-09T06:02:00.813471+0000","last_undegraded":"2023-10-09T06:02:00.813471+0000","last_fullsized":"2023-10-09T06:02:00.813471+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:41.753506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:41.753506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:32.324674+0000","scrub_duration":0.0022121929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":179,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.872600+0000","last_change":"2023-10-09T05:53:59.212161+0000","last_active":"2023-10-09T06:02:00.872600+0000","last_peered":"2023-10-09T06:02:00.872600+0000","last_clean":"2023-10-09T06:02:00.872600+0000","last_became_active":"2023-10-09T05:53:59.193814+0000","last_became_peered":"2023-10-09T05:53:59.193814+0000","last_unstale":"2023-10-09T06:02:00.872600+0000","last_undegraded":"2023-10-09T06:02:00.872600+0000","last_fullsized":"2023-10-09T06:02:00.872600+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:10.502408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":390,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.812681+0000","last_change":"2023-10-09T06:01:54.757292+0000","last_active":"2023-10-09T06:02:00.812681+0000","last_peered":"2023-10-09T06:02:00.812681+0000","last_clean":"2023-10-09T06:02:00.812681+0000","last_became_active":"2023-10-09T06:01:54.756872+0000","last_became_peered":"2023-10-09T06:01:54.756872+0000","last_unstale":"2023-10-09T06:02:00.812681+0000","last_undegraded":"2023-10-09T06:02:00.812681+0000","last_fullsized":"2023-10-09T06:02:00.812681+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:54.248077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"341'12127","reported_seq":14509,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-09T06:02:05.556802+0000","last_change":"2023-10-09T06:01:32.900283+0000","last_active":"2023-10-09T06:02:05.556802+0000","last_peered":"2023-10-09T06:02:05.556802+0000","last_clean":"2023-10-09T06:02:05.556802+0000","last_became_active":"2023-10-09T06:01:32.841280+0000","last_became_peered":"2023-10-09T06:01:32.841280+0000","last_unstale":"2023-10-09T06:02:05.556802+0000","last_undegraded":"2023-10-09T06:02:05.556802+0000","last_fullsized":"2023-10-09T06:02:05.556802+0000","mapping_epoch":326,"log_start":"308'6300","ondisk_log_start":"308'6300","created":285,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5827,"log_dups_size":0,"ondisk_log_size":5827,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:51.847473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64946176,"num_objects":991,"num_object_clones":0,"num_object_copies":2973,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":991,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12127,"num_write_kb":419776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1140,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":506,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-09T06:02:03.461860+0000","last_change":"2023-10-09T06:02:03.461860+0000","last_active":"2023-10-09T06:02:03.461860+0000","last_peered":"2023-10-09T06:02:03.461860+0000","last_clean":"2023-10-09T06:02:03.461860+0000","last_became_active":"2023-10-09T06:01:22.337294+0000","last_became_peered":"2023-10-09T06:01:22.337294+0000","last_unstale":"2023-10-09T06:02:03.461860+0000","last_undegraded":"2023-10-09T06:02:03.461860+0000","last_fullsized":"2023-10-09T06:02:03.461860+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.043489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.043489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":105,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.811556+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T06:02:00.811556+0000","last_peered":"2023-10-09T06:02:00.811556+0000","last_clean":"2023-10-09T06:02:00.811556+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T06:02:00.811556+0000","last_undegraded":"2023-10-09T06:02:00.811556+0000","last_fullsized":"2023-10-09T06:02:00.811556+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":377,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.807802+0000","last_change":"2023-10-09T06:01:54.753427+0000","last_active":"2023-10-09T06:02:00.807802+0000","last_peered":"2023-10-09T06:02:00.807802+0000","last_clean":"2023-10-09T06:02:00.807802+0000","last_became_active":"2023-10-09T06:01:54.753099+0000","last_became_peered":"2023-10-09T06:01:54.753099+0000","last_unstale":"2023-10-09T06:02:00.807802+0000","last_undegraded":"2023-10-09T06:02:00.807802+0000","last_fullsized":"2023-10-09T06:02:00.807802+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:07.504657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"341'11946","reported_seq":12020,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-09T06:02:05.037397+0000","last_change":"2023-10-09T06:01:41.970909+0000","last_active":"2023-10-09T06:02:05.037397+0000","last_peered":"2023-10-09T06:02:05.037397+0000","last_clean":"2023-10-09T06:02:05.037397+0000","last_became_active":"2023-10-09T06:01:41.939476+0000","last_became_peered":"2023-10-09T06:01:41.939476+0000","last_unstale":"2023-10-09T06:02:05.037397+0000","last_undegraded":"2023-10-09T06:02:05.037397+0000","last_fullsized":"2023-10-09T06:02:05.037397+0000","mapping_epoch":328,"log_start":"309'6515","ondisk_log_start":"309'6515","created":285,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5431,"log_dups_size":0,"ondisk_log_size":5431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:44:10.139800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68681728,"num_objects":1048,"num_object_clones":0,"num_object_copies":3144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1048,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11946,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":784,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.872102+0000","last_change":"2023-10-09T05:59:55.438018+0000","last_active":"2023-10-09T06:02:00.872102+0000","last_peered":"2023-10-09T06:02:00.872102+0000","last_clean":"2023-10-09T06:02:00.872102+0000","last_became_active":"2023-10-09T05:59:32.538631+0000","last_became_peered":"2023-10-09T05:59:32.538631+0000","last_unstale":"2023-10-09T06:02:00.872102+0000","last_undegraded":"2023-10-09T06:02:00.872102+0000","last_fullsized":"2023-10-09T06:02:00.872102+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:55.437947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:55.437947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:37.566205+0000","scrub_duration":0.0091262979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":124,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.812661+0000","last_change":"2023-10-09T06:01:22.355503+0000","last_active":"2023-10-09T06:02:00.812661+0000","last_peered":"2023-10-09T06:02:00.812661+0000","last_clean":"2023-10-09T06:02:00.812661+0000","last_became_active":"2023-10-09T06:01:22.354704+0000","last_became_peered":"2023-10-09T06:01:22.354704+0000","last_unstale":"2023-10-09T06:02:00.812661+0000","last_undegraded":"2023-10-09T06:02:00.812661+0000","last_fullsized":"2023-10-09T06:02:00.812661+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:56:21.446402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":295,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.810432+0000","last_change":"2023-10-09T06:01:22.332033+0000","last_active":"2023-10-09T06:02:00.810432+0000","last_peered":"2023-10-09T06:02:00.810432+0000","last_clean":"2023-10-09T06:02:00.810432+0000","last_became_active":"2023-10-09T06:01:22.329825+0000","last_became_peered":"2023-10-09T06:01:22.329825+0000","last_unstale":"2023-10-09T06:02:00.810432+0000","last_undegraded":"2023-10-09T06:02:00.810432+0000","last_fullsized":"2023-10-09T06:02:00.810432+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:54.046681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"341'12200","reported_seq":12266,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-09T06:02:02.711094+0000","last_change":"2023-10-09T05:58:27.405765+0000","last_active":"2023-10-09T06:02:02.711094+0000","last_peered":"2023-10-09T06:02:02.711094+0000","last_clean":"2023-10-09T06:02:02.711094+0000","last_became_active":"2023-10-09T05:58:27.405447+0000","last_became_peered":"2023-10-09T05:58:27.405447+0000","last_unstale":"2023-10-09T06:02:02.711094+0000","last_undegraded":"2023-10-09T06:02:02.711094+0000","last_fullsized":"2023-10-09T06:02:02.711094+0000","mapping_epoch":285,"log_start":"298'4600","ondisk_log_start":"298'4600","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":7600,"log_dups_size":0,"ondisk_log_size":7600,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:52.191564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73793536,"num_objects":1126,"num_object_clones":0,"num_object_copies":3378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1126,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12200,"num_write_kb":426432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":530,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.811049+0000","last_change":"2023-10-09T05:59:58.282543+0000","last_active":"2023-10-09T06:02:00.811049+0000","last_peered":"2023-10-09T06:02:00.811049+0000","last_clean":"2023-10-09T06:02:00.811049+0000","last_became_active":"2023-10-09T05:59:32.552960+0000","last_became_peered":"2023-10-09T05:59:32.552960+0000","last_unstale":"2023-10-09T06:02:00.811049+0000","last_undegraded":"2023-10-09T06:02:00.811049+0000","last_fullsized":"2023-10-09T06:02:00.811049+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.282393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.282393+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:10.201323+0000","scrub_duration":0.01402403,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":378,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.812559+0000","last_change":"2023-10-09T06:01:54.751188+0000","last_active":"2023-10-09T06:02:00.812559+0000","last_peered":"2023-10-09T06:02:00.812559+0000","last_clean":"2023-10-09T06:02:00.812559+0000","last_became_active":"2023-10-09T06:01:54.750870+0000","last_became_peered":"2023-10-09T06:01:54.750870+0000","last_unstale":"2023-10-09T06:02:00.812559+0000","last_undegraded":"2023-10-09T06:02:00.812559+0000","last_fullsized":"2023-10-09T06:02:00.812559+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:01.985974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":369,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.810154+0000","last_change":"2023-10-09T06:01:54.757667+0000","last_active":"2023-10-09T06:02:00.810154+0000","last_peered":"2023-10-09T06:02:00.810154+0000","last_clean":"2023-10-09T06:02:00.810154+0000","last_became_active":"2023-10-09T06:01:54.757367+0000","last_became_peered":"2023-10-09T06:01:54.757367+0000","last_unstale":"2023-10-09T06:02:00.810154+0000","last_undegraded":"2023-10-09T06:02:00.810154+0000","last_fullsized":"2023-10-09T06:02:00.810154+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:45.022566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"341'12107","reported_seq":13961,"reported_epoch":341,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T06:02:03.619371+0000","last_change":"2023-10-09T06:02:00.812333+0000","last_active":"2023-10-09T06:02:03.619371+0000","last_peered":"2023-10-09T06:02:03.619371+0000","last_clean":"2023-10-09T06:01:53.674923+0000","last_became_active":"2023-10-09T06:01:55.886238+0000","last_became_peered":"2023-10-09T06:01:55.886238+0000","last_unstale":"2023-10-09T06:02:03.619371+0000","last_undegraded":"2023-10-09T06:02:03.619371+0000","last_fullsized":"2023-10-09T06:02:03.619371+0000","mapping_epoch":334,"log_start":"307'6000","ondisk_log_start":"307'6000","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":6107,"log_dups_size":0,"ondisk_log_size":6107,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:52:15.708705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70647832,"num_objects":1079,"num_object_clones":0,"num_object_copies":3237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":240,"num_objects_unfound":0,"num_objects_dirty":1079,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12107,"num_write_kb":421889,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":888,"num_bytes_recovered":58130456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,0,3],"avail_no_missing":["6","0","3"],"object_location_counts":[{"shards":"0,3,6","objects":1079}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":619,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.805465+0000","last_change":"2023-10-09T06:01:57.796523+0000","last_active":"2023-10-09T06:02:00.805465+0000","last_peered":"2023-10-09T06:02:00.805465+0000","last_clean":"2023-10-09T06:02:00.805465+0000","last_became_active":"2023-10-09T06:01:57.796145+0000","last_became_peered":"2023-10-09T06:01:57.796145+0000","last_unstale":"2023-10-09T06:02:00.805465+0000","last_undegraded":"2023-10-09T06:02:00.805465+0000","last_fullsized":"2023-10-09T06:02:00.805465+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.373748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.373748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:52.076986+0000","scrub_duration":0.0024576450000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":311,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.809849+0000","last_change":"2023-10-09T06:01:22.334879+0000","last_active":"2023-10-09T06:02:00.809849+0000","last_peered":"2023-10-09T06:02:00.809849+0000","last_clean":"2023-10-09T06:02:00.809849+0000","last_became_active":"2023-10-09T06:01:22.334609+0000","last_became_peered":"2023-10-09T06:01:22.334609+0000","last_unstale":"2023-10-09T06:02:00.809849+0000","last_undegraded":"2023-10-09T06:02:00.809849+0000","last_fullsized":"2023-10-09T06:02:00.809849+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:36.094645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":352,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.810099+0000","last_change":"2023-10-09T06:01:54.757024+0000","last_active":"2023-10-09T06:02:00.810099+0000","last_peered":"2023-10-09T06:02:00.810099+0000","last_clean":"2023-10-09T06:02:00.810099+0000","last_became_active":"2023-10-09T06:01:54.756713+0000","last_became_peered":"2023-10-09T06:01:54.756713+0000","last_unstale":"2023-10-09T06:02:00.810099+0000","last_undegraded":"2023-10-09T06:02:00.810099+0000","last_fullsized":"2023-10-09T06:02:00.810099+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:26.229374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"341'11922","reported_seq":11986,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-09T06:02:03.581006+0000","last_change":"2023-10-09T05:58:27.376357+0000","last_active":"2023-10-09T06:02:03.581006+0000","last_peered":"2023-10-09T06:02:03.581006+0000","last_clean":"2023-10-09T06:02:03.581006+0000","last_became_active":"2023-10-09T05:58:27.376017+0000","last_became_peered":"2023-10-09T05:58:27.376017+0000","last_unstale":"2023-10-09T06:02:03.581006+0000","last_undegraded":"2023-10-09T06:02:03.581006+0000","last_fullsized":"2023-10-09T06:02:03.581006+0000","mapping_epoch":285,"log_start":"307'5800","ondisk_log_start":"307'5800","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":6122,"log_dups_size":0,"ondisk_log_size":6122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:30.067479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71958528,"num_objects":1098,"num_object_clones":0,"num_object_copies":3294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1098,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11922,"num_write_kb":416640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":494,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.812468+0000","last_change":"2023-10-09T05:59:52.129883+0000","last_active":"2023-10-09T06:02:00.812468+0000","last_peered":"2023-10-09T06:02:00.812468+0000","last_clean":"2023-10-09T06:02:00.812468+0000","last_became_active":"2023-10-09T05:59:32.529083+0000","last_became_peered":"2023-10-09T05:59:32.529083+0000","last_unstale":"2023-10-09T06:02:00.812468+0000","last_undegraded":"2023-10-09T06:02:00.812468+0000","last_fullsized":"2023-10-09T06:02:00.812468+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:52.129816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:52.129816+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:38.309831+0000","scrub_duration":0.0039938459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":129,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.811124+0000","last_change":"2023-10-09T06:01:54.761275+0000","last_active":"2023-10-09T06:02:00.811124+0000","last_peered":"2023-10-09T06:02:00.811124+0000","last_clean":"2023-10-09T06:02:00.811124+0000","last_became_active":"2023-10-09T06:01:54.760782+0000","last_became_peered":"2023-10-09T06:01:54.760782+0000","last_unstale":"2023-10-09T06:02:00.811124+0000","last_undegraded":"2023-10-09T06:02:00.811124+0000","last_fullsized":"2023-10-09T06:02:00.811124+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:28.119846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":18,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.806255+0000","last_change":"2023-10-09T06:01:54.761768+0000","last_active":"2023-10-09T06:02:00.806255+0000","last_peered":"2023-10-09T06:02:00.806255+0000","last_clean":"2023-10-09T06:02:00.806255+0000","last_became_active":"2023-10-09T06:01:54.761443+0000","last_became_peered":"2023-10-09T06:01:54.761443+0000","last_unstale":"2023-10-09T06:02:00.806255+0000","last_undegraded":"2023-10-09T06:02:00.806255+0000","last_fullsized":"2023-10-09T06:02:00.806255+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:51.555987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"341'12107","reported_seq":14453,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-09T06:02:05.397108+0000","last_change":"2023-10-09T06:02:00.868474+0000","last_active":"2023-10-09T06:02:05.397108+0000","last_peered":"2023-10-09T06:02:05.397108+0000","last_clean":"2023-10-09T06:02:05.397108+0000","last_became_active":"2023-10-09T06:02:00.813636+0000","last_became_peered":"2023-10-09T06:02:00.813636+0000","last_unstale":"2023-10-09T06:02:05.397108+0000","last_undegraded":"2023-10-09T06:02:05.397108+0000","last_fullsized":"2023-10-09T06:02:05.397108+0000","mapping_epoch":339,"log_start":"310'6749","ondisk_log_start":"310'6749","created":285,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5358,"log_dups_size":0,"ondisk_log_size":5358,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:04.801455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66781184,"num_objects":1019,"num_object_clones":0,"num_object_copies":3057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1019,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12107,"num_write_kb":420032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1126,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":764,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.813306+0000","last_change":"2023-10-09T05:59:56.092654+0000","last_active":"2023-10-09T06:02:00.813306+0000","last_peered":"2023-10-09T06:02:00.813306+0000","last_clean":"2023-10-09T06:02:00.813306+0000","last_became_active":"2023-10-09T05:59:32.529949+0000","last_became_peered":"2023-10-09T05:59:32.529949+0000","last_unstale":"2023-10-09T06:02:00.813306+0000","last_undegraded":"2023-10-09T06:02:00.813306+0000","last_fullsized":"2023-10-09T06:02:00.813306+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.092586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.092586+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:15:13.234903+0000","scrub_duration":0.0013484219999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.805732+0000","last_change":"2023-10-09T06:01:54.823231+0000","last_active":"2023-10-09T06:02:00.805732+0000","last_peered":"2023-10-09T06:02:00.805732+0000","last_clean":"2023-10-09T06:02:00.805732+0000","last_became_active":"2023-10-09T06:01:54.822755+0000","last_became_peered":"2023-10-09T06:01:54.822755+0000","last_unstale":"2023-10-09T06:02:00.805732+0000","last_undegraded":"2023-10-09T06:02:00.805732+0000","last_fullsized":"2023-10-09T06:02:00.805732+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:17.592052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":105,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.811692+0000","last_change":"2023-10-09T05:57:23.468480+0000","last_active":"2023-10-09T06:02:00.811692+0000","last_peered":"2023-10-09T06:02:00.811692+0000","last_clean":"2023-10-09T06:02:00.811692+0000","last_became_active":"2023-10-09T05:57:23.468179+0000","last_became_peered":"2023-10-09T05:57:23.468179+0000","last_unstale":"2023-10-09T06:02:00.811692+0000","last_undegraded":"2023-10-09T06:02:00.811692+0000","last_fullsized":"2023-10-09T06:02:00.811692+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:18.157188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":18,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.805126+0000","last_change":"2023-10-09T06:01:54.751385+0000","last_active":"2023-10-09T06:02:00.805126+0000","last_peered":"2023-10-09T06:02:00.805126+0000","last_clean":"2023-10-09T06:02:00.805126+0000","last_became_active":"2023-10-09T06:01:54.751041+0000","last_became_peered":"2023-10-09T06:01:54.751041+0000","last_unstale":"2023-10-09T06:02:00.805126+0000","last_undegraded":"2023-10-09T06:02:00.805126+0000","last_fullsized":"2023-10-09T06:02:00.805126+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:03.424836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"341'11712","reported_seq":11775,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-09T06:02:03.613311+0000","last_change":"2023-10-09T05:58:27.378776+0000","last_active":"2023-10-09T06:02:03.613311+0000","last_peered":"2023-10-09T06:02:03.613311+0000","last_clean":"2023-10-09T06:02:03.613311+0000","last_became_active":"2023-10-09T05:58:27.378504+0000","last_became_peered":"2023-10-09T05:58:27.378504+0000","last_unstale":"2023-10-09T06:02:03.613311+0000","last_undegraded":"2023-10-09T06:02:03.613311+0000","last_fullsized":"2023-10-09T06:02:03.613311+0000","mapping_epoch":285,"log_start":"307'5600","ondisk_log_start":"307'5600","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":6112,"log_dups_size":0,"ondisk_log_size":6112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:27.090596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74448896,"num_objects":1136,"num_object_clones":0,"num_object_copies":3408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1136,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11712,"num_write_kb":411136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":545,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.807979+0000","last_change":"2023-10-09T05:58:36.225081+0000","last_active":"2023-10-09T06:02:00.807979+0000","last_peered":"2023-10-09T06:02:00.807979+0000","last_clean":"2023-10-09T06:02:00.807979+0000","last_became_active":"2023-10-09T05:56:49.237269+0000","last_became_peered":"2023-10-09T05:56:49.237269+0000","last_unstale":"2023-10-09T06:02:00.807979+0000","last_undegraded":"2023-10-09T06:02:00.807979+0000","last_fullsized":"2023-10-09T06:02:00.807979+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:02.908583+0000","scrub_duration":0.0016660799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.806033+0000","last_change":"2023-10-09T06:01:54.754294+0000","last_active":"2023-10-09T06:02:00.806033+0000","last_peered":"2023-10-09T06:02:00.806033+0000","last_clean":"2023-10-09T06:02:00.806033+0000","last_became_active":"2023-10-09T06:01:54.754024+0000","last_became_peered":"2023-10-09T06:01:54.754024+0000","last_unstale":"2023-10-09T06:02:00.806033+0000","last_undegraded":"2023-10-09T06:02:00.806033+0000","last_fullsized":"2023-10-09T06:02:00.806033+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:19.746882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":304,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.809703+0000","last_change":"2023-10-09T06:01:22.335651+0000","last_active":"2023-10-09T06:02:00.809703+0000","last_peered":"2023-10-09T06:02:00.809703+0000","last_clean":"2023-10-09T06:02:00.809703+0000","last_became_active":"2023-10-09T06:01:22.334756+0000","last_became_peered":"2023-10-09T06:01:22.334756+0000","last_unstale":"2023-10-09T06:02:00.809703+0000","last_undegraded":"2023-10-09T06:02:00.809703+0000","last_fullsized":"2023-10-09T06:02:00.809703+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:33.123496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":18,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.805802+0000","last_change":"2023-10-09T06:01:54.760832+0000","last_active":"2023-10-09T06:02:00.805802+0000","last_peered":"2023-10-09T06:02:00.805802+0000","last_clean":"2023-10-09T06:02:00.805802+0000","last_became_active":"2023-10-09T06:01:54.760137+0000","last_became_peered":"2023-10-09T06:01:54.760137+0000","last_unstale":"2023-10-09T06:02:00.805802+0000","last_undegraded":"2023-10-09T06:02:00.805802+0000","last_fullsized":"2023-10-09T06:02:00.805802+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:19:58.776604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"341'12117","reported_seq":14856,"reported_epoch":341,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:02:04.805993+0000","last_change":"2023-10-09T06:02:01.853674+0000","last_active":"2023-10-09T06:02:04.805993+0000","last_peered":"2023-10-09T06:02:04.805993+0000","last_clean":"2023-10-09T06:00:59.244996+0000","last_became_active":"2023-10-09T06:02:01.823101+0000","last_became_peered":"2023-10-09T06:02:01.823101+0000","last_unstale":"2023-10-09T06:02:04.805993+0000","last_undegraded":"2023-10-09T06:02:04.805993+0000","last_fullsized":"2023-10-09T06:02:04.805993+0000","mapping_epoch":340,"log_start":"308'6200","ondisk_log_start":"308'6200","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5917,"log_dups_size":0,"ondisk_log_size":5917,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:01.623860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69271552,"num_objects":1057,"num_object_clones":0,"num_object_copies":3171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":871,"num_objects_unfound":0,"num_objects_dirty":1057,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12117,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1317,"num_bytes_recovered":14548992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,6","objects":1057}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":665,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.872226+0000","last_change":"2023-10-09T06:01:22.335906+0000","last_active":"2023-10-09T06:02:00.872226+0000","last_peered":"2023-10-09T06:02:00.872226+0000","last_clean":"2023-10-09T06:02:00.872226+0000","last_became_active":"2023-10-09T06:01:22.335590+0000","last_became_peered":"2023-10-09T06:01:22.335590+0000","last_unstale":"2023-10-09T06:02:00.872226+0000","last_undegraded":"2023-10-09T06:02:00.872226+0000","last_fullsized":"2023-10-09T06:02:00.872226+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:55.897812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:55.897812+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:02.771428+0000","scrub_duration":0.0014511420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":82,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.811867+0000","last_change":"2023-10-09T05:58:59.325257+0000","last_active":"2023-10-09T06:02:00.811867+0000","last_peered":"2023-10-09T06:02:00.811867+0000","last_clean":"2023-10-09T06:02:00.811867+0000","last_became_active":"2023-10-09T05:58:59.324993+0000","last_became_peered":"2023-10-09T05:58:59.324993+0000","last_unstale":"2023-10-09T06:02:00.811867+0000","last_undegraded":"2023-10-09T06:02:00.811867+0000","last_fullsized":"2023-10-09T06:02:00.811867+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:20.825840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":18,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.805053+0000","last_change":"2023-10-09T06:01:54.755425+0000","last_active":"2023-10-09T06:02:00.805053+0000","last_peered":"2023-10-09T06:02:00.805053+0000","last_clean":"2023-10-09T06:02:00.805053+0000","last_became_active":"2023-10-09T06:01:54.755129+0000","last_became_peered":"2023-10-09T06:01:54.755129+0000","last_unstale":"2023-10-09T06:02:00.805053+0000","last_undegraded":"2023-10-09T06:02:00.805053+0000","last_fullsized":"2023-10-09T06:02:00.805053+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:01:58.037849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":18,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.806173+0000","last_change":"2023-10-09T06:01:54.751198+0000","last_active":"2023-10-09T06:02:00.806173+0000","last_peered":"2023-10-09T06:02:00.806173+0000","last_clean":"2023-10-09T06:02:00.806173+0000","last_became_active":"2023-10-09T06:01:54.750862+0000","last_became_peered":"2023-10-09T06:01:54.750862+0000","last_unstale":"2023-10-09T06:02:00.806173+0000","last_undegraded":"2023-10-09T06:02:00.806173+0000","last_fullsized":"2023-10-09T06:02:00.806173+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:35.203567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"341'12178","reported_seq":12252,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-09T06:02:05.556830+0000","last_change":"2023-10-09T06:01:41.981434+0000","last_active":"2023-10-09T06:02:05.556830+0000","last_peered":"2023-10-09T06:02:05.556830+0000","last_clean":"2023-10-09T06:02:05.556830+0000","last_became_active":"2023-10-09T06:01:41.941614+0000","last_became_peered":"2023-10-09T06:01:41.941614+0000","last_unstale":"2023-10-09T06:02:05.556830+0000","last_undegraded":"2023-10-09T06:02:05.556830+0000","last_fullsized":"2023-10-09T06:02:05.556830+0000","mapping_epoch":328,"log_start":"308'6252","ondisk_log_start":"308'6252","created":285,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5926,"log_dups_size":0,"ondisk_log_size":5926,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:24.616533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68026368,"num_objects":1038,"num_object_clones":0,"num_object_copies":3114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12178,"num_write_kb":422912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":212,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.872172+0000","last_change":"2023-10-09T05:58:59.330109+0000","last_active":"2023-10-09T06:02:00.872172+0000","last_peered":"2023-10-09T06:02:00.872172+0000","last_clean":"2023-10-09T06:02:00.872172+0000","last_became_active":"2023-10-09T05:58:59.329808+0000","last_became_peered":"2023-10-09T05:58:59.329808+0000","last_unstale":"2023-10-09T06:02:00.872172+0000","last_undegraded":"2023-10-09T06:02:00.872172+0000","last_fullsized":"2023-10-09T06:02:00.872172+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:30.680622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":745,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.872114+0000","last_change":"2023-10-09T05:58:37.782237+0000","last_active":"2023-10-09T06:02:00.872114+0000","last_peered":"2023-10-09T06:02:00.872114+0000","last_clean":"2023-10-09T06:02:00.872114+0000","last_became_active":"2023-10-09T05:53:59.202004+0000","last_became_peered":"2023-10-09T05:53:59.202004+0000","last_unstale":"2023-10-09T06:02:00.872114+0000","last_undegraded":"2023-10-09T06:02:00.872114+0000","last_fullsized":"2023-10-09T06:02:00.872114+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:37.782173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:37.782173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:20.787963+0000","scrub_duration":0.001325908,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":105,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.811529+0000","last_change":"2023-10-09T05:57:23.479174+0000","last_active":"2023-10-09T06:02:00.811529+0000","last_peered":"2023-10-09T06:02:00.811529+0000","last_clean":"2023-10-09T06:02:00.811529+0000","last_became_active":"2023-10-09T05:57:23.478897+0000","last_became_peered":"2023-10-09T05:57:23.478897+0000","last_unstale":"2023-10-09T06:02:00.811529+0000","last_undegraded":"2023-10-09T06:02:00.811529+0000","last_fullsized":"2023-10-09T06:02:00.811529+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:00.160042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":320,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.820716+0000","last_change":"2023-10-09T06:01:22.340980+0000","last_active":"2023-10-09T06:02:00.820716+0000","last_peered":"2023-10-09T06:02:00.820716+0000","last_clean":"2023-10-09T06:02:00.820716+0000","last_became_active":"2023-10-09T06:01:22.340690+0000","last_became_peered":"2023-10-09T06:01:22.340690+0000","last_unstale":"2023-10-09T06:02:00.820716+0000","last_undegraded":"2023-10-09T06:02:00.820716+0000","last_fullsized":"2023-10-09T06:02:00.820716+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:45.161101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"341'12360","reported_seq":12424,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-09T06:02:05.560785+0000","last_change":"2023-10-09T05:58:27.375153+0000","last_active":"2023-10-09T06:02:05.560785+0000","last_peered":"2023-10-09T06:02:05.560785+0000","last_clean":"2023-10-09T06:02:05.560785+0000","last_became_active":"2023-10-09T05:58:27.374820+0000","last_became_peered":"2023-10-09T05:58:27.374820+0000","last_unstale":"2023-10-09T06:02:05.560785+0000","last_undegraded":"2023-10-09T06:02:05.560785+0000","last_fullsized":"2023-10-09T06:02:05.560785+0000","mapping_epoch":285,"log_start":"309'6500","ondisk_log_start":"309'6500","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5860,"log_dups_size":0,"ondisk_log_size":5860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:08.834399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71303168,"num_objects":1088,"num_object_clones":0,"num_object_copies":3264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12360,"num_write_kb":430336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":522,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.805361+0000","last_change":"2023-10-09T06:01:57.804327+0000","last_active":"2023-10-09T06:02:00.805361+0000","last_peered":"2023-10-09T06:02:00.805361+0000","last_clean":"2023-10-09T06:02:00.805361+0000","last_became_active":"2023-10-09T06:01:57.803970+0000","last_became_peered":"2023-10-09T06:01:57.803970+0000","last_unstale":"2023-10-09T06:02:00.805361+0000","last_undegraded":"2023-10-09T06:02:00.805361+0000","last_fullsized":"2023-10-09T06:02:00.805361+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:40.724331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:40.724331+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":217,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.872029+0000","last_change":"2023-10-09T06:01:54.751146+0000","last_active":"2023-10-09T06:02:00.872029+0000","last_peered":"2023-10-09T06:02:00.872029+0000","last_clean":"2023-10-09T06:02:00.872029+0000","last_became_active":"2023-10-09T06:01:54.750833+0000","last_became_peered":"2023-10-09T06:01:54.750833+0000","last_unstale":"2023-10-09T06:02:00.872029+0000","last_undegraded":"2023-10-09T06:02:00.872029+0000","last_fullsized":"2023-10-09T06:02:00.872029+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:54.035203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":294,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.808092+0000","last_change":"2023-10-09T06:01:22.320317+0000","last_active":"2023-10-09T06:02:00.808092+0000","last_peered":"2023-10-09T06:02:00.808092+0000","last_clean":"2023-10-09T06:02:00.808092+0000","last_became_active":"2023-10-09T06:01:22.319989+0000","last_became_peered":"2023-10-09T06:01:22.319989+0000","last_unstale":"2023-10-09T06:02:00.808092+0000","last_undegraded":"2023-10-09T06:02:00.808092+0000","last_fullsized":"2023-10-09T06:02:00.808092+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:06.219647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":405,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.808259+0000","last_change":"2023-10-09T06:01:54.752942+0000","last_active":"2023-10-09T06:02:00.808259+0000","last_peered":"2023-10-09T06:02:00.808259+0000","last_clean":"2023-10-09T06:02:00.808259+0000","last_became_active":"2023-10-09T06:01:54.752593+0000","last_became_peered":"2023-10-09T06:01:54.752593+0000","last_unstale":"2023-10-09T06:02:00.808259+0000","last_undegraded":"2023-10-09T06:02:00.808259+0000","last_fullsized":"2023-10-09T06:02:00.808259+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:10.005606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"341'12192","reported_seq":12268,"reported_epoch":341,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:02:02.452095+0000","last_change":"2023-10-09T06:02:00.390881+0000","last_active":"2023-10-09T06:02:02.452095+0000","last_peered":"2023-10-09T06:02:02.452095+0000","last_clean":"2023-10-09T06:01:53.717602+0000","last_became_active":"2023-10-09T06:01:55.880221+0000","last_became_peered":"2023-10-09T06:01:55.880221+0000","last_unstale":"2023-10-09T06:02:02.452095+0000","last_undegraded":"2023-10-09T06:02:02.452095+0000","last_fullsized":"2023-10-09T06:02:02.452095+0000","mapping_epoch":334,"log_start":"310'6905","ondisk_log_start":"310'6905","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5287,"log_dups_size":0,"ondisk_log_size":5287,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:18.588132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73793536,"num_objects":1126,"num_object_clones":0,"num_object_copies":3378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1126,"num_objects_unfound":0,"num_objects_dirty":1126,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12192,"num_write_kb":426176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[2,5,3],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":1126}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":754,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.813219+0000","last_change":"2023-10-09T05:59:54.074413+0000","last_active":"2023-10-09T06:02:00.813219+0000","last_peered":"2023-10-09T06:02:00.813219+0000","last_clean":"2023-10-09T06:02:00.813219+0000","last_became_active":"2023-10-09T05:59:32.538959+0000","last_became_peered":"2023-10-09T05:59:32.538959+0000","last_unstale":"2023-10-09T06:02:00.813219+0000","last_undegraded":"2023-10-09T06:02:00.813219+0000","last_fullsized":"2023-10-09T06:02:00.813219+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:54.074310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:54.074310+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:10.811269+0000","scrub_duration":0.011040648,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":192,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.872468+0000","last_change":"2023-10-09T06:01:22.337610+0000","last_active":"2023-10-09T06:02:00.872468+0000","last_peered":"2023-10-09T06:02:00.872468+0000","last_clean":"2023-10-09T06:02:00.872468+0000","last_became_active":"2023-10-09T06:01:22.336719+0000","last_became_peered":"2023-10-09T06:01:22.336719+0000","last_unstale":"2023-10-09T06:02:00.872468+0000","last_undegraded":"2023-10-09T06:02:00.872468+0000","last_fullsized":"2023-10-09T06:02:00.872468+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:33:29.587947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":18,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.805881+0000","last_change":"2023-10-09T06:01:54.756606+0000","last_active":"2023-10-09T06:02:00.805881+0000","last_peered":"2023-10-09T06:02:00.805881+0000","last_clean":"2023-10-09T06:02:00.805881+0000","last_became_active":"2023-10-09T06:01:54.755869+0000","last_became_peered":"2023-10-09T06:01:54.755869+0000","last_unstale":"2023-10-09T06:02:00.805881+0000","last_undegraded":"2023-10-09T06:02:00.805881+0000","last_fullsized":"2023-10-09T06:02:00.805881+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:29.672095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":113,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.872244+0000","last_change":"2023-10-09T06:01:41.938541+0000","last_active":"2023-10-09T06:02:00.872244+0000","last_peered":"2023-10-09T06:02:00.872244+0000","last_clean":"2023-10-09T06:02:00.872244+0000","last_became_active":"2023-10-09T06:01:41.938289+0000","last_became_peered":"2023-10-09T06:01:41.938289+0000","last_unstale":"2023-10-09T06:02:00.872244+0000","last_undegraded":"2023-10-09T06:02:00.872244+0000","last_fullsized":"2023-10-09T06:02:00.872244+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:29.735530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.3","version":"341'12070","reported_seq":14368,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-09T06:02:05.069789+0000","last_change":"2023-10-09T06:01:48.374012+0000","last_active":"2023-10-09T06:02:05.069789+0000","last_peered":"2023-10-09T06:02:05.069789+0000","last_clean":"2023-10-09T06:02:05.069789+0000","last_became_active":"2023-10-09T06:01:48.338444+0000","last_became_peered":"2023-10-09T06:01:48.338444+0000","last_unstale":"2023-10-09T06:02:05.069789+0000","last_undegraded":"2023-10-09T06:02:05.069789+0000","last_fullsized":"2023-10-09T06:02:05.069789+0000","mapping_epoch":331,"log_start":"310'6700","ondisk_log_start":"310'6700","created":285,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5370,"log_dups_size":0,"ondisk_log_size":5370,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:54.361781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70516736,"num_objects":1076,"num_object_clones":0,"num_object_copies":3228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12070,"num_write_kb":420672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1103,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":721,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.810295+0000","last_change":"2023-10-09T06:01:22.328323+0000","last_active":"2023-10-09T06:02:00.810295+0000","last_peered":"2023-10-09T06:02:00.810295+0000","last_clean":"2023-10-09T06:02:00.810295+0000","last_became_active":"2023-10-09T06:01:22.327941+0000","last_became_peered":"2023-10-09T06:01:22.327941+0000","last_unstale":"2023-10-09T06:02:00.810295+0000","last_undegraded":"2023-10-09T06:02:00.810295+0000","last_fullsized":"2023-10-09T06:02:00.810295+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:54.020712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:54.020712+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":348,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.812310+0000","last_change":"2023-10-09T05:53:59.228948+0000","last_active":"2023-10-09T06:02:00.812310+0000","last_peered":"2023-10-09T06:02:00.812310+0000","last_clean":"2023-10-09T06:02:00.812310+0000","last_became_active":"2023-10-09T05:53:59.186359+0000","last_became_peered":"2023-10-09T05:53:59.186359+0000","last_unstale":"2023-10-09T06:02:00.812310+0000","last_undegraded":"2023-10-09T06:02:00.812310+0000","last_fullsized":"2023-10-09T06:02:00.812310+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:39.990576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.806088+0000","last_change":"2023-10-09T06:01:54.753800+0000","last_active":"2023-10-09T06:02:00.806088+0000","last_peered":"2023-10-09T06:02:00.806088+0000","last_clean":"2023-10-09T06:02:00.806088+0000","last_became_active":"2023-10-09T06:01:54.751920+0000","last_became_peered":"2023-10-09T06:01:54.751920+0000","last_unstale":"2023-10-09T06:02:00.806088+0000","last_undegraded":"2023-10-09T06:02:00.806088+0000","last_fullsized":"2023-10-09T06:02:00.806088+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:45.359801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":105,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.811252+0000","last_change":"2023-10-09T05:57:23.477143+0000","last_active":"2023-10-09T06:02:00.811252+0000","last_peered":"2023-10-09T06:02:00.811252+0000","last_clean":"2023-10-09T06:02:00.811252+0000","last_became_active":"2023-10-09T05:57:23.475655+0000","last_became_peered":"2023-10-09T05:57:23.475655+0000","last_unstale":"2023-10-09T06:02:00.811252+0000","last_undegraded":"2023-10-09T06:02:00.811252+0000","last_fullsized":"2023-10-09T06:02:00.811252+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:29.528023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"341'12009","reported_seq":12084,"reported_epoch":341,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:02:05.037707+0000","last_change":"2023-10-09T06:01:55.947190+0000","last_active":"2023-10-09T06:02:05.037707+0000","last_peered":"2023-10-09T06:02:05.037707+0000","last_clean":"2023-10-09T06:01:53.726631+0000","last_became_active":"2023-10-09T06:01:55.885836+0000","last_became_peered":"2023-10-09T06:01:55.885836+0000","last_unstale":"2023-10-09T06:02:05.037707+0000","last_undegraded":"2023-10-09T06:02:05.037707+0000","last_fullsized":"2023-10-09T06:02:05.037707+0000","mapping_epoch":334,"log_start":"310'6600","ondisk_log_start":"310'6600","created":285,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:58:26.345875+0000","last_clean_scrub_stamp":"2023-10-09T05:58:26.345875+0000","objects_scrubbed":0,"log_size":5409,"log_dups_size":0,"ondisk_log_size":5409,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:07.329582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65994752,"num_objects":1007,"num_object_clones":0,"num_object_copies":3021,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1007,"num_objects_unfound":0,"num_objects_dirty":1007,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12009,"num_write_kb":416512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,2,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":1007}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":599,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.811422+0000","last_change":"2023-10-09T05:58:34.333129+0000","last_active":"2023-10-09T06:02:00.811422+0000","last_peered":"2023-10-09T06:02:00.811422+0000","last_clean":"2023-10-09T06:02:00.811422+0000","last_became_active":"2023-10-09T05:57:26.502911+0000","last_became_peered":"2023-10-09T05:57:26.502911+0000","last_unstale":"2023-10-09T06:02:00.811422+0000","last_undegraded":"2023-10-09T06:02:00.811422+0000","last_fullsized":"2023-10-09T06:02:00.811422+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:25.230712+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":345,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.813425+0000","last_change":"2023-10-09T05:57:58.832790+0000","last_active":"2023-10-09T06:02:00.813425+0000","last_peered":"2023-10-09T06:02:00.813425+0000","last_clean":"2023-10-09T06:02:00.813425+0000","last_became_active":"2023-10-09T05:57:58.832471+0000","last_became_peered":"2023-10-09T05:57:58.832471+0000","last_unstale":"2023-10-09T06:02:00.813425+0000","last_undegraded":"2023-10-09T06:02:00.813425+0000","last_fullsized":"2023-10-09T06:02:00.813425+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:39.358174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":410,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.808376+0000","last_change":"2023-10-09T06:01:54.776025+0000","last_active":"2023-10-09T06:02:00.808376+0000","last_peered":"2023-10-09T06:02:00.808376+0000","last_clean":"2023-10-09T06:02:00.808376+0000","last_became_active":"2023-10-09T06:01:54.753050+0000","last_became_peered":"2023-10-09T06:01:54.753050+0000","last_unstale":"2023-10-09T06:02:00.808376+0000","last_undegraded":"2023-10-09T06:02:00.808376+0000","last_fullsized":"2023-10-09T06:02:00.808376+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:58:24.000420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":287,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.810288+0000","last_change":"2023-10-09T06:01:22.334081+0000","last_active":"2023-10-09T06:02:00.810288+0000","last_peered":"2023-10-09T06:02:00.810288+0000","last_clean":"2023-10-09T06:02:00.810288+0000","last_became_active":"2023-10-09T06:01:22.333663+0000","last_became_peered":"2023-10-09T06:01:22.333663+0000","last_unstale":"2023-10-09T06:02:00.810288+0000","last_undegraded":"2023-10-09T06:02:00.810288+0000","last_fullsized":"2023-10-09T06:02:00.810288+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:55.809259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":567,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-09T06:02:01.813595+0000","last_change":"2023-10-09T06:00:06.181968+0000","last_active":"2023-10-09T06:02:01.813595+0000","last_peered":"2023-10-09T06:02:01.813595+0000","last_clean":"2023-10-09T06:02:01.813595+0000","last_became_active":"2023-10-09T05:59:40.324653+0000","last_became_peered":"2023-10-09T05:59:40.324653+0000","last_unstale":"2023-10-09T06:02:01.813595+0000","last_undegraded":"2023-10-09T06:02:01.813595+0000","last_fullsized":"2023-10-09T06:02:01.813595+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.181811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.181811+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:05:16.987523+0000","scrub_duration":0.011553313000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":286,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.808020+0000","last_change":"2023-10-09T06:01:22.349718+0000","last_active":"2023-10-09T06:02:00.808020+0000","last_peered":"2023-10-09T06:02:00.808020+0000","last_clean":"2023-10-09T06:02:00.808020+0000","last_became_active":"2023-10-09T06:01:22.349401+0000","last_became_peered":"2023-10-09T06:01:22.349401+0000","last_unstale":"2023-10-09T06:02:00.808020+0000","last_undegraded":"2023-10-09T06:02:00.808020+0000","last_fullsized":"2023-10-09T06:02:00.808020+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:08.983715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":18,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.805955+0000","last_change":"2023-10-09T06:01:54.761007+0000","last_active":"2023-10-09T06:02:00.805955+0000","last_peered":"2023-10-09T06:02:00.805955+0000","last_clean":"2023-10-09T06:02:00.805955+0000","last_became_active":"2023-10-09T06:01:54.760538+0000","last_became_peered":"2023-10-09T06:01:54.760538+0000","last_unstale":"2023-10-09T06:02:00.805955+0000","last_undegraded":"2023-10-09T06:02:00.805955+0000","last_fullsized":"2023-10-09T06:02:00.805955+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:08.772179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":684,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.811553+0000","last_change":"2023-10-09T06:00:04.215425+0000","last_active":"2023-10-09T06:02:00.811553+0000","last_peered":"2023-10-09T06:02:00.811553+0000","last_clean":"2023-10-09T06:02:00.811553+0000","last_became_active":"2023-10-09T05:59:40.326685+0000","last_became_peered":"2023-10-09T05:59:40.326685+0000","last_unstale":"2023-10-09T06:02:00.811553+0000","last_undegraded":"2023-10-09T06:02:00.811553+0000","last_fullsized":"2023-10-09T06:02:00.811553+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:04.215270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:04.215270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:03.963694+0000","scrub_duration":0.002850652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":112,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.810348+0000","last_change":"2023-10-09T06:01:54.824780+0000","last_active":"2023-10-09T06:02:00.810348+0000","last_peered":"2023-10-09T06:02:00.810348+0000","last_clean":"2023-10-09T06:02:00.810348+0000","last_became_active":"2023-10-09T06:01:54.757727+0000","last_became_peered":"2023-10-09T06:01:54.757727+0000","last_unstale":"2023-10-09T06:02:00.810348+0000","last_undegraded":"2023-10-09T06:02:00.810348+0000","last_fullsized":"2023-10-09T06:02:00.810348+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:44.132556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":359,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.809953+0000","last_change":"2023-10-09T06:01:54.764989+0000","last_active":"2023-10-09T06:02:00.809953+0000","last_peered":"2023-10-09T06:02:00.809953+0000","last_clean":"2023-10-09T06:02:00.809953+0000","last_became_active":"2023-10-09T06:01:54.764790+0000","last_became_peered":"2023-10-09T06:01:54.764790+0000","last_unstale":"2023-10-09T06:02:00.809953+0000","last_undegraded":"2023-10-09T06:02:00.809953+0000","last_fullsized":"2023-10-09T06:02:00.809953+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:14:07.208858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":98,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.809896+0000","last_change":"2023-10-09T06:01:22.342190+0000","last_active":"2023-10-09T06:02:00.809896+0000","last_peered":"2023-10-09T06:02:00.809896+0000","last_clean":"2023-10-09T06:02:00.809896+0000","last_became_active":"2023-10-09T06:01:22.341855+0000","last_became_peered":"2023-10-09T06:01:22.341855+0000","last_unstale":"2023-10-09T06:02:00.809896+0000","last_undegraded":"2023-10-09T06:02:00.809896+0000","last_fullsized":"2023-10-09T06:02:00.809896+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:01:21.322371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":329,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.809676+0000","last_change":"2023-10-09T06:01:22.326786+0000","last_active":"2023-10-09T06:02:00.809676+0000","last_peered":"2023-10-09T06:02:00.809676+0000","last_clean":"2023-10-09T06:02:00.809676+0000","last_became_active":"2023-10-09T06:01:22.326474+0000","last_became_peered":"2023-10-09T06:01:22.326474+0000","last_unstale":"2023-10-09T06:02:00.809676+0000","last_undegraded":"2023-10-09T06:02:00.809676+0000","last_fullsized":"2023-10-09T06:02:00.809676+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:17:48.846619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":607,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.872389+0000","last_change":"2023-10-09T06:01:57.801991+0000","last_active":"2023-10-09T06:02:00.872389+0000","last_peered":"2023-10-09T06:02:00.872389+0000","last_clean":"2023-10-09T06:02:00.872389+0000","last_became_active":"2023-10-09T06:01:57.801579+0000","last_became_peered":"2023-10-09T06:01:57.801579+0000","last_unstale":"2023-10-09T06:02:00.872389+0000","last_undegraded":"2023-10-09T06:02:00.872389+0000","last_fullsized":"2023-10-09T06:02:00.872389+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:00.456132+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:00.456132+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:48.548353+0000","scrub_duration":0.0014992199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":105,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.811887+0000","last_change":"2023-10-09T05:57:23.479560+0000","last_active":"2023-10-09T06:02:00.811887+0000","last_peered":"2023-10-09T06:02:00.811887+0000","last_clean":"2023-10-09T06:02:00.811887+0000","last_became_active":"2023-10-09T05:57:23.479177+0000","last_became_peered":"2023-10-09T05:57:23.479177+0000","last_unstale":"2023-10-09T06:02:00.811887+0000","last_undegraded":"2023-10-09T06:02:00.811887+0000","last_fullsized":"2023-10-09T06:02:00.811887+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:41.925161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":104,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.811846+0000","last_change":"2023-10-09T05:57:23.475179+0000","last_active":"2023-10-09T06:02:00.811846+0000","last_peered":"2023-10-09T06:02:00.811846+0000","last_clean":"2023-10-09T06:02:00.811846+0000","last_became_active":"2023-10-09T05:57:23.474904+0000","last_became_peered":"2023-10-09T05:57:23.474904+0000","last_unstale":"2023-10-09T06:02:00.811846+0000","last_undegraded":"2023-10-09T06:02:00.811846+0000","last_fullsized":"2023-10-09T06:02:00.811846+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:54.587601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":612,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.805736+0000","last_change":"2023-10-09T06:01:57.817554+0000","last_active":"2023-10-09T06:02:00.805736+0000","last_peered":"2023-10-09T06:02:00.805736+0000","last_clean":"2023-10-09T06:02:00.805736+0000","last_became_active":"2023-10-09T06:01:57.817191+0000","last_became_peered":"2023-10-09T06:01:57.817191+0000","last_unstale":"2023-10-09T06:02:00.805736+0000","last_undegraded":"2023-10-09T06:02:00.805736+0000","last_fullsized":"2023-10-09T06:02:00.805736+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:07.172878+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:07.172878+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":309,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.808806+0000","last_change":"2023-10-09T05:57:10.379258+0000","last_active":"2023-10-09T06:02:00.808806+0000","last_peered":"2023-10-09T06:02:00.808806+0000","last_clean":"2023-10-09T06:02:00.808806+0000","last_became_active":"2023-10-09T05:57:10.359745+0000","last_became_peered":"2023-10-09T05:57:10.359745+0000","last_unstale":"2023-10-09T06:02:00.808806+0000","last_undegraded":"2023-10-09T06:02:00.808806+0000","last_fullsized":"2023-10-09T06:02:00.808806+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:14.750256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":352,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:02:00.809439+0000","last_change":"2023-10-09T06:01:54.757112+0000","last_active":"2023-10-09T06:02:00.809439+0000","last_peered":"2023-10-09T06:02:00.809439+0000","last_clean":"2023-10-09T06:02:00.809439+0000","last_became_active":"2023-10-09T06:01:54.756793+0000","last_became_peered":"2023-10-09T06:01:54.756793+0000","last_unstale":"2023-10-09T06:02:00.809439+0000","last_undegraded":"2023-10-09T06:02:00.809439+0000","last_fullsized":"2023-10-09T06:02:00.809439+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:31.657029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":1124532248,"num_objects":17160,"num_object_clones":0,"num_object_copies":51480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5495,"num_objects_unfound":0,"num_objects_dirty":17160,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":193542,"num_write_kb":6742401,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6760,"num_bytes_recovered":72679448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3431284736,"data_stored":3431268448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":93721,"ondisk_log_size":93721,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":1324,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10938,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":746905,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":130,"num_read_kb":243,"num_write":63,"num_write_kb":641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":469612,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234269,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":766780,"kb_used_data":469492,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":92978372,"statfs":{"total":95995035648,"available":95209852928,"internally_reserved":0,"allocated":480759808,"data_stored":479663888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:01:05 2023","interfaces":[{"interface":"back","average":{"1min":9.7439999999999998,"5min":5.6040000000000001,"15min":2.968},"min":{"1min":0.23400000000000001,"5min":0.22500000000000001,"15min":0.22},"max":{"1min":165.792,"5min":165.792,"15min":165.792},"last":0.317},{"interface":"front","average":{"1min":10.246,"5min":5.6920000000000002,"15min":2.9430000000000001},"min":{"1min":0.23899999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":166.00700000000001,"5min":166.00700000000001,"15min":166.00700000000001},"last":2.0830000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:01:40 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":1.7010000000000001,"15min":1.087},"min":{"1min":0.309,"5min":0.249,"15min":0.22900000000000001},"max":{"1min":2.4489999999999998,"5min":35.113999999999997,"15min":35.113999999999997},"last":1.6379999999999999},{"interface":"front","average":{"1min":0.98599999999999999,"5min":1.4330000000000001,"15min":0.95999999999999996},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":3.528,"5min":34.768999999999998,"15min":34.768999999999998},"last":1.9950000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:01:05 2023","interfaces":[{"interface":"back","average":{"1min":8.6890000000000001,"5min":5.423,"15min":2.782},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":127.86499999999999,"5min":127.86499999999999,"15min":127.86499999999999},"last":2.3109999999999999},{"interface":"front","average":{"1min":8.4410000000000007,"5min":5.1980000000000004,"15min":2.7410000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":129.24000000000001,"5min":129.24000000000001,"15min":129.24000000000001},"last":2.5569999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:01:40 2023","interfaces":[{"interface":"back","average":{"1min":11.577,"5min":9.9809999999999999,"15min":5.6239999999999997},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33300000000000002},"max":{"1min":164.16999999999999,"5min":164.16999999999999,"15min":164.16999999999999},"last":1.871},{"interface":"front","average":{"1min":11.983000000000001,"5min":9.8729999999999993,"15min":5.5190000000000001},"min":{"1min":0.48299999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":164.40700000000001,"5min":164.40700000000001,"15min":164.40700000000001},"last":2.2629999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:00:20 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":0.85299999999999998,"15min":0.67800000000000005},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.24399999999999999},"max":{"1min":11.635,"5min":11.635,"15min":11.635},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.5069999999999999,"5min":0.85699999999999998,"15min":0.753},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":6.085,"5min":6.085,"15min":7.6120000000000001},"last":0.58499999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:01:23 2023","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":1.0009999999999999,"15min":0.81499999999999995},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27500000000000002},"max":{"1min":2.6219999999999999,"5min":15.622,"15min":15.622},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.80200000000000005,"15min":0.69299999999999995},"min":{"1min":0.309,"5min":0.309,"15min":0.30499999999999999},"max":{"1min":4.133,"5min":4.133,"15min":4.133},"last":1.899}]},{"osd":6,"last update":"Mon Oct 9 06:01:23 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.94599999999999995,"15min":1.3320000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.4220000000000002,"5min":12.727,"15min":131.94399999999999},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.879,"5min":0.91700000000000004,"15min":1.3480000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.29999999999999999},"max":{"1min":5.4870000000000001,"5min":5.4870000000000001,"15min":135.23400000000001},"last":0.47799999999999998}]}]},{"osd":6,"up_from":46,"seq":197568495905,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":729388,"kb_used_data":431972,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93015764,"statfs":{"total":95995035648,"available":95248142336,"internally_reserved":0,"allocated":442339328,"data_stored":441258861,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:01:06 2023","interfaces":[{"interface":"back","average":{"1min":12.595000000000001,"5min":4.1280000000000001,"15min":2.048},"min":{"1min":0.22900000000000001,"5min":0.224,"15min":0.224},"max":{"1min":165.124,"5min":165.124,"15min":165.124},"last":3.302},{"interface":"front","average":{"1min":12.16,"5min":4.2709999999999999,"15min":2.1280000000000001},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":165.221,"5min":165.221,"15min":165.221},"last":0.45400000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:01:47 2023","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":1.2649999999999999,"15min":0.92500000000000004},"min":{"1min":0.27000000000000002,"5min":0.246,"15min":0.23899999999999999},"max":{"1min":3.3919999999999999,"5min":17.341000000000001,"15min":17.341000000000001},"last":1.7929999999999999},{"interface":"front","average":{"1min":1.327,"5min":1.7130000000000001,"15min":1.1040000000000001},"min":{"1min":0.315,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.5579999999999998,"5min":22.445,"15min":22.445},"last":0.36499999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:01:06 2023","interfaces":[{"interface":"back","average":{"1min":1.5900000000000001,"5min":2.7709999999999999,"15min":1.49},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":5.1349999999999998,"5min":61.088999999999999,"15min":61.088999999999999},"last":4.5720000000000001},{"interface":"front","average":{"1min":1.841,"5min":2.7930000000000001,"15min":1.504},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.25600000000000001},"max":{"1min":8.4659999999999993,"5min":61.082000000000001,"15min":61.082000000000001},"last":1.0740000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:01:47 2023","interfaces":[{"interface":"back","average":{"1min":14.416,"5min":6.3220000000000001,"15min":3.5640000000000001},"min":{"1min":0.307,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":86.099000000000004,"5min":86.099000000000004,"15min":93.626000000000005},"last":1.343},{"interface":"front","average":{"1min":14.617000000000001,"5min":6.3079999999999998,"15min":3.468},"min":{"1min":0.35599999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":83.126999999999995,"5min":83.126999999999995,"15min":93.643000000000001},"last":3.4540000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:00:52 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.99199999999999999,"15min":0.80600000000000005},"min":{"1min":0.29399999999999998,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":3.6429999999999998,"5min":5.1070000000000002,"15min":7.9100000000000001},"last":0.32900000000000001},{"interface":"front","average":{"1min":1.079,"5min":1.0449999999999999,"15min":0.82499999999999996},"min":{"1min":0.311,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.085,"5min":8.734,"15min":8.734},"last":0.39000000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:01:56 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.024,"15min":1.002},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":4.5609999999999999,"5min":5.5910000000000002,"15min":45.933999999999997},"last":0.498},{"interface":"front","average":{"1min":0.92200000000000004,"5min":0.94299999999999995,"15min":1.0209999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":4.415,"5min":4.976,"15min":45.917999999999999},"last":0.48199999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:01:18 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":1.6240000000000001,"15min":1.1479999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.843,"5min":52.042999999999999,"15min":52.042999999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.65800000000000003,"5min":1.3959999999999999,"15min":1.0589999999999999},"min":{"1min":0.33700000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.1699999999999999,"5min":51.259,"15min":51.259},"last":0.51900000000000002}]}]},{"osd":1,"up_from":89,"seq":382252089494,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":271368,"kb_used_data":74432,"kb_used_omap":9,"kb_used_meta":196918,"kb_avail":93473784,"statfs":{"total":95995035648,"available":95717154816,"internally_reserved":0,"allocated":76218368,"data_stored":74233020,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":201645033},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:01:10 2023","interfaces":[{"interface":"back","average":{"1min":10.433999999999999,"5min":3.5840000000000001,"15min":1.53},"min":{"1min":0.218,"5min":0.191,"15min":0.191},"max":{"1min":132.04499999999999,"5min":132.04499999999999,"15min":132.04499999999999},"last":5.8630000000000004},{"interface":"front","average":{"1min":10.818,"5min":3.831,"15min":1.9039999999999999},"min":{"1min":0.25700000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":132.91200000000001,"5min":132.91200000000001,"15min":132.91200000000001},"last":5.8300000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:01:22 2023","interfaces":[{"interface":"back","average":{"1min":7.04,"5min":7.1740000000000004,"15min":2.8879999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.20100000000000001},"max":{"1min":129.053,"5min":204.989,"15min":204.989},"last":1.7789999999999999},{"interface":"front","average":{"1min":7.3780000000000001,"5min":7.1589999999999998,"15min":3.0310000000000001},"min":{"1min":0.222,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":129.077,"5min":203.65899999999999,"15min":203.65899999999999},"last":8.3919999999999995}]},{"osd":3,"last update":"Mon Oct 9 06:01:42 2023","interfaces":[{"interface":"back","average":{"1min":6.5730000000000004,"5min":6.4489999999999998,"15min":4.556},"min":{"1min":0.27600000000000002,"5min":0.26000000000000001,"15min":0.214},"max":{"1min":72.536000000000001,"5min":213.654,"15min":284.06099999999998},"last":5.8979999999999997},{"interface":"front","average":{"1min":6.2359999999999998,"5min":6.3159999999999998,"15min":4.6269999999999998},"min":{"1min":0.30099999999999999,"5min":0.28100000000000003,"15min":0.27900000000000003},"max":{"1min":68.575999999999993,"5min":213.77199999999999,"15min":284.97300000000001},"last":4.6459999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:00:32 2023","interfaces":[{"interface":"back","average":{"1min":1.7669999999999999,"5min":1.9750000000000001,"15min":1.2},"min":{"1min":0.33100000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.6340000000000003,"5min":61.183999999999997,"15min":61.183999999999997},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.2210000000000001,"5min":1.7729999999999999,"15min":1.0169999999999999},"min":{"1min":0.40500000000000003,"5min":0.315,"15min":0.29099999999999998},"max":{"1min":5.391,"5min":61.162999999999997,"15min":61.162999999999997},"last":2.504}]},{"osd":5,"last update":"Mon Oct 9 06:01:36 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":1.02,"15min":1.129},"min":{"1min":0.34200000000000003,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.7649999999999997,"5min":6.2949999999999999,"15min":64.584999999999994},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.93000000000000005,"5min":1.244,"15min":1.2310000000000001},"min":{"1min":0.34300000000000003,"5min":0.315,"15min":0.315},"max":{"1min":3.9710000000000001,"5min":8.9369999999999994,"15min":64.150999999999996},"last":2.573}]},{"osd":6,"last update":"Mon Oct 9 06:01:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":1.169,"15min":1.8180000000000001},"min":{"1min":0.318,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.8620000000000001,"5min":6.3200000000000003,"15min":204.62700000000001},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.143,"5min":1.415,"15min":1.925},"min":{"1min":0.38300000000000001,"5min":0.36599999999999999,"15min":0.35499999999999998},"max":{"1min":4.1980000000000004,"5min":7.7270000000000003,"15min":204.285},"last":2.544}]},{"osd":7,"last update":"Mon Oct 9 06:01:36 2023","interfaces":[{"interface":"back","average":{"1min":5.9279999999999999,"5min":2.3839999999999999,"15min":1.694},"min":{"1min":0.41899999999999998,"5min":0.314,"15min":0.314},"max":{"1min":108.553,"5min":108.553,"15min":108.553},"last":2.6000000000000001},{"interface":"front","average":{"1min":5.657,"5min":2.2130000000000001,"15min":1.4790000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38},"max":{"1min":108.56999999999999,"5min":108.56999999999999,"15min":108.56999999999999},"last":1.49}]}]},{"osd":0,"up_from":129,"seq":554050781296,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":792912,"kb_used_data":549960,"kb_used_omap":29,"kb_used_meta":242914,"kb_avail":92952240,"statfs":{"total":95995035648,"available":95183093760,"internally_reserved":0,"allocated":563159040,"data_stored":561285064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30322,"internal_metadata":248744334},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:01:14 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":1.0449999999999999,"15min":0.85499999999999998},"min":{"1min":0.26500000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":5.4580000000000002,"5min":8.3300000000000001,"15min":9.1720000000000006},"last":0.71199999999999997},{"interface":"front","average":{"1min":1.1719999999999999,"5min":1.0529999999999999,"15min":1.0409999999999999},"min":{"1min":0.32000000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":3.6379999999999999,"5min":6.9470000000000001,"15min":6.9470000000000001},"last":1.5149999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:01:14 2023","interfaces":[{"interface":"back","average":{"1min":1.9730000000000001,"5min":1.7110000000000001,"15min":1.1890000000000001},"min":{"1min":0.32900000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.4779999999999998,"5min":13.471,"15min":13.471},"last":1.704},{"interface":"front","average":{"1min":4.5830000000000002,"5min":2.165,"15min":1.3460000000000001},"min":{"1min":0.313,"5min":0.224,"15min":0.224},"max":{"1min":18.018000000000001,"5min":18.018000000000001,"15min":18.018000000000001},"last":14.172000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:01:14 2023","interfaces":[{"interface":"back","average":{"1min":2.9399999999999999,"5min":6.2910000000000004,"15min":2.6219999999999999},"min":{"1min":0.37,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":14.331,"5min":134.185,"15min":134.185},"last":0.81100000000000005},{"interface":"front","average":{"1min":4.4340000000000002,"5min":6.4359999999999999,"15min":2.7269999999999999},"min":{"1min":0.34599999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":15.811,"5min":134.16800000000001,"15min":134.16800000000001},"last":1.444}]},{"osd":4,"last update":"Mon Oct 9 06:00:10 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":3.9740000000000002,"15min":2.101},"min":{"1min":0.29999999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.9929999999999999,"5min":149.07400000000001,"15min":149.07400000000001},"last":1.4790000000000001},{"interface":"front","average":{"1min":1.5349999999999999,"5min":4.0700000000000003,"15min":1.9259999999999999},"min":{"1min":0.34000000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.8979999999999997,"5min":149.12700000000001,"15min":149.12700000000001},"last":0.40300000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:01:14 2023","interfaces":[{"interface":"back","average":{"1min":3.1819999999999999,"5min":3.3199999999999998,"15min":1.8129999999999999},"min":{"1min":0.46200000000000002,"5min":0.314,"15min":0.314},"max":{"1min":9.7780000000000005,"5min":100.25,"15min":100.25},"last":1.5049999999999999},{"interface":"front","average":{"1min":1.1899999999999999,"5min":2.9249999999999998,"15min":1.579},"min":{"1min":0.379,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":5.2199999999999998,"5min":102.40300000000001,"15min":102.40300000000001},"last":0.442}]},{"osd":6,"last update":"Mon Oct 9 06:01:14 2023","interfaces":[{"interface":"back","average":{"1min":3.597,"5min":1.661,"15min":1.3320000000000001},"min":{"1min":0.36699999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":9.8059999999999992,"5min":9.8059999999999992,"15min":9.8059999999999992},"last":1.532},{"interface":"front","average":{"1min":1.117,"5min":1.3360000000000001,"15min":1.1970000000000001},"min":{"1min":0.41599999999999998,"5min":0.38100000000000001,"15min":0.378},"max":{"1min":5.1950000000000003,"5min":8.0510000000000002,"15min":9.141},"last":0.49199999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:01:14 2023","interfaces":[{"interface":"back","average":{"1min":1.252,"5min":3.024,"15min":1.879},"min":{"1min":0.45000000000000001,"5min":0.42399999999999999,"15min":0.41999999999999998},"max":{"1min":5.1669999999999998,"5min":103.866,"15min":103.866},"last":0.51500000000000001},{"interface":"front","average":{"1min":1.325,"5min":2.96,"15min":1.819},"min":{"1min":0.42199999999999999,"5min":0.378,"15min":0.378},"max":{"1min":4.0460000000000003,"5min":103.848,"15min":103.848},"last":0.54600000000000004}]}]},{"osd":2,"up_from":94,"seq":403726925985,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1060812,"kb_used_data":593220,"kb_used_omap":81,"kb_used_meta":467310,"kb_avail":92684340,"statfs":{"total":95995035648,"available":94908764160,"internally_reserved":0,"allocated":607457280,"data_stored":605701452,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":478525658},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:01:18 2023","interfaces":[{"interface":"back","average":{"1min":6.6710000000000003,"5min":4.3079999999999998,"15min":2.3690000000000002},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20399999999999999},"max":{"1min":70.721999999999994,"5min":70.721999999999994,"15min":70.721999999999994},"last":48.720999999999997},{"interface":"front","average":{"1min":6.3600000000000003,"5min":4.4969999999999999,"15min":2.355},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":72.382999999999996,"5min":72.382999999999996,"15min":72.382999999999996},"last":48.518000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:01:09 2023","interfaces":[{"interface":"back","average":{"1min":1.9650000000000001,"5min":1.391,"15min":1.5820000000000001},"min":{"1min":0.27900000000000003,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.1219999999999999,"5min":9.8949999999999996,"15min":163.24199999999999},"last":0.34599999999999997},{"interface":"front","average":{"1min":1.524,"5min":1.5529999999999999,"15min":1.5569999999999999},"min":{"1min":0.32100000000000001,"5min":0.223,"15min":0.214},"max":{"1min":4.6900000000000004,"5min":12.278,"15min":163.27000000000001},"last":0.439}]},{"osd":3,"last update":"Mon Oct 9 06:01:41 2023","interfaces":[{"interface":"back","average":{"1min":5.9459999999999997,"5min":5.4939999999999998,"15min":3.0009999999999999},"min":{"1min":0.26000000000000001,"5min":0.24099999999999999,"15min":0.23899999999999999},"max":{"1min":69.390000000000001,"5min":144.88200000000001,"15min":170.07599999999999},"last":0.36199999999999999},{"interface":"front","average":{"1min":5.1790000000000003,"5min":5.29,"15min":2.5339999999999998},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25},"max":{"1min":69.337999999999994,"5min":140.79499999999999,"15min":140.79499999999999},"last":0.41899999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:00:07 2023","interfaces":[{"interface":"back","average":{"1min":1.4630000000000001,"5min":1.2070000000000001,"15min":1.0189999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28199999999999997},"max":{"1min":5.7690000000000001,"5min":8.6280000000000001,"15min":11.372999999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":2.7730000000000001,"5min":1.3939999999999999,"15min":1.0629999999999999},"min":{"1min":0.379,"5min":0.33200000000000002,"15min":0.26800000000000002},"max":{"1min":21.477,"5min":21.477,"15min":21.477},"last":0.53300000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:01:09 2023","interfaces":[{"interface":"back","average":{"1min":1.415,"5min":1.621,"15min":1.1699999999999999},"min":{"1min":0.38500000000000001,"5min":0.30599999999999999,"15min":0.28999999999999998},"max":{"1min":4.5030000000000001,"5min":8.8819999999999997,"15min":13.414},"last":0.46999999999999997},{"interface":"front","average":{"1min":1.8720000000000001,"5min":1.502,"15min":1.135},"min":{"1min":0.35299999999999998,"5min":0.313,"15min":0.29799999999999999},"max":{"1min":7.6900000000000004,"5min":8.5570000000000004,"15min":13.438000000000001},"last":0.57499999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:01:09 2023","interfaces":[{"interface":"back","average":{"1min":1.8779999999999999,"5min":1.591,"15min":1.1739999999999999},"min":{"1min":0.35299999999999998,"5min":0.31900000000000001,"15min":0.29399999999999998},"max":{"1min":6.3979999999999997,"5min":21.524999999999999,"15min":21.524999999999999},"last":0.503},{"interface":"front","average":{"1min":1.8640000000000001,"5min":1.6799999999999999,"15min":1.1120000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":9.9540000000000006,"5min":9.9540000000000006,"15min":9.9540000000000006},"last":0.58899999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:01:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":2.1920000000000002,"15min":1.4750000000000001},"min":{"1min":0.41399999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":4.6219999999999999,"5min":55.052,"15min":55.052},"last":0.55000000000000004},{"interface":"front","average":{"1min":1.6319999999999999,"5min":1.5740000000000001,"15min":1.2350000000000001},"min":{"1min":0.38500000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":7.6619999999999999,"5min":8.6010000000000009,"15min":40.113},"last":0.61099999999999999}]}]},{"osd":3,"up_from":99,"seq":425201762459,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1144620,"kb_used_data":673700,"kb_used_omap":7,"kb_used_meta":470840,"kb_avail":92600532,"statfs":{"total":95995035648,"available":94822944768,"internally_reserved":0,"allocated":689868800,"data_stored":687904711,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":482140776},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:01:14 2023","interfaces":[{"interface":"back","average":{"1min":8.8699999999999992,"5min":2.8109999999999999,"15min":1.865},"min":{"1min":0.20499999999999999,"5min":0.186,"15min":0.186},"max":{"1min":109.976,"5min":109.976,"15min":109.976},"last":4.8019999999999996},{"interface":"front","average":{"1min":9.157,"5min":3.125,"15min":2.0379999999999998},"min":{"1min":0.23100000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":107.161,"5min":107.161,"15min":107.161},"last":1.865}]},{"osd":1,"last update":"Mon Oct 9 06:01:46 2023","interfaces":[{"interface":"back","average":{"1min":2.5329999999999999,"5min":1.714,"15min":1.484},"min":{"1min":0.23699999999999999,"5min":0.217,"15min":0.20999999999999999},"max":{"1min":26.538,"5min":26.538,"15min":127.226},"last":9.609},{"interface":"front","average":{"1min":2.9540000000000002,"5min":2.806,"15min":1.871},"min":{"1min":0.28899999999999998,"5min":0.248,"15min":0.21099999999999999},"max":{"1min":7.2990000000000004,"5min":106.95699999999999,"15min":127.468},"last":5.0609999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:01:46 2023","interfaces":[{"interface":"back","average":{"1min":2.657,"5min":3.742,"15min":1.855},"min":{"1min":0.29399999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.6870000000000003,"5min":107.01300000000001,"15min":107.01300000000001},"last":9.577},{"interface":"front","average":{"1min":2.0640000000000001,"5min":3.7349999999999999,"15min":2.0489999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":5.8440000000000003,"5min":109.807,"15min":109.807},"last":1.5369999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:00:44 2023","interfaces":[{"interface":"back","average":{"1min":2.3820000000000001,"5min":3.0419999999999998,"15min":1.5209999999999999},"min":{"1min":0.33500000000000002,"5min":0.33000000000000002,"15min":0.28199999999999997},"max":{"1min":9.3070000000000004,"5min":142.994,"15min":142.994},"last":7.548},{"interface":"front","average":{"1min":2.516,"5min":3.004,"15min":1.5109999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.0339999999999998,"5min":143.411,"15min":143.411},"last":1.0449999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:01:46 2023","interfaces":[{"interface":"back","average":{"1min":2.3380000000000001,"5min":1.964,"15min":1.2050000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":22.670000000000002,"5min":22.670000000000002,"15min":22.670000000000002},"last":0.90000000000000002},{"interface":"front","average":{"1min":2.7429999999999999,"5min":2.0139999999999998,"15min":1.1679999999999999},"min":{"1min":0.34300000000000003,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.1959999999999997,"5min":18.013999999999999,"15min":18.013999999999999},"last":7.5010000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:01:46 2023","interfaces":[{"interface":"back","average":{"1min":11.558999999999999,"5min":6.6379999999999999,"15min":2.8250000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.30499999999999999},"max":{"1min":101.59999999999999,"5min":152.69,"15min":152.69},"last":0.83899999999999997},{"interface":"front","average":{"1min":11.509,"5min":6.7759999999999998,"15min":2.8319999999999999},"min":{"1min":0.42499999999999999,"5min":0.318,"15min":0.318},"max":{"1min":101.583,"5min":152.70400000000001,"15min":152.70400000000001},"last":7.5739999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:01:46 2023","interfaces":[{"interface":"back","average":{"1min":11.571,"5min":6.7110000000000003,"15min":2.79},"min":{"1min":0.377,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":104.65000000000001,"5min":149.93600000000001,"15min":149.93600000000001},"last":7.5979999999999999},{"interface":"front","average":{"1min":11.156000000000001,"5min":6.5339999999999998,"15min":2.778},"min":{"1min":0.38,"5min":0.379,"15min":0.379},"max":{"1min":104.69799999999999,"5min":150.12299999999999,"15min":150.12299999999999},"last":1.012}]}]},{"osd":4,"up_from":322,"seq":1382979469322,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":679032,"kb_used_data":344368,"kb_used_omap":35,"kb_used_meta":334620,"kb_avail":93066120,"statfs":{"total":95995035648,"available":95299706880,"internally_reserved":0,"allocated":352632832,"data_stored":351540717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36552,"internal_metadata":342651192},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32200000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.97699999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.503}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.85099999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28999999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.6360000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.79500000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757542,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":584492,"kb_used_data":283108,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93160660,"statfs":{"total":95995035648,"available":95396515840,"internally_reserved":0,"allocated":289902592,"data_stored":288819053,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:01:22 2023","interfaces":[{"interface":"back","average":{"1min":5.5229999999999997,"5min":2.2490000000000001,"15min":1.1819999999999999},"min":{"1min":0.23699999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":46.185000000000002,"5min":46.185000000000002,"15min":46.185000000000002},"last":1.5629999999999999},{"interface":"front","average":{"1min":5.9059999999999997,"5min":2.6240000000000001,"15min":1.3859999999999999},"min":{"1min":0.30599999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":46.222999999999999,"5min":46.222999999999999,"15min":46.222999999999999},"last":0.318}]},{"osd":1,"last update":"Mon Oct 9 06:01:39 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":1.3700000000000001,"15min":1.8720000000000001},"min":{"1min":0.33600000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.9790000000000001,"5min":6.8410000000000002,"15min":114.247},"last":2.0979999999999999},{"interface":"front","average":{"1min":1.153,"5min":0.86199999999999999,"15min":1.597},"min":{"1min":0.29899999999999999,"5min":0.28100000000000003,"15min":0.26400000000000001},"max":{"1min":2.887,"5min":3.367,"15min":114.678},"last":4.6929999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:01:06 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":3.984,"15min":2.3330000000000002},"min":{"1min":0.36099999999999999,"5min":0.28499999999999998,"15min":0.25},"max":{"1min":5.6600000000000001,"5min":217.06399999999999,"15min":217.06399999999999},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.4430000000000001,"5min":3.6869999999999998,"15min":2.1819999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.27600000000000002},"max":{"1min":5.8259999999999996,"5min":217.018,"15min":217.018},"last":1.5940000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:01:39 2023","interfaces":[{"interface":"back","average":{"1min":16.890999999999998,"5min":9.0679999999999996,"15min":6.2290000000000001},"min":{"1min":0.31900000000000001,"5min":0.29799999999999999,"15min":0.27500000000000002},"max":{"1min":97.403000000000006,"5min":211.40199999999999,"15min":272.81299999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":16.716000000000001,"5min":9.1500000000000004,"15min":6.4349999999999996},"min":{"1min":0.51200000000000001,"5min":0.317,"15min":0.317},"max":{"1min":94.423000000000002,"5min":211.578,"15min":273.00999999999999},"last":0.41799999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:00:33 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.79500000000000004,"15min":0.66200000000000003},"min":{"1min":0.308,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.2120000000000002,"5min":13.052,"15min":13.052},"last":3.0139999999999998},{"interface":"front","average":{"1min":0.79300000000000004,"5min":0.71599999999999997,"15min":0.63800000000000001},"min":{"1min":0.32100000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.069,"5min":3.879,"15min":13.050000000000001},"last":3.0880000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:01:42 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":2.1800000000000002,"15min":1.1659999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":3.6389999999999998,"5min":126.89400000000001,"15min":126.89400000000001},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.88100000000000001,"5min":2.1579999999999999,"15min":1.2290000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26200000000000001},"max":{"1min":3.1709999999999998,"5min":126.616,"15min":126.616},"last":3.8279999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:01:22 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":3.706,"15min":1.7949999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.29999999999999999},"max":{"1min":6.2590000000000003,"5min":133.27099999999999,"15min":133.27099999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.86099999999999999,"5min":3.4969999999999999,"15min":1.597},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.32000000000000001},"max":{"1min":4.6059999999999999,"5min":135.09200000000001,"15min":135.09200000000001},"last":3.5579999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1474,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4889,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":557453312,"data_stored":557449240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":75304960,"data_stored":75300888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":596115456,"data_stored":596115456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":682299392,"data_stored":682295320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":338362368,"data_stored":338362368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":276103168,"data_stored":276103168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":435490816,"data_stored":435486744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":470155264,"data_stored":470155264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:02:10.499 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 31 2023-10-09T06:02:10.552 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:10.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:10 smithi156 ceph-mon[195101]: failed to encode map e344 with expected crc 2023-10-09T06:02:10.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:10 smithi156 ceph-mon[195101]: failed to encode map e344 with expected crc 2023-10-09T06:02:10.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:10 smithi156 ceph-mon[195101]: failed to encode map e344 with expected crc 2023-10-09T06:02:10.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:10 smithi156 ceph-mon[195101]: pgmap v812: 119 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 114 active+clean; 1007 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 4431/48360 objects misplaced (9.163%); 19 MiB/s, 319 objects/s recovering 2023-10-09T06:02:10.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:10 smithi156 ceph-mon[195101]: 1.11 scrub starts 2023-10-09T06:02:10.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:10 smithi064 ceph-mon[220044]: failed to encode map e344 with expected crc 2023-10-09T06:02:10.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:10 smithi064 ceph-mon[220044]: failed to encode map e344 with expected crc 2023-10-09T06:02:10.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:10 smithi064 ceph-mon[220044]: failed to encode map e344 with expected crc 2023-10-09T06:02:10.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:10 smithi064 ceph-mon[220044]: pgmap v812: 119 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 114 active+clean; 1007 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 4431/48360 objects misplaced (9.163%); 19 MiB/s, 319 objects/s recovering 2023-10-09T06:02:10.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:10 smithi064 ceph-mon[220044]: 1.11 scrub starts 2023-10-09T06:02:11.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:11 smithi064 ceph-mon[220044]: 1.11 deep-scrub starts 2023-10-09T06:02:11.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:11 smithi064 ceph-mon[220044]: pgmap v813: 119 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 113 active+clean; 886 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 3874/42576 objects misplaced (9.099%); 9.6 MiB/s, 171 objects/s recovering 2023-10-09T06:02:11.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:11 smithi064 ceph-mon[212750]: 1.11 deep-scrub starts 2023-10-09T06:02:11.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:11 smithi064 ceph-mon[212750]: pgmap v813: 119 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 113 active+clean; 886 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 3874/42576 objects misplaced (9.099%); 9.6 MiB/s, 171 objects/s recovering 2023-10-09T06:02:11.607 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:11.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:11.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:11.716 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:11.823 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:11.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:11 smithi156 ceph-mon[195101]: 1.11 deep-scrub starts 2023-10-09T06:02:11.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:11 smithi156 ceph-mon[195101]: pgmap v813: 119 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 113 active+clean; 886 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 3874/42576 objects misplaced (9.099%); 9.6 MiB/s, 171 objects/s recovering 2023-10-09T06:02:12.040 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:12.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:12.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:12.108 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:12.146 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:12.231 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:12.305 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:02:12.305 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:02:12.671 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:02:12.689 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:12.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:12.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:12.708 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:02:12.749 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:02:12.797 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:02:12.809 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:12.831 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:02:12.844 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:12.873 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:02:12.903 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:12.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:12.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:12.919 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:02:12.955 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:02:12.965 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:02:12.966 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:02:12.966 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:02:12.966 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:02:12.966 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:02:12.966 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:02:12.966 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:02:12.966 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:02:13.012 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:13.059 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:13.115 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:13.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:13.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:13.220 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:13.301 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:13.330 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:02:13.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:13 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1378939336' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:13.712 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:13 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1378939336' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:13.712 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:13 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1378939336' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:13.808 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:02:13.837 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:13.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:13.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:13.941 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:13.998 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:14.211 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:02:14.551 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:14.554 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:14 smithi064 ceph-mon[212750]: pgmap v814: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 4 active+remapped+backfill_wait, 113 active+clean; 846 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 3169/40653 objects misplaced (7.795%); 11 MiB/s, 174 objects/s recovering 2023-10-09T06:02:14.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:14 smithi064 ceph-mon[212750]: 1.11 scrub starts 2023-10-09T06:02:14.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:02:14.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:14.555 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:14 smithi064 ceph-mon[220044]: pgmap v814: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 4 active+remapped+backfill_wait, 113 active+clean; 846 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 3169/40653 objects misplaced (7.795%); 11 MiB/s, 174 objects/s recovering 2023-10-09T06:02:14.555 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:14 smithi064 ceph-mon[220044]: 1.11 scrub starts 2023-10-09T06:02:14.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:02:14.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:14.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:14.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:14.658 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:14.682 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:14 smithi156 ceph-mon[195101]: pgmap v814: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 4 active+remapped+backfill_wait, 113 active+clean; 846 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 3169/40653 objects misplaced (7.795%); 11 MiB/s, 174 objects/s recovering 2023-10-09T06:02:14.682 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:14 smithi156 ceph-mon[195101]: 1.11 scrub starts 2023-10-09T06:02:14.683 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:02:14.683 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:14.685 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:14.743 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:14.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:14.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:14.846 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:14.877 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:14.921 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:14.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:14.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:15.025 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:15.051 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:15.203 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:15.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:15.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:15.306 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:02:15.333 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:15.375 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:02:15.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:15.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:02:15.479 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:02:15.523 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:15.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:15 smithi156 ceph-mon[195101]: osdmap e345: 8 total, 8 up, 8 in 2023-10-09T06:02:15.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:15 smithi156 ceph-mon[195101]: failed to encode map e345 with expected crc 2023-10-09T06:02:15.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:15 smithi156 ceph-mon[195101]: failed to encode map e345 with expected crc 2023-10-09T06:02:15.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:15 smithi156 ceph-mon[195101]: failed to encode map e345 with expected crc 2023-10-09T06:02:15.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:15 smithi156 ceph-mon[195101]: failed to encode map e345 with expected crc 2023-10-09T06:02:15.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:15 smithi156 ceph-mon[195101]: failed to encode map e345 with expected crc 2023-10-09T06:02:15.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:15 smithi156 ceph-mon[195101]: failed to encode map e345 with expected crc 2023-10-09T06:02:15.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:15 smithi156 ceph-mon[195101]: failed to encode map e345 with expected crc 2023-10-09T06:02:15.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:15 smithi156 ceph-mon[195101]: failed to encode map e345 with expected crc 2023-10-09T06:02:15.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:02:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:02:15] "GET /metrics HTTP/1.1" 200 36413 "" "Prometheus/2.43.0" 2023-10-09T06:02:15.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:15 smithi064 ceph-mon[212750]: osdmap e345: 8 total, 8 up, 8 in 2023-10-09T06:02:15.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:15 smithi064 ceph-mon[212750]: failed to encode map e345 with expected crc 2023-10-09T06:02:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:15 smithi064 ceph-mon[212750]: failed to encode map e345 with expected crc 2023-10-09T06:02:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:15 smithi064 ceph-mon[212750]: failed to encode map e345 with expected crc 2023-10-09T06:02:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:15 smithi064 ceph-mon[212750]: failed to encode map e345 with expected crc 2023-10-09T06:02:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:15 smithi064 ceph-mon[212750]: failed to encode map e345 with expected crc 2023-10-09T06:02:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:15 smithi064 ceph-mon[212750]: failed to encode map e345 with expected crc 2023-10-09T06:02:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:15 smithi064 ceph-mon[212750]: failed to encode map e345 with expected crc 2023-10-09T06:02:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:15 smithi064 ceph-mon[212750]: failed to encode map e345 with expected crc 2023-10-09T06:02:15.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:15 smithi064 ceph-mon[220044]: osdmap e345: 8 total, 8 up, 8 in 2023-10-09T06:02:15.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:15 smithi064 ceph-mon[220044]: failed to encode map e345 with expected crc 2023-10-09T06:02:15.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:15 smithi064 ceph-mon[220044]: failed to encode map e345 with expected crc 2023-10-09T06:02:15.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:15 smithi064 ceph-mon[220044]: failed to encode map e345 with expected crc 2023-10-09T06:02:15.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:15 smithi064 ceph-mon[220044]: failed to encode map e345 with expected crc 2023-10-09T06:02:15.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:15 smithi064 ceph-mon[220044]: failed to encode map e345 with expected crc 2023-10-09T06:02:15.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:15 smithi064 ceph-mon[220044]: failed to encode map e345 with expected crc 2023-10-09T06:02:15.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:15 smithi064 ceph-mon[220044]: failed to encode map e345 with expected crc 2023-10-09T06:02:15.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:15 smithi064 ceph-mon[220044]: failed to encode map e345 with expected crc 2023-10-09T06:02:15.790 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:02:15.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:15.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:02:15.872 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:15.894 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:15.989 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:16.265 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:16.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:16.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:16.374 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:16.423 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:16.485 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:02:16.485 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:02:16.485 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:16.485 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45263.0:200749 7.1 7:88b81feb:::benchmark_data_smithi064_398483_object95396:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T06:02:16.486 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:02:16.480131+0000", 2023-10-09T06:02:16.486 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0047508209999999997, 2023-10-09T06:02:16.486 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0047841359999999996, 2023-10-09T06:02:16.486 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:02:16.486 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:02:16.486 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:02:16.487 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45263", 2023-10-09T06:02:16.487 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:02:16.487 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 200749 2023-10-09T06:02:16.487 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:16.487 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:02:16.488 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:16.488 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:02:16.488 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:16.480131+0000", 2023-10-09T06:02:16.488 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:16.488 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:16.489 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:16.489 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:02:16.489 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:16.480131+0000", 2023-10-09T06:02:16.489 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:16.489 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:16.490 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:16.490 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:02:16.490 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:16.480129+0000", 2023-10-09T06:02:16.491 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T06:02:16.491 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:16.491 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:16.491 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:02:16.492 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:16.480141+0000", 2023-10-09T06:02:16.492 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1656e-05 2023-10-09T06:02:16.492 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:16.492 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:16.492 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:02:16.492 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:16.480153+0000", 2023-10-09T06:02:16.492 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2551999999999999e-05 2023-10-09T06:02:16.492 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:16.492 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:16.492 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:02:16.493 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:16.480160+0000", 2023-10-09T06:02:16.493 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9460000000000001e-06 2023-10-09T06:02:16.493 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:16.493 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:16.493 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:02:16.493 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:16.480292+0000", 2023-10-09T06:02:16.493 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013179000000000001 2023-10-09T06:02:16.493 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:16.493 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:16.493 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:02:16.494 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:16.480450+0000", 2023-10-09T06:02:16.494 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015828700000000001 2023-10-09T06:02:16.494 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:16.494 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:16.494 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:02:16.494 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:16.480589+0000", 2023-10-09T06:02:16.494 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013856799999999999 2023-10-09T06:02:16.494 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:16.494 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:16.495 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:02:16.495 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:16.483756+0000", 2023-10-09T06:02:16.495 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0031665629999999998 2023-10-09T06:02:16.495 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:16.495 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:16.495 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:16.495 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:16.484168+0000", 2023-10-09T06:02:16.495 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00041271400000000001 2023-10-09T06:02:16.495 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:16.495 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:02:16.495 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:16.496 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:16.496 INFO:teuthology.orchestra.run.smithi156.stdout: ], 2023-10-09T06:02:16.496 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 1 2023-10-09T06:02:16.496 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:02:16.506 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:16.512 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:16 smithi156 ceph-mon[195101]: 1.17 scrub starts 2023-10-09T06:02:16.512 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:16 smithi156 ceph-mon[195101]: pgmap v816: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 4 active+remapped+backfill_wait, 113 active+clean; 806 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 3169/38745 objects misplaced (8.179%); 9.4 MiB/s, 150 objects/s recovering 2023-10-09T06:02:16.512 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:16 smithi156 ceph-mon[195101]: 1.11 deep-scrub starts 2023-10-09T06:02:16.513 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:16 smithi156 ceph-mon[195101]: 1.1a scrub starts 2023-10-09T06:02:16.513 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:16 smithi156 ceph-mon[195101]: osdmap e346: 8 total, 8 up, 8 in 2023-10-09T06:02:16.513 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:16 smithi156 ceph-mon[195101]: failed to encode map e346 with expected crc 2023-10-09T06:02:16.513 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:16 smithi156 ceph-mon[195101]: failed to encode map e346 with expected crc 2023-10-09T06:02:16.513 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:16 smithi156 ceph-mon[195101]: failed to encode map e346 with expected crc 2023-10-09T06:02:16.513 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:16 smithi156 ceph-mon[195101]: failed to encode map e346 with expected crc 2023-10-09T06:02:16.514 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:16 smithi156 ceph-mon[195101]: failed to encode map e346 with expected crc 2023-10-09T06:02:16.514 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:16 smithi156 ceph-mon[195101]: failed to encode map e346 with expected crc 2023-10-09T06:02:16.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:16.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:16.553 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:16 smithi064 ceph-mon[212750]: 1.17 scrub starts 2023-10-09T06:02:16.554 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:16 smithi064 ceph-mon[212750]: pgmap v816: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 4 active+remapped+backfill_wait, 113 active+clean; 806 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 3169/38745 objects misplaced (8.179%); 9.4 MiB/s, 150 objects/s recovering 2023-10-09T06:02:16.554 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:16 smithi064 ceph-mon[212750]: 1.11 deep-scrub starts 2023-10-09T06:02:16.554 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:16 smithi064 ceph-mon[212750]: 1.1a scrub starts 2023-10-09T06:02:16.554 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:16 smithi064 ceph-mon[212750]: osdmap e346: 8 total, 8 up, 8 in 2023-10-09T06:02:16.554 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:16 smithi064 ceph-mon[212750]: failed to encode map e346 with expected crc 2023-10-09T06:02:16.554 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:16 smithi064 ceph-mon[212750]: failed to encode map e346 with expected crc 2023-10-09T06:02:16.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:16 smithi064 ceph-mon[212750]: failed to encode map e346 with expected crc 2023-10-09T06:02:16.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:16 smithi064 ceph-mon[212750]: failed to encode map e346 with expected crc 2023-10-09T06:02:16.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:16 smithi064 ceph-mon[212750]: failed to encode map e346 with expected crc 2023-10-09T06:02:16.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:16 smithi064 ceph-mon[212750]: failed to encode map e346 with expected crc 2023-10-09T06:02:16.555 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:16 smithi064 ceph-mon[220044]: 1.17 scrub starts 2023-10-09T06:02:16.555 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:16 smithi064 ceph-mon[220044]: pgmap v816: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 4 active+remapped+backfill_wait, 113 active+clean; 806 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 3169/38745 objects misplaced (8.179%); 9.4 MiB/s, 150 objects/s recovering 2023-10-09T06:02:16.555 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:16 smithi064 ceph-mon[220044]: 1.11 deep-scrub starts 2023-10-09T06:02:16.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:16 smithi064 ceph-mon[220044]: 1.1a scrub starts 2023-10-09T06:02:16.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:16 smithi064 ceph-mon[220044]: osdmap e346: 8 total, 8 up, 8 in 2023-10-09T06:02:16.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:16 smithi064 ceph-mon[220044]: failed to encode map e346 with expected crc 2023-10-09T06:02:16.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:16 smithi064 ceph-mon[220044]: failed to encode map e346 with expected crc 2023-10-09T06:02:16.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:16 smithi064 ceph-mon[220044]: failed to encode map e346 with expected crc 2023-10-09T06:02:16.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:16 smithi064 ceph-mon[220044]: failed to encode map e346 with expected crc 2023-10-09T06:02:16.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:16 smithi064 ceph-mon[220044]: failed to encode map e346 with expected crc 2023-10-09T06:02:16.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:16 smithi064 ceph-mon[220044]: failed to encode map e346 with expected crc 2023-10-09T06:02:16.616 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:16.651 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:16.729 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:16.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:16.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:16.822 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:02:16.836 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:16.872 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:16.928 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:16.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:16.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:17.031 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:17.128 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:17.244 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:17.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:17.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:17.333 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 1 pgp_num to 31 2023-10-09T06:02:17.348 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:02:17.374 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:17.417 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:02:17.445 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:02:17.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:17.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:02:17.553 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:02:17.579 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:17.622 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:02:17.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:17 smithi156 ceph-mon[195101]: 1.17 scrub starts 2023-10-09T06:02:17.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:17 smithi156 ceph-mon[195101]: 1.e scrub starts 2023-10-09T06:02:17.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:17 smithi156 ceph-mon[195101]: 1.e scrub ok 2023-10-09T06:02:17.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:17 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2782407008' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-10-09T06:02:17.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:17 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2782407008' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2023-10-09T06:02:17.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:17 smithi156 ceph-mon[195101]: osdmap e347: 8 total, 8 up, 8 in 2023-10-09T06:02:17.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:17.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:02:17.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:17 smithi064 ceph-mon[212750]: 1.17 scrub starts 2023-10-09T06:02:17.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:17 smithi064 ceph-mon[212750]: 1.e scrub starts 2023-10-09T06:02:17.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:17 smithi064 ceph-mon[212750]: 1.e scrub ok 2023-10-09T06:02:17.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:17 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2782407008' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-10-09T06:02:17.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:17 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2782407008' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2023-10-09T06:02:17.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:17 smithi064 ceph-mon[212750]: osdmap e347: 8 total, 8 up, 8 in 2023-10-09T06:02:17.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:17 smithi064 ceph-mon[220044]: 1.17 scrub starts 2023-10-09T06:02:17.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:17 smithi064 ceph-mon[220044]: 1.e scrub starts 2023-10-09T06:02:17.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:17 smithi064 ceph-mon[220044]: 1.e scrub ok 2023-10-09T06:02:17.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:17 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2782407008' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-10-09T06:02:17.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:17 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2782407008' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2023-10-09T06:02:17.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:17 smithi064 ceph-mon[220044]: osdmap e347: 8 total, 8 up, 8 in 2023-10-09T06:02:17.728 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:17.759 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:17.818 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:02:18.572 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:18.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:18.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:18.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:18 smithi156 ceph-mon[195101]: 1.11 scrub starts 2023-10-09T06:02:18.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:18 smithi156 ceph-mon[195101]: 1.17 scrub starts 2023-10-09T06:02:18.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:18 smithi156 ceph-mon[195101]: pgmap v818: 119 pgs: 1 peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 114 active+clean; 691 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 2260/33204 objects misplaced (6.806%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T06:02:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:18 smithi156 ceph-mon[195101]: failed to encode map e347 with expected crc 2023-10-09T06:02:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:18 smithi156 ceph-mon[195101]: 1.5 scrub starts 2023-10-09T06:02:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:18 smithi156 ceph-mon[195101]: 1.5 scrub ok 2023-10-09T06:02:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:18 smithi156 ceph-mon[195101]: failed to encode map e347 with expected crc 2023-10-09T06:02:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:18 smithi156 ceph-mon[195101]: failed to encode map e347 with expected crc 2023-10-09T06:02:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:18 smithi156 ceph-mon[195101]: failed to encode map e347 with expected crc 2023-10-09T06:02:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:18 smithi156 ceph-mon[195101]: failed to encode map e347 with expected crc 2023-10-09T06:02:18.675 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:18.678 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:18 smithi064 ceph-mon[212750]: 1.11 scrub starts 2023-10-09T06:02:18.678 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:18 smithi064 ceph-mon[212750]: 1.17 scrub starts 2023-10-09T06:02:18.678 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:18 smithi064 ceph-mon[212750]: pgmap v818: 119 pgs: 1 peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 114 active+clean; 691 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 2260/33204 objects misplaced (6.806%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T06:02:18.678 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:18 smithi064 ceph-mon[212750]: failed to encode map e347 with expected crc 2023-10-09T06:02:18.678 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:18 smithi064 ceph-mon[212750]: 1.5 scrub starts 2023-10-09T06:02:18.679 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:18 smithi064 ceph-mon[212750]: 1.5 scrub ok 2023-10-09T06:02:18.679 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:18 smithi064 ceph-mon[212750]: failed to encode map e347 with expected crc 2023-10-09T06:02:18.679 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:18 smithi064 ceph-mon[212750]: failed to encode map e347 with expected crc 2023-10-09T06:02:18.679 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:18 smithi064 ceph-mon[212750]: failed to encode map e347 with expected crc 2023-10-09T06:02:18.679 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:18 smithi064 ceph-mon[212750]: failed to encode map e347 with expected crc 2023-10-09T06:02:18.679 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:18 smithi064 ceph-mon[220044]: 1.11 scrub starts 2023-10-09T06:02:18.679 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:18 smithi064 ceph-mon[220044]: 1.17 scrub starts 2023-10-09T06:02:18.679 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:18 smithi064 ceph-mon[220044]: pgmap v818: 119 pgs: 1 peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 114 active+clean; 691 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 2260/33204 objects misplaced (6.806%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T06:02:18.679 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:18 smithi064 ceph-mon[220044]: failed to encode map e347 with expected crc 2023-10-09T06:02:18.679 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:18 smithi064 ceph-mon[220044]: 1.5 scrub starts 2023-10-09T06:02:18.680 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:18 smithi064 ceph-mon[220044]: 1.5 scrub ok 2023-10-09T06:02:18.680 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:18 smithi064 ceph-mon[220044]: failed to encode map e347 with expected crc 2023-10-09T06:02:18.680 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:18 smithi064 ceph-mon[220044]: failed to encode map e347 with expected crc 2023-10-09T06:02:18.680 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:18 smithi064 ceph-mon[220044]: failed to encode map e347 with expected crc 2023-10-09T06:02:18.680 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:18 smithi064 ceph-mon[220044]: failed to encode map e347 with expected crc 2023-10-09T06:02:18.710 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:18.968 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:18.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:18.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:19.072 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:19.152 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:19.163 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:19.271 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:19.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:19.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:19.378 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:19.487 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:19.546 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:19 smithi064 ceph-mon[212750]: 1.11 deep-scrub starts 2023-10-09T06:02:19.546 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:19 smithi064 ceph-mon[212750]: 1.17 scrub starts 2023-10-09T06:02:19.546 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:19 smithi064 ceph-mon[212750]: 1.1a scrub starts 2023-10-09T06:02:19.546 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:19 smithi064 ceph-mon[212750]: failed to encode map e348 with expected crc 2023-10-09T06:02:19.546 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:19 smithi064 ceph-mon[212750]: osdmap e348: 8 total, 8 up, 8 in 2023-10-09T06:02:19.546 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:19 smithi064 ceph-mon[212750]: failed to encode map e348 with expected crc 2023-10-09T06:02:19.546 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:19 smithi064 ceph-mon[212750]: failed to encode map e348 with expected crc 2023-10-09T06:02:19.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:19 smithi064 ceph-mon[220044]: 1.11 deep-scrub starts 2023-10-09T06:02:19.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:19 smithi064 ceph-mon[220044]: 1.17 scrub starts 2023-10-09T06:02:19.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:19 smithi064 ceph-mon[220044]: 1.1a scrub starts 2023-10-09T06:02:19.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:19 smithi064 ceph-mon[220044]: failed to encode map e348 with expected crc 2023-10-09T06:02:19.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:19 smithi064 ceph-mon[220044]: osdmap e348: 8 total, 8 up, 8 in 2023-10-09T06:02:19.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:19 smithi064 ceph-mon[220044]: failed to encode map e348 with expected crc 2023-10-09T06:02:19.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:19 smithi064 ceph-mon[220044]: failed to encode map e348 with expected crc 2023-10-09T06:02:19.574 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:02:19.574 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:02:19.574 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T06:02:19.574 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T06:02:19.574 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:02:19.599 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:19 smithi156 ceph-mon[195101]: 1.11 deep-scrub starts 2023-10-09T06:02:19.599 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:19 smithi156 ceph-mon[195101]: 1.17 scrub starts 2023-10-09T06:02:19.599 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:19 smithi156 ceph-mon[195101]: 1.1a scrub starts 2023-10-09T06:02:19.599 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:19 smithi156 ceph-mon[195101]: failed to encode map e348 with expected crc 2023-10-09T06:02:19.599 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:19 smithi156 ceph-mon[195101]: osdmap e348: 8 total, 8 up, 8 in 2023-10-09T06:02:19.599 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:19 smithi156 ceph-mon[195101]: failed to encode map e348 with expected crc 2023-10-09T06:02:19.599 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:19 smithi156 ceph-mon[195101]: failed to encode map e348 with expected crc 2023-10-09T06:02:19.624 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:19.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:19.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:19.739 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:19.815 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:20.018 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:20.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:20.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:20.122 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:20.170 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:20.422 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:02:20.428 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:20 smithi156 ceph-mon[195101]: 1.11 scrub starts 2023-10-09T06:02:20.428 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:20 smithi156 ceph-mon[195101]: failed to encode map e348 with expected crc 2023-10-09T06:02:20.428 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:20 smithi156 ceph-mon[195101]: pgmap v821: 119 pgs: 1 peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 114 active+clean; 644 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 2078/30975 objects misplaced (6.709%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:02:20.428 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:20 smithi156 ceph-mon[195101]: 1.1a scrub starts 2023-10-09T06:02:20.428 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:20 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3303597249' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:02:20.428 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:20 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:02:20.429 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:02:20.438 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:20.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:20.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:20.546 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:20.577 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:20.643 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:20.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:20.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:20.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:20 smithi064 ceph-mon[212750]: 1.11 scrub starts 2023-10-09T06:02:20.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:20 smithi064 ceph-mon[212750]: failed to encode map e348 with expected crc 2023-10-09T06:02:20.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:20 smithi064 ceph-mon[212750]: pgmap v821: 119 pgs: 1 peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 114 active+clean; 644 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 2078/30975 objects misplaced (6.709%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:02:20.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:20 smithi064 ceph-mon[212750]: 1.1a scrub starts 2023-10-09T06:02:20.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:20 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3303597249' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:02:20.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:20 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:02:20.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:20 smithi064 ceph-mon[220044]: 1.11 scrub starts 2023-10-09T06:02:20.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:20 smithi064 ceph-mon[220044]: failed to encode map e348 with expected crc 2023-10-09T06:02:20.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:20 smithi064 ceph-mon[220044]: pgmap v821: 119 pgs: 1 peering, 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 114 active+clean; 644 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 2078/30975 objects misplaced (6.709%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:02:20.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:20 smithi064 ceph-mon[220044]: 1.1a scrub starts 2023-10-09T06:02:20.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:20 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3303597249' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:02:20.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:20 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:02:20.753 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:20.835 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:20.885 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:02:21.478 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:21.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:21.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:21.587 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:21.632 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:21.691 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:21.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:21.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:21.701 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:21 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:02:21.701 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:21 smithi156 ceph-mon[195101]: failed to encode map e349 with expected crc 2023-10-09T06:02:21.701 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:21 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:02:21.701 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:21 smithi156 ceph-mon[195101]: osdmap e349: 8 total, 8 up, 8 in 2023-10-09T06:02:21.701 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:21 smithi156 ceph-mon[195101]: failed to encode map e349 with expected crc 2023-10-09T06:02:21.701 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:21 smithi156 ceph-mon[195101]: failed to encode map e349 with expected crc 2023-10-09T06:02:21.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:21 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:02:21.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:21 smithi064 ceph-mon[220044]: failed to encode map e349 with expected crc 2023-10-09T06:02:21.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:21 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:02:21.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:21 smithi064 ceph-mon[220044]: osdmap e349: 8 total, 8 up, 8 in 2023-10-09T06:02:21.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:21 smithi064 ceph-mon[220044]: failed to encode map e349 with expected crc 2023-10-09T06:02:21.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:21 smithi064 ceph-mon[220044]: failed to encode map e349 with expected crc 2023-10-09T06:02:21.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:21 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:02:21.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:21 smithi064 ceph-mon[212750]: failed to encode map e349 with expected crc 2023-10-09T06:02:21.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:21 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:02:21.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:21 smithi064 ceph-mon[212750]: osdmap e349: 8 total, 8 up, 8 in 2023-10-09T06:02:21.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:21 smithi064 ceph-mon[212750]: failed to encode map e349 with expected crc 2023-10-09T06:02:21.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:21 smithi064 ceph-mon[212750]: failed to encode map e349 with expected crc 2023-10-09T06:02:21.802 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:21.903 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:22.076 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:22.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:22.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:22.193 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:22.240 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:22.361 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:22.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:22.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:22.425 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:22 smithi064 ceph-mon[212750]: pgmap v823: 119 pgs: 1 active+clean+scrubbing, 3 active+remapped+backfill_wait, 115 active+clean; 532 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 984/25590 objects misplaced (3.845%); 10 MiB/s, 163 objects/s recovering 2023-10-09T06:02:22.464 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:22.512 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:22.537 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:02:22.576 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:02:22.587 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:22.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:22.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:22.594 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:22 smithi156 ceph-mon[195101]: pgmap v823: 119 pgs: 1 active+clean+scrubbing, 3 active+remapped+backfill_wait, 115 active+clean; 532 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 984/25590 objects misplaced (3.845%); 10 MiB/s, 163 objects/s recovering 2023-10-09T06:02:22.608 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:02:22.608 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T06:02:22.608 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T06:02:22.608 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:02:22.608 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.608 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45263.0:166114 7.1 7.6997e1e1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-10-09T06:02:22.608 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:01:21.468148+0000", 2023-10-09T06:02:22.608 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 61.137909569999998, 2023-10-09T06:02:22.609 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.84786050599999996, 2023-10-09T06:02:22.609 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:02:22.609 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:02:22.609 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:02:22.609 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45263", 2023-10-09T06:02:22.609 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:02:22.609 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 166114 2023-10-09T06:02:22.609 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.609 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:02:22.609 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.609 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:02:22.610 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468148+0000", 2023-10-09T06:02:22.610 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.610 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.610 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.610 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:02:22.610 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468148+0000", 2023-10-09T06:02:22.610 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.610 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.610 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.610 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:02:22.611 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468147+0000", 2023-10-09T06:02:22.611 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:02:22.611 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.611 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.611 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:02:22.611 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468152+0000", 2023-10-09T06:02:22.611 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0479999999999998e-06 2023-10-09T06:02:22.611 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.611 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.611 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:02:22.611 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468160+0000", 2023-10-09T06:02:22.612 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4510000000000003e-06 2023-10-09T06:02:22.612 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.612 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.612 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:02:22.612 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468165+0000", 2023-10-09T06:02:22.612 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8880000000000002e-06 2023-10-09T06:02:22.612 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.612 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.612 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:02:22.612 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468214+0000", 2023-10-09T06:02:22.612 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.884e-05 2023-10-09T06:02:22.613 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.613 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.613 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:02:22.613 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468225+0000", 2023-10-09T06:02:22.613 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1061e-05 2023-10-09T06:02:22.613 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.613 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.613 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:02:22.613 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:22.315992+0000", 2023-10-09T06:02:22.613 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.84776714900000005 2023-10-09T06:02:22.613 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.614 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.614 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:02:22.614 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:22.316009+0000", 2023-10-09T06:02:22.614 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6137000000000002e-05 2023-10-09T06:02:22.614 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.614 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:02:22.614 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.614 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.614 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.614 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45263.0:166115 7.1 7.a53a721 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-10-09T06:02:22.614 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:01:21.468174+0000", 2023-10-09T06:02:22.615 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 61.137883430999999, 2023-10-09T06:02:22.615 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.84806625999999996, 2023-10-09T06:02:22.615 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:02:22.615 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:02:22.615 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:02:22.615 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45263", 2023-10-09T06:02:22.615 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:02:22.615 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 166115 2023-10-09T06:02:22.615 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.615 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:02:22.615 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.615 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:02:22.616 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468174+0000", 2023-10-09T06:02:22.616 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.616 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.616 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.616 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:02:22.616 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468174+0000", 2023-10-09T06:02:22.616 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.616 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.616 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.616 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:02:22.616 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468174+0000", 2023-10-09T06:02:22.616 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:02:22.617 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.617 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.617 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:02:22.617 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468176+0000", 2023-10-09T06:02:22.617 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.655e-06 2023-10-09T06:02:22.617 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.617 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.617 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:02:22.617 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468188+0000", 2023-10-09T06:02:22.617 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1212999999999999e-05 2023-10-09T06:02:22.617 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.617 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.618 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:02:22.618 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468192+0000", 2023-10-09T06:02:22.618 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2760000000000002e-06 2023-10-09T06:02:22.618 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.618 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.618 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:02:22.618 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468250+0000", 2023-10-09T06:02:22.618 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8267000000000003e-05 2023-10-09T06:02:22.618 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.618 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.618 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:02:22.618 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468256+0000", 2023-10-09T06:02:22.619 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.553e-06 2023-10-09T06:02:22.619 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.619 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.619 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:02:22.619 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:22.316234+0000", 2023-10-09T06:02:22.619 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.84797834599999999 2023-10-09T06:02:22.619 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.619 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.619 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:02:22.619 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:22.316241+0000", 2023-10-09T06:02:22.619 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4110000000000001e-06 2023-10-09T06:02:22.619 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.620 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:02:22.620 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.620 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.620 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.620 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45263.0:166773 7.9 e324/323 7:9accf9c9:::benchmark_data_smithi064_398483_object61420:head v 324'10500, mlcod=315'9253)", 2023-10-09T06:02:22.620 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:01:24.445502+0000", 2023-10-09T06:02:22.620 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 58.160555924000001, 2023-10-09T06:02:22.620 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.034357862000000003, 2023-10-09T06:02:22.620 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:02:22.620 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:22.620 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:02:22.621 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.621 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:02:22.621 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.445502+0000", 2023-10-09T06:02:22.621 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.621 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.621 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.621 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:02:22.621 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.445502+0000", 2023-10-09T06:02:22.621 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.621 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.621 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.621 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:02:22.622 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.445501+0000", 2023-10-09T06:02:22.622 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:02:22.622 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.622 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.622 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:02:22.622 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.445507+0000", 2023-10-09T06:02:22.622 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2630000000000003e-06 2023-10-09T06:02:22.622 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.622 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.622 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:02:22.622 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.445518+0000", 2023-10-09T06:02:22.622 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1026000000000001e-05 2023-10-09T06:02:22.623 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.623 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.623 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:02:22.623 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.445522+0000", 2023-10-09T06:02:22.623 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.121e-06 2023-10-09T06:02:22.623 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.623 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.623 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:02:22.623 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.449987+0000", 2023-10-09T06:02:22.623 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.004465096 2023-10-09T06:02:22.623 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.623 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.624 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:02:22.624 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.450010+0000", 2023-10-09T06:02:22.624 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3193999999999998e-05 2023-10-09T06:02:22.624 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.624 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.624 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:02:22.624 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.479815+0000", 2023-10-09T06:02:22.624 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.029804415000000001 2023-10-09T06:02:22.624 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.624 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.624 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:02:22.624 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.479860+0000", 2023-10-09T06:02:22.625 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5116e-05 2023-10-09T06:02:22.625 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.625 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:02:22.625 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.625 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.625 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.625 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45263.0:166775 7.9 e324/323 7:9d8ee137:::benchmark_data_smithi064_398483_object61422:head v 324'10501, mlcod=315'9253)", 2023-10-09T06:02:22.625 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:01:24.449866+0000", 2023-10-09T06:02:22.625 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 58.156192040999997, 2023-10-09T06:02:22.625 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.037012376, 2023-10-09T06:02:22.625 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:02:22.626 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:22.626 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:02:22.626 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.626 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:02:22.626 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.449866+0000", 2023-10-09T06:02:22.626 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.626 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.626 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.626 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:02:22.626 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.449866+0000", 2023-10-09T06:02:22.626 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.626 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.627 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.627 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:02:22.627 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.449865+0000", 2023-10-09T06:02:22.627 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:02:22.627 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.627 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.627 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:02:22.627 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.449880+0000", 2023-10-09T06:02:22.627 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4691e-05 2023-10-09T06:02:22.627 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.627 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.627 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:02:22.628 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.449891+0000", 2023-10-09T06:02:22.628 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0801999999999999e-05 2023-10-09T06:02:22.628 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.628 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.628 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:02:22.628 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.449899+0000", 2023-10-09T06:02:22.628 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0269999999999995e-06 2023-10-09T06:02:22.628 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.628 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.628 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:02:22.628 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.465431+0000", 2023-10-09T06:02:22.628 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015532225 2023-10-09T06:02:22.629 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.629 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.629 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:02:22.629 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.465457+0000", 2023-10-09T06:02:22.629 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.5772e-05 2023-10-09T06:02:22.629 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.629 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.629 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:02:22.629 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.486846+0000", 2023-10-09T06:02:22.629 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.021388933999999998 2023-10-09T06:02:22.629 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.629 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.630 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:02:22.630 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.486878+0000", 2023-10-09T06:02:22.630 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.2434999999999997e-05 2023-10-09T06:02:22.630 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.630 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:02:22.630 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.630 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.630 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.630 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45263.0:166786 7.1 e324/323 7:89fc282a:::benchmark_data_smithi064_398483_object61433:head v 324'10410, mlcod=315'9249)", 2023-10-09T06:02:22.630 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:01:24.451959+0000", 2023-10-09T06:02:22.630 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 58.154098298999997, 2023-10-09T06:02:22.631 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.034879764000000001, 2023-10-09T06:02:22.631 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:02:22.631 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:22.631 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:02:22.631 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.631 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:02:22.631 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.451959+0000", 2023-10-09T06:02:22.631 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.631 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.631 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.631 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:02:22.631 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.451959+0000", 2023-10-09T06:02:22.632 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.632 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.632 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.632 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:02:22.632 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.451958+0000", 2023-10-09T06:02:22.632 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:22.632 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.632 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.632 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:02:22.632 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.451973+0000", 2023-10-09T06:02:22.632 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5078e-05 2023-10-09T06:02:22.632 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.633 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.633 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:02:22.633 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.451991+0000", 2023-10-09T06:02:22.633 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7499e-05 2023-10-09T06:02:22.633 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.633 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.633 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:02:22.633 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.452000+0000", 2023-10-09T06:02:22.633 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9269999999999994e-06 2023-10-09T06:02:22.633 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.633 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.633 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:02:22.634 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.460784+0000", 2023-10-09T06:02:22.634 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0087840339999999996 2023-10-09T06:02:22.634 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.634 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.634 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:02:22.634 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.460811+0000", 2023-10-09T06:02:22.634 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.6974999999999998e-05 2023-10-09T06:02:22.634 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.634 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.634 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:02:22.635 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.486798+0000", 2023-10-09T06:02:22.635 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.025987110000000001 2023-10-09T06:02:22.635 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.635 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.635 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:02:22.635 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.486839+0000", 2023-10-09T06:02:22.635 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1384999999999998e-05 2023-10-09T06:02:22.635 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.635 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:02:22.635 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.635 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.636 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.636 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45263.0:166795 7.1 e324/323 7:8459951c:::benchmark_data_smithi064_398483_object61442:head v 324'10411, mlcod=315'9249)", 2023-10-09T06:02:22.636 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:01:24.456681+0000", 2023-10-09T06:02:22.636 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 58.149376908999997, 2023-10-09T06:02:22.636 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.048266389, 2023-10-09T06:02:22.636 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:02:22.636 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:22.636 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:02:22.636 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.636 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:02:22.636 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.456681+0000", 2023-10-09T06:02:22.637 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.637 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.637 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.637 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:02:22.637 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.456681+0000", 2023-10-09T06:02:22.637 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.637 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.637 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.637 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:02:22.637 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.456680+0000", 2023-10-09T06:02:22.637 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:02:22.637 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.638 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.638 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:02:22.638 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.456690+0000", 2023-10-09T06:02:22.638 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7860000000000008e-06 2023-10-09T06:02:22.638 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.638 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.638 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:02:22.638 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.456699+0000", 2023-10-09T06:02:22.638 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.5899999999999997e-06 2023-10-09T06:02:22.638 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.638 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.639 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:02:22.639 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.456706+0000", 2023-10-09T06:02:22.639 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5520000000000001e-06 2023-10-09T06:02:22.639 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.639 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.639 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:02:22.639 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.469593+0000", 2023-10-09T06:02:22.639 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.01288721 2023-10-09T06:02:22.639 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.639 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.639 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:02:22.639 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.469621+0000", 2023-10-09T06:02:22.640 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.8181e-05 2023-10-09T06:02:22.640 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.640 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.640 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:02:22.640 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.504907+0000", 2023-10-09T06:02:22.640 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.035286035 2023-10-09T06:02:22.640 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.640 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.640 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:02:22.640 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.504947+0000", 2023-10-09T06:02:22.640 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9913000000000002e-05 2023-10-09T06:02:22.641 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.641 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:02:22.641 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.641 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.641 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.641 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45263.0:166793 7.9 e324/323 7:9afa3dfb:::benchmark_data_smithi064_398483_object61440:head v 324'10502, mlcod=315'9253)", 2023-10-09T06:02:22.641 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:01:24.460376+0000", 2023-10-09T06:02:22.641 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 58.145681394, 2023-10-09T06:02:22.641 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.035528401000000001, 2023-10-09T06:02:22.641 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:02:22.641 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:22.641 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:02:22.642 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.642 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:02:22.642 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.460376+0000", 2023-10-09T06:02:22.642 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.642 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.642 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.642 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:02:22.642 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.460376+0000", 2023-10-09T06:02:22.642 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.642 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.642 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.643 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:02:22.643 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.460375+0000", 2023-10-09T06:02:22.643 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:22.643 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.643 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.643 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:02:22.643 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.460384+0000", 2023-10-09T06:02:22.643 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.721e-06 2023-10-09T06:02:22.643 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.643 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.643 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:02:22.643 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.460406+0000", 2023-10-09T06:02:22.644 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1922000000000001e-05 2023-10-09T06:02:22.644 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.644 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.644 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:02:22.644 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.460415+0000", 2023-10-09T06:02:22.644 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1540000000000008e-06 2023-10-09T06:02:22.644 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.644 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.644 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:02:22.644 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.475972+0000", 2023-10-09T06:02:22.644 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015556876000000001 2023-10-09T06:02:22.644 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.645 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.645 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:02:22.645 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.476000+0000", 2023-10-09T06:02:22.645 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.8509000000000001e-05 2023-10-09T06:02:22.645 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.645 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.645 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:02:22.645 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.495863+0000", 2023-10-09T06:02:22.645 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.019862919999999999 2023-10-09T06:02:22.645 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.645 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.645 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:02:22.646 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.495905+0000", 2023-10-09T06:02:22.646 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1591000000000002e-05 2023-10-09T06:02:22.646 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.646 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:02:22.646 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.646 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.646 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.646 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45263.0:166804 7.1 e324/323 7:81891c03:::benchmark_data_smithi064_398483_object61451:head v 324'10412, mlcod=315'9249)", 2023-10-09T06:02:22.646 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:01:24.470201+0000", 2023-10-09T06:02:22.646 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 58.135856578999999, 2023-10-09T06:02:22.646 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.034771516000000002, 2023-10-09T06:02:22.646 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:02:22.647 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:22.647 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:02:22.647 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.647 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:02:22.647 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.470201+0000", 2023-10-09T06:02:22.647 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.647 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.647 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.647 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:02:22.647 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.470201+0000", 2023-10-09T06:02:22.647 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.647 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.648 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.648 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:02:22.648 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.470200+0000", 2023-10-09T06:02:22.648 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:02:22.648 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.648 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.648 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:02:22.648 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.470209+0000", 2023-10-09T06:02:22.648 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9500000000000007e-06 2023-10-09T06:02:22.648 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.648 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.648 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:02:22.649 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.470223+0000", 2023-10-09T06:02:22.649 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3402e-05 2023-10-09T06:02:22.649 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.649 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.649 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:02:22.649 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.470229+0000", 2023-10-09T06:02:22.649 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8300000000000001e-06 2023-10-09T06:02:22.649 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.649 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.649 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:02:22.649 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.486898+0000", 2023-10-09T06:02:22.649 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.016669534 2023-10-09T06:02:22.650 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.650 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.650 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:02:22.650 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.486923+0000", 2023-10-09T06:02:22.650 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4854e-05 2023-10-09T06:02:22.650 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.650 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.650 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:02:22.650 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.504951+0000", 2023-10-09T06:02:22.650 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018027964 2023-10-09T06:02:22.650 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.650 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.651 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:02:22.651 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.504973+0000", 2023-10-09T06:02:22.651 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1670000000000001e-05 2023-10-09T06:02:22.651 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.651 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:02:22.651 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.651 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.651 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.651 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45263.0:166809 7.1 e324/323 7:859a279a:::benchmark_data_smithi064_398483_object61456:head v 324'10413, mlcod=315'9249)", 2023-10-09T06:02:22.651 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:01:24.476534+0000", 2023-10-09T06:02:22.651 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 58.129524046, 2023-10-09T06:02:22.651 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.033034652999999997, 2023-10-09T06:02:22.652 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:02:22.652 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:22.652 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:02:22.652 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.652 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:02:22.652 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.476534+0000", 2023-10-09T06:02:22.652 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.652 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.652 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.652 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:02:22.652 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.476534+0000", 2023-10-09T06:02:22.652 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.653 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.653 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.653 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:02:22.653 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.476533+0000", 2023-10-09T06:02:22.653 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:02:22.653 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.653 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.653 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:02:22.653 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.476544+0000", 2023-10-09T06:02:22.653 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1255e-05 2023-10-09T06:02:22.653 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.653 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.654 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:02:22.654 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.476559+0000", 2023-10-09T06:02:22.654 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5191000000000001e-05 2023-10-09T06:02:22.654 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.654 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.654 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:02:22.654 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.476566+0000", 2023-10-09T06:02:22.654 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.404e-06 2023-10-09T06:02:22.654 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.654 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.654 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:02:22.655 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.491461+0000", 2023-10-09T06:02:22.655 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.014895736999999999 2023-10-09T06:02:22.655 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.655 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.655 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:02:22.655 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.491487+0000", 2023-10-09T06:02:22.655 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.5792e-05 2023-10-09T06:02:22.655 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.655 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.655 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:02:22.655 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.509311+0000", 2023-10-09T06:02:22.655 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.017823987999999999 2023-10-09T06:02:22.656 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.656 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.656 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:02:22.656 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:24.509568+0000", 2023-10-09T06:02:22.656 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00025728099999999999 2023-10-09T06:02:22.656 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.656 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:02:22.656 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.656 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.656 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.656 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45263.0:170440 7.3 e325/323 7:ccc849ff:::benchmark_data_smithi064_398483_object65087:head v 325'10593, mlcod=315'9230)", 2023-10-09T06:02:22.656 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:01:28.778123+0000", 2023-10-09T06:02:22.657 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 53.827934968000001, 2023-10-09T06:02:22.657 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.038656342000000003, 2023-10-09T06:02:22.657 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:02:22.657 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:22.657 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:02:22.657 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.657 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:02:22.657 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:28.778123+0000", 2023-10-09T06:02:22.657 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.657 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.657 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.657 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:02:22.658 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:28.778123+0000", 2023-10-09T06:02:22.658 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.658 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.658 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.658 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:02:22.658 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:28.778122+0000", 2023-10-09T06:02:22.658 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:02:22.658 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.658 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.658 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:02:22.658 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:28.778128+0000", 2023-10-09T06:02:22.659 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9599999999999997e-06 2023-10-09T06:02:22.659 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.659 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.659 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:02:22.659 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:28.778136+0000", 2023-10-09T06:02:22.659 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.371e-06 2023-10-09T06:02:22.659 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.659 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.659 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:02:22.659 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:28.778141+0000", 2023-10-09T06:02:22.659 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1620000000000001e-06 2023-10-09T06:02:22.659 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.660 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.660 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:02:22.660 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:28.778179+0000", 2023-10-09T06:02:22.660 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8806000000000001e-05 2023-10-09T06:02:22.660 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.660 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.660 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:02:22.660 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:28.778202+0000", 2023-10-09T06:02:22.660 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2289e-05 2023-10-09T06:02:22.660 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.660 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.660 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:02:22.661 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:28.783272+0000", 2023-10-09T06:02:22.661 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0050699309999999997 2023-10-09T06:02:22.661 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.661 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.661 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:02:22.661 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:28.816779+0000", 2023-10-09T06:02:22.661 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.033507442999999998 2023-10-09T06:02:22.661 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.661 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:02:22.661 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.661 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.661 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.662 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45263.0:170444 7.9 e325/323 7:9ffb2999:::benchmark_data_smithi064_398483_object65091:head v 325'10744, mlcod=315'9253)", 2023-10-09T06:02:22.662 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:01:28.780883+0000", 2023-10-09T06:02:22.662 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 53.825174965999999, 2023-10-09T06:02:22.662 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.035936067000000002, 2023-10-09T06:02:22.662 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:02:22.662 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:22.662 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:02:22.662 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.662 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:02:22.662 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:28.780883+0000", 2023-10-09T06:02:22.662 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.662 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.663 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.663 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:02:22.663 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:28.780883+0000", 2023-10-09T06:02:22.663 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.663 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.663 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.663 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:02:22.663 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:28.780881+0000", 2023-10-09T06:02:22.663 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:22.663 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.663 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.664 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:02:22.664 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:28.780890+0000", 2023-10-09T06:02:22.664 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.242e-06 2023-10-09T06:02:22.664 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.664 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.664 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:02:22.664 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:28.780900+0000", 2023-10-09T06:02:22.664 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0652000000000001e-05 2023-10-09T06:02:22.664 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.664 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.664 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:02:22.664 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:28.780907+0000", 2023-10-09T06:02:22.665 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1689999999999997e-06 2023-10-09T06:02:22.665 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.665 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.665 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:02:22.665 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:28.780954+0000", 2023-10-09T06:02:22.665 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7425000000000001e-05 2023-10-09T06:02:22.665 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.665 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.665 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:02:22.665 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:28.780976+0000", 2023-10-09T06:02:22.665 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2453e-05 2023-10-09T06:02:22.665 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.666 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.666 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:02:22.666 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:28.816778+0000", 2023-10-09T06:02:22.666 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.035801604000000001 2023-10-09T06:02:22.666 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.666 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.666 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:02:22.666 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:28.816819+0000", 2023-10-09T06:02:22.666 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0751999999999999e-05 2023-10-09T06:02:22.666 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.666 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:02:22.666 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.667 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.667 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.667 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45263.0:174382 7.3 e327/323 7:caf7a445:::benchmark_data_smithi064_398483_object69029:head v 327'10833, mlcod=315'9230)", 2023-10-09T06:02:22.667 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:01:34.444802+0000", 2023-10-09T06:02:22.667 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 48.161255435000001, 2023-10-09T06:02:22.667 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.18731925899999999, 2023-10-09T06:02:22.667 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:02:22.667 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:22.667 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:02:22.667 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.667 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:02:22.668 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444802+0000", 2023-10-09T06:02:22.668 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.668 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.668 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.668 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:02:22.668 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444802+0000", 2023-10-09T06:02:22.668 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.668 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.668 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.668 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:02:22.668 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444802+0000", 2023-10-09T06:02:22.668 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:02:22.669 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.669 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.669 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:02:22.669 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444806+0000", 2023-10-09T06:02:22.669 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5040000000000004e-06 2023-10-09T06:02:22.669 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.669 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.669 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:02:22.669 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444814+0000", 2023-10-09T06:02:22.669 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3919999999999997e-06 2023-10-09T06:02:22.669 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.670 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.670 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:02:22.670 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444817+0000", 2023-10-09T06:02:22.670 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6540000000000001e-06 2023-10-09T06:02:22.670 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.670 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.670 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:02:22.670 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444874+0000", 2023-10-09T06:02:22.670 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6635000000000002e-05 2023-10-09T06:02:22.670 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.670 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.670 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:02:22.671 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444889+0000", 2023-10-09T06:02:22.671 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5128e-05 2023-10-09T06:02:22.671 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.671 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.671 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:02:22.671 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.632091+0000", 2023-10-09T06:02:22.671 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.18720204500000001 2023-10-09T06:02:22.671 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.671 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.671 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:02:22.671 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.632122+0000", 2023-10-09T06:02:22.671 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0380999999999999e-05 2023-10-09T06:02:22.672 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.672 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:02:22.672 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.672 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.672 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.672 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45263.0:174383 7.1 e327/323 7:8ed0fc45:::benchmark_data_smithi064_398483_object69030:head v 327'10884, mlcod=315'9249)", 2023-10-09T06:02:22.672 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:01:34.444859+0000", 2023-10-09T06:02:22.672 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 48.161198524, 2023-10-09T06:02:22.672 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.191513771, 2023-10-09T06:02:22.672 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:02:22.672 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:22.672 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:02:22.673 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.673 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:02:22.673 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444859+0000", 2023-10-09T06:02:22.673 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.673 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.673 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.673 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:02:22.673 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444859+0000", 2023-10-09T06:02:22.673 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.673 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.673 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.674 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:02:22.674 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444859+0000", 2023-10-09T06:02:22.674 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:02:22.674 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.674 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.674 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:02:22.674 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444862+0000", 2023-10-09T06:02:22.674 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0620000000000001e-06 2023-10-09T06:02:22.674 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.674 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.674 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:02:22.674 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444869+0000", 2023-10-09T06:02:22.675 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4379999999999998e-06 2023-10-09T06:02:22.675 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.675 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.675 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:02:22.675 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444872+0000", 2023-10-09T06:02:22.675 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.994e-06 2023-10-09T06:02:22.675 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.675 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.675 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:02:22.675 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444903+0000", 2023-10-09T06:02:22.675 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0527000000000001e-05 2023-10-09T06:02:22.676 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.676 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.676 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:02:22.676 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444916+0000", 2023-10-09T06:02:22.676 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3030999999999999e-05 2023-10-09T06:02:22.676 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.676 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.676 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:02:22.676 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.636329+0000", 2023-10-09T06:02:22.676 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.191413368 2023-10-09T06:02:22.676 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.676 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.677 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:02:22.677 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.636373+0000", 2023-10-09T06:02:22.677 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3650999999999999e-05 2023-10-09T06:02:22.677 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.677 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:02:22.677 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.677 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.677 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.677 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45263.0:174381 7.9 e327/323 7:94e65e08:::benchmark_data_smithi064_398483_object69028:head v 327'10979, mlcod=318'9703)", 2023-10-09T06:02:22.677 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:01:34.445377+0000", 2023-10-09T06:02:22.677 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 48.160680571, 2023-10-09T06:02:22.678 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.193574469, 2023-10-09T06:02:22.678 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:02:22.678 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:22.678 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:02:22.678 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.678 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:02:22.678 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.445377+0000", 2023-10-09T06:02:22.678 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.678 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.678 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.678 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:02:22.679 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.445377+0000", 2023-10-09T06:02:22.679 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.679 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.679 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.679 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:02:22.679 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.445377+0000", 2023-10-09T06:02:22.679 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:02:22.679 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.679 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.679 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:02:22.679 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.445383+0000", 2023-10-09T06:02:22.680 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9270000000000002e-06 2023-10-09T06:02:22.680 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.680 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.680 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:02:22.680 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.445388+0000", 2023-10-09T06:02:22.680 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9420000000000003e-06 2023-10-09T06:02:22.680 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.680 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.680 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:02:22.680 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.445391+0000", 2023-10-09T06:02:22.680 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6090000000000001e-06 2023-10-09T06:02:22.680 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.681 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.681 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:02:22.681 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.445419+0000", 2023-10-09T06:02:22.681 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7396e-05 2023-10-09T06:02:22.681 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.681 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.681 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:02:22.681 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.445438+0000", 2023-10-09T06:02:22.681 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9558000000000001e-05 2023-10-09T06:02:22.681 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.681 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.681 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:02:22.682 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.638912+0000", 2023-10-09T06:02:22.682 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.19347399400000001 2023-10-09T06:02:22.682 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.682 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.682 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:02:22.682 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.638952+0000", 2023-10-09T06:02:22.682 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9363000000000002e-05 2023-10-09T06:02:22.682 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.682 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:02:22.682 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.682 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.683 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.683 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45263.0:199564 7.1 7:8455ddc3:::benchmark_data_smithi064_398483_object94211:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2023-10-09T06:02:22.683 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:02:14.067621+0000", 2023-10-09T06:02:22.683 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 8.5384366269999994, 2023-10-09T06:02:22.683 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.104825548, 2023-10-09T06:02:22.683 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:02:22.683 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:22.683 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:02:22.683 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45263", 2023-10-09T06:02:22.683 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:02:22.683 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 199564 2023-10-09T06:02:22.683 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.684 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:02:22.684 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.684 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:02:22.684 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:14.067621+0000", 2023-10-09T06:02:22.684 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.684 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.684 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.684 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:02:22.684 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:14.067621+0000", 2023-10-09T06:02:22.684 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.684 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.685 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.685 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:02:22.685 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:14.067619+0000", 2023-10-09T06:02:22.685 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999976 2023-10-09T06:02:22.685 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.685 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.685 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:02:22.685 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:14.067627+0000", 2023-10-09T06:02:22.685 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9780000000000005e-06 2023-10-09T06:02:22.685 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.685 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.685 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:02:22.686 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:14.067646+0000", 2023-10-09T06:02:22.686 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8746e-05 2023-10-09T06:02:22.686 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.686 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.686 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:02:22.686 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:14.067653+0000", 2023-10-09T06:02:22.686 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2189999999999998e-06 2023-10-09T06:02:22.686 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.686 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.686 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:02:22.686 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:14.067699+0000", 2023-10-09T06:02:22.686 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6606000000000001e-05 2023-10-09T06:02:22.687 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.687 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.687 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:02:22.687 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:14.067835+0000", 2023-10-09T06:02:22.687 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013606299999999999 2023-10-09T06:02:22.687 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.687 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.687 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:02:22.687 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:14.067952+0000", 2023-10-09T06:02:22.687 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000116558 2023-10-09T06:02:22.687 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.688 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.688 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:02:22.688 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:14.070603+0000", 2023-10-09T06:02:22.688 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00265146 2023-10-09T06:02:22.688 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.688 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.688 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:22.688 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:14.073180+0000", 2023-10-09T06:02:22.688 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0025764719999999998 2023-10-09T06:02:22.688 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.688 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.688 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:22.689 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:14.172354+0000", 2023-10-09T06:02:22.689 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.099174444 2023-10-09T06:02:22.689 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.689 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.689 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:02:22.689 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:14.172404+0000", 2023-10-09T06:02:22.689 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9555000000000002e-05 2023-10-09T06:02:22.689 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.689 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.689 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:02:22.689 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:14.172447+0000", 2023-10-09T06:02:22.689 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2698999999999999e-05 2023-10-09T06:02:22.690 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.690 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:02:22.690 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.690 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.690 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.690 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45263.0:201287 7.1 7:890025c2:::benchmark_data_smithi064_398483_object95934:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T06:02:22.690 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:02:17.380816+0000", 2023-10-09T06:02:22.690 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 5.2252420690000001, 2023-10-09T06:02:22.690 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.038631440000000003, 2023-10-09T06:02:22.690 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:02:22.690 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:22.690 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:02:22.691 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45263", 2023-10-09T06:02:22.691 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:02:22.691 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 201287 2023-10-09T06:02:22.691 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.691 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:02:22.691 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.691 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:02:22.691 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:17.380816+0000", 2023-10-09T06:02:22.691 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.691 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.691 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.692 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:02:22.692 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:17.380816+0000", 2023-10-09T06:02:22.692 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.692 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.692 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.692 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:02:22.692 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:17.380813+0000", 2023-10-09T06:02:22.692 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999976 2023-10-09T06:02:22.692 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.692 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.692 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:02:22.692 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:17.380823+0000", 2023-10-09T06:02:22.693 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2240000000000001e-06 2023-10-09T06:02:22.693 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.693 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.693 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:02:22.693 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:17.380840+0000", 2023-10-09T06:02:22.693 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7595999999999999e-05 2023-10-09T06:02:22.693 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.693 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.693 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:02:22.693 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:17.380848+0000", 2023-10-09T06:02:22.693 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7230000000000004e-06 2023-10-09T06:02:22.693 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.694 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.694 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:02:22.694 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:17.381066+0000", 2023-10-09T06:02:22.694 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00021833000000000001 2023-10-09T06:02:22.694 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.694 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.694 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:02:22.694 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:17.381208+0000", 2023-10-09T06:02:22.694 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000141979 2023-10-09T06:02:22.694 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.694 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.695 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:02:22.695 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:17.381283+0000", 2023-10-09T06:02:22.695 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5044999999999997e-05 2023-10-09T06:02:22.695 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.695 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.695 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:22.695 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:17.388124+0000", 2023-10-09T06:02:22.695 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0068406639999999998 2023-10-09T06:02:22.695 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.695 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.695 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:02:22.696 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:17.390393+0000", 2023-10-09T06:02:22.696 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00226917 2023-10-09T06:02:22.696 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.696 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.696 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:22.696 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:17.419347+0000", 2023-10-09T06:02:22.696 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.028954177000000001 2023-10-09T06:02:22.696 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.696 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.696 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:02:22.696 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:17.419403+0000", 2023-10-09T06:02:22.696 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5782999999999998e-05 2023-10-09T06:02:22.697 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.697 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.697 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:02:22.697 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:17.419447+0000", 2023-10-09T06:02:22.697 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3968999999999999e-05 2023-10-09T06:02:22.697 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.697 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:02:22.697 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.697 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.697 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.697 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45263.0:201414 7.4 e348/340 7:28a9459f:::benchmark_data_smithi064_398483_object96061:head v 348'12590, mlcod=347'12588)", 2023-10-09T06:02:22.697 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:02:18.496573+0000", 2023-10-09T06:02:22.698 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 4.1094848869999998, 2023-10-09T06:02:22.698 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.091274991, 2023-10-09T06:02:22.698 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:02:22.698 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:22.698 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:02:22.698 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.698 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:02:22.698 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.496573+0000", 2023-10-09T06:02:22.698 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.698 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.698 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.699 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:02:22.699 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.496573+0000", 2023-10-09T06:02:22.699 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.699 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.699 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.699 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:02:22.699 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.496572+0000", 2023-10-09T06:02:22.699 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:02:22.699 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.699 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.699 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:02:22.699 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.496583+0000", 2023-10-09T06:02:22.699 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1178e-05 2023-10-09T06:02:22.700 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.700 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.700 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:02:22.700 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.576246+0000", 2023-10-09T06:02:22.700 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.079662469999999999 2023-10-09T06:02:22.700 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.700 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.700 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:02:22.700 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.576255+0000", 2023-10-09T06:02:22.700 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3230000000000004e-06 2023-10-09T06:02:22.700 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.701 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.701 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:02:22.701 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.576360+0000", 2023-10-09T06:02:22.701 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010498899999999999 2023-10-09T06:02:22.701 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.701 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.701 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:02:22.701 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.576382+0000", 2023-10-09T06:02:22.701 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2090999999999999e-05 2023-10-09T06:02:22.701 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.701 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.701 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:02:22.702 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.587810+0000", 2023-10-09T06:02:22.702 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011427619 2023-10-09T06:02:22.702 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.702 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.702 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:02:22.702 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.587848+0000", 2023-10-09T06:02:22.702 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8071999999999997e-05 2023-10-09T06:02:22.702 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.702 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:02:22.702 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.703 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.703 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.703 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45263.0:202089 7.1 7:81989e88:::benchmark_data_smithi064_398483_object96736:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-10-09T06:02:22.703 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:02:19.489809+0000", 2023-10-09T06:02:22.703 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 3.1162491659999998, 2023-10-09T06:02:22.703 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.053726133000000002, 2023-10-09T06:02:22.703 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:02:22.703 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:22.703 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:02:22.703 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45263", 2023-10-09T06:02:22.703 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:02:22.703 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 202089 2023-10-09T06:02:22.704 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.704 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:02:22.704 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.704 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:02:22.704 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.489809+0000", 2023-10-09T06:02:22.704 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.704 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.704 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.704 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:02:22.704 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.489809+0000", 2023-10-09T06:02:22.704 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.704 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.705 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.705 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:02:22.705 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.489807+0000", 2023-10-09T06:02:22.705 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:22.705 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.705 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.705 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:02:22.705 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.489818+0000", 2023-10-09T06:02:22.705 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0842000000000001e-05 2023-10-09T06:02:22.705 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.705 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.705 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:02:22.706 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.489833+0000", 2023-10-09T06:02:22.706 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4523e-05 2023-10-09T06:02:22.706 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.706 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.706 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:02:22.706 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.489839+0000", 2023-10-09T06:02:22.706 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4019999999999999e-06 2023-10-09T06:02:22.706 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.706 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.706 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:02:22.706 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.489895+0000", 2023-10-09T06:02:22.706 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5627000000000001e-05 2023-10-09T06:02:22.707 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.707 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.707 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:02:22.707 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.490049+0000", 2023-10-09T06:02:22.707 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000154382 2023-10-09T06:02:22.707 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.707 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.707 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:02:22.707 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.490189+0000", 2023-10-09T06:02:22.707 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000140168 2023-10-09T06:02:22.707 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.707 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.708 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:02:22.708 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.493223+0000", 2023-10-09T06:02:22.708 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0030335280000000002 2023-10-09T06:02:22.708 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.708 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.708 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:22.708 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.493636+0000", 2023-10-09T06:02:22.708 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00041289000000000001 2023-10-09T06:02:22.708 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.708 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.708 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:22.708 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.543434+0000", 2023-10-09T06:02:22.709 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.049798569000000001 2023-10-09T06:02:22.709 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.709 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.709 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:02:22.709 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.543475+0000", 2023-10-09T06:02:22.709 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0960000000000001e-05 2023-10-09T06:02:22.709 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.709 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.709 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:02:22.709 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.543535+0000", 2023-10-09T06:02:22.709 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9567000000000001e-05 2023-10-09T06:02:22.710 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.710 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:02:22.710 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.710 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.710 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.710 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45263.0:202098 7.1 7:89419be0:::benchmark_data_smithi064_398483_object96745:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-10-09T06:02:22.710 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:02:19.508533+0000", 2023-10-09T06:02:22.710 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 3.0975248280000001, 2023-10-09T06:02:22.710 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.035171501000000001, 2023-10-09T06:02:22.710 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:02:22.710 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:22.711 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:02:22.711 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45263", 2023-10-09T06:02:22.711 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:02:22.711 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 202098 2023-10-09T06:02:22.711 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.711 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:02:22.711 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.711 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:02:22.711 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.508533+0000", 2023-10-09T06:02:22.711 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.711 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.711 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.712 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:02:22.712 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.508533+0000", 2023-10-09T06:02:22.712 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.712 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.712 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.712 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:02:22.712 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.508531+0000", 2023-10-09T06:02:22.712 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T06:02:22.712 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.712 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.713 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:02:22.713 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.508539+0000", 2023-10-09T06:02:22.713 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9179999999999994e-06 2023-10-09T06:02:22.713 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.713 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.713 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:02:22.713 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.508557+0000", 2023-10-09T06:02:22.713 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7518999999999999e-05 2023-10-09T06:02:22.713 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.713 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.713 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:02:22.714 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.508563+0000", 2023-10-09T06:02:22.714 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.601e-06 2023-10-09T06:02:22.714 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.714 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.714 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:02:22.714 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.508602+0000", 2023-10-09T06:02:22.714 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8498000000000003e-05 2023-10-09T06:02:22.714 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.714 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.714 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:02:22.714 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.508710+0000", 2023-10-09T06:02:22.715 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010809100000000001 2023-10-09T06:02:22.715 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.715 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.715 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:02:22.715 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.508794+0000", 2023-10-09T06:02:22.715 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.4469999999999996e-05 2023-10-09T06:02:22.715 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.715 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.715 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:02:22.715 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.511591+0000", 2023-10-09T06:02:22.715 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0027963520000000002 2023-10-09T06:02:22.716 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.716 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.716 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:22.716 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.512262+0000", 2023-10-09T06:02:22.716 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00067134400000000004 2023-10-09T06:02:22.716 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.716 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.716 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:22.716 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.543624+0000", 2023-10-09T06:02:22.716 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.031361972000000002 2023-10-09T06:02:22.716 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.717 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.717 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:02:22.717 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.543668+0000", 2023-10-09T06:02:22.717 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3778e-05 2023-10-09T06:02:22.717 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.717 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.717 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:02:22.717 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:19.543704+0000", 2023-10-09T06:02:22.717 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6757000000000001e-05 2023-10-09T06:02:22.717 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.717 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:02:22.717 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.718 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.718 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.718 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45263.0:203132 7.1 7:8fb42464:::benchmark_data_smithi064_398483_object97779:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e349)", 2023-10-09T06:02:22.718 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:02:21.255037+0000", 2023-10-09T06:02:22.718 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 1.3510207949999999, 2023-10-09T06:02:22.718 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.104856966, 2023-10-09T06:02:22.718 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:02:22.718 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:22.718 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:02:22.718 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45263", 2023-10-09T06:02:22.718 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:02:22.719 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 203132 2023-10-09T06:02:22.719 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.719 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:02:22.719 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.719 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:02:22.719 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:21.255037+0000", 2023-10-09T06:02:22.719 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.719 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.719 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.719 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:02:22.719 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:21.255037+0000", 2023-10-09T06:02:22.719 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:02:22.720 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.720 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.720 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:02:22.720 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:21.255035+0000", 2023-10-09T06:02:22.720 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T06:02:22.720 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.720 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.720 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:02:22.720 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:21.255045+0000", 2023-10-09T06:02:22.720 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.023e-05 2023-10-09T06:02:22.720 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.720 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.721 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:02:22.721 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:21.255067+0000", 2023-10-09T06:02:22.721 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1840000000000001e-05 2023-10-09T06:02:22.721 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.721 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.721 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:02:22.721 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:21.255076+0000", 2023-10-09T06:02:22.721 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6999999999999997e-06 2023-10-09T06:02:22.721 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.721 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.721 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:02:22.722 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:21.255129+0000", 2023-10-09T06:02:22.722 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3597999999999997e-05 2023-10-09T06:02:22.722 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.722 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.722 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:02:22.722 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:21.255233+0000", 2023-10-09T06:02:22.722 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000103899 2023-10-09T06:02:22.722 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.722 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.722 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:02:22.722 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:21.255318+0000", 2023-10-09T06:02:22.722 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.4268e-05 2023-10-09T06:02:22.723 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.723 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.723 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:02:22.723 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:21.258246+0000", 2023-10-09T06:02:22.723 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00292881 2023-10-09T06:02:22.723 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.723 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.723 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:22.723 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:21.258496+0000", 2023-10-09T06:02:22.723 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00024981500000000002 2023-10-09T06:02:22.723 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.723 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.724 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:22.724 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:21.359787+0000", 2023-10-09T06:02:22.724 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.101290571 2023-10-09T06:02:22.724 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.724 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.724 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:02:22.724 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:21.359846+0000", 2023-10-09T06:02:22.724 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9407999999999998e-05 2023-10-09T06:02:22.724 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:02:22.724 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:02:22.724 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:02:22.724 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:21.359894+0000", 2023-10-09T06:02:22.725 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7623999999999997e-05 2023-10-09T06:02:22.725 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.725 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:02:22.725 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.725 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:02:22.725 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:02:22.725 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:02:22.725 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:22.726 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:02:22.726 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:02:22.727 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:22 smithi064 ceph-mon[220044]: pgmap v823: 119 pgs: 1 active+clean+scrubbing, 3 active+remapped+backfill_wait, 115 active+clean; 532 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 984/25590 objects misplaced (3.845%); 10 MiB/s, 163 objects/s recovering 2023-10-09T06:02:22.729 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:02:22.773 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:02:22.810 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:02:22.833 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:22.851 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:02:22.872 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:02:22.872 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:02:22.872 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:02:22.872 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:02:22.872 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:02:22.872 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:02:22.872 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:02:22.872 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:02:23.164 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:23.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:23.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:23.268 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:23.312 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:23.507 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:23.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:23.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:23.559 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:02:23.611 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:23.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:23 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1421688834' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:23.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:23 smithi156 ceph-mon[195101]: pgmap v824: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 115 active+clean; 509 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 481 op/s; 460/24498 objects misplaced (1.878%); 14 MiB/s, 219 objects/s recovering 2023-10-09T06:02:23.638 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:23.684 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:23.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:23.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:23.708 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 0, 4] 2023-10-09T06:02:23.708 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T06:02:23.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:23 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1421688834' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:23.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:23 smithi064 ceph-mon[212750]: pgmap v824: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 115 active+clean; 509 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 481 op/s; 460/24498 objects misplaced (1.878%); 14 MiB/s, 219 objects/s recovering 2023-10-09T06:02:23.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:23 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1421688834' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:23.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:23 smithi064 ceph-mon[220044]: pgmap v824: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 115 active+clean; 509 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 481 op/s; 460/24498 objects misplaced (1.878%); 14 MiB/s, 219 objects/s recovering 2023-10-09T06:02:23.787 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:23.813 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:02:23.886 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:24.245 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:24.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:24.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:24.352 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:24.385 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:24.392 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:24.538 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:24.563 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:24.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:24.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:24.578 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:24 smithi064 ceph-mon[220044]: osdmap e350: 8 total, 8 up, 8 in 2023-10-09T06:02:24.579 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:24 smithi064 ceph-mon[220044]: failed to encode map e350 with expected crc 2023-10-09T06:02:24.579 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:24 smithi064 ceph-mon[220044]: failed to encode map e350 with expected crc 2023-10-09T06:02:24.579 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:24 smithi064 ceph-mon[220044]: failed to encode map e350 with expected crc 2023-10-09T06:02:24.579 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:24 smithi064 ceph-mon[220044]: failed to encode map e350 with expected crc 2023-10-09T06:02:24.582 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:24 smithi064 ceph-mon[212750]: osdmap e350: 8 total, 8 up, 8 in 2023-10-09T06:02:24.582 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:24 smithi064 ceph-mon[212750]: failed to encode map e350 with expected crc 2023-10-09T06:02:24.582 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:24 smithi064 ceph-mon[212750]: failed to encode map e350 with expected crc 2023-10-09T06:02:24.582 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:24 smithi064 ceph-mon[212750]: failed to encode map e350 with expected crc 2023-10-09T06:02:24.582 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:24 smithi064 ceph-mon[212750]: failed to encode map e350 with expected crc 2023-10-09T06:02:24.676 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:24.747 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:24.825 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:24.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:24.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:24.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:24 smithi156 ceph-mon[195101]: osdmap e350: 8 total, 8 up, 8 in 2023-10-09T06:02:24.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:24 smithi156 ceph-mon[195101]: failed to encode map e350 with expected crc 2023-10-09T06:02:24.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:24 smithi156 ceph-mon[195101]: failed to encode map e350 with expected crc 2023-10-09T06:02:24.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:24 smithi156 ceph-mon[195101]: failed to encode map e350 with expected crc 2023-10-09T06:02:24.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:24 smithi156 ceph-mon[195101]: failed to encode map e350 with expected crc 2023-10-09T06:02:24.929 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:24.957 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:24.998 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:25.155 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:25.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:25.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:25.260 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:25.289 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:25.334 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:25.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:25.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:25.413 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:02:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:02:25] "GET /metrics HTTP/1.1" 200 36413 "" "Prometheus/2.43.0" 2023-10-09T06:02:25.440 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:25.478 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:25.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:25 smithi156 ceph-mon[195101]: osdmap e351: 8 total, 8 up, 8 in 2023-10-09T06:02:25.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:25 smithi156 ceph-mon[195101]: failed to encode map e351 with expected crc 2023-10-09T06:02:25.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:25 smithi156 ceph-mon[195101]: failed to encode map e351 with expected crc 2023-10-09T06:02:25.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:25 smithi156 ceph-mon[195101]: failed to encode map e351 with expected crc 2023-10-09T06:02:25.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:25 smithi156 ceph-mon[195101]: failed to encode map e351 with expected crc 2023-10-09T06:02:25.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:25 smithi156 ceph-mon[195101]: pgmap v827: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 115 active+clean; 470 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 460/22596 objects misplaced (2.036%); 14 MiB/s, 219 objects/s recovering 2023-10-09T06:02:25.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:25 smithi156 ceph-mon[195101]: osdmap e352: 8 total, 8 up, 8 in 2023-10-09T06:02:25.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:25 smithi064 ceph-mon[220044]: osdmap e351: 8 total, 8 up, 8 in 2023-10-09T06:02:25.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:25 smithi064 ceph-mon[220044]: failed to encode map e351 with expected crc 2023-10-09T06:02:25.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:25 smithi064 ceph-mon[220044]: failed to encode map e351 with expected crc 2023-10-09T06:02:25.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:25 smithi064 ceph-mon[220044]: failed to encode map e351 with expected crc 2023-10-09T06:02:25.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:25 smithi064 ceph-mon[220044]: failed to encode map e351 with expected crc 2023-10-09T06:02:25.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:25 smithi064 ceph-mon[220044]: pgmap v827: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 115 active+clean; 470 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 460/22596 objects misplaced (2.036%); 14 MiB/s, 219 objects/s recovering 2023-10-09T06:02:25.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:25 smithi064 ceph-mon[220044]: osdmap e352: 8 total, 8 up, 8 in 2023-10-09T06:02:25.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:25 smithi064 ceph-mon[212750]: osdmap e351: 8 total, 8 up, 8 in 2023-10-09T06:02:25.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:25 smithi064 ceph-mon[212750]: failed to encode map e351 with expected crc 2023-10-09T06:02:25.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:25 smithi064 ceph-mon[212750]: failed to encode map e351 with expected crc 2023-10-09T06:02:25.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:25 smithi064 ceph-mon[212750]: failed to encode map e351 with expected crc 2023-10-09T06:02:25.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:25 smithi064 ceph-mon[212750]: failed to encode map e351 with expected crc 2023-10-09T06:02:25.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:25 smithi064 ceph-mon[212750]: pgmap v827: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 115 active+clean; 470 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 460/22596 objects misplaced (2.036%); 14 MiB/s, 219 objects/s recovering 2023-10-09T06:02:25.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:25 smithi064 ceph-mon[212750]: osdmap e352: 8 total, 8 up, 8 in 2023-10-09T06:02:26.257 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:02:26.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:26 smithi064 ceph-mon[220044]: failed to encode map e352 with expected crc 2023-10-09T06:02:26.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:26 smithi064 ceph-mon[220044]: failed to encode map e352 with expected crc 2023-10-09T06:02:26.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:26 smithi064 ceph-mon[220044]: failed to encode map e352 with expected crc 2023-10-09T06:02:26.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:26 smithi064 ceph-mon[220044]: failed to encode map e352 with expected crc 2023-10-09T06:02:26.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:26 smithi064 ceph-mon[220044]: osdmap e353: 8 total, 8 up, 8 in 2023-10-09T06:02:26.724 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:26 smithi064 ceph-mon[212750]: failed to encode map e352 with expected crc 2023-10-09T06:02:26.724 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:26 smithi064 ceph-mon[212750]: failed to encode map e352 with expected crc 2023-10-09T06:02:26.724 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:26 smithi064 ceph-mon[212750]: failed to encode map e352 with expected crc 2023-10-09T06:02:26.724 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:26 smithi064 ceph-mon[212750]: failed to encode map e352 with expected crc 2023-10-09T06:02:26.725 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:26 smithi064 ceph-mon[212750]: osdmap e353: 8 total, 8 up, 8 in 2023-10-09T06:02:26.859 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:26.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:26.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:26.876 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:26 smithi156 ceph-mon[195101]: failed to encode map e352 with expected crc 2023-10-09T06:02:26.876 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:26 smithi156 ceph-mon[195101]: failed to encode map e352 with expected crc 2023-10-09T06:02:26.876 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:26 smithi156 ceph-mon[195101]: failed to encode map e352 with expected crc 2023-10-09T06:02:26.876 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:26 smithi156 ceph-mon[195101]: failed to encode map e352 with expected crc 2023-10-09T06:02:26.877 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:26 smithi156 ceph-mon[195101]: osdmap e353: 8 total, 8 up, 8 in 2023-10-09T06:02:26.964 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:27.029 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:27.630 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:27.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:27.646 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:27 smithi064 ceph-mon[212750]: failed to encode map e353 with expected crc 2023-10-09T06:02:27.646 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:27 smithi064 ceph-mon[212750]: failed to encode map e353 with expected crc 2023-10-09T06:02:27.646 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:27 smithi064 ceph-mon[212750]: failed to encode map e353 with expected crc 2023-10-09T06:02:27.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:27 smithi064 ceph-mon[212750]: failed to encode map e353 with expected crc 2023-10-09T06:02:27.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:27 smithi064 ceph-mon[212750]: pgmap v830: 119 pgs: 1 active+clean+remapped, 1 peering, 117 active+clean; 349 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 5.0 MiB/s, 80 objects/s recovering 2023-10-09T06:02:27.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:27.648 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:27 smithi064 ceph-mon[220044]: failed to encode map e353 with expected crc 2023-10-09T06:02:27.648 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:27 smithi064 ceph-mon[220044]: failed to encode map e353 with expected crc 2023-10-09T06:02:27.648 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:27 smithi064 ceph-mon[220044]: failed to encode map e353 with expected crc 2023-10-09T06:02:27.648 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:27 smithi064 ceph-mon[220044]: failed to encode map e353 with expected crc 2023-10-09T06:02:27.648 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:27 smithi064 ceph-mon[220044]: pgmap v830: 119 pgs: 1 active+clean+remapped, 1 peering, 117 active+clean; 349 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 5.0 MiB/s, 80 objects/s recovering 2023-10-09T06:02:27.748 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:27.776 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:27 smithi156 ceph-mon[195101]: failed to encode map e353 with expected crc 2023-10-09T06:02:27.777 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:27 smithi156 ceph-mon[195101]: failed to encode map e353 with expected crc 2023-10-09T06:02:27.777 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:27 smithi156 ceph-mon[195101]: failed to encode map e353 with expected crc 2023-10-09T06:02:27.777 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:27 smithi156 ceph-mon[195101]: failed to encode map e353 with expected crc 2023-10-09T06:02:27.777 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:27 smithi156 ceph-mon[195101]: pgmap v830: 119 pgs: 1 active+clean+remapped, 1 peering, 117 active+clean; 349 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 5.0 MiB/s, 80 objects/s recovering 2023-10-09T06:02:27.780 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:27.838 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:27.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:27.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:27.945 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:28.032 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:28.352 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:28.432 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:28.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:28.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:28.540 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:28.663 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:28.890 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:28.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:28.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:29.022 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:29.050 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:29.095 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:02:29.095 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:02:29.102 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:29.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:29.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:29.208 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:29.303 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:29.562 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:02:29.564 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T06:02:29.708 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:29.710 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:29 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:02:29.710 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:29.710 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:29 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3354936559' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:02:29.710 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:29 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:02:29.710 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:29 smithi064 ceph-mon[212750]: pgmap v831: 119 pgs: 1 active+clean+remapped, 1 peering, 117 active+clean; 315 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 3.8 MiB/s, 60 objects/s recovering 2023-10-09T06:02:29.710 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:29 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3029423519' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:02:29.710 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:29 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:02:29.713 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:29 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:02:29.713 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:29.713 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:29 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3354936559' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:02:29.713 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:29 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:02:29.713 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:29 smithi064 ceph-mon[220044]: pgmap v831: 119 pgs: 1 active+clean+remapped, 1 peering, 117 active+clean; 315 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 3.8 MiB/s, 60 objects/s recovering 2023-10-09T06:02:29.713 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:29 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3029423519' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:02:29.713 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:29 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:02:29.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:29.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:29.815 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:02:29.851 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:29 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:02:29.851 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:29.851 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:29 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3354936559' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:02:29.851 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:29 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:02:29.851 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:29 smithi156 ceph-mon[195101]: pgmap v831: 119 pgs: 1 active+clean+remapped, 1 peering, 117 active+clean; 315 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 3.8 MiB/s, 60 objects/s recovering 2023-10-09T06:02:29.851 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:29 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3029423519' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:02:29.851 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:29 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:02:29.854 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:29.915 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:02:29.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:29.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:02:30.022 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:30.084 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:30.561 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:30.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:30.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:30.665 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:30.703 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:30.777 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:30.783 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:30 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:02:30.783 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:30 smithi156 ceph-mon[195101]: failed to encode map e354 with expected crc 2023-10-09T06:02:30.783 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:30 smithi156 ceph-mon[195101]: failed to encode map e354 with expected crc 2023-10-09T06:02:30.783 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:30 smithi156 ceph-mon[195101]: failed to encode map e354 with expected crc 2023-10-09T06:02:30.783 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:30 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:02:30.783 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:30 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:02:30.784 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:30 smithi156 ceph-mon[195101]: osdmap e354: 8 total, 8 up, 8 in 2023-10-09T06:02:30.784 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:30 smithi156 ceph-mon[195101]: failed to encode map e354 with expected crc 2023-10-09T06:02:30.784 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:30 smithi156 ceph-mon[195101]: failed to encode map e354 with expected crc 2023-10-09T06:02:30.784 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:30 smithi156 ceph-mon[195101]: failed to encode map e354 with expected crc 2023-10-09T06:02:30.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:30.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:30.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:30 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:02:30.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:30 smithi064 ceph-mon[212750]: failed to encode map e354 with expected crc 2023-10-09T06:02:30.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:30 smithi064 ceph-mon[212750]: failed to encode map e354 with expected crc 2023-10-09T06:02:30.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:30 smithi064 ceph-mon[212750]: failed to encode map e354 with expected crc 2023-10-09T06:02:30.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:30 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:02:30.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:30 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:02:30.822 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:30 smithi064 ceph-mon[212750]: osdmap e354: 8 total, 8 up, 8 in 2023-10-09T06:02:30.822 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:30 smithi064 ceph-mon[212750]: failed to encode map e354 with expected crc 2023-10-09T06:02:30.822 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:30 smithi064 ceph-mon[212750]: failed to encode map e354 with expected crc 2023-10-09T06:02:30.822 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:30 smithi064 ceph-mon[212750]: failed to encode map e354 with expected crc 2023-10-09T06:02:30.822 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:30 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:02:30.822 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:30 smithi064 ceph-mon[220044]: failed to encode map e354 with expected crc 2023-10-09T06:02:30.822 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:30 smithi064 ceph-mon[220044]: failed to encode map e354 with expected crc 2023-10-09T06:02:30.822 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:30 smithi064 ceph-mon[220044]: failed to encode map e354 with expected crc 2023-10-09T06:02:30.822 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:30 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:02:30.823 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:30 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:02:30.823 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:30 smithi064 ceph-mon[220044]: osdmap e354: 8 total, 8 up, 8 in 2023-10-09T06:02:30.823 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:30 smithi064 ceph-mon[220044]: failed to encode map e354 with expected crc 2023-10-09T06:02:30.823 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:30 smithi064 ceph-mon[220044]: failed to encode map e354 with expected crc 2023-10-09T06:02:30.823 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:30 smithi064 ceph-mon[220044]: failed to encode map e354 with expected crc 2023-10-09T06:02:30.885 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:30.913 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:30.966 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:30.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:30.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:31.074 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:31.146 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:31.519 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:02:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:31 smithi156 ceph-mon[195101]: pgmap v833: 119 pgs: 119 active+clean; 173 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 3.4 MiB/s, 53 objects/s recovering 2023-10-09T06:02:31.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:31 smithi064 ceph-mon[212750]: pgmap v833: 119 pgs: 119 active+clean; 173 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 3.4 MiB/s, 53 objects/s recovering 2023-10-09T06:02:31.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:31 smithi064 ceph-mon[220044]: pgmap v833: 119 pgs: 119 active+clean; 173 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 3.4 MiB/s, 53 objects/s recovering 2023-10-09T06:02:32.060 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:32.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:32.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:32.171 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:32.258 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:32.356 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T06:02:32.502 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:32.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:32.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:32.617 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:32.650 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:32.723 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:32.730 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:32.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:32.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:32.833 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:32.869 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:32.924 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:32.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:32.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:33.030 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:33.064 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:33.120 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:33.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:33.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:33.227 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:33.298 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Removed 105350 objects 2023-10-09T06:02:33.299 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Clean up completed and total clean up time :144.281 2023-10-09T06:02:33.301 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:02:33.302 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:33.322 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:02:33.348 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:02:33.368 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:02:33.388 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:02:33.409 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:02:33.432 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:02:33.459 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:02:33.460 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:02:33.461 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:02:33.461 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:02:33.461 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:02:33.461 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:02:33.462 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:02:33.462 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:02:33.462 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:02:33.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:33 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1021721140' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:33.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:33 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1021721140' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:33.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:33 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1021721140' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:34.433 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:02:34.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:34 smithi156 ceph-mon[195101]: pgmap v834: 119 pgs: 119 active+clean; 158 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 485 op/s 2023-10-09T06:02:34.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:34 smithi064 ceph-mon[212750]: pgmap v834: 119 pgs: 119 active+clean; 158 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 485 op/s 2023-10-09T06:02:34.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:34 smithi064 ceph-mon[220044]: pgmap v834: 119 pgs: 119 active+clean; 158 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 485 op/s 2023-10-09T06:02:34.730 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:34.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:34.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:34.834 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:34.868 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:34.923 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:34.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:34.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:34.952 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:35.027 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:02:35.059 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:35.109 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:02:35.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:35.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:02:35.216 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:35.246 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:35.274 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:35.290 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:35.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:35.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:35.358 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:02:35.358 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T06:02:35.358 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T06:02:35.359 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T06:02:35.359 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:02:35.384 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:02:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:02:35] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.43.0" 2023-10-09T06:02:35.397 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:35.424 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:35.465 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:35.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:35.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:35.574 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:35.605 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:36.183 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:02:36.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:36 smithi156 ceph-mon[195101]: pgmap v835: 119 pgs: 119 active+clean; 114 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 450 op/s 2023-10-09T06:02:36.711 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:36.712 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:36 smithi064 ceph-mon[220044]: pgmap v835: 119 pgs: 119 active+clean; 114 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 450 op/s 2023-10-09T06:02:36.713 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:36 smithi064 ceph-mon[212750]: pgmap v835: 119 pgs: 119 active+clean; 114 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 450 op/s 2023-10-09T06:02:36.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:36.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:36.814 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:36.844 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:36.857 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-10-09T06:02:36.857 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-10-09T06:02:36.866 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:02:36.902 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:36.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:36.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:37.006 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:02:37.038 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:37.086 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:02:37.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:37.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:02:37.192 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:37.221 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:37.749 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:37.835 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:37.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:37.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:37.938 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:37.970 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:38.017 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:38.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:38.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:38.124 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:38.158 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:38.187 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:38.316 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:38.366 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:38 smithi064 ceph-mon[212750]: pgmap v836: 119 pgs: 119 active+clean; 2.3 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 553 op/s 2023-10-09T06:02:38.366 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:38 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2988938917' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:02:38.366 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:38 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:02:38.367 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:38 smithi064 ceph-mon[220044]: pgmap v836: 119 pgs: 119 active+clean; 2.3 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 553 op/s 2023-10-09T06:02:38.367 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:38 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2988938917' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:02:38.367 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:38 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:02:38.369 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T06:02:38.465 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:38.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:38.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:38.570 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:38.599 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:38.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:38 smithi156 ceph-mon[195101]: pgmap v836: 119 pgs: 119 active+clean; 2.3 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 553 op/s 2023-10-09T06:02:38.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:38 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2988938917' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:02:38.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:38 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:02:38.645 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:38.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:38.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:38.750 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:38.777 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:38.818 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:38.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:38.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:38.924 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:38.950 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:38.991 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:38.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:38.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:39.096 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:39.122 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:39.164 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:39.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:39.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:39.271 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:39.301 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:39.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:39 smithi064 ceph-mon[212750]: failed to encode map e355 with expected crc 2023-10-09T06:02:39.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:39 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:02:39.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:39 smithi064 ceph-mon[212750]: osdmap e355: 8 total, 8 up, 8 in 2023-10-09T06:02:39.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:39 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:02:39.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:39 smithi064 ceph-mon[220044]: failed to encode map e355 with expected crc 2023-10-09T06:02:39.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:39 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:02:39.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:39 smithi064 ceph-mon[220044]: osdmap e355: 8 total, 8 up, 8 in 2023-10-09T06:02:39.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:39 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:02:39.501 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:39.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:39.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:39.605 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:39.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:39 smithi156 ceph-mon[195101]: failed to encode map e355 with expected crc 2023-10-09T06:02:39.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:39 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:02:39.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:39 smithi156 ceph-mon[195101]: osdmap e355: 8 total, 8 up, 8 in 2023-10-09T06:02:39.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:39 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:02:39.637 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:40.101 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T06:02:40.371 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:40 smithi064 ceph-mon[220044]: pgmap v838: 119 pgs: 119 active+clean; 709 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 542 op/s 2023-10-09T06:02:40.372 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:40 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2381763936' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:02:40.384 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:02:40.587 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:40.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:40.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:40.691 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:40.718 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:40 smithi064 ceph-mon[212750]: pgmap v838: 119 pgs: 119 active+clean; 709 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 542 op/s 2023-10-09T06:02:40.718 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:40 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2381763936' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:02:40.721 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:40.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:40 smithi156 ceph-mon[195101]: pgmap v838: 119 pgs: 119 active+clean; 709 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 542 op/s 2023-10-09T06:02:40.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:40 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2381763936' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:02:41.264 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:02:41.264 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T06:02:41.265 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T06:02:41.265 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:02:41.265 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.265 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:99073 6.4 6:27b1f89e:::benchmark_data_smithi064_284193_object2530:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-10-09T06:02:41.265 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:52:42.783709+0000", 2023-10-09T06:02:41.265 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 598.47823541499997, 2023-10-09T06:02:41.265 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4128961629999992, 2023-10-09T06:02:41.265 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:02:41.265 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:41.265 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:02:41.265 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:02:41.266 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:02:41.266 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 99073 2023-10-09T06:02:41.266 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.266 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:02:41.266 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.266 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:02:41.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:42.783709+0000", 2023-10-09T06:02:41.266 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.266 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.266 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:02:41.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:42.783709+0000", 2023-10-09T06:02:41.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:02:41.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:42.783708+0000", 2023-10-09T06:02:41.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:41.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:02:41.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:42.783714+0000", 2023-10-09T06:02:41.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.686e-06 2023-10-09T06:02:41.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.268 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.268 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:02:41.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:42.783728+0000", 2023-10-09T06:02:41.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3182e-05 2023-10-09T06:02:41.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.268 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.268 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:02:41.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:42.783735+0000", 2023-10-09T06:02:41.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3300000000000001e-06 2023-10-09T06:02:41.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.268 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:52.071234+0000", 2023-10-09T06:02:41.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2874990569999998 2023-10-09T06:02:41.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:02:41.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:52.083105+0000", 2023-10-09T06:02:41.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011871532000000001 2023-10-09T06:02:41.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:02:41.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:52.097849+0000", 2023-10-09T06:02:41.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014743898 2023-10-09T06:02:41.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:02:41.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:52.148584+0000", 2023-10-09T06:02:41.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.050735114999999997 2023-10-09T06:02:41.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:41.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:52.154161+0000", 2023-10-09T06:02:41.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.005577009 2023-10-09T06:02:41.271 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.271 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:41.271 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:52.160285+0000", 2023-10-09T06:02:41.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061234569999999997 2023-10-09T06:02:41.271 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.271 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:02:41.271 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:52.168785+0000", 2023-10-09T06:02:41.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085001279999999992 2023-10-09T06:02:41.271 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.271 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:02:41.272 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:52:52.196605+0000", 2023-10-09T06:02:41.272 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.027820107 2023-10-09T06:02:41.272 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.272 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:02:41.272 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.272 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.272 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.272 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104142 6.1 6:8dcfcc8f:::benchmark_data_smithi064_284193_object7599:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T06:02:41.272 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:40.204920+0000", 2023-10-09T06:02:41.272 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 481.05702447700003, 2023-10-09T06:02:41.272 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6444858290000006, 2023-10-09T06:02:41.272 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:02:41.273 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:02:41.273 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:02:41.273 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:02:41.273 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:02:41.273 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104142 2023-10-09T06:02:41.273 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.273 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:02:41.273 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.273 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:02:41.273 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.204920+0000", 2023-10-09T06:02:41.273 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.273 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.274 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:02:41.274 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.204920+0000", 2023-10-09T06:02:41.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.274 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:02:41.274 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.204917+0000", 2023-10-09T06:02:41.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T06:02:41.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:02:41.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.204929+0000", 2023-10-09T06:02:41.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.163e-05 2023-10-09T06:02:41.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:02:41.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.204945+0000", 2023-10-09T06:02:41.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5988e-05 2023-10-09T06:02:41.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:02:41.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.204955+0000", 2023-10-09T06:02:41.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9219999999999992e-06 2023-10-09T06:02:41.276 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.276 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.554927+0000", 2023-10-09T06:02:41.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.349971901 2023-10-09T06:02:41.276 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:02:41.276 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.582332+0000", 2023-10-09T06:02:41.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.027405307 2023-10-09T06:02:41.276 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:02:41.277 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.615389+0000", 2023-10-09T06:02:41.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.033057214000000001 2023-10-09T06:02:41.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.277 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.277 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849384+0000", 2023-10-09T06:02:41.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.23399434 2023-10-09T06:02:41.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.277 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:02:41.277 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849406+0000", 2023-10-09T06:02:41.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2062999999999999e-05 2023-10-09T06:02:41.278 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.278 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:02:41.278 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.278 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.278 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104152 6.1 6.22f756c1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T06:02:41.278 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:40.286265+0000", 2023-10-09T06:02:41.278 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 480.97567940900001, 2023-10-09T06:02:41.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5631822680000003, 2023-10-09T06:02:41.278 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:02:41.278 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:02:41.278 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:02:41.279 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:02:41.279 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:02:41.279 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104152 2023-10-09T06:02:41.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.279 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:02:41.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:02:41.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.286265+0000", 2023-10-09T06:02:41.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:02:41.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.286265+0000", 2023-10-09T06:02:41.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:02:41.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.286263+0000", 2023-10-09T06:02:41.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:41.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:02:41.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.286271+0000", 2023-10-09T06:02:41.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9289999999999998e-06 2023-10-09T06:02:41.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.281 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:02:41.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.286285+0000", 2023-10-09T06:02:41.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3209e-05 2023-10-09T06:02:41.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.281 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:02:41.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.286293+0000", 2023-10-09T06:02:41.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4379999999999995e-06 2023-10-09T06:02:41.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849433+0000", 2023-10-09T06:02:41.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5631398769999993 2023-10-09T06:02:41.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:02:41.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849447+0000", 2023-10-09T06:02:41.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4353e-05 2023-10-09T06:02:41.282 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.282 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:02:41.282 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.283 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104178 6.1 6.53d24ae1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T06:02:41.283 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:40.851048+0000", 2023-10-09T06:02:41.283 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 480.4108966, 2023-10-09T06:02:41.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9984412760000003, 2023-10-09T06:02:41.283 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:02:41.283 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:02:41.283 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:02:41.283 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:02:41.283 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:02:41.283 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104178 2023-10-09T06:02:41.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.284 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:02:41.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:02:41.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.851048+0000", 2023-10-09T06:02:41.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:02:41.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.851048+0000", 2023-10-09T06:02:41.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:02:41.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.851046+0000", 2023-10-09T06:02:41.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:02:41.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:02:41.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.851055+0000", 2023-10-09T06:02:41.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9400000000000008e-06 2023-10-09T06:02:41.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:02:41.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.851070+0000", 2023-10-09T06:02:41.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5096000000000001e-05 2023-10-09T06:02:41.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:02:41.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:40.851082+0000", 2023-10-09T06:02:41.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2004000000000001e-05 2023-10-09T06:02:41.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849474+0000", 2023-10-09T06:02:41.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9983919610000003 2023-10-09T06:02:41.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.287 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:02:41.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849489+0000", 2023-10-09T06:02:41.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5037e-05 2023-10-09T06:02:41.287 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.287 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:02:41.287 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.287 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104195 6.1 6.f5ea83b1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T06:02:41.288 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:41.193752+0000", 2023-10-09T06:02:41.288 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 480.06819210899999, 2023-10-09T06:02:41.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.655772367, 2023-10-09T06:02:41.288 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:02:41.288 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:02:41.288 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:02:41.288 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:02:41.288 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:02:41.288 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104195 2023-10-09T06:02:41.288 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.288 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:02:41.288 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:02:41.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.193752+0000", 2023-10-09T06:02:41.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:02:41.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.193752+0000", 2023-10-09T06:02:41.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:02:41.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.193750+0000", 2023-10-09T06:02:41.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999971 2023-10-09T06:02:41.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:02:41.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.193760+0000", 2023-10-09T06:02:41.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0652999999999999e-05 2023-10-09T06:02:41.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:02:41.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.193778+0000", 2023-10-09T06:02:41.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7849000000000001e-05 2023-10-09T06:02:41.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.291 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:02:41.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.193788+0000", 2023-10-09T06:02:41.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8609999999999993e-06 2023-10-09T06:02:41.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.291 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849512+0000", 2023-10-09T06:02:41.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6557239150000003 2023-10-09T06:02:41.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:02:41.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849525+0000", 2023-10-09T06:02:41.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2850999999999999e-05 2023-10-09T06:02:41.292 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.292 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:02:41.292 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.292 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104206 6.1 6.fe55d9d1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T06:02:41.292 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:41.359887+0000", 2023-10-09T06:02:41.292 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 479.90205741300002, 2023-10-09T06:02:41.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4896904470000001, 2023-10-09T06:02:41.293 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:02:41.293 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:02:41.293 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:02:41.293 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:02:41.293 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:02:41.293 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104206 2023-10-09T06:02:41.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.293 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:02:41.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:02:41.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.359887+0000", 2023-10-09T06:02:41.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:02:41.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.359887+0000", 2023-10-09T06:02:41.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:02:41.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.359885+0000", 2023-10-09T06:02:41.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:02:41.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:02:41.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.359894+0000", 2023-10-09T06:02:41.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9309999999999997e-06 2023-10-09T06:02:41.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:02:41.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.359908+0000", 2023-10-09T06:02:41.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4185999999999999e-05 2023-10-09T06:02:41.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:02:41.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.359919+0000", 2023-10-09T06:02:41.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0611000000000001e-05 2023-10-09T06:02:41.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849563+0000", 2023-10-09T06:02:41.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4896442780000001 2023-10-09T06:02:41.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:02:41.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849577+0000", 2023-10-09T06:02:41.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4161999999999999e-05 2023-10-09T06:02:41.297 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.297 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:02:41.297 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.297 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104234 6.1 6.d0896e1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T06:02:41.297 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:41.671864+0000", 2023-10-09T06:02:41.297 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 479.59008064, 2023-10-09T06:02:41.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1777485309999998, 2023-10-09T06:02:41.297 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:02:41.298 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:02:41.298 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:02:41.298 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:02:41.298 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:02:41.298 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104234 2023-10-09T06:02:41.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.298 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:02:41.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:02:41.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.671864+0000", 2023-10-09T06:02:41.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:02:41.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.671864+0000", 2023-10-09T06:02:41.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:02:41.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.671862+0000", 2023-10-09T06:02:41.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:02:41.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:02:41.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.671872+0000", 2023-10-09T06:02:41.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0444e-05 2023-10-09T06:02:41.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:02:41.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.671891+0000", 2023-10-09T06:02:41.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8637000000000001e-05 2023-10-09T06:02:41.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:02:41.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.671901+0000", 2023-10-09T06:02:41.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7079999999999992e-06 2023-10-09T06:02:41.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849599+0000", 2023-10-09T06:02:41.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1776986379999999 2023-10-09T06:02:41.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:02:41.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849612+0000", 2023-10-09T06:02:41.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2996999999999999e-05 2023-10-09T06:02:41.302 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.302 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:02:41.302 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.302 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104244 6.1 6.700f7241 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T06:02:41.302 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:41.971632+0000", 2023-10-09T06:02:41.302 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 479.29031206000002, 2023-10-09T06:02:41.302 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8780479510000001, 2023-10-09T06:02:41.302 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:02:41.302 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:02:41.302 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:02:41.303 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:02:41.303 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:02:41.303 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104244 2023-10-09T06:02:41.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.303 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:02:41.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.303 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:02:41.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.971632+0000", 2023-10-09T06:02:41.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:02:41.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.971632+0000", 2023-10-09T06:02:41.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:02:41.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.971630+0000", 2023-10-09T06:02:41.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:02:41.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:02:41.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.971640+0000", 2023-10-09T06:02:41.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0138e-05 2023-10-09T06:02:41.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:02:41.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.971663+0000", 2023-10-09T06:02:41.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2561999999999998e-05 2023-10-09T06:02:41.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:02:41.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:41.971675+0000", 2023-10-09T06:02:41.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1970000000000001e-05 2023-10-09T06:02:41.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849666+0000", 2023-10-09T06:02:41.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8779913940000004 2023-10-09T06:02:41.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:02:41.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849680+0000", 2023-10-09T06:02:41.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3917000000000001e-05 2023-10-09T06:02:41.306 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.306 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:02:41.307 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.307 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104255 6.1 6.f28f24e1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T06:02:41.307 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:42.044751+0000", 2023-10-09T06:02:41.307 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 479.21719345100001, 2023-10-09T06:02:41.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8049857540000005, 2023-10-09T06:02:41.307 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:02:41.307 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:02:41.307 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:02:41.307 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:02:41.307 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:02:41.308 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104255 2023-10-09T06:02:41.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.308 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:02:41.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:02:41.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.044751+0000", 2023-10-09T06:02:41.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:02:41.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.044751+0000", 2023-10-09T06:02:41.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:02:41.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.044749+0000", 2023-10-09T06:02:41.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T06:02:41.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:02:41.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.044759+0000", 2023-10-09T06:02:41.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0732e-05 2023-10-09T06:02:41.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:02:41.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.044779+0000", 2023-10-09T06:02:41.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9806e-05 2023-10-09T06:02:41.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:02:41.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.044791+0000", 2023-10-09T06:02:41.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1562999999999999e-05 2023-10-09T06:02:41.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849721+0000", 2023-10-09T06:02:41.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8049299540000003 2023-10-09T06:02:41.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:02:41.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849737+0000", 2023-10-09T06:02:41.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6058000000000001e-05 2023-10-09T06:02:41.311 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.311 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:02:41.311 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.311 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104262 6.1 6.9203e6d1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T06:02:41.312 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:42.111284+0000", 2023-10-09T06:02:41.312 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 479.150660641, 2023-10-09T06:02:41.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.738488383, 2023-10-09T06:02:41.312 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:02:41.312 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:02:41.312 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:02:41.312 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:02:41.312 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:02:41.312 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104262 2023-10-09T06:02:41.312 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.312 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:02:41.312 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:02:41.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.111284+0000", 2023-10-09T06:02:41.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:02:41.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.111284+0000", 2023-10-09T06:02:41.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:02:41.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.111282+0000", 2023-10-09T06:02:41.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:41.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:02:41.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.111289+0000", 2023-10-09T06:02:41.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7549999999999997e-06 2023-10-09T06:02:41.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:02:41.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.111303+0000", 2023-10-09T06:02:41.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4024e-05 2023-10-09T06:02:41.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.315 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:02:41.315 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.111311+0000", 2023-10-09T06:02:41.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2930000000000001e-06 2023-10-09T06:02:41.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.315 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.315 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849760+0000", 2023-10-09T06:02:41.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7384484999999996 2023-10-09T06:02:41.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:02:41.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849772+0000", 2023-10-09T06:02:41.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2429000000000001e-05 2023-10-09T06:02:41.316 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.316 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:02:41.316 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.316 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104285 6.1 6.2bf38801 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T06:02:41.316 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:42.596255+0000", 2023-10-09T06:02:41.316 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 478.66568949999998, 2023-10-09T06:02:41.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2535693370000001, 2023-10-09T06:02:41.317 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:02:41.317 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:02:41.317 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:02:41.317 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:02:41.317 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:02:41.317 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104285 2023-10-09T06:02:41.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.317 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:02:41.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:02:41.317 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.596255+0000", 2023-10-09T06:02:41.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.318 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.318 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.318 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:02:41.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.596255+0000", 2023-10-09T06:02:41.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.318 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.318 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.318 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:02:41.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.596253+0000", 2023-10-09T06:02:41.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T06:02:41.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.319 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:02:41.319 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.596264+0000", 2023-10-09T06:02:41.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1596e-05 2023-10-09T06:02:41.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.319 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:02:41.319 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.596292+0000", 2023-10-09T06:02:41.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7569e-05 2023-10-09T06:02:41.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:02:41.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.596305+0000", 2023-10-09T06:02:41.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2629999999999999e-05 2023-10-09T06:02:41.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849810+0000", 2023-10-09T06:02:41.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2535054219999999 2023-10-09T06:02:41.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:02:41.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849824+0000", 2023-10-09T06:02:41.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4324999999999999e-05 2023-10-09T06:02:41.321 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.321 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:02:41.321 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.321 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:104290 6.1 6.415302b1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e186)", 2023-10-09T06:02:41.321 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:54:42.695406+0000", 2023-10-09T06:02:41.321 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 478.56653853500001, 2023-10-09T06:02:41.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1544539709999997, 2023-10-09T06:02:41.321 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:02:41.321 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:02:41.321 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:02:41.322 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:02:41.322 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:02:41.322 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104290 2023-10-09T06:02:41.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.322 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:02:41.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:02:41.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.695406+0000", 2023-10-09T06:02:41.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:02:41.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.695406+0000", 2023-10-09T06:02:41.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:02:41.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.695404+0000", 2023-10-09T06:02:41.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:02:41.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:02:41.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.695413+0000", 2023-10-09T06:02:41.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5669999999999995e-06 2023-10-09T06:02:41.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:02:41.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.695428+0000", 2023-10-09T06:02:41.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5549999999999999e-05 2023-10-09T06:02:41.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:02:41.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:42.695438+0000", 2023-10-09T06:02:41.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6479999999999998e-06 2023-10-09T06:02:41.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.325 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.325 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.325 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849847+0000", 2023-10-09T06:02:41.325 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1544087489999999 2023-10-09T06:02:41.325 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.325 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.325 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:02:41.325 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:54:48.849860+0000", 2023-10-09T06:02:41.325 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3277e-05 2023-10-09T06:02:41.325 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.325 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:02:41.325 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.325 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.326 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119186 6.21 6:86b6e4ca:::benchmark_data_smithi064_284193_object22643:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e214)", 2023-10-09T06:02:41.326 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:44.656755+0000", 2023-10-09T06:02:41.326 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 416.605189688, 2023-10-09T06:02:41.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1584478259999997, 2023-10-09T06:02:41.326 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:02:41.326 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:41.326 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:02:41.326 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:02:41.326 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:02:41.326 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119186 2023-10-09T06:02:41.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.327 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:02:41.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:02:41.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656755+0000", 2023-10-09T06:02:41.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:02:41.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656755+0000", 2023-10-09T06:02:41.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:02:41.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656753+0000", 2023-10-09T06:02:41.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:02:41.328 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:02:41.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656760+0000", 2023-10-09T06:02:41.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8290000000000002e-06 2023-10-09T06:02:41.328 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:02:41.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656773+0000", 2023-10-09T06:02:41.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3166000000000001e-05 2023-10-09T06:02:41.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.329 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:02:41.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656783+0000", 2023-10-09T06:02:41.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0210000000000001e-05 2023-10-09T06:02:41.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.329 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.658579+0000", 2023-10-09T06:02:41.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001795488 2023-10-09T06:02:41.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:02:41.330 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.658592+0000", 2023-10-09T06:02:41.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.278e-05 2023-10-09T06:02:41.330 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.330 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.743680+0000", 2023-10-09T06:02:41.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0850882960000003 2023-10-09T06:02:41.330 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T06:02:41.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.749035+0000", 2023-10-09T06:02:41.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053553639999999996 2023-10-09T06:02:41.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.776409+0000", 2023-10-09T06:02:41.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.027373859 2023-10-09T06:02:41.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:02:41.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.778175+0000", 2023-10-09T06:02:41.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001765953 2023-10-09T06:02:41.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:02:41.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.781558+0000", 2023-10-09T06:02:41.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033831220000000001 2023-10-09T06:02:41.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:41.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.784638+0000", 2023-10-09T06:02:41.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030801969999999998 2023-10-09T06:02:41.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:41.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.785224+0000", 2023-10-09T06:02:41.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00058571999999999997 2023-10-09T06:02:41.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:02:41.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.811281+0000", 2023-10-09T06:02:41.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.026056551000000001 2023-10-09T06:02:41.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:02:41.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.811998+0000", 2023-10-09T06:02:41.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00071781999999999998 2023-10-09T06:02:41.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.334 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:02:41.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.815203+0000", 2023-10-09T06:02:41.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032041819999999999 2023-10-09T06:02:41.334 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.334 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:02:41.334 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.335 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119190 6.11 6:89fd5615:::benchmark_data_smithi064_284193_object22647:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e214)", 2023-10-09T06:02:41.335 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:44.656790+0000", 2023-10-09T06:02:41.335 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 416.60515470399997, 2023-10-09T06:02:41.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0863524140000003, 2023-10-09T06:02:41.335 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:02:41.335 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:41.335 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:02:41.335 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:02:41.335 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:02:41.335 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119190 2023-10-09T06:02:41.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.335 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:02:41.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:02:41.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656790+0000", 2023-10-09T06:02:41.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:02:41.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656790+0000", 2023-10-09T06:02:41.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:02:41.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656789+0000", 2023-10-09T06:02:41.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:02:41.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:02:41.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656792+0000", 2023-10-09T06:02:41.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6599999999999999e-06 2023-10-09T06:02:41.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:02:41.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656798+0000", 2023-10-09T06:02:41.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9449999999999998e-06 2023-10-09T06:02:41.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.338 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:02:41.338 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656803+0000", 2023-10-09T06:02:41.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6060000000000002e-06 2023-10-09T06:02:41.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.338 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.338 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.658616+0000", 2023-10-09T06:02:41.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018122049999999999 2023-10-09T06:02:41.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:02:41.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.658631+0000", 2023-10-09T06:02:41.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4922999999999999e-05 2023-10-09T06:02:41.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.663857+0000", 2023-10-09T06:02:41.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0052261009999999 2023-10-09T06:02:41.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T06:02:41.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.673837+0000", 2023-10-09T06:02:41.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0099803089999999997 2023-10-09T06:02:41.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.340 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.705811+0000", 2023-10-09T06:02:41.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.031974225000000002 2023-10-09T06:02:41.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.340 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:02:41.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.708838+0000", 2023-10-09T06:02:41.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030267570000000001 2023-10-09T06:02:41.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:02:41.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.715396+0000", 2023-10-09T06:02:41.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065577710000000004 2023-10-09T06:02:41.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:41.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.731615+0000", 2023-10-09T06:02:41.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016219506000000002 2023-10-09T06:02:41.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:41.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.732772+0000", 2023-10-09T06:02:41.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011562300000000001 2023-10-09T06:02:41.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.342 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:02:41.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.733208+0000", 2023-10-09T06:02:41.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000436728 2023-10-09T06:02:41.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.342 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:02:41.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.734975+0000", 2023-10-09T06:02:41.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001766739 2023-10-09T06:02:41.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:02:41.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.743142+0000", 2023-10-09T06:02:41.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0081671590000000002 2023-10-09T06:02:41.343 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.343 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:02:41.343 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.343 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119234 6.11 6:8be56ac0:::benchmark_data_smithi064_284193_object22691:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-10-09T06:02:41.344 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:44.944674+0000", 2023-10-09T06:02:41.344 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 416.31727052100001, 2023-10-09T06:02:41.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9036751780000003, 2023-10-09T06:02:41.344 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:02:41.344 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:41.344 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:02:41.344 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:02:41.344 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:02:41.344 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119234 2023-10-09T06:02:41.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.344 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:02:41.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:02:41.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.944674+0000", 2023-10-09T06:02:41.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.345 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:02:41.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.944674+0000", 2023-10-09T06:02:41.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.345 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:02:41.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.944672+0000", 2023-10-09T06:02:41.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:02:41.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:02:41.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.944681+0000", 2023-10-09T06:02:41.346 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1509999999999993e-06 2023-10-09T06:02:41.346 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:02:41.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.944699+0000", 2023-10-09T06:02:41.346 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7589e-05 2023-10-09T06:02:41.346 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.347 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:02:41.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.944708+0000", 2023-10-09T06:02:41.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9090000000000006e-06 2023-10-09T06:02:41.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.347 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.944762+0000", 2023-10-09T06:02:41.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4095000000000003e-05 2023-10-09T06:02:41.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.347 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:02:41.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.944774+0000", 2023-10-09T06:02:41.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2248e-05 2023-10-09T06:02:41.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.720858+0000", 2023-10-09T06:02:41.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7760837649999992 2023-10-09T06:02:41.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T06:02:41.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.730024+0000", 2023-10-09T06:02:41.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0091661759999999998 2023-10-09T06:02:41.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.775720+0000", 2023-10-09T06:02:41.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.045695982000000003 2023-10-09T06:02:41.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:02:41.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.782885+0000", 2023-10-09T06:02:41.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0071655690000000001 2023-10-09T06:02:41.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:02:41.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.792563+0000", 2023-10-09T06:02:41.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0096778569999999998 2023-10-09T06:02:41.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:41.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.810903+0000", 2023-10-09T06:02:41.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018339273999999999 2023-10-09T06:02:41.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:41.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.812158+0000", 2023-10-09T06:02:41.351 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001255775 2023-10-09T06:02:41.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:02:41.351 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.836393+0000", 2023-10-09T06:02:41.351 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.024234970000000002 2023-10-09T06:02:41.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:02:41.351 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.839508+0000", 2023-10-09T06:02:41.351 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031143970000000001 2023-10-09T06:02:41.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.352 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.352 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:02:41.352 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.848349+0000", 2023-10-09T06:02:41.352 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0088413299999999997 2023-10-09T06:02:41.352 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.352 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:02:41.352 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.352 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.352 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.352 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119237 6.11 6:8ee80634:::benchmark_data_smithi064_284193_object22694:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-10-09T06:02:41.352 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:45.702303+0000", 2023-10-09T06:02:41.352 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 415.559641747, 2023-10-09T06:02:41.353 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1973806820000004, 2023-10-09T06:02:41.353 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:02:41.353 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:41.353 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:02:41.353 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:02:41.353 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:02:41.353 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119237 2023-10-09T06:02:41.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.353 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:02:41.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.353 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:02:41.353 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.702303+0000", 2023-10-09T06:02:41.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.354 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:02:41.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.702303+0000", 2023-10-09T06:02:41.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.354 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:02:41.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.702301+0000", 2023-10-09T06:02:41.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:02:41.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.355 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:02:41.355 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.702308+0000", 2023-10-09T06:02:41.355 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2830000000000004e-06 2023-10-09T06:02:41.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.355 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:02:41.355 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.716185+0000", 2023-10-09T06:02:41.355 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013876292 2023-10-09T06:02:41.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.355 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:02:41.356 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.716200+0000", 2023-10-09T06:02:41.356 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5145e-05 2023-10-09T06:02:41.356 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.356 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.356 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.798524+0000", 2023-10-09T06:02:41.356 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0823238269999997 2023-10-09T06:02:41.356 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.356 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T06:02:41.356 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.809274+0000", 2023-10-09T06:02:41.356 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010750116000000001 2023-10-09T06:02:41.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.848945+0000", 2023-10-09T06:02:41.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.039671062999999999 2023-10-09T06:02:41.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:02:41.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.854842+0000", 2023-10-09T06:02:41.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058972850000000004 2023-10-09T06:02:41.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.358 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:02:41.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.862009+0000", 2023-10-09T06:02:41.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0071667279999999998 2023-10-09T06:02:41.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.358 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:41.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.885199+0000", 2023-10-09T06:02:41.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023190399 2023-10-09T06:02:41.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.358 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:02:41.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.885882+0000", 2023-10-09T06:02:41.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00068252700000000005 2023-10-09T06:02:41.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:41.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.888407+0000", 2023-10-09T06:02:41.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025256649999999999 2023-10-09T06:02:41.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:02:41.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.890797+0000", 2023-10-09T06:02:41.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0023894400000000001 2023-10-09T06:02:41.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:02:41.360 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.899683+0000", 2023-10-09T06:02:41.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0088864919999999993 2023-10-09T06:02:41.360 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.360 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:02:41.360 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.360 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119262 6.21 6:845e8c29:::benchmark_data_smithi064_284193_object22719:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-10-09T06:02:41.360 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:45.846736+0000", 2023-10-09T06:02:41.360 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 415.41520830600001, 2023-10-09T06:02:41.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0664286440000001, 2023-10-09T06:02:41.360 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:02:41.360 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:41.360 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:02:41.361 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:02:41.361 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:02:41.361 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119262 2023-10-09T06:02:41.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.361 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:02:41.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:02:41.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.846736+0000", 2023-10-09T06:02:41.361 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:02:41.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.846736+0000", 2023-10-09T06:02:41.361 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.362 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:02:41.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.846734+0000", 2023-10-09T06:02:41.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:02:41.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.362 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:02:41.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.846743+0000", 2023-10-09T06:02:41.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0820000000000005e-06 2023-10-09T06:02:41.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:02:41.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.846757+0000", 2023-10-09T06:02:41.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4083999999999999e-05 2023-10-09T06:02:41.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:02:41.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.846765+0000", 2023-10-09T06:02:41.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3339999999999996e-06 2023-10-09T06:02:41.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.848937+0000", 2023-10-09T06:02:41.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0021717930000005 2023-10-09T06:02:41.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.364 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T06:02:41.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.854164+0000", 2023-10-09T06:02:41.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0052279120000000004 2023-10-09T06:02:41.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.364 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.900206+0000", 2023-10-09T06:02:41.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.046042012 2023-10-09T06:02:41.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:02:41.365 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.901864+0000", 2023-10-09T06:02:41.365 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001657258 2023-10-09T06:02:41.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:02:41.365 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.905026+0000", 2023-10-09T06:02:41.365 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031627719999999999 2023-10-09T06:02:41.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:41.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.908161+0000", 2023-10-09T06:02:41.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031345349999999999 2023-10-09T06:02:41.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:02:41.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.908349+0000", 2023-10-09T06:02:41.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000187911 2023-10-09T06:02:41.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:41.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.908935+0000", 2023-10-09T06:02:41.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00058620600000000003 2023-10-09T06:02:41.367 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:02:41.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.909877+0000", 2023-10-09T06:02:41.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00094206200000000002 2023-10-09T06:02:41.367 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:02:41.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.913165+0000", 2023-10-09T06:02:41.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003287594 2023-10-09T06:02:41.367 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.368 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:02:41.368 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.368 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119308 6.11 6:8cb42c43:::benchmark_data_smithi064_284193_object22765:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-10-09T06:02:41.368 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:46.087787+0000", 2023-10-09T06:02:41.368 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 415.17415779599997, 2023-10-09T06:02:41.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8988220839999999, 2023-10-09T06:02:41.368 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:02:41.368 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:41.368 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:02:41.368 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:02:41.369 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:02:41.369 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119308 2023-10-09T06:02:41.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.369 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:02:41.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:02:41.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.087787+0000", 2023-10-09T06:02:41.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:02:41.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.087787+0000", 2023-10-09T06:02:41.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:02:41.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.087785+0000", 2023-10-09T06:02:41.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:02:41.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:02:41.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.087796+0000", 2023-10-09T06:02:41.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1157e-05 2023-10-09T06:02:41.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:02:41.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.087811+0000", 2023-10-09T06:02:41.371 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5238999999999999e-05 2023-10-09T06:02:41.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:02:41.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.087822+0000", 2023-10-09T06:02:41.371 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1399e-05 2023-10-09T06:02:41.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.867168+0000", 2023-10-09T06:02:41.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7793453689999996 2023-10-09T06:02:41.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T06:02:41.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.883209+0000", 2023-10-09T06:02:41.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016041239999999998 2023-10-09T06:02:41.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.922185+0000", 2023-10-09T06:02:41.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.038975888 2023-10-09T06:02:41.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:02:41.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.928697+0000", 2023-10-09T06:02:41.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065118830000000004 2023-10-09T06:02:41.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:02:41.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.941529+0000", 2023-10-09T06:02:41.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012832487 2023-10-09T06:02:41.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:02:41.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.966726+0000", 2023-10-09T06:02:41.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.025196347000000001 2023-10-09T06:02:41.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:41.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.968542+0000", 2023-10-09T06:02:41.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001816736 2023-10-09T06:02:41.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:41.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.970863+0000", 2023-10-09T06:02:41.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00232102 2023-10-09T06:02:41.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:02:41.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.974139+0000", 2023-10-09T06:02:41.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003276143 2023-10-09T06:02:41.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:02:41.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.986609+0000", 2023-10-09T06:02:41.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012469236 2023-10-09T06:02:41.375 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.376 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:02:41.376 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.376 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119315 6.21 6:844be45b:::benchmark_data_smithi064_284193_object22772:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-10-09T06:02:41.376 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:46.187262+0000", 2023-10-09T06:02:41.376 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 415.07468274399997, 2023-10-09T06:02:41.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7650340010000001, 2023-10-09T06:02:41.376 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:02:41.376 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:41.376 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:02:41.376 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:02:41.377 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:02:41.377 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119315 2023-10-09T06:02:41.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.377 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:02:41.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:02:41.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.187262+0000", 2023-10-09T06:02:41.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:02:41.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.187262+0000", 2023-10-09T06:02:41.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:02:41.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.187260+0000", 2023-10-09T06:02:41.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:02:41.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:02:41.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.187270+0000", 2023-10-09T06:02:41.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0227000000000001e-05 2023-10-09T06:02:41.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:02:41.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.187287+0000", 2023-10-09T06:02:41.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6762e-05 2023-10-09T06:02:41.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:02:41.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.187298+0000", 2023-10-09T06:02:41.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1574e-05 2023-10-09T06:02:41.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.913351+0000", 2023-10-09T06:02:41.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7260523179999998 2023-10-09T06:02:41.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T06:02:41.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.917641+0000", 2023-10-09T06:02:41.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042900730000000002 2023-10-09T06:02:41.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.930458+0000", 2023-10-09T06:02:41.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012817034999999999 2023-10-09T06:02:41.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:02:41.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.932203+0000", 2023-10-09T06:02:41.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001745459 2023-10-09T06:02:41.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:02:41.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.935768+0000", 2023-10-09T06:02:41.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00356522 2023-10-09T06:02:41.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:41.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.938933+0000", 2023-10-09T06:02:41.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031648140000000002 2023-10-09T06:02:41.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:41.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.947606+0000", 2023-10-09T06:02:41.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086732140000000003 2023-10-09T06:02:41.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:02:41.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.947767+0000", 2023-10-09T06:02:41.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016054900000000001 2023-10-09T06:02:41.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.383 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.383 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:02:41.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.948714+0000", 2023-10-09T06:02:41.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00094673999999999995 2023-10-09T06:02:41.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.383 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.383 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:02:41.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.952296+0000", 2023-10-09T06:02:41.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035820740000000002 2023-10-09T06:02:41.383 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.384 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:02:41.384 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.384 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.384 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119330 6.11 6:8f58021f:::benchmark_data_smithi064_284193_object22787:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-10-09T06:02:41.384 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:46.350194+0000", 2023-10-09T06:02:41.384 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 414.91175068199999, 2023-10-09T06:02:41.384 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7595639490000003, 2023-10-09T06:02:41.384 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:02:41.384 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:02:41.384 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:02:41.384 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:02:41.385 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:02:41.385 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119330 2023-10-09T06:02:41.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.385 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:02:41.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:02:41.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.350194+0000", 2023-10-09T06:02:41.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:02:41.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.350194+0000", 2023-10-09T06:02:41.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:02:41.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:02:41.386 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.350192+0000", 2023-10-09T06:02:41.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:02:41.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:02:41.386 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.350201+0000", 2023-10-09T06:02:41.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0410000000000003e-06 2023-10-09T06:02:41.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.387 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:02:41.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.350222+0000", 2023-10-09T06:02:41.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0968e-05 2023-10-09T06:02:41.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.387 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:02:41.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.350232+0000", 2023-10-09T06:02:41.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8330000000000006e-06 2023-10-09T06:02:41.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.387 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.388 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.947802+0000", 2023-10-09T06:02:41.388 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5975698659999997 2023-10-09T06:02:41.388 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T06:02:41.388 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.966003+0000", 2023-10-09T06:02:41.388 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018200917 2023-10-09T06:02:41.388 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:02:41.388 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:54.029395+0000", 2023-10-09T06:02:41.388 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.063392047000000007 2023-10-09T06:02:41.388 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:02:41.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:54.038664+0000", 2023-10-09T06:02:41.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0092691679999999995 2023-10-09T06:02:41.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:02:41.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:54.047296+0000", 2023-10-09T06:02:41.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086318609999999994 2023-10-09T06:02:41.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:41.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:54.067975+0000", 2023-10-09T06:02:41.390 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020679792999999998 2023-10-09T06:02:41.390 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:02:41.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:54.070461+0000", 2023-10-09T06:02:41.390 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024859119999999998 2023-10-09T06:02:41.390 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:02:41.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:54.094248+0000", 2023-10-09T06:02:41.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023787081000000002 2023-10-09T06:02:41.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.391 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.391 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:02:41.391 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:54.098008+0000", 2023-10-09T06:02:41.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037596750000000001 2023-10-09T06:02:41.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:02:41.391 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:02:41.391 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:02:41.391 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:54.109758+0000", 2023-10-09T06:02:41.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011749625 2023-10-09T06:02:41.391 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.392 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:02:41.392 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.392 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:02:41.392 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:02:41.392 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:02:41.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:41 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:02:41.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:41 smithi064 ceph-mon[220044]: failed to encode map e356 with expected crc 2023-10-09T06:02:41.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:41 smithi064 ceph-mon[220044]: failed to encode map e356 with expected crc 2023-10-09T06:02:41.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:41 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2381763936' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:02:41.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:41 smithi064 ceph-mon[220044]: osdmap e356: 8 total, 8 up, 8 in 2023-10-09T06:02:41.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:41 smithi064 ceph-mon[220044]: 1.12 scrub starts 2023-10-09T06:02:41.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:41 smithi064 ceph-mon[220044]: failed to encode map e356 with expected crc 2023-10-09T06:02:41.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:41 smithi064 ceph-mon[220044]: failed to encode map e356 with expected crc 2023-10-09T06:02:41.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:41 smithi064 ceph-mon[220044]: 1.12 scrub ok 2023-10-09T06:02:41.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:41 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:02:41.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:41 smithi064 ceph-mon[212750]: failed to encode map e356 with expected crc 2023-10-09T06:02:41.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:41 smithi064 ceph-mon[212750]: failed to encode map e356 with expected crc 2023-10-09T06:02:41.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:41 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2381763936' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:02:41.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:41 smithi064 ceph-mon[212750]: osdmap e356: 8 total, 8 up, 8 in 2023-10-09T06:02:41.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:41 smithi064 ceph-mon[212750]: 1.12 scrub starts 2023-10-09T06:02:41.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:41 smithi064 ceph-mon[212750]: failed to encode map e356 with expected crc 2023-10-09T06:02:41.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:41 smithi064 ceph-mon[212750]: failed to encode map e356 with expected crc 2023-10-09T06:02:41.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:41 smithi064 ceph-mon[212750]: 1.12 scrub ok 2023-10-09T06:02:41.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:41 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:02:41.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:41 smithi156 ceph-mon[195101]: failed to encode map e356 with expected crc 2023-10-09T06:02:41.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:41 smithi156 ceph-mon[195101]: failed to encode map e356 with expected crc 2023-10-09T06:02:41.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:41 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2381763936' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:02:41.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:41 smithi156 ceph-mon[195101]: osdmap e356: 8 total, 8 up, 8 in 2023-10-09T06:02:41.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:41 smithi156 ceph-mon[195101]: 1.12 scrub starts 2023-10-09T06:02:41.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:41 smithi156 ceph-mon[195101]: failed to encode map e356 with expected crc 2023-10-09T06:02:41.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:41 smithi156 ceph-mon[195101]: failed to encode map e356 with expected crc 2023-10-09T06:02:41.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:41 smithi156 ceph-mon[195101]: 1.12 scrub ok 2023-10-09T06:02:42.321 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:42.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:42.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:42.425 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:42.430 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:02:42.459 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:42.515 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:42.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:42 smithi064 ceph-mon[212750]: pgmap v840: 119 pgs: 119 active+clean; 709 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 314 op/s 2023-10-09T06:02:42.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:42 smithi064 ceph-mon[220044]: pgmap v840: 119 pgs: 119 active+clean; 709 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 314 op/s 2023-10-09T06:02:42.716 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:42.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:42.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:42.822 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:42.845 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:02:42.850 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:42 smithi156 ceph-mon[195101]: pgmap v840: 119 pgs: 119 active+clean; 709 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 314 op/s 2023-10-09T06:02:42.853 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:42.869 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:02:42.892 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:02:42.902 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:42.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:42.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:42.914 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:02:42.934 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:02:42.955 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:02:42.977 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:02:42.999 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:02:43.000 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:02:43.000 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:02:43.000 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:02:43.000 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:02:43.000 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:02:43.001 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:02:43.001 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:02:43.001 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:02:43.007 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:43.038 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:43.193 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:43.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:43.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:43.297 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:43.325 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:43.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:43 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3239824059' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:43.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:43 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3239824059' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:43.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:43 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3239824059' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:44.126 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:44.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:44.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:44.215 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:44.230 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:44.264 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:44.357 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:44.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:44.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:44.397 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:02:44.431 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_1' removed 2023-10-09T06:02:44.461 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:44.490 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:44.534 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:44.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:44.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:44.639 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:02:44.664 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:44 smithi156 ceph-mon[195101]: pgmap v841: 119 pgs: 119 active+clean; 709 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 226 op/s 2023-10-09T06:02:44.664 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:44 smithi156 ceph-mon[195101]: 1.11 scrub starts 2023-10-09T06:02:44.664 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:44 smithi156 ceph-mon[195101]: 1.11 scrub ok 2023-10-09T06:02:44.664 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:44 smithi156 ceph-mon[195101]: 1.1a scrub starts 2023-10-09T06:02:44.664 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:44 smithi156 ceph-mon[195101]: 1.1a scrub ok 2023-10-09T06:02:44.664 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:44 smithi156 ceph-mon[195101]: 1.13 scrub starts 2023-10-09T06:02:44.664 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:44 smithi156 ceph-mon[195101]: 1.13 scrub ok 2023-10-09T06:02:44.665 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:44 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2763847397' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:02:44.665 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:44.666 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:44.709 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:02:44.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:44.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:02:44.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:44 smithi064 ceph-mon[212750]: pgmap v841: 119 pgs: 119 active+clean; 709 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 226 op/s 2023-10-09T06:02:44.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:44 smithi064 ceph-mon[212750]: 1.11 scrub starts 2023-10-09T06:02:44.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:44 smithi064 ceph-mon[212750]: 1.11 scrub ok 2023-10-09T06:02:44.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:44 smithi064 ceph-mon[212750]: 1.1a scrub starts 2023-10-09T06:02:44.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:44 smithi064 ceph-mon[212750]: 1.1a scrub ok 2023-10-09T06:02:44.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:44 smithi064 ceph-mon[212750]: 1.13 scrub starts 2023-10-09T06:02:44.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:44 smithi064 ceph-mon[212750]: 1.13 scrub ok 2023-10-09T06:02:44.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:44 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2763847397' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:02:44.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:44.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:44 smithi064 ceph-mon[220044]: pgmap v841: 119 pgs: 119 active+clean; 709 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 226 op/s 2023-10-09T06:02:44.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:44 smithi064 ceph-mon[220044]: 1.11 scrub starts 2023-10-09T06:02:44.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:44 smithi064 ceph-mon[220044]: 1.11 scrub ok 2023-10-09T06:02:44.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:44 smithi064 ceph-mon[220044]: 1.1a scrub starts 2023-10-09T06:02:44.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:44 smithi064 ceph-mon[220044]: 1.1a scrub ok 2023-10-09T06:02:44.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:44 smithi064 ceph-mon[220044]: 1.13 scrub starts 2023-10-09T06:02:44.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:44 smithi064 ceph-mon[220044]: 1.13 scrub ok 2023-10-09T06:02:44.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:44 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2763847397' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:02:44.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:44.816 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:44.841 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:44.921 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:44.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:44.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:45.039 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:02:45.065 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:45.107 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:02:45.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:45.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:02:45.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:45.242 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:45.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:45 smithi064 ceph-mon[212750]: failed to encode map e357 with expected crc 2023-10-09T06:02:45.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:45 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2763847397' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:02:45.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:45 smithi064 ceph-mon[212750]: osdmap e357: 8 total, 8 up, 8 in 2023-10-09T06:02:45.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:45 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1549899401' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:02:45.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:45 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:02:45.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:45 smithi064 ceph-mon[220044]: failed to encode map e357 with expected crc 2023-10-09T06:02:45.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:45 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2763847397' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:02:45.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:45 smithi064 ceph-mon[220044]: osdmap e357: 8 total, 8 up, 8 in 2023-10-09T06:02:45.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:45 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1549899401' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:02:45.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:45 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:02:45.537 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:02:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:02:45] "GET /metrics HTTP/1.1" 200 36341 "" "Prometheus/2.43.0" 2023-10-09T06:02:45.701 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T06:02:45.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:45 smithi156 ceph-mon[195101]: failed to encode map e357 with expected crc 2023-10-09T06:02:45.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:45 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2763847397' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:02:45.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:45 smithi156 ceph-mon[195101]: osdmap e357: 8 total, 8 up, 8 in 2023-10-09T06:02:45.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:45 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1549899401' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:02:45.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:45 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:02:46.086 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:02:46.137 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T06:02:46.138 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T06:02:46.138 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-10-09T06:02:46.138 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-10-09T06:02:46.196 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:46.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:46.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:46.299 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:46.332 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:46.380 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:46.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:46.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:46.488 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:46.517 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:46.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:46 smithi156 ceph-mon[195101]: pgmap v843: 103 pgs: 103 active+clean; 709 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:46.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:46 smithi156 ceph-mon[195101]: 1.3 scrub starts 2023-10-09T06:02:46.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:46 smithi156 ceph-mon[195101]: 1.3 scrub ok 2023-10-09T06:02:46.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:46 smithi156 ceph-mon[195101]: failed to encode map e358 with expected crc 2023-10-09T06:02:46.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:46 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:02:46.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:46 smithi156 ceph-mon[195101]: osdmap e358: 8 total, 8 up, 8 in 2023-10-09T06:02:46.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:46 smithi156 ceph-mon[195101]: failed to encode map e358 with expected crc 2023-10-09T06:02:46.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:46 smithi156 ceph-mon[195101]: failed to encode map e358 with expected crc 2023-10-09T06:02:46.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:46 smithi064 ceph-mon[220044]: pgmap v843: 103 pgs: 103 active+clean; 709 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:46.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:46 smithi064 ceph-mon[220044]: 1.3 scrub starts 2023-10-09T06:02:46.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:46 smithi064 ceph-mon[220044]: 1.3 scrub ok 2023-10-09T06:02:46.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:46 smithi064 ceph-mon[220044]: failed to encode map e358 with expected crc 2023-10-09T06:02:46.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:46 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:02:46.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:46 smithi064 ceph-mon[220044]: osdmap e358: 8 total, 8 up, 8 in 2023-10-09T06:02:46.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:46 smithi064 ceph-mon[220044]: failed to encode map e358 with expected crc 2023-10-09T06:02:46.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:46 smithi064 ceph-mon[220044]: failed to encode map e358 with expected crc 2023-10-09T06:02:46.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:46 smithi064 ceph-mon[212750]: pgmap v843: 103 pgs: 103 active+clean; 709 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:46.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:46 smithi064 ceph-mon[212750]: 1.3 scrub starts 2023-10-09T06:02:46.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:46 smithi064 ceph-mon[212750]: 1.3 scrub ok 2023-10-09T06:02:46.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:46 smithi064 ceph-mon[212750]: failed to encode map e358 with expected crc 2023-10-09T06:02:46.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:46 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:02:46.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:46 smithi064 ceph-mon[212750]: osdmap e358: 8 total, 8 up, 8 in 2023-10-09T06:02:46.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:46 smithi064 ceph-mon[212750]: failed to encode map e358 with expected crc 2023-10-09T06:02:46.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:46 smithi064 ceph-mon[212750]: failed to encode map e358 with expected crc 2023-10-09T06:02:47.128 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T06:02:47.129 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:47.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:47.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:47.207 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:47.232 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:02:47.264 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:47.313 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:02:47.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:47.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:02:47.322 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:02:47.322 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:02:47.420 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:47.448 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:47.492 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:47.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:47.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:47.597 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:47.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:47 smithi156 ceph-mon[195101]: failed to encode map e358 with expected crc 2023-10-09T06:02:47.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:47 smithi156 ceph-mon[195101]: 1.1d scrub starts 2023-10-09T06:02:47.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:47 smithi156 ceph-mon[195101]: 1.1d scrub ok 2023-10-09T06:02:47.623 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:47 smithi064 ceph-mon[212750]: failed to encode map e358 with expected crc 2023-10-09T06:02:47.623 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:47 smithi064 ceph-mon[212750]: 1.1d scrub starts 2023-10-09T06:02:47.623 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:47 smithi064 ceph-mon[212750]: 1.1d scrub ok 2023-10-09T06:02:47.623 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:47 smithi064 ceph-mon[220044]: failed to encode map e358 with expected crc 2023-10-09T06:02:47.623 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:47 smithi064 ceph-mon[220044]: 1.1d scrub starts 2023-10-09T06:02:47.623 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:47 smithi064 ceph-mon[220044]: 1.1d scrub ok 2023-10-09T06:02:47.626 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:48.027 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:48.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:48.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:48.106 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:02:48.129 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:48.177 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:48.245 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:48.384 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:48.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:48.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:48.490 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:48.517 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:48.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:48 smithi064 ceph-mon[212750]: pgmap v845: 103 pgs: 103 active+clean; 612 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:48.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:48 smithi064 ceph-mon[220044]: pgmap v845: 103 pgs: 103 active+clean; 612 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:48.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:48 smithi156 ceph-mon[195101]: pgmap v845: 103 pgs: 103 active+clean; 612 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:48.883 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:48.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:48.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:48.988 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:02:48.990 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:49.019 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:49.020 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:49.068 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:02:49.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:49.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:02:49.174 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:49.204 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:49.266 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:49.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:49.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:49.370 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:49.399 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:49.596 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:49.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:49 smithi064 ceph-mon[212750]: 1.17 scrub starts 2023-10-09T06:02:49.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:49 smithi064 ceph-mon[212750]: 1.17 scrub ok 2023-10-09T06:02:49.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:49 smithi064 ceph-mon[220044]: 1.17 scrub starts 2023-10-09T06:02:49.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:49 smithi064 ceph-mon[220044]: 1.17 scrub ok 2023-10-09T06:02:49.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:49 smithi156 ceph-mon[195101]: 1.17 scrub starts 2023-10-09T06:02:49.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:49 smithi156 ceph-mon[195101]: 1.17 scrub ok 2023-10-09T06:02:50.254 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:50.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:50.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:50.358 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:50.388 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:50.428 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:02:50.650 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:50 smithi064 ceph-mon[212750]: pgmap v846: 103 pgs: 103 active+clean; 612 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:50.650 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:50 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/185467047' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:02:50.650 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:50 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:02:50.651 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:50 smithi064 ceph-mon[220044]: pgmap v846: 103 pgs: 103 active+clean; 612 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:50.651 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:50 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/185467047' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:02:50.651 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:50 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:02:50.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:50 smithi156 ceph-mon[195101]: pgmap v846: 103 pgs: 103 active+clean; 612 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:50.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:50 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/185467047' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:02:50.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:50 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:02:51.702 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:51 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:02:51.702 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:51 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:02:51.702 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:51 smithi064 ceph-mon[212750]: failed to encode map e359 with expected crc 2023-10-09T06:02:51.702 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:51 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:02:51.702 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:51 smithi064 ceph-mon[212750]: osdmap e359: 8 total, 8 up, 8 in 2023-10-09T06:02:51.703 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:51 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:02:51.703 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:51 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T06:02:51.703 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:51 smithi064 ceph-mon[220044]: failed to encode map e359 with expected crc 2023-10-09T06:02:51.703 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:51 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:02:51.703 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:51 smithi064 ceph-mon[220044]: osdmap e359: 8 total, 8 up, 8 in 2023-10-09T06:02:51.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:51 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:02:51.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:51 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T06:02:51.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:51 smithi156 ceph-mon[195101]: failed to encode map e359 with expected crc 2023-10-09T06:02:51.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:51 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:02:51.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:51 smithi156 ceph-mon[195101]: osdmap e359: 8 total, 8 up, 8 in 2023-10-09T06:02:52.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:52 smithi064 ceph-mon[212750]: pgmap v848: 103 pgs: 103 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:52.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:52 smithi064 ceph-mon[220044]: pgmap v848: 103 pgs: 103 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:52.849 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:52.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:52.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:52.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:52 smithi156 ceph-mon[195101]: pgmap v848: 103 pgs: 103 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:52.955 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:52.989 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:53.094 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:02:53.114 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:02:53.140 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:02:53.166 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:02:53.192 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:02:53.218 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:02:53.237 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:02:53.259 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:02:53.260 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:02:53.260 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:02:53.260 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:02:53.261 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:02:53.261 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:02:53.261 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:02:53.261 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:02:53.261 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:02:53.349 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:53.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:53.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:53.456 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:53.489 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:53.592 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:53.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:53.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:53.696 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:53.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:53 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2205983964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:53.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:53 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2205983964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:53.723 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:53.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:53 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2205983964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:02:54.380 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:02:54.380 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T06:02:54.380 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T06:02:54.380 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T06:02:54.380 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:02:54.422 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:54.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:54.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:54.452 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_2' created 2023-10-09T06:02:54.526 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:54.557 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:54.604 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:54.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:54.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:54.671 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:02:54.697 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:54 smithi064 ceph-mon[212750]: pgmap v849: 103 pgs: 103 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:54.697 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:54 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1544974372' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T06:02:54.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:54 smithi064 ceph-mon[220044]: pgmap v849: 103 pgs: 103 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:54.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:54 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1544974372' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T06:02:54.709 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:02:54.735 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:54 smithi156 ceph-mon[195101]: pgmap v849: 103 pgs: 103 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:54.735 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:54 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1544974372' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T06:02:54.737 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:54.781 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:02:54.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:54.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:02:54.888 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:54.915 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:55.447 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:55 smithi064 ceph-mon[212750]: failed to encode map e360 with expected crc 2023-10-09T06:02:55.447 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:55 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1544974372' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T06:02:55.447 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:55 smithi064 ceph-mon[212750]: osdmap e360: 8 total, 8 up, 8 in 2023-10-09T06:02:55.447 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:55 smithi064 ceph-mon[212750]: failed to encode map e360 with expected crc 2023-10-09T06:02:55.447 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:55 smithi064 ceph-mon[212750]: failed to encode map e360 with expected crc 2023-10-09T06:02:55.447 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:55 smithi064 ceph-mon[212750]: failed to encode map e360 with expected crc 2023-10-09T06:02:55.447 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:55 smithi064 ceph-mon[212750]: failed to encode map e360 with expected crc 2023-10-09T06:02:55.447 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:55 smithi064 ceph-mon[212750]: failed to encode map e360 with expected crc 2023-10-09T06:02:55.447 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:55 smithi064 ceph-mon[212750]: pgmap v851: 119 pgs: 16 unknown, 103 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:55.448 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:55 smithi064 ceph-mon[220044]: failed to encode map e360 with expected crc 2023-10-09T06:02:55.448 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:55 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1544974372' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T06:02:55.448 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:55 smithi064 ceph-mon[220044]: osdmap e360: 8 total, 8 up, 8 in 2023-10-09T06:02:55.448 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:55 smithi064 ceph-mon[220044]: failed to encode map e360 with expected crc 2023-10-09T06:02:55.448 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:55 smithi064 ceph-mon[220044]: failed to encode map e360 with expected crc 2023-10-09T06:02:55.448 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:55 smithi064 ceph-mon[220044]: failed to encode map e360 with expected crc 2023-10-09T06:02:55.448 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:55 smithi064 ceph-mon[220044]: failed to encode map e360 with expected crc 2023-10-09T06:02:55.448 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:55 smithi064 ceph-mon[220044]: failed to encode map e360 with expected crc 2023-10-09T06:02:55.448 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:55 smithi064 ceph-mon[220044]: pgmap v851: 119 pgs: 16 unknown, 103 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:55.449 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:02:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:02:55] "GET /metrics HTTP/1.1" 200 36341 "" "Prometheus/2.43.0" 2023-10-09T06:02:55.674 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:55.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:55.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:55.757 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-10-09T06:02:55.778 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:02:55.779 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:02:55.851 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:55.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:55 smithi156 ceph-mon[195101]: failed to encode map e360 with expected crc 2023-10-09T06:02:55.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:55 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1544974372' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T06:02:55.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:55 smithi156 ceph-mon[195101]: osdmap e360: 8 total, 8 up, 8 in 2023-10-09T06:02:55.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:55 smithi156 ceph-mon[195101]: failed to encode map e360 with expected crc 2023-10-09T06:02:55.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:55 smithi156 ceph-mon[195101]: failed to encode map e360 with expected crc 2023-10-09T06:02:55.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:55 smithi156 ceph-mon[195101]: failed to encode map e360 with expected crc 2023-10-09T06:02:55.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:55 smithi156 ceph-mon[195101]: failed to encode map e360 with expected crc 2023-10-09T06:02:55.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:55 smithi156 ceph-mon[195101]: failed to encode map e360 with expected crc 2023-10-09T06:02:55.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:55 smithi156 ceph-mon[195101]: pgmap v851: 119 pgs: 16 unknown, 103 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:55.909 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:02:55.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:55.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:02:55.975 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:56.013 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:56.044 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:56.150 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:56.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:56.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:56.227 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:02:56.236 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:56.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:02:56.288 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:56.347 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:02:56.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:56.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:02:56.452 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:56.459 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T06:02:56.487 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:56.553 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:56 smithi064 ceph-mon[220044]: osdmap e361: 8 total, 8 up, 8 in 2023-10-09T06:02:56.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:56 smithi064 ceph-mon[220044]: failed to encode map e361 with expected crc 2023-10-09T06:02:56.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:56 smithi064 ceph-mon[220044]: failed to encode map e361 with expected crc 2023-10-09T06:02:56.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:56 smithi064 ceph-mon[220044]: failed to encode map e361 with expected crc 2023-10-09T06:02:56.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:56 smithi064 ceph-mon[220044]: failed to encode map e361 with expected crc 2023-10-09T06:02:56.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:56 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4144884881' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:02:56.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:56 smithi064 ceph-mon[212750]: osdmap e361: 8 total, 8 up, 8 in 2023-10-09T06:02:56.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:56 smithi064 ceph-mon[212750]: failed to encode map e361 with expected crc 2023-10-09T06:02:56.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:56 smithi064 ceph-mon[212750]: failed to encode map e361 with expected crc 2023-10-09T06:02:56.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:56 smithi064 ceph-mon[212750]: failed to encode map e361 with expected crc 2023-10-09T06:02:56.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:56 smithi064 ceph-mon[212750]: failed to encode map e361 with expected crc 2023-10-09T06:02:56.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:56 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4144884881' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:02:56.637 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:56.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:56 smithi156 ceph-mon[195101]: osdmap e361: 8 total, 8 up, 8 in 2023-10-09T06:02:56.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:56 smithi156 ceph-mon[195101]: failed to encode map e361 with expected crc 2023-10-09T06:02:56.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:56 smithi156 ceph-mon[195101]: failed to encode map e361 with expected crc 2023-10-09T06:02:56.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:56 smithi156 ceph-mon[195101]: failed to encode map e361 with expected crc 2023-10-09T06:02:56.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:56 smithi156 ceph-mon[195101]: failed to encode map e361 with expected crc 2023-10-09T06:02:56.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:56 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4144884881' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:02:57.438 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:57.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:57.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:57.542 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:02:57.568 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:57 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4144884881' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:02:57.568 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:57 smithi064 ceph-mon[212750]: osdmap e362: 8 total, 8 up, 8 in 2023-10-09T06:02:57.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:57 smithi064 ceph-mon[212750]: failed to encode map e362 with expected crc 2023-10-09T06:02:57.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:57 smithi064 ceph-mon[212750]: failed to encode map e362 with expected crc 2023-10-09T06:02:57.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:57 smithi064 ceph-mon[212750]: failed to encode map e362 with expected crc 2023-10-09T06:02:57.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:57 smithi064 ceph-mon[212750]: failed to encode map e362 with expected crc 2023-10-09T06:02:57.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:57 smithi064 ceph-mon[212750]: failed to encode map e362 with expected crc 2023-10-09T06:02:57.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:57 smithi064 ceph-mon[212750]: failed to encode map e362 with expected crc 2023-10-09T06:02:57.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:57 smithi064 ceph-mon[212750]: pgmap v854: 119 pgs: 13 creating+peering, 2 unknown, 104 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:57.569 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:57 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4144884881' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:02:57.569 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:57 smithi064 ceph-mon[220044]: osdmap e362: 8 total, 8 up, 8 in 2023-10-09T06:02:57.570 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:57 smithi064 ceph-mon[220044]: failed to encode map e362 with expected crc 2023-10-09T06:02:57.570 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:57 smithi064 ceph-mon[220044]: failed to encode map e362 with expected crc 2023-10-09T06:02:57.570 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:57 smithi064 ceph-mon[220044]: failed to encode map e362 with expected crc 2023-10-09T06:02:57.570 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:57 smithi064 ceph-mon[220044]: failed to encode map e362 with expected crc 2023-10-09T06:02:57.570 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:57 smithi064 ceph-mon[220044]: failed to encode map e362 with expected crc 2023-10-09T06:02:57.570 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:57 smithi064 ceph-mon[220044]: failed to encode map e362 with expected crc 2023-10-09T06:02:57.570 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:57 smithi064 ceph-mon[220044]: pgmap v854: 119 pgs: 13 creating+peering, 2 unknown, 104 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:57.572 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:57.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:57 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4144884881' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:02:57.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:57 smithi156 ceph-mon[195101]: osdmap e362: 8 total, 8 up, 8 in 2023-10-09T06:02:57.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:57 smithi156 ceph-mon[195101]: failed to encode map e362 with expected crc 2023-10-09T06:02:57.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:57 smithi156 ceph-mon[195101]: failed to encode map e362 with expected crc 2023-10-09T06:02:57.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:57 smithi156 ceph-mon[195101]: failed to encode map e362 with expected crc 2023-10-09T06:02:57.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:57 smithi156 ceph-mon[195101]: failed to encode map e362 with expected crc 2023-10-09T06:02:57.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:57 smithi156 ceph-mon[195101]: failed to encode map e362 with expected crc 2023-10-09T06:02:57.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:57 smithi156 ceph-mon[195101]: failed to encode map e362 with expected crc 2023-10-09T06:02:57.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:57 smithi156 ceph-mon[195101]: pgmap v854: 119 pgs: 13 creating+peering, 2 unknown, 104 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:58.024 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:02:58.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:58.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:02:58.122 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T06:02:58.128 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:58.160 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:58.209 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:58.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:58.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:58.315 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:58.344 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:58.389 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:58.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:58.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:58.495 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:58.524 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:58.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:58 smithi064 ceph-mon[212750]: failed to encode map e362 with expected crc 2023-10-09T06:02:58.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:58 smithi064 ceph-mon[220044]: failed to encode map e362 with expected crc 2023-10-09T06:02:58.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:58 smithi156 ceph-mon[195101]: failed to encode map e362 with expected crc 2023-10-09T06:02:59.109 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:59.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:59.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:59.213 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:02:59.245 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:59.292 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:02:59.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:59.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:02:59.396 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:02:59.425 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:59.469 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:02:59.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:59.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:02:59.573 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:59.605 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:02:59.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:59 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:02:59.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:59.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:02:59 smithi156 ceph-mon[195101]: pgmap v855: 119 pgs: 13 creating+peering, 106 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:59.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:59 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:02:59.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:59.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:02:59 smithi064 ceph-mon[212750]: pgmap v855: 119 pgs: 13 creating+peering, 106 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:59.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:59 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:02:59.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:59.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:02:59 smithi064 ceph-mon[220044]: pgmap v855: 119 pgs: 13 creating+peering, 106 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:59.838 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:59.845 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:02:59.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:02:59.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:02:59.951 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:02:59.980 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:00.170 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:00.179 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:00.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:00.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:00.219 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:03:00.220 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T06:03:00.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T06:03:00.220 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:03:00.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.220 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:159588 6.f 6:f2345e47:::benchmark_data_smithi064_284193_object63045:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e249)", 2023-10-09T06:03:00.220 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:57:23.458648+0000", 2023-10-09T06:03:00.220 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 336.75845622200001, 2023-10-09T06:03:00.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.017528263, 2023-10-09T06:03:00.220 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:03:00.220 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:00.221 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:03:00.221 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:03:00.221 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:03:00.221 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 159588 2023-10-09T06:03:00.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.221 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:03:00.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:03:00.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458648+0000", 2023-10-09T06:03:00.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:03:00.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458648+0000", 2023-10-09T06:03:00.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:03:00.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458637+0000", 2023-10-09T06:03:00.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999989 2023-10-09T06:03:00.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:03:00.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458653+0000", 2023-10-09T06:03:00.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5855999999999999e-05 2023-10-09T06:03:00.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:03:00.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458665+0000", 2023-10-09T06:03:00.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2064e-05 2023-10-09T06:03:00.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:03:00.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458674+0000", 2023-10-09T06:03:00.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3950000000000007e-06 2023-10-09T06:03:00.224 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.460075+0000", 2023-10-09T06:03:00.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001401086 2023-10-09T06:03:00.224 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:03:00.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.460087+0000", 2023-10-09T06:03:00.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1527e-05 2023-10-09T06:03:00.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.225 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.225 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.461560+0000", 2023-10-09T06:03:00.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0014731719999999 2023-10-09T06:03:00.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.225 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:03:00.225 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.462110+0000", 2023-10-09T06:03:00.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00054960100000000004 2023-10-09T06:03:00.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:03:00.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.463093+0000", 2023-10-09T06:03:00.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00098321699999999999 2023-10-09T06:03:00.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.467039+0000", 2023-10-09T06:03:00.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039463830000000004 2023-10-09T06:03:00.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.227 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.470896+0000", 2023-10-09T06:03:00.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003856805 2023-10-09T06:03:00.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.227 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:03:00.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.475276+0000", 2023-10-09T06:03:00.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043798400000000003 2023-10-09T06:03:00.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.227 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:03:00.228 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.475478+0000", 2023-10-09T06:03:00.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020193899999999999 2023-10-09T06:03:00.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:03:00.228 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.476176+0000", 2023-10-09T06:03:00.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00069817800000000004 2023-10-09T06:03:00.228 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.228 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:03:00.228 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.229 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:159600 6.1f 6:f9ce68c5:::benchmark_data_smithi064_284193_object63057:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e249)", 2023-10-09T06:03:00.229 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:57:23.458682+0000", 2023-10-09T06:03:00.229 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 336.75842190700001, 2023-10-09T06:03:00.229 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0352311799999998, 2023-10-09T06:03:00.229 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:03:00.229 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:00.229 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:03:00.229 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:03:00.229 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:03:00.229 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 159600 2023-10-09T06:03:00.229 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.229 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:03:00.229 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:03:00.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458682+0000", 2023-10-09T06:03:00.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:03:00.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458682+0000", 2023-10-09T06:03:00.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:03:00.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458681+0000", 2023-10-09T06:03:00.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:00.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:03:00.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458684+0000", 2023-10-09T06:03:00.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4700000000000001e-06 2023-10-09T06:03:00.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:03:00.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458690+0000", 2023-10-09T06:03:00.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.993e-06 2023-10-09T06:03:00.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.232 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.232 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:03:00.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458695+0000", 2023-10-09T06:03:00.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8300000000000003e-06 2023-10-09T06:03:00.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.232 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.232 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.460091+0000", 2023-10-09T06:03:00.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013964229999999999 2023-10-09T06:03:00.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.233 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:03:00.233 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.460101+0000", 2023-10-09T06:03:00.233 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0040999999999999e-05 2023-10-09T06:03:00.233 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.233 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.233 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.467208+0000", 2023-10-09T06:03:00.233 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.007106442 2023-10-09T06:03:00.233 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.234 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:03:00.234 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.467772+0000", 2023-10-09T06:03:00.234 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00056436500000000005 2023-10-09T06:03:00.234 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.234 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:03:00.234 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.468789+0000", 2023-10-09T06:03:00.234 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001016711 2023-10-09T06:03:00.234 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.234 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.234 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.475179+0000", 2023-10-09T06:03:00.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063902730000000001 2023-10-09T06:03:00.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.235 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.475320+0000", 2023-10-09T06:03:00.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000141423 2023-10-09T06:03:00.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.235 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:03:00.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.492859+0000", 2023-10-09T06:03:00.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017538893 2023-10-09T06:03:00.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:03:00.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.493094+0000", 2023-10-09T06:03:00.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00023475199999999999 2023-10-09T06:03:00.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:03:00.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.493913+0000", 2023-10-09T06:03:00.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00081901499999999998 2023-10-09T06:03:00.236 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.236 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:03:00.236 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.237 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.237 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.237 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:188635 7.0 7:033812fe:::benchmark_data_smithi064_398483_object83282:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T06:03:00.237 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:01:54.738354+0000", 2023-10-09T06:03:00.237 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 65.478749683999993, 2023-10-09T06:03:00.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.184445381, 2023-10-09T06:03:00.237 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:03:00.237 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:03:00.237 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:03:00.237 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:03:00.237 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:03:00.237 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 188635 2023-10-09T06:03:00.238 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.238 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:03:00.238 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.238 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:03:00.238 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738354+0000", 2023-10-09T06:03:00.238 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.238 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.238 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.238 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:03:00.238 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738354+0000", 2023-10-09T06:03:00.238 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.238 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.238 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.239 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:03:00.239 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738353+0000", 2023-10-09T06:03:00.239 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:03:00.239 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.239 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.239 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:03:00.239 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738359+0000", 2023-10-09T06:03:00.239 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8730000000000001e-06 2023-10-09T06:03:00.239 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.239 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.239 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:03:00.239 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738372+0000", 2023-10-09T06:03:00.240 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2434e-05 2023-10-09T06:03:00.240 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.240 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.240 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:03:00.240 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738382+0000", 2023-10-09T06:03:00.240 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0284e-05 2023-10-09T06:03:00.240 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.240 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.240 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.240 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738635+0000", 2023-10-09T06:03:00.240 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00025253899999999998 2023-10-09T06:03:00.240 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.241 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.241 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:03:00.241 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738645+0000", 2023-10-09T06:03:00.241 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0669000000000001e-05 2023-10-09T06:03:00.241 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.241 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.241 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.241 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.920806+0000", 2023-10-09T06:03:00.241 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1821608100000001 2023-10-09T06:03:00.241 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.241 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.241 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:03:00.241 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.922800+0000", 2023-10-09T06:03:00.242 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019933519999999999 2023-10-09T06:03:00.242 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.242 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:03:00.242 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.242 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.242 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.242 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:188644 7.0 7:0ac6db4a:::benchmark_data_smithi064_398483_object83291:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T06:03:00.242 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:01:54.738395+0000", 2023-10-09T06:03:00.242 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 65.478708424000004, 2023-10-09T06:03:00.242 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.21876129, 2023-10-09T06:03:00.242 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:03:00.242 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:00.243 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:03:00.243 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:03:00.243 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:03:00.243 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 188644 2023-10-09T06:03:00.243 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.243 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:03:00.243 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.243 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:03:00.243 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738395+0000", 2023-10-09T06:03:00.243 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.243 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.243 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.243 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:03:00.244 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738395+0000", 2023-10-09T06:03:00.244 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.244 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.244 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.244 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:03:00.244 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738395+0000", 2023-10-09T06:03:00.244 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:00.244 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.244 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.244 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:03:00.244 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738398+0000", 2023-10-09T06:03:00.244 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3699999999999999e-06 2023-10-09T06:03:00.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.245 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.245 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:03:00.245 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738405+0000", 2023-10-09T06:03:00.245 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1069999999999999e-06 2023-10-09T06:03:00.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.245 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.245 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:03:00.245 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738411+0000", 2023-10-09T06:03:00.245 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7910000000000001e-06 2023-10-09T06:03:00.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.245 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.245 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.246 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738669+0000", 2023-10-09T06:03:00.246 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00025792099999999999 2023-10-09T06:03:00.246 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.246 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:03:00.246 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738677+0000", 2023-10-09T06:03:00.246 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.464e-06 2023-10-09T06:03:00.246 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.246 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.246 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.923009+0000", 2023-10-09T06:03:00.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1843328120000001 2023-10-09T06:03:00.247 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.247 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.247 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:03:00.247 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.923497+0000", 2023-10-09T06:03:00.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00048763799999999998 2023-10-09T06:03:00.247 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.247 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.247 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,3,5", 2023-10-09T06:03:00.247 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.923579+0000", 2023-10-09T06:03:00.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1662999999999994e-05 2023-10-09T06:03:00.247 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.248 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.248 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.248 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.928612+0000", 2023-10-09T06:03:00.248 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050336030000000002 2023-10-09T06:03:00.248 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.248 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.248 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:03:00.248 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.930788+0000", 2023-10-09T06:03:00.248 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0021759819999999999 2023-10-09T06:03:00.248 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.248 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.248 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.249 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.940855+0000", 2023-10-09T06:03:00.249 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010066607999999999 2023-10-09T06:03:00.249 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.249 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.249 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.249 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.957000+0000", 2023-10-09T06:03:00.249 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016145249 2023-10-09T06:03:00.249 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.249 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.249 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:03:00.249 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.957047+0000", 2023-10-09T06:03:00.249 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6553e-05 2023-10-09T06:03:00.249 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.250 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.250 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:03:00.250 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.957157+0000", 2023-10-09T06:03:00.250 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110024 2023-10-09T06:03:00.250 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.250 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:03:00.250 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.250 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.250 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.250 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:188650 7.0 7:0e595adb:::benchmark_data_smithi064_398483_object83297:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T06:03:00.250 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:01:54.738416+0000", 2023-10-09T06:03:00.250 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 65.478687949999994, 2023-10-09T06:03:00.251 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2189154719999999, 2023-10-09T06:03:00.251 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:03:00.251 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:00.251 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:03:00.251 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:03:00.251 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:03:00.251 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 188650 2023-10-09T06:03:00.251 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.251 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:03:00.251 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.251 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:03:00.251 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738416+0000", 2023-10-09T06:03:00.252 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.252 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.252 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.252 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:03:00.252 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738416+0000", 2023-10-09T06:03:00.252 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.252 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.252 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.252 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:03:00.252 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738415+0000", 2023-10-09T06:03:00.252 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:00.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.253 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:03:00.253 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738418+0000", 2023-10-09T06:03:00.253 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8439999999999998e-06 2023-10-09T06:03:00.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.253 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:03:00.253 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738425+0000", 2023-10-09T06:03:00.253 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6810000000000001e-06 2023-10-09T06:03:00.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:03:00.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738430+0000", 2023-10-09T06:03:00.254 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4140000000000002e-06 2023-10-09T06:03:00.254 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.254 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738697+0000", 2023-10-09T06:03:00.254 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00026689599999999997 2023-10-09T06:03:00.254 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.254 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:03:00.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738704+0000", 2023-10-09T06:03:00.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6900000000000003e-06 2023-10-09T06:03:00.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.255 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.255 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.925969+0000", 2023-10-09T06:03:00.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.187264855 2023-10-09T06:03:00.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.255 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:03:00.255 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.926127+0000", 2023-10-09T06:03:00.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015809 2023-10-09T06:03:00.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.256 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.256 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,3,5", 2023-10-09T06:03:00.256 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.926242+0000", 2023-10-09T06:03:00.256 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011499 2023-10-09T06:03:00.256 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.256 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.256 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:03:00.256 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.930875+0000", 2023-10-09T06:03:00.256 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046326539999999999 2023-10-09T06:03:00.256 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.256 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.257 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.257 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.938798+0000", 2023-10-09T06:03:00.257 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0079239329999999993 2023-10-09T06:03:00.257 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.257 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.257 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.257 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.940890+0000", 2023-10-09T06:03:00.257 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020916749999999999 2023-10-09T06:03:00.257 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.257 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.257 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.257 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.957195+0000", 2023-10-09T06:03:00.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016304822 2023-10-09T06:03:00.258 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.258 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.258 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:03:00.258 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.957234+0000", 2023-10-09T06:03:00.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9288000000000001e-05 2023-10-09T06:03:00.258 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.258 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.258 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:03:00.258 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.957331+0000", 2023-10-09T06:03:00.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7102999999999995e-05 2023-10-09T06:03:00.258 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.259 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:03:00.259 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.259 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.259 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.259 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:188668 7.b 7:dc65474a:::benchmark_data_smithi064_398483_object83315:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T06:03:00.259 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:01:54.738435+0000", 2023-10-09T06:03:00.259 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 65.478668974000001, 2023-10-09T06:03:00.259 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2254298269999999, 2023-10-09T06:03:00.259 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:03:00.259 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:00.259 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:03:00.259 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:03:00.260 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:03:00.260 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 188668 2023-10-09T06:03:00.260 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.260 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:03:00.260 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.260 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:03:00.260 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738435+0000", 2023-10-09T06:03:00.260 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.260 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.260 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.260 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:03:00.260 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738435+0000", 2023-10-09T06:03:00.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.261 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.261 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:03:00.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738434+0000", 2023-10-09T06:03:00.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:00.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.261 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.261 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:03:00.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738437+0000", 2023-10-09T06:03:00.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.554e-06 2023-10-09T06:03:00.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:03:00.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738444+0000", 2023-10-09T06:03:00.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7299999999999999e-06 2023-10-09T06:03:00.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:03:00.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738449+0000", 2023-10-09T06:03:00.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2499999999999997e-06 2023-10-09T06:03:00.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738502+0000", 2023-10-09T06:03:00.263 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2732000000000003e-05 2023-10-09T06:03:00.263 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.263 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.263 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:03:00.263 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738513+0000", 2023-10-09T06:03:00.263 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1175999999999999e-05 2023-10-09T06:03:00.263 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.263 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.263 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.264 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.941813+0000", 2023-10-09T06:03:00.264 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.20329978 2023-10-09T06:03:00.264 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.264 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.264 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:03:00.264 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.944592+0000", 2023-10-09T06:03:00.264 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002779751 2023-10-09T06:03:00.264 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.264 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.264 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T06:03:00.264 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.944739+0000", 2023-10-09T06:03:00.264 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014661399999999999 2023-10-09T06:03:00.265 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.265 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.265 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:03:00.265 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.959644+0000", 2023-10-09T06:03:00.265 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014904914999999999 2023-10-09T06:03:00.265 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.265 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.265 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.265 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.959694+0000", 2023-10-09T06:03:00.265 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9805999999999997e-05 2023-10-09T06:03:00.265 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.265 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.266 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.959838+0000", 2023-10-09T06:03:00.266 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014395200000000001 2023-10-09T06:03:00.266 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.266 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.266 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.963712+0000", 2023-10-09T06:03:00.266 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038743050000000002 2023-10-09T06:03:00.266 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.266 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.266 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:03:00.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.963777+0000", 2023-10-09T06:03:00.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4559000000000003e-05 2023-10-09T06:03:00.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:03:00.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.963865+0000", 2023-10-09T06:03:00.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8127000000000005e-05 2023-10-09T06:03:00.267 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.267 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:03:00.267 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.267 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:188671 7.b 7:d38751a4:::benchmark_data_smithi064_398483_object83318:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T06:03:00.268 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:01:54.738460+0000", 2023-10-09T06:03:00.268 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 65.478643961000003, 2023-10-09T06:03:00.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2255200310000001, 2023-10-09T06:03:00.268 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:03:00.268 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:00.268 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:03:00.268 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:03:00.268 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:03:00.268 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 188671 2023-10-09T06:03:00.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.268 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:03:00.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:03:00.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738460+0000", 2023-10-09T06:03:00.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:03:00.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738460+0000", 2023-10-09T06:03:00.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:03:00.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738459+0000", 2023-10-09T06:03:00.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:00.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:03:00.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738462+0000", 2023-10-09T06:03:00.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8490000000000002e-06 2023-10-09T06:03:00.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:03:00.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738471+0000", 2023-10-09T06:03:00.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4309999999999994e-06 2023-10-09T06:03:00.271 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.271 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:03:00.271 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738476+0000", 2023-10-09T06:03:00.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5119999999999999e-06 2023-10-09T06:03:00.271 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.271 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.271 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738536+0000", 2023-10-09T06:03:00.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9747000000000002e-05 2023-10-09T06:03:00.271 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.272 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.272 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:03:00.272 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738543+0000", 2023-10-09T06:03:00.272 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0700000000000001e-06 2023-10-09T06:03:00.272 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.272 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.272 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.272 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.948303+0000", 2023-10-09T06:03:00.272 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.209759719 2023-10-09T06:03:00.272 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.272 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.272 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:03:00.273 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.948403+0000", 2023-10-09T06:03:00.273 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000100246 2023-10-09T06:03:00.273 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.273 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.273 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T06:03:00.273 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.948472+0000", 2023-10-09T06:03:00.273 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8626000000000003e-05 2023-10-09T06:03:00.273 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.273 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.273 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:03:00.273 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.959651+0000", 2023-10-09T06:03:00.273 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011179022 2023-10-09T06:03:00.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.274 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.274 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.960117+0000", 2023-10-09T06:03:00.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000465969 2023-10-09T06:03:00.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.274 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.274 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.960161+0000", 2023-10-09T06:03:00.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3989999999999997e-05 2023-10-09T06:03:00.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.963898+0000", 2023-10-09T06:03:00.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037376639999999999 2023-10-09T06:03:00.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:03:00.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.963924+0000", 2023-10-09T06:03:00.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5548999999999999e-05 2023-10-09T06:03:00.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:03:00.276 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.963980+0000", 2023-10-09T06:03:00.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6088000000000002e-05 2023-10-09T06:03:00.276 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.276 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:03:00.276 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.276 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.276 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:188672 7.b 7:d57825da:::benchmark_data_smithi064_398483_object83319:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T06:03:00.276 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:01:54.738493+0000", 2023-10-09T06:03:00.276 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 65.478610930000002, 2023-10-09T06:03:00.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.225582057, 2023-10-09T06:03:00.277 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:03:00.277 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:00.277 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:03:00.277 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:03:00.277 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:03:00.277 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 188672 2023-10-09T06:03:00.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.277 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:03:00.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.277 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:03:00.277 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738493+0000", 2023-10-09T06:03:00.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.278 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.278 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:03:00.278 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738493+0000", 2023-10-09T06:03:00.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.278 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.278 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:03:00.278 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738492+0000", 2023-10-09T06:03:00.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:00.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:03:00.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738495+0000", 2023-10-09T06:03:00.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9330000000000001e-06 2023-10-09T06:03:00.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:03:00.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738503+0000", 2023-10-09T06:03:00.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2699999999999999e-06 2023-10-09T06:03:00.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:03:00.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738509+0000", 2023-10-09T06:03:00.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2009999999999996e-06 2023-10-09T06:03:00.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738564+0000", 2023-10-09T06:03:00.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5197e-05 2023-10-09T06:03:00.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:03:00.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738571+0000", 2023-10-09T06:03:00.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0160000000000003e-06 2023-10-09T06:03:00.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.281 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.951402+0000", 2023-10-09T06:03:00.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2128306740000001 2023-10-09T06:03:00.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.281 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:03:00.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.951518+0000", 2023-10-09T06:03:00.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116768 2023-10-09T06:03:00.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T06:03:00.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.951641+0000", 2023-10-09T06:03:00.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012232599999999999 2023-10-09T06:03:00.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.960194+0000", 2023-10-09T06:03:00.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085529860000000003 2023-10-09T06:03:00.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.960226+0000", 2023-10-09T06:03:00.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2001000000000001e-05 2023-10-09T06:03:00.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:03:00.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.960368+0000", 2023-10-09T06:03:00.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014181900000000001 2023-10-09T06:03:00.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.964000+0000", 2023-10-09T06:03:00.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0036322279999999999 2023-10-09T06:03:00.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:03:00.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.964022+0000", 2023-10-09T06:03:00.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2167e-05 2023-10-09T06:03:00.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:03:00.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.964075+0000", 2023-10-09T06:03:00.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2975e-05 2023-10-09T06:03:00.284 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.284 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:03:00.284 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.285 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:188677 7.0 7:0d2b7d37:::benchmark_data_smithi064_398483_object83324:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T06:03:00.285 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:01:54.738516+0000", 2023-10-09T06:03:00.285 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 65.478588290000005, 2023-10-09T06:03:00.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.218979737, 2023-10-09T06:03:00.285 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:03:00.285 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:00.285 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:03:00.285 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:03:00.285 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:03:00.285 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 188677 2023-10-09T06:03:00.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.286 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:03:00.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:03:00.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738516+0000", 2023-10-09T06:03:00.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:03:00.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738516+0000", 2023-10-09T06:03:00.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.287 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:03:00.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738515+0000", 2023-10-09T06:03:00.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:00.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.287 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:03:00.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738518+0000", 2023-10-09T06:03:00.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7879999999999998e-06 2023-10-09T06:03:00.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.287 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:03:00.288 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738524+0000", 2023-10-09T06:03:00.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2559999999999999e-06 2023-10-09T06:03:00.288 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.288 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.288 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:03:00.288 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738529+0000", 2023-10-09T06:03:00.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2480000000000004e-06 2023-10-09T06:03:00.288 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.288 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.288 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.288 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738726+0000", 2023-10-09T06:03:00.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000196928 2023-10-09T06:03:00.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:03:00.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738735+0000", 2023-10-09T06:03:00.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4079999999999998e-06 2023-10-09T06:03:00.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.928644+0000", 2023-10-09T06:03:00.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.189909745 2023-10-09T06:03:00.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:03:00.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.928727+0000", 2023-10-09T06:03:00.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2735e-05 2023-10-09T06:03:00.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,3,5", 2023-10-09T06:03:00.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.928798+0000", 2023-10-09T06:03:00.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1055000000000003e-05 2023-10-09T06:03:00.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:03:00.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.930994+0000", 2023-10-09T06:03:00.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002196141 2023-10-09T06:03:00.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.291 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.938838+0000", 2023-10-09T06:03:00.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0078432740000000008 2023-10-09T06:03:00.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.291 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.942721+0000", 2023-10-09T06:03:00.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038830990000000001 2023-10-09T06:03:00.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.957366+0000", 2023-10-09T06:03:00.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014644885999999999 2023-10-09T06:03:00.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:03:00.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.957403+0000", 2023-10-09T06:03:00.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6959000000000003e-05 2023-10-09T06:03:00.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:03:00.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.957495+0000", 2023-10-09T06:03:00.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2639000000000005e-05 2023-10-09T06:03:00.293 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.293 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:03:00.293 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.293 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:188681 7.b 7:da89e078:::benchmark_data_smithi064_398483_object83328:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T06:03:00.293 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:01:54.859467+0000", 2023-10-09T06:03:00.293 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 65.357637264000005, 2023-10-09T06:03:00.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.104714006, 2023-10-09T06:03:00.293 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:03:00.293 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:00.293 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:03:00.294 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:03:00.294 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:03:00.294 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 188681 2023-10-09T06:03:00.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.294 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:03:00.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:03:00.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.859467+0000", 2023-10-09T06:03:00.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:03:00.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.859467+0000", 2023-10-09T06:03:00.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:03:00.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.859464+0000", 2023-10-09T06:03:00.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:03:00.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:03:00.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.859475+0000", 2023-10-09T06:03:00.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0161e-05 2023-10-09T06:03:00.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:03:00.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.859496+0000", 2023-10-09T06:03:00.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1185000000000001e-05 2023-10-09T06:03:00.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:03:00.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.859505+0000", 2023-10-09T06:03:00.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4360000000000008e-06 2023-10-09T06:03:00.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.859568+0000", 2023-10-09T06:03:00.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2723000000000002e-05 2023-10-09T06:03:00.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:03:00.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.859581+0000", 2023-10-09T06:03:00.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3193999999999999e-05 2023-10-09T06:03:00.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.956496+0000", 2023-10-09T06:03:00.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.096915267 2023-10-09T06:03:00.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:03:00.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.956642+0000", 2023-10-09T06:03:00.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014533699999999999 2023-10-09T06:03:00.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T06:03:00.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.956777+0000", 2023-10-09T06:03:00.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013512299999999999 2023-10-09T06:03:00.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:03:00.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.960374+0000", 2023-10-09T06:03:00.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003597303 2023-10-09T06:03:00.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.963246+0000", 2023-10-09T06:03:00.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028718070000000001 2023-10-09T06:03:00.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.963266+0000", 2023-10-09T06:03:00.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9972000000000001e-05 2023-10-09T06:03:00.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.964096+0000", 2023-10-09T06:03:00.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00082984599999999997 2023-10-09T06:03:00.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:03:00.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.964118+0000", 2023-10-09T06:03:00.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1908000000000001e-05 2023-10-09T06:03:00.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:03:00.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.964181+0000", 2023-10-09T06:03:00.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2834000000000002e-05 2023-10-09T06:03:00.301 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.301 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:03:00.301 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.301 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:201273 7.e 7:79b1205a:::benchmark_data_smithi064_398483_object95920:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T06:03:00.301 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:02:17.220851+0000", 2023-10-09T06:03:00.301 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 42.996253283000001, 2023-10-09T06:03:00.302 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15538379799999999, 2023-10-09T06:03:00.302 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:03:00.302 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:00.302 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:03:00.302 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:03:00.302 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:03:00.302 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 201273 2023-10-09T06:03:00.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.302 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:03:00.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.302 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:03:00.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220851+0000", 2023-10-09T06:03:00.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.303 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:03:00.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220851+0000", 2023-10-09T06:03:00.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.303 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:03:00.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220849+0000", 2023-10-09T06:03:00.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:03:00.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:03:00.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220856+0000", 2023-10-09T06:03:00.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8430000000000004e-06 2023-10-09T06:03:00.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:03:00.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220869+0000", 2023-10-09T06:03:00.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2901999999999999e-05 2023-10-09T06:03:00.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:03:00.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220878+0000", 2023-10-09T06:03:00.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8040000000000007e-06 2023-10-09T06:03:00.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220939+0000", 2023-10-09T06:03:00.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1678000000000001e-05 2023-10-09T06:03:00.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:03:00.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.221067+0000", 2023-10-09T06:03:00.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000127479 2023-10-09T06:03:00.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:03:00.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.221185+0000", 2023-10-09T06:03:00.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000117911 2023-10-09T06:03:00.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.228808+0000", 2023-10-09T06:03:00.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0076236990000000003 2023-10-09T06:03:00.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:03:00.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.235105+0000", 2023-10-09T06:03:00.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062963810000000002 2023-10-09T06:03:00.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.376133+0000", 2023-10-09T06:03:00.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14102801100000001 2023-10-09T06:03:00.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:03:00.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.376189+0000", 2023-10-09T06:03:00.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.643e-05 2023-10-09T06:03:00.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:03:00.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.376234+0000", 2023-10-09T06:03:00.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5046999999999997e-05 2023-10-09T06:03:00.308 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.308 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:03:00.308 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.309 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:201274 7.0 7:0b6d1882:::benchmark_data_smithi064_398483_object95921:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T06:03:00.309 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:02:17.220892+0000", 2023-10-09T06:03:00.309 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 42.996211326999997, 2023-10-09T06:03:00.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18597917999999999, 2023-10-09T06:03:00.309 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:03:00.309 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:00.309 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:03:00.309 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:03:00.309 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:03:00.309 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 201274 2023-10-09T06:03:00.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.310 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:03:00.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:03:00.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220892+0000", 2023-10-09T06:03:00.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:03:00.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220892+0000", 2023-10-09T06:03:00.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:03:00.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220892+0000", 2023-10-09T06:03:00.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:00.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:03:00.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220895+0000", 2023-10-09T06:03:00.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8480000000000001e-06 2023-10-09T06:03:00.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:03:00.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220901+0000", 2023-10-09T06:03:00.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.037e-06 2023-10-09T06:03:00.312 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.312 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.312 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:03:00.312 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220906+0000", 2023-10-09T06:03:00.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0989999999999998e-06 2023-10-09T06:03:00.312 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.312 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.312 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.312 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.222196+0000", 2023-10-09T06:03:00.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001289789 2023-10-09T06:03:00.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:03:00.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.222333+0000", 2023-10-09T06:03:00.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000137179 2023-10-09T06:03:00.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,3,5", 2023-10-09T06:03:00.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.222448+0000", 2023-10-09T06:03:00.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000114509 2023-10-09T06:03:00.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:03:00.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.225473+0000", 2023-10-09T06:03:00.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030259430000000001 2023-10-09T06:03:00.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.228901+0000", 2023-10-09T06:03:00.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034273670000000002 2023-10-09T06:03:00.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.232731+0000", 2023-10-09T06:03:00.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038302480000000001 2023-10-09T06:03:00.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.315 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.315 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.406688+0000", 2023-10-09T06:03:00.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.173957205 2023-10-09T06:03:00.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.315 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:03:00.315 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.406746+0000", 2023-10-09T06:03:00.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8065000000000001e-05 2023-10-09T06:03:00.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:03:00.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.406872+0000", 2023-10-09T06:03:00.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000125334 2023-10-09T06:03:00.316 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.316 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:03:00.316 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.316 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:202667 7.0 7:0aae096f:::benchmark_data_smithi064_398483_object97314:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-10-09T06:03:00.316 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:02:20.340853+0000", 2023-10-09T06:03:00.317 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 39.876250433999999, 2023-10-09T06:03:00.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25382495399999999, 2023-10-09T06:03:00.317 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:03:00.318 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:00.318 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:03:00.318 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:03:00.318 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:03:00.318 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 202667 2023-10-09T06:03:00.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.319 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:03:00.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.319 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:03:00.319 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.340853+0000", 2023-10-09T06:03:00.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:03:00.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.340853+0000", 2023-10-09T06:03:00.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:03:00.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.340852+0000", 2023-10-09T06:03:00.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:03:00.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:03:00.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.340859+0000", 2023-10-09T06:03:00.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.729e-06 2023-10-09T06:03:00.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:03:00.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.340873+0000", 2023-10-09T06:03:00.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.331e-05 2023-10-09T06:03:00.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:03:00.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.340882+0000", 2023-10-09T06:03:00.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3260000000000002e-06 2023-10-09T06:03:00.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.340930+0000", 2023-10-09T06:03:00.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7539e-05 2023-10-09T06:03:00.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:03:00.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.341069+0000", 2023-10-09T06:03:00.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000138999 2023-10-09T06:03:00.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,3,5", 2023-10-09T06:03:00.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.341190+0000", 2023-10-09T06:03:00.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0001211 2023-10-09T06:03:00.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:03:00.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594377+0000", 2023-10-09T06:03:00.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25318729800000001 2023-10-09T06:03:00.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.325 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.325 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594424+0000", 2023-10-09T06:03:00.325 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7475999999999997e-05 2023-10-09T06:03:00.325 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.325 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.325 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.326 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594460+0000", 2023-10-09T06:03:00.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6032999999999999e-05 2023-10-09T06:03:00.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.326 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.326 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594499+0000", 2023-10-09T06:03:00.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8976000000000001e-05 2023-10-09T06:03:00.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:03:00.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594556+0000", 2023-10-09T06:03:00.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6440999999999997e-05 2023-10-09T06:03:00.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:03:00.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594678+0000", 2023-10-09T06:03:00.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012243700000000001 2023-10-09T06:03:00.328 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.328 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:03:00.328 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.328 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.328 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45263.0:202666 7.5 e348/285 7:a4298507:::benchmark_data_smithi064_398483_object97313:head v 348'12941, mlcod=348'12939)", 2023-10-09T06:03:00.328 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:02:20.341200+0000", 2023-10-09T06:03:00.329 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 39.875904255999998, 2023-10-09T06:03:00.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25306627599999998, 2023-10-09T06:03:00.329 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:03:00.329 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:00.329 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:03:00.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:03:00.330 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.341200+0000", 2023-10-09T06:03:00.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.330 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:03:00.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.341200+0000", 2023-10-09T06:03:00.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:03:00.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.341199+0000", 2023-10-09T06:03:00.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:03:00.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:03:00.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.341206+0000", 2023-10-09T06:03:00.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1110000000000002e-06 2023-10-09T06:03:00.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:03:00.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.341218+0000", 2023-10-09T06:03:00.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1789e-05 2023-10-09T06:03:00.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.334 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:03:00.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.341225+0000", 2023-10-09T06:03:00.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0690000000000004e-06 2023-10-09T06:03:00.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.341270+0000", 2023-10-09T06:03:00.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5640999999999999e-05 2023-10-09T06:03:00.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:03:00.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.341292+0000", 2023-10-09T06:03:00.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1225000000000001e-05 2023-10-09T06:03:00.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:03:00.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.345678+0000", 2023-10-09T06:03:00.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043867799999999998 2023-10-09T06:03:00.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:03:00.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594266+0000", 2023-10-09T06:03:00.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24858746800000001 2023-10-09T06:03:00.336 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.336 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:03:00.337 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.337 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPushReply(7.0 348/334 [PushReplyOp(7:083777c0:::benchmark_data_smithi064_398483_object97370:head)])", 2023-10-09T06:03:00.337 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:02:20.344731+0000", 2023-10-09T06:03:00.337 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 39.872372546999998, 2023-10-09T06:03:00.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24962795099999999, 2023-10-09T06:03:00.337 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:03:00.337 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:03:00.337 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:03:00.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:03:00.338 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.344731+0000", 2023-10-09T06:03:00.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.338 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:03:00.338 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.344731+0000", 2023-10-09T06:03:00.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:03:00.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.344731+0000", 2023-10-09T06:03:00.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:00.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:03:00.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.344735+0000", 2023-10-09T06:03:00.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0999999999999997e-06 2023-10-09T06:03:00.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:03:00.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.344743+0000", 2023-10-09T06:03:00.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0579999999999998e-06 2023-10-09T06:03:00.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.340 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:03:00.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.344751+0000", 2023-10-09T06:03:00.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.303e-06 2023-10-09T06:03:00.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.344812+0000", 2023-10-09T06:03:00.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0460000000000001e-05 2023-10-09T06:03:00.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:03:00.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594359+0000", 2023-10-09T06:03:00.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24954735 2023-10-09T06:03:00.342 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.342 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:03:00.342 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.342 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.45263.0:202667 7.0 e348/334 ondisk, result = 0)", 2023-10-09T06:03:00.342 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:02:20.347781+0000", 2023-10-09T06:03:00.342 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 39.869323252000001, 2023-10-09T06:03:00.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24668209799999999, 2023-10-09T06:03:00.342 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:03:00.343 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:03:00.343 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:03:00.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:03:00.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.347781+0000", 2023-10-09T06:03:00.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:03:00.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.347781+0000", 2023-10-09T06:03:00.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:03:00.344 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.347780+0000", 2023-10-09T06:03:00.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:03:00.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:03:00.344 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.347786+0000", 2023-10-09T06:03:00.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7920000000000004e-06 2023-10-09T06:03:00.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:03:00.344 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594128+0000", 2023-10-09T06:03:00.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24634131400000001 2023-10-09T06:03:00.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.345 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:03:00.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594142+0000", 2023-10-09T06:03:00.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4664000000000001e-05 2023-10-09T06:03:00.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.345 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594448+0000", 2023-10-09T06:03:00.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00030556099999999998 2023-10-09T06:03:00.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:03:00.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594455+0000", 2023-10-09T06:03:00.346 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9299999999999997e-06 2023-10-09T06:03:00.346 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:03:00.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594463+0000", 2023-10-09T06:03:00.346 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7600000000000002e-06 2023-10-09T06:03:00.346 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.346 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:03:00.346 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.347 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.45263.0:202667 7.0 e348/334 ondisk, result = 0)", 2023-10-09T06:03:00.347 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:02:20.348183+0000", 2023-10-09T06:03:00.347 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 39.868920682000002, 2023-10-09T06:03:00.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.246244344, 2023-10-09T06:03:00.347 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:03:00.347 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:03:00.347 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:03:00.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:03:00.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.348183+0000", 2023-10-09T06:03:00.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:03:00.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.348183+0000", 2023-10-09T06:03:00.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:03:00.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.348183+0000", 2023-10-09T06:03:00.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:00.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:03:00.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.348187+0000", 2023-10-09T06:03:00.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5950000000000001e-06 2023-10-09T06:03:00.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:03:00.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594117+0000", 2023-10-09T06:03:00.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24592978099999999 2023-10-09T06:03:00.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:03:00.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594130+0000", 2023-10-09T06:03:00.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3023999999999999e-05 2023-10-09T06:03:00.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594407+0000", 2023-10-09T06:03:00.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00027666300000000002 2023-10-09T06:03:00.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:03:00.351 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594418+0000", 2023-10-09T06:03:00.351 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0890000000000001e-05 2023-10-09T06:03:00.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:03:00.351 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594427+0000", 2023-10-09T06:03:00.351 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9469999999999998e-06 2023-10-09T06:03:00.352 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.352 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:03:00.352 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.352 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.352 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.352 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:203099 7.0 7:0ca84f41:::benchmark_data_smithi064_398483_object97746:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e349)", 2023-10-09T06:03:00.352 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:02:21.080783+0000", 2023-10-09T06:03:00.352 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 39.136321221000003, 2023-10-09T06:03:00.352 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.17404369, 2023-10-09T06:03:00.352 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:03:00.352 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:00.352 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:03:00.353 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:03:00.353 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:03:00.353 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 203099 2023-10-09T06:03:00.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.353 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:03:00.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.353 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:03:00.353 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.080783+0000", 2023-10-09T06:03:00.353 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.353 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:03:00.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.080783+0000", 2023-10-09T06:03:00.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.354 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:03:00.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.080782+0000", 2023-10-09T06:03:00.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:00.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.354 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:03:00.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.080785+0000", 2023-10-09T06:03:00.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4430000000000002e-06 2023-10-09T06:03:00.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.355 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:03:00.355 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.080790+0000", 2023-10-09T06:03:00.355 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.507e-06 2023-10-09T06:03:00.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:03:00.356 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.080795+0000", 2023-10-09T06:03:00.356 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5260000000000004e-06 2023-10-09T06:03:00.356 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.356 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.356 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093828+0000", 2023-10-09T06:03:00.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013033756000000001 2023-10-09T06:03:00.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:03:00.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093950+0000", 2023-10-09T06:03:00.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012169599999999999 2023-10-09T06:03:00.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,3,5", 2023-10-09T06:03:00.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.094070+0000", 2023-10-09T06:03:00.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011979100000000001 2023-10-09T06:03:00.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.358 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.106906+0000", 2023-10-09T06:03:00.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0128361 2023-10-09T06:03:00.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.358 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:03:00.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.106918+0000", 2023-10-09T06:03:00.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1938e-05 2023-10-09T06:03:00.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.107211+0000", 2023-10-09T06:03:00.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000293147 2023-10-09T06:03:00.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.254679+0000", 2023-10-09T06:03:00.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14746752099999999 2023-10-09T06:03:00.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.360 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:03:00.360 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.254736+0000", 2023-10-09T06:03:00.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7077999999999998e-05 2023-10-09T06:03:00.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.360 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:03:00.360 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.254826+0000", 2023-10-09T06:03:00.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0613999999999997e-05 2023-10-09T06:03:00.360 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.360 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:03:00.360 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.361 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:203102 7.0 7:05c12b3f:::benchmark_data_smithi064_398483_object97749:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e349)", 2023-10-09T06:03:00.361 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:02:21.093652+0000", 2023-10-09T06:03:00.361 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 39.123452166, 2023-10-09T06:03:00.361 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16137338300000001, 2023-10-09T06:03:00.361 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:03:00.361 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:00.362 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:03:00.362 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:03:00.362 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:03:00.362 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 203102 2023-10-09T06:03:00.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.362 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:03:00.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.362 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:03:00.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093652+0000", 2023-10-09T06:03:00.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:03:00.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093652+0000", 2023-10-09T06:03:00.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:03:00.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093650+0000", 2023-10-09T06:03:00.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:03:00.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.364 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:03:00.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093658+0000", 2023-10-09T06:03:00.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3819999999999998e-06 2023-10-09T06:03:00.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.364 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:03:00.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093671+0000", 2023-10-09T06:03:00.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2763e-05 2023-10-09T06:03:00.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:03:00.365 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093679+0000", 2023-10-09T06:03:00.365 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4449999999999996e-06 2023-10-09T06:03:00.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.097473+0000", 2023-10-09T06:03:00.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037941619999999998 2023-10-09T06:03:00.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:03:00.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.097609+0000", 2023-10-09T06:03:00.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013628800000000001 2023-10-09T06:03:00.367 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,3,5", 2023-10-09T06:03:00.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.097747+0000", 2023-10-09T06:03:00.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000137589 2023-10-09T06:03:00.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.368 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:03:00.368 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.106926+0000", 2023-10-09T06:03:00.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0091790280000000005 2023-10-09T06:03:00.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.106963+0000", 2023-10-09T06:03:00.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6659999999999998e-05 2023-10-09T06:03:00.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.254542+0000", 2023-10-09T06:03:00.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14757968599999999 2023-10-09T06:03:00.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.254872+0000", 2023-10-09T06:03:00.371 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00032967700000000001 2023-10-09T06:03:00.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:03:00.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.254922+0000", 2023-10-09T06:03:00.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0275999999999998e-05 2023-10-09T06:03:00.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:03:00.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.255025+0000", 2023-10-09T06:03:00.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000102693 2023-10-09T06:03:00.373 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.373 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:03:00.373 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.374 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:203105 7.0 7:042c63f5:::benchmark_data_smithi064_398483_object97752:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e349)", 2023-10-09T06:03:00.374 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:02:21.093975+0000", 2023-10-09T06:03:00.374 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 39.123128627, 2023-10-09T06:03:00.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16126423000000001, 2023-10-09T06:03:00.374 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:03:00.375 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:00.375 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:03:00.375 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:03:00.375 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:03:00.375 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 203105 2023-10-09T06:03:00.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.376 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:03:00.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.376 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:03:00.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093975+0000", 2023-10-09T06:03:00.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:03:00.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093975+0000", 2023-10-09T06:03:00.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:03:00.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:03:00.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093974+0000", 2023-10-09T06:03:00.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:03:00.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:03:00.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093978+0000", 2023-10-09T06:03:00.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0509999999999998e-06 2023-10-09T06:03:00.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:03:00.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093989+0000", 2023-10-09T06:03:00.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0478e-05 2023-10-09T06:03:00.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:03:00.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093996+0000", 2023-10-09T06:03:00.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2069999999999998e-06 2023-10-09T06:03:00.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:03:00.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.100876+0000", 2023-10-09T06:03:00.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0068799109999999998 2023-10-09T06:03:00.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:03:00.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.101009+0000", 2023-10-09T06:03:00.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013321599999999999 2023-10-09T06:03:00.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,3,5", 2023-10-09T06:03:00.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.101139+0000", 2023-10-09T06:03:00.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012954899999999999 2023-10-09T06:03:00.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.107003+0000", 2023-10-09T06:03:00.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058646050000000002 2023-10-09T06:03:00.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:03:00.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.107020+0000", 2023-10-09T06:03:00.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6518999999999999e-05 2023-10-09T06:03:00.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.254642+0000", 2023-10-09T06:03:00.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14762177100000001 2023-10-09T06:03:00.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:00.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.255083+0000", 2023-10-09T06:03:00.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000441562 2023-10-09T06:03:00.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:03:00.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.255137+0000", 2023-10-09T06:03:00.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3751000000000001e-05 2023-10-09T06:03:00.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:03:00.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:03:00.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:03:00.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.255239+0000", 2023-10-09T06:03:00.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000102372 2023-10-09T06:03:00.382 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.382 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:03:00.382 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.382 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:03:00.383 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:03:00.383 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:03:00.383 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:00.413 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:00.459 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:00.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:00.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:00.505 INFO:teuthology.orchestra.run.smithi064.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-10-09T06:03:00.564 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:00.592 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:00.636 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:00.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:00.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:00.741 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:00.769 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:00 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1753088901' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:03:00.769 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:00 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1753088901' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:03:00.772 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:00.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:00 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1753088901' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:03:01.513 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:03:01.577 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:01.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:01.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:01.681 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:01.710 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:01.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:01 smithi064 ceph-mon[212750]: failed to encode map e363 with expected crc 2023-10-09T06:03:01.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:01 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1753088901' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:03:01.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:01 smithi064 ceph-mon[212750]: osdmap e363: 8 total, 8 up, 8 in 2023-10-09T06:03:01.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:01 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/782407439' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:03:01.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:01 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:03:01.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:01 smithi064 ceph-mon[212750]: pgmap v857: 119 pgs: 119 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:01.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:01 smithi064 ceph-mon[212750]: failed to encode map e363 with expected crc 2023-10-09T06:03:01.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:01 smithi064 ceph-mon[212750]: failed to encode map e363 with expected crc 2023-10-09T06:03:01.868 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:01 smithi064 ceph-mon[212750]: failed to encode map e363 with expected crc 2023-10-09T06:03:01.868 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:01 smithi064 ceph-mon[220044]: failed to encode map e363 with expected crc 2023-10-09T06:03:01.868 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:01 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1753088901' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:03:01.868 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:01 smithi064 ceph-mon[220044]: osdmap e363: 8 total, 8 up, 8 in 2023-10-09T06:03:01.868 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:01 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/782407439' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:03:01.868 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:01 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:03:01.868 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:01 smithi064 ceph-mon[220044]: pgmap v857: 119 pgs: 119 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:01.868 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:01 smithi064 ceph-mon[220044]: failed to encode map e363 with expected crc 2023-10-09T06:03:01.868 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:01 smithi064 ceph-mon[220044]: failed to encode map e363 with expected crc 2023-10-09T06:03:01.869 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:01 smithi064 ceph-mon[220044]: failed to encode map e363 with expected crc 2023-10-09T06:03:01.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:01 smithi156 ceph-mon[195101]: failed to encode map e363 with expected crc 2023-10-09T06:03:01.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:01 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1753088901' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:03:01.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:01 smithi156 ceph-mon[195101]: osdmap e363: 8 total, 8 up, 8 in 2023-10-09T06:03:01.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:01 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/782407439' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:03:01.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:01 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:03:01.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:01 smithi156 ceph-mon[195101]: pgmap v857: 119 pgs: 119 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:01.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:01 smithi156 ceph-mon[195101]: failed to encode map e363 with expected crc 2023-10-09T06:03:01.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:01 smithi156 ceph-mon[195101]: failed to encode map e363 with expected crc 2023-10-09T06:03:01.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:01 smithi156 ceph-mon[195101]: failed to encode map e363 with expected crc 2023-10-09T06:03:02.539 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:03:02.660 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:02.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:02.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:02.666 DEBUG:teuthology.orchestra.run.smithi064:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-10-09T06:03:02.668 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T06:03:02.668 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T06:03:02.701 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:02 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:03:02.701 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:02 smithi064 ceph-mon[212750]: failed to encode map e364 with expected crc 2023-10-09T06:03:02.701 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:02 smithi064 ceph-mon[212750]: failed to encode map e364 with expected crc 2023-10-09T06:03:02.701 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:02 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:03:02.701 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:02 smithi064 ceph-mon[212750]: osdmap e364: 8 total, 8 up, 8 in 2023-10-09T06:03:02.702 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:02 smithi064 ceph-mon[212750]: failed to encode map e364 with expected crc 2023-10-09T06:03:02.702 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:02 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:03:02.702 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:02 smithi064 ceph-mon[220044]: failed to encode map e364 with expected crc 2023-10-09T06:03:02.702 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:02 smithi064 ceph-mon[220044]: failed to encode map e364 with expected crc 2023-10-09T06:03:02.702 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:02 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:03:02.702 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:02 smithi064 ceph-mon[220044]: osdmap e364: 8 total, 8 up, 8 in 2023-10-09T06:03:02.702 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:02 smithi064 ceph-mon[220044]: failed to encode map e364 with expected crc 2023-10-09T06:03:02.719 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:hints = 1 2023-10-09T06:03:02.720 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T06:03:02.720 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object prefix: benchmark_data_smithi064_457003 2023-10-09T06:03:02.720 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:03:02.720 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 0 0 0 0 0 0 - 0 2023-10-09T06:03:02.764 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:02.832 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:02.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:02 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:03:02.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:02 smithi156 ceph-mon[195101]: failed to encode map e364 with expected crc 2023-10-09T06:03:02.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:02 smithi156 ceph-mon[195101]: failed to encode map e364 with expected crc 2023-10-09T06:03:02.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:02 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:03:02.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:02 smithi156 ceph-mon[195101]: osdmap e364: 8 total, 8 up, 8 in 2023-10-09T06:03:02.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:02 smithi156 ceph-mon[195101]: failed to encode map e364 with expected crc 2023-10-09T06:03:03.720 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 1 16 3208 3192 199.488 199.5 0.00363581 0.00499501 2023-10-09T06:03:03.792 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:03.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:03.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:03.802 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:03 smithi064 ceph-mon[212750]: failed to encode map e364 with expected crc 2023-10-09T06:03:03.802 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:03 smithi064 ceph-mon[212750]: pgmap v859: 119 pgs: 119 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:03.802 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:03 smithi064 ceph-mon[220044]: failed to encode map e364 with expected crc 2023-10-09T06:03:03.803 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:03 smithi064 ceph-mon[220044]: pgmap v859: 119 pgs: 119 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:03.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:03 smithi156 ceph-mon[195101]: failed to encode map e364 with expected crc 2023-10-09T06:03:03.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:03 smithi156 ceph-mon[195101]: pgmap v859: 119 pgs: 119 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:03.903 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:03.931 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:03.973 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:03.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:03.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:04.079 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:04.122 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:04.200 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:04.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:04.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:04.307 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:04.348 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:04.424 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:04.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:04.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:04.469 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:03:04.533 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:04.567 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:04.620 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:04.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:04.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:04.641 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:04.721 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 2 16 5827 5811 181.443 163.688 0.00971297 0.00549032 2023-10-09T06:03:04.726 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:04.764 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:04.835 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:04.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:04.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:04.941 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:04.977 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:05.036 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:05.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:05.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:05.120 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 06:03:04 smithi156 conmon[175419]: logger=cleanup t=2023-10-09T06:03:04.971681614Z level=info msg="Completed cleanup jobs" duration=13.012727ms 2023-10-09T06:03:05.142 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:05.215 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:05.307 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:03:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:03:05] "GET /metrics HTTP/1.1" 200 36316 "" "Prometheus/2.43.0" 2023-10-09T06:03:05.433 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:03:05.434 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:05.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:05.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:05.481 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:03:05.527 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:03:05.540 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:05.577 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:03:05.606 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:05.619 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:03:05.661 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:03:05.704 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:03:05.721 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 3 16 8177 8161 169.922 146.875 0.00986968 0.00587423 2023-10-09T06:03:05.751 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:03:05.756 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:03:05.756 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:03:05.756 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:03:05.757 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:03:05.757 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:03:05.757 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:03:05.757 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:03:05.757 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:03:05.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:05 smithi064 ceph-mon[212750]: pgmap v860: 119 pgs: 119 active+clean; 24 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s wr, 46 op/s 2023-10-09T06:03:05.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:05 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/197289314' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:05.869 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:05.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:05.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:05.978 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:06.036 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:06.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:05 smithi156 ceph-mon[195101]: pgmap v860: 119 pgs: 119 active+clean; 24 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s wr, 46 op/s 2023-10-09T06:03:06.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:05 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/197289314' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:06.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:05 smithi064 ceph-mon[220044]: pgmap v860: 119 pgs: 119 active+clean; 24 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s wr, 46 op/s 2023-10-09T06:03:06.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:05 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/197289314' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:06.722 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 4 16 10565 10549 164.75 149.25 0.00419926 0.00606189 2023-10-09T06:03:06.877 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T06:03:07.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:06 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:03:07.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:06 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1791278118' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:03:07.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:06 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:03:07.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:06 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:03:07.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:06 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1791278118' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:03:07.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:06 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:03:07.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:06 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:03:07.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:06 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1791278118' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:03:07.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:06 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:03:07.722 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 5 16 13283 13267 165.772 169.875 0.0126761 0.00602527 2023-10-09T06:03:08.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:07 smithi156 ceph-mon[195101]: pgmap v861: 119 pgs: 119 active+clean; 408 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2023-10-09T06:03:08.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:07 smithi156 ceph-mon[195101]: failed to encode map e365 with expected crc 2023-10-09T06:03:08.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:07 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:03:08.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:07 smithi156 ceph-mon[195101]: failed to encode map e365 with expected crc 2023-10-09T06:03:08.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:07 smithi156 ceph-mon[195101]: osdmap e365: 8 total, 8 up, 8 in 2023-10-09T06:03:08.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:07 smithi156 ceph-mon[195101]: failed to encode map e365 with expected crc 2023-10-09T06:03:08.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:07 smithi156 ceph-mon[195101]: failed to encode map e365 with expected crc 2023-10-09T06:03:08.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:07 smithi156 ceph-mon[195101]: failed to encode map e365 with expected crc 2023-10-09T06:03:08.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:07 smithi156 ceph-mon[195101]: failed to encode map e365 with expected crc 2023-10-09T06:03:08.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:07 smithi156 ceph-mon[195101]: failed to encode map e365 with expected crc 2023-10-09T06:03:08.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:07 smithi064 ceph-mon[212750]: pgmap v861: 119 pgs: 119 active+clean; 408 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2023-10-09T06:03:08.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:07 smithi064 ceph-mon[212750]: failed to encode map e365 with expected crc 2023-10-09T06:03:08.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:07 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:03:08.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:07 smithi064 ceph-mon[212750]: failed to encode map e365 with expected crc 2023-10-09T06:03:08.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:07 smithi064 ceph-mon[212750]: osdmap e365: 8 total, 8 up, 8 in 2023-10-09T06:03:08.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:07 smithi064 ceph-mon[212750]: failed to encode map e365 with expected crc 2023-10-09T06:03:08.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:07 smithi064 ceph-mon[212750]: failed to encode map e365 with expected crc 2023-10-09T06:03:08.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:07 smithi064 ceph-mon[212750]: failed to encode map e365 with expected crc 2023-10-09T06:03:08.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:07 smithi064 ceph-mon[212750]: failed to encode map e365 with expected crc 2023-10-09T06:03:08.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:07 smithi064 ceph-mon[212750]: failed to encode map e365 with expected crc 2023-10-09T06:03:08.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:07 smithi064 ceph-mon[220044]: pgmap v861: 119 pgs: 119 active+clean; 408 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2023-10-09T06:03:08.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:07 smithi064 ceph-mon[220044]: failed to encode map e365 with expected crc 2023-10-09T06:03:08.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:07 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:03:08.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:07 smithi064 ceph-mon[220044]: failed to encode map e365 with expected crc 2023-10-09T06:03:08.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:07 smithi064 ceph-mon[220044]: osdmap e365: 8 total, 8 up, 8 in 2023-10-09T06:03:08.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:07 smithi064 ceph-mon[220044]: failed to encode map e365 with expected crc 2023-10-09T06:03:08.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:07 smithi064 ceph-mon[220044]: failed to encode map e365 with expected crc 2023-10-09T06:03:08.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:07 smithi064 ceph-mon[220044]: failed to encode map e365 with expected crc 2023-10-09T06:03:08.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:07 smithi064 ceph-mon[220044]: failed to encode map e365 with expected crc 2023-10-09T06:03:08.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:07 smithi064 ceph-mon[220044]: failed to encode map e365 with expected crc 2023-10-09T06:03:08.723 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 6 16 15837 15821 164.712 159.625 0.0072744 0.00606391 2023-10-09T06:03:09.724 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 7 16 17593 17577 156.838 109.75 0.00440453 0.00636993 2023-10-09T06:03:09.793 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:09.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:09.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:09.901 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:09.987 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T06:03:09.998 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:10.037 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:10.059 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:09 smithi064 ceph-mon[220044]: pgmap v863: 119 pgs: 119 active+clean; 506 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T06:03:10.059 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:09 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:10.062 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:09 smithi064 ceph-mon[212750]: pgmap v863: 119 pgs: 119 active+clean; 506 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T06:03:10.062 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:09 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:10.364 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:10.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:09 smithi156 ceph-mon[195101]: pgmap v863: 119 pgs: 119 active+clean; 506 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T06:03:10.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:09 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:10.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:10.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:10.473 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:10.550 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:10.724 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 8 16 19313 19297 150.672 107.5 0.00669142 0.00662972 2023-10-09T06:03:10.778 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:03:10.778 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:03:11.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:10 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:11.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:10 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:11.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:10 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:11.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:10 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:11.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:10 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:11.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:10 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:11.724 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 9 16 21674 21658 150.324 147.562 0.00135907 0.00662969 2023-10-09T06:03:12.089 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:03:12.274 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:12.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:12.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:12.280 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:12 smithi064 ceph-mon[220044]: pgmap v864: 119 pgs: 119 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 122 MiB/s wr, 1.94k op/s 2023-10-09T06:03:12.280 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:12 smithi064 ceph-mon[212750]: pgmap v864: 119 pgs: 119 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 122 MiB/s wr, 1.94k op/s 2023-10-09T06:03:12.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:12 smithi156 ceph-mon[195101]: pgmap v864: 119 pgs: 119 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 122 MiB/s wr, 1.94k op/s 2023-10-09T06:03:12.380 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:12.408 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:12.724 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 10 16 23430 23414 146.267 109.75 0.00563004 0.00682169 2023-10-09T06:03:13.364 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:13.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:13.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:13.437 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:13.457 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:03:13.469 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:03:13.504 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:13.554 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:03:13.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:13.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:03:13.625 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:13.660 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:03:13.694 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:13.725 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 11 16 24873 24857 141.165 90.1875 0.0135863 0.00707604 2023-10-09T06:03:13.750 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:03:13.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:13.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:03:13.856 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:13.935 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:14.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:14 smithi064 ceph-mon[212750]: pgmap v865: 119 pgs: 119 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-10-09T06:03:14.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:14.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:14.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:14 smithi064 ceph-mon[220044]: pgmap v865: 119 pgs: 119 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-10-09T06:03:14.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:14.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:14.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:14 smithi156 ceph-mon[195101]: pgmap v865: 119 pgs: 119 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-10-09T06:03:14.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:14.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:14.725 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 12 16 26609 26593 138.443 108.5 0.00629262 0.00721796 2023-10-09T06:03:15.397 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:03:15.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:15 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/478707158' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:03:15.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:15 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:03:15.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:15 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/478707158' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:03:15.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:15 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:03:15.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:15 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/478707158' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:03:15.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:15 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:03:15.720 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:03:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:03:15] "GET /metrics HTTP/1.1" 200 36373 "" "Prometheus/2.43.0" 2023-10-09T06:03:15.725 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 13 16 28544 28528 137.096 120.938 0.00176137 0.0072481 2023-10-09T06:03:16.228 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:16.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:16.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:16.336 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:16.420 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:16.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:16 smithi064 ceph-mon[212750]: pgmap v866: 119 pgs: 119 active+clean; 1.2 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-10-09T06:03:16.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:16 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:03:16.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:16 smithi064 ceph-mon[212750]: failed to encode map e366 with expected crc 2023-10-09T06:03:16.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:16 smithi064 ceph-mon[212750]: failed to encode map e366 with expected crc 2023-10-09T06:03:16.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:16 smithi064 ceph-mon[212750]: failed to encode map e366 with expected crc 2023-10-09T06:03:16.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:16 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:03:16.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:16 smithi064 ceph-mon[212750]: osdmap e366: 8 total, 8 up, 8 in 2023-10-09T06:03:16.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:16 smithi064 ceph-mon[212750]: failed to encode map e366 with expected crc 2023-10-09T06:03:16.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:16 smithi064 ceph-mon[212750]: failed to encode map e366 with expected crc 2023-10-09T06:03:16.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:16 smithi064 ceph-mon[212750]: failed to encode map e366 with expected crc 2023-10-09T06:03:16.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:16 smithi064 ceph-mon[212750]: failed to encode map e366 with expected crc 2023-10-09T06:03:16.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:16 smithi064 ceph-mon[220044]: pgmap v866: 119 pgs: 119 active+clean; 1.2 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-10-09T06:03:16.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:16 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:03:16.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:16 smithi064 ceph-mon[220044]: failed to encode map e366 with expected crc 2023-10-09T06:03:16.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:16 smithi064 ceph-mon[220044]: failed to encode map e366 with expected crc 2023-10-09T06:03:16.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:16 smithi064 ceph-mon[220044]: failed to encode map e366 with expected crc 2023-10-09T06:03:16.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:16 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:03:16.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:16 smithi064 ceph-mon[220044]: osdmap e366: 8 total, 8 up, 8 in 2023-10-09T06:03:16.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:16 smithi064 ceph-mon[220044]: failed to encode map e366 with expected crc 2023-10-09T06:03:16.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:16 smithi064 ceph-mon[220044]: failed to encode map e366 with expected crc 2023-10-09T06:03:16.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:16 smithi064 ceph-mon[220044]: failed to encode map e366 with expected crc 2023-10-09T06:03:16.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:16 smithi064 ceph-mon[220044]: failed to encode map e366 with expected crc 2023-10-09T06:03:16.624 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:16.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:16.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:16.725 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 14 16 30272 30256 135.018 108 0.00503357 0.00740173 2023-10-09T06:03:16.733 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:16.800 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:16 smithi156 ceph-mon[195101]: pgmap v866: 119 pgs: 119 active+clean; 1.2 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-10-09T06:03:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:16 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:03:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:16 smithi156 ceph-mon[195101]: failed to encode map e366 with expected crc 2023-10-09T06:03:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:16 smithi156 ceph-mon[195101]: failed to encode map e366 with expected crc 2023-10-09T06:03:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:16 smithi156 ceph-mon[195101]: failed to encode map e366 with expected crc 2023-10-09T06:03:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:16 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:03:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:16 smithi156 ceph-mon[195101]: osdmap e366: 8 total, 8 up, 8 in 2023-10-09T06:03:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:16 smithi156 ceph-mon[195101]: failed to encode map e366 with expected crc 2023-10-09T06:03:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:16 smithi156 ceph-mon[195101]: failed to encode map e366 with expected crc 2023-10-09T06:03:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:16 smithi156 ceph-mon[195101]: failed to encode map e366 with expected crc 2023-10-09T06:03:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:16 smithi156 ceph-mon[195101]: failed to encode map e366 with expected crc 2023-10-09T06:03:17.726 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 15 16 31897 31881 132.787 101.562 0.00187029 0.00749272 2023-10-09T06:03:18.430 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:18.553 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:18.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:18.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:18.664 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:18.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:18 smithi064 ceph-mon[212750]: pgmap v868: 119 pgs: 119 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-09T06:03:18.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:18 smithi064 ceph-mon[220044]: pgmap v868: 119 pgs: 119 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-09T06:03:18.726 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 16 16 33212 33196 129.616 82.1875 0.00972929 0.0077086 2023-10-09T06:03:18.741 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:18.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:18 smithi156 ceph-mon[195101]: pgmap v868: 119 pgs: 119 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-09T06:03:18.923 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:18.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:18.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:19.028 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:19.101 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:19.398 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:19 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4249100803' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:03:19.398 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:19 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:03:19.438 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T06:03:19.521 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:03:19.568 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:03:19.611 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:03:19.648 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:03:19.681 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:03:19.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:19 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4249100803' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:03:19.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:19 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:03:19.723 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:03:19.727 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 17 16 34554 34538 126.919 83.875 0.00633165 0.00787267 2023-10-09T06:03:19.766 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:03:19.804 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:03:19.818 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:03:19.819 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:03:19.819 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:03:19.819 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:03:19.819 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:03:19.819 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:03:19.819 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:03:19.819 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:03:19.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:19 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4249100803' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:03:19.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:19 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:03:20.028 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:20.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:20.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:20.134 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:20.189 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:20.262 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:20.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:20.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:20.367 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:20.396 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:20.458 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:03:20.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:20 smithi156 ceph-mon[195101]: pgmap v869: 119 pgs: 119 active+clean; 1.7 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-10-09T06:03:20.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:20 smithi156 ceph-mon[195101]: failed to encode map e367 with expected crc 2023-10-09T06:03:20.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:20 smithi156 ceph-mon[195101]: failed to encode map e367 with expected crc 2023-10-09T06:03:20.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:20 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:03:20.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:20 smithi156 ceph-mon[195101]: osdmap e367: 8 total, 8 up, 8 in 2023-10-09T06:03:20.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:20 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4069101476' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:20.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:20 smithi064 ceph-mon[212750]: pgmap v869: 119 pgs: 119 active+clean; 1.7 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-10-09T06:03:20.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:20 smithi064 ceph-mon[212750]: failed to encode map e367 with expected crc 2023-10-09T06:03:20.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:20 smithi064 ceph-mon[212750]: failed to encode map e367 with expected crc 2023-10-09T06:03:20.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:20 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:03:20.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:20 smithi064 ceph-mon[212750]: osdmap e367: 8 total, 8 up, 8 in 2023-10-09T06:03:20.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:20 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4069101476' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:20.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:20 smithi064 ceph-mon[220044]: pgmap v869: 119 pgs: 119 active+clean; 1.7 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-10-09T06:03:20.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:20 smithi064 ceph-mon[220044]: failed to encode map e367 with expected crc 2023-10-09T06:03:20.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:20 smithi064 ceph-mon[220044]: failed to encode map e367 with expected crc 2023-10-09T06:03:20.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:20 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:03:20.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:20 smithi064 ceph-mon[220044]: osdmap e367: 8 total, 8 up, 8 in 2023-10-09T06:03:20.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:20 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4069101476' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:20.727 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 18 16 35794 35778 124.174 77.5 0.00627218 0.00804669 2023-10-09T06:03:21.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:21 smithi064 ceph-mon[220044]: failed to encode map e367 with expected crc 2023-10-09T06:03:21.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:21 smithi064 ceph-mon[220044]: failed to encode map e367 with expected crc 2023-10-09T06:03:21.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:21 smithi064 ceph-mon[212750]: failed to encode map e367 with expected crc 2023-10-09T06:03:21.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:21 smithi064 ceph-mon[212750]: failed to encode map e367 with expected crc 2023-10-09T06:03:21.728 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 19 16 37082 37066 121.875 80.5 0.00197248 0.0081831 2023-10-09T06:03:21.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:21 smithi156 ceph-mon[195101]: failed to encode map e367 with expected crc 2023-10-09T06:03:21.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:21 smithi156 ceph-mon[195101]: failed to encode map e367 with expected crc 2023-10-09T06:03:21.897 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:21.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:21.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:22.001 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:22.066 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:22.087 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T06:03:22.377 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:22.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:22.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:22.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:22 smithi064 ceph-mon[220044]: pgmap v871: 119 pgs: 119 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 132 MiB/s wr, 2.12k op/s 2023-10-09T06:03:22.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:22 smithi064 ceph-mon[212750]: pgmap v871: 119 pgs: 119 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 132 MiB/s wr, 2.12k op/s 2023-10-09T06:03:22.487 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:22.561 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:22.729 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-10-09T06:03:22.728702+0000 min lat: 0.0010326 max lat: 0.265624 avg lat: 0.00837179 2023-10-09T06:03:22.729 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:03:22.729 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 20 16 38229 38213 119.366 71.6875 0.00472664 0.00837179 2023-10-09T06:03:22.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:22 smithi156 ceph-mon[195101]: pgmap v871: 119 pgs: 119 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 132 MiB/s wr, 2.12k op/s 2023-10-09T06:03:22.933 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:22.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:22.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:23.038 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:23.084 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:23.153 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:23.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:23.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:23.185 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:23.260 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:23.319 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:03:23.319 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T06:03:23.319 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T06:03:23.320 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T06:03:23.320 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:03:23.329 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:23.617 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:23.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:23.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:23.727 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:23.728 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 21 16 39700 39684 118.06 91.9375 0.00971213 0.00846573 2023-10-09T06:03:23.772 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:24.339 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:03:24.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:24 smithi064 ceph-mon[212750]: pgmap v872: 119 pgs: 119 active+clean; 2.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 123 MiB/s wr, 1.98k op/s 2023-10-09T06:03:24.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:24 smithi064 ceph-mon[220044]: pgmap v872: 119 pgs: 119 active+clean; 2.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 123 MiB/s wr, 1.98k op/s 2023-10-09T06:03:24.729 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 22 16 40996 40980 116.369 81 0.149536 0.00857175 2023-10-09T06:03:24.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:24 smithi156 ceph-mon[195101]: pgmap v872: 119 pgs: 119 active+clean; 2.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 123 MiB/s wr, 1.98k op/s 2023-10-09T06:03:25.173 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:25.175 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:25.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:25.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:25.214 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:03:25.281 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:25.401 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:25.450 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:03:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:03:25] "GET /metrics HTTP/1.1" 200 36373 "" "Prometheus/2.43.0" 2023-10-09T06:03:25.533 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:25.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:25.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:25.638 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:03:25.683 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:25.730 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 23 16 42307 42291 114.872 81.9375 0.0211652 0.00869734 2023-10-09T06:03:25.750 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:03:25.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:25.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:03:25.855 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:25.909 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:25.981 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:25.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:25.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:26.085 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:26.111 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:26.174 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:26.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:26.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:26.282 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:26.343 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:26.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:26 smithi064 ceph-mon[212750]: pgmap v873: 119 pgs: 119 active+clean; 2.2 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T06:03:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:26 smithi064 ceph-mon[220044]: pgmap v873: 119 pgs: 119 active+clean; 2.2 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T06:03:26.729 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 24 16 43209 43193 112.435 56.375 0.00257263 0.00887572 2023-10-09T06:03:26.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:26 smithi156 ceph-mon[195101]: pgmap v873: 119 pgs: 119 active+clean; 2.2 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T06:03:27.730 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 25 16 44456 44440 111.055 77.9375 0.00653755 0.00899715 2023-10-09T06:03:27.971 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:27.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:27.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:28.023 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:28.078 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:28.107 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:28.167 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:28.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:28.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:28.202 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:28.276 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:03:28.320 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:28.375 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:03:28.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:28.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:03:28.481 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:28.516 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:28.578 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:28.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:28.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:28.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:28 smithi064 ceph-mon[220044]: pgmap v874: 119 pgs: 119 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T06:03:28.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:28 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:28.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:28 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:28.641 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:28 smithi064 ceph-mon[212750]: pgmap v874: 119 pgs: 119 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T06:03:28.641 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:28 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:28.641 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:28 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:28.688 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:03:28.719 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:28 smithi156 ceph-mon[195101]: pgmap v874: 119 pgs: 119 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T06:03:28.719 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:28 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:28.719 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:28 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:28.722 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:28.730 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 26 16 45710 45694 109.798 78.375 0.0116007 0.00909922 2023-10-09T06:03:28.784 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:03:28.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:28.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:03:28.946 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:29.036 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:29.461 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:03:29.609 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:29.609 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:29 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2646198656' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:03:29.609 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:03:29.609 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:03:29.609 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:29 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:29.609 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:29.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:29 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2646198656' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:03:29.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:03:29.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:03:29.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:29 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:29.730 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 27 16 46827 46811 108.318 69.8125 0.00153521 0.0092148 2023-10-09T06:03:29.858 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:29.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:29.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:29 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2646198656' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:03:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:03:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:03:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:29 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:29.967 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:30.014 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:30.094 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:30.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:30.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:30.203 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:30.252 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:30.295 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:30.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:30.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:30.400 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:30.431 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:30.473 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T06:03:30.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:30 smithi156 ceph-mon[195101]: pgmap v875: 119 pgs: 119 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T06:03:30.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:30 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:03:30.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:30 smithi156 ceph-mon[195101]: failed to encode map e368 with expected crc 2023-10-09T06:03:30.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:30 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2646198656' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:03:30.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:30 smithi156 ceph-mon[195101]: osdmap e368: 8 total, 8 up, 8 in 2023-10-09T06:03:30.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:30 smithi156 ceph-mon[195101]: failed to encode map e368 with expected crc 2023-10-09T06:03:30.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:30 smithi156 ceph-mon[195101]: failed to encode map e368 with expected crc 2023-10-09T06:03:30.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:30 smithi156 ceph-mon[195101]: failed to encode map e368 with expected crc 2023-10-09T06:03:30.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:30 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1296174659' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:03:30.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:30 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:03:30.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:30 smithi064 ceph-mon[212750]: pgmap v875: 119 pgs: 119 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T06:03:30.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:30 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:03:30.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:30 smithi064 ceph-mon[212750]: failed to encode map e368 with expected crc 2023-10-09T06:03:30.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:30 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2646198656' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:03:30.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:30 smithi064 ceph-mon[212750]: osdmap e368: 8 total, 8 up, 8 in 2023-10-09T06:03:30.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:30 smithi064 ceph-mon[212750]: failed to encode map e368 with expected crc 2023-10-09T06:03:30.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:30 smithi064 ceph-mon[212750]: failed to encode map e368 with expected crc 2023-10-09T06:03:30.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:30 smithi064 ceph-mon[212750]: failed to encode map e368 with expected crc 2023-10-09T06:03:30.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:30 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1296174659' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:03:30.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:30 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:03:30.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:30 smithi064 ceph-mon[220044]: pgmap v875: 119 pgs: 119 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T06:03:30.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:30 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:03:30.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:30 smithi064 ceph-mon[220044]: failed to encode map e368 with expected crc 2023-10-09T06:03:30.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:30 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2646198656' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:03:30.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:30 smithi064 ceph-mon[220044]: osdmap e368: 8 total, 8 up, 8 in 2023-10-09T06:03:30.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:30 smithi064 ceph-mon[220044]: failed to encode map e368 with expected crc 2023-10-09T06:03:30.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:30 smithi064 ceph-mon[220044]: failed to encode map e368 with expected crc 2023-10-09T06:03:30.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:30 smithi064 ceph-mon[220044]: failed to encode map e368 with expected crc 2023-10-09T06:03:30.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:30 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1296174659' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:03:30.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:30 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:03:30.730 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 28 16 47907 47891 106.86 67.5 0.00616317 0.00935244 2023-10-09T06:03:31.731 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 29 16 48997 48981 105.522 68.125 0.00763304 0.0094717 2023-10-09T06:03:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:31 smithi156 ceph-mon[195101]: failed to encode map e369 with expected crc 2023-10-09T06:03:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:31 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:03:31.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:31 smithi156 ceph-mon[195101]: osdmap e369: 8 total, 8 up, 8 in 2023-10-09T06:03:31.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:31 smithi156 ceph-mon[195101]: failed to encode map e369 with expected crc 2023-10-09T06:03:31.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:31 smithi156 ceph-mon[195101]: failed to encode map e369 with expected crc 2023-10-09T06:03:31.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:31 smithi156 ceph-mon[195101]: failed to encode map e369 with expected crc 2023-10-09T06:03:31.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:31 smithi156 ceph-mon[195101]: pgmap v878: 119 pgs: 119 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2023-10-09T06:03:31.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:31 smithi064 ceph-mon[212750]: failed to encode map e369 with expected crc 2023-10-09T06:03:31.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:31 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:03:31.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:31 smithi064 ceph-mon[212750]: osdmap e369: 8 total, 8 up, 8 in 2023-10-09T06:03:31.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:31 smithi064 ceph-mon[212750]: failed to encode map e369 with expected crc 2023-10-09T06:03:31.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:31 smithi064 ceph-mon[212750]: failed to encode map e369 with expected crc 2023-10-09T06:03:31.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:31 smithi064 ceph-mon[212750]: failed to encode map e369 with expected crc 2023-10-09T06:03:31.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:31 smithi064 ceph-mon[212750]: pgmap v878: 119 pgs: 119 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2023-10-09T06:03:31.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:31 smithi064 ceph-mon[220044]: failed to encode map e369 with expected crc 2023-10-09T06:03:31.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:31 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:03:31.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:31 smithi064 ceph-mon[220044]: osdmap e369: 8 total, 8 up, 8 in 2023-10-09T06:03:31.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:31 smithi064 ceph-mon[220044]: failed to encode map e369 with expected crc 2023-10-09T06:03:31.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:31 smithi064 ceph-mon[220044]: failed to encode map e369 with expected crc 2023-10-09T06:03:31.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:31 smithi064 ceph-mon[220044]: failed to encode map e369 with expected crc 2023-10-09T06:03:31.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:31 smithi064 ceph-mon[220044]: pgmap v878: 119 pgs: 119 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2023-10-09T06:03:32.731 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 30 16 50262 50246 104.64 79.0625 0.002549 0.00953939 2023-10-09T06:03:32.939 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:32.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:32.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:33.045 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:33.079 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T06:03:33.123 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:33.466 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:33.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:33.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:33.572 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:33.640 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:33.731 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 31 16 51339 51323 103.436 67.3125 0.0274772 0.00966193 2023-10-09T06:03:33.764 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:33.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:33.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:33.847 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:03:33.870 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:33.883 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:03:33.925 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:03:33.938 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:33.985 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:03:34.031 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:03:34.065 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:03:34.097 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:03:34.148 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:03:34.156 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:03:34.156 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:03:34.157 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:03:34.157 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:03:34.157 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:03:34.157 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:03:34.157 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:03:34.157 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:03:34.168 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:34.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:34.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:34.279 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:34.300 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:03:34.300 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:03:34.356 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:34.454 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:03:34.475 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:34 smithi064 ceph-mon[212750]: pgmap v879: 119 pgs: 119 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T06:03:34.475 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:34 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3693504616' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:34.476 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:34 smithi064 ceph-mon[220044]: pgmap v879: 119 pgs: 119 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T06:03:34.476 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:34 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3693504616' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:34.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:34 smithi156 ceph-mon[195101]: pgmap v879: 119 pgs: 119 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T06:03:34.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:34 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3693504616' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:34.731 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 32 16 52526 52510 102.522 74.1875 0.0221323 0.00974916 2023-10-09T06:03:34.941 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:34.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:34.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:35.046 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:35.569 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:03:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:03:35] "GET /metrics HTTP/1.1" 200 36378 "" "Prometheus/2.43.0" 2023-10-09T06:03:35.575 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:35.731 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 33 16 53900 53884 102.017 85.875 0.0173552 0.00979749 2023-10-09T06:03:36.436 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:36.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:36.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:36.547 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:36.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:36 smithi064 ceph-mon[212750]: pgmap v880: 119 pgs: 119 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s 2023-10-09T06:03:36.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:36 smithi064 ceph-mon[220044]: pgmap v880: 119 pgs: 119 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s 2023-10-09T06:03:36.652 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:36.678 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:03:36.718 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:36 smithi156 ceph-mon[195101]: pgmap v880: 119 pgs: 119 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s 2023-10-09T06:03:36.731 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 34 16 54988 54972 101.017 68 0.0177043 0.00989442 2023-10-09T06:03:36.737 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:36.853 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:36.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:36.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:36.942 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:36.959 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:37.013 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:37.090 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:37.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:37.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:37.200 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:37.241 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:03:37.282 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:37.731 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 35 16 56278 56262 100.434 80.625 0.0133541 0.00995267 2023-10-09T06:03:38.103 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:38.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:38.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:38.213 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:38.264 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:38.623 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:03:38.654 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:38 smithi064 ceph-mon[212750]: pgmap v881: 119 pgs: 119 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T06:03:38.655 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:38 smithi064 ceph-mon[220044]: pgmap v881: 119 pgs: 119 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T06:03:38.732 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 36 16 57256 57240 99.3422 61.125 0.0099161 0.0100609 2023-10-09T06:03:38.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:38 smithi156 ceph-mon[195101]: pgmap v881: 119 pgs: 119 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T06:03:39.593 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:39.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:39.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:39.710 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:03:39.731 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 37 16 58536 58520 98.8194 80 0.00762977 0.0101145 2023-10-09T06:03:39.753 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:39.827 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:03:39.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:39.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:03:39.936 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:40.057 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:40.402 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:40.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:40.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:40.511 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:40.557 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:40.580 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:40.732 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 38 16 59496 59480 97.7979 60 0.00362176 0.0101723 2023-10-09T06:03:40.737 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:40 smithi064 ceph-mon[212750]: pgmap v882: 119 pgs: 119 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T06:03:40.737 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:40 smithi064 ceph-mon[220044]: pgmap v882: 119 pgs: 119 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T06:03:40.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:40 smithi156 ceph-mon[195101]: pgmap v882: 119 pgs: 119 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T06:03:40.956 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:40.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:40.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:41.060 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:41.430 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:03:41.452 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T06:03:41.551 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:41.559 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:41 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1486842460' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:03:41.559 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:41 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:03:41.559 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:41 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1923961574' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:03:41.680 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:41 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1486842460' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:03:41.680 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:41 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:03:41.680 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:41 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1923961574' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:03:41.681 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:41 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1486842460' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:03:41.681 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:41 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:03:41.681 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:41 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1923961574' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:03:41.683 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:41.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:41.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:41.732 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 39 16 60584 60568 97.033 68 0.0156296 0.0103012 2023-10-09T06:03:41.796 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:03:41.828 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:41.874 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:03:41.875 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:03:41.913 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:03:41.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:41.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:03:42.018 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:42.086 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:42.611 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:42.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:42.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:42.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:42 smithi064 ceph-mon[212750]: pgmap v883: 119 pgs: 119 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:03:42.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:42 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:03:42.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:42 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:03:42.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:42 smithi064 ceph-mon[212750]: failed to encode map e370 with expected crc 2023-10-09T06:03:42.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:42 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:03:42.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:42 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1923961574' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:03:42.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:42 smithi064 ceph-mon[212750]: osdmap e370: 8 total, 8 up, 8 in 2023-10-09T06:03:42.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:42 smithi064 ceph-mon[212750]: failed to encode map e370 with expected crc 2023-10-09T06:03:42.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:42 smithi064 ceph-mon[212750]: failed to encode map e370 with expected crc 2023-10-09T06:03:42.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:42 smithi064 ceph-mon[220044]: pgmap v883: 119 pgs: 119 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:03:42.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:42 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:03:42.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:42 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T06:03:42.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:42 smithi064 ceph-mon[220044]: failed to encode map e370 with expected crc 2023-10-09T06:03:42.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:42 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:03:42.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:42 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1923961574' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:03:42.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:42 smithi064 ceph-mon[220044]: osdmap e370: 8 total, 8 up, 8 in 2023-10-09T06:03:42.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:42 smithi064 ceph-mon[220044]: failed to encode map e370 with expected crc 2023-10-09T06:03:42.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:42 smithi064 ceph-mon[220044]: failed to encode map e370 with expected crc 2023-10-09T06:03:42.729 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:42.732 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-10-09T06:03:42.733069+0000 min lat: 0.0010326 max lat: 0.330132 avg lat: 0.0103857 2023-10-09T06:03:42.732 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:03:42.733 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 40 15 61632 61617 96.2462 65.5625 0.00802773 0.0103857 2023-10-09T06:03:42.767 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:42 smithi156 ceph-mon[195101]: pgmap v883: 119 pgs: 119 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:03:42.767 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:42 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:03:42.767 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:42 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T06:03:42.767 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:42 smithi156 ceph-mon[195101]: failed to encode map e370 with expected crc 2023-10-09T06:03:42.767 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:42 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:03:42.767 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:42 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1923961574' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:03:42.767 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:42 smithi156 ceph-mon[195101]: osdmap e370: 8 total, 8 up, 8 in 2023-10-09T06:03:42.767 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:42 smithi156 ceph-mon[195101]: failed to encode map e370 with expected crc 2023-10-09T06:03:42.768 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:42 smithi156 ceph-mon[195101]: failed to encode map e370 with expected crc 2023-10-09T06:03:42.772 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:42.837 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:42.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:42.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:42.848 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:03:42.943 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:42.981 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:43.028 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:43.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:43.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:43.135 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:43.303 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:43.389 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:03:43.666 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:43 smithi156 ceph-mon[195101]: failed to encode map e370 with expected crc 2023-10-09T06:03:43.666 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:43 smithi156 ceph-mon[195101]: failed to encode map e370 with expected crc 2023-10-09T06:03:43.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:43 smithi064 ceph-mon[212750]: failed to encode map e370 with expected crc 2023-10-09T06:03:43.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:43 smithi064 ceph-mon[212750]: failed to encode map e370 with expected crc 2023-10-09T06:03:43.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:43 smithi064 ceph-mon[220044]: failed to encode map e370 with expected crc 2023-10-09T06:03:43.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:43 smithi064 ceph-mon[220044]: failed to encode map e370 with expected crc 2023-10-09T06:03:43.732 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 41 16 62407 62391 95.0786 48.375 0.00265714 0.0104795 2023-10-09T06:03:44.329 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T06:03:44.329 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:03:44.356 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:44.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:44.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:44.467 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:44.566 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:44 smithi064 ceph-mon[212750]: pgmap v885: 119 pgs: 119 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:03:44.566 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:44.566 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:44.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:44 smithi064 ceph-mon[220044]: pgmap v885: 119 pgs: 119 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:03:44.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:44.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:44.570 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:44.733 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 42 16 63537 63521 94.4963 70.625 0.0113368 0.0105772 2023-10-09T06:03:44.763 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:44 smithi156 ceph-mon[195101]: pgmap v885: 119 pgs: 119 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:03:44.763 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:44.763 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:44.789 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:44.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:44.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:44.905 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:03:44.960 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:45.096 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:03:45.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:45.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:03:45.148 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:03:45.187 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:03:45.202 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:03:45.228 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:03:45.244 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:45.280 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:03:45.307 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:03:45.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:45.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:03:45.324 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:03:45.364 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:03:45.407 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:03:45.407 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:03:45.407 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:03:45.407 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T06:03:45.408 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T06:03:45.408 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:03:45.408 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:45.417 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:45.446 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:03:45.453 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:03:45.453 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:03:45.453 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:03:45.453 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:03:45.453 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:03:45.453 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:03:45.454 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:03:45.454 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:03:45.507 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:45.720 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:03:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:03:45] "GET /metrics HTTP/1.1" 200 36379 "" "Prometheus/2.43.0" 2023-10-09T06:03:45.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:45 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/49260138' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:45.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:45 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/49260138' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:45.735 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 43 16 64640 64624 93.897 68.9375 0.00763922 0.0106441 2023-10-09T06:03:45.796 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:45.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:45.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:45.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:45 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/49260138' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:45.901 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:45.928 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:46.166 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:46.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:46.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:46.270 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:46.313 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:03:46.322 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:03:46.332 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:46.438 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:46.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:46.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:46.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:46 smithi064 ceph-mon[220044]: pgmap v886: 119 pgs: 119 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:03:46.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:46 smithi064 ceph-mon[212750]: pgmap v886: 119 pgs: 119 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:03:46.546 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:46.597 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:46 smithi156 ceph-mon[195101]: pgmap v886: 119 pgs: 119 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:03:46.597 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:46.683 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:46.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:46.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:46.778 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 44 16 65647 65631 93.1935 62.9375 0.0189917 0.0107254 2023-10-09T06:03:46.790 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:46.846 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:46.938 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:46.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:46.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:46.974 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:03:47.046 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:47.085 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:47.735 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 45 16 66713 66697 92.6029 66.625 0.0128871 0.0107923 2023-10-09T06:03:47.779 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:47.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:47.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:47.884 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:47.912 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:47.956 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:47.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:47.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:48.059 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:48.111 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:48.274 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:48.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:48.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:48.384 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:03:48.436 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:48.441 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:48.535 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:03:48.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:48.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:03:48.645 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:48.648 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:48 smithi064 ceph-mon[220044]: pgmap v887: 119 pgs: 119 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T06:03:48.648 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:48 smithi064 ceph-mon[212750]: pgmap v887: 119 pgs: 119 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T06:03:48.681 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:48 smithi156 ceph-mon[195101]: pgmap v887: 119 pgs: 119 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T06:03:48.689 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:48.736 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 46 16 67569 67553 91.7517 53.5 0.0101238 0.0108943 2023-10-09T06:03:48.781 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:48.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:48.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:48.885 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:48.966 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:49.035 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:03:49.035 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T06:03:49.035 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T06:03:49.035 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:03:49.035 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.036 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:110007 6.1d 6:b8aa09df:::benchmark_data_smithi064_284193_object13464:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2023-10-09T06:03:49.036 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:22.341065+0000", 2023-10-09T06:03:49.036 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 506.69123236899998, 2023-10-09T06:03:49.036 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9528985840000002, 2023-10-09T06:03:49.036 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:03:49.036 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.036 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:03:49.036 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:03:49.036 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:03:49.036 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 110007 2023-10-09T06:03:49.036 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.036 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:03:49.037 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.037 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:03:49.037 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.341065+0000", 2023-10-09T06:03:49.037 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.037 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.037 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.037 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:03:49.037 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.341065+0000", 2023-10-09T06:03:49.037 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.037 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.037 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.038 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:03:49.038 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.341064+0000", 2023-10-09T06:03:49.038 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:03:49.038 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.038 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.038 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:03:49.038 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.341070+0000", 2023-10-09T06:03:49.038 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6549999999999998e-06 2023-10-09T06:03:49.038 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.039 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.039 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:03:49.039 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.341087+0000", 2023-10-09T06:03:49.039 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6824999999999999e-05 2023-10-09T06:03:49.039 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.039 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.039 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.039 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.341093+0000", 2023-10-09T06:03:49.039 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8510000000000004e-06 2023-10-09T06:03:49.039 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.040 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.040 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.040 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.341175+0000", 2023-10-09T06:03:49.040 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.2087000000000002e-05 2023-10-09T06:03:49.040 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.040 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.040 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.040 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:22.341186+0000", 2023-10-09T06:03:49.040 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0701e-05 2023-10-09T06:03:49.041 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.041 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.041 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.041 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.172841+0000", 2023-10-09T06:03:49.041 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.83165533400000002 2023-10-09T06:03:49.041 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.041 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.041 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.041 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:23.172847+0000", 2023-10-09T06:03:49.042 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.891e-06 2023-10-09T06:03:49.042 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.042 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.042 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.042 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.286688+0000", 2023-10-09T06:03:49.042 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1138407260000001 2023-10-09T06:03:49.042 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.042 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.042 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:03:49.043 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.286821+0000", 2023-10-09T06:03:49.043 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013355999999999999 2023-10-09T06:03:49.043 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.043 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.043 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1,2", 2023-10-09T06:03:49.043 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.287004+0000", 2023-10-09T06:03:49.043 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00018287199999999999 2023-10-09T06:03:49.043 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.044 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.044 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:03:49.044 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.291811+0000", 2023-10-09T06:03:49.044 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0048072410000000003 2023-10-09T06:03:49.044 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.044 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.044 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.044 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.293518+0000", 2023-10-09T06:03:49.044 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0017064420000000001 2023-10-09T06:03:49.044 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.045 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.045 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.045 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.293622+0000", 2023-10-09T06:03:49.045 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000104602 2023-10-09T06:03:49.045 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.045 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.045 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.045 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.293839+0000", 2023-10-09T06:03:49.045 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000217054 2023-10-09T06:03:49.045 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.046 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.046 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:03:49.046 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.293884+0000", 2023-10-09T06:03:49.046 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4354000000000003e-05 2023-10-09T06:03:49.046 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.046 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.046 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:03:49.046 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:24.293964+0000", 2023-10-09T06:03:49.046 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9850000000000003e-05 2023-10-09T06:03:49.046 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.046 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:03:49.046 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.047 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.047 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.047 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:119182 6.d 6:b79c5f24:::benchmark_data_smithi064_284193_object22639:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e212)", 2023-10-09T06:03:49.047 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:43.662278+0000", 2023-10-09T06:03:49.047 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 485.37001929100001, 2023-10-09T06:03:49.047 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0380490080000002, 2023-10-09T06:03:49.047 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:03:49.047 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.047 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:03:49.047 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:03:49.048 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:03:49.048 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 119182 2023-10-09T06:03:49.048 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.048 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:03:49.048 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.048 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:03:49.048 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.662278+0000", 2023-10-09T06:03:49.048 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.048 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.049 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.049 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:03:49.049 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.662278+0000", 2023-10-09T06:03:49.049 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.049 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.049 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.049 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:03:49.049 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.662277+0000", 2023-10-09T06:03:49.049 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:03:49.049 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.050 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.050 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:03:49.050 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.662281+0000", 2023-10-09T06:03:49.050 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3100000000000002e-06 2023-10-09T06:03:49.050 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.050 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.050 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:03:49.050 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.662292+0000", 2023-10-09T06:03:49.050 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0444e-05 2023-10-09T06:03:49.050 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.050 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.051 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.051 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.662299+0000", 2023-10-09T06:03:49.051 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6140000000000003e-06 2023-10-09T06:03:49.051 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.051 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.051 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.051 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.680967+0000", 2023-10-09T06:03:49.051 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018667429999999999 2023-10-09T06:03:49.051 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.051 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.051 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.051 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:43.680979+0000", 2023-10-09T06:03:49.052 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2771e-05 2023-10-09T06:03:49.052 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.052 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.052 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.052 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.668837+0000", 2023-10-09T06:03:49.052 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.98785785100000001 2023-10-09T06:03:49.052 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.052 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.052 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.052 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.668849+0000", 2023-10-09T06:03:49.052 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1681e-05 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.689311+0000", 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0204623779999999 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.689472+0000", 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00016038200000000001 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.053 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.054 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:03:49.054 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.689627+0000", 2023-10-09T06:03:49.054 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000155284 2023-10-09T06:03:49.054 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.054 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.054 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:03:49.054 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.691124+0000", 2023-10-09T06:03:49.054 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0014973219999999999 2023-10-09T06:03:49.054 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.055 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.055 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.055 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.700147+0000", 2023-10-09T06:03:49.055 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0090226330000000004 2023-10-09T06:03:49.055 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.055 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.055 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.055 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.700223+0000", 2023-10-09T06:03:49.055 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.648e-05 2023-10-09T06:03:49.055 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.700284+0000", 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0338999999999999e-05 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.700327+0000", 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3349000000000001e-05 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.056 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.057 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.057 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:119212 6.1d 6:bf532f68:::benchmark_data_smithi064_284193_object22669:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2023-10-09T06:03:49.057 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:55:44.125794+0000", 2023-10-09T06:03:49.057 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 484.90650376600001, 2023-10-09T06:03:49.057 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5860949519999998, 2023-10-09T06:03:49.057 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:03:49.057 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.057 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:03:49.057 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:03:49.057 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:03:49.057 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 119212 2023-10-09T06:03:49.057 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.058 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:03:49.058 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.058 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:03:49.058 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.125794+0000", 2023-10-09T06:03:49.058 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.058 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.058 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.058 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:03:49.058 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.125794+0000", 2023-10-09T06:03:49.058 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.058 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.059 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.059 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:03:49.059 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.125792+0000", 2023-10-09T06:03:49.059 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:03:49.059 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.059 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.059 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:03:49.059 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.125799+0000", 2023-10-09T06:03:49.059 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7340000000000002e-06 2023-10-09T06:03:49.059 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.059 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.059 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.125818+0000", 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9021e-05 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.125824+0000", 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7479999999999999e-06 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.125878+0000", 2023-10-09T06:03:49.060 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4055999999999999e-05 2023-10-09T06:03:49.061 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.061 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.061 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.061 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.125887+0000", 2023-10-09T06:03:49.061 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.4939999999999992e-06 2023-10-09T06:03:49.061 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.061 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.061 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.061 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.672545+0000", 2023-10-09T06:03:49.062 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.54665808100000002 2023-10-09T06:03:49.062 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.062 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.062 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.062 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:44.672554+0000", 2023-10-09T06:03:49.062 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1819999999999995e-06 2023-10-09T06:03:49.062 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.062 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.062 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.062 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.698457+0000", 2023-10-09T06:03:49.062 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0259025020000001 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.698603+0000", 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014633800000000001 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.698704+0000", 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010069900000000001 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.063 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.064 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:03:49.064 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.700342+0000", 2023-10-09T06:03:49.064 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0016376730000000001 2023-10-09T06:03:49.064 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.064 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.064 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.064 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.708516+0000", 2023-10-09T06:03:49.064 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0081747599999999997 2023-10-09T06:03:49.064 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.064 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.064 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.064 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.711810+0000", 2023-10-09T06:03:49.065 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.003293921 2023-10-09T06:03:49.065 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.065 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.065 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:03:49.065 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.711852+0000", 2023-10-09T06:03:49.065 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1709000000000003e-05 2023-10-09T06:03:49.065 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.065 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.065 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:03:49.065 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:55:45.711889+0000", 2023-10-09T06:03:49.065 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6625999999999998e-05 2023-10-09T06:03:49.065 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.066 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:03:49.066 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.066 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.066 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.066 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:139361 6.24 6:25f0cdb9:::benchmark_data_smithi064_284193_object42818:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-10-09T06:03:49.066 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:56:40.361425+0000", 2023-10-09T06:03:49.066 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 428.67087200499998, 2023-10-09T06:03:49.066 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.982316784, 2023-10-09T06:03:49.066 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:03:49.066 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.066 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:03:49.067 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:03:49.067 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:03:49.067 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 139361 2023-10-09T06:03:49.067 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.067 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:03:49.067 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.067 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:03:49.067 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.361425+0000", 2023-10-09T06:03:49.067 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.067 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.067 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.067 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:03:49.068 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.361425+0000", 2023-10-09T06:03:49.068 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.068 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.068 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.068 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:03:49.068 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.361425+0000", 2023-10-09T06:03:49.068 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:03:49.068 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.068 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.068 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:03:49.068 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.361430+0000", 2023-10-09T06:03:49.068 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9989999999999999e-06 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.361439+0000", 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2439999999999999e-06 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.361444+0000", 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4480000000000002e-06 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.069 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.070 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.070 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.361547+0000", 2023-10-09T06:03:49.070 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000102945 2023-10-09T06:03:49.070 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.070 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.070 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.070 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.361564+0000", 2023-10-09T06:03:49.070 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6821e-05 2023-10-09T06:03:49.070 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.070 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.070 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.246627+0000", 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.88506318399999995 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.246634+0000", 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.832e-06 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.331376+0000", 2023-10-09T06:03:49.071 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0847416480000001 2023-10-09T06:03:49.072 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.072 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.072 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:03:49.072 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.331564+0000", 2023-10-09T06:03:49.072 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000188068 2023-10-09T06:03:49.072 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.072 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.072 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,4,6", 2023-10-09T06:03:49.072 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.331809+0000", 2023-10-09T06:03:49.072 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00024512299999999998 2023-10-09T06:03:49.072 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.335159+0000", 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0033495970000000002 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.339819+0000", 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0046607259999999996 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.073 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.074 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.340879+0000", 2023-10-09T06:03:49.074 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0010600850000000001 2023-10-09T06:03:49.074 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.074 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.074 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.074 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.342737+0000", 2023-10-09T06:03:49.074 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0018573859999999999 2023-10-09T06:03:49.074 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.074 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.074 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:03:49.075 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.342805+0000", 2023-10-09T06:03:49.075 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8281999999999994e-05 2023-10-09T06:03:49.075 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.075 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.075 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:03:49.075 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.343742+0000", 2023-10-09T06:03:49.075 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00093714299999999998 2023-10-09T06:03:49.075 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.075 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:03:49.076 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.076 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.076 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.076 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:139371 6.2d 6:b70b29b6:::benchmark_data_smithi064_284193_object42828:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-10-09T06:03:49.076 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:56:40.373512+0000", 2023-10-09T06:03:49.076 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 428.658785841, 2023-10-09T06:03:49.076 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8958427599999998, 2023-10-09T06:03:49.076 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:03:49.076 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.077 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:03:49.077 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:03:49.077 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:03:49.077 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 139371 2023-10-09T06:03:49.077 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.077 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:03:49.077 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.077 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:03:49.077 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.373512+0000", 2023-10-09T06:03:49.078 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.078 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.078 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.078 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:03:49.078 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.373512+0000", 2023-10-09T06:03:49.078 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.078 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.078 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.078 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:03:49.079 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.373510+0000", 2023-10-09T06:03:49.079 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:03:49.079 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.079 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.079 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:03:49.079 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.373516+0000", 2023-10-09T06:03:49.079 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6860000000000003e-06 2023-10-09T06:03:49.079 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.079 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.079 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:03:49.079 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.373531+0000", 2023-10-09T06:03:49.079 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4501000000000001e-05 2023-10-09T06:03:49.080 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.080 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.080 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.080 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.373537+0000", 2023-10-09T06:03:49.080 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1210000000000003e-06 2023-10-09T06:03:49.080 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.080 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.080 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.080 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.373583+0000", 2023-10-09T06:03:49.080 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5803999999999997e-05 2023-10-09T06:03:49.080 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.080 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.081 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.081 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.373590+0000", 2023-10-09T06:03:49.081 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4499999999999998e-06 2023-10-09T06:03:49.081 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.081 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.081 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.081 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.246594+0000", 2023-10-09T06:03:49.081 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.87300429599999996 2023-10-09T06:03:49.081 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.081 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.081 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.081 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.246602+0000", 2023-10-09T06:03:49.082 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0700000000000007e-06 2023-10-09T06:03:49.082 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.082 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.082 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.082 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.262220+0000", 2023-10-09T06:03:49.082 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.015617977 2023-10-09T06:03:49.082 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.082 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.082 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:03:49.082 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.262369+0000", 2023-10-09T06:03:49.082 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014906199999999999 2023-10-09T06:03:49.083 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.083 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.083 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1,2", 2023-10-09T06:03:49.083 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.262602+0000", 2023-10-09T06:03:49.083 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00023242900000000001 2023-10-09T06:03:49.083 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.083 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.083 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:03:49.083 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.266358+0000", 2023-10-09T06:03:49.083 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0037562889999999999 2023-10-09T06:03:49.083 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.083 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.084 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.084 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.266556+0000", 2023-10-09T06:03:49.084 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00019816500000000001 2023-10-09T06:03:49.084 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.084 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.084 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.084 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.266738+0000", 2023-10-09T06:03:49.084 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00018129999999999999 2023-10-09T06:03:49.084 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.084 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.084 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.084 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.269182+0000", 2023-10-09T06:03:49.085 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002444014 2023-10-09T06:03:49.085 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.085 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.085 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:03:49.085 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.269239+0000", 2023-10-09T06:03:49.085 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7680999999999999e-05 2023-10-09T06:03:49.085 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.085 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.085 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:03:49.085 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.269354+0000", 2023-10-09T06:03:49.085 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000115093 2023-10-09T06:03:49.085 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.086 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:03:49.086 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.086 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.086 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.086 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:139385 6.24 6:2459843b:::benchmark_data_smithi064_284193_object42842:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-10-09T06:03:49.086 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:56:40.417282+0000", 2023-10-09T06:03:49.086 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 428.61501506799999, 2023-10-09T06:03:49.086 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9290033069999999, 2023-10-09T06:03:49.086 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:03:49.086 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.086 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:03:49.086 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:03:49.087 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:03:49.087 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 139385 2023-10-09T06:03:49.087 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.087 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:03:49.087 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.087 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:03:49.087 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.417282+0000", 2023-10-09T06:03:49.087 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.087 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.087 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.087 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:03:49.088 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.417282+0000", 2023-10-09T06:03:49.088 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.088 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.088 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.088 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:03:49.088 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.417281+0000", 2023-10-09T06:03:49.088 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T06:03:49.088 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.088 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.088 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:03:49.088 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.417288+0000", 2023-10-09T06:03:49.088 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7020000000000001e-06 2023-10-09T06:03:49.089 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.089 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.089 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:03:49.089 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.417309+0000", 2023-10-09T06:03:49.089 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0528999999999999e-05 2023-10-09T06:03:49.089 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.089 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.089 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.089 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.417316+0000", 2023-10-09T06:03:49.089 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1679999999999998e-06 2023-10-09T06:03:49.089 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.089 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.090 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.090 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.417374+0000", 2023-10-09T06:03:49.090 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7525999999999998e-05 2023-10-09T06:03:49.090 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.090 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.090 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.090 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:40.417382+0000", 2023-10-09T06:03:49.090 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6389999999999998e-06 2023-10-09T06:03:49.090 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.090 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.090 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.090 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.246740+0000", 2023-10-09T06:03:49.091 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.82935787800000005 2023-10-09T06:03:49.091 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.091 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.091 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.091 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:41.246746+0000", 2023-10-09T06:03:49.091 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9780000000000002e-06 2023-10-09T06:03:49.091 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.091 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.091 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.091 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.332776+0000", 2023-10-09T06:03:49.091 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0860301640000001 2023-10-09T06:03:49.091 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.092 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.092 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:03:49.092 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.332915+0000", 2023-10-09T06:03:49.092 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000138808 2023-10-09T06:03:49.092 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.092 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.092 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,4,6", 2023-10-09T06:03:49.092 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.333118+0000", 2023-10-09T06:03:49.092 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00020254600000000001 2023-10-09T06:03:49.092 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.092 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.092 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:03:49.093 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.336415+0000", 2023-10-09T06:03:49.093 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0032973070000000002 2023-10-09T06:03:49.093 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.093 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.093 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.093 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.341347+0000", 2023-10-09T06:03:49.093 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0049323400000000003 2023-10-09T06:03:49.093 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.093 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.093 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.093 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.343782+0000", 2023-10-09T06:03:49.093 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0024349580000000001 2023-10-09T06:03:49.094 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.094 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.094 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.094 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.345379+0000", 2023-10-09T06:03:49.094 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0015963869999999999 2023-10-09T06:03:49.094 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.094 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.094 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:03:49.094 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.345456+0000", 2023-10-09T06:03:49.094 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7929000000000006e-05 2023-10-09T06:03:49.094 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.095 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.095 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:03:49.095 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:56:42.346286+0000", 2023-10-09T06:03:49.095 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000829194 2023-10-09T06:03:49.095 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.095 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:03:49.095 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.095 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.095 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.095 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:159581 6.4 6:210f1082:::benchmark_data_smithi064_284193_object63038:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2023-10-09T06:03:49.095 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:22.433041+0000", 2023-10-09T06:03:49.095 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 386.59925599100001, 2023-10-09T06:03:49.096 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.278757835, 2023-10-09T06:03:49.096 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:03:49.096 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:03:49.096 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:03:49.096 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:03:49.096 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:03:49.096 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 159581 2023-10-09T06:03:49.096 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.096 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:03:49.096 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.096 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:03:49.096 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.433041+0000", 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.433041+0000", 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.433040+0000", 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T06:03:49.097 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.098 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.098 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:03:49.098 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.433048+0000", 2023-10-09T06:03:49.098 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8399999999999995e-06 2023-10-09T06:03:49.098 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.098 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.098 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:03:49.098 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.433060+0000", 2023-10-09T06:03:49.098 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.279e-05 2023-10-09T06:03:49.098 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.098 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.099 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.099 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.433067+0000", 2023-10-09T06:03:49.099 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.917e-06 2023-10-09T06:03:49.099 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.099 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.099 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.099 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.442016+0000", 2023-10-09T06:03:49.099 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0089491350000000004 2023-10-09T06:03:49.099 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.099 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.099 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:03:49.099 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.442164+0000", 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014801899999999999 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,4,6", 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.442457+0000", 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00029275900000000002 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.445654+0000", 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0031968539999999998 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.100 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.101 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.101 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.447463+0000", 2023-10-09T06:03:49.101 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001809251 2023-10-09T06:03:49.101 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.101 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.101 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.101 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.458928+0000", 2023-10-09T06:03:49.101 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011464215 2023-10-09T06:03:49.101 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.101 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.101 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.101 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.458940+0000", 2023-10-09T06:03:49.102 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2177e-05 2023-10-09T06:03:49.102 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.102 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.102 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.102 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:23.465556+0000", 2023-10-09T06:03:49.102 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0066167749999999 2023-10-09T06:03:49.102 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.102 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.102 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.102 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:23.465569+0000", 2023-10-09T06:03:49.102 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2317000000000001e-05 2023-10-09T06:03:49.102 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.103 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.103 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.103 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.687557+0000", 2023-10-09T06:03:49.103 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.221987945 2023-10-09T06:03:49.103 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.103 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.103 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for missing object", 2023-10-09T06:03:49.103 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.687650+0000", 2023-10-09T06:03:49.103 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3535999999999993e-05 2023-10-09T06:03:49.103 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.103 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.103 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.104 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.711751+0000", 2023-10-09T06:03:49.104 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.02410052 2023-10-09T06:03:49.104 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.104 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.104 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:03:49.104 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.711799+0000", 2023-10-09T06:03:49.104 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8480999999999999e-05 2023-10-09T06:03:49.104 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.104 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:03:49.104 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.104 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:159597 6.4 6:223d96ac:::benchmark_data_smithi064_284193_object63054:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:22.459109+0000", 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 386.573188829, 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2528554199999999, 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 159597 2023-10-09T06:03:49.105 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.106 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:03:49.106 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.106 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:03:49.106 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.459109+0000", 2023-10-09T06:03:49.106 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.106 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.106 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.106 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:03:49.106 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.459109+0000", 2023-10-09T06:03:49.106 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.106 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.106 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.107 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:03:49.107 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.459107+0000", 2023-10-09T06:03:49.107 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:03:49.107 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.107 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.107 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:03:49.107 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.459116+0000", 2023-10-09T06:03:49.107 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9209999999999998e-06 2023-10-09T06:03:49.107 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.107 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.107 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:03:49.107 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.459129+0000", 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2853e-05 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.459136+0000", 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0849999999999999e-06 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.465479+0000", 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0063422599999999997 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.108 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:22.465488+0000", 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9420000000000001e-06 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:23.465654+0000", 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.000166192 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:03:49.109 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:23.465664+0000", 2023-10-09T06:03:49.110 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0171e-05 2023-10-09T06:03:49.110 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.110 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.110 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.110 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.687663+0000", 2023-10-09T06:03:49.110 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.221999211 2023-10-09T06:03:49.110 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.110 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.110 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:03:49.110 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.687741+0000", 2023-10-09T06:03:49.110 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8171999999999996e-05 2023-10-09T06:03:49.110 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.111 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.111 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:03:49.111 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.687967+0000", 2023-10-09T06:03:49.111 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00022523199999999999 2023-10-09T06:03:49.111 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.111 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.111 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:03:49.111 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.701380+0000", 2023-10-09T06:03:49.111 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.01341375 2023-10-09T06:03:49.111 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.111 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.711527+0000", 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010146919000000001 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.711831+0000", 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000303455 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:03:49.112 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.711859+0000", 2023-10-09T06:03:49.113 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7990000000000001e-05 2023-10-09T06:03:49.113 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.113 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.113 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:03:49.113 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:24.711964+0000", 2023-10-09T06:03:49.113 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010540900000000001 2023-10-09T06:03:49.113 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.113 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:03:49.113 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.113 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.113 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.113 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45263.0:199820 7.b e346/345 7:d94c8ab5:::benchmark_data_smithi064_398483_object94467:head v 346'12559, mlcod=346'12555)", 2023-10-09T06:03:49.114 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:02:15.470923+0000", 2023-10-09T06:03:49.114 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 93.561374608999998, 2023-10-09T06:03:49.114 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.030838332999999999, 2023-10-09T06:03:49.114 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:03:49.114 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.114 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:03:49.114 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.114 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:03:49.114 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:15.470923+0000", 2023-10-09T06:03:49.114 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.114 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.114 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.115 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:03:49.115 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:15.470923+0000", 2023-10-09T06:03:49.115 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.115 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.115 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.115 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:03:49.115 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:15.470922+0000", 2023-10-09T06:03:49.115 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:49.115 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.115 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.115 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:03:49.115 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:15.470929+0000", 2023-10-09T06:03:49.116 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5740000000000001e-06 2023-10-09T06:03:49.116 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.116 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.116 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:03:49.116 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:15.470938+0000", 2023-10-09T06:03:49.116 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.679e-06 2023-10-09T06:03:49.116 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.116 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.116 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.116 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:15.470944+0000", 2023-10-09T06:03:49.116 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0499999999999999e-06 2023-10-09T06:03:49.116 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.117 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.117 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.117 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:15.473002+0000", 2023-10-09T06:03:49.117 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002058149 2023-10-09T06:03:49.117 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.117 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.117 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:03:49.117 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:15.473016+0000", 2023-10-09T06:03:49.117 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3805000000000001e-05 2023-10-09T06:03:49.117 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.117 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.117 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:03:49.118 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:15.501716+0000", 2023-10-09T06:03:49.118 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.028700355 2023-10-09T06:03:49.118 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.118 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.118 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:03:49.118 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:15.501761+0000", 2023-10-09T06:03:49.118 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5312999999999998e-05 2023-10-09T06:03:49.118 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.118 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:03:49.118 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.118 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.118 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.119 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45263.0:199822 7.b e346/345 7:dcc69d2b:::benchmark_data_smithi064_398483_object94469:head v 346'12560, mlcod=346'12555)", 2023-10-09T06:03:49.119 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:02:15.473306+0000", 2023-10-09T06:03:49.119 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 93.558991078000005, 2023-10-09T06:03:49.119 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.028485715000000002, 2023-10-09T06:03:49.119 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:03:49.119 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.119 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:03:49.119 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.119 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:03:49.119 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:15.473306+0000", 2023-10-09T06:03:49.119 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.119 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.120 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.120 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:03:49.120 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:15.473306+0000", 2023-10-09T06:03:49.120 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.120 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.120 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.120 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:03:49.120 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:15.473306+0000", 2023-10-09T06:03:49.120 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:49.120 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.120 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.120 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:03:49.121 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:15.473312+0000", 2023-10-09T06:03:49.121 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4640000000000003e-06 2023-10-09T06:03:49.121 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.121 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.121 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:03:49.121 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:15.473322+0000", 2023-10-09T06:03:49.121 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.6670000000000008e-06 2023-10-09T06:03:49.121 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.121 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.121 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.121 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:15.473327+0000", 2023-10-09T06:03:49.121 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0819999999999998e-06 2023-10-09T06:03:49.122 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.122 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.122 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.122 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:15.490489+0000", 2023-10-09T06:03:49.122 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.017162141999999998 2023-10-09T06:03:49.122 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.122 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.122 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:03:49.122 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:15.490511+0000", 2023-10-09T06:03:49.122 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1464e-05 2023-10-09T06:03:49.122 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.122 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.123 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:03:49.123 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:15.501767+0000", 2023-10-09T06:03:49.123 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011256962000000001 2023-10-09T06:03:49.123 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.123 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.123 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:03:49.123 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:15.501792+0000", 2023-10-09T06:03:49.123 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4587999999999999e-05 2023-10-09T06:03:49.123 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.123 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:03:49.123 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.123 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.124 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.124 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45263.0:201410 7.e e348/285 7:726bdc74:::benchmark_data_smithi064_398483_object96057:head v 348'12737, mlcod=347'12735)", 2023-10-09T06:03:49.124 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:02:18.491410+0000", 2023-10-09T06:03:49.124 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 90.540887634000001, 2023-10-09T06:03:49.124 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.096427714999999997, 2023-10-09T06:03:49.124 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:03:49.124 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.124 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:03:49.124 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.124 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:03:49.124 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.491410+0000", 2023-10-09T06:03:49.124 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.125 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.125 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.125 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:03:49.125 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.491410+0000", 2023-10-09T06:03:49.125 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.125 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.125 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.125 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:03:49.125 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.491409+0000", 2023-10-09T06:03:49.125 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:49.125 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.125 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.126 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:03:49.126 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.491417+0000", 2023-10-09T06:03:49.126 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8720000000000002e-06 2023-10-09T06:03:49.126 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.126 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.126 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:03:49.126 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.491425+0000", 2023-10-09T06:03:49.126 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.2979999999999992e-06 2023-10-09T06:03:49.126 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.126 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.126 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.126 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.491430+0000", 2023-10-09T06:03:49.127 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2150000000000003e-06 2023-10-09T06:03:49.127 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.127 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.127 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.127 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.491472+0000", 2023-10-09T06:03:49.127 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2517e-05 2023-10-09T06:03:49.127 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.127 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.127 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:03:49.127 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.491491+0000", 2023-10-09T06:03:49.127 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9026e-05 2023-10-09T06:03:49.127 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.128 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.128 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:03:49.128 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.587793+0000", 2023-10-09T06:03:49.128 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.096301605999999998 2023-10-09T06:03:49.128 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.128 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.128 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:03:49.128 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.587838+0000", 2023-10-09T06:03:49.128 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4812999999999999e-05 2023-10-09T06:03:49.128 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.128 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45263.0:201412 7.e e348/285 7:7a811a08:::benchmark_data_smithi064_398483_object96059:head v 348'12738, mlcod=347'12735)", 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:02:18.496540+0000", 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 90.535757029999999, 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.091879891000000005, 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.129 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:03:49.130 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.496540+0000", 2023-10-09T06:03:49.130 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.130 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.130 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.130 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:03:49.130 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.496540+0000", 2023-10-09T06:03:49.130 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.130 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.130 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.130 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:03:49.130 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.496540+0000", 2023-10-09T06:03:49.130 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:49.131 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.131 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.131 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:03:49.131 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.496548+0000", 2023-10-09T06:03:49.131 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7190000000000001e-06 2023-10-09T06:03:49.131 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.131 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.131 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:03:49.131 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.579360+0000", 2023-10-09T06:03:49.131 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.082812151000000001 2023-10-09T06:03:49.131 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.131 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.132 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.132 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.579373+0000", 2023-10-09T06:03:49.132 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2999999999999999e-05 2023-10-09T06:03:49.132 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.132 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.132 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.132 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.583577+0000", 2023-10-09T06:03:49.132 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0042038350000000004 2023-10-09T06:03:49.132 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.132 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.133 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:03:49.133 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.583597+0000", 2023-10-09T06:03:49.133 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0599e-05 2023-10-09T06:03:49.133 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.133 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.133 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:03:49.133 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.588381+0000", 2023-10-09T06:03:49.133 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0047842379999999997 2023-10-09T06:03:49.133 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.133 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.133 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:03:49.133 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:18.588420+0000", 2023-10-09T06:03:49.134 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8948e-05 2023-10-09T06:03:49.134 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.134 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:03:49.134 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.134 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.134 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.134 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:64670 8.a e370/360 8:5d1c43ba:::benchmark_data_smithi064_457003_object64669:head v 370'4159, mlcod=370'4157)", 2023-10-09T06:03:49.134 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:03:45.761053+0000", 2023-10-09T06:03:49.134 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 3.2712446430000002, 2023-10-09T06:03:49.134 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.17253448699999999, 2023-10-09T06:03:49.134 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:03:49.135 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.135 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:03:49.135 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.135 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:03:49.135 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.761053+0000", 2023-10-09T06:03:49.135 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.135 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.135 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.135 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:03:49.135 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.761053+0000", 2023-10-09T06:03:49.135 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.135 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.136 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.136 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:03:49.136 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.761052+0000", 2023-10-09T06:03:49.136 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:49.136 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.136 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.136 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:03:49.136 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.761149+0000", 2023-10-09T06:03:49.136 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.6750999999999996e-05 2023-10-09T06:03:49.136 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.136 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.136 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.761155+0000", 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1399999999999997e-06 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.761159+0000", 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6849999999999999e-06 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.761196+0000", 2023-10-09T06:03:49.137 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7722999999999998e-05 2023-10-09T06:03:49.138 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.138 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.138 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:03:49.138 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.761209+0000", 2023-10-09T06:03:49.138 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.26e-05 2023-10-09T06:03:49.138 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.138 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.138 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:03:49.138 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.933547+0000", 2023-10-09T06:03:49.138 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.17233832399999999 2023-10-09T06:03:49.138 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.933587+0000", 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9922000000000002e-05 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:64669 8.8 e370/360 8:123de68f:::benchmark_data_smithi064_457003_object64668:head v 370'3998, mlcod=370'3995)", 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:03:45.761261+0000", 2023-10-09T06:03:49.139 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 3.2710362430000002, 2023-10-09T06:03:49.140 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.176026451, 2023-10-09T06:03:49.140 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:03:49.140 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.140 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:03:49.140 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.140 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:03:49.140 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.761261+0000", 2023-10-09T06:03:49.140 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.140 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.140 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.140 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:03:49.140 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.761261+0000", 2023-10-09T06:03:49.141 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.141 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.141 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.141 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:03:49.141 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.761261+0000", 2023-10-09T06:03:49.141 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:49.141 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.141 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.141 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:03:49.141 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.761362+0000", 2023-10-09T06:03:49.141 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000101599 2023-10-09T06:03:49.141 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.142 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.142 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:03:49.142 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.761371+0000", 2023-10-09T06:03:49.142 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9020000000000005e-06 2023-10-09T06:03:49.142 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.142 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.142 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.142 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.761376+0000", 2023-10-09T06:03:49.142 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.259e-06 2023-10-09T06:03:49.142 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.142 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.143 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.143 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.933401+0000", 2023-10-09T06:03:49.143 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.172024707 2023-10-09T06:03:49.143 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.143 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.143 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:03:49.143 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.933425+0000", 2023-10-09T06:03:49.143 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3516e-05 2023-10-09T06:03:49.143 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.143 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.143 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:03:49.143 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.937240+0000", 2023-10-09T06:03:49.144 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.003815854 2023-10-09T06:03:49.144 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.144 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.144 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:03:49.144 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.937288+0000", 2023-10-09T06:03:49.144 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7178999999999997e-05 2023-10-09T06:03:49.144 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.144 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:03:49.144 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.144 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.144 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.144 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:64674 8.3 e370/360 8:c0f06225:::benchmark_data_smithi064_457003_object64673:head v 370'4109, mlcod=370'4107)", 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:03:45.765173+0000", 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 3.2671239979999998, 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.174960429, 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.765173+0000", 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.145 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.146 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:03:49.146 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.765173+0000", 2023-10-09T06:03:49.146 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.146 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.146 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.146 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:03:49.146 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.765173+0000", 2023-10-09T06:03:49.146 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:49.146 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.146 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.146 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:03:49.146 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.765287+0000", 2023-10-09T06:03:49.147 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000114541 2023-10-09T06:03:49.147 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.147 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.147 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:03:49.147 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.932876+0000", 2023-10-09T06:03:49.147 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.16758910699999999 2023-10-09T06:03:49.147 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.147 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.147 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.147 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.932890+0000", 2023-10-09T06:03:49.147 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.384e-05 2023-10-09T06:03:49.147 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.148 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.148 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.148 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.933074+0000", 2023-10-09T06:03:49.148 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00018350899999999999 2023-10-09T06:03:49.148 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.148 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.148 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:03:49.148 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.933096+0000", 2023-10-09T06:03:49.148 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1735000000000001e-05 2023-10-09T06:03:49.148 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.148 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.148 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:03:49.149 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.940069+0000", 2023-10-09T06:03:49.149 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0069738059999999999 2023-10-09T06:03:49.149 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.149 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.149 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:03:49.149 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.940134+0000", 2023-10-09T06:03:49.149 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4529999999999994e-05 2023-10-09T06:03:49.149 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.149 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:03:49.149 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.149 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.149 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.150 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:66801 8.8 e370/360 8:1e71bc11:::benchmark_data_smithi064_457003_object66800:head v 370'4119, mlcod=370'4117)", 2023-10-09T06:03:49.150 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:03:47.803790+0000", 2023-10-09T06:03:49.150 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 1.2285071379999999, 2023-10-09T06:03:49.150 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.244972208, 2023-10-09T06:03:49.150 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:03:49.150 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.150 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:03:49.150 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.150 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:03:49.150 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.803790+0000", 2023-10-09T06:03:49.150 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.150 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.151 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.151 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:03:49.151 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.803790+0000", 2023-10-09T06:03:49.151 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.151 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.151 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.151 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:03:49.151 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.803789+0000", 2023-10-09T06:03:49.151 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:03:49.151 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.151 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.151 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:03:49.152 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.803850+0000", 2023-10-09T06:03:49.152 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0393000000000003e-05 2023-10-09T06:03:49.152 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.152 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.152 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:03:49.152 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.803861+0000", 2023-10-09T06:03:49.152 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0635999999999999e-05 2023-10-09T06:03:49.152 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.152 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.152 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.152 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.803866+0000", 2023-10-09T06:03:49.152 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5310000000000001e-06 2023-10-09T06:03:49.153 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.153 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.153 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.153 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.803906+0000", 2023-10-09T06:03:49.153 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0277000000000001e-05 2023-10-09T06:03:49.153 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.153 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.153 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:03:49.153 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.803924+0000", 2023-10-09T06:03:49.153 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8063e-05 2023-10-09T06:03:49.153 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.153 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.154 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:03:49.154 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.805945+0000", 2023-10-09T06:03:49.154 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0020206209999999998 2023-10-09T06:03:49.154 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.154 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.154 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:03:49.154 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048763+0000", 2023-10-09T06:03:49.154 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24281751800000001 2023-10-09T06:03:49.154 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.154 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:03:49.154 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.155 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.155 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.155 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:66805 8.d e370/360 8:beda68c4:::benchmark_data_smithi064_457003_object66804:head v 370'4056, mlcod=370'4054)", 2023-10-09T06:03:49.155 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:03:47.805362+0000", 2023-10-09T06:03:49.155 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 1.2269356519999999, 2023-10-09T06:03:49.155 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24348254499999999, 2023-10-09T06:03:49.155 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:03:49.155 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.155 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:03:49.155 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.155 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:03:49.155 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.805362+0000", 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.805362+0000", 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.805361+0000", 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.156 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.157 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:03:49.157 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.805385+0000", 2023-10-09T06:03:49.157 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4508999999999999e-05 2023-10-09T06:03:49.157 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.157 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.157 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:03:49.157 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.805396+0000", 2023-10-09T06:03:49.157 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0275000000000001e-05 2023-10-09T06:03:49.157 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.157 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.157 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.157 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.805402+0000", 2023-10-09T06:03:49.158 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1670000000000004e-06 2023-10-09T06:03:49.158 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.158 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.158 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.158 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.805446+0000", 2023-10-09T06:03:49.158 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4209999999999999e-05 2023-10-09T06:03:49.158 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.158 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.158 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:03:49.158 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.805464+0000", 2023-10-09T06:03:49.158 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7649e-05 2023-10-09T06:03:49.159 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.159 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.159 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:03:49.159 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048738+0000", 2023-10-09T06:03:49.159 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.243274142 2023-10-09T06:03:49.159 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.159 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.159 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:03:49.159 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048844+0000", 2023-10-09T06:03:49.159 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010647200000000001 2023-10-09T06:03:49.159 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.159 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:03:49.159 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.159 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.160 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.160 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:66804 8.a e370/360 8:503f273d:::benchmark_data_smithi064_457003_object66803:head v 370'4302, mlcod=370'4300)", 2023-10-09T06:03:49.160 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:03:47.807368+0000", 2023-10-09T06:03:49.160 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 1.224929578, 2023-10-09T06:03:49.160 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24623639999999999, 2023-10-09T06:03:49.160 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:03:49.160 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.160 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:03:49.160 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.160 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:03:49.160 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.807368+0000", 2023-10-09T06:03:49.160 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.161 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.161 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.161 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:03:49.161 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.807368+0000", 2023-10-09T06:03:49.161 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.161 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.161 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.161 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:03:49.161 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.807367+0000", 2023-10-09T06:03:49.161 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:49.161 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.161 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.162 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:03:49.162 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.807414+0000", 2023-10-09T06:03:49.162 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6770000000000001e-05 2023-10-09T06:03:49.162 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.162 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.162 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:03:49.162 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048692+0000", 2023-10-09T06:03:49.162 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24127759100000001 2023-10-09T06:03:49.162 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.162 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.162 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.162 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048718+0000", 2023-10-09T06:03:49.163 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.6829e-05 2023-10-09T06:03:49.163 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.163 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.163 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.163 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048837+0000", 2023-10-09T06:03:49.163 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011845099999999999 2023-10-09T06:03:49.163 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.163 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.163 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:03:49.163 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048858+0000", 2023-10-09T06:03:49.163 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1131000000000001e-05 2023-10-09T06:03:49.163 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.053554+0000", 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0046963669999999999 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.053604+0000", 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9839000000000002e-05 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:03:49.164 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.165 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.165 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.165 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:66806 8.d e370/360 8:ba263c5e:::benchmark_data_smithi064_457003_object66805:head v 370'4057, mlcod=370'4054)", 2023-10-09T06:03:49.165 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:03:47.808240+0000", 2023-10-09T06:03:49.165 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 1.224057255, 2023-10-09T06:03:49.165 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24444445000000001, 2023-10-09T06:03:49.165 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:03:49.165 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.165 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:03:49.165 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.165 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:03:49.165 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.808240+0000", 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.808240+0000", 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.808240+0000", 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.166 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.167 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:03:49.167 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.808280+0000", 2023-10-09T06:03:49.167 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0431e-05 2023-10-09T06:03:49.167 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.167 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.167 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:03:49.167 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048727+0000", 2023-10-09T06:03:49.167 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.240446787 2023-10-09T06:03:49.167 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.167 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.167 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.169 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048747+0000", 2023-10-09T06:03:49.169 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0438999999999999e-05 2023-10-09T06:03:49.169 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.169 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.169 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.169 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048872+0000", 2023-10-09T06:03:49.169 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012439300000000001 2023-10-09T06:03:49.169 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.170 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.170 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:03:49.170 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048889+0000", 2023-10-09T06:03:49.170 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7540000000000001e-05 2023-10-09T06:03:49.170 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.170 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.170 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:03:49.170 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.052646+0000", 2023-10-09T06:03:49.170 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0037566460000000002 2023-10-09T06:03:49.170 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.170 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.171 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:03:49.171 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.052685+0000", 2023-10-09T06:03:49.171 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8875999999999998e-05 2023-10-09T06:03:49.171 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.171 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:03:49.171 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.171 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.171 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.171 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:66808 8.8 e370/360 8:173e3c76:::benchmark_data_smithi064_457003_object66807:head v 370'4120, mlcod=370'4117)", 2023-10-09T06:03:49.171 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:03:47.809035+0000", 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 1.2232622559999999, 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24526658800000001, 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.809035+0000", 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:03:49.172 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.809035+0000", 2023-10-09T06:03:49.173 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:03:49.173 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.173 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.173 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:03:49.173 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.809035+0000", 2023-10-09T06:03:49.173 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:03:49.173 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.173 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.173 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:03:49.173 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.809092+0000", 2023-10-09T06:03:49.173 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7357e-05 2023-10-09T06:03:49.173 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.174 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.174 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:03:49.174 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048698+0000", 2023-10-09T06:03:49.174 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.239606072 2023-10-09T06:03:49.174 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.174 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.174 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:03:49.174 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048723+0000", 2023-10-09T06:03:49.174 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.5075999999999998e-05 2023-10-09T06:03:49.174 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.174 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.174 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048829+0000", 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000105544 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048855+0000", 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.6302999999999999e-05 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.054261+0000", 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0054058939999999996 2023-10-09T06:03:49.175 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:03:49.176 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:03:49.176 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:03:49.176 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.054302+0000", 2023-10-09T06:03:49.176 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1029999999999998e-05 2023-10-09T06:03:49.176 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.176 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:03:49.176 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.176 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:03:49.176 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:03:49.176 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:03:49.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:49.177 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:49.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:49.278 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:49.293 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:03:49.310 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T06:03:49.370 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:49.476 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:49.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:49.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:49.580 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:49.633 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:49.738 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 47 16 68401 68385 90.9023 52 0.0131035 0.0109959 2023-10-09T06:03:49.924 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:03:49.973 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:03:50.461 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:03:50.714 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:50 smithi156 ceph-mon[195101]: pgmap v888: 119 pgs: 119 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T06:03:50.715 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:50 smithi156 ceph-mon[195101]: from='client.45665 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:03:50.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:50 smithi064 ceph-mon[220044]: pgmap v888: 119 pgs: 119 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T06:03:50.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:50 smithi064 ceph-mon[220044]: from='client.45665 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:03:50.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:50 smithi064 ceph-mon[212750]: pgmap v888: 119 pgs: 119 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T06:03:50.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:50 smithi064 ceph-mon[212750]: from='client.45665 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:03:50.738 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 48 16 69204 69188 90.0542 50.1875 0.00884637 0.0110995 2023-10-09T06:03:50.935 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:50.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:50.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:51.086 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:51.187 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:51.209 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":888,"stamp":"2023-10-09T06:03:48.818623+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4228585376,"num_objects":64538,"num_object_clones":0,"num_object_copies":193614,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":64538,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64575,"num_write_kb":4128925,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":1324,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":69560,"ondisk_log_size":69560,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15530376,"kb_used_data":12366600,"kb_used_omap":163,"kb_used_meta":3163548,"kb_avail":734430840,"statfs":{"total":767960285184,"available":752057180160,"internally_reserved":0,"allocated":12663398400,"data_stored":12650786647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":167439,"internal_metadata":3239473649},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":711458816,"num_objects":10856,"num_object_clones":0,"num_object_copies":32568,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10856,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10856,"num_write_kb":694784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005865"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":600,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.492899+0000","last_change":"2023-10-09T06:00:05.413041+0000","last_active":"2023-10-09T06:03:30.492899+0000","last_peered":"2023-10-09T06:03:30.492899+0000","last_clean":"2023-10-09T06:03:30.492899+0000","last_became_active":"2023-10-09T05:59:40.308608+0000","last_became_peered":"2023-10-09T05:59:40.308608+0000","last_unstale":"2023-10-09T06:03:30.492899+0000","last_undegraded":"2023-10-09T06:03:30.492899+0000","last_fullsized":"2023-10-09T06:03:30.492899+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:05.412941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:05.412941+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:53.518055+0000","scrub_duration":0.0061014329999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":418,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.470166+0000","last_change":"2023-10-09T06:01:54.757141+0000","last_active":"2023-10-09T06:03:30.470166+0000","last_peered":"2023-10-09T06:03:30.470166+0000","last_clean":"2023-10-09T06:03:30.470166+0000","last_became_active":"2023-10-09T06:01:54.756855+0000","last_became_peered":"2023-10-09T06:01:54.756855+0000","last_unstale":"2023-10-09T06:03:30.470166+0000","last_undegraded":"2023-10-09T06:03:30.470166+0000","last_fullsized":"2023-10-09T06:03:30.470166+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:57.741319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":158,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.456520+0000","last_change":"2023-10-09T06:01:54.762232+0000","last_active":"2023-10-09T06:03:30.456520+0000","last_peered":"2023-10-09T06:03:30.456520+0000","last_clean":"2023-10-09T06:03:30.456520+0000","last_became_active":"2023-10-09T06:01:54.761935+0000","last_became_peered":"2023-10-09T06:01:54.761935+0000","last_unstale":"2023-10-09T06:03:30.456520+0000","last_undegraded":"2023-10-09T06:03:30.456520+0000","last_fullsized":"2023-10-09T06:03:30.456520+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:20.380704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":652,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.470053+0000","last_change":"2023-10-09T06:02:01.468449+0000","last_active":"2023-10-09T06:03:30.470053+0000","last_peered":"2023-10-09T06:03:30.470053+0000","last_clean":"2023-10-09T06:03:30.470053+0000","last_became_active":"2023-10-09T06:01:22.334923+0000","last_became_peered":"2023-10-09T06:01:22.334923+0000","last_unstale":"2023-10-09T06:03:30.470053+0000","last_undegraded":"2023-10-09T06:03:30.470053+0000","last_fullsized":"2023-10-09T06:03:30.470053+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:01.468346+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:01.468346+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:20:04.967048+0000","scrub_duration":0.0016258220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.463363+0000","last_change":"2023-10-09T06:01:54.762060+0000","last_active":"2023-10-09T06:03:30.463363+0000","last_peered":"2023-10-09T06:03:30.463363+0000","last_clean":"2023-10-09T06:03:30.463363+0000","last_became_active":"2023-10-09T06:01:54.761757+0000","last_became_peered":"2023-10-09T06:01:54.761757+0000","last_unstale":"2023-10-09T06:03:30.463363+0000","last_undegraded":"2023-10-09T06:03:30.463363+0000","last_fullsized":"2023-10-09T06:03:30.463363+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:40.294232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":330,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.493270+0000","last_change":"2023-10-09T06:01:41.942412+0000","last_active":"2023-10-09T06:03:30.493270+0000","last_peered":"2023-10-09T06:03:30.493270+0000","last_clean":"2023-10-09T06:03:30.493270+0000","last_became_active":"2023-10-09T06:01:41.941186+0000","last_became_peered":"2023-10-09T06:01:41.941186+0000","last_unstale":"2023-10-09T06:03:30.493270+0000","last_undegraded":"2023-10-09T06:03:30.493270+0000","last_fullsized":"2023-10-09T06:03:30.493270+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:31.680824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":614,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.469931+0000","last_change":"2023-10-09T06:02:43.293521+0000","last_active":"2023-10-09T06:03:30.469931+0000","last_peered":"2023-10-09T06:03:30.469931+0000","last_clean":"2023-10-09T06:03:30.469931+0000","last_became_active":"2023-10-09T06:01:57.803642+0000","last_became_peered":"2023-10-09T06:01:57.803642+0000","last_unstale":"2023-10-09T06:03:30.469931+0000","last_undegraded":"2023-10-09T06:03:30.469931+0000","last_fullsized":"2023-10-09T06:03:30.469931+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.293426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.293426+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:45.626761+0000","scrub_duration":0.001587946,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":143,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.481432+0000","last_change":"2023-10-09T06:01:54.760903+0000","last_active":"2023-10-09T06:03:30.481432+0000","last_peered":"2023-10-09T06:03:30.481432+0000","last_clean":"2023-10-09T06:03:30.481432+0000","last_became_active":"2023-10-09T06:01:54.760338+0000","last_became_peered":"2023-10-09T06:01:54.760338+0000","last_unstale":"2023-10-09T06:03:30.481432+0000","last_undegraded":"2023-10-09T06:03:30.481432+0000","last_fullsized":"2023-10-09T06:03:30.481432+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:53.904647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":406,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.475141+0000","last_change":"2023-10-09T06:01:54.757269+0000","last_active":"2023-10-09T06:03:30.475141+0000","last_peered":"2023-10-09T06:03:30.475141+0000","last_clean":"2023-10-09T06:03:30.475141+0000","last_became_active":"2023-10-09T06:01:54.756894+0000","last_became_peered":"2023-10-09T06:01:54.756894+0000","last_unstale":"2023-10-09T06:03:30.475141+0000","last_undegraded":"2023-10-09T06:03:30.475141+0000","last_fullsized":"2023-10-09T06:03:30.475141+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:45.212821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":689,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.493182+0000","last_change":"2023-10-09T06:01:57.804160+0000","last_active":"2023-10-09T06:03:30.493182+0000","last_peered":"2023-10-09T06:03:30.493182+0000","last_clean":"2023-10-09T06:03:30.493182+0000","last_became_active":"2023-10-09T06:01:57.803424+0000","last_became_peered":"2023-10-09T06:01:57.803424+0000","last_unstale":"2023-10-09T06:03:30.493182+0000","last_undegraded":"2023-10-09T06:03:30.493182+0000","last_fullsized":"2023-10-09T06:03:30.493182+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.384407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.384407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:42.857346+0000","scrub_duration":0.0040651960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":248,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.456902+0000","last_change":"2023-10-09T06:01:41.939852+0000","last_active":"2023-10-09T06:03:30.456902+0000","last_peered":"2023-10-09T06:03:30.456902+0000","last_clean":"2023-10-09T06:03:30.456902+0000","last_became_active":"2023-10-09T06:01:41.939556+0000","last_became_peered":"2023-10-09T06:01:41.939556+0000","last_unstale":"2023-10-09T06:03:30.456902+0000","last_undegraded":"2023-10-09T06:03:30.456902+0000","last_fullsized":"2023-10-09T06:03:30.456902+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:06.980511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":158,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.481473+0000","last_change":"2023-10-09T06:01:54.761101+0000","last_active":"2023-10-09T06:03:30.481473+0000","last_peered":"2023-10-09T06:03:30.481473+0000","last_clean":"2023-10-09T06:03:30.481473+0000","last_became_active":"2023-10-09T06:01:54.760571+0000","last_became_peered":"2023-10-09T06:01:54.760571+0000","last_unstale":"2023-10-09T06:03:30.481473+0000","last_undegraded":"2023-10-09T06:03:30.481473+0000","last_fullsized":"2023-10-09T06:03:30.481473+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:56.812358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":651,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.471347+0000","last_change":"2023-10-09T06:01:22.334061+0000","last_active":"2023-10-09T06:03:30.471347+0000","last_peered":"2023-10-09T06:03:30.471347+0000","last_clean":"2023-10-09T06:03:30.471347+0000","last_became_active":"2023-10-09T06:01:22.333698+0000","last_became_peered":"2023-10-09T06:01:22.333698+0000","last_unstale":"2023-10-09T06:03:30.471347+0000","last_undegraded":"2023-10-09T06:03:30.471347+0000","last_fullsized":"2023-10-09T06:03:30.471347+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.913004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.913004+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:57.020327+0000","scrub_duration":0.0017791980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":419,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.493404+0000","last_change":"2023-10-09T06:01:54.766344+0000","last_active":"2023-10-09T06:03:30.493404+0000","last_peered":"2023-10-09T06:03:30.493404+0000","last_clean":"2023-10-09T06:03:30.493404+0000","last_became_active":"2023-10-09T06:01:54.766086+0000","last_became_peered":"2023-10-09T06:01:54.766086+0000","last_unstale":"2023-10-09T06:03:30.493404+0000","last_undegraded":"2023-10-09T06:03:30.493404+0000","last_fullsized":"2023-10-09T06:03:30.493404+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:47.163700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":316,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.455984+0000","last_change":"2023-10-09T05:53:59.185702+0000","last_active":"2023-10-09T06:03:30.455984+0000","last_peered":"2023-10-09T06:03:30.455984+0000","last_clean":"2023-10-09T06:03:30.455984+0000","last_became_active":"2023-10-09T05:53:59.185373+0000","last_became_peered":"2023-10-09T05:53:59.185373+0000","last_unstale":"2023-10-09T06:03:30.455984+0000","last_undegraded":"2023-10-09T06:03:30.455984+0000","last_fullsized":"2023-10-09T06:03:30.455984+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:50.756967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":787,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.475221+0000","last_change":"2023-10-09T06:02:48.374519+0000","last_active":"2023-10-09T06:03:30.475221+0000","last_peered":"2023-10-09T06:03:30.475221+0000","last_clean":"2023-10-09T06:03:30.475221+0000","last_became_active":"2023-10-09T06:01:41.939691+0000","last_became_peered":"2023-10-09T06:01:41.939691+0000","last_unstale":"2023-10-09T06:03:30.475221+0000","last_undegraded":"2023-10-09T06:03:30.475221+0000","last_fullsized":"2023-10-09T06:03:30.475221+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:48.374432+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:48.374432+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:54.162995+0000","scrub_duration":0.001678585,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":227,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.493631+0000","last_change":"2023-10-09T06:01:22.337444+0000","last_active":"2023-10-09T06:03:30.493631+0000","last_peered":"2023-10-09T06:03:30.493631+0000","last_clean":"2023-10-09T06:03:30.493631+0000","last_became_active":"2023-10-09T06:01:22.336348+0000","last_became_peered":"2023-10-09T06:01:22.336348+0000","last_unstale":"2023-10-09T06:03:30.493631+0000","last_undegraded":"2023-10-09T06:03:30.493631+0000","last_fullsized":"2023-10-09T06:03:30.493631+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:43:11.149649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.462568+0000","last_change":"2023-10-09T06:01:54.755723+0000","last_active":"2023-10-09T06:03:30.462568+0000","last_peered":"2023-10-09T06:03:30.462568+0000","last_clean":"2023-10-09T06:03:30.462568+0000","last_became_active":"2023-10-09T06:01:54.755334+0000","last_became_peered":"2023-10-09T06:01:54.755334+0000","last_unstale":"2023-10-09T06:03:30.462568+0000","last_undegraded":"2023-10-09T06:03:30.462568+0000","last_fullsized":"2023-10-09T06:03:30.462568+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:12.041652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":522,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.462425+0000","last_change":"2023-10-09T06:01:57.807839+0000","last_active":"2023-10-09T06:03:30.462425+0000","last_peered":"2023-10-09T06:03:30.462425+0000","last_clean":"2023-10-09T06:03:30.462425+0000","last_became_active":"2023-10-09T06:01:57.807451+0000","last_became_peered":"2023-10-09T06:01:57.807451+0000","last_unstale":"2023-10-09T06:03:30.462425+0000","last_undegraded":"2023-10-09T06:03:30.462425+0000","last_fullsized":"2023-10-09T06:03:30.462425+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.339121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.339121+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:47.740601+0000","scrub_duration":0.010474344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":364,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.470470+0000","last_change":"2023-10-09T06:01:22.335608+0000","last_active":"2023-10-09T06:03:30.470470+0000","last_peered":"2023-10-09T06:03:30.470470+0000","last_clean":"2023-10-09T06:03:30.470470+0000","last_became_active":"2023-10-09T06:01:22.335296+0000","last_became_peered":"2023-10-09T06:01:22.335296+0000","last_unstale":"2023-10-09T06:03:30.470470+0000","last_undegraded":"2023-10-09T06:03:30.470470+0000","last_fullsized":"2023-10-09T06:03:30.470470+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:21.843341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":382,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.470182+0000","last_change":"2023-10-09T06:01:54.764916+0000","last_active":"2023-10-09T06:03:30.470182+0000","last_peered":"2023-10-09T06:03:30.470182+0000","last_clean":"2023-10-09T06:03:30.470182+0000","last_became_active":"2023-10-09T06:01:54.764705+0000","last_became_peered":"2023-10-09T06:01:54.764705+0000","last_unstale":"2023-10-09T06:03:30.470182+0000","last_undegraded":"2023-10-09T06:03:30.470182+0000","last_fullsized":"2023-10-09T06:03:30.470182+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:12.527719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":544,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.455519+0000","last_change":"2023-10-09T06:02:42.900973+0000","last_active":"2023-10-09T06:03:30.455519+0000","last_peered":"2023-10-09T06:03:30.455519+0000","last_clean":"2023-10-09T06:03:30.455519+0000","last_became_active":"2023-10-09T06:01:41.939455+0000","last_became_peered":"2023-10-09T06:01:41.939455+0000","last_unstale":"2023-10-09T06:03:30.455519+0000","last_undegraded":"2023-10-09T06:03:30.455519+0000","last_fullsized":"2023-10-09T06:03:30.455519+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:42.900876+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:42.900876+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:37.457334+0000","scrub_duration":0.0014928649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":229,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.492222+0000","last_change":"2023-10-09T06:01:22.339830+0000","last_active":"2023-10-09T06:03:30.492222+0000","last_peered":"2023-10-09T06:03:30.492222+0000","last_clean":"2023-10-09T06:03:30.492222+0000","last_became_active":"2023-10-09T06:01:22.339525+0000","last_became_peered":"2023-10-09T06:01:22.339525+0000","last_unstale":"2023-10-09T06:03:30.492222+0000","last_undegraded":"2023-10-09T06:03:30.492222+0000","last_fullsized":"2023-10-09T06:03:30.492222+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:57.521163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":411,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.481717+0000","last_change":"2023-10-09T06:01:54.761558+0000","last_active":"2023-10-09T06:03:30.481717+0000","last_peered":"2023-10-09T06:03:30.481717+0000","last_clean":"2023-10-09T06:03:30.481717+0000","last_became_active":"2023-10-09T06:01:54.761251+0000","last_became_peered":"2023-10-09T06:01:54.761251+0000","last_unstale":"2023-10-09T06:03:30.481717+0000","last_undegraded":"2023-10-09T06:03:30.481717+0000","last_fullsized":"2023-10-09T06:03:30.481717+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:53.245267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":793,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.493189+0000","last_change":"2023-10-09T05:59:15.170543+0000","last_active":"2023-10-09T06:03:30.493189+0000","last_peered":"2023-10-09T06:03:30.493189+0000","last_clean":"2023-10-09T06:03:30.493189+0000","last_became_active":"2023-10-09T05:58:21.332533+0000","last_became_peered":"2023-10-09T05:58:21.332533+0000","last_unstale":"2023-10-09T06:03:30.493189+0000","last_undegraded":"2023-10-09T06:03:30.493189+0000","last_fullsized":"2023-10-09T06:03:30.493189+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:33.995695+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":295,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.492123+0000","last_change":"2023-10-09T06:01:41.942115+0000","last_active":"2023-10-09T06:03:30.492123+0000","last_peered":"2023-10-09T06:03:30.492123+0000","last_clean":"2023-10-09T06:03:30.492123+0000","last_became_active":"2023-10-09T06:01:41.941827+0000","last_became_peered":"2023-10-09T06:01:41.941827+0000","last_unstale":"2023-10-09T06:03:30.492123+0000","last_undegraded":"2023-10-09T06:03:30.492123+0000","last_fullsized":"2023-10-09T06:03:30.492123+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:51:15.226428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":293,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.493380+0000","last_change":"2023-10-09T05:56:49.237784+0000","last_active":"2023-10-09T06:03:30.493380+0000","last_peered":"2023-10-09T06:03:30.493380+0000","last_clean":"2023-10-09T06:03:30.493380+0000","last_became_active":"2023-10-09T05:56:49.237417+0000","last_became_peered":"2023-10-09T05:56:49.237417+0000","last_unstale":"2023-10-09T06:03:30.493380+0000","last_undegraded":"2023-10-09T06:03:30.493380+0000","last_fullsized":"2023-10-09T06:03:30.493380+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:33.129268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":569,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.464182+0000","last_change":"2023-10-09T06:02:43.316814+0000","last_active":"2023-10-09T06:03:30.464182+0000","last_peered":"2023-10-09T06:03:30.464182+0000","last_clean":"2023-10-09T06:03:30.464182+0000","last_became_active":"2023-10-09T06:01:57.807514+0000","last_became_peered":"2023-10-09T06:01:57.807514+0000","last_unstale":"2023-10-09T06:03:30.464182+0000","last_undegraded":"2023-10-09T06:03:30.464182+0000","last_fullsized":"2023-10-09T06:03:30.464182+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.316697+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.316697+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:15.123801+0000","scrub_duration":0.001666561,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":223,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.492209+0000","last_change":"2023-10-09T06:01:22.337171+0000","last_active":"2023-10-09T06:03:30.492209+0000","last_peered":"2023-10-09T06:03:30.492209+0000","last_clean":"2023-10-09T06:03:30.492209+0000","last_became_active":"2023-10-09T06:01:22.335438+0000","last_became_peered":"2023-10-09T06:01:22.335438+0000","last_unstale":"2023-10-09T06:03:30.492209+0000","last_undegraded":"2023-10-09T06:03:30.492209+0000","last_fullsized":"2023-10-09T06:03:30.492209+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:25.036448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":292,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.456855+0000","last_change":"2023-10-09T06:01:41.941888+0000","last_active":"2023-10-09T06:03:30.456855+0000","last_peered":"2023-10-09T06:03:30.456855+0000","last_clean":"2023-10-09T06:03:30.456855+0000","last_became_active":"2023-10-09T06:01:41.941413+0000","last_became_peered":"2023-10-09T06:01:41.941413+0000","last_unstale":"2023-10-09T06:03:30.456855+0000","last_undegraded":"2023-10-09T06:03:30.456855+0000","last_fullsized":"2023-10-09T06:03:30.456855+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:32:41.773690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":621,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.462826+0000","last_change":"2023-10-09T06:02:40.388302+0000","last_active":"2023-10-09T06:03:30.462826+0000","last_peered":"2023-10-09T06:03:30.462826+0000","last_clean":"2023-10-09T06:03:30.462826+0000","last_became_active":"2023-10-09T06:01:57.803733+0000","last_became_peered":"2023-10-09T06:01:57.803733+0000","last_unstale":"2023-10-09T06:03:30.462826+0000","last_undegraded":"2023-10-09T06:03:30.462826+0000","last_fullsized":"2023-10-09T06:03:30.462826+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:40.388125+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:40.388125+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:17.919838+0000","scrub_duration":0.0055576990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":355,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.474600+0000","last_change":"2023-10-09T05:51:57.091639+0000","last_active":"2023-10-09T06:03:30.474600+0000","last_peered":"2023-10-09T06:03:30.474600+0000","last_clean":"2023-10-09T06:03:30.474600+0000","last_became_active":"2023-10-09T05:51:57.091284+0000","last_became_peered":"2023-10-09T05:51:57.091284+0000","last_unstale":"2023-10-09T06:03:30.474600+0000","last_undegraded":"2023-10-09T06:03:30.474600+0000","last_fullsized":"2023-10-09T06:03:30.474600+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:10.553453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":152,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.456964+0000","last_change":"2023-10-09T06:01:54.760790+0000","last_active":"2023-10-09T06:03:30.456964+0000","last_peered":"2023-10-09T06:03:30.456964+0000","last_clean":"2023-10-09T06:03:30.456964+0000","last_became_active":"2023-10-09T06:01:54.760439+0000","last_became_peered":"2023-10-09T06:01:54.760439+0000","last_unstale":"2023-10-09T06:03:30.456964+0000","last_undegraded":"2023-10-09T06:03:30.456964+0000","last_fullsized":"2023-10-09T06:03:30.456964+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:50.428356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"370'4040","reported_seq":4058,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:45.398926+0000","last_change":"2023-10-09T06:02:55.482628+0000","last_active":"2023-10-09T06:03:45.398926+0000","last_peered":"2023-10-09T06:03:45.398926+0000","last_clean":"2023-10-09T06:03:45.398926+0000","last_became_active":"2023-10-09T06:02:55.482332+0000","last_became_peered":"2023-10-09T06:02:55.482332+0000","last_unstale":"2023-10-09T06:03:45.398926+0000","last_undegraded":"2023-10-09T06:03:45.398926+0000","last_fullsized":"2023-10-09T06:03:45.398926+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4040,"log_dups_size":0,"ondisk_log_size":4040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:58:21.128696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264765440,"num_objects":4040,"num_object_clones":0,"num_object_copies":12120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4040,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4040,"num_write_kb":258560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":746,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.493265+0000","last_change":"2023-10-09T06:01:57.804339+0000","last_active":"2023-10-09T06:03:30.493265+0000","last_peered":"2023-10-09T06:03:30.493265+0000","last_clean":"2023-10-09T06:03:30.493265+0000","last_became_active":"2023-10-09T06:01:57.803844+0000","last_became_peered":"2023-10-09T06:01:57.803844+0000","last_unstale":"2023-10-09T06:03:30.493265+0000","last_undegraded":"2023-10-09T06:03:30.493265+0000","last_fullsized":"2023-10-09T06:03:30.493265+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:53.469394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:53.469394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:51.125394+0000","scrub_duration":0.0059716659999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":227,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.492422+0000","last_change":"2023-10-09T06:01:22.336769+0000","last_active":"2023-10-09T06:03:30.492422+0000","last_peered":"2023-10-09T06:03:30.492422+0000","last_clean":"2023-10-09T06:03:30.492422+0000","last_became_active":"2023-10-09T06:01:22.336264+0000","last_became_peered":"2023-10-09T06:01:22.336264+0000","last_unstale":"2023-10-09T06:03:30.492422+0000","last_undegraded":"2023-10-09T06:03:30.492422+0000","last_fullsized":"2023-10-09T06:03:30.492422+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:17.065771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":276,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.493030+0000","last_change":"2023-10-09T05:56:49.234185+0000","last_active":"2023-10-09T06:03:30.493030+0000","last_peered":"2023-10-09T06:03:30.493030+0000","last_clean":"2023-10-09T06:03:30.493030+0000","last_became_active":"2023-10-09T05:56:49.233884+0000","last_became_peered":"2023-10-09T05:56:49.233884+0000","last_unstale":"2023-10-09T06:03:30.493030+0000","last_undegraded":"2023-10-09T06:03:30.493030+0000","last_fullsized":"2023-10-09T06:03:30.493030+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:12.142436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"370'4233","reported_seq":4251,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:48.592307+0000","last_change":"2023-10-09T06:02:55.459779+0000","last_active":"2023-10-09T06:03:48.592307+0000","last_peered":"2023-10-09T06:03:48.592307+0000","last_clean":"2023-10-09T06:03:48.592307+0000","last_became_active":"2023-10-09T06:02:55.458593+0000","last_became_peered":"2023-10-09T06:02:55.458593+0000","last_unstale":"2023-10-09T06:03:48.592307+0000","last_undegraded":"2023-10-09T06:03:48.592307+0000","last_fullsized":"2023-10-09T06:03:48.592307+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4233,"log_dups_size":0,"ondisk_log_size":4233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:03.468621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277413888,"num_objects":4233,"num_object_clones":0,"num_object_copies":12699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4233,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4233,"num_write_kb":270912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":677,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.481284+0000","last_change":"2023-10-09T06:00:03.241479+0000","last_active":"2023-10-09T06:03:30.481284+0000","last_peered":"2023-10-09T06:03:30.481284+0000","last_clean":"2023-10-09T06:03:30.481284+0000","last_became_active":"2023-10-09T05:59:32.530753+0000","last_became_peered":"2023-10-09T05:59:32.530753+0000","last_unstale":"2023-10-09T06:03:30.481284+0000","last_undegraded":"2023-10-09T06:03:30.481284+0000","last_fullsized":"2023-10-09T06:03:30.481284+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:03.241335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:03.241335+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:54.822114+0000","scrub_duration":0.017780547000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":283,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.493485+0000","last_change":"2023-10-09T06:01:54.826540+0000","last_active":"2023-10-09T06:03:30.493485+0000","last_peered":"2023-10-09T06:03:30.493485+0000","last_clean":"2023-10-09T06:03:30.493485+0000","last_became_active":"2023-10-09T06:01:54.766205+0000","last_became_peered":"2023-10-09T06:01:54.766205+0000","last_unstale":"2023-10-09T06:03:30.493485+0000","last_undegraded":"2023-10-09T06:03:30.493485+0000","last_fullsized":"2023-10-09T06:03:30.493485+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:45.220336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":408,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.469739+0000","last_change":"2023-10-09T06:01:54.760367+0000","last_active":"2023-10-09T06:03:30.469739+0000","last_peered":"2023-10-09T06:03:30.469739+0000","last_clean":"2023-10-09T06:03:30.469739+0000","last_became_active":"2023-10-09T06:01:54.760087+0000","last_became_peered":"2023-10-09T06:01:54.760087+0000","last_unstale":"2023-10-09T06:03:30.469739+0000","last_undegraded":"2023-10-09T06:03:30.469739+0000","last_fullsized":"2023-10-09T06:03:30.469739+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:26.000104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"370'3978","reported_seq":3997,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:45.183380+0000","last_change":"2023-10-09T06:02:55.459326+0000","last_active":"2023-10-09T06:03:45.183380+0000","last_peered":"2023-10-09T06:03:45.183380+0000","last_clean":"2023-10-09T06:03:45.183380+0000","last_became_active":"2023-10-09T06:02:55.458307+0000","last_became_peered":"2023-10-09T06:02:55.458307+0000","last_unstale":"2023-10-09T06:03:45.183380+0000","last_undegraded":"2023-10-09T06:03:45.183380+0000","last_fullsized":"2023-10-09T06:03:45.183380+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":3978,"log_dups_size":0,"ondisk_log_size":3978,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:04.011767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260702208,"num_objects":3978,"num_object_clones":0,"num_object_copies":11934,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3978,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3978,"num_write_kb":254592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":809,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.475279+0000","last_change":"2023-10-09T06:01:22.326209+0000","last_active":"2023-10-09T06:03:30.475279+0000","last_peered":"2023-10-09T06:03:30.475279+0000","last_clean":"2023-10-09T06:03:30.475279+0000","last_became_active":"2023-10-09T06:01:22.325866+0000","last_became_peered":"2023-10-09T06:01:22.325866+0000","last_unstale":"2023-10-09T06:03:30.475279+0000","last_undegraded":"2023-10-09T06:03:30.475279+0000","last_fullsized":"2023-10-09T06:03:30.475279+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:41.753506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:41.753506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:32.324674+0000","scrub_duration":0.0022121929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":208,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.493641+0000","last_change":"2023-10-09T05:53:59.212161+0000","last_active":"2023-10-09T06:03:30.493641+0000","last_peered":"2023-10-09T06:03:30.493641+0000","last_clean":"2023-10-09T06:03:30.493641+0000","last_became_active":"2023-10-09T05:53:59.193814+0000","last_became_peered":"2023-10-09T05:53:59.193814+0000","last_unstale":"2023-10-09T06:03:30.493641+0000","last_undegraded":"2023-10-09T06:03:30.493641+0000","last_fullsized":"2023-10-09T06:03:30.493641+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:10.502408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":419,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.474886+0000","last_change":"2023-10-09T06:01:54.757292+0000","last_active":"2023-10-09T06:03:30.474886+0000","last_peered":"2023-10-09T06:03:30.474886+0000","last_clean":"2023-10-09T06:03:30.474886+0000","last_became_active":"2023-10-09T06:01:54.756872+0000","last_became_peered":"2023-10-09T06:01:54.756872+0000","last_unstale":"2023-10-09T06:03:30.474886+0000","last_undegraded":"2023-10-09T06:03:30.474886+0000","last_fullsized":"2023-10-09T06:03:30.474886+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:54.248077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"370'3919","reported_seq":3937,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:45.002397+0000","last_change":"2023-10-09T06:02:55.459796+0000","last_active":"2023-10-09T06:03:45.002397+0000","last_peered":"2023-10-09T06:03:45.002397+0000","last_clean":"2023-10-09T06:03:45.002397+0000","last_became_active":"2023-10-09T06:02:55.459408+0000","last_became_peered":"2023-10-09T06:02:55.459408+0000","last_unstale":"2023-10-09T06:03:45.002397+0000","last_undegraded":"2023-10-09T06:03:45.002397+0000","last_fullsized":"2023-10-09T06:03:45.002397+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":3919,"log_dups_size":0,"ondisk_log_size":3919,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:54.450890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256835584,"num_objects":3919,"num_object_clones":0,"num_object_copies":11757,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3919,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3919,"num_write_kb":250816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":545,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.470437+0000","last_change":"2023-10-09T06:02:16.361201+0000","last_active":"2023-10-09T06:03:30.470437+0000","last_peered":"2023-10-09T06:03:30.470437+0000","last_clean":"2023-10-09T06:03:30.470437+0000","last_became_active":"2023-10-09T06:01:22.337294+0000","last_became_peered":"2023-10-09T06:01:22.337294+0000","last_unstale":"2023-10-09T06:03:30.470437+0000","last_undegraded":"2023-10-09T06:03:30.470437+0000","last_fullsized":"2023-10-09T06:03:30.470437+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:16.361097+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:16.361097+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:16:25.401939+0000","scrub_duration":0.028533749000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":134,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.481632+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T06:03:30.481632+0000","last_peered":"2023-10-09T06:03:30.481632+0000","last_clean":"2023-10-09T06:03:30.481632+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T06:03:30.481632+0000","last_undegraded":"2023-10-09T06:03:30.481632+0000","last_fullsized":"2023-10-09T06:03:30.481632+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":407,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.469309+0000","last_change":"2023-10-09T06:01:54.753427+0000","last_active":"2023-10-09T06:03:30.469309+0000","last_peered":"2023-10-09T06:03:30.469309+0000","last_clean":"2023-10-09T06:03:30.469309+0000","last_became_active":"2023-10-09T06:01:54.753099+0000","last_became_peered":"2023-10-09T06:01:54.753099+0000","last_unstale":"2023-10-09T06:03:30.469309+0000","last_undegraded":"2023-10-09T06:03:30.469309+0000","last_fullsized":"2023-10-09T06:03:30.469309+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:07.504657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"370'3903","reported_seq":3922,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:45.086452+0000","last_change":"2023-10-09T06:02:55.482396+0000","last_active":"2023-10-09T06:03:45.086452+0000","last_peered":"2023-10-09T06:03:45.086452+0000","last_clean":"2023-10-09T06:03:45.086452+0000","last_became_active":"2023-10-09T06:02:55.482125+0000","last_became_peered":"2023-10-09T06:02:55.482125+0000","last_unstale":"2023-10-09T06:03:45.086452+0000","last_undegraded":"2023-10-09T06:03:45.086452+0000","last_fullsized":"2023-10-09T06:03:45.086452+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":3903,"log_dups_size":0,"ondisk_log_size":3903,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:58.878190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255787008,"num_objects":3903,"num_object_clones":0,"num_object_copies":11709,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3903,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3903,"num_write_kb":249792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":813,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.492314+0000","last_change":"2023-10-09T05:59:55.438018+0000","last_active":"2023-10-09T06:03:30.492314+0000","last_peered":"2023-10-09T06:03:30.492314+0000","last_clean":"2023-10-09T06:03:30.492314+0000","last_became_active":"2023-10-09T05:59:32.538631+0000","last_became_peered":"2023-10-09T05:59:32.538631+0000","last_unstale":"2023-10-09T06:03:30.492314+0000","last_undegraded":"2023-10-09T06:03:30.492314+0000","last_fullsized":"2023-10-09T06:03:30.492314+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:55.437947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:55.437947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:37.566205+0000","scrub_duration":0.0091262979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":153,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.481491+0000","last_change":"2023-10-09T06:01:22.355503+0000","last_active":"2023-10-09T06:03:30.481491+0000","last_peered":"2023-10-09T06:03:30.481491+0000","last_clean":"2023-10-09T06:03:30.481491+0000","last_became_active":"2023-10-09T06:01:22.354704+0000","last_became_peered":"2023-10-09T06:01:22.354704+0000","last_unstale":"2023-10-09T06:03:30.481491+0000","last_undegraded":"2023-10-09T06:03:30.481491+0000","last_fullsized":"2023-10-09T06:03:30.481491+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:56:21.446402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":324,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.470409+0000","last_change":"2023-10-09T06:01:22.332033+0000","last_active":"2023-10-09T06:03:30.470409+0000","last_peered":"2023-10-09T06:03:30.470409+0000","last_clean":"2023-10-09T06:03:30.470409+0000","last_became_active":"2023-10-09T06:01:22.329825+0000","last_became_peered":"2023-10-09T06:01:22.329825+0000","last_unstale":"2023-10-09T06:03:30.470409+0000","last_undegraded":"2023-10-09T06:03:30.470409+0000","last_fullsized":"2023-10-09T06:03:30.470409+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:54.046681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"370'4051","reported_seq":4070,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:45.003886+0000","last_change":"2023-10-09T06:02:55.455129+0000","last_active":"2023-10-09T06:03:45.003886+0000","last_peered":"2023-10-09T06:03:45.003886+0000","last_clean":"2023-10-09T06:03:45.003886+0000","last_became_active":"2023-10-09T06:02:55.454836+0000","last_became_peered":"2023-10-09T06:02:55.454836+0000","last_unstale":"2023-10-09T06:03:45.003886+0000","last_undegraded":"2023-10-09T06:03:45.003886+0000","last_fullsized":"2023-10-09T06:03:45.003886+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4051,"log_dups_size":0,"ondisk_log_size":4051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:29.116614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265486336,"num_objects":4051,"num_object_clones":0,"num_object_copies":12153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4051,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4051,"num_write_kb":259264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":560,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.481337+0000","last_change":"2023-10-09T05:59:58.282543+0000","last_active":"2023-10-09T06:03:30.481337+0000","last_peered":"2023-10-09T06:03:30.481337+0000","last_clean":"2023-10-09T06:03:30.481337+0000","last_became_active":"2023-10-09T05:59:32.552960+0000","last_became_peered":"2023-10-09T05:59:32.552960+0000","last_unstale":"2023-10-09T06:03:30.481337+0000","last_undegraded":"2023-10-09T06:03:30.481337+0000","last_fullsized":"2023-10-09T06:03:30.481337+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.282393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.282393+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:10.201323+0000","scrub_duration":0.01402403,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":407,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.474807+0000","last_change":"2023-10-09T06:01:54.751188+0000","last_active":"2023-10-09T06:03:30.474807+0000","last_peered":"2023-10-09T06:03:30.474807+0000","last_clean":"2023-10-09T06:03:30.474807+0000","last_became_active":"2023-10-09T06:01:54.750870+0000","last_became_peered":"2023-10-09T06:01:54.750870+0000","last_unstale":"2023-10-09T06:03:30.474807+0000","last_undegraded":"2023-10-09T06:03:30.474807+0000","last_fullsized":"2023-10-09T06:03:30.474807+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:01.985974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":398,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.470513+0000","last_change":"2023-10-09T06:01:54.757667+0000","last_active":"2023-10-09T06:03:30.470513+0000","last_peered":"2023-10-09T06:03:30.470513+0000","last_clean":"2023-10-09T06:03:30.470513+0000","last_became_active":"2023-10-09T06:01:54.757367+0000","last_became_peered":"2023-10-09T06:01:54.757367+0000","last_unstale":"2023-10-09T06:03:30.470513+0000","last_undegraded":"2023-10-09T06:03:30.470513+0000","last_fullsized":"2023-10-09T06:03:30.470513+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:45.022566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"370'3996","reported_seq":4014,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:45.410475+0000","last_change":"2023-10-09T06:02:55.484013+0000","last_active":"2023-10-09T06:03:45.410475+0000","last_peered":"2023-10-09T06:03:45.410475+0000","last_clean":"2023-10-09T06:03:45.410475+0000","last_became_active":"2023-10-09T06:02:55.483714+0000","last_became_peered":"2023-10-09T06:02:55.483714+0000","last_unstale":"2023-10-09T06:03:45.410475+0000","last_undegraded":"2023-10-09T06:03:45.410475+0000","last_fullsized":"2023-10-09T06:03:45.410475+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":3996,"log_dups_size":0,"ondisk_log_size":3996,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:23:48.138776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261881856,"num_objects":3996,"num_object_clones":0,"num_object_copies":11988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3996,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3996,"num_write_kb":255744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":648,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.464013+0000","last_change":"2023-10-09T06:01:57.796523+0000","last_active":"2023-10-09T06:03:30.464013+0000","last_peered":"2023-10-09T06:03:30.464013+0000","last_clean":"2023-10-09T06:03:30.464013+0000","last_became_active":"2023-10-09T06:01:57.796145+0000","last_became_peered":"2023-10-09T06:01:57.796145+0000","last_unstale":"2023-10-09T06:03:30.464013+0000","last_undegraded":"2023-10-09T06:03:30.464013+0000","last_fullsized":"2023-10-09T06:03:30.464013+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.373748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.373748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:52.076986+0000","scrub_duration":0.0024576450000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":340,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.469831+0000","last_change":"2023-10-09T06:01:22.334879+0000","last_active":"2023-10-09T06:03:30.469831+0000","last_peered":"2023-10-09T06:03:30.469831+0000","last_clean":"2023-10-09T06:03:30.469831+0000","last_became_active":"2023-10-09T06:01:22.334609+0000","last_became_peered":"2023-10-09T06:01:22.334609+0000","last_unstale":"2023-10-09T06:03:30.469831+0000","last_undegraded":"2023-10-09T06:03:30.469831+0000","last_fullsized":"2023-10-09T06:03:30.469831+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:36.094645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":381,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.470380+0000","last_change":"2023-10-09T06:01:54.757024+0000","last_active":"2023-10-09T06:03:30.470380+0000","last_peered":"2023-10-09T06:03:30.470380+0000","last_clean":"2023-10-09T06:03:30.470380+0000","last_became_active":"2023-10-09T06:01:54.756713+0000","last_became_peered":"2023-10-09T06:01:54.756713+0000","last_unstale":"2023-10-09T06:03:30.470380+0000","last_undegraded":"2023-10-09T06:03:30.470380+0000","last_fullsized":"2023-10-09T06:03:30.470380+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:26.229374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"370'4057","reported_seq":4076,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:45.004841+0000","last_change":"2023-10-09T06:02:55.481570+0000","last_active":"2023-10-09T06:03:45.004841+0000","last_peered":"2023-10-09T06:03:45.004841+0000","last_clean":"2023-10-09T06:03:45.004841+0000","last_became_active":"2023-10-09T06:02:55.481243+0000","last_became_peered":"2023-10-09T06:02:55.481243+0000","last_unstale":"2023-10-09T06:03:45.004841+0000","last_undegraded":"2023-10-09T06:03:45.004841+0000","last_fullsized":"2023-10-09T06:03:45.004841+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4057,"log_dups_size":0,"ondisk_log_size":4057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:35.076981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265879552,"num_objects":4057,"num_object_clones":0,"num_object_copies":12171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4057,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4057,"num_write_kb":259648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":523,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.474751+0000","last_change":"2023-10-09T05:59:52.129883+0000","last_active":"2023-10-09T06:03:30.474751+0000","last_peered":"2023-10-09T06:03:30.474751+0000","last_clean":"2023-10-09T06:03:30.474751+0000","last_became_active":"2023-10-09T05:59:32.529083+0000","last_became_peered":"2023-10-09T05:59:32.529083+0000","last_unstale":"2023-10-09T06:03:30.474751+0000","last_undegraded":"2023-10-09T06:03:30.474751+0000","last_fullsized":"2023-10-09T06:03:30.474751+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:52.129816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:52.129816+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:38.309831+0000","scrub_duration":0.0039938459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":158,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.481389+0000","last_change":"2023-10-09T06:01:54.761275+0000","last_active":"2023-10-09T06:03:30.481389+0000","last_peered":"2023-10-09T06:03:30.481389+0000","last_clean":"2023-10-09T06:03:30.481389+0000","last_became_active":"2023-10-09T06:01:54.760782+0000","last_became_peered":"2023-10-09T06:01:54.760782+0000","last_unstale":"2023-10-09T06:03:30.481389+0000","last_undegraded":"2023-10-09T06:03:30.481389+0000","last_fullsized":"2023-10-09T06:03:30.481389+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:28.119846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":47,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.463493+0000","last_change":"2023-10-09T06:01:54.761768+0000","last_active":"2023-10-09T06:03:30.463493+0000","last_peered":"2023-10-09T06:03:30.463493+0000","last_clean":"2023-10-09T06:03:30.463493+0000","last_became_active":"2023-10-09T06:01:54.761443+0000","last_became_peered":"2023-10-09T06:01:54.761443+0000","last_unstale":"2023-10-09T06:03:30.463493+0000","last_undegraded":"2023-10-09T06:03:30.463493+0000","last_fullsized":"2023-10-09T06:03:30.463493+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:51.555987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"370'4061","reported_seq":4080,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:45.176573+0000","last_change":"2023-10-09T06:02:55.459982+0000","last_active":"2023-10-09T06:03:45.176573+0000","last_peered":"2023-10-09T06:03:45.176573+0000","last_clean":"2023-10-09T06:03:45.176573+0000","last_became_active":"2023-10-09T06:02:55.459651+0000","last_became_peered":"2023-10-09T06:02:55.459651+0000","last_unstale":"2023-10-09T06:03:45.176573+0000","last_undegraded":"2023-10-09T06:03:45.176573+0000","last_fullsized":"2023-10-09T06:03:45.176573+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4061,"log_dups_size":0,"ondisk_log_size":4061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:55.260004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266141696,"num_objects":4061,"num_object_clones":0,"num_object_copies":12183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4061,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4061,"num_write_kb":259904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":793,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.475186+0000","last_change":"2023-10-09T05:59:56.092654+0000","last_active":"2023-10-09T06:03:30.475186+0000","last_peered":"2023-10-09T06:03:30.475186+0000","last_clean":"2023-10-09T06:03:30.475186+0000","last_became_active":"2023-10-09T05:59:32.529949+0000","last_became_peered":"2023-10-09T05:59:32.529949+0000","last_unstale":"2023-10-09T06:03:30.475186+0000","last_undegraded":"2023-10-09T06:03:30.475186+0000","last_fullsized":"2023-10-09T06:03:30.475186+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.092586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.092586+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:15:13.234903+0000","scrub_duration":0.0013484219999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":47,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.462706+0000","last_change":"2023-10-09T06:01:54.823231+0000","last_active":"2023-10-09T06:03:30.462706+0000","last_peered":"2023-10-09T06:03:30.462706+0000","last_clean":"2023-10-09T06:03:30.462706+0000","last_became_active":"2023-10-09T06:01:54.822755+0000","last_became_peered":"2023-10-09T06:01:54.822755+0000","last_unstale":"2023-10-09T06:03:30.462706+0000","last_undegraded":"2023-10-09T06:03:30.462706+0000","last_fullsized":"2023-10-09T06:03:30.462706+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:17.592052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":135,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:40.150336+0000","last_change":"2023-10-09T05:57:23.468480+0000","last_active":"2023-10-09T06:03:40.150336+0000","last_peered":"2023-10-09T06:03:40.150336+0000","last_clean":"2023-10-09T06:03:40.150336+0000","last_became_active":"2023-10-09T05:57:23.468179+0000","last_became_peered":"2023-10-09T05:57:23.468179+0000","last_unstale":"2023-10-09T06:03:40.150336+0000","last_undegraded":"2023-10-09T06:03:40.150336+0000","last_fullsized":"2023-10-09T06:03:40.150336+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:18.157188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":47,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.462645+0000","last_change":"2023-10-09T06:01:54.751385+0000","last_active":"2023-10-09T06:03:30.462645+0000","last_peered":"2023-10-09T06:03:30.462645+0000","last_clean":"2023-10-09T06:03:30.462645+0000","last_became_active":"2023-10-09T06:01:54.751041+0000","last_became_peered":"2023-10-09T06:01:54.751041+0000","last_unstale":"2023-10-09T06:03:30.462645+0000","last_undegraded":"2023-10-09T06:03:30.462645+0000","last_fullsized":"2023-10-09T06:03:30.462645+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:03.424836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"370'3960","reported_seq":3979,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:45.149564+0000","last_change":"2023-10-09T06:02:55.481800+0000","last_active":"2023-10-09T06:03:45.149564+0000","last_peered":"2023-10-09T06:03:45.149564+0000","last_clean":"2023-10-09T06:03:45.149564+0000","last_became_active":"2023-10-09T06:02:55.481441+0000","last_became_peered":"2023-10-09T06:02:55.481441+0000","last_unstale":"2023-10-09T06:03:45.149564+0000","last_undegraded":"2023-10-09T06:03:45.149564+0000","last_fullsized":"2023-10-09T06:03:45.149564+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":3960,"log_dups_size":0,"ondisk_log_size":3960,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:37.737097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259522560,"num_objects":3960,"num_object_clones":0,"num_object_copies":11880,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3960,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3960,"num_write_kb":253440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":575,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.516886+0000","last_change":"2023-10-09T05:58:36.225081+0000","last_active":"2023-10-09T06:03:30.516886+0000","last_peered":"2023-10-09T06:03:30.516886+0000","last_clean":"2023-10-09T06:03:30.516886+0000","last_became_active":"2023-10-09T05:56:49.237269+0000","last_became_peered":"2023-10-09T05:56:49.237269+0000","last_unstale":"2023-10-09T06:03:30.516886+0000","last_undegraded":"2023-10-09T06:03:30.516886+0000","last_fullsized":"2023-10-09T06:03:30.516886+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:02.908583+0000","scrub_duration":0.0016660799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":47,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.462715+0000","last_change":"2023-10-09T06:01:54.754294+0000","last_active":"2023-10-09T06:03:30.462715+0000","last_peered":"2023-10-09T06:03:30.462715+0000","last_clean":"2023-10-09T06:03:30.462715+0000","last_became_active":"2023-10-09T06:01:54.754024+0000","last_became_peered":"2023-10-09T06:01:54.754024+0000","last_unstale":"2023-10-09T06:03:30.462715+0000","last_undegraded":"2023-10-09T06:03:30.462715+0000","last_fullsized":"2023-10-09T06:03:30.462715+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:19.746882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":333,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.469768+0000","last_change":"2023-10-09T06:01:22.335651+0000","last_active":"2023-10-09T06:03:30.469768+0000","last_peered":"2023-10-09T06:03:30.469768+0000","last_clean":"2023-10-09T06:03:30.469768+0000","last_became_active":"2023-10-09T06:01:22.334756+0000","last_became_peered":"2023-10-09T06:01:22.334756+0000","last_unstale":"2023-10-09T06:03:30.469768+0000","last_undegraded":"2023-10-09T06:03:30.469768+0000","last_fullsized":"2023-10-09T06:03:30.469768+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:33.123496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":47,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.462774+0000","last_change":"2023-10-09T06:01:54.760832+0000","last_active":"2023-10-09T06:03:30.462774+0000","last_peered":"2023-10-09T06:03:30.462774+0000","last_clean":"2023-10-09T06:03:30.462774+0000","last_became_active":"2023-10-09T06:01:54.760137+0000","last_became_peered":"2023-10-09T06:01:54.760137+0000","last_unstale":"2023-10-09T06:03:30.462774+0000","last_undegraded":"2023-10-09T06:03:30.462774+0000","last_fullsized":"2023-10-09T06:03:30.462774+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:19:58.776604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"370'4002","reported_seq":4020,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:45.144663+0000","last_change":"2023-10-09T06:02:55.454431+0000","last_active":"2023-10-09T06:03:45.144663+0000","last_peered":"2023-10-09T06:03:45.144663+0000","last_clean":"2023-10-09T06:03:45.144663+0000","last_became_active":"2023-10-09T06:02:55.454117+0000","last_became_peered":"2023-10-09T06:02:55.454117+0000","last_unstale":"2023-10-09T06:03:45.144663+0000","last_undegraded":"2023-10-09T06:03:45.144663+0000","last_fullsized":"2023-10-09T06:03:45.144663+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4002,"log_dups_size":0,"ondisk_log_size":4002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:02.450452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262275072,"num_objects":4002,"num_object_clones":0,"num_object_copies":12006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4002,"num_write_kb":256128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":694,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.492043+0000","last_change":"2023-10-09T06:01:22.335906+0000","last_active":"2023-10-09T06:03:30.492043+0000","last_peered":"2023-10-09T06:03:30.492043+0000","last_clean":"2023-10-09T06:03:30.492043+0000","last_became_active":"2023-10-09T06:01:22.335590+0000","last_became_peered":"2023-10-09T06:01:22.335590+0000","last_unstale":"2023-10-09T06:03:30.492043+0000","last_undegraded":"2023-10-09T06:03:30.492043+0000","last_fullsized":"2023-10-09T06:03:30.492043+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:55.897812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:55.897812+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:02.771428+0000","scrub_duration":0.0014511420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":112,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.455671+0000","last_change":"2023-10-09T05:58:59.325257+0000","last_active":"2023-10-09T06:03:30.455671+0000","last_peered":"2023-10-09T06:03:30.455671+0000","last_clean":"2023-10-09T06:03:30.455671+0000","last_became_active":"2023-10-09T05:58:59.324993+0000","last_became_peered":"2023-10-09T05:58:59.324993+0000","last_unstale":"2023-10-09T06:03:30.455671+0000","last_undegraded":"2023-10-09T06:03:30.455671+0000","last_fullsized":"2023-10-09T06:03:30.455671+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:20.825840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.461836+0000","last_change":"2023-10-09T06:01:54.755425+0000","last_active":"2023-10-09T06:03:30.461836+0000","last_peered":"2023-10-09T06:03:30.461836+0000","last_clean":"2023-10-09T06:03:30.461836+0000","last_became_active":"2023-10-09T06:01:54.755129+0000","last_became_peered":"2023-10-09T06:01:54.755129+0000","last_unstale":"2023-10-09T06:03:30.461836+0000","last_undegraded":"2023-10-09T06:03:30.461836+0000","last_fullsized":"2023-10-09T06:03:30.461836+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:01:58.037849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":47,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.463428+0000","last_change":"2023-10-09T06:01:54.751198+0000","last_active":"2023-10-09T06:03:30.463428+0000","last_peered":"2023-10-09T06:03:30.463428+0000","last_clean":"2023-10-09T06:03:30.463428+0000","last_became_active":"2023-10-09T06:01:54.750862+0000","last_became_peered":"2023-10-09T06:01:54.750862+0000","last_unstale":"2023-10-09T06:03:30.463428+0000","last_undegraded":"2023-10-09T06:03:30.463428+0000","last_fullsized":"2023-10-09T06:03:30.463428+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:35.203567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"370'4203","reported_seq":4221,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:48.622486+0000","last_change":"2023-10-09T06:02:55.481788+0000","last_active":"2023-10-09T06:03:48.622486+0000","last_peered":"2023-10-09T06:03:48.622486+0000","last_clean":"2023-10-09T06:03:48.622486+0000","last_became_active":"2023-10-09T06:02:55.481460+0000","last_became_peered":"2023-10-09T06:02:55.481460+0000","last_unstale":"2023-10-09T06:03:48.622486+0000","last_undegraded":"2023-10-09T06:03:48.622486+0000","last_fullsized":"2023-10-09T06:03:48.622486+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4203,"log_dups_size":0,"ondisk_log_size":4203,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:31.125821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275447808,"num_objects":4203,"num_object_clones":0,"num_object_copies":12609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4203,"num_write_kb":268992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":241,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.492035+0000","last_change":"2023-10-09T05:58:59.330109+0000","last_active":"2023-10-09T06:03:30.492035+0000","last_peered":"2023-10-09T06:03:30.492035+0000","last_clean":"2023-10-09T06:03:30.492035+0000","last_became_active":"2023-10-09T05:58:59.329808+0000","last_became_peered":"2023-10-09T05:58:59.329808+0000","last_unstale":"2023-10-09T06:03:30.492035+0000","last_undegraded":"2023-10-09T06:03:30.492035+0000","last_fullsized":"2023-10-09T06:03:30.492035+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:30.680622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":774,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.493196+0000","last_change":"2023-10-09T05:58:37.782237+0000","last_active":"2023-10-09T06:03:30.493196+0000","last_peered":"2023-10-09T06:03:30.493196+0000","last_clean":"2023-10-09T06:03:30.493196+0000","last_became_active":"2023-10-09T05:53:59.202004+0000","last_became_peered":"2023-10-09T05:53:59.202004+0000","last_unstale":"2023-10-09T06:03:30.493196+0000","last_undegraded":"2023-10-09T06:03:30.493196+0000","last_fullsized":"2023-10-09T06:03:30.493196+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:37.782173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:37.782173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:20.787963+0000","scrub_duration":0.001325908,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":135,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:40.150275+0000","last_change":"2023-10-09T05:57:23.479174+0000","last_active":"2023-10-09T06:03:40.150275+0000","last_peered":"2023-10-09T06:03:40.150275+0000","last_clean":"2023-10-09T06:03:40.150275+0000","last_became_active":"2023-10-09T05:57:23.478897+0000","last_became_peered":"2023-10-09T05:57:23.478897+0000","last_unstale":"2023-10-09T06:03:40.150275+0000","last_undegraded":"2023-10-09T06:03:40.150275+0000","last_fullsized":"2023-10-09T06:03:40.150275+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:00.160042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":349,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.474997+0000","last_change":"2023-10-09T06:01:22.340980+0000","last_active":"2023-10-09T06:03:30.474997+0000","last_peered":"2023-10-09T06:03:30.474997+0000","last_clean":"2023-10-09T06:03:30.474997+0000","last_became_active":"2023-10-09T06:01:22.340690+0000","last_became_peered":"2023-10-09T06:01:22.340690+0000","last_unstale":"2023-10-09T06:03:30.474997+0000","last_undegraded":"2023-10-09T06:03:30.474997+0000","last_fullsized":"2023-10-09T06:03:30.474997+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:45.161101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"370'4111","reported_seq":4130,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:45.029711+0000","last_change":"2023-10-09T06:02:55.481658+0000","last_active":"2023-10-09T06:03:45.029711+0000","last_peered":"2023-10-09T06:03:45.029711+0000","last_clean":"2023-10-09T06:03:45.029711+0000","last_became_active":"2023-10-09T06:02:55.481365+0000","last_became_peered":"2023-10-09T06:02:55.481365+0000","last_unstale":"2023-10-09T06:03:45.029711+0000","last_undegraded":"2023-10-09T06:03:45.029711+0000","last_fullsized":"2023-10-09T06:03:45.029711+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4111,"log_dups_size":0,"ondisk_log_size":4111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:59.430808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269418496,"num_objects":4111,"num_object_clones":0,"num_object_copies":12333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4111,"num_write_kb":263104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":563,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.464097+0000","last_change":"2023-10-09T06:02:45.280115+0000","last_active":"2023-10-09T06:03:30.464097+0000","last_peered":"2023-10-09T06:03:30.464097+0000","last_clean":"2023-10-09T06:03:30.464097+0000","last_became_active":"2023-10-09T06:01:57.803970+0000","last_became_peered":"2023-10-09T06:01:57.803970+0000","last_unstale":"2023-10-09T06:03:30.464097+0000","last_undegraded":"2023-10-09T06:03:30.464097+0000","last_fullsized":"2023-10-09T06:03:30.464097+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:45.280021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:45.280021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:30.141864+0000","scrub_duration":0.001913166,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":246,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.493129+0000","last_change":"2023-10-09T06:01:54.751146+0000","last_active":"2023-10-09T06:03:30.493129+0000","last_peered":"2023-10-09T06:03:30.493129+0000","last_clean":"2023-10-09T06:03:30.493129+0000","last_became_active":"2023-10-09T06:01:54.750833+0000","last_became_peered":"2023-10-09T06:01:54.750833+0000","last_unstale":"2023-10-09T06:03:30.493129+0000","last_undegraded":"2023-10-09T06:03:30.493129+0000","last_fullsized":"2023-10-09T06:03:30.493129+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:54.035203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":324,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.517066+0000","last_change":"2023-10-09T06:01:22.320317+0000","last_active":"2023-10-09T06:03:30.517066+0000","last_peered":"2023-10-09T06:03:30.517066+0000","last_clean":"2023-10-09T06:03:30.517066+0000","last_became_active":"2023-10-09T06:01:22.319989+0000","last_became_peered":"2023-10-09T06:01:22.319989+0000","last_unstale":"2023-10-09T06:03:30.517066+0000","last_undegraded":"2023-10-09T06:03:30.517066+0000","last_fullsized":"2023-10-09T06:03:30.517066+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:06.219647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":435,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.469587+0000","last_change":"2023-10-09T06:01:54.752942+0000","last_active":"2023-10-09T06:03:30.469587+0000","last_peered":"2023-10-09T06:03:30.469587+0000","last_clean":"2023-10-09T06:03:30.469587+0000","last_became_active":"2023-10-09T06:01:54.752593+0000","last_became_peered":"2023-10-09T06:01:54.752593+0000","last_unstale":"2023-10-09T06:03:30.469587+0000","last_undegraded":"2023-10-09T06:03:30.469587+0000","last_fullsized":"2023-10-09T06:03:30.469587+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:10.005606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"370'4103","reported_seq":4122,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:45.139282+0000","last_change":"2023-10-09T06:02:55.454444+0000","last_active":"2023-10-09T06:03:45.139282+0000","last_peered":"2023-10-09T06:03:45.139282+0000","last_clean":"2023-10-09T06:03:45.139282+0000","last_became_active":"2023-10-09T06:02:55.454144+0000","last_became_peered":"2023-10-09T06:02:55.454144+0000","last_unstale":"2023-10-09T06:03:45.139282+0000","last_undegraded":"2023-10-09T06:03:45.139282+0000","last_fullsized":"2023-10-09T06:03:45.139282+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4103,"log_dups_size":0,"ondisk_log_size":4103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:45:20.421821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268894208,"num_objects":4103,"num_object_clones":0,"num_object_copies":12309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4103,"num_write_kb":262592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":783,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.475240+0000","last_change":"2023-10-09T05:59:54.074413+0000","last_active":"2023-10-09T06:03:30.475240+0000","last_peered":"2023-10-09T06:03:30.475240+0000","last_clean":"2023-10-09T06:03:30.475240+0000","last_became_active":"2023-10-09T05:59:32.538959+0000","last_became_peered":"2023-10-09T05:59:32.538959+0000","last_unstale":"2023-10-09T06:03:30.475240+0000","last_undegraded":"2023-10-09T06:03:30.475240+0000","last_fullsized":"2023-10-09T06:03:30.475240+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:54.074310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:54.074310+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:10.811269+0000","scrub_duration":0.011040648,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":221,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.493450+0000","last_change":"2023-10-09T06:01:22.337610+0000","last_active":"2023-10-09T06:03:30.493450+0000","last_peered":"2023-10-09T06:03:30.493450+0000","last_clean":"2023-10-09T06:03:30.493450+0000","last_became_active":"2023-10-09T06:01:22.336719+0000","last_became_peered":"2023-10-09T06:01:22.336719+0000","last_unstale":"2023-10-09T06:03:30.493450+0000","last_undegraded":"2023-10-09T06:03:30.493450+0000","last_fullsized":"2023-10-09T06:03:30.493450+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:33:29.587947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.462846+0000","last_change":"2023-10-09T06:01:54.756606+0000","last_active":"2023-10-09T06:03:30.462846+0000","last_peered":"2023-10-09T06:03:30.462846+0000","last_clean":"2023-10-09T06:03:30.462846+0000","last_became_active":"2023-10-09T06:01:54.755869+0000","last_became_peered":"2023-10-09T06:01:54.755869+0000","last_unstale":"2023-10-09T06:03:30.462846+0000","last_undegraded":"2023-10-09T06:03:30.462846+0000","last_fullsized":"2023-10-09T06:03:30.462846+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:29.672095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":142,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.492118+0000","last_change":"2023-10-09T06:01:41.938541+0000","last_active":"2023-10-09T06:03:30.492118+0000","last_peered":"2023-10-09T06:03:30.492118+0000","last_clean":"2023-10-09T06:03:30.492118+0000","last_became_active":"2023-10-09T06:01:41.938289+0000","last_became_peered":"2023-10-09T06:01:41.938289+0000","last_unstale":"2023-10-09T06:03:30.492118+0000","last_undegraded":"2023-10-09T06:03:30.492118+0000","last_fullsized":"2023-10-09T06:03:30.492118+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:29.735530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.c","version":"370'3962","reported_seq":3981,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:45.184782+0000","last_change":"2023-10-09T06:02:55.458266+0000","last_active":"2023-10-09T06:03:45.184782+0000","last_peered":"2023-10-09T06:03:45.184782+0000","last_clean":"2023-10-09T06:03:45.184782+0000","last_became_active":"2023-10-09T06:02:55.457936+0000","last_became_peered":"2023-10-09T06:02:55.457936+0000","last_unstale":"2023-10-09T06:03:45.184782+0000","last_undegraded":"2023-10-09T06:03:45.184782+0000","last_fullsized":"2023-10-09T06:03:45.184782+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":3962,"log_dups_size":0,"ondisk_log_size":3962,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:44:16.198311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259653632,"num_objects":3962,"num_object_clones":0,"num_object_copies":11886,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3962,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3962,"num_write_kb":253568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":763,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.470567+0000","last_change":"2023-10-09T06:02:17.344619+0000","last_active":"2023-10-09T06:03:30.470567+0000","last_peered":"2023-10-09T06:03:30.470567+0000","last_clean":"2023-10-09T06:03:30.470567+0000","last_became_active":"2023-10-09T06:01:22.327941+0000","last_became_peered":"2023-10-09T06:01:22.327941+0000","last_unstale":"2023-10-09T06:03:30.470567+0000","last_undegraded":"2023-10-09T06:03:30.470567+0000","last_fullsized":"2023-10-09T06:03:30.470567+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:17.344518+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:17.344518+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:26.950368+0000","scrub_duration":0.0015914099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":378,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.481325+0000","last_change":"2023-10-09T05:53:59.228948+0000","last_active":"2023-10-09T06:03:30.481325+0000","last_peered":"2023-10-09T06:03:30.481325+0000","last_clean":"2023-10-09T06:03:30.481325+0000","last_became_active":"2023-10-09T05:53:59.186359+0000","last_became_peered":"2023-10-09T05:53:59.186359+0000","last_unstale":"2023-10-09T06:03:30.481325+0000","last_undegraded":"2023-10-09T06:03:30.481325+0000","last_fullsized":"2023-10-09T06:03:30.481325+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:39.990576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":47,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.462346+0000","last_change":"2023-10-09T06:01:54.753800+0000","last_active":"2023-10-09T06:03:30.462346+0000","last_peered":"2023-10-09T06:03:30.462346+0000","last_clean":"2023-10-09T06:03:30.462346+0000","last_became_active":"2023-10-09T06:01:54.751920+0000","last_became_peered":"2023-10-09T06:01:54.751920+0000","last_unstale":"2023-10-09T06:03:30.462346+0000","last_undegraded":"2023-10-09T06:03:30.462346+0000","last_fullsized":"2023-10-09T06:03:30.462346+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:45.359801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":135,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.481523+0000","last_change":"2023-10-09T05:57:23.477143+0000","last_active":"2023-10-09T06:03:30.481523+0000","last_peered":"2023-10-09T06:03:30.481523+0000","last_clean":"2023-10-09T06:03:30.481523+0000","last_became_active":"2023-10-09T05:57:23.475655+0000","last_became_peered":"2023-10-09T05:57:23.475655+0000","last_unstale":"2023-10-09T06:03:30.481523+0000","last_undegraded":"2023-10-09T06:03:30.481523+0000","last_fullsized":"2023-10-09T06:03:30.481523+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:29.528023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"370'3935","reported_seq":3953,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:45.753551+0000","last_change":"2023-10-09T06:02:55.481513+0000","last_active":"2023-10-09T06:03:45.753551+0000","last_peered":"2023-10-09T06:03:45.753551+0000","last_clean":"2023-10-09T06:03:45.753551+0000","last_became_active":"2023-10-09T06:02:55.481226+0000","last_became_peered":"2023-10-09T06:02:55.481226+0000","last_unstale":"2023-10-09T06:03:45.753551+0000","last_undegraded":"2023-10-09T06:03:45.753551+0000","last_fullsized":"2023-10-09T06:03:45.753551+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":3935,"log_dups_size":0,"ondisk_log_size":3935,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:12.967950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257884160,"num_objects":3935,"num_object_clones":0,"num_object_copies":11805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3935,"num_write_kb":251840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":629,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.405480+0000","last_change":"2023-10-09T05:58:34.333129+0000","last_active":"2023-10-09T06:03:41.405480+0000","last_peered":"2023-10-09T06:03:41.405480+0000","last_clean":"2023-10-09T06:03:41.405480+0000","last_became_active":"2023-10-09T05:57:26.502911+0000","last_became_peered":"2023-10-09T05:57:26.502911+0000","last_unstale":"2023-10-09T06:03:41.405480+0000","last_undegraded":"2023-10-09T06:03:41.405480+0000","last_fullsized":"2023-10-09T06:03:41.405480+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:25.230712+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":374,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.475232+0000","last_change":"2023-10-09T05:57:58.832790+0000","last_active":"2023-10-09T06:03:30.475232+0000","last_peered":"2023-10-09T06:03:30.475232+0000","last_clean":"2023-10-09T06:03:30.475232+0000","last_became_active":"2023-10-09T05:57:58.832471+0000","last_became_peered":"2023-10-09T05:57:58.832471+0000","last_unstale":"2023-10-09T06:03:30.475232+0000","last_undegraded":"2023-10-09T06:03:30.475232+0000","last_fullsized":"2023-10-09T06:03:30.475232+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:39.358174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":440,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.469676+0000","last_change":"2023-10-09T06:01:54.776025+0000","last_active":"2023-10-09T06:03:30.469676+0000","last_peered":"2023-10-09T06:03:30.469676+0000","last_clean":"2023-10-09T06:03:30.469676+0000","last_became_active":"2023-10-09T06:01:54.753050+0000","last_became_peered":"2023-10-09T06:01:54.753050+0000","last_unstale":"2023-10-09T06:03:30.469676+0000","last_undegraded":"2023-10-09T06:03:30.469676+0000","last_fullsized":"2023-10-09T06:03:30.469676+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:58:24.000420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":316,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.469845+0000","last_change":"2023-10-09T06:01:22.334081+0000","last_active":"2023-10-09T06:03:30.469845+0000","last_peered":"2023-10-09T06:03:30.469845+0000","last_clean":"2023-10-09T06:03:30.469845+0000","last_became_active":"2023-10-09T06:01:22.333663+0000","last_became_peered":"2023-10-09T06:01:22.333663+0000","last_unstale":"2023-10-09T06:03:30.469845+0000","last_undegraded":"2023-10-09T06:03:30.469845+0000","last_fullsized":"2023-10-09T06:03:30.469845+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:55.809259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":596,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.455421+0000","last_change":"2023-10-09T06:00:06.181968+0000","last_active":"2023-10-09T06:03:30.455421+0000","last_peered":"2023-10-09T06:03:30.455421+0000","last_clean":"2023-10-09T06:03:30.455421+0000","last_became_active":"2023-10-09T05:59:40.324653+0000","last_became_peered":"2023-10-09T05:59:40.324653+0000","last_unstale":"2023-10-09T06:03:30.455421+0000","last_undegraded":"2023-10-09T06:03:30.455421+0000","last_fullsized":"2023-10-09T06:03:30.455421+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.181811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.181811+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:05:16.987523+0000","scrub_duration":0.011553313000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":316,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.469414+0000","last_change":"2023-10-09T06:01:22.349718+0000","last_active":"2023-10-09T06:03:30.469414+0000","last_peered":"2023-10-09T06:03:30.469414+0000","last_clean":"2023-10-09T06:03:30.469414+0000","last_became_active":"2023-10-09T06:01:22.349401+0000","last_became_peered":"2023-10-09T06:01:22.349401+0000","last_unstale":"2023-10-09T06:03:30.469414+0000","last_undegraded":"2023-10-09T06:03:30.469414+0000","last_fullsized":"2023-10-09T06:03:30.469414+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:08.983715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":47,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.462916+0000","last_change":"2023-10-09T06:01:54.761007+0000","last_active":"2023-10-09T06:03:30.462916+0000","last_peered":"2023-10-09T06:03:30.462916+0000","last_clean":"2023-10-09T06:03:30.462916+0000","last_became_active":"2023-10-09T06:01:54.760538+0000","last_became_peered":"2023-10-09T06:01:54.760538+0000","last_unstale":"2023-10-09T06:03:30.462916+0000","last_undegraded":"2023-10-09T06:03:30.462916+0000","last_fullsized":"2023-10-09T06:03:30.462916+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:08.772179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":714,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:40.150371+0000","last_change":"2023-10-09T06:00:04.215425+0000","last_active":"2023-10-09T06:03:40.150371+0000","last_peered":"2023-10-09T06:03:40.150371+0000","last_clean":"2023-10-09T06:03:40.150371+0000","last_became_active":"2023-10-09T05:59:40.326685+0000","last_became_peered":"2023-10-09T05:59:40.326685+0000","last_unstale":"2023-10-09T06:03:40.150371+0000","last_undegraded":"2023-10-09T06:03:40.150371+0000","last_fullsized":"2023-10-09T06:03:40.150371+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:04.215270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:04.215270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:03.963694+0000","scrub_duration":0.002850652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":141,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.470331+0000","last_change":"2023-10-09T06:01:54.824780+0000","last_active":"2023-10-09T06:03:30.470331+0000","last_peered":"2023-10-09T06:03:30.470331+0000","last_clean":"2023-10-09T06:03:30.470331+0000","last_became_active":"2023-10-09T06:01:54.757727+0000","last_became_peered":"2023-10-09T06:01:54.757727+0000","last_unstale":"2023-10-09T06:03:30.470331+0000","last_undegraded":"2023-10-09T06:03:30.470331+0000","last_fullsized":"2023-10-09T06:03:30.470331+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:44.132556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":388,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.469986+0000","last_change":"2023-10-09T06:01:54.764989+0000","last_active":"2023-10-09T06:03:30.469986+0000","last_peered":"2023-10-09T06:03:30.469986+0000","last_clean":"2023-10-09T06:03:30.469986+0000","last_became_active":"2023-10-09T06:01:54.764790+0000","last_became_peered":"2023-10-09T06:01:54.764790+0000","last_unstale":"2023-10-09T06:03:30.469986+0000","last_undegraded":"2023-10-09T06:03:30.469986+0000","last_fullsized":"2023-10-09T06:03:30.469986+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:14:07.208858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":127,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.470361+0000","last_change":"2023-10-09T06:01:22.342190+0000","last_active":"2023-10-09T06:03:30.470361+0000","last_peered":"2023-10-09T06:03:30.470361+0000","last_clean":"2023-10-09T06:03:30.470361+0000","last_became_active":"2023-10-09T06:01:22.341855+0000","last_became_peered":"2023-10-09T06:01:22.341855+0000","last_unstale":"2023-10-09T06:03:30.470361+0000","last_undegraded":"2023-10-09T06:03:30.470361+0000","last_fullsized":"2023-10-09T06:03:30.470361+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:01:21.322371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":358,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.470084+0000","last_change":"2023-10-09T06:01:22.326786+0000","last_active":"2023-10-09T06:03:30.470084+0000","last_peered":"2023-10-09T06:03:30.470084+0000","last_clean":"2023-10-09T06:03:30.470084+0000","last_became_active":"2023-10-09T06:01:22.326474+0000","last_became_peered":"2023-10-09T06:01:22.326474+0000","last_unstale":"2023-10-09T06:03:30.470084+0000","last_undegraded":"2023-10-09T06:03:30.470084+0000","last_fullsized":"2023-10-09T06:03:30.470084+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:17:48.846619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":636,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.492348+0000","last_change":"2023-10-09T06:01:57.801991+0000","last_active":"2023-10-09T06:03:30.492348+0000","last_peered":"2023-10-09T06:03:30.492348+0000","last_clean":"2023-10-09T06:03:30.492348+0000","last_became_active":"2023-10-09T06:01:57.801579+0000","last_became_peered":"2023-10-09T06:01:57.801579+0000","last_unstale":"2023-10-09T06:03:30.492348+0000","last_undegraded":"2023-10-09T06:03:30.492348+0000","last_fullsized":"2023-10-09T06:03:30.492348+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:00.456132+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:00.456132+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:48.548353+0000","scrub_duration":0.0014992199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":135,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.456936+0000","last_change":"2023-10-09T05:57:23.479560+0000","last_active":"2023-10-09T06:03:30.456936+0000","last_peered":"2023-10-09T06:03:30.456936+0000","last_clean":"2023-10-09T06:03:30.456936+0000","last_became_active":"2023-10-09T05:57:23.479177+0000","last_became_peered":"2023-10-09T05:57:23.479177+0000","last_unstale":"2023-10-09T06:03:30.456936+0000","last_undegraded":"2023-10-09T06:03:30.456936+0000","last_fullsized":"2023-10-09T06:03:30.456936+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:41.925161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":134,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.455773+0000","last_change":"2023-10-09T05:57:23.475179+0000","last_active":"2023-10-09T06:03:30.455773+0000","last_peered":"2023-10-09T06:03:30.455773+0000","last_clean":"2023-10-09T06:03:30.455773+0000","last_became_active":"2023-10-09T05:57:23.474904+0000","last_became_peered":"2023-10-09T05:57:23.474904+0000","last_unstale":"2023-10-09T06:03:30.455773+0000","last_undegraded":"2023-10-09T06:03:30.455773+0000","last_fullsized":"2023-10-09T06:03:30.455773+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:54.587601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":653,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.461922+0000","last_change":"2023-10-09T06:02:46.290692+0000","last_active":"2023-10-09T06:03:30.461922+0000","last_peered":"2023-10-09T06:03:30.461922+0000","last_clean":"2023-10-09T06:03:30.461922+0000","last_became_active":"2023-10-09T06:01:57.817191+0000","last_became_peered":"2023-10-09T06:01:57.817191+0000","last_unstale":"2023-10-09T06:03:30.461922+0000","last_undegraded":"2023-10-09T06:03:30.461922+0000","last_fullsized":"2023-10-09T06:03:30.461922+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:46.290510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:46.290510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:09.771203+0000","scrub_duration":0.002627819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":339,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.469485+0000","last_change":"2023-10-09T05:57:10.379258+0000","last_active":"2023-10-09T06:03:30.469485+0000","last_peered":"2023-10-09T06:03:30.469485+0000","last_clean":"2023-10-09T06:03:30.469485+0000","last_became_active":"2023-10-09T05:57:10.359745+0000","last_became_peered":"2023-10-09T05:57:10.359745+0000","last_unstale":"2023-10-09T06:03:30.469485+0000","last_undegraded":"2023-10-09T06:03:30.469485+0000","last_fullsized":"2023-10-09T06:03:30.469485+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:14.750256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":381,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:03:30.469651+0000","last_change":"2023-10-09T06:01:54.757112+0000","last_active":"2023-10-09T06:03:30.469651+0000","last_peered":"2023-10-09T06:03:30.469651+0000","last_clean":"2023-10-09T06:03:30.469651+0000","last_became_active":"2023-10-09T06:01:54.756793+0000","last_became_peered":"2023-10-09T06:01:54.756793+0000","last_unstale":"2023-10-09T06:03:30.469651+0000","last_undegraded":"2023-10-09T06:03:30.469651+0000","last_fullsized":"2023-10-09T06:03:30.469651+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:31.657029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":1324,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10938,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4227989504,"num_objects":64514,"num_object_clones":0,"num_object_copies":193542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":64514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64514,"num_write_kb":4128896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12722044928,"data_stored":12722044928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":64514,"ondisk_log_size":64514,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234289,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1572936,"kb_used_data":1275648,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":92172216,"statfs":{"total":95995035648,"available":94384349184,"internally_reserved":0,"allocated":1306263552,"data_stored":1305128535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:03:07 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":5.5570000000000004,"15min":2.9340000000000002},"min":{"1min":0.25600000000000001,"5min":0.22500000000000001,"15min":0.22},"max":{"1min":4.3410000000000002,"5min":165.792,"15min":165.792},"last":1.4710000000000001},{"interface":"front","average":{"1min":1.147,"5min":5.7679999999999998,"15min":2.9500000000000002},"min":{"1min":0.35599999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":4.5670000000000002,"5min":166.00700000000001,"15min":166.00700000000001},"last":0.28899999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:03:43 2023","interfaces":[{"interface":"back","average":{"1min":2.968,"5min":1.6399999999999999,"15min":1.331},"min":{"1min":0.25,"5min":0.249,"15min":0.22900000000000001},"max":{"1min":38.118000000000002,"5min":38.118000000000002,"15min":38.118000000000002},"last":0.25},{"interface":"front","average":{"1min":2.4940000000000002,"5min":1.484,"15min":1.1739999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":36.478999999999999,"5min":36.478999999999999,"15min":36.478999999999999},"last":0.505}]},{"osd":2,"last update":"Mon Oct 9 06:03:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":3.1190000000000002,"15min":2.887},"min":{"1min":0.309,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.1080000000000001,"5min":127.86499999999999,"15min":127.86499999999999},"last":1.429},{"interface":"front","average":{"1min":1.2150000000000001,"5min":2.7850000000000001,"15min":2.8410000000000002},"min":{"1min":0.43099999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":7.2240000000000002,"5min":129.24000000000001,"15min":129.24000000000001},"last":0.56000000000000005}]},{"osd":3,"last update":"Mon Oct 9 06:03:43 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":8.8309999999999995,"15min":5.6859999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":2.827,"5min":164.16999999999999,"15min":164.16999999999999},"last":0.58699999999999997},{"interface":"front","average":{"1min":1.2250000000000001,"5min":8.9550000000000001,"15min":5.6360000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.30299999999999999},"max":{"1min":3.5619999999999998,"5min":164.40700000000001,"15min":164.40700000000001},"last":3.5619999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:03:22 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.753,"15min":0.78700000000000003},"min":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.671,"5min":3.456,"15min":3.456},"last":1.3560000000000001},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.78800000000000003,"15min":0.83199999999999996},"min":{"1min":0.35199999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.98299999999999998,"5min":3.5939999999999999,"15min":3.5939999999999999},"last":0.60999999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:03:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.748,"15min":0.84799999999999998},"min":{"1min":0.309,"5min":0.27600000000000002,"15min":0.27500000000000002},"max":{"1min":2.7930000000000001,"5min":3.5230000000000001,"15min":15.622},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.85199999999999998,"15min":0.73399999999999999},"min":{"1min":0.376,"5min":0.309,"15min":0.30499999999999999},"max":{"1min":1.6539999999999999,"5min":4.133,"15min":4.133},"last":0.54000000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:03:26 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.81799999999999995,"15min":1.3759999999999999},"min":{"1min":0.34699999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.0339999999999998,"5min":3.5379999999999998,"15min":131.94399999999999},"last":1.2190000000000001},{"interface":"front","average":{"1min":0.629,"5min":0.90000000000000002,"15min":1.3799999999999999},"min":{"1min":0.36399999999999999,"5min":0.313,"15min":0.29999999999999999},"max":{"1min":1.6339999999999999,"5min":5.4870000000000001,"15min":135.23400000000001},"last":1.397}]}]},{"osd":6,"up_from":46,"seq":197568495926,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1924852,"kb_used_data":1339180,"kb_used_omap":0,"kb_used_meta":585664,"kb_avail":91820300,"statfs":{"total":95995035648,"available":94023987200,"internally_reserved":0,"allocated":1371320320,"data_stored":1370204828,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599719936},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:03:08 2023","interfaces":[{"interface":"back","average":{"1min":1.794,"5min":3.7989999999999999,"15min":2.165},"min":{"1min":0.26000000000000001,"5min":0.22900000000000001,"15min":0.224},"max":{"1min":25.933,"5min":165.124,"15min":165.124},"last":0.318},{"interface":"front","average":{"1min":1.996,"5min":3.9180000000000001,"15min":2.2050000000000001},"min":{"1min":0.29199999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":25.611000000000001,"5min":165.221,"15min":165.221},"last":1.2090000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:02:50 2023","interfaces":[{"interface":"back","average":{"1min":1.371,"5min":1.208,"15min":0.97699999999999998},"min":{"1min":0.29099999999999998,"5min":0.27000000000000002,"15min":0.23899999999999999},"max":{"1min":9.8819999999999997,"5min":9.8819999999999997,"15min":17.341000000000001},"last":6.5},{"interface":"front","average":{"1min":0.86499999999999999,"5min":1.512,"15min":1.123},"min":{"1min":0.32100000000000001,"5min":0.29199999999999998,"15min":0.27000000000000002},"max":{"1min":2.1299999999999999,"5min":22.445,"15min":22.445},"last":3.9420000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:03:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5720000000000001,"5min":2.202,"15min":1.7430000000000001},"min":{"1min":0.313,"5min":0.26900000000000002,"15min":0.26700000000000002},"max":{"1min":13.917,"5min":27.224,"15min":61.088999999999999},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.80800000000000005,"5min":2.0950000000000002,"15min":1.7010000000000001},"min":{"1min":0.34999999999999998,"5min":0.28899999999999998,"15min":0.25600000000000001},"max":{"1min":4.415,"5min":27.177,"15min":61.082000000000001},"last":0.45600000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:02:50 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":5.9900000000000002,"15min":3.593},"min":{"1min":0.34399999999999997,"5min":0.307,"15min":0.29599999999999999},"max":{"1min":2.2240000000000002,"5min":86.099000000000004,"15min":93.626000000000005},"last":2.081},{"interface":"front","average":{"1min":1.1879999999999999,"5min":5.9969999999999999,"15min":3.5089999999999999},"min":{"1min":0.34200000000000003,"5min":0.314,"15min":0.28499999999999998},"max":{"1min":4.2050000000000001,"5min":83.126999999999995,"15min":93.643000000000001},"last":4.0229999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:03:22 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":1.151,"15min":1.248},"min":{"1min":0.26400000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":1.6419999999999999,"5min":4.673,"15min":4.673},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.68300000000000005,"5min":1.0149999999999999,"15min":1.0700000000000001},"min":{"1min":0.28199999999999997,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":2.9049999999999998,"5min":4.399,"15min":4.399},"last":0.79400000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:02:56 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.81000000000000005,"15min":1.002},"min":{"1min":0.29799999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.88800000000000001,"5min":4.9699999999999998,"15min":45.933999999999997},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.84599999999999997,"15min":1.0309999999999999},"min":{"1min":0.27700000000000002,"5min":0.255,"15min":0.255},"max":{"1min":2.7469999999999999,"5min":4.976,"15min":45.917999999999999},"last":1.9950000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:03:22 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.1220000000000001,"15min":1.2370000000000001},"min":{"1min":0.32500000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.2869999999999999,"5min":6.4749999999999996,"15min":52.042999999999999},"last":0.82199999999999995},{"interface":"front","average":{"1min":0.91100000000000003,"5min":1.0109999999999999,"15min":1.141},"min":{"1min":0.35399999999999998,"5min":0.33100000000000002,"15min":0.32200000000000001},"max":{"1min":3.536,"5min":6.4589999999999996,"15min":51.259},"last":2.0449999999999999}]}]},{"osd":1,"up_from":89,"seq":382252089514,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2055876,"kb_used_data":1790844,"kb_used_omap":9,"kb_used_meta":265014,"kb_avail":91689276,"statfs":{"total":95995035648,"available":93889818624,"internally_reserved":0,"allocated":1833824256,"data_stored":1831695580,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":271375337},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:03:15 2023","interfaces":[{"interface":"back","average":{"1min":8.8889999999999993,"5min":5.1909999999999998,"15min":2.323},"min":{"1min":0.23699999999999999,"5min":0.20100000000000001,"15min":0.191},"max":{"1min":85.495999999999995,"5min":132.04499999999999,"15min":132.04499999999999},"last":0.82199999999999995},{"interface":"front","average":{"1min":8.9659999999999993,"5min":5.5519999999999996,"15min":2.6419999999999999},"min":{"1min":0.27100000000000002,"5min":0.23599999999999999,"15min":0.21299999999999999},"max":{"1min":85.900999999999996,"5min":132.91200000000001,"15min":132.91200000000001},"last":0.65100000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":3.5169999999999999,"15min":2.9260000000000002},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20100000000000001},"max":{"1min":3.774,"5min":129.053,"15min":204.989},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.994,"5min":3.73,"15min":3.1629999999999998},"min":{"1min":0.28299999999999997,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":12.728,"5min":129.077,"15min":203.65899999999999},"last":1.079}]},{"osd":3,"last update":"Mon Oct 9 06:02:43 2023","interfaces":[{"interface":"back","average":{"1min":7.5579999999999998,"5min":6.0049999999999999,"15min":5.0330000000000004},"min":{"1min":0.32100000000000001,"5min":0.27600000000000002,"15min":0.214},"max":{"1min":112.20699999999999,"5min":213.654,"15min":284.06099999999998},"last":1.3520000000000001},{"interface":"front","average":{"1min":7.4870000000000001,"5min":5.9539999999999997,"15min":5.0990000000000002},"min":{"1min":0.34399999999999997,"5min":0.28100000000000003,"15min":0.27900000000000003},"max":{"1min":112.901,"5min":213.77199999999999,"15min":284.97300000000001},"last":1.4390000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:03:25 2023","interfaces":[{"interface":"back","average":{"1min":1.298,"5min":1.0860000000000001,"15min":1.05},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.0429999999999993,"5min":8.0429999999999993,"15min":8.0429999999999993},"last":0.83999999999999997},{"interface":"front","average":{"1min":0.89000000000000001,"5min":1.4550000000000001,"15min":1.5489999999999999},"min":{"1min":0.51100000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.9259999999999999,"5min":9.8629999999999995,"15min":9.8629999999999995},"last":0.46400000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:03:37 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.0640000000000001,"15min":1.198},"min":{"1min":0.372,"5min":0.34200000000000003,"15min":0.32800000000000001},"max":{"1min":4.3250000000000002,"5min":8.0980000000000008,"15min":64.584999999999994},"last":0.80500000000000005},{"interface":"front","average":{"1min":1.5449999999999999,"5min":1.5429999999999999,"15min":1.373},"min":{"1min":0.51200000000000001,"5min":0.34300000000000003,"15min":0.315},"max":{"1min":8.3390000000000004,"5min":9.827,"15min":64.150999999999996},"last":0.503}]},{"osd":6,"last update":"Mon Oct 9 06:03:37 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.147,"15min":1.899},"min":{"1min":0.33100000000000002,"5min":0.318,"15min":0.30099999999999999},"max":{"1min":4.3570000000000002,"5min":8.0700000000000003,"15min":204.62700000000001},"last":0.86399999999999999},{"interface":"front","average":{"1min":1.55,"5min":1.6619999999999999,"15min":2.0830000000000002},"min":{"1min":0.54300000000000004,"5min":0.38300000000000001,"15min":0.35499999999999998},"max":{"1min":8.3919999999999995,"5min":8.4990000000000006,"15min":204.285},"last":0.61799999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:03:37 2023","interfaces":[{"interface":"back","average":{"1min":1.5820000000000001,"5min":2.5649999999999999,"15min":1.823},"min":{"1min":0.54300000000000004,"5min":0.41899999999999998,"15min":0.314},"max":{"1min":8.4329999999999998,"5min":108.553,"15min":108.553},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.2150000000000001,"5min":2.3149999999999999,"15min":1.5960000000000001},"min":{"1min":0.39300000000000002,"5min":0.38100000000000001,"15min":0.38},"max":{"1min":7.9359999999999999,"5min":108.56999999999999,"15min":108.56999999999999},"last":0.55200000000000005}]}]},{"osd":0,"up_from":129,"seq":554050781316,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1897460,"kb_used_data":1542124,"kb_used_omap":29,"kb_used_meta":355298,"kb_avail":91847692,"statfs":{"total":95995035648,"available":94052036608,"internally_reserved":0,"allocated":1579134976,"data_stored":1577140383,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30322,"internal_metadata":363825550},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:03:16 2023","interfaces":[{"interface":"back","average":{"1min":1.724,"5min":1.462,"15min":1.0369999999999999},"min":{"1min":0.224,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":22,"5min":22.009,"15min":22.009},"last":1.0860000000000001},{"interface":"front","average":{"1min":1.714,"5min":1.5309999999999999,"15min":1.1759999999999999},"min":{"1min":0.25,"5min":0.248,"15min":0.24099999999999999},"max":{"1min":15.173,"5min":20.082000000000001,"15min":20.082000000000001},"last":1.504}]},{"osd":2,"last update":"Mon Oct 9 06:03:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":2.0449999999999999,"15min":1.429},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.23000000000000001},"max":{"1min":7.133,"5min":52.405999999999999,"15min":52.405999999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.1160000000000001,"5min":2.8980000000000001,"15min":1.675},"min":{"1min":0.28599999999999998,"5min":0.219,"15min":0.219},"max":{"1min":8.1690000000000005,"5min":52.645000000000003,"15min":52.645000000000003},"last":0.27100000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:03:16 2023","interfaces":[{"interface":"back","average":{"1min":1.2450000000000001,"5min":5.7329999999999997,"15min":2.714},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":7.2080000000000002,"5min":134.185,"15min":134.185},"last":0.32200000000000001},{"interface":"front","average":{"1min":1.347,"5min":6.1539999999999999,"15min":2.827},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":8.3239999999999998,"5min":134.16800000000001,"15min":134.16800000000001},"last":0.46600000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:03:21 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.7669999999999999,"15min":1.871},"min":{"1min":0.39600000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.1419999999999995,"5min":10.69,"15min":10.69},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.161,"5min":1.349,"15min":1.3799999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":8.3079999999999998,"5min":8.3079999999999998,"15min":8.3079999999999998},"last":1.246}]},{"osd":5,"last update":"Mon Oct 9 06:03:16 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":2.077,"15min":1.8919999999999999},"min":{"1min":0.44500000000000001,"5min":0.318,"15min":0.314},"max":{"1min":8.109,"5min":39.188000000000002,"15min":100.25},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.206,"5min":1.53,"15min":1.633},"min":{"1min":0.39700000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":8.452,"5min":39.143000000000001,"15min":102.40300000000001},"last":1.141}]},{"osd":6,"last update":"Mon Oct 9 06:03:16 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":1.964,"15min":1.4179999999999999},"min":{"1min":0.46300000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":8.0760000000000005,"5min":16.849,"15min":16.849},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.224,"5min":1.4410000000000001,"15min":1.3},"min":{"1min":0.46899999999999997,"5min":0.39000000000000001,"15min":0.378},"max":{"1min":8.3490000000000002,"5min":17.869,"15min":17.869},"last":1.268}]},{"osd":7,"last update":"Mon Oct 9 06:03:16 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":2.173,"15min":1.8799999999999999},"min":{"1min":0.499,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":8.4779999999999998,"5min":103.866,"15min":103.866},"last":1.1699999999999999},{"interface":"front","average":{"1min":1.083,"5min":2.3380000000000001,"15min":1.845},"min":{"1min":0.439,"5min":0.38300000000000001,"15min":0.378},"max":{"1min":7.1719999999999997,"5min":103.848,"15min":103.848},"last":0.41599999999999998}]}]},{"osd":2,"up_from":94,"seq":403726926006,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2113396,"kb_used_data":1613996,"kb_used_omap":81,"kb_used_meta":499310,"kb_avail":91631756,"statfs":{"total":95995035648,"available":93830918144,"internally_reserved":0,"allocated":1652731904,"data_stored":1650851408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":511293658},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:03:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":3.851,"15min":2.5350000000000001},"min":{"1min":0.20799999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":7.6580000000000004,"5min":70.721999999999994,"15min":70.721999999999994},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.82599999999999996,"5min":3.7879999999999998,"15min":2.468},"min":{"1min":0.224,"5min":0.215,"15min":0.215},"max":{"1min":3.9140000000000001,"5min":72.382999999999996,"15min":72.382999999999996},"last":0.49099999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:03:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":1.6850000000000001,"15min":1.7749999999999999},"min":{"1min":0.22900000000000001,"5min":0.224,"15min":0.21099999999999999},"max":{"1min":7.6210000000000004,"5min":14.226000000000001,"15min":163.24199999999999},"last":0.377},{"interface":"front","average":{"1min":0.82399999999999995,"5min":1.7929999999999999,"15min":1.734},"min":{"1min":0.27000000000000002,"5min":0.252,"15min":0.214},"max":{"1min":4.2229999999999999,"5min":13.625,"15min":163.27000000000001},"last":2.3260000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:03:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":4.423,"15min":3.1459999999999999},"min":{"1min":0.25800000000000001,"5min":0.24099999999999999,"15min":0.23899999999999999},"max":{"1min":5.149,"5min":144.88200000000001,"15min":170.07599999999999},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.615,"5min":4.6849999999999996,"15min":2.794},"min":{"1min":0.29699999999999999,"5min":0.25900000000000001,"15min":0.25},"max":{"1min":7.4000000000000004,"5min":140.79499999999999,"15min":140.79499999999999},"last":0.51000000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:03:26 2023","interfaces":[{"interface":"back","average":{"1min":1.264,"5min":1.631,"15min":1.6919999999999999},"min":{"1min":0.32300000000000001,"5min":0.315,"15min":0.315},"max":{"1min":5.9379999999999997,"5min":7.9989999999999997,"15min":7.9989999999999997},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.4259999999999999,"5min":1.8069999999999999,"15min":1.871},"min":{"1min":0.38,"5min":0.31,"15min":0.31},"max":{"1min":6.4210000000000003,"5min":7.2590000000000003,"15min":7.2590000000000003},"last":0.44400000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:03:12 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":1.4339999999999999,"15min":1.2470000000000001},"min":{"1min":0.42799999999999999,"5min":0.34599999999999997,"15min":0.28999999999999998},"max":{"1min":2.3130000000000002,"5min":8.8819999999999997,"15min":13.414},"last":1.4359999999999999},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.4670000000000001,"15min":1.2430000000000001},"min":{"1min":0.41399999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.1880000000000002,"5min":7.6900000000000004,"15min":13.438000000000001},"last":0.47899999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:03:12 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":2.0990000000000002,"15min":1.389},"min":{"1min":0.38,"5min":0.34699999999999998,"15min":0.29399999999999998},"max":{"1min":3.8769999999999998,"5min":30.420000000000002,"15min":30.420000000000002},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.83999999999999997,"5min":1.921,"15min":1.319},"min":{"1min":0.40100000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.4489999999999998,"5min":30.713000000000001,"15min":30.713000000000001},"last":0.53500000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:03:12 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":2.1280000000000001,"15min":1.7589999999999999},"min":{"1min":0.41199999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":2.2890000000000001,"5min":33.856000000000002,"15min":55.052},"last":0.871},{"interface":"front","average":{"1min":1.0660000000000001,"5min":2.0619999999999998,"15min":1.544},"min":{"1min":0.45200000000000001,"5min":0.35899999999999999,"15min":0.35599999999999998},"max":{"1min":3.214,"5min":33.098999999999997,"15min":40.113},"last":1.0840000000000001}]}]},{"osd":3,"up_from":99,"seq":425201762479,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2047268,"kb_used_data":1518172,"kb_used_omap":7,"kb_used_meta":529080,"kb_avail":91697884,"statfs":{"total":95995035648,"available":93898633216,"internally_reserved":0,"allocated":1554608128,"data_stored":1552504497,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":541778536},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:03:21 2023","interfaces":[{"interface":"back","average":{"1min":4.9530000000000003,"5min":4.4000000000000004,"15min":2.2930000000000001},"min":{"1min":0.23799999999999999,"5min":0.20499999999999999,"15min":0.186},"max":{"1min":84.673000000000002,"5min":109.976,"15min":109.976},"last":3.4020000000000001},{"interface":"front","average":{"1min":4.9809999999999999,"5min":4.5270000000000001,"15min":2.4940000000000002},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":84.439999999999998,"5min":107.161,"15min":107.161},"last":3.2120000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:02:46 2023","interfaces":[{"interface":"back","average":{"1min":2.4609999999999999,"5min":1.925,"15min":1.623},"min":{"1min":0.26800000000000002,"5min":0.23300000000000001,"15min":0.20999999999999999},"max":{"1min":10.984999999999999,"5min":26.538,"15min":127.226},"last":0.51900000000000002},{"interface":"front","average":{"1min":2.5649999999999999,"5min":3.1800000000000002,"15min":2.016},"min":{"1min":0.29199999999999998,"5min":0.248,"15min":0.21099999999999999},"max":{"1min":19.751999999999999,"5min":106.95699999999999,"15min":127.468},"last":1.081}]},{"osd":2,"last update":"Mon Oct 9 06:02:46 2023","interfaces":[{"interface":"back","average":{"1min":2.0569999999999999,"5min":4.0030000000000001,"15min":1.9670000000000001},"min":{"1min":0.27000000000000002,"5min":0.23499999999999999,"15min":0.23200000000000001},"max":{"1min":9.577,"5min":107.01300000000001,"15min":107.01300000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":2.181,"5min":3.9180000000000001,"15min":2.1659999999999999},"min":{"1min":0.28199999999999997,"5min":0.249,"15min":0.249},"max":{"1min":11.380000000000001,"5min":109.807,"15min":109.807},"last":3.4740000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:03:26 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":5.8869999999999996,"15min":6.6980000000000004},"min":{"1min":0.34499999999999997,"5min":0.254,"15min":0.254},"max":{"1min":3.8809999999999998,"5min":99.876000000000005,"15min":99.876000000000005},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.92100000000000004,"5min":5.508,"15min":6.2720000000000002},"min":{"1min":0.45200000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":4.5439999999999996,"5min":99.914000000000001,"15min":99.914000000000001},"last":1.119}]},{"osd":5,"last update":"Mon Oct 9 06:02:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":1.925,"15min":1.2629999999999999},"min":{"1min":0.42499999999999999,"5min":0.308,"15min":0.308},"max":{"1min":9.0129999999999999,"5min":22.670000000000002,"15min":22.670000000000002},"last":0.77200000000000002},{"interface":"front","average":{"1min":2.278,"5min":2.0920000000000001,"15min":1.2769999999999999},"min":{"1min":0.44400000000000001,"5min":0.34300000000000003,"15min":0.28199999999999997},"max":{"1min":7.5010000000000003,"5min":7.7110000000000003,"15min":18.013999999999999},"last":0.85499999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:02:46 2023","interfaces":[{"interface":"back","average":{"1min":1.7929999999999999,"5min":6.218,"15min":2.903},"min":{"1min":0.42599999999999999,"5min":0.33100000000000002,"15min":0.30499999999999999},"max":{"1min":9.0419999999999998,"5min":152.69,"15min":152.69},"last":0.81000000000000005},{"interface":"front","average":{"1min":2.2130000000000001,"5min":6.5060000000000002,"15min":2.9340000000000002},"min":{"1min":0.46400000000000002,"5min":0.39700000000000002,"15min":0.318},"max":{"1min":7.5739999999999998,"5min":152.70400000000001,"15min":152.70400000000001},"last":0.55400000000000005}]},{"osd":7,"last update":"Mon Oct 9 06:02:46 2023","interfaces":[{"interface":"back","average":{"1min":2.4009999999999998,"5min":6.7800000000000002,"15min":2.9079999999999999},"min":{"1min":0.44,"5min":0.377,"15min":0.34200000000000003},"max":{"1min":7.5979999999999999,"5min":149.93600000000001,"15min":149.93600000000001},"last":0.89500000000000002},{"interface":"front","average":{"1min":1.915,"5min":6.7039999999999997,"15min":2.8610000000000002},"min":{"1min":0.41499999999999998,"5min":0.379,"15min":0.379},"max":{"1min":8.7240000000000002,"5min":150.12299999999999,"15min":150.12299999999999},"last":1.163}]}]},{"osd":4,"up_from":322,"seq":1382979469342,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1621188,"kb_used_data":1290620,"kb_used_omap":35,"kb_used_meta":330524,"kb_avail":92123964,"statfs":{"total":95995035648,"available":94334939136,"internally_reserved":0,"allocated":1321594880,"data_stored":1320463668,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36552,"internal_metadata":338456888},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":1.131,"15min":1.2210000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":1.5,"5min":4.2720000000000002,"15min":4.2720000000000002},"last":1.0029999999999999},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.7,"15min":1.794},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":8.3719999999999999,"5min":8.3719999999999999,"15min":8.3719999999999999},"last":0.38800000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":1.6899999999999999,"15min":1.8109999999999999},"min":{"1min":0.30199999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.1110000000000002,"5min":6.3810000000000002,"15min":6.3810000000000002},"last":78.891000000000005},{"interface":"front","average":{"1min":0.84799999999999998,"5min":1.3919999999999999,"15min":1.4830000000000001},"min":{"1min":0.316,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.0920000000000001,"5min":12.413,"15min":12.413},"last":78.921000000000006}]},{"osd":2,"last update":"Mon Oct 9 06:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.83699999999999997,"5min":1.6739999999999999,"15min":1.8140000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.6080000000000001,"5min":4.8840000000000003,"15min":4.8840000000000003},"last":1.0780000000000001},{"interface":"front","average":{"1min":0.85199999999999998,"5min":1.966,"15min":2.1520000000000001},"min":{"1min":0.34399999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.9929999999999999,"5min":11.381,"15min":11.381},"last":0.42599999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:03:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.9019999999999999,"15min":2.0379999999999998},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":3.5539999999999998,"5min":4.9009999999999998,"15min":4.9009999999999998},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.2150000000000001,"5min":1.6499999999999999,"15min":1.722},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":5.931,"5min":7.5960000000000001,"15min":7.5960000000000001},"last":1.046}]},{"osd":5,"last update":"Mon Oct 9 06:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.85499999999999998,"15min":0.89500000000000002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.226,"5min":5.7629999999999999,"15min":5.7629999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.503,"5min":1.3,"15min":1.4330000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.2689999999999999,"5min":4.8369999999999997,"15min":4.8369999999999997},"last":0.371}]},{"osd":6,"last update":"Mon Oct 9 06:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":3.5419999999999998,"15min":4.0369999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.1040000000000001,"5min":53.744999999999997,"15min":53.744999999999997},"last":2.3250000000000002},{"interface":"front","average":{"1min":0.52900000000000003,"5min":3.7309999999999999,"15min":4.2649999999999997},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.048,"5min":53.798000000000002,"15min":53.798000000000002},"last":0.746}]},{"osd":7,"last update":"Mon Oct 9 06:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":5.3380000000000001,"15min":6.1260000000000003},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.4119999999999999,"5min":79.173000000000002,"15min":79.173000000000002},"last":2.141},{"interface":"front","average":{"1min":0.64600000000000002,"5min":4.3280000000000003,"15min":4.9420000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.5669999999999999,"5min":78.894000000000005,"15min":78.894000000000005},"last":1.244}]}]},{"osd":5,"up_from":38,"seq":163208757562,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2297400,"kb_used_data":1996016,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":91447752,"statfs":{"total":95995035648,"available":93642498048,"internally_reserved":0,"allocated":2043920384,"data_stored":2042797748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:03:27 2023","interfaces":[{"interface":"back","average":{"1min":4.0800000000000001,"5min":2.5379999999999998,"15min":1.4430000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.22500000000000001},"max":{"1min":69.088999999999999,"5min":69.088999999999999,"15min":69.088999999999999},"last":0.314},{"interface":"front","average":{"1min":4.024,"5min":2.8759999999999999,"15min":1.663},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":69.073999999999998,"5min":69.073999999999998,"15min":69.073999999999998},"last":1.276}]},{"osd":1,"last update":"Mon Oct 9 06:03:43 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":2.347,"15min":2.2709999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.26000000000000001},"max":{"1min":4.0179999999999998,"5min":102.642,"15min":114.247},"last":1.349},{"interface":"front","average":{"1min":0.96899999999999997,"5min":2.0099999999999998,"15min":2.0110000000000001},"min":{"1min":0.28899999999999998,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":2.5009999999999999,"5min":102.45999999999999,"15min":114.678},"last":0.34499999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:03:12 2023","interfaces":[{"interface":"back","average":{"1min":7.4580000000000002,"5min":3.8380000000000001,"15min":3.0990000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.25},"max":{"1min":129.21199999999999,"5min":129.21199999999999,"15min":217.06399999999999},"last":1.409},{"interface":"front","average":{"1min":7.6970000000000001,"5min":3.5030000000000001,"15min":2.9430000000000001},"min":{"1min":0.38100000000000001,"5min":0.28199999999999997,"15min":0.27600000000000002},"max":{"1min":129.042,"5min":129.042,"15min":217.018},"last":0.48899999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:03:43 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":8.0969999999999995,"15min":6.8780000000000001},"min":{"1min":0.39700000000000002,"5min":0.31900000000000001,"15min":0.27500000000000002},"max":{"1min":8.0670000000000002,"5min":97.403000000000006,"15min":272.81299999999999},"last":0.71299999999999997},{"interface":"front","average":{"1min":1.2290000000000001,"5min":7.8719999999999999,"15min":7.0019999999999998},"min":{"1min":0.47699999999999998,"5min":0.32200000000000001,"15min":0.317},"max":{"1min":2.8490000000000002,"5min":94.423000000000002,"15min":273.00999999999999},"last":1.387}]},{"osd":4,"last update":"Mon Oct 9 06:03:27 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.82199999999999995,"15min":0.88},"min":{"1min":0.28000000000000003,"5min":0.249,"15min":0.249},"max":{"1min":0.82299999999999995,"5min":4.9720000000000004,"15min":4.9720000000000004},"last":0.374},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.0860000000000001,"15min":1.1080000000000001},"min":{"1min":0.27000000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.8029999999999999,"5min":5.8879999999999999,"15min":5.8879999999999999},"last":0.75}]},{"osd":6,"last update":"Mon Oct 9 06:02:44 2023","interfaces":[{"interface":"back","average":{"1min":1.5980000000000001,"5min":2.2469999999999999,"15min":1.2350000000000001},"min":{"1min":0.312,"5min":0.251,"15min":0.251},"max":{"1min":9.0909999999999993,"5min":126.89400000000001,"15min":126.89400000000001},"last":0.78400000000000003},{"interface":"front","average":{"1min":1.4139999999999999,"5min":2.254,"15min":1.2849999999999999},"min":{"1min":0.32300000000000001,"5min":0.26700000000000002,"15min":0.26200000000000001},"max":{"1min":5.4279999999999999,"5min":126.616,"15min":126.616},"last":1.367}]},{"osd":7,"last update":"Mon Oct 9 06:03:27 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":3.6779999999999999,"15min":1.8640000000000001},"min":{"1min":0.34599999999999997,"5min":0.308,"15min":0.29999999999999999},"max":{"1min":2.879,"5min":133.27099999999999,"15min":133.27099999999999},"last":1.3169999999999999},{"interface":"front","average":{"1min":0.85099999999999998,"5min":3.5619999999999998,"15min":1.661},"min":{"1min":0.442,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.4620000000000002,"5min":135.09200000000001,"15min":135.09200000000001},"last":0.51600000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1474,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4889,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1574240256,"data_stored":1574240256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1843396608,"data_stored":1843396608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1655767040,"data_stored":1655767040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1563426816,"data_stored":1563426816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1321336832,"data_stored":1321336832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2071855104,"data_stored":2071855104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1380974592,"data_stored":1380974592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1311047680,"data_stored":1311047680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:03:51.219 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:03:51.310 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:51.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:51.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:51.358 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:51.417 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:51.495 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:51.744 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:51.746 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 49 16 70211 70195 89.4866 62.9375 0.00689971 0.0111689 2023-10-09T06:03:51.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:51.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:51.859 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:51.918 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:52.029 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:52.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:52.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:52.134 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:52.221 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:52.404 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:03:52.404 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:03:52.404 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 0 2023-10-09T06:03:52.404 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:03:52.470 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:03:52.645 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:52.731 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:52 smithi064 ceph-mon[212750]: pgmap v889: 119 pgs: 119 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s 2023-10-09T06:03:52.731 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:52 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3476419599' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:03:52.731 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:52 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:03:52.732 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:52 smithi064 ceph-mon[220044]: pgmap v889: 119 pgs: 119 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s 2023-10-09T06:03:52.732 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:52 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3476419599' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:03:52.732 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:52 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:03:52.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:52.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:52.746 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 50 16 71068 71052 88.7683 53.5625 0.00295159 0.0112455 2023-10-09T06:03:52.834 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:52.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:52 smithi156 ceph-mon[195101]: pgmap v889: 119 pgs: 119 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s 2023-10-09T06:03:52.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:52 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3476419599' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:03:52.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:52 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:03:52.913 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:53.389 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:03:53.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:53 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:03:53.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:53 smithi156 ceph-mon[195101]: failed to encode map e371 with expected crc 2023-10-09T06:03:53.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:53 smithi156 ceph-mon[195101]: failed to encode map e371 with expected crc 2023-10-09T06:03:53.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:53 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:03:53.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:53 smithi156 ceph-mon[195101]: osdmap e371: 8 total, 8 up, 8 in 2023-10-09T06:03:53.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:53 smithi156 ceph-mon[195101]: failed to encode map e371 with expected crc 2023-10-09T06:03:53.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:53 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:03:53.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:53 smithi064 ceph-mon[212750]: failed to encode map e371 with expected crc 2023-10-09T06:03:53.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:53 smithi064 ceph-mon[212750]: failed to encode map e371 with expected crc 2023-10-09T06:03:53.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:53 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:03:53.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:53 smithi064 ceph-mon[212750]: osdmap e371: 8 total, 8 up, 8 in 2023-10-09T06:03:53.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:53 smithi064 ceph-mon[212750]: failed to encode map e371 with expected crc 2023-10-09T06:03:53.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:53 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:03:53.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:53 smithi064 ceph-mon[220044]: failed to encode map e371 with expected crc 2023-10-09T06:03:53.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:53 smithi064 ceph-mon[220044]: failed to encode map e371 with expected crc 2023-10-09T06:03:53.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:53 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:03:53.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:53 smithi064 ceph-mon[220044]: osdmap e371: 8 total, 8 up, 8 in 2023-10-09T06:03:53.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:53 smithi064 ceph-mon[220044]: failed to encode map e371 with expected crc 2023-10-09T06:03:53.746 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 51 16 71974 71958 88.1382 56.625 0.00295199 0.011331 2023-10-09T06:03:53.864 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:03:54.059 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:54.073 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:54.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:54.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:54.181 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:54.228 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:54.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:54 smithi064 ceph-mon[220044]: pgmap v891: 119 pgs: 119 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2023-10-09T06:03:54.640 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:54 smithi064 ceph-mon[212750]: pgmap v891: 119 pgs: 119 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2023-10-09T06:03:54.721 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:54.727 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:54 smithi156 ceph-mon[195101]: pgmap v891: 119 pgs: 119 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2023-10-09T06:03:54.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:54.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:54.746 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 52 16 72878 72862 87.53 56.5 0.00721915 0.0114195 2023-10-09T06:03:54.831 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:54.916 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:55.170 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:03:55.170 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:03:55.170 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:03:55.170 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:03:55.177 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:03:55.222 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:03:55.224 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:55.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:55.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:55.258 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:03:55.330 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:55.400 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:55.479 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:03:55.479 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:03:55.479 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:03:55.479 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:03:55.479 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:03:55.479 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:03:55.479 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:03:55.479 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:03:55.479 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:03:55.482 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:03:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:03:55] "GET /metrics HTTP/1.1" 200 36379 "" "Prometheus/2.43.0" 2023-10-09T06:03:55.483 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:55 smithi064 ceph-mon[212750]: pgmap v892: 119 pgs: 119 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2023-10-09T06:03:55.483 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:55 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3097267485' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:55.490 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:03:55.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:55.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:03:55.594 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:55.611 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:55 smithi156 ceph-mon[195101]: pgmap v892: 119 pgs: 119 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2023-10-09T06:03:55.611 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:55 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3097267485' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:55.662 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:55.748 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 53 16 73880 73864 87.0589 62.625 0.0118095 0.0114814 2023-10-09T06:03:55.904 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:03:55.904 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:03:55.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:55 smithi064 ceph-mon[220044]: pgmap v892: 119 pgs: 119 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2023-10-09T06:03:55.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:55 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3097267485' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:03:56.078 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:03:56.326 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:56.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:56.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:56.441 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:56.488 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:56.664 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:56.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:56.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:56.748 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 54 16 74830 74814 86.5452 59.375 0.00893331 0.0115495 2023-10-09T06:03:56.768 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:56.786 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:03:56.851 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:57.023 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:57.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:57.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:57.129 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:03:57.190 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:57.339 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:03:57.355 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:03:57.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:57.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:03:57.459 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:57.463 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:03:57.557 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:57.750 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 55 16 75817 75801 86.0898 61.6875 0.0176797 0.0116103 2023-10-09T06:03:57.813 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:03:57.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:57.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:03:57.835 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:03:57.855 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T06:03:57.926 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:03:57.974 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:58.061 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:03:58.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:58.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:03:58.165 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:03:58.195 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:58.241 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:03:58.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:58.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:03:58.345 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:03:58.423 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:58.464 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:03:58.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:58.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:03:58.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:58 smithi064 ceph-mon[220044]: pgmap v893: 119 pgs: 119 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s 2023-10-09T06:03:58.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:58 smithi064 ceph-mon[212750]: pgmap v893: 119 pgs: 119 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s 2023-10-09T06:03:58.568 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:58.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:58 smithi156 ceph-mon[195101]: pgmap v893: 119 pgs: 119 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s 2023-10-09T06:03:58.626 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:58.750 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 56 16 76524 76508 85.3418 44.1875 0.00369225 0.011705 2023-10-09T06:03:58.767 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:58.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:58.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:58.876 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:03:58.903 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:58.998 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:03:59.046 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:03:59.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:59.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:03:59.152 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:03:59.225 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:59.304 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:03:59.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:59.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:03:59.333 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:03:59.333 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:03:59.409 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:03:59.461 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:59 smithi156 ceph-mon[195101]: from='client.35709 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:03:59.461 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:59 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:59.461 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:59.462 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-10-09T06:03:59.462 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-10-09T06:03:59.462 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:59 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-10-09T06:03:59.462 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-10-09T06:03:59.462 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:59 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-10-09T06:03:59.462 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:03:59 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-10-09T06:03:59.469 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:59.538 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:03:59.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:03:59.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:03:59.649 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:03:59.676 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:59 smithi064 ceph-mon[212750]: from='client.35709 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:03:59.676 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:59 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:59.676 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:59.676 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-10-09T06:03:59.676 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-10-09T06:03:59.676 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:59 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-10-09T06:03:59.676 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-10-09T06:03:59.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:59 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-10-09T06:03:59.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:03:59 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-10-09T06:03:59.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:59 smithi064 ceph-mon[220044]: from='client.35709 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:03:59.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:59 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:03:59.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:59.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-10-09T06:03:59.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-10-09T06:03:59.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:59 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-10-09T06:03:59.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-10-09T06:03:59.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:59 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-10-09T06:03:59.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:03:59 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-10-09T06:03:59.678 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:03:59.750 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 57 16 77263 77247 84.655 46.1875 0.00317688 0.011785 2023-10-09T06:04:00.004 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:00.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:00.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:00.044 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":893,"stamp":"2023-10-09T06:03:56.821609+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4798027680,"num_objects":73227,"num_object_clones":0,"num_object_copies":219681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":73227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":73264,"num_write_kb":4685021,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":1324,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":78249,"ondisk_log_size":78249,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17128724,"kb_used_data":13960852,"kb_used_omap":163,"kb_used_meta":3167644,"kb_avail":732832492,"statfs":{"total":767960285184,"available":750420471808,"internally_reserved":0,"allocated":14295912448,"data_stored":14283288585,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":167439,"internal_metadata":3243667953},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[9,0,1,10,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":613941248,"num_objects":9368,"num_object_clones":0,"num_object_copies":28104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9368,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9368,"num_write_kb":599552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003800"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":601,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.426454+0000","last_change":"2023-10-09T06:00:05.413041+0000","last_active":"2023-10-09T06:03:41.426454+0000","last_peered":"2023-10-09T06:03:41.426454+0000","last_clean":"2023-10-09T06:03:41.426454+0000","last_became_active":"2023-10-09T05:59:40.308608+0000","last_became_peered":"2023-10-09T05:59:40.308608+0000","last_unstale":"2023-10-09T06:03:41.426454+0000","last_undegraded":"2023-10-09T06:03:41.426454+0000","last_fullsized":"2023-10-09T06:03:41.426454+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:05.412941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:05.412941+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:53.518055+0000","scrub_duration":0.0061014329999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":419,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.421467+0000","last_change":"2023-10-09T06:01:54.757141+0000","last_active":"2023-10-09T06:03:41.421467+0000","last_peered":"2023-10-09T06:03:41.421467+0000","last_clean":"2023-10-09T06:03:41.421467+0000","last_became_active":"2023-10-09T06:01:54.756855+0000","last_became_peered":"2023-10-09T06:01:54.756855+0000","last_unstale":"2023-10-09T06:03:41.421467+0000","last_undegraded":"2023-10-09T06:03:41.421467+0000","last_fullsized":"2023-10-09T06:03:41.421467+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:57.741319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":159,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.405051+0000","last_change":"2023-10-09T06:01:54.762232+0000","last_active":"2023-10-09T06:03:41.405051+0000","last_peered":"2023-10-09T06:03:41.405051+0000","last_clean":"2023-10-09T06:03:41.405051+0000","last_became_active":"2023-10-09T06:01:54.761935+0000","last_became_peered":"2023-10-09T06:01:54.761935+0000","last_unstale":"2023-10-09T06:03:41.405051+0000","last_undegraded":"2023-10-09T06:03:41.405051+0000","last_fullsized":"2023-10-09T06:03:41.405051+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:20.380704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":653,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.421347+0000","last_change":"2023-10-09T06:02:01.468449+0000","last_active":"2023-10-09T06:03:41.421347+0000","last_peered":"2023-10-09T06:03:41.421347+0000","last_clean":"2023-10-09T06:03:41.421347+0000","last_became_active":"2023-10-09T06:01:22.334923+0000","last_became_peered":"2023-10-09T06:01:22.334923+0000","last_unstale":"2023-10-09T06:03:41.421347+0000","last_undegraded":"2023-10-09T06:03:41.421347+0000","last_fullsized":"2023-10-09T06:03:41.421347+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:01.468346+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:01.468346+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:20:04.967048+0000","scrub_duration":0.0016258220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.413721+0000","last_change":"2023-10-09T06:01:54.762060+0000","last_active":"2023-10-09T06:03:41.413721+0000","last_peered":"2023-10-09T06:03:41.413721+0000","last_clean":"2023-10-09T06:03:41.413721+0000","last_became_active":"2023-10-09T06:01:54.761757+0000","last_became_peered":"2023-10-09T06:01:54.761757+0000","last_unstale":"2023-10-09T06:03:41.413721+0000","last_undegraded":"2023-10-09T06:03:41.413721+0000","last_fullsized":"2023-10-09T06:03:41.413721+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:40.294232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":331,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.428312+0000","last_change":"2023-10-09T06:01:41.942412+0000","last_active":"2023-10-09T06:03:41.428312+0000","last_peered":"2023-10-09T06:03:41.428312+0000","last_clean":"2023-10-09T06:03:41.428312+0000","last_became_active":"2023-10-09T06:01:41.941186+0000","last_became_peered":"2023-10-09T06:01:41.941186+0000","last_unstale":"2023-10-09T06:03:41.428312+0000","last_undegraded":"2023-10-09T06:03:41.428312+0000","last_fullsized":"2023-10-09T06:03:41.428312+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:31.680824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":615,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.421217+0000","last_change":"2023-10-09T06:02:43.293521+0000","last_active":"2023-10-09T06:03:41.421217+0000","last_peered":"2023-10-09T06:03:41.421217+0000","last_clean":"2023-10-09T06:03:41.421217+0000","last_became_active":"2023-10-09T06:01:57.803642+0000","last_became_peered":"2023-10-09T06:01:57.803642+0000","last_unstale":"2023-10-09T06:03:41.421217+0000","last_undegraded":"2023-10-09T06:03:41.421217+0000","last_fullsized":"2023-10-09T06:03:41.421217+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.293426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.293426+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:45.626761+0000","scrub_duration":0.001587946,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":144,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.404811+0000","last_change":"2023-10-09T06:01:54.760903+0000","last_active":"2023-10-09T06:03:41.404811+0000","last_peered":"2023-10-09T06:03:41.404811+0000","last_clean":"2023-10-09T06:03:41.404811+0000","last_became_active":"2023-10-09T06:01:54.760338+0000","last_became_peered":"2023-10-09T06:01:54.760338+0000","last_unstale":"2023-10-09T06:03:41.404811+0000","last_undegraded":"2023-10-09T06:03:41.404811+0000","last_fullsized":"2023-10-09T06:03:41.404811+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:53.904647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":407,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.444818+0000","last_change":"2023-10-09T06:01:54.757269+0000","last_active":"2023-10-09T06:03:41.444818+0000","last_peered":"2023-10-09T06:03:41.444818+0000","last_clean":"2023-10-09T06:03:41.444818+0000","last_became_active":"2023-10-09T06:01:54.756894+0000","last_became_peered":"2023-10-09T06:01:54.756894+0000","last_unstale":"2023-10-09T06:03:41.444818+0000","last_undegraded":"2023-10-09T06:03:41.444818+0000","last_fullsized":"2023-10-09T06:03:41.444818+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:45.212821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":690,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.426934+0000","last_change":"2023-10-09T06:01:57.804160+0000","last_active":"2023-10-09T06:03:41.426934+0000","last_peered":"2023-10-09T06:03:41.426934+0000","last_clean":"2023-10-09T06:03:41.426934+0000","last_became_active":"2023-10-09T06:01:57.803424+0000","last_became_peered":"2023-10-09T06:01:57.803424+0000","last_unstale":"2023-10-09T06:03:41.426934+0000","last_undegraded":"2023-10-09T06:03:41.426934+0000","last_fullsized":"2023-10-09T06:03:41.426934+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.384407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.384407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:42.857346+0000","scrub_duration":0.0040651960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":249,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.405551+0000","last_change":"2023-10-09T06:01:41.939852+0000","last_active":"2023-10-09T06:03:41.405551+0000","last_peered":"2023-10-09T06:03:41.405551+0000","last_clean":"2023-10-09T06:03:41.405551+0000","last_became_active":"2023-10-09T06:01:41.939556+0000","last_became_peered":"2023-10-09T06:01:41.939556+0000","last_unstale":"2023-10-09T06:03:41.405551+0000","last_undegraded":"2023-10-09T06:03:41.405551+0000","last_fullsized":"2023-10-09T06:03:41.405551+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:06.980511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":159,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.404880+0000","last_change":"2023-10-09T06:01:54.761101+0000","last_active":"2023-10-09T06:03:41.404880+0000","last_peered":"2023-10-09T06:03:41.404880+0000","last_clean":"2023-10-09T06:03:41.404880+0000","last_became_active":"2023-10-09T06:01:54.760571+0000","last_became_peered":"2023-10-09T06:01:54.760571+0000","last_unstale":"2023-10-09T06:03:41.404880+0000","last_undegraded":"2023-10-09T06:03:41.404880+0000","last_fullsized":"2023-10-09T06:03:41.404880+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:56.812358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":652,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.433802+0000","last_change":"2023-10-09T06:01:22.334061+0000","last_active":"2023-10-09T06:03:41.433802+0000","last_peered":"2023-10-09T06:03:41.433802+0000","last_clean":"2023-10-09T06:03:41.433802+0000","last_became_active":"2023-10-09T06:01:22.333698+0000","last_became_peered":"2023-10-09T06:01:22.333698+0000","last_unstale":"2023-10-09T06:03:41.433802+0000","last_undegraded":"2023-10-09T06:03:41.433802+0000","last_fullsized":"2023-10-09T06:03:41.433802+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.913004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.913004+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:57.020327+0000","scrub_duration":0.0017791980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":420,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.428429+0000","last_change":"2023-10-09T06:01:54.766344+0000","last_active":"2023-10-09T06:03:41.428429+0000","last_peered":"2023-10-09T06:03:41.428429+0000","last_clean":"2023-10-09T06:03:41.428429+0000","last_became_active":"2023-10-09T06:01:54.766086+0000","last_became_peered":"2023-10-09T06:01:54.766086+0000","last_unstale":"2023-10-09T06:03:41.428429+0000","last_undegraded":"2023-10-09T06:03:41.428429+0000","last_fullsized":"2023-10-09T06:03:41.428429+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:47.163700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":317,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.411409+0000","last_change":"2023-10-09T05:53:59.185702+0000","last_active":"2023-10-09T06:03:41.411409+0000","last_peered":"2023-10-09T06:03:41.411409+0000","last_clean":"2023-10-09T06:03:41.411409+0000","last_became_active":"2023-10-09T05:53:59.185373+0000","last_became_peered":"2023-10-09T05:53:59.185373+0000","last_unstale":"2023-10-09T06:03:41.411409+0000","last_undegraded":"2023-10-09T06:03:41.411409+0000","last_fullsized":"2023-10-09T06:03:41.411409+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:50.756967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":788,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.444937+0000","last_change":"2023-10-09T06:02:48.374519+0000","last_active":"2023-10-09T06:03:41.444937+0000","last_peered":"2023-10-09T06:03:41.444937+0000","last_clean":"2023-10-09T06:03:41.444937+0000","last_became_active":"2023-10-09T06:01:41.939691+0000","last_became_peered":"2023-10-09T06:01:41.939691+0000","last_unstale":"2023-10-09T06:03:41.444937+0000","last_undegraded":"2023-10-09T06:03:41.444937+0000","last_fullsized":"2023-10-09T06:03:41.444937+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:48.374432+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:48.374432+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:54.162995+0000","scrub_duration":0.001678585,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":228,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.427204+0000","last_change":"2023-10-09T06:01:22.337444+0000","last_active":"2023-10-09T06:03:41.427204+0000","last_peered":"2023-10-09T06:03:41.427204+0000","last_clean":"2023-10-09T06:03:41.427204+0000","last_became_active":"2023-10-09T06:01:22.336348+0000","last_became_peered":"2023-10-09T06:01:22.336348+0000","last_unstale":"2023-10-09T06:03:41.427204+0000","last_undegraded":"2023-10-09T06:03:41.427204+0000","last_fullsized":"2023-10-09T06:03:41.427204+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:43:11.149649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.414397+0000","last_change":"2023-10-09T06:01:54.755723+0000","last_active":"2023-10-09T06:03:41.414397+0000","last_peered":"2023-10-09T06:03:41.414397+0000","last_clean":"2023-10-09T06:03:41.414397+0000","last_became_active":"2023-10-09T06:01:54.755334+0000","last_became_peered":"2023-10-09T06:01:54.755334+0000","last_unstale":"2023-10-09T06:03:41.414397+0000","last_undegraded":"2023-10-09T06:03:41.414397+0000","last_fullsized":"2023-10-09T06:03:41.414397+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:12.041652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":523,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.414304+0000","last_change":"2023-10-09T06:01:57.807839+0000","last_active":"2023-10-09T06:03:41.414304+0000","last_peered":"2023-10-09T06:03:41.414304+0000","last_clean":"2023-10-09T06:03:41.414304+0000","last_became_active":"2023-10-09T06:01:57.807451+0000","last_became_peered":"2023-10-09T06:01:57.807451+0000","last_unstale":"2023-10-09T06:03:41.414304+0000","last_undegraded":"2023-10-09T06:03:41.414304+0000","last_fullsized":"2023-10-09T06:03:41.414304+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.339121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.339121+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:47.740601+0000","scrub_duration":0.010474344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":365,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.423271+0000","last_change":"2023-10-09T06:01:22.335608+0000","last_active":"2023-10-09T06:03:41.423271+0000","last_peered":"2023-10-09T06:03:41.423271+0000","last_clean":"2023-10-09T06:03:41.423271+0000","last_became_active":"2023-10-09T06:01:22.335296+0000","last_became_peered":"2023-10-09T06:01:22.335296+0000","last_unstale":"2023-10-09T06:03:41.423271+0000","last_undegraded":"2023-10-09T06:03:41.423271+0000","last_fullsized":"2023-10-09T06:03:41.423271+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:21.843341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":383,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.421498+0000","last_change":"2023-10-09T06:01:54.764916+0000","last_active":"2023-10-09T06:03:41.421498+0000","last_peered":"2023-10-09T06:03:41.421498+0000","last_clean":"2023-10-09T06:03:41.421498+0000","last_became_active":"2023-10-09T06:01:54.764705+0000","last_became_peered":"2023-10-09T06:01:54.764705+0000","last_unstale":"2023-10-09T06:03:41.421498+0000","last_undegraded":"2023-10-09T06:03:41.421498+0000","last_fullsized":"2023-10-09T06:03:41.421498+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:12.527719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":545,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.410875+0000","last_change":"2023-10-09T06:02:42.900973+0000","last_active":"2023-10-09T06:03:41.410875+0000","last_peered":"2023-10-09T06:03:41.410875+0000","last_clean":"2023-10-09T06:03:41.410875+0000","last_became_active":"2023-10-09T06:01:41.939455+0000","last_became_peered":"2023-10-09T06:01:41.939455+0000","last_unstale":"2023-10-09T06:03:41.410875+0000","last_undegraded":"2023-10-09T06:03:41.410875+0000","last_fullsized":"2023-10-09T06:03:41.410875+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:42.900876+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:42.900876+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:37.457334+0000","scrub_duration":0.0014928649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":230,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.426887+0000","last_change":"2023-10-09T06:01:22.339830+0000","last_active":"2023-10-09T06:03:41.426887+0000","last_peered":"2023-10-09T06:03:41.426887+0000","last_clean":"2023-10-09T06:03:41.426887+0000","last_became_active":"2023-10-09T06:01:22.339525+0000","last_became_peered":"2023-10-09T06:01:22.339525+0000","last_unstale":"2023-10-09T06:03:41.426887+0000","last_undegraded":"2023-10-09T06:03:41.426887+0000","last_fullsized":"2023-10-09T06:03:41.426887+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:57.521163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":412,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.405837+0000","last_change":"2023-10-09T06:01:54.761558+0000","last_active":"2023-10-09T06:03:41.405837+0000","last_peered":"2023-10-09T06:03:41.405837+0000","last_clean":"2023-10-09T06:03:41.405837+0000","last_became_active":"2023-10-09T06:01:54.761251+0000","last_became_peered":"2023-10-09T06:01:54.761251+0000","last_unstale":"2023-10-09T06:03:41.405837+0000","last_undegraded":"2023-10-09T06:03:41.405837+0000","last_fullsized":"2023-10-09T06:03:41.405837+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:53.245267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":794,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.428260+0000","last_change":"2023-10-09T05:59:15.170543+0000","last_active":"2023-10-09T06:03:41.428260+0000","last_peered":"2023-10-09T06:03:41.428260+0000","last_clean":"2023-10-09T06:03:41.428260+0000","last_became_active":"2023-10-09T05:58:21.332533+0000","last_became_peered":"2023-10-09T05:58:21.332533+0000","last_unstale":"2023-10-09T06:03:41.428260+0000","last_undegraded":"2023-10-09T06:03:41.428260+0000","last_fullsized":"2023-10-09T06:03:41.428260+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:33.995695+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":296,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.426698+0000","last_change":"2023-10-09T06:01:41.942115+0000","last_active":"2023-10-09T06:03:41.426698+0000","last_peered":"2023-10-09T06:03:41.426698+0000","last_clean":"2023-10-09T06:03:41.426698+0000","last_became_active":"2023-10-09T06:01:41.941827+0000","last_became_peered":"2023-10-09T06:01:41.941827+0000","last_unstale":"2023-10-09T06:03:41.426698+0000","last_undegraded":"2023-10-09T06:03:41.426698+0000","last_fullsized":"2023-10-09T06:03:41.426698+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:51:15.226428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":294,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.427092+0000","last_change":"2023-10-09T05:56:49.237784+0000","last_active":"2023-10-09T06:03:41.427092+0000","last_peered":"2023-10-09T06:03:41.427092+0000","last_clean":"2023-10-09T06:03:41.427092+0000","last_became_active":"2023-10-09T05:56:49.237417+0000","last_became_peered":"2023-10-09T05:56:49.237417+0000","last_unstale":"2023-10-09T06:03:41.427092+0000","last_undegraded":"2023-10-09T06:03:41.427092+0000","last_fullsized":"2023-10-09T06:03:41.427092+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:33.129268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":570,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.414087+0000","last_change":"2023-10-09T06:02:43.316814+0000","last_active":"2023-10-09T06:03:41.414087+0000","last_peered":"2023-10-09T06:03:41.414087+0000","last_clean":"2023-10-09T06:03:41.414087+0000","last_became_active":"2023-10-09T06:01:57.807514+0000","last_became_peered":"2023-10-09T06:01:57.807514+0000","last_unstale":"2023-10-09T06:03:41.414087+0000","last_undegraded":"2023-10-09T06:03:41.414087+0000","last_fullsized":"2023-10-09T06:03:41.414087+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.316697+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.316697+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:15.123801+0000","scrub_duration":0.001666561,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":224,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.426559+0000","last_change":"2023-10-09T06:01:22.337171+0000","last_active":"2023-10-09T06:03:41.426559+0000","last_peered":"2023-10-09T06:03:41.426559+0000","last_clean":"2023-10-09T06:03:41.426559+0000","last_became_active":"2023-10-09T06:01:22.335438+0000","last_became_peered":"2023-10-09T06:01:22.335438+0000","last_unstale":"2023-10-09T06:03:41.426559+0000","last_undegraded":"2023-10-09T06:03:41.426559+0000","last_fullsized":"2023-10-09T06:03:41.426559+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:25.036448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":293,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.405911+0000","last_change":"2023-10-09T06:01:41.941888+0000","last_active":"2023-10-09T06:03:41.405911+0000","last_peered":"2023-10-09T06:03:41.405911+0000","last_clean":"2023-10-09T06:03:41.405911+0000","last_became_active":"2023-10-09T06:01:41.941413+0000","last_became_peered":"2023-10-09T06:01:41.941413+0000","last_unstale":"2023-10-09T06:03:41.405911+0000","last_undegraded":"2023-10-09T06:03:41.405911+0000","last_fullsized":"2023-10-09T06:03:41.405911+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:32:41.773690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":622,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.424306+0000","last_change":"2023-10-09T06:02:40.388302+0000","last_active":"2023-10-09T06:03:41.424306+0000","last_peered":"2023-10-09T06:03:41.424306+0000","last_clean":"2023-10-09T06:03:41.424306+0000","last_became_active":"2023-10-09T06:01:57.803733+0000","last_became_peered":"2023-10-09T06:01:57.803733+0000","last_unstale":"2023-10-09T06:03:41.424306+0000","last_undegraded":"2023-10-09T06:03:41.424306+0000","last_fullsized":"2023-10-09T06:03:41.424306+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:40.388125+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:40.388125+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:17.919838+0000","scrub_duration":0.0055576990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":356,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.444436+0000","last_change":"2023-10-09T05:51:57.091639+0000","last_active":"2023-10-09T06:03:41.444436+0000","last_peered":"2023-10-09T06:03:41.444436+0000","last_clean":"2023-10-09T06:03:41.444436+0000","last_became_active":"2023-10-09T05:51:57.091284+0000","last_became_peered":"2023-10-09T05:51:57.091284+0000","last_unstale":"2023-10-09T06:03:41.444436+0000","last_undegraded":"2023-10-09T06:03:41.444436+0000","last_fullsized":"2023-10-09T06:03:41.444436+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:10.553453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":153,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.405680+0000","last_change":"2023-10-09T06:01:54.760790+0000","last_active":"2023-10-09T06:03:41.405680+0000","last_peered":"2023-10-09T06:03:41.405680+0000","last_clean":"2023-10-09T06:03:41.405680+0000","last_became_active":"2023-10-09T06:01:54.760439+0000","last_became_peered":"2023-10-09T06:01:54.760439+0000","last_unstale":"2023-10-09T06:03:41.405680+0000","last_undegraded":"2023-10-09T06:03:41.405680+0000","last_fullsized":"2023-10-09T06:03:41.405680+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:50.428356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"371'4602","reported_seq":4621,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-10-09T06:03:55.426321+0000","last_change":"2023-10-09T06:02:55.482628+0000","last_active":"2023-10-09T06:03:55.426321+0000","last_peered":"2023-10-09T06:03:55.426321+0000","last_clean":"2023-10-09T06:03:55.426321+0000","last_became_active":"2023-10-09T06:02:55.482332+0000","last_became_peered":"2023-10-09T06:02:55.482332+0000","last_unstale":"2023-10-09T06:03:55.426321+0000","last_undegraded":"2023-10-09T06:03:55.426321+0000","last_fullsized":"2023-10-09T06:03:55.426321+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4602,"log_dups_size":0,"ondisk_log_size":4602,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:58:21.128696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301596672,"num_objects":4602,"num_object_clones":0,"num_object_copies":13806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4602,"num_write_kb":294528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":747,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.427002+0000","last_change":"2023-10-09T06:01:57.804339+0000","last_active":"2023-10-09T06:03:41.427002+0000","last_peered":"2023-10-09T06:03:41.427002+0000","last_clean":"2023-10-09T06:03:41.427002+0000","last_became_active":"2023-10-09T06:01:57.803844+0000","last_became_peered":"2023-10-09T06:01:57.803844+0000","last_unstale":"2023-10-09T06:03:41.427002+0000","last_undegraded":"2023-10-09T06:03:41.427002+0000","last_fullsized":"2023-10-09T06:03:41.427002+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:53.469394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:53.469394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:51.125394+0000","scrub_duration":0.0059716659999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":228,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.427175+0000","last_change":"2023-10-09T06:01:22.336769+0000","last_active":"2023-10-09T06:03:41.427175+0000","last_peered":"2023-10-09T06:03:41.427175+0000","last_clean":"2023-10-09T06:03:41.427175+0000","last_became_active":"2023-10-09T06:01:22.336264+0000","last_became_peered":"2023-10-09T06:01:22.336264+0000","last_unstale":"2023-10-09T06:03:41.427175+0000","last_undegraded":"2023-10-09T06:03:41.427175+0000","last_fullsized":"2023-10-09T06:03:41.427175+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:17.065771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":277,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.426552+0000","last_change":"2023-10-09T05:56:49.234185+0000","last_active":"2023-10-09T06:03:41.426552+0000","last_peered":"2023-10-09T06:03:41.426552+0000","last_clean":"2023-10-09T06:03:41.426552+0000","last_became_active":"2023-10-09T05:56:49.233884+0000","last_became_peered":"2023-10-09T05:56:49.233884+0000","last_unstale":"2023-10-09T06:03:41.426552+0000","last_undegraded":"2023-10-09T06:03:41.426552+0000","last_fullsized":"2023-10-09T06:03:41.426552+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:12.142436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"371'4510","reported_seq":4529,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-10-09T06:03:53.614440+0000","last_change":"2023-10-09T06:02:55.459779+0000","last_active":"2023-10-09T06:03:53.614440+0000","last_peered":"2023-10-09T06:03:53.614440+0000","last_clean":"2023-10-09T06:03:53.614440+0000","last_became_active":"2023-10-09T06:02:55.458593+0000","last_became_peered":"2023-10-09T06:02:55.458593+0000","last_unstale":"2023-10-09T06:03:53.614440+0000","last_undegraded":"2023-10-09T06:03:53.614440+0000","last_fullsized":"2023-10-09T06:03:53.614440+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4510,"log_dups_size":0,"ondisk_log_size":4510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:03.468621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295567360,"num_objects":4510,"num_object_clones":0,"num_object_copies":13530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4510,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4510,"num_write_kb":288640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":678,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.405599+0000","last_change":"2023-10-09T06:00:03.241479+0000","last_active":"2023-10-09T06:03:41.405599+0000","last_peered":"2023-10-09T06:03:41.405599+0000","last_clean":"2023-10-09T06:03:41.405599+0000","last_became_active":"2023-10-09T05:59:32.530753+0000","last_became_peered":"2023-10-09T05:59:32.530753+0000","last_unstale":"2023-10-09T06:03:41.405599+0000","last_undegraded":"2023-10-09T06:03:41.405599+0000","last_fullsized":"2023-10-09T06:03:41.405599+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:03.241335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:03.241335+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:54.822114+0000","scrub_duration":0.017780547000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":284,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.428457+0000","last_change":"2023-10-09T06:01:54.826540+0000","last_active":"2023-10-09T06:03:41.428457+0000","last_peered":"2023-10-09T06:03:41.428457+0000","last_clean":"2023-10-09T06:03:41.428457+0000","last_became_active":"2023-10-09T06:01:54.766205+0000","last_became_peered":"2023-10-09T06:01:54.766205+0000","last_unstale":"2023-10-09T06:03:41.428457+0000","last_undegraded":"2023-10-09T06:03:41.428457+0000","last_fullsized":"2023-10-09T06:03:41.428457+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:45.220336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":409,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.420873+0000","last_change":"2023-10-09T06:01:54.760367+0000","last_active":"2023-10-09T06:03:41.420873+0000","last_peered":"2023-10-09T06:03:41.420873+0000","last_clean":"2023-10-09T06:03:41.420873+0000","last_became_active":"2023-10-09T06:01:54.760087+0000","last_became_peered":"2023-10-09T06:01:54.760087+0000","last_unstale":"2023-10-09T06:03:41.420873+0000","last_undegraded":"2023-10-09T06:03:41.420873+0000","last_fullsized":"2023-10-09T06:03:41.420873+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:26.000104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"371'4578","reported_seq":4598,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-10-09T06:03:55.203613+0000","last_change":"2023-10-09T06:02:55.459326+0000","last_active":"2023-10-09T06:03:55.203613+0000","last_peered":"2023-10-09T06:03:55.203613+0000","last_clean":"2023-10-09T06:03:55.203613+0000","last_became_active":"2023-10-09T06:02:55.458307+0000","last_became_peered":"2023-10-09T06:02:55.458307+0000","last_unstale":"2023-10-09T06:03:55.203613+0000","last_undegraded":"2023-10-09T06:03:55.203613+0000","last_fullsized":"2023-10-09T06:03:55.203613+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4578,"log_dups_size":0,"ondisk_log_size":4578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:04.011767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300023808,"num_objects":4578,"num_object_clones":0,"num_object_copies":13734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4578,"num_write_kb":292992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":810,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.445060+0000","last_change":"2023-10-09T06:01:22.326209+0000","last_active":"2023-10-09T06:03:41.445060+0000","last_peered":"2023-10-09T06:03:41.445060+0000","last_clean":"2023-10-09T06:03:41.445060+0000","last_became_active":"2023-10-09T06:01:22.325866+0000","last_became_peered":"2023-10-09T06:01:22.325866+0000","last_unstale":"2023-10-09T06:03:41.445060+0000","last_undegraded":"2023-10-09T06:03:41.445060+0000","last_fullsized":"2023-10-09T06:03:41.445060+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:41.753506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:41.753506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:32.324674+0000","scrub_duration":0.0022121929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":209,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.427269+0000","last_change":"2023-10-09T05:53:59.212161+0000","last_active":"2023-10-09T06:03:41.427269+0000","last_peered":"2023-10-09T06:03:41.427269+0000","last_clean":"2023-10-09T06:03:41.427269+0000","last_became_active":"2023-10-09T05:53:59.193814+0000","last_became_peered":"2023-10-09T05:53:59.193814+0000","last_unstale":"2023-10-09T06:03:41.427269+0000","last_undegraded":"2023-10-09T06:03:41.427269+0000","last_fullsized":"2023-10-09T06:03:41.427269+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:10.502408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":420,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.445249+0000","last_change":"2023-10-09T06:01:54.757292+0000","last_active":"2023-10-09T06:03:41.445249+0000","last_peered":"2023-10-09T06:03:41.445249+0000","last_clean":"2023-10-09T06:03:41.445249+0000","last_became_active":"2023-10-09T06:01:54.756872+0000","last_became_peered":"2023-10-09T06:01:54.756872+0000","last_unstale":"2023-10-09T06:03:41.445249+0000","last_undegraded":"2023-10-09T06:03:41.445249+0000","last_fullsized":"2023-10-09T06:03:41.445249+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:54.248077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"371'4491","reported_seq":4511,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-10-09T06:03:55.019129+0000","last_change":"2023-10-09T06:02:55.459796+0000","last_active":"2023-10-09T06:03:55.019129+0000","last_peered":"2023-10-09T06:03:55.019129+0000","last_clean":"2023-10-09T06:03:55.019129+0000","last_became_active":"2023-10-09T06:02:55.459408+0000","last_became_peered":"2023-10-09T06:02:55.459408+0000","last_unstale":"2023-10-09T06:03:55.019129+0000","last_undegraded":"2023-10-09T06:03:55.019129+0000","last_fullsized":"2023-10-09T06:03:55.019129+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4491,"log_dups_size":0,"ondisk_log_size":4491,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:54.450890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294322176,"num_objects":4491,"num_object_clones":0,"num_object_copies":13473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4491,"num_write_kb":287424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":546,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.423559+0000","last_change":"2023-10-09T06:02:16.361201+0000","last_active":"2023-10-09T06:03:41.423559+0000","last_peered":"2023-10-09T06:03:41.423559+0000","last_clean":"2023-10-09T06:03:41.423559+0000","last_became_active":"2023-10-09T06:01:22.337294+0000","last_became_peered":"2023-10-09T06:01:22.337294+0000","last_unstale":"2023-10-09T06:03:41.423559+0000","last_undegraded":"2023-10-09T06:03:41.423559+0000","last_fullsized":"2023-10-09T06:03:41.423559+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:16.361097+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:16.361097+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:16:25.401939+0000","scrub_duration":0.028533749000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":135,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.405701+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T06:03:41.405701+0000","last_peered":"2023-10-09T06:03:41.405701+0000","last_clean":"2023-10-09T06:03:41.405701+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T06:03:41.405701+0000","last_undegraded":"2023-10-09T06:03:41.405701+0000","last_fullsized":"2023-10-09T06:03:41.405701+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":408,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.427644+0000","last_change":"2023-10-09T06:01:54.753427+0000","last_active":"2023-10-09T06:03:41.427644+0000","last_peered":"2023-10-09T06:03:41.427644+0000","last_clean":"2023-10-09T06:03:41.427644+0000","last_became_active":"2023-10-09T06:01:54.753099+0000","last_became_peered":"2023-10-09T06:01:54.753099+0000","last_unstale":"2023-10-09T06:03:41.427644+0000","last_undegraded":"2023-10-09T06:03:41.427644+0000","last_fullsized":"2023-10-09T06:03:41.427644+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:07.504657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"371'4459","reported_seq":4478,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-10-09T06:03:55.143914+0000","last_change":"2023-10-09T06:02:55.482396+0000","last_active":"2023-10-09T06:03:55.143914+0000","last_peered":"2023-10-09T06:03:55.143914+0000","last_clean":"2023-10-09T06:03:55.143914+0000","last_became_active":"2023-10-09T06:02:55.482125+0000","last_became_peered":"2023-10-09T06:02:55.482125+0000","last_unstale":"2023-10-09T06:03:55.143914+0000","last_undegraded":"2023-10-09T06:03:55.143914+0000","last_fullsized":"2023-10-09T06:03:55.143914+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4459,"log_dups_size":0,"ondisk_log_size":4459,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:58.878190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292225024,"num_objects":4459,"num_object_clones":0,"num_object_copies":13377,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4459,"num_write_kb":285376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":814,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.426666+0000","last_change":"2023-10-09T05:59:55.438018+0000","last_active":"2023-10-09T06:03:41.426666+0000","last_peered":"2023-10-09T06:03:41.426666+0000","last_clean":"2023-10-09T06:03:41.426666+0000","last_became_active":"2023-10-09T05:59:32.538631+0000","last_became_peered":"2023-10-09T05:59:32.538631+0000","last_unstale":"2023-10-09T06:03:41.426666+0000","last_undegraded":"2023-10-09T06:03:41.426666+0000","last_fullsized":"2023-10-09T06:03:41.426666+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:55.437947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:55.437947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:37.566205+0000","scrub_duration":0.0091262979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":154,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.411201+0000","last_change":"2023-10-09T06:01:22.355503+0000","last_active":"2023-10-09T06:03:41.411201+0000","last_peered":"2023-10-09T06:03:41.411201+0000","last_clean":"2023-10-09T06:03:41.411201+0000","last_became_active":"2023-10-09T06:01:22.354704+0000","last_became_peered":"2023-10-09T06:01:22.354704+0000","last_unstale":"2023-10-09T06:03:41.411201+0000","last_undegraded":"2023-10-09T06:03:41.411201+0000","last_fullsized":"2023-10-09T06:03:41.411201+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:56:21.446402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":325,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.423192+0000","last_change":"2023-10-09T06:01:22.332033+0000","last_active":"2023-10-09T06:03:41.423192+0000","last_peered":"2023-10-09T06:03:41.423192+0000","last_clean":"2023-10-09T06:03:41.423192+0000","last_became_active":"2023-10-09T06:01:22.329825+0000","last_became_peered":"2023-10-09T06:01:22.329825+0000","last_unstale":"2023-10-09T06:03:41.423192+0000","last_undegraded":"2023-10-09T06:03:41.423192+0000","last_fullsized":"2023-10-09T06:03:41.423192+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:54.046681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"371'4675","reported_seq":4695,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-10-09T06:03:55.010949+0000","last_change":"2023-10-09T06:02:55.455129+0000","last_active":"2023-10-09T06:03:55.010949+0000","last_peered":"2023-10-09T06:03:55.010949+0000","last_clean":"2023-10-09T06:03:55.010949+0000","last_became_active":"2023-10-09T06:02:55.454836+0000","last_became_peered":"2023-10-09T06:02:55.454836+0000","last_unstale":"2023-10-09T06:03:55.010949+0000","last_undegraded":"2023-10-09T06:03:55.010949+0000","last_fullsized":"2023-10-09T06:03:55.010949+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4675,"log_dups_size":0,"ondisk_log_size":4675,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:29.116614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306380800,"num_objects":4675,"num_object_clones":0,"num_object_copies":14025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4675,"num_write_kb":299200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":561,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.404652+0000","last_change":"2023-10-09T05:59:58.282543+0000","last_active":"2023-10-09T06:03:41.404652+0000","last_peered":"2023-10-09T06:03:41.404652+0000","last_clean":"2023-10-09T06:03:41.404652+0000","last_became_active":"2023-10-09T05:59:32.552960+0000","last_became_peered":"2023-10-09T05:59:32.552960+0000","last_unstale":"2023-10-09T06:03:41.404652+0000","last_undegraded":"2023-10-09T06:03:41.404652+0000","last_fullsized":"2023-10-09T06:03:41.404652+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.282393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.282393+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:10.201323+0000","scrub_duration":0.01402403,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":408,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.444985+0000","last_change":"2023-10-09T06:01:54.751188+0000","last_active":"2023-10-09T06:03:41.444985+0000","last_peered":"2023-10-09T06:03:41.444985+0000","last_clean":"2023-10-09T06:03:41.444985+0000","last_became_active":"2023-10-09T06:01:54.750870+0000","last_became_peered":"2023-10-09T06:01:54.750870+0000","last_unstale":"2023-10-09T06:03:41.444985+0000","last_undegraded":"2023-10-09T06:03:41.444985+0000","last_fullsized":"2023-10-09T06:03:41.444985+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:01.985974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":399,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.423423+0000","last_change":"2023-10-09T06:01:54.757667+0000","last_active":"2023-10-09T06:03:41.423423+0000","last_peered":"2023-10-09T06:03:41.423423+0000","last_clean":"2023-10-09T06:03:41.423423+0000","last_became_active":"2023-10-09T06:01:54.757367+0000","last_became_peered":"2023-10-09T06:01:54.757367+0000","last_unstale":"2023-10-09T06:03:41.423423+0000","last_undegraded":"2023-10-09T06:03:41.423423+0000","last_fullsized":"2023-10-09T06:03:41.423423+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:45.022566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"371'4542","reported_seq":4561,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-10-09T06:03:55.414250+0000","last_change":"2023-10-09T06:02:55.484013+0000","last_active":"2023-10-09T06:03:55.414250+0000","last_peered":"2023-10-09T06:03:55.414250+0000","last_clean":"2023-10-09T06:03:55.414250+0000","last_became_active":"2023-10-09T06:02:55.483714+0000","last_became_peered":"2023-10-09T06:02:55.483714+0000","last_unstale":"2023-10-09T06:03:55.414250+0000","last_undegraded":"2023-10-09T06:03:55.414250+0000","last_fullsized":"2023-10-09T06:03:55.414250+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4542,"log_dups_size":0,"ondisk_log_size":4542,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:23:48.138776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297664512,"num_objects":4542,"num_object_clones":0,"num_object_copies":13626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4542,"num_write_kb":290688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":649,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.413971+0000","last_change":"2023-10-09T06:01:57.796523+0000","last_active":"2023-10-09T06:03:41.413971+0000","last_peered":"2023-10-09T06:03:41.413971+0000","last_clean":"2023-10-09T06:03:41.413971+0000","last_became_active":"2023-10-09T06:01:57.796145+0000","last_became_peered":"2023-10-09T06:01:57.796145+0000","last_unstale":"2023-10-09T06:03:41.413971+0000","last_undegraded":"2023-10-09T06:03:41.413971+0000","last_fullsized":"2023-10-09T06:03:41.413971+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.373748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.373748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:52.076986+0000","scrub_duration":0.0024576450000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":341,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.421018+0000","last_change":"2023-10-09T06:01:22.334879+0000","last_active":"2023-10-09T06:03:41.421018+0000","last_peered":"2023-10-09T06:03:41.421018+0000","last_clean":"2023-10-09T06:03:41.421018+0000","last_became_active":"2023-10-09T06:01:22.334609+0000","last_became_peered":"2023-10-09T06:01:22.334609+0000","last_unstale":"2023-10-09T06:03:41.421018+0000","last_undegraded":"2023-10-09T06:03:41.421018+0000","last_fullsized":"2023-10-09T06:03:41.421018+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:36.094645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":382,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.423526+0000","last_change":"2023-10-09T06:01:54.757024+0000","last_active":"2023-10-09T06:03:41.423526+0000","last_peered":"2023-10-09T06:03:41.423526+0000","last_clean":"2023-10-09T06:03:41.423526+0000","last_became_active":"2023-10-09T06:01:54.756713+0000","last_became_peered":"2023-10-09T06:01:54.756713+0000","last_unstale":"2023-10-09T06:03:41.423526+0000","last_undegraded":"2023-10-09T06:03:41.423526+0000","last_fullsized":"2023-10-09T06:03:41.423526+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:26.229374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"371'4660","reported_seq":4680,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-10-09T06:03:55.038782+0000","last_change":"2023-10-09T06:02:55.481570+0000","last_active":"2023-10-09T06:03:55.038782+0000","last_peered":"2023-10-09T06:03:55.038782+0000","last_clean":"2023-10-09T06:03:55.038782+0000","last_became_active":"2023-10-09T06:02:55.481243+0000","last_became_peered":"2023-10-09T06:02:55.481243+0000","last_unstale":"2023-10-09T06:03:55.038782+0000","last_undegraded":"2023-10-09T06:03:55.038782+0000","last_fullsized":"2023-10-09T06:03:55.038782+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4660,"log_dups_size":0,"ondisk_log_size":4660,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:35.076981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305397760,"num_objects":4660,"num_object_clones":0,"num_object_copies":13980,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4660,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4660,"num_write_kb":298240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":524,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.444702+0000","last_change":"2023-10-09T05:59:52.129883+0000","last_active":"2023-10-09T06:03:41.444702+0000","last_peered":"2023-10-09T06:03:41.444702+0000","last_clean":"2023-10-09T06:03:41.444702+0000","last_became_active":"2023-10-09T05:59:32.529083+0000","last_became_peered":"2023-10-09T05:59:32.529083+0000","last_unstale":"2023-10-09T06:03:41.444702+0000","last_undegraded":"2023-10-09T06:03:41.444702+0000","last_fullsized":"2023-10-09T06:03:41.444702+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:52.129816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:52.129816+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:38.309831+0000","scrub_duration":0.0039938459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":160,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-10-09T06:03:52.486353+0000","last_change":"2023-10-09T06:01:54.761275+0000","last_active":"2023-10-09T06:03:52.486353+0000","last_peered":"2023-10-09T06:03:52.486353+0000","last_clean":"2023-10-09T06:03:52.486353+0000","last_became_active":"2023-10-09T06:01:54.760782+0000","last_became_peered":"2023-10-09T06:01:54.760782+0000","last_unstale":"2023-10-09T06:03:52.486353+0000","last_undegraded":"2023-10-09T06:03:52.486353+0000","last_fullsized":"2023-10-09T06:03:52.486353+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:28.119846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.414658+0000","last_change":"2023-10-09T06:01:54.761768+0000","last_active":"2023-10-09T06:03:41.414658+0000","last_peered":"2023-10-09T06:03:41.414658+0000","last_clean":"2023-10-09T06:03:41.414658+0000","last_became_active":"2023-10-09T06:01:54.761443+0000","last_became_peered":"2023-10-09T06:01:54.761443+0000","last_unstale":"2023-10-09T06:03:41.414658+0000","last_undegraded":"2023-10-09T06:03:41.414658+0000","last_fullsized":"2023-10-09T06:03:41.414658+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:51.555987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"371'4601","reported_seq":4621,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-10-09T06:03:55.206184+0000","last_change":"2023-10-09T06:02:55.459982+0000","last_active":"2023-10-09T06:03:55.206184+0000","last_peered":"2023-10-09T06:03:55.206184+0000","last_clean":"2023-10-09T06:03:55.206184+0000","last_became_active":"2023-10-09T06:02:55.459651+0000","last_became_peered":"2023-10-09T06:02:55.459651+0000","last_unstale":"2023-10-09T06:03:55.206184+0000","last_undegraded":"2023-10-09T06:03:55.206184+0000","last_fullsized":"2023-10-09T06:03:55.206184+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4601,"log_dups_size":0,"ondisk_log_size":4601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:55.260004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301531136,"num_objects":4601,"num_object_clones":0,"num_object_copies":13803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4601,"num_write_kb":294464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":794,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.444925+0000","last_change":"2023-10-09T05:59:56.092654+0000","last_active":"2023-10-09T06:03:41.444925+0000","last_peered":"2023-10-09T06:03:41.444925+0000","last_clean":"2023-10-09T06:03:41.444925+0000","last_became_active":"2023-10-09T05:59:32.529949+0000","last_became_peered":"2023-10-09T05:59:32.529949+0000","last_unstale":"2023-10-09T06:03:41.444925+0000","last_undegraded":"2023-10-09T06:03:41.444925+0000","last_fullsized":"2023-10-09T06:03:41.444925+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.092586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.092586+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:15:13.234903+0000","scrub_duration":0.0013484219999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":48,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.414030+0000","last_change":"2023-10-09T06:01:54.823231+0000","last_active":"2023-10-09T06:03:41.414030+0000","last_peered":"2023-10-09T06:03:41.414030+0000","last_clean":"2023-10-09T06:03:41.414030+0000","last_became_active":"2023-10-09T06:01:54.822755+0000","last_became_peered":"2023-10-09T06:01:54.822755+0000","last_unstale":"2023-10-09T06:03:41.414030+0000","last_undegraded":"2023-10-09T06:03:41.414030+0000","last_fullsized":"2023-10-09T06:03:41.414030+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:17.592052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":136,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.404880+0000","last_change":"2023-10-09T05:57:23.468480+0000","last_active":"2023-10-09T06:03:41.404880+0000","last_peered":"2023-10-09T06:03:41.404880+0000","last_clean":"2023-10-09T06:03:41.404880+0000","last_became_active":"2023-10-09T05:57:23.468179+0000","last_became_peered":"2023-10-09T05:57:23.468179+0000","last_unstale":"2023-10-09T06:03:41.404880+0000","last_undegraded":"2023-10-09T06:03:41.404880+0000","last_fullsized":"2023-10-09T06:03:41.404880+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:18.157188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":48,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.414510+0000","last_change":"2023-10-09T06:01:54.751385+0000","last_active":"2023-10-09T06:03:41.414510+0000","last_peered":"2023-10-09T06:03:41.414510+0000","last_clean":"2023-10-09T06:03:41.414510+0000","last_became_active":"2023-10-09T06:01:54.751041+0000","last_became_peered":"2023-10-09T06:01:54.751041+0000","last_unstale":"2023-10-09T06:03:41.414510+0000","last_undegraded":"2023-10-09T06:03:41.414510+0000","last_fullsized":"2023-10-09T06:03:41.414510+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:03.424836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"371'4555","reported_seq":4575,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-10-09T06:03:55.160397+0000","last_change":"2023-10-09T06:02:55.481800+0000","last_active":"2023-10-09T06:03:55.160397+0000","last_peered":"2023-10-09T06:03:55.160397+0000","last_clean":"2023-10-09T06:03:55.160397+0000","last_became_active":"2023-10-09T06:02:55.481441+0000","last_became_peered":"2023-10-09T06:02:55.481441+0000","last_unstale":"2023-10-09T06:03:55.160397+0000","last_undegraded":"2023-10-09T06:03:55.160397+0000","last_fullsized":"2023-10-09T06:03:55.160397+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4555,"log_dups_size":0,"ondisk_log_size":4555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:37.737097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298516480,"num_objects":4555,"num_object_clones":0,"num_object_copies":13665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4555,"num_write_kb":291520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":576,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.427540+0000","last_change":"2023-10-09T05:58:36.225081+0000","last_active":"2023-10-09T06:03:41.427540+0000","last_peered":"2023-10-09T06:03:41.427540+0000","last_clean":"2023-10-09T06:03:41.427540+0000","last_became_active":"2023-10-09T05:56:49.237269+0000","last_became_peered":"2023-10-09T05:56:49.237269+0000","last_unstale":"2023-10-09T06:03:41.427540+0000","last_undegraded":"2023-10-09T06:03:41.427540+0000","last_fullsized":"2023-10-09T06:03:41.427540+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:02.908583+0000","scrub_duration":0.0016660799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":48,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.414605+0000","last_change":"2023-10-09T06:01:54.754294+0000","last_active":"2023-10-09T06:03:41.414605+0000","last_peered":"2023-10-09T06:03:41.414605+0000","last_clean":"2023-10-09T06:03:41.414605+0000","last_became_active":"2023-10-09T06:01:54.754024+0000","last_became_peered":"2023-10-09T06:01:54.754024+0000","last_unstale":"2023-10-09T06:03:41.414605+0000","last_undegraded":"2023-10-09T06:03:41.414605+0000","last_fullsized":"2023-10-09T06:03:41.414605+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:19.746882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":334,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.421204+0000","last_change":"2023-10-09T06:01:22.335651+0000","last_active":"2023-10-09T06:03:41.421204+0000","last_peered":"2023-10-09T06:03:41.421204+0000","last_clean":"2023-10-09T06:03:41.421204+0000","last_became_active":"2023-10-09T06:01:22.334756+0000","last_became_peered":"2023-10-09T06:01:22.334756+0000","last_unstale":"2023-10-09T06:03:41.421204+0000","last_undegraded":"2023-10-09T06:03:41.421204+0000","last_fullsized":"2023-10-09T06:03:41.421204+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:33.123496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":48,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.414119+0000","last_change":"2023-10-09T06:01:54.760832+0000","last_active":"2023-10-09T06:03:41.414119+0000","last_peered":"2023-10-09T06:03:41.414119+0000","last_clean":"2023-10-09T06:03:41.414119+0000","last_became_active":"2023-10-09T06:01:54.760137+0000","last_became_peered":"2023-10-09T06:01:54.760137+0000","last_unstale":"2023-10-09T06:03:41.414119+0000","last_undegraded":"2023-10-09T06:03:41.414119+0000","last_fullsized":"2023-10-09T06:03:41.414119+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:19:58.776604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"371'4574","reported_seq":4594,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-10-09T06:03:55.170141+0000","last_change":"2023-10-09T06:02:55.454431+0000","last_active":"2023-10-09T06:03:55.170141+0000","last_peered":"2023-10-09T06:03:55.170141+0000","last_clean":"2023-10-09T06:03:55.170141+0000","last_became_active":"2023-10-09T06:02:55.454117+0000","last_became_peered":"2023-10-09T06:02:55.454117+0000","last_unstale":"2023-10-09T06:03:55.170141+0000","last_undegraded":"2023-10-09T06:03:55.170141+0000","last_fullsized":"2023-10-09T06:03:55.170141+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4574,"log_dups_size":0,"ondisk_log_size":4574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:02.450452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299761664,"num_objects":4574,"num_object_clones":0,"num_object_copies":13722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4574,"num_write_kb":292736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":695,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.426680+0000","last_change":"2023-10-09T06:01:22.335906+0000","last_active":"2023-10-09T06:03:41.426680+0000","last_peered":"2023-10-09T06:03:41.426680+0000","last_clean":"2023-10-09T06:03:41.426680+0000","last_became_active":"2023-10-09T06:01:22.335590+0000","last_became_peered":"2023-10-09T06:01:22.335590+0000","last_unstale":"2023-10-09T06:03:41.426680+0000","last_undegraded":"2023-10-09T06:03:41.426680+0000","last_fullsized":"2023-10-09T06:03:41.426680+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:55.897812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:55.897812+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:02.771428+0000","scrub_duration":0.0014511420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":113,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.411137+0000","last_change":"2023-10-09T05:58:59.325257+0000","last_active":"2023-10-09T06:03:41.411137+0000","last_peered":"2023-10-09T06:03:41.411137+0000","last_clean":"2023-10-09T06:03:41.411137+0000","last_became_active":"2023-10-09T05:58:59.324993+0000","last_became_peered":"2023-10-09T05:58:59.324993+0000","last_unstale":"2023-10-09T06:03:41.411137+0000","last_undegraded":"2023-10-09T06:03:41.411137+0000","last_fullsized":"2023-10-09T06:03:41.411137+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:20.825840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":49,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-10-09T06:03:52.465480+0000","last_change":"2023-10-09T06:01:54.755425+0000","last_active":"2023-10-09T06:03:52.465480+0000","last_peered":"2023-10-09T06:03:52.465480+0000","last_clean":"2023-10-09T06:03:52.465480+0000","last_became_active":"2023-10-09T06:01:54.755129+0000","last_became_peered":"2023-10-09T06:01:54.755129+0000","last_unstale":"2023-10-09T06:03:52.465480+0000","last_undegraded":"2023-10-09T06:03:52.465480+0000","last_fullsized":"2023-10-09T06:03:52.465480+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:01:58.037849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":48,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.414643+0000","last_change":"2023-10-09T06:01:54.751198+0000","last_active":"2023-10-09T06:03:41.414643+0000","last_peered":"2023-10-09T06:03:41.414643+0000","last_clean":"2023-10-09T06:03:41.414643+0000","last_became_active":"2023-10-09T06:01:54.750862+0000","last_became_peered":"2023-10-09T06:01:54.750862+0000","last_unstale":"2023-10-09T06:03:41.414643+0000","last_undegraded":"2023-10-09T06:03:41.414643+0000","last_fullsized":"2023-10-09T06:03:41.414643+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:35.203567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"371'4495","reported_seq":4514,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-10-09T06:03:53.626850+0000","last_change":"2023-10-09T06:02:55.481788+0000","last_active":"2023-10-09T06:03:53.626850+0000","last_peered":"2023-10-09T06:03:53.626850+0000","last_clean":"2023-10-09T06:03:53.626850+0000","last_became_active":"2023-10-09T06:02:55.481460+0000","last_became_peered":"2023-10-09T06:02:55.481460+0000","last_unstale":"2023-10-09T06:03:53.626850+0000","last_undegraded":"2023-10-09T06:03:53.626850+0000","last_fullsized":"2023-10-09T06:03:53.626850+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4495,"log_dups_size":0,"ondisk_log_size":4495,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:31.125821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294584320,"num_objects":4495,"num_object_clones":0,"num_object_copies":13485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4495,"num_write_kb":287680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":242,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.426680+0000","last_change":"2023-10-09T05:58:59.330109+0000","last_active":"2023-10-09T06:03:41.426680+0000","last_peered":"2023-10-09T06:03:41.426680+0000","last_clean":"2023-10-09T06:03:41.426680+0000","last_became_active":"2023-10-09T05:58:59.329808+0000","last_became_peered":"2023-10-09T05:58:59.329808+0000","last_unstale":"2023-10-09T06:03:41.426680+0000","last_undegraded":"2023-10-09T06:03:41.426680+0000","last_fullsized":"2023-10-09T06:03:41.426680+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:30.680622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":775,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.428311+0000","last_change":"2023-10-09T05:58:37.782237+0000","last_active":"2023-10-09T06:03:41.428311+0000","last_peered":"2023-10-09T06:03:41.428311+0000","last_clean":"2023-10-09T06:03:41.428311+0000","last_became_active":"2023-10-09T05:53:59.202004+0000","last_became_peered":"2023-10-09T05:53:59.202004+0000","last_unstale":"2023-10-09T06:03:41.428311+0000","last_undegraded":"2023-10-09T06:03:41.428311+0000","last_fullsized":"2023-10-09T06:03:41.428311+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:37.782173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:37.782173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:20.787963+0000","scrub_duration":0.001325908,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":136,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.404721+0000","last_change":"2023-10-09T05:57:23.479174+0000","last_active":"2023-10-09T06:03:41.404721+0000","last_peered":"2023-10-09T06:03:41.404721+0000","last_clean":"2023-10-09T06:03:41.404721+0000","last_became_active":"2023-10-09T05:57:23.478897+0000","last_became_peered":"2023-10-09T05:57:23.478897+0000","last_unstale":"2023-10-09T06:03:41.404721+0000","last_undegraded":"2023-10-09T06:03:41.404721+0000","last_fullsized":"2023-10-09T06:03:41.404721+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:00.160042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":350,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.445014+0000","last_change":"2023-10-09T06:01:22.340980+0000","last_active":"2023-10-09T06:03:41.445014+0000","last_peered":"2023-10-09T06:03:41.445014+0000","last_clean":"2023-10-09T06:03:41.445014+0000","last_became_active":"2023-10-09T06:01:22.340690+0000","last_became_peered":"2023-10-09T06:01:22.340690+0000","last_unstale":"2023-10-09T06:03:41.445014+0000","last_undegraded":"2023-10-09T06:03:41.445014+0000","last_fullsized":"2023-10-09T06:03:41.445014+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:45.161101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"371'4715","reported_seq":4735,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-10-09T06:03:55.025948+0000","last_change":"2023-10-09T06:02:55.481658+0000","last_active":"2023-10-09T06:03:55.025948+0000","last_peered":"2023-10-09T06:03:55.025948+0000","last_clean":"2023-10-09T06:03:55.025948+0000","last_became_active":"2023-10-09T06:02:55.481365+0000","last_became_peered":"2023-10-09T06:02:55.481365+0000","last_unstale":"2023-10-09T06:03:55.025948+0000","last_undegraded":"2023-10-09T06:03:55.025948+0000","last_fullsized":"2023-10-09T06:03:55.025948+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4715,"log_dups_size":0,"ondisk_log_size":4715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:59.430808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309002240,"num_objects":4715,"num_object_clones":0,"num_object_copies":14145,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4715,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4715,"num_write_kb":301760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":564,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.414333+0000","last_change":"2023-10-09T06:02:45.280115+0000","last_active":"2023-10-09T06:03:41.414333+0000","last_peered":"2023-10-09T06:03:41.414333+0000","last_clean":"2023-10-09T06:03:41.414333+0000","last_became_active":"2023-10-09T06:01:57.803970+0000","last_became_peered":"2023-10-09T06:01:57.803970+0000","last_unstale":"2023-10-09T06:03:41.414333+0000","last_undegraded":"2023-10-09T06:03:41.414333+0000","last_fullsized":"2023-10-09T06:03:41.414333+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:45.280021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:45.280021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:30.141864+0000","scrub_duration":0.001913166,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":247,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.426684+0000","last_change":"2023-10-09T06:01:54.751146+0000","last_active":"2023-10-09T06:03:41.426684+0000","last_peered":"2023-10-09T06:03:41.426684+0000","last_clean":"2023-10-09T06:03:41.426684+0000","last_became_active":"2023-10-09T06:01:54.750833+0000","last_became_peered":"2023-10-09T06:01:54.750833+0000","last_unstale":"2023-10-09T06:03:41.426684+0000","last_undegraded":"2023-10-09T06:03:41.426684+0000","last_fullsized":"2023-10-09T06:03:41.426684+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:54.035203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":325,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.427735+0000","last_change":"2023-10-09T06:01:22.320317+0000","last_active":"2023-10-09T06:03:41.427735+0000","last_peered":"2023-10-09T06:03:41.427735+0000","last_clean":"2023-10-09T06:03:41.427735+0000","last_became_active":"2023-10-09T06:01:22.319989+0000","last_became_peered":"2023-10-09T06:01:22.319989+0000","last_unstale":"2023-10-09T06:03:41.427735+0000","last_undegraded":"2023-10-09T06:03:41.427735+0000","last_fullsized":"2023-10-09T06:03:41.427735+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:06.219647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":436,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.428364+0000","last_change":"2023-10-09T06:01:54.752942+0000","last_active":"2023-10-09T06:03:41.428364+0000","last_peered":"2023-10-09T06:03:41.428364+0000","last_clean":"2023-10-09T06:03:41.428364+0000","last_became_active":"2023-10-09T06:01:54.752593+0000","last_became_peered":"2023-10-09T06:01:54.752593+0000","last_unstale":"2023-10-09T06:03:41.428364+0000","last_undegraded":"2023-10-09T06:03:41.428364+0000","last_fullsized":"2023-10-09T06:03:41.428364+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:10.005606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"371'4690","reported_seq":4710,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-10-09T06:03:55.178354+0000","last_change":"2023-10-09T06:02:55.454444+0000","last_active":"2023-10-09T06:03:55.178354+0000","last_peered":"2023-10-09T06:03:55.178354+0000","last_clean":"2023-10-09T06:03:55.178354+0000","last_became_active":"2023-10-09T06:02:55.454144+0000","last_became_peered":"2023-10-09T06:02:55.454144+0000","last_unstale":"2023-10-09T06:03:55.178354+0000","last_undegraded":"2023-10-09T06:03:55.178354+0000","last_fullsized":"2023-10-09T06:03:55.178354+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4690,"log_dups_size":0,"ondisk_log_size":4690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:45:20.421821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307363840,"num_objects":4690,"num_object_clones":0,"num_object_copies":14070,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4690,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4690,"num_write_kb":300160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":784,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.445263+0000","last_change":"2023-10-09T05:59:54.074413+0000","last_active":"2023-10-09T06:03:41.445263+0000","last_peered":"2023-10-09T06:03:41.445263+0000","last_clean":"2023-10-09T06:03:41.445263+0000","last_became_active":"2023-10-09T05:59:32.538959+0000","last_became_peered":"2023-10-09T05:59:32.538959+0000","last_unstale":"2023-10-09T06:03:41.445263+0000","last_undegraded":"2023-10-09T06:03:41.445263+0000","last_fullsized":"2023-10-09T06:03:41.445263+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:54.074310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:54.074310+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:10.811269+0000","scrub_duration":0.011040648,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":222,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.427099+0000","last_change":"2023-10-09T06:01:22.337610+0000","last_active":"2023-10-09T06:03:41.427099+0000","last_peered":"2023-10-09T06:03:41.427099+0000","last_clean":"2023-10-09T06:03:41.427099+0000","last_became_active":"2023-10-09T06:01:22.336719+0000","last_became_peered":"2023-10-09T06:01:22.336719+0000","last_unstale":"2023-10-09T06:03:41.427099+0000","last_undegraded":"2023-10-09T06:03:41.427099+0000","last_fullsized":"2023-10-09T06:03:41.427099+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:33:29.587947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.414184+0000","last_change":"2023-10-09T06:01:54.756606+0000","last_active":"2023-10-09T06:03:41.414184+0000","last_peered":"2023-10-09T06:03:41.414184+0000","last_clean":"2023-10-09T06:03:41.414184+0000","last_became_active":"2023-10-09T06:01:54.755869+0000","last_became_peered":"2023-10-09T06:01:54.755869+0000","last_unstale":"2023-10-09T06:03:41.414184+0000","last_undegraded":"2023-10-09T06:03:41.414184+0000","last_fullsized":"2023-10-09T06:03:41.414184+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:29.672095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":143,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.426805+0000","last_change":"2023-10-09T06:01:41.938541+0000","last_active":"2023-10-09T06:03:41.426805+0000","last_peered":"2023-10-09T06:03:41.426805+0000","last_clean":"2023-10-09T06:03:41.426805+0000","last_became_active":"2023-10-09T06:01:41.938289+0000","last_became_peered":"2023-10-09T06:01:41.938289+0000","last_unstale":"2023-10-09T06:03:41.426805+0000","last_undegraded":"2023-10-09T06:03:41.426805+0000","last_fullsized":"2023-10-09T06:03:41.426805+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:29.735530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.c","version":"371'4528","reported_seq":4546,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-10-09T06:03:55.221084+0000","last_change":"2023-10-09T06:02:55.458266+0000","last_active":"2023-10-09T06:03:55.221084+0000","last_peered":"2023-10-09T06:03:55.221084+0000","last_clean":"2023-10-09T06:03:55.221084+0000","last_became_active":"2023-10-09T06:02:55.457936+0000","last_became_peered":"2023-10-09T06:02:55.457936+0000","last_unstale":"2023-10-09T06:03:55.221084+0000","last_undegraded":"2023-10-09T06:03:55.221084+0000","last_fullsized":"2023-10-09T06:03:55.221084+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4528,"log_dups_size":0,"ondisk_log_size":4528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:44:16.198311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296747008,"num_objects":4528,"num_object_clones":0,"num_object_copies":13584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4528,"num_write_kb":289792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":764,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.423541+0000","last_change":"2023-10-09T06:02:17.344619+0000","last_active":"2023-10-09T06:03:41.423541+0000","last_peered":"2023-10-09T06:03:41.423541+0000","last_clean":"2023-10-09T06:03:41.423541+0000","last_became_active":"2023-10-09T06:01:22.327941+0000","last_became_peered":"2023-10-09T06:01:22.327941+0000","last_unstale":"2023-10-09T06:03:41.423541+0000","last_undegraded":"2023-10-09T06:03:41.423541+0000","last_fullsized":"2023-10-09T06:03:41.423541+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:17.344518+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:17.344518+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:26.950368+0000","scrub_duration":0.0015914099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":379,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.405803+0000","last_change":"2023-10-09T05:53:59.228948+0000","last_active":"2023-10-09T06:03:41.405803+0000","last_peered":"2023-10-09T06:03:41.405803+0000","last_clean":"2023-10-09T06:03:41.405803+0000","last_became_active":"2023-10-09T05:53:59.186359+0000","last_became_peered":"2023-10-09T05:53:59.186359+0000","last_unstale":"2023-10-09T06:03:41.405803+0000","last_undegraded":"2023-10-09T06:03:41.405803+0000","last_fullsized":"2023-10-09T06:03:41.405803+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:39.990576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":48,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.414170+0000","last_change":"2023-10-09T06:01:54.753800+0000","last_active":"2023-10-09T06:03:41.414170+0000","last_peered":"2023-10-09T06:03:41.414170+0000","last_clean":"2023-10-09T06:03:41.414170+0000","last_became_active":"2023-10-09T06:01:54.751920+0000","last_became_peered":"2023-10-09T06:01:54.751920+0000","last_unstale":"2023-10-09T06:03:41.414170+0000","last_undegraded":"2023-10-09T06:03:41.414170+0000","last_fullsized":"2023-10-09T06:03:41.414170+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:45.359801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":136,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.404949+0000","last_change":"2023-10-09T05:57:23.477143+0000","last_active":"2023-10-09T06:03:41.404949+0000","last_peered":"2023-10-09T06:03:41.404949+0000","last_clean":"2023-10-09T06:03:41.404949+0000","last_became_active":"2023-10-09T05:57:23.475655+0000","last_became_peered":"2023-10-09T05:57:23.475655+0000","last_unstale":"2023-10-09T06:03:41.404949+0000","last_undegraded":"2023-10-09T06:03:41.404949+0000","last_fullsized":"2023-10-09T06:03:41.404949+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:29.528023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"371'4528","reported_seq":4547,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-10-09T06:03:55.932632+0000","last_change":"2023-10-09T06:02:55.481513+0000","last_active":"2023-10-09T06:03:55.932632+0000","last_peered":"2023-10-09T06:03:55.932632+0000","last_clean":"2023-10-09T06:03:55.932632+0000","last_became_active":"2023-10-09T06:02:55.481226+0000","last_became_peered":"2023-10-09T06:02:55.481226+0000","last_unstale":"2023-10-09T06:03:55.932632+0000","last_undegraded":"2023-10-09T06:03:55.932632+0000","last_fullsized":"2023-10-09T06:03:55.932632+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":360,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:02:54.432934+0000","last_clean_scrub_stamp":"2023-10-09T06:02:54.432934+0000","objects_scrubbed":0,"log_size":4528,"log_dups_size":0,"ondisk_log_size":4528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:12.967950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296747008,"num_objects":4528,"num_object_clones":0,"num_object_copies":13584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4528,"num_write_kb":289792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":630,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:45.158599+0000","last_change":"2023-10-09T05:58:34.333129+0000","last_active":"2023-10-09T06:03:45.158599+0000","last_peered":"2023-10-09T06:03:45.158599+0000","last_clean":"2023-10-09T06:03:45.158599+0000","last_became_active":"2023-10-09T05:57:26.502911+0000","last_became_peered":"2023-10-09T05:57:26.502911+0000","last_unstale":"2023-10-09T06:03:45.158599+0000","last_undegraded":"2023-10-09T06:03:45.158599+0000","last_fullsized":"2023-10-09T06:03:45.158599+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:25.230712+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":375,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.445045+0000","last_change":"2023-10-09T05:57:58.832790+0000","last_active":"2023-10-09T06:03:41.445045+0000","last_peered":"2023-10-09T06:03:41.445045+0000","last_clean":"2023-10-09T06:03:41.445045+0000","last_became_active":"2023-10-09T05:57:58.832471+0000","last_became_peered":"2023-10-09T05:57:58.832471+0000","last_unstale":"2023-10-09T06:03:41.445045+0000","last_undegraded":"2023-10-09T06:03:41.445045+0000","last_fullsized":"2023-10-09T06:03:41.445045+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:39.358174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":441,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.428556+0000","last_change":"2023-10-09T06:01:54.776025+0000","last_active":"2023-10-09T06:03:41.428556+0000","last_peered":"2023-10-09T06:03:41.428556+0000","last_clean":"2023-10-09T06:03:41.428556+0000","last_became_active":"2023-10-09T06:01:54.753050+0000","last_became_peered":"2023-10-09T06:01:54.753050+0000","last_unstale":"2023-10-09T06:03:41.428556+0000","last_undegraded":"2023-10-09T06:03:41.428556+0000","last_fullsized":"2023-10-09T06:03:41.428556+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:58:24.000420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":317,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.421044+0000","last_change":"2023-10-09T06:01:22.334081+0000","last_active":"2023-10-09T06:03:41.421044+0000","last_peered":"2023-10-09T06:03:41.421044+0000","last_clean":"2023-10-09T06:03:41.421044+0000","last_became_active":"2023-10-09T06:01:22.333663+0000","last_became_peered":"2023-10-09T06:01:22.333663+0000","last_unstale":"2023-10-09T06:03:41.421044+0000","last_undegraded":"2023-10-09T06:03:41.421044+0000","last_fullsized":"2023-10-09T06:03:41.421044+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:55.809259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":597,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.405904+0000","last_change":"2023-10-09T06:00:06.181968+0000","last_active":"2023-10-09T06:03:41.405904+0000","last_peered":"2023-10-09T06:03:41.405904+0000","last_clean":"2023-10-09T06:03:41.405904+0000","last_became_active":"2023-10-09T05:59:40.324653+0000","last_became_peered":"2023-10-09T05:59:40.324653+0000","last_unstale":"2023-10-09T06:03:41.405904+0000","last_undegraded":"2023-10-09T06:03:41.405904+0000","last_fullsized":"2023-10-09T06:03:41.405904+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.181811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.181811+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:05:16.987523+0000","scrub_duration":0.011553313000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":317,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.428316+0000","last_change":"2023-10-09T06:01:22.349718+0000","last_active":"2023-10-09T06:03:41.428316+0000","last_peered":"2023-10-09T06:03:41.428316+0000","last_clean":"2023-10-09T06:03:41.428316+0000","last_became_active":"2023-10-09T06:01:22.349401+0000","last_became_peered":"2023-10-09T06:01:22.349401+0000","last_unstale":"2023-10-09T06:03:41.428316+0000","last_undegraded":"2023-10-09T06:03:41.428316+0000","last_fullsized":"2023-10-09T06:03:41.428316+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:08.983715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":48,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.414198+0000","last_change":"2023-10-09T06:01:54.761007+0000","last_active":"2023-10-09T06:03:41.414198+0000","last_peered":"2023-10-09T06:03:41.414198+0000","last_clean":"2023-10-09T06:03:41.414198+0000","last_became_active":"2023-10-09T06:01:54.760538+0000","last_became_peered":"2023-10-09T06:01:54.760538+0000","last_unstale":"2023-10-09T06:03:41.414198+0000","last_undegraded":"2023-10-09T06:03:41.414198+0000","last_fullsized":"2023-10-09T06:03:41.414198+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:08.772179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":715,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.405586+0000","last_change":"2023-10-09T06:00:04.215425+0000","last_active":"2023-10-09T06:03:41.405586+0000","last_peered":"2023-10-09T06:03:41.405586+0000","last_clean":"2023-10-09T06:03:41.405586+0000","last_became_active":"2023-10-09T05:59:40.326685+0000","last_became_peered":"2023-10-09T05:59:40.326685+0000","last_unstale":"2023-10-09T06:03:41.405586+0000","last_undegraded":"2023-10-09T06:03:41.405586+0000","last_fullsized":"2023-10-09T06:03:41.405586+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:04.215270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:04.215270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:03.963694+0000","scrub_duration":0.002850652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":142,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.423214+0000","last_change":"2023-10-09T06:01:54.824780+0000","last_active":"2023-10-09T06:03:41.423214+0000","last_peered":"2023-10-09T06:03:41.423214+0000","last_clean":"2023-10-09T06:03:41.423214+0000","last_became_active":"2023-10-09T06:01:54.757727+0000","last_became_peered":"2023-10-09T06:01:54.757727+0000","last_unstale":"2023-10-09T06:03:41.423214+0000","last_undegraded":"2023-10-09T06:03:41.423214+0000","last_fullsized":"2023-10-09T06:03:41.423214+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:44.132556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":389,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.421375+0000","last_change":"2023-10-09T06:01:54.764989+0000","last_active":"2023-10-09T06:03:41.421375+0000","last_peered":"2023-10-09T06:03:41.421375+0000","last_clean":"2023-10-09T06:03:41.421375+0000","last_became_active":"2023-10-09T06:01:54.764790+0000","last_became_peered":"2023-10-09T06:01:54.764790+0000","last_unstale":"2023-10-09T06:03:41.421375+0000","last_undegraded":"2023-10-09T06:03:41.421375+0000","last_fullsized":"2023-10-09T06:03:41.421375+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:14:07.208858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":128,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.423295+0000","last_change":"2023-10-09T06:01:22.342190+0000","last_active":"2023-10-09T06:03:41.423295+0000","last_peered":"2023-10-09T06:03:41.423295+0000","last_clean":"2023-10-09T06:03:41.423295+0000","last_became_active":"2023-10-09T06:01:22.341855+0000","last_became_peered":"2023-10-09T06:01:22.341855+0000","last_unstale":"2023-10-09T06:03:41.423295+0000","last_undegraded":"2023-10-09T06:03:41.423295+0000","last_fullsized":"2023-10-09T06:03:41.423295+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:01:21.322371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":359,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.421412+0000","last_change":"2023-10-09T06:01:22.326786+0000","last_active":"2023-10-09T06:03:41.421412+0000","last_peered":"2023-10-09T06:03:41.421412+0000","last_clean":"2023-10-09T06:03:41.421412+0000","last_became_active":"2023-10-09T06:01:22.326474+0000","last_became_peered":"2023-10-09T06:01:22.326474+0000","last_unstale":"2023-10-09T06:03:41.421412+0000","last_undegraded":"2023-10-09T06:03:41.421412+0000","last_fullsized":"2023-10-09T06:03:41.421412+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:17:48.846619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":637,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.426869+0000","last_change":"2023-10-09T06:01:57.801991+0000","last_active":"2023-10-09T06:03:41.426869+0000","last_peered":"2023-10-09T06:03:41.426869+0000","last_clean":"2023-10-09T06:03:41.426869+0000","last_became_active":"2023-10-09T06:01:57.801579+0000","last_became_peered":"2023-10-09T06:01:57.801579+0000","last_unstale":"2023-10-09T06:03:41.426869+0000","last_undegraded":"2023-10-09T06:03:41.426869+0000","last_fullsized":"2023-10-09T06:03:41.426869+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:00.456132+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:00.456132+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:48.548353+0000","scrub_duration":0.0014992199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":136,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.411007+0000","last_change":"2023-10-09T05:57:23.479560+0000","last_active":"2023-10-09T06:03:41.411007+0000","last_peered":"2023-10-09T06:03:41.411007+0000","last_clean":"2023-10-09T06:03:41.411007+0000","last_became_active":"2023-10-09T05:57:23.479177+0000","last_became_peered":"2023-10-09T05:57:23.479177+0000","last_unstale":"2023-10-09T06:03:41.411007+0000","last_undegraded":"2023-10-09T06:03:41.411007+0000","last_fullsized":"2023-10-09T06:03:41.411007+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:41.925161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":135,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.411221+0000","last_change":"2023-10-09T05:57:23.475179+0000","last_active":"2023-10-09T06:03:41.411221+0000","last_peered":"2023-10-09T06:03:41.411221+0000","last_clean":"2023-10-09T06:03:41.411221+0000","last_became_active":"2023-10-09T05:57:23.474904+0000","last_became_peered":"2023-10-09T05:57:23.474904+0000","last_unstale":"2023-10-09T06:03:41.411221+0000","last_undegraded":"2023-10-09T06:03:41.411221+0000","last_fullsized":"2023-10-09T06:03:41.411221+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:54.587601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":654,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.418517+0000","last_change":"2023-10-09T06:02:46.290692+0000","last_active":"2023-10-09T06:03:41.418517+0000","last_peered":"2023-10-09T06:03:41.418517+0000","last_clean":"2023-10-09T06:03:41.418517+0000","last_became_active":"2023-10-09T06:01:57.817191+0000","last_became_peered":"2023-10-09T06:01:57.817191+0000","last_unstale":"2023-10-09T06:03:41.418517+0000","last_undegraded":"2023-10-09T06:03:41.418517+0000","last_fullsized":"2023-10-09T06:03:41.418517+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:46.290510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:46.290510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:09.771203+0000","scrub_duration":0.002627819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":340,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.427943+0000","last_change":"2023-10-09T05:57:10.379258+0000","last_active":"2023-10-09T06:03:41.427943+0000","last_peered":"2023-10-09T06:03:41.427943+0000","last_clean":"2023-10-09T06:03:41.427943+0000","last_became_active":"2023-10-09T05:57:10.359745+0000","last_became_peered":"2023-10-09T05:57:10.359745+0000","last_unstale":"2023-10-09T06:03:41.427943+0000","last_undegraded":"2023-10-09T06:03:41.427943+0000","last_fullsized":"2023-10-09T06:03:41.427943+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:14.750256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":382,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-10-09T06:03:41.420708+0000","last_change":"2023-10-09T06:01:54.757112+0000","last_active":"2023-10-09T06:03:41.420708+0000","last_peered":"2023-10-09T06:03:41.420708+0000","last_clean":"2023-10-09T06:03:41.420708+0000","last_became_active":"2023-10-09T06:01:54.756793+0000","last_became_peered":"2023-10-09T06:01:54.756793+0000","last_unstale":"2023-10-09T06:03:41.420708+0000","last_undegraded":"2023-10-09T06:03:41.420708+0000","last_fullsized":"2023-10-09T06:03:41.420708+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:31.657029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":1324,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10938,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4797431808,"num_objects":73203,"num_object_clones":0,"num_object_copies":219609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":73203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":73203,"num_write_kb":4684992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14328266752,"data_stored":14328266752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":73203,"ondisk_log_size":73203,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234291,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1776464,"kb_used_data":1466888,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":91968688,"statfs":{"total":95995035648,"available":94175936512,"internally_reserved":0,"allocated":1502093312,"data_stored":1500958204,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:03:07 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":5.5570000000000004,"15min":2.9340000000000002},"min":{"1min":0.25600000000000001,"5min":0.22500000000000001,"15min":0.22},"max":{"1min":4.3410000000000002,"5min":165.792,"15min":165.792},"last":4.9109999999999996},{"interface":"front","average":{"1min":1.147,"5min":5.7679999999999998,"15min":2.9500000000000002},"min":{"1min":0.35599999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":4.5670000000000002,"5min":166.00700000000001,"15min":166.00700000000001},"last":0.40799999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:03:43 2023","interfaces":[{"interface":"back","average":{"1min":2.968,"5min":1.6399999999999999,"15min":1.331},"min":{"1min":0.25,"5min":0.249,"15min":0.22900000000000001},"max":{"1min":38.118000000000002,"5min":38.118000000000002,"15min":38.118000000000002},"last":2.0110000000000001},{"interface":"front","average":{"1min":2.4940000000000002,"5min":1.484,"15min":1.1739999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":36.478999999999999,"5min":36.478999999999999,"15min":36.478999999999999},"last":4.6360000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:03:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":3.1190000000000002,"15min":2.887},"min":{"1min":0.309,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.1080000000000001,"5min":127.86499999999999,"15min":127.86499999999999},"last":1.879},{"interface":"front","average":{"1min":1.2150000000000001,"5min":2.7850000000000001,"15min":2.8410000000000002},"min":{"1min":0.43099999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":7.2240000000000002,"5min":129.24000000000001,"15min":129.24000000000001},"last":5.0090000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:03:43 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":8.8309999999999995,"15min":5.6859999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":2.827,"5min":164.16999999999999,"15min":164.16999999999999},"last":1.9930000000000001},{"interface":"front","average":{"1min":1.2250000000000001,"5min":8.9550000000000001,"15min":5.6360000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.30299999999999999},"max":{"1min":3.5619999999999998,"5min":164.40700000000001,"15min":164.40700000000001},"last":4.8760000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:03:22 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.753,"15min":0.78700000000000003},"min":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.671,"5min":3.456,"15min":3.456},"last":1.9450000000000001},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.78800000000000003,"15min":0.83199999999999996},"min":{"1min":0.35199999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.98299999999999998,"5min":3.5939999999999999,"15min":3.5939999999999999},"last":4.593}]},{"osd":5,"last update":"Mon Oct 9 06:03:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.748,"15min":0.84799999999999998},"min":{"1min":0.309,"5min":0.27600000000000002,"15min":0.27500000000000002},"max":{"1min":2.7930000000000001,"5min":3.5230000000000001,"15min":15.622},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.85199999999999998,"15min":0.73399999999999999},"min":{"1min":0.376,"5min":0.309,"15min":0.30499999999999999},"max":{"1min":1.6539999999999999,"5min":4.133,"15min":4.133},"last":2.0329999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:03:26 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.81799999999999995,"15min":1.3759999999999999},"min":{"1min":0.34699999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.0339999999999998,"5min":3.5379999999999998,"15min":131.94399999999999},"last":2.0529999999999999},{"interface":"front","average":{"1min":0.629,"5min":0.90000000000000002,"15min":1.3799999999999999},"min":{"1min":0.36399999999999999,"5min":0.313,"15min":0.29999999999999999},"max":{"1min":1.6339999999999999,"5min":5.4870000000000001,"15min":135.23400000000001},"last":1.9159999999999999}]}]},{"osd":6,"up_from":46,"seq":197568495927,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2027196,"kb_used_data":1433332,"kb_used_omap":0,"kb_used_meta":593856,"kb_avail":91717956,"statfs":{"total":95995035648,"available":93919186944,"internally_reserved":0,"allocated":1467731968,"data_stored":1466616385,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608108544},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:03:08 2023","interfaces":[{"interface":"back","average":{"1min":1.794,"5min":3.7989999999999999,"15min":2.165},"min":{"1min":0.26000000000000001,"5min":0.22900000000000001,"15min":0.224},"max":{"1min":25.933,"5min":165.124,"15min":165.124},"last":0.56899999999999995},{"interface":"front","average":{"1min":1.996,"5min":3.9180000000000001,"15min":2.2050000000000001},"min":{"1min":0.29199999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":25.611000000000001,"5min":165.221,"15min":165.221},"last":0.53400000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:02:50 2023","interfaces":[{"interface":"back","average":{"1min":1.371,"5min":1.208,"15min":0.97699999999999998},"min":{"1min":0.29099999999999998,"5min":0.27000000000000002,"15min":0.23899999999999999},"max":{"1min":9.8819999999999997,"5min":9.8819999999999997,"15min":17.341000000000001},"last":0.86199999999999999},{"interface":"front","average":{"1min":0.86499999999999999,"5min":1.512,"15min":1.123},"min":{"1min":0.32100000000000001,"5min":0.29199999999999998,"15min":0.27000000000000002},"max":{"1min":2.1299999999999999,"5min":22.445,"15min":22.445},"last":0.62}]},{"osd":2,"last update":"Mon Oct 9 06:03:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5720000000000001,"5min":2.202,"15min":1.7430000000000001},"min":{"1min":0.313,"5min":0.26900000000000002,"15min":0.26700000000000002},"max":{"1min":13.917,"5min":27.224,"15min":61.088999999999999},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.80800000000000005,"5min":2.0950000000000002,"15min":1.7010000000000001},"min":{"1min":0.34999999999999998,"5min":0.28899999999999998,"15min":0.25600000000000001},"max":{"1min":4.415,"5min":27.177,"15min":61.082000000000001},"last":1.129}]},{"osd":3,"last update":"Mon Oct 9 06:02:50 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":5.9900000000000002,"15min":3.593},"min":{"1min":0.34399999999999997,"5min":0.307,"15min":0.29599999999999999},"max":{"1min":2.2240000000000002,"5min":86.099000000000004,"15min":93.626000000000005},"last":1.056},{"interface":"front","average":{"1min":1.1879999999999999,"5min":5.9969999999999999,"15min":3.5089999999999999},"min":{"1min":0.34200000000000003,"5min":0.314,"15min":0.28499999999999998},"max":{"1min":4.2050000000000001,"5min":83.126999999999995,"15min":93.643000000000001},"last":0.68000000000000005}]},{"osd":4,"last update":"Mon Oct 9 06:03:22 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":1.151,"15min":1.248},"min":{"1min":0.26400000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":1.6419999999999999,"5min":4.673,"15min":4.673},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.68300000000000005,"5min":1.0149999999999999,"15min":1.0700000000000001},"min":{"1min":0.28199999999999997,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":2.9049999999999998,"5min":4.399,"15min":4.399},"last":0.61099999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:02:56 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.81000000000000005,"15min":1.002},"min":{"1min":0.29799999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.88800000000000001,"5min":4.9699999999999998,"15min":45.933999999999997},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.84599999999999997,"15min":1.0309999999999999},"min":{"1min":0.27700000000000002,"5min":0.255,"15min":0.255},"max":{"1min":2.7469999999999999,"5min":4.976,"15min":45.917999999999999},"last":0.48999999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:03:22 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.1220000000000001,"15min":1.2370000000000001},"min":{"1min":0.32500000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.2869999999999999,"5min":6.4749999999999996,"15min":52.042999999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.91100000000000003,"5min":1.0109999999999999,"15min":1.141},"min":{"1min":0.35399999999999998,"5min":0.33100000000000002,"15min":0.32200000000000001},"max":{"1min":3.536,"5min":6.4589999999999996,"15min":51.259},"last":0.52400000000000002}]}]},{"osd":1,"up_from":89,"seq":382252089516,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2315984,"kb_used_data":2055048,"kb_used_omap":9,"kb_used_meta":260918,"kb_avail":91429168,"statfs":{"total":95995035648,"available":93623468032,"internally_reserved":0,"allocated":2104369152,"data_stored":2102236591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":267181033},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,0,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:03:15 2023","interfaces":[{"interface":"back","average":{"1min":8.8889999999999993,"5min":5.1909999999999998,"15min":2.323},"min":{"1min":0.23699999999999999,"5min":0.20100000000000001,"15min":0.191},"max":{"1min":85.495999999999995,"5min":132.04499999999999,"15min":132.04499999999999},"last":4.452},{"interface":"front","average":{"1min":8.9659999999999993,"5min":5.5519999999999996,"15min":2.6419999999999999},"min":{"1min":0.27100000000000002,"5min":0.23599999999999999,"15min":0.21299999999999999},"max":{"1min":85.900999999999996,"5min":132.91200000000001,"15min":132.91200000000001},"last":4.5389999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":3.5169999999999999,"15min":2.9260000000000002},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20100000000000001},"max":{"1min":3.774,"5min":129.053,"15min":204.989},"last":4.4939999999999998},{"interface":"front","average":{"1min":1.994,"5min":3.73,"15min":3.1629999999999998},"min":{"1min":0.28299999999999997,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":12.728,"5min":129.077,"15min":203.65899999999999},"last":4.3029999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:03:46 2023","interfaces":[{"interface":"back","average":{"1min":1.3440000000000001,"5min":6.085,"15min":5.0960000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.214},"max":{"1min":7.9669999999999996,"5min":213.654,"15min":284.06099999999998},"last":5.2510000000000003},{"interface":"front","average":{"1min":1.556,"5min":6.0800000000000001,"15min":5.1760000000000002},"min":{"1min":0.30499999999999999,"5min":0.28100000000000003,"15min":0.27900000000000003},"max":{"1min":6.9089999999999998,"5min":213.77199999999999,"15min":284.97300000000001},"last":0.504}]},{"osd":4,"last update":"Mon Oct 9 06:03:25 2023","interfaces":[{"interface":"back","average":{"1min":1.298,"5min":1.0860000000000001,"15min":1.05},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.0429999999999993,"5min":8.0429999999999993,"15min":8.0429999999999993},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.89000000000000001,"5min":1.4550000000000001,"15min":1.5489999999999999},"min":{"1min":0.51100000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.9259999999999999,"5min":9.8629999999999995,"15min":9.8629999999999995},"last":1.1299999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:03:37 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.0640000000000001,"15min":1.198},"min":{"1min":0.372,"5min":0.34200000000000003,"15min":0.32800000000000001},"max":{"1min":4.3250000000000002,"5min":8.0980000000000008,"15min":64.584999999999994},"last":0.57599999999999996},{"interface":"front","average":{"1min":1.5449999999999999,"5min":1.5429999999999999,"15min":1.373},"min":{"1min":0.51200000000000001,"5min":0.34300000000000003,"15min":0.315},"max":{"1min":8.3390000000000004,"5min":9.827,"15min":64.150999999999996},"last":1.1679999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:03:37 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.147,"15min":1.899},"min":{"1min":0.33100000000000002,"5min":0.318,"15min":0.30099999999999999},"max":{"1min":4.3570000000000002,"5min":8.0700000000000003,"15min":204.62700000000001},"last":0.61699999999999999},{"interface":"front","average":{"1min":1.55,"5min":1.6619999999999999,"15min":2.0830000000000002},"min":{"1min":0.54300000000000004,"5min":0.38300000000000001,"15min":0.35499999999999998},"max":{"1min":8.3919999999999995,"5min":8.4990000000000006,"15min":204.285},"last":1.2390000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:03:37 2023","interfaces":[{"interface":"back","average":{"1min":1.5820000000000001,"5min":2.5649999999999999,"15min":1.823},"min":{"1min":0.54300000000000004,"5min":0.41899999999999998,"15min":0.314},"max":{"1min":8.4329999999999998,"5min":108.553,"15min":108.553},"last":1.2090000000000001},{"interface":"front","average":{"1min":1.2150000000000001,"5min":2.3149999999999999,"15min":1.5960000000000001},"min":{"1min":0.39300000000000002,"5min":0.38100000000000001,"15min":0.38},"max":{"1min":7.9359999999999999,"5min":108.56999999999999,"15min":108.56999999999999},"last":4.5739999999999998}]}]},{"osd":0,"up_from":129,"seq":554050781318,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2112576,"kb_used_data":1761336,"kb_used_omap":29,"kb_used_meta":351202,"kb_avail":91632576,"statfs":{"total":95995035648,"available":93831757824,"internally_reserved":0,"allocated":1803608064,"data_stored":1801609586,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30322,"internal_metadata":359631246},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,5],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:03:16 2023","interfaces":[{"interface":"back","average":{"1min":1.724,"5min":1.462,"15min":1.0369999999999999},"min":{"1min":0.224,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":22,"5min":22.009,"15min":22.009},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.714,"5min":1.5309999999999999,"15min":1.1759999999999999},"min":{"1min":0.25,"5min":0.248,"15min":0.24099999999999999},"max":{"1min":15.173,"5min":20.082000000000001,"15min":20.082000000000001},"last":2.2930000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:03:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":2.0449999999999999,"15min":1.429},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.23000000000000001},"max":{"1min":7.133,"5min":52.405999999999999,"15min":52.405999999999999},"last":2.2480000000000002},{"interface":"front","average":{"1min":1.1160000000000001,"5min":2.8980000000000001,"15min":1.675},"min":{"1min":0.28599999999999998,"5min":0.219,"15min":0.219},"max":{"1min":8.1690000000000005,"5min":52.645000000000003,"15min":52.645000000000003},"last":0.44700000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:03:16 2023","interfaces":[{"interface":"back","average":{"1min":1.2450000000000001,"5min":5.7329999999999997,"15min":2.714},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":7.2080000000000002,"5min":134.185,"15min":134.185},"last":2.3769999999999998},{"interface":"front","average":{"1min":1.347,"5min":6.1539999999999999,"15min":2.827},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":8.3239999999999998,"5min":134.16800000000001,"15min":134.16800000000001},"last":0.66500000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:03:21 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.7669999999999999,"15min":1.871},"min":{"1min":0.39600000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.1419999999999995,"5min":10.69,"15min":10.69},"last":0.88700000000000001},{"interface":"front","average":{"1min":1.161,"5min":1.349,"15min":1.3799999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":8.3079999999999998,"5min":8.3079999999999998,"15min":8.3079999999999998},"last":1.7}]},{"osd":5,"last update":"Mon Oct 9 06:03:16 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":2.077,"15min":1.8919999999999999},"min":{"1min":0.44500000000000001,"5min":0.318,"15min":0.314},"max":{"1min":8.109,"5min":39.188000000000002,"15min":100.25},"last":0.749},{"interface":"front","average":{"1min":1.206,"5min":1.53,"15min":1.633},"min":{"1min":0.39700000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":8.452,"5min":39.143000000000001,"15min":102.40300000000001},"last":2.7810000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:03:16 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":1.964,"15min":1.4179999999999999},"min":{"1min":0.46300000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":8.0760000000000005,"5min":16.849,"15min":16.849},"last":2.9430000000000001},{"interface":"front","average":{"1min":1.224,"5min":1.4410000000000001,"15min":1.3},"min":{"1min":0.46899999999999997,"5min":0.39000000000000001,"15min":0.378},"max":{"1min":8.3490000000000002,"5min":17.869,"15min":17.869},"last":0.77400000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:03:16 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":2.173,"15min":1.8799999999999999},"min":{"1min":0.499,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":8.4779999999999998,"5min":103.866,"15min":103.866},"last":0.86499999999999999},{"interface":"front","average":{"1min":1.083,"5min":2.3380000000000001,"15min":1.845},"min":{"1min":0.439,"5min":0.38300000000000001,"15min":0.378},"max":{"1min":7.1719999999999997,"5min":103.848,"15min":103.848},"last":2.3420000000000001}]}]},{"osd":2,"up_from":94,"seq":403726926007,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2224316,"kb_used_data":1720820,"kb_used_omap":81,"kb_used_meta":503406,"kb_avail":91520836,"statfs":{"total":95995035648,"available":93717336064,"internally_reserved":0,"allocated":1762119680,"data_stored":1760239093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":515487962},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:03:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":3.851,"15min":2.5350000000000001},"min":{"1min":0.20799999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":7.6580000000000004,"5min":70.721999999999994,"15min":70.721999999999994},"last":6.1180000000000003},{"interface":"front","average":{"1min":0.82599999999999996,"5min":3.7879999999999998,"15min":2.468},"min":{"1min":0.224,"5min":0.215,"15min":0.215},"max":{"1min":3.9140000000000001,"5min":72.382999999999996,"15min":72.382999999999996},"last":3.782}]},{"osd":1,"last update":"Mon Oct 9 06:03:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":1.6850000000000001,"15min":1.7749999999999999},"min":{"1min":0.22900000000000001,"5min":0.224,"15min":0.21099999999999999},"max":{"1min":7.6210000000000004,"5min":14.226000000000001,"15min":163.24199999999999},"last":6.2000000000000002},{"interface":"front","average":{"1min":0.82399999999999995,"5min":1.7929999999999999,"15min":1.734},"min":{"1min":0.27000000000000002,"5min":0.252,"15min":0.214},"max":{"1min":4.2229999999999999,"5min":13.625,"15min":163.27000000000001},"last":3.819}]},{"osd":3,"last update":"Mon Oct 9 06:03:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":4.423,"15min":3.1459999999999999},"min":{"1min":0.25800000000000001,"5min":0.24099999999999999,"15min":0.23899999999999999},"max":{"1min":5.149,"5min":144.88200000000001,"15min":170.07599999999999},"last":1.8009999999999999},{"interface":"front","average":{"1min":1.615,"5min":4.6849999999999996,"15min":2.794},"min":{"1min":0.29699999999999999,"5min":0.25900000000000001,"15min":0.25},"max":{"1min":7.4000000000000004,"5min":140.79499999999999,"15min":140.79499999999999},"last":6.1639999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:03:26 2023","interfaces":[{"interface":"back","average":{"1min":1.264,"5min":1.631,"15min":1.6919999999999999},"min":{"1min":0.32300000000000001,"5min":0.315,"15min":0.315},"max":{"1min":5.9379999999999997,"5min":7.9989999999999997,"15min":7.9989999999999997},"last":3.851},{"interface":"front","average":{"1min":1.4259999999999999,"5min":1.8069999999999999,"15min":1.871},"min":{"1min":0.38,"5min":0.31,"15min":0.31},"max":{"1min":6.4210000000000003,"5min":7.2590000000000003,"15min":7.2590000000000003},"last":6.2350000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:03:12 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":1.4339999999999999,"15min":1.2470000000000001},"min":{"1min":0.42799999999999999,"5min":0.34599999999999997,"15min":0.28999999999999998},"max":{"1min":2.3130000000000002,"5min":8.8819999999999997,"15min":13.414},"last":1.7749999999999999},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.4670000000000001,"15min":1.2430000000000001},"min":{"1min":0.41399999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.1880000000000002,"5min":7.6900000000000004,"15min":13.438000000000001},"last":6.2709999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:03:12 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":2.0990000000000002,"15min":1.389},"min":{"1min":0.38,"5min":0.34699999999999998,"15min":0.29399999999999998},"max":{"1min":3.8769999999999998,"5min":30.420000000000002,"15min":30.420000000000002},"last":5.4169999999999998},{"interface":"front","average":{"1min":0.83999999999999997,"5min":1.921,"15min":1.319},"min":{"1min":0.40100000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.4489999999999998,"5min":30.713000000000001,"15min":30.713000000000001},"last":1.823}]},{"osd":7,"last update":"Mon Oct 9 06:03:12 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":2.1280000000000001,"15min":1.7589999999999999},"min":{"1min":0.41199999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":2.2890000000000001,"5min":33.856000000000002,"15min":55.052},"last":1.7410000000000001},{"interface":"front","average":{"1min":1.0660000000000001,"5min":2.0619999999999998,"15min":1.544},"min":{"1min":0.45200000000000001,"5min":0.35899999999999999,"15min":0.35599999999999998},"max":{"1min":3.214,"5min":33.098999999999997,"15min":40.113},"last":6.3070000000000004}]}]},{"osd":3,"up_from":99,"seq":425201762481,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2262960,"kb_used_data":1737960,"kb_used_omap":7,"kb_used_meta":524984,"kb_avail":91482192,"statfs":{"total":95995035648,"available":93677764608,"internally_reserved":0,"allocated":1779671040,"data_stored":1777563524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":537584232},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:03:21 2023","interfaces":[{"interface":"back","average":{"1min":4.9530000000000003,"5min":4.4000000000000004,"15min":2.2930000000000001},"min":{"1min":0.23799999999999999,"5min":0.20499999999999999,"15min":0.186},"max":{"1min":84.673000000000002,"5min":109.976,"15min":109.976},"last":7.3200000000000003},{"interface":"front","average":{"1min":4.9809999999999999,"5min":4.5270000000000001,"15min":2.4940000000000002},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":84.439999999999998,"5min":107.161,"15min":107.161},"last":0.23999999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:03:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3680000000000001,"5min":1.966,"15min":1.6899999999999999},"min":{"1min":0.23999999999999999,"5min":0.23300000000000001,"15min":0.20999999999999999},"max":{"1min":8.3339999999999996,"5min":26.538,"15min":127.226},"last":7.0789999999999997},{"interface":"front","average":{"1min":1.2390000000000001,"5min":3.2330000000000001,"15min":2.0720000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.21099999999999999},"max":{"1min":8.3879999999999999,"5min":106.95699999999999,"15min":127.468},"last":7.2460000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:03:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0049999999999999,"5min":3.7149999999999999,"15min":2.1419999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":41.945999999999998,"5min":107.01300000000001,"15min":107.01300000000001},"last":0.27100000000000002},{"interface":"front","average":{"1min":3.1190000000000002,"5min":3.6549999999999998,"15min":2.3450000000000002},"min":{"1min":0.28100000000000003,"5min":0.249,"15min":0.249},"max":{"1min":41.475999999999999,"5min":109.807,"15min":109.807},"last":7.2290000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:03:26 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":5.8869999999999996,"15min":6.6980000000000004},"min":{"1min":0.34499999999999997,"5min":0.254,"15min":0.254},"max":{"1min":3.8809999999999998,"5min":99.876000000000005,"15min":99.876000000000005},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.92100000000000004,"5min":5.508,"15min":6.2720000000000002},"min":{"1min":0.45200000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":4.5439999999999996,"5min":99.914000000000001,"15min":99.914000000000001},"last":7.181}]},{"osd":5,"last update":"Mon Oct 9 06:03:50 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":1.7310000000000001,"15min":1.284},"min":{"1min":0.32200000000000001,"5min":0.308,"15min":0.308},"max":{"1min":3.101,"5min":22.670000000000002,"15min":22.670000000000002},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.0980000000000001,"5min":2.04,"15min":1.3080000000000001},"min":{"1min":0.37,"5min":0.34300000000000003,"15min":0.28199999999999997},"max":{"1min":3.4329999999999998,"5min":7.7110000000000003,"15min":18.013999999999999},"last":4.7300000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:03:50 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":6.1580000000000004,"15min":2.919},"min":{"1min":0.379,"5min":0.33100000000000002,"15min":0.30499999999999999},"max":{"1min":3.0640000000000001,"5min":152.69,"15min":152.69},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.081,"5min":6.4290000000000003,"15min":2.96},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.318},"max":{"1min":3.3929999999999998,"5min":152.70400000000001,"15min":152.70400000000001},"last":4.798}]},{"osd":7,"last update":"Mon Oct 9 06:03:50 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":6.6989999999999998,"15min":2.9390000000000001},"min":{"1min":0.39000000000000001,"5min":0.377,"15min":0.34200000000000003},"max":{"1min":3.3610000000000002,"5min":149.93600000000001,"15min":149.93600000000001},"last":4.774},{"interface":"front","average":{"1min":1.0449999999999999,"5min":6.5810000000000004,"15min":2.887},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":5.2629999999999999,"5min":150.12299999999999,"15min":150.12299999999999},"last":7.2110000000000003}]}]},{"osd":4,"up_from":322,"seq":1382979469344,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1793548,"kb_used_data":1471172,"kb_used_omap":35,"kb_used_meta":322332,"kb_avail":91951604,"statfs":{"total":95995035648,"available":94158442496,"internally_reserved":0,"allocated":1506480128,"data_stored":1505348825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36552,"internal_metadata":330068280},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":1.131,"15min":1.2210000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":1.5,"5min":4.2720000000000002,"15min":4.2720000000000002},"last":0.61299999999999999},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.7,"15min":1.794},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":8.3719999999999999,"5min":8.3719999999999999,"15min":8.3719999999999999},"last":1.5640000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":1.6899999999999999,"15min":1.8109999999999999},"min":{"1min":0.30199999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.1110000000000002,"5min":6.3810000000000002,"15min":6.3810000000000002},"last":2.069},{"interface":"front","average":{"1min":0.84799999999999998,"5min":1.3919999999999999,"15min":1.4830000000000001},"min":{"1min":0.316,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.0920000000000001,"5min":12.413,"15min":12.413},"last":3.0190000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.83699999999999997,"5min":1.6739999999999999,"15min":1.8140000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.6080000000000001,"5min":4.8840000000000003,"15min":4.8840000000000003},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.85199999999999998,"5min":1.966,"15min":2.1520000000000001},"min":{"1min":0.34399999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.9929999999999999,"5min":11.381,"15min":11.381},"last":0.501}]},{"osd":3,"last update":"Mon Oct 9 06:03:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.9019999999999999,"15min":2.0379999999999998},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":3.5539999999999998,"5min":4.9009999999999998,"15min":4.9009999999999998},"last":0.59499999999999997},{"interface":"front","average":{"1min":1.2150000000000001,"5min":1.6499999999999999,"15min":1.722},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":5.931,"5min":7.5960000000000001,"15min":7.5960000000000001},"last":0.379}]},{"osd":5,"last update":"Mon Oct 9 06:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.85499999999999998,"15min":0.89500000000000002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.226,"5min":5.7629999999999999,"15min":5.7629999999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.503,"5min":1.3,"15min":1.4330000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.2689999999999999,"5min":4.8369999999999997,"15min":4.8369999999999997},"last":0.51100000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":3.5419999999999998,"15min":4.0369999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.1040000000000001,"5min":53.744999999999997,"15min":53.744999999999997},"last":2.2450000000000001},{"interface":"front","average":{"1min":0.52900000000000003,"5min":3.7309999999999999,"15min":4.2649999999999997},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.048,"5min":53.798000000000002,"15min":53.798000000000002},"last":0.63300000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":5.3380000000000001,"15min":6.1260000000000003},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.4119999999999999,"5min":79.173000000000002,"15min":79.173000000000002},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.64600000000000002,"5min":4.3280000000000003,"15min":4.9420000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.5669999999999999,"5min":78.894000000000005,"15min":78.894000000000005},"last":0.80200000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757564,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2615680,"kb_used_data":2314296,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":91129472,"statfs":{"total":95995035648,"available":93316579328,"internally_reserved":0,"allocated":2369839104,"data_stored":2368716377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:03:27 2023","interfaces":[{"interface":"back","average":{"1min":4.0800000000000001,"5min":2.5379999999999998,"15min":1.4430000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.22500000000000001},"max":{"1min":69.088999999999999,"5min":69.088999999999999,"15min":69.088999999999999},"last":4.5759999999999996},{"interface":"front","average":{"1min":4.024,"5min":2.8759999999999999,"15min":1.663},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":69.073999999999998,"5min":69.073999999999998,"15min":69.073999999999998},"last":2.5590000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:03:43 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":2.347,"15min":2.2709999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.26000000000000001},"max":{"1min":4.0179999999999998,"5min":102.642,"15min":114.247},"last":2.5830000000000002},{"interface":"front","average":{"1min":0.96899999999999997,"5min":2.0099999999999998,"15min":2.0110000000000001},"min":{"1min":0.28899999999999998,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":2.5009999999999999,"5min":102.45999999999999,"15min":114.678},"last":4.6660000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:03:12 2023","interfaces":[{"interface":"back","average":{"1min":7.4580000000000002,"5min":3.8380000000000001,"15min":3.0990000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.25},"max":{"1min":129.21199999999999,"5min":129.21199999999999,"15min":217.06399999999999},"last":2.7149999999999999},{"interface":"front","average":{"1min":7.6970000000000001,"5min":3.5030000000000001,"15min":2.9430000000000001},"min":{"1min":0.38100000000000001,"5min":0.28199999999999997,"15min":0.27600000000000002},"max":{"1min":129.042,"5min":129.042,"15min":217.018},"last":4.6429999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:03:43 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":8.0969999999999995,"15min":6.8780000000000001},"min":{"1min":0.39700000000000002,"5min":0.31900000000000001,"15min":0.27500000000000002},"max":{"1min":8.0670000000000002,"5min":97.403000000000006,"15min":272.81299999999999},"last":2.5030000000000001},{"interface":"front","average":{"1min":1.2290000000000001,"5min":7.8719999999999999,"15min":7.0019999999999998},"min":{"1min":0.47699999999999998,"5min":0.32200000000000001,"15min":0.317},"max":{"1min":2.8490000000000002,"5min":94.423000000000002,"15min":273.00999999999999},"last":2.742}]},{"osd":4,"last update":"Mon Oct 9 06:03:27 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.82199999999999995,"15min":0.88},"min":{"1min":0.28000000000000003,"5min":0.249,"15min":0.249},"max":{"1min":0.82299999999999995,"5min":4.9720000000000004,"15min":4.9720000000000004},"last":7.6859999999999999},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.0860000000000001,"15min":1.1080000000000001},"min":{"1min":0.27000000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.8029999999999999,"5min":5.8879999999999999,"15min":5.8879999999999999},"last":4.1760000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:03:45 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":2.2629999999999999,"15min":1.274},"min":{"1min":0.36099999999999999,"5min":0.251,"15min":0.251},"max":{"1min":5.0220000000000002,"5min":126.89400000000001,"15min":126.89400000000001},"last":3.5339999999999998},{"interface":"front","average":{"1min":0.88700000000000001,"5min":2.258,"15min":1.3080000000000001},"min":{"1min":0.34699999999999998,"5min":0.26700000000000002,"15min":0.26200000000000001},"max":{"1min":3.0830000000000002,"5min":126.616,"15min":126.616},"last":2.488}]},{"osd":7,"last update":"Mon Oct 9 06:03:27 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":3.6779999999999999,"15min":1.8640000000000001},"min":{"1min":0.34599999999999997,"5min":0.308,"15min":0.29999999999999999},"max":{"1min":2.879,"5min":133.27099999999999,"15min":133.27099999999999},"last":2.5329999999999999},{"interface":"front","average":{"1min":0.85099999999999998,"5min":3.5619999999999998,"15min":1.661},"min":{"1min":0.442,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.4620000000000002,"5min":135.09200000000001,"15min":135.09200000000001},"last":4.6180000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1474,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4889,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1801322496,"data_stored":1801322496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2112749568,"data_stored":2112749568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1761083392,"data_stored":1761083392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1788936192,"data_stored":1788936192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1506082816,"data_stored":1506082816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2377252864,"data_stored":2377252864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1473511424,"data_stored":1473511424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1507328000,"data_stored":1507328000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:04:00.051 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T06:04:00.051 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:04:00.052 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-10-09T06:04:00.052 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:04:00.052 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:04:00.052 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-10-09T06:04:00.143 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:00.176 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:04:00.196 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:00.285 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:00.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:00.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:00.386 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:00 smithi064 ceph-mon[212750]: pgmap v894: 119 pgs: 119 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s 2023-10-09T06:04:00.386 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:00 smithi064 ceph-mon[212750]: failed to encode map e372 with expected crc 2023-10-09T06:04:00.386 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:00 smithi064 ceph-mon[212750]: failed to encode map e372 with expected crc 2023-10-09T06:04:00.386 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:00 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2023-10-09T06:04:00.386 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:00 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-10-09T06:04:00.386 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:00 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-10-09T06:04:00.386 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:00 smithi064 ceph-mon[212750]: osdmap e372: 8 total, 8 up, 8 in 2023-10-09T06:04:00.387 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:00 smithi064 ceph-mon[220044]: pgmap v894: 119 pgs: 119 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s 2023-10-09T06:04:00.387 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:00 smithi064 ceph-mon[220044]: failed to encode map e372 with expected crc 2023-10-09T06:04:00.387 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:00 smithi064 ceph-mon[220044]: failed to encode map e372 with expected crc 2023-10-09T06:04:00.387 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:00 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2023-10-09T06:04:00.387 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:00 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-10-09T06:04:00.387 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:00 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-10-09T06:04:00.387 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:00 smithi064 ceph-mon[220044]: osdmap e372: 8 total, 8 up, 8 in 2023-10-09T06:04:00.390 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:00.519 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:00.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:00 smithi156 ceph-mon[195101]: pgmap v894: 119 pgs: 119 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s 2023-10-09T06:04:00.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:00 smithi156 ceph-mon[195101]: failed to encode map e372 with expected crc 2023-10-09T06:04:00.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:00 smithi156 ceph-mon[195101]: failed to encode map e372 with expected crc 2023-10-09T06:04:00.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:00 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2023-10-09T06:04:00.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:00 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-10-09T06:04:00.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:00 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-10-09T06:04:00.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:00 smithi156 ceph-mon[195101]: osdmap e372: 8 total, 8 up, 8 in 2023-10-09T06:04:00.662 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:00.697 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:00.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:00.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:00.713 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:04:00.750 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 58 15 78120 78105 84.1201 53.625 0.0247915 0.0118832 2023-10-09T06:04:00.805 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:00.883 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:01.000 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:01.060 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:01.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:01.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:01.166 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:01.241 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:01.394 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:01 smithi064 ceph-mon[220044]: failed to encode map e372 with expected crc 2023-10-09T06:04:01.394 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:01 smithi064 ceph-mon[220044]: osdmap e373: 8 total, 8 up, 8 in 2023-10-09T06:04:01.394 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:01 smithi064 ceph-mon[220044]: failed to encode map e373 with expected crc 2023-10-09T06:04:01.394 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:01 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2779692149' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:01.548 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:04:01.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:01 smithi064 ceph-mon[212750]: failed to encode map e372 with expected crc 2023-10-09T06:04:01.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:01 smithi064 ceph-mon[212750]: osdmap e373: 8 total, 8 up, 8 in 2023-10-09T06:04:01.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:01 smithi064 ceph-mon[212750]: failed to encode map e373 with expected crc 2023-10-09T06:04:01.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:01 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2779692149' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:01.752 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 59 16 78980 78964 83.6018 53.6875 0.0180197 0.0119567 2023-10-09T06:04:01.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:01 smithi156 ceph-mon[195101]: failed to encode map e372 with expected crc 2023-10-09T06:04:01.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:01 smithi156 ceph-mon[195101]: osdmap e373: 8 total, 8 up, 8 in 2023-10-09T06:04:01.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:01 smithi156 ceph-mon[195101]: failed to encode map e373 with expected crc 2023-10-09T06:04:01.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:01 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2779692149' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:02.580 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:04:02.580 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:04:02.603 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:02 smithi156 ceph-mon[195101]: failed to encode map e373 with expected crc 2023-10-09T06:04:02.603 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:02 smithi156 ceph-mon[195101]: pgmap v897: 119 pgs: 2 peering, 117 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T06:04:02.604 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:02 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:02.604 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:02 smithi156 ceph-mon[195101]: failed to encode map e374 with expected crc 2023-10-09T06:04:02.604 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:02 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2779692149' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:04:02.604 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:02 smithi156 ceph-mon[195101]: osdmap e374: 8 total, 8 up, 8 in 2023-10-09T06:04:02.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:02 smithi064 ceph-mon[212750]: failed to encode map e373 with expected crc 2023-10-09T06:04:02.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:02 smithi064 ceph-mon[212750]: pgmap v897: 119 pgs: 2 peering, 117 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T06:04:02.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:02 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:02.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:02 smithi064 ceph-mon[212750]: failed to encode map e374 with expected crc 2023-10-09T06:04:02.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:02 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2779692149' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:04:02.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:02 smithi064 ceph-mon[212750]: osdmap e374: 8 total, 8 up, 8 in 2023-10-09T06:04:02.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:02 smithi064 ceph-mon[220044]: failed to encode map e373 with expected crc 2023-10-09T06:04:02.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:02 smithi064 ceph-mon[220044]: pgmap v897: 119 pgs: 2 peering, 117 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T06:04:02.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:02 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:02.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:02 smithi064 ceph-mon[220044]: failed to encode map e374 with expected crc 2023-10-09T06:04:02.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:02 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2779692149' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:04:02.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:02 smithi064 ceph-mon[220044]: osdmap e374: 8 total, 8 up, 8 in 2023-10-09T06:04:02.752 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-10-09T06:04:02.753371+0000 min lat: 0.0010326 max lat: 0.330132 avg lat: 0.012005 2023-10-09T06:04:02.753 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:04:02.753 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 60 16 79993 79977 83.2637 63.3125 0.0120191 0.012005 2023-10-09T06:04:03.571 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:04:03.746 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:03.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:03.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:03.755 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 61 16 80936 80920 82.8633 58.9375 0.00965283 0.0120628 2023-10-09T06:04:03.856 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:04.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:04 smithi064 ceph-mon[212750]: pgmap v899: 119 pgs: 2 peering, 117 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 45 MiB/s wr, 720 op/s 2023-10-09T06:04:04.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:04 smithi064 ceph-mon[220044]: pgmap v899: 119 pgs: 2 peering, 117 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 45 MiB/s wr, 720 op/s 2023-10-09T06:04:04.908 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 62 16 81802 81786 82.3988 54.125 0.00989146 0.0121302 2023-10-09T06:04:05.001 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:04:05.001 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:04:05.002 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:04:05.002 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:04:05.002 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:04:05.003 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:04 smithi156 ceph-mon[195101]: pgmap v899: 119 pgs: 2 peering, 117 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 45 MiB/s wr, 720 op/s 2023-10-09T06:04:05.054 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:04:05.064 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:04:05.239 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:05.240 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:04:05.240 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:04:05.240 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:04:05.337 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:04:05.338 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:04:05.338 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:04:05.338 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:04:05.338 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:04:05.338 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:04:05.338 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:04:05.351 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:05.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:05.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:05.413 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:04:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:04:05] "GET /metrics HTTP/1.1" 200 36381 "" "Prometheus/2.43.0" 2023-10-09T06:04:05.459 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:04:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T06:04:05.502 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:04:05.503 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:05.577 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:05.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:05 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/852585508' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:05.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:05 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/852585508' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:05.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:05 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/852585508' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:05.755 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 63 16 82586 82570 81.8688 49 0.00569033 0.0122096 2023-10-09T06:04:05.915 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:04:05.915 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:04:05.915 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T06:04:05.915 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T06:04:05.915 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:04:06.371 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:06.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:06.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:06.477 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:06.539 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:06.676 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:06 smithi064 ceph-mon[212750]: pgmap v900: 119 pgs: 2 peering, 117 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 45 MiB/s wr, 713 op/s 2023-10-09T06:04:06.676 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:06 smithi064 ceph-mon[220044]: pgmap v900: 119 pgs: 2 peering, 117 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 45 MiB/s wr, 713 op/s 2023-10-09T06:04:06.751 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:06 smithi156 ceph-mon[195101]: pgmap v900: 119 pgs: 2 peering, 117 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 45 MiB/s wr, 713 op/s 2023-10-09T06:04:06.755 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 64 16 83465 83449 81.4481 54.9375 0.00311907 0.0122669 2023-10-09T06:04:06.769 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:04:06.780 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:06.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:06.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:06.891 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:06.908 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:04:06.965 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:07.038 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:07.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:07.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:07.151 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:07.226 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:07.321 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:04:07.380 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:07.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:07.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:07.493 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:07.564 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:07.659 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:07.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:07.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:07.755 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 65 16 84411 84395 81.1048 59.125 0.0240117 0.012326 2023-10-09T06:04:07.765 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:07.768 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:07.841 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:07.952 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:07.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:07.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:08.062 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:08.091 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:08.139 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:08.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:08.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:08.243 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:08.272 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:08.412 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:04:08.425 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:08 smithi064 ceph-mon[212750]: pgmap v901: 119 pgs: 119 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T06:04:08.427 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:08 smithi064 ceph-mon[220044]: pgmap v901: 119 pgs: 119 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T06:04:08.539 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:08.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:08.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:08.651 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:08.719 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:08.755 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 66 16 85139 85123 80.5655 45.5 0.00789009 0.0124077 2023-10-09T06:04:08.797 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:08.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:08.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:08.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:08 smithi156 ceph-mon[195101]: pgmap v901: 119 pgs: 119 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T06:04:08.901 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:08.931 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:09.104 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:09.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:09.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:09.175 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:04:09.175 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T06:04:09.175 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T06:04:09.175 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:04:09.176 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.176 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:104350 6.15 6:af513ac9:::benchmark_data_smithi064_284193_object7807:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2023-10-09T06:04:09.176 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:54:45.800000+0000", 2023-10-09T06:04:09.176 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 563.36908408900001, 2023-10-09T06:04:09.176 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.898542277, 2023-10-09T06:04:09.176 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:09.176 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:09.176 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:09.176 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:04:09.176 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:04:09.176 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 104350 2023-10-09T06:04:09.177 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.177 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:09.177 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.177 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:09.177 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.800000+0000", 2023-10-09T06:04:09.177 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.177 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.177 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.177 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:09.177 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.800000+0000", 2023-10-09T06:04:09.177 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.178 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.178 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.178 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:09.178 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.799999+0000", 2023-10-09T06:04:09.178 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:04:09.178 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.178 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.178 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:09.178 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.800004+0000", 2023-10-09T06:04:09.178 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4840000000000003e-06 2023-10-09T06:04:09.178 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.178 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.179 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:09.179 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.800015+0000", 2023-10-09T06:04:09.179 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0581000000000001e-05 2023-10-09T06:04:09.179 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.179 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.179 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:09.179 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.800030+0000", 2023-10-09T06:04:09.179 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4489e-05 2023-10-09T06:04:09.179 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.179 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.179 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.180 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.800068+0000", 2023-10-09T06:04:09.180 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8318999999999997e-05 2023-10-09T06:04:09.180 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.180 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.180 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.180 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:45.800076+0000", 2023-10-09T06:04:09.180 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6690000000000006e-06 2023-10-09T06:04:09.180 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.180 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.180 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.180 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:46.648630+0000", 2023-10-09T06:04:09.181 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.84855464599999997 2023-10-09T06:04:09.181 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.181 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.181 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.181 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:46.648639+0000", 2023-10-09T06:04:09.181 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.5259999999999993e-06 2023-10-09T06:04:09.181 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.181 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.181 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.181 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:47.674779+0000", 2023-10-09T06:04:09.181 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.026139881 2023-10-09T06:04:09.182 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.182 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.182 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:09.182 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:47.674949+0000", 2023-10-09T06:04:09.182 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000170765 2023-10-09T06:04:09.182 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.182 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.182 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:04:09.182 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:47.675251+0000", 2023-10-09T06:04:09.182 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000301682 2023-10-09T06:04:09.182 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.183 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.183 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.183 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:47.697358+0000", 2023-10-09T06:04:09.183 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.022106995000000001 2023-10-09T06:04:09.183 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.183 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.183 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:09.183 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:47.697368+0000", 2023-10-09T06:04:09.183 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0220999999999999e-05 2023-10-09T06:04:09.183 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.184 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.184 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.184 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:47.697409+0000", 2023-10-09T06:04:09.184 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0881999999999999e-05 2023-10-09T06:04:09.184 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.184 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.184 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:09.184 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:47.697470+0000", 2023-10-09T06:04:09.184 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0804000000000003e-05 2023-10-09T06:04:09.184 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.185 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.185 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:09.185 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:54:47.698542+0000", 2023-10-09T06:04:09.185 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001072386 2023-10-09T06:04:09.185 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.185 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:09.185 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.185 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.185 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.185 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:185875 6.15 6:aa8ef49b:::benchmark_data_smithi064_284193_object89332:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-10-09T06:04:09.185 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.701818+0000", 2023-10-09T06:04:09.186 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 378.46726603000002, 2023-10-09T06:04:09.186 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.183023392, 2023-10-09T06:04:09.186 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:09.186 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:04:09.186 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:09.186 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:04:09.186 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:04:09.186 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 185875 2023-10-09T06:04:09.186 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.186 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:09.186 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.187 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:09.187 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.701818+0000", 2023-10-09T06:04:09.187 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.187 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.187 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.187 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:09.187 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.701818+0000", 2023-10-09T06:04:09.187 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.187 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.187 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.187 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:09.187 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.701818+0000", 2023-10-09T06:04:09.188 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:04:09.188 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.188 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.188 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:09.188 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.701821+0000", 2023-10-09T06:04:09.188 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1410000000000001e-06 2023-10-09T06:04:09.188 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.188 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.188 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:09.188 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.701831+0000", 2023-10-09T06:04:09.188 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0142e-05 2023-10-09T06:04:09.189 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.189 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.189 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:09.189 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.701835+0000", 2023-10-09T06:04:09.189 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2919999999999997e-06 2023-10-09T06:04:09.189 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.189 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.189 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.189 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.707726+0000", 2023-10-09T06:04:09.189 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0058907179999999996 2023-10-09T06:04:09.189 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.189 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.190 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:09.190 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.707842+0000", 2023-10-09T06:04:09.190 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000115937 2023-10-09T06:04:09.190 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.190 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.190 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:04:09.190 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.708169+0000", 2023-10-09T06:04:09.190 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00032739800000000001 2023-10-09T06:04:09.190 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.190 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.190 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:09.191 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.711326+0000", 2023-10-09T06:04:09.191 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.003157235 2023-10-09T06:04:09.191 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.191 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.191 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.191 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.732073+0000", 2023-10-09T06:04:09.191 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.020746568999999999 2023-10-09T06:04:09.191 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.191 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.191 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.191 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.732081+0000", 2023-10-09T06:04:09.191 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8420000000000005e-06 2023-10-09T06:04:09.192 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.192 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.192 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.192 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735784+0000", 2023-10-09T06:04:09.192 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.003702782 2023-10-09T06:04:09.192 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.192 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.192 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.192 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735796+0000", 2023-10-09T06:04:09.192 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2009e-05 2023-10-09T06:04:09.192 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.193 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.193 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.193 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.884749+0000", 2023-10-09T06:04:09.193 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.148953603 2023-10-09T06:04:09.193 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.193 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.193 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:09.193 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.884842+0000", 2023-10-09T06:04:09.193 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2200000000000005e-05 2023-10-09T06:04:09.193 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.193 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:09.193 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.194 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.194 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.194 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:185894 6.15 6:aea4a65e:::benchmark_data_smithi064_284193_object89351:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:04:09.194 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.718597+0000", 2023-10-09T06:04:09.194 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 378.45048720599999, 2023-10-09T06:04:09.194 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1840398429999999, 2023-10-09T06:04:09.194 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:09.194 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:09.194 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:09.194 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:04:09.194 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:04:09.194 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 185894 2023-10-09T06:04:09.195 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.195 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:09.195 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.195 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:09.195 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.718597+0000", 2023-10-09T06:04:09.195 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.195 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.195 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.195 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:09.195 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.718597+0000", 2023-10-09T06:04:09.195 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.196 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.196 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.196 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:09.196 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.718596+0000", 2023-10-09T06:04:09.196 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:04:09.196 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.196 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.196 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:09.196 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.718616+0000", 2023-10-09T06:04:09.196 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0449999999999999e-05 2023-10-09T06:04:09.196 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.196 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.197 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:09.197 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.718644+0000", 2023-10-09T06:04:09.197 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7945000000000001e-05 2023-10-09T06:04:09.197 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.197 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.197 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:09.197 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.718651+0000", 2023-10-09T06:04:09.197 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6490000000000002e-06 2023-10-09T06:04:09.197 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.197 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.197 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.197 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.718696+0000", 2023-10-09T06:04:09.198 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4870000000000002e-05 2023-10-09T06:04:09.198 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.198 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.198 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op must wait for map", 2023-10-09T06:04:09.198 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.718703+0000", 2023-10-09T06:04:09.198 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2409999999999998e-06 2023-10-09T06:04:09.198 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.198 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.198 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.198 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.732103+0000", 2023-10-09T06:04:09.198 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013399708999999999 2023-10-09T06:04:09.199 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.199 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.199 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.199 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.732109+0000", 2023-10-09T06:04:09.199 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4869999999999999e-06 2023-10-09T06:04:09.199 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.199 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.199 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.199 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735816+0000", 2023-10-09T06:04:09.199 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.003706183 2023-10-09T06:04:09.199 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.200 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.200 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.200 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735822+0000", 2023-10-09T06:04:09.200 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8580000000000002e-06 2023-10-09T06:04:09.200 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.200 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.200 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.200 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.884859+0000", 2023-10-09T06:04:09.200 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1490370620000001 2023-10-09T06:04:09.200 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.200 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.200 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:09.201 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.884937+0000", 2023-10-09T06:04:09.201 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7639000000000007e-05 2023-10-09T06:04:09.201 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.201 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.201 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T06:04:09.201 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.885342+0000", 2023-10-09T06:04:09.201 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00040508599999999998 2023-10-09T06:04:09.201 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.201 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.201 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:09.201 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.893721+0000", 2023-10-09T06:04:09.201 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0083789090000000004 2023-10-09T06:04:09.202 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.202 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.202 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.202 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902271+0000", 2023-10-09T06:04:09.202 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0085503039999999999 2023-10-09T06:04:09.202 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.202 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.202 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.202 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902383+0000", 2023-10-09T06:04:09.202 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000111969 2023-10-09T06:04:09.202 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.203 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.203 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.203 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902489+0000", 2023-10-09T06:04:09.203 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010552099999999999 2023-10-09T06:04:09.203 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.203 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.203 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:09.203 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902536+0000", 2023-10-09T06:04:09.203 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7466999999999998e-05 2023-10-09T06:04:09.203 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.203 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.203 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:09.204 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902637+0000", 2023-10-09T06:04:09.204 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010043999999999999 2023-10-09T06:04:09.204 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.204 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:09.204 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.204 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.204 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.204 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:185895 6.15 6:a9b7b9d9:::benchmark_data_smithi064_284193_object89352:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:04:09.204 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.726247+0000", 2023-10-09T06:04:09.204 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 378.44283753299999, 2023-10-09T06:04:09.204 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1767582440000002, 2023-10-09T06:04:09.205 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:09.205 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:09.205 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:09.205 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:04:09.205 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:04:09.205 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 185895 2023-10-09T06:04:09.205 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.205 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:09.205 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.205 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:09.205 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.726247+0000", 2023-10-09T06:04:09.205 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.206 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.206 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.206 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:09.206 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.726247+0000", 2023-10-09T06:04:09.206 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.206 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.206 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.206 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:09.206 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.726245+0000", 2023-10-09T06:04:09.206 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:04:09.206 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.207 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.207 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:09.207 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.726252+0000", 2023-10-09T06:04:09.207 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1350000000000003e-06 2023-10-09T06:04:09.207 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.207 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.207 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:09.207 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.726264+0000", 2023-10-09T06:04:09.207 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2021000000000001e-05 2023-10-09T06:04:09.207 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.207 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.207 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:09.208 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.726271+0000", 2023-10-09T06:04:09.208 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6869999999999997e-06 2023-10-09T06:04:09.208 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.208 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.208 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.208 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.726427+0000", 2023-10-09T06:04:09.208 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015600299999999999 2023-10-09T06:04:09.208 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.208 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.208 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting_for_map not empty", 2023-10-09T06:04:09.208 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.726433+0000", 2023-10-09T06:04:09.208 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9449999999999998e-06 2023-10-09T06:04:09.209 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.209 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.209 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.209 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.732127+0000", 2023-10-09T06:04:09.209 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0056935670000000001 2023-10-09T06:04:09.209 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.209 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.209 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.209 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.732133+0000", 2023-10-09T06:04:09.209 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1999999999999999e-06 2023-10-09T06:04:09.209 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.210 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.210 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.210 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735842+0000", 2023-10-09T06:04:09.210 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0037096569999999 2023-10-09T06:04:09.210 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.210 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.210 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.210 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735851+0000", 2023-10-09T06:04:09.210 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.3650000000000004e-06 2023-10-09T06:04:09.210 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.211 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.211 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.211 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887467+0000", 2023-10-09T06:04:09.211 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1516161149999999 2023-10-09T06:04:09.211 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.211 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.211 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:09.211 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887570+0000", 2023-10-09T06:04:09.211 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000103458 2023-10-09T06:04:09.211 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.211 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.211 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T06:04:09.212 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887826+0000", 2023-10-09T06:04:09.212 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000255212 2023-10-09T06:04:09.212 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.212 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.212 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:09.212 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.893729+0000", 2023-10-09T06:04:09.212 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0059037949999999999 2023-10-09T06:04:09.212 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.212 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.212 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.212 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902299+0000", 2023-10-09T06:04:09.212 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0085701249999999996 2023-10-09T06:04:09.213 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.213 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.213 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.213 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902678+0000", 2023-10-09T06:04:09.213 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000379069 2023-10-09T06:04:09.213 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.213 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.213 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.213 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902845+0000", 2023-10-09T06:04:09.213 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00016605400000000001 2023-10-09T06:04:09.213 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.213 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.214 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:09.214 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902896+0000", 2023-10-09T06:04:09.214 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1298999999999998e-05 2023-10-09T06:04:09.214 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.214 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.214 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:09.214 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.903005+0000", 2023-10-09T06:04:09.214 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000109002 2023-10-09T06:04:09.214 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.214 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:09.214 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.215 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.215 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.215 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:185920 6.15 6:aed0640a:::benchmark_data_smithi064_284193_object89377:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:04:09.215 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.758471+0000", 2023-10-09T06:04:09.215 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 378.41061275800001, 2023-10-09T06:04:09.215 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1447898919999999, 2023-10-09T06:04:09.215 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:09.215 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:09.215 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:09.215 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:04:09.215 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:04:09.215 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 185920 2023-10-09T06:04:09.216 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.216 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:09.216 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.216 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:09.216 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.758471+0000", 2023-10-09T06:04:09.216 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.216 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.216 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.216 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:09.216 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.758471+0000", 2023-10-09T06:04:09.216 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.216 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.217 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.217 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:09.217 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.758470+0000", 2023-10-09T06:04:09.217 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:04:09.217 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.217 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.217 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:09.217 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.758476+0000", 2023-10-09T06:04:09.217 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3690000000000003e-06 2023-10-09T06:04:09.217 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.217 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.217 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:09.218 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.758490+0000", 2023-10-09T06:04:09.218 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3767e-05 2023-10-09T06:04:09.218 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.218 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.218 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:09.218 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.758496+0000", 2023-10-09T06:04:09.218 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0610000000000001e-06 2023-10-09T06:04:09.218 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.218 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.218 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.218 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.758550+0000", 2023-10-09T06:04:09.219 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3894000000000003e-05 2023-10-09T06:04:09.219 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.219 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.219 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.219 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.758556+0000", 2023-10-09T06:04:09.219 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6799999999999998e-06 2023-10-09T06:04:09.219 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.219 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.219 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.219 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735869+0000", 2023-10-09T06:04:09.219 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.97731343199999998 2023-10-09T06:04:09.219 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.220 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.220 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.220 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735875+0000", 2023-10-09T06:04:09.220 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0639999999999999e-06 2023-10-09T06:04:09.220 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.220 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.220 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.220 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.889339+0000", 2023-10-09T06:04:09.220 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1534636359999999 2023-10-09T06:04:09.220 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.220 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.220 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:09.221 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.889423+0000", 2023-10-09T06:04:09.221 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.4303999999999998e-05 2023-10-09T06:04:09.221 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.221 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.221 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T06:04:09.221 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.889619+0000", 2023-10-09T06:04:09.221 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000195606 2023-10-09T06:04:09.221 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.221 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.221 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:09.221 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.896800+0000", 2023-10-09T06:04:09.221 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0071807390000000002 2023-10-09T06:04:09.222 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.222 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.222 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.222 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902329+0000", 2023-10-09T06:04:09.222 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0055295759999999996 2023-10-09T06:04:09.222 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.222 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.222 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.222 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902708+0000", 2023-10-09T06:04:09.222 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00037905300000000002 2023-10-09T06:04:09.222 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.222 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.223 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.223 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.903105+0000", 2023-10-09T06:04:09.223 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00039682200000000002 2023-10-09T06:04:09.223 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.223 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.223 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:09.223 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.903155+0000", 2023-10-09T06:04:09.223 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9982999999999999e-05 2023-10-09T06:04:09.223 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.223 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.223 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:09.223 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.903261+0000", 2023-10-09T06:04:09.224 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010619399999999999 2023-10-09T06:04:09.224 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.224 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:09.224 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.224 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.224 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.224 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:185935 6.25 6:a62e1056:::benchmark_data_smithi064_284193_object89392:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:04:09.224 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.768157+0000", 2023-10-09T06:04:09.224 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 378.40092669900002, 2023-10-09T06:04:09.224 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1328383999999998, 2023-10-09T06:04:09.224 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:09.224 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:09.225 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:09.225 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:04:09.225 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:04:09.225 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 185935 2023-10-09T06:04:09.225 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.225 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:09.225 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.225 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:09.225 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.768157+0000", 2023-10-09T06:04:09.225 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.225 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.226 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.226 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:09.226 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.768157+0000", 2023-10-09T06:04:09.226 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.226 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.226 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.226 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:09.226 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.768157+0000", 2023-10-09T06:04:09.226 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:04:09.226 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.226 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.227 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:09.227 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.768166+0000", 2023-10-09T06:04:09.227 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9919999999999996e-06 2023-10-09T06:04:09.227 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.227 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.227 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:09.227 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.768179+0000", 2023-10-09T06:04:09.227 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3552e-05 2023-10-09T06:04:09.227 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.227 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.227 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:09.227 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.768186+0000", 2023-10-09T06:04:09.228 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5459999999999997e-06 2023-10-09T06:04:09.228 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.228 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.228 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.228 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.768252+0000", 2023-10-09T06:04:09.228 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6629000000000002e-05 2023-10-09T06:04:09.228 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.228 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.228 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.228 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.768259+0000", 2023-10-09T06:04:09.228 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4389999999999997e-06 2023-10-09T06:04:09.229 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.229 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.229 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.229 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737077+0000", 2023-10-09T06:04:09.229 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96881812599999995 2023-10-09T06:04:09.229 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.229 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.229 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.229 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737085+0000", 2023-10-09T06:04:09.229 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.4970000000000001e-06 2023-10-09T06:04:09.229 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.229 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.230 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.230 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.870073+0000", 2023-10-09T06:04:09.230 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1329872590000001 2023-10-09T06:04:09.230 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.230 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.230 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:09.230 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.870168+0000", 2023-10-09T06:04:09.230 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.5539999999999997e-05 2023-10-09T06:04:09.230 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.230 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.230 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:04:09.230 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.870401+0000", 2023-10-09T06:04:09.231 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000232465 2023-10-09T06:04:09.231 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.231 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.231 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:09.231 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887473+0000", 2023-10-09T06:04:09.231 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.017071975999999999 2023-10-09T06:04:09.231 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.231 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.231 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.231 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900782+0000", 2023-10-09T06:04:09.231 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013308943 2023-10-09T06:04:09.232 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.232 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.232 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.232 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900835+0000", 2023-10-09T06:04:09.232 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3684999999999998e-05 2023-10-09T06:04:09.232 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.232 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.232 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:09.232 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900869+0000", 2023-10-09T06:04:09.232 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.3395000000000001e-05 2023-10-09T06:04:09.232 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.232 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.233 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:09.233 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900996+0000", 2023-10-09T06:04:09.233 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012716 2023-10-09T06:04:09.233 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.233 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:09.233 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.233 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.233 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.233 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:185975 6.15 6:ad237c98:::benchmark_data_smithi064_284193_object89432:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:04:09.233 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.808092+0000", 2023-10-09T06:04:09.233 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 378.36099234300002, 2023-10-09T06:04:09.233 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.100164849, 2023-10-09T06:04:09.234 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:09.234 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:09.234 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:09.234 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:04:09.234 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:04:09.234 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 185975 2023-10-09T06:04:09.234 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.234 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:09.234 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.234 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:09.234 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.808092+0000", 2023-10-09T06:04:09.235 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.235 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.235 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.235 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:09.235 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.808092+0000", 2023-10-09T06:04:09.235 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.235 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.235 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.235 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:09.235 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.808091+0000", 2023-10-09T06:04:09.235 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:04:09.235 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.236 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.236 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:09.236 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.808098+0000", 2023-10-09T06:04:09.236 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8979999999999999e-06 2023-10-09T06:04:09.236 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.236 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.236 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:09.236 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.808110+0000", 2023-10-09T06:04:09.236 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.269e-05 2023-10-09T06:04:09.236 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.236 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.236 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:09.237 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.808118+0000", 2023-10-09T06:04:09.237 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1369999999999996e-06 2023-10-09T06:04:09.237 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.237 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.237 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.237 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.808181+0000", 2023-10-09T06:04:09.237 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3409000000000002e-05 2023-10-09T06:04:09.237 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.237 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.237 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.237 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.808190+0000", 2023-10-09T06:04:09.238 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8319999999999995e-06 2023-10-09T06:04:09.238 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.238 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.238 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.238 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735892+0000", 2023-10-09T06:04:09.238 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.92770270899999996 2023-10-09T06:04:09.238 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.238 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.238 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.238 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735904+0000", 2023-10-09T06:04:09.238 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1919999999999999e-05 2023-10-09T06:04:09.238 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.239 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.239 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.239 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.890799+0000", 2023-10-09T06:04:09.239 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1548949850000001 2023-10-09T06:04:09.239 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.239 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.239 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:09.239 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.890900+0000", 2023-10-09T06:04:09.239 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000100787 2023-10-09T06:04:09.239 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.239 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.240 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T06:04:09.240 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.891095+0000", 2023-10-09T06:04:09.240 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00019488099999999999 2023-10-09T06:04:09.240 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.240 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.240 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:09.240 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.896810+0000", 2023-10-09T06:04:09.240 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0057148370000000004 2023-10-09T06:04:09.240 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.240 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.240 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.240 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902357+0000", 2023-10-09T06:04:09.241 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0055466739999999997 2023-10-09T06:04:09.241 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.241 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.241 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.241 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902737+0000", 2023-10-09T06:04:09.241 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00038066800000000002 2023-10-09T06:04:09.241 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.241 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.241 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.241 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908154+0000", 2023-10-09T06:04:09.241 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0054167620000000003 2023-10-09T06:04:09.241 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.242 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.242 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:09.242 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908191+0000", 2023-10-09T06:04:09.242 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7197000000000003e-05 2023-10-09T06:04:09.242 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.242 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.242 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:09.242 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908257+0000", 2023-10-09T06:04:09.242 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5439999999999997e-05 2023-10-09T06:04:09.242 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.242 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:09.243 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.243 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.243 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.243 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:185985 6.5 6:a34f1df4:::benchmark_data_smithi064_284193_object89442:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:04:09.243 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.815985+0000", 2023-10-09T06:04:09.243 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 378.35309909199998, 2023-10-09T06:04:09.243 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0848691029999999, 2023-10-09T06:04:09.243 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:09.243 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:09.243 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:09.243 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:04:09.245 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:04:09.245 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 185985 2023-10-09T06:04:09.245 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.246 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:09.246 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.246 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:09.246 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.815985+0000", 2023-10-09T06:04:09.246 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.247 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.247 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.247 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:09.247 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.815985+0000", 2023-10-09T06:04:09.247 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.248 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.248 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.248 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:09.248 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.815984+0000", 2023-10-09T06:04:09.248 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:04:09.249 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.249 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.249 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:09.249 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.815991+0000", 2023-10-09T06:04:09.249 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.973e-06 2023-10-09T06:04:09.250 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.250 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.250 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:09.250 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.816004+0000", 2023-10-09T06:04:09.250 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3162999999999999e-05 2023-10-09T06:04:09.251 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.251 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.251 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:09.251 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.816009+0000", 2023-10-09T06:04:09.251 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8049999999999997e-06 2023-10-09T06:04:09.252 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.252 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.252 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.252 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.816081+0000", 2023-10-09T06:04:09.252 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2614000000000006e-05 2023-10-09T06:04:09.252 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.252 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.253 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.253 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.816093+0000", 2023-10-09T06:04:09.253 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1919999999999999e-05 2023-10-09T06:04:09.253 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.253 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.253 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.254 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.738574+0000", 2023-10-09T06:04:09.254 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.922481047 2023-10-09T06:04:09.254 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.254 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.254 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.255 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.738585+0000", 2023-10-09T06:04:09.255 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0457e-05 2023-10-09T06:04:09.255 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.255 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.255 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.255 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.860265+0000", 2023-10-09T06:04:09.256 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1216806349999999 2023-10-09T06:04:09.256 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.256 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.256 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:09.256 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.860401+0000", 2023-10-09T06:04:09.257 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000135158 2023-10-09T06:04:09.257 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.257 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.257 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:04:09.257 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.860685+0000", 2023-10-09T06:04:09.257 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00028465499999999998 2023-10-09T06:04:09.258 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.258 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.258 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.258 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.875933+0000", 2023-10-09T06:04:09.258 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015247671000000001 2023-10-09T06:04:09.259 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.259 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.259 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:09.259 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887462+0000", 2023-10-09T06:04:09.259 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011528974000000001 2023-10-09T06:04:09.259 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.260 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.260 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.260 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900446+0000", 2023-10-09T06:04:09.260 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012984446 2023-10-09T06:04:09.260 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.261 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.261 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:09.261 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900505+0000", 2023-10-09T06:04:09.261 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8844000000000001e-05 2023-10-09T06:04:09.261 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.261 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.262 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:09.262 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900854+0000", 2023-10-09T06:04:09.262 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00034893000000000001 2023-10-09T06:04:09.262 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.262 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:09.263 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.263 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.263 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.263 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186010 6.25 6:a4768b4e:::benchmark_data_smithi064_284193_object89467:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:04:09.263 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.842412+0000", 2023-10-09T06:04:09.264 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 378.32667170899998, 2023-10-09T06:04:09.264 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.058999177, 2023-10-09T06:04:09.264 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:09.264 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:09.264 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:09.264 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:04:09.264 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:04:09.265 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186010 2023-10-09T06:04:09.265 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.265 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:09.265 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.265 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:09.265 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842412+0000", 2023-10-09T06:04:09.265 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.266 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.266 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.266 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:09.266 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842412+0000", 2023-10-09T06:04:09.266 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.266 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.267 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.267 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:09.267 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842411+0000", 2023-10-09T06:04:09.267 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:04:09.267 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.267 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.267 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:09.267 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842424+0000", 2023-10-09T06:04:09.268 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2785999999999999e-05 2023-10-09T06:04:09.268 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.268 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.268 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:09.268 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842440+0000", 2023-10-09T06:04:09.268 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5608e-05 2023-10-09T06:04:09.268 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.268 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.268 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:09.268 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842445+0000", 2023-10-09T06:04:09.268 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5300000000000004e-06 2023-10-09T06:04:09.269 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.269 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.269 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.269 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842503+0000", 2023-10-09T06:04:09.269 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7425e-05 2023-10-09T06:04:09.269 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.269 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.269 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.269 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842512+0000", 2023-10-09T06:04:09.269 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3090000000000001e-06 2023-10-09T06:04:09.269 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.269 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.270 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.270 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737132+0000", 2023-10-09T06:04:09.270 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.89461971600000001 2023-10-09T06:04:09.270 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.270 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.270 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.270 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737140+0000", 2023-10-09T06:04:09.270 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0819999999999999e-06 2023-10-09T06:04:09.270 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.270 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.270 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.271 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.871339+0000", 2023-10-09T06:04:09.271 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1341993539999999 2023-10-09T06:04:09.271 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.271 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.271 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:09.271 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.871455+0000", 2023-10-09T06:04:09.271 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000116223 2023-10-09T06:04:09.271 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.271 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.271 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:04:09.271 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.871601+0000", 2023-10-09T06:04:09.272 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000145215 2023-10-09T06:04:09.272 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.272 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.272 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:09.272 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887480+0000", 2023-10-09T06:04:09.272 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015879000000000001 2023-10-09T06:04:09.272 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.272 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.272 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.272 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900800+0000", 2023-10-09T06:04:09.272 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013320784 2023-10-09T06:04:09.272 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.273 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.273 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.273 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901152+0000", 2023-10-09T06:04:09.273 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00035196099999999997 2023-10-09T06:04:09.273 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.273 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.273 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:09.273 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901201+0000", 2023-10-09T06:04:09.273 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9029000000000001e-05 2023-10-09T06:04:09.273 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.273 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.274 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:09.274 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901412+0000", 2023-10-09T06:04:09.274 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00021017099999999999 2023-10-09T06:04:09.274 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.274 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:09.274 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.274 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.274 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.274 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186011 6.25 6:a4aaf67a:::benchmark_data_smithi064_284193_object89468:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:04:09.274 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.842461+0000", 2023-10-09T06:04:09.274 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 378.326623316, 2023-10-09T06:04:09.275 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.059291977, 2023-10-09T06:04:09.275 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:09.275 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:09.275 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:09.275 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:04:09.275 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:04:09.275 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186011 2023-10-09T06:04:09.275 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.275 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:09.275 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.275 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:09.275 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842461+0000", 2023-10-09T06:04:09.276 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.276 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.276 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.276 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:09.276 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842461+0000", 2023-10-09T06:04:09.276 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.276 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.276 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.276 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:09.276 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842460+0000", 2023-10-09T06:04:09.276 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:04:09.276 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.277 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.277 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:09.277 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842464+0000", 2023-10-09T06:04:09.277 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5949999999999999e-06 2023-10-09T06:04:09.277 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.277 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.277 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:09.277 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842470+0000", 2023-10-09T06:04:09.277 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4500000000000001e-06 2023-10-09T06:04:09.277 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.277 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.277 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:09.278 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842475+0000", 2023-10-09T06:04:09.278 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1629999999999998e-06 2023-10-09T06:04:09.278 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.278 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.278 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.278 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.844758+0000", 2023-10-09T06:04:09.278 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002283799 2023-10-09T06:04:09.278 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.278 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.278 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.278 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.844765+0000", 2023-10-09T06:04:09.279 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5250000000000002e-06 2023-10-09T06:04:09.279 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.279 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.279 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.279 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737182+0000", 2023-10-09T06:04:09.279 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.89241688699999999 2023-10-09T06:04:09.279 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.279 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.279 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.280 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737191+0000", 2023-10-09T06:04:09.280 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0219999999999993e-06 2023-10-09T06:04:09.280 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.280 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.280 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.280 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.872509+0000", 2023-10-09T06:04:09.280 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1353186339999999 2023-10-09T06:04:09.280 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.280 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.280 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:09.281 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.872596+0000", 2023-10-09T06:04:09.281 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6112000000000005e-05 2023-10-09T06:04:09.281 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.281 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.281 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:04:09.281 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.872731+0000", 2023-10-09T06:04:09.281 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000134964 2023-10-09T06:04:09.281 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.281 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.281 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:09.281 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887486+0000", 2023-10-09T06:04:09.282 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.014755367 2023-10-09T06:04:09.282 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.282 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.282 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.282 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900818+0000", 2023-10-09T06:04:09.282 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013332274 2023-10-09T06:04:09.282 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.282 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.283 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.283 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901502+0000", 2023-10-09T06:04:09.283 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00068389899999999997 2023-10-09T06:04:09.283 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.283 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.283 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:09.283 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901562+0000", 2023-10-09T06:04:09.283 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0087999999999998e-05 2023-10-09T06:04:09.283 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.283 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.283 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:09.284 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901753+0000", 2023-10-09T06:04:09.284 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00019062199999999999 2023-10-09T06:04:09.284 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.284 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:09.284 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.284 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.284 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.284 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186073 6.15 6:ac3db1f6:::benchmark_data_smithi064_284193_object89530:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:04:09.284 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.899281+0000", 2023-10-09T06:04:09.284 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 378.26980302300001, 2023-10-09T06:04:09.284 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0091446479999999, 2023-10-09T06:04:09.285 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:09.285 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:09.285 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:09.285 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:04:09.285 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:04:09.285 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186073 2023-10-09T06:04:09.285 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.285 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:09.285 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.285 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:09.285 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.899281+0000", 2023-10-09T06:04:09.286 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.286 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.286 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.286 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:09.286 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.899281+0000", 2023-10-09T06:04:09.286 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.286 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.286 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.287 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:09.287 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.899280+0000", 2023-10-09T06:04:09.287 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:04:09.287 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.287 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.287 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:09.287 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.899285+0000", 2023-10-09T06:04:09.287 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7600000000000002e-06 2023-10-09T06:04:09.287 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.287 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.288 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:09.288 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.899294+0000", 2023-10-09T06:04:09.288 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.397e-06 2023-10-09T06:04:09.288 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.288 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.288 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:09.288 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.899300+0000", 2023-10-09T06:04:09.288 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.412e-06 2023-10-09T06:04:09.288 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.288 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.288 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.289 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.899361+0000", 2023-10-09T06:04:09.289 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1532999999999995e-05 2023-10-09T06:04:09.289 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.289 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.289 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.289 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.899370+0000", 2023-10-09T06:04:09.289 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.4649999999999994e-06 2023-10-09T06:04:09.289 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.289 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.289 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.289 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735922+0000", 2023-10-09T06:04:09.290 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.83655174300000001 2023-10-09T06:04:09.290 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.290 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.290 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.290 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735928+0000", 2023-10-09T06:04:09.290 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4790000000000001e-06 2023-10-09T06:04:09.290 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.290 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.290 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.290 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.892186+0000", 2023-10-09T06:04:09.291 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.156257533 2023-10-09T06:04:09.291 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.291 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.291 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:09.291 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.892275+0000", 2023-10-09T06:04:09.291 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9951000000000007e-05 2023-10-09T06:04:09.291 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.291 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.291 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T06:04:09.291 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.892504+0000", 2023-10-09T06:04:09.291 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00022880200000000001 2023-10-09T06:04:09.292 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.292 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.292 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:09.292 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900294+0000", 2023-10-09T06:04:09.292 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.007790033 2023-10-09T06:04:09.292 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.292 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.292 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.292 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902411+0000", 2023-10-09T06:04:09.292 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0021167360000000001 2023-10-09T06:04:09.292 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.292 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.293 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.293 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.906259+0000", 2023-10-09T06:04:09.293 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0038482970000000001 2023-10-09T06:04:09.293 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.293 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.293 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.293 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908294+0000", 2023-10-09T06:04:09.293 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00203426 2023-10-09T06:04:09.293 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.293 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.293 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:09.294 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908329+0000", 2023-10-09T06:04:09.294 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5098999999999997e-05 2023-10-09T06:04:09.294 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.294 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.294 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:09.294 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908426+0000", 2023-10-09T06:04:09.294 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7058000000000005e-05 2023-10-09T06:04:09.294 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.294 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:09.295 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.295 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.295 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.295 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186096 6.25 6:a53a4882:::benchmark_data_smithi064_284193_object89553:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:04:09.295 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.926438+0000", 2023-10-09T06:04:09.295 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 378.24264645900001, 2023-10-09T06:04:09.295 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.975667222, 2023-10-09T06:04:09.295 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:09.295 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:09.295 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:09.296 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:04:09.296 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:04:09.296 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186096 2023-10-09T06:04:09.296 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.296 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:09.296 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.297 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:09.297 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.926438+0000", 2023-10-09T06:04:09.297 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.297 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.297 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.298 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:09.298 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.926438+0000", 2023-10-09T06:04:09.298 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.298 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.298 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.298 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:09.298 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.926436+0000", 2023-10-09T06:04:09.298 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:04:09.298 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.299 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.299 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:09.299 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.926443+0000", 2023-10-09T06:04:09.299 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3099999999999997e-06 2023-10-09T06:04:09.299 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.299 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.299 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:09.299 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.926455+0000", 2023-10-09T06:04:09.299 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2026e-05 2023-10-09T06:04:09.299 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.299 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.299 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:09.300 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.926460+0000", 2023-10-09T06:04:09.300 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8379999999999998e-06 2023-10-09T06:04:09.300 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.300 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.300 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.300 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.926522+0000", 2023-10-09T06:04:09.300 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1502999999999997e-05 2023-10-09T06:04:09.300 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.300 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.300 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.300 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.926531+0000", 2023-10-09T06:04:09.300 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6559999999999998e-06 2023-10-09T06:04:09.301 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.301 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.301 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.301 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737222+0000", 2023-10-09T06:04:09.301 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.81069113400000004 2023-10-09T06:04:09.301 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.301 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.301 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.301 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737229+0000", 2023-10-09T06:04:09.302 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6669999999999996e-06 2023-10-09T06:04:09.302 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.302 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.302 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.302 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.873463+0000", 2023-10-09T06:04:09.302 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1362332390000001 2023-10-09T06:04:09.302 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.302 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.302 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:09.303 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.873604+0000", 2023-10-09T06:04:09.303 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000141108 2023-10-09T06:04:09.303 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.303 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.303 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:04:09.303 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.873782+0000", 2023-10-09T06:04:09.303 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00017831699999999999 2023-10-09T06:04:09.303 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.303 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.303 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:09.303 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887492+0000", 2023-10-09T06:04:09.304 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013710119 2023-10-09T06:04:09.304 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.304 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.304 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.304 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901054+0000", 2023-10-09T06:04:09.304 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013561478 2023-10-09T06:04:09.304 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.304 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.304 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.305 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901854+0000", 2023-10-09T06:04:09.305 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00079998700000000005 2023-10-09T06:04:09.305 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.305 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.305 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:09.305 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901914+0000", 2023-10-09T06:04:09.305 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0770000000000003e-05 2023-10-09T06:04:09.305 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.305 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.305 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:09.305 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902105+0000", 2023-10-09T06:04:09.306 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000190497 2023-10-09T06:04:09.306 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.306 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:09.306 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.306 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.306 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.306 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186101 6.15 6:ac185365:::benchmark_data_smithi064_284193_object89558:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:04:09.306 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.933029+0000", 2023-10-09T06:04:09.306 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 378.23605562799997, 2023-10-09T06:04:09.306 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.97558872, 2023-10-09T06:04:09.306 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:09.307 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:09.307 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:09.307 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:04:09.307 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:04:09.307 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186101 2023-10-09T06:04:09.307 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.307 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:09.307 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.307 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:09.307 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.933029+0000", 2023-10-09T06:04:09.307 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.307 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.308 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.308 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:09.308 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.933029+0000", 2023-10-09T06:04:09.308 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.308 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.308 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.308 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:09.308 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.933027+0000", 2023-10-09T06:04:09.308 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:04:09.308 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.308 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.309 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:09.309 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.933033+0000", 2023-10-09T06:04:09.309 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1380000000000004e-06 2023-10-09T06:04:09.309 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.309 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.309 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:09.309 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.933048+0000", 2023-10-09T06:04:09.309 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4273e-05 2023-10-09T06:04:09.309 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.309 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.309 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:09.310 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.933053+0000", 2023-10-09T06:04:09.310 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.699e-06 2023-10-09T06:04:09.310 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.310 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.310 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.310 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.933117+0000", 2023-10-09T06:04:09.310 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3668000000000001e-05 2023-10-09T06:04:09.310 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.310 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.310 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.310 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.933125+0000", 2023-10-09T06:04:09.311 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8990000000000001e-06 2023-10-09T06:04:09.311 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.311 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.311 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.311 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735946+0000", 2023-10-09T06:04:09.311 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.802820796 2023-10-09T06:04:09.311 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.311 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.311 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.311 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735952+0000", 2023-10-09T06:04:09.311 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3230000000000003e-06 2023-10-09T06:04:09.312 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.312 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.312 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.312 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.893777+0000", 2023-10-09T06:04:09.312 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1578248010000001 2023-10-09T06:04:09.312 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.312 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.312 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:09.312 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.893902+0000", 2023-10-09T06:04:09.312 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012566899999999999 2023-10-09T06:04:09.312 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.312 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.313 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T06:04:09.313 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.894163+0000", 2023-10-09T06:04:09.313 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000260253 2023-10-09T06:04:09.313 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.313 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.313 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:09.313 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900304+0000", 2023-10-09T06:04:09.313 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0061414939999999999 2023-10-09T06:04:09.313 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.313 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.313 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.314 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902438+0000", 2023-10-09T06:04:09.314 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0021343109999999998 2023-10-09T06:04:09.314 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.314 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.314 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.314 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.906306+0000", 2023-10-09T06:04:09.314 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0038679690000000002 2023-10-09T06:04:09.314 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.314 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.314 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.314 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908502+0000", 2023-10-09T06:04:09.315 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00219549 2023-10-09T06:04:09.315 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.315 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.315 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:09.315 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908544+0000", 2023-10-09T06:04:09.315 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2407000000000002e-05 2023-10-09T06:04:09.315 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.315 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.315 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:09.315 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908617+0000", 2023-10-09T06:04:09.315 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2891000000000004e-05 2023-10-09T06:04:09.316 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.316 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:09.316 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.316 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.316 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.316 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186105 6.15 6:a80320df:::benchmark_data_smithi064_284193_object89562:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:04:09.316 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.936806+0000", 2023-10-09T06:04:09.316 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 378.23227819200002, 2023-10-09T06:04:09.316 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.971922425, 2023-10-09T06:04:09.316 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:09.316 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:09.316 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:09.317 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:04:09.317 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:04:09.317 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186105 2023-10-09T06:04:09.317 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.317 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:09.317 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.317 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:09.317 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.936806+0000", 2023-10-09T06:04:09.317 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.317 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.317 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.318 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:09.318 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.936806+0000", 2023-10-09T06:04:09.318 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.318 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.318 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.318 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:09.318 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.936805+0000", 2023-10-09T06:04:09.318 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:04:09.318 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.318 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.318 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:09.319 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.936812+0000", 2023-10-09T06:04:09.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0870000000000001e-06 2023-10-09T06:04:09.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.319 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.319 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:09.319 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.936826+0000", 2023-10-09T06:04:09.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4511999999999999e-05 2023-10-09T06:04:09.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.319 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.319 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:09.319 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.936833+0000", 2023-10-09T06:04:09.320 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9140000000000002e-06 2023-10-09T06:04:09.320 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.320 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.320 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.320 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.936903+0000", 2023-10-09T06:04:09.320 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0197999999999994e-05 2023-10-09T06:04:09.320 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.320 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.320 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.320 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.936913+0000", 2023-10-09T06:04:09.320 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3549999999999994e-06 2023-10-09T06:04:09.320 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.321 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.321 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.321 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735976+0000", 2023-10-09T06:04:09.321 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.79906334099999998 2023-10-09T06:04:09.321 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.321 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.321 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.321 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735983+0000", 2023-10-09T06:04:09.321 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8859999999999998e-06 2023-10-09T06:04:09.321 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.321 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.321 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.322 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.895376+0000", 2023-10-09T06:04:09.322 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.159393398 2023-10-09T06:04:09.322 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.322 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.322 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:09.322 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.895479+0000", 2023-10-09T06:04:09.322 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000103028 2023-10-09T06:04:09.322 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.322 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.322 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T06:04:09.323 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.895680+0000", 2023-10-09T06:04:09.323 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000200216 2023-10-09T06:04:09.323 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.323 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.323 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:09.323 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900382+0000", 2023-10-09T06:04:09.323 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0047027529999999996 2023-10-09T06:04:09.323 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.323 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.323 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.323 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902767+0000", 2023-10-09T06:04:09.323 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0023844389999999999 2023-10-09T06:04:09.324 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.324 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.324 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.324 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.906331+0000", 2023-10-09T06:04:09.324 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0035642719999999998 2023-10-09T06:04:09.324 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.324 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.324 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.324 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908649+0000", 2023-10-09T06:04:09.324 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0023178819999999998 2023-10-09T06:04:09.325 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.325 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.325 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:09.325 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908673+0000", 2023-10-09T06:04:09.325 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4329999999999999e-05 2023-10-09T06:04:09.325 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.325 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.325 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:09.325 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908728+0000", 2023-10-09T06:04:09.325 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5074999999999997e-05 2023-10-09T06:04:09.325 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.326 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:09.326 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.326 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.326 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.326 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186107 6.15 6:adc2ed22:::benchmark_data_smithi064_284193_object89564:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:04:09.326 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.937578+0000", 2023-10-09T06:04:09.326 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 378.23150618199998, 2023-10-09T06:04:09.326 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9730703780000001, 2023-10-09T06:04:09.326 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:09.326 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:09.327 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:09.327 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:04:09.327 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:04:09.327 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186107 2023-10-09T06:04:09.327 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.327 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:09.327 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.327 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:09.327 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.937578+0000", 2023-10-09T06:04:09.327 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.327 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.328 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.328 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:09.328 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.937578+0000", 2023-10-09T06:04:09.328 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.328 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.328 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.328 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:09.328 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.937577+0000", 2023-10-09T06:04:09.328 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:04:09.328 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.328 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.329 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:09.329 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.937586+0000", 2023-10-09T06:04:09.329 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0250000000000008e-06 2023-10-09T06:04:09.329 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.329 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.329 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:09.329 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.937601+0000", 2023-10-09T06:04:09.329 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5301000000000001e-05 2023-10-09T06:04:09.329 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.329 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.329 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:09.330 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.937608+0000", 2023-10-09T06:04:09.330 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1689999999999997e-06 2023-10-09T06:04:09.330 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.330 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.330 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.330 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.937654+0000", 2023-10-09T06:04:09.330 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6179e-05 2023-10-09T06:04:09.330 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.330 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.330 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.331 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.937660+0000", 2023-10-09T06:04:09.331 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3640000000000004e-06 2023-10-09T06:04:09.331 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.331 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.331 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.331 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.736002+0000", 2023-10-09T06:04:09.331 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.79834211899999996 2023-10-09T06:04:09.331 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.331 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.331 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.331 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.736009+0000", 2023-10-09T06:04:09.331 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4899999999999997e-06 2023-10-09T06:04:09.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.332 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.332 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.332 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.896857+0000", 2023-10-09T06:04:09.332 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1608479190000001 2023-10-09T06:04:09.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.332 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.332 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:09.332 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.896983+0000", 2023-10-09T06:04:09.332 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000125962 2023-10-09T06:04:09.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.333 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.333 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T06:04:09.333 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.897240+0000", 2023-10-09T06:04:09.333 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000257753 2023-10-09T06:04:09.333 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.333 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.333 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:09.333 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900398+0000", 2023-10-09T06:04:09.333 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0031575539999999999 2023-10-09T06:04:09.333 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.333 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.334 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.334 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902797+0000", 2023-10-09T06:04:09.334 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0023991149999999998 2023-10-09T06:04:09.334 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.334 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.334 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.334 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908771+0000", 2023-10-09T06:04:09.334 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0059742579999999997 2023-10-09T06:04:09.334 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.334 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.334 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.335 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.910483+0000", 2023-10-09T06:04:09.335 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0017115489999999999 2023-10-09T06:04:09.335 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.335 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.335 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:09.335 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.910530+0000", 2023-10-09T06:04:09.335 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7488999999999999e-05 2023-10-09T06:04:09.335 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.335 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.335 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:09.335 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.910648+0000", 2023-10-09T06:04:09.335 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000117941 2023-10-09T06:04:09.336 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.336 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:09.336 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.336 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.336 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.336 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186109 6.15 6:ac6fdfb9:::benchmark_data_smithi064_284193_object89566:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:04:09.336 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.940173+0000", 2023-10-09T06:04:09.336 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 378.22891101099998, 2023-10-09T06:04:09.336 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9706640229999999, 2023-10-09T06:04:09.336 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:09.336 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:09.337 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:09.337 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:04:09.337 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:04:09.337 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186109 2023-10-09T06:04:09.337 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.338 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:09.338 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.338 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:09.338 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.940173+0000", 2023-10-09T06:04:09.338 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.338 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.339 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.339 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:09.339 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.940173+0000", 2023-10-09T06:04:09.339 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.339 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.339 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.340 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:09.340 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.940172+0000", 2023-10-09T06:04:09.340 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:04:09.340 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.340 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.340 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:09.340 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.940177+0000", 2023-10-09T06:04:09.341 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4750000000000004e-06 2023-10-09T06:04:09.341 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.341 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.341 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:09.341 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.940188+0000", 2023-10-09T06:04:09.341 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1508999999999999e-05 2023-10-09T06:04:09.341 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.342 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.342 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:09.342 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.940193+0000", 2023-10-09T06:04:09.342 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.147e-06 2023-10-09T06:04:09.342 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.342 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.343 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.343 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.940244+0000", 2023-10-09T06:04:09.343 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0552e-05 2023-10-09T06:04:09.343 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.343 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.344 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.344 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.940254+0000", 2023-10-09T06:04:09.344 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0096e-05 2023-10-09T06:04:09.344 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.344 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.344 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.344 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.736034+0000", 2023-10-09T06:04:09.345 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.79577992900000005 2023-10-09T06:04:09.345 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.345 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.345 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.345 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.736041+0000", 2023-10-09T06:04:09.345 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.849e-06 2023-10-09T06:04:09.345 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.345 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.346 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.346 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.898427+0000", 2023-10-09T06:04:09.346 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.162385929 2023-10-09T06:04:09.346 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.346 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.346 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:09.346 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.898534+0000", 2023-10-09T06:04:09.347 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000107227 2023-10-09T06:04:09.347 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.347 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.347 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T06:04:09.347 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.898738+0000", 2023-10-09T06:04:09.347 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000203928 2023-10-09T06:04:09.347 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.348 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.348 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:09.348 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900865+0000", 2023-10-09T06:04:09.348 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0021271129999999999 2023-10-09T06:04:09.348 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.348 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.349 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.349 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.903326+0000", 2023-10-09T06:04:09.349 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002460991 2023-10-09T06:04:09.349 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.349 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.349 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.349 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908797+0000", 2023-10-09T06:04:09.350 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0054708930000000001 2023-10-09T06:04:09.350 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.350 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.350 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.350 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.910685+0000", 2023-10-09T06:04:09.350 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0018880940000000001 2023-10-09T06:04:09.351 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.351 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.351 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:09.351 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.910732+0000", 2023-10-09T06:04:09.351 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7129000000000002e-05 2023-10-09T06:04:09.352 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.352 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.352 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:09.352 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.910837+0000", 2023-10-09T06:04:09.352 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000104962 2023-10-09T06:04:09.353 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.353 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:09.353 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.353 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.353 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.353 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186112 6.25 6:a55bc7ba:::benchmark_data_smithi064_284193_object89569:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:04:09.354 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.946091+0000", 2023-10-09T06:04:09.354 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 378.222992723, 2023-10-09T06:04:09.354 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.956330801, 2023-10-09T06:04:09.354 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:09.354 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:09.354 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:09.355 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:04:09.355 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:04:09.355 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186112 2023-10-09T06:04:09.355 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.355 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:09.355 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.355 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:09.356 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.946091+0000", 2023-10-09T06:04:09.356 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.356 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.356 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.356 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:09.357 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.946091+0000", 2023-10-09T06:04:09.357 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.357 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.357 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.357 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:09.357 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.946091+0000", 2023-10-09T06:04:09.358 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:04:09.358 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.358 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.358 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:09.358 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.946095+0000", 2023-10-09T06:04:09.358 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1099999999999996e-06 2023-10-09T06:04:09.358 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.358 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.359 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:09.359 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.946104+0000", 2023-10-09T06:04:09.359 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.687e-06 2023-10-09T06:04:09.359 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.359 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.359 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:09.359 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.946109+0000", 2023-10-09T06:04:09.359 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2700000000000004e-06 2023-10-09T06:04:09.359 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.359 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.360 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.360 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.946170+0000", 2023-10-09T06:04:09.360 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1313000000000001e-05 2023-10-09T06:04:09.360 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.360 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.360 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.360 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.946181+0000", 2023-10-09T06:04:09.360 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0849000000000001e-05 2023-10-09T06:04:09.360 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.360 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.360 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.360 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737296+0000", 2023-10-09T06:04:09.361 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.79111525800000004 2023-10-09T06:04:09.361 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.361 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.361 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:09.361 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737305+0000", 2023-10-09T06:04:09.361 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0000000000000002e-06 2023-10-09T06:04:09.361 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.361 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.361 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.361 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.874487+0000", 2023-10-09T06:04:09.361 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.137181663 2023-10-09T06:04:09.362 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.362 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.362 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:09.362 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.874590+0000", 2023-10-09T06:04:09.362 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000102778 2023-10-09T06:04:09.362 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.362 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.362 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:04:09.362 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.874748+0000", 2023-10-09T06:04:09.362 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015821699999999999 2023-10-09T06:04:09.363 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.363 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.363 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:09.363 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887498+0000", 2023-10-09T06:04:09.363 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012750441 2023-10-09T06:04:09.363 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.363 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.363 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.363 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901113+0000", 2023-10-09T06:04:09.363 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013614316 2023-10-09T06:04:09.363 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.363 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.364 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.364 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902187+0000", 2023-10-09T06:04:09.364 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00107467 2023-10-09T06:04:09.364 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.364 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.364 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:09.364 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902236+0000", 2023-10-09T06:04:09.364 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8364e-05 2023-10-09T06:04:09.364 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.364 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.364 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:09.365 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902422+0000", 2023-10-09T06:04:09.365 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00018654000000000001 2023-10-09T06:04:09.365 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.365 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:09.365 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.365 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.365 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.365 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45263.0:202666 7.5 7:a4298507:::benchmark_data_smithi064_398483_object97313:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-10-09T06:04:09.365 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:02:20.340716+0000", 2023-10-09T06:04:09.365 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 108.82836833499999, 2023-10-09T06:04:09.365 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.25391414099999998, 2023-10-09T06:04:09.365 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:09.366 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:09.366 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:09.366 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45263", 2023-10-09T06:04:09.366 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:04:09.366 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 202666 2023-10-09T06:04:09.366 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.366 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:09.366 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.366 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:09.366 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:20.340716+0000", 2023-10-09T06:04:09.366 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.367 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.367 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.367 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:09.367 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:20.340716+0000", 2023-10-09T06:04:09.367 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.367 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.367 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.367 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:09.367 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:20.340714+0000", 2023-10-09T06:04:09.367 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:04:09.367 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.367 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.368 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:09.368 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:20.340724+0000", 2023-10-09T06:04:09.368 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.006e-05 2023-10-09T06:04:09.368 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.368 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.368 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:09.368 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:20.340738+0000", 2023-10-09T06:04:09.368 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3729e-05 2023-10-09T06:04:09.368 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.369 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.369 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:09.369 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:20.340743+0000", 2023-10-09T06:04:09.369 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9969999999999998e-06 2023-10-09T06:04:09.369 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.369 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.369 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.369 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:20.340800+0000", 2023-10-09T06:04:09.369 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7293999999999997e-05 2023-10-09T06:04:09.369 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.369 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.370 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:09.370 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:20.340941+0000", 2023-10-09T06:04:09.370 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014091799999999999 2023-10-09T06:04:09.370 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.370 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.370 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:04:09.370 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:20.341120+0000", 2023-10-09T06:04:09.370 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00017834799999999999 2023-10-09T06:04:09.370 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.370 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.370 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:09.371 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:20.344451+0000", 2023-10-09T06:04:09.371 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0033314899999999999 2023-10-09T06:04:09.371 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.371 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.371 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.371 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:20.348235+0000", 2023-10-09T06:04:09.371 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0037839700000000002 2023-10-09T06:04:09.371 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.371 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.371 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.371 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:20.594531+0000", 2023-10-09T06:04:09.372 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24629593699999999 2023-10-09T06:04:09.372 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.372 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.372 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:09.372 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:20.594587+0000", 2023-10-09T06:04:09.372 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5850999999999998e-05 2023-10-09T06:04:09.372 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.372 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.372 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:09.372 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:02:20.594630+0000", 2023-10-09T06:04:09.372 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3037000000000001e-05 2023-10-09T06:04:09.373 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.373 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:09.373 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.373 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.373 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.373 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:66805 8.d 8:beda68c4:::benchmark_data_smithi064_457003_object66804:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e370)", 2023-10-09T06:04:09.373 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:03:47.804789+0000", 2023-10-09T06:04:09.373 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 21.364295159000001, 2023-10-09T06:04:09.373 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24902305499999999, 2023-10-09T06:04:09.374 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:09.374 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:09.374 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:09.374 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:04:09.374 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:04:09.374 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 66805 2023-10-09T06:04:09.374 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.374 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:09.374 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.375 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:09.375 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.804789+0000", 2023-10-09T06:04:09.375 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.375 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.375 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.375 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:09.375 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.804789+0000", 2023-10-09T06:04:09.375 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.375 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.375 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.375 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:09.376 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.804787+0000", 2023-10-09T06:04:09.376 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T06:04:09.376 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.376 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.376 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:09.376 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.804830+0000", 2023-10-09T06:04:09.376 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2784999999999998e-05 2023-10-09T06:04:09.376 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.376 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.376 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:09.376 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.804845+0000", 2023-10-09T06:04:09.376 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5191000000000001e-05 2023-10-09T06:04:09.377 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.377 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.377 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:09.377 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.804852+0000", 2023-10-09T06:04:09.377 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5679999999999996e-06 2023-10-09T06:04:09.377 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.377 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.377 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.377 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.804905+0000", 2023-10-09T06:04:09.377 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2880999999999997e-05 2023-10-09T06:04:09.377 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.377 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.378 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:09.378 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.805045+0000", 2023-10-09T06:04:09.378 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014043899999999999 2023-10-09T06:04:09.378 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.378 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.378 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:04:09.378 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.805187+0000", 2023-10-09T06:04:09.378 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000141839 2023-10-09T06:04:09.378 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.378 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.379 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:09.379 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.807163+0000", 2023-10-09T06:04:09.379 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0019761650000000002 2023-10-09T06:04:09.379 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.379 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.379 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.379 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.809580+0000", 2023-10-09T06:04:09.379 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0024167569999999998 2023-10-09T06:04:09.379 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.379 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.379 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.380 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.053712+0000", 2023-10-09T06:04:09.380 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.244131981 2023-10-09T06:04:09.380 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.380 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.380 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:09.380 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.053769+0000", 2023-10-09T06:04:09.380 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7031999999999999e-05 2023-10-09T06:04:09.380 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.380 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.380 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:09.380 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.053812+0000", 2023-10-09T06:04:09.380 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3303000000000002e-05 2023-10-09T06:04:09.381 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.381 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:09.381 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.381 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.381 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.381 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:66806 8.d 8:ba263c5e:::benchmark_data_smithi064_457003_object66805:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e370)", 2023-10-09T06:04:09.381 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:03:47.807829+0000", 2023-10-09T06:04:09.381 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 21.361255287999999, 2023-10-09T06:04:09.381 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.25023572900000002, 2023-10-09T06:04:09.382 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:09.382 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:09.382 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:09.382 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:04:09.382 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:04:09.382 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 66806 2023-10-09T06:04:09.382 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.382 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:09.382 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.382 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:09.383 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.807829+0000", 2023-10-09T06:04:09.383 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.383 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.383 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.383 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:09.383 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.807829+0000", 2023-10-09T06:04:09.383 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:09.383 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.383 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.383 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:09.383 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.807828+0000", 2023-10-09T06:04:09.383 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:04:09.384 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.384 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.384 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:09.384 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.807866+0000", 2023-10-09T06:04:09.384 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.79e-05 2023-10-09T06:04:09.384 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.384 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.384 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:09.384 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.807877+0000", 2023-10-09T06:04:09.384 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1038e-05 2023-10-09T06:04:09.384 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.384 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.385 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:09.385 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.807898+0000", 2023-10-09T06:04:09.385 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0880999999999999e-05 2023-10-09T06:04:09.385 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.385 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.385 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:09.385 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.807949+0000", 2023-10-09T06:04:09.385 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1190000000000003e-05 2023-10-09T06:04:09.385 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.386 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.386 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:09.386 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.808059+0000", 2023-10-09T06:04:09.386 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000110065 2023-10-09T06:04:09.386 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.386 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.386 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:04:09.386 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.808147+0000", 2023-10-09T06:04:09.386 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.7713000000000005e-05 2023-10-09T06:04:09.386 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.387 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.387 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:09.387 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.809657+0000", 2023-10-09T06:04:09.387 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001509987 2023-10-09T06:04:09.387 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.387 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.387 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.387 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.053847+0000", 2023-10-09T06:04:09.387 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24418989899999999 2023-10-09T06:04:09.387 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.387 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.388 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:09.388 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.057977+0000", 2023-10-09T06:04:09.388 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0041304920000000004 2023-10-09T06:04:09.388 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.388 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.388 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:09.388 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.058030+0000", 2023-10-09T06:04:09.388 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2364000000000002e-05 2023-10-09T06:04:09.388 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:09.388 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:09.388 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:09.388 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.058065+0000", 2023-10-09T06:04:09.389 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5048000000000001e-05 2023-10-09T06:04:09.389 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.389 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:09.389 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.389 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:09.389 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:09.389 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:04:09.389 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:09.444 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:09.527 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:09.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:09.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:09.623 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:09.635 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:09.661 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:09.702 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:09.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:09.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:09.755 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 67 16 85778 85762 79.9593 39.9375 0.00419884 0.0124714 2023-10-09T06:04:09.805 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:09.895 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:10.016 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:10.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:10.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:10.094 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:04:10.126 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:10.172 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:10.235 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:10.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:10.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:10.343 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:10.385 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:10.447 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:10.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:10.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:10.554 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:04:10.597 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:10.674 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:04:10.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:10.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:04:10.683 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:10 smithi156 ceph-mon[195101]: pgmap v902: 119 pgs: 119 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 39 MiB/s wr, 625 op/s 2023-10-09T06:04:10.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:10 smithi064 ceph-mon[212750]: pgmap v902: 119 pgs: 119 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 39 MiB/s wr, 625 op/s 2023-10-09T06:04:10.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:10 smithi064 ceph-mon[220044]: pgmap v902: 119 pgs: 119 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 39 MiB/s wr, 625 op/s 2023-10-09T06:04:10.755 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 68 16 86653 86637 79.5878 54.6875 0.0157718 0.0125599 2023-10-09T06:04:10.784 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:04:10.826 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:10.908 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:04:10.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:10.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:04:11.016 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:11.065 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:11.146 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:11.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:11.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:11.252 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:11.328 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:11.449 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:11.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:11.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:11.505 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:04:11.560 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:11.608 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:11.654 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:11.682 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:11.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:11.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:11.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:11 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1780651807' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:04:11.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:11 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:04:11.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:11 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1780651807' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:04:11.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:11 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:04:11.755 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 69 16 87611 87595 79.3022 59.875 0.0151222 0.0126054 2023-10-09T06:04:11.793 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:04:11.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:11 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1780651807' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:04:11.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:11 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:04:11.871 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:12.341 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 0, 4] 2023-10-09T06:04:12.341 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:04:12.341 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-10-09T06:04:12.341 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:04:12.341 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:04:12.342 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-10-09T06:04:12.696 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:04:12.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:12 smithi064 ceph-mon[220044]: pgmap v903: 119 pgs: 119 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 850 op/s 2023-10-09T06:04:12.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:12 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:12.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:12 smithi064 ceph-mon[220044]: failed to encode map e375 with expected crc 2023-10-09T06:04:12.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:12 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:04:12.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:12 smithi064 ceph-mon[220044]: osdmap e375: 8 total, 8 up, 8 in 2023-10-09T06:04:12.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:12 smithi064 ceph-mon[220044]: failed to encode map e375 with expected crc 2023-10-09T06:04:12.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:12.711 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:12 smithi064 ceph-mon[212750]: pgmap v903: 119 pgs: 119 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 850 op/s 2023-10-09T06:04:12.711 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:12 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:12.711 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:12 smithi064 ceph-mon[212750]: failed to encode map e375 with expected crc 2023-10-09T06:04:12.711 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:12 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:04:12.711 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:12 smithi064 ceph-mon[212750]: osdmap e375: 8 total, 8 up, 8 in 2023-10-09T06:04:12.711 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:12 smithi064 ceph-mon[212750]: failed to encode map e375 with expected crc 2023-10-09T06:04:12.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:04:12.755 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 70 16 88429 88413 78.8997 51.125 0.00757541 0.012669 2023-10-09T06:04:12.812 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:12.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:12 smithi156 ceph-mon[195101]: pgmap v903: 119 pgs: 119 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 850 op/s 2023-10-09T06:04:12.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:12 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:12.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:12 smithi156 ceph-mon[195101]: failed to encode map e375 with expected crc 2023-10-09T06:04:12.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:12 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:04:12.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:12 smithi156 ceph-mon[195101]: osdmap e375: 8 total, 8 up, 8 in 2023-10-09T06:04:12.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:12 smithi156 ceph-mon[195101]: failed to encode map e375 with expected crc 2023-10-09T06:04:12.955 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:13.098 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:04:13.137 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:04:13.160 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:13.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:13.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:13.174 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:04:13.217 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:04:13.246 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:04:13.275 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:13.281 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:04:13.324 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:04:13.353 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:04:13.370 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:04:13.370 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:04:13.370 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:04:13.370 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:04:13.370 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:04:13.370 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:04:13.370 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:04:13.370 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:04:13.371 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:13.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:13 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3878588405' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:13.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:13 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3878588405' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:13.756 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 71 16 89302 89286 78.557 54.5625 0.00347285 0.0127191 2023-10-09T06:04:13.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:13 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3878588405' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:14.576 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:14.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:14.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:14.690 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:14.746 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:14 smithi156 ceph-mon[195101]: pgmap v905: 119 pgs: 119 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s 2023-10-09T06:04:14.746 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:14.746 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:04:14.747 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:14.760 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 72 16 90162 90146 78.2096 53.75 0.00815221 0.0127813 2023-10-09T06:04:14.849 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:14.856 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:04:14.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:14.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:14.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:14 smithi064 ceph-mon[212750]: pgmap v905: 119 pgs: 119 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s 2023-10-09T06:04:14.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:14.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:04:14.916 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:14 smithi064 ceph-mon[220044]: pgmap v905: 119 pgs: 119 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s 2023-10-09T06:04:14.916 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:14.916 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:04:14.959 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:04:15.004 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:15.067 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:04:15.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:15.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:04:15.176 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:15.261 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:15.341 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:15.399 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:15.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:15.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:15.506 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:15.507 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:04:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:04:15] "GET /metrics HTTP/1.1" 200 36379 "" "Prometheus/2.43.0" 2023-10-09T06:04:15.545 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:15.622 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:15.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:15.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:15.686 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:15.728 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:15.761 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 73 16 91069 91053 77.9124 56.6875 0.0179006 0.0128294 2023-10-09T06:04:15.764 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:15 smithi156 ceph-mon[195101]: pgmap v906: 119 pgs: 119 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s 2023-10-09T06:04:15.772 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:15.777 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:15 smithi064 ceph-mon[212750]: pgmap v906: 119 pgs: 119 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s 2023-10-09T06:04:15.777 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:15 smithi064 ceph-mon[220044]: pgmap v906: 119 pgs: 119 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s 2023-10-09T06:04:15.829 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:15.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:15.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:15.935 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:04:16.022 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:16.259 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:04:16.259 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:04:16.329 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:04:16.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:16.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:04:16.441 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:16.523 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:16.761 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 74 16 91935 91919 77.591 54.125 0.00857695 0.0128835 2023-10-09T06:04:17.262 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:04:17.761 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 75 16 92653 92637 77.1549 44.875 0.0792111 0.0129465 2023-10-09T06:04:18.140 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:18.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:18.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:18.247 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:04:18.340 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:18.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:18 smithi064 ceph-mon[212750]: pgmap v907: 119 pgs: 119 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s 2023-10-09T06:04:18.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:18 smithi064 ceph-mon[220044]: pgmap v907: 119 pgs: 119 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s 2023-10-09T06:04:18.761 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 76 16 93471 93455 76.8125 51.125 0.02098 0.0130143 2023-10-09T06:04:18.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:18 smithi156 ceph-mon[195101]: pgmap v907: 119 pgs: 119 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s 2023-10-09T06:04:18.950 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:04:18.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:18.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:04:19.059 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:19.071 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:04:19.136 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:19.421 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:19.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:19.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:19.530 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:19.590 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:19.659 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:19.665 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:19.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:19.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:19.763 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 77 16 94293 94277 76.4806 51.375 0.0133866 0.0130687 2023-10-09T06:04:19.772 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:19.785 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:04:19.785 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "" 2023-10-09T06:04:19.785 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:04:19.810 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:19.886 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:19.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:19.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:20.000 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:20.046 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:20.127 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:20.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:20.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:20.235 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:20.288 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:20.374 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:20.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:20.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:20.446 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:04:20.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:20 smithi064 ceph-mon[212750]: pgmap v908: 119 pgs: 119 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s 2023-10-09T06:04:20.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:20 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2319417622' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:20.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:20 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:20.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:20 smithi064 ceph-mon[220044]: pgmap v908: 119 pgs: 119 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s 2023-10-09T06:04:20.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:20 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2319417622' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:20.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:20 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:20.480 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:20.532 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:20.612 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:20.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:20.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:20.617 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:20 smithi156 ceph-mon[195101]: pgmap v908: 119 pgs: 119 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s 2023-10-09T06:04:20.618 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:20 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2319417622' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:20.618 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:20 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:20.718 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:04:20.763 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 78 16 95204 95188 76.2301 56.9375 0.00898748 0.013113 2023-10-09T06:04:20.797 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:21.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:21 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:04:21.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:21 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T06:04:21.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:21 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:04:21.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:21 smithi064 ceph-mon[220044]: osdmap e376: 8 total, 8 up, 8 in 2023-10-09T06:04:21.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:21 smithi064 ceph-mon[220044]: failed to encode map e376 with expected crc 2023-10-09T06:04:21.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:21 smithi064 ceph-mon[220044]: failed to encode map e376 with expected crc 2023-10-09T06:04:21.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:21 smithi064 ceph-mon[220044]: failed to encode map e376 with expected crc 2023-10-09T06:04:21.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:21 smithi064 ceph-mon[220044]: failed to encode map e376 with expected crc 2023-10-09T06:04:21.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:21 smithi064 ceph-mon[220044]: failed to encode map e376 with expected crc 2023-10-09T06:04:21.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:21 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:04:21.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:21 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:04:21.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:21 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:04:21.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:21 smithi064 ceph-mon[212750]: osdmap e376: 8 total, 8 up, 8 in 2023-10-09T06:04:21.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:21 smithi064 ceph-mon[212750]: failed to encode map e376 with expected crc 2023-10-09T06:04:21.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:21 smithi064 ceph-mon[212750]: failed to encode map e376 with expected crc 2023-10-09T06:04:21.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:21 smithi064 ceph-mon[212750]: failed to encode map e376 with expected crc 2023-10-09T06:04:21.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:21 smithi064 ceph-mon[212750]: failed to encode map e376 with expected crc 2023-10-09T06:04:21.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:21 smithi064 ceph-mon[212750]: failed to encode map e376 with expected crc 2023-10-09T06:04:21.768 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 79 16 96066 96050 75.9431 53.875 0.0157104 0.0131623 2023-10-09T06:04:21.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:21 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:04:21.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:21 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T06:04:21.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:21 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:04:21.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:21 smithi156 ceph-mon[195101]: osdmap e376: 8 total, 8 up, 8 in 2023-10-09T06:04:21.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:21 smithi156 ceph-mon[195101]: failed to encode map e376 with expected crc 2023-10-09T06:04:21.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:21 smithi156 ceph-mon[195101]: failed to encode map e376 with expected crc 2023-10-09T06:04:21.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:21 smithi156 ceph-mon[195101]: failed to encode map e376 with expected crc 2023-10-09T06:04:21.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:21 smithi156 ceph-mon[195101]: failed to encode map e376 with expected crc 2023-10-09T06:04:21.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:21 smithi156 ceph-mon[195101]: failed to encode map e376 with expected crc 2023-10-09T06:04:22.140 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:04:22.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:22.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:04:22.250 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:22.277 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:22.335 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:22.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:22.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:22.443 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:22.487 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:22.569 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:22.575 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:22 smithi156 ceph-mon[195101]: pgmap v910: 119 pgs: 119 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s 2023-10-09T06:04:22.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:22.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:22.676 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:22.677 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:22.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:22 smithi064 ceph-mon[212750]: pgmap v910: 119 pgs: 119 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s 2023-10-09T06:04:22.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:22 smithi064 ceph-mon[220044]: pgmap v910: 119 pgs: 119 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s 2023-10-09T06:04:22.771 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-10-09T06:04:22.768232+0000 min lat: 0.0010326 max lat: 0.330132 avg lat: 0.013224 2023-10-09T06:04:22.771 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:04:22.771 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 80 16 96824 96808 75.5861 47.375 0.024252 0.013224 2023-10-09T06:04:22.799 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:23.213 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:23.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:23.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:23.317 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:23.401 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:23.590 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:04:23.623 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:04:23.673 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:04:23.694 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:23.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:23.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:23.715 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:04:23.758 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:04:23.769 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 81 15 97672 97657 75.3069 53.0625 0.0113 0.0132739 2023-10-09T06:04:23.791 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:04:23.795 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:04:23.815 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:04:23.826 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:04:23.870 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:04:23.898 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:23.898 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:04:23.898 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:04:23.898 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:04:23.898 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:04:23.898 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:04:23.899 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:04:23.899 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:04:23.899 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:04:24.034 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:04:24.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:24.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:04:24.148 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:24.184 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:24.270 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:24.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:24.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:24.378 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:24.436 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:24 smithi156 ceph-mon[195101]: pgmap v911: 119 pgs: 119 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s 2023-10-09T06:04:24.436 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:24 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3535467406' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:24.473 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:24.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:24 smithi064 ceph-mon[212750]: pgmap v911: 119 pgs: 119 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s 2023-10-09T06:04:24.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:24 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3535467406' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:24.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:24 smithi064 ceph-mon[220044]: pgmap v911: 119 pgs: 119 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s 2023-10-09T06:04:24.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:24 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3535467406' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:24.771 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 82 16 98434 98418 74.9664 47.5625 0.00644886 0.0133337 2023-10-09T06:04:25.517 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:25.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:25.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:25.628 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:25.655 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:04:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:04:25] "GET /metrics HTTP/1.1" 200 36379 "" "Prometheus/2.43.0" 2023-10-09T06:04:25.746 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:25.779 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 83 15 99263 99248 74.6818 51.875 0.00876478 0.0133855 2023-10-09T06:04:25.920 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:25.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:25.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:26.027 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:26.038 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:26.101 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:26.205 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:04:26.206 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T06:04:26.206 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T06:04:26.206 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T06:04:26.206 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:04:26.228 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:26.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:26.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:26.338 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:26.404 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:26.504 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:26 smithi064 ceph-mon[212750]: pgmap v912: 119 pgs: 119 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2023-10-09T06:04:26.504 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:26 smithi064 ceph-mon[220044]: pgmap v912: 119 pgs: 119 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2023-10-09T06:04:26.577 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:26.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:26.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:26.686 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:26.714 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:26.758 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:26.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:26.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:26.779 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 84 16 100043 100027 74.3725 48.6875 0.00366759 0.0134271 2023-10-09T06:04:26.862 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:26.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:26 smithi156 ceph-mon[195101]: pgmap v912: 119 pgs: 119 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2023-10-09T06:04:26.889 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:27.311 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:04:27.362 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:27.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:27.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:27.455 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 0, 4] 2023-10-09T06:04:27.455 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:04:27.455 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-10-09T06:04:27.455 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:04:27.455 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:04:27.455 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-10-09T06:04:27.478 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:27.508 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:27.573 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:27.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:27.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:27.683 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:27.684 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:04:27.717 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:27.779 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 85 16 100695 100679 73.9771 40.75 0.136478 0.0135106 2023-10-09T06:04:28.064 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:28.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:28.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:28.177 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:28.212 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:28.270 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:28.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:28.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:28.369 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:28.380 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:28.409 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:28.451 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:28.452 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:28 smithi064 ceph-mon[212750]: pgmap v913: 119 pgs: 119 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s 2023-10-09T06:04:28.465 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:28.469 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:28 smithi156 ceph-mon[195101]: pgmap v913: 119 pgs: 119 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s 2023-10-09T06:04:28.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:28.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:28.571 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:04:28.664 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:28.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:28 smithi064 ceph-mon[220044]: pgmap v913: 119 pgs: 119 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s 2023-10-09T06:04:28.783 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 86 16 101459 101443 73.669 47.75 0.0241387 0.0135692 2023-10-09T06:04:28.899 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:29.018 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:04:29.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:29.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:04:29.136 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:29.168 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:29.220 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:29.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:29.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:29.328 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:29.357 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:29.410 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:29.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:29.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:29.512 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:29 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:04:29.513 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:29.513 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:29 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3250487688' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:04:29.513 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:29 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:04:29.513 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:04:29.516 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:29.560 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:04:29.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:29 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:04:29.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:29.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:29 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3250487688' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:04:29.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:29 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:04:29.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:04:29.592 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:29 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:04:29.592 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:29.592 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:29 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3250487688' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:04:29.592 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:29 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:04:29.593 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:04:29.620 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:29.783 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 87 16 102168 102152 73.3317 44.3125 0.0206995 0.0136321 2023-10-09T06:04:30.371 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:30.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:30.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:30.480 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:30.520 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:30.708 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:30 smithi064 ceph-mon[212750]: pgmap v914: 119 pgs: 119 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 803 op/s 2023-10-09T06:04:30.708 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:30 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:30.708 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:30 smithi064 ceph-mon[212750]: failed to encode map e377 with expected crc 2023-10-09T06:04:30.708 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:30 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:04:30.708 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:30 smithi064 ceph-mon[212750]: osdmap e377: 8 total, 8 up, 8 in 2023-10-09T06:04:30.709 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:30 smithi064 ceph-mon[220044]: pgmap v914: 119 pgs: 119 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 803 op/s 2023-10-09T06:04:30.709 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:30 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:30.709 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:30 smithi064 ceph-mon[220044]: failed to encode map e377 with expected crc 2023-10-09T06:04:30.709 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:30 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:04:30.709 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:30 smithi064 ceph-mon[220044]: osdmap e377: 8 total, 8 up, 8 in 2023-10-09T06:04:30.763 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:30.767 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:30 smithi156 ceph-mon[195101]: pgmap v914: 119 pgs: 119 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 803 op/s 2023-10-09T06:04:30.768 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:30 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:30.768 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:30 smithi156 ceph-mon[195101]: failed to encode map e377 with expected crc 2023-10-09T06:04:30.768 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:30 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:04:30.768 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:30 smithi156 ceph-mon[195101]: osdmap e377: 8 total, 8 up, 8 in 2023-10-09T06:04:30.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:30.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:30.783 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 88 16 102953 102937 73.056 49.0625 0.0151245 0.0136786 2023-10-09T06:04:30.869 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:30.927 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:31.018 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:31.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:31.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:31.340 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:31.423 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:31.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:31 smithi064 ceph-mon[212750]: pgmap v916: 119 pgs: 119 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 793 op/s 2023-10-09T06:04:31.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:31 smithi064 ceph-mon[220044]: pgmap v916: 119 pgs: 119 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 793 op/s 2023-10-09T06:04:31.783 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 89 16 103777 103761 72.8139 51.5 0.0174752 0.0137291 2023-10-09T06:04:31.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:31 smithi156 ceph-mon[195101]: pgmap v916: 119 pgs: 119 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 793 op/s 2023-10-09T06:04:32.739 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:32.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:32.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:32.783 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total time run: 90.0125 2023-10-09T06:04:32.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total writes made: 104515 2023-10-09T06:04:32.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Write size: 65536 2023-10-09T06:04:32.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object size: 65536 2023-10-09T06:04:32.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Bandwidth (MB/sec): 72.5697 2023-10-09T06:04:32.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Bandwidth: 31.9913 2023-10-09T06:04:32.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max bandwidth (MB/sec): 199.5 2023-10-09T06:04:32.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min bandwidth (MB/sec): 39.9375 2023-10-09T06:04:32.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average IOPS: 1161 2023-10-09T06:04:32.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev IOPS: 511.861 2023-10-09T06:04:32.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max IOPS: 3192 2023-10-09T06:04:32.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min IOPS: 639 2023-10-09T06:04:32.785 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average Latency(s): 0.0137754 2023-10-09T06:04:32.785 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Latency(s): 0.0177021 2023-10-09T06:04:32.785 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max latency(s): 0.330132 2023-10-09T06:04:32.785 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min latency(s): 0.0010326 2023-10-09T06:04:32.789 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T06:04:32.843 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:32.938 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:33.080 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:33.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:33.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:33.187 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:33.252 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:33.448 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:04:33.448 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T06:04:33.448 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:04:33.534 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:33.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:33.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:33.654 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:33.705 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:33.849 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:04:33.893 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:04:33.932 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:04:33.933 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:04:33.937 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:04:33.976 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:04:34.026 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:04:34.054 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:04:34.094 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:04:34.128 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:04:34.153 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:04:34.153 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:04:34.153 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:04:34.153 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:04:34.153 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:04:34.153 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:04:34.153 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:04:34.153 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:04:34.481 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:04:34.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:34 smithi064 ceph-mon[212750]: pgmap v917: 119 pgs: 119 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s 2023-10-09T06:04:34.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:34 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/581907688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:34.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:34 smithi064 ceph-mon[220044]: pgmap v917: 119 pgs: 119 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s 2023-10-09T06:04:34.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:34 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/581907688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:34.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:34 smithi156 ceph-mon[195101]: pgmap v917: 119 pgs: 119 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s 2023-10-09T06:04:34.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:34 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/581907688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:35.007 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:35.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:35.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:35.123 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:35.168 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:35.477 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:04:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:04:35] "GET /metrics HTTP/1.1" 200 36380 "" "Prometheus/2.43.0" 2023-10-09T06:04:36.035 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:36.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:36.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:36.143 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:36.213 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:04:36.243 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:36.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:36 smithi064 ceph-mon[212750]: pgmap v918: 119 pgs: 119 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 46 MiB/s wr, 782 op/s 2023-10-09T06:04:36.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:36 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:04:36.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:36 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:04:36.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:36 smithi064 ceph-mon[220044]: pgmap v918: 119 pgs: 119 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 46 MiB/s wr, 782 op/s 2023-10-09T06:04:36.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:36 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:04:36.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:36 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:04:36.478 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:36.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:36.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:36.587 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:36.606 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:36.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:36 smithi156 ceph-mon[195101]: pgmap v918: 119 pgs: 119 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 46 MiB/s wr, 782 op/s 2023-10-09T06:04:36.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:36 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:04:36.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:36 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:04:36.647 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:36.731 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:04:36.812 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:36.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:36.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:36.922 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:04:36.972 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:37.053 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:04:37.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:37.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:04:37.160 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:37.224 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:37.308 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:37.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:37.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:37.412 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:37.485 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:37.641 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:37.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:37.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:37.746 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:37.810 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:38.242 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:04:38.439 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:38.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:38.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:38.544 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:38.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:38 smithi064 ceph-mon[212750]: pgmap v919: 119 pgs: 119 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 38 MiB/s wr, 771 op/s 2023-10-09T06:04:38.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:38 smithi064 ceph-mon[220044]: pgmap v919: 119 pgs: 119 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 38 MiB/s wr, 771 op/s 2023-10-09T06:04:38.663 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:38.709 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:04:38.709 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:04:38.735 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:38 smithi156 ceph-mon[195101]: pgmap v919: 119 pgs: 119 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 38 MiB/s wr, 771 op/s 2023-10-09T06:04:39.004 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:39.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:39.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:39.113 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:39.143 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:39.165 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:39.187 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:39.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:39.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:39.290 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:39.331 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:39.527 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:39.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:39.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:39.586 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:04:39.633 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:39.749 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:40.125 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:04:40.446 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:04:40.678 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:40.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:40 smithi064 ceph-mon[212750]: pgmap v920: 119 pgs: 119 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 34 MiB/s wr, 773 op/s 2023-10-09T06:04:40.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:40 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2525834126' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:40.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:40 smithi064 ceph-mon[220044]: pgmap v920: 119 pgs: 119 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 34 MiB/s wr, 773 op/s 2023-10-09T06:04:40.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:40 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2525834126' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:40.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:40.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:40.733 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:40 smithi156 ceph-mon[195101]: pgmap v920: 119 pgs: 119 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 34 MiB/s wr, 773 op/s 2023-10-09T06:04:40.733 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:40 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2525834126' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:40.791 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:40.841 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:40.884 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 0, 4] 2023-10-09T06:04:40.884 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:04:40.884 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-09T06:04:40.884 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:04:40.885 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:04:40.885 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-09T06:04:41.716 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:41 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:41.716 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:41 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2525834126' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:04:41.716 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:41 smithi156 ceph-mon[195101]: osdmap e378: 8 total, 8 up, 8 in 2023-10-09T06:04:41.716 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:41 smithi156 ceph-mon[195101]: failed to encode map e378 with expected crc 2023-10-09T06:04:41.716 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:41 smithi156 ceph-mon[195101]: failed to encode map e378 with expected crc 2023-10-09T06:04:41.716 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:41 smithi156 ceph-mon[195101]: failed to encode map e378 with expected crc 2023-10-09T06:04:41.716 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:41 smithi156 ceph-mon[195101]: failed to encode map e378 with expected crc 2023-10-09T06:04:41.716 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:41 smithi156 ceph-mon[195101]: failed to encode map e378 with expected crc 2023-10-09T06:04:41.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:41 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:41.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:41 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2525834126' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:04:41.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:41 smithi064 ceph-mon[212750]: osdmap e378: 8 total, 8 up, 8 in 2023-10-09T06:04:41.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:41 smithi064 ceph-mon[212750]: failed to encode map e378 with expected crc 2023-10-09T06:04:41.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:41 smithi064 ceph-mon[212750]: failed to encode map e378 with expected crc 2023-10-09T06:04:41.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:41 smithi064 ceph-mon[212750]: failed to encode map e378 with expected crc 2023-10-09T06:04:41.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:41 smithi064 ceph-mon[212750]: failed to encode map e378 with expected crc 2023-10-09T06:04:41.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:41 smithi064 ceph-mon[212750]: failed to encode map e378 with expected crc 2023-10-09T06:04:41.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:41 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:41.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:41 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2525834126' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:04:41.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:41 smithi064 ceph-mon[220044]: osdmap e378: 8 total, 8 up, 8 in 2023-10-09T06:04:41.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:41 smithi064 ceph-mon[220044]: failed to encode map e378 with expected crc 2023-10-09T06:04:41.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:41 smithi064 ceph-mon[220044]: failed to encode map e378 with expected crc 2023-10-09T06:04:41.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:41 smithi064 ceph-mon[220044]: failed to encode map e378 with expected crc 2023-10-09T06:04:41.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:41 smithi064 ceph-mon[220044]: failed to encode map e378 with expected crc 2023-10-09T06:04:41.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:41 smithi064 ceph-mon[220044]: failed to encode map e378 with expected crc 2023-10-09T06:04:41.833 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:41.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:41.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:41.938 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:04:41.996 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:04:41.996 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:04:42.015 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:42.305 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:04:42.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:42.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:04:42.423 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:42.434 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:42.453 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:42.497 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:42.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:42.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:42.600 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:04:42.705 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:42 smithi064 ceph-mon[212750]: pgmap v922: 119 pgs: 119 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 15 MiB/s wr, 805 op/s 2023-10-09T06:04:42.706 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:42 smithi064 ceph-mon[220044]: pgmap v922: 119 pgs: 119 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 15 MiB/s wr, 805 op/s 2023-10-09T06:04:42.711 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:42.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:42 smithi156 ceph-mon[195101]: pgmap v922: 119 pgs: 119 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 15 MiB/s wr, 805 op/s 2023-10-09T06:04:42.889 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:04:42.896 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:04:42.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:42.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:04:43.009 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:04:43.009 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:43.058 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:43.064 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:04:43.099 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:04:43.138 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:04:43.170 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:43.171 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:04:43.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:43.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:43.225 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:04:43.263 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:04:43.275 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:43.301 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:04:43.302 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:04:43.302 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:04:43.302 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:04:43.302 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:04:43.302 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:04:43.302 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:04:43.303 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:04:43.303 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:04:43.342 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:43.386 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:04:43.604 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:43.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:43.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:43.718 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:43.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:43 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/383086670' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:43.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:43 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/383086670' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:43.805 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:43.829 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:43 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/383086670' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:44.229 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:04:44.286 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:44.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:44.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:44.391 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:44.435 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:44.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:44 smithi064 ceph-mon[220044]: pgmap v923: 119 pgs: 119 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 14 MiB/s wr, 811 op/s 2023-10-09T06:04:44.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:04:44.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:44.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:44 smithi064 ceph-mon[212750]: pgmap v923: 119 pgs: 119 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 14 MiB/s wr, 811 op/s 2023-10-09T06:04:44.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:04:44.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:44.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:44 smithi156 ceph-mon[195101]: pgmap v923: 119 pgs: 119 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 14 MiB/s wr, 811 op/s 2023-10-09T06:04:44.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:04:44.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:44.910 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:44.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:44.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:45.018 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:45.108 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:45.299 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:04:45.300 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T06:04:45.300 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T06:04:45.300 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:04:45.300 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.300 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45263.0:166114 7.1 7.6997e1e1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-10-09T06:04:45.300 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:01:21.468148+0000", 2023-10-09T06:04:45.300 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 203.82834884499999, 2023-10-09T06:04:45.300 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.84786050599999996, 2023-10-09T06:04:45.300 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:45.300 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:04:45.301 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:45.301 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45263", 2023-10-09T06:04:45.301 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:04:45.301 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 166114 2023-10-09T06:04:45.301 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.301 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:45.301 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.301 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:45.301 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468148+0000", 2023-10-09T06:04:45.301 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.301 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.302 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.302 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:45.302 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468148+0000", 2023-10-09T06:04:45.302 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.302 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.302 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.302 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:45.302 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468147+0000", 2023-10-09T06:04:45.302 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:04:45.303 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.303 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.303 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:45.303 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468152+0000", 2023-10-09T06:04:45.303 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0479999999999998e-06 2023-10-09T06:04:45.303 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.303 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.303 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:45.303 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468160+0000", 2023-10-09T06:04:45.303 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4510000000000003e-06 2023-10-09T06:04:45.304 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.304 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.304 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:45.304 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468165+0000", 2023-10-09T06:04:45.304 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8880000000000002e-06 2023-10-09T06:04:45.304 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.304 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.304 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:45.304 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468214+0000", 2023-10-09T06:04:45.304 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.884e-05 2023-10-09T06:04:45.304 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.304 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.305 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:45.305 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468225+0000", 2023-10-09T06:04:45.305 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1061e-05 2023-10-09T06:04:45.305 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.305 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.305 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:45.305 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:22.315992+0000", 2023-10-09T06:04:45.305 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.84776714900000005 2023-10-09T06:04:45.305 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.305 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.305 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:45.305 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:22.316009+0000", 2023-10-09T06:04:45.306 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6137000000000002e-05 2023-10-09T06:04:45.306 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.306 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:45.306 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.306 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.306 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.306 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45263.0:166115 7.1 7.a53a721 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-10-09T06:04:45.306 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:01:21.468174+0000", 2023-10-09T06:04:45.306 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 203.82832270599999, 2023-10-09T06:04:45.306 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.84806625999999996, 2023-10-09T06:04:45.306 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:45.307 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:04:45.307 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:45.307 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45263", 2023-10-09T06:04:45.307 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:04:45.307 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 166115 2023-10-09T06:04:45.307 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.307 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:45.307 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.307 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:45.307 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468174+0000", 2023-10-09T06:04:45.307 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.307 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.307 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.308 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:45.308 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468174+0000", 2023-10-09T06:04:45.308 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.308 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.308 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.308 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:45.308 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468174+0000", 2023-10-09T06:04:45.308 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:04:45.308 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.308 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.308 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:45.308 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468176+0000", 2023-10-09T06:04:45.309 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.655e-06 2023-10-09T06:04:45.309 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.309 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.309 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:45.309 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468188+0000", 2023-10-09T06:04:45.309 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1212999999999999e-05 2023-10-09T06:04:45.309 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.309 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.309 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:45.309 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468192+0000", 2023-10-09T06:04:45.309 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2760000000000002e-06 2023-10-09T06:04:45.309 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.310 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.310 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:45.310 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468250+0000", 2023-10-09T06:04:45.310 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8267000000000003e-05 2023-10-09T06:04:45.310 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.310 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.310 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:04:45.310 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468256+0000", 2023-10-09T06:04:45.310 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.553e-06 2023-10-09T06:04:45.310 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.310 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.311 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:45.311 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:22.316234+0000", 2023-10-09T06:04:45.311 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.84797834599999999 2023-10-09T06:04:45.311 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.311 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.311 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:45.311 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:22.316241+0000", 2023-10-09T06:04:45.311 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4110000000000001e-06 2023-10-09T06:04:45.311 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.312 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:45.312 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.312 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.312 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.312 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45263.0:174382 7.3 e327/323 7:caf7a445:::benchmark_data_smithi064_398483_object69029:head v 327'10833, mlcod=315'9230)", 2023-10-09T06:04:45.312 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:01:34.444802+0000", 2023-10-09T06:04:45.312 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 190.85169471, 2023-10-09T06:04:45.312 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.18731925899999999, 2023-10-09T06:04:45.312 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:45.312 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:45.312 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:45.313 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.313 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:45.313 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444802+0000", 2023-10-09T06:04:45.313 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.313 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.313 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.313 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:45.313 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444802+0000", 2023-10-09T06:04:45.313 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.313 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.313 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.313 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:45.314 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444802+0000", 2023-10-09T06:04:45.314 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:04:45.314 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.314 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.314 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:45.314 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444806+0000", 2023-10-09T06:04:45.314 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5040000000000004e-06 2023-10-09T06:04:45.314 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.314 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.315 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:45.315 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444814+0000", 2023-10-09T06:04:45.315 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3919999999999997e-06 2023-10-09T06:04:45.315 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.315 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.315 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:45.315 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444817+0000", 2023-10-09T06:04:45.315 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6540000000000001e-06 2023-10-09T06:04:45.315 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.315 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.316 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:45.316 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444874+0000", 2023-10-09T06:04:45.316 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6635000000000002e-05 2023-10-09T06:04:45.316 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.316 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.316 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:45.316 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444889+0000", 2023-10-09T06:04:45.316 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5128e-05 2023-10-09T06:04:45.316 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.316 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.316 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:45.316 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.632091+0000", 2023-10-09T06:04:45.317 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.18720204500000001 2023-10-09T06:04:45.317 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.317 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.317 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:45.317 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.632122+0000", 2023-10-09T06:04:45.317 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0380999999999999e-05 2023-10-09T06:04:45.317 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.317 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:45.317 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.317 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.317 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.317 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45263.0:174383 7.1 e327/323 7:8ed0fc45:::benchmark_data_smithi064_398483_object69030:head v 327'10884, mlcod=315'9249)", 2023-10-09T06:04:45.318 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:01:34.444859+0000", 2023-10-09T06:04:45.318 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 190.851637799, 2023-10-09T06:04:45.318 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.191513771, 2023-10-09T06:04:45.318 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:45.318 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:45.318 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:45.318 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.318 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:45.318 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444859+0000", 2023-10-09T06:04:45.318 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.318 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.318 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.319 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:45.319 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444859+0000", 2023-10-09T06:04:45.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.319 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.319 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:45.319 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444859+0000", 2023-10-09T06:04:45.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:04:45.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.319 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.319 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:45.319 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444862+0000", 2023-10-09T06:04:45.320 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0620000000000001e-06 2023-10-09T06:04:45.320 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.320 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.320 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:45.320 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444869+0000", 2023-10-09T06:04:45.320 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4379999999999998e-06 2023-10-09T06:04:45.320 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.320 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.320 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:45.320 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444872+0000", 2023-10-09T06:04:45.321 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.994e-06 2023-10-09T06:04:45.321 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.321 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.321 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:45.321 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444903+0000", 2023-10-09T06:04:45.321 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0527000000000001e-05 2023-10-09T06:04:45.321 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.321 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.321 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:45.321 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.444916+0000", 2023-10-09T06:04:45.321 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3030999999999999e-05 2023-10-09T06:04:45.322 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.322 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.322 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:45.322 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.636329+0000", 2023-10-09T06:04:45.322 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.191413368 2023-10-09T06:04:45.322 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.322 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.322 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:45.322 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.636373+0000", 2023-10-09T06:04:45.322 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3650999999999999e-05 2023-10-09T06:04:45.322 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.322 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:45.323 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.323 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.323 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.323 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45263.0:174381 7.9 e327/323 7:94e65e08:::benchmark_data_smithi064_398483_object69028:head v 327'10979, mlcod=318'9703)", 2023-10-09T06:04:45.323 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:01:34.445377+0000", 2023-10-09T06:04:45.323 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 190.85111984599999, 2023-10-09T06:04:45.323 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.193574469, 2023-10-09T06:04:45.323 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:45.323 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:45.323 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:45.323 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.325 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:45.325 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.445377+0000", 2023-10-09T06:04:45.325 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.325 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.325 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.325 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:45.325 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.445377+0000", 2023-10-09T06:04:45.325 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.325 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.326 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.326 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:45.326 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.445377+0000", 2023-10-09T06:04:45.326 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:04:45.326 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.326 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.326 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:45.326 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.445383+0000", 2023-10-09T06:04:45.326 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9270000000000002e-06 2023-10-09T06:04:45.326 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.326 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.327 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:45.327 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.445388+0000", 2023-10-09T06:04:45.327 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9420000000000003e-06 2023-10-09T06:04:45.327 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.327 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.327 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:45.327 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.445391+0000", 2023-10-09T06:04:45.327 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6090000000000001e-06 2023-10-09T06:04:45.327 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.327 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.327 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:45.327 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.445419+0000", 2023-10-09T06:04:45.328 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7396e-05 2023-10-09T06:04:45.328 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.328 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.328 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:45.328 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.445438+0000", 2023-10-09T06:04:45.328 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9558000000000001e-05 2023-10-09T06:04:45.328 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.328 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.328 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:45.328 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.638912+0000", 2023-10-09T06:04:45.328 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.19347399400000001 2023-10-09T06:04:45.328 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.329 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.329 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:45.329 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:34.638952+0000", 2023-10-09T06:04:45.329 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9363000000000002e-05 2023-10-09T06:04:45.329 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.329 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:45.329 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.329 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.329 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.329 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:40962 8.2 8:432ae0d6:::benchmark_data_smithi064_457003_object40961:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T06:04:45.329 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:03:24.580223+0000", 2023-10-09T06:04:45.329 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 80.716273834999996, 2023-10-09T06:04:45.330 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.144240487, 2023-10-09T06:04:45.330 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:45.330 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:45.330 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:45.330 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:04:45.330 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:04:45.330 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 40962 2023-10-09T06:04:45.330 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.330 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:45.330 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.330 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:45.331 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:24.580223+0000", 2023-10-09T06:04:45.331 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.331 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.331 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.331 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:45.331 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:24.580223+0000", 2023-10-09T06:04:45.331 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.331 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.331 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.331 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:45.331 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:24.580222+0000", 2023-10-09T06:04:45.331 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:04:45.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.332 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.332 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:45.332 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:24.580271+0000", 2023-10-09T06:04:45.332 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9342000000000003e-05 2023-10-09T06:04:45.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.332 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.332 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:45.332 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:24.580286+0000", 2023-10-09T06:04:45.332 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5387999999999999e-05 2023-10-09T06:04:45.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.332 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.333 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:45.333 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:24.580294+0000", 2023-10-09T06:04:45.333 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4660000000000002e-06 2023-10-09T06:04:45.333 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.333 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.333 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:45.333 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:24.580350+0000", 2023-10-09T06:04:45.333 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.63e-05 2023-10-09T06:04:45.333 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.333 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.333 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:45.333 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:24.580454+0000", 2023-10-09T06:04:45.334 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000103552 2023-10-09T06:04:45.334 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.334 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.334 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:04:45.334 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:24.580557+0000", 2023-10-09T06:04:45.334 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010303500000000001 2023-10-09T06:04:45.334 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.334 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.334 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:45.334 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:24.582178+0000", 2023-10-09T06:04:45.334 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001621141 2023-10-09T06:04:45.335 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.335 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.335 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:45.335 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:24.583134+0000", 2023-10-09T06:04:45.335 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00095563699999999996 2023-10-09T06:04:45.335 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.335 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.335 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:45.335 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:24.724356+0000", 2023-10-09T06:04:45.335 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.14122231199999999 2023-10-09T06:04:45.335 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.335 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.336 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:45.336 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:24.724416+0000", 2023-10-09T06:04:45.336 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9812000000000003e-05 2023-10-09T06:04:45.336 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.336 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.336 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:45.336 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:24.724464+0000", 2023-10-09T06:04:45.336 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7982999999999998e-05 2023-10-09T06:04:45.336 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.336 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:45.336 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.337 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.337 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.337 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:50268 8.2 8:43b0c6ed:::benchmark_data_smithi064_457003_object50267:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2023-10-09T06:04:45.337 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:03:32.767188+0000", 2023-10-09T06:04:45.337 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 72.529308936999996, 2023-10-09T06:04:45.337 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.153225635, 2023-10-09T06:04:45.337 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:45.337 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:45.337 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:45.337 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:04:45.337 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:04:45.337 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 50268 2023-10-09T06:04:45.338 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.338 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:45.338 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.338 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:45.338 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:32.767188+0000", 2023-10-09T06:04:45.338 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.338 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.338 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.338 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:45.338 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:32.767188+0000", 2023-10-09T06:04:45.338 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.338 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.339 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.339 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:45.339 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:32.767187+0000", 2023-10-09T06:04:45.339 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:04:45.339 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.339 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.339 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:45.339 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:32.767251+0000", 2023-10-09T06:04:45.339 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3848000000000003e-05 2023-10-09T06:04:45.339 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.340 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.340 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:45.340 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:32.767267+0000", 2023-10-09T06:04:45.340 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6526999999999999e-05 2023-10-09T06:04:45.340 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.340 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.340 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:45.340 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:32.767275+0000", 2023-10-09T06:04:45.340 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6879999999999999e-06 2023-10-09T06:04:45.340 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.340 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.340 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:45.341 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:32.767328+0000", 2023-10-09T06:04:45.341 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3412999999999998e-05 2023-10-09T06:04:45.341 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.341 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.341 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:45.341 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:32.767474+0000", 2023-10-09T06:04:45.341 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000146224 2023-10-09T06:04:45.341 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.341 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.341 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:04:45.342 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:32.767639+0000", 2023-10-09T06:04:45.342 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00016468799999999999 2023-10-09T06:04:45.342 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.342 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.342 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:45.342 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:32.769645+0000", 2023-10-09T06:04:45.342 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0020060289999999999 2023-10-09T06:04:45.342 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.342 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.342 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:45.342 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:32.771364+0000", 2023-10-09T06:04:45.342 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001718359 2023-10-09T06:04:45.343 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.343 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.343 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:45.343 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:32.920352+0000", 2023-10-09T06:04:45.343 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.14898856699999999 2023-10-09T06:04:45.343 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.343 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.343 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:45.343 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:32.920388+0000", 2023-10-09T06:04:45.343 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6282000000000003e-05 2023-10-09T06:04:45.343 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.343 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.344 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:45.344 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:32.920414+0000", 2023-10-09T06:04:45.344 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.5269000000000002e-05 2023-10-09T06:04:45.344 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.344 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:45.344 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.344 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.344 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.344 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:64675 8.e e370/360 8:74d7593a:::benchmark_data_smithi064_457003_object64674:head v 370'4093, mlcod=370'4091)", 2023-10-09T06:04:45.344 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:03:45.766584+0000", 2023-10-09T06:04:45.344 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 59.529912797999998, 2023-10-09T06:04:45.345 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.169625003, 2023-10-09T06:04:45.345 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:45.345 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:45.345 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:45.345 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.345 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:45.345 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.766584+0000", 2023-10-09T06:04:45.345 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.345 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.345 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.345 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:45.345 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.766584+0000", 2023-10-09T06:04:45.346 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.346 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.346 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.346 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:45.346 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.766583+0000", 2023-10-09T06:04:45.346 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:04:45.346 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.346 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.346 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:45.346 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.766625+0000", 2023-10-09T06:04:45.346 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1288999999999997e-05 2023-10-09T06:04:45.346 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.347 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.347 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:45.347 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.766635+0000", 2023-10-09T06:04:45.347 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.9809999999999998e-06 2023-10-09T06:04:45.347 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.347 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.347 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:45.347 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.766639+0000", 2023-10-09T06:04:45.347 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2289999999999997e-06 2023-10-09T06:04:45.347 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.347 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.347 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:45.348 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.933012+0000", 2023-10-09T06:04:45.348 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.16637262 2023-10-09T06:04:45.348 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.348 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.348 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:45.348 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.933043+0000", 2023-10-09T06:04:45.348 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1161000000000002e-05 2023-10-09T06:04:45.348 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.348 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.348 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:45.349 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.936139+0000", 2023-10-09T06:04:45.349 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0030965699999999999 2023-10-09T06:04:45.349 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.349 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.349 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:45.349 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:45.936209+0000", 2023-10-09T06:04:45.349 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9863999999999994e-05 2023-10-09T06:04:45.349 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.349 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:45.349 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.349 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.350 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.350 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:66811 8.2 8:407b9f5e:::benchmark_data_smithi064_457003_object66810:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e370)", 2023-10-09T06:04:45.350 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:03:47.811912+0000", 2023-10-09T06:04:45.350 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 57.484585228999997, 2023-10-09T06:04:45.350 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24319370400000001, 2023-10-09T06:04:45.350 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:45.350 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:45.350 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:45.350 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:04:45.350 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:04:45.350 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 66811 2023-10-09T06:04:45.351 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.351 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:45.351 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.351 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:45.351 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.811912+0000", 2023-10-09T06:04:45.351 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.351 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.351 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.351 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:45.351 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.811912+0000", 2023-10-09T06:04:45.351 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.351 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.352 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.352 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:45.352 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.811911+0000", 2023-10-09T06:04:45.352 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:04:45.352 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.352 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.352 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:45.352 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.811960+0000", 2023-10-09T06:04:45.352 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9636999999999999e-05 2023-10-09T06:04:45.352 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.352 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.352 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:45.353 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.811971+0000", 2023-10-09T06:04:45.353 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1143e-05 2023-10-09T06:04:45.353 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.353 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.353 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:45.353 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.811976+0000", 2023-10-09T06:04:45.353 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9599999999999999e-06 2023-10-09T06:04:45.353 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.353 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.353 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:45.353 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.812008+0000", 2023-10-09T06:04:45.354 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.2153000000000002e-05 2023-10-09T06:04:45.354 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.354 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.354 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:45.354 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.812095+0000", 2023-10-09T06:04:45.354 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6361999999999997e-05 2023-10-09T06:04:45.354 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.354 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.354 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:04:45.354 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.812179+0000", 2023-10-09T06:04:45.354 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.4025999999999998e-05 2023-10-09T06:04:45.354 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.355 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.355 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:45.355 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.813938+0000", 2023-10-09T06:04:45.355 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001759549 2023-10-09T06:04:45.355 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.355 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.355 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:45.355 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.814890+0000", 2023-10-09T06:04:45.355 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00095179900000000003 2023-10-09T06:04:45.355 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.355 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.355 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:45.356 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.054998+0000", 2023-10-09T06:04:45.356 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24010741199999999 2023-10-09T06:04:45.356 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.356 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.356 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:45.356 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.055062+0000", 2023-10-09T06:04:45.356 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4838000000000005e-05 2023-10-09T06:04:45.356 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.356 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.356 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:45.356 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.055105+0000", 2023-10-09T06:04:45.356 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2979999999999998e-05 2023-10-09T06:04:45.357 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.357 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:45.357 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.357 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.357 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.357 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:68187 8.4 8:2b957196:::benchmark_data_smithi064_457003_object68186:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e370)", 2023-10-09T06:04:45.357 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:03:49.356418+0000", 2023-10-09T06:04:45.357 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 55.940079267999998, 2023-10-09T06:04:45.357 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.22125428599999999, 2023-10-09T06:04:45.357 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:45.357 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:45.357 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:45.358 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:04:45.358 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:04:45.358 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 68187 2023-10-09T06:04:45.358 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.358 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:45.358 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.358 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:45.358 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.356418+0000", 2023-10-09T06:04:45.358 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.358 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.358 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.358 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:45.359 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.356418+0000", 2023-10-09T06:04:45.359 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.359 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.359 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.359 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:45.359 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.356417+0000", 2023-10-09T06:04:45.359 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:04:45.359 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.359 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.359 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:45.359 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.356493+0000", 2023-10-09T06:04:45.360 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6738999999999998e-05 2023-10-09T06:04:45.360 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.360 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.360 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:45.360 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.356507+0000", 2023-10-09T06:04:45.360 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3444999999999999e-05 2023-10-09T06:04:45.360 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.360 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.360 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:45.360 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.356512+0000", 2023-10-09T06:04:45.360 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5140000000000001e-06 2023-10-09T06:04:45.360 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.361 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.361 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:45.361 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.356550+0000", 2023-10-09T06:04:45.361 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7744999999999999e-05 2023-10-09T06:04:45.361 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.361 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.361 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:45.361 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.356653+0000", 2023-10-09T06:04:45.361 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000103034 2023-10-09T06:04:45.361 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.361 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.361 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:04:45.362 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.356751+0000", 2023-10-09T06:04:45.362 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.8065000000000004e-05 2023-10-09T06:04:45.362 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.362 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.362 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:45.362 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.359884+0000", 2023-10-09T06:04:45.362 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0031327909999999998 2023-10-09T06:04:45.362 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.362 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.362 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:45.363 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.362136+0000", 2023-10-09T06:04:45.363 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0022520259999999999 2023-10-09T06:04:45.363 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.363 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.363 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:45.363 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.577574+0000", 2023-10-09T06:04:45.363 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.215438413 2023-10-09T06:04:45.363 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.363 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.363 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:45.363 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.577628+0000", 2023-10-09T06:04:45.364 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3297000000000001e-05 2023-10-09T06:04:45.364 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.364 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.364 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:45.364 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.577672+0000", 2023-10-09T06:04:45.364 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4360999999999998e-05 2023-10-09T06:04:45.364 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.364 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:45.364 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.364 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.364 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.364 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:68188 8.4 8:2b2d31ce:::benchmark_data_smithi064_457003_object68187:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e370)", 2023-10-09T06:04:45.365 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:03:49.356524+0000", 2023-10-09T06:04:45.365 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 55.939972603999998, 2023-10-09T06:04:45.365 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.22125093800000001, 2023-10-09T06:04:45.365 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:45.365 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:45.365 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:45.365 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:04:45.365 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:04:45.365 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 68188 2023-10-09T06:04:45.365 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.365 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:45.365 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.366 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:45.366 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.356524+0000", 2023-10-09T06:04:45.366 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.366 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.366 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.366 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:45.366 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.356524+0000", 2023-10-09T06:04:45.366 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.366 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.366 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.366 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:45.367 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.356524+0000", 2023-10-09T06:04:45.367 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:04:45.367 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.367 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.367 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:45.367 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.356557+0000", 2023-10-09T06:04:45.367 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.3148000000000002e-05 2023-10-09T06:04:45.367 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.367 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.367 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:45.367 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.356564+0000", 2023-10-09T06:04:45.367 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0180000000000004e-06 2023-10-09T06:04:45.368 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.368 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.368 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:45.368 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.356568+0000", 2023-10-09T06:04:45.368 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3440000000000003e-06 2023-10-09T06:04:45.368 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.368 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.368 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:45.368 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.358423+0000", 2023-10-09T06:04:45.368 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0018546439999999999 2023-10-09T06:04:45.368 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.368 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.369 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:45.369 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.358511+0000", 2023-10-09T06:04:45.369 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.7928000000000002e-05 2023-10-09T06:04:45.369 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.369 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.369 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:04:45.369 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.358603+0000", 2023-10-09T06:04:45.369 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2138000000000005e-05 2023-10-09T06:04:45.369 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.369 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.370 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:45.370 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.360018+0000", 2023-10-09T06:04:45.370 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0014144559999999999 2023-10-09T06:04:45.370 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.370 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.370 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:45.370 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.362476+0000", 2023-10-09T06:04:45.370 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002458815 2023-10-09T06:04:45.370 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.370 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.370 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:45.370 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.577705+0000", 2023-10-09T06:04:45.371 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.21522841100000001 2023-10-09T06:04:45.371 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.371 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.371 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:45.371 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.577739+0000", 2023-10-09T06:04:45.371 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.4671000000000001e-05 2023-10-09T06:04:45.371 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.371 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.371 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:45.371 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.577775+0000", 2023-10-09T06:04:45.371 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5877000000000002e-05 2023-10-09T06:04:45.371 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.372 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:45.372 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.372 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.372 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.372 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:72676 8.b e371/360 8:db64baf3:::benchmark_data_smithi064_457003_object72675:head v 371'4528, mlcod=371'4526)", 2023-10-09T06:04:45.372 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:03:54.480642+0000", 2023-10-09T06:04:45.372 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 50.815854573999999, 2023-10-09T06:04:45.372 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.13130855799999999, 2023-10-09T06:04:45.372 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:45.372 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:45.372 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:45.372 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.373 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:45.373 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:54.480642+0000", 2023-10-09T06:04:45.373 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.373 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.373 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.373 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:45.373 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:54.480642+0000", 2023-10-09T06:04:45.373 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.373 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.373 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.373 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:45.374 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:54.480642+0000", 2023-10-09T06:04:45.374 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:04:45.374 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.374 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.374 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:45.374 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:54.480683+0000", 2023-10-09T06:04:45.374 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1048999999999999e-05 2023-10-09T06:04:45.374 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.374 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.374 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:45.374 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:54.598084+0000", 2023-10-09T06:04:45.374 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.117401597 2023-10-09T06:04:45.375 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.375 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.375 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:45.375 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:54.598099+0000", 2023-10-09T06:04:45.375 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4547e-05 2023-10-09T06:04:45.375 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.375 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.375 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:45.375 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:54.598243+0000", 2023-10-09T06:04:45.375 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014377699999999999 2023-10-09T06:04:45.375 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.376 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.376 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:45.376 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:54.598263+0000", 2023-10-09T06:04:45.376 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0808000000000001e-05 2023-10-09T06:04:45.376 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.376 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.376 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:45.376 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:54.611900+0000", 2023-10-09T06:04:45.376 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013637056 2023-10-09T06:04:45.376 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.376 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.377 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:45.377 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:54.611951+0000", 2023-10-09T06:04:45.377 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0581999999999998e-05 2023-10-09T06:04:45.377 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.377 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:45.377 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.377 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.377 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.377 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:85738 8.4 8:23188487:::benchmark_data_smithi064_457003_object85737:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-10-09T06:04:45.377 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:04:09.562934+0000", 2023-10-09T06:04:45.377 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 35.733562786999997, 2023-10-09T06:04:45.377 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.20575234100000001, 2023-10-09T06:04:45.378 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:45.378 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:45.378 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:45.378 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:04:45.378 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:04:45.378 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 85738 2023-10-09T06:04:45.378 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.378 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:45.378 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.378 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:45.378 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.562934+0000", 2023-10-09T06:04:45.378 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.379 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.379 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.379 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:45.379 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.562934+0000", 2023-10-09T06:04:45.379 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.379 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.379 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.379 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:45.379 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.562933+0000", 2023-10-09T06:04:45.379 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:04:45.379 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.379 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.380 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:45.380 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.562997+0000", 2023-10-09T06:04:45.380 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4661999999999999e-05 2023-10-09T06:04:45.380 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.380 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.380 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:45.380 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.563010+0000", 2023-10-09T06:04:45.380 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.289e-05 2023-10-09T06:04:45.380 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.380 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.381 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:45.381 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.563016+0000", 2023-10-09T06:04:45.381 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8889999999999999e-06 2023-10-09T06:04:45.381 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.381 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.381 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:45.381 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.563080+0000", 2023-10-09T06:04:45.381 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3880000000000005e-05 2023-10-09T06:04:45.381 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.381 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.381 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:45.381 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.563201+0000", 2023-10-09T06:04:45.382 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012068100000000001 2023-10-09T06:04:45.382 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.382 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.382 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:04:45.382 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.563335+0000", 2023-10-09T06:04:45.382 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013386800000000001 2023-10-09T06:04:45.382 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.382 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.382 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:45.382 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.566030+0000", 2023-10-09T06:04:45.382 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0026955159999999998 2023-10-09T06:04:45.383 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.383 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.383 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:45.383 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.566563+0000", 2023-10-09T06:04:45.383 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00053269899999999998 2023-10-09T06:04:45.383 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.383 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.383 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:45.383 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.768581+0000", 2023-10-09T06:04:45.383 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.20201845199999999 2023-10-09T06:04:45.383 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.384 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.384 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:45.384 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.768636+0000", 2023-10-09T06:04:45.384 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4490000000000002e-05 2023-10-09T06:04:45.384 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.384 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.384 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:45.384 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.768687+0000", 2023-10-09T06:04:45.384 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0858e-05 2023-10-09T06:04:45.384 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.384 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:45.384 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.385 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.385 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.385 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:85744 8.4 8:263bf16b:::benchmark_data_smithi064_457003_object85743:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-10-09T06:04:45.385 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:04:09.569216+0000", 2023-10-09T06:04:45.385 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 35.727280630000003, 2023-10-09T06:04:45.385 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.21085531199999999, 2023-10-09T06:04:45.385 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:45.385 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:45.386 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:45.386 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:04:45.386 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:04:45.386 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 85744 2023-10-09T06:04:45.386 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.386 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:45.386 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.386 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:45.386 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.569216+0000", 2023-10-09T06:04:45.386 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.386 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.387 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.387 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:45.387 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.569216+0000", 2023-10-09T06:04:45.387 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.387 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.387 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.387 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:45.387 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.569215+0000", 2023-10-09T06:04:45.387 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:04:45.387 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.387 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.388 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:45.388 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.569279+0000", 2023-10-09T06:04:45.388 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4042999999999997e-05 2023-10-09T06:04:45.388 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.388 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.388 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:45.388 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.569291+0000", 2023-10-09T06:04:45.388 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2428e-05 2023-10-09T06:04:45.388 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.388 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.388 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:45.389 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.569298+0000", 2023-10-09T06:04:45.389 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4300000000000003e-06 2023-10-09T06:04:45.389 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.389 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.389 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:45.389 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.569339+0000", 2023-10-09T06:04:45.389 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0939999999999998e-05 2023-10-09T06:04:45.389 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.389 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.389 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:45.389 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.569417+0000", 2023-10-09T06:04:45.390 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8756000000000003e-05 2023-10-09T06:04:45.390 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.390 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.390 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:04:45.390 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.569500+0000", 2023-10-09T06:04:45.390 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.2663000000000005e-05 2023-10-09T06:04:45.390 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.390 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.390 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:45.390 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.768731+0000", 2023-10-09T06:04:45.390 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.19923073499999999 2023-10-09T06:04:45.390 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.391 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.391 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:45.391 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.768765+0000", 2023-10-09T06:04:45.391 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.3782000000000003e-05 2023-10-09T06:04:45.391 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.391 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.391 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:45.391 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.779966+0000", 2023-10-09T06:04:45.391 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011201496999999999 2023-10-09T06:04:45.391 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.392 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.392 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:45.392 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.780034+0000", 2023-10-09T06:04:45.392 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7929999999999995e-05 2023-10-09T06:04:45.392 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.392 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.392 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:45.392 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.780072+0000", 2023-10-09T06:04:45.392 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7629000000000002e-05 2023-10-09T06:04:45.392 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.392 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:45.392 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.393 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.393 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.393 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:85743 8.5 e374/360 8:a9f98ee6:::benchmark_data_smithi064_457003_object85742:head v 374'5344, mlcod=374'5342)", 2023-10-09T06:04:45.393 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:04:09.569512+0000", 2023-10-09T06:04:45.393 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 35.726985071000001, 2023-10-09T06:04:45.393 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.19902925399999999, 2023-10-09T06:04:45.393 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:45.393 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:45.393 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:45.393 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.393 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:45.393 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.569512+0000", 2023-10-09T06:04:45.394 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.394 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.394 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.394 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:45.394 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.569512+0000", 2023-10-09T06:04:45.394 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.394 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.394 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.394 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:45.394 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.569511+0000", 2023-10-09T06:04:45.394 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:04:45.395 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.395 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.395 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:45.395 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.569533+0000", 2023-10-09T06:04:45.395 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2086000000000002e-05 2023-10-09T06:04:45.395 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.395 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.395 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:45.395 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.569544+0000", 2023-10-09T06:04:45.395 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0427e-05 2023-10-09T06:04:45.396 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.396 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.396 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:45.396 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.569550+0000", 2023-10-09T06:04:45.396 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7169999999999996e-06 2023-10-09T06:04:45.396 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.396 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.396 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:45.396 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.569594+0000", 2023-10-09T06:04:45.396 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4298000000000002e-05 2023-10-09T06:04:45.396 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.397 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.397 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:45.397 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.569612+0000", 2023-10-09T06:04:45.397 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8529000000000002e-05 2023-10-09T06:04:45.397 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.397 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.397 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:45.397 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.572050+0000", 2023-10-09T06:04:45.397 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002438122 2023-10-09T06:04:45.397 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.397 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.397 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:45.398 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.768541+0000", 2023-10-09T06:04:45.398 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.196490687 2023-10-09T06:04:45.398 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.398 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:45.398 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.398 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.398 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.398 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop_reply(client.45605.0:85738 8.4 e374/360 ondisk, result = 0)", 2023-10-09T06:04:45.398 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:04:09.570931+0000", 2023-10-09T06:04:45.398 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 35.725565689, 2023-10-09T06:04:45.398 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.19776560800000001, 2023-10-09T06:04:45.399 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:45.399 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:04:45.399 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:45.399 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.399 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:45.399 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.570931+0000", 2023-10-09T06:04:45.399 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.399 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.399 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.399 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:45.399 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.570931+0000", 2023-10-09T06:04:45.400 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.400 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.400 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.400 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:45.400 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.570931+0000", 2023-10-09T06:04:45.400 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:04:45.400 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.400 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.400 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:45.400 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.570935+0000", 2023-10-09T06:04:45.400 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7910000000000004e-06 2023-10-09T06:04:45.401 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.401 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.401 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:45.401 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.570946+0000", 2023-10-09T06:04:45.401 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0403e-05 2023-10-09T06:04:45.401 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.401 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.401 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:45.401 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.570951+0000", 2023-10-09T06:04:45.401 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5169999999999999e-06 2023-10-09T06:04:45.401 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.402 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.402 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:45.402 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.768561+0000", 2023-10-09T06:04:45.402 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.19760997499999999 2023-10-09T06:04:45.402 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.402 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.402 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:45.402 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.768573+0000", 2023-10-09T06:04:45.402 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1474999999999999e-05 2023-10-09T06:04:45.402 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.402 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.403 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:45.403 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.768697+0000", 2023-10-09T06:04:45.403 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012404699999999999 2023-10-09T06:04:45.403 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.403 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:45.403 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.403 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.403 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.403 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop_reply(client.45605.0:85744 8.4 e374/360 ondisk, result = 0)", 2023-10-09T06:04:45.403 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:04:09.572461+0000", 2023-10-09T06:04:45.403 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 35.724036040000001, 2023-10-09T06:04:45.404 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.196271573, 2023-10-09T06:04:45.404 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:45.404 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:04:45.404 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:45.404 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.404 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:45.404 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.572461+0000", 2023-10-09T06:04:45.404 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.404 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.404 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.404 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:45.405 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.572461+0000", 2023-10-09T06:04:45.405 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.405 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.405 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.405 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:45.405 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.572460+0000", 2023-10-09T06:04:45.405 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:04:45.405 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.405 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.405 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:45.405 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.572464+0000", 2023-10-09T06:04:45.406 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0419999999999996e-06 2023-10-09T06:04:45.406 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.406 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.406 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:45.406 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.768498+0000", 2023-10-09T06:04:45.406 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.19603383399999999 2023-10-09T06:04:45.406 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.406 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.406 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:45.406 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.768514+0000", 2023-10-09T06:04:45.407 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5999999999999999e-05 2023-10-09T06:04:45.407 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.407 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.407 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:45.407 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.768719+0000", 2023-10-09T06:04:45.407 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00020441599999999999 2023-10-09T06:04:45.407 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.407 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.407 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:45.407 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.768725+0000", 2023-10-09T06:04:45.407 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0090000000000004e-06 2023-10-09T06:04:45.408 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.408 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.408 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:45.408 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.768733+0000", 2023-10-09T06:04:45.408 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8269999999999998e-06 2023-10-09T06:04:45.408 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.408 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:45.408 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.408 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.408 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.408 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:85745 8.5 e374/360 8:a6af8761:::benchmark_data_smithi064_457003_object85744:head v 374'5345, mlcod=374'5342)", 2023-10-09T06:04:45.409 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:04:09.574349+0000", 2023-10-09T06:04:45.409 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 35.722148224999998, 2023-10-09T06:04:45.409 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.19984295999999999, 2023-10-09T06:04:45.409 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:45.409 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:45.409 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:45.409 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.409 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:45.409 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.574349+0000", 2023-10-09T06:04:45.410 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.410 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.410 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.410 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:45.410 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.574349+0000", 2023-10-09T06:04:45.410 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.410 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.410 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.410 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:45.410 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.574348+0000", 2023-10-09T06:04:45.410 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:04:45.411 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.411 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.411 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:45.411 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.574372+0000", 2023-10-09T06:04:45.411 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4093000000000001e-05 2023-10-09T06:04:45.411 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.411 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.411 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:45.411 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.768521+0000", 2023-10-09T06:04:45.411 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.19414836999999999 2023-10-09T06:04:45.411 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.412 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.412 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:45.412 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.768528+0000", 2023-10-09T06:04:45.412 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2259999999999999e-06 2023-10-09T06:04:45.412 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.412 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.412 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:45.412 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.768570+0000", 2023-10-09T06:04:45.412 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1819000000000001e-05 2023-10-09T06:04:45.412 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.413 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.413 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:45.413 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.768588+0000", 2023-10-09T06:04:45.413 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8134999999999998e-05 2023-10-09T06:04:45.413 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.413 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.413 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:45.413 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.774142+0000", 2023-10-09T06:04:45.413 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0055539209999999999 2023-10-09T06:04:45.413 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.414 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.414 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:45.414 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.774192+0000", 2023-10-09T06:04:45.414 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0056999999999999e-05 2023-10-09T06:04:45.414 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.414 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:45.414 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.414 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.414 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.414 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:85750 8.4 8:247c096d:::benchmark_data_smithi064_457003_object85749:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-10-09T06:04:45.414 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:04:09.578071+0000", 2023-10-09T06:04:45.415 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 35.718426094000002, 2023-10-09T06:04:45.415 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.202157754, 2023-10-09T06:04:45.415 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:45.415 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:45.415 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:45.415 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:04:45.415 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:04:45.415 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 85750 2023-10-09T06:04:45.415 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.415 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:45.416 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.416 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:45.416 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.578071+0000", 2023-10-09T06:04:45.416 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.416 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.416 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.416 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:45.416 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.578071+0000", 2023-10-09T06:04:45.416 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.416 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.416 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.417 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:45.417 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.578069+0000", 2023-10-09T06:04:45.417 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:04:45.417 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.417 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.417 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:45.417 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.578112+0000", 2023-10-09T06:04:45.417 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2592000000000001e-05 2023-10-09T06:04:45.417 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.417 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.417 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:45.418 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.768510+0000", 2023-10-09T06:04:45.418 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.19039865 2023-10-09T06:04:45.418 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.418 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.418 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:45.418 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.768523+0000", 2023-10-09T06:04:45.418 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2968e-05 2023-10-09T06:04:45.418 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.418 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.418 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:45.418 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.768787+0000", 2023-10-09T06:04:45.419 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00026346099999999999 2023-10-09T06:04:45.419 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.419 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.419 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:45.419 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.768904+0000", 2023-10-09T06:04:45.419 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000117086 2023-10-09T06:04:45.419 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.419 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.419 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:04:45.419 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.769051+0000", 2023-10-09T06:04:45.419 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014662799999999999 2023-10-09T06:04:45.419 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.419 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.420 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:45.420 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.780114+0000", 2023-10-09T06:04:45.420 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011063351000000001 2023-10-09T06:04:45.420 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.420 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.420 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:45.420 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.780120+0000", 2023-10-09T06:04:45.420 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9689999999999999e-06 2023-10-09T06:04:45.420 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.420 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.420 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:45.421 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.780163+0000", 2023-10-09T06:04:45.421 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2817e-05 2023-10-09T06:04:45.421 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.421 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.421 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:45.421 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.780198+0000", 2023-10-09T06:04:45.421 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5716000000000002e-05 2023-10-09T06:04:45.421 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.421 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.421 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:45.421 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:09.780229+0000", 2023-10-09T06:04:45.422 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0156000000000001e-05 2023-10-09T06:04:45.422 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.422 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:45.422 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.422 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.422 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.422 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:102740 8.2 8:42471e85:::benchmark_data_smithi064_457003_object102739:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e377)", 2023-10-09T06:04:45.422 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:04:30.414271+0000", 2023-10-09T06:04:45.422 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 14.882226037000001, 2023-10-09T06:04:45.422 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.14542585999999999, 2023-10-09T06:04:45.422 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:04:45.423 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:04:45.423 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:04:45.423 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:04:45.423 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:04:45.423 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 102740 2023-10-09T06:04:45.423 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.423 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:04:45.423 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.423 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:04:45.423 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:30.414271+0000", 2023-10-09T06:04:45.423 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.423 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.424 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.424 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:04:45.424 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:30.414271+0000", 2023-10-09T06:04:45.424 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:04:45.424 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.424 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.424 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:04:45.424 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:30.414269+0000", 2023-10-09T06:04:45.424 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:04:45.424 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.424 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.425 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:04:45.425 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:30.414310+0000", 2023-10-09T06:04:45.425 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0640999999999999e-05 2023-10-09T06:04:45.425 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.425 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.425 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:04:45.425 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:30.414327+0000", 2023-10-09T06:04:45.425 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6597000000000001e-05 2023-10-09T06:04:45.425 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.425 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.425 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:04:45.425 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:30.414334+0000", 2023-10-09T06:04:45.425 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2450000000000001e-06 2023-10-09T06:04:45.426 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.426 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.426 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:04:45.426 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:30.414389+0000", 2023-10-09T06:04:45.426 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.516e-05 2023-10-09T06:04:45.426 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.426 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.426 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:04:45.426 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:30.414536+0000", 2023-10-09T06:04:45.426 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000146957 2023-10-09T06:04:45.426 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.427 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.427 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:04:45.427 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:30.414640+0000", 2023-10-09T06:04:45.427 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000103637 2023-10-09T06:04:45.427 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.427 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.427 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:04:45.427 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:30.417353+0000", 2023-10-09T06:04:45.427 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0027139 2023-10-09T06:04:45.427 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.427 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.427 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:45.428 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:30.420596+0000", 2023-10-09T06:04:45.428 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.003242801 2023-10-09T06:04:45.428 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.428 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.428 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:45.428 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:30.559511+0000", 2023-10-09T06:04:45.428 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.13891429899999999 2023-10-09T06:04:45.428 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.428 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.428 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:04:45.428 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:30.559567+0000", 2023-10-09T06:04:45.429 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6217000000000001e-05 2023-10-09T06:04:45.429 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:04:45.429 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:04:45.429 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:04:45.429 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:30.559697+0000", 2023-10-09T06:04:45.429 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013000899999999999 2023-10-09T06:04:45.429 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.429 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:45.429 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.429 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:04:45.429 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:04:45.429 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:04:45.430 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:45.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:45.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:45.541 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:04:45.602 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:45.650 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:04:45.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:45.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:04:45.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:04:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:04:45] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.43.0" 2023-10-09T06:04:45.754 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:04:45.787 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:45.832 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:04:45.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:45.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:04:45.938 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:04:45.972 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:46.192 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:04:46.193 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:46.196 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:04:46.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:46.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:04:46.304 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:46.368 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:46 smithi064 ceph-mon[212750]: pgmap v924: 119 pgs: 119 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 7.1 MiB/s wr, 811 op/s 2023-10-09T06:04:46.369 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:46 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:04:46.369 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:46 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:04:46.369 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:46 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:04:46.369 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:46 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:04:46.369 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:46 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:04:46.369 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:46 smithi064 ceph-mon[220044]: pgmap v924: 119 pgs: 119 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 7.1 MiB/s wr, 811 op/s 2023-10-09T06:04:46.369 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:46 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:04:46.369 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:46 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:04:46.369 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:46 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:04:46.370 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:46 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:04:46.370 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:46 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:04:46.373 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:46.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:46 smithi156 ceph-mon[195101]: pgmap v924: 119 pgs: 119 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 7.1 MiB/s wr, 811 op/s 2023-10-09T06:04:46.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:46 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:04:46.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:46 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:04:46.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:46 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:04:46.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:46 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:04:46.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:46 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:04:46.718 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:46.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:46.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:46.825 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:04:46.873 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:46.949 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:04:46.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:46.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:04:47.015 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:04:47.015 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "" 2023-10-09T06:04:47.015 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:04:47.042 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:04:47.057 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:04:47.107 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:47.179 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:04:47.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:47.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:04:47.289 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:47.378 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:48.679 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:48 smithi064 ceph-mon[212750]: pgmap v925: 119 pgs: 119 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2023-10-09T06:04:48.679 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:48 smithi064 ceph-mon[220044]: pgmap v925: 119 pgs: 119 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2023-10-09T06:04:48.852 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:48.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:48.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:48.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:48 smithi156 ceph-mon[195101]: pgmap v925: 119 pgs: 119 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2023-10-09T06:04:48.948 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:48.968 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:04:49.077 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:49.479 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:04:49.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:49.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:04:49.528 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:49.591 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:49.637 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:49.703 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:49.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:49.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:49.812 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:49.845 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:49.913 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:49.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:49.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:50.023 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:50.090 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:50.427 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:04:50.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:50 smithi064 ceph-mon[212750]: pgmap v926: 119 pgs: 119 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2023-10-09T06:04:50.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:50 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2695066862' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:04:50.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:50 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:04:50.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:50 smithi064 ceph-mon[220044]: pgmap v926: 119 pgs: 119 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2023-10-09T06:04:50.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:50 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2695066862' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:04:50.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:50 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:04:50.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:50 smithi156 ceph-mon[195101]: pgmap v926: 119 pgs: 119 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2023-10-09T06:04:50.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:50 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2695066862' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:04:50.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:50 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:04:51.444 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:51 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:51.445 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:51 smithi064 ceph-mon[212750]: failed to encode map e379 with expected crc 2023-10-09T06:04:51.445 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:51 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:04:51.445 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:51 smithi064 ceph-mon[212750]: osdmap e379: 8 total, 8 up, 8 in 2023-10-09T06:04:51.445 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:51 smithi064 ceph-mon[212750]: failed to encode map e379 with expected crc 2023-10-09T06:04:51.445 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:51 smithi064 ceph-mon[212750]: failed to encode map e379 with expected crc 2023-10-09T06:04:51.445 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:51 smithi064 ceph-mon[212750]: failed to encode map e379 with expected crc 2023-10-09T06:04:51.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:51 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:51.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:51 smithi064 ceph-mon[220044]: failed to encode map e379 with expected crc 2023-10-09T06:04:51.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:51 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:04:51.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:51 smithi064 ceph-mon[220044]: osdmap e379: 8 total, 8 up, 8 in 2023-10-09T06:04:51.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:51 smithi064 ceph-mon[220044]: failed to encode map e379 with expected crc 2023-10-09T06:04:51.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:51 smithi064 ceph-mon[220044]: failed to encode map e379 with expected crc 2023-10-09T06:04:51.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:51 smithi064 ceph-mon[220044]: failed to encode map e379 with expected crc 2023-10-09T06:04:51.824 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:51.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:51.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:51.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:51 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:04:51.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:51 smithi156 ceph-mon[195101]: failed to encode map e379 with expected crc 2023-10-09T06:04:51.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:51 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:04:51.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:51 smithi156 ceph-mon[195101]: osdmap e379: 8 total, 8 up, 8 in 2023-10-09T06:04:51.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:51 smithi156 ceph-mon[195101]: failed to encode map e379 with expected crc 2023-10-09T06:04:51.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:51 smithi156 ceph-mon[195101]: failed to encode map e379 with expected crc 2023-10-09T06:04:51.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:51 smithi156 ceph-mon[195101]: failed to encode map e379 with expected crc 2023-10-09T06:04:51.935 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:51.983 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:52.056 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:52.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:52.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:52.172 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:52.201 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:52.247 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:52.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:52.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:52.351 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:52.426 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:52.705 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:52.709 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:52 smithi064 ceph-mon[212750]: failed to encode map e379 with expected crc 2023-10-09T06:04:52.709 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:52 smithi064 ceph-mon[212750]: pgmap v928: 119 pgs: 119 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 780 op/s 2023-10-09T06:04:52.709 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:52 smithi064 ceph-mon[220044]: failed to encode map e379 with expected crc 2023-10-09T06:04:52.709 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:52 smithi064 ceph-mon[220044]: pgmap v928: 119 pgs: 119 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 780 op/s 2023-10-09T06:04:52.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:52.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:52.811 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:04:52.817 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:04:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:04:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:120404 8.3 8:c4a51686:::benchmark_data_smithi064_457003_object15886:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-10-09T06:04:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:04:52.793521+0000", 2023-10-09T06:04:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.014242988999999999, 2023-10-09T06:04:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014277754, 2023-10-09T06:04:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:04:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:04:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:04:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:04:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:04:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 120404 2023-10-09T06:04:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:04:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:04:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.793521+0000", 2023-10-09T06:04:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:04:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:04:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.793521+0000", 2023-10-09T06:04:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:04:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:04:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.793520+0000", 2023-10-09T06:04:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:04:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:04:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.793522+0000", 2023-10-09T06:04:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9290000000000001e-06 2023-10-09T06:04:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:04:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.793525+0000", 2023-10-09T06:04:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.066e-06 2023-10-09T06:04:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:04:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.793528+0000", 2023-10-09T06:04:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.695e-06 2023-10-09T06:04:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:04:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.803350+0000", 2023-10-09T06:04:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0098224990000000002 2023-10-09T06:04:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:04:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.803480+0000", 2023-10-09T06:04:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012998299999999999 2023-10-09T06:04:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:04:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.803609+0000", 2023-10-09T06:04:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012860700000000001 2023-10-09T06:04:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.807161+0000", 2023-10-09T06:04:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035521839999999999 2023-10-09T06:04:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:04:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.807279+0000", 2023-10-09T06:04:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000117583 2023-10-09T06:04:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:04:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:04:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:04:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:120410 8.1 8:8f4dc76b:::benchmark_data_smithi064_457003_object15892:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-10-09T06:04:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:04:52.799812+0000", 2023-10-09T06:04:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0079518109999999996, 2023-10-09T06:04:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0080995560000000008, 2023-10-09T06:04:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:04:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:04:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:04:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:04:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:04:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 120410 2023-10-09T06:04:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:04:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:04:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.799812+0000", 2023-10-09T06:04:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:04:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:04:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.799812+0000", 2023-10-09T06:04:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:04:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:04:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.799810+0000", 2023-10-09T06:04:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:04:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:04:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.799817+0000", 2023-10-09T06:04:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2570000000000004e-06 2023-10-09T06:04:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:04:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.799829+0000", 2023-10-09T06:04:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2130000000000001e-05 2023-10-09T06:04:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:04:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.799837+0000", 2023-10-09T06:04:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2689999999999999e-06 2023-10-09T06:04:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:04:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.800078+0000", 2023-10-09T06:04:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000241189 2023-10-09T06:04:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:04:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.800210+0000", 2023-10-09T06:04:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013171 2023-10-09T06:04:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:04:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.800335+0000", 2023-10-09T06:04:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012537599999999999 2023-10-09T06:04:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:04:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.804034+0000", 2023-10-09T06:04:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003698322 2023-10-09T06:04:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:04:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.805794+0000", 2023-10-09T06:04:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0017597509999999999 2023-10-09T06:04:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:04:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:04:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:04:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:120413 8.7 8:e3f6909f:::benchmark_data_smithi064_457003_object15895:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-10-09T06:04:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:04:52.803845+0000", 2023-10-09T06:04:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0039185390000000004, 2023-10-09T06:04:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041304810000000001, 2023-10-09T06:04:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:04:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:04:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:04:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:04:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:04:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 120413 2023-10-09T06:04:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:04:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:04:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.803845+0000", 2023-10-09T06:04:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:04:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:04:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.803845+0000", 2023-10-09T06:04:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:04:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:04:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.803844+0000", 2023-10-09T06:04:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:04:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:04:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.803849+0000", 2023-10-09T06:04:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.938e-06 2023-10-09T06:04:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:04:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.803859+0000", 2023-10-09T06:04:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6819999999999998e-06 2023-10-09T06:04:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:04:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.803866+0000", 2023-10-09T06:04:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3420000000000002e-06 2023-10-09T06:04:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:04:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.807500+0000", 2023-10-09T06:04:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003634473 2023-10-09T06:04:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:04:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.807630+0000", 2023-10-09T06:04:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000129622 2023-10-09T06:04:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:04:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.807746+0000", 2023-10-09T06:04:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116123 2023-10-09T06:04:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:04:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:04:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:04:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:120414 8.7 8.49ba9137 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-10-09T06:04:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:04:52.804591+0000", 2023-10-09T06:04:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0031721739999999998, 2023-10-09T06:04:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034623779999999999, 2023-10-09T06:04:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:04:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-10-09T06:04:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:04:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:04:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:04:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 120414 2023-10-09T06:04:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:04:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:04:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.804591+0000", 2023-10-09T06:04:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:04:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:04:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.804591+0000", 2023-10-09T06:04:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:04:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:04:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.804590+0000", 2023-10-09T06:04:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:04:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:04:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.804596+0000", 2023-10-09T06:04:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9030000000000001e-06 2023-10-09T06:04:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:04:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.804608+0000", 2023-10-09T06:04:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1379000000000001e-05 2023-10-09T06:04:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:04:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:04:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:04:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.804613+0000", 2023-10-09T06:04:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2839999999999997e-06 2023-10-09T06:04:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:04:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:04:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:04:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:04:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-10-09T06:04:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 4 2023-10-09T06:04:52.840 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:04:52.842 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:52 smithi156 ceph-mon[195101]: failed to encode map e379 with expected crc 2023-10-09T06:04:52.842 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:52 smithi156 ceph-mon[195101]: pgmap v928: 119 pgs: 119 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 780 op/s 2023-10-09T06:04:52.845 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:52.930 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:04:52.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:52.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:04:53.043 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:53.143 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:53.777 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:04:53.828 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:53.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:53.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:53.847 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:04:53.939 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:53.967 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:53.995 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 0, 4] 2023-10-09T06:04:53.996 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:04:53.996 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T06:04:53.996 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:04:53.996 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:04:53.996 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T06:04:54.031 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:54.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:54.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:54.141 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:54.174 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:04:54.180 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:54.224 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:04:54.258 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:54.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:54.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:54.267 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:04:54.340 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:04:54.364 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:54.381 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:04:54.426 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:04:54.446 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:54.483 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:04:54.518 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:04:54.519 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:04:54.527 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:04:54.527 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:04:54.527 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:04:54.527 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:04:54.527 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:04:54.527 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:04:54.528 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:04:54.528 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:04:54.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:54 smithi156 ceph-mon[195101]: pgmap v929: 119 pgs: 119 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2023-10-09T06:04:54.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:54 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/733787814' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:54.703 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:54 smithi064 ceph-mon[212750]: pgmap v929: 119 pgs: 119 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2023-10-09T06:04:54.703 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:54 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/733787814' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:54.703 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:54 smithi064 ceph-mon[220044]: pgmap v929: 119 pgs: 119 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2023-10-09T06:04:54.704 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:54 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/733787814' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:04:54.762 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:54.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:54.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:54.870 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:54.925 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:55.070 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:55.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:55.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:55.173 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:04:55.242 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:55.326 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:04:55.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:55.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:04:55.430 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:04:55.456 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:55.463 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:55.511 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:55.532 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:04:55.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:55.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:04:55.650 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:55.717 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:04:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:04:55] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.43.0" 2023-10-09T06:04:55.718 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:55.885 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:55.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:55.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:55.995 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:04:56.048 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:56.124 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:04:56.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:56.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:04:56.229 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:04:56.297 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:56.439 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:04:56.439 INFO:teuthology.orchestra.run.smithi156.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T06:04:56.439 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:04:56.459 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:56 smithi156 ceph-mon[195101]: pgmap v930: 119 pgs: 119 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2023-10-09T06:04:56.681 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:04:56.684 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:56 smithi064 ceph-mon[220044]: pgmap v930: 119 pgs: 119 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2023-10-09T06:04:56.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:56.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:04:56.686 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:56 smithi064 ceph-mon[212750]: pgmap v930: 119 pgs: 119 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2023-10-09T06:04:56.786 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:56.873 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:57.292 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:57.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:57.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:57.406 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:04:57.454 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:57.486 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:04:57.530 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:04:57.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:57.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:04:57.638 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:57.733 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:57.880 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:57.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:57.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:57.988 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:58.133 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:58.314 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:04:58.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:58.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:04:58.422 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:04:58.501 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:58.621 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:58 smithi064 ceph-mon[212750]: pgmap v931: 119 pgs: 119 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 781 op/s 2023-10-09T06:04:58.621 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:58 smithi064 ceph-mon[220044]: pgmap v931: 119 pgs: 119 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 781 op/s 2023-10-09T06:04:58.677 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:04:58.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:58.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:04:58.785 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:04:58.818 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:58.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:58 smithi156 ceph-mon[195101]: pgmap v931: 119 pgs: 119 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 781 op/s 2023-10-09T06:04:58.926 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:58.985 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:04:58.985 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:04:59.293 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:04:59.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:04:59.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:04:59.408 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:04:59.484 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:04:59.521 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:04:59.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:59.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:59 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1248633662' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:59.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:04:59 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:59.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:59.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:59 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1248633662' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:59.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:04:59 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:59.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:59.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:59 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1248633662' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:04:59.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:04:59 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:05:00.715 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:00.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:00.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:00.743 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:00 smithi064 ceph-mon[212750]: pgmap v932: 119 pgs: 119 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 778 op/s 2023-10-09T06:05:00.744 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:00 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:05:00.744 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:00 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:05:00.744 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:00 smithi064 ceph-mon[212750]: failed to encode map e380 with expected crc 2023-10-09T06:05:00.744 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:00 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:05:00.744 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:00 smithi064 ceph-mon[212750]: osdmap e380: 8 total, 8 up, 8 in 2023-10-09T06:05:00.744 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:00 smithi064 ceph-mon[212750]: failed to encode map e380 with expected crc 2023-10-09T06:05:00.744 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:00 smithi064 ceph-mon[220044]: pgmap v932: 119 pgs: 119 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 778 op/s 2023-10-09T06:05:00.744 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:00 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:05:00.745 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:00 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T06:05:00.745 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:00 smithi064 ceph-mon[220044]: failed to encode map e380 with expected crc 2023-10-09T06:05:00.745 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:00 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:05:00.745 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:00 smithi064 ceph-mon[220044]: osdmap e380: 8 total, 8 up, 8 in 2023-10-09T06:05:00.745 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:00 smithi064 ceph-mon[220044]: failed to encode map e380 with expected crc 2023-10-09T06:05:00.831 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:05:00.859 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:00.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:00 smithi156 ceph-mon[195101]: pgmap v932: 119 pgs: 119 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 778 op/s 2023-10-09T06:05:00.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:00 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:05:00.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:00 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T06:05:00.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:00 smithi156 ceph-mon[195101]: failed to encode map e380 with expected crc 2023-10-09T06:05:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:00 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:05:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:00 smithi156 ceph-mon[195101]: osdmap e380: 8 total, 8 up, 8 in 2023-10-09T06:05:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:00 smithi156 ceph-mon[195101]: failed to encode map e380 with expected crc 2023-10-09T06:05:01.338 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:05:01.495 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:05:01.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:01.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:05:01.605 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:05:01.656 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:02.156 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:05:02.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:02.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:05:02.263 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:02.308 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:02.340 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7, 1] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 0, 4] 2023-10-09T06:05:02.341 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:05:02.341 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [0, 2, 3, 4, 5, 6, 7, 1] 2023-10-09T06:05:02.341 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd out 2 2023-10-09T06:05:02.393 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:02.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:02.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:02.431 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:02.501 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:02.542 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:02.607 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:02.612 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:02 smithi156 ceph-mon[195101]: pgmap v934: 119 pgs: 119 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2023-10-09T06:05:02.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:02.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:02.714 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:02.750 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:02.818 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:02.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:02.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:02.823 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:02 smithi064 ceph-mon[212750]: pgmap v934: 119 pgs: 119 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2023-10-09T06:05:02.824 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:02 smithi064 ceph-mon[220044]: pgmap v934: 119 pgs: 119 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2023-10-09T06:05:02.923 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:05:02.983 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:03.146 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:05:03.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:03.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:05:03.156 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:05:03.186 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:05:03.219 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:05:03.257 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:03.264 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:05:03.310 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:05:03.324 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:03.365 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:05:03.418 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:05:03.466 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:05:03.474 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:05:03.474 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:05:03.474 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:05:03.474 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:05:03.474 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:05:03.474 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:05:03.474 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:05:03.474 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:05:03.488 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:03.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:03.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:03.597 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:03.651 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:03.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:03 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3627388103' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:03.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:03 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3627388103' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:03.724 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:03.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:03.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:03.839 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:03.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:03 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3627388103' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:03.899 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:03.957 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:05:03.999 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:04.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:04.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:04.113 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:04.161 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:04.857 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:04.859 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:04 smithi064 ceph-mon[220044]: pgmap v935: 119 pgs: 119 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-10-09T06:05:04.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:04.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:04.860 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:04 smithi064 ceph-mon[212750]: pgmap v935: 119 pgs: 119 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-10-09T06:05:04.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:04 smithi156 ceph-mon[195101]: pgmap v935: 119 pgs: 119 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-10-09T06:05:04.961 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:05.006 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:05.224 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:05.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:05.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:05.334 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:05.380 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:05.433 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:05.445 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:05.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:05.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:05.477 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:05:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:05:05] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.43.0" 2023-10-09T06:05:05.554 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:05.587 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:05.657 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:05.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:05.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:05.766 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:05.821 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:05.899 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:05.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:05.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:05.975 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:06.011 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:06.050 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:06.114 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:06.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:06.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:06.126 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:05:06.127 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T06:05:06.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T06:05:06.127 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:05:06.127 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.127 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119186 6.21 6:86b6e4ca:::benchmark_data_smithi064_284193_object22643:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e214)", 2023-10-09T06:05:06.127 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:44.656755+0000", 2023-10-09T06:05:06.127 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 561.46283929799995, 2023-10-09T06:05:06.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1584478259999997, 2023-10-09T06:05:06.127 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:06.127 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.128 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:06.128 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:05:06.128 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:05:06.128 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119186 2023-10-09T06:05:06.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.128 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:06.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.128 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:06.128 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656755+0000", 2023-10-09T06:05:06.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:06.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656755+0000", 2023-10-09T06:05:06.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:06.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656753+0000", 2023-10-09T06:05:06.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:05:06.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:06.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656760+0000", 2023-10-09T06:05:06.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8290000000000002e-06 2023-10-09T06:05:06.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:06.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656773+0000", 2023-10-09T06:05:06.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3166000000000001e-05 2023-10-09T06:05:06.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656783+0000", 2023-10-09T06:05:06.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0210000000000001e-05 2023-10-09T06:05:06.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.658579+0000", 2023-10-09T06:05:06.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001795488 2023-10-09T06:05:06.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:05:06.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.658592+0000", 2023-10-09T06:05:06.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.278e-05 2023-10-09T06:05:06.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.743680+0000", 2023-10-09T06:05:06.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0850882960000003 2023-10-09T06:05:06.132 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:06.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.749035+0000", 2023-10-09T06:05:06.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053553639999999996 2023-10-09T06:05:06.132 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.133 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.776409+0000", 2023-10-09T06:05:06.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.027373859 2023-10-09T06:05:06.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.133 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:06.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.778175+0000", 2023-10-09T06:05:06.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001765953 2023-10-09T06:05:06.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.133 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:05:06.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.781558+0000", 2023-10-09T06:05:06.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033831220000000001 2023-10-09T06:05:06.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.784638+0000", 2023-10-09T06:05:06.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030801969999999998 2023-10-09T06:05:06.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.785224+0000", 2023-10-09T06:05:06.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00058571999999999997 2023-10-09T06:05:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.811281+0000", 2023-10-09T06:05:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.026056551000000001 2023-10-09T06:05:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.811998+0000", 2023-10-09T06:05:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00071781999999999998 2023-10-09T06:05:06.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:06.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.815203+0000", 2023-10-09T06:05:06.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032041819999999999 2023-10-09T06:05:06.136 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.136 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:06.136 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.136 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119190 6.11 6:89fd5615:::benchmark_data_smithi064_284193_object22647:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e214)", 2023-10-09T06:05:06.136 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:44.656790+0000", 2023-10-09T06:05:06.136 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 561.46280431399998, 2023-10-09T06:05:06.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0863524140000003, 2023-10-09T06:05:06.137 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:06.137 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.137 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:06.137 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:05:06.137 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:05:06.137 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119190 2023-10-09T06:05:06.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.137 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:06.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:06.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656790+0000", 2023-10-09T06:05:06.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:06.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656790+0000", 2023-10-09T06:05:06.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:06.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656789+0000", 2023-10-09T06:05:06.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:05:06.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:06.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656792+0000", 2023-10-09T06:05:06.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6599999999999999e-06 2023-10-09T06:05:06.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:06.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656798+0000", 2023-10-09T06:05:06.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9449999999999998e-06 2023-10-09T06:05:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.656803+0000", 2023-10-09T06:05:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6060000000000002e-06 2023-10-09T06:05:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.658616+0000", 2023-10-09T06:05:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018122049999999999 2023-10-09T06:05:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:05:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.658631+0000", 2023-10-09T06:05:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4922999999999999e-05 2023-10-09T06:05:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.663857+0000", 2023-10-09T06:05:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0052261009999999 2023-10-09T06:05:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:06.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.673837+0000", 2023-10-09T06:05:06.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0099803089999999997 2023-10-09T06:05:06.142 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.142 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.705811+0000", 2023-10-09T06:05:06.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.031974225000000002 2023-10-09T06:05:06.142 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.142 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:06.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.708838+0000", 2023-10-09T06:05:06.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030267570000000001 2023-10-09T06:05:06.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.143 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:05:06.143 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.715396+0000", 2023-10-09T06:05:06.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065577710000000004 2023-10-09T06:05:06.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.143 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.143 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.731615+0000", 2023-10-09T06:05:06.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016219506000000002 2023-10-09T06:05:06.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.732772+0000", 2023-10-09T06:05:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011562300000000001 2023-10-09T06:05:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.733208+0000", 2023-10-09T06:05:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000436728 2023-10-09T06:05:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.734975+0000", 2023-10-09T06:05:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001766739 2023-10-09T06:05:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.743142+0000", 2023-10-09T06:05:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0081671590000000002 2023-10-09T06:05:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.146 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119234 6.11 6:8be56ac0:::benchmark_data_smithi064_284193_object22691:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-10-09T06:05:06.146 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:44.944674+0000", 2023-10-09T06:05:06.146 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 561.17492013100002, 2023-10-09T06:05:06.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9036751780000003, 2023-10-09T06:05:06.146 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:06.147 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.147 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:06.147 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:05:06.147 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:05:06.147 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119234 2023-10-09T06:05:06.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.147 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:06.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.944674+0000", 2023-10-09T06:05:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.944674+0000", 2023-10-09T06:05:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.944672+0000", 2023-10-09T06:05:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:05:06.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.149 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:06.149 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.944681+0000", 2023-10-09T06:05:06.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1509999999999993e-06 2023-10-09T06:05:06.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.149 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:06.149 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.944699+0000", 2023-10-09T06:05:06.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7589e-05 2023-10-09T06:05:06.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.150 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.944708+0000", 2023-10-09T06:05:06.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9090000000000006e-06 2023-10-09T06:05:06.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.150 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.944762+0000", 2023-10-09T06:05:06.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4095000000000003e-05 2023-10-09T06:05:06.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:05:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:44.944774+0000", 2023-10-09T06:05:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2248e-05 2023-10-09T06:05:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.720858+0000", 2023-10-09T06:05:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7760837649999992 2023-10-09T06:05:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.730024+0000", 2023-10-09T06:05:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0091661759999999998 2023-10-09T06:05:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.775720+0000", 2023-10-09T06:05:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.045695982000000003 2023-10-09T06:05:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:06.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.782885+0000", 2023-10-09T06:05:06.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0071655690000000001 2023-10-09T06:05:06.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.153 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.153 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:05:06.153 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.792563+0000", 2023-10-09T06:05:06.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0096778569999999998 2023-10-09T06:05:06.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.153 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.153 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.153 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.810903+0000", 2023-10-09T06:05:06.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018339273999999999 2023-10-09T06:05:06.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.812158+0000", 2023-10-09T06:05:06.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001255775 2023-10-09T06:05:06.154 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:06.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.836393+0000", 2023-10-09T06:05:06.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.024234970000000002 2023-10-09T06:05:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.839508+0000", 2023-10-09T06:05:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031143970000000001 2023-10-09T06:05:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.848349+0000", 2023-10-09T06:05:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0088413299999999997 2023-10-09T06:05:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.155 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119237 6.11 6:8ee80634:::benchmark_data_smithi064_284193_object22694:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-10-09T06:05:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:45.702303+0000", 2023-10-09T06:05:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 560.41729135699995, 2023-10-09T06:05:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1973806820000004, 2023-10-09T06:05:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:05:06.156 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:05:06.157 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119237 2023-10-09T06:05:06.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.157 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:06.157 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.157 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:06.157 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.702303+0000", 2023-10-09T06:05:06.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.157 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.157 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:06.157 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.702303+0000", 2023-10-09T06:05:06.158 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:06.158 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.702301+0000", 2023-10-09T06:05:06.158 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:05:06.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.702308+0000", 2023-10-09T06:05:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2830000000000004e-06 2023-10-09T06:05:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.716185+0000", 2023-10-09T06:05:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013876292 2023-10-09T06:05:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.716200+0000", 2023-10-09T06:05:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5145e-05 2023-10-09T06:05:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.798524+0000", 2023-10-09T06:05:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0823238269999997 2023-10-09T06:05:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.809274+0000", 2023-10-09T06:05:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010750116000000001 2023-10-09T06:05:06.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.161 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.848945+0000", 2023-10-09T06:05:06.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.039671062999999999 2023-10-09T06:05:06.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.161 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:06.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.854842+0000", 2023-10-09T06:05:06.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058972850000000004 2023-10-09T06:05:06.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.161 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:05:06.162 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.862009+0000", 2023-10-09T06:05:06.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0071667279999999998 2023-10-09T06:05:06.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.162 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.885199+0000", 2023-10-09T06:05:06.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023190399 2023-10-09T06:05:06.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.885882+0000", 2023-10-09T06:05:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00068252700000000005 2023-10-09T06:05:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.888407+0000", 2023-10-09T06:05:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025256649999999999 2023-10-09T06:05:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:06.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.890797+0000", 2023-10-09T06:05:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0023894400000000001 2023-10-09T06:05:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.899683+0000", 2023-10-09T06:05:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0088864919999999993 2023-10-09T06:05:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.164 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119262 6.21 6:845e8c29:::benchmark_data_smithi064_284193_object22719:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-10-09T06:05:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:45.846736+0000", 2023-10-09T06:05:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 560.27285791600002, 2023-10-09T06:05:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0664286440000001, 2023-10-09T06:05:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:05:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:05:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119262 2023-10-09T06:05:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.165 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:06.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:06.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.846736+0000", 2023-10-09T06:05:06.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:06.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.846736+0000", 2023-10-09T06:05:06.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.846734+0000", 2023-10-09T06:05:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:05:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.846743+0000", 2023-10-09T06:05:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0820000000000005e-06 2023-10-09T06:05:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.846757+0000", 2023-10-09T06:05:06.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4083999999999999e-05 2023-10-09T06:05:06.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:45.846765+0000", 2023-10-09T06:05:06.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3339999999999996e-06 2023-10-09T06:05:06.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.848937+0000", 2023-10-09T06:05:06.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0021717930000005 2023-10-09T06:05:06.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:06.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.854164+0000", 2023-10-09T06:05:06.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0052279120000000004 2023-10-09T06:05:06.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.900206+0000", 2023-10-09T06:05:06.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.046042012 2023-10-09T06:05:06.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:06.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.901864+0000", 2023-10-09T06:05:06.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001657258 2023-10-09T06:05:06.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:05:06.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.905026+0000", 2023-10-09T06:05:06.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031627719999999999 2023-10-09T06:05:06.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.908161+0000", 2023-10-09T06:05:06.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031345349999999999 2023-10-09T06:05:06.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:06.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.908349+0000", 2023-10-09T06:05:06.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000187911 2023-10-09T06:05:06.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.908935+0000", 2023-10-09T06:05:06.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00058620600000000003 2023-10-09T06:05:06.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:06.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.909877+0000", 2023-10-09T06:05:06.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00094206200000000002 2023-10-09T06:05:06.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.913165+0000", 2023-10-09T06:05:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003287594 2023-10-09T06:05:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119308 6.11 6:8cb42c43:::benchmark_data_smithi064_284193_object22765:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-10-09T06:05:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:46.087787+0000", 2023-10-09T06:05:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 560.03180740599998, 2023-10-09T06:05:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8988220839999999, 2023-10-09T06:05:06.173 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:06.174 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.174 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:06.174 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:05:06.174 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:05:06.174 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119308 2023-10-09T06:05:06.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.174 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:06.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:06.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.087787+0000", 2023-10-09T06:05:06.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.087787+0000", 2023-10-09T06:05:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.087785+0000", 2023-10-09T06:05:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:05:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:06.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.087796+0000", 2023-10-09T06:05:06.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1157e-05 2023-10-09T06:05:06.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:06.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.087811+0000", 2023-10-09T06:05:06.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5238999999999999e-05 2023-10-09T06:05:06.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.087822+0000", 2023-10-09T06:05:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1399e-05 2023-10-09T06:05:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.867168+0000", 2023-10-09T06:05:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7793453689999996 2023-10-09T06:05:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.883209+0000", 2023-10-09T06:05:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016041239999999998 2023-10-09T06:05:06.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.922185+0000", 2023-10-09T06:05:06.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.038975888 2023-10-09T06:05:06.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:06.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.928697+0000", 2023-10-09T06:05:06.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065118830000000004 2023-10-09T06:05:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:05:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.941529+0000", 2023-10-09T06:05:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012832487 2023-10-09T06:05:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.966726+0000", 2023-10-09T06:05:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.025196347000000001 2023-10-09T06:05:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.968542+0000", 2023-10-09T06:05:06.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001816736 2023-10-09T06:05:06.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.970863+0000", 2023-10-09T06:05:06.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00232102 2023-10-09T06:05:06.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:06.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.974139+0000", 2023-10-09T06:05:06.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003276143 2023-10-09T06:05:06.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:06.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.986609+0000", 2023-10-09T06:05:06.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012469236 2023-10-09T06:05:06.181 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.181 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:06.181 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.182 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119315 6.21 6:844be45b:::benchmark_data_smithi064_284193_object22772:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-10-09T06:05:06.182 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:46.187262+0000", 2023-10-09T06:05:06.182 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 559.93233235399998, 2023-10-09T06:05:06.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7650340010000001, 2023-10-09T06:05:06.182 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:06.182 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.182 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:06.182 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:05:06.182 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:05:06.182 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119315 2023-10-09T06:05:06.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.187262+0000", 2023-10-09T06:05:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.187262+0000", 2023-10-09T06:05:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.187260+0000", 2023-10-09T06:05:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:05:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.187270+0000", 2023-10-09T06:05:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0227000000000001e-05 2023-10-09T06:05:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:06.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.187287+0000", 2023-10-09T06:05:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6762e-05 2023-10-09T06:05:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.187298+0000", 2023-10-09T06:05:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1574e-05 2023-10-09T06:05:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.913351+0000", 2023-10-09T06:05:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7260523179999998 2023-10-09T06:05:06.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:06.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.917641+0000", 2023-10-09T06:05:06.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042900730000000002 2023-10-09T06:05:06.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.930458+0000", 2023-10-09T06:05:06.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012817034999999999 2023-10-09T06:05:06.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.932203+0000", 2023-10-09T06:05:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001745459 2023-10-09T06:05:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:05:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.935768+0000", 2023-10-09T06:05:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00356522 2023-10-09T06:05:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.938933+0000", 2023-10-09T06:05:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031648140000000002 2023-10-09T06:05:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.947606+0000", 2023-10-09T06:05:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086732140000000003 2023-10-09T06:05:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.947767+0000", 2023-10-09T06:05:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016054900000000001 2023-10-09T06:05:06.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.948714+0000", 2023-10-09T06:05:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00094673999999999995 2023-10-09T06:05:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.952296+0000", 2023-10-09T06:05:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035820740000000002 2023-10-09T06:05:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.190 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:119330 6.11 6:8f58021f:::benchmark_data_smithi064_284193_object22787:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-10-09T06:05:06.190 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:55:46.350194+0000", 2023-10-09T06:05:06.190 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 559.769400292, 2023-10-09T06:05:06.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7595639490000003, 2023-10-09T06:05:06.190 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:06.190 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.190 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:06.190 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:05:06.190 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:05:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119330 2023-10-09T06:05:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.350194+0000", 2023-10-09T06:05:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.350194+0000", 2023-10-09T06:05:06.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.350192+0000", 2023-10-09T06:05:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:05:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.350201+0000", 2023-10-09T06:05:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0410000000000003e-06 2023-10-09T06:05:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.350222+0000", 2023-10-09T06:05:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0968e-05 2023-10-09T06:05:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:46.350232+0000", 2023-10-09T06:05:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8330000000000006e-06 2023-10-09T06:05:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.947802+0000", 2023-10-09T06:05:06.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5975698659999997 2023-10-09T06:05:06.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-10-09T06:05:06.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:53.966003+0000", 2023-10-09T06:05:06.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018200917 2023-10-09T06:05:06.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:54.029395+0000", 2023-10-09T06:05:06.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.063392047000000007 2023-10-09T06:05:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:54.038664+0000", 2023-10-09T06:05:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0092691679999999995 2023-10-09T06:05:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:05:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:54.047296+0000", 2023-10-09T06:05:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086318609999999994 2023-10-09T06:05:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:54.067975+0000", 2023-10-09T06:05:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020679792999999998 2023-10-09T06:05:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:54.070461+0000", 2023-10-09T06:05:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024859119999999998 2023-10-09T06:05:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:54.094248+0000", 2023-10-09T06:05:06.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023787081000000002 2023-10-09T06:05:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:54.098008+0000", 2023-10-09T06:05:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037596750000000001 2023-10-09T06:05:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:55:54.109758+0000", 2023-10-09T06:05:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011749625 2023-10-09T06:05:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.197 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:06.198 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.198 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:84654 8.7 8:e2f8af57:::benchmark_data_smithi064_457003_object84653:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-10-09T06:05:06.198 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:04:08.017817+0000", 2023-10-09T06:05:06.198 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 58.101777147, 2023-10-09T06:05:06.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.267264591, 2023-10-09T06:05:06.198 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:06.198 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.198 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:06.198 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:05:06.198 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:05:06.199 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 84654 2023-10-09T06:05:06.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.199 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:06.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:06.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:08.017817+0000", 2023-10-09T06:05:06.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:06.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:08.017817+0000", 2023-10-09T06:05:06.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:06.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:08.017816+0000", 2023-10-09T06:05:06.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:05:06.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:06.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:08.017839+0000", 2023-10-09T06:05:06.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2858999999999999e-05 2023-10-09T06:05:06.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:06.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:08.017847+0000", 2023-10-09T06:05:06.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2359999999999998e-06 2023-10-09T06:05:06.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:08.017852+0000", 2023-10-09T06:05:06.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4689999999999996e-06 2023-10-09T06:05:06.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:08.020946+0000", 2023-10-09T06:05:06.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030937009999999999 2023-10-09T06:05:06.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:06.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:08.021113+0000", 2023-10-09T06:05:06.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016726100000000001 2023-10-09T06:05:06.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:05:06.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:08.021289+0000", 2023-10-09T06:05:06.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000175613 2023-10-09T06:05:06.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:06.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:08.024152+0000", 2023-10-09T06:05:06.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028636949999999999 2023-10-09T06:05:06.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:08.032997+0000", 2023-10-09T06:05:06.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.008844569 2023-10-09T06:05:06.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:08.284988+0000", 2023-10-09T06:05:06.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25199083 2023-10-09T06:05:06.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:06.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:08.285041+0000", 2023-10-09T06:05:06.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3026999999999999e-05 2023-10-09T06:05:06.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:06.205 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:08.285081+0000", 2023-10-09T06:05:06.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0810999999999999e-05 2023-10-09T06:05:06.205 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.205 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:06.205 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.206 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:113837 8.7 8:ed9e6c6a:::benchmark_data_smithi064_457003_object9319:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-10-09T06:05:06.206 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:04:44.113645+0000", 2023-10-09T06:05:06.206 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 22.005948919000001, 2023-10-09T06:05:06.206 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25526415499999999, 2023-10-09T06:05:06.206 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:06.206 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.206 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:06.207 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:05:06.207 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:05:06.207 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 113837 2023-10-09T06:05:06.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.207 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:06.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:06.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.113645+0000", 2023-10-09T06:05:06.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:06.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.113645+0000", 2023-10-09T06:05:06.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:06.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.113643+0000", 2023-10-09T06:05:06.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:05:06.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.210 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:06.210 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.113651+0000", 2023-10-09T06:05:06.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8180000000000004e-06 2023-10-09T06:05:06.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.211 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:06.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.113668+0000", 2023-10-09T06:05:06.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6940999999999999e-05 2023-10-09T06:05:06.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.113677+0000", 2023-10-09T06:05:06.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5390000000000007e-06 2023-10-09T06:05:06.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.113728+0000", 2023-10-09T06:05:06.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1663000000000003e-05 2023-10-09T06:05:06.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:06.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.113837+0000", 2023-10-09T06:05:06.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010853699999999999 2023-10-09T06:05:06.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.214 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:05:06.214 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.113942+0000", 2023-10-09T06:05:06.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000105164 2023-10-09T06:05:06.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.214 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:06.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.117443+0000", 2023-10-09T06:05:06.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035004110000000001 2023-10-09T06:05:06.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.117634+0000", 2023-10-09T06:05:06.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019185799999999999 2023-10-09T06:05:06.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.368822+0000", 2023-10-09T06:05:06.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.251187875 2023-10-09T06:05:06.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:06.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.368869+0000", 2023-10-09T06:05:06.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7052000000000003e-05 2023-10-09T06:05:06.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:06.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.368909+0000", 2023-10-09T06:05:06.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9950000000000002e-05 2023-10-09T06:05:06.216 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.217 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:06.217 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.217 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:113842 8.7 8:ed5243a2:::benchmark_data_smithi064_457003_object9324:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-10-09T06:05:06.217 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:04:44.118705+0000", 2023-10-09T06:05:06.217 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 22.000889009000002, 2023-10-09T06:05:06.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25030826, 2023-10-09T06:05:06.217 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:06.217 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.217 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:06.217 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:05:06.218 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:05:06.218 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 113842 2023-10-09T06:05:06.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.218 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:06.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.218 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:06.218 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.118705+0000", 2023-10-09T06:05:06.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.218 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:06.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.118705+0000", 2023-10-09T06:05:06.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:06.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.118703+0000", 2023-10-09T06:05:06.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:05:06.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:06.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.118710+0000", 2023-10-09T06:05:06.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7229999999999998e-06 2023-10-09T06:05:06.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.220 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:06.220 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.118727+0000", 2023-10-09T06:05:06.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6453e-05 2023-10-09T06:05:06.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.220 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.220 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.118734+0000", 2023-10-09T06:05:06.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8420000000000005e-06 2023-10-09T06:05:06.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.118805+0000", 2023-10-09T06:05:06.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0585000000000003e-05 2023-10-09T06:05:06.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:06.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.118945+0000", 2023-10-09T06:05:06.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000140469 2023-10-09T06:05:06.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:05:06.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.119085+0000", 2023-10-09T06:05:06.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013911800000000001 2023-10-09T06:05:06.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:06.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.125101+0000", 2023-10-09T06:05:06.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060166519999999999 2023-10-09T06:05:06.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.125126+0000", 2023-10-09T06:05:06.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4743e-05 2023-10-09T06:05:06.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.368948+0000", 2023-10-09T06:05:06.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24382216500000001 2023-10-09T06:05:06.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:06.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.368980+0000", 2023-10-09T06:05:06.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1652999999999997e-05 2023-10-09T06:05:06.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:06.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.369013+0000", 2023-10-09T06:05:06.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3544999999999998e-05 2023-10-09T06:05:06.224 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.224 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:06.224 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.224 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.224 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:113845 8.7 8:e541ca4a:::benchmark_data_smithi064_457003_object9327:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-10-09T06:05:06.224 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:04:44.118749+0000", 2023-10-09T06:05:06.224 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 22.000845194, 2023-10-09T06:05:06.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.253492842, 2023-10-09T06:05:06.225 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:06.225 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.225 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:06.225 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:05:06.225 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:05:06.225 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 113845 2023-10-09T06:05:06.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.225 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:06.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.225 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:06.225 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.118749+0000", 2023-10-09T06:05:06.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:06.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.118749+0000", 2023-10-09T06:05:06.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:06.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.118748+0000", 2023-10-09T06:05:06.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:05:06.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.227 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:06.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.118751+0000", 2023-10-09T06:05:06.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.694e-06 2023-10-09T06:05:06.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.227 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:06.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.118757+0000", 2023-10-09T06:05:06.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3380000000000001e-06 2023-10-09T06:05:06.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.227 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.118764+0000", 2023-10-09T06:05:06.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9290000000000001e-06 2023-10-09T06:05:06.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.228 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.121954+0000", 2023-10-09T06:05:06.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031891649999999999 2023-10-09T06:05:06.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:06.228 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.122081+0000", 2023-10-09T06:05:06.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012695100000000001 2023-10-09T06:05:06.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.229 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.229 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:05:06.229 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.122197+0000", 2023-10-09T06:05:06.229 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116675 2023-10-09T06:05:06.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:06.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.125281+0000", 2023-10-09T06:05:06.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030838660000000002 2023-10-09T06:05:06.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.125676+0000", 2023-10-09T06:05:06.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00039473500000000001 2023-10-09T06:05:06.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.372153+0000", 2023-10-09T06:05:06.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24647673 2023-10-09T06:05:06.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.232 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.232 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:06.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.372202+0000", 2023-10-09T06:05:06.233 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9971e-05 2023-10-09T06:05:06.233 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.233 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:06.233 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.372242+0000", 2023-10-09T06:05:06.233 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9198e-05 2023-10-09T06:05:06.233 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.234 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:06.234 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.234 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.234 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:113849 8.3 8:c60b32df:::benchmark_data_smithi064_457003_object9331:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-10-09T06:05:06.234 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:04:44.126668+0000", 2023-10-09T06:05:06.235 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 21.992925710000002, 2023-10-09T06:05:06.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.241902283, 2023-10-09T06:05:06.235 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:06.235 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.235 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:06.235 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:05:06.235 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:05:06.236 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 113849 2023-10-09T06:05:06.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.236 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:06.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:06.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.126668+0000", 2023-10-09T06:05:06.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.237 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.237 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:06.237 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.126668+0000", 2023-10-09T06:05:06.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.237 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.237 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.237 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:06.238 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.126667+0000", 2023-10-09T06:05:06.238 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:05:06.238 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.238 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.238 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:06.238 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.126674+0000", 2023-10-09T06:05:06.239 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3470000000000001e-06 2023-10-09T06:05:06.239 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.239 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.239 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:06.239 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.126690+0000", 2023-10-09T06:05:06.239 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6368999999999999e-05 2023-10-09T06:05:06.240 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.240 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.240 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.240 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.126698+0000", 2023-10-09T06:05:06.240 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3629999999999996e-06 2023-10-09T06:05:06.240 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.240 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.241 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.241 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.126745+0000", 2023-10-09T06:05:06.241 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6885999999999998e-05 2023-10-09T06:05:06.241 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.241 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.241 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:06.242 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.126869+0000", 2023-10-09T06:05:06.242 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000124537 2023-10-09T06:05:06.242 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.242 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.242 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:05:06.242 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.126989+0000", 2023-10-09T06:05:06.243 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000119449 2023-10-09T06:05:06.243 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.243 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.243 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:06.243 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.130470+0000", 2023-10-09T06:05:06.243 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034818280000000002 2023-10-09T06:05:06.244 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.244 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.244 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.244 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.365586+0000", 2023-10-09T06:05:06.244 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.23511591800000001 2023-10-09T06:05:06.244 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.244 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.245 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.245 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.368475+0000", 2023-10-09T06:05:06.245 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028883390000000002 2023-10-09T06:05:06.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.245 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.245 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:06.246 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.368527+0000", 2023-10-09T06:05:06.246 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1864999999999999e-05 2023-10-09T06:05:06.246 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.246 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:06.246 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.368571+0000", 2023-10-09T06:05:06.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4048000000000002e-05 2023-10-09T06:05:06.247 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.247 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:06.247 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.247 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.247 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.248 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:113853 8.7 8:ed45ca9b:::benchmark_data_smithi064_457003_object9335:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-10-09T06:05:06.248 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:04:44.133210+0000", 2023-10-09T06:05:06.248 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 21.986383928999999, 2023-10-09T06:05:06.248 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.240108289, 2023-10-09T06:05:06.248 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:06.248 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.249 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:06.249 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:05:06.249 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:05:06.249 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 113853 2023-10-09T06:05:06.249 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.249 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:06.249 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.249 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:06.249 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.133210+0000", 2023-10-09T06:05:06.249 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.250 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.250 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.250 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:06.250 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.133210+0000", 2023-10-09T06:05:06.250 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.250 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.250 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.250 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:06.250 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.133209+0000", 2023-10-09T06:05:06.250 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:05:06.250 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.251 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.251 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:06.251 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.133214+0000", 2023-10-09T06:05:06.251 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.609e-06 2023-10-09T06:05:06.251 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.251 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.251 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:06.251 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.133227+0000", 2023-10-09T06:05:06.251 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2181000000000001e-05 2023-10-09T06:05:06.251 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.251 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.251 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.251 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.133233+0000", 2023-10-09T06:05:06.252 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1720000000000004e-06 2023-10-09T06:05:06.252 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.252 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.252 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.252 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.133272+0000", 2023-10-09T06:05:06.252 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9394999999999998e-05 2023-10-09T06:05:06.252 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.252 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.252 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:06.252 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.133379+0000", 2023-10-09T06:05:06.252 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000107182 2023-10-09T06:05:06.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.253 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:05:06.253 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.133462+0000", 2023-10-09T06:05:06.253 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2457e-05 2023-10-09T06:05:06.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.253 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:06.253 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.136828+0000", 2023-10-09T06:05:06.253 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033664620000000002 2023-10-09T06:05:06.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.136976+0000", 2023-10-09T06:05:06.254 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014788900000000001 2023-10-09T06:05:06.254 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.254 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.373233+0000", 2023-10-09T06:05:06.254 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.236257048 2023-10-09T06:05:06.254 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.254 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:06.255 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.373282+0000", 2023-10-09T06:05:06.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8656000000000003e-05 2023-10-09T06:05:06.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.255 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:06.255 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:44.373318+0000", 2023-10-09T06:05:06.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6470999999999997e-05 2023-10-09T06:05:06.255 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.255 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:06.255 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.256 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.256 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:118598 8.7 8:e1f40890:::benchmark_data_smithi064_457003_object14080:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-10-09T06:05:06.256 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:04:50.342849+0000", 2023-10-09T06:05:06.256 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 15.776744678, 2023-10-09T06:05:06.256 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.23012328400000001, 2023-10-09T06:05:06.256 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:06.256 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.256 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:06.256 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:05:06.256 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:05:06.256 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 118598 2023-10-09T06:05:06.257 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.257 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:06.257 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.257 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:06.257 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:50.342849+0000", 2023-10-09T06:05:06.257 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.257 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.257 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.257 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:06.257 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:50.342849+0000", 2023-10-09T06:05:06.257 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.257 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.258 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.258 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:06.258 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:50.342848+0000", 2023-10-09T06:05:06.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:05:06.258 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.258 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.258 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:06.258 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:50.342855+0000", 2023-10-09T06:05:06.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.708e-06 2023-10-09T06:05:06.258 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.258 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.259 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:06.259 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:50.342871+0000", 2023-10-09T06:05:06.259 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6477000000000002e-05 2023-10-09T06:05:06.259 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.259 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.259 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.259 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:50.342880+0000", 2023-10-09T06:05:06.259 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9139999999999997e-06 2023-10-09T06:05:06.259 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.259 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.259 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.259 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:50.342938+0000", 2023-10-09T06:05:06.260 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8077e-05 2023-10-09T06:05:06.260 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.260 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.260 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:06.260 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:50.343066+0000", 2023-10-09T06:05:06.260 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000128095 2023-10-09T06:05:06.260 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.260 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.260 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:05:06.260 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:50.343130+0000", 2023-10-09T06:05:06.260 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3441999999999993e-05 2023-10-09T06:05:06.260 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.261 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.261 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:06.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:50.348793+0000", 2023-10-09T06:05:06.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056630079999999998 2023-10-09T06:05:06.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.261 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.261 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:50.358077+0000", 2023-10-09T06:05:06.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0092842440000000005 2023-10-09T06:05:06.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:50.572883+0000", 2023-10-09T06:05:06.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21480574699999999 2023-10-09T06:05:06.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:06.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:50.572933+0000", 2023-10-09T06:05:06.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0754000000000002e-05 2023-10-09T06:05:06.263 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.263 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.263 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:06.263 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:50.572973+0000", 2023-10-09T06:05:06.263 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9162999999999998e-05 2023-10-09T06:05:06.263 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.263 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:06.263 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.263 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.263 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.263 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:119946 8.3 8:c4123a19:::benchmark_data_smithi064_457003_object15428:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-10-09T06:05:06.264 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:04:52.102783+0000", 2023-10-09T06:05:06.264 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 14.016811237000001, 2023-10-09T06:05:06.264 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18337737600000001, 2023-10-09T06:05:06.264 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:06.264 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.264 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:06.264 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:05:06.264 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:05:06.264 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119946 2023-10-09T06:05:06.264 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.264 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:06.264 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.265 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:06.265 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.102783+0000", 2023-10-09T06:05:06.265 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.265 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.265 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.265 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:06.265 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.102783+0000", 2023-10-09T06:05:06.265 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.265 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.265 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.265 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:06.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.102782+0000", 2023-10-09T06:05:06.266 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:05:06.266 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.266 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.266 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:06.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.102785+0000", 2023-10-09T06:05:06.266 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5669999999999999e-06 2023-10-09T06:05:06.266 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.266 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:06.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.102793+0000", 2023-10-09T06:05:06.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.072e-06 2023-10-09T06:05:06.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.102799+0000", 2023-10-09T06:05:06.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7019999999999998e-06 2023-10-09T06:05:06.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.106101+0000", 2023-10-09T06:05:06.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033022609999999999 2023-10-09T06:05:06.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.268 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.268 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:06.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.106229+0000", 2023-10-09T06:05:06.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012782099999999999 2023-10-09T06:05:06.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.268 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.268 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:05:06.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.106335+0000", 2023-10-09T06:05:06.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010609500000000001 2023-10-09T06:05:06.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.112606+0000", 2023-10-09T06:05:06.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062707329999999997 2023-10-09T06:05:06.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:06.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.117850+0000", 2023-10-09T06:05:06.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0052438559999999999 2023-10-09T06:05:06.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.286087+0000", 2023-10-09T06:05:06.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.168237304 2023-10-09T06:05:06.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:06.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.286126+0000", 2023-10-09T06:05:06.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8676e-05 2023-10-09T06:05:06.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.271 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:06.271 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.286160+0000", 2023-10-09T06:05:06.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4270999999999998e-05 2023-10-09T06:05:06.271 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.271 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:06.271 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.271 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.271 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:119947 8.7 8:e0989ef3:::benchmark_data_smithi064_457003_object15429:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-10-09T06:05:06.271 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:04:52.102804+0000", 2023-10-09T06:05:06.271 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 14.01678993, 2023-10-09T06:05:06.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.180135775, 2023-10-09T06:05:06.272 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:06.272 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.272 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:06.272 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:05:06.272 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:05:06.272 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119947 2023-10-09T06:05:06.272 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.273 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:06.273 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.273 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:06.273 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.102804+0000", 2023-10-09T06:05:06.273 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.273 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.274 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:06.274 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.102804+0000", 2023-10-09T06:05:06.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:06.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.102804+0000", 2023-10-09T06:05:06.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:05:06.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:06.276 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:06.277 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.102806+0000", 2023-10-09T06:05:06.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1600000000000001e-06 2023-10-09T06:05:06.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.278 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:06.278 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.102811+0000", 2023-10-09T06:05:06.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3539999999999999e-06 2023-10-09T06:05:06.278 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.102816+0000", 2023-10-09T06:05:06.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7659999999999998e-06 2023-10-09T06:05:06.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.102868+0000", 2023-10-09T06:05:06.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1592999999999999e-05 2023-10-09T06:05:06.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:06.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.102976+0000", 2023-10-09T06:05:06.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108183 2023-10-09T06:05:06.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.281 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:05:06.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.103084+0000", 2023-10-09T06:05:06.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108038 2023-10-09T06:05:06.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:06.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.109712+0000", 2023-10-09T06:05:06.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0066277769999999996 2023-10-09T06:05:06.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.282455+0000", 2023-10-09T06:05:06.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.17274299700000001 2023-10-09T06:05:06.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.282856+0000", 2023-10-09T06:05:06.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00040180800000000003 2023-10-09T06:05:06.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:06.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.282901+0000", 2023-10-09T06:05:06.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4314000000000003e-05 2023-10-09T06:05:06.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:06.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.282940+0000", 2023-10-09T06:05:06.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9125999999999998e-05 2023-10-09T06:05:06.286 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.286 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:06.286 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.287 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:119956 8.7 8:e1e441fd:::benchmark_data_smithi064_457003_object15438:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-10-09T06:05:06.287 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:04:52.109788+0000", 2023-10-09T06:05:06.287 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 14.009806139, 2023-10-09T06:05:06.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.17324589000000001, 2023-10-09T06:05:06.287 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:06.288 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.288 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:06.288 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:05:06.288 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:05:06.288 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119956 2023-10-09T06:05:06.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.289 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:06.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:06.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.109788+0000", 2023-10-09T06:05:06.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:06.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.109788+0000", 2023-10-09T06:05:06.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.291 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:06.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.109786+0000", 2023-10-09T06:05:06.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:05:06.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:06.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.109792+0000", 2023-10-09T06:05:06.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4040000000000003e-06 2023-10-09T06:05:06.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:06.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.109802+0000", 2023-10-09T06:05:06.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0057999999999999e-05 2023-10-09T06:05:06.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.109810+0000", 2023-10-09T06:05:06.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2770000000000005e-06 2023-10-09T06:05:06.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.109858+0000", 2023-10-09T06:05:06.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7882e-05 2023-10-09T06:05:06.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:06.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.109982+0000", 2023-10-09T06:05:06.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000123437 2023-10-09T06:05:06.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:05:06.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.110100+0000", 2023-10-09T06:05:06.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118095 2023-10-09T06:05:06.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:06.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.117919+0000", 2023-10-09T06:05:06.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0078190940000000004 2023-10-09T06:05:06.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.282484+0000", 2023-10-09T06:05:06.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.164565453 2023-10-09T06:05:06.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.282972+0000", 2023-10-09T06:05:06.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00048761399999999998 2023-10-09T06:05:06.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:06.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.283004+0000", 2023-10-09T06:05:06.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2546999999999999e-05 2023-10-09T06:05:06.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:06.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.283034+0000", 2023-10-09T06:05:06.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9482e-05 2023-10-09T06:05:06.301 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.301 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:06.301 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.301 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:119957 8.1 8:8d3cbae1:::benchmark_data_smithi064_457003_object15439:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-10-09T06:05:06.302 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:04:52.113840+0000", 2023-10-09T06:05:06.302 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 14.005753822000001, 2023-10-09T06:05:06.302 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.182580345, 2023-10-09T06:05:06.302 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:06.302 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.302 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:06.303 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:05:06.303 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:05:06.303 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119957 2023-10-09T06:05:06.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.303 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:06.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:06.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.113840+0000", 2023-10-09T06:05:06.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:06.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.113840+0000", 2023-10-09T06:05:06.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:06.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.113839+0000", 2023-10-09T06:05:06.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:05:06.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:06.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.113845+0000", 2023-10-09T06:05:06.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5420000000000002e-06 2023-10-09T06:05:06.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:06.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.113858+0000", 2023-10-09T06:05:06.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2306e-05 2023-10-09T06:05:06.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.113865+0000", 2023-10-09T06:05:06.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6809999999999998e-06 2023-10-09T06:05:06.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.117568+0000", 2023-10-09T06:05:06.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037030829999999998 2023-10-09T06:05:06.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:06.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.117719+0000", 2023-10-09T06:05:06.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000150727 2023-10-09T06:05:06.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:05:06.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.117845+0000", 2023-10-09T06:05:06.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012641099999999999 2023-10-09T06:05:06.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:06.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.121247+0000", 2023-10-09T06:05:06.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003401112 2023-10-09T06:05:06.312 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.312 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.312 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.312 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.282050+0000", 2023-10-09T06:05:06.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16080328299999999 2023-10-09T06:05:06.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.296341+0000", 2023-10-09T06:05:06.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014290951 2023-10-09T06:05:06.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:06.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.296385+0000", 2023-10-09T06:05:06.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3979e-05 2023-10-09T06:05:06.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.315 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:06.315 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.296421+0000", 2023-10-09T06:05:06.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5757999999999999e-05 2023-10-09T06:05:06.315 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.315 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:06.315 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.316 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:119960 8.a 8:5d6e653e:::benchmark_data_smithi064_457003_object15442:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-10-09T06:05:06.316 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:04:52.123412+0000", 2023-10-09T06:05:06.316 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 13.996182041000001, 2023-10-09T06:05:06.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18831036500000001, 2023-10-09T06:05:06.316 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:06.316 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:06.316 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:06.317 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:05:06.317 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:05:06.317 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 119960 2023-10-09T06:05:06.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.317 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:06.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:06.317 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.123412+0000", 2023-10-09T06:05:06.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.318 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.318 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.318 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:06.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.123412+0000", 2023-10-09T06:05:06.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:06.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.319 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:06.319 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.123412+0000", 2023-10-09T06:05:06.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:05:06.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.319 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:06.319 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.123414+0000", 2023-10-09T06:05:06.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.83e-06 2023-10-09T06:05:06.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:06.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.123420+0000", 2023-10-09T06:05:06.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0379999999999996e-06 2023-10-09T06:05:06.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:06.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.123426+0000", 2023-10-09T06:05:06.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9200000000000001e-06 2023-10-09T06:05:06.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:06.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.285499+0000", 2023-10-09T06:05:06.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16207290799999999 2023-10-09T06:05:06.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:06.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.285604+0000", 2023-10-09T06:05:06.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000104522 2023-10-09T06:05:06.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:05:06.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.285729+0000", 2023-10-09T06:05:06.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012520000000000001 2023-10-09T06:05:06.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:06.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.292141+0000", 2023-10-09T06:05:06.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006412058 2023-10-09T06:05:06.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.292181+0000", 2023-10-09T06:05:06.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9975999999999998e-05 2023-10-09T06:05:06.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:06.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.311631+0000", 2023-10-09T06:05:06.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01945011 2023-10-09T06:05:06.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:06.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.311680+0000", 2023-10-09T06:05:06.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9273000000000001e-05 2023-10-09T06:05:06.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:06.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:06.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:06.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:04:52.311722+0000", 2023-10-09T06:05:06.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1916999999999999e-05 2023-10-09T06:05:06.324 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.324 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:06.324 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.324 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:06.324 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:06.324 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:05:06.325 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:06.388 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:06.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:06.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:06.498 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:06.537 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:06.614 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:06.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:06 smithi156 ceph-mon[195101]: pgmap v936: 119 pgs: 119 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2023-10-09T06:05:06.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:06.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:06.723 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:05:06.776 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:06 smithi064 ceph-mon[212750]: pgmap v936: 119 pgs: 119 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2023-10-09T06:05:06.778 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:06 smithi064 ceph-mon[220044]: pgmap v936: 119 pgs: 119 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2023-10-09T06:05:06.791 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:07.414 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:05:08.079 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:05:08.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:08.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:05:08.126 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:05:08.185 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:08.216 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:08.284 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:08.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:08.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:08.395 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:08.436 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:08.483 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:08.488 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:08 smithi156 ceph-mon[195101]: pgmap v937: 119 pgs: 119 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2023-10-09T06:05:08.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:08.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:08.590 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:08.664 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:08.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:08 smithi064 ceph-mon[212750]: pgmap v937: 119 pgs: 119 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2023-10-09T06:05:08.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:08 smithi064 ceph-mon[220044]: pgmap v937: 119 pgs: 119 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2023-10-09T06:05:08.867 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:08.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:08.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:08.972 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:09.005 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:09.066 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:09.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:09.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:09.172 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:09.232 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:09.298 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:09.321 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:09.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:09.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:09.346 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:09.427 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:09.470 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:09.553 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:09.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:09.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:09.657 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:09.699 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:09.781 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:09.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:09.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:09.887 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:09.981 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:10.050 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:09 smithi064 conmon[212726]: 2023-10-09T06:05:09.787+0000 7f8823f23700 -1 mon.a@0(leader).osd e380 definitely_dead 0 2023-10-09T06:05:10.557 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:05:10.561 INFO:teuthology.orchestra.run.smithi064.stderr:marked out osd.2. 2023-10-09T06:05:10.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:10 smithi064 ceph-mon[212750]: pgmap v938: 119 pgs: 119 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 824 op/s 2023-10-09T06:05:10.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:10 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2080167979' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T06:05:10.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:10 smithi064 ceph-mon[212750]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T06:05:10.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:10 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1120156301' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:05:10.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:10 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:05:10.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:10 smithi064 ceph-mon[220044]: pgmap v938: 119 pgs: 119 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 824 op/s 2023-10-09T06:05:10.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:10 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2080167979' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T06:05:10.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:10 smithi064 ceph-mon[220044]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T06:05:10.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:10 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1120156301' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:05:10.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:10 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:05:10.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:10 smithi156 ceph-mon[195101]: pgmap v938: 119 pgs: 119 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 824 op/s 2023-10-09T06:05:10.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:10 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2080167979' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T06:05:10.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:10 smithi156 ceph-mon[195101]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T06:05:10.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:10 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1120156301' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:05:10.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:10 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:05:11.474 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:11.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:11.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:11.584 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:11.615 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:11.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:11 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:05:11.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:11 smithi064 ceph-mon[212750]: failed to encode map e381 with expected crc 2023-10-09T06:05:11.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:11 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2080167979' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T06:05:11.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:11 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:05:11.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:11 smithi064 ceph-mon[212750]: osdmap e381: 8 total, 8 up, 7 in 2023-10-09T06:05:11.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:11 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:05:11.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:11 smithi064 ceph-mon[220044]: failed to encode map e381 with expected crc 2023-10-09T06:05:11.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:11 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2080167979' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T06:05:11.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:11 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:05:11.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:11 smithi064 ceph-mon[220044]: osdmap e381: 8 total, 8 up, 7 in 2023-10-09T06:05:11.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:11 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:05:11.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:11 smithi156 ceph-mon[195101]: failed to encode map e381 with expected crc 2023-10-09T06:05:11.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:11 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2080167979' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T06:05:11.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:11 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:05:11.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:11 smithi156 ceph-mon[195101]: osdmap e381: 8 total, 8 up, 7 in 2023-10-09T06:05:12.638 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:12.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:12.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:12.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:12 smithi064 ceph-mon[212750]: pgmap v940: 119 pgs: 6 peering, 113 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 776 op/s 2023-10-09T06:05:12.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:12 smithi064 ceph-mon[212750]: failed to encode map e382 with expected crc 2023-10-09T06:05:12.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:12 smithi064 ceph-mon[212750]: osdmap e382: 8 total, 8 up, 7 in 2023-10-09T06:05:12.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:12 smithi064 ceph-mon[212750]: failed to encode map e382 with expected crc 2023-10-09T06:05:12.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:12 smithi064 ceph-mon[212750]: failed to encode map e382 with expected crc 2023-10-09T06:05:12.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:12 smithi064 ceph-mon[212750]: failed to encode map e382 with expected crc 2023-10-09T06:05:12.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:12 smithi064 ceph-mon[212750]: failed to encode map e382 with expected crc 2023-10-09T06:05:12.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:12 smithi064 ceph-mon[212750]: failed to encode map e382 with expected crc 2023-10-09T06:05:12.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:12 smithi064 ceph-mon[212750]: failed to encode map e382 with expected crc 2023-10-09T06:05:12.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:12 smithi064 ceph-mon[212750]: failed to encode map e382 with expected crc 2023-10-09T06:05:12.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:12 smithi064 ceph-mon[220044]: pgmap v940: 119 pgs: 6 peering, 113 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 776 op/s 2023-10-09T06:05:12.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:12 smithi064 ceph-mon[220044]: failed to encode map e382 with expected crc 2023-10-09T06:05:12.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:12 smithi064 ceph-mon[220044]: osdmap e382: 8 total, 8 up, 7 in 2023-10-09T06:05:12.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:12 smithi064 ceph-mon[220044]: failed to encode map e382 with expected crc 2023-10-09T06:05:12.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:12 smithi064 ceph-mon[220044]: failed to encode map e382 with expected crc 2023-10-09T06:05:12.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:12 smithi064 ceph-mon[220044]: failed to encode map e382 with expected crc 2023-10-09T06:05:12.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:12 smithi064 ceph-mon[220044]: failed to encode map e382 with expected crc 2023-10-09T06:05:12.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:12 smithi064 ceph-mon[220044]: failed to encode map e382 with expected crc 2023-10-09T06:05:12.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:12 smithi064 ceph-mon[220044]: failed to encode map e382 with expected crc 2023-10-09T06:05:12.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:12 smithi064 ceph-mon[220044]: failed to encode map e382 with expected crc 2023-10-09T06:05:12.748 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:12.862 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:12.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:12 smithi156 ceph-mon[195101]: pgmap v940: 119 pgs: 6 peering, 113 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 776 op/s 2023-10-09T06:05:12.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:12 smithi156 ceph-mon[195101]: failed to encode map e382 with expected crc 2023-10-09T06:05:12.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:12 smithi156 ceph-mon[195101]: osdmap e382: 8 total, 8 up, 7 in 2023-10-09T06:05:12.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:12 smithi156 ceph-mon[195101]: failed to encode map e382 with expected crc 2023-10-09T06:05:12.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:12 smithi156 ceph-mon[195101]: failed to encode map e382 with expected crc 2023-10-09T06:05:12.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:12 smithi156 ceph-mon[195101]: failed to encode map e382 with expected crc 2023-10-09T06:05:12.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:12 smithi156 ceph-mon[195101]: failed to encode map e382 with expected crc 2023-10-09T06:05:12.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:12 smithi156 ceph-mon[195101]: failed to encode map e382 with expected crc 2023-10-09T06:05:12.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:12 smithi156 ceph-mon[195101]: failed to encode map e382 with expected crc 2023-10-09T06:05:12.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:12 smithi156 ceph-mon[195101]: failed to encode map e382 with expected crc 2023-10-09T06:05:13.171 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:13.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:13.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:13.283 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:13.371 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:13.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:13 smithi064 ceph-mon[212750]: 8.1 starting backfill to osd.5 from (0'0,0'0] MAX to 380'8467 2023-10-09T06:05:13.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:13 smithi064 ceph-mon[212750]: osdmap e383: 8 total, 8 up, 7 in 2023-10-09T06:05:13.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:13 smithi064 ceph-mon[212750]: 8.e starting backfill to osd.7 from (0'0,0'0] MAX to 380'8492 2023-10-09T06:05:13.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:13 smithi064 ceph-mon[212750]: failed to encode map e383 with expected crc 2023-10-09T06:05:13.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:13 smithi064 ceph-mon[212750]: failed to encode map e383 with expected crc 2023-10-09T06:05:13.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:13 smithi064 ceph-mon[212750]: failed to encode map e383 with expected crc 2023-10-09T06:05:13.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:13 smithi064 ceph-mon[212750]: failed to encode map e383 with expected crc 2023-10-09T06:05:13.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:13 smithi064 ceph-mon[212750]: 8.a starting backfill to osd.3 from (0'0,0'0] MAX to 380'8622 2023-10-09T06:05:13.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:13 smithi064 ceph-mon[212750]: 8.4 starting backfill to osd.6 from (0'0,0'0] MAX to 380'8462 2023-10-09T06:05:13.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:13 smithi064 ceph-mon[212750]: 8.d starting backfill to osd.6 from (0'0,0'0] MAX to 380'8217 2023-10-09T06:05:13.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:13 smithi064 ceph-mon[212750]: failed to encode map e383 with expected crc 2023-10-09T06:05:13.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:13 smithi064 ceph-mon[212750]: 8.f starting backfill to osd.4 from (0'0,0'0] MAX to 380'8472 2023-10-09T06:05:13.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:13 smithi064 ceph-mon[220044]: 8.1 starting backfill to osd.5 from (0'0,0'0] MAX to 380'8467 2023-10-09T06:05:13.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:13 smithi064 ceph-mon[220044]: osdmap e383: 8 total, 8 up, 7 in 2023-10-09T06:05:13.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:13 smithi064 ceph-mon[220044]: 8.e starting backfill to osd.7 from (0'0,0'0] MAX to 380'8492 2023-10-09T06:05:13.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:13 smithi064 ceph-mon[220044]: failed to encode map e383 with expected crc 2023-10-09T06:05:13.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:13 smithi064 ceph-mon[220044]: failed to encode map e383 with expected crc 2023-10-09T06:05:13.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:13 smithi064 ceph-mon[220044]: failed to encode map e383 with expected crc 2023-10-09T06:05:13.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:13 smithi064 ceph-mon[220044]: failed to encode map e383 with expected crc 2023-10-09T06:05:13.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:13 smithi064 ceph-mon[220044]: 8.a starting backfill to osd.3 from (0'0,0'0] MAX to 380'8622 2023-10-09T06:05:13.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:13 smithi064 ceph-mon[220044]: 8.4 starting backfill to osd.6 from (0'0,0'0] MAX to 380'8462 2023-10-09T06:05:13.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:13 smithi064 ceph-mon[220044]: 8.d starting backfill to osd.6 from (0'0,0'0] MAX to 380'8217 2023-10-09T06:05:13.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:13 smithi064 ceph-mon[220044]: failed to encode map e383 with expected crc 2023-10-09T06:05:13.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:13 smithi064 ceph-mon[220044]: 8.f starting backfill to osd.4 from (0'0,0'0] MAX to 380'8472 2023-10-09T06:05:13.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:13 smithi156 ceph-mon[195101]: 8.1 starting backfill to osd.5 from (0'0,0'0] MAX to 380'8467 2023-10-09T06:05:13.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:13 smithi156 ceph-mon[195101]: osdmap e383: 8 total, 8 up, 7 in 2023-10-09T06:05:13.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:13 smithi156 ceph-mon[195101]: 8.e starting backfill to osd.7 from (0'0,0'0] MAX to 380'8492 2023-10-09T06:05:13.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:13 smithi156 ceph-mon[195101]: failed to encode map e383 with expected crc 2023-10-09T06:05:13.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:13 smithi156 ceph-mon[195101]: failed to encode map e383 with expected crc 2023-10-09T06:05:13.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:13 smithi156 ceph-mon[195101]: failed to encode map e383 with expected crc 2023-10-09T06:05:13.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:13 smithi156 ceph-mon[195101]: failed to encode map e383 with expected crc 2023-10-09T06:05:13.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:13 smithi156 ceph-mon[195101]: 8.a starting backfill to osd.3 from (0'0,0'0] MAX to 380'8622 2023-10-09T06:05:13.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:13 smithi156 ceph-mon[195101]: 8.4 starting backfill to osd.6 from (0'0,0'0] MAX to 380'8462 2023-10-09T06:05:13.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:13 smithi156 ceph-mon[195101]: 8.d starting backfill to osd.6 from (0'0,0'0] MAX to 380'8217 2023-10-09T06:05:13.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:13 smithi156 ceph-mon[195101]: failed to encode map e383 with expected crc 2023-10-09T06:05:13.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:13 smithi156 ceph-mon[195101]: 8.f starting backfill to osd.4 from (0'0,0'0] MAX to 380'8472 2023-10-09T06:05:14.061 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:14.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:14.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:14.170 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:05:14.215 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:05:14.215 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:05:14.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:14.215 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45605.0:135642 8.e e384/382 8:70f80985:::benchmark_data_smithi064_457003_object31124:head v 384'8574, mlcod=384'8572)", 2023-10-09T06:05:14.215 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:14.208725+0000", 2023-10-09T06:05:14.215 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0042589460000000004, 2023-10-09T06:05:14.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042910329999999997, 2023-10-09T06:05:14.215 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:14.216 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:05:14.216 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:14.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:14.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:14.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:14.208725+0000", 2023-10-09T06:05:14.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:14.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:14.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:14.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:14.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:14.208725+0000", 2023-10-09T06:05:14.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:14.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:14.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:14.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:14.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:14.208724+0000", 2023-10-09T06:05:14.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:05:14.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:14.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:14.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:14.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:14.208733+0000", 2023-10-09T06:05:14.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.054e-06 2023-10-09T06:05:14.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:14.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:14.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:14.218 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:14.208746+0000", 2023-10-09T06:05:14.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3359e-05 2023-10-09T06:05:14.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:14.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:14.218 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:14.218 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:14.208754+0000", 2023-10-09T06:05:14.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4419999999999998e-06 2023-10-09T06:05:14.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:14.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:14.218 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:14.218 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:14.210804+0000", 2023-10-09T06:05:14.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020491329999999999 2023-10-09T06:05:14.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:14.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:14.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:14.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:14.210829+0000", 2023-10-09T06:05:14.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5185e-05 2023-10-09T06:05:14.219 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:14.219 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:14.219 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:14.219 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:14.219 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-10-09T06:05:14.219 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 1 2023-10-09T06:05:14.220 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:05:14.254 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:14.458 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[212750]: pgmap v943: 119 pgs: 12 peering, 107 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 4.7 KiB/s rd, 12 KiB/s wr, 615 op/s; 8.7 KiB/s, 0 objects/s recovering 2023-10-09T06:05:14.459 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:05:14.463 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:05:14.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:14.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:05:14.495 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:05:14.560 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:05:14.569 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:14.609 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:05:14.617 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:14.619 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:05:14.645 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:05:14.677 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:05:14.686 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:14.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:14.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:14.715 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:05:14.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[220044]: pgmap v943: 119 pgs: 12 peering, 107 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 4.7 KiB/s rd, 12 KiB/s wr, 615 op/s; 8.7 KiB/s, 0 objects/s recovering 2023-10-09T06:05:14.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:05:14.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[220044]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[220044]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[220044]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[220044]: osdmap e384: 8 total, 8 up, 7 in 2023-10-09T06:05:14.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[220044]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[220044]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[220044]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[220044]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[220044]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[220044]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:05:14.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:14.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/136103248' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:14.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[212750]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[212750]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[212750]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[212750]: osdmap e384: 8 total, 8 up, 7 in 2023-10-09T06:05:14.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[212750]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[212750]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[212750]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[212750]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[212750]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[212750]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:05:14.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:14.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:14 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/136103248' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:14.754 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:05:14.790 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:05:14.792 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:14.801 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:05:14.801 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:05:14.801 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:05:14.801 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:05:14.802 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:05:14.802 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:05:14.802 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:05:14.802 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:05:14.824 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:14 smithi156 ceph-mon[195101]: pgmap v943: 119 pgs: 12 peering, 107 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 4.7 KiB/s rd, 12 KiB/s wr, 615 op/s; 8.7 KiB/s, 0 objects/s recovering 2023-10-09T06:05:14.825 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:14 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:05:14.825 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:14 smithi156 ceph-mon[195101]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.825 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:14 smithi156 ceph-mon[195101]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.825 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:14 smithi156 ceph-mon[195101]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.825 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:14 smithi156 ceph-mon[195101]: osdmap e384: 8 total, 8 up, 7 in 2023-10-09T06:05:14.825 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:14 smithi156 ceph-mon[195101]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.825 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:14 smithi156 ceph-mon[195101]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.825 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:14 smithi156 ceph-mon[195101]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.826 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:14 smithi156 ceph-mon[195101]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.826 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:14 smithi156 ceph-mon[195101]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.826 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:14 smithi156 ceph-mon[195101]: failed to encode map e384 with expected crc 2023-10-09T06:05:14.826 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:05:14.826 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:14.826 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:14 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/136103248' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:14.827 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:14.931 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:14.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:14.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:15.043 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:15.086 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:15.173 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:15.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:15.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:15.280 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:15.342 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:15.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:15 smithi064 ceph-mon[212750]: osdmap e385: 8 total, 8 up, 7 in 2023-10-09T06:05:15.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:15 smithi064 ceph-mon[212750]: failed to encode map e385 with expected crc 2023-10-09T06:05:15.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:15 smithi064 ceph-mon[212750]: failed to encode map e385 with expected crc 2023-10-09T06:05:15.621 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:15 smithi064 ceph-mon[212750]: failed to encode map e385 with expected crc 2023-10-09T06:05:15.621 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:15 smithi064 ceph-mon[212750]: failed to encode map e385 with expected crc 2023-10-09T06:05:15.621 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:15 smithi064 ceph-mon[212750]: failed to encode map e385 with expected crc 2023-10-09T06:05:15.621 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:15 smithi064 ceph-mon[212750]: failed to encode map e385 with expected crc 2023-10-09T06:05:15.621 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:15 smithi064 ceph-mon[220044]: osdmap e385: 8 total, 8 up, 7 in 2023-10-09T06:05:15.621 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:15 smithi064 ceph-mon[220044]: failed to encode map e385 with expected crc 2023-10-09T06:05:15.621 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:15 smithi064 ceph-mon[220044]: failed to encode map e385 with expected crc 2023-10-09T06:05:15.621 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:15 smithi064 ceph-mon[220044]: failed to encode map e385 with expected crc 2023-10-09T06:05:15.621 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:15 smithi064 ceph-mon[220044]: failed to encode map e385 with expected crc 2023-10-09T06:05:15.622 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:15 smithi064 ceph-mon[220044]: failed to encode map e385 with expected crc 2023-10-09T06:05:15.622 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:15 smithi064 ceph-mon[220044]: failed to encode map e385 with expected crc 2023-10-09T06:05:15.622 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:05:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:05:15] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.43.0" 2023-10-09T06:05:15.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:15 smithi156 ceph-mon[195101]: osdmap e385: 8 total, 8 up, 7 in 2023-10-09T06:05:15.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:15 smithi156 ceph-mon[195101]: failed to encode map e385 with expected crc 2023-10-09T06:05:15.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:15 smithi156 ceph-mon[195101]: failed to encode map e385 with expected crc 2023-10-09T06:05:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:15 smithi156 ceph-mon[195101]: failed to encode map e385 with expected crc 2023-10-09T06:05:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:15 smithi156 ceph-mon[195101]: failed to encode map e385 with expected crc 2023-10-09T06:05:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:15 smithi156 ceph-mon[195101]: failed to encode map e385 with expected crc 2023-10-09T06:05:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:15 smithi156 ceph-mon[195101]: failed to encode map e385 with expected crc 2023-10-09T06:05:16.208 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:16.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:16.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:16.323 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:16.388 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:16.482 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:16.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:16.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:16.496 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:05:16.590 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:16.625 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:16.717 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:16.727 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:16 smithi156 ceph-mon[195101]: pgmap v946: 119 pgs: 1 active+remapped+backfilling, 13 peering, 105 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 257 op/s; 4401/222708 objects misplaced (1.976%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T06:05:16.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:16.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:16.728 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:16.829 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:16.878 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:16.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:16 smithi064 ceph-mon[212750]: pgmap v946: 119 pgs: 1 active+remapped+backfilling, 13 peering, 105 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 257 op/s; 4401/222708 objects misplaced (1.976%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T06:05:16.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:16 smithi064 ceph-mon[220044]: pgmap v946: 119 pgs: 1 active+remapped+backfilling, 13 peering, 105 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 257 op/s; 4401/222708 objects misplaced (1.976%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T06:05:16.984 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:16.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:16.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:17.094 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:05:17.211 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:17.317 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:05:17.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:17.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:05:17.420 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:17.517 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:17.675 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:17.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:17.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:17.785 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:17.834 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:17.930 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:17.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:17.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:18.040 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:18.119 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:18.266 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:18.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:18.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:18.374 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:18.400 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:18.499 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:18.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:18.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:18.540 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:05:18.606 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:18.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:18 smithi064 ceph-mon[212750]: pgmap v947: 119 pgs: 2 active+remapped+backfill_wait, 4 active+remapped+backfilling, 1 peering, 112 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 400 op/s; 25336/218655 objects misplaced (11.587%); 23 MiB/s, 1 keys/s, 362 objects/s recovering 2023-10-09T06:05:18.692 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:18 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:05:18.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:18 smithi064 ceph-mon[220044]: pgmap v947: 119 pgs: 2 active+remapped+backfill_wait, 4 active+remapped+backfilling, 1 peering, 112 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 400 op/s; 25336/218655 objects misplaced (11.587%); 23 MiB/s, 1 keys/s, 362 objects/s recovering 2023-10-09T06:05:18.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:18 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:05:18.702 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:18.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:18 smithi156 ceph-mon[195101]: pgmap v947: 119 pgs: 2 active+remapped+backfill_wait, 4 active+remapped+backfilling, 1 peering, 112 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 400 op/s; 25336/218655 objects misplaced (11.587%); 23 MiB/s, 1 keys/s, 362 objects/s recovering 2023-10-09T06:05:18.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:18 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:05:18.948 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:18.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:18.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:18.991 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:19.062 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:19.115 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:19.179 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1] out_osds: [2] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 0, 4] 2023-10-09T06:05:19.179 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:05:19.180 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T06:05:19.180 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 1.0 2023-10-09T06:05:19.209 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:19.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:19.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:19.317 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:19.368 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:19.448 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:19.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:19.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:19.562 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:19.669 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:19.718 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:19.883 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:19.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:19.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:19.985 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:05:20.077 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:20.455 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:05:20.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:20 smithi064 ceph-mon[212750]: pgmap v948: 119 pgs: 2 active+remapped+backfill_wait, 4 active+remapped+backfilling, 113 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 402 op/s; 25336/217431 objects misplaced (11.652%); 19 MiB/s, 1 keys/s, 305 objects/s recovering 2023-10-09T06:05:20.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:20 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1019760970' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:05:20.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:20 smithi064 ceph-mon[220044]: pgmap v948: 119 pgs: 2 active+remapped+backfill_wait, 4 active+remapped+backfilling, 113 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 402 op/s; 25336/217431 objects misplaced (11.652%); 19 MiB/s, 1 keys/s, 305 objects/s recovering 2023-10-09T06:05:20.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:20 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1019760970' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:05:20.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:20 smithi156 ceph-mon[195101]: pgmap v948: 119 pgs: 2 active+remapped+backfill_wait, 4 active+remapped+backfilling, 113 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 402 op/s; 25336/217431 objects misplaced (11.652%); 19 MiB/s, 1 keys/s, 305 objects/s recovering 2023-10-09T06:05:20.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:20 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1019760970' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:05:21.656 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:05:21.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:21.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:05:21.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:21 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:05:21.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:21 smithi064 ceph-mon[212750]: failed to encode map e386 with expected crc 2023-10-09T06:05:21.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:21 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1019760970' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:05:21.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:21 smithi064 ceph-mon[212750]: failed to encode map e386 with expected crc 2023-10-09T06:05:21.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:21 smithi064 ceph-mon[212750]: osdmap e386: 8 total, 8 up, 7 in 2023-10-09T06:05:21.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:21 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:05:21.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:21 smithi064 ceph-mon[220044]: failed to encode map e386 with expected crc 2023-10-09T06:05:21.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:21 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1019760970' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:05:21.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:21 smithi064 ceph-mon[220044]: failed to encode map e386 with expected crc 2023-10-09T06:05:21.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:21 smithi064 ceph-mon[220044]: osdmap e386: 8 total, 8 up, 7 in 2023-10-09T06:05:21.763 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:21.807 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:21 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:05:21.807 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:21 smithi156 ceph-mon[195101]: failed to encode map e386 with expected crc 2023-10-09T06:05:21.807 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:21 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1019760970' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:05:21.807 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:21 smithi156 ceph-mon[195101]: failed to encode map e386 with expected crc 2023-10-09T06:05:21.808 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:21 smithi156 ceph-mon[195101]: osdmap e386: 8 total, 8 up, 7 in 2023-10-09T06:05:21.810 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:21.871 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:21.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:21.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:21.985 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:22.047 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:22.546 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:22.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:22.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:22.643 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:22 smithi064 ceph-mon[212750]: pgmap v950: 119 pgs: 2 active+remapped+backfill_wait, 4 active+remapped+backfilling, 113 active+clean; 4.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 22553/208431 objects misplaced (10.820%); 31 MiB/s, 1 keys/s, 494 objects/s recovering 2023-10-09T06:05:22.644 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:22 smithi064 ceph-mon[220044]: pgmap v950: 119 pgs: 2 active+remapped+backfill_wait, 4 active+remapped+backfilling, 113 active+clean; 4.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 22553/208431 objects misplaced (10.820%); 31 MiB/s, 1 keys/s, 494 objects/s recovering 2023-10-09T06:05:22.658 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:22.740 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:22.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:22 smithi156 ceph-mon[195101]: pgmap v950: 119 pgs: 2 active+remapped+backfill_wait, 4 active+remapped+backfilling, 113 active+clean; 4.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 22553/208431 objects misplaced (10.820%); 31 MiB/s, 1 keys/s, 494 objects/s recovering 2023-10-09T06:05:23.069 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:23.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:23.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:23.182 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:23.234 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:23.265 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:05:23.265 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:05:23.356 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:23.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:23.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:23.471 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:23.531 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:24.244 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:24.470 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:05:24.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:24 smithi064 ceph-mon[212750]: pgmap v951: 119 pgs: 2 active+remapped+backfill_wait, 4 active+remapped+backfilling, 113 active+clean; 4.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 20774/207738 objects misplaced (10.000%); 40 MiB/s, 1 keys/s, 632 objects/s recovering 2023-10-09T06:05:24.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:24 smithi064 ceph-mon[220044]: pgmap v951: 119 pgs: 2 active+remapped+backfill_wait, 4 active+remapped+backfilling, 113 active+clean; 4.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 20774/207738 objects misplaced (10.000%); 40 MiB/s, 1 keys/s, 632 objects/s recovering 2023-10-09T06:05:24.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:24 smithi156 ceph-mon[195101]: pgmap v951: 119 pgs: 2 active+remapped+backfill_wait, 4 active+remapped+backfilling, 113 active+clean; 4.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 20774/207738 objects misplaced (10.000%); 40 MiB/s, 1 keys/s, 632 objects/s recovering 2023-10-09T06:05:25.310 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:25.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:25.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:25.417 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:25.453 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:25.454 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:05:25.535 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:25.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:25.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:25.592 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:05:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:05:25] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.43.0" 2023-10-09T06:05:25.642 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:25.697 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:25.777 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:25.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:25.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:25.886 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:25.927 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:26.011 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:26.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:26.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:26.126 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:26.188 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:05:26.231 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:26.232 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:05:26.279 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:05:26.329 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:05:26.404 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:05:26.409 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:26.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:26.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:26.450 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:05:26.486 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:05:26.528 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:26.532 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:05:26.545 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:05:26.545 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:05:26.546 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:05:26.546 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:05:26.546 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:05:26.546 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:05:26.546 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:05:26.546 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:05:26.579 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:26.662 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:26.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:26.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:26.776 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:26.819 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:26 smithi064 ceph-mon[212750]: pgmap v952: 119 pgs: 2 active+remapped+backfill_wait, 4 active+remapped+backfilling, 113 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 644 op/s; 19992/203358 objects misplaced (9.831%); 36 MiB/s, 0 keys/s, 570 objects/s recovering 2023-10-09T06:05:26.819 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:26 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3622488862' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:26.821 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:26 smithi064 ceph-mon[220044]: pgmap v952: 119 pgs: 2 active+remapped+backfill_wait, 4 active+remapped+backfilling, 113 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 644 op/s; 19992/203358 objects misplaced (9.831%); 36 MiB/s, 0 keys/s, 570 objects/s recovering 2023-10-09T06:05:26.821 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:26 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3622488862' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:26.823 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:26 smithi156 ceph-mon[195101]: pgmap v952: 119 pgs: 2 active+remapped+backfill_wait, 4 active+remapped+backfilling, 113 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 644 op/s; 19992/203358 objects misplaced (9.831%); 36 MiB/s, 0 keys/s, 570 objects/s recovering 2023-10-09T06:05:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:26 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3622488862' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:27.771 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:27.780 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:27 smithi064 ceph-mon[212750]: failed to encode map e387 with expected crc 2023-10-09T06:05:27.780 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:27 smithi064 ceph-mon[212750]: osdmap e387: 8 total, 8 up, 7 in 2023-10-09T06:05:27.780 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:27 smithi064 ceph-mon[212750]: failed to encode map e387 with expected crc 2023-10-09T06:05:27.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:27.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:27.782 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:27 smithi064 ceph-mon[220044]: failed to encode map e387 with expected crc 2023-10-09T06:05:27.783 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:27 smithi064 ceph-mon[220044]: osdmap e387: 8 total, 8 up, 7 in 2023-10-09T06:05:27.783 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:27 smithi064 ceph-mon[220044]: failed to encode map e387 with expected crc 2023-10-09T06:05:27.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:27 smithi156 ceph-mon[195101]: failed to encode map e387 with expected crc 2023-10-09T06:05:27.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:27 smithi156 ceph-mon[195101]: osdmap e387: 8 total, 8 up, 7 in 2023-10-09T06:05:27.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:27 smithi156 ceph-mon[195101]: failed to encode map e387 with expected crc 2023-10-09T06:05:27.882 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:27.935 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:28.354 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:28.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:28.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:28.461 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:28.488 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:28.697 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:28.707 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:28 smithi064 ceph-mon[212750]: pgmap v954: 119 pgs: 1 active+clean+remapped, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 113 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 899 op/s; 16437/195807 objects misplaced (8.394%); 51 MiB/s, 820 objects/s recovering 2023-10-09T06:05:28.707 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:28 smithi064 ceph-mon[212750]: failed to encode map e388 with expected crc 2023-10-09T06:05:28.707 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:28 smithi064 ceph-mon[212750]: failed to encode map e388 with expected crc 2023-10-09T06:05:28.707 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:28 smithi064 ceph-mon[212750]: failed to encode map e388 with expected crc 2023-10-09T06:05:28.707 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:28 smithi064 ceph-mon[212750]: osdmap e388: 8 total, 8 up, 7 in 2023-10-09T06:05:28.707 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:28 smithi064 ceph-mon[212750]: failed to encode map e388 with expected crc 2023-10-09T06:05:28.707 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:28 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:05:28.707 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:28 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:28.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:28 smithi064 ceph-mon[220044]: pgmap v954: 119 pgs: 1 active+clean+remapped, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 113 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 899 op/s; 16437/195807 objects misplaced (8.394%); 51 MiB/s, 820 objects/s recovering 2023-10-09T06:05:28.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:28 smithi064 ceph-mon[220044]: failed to encode map e388 with expected crc 2023-10-09T06:05:28.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:28 smithi064 ceph-mon[220044]: failed to encode map e388 with expected crc 2023-10-09T06:05:28.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:28 smithi064 ceph-mon[220044]: failed to encode map e388 with expected crc 2023-10-09T06:05:28.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:28 smithi064 ceph-mon[220044]: osdmap e388: 8 total, 8 up, 7 in 2023-10-09T06:05:28.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:28 smithi064 ceph-mon[220044]: failed to encode map e388 with expected crc 2023-10-09T06:05:28.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:28 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:05:28.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:28 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:28.746 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:28.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:28.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:28.856 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:28.870 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:28 smithi156 ceph-mon[195101]: pgmap v954: 119 pgs: 1 active+clean+remapped, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 113 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 899 op/s; 16437/195807 objects misplaced (8.394%); 51 MiB/s, 820 objects/s recovering 2023-10-09T06:05:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:28 smithi156 ceph-mon[195101]: failed to encode map e388 with expected crc 2023-10-09T06:05:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:28 smithi156 ceph-mon[195101]: failed to encode map e388 with expected crc 2023-10-09T06:05:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:28 smithi156 ceph-mon[195101]: failed to encode map e388 with expected crc 2023-10-09T06:05:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:28 smithi156 ceph-mon[195101]: osdmap e388: 8 total, 8 up, 7 in 2023-10-09T06:05:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:28 smithi156 ceph-mon[195101]: failed to encode map e388 with expected crc 2023-10-09T06:05:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:28 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:05:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:28 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:28.961 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:29.110 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:29.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:29.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:29.219 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:29.289 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:29.378 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:29.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:29.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:29.491 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:29.552 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:29.665 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:29.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:29.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:29.780 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:05:29.852 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:30.242 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:05:30.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:30.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:05:30.346 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:30.429 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:30.785 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:05:30.812 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:30 smithi064 ceph-mon[212750]: pgmap v956: 119 pgs: 1 active+clean+remapped, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 113 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 16437/194685 objects misplaced (8.443%); 38 MiB/s, 608 objects/s recovering 2023-10-09T06:05:30.813 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:30 smithi064 ceph-mon[220044]: pgmap v956: 119 pgs: 1 active+clean+remapped, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 113 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 16437/194685 objects misplaced (8.443%); 38 MiB/s, 608 objects/s recovering 2023-10-09T06:05:30.828 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:30.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:30.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:30.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:30 smithi156 ceph-mon[195101]: pgmap v956: 119 pgs: 1 active+clean+remapped, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 113 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 16437/194685 objects misplaced (8.443%); 38 MiB/s, 608 objects/s recovering 2023-10-09T06:05:30.932 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:30.978 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:31.061 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:31.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:31.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:31.167 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:31.196 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:31.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:31 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'5':'58074','6':'54165'}"}]: dispatch 2023-10-09T06:05:31.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:31 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'5':'58074','6':'54165'}"}]: dispatch 2023-10-09T06:05:31.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:31 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'5':'58074','6':'54165'}"}]: dispatch 2023-10-09T06:05:31.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:31 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'5':'58074','6':'54165'}"}]: dispatch 2023-10-09T06:05:31.551 INFO:teuthology.orchestra.run.smithi064.stdout:moved 6 / 357 (1.68067%) 2023-10-09T06:05:31.551 INFO:teuthology.orchestra.run.smithi064.stdout:avg 51 2023-10-09T06:05:31.551 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 3.02372 -> 3.42261 (expected baseline 6.61168) 2023-10-09T06:05:31.551 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.0 with 47 -> 48 pgs (0.921569 -> 0.941176 * mean) 2023-10-09T06:05:31.551 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.1 with 56 -> 57 pgs (1.09804 -> 1.11765 * mean) 2023-10-09T06:05:31.551 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:05:31.552 INFO:teuthology.orchestra.run.smithi064.stdout:oload 110 2023-10-09T06:05:31.552 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 1 2023-10-09T06:05:31.552 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-10-09T06:05:31.552 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0219 2023-10-09T06:05:31.552 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.0241 2023-10-09T06:05:31.552 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 weight 1.0000 -> 0.8265 2023-10-09T06:05:31.552 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 weight 1.0000 -> 0.8861 2023-10-09T06:05:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:31 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'5':'58074','6':'54165'}"}]: dispatch 2023-10-09T06:05:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:31 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'5':'58074','6':'54165'}"}]: dispatch 2023-10-09T06:05:32.071 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:32.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:32.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:32.176 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:32.206 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:32.256 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:32.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:32.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:32.362 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:32.398 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:32.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[220044]: pgmap v957: 119 pgs: 1 active+clean+remapped, 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 113 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 871 op/s; 11332/186816 objects misplaced (6.066%); 60 MiB/s, 952 objects/s recovering 2023-10-09T06:05:32.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[220044]: from='client.45758 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:05:32.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'5':'58074','6':'54165'}"}]': finished 2023-10-09T06:05:32.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[220044]: osdmap e389: 8 total, 8 up, 7 in 2023-10-09T06:05:32.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[220044]: failed to encode map e389 with expected crc 2023-10-09T06:05:32.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[220044]: failed to encode map e389 with expected crc 2023-10-09T06:05:32.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[220044]: failed to encode map e389 with expected crc 2023-10-09T06:05:32.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[220044]: failed to encode map e389 with expected crc 2023-10-09T06:05:32.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[220044]: failed to encode map e389 with expected crc 2023-10-09T06:05:32.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[220044]: failed to encode map e389 with expected crc 2023-10-09T06:05:32.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[220044]: osdmap e390: 8 total, 8 up, 7 in 2023-10-09T06:05:32.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[220044]: failed to encode map e390 with expected crc 2023-10-09T06:05:32.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[220044]: failed to encode map e390 with expected crc 2023-10-09T06:05:32.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[220044]: failed to encode map e390 with expected crc 2023-10-09T06:05:32.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[212750]: pgmap v957: 119 pgs: 1 active+clean+remapped, 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 113 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 871 op/s; 11332/186816 objects misplaced (6.066%); 60 MiB/s, 952 objects/s recovering 2023-10-09T06:05:32.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[212750]: from='client.45758 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:05:32.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'5':'58074','6':'54165'}"}]': finished 2023-10-09T06:05:32.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[212750]: osdmap e389: 8 total, 8 up, 7 in 2023-10-09T06:05:32.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[212750]: failed to encode map e389 with expected crc 2023-10-09T06:05:32.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[212750]: failed to encode map e389 with expected crc 2023-10-09T06:05:32.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[212750]: failed to encode map e389 with expected crc 2023-10-09T06:05:32.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[212750]: failed to encode map e389 with expected crc 2023-10-09T06:05:32.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[212750]: failed to encode map e389 with expected crc 2023-10-09T06:05:32.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[212750]: failed to encode map e389 with expected crc 2023-10-09T06:05:32.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[212750]: osdmap e390: 8 total, 8 up, 7 in 2023-10-09T06:05:32.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[212750]: failed to encode map e390 with expected crc 2023-10-09T06:05:32.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[212750]: failed to encode map e390 with expected crc 2023-10-09T06:05:32.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:32 smithi064 ceph-mon[212750]: failed to encode map e390 with expected crc 2023-10-09T06:05:32.786 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T06:05:32.787 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-10-09T06:05:32.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:32 smithi156 ceph-mon[195101]: pgmap v957: 119 pgs: 1 active+clean+remapped, 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 113 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 871 op/s; 11332/186816 objects misplaced (6.066%); 60 MiB/s, 952 objects/s recovering 2023-10-09T06:05:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:32 smithi156 ceph-mon[195101]: from='client.45758 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:05:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:32 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'5':'58074','6':'54165'}"}]': finished 2023-10-09T06:05:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:32 smithi156 ceph-mon[195101]: osdmap e389: 8 total, 8 up, 7 in 2023-10-09T06:05:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:32 smithi156 ceph-mon[195101]: failed to encode map e389 with expected crc 2023-10-09T06:05:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:32 smithi156 ceph-mon[195101]: failed to encode map e389 with expected crc 2023-10-09T06:05:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:32 smithi156 ceph-mon[195101]: failed to encode map e389 with expected crc 2023-10-09T06:05:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:32 smithi156 ceph-mon[195101]: failed to encode map e389 with expected crc 2023-10-09T06:05:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:32 smithi156 ceph-mon[195101]: failed to encode map e389 with expected crc 2023-10-09T06:05:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:32 smithi156 ceph-mon[195101]: failed to encode map e389 with expected crc 2023-10-09T06:05:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:32 smithi156 ceph-mon[195101]: osdmap e390: 8 total, 8 up, 7 in 2023-10-09T06:05:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:32 smithi156 ceph-mon[195101]: failed to encode map e390 with expected crc 2023-10-09T06:05:32.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:32 smithi156 ceph-mon[195101]: failed to encode map e390 with expected crc 2023-10-09T06:05:32.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:32 smithi156 ceph-mon[195101]: failed to encode map e390 with expected crc 2023-10-09T06:05:32.903 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:32.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:32.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:33.011 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:33.013 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:33.043 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:33.100 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:33.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:33.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:33.205 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:33.248 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:33.323 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:33.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:33.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:33.430 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:33.434 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:05:33.441 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:05:33.441 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T06:05:33.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T06:05:33.441 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:05:33.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.441 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:159588 6.f 6:f2345e47:::benchmark_data_smithi064_284193_object63045:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e249)", 2023-10-09T06:05:33.441 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:57:23.458648+0000", 2023-10-09T06:05:33.442 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 489.97647784600002, 2023-10-09T06:05:33.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.017528263, 2023-10-09T06:05:33.442 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:33.442 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:33.442 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:33.442 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:05:33.442 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:05:33.442 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 159588 2023-10-09T06:05:33.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.443 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:33.443 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.443 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:33.443 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458648+0000", 2023-10-09T06:05:33.443 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.443 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.443 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.443 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:33.443 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458648+0000", 2023-10-09T06:05:33.443 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.443 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.444 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:33.444 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458637+0000", 2023-10-09T06:05:33.444 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999989 2023-10-09T06:05:33.444 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.444 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:33.444 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458653+0000", 2023-10-09T06:05:33.444 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5855999999999999e-05 2023-10-09T06:05:33.444 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.444 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:33.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458665+0000", 2023-10-09T06:05:33.445 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2064e-05 2023-10-09T06:05:33.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:33.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458674+0000", 2023-10-09T06:05:33.445 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3950000000000007e-06 2023-10-09T06:05:33.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.460075+0000", 2023-10-09T06:05:33.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001401086 2023-10-09T06:05:33.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.446 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:05:33.446 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.460087+0000", 2023-10-09T06:05:33.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1527e-05 2023-10-09T06:05:33.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.446 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.446 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.461560+0000", 2023-10-09T06:05:33.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0014731719999999 2023-10-09T06:05:33.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:33.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.462110+0000", 2023-10-09T06:05:33.447 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00054960100000000004 2023-10-09T06:05:33.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:05:33.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.463093+0000", 2023-10-09T06:05:33.447 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00098321699999999999 2023-10-09T06:05:33.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.467039+0000", 2023-10-09T06:05:33.448 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039463830000000004 2023-10-09T06:05:33.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.448 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.448 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.470896+0000", 2023-10-09T06:05:33.448 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003856805 2023-10-09T06:05:33.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.448 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.448 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:33.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.475276+0000", 2023-10-09T06:05:33.448 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043798400000000003 2023-10-09T06:05:33.449 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:33.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.475478+0000", 2023-10-09T06:05:33.449 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020193899999999999 2023-10-09T06:05:33.449 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:33.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.476176+0000", 2023-10-09T06:05:33.449 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00069817800000000004 2023-10-09T06:05:33.449 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.449 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:33.450 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.450 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.450 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.450 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44588.0:159600 6.1f 6:f9ce68c5:::benchmark_data_smithi064_284193_object63057:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e249)", 2023-10-09T06:05:33.450 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T05:57:23.458682+0000", 2023-10-09T06:05:33.450 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 489.97644353099997, 2023-10-09T06:05:33.450 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0352311799999998, 2023-10-09T06:05:33.450 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:33.450 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:33.450 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:33.450 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44588", 2023-10-09T06:05:33.451 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:05:33.451 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 159600 2023-10-09T06:05:33.451 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.451 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:33.451 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.451 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:33.451 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458682+0000", 2023-10-09T06:05:33.451 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.451 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.451 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.451 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:33.451 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458682+0000", 2023-10-09T06:05:33.452 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.452 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.452 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.452 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:33.452 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458681+0000", 2023-10-09T06:05:33.452 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:05:33.452 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.452 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.452 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:33.452 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458684+0000", 2023-10-09T06:05:33.452 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4700000000000001e-06 2023-10-09T06:05:33.452 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.453 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.453 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:33.453 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458690+0000", 2023-10-09T06:05:33.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.993e-06 2023-10-09T06:05:33.453 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.453 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.453 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:33.453 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.458695+0000", 2023-10-09T06:05:33.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8300000000000003e-06 2023-10-09T06:05:33.453 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.453 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.454 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.460091+0000", 2023-10-09T06:05:33.454 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013964229999999999 2023-10-09T06:05:33.454 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.454 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:05:33.454 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:23.460101+0000", 2023-10-09T06:05:33.454 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0040999999999999e-05 2023-10-09T06:05:33.454 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.454 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.454 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.467208+0000", 2023-10-09T06:05:33.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.007106442 2023-10-09T06:05:33.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.455 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:33.455 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.467772+0000", 2023-10-09T06:05:33.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00056436500000000005 2023-10-09T06:05:33.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.455 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:05:33.455 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.468789+0000", 2023-10-09T06:05:33.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001016711 2023-10-09T06:05:33.456 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.456 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.456 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.475179+0000", 2023-10-09T06:05:33.456 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063902730000000001 2023-10-09T06:05:33.456 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.456 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.456 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.475320+0000", 2023-10-09T06:05:33.456 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000141423 2023-10-09T06:05:33.456 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:33.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.492859+0000", 2023-10-09T06:05:33.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017538893 2023-10-09T06:05:33.457 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:33.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.493094+0000", 2023-10-09T06:05:33.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00023475199999999999 2023-10-09T06:05:33.457 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:33.458 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T05:57:25.493913+0000", 2023-10-09T06:05:33.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00081901499999999998 2023-10-09T06:05:33.458 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.458 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:33.458 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.458 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.458 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.458 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:188635 7.0 7:033812fe:::benchmark_data_smithi064_398483_object83282:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T06:05:33.458 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:01:54.738354+0000", 2023-10-09T06:05:33.458 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 218.696771308, 2023-10-09T06:05:33.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.184445381, 2023-10-09T06:05:33.459 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:33.459 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:05:33.459 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:33.459 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:05:33.459 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:05:33.459 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 188635 2023-10-09T06:05:33.459 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.459 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:33.459 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.459 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:33.459 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738354+0000", 2023-10-09T06:05:33.460 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:33.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738354+0000", 2023-10-09T06:05:33.460 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:33.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738353+0000", 2023-10-09T06:05:33.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:05:33.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.461 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:33.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738359+0000", 2023-10-09T06:05:33.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8730000000000001e-06 2023-10-09T06:05:33.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.461 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:33.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738372+0000", 2023-10-09T06:05:33.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2434e-05 2023-10-09T06:05:33.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.462 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:33.462 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738382+0000", 2023-10-09T06:05:33.462 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0284e-05 2023-10-09T06:05:33.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.462 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.462 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738635+0000", 2023-10-09T06:05:33.462 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00025253899999999998 2023-10-09T06:05:33.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:05:33.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738645+0000", 2023-10-09T06:05:33.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0669000000000001e-05 2023-10-09T06:05:33.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.463 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.920806+0000", 2023-10-09T06:05:33.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1821608100000001 2023-10-09T06:05:33.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.463 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:33.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.922800+0000", 2023-10-09T06:05:33.464 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019933519999999999 2023-10-09T06:05:33.464 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.464 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:33.464 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.464 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.464 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:188644 7.0 7:0ac6db4a:::benchmark_data_smithi064_398483_object83291:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T06:05:33.464 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:01:54.738395+0000", 2023-10-09T06:05:33.464 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 218.69673004800001, 2023-10-09T06:05:33.464 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.21876129, 2023-10-09T06:05:33.464 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:33.465 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:33.465 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:33.465 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:05:33.465 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:05:33.465 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 188644 2023-10-09T06:05:33.465 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.465 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:33.465 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.465 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:33.465 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738395+0000", 2023-10-09T06:05:33.465 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.465 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.466 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:33.466 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738395+0000", 2023-10-09T06:05:33.466 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.466 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.466 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:33.466 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738395+0000", 2023-10-09T06:05:33.466 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:05:33.466 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.466 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:33.467 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738398+0000", 2023-10-09T06:05:33.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3699999999999999e-06 2023-10-09T06:05:33.467 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.467 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.467 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:33.467 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738405+0000", 2023-10-09T06:05:33.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1069999999999999e-06 2023-10-09T06:05:33.467 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.467 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.467 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:33.467 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738411+0000", 2023-10-09T06:05:33.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7910000000000001e-06 2023-10-09T06:05:33.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738669+0000", 2023-10-09T06:05:33.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00025792099999999999 2023-10-09T06:05:33.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:05:33.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738677+0000", 2023-10-09T06:05:33.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.464e-06 2023-10-09T06:05:33.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.469 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.923009+0000", 2023-10-09T06:05:33.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1843328120000001 2023-10-09T06:05:33.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:33.469 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.923497+0000", 2023-10-09T06:05:33.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00048763799999999998 2023-10-09T06:05:33.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,3,5", 2023-10-09T06:05:33.470 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.923579+0000", 2023-10-09T06:05:33.470 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1662999999999994e-05 2023-10-09T06:05:33.470 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.470 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.470 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.928612+0000", 2023-10-09T06:05:33.470 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050336030000000002 2023-10-09T06:05:33.470 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.470 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:33.470 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.930788+0000", 2023-10-09T06:05:33.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0021759819999999999 2023-10-09T06:05:33.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.940855+0000", 2023-10-09T06:05:33.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010066607999999999 2023-10-09T06:05:33.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.957000+0000", 2023-10-09T06:05:33.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016145249 2023-10-09T06:05:33.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:33.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.957047+0000", 2023-10-09T06:05:33.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6553e-05 2023-10-09T06:05:33.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:33.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.957157+0000", 2023-10-09T06:05:33.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110024 2023-10-09T06:05:33.472 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.472 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:33.473 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.473 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:188650 7.0 7:0e595adb:::benchmark_data_smithi064_398483_object83297:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T06:05:33.473 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:01:54.738416+0000", 2023-10-09T06:05:33.473 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 218.69670957400001, 2023-10-09T06:05:33.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2189154719999999, 2023-10-09T06:05:33.473 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:33.473 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:33.473 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:33.473 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:05:33.474 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:05:33.474 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 188650 2023-10-09T06:05:33.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.474 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:33.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:33.474 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738416+0000", 2023-10-09T06:05:33.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:33.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738416+0000", 2023-10-09T06:05:33.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:33.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738415+0000", 2023-10-09T06:05:33.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:05:33.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:33.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738418+0000", 2023-10-09T06:05:33.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8439999999999998e-06 2023-10-09T06:05:33.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:33.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738425+0000", 2023-10-09T06:05:33.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6810000000000001e-06 2023-10-09T06:05:33.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:33.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738430+0000", 2023-10-09T06:05:33.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4140000000000002e-06 2023-10-09T06:05:33.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738697+0000", 2023-10-09T06:05:33.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00026689599999999997 2023-10-09T06:05:33.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:05:33.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738704+0000", 2023-10-09T06:05:33.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6900000000000003e-06 2023-10-09T06:05:33.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.925969+0000", 2023-10-09T06:05:33.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.187264855 2023-10-09T06:05:33.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:33.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.926127+0000", 2023-10-09T06:05:33.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015809 2023-10-09T06:05:33.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,3,5", 2023-10-09T06:05:33.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.926242+0000", 2023-10-09T06:05:33.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011499 2023-10-09T06:05:33.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:33.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.930875+0000", 2023-10-09T06:05:33.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046326539999999999 2023-10-09T06:05:33.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.938798+0000", 2023-10-09T06:05:33.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0079239329999999993 2023-10-09T06:05:33.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.940890+0000", 2023-10-09T06:05:33.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020916749999999999 2023-10-09T06:05:33.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.957195+0000", 2023-10-09T06:05:33.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016304822 2023-10-09T06:05:33.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:33.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.957234+0000", 2023-10-09T06:05:33.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9288000000000001e-05 2023-10-09T06:05:33.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:33.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.957331+0000", 2023-10-09T06:05:33.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7102999999999995e-05 2023-10-09T06:05:33.481 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.481 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:33.481 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.482 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:188668 7.b 7:dc65474a:::benchmark_data_smithi064_398483_object83315:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T06:05:33.482 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:01:54.738435+0000", 2023-10-09T06:05:33.482 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 218.696690598, 2023-10-09T06:05:33.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2254298269999999, 2023-10-09T06:05:33.482 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:33.482 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:33.482 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:33.482 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:05:33.482 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:05:33.483 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 188668 2023-10-09T06:05:33.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.483 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:33.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:33.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738435+0000", 2023-10-09T06:05:33.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:33.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738435+0000", 2023-10-09T06:05:33.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:33.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738434+0000", 2023-10-09T06:05:33.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:05:33.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:33.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738437+0000", 2023-10-09T06:05:33.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.554e-06 2023-10-09T06:05:33.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:33.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738444+0000", 2023-10-09T06:05:33.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7299999999999999e-06 2023-10-09T06:05:33.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:33.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738449+0000", 2023-10-09T06:05:33.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2499999999999997e-06 2023-10-09T06:05:33.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738502+0000", 2023-10-09T06:05:33.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2732000000000003e-05 2023-10-09T06:05:33.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:05:33.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738513+0000", 2023-10-09T06:05:33.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1175999999999999e-05 2023-10-09T06:05:33.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.941813+0000", 2023-10-09T06:05:33.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.20329978 2023-10-09T06:05:33.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:33.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.944592+0000", 2023-10-09T06:05:33.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002779751 2023-10-09T06:05:33.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T06:05:33.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.944739+0000", 2023-10-09T06:05:33.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014661399999999999 2023-10-09T06:05:33.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:33.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.959644+0000", 2023-10-09T06:05:33.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014904914999999999 2023-10-09T06:05:33.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.959694+0000", 2023-10-09T06:05:33.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9805999999999997e-05 2023-10-09T06:05:33.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.959838+0000", 2023-10-09T06:05:33.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014395200000000001 2023-10-09T06:05:33.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.963712+0000", 2023-10-09T06:05:33.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038743050000000002 2023-10-09T06:05:33.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:33.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.963777+0000", 2023-10-09T06:05:33.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4559000000000003e-05 2023-10-09T06:05:33.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:33.490 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.963865+0000", 2023-10-09T06:05:33.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8127000000000005e-05 2023-10-09T06:05:33.490 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.490 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:33.490 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.491 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:188671 7.b 7:d38751a4:::benchmark_data_smithi064_398483_object83318:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T06:05:33.491 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:01:54.738460+0000", 2023-10-09T06:05:33.491 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 218.69666558500001, 2023-10-09T06:05:33.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2255200310000001, 2023-10-09T06:05:33.491 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:33.491 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:33.491 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:33.491 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:05:33.491 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:05:33.491 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 188671 2023-10-09T06:05:33.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.491 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:33.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:33.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738460+0000", 2023-10-09T06:05:33.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:33.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738460+0000", 2023-10-09T06:05:33.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:33.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738459+0000", 2023-10-09T06:05:33.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:05:33.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:33.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738462+0000", 2023-10-09T06:05:33.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8490000000000002e-06 2023-10-09T06:05:33.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:33.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738471+0000", 2023-10-09T06:05:33.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4309999999999994e-06 2023-10-09T06:05:33.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:33.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738476+0000", 2023-10-09T06:05:33.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5119999999999999e-06 2023-10-09T06:05:33.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738536+0000", 2023-10-09T06:05:33.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9747000000000002e-05 2023-10-09T06:05:33.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.495 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:05:33.495 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738543+0000", 2023-10-09T06:05:33.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0700000000000001e-06 2023-10-09T06:05:33.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.495 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.495 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.948303+0000", 2023-10-09T06:05:33.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.209759719 2023-10-09T06:05:33.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:33.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.948403+0000", 2023-10-09T06:05:33.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000100246 2023-10-09T06:05:33.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T06:05:33.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.948472+0000", 2023-10-09T06:05:33.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8626000000000003e-05 2023-10-09T06:05:33.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:33.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.959651+0000", 2023-10-09T06:05:33.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011179022 2023-10-09T06:05:33.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.497 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.960117+0000", 2023-10-09T06:05:33.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000465969 2023-10-09T06:05:33.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.497 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.960161+0000", 2023-10-09T06:05:33.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3989999999999997e-05 2023-10-09T06:05:33.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.498 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.963898+0000", 2023-10-09T06:05:33.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037376639999999999 2023-10-09T06:05:33.498 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.498 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:33.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.963924+0000", 2023-10-09T06:05:33.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5548999999999999e-05 2023-10-09T06:05:33.498 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.499 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:33.499 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.963980+0000", 2023-10-09T06:05:33.499 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6088000000000002e-05 2023-10-09T06:05:33.499 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.499 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:33.499 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.499 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:188672 7.b 7:d57825da:::benchmark_data_smithi064_398483_object83319:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T06:05:33.499 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:01:54.738493+0000", 2023-10-09T06:05:33.499 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 218.69663255399999, 2023-10-09T06:05:33.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.225582057, 2023-10-09T06:05:33.500 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:33.500 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:33.500 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:33.500 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:05:33.500 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:05:33.500 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 188672 2023-10-09T06:05:33.500 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.500 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:33.500 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.500 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:33.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738493+0000", 2023-10-09T06:05:33.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:33.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738493+0000", 2023-10-09T06:05:33.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:33.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738492+0000", 2023-10-09T06:05:33.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:05:33.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:33.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738495+0000", 2023-10-09T06:05:33.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9330000000000001e-06 2023-10-09T06:05:33.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:33.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738503+0000", 2023-10-09T06:05:33.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2699999999999999e-06 2023-10-09T06:05:33.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:33.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738509+0000", 2023-10-09T06:05:33.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2009999999999996e-06 2023-10-09T06:05:33.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738564+0000", 2023-10-09T06:05:33.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5197e-05 2023-10-09T06:05:33.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:05:33.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738571+0000", 2023-10-09T06:05:33.504 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0160000000000003e-06 2023-10-09T06:05:33.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.951402+0000", 2023-10-09T06:05:33.504 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2128306740000001 2023-10-09T06:05:33.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:33.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.951518+0000", 2023-10-09T06:05:33.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116768 2023-10-09T06:05:33.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T06:05:33.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.951641+0000", 2023-10-09T06:05:33.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012232599999999999 2023-10-09T06:05:33.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.960194+0000", 2023-10-09T06:05:33.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085529860000000003 2023-10-09T06:05:33.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.960226+0000", 2023-10-09T06:05:33.506 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2001000000000001e-05 2023-10-09T06:05:33.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:33.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.960368+0000", 2023-10-09T06:05:33.506 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014181900000000001 2023-10-09T06:05:33.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.507 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.964000+0000", 2023-10-09T06:05:33.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0036322279999999999 2023-10-09T06:05:33.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.507 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.507 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:33.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.964022+0000", 2023-10-09T06:05:33.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2167e-05 2023-10-09T06:05:33.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.507 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.507 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:33.508 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.964075+0000", 2023-10-09T06:05:33.508 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2975e-05 2023-10-09T06:05:33.508 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.508 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:33.508 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.508 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:188677 7.0 7:0d2b7d37:::benchmark_data_smithi064_398483_object83324:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T06:05:33.508 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:01:54.738516+0000", 2023-10-09T06:05:33.508 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 218.69660991399999, 2023-10-09T06:05:33.508 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.218979737, 2023-10-09T06:05:33.508 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:33.509 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:33.509 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:33.509 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:05:33.509 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:05:33.509 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 188677 2023-10-09T06:05:33.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.509 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:33.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:33.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738516+0000", 2023-10-09T06:05:33.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:33.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738516+0000", 2023-10-09T06:05:33.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:33.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738515+0000", 2023-10-09T06:05:33.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:05:33.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:33.511 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738518+0000", 2023-10-09T06:05:33.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7879999999999998e-06 2023-10-09T06:05:33.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:33.511 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738524+0000", 2023-10-09T06:05:33.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2559999999999999e-06 2023-10-09T06:05:33.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:33.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738529+0000", 2023-10-09T06:05:33.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2480000000000004e-06 2023-10-09T06:05:33.512 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738726+0000", 2023-10-09T06:05:33.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000196928 2023-10-09T06:05:33.512 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:05:33.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.738735+0000", 2023-10-09T06:05:33.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4079999999999998e-06 2023-10-09T06:05:33.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.513 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.513 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.513 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.928644+0000", 2023-10-09T06:05:33.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.189909745 2023-10-09T06:05:33.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.513 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.513 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:33.513 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.928727+0000", 2023-10-09T06:05:33.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2735e-05 2023-10-09T06:05:33.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.513 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,3,5", 2023-10-09T06:05:33.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.928798+0000", 2023-10-09T06:05:33.514 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1055000000000003e-05 2023-10-09T06:05:33.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:33.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.930994+0000", 2023-10-09T06:05:33.514 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002196141 2023-10-09T06:05:33.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.938838+0000", 2023-10-09T06:05:33.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0078432740000000008 2023-10-09T06:05:33.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.942721+0000", 2023-10-09T06:05:33.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038830990000000001 2023-10-09T06:05:33.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.957366+0000", 2023-10-09T06:05:33.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014644885999999999 2023-10-09T06:05:33.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.516 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:33.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.957403+0000", 2023-10-09T06:05:33.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6959000000000003e-05 2023-10-09T06:05:33.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.516 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:33.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.957495+0000", 2023-10-09T06:05:33.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2639000000000005e-05 2023-10-09T06:05:33.516 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.517 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:33.517 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.517 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:188681 7.b 7:da89e078:::benchmark_data_smithi064_398483_object83328:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2023-10-09T06:05:33.517 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:01:54.859467+0000", 2023-10-09T06:05:33.517 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 218.57565888799999, 2023-10-09T06:05:33.517 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.104714006, 2023-10-09T06:05:33.517 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:33.517 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:33.517 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:33.517 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:05:33.518 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:05:33.518 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 188681 2023-10-09T06:05:33.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.518 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:33.518 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.518 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:33.518 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.859467+0000", 2023-10-09T06:05:33.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.518 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.518 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:33.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.859467+0000", 2023-10-09T06:05:33.519 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:33.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.859464+0000", 2023-10-09T06:05:33.519 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:05:33.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:33.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.859475+0000", 2023-10-09T06:05:33.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0161e-05 2023-10-09T06:05:33.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.520 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:33.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.859496+0000", 2023-10-09T06:05:33.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1185000000000001e-05 2023-10-09T06:05:33.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.520 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:33.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.859505+0000", 2023-10-09T06:05:33.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4360000000000008e-06 2023-10-09T06:05:33.521 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.521 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.859568+0000", 2023-10-09T06:05:33.521 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2723000000000002e-05 2023-10-09T06:05:33.521 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:05:33.521 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.859581+0000", 2023-10-09T06:05:33.521 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3193999999999999e-05 2023-10-09T06:05:33.521 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.956496+0000", 2023-10-09T06:05:33.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.096915267 2023-10-09T06:05:33.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:33.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.956642+0000", 2023-10-09T06:05:33.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014533699999999999 2023-10-09T06:05:33.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T06:05:33.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.956777+0000", 2023-10-09T06:05:33.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013512299999999999 2023-10-09T06:05:33.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:33.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.960374+0000", 2023-10-09T06:05:33.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003597303 2023-10-09T06:05:33.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.963246+0000", 2023-10-09T06:05:33.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028718070000000001 2023-10-09T06:05:33.524 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.524 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.524 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.524 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.963266+0000", 2023-10-09T06:05:33.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9972000000000001e-05 2023-10-09T06:05:33.524 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.524 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.524 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.524 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.964096+0000", 2023-10-09T06:05:33.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00082984599999999997 2023-10-09T06:05:33.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.525 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:33.525 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.964118+0000", 2023-10-09T06:05:33.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1908000000000001e-05 2023-10-09T06:05:33.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.525 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:33.525 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.964181+0000", 2023-10-09T06:05:33.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2834000000000002e-05 2023-10-09T06:05:33.525 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.526 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:33.526 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.528 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:201273 7.e 7:79b1205a:::benchmark_data_smithi064_398483_object95920:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T06:05:33.528 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:02:17.220851+0000", 2023-10-09T06:05:33.528 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 196.214274907, 2023-10-09T06:05:33.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15538379799999999, 2023-10-09T06:05:33.528 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:33.528 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:33.528 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:33.528 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:05:33.528 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:05:33.529 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 201273 2023-10-09T06:05:33.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.529 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:33.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:33.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220851+0000", 2023-10-09T06:05:33.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:33.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220851+0000", 2023-10-09T06:05:33.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:33.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220849+0000", 2023-10-09T06:05:33.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:05:33.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:33.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220856+0000", 2023-10-09T06:05:33.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8430000000000004e-06 2023-10-09T06:05:33.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:33.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220869+0000", 2023-10-09T06:05:33.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2901999999999999e-05 2023-10-09T06:05:33.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:33.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220878+0000", 2023-10-09T06:05:33.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8040000000000007e-06 2023-10-09T06:05:33.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.532 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.532 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220939+0000", 2023-10-09T06:05:33.532 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1678000000000001e-05 2023-10-09T06:05:33.532 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.532 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:33.532 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.221067+0000", 2023-10-09T06:05:33.532 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000127479 2023-10-09T06:05:33.532 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.532 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:05:33.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.221185+0000", 2023-10-09T06:05:33.533 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000117911 2023-10-09T06:05:33.533 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.533 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.533 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.228808+0000", 2023-10-09T06:05:33.533 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0076236990000000003 2023-10-09T06:05:33.533 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.533 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.533 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:33.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.235105+0000", 2023-10-09T06:05:33.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062963810000000002 2023-10-09T06:05:33.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.534 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.534 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.376133+0000", 2023-10-09T06:05:33.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14102801100000001 2023-10-09T06:05:33.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.534 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:33.534 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.376189+0000", 2023-10-09T06:05:33.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.643e-05 2023-10-09T06:05:33.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.535 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:33.535 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.376234+0000", 2023-10-09T06:05:33.535 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5046999999999997e-05 2023-10-09T06:05:33.535 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.535 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:33.535 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.535 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:201274 7.0 7:0b6d1882:::benchmark_data_smithi064_398483_object95921:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T06:05:33.535 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:02:17.220892+0000", 2023-10-09T06:05:33.536 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 196.21423295100001, 2023-10-09T06:05:33.536 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18597917999999999, 2023-10-09T06:05:33.536 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:33.536 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:33.536 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:33.536 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:05:33.536 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:05:33.536 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 201274 2023-10-09T06:05:33.536 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.536 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:33.536 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.536 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:33.537 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220892+0000", 2023-10-09T06:05:33.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.537 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.537 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.537 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:33.537 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220892+0000", 2023-10-09T06:05:33.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.537 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.537 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.537 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:33.537 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220892+0000", 2023-10-09T06:05:33.538 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:05:33.538 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.538 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:33.538 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220895+0000", 2023-10-09T06:05:33.538 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8480000000000001e-06 2023-10-09T06:05:33.538 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.538 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:33.538 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220901+0000", 2023-10-09T06:05:33.538 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.037e-06 2023-10-09T06:05:33.538 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.539 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.539 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:33.539 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.220906+0000", 2023-10-09T06:05:33.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0989999999999998e-06 2023-10-09T06:05:33.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.539 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.539 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.539 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.222196+0000", 2023-10-09T06:05:33.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001289789 2023-10-09T06:05:33.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.539 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.539 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:33.540 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.222333+0000", 2023-10-09T06:05:33.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000137179 2023-10-09T06:05:33.540 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.540 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.540 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,3,5", 2023-10-09T06:05:33.540 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.222448+0000", 2023-10-09T06:05:33.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000114509 2023-10-09T06:05:33.540 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.540 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.540 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:33.541 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.225473+0000", 2023-10-09T06:05:33.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030259430000000001 2023-10-09T06:05:33.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.541 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.541 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.541 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.228901+0000", 2023-10-09T06:05:33.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034273670000000002 2023-10-09T06:05:33.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.541 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.541 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.541 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.232731+0000", 2023-10-09T06:05:33.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038302480000000001 2023-10-09T06:05:33.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.542 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.542 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.406688+0000", 2023-10-09T06:05:33.542 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.173957205 2023-10-09T06:05:33.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.542 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:33.542 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.406746+0000", 2023-10-09T06:05:33.542 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8065000000000001e-05 2023-10-09T06:05:33.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.543 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.543 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:33.543 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.406872+0000", 2023-10-09T06:05:33.543 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000125334 2023-10-09T06:05:33.543 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.543 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:33.543 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.543 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.543 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.543 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:202667 7.0 7:0aae096f:::benchmark_data_smithi064_398483_object97314:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-10-09T06:05:33.543 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:02:20.340853+0000", 2023-10-09T06:05:33.544 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 193.094272058, 2023-10-09T06:05:33.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25382495399999999, 2023-10-09T06:05:33.544 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:33.544 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:33.544 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:33.544 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:05:33.544 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:05:33.544 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 202667 2023-10-09T06:05:33.544 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.544 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:33.544 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.544 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:33.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.340853+0000", 2023-10-09T06:05:33.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.545 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:33.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.340853+0000", 2023-10-09T06:05:33.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.545 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:33.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.340852+0000", 2023-10-09T06:05:33.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:05:33.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.546 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:33.546 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.340859+0000", 2023-10-09T06:05:33.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.729e-06 2023-10-09T06:05:33.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.546 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:33.546 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.340873+0000", 2023-10-09T06:05:33.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.331e-05 2023-10-09T06:05:33.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.547 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.547 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:33.547 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.340882+0000", 2023-10-09T06:05:33.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3260000000000002e-06 2023-10-09T06:05:33.547 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.547 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.547 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.547 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.340930+0000", 2023-10-09T06:05:33.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7539e-05 2023-10-09T06:05:33.547 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.547 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.547 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:33.548 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.341069+0000", 2023-10-09T06:05:33.548 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000138999 2023-10-09T06:05:33.548 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.548 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.548 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,3,5", 2023-10-09T06:05:33.548 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.341190+0000", 2023-10-09T06:05:33.548 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0001211 2023-10-09T06:05:33.548 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.548 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.548 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:33.548 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594377+0000", 2023-10-09T06:05:33.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25318729800000001 2023-10-09T06:05:33.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.549 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.549 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594424+0000", 2023-10-09T06:05:33.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7475999999999997e-05 2023-10-09T06:05:33.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.549 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.549 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594460+0000", 2023-10-09T06:05:33.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6032999999999999e-05 2023-10-09T06:05:33.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594499+0000", 2023-10-09T06:05:33.550 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8976000000000001e-05 2023-10-09T06:05:33.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:33.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594556+0000", 2023-10-09T06:05:33.550 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6440999999999997e-05 2023-10-09T06:05:33.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:33.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594678+0000", 2023-10-09T06:05:33.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012243700000000001 2023-10-09T06:05:33.551 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.551 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:33.551 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.551 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.45263.0:202666 7.5 e348/285 7:a4298507:::benchmark_data_smithi064_398483_object97313:head v 348'12941, mlcod=348'12939)", 2023-10-09T06:05:33.551 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:02:20.341200+0000", 2023-10-09T06:05:33.552 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 193.09392588, 2023-10-09T06:05:33.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25306627599999998, 2023-10-09T06:05:33.552 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:33.552 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:33.552 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:33.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:33.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.341200+0000", 2023-10-09T06:05:33.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:33.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.341200+0000", 2023-10-09T06:05:33.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:33.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.341199+0000", 2023-10-09T06:05:33.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:05:33.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:33.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.341206+0000", 2023-10-09T06:05:33.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1110000000000002e-06 2023-10-09T06:05:33.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:33.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.341218+0000", 2023-10-09T06:05:33.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1789e-05 2023-10-09T06:05:33.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:33.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.341225+0000", 2023-10-09T06:05:33.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0690000000000004e-06 2023-10-09T06:05:33.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.341270+0000", 2023-10-09T06:05:33.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5640999999999999e-05 2023-10-09T06:05:33.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:33.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.341292+0000", 2023-10-09T06:05:33.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1225000000000001e-05 2023-10-09T06:05:33.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:33.556 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.345678+0000", 2023-10-09T06:05:33.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043867799999999998 2023-10-09T06:05:33.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:33.556 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594266+0000", 2023-10-09T06:05:33.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24858746800000001 2023-10-09T06:05:33.556 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.556 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:33.556 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.557 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.557 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPushReply(7.0 348/334 [PushReplyOp(7:083777c0:::benchmark_data_smithi064_398483_object97370:head)])", 2023-10-09T06:05:33.557 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:02:20.344731+0000", 2023-10-09T06:05:33.557 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 193.09039417100001, 2023-10-09T06:05:33.557 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24962795099999999, 2023-10-09T06:05:33.557 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:33.557 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:05:33.557 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:33.557 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.557 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:33.557 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.344731+0000", 2023-10-09T06:05:33.557 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.558 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.558 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.558 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:33.558 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.344731+0000", 2023-10-09T06:05:33.558 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.558 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.558 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.558 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:33.558 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.344731+0000", 2023-10-09T06:05:33.558 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:05:33.558 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.559 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:33.559 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.344735+0000", 2023-10-09T06:05:33.559 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0999999999999997e-06 2023-10-09T06:05:33.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.559 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:33.559 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.344743+0000", 2023-10-09T06:05:33.559 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0579999999999998e-06 2023-10-09T06:05:33.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.559 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:33.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.344751+0000", 2023-10-09T06:05:33.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.303e-06 2023-10-09T06:05:33.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.344812+0000", 2023-10-09T06:05:33.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0460000000000001e-05 2023-10-09T06:05:33.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:33.561 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594359+0000", 2023-10-09T06:05:33.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24954735 2023-10-09T06:05:33.561 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.561 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:33.561 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.561 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.561 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.45263.0:202667 7.0 e348/334 ondisk, result = 0)", 2023-10-09T06:05:33.561 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:02:20.347781+0000", 2023-10-09T06:05:33.561 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 193.087344876, 2023-10-09T06:05:33.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24668209799999999, 2023-10-09T06:05:33.561 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:33.562 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:05:33.562 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:33.562 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.562 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:33.562 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.347781+0000", 2023-10-09T06:05:33.562 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.562 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.562 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.562 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:33.562 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.347781+0000", 2023-10-09T06:05:33.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:33.563 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.347780+0000", 2023-10-09T06:05:33.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:05:33.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:33.563 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.347786+0000", 2023-10-09T06:05:33.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7920000000000004e-06 2023-10-09T06:05:33.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:33.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594128+0000", 2023-10-09T06:05:33.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24634131400000001 2023-10-09T06:05:33.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:33.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594142+0000", 2023-10-09T06:05:33.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4664000000000001e-05 2023-10-09T06:05:33.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594448+0000", 2023-10-09T06:05:33.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00030556099999999998 2023-10-09T06:05:33.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:33.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594455+0000", 2023-10-09T06:05:33.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9299999999999997e-06 2023-10-09T06:05:33.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:33.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594463+0000", 2023-10-09T06:05:33.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7600000000000002e-06 2023-10-09T06:05:33.566 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.566 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:33.566 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.566 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.566 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.45263.0:202667 7.0 e348/334 ondisk, result = 0)", 2023-10-09T06:05:33.566 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:02:20.348183+0000", 2023-10-09T06:05:33.566 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 193.086942306, 2023-10-09T06:05:33.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.246244344, 2023-10-09T06:05:33.566 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:33.566 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:05:33.567 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:33.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.567 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:33.567 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.348183+0000", 2023-10-09T06:05:33.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.567 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.567 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:33.567 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.348183+0000", 2023-10-09T06:05:33.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.567 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.568 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:33.568 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.348183+0000", 2023-10-09T06:05:33.568 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:05:33.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.568 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.568 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:33.568 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.348187+0000", 2023-10-09T06:05:33.568 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5950000000000001e-06 2023-10-09T06:05:33.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.568 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.568 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:33.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594117+0000", 2023-10-09T06:05:33.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24592978099999999 2023-10-09T06:05:33.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:33.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594130+0000", 2023-10-09T06:05:33.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3023999999999999e-05 2023-10-09T06:05:33.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594407+0000", 2023-10-09T06:05:33.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00027666300000000002 2023-10-09T06:05:33.570 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.570 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.570 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:33.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594418+0000", 2023-10-09T06:05:33.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0890000000000001e-05 2023-10-09T06:05:33.570 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.570 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.570 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:33.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:20.594427+0000", 2023-10-09T06:05:33.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9469999999999998e-06 2023-10-09T06:05:33.570 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.571 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:33.571 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.571 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.571 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.571 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:203099 7.0 7:0ca84f41:::benchmark_data_smithi064_398483_object97746:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e349)", 2023-10-09T06:05:33.571 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:02:21.080783+0000", 2023-10-09T06:05:33.571 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 192.35434284499999, 2023-10-09T06:05:33.571 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.17404369, 2023-10-09T06:05:33.571 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:33.571 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:33.572 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:33.572 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:05:33.572 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:05:33.572 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 203099 2023-10-09T06:05:33.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.572 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:33.572 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.572 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:33.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.080783+0000", 2023-10-09T06:05:33.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:33.573 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.080783+0000", 2023-10-09T06:05:33.573 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:33.573 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.080782+0000", 2023-10-09T06:05:33.573 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:05:33.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:33.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.080785+0000", 2023-10-09T06:05:33.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4430000000000002e-06 2023-10-09T06:05:33.574 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.574 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.574 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:33.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.080790+0000", 2023-10-09T06:05:33.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.507e-06 2023-10-09T06:05:33.574 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.574 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.574 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:33.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.080795+0000", 2023-10-09T06:05:33.575 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5260000000000004e-06 2023-10-09T06:05:33.575 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.575 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.575 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.575 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093828+0000", 2023-10-09T06:05:33.575 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013033756000000001 2023-10-09T06:05:33.575 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.575 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.575 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:33.575 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093950+0000", 2023-10-09T06:05:33.575 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012169599999999999 2023-10-09T06:05:33.575 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.576 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.576 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,3,5", 2023-10-09T06:05:33.576 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.094070+0000", 2023-10-09T06:05:33.576 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011979100000000001 2023-10-09T06:05:33.576 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.576 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.576 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.576 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.106906+0000", 2023-10-09T06:05:33.576 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0128361 2023-10-09T06:05:33.576 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.576 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:33.577 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.106918+0000", 2023-10-09T06:05:33.577 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1938e-05 2023-10-09T06:05:33.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.577 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.107211+0000", 2023-10-09T06:05:33.577 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000293147 2023-10-09T06:05:33.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.577 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.254679+0000", 2023-10-09T06:05:33.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14746752099999999 2023-10-09T06:05:33.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.578 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.578 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:33.578 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.254736+0000", 2023-10-09T06:05:33.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7077999999999998e-05 2023-10-09T06:05:33.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.578 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.578 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:33.578 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.254826+0000", 2023-10-09T06:05:33.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0613999999999997e-05 2023-10-09T06:05:33.579 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.579 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:33.579 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.579 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.579 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:203102 7.0 7:05c12b3f:::benchmark_data_smithi064_398483_object97749:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e349)", 2023-10-09T06:05:33.579 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:02:21.093652+0000", 2023-10-09T06:05:33.579 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 192.34147379000001, 2023-10-09T06:05:33.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16137338300000001, 2023-10-09T06:05:33.579 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:33.579 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:33.580 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:33.580 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:05:33.580 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:05:33.580 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 203102 2023-10-09T06:05:33.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.580 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:33.580 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.580 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:33.580 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093652+0000", 2023-10-09T06:05:33.580 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.581 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:33.581 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093652+0000", 2023-10-09T06:05:33.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.581 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.581 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:33.581 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093650+0000", 2023-10-09T06:05:33.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:05:33.581 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.581 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:33.582 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093658+0000", 2023-10-09T06:05:33.582 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3819999999999998e-06 2023-10-09T06:05:33.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.582 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:33.582 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093671+0000", 2023-10-09T06:05:33.582 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2763e-05 2023-10-09T06:05:33.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.582 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:33.582 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093679+0000", 2023-10-09T06:05:33.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4449999999999996e-06 2023-10-09T06:05:33.583 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.583 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.583 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.097473+0000", 2023-10-09T06:05:33.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037941619999999998 2023-10-09T06:05:33.583 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.583 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.583 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:33.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.097609+0000", 2023-10-09T06:05:33.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013628800000000001 2023-10-09T06:05:33.584 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.584 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.584 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,3,5", 2023-10-09T06:05:33.584 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.097747+0000", 2023-10-09T06:05:33.584 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000137589 2023-10-09T06:05:33.584 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.584 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.584 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:33.584 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.106926+0000", 2023-10-09T06:05:33.584 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0091790280000000005 2023-10-09T06:05:33.584 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.585 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.585 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.106963+0000", 2023-10-09T06:05:33.585 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6659999999999998e-05 2023-10-09T06:05:33.585 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.585 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.585 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.254542+0000", 2023-10-09T06:05:33.585 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14757968599999999 2023-10-09T06:05:33.585 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.585 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.585 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.254872+0000", 2023-10-09T06:05:33.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00032967700000000001 2023-10-09T06:05:33.586 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.586 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.586 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:33.586 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.254922+0000", 2023-10-09T06:05:33.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0275999999999998e-05 2023-10-09T06:05:33.586 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.586 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.586 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:33.586 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.255025+0000", 2023-10-09T06:05:33.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000102693 2023-10-09T06:05:33.587 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.587 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:33.587 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.587 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:203105 7.0 7:042c63f5:::benchmark_data_smithi064_398483_object97752:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e349)", 2023-10-09T06:05:33.587 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:02:21.093975+0000", 2023-10-09T06:05:33.587 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 192.34115025099999, 2023-10-09T06:05:33.587 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16126423000000001, 2023-10-09T06:05:33.587 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:05:33.587 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:05:33.588 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:05:33.588 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:05:33.588 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:05:33.588 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 203105 2023-10-09T06:05:33.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.588 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:05:33.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:05:33.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093975+0000", 2023-10-09T06:05:33.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:05:33.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093975+0000", 2023-10-09T06:05:33.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:05:33.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:05:33.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093974+0000", 2023-10-09T06:05:33.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:05:33.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:05:33.590 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093978+0000", 2023-10-09T06:05:33.590 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0509999999999998e-06 2023-10-09T06:05:33.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.590 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:05:33.590 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093989+0000", 2023-10-09T06:05:33.590 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0478e-05 2023-10-09T06:05:33.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.590 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:05:33.590 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.093996+0000", 2023-10-09T06:05:33.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2069999999999998e-06 2023-10-09T06:05:33.591 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.591 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.591 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:05:33.591 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.100876+0000", 2023-10-09T06:05:33.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0068799109999999998 2023-10-09T06:05:33.591 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.591 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.591 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:05:33.591 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.101009+0000", 2023-10-09T06:05:33.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013321599999999999 2023-10-09T06:05:33.591 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.592 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.592 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,3,5", 2023-10-09T06:05:33.592 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.101139+0000", 2023-10-09T06:05:33.592 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012954899999999999 2023-10-09T06:05:33.592 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.592 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.592 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.592 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.107003+0000", 2023-10-09T06:05:33.592 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058646050000000002 2023-10-09T06:05:33.592 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.592 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:05:33.593 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.107020+0000", 2023-10-09T06:05:33.593 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6518999999999999e-05 2023-10-09T06:05:33.593 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.593 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.593 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.254642+0000", 2023-10-09T06:05:33.593 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14762177100000001 2023-10-09T06:05:33.593 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.593 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:05:33.593 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.255083+0000", 2023-10-09T06:05:33.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000441562 2023-10-09T06:05:33.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.594 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.594 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:05:33.594 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.255137+0000", 2023-10-09T06:05:33.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3751000000000001e-05 2023-10-09T06:05:33.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:05:33.594 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:05:33.594 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:05:33.594 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:21.255239+0000", 2023-10-09T06:05:33.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000102372 2023-10-09T06:05:33.595 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.595 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:33.595 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.595 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:05:33.595 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:05:33.595 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:05:33.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:33.596 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:33.596 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:33.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:33.697 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:33.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:33 smithi064 ceph-mon[220044]: failed to encode map e390 with expected crc 2023-10-09T06:05:33.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:33 smithi064 ceph-mon[220044]: failed to encode map e390 with expected crc 2023-10-09T06:05:33.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:33 smithi064 ceph-mon[220044]: failed to encode map e390 with expected crc 2023-10-09T06:05:33.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:33 smithi064 ceph-mon[220044]: failed to encode map e390 with expected crc 2023-10-09T06:05:33.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:33 smithi064 ceph-mon[220044]: failed to encode map e390 with expected crc 2023-10-09T06:05:33.725 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:33 smithi064 ceph-mon[220044]: failed to encode map e390 with expected crc 2023-10-09T06:05:33.725 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:33 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1733735103' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:05:33.725 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:33 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:05:33.725 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:33 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1733735103' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:05:33.725 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:33 smithi064 ceph-mon[220044]: osdmap e391: 8 total, 8 up, 7 in 2023-10-09T06:05:33.725 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:33 smithi064 ceph-mon[212750]: failed to encode map e390 with expected crc 2023-10-09T06:05:33.725 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:33 smithi064 ceph-mon[212750]: failed to encode map e390 with expected crc 2023-10-09T06:05:33.725 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:33 smithi064 ceph-mon[212750]: failed to encode map e390 with expected crc 2023-10-09T06:05:33.726 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:33 smithi064 ceph-mon[212750]: failed to encode map e390 with expected crc 2023-10-09T06:05:33.726 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:33 smithi064 ceph-mon[212750]: failed to encode map e390 with expected crc 2023-10-09T06:05:33.726 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:33 smithi064 ceph-mon[212750]: failed to encode map e390 with expected crc 2023-10-09T06:05:33.726 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:33 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1733735103' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:05:33.726 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:33 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:05:33.726 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:33 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1733735103' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:05:33.726 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:33 smithi064 ceph-mon[212750]: osdmap e391: 8 total, 8 up, 7 in 2023-10-09T06:05:33.765 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:33 smithi156 ceph-mon[195101]: failed to encode map e390 with expected crc 2023-10-09T06:05:33.765 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:33 smithi156 ceph-mon[195101]: failed to encode map e390 with expected crc 2023-10-09T06:05:33.766 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:33 smithi156 ceph-mon[195101]: failed to encode map e390 with expected crc 2023-10-09T06:05:33.766 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:33 smithi156 ceph-mon[195101]: failed to encode map e390 with expected crc 2023-10-09T06:05:33.766 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:33 smithi156 ceph-mon[195101]: failed to encode map e390 with expected crc 2023-10-09T06:05:33.766 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:33 smithi156 ceph-mon[195101]: failed to encode map e390 with expected crc 2023-10-09T06:05:33.766 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:33 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1733735103' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:05:33.766 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:33 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:05:33.766 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:33 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1733735103' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:05:33.766 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:33 smithi156 ceph-mon[195101]: osdmap e391: 8 total, 8 up, 7 in 2023-10-09T06:05:33.768 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:33.868 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:33.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:33.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:33.988 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:34.082 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:34.636 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:34.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:34.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:34.741 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:34.773 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:34.826 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:34.830 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:34 smithi156 ceph-mon[195101]: pgmap v960: 119 pgs: 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 114 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 11332/186201 objects misplaced (6.086%); 46 MiB/s, 733 objects/s recovering 2023-10-09T06:05:34.830 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:34 smithi156 ceph-mon[195101]: failed to encode map e391 with expected crc 2023-10-09T06:05:34.830 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:34 smithi156 ceph-mon[195101]: failed to encode map e391 with expected crc 2023-10-09T06:05:34.830 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:34 smithi156 ceph-mon[195101]: failed to encode map e391 with expected crc 2023-10-09T06:05:34.830 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:34 smithi156 ceph-mon[195101]: failed to encode map e391 with expected crc 2023-10-09T06:05:34.830 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:34 smithi156 ceph-mon[195101]: failed to encode map e391 with expected crc 2023-10-09T06:05:34.830 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:34 smithi156 ceph-mon[195101]: 8.d starting backfill to osd.1 from (0'0,0'0] MAX to 388'9080 2023-10-09T06:05:34.830 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:34 smithi156 ceph-mon[195101]: osdmap e392: 8 total, 8 up, 7 in 2023-10-09T06:05:34.830 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:34 smithi156 ceph-mon[195101]: failed to encode map e392 with expected crc 2023-10-09T06:05:34.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:34.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:34.932 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:34.963 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:34.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:34 smithi064 ceph-mon[220044]: pgmap v960: 119 pgs: 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 114 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 11332/186201 objects misplaced (6.086%); 46 MiB/s, 733 objects/s recovering 2023-10-09T06:05:34.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:34 smithi064 ceph-mon[220044]: failed to encode map e391 with expected crc 2023-10-09T06:05:34.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:34 smithi064 ceph-mon[220044]: failed to encode map e391 with expected crc 2023-10-09T06:05:34.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:34 smithi064 ceph-mon[220044]: failed to encode map e391 with expected crc 2023-10-09T06:05:34.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:34 smithi064 ceph-mon[220044]: failed to encode map e391 with expected crc 2023-10-09T06:05:34.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:34 smithi064 ceph-mon[220044]: failed to encode map e391 with expected crc 2023-10-09T06:05:34.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:34 smithi064 ceph-mon[220044]: 8.d starting backfill to osd.1 from (0'0,0'0] MAX to 388'9080 2023-10-09T06:05:34.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:34 smithi064 ceph-mon[220044]: osdmap e392: 8 total, 8 up, 7 in 2023-10-09T06:05:34.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:34 smithi064 ceph-mon[220044]: failed to encode map e392 with expected crc 2023-10-09T06:05:34.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:34 smithi064 ceph-mon[212750]: pgmap v960: 119 pgs: 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 114 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 11332/186201 objects misplaced (6.086%); 46 MiB/s, 733 objects/s recovering 2023-10-09T06:05:34.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:34 smithi064 ceph-mon[212750]: failed to encode map e391 with expected crc 2023-10-09T06:05:34.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:34 smithi064 ceph-mon[212750]: failed to encode map e391 with expected crc 2023-10-09T06:05:34.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:34 smithi064 ceph-mon[212750]: failed to encode map e391 with expected crc 2023-10-09T06:05:34.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:34 smithi064 ceph-mon[212750]: failed to encode map e391 with expected crc 2023-10-09T06:05:34.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:34 smithi064 ceph-mon[212750]: failed to encode map e391 with expected crc 2023-10-09T06:05:34.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:34 smithi064 ceph-mon[212750]: 8.d starting backfill to osd.1 from (0'0,0'0] MAX to 388'9080 2023-10-09T06:05:34.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:34 smithi064 ceph-mon[212750]: osdmap e392: 8 total, 8 up, 7 in 2023-10-09T06:05:34.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:34 smithi064 ceph-mon[212750]: failed to encode map e392 with expected crc 2023-10-09T06:05:35.041 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:35.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:35.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:35.147 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:35.178 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:35.233 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:35.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:35.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:35.339 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:35.370 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:35.424 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:35.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:35.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:35.533 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:35.571 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:35 smithi064 ceph-mon[212750]: failed to encode map e392 with expected crc 2023-10-09T06:05:35.572 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:05:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:05:35] "GET /metrics HTTP/1.1" 200 36413 "" "Prometheus/2.43.0" 2023-10-09T06:05:35.595 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:35.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:35 smithi156 ceph-mon[195101]: failed to encode map e392 with expected crc 2023-10-09T06:05:35.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:35 smithi156 ceph-mon[195101]: failed to encode map e392 with expected crc 2023-10-09T06:05:35.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:35 smithi156 ceph-mon[195101]: failed to encode map e392 with expected crc 2023-10-09T06:05:35.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:35 smithi156 ceph-mon[195101]: pgmap v963: 119 pgs: 1 peering, 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 113 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 296 op/s; 10032/183258 objects misplaced (5.474%); 19 MiB/s, 309 objects/s recovering 2023-10-09T06:05:35.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:35 smithi156 ceph-mon[195101]: osdmap e393: 8 total, 8 up, 7 in 2023-10-09T06:05:35.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:35 smithi064 ceph-mon[212750]: failed to encode map e392 with expected crc 2023-10-09T06:05:35.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:35 smithi064 ceph-mon[212750]: failed to encode map e392 with expected crc 2023-10-09T06:05:35.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:35 smithi064 ceph-mon[212750]: pgmap v963: 119 pgs: 1 peering, 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 113 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 296 op/s; 10032/183258 objects misplaced (5.474%); 19 MiB/s, 309 objects/s recovering 2023-10-09T06:05:35.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:35 smithi064 ceph-mon[212750]: osdmap e393: 8 total, 8 up, 7 in 2023-10-09T06:05:35.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:35 smithi064 ceph-mon[220044]: failed to encode map e392 with expected crc 2023-10-09T06:05:35.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:35 smithi064 ceph-mon[220044]: failed to encode map e392 with expected crc 2023-10-09T06:05:35.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:35 smithi064 ceph-mon[220044]: failed to encode map e392 with expected crc 2023-10-09T06:05:35.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:35 smithi064 ceph-mon[220044]: pgmap v963: 119 pgs: 1 peering, 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 113 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 296 op/s; 10032/183258 objects misplaced (5.474%); 19 MiB/s, 309 objects/s recovering 2023-10-09T06:05:35.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:35 smithi064 ceph-mon[220044]: osdmap e393: 8 total, 8 up, 7 in 2023-10-09T06:05:36.244 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:36.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:36.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:36.349 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:36.377 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:36.432 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:36.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:36.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:36.546 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:36.616 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:36.697 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:36.701 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:36 smithi156 ceph-mon[195101]: failed to encode map e393 with expected crc 2023-10-09T06:05:36.701 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:36 smithi156 ceph-mon[195101]: failed to encode map e393 with expected crc 2023-10-09T06:05:36.701 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:36 smithi156 ceph-mon[195101]: failed to encode map e393 with expected crc 2023-10-09T06:05:36.701 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:36 smithi156 ceph-mon[195101]: failed to encode map e393 with expected crc 2023-10-09T06:05:36.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:36.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:36.803 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:36.829 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:36.870 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:36.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:36.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:36.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:36 smithi064 ceph-mon[212750]: failed to encode map e393 with expected crc 2023-10-09T06:05:36.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:36 smithi064 ceph-mon[212750]: failed to encode map e393 with expected crc 2023-10-09T06:05:36.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:36 smithi064 ceph-mon[212750]: failed to encode map e393 with expected crc 2023-10-09T06:05:36.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:36 smithi064 ceph-mon[212750]: failed to encode map e393 with expected crc 2023-10-09T06:05:36.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:36 smithi064 ceph-mon[220044]: failed to encode map e393 with expected crc 2023-10-09T06:05:36.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:36 smithi064 ceph-mon[220044]: failed to encode map e393 with expected crc 2023-10-09T06:05:36.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:36 smithi064 ceph-mon[220044]: failed to encode map e393 with expected crc 2023-10-09T06:05:36.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:36 smithi064 ceph-mon[220044]: failed to encode map e393 with expected crc 2023-10-09T06:05:36.976 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:37.021 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:37.074 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:37.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:37.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:37.183 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:37.211 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:37.259 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:37.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:37.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:37.338 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:05:37.364 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:37.392 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:37.429 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:37.441 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:37.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:37.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:37.545 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:37.629 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:37.794 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:37.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:37.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:37 smithi156 ceph-mon[195101]: osdmap e394: 8 total, 8 up, 7 in 2023-10-09T06:05:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:37 smithi156 ceph-mon[195101]: failed to encode map e394 with expected crc 2023-10-09T06:05:37.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:37 smithi156 ceph-mon[195101]: failed to encode map e394 with expected crc 2023-10-09T06:05:37.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:37 smithi156 ceph-mon[195101]: failed to encode map e394 with expected crc 2023-10-09T06:05:37.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:37 smithi156 ceph-mon[195101]: failed to encode map e394 with expected crc 2023-10-09T06:05:37.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:37 smithi156 ceph-mon[195101]: failed to encode map e394 with expected crc 2023-10-09T06:05:37.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:37 smithi156 ceph-mon[195101]: failed to encode map e394 with expected crc 2023-10-09T06:05:37.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:37 smithi156 ceph-mon[195101]: pgmap v966: 119 pgs: 1 active+remapped+backfill_wait, 2 peering, 4 active+remapped+backfilling, 112 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 8076/179496 objects misplaced (4.499%); 103 MiB/s, 1.65k objects/s recovering 2023-10-09T06:05:37.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:37 smithi156 ceph-mon[195101]: osdmap e395: 8 total, 8 up, 7 in 2023-10-09T06:05:37.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:37 smithi156 ceph-mon[195101]: failed to encode map e395 with expected crc 2023-10-09T06:05:37.888 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:37 smithi064 ceph-mon[212750]: osdmap e394: 8 total, 8 up, 7 in 2023-10-09T06:05:37.888 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:37 smithi064 ceph-mon[212750]: failed to encode map e394 with expected crc 2023-10-09T06:05:37.888 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:37 smithi064 ceph-mon[212750]: failed to encode map e394 with expected crc 2023-10-09T06:05:37.888 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:37 smithi064 ceph-mon[212750]: failed to encode map e394 with expected crc 2023-10-09T06:05:37.888 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:37 smithi064 ceph-mon[212750]: failed to encode map e394 with expected crc 2023-10-09T06:05:37.888 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:37 smithi064 ceph-mon[212750]: failed to encode map e394 with expected crc 2023-10-09T06:05:37.888 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:37 smithi064 ceph-mon[212750]: failed to encode map e394 with expected crc 2023-10-09T06:05:37.888 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:37 smithi064 ceph-mon[212750]: pgmap v966: 119 pgs: 1 active+remapped+backfill_wait, 2 peering, 4 active+remapped+backfilling, 112 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 8076/179496 objects misplaced (4.499%); 103 MiB/s, 1.65k objects/s recovering 2023-10-09T06:05:37.888 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:37 smithi064 ceph-mon[212750]: osdmap e395: 8 total, 8 up, 7 in 2023-10-09T06:05:37.889 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:37 smithi064 ceph-mon[212750]: failed to encode map e395 with expected crc 2023-10-09T06:05:37.889 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:37 smithi064 ceph-mon[220044]: osdmap e394: 8 total, 8 up, 7 in 2023-10-09T06:05:37.889 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:37 smithi064 ceph-mon[220044]: failed to encode map e394 with expected crc 2023-10-09T06:05:37.889 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:37 smithi064 ceph-mon[220044]: failed to encode map e394 with expected crc 2023-10-09T06:05:37.889 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:37 smithi064 ceph-mon[220044]: failed to encode map e394 with expected crc 2023-10-09T06:05:37.889 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:37 smithi064 ceph-mon[220044]: failed to encode map e394 with expected crc 2023-10-09T06:05:37.889 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:37 smithi064 ceph-mon[220044]: failed to encode map e394 with expected crc 2023-10-09T06:05:37.889 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:37 smithi064 ceph-mon[220044]: failed to encode map e394 with expected crc 2023-10-09T06:05:37.890 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:37 smithi064 ceph-mon[220044]: pgmap v966: 119 pgs: 1 active+remapped+backfill_wait, 2 peering, 4 active+remapped+backfilling, 112 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 8076/179496 objects misplaced (4.499%); 103 MiB/s, 1.65k objects/s recovering 2023-10-09T06:05:37.890 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:37 smithi064 ceph-mon[220044]: osdmap e395: 8 total, 8 up, 7 in 2023-10-09T06:05:37.890 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:37 smithi064 ceph-mon[220044]: failed to encode map e395 with expected crc 2023-10-09T06:05:37.904 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:37.967 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:38.527 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:05:38.527 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:05:38.527 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:05:38.527 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:05:38.527 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:05:38.557 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:38.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:38.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:38.567 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:05:38.610 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:05:38.647 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:05:38.653 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:05:38.653 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:05:38.653 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:05:38.653 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:05:38.653 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:05:38.653 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:05:38.653 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:05:38.653 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:05:38.666 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:38.738 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:38.802 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:38.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:38.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:38.907 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:38.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:38 smithi064 ceph-mon[212750]: failed to encode map e395 with expected crc 2023-10-09T06:05:38.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:38 smithi064 ceph-mon[212750]: failed to encode map e395 with expected crc 2023-10-09T06:05:38.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:38 smithi064 ceph-mon[212750]: failed to encode map e395 with expected crc 2023-10-09T06:05:38.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:38 smithi064 ceph-mon[212750]: failed to encode map e395 with expected crc 2023-10-09T06:05:38.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:38 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3599044398' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:38.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:38 smithi064 ceph-mon[212750]: osdmap e396: 8 total, 8 up, 7 in 2023-10-09T06:05:38.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:38 smithi064 ceph-mon[212750]: failed to encode map e396 with expected crc 2023-10-09T06:05:38.952 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:38 smithi064 ceph-mon[220044]: failed to encode map e395 with expected crc 2023-10-09T06:05:38.952 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:38 smithi064 ceph-mon[220044]: failed to encode map e395 with expected crc 2023-10-09T06:05:38.952 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:38 smithi064 ceph-mon[220044]: failed to encode map e395 with expected crc 2023-10-09T06:05:38.952 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:38 smithi064 ceph-mon[220044]: failed to encode map e395 with expected crc 2023-10-09T06:05:38.952 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:38 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3599044398' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:38.952 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:38 smithi064 ceph-mon[220044]: osdmap e396: 8 total, 8 up, 7 in 2023-10-09T06:05:38.952 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:38 smithi064 ceph-mon[220044]: failed to encode map e396 with expected crc 2023-10-09T06:05:38.965 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:39.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:38 smithi156 ceph-mon[195101]: failed to encode map e395 with expected crc 2023-10-09T06:05:39.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:38 smithi156 ceph-mon[195101]: failed to encode map e395 with expected crc 2023-10-09T06:05:39.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:38 smithi156 ceph-mon[195101]: failed to encode map e395 with expected crc 2023-10-09T06:05:39.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:38 smithi156 ceph-mon[195101]: failed to encode map e395 with expected crc 2023-10-09T06:05:39.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:38 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3599044398' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:39.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:38 smithi156 ceph-mon[195101]: osdmap e396: 8 total, 8 up, 7 in 2023-10-09T06:05:39.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:38 smithi156 ceph-mon[195101]: failed to encode map e396 with expected crc 2023-10-09T06:05:39.261 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:05:39.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:39 smithi064 ceph-mon[212750]: failed to encode map e396 with expected crc 2023-10-09T06:05:39.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:39 smithi064 ceph-mon[212750]: failed to encode map e396 with expected crc 2023-10-09T06:05:39.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:39 smithi064 ceph-mon[212750]: failed to encode map e396 with expected crc 2023-10-09T06:05:39.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:39 smithi064 ceph-mon[212750]: pgmap v969: 119 pgs: 1 active+remapped+backfill_wait, 2 peering, 4 active+remapped+backfilling, 112 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 383 op/s; 8076/178653 objects misplaced (4.520%); 84 MiB/s, 1.34k objects/s recovering 2023-10-09T06:05:39.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:39 smithi064 ceph-mon[220044]: failed to encode map e396 with expected crc 2023-10-09T06:05:39.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:39 smithi064 ceph-mon[220044]: failed to encode map e396 with expected crc 2023-10-09T06:05:39.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:39 smithi064 ceph-mon[220044]: failed to encode map e396 with expected crc 2023-10-09T06:05:39.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:39 smithi064 ceph-mon[220044]: pgmap v969: 119 pgs: 1 active+remapped+backfill_wait, 2 peering, 4 active+remapped+backfilling, 112 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 383 op/s; 8076/178653 objects misplaced (4.520%); 84 MiB/s, 1.34k objects/s recovering 2023-10-09T06:05:40.010 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:40.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:40.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:40.117 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:40.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:39 smithi156 ceph-mon[195101]: failed to encode map e396 with expected crc 2023-10-09T06:05:40.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:39 smithi156 ceph-mon[195101]: failed to encode map e396 with expected crc 2023-10-09T06:05:40.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:39 smithi156 ceph-mon[195101]: failed to encode map e396 with expected crc 2023-10-09T06:05:40.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:39 smithi156 ceph-mon[195101]: pgmap v969: 119 pgs: 1 active+remapped+backfill_wait, 2 peering, 4 active+remapped+backfilling, 112 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 383 op/s; 8076/178653 objects misplaced (4.520%); 84 MiB/s, 1.34k objects/s recovering 2023-10-09T06:05:40.169 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:40.182 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:40.211 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:40.254 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:40.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:40.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:40.366 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:40.422 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:40.817 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:40.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:40.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:40.925 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:41.043 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:41.096 INFO:teuthology.orchestra.run.smithi064.stdout:moved 0 / 357 (0%) 2023-10-09T06:05:41.096 INFO:teuthology.orchestra.run.smithi064.stdout:avg 51 2023-10-09T06:05:41.096 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 3.42261 -> 3.42261 (expected baseline 6.61168) 2023-10-09T06:05:41.097 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.5 with 46 -> 46 pgs (0.901961 -> 0.901961 * mean) 2023-10-09T06:05:41.097 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.1 with 57 -> 57 pgs (1.11765 -> 1.11765 * mean) 2023-10-09T06:05:41.097 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:05:41.097 INFO:teuthology.orchestra.run.smithi064.stdout:oload 1000 2023-10-09T06:05:41.097 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 0.05 2023-10-09T06:05:41.097 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-10-09T06:05:41.097 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0220 2023-10-09T06:05:41.098 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.2204 2023-10-09T06:05:41.098 INFO:teuthology.orchestra.run.smithi064.stderr:no change 2023-10-09T06:05:42.086 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:05:42.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:41 smithi064 ceph-mon[212750]: pgmap v970: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 6009/171492 objects misplaced (3.504%); 52 MiB/s, 830 objects/s recovering 2023-10-09T06:05:42.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:41 smithi064 ceph-mon[212750]: from='client.35790 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:05:42.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:41 smithi064 ceph-mon[220044]: pgmap v970: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 6009/171492 objects misplaced (3.504%); 52 MiB/s, 830 objects/s recovering 2023-10-09T06:05:42.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:41 smithi064 ceph-mon[220044]: from='client.35790 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:05:42.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:41 smithi156 ceph-mon[195101]: pgmap v970: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 6009/171492 objects misplaced (3.504%); 52 MiB/s, 830 objects/s recovering 2023-10-09T06:05:42.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:41 smithi156 ceph-mon[195101]: from='client.35790 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:05:42.656 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:42.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:42.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:42.761 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:42.836 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:42.924 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:42.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:42.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:43.034 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:43.077 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:43.121 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:43.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:43.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:43.227 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:43.292 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:43.402 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:43.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:43.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:43.467 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T06:05:43.467 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-10-09T06:05:43.509 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:43.588 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:43.793 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:43.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:43.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:43.892 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:43.916 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:43.994 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:44.103 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:44.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:44.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:44.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:44.289 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:44.506 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:44.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:44.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:44.614 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:44.653 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:44.670 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:05:44.670 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:05:44.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:44 smithi064 ceph-mon[220044]: pgmap v971: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 458 op/s; 4539/170889 objects misplaced (2.656%); 32 MiB/s, 514 objects/s recovering 2023-10-09T06:05:44.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:44.709 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:44 smithi064 ceph-mon[212750]: pgmap v971: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 458 op/s; 4539/170889 objects misplaced (2.656%); 32 MiB/s, 514 objects/s recovering 2023-10-09T06:05:44.710 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:44.736 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:44.745 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:44 smithi156 ceph-mon[195101]: pgmap v971: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 458 op/s; 4539/170889 objects misplaced (2.656%); 32 MiB/s, 514 objects/s recovering 2023-10-09T06:05:44.745 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:44.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:44.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:44.847 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:44.907 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:45.014 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:45.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:45.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:45.126 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:45.170 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:45.282 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:45.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:45.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:45.391 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:05:45.448 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:45.486 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:05:45.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:45 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3352044284' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:05:45.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:45 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3352044284' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:05:45.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:45 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3352044284' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:05:45.720 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:05:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:05:45] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.43.0" 2023-10-09T06:05:46.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[220044]: pgmap v972: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 4539/165948 objects misplaced (2.735%); 25 MiB/s, 402 objects/s recovering 2023-10-09T06:05:46.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:05:46.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T06:05:46.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[220044]: failed to encode map e397 with expected crc 2023-10-09T06:05:46.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[220044]: failed to encode map e397 with expected crc 2023-10-09T06:05:46.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[220044]: failed to encode map e397 with expected crc 2023-10-09T06:05:46.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[220044]: failed to encode map e397 with expected crc 2023-10-09T06:05:46.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[220044]: failed to encode map e397 with expected crc 2023-10-09T06:05:46.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[220044]: failed to encode map e397 with expected crc 2023-10-09T06:05:46.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3352044284' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:05:46.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[220044]: osdmap e397: 8 total, 8 up, 7 in 2023-10-09T06:05:46.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[220044]: failed to encode map e397 with expected crc 2023-10-09T06:05:46.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[220044]: failed to encode map e397 with expected crc 2023-10-09T06:05:46.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:05:46.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[212750]: pgmap v972: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 4539/165948 objects misplaced (2.735%); 25 MiB/s, 402 objects/s recovering 2023-10-09T06:05:46.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:05:46.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:05:46.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[212750]: failed to encode map e397 with expected crc 2023-10-09T06:05:46.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[212750]: failed to encode map e397 with expected crc 2023-10-09T06:05:46.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[212750]: failed to encode map e397 with expected crc 2023-10-09T06:05:46.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[212750]: failed to encode map e397 with expected crc 2023-10-09T06:05:46.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[212750]: failed to encode map e397 with expected crc 2023-10-09T06:05:46.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[212750]: failed to encode map e397 with expected crc 2023-10-09T06:05:46.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3352044284' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:05:46.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[212750]: osdmap e397: 8 total, 8 up, 7 in 2023-10-09T06:05:46.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[212750]: failed to encode map e397 with expected crc 2023-10-09T06:05:46.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[212750]: failed to encode map e397 with expected crc 2023-10-09T06:05:46.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:46 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:05:46.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:46 smithi156 ceph-mon[195101]: pgmap v972: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 4539/165948 objects misplaced (2.735%); 25 MiB/s, 402 objects/s recovering 2023-10-09T06:05:46.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:46 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:05:46.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:46 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T06:05:46.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:46 smithi156 ceph-mon[195101]: failed to encode map e397 with expected crc 2023-10-09T06:05:46.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:46 smithi156 ceph-mon[195101]: failed to encode map e397 with expected crc 2023-10-09T06:05:46.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:46 smithi156 ceph-mon[195101]: failed to encode map e397 with expected crc 2023-10-09T06:05:46.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:46 smithi156 ceph-mon[195101]: failed to encode map e397 with expected crc 2023-10-09T06:05:46.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:46 smithi156 ceph-mon[195101]: failed to encode map e397 with expected crc 2023-10-09T06:05:46.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:46 smithi156 ceph-mon[195101]: failed to encode map e397 with expected crc 2023-10-09T06:05:46.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:46 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3352044284' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:05:46.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:46 smithi156 ceph-mon[195101]: osdmap e397: 8 total, 8 up, 7 in 2023-10-09T06:05:46.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:46 smithi156 ceph-mon[195101]: failed to encode map e397 with expected crc 2023-10-09T06:05:46.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:46 smithi156 ceph-mon[195101]: failed to encode map e397 with expected crc 2023-10-09T06:05:46.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:46 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:05:47.111 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:05:47.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:47.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:05:47.152 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:05:47.217 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:47.280 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:47.361 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:47.374 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:47.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:47.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:47.481 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:47.600 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:47.848 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:47.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:47.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:47.931 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:05:47.963 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:47.975 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:05:48.018 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:05:48.043 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:48.064 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:05:48.100 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:05:48.138 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:05:48.175 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:05:48.211 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:05:48.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:47 smithi064 ceph-mon[212750]: pgmap v974: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 855 op/s; 2971/157818 objects misplaced (1.883%); 33 MiB/s, 534 objects/s recovering 2023-10-09T06:05:48.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:47 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:05:48.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:47 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:05:48.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:47 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:05:48.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:47 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:05:48.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:47 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2138003601' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:48.226 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:47 smithi064 ceph-mon[220044]: pgmap v974: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 855 op/s; 2971/157818 objects misplaced (1.883%); 33 MiB/s, 534 objects/s recovering 2023-10-09T06:05:48.227 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:47 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:05:48.227 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:47 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:05:48.227 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:47 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:05:48.227 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:47 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:05:48.227 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:47 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2138003601' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:48.231 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:05:48.231 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:05:48.231 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:05:48.231 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:05:48.231 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:05:48.231 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:05:48.231 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:05:48.232 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:05:48.313 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:48.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:48.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:48.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:47 smithi156 ceph-mon[195101]: pgmap v974: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 855 op/s; 2971/157818 objects misplaced (1.883%); 33 MiB/s, 534 objects/s recovering 2023-10-09T06:05:48.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:47 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:05:48.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:47 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:05:48.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:47 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:05:48.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:47 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:05:48.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:47 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2138003601' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:48.430 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:05:48.492 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:49.322 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:05:49.392 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:05:49.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:49.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:05:49.495 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:49.570 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:49.678 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:49.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:49.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:49.786 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:49.833 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:49.911 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:50.057 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:50.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:50.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:50.163 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:50.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:49 smithi064 ceph-mon[212750]: pgmap v975: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 741 op/s; 2971/156393 objects misplaced (1.900%); 28 MiB/s, 451 objects/s recovering 2023-10-09T06:05:50.225 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:49 smithi064 ceph-mon[220044]: pgmap v975: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 741 op/s; 2971/156393 objects misplaced (1.900%); 28 MiB/s, 451 objects/s recovering 2023-10-09T06:05:50.287 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:50.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:49 smithi156 ceph-mon[195101]: pgmap v975: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 741 op/s; 2971/156393 objects misplaced (1.900%); 28 MiB/s, 451 objects/s recovering 2023-10-09T06:05:50.416 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:05:50.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:50.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:05:50.529 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:50.637 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:50.688 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:50.721 INFO:teuthology.orchestra.run.smithi064.stdout:moved 8 / 357 (2.2409%) 2023-10-09T06:05:50.721 INFO:teuthology.orchestra.run.smithi064.stdout:avg 51 2023-10-09T06:05:50.721 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 3.42261 -> 5.23723 (expected baseline 6.61168) 2023-10-09T06:05:50.721 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.5 with 46 -> 44 pgs (0.901961 -> 0.862745 * mean) 2023-10-09T06:05:50.721 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.1 with 57 -> 59 pgs (1.11765 -> 1.15686 * mean) 2023-10-09T06:05:50.721 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:05:50.722 INFO:teuthology.orchestra.run.smithi064.stdout:oload 110 2023-10-09T06:05:50.722 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 0.05 2023-10-09T06:05:50.722 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-10-09T06:05:50.722 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0207 2023-10-09T06:05:50.722 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.0228 2023-10-09T06:05:50.722 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 weight 0.8265 -> 0.7765 2023-10-09T06:05:50.722 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 weight 0.8861 -> 0.8362 2023-10-09T06:05:50.754 INFO:teuthology.orchestra.run.smithi064.stderr:no change 2023-10-09T06:05:50.922 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:05:50 smithi064 conmon[198859]: 2023-10-09T06:05:50.720+0000 7fcb872b4700 -1 mgr.server reply reply (2) No such file or directory no change 2023-10-09T06:05:51.886 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:05:51.887 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:51.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:51.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:51.991 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:52.022 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:52.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:52 smithi064 ceph-mon[212750]: from='client.45782 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:05:52.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:52 smithi064 ceph-mon[212750]: pgmap v976: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 801 op/s; 2298/147450 objects misplaced (1.558%); 20 MiB/s, 321 objects/s recovering 2023-10-09T06:05:52.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:52 smithi064 ceph-mon[220044]: from='client.45782 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:05:52.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:52 smithi064 ceph-mon[220044]: pgmap v976: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 801 op/s; 2298/147450 objects misplaced (1.558%); 20 MiB/s, 321 objects/s recovering 2023-10-09T06:05:52.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:52 smithi156 ceph-mon[195101]: from='client.45782 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:05:52.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:52 smithi156 ceph-mon[195101]: pgmap v976: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 801 op/s; 2298/147450 objects misplaced (1.558%); 20 MiB/s, 321 objects/s recovering 2023-10-09T06:05:52.427 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:52.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:52.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:52.437 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T06:05:52.437 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 1.0 2023-10-09T06:05:52.536 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:52.579 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:52.634 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:52.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:52.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:52.746 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:05:52.790 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:52.845 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:05:52.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:52.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:05:52.950 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:52.976 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:53.021 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:53.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:53.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:53.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:53 smithi156 ceph-mon[195101]: 1.14 scrub starts 2023-10-09T06:05:53.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:53 smithi156 ceph-mon[195101]: failed to encode map e398 with expected crc 2023-10-09T06:05:53.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:53 smithi156 ceph-mon[195101]: osdmap e398: 8 total, 8 up, 7 in 2023-10-09T06:05:53.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:53 smithi156 ceph-mon[195101]: failed to encode map e398 with expected crc 2023-10-09T06:05:53.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:53 smithi156 ceph-mon[195101]: osdmap e399: 8 total, 8 up, 7 in 2023-10-09T06:05:53.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:53 smithi156 ceph-mon[195101]: failed to encode map e399 with expected crc 2023-10-09T06:05:53.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:53 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:05:53.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:53 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:05:53.131 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:53.166 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:53.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:53 smithi064 ceph-mon[212750]: 1.14 scrub starts 2023-10-09T06:05:53.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:53 smithi064 ceph-mon[212750]: failed to encode map e398 with expected crc 2023-10-09T06:05:53.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:53 smithi064 ceph-mon[212750]: osdmap e398: 8 total, 8 up, 7 in 2023-10-09T06:05:53.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:53 smithi064 ceph-mon[212750]: failed to encode map e398 with expected crc 2023-10-09T06:05:53.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:53 smithi064 ceph-mon[212750]: osdmap e399: 8 total, 8 up, 7 in 2023-10-09T06:05:53.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:53 smithi064 ceph-mon[212750]: failed to encode map e399 with expected crc 2023-10-09T06:05:53.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:53 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:05:53.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:53 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:05:53.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:53 smithi064 ceph-mon[220044]: 1.14 scrub starts 2023-10-09T06:05:53.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:53 smithi064 ceph-mon[220044]: failed to encode map e398 with expected crc 2023-10-09T06:05:53.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:53 smithi064 ceph-mon[220044]: osdmap e398: 8 total, 8 up, 7 in 2023-10-09T06:05:53.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:53 smithi064 ceph-mon[220044]: failed to encode map e398 with expected crc 2023-10-09T06:05:53.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:53 smithi064 ceph-mon[220044]: osdmap e399: 8 total, 8 up, 7 in 2023-10-09T06:05:53.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:53 smithi064 ceph-mon[220044]: failed to encode map e399 with expected crc 2023-10-09T06:05:53.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:53 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:05:53.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:53 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:05:53.885 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:53.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:53.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:53.982 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:53.992 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:05:54.094 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:54.225 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:54 smithi064 ceph-mon[220044]: failed to encode map e399 with expected crc 2023-10-09T06:05:54.225 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:54 smithi064 ceph-mon[220044]: failed to encode map e399 with expected crc 2023-10-09T06:05:54.225 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:54 smithi064 ceph-mon[220044]: pgmap v979: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 602 op/s; 1689/146712 objects misplaced (1.151%); 16 MiB/s, 250 objects/s recovering 2023-10-09T06:05:54.225 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:54 smithi064 ceph-mon[220044]: osdmap e400: 8 total, 8 up, 7 in 2023-10-09T06:05:54.225 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:54 smithi064 ceph-mon[220044]: failed to encode map e400 with expected crc 2023-10-09T06:05:54.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:54 smithi064 ceph-mon[212750]: failed to encode map e399 with expected crc 2023-10-09T06:05:54.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:54 smithi064 ceph-mon[212750]: failed to encode map e399 with expected crc 2023-10-09T06:05:54.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:54 smithi064 ceph-mon[212750]: pgmap v979: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 602 op/s; 1689/146712 objects misplaced (1.151%); 16 MiB/s, 250 objects/s recovering 2023-10-09T06:05:54.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:54 smithi064 ceph-mon[212750]: osdmap e400: 8 total, 8 up, 7 in 2023-10-09T06:05:54.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:54 smithi064 ceph-mon[212750]: failed to encode map e400 with expected crc 2023-10-09T06:05:54.355 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:05:54.355 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T06:05:54.356 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T06:05:54.356 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T06:05:54.356 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:05:54.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:54 smithi156 ceph-mon[195101]: failed to encode map e399 with expected crc 2023-10-09T06:05:54.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:54 smithi156 ceph-mon[195101]: failed to encode map e399 with expected crc 2023-10-09T06:05:54.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:54 smithi156 ceph-mon[195101]: pgmap v979: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 602 op/s; 1689/146712 objects misplaced (1.151%); 16 MiB/s, 250 objects/s recovering 2023-10-09T06:05:54.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:54 smithi156 ceph-mon[195101]: osdmap e400: 8 total, 8 up, 7 in 2023-10-09T06:05:54.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:54 smithi156 ceph-mon[195101]: failed to encode map e400 with expected crc 2023-10-09T06:05:54.401 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:54.442 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:05:54.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:54.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:05:54.564 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:54.628 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:54.697 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:54.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:54.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:54.803 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:54.881 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:55.031 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:55 smithi156 ceph-mon[195101]: failed to encode map e400 with expected crc 2023-10-09T06:05:55.031 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:55 smithi156 ceph-mon[195101]: failed to encode map e400 with expected crc 2023-10-09T06:05:55.031 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:55 smithi156 ceph-mon[195101]: failed to encode map e400 with expected crc 2023-10-09T06:05:55.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:55 smithi064 ceph-mon[220044]: failed to encode map e400 with expected crc 2023-10-09T06:05:55.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:55 smithi064 ceph-mon[220044]: failed to encode map e400 with expected crc 2023-10-09T06:05:55.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:55 smithi064 ceph-mon[220044]: failed to encode map e400 with expected crc 2023-10-09T06:05:55.308 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:55 smithi064 ceph-mon[212750]: failed to encode map e400 with expected crc 2023-10-09T06:05:55.308 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:55 smithi064 ceph-mon[212750]: failed to encode map e400 with expected crc 2023-10-09T06:05:55.308 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:55 smithi064 ceph-mon[212750]: failed to encode map e400 with expected crc 2023-10-09T06:05:55.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:05:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:05:55] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.43.0" 2023-10-09T06:05:56.061 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:05:56.147 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:56.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:56.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:56.256 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:05:56.320 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:56 smithi156 ceph-mon[195101]: pgmap v981: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 750 op/s; 1689/142884 objects misplaced (1.182%); 12 MiB/s, 188 objects/s recovering 2023-10-09T06:05:56.324 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:56.385 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:56 smithi064 ceph-mon[220044]: pgmap v981: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 750 op/s; 1689/142884 objects misplaced (1.182%); 12 MiB/s, 188 objects/s recovering 2023-10-09T06:05:56.386 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:56 smithi064 ceph-mon[212750]: pgmap v981: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 750 op/s; 1689/142884 objects misplaced (1.182%); 12 MiB/s, 188 objects/s recovering 2023-10-09T06:05:56.399 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:05:56.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:56.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:05:56.503 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:56.529 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:56.598 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:56.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:56.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:56.707 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:56.759 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:56.875 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:56.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:56.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:56.988 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:05:57.029 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:57.047 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:05:57.085 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:05:57.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:57.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:05:57.192 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:57.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:57 smithi064 ceph-mon[220044]: 1.14 scrub starts 2023-10-09T06:05:57.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:57 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/56480930' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:05:57.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:57 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:05:57.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:57 smithi064 ceph-mon[212750]: 1.14 scrub starts 2023-10-09T06:05:57.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:57 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/56480930' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:05:57.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:57 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:05:57.282 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:57.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:57 smithi156 ceph-mon[195101]: 1.14 scrub starts 2023-10-09T06:05:57.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:57 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/56480930' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:05:57.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:57 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:05:57.692 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:57.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:57.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:57.802 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:57.841 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:58.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:58 smithi156 ceph-mon[195101]: pgmap v982: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 627/136341 objects misplaced (0.460%); 16 MiB/s, 261 objects/s recovering 2023-10-09T06:05:58.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:58 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:05:58.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:58 smithi156 ceph-mon[195101]: failed to encode map e401 with expected crc 2023-10-09T06:05:58.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:58 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:05:58.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:58 smithi156 ceph-mon[195101]: osdmap e401: 8 total, 8 up, 7 in 2023-10-09T06:05:58.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:58 smithi156 ceph-mon[195101]: failed to encode map e401 with expected crc 2023-10-09T06:05:58.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:58 smithi156 ceph-mon[195101]: failed to encode map e401 with expected crc 2023-10-09T06:05:58.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:58 smithi156 ceph-mon[195101]: failed to encode map e401 with expected crc 2023-10-09T06:05:58.441 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:58.446 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:58 smithi064 ceph-mon[212750]: pgmap v982: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 627/136341 objects misplaced (0.460%); 16 MiB/s, 261 objects/s recovering 2023-10-09T06:05:58.446 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:58 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:05:58.446 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:58 smithi064 ceph-mon[212750]: failed to encode map e401 with expected crc 2023-10-09T06:05:58.446 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:58 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:05:58.446 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:58 smithi064 ceph-mon[212750]: osdmap e401: 8 total, 8 up, 7 in 2023-10-09T06:05:58.446 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:58 smithi064 ceph-mon[212750]: failed to encode map e401 with expected crc 2023-10-09T06:05:58.446 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:58 smithi064 ceph-mon[212750]: failed to encode map e401 with expected crc 2023-10-09T06:05:58.446 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:58 smithi064 ceph-mon[212750]: failed to encode map e401 with expected crc 2023-10-09T06:05:58.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:58.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:58.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:58 smithi064 ceph-mon[220044]: pgmap v982: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 627/136341 objects misplaced (0.460%); 16 MiB/s, 261 objects/s recovering 2023-10-09T06:05:58.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:58 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:05:58.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:58 smithi064 ceph-mon[220044]: failed to encode map e401 with expected crc 2023-10-09T06:05:58.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:58 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:05:58.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:58 smithi064 ceph-mon[220044]: osdmap e401: 8 total, 8 up, 7 in 2023-10-09T06:05:58.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:58 smithi064 ceph-mon[220044]: failed to encode map e401 with expected crc 2023-10-09T06:05:58.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:58 smithi064 ceph-mon[220044]: failed to encode map e401 with expected crc 2023-10-09T06:05:58.450 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:58 smithi064 ceph-mon[220044]: failed to encode map e401 with expected crc 2023-10-09T06:05:58.548 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:58.685 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:58.770 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:58.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:58.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:58.877 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:05:58.908 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:58.936 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:05:58.974 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:05:58.979 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:05:58.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:58.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:05:59.018 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:05:59.050 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:59.053 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:05:59.081 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:05:59.097 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:05:59.139 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:05:59.168 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:05:59.174 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:05:59.230 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:05:59.272 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:05:59.273 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:05:59.273 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:05:59.273 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:05:59.273 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:05:59.273 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:05:59.273 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:05:59.273 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:05:59.298 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:05:59.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:05:59.310 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:59.310 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:05:59 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1901534131' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:59.310 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:59.310 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:05:59 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1901534131' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:59.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:05:59.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:59.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:05:59 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1901534131' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:05:59.411 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:05:59.483 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:00.064 INFO:teuthology.orchestra.run.smithi064.stdout:moved 18 / 357 (5.04202%) 2023-10-09T06:06:00.064 INFO:teuthology.orchestra.run.smithi064.stdout:avg 51 2023-10-09T06:06:00.064 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 3.42261 -> 7.70899 (expected baseline 6.61168) 2023-10-09T06:06:00.064 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.5 with 46 -> 40 pgs (0.901961 -> 0.784314 * mean) 2023-10-09T06:06:00.064 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.1 with 57 -> 59 pgs (1.11765 -> 1.15686 * mean) 2023-10-09T06:06:00.064 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:06:00.064 INFO:teuthology.orchestra.run.smithi064.stdout:oload 110 2023-10-09T06:06:00.064 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 1 2023-10-09T06:06:00.064 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-10-09T06:06:00.065 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0188 2023-10-09T06:06:00.065 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.0207 2023-10-09T06:06:00.065 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 weight 0.8265 -> 0.6914 2023-10-09T06:06:00.065 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 weight 0.8861 -> 0.7847 2023-10-09T06:06:00.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:00 smithi156 ceph-mon[195101]: pgmap v984: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 627/134574 objects misplaced (0.466%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T06:06:00.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:00 smithi156 ceph-mon[195101]: from='client.35808 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:00.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:00 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'5':'51425','6':'45314'}"}]: dispatch 2023-10-09T06:06:00.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:00 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'5':'51425','6':'45314'}"}]: dispatch 2023-10-09T06:06:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:00 smithi064 ceph-mon[212750]: pgmap v984: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 627/134574 objects misplaced (0.466%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T06:06:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:00 smithi064 ceph-mon[212750]: from='client.35808 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:00 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'5':'51425','6':'45314'}"}]: dispatch 2023-10-09T06:06:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:00 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'5':'51425','6':'45314'}"}]: dispatch 2023-10-09T06:06:00.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:00 smithi064 ceph-mon[220044]: pgmap v984: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 627/134574 objects misplaced (0.466%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T06:06:00.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:00 smithi064 ceph-mon[220044]: from='client.35808 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:00.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:00 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'5':'51425','6':'45314'}"}]: dispatch 2023-10-09T06:06:00.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:00 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'5':'51425','6':'45314'}"}]: dispatch 2023-10-09T06:06:00.531 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:00.532 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:06:00.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:00.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:00.636 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:00.665 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:00.708 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:00.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:00.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:00.813 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:00.855 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:01.082 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:01 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'5':'51425','6':'45314'}"}]': finished 2023-10-09T06:06:01.082 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:01 smithi156 ceph-mon[195101]: osdmap e402: 8 total, 8 up, 7 in 2023-10-09T06:06:01.083 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:01 smithi156 ceph-mon[195101]: failed to encode map e402 with expected crc 2023-10-09T06:06:01.083 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:01 smithi156 ceph-mon[195101]: failed to encode map e402 with expected crc 2023-10-09T06:06:01.083 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:01 smithi156 ceph-mon[195101]: failed to encode map e402 with expected crc 2023-10-09T06:06:01.083 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:01 smithi156 ceph-mon[195101]: failed to encode map e402 with expected crc 2023-10-09T06:06:01.083 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:01 smithi156 ceph-mon[195101]: osdmap e403: 8 total, 8 up, 7 in 2023-10-09T06:06:01.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:01 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'5':'51425','6':'45314'}"}]': finished 2023-10-09T06:06:01.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:01 smithi064 ceph-mon[212750]: osdmap e402: 8 total, 8 up, 7 in 2023-10-09T06:06:01.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:01 smithi064 ceph-mon[212750]: failed to encode map e402 with expected crc 2023-10-09T06:06:01.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:01 smithi064 ceph-mon[212750]: failed to encode map e402 with expected crc 2023-10-09T06:06:01.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:01 smithi064 ceph-mon[212750]: failed to encode map e402 with expected crc 2023-10-09T06:06:01.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:01 smithi064 ceph-mon[212750]: failed to encode map e402 with expected crc 2023-10-09T06:06:01.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:01 smithi064 ceph-mon[212750]: osdmap e403: 8 total, 8 up, 7 in 2023-10-09T06:06:01.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:01 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'5':'51425','6':'45314'}"}]': finished 2023-10-09T06:06:01.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:01 smithi064 ceph-mon[220044]: osdmap e402: 8 total, 8 up, 7 in 2023-10-09T06:06:01.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:01 smithi064 ceph-mon[220044]: failed to encode map e402 with expected crc 2023-10-09T06:06:01.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:01 smithi064 ceph-mon[220044]: failed to encode map e402 with expected crc 2023-10-09T06:06:01.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:01 smithi064 ceph-mon[220044]: failed to encode map e402 with expected crc 2023-10-09T06:06:01.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:01 smithi064 ceph-mon[220044]: failed to encode map e402 with expected crc 2023-10-09T06:06:01.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:01 smithi064 ceph-mon[220044]: osdmap e403: 8 total, 8 up, 7 in 2023-10-09T06:06:02.059 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:02.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:02.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:02.129 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T06:06:02.129 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-10-09T06:06:02.167 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:02.196 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:02.290 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:02.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:02.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:02.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:02 smithi156 ceph-mon[195101]: pgmap v986: 119 pgs: 1 remapped+peering, 8 peering, 110 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s; 13 MiB/s, 200 objects/s recovering 2023-10-09T06:06:02.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:02 smithi156 ceph-mon[195101]: 8.a continuing backfill to osd.3 from (374'5477,401'10770] 8:5f44e167:::benchmark_data_smithi064_457003_object91091:head to 401'10770 2023-10-09T06:06:02.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:02 smithi156 ceph-mon[195101]: 8.a starting backfill to osd.4 from (0'0,0'0] MAX to 401'10770 2023-10-09T06:06:02.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:02 smithi156 ceph-mon[195101]: failed to encode map e403 with expected crc 2023-10-09T06:06:02.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:02 smithi156 ceph-mon[195101]: failed to encode map e403 with expected crc 2023-10-09T06:06:02.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:02 smithi156 ceph-mon[195101]: failed to encode map e403 with expected crc 2023-10-09T06:06:02.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:02 smithi156 ceph-mon[195101]: failed to encode map e403 with expected crc 2023-10-09T06:06:02.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:02 smithi156 ceph-mon[195101]: failed to encode map e403 with expected crc 2023-10-09T06:06:02.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:02 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T06:06:02.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:02 smithi156 ceph-mon[195101]: osdmap e404: 8 total, 8 up, 7 in 2023-10-09T06:06:02.384 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:02.398 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:02.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:02 smithi064 ceph-mon[212750]: pgmap v986: 119 pgs: 1 remapped+peering, 8 peering, 110 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s; 13 MiB/s, 200 objects/s recovering 2023-10-09T06:06:02.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:02 smithi064 ceph-mon[212750]: 8.a continuing backfill to osd.3 from (374'5477,401'10770] 8:5f44e167:::benchmark_data_smithi064_457003_object91091:head to 401'10770 2023-10-09T06:06:02.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:02 smithi064 ceph-mon[212750]: 8.a starting backfill to osd.4 from (0'0,0'0] MAX to 401'10770 2023-10-09T06:06:02.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:02 smithi064 ceph-mon[212750]: failed to encode map e403 with expected crc 2023-10-09T06:06:02.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:02 smithi064 ceph-mon[212750]: failed to encode map e403 with expected crc 2023-10-09T06:06:02.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:02 smithi064 ceph-mon[212750]: failed to encode map e403 with expected crc 2023-10-09T06:06:02.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:02 smithi064 ceph-mon[212750]: failed to encode map e403 with expected crc 2023-10-09T06:06:02.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:02 smithi064 ceph-mon[212750]: failed to encode map e403 with expected crc 2023-10-09T06:06:02.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:02 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T06:06:02.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:02 smithi064 ceph-mon[212750]: osdmap e404: 8 total, 8 up, 7 in 2023-10-09T06:06:02.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:02 smithi064 ceph-mon[220044]: pgmap v986: 119 pgs: 1 remapped+peering, 8 peering, 110 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s; 13 MiB/s, 200 objects/s recovering 2023-10-09T06:06:02.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:02 smithi064 ceph-mon[220044]: 8.a continuing backfill to osd.3 from (374'5477,401'10770] 8:5f44e167:::benchmark_data_smithi064_457003_object91091:head to 401'10770 2023-10-09T06:06:02.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:02 smithi064 ceph-mon[220044]: 8.a starting backfill to osd.4 from (0'0,0'0] MAX to 401'10770 2023-10-09T06:06:02.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:02 smithi064 ceph-mon[220044]: failed to encode map e403 with expected crc 2023-10-09T06:06:02.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:02 smithi064 ceph-mon[220044]: failed to encode map e403 with expected crc 2023-10-09T06:06:02.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:02 smithi064 ceph-mon[220044]: failed to encode map e403 with expected crc 2023-10-09T06:06:02.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:02 smithi064 ceph-mon[220044]: failed to encode map e403 with expected crc 2023-10-09T06:06:02.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:02 smithi064 ceph-mon[220044]: failed to encode map e403 with expected crc 2023-10-09T06:06:02.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:02 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T06:06:02.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:02 smithi064 ceph-mon[220044]: osdmap e404: 8 total, 8 up, 7 in 2023-10-09T06:06:02.478 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:02.528 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:06:02.590 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:02.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:02.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:02.700 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:02.741 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:02.813 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:02.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:02.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:02.920 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:02.954 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:03.016 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:03.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:03.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:03.058 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:06:03.058 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:06:03.124 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:03.234 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:03.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:03 smithi156 ceph-mon[195101]: failed to encode map e404 with expected crc 2023-10-09T06:06:03.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:03 smithi156 ceph-mon[195101]: failed to encode map e404 with expected crc 2023-10-09T06:06:03.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:03 smithi156 ceph-mon[195101]: failed to encode map e404 with expected crc 2023-10-09T06:06:03.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:03 smithi156 ceph-mon[195101]: failed to encode map e404 with expected crc 2023-10-09T06:06:03.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:03 smithi156 ceph-mon[195101]: 8.9 starting backfill to osd.3 from (0'0,0'0] MAX to 401'10443 2023-10-09T06:06:03.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:03 smithi156 ceph-mon[195101]: 8.5 starting backfill to osd.0 from (0'0,0'0] MAX to 401'10508 2023-10-09T06:06:03.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:03 smithi156 ceph-mon[195101]: osdmap e405: 8 total, 8 up, 7 in 2023-10-09T06:06:03.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:03 smithi064 ceph-mon[220044]: failed to encode map e404 with expected crc 2023-10-09T06:06:03.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:03 smithi064 ceph-mon[220044]: failed to encode map e404 with expected crc 2023-10-09T06:06:03.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:03 smithi064 ceph-mon[220044]: failed to encode map e404 with expected crc 2023-10-09T06:06:03.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:03 smithi064 ceph-mon[220044]: failed to encode map e404 with expected crc 2023-10-09T06:06:03.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:03 smithi064 ceph-mon[220044]: 8.9 starting backfill to osd.3 from (0'0,0'0] MAX to 401'10443 2023-10-09T06:06:03.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:03 smithi064 ceph-mon[220044]: 8.5 starting backfill to osd.0 from (0'0,0'0] MAX to 401'10508 2023-10-09T06:06:03.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:03 smithi064 ceph-mon[220044]: osdmap e405: 8 total, 8 up, 7 in 2023-10-09T06:06:03.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:03 smithi064 ceph-mon[212750]: failed to encode map e404 with expected crc 2023-10-09T06:06:03.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:03 smithi064 ceph-mon[212750]: failed to encode map e404 with expected crc 2023-10-09T06:06:03.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:03 smithi064 ceph-mon[212750]: failed to encode map e404 with expected crc 2023-10-09T06:06:03.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:03 smithi064 ceph-mon[212750]: failed to encode map e404 with expected crc 2023-10-09T06:06:03.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:03 smithi064 ceph-mon[212750]: 8.9 starting backfill to osd.3 from (0'0,0'0] MAX to 401'10443 2023-10-09T06:06:03.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:03 smithi064 ceph-mon[212750]: 8.5 starting backfill to osd.0 from (0'0,0'0] MAX to 401'10508 2023-10-09T06:06:03.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:03 smithi064 ceph-mon[212750]: osdmap e405: 8 total, 8 up, 7 in 2023-10-09T06:06:04.255 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:04.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:04.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:04.369 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:04 smithi156 ceph-mon[195101]: failed to encode map e405 with expected crc 2023-10-09T06:06:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:04 smithi156 ceph-mon[195101]: failed to encode map e405 with expected crc 2023-10-09T06:06:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:04 smithi156 ceph-mon[195101]: pgmap v990: 119 pgs: 1 remapped+peering, 12 peering, 106 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 913 op/s; 7.8 MiB/s, 124 objects/s recovering 2023-10-09T06:06:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:04 smithi156 ceph-mon[195101]: osdmap e406: 8 total, 8 up, 7 in 2023-10-09T06:06:04.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:04 smithi156 ceph-mon[195101]: failed to encode map e406 with expected crc 2023-10-09T06:06:04.394 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:04.434 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:04.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:04.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:04.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:04 smithi064 ceph-mon[212750]: failed to encode map e405 with expected crc 2023-10-09T06:06:04.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:04 smithi064 ceph-mon[212750]: failed to encode map e405 with expected crc 2023-10-09T06:06:04.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:04 smithi064 ceph-mon[212750]: pgmap v990: 119 pgs: 1 remapped+peering, 12 peering, 106 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 913 op/s; 7.8 MiB/s, 124 objects/s recovering 2023-10-09T06:06:04.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:04 smithi064 ceph-mon[212750]: osdmap e406: 8 total, 8 up, 7 in 2023-10-09T06:06:04.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:04 smithi064 ceph-mon[212750]: failed to encode map e406 with expected crc 2023-10-09T06:06:04.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:04 smithi064 ceph-mon[220044]: failed to encode map e405 with expected crc 2023-10-09T06:06:04.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:04 smithi064 ceph-mon[220044]: failed to encode map e405 with expected crc 2023-10-09T06:06:04.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:04 smithi064 ceph-mon[220044]: pgmap v990: 119 pgs: 1 remapped+peering, 12 peering, 106 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 913 op/s; 7.8 MiB/s, 124 objects/s recovering 2023-10-09T06:06:04.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:04 smithi064 ceph-mon[220044]: osdmap e406: 8 total, 8 up, 7 in 2023-10-09T06:06:04.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:04 smithi064 ceph-mon[220044]: failed to encode map e406 with expected crc 2023-10-09T06:06:04.540 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:04.597 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:04.714 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:04.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:04.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:04.830 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:04.889 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:06:04.890 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:05.030 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:05.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:05.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:05.143 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:05.144 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:05.174 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:05.212 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:05.365 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:05.375 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:05 smithi064 ceph-mon[212750]: failed to encode map e406 with expected crc 2023-10-09T06:06:05.375 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:05 smithi064 ceph-mon[212750]: failed to encode map e406 with expected crc 2023-10-09T06:06:05.376 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:05 smithi064 ceph-mon[212750]: failed to encode map e406 with expected crc 2023-10-09T06:06:05.376 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:05 smithi064 ceph-mon[212750]: failed to encode map e406 with expected crc 2023-10-09T06:06:05.376 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:05 smithi064 ceph-mon[220044]: failed to encode map e406 with expected crc 2023-10-09T06:06:05.376 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:05 smithi064 ceph-mon[220044]: failed to encode map e406 with expected crc 2023-10-09T06:06:05.376 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:05 smithi064 ceph-mon[220044]: failed to encode map e406 with expected crc 2023-10-09T06:06:05.377 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:05 smithi064 ceph-mon[220044]: failed to encode map e406 with expected crc 2023-10-09T06:06:05.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:05.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:05.378 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:06:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:06:05] "GET /metrics HTTP/1.1" 200 36437 "" "Prometheus/2.43.0" 2023-10-09T06:06:05.380 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:05 smithi156 ceph-mon[195101]: failed to encode map e406 with expected crc 2023-10-09T06:06:05.380 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:05 smithi156 ceph-mon[195101]: failed to encode map e406 with expected crc 2023-10-09T06:06:05.380 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:05 smithi156 ceph-mon[195101]: failed to encode map e406 with expected crc 2023-10-09T06:06:05.380 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:05 smithi156 ceph-mon[195101]: failed to encode map e406 with expected crc 2023-10-09T06:06:05.475 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:06:05.478 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:05.547 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:05.706 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:05.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:05.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:05.822 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:05.919 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:06.116 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:06.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:06.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:06.231 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:06.305 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:06 smithi064 ceph-mon[212750]: pgmap v992: 119 pgs: 1 active+remapped+backfill_wait, 1 remapped+peering, 12 peering, 105 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 227 op/s; 2489/121428 objects misplaced (2.050%) 2023-10-09T06:06:06.305 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:06 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:06:06.305 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:06 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:06:06.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:06 smithi064 ceph-mon[220044]: pgmap v992: 119 pgs: 1 active+remapped+backfill_wait, 1 remapped+peering, 12 peering, 105 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 227 op/s; 2489/121428 objects misplaced (2.050%) 2023-10-09T06:06:06.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:06 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:06:06.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:06 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:06:06.315 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:06.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:06 smithi156 ceph-mon[195101]: pgmap v992: 119 pgs: 1 active+remapped+backfill_wait, 1 remapped+peering, 12 peering, 105 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 227 op/s; 2489/121428 objects misplaced (2.050%) 2023-10-09T06:06:06.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:06 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:06:06.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:06 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:06:07.236 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:06:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:07 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:07 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:06:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:07 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:06:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:07 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/721755133' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:07 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:07 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:06:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:07 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:06:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:07 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/721755133' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:07.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:07 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:07.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:07 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:06:07.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:07 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:06:07.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:07 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/721755133' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:08.179 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:08.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:08.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:08.287 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:08.342 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:08.413 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:08.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:08.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:08.419 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:08 smithi156 ceph-mon[195101]: pgmap v993: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 116 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 433 op/s; 6121/116031 objects misplaced (5.275%); 14 MiB/s, 227 objects/s recovering 2023-10-09T06:06:08.419 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:08 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-10-09T06:06:08.419 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:08 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:08.419 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:08 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/721755133' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:06:08.419 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:08 smithi156 ceph-mon[195101]: osdmap e407: 8 total, 8 up, 7 in 2023-10-09T06:06:08.419 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:08 smithi156 ceph-mon[195101]: failed to encode map e407 with expected crc 2023-10-09T06:06:08.419 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:08 smithi156 ceph-mon[195101]: failed to encode map e407 with expected crc 2023-10-09T06:06:08.419 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:08 smithi156 ceph-mon[195101]: failed to encode map e407 with expected crc 2023-10-09T06:06:08.419 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:08 smithi156 ceph-mon[195101]: failed to encode map e407 with expected crc 2023-10-09T06:06:08.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:08 smithi064 ceph-mon[212750]: pgmap v993: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 116 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 433 op/s; 6121/116031 objects misplaced (5.275%); 14 MiB/s, 227 objects/s recovering 2023-10-09T06:06:08.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:08 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-10-09T06:06:08.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:08 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:08.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:08 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/721755133' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:06:08.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:08 smithi064 ceph-mon[212750]: osdmap e407: 8 total, 8 up, 7 in 2023-10-09T06:06:08.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:08 smithi064 ceph-mon[212750]: failed to encode map e407 with expected crc 2023-10-09T06:06:08.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:08 smithi064 ceph-mon[212750]: failed to encode map e407 with expected crc 2023-10-09T06:06:08.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:08 smithi064 ceph-mon[212750]: failed to encode map e407 with expected crc 2023-10-09T06:06:08.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:08 smithi064 ceph-mon[212750]: failed to encode map e407 with expected crc 2023-10-09T06:06:08.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:08 smithi064 ceph-mon[220044]: pgmap v993: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 116 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 433 op/s; 6121/116031 objects misplaced (5.275%); 14 MiB/s, 227 objects/s recovering 2023-10-09T06:06:08.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:08 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-10-09T06:06:08.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:08 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:08.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:08 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/721755133' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:06:08.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:08 smithi064 ceph-mon[220044]: osdmap e407: 8 total, 8 up, 7 in 2023-10-09T06:06:08.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:08 smithi064 ceph-mon[220044]: failed to encode map e407 with expected crc 2023-10-09T06:06:08.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:08 smithi064 ceph-mon[220044]: failed to encode map e407 with expected crc 2023-10-09T06:06:08.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:08 smithi064 ceph-mon[220044]: failed to encode map e407 with expected crc 2023-10-09T06:06:08.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:08 smithi064 ceph-mon[220044]: failed to encode map e407 with expected crc 2023-10-09T06:06:08.519 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:08.548 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:08.594 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:08.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:08.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:08.698 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:08.764 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:08.950 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:08.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:08.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:09.058 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:09.122 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:09.211 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:09.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:09.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:09.315 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:09.349 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:09.429 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:09.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:09.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:09.538 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:09.637 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:09.783 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:09.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:09.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:09.824 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:06:09.869 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:06:09.898 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:09.912 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:06:09.953 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:09.973 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:06:09.995 INFO:teuthology.orchestra.run.smithi064.stdout:moved 0 / 357 (0%) 2023-10-09T06:06:09.995 INFO:teuthology.orchestra.run.smithi064.stdout:avg 51 2023-10-09T06:06:09.995 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 7.70899 -> 7.70899 (expected baseline 6.61168) 2023-10-09T06:06:09.995 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.6 with 39 -> 39 pgs (0.764706 -> 0.764706 * mean) 2023-10-09T06:06:09.995 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.1 with 59 -> 59 pgs (1.15686 -> 1.15686 * mean) 2023-10-09T06:06:09.995 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:06:09.995 INFO:teuthology.orchestra.run.smithi064.stdout:oload 1000 2023-10-09T06:06:09.996 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 1 2023-10-09T06:06:09.996 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-10-09T06:06:09.996 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0170 2023-10-09T06:06:09.996 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.1697 2023-10-09T06:06:10.010 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:06:10.012 INFO:teuthology.orchestra.run.smithi064.stderr:no change 2023-10-09T06:06:10.047 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:10.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:10.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:10.055 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:06:10.095 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:06:10.150 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:06:10.151 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:10.152 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:06:10.152 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:06:10.152 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:06:10.152 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:06:10.153 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:06:10.153 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:06:10.153 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:06:10.153 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:06:10.198 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:10 smithi156 ceph-mon[195101]: pgmap v995: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 433 op/s; 5856/115236 objects misplaced (5.082%); 13 MiB/s, 205 objects/s recovering 2023-10-09T06:06:10.198 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:10 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1305558848' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:10.203 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:10.315 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:10.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:10.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:10.419 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:10.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:10 smithi064 ceph-mon[212750]: pgmap v995: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 433 op/s; 5856/115236 objects misplaced (5.082%); 13 MiB/s, 205 objects/s recovering 2023-10-09T06:06:10.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:10 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1305558848' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:10.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:10 smithi064 ceph-mon[220044]: pgmap v995: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 433 op/s; 5856/115236 objects misplaced (5.082%); 13 MiB/s, 205 objects/s recovering 2023-10-09T06:06:10.474 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:10 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1305558848' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:10.493 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:10.623 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:10.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:10.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:10.732 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:10.805 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:10.942 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:06:11.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:11 smithi064 ceph-mon[212750]: from='client.55648 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:11.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:11 smithi064 ceph-mon[220044]: from='client.55648 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:11.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:11 smithi156 ceph-mon[195101]: from='client.55648 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:11.862 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:11.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:11.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:11.973 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:12.011 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:12.092 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:12.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:12.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:12.148 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:12.205 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:12.284 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:12.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:12 smithi156 ceph-mon[195101]: pgmap v996: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 2.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 4147/104052 objects misplaced (3.986%); 21 MiB/s, 338 objects/s recovering 2023-10-09T06:06:12.450 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:12 smithi064 ceph-mon[212750]: pgmap v996: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 2.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 4147/104052 objects misplaced (3.986%); 21 MiB/s, 338 objects/s recovering 2023-10-09T06:06:12.450 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:12 smithi064 ceph-mon[220044]: pgmap v996: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 2.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 4147/104052 objects misplaced (3.986%); 21 MiB/s, 338 objects/s recovering 2023-10-09T06:06:12.493 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:12.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:12.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:12.602 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:12.667 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:12.757 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:12.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:12.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:12.864 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:12.937 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:13.065 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:13.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:13.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:13.169 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:13.223 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:13.269 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:13.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:13.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:13.372 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:13.410 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:13.666 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:13.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:13.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:13.775 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:13.818 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:13.895 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:06:14.114 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:14.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:14.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:14.230 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:14.318 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:14.436 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:14.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:14.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:14.474 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:14 smithi064 ceph-mon[212750]: pgmap v997: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 2.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 720 op/s; 4147/103200 objects misplaced (4.018%); 18 MiB/s, 287 objects/s recovering 2023-10-09T06:06:14.474 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:14.475 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:14 smithi064 ceph-mon[220044]: pgmap v997: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 2.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 720 op/s; 4147/103200 objects misplaced (4.018%); 18 MiB/s, 287 objects/s recovering 2023-10-09T06:06:14.475 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:14.542 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:14.608 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:14.686 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:14.725 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:14.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:14.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:14.735 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:14 smithi156 ceph-mon[195101]: pgmap v997: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 2.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 720 op/s; 4147/103200 objects misplaced (4.018%); 18 MiB/s, 287 objects/s recovering 2023-10-09T06:06:14.735 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:14.742 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:06:14.742 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:06:14.742 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.742 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:178441 8.a e407/402 8:5621b4c5:::benchmark_data_smithi064_457003_object73923:head v 407'11446, mlcod=407'11444)", 2023-10-09T06:06:14.743 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:14.732915+0000", 2023-10-09T06:06:14.743 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0032419359999999999, 2023-10-09T06:06:14.743 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0032725639999999999, 2023-10-09T06:06:14.743 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:14.743 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:06:14.743 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:14.743 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.744 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:14.744 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.732915+0000", 2023-10-09T06:06:14.744 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:14.744 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.744 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.744 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:14.745 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.732915+0000", 2023-10-09T06:06:14.745 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:14.745 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.745 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.745 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:14.745 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.732914+0000", 2023-10-09T06:06:14.745 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:06:14.746 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.746 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.746 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:14.746 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.732926+0000", 2023-10-09T06:06:14.746 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1216000000000001e-05 2023-10-09T06:06:14.746 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.746 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.747 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:14.747 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.732935+0000", 2023-10-09T06:06:14.747 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3680000000000008e-06 2023-10-09T06:06:14.747 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.747 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.747 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:14.747 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.732940+0000", 2023-10-09T06:06:14.748 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3059999999999997e-06 2023-10-09T06:06:14.748 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.748 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.748 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:14.748 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.733568+0000", 2023-10-09T06:06:14.748 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00062807 2023-10-09T06:06:14.749 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.749 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.749 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:14.749 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.733592+0000", 2023-10-09T06:06:14.749 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3281999999999998e-05 2023-10-09T06:06:14.749 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:14.749 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:14.750 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:14.750 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.750 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.750 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:178435 8.d e407/398 8:b80ccc5a:::benchmark_data_smithi064_457003_object73917:head v 407'10923, mlcod=407'10920)", 2023-10-09T06:06:14.750 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:14.733269+0000", 2023-10-09T06:06:14.750 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0028882809999999999, 2023-10-09T06:06:14.751 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0029923689999999999, 2023-10-09T06:06:14.751 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:14.751 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:06:14.751 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:14.751 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.751 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:14.751 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.733269+0000", 2023-10-09T06:06:14.752 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:14.752 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.752 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.752 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:14.752 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.733269+0000", 2023-10-09T06:06:14.752 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:14.752 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.753 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.753 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:14.753 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.733268+0000", 2023-10-09T06:06:14.753 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:06:14.753 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.753 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.754 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:14.754 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.733276+0000", 2023-10-09T06:06:14.754 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1540000000000002e-06 2023-10-09T06:06:14.754 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.754 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.754 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:14.754 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.733292+0000", 2023-10-09T06:06:14.755 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5775999999999999e-05 2023-10-09T06:06:14.755 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.755 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.755 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:14.755 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.733298+0000", 2023-10-09T06:06:14.755 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7579999999999998e-06 2023-10-09T06:06:14.755 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.756 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.756 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:14.756 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.734444+0000", 2023-10-09T06:06:14.756 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0011462309999999999 2023-10-09T06:06:14.756 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.756 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.756 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:14.757 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.734467+0000", 2023-10-09T06:06:14.757 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3079e-05 2023-10-09T06:06:14.757 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:14.757 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:14.757 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:14.757 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.757 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.757 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:178437 8.d e407/398)", 2023-10-09T06:06:14.757 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:14.733410+0000", 2023-10-09T06:06:14.758 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0027471259999999999, 2023-10-09T06:06:14.758 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0029082040000000002, 2023-10-09T06:06:14.758 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:14.758 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "queued for pg", 2023-10-09T06:06:14.758 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:14.758 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.758 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:14.759 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.733410+0000", 2023-10-09T06:06:14.759 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:14.759 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.759 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.759 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:14.759 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.733410+0000", 2023-10-09T06:06:14.759 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:14.760 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.760 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.760 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:14.760 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.733409+0000", 2023-10-09T06:06:14.760 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:06:14.760 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.760 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.761 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:14.761 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.733416+0000", 2023-10-09T06:06:14.761 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1589999999999998e-06 2023-10-09T06:06:14.761 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.761 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.761 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:14.761 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.733426+0000", 2023-10-09T06:06:14.761 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0003999999999999e-05 2023-10-09T06:06:14.761 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.762 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.762 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:14.762 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.733431+0000", 2023-10-09T06:06:14.762 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3329999999999996e-06 2023-10-09T06:06:14.762 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:14.762 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:14.762 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:14.762 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.763 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.763 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:178440 8.2 e407/360 8:4005013e:::benchmark_data_smithi064_457003_object73922:head v 407'11081, mlcod=407'11079)", 2023-10-09T06:06:14.763 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:14.734110+0000", 2023-10-09T06:06:14.763 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0020471249999999999, 2023-10-09T06:06:14.763 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002255483, 2023-10-09T06:06:14.763 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:14.763 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:06:14.764 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:14.764 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.764 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:14.764 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.734110+0000", 2023-10-09T06:06:14.764 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:14.764 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.764 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.765 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:14.765 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.734110+0000", 2023-10-09T06:06:14.765 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:14.765 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.765 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.765 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:14.765 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.734109+0000", 2023-10-09T06:06:14.766 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:06:14.766 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.766 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.766 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:14.766 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.734116+0000", 2023-10-09T06:06:14.766 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.268e-06 2023-10-09T06:06:14.767 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.767 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.767 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:14.767 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.734129+0000", 2023-10-09T06:06:14.767 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3543000000000001e-05 2023-10-09T06:06:14.767 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.767 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.767 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:14.768 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.734134+0000", 2023-10-09T06:06:14.768 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2909999999999998e-06 2023-10-09T06:06:14.768 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.768 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.768 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:14.768 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.736195+0000", 2023-10-09T06:06:14.768 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0020610020000000001 2023-10-09T06:06:14.769 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:14.769 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:14.769 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:14.769 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:14.736216+0000", 2023-10-09T06:06:14.769 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0055999999999999e-05 2023-10-09T06:06:14.769 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:14.769 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:14.770 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:14.770 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:14.770 INFO:teuthology.orchestra.run.smithi156.stdout: ], 2023-10-09T06:06:14.770 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 4 2023-10-09T06:06:14.770 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:06:14.836 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:14.909 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:15.012 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:15.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:15.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:15.125 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:15.187 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:15.248 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:15.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:15.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:15.303 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:06:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:06:15] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.43.0" 2023-10-09T06:06:15.357 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:15.387 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:15.433 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:15.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:15.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:15.458 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:06:15.537 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:15.580 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:15.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:15 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/6271006' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:06:15.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:15 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/6271006' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:06:15.867 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:15.868 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:15 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/6271006' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:06:15.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:15.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:15.939 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:06:15.971 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:16.010 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:16.150 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:16.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:16.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:16.262 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:16.332 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:16.508 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:06:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:16 smithi064 ceph-mon[212750]: pgmap v998: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 2.0 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 3885/99363 objects misplaced (3.910%); 17 MiB/s, 270 objects/s recovering 2023-10-09T06:06:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:16 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:16 smithi064 ceph-mon[212750]: failed to encode map e408 with expected crc 2023-10-09T06:06:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:16 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/6271006' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:06:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:16 smithi064 ceph-mon[212750]: osdmap e408: 8 total, 8 up, 7 in 2023-10-09T06:06:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:16 smithi064 ceph-mon[212750]: failed to encode map e408 with expected crc 2023-10-09T06:06:16.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:16 smithi064 ceph-mon[212750]: failed to encode map e408 with expected crc 2023-10-09T06:06:16.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:16 smithi064 ceph-mon[220044]: pgmap v998: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 2.0 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 3885/99363 objects misplaced (3.910%); 17 MiB/s, 270 objects/s recovering 2023-10-09T06:06:16.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:16 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:16.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:16 smithi064 ceph-mon[220044]: failed to encode map e408 with expected crc 2023-10-09T06:06:16.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:16 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/6271006' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:06:16.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:16 smithi064 ceph-mon[220044]: osdmap e408: 8 total, 8 up, 7 in 2023-10-09T06:06:16.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:16 smithi064 ceph-mon[220044]: failed to encode map e408 with expected crc 2023-10-09T06:06:16.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:16 smithi064 ceph-mon[220044]: failed to encode map e408 with expected crc 2023-10-09T06:06:16.823 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:16 smithi156 ceph-mon[195101]: pgmap v998: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 2.0 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 3885/99363 objects misplaced (3.910%); 17 MiB/s, 270 objects/s recovering 2023-10-09T06:06:16.823 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:16 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:16.824 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:16 smithi156 ceph-mon[195101]: failed to encode map e408 with expected crc 2023-10-09T06:06:16.824 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:16 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/6271006' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:06:16.824 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:16 smithi156 ceph-mon[195101]: osdmap e408: 8 total, 8 up, 7 in 2023-10-09T06:06:16.824 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:16 smithi156 ceph-mon[195101]: failed to encode map e408 with expected crc 2023-10-09T06:06:16.824 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:16 smithi156 ceph-mon[195101]: failed to encode map e408 with expected crc 2023-10-09T06:06:16.867 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1] out_osds: [2] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 0, 4] 2023-10-09T06:06:16.867 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:06:16.867 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-10-09T06:06:16.868 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:06:16.868 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:06:16.868 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-10-09T06:06:16.944 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:16.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:16.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:17.060 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:17.137 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:17.307 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:17.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:17.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:17.418 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:17.505 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:17.574 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:17.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:17.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:17.606 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:17.611 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:06:17.638 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:06:17.659 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:06:17.679 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:17.680 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:06:17.709 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:17.744 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:06:17.783 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:06:17.832 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:06:17.849 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:17.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:17.858 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:17 smithi064 ceph-mon[212750]: 1.14 scrub starts 2023-10-09T06:06:17.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:17 smithi064 ceph-mon[220044]: 1.14 scrub starts 2023-10-09T06:06:17.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:17.864 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:06:17.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:17 smithi156 ceph-mon[195101]: 1.14 scrub starts 2023-10-09T06:06:17.880 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:06:17.880 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:06:17.880 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:06:17.880 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:06:17.880 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:06:17.880 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:06:17.880 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:06:17.880 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:06:17.960 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:18.042 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:18.201 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:18.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:18.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:18.311 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:18.376 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:18.574 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:18.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:18.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:18 smithi156 ceph-mon[195101]: pgmap v1000: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 1.9 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 862 op/s; 2517/91038 objects misplaced (2.765%); 17 MiB/s, 268 objects/s recovering 2023-10-09T06:06:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:18 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2826568530' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:18.679 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:18.683 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:06:18.683 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:06:18.707 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:18.751 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:18.753 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:18 smithi064 ceph-mon[212750]: pgmap v1000: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 1.9 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 862 op/s; 2517/91038 objects misplaced (2.765%); 17 MiB/s, 268 objects/s recovering 2023-10-09T06:06:18.753 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:18 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2826568530' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:18.753 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:18 smithi064 ceph-mon[220044]: pgmap v1000: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 1.9 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 862 op/s; 2517/91038 objects misplaced (2.765%); 17 MiB/s, 268 objects/s recovering 2023-10-09T06:06:18.753 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:18 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2826568530' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:18.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:18.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:18.855 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:18.901 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:06:18.902 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:18.955 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:18.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:18.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:19.059 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:19.087 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:19.131 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:19.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:19.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:19.235 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:19.261 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:19.304 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:19.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:19.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:19.407 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:19.436 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:19.481 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:19.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:19.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:19.584 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:19.667 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:19.791 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:19.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:19.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:19.795 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:19 smithi064 ceph-mon[212750]: osdmap e409: 8 total, 8 up, 7 in 2023-10-09T06:06:19.796 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:19 smithi064 ceph-mon[212750]: failed to encode map e409 with expected crc 2023-10-09T06:06:19.796 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:19 smithi064 ceph-mon[212750]: failed to encode map e409 with expected crc 2023-10-09T06:06:19.796 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:19 smithi064 ceph-mon[212750]: failed to encode map e409 with expected crc 2023-10-09T06:06:19.796 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:19 smithi064 ceph-mon[212750]: failed to encode map e409 with expected crc 2023-10-09T06:06:19.796 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:19 smithi064 ceph-mon[212750]: failed to encode map e409 with expected crc 2023-10-09T06:06:19.796 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:19 smithi064 ceph-mon[212750]: failed to encode map e409 with expected crc 2023-10-09T06:06:19.796 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:19 smithi064 ceph-mon[220044]: osdmap e409: 8 total, 8 up, 7 in 2023-10-09T06:06:19.796 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:19 smithi064 ceph-mon[220044]: failed to encode map e409 with expected crc 2023-10-09T06:06:19.796 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:19 smithi064 ceph-mon[220044]: failed to encode map e409 with expected crc 2023-10-09T06:06:19.797 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:19 smithi064 ceph-mon[220044]: failed to encode map e409 with expected crc 2023-10-09T06:06:19.797 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:19 smithi064 ceph-mon[220044]: failed to encode map e409 with expected crc 2023-10-09T06:06:19.797 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:19 smithi064 ceph-mon[220044]: failed to encode map e409 with expected crc 2023-10-09T06:06:19.797 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:19 smithi064 ceph-mon[220044]: failed to encode map e409 with expected crc 2023-10-09T06:06:19.844 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:19.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:19 smithi156 ceph-mon[195101]: osdmap e409: 8 total, 8 up, 7 in 2023-10-09T06:06:19.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:19 smithi156 ceph-mon[195101]: failed to encode map e409 with expected crc 2023-10-09T06:06:19.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:19 smithi156 ceph-mon[195101]: failed to encode map e409 with expected crc 2023-10-09T06:06:19.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:19 smithi156 ceph-mon[195101]: failed to encode map e409 with expected crc 2023-10-09T06:06:19.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:19 smithi156 ceph-mon[195101]: failed to encode map e409 with expected crc 2023-10-09T06:06:19.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:19 smithi156 ceph-mon[195101]: failed to encode map e409 with expected crc 2023-10-09T06:06:19.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:19 smithi156 ceph-mon[195101]: failed to encode map e409 with expected crc 2023-10-09T06:06:19.896 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:19.969 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:06:19.985 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:20.243 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:20.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:20.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:20.333 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:06:20.333 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "" 2023-10-09T06:06:20.333 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:06:20.349 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:20.428 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:20.505 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:20 smithi064 ceph-mon[212750]: pgmap v1002: 119 pgs: 3 active+remapped+backfilling, 116 active+clean; 1.8 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 2153/90234 objects misplaced (2.386%); 10 MiB/s, 163 objects/s recovering 2023-10-09T06:06:20.505 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:20 smithi064 ceph-mon[212750]: failed to encode map e410 with expected crc 2023-10-09T06:06:20.505 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:20 smithi064 ceph-mon[212750]: osdmap e410: 8 total, 8 up, 7 in 2023-10-09T06:06:20.505 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:20 smithi064 ceph-mon[212750]: failed to encode map e410 with expected crc 2023-10-09T06:06:20.505 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:20 smithi064 ceph-mon[212750]: failed to encode map e410 with expected crc 2023-10-09T06:06:20.505 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:20 smithi064 ceph-mon[212750]: failed to encode map e410 with expected crc 2023-10-09T06:06:20.505 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:20 smithi064 ceph-mon[212750]: failed to encode map e410 with expected crc 2023-10-09T06:06:20.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:20 smithi064 ceph-mon[220044]: pgmap v1002: 119 pgs: 3 active+remapped+backfilling, 116 active+clean; 1.8 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 2153/90234 objects misplaced (2.386%); 10 MiB/s, 163 objects/s recovering 2023-10-09T06:06:20.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:20 smithi064 ceph-mon[220044]: failed to encode map e410 with expected crc 2023-10-09T06:06:20.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:20 smithi064 ceph-mon[220044]: osdmap e410: 8 total, 8 up, 7 in 2023-10-09T06:06:20.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:20 smithi064 ceph-mon[220044]: failed to encode map e410 with expected crc 2023-10-09T06:06:20.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:20 smithi064 ceph-mon[220044]: failed to encode map e410 with expected crc 2023-10-09T06:06:20.508 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:20 smithi064 ceph-mon[220044]: failed to encode map e410 with expected crc 2023-10-09T06:06:20.508 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:20 smithi064 ceph-mon[220044]: failed to encode map e410 with expected crc 2023-10-09T06:06:20.531 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:20.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:20.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:20.632 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:06:20.641 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:20.701 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:20 smithi156 ceph-mon[195101]: pgmap v1002: 119 pgs: 3 active+remapped+backfilling, 116 active+clean; 1.8 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 2153/90234 objects misplaced (2.386%); 10 MiB/s, 163 objects/s recovering 2023-10-09T06:06:20.701 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:20 smithi156 ceph-mon[195101]: failed to encode map e410 with expected crc 2023-10-09T06:06:20.702 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:20 smithi156 ceph-mon[195101]: osdmap e410: 8 total, 8 up, 7 in 2023-10-09T06:06:20.702 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:20 smithi156 ceph-mon[195101]: failed to encode map e410 with expected crc 2023-10-09T06:06:20.702 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:20 smithi156 ceph-mon[195101]: failed to encode map e410 with expected crc 2023-10-09T06:06:20.702 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:20 smithi156 ceph-mon[195101]: failed to encode map e410 with expected crc 2023-10-09T06:06:20.702 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:20 smithi156 ceph-mon[195101]: failed to encode map e410 with expected crc 2023-10-09T06:06:20.706 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:20.827 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:20.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:20.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:20.934 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:20.964 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:21.613 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:06:21.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:21 smithi156 ceph-mon[195101]: 1.14 scrub starts 2023-10-09T06:06:21.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:21 smithi156 ceph-mon[195101]: 1.14 scrub ok 2023-10-09T06:06:21.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:21 smithi064 ceph-mon[220044]: 1.14 scrub starts 2023-10-09T06:06:21.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:21 smithi064 ceph-mon[220044]: 1.14 scrub ok 2023-10-09T06:06:21.673 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:21 smithi064 ceph-mon[212750]: 1.14 scrub starts 2023-10-09T06:06:21.673 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:21 smithi064 ceph-mon[212750]: 1.14 scrub ok 2023-10-09T06:06:21.803 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:21.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:21.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:21.911 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:22.001 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:22.162 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:22.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:22.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:22.269 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:22.342 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:22.428 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:22.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:22.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:22.513 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:22.537 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:22.569 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:22.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:22 smithi156 ceph-mon[195101]: pgmap v1004: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.7 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 991 op/s; 1598/81501 objects misplaced (1.961%); 18 MiB/s, 295 objects/s recovering 2023-10-09T06:06:22.749 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:22.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:22.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:22.811 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:22 smithi064 ceph-mon[212750]: pgmap v1004: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.7 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 991 op/s; 1598/81501 objects misplaced (1.961%); 18 MiB/s, 295 objects/s recovering 2023-10-09T06:06:22.813 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:22 smithi064 ceph-mon[220044]: pgmap v1004: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.7 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 991 op/s; 1598/81501 objects misplaced (1.961%); 18 MiB/s, 295 objects/s recovering 2023-10-09T06:06:22.854 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:22.925 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:23.039 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:06:23.039 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:06:23.050 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:23.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:23.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:23.159 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:23.222 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:23.364 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:23.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:23.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:23.478 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:23.523 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:23.555 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:06:23.573 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:23.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:23.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:23.677 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:23.721 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:23.798 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:23.800 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:23 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1123836417' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:23.800 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:23 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:23.800 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:23 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1123836417' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:23.800 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:23 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:23.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:23.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:23.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:23 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1123836417' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:23 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:23.902 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:23.959 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:24.182 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:24.184 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:06:24.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:24.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:24.287 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:24.317 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:24.551 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:24 smithi156 ceph-mon[195101]: pgmap v1005: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.6 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 1598/80841 objects misplaced (1.977%); 11 MiB/s, 171 objects/s recovering 2023-10-09T06:06:24.551 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:24 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:06:24.552 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:24 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T06:06:24.552 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:24 smithi156 ceph-mon[195101]: failed to encode map e411 with expected crc 2023-10-09T06:06:24.552 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:24 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:06:24.552 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:24 smithi156 ceph-mon[195101]: osdmap e411: 8 total, 8 up, 7 in 2023-10-09T06:06:24.552 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:24 smithi156 ceph-mon[195101]: failed to encode map e411 with expected crc 2023-10-09T06:06:24.552 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:24 smithi156 ceph-mon[195101]: failed to encode map e411 with expected crc 2023-10-09T06:06:24.552 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:24 smithi156 ceph-mon[195101]: failed to encode map e411 with expected crc 2023-10-09T06:06:24.645 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:24.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:24.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:24.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:24 smithi064 ceph-mon[212750]: pgmap v1005: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.6 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 1598/80841 objects misplaced (1.977%); 11 MiB/s, 171 objects/s recovering 2023-10-09T06:06:24.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:24 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:06:24.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:24 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:06:24.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:24 smithi064 ceph-mon[212750]: failed to encode map e411 with expected crc 2023-10-09T06:06:24.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:24 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:06:24.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:24 smithi064 ceph-mon[212750]: osdmap e411: 8 total, 8 up, 7 in 2023-10-09T06:06:24.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:24 smithi064 ceph-mon[212750]: failed to encode map e411 with expected crc 2023-10-09T06:06:24.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:24 smithi064 ceph-mon[212750]: failed to encode map e411 with expected crc 2023-10-09T06:06:24.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:24 smithi064 ceph-mon[212750]: failed to encode map e411 with expected crc 2023-10-09T06:06:24.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:24 smithi064 ceph-mon[220044]: pgmap v1005: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.6 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 1598/80841 objects misplaced (1.977%); 11 MiB/s, 171 objects/s recovering 2023-10-09T06:06:24.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:24 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:06:24.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:24 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T06:06:24.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:24 smithi064 ceph-mon[220044]: failed to encode map e411 with expected crc 2023-10-09T06:06:24.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:24 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:06:24.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:24 smithi064 ceph-mon[220044]: osdmap e411: 8 total, 8 up, 7 in 2023-10-09T06:06:24.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:24 smithi064 ceph-mon[220044]: failed to encode map e411 with expected crc 2023-10-09T06:06:24.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:24 smithi064 ceph-mon[220044]: failed to encode map e411 with expected crc 2023-10-09T06:06:24.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:24 smithi064 ceph-mon[220044]: failed to encode map e411 with expected crc 2023-10-09T06:06:24.751 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:24.775 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:06:24.815 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:25.028 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:25.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:25.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:25.139 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:25.177 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:06:25.212 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:25.220 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:06:25.254 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:06:25.289 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:06:25.309 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:25.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:25.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:25.330 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:06:25.365 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:06:25.409 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:06:25.415 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:25.442 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:06:25.453 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:06:25.453 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:06:25.454 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:06:25.454 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:06:25.454 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:06:25.454 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:06:25.454 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:06:25.454 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:06:25.461 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:25.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:06:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:06:25] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.43.0" 2023-10-09T06:06:25.552 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:25.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:25.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:25.659 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:25.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:25 smithi156 ceph-mon[195101]: failed to encode map e411 with expected crc 2023-10-09T06:06:25.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:25 smithi156 ceph-mon[195101]: failed to encode map e411 with expected crc 2023-10-09T06:06:25.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:25 smithi156 ceph-mon[195101]: failed to encode map e411 with expected crc 2023-10-09T06:06:25.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:25 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3755592323' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:25.918 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:25.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:25 smithi064 ceph-mon[212750]: failed to encode map e411 with expected crc 2023-10-09T06:06:25.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:25 smithi064 ceph-mon[212750]: failed to encode map e411 with expected crc 2023-10-09T06:06:25.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:25 smithi064 ceph-mon[212750]: failed to encode map e411 with expected crc 2023-10-09T06:06:25.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:25 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3755592323' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:25.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:25 smithi064 ceph-mon[220044]: failed to encode map e411 with expected crc 2023-10-09T06:06:25.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:25 smithi064 ceph-mon[220044]: failed to encode map e411 with expected crc 2023-10-09T06:06:25.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:25 smithi064 ceph-mon[220044]: failed to encode map e411 with expected crc 2023-10-09T06:06:25.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:25 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3755592323' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:26.010 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:26.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:26.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:26.116 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:26.162 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:26.247 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:26.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:26.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:26.355 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:26.401 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:26.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:26 smithi156 ceph-mon[195101]: pgmap v1007: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 588/76983 objects misplaced (0.764%); 14 MiB/s, 229 objects/s recovering 2023-10-09T06:06:26.634 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:26.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:26.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:26.661 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:06:26.753 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:26.834 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:26 smithi064 ceph-mon[212750]: pgmap v1007: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 588/76983 objects misplaced (0.764%); 14 MiB/s, 229 objects/s recovering 2023-10-09T06:06:26.834 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:26 smithi064 ceph-mon[220044]: pgmap v1007: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 588/76983 objects misplaced (0.764%); 14 MiB/s, 229 objects/s recovering 2023-10-09T06:06:26.835 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:26.908 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1] out_osds: [2] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 0, 4] 2023-10-09T06:06:26.908 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:06:26.908 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T06:06:26.908 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:06:26.908 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:06:26.909 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T06:06:26.962 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:26.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:26.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:27.074 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:27.084 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:06:27.084 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:06:27.084 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.084 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:188018 8.9 8:9f384090:::benchmark_data_smithi064_457003_object83500:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2023-10-09T06:06:27.084 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:27.076013+0000", 2023-10-09T06:06:27.084 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0053637390000000002, 2023-10-09T06:06:27.085 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0053952410000000003, 2023-10-09T06:06:27.085 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:27.085 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:06:27.085 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:06:27.085 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:06:27.085 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:06:27.085 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188018 2023-10-09T06:06:27.085 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.085 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:27.085 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.086 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:27.086 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.076013+0000", 2023-10-09T06:06:27.086 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:27.086 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.086 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.086 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:27.086 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.076013+0000", 2023-10-09T06:06:27.086 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:27.086 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.086 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.087 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:27.087 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.076012+0000", 2023-10-09T06:06:27.087 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:06:27.087 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.087 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.087 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:27.087 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.076018+0000", 2023-10-09T06:06:27.087 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4010000000000002e-06 2023-10-09T06:06:27.087 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.087 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.087 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:27.088 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.076045+0000", 2023-10-09T06:06:27.088 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.6234e-05 2023-10-09T06:06:27.088 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.088 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.088 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:27.088 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.076051+0000", 2023-10-09T06:06:27.088 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2299999999999996e-06 2023-10-09T06:06:27.088 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.088 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.088 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:27.088 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.076100+0000", 2023-10-09T06:06:27.089 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9469999999999999e-05 2023-10-09T06:06:27.089 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.089 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.089 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:27.089 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.076232+0000", 2023-10-09T06:06:27.089 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013155400000000001 2023-10-09T06:06:27.089 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.089 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.089 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T06:06:27.089 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.076366+0000", 2023-10-09T06:06:27.089 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013425199999999999 2023-10-09T06:06:27.090 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.090 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.090 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:06:27.090 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.079696+0000", 2023-10-09T06:06:27.090 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0033300449999999998 2023-10-09T06:06:27.090 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:27.090 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:27.090 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:27.090 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.090 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.090 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:188019 8.4 e411/392 8:2c2a09d5:::benchmark_data_smithi064_457003_object83501:head v 411'11756, mlcod=411'11754)", 2023-10-09T06:06:27.090 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:27.080248+0000", 2023-10-09T06:06:27.091 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0011287949999999999, 2023-10-09T06:06:27.091 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001246563, 2023-10-09T06:06:27.091 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:27.091 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:06:27.091 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:27.091 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.091 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:27.091 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.080248+0000", 2023-10-09T06:06:27.091 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:27.091 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.091 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.091 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:27.092 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.080248+0000", 2023-10-09T06:06:27.092 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:27.092 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.092 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.092 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:27.092 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.080248+0000", 2023-10-09T06:06:27.092 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:06:27.092 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.092 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.092 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:27.092 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.080254+0000", 2023-10-09T06:06:27.093 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6880000000000002e-06 2023-10-09T06:06:27.093 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.093 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.093 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:27.093 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.080266+0000", 2023-10-09T06:06:27.093 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2149e-05 2023-10-09T06:06:27.093 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.093 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.093 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:27.093 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.080272+0000", 2023-10-09T06:06:27.093 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4609999999999999e-06 2023-10-09T06:06:27.093 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.094 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.094 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:27.094 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.080323+0000", 2023-10-09T06:06:27.094 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0881999999999998e-05 2023-10-09T06:06:27.094 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.094 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.094 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:27.094 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.080346+0000", 2023-10-09T06:06:27.094 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3204999999999999e-05 2023-10-09T06:06:27.094 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:27.094 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:27.094 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:27.095 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.095 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.095 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:188020 8.9 8.14090e39 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e411)", 2023-10-09T06:06:27.095 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:27.080771+0000", 2023-10-09T06:06:27.095 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.00060612800000000005, 2023-10-09T06:06:27.095 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00078174500000000003, 2023-10-09T06:06:27.095 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:27.095 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "queued for pg", 2023-10-09T06:06:27.095 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:06:27.095 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:06:27.095 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:06:27.095 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188020 2023-10-09T06:06:27.096 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.096 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:27.096 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.096 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:27.096 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.080771+0000", 2023-10-09T06:06:27.096 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:27.096 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.096 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.096 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:27.096 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.080771+0000", 2023-10-09T06:06:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.080770+0000", 2023-10-09T06:06:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:06:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.080776+0000", 2023-10-09T06:06:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.173e-06 2023-10-09T06:06:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.097 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.098 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:27.098 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.080788+0000", 2023-10-09T06:06:27.098 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1759999999999999e-05 2023-10-09T06:06:27.098 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.098 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.098 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:27.098 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.080793+0000", 2023-10-09T06:06:27.098 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8370000000000002e-06 2023-10-09T06:06:27.098 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:27.098 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:27.098 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:27.099 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.099 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.099 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop_reply(client.45605.0:188018 8.9 e411/403)", 2023-10-09T06:06:27.099 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:27.081147+0000", 2023-10-09T06:06:27.099 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.000229886, 2023-10-09T06:06:27.099 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00045554299999999999, 2023-10-09T06:06:27.099 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:27.099 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "queued for pg", 2023-10-09T06:06:27.099 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:27.099 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.099 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:27.099 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.081147+0000", 2023-10-09T06:06:27.100 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:27.100 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.100 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.100 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:27.100 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.081147+0000", 2023-10-09T06:06:27.100 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:27.100 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.100 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.100 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:27.100 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.081147+0000", 2023-10-09T06:06:27.101 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:06:27.101 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.101 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.101 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:27.101 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.081151+0000", 2023-10-09T06:06:27.101 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0450000000000002e-06 2023-10-09T06:06:27.101 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.101 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.101 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:27.101 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.081159+0000", 2023-10-09T06:06:27.101 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9869999999999999e-06 2023-10-09T06:06:27.102 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:27.102 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:27.102 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:27.102 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.081164+0000", 2023-10-09T06:06:27.102 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6770000000000003e-06 2023-10-09T06:06:27.102 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:27.102 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:27.102 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:27.102 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:27.102 INFO:teuthology.orchestra.run.smithi156.stdout: ], 2023-10-09T06:06:27.102 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 4 2023-10-09T06:06:27.102 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:06:27.118 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:27.203 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:27.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:27.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:27.285 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:27.312 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:27.374 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:28.107 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:28.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:28.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:28.175 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:06:28.210 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:28.218 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:06:28.241 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:28.288 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:28.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:28.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:28.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:28.420 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:28.653 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:06:28.687 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:28 smithi064 ceph-mon[212750]: pgmap v1008: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.4 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 914 op/s; 588/68244 objects misplaced (0.862%); 11 MiB/s, 171 objects/s recovering 2023-10-09T06:06:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:28 smithi064 ceph-mon[212750]: osdmap e412: 8 total, 8 up, 7 in 2023-10-09T06:06:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:28 smithi064 ceph-mon[212750]: failed to encode map e412 with expected crc 2023-10-09T06:06:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:28 smithi064 ceph-mon[212750]: failed to encode map e412 with expected crc 2023-10-09T06:06:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:28 smithi064 ceph-mon[212750]: failed to encode map e412 with expected crc 2023-10-09T06:06:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:28 smithi064 ceph-mon[212750]: failed to encode map e412 with expected crc 2023-10-09T06:06:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:28 smithi064 ceph-mon[212750]: failed to encode map e412 with expected crc 2023-10-09T06:06:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:28 smithi064 ceph-mon[212750]: failed to encode map e412 with expected crc 2023-10-09T06:06:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:28 smithi064 ceph-mon[212750]: failed to encode map e412 with expected crc 2023-10-09T06:06:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:28 smithi064 ceph-mon[212750]: failed to encode map e412 with expected crc 2023-10-09T06:06:28.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:28 smithi064 ceph-mon[220044]: pgmap v1008: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.4 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 914 op/s; 588/68244 objects misplaced (0.862%); 11 MiB/s, 171 objects/s recovering 2023-10-09T06:06:28.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:28 smithi064 ceph-mon[220044]: osdmap e412: 8 total, 8 up, 7 in 2023-10-09T06:06:28.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:28 smithi064 ceph-mon[220044]: failed to encode map e412 with expected crc 2023-10-09T06:06:28.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:28 smithi064 ceph-mon[220044]: failed to encode map e412 with expected crc 2023-10-09T06:06:28.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:28 smithi064 ceph-mon[220044]: failed to encode map e412 with expected crc 2023-10-09T06:06:28.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:28 smithi064 ceph-mon[220044]: failed to encode map e412 with expected crc 2023-10-09T06:06:28.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:28 smithi064 ceph-mon[220044]: failed to encode map e412 with expected crc 2023-10-09T06:06:28.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:28 smithi064 ceph-mon[220044]: failed to encode map e412 with expected crc 2023-10-09T06:06:28.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:28 smithi064 ceph-mon[220044]: failed to encode map e412 with expected crc 2023-10-09T06:06:28.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:28 smithi064 ceph-mon[220044]: failed to encode map e412 with expected crc 2023-10-09T06:06:28.697 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:28.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:28.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:28.791 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:28 smithi156 ceph-mon[195101]: pgmap v1008: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.4 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 914 op/s; 588/68244 objects misplaced (0.862%); 11 MiB/s, 171 objects/s recovering 2023-10-09T06:06:28.791 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:28 smithi156 ceph-mon[195101]: osdmap e412: 8 total, 8 up, 7 in 2023-10-09T06:06:28.791 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:28 smithi156 ceph-mon[195101]: failed to encode map e412 with expected crc 2023-10-09T06:06:28.791 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:28 smithi156 ceph-mon[195101]: failed to encode map e412 with expected crc 2023-10-09T06:06:28.792 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:28 smithi156 ceph-mon[195101]: failed to encode map e412 with expected crc 2023-10-09T06:06:28.792 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:28 smithi156 ceph-mon[195101]: failed to encode map e412 with expected crc 2023-10-09T06:06:28.792 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:28 smithi156 ceph-mon[195101]: failed to encode map e412 with expected crc 2023-10-09T06:06:28.792 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:28 smithi156 ceph-mon[195101]: failed to encode map e412 with expected crc 2023-10-09T06:06:28.792 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:28 smithi156 ceph-mon[195101]: failed to encode map e412 with expected crc 2023-10-09T06:06:28.792 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:28 smithi156 ceph-mon[195101]: failed to encode map e412 with expected crc 2023-10-09T06:06:28.813 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:28.870 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:28.927 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:28.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:28.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:29.031 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:29.079 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:29.336 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:06:29.365 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:29.372 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:29.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:29.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:29.450 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:29 smithi064 ceph-mon[212750]: failed to encode map e413 with expected crc 2023-10-09T06:06:29.450 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:29 smithi064 ceph-mon[212750]: failed to encode map e413 with expected crc 2023-10-09T06:06:29.450 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:29 smithi064 ceph-mon[212750]: osdmap e413: 8 total, 8 up, 7 in 2023-10-09T06:06:29.450 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:29 smithi064 ceph-mon[212750]: failed to encode map e413 with expected crc 2023-10-09T06:06:29.450 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:29 smithi064 ceph-mon[212750]: failed to encode map e413 with expected crc 2023-10-09T06:06:29.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:29 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:06:29.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:29.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:29 smithi064 ceph-mon[220044]: failed to encode map e413 with expected crc 2023-10-09T06:06:29.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:29 smithi064 ceph-mon[220044]: failed to encode map e413 with expected crc 2023-10-09T06:06:29.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:29 smithi064 ceph-mon[220044]: osdmap e413: 8 total, 8 up, 7 in 2023-10-09T06:06:29.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:29 smithi064 ceph-mon[220044]: failed to encode map e413 with expected crc 2023-10-09T06:06:29.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:29 smithi064 ceph-mon[220044]: failed to encode map e413 with expected crc 2023-10-09T06:06:29.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:29 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:06:29.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:29.479 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:29.569 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:29.617 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:29 smithi156 ceph-mon[195101]: failed to encode map e413 with expected crc 2023-10-09T06:06:29.617 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:29 smithi156 ceph-mon[195101]: failed to encode map e413 with expected crc 2023-10-09T06:06:29.617 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:29 smithi156 ceph-mon[195101]: osdmap e413: 8 total, 8 up, 7 in 2023-10-09T06:06:29.617 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:29 smithi156 ceph-mon[195101]: failed to encode map e413 with expected crc 2023-10-09T06:06:29.617 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:29 smithi156 ceph-mon[195101]: failed to encode map e413 with expected crc 2023-10-09T06:06:29.617 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:29 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:06:29.618 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:29.681 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:29.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:29.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:29.786 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:29.859 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:30.548 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:06:30.793 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:30 smithi156 ceph-mon[195101]: pgmap v1011: 119 pgs: 119 active+clean; 1.4 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 740 op/s; 15 MiB/s, 240 objects/s recovering 2023-10-09T06:06:30.794 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:30 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3009250058' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:06:30.794 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:30 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:06:30.847 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:30 smithi064 ceph-mon[212750]: pgmap v1011: 119 pgs: 119 active+clean; 1.4 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 740 op/s; 15 MiB/s, 240 objects/s recovering 2023-10-09T06:06:30.847 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:30 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3009250058' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:06:30.847 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:30 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:06:30.848 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:30 smithi064 ceph-mon[220044]: pgmap v1011: 119 pgs: 119 active+clean; 1.4 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 740 op/s; 15 MiB/s, 240 objects/s recovering 2023-10-09T06:06:30.848 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:30 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3009250058' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:06:30.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:30 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:06:30.888 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:06:30.888 INFO:teuthology.orchestra.run.smithi156.stdout: "success": "" 2023-10-09T06:06:30.888 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:06:31.037 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:31.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:31.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:31.141 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:31.220 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:31.405 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:31.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:31.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:31.518 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:31.564 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:31.662 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:31.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:31.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:31.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:31 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:31.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:31 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:06:31.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:31 smithi064 ceph-mon[212750]: osdmap e414: 8 total, 8 up, 7 in 2023-10-09T06:06:31.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:31 smithi064 ceph-mon[212750]: failed to encode map e414 with expected crc 2023-10-09T06:06:31.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:31 smithi064 ceph-mon[212750]: failed to encode map e414 with expected crc 2023-10-09T06:06:31.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:31 smithi064 ceph-mon[212750]: failed to encode map e414 with expected crc 2023-10-09T06:06:31.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:31 smithi064 ceph-mon[212750]: failed to encode map e414 with expected crc 2023-10-09T06:06:31.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:31 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:31.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:31 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:06:31.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:31 smithi064 ceph-mon[220044]: osdmap e414: 8 total, 8 up, 7 in 2023-10-09T06:06:31.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:31 smithi064 ceph-mon[220044]: failed to encode map e414 with expected crc 2023-10-09T06:06:31.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:31 smithi064 ceph-mon[220044]: failed to encode map e414 with expected crc 2023-10-09T06:06:31.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:31 smithi064 ceph-mon[220044]: failed to encode map e414 with expected crc 2023-10-09T06:06:31.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:31 smithi064 ceph-mon[220044]: failed to encode map e414 with expected crc 2023-10-09T06:06:31.772 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:31.817 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:31 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:31.817 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:31 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:06:31.817 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:31 smithi156 ceph-mon[195101]: osdmap e414: 8 total, 8 up, 7 in 2023-10-09T06:06:31.817 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:31 smithi156 ceph-mon[195101]: failed to encode map e414 with expected crc 2023-10-09T06:06:31.817 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:31 smithi156 ceph-mon[195101]: failed to encode map e414 with expected crc 2023-10-09T06:06:31.818 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:31 smithi156 ceph-mon[195101]: failed to encode map e414 with expected crc 2023-10-09T06:06:31.818 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:31 smithi156 ceph-mon[195101]: failed to encode map e414 with expected crc 2023-10-09T06:06:31.823 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:31.918 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:31.924 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:06:31.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:31.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:31.960 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:06:31.986 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:06:31.986 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:06:31.986 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T06:06:31.986 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T06:06:31.986 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:06:31.992 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:06:32.026 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:32.031 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:06:32.070 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:06:32.083 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:32.107 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:06:32.158 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:06:32.167 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:32.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:32.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:32.204 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:06:32.208 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:06:32.208 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:06:32.208 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:06:32.208 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:06:32.209 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:06:32.209 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:06:32.209 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:06:32.209 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:06:32.273 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:32.338 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:32.427 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:32.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:32.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:32.538 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:32.602 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:32.705 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:32.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:32.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:32.713 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:32 smithi064 ceph-mon[212750]: pgmap v1013: 119 pgs: 119 active+clean; 1.2 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 5.6 MiB/s, 89 objects/s recovering 2023-10-09T06:06:32.713 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:32 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4093599076' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:32.714 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:32 smithi064 ceph-mon[220044]: pgmap v1013: 119 pgs: 119 active+clean; 1.2 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 5.6 MiB/s, 89 objects/s recovering 2023-10-09T06:06:32.714 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:32 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4093599076' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:32.813 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:32.842 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:32 smithi156 ceph-mon[195101]: pgmap v1013: 119 pgs: 119 active+clean; 1.2 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 5.6 MiB/s, 89 objects/s recovering 2023-10-09T06:06:32.843 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:32 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4093599076' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:32.871 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:32.885 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:06:32.973 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:32.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:32.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:33.085 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:33.143 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:33.277 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:33.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:33.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:33.390 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:33.442 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:06:33.455 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:33.540 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:33.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:33.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:33.574 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:06:33.651 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:33.704 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:33.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:33.858 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:33.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:33.959 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:34.010 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:34.060 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:34.256 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:34.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:34.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:34.362 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:34.423 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:34.680 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:34 smithi064 ceph-mon[220044]: pgmap v1014: 119 pgs: 119 active+clean; 1.2 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 5.6 MiB/s, 89 objects/s recovering 2023-10-09T06:06:34.680 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:34 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:06:34.682 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:34 smithi064 ceph-mon[212750]: pgmap v1014: 119 pgs: 119 active+clean; 1.2 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 5.6 MiB/s, 89 objects/s recovering 2023-10-09T06:06:34.682 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:34 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:06:34.722 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:34.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:34.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:34.829 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:34.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:34 smithi156 ceph-mon[195101]: pgmap v1014: 119 pgs: 119 active+clean; 1.2 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 5.6 MiB/s, 89 objects/s recovering 2023-10-09T06:06:34.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:34 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:06:34.893 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:34.996 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:35.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:35.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:35.102 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:35.138 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:35.184 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:35.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:35.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:35.213 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:06:35.289 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:35.305 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:06:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:06:35] "GET /metrics HTTP/1.1" 200 36425 "" "Prometheus/2.43.0" 2023-10-09T06:06:35.368 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:35.622 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:06:35.623 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:06:35.693 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:35.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:35.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:35.814 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:35.865 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:35.944 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:35.963 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:35.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:35.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:36.067 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:36.103 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:36.191 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:36.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:36.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:36.297 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:36.339 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:36.527 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:06:36.532 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:36.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:36.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:36.589 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1] out_osds: [2] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 0, 4] 2023-10-09T06:06:36.590 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:06:36.590 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:06:36.590 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 1.000000 2023-10-09T06:06:36.590 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-10-09T06:06:36.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:36 smithi156 ceph-mon[195101]: pgmap v1015: 119 pgs: 119 active+clean; 1.1 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 4.5 MiB/s, 72 objects/s recovering 2023-10-09T06:06:36.640 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:36.672 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:36 smithi064 ceph-mon[212750]: pgmap v1015: 119 pgs: 119 active+clean; 1.1 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 4.5 MiB/s, 72 objects/s recovering 2023-10-09T06:06:36.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:36 smithi064 ceph-mon[220044]: pgmap v1015: 119 pgs: 119 active+clean; 1.1 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 4.5 MiB/s, 72 objects/s recovering 2023-10-09T06:06:36.711 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:36.825 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:36.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:36.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:36.931 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:36.979 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:37.032 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:06:37.088 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:37.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:37.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:37.198 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:37.255 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:37.532 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:37.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:37.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:37.611 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:06:37.640 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:37.719 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:37.735 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:37 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1680585046' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:37.932 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:37 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1680585046' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:37.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:37 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1680585046' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:38.392 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:06:38.392 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:06:38.392 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:06:38.422 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:06:38.458 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:06:38.492 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:06:38.524 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:06:38.566 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:06:38.575 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:06:38.575 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:06:38.575 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:06:38.576 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:06:38.576 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:06:38.576 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:06:38.576 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:06:38.576 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:06:38.587 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:38 smithi156 ceph-mon[195101]: pgmap v1016: 119 pgs: 119 active+clean; 883 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 4.0 MiB/s, 63 objects/s recovering 2023-10-09T06:06:38.587 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:38 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:38.587 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:38 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1680585046' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:06:38.587 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:38 smithi156 ceph-mon[195101]: osdmap e415: 8 total, 8 up, 7 in 2023-10-09T06:06:38.587 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:38 smithi156 ceph-mon[195101]: failed to encode map e415 with expected crc 2023-10-09T06:06:38.587 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:38 smithi156 ceph-mon[195101]: failed to encode map e415 with expected crc 2023-10-09T06:06:38.587 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:38 smithi156 ceph-mon[195101]: failed to encode map e415 with expected crc 2023-10-09T06:06:38.587 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:38 smithi156 ceph-mon[195101]: failed to encode map e415 with expected crc 2023-10-09T06:06:38.588 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:38 smithi156 ceph-mon[195101]: failed to encode map e415 with expected crc 2023-10-09T06:06:38.588 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:38 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/562358363' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:38.828 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:38.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:38.839 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:38 smithi064 ceph-mon[220044]: pgmap v1016: 119 pgs: 119 active+clean; 883 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 4.0 MiB/s, 63 objects/s recovering 2023-10-09T06:06:38.839 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:38 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:38.839 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:38 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1680585046' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:06:38.839 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:38 smithi064 ceph-mon[220044]: osdmap e415: 8 total, 8 up, 7 in 2023-10-09T06:06:38.839 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:38 smithi064 ceph-mon[220044]: failed to encode map e415 with expected crc 2023-10-09T06:06:38.840 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:38 smithi064 ceph-mon[220044]: failed to encode map e415 with expected crc 2023-10-09T06:06:38.840 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:38 smithi064 ceph-mon[220044]: failed to encode map e415 with expected crc 2023-10-09T06:06:38.840 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:38 smithi064 ceph-mon[220044]: failed to encode map e415 with expected crc 2023-10-09T06:06:38.840 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:38 smithi064 ceph-mon[220044]: failed to encode map e415 with expected crc 2023-10-09T06:06:38.840 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:38 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/562358363' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:38.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:38.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:38 smithi064 ceph-mon[212750]: pgmap v1016: 119 pgs: 119 active+clean; 883 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 4.0 MiB/s, 63 objects/s recovering 2023-10-09T06:06:38.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:38 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:38.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:38 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1680585046' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:06:38.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:38 smithi064 ceph-mon[212750]: osdmap e415: 8 total, 8 up, 7 in 2023-10-09T06:06:38.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:38 smithi064 ceph-mon[212750]: failed to encode map e415 with expected crc 2023-10-09T06:06:38.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:38 smithi064 ceph-mon[212750]: failed to encode map e415 with expected crc 2023-10-09T06:06:38.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:38 smithi064 ceph-mon[212750]: failed to encode map e415 with expected crc 2023-10-09T06:06:38.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:38 smithi064 ceph-mon[212750]: failed to encode map e415 with expected crc 2023-10-09T06:06:38.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:38 smithi064 ceph-mon[212750]: failed to encode map e415 with expected crc 2023-10-09T06:06:38.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:38 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/562358363' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:38.941 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:38.978 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:38.988 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:06:38.988 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:06:38.988 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 0 2023-10-09T06:06:38.988 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:06:39.481 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:39.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:39.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:39.590 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:39.660 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:39.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:39 smithi156 ceph-mon[195101]: pgmap v1018: 119 pgs: 119 active+clean; 867 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 719 op/s 2023-10-09T06:06:39.889 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:39 smithi064 ceph-mon[212750]: pgmap v1018: 119 pgs: 119 active+clean; 867 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 719 op/s 2023-10-09T06:06:39.889 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:39 smithi064 ceph-mon[220044]: pgmap v1018: 119 pgs: 119 active+clean; 867 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 719 op/s 2023-10-09T06:06:39.907 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:06:39.928 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:39.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:39.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:40.044 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:40.109 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:40.210 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:40.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:40.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:40.315 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:40.368 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:40.474 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:40.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:40.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:40.525 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:06:40.582 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:40.649 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:40.753 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:40.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:40.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:40.860 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:40.915 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:41.162 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:41.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:41.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:41.270 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:41.313 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:06:41.343 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:41.494 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:41.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:41.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:41.601 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:41.602 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:06:41.647 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:41.881 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:41.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:41.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:41.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:41 smithi064 ceph-mon[212750]: pgmap v1019: 119 pgs: 119 active+clean; 616 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 932 op/s 2023-10-09T06:06:41.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:41 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3019286413' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:06:41.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:41 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:06:41.986 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:42.003 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-10-09T06:06:42.045 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:42.045 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:42.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:41 smithi064 ceph-mon[220044]: pgmap v1019: 119 pgs: 119 active+clean; 616 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 932 op/s 2023-10-09T06:06:42.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:41 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3019286413' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:06:42.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:41 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:06:42.314 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:42.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:42.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:42.355 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:41 smithi156 ceph-mon[195101]: pgmap v1019: 119 pgs: 119 active+clean; 616 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 932 op/s 2023-10-09T06:06:42.355 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:41 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3019286413' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:06:42.355 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:41 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:06:42.422 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:42.481 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:42.603 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:42.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:42.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:42.664 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:06:42.665 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:06:42.665 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T06:06:42.665 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T06:06:42.665 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:06:42.707 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:42.735 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:42.779 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:42.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:42.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:42.883 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:42.936 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:42.981 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:42.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:42.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:43.084 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:43.110 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:42 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:06:43.110 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:42 smithi064 ceph-mon[220044]: osdmap e416: 8 total, 8 up, 7 in 2023-10-09T06:06:43.110 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:42 smithi064 ceph-mon[220044]: failed to encode map e416 with expected crc 2023-10-09T06:06:43.110 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:42 smithi064 ceph-mon[220044]: failed to encode map e416 with expected crc 2023-10-09T06:06:43.110 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:42 smithi064 ceph-mon[220044]: failed to encode map e416 with expected crc 2023-10-09T06:06:43.110 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:42 smithi064 ceph-mon[220044]: failed to encode map e416 with expected crc 2023-10-09T06:06:43.110 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:42 smithi064 ceph-mon[220044]: failed to encode map e416 with expected crc 2023-10-09T06:06:43.110 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:42 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:06:43.111 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:42 smithi064 ceph-mon[212750]: osdmap e416: 8 total, 8 up, 7 in 2023-10-09T06:06:43.111 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:42 smithi064 ceph-mon[212750]: failed to encode map e416 with expected crc 2023-10-09T06:06:43.111 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:42 smithi064 ceph-mon[212750]: failed to encode map e416 with expected crc 2023-10-09T06:06:43.111 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:42 smithi064 ceph-mon[212750]: failed to encode map e416 with expected crc 2023-10-09T06:06:43.111 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:42 smithi064 ceph-mon[212750]: failed to encode map e416 with expected crc 2023-10-09T06:06:43.111 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:42 smithi064 ceph-mon[212750]: failed to encode map e416 with expected crc 2023-10-09T06:06:43.112 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:43.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:42 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:06:43.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:42 smithi156 ceph-mon[195101]: osdmap e416: 8 total, 8 up, 7 in 2023-10-09T06:06:43.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:42 smithi156 ceph-mon[195101]: failed to encode map e416 with expected crc 2023-10-09T06:06:43.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:42 smithi156 ceph-mon[195101]: failed to encode map e416 with expected crc 2023-10-09T06:06:43.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:42 smithi156 ceph-mon[195101]: failed to encode map e416 with expected crc 2023-10-09T06:06:43.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:42 smithi156 ceph-mon[195101]: failed to encode map e416 with expected crc 2023-10-09T06:06:43.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:42 smithi156 ceph-mon[195101]: failed to encode map e416 with expected crc 2023-10-09T06:06:43.388 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:43.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:43.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:43.505 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:43.535 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:06:43.566 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:43.622 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:43.833 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:43.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:43.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:43.947 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:44.023 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:44.132 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:44.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:44.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:44.158 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:06:44.240 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:44.326 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:44.476 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:44.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:44.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:44.581 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:44.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:44 smithi156 ceph-mon[195101]: pgmap v1021: 119 pgs: 119 active+clean; 596 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 979 op/s 2023-10-09T06:06:44.656 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:06:44.656 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:44.657 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:44.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:44 smithi064 ceph-mon[220044]: pgmap v1021: 119 pgs: 119 active+clean; 596 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 979 op/s 2023-10-09T06:06:44.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:06:44.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:44.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:44 smithi064 ceph-mon[212750]: pgmap v1021: 119 pgs: 119 active+clean; 596 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 979 op/s 2023-10-09T06:06:44.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:06:44.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:44.851 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:44.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:44.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:44.956 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:45.019 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:45.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:06:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:06:45] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.43.0" 2023-10-09T06:06:45.826 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:45.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:45.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:45.931 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:46.007 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:46.240 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:46.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:46.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:46.315 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:06:46.315 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T06:06:46.316 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T06:06:46.316 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:06:46.316 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.316 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:185875 6.15 6:aa8ef49b:::benchmark_data_smithi064_284193_object89332:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-10-09T06:06:46.316 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.701818+0000", 2023-10-09T06:06:46.316 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 535.60686951499997, 2023-10-09T06:06:46.316 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.183023392, 2023-10-09T06:06:46.316 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:46.316 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:06:46.316 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:06:46.316 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:06:46.317 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:06:46.317 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 185875 2023-10-09T06:06:46.317 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.317 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:46.317 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.317 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:46.317 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.701818+0000", 2023-10-09T06:06:46.317 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.317 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.317 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.317 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:46.318 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.701818+0000", 2023-10-09T06:06:46.318 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.318 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.318 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.318 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:46.318 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.701818+0000", 2023-10-09T06:06:46.318 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:06:46.318 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.318 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.318 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:46.318 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.701821+0000", 2023-10-09T06:06:46.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1410000000000001e-06 2023-10-09T06:06:46.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.319 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.319 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:46.319 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.701831+0000", 2023-10-09T06:06:46.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0142e-05 2023-10-09T06:06:46.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.319 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.319 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.319 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.701835+0000", 2023-10-09T06:06:46.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2919999999999997e-06 2023-10-09T06:06:46.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.320 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.320 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.320 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.707726+0000", 2023-10-09T06:06:46.320 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0058907179999999996 2023-10-09T06:06:46.320 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.320 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.320 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:46.320 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.707842+0000", 2023-10-09T06:06:46.320 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000115937 2023-10-09T06:06:46.320 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.320 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.320 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:06:46.321 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.708169+0000", 2023-10-09T06:06:46.321 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00032739800000000001 2023-10-09T06:06:46.321 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.321 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.321 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:06:46.321 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.711326+0000", 2023-10-09T06:06:46.321 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.003157235 2023-10-09T06:06:46.321 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.321 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.321 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.321 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.732073+0000", 2023-10-09T06:06:46.321 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.020746568999999999 2023-10-09T06:06:46.322 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.322 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.322 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.322 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.732081+0000", 2023-10-09T06:06:46.322 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8420000000000005e-06 2023-10-09T06:06:46.322 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.322 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.322 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.322 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735784+0000", 2023-10-09T06:06:46.322 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.003702782 2023-10-09T06:06:46.322 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.322 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.323 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.323 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735796+0000", 2023-10-09T06:06:46.323 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2009e-05 2023-10-09T06:06:46.323 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.323 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.323 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.323 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.884749+0000", 2023-10-09T06:06:46.323 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.148953603 2023-10-09T06:06:46.323 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.324 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.324 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:06:46.324 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.884842+0000", 2023-10-09T06:06:46.324 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2200000000000005e-05 2023-10-09T06:06:46.324 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.324 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:46.324 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.324 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.324 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.324 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:185894 6.15 6:aea4a65e:::benchmark_data_smithi064_284193_object89351:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:06:46.324 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.718597+0000", 2023-10-09T06:06:46.324 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 535.590090691, 2023-10-09T06:06:46.325 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1840398429999999, 2023-10-09T06:06:46.325 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:46.325 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.325 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:06:46.325 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:06:46.325 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:06:46.325 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 185894 2023-10-09T06:06:46.325 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.325 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:46.325 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.325 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:46.325 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.718597+0000", 2023-10-09T06:06:46.326 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.326 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.326 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.326 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:46.326 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.718597+0000", 2023-10-09T06:06:46.326 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.326 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.326 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.326 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:46.326 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.718596+0000", 2023-10-09T06:06:46.326 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:06:46.327 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.327 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.327 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:46.327 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.718616+0000", 2023-10-09T06:06:46.327 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0449999999999999e-05 2023-10-09T06:06:46.327 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.327 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.327 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:46.327 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.718644+0000", 2023-10-09T06:06:46.327 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7945000000000001e-05 2023-10-09T06:06:46.327 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.327 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.327 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.328 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.718651+0000", 2023-10-09T06:06:46.328 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6490000000000002e-06 2023-10-09T06:06:46.328 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.328 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.328 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.328 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.718696+0000", 2023-10-09T06:06:46.328 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4870000000000002e-05 2023-10-09T06:06:46.328 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.328 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.328 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op must wait for map", 2023-10-09T06:06:46.328 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.718703+0000", 2023-10-09T06:06:46.328 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2409999999999998e-06 2023-10-09T06:06:46.329 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.329 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.329 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.329 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.732103+0000", 2023-10-09T06:06:46.329 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013399708999999999 2023-10-09T06:06:46.329 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.329 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.329 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.329 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.732109+0000", 2023-10-09T06:06:46.329 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4869999999999999e-06 2023-10-09T06:06:46.329 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.329 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.330 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.330 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735816+0000", 2023-10-09T06:06:46.330 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.003706183 2023-10-09T06:06:46.330 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.330 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.330 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.330 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735822+0000", 2023-10-09T06:06:46.330 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8580000000000002e-06 2023-10-09T06:06:46.330 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.330 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.330 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.330 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.884859+0000", 2023-10-09T06:06:46.330 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1490370620000001 2023-10-09T06:06:46.331 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.331 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.331 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:46.331 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.884937+0000", 2023-10-09T06:06:46.331 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7639000000000007e-05 2023-10-09T06:06:46.331 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.331 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.331 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T06:06:46.331 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.885342+0000", 2023-10-09T06:06:46.331 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00040508599999999998 2023-10-09T06:06:46.331 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.331 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.332 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:06:46.332 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.893721+0000", 2023-10-09T06:06:46.332 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0083789090000000004 2023-10-09T06:06:46.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.332 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.332 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.332 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902271+0000", 2023-10-09T06:06:46.332 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0085503039999999999 2023-10-09T06:06:46.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.332 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.332 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.333 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902383+0000", 2023-10-09T06:06:46.333 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000111969 2023-10-09T06:06:46.333 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.333 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.333 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.333 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902489+0000", 2023-10-09T06:06:46.333 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010552099999999999 2023-10-09T06:06:46.333 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.333 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.333 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:06:46.333 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902536+0000", 2023-10-09T06:06:46.333 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7466999999999998e-05 2023-10-09T06:06:46.334 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.334 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.334 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:06:46.334 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902637+0000", 2023-10-09T06:06:46.334 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010043999999999999 2023-10-09T06:06:46.334 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.334 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:46.334 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.334 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.334 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.334 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:185895 6.15 6:a9b7b9d9:::benchmark_data_smithi064_284193_object89352:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:06:46.335 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.726247+0000", 2023-10-09T06:06:46.335 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 535.58244101800005, 2023-10-09T06:06:46.335 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1767582440000002, 2023-10-09T06:06:46.335 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:46.335 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.335 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:06:46.335 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:06:46.335 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:06:46.335 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 185895 2023-10-09T06:06:46.335 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.335 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:46.336 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.336 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:46.336 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.726247+0000", 2023-10-09T06:06:46.336 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.336 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.336 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.336 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:46.336 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.726247+0000", 2023-10-09T06:06:46.336 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.336 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.336 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.336 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:46.337 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.726245+0000", 2023-10-09T06:06:46.337 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:06:46.337 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.337 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.337 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:46.337 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.726252+0000", 2023-10-09T06:06:46.337 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1350000000000003e-06 2023-10-09T06:06:46.337 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.337 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.337 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:46.337 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.726264+0000", 2023-10-09T06:06:46.337 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2021000000000001e-05 2023-10-09T06:06:46.337 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.338 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.338 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.338 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.726271+0000", 2023-10-09T06:06:46.338 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6869999999999997e-06 2023-10-09T06:06:46.338 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.338 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.338 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.338 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.726427+0000", 2023-10-09T06:06:46.338 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015600299999999999 2023-10-09T06:06:46.338 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.338 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.339 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting_for_map not empty", 2023-10-09T06:06:46.339 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.726433+0000", 2023-10-09T06:06:46.339 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9449999999999998e-06 2023-10-09T06:06:46.339 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.339 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.339 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.339 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.732127+0000", 2023-10-09T06:06:46.339 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0056935670000000001 2023-10-09T06:06:46.339 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.339 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.339 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.339 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.732133+0000", 2023-10-09T06:06:46.340 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1999999999999999e-06 2023-10-09T06:06:46.340 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.340 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.340 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.340 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735842+0000", 2023-10-09T06:06:46.340 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0037096569999999 2023-10-09T06:06:46.340 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.340 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.340 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.340 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735851+0000", 2023-10-09T06:06:46.340 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.3650000000000004e-06 2023-10-09T06:06:46.340 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.340 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.341 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.341 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887467+0000", 2023-10-09T06:06:46.341 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1516161149999999 2023-10-09T06:06:46.341 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.341 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.341 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:46.341 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887570+0000", 2023-10-09T06:06:46.341 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000103458 2023-10-09T06:06:46.341 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.341 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.341 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T06:06:46.341 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887826+0000", 2023-10-09T06:06:46.342 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000255212 2023-10-09T06:06:46.342 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.342 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.342 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:06:46.342 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.893729+0000", 2023-10-09T06:06:46.342 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0059037949999999999 2023-10-09T06:06:46.342 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.342 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.342 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.342 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902299+0000", 2023-10-09T06:06:46.342 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0085701249999999996 2023-10-09T06:06:46.342 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.343 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.343 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.343 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902678+0000", 2023-10-09T06:06:46.343 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000379069 2023-10-09T06:06:46.343 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.343 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.343 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.343 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902845+0000", 2023-10-09T06:06:46.343 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00016605400000000001 2023-10-09T06:06:46.343 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.343 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.343 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:06:46.344 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902896+0000", 2023-10-09T06:06:46.344 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1298999999999998e-05 2023-10-09T06:06:46.344 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.344 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.344 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:06:46.344 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.903005+0000", 2023-10-09T06:06:46.344 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000109002 2023-10-09T06:06:46.344 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.344 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:46.344 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.344 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.344 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.345 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:185920 6.15 6:aed0640a:::benchmark_data_smithi064_284193_object89377:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:06:46.345 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.758471+0000", 2023-10-09T06:06:46.345 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 535.55021624300002, 2023-10-09T06:06:46.345 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1447898919999999, 2023-10-09T06:06:46.345 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:46.345 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.345 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:06:46.345 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:06:46.345 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:06:46.345 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 185920 2023-10-09T06:06:46.345 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.345 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:46.346 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.346 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:46.346 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.758471+0000", 2023-10-09T06:06:46.346 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.346 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.346 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.346 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:46.346 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.758471+0000", 2023-10-09T06:06:46.346 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.346 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.346 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.346 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:46.347 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.758470+0000", 2023-10-09T06:06:46.347 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:06:46.347 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.347 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.347 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:46.347 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.758476+0000", 2023-10-09T06:06:46.347 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3690000000000003e-06 2023-10-09T06:06:46.347 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.347 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.347 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:46.347 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.758490+0000", 2023-10-09T06:06:46.347 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3767e-05 2023-10-09T06:06:46.347 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.348 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.348 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.348 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.758496+0000", 2023-10-09T06:06:46.348 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0610000000000001e-06 2023-10-09T06:06:46.348 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.348 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.348 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.348 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.758550+0000", 2023-10-09T06:06:46.348 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3894000000000003e-05 2023-10-09T06:06:46.348 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.348 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.349 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.349 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.758556+0000", 2023-10-09T06:06:46.349 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6799999999999998e-06 2023-10-09T06:06:46.349 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.349 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.349 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.349 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735869+0000", 2023-10-09T06:06:46.349 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.97731343199999998 2023-10-09T06:06:46.349 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.349 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.349 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.349 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735875+0000", 2023-10-09T06:06:46.350 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0639999999999999e-06 2023-10-09T06:06:46.350 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.350 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.350 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.350 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.889339+0000", 2023-10-09T06:06:46.350 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1534636359999999 2023-10-09T06:06:46.350 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.350 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.350 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:46.350 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.889423+0000", 2023-10-09T06:06:46.350 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.4303999999999998e-05 2023-10-09T06:06:46.350 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.351 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.351 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T06:06:46.351 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.889619+0000", 2023-10-09T06:06:46.351 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000195606 2023-10-09T06:06:46.351 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.351 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.351 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:06:46.351 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.896800+0000", 2023-10-09T06:06:46.351 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0071807390000000002 2023-10-09T06:06:46.351 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.351 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.352 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.352 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902329+0000", 2023-10-09T06:06:46.352 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0055295759999999996 2023-10-09T06:06:46.352 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.352 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.352 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.352 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902708+0000", 2023-10-09T06:06:46.352 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00037905300000000002 2023-10-09T06:06:46.353 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.353 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.353 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.353 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.903105+0000", 2023-10-09T06:06:46.353 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00039682200000000002 2023-10-09T06:06:46.353 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.353 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.353 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:06:46.353 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.903155+0000", 2023-10-09T06:06:46.353 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9982999999999999e-05 2023-10-09T06:06:46.353 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.354 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.354 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:06:46.354 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.903261+0000", 2023-10-09T06:06:46.354 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010619399999999999 2023-10-09T06:06:46.354 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.354 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:46.354 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.354 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.354 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.354 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:185935 6.25 6:a62e1056:::benchmark_data_smithi064_284193_object89392:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:06:46.355 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.768157+0000", 2023-10-09T06:06:46.355 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 535.54053018399998, 2023-10-09T06:06:46.355 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1328383999999998, 2023-10-09T06:06:46.355 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:46.355 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.355 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:06:46.355 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:06:46.355 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:06:46.355 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 185935 2023-10-09T06:06:46.355 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.356 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:46.356 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.356 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:46.356 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.768157+0000", 2023-10-09T06:06:46.356 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.356 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.356 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.356 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:46.356 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.768157+0000", 2023-10-09T06:06:46.356 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.356 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.356 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.357 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:46.357 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.768157+0000", 2023-10-09T06:06:46.357 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:06:46.357 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.357 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.357 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:46.357 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.768166+0000", 2023-10-09T06:06:46.357 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9919999999999996e-06 2023-10-09T06:06:46.357 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.357 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.358 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:46.358 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.768179+0000", 2023-10-09T06:06:46.358 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3552e-05 2023-10-09T06:06:46.358 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.358 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.358 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.358 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.768186+0000", 2023-10-09T06:06:46.358 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5459999999999997e-06 2023-10-09T06:06:46.358 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.358 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.359 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.359 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.768252+0000", 2023-10-09T06:06:46.359 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6629000000000002e-05 2023-10-09T06:06:46.359 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.359 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.359 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.359 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.768259+0000", 2023-10-09T06:06:46.359 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4389999999999997e-06 2023-10-09T06:06:46.359 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.359 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.359 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.359 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737077+0000", 2023-10-09T06:06:46.360 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96881812599999995 2023-10-09T06:06:46.360 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.360 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.360 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.360 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737085+0000", 2023-10-09T06:06:46.360 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.4970000000000001e-06 2023-10-09T06:06:46.360 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.360 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.360 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.360 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.870073+0000", 2023-10-09T06:06:46.360 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1329872590000001 2023-10-09T06:06:46.361 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.361 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.361 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:46.361 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.870168+0000", 2023-10-09T06:06:46.361 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.5539999999999997e-05 2023-10-09T06:06:46.361 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.361 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.361 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:06:46.361 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.870401+0000", 2023-10-09T06:06:46.361 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000232465 2023-10-09T06:06:46.361 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.361 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.362 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:06:46.362 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887473+0000", 2023-10-09T06:06:46.362 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.017071975999999999 2023-10-09T06:06:46.362 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.362 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.362 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.362 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900782+0000", 2023-10-09T06:06:46.362 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013308943 2023-10-09T06:06:46.362 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.362 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.362 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.362 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900835+0000", 2023-10-09T06:06:46.363 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3684999999999998e-05 2023-10-09T06:06:46.363 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.363 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.363 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:06:46.363 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900869+0000", 2023-10-09T06:06:46.363 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.3395000000000001e-05 2023-10-09T06:06:46.363 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.363 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.363 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:06:46.363 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900996+0000", 2023-10-09T06:06:46.363 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012716 2023-10-09T06:06:46.363 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.364 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:46.364 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.364 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.364 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.364 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:185975 6.15 6:ad237c98:::benchmark_data_smithi064_284193_object89432:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:06:46.364 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.808092+0000", 2023-10-09T06:06:46.364 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 535.50059582799997, 2023-10-09T06:06:46.364 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.100164849, 2023-10-09T06:06:46.364 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:46.364 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.364 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:06:46.365 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:06:46.365 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:06:46.365 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 185975 2023-10-09T06:06:46.365 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.365 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:46.365 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.365 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:46.365 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.808092+0000", 2023-10-09T06:06:46.365 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.365 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.365 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.365 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:46.366 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.808092+0000", 2023-10-09T06:06:46.366 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.366 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.366 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.366 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:46.366 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.808091+0000", 2023-10-09T06:06:46.366 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:06:46.366 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.366 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.366 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:46.366 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.808098+0000", 2023-10-09T06:06:46.367 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8979999999999999e-06 2023-10-09T06:06:46.367 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.367 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.367 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:46.367 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.808110+0000", 2023-10-09T06:06:46.367 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.269e-05 2023-10-09T06:06:46.367 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.367 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.367 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.367 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.808118+0000", 2023-10-09T06:06:46.367 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1369999999999996e-06 2023-10-09T06:06:46.367 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.367 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.368 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.368 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.808181+0000", 2023-10-09T06:06:46.368 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3409000000000002e-05 2023-10-09T06:06:46.368 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.368 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.368 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.368 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.808190+0000", 2023-10-09T06:06:46.368 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8319999999999995e-06 2023-10-09T06:06:46.368 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.368 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.368 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.369 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735892+0000", 2023-10-09T06:06:46.369 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.92770270899999996 2023-10-09T06:06:46.369 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.369 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.369 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.369 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735904+0000", 2023-10-09T06:06:46.369 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1919999999999999e-05 2023-10-09T06:06:46.369 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.369 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.369 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.369 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.890799+0000", 2023-10-09T06:06:46.370 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1548949850000001 2023-10-09T06:06:46.370 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.370 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.370 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:46.370 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.890900+0000", 2023-10-09T06:06:46.370 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000100787 2023-10-09T06:06:46.370 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.370 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.370 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T06:06:46.370 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.891095+0000", 2023-10-09T06:06:46.370 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00019488099999999999 2023-10-09T06:06:46.371 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.371 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.371 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:06:46.371 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.896810+0000", 2023-10-09T06:06:46.371 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0057148370000000004 2023-10-09T06:06:46.371 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.371 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.371 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.371 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902357+0000", 2023-10-09T06:06:46.371 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0055466739999999997 2023-10-09T06:06:46.371 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.372 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.372 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.372 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902737+0000", 2023-10-09T06:06:46.372 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00038066800000000002 2023-10-09T06:06:46.372 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.372 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.372 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.372 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908154+0000", 2023-10-09T06:06:46.372 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0054167620000000003 2023-10-09T06:06:46.372 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.372 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.373 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:06:46.373 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908191+0000", 2023-10-09T06:06:46.373 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7197000000000003e-05 2023-10-09T06:06:46.373 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.373 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.373 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:06:46.373 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908257+0000", 2023-10-09T06:06:46.373 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5439999999999997e-05 2023-10-09T06:06:46.373 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.373 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:46.374 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.374 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.374 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.374 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:185985 6.5 6:a34f1df4:::benchmark_data_smithi064_284193_object89442:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:06:46.374 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.815985+0000", 2023-10-09T06:06:46.374 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 535.49270257700005, 2023-10-09T06:06:46.374 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0848691029999999, 2023-10-09T06:06:46.374 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:46.374 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.374 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:06:46.374 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:06:46.375 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:06:46.375 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 185985 2023-10-09T06:06:46.375 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.375 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:46.375 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.375 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:46.375 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.815985+0000", 2023-10-09T06:06:46.375 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.375 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.375 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.375 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:46.376 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.815985+0000", 2023-10-09T06:06:46.376 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.376 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.376 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.376 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:46.376 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.815984+0000", 2023-10-09T06:06:46.376 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:06:46.376 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.376 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.376 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:46.376 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.815991+0000", 2023-10-09T06:06:46.376 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.973e-06 2023-10-09T06:06:46.377 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.377 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.377 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:46.377 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.816004+0000", 2023-10-09T06:06:46.377 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3162999999999999e-05 2023-10-09T06:06:46.377 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.377 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.377 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.377 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.816009+0000", 2023-10-09T06:06:46.377 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8049999999999997e-06 2023-10-09T06:06:46.378 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.378 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.378 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.378 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.816081+0000", 2023-10-09T06:06:46.378 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2614000000000006e-05 2023-10-09T06:06:46.378 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.378 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.378 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.378 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.816093+0000", 2023-10-09T06:06:46.378 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1919999999999999e-05 2023-10-09T06:06:46.378 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.379 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.379 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.379 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.738574+0000", 2023-10-09T06:06:46.379 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.922481047 2023-10-09T06:06:46.379 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.379 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.379 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.379 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.738585+0000", 2023-10-09T06:06:46.379 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0457e-05 2023-10-09T06:06:46.379 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.379 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.380 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.380 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.860265+0000", 2023-10-09T06:06:46.380 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1216806349999999 2023-10-09T06:06:46.380 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.380 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.380 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:46.380 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.860401+0000", 2023-10-09T06:06:46.380 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000135158 2023-10-09T06:06:46.380 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.380 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.380 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:06:46.381 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.860685+0000", 2023-10-09T06:06:46.381 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00028465499999999998 2023-10-09T06:06:46.381 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.381 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.381 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.381 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.875933+0000", 2023-10-09T06:06:46.381 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015247671000000001 2023-10-09T06:06:46.381 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.381 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.381 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:06:46.381 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887462+0000", 2023-10-09T06:06:46.382 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011528974000000001 2023-10-09T06:06:46.382 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.382 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.382 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.382 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900446+0000", 2023-10-09T06:06:46.382 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012984446 2023-10-09T06:06:46.382 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.382 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.382 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:06:46.382 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900505+0000", 2023-10-09T06:06:46.383 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8844000000000001e-05 2023-10-09T06:06:46.383 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.383 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.383 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:06:46.383 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900854+0000", 2023-10-09T06:06:46.383 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00034893000000000001 2023-10-09T06:06:46.383 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.383 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:46.383 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.383 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.383 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.383 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186010 6.25 6:a4768b4e:::benchmark_data_smithi064_284193_object89467:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:06:46.384 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.842412+0000", 2023-10-09T06:06:46.384 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 535.46627519399999, 2023-10-09T06:06:46.384 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.058999177, 2023-10-09T06:06:46.384 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:46.384 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.384 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:06:46.384 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:06:46.384 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:06:46.384 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186010 2023-10-09T06:06:46.384 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.384 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:46.385 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.385 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:46.385 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842412+0000", 2023-10-09T06:06:46.385 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.385 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.385 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.385 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:46.385 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842412+0000", 2023-10-09T06:06:46.385 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.385 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.385 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.385 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:46.386 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842411+0000", 2023-10-09T06:06:46.386 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:06:46.386 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.386 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.386 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:46.386 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842424+0000", 2023-10-09T06:06:46.386 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2785999999999999e-05 2023-10-09T06:06:46.386 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.386 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.386 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:46.387 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842440+0000", 2023-10-09T06:06:46.387 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5608e-05 2023-10-09T06:06:46.387 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.387 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.387 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.387 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842445+0000", 2023-10-09T06:06:46.387 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5300000000000004e-06 2023-10-09T06:06:46.387 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.387 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.387 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.387 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842503+0000", 2023-10-09T06:06:46.387 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7425e-05 2023-10-09T06:06:46.388 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.388 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.388 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.388 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842512+0000", 2023-10-09T06:06:46.388 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3090000000000001e-06 2023-10-09T06:06:46.388 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.388 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.388 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.388 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737132+0000", 2023-10-09T06:06:46.388 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.89461971600000001 2023-10-09T06:06:46.388 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.389 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.389 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.389 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737140+0000", 2023-10-09T06:06:46.389 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0819999999999999e-06 2023-10-09T06:06:46.389 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.389 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.389 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.389 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.871339+0000", 2023-10-09T06:06:46.389 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1341993539999999 2023-10-09T06:06:46.389 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.389 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.390 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:46.390 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.871455+0000", 2023-10-09T06:06:46.390 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000116223 2023-10-09T06:06:46.390 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.390 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.390 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:06:46.390 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.871601+0000", 2023-10-09T06:06:46.390 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000145215 2023-10-09T06:06:46.390 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.390 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.391 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:06:46.391 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887480+0000", 2023-10-09T06:06:46.391 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015879000000000001 2023-10-09T06:06:46.391 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.391 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.391 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.391 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900800+0000", 2023-10-09T06:06:46.391 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013320784 2023-10-09T06:06:46.391 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.391 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.391 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.392 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901152+0000", 2023-10-09T06:06:46.392 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00035196099999999997 2023-10-09T06:06:46.392 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.392 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.392 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:06:46.392 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901201+0000", 2023-10-09T06:06:46.392 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9029000000000001e-05 2023-10-09T06:06:46.392 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.392 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.392 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:06:46.392 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901412+0000", 2023-10-09T06:06:46.392 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00021017099999999999 2023-10-09T06:06:46.393 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.393 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:46.393 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.393 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.393 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.393 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186011 6.25 6:a4aaf67a:::benchmark_data_smithi064_284193_object89468:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:06:46.393 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.842461+0000", 2023-10-09T06:06:46.393 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 535.46622680099995, 2023-10-09T06:06:46.393 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.059291977, 2023-10-09T06:06:46.393 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:46.393 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.394 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:06:46.394 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:06:46.394 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:06:46.394 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186011 2023-10-09T06:06:46.394 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.394 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:46.394 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.394 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:46.394 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842461+0000", 2023-10-09T06:06:46.394 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.395 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.395 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.395 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:46.395 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842461+0000", 2023-10-09T06:06:46.395 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.395 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.395 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.395 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:46.395 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842460+0000", 2023-10-09T06:06:46.395 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:06:46.395 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.396 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.396 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:46.396 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842464+0000", 2023-10-09T06:06:46.396 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5949999999999999e-06 2023-10-09T06:06:46.396 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.396 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.396 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:46.396 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842470+0000", 2023-10-09T06:06:46.396 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4500000000000001e-06 2023-10-09T06:06:46.396 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.396 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.397 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.397 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.842475+0000", 2023-10-09T06:06:46.397 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1629999999999998e-06 2023-10-09T06:06:46.397 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.397 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.397 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.397 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.844758+0000", 2023-10-09T06:06:46.397 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002283799 2023-10-09T06:06:46.397 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.397 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.398 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.398 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.844765+0000", 2023-10-09T06:06:46.398 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5250000000000002e-06 2023-10-09T06:06:46.398 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.398 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.398 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.398 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737182+0000", 2023-10-09T06:06:46.398 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.89241688699999999 2023-10-09T06:06:46.398 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.398 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.399 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.399 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737191+0000", 2023-10-09T06:06:46.399 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0219999999999993e-06 2023-10-09T06:06:46.399 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.399 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.399 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.399 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.872509+0000", 2023-10-09T06:06:46.399 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1353186339999999 2023-10-09T06:06:46.399 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.399 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.399 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:46.400 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.872596+0000", 2023-10-09T06:06:46.400 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6112000000000005e-05 2023-10-09T06:06:46.400 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.400 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.400 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:06:46.400 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.872731+0000", 2023-10-09T06:06:46.400 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000134964 2023-10-09T06:06:46.400 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.400 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.400 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:06:46.400 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887486+0000", 2023-10-09T06:06:46.400 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.014755367 2023-10-09T06:06:46.401 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.401 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.401 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.401 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900818+0000", 2023-10-09T06:06:46.401 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013332274 2023-10-09T06:06:46.401 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.401 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.401 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.401 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901502+0000", 2023-10-09T06:06:46.401 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00068389899999999997 2023-10-09T06:06:46.401 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.401 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.401 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:06:46.402 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901562+0000", 2023-10-09T06:06:46.402 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0087999999999998e-05 2023-10-09T06:06:46.402 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.402 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.402 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:06:46.402 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901753+0000", 2023-10-09T06:06:46.402 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00019062199999999999 2023-10-09T06:06:46.402 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.402 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:46.402 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.402 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.403 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.403 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186073 6.15 6:ac3db1f6:::benchmark_data_smithi064_284193_object89530:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:06:46.403 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.899281+0000", 2023-10-09T06:06:46.403 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 535.40940650799996, 2023-10-09T06:06:46.403 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0091446479999999, 2023-10-09T06:06:46.403 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:46.403 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.403 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:06:46.403 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:06:46.403 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:06:46.403 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186073 2023-10-09T06:06:46.404 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.404 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:46.404 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.404 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:46.404 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.899281+0000", 2023-10-09T06:06:46.404 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.404 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.404 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.404 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:46.404 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.899281+0000", 2023-10-09T06:06:46.404 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.405 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.405 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.405 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:46.405 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.899280+0000", 2023-10-09T06:06:46.405 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:06:46.405 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.405 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.405 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:46.405 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.899285+0000", 2023-10-09T06:06:46.405 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7600000000000002e-06 2023-10-09T06:06:46.406 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.406 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.406 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:46.406 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.899294+0000", 2023-10-09T06:06:46.406 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.397e-06 2023-10-09T06:06:46.406 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.406 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.406 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.406 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.899300+0000", 2023-10-09T06:06:46.406 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.412e-06 2023-10-09T06:06:46.407 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.407 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.407 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.407 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.899361+0000", 2023-10-09T06:06:46.407 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1532999999999995e-05 2023-10-09T06:06:46.407 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.407 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.407 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.407 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.899370+0000", 2023-10-09T06:06:46.408 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.4649999999999994e-06 2023-10-09T06:06:46.408 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.408 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.408 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.408 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735922+0000", 2023-10-09T06:06:46.408 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.83655174300000001 2023-10-09T06:06:46.408 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.408 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.408 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.408 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735928+0000", 2023-10-09T06:06:46.409 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4790000000000001e-06 2023-10-09T06:06:46.409 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.409 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.409 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.409 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.892186+0000", 2023-10-09T06:06:46.409 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.156257533 2023-10-09T06:06:46.409 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.409 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.409 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:46.409 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.892275+0000", 2023-10-09T06:06:46.409 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9951000000000007e-05 2023-10-09T06:06:46.409 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.410 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.410 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T06:06:46.410 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.892504+0000", 2023-10-09T06:06:46.410 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00022880200000000001 2023-10-09T06:06:46.410 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.410 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.410 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:06:46.410 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900294+0000", 2023-10-09T06:06:46.410 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.007790033 2023-10-09T06:06:46.411 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.411 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.411 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.411 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902411+0000", 2023-10-09T06:06:46.411 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0021167360000000001 2023-10-09T06:06:46.411 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.411 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.411 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.411 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.906259+0000", 2023-10-09T06:06:46.411 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0038482970000000001 2023-10-09T06:06:46.411 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.412 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.412 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.412 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908294+0000", 2023-10-09T06:06:46.412 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00203426 2023-10-09T06:06:46.412 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.412 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.412 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:06:46.412 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908329+0000", 2023-10-09T06:06:46.412 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5098999999999997e-05 2023-10-09T06:06:46.412 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.413 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.413 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:06:46.413 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908426+0000", 2023-10-09T06:06:46.413 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7058000000000005e-05 2023-10-09T06:06:46.413 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.413 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:46.413 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.413 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.413 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.413 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186096 6.25 6:a53a4882:::benchmark_data_smithi064_284193_object89553:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:06:46.414 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.926438+0000", 2023-10-09T06:06:46.414 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 535.38224994400002, 2023-10-09T06:06:46.414 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.975667222, 2023-10-09T06:06:46.414 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:46.414 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.414 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:06:46.414 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:06:46.414 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:06:46.414 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186096 2023-10-09T06:06:46.415 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.415 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:46.415 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.415 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:46.415 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.926438+0000", 2023-10-09T06:06:46.415 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.415 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.415 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.415 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:46.415 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.926438+0000", 2023-10-09T06:06:46.415 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.416 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.416 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.416 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:46.416 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.926436+0000", 2023-10-09T06:06:46.416 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:06:46.416 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.416 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.416 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:46.416 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.926443+0000", 2023-10-09T06:06:46.416 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3099999999999997e-06 2023-10-09T06:06:46.416 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.416 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.417 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:46.417 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.926455+0000", 2023-10-09T06:06:46.417 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2026e-05 2023-10-09T06:06:46.417 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.417 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.417 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.417 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.926460+0000", 2023-10-09T06:06:46.417 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8379999999999998e-06 2023-10-09T06:06:46.417 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.417 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.418 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.418 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.926522+0000", 2023-10-09T06:06:46.418 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1502999999999997e-05 2023-10-09T06:06:46.418 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.418 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.418 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.418 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.926531+0000", 2023-10-09T06:06:46.418 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6559999999999998e-06 2023-10-09T06:06:46.418 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.418 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.419 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.419 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737222+0000", 2023-10-09T06:06:46.419 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.81069113400000004 2023-10-09T06:06:46.419 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.419 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.419 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.419 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.737229+0000", 2023-10-09T06:06:46.419 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6669999999999996e-06 2023-10-09T06:06:46.419 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.419 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.419 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.420 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.873463+0000", 2023-10-09T06:06:46.420 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1362332390000001 2023-10-09T06:06:46.420 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.420 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.420 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:46.420 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.873604+0000", 2023-10-09T06:06:46.420 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000141108 2023-10-09T06:06:46.420 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.420 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.420 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:06:46.420 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.873782+0000", 2023-10-09T06:06:46.420 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00017831699999999999 2023-10-09T06:06:46.421 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.421 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.421 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:06:46.421 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.887492+0000", 2023-10-09T06:06:46.421 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013710119 2023-10-09T06:06:46.421 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.421 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.421 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.421 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901054+0000", 2023-10-09T06:06:46.421 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013561478 2023-10-09T06:06:46.421 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.422 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.422 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.422 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901854+0000", 2023-10-09T06:06:46.422 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00079998700000000005 2023-10-09T06:06:46.422 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.422 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.422 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:06:46.422 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.901914+0000", 2023-10-09T06:06:46.422 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0770000000000003e-05 2023-10-09T06:06:46.422 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.422 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.422 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:06:46.423 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902105+0000", 2023-10-09T06:06:46.423 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000190497 2023-10-09T06:06:46.423 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.423 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:46.423 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.423 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.423 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.423 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186101 6.15 6:ac185365:::benchmark_data_smithi064_284193_object89558:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:06:46.423 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.933029+0000", 2023-10-09T06:06:46.423 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 535.37565911299998, 2023-10-09T06:06:46.423 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.97558872, 2023-10-09T06:06:46.423 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:46.424 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.424 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:06:46.424 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:06:46.424 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:06:46.424 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186101 2023-10-09T06:06:46.424 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.424 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:46.424 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.424 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:46.424 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.933029+0000", 2023-10-09T06:06:46.424 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.425 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.425 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.425 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:46.425 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.933029+0000", 2023-10-09T06:06:46.425 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.425 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.425 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.425 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:46.425 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.933027+0000", 2023-10-09T06:06:46.425 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:06:46.425 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.426 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.426 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:46.426 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.933033+0000", 2023-10-09T06:06:46.426 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1380000000000004e-06 2023-10-09T06:06:46.426 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.426 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.426 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:46.426 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.933048+0000", 2023-10-09T06:06:46.426 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4273e-05 2023-10-09T06:06:46.426 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.426 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.426 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.427 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.933053+0000", 2023-10-09T06:06:46.427 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.699e-06 2023-10-09T06:06:46.427 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.427 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.427 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.427 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.933117+0000", 2023-10-09T06:06:46.427 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3668000000000001e-05 2023-10-09T06:06:46.427 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.427 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.427 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.427 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.933125+0000", 2023-10-09T06:06:46.427 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8990000000000001e-06 2023-10-09T06:06:46.427 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.428 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.428 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.428 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735946+0000", 2023-10-09T06:06:46.428 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.802820796 2023-10-09T06:06:46.428 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.428 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.428 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.428 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735952+0000", 2023-10-09T06:06:46.428 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3230000000000003e-06 2023-10-09T06:06:46.428 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.428 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.429 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.429 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.893777+0000", 2023-10-09T06:06:46.429 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1578248010000001 2023-10-09T06:06:46.429 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.429 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.429 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:46.429 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.893902+0000", 2023-10-09T06:06:46.429 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012566899999999999 2023-10-09T06:06:46.429 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.429 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.429 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T06:06:46.430 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.894163+0000", 2023-10-09T06:06:46.430 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000260253 2023-10-09T06:06:46.430 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.430 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.430 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:06:46.430 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900304+0000", 2023-10-09T06:06:46.430 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0061414939999999999 2023-10-09T06:06:46.430 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.430 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.430 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.431 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902438+0000", 2023-10-09T06:06:46.431 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0021343109999999998 2023-10-09T06:06:46.431 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.431 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.431 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.431 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.906306+0000", 2023-10-09T06:06:46.431 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0038679690000000002 2023-10-09T06:06:46.431 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.431 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.431 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.431 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908502+0000", 2023-10-09T06:06:46.431 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00219549 2023-10-09T06:06:46.433 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.433 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.433 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:06:46.433 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908544+0000", 2023-10-09T06:06:46.433 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2407000000000002e-05 2023-10-09T06:06:46.433 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.433 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.433 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:06:46.433 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908617+0000", 2023-10-09T06:06:46.434 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2891000000000004e-05 2023-10-09T06:06:46.434 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.434 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:46.434 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.434 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.434 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.434 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186105 6.15 6:a80320df:::benchmark_data_smithi064_284193_object89562:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:06:46.434 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.936806+0000", 2023-10-09T06:06:46.434 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 535.37188167700003, 2023-10-09T06:06:46.434 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.971922425, 2023-10-09T06:06:46.435 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:46.435 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.435 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:06:46.435 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:06:46.435 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:06:46.435 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186105 2023-10-09T06:06:46.435 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.435 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:46.435 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.435 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:46.436 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.936806+0000", 2023-10-09T06:06:46.436 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.436 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.436 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.436 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:46.436 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.936806+0000", 2023-10-09T06:06:46.436 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.436 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.436 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.436 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:46.436 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.936805+0000", 2023-10-09T06:06:46.436 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:06:46.436 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.437 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.437 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:46.437 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.936812+0000", 2023-10-09T06:06:46.437 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0870000000000001e-06 2023-10-09T06:06:46.437 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.437 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.437 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:46.437 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.936826+0000", 2023-10-09T06:06:46.437 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4511999999999999e-05 2023-10-09T06:06:46.437 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.437 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.438 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.438 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.936833+0000", 2023-10-09T06:06:46.438 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9140000000000002e-06 2023-10-09T06:06:46.438 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.438 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.438 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.438 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.936903+0000", 2023-10-09T06:06:46.438 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0197999999999994e-05 2023-10-09T06:06:46.438 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.438 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.438 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.439 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.936913+0000", 2023-10-09T06:06:46.439 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3549999999999994e-06 2023-10-09T06:06:46.439 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.439 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.439 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.439 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735976+0000", 2023-10-09T06:06:46.439 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.79906334099999998 2023-10-09T06:06:46.439 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.439 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.439 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.439 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.735983+0000", 2023-10-09T06:06:46.440 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8859999999999998e-06 2023-10-09T06:06:46.440 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.440 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.440 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.440 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.895376+0000", 2023-10-09T06:06:46.440 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.159393398 2023-10-09T06:06:46.440 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.440 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.440 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:46.440 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.895479+0000", 2023-10-09T06:06:46.440 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000103028 2023-10-09T06:06:46.441 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.441 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.441 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T06:06:46.441 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.895680+0000", 2023-10-09T06:06:46.441 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000200216 2023-10-09T06:06:46.441 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.441 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.441 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:06:46.441 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900382+0000", 2023-10-09T06:06:46.441 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0047027529999999996 2023-10-09T06:06:46.441 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.442 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.442 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.442 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902767+0000", 2023-10-09T06:06:46.442 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0023844389999999999 2023-10-09T06:06:46.442 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.442 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.442 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.442 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.906331+0000", 2023-10-09T06:06:46.442 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0035642719999999998 2023-10-09T06:06:46.442 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.442 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.442 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.443 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908649+0000", 2023-10-09T06:06:46.443 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0023178819999999998 2023-10-09T06:06:46.443 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.443 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.443 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:06:46.443 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908673+0000", 2023-10-09T06:06:46.443 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4329999999999999e-05 2023-10-09T06:06:46.443 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.443 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.443 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:06:46.443 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908728+0000", 2023-10-09T06:06:46.443 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5074999999999997e-05 2023-10-09T06:06:46.444 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.444 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:46.444 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.444 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.444 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.444 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186107 6.15 6:adc2ed22:::benchmark_data_smithi064_284193_object89564:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:06:46.444 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.937578+0000", 2023-10-09T06:06:46.444 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 535.37110966700004, 2023-10-09T06:06:46.444 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9730703780000001, 2023-10-09T06:06:46.444 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:46.444 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.444 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:06:46.445 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:06:46.445 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:06:46.445 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186107 2023-10-09T06:06:46.445 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.445 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:46.445 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.445 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:46.445 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.937578+0000", 2023-10-09T06:06:46.445 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.445 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.445 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.445 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:46.446 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.937578+0000", 2023-10-09T06:06:46.446 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.446 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.446 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.446 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:46.446 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.937577+0000", 2023-10-09T06:06:46.446 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:06:46.446 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.446 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.446 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:46.447 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.937586+0000", 2023-10-09T06:06:46.447 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0250000000000008e-06 2023-10-09T06:06:46.447 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.447 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.447 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:46.447 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.937601+0000", 2023-10-09T06:06:46.447 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5301000000000001e-05 2023-10-09T06:06:46.447 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.447 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.447 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.447 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.937608+0000", 2023-10-09T06:06:46.447 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1689999999999997e-06 2023-10-09T06:06:46.448 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.448 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.448 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.448 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.937654+0000", 2023-10-09T06:06:46.448 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6179e-05 2023-10-09T06:06:46.448 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.448 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.448 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.448 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.937660+0000", 2023-10-09T06:06:46.448 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3640000000000004e-06 2023-10-09T06:06:46.448 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.449 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.449 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.449 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.736002+0000", 2023-10-09T06:06:46.449 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.79834211899999996 2023-10-09T06:06:46.449 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.449 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.449 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.449 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.736009+0000", 2023-10-09T06:06:46.449 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4899999999999997e-06 2023-10-09T06:06:46.449 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.449 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.450 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.450 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.896857+0000", 2023-10-09T06:06:46.450 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1608479190000001 2023-10-09T06:06:46.450 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.450 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.450 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:46.450 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.896983+0000", 2023-10-09T06:06:46.450 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000125962 2023-10-09T06:06:46.450 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.450 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.451 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T06:06:46.451 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.897240+0000", 2023-10-09T06:06:46.451 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000257753 2023-10-09T06:06:46.451 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.451 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.451 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:06:46.451 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900398+0000", 2023-10-09T06:06:46.451 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0031575539999999999 2023-10-09T06:06:46.451 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.451 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.451 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.451 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.902797+0000", 2023-10-09T06:06:46.452 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0023991149999999998 2023-10-09T06:06:46.452 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.452 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.452 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.452 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908771+0000", 2023-10-09T06:06:46.452 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0059742579999999997 2023-10-09T06:06:46.452 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.452 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.452 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.452 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.910483+0000", 2023-10-09T06:06:46.453 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0017115489999999999 2023-10-09T06:06:46.453 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.453 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.453 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:06:46.453 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.910530+0000", 2023-10-09T06:06:46.453 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7488999999999999e-05 2023-10-09T06:06:46.453 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.453 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.453 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:06:46.453 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.910648+0000", 2023-10-09T06:06:46.453 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000117941 2023-10-09T06:06:46.453 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.454 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:46.454 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.454 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.454 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.454 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.44588.0:186109 6.15 6:ac6fdfb9:::benchmark_data_smithi064_284193_object89566:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T06:06:46.454 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T05:57:50.940173+0000", 2023-10-09T06:06:46.454 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 535.36851449599999, 2023-10-09T06:06:46.454 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9706640229999999, 2023-10-09T06:06:46.454 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:46.454 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.455 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:06:46.455 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.44588", 2023-10-09T06:06:46.455 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/3217670227", 2023-10-09T06:06:46.455 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 186109 2023-10-09T06:06:46.455 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.455 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:46.455 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.455 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:46.455 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.940173+0000", 2023-10-09T06:06:46.455 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.455 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.455 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.456 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:46.456 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.940173+0000", 2023-10-09T06:06:46.456 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.456 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.456 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.456 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:46.456 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.940172+0000", 2023-10-09T06:06:46.456 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:06:46.456 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.456 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.456 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:46.456 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.940177+0000", 2023-10-09T06:06:46.457 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4750000000000004e-06 2023-10-09T06:06:46.457 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.457 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.457 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:46.457 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.940188+0000", 2023-10-09T06:06:46.457 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1508999999999999e-05 2023-10-09T06:06:46.457 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.457 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.457 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.457 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.940193+0000", 2023-10-09T06:06:46.457 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.147e-06 2023-10-09T06:06:46.458 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.458 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.458 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.458 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.940244+0000", 2023-10-09T06:06:46.458 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0552e-05 2023-10-09T06:06:46.458 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.458 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.458 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.458 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:50.940254+0000", 2023-10-09T06:06:46.458 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0096e-05 2023-10-09T06:06:46.458 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.459 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.459 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.459 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.736034+0000", 2023-10-09T06:06:46.459 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.79577992900000005 2023-10-09T06:06:46.459 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.459 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.459 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.459 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:51.736041+0000", 2023-10-09T06:06:46.459 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.849e-06 2023-10-09T06:06:46.459 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.459 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.460 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.460 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.898427+0000", 2023-10-09T06:06:46.460 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.162385929 2023-10-09T06:06:46.460 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.460 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.460 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:46.460 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.898534+0000", 2023-10-09T06:06:46.460 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000107227 2023-10-09T06:06:46.460 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.460 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.460 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T06:06:46.461 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.898738+0000", 2023-10-09T06:06:46.461 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000203928 2023-10-09T06:06:46.461 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.461 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.461 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:06:46.461 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.900865+0000", 2023-10-09T06:06:46.461 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0021271129999999999 2023-10-09T06:06:46.461 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.461 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.461 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.461 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.903326+0000", 2023-10-09T06:06:46.462 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002460991 2023-10-09T06:06:46.462 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.462 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.462 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.462 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.908797+0000", 2023-10-09T06:06:46.462 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0054708930000000001 2023-10-09T06:06:46.462 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.462 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.462 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.462 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.910685+0000", 2023-10-09T06:06:46.463 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0018880940000000001 2023-10-09T06:06:46.463 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.463 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.463 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:06:46.463 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.910732+0000", 2023-10-09T06:06:46.463 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7129000000000002e-05 2023-10-09T06:06:46.463 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.463 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.463 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:06:46.463 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T05:57:52.910837+0000", 2023-10-09T06:06:46.463 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000104962 2023-10-09T06:06:46.464 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.464 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:46.464 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.464 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.464 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.464 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:134387 8.d 8:b264af51:::benchmark_data_smithi064_457003_object29869:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-10-09T06:06:46.464 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:10.564773+0000", 2023-10-09T06:06:46.464 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 95.743914203000003, 2023-10-09T06:06:46.464 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1306900139999998, 2023-10-09T06:06:46.464 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:46.464 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.465 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:06:46.465 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:06:46.465 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:06:46.465 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 134387 2023-10-09T06:06:46.465 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.465 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:46.465 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.465 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:46.465 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.564773+0000", 2023-10-09T06:06:46.465 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.466 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.466 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.466 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:46.466 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.564773+0000", 2023-10-09T06:06:46.466 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.466 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.466 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.466 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:46.466 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.564772+0000", 2023-10-09T06:06:46.466 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:06:46.466 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.467 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.467 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:46.467 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.564778+0000", 2023-10-09T06:06:46.467 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3600000000000004e-06 2023-10-09T06:06:46.467 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.467 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.467 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:46.467 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.564792+0000", 2023-10-09T06:06:46.467 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4109e-05 2023-10-09T06:06:46.467 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.467 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.468 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.468 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.564798+0000", 2023-10-09T06:06:46.468 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1829999999999999e-06 2023-10-09T06:06:46.468 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.468 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.468 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.468 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.564842+0000", 2023-10-09T06:06:46.468 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4147999999999998e-05 2023-10-09T06:06:46.468 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.468 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.468 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.469 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.564849+0000", 2023-10-09T06:06:46.469 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.652e-06 2023-10-09T06:06:46.469 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.469 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.469 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.469 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:11.450042+0000", 2023-10-09T06:06:46.469 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.88519274999999997 2023-10-09T06:06:46.469 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.469 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.469 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.469 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:11.450054+0000", 2023-10-09T06:06:46.470 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2559e-05 2023-10-09T06:06:46.470 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.470 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.470 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.470 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.582018+0000", 2023-10-09T06:06:46.470 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1319637280000001 2023-10-09T06:06:46.470 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.470 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.470 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:46.470 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.582181+0000", 2023-10-09T06:06:46.471 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00016312600000000001 2023-10-09T06:06:46.471 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.471 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.471 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:06:46.471 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.582742+0000", 2023-10-09T06:06:46.471 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00056150299999999998 2023-10-09T06:06:46.471 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.471 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.471 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:06:46.471 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.590865+0000", 2023-10-09T06:06:46.471 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0081227209999999994 2023-10-09T06:06:46.471 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.471 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.472 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.472 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.591171+0000", 2023-10-09T06:06:46.472 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00030548100000000003 2023-10-09T06:06:46.472 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.472 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.472 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.472 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.591371+0000", 2023-10-09T06:06:46.472 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00019998600000000001 2023-10-09T06:06:46.472 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.472 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.472 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.473 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.695356+0000", 2023-10-09T06:06:46.473 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.103985074 2023-10-09T06:06:46.473 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.473 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.473 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:06:46.473 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.695398+0000", 2023-10-09T06:06:46.473 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2641000000000001e-05 2023-10-09T06:06:46.473 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.473 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.473 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:06:46.474 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.695463+0000", 2023-10-09T06:06:46.474 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5043999999999996e-05 2023-10-09T06:06:46.474 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.474 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:46.474 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.474 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.474 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.474 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:134404 8.d 8:bd17c3be:::benchmark_data_smithi064_457003_object29886:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-10-09T06:06:46.474 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:10.590604+0000", 2023-10-09T06:06:46.474 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 95.718083266999997, 2023-10-09T06:06:46.474 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1059837670000001, 2023-10-09T06:06:46.475 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:46.475 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.475 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:06:46.475 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:06:46.475 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:06:46.475 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 134404 2023-10-09T06:06:46.475 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.475 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:46.475 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.475 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:46.476 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590604+0000", 2023-10-09T06:06:46.476 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.476 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.476 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.476 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:46.476 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590604+0000", 2023-10-09T06:06:46.476 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.476 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.476 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.476 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:46.477 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590603+0000", 2023-10-09T06:06:46.477 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:06:46.477 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.477 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.477 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:46.477 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590609+0000", 2023-10-09T06:06:46.477 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6620000000000002e-06 2023-10-09T06:06:46.477 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.477 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.477 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:46.478 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590621+0000", 2023-10-09T06:06:46.478 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.254e-05 2023-10-09T06:06:46.478 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.478 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.478 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.478 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590626+0000", 2023-10-09T06:06:46.478 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6840000000000004e-06 2023-10-09T06:06:46.478 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.478 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.478 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.479 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590680+0000", 2023-10-09T06:06:46.479 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4625e-05 2023-10-09T06:06:46.479 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.479 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.479 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.479 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590691+0000", 2023-10-09T06:06:46.479 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0981999999999999e-05 2023-10-09T06:06:46.479 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.479 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.480 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.480 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:11.450082+0000", 2023-10-09T06:06:46.480 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.85939068100000005 2023-10-09T06:06:46.480 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.480 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.480 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.480 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:11.450091+0000", 2023-10-09T06:06:46.480 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.5590000000000006e-06 2023-10-09T06:06:46.480 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.480 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.481 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.481 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.587676+0000", 2023-10-09T06:06:46.481 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1375855880000001 2023-10-09T06:06:46.481 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.481 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.481 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:46.481 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.587819+0000", 2023-10-09T06:06:46.481 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014301599999999999 2023-10-09T06:06:46.481 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.481 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.481 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:06:46.482 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.587967+0000", 2023-10-09T06:06:46.482 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000148133 2023-10-09T06:06:46.482 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.482 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.482 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:06:46.482 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.591403+0000", 2023-10-09T06:06:46.482 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.003435235 2023-10-09T06:06:46.482 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.482 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.482 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.482 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.596927+0000", 2023-10-09T06:06:46.483 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0055241159999999999 2023-10-09T06:06:46.483 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.483 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.483 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.483 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.597280+0000", 2023-10-09T06:06:46.483 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00035289300000000001 2023-10-09T06:06:46.483 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.483 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.483 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.483 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.696456+0000", 2023-10-09T06:06:46.483 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.099176147000000006 2023-10-09T06:06:46.484 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.484 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.484 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:06:46.484 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.696499+0000", 2023-10-09T06:06:46.484 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3584999999999997e-05 2023-10-09T06:06:46.484 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.484 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.484 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:06:46.484 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.696588+0000", 2023-10-09T06:06:46.484 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8826000000000006e-05 2023-10-09T06:06:46.484 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.485 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:46.485 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.485 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.485 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.485 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:148095 8.d 8:bd028436:::benchmark_data_smithi064_457003_object43577:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e389)", 2023-10-09T06:06:46.485 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:31.553767+0000", 2023-10-09T06:06:46.485 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 74.754920511999998, 2023-10-09T06:06:46.485 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0552826390000001, 2023-10-09T06:06:46.485 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:46.485 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.485 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:06:46.486 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:06:46.486 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:06:46.486 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 148095 2023-10-09T06:06:46.486 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.486 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:46.486 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.486 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:46.486 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.553767+0000", 2023-10-09T06:06:46.486 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.486 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.486 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.487 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:46.487 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.553767+0000", 2023-10-09T06:06:46.487 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.487 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.487 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.487 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:46.487 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.553766+0000", 2023-10-09T06:06:46.487 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:06:46.487 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:46.489 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.489 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.489 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:46.489 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.553770+0000", 2023-10-09T06:06:46.489 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1250000000000003e-06 2023-10-09T06:06:46.489 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.489 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.489 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:46.489 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.553779+0000", 2023-10-09T06:06:46.489 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.5300000000000002e-06 2023-10-09T06:06:46.490 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.490 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.490 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.490 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.553785+0000", 2023-10-09T06:06:46.490 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6350000000000003e-06 2023-10-09T06:06:46.490 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.490 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.490 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.490 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.554078+0000", 2023-10-09T06:06:46.490 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00029307500000000001 2023-10-09T06:06:46.490 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.491 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.491 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.491 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.554086+0000", 2023-10-09T06:06:46.491 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.3580000000000003e-06 2023-10-09T06:06:46.491 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.491 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.491 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.491 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:32.420947+0000", 2023-10-09T06:06:46.491 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.86686022200000001 2023-10-09T06:06:46.491 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.491 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.492 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.492 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:32.420958+0000", 2023-10-09T06:06:46.492 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1759e-05 2023-10-09T06:06:46.492 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.492 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.492 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.492 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.556878+0000", 2023-10-09T06:06:46.492 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.135919782 2023-10-09T06:06:46.492 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.492 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.492 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:46.492 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.556994+0000", 2023-10-09T06:06:46.493 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011585499999999999 2023-10-09T06:06:46.493 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.493 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.493 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T06:06:46.493 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.557534+0000", 2023-10-09T06:06:46.493 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00054018100000000004 2023-10-09T06:06:46.493 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.493 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.493 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:06:46.493 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.567819+0000", 2023-10-09T06:06:46.493 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010284695999999999 2023-10-09T06:06:46.493 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.494 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.494 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.494 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.608747+0000", 2023-10-09T06:06:46.494 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.040927951999999997 2023-10-09T06:06:46.494 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.494 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.494 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.494 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.608813+0000", 2023-10-09T06:06:46.494 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5950000000000004e-05 2023-10-09T06:06:46.494 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.494 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.495 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.495 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.608861+0000", 2023-10-09T06:06:46.495 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8052e-05 2023-10-09T06:06:46.495 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.495 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.495 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:06:46.495 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.608911+0000", 2023-10-09T06:06:46.495 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0008e-05 2023-10-09T06:06:46.495 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.495 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.495 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:06:46.495 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.609050+0000", 2023-10-09T06:06:46.496 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013881700000000001 2023-10-09T06:06:46.496 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.496 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:46.496 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.496 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.496 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.496 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:148097 8.d 8:be110a7d:::benchmark_data_smithi064_457003_object43579:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e389)", 2023-10-09T06:06:46.496 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:31.553792+0000", 2023-10-09T06:06:46.496 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 74.754895571999995, 2023-10-09T06:06:46.496 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0555980979999999, 2023-10-09T06:06:46.496 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:46.496 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.497 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:06:46.497 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:06:46.497 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:06:46.497 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 148097 2023-10-09T06:06:46.497 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.497 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:46.497 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.497 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:46.497 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.553792+0000", 2023-10-09T06:06:46.497 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.497 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.498 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.498 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:46.498 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.553792+0000", 2023-10-09T06:06:46.498 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.498 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.498 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.498 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:46.498 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.553792+0000", 2023-10-09T06:06:46.498 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:06:46.498 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.498 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.498 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:46.499 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.553794+0000", 2023-10-09T06:06:46.499 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2819999999999999e-06 2023-10-09T06:06:46.499 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.499 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.499 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:46.499 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.553800+0000", 2023-10-09T06:06:46.499 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4689999999999996e-06 2023-10-09T06:06:46.499 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.499 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.499 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.499 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.553803+0000", 2023-10-09T06:06:46.500 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8099999999999999e-06 2023-10-09T06:06:46.500 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.500 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.500 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.500 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.554150+0000", 2023-10-09T06:06:46.500 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00034632100000000002 2023-10-09T06:06:46.500 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.500 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.500 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.500 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.554156+0000", 2023-10-09T06:06:46.500 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6200000000000001e-06 2023-10-09T06:06:46.501 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.501 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.501 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.501 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:32.420988+0000", 2023-10-09T06:06:46.501 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.866831295 2023-10-09T06:06:46.501 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.501 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.501 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.501 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:32.420995+0000", 2023-10-09T06:06:46.501 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6299999999999998e-06 2023-10-09T06:06:46.501 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.502 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.502 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.502 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.561631+0000", 2023-10-09T06:06:46.502 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1406362379999999 2023-10-09T06:06:46.502 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.502 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.502 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:46.502 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.561721+0000", 2023-10-09T06:06:46.502 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9339999999999995e-05 2023-10-09T06:06:46.502 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.502 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.503 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T06:06:46.503 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.561833+0000", 2023-10-09T06:06:46.503 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000112286 2023-10-09T06:06:46.503 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.503 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.503 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:06:46.503 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.571366+0000", 2023-10-09T06:06:46.503 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0095330120000000004 2023-10-09T06:06:46.503 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.503 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.503 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.503 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.609100+0000", 2023-10-09T06:06:46.504 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.037734151 2023-10-09T06:06:46.504 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.504 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.504 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.504 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.609154+0000", 2023-10-09T06:06:46.504 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3628999999999998e-05 2023-10-09T06:06:46.504 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.504 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.504 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.504 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.609217+0000", 2023-10-09T06:06:46.505 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3595000000000004e-05 2023-10-09T06:06:46.505 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.505 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.505 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:06:46.505 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.609272+0000", 2023-10-09T06:06:46.505 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4932000000000002e-05 2023-10-09T06:06:46.505 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.505 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.505 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:06:46.505 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.609390+0000", 2023-10-09T06:06:46.505 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000117782 2023-10-09T06:06:46.506 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.506 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:46.506 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.506 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.506 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.506 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:148135 8.d 8:b2daf003:::benchmark_data_smithi064_457003_object43617:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e389)", 2023-10-09T06:06:46.506 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:31.608060+0000", 2023-10-09T06:06:46.506 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 74.700627372, 2023-10-09T06:06:46.506 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0017507889999999, 2023-10-09T06:06:46.506 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:06:46.506 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:06:46.507 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:06:46.507 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:06:46.507 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:06:46.507 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 148135 2023-10-09T06:06:46.507 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.507 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:06:46.507 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.507 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:06:46.507 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.608060+0000", 2023-10-09T06:06:46.507 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.507 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.507 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.508 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:06:46.508 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.608060+0000", 2023-10-09T06:06:46.508 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:06:46.508 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.508 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.508 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:06:46.508 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.608059+0000", 2023-10-09T06:06:46.508 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:06:46.508 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.508 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.508 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:06:46.508 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.608067+0000", 2023-10-09T06:06:46.509 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5000000000000002e-06 2023-10-09T06:06:46.509 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.509 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.509 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:06:46.509 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.608083+0000", 2023-10-09T06:06:46.509 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6328e-05 2023-10-09T06:06:46.509 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.509 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.509 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:06:46.509 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.608090+0000", 2023-10-09T06:06:46.510 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3509999999999996e-06 2023-10-09T06:06:46.510 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.510 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.510 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.510 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.608152+0000", 2023-10-09T06:06:46.510 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1729000000000005e-05 2023-10-09T06:06:46.510 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.510 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.510 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.510 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.608162+0000", 2023-10-09T06:06:46.510 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0149999999999999e-05 2023-10-09T06:06:46.511 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.511 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.511 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.511 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:32.421010+0000", 2023-10-09T06:06:46.511 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.81284742899999995 2023-10-09T06:06:46.511 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.511 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.511 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:06:46.511 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:32.421014+0000", 2023-10-09T06:06:46.511 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8770000000000003e-06 2023-10-09T06:06:46.511 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.512 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.512 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:06:46.512 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.564679+0000", 2023-10-09T06:06:46.512 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.143665806 2023-10-09T06:06:46.512 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.512 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.512 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:06:46.512 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.564811+0000", 2023-10-09T06:06:46.512 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000131505 2023-10-09T06:06:46.512 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.512 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.513 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T06:06:46.513 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.564954+0000", 2023-10-09T06:06:46.513 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014360299999999999 2023-10-09T06:06:46.513 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.513 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.513 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:06:46.513 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.574596+0000", 2023-10-09T06:06:46.513 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0096416899999999996 2023-10-09T06:06:46.513 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.513 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.513 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.513 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.609438+0000", 2023-10-09T06:06:46.514 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.034841346000000002 2023-10-09T06:06:46.514 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.514 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.514 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.514 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.609592+0000", 2023-10-09T06:06:46.514 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000154716 2023-10-09T06:06:46.514 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.514 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.514 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:06:46.514 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.609634+0000", 2023-10-09T06:06:46.514 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2018999999999999e-05 2023-10-09T06:06:46.515 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.515 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.515 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:06:46.515 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.609683+0000", 2023-10-09T06:06:46.515 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8888999999999999e-05 2023-10-09T06:06:46.515 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:06:46.515 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:06:46.515 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:06:46.515 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.609811+0000", 2023-10-09T06:06:46.515 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012789999999999999 2023-10-09T06:06:46.515 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.516 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:46.516 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.516 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:06:46.516 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:06:46.516 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:06:46.558 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:06:46.580 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:06:46.580 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:46.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:46 smithi156 ceph-mon[195101]: pgmap v1022: 119 pgs: 119 active+clean; 508 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 749 op/s 2023-10-09T06:06:46.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:46 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3772943304' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:06:46.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:46 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1835400546' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:46.624 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:06:46.673 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:06:46.731 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:06:46.773 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:06:46.786 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:46.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:46.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:46.794 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:46 smithi064 ceph-mon[220044]: pgmap v1022: 119 pgs: 119 active+clean; 508 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 749 op/s 2023-10-09T06:06:46.795 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:46 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3772943304' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:06:46.795 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:46 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1835400546' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:46.795 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:46 smithi064 ceph-mon[212750]: pgmap v1022: 119 pgs: 119 active+clean; 508 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 749 op/s 2023-10-09T06:06:46.795 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:46 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3772943304' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:06:46.795 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:46 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1835400546' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:46.810 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:06:46.848 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:06:46.878 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:06:46.883 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:06:46.883 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:06:46.883 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:06:46.884 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:06:46.884 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:06:46.884 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:06:46.884 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:06:46.884 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:06:46.895 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:46.942 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:47.050 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:47.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:47.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:47.156 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:47.188 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:47.268 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:06:47.273 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:47.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:47.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:47.378 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:47.464 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:47.531 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:47 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:47.531 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:47 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3772943304' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:06:47.531 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:47 smithi156 ceph-mon[195101]: osdmap e417: 8 total, 8 up, 7 in 2023-10-09T06:06:47.531 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:47 smithi156 ceph-mon[195101]: failed to encode map e417 with expected crc 2023-10-09T06:06:47.531 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:47 smithi156 ceph-mon[195101]: failed to encode map e417 with expected crc 2023-10-09T06:06:47.531 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:47 smithi156 ceph-mon[195101]: failed to encode map e417 with expected crc 2023-10-09T06:06:47.531 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:47 smithi156 ceph-mon[195101]: failed to encode map e417 with expected crc 2023-10-09T06:06:47.635 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:47 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:47.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:47 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3772943304' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:06:47.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:47 smithi064 ceph-mon[220044]: osdmap e417: 8 total, 8 up, 7 in 2023-10-09T06:06:47.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:47 smithi064 ceph-mon[220044]: failed to encode map e417 with expected crc 2023-10-09T06:06:47.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:47 smithi064 ceph-mon[220044]: failed to encode map e417 with expected crc 2023-10-09T06:06:47.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:47 smithi064 ceph-mon[220044]: failed to encode map e417 with expected crc 2023-10-09T06:06:47.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:47 smithi064 ceph-mon[220044]: failed to encode map e417 with expected crc 2023-10-09T06:06:47.637 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:47 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:06:47.637 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:47 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3772943304' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:06:47.637 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:47 smithi064 ceph-mon[212750]: osdmap e417: 8 total, 8 up, 7 in 2023-10-09T06:06:47.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:47 smithi064 ceph-mon[212750]: failed to encode map e417 with expected crc 2023-10-09T06:06:47.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:47 smithi064 ceph-mon[212750]: failed to encode map e417 with expected crc 2023-10-09T06:06:47.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:47 smithi064 ceph-mon[212750]: failed to encode map e417 with expected crc 2023-10-09T06:06:47.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:47 smithi064 ceph-mon[212750]: failed to encode map e417 with expected crc 2023-10-09T06:06:48.100 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:48.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:48.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:48.209 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:48.259 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:48.336 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:48.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:48.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:48.346 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:48.416 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1] out_osds: [2] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 0, 4] 2023-10-09T06:06:48.416 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:06:48.416 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:06:48.416 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.332584 2023-10-09T06:06:48.417 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 0.33258394583658757 2023-10-09T06:06:48.442 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:48.532 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:48.572 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:48 smithi156 ceph-mon[195101]: pgmap v1024: 119 pgs: 119 active+clean; 315 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s 2023-10-09T06:06:48.692 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:48.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:48.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:48.765 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:48 smithi064 ceph-mon[212750]: pgmap v1024: 119 pgs: 119 active+clean; 315 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s 2023-10-09T06:06:48.772 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:48 smithi064 ceph-mon[220044]: pgmap v1024: 119 pgs: 119 active+clean; 315 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s 2023-10-09T06:06:48.800 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:48.864 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:49.183 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:49.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:49.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:49.293 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:49.347 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:49.446 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:49.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:49.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:49.551 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:49.578 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:49.629 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:49.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:49.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:49.740 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:49.804 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:49.954 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:49.967 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:06:49.973 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:49.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:49.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:50.081 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:50.192 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:06:50.211 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:50.484 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Removed 104515 objects 2023-10-09T06:06:50.484 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Clean up completed and total clean up time :137.695 2023-10-09T06:06:50.550 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:06:50.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:50.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:06:50.573 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:50 smithi064 ceph-mon[220044]: pgmap v1025: 119 pgs: 119 active+clean; 296 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 638 op/s 2023-10-09T06:06:50.655 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:50.683 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:50.724 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:50.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:50.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:50.828 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:50.852 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:50 smithi156 ceph-mon[195101]: pgmap v1025: 119 pgs: 119 active+clean; 296 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 638 op/s 2023-10-09T06:06:50.854 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:50.895 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:50.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:50.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:50.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:50 smithi064 ceph-mon[212750]: pgmap v1025: 119 pgs: 119 active+clean; 296 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 638 op/s 2023-10-09T06:06:51.001 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:51.028 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:51.069 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:51.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:51.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:51.175 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:51.176 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:06:51.176 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:06:51.206 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:51.755 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:51.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:51.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:51.846 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:06:51.859 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:51.887 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:51.912 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:51.929 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:51.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:51.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:51.940 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:52.036 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:52.062 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:52.102 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:52.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:52.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:52.208 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:52.234 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:52.275 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:52.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:52.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:52.380 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:52.406 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:52.448 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:52.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:52.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:52.553 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:52.585 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:52 smithi064 ceph-mon[212750]: pgmap v1026: 119 pgs: 119 active+clean; 34 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s 2023-10-09T06:06:52.585 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:52 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2308154735' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.33258394583658757}]: dispatch 2023-10-09T06:06:52.586 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:52 smithi064 ceph-mon[220044]: pgmap v1026: 119 pgs: 119 active+clean; 34 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s 2023-10-09T06:06:52.586 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:52 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2308154735' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.33258394583658757}]: dispatch 2023-10-09T06:06:52.588 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:52.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:52 smithi156 ceph-mon[195101]: pgmap v1026: 119 pgs: 119 active+clean; 34 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s 2023-10-09T06:06:52.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:52 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2308154735' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.33258394583658757}]: dispatch 2023-10-09T06:06:53.388 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:53.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:53.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:53.453 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.1 primary-affinity to 0.332584 (8217962) 2023-10-09T06:06:53.493 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:53.520 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:54.079 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:54.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:54.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:54.183 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:54.216 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:54.230 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:54.263 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:54.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:54.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:54.368 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:54.396 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:54.439 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:54.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:54.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:54.514 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-10-09T06:06:54.515 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-10-09T06:06:54.544 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:54.593 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:54.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:54 smithi156 ceph-mon[195101]: pgmap v1027: 119 pgs: 119 active+clean; 18 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 923 op/s 2023-10-09T06:06:54.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:54 smithi156 ceph-mon[195101]: failed to encode map e418 with expected crc 2023-10-09T06:06:54.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:54 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2308154735' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.33258394583658757}]': finished 2023-10-09T06:06:54.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:54 smithi156 ceph-mon[195101]: osdmap e418: 8 total, 8 up, 7 in 2023-10-09T06:06:54.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:54 smithi156 ceph-mon[195101]: failed to encode map e418 with expected crc 2023-10-09T06:06:54.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:54 smithi156 ceph-mon[195101]: failed to encode map e418 with expected crc 2023-10-09T06:06:54.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:54 smithi156 ceph-mon[195101]: failed to encode map e418 with expected crc 2023-10-09T06:06:54.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:54 smithi156 ceph-mon[195101]: failed to encode map e418 with expected crc 2023-10-09T06:06:54.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:54 smithi156 ceph-mon[195101]: failed to encode map e418 with expected crc 2023-10-09T06:06:54.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:54 smithi156 ceph-mon[195101]: failed to encode map e418 with expected crc 2023-10-09T06:06:54.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:54 smithi064 ceph-mon[212750]: pgmap v1027: 119 pgs: 119 active+clean; 18 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 923 op/s 2023-10-09T06:06:54.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:54 smithi064 ceph-mon[212750]: failed to encode map e418 with expected crc 2023-10-09T06:06:54.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:54 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2308154735' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.33258394583658757}]': finished 2023-10-09T06:06:54.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:54 smithi064 ceph-mon[212750]: osdmap e418: 8 total, 8 up, 7 in 2023-10-09T06:06:54.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:54 smithi064 ceph-mon[212750]: failed to encode map e418 with expected crc 2023-10-09T06:06:54.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:54 smithi064 ceph-mon[212750]: failed to encode map e418 with expected crc 2023-10-09T06:06:54.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:54 smithi064 ceph-mon[212750]: failed to encode map e418 with expected crc 2023-10-09T06:06:54.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:54 smithi064 ceph-mon[212750]: failed to encode map e418 with expected crc 2023-10-09T06:06:54.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:54 smithi064 ceph-mon[212750]: failed to encode map e418 with expected crc 2023-10-09T06:06:54.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:54 smithi064 ceph-mon[212750]: failed to encode map e418 with expected crc 2023-10-09T06:06:54.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:54 smithi064 ceph-mon[220044]: pgmap v1027: 119 pgs: 119 active+clean; 18 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 923 op/s 2023-10-09T06:06:54.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:54 smithi064 ceph-mon[220044]: failed to encode map e418 with expected crc 2023-10-09T06:06:54.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:54 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2308154735' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.33258394583658757}]': finished 2023-10-09T06:06:54.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:54 smithi064 ceph-mon[220044]: osdmap e418: 8 total, 8 up, 7 in 2023-10-09T06:06:54.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:54 smithi064 ceph-mon[220044]: failed to encode map e418 with expected crc 2023-10-09T06:06:54.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:54 smithi064 ceph-mon[220044]: failed to encode map e418 with expected crc 2023-10-09T06:06:54.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:54 smithi064 ceph-mon[220044]: failed to encode map e418 with expected crc 2023-10-09T06:06:54.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:54 smithi064 ceph-mon[220044]: failed to encode map e418 with expected crc 2023-10-09T06:06:54.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:54 smithi064 ceph-mon[220044]: failed to encode map e418 with expected crc 2023-10-09T06:06:54.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:54 smithi064 ceph-mon[220044]: failed to encode map e418 with expected crc 2023-10-09T06:06:54.747 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:54.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:54.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:54.825 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:06:54.848 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:06:54.853 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:54.870 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:06:54.880 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:54.893 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:06:54.914 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:06:54.922 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:54.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:54.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:54.934 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:06:54.953 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:06:54.976 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:06:54.977 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:06:54.977 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:06:54.977 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:06:54.977 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:06:54.977 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:06:54.978 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:06:54.978 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:06:54.978 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:06:55.027 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:06:55.063 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:55.460 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:06:55.676 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:06:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:06:55] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.43.0" 2023-10-09T06:06:55.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:55 smithi064 ceph-mon[212750]: osdmap e419: 8 total, 8 up, 7 in 2023-10-09T06:06:55.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:55 smithi064 ceph-mon[212750]: failed to encode map e419 with expected crc 2023-10-09T06:06:55.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:55 smithi064 ceph-mon[212750]: failed to encode map e419 with expected crc 2023-10-09T06:06:55.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:55 smithi064 ceph-mon[212750]: failed to encode map e419 with expected crc 2023-10-09T06:06:55.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:55 smithi064 ceph-mon[212750]: failed to encode map e419 with expected crc 2023-10-09T06:06:55.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:55 smithi064 ceph-mon[212750]: failed to encode map e419 with expected crc 2023-10-09T06:06:55.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:55 smithi064 ceph-mon[212750]: failed to encode map e419 with expected crc 2023-10-09T06:06:55.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:55 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3026254671' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:55.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:55 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/580995765' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:55.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:55 smithi064 ceph-mon[220044]: osdmap e419: 8 total, 8 up, 7 in 2023-10-09T06:06:55.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:55 smithi064 ceph-mon[220044]: failed to encode map e419 with expected crc 2023-10-09T06:06:55.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:55 smithi064 ceph-mon[220044]: failed to encode map e419 with expected crc 2023-10-09T06:06:55.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:55 smithi064 ceph-mon[220044]: failed to encode map e419 with expected crc 2023-10-09T06:06:55.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:55 smithi064 ceph-mon[220044]: failed to encode map e419 with expected crc 2023-10-09T06:06:55.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:55 smithi064 ceph-mon[220044]: failed to encode map e419 with expected crc 2023-10-09T06:06:55.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:55 smithi064 ceph-mon[220044]: failed to encode map e419 with expected crc 2023-10-09T06:06:55.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:55 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3026254671' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:55.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:55 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/580995765' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:55.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:55 smithi156 ceph-mon[195101]: osdmap e419: 8 total, 8 up, 7 in 2023-10-09T06:06:55.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:55 smithi156 ceph-mon[195101]: failed to encode map e419 with expected crc 2023-10-09T06:06:55.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:55 smithi156 ceph-mon[195101]: failed to encode map e419 with expected crc 2023-10-09T06:06:55.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:55 smithi156 ceph-mon[195101]: failed to encode map e419 with expected crc 2023-10-09T06:06:55.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:55 smithi156 ceph-mon[195101]: failed to encode map e419 with expected crc 2023-10-09T06:06:55.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:55 smithi156 ceph-mon[195101]: failed to encode map e419 with expected crc 2023-10-09T06:06:55.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:55 smithi156 ceph-mon[195101]: failed to encode map e419 with expected crc 2023-10-09T06:06:55.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:55 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3026254671' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:06:55.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:55 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/580995765' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:06:56.166 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:06:56.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:56.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:06:56.246 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:56.270 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:56.302 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:56.351 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:56.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:56.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:56.456 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:56.485 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:56.530 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:56.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:56.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:56.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:56 smithi156 ceph-mon[195101]: pgmap v1030: 119 pgs: 3 peering, 116 active+clean; 582 KiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 628 op/s 2023-10-09T06:06:56.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:56 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:06:56.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:56 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:06:56.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:56 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T06:06:56.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:56 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/580995765' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:06:56.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:56 smithi156 ceph-mon[195101]: osdmap e420: 8 total, 8 up, 7 in 2023-10-09T06:06:56.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:56 smithi156 ceph-mon[195101]: failed to encode map e420 with expected crc 2023-10-09T06:06:56.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:56 smithi156 ceph-mon[195101]: failed to encode map e420 with expected crc 2023-10-09T06:06:56.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:56 smithi156 ceph-mon[195101]: failed to encode map e420 with expected crc 2023-10-09T06:06:56.636 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:06:56.663 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:56.704 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:06:56.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:56.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:06:56.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:56 smithi064 ceph-mon[212750]: pgmap v1030: 119 pgs: 3 peering, 116 active+clean; 582 KiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 628 op/s 2023-10-09T06:06:56.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:56 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:06:56.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:56 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:06:56.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:56 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:06:56.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:56 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/580995765' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:06:56.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:56 smithi064 ceph-mon[212750]: osdmap e420: 8 total, 8 up, 7 in 2023-10-09T06:06:56.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:56 smithi064 ceph-mon[212750]: failed to encode map e420 with expected crc 2023-10-09T06:06:56.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:56 smithi064 ceph-mon[212750]: failed to encode map e420 with expected crc 2023-10-09T06:06:56.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:56 smithi064 ceph-mon[212750]: failed to encode map e420 with expected crc 2023-10-09T06:06:56.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:56 smithi064 ceph-mon[220044]: pgmap v1030: 119 pgs: 3 peering, 116 active+clean; 582 KiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 628 op/s 2023-10-09T06:06:56.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:56 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:06:56.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:56 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:06:56.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:56 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T06:06:56.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:56 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/580995765' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:06:56.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:56 smithi064 ceph-mon[220044]: osdmap e420: 8 total, 8 up, 7 in 2023-10-09T06:06:56.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:56 smithi064 ceph-mon[220044]: failed to encode map e420 with expected crc 2023-10-09T06:06:56.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:56 smithi064 ceph-mon[220044]: failed to encode map e420 with expected crc 2023-10-09T06:06:56.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:56 smithi064 ceph-mon[220044]: failed to encode map e420 with expected crc 2023-10-09T06:06:56.811 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:56.837 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:56.879 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:56.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:56.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:56.987 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:06:57.013 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:57.054 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:06:57.057 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:06:57.057 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T06:06:57.057 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T06:06:57.057 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T06:06:57.057 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:06:57.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:57.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:06:57.160 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:57.190 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:57.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:57 smithi064 ceph-mon[212750]: failed to encode map e420 with expected crc 2023-10-09T06:06:57.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:57 smithi064 ceph-mon[212750]: failed to encode map e420 with expected crc 2023-10-09T06:06:57.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:57 smithi064 ceph-mon[220044]: failed to encode map e420 with expected crc 2023-10-09T06:06:57.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:57 smithi064 ceph-mon[220044]: failed to encode map e420 with expected crc 2023-10-09T06:06:57.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:57 smithi156 ceph-mon[195101]: failed to encode map e420 with expected crc 2023-10-09T06:06:57.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:57 smithi156 ceph-mon[195101]: failed to encode map e420 with expected crc 2023-10-09T06:06:58.119 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:06:58.139 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:06:58.291 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:58.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:58.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:58.396 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:06:58.429 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:58.461 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:58 smithi064 ceph-mon[220044]: pgmap v1032: 119 pgs: 3 peering, 116 active+clean; 599 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 4.5 KiB/s rd, 12 KiB/s wr, 92 op/s; 8.7 KiB/s, 0 objects/s recovering 2023-10-09T06:06:58.478 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:06:58.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:58.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:06:58.566 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:58.584 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:58.612 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:58.656 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:58.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:58.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:58.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:58 smithi064 ceph-mon[212750]: pgmap v1032: 119 pgs: 3 peering, 116 active+clean; 599 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 4.5 KiB/s rd, 12 KiB/s wr, 92 op/s; 8.7 KiB/s, 0 objects/s recovering 2023-10-09T06:06:58.761 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:06:58.796 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:58.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:58 smithi156 ceph-mon[195101]: pgmap v1032: 119 pgs: 3 peering, 116 active+clean; 599 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 4.5 KiB/s rd, 12 KiB/s wr, 92 op/s; 8.7 KiB/s, 0 objects/s recovering 2023-10-09T06:06:58.898 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:06:58.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:58.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:06:58.901 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:58.910 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:06:59.002 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:06:59.030 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:59.074 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:06:59.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:59.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:06:59.181 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:06:59.217 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:06:59.256 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1] out_osds: [2] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 0, 4] 2023-10-09T06:06:59.256 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:06:59.256 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:06:59.256 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 1.000000 2023-10-09T06:06:59.257 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-10-09T06:06:59.555 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_2' removed 2023-10-09T06:06:59.843 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:59 smithi064 ceph-mon[220044]: 1.3 scrub starts 2023-10-09T06:06:59.843 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:59 smithi064 ceph-mon[220044]: 1.3 scrub ok 2023-10-09T06:06:59.843 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:59 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:06:59.843 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:59.843 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:06:59 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2457202627' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:06:59.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:59 smithi064 ceph-mon[212750]: 1.3 scrub starts 2023-10-09T06:06:59.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:59 smithi064 ceph-mon[212750]: 1.3 scrub ok 2023-10-09T06:06:59.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:59 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:06:59.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:59.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:06:59 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2457202627' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:06:59.844 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:06:59.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:06:59.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:06:59.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:59 smithi156 ceph-mon[195101]: 1.3 scrub starts 2023-10-09T06:06:59.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:59 smithi156 ceph-mon[195101]: 1.3 scrub ok 2023-10-09T06:06:59.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:59 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:06:59.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:59.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:06:59 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2457202627' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:06:59.947 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:06:59.975 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:00.174 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:00.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:00.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:00.279 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:00.306 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:00.347 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:00.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:00.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:00.454 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:00.480 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:00.521 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:00.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:00.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:00.614 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T06:07:00.615 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T06:07:00.615 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-10-09T06:07:00.615 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-10-09T06:07:00.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:00 smithi156 ceph-mon[195101]: pgmap v1033: 119 pgs: 119 active+clean; 599 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 4.5 KiB/s rd, 12 KiB/s wr, 50 op/s; 8.7 KiB/s, 0 objects/s recovering 2023-10-09T06:07:00.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:00 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T06:07:00.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:00 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T06:07:00.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:00 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2457202627' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:07:00.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:00 smithi156 ceph-mon[195101]: osdmap e421: 8 total, 8 up, 7 in 2023-10-09T06:07:00.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:00 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2023-10-09T06:07:00.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:00 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2023-10-09T06:07:00.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:00 smithi156 ceph-mon[195101]: failed to encode map e421 with expected crc 2023-10-09T06:07:00.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:00 smithi156 ceph-mon[195101]: failed to encode map e421 with expected crc 2023-10-09T06:07:00.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:00 smithi156 ceph-mon[195101]: failed to encode map e421 with expected crc 2023-10-09T06:07:00.627 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:00.653 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:00.695 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:00.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:00.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:00.765 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:00 smithi064 ceph-mon[212750]: pgmap v1033: 119 pgs: 119 active+clean; 599 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 4.5 KiB/s rd, 12 KiB/s wr, 50 op/s; 8.7 KiB/s, 0 objects/s recovering 2023-10-09T06:07:00.765 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:00 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T06:07:00.765 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:00 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:07:00.765 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:00 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2457202627' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:07:00.765 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:00 smithi064 ceph-mon[212750]: osdmap e421: 8 total, 8 up, 7 in 2023-10-09T06:07:00.766 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:00 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2023-10-09T06:07:00.766 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:00 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2023-10-09T06:07:00.766 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:00 smithi064 ceph-mon[212750]: failed to encode map e421 with expected crc 2023-10-09T06:07:00.766 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:00 smithi064 ceph-mon[212750]: failed to encode map e421 with expected crc 2023-10-09T06:07:00.766 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:00 smithi064 ceph-mon[212750]: failed to encode map e421 with expected crc 2023-10-09T06:07:00.766 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:00 smithi064 ceph-mon[220044]: pgmap v1033: 119 pgs: 119 active+clean; 599 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 4.5 KiB/s rd, 12 KiB/s wr, 50 op/s; 8.7 KiB/s, 0 objects/s recovering 2023-10-09T06:07:00.766 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:00 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T06:07:00.766 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:00 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T06:07:00.766 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:00 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2457202627' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:07:00.766 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:00 smithi064 ceph-mon[220044]: osdmap e421: 8 total, 8 up, 7 in 2023-10-09T06:07:00.767 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:00 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2023-10-09T06:07:00.767 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:00 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2023-10-09T06:07:00.767 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:00 smithi064 ceph-mon[220044]: failed to encode map e421 with expected crc 2023-10-09T06:07:00.767 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:00 smithi064 ceph-mon[220044]: failed to encode map e421 with expected crc 2023-10-09T06:07:00.767 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:00 smithi064 ceph-mon[220044]: failed to encode map e421 with expected crc 2023-10-09T06:07:00.801 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:00.827 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:00.869 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:00.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:00.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:00.975 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:01.008 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:01.154 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:01.162 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:01.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:01.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:01.167 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:07:01.191 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:07:01.210 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:07:01.231 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:07:01.250 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:07:01.266 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:01.274 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:07:01.294 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:07:01.297 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:01.322 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:07:01.324 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:07:01.325 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:07:01.325 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:07:01.325 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:07:01.325 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:07:01.325 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:07:01.325 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:07:01.325 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:07:01.428 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:01.458 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:01.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:01.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:01.475 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:07:01.475 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:07:01.562 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:01.595 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:01.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:01 smithi064 ceph-mon[220044]: failed to encode map e421 with expected crc 2023-10-09T06:07:01.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:01 smithi064 ceph-mon[220044]: failed to encode map e421 with expected crc 2023-10-09T06:07:01.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:01 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2023-10-09T06:07:01.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:01 smithi064 ceph-mon[220044]: osdmap e422: 8 total, 8 up, 7 in 2023-10-09T06:07:01.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:01 smithi064 ceph-mon[220044]: failed to encode map e422 with expected crc 2023-10-09T06:07:01.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:01 smithi064 ceph-mon[220044]: failed to encode map e422 with expected crc 2023-10-09T06:07:01.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:01 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3603540875' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:01.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:01 smithi064 ceph-mon[212750]: failed to encode map e421 with expected crc 2023-10-09T06:07:01.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:01 smithi064 ceph-mon[212750]: failed to encode map e421 with expected crc 2023-10-09T06:07:01.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:01 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2023-10-09T06:07:01.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:01 smithi064 ceph-mon[212750]: osdmap e422: 8 total, 8 up, 7 in 2023-10-09T06:07:01.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:01 smithi064 ceph-mon[212750]: failed to encode map e422 with expected crc 2023-10-09T06:07:01.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:01 smithi064 ceph-mon[212750]: failed to encode map e422 with expected crc 2023-10-09T06:07:01.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:01 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3603540875' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:01.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:01 smithi156 ceph-mon[195101]: failed to encode map e421 with expected crc 2023-10-09T06:07:01.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:01 smithi156 ceph-mon[195101]: failed to encode map e421 with expected crc 2023-10-09T06:07:01.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:01 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2023-10-09T06:07:01.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:01 smithi156 ceph-mon[195101]: osdmap e422: 8 total, 8 up, 7 in 2023-10-09T06:07:01.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:01 smithi156 ceph-mon[195101]: failed to encode map e422 with expected crc 2023-10-09T06:07:01.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:01 smithi156 ceph-mon[195101]: failed to encode map e422 with expected crc 2023-10-09T06:07:01.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:01 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3603540875' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:02.246 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:02.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:02.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:02.350 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:02.382 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:02.431 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:02.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:02.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:02.539 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:02.567 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:02.866 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:02.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:02 smithi064 ceph-mon[212750]: pgmap v1036: 103 pgs: 103 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:07:02.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:02 smithi064 ceph-mon[212750]: failed to encode map e423 with expected crc 2023-10-09T06:07:02.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:02 smithi064 ceph-mon[212750]: osdmap e423: 8 total, 8 up, 7 in 2023-10-09T06:07:02.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:02 smithi064 ceph-mon[212750]: failed to encode map e423 with expected crc 2023-10-09T06:07:02.868 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:02 smithi064 ceph-mon[220044]: pgmap v1036: 103 pgs: 103 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:07:02.868 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:02 smithi064 ceph-mon[220044]: failed to encode map e423 with expected crc 2023-10-09T06:07:02.868 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:02 smithi064 ceph-mon[220044]: osdmap e423: 8 total, 8 up, 7 in 2023-10-09T06:07:02.868 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:02 smithi064 ceph-mon[220044]: failed to encode map e423 with expected crc 2023-10-09T06:07:02.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:02.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:02.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:02 smithi156 ceph-mon[195101]: pgmap v1036: 103 pgs: 103 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:07:02.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:02 smithi156 ceph-mon[195101]: failed to encode map e423 with expected crc 2023-10-09T06:07:02.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:02 smithi156 ceph-mon[195101]: osdmap e423: 8 total, 8 up, 7 in 2023-10-09T06:07:02.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:02 smithi156 ceph-mon[195101]: failed to encode map e423 with expected crc 2023-10-09T06:07:02.970 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:03.001 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:03.036 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:03.050 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:03.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:03.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:03.060 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:07:03.155 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:03.183 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:03.225 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:03.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:03.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:03.330 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:03.358 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:03.451 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:07:03.710 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:03.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:03.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:03.814 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:03.849 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:04.260 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:07:04.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:04 smithi064 ceph-mon[212750]: pgmap v1038: 103 pgs: 103 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:07:04.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:04 smithi064 ceph-mon[220044]: pgmap v1038: 103 pgs: 103 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:07:04.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:04 smithi156 ceph-mon[195101]: pgmap v1038: 103 pgs: 103 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:07:05.049 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:05.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:05.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:05.155 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:05.210 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:05.369 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:05.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:05.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:05.421 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:05.472 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:07:05.472 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:07:05.472 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:05.502 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:05.552 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-10-09T06:07:05.568 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:05 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2074326902' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:07:05.568 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:05 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:07:05.568 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:05 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2074326902' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:07:05.569 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:05 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:07:05.569 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:07:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:07:05] "GET /metrics HTTP/1.1" 200 36416 "" "Prometheus/2.43.0" 2023-10-09T06:07:05.655 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:05.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:05.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:05.759 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:05.784 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:05 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2074326902' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:07:05.785 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:05 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:07:05.786 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:05.827 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:05.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:05.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:05.930 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:05.962 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:06.107 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:07:06.451 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:07:06.550 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_3' created 2023-10-09T06:07:06.554 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:07:06.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:06 smithi064 ceph-mon[212750]: pgmap v1039: 103 pgs: 103 active+clean; 582 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:07:06.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:06 smithi064 ceph-mon[212750]: failed to encode map e424 with expected crc 2023-10-09T06:07:06.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:06 smithi064 ceph-mon[212750]: failed to encode map e424 with expected crc 2023-10-09T06:07:06.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:06 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:07:06.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:06 smithi064 ceph-mon[212750]: osdmap e424: 8 total, 8 up, 7 in 2023-10-09T06:07:06.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:06 smithi064 ceph-mon[212750]: failed to encode map e424 with expected crc 2023-10-09T06:07:06.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:06 smithi064 ceph-mon[212750]: failed to encode map e424 with expected crc 2023-10-09T06:07:06.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:06 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/423222579' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:07:06.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:06 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2858104925' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T06:07:06.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:06 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T06:07:06.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:06 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:06.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:06 smithi064 ceph-mon[220044]: pgmap v1039: 103 pgs: 103 active+clean; 582 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:07:06.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:06 smithi064 ceph-mon[220044]: failed to encode map e424 with expected crc 2023-10-09T06:07:06.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:06 smithi064 ceph-mon[220044]: failed to encode map e424 with expected crc 2023-10-09T06:07:06.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:06 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:07:06.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:06 smithi064 ceph-mon[220044]: osdmap e424: 8 total, 8 up, 7 in 2023-10-09T06:07:06.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:06 smithi064 ceph-mon[220044]: failed to encode map e424 with expected crc 2023-10-09T06:07:06.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:06 smithi064 ceph-mon[220044]: failed to encode map e424 with expected crc 2023-10-09T06:07:06.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:06 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/423222579' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:07:06.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:06 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2858104925' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T06:07:06.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:06 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T06:07:06.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:06 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:06.825 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:06 smithi156 ceph-mon[195101]: pgmap v1039: 103 pgs: 103 active+clean; 582 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:07:06.826 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:06 smithi156 ceph-mon[195101]: failed to encode map e424 with expected crc 2023-10-09T06:07:06.826 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:06 smithi156 ceph-mon[195101]: failed to encode map e424 with expected crc 2023-10-09T06:07:06.826 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:06 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:07:06.826 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:06 smithi156 ceph-mon[195101]: osdmap e424: 8 total, 8 up, 7 in 2023-10-09T06:07:06.826 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:06 smithi156 ceph-mon[195101]: failed to encode map e424 with expected crc 2023-10-09T06:07:06.826 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:06 smithi156 ceph-mon[195101]: failed to encode map e424 with expected crc 2023-10-09T06:07:06.826 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:06 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/423222579' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:07:06.826 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:06 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2858104925' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T06:07:06.826 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:06 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T06:07:06.826 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:06 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:07.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:07.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[220044]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[220044]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[220044]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[220044]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[220044]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[220044]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/423222579' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:07:07.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T06:07:07.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[220044]: osdmap e425: 8 total, 8 up, 7 in 2023-10-09T06:07:07.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[220044]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[220044]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[220044]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[220044]: osdmap e426: 8 total, 8 up, 7 in 2023-10-09T06:07:07.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:07.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[212750]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[212750]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[212750]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[212750]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[212750]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[212750]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/423222579' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:07:07.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T06:07:07.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[212750]: osdmap e425: 8 total, 8 up, 7 in 2023-10-09T06:07:07.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[212750]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[212750]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[212750]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:07 smithi064 ceph-mon[212750]: osdmap e426: 8 total, 8 up, 7 in 2023-10-09T06:07:07.847 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:07 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:07.848 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:07 smithi156 ceph-mon[195101]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.848 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:07 smithi156 ceph-mon[195101]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.848 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:07 smithi156 ceph-mon[195101]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.848 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:07 smithi156 ceph-mon[195101]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.848 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:07 smithi156 ceph-mon[195101]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.848 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:07 smithi156 ceph-mon[195101]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.848 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:07 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/423222579' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:07:07.848 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:07 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T06:07:07.848 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:07 smithi156 ceph-mon[195101]: osdmap e425: 8 total, 8 up, 7 in 2023-10-09T06:07:07.848 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:07 smithi156 ceph-mon[195101]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.849 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:07 smithi156 ceph-mon[195101]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.849 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:07 smithi156 ceph-mon[195101]: failed to encode map e425 with expected crc 2023-10-09T06:07:07.849 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:07 smithi156 ceph-mon[195101]: osdmap e426: 8 total, 8 up, 7 in 2023-10-09T06:07:07.901 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-10-09T06:07:07.951 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:07.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:07.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:08.055 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:08.086 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:08.137 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:08.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:08.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:08.240 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:08.272 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:08.320 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:07:08.324 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:08.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:08.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:08.346 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:07:08.365 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:07:08.380 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:07:08.380 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:07:08.383 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:07:08.406 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:07:08.428 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:08.430 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:07:08.451 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:07:08.458 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:08.470 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:07:08.471 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:07:08.471 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:07:08.471 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:07:08.471 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:07:08.471 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:07:08.472 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:07:08.472 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:07:08.472 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:07:08.478 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:08.562 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:08.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:08.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:08.666 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:08.690 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:08 smithi156 ceph-mon[195101]: pgmap v1042: 119 pgs: 16 unknown, 103 active+clean; 582 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:07:08.690 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:08 smithi156 ceph-mon[195101]: failed to encode map e426 with expected crc 2023-10-09T06:07:08.690 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:08 smithi156 ceph-mon[195101]: failed to encode map e426 with expected crc 2023-10-09T06:07:08.690 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:08 smithi156 ceph-mon[195101]: failed to encode map e426 with expected crc 2023-10-09T06:07:08.690 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:08 smithi156 ceph-mon[195101]: failed to encode map e426 with expected crc 2023-10-09T06:07:08.690 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:08 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1245626746' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:08.690 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:08 smithi156 ceph-mon[195101]: osdmap e427: 8 total, 8 up, 7 in 2023-10-09T06:07:08.692 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:08.756 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:08 smithi064 ceph-mon[212750]: pgmap v1042: 119 pgs: 16 unknown, 103 active+clean; 582 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:07:08.756 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:08 smithi064 ceph-mon[212750]: failed to encode map e426 with expected crc 2023-10-09T06:07:08.756 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:08 smithi064 ceph-mon[212750]: failed to encode map e426 with expected crc 2023-10-09T06:07:08.756 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:08 smithi064 ceph-mon[212750]: failed to encode map e426 with expected crc 2023-10-09T06:07:08.756 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:08 smithi064 ceph-mon[212750]: failed to encode map e426 with expected crc 2023-10-09T06:07:08.756 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:08 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1245626746' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:08.756 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:08 smithi064 ceph-mon[212750]: osdmap e427: 8 total, 8 up, 7 in 2023-10-09T06:07:08.757 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:08 smithi064 ceph-mon[220044]: pgmap v1042: 119 pgs: 16 unknown, 103 active+clean; 582 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:07:08.757 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:08 smithi064 ceph-mon[220044]: failed to encode map e426 with expected crc 2023-10-09T06:07:08.757 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:08 smithi064 ceph-mon[220044]: failed to encode map e426 with expected crc 2023-10-09T06:07:08.757 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:08 smithi064 ceph-mon[220044]: failed to encode map e426 with expected crc 2023-10-09T06:07:08.757 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:08 smithi064 ceph-mon[220044]: failed to encode map e426 with expected crc 2023-10-09T06:07:08.757 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:08 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1245626746' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:08.757 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:08 smithi064 ceph-mon[220044]: osdmap e427: 8 total, 8 up, 7 in 2023-10-09T06:07:09.195 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:07:09.245 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:09.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:09.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:09.351 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:09.378 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:09.481 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:09.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:09.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:09.585 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:09.616 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:09.655 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:07:09.666 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:09.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:09.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:09.770 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:09.790 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:07:09.796 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:09 smithi156 ceph-mon[195101]: failed to encode map e427 with expected crc 2023-10-09T06:07:09.796 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:09 smithi156 ceph-mon[195101]: failed to encode map e427 with expected crc 2023-10-09T06:07:09.796 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:09 smithi156 ceph-mon[195101]: failed to encode map e427 with expected crc 2023-10-09T06:07:09.799 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:09.800 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:09 smithi064 ceph-mon[212750]: failed to encode map e427 with expected crc 2023-10-09T06:07:09.800 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:09 smithi064 ceph-mon[212750]: failed to encode map e427 with expected crc 2023-10-09T06:07:09.800 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:09 smithi064 ceph-mon[212750]: failed to encode map e427 with expected crc 2023-10-09T06:07:09.800 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:09 smithi064 ceph-mon[220044]: failed to encode map e427 with expected crc 2023-10-09T06:07:09.801 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:09 smithi064 ceph-mon[220044]: failed to encode map e427 with expected crc 2023-10-09T06:07:09.801 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:09 smithi064 ceph-mon[220044]: failed to encode map e427 with expected crc 2023-10-09T06:07:09.854 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:09.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:09.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:09.958 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:09.983 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:10.025 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:10.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:10.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:10.128 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:10.157 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:10.201 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:10.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:10.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:10.305 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:10.333 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:10.396 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:10.710 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:10 smithi064 ceph-mon[212750]: pgmap v1045: 119 pgs: 15 unknown, 104 active+clean; 582 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:07:10.710 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:10 smithi064 ceph-mon[220044]: pgmap v1045: 119 pgs: 15 unknown, 104 active+clean; 582 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:07:10.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:10 smithi156 ceph-mon[195101]: pgmap v1045: 119 pgs: 15 unknown, 104 active+clean; 582 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:07:10.930 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:10.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:10.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:11.034 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:11.065 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:11.115 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:11.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:11.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:11.219 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:11.246 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:11.509 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:07:11.571 INFO:teuthology.orchestra.run.smithi064.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-10-09T06:07:11.614 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:07:11.615 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T06:07:11.615 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T06:07:11.615 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:07:11.615 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.615 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45263.0:166115 7.1 7.a53a721 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-10-09T06:07:11.615 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:01:21.468174+0000", 2023-10-09T06:07:11.615 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 350.14311018799998, 2023-10-09T06:07:11.615 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.84806625999999996, 2023-10-09T06:07:11.615 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:07:11.615 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:07:11.616 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:07:11.616 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45263", 2023-10-09T06:07:11.616 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:07:11.616 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 166115 2023-10-09T06:07:11.616 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.616 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:07:11.616 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.616 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:07:11.616 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468174+0000", 2023-10-09T06:07:11.616 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.616 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.616 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.617 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:07:11.617 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468174+0000", 2023-10-09T06:07:11.617 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.617 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.617 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.617 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:07:11.617 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468174+0000", 2023-10-09T06:07:11.617 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:07:11.617 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.617 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.617 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:07:11.618 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468176+0000", 2023-10-09T06:07:11.618 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.655e-06 2023-10-09T06:07:11.618 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.618 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.618 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:07:11.618 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468188+0000", 2023-10-09T06:07:11.618 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1212999999999999e-05 2023-10-09T06:07:11.618 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.618 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.618 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:07:11.618 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468192+0000", 2023-10-09T06:07:11.618 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2760000000000002e-06 2023-10-09T06:07:11.619 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.619 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.619 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.619 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468250+0000", 2023-10-09T06:07:11.619 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8267000000000003e-05 2023-10-09T06:07:11.619 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.619 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.619 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:07:11.619 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:21.468256+0000", 2023-10-09T06:07:11.619 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.553e-06 2023-10-09T06:07:11.619 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.619 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.620 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.620 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:22.316234+0000", 2023-10-09T06:07:11.620 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.84797834599999999 2023-10-09T06:07:11.620 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.620 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.620 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:07:11.620 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:22.316241+0000", 2023-10-09T06:07:11.620 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4110000000000001e-06 2023-10-09T06:07:11.620 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.620 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:07:11.620 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.620 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.621 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.621 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:134383 8.4 8:22d7e2f9:::benchmark_data_smithi064_457003_object29865:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2023-10-09T06:07:11.621 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:10.426661+0000", 2023-10-09T06:07:11.621 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 121.184623716, 2023-10-09T06:07:11.621 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1812402510000002, 2023-10-09T06:07:11.621 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:07:11.621 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:07:11.621 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:07:11.621 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:07:11.621 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:11.621 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 134383 2023-10-09T06:07:11.622 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.622 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:07:11.622 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.622 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:07:11.622 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.426661+0000", 2023-10-09T06:07:11.622 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.622 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.622 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.622 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:07:11.622 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.426661+0000", 2023-10-09T06:07:11.623 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.623 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.623 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.623 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:07:11.623 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.426660+0000", 2023-10-09T06:07:11.623 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:07:11.623 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.623 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.623 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:07:11.623 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.426664+0000", 2023-10-09T06:07:11.623 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3610000000000003e-06 2023-10-09T06:07:11.624 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.624 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.624 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:07:11.624 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.426676+0000", 2023-10-09T06:07:11.624 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1798e-05 2023-10-09T06:07:11.624 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.624 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.624 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:07:11.624 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.426680+0000", 2023-10-09T06:07:11.624 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2010000000000001e-06 2023-10-09T06:07:11.624 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.625 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.625 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.625 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.426731+0000", 2023-10-09T06:07:11.625 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0767999999999999e-05 2023-10-09T06:07:11.625 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.625 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.625 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:07:11.625 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.426864+0000", 2023-10-09T06:07:11.625 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013331000000000001 2023-10-09T06:07:11.626 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.626 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.626 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:07:11.626 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.427008+0000", 2023-10-09T06:07:11.626 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014342700000000001 2023-10-09T06:07:11.626 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.626 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.626 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.626 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.436794+0000", 2023-10-09T06:07:11.626 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0097867189999999993 2023-10-09T06:07:11.626 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.626 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.627 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:07:11.627 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.436805+0000", 2023-10-09T06:07:11.627 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0703999999999999e-05 2023-10-09T06:07:11.629 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.629 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.629 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.629 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:11.450433+0000", 2023-10-09T06:07:11.629 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.01362781 2023-10-09T06:07:11.629 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.629 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.629 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:07:11.629 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:11.450445+0000", 2023-10-09T06:07:11.630 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1657999999999999e-05 2023-10-09T06:07:11.630 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.630 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.630 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.630 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.607824+0000", 2023-10-09T06:07:11.630 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1573795309999999 2023-10-09T06:07:11.630 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.630 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.630 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:07:11.630 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.607901+0000", 2023-10-09T06:07:11.631 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6910999999999996e-05 2023-10-09T06:07:11.631 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.631 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:07:11.631 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.631 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.631 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.631 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:134384 8.4 8:2434c918:::benchmark_data_smithi064_457003_object29866:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2023-10-09T06:07:11.631 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:10.429188+0000", 2023-10-09T06:07:11.631 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 121.182095967, 2023-10-09T06:07:11.631 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1788163570000001, 2023-10-09T06:07:11.631 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:07:11.631 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:07:11.632 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:07:11.632 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:07:11.632 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:11.632 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 134384 2023-10-09T06:07:11.632 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.632 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:07:11.632 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.632 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:07:11.632 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.429188+0000", 2023-10-09T06:07:11.632 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.632 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.632 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:07:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.429188+0000", 2023-10-09T06:07:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:07:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.429187+0000", 2023-10-09T06:07:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:07:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.429193+0000", 2023-10-09T06:07:11.633 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.049e-06 2023-10-09T06:07:11.634 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.634 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.634 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:07:11.634 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.429207+0000", 2023-10-09T06:07:11.634 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4011e-05 2023-10-09T06:07:11.634 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.634 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.634 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:07:11.634 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.429216+0000", 2023-10-09T06:07:11.634 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8629999999999997e-06 2023-10-09T06:07:11.635 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.635 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.635 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.635 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.429661+0000", 2023-10-09T06:07:11.635 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00044492400000000002 2023-10-09T06:07:11.635 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.635 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.635 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:07:11.635 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.429777+0000", 2023-10-09T06:07:11.635 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000116107 2023-10-09T06:07:11.635 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.635 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.636 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:07:11.636 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.429856+0000", 2023-10-09T06:07:11.636 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8425999999999998e-05 2023-10-09T06:07:11.636 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.636 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.636 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.636 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.436827+0000", 2023-10-09T06:07:11.636 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0069719680000000003 2023-10-09T06:07:11.636 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.636 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.636 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:07:11.637 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.436833+0000", 2023-10-09T06:07:11.637 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6529999999999999e-06 2023-10-09T06:07:11.637 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.637 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.637 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.637 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:11.450477+0000", 2023-10-09T06:07:11.637 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0136439020000001 2023-10-09T06:07:11.637 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.637 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.637 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:07:11.637 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:11.450484+0000", 2023-10-09T06:07:11.637 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3440000000000003e-06 2023-10-09T06:07:11.638 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.638 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.638 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.638 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.607929+0000", 2023-10-09T06:07:11.638 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1574441870000001 2023-10-09T06:07:11.638 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.638 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.638 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:07:11.638 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.608005+0000", 2023-10-09T06:07:11.638 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6285000000000006e-05 2023-10-09T06:07:11.638 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.638 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:07:11.639 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.639 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.639 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.639 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:134394 8.4 8:2f57916d:::benchmark_data_smithi064_457003_object29876:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-10-09T06:07:11.639 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:10.577118+0000", 2023-10-09T06:07:11.639 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 121.034166145, 2023-10-09T06:07:11.639 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.121318697, 2023-10-09T06:07:11.639 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:07:11.639 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:11.639 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:07:11.639 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:07:11.640 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:11.640 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 134394 2023-10-09T06:07:11.640 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.640 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:07:11.640 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.640 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:07:11.640 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.577118+0000", 2023-10-09T06:07:11.640 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.640 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.640 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.640 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:07:11.640 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.577118+0000", 2023-10-09T06:07:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:07:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.577117+0000", 2023-10-09T06:07:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:07:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.577123+0000", 2023-10-09T06:07:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1929999999999998e-06 2023-10-09T06:07:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.641 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.642 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:07:11.642 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.577140+0000", 2023-10-09T06:07:11.642 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7090000000000001e-05 2023-10-09T06:07:11.642 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.642 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.642 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:07:11.642 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.577147+0000", 2023-10-09T06:07:11.642 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2280000000000001e-06 2023-10-09T06:07:11.642 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.642 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.642 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.577209+0000", 2023-10-09T06:07:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1564999999999998e-05 2023-10-09T06:07:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:07:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.577218+0000", 2023-10-09T06:07:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0529999999999996e-06 2023-10-09T06:07:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:11.450510+0000", 2023-10-09T06:07:11.643 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.87329198299999999 2023-10-09T06:07:11.644 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.644 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.644 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:07:11.644 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:11.452323+0000", 2023-10-09T06:07:11.644 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001812917 2023-10-09T06:07:11.644 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.644 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.644 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.644 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.608037+0000", 2023-10-09T06:07:11.644 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1557139999999999 2023-10-09T06:07:11.644 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.644 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.645 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:07:11.645 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.608112+0000", 2023-10-09T06:07:11.645 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5640000000000001e-05 2023-10-09T06:07:11.645 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.645 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.645 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:07:11.645 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.608577+0000", 2023-10-09T06:07:11.645 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00046510900000000001 2023-10-09T06:07:11.645 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.645 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.645 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:07:11.645 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.616852+0000", 2023-10-09T06:07:11.646 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0082743859999999999 2023-10-09T06:07:11.646 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.646 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.646 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.646 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.616922+0000", 2023-10-09T06:07:11.646 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0260999999999997e-05 2023-10-09T06:07:11.646 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.646 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.646 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.646 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.616961+0000", 2023-10-09T06:07:11.646 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9382999999999999e-05 2023-10-09T06:07:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.698270+0000", 2023-10-09T06:07:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.081308510000000001 2023-10-09T06:07:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:07:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.698319+0000", 2023-10-09T06:07:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9443000000000001e-05 2023-10-09T06:07:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.647 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:07:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.698437+0000", 2023-10-09T06:07:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011770800000000001 2023-10-09T06:07:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:07:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:134403 8.4 8:2a26dc4b:::benchmark_data_smithi064_457003_object29885:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-10-09T06:07:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:10.590324+0000", 2023-10-09T06:07:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 121.020960531, 2023-10-09T06:07:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1145394660000001, 2023-10-09T06:07:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:07:11.648 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:11.649 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:07:11.649 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:07:11.649 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:11.649 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 134403 2023-10-09T06:07:11.649 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.649 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:07:11.649 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.649 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:07:11.649 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590324+0000", 2023-10-09T06:07:11.649 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.649 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.649 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.649 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:07:11.650 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590324+0000", 2023-10-09T06:07:11.650 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.650 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.650 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.650 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:07:11.650 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590322+0000", 2023-10-09T06:07:11.650 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:11.650 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.650 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.650 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:07:11.650 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590331+0000", 2023-10-09T06:07:11.650 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2590000000000006e-06 2023-10-09T06:07:11.651 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.651 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.651 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:07:11.651 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590346+0000", 2023-10-09T06:07:11.651 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4663e-05 2023-10-09T06:07:11.651 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.651 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.651 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:07:11.651 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590354+0000", 2023-10-09T06:07:11.651 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6939999999999995e-06 2023-10-09T06:07:11.651 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.651 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.652 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.652 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590417+0000", 2023-10-09T06:07:11.652 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3152999999999997e-05 2023-10-09T06:07:11.652 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.652 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.652 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:07:11.652 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590430+0000", 2023-10-09T06:07:11.652 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.288e-05 2023-10-09T06:07:11.652 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.652 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.652 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:11.452416+0000", 2023-10-09T06:07:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.86198688400000001 2023-10-09T06:07:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:07:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:11.452427+0000", 2023-10-09T06:07:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0414999999999999e-05 2023-10-09T06:07:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.613291+0000", 2023-10-09T06:07:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.160863693 2023-10-09T06:07:11.653 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.654 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.654 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:07:11.654 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.613436+0000", 2023-10-09T06:07:11.654 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014545000000000001 2023-10-09T06:07:11.654 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.654 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.654 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:07:11.654 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.613593+0000", 2023-10-09T06:07:11.654 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015705600000000001 2023-10-09T06:07:11.654 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.654 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.655 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.655 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.620497+0000", 2023-10-09T06:07:11.655 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0069035349999999997 2023-10-09T06:07:11.655 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.655 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.655 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.655 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.620582+0000", 2023-10-09T06:07:11.655 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.5128000000000002e-05 2023-10-09T06:07:11.655 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.655 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.655 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:07:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.623088+0000", 2023-10-09T06:07:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0025060540000000002 2023-10-09T06:07:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.704684+0000", 2023-10-09T06:07:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.081596740000000001 2023-10-09T06:07:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:07:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.704743+0000", 2023-10-09T06:07:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8674000000000001e-05 2023-10-09T06:07:11.656 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.657 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.657 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:07:11.657 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.704863+0000", 2023-10-09T06:07:11.657 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000120219 2023-10-09T06:07:11.657 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.657 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:07:11.657 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.657 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.657 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.657 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149154 8.4 8:20798e00:::benchmark_data_smithi064_457003_object44636:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e391)", 2023-10-09T06:07:11.657 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.422228+0000", 2023-10-09T06:07:11.657 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 97.189056117000007, 2023-10-09T06:07:11.658 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.034537332, 2023-10-09T06:07:11.658 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:07:11.658 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:07:11.658 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:07:11.658 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:07:11.658 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:11.658 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149154 2023-10-09T06:07:11.658 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.658 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:07:11.658 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.658 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:07:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.422228+0000", 2023-10-09T06:07:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:07:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.422228+0000", 2023-10-09T06:07:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:07:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.422227+0000", 2023-10-09T06:07:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:11.659 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:07:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.422234+0000", 2023-10-09T06:07:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0060000000000003e-06 2023-10-09T06:07:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:07:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.422247+0000", 2023-10-09T06:07:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2954e-05 2023-10-09T06:07:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:07:11.660 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.422253+0000", 2023-10-09T06:07:11.661 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5860000000000001e-06 2023-10-09T06:07:11.661 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.661 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.661 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.661 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.422301+0000", 2023-10-09T06:07:11.661 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7951000000000002e-05 2023-10-09T06:07:11.661 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.661 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.661 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:07:11.661 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.422419+0000", 2023-10-09T06:07:11.661 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000118121 2023-10-09T06:07:11.661 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.662 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.662 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:07:11.662 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.422533+0000", 2023-10-09T06:07:11.662 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011380899999999999 2023-10-09T06:07:11.662 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.662 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.662 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.662 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.428222+0000", 2023-10-09T06:07:11.662 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0056886109999999997 2023-10-09T06:07:11.662 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.662 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.430128+0000", 2023-10-09T06:07:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0019062689999999999 2023-10-09T06:07:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:07:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.436483+0000", 2023-10-09T06:07:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0063544279999999996 2023-10-09T06:07:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.442211+0000", 2023-10-09T06:07:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0057284190000000002 2023-10-09T06:07:11.663 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.664 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.664 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:07:11.664 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.442220+0000", 2023-10-09T06:07:11.664 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3160000000000003e-06 2023-10-09T06:07:11.664 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.664 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.664 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.664 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.456686+0000", 2023-10-09T06:07:11.664 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.014465739 2023-10-09T06:07:11.664 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.664 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.664 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:07:11.665 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.456766+0000", 2023-10-09T06:07:11.665 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9617999999999996e-05 2023-10-09T06:07:11.665 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.665 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:07:11.665 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.665 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.665 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.665 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149161 8.4 8:2898f062:::benchmark_data_smithi064_457003_object44643:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:07:11.665 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.439170+0000", 2023-10-09T06:07:11.665 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 97.172114531999995, 2023-10-09T06:07:11.665 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.061805517, 2023-10-09T06:07:11.665 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:07:11.666 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:11.666 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:07:11.666 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:07:11.666 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:11.666 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149161 2023-10-09T06:07:11.666 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.666 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:07:11.666 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.666 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:07:11.666 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.439170+0000", 2023-10-09T06:07:11.666 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.667 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.667 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.667 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:07:11.667 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.439170+0000", 2023-10-09T06:07:11.667 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.667 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.667 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.667 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:07:11.667 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.439168+0000", 2023-10-09T06:07:11.667 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:11.667 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.667 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.668 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:07:11.668 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.439179+0000", 2023-10-09T06:07:11.668 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0719e-05 2023-10-09T06:07:11.668 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.668 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.668 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:07:11.668 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.439192+0000", 2023-10-09T06:07:11.668 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2734e-05 2023-10-09T06:07:11.668 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.668 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.668 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:07:11.668 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.439199+0000", 2023-10-09T06:07:11.669 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5570000000000001e-06 2023-10-09T06:07:11.669 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.669 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.669 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.669 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.442783+0000", 2023-10-09T06:07:11.669 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0035848360000000001 2023-10-09T06:07:11.669 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.669 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.669 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:07:11.669 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.442787+0000", 2023-10-09T06:07:11.669 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0080000000000004e-06 2023-10-09T06:07:11.669 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.670 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.670 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.670 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.456792+0000", 2023-10-09T06:07:11.670 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.014004393 2023-10-09T06:07:11.670 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.670 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.670 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:07:11.670 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.456908+0000", 2023-10-09T06:07:11.670 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000116222 2023-10-09T06:07:11.670 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.670 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.670 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:07:11.671 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.457059+0000", 2023-10-09T06:07:11.671 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000150543 2023-10-09T06:07:11.671 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.671 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.671 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:07:11.671 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.468559+0000", 2023-10-09T06:07:11.671 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011500836 2023-10-09T06:07:11.671 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.671 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.671 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.671 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.500849+0000", 2023-10-09T06:07:11.671 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.032290039999999999 2023-10-09T06:07:11.672 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.672 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.672 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.672 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.500886+0000", 2023-10-09T06:07:11.672 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6297999999999997e-05 2023-10-09T06:07:11.672 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.672 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.672 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:07:11.672 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.500935+0000", 2023-10-09T06:07:11.672 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9323999999999997e-05 2023-10-09T06:07:11.672 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.672 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:07:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.500975+0000", 2023-10-09T06:07:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0432999999999997e-05 2023-10-09T06:07:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:07:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149163 8.4 8:2aaf630e:::benchmark_data_smithi064_457003_object44645:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:07:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.439209+0000", 2023-10-09T06:07:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 97.172075695000004, 2023-10-09T06:07:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.061909999, 2023-10-09T06:07:11.673 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:07:11.674 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:11.674 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:07:11.674 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:07:11.674 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:11.674 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149163 2023-10-09T06:07:11.674 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.674 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:07:11.674 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.674 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:07:11.674 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.439209+0000", 2023-10-09T06:07:11.674 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.674 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.675 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.675 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:07:11.675 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.439209+0000", 2023-10-09T06:07:11.675 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.675 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.675 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.675 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:07:11.675 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.439208+0000", 2023-10-09T06:07:11.675 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:07:11.675 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.675 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.675 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:07:11.676 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.439211+0000", 2023-10-09T06:07:11.676 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7379999999999999e-06 2023-10-09T06:07:11.676 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.676 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.676 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:07:11.676 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.439222+0000", 2023-10-09T06:07:11.676 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0821e-05 2023-10-09T06:07:11.676 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.676 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.676 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:07:11.676 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.439226+0000", 2023-10-09T06:07:11.676 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7809999999999999e-06 2023-10-09T06:07:11.677 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.677 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.677 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.677 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.445449+0000", 2023-10-09T06:07:11.677 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0062232609999999999 2023-10-09T06:07:11.677 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.677 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.677 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:07:11.677 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.445458+0000", 2023-10-09T06:07:11.677 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6109999999999994e-06 2023-10-09T06:07:11.677 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.678 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.678 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.678 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.459701+0000", 2023-10-09T06:07:11.678 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0142438330000001 2023-10-09T06:07:11.678 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.678 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.678 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:07:11.678 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.459837+0000", 2023-10-09T06:07:11.678 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000135365 2023-10-09T06:07:11.678 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.678 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.678 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:07:11.679 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.459967+0000", 2023-10-09T06:07:11.679 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012998099999999999 2023-10-09T06:07:11.679 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.679 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.679 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:07:11.679 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.483196+0000", 2023-10-09T06:07:11.679 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.02322886 2023-10-09T06:07:11.679 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.679 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.679 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.679 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501029+0000", 2023-10-09T06:07:11.680 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.017833762999999999 2023-10-09T06:07:11.680 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.680 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.680 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.680 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501057+0000", 2023-10-09T06:07:11.680 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.8053999999999999e-05 2023-10-09T06:07:11.680 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.680 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.680 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:07:11.680 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501085+0000", 2023-10-09T06:07:11.680 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7274e-05 2023-10-09T06:07:11.681 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.681 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.681 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:07:11.681 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501119+0000", 2023-10-09T06:07:11.681 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.4059e-05 2023-10-09T06:07:11.681 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.681 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:07:11.681 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.681 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.681 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.681 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149178 8.4 8:2eb2dbb9:::benchmark_data_smithi064_457003_object44660:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:07:11.682 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.449151+0000", 2023-10-09T06:07:11.682 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 97.162133307000005, 2023-10-09T06:07:11.682 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0521200980000001, 2023-10-09T06:07:11.682 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:07:11.682 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:11.682 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:07:11.682 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:07:11.682 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:11.682 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149178 2023-10-09T06:07:11.682 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.683 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:07:11.683 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.683 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:07:11.683 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.449151+0000", 2023-10-09T06:07:11.683 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.683 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.683 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.683 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:07:11.683 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.449151+0000", 2023-10-09T06:07:11.683 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.683 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.683 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.684 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:07:11.684 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.449150+0000", 2023-10-09T06:07:11.684 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:11.684 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.684 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.684 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:07:11.684 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.449156+0000", 2023-10-09T06:07:11.684 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3999999999999997e-06 2023-10-09T06:07:11.684 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.684 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.684 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:07:11.685 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.449168+0000", 2023-10-09T06:07:11.685 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1606999999999999e-05 2023-10-09T06:07:11.685 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.685 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.685 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:07:11.685 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.449174+0000", 2023-10-09T06:07:11.685 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4470000000000003e-06 2023-10-09T06:07:11.685 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.685 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.685 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.685 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.449221+0000", 2023-10-09T06:07:11.685 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6761999999999997e-05 2023-10-09T06:07:11.686 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.686 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.686 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:07:11.686 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.449228+0000", 2023-10-09T06:07:11.686 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4900000000000003e-06 2023-10-09T06:07:11.686 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.686 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.686 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.686 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.462573+0000", 2023-10-09T06:07:11.686 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.013344947 2023-10-09T06:07:11.686 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.686 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:07:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.462663+0000", 2023-10-09T06:07:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9839999999999994e-05 2023-10-09T06:07:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:07:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.462739+0000", 2023-10-09T06:07:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5459999999999999e-05 2023-10-09T06:07:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:07:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.483201+0000", 2023-10-09T06:07:11.687 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.020462620000000001 2023-10-09T06:07:11.688 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.688 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.688 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.688 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501192+0000", 2023-10-09T06:07:11.688 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.017991333000000002 2023-10-09T06:07:11.688 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.688 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.688 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.688 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501220+0000", 2023-10-09T06:07:11.688 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7775000000000001e-05 2023-10-09T06:07:11.688 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.688 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.689 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:07:11.689 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501244+0000", 2023-10-09T06:07:11.689 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3858000000000001e-05 2023-10-09T06:07:11.689 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.689 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.689 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:07:11.689 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501271+0000", 2023-10-09T06:07:11.689 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7115e-05 2023-10-09T06:07:11.689 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.689 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:07:11.689 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.689 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.690 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.690 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149189 8.4 8:2633a930:::benchmark_data_smithi064_457003_object44671:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:07:11.690 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.470034+0000", 2023-10-09T06:07:11.690 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 97.141250278000001, 2023-10-09T06:07:11.690 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.03132147, 2023-10-09T06:07:11.690 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:07:11.690 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:11.690 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:07:11.690 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:07:11.690 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:11.690 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149189 2023-10-09T06:07:11.690 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.691 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:07:11.691 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.691 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:07:11.691 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470034+0000", 2023-10-09T06:07:11.691 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.691 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.691 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.691 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:07:11.691 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470034+0000", 2023-10-09T06:07:11.691 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.691 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.691 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.692 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:07:11.692 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470034+0000", 2023-10-09T06:07:11.692 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:07:11.692 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.692 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.692 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:07:11.692 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470037+0000", 2023-10-09T06:07:11.692 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7799999999999998e-06 2023-10-09T06:07:11.692 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.692 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.692 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:07:11.692 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470045+0000", 2023-10-09T06:07:11.693 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5689999999999999e-06 2023-10-09T06:07:11.693 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.693 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.693 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:07:11.693 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470050+0000", 2023-10-09T06:07:11.693 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0599999999999998e-06 2023-10-09T06:07:11.693 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.693 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.693 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.693 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470099+0000", 2023-10-09T06:07:11.693 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9329000000000002e-05 2023-10-09T06:07:11.693 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.694 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.694 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:07:11.694 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470108+0000", 2023-10-09T06:07:11.694 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.5180000000000004e-06 2023-10-09T06:07:11.694 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.694 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.694 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.694 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.465423+0000", 2023-10-09T06:07:11.694 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.995315328 2023-10-09T06:07:11.694 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.694 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.694 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:07:11.695 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.465549+0000", 2023-10-09T06:07:11.695 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012567899999999999 2023-10-09T06:07:11.695 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.695 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.695 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:07:11.695 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.465702+0000", 2023-10-09T06:07:11.695 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000153401 2023-10-09T06:07:11.695 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.695 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.695 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:07:11.695 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.485892+0000", 2023-10-09T06:07:11.695 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.020189628000000001 2023-10-09T06:07:11.696 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.696 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.696 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.696 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501297+0000", 2023-10-09T06:07:11.696 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015405106999999999 2023-10-09T06:07:11.696 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.696 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.696 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.696 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501314+0000", 2023-10-09T06:07:11.696 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6880000000000001e-05 2023-10-09T06:07:11.696 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.696 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:07:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501337+0000", 2023-10-09T06:07:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3289999999999999e-05 2023-10-09T06:07:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:07:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501356+0000", 2023-10-09T06:07:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8420999999999999e-05 2023-10-09T06:07:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:07:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.697 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.698 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149190 8.4 8:2b47ddf4:::benchmark_data_smithi064_457003_object44672:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:07:11.698 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.470066+0000", 2023-10-09T06:07:11.698 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 97.141218081999995, 2023-10-09T06:07:11.698 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.031372773, 2023-10-09T06:07:11.698 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:07:11.698 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:11.698 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:07:11.698 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:07:11.698 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:11.698 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149190 2023-10-09T06:07:11.698 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.699 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:07:11.699 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.699 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:07:11.699 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470066+0000", 2023-10-09T06:07:11.699 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.699 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.699 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.699 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:07:11.699 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470066+0000", 2023-10-09T06:07:11.699 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.699 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.699 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:07:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470066+0000", 2023-10-09T06:07:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:07:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:07:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470068+0000", 2023-10-09T06:07:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2129999999999998e-06 2023-10-09T06:07:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:07:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470073+0000", 2023-10-09T06:07:11.700 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0340000000000004e-06 2023-10-09T06:07:11.701 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.701 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.701 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:07:11.701 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470076+0000", 2023-10-09T06:07:11.701 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7779999999999999e-06 2023-10-09T06:07:11.701 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.701 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.701 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.701 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470127+0000", 2023-10-09T06:07:11.701 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1140000000000002e-05 2023-10-09T06:07:11.701 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.701 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.702 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:07:11.702 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470132+0000", 2023-10-09T06:07:11.702 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.028e-06 2023-10-09T06:07:11.702 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.702 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.702 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.702 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.468594+0000", 2023-10-09T06:07:11.702 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99846229600000003 2023-10-09T06:07:11.702 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.702 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.702 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:07:11.702 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.468669+0000", 2023-10-09T06:07:11.703 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4534000000000002e-05 2023-10-09T06:07:11.703 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.703 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.703 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:07:11.703 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.468766+0000", 2023-10-09T06:07:11.703 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7283999999999999e-05 2023-10-09T06:07:11.703 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.703 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.703 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:07:11.703 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.485898+0000", 2023-10-09T06:07:11.703 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.017131825 2023-10-09T06:07:11.703 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.704 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.704 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.704 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501375+0000", 2023-10-09T06:07:11.704 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.01547684 2023-10-09T06:07:11.704 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.704 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.704 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.704 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501397+0000", 2023-10-09T06:07:11.704 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2228000000000001e-05 2023-10-09T06:07:11.704 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.704 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.704 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:07:11.705 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501421+0000", 2023-10-09T06:07:11.705 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3438000000000002e-05 2023-10-09T06:07:11.705 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.705 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.705 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:07:11.705 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501439+0000", 2023-10-09T06:07:11.705 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8499e-05 2023-10-09T06:07:11.705 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.705 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:07:11.705 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.705 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.705 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.705 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149203 8.4 8:2695d781:::benchmark_data_smithi064_457003_object44685:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:07:11.706 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.485058+0000", 2023-10-09T06:07:11.706 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 97.126226668000001, 2023-10-09T06:07:11.706 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.017548871, 2023-10-09T06:07:11.706 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:07:11.706 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:11.706 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:07:11.706 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:07:11.706 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:11.706 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149203 2023-10-09T06:07:11.706 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.706 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:07:11.707 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.707 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:07:11.707 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.485058+0000", 2023-10-09T06:07:11.707 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.707 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.707 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.707 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:07:11.707 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.485058+0000", 2023-10-09T06:07:11.707 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.707 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.707 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:07:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.485056+0000", 2023-10-09T06:07:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:07:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.485063+0000", 2023-10-09T06:07:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8580000000000002e-06 2023-10-09T06:07:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:07:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.485077+0000", 2023-10-09T06:07:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3822000000000001e-05 2023-10-09T06:07:11.708 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.709 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.709 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:07:11.709 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.485084+0000", 2023-10-09T06:07:11.709 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5849999999999996e-06 2023-10-09T06:07:11.709 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.709 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.709 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.709 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.485169+0000", 2023-10-09T06:07:11.709 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.5824999999999999e-05 2023-10-09T06:07:11.709 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.709 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.710 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:07:11.710 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.485178+0000", 2023-10-09T06:07:11.710 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1769999999999998e-06 2023-10-09T06:07:11.710 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.710 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.710 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.710 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.471219+0000", 2023-10-09T06:07:11.710 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.98604176700000001 2023-10-09T06:07:11.710 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.710 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.710 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:07:11.711 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.471328+0000", 2023-10-09T06:07:11.711 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000108862 2023-10-09T06:07:11.711 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.711 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.711 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:07:11.711 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.471447+0000", 2023-10-09T06:07:11.711 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000118905 2023-10-09T06:07:11.711 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.711 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.711 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:07:11.711 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.485903+0000", 2023-10-09T06:07:11.711 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.014455956000000001 2023-10-09T06:07:11.712 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.712 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.712 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.712 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501458+0000", 2023-10-09T06:07:11.712 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015554869000000001 2023-10-09T06:07:11.712 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.712 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.712 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.712 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.502526+0000", 2023-10-09T06:07:11.712 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0010681250000000001 2023-10-09T06:07:11.712 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.712 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.713 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:07:11.713 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.502573+0000", 2023-10-09T06:07:11.713 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6508999999999998e-05 2023-10-09T06:07:11.713 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.713 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.713 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:07:11.713 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.502607+0000", 2023-10-09T06:07:11.713 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.4057000000000003e-05 2023-10-09T06:07:11.713 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.713 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:07:11.713 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.713 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.714 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.714 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149220 8.4 8:2e455896:::benchmark_data_smithi064_457003_object44702:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:07:11.714 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.508273+0000", 2023-10-09T06:07:11.714 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 97.103011522000003, 2023-10-09T06:07:11.714 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.994475785, 2023-10-09T06:07:11.714 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:07:11.714 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:11.714 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:07:11.714 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:07:11.714 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:11.714 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149220 2023-10-09T06:07:11.715 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.715 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:07:11.715 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.715 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:07:11.715 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.508273+0000", 2023-10-09T06:07:11.715 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.715 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.715 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.715 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:07:11.715 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.508273+0000", 2023-10-09T06:07:11.715 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.715 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.715 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.716 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:07:11.716 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.508272+0000", 2023-10-09T06:07:11.716 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:07:11.716 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.716 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.716 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:07:11.716 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.508278+0000", 2023-10-09T06:07:11.716 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8749999999999997e-06 2023-10-09T06:07:11.716 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.716 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.716 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:07:11.716 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.508292+0000", 2023-10-09T06:07:11.717 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4306e-05 2023-10-09T06:07:11.717 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.717 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.717 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:07:11.717 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.508299+0000", 2023-10-09T06:07:11.717 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9199999999999998e-06 2023-10-09T06:07:11.717 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.717 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.717 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.717 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.508352+0000", 2023-10-09T06:07:11.717 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2942000000000002e-05 2023-10-09T06:07:11.717 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.718 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.718 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:07:11.718 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.508358+0000", 2023-10-09T06:07:11.718 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3960000000000003e-06 2023-10-09T06:07:11.718 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.718 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.718 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.718 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.474150+0000", 2023-10-09T06:07:11.718 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96579230199999999 2023-10-09T06:07:11.718 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.718 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.719 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:07:11.719 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.474258+0000", 2023-10-09T06:07:11.719 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000107396 2023-10-09T06:07:11.719 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.719 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.719 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:07:11.719 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.474372+0000", 2023-10-09T06:07:11.719 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000114195 2023-10-09T06:07:11.719 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.719 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.719 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:07:11.719 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.485909+0000", 2023-10-09T06:07:11.720 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.01153674 2023-10-09T06:07:11.720 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.720 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.720 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.720 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501474+0000", 2023-10-09T06:07:11.720 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015565422000000001 2023-10-09T06:07:11.720 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.720 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.720 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.720 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.502662+0000", 2023-10-09T06:07:11.720 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0011879200000000001 2023-10-09T06:07:11.720 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.721 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.721 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:07:11.721 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.502706+0000", 2023-10-09T06:07:11.721 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3810999999999998e-05 2023-10-09T06:07:11.721 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.721 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.721 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:07:11.721 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.502749+0000", 2023-10-09T06:07:11.721 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2738000000000003e-05 2023-10-09T06:07:11.721 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.721 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:07:11.721 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.722 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.722 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.722 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149247 8.4 8:2a12e90d:::benchmark_data_smithi064_457003_object44729:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:07:11.722 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.538893+0000", 2023-10-09T06:07:11.722 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 97.072391796999995, 2023-10-09T06:07:11.722 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96400028800000004, 2023-10-09T06:07:11.722 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:07:11.722 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:11.722 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:07:11.722 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:07:11.722 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:11.723 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149247 2023-10-09T06:07:11.723 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.723 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:07:11.723 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.723 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:07:11.723 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.538893+0000", 2023-10-09T06:07:11.723 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.723 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.723 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.723 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:07:11.723 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.538893+0000", 2023-10-09T06:07:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:07:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.538891+0000", 2023-10-09T06:07:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:07:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.538898+0000", 2023-10-09T06:07:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7630000000000003e-06 2023-10-09T06:07:11.724 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.725 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.725 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:07:11.725 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.538911+0000", 2023-10-09T06:07:11.725 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3047000000000001e-05 2023-10-09T06:07:11.725 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.725 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.725 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:07:11.725 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.538918+0000", 2023-10-09T06:07:11.725 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5520000000000001e-06 2023-10-09T06:07:11.725 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.725 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.726 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.726 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.538968+0000", 2023-10-09T06:07:11.726 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0691e-05 2023-10-09T06:07:11.726 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.726 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.726 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:07:11.726 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.538977+0000", 2023-10-09T06:07:11.726 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8079999999999994e-06 2023-10-09T06:07:11.726 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.726 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.726 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.477116+0000", 2023-10-09T06:07:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.93813863799999997 2023-10-09T06:07:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:07:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.477219+0000", 2023-10-09T06:07:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000103694 2023-10-09T06:07:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:07:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.477336+0000", 2023-10-09T06:07:11.727 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000116764 2023-10-09T06:07:11.728 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.728 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.728 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:07:11.728 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.485914+0000", 2023-10-09T06:07:11.728 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0085778710000000008 2023-10-09T06:07:11.728 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.728 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.728 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.728 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501493+0000", 2023-10-09T06:07:11.728 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015579434 2023-10-09T06:07:11.728 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.729 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.729 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.729 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.502811+0000", 2023-10-09T06:07:11.729 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0013172539999999999 2023-10-09T06:07:11.729 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.729 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.729 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:07:11.729 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.502859+0000", 2023-10-09T06:07:11.729 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8408000000000001e-05 2023-10-09T06:07:11.729 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.729 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.729 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:07:11.730 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.502893+0000", 2023-10-09T06:07:11.730 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.3868999999999997e-05 2023-10-09T06:07:11.730 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.730 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:07:11.730 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.730 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.730 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.730 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149253 8.4 8:270bc248:::benchmark_data_smithi064_457003_object44735:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:07:11.730 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.547086+0000", 2023-10-09T06:07:11.730 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 97.064198114999996, 2023-10-09T06:07:11.730 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.95591631499999996, 2023-10-09T06:07:11.731 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:07:11.731 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:11.731 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:07:11.731 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:07:11.731 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:11.731 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149253 2023-10-09T06:07:11.731 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.731 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:07:11.731 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.731 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:07:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547086+0000", 2023-10-09T06:07:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:07:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547086+0000", 2023-10-09T06:07:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:07:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547085+0000", 2023-10-09T06:07:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:11.732 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.733 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.733 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:07:11.733 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547091+0000", 2023-10-09T06:07:11.733 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5610000000000004e-06 2023-10-09T06:07:11.733 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.733 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.733 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:07:11.733 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547104+0000", 2023-10-09T06:07:11.733 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2527999999999999e-05 2023-10-09T06:07:11.733 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.733 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.733 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:07:11.734 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547109+0000", 2023-10-09T06:07:11.734 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4940000000000003e-06 2023-10-09T06:07:11.734 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.734 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.734 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.734 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547160+0000", 2023-10-09T06:07:11.734 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0955000000000003e-05 2023-10-09T06:07:11.734 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.734 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.734 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:07:11.734 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547167+0000", 2023-10-09T06:07:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4570000000000002e-06 2023-10-09T06:07:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.480627+0000", 2023-10-09T06:07:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.93346054199999995 2023-10-09T06:07:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:07:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.480719+0000", 2023-10-09T06:07:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2038000000000002e-05 2023-10-09T06:07:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.735 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.736 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:07:11.736 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.480800+0000", 2023-10-09T06:07:11.736 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0419000000000004e-05 2023-10-09T06:07:11.736 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.736 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.736 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:07:11.736 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.488792+0000", 2023-10-09T06:07:11.736 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0079922489999999999 2023-10-09T06:07:11.736 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.736 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.736 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.736 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501516+0000", 2023-10-09T06:07:11.737 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012724231000000001 2023-10-09T06:07:11.737 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.737 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.737 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.737 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.502932+0000", 2023-10-09T06:07:11.737 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001415302 2023-10-09T06:07:11.737 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.737 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.737 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:07:11.737 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.502969+0000", 2023-10-09T06:07:11.737 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7469000000000003e-05 2023-10-09T06:07:11.738 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.738 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.738 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:07:11.738 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.503003+0000", 2023-10-09T06:07:11.738 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.366e-05 2023-10-09T06:07:11.738 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.738 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:07:11.738 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.738 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.738 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.738 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149254 8.4 8:2ee8489f:::benchmark_data_smithi064_457003_object44736:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:07:11.739 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.547125+0000", 2023-10-09T06:07:11.739 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 97.064159916999998, 2023-10-09T06:07:11.739 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.95964453599999999, 2023-10-09T06:07:11.739 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:07:11.739 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:11.739 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:07:11.739 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:07:11.739 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:11.739 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149254 2023-10-09T06:07:11.739 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.739 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:07:11.740 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.740 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:07:11.740 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547125+0000", 2023-10-09T06:07:11.740 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.740 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.740 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.740 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:07:11.740 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547125+0000", 2023-10-09T06:07:11.740 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.740 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.740 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.740 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:07:11.741 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547124+0000", 2023-10-09T06:07:11.741 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:07:11.741 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.741 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.741 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:07:11.741 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547127+0000", 2023-10-09T06:07:11.741 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.117e-06 2023-10-09T06:07:11.741 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.741 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.741 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:07:11.741 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547134+0000", 2023-10-09T06:07:11.742 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3400000000000003e-06 2023-10-09T06:07:11.742 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.742 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.742 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:07:11.742 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547138+0000", 2023-10-09T06:07:11.742 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.002e-06 2023-10-09T06:07:11.742 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.742 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.742 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.742 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547190+0000", 2023-10-09T06:07:11.742 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2871000000000003e-05 2023-10-09T06:07:11.743 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.743 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.743 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:07:11.743 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547199+0000", 2023-10-09T06:07:11.743 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0330000000000008e-06 2023-10-09T06:07:11.743 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.743 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.743 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.743 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.483229+0000", 2023-10-09T06:07:11.743 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.93603051800000003 2023-10-09T06:07:11.743 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.743 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.744 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:07:11.744 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.483338+0000", 2023-10-09T06:07:11.744 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000108942 2023-10-09T06:07:11.744 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.744 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.744 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:07:11.744 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.483456+0000", 2023-10-09T06:07:11.744 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000117689 2023-10-09T06:07:11.744 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.744 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.744 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:07:11.744 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.497626+0000", 2023-10-09T06:07:11.745 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.014170075000000001 2023-10-09T06:07:11.745 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.745 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.745 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.745 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501553+0000", 2023-10-09T06:07:11.745 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0039269229999999997 2023-10-09T06:07:11.745 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.745 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.745 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.745 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.506695+0000", 2023-10-09T06:07:11.745 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0051421440000000004 2023-10-09T06:07:11.746 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.746 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.746 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:07:11.746 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.506732+0000", 2023-10-09T06:07:11.746 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7499000000000001e-05 2023-10-09T06:07:11.746 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.746 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.746 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:07:11.746 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.506769+0000", 2023-10-09T06:07:11.746 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6773000000000002e-05 2023-10-09T06:07:11.746 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.746 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:07:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149256 8.4 8:21764160:::benchmark_data_smithi064_457003_object44738:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:07:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.550527+0000", 2023-10-09T06:07:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 97.060757910999996, 2023-10-09T06:07:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.95636602400000004, 2023-10-09T06:07:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:07:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:07:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:07:11.747 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:11.748 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149256 2023-10-09T06:07:11.748 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.748 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:07:11.748 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.748 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:07:11.748 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.550527+0000", 2023-10-09T06:07:11.748 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.748 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.748 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.748 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:07:11.748 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.550527+0000", 2023-10-09T06:07:11.748 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.749 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.749 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.749 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:07:11.749 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.550525+0000", 2023-10-09T06:07:11.749 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:07:11.749 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.749 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.749 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:07:11.749 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.550533+0000", 2023-10-09T06:07:11.749 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8900000000000007e-06 2023-10-09T06:07:11.749 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.749 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.750 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:07:11.750 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.550546+0000", 2023-10-09T06:07:11.750 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2383e-05 2023-10-09T06:07:11.750 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.750 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.750 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:07:11.750 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.550551+0000", 2023-10-09T06:07:11.750 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2499999999999997e-06 2023-10-09T06:07:11.750 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.750 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.750 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.750 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.550589+0000", 2023-10-09T06:07:11.751 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7891e-05 2023-10-09T06:07:11.751 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.751 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.751 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:07:11.751 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.550595+0000", 2023-10-09T06:07:11.751 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7799999999999997e-06 2023-10-09T06:07:11.751 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.751 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.751 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.751 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.485946+0000", 2023-10-09T06:07:11.751 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.93535175100000001 2023-10-09T06:07:11.751 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.752 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.752 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:07:11.752 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.486048+0000", 2023-10-09T06:07:11.752 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000101838 2023-10-09T06:07:11.752 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.752 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.752 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:07:11.752 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.486176+0000", 2023-10-09T06:07:11.752 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012777900000000001 2023-10-09T06:07:11.752 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.752 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.752 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:07:11.753 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.500497+0000", 2023-10-09T06:07:11.753 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.014320502000000001 2023-10-09T06:07:11.753 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.753 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.753 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.753 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501581+0000", 2023-10-09T06:07:11.753 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0010841869999999999 2023-10-09T06:07:11.753 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.753 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.753 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.753 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.506806+0000", 2023-10-09T06:07:11.753 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0052252360000000003 2023-10-09T06:07:11.753 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.754 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.754 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:07:11.754 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.506848+0000", 2023-10-09T06:07:11.754 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2057000000000001e-05 2023-10-09T06:07:11.754 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.754 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.754 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:07:11.754 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.506893+0000", 2023-10-09T06:07:11.754 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4580999999999999e-05 2023-10-09T06:07:11.754 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.754 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:07:11.754 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.755 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.755 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.755 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149271 8.4 8:24d9a3e3:::benchmark_data_smithi064_457003_object44753:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:07:11.755 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.574010+0000", 2023-10-09T06:07:11.755 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 97.037274285999999, 2023-10-09T06:07:11.755 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.93297265799999995, 2023-10-09T06:07:11.755 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:07:11.755 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:11.755 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:07:11.755 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:07:11.755 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:11.755 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149271 2023-10-09T06:07:11.756 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.756 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:07:11.756 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.756 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:07:11.756 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.574010+0000", 2023-10-09T06:07:11.756 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.756 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.756 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.756 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:07:11.756 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.574010+0000", 2023-10-09T06:07:11.756 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.756 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.757 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.757 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:07:11.757 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.574009+0000", 2023-10-09T06:07:11.757 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:11.757 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.757 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.757 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:07:11.757 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.574015+0000", 2023-10-09T06:07:11.757 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2020000000000001e-06 2023-10-09T06:07:11.757 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.757 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.757 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:07:11.757 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.574035+0000", 2023-10-09T06:07:11.758 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0281000000000001e-05 2023-10-09T06:07:11.758 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.758 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.758 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:07:11.758 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.574043+0000", 2023-10-09T06:07:11.758 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3989999999999998e-06 2023-10-09T06:07:11.758 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.758 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.758 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.758 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.574097+0000", 2023-10-09T06:07:11.758 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4465000000000001e-05 2023-10-09T06:07:11.759 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.759 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.759 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:07:11.759 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.574113+0000", 2023-10-09T06:07:11.759 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5620999999999998e-05 2023-10-09T06:07:11.759 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.759 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.759 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.759 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.488847+0000", 2023-10-09T06:07:11.759 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.91473444800000003 2023-10-09T06:07:11.759 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.759 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.760 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:07:11.760 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.488959+0000", 2023-10-09T06:07:11.760 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011193599999999999 2023-10-09T06:07:11.760 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.760 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.760 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:07:11.760 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.489072+0000", 2023-10-09T06:07:11.760 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000112901 2023-10-09T06:07:11.760 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.760 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.760 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:07:11.760 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.500503+0000", 2023-10-09T06:07:11.761 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011431062000000001 2023-10-09T06:07:11.761 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.761 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.761 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.761 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501608+0000", 2023-10-09T06:07:11.761 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0011049300000000001 2023-10-09T06:07:11.761 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.761 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.761 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.761 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.506936+0000", 2023-10-09T06:07:11.761 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0053282650000000004 2023-10-09T06:07:11.761 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.761 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.762 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:07:11.762 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.506960+0000", 2023-10-09T06:07:11.762 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3886000000000001e-05 2023-10-09T06:07:11.762 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.762 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.762 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:07:11.762 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.506983+0000", 2023-10-09T06:07:11.762 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2784000000000001e-05 2023-10-09T06:07:11.762 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.762 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:07:11.762 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.763 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.763 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.763 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149274 8.4 8:2a0dabef:::benchmark_data_smithi064_457003_object44756:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:07:11.763 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.579373+0000", 2023-10-09T06:07:11.763 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 97.031911367000006, 2023-10-09T06:07:11.763 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.92771710500000004, 2023-10-09T06:07:11.763 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:07:11.763 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:11.763 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:07:11.763 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:07:11.763 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:11.763 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149274 2023-10-09T06:07:11.764 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.764 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:07:11.764 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.764 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:07:11.764 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.579373+0000", 2023-10-09T06:07:11.764 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.764 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.764 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.764 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:07:11.764 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.579373+0000", 2023-10-09T06:07:11.764 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.764 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.765 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.765 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:07:11.765 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.579372+0000", 2023-10-09T06:07:11.765 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:11.765 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.765 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.765 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:07:11.765 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.579381+0000", 2023-10-09T06:07:11.765 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2900000000000008e-06 2023-10-09T06:07:11.765 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.765 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.766 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:07:11.766 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.579398+0000", 2023-10-09T06:07:11.766 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7119999999999999e-05 2023-10-09T06:07:11.766 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.766 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.766 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:07:11.766 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.579405+0000", 2023-10-09T06:07:11.766 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.323e-06 2023-10-09T06:07:11.766 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.766 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.766 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.766 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.579464+0000", 2023-10-09T06:07:11.767 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8386e-05 2023-10-09T06:07:11.767 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.767 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.767 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:07:11.767 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.579474+0000", 2023-10-09T06:07:11.767 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0407999999999999e-05 2023-10-09T06:07:11.767 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.767 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.767 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.767 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.491600+0000", 2023-10-09T06:07:11.767 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.91212597299999998 2023-10-09T06:07:11.767 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.768 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.768 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:07:11.768 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.491704+0000", 2023-10-09T06:07:11.768 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000103558 2023-10-09T06:07:11.768 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.768 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.768 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:07:11.768 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.491821+0000", 2023-10-09T06:07:11.768 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000117 2023-10-09T06:07:11.768 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.768 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.768 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:07:11.768 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.500509+0000", 2023-10-09T06:07:11.769 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0086884040000000003 2023-10-09T06:07:11.769 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.769 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.769 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.769 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501631+0000", 2023-10-09T06:07:11.769 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0011220850000000001 2023-10-09T06:07:11.769 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.769 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.769 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.769 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.507042+0000", 2023-10-09T06:07:11.769 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0054110019999999998 2023-10-09T06:07:11.769 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.770 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.770 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:07:11.770 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.507069+0000", 2023-10-09T06:07:11.770 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7005999999999999e-05 2023-10-09T06:07:11.770 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.770 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.770 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:07:11.770 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.507090+0000", 2023-10-09T06:07:11.770 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0981000000000001e-05 2023-10-09T06:07:11.770 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.770 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:07:11.771 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.771 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.771 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.771 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149287 8.4 8:2faede79:::benchmark_data_smithi064_457003_object44769:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:07:11.771 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.613919+0000", 2023-10-09T06:07:11.771 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 96.997365087999995, 2023-10-09T06:07:11.771 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.89325934699999998, 2023-10-09T06:07:11.771 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:07:11.771 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:11.771 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:07:11.771 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:07:11.771 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:11.771 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149287 2023-10-09T06:07:11.772 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.772 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:07:11.772 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.772 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:07:11.772 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.613919+0000", 2023-10-09T06:07:11.772 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.772 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.772 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.772 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:07:11.772 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.613919+0000", 2023-10-09T06:07:11.772 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:07:11.772 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.773 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.773 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:07:11.773 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.613918+0000", 2023-10-09T06:07:11.773 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:11.773 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.773 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.773 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:07:11.773 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.613924+0000", 2023-10-09T06:07:11.773 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2269999999999998e-06 2023-10-09T06:07:11.773 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.773 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.773 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:07:11.774 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.613936+0000", 2023-10-09T06:07:11.774 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2027e-05 2023-10-09T06:07:11.774 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.774 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.774 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:07:11.774 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.613942+0000", 2023-10-09T06:07:11.774 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8830000000000003e-06 2023-10-09T06:07:11.774 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.774 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.774 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.774 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.613992+0000", 2023-10-09T06:07:11.775 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9536999999999997e-05 2023-10-09T06:07:11.775 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.775 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.775 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:07:11.775 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.614001+0000", 2023-10-09T06:07:11.775 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0059999999999998e-06 2023-10-09T06:07:11.775 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.775 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.775 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:07:11.775 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.494720+0000", 2023-10-09T06:07:11.775 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.88071967900000003 2023-10-09T06:07:11.775 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.776 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.776 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:07:11.776 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.494855+0000", 2023-10-09T06:07:11.776 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013431700000000001 2023-10-09T06:07:11.776 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.776 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.776 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:07:11.776 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.494986+0000", 2023-10-09T06:07:11.776 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013104999999999999 2023-10-09T06:07:11.776 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.776 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.777 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:07:11.777 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.500791+0000", 2023-10-09T06:07:11.777 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0058056239999999997 2023-10-09T06:07:11.777 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.777 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.777 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.777 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501647+0000", 2023-10-09T06:07:11.777 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00085574000000000002 2023-10-09T06:07:11.777 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.777 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.777 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:11.777 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.507131+0000", 2023-10-09T06:07:11.778 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0054840169999999999 2023-10-09T06:07:11.778 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.778 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.778 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:07:11.778 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.507157+0000", 2023-10-09T06:07:11.778 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.5667e-05 2023-10-09T06:07:11.778 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:07:11.778 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:07:11.778 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:07:11.778 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.507179+0000", 2023-10-09T06:07:11.778 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2002e-05 2023-10-09T06:07:11.779 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.779 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:07:11.779 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.779 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:07:11.779 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:07:11.779 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:07:11.780 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:11 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4081263397' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:07:11.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:11.780 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:11.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:11.791 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:11 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4081263397' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:07:11.791 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:11 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4081263397' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:07:11.882 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:11.914 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:11.957 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:11.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:11.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:12.061 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:12.089 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:12.323 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:12.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:12.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:12.426 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:12.460 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:12.597 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:07:12.613 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:12.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:12.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:12.717 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:12.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:12 smithi064 ceph-mon[212750]: pgmap v1046: 119 pgs: 6 unknown, 113 active+clean; 582 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:07:12.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:12 smithi064 ceph-mon[212750]: failed to encode map e428 with expected crc 2023-10-09T06:07:12.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:12 smithi064 ceph-mon[212750]: failed to encode map e428 with expected crc 2023-10-09T06:07:12.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:12 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4081263397' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:07:12.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:12 smithi064 ceph-mon[212750]: osdmap e428: 8 total, 8 up, 7 in 2023-10-09T06:07:12.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:12 smithi064 ceph-mon[212750]: failed to encode map e428 with expected crc 2023-10-09T06:07:12.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:12 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:12.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:12 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:12.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:12 smithi064 ceph-mon[220044]: pgmap v1046: 119 pgs: 6 unknown, 113 active+clean; 582 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:07:12.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:12 smithi064 ceph-mon[220044]: failed to encode map e428 with expected crc 2023-10-09T06:07:12.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:12 smithi064 ceph-mon[220044]: failed to encode map e428 with expected crc 2023-10-09T06:07:12.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:12 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4081263397' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:07:12.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:12 smithi064 ceph-mon[220044]: osdmap e428: 8 total, 8 up, 7 in 2023-10-09T06:07:12.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:12 smithi064 ceph-mon[220044]: failed to encode map e428 with expected crc 2023-10-09T06:07:12.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:12 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:12.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:12 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:12.745 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:12.770 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:12.787 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:12.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:12.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:12.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:12 smithi156 ceph-mon[195101]: pgmap v1046: 119 pgs: 6 unknown, 113 active+clean; 582 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:07:12.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:12 smithi156 ceph-mon[195101]: failed to encode map e428 with expected crc 2023-10-09T06:07:12.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:12 smithi156 ceph-mon[195101]: failed to encode map e428 with expected crc 2023-10-09T06:07:12.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:12 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4081263397' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:07:12.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:12 smithi156 ceph-mon[195101]: osdmap e428: 8 total, 8 up, 7 in 2023-10-09T06:07:12.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:12 smithi156 ceph-mon[195101]: failed to encode map e428 with expected crc 2023-10-09T06:07:12.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:12 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:12.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:12 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:12.893 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:12.919 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:12.926 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1] out_osds: [2] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 0, 4] 2023-10-09T06:07:12.926 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:07:12.962 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:12.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:12.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:13.069 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:13.096 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:13.137 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:13.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:13.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:13.243 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:13.269 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:13.311 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:13.312 DEBUG:teuthology.orchestra.run.smithi064:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-10-09T06:07:13.313 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T06:07:13.313 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:07:13.315 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T06:07:13.315 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T06:07:13.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:13.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:13.418 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:13.444 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:13.485 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:13.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:13.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:13.584 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:07:13.590 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:13.606 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:13 smithi064 ceph-mon[212750]: failed to encode map e428 with expected crc 2023-10-09T06:07:13.606 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:13 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1250731012' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:13.606 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:13 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:13.606 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:13 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:13.607 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:13 smithi064 ceph-mon[220044]: failed to encode map e428 with expected crc 2023-10-09T06:07:13.607 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:13 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1250731012' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:13.607 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:13 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:13.607 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:13 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:13.620 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:13.687 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:hints = 1 2023-10-09T06:07:13.687 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T06:07:13.688 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object prefix: benchmark_data_smithi064_511911 2023-10-09T06:07:13.688 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:07:13.688 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 0 0 0 0 0 0 - 0 2023-10-09T06:07:13.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:13 smithi156 ceph-mon[195101]: failed to encode map e428 with expected crc 2023-10-09T06:07:13.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:13 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1250731012' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:13.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:13 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:13.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:13 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:14.688 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 1 16 3140 3124 195.245 195.25 0.0012637 0.00498173 2023-10-09T06:07:14.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:14 smithi156 ceph-mon[195101]: pgmap v1048: 119 pgs: 119 active+clean; 582 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:07:14.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:14 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:14.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:14 smithi156 ceph-mon[195101]: failed to encode map e429 with expected crc 2023-10-09T06:07:14.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:14 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1250731012' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:07:14.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:14 smithi156 ceph-mon[195101]: osdmap e429: 8 total, 8 up, 7 in 2023-10-09T06:07:14.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:14 smithi156 ceph-mon[195101]: failed to encode map e429 with expected crc 2023-10-09T06:07:14.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:14 smithi156 ceph-mon[195101]: failed to encode map e429 with expected crc 2023-10-09T06:07:14.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:14 smithi156 ceph-mon[195101]: failed to encode map e429 with expected crc 2023-10-09T06:07:14.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:14 smithi156 ceph-mon[195101]: failed to encode map e429 with expected crc 2023-10-09T06:07:14.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:14 smithi156 ceph-mon[195101]: failed to encode map e429 with expected crc 2023-10-09T06:07:14.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:14 smithi156 ceph-mon[195101]: failed to encode map e429 with expected crc 2023-10-09T06:07:14.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:14.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:14.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[212750]: pgmap v1048: 119 pgs: 119 active+clean; 582 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:07:14.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:14.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[212750]: failed to encode map e429 with expected crc 2023-10-09T06:07:14.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1250731012' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:07:14.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[212750]: osdmap e429: 8 total, 8 up, 7 in 2023-10-09T06:07:14.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[212750]: failed to encode map e429 with expected crc 2023-10-09T06:07:14.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[212750]: failed to encode map e429 with expected crc 2023-10-09T06:07:14.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[212750]: failed to encode map e429 with expected crc 2023-10-09T06:07:14.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[212750]: failed to encode map e429 with expected crc 2023-10-09T06:07:14.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[212750]: failed to encode map e429 with expected crc 2023-10-09T06:07:14.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[212750]: failed to encode map e429 with expected crc 2023-10-09T06:07:14.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:14.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:14.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[220044]: pgmap v1048: 119 pgs: 119 active+clean; 582 KiB data, 3.8 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:07:14.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:14.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[220044]: failed to encode map e429 with expected crc 2023-10-09T06:07:14.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1250731012' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:07:14.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[220044]: osdmap e429: 8 total, 8 up, 7 in 2023-10-09T06:07:14.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[220044]: failed to encode map e429 with expected crc 2023-10-09T06:07:14.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[220044]: failed to encode map e429 with expected crc 2023-10-09T06:07:14.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[220044]: failed to encode map e429 with expected crc 2023-10-09T06:07:14.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[220044]: failed to encode map e429 with expected crc 2023-10-09T06:07:14.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[220044]: failed to encode map e429 with expected crc 2023-10-09T06:07:14.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[220044]: failed to encode map e429 with expected crc 2023-10-09T06:07:14.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:14.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:15.212 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:15.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:15.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:15.324 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:15.370 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:15.425 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:15.480 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:07:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:07:15] "GET /metrics HTTP/1.1" 200 36342 "" "Prometheus/2.43.0" 2023-10-09T06:07:15.685 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:15.685 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:15.687 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 2 16 5362 5346 167.053 138.875 0.00452219 0.00596362 2023-10-09T06:07:15.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:15.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:15.795 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:15.797 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:15 smithi064 ceph-mon[220044]: failed to encode map e429 with expected crc 2023-10-09T06:07:15.797 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:15 smithi064 ceph-mon[220044]: pgmap v1050: 119 pgs: 119 active+clean; 49 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 7.6 MiB/s wr, 121 op/s 2023-10-09T06:07:15.798 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:15 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:15.798 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:15 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:07:15.798 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:15 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:15.798 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:15 smithi064 ceph-mon[212750]: failed to encode map e429 with expected crc 2023-10-09T06:07:15.798 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:15 smithi064 ceph-mon[212750]: pgmap v1050: 119 pgs: 119 active+clean; 49 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 7.6 MiB/s wr, 121 op/s 2023-10-09T06:07:15.799 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:15 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:15.799 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:15 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:07:15.799 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:15 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:15.829 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:15.862 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:07:15.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:15 smithi156 ceph-mon[195101]: failed to encode map e429 with expected crc 2023-10-09T06:07:15.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:15 smithi156 ceph-mon[195101]: pgmap v1050: 119 pgs: 119 active+clean; 49 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 7.6 MiB/s wr, 121 op/s 2023-10-09T06:07:15.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:15 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:15.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:15 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:07:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:15 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:15.903 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:07:15.905 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:15.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:15.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:15.950 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:07:15.996 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:07:16.010 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:16.042 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:07:16.071 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:16.079 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:07:16.120 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:07:16.162 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:07:16.170 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:07:16.170 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:07:16.170 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:07:16.170 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:07:16.170 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:07:16.170 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:07:16.170 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:07:16.170 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:07:16.630 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:16.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:16.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:16.688 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 3 16 8050 8034 167.366 168 0.00565968 0.00596893 2023-10-09T06:07:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:16 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3101393713' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:16 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3101393713' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:16.737 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:16.784 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:16 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3101393713' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:16.879 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:16.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:16.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:16.987 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:17.032 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:17.123 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:17.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:17.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:17.229 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:17.255 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:17.297 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:17.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:17.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:17.365 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:07:17.402 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:17.448 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:17.529 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:17.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:17.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:17.636 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:17.679 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:17.688 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 4 16 10776 10760 168.114 170.375 0.00509559 0.00594097 2023-10-09T06:07:17.748 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:17.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:17.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:17.856 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:17.926 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:18.689 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 5 16 13370 13354 166.89 162.125 0.00532543 0.00598533 2023-10-09T06:07:18.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:18 smithi064 ceph-mon[220044]: pgmap v1051: 119 pgs: 119 active+clean; 288 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 36 MiB/s wr, 573 op/s 2023-10-09T06:07:18.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:18 smithi064 ceph-mon[212750]: pgmap v1051: 119 pgs: 119 active+clean; 288 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 36 MiB/s wr, 573 op/s 2023-10-09T06:07:18.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:18 smithi156 ceph-mon[195101]: pgmap v1051: 119 pgs: 119 active+clean; 288 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 36 MiB/s wr, 573 op/s 2023-10-09T06:07:19.246 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:19.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:19.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:19.353 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:19.401 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:19.491 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:19.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:19.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:19.600 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:19.647 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:19.689 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 6 16 15971 15955 166.166 162.562 0.00292582 0.00601133 2023-10-09T06:07:19.739 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:19.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:19.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:19.845 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:19.885 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:19.952 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:19.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:19.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:20.061 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:20.106 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:20.178 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:20.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:20.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:20.185 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:20.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:20 smithi064 ceph-mon[212750]: pgmap v1052: 119 pgs: 119 active+clean; 343 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 43 MiB/s wr, 684 op/s 2023-10-09T06:07:20.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:20 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:20.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:20 smithi064 ceph-mon[220044]: pgmap v1052: 119 pgs: 119 active+clean; 343 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 43 MiB/s wr, 684 op/s 2023-10-09T06:07:20.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:20 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:20.283 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:20.307 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:07:20.307 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:07:20.317 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:20 smithi156 ceph-mon[195101]: pgmap v1052: 119 pgs: 119 active+clean; 343 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 43 MiB/s wr, 684 op/s 2023-10-09T06:07:20.317 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:20 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:20.320 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:20.410 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:20.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:20.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:20.517 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:20.564 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:20.638 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:20.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:20.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:20.689 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 7 15 18311 18296 163.328 146.312 0.00356578 0.00611684 2023-10-09T06:07:20.744 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:20.793 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:20.880 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:20.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:20.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:20.986 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:21.050 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:21.054 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:07:21.081 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T06:07:21.689 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 8 16 20260 20244 158.13 121.75 0.00935836 0.00631696 2023-10-09T06:07:21.905 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:07:22.112 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:07:22.142 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:22.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:22.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:22.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:22 smithi064 ceph-mon[212750]: pgmap v1053: 119 pgs: 119 active+clean; 800 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-10-09T06:07:22.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:22 smithi064 ceph-mon[212750]: from='client.35946 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:07:22.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:22 smithi064 ceph-mon[220044]: pgmap v1053: 119 pgs: 119 active+clean; 800 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-10-09T06:07:22.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:22 smithi064 ceph-mon[220044]: from='client.35946 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:07:22.253 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:22.308 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:22 smithi156 ceph-mon[195101]: pgmap v1053: 119 pgs: 119 active+clean; 800 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-10-09T06:07:22.308 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:22 smithi156 ceph-mon[195101]: from='client.35946 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:07:22.314 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:22.428 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:22.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:22.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:22.535 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:22.577 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:22.671 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:22.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:22.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:22.689 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 9 15 22435 22420 155.67 136 0.00320749 0.00641808 2023-10-09T06:07:22.779 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:22.830 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1053,"stamp":"2023-10-09T06:07:20.911015+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":838473632,"num_objects":12809,"num_object_clones":0,"num_object_copies":38427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12846,"num_write_kb":818269,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":264670,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":17831,"ondisk_log_size":17831,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":360,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6412324,"kb_used_data":2499564,"kb_used_omap":81,"kb_used_meta":3912622,"kb_avail":649803740,"statfs":{"total":671965249536,"available":665399029760,"internally_reserved":0,"allocated":2559553536,"data_stored":2547586321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83689,"internal_metadata":4006525207},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":837877760,"num_objects":12785,"num_object_clones":0,"num_object_copies":38355,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12785,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12785,"num_write_kb":818240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.345625"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":683,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.717314+0000","last_change":"2023-10-09T06:05:14.494993+0000","last_active":"2023-10-09T06:07:11.717314+0000","last_peered":"2023-10-09T06:07:11.717314+0000","last_clean":"2023-10-09T06:07:11.717314+0000","last_became_active":"2023-10-09T06:05:14.494626+0000","last_became_peered":"2023-10-09T06:05:14.494626+0000","last_unstale":"2023-10-09T06:07:11.717314+0000","last_undegraded":"2023-10-09T06:07:11.717314+0000","last_fullsized":"2023-10-09T06:07:11.717314+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:05.412941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:05.412941+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:21.757804+0000","scrub_duration":0.0061014329999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":487,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.569846+0000","last_change":"2023-10-09T06:05:11.462278+0000","last_active":"2023-10-09T06:07:11.569846+0000","last_peered":"2023-10-09T06:07:11.569846+0000","last_clean":"2023-10-09T06:07:11.569846+0000","last_became_active":"2023-10-09T06:05:11.460942+0000","last_became_peered":"2023-10-09T06:05:11.460942+0000","last_unstale":"2023-10-09T06:07:11.569846+0000","last_undegraded":"2023-10-09T06:07:11.569846+0000","last_fullsized":"2023-10-09T06:07:11.569846+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:07.197176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":236,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.715416+0000","last_change":"2023-10-09T06:05:11.451972+0000","last_active":"2023-10-09T06:07:11.715416+0000","last_peered":"2023-10-09T06:07:11.715416+0000","last_clean":"2023-10-09T06:07:11.715416+0000","last_became_active":"2023-10-09T06:05:11.451448+0000","last_became_peered":"2023-10-09T06:05:11.451448+0000","last_unstale":"2023-10-09T06:07:11.715416+0000","last_undegraded":"2023-10-09T06:07:11.715416+0000","last_fullsized":"2023-10-09T06:07:11.715416+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:39.183621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":711,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.570311+0000","last_change":"2023-10-09T06:02:01.468449+0000","last_active":"2023-10-09T06:07:11.570311+0000","last_peered":"2023-10-09T06:07:11.570311+0000","last_clean":"2023-10-09T06:07:11.570311+0000","last_became_active":"2023-10-09T06:01:22.334923+0000","last_became_peered":"2023-10-09T06:01:22.334923+0000","last_unstale":"2023-10-09T06:07:11.570311+0000","last_undegraded":"2023-10-09T06:07:11.570311+0000","last_fullsized":"2023-10-09T06:07:11.570311+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:01.468346+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:01.468346+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:20:04.967048+0000","scrub_duration":0.0016258220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":108,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.567193+0000","last_change":"2023-10-09T06:01:54.762060+0000","last_active":"2023-10-09T06:07:11.567193+0000","last_peered":"2023-10-09T06:07:11.567193+0000","last_clean":"2023-10-09T06:07:11.567193+0000","last_became_active":"2023-10-09T06:01:54.761757+0000","last_became_peered":"2023-10-09T06:01:54.761757+0000","last_unstale":"2023-10-09T06:07:11.567193+0000","last_undegraded":"2023-10-09T06:07:11.567193+0000","last_fullsized":"2023-10-09T06:07:11.567193+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:40.294232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":389,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.719085+0000","last_change":"2023-10-09T06:01:41.942412+0000","last_active":"2023-10-09T06:07:11.719085+0000","last_peered":"2023-10-09T06:07:11.719085+0000","last_clean":"2023-10-09T06:07:11.719085+0000","last_became_active":"2023-10-09T06:01:41.941186+0000","last_became_peered":"2023-10-09T06:01:41.941186+0000","last_unstale":"2023-10-09T06:07:11.719085+0000","last_undegraded":"2023-10-09T06:07:11.719085+0000","last_fullsized":"2023-10-09T06:07:11.719085+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:31.680824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":673,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.569664+0000","last_change":"2023-10-09T06:02:43.293521+0000","last_active":"2023-10-09T06:07:11.569664+0000","last_peered":"2023-10-09T06:07:11.569664+0000","last_clean":"2023-10-09T06:07:11.569664+0000","last_became_active":"2023-10-09T06:01:57.803642+0000","last_became_peered":"2023-10-09T06:01:57.803642+0000","last_unstale":"2023-10-09T06:07:11.569664+0000","last_undegraded":"2023-10-09T06:07:11.569664+0000","last_fullsized":"2023-10-09T06:07:11.569664+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.293426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.293426+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:45.626761+0000","scrub_duration":0.001587946,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":204,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.715427+0000","last_change":"2023-10-09T06:01:54.760903+0000","last_active":"2023-10-09T06:07:11.715427+0000","last_peered":"2023-10-09T06:07:11.715427+0000","last_clean":"2023-10-09T06:07:11.715427+0000","last_became_active":"2023-10-09T06:01:54.760338+0000","last_became_peered":"2023-10-09T06:01:54.760338+0000","last_unstale":"2023-10-09T06:07:11.715427+0000","last_undegraded":"2023-10-09T06:07:11.715427+0000","last_fullsized":"2023-10-09T06:07:11.715427+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:53.904647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":465,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.718682+0000","last_change":"2023-10-09T06:01:54.757269+0000","last_active":"2023-10-09T06:07:11.718682+0000","last_peered":"2023-10-09T06:07:11.718682+0000","last_clean":"2023-10-09T06:07:11.718682+0000","last_became_active":"2023-10-09T06:01:54.756894+0000","last_became_peered":"2023-10-09T06:01:54.756894+0000","last_unstale":"2023-10-09T06:07:11.718682+0000","last_undegraded":"2023-10-09T06:07:11.718682+0000","last_fullsized":"2023-10-09T06:07:11.718682+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:45.212821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":748,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.718148+0000","last_change":"2023-10-09T06:01:57.804160+0000","last_active":"2023-10-09T06:07:11.718148+0000","last_peered":"2023-10-09T06:07:11.718148+0000","last_clean":"2023-10-09T06:07:11.718148+0000","last_became_active":"2023-10-09T06:01:57.803424+0000","last_became_peered":"2023-10-09T06:01:57.803424+0000","last_unstale":"2023-10-09T06:07:11.718148+0000","last_undegraded":"2023-10-09T06:07:11.718148+0000","last_fullsized":"2023-10-09T06:07:11.718148+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.384407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.384407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:42.857346+0000","scrub_duration":0.0040651960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":309,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.715922+0000","last_change":"2023-10-09T06:01:41.939852+0000","last_active":"2023-10-09T06:07:11.715922+0000","last_peered":"2023-10-09T06:07:11.715922+0000","last_clean":"2023-10-09T06:07:11.715922+0000","last_became_active":"2023-10-09T06:01:41.939556+0000","last_became_peered":"2023-10-09T06:01:41.939556+0000","last_unstale":"2023-10-09T06:07:11.715922+0000","last_undegraded":"2023-10-09T06:07:11.715922+0000","last_fullsized":"2023-10-09T06:07:11.715922+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:06.980511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":246,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.715506+0000","last_change":"2023-10-09T06:06:01.067544+0000","last_active":"2023-10-09T06:07:11.715506+0000","last_peered":"2023-10-09T06:07:11.715506+0000","last_clean":"2023-10-09T06:07:11.715506+0000","last_became_active":"2023-10-09T06:06:01.067188+0000","last_became_peered":"2023-10-09T06:06:01.067188+0000","last_unstale":"2023-10-09T06:07:11.715506+0000","last_undegraded":"2023-10-09T06:07:11.715506+0000","last_fullsized":"2023-10-09T06:07:11.715506+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:43.464186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":575,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.718562+0000","last_change":"2023-10-09T06:06:20.742655+0000","last_active":"2023-10-09T06:07:11.718562+0000","last_peered":"2023-10-09T06:07:11.718562+0000","last_clean":"2023-10-09T06:07:11.718562+0000","last_became_active":"2023-10-09T06:05:14.494869+0000","last_became_peered":"2023-10-09T06:05:14.494869+0000","last_unstale":"2023-10-09T06:07:11.718562+0000","last_undegraded":"2023-10-09T06:07:11.718562+0000","last_fullsized":"2023-10-09T06:07:11.718562+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:20.742561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:20.742561+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:28.688207+0000","scrub_duration":0.0027005039999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":478,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.719164+0000","last_change":"2023-10-09T06:01:54.766344+0000","last_active":"2023-10-09T06:07:11.719164+0000","last_peered":"2023-10-09T06:07:11.719164+0000","last_clean":"2023-10-09T06:07:11.719164+0000","last_became_active":"2023-10-09T06:01:54.766086+0000","last_became_peered":"2023-10-09T06:01:54.766086+0000","last_unstale":"2023-10-09T06:07:11.719164+0000","last_undegraded":"2023-10-09T06:07:11.719164+0000","last_fullsized":"2023-10-09T06:07:11.719164+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:47.163700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":376,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.716159+0000","last_change":"2023-10-09T05:53:59.185702+0000","last_active":"2023-10-09T06:07:11.716159+0000","last_peered":"2023-10-09T06:07:11.716159+0000","last_clean":"2023-10-09T06:07:11.716159+0000","last_became_active":"2023-10-09T05:53:59.185373+0000","last_became_peered":"2023-10-09T05:53:59.185373+0000","last_unstale":"2023-10-09T06:07:11.716159+0000","last_undegraded":"2023-10-09T06:07:11.716159+0000","last_fullsized":"2023-10-09T06:07:11.716159+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:50.756967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":846,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.718749+0000","last_change":"2023-10-09T06:02:48.374519+0000","last_active":"2023-10-09T06:07:11.718749+0000","last_peered":"2023-10-09T06:07:11.718749+0000","last_clean":"2023-10-09T06:07:11.718749+0000","last_became_active":"2023-10-09T06:01:41.939691+0000","last_became_peered":"2023-10-09T06:01:41.939691+0000","last_unstale":"2023-10-09T06:07:11.718749+0000","last_undegraded":"2023-10-09T06:07:11.718749+0000","last_fullsized":"2023-10-09T06:07:11.718749+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:48.374432+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:48.374432+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:54.162995+0000","scrub_duration":0.001678585,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":286,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.718500+0000","last_change":"2023-10-09T06:01:22.337444+0000","last_active":"2023-10-09T06:07:11.718500+0000","last_peered":"2023-10-09T06:07:11.718500+0000","last_clean":"2023-10-09T06:07:11.718500+0000","last_became_active":"2023-10-09T06:01:22.336348+0000","last_became_peered":"2023-10-09T06:01:22.336348+0000","last_unstale":"2023-10-09T06:07:11.718500+0000","last_undegraded":"2023-10-09T06:07:11.718500+0000","last_fullsized":"2023-10-09T06:07:11.718500+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:43:11.149649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":265,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.567420+0000","last_change":"2023-10-09T06:07:06.554075+0000","last_active":"2023-10-09T06:07:11.567420+0000","last_peered":"2023-10-09T06:07:11.567420+0000","last_clean":"2023-10-09T06:07:11.567420+0000","last_became_active":"2023-10-09T06:07:06.553770+0000","last_became_peered":"2023-10-09T06:07:06.553770+0000","last_unstale":"2023-10-09T06:07:11.567420+0000","last_undegraded":"2023-10-09T06:07:11.567420+0000","last_fullsized":"2023-10-09T06:07:11.567420+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:28.268563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":583,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.568869+0000","last_change":"2023-10-09T06:01:57.807839+0000","last_active":"2023-10-09T06:07:11.568869+0000","last_peered":"2023-10-09T06:07:11.568869+0000","last_clean":"2023-10-09T06:07:11.568869+0000","last_became_active":"2023-10-09T06:01:57.807451+0000","last_became_peered":"2023-10-09T06:01:57.807451+0000","last_unstale":"2023-10-09T06:07:11.568869+0000","last_undegraded":"2023-10-09T06:07:11.568869+0000","last_fullsized":"2023-10-09T06:07:11.568869+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.339121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.339121+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:47.740601+0000","scrub_duration":0.010474344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":423,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.570533+0000","last_change":"2023-10-09T06:01:22.335608+0000","last_active":"2023-10-09T06:07:11.570533+0000","last_peered":"2023-10-09T06:07:11.570533+0000","last_clean":"2023-10-09T06:07:11.570533+0000","last_became_active":"2023-10-09T06:01:22.335296+0000","last_became_peered":"2023-10-09T06:01:22.335296+0000","last_unstale":"2023-10-09T06:07:11.570533+0000","last_undegraded":"2023-10-09T06:07:11.570533+0000","last_fullsized":"2023-10-09T06:07:11.570533+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:21.843341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":441,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.570620+0000","last_change":"2023-10-09T06:01:54.764916+0000","last_active":"2023-10-09T06:07:11.570620+0000","last_peered":"2023-10-09T06:07:11.570620+0000","last_clean":"2023-10-09T06:07:11.570620+0000","last_became_active":"2023-10-09T06:01:54.764705+0000","last_became_peered":"2023-10-09T06:01:54.764705+0000","last_unstale":"2023-10-09T06:07:11.570620+0000","last_undegraded":"2023-10-09T06:07:11.570620+0000","last_fullsized":"2023-10-09T06:07:11.570620+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:12.527719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":622,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.715848+0000","last_change":"2023-10-09T06:06:03.463559+0000","last_active":"2023-10-09T06:07:11.715848+0000","last_peered":"2023-10-09T06:07:11.715848+0000","last_clean":"2023-10-09T06:07:11.715848+0000","last_became_active":"2023-10-09T06:06:03.463215+0000","last_became_peered":"2023-10-09T06:06:03.463215+0000","last_unstale":"2023-10-09T06:07:11.715848+0000","last_undegraded":"2023-10-09T06:07:11.715848+0000","last_fullsized":"2023-10-09T06:07:11.715848+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:42.900876+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:42.900876+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:24:34.635157+0000","scrub_duration":0.0014928649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":304,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.718515+0000","last_change":"2023-10-09T06:05:11.465936+0000","last_active":"2023-10-09T06:07:11.718515+0000","last_peered":"2023-10-09T06:07:11.718515+0000","last_clean":"2023-10-09T06:07:11.718515+0000","last_became_active":"2023-10-09T06:05:11.459719+0000","last_became_peered":"2023-10-09T06:05:11.459719+0000","last_unstale":"2023-10-09T06:07:11.718515+0000","last_undegraded":"2023-10-09T06:07:11.718515+0000","last_fullsized":"2023-10-09T06:07:11.718515+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:45.174267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":471,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.715760+0000","last_change":"2023-10-09T06:01:54.761558+0000","last_active":"2023-10-09T06:07:11.715760+0000","last_peered":"2023-10-09T06:07:11.715760+0000","last_clean":"2023-10-09T06:07:11.715760+0000","last_became_active":"2023-10-09T06:01:54.761251+0000","last_became_peered":"2023-10-09T06:01:54.761251+0000","last_unstale":"2023-10-09T06:07:11.715760+0000","last_undegraded":"2023-10-09T06:07:11.715760+0000","last_fullsized":"2023-10-09T06:07:11.715760+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:53.245267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":872,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.717493+0000","last_change":"2023-10-09T06:06:03.453188+0000","last_active":"2023-10-09T06:07:11.717493+0000","last_peered":"2023-10-09T06:07:11.717493+0000","last_clean":"2023-10-09T06:07:11.717493+0000","last_became_active":"2023-10-09T06:06:03.452853+0000","last_became_peered":"2023-10-09T06:06:03.452853+0000","last_unstale":"2023-10-09T06:07:11.717493+0000","last_undegraded":"2023-10-09T06:07:11.717493+0000","last_fullsized":"2023-10-09T06:07:11.717493+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:26:21.994828+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":354,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.718178+0000","last_change":"2023-10-09T06:01:41.942115+0000","last_active":"2023-10-09T06:07:11.718178+0000","last_peered":"2023-10-09T06:07:11.718178+0000","last_clean":"2023-10-09T06:07:11.718178+0000","last_became_active":"2023-10-09T06:01:41.941827+0000","last_became_peered":"2023-10-09T06:01:41.941827+0000","last_unstale":"2023-10-09T06:07:11.718178+0000","last_undegraded":"2023-10-09T06:07:11.718178+0000","last_fullsized":"2023-10-09T06:07:11.718178+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:51:15.226428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":362,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.718655+0000","last_change":"2023-10-09T06:06:01.071454+0000","last_active":"2023-10-09T06:07:11.718655+0000","last_peered":"2023-10-09T06:07:11.718655+0000","last_clean":"2023-10-09T06:07:11.718655+0000","last_became_active":"2023-10-09T06:06:01.071137+0000","last_became_peered":"2023-10-09T06:06:01.071137+0000","last_unstale":"2023-10-09T06:07:11.718655+0000","last_undegraded":"2023-10-09T06:07:11.718655+0000","last_fullsized":"2023-10-09T06:07:11.718655+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:58:52.761376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":632,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.568242+0000","last_change":"2023-10-09T06:07:06.561922+0000","last_active":"2023-10-09T06:07:11.568242+0000","last_peered":"2023-10-09T06:07:11.568242+0000","last_clean":"2023-10-09T06:07:11.568242+0000","last_became_active":"2023-10-09T06:07:06.561625+0000","last_became_peered":"2023-10-09T06:07:06.561625+0000","last_unstale":"2023-10-09T06:07:11.568242+0000","last_undegraded":"2023-10-09T06:07:11.568242+0000","last_fullsized":"2023-10-09T06:07:11.568242+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.316697+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.316697+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:08.265733+0000","scrub_duration":0.001666561,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":282,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.717656+0000","last_change":"2023-10-09T06:01:22.337171+0000","last_active":"2023-10-09T06:07:11.717656+0000","last_peered":"2023-10-09T06:07:11.717656+0000","last_clean":"2023-10-09T06:07:11.717656+0000","last_became_active":"2023-10-09T06:01:22.335438+0000","last_became_peered":"2023-10-09T06:01:22.335438+0000","last_unstale":"2023-10-09T06:07:11.717656+0000","last_undegraded":"2023-10-09T06:07:11.717656+0000","last_fullsized":"2023-10-09T06:07:11.717656+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:25.036448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":353,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.716252+0000","last_change":"2023-10-09T06:01:41.941888+0000","last_active":"2023-10-09T06:07:11.716252+0000","last_peered":"2023-10-09T06:07:11.716252+0000","last_clean":"2023-10-09T06:07:11.716252+0000","last_became_active":"2023-10-09T06:01:41.941413+0000","last_became_peered":"2023-10-09T06:01:41.941413+0000","last_unstale":"2023-10-09T06:07:11.716252+0000","last_undegraded":"2023-10-09T06:07:11.716252+0000","last_fullsized":"2023-10-09T06:07:11.716252+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:32:41.773690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":682,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.567934+0000","last_change":"2023-10-09T06:02:40.388302+0000","last_active":"2023-10-09T06:07:11.567934+0000","last_peered":"2023-10-09T06:07:11.567934+0000","last_clean":"2023-10-09T06:07:11.567934+0000","last_became_active":"2023-10-09T06:01:57.803733+0000","last_became_peered":"2023-10-09T06:01:57.803733+0000","last_unstale":"2023-10-09T06:07:11.567934+0000","last_undegraded":"2023-10-09T06:07:11.567934+0000","last_fullsized":"2023-10-09T06:07:11.567934+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:40.388125+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:40.388125+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:17.919838+0000","scrub_duration":0.0055576990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":434,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.717419+0000","last_change":"2023-10-09T06:05:32.448360+0000","last_active":"2023-10-09T06:07:11.717419+0000","last_peered":"2023-10-09T06:07:11.717419+0000","last_clean":"2023-10-09T06:07:11.717419+0000","last_became_active":"2023-10-09T06:05:32.439146+0000","last_became_peered":"2023-10-09T06:05:32.439146+0000","last_unstale":"2023-10-09T06:07:11.717419+0000","last_undegraded":"2023-10-09T06:07:11.717419+0000","last_fullsized":"2023-10-09T06:07:11.717419+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:39.952582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":222,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.716004+0000","last_change":"2023-10-09T06:07:02.023531+0000","last_active":"2023-10-09T06:07:11.716004+0000","last_peered":"2023-10-09T06:07:11.716004+0000","last_clean":"2023-10-09T06:07:11.716004+0000","last_became_active":"2023-10-09T06:07:02.023280+0000","last_became_peered":"2023-10-09T06:07:02.023280+0000","last_unstale":"2023-10-09T06:07:11.716004+0000","last_undegraded":"2023-10-09T06:07:11.716004+0000","last_fullsized":"2023-10-09T06:07:11.716004+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:53.202101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"429'384","reported_seq":396,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:07:16.009006+0000","last_change":"2023-10-09T06:07:08.300714+0000","last_active":"2023-10-09T06:07:16.009006+0000","last_peered":"2023-10-09T06:07:16.009006+0000","last_clean":"2023-10-09T06:07:16.009006+0000","last_became_active":"2023-10-09T06:07:08.300406+0000","last_became_peered":"2023-10-09T06:07:08.300406+0000","last_unstale":"2023-10-09T06:07:16.009006+0000","last_undegraded":"2023-10-09T06:07:16.009006+0000","last_fullsized":"2023-10-09T06:07:16.009006+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":384,"log_dups_size":0,"ondisk_log_size":384,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:36.533742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25165824,"num_objects":384,"num_object_clones":0,"num_object_copies":1152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":384,"num_write_kb":24576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":805,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.718276+0000","last_change":"2023-10-09T06:01:57.804339+0000","last_active":"2023-10-09T06:07:11.718276+0000","last_peered":"2023-10-09T06:07:11.718276+0000","last_clean":"2023-10-09T06:07:11.718276+0000","last_became_active":"2023-10-09T06:01:57.803844+0000","last_became_peered":"2023-10-09T06:01:57.803844+0000","last_unstale":"2023-10-09T06:07:11.718276+0000","last_undegraded":"2023-10-09T06:07:11.718276+0000","last_fullsized":"2023-10-09T06:07:11.718276+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:53.469394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:53.469394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:51.125394+0000","scrub_duration":0.0059716659999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":286,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.719133+0000","last_change":"2023-10-09T06:01:22.336769+0000","last_active":"2023-10-09T06:07:11.719133+0000","last_peered":"2023-10-09T06:07:11.719133+0000","last_clean":"2023-10-09T06:07:11.719133+0000","last_became_active":"2023-10-09T06:01:22.336264+0000","last_became_peered":"2023-10-09T06:01:22.336264+0000","last_unstale":"2023-10-09T06:07:11.719133+0000","last_undegraded":"2023-10-09T06:07:11.719133+0000","last_fullsized":"2023-10-09T06:07:11.719133+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:17.065771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":345,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.717403+0000","last_change":"2023-10-09T06:06:01.074296+0000","last_active":"2023-10-09T06:07:11.717403+0000","last_peered":"2023-10-09T06:07:11.717403+0000","last_clean":"2023-10-09T06:07:11.717403+0000","last_became_active":"2023-10-09T06:06:01.073655+0000","last_became_peered":"2023-10-09T06:06:01.073655+0000","last_unstale":"2023-10-09T06:07:11.717403+0000","last_undegraded":"2023-10-09T06:07:11.717403+0000","last_fullsized":"2023-10-09T06:07:11.717403+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:20.319425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"429'1110","reported_seq":1123,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:07:20.502416+0000","last_change":"2023-10-09T06:07:07.580215+0000","last_active":"2023-10-09T06:07:20.502416+0000","last_peered":"2023-10-09T06:07:20.502416+0000","last_clean":"2023-10-09T06:07:20.502416+0000","last_became_active":"2023-10-09T06:07:07.579874+0000","last_became_peered":"2023-10-09T06:07:07.579874+0000","last_unstale":"2023-10-09T06:07:20.502416+0000","last_undegraded":"2023-10-09T06:07:20.502416+0000","last_fullsized":"2023-10-09T06:07:20.502416+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":1110,"log_dups_size":0,"ondisk_log_size":1110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:04.426865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72744960,"num_objects":1110,"num_object_clones":0,"num_object_copies":3330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1110,"num_write_kb":71040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":738,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.715486+0000","last_change":"2023-10-09T06:00:03.241479+0000","last_active":"2023-10-09T06:07:11.715486+0000","last_peered":"2023-10-09T06:07:11.715486+0000","last_clean":"2023-10-09T06:07:11.715486+0000","last_became_active":"2023-10-09T05:59:32.530753+0000","last_became_peered":"2023-10-09T05:59:32.530753+0000","last_unstale":"2023-10-09T06:07:11.715486+0000","last_undegraded":"2023-10-09T06:07:11.715486+0000","last_fullsized":"2023-10-09T06:07:11.715486+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:03.241335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:03.241335+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:54.822114+0000","scrub_duration":0.017780547000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":342,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.719197+0000","last_change":"2023-10-09T06:01:54.826540+0000","last_active":"2023-10-09T06:07:11.719197+0000","last_peered":"2023-10-09T06:07:11.719197+0000","last_clean":"2023-10-09T06:07:11.719197+0000","last_became_active":"2023-10-09T06:01:54.766205+0000","last_became_peered":"2023-10-09T06:01:54.766205+0000","last_unstale":"2023-10-09T06:07:11.719197+0000","last_undegraded":"2023-10-09T06:07:11.719197+0000","last_fullsized":"2023-10-09T06:07:11.719197+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:45.220336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":467,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.569349+0000","last_change":"2023-10-09T06:01:54.760367+0000","last_active":"2023-10-09T06:07:11.569349+0000","last_peered":"2023-10-09T06:07:11.569349+0000","last_clean":"2023-10-09T06:07:11.569349+0000","last_became_active":"2023-10-09T06:01:54.760087+0000","last_became_peered":"2023-10-09T06:01:54.760087+0000","last_unstale":"2023-10-09T06:07:11.569349+0000","last_undegraded":"2023-10-09T06:07:11.569349+0000","last_fullsized":"2023-10-09T06:07:11.569349+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:26.000104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"429'379","reported_seq":391,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:07:16.005443+0000","last_change":"2023-10-09T06:07:07.880402+0000","last_active":"2023-10-09T06:07:16.005443+0000","last_peered":"2023-10-09T06:07:16.005443+0000","last_clean":"2023-10-09T06:07:16.005443+0000","last_became_active":"2023-10-09T06:07:07.880006+0000","last_became_peered":"2023-10-09T06:07:07.880006+0000","last_unstale":"2023-10-09T06:07:16.005443+0000","last_undegraded":"2023-10-09T06:07:16.005443+0000","last_fullsized":"2023-10-09T06:07:16.005443+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":379,"log_dups_size":0,"ondisk_log_size":379,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:17.652201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24838144,"num_objects":379,"num_object_clones":0,"num_object_copies":1137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":379,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":379,"num_write_kb":24256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":868,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.718835+0000","last_change":"2023-10-09T06:01:22.326209+0000","last_active":"2023-10-09T06:07:11.718835+0000","last_peered":"2023-10-09T06:07:11.718835+0000","last_clean":"2023-10-09T06:07:11.718835+0000","last_became_active":"2023-10-09T06:01:22.325866+0000","last_became_peered":"2023-10-09T06:01:22.325866+0000","last_unstale":"2023-10-09T06:07:11.718835+0000","last_undegraded":"2023-10-09T06:07:11.718835+0000","last_fullsized":"2023-10-09T06:07:11.718835+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:41.753506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:41.753506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:32.324674+0000","scrub_duration":0.0022121929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":281,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.718592+0000","last_change":"2023-10-09T06:06:01.078672+0000","last_active":"2023-10-09T06:07:11.718592+0000","last_peered":"2023-10-09T06:07:11.718592+0000","last_clean":"2023-10-09T06:07:11.718592+0000","last_became_active":"2023-10-09T06:06:01.072297+0000","last_became_peered":"2023-10-09T06:06:01.072297+0000","last_unstale":"2023-10-09T06:07:11.718592+0000","last_undegraded":"2023-10-09T06:07:11.718592+0000","last_fullsized":"2023-10-09T06:07:11.718592+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:35.905773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":412,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.567659+0000","last_change":"2023-10-09T06:06:01.077340+0000","last_active":"2023-10-09T06:07:11.567659+0000","last_peered":"2023-10-09T06:07:11.567659+0000","last_clean":"2023-10-09T06:07:11.567659+0000","last_became_active":"2023-10-09T06:06:01.076610+0000","last_became_peered":"2023-10-09T06:06:01.076610+0000","last_unstale":"2023-10-09T06:07:11.567659+0000","last_undegraded":"2023-10-09T06:07:11.567659+0000","last_fullsized":"2023-10-09T06:07:11.567659+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:45.158282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"429'1130","reported_seq":1143,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:07:20.478763+0000","last_change":"2023-10-09T06:07:07.880093+0000","last_active":"2023-10-09T06:07:20.478763+0000","last_peered":"2023-10-09T06:07:20.478763+0000","last_clean":"2023-10-09T06:07:20.478763+0000","last_became_active":"2023-10-09T06:07:07.879786+0000","last_became_peered":"2023-10-09T06:07:07.879786+0000","last_unstale":"2023-10-09T06:07:20.478763+0000","last_undegraded":"2023-10-09T06:07:20.478763+0000","last_fullsized":"2023-10-09T06:07:20.478763+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":1130,"log_dups_size":0,"ondisk_log_size":1130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:04.721505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74055680,"num_objects":1130,"num_object_clones":0,"num_object_copies":3390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1130,"num_write_kb":72320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":604,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.571483+0000","last_change":"2023-10-09T06:02:16.361201+0000","last_active":"2023-10-09T06:07:11.571483+0000","last_peered":"2023-10-09T06:07:11.571483+0000","last_clean":"2023-10-09T06:07:11.571483+0000","last_became_active":"2023-10-09T06:01:22.337294+0000","last_became_peered":"2023-10-09T06:01:22.337294+0000","last_unstale":"2023-10-09T06:07:11.571483+0000","last_undegraded":"2023-10-09T06:07:11.571483+0000","last_fullsized":"2023-10-09T06:07:11.571483+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:16.361097+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:16.361097+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:16:25.401939+0000","scrub_duration":0.028533749000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":195,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.715603+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T06:07:11.715603+0000","last_peered":"2023-10-09T06:07:11.715603+0000","last_clean":"2023-10-09T06:07:11.715603+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T06:07:11.715603+0000","last_undegraded":"2023-10-09T06:07:11.715603+0000","last_fullsized":"2023-10-09T06:07:11.715603+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":467,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.715922+0000","last_change":"2023-10-09T06:01:54.753427+0000","last_active":"2023-10-09T06:07:11.715922+0000","last_peered":"2023-10-09T06:07:11.715922+0000","last_clean":"2023-10-09T06:07:11.715922+0000","last_became_active":"2023-10-09T06:01:54.753099+0000","last_became_peered":"2023-10-09T06:01:54.753099+0000","last_unstale":"2023-10-09T06:07:11.715922+0000","last_undegraded":"2023-10-09T06:07:11.715922+0000","last_fullsized":"2023-10-09T06:07:11.715922+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:07.504657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"429'1032","reported_seq":1045,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:07:19.659248+0000","last_change":"2023-10-09T06:07:07.547742+0000","last_active":"2023-10-09T06:07:19.659248+0000","last_peered":"2023-10-09T06:07:19.659248+0000","last_clean":"2023-10-09T06:07:19.659248+0000","last_became_active":"2023-10-09T06:07:07.547431+0000","last_became_peered":"2023-10-09T06:07:07.547431+0000","last_unstale":"2023-10-09T06:07:19.659248+0000","last_undegraded":"2023-10-09T06:07:19.659248+0000","last_fullsized":"2023-10-09T06:07:19.659248+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":1032,"log_dups_size":0,"ondisk_log_size":1032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:35.316678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67633152,"num_objects":1032,"num_object_clones":0,"num_object_copies":3096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1032,"num_write_kb":66048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":892,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.717846+0000","last_change":"2023-10-09T06:05:14.490665+0000","last_active":"2023-10-09T06:07:11.717846+0000","last_peered":"2023-10-09T06:07:11.717846+0000","last_clean":"2023-10-09T06:07:11.717846+0000","last_became_active":"2023-10-09T06:05:14.490318+0000","last_became_peered":"2023-10-09T06:05:14.490318+0000","last_unstale":"2023-10-09T06:07:11.717846+0000","last_undegraded":"2023-10-09T06:07:11.717846+0000","last_fullsized":"2023-10-09T06:07:11.717846+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:55.437947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:55.437947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:26.492183+0000","scrub_duration":0.0091262979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":214,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.715246+0000","last_change":"2023-10-09T06:01:22.355503+0000","last_active":"2023-10-09T06:07:11.715246+0000","last_peered":"2023-10-09T06:07:11.715246+0000","last_clean":"2023-10-09T06:07:11.715246+0000","last_became_active":"2023-10-09T06:01:22.354704+0000","last_became_peered":"2023-10-09T06:01:22.354704+0000","last_unstale":"2023-10-09T06:07:11.715246+0000","last_undegraded":"2023-10-09T06:07:11.715246+0000","last_fullsized":"2023-10-09T06:07:11.715246+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:56:21.446402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":383,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.570458+0000","last_change":"2023-10-09T06:01:22.332033+0000","last_active":"2023-10-09T06:07:11.570458+0000","last_peered":"2023-10-09T06:07:11.570458+0000","last_clean":"2023-10-09T06:07:11.570458+0000","last_became_active":"2023-10-09T06:01:22.329825+0000","last_became_peered":"2023-10-09T06:01:22.329825+0000","last_unstale":"2023-10-09T06:07:11.570458+0000","last_undegraded":"2023-10-09T06:07:11.570458+0000","last_fullsized":"2023-10-09T06:07:11.570458+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:54.046681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"429'1178","reported_seq":1191,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:07:20.505507+0000","last_change":"2023-10-09T06:07:08.301098+0000","last_active":"2023-10-09T06:07:20.505507+0000","last_peered":"2023-10-09T06:07:20.505507+0000","last_clean":"2023-10-09T06:07:20.505507+0000","last_became_active":"2023-10-09T06:07:08.300784+0000","last_became_peered":"2023-10-09T06:07:08.300784+0000","last_unstale":"2023-10-09T06:07:20.505507+0000","last_undegraded":"2023-10-09T06:07:20.505507+0000","last_fullsized":"2023-10-09T06:07:20.505507+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":1178,"log_dups_size":0,"ondisk_log_size":1178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:43.574832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77201408,"num_objects":1178,"num_object_clones":0,"num_object_copies":3534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1178,"num_write_kb":75392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":654,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.715187+0000","last_change":"2023-10-09T06:05:35.430902+0000","last_active":"2023-10-09T06:07:11.715187+0000","last_peered":"2023-10-09T06:07:11.715187+0000","last_clean":"2023-10-09T06:07:11.715187+0000","last_became_active":"2023-10-09T06:05:35.430492+0000","last_became_peered":"2023-10-09T06:05:35.430492+0000","last_unstale":"2023-10-09T06:07:11.715187+0000","last_undegraded":"2023-10-09T06:07:11.715187+0000","last_fullsized":"2023-10-09T06:07:11.715187+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.282393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.282393+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:19.813132+0000","scrub_duration":0.01402403,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":466,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.717702+0000","last_change":"2023-10-09T06:01:54.751188+0000","last_active":"2023-10-09T06:07:11.717702+0000","last_peered":"2023-10-09T06:07:11.717702+0000","last_clean":"2023-10-09T06:07:11.717702+0000","last_became_active":"2023-10-09T06:01:54.750870+0000","last_became_peered":"2023-10-09T06:01:54.750870+0000","last_unstale":"2023-10-09T06:07:11.717702+0000","last_undegraded":"2023-10-09T06:07:11.717702+0000","last_fullsized":"2023-10-09T06:07:11.717702+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:01.985974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":467,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.570747+0000","last_change":"2023-10-09T06:05:11.462500+0000","last_active":"2023-10-09T06:07:11.570747+0000","last_peered":"2023-10-09T06:07:11.570747+0000","last_clean":"2023-10-09T06:07:11.570747+0000","last_became_active":"2023-10-09T06:05:11.462181+0000","last_became_peered":"2023-10-09T06:05:11.462181+0000","last_unstale":"2023-10-09T06:07:11.570747+0000","last_undegraded":"2023-10-09T06:07:11.570747+0000","last_fullsized":"2023-10-09T06:07:11.570747+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:54.880123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"429'424","reported_seq":436,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:07:16.010968+0000","last_change":"2023-10-09T06:07:07.547801+0000","last_active":"2023-10-09T06:07:16.010968+0000","last_peered":"2023-10-09T06:07:16.010968+0000","last_clean":"2023-10-09T06:07:16.010968+0000","last_became_active":"2023-10-09T06:07:07.547533+0000","last_became_peered":"2023-10-09T06:07:07.547533+0000","last_unstale":"2023-10-09T06:07:16.010968+0000","last_undegraded":"2023-10-09T06:07:16.010968+0000","last_fullsized":"2023-10-09T06:07:16.010968+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":424,"log_dups_size":0,"ondisk_log_size":424,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:43:16.000531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27787264,"num_objects":424,"num_object_clones":0,"num_object_copies":1272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":424,"num_write_kb":27136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":711,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.568143+0000","last_change":"2023-10-09T06:07:06.559850+0000","last_active":"2023-10-09T06:07:11.568143+0000","last_peered":"2023-10-09T06:07:11.568143+0000","last_clean":"2023-10-09T06:07:11.568143+0000","last_became_active":"2023-10-09T06:07:06.559225+0000","last_became_peered":"2023-10-09T06:07:06.559225+0000","last_unstale":"2023-10-09T06:07:11.568143+0000","last_undegraded":"2023-10-09T06:07:11.568143+0000","last_fullsized":"2023-10-09T06:07:11.568143+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.373748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.373748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:11.407920+0000","scrub_duration":0.0024576450000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":409,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.569479+0000","last_change":"2023-10-09T06:05:11.461362+0000","last_active":"2023-10-09T06:07:11.569479+0000","last_peered":"2023-10-09T06:07:11.569479+0000","last_clean":"2023-10-09T06:07:11.569479+0000","last_became_active":"2023-10-09T06:05:11.460932+0000","last_became_peered":"2023-10-09T06:05:11.460932+0000","last_unstale":"2023-10-09T06:07:11.569479+0000","last_undegraded":"2023-10-09T06:07:11.569479+0000","last_fullsized":"2023-10-09T06:07:11.569479+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:49.980228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":450,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.571373+0000","last_change":"2023-10-09T06:06:01.068708+0000","last_active":"2023-10-09T06:07:11.571373+0000","last_peered":"2023-10-09T06:07:11.571373+0000","last_clean":"2023-10-09T06:07:11.571373+0000","last_became_active":"2023-10-09T06:06:01.068474+0000","last_became_peered":"2023-10-09T06:06:01.068474+0000","last_unstale":"2023-10-09T06:07:11.571373+0000","last_undegraded":"2023-10-09T06:07:11.571373+0000","last_fullsized":"2023-10-09T06:07:11.571373+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:31.749069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"429'1002","reported_seq":1015,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:07:19.662868+0000","last_change":"2023-10-09T06:07:07.547788+0000","last_active":"2023-10-09T06:07:19.662868+0000","last_peered":"2023-10-09T06:07:19.662868+0000","last_clean":"2023-10-09T06:07:19.662868+0000","last_became_active":"2023-10-09T06:07:07.547469+0000","last_became_peered":"2023-10-09T06:07:07.547469+0000","last_unstale":"2023-10-09T06:07:19.662868+0000","last_undegraded":"2023-10-09T06:07:19.662868+0000","last_fullsized":"2023-10-09T06:07:19.662868+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":1002,"log_dups_size":0,"ondisk_log_size":1002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:40:09.252838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65667072,"num_objects":1002,"num_object_clones":0,"num_object_copies":3006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1002,"num_write_kb":64128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":602,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.717609+0000","last_change":"2023-10-09T06:05:14.496514+0000","last_active":"2023-10-09T06:07:11.717609+0000","last_peered":"2023-10-09T06:07:11.717609+0000","last_clean":"2023-10-09T06:07:11.717609+0000","last_became_active":"2023-10-09T06:05:14.496164+0000","last_became_peered":"2023-10-09T06:05:14.496164+0000","last_unstale":"2023-10-09T06:07:11.717609+0000","last_undegraded":"2023-10-09T06:07:11.717609+0000","last_fullsized":"2023-10-09T06:07:11.717609+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:52.129816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:52.129816+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:22.553934+0000","scrub_duration":0.0039938459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":219,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.715342+0000","last_change":"2023-10-09T06:01:54.761275+0000","last_active":"2023-10-09T06:07:11.715342+0000","last_peered":"2023-10-09T06:07:11.715342+0000","last_clean":"2023-10-09T06:07:11.715342+0000","last_became_active":"2023-10-09T06:01:54.760782+0000","last_became_peered":"2023-10-09T06:01:54.760782+0000","last_unstale":"2023-10-09T06:07:11.715342+0000","last_undegraded":"2023-10-09T06:07:11.715342+0000","last_fullsized":"2023-10-09T06:07:11.715342+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:28.119846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":108,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.568498+0000","last_change":"2023-10-09T06:01:54.761768+0000","last_active":"2023-10-09T06:07:11.568498+0000","last_peered":"2023-10-09T06:07:11.568498+0000","last_clean":"2023-10-09T06:07:11.568498+0000","last_became_active":"2023-10-09T06:01:54.761443+0000","last_became_peered":"2023-10-09T06:01:54.761443+0000","last_unstale":"2023-10-09T06:07:11.568498+0000","last_undegraded":"2023-10-09T06:07:11.568498+0000","last_fullsized":"2023-10-09T06:07:11.568498+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:51.555987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"429'1066","reported_seq":1079,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:07:20.498316+0000","last_change":"2023-10-09T06:07:07.880280+0000","last_active":"2023-10-09T06:07:20.498316+0000","last_peered":"2023-10-09T06:07:20.498316+0000","last_clean":"2023-10-09T06:07:20.498316+0000","last_became_active":"2023-10-09T06:07:07.879952+0000","last_became_peered":"2023-10-09T06:07:07.879952+0000","last_unstale":"2023-10-09T06:07:20.498316+0000","last_undegraded":"2023-10-09T06:07:20.498316+0000","last_fullsized":"2023-10-09T06:07:20.498316+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":1066,"log_dups_size":0,"ondisk_log_size":1066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:10.864472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69861376,"num_objects":1066,"num_object_clones":0,"num_object_copies":3198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1066,"num_write_kb":68224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":852,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.718730+0000","last_change":"2023-10-09T05:59:56.092654+0000","last_active":"2023-10-09T06:07:11.718730+0000","last_peered":"2023-10-09T06:07:11.718730+0000","last_clean":"2023-10-09T06:07:11.718730+0000","last_became_active":"2023-10-09T05:59:32.529949+0000","last_became_peered":"2023-10-09T05:59:32.529949+0000","last_unstale":"2023-10-09T06:07:11.718730+0000","last_undegraded":"2023-10-09T06:07:11.718730+0000","last_fullsized":"2023-10-09T06:07:11.718730+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.092586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.092586+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:15:13.234903+0000","scrub_duration":0.0013484219999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":137,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.568324+0000","last_change":"2023-10-09T06:07:06.564367+0000","last_active":"2023-10-09T06:07:11.568324+0000","last_peered":"2023-10-09T06:07:11.568324+0000","last_clean":"2023-10-09T06:07:11.568324+0000","last_became_active":"2023-10-09T06:07:06.563982+0000","last_became_peered":"2023-10-09T06:07:06.563982+0000","last_unstale":"2023-10-09T06:07:11.568324+0000","last_undegraded":"2023-10-09T06:07:11.568324+0000","last_fullsized":"2023-10-09T06:07:11.568324+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:16.720442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":204,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.715438+0000","last_change":"2023-10-09T06:05:11.461458+0000","last_active":"2023-10-09T06:07:11.715438+0000","last_peered":"2023-10-09T06:07:11.715438+0000","last_clean":"2023-10-09T06:07:11.715438+0000","last_became_active":"2023-10-09T06:05:11.461054+0000","last_became_peered":"2023-10-09T06:05:11.461054+0000","last_unstale":"2023-10-09T06:07:11.715438+0000","last_undegraded":"2023-10-09T06:07:11.715438+0000","last_fullsized":"2023-10-09T06:07:11.715438+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:23:03.762018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":117,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.567801+0000","last_change":"2023-10-09T06:06:01.077664+0000","last_active":"2023-10-09T06:07:11.567801+0000","last_peered":"2023-10-09T06:07:11.567801+0000","last_clean":"2023-10-09T06:07:11.567801+0000","last_became_active":"2023-10-09T06:06:01.077052+0000","last_became_peered":"2023-10-09T06:06:01.077052+0000","last_unstale":"2023-10-09T06:07:11.567801+0000","last_undegraded":"2023-10-09T06:07:11.567801+0000","last_fullsized":"2023-10-09T06:07:11.567801+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:24.492106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"429'394","reported_seq":406,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:07:16.014396+0000","last_change":"2023-10-09T06:07:08.300794+0000","last_active":"2023-10-09T06:07:16.014396+0000","last_peered":"2023-10-09T06:07:16.014396+0000","last_clean":"2023-10-09T06:07:16.014396+0000","last_became_active":"2023-10-09T06:07:08.300476+0000","last_became_peered":"2023-10-09T06:07:08.300476+0000","last_unstale":"2023-10-09T06:07:16.014396+0000","last_undegraded":"2023-10-09T06:07:16.014396+0000","last_fullsized":"2023-10-09T06:07:16.014396+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":394,"log_dups_size":0,"ondisk_log_size":394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:43.762010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25821184,"num_objects":394,"num_object_clones":0,"num_object_copies":1182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":394,"num_write_kb":25216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":653,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:07:13.691287+0000","last_change":"2023-10-09T06:05:14.492054+0000","last_active":"2023-10-09T06:07:13.691287+0000","last_peered":"2023-10-09T06:07:13.691287+0000","last_clean":"2023-10-09T06:07:13.691287+0000","last_became_active":"2023-10-09T06:05:14.491694+0000","last_became_peered":"2023-10-09T06:05:14.491694+0000","last_unstale":"2023-10-09T06:07:13.691287+0000","last_undegraded":"2023-10-09T06:07:13.691287+0000","last_fullsized":"2023-10-09T06:07:13.691287+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:24.863363+0000","scrub_duration":0.0016660799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":108,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.567878+0000","last_change":"2023-10-09T06:01:54.754294+0000","last_active":"2023-10-09T06:07:11.567878+0000","last_peered":"2023-10-09T06:07:11.567878+0000","last_clean":"2023-10-09T06:07:11.567878+0000","last_became_active":"2023-10-09T06:01:54.754024+0000","last_became_peered":"2023-10-09T06:01:54.754024+0000","last_unstale":"2023-10-09T06:07:11.567878+0000","last_undegraded":"2023-10-09T06:07:11.567878+0000","last_fullsized":"2023-10-09T06:07:11.567878+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:19.746882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":392,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.570075+0000","last_change":"2023-10-09T06:01:22.335651+0000","last_active":"2023-10-09T06:07:11.570075+0000","last_peered":"2023-10-09T06:07:11.570075+0000","last_clean":"2023-10-09T06:07:11.570075+0000","last_became_active":"2023-10-09T06:01:22.334756+0000","last_became_peered":"2023-10-09T06:01:22.334756+0000","last_unstale":"2023-10-09T06:07:11.570075+0000","last_undegraded":"2023-10-09T06:07:11.570075+0000","last_fullsized":"2023-10-09T06:07:11.570075+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:33.123496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":242,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.568326+0000","last_change":"2023-10-09T06:07:06.561537+0000","last_active":"2023-10-09T06:07:11.568326+0000","last_peered":"2023-10-09T06:07:11.568326+0000","last_clean":"2023-10-09T06:07:11.568326+0000","last_became_active":"2023-10-09T06:07:06.561123+0000","last_became_peered":"2023-10-09T06:07:06.561123+0000","last_unstale":"2023-10-09T06:07:11.568326+0000","last_undegraded":"2023-10-09T06:07:11.568326+0000","last_fullsized":"2023-10-09T06:07:11.568326+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:48.858610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"429'1076","reported_seq":1089,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:07:20.504092+0000","last_change":"2023-10-09T06:07:07.546882+0000","last_active":"2023-10-09T06:07:20.504092+0000","last_peered":"2023-10-09T06:07:20.504092+0000","last_clean":"2023-10-09T06:07:20.504092+0000","last_became_active":"2023-10-09T06:07:07.546552+0000","last_became_peered":"2023-10-09T06:07:07.546552+0000","last_unstale":"2023-10-09T06:07:20.504092+0000","last_undegraded":"2023-10-09T06:07:20.504092+0000","last_fullsized":"2023-10-09T06:07:20.504092+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":1076,"log_dups_size":0,"ondisk_log_size":1076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:57.434915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70516736,"num_objects":1076,"num_object_clones":0,"num_object_copies":3228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1076,"num_write_kb":68864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":753,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.718165+0000","last_change":"2023-10-09T06:01:22.335906+0000","last_active":"2023-10-09T06:07:11.718165+0000","last_peered":"2023-10-09T06:07:11.718165+0000","last_clean":"2023-10-09T06:07:11.718165+0000","last_became_active":"2023-10-09T06:01:22.335590+0000","last_became_peered":"2023-10-09T06:01:22.335590+0000","last_unstale":"2023-10-09T06:07:11.718165+0000","last_undegraded":"2023-10-09T06:07:11.718165+0000","last_fullsized":"2023-10-09T06:07:11.718165+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:55.897812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:55.897812+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:02.771428+0000","scrub_duration":0.0014511420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":182,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:07:13.578207+0000","last_change":"2023-10-09T06:05:11.457098+0000","last_active":"2023-10-09T06:07:13.578207+0000","last_peered":"2023-10-09T06:07:13.578207+0000","last_clean":"2023-10-09T06:07:13.578207+0000","last_became_active":"2023-10-09T06:05:11.456658+0000","last_became_peered":"2023-10-09T06:05:11.456658+0000","last_unstale":"2023-10-09T06:07:13.578207+0000","last_undegraded":"2023-10-09T06:07:13.578207+0000","last_fullsized":"2023-10-09T06:07:13.578207+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:12.553278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":168,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.567087+0000","last_change":"2023-10-09T06:07:06.554067+0000","last_active":"2023-10-09T06:07:11.567087+0000","last_peered":"2023-10-09T06:07:11.567087+0000","last_clean":"2023-10-09T06:07:11.567087+0000","last_became_active":"2023-10-09T06:07:06.553769+0000","last_became_peered":"2023-10-09T06:07:06.553769+0000","last_unstale":"2023-10-09T06:07:11.567087+0000","last_undegraded":"2023-10-09T06:07:11.567087+0000","last_fullsized":"2023-10-09T06:07:11.567087+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:23:24.186689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":108,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.568457+0000","last_change":"2023-10-09T06:01:54.751198+0000","last_active":"2023-10-09T06:07:11.568457+0000","last_peered":"2023-10-09T06:07:11.568457+0000","last_clean":"2023-10-09T06:07:11.568457+0000","last_became_active":"2023-10-09T06:01:54.750862+0000","last_became_peered":"2023-10-09T06:01:54.750862+0000","last_unstale":"2023-10-09T06:07:11.568457+0000","last_undegraded":"2023-10-09T06:07:11.568457+0000","last_fullsized":"2023-10-09T06:07:11.568457+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:35.203567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"429'1009","reported_seq":1020,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:07:19.670311+0000","last_change":"2023-10-09T06:07:08.300703+0000","last_active":"2023-10-09T06:07:19.670311+0000","last_peered":"2023-10-09T06:07:19.670311+0000","last_clean":"2023-10-09T06:07:19.670311+0000","last_became_active":"2023-10-09T06:07:08.300414+0000","last_became_peered":"2023-10-09T06:07:08.300414+0000","last_unstale":"2023-10-09T06:07:19.670311+0000","last_undegraded":"2023-10-09T06:07:19.670311+0000","last_fullsized":"2023-10-09T06:07:19.670311+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":1009,"log_dups_size":0,"ondisk_log_size":1009,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:37:03.895557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66125824,"num_objects":1009,"num_object_clones":0,"num_object_copies":3027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1009,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1009,"num_write_kb":64576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":310,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.718293+0000","last_change":"2023-10-09T06:05:11.457391+0000","last_active":"2023-10-09T06:07:11.718293+0000","last_peered":"2023-10-09T06:07:11.718293+0000","last_clean":"2023-10-09T06:07:11.718293+0000","last_became_active":"2023-10-09T06:05:11.457072+0000","last_became_peered":"2023-10-09T06:05:11.457072+0000","last_unstale":"2023-10-09T06:07:11.718293+0000","last_undegraded":"2023-10-09T06:07:11.718293+0000","last_fullsized":"2023-10-09T06:07:11.718293+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:19:46.357642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":833,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.717580+0000","last_change":"2023-10-09T05:58:37.782237+0000","last_active":"2023-10-09T06:07:11.717580+0000","last_peered":"2023-10-09T06:07:11.717580+0000","last_clean":"2023-10-09T06:07:11.717580+0000","last_became_active":"2023-10-09T05:53:59.202004+0000","last_became_peered":"2023-10-09T05:53:59.202004+0000","last_unstale":"2023-10-09T06:07:11.717580+0000","last_undegraded":"2023-10-09T06:07:11.717580+0000","last_fullsized":"2023-10-09T06:07:11.717580+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:37.782173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:37.782173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:20.787963+0000","scrub_duration":0.001325908,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":195,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.715817+0000","last_change":"2023-10-09T05:57:23.479174+0000","last_active":"2023-10-09T06:07:11.715817+0000","last_peered":"2023-10-09T06:07:11.715817+0000","last_clean":"2023-10-09T06:07:11.715817+0000","last_became_active":"2023-10-09T05:57:23.478897+0000","last_became_peered":"2023-10-09T05:57:23.478897+0000","last_unstale":"2023-10-09T06:07:11.715817+0000","last_undegraded":"2023-10-09T06:07:11.715817+0000","last_fullsized":"2023-10-09T06:07:11.715817+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:00.160042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":332,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.570697+0000","last_change":"2023-10-09T06:06:01.071756+0000","last_active":"2023-10-09T06:07:11.570697+0000","last_peered":"2023-10-09T06:07:11.570697+0000","last_clean":"2023-10-09T06:07:11.570697+0000","last_became_active":"2023-10-09T06:06:01.071341+0000","last_became_peered":"2023-10-09T06:06:01.071341+0000","last_unstale":"2023-10-09T06:07:11.570697+0000","last_undegraded":"2023-10-09T06:07:11.570697+0000","last_fullsized":"2023-10-09T06:07:11.570697+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:17.804204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"429'884","reported_seq":896,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:07:18.698236+0000","last_change":"2023-10-09T06:07:08.300727+0000","last_active":"2023-10-09T06:07:18.698236+0000","last_peered":"2023-10-09T06:07:18.698236+0000","last_clean":"2023-10-09T06:07:18.698236+0000","last_became_active":"2023-10-09T06:07:08.300436+0000","last_became_peered":"2023-10-09T06:07:08.300436+0000","last_unstale":"2023-10-09T06:07:18.698236+0000","last_undegraded":"2023-10-09T06:07:18.698236+0000","last_fullsized":"2023-10-09T06:07:18.698236+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":884,"log_dups_size":0,"ondisk_log_size":884,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:50:16.105023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57933824,"num_objects":884,"num_object_clones":0,"num_object_copies":2652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":884,"num_write_kb":56576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":643,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.568176+0000","last_change":"2023-10-09T06:07:06.560255+0000","last_active":"2023-10-09T06:07:11.568176+0000","last_peered":"2023-10-09T06:07:11.568176+0000","last_clean":"2023-10-09T06:07:11.568176+0000","last_became_active":"2023-10-09T06:07:06.559524+0000","last_became_peered":"2023-10-09T06:07:06.559524+0000","last_unstale":"2023-10-09T06:07:11.568176+0000","last_undegraded":"2023-10-09T06:07:11.568176+0000","last_fullsized":"2023-10-09T06:07:11.568176+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:57.627296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:57.627296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:41.099391+0000","scrub_duration":0.001913166,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":305,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.717454+0000","last_change":"2023-10-09T06:01:54.751146+0000","last_active":"2023-10-09T06:07:11.717454+0000","last_peered":"2023-10-09T06:07:11.717454+0000","last_clean":"2023-10-09T06:07:11.717454+0000","last_became_active":"2023-10-09T06:01:54.750833+0000","last_became_peered":"2023-10-09T06:01:54.750833+0000","last_unstale":"2023-10-09T06:07:11.717454+0000","last_undegraded":"2023-10-09T06:07:11.717454+0000","last_fullsized":"2023-10-09T06:07:11.717454+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:54.035203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":385,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.716781+0000","last_change":"2023-10-09T06:01:22.320317+0000","last_active":"2023-10-09T06:07:11.716781+0000","last_peered":"2023-10-09T06:07:11.716781+0000","last_clean":"2023-10-09T06:07:11.716781+0000","last_became_active":"2023-10-09T06:01:22.319989+0000","last_became_peered":"2023-10-09T06:01:22.319989+0000","last_unstale":"2023-10-09T06:07:11.716781+0000","last_undegraded":"2023-10-09T06:07:11.716781+0000","last_fullsized":"2023-10-09T06:07:11.716781+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:06.219647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":505,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:07:13.691706+0000","last_change":"2023-10-09T06:05:11.456603+0000","last_active":"2023-10-09T06:07:13.691706+0000","last_peered":"2023-10-09T06:07:13.691706+0000","last_clean":"2023-10-09T06:07:13.691706+0000","last_became_active":"2023-10-09T06:05:11.456336+0000","last_became_peered":"2023-10-09T06:05:11.456336+0000","last_unstale":"2023-10-09T06:07:13.691706+0000","last_undegraded":"2023-10-09T06:07:13.691706+0000","last_fullsized":"2023-10-09T06:07:13.691706+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:42.222096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"429'374","reported_seq":386,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:07:16.021816+0000","last_change":"2023-10-09T06:07:07.579770+0000","last_active":"2023-10-09T06:07:16.021816+0000","last_peered":"2023-10-09T06:07:16.021816+0000","last_clean":"2023-10-09T06:07:16.021816+0000","last_became_active":"2023-10-09T06:07:07.579317+0000","last_became_peered":"2023-10-09T06:07:07.579317+0000","last_unstale":"2023-10-09T06:07:16.021816+0000","last_undegraded":"2023-10-09T06:07:16.021816+0000","last_fullsized":"2023-10-09T06:07:16.021816+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":374,"log_dups_size":0,"ondisk_log_size":374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:30.621836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24510464,"num_objects":374,"num_object_clones":0,"num_object_copies":1122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":374,"num_write_kb":23936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":842,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.718988+0000","last_change":"2023-10-09T05:59:54.074413+0000","last_active":"2023-10-09T06:07:11.718988+0000","last_peered":"2023-10-09T06:07:11.718988+0000","last_clean":"2023-10-09T06:07:11.718988+0000","last_became_active":"2023-10-09T05:59:32.538959+0000","last_became_peered":"2023-10-09T05:59:32.538959+0000","last_unstale":"2023-10-09T06:07:11.718988+0000","last_undegraded":"2023-10-09T06:07:11.718988+0000","last_fullsized":"2023-10-09T06:07:11.718988+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:54.074310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:54.074310+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:10.811269+0000","scrub_duration":0.011040648,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":280,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.718391+0000","last_change":"2023-10-09T06:01:22.337610+0000","last_active":"2023-10-09T06:07:11.718391+0000","last_peered":"2023-10-09T06:07:11.718391+0000","last_clean":"2023-10-09T06:07:11.718391+0000","last_became_active":"2023-10-09T06:01:22.336719+0000","last_became_peered":"2023-10-09T06:01:22.336719+0000","last_unstale":"2023-10-09T06:07:11.718391+0000","last_undegraded":"2023-10-09T06:07:11.718391+0000","last_fullsized":"2023-10-09T06:07:11.718391+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:33:29.587947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":117,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.568386+0000","last_change":"2023-10-09T06:05:11.460787+0000","last_active":"2023-10-09T06:07:11.568386+0000","last_peered":"2023-10-09T06:07:11.568386+0000","last_clean":"2023-10-09T06:07:11.568386+0000","last_became_active":"2023-10-09T06:05:11.460451+0000","last_became_peered":"2023-10-09T06:05:11.460451+0000","last_unstale":"2023-10-09T06:07:11.568386+0000","last_undegraded":"2023-10-09T06:07:11.568386+0000","last_fullsized":"2023-10-09T06:07:11.568386+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:32:05.202156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":201,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.718362+0000","last_change":"2023-10-09T06:01:41.938541+0000","last_active":"2023-10-09T06:07:11.718362+0000","last_peered":"2023-10-09T06:07:11.718362+0000","last_clean":"2023-10-09T06:07:11.718362+0000","last_became_active":"2023-10-09T06:01:41.938289+0000","last_became_peered":"2023-10-09T06:01:41.938289+0000","last_unstale":"2023-10-09T06:07:11.718362+0000","last_undegraded":"2023-10-09T06:07:11.718362+0000","last_fullsized":"2023-10-09T06:07:11.718362+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:29.735530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.d","version":"429'387","reported_seq":399,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:07:16.009120+0000","last_change":"2023-10-09T06:07:07.883000+0000","last_active":"2023-10-09T06:07:16.009120+0000","last_peered":"2023-10-09T06:07:16.009120+0000","last_clean":"2023-10-09T06:07:16.009120+0000","last_became_active":"2023-10-09T06:07:07.882612+0000","last_became_peered":"2023-10-09T06:07:07.882612+0000","last_unstale":"2023-10-09T06:07:16.009120+0000","last_undegraded":"2023-10-09T06:07:16.009120+0000","last_fullsized":"2023-10-09T06:07:16.009120+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":387,"log_dups_size":0,"ondisk_log_size":387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:14:21.633273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25362432,"num_objects":387,"num_object_clones":0,"num_object_copies":1161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":387,"num_write_kb":24768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":862,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.570824+0000","last_change":"2023-10-09T06:06:03.450574+0000","last_active":"2023-10-09T06:07:11.570824+0000","last_peered":"2023-10-09T06:07:11.570824+0000","last_clean":"2023-10-09T06:07:11.570824+0000","last_became_active":"2023-10-09T06:06:03.450258+0000","last_became_peered":"2023-10-09T06:06:03.450258+0000","last_unstale":"2023-10-09T06:07:11.570824+0000","last_undegraded":"2023-10-09T06:07:11.570824+0000","last_fullsized":"2023-10-09T06:07:11.570824+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:17.344518+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:17.344518+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:35.038506+0000","scrub_duration":0.0015914099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":439,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.716395+0000","last_change":"2023-10-09T05:53:59.228948+0000","last_active":"2023-10-09T06:07:11.716395+0000","last_peered":"2023-10-09T06:07:11.716395+0000","last_clean":"2023-10-09T06:07:11.716395+0000","last_became_active":"2023-10-09T05:53:59.186359+0000","last_became_peered":"2023-10-09T05:53:59.186359+0000","last_unstale":"2023-10-09T06:07:11.716395+0000","last_undegraded":"2023-10-09T06:07:11.716395+0000","last_fullsized":"2023-10-09T06:07:11.716395+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:39.990576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":224,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.568826+0000","last_change":"2023-10-09T06:07:06.560512+0000","last_active":"2023-10-09T06:07:11.568826+0000","last_peered":"2023-10-09T06:07:11.568826+0000","last_clean":"2023-10-09T06:07:11.568826+0000","last_became_active":"2023-10-09T06:07:06.560204+0000","last_became_peered":"2023-10-09T06:07:06.560204+0000","last_unstale":"2023-10-09T06:07:11.568826+0000","last_undegraded":"2023-10-09T06:07:11.568826+0000","last_fullsized":"2023-10-09T06:07:11.568826+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:19.102248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":195,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.715568+0000","last_change":"2023-10-09T05:57:23.477143+0000","last_active":"2023-10-09T06:07:11.715568+0000","last_peered":"2023-10-09T06:07:11.715568+0000","last_clean":"2023-10-09T06:07:11.715568+0000","last_became_active":"2023-10-09T05:57:23.475655+0000","last_became_peered":"2023-10-09T05:57:23.475655+0000","last_unstale":"2023-10-09T06:07:11.715568+0000","last_undegraded":"2023-10-09T06:07:11.715568+0000","last_fullsized":"2023-10-09T06:07:11.715568+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:29.528023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"429'956","reported_seq":968,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:07:19.676172+0000","last_change":"2023-10-09T06:07:07.580272+0000","last_active":"2023-10-09T06:07:19.676172+0000","last_peered":"2023-10-09T06:07:19.676172+0000","last_clean":"2023-10-09T06:07:19.676172+0000","last_became_active":"2023-10-09T06:07:07.579909+0000","last_became_peered":"2023-10-09T06:07:07.579909+0000","last_unstale":"2023-10-09T06:07:19.676172+0000","last_undegraded":"2023-10-09T06:07:19.676172+0000","last_fullsized":"2023-10-09T06:07:19.676172+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":956,"log_dups_size":0,"ondisk_log_size":956,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:18.048371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62652416,"num_objects":956,"num_object_clones":0,"num_object_copies":2868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":956,"num_write_kb":61184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":689,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.715337+0000","last_change":"2023-10-09T05:58:34.333129+0000","last_active":"2023-10-09T06:07:11.715337+0000","last_peered":"2023-10-09T06:07:11.715337+0000","last_clean":"2023-10-09T06:07:11.715337+0000","last_became_active":"2023-10-09T05:57:26.502911+0000","last_became_peered":"2023-10-09T05:57:26.502911+0000","last_unstale":"2023-10-09T06:07:11.715337+0000","last_undegraded":"2023-10-09T06:07:11.715337+0000","last_fullsized":"2023-10-09T06:07:11.715337+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:25.230712+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":443,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.718825+0000","last_change":"2023-10-09T06:05:11.463130+0000","last_active":"2023-10-09T06:07:11.718825+0000","last_peered":"2023-10-09T06:07:11.718825+0000","last_clean":"2023-10-09T06:07:11.718825+0000","last_became_active":"2023-10-09T06:05:11.461340+0000","last_became_peered":"2023-10-09T06:05:11.461340+0000","last_unstale":"2023-10-09T06:07:11.718825+0000","last_undegraded":"2023-10-09T06:07:11.718825+0000","last_fullsized":"2023-10-09T06:07:11.718825+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:19.274025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":501,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:07:13.801240+0000","last_change":"2023-10-09T06:01:54.776025+0000","last_active":"2023-10-09T06:07:13.801240+0000","last_peered":"2023-10-09T06:07:13.801240+0000","last_clean":"2023-10-09T06:07:13.801240+0000","last_became_active":"2023-10-09T06:01:54.753050+0000","last_became_peered":"2023-10-09T06:01:54.753050+0000","last_unstale":"2023-10-09T06:07:13.801240+0000","last_undegraded":"2023-10-09T06:07:13.801240+0000","last_fullsized":"2023-10-09T06:07:13.801240+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:58:24.000420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":375,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.569554+0000","last_change":"2023-10-09T06:01:22.334081+0000","last_active":"2023-10-09T06:07:11.569554+0000","last_peered":"2023-10-09T06:07:11.569554+0000","last_clean":"2023-10-09T06:07:11.569554+0000","last_became_active":"2023-10-09T06:01:22.333663+0000","last_became_peered":"2023-10-09T06:01:22.333663+0000","last_unstale":"2023-10-09T06:07:11.569554+0000","last_undegraded":"2023-10-09T06:07:11.569554+0000","last_fullsized":"2023-10-09T06:07:11.569554+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:55.809259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":657,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.715753+0000","last_change":"2023-10-09T06:00:06.181968+0000","last_active":"2023-10-09T06:07:11.715753+0000","last_peered":"2023-10-09T06:07:11.715753+0000","last_clean":"2023-10-09T06:07:11.715753+0000","last_became_active":"2023-10-09T05:59:40.324653+0000","last_became_peered":"2023-10-09T05:59:40.324653+0000","last_unstale":"2023-10-09T06:07:11.715753+0000","last_undegraded":"2023-10-09T06:07:11.715753+0000","last_fullsized":"2023-10-09T06:07:11.715753+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.181811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.181811+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:05:16.987523+0000","scrub_duration":0.011553313000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":377,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.716090+0000","last_change":"2023-10-09T06:01:22.349718+0000","last_active":"2023-10-09T06:07:11.716090+0000","last_peered":"2023-10-09T06:07:11.716090+0000","last_clean":"2023-10-09T06:07:11.716090+0000","last_became_active":"2023-10-09T06:01:22.349401+0000","last_became_peered":"2023-10-09T06:01:22.349401+0000","last_unstale":"2023-10-09T06:07:11.716090+0000","last_undegraded":"2023-10-09T06:07:11.716090+0000","last_fullsized":"2023-10-09T06:07:11.716090+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:08.983715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":119,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.568414+0000","last_change":"2023-10-09T06:07:06.561382+0000","last_active":"2023-10-09T06:07:11.568414+0000","last_peered":"2023-10-09T06:07:11.568414+0000","last_clean":"2023-10-09T06:07:11.568414+0000","last_became_active":"2023-10-09T06:07:06.560583+0000","last_became_peered":"2023-10-09T06:07:06.560583+0000","last_unstale":"2023-10-09T06:07:11.568414+0000","last_undegraded":"2023-10-09T06:07:11.568414+0000","last_fullsized":"2023-10-09T06:07:11.568414+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:14.594951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":774,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.715967+0000","last_change":"2023-10-09T06:00:04.215425+0000","last_active":"2023-10-09T06:07:11.715967+0000","last_peered":"2023-10-09T06:07:11.715967+0000","last_clean":"2023-10-09T06:07:11.715967+0000","last_became_active":"2023-10-09T05:59:40.326685+0000","last_became_peered":"2023-10-09T05:59:40.326685+0000","last_unstale":"2023-10-09T06:07:11.715967+0000","last_undegraded":"2023-10-09T06:07:11.715967+0000","last_fullsized":"2023-10-09T06:07:11.715967+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:04.215270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:04.215270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:03.963694+0000","scrub_duration":0.002850652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":200,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.570412+0000","last_change":"2023-10-09T06:01:54.824780+0000","last_active":"2023-10-09T06:07:11.570412+0000","last_peered":"2023-10-09T06:07:11.570412+0000","last_clean":"2023-10-09T06:07:11.570412+0000","last_became_active":"2023-10-09T06:01:54.757727+0000","last_became_peered":"2023-10-09T06:01:54.757727+0000","last_unstale":"2023-10-09T06:07:11.570412+0000","last_undegraded":"2023-10-09T06:07:11.570412+0000","last_fullsized":"2023-10-09T06:07:11.570412+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:44.132556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":447,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.570398+0000","last_change":"2023-10-09T06:01:54.764989+0000","last_active":"2023-10-09T06:07:11.570398+0000","last_peered":"2023-10-09T06:07:11.570398+0000","last_clean":"2023-10-09T06:07:11.570398+0000","last_became_active":"2023-10-09T06:01:54.764790+0000","last_became_peered":"2023-10-09T06:01:54.764790+0000","last_unstale":"2023-10-09T06:07:11.570398+0000","last_undegraded":"2023-10-09T06:07:11.570398+0000","last_fullsized":"2023-10-09T06:07:11.570398+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:14:07.208858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":186,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.570067+0000","last_change":"2023-10-09T06:01:22.342190+0000","last_active":"2023-10-09T06:07:11.570067+0000","last_peered":"2023-10-09T06:07:11.570067+0000","last_clean":"2023-10-09T06:07:11.570067+0000","last_became_active":"2023-10-09T06:01:22.341855+0000","last_became_peered":"2023-10-09T06:01:22.341855+0000","last_unstale":"2023-10-09T06:07:11.570067+0000","last_undegraded":"2023-10-09T06:07:11.570067+0000","last_fullsized":"2023-10-09T06:07:11.570067+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:01:21.322371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":427,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.569810+0000","last_change":"2023-10-09T06:05:11.455187+0000","last_active":"2023-10-09T06:07:11.569810+0000","last_peered":"2023-10-09T06:07:11.569810+0000","last_clean":"2023-10-09T06:07:11.569810+0000","last_became_active":"2023-10-09T06:05:11.454866+0000","last_became_peered":"2023-10-09T06:05:11.454866+0000","last_unstale":"2023-10-09T06:07:11.569810+0000","last_undegraded":"2023-10-09T06:07:11.569810+0000","last_fullsized":"2023-10-09T06:07:11.569810+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:39.993744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":695,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.718322+0000","last_change":"2023-10-09T06:01:57.801991+0000","last_active":"2023-10-09T06:07:11.718322+0000","last_peered":"2023-10-09T06:07:11.718322+0000","last_clean":"2023-10-09T06:07:11.718322+0000","last_became_active":"2023-10-09T06:01:57.801579+0000","last_became_peered":"2023-10-09T06:01:57.801579+0000","last_unstale":"2023-10-09T06:07:11.718322+0000","last_undegraded":"2023-10-09T06:07:11.718322+0000","last_fullsized":"2023-10-09T06:07:11.718322+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:00.456132+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:00.456132+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:48.548353+0000","scrub_duration":0.0014992199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":213,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.716284+0000","last_change":"2023-10-09T06:06:01.074307+0000","last_active":"2023-10-09T06:07:11.716284+0000","last_peered":"2023-10-09T06:07:11.716284+0000","last_clean":"2023-10-09T06:07:11.716284+0000","last_became_active":"2023-10-09T06:06:01.073884+0000","last_became_peered":"2023-10-09T06:06:01.073884+0000","last_unstale":"2023-10-09T06:07:11.716284+0000","last_undegraded":"2023-10-09T06:07:11.716284+0000","last_fullsized":"2023-10-09T06:07:11.716284+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:15.768039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":204,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.715538+0000","last_change":"2023-10-09T06:05:11.456988+0000","last_active":"2023-10-09T06:07:11.715538+0000","last_peered":"2023-10-09T06:07:11.715538+0000","last_clean":"2023-10-09T06:07:11.715538+0000","last_became_active":"2023-10-09T06:05:11.456663+0000","last_became_peered":"2023-10-09T06:05:11.456663+0000","last_unstale":"2023-10-09T06:07:11.715538+0000","last_undegraded":"2023-10-09T06:07:11.715538+0000","last_fullsized":"2023-10-09T06:07:11.715538+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:13.286427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":716,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.569078+0000","last_change":"2023-10-09T06:07:06.560275+0000","last_active":"2023-10-09T06:07:11.569078+0000","last_peered":"2023-10-09T06:07:11.569078+0000","last_clean":"2023-10-09T06:07:11.569078+0000","last_became_active":"2023-10-09T06:07:06.559953+0000","last_became_peered":"2023-10-09T06:07:06.559953+0000","last_unstale":"2023-10-09T06:07:11.569078+0000","last_undegraded":"2023-10-09T06:07:11.569078+0000","last_fullsized":"2023-10-09T06:07:11.569078+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:46.290510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:46.290510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:10.910416+0000","scrub_duration":0.002627819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":415,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.715535+0000","last_change":"2023-10-09T06:05:11.486450+0000","last_active":"2023-10-09T06:07:11.715535+0000","last_peered":"2023-10-09T06:07:11.715535+0000","last_clean":"2023-10-09T06:07:11.715535+0000","last_became_active":"2023-10-09T06:05:11.459761+0000","last_became_peered":"2023-10-09T06:05:11.459761+0000","last_unstale":"2023-10-09T06:07:11.715535+0000","last_undegraded":"2023-10-09T06:07:11.715535+0000","last_fullsized":"2023-10-09T06:07:11.715535+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:31.896189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":450,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:07:11.569292+0000","last_change":"2023-10-09T06:06:01.068828+0000","last_active":"2023-10-09T06:07:11.569292+0000","last_peered":"2023-10-09T06:07:11.569292+0000","last_clean":"2023-10-09T06:07:11.569292+0000","last_became_active":"2023-10-09T06:06:01.068491+0000","last_became_peered":"2023-10-09T06:06:01.068491+0000","last_unstale":"2023-10-09T06:07:11.569292+0000","last_undegraded":"2023-10-09T06:07:11.569292+0000","last_fullsized":"2023-10-09T06:07:11.569292+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:25.661796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":1982,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10938,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":837877760,"num_objects":12785,"num_object_clones":0,"num_object_copies":38355,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12785,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12785,"num_write_kb":818240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2797600768,"data_stored":2797600768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":12785,"ondisk_log_size":12785,"up":48,"acting":48,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234331,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":764476,"kb_used_data":174708,"kb_used_omap":0,"kb_used_meta":589760,"kb_avail":92980676,"statfs":{"total":95995035648,"available":95212212224,"internally_reserved":0,"allocated":178900992,"data_stored":177670526,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603914240},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":2.7919999999999998,"15min":3.2309999999999999},"min":{"1min":0.22700000000000001,"5min":0.218,"15min":0.218},"max":{"1min":5.0229999999999997,"5min":99.459999999999994,"15min":165.792},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.5800000000000001,"5min":3.0670000000000002,"15min":3.3700000000000001},"min":{"1min":0.26700000000000002,"5min":0.24399999999999999,"15min":0.21299999999999999},"max":{"1min":9.7149999999999999,"5min":100.096,"15min":166.00700000000001},"last":0.71999999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:06:49 2023","interfaces":[{"interface":"back","average":{"1min":2.2839999999999998,"5min":3.964,"15min":2.222},"min":{"1min":0.30499999999999999,"5min":0.25,"15min":0.22900000000000001},"max":{"1min":7.5940000000000003,"5min":64.760000000000005,"15min":64.760000000000005},"last":0.76100000000000001},{"interface":"front","average":{"1min":1.7509999999999999,"5min":3.9620000000000002,"15min":2.0840000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.0339999999999998,"5min":65.102000000000004,"15min":65.102000000000004},"last":1.0509999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.27,"5min":1.6020000000000001,"15min":2.9180000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.8319999999999999,"5min":16.878,"15min":127.86499999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.3680000000000001,"5min":1.7969999999999999,"15min":2.8650000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.29599999999999999},"max":{"1min":5.1699999999999999,"5min":12.456,"15min":129.24000000000001},"last":0.53800000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:06:49 2023","interfaces":[{"interface":"back","average":{"1min":2.226,"5min":2.3570000000000002,"15min":5.5439999999999996},"min":{"1min":0.373,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":7.7439999999999998,"5min":36.402000000000001,"15min":164.16999999999999},"last":1.115},{"interface":"front","average":{"1min":2.2269999999999999,"5min":2.5350000000000001,"15min":5.5060000000000002},"min":{"1min":0.372,"5min":0.34399999999999997,"15min":0.30299999999999999},"max":{"1min":6.069,"5min":36.872,"15min":164.40700000000001},"last":0.47699999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:06:25 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":0.78000000000000003,"15min":0.79600000000000004},"min":{"1min":0.27000000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.2999999999999998,"5min":5.5570000000000004,"15min":5.5570000000000004},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.2150000000000001,"5min":0.80400000000000005,"15min":0.83799999999999997},"min":{"1min":0.34200000000000003,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.669,"5min":4.593,"15min":4.593},"last":1.0840000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:06:38 2023","interfaces":[{"interface":"back","average":{"1min":1.5329999999999999,"5min":0.996,"15min":0.97999999999999998},"min":{"1min":0.28199999999999997,"5min":0.247,"15min":0.247},"max":{"1min":9.7910000000000004,"5min":9.7910000000000004,"15min":15.622},"last":2.25},{"interface":"front","average":{"1min":1.345,"5min":1.1000000000000001,"15min":0.86699999999999999},"min":{"1min":0.27400000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":9.3409999999999993,"5min":9.3409999999999993,"15min":9.3409999999999993},"last":0.50900000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:06:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0619999999999998,"5min":1.1759999999999999,"15min":1.5369999999999999},"min":{"1min":0.318,"5min":0.28499999999999998,"15min":0.28000000000000003},"max":{"1min":6.1379999999999999,"5min":6.6219999999999999,"15min":131.94399999999999},"last":0.97799999999999998},{"interface":"front","average":{"1min":1.429,"5min":0.97399999999999998,"15min":1.476},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.313},"max":{"1min":4.7690000000000001,"5min":5.5800000000000001,"15min":135.23400000000001},"last":0.39800000000000002}]}]},{"osd":6,"up_from":46,"seq":197568495968,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1110896,"kb_used_data":213352,"kb_used_omap":0,"kb_used_meta":897536,"kb_avail":92634256,"statfs":{"total":95995035648,"available":94857478144,"internally_reserved":0,"allocated":218472448,"data_stored":217253903,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919076864},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":5.782,"15min":3.6629999999999998},"min":{"1min":0.27100000000000002,"5min":0.24299999999999999,"15min":0.224},"max":{"1min":5.0289999999999999,"5min":225.422,"15min":225.422},"last":0.70399999999999996},{"interface":"front","average":{"1min":2.0329999999999999,"5min":6.1470000000000002,"15min":3.794},"min":{"1min":0.28899999999999998,"5min":0.23000000000000001,"15min":0.22800000000000001},"max":{"1min":15.311999999999999,"5min":225.255,"15min":225.255},"last":0.316}]},{"osd":1,"last update":"Mon Oct 9 06:07:03 2023","interfaces":[{"interface":"back","average":{"1min":6.3220000000000001,"5min":6.5259999999999998,"15min":2.875},"min":{"1min":0.29999999999999999,"5min":0.28399999999999997,"15min":0.246},"max":{"1min":77.930999999999997,"5min":109.101,"15min":109.101},"last":0.69499999999999995},{"interface":"front","average":{"1min":7.1689999999999996,"5min":6.3680000000000003,"15min":3.016},"min":{"1min":0.39700000000000002,"5min":0.27900000000000003,"15min":0.27000000000000002},"max":{"1min":77.911000000000001,"5min":109.071,"15min":109.071},"last":0.80200000000000005}]},{"osd":2,"last update":"Mon Oct 9 06:07:13 2023","interfaces":[{"interface":"back","average":{"1min":2.3500000000000001,"5min":1.976,"15min":2.0790000000000002},"min":{"1min":0.28299999999999997,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":15.374000000000001,"5min":15.374000000000001,"15min":61.088999999999999},"last":0.64000000000000001},{"interface":"front","average":{"1min":1.6399999999999999,"5min":1.7210000000000001,"15min":2.016},"min":{"1min":0.32800000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":15.429,"5min":15.429,"15min":61.082000000000001},"last":0.65300000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:06:57 2023","interfaces":[{"interface":"back","average":{"1min":9.4359999999999999,"5min":6.4950000000000001,"15min":5.5110000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.29599999999999999},"max":{"1min":138.13300000000001,"5min":138.13300000000001,"15min":138.13300000000001},"last":0.60099999999999998},{"interface":"front","average":{"1min":9.9280000000000008,"5min":6.6230000000000002,"15min":5.4770000000000003},"min":{"1min":0.36599999999999999,"5min":0.34200000000000003,"15min":0.28499999999999998},"max":{"1min":137.215,"5min":137.215,"15min":137.215},"last":0.72399999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:06:33 2023","interfaces":[{"interface":"back","average":{"1min":1.829,"5min":1.3759999999999999,"15min":1.323},"min":{"1min":0.28100000000000003,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":16.207000000000001,"5min":16.207000000000001,"15min":16.207000000000001},"last":0.68500000000000005},{"interface":"front","average":{"1min":2.7759999999999998,"5min":1.5,"15min":1.232},"min":{"1min":0.30599999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":21.927,"5min":21.927,"15min":21.927},"last":0.48599999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:07:03 2023","interfaces":[{"interface":"back","average":{"1min":1.9510000000000001,"5min":1.379,"15min":1.2829999999999999},"min":{"1min":0.37,"5min":0.29799999999999999,"15min":0.23899999999999999},"max":{"1min":15.342000000000001,"5min":15.342000000000001,"15min":45.933999999999997},"last":0.74099999999999999},{"interface":"front","average":{"1min":1.833,"5min":1.4319999999999999,"15min":1.2689999999999999},"min":{"1min":0.29199999999999998,"5min":0.27700000000000002,"15min":0.255},"max":{"1min":15.358000000000001,"5min":15.358000000000001,"15min":45.917999999999999},"last":0.66900000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:06:33 2023","interfaces":[{"interface":"back","average":{"1min":2.6230000000000002,"5min":1.4350000000000001,"15min":1.4419999999999999},"min":{"1min":0.312,"5min":0.29399999999999998,"15min":0.27000000000000002},"max":{"1min":21.98,"5min":21.98,"15min":52.042999999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":2.2349999999999999,"5min":1.538,"15min":1.353},"min":{"1min":0.33300000000000002,"5min":0.33100000000000002,"15min":0.32200000000000001},"max":{"1min":18.241,"5min":18.241,"15min":51.259},"last":0.625}]}]},{"osd":1,"up_from":89,"seq":382252089557,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1022560,"kb_used_data":639384,"kb_used_omap":9,"kb_used_meta":383158,"kb_avail":92722592,"statfs":{"total":95995035648,"available":94947934208,"internally_reserved":0,"allocated":654729216,"data_stored":652290224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":392354793},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:06:24 2023","interfaces":[{"interface":"back","average":{"1min":6.7919999999999998,"5min":5.4859999999999998,"15min":3.2029999999999998},"min":{"1min":0.21199999999999999,"5min":0.20100000000000001,"15min":0.191},"max":{"1min":116.84099999999999,"5min":116.84099999999999,"15min":132.04499999999999},"last":0.86499999999999999},{"interface":"front","average":{"1min":7.024,"5min":5.5430000000000001,"15min":3.4039999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.21299999999999999},"max":{"1min":116.941,"5min":116.941,"15min":132.91200000000001},"last":0.59399999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:06:32 2023","interfaces":[{"interface":"back","average":{"1min":1.4079999999999999,"5min":1.5389999999999999,"15min":3.2069999999999999},"min":{"1min":0.27200000000000002,"5min":0.20799999999999999,"15min":0.20100000000000001},"max":{"1min":9.7550000000000008,"5min":11.631,"15min":204.989},"last":0.96799999999999997},{"interface":"front","average":{"1min":1.4630000000000001,"5min":1.8420000000000001,"15min":3.4769999999999999},"min":{"1min":0.312,"5min":0.26400000000000001,"15min":0.20300000000000001},"max":{"1min":10.699,"5min":12.728,"15min":203.65899999999999},"last":1.0449999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:06:51 2023","interfaces":[{"interface":"back","average":{"1min":9.0359999999999996,"5min":4.923,"15min":5.2430000000000003},"min":{"1min":0.317,"5min":0.25600000000000001,"15min":0.214},"max":{"1min":142.97,"5min":142.97,"15min":213.654},"last":0.27100000000000002},{"interface":"front","average":{"1min":8.8829999999999991,"5min":4.9029999999999996,"15min":5.306},"min":{"1min":0.41099999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":142.94800000000001,"5min":142.94800000000001,"15min":213.77199999999999},"last":0.41299999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:06:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1499999999999999,"5min":1.6919999999999999,"15min":1.252},"min":{"1min":0.40799999999999997,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":10.782,"5min":10.782,"15min":10.782},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.7929999999999999,"5min":1.738,"15min":1.6439999999999999},"min":{"1min":0.35999999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":5.2009999999999996,"5min":9.8629999999999995,"15min":9.8629999999999995},"last":1.234}]},{"osd":5,"last update":"Mon Oct 9 06:06:42 2023","interfaces":[{"interface":"back","average":{"1min":1.738,"5min":1.9299999999999999,"15min":1.5609999999999999},"min":{"1min":0.35499999999999998,"5min":0.34300000000000003,"15min":0.32800000000000001},"max":{"1min":10.750999999999999,"5min":36.356999999999999,"15min":64.584999999999994},"last":0.47699999999999998},{"interface":"front","average":{"1min":2.21,"5min":2.4060000000000001,"15min":1.8049999999999999},"min":{"1min":0.376,"5min":0.35799999999999998,"15min":0.315},"max":{"1min":8.2370000000000001,"5min":36.268000000000001,"15min":64.150999999999996},"last":1.3560000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:06:42 2023","interfaces":[{"interface":"back","average":{"1min":2.1669999999999998,"5min":1.7150000000000001,"15min":2.1749999999999998},"min":{"1min":0.379,"5min":0.33100000000000002,"15min":0.30099999999999999},"max":{"1min":10.81,"5min":10.81,"15min":204.62700000000001},"last":0.85299999999999998},{"interface":"front","average":{"1min":2.0259999999999998,"5min":2.169,"15min":2.4140000000000001},"min":{"1min":0.40100000000000002,"5min":0.39900000000000002,"15min":0.35499999999999998},"max":{"1min":8.3490000000000002,"5min":9.2949999999999999,"15min":204.285},"last":1.258}]},{"osd":7,"last update":"Mon Oct 9 06:06:42 2023","interfaces":[{"interface":"back","average":{"1min":2.0819999999999999,"5min":2.4540000000000002,"15min":2.2690000000000001},"min":{"1min":0.42999999999999999,"5min":0.39400000000000002,"15min":0.314},"max":{"1min":6.9379999999999997,"5min":37.963999999999999,"15min":108.553},"last":1.276},{"interface":"front","average":{"1min":2.1480000000000001,"5min":2.0790000000000002,"15min":1.9470000000000001},"min":{"1min":0.44400000000000001,"5min":0.39300000000000002,"15min":0.38},"max":{"1min":11.869,"5min":37.950000000000003,"15min":108.56999999999999},"last":0.46800000000000003}]}]},{"osd":0,"up_from":129,"seq":554050781359,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":804932,"kb_used_data":365436,"kb_used_omap":29,"kb_used_meta":439458,"kb_avail":92940220,"statfs":{"total":95995035648,"available":95170785280,"internally_reserved":0,"allocated":374206464,"data_stored":371971921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30322,"internal_metadata":450005390},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:07:20 2023","interfaces":[{"interface":"back","average":{"1min":12.545,"5min":7.6920000000000002,"15min":3.2989999999999999},"min":{"1min":0.223,"5min":0.189,"15min":0.189},"max":{"1min":189.327,"5min":223.09100000000001,"15min":223.09100000000001},"last":1.3620000000000001},{"interface":"front","average":{"1min":12.852,"5min":8.3219999999999992,"15min":3.5299999999999998},"min":{"1min":0.29399999999999998,"5min":0.24399999999999999,"15min":0.24099999999999999},"max":{"1min":192.80500000000001,"5min":223.161,"15min":223.161},"last":0.98399999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:07:20 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":1.627,"15min":1.615},"min":{"1min":0.29399999999999998,"5min":0.245,"15min":0.23000000000000001},"max":{"1min":2.8279999999999998,"5min":16.152999999999999,"15min":52.405999999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.71599999999999997,"5min":1.5920000000000001,"15min":1.863},"min":{"1min":0.254,"5min":0.24399999999999999,"15min":0.219},"max":{"1min":3.3050000000000002,"5min":22.844000000000001,"15min":52.645000000000003},"last":0.52800000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:07:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":2.661,"15min":3.3119999999999998},"min":{"1min":0.28699999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":3.9630000000000001,"5min":70.662000000000006,"15min":134.185},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.98399999999999999,"5min":2.5939999999999999,"15min":3.3610000000000002},"min":{"1min":0.316,"5min":0.25,"15min":0.25},"max":{"1min":3.589,"5min":70.575000000000003,"15min":134.16800000000001},"last":1.238}]},{"osd":4,"last update":"Mon Oct 9 06:06:36 2023","interfaces":[{"interface":"back","average":{"1min":7.484,"5min":3.573,"15min":2.4729999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":118.82299999999999,"5min":118.82299999999999,"15min":118.82299999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":7.7199999999999998,"5min":3.3119999999999998,"15min":2.0339999999999998},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":118.89400000000001,"5min":118.89400000000001,"15min":118.89400000000001},"last":0.60299999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:07:20 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":2.21,"15min":2.2370000000000001},"min":{"1min":0.35599999999999998,"5min":0.35199999999999998,"15min":0.314},"max":{"1min":5.2640000000000002,"5min":70.638000000000005,"15min":100.25},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.69499999999999995,"5min":1.8200000000000001,"15min":1.915},"min":{"1min":0.44,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.04,"5min":70.518000000000001,"15min":102.40300000000001},"last":0.57199999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:07:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":2.3980000000000001,"15min":1.7769999999999999},"min":{"1min":0.38500000000000001,"5min":0.34399999999999997,"15min":0.33800000000000002},"max":{"1min":12.967000000000001,"5min":70.745999999999995,"15min":70.745999999999995},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.1799999999999999,"5min":1.9730000000000001,"15min":1.55},"min":{"1min":0.46300000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":5.1900000000000004,"5min":70.597999999999999,"15min":70.597999999999999},"last":0.629}]},{"osd":7,"last update":"Mon Oct 9 06:07:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":2.0950000000000002,"15min":2.117},"min":{"1min":0.38600000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":2.129,"5min":70.840000000000003,"15min":103.866},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.88700000000000001,"5min":1.4450000000000001,"15min":1.879},"min":{"1min":0.42699999999999999,"5min":0.41599999999999998,"15min":0.378},"max":{"1min":2.8559999999999999,"5min":7.6609999999999996,"15min":103.848},"last":0.51300000000000001}]}]},{"osd":2,"up_from":0,"seq":403726926048,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":99,"seq":425201762522,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1184820,"kb_used_data":502380,"kb_used_omap":7,"kb_used_meta":682424,"kb_avail":92560332,"statfs":{"total":95995035648,"available":94781779968,"internally_reserved":0,"allocated":514437120,"data_stored":512033171,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":698802792},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:06:31 2023","interfaces":[{"interface":"back","average":{"1min":5.0750000000000002,"5min":6.0910000000000002,"15min":3.379},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.186},"max":{"1min":42.289999999999999,"5min":131.363,"15min":131.363},"last":3.7240000000000002},{"interface":"front","average":{"1min":4.1189999999999998,"5min":5.6660000000000004,"15min":3.3959999999999999},"min":{"1min":0.23400000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":42.265000000000001,"5min":131.26300000000001,"15min":131.26300000000001},"last":2.6709999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:06:55 2023","interfaces":[{"interface":"back","average":{"1min":3.0369999999999999,"5min":4.5540000000000003,"15min":2.4740000000000002},"min":{"1min":0.27300000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":20.635999999999999,"5min":207.75299999999999,"15min":207.75299999999999},"last":2.5720000000000001},{"interface":"front","average":{"1min":2.5489999999999999,"5min":4.8140000000000001,"15min":2.9199999999999999},"min":{"1min":0.26900000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":11.336,"5min":207.12200000000001,"15min":207.12200000000001},"last":4.0449999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:06:55 2023","interfaces":[{"interface":"back","average":{"1min":2.524,"5min":2.2869999999999999,"15min":2.258},"min":{"1min":0.26300000000000001,"5min":0.223,"15min":0.223},"max":{"1min":7.4470000000000001,"5min":41.945999999999998,"15min":107.01300000000001},"last":2.7519999999999998},{"interface":"front","average":{"1min":2.1200000000000001,"5min":2.5670000000000002,"15min":2.4910000000000001},"min":{"1min":0.35199999999999998,"5min":0.28100000000000003,"15min":0.249},"max":{"1min":11.210000000000001,"5min":41.475999999999999,"15min":109.807},"last":4.6449999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:06:31 2023","interfaces":[{"interface":"back","average":{"1min":2.661,"5min":3.02,"15min":5.742},"min":{"1min":0.35799999999999998,"5min":0.254,"15min":0.254},"max":{"1min":7.7779999999999996,"5min":99.876000000000005,"15min":99.876000000000005},"last":2.7789999999999999},{"interface":"front","average":{"1min":3.1920000000000002,"5min":3.2799999999999998,"15min":5.5300000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":10.255000000000001,"5min":99.914000000000001,"15min":99.914000000000001},"last":2.1259999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:06:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":1.657,"15min":1.5489999999999999},"min":{"1min":0.34000000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.5979999999999999,"5min":27.937000000000001,"15min":27.937000000000001},"last":2.6219999999999999},{"interface":"front","average":{"1min":2.032,"5min":4.5430000000000001,"15min":2.4609999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.28199999999999997},"max":{"1min":5.9950000000000001,"5min":235.56,"15min":235.56},"last":2.863}]},{"osd":6,"last update":"Mon Oct 9 06:06:55 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":5.3949999999999996,"15min":4.4080000000000004},"min":{"1min":0.374,"5min":0.33600000000000002,"15min":0.30499999999999999},"max":{"1min":6.8550000000000004,"5min":233.26599999999999,"15min":233.26599999999999},"last":2.702},{"interface":"front","average":{"1min":1.7330000000000001,"5min":5.7919999999999998,"15min":4.5270000000000001},"min":{"1min":0.39200000000000002,"5min":0.39000000000000001,"15min":0.318},"max":{"1min":6.085,"5min":233.24100000000001,"15min":233.24100000000001},"last":2.8159999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:06:55 2023","interfaces":[{"interface":"back","average":{"1min":2.5960000000000001,"5min":4.399,"15min":4.0250000000000004},"min":{"1min":0.45300000000000001,"5min":0.39000000000000001,"15min":0.34200000000000003},"max":{"1min":8.2309999999999999,"5min":230.69399999999999,"15min":230.69399999999999},"last":2.8460000000000001},{"interface":"front","average":{"1min":2.3769999999999998,"5min":4.3419999999999996,"15min":3.9830000000000001},"min":{"1min":0.46100000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":10.301,"5min":230.67500000000001,"15min":230.67500000000001},"last":2.1640000000000001}]}]},{"osd":4,"up_from":322,"seq":1382979469385,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":544184,"kb_used_data":213616,"kb_used_omap":35,"kb_used_meta":330524,"kb_avail":93200968,"statfs":{"total":95995035648,"available":95437791232,"internally_reserved":0,"allocated":218742784,"data_stored":217516123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36552,"internal_metadata":338456888},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:06:32 2023","interfaces":[{"interface":"back","average":{"1min":5.2629999999999999,"5min":2.117,"15min":1.5489999999999999},"min":{"1min":0.33400000000000002,"5min":0.223,"15min":0.223},"max":{"1min":45.460000000000001,"5min":45.460000000000001,"15min":45.460000000000001},"last":0.78300000000000003},{"interface":"front","average":{"1min":4.3639999999999999,"5min":2.2709999999999999,"15min":1.9850000000000001},"min":{"1min":0.249,"5min":0.247,"15min":0.247},"max":{"1min":45.476999999999997,"5min":45.476999999999997,"15min":45.476999999999997},"last":0.60599999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:06:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1070000000000002,"5min":3.3980000000000001,"15min":2.3799999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":8.4280000000000008,"5min":78.891000000000005,"15min":78.891000000000005},"last":1.448},{"interface":"front","average":{"1min":2.403,"5min":3.3820000000000001,"15min":2.1459999999999999},"min":{"1min":0.27700000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":11.231999999999999,"5min":78.921000000000006,"15min":78.921000000000006},"last":0.41099999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:06:32 2023","interfaces":[{"interface":"back","average":{"1min":2.073,"5min":1.7150000000000001,"15min":1.827},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.9189999999999996,"5min":7.9189999999999996,"15min":7.9189999999999996},"last":0.44800000000000001},{"interface":"front","average":{"1min":2.1989999999999998,"5min":1.6579999999999999,"15min":2.0489999999999999},"min":{"1min":0.312,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.8710000000000004,"5min":11.381,"15min":11.381},"last":0.63200000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:06:32 2023","interfaces":[{"interface":"back","average":{"1min":2.4900000000000002,"5min":2.9689999999999999,"15min":2.3940000000000001},"min":{"1min":0.34300000000000003,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":8.1649999999999991,"5min":58.398000000000003,"15min":58.398000000000003},"last":2.2000000000000002},{"interface":"front","average":{"1min":1.9430000000000001,"5min":2.4910000000000001,"15min":2.0030000000000001},"min":{"1min":0.39500000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":8.4920000000000009,"5min":58.305999999999997,"15min":58.305999999999997},"last":0.47699999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:06:32 2023","interfaces":[{"interface":"back","average":{"1min":1.3879999999999999,"5min":1.048,"15min":0.95999999999999996},"min":{"1min":0.27600000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.5209999999999999,"5min":5.7629999999999999,"15min":5.7629999999999999},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.5780000000000001,"5min":1.3200000000000001,"15min":1.4399999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":5.5830000000000002,"5min":12.087,"15min":12.087},"last":0.81200000000000006}]},{"osd":6,"last update":"Mon Oct 9 06:06:32 2023","interfaces":[{"interface":"back","average":{"1min":2,"5min":1.9419999999999999,"15min":3.504},"min":{"1min":0.33800000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.6820000000000004,"5min":53.744999999999997,"15min":53.744999999999997},"last":0.54700000000000004},{"interface":"front","average":{"1min":2.2240000000000002,"5min":2.0830000000000002,"15min":3.7149999999999999},"min":{"1min":0.33400000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":15.522,"5min":53.798000000000002,"15min":53.798000000000002},"last":0.876}]},{"osd":7,"last update":"Mon Oct 9 06:06:32 2023","interfaces":[{"interface":"back","average":{"1min":1.839,"5min":2.3450000000000002,"15min":5.1280000000000001},"min":{"1min":0.34300000000000003,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":8.3369999999999997,"5min":79.173000000000002,"15min":79.173000000000002},"last":0.84699999999999998},{"interface":"front","average":{"1min":1.633,"5min":2.0960000000000001,"15min":4.1980000000000004},"min":{"1min":0.34899999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.4690000000000003,"5min":78.894000000000005,"15min":78.894000000000005},"last":1.1850000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757605,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":980456,"kb_used_data":390688,"kb_used_omap":0,"kb_used_meta":589760,"kb_avail":92764696,"statfs":{"total":95995035648,"available":94991048704,"internally_reserved":0,"allocated":400064512,"data_stored":398850453,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603914240},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:06:34 2023","interfaces":[{"interface":"back","average":{"1min":2.5939999999999999,"5min":2.1789999999999998,"15min":1.673},"min":{"1min":0.23599999999999999,"5min":0.23499999999999999,"15min":0.22500000000000001},"max":{"1min":19.119,"5min":69.088999999999999,"15min":69.088999999999999},"last":1.49},{"interface":"front","average":{"1min":2.5009999999999999,"5min":2.3889999999999998,"15min":1.917},"min":{"1min":0.33500000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":19.161000000000001,"5min":69.073999999999998,"15min":69.073999999999998},"last":1.5309999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:06:52 2023","interfaces":[{"interface":"back","average":{"1min":1.198,"5min":4.016,"15min":2.6749999999999998},"min":{"1min":0.27400000000000002,"5min":0.26800000000000002,"15min":0.26000000000000001},"max":{"1min":4.9009999999999998,"5min":102.642,"15min":102.642},"last":2.4380000000000002},{"interface":"front","average":{"1min":1.6879999999999999,"5min":4.3159999999999998,"15min":2.4780000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.26400000000000001},"max":{"1min":5.2290000000000001,"5min":102.45999999999999,"15min":102.45999999999999},"last":0.317}]},{"osd":2,"last update":"Mon Oct 9 06:06:19 2023","interfaces":[{"interface":"back","average":{"1min":1.726,"5min":3.7000000000000002,"15min":3.3740000000000001},"min":{"1min":0.36799999999999999,"5min":0.254,"15min":0.25},"max":{"1min":4.9770000000000003,"5min":129.21199999999999,"15min":217.06399999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.889,"5min":3.8250000000000002,"15min":3.2610000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.27600000000000002},"max":{"1min":7.968,"5min":129.042,"15min":217.018},"last":0.59399999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:06:52 2023","interfaces":[{"interface":"back","average":{"1min":2.4180000000000001,"5min":4.9340000000000002,"15min":6.8689999999999998},"min":{"1min":0.379,"5min":0.312,"15min":0.27500000000000002},"max":{"1min":8.6470000000000002,"5min":91.528999999999996,"15min":211.40199999999999},"last":1.232},{"interface":"front","average":{"1min":1.9330000000000001,"5min":4.7270000000000003,"15min":6.9530000000000003},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.5380000000000003,"5min":91.435000000000002,"15min":211.578},"last":0.371}]},{"osd":4,"last update":"Mon Oct 9 06:06:34 2023","interfaces":[{"interface":"back","average":{"1min":1.732,"5min":1.0549999999999999,"15min":0.95699999999999996},"min":{"1min":0.27400000000000002,"5min":0.249,"15min":0.249},"max":{"1min":8.1539999999999999,"5min":8.1539999999999999,"15min":8.1539999999999999},"last":0.27900000000000003},{"interface":"front","average":{"1min":1.367,"5min":1.034,"15min":1.091},"min":{"1min":0.28299999999999997,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.335,"5min":5.8879999999999999,"15min":5.8879999999999999},"last":0.42899999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:06:52 2023","interfaces":[{"interface":"back","average":{"1min":1.73,"5min":1.391,"15min":1.446},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.251},"max":{"1min":8.6809999999999992,"5min":9.0909999999999993,"15min":126.89400000000001},"last":0.38700000000000001},{"interface":"front","average":{"1min":1.4279999999999999,"5min":1.4259999999999999,"15min":1.5},"min":{"1min":0.33300000000000002,"5min":0.28000000000000003,"15min":0.26200000000000001},"max":{"1min":4.7229999999999999,"5min":9.9009999999999998,"15min":126.616},"last":0.33100000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:06:34 2023","interfaces":[{"interface":"back","average":{"1min":1.9990000000000001,"5min":1.423,"15min":2.0680000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.29999999999999999},"max":{"1min":6.274,"5min":9.9380000000000006,"15min":133.27099999999999},"last":0.73099999999999998},{"interface":"front","average":{"1min":2.2240000000000002,"5min":1.3700000000000001,"15min":1.8520000000000001},"min":{"1min":0.35999999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":9.9529999999999994,"5min":9.9529999999999994,"15min":135.09200000000001},"last":0.34499999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1474,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4889,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":369950720,"data_stored":369950720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":660733952,"data_stored":660733952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":583073792,"data_stored":583073792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":218169344,"data_stored":218169344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":457703424,"data_stored":457703424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":226099200,"data_stored":226099200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":281870336,"data_stored":281870336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:07:22.838 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T06:07:22.838 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 41 2023-10-09T06:07:22.862 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:22.975 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:22.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:22.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:23.081 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:23.163 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:23.240 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:23.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:23.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:23.345 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:23.404 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:23.674 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:23.689 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 10 16 24442 24426 152.639 125.375 0.0114584 0.00654633 2023-10-09T06:07:23.764 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:23.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:23.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:23.881 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:23.885 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:23.916 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:23.956 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:24.130 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:24.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:24.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:24.245 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:24.289 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:24.383 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:24.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:24.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:24.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:24 smithi064 ceph-mon[212750]: pgmap v1054: 119 pgs: 119 active+clean; 1.1 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T06:07:24.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:24 smithi064 ceph-mon[220044]: pgmap v1054: 119 pgs: 119 active+clean; 1.1 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T06:07:24.487 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:24.526 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:24.596 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:24.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:24.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:24.604 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:24 smithi156 ceph-mon[195101]: pgmap v1054: 119 pgs: 119 active+clean; 1.1 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T06:07:24.690 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 11 16 26352 26336 149.61 119.375 0.0060142 0.00667826 2023-10-09T06:07:24.704 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:24.740 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:24.810 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:24.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:24.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:24.917 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:24.959 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:25.047 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:25.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:25.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:25.154 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:25.207 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:25.262 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:25.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:25.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:25.304 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:07:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:07:25] "GET /metrics HTTP/1.1" 200 36342 "" "Prometheus/2.43.0" 2023-10-09T06:07:25.370 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:25.451 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:25.502 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:07:25.690 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 12 16 28080 28064 146.142 108 0.00656252 0.0068373 2023-10-09T06:07:25.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:25 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/370925190' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:07:25.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:25 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/370925190' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:07:25.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:25 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/370925190' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:07:26.690 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 13 16 30150 30134 144.851 129.375 0.00284948 0.00689914 2023-10-09T06:07:26.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:26 smithi064 ceph-mon[212750]: pgmap v1055: 119 pgs: 119 active+clean; 1.3 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-10-09T06:07:26.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:26 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:26.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:26 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/370925190' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:07:26.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:26 smithi064 ceph-mon[212750]: osdmap e430: 8 total, 8 up, 7 in 2023-10-09T06:07:26.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:26 smithi064 ceph-mon[212750]: failed to encode map e430 with expected crc 2023-10-09T06:07:26.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:26 smithi064 ceph-mon[212750]: failed to encode map e430 with expected crc 2023-10-09T06:07:26.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:26 smithi064 ceph-mon[212750]: failed to encode map e430 with expected crc 2023-10-09T06:07:26.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:26 smithi064 ceph-mon[220044]: pgmap v1055: 119 pgs: 119 active+clean; 1.3 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-10-09T06:07:26.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:26 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:26.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:26 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/370925190' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:07:26.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:26 smithi064 ceph-mon[220044]: osdmap e430: 8 total, 8 up, 7 in 2023-10-09T06:07:26.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:26 smithi064 ceph-mon[220044]: failed to encode map e430 with expected crc 2023-10-09T06:07:26.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:26 smithi064 ceph-mon[220044]: failed to encode map e430 with expected crc 2023-10-09T06:07:26.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:26 smithi064 ceph-mon[220044]: failed to encode map e430 with expected crc 2023-10-09T06:07:26.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:26 smithi156 ceph-mon[195101]: pgmap v1055: 119 pgs: 119 active+clean; 1.3 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-10-09T06:07:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:26 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:26 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/370925190' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:07:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:26 smithi156 ceph-mon[195101]: osdmap e430: 8 total, 8 up, 7 in 2023-10-09T06:07:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:26 smithi156 ceph-mon[195101]: failed to encode map e430 with expected crc 2023-10-09T06:07:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:26 smithi156 ceph-mon[195101]: failed to encode map e430 with expected crc 2023-10-09T06:07:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:26 smithi156 ceph-mon[195101]: failed to encode map e430 with expected crc 2023-10-09T06:07:27.170 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:27.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:27.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:27.279 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:27.343 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:27.596 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:27.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:27.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:27.690 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 14 16 31914 31898 142.379 110.25 0.00224608 0.00699599 2023-10-09T06:07:27.700 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:27.759 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:28.251 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:28.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:28.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:28.357 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:28.396 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:28.473 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:28.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:28.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:28.580 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:28.647 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:28 smithi064 ceph-mon[220044]: pgmap v1057: 119 pgs: 119 active+clean; 1.7 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-10-09T06:07:28.649 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:28 smithi064 ceph-mon[212750]: pgmap v1057: 119 pgs: 119 active+clean; 1.7 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-10-09T06:07:28.652 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:28.690 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 15 16 33753 33737 140.549 114.938 0.00497859 0.00710989 2023-10-09T06:07:28.822 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:28.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:28.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:28.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:28 smithi156 ceph-mon[195101]: pgmap v1057: 119 pgs: 119 active+clean; 1.7 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-10-09T06:07:28.931 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:28.968 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:29.019 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:07:29.030 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:29.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:29.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:29.058 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:07:29.110 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:07:29.135 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:29.149 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:07:29.199 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:07:29.211 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:07:29.211 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T06:07:29.211 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T06:07:29.213 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T06:07:29.213 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:07:29.229 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:29.255 INFO:teuthology.orchestra.run.smithi064.stderr:Error EINVAL: specified pg_num 41 < pg_num_max 32 2023-10-09T06:07:29.256 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:07:29.290 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:07:29.330 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:07:29.339 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:07:29.339 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:07:29.339 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:07:29.339 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:07:29.339 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:07:29.339 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:07:29.339 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:07:29.339 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:07:29.690 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 16 16 35294 35278 137.784 96.3125 0.007282 0.00725232 2023-10-09T06:07:29.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:29.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:29 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3304214005' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:29.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:29 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1574869746' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-10-09T06:07:29.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:29.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:29 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3304214005' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:29.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:29 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1574869746' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-10-09T06:07:29.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:29.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:29 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3304214005' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:29.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:29 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1574869746' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-10-09T06:07:30.138 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:07:30.690 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 17 16 36854 36838 135.414 97.5 0.00350442 0.00735162 2023-10-09T06:07:30.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:30 smithi156 ceph-mon[195101]: pgmap v1058: 119 pgs: 119 active+clean; 1.7 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 142 MiB/s wr, 2.27k op/s 2023-10-09T06:07:30.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:30 smithi064 ceph-mon[220044]: pgmap v1058: 119 pgs: 119 active+clean; 1.7 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 142 MiB/s wr, 2.27k op/s 2023-10-09T06:07:30.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:30 smithi064 ceph-mon[212750]: pgmap v1058: 119 pgs: 119 active+clean; 1.7 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 142 MiB/s wr, 2.27k op/s 2023-10-09T06:07:31.652 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T06:07:31.694 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 18 16 38364 38348 133.108 94.375 0.00569203 0.00750724 2023-10-09T06:07:31.730 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:31.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:31.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:31.741 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:07:31.838 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:31.877 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:31.962 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:31.965 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:31.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:31.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:32.068 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:32.114 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:32.198 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:32.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:32.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:32.307 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:32.385 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:32.537 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:32.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:32 smithi064 ceph-mon[220044]: pgmap v1059: 119 pgs: 119 active+clean; 2.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-10-09T06:07:32.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:32.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:32.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:32 smithi064 ceph-mon[212750]: pgmap v1059: 119 pgs: 119 active+clean; 2.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-10-09T06:07:32.642 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:32.694 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 19 16 39939 39923 131.284 98.4375 0.0116904 0.00761127 2023-10-09T06:07:32.708 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:32.815 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:32.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:32.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:32 smithi156 ceph-mon[195101]: pgmap v1059: 119 pgs: 119 active+clean; 2.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-10-09T06:07:32.922 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:32.970 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:33.012 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:33.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:33.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:33.119 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:33.147 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:33.244 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:33.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:33.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:33.350 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:33.406 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:33.540 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:33.566 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:33.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:33.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:33.675 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:33.694 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-10-09T06:07:33.695154+0000 min lat: 0.00120925 max lat: 0.254013 avg lat: 0.00774556 2023-10-09T06:07:33.694 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:07:33.694 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 20 16 41316 41300 129.019 86.0625 0.00510793 0.00774556 2023-10-09T06:07:33.748 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:33.821 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:07:33.852 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:33.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:33.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:33.962 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:34.086 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:34.226 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:34.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:34.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:34.331 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:34.392 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:34.475 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:34.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:34.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:34.587 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:34.641 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:34.695 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 21 16 42741 42725 127.117 89.0625 0.00316851 0.00785655 2023-10-09T06:07:34.697 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:34.700 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:34 smithi156 ceph-mon[195101]: pgmap v1060: 119 pgs: 119 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T06:07:34.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:34.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:34.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:34 smithi064 ceph-mon[212750]: pgmap v1060: 119 pgs: 119 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T06:07:34.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:34 smithi064 ceph-mon[220044]: pgmap v1060: 119 pgs: 119 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T06:07:34.803 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:34.850 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:34.949 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:34.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:34.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:35.056 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:35.114 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:35.226 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:35.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:35.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:35.334 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:35.383 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:35.456 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:35.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:35.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:35.472 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:35.472 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:07:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:07:35] "GET /metrics HTTP/1.1" 200 36403 "" "Prometheus/2.43.0" 2023-10-09T06:07:35.548 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:07:35.562 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:35.607 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:35.685 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:35.694 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 22 16 44147 44131 125.333 87.875 0.0108234 0.00797241 2023-10-09T06:07:35.696 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:35 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2085408380' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:35.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:35.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:35.771 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:35 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2085408380' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:35.774 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:35 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2085408380' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:35.800 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:35.861 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:36.632 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:36.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:36.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:36.652 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:07:36.652 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T06:07:36.652 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T06:07:36.652 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:07:36.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.652 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:134378 8.a 8:5cb190c4:::benchmark_data_smithi064_457003_object29860:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2023-10-09T06:07:36.653 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:10.422104+0000", 2023-10-09T06:07:36.653 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 146.225966513, 2023-10-09T06:07:36.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1862699270000001, 2023-10-09T06:07:36.653 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:07:36.653 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:07:36.653 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:07:36.653 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:07:36.653 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:36.653 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 134378 2023-10-09T06:07:36.654 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.654 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:07:36.654 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.654 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:07:36.654 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.422104+0000", 2023-10-09T06:07:36.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.654 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.654 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.654 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:07:36.655 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.422104+0000", 2023-10-09T06:07:36.655 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.655 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.655 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.655 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:07:36.655 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.422103+0000", 2023-10-09T06:07:36.655 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:36.655 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.655 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.656 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:07:36.656 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.422109+0000", 2023-10-09T06:07:36.656 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1550000000000004e-06 2023-10-09T06:07:36.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.656 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:07:36.656 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.422121+0000", 2023-10-09T06:07:36.656 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2243e-05 2023-10-09T06:07:36.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:07:36.657 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.422129+0000", 2023-10-09T06:07:36.657 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5939999999999996e-06 2023-10-09T06:07:36.657 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.657 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.657 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.422179+0000", 2023-10-09T06:07:36.657 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0278000000000002e-05 2023-10-09T06:07:36.657 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.658 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:07:36.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.422279+0000", 2023-10-09T06:07:36.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9932000000000005e-05 2023-10-09T06:07:36.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.658 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:07:36.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.422392+0000", 2023-10-09T06:07:36.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000113328 2023-10-09T06:07:36.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:07:36.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.425990+0000", 2023-10-09T06:07:36.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035977890000000001 2023-10-09T06:07:36.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.426129+0000", 2023-10-09T06:07:36.660 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013879100000000001 2023-10-09T06:07:36.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.660 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.660 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.578547+0000", 2023-10-09T06:07:36.660 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15241822299999999 2023-10-09T06:07:36.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.660 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:07:36.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.578560+0000", 2023-10-09T06:07:36.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.279e-05 2023-10-09T06:07:36.661 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.661 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.661 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:11.444879+0000", 2023-10-09T06:07:36.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.86631858500000003 2023-10-09T06:07:36.661 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.661 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.662 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:07:36.662 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:11.444894+0000", 2023-10-09T06:07:36.662 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5071e-05 2023-10-09T06:07:36.662 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.662 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.662 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.662 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.608296+0000", 2023-10-09T06:07:36.662 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.163402286 2023-10-09T06:07:36.662 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.663 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.663 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:07:36.663 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.608374+0000", 2023-10-09T06:07:36.663 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8324000000000004e-05 2023-10-09T06:07:36.663 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.663 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:07:36.663 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.663 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.663 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.664 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:134382 8.1 8:8d031598:::benchmark_data_smithi064_457003_object29864:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2023-10-09T06:07:36.664 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:10.426471+0000", 2023-10-09T06:07:36.664 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 146.221600276, 2023-10-09T06:07:36.664 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1983307280000002, 2023-10-09T06:07:36.664 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:07:36.664 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:07:36.664 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:07:36.664 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:07:36.664 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:36.664 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 134382 2023-10-09T06:07:36.664 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.665 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:07:36.665 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.665 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:07:36.665 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.426471+0000", 2023-10-09T06:07:36.665 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.665 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.665 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.665 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:07:36.665 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.426471+0000", 2023-10-09T06:07:36.665 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.665 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.665 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.666 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:07:36.666 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.426469+0000", 2023-10-09T06:07:36.666 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:36.666 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.666 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.666 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:07:36.666 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.426474+0000", 2023-10-09T06:07:36.666 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.13e-06 2023-10-09T06:07:36.666 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.666 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.667 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:07:36.667 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.426486+0000", 2023-10-09T06:07:36.667 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1987999999999999e-05 2023-10-09T06:07:36.667 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.667 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.667 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:07:36.667 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.426494+0000", 2023-10-09T06:07:36.667 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3930000000000002e-06 2023-10-09T06:07:36.667 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.667 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.668 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.668 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.426538+0000", 2023-10-09T06:07:36.668 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4017000000000002e-05 2023-10-09T06:07:36.668 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.668 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.668 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:07:36.668 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.426654+0000", 2023-10-09T06:07:36.668 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116385 2023-10-09T06:07:36.668 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.668 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.668 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:07:36.668 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.426727+0000", 2023-10-09T06:07:36.669 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3096000000000006e-05 2023-10-09T06:07:36.669 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.669 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.669 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.669 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.554596+0000", 2023-10-09T06:07:36.669 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.127868809 2023-10-09T06:07:36.669 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.669 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.669 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:07:36.669 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560066+0000", 2023-10-09T06:07:36.669 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0054699459999999998 2023-10-09T06:07:36.670 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.670 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.670 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.670 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.581043+0000", 2023-10-09T06:07:36.670 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020976787 2023-10-09T06:07:36.670 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.670 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.670 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:07:36.670 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.581055+0000", 2023-10-09T06:07:36.670 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1701e-05 2023-10-09T06:07:36.670 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.671 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.671 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.671 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:11.443633+0000", 2023-10-09T06:07:36.671 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.862578501 2023-10-09T06:07:36.671 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.671 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.671 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:07:36.671 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:11.443648+0000", 2023-10-09T06:07:36.671 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5058999999999999e-05 2023-10-09T06:07:36.671 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.672 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.672 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.672 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.624713+0000", 2023-10-09T06:07:36.672 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.181064774 2023-10-09T06:07:36.672 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.672 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.672 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:07:36.672 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.624801+0000", 2023-10-09T06:07:36.672 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8348000000000002e-05 2023-10-09T06:07:36.672 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.672 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:07:36.673 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.673 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.673 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.673 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:134398 8.1 8:824844e4:::benchmark_data_smithi064_457003_object29880:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-10-09T06:07:36.673 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:10.582480+0000", 2023-10-09T06:07:36.673 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 146.06559117899999, 2023-10-09T06:07:36.673 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1200893710000002, 2023-10-09T06:07:36.673 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:07:36.673 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:36.673 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:07:36.673 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:07:36.674 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:36.674 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 134398 2023-10-09T06:07:36.674 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.674 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:07:36.674 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.674 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:07:36.674 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.582480+0000", 2023-10-09T06:07:36.674 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.674 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.674 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.674 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:07:36.675 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.582480+0000", 2023-10-09T06:07:36.675 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.675 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.675 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.675 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:07:36.675 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.582479+0000", 2023-10-09T06:07:36.675 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:07:36.675 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.675 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.675 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:07:36.675 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.582484+0000", 2023-10-09T06:07:36.675 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4920000000000001e-06 2023-10-09T06:07:36.676 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.676 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.676 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:07:36.676 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.582499+0000", 2023-10-09T06:07:36.676 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5311e-05 2023-10-09T06:07:36.676 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.676 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.676 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:07:36.676 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.582508+0000", 2023-10-09T06:07:36.676 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0560000000000005e-06 2023-10-09T06:07:36.677 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.677 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.677 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.677 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.582557+0000", 2023-10-09T06:07:36.677 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9972000000000002e-05 2023-10-09T06:07:36.677 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.677 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.677 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:07:36.677 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.582570+0000", 2023-10-09T06:07:36.677 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2850999999999999e-05 2023-10-09T06:07:36.677 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.678 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.678 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.678 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:11.443677+0000", 2023-10-09T06:07:36.678 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.86110640199999999 2023-10-09T06:07:36.678 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.678 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.678 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:07:36.678 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:11.443685+0000", 2023-10-09T06:07:36.678 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4780000000000008e-06 2023-10-09T06:07:36.678 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.678 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.678 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.679 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.625063+0000", 2023-10-09T06:07:36.679 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.181377425 2023-10-09T06:07:36.679 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.679 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.679 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:07:36.679 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.625178+0000", 2023-10-09T06:07:36.679 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000115757 2023-10-09T06:07:36.679 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.679 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.679 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2,5", 2023-10-09T06:07:36.680 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.625758+0000", 2023-10-09T06:07:36.680 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00058000499999999997 2023-10-09T06:07:36.680 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.680 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.680 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.680 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.641712+0000", 2023-10-09T06:07:36.680 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015953261 2023-10-09T06:07:36.680 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.680 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.681 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:07:36.681 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.642831+0000", 2023-10-09T06:07:36.681 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011196699999999999 2023-10-09T06:07:36.681 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.681 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.681 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.681 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.642873+0000", 2023-10-09T06:07:36.681 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1548000000000003e-05 2023-10-09T06:07:36.681 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.681 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.681 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.682 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.702406+0000", 2023-10-09T06:07:36.682 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.059533448000000003 2023-10-09T06:07:36.682 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.682 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.682 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:07:36.682 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.702459+0000", 2023-10-09T06:07:36.682 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3112999999999998e-05 2023-10-09T06:07:36.682 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.682 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.682 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:07:36.682 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.702569+0000", 2023-10-09T06:07:36.683 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010953199999999999 2023-10-09T06:07:36.683 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.683 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:07:36.683 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.683 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.683 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.683 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:134410 8.1 8:8d635155:::benchmark_data_smithi064_457003_object29892:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-10-09T06:07:36.683 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:10.604120+0000", 2023-10-09T06:07:36.683 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 146.043950978, 2023-10-09T06:07:36.683 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.098642146, 2023-10-09T06:07:36.683 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:07:36.683 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:36.684 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:07:36.684 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:07:36.684 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:36.684 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 134410 2023-10-09T06:07:36.684 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.684 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:07:36.684 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.684 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:07:36.684 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.604120+0000", 2023-10-09T06:07:36.684 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.684 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.685 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.685 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:07:36.685 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.604120+0000", 2023-10-09T06:07:36.685 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.685 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.685 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.685 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:07:36.685 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.604118+0000", 2023-10-09T06:07:36.685 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:36.685 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.685 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.686 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:07:36.686 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.604124+0000", 2023-10-09T06:07:36.686 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2600000000000002e-06 2023-10-09T06:07:36.686 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.686 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.686 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:07:36.686 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.604140+0000", 2023-10-09T06:07:36.686 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5883999999999999e-05 2023-10-09T06:07:36.686 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.686 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.686 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:07:36.687 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.604147+0000", 2023-10-09T06:07:36.687 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4559999999999997e-06 2023-10-09T06:07:36.687 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.687 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.687 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.687 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.604192+0000", 2023-10-09T06:07:36.687 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5639999999999997e-05 2023-10-09T06:07:36.687 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.687 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.687 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:07:36.687 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.604199+0000", 2023-10-09T06:07:36.688 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6719999999999998e-06 2023-10-09T06:07:36.688 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.688 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.688 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.688 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:11.443713+0000", 2023-10-09T06:07:36.688 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.83951366599999999 2023-10-09T06:07:36.688 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.688 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.688 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:07:36.689 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:11.443721+0000", 2023-10-09T06:07:36.689 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7239999999999998e-06 2023-10-09T06:07:36.689 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.689 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.689 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.689 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.633391+0000", 2023-10-09T06:07:36.689 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1896695770000001 2023-10-09T06:07:36.689 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.689 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.689 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:07:36.690 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.633512+0000", 2023-10-09T06:07:36.690 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012129399999999999 2023-10-09T06:07:36.690 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.690 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.690 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2,5", 2023-10-09T06:07:36.690 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.633669+0000", 2023-10-09T06:07:36.690 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015691300000000001 2023-10-09T06:07:36.690 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.691 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.691 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.691 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.641754+0000", 2023-10-09T06:07:36.691 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0080843709999999999 2023-10-09T06:07:36.691 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.691 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.691 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:07:36.691 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.642881+0000", 2023-10-09T06:07:36.691 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001127213 2023-10-09T06:07:36.692 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.692 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.692 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.692 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.645184+0000", 2023-10-09T06:07:36.692 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0023028290000000002 2023-10-09T06:07:36.692 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.692 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.692 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.692 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.702606+0000", 2023-10-09T06:07:36.692 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.057422478999999998 2023-10-09T06:07:36.693 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.693 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.693 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:07:36.693 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.702661+0000", 2023-10-09T06:07:36.693 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5365000000000003e-05 2023-10-09T06:07:36.693 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.693 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.693 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:07:36.693 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.702762+0000", 2023-10-09T06:07:36.693 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000100512 2023-10-09T06:07:36.694 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.694 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:07:36.694 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.694 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.694 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.694 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:162053 8.1 8:8f3c67d0:::benchmark_data_smithi064_457003_object57535:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:07:36.694 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:52.493183+0000", 2023-10-09T06:07:36.694 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 104.154887519, 2023-10-09T06:07:36.694 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.969793028, 2023-10-09T06:07:36.694 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:07:36.694 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:36.695 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:07:36.695 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:07:36.695 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:36.695 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 162053 2023-10-09T06:07:36.695 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.695 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:07:36.695 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.695 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:07:36.695 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.493183+0000", 2023-10-09T06:07:36.695 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.695 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.696 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.696 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:07:36.696 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.493183+0000", 2023-10-09T06:07:36.696 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.696 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.696 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.696 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:07:36.696 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.493182+0000", 2023-10-09T06:07:36.696 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:36.696 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.696 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.697 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:07:36.697 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.493189+0000", 2023-10-09T06:07:36.697 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1770000000000001e-06 2023-10-09T06:07:36.697 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.697 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.697 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:07:36.697 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.493207+0000", 2023-10-09T06:07:36.697 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8340000000000001e-05 2023-10-09T06:07:36.697 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.697 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.697 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:07:36.698 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.493217+0000", 2023-10-09T06:07:36.698 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.363e-06 2023-10-09T06:07:36.698 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.698 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.698 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.698 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.493268+0000", 2023-10-09T06:07:36.698 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1808999999999998e-05 2023-10-09T06:07:36.698 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.698 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.699 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:07:36.699 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.493278+0000", 2023-10-09T06:07:36.699 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.363e-06 2023-10-09T06:07:36.699 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.699 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.699 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.699 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.455855+0000", 2023-10-09T06:07:36.699 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96257762899999999 2023-10-09T06:07:36.699 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.700 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.700 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:07:36.700 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.456014+0000", 2023-10-09T06:07:36.700 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015897599999999999 2023-10-09T06:07:36.700 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.700 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.700 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:07:36.700 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.456438+0000", 2023-10-09T06:07:36.700 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00042368400000000002 2023-10-09T06:07:36.701 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.701 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.701 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.701 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.459389+0000", 2023-10-09T06:07:36.701 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029509670000000001 2023-10-09T06:07:36.701 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.701 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.701 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:07:36.701 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.461119+0000", 2023-10-09T06:07:36.701 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0017302859999999999 2023-10-09T06:07:36.702 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.702 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.702 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.702 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.462880+0000", 2023-10-09T06:07:36.702 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001760899 2023-10-09T06:07:36.702 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.702 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.702 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:07:36.702 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.462935+0000", 2023-10-09T06:07:36.702 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4991999999999998e-05 2023-10-09T06:07:36.702 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.703 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.703 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:07:36.703 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.462976+0000", 2023-10-09T06:07:36.703 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1109000000000002e-05 2023-10-09T06:07:36.703 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.703 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:07:36.703 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.703 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.703 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.703 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:162062 8.1 8:8e8d146e:::benchmark_data_smithi064_457003_object57544:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:07:36.704 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:52.493230+0000", 2023-10-09T06:07:36.704 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 104.154840561, 2023-10-09T06:07:36.704 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96999237299999996, 2023-10-09T06:07:36.704 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:07:36.704 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:36.704 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:07:36.704 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:07:36.704 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:36.704 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 162062 2023-10-09T06:07:36.704 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.705 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:07:36.705 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.705 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:07:36.705 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.493230+0000", 2023-10-09T06:07:36.705 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.705 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.705 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.705 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:07:36.705 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.493230+0000", 2023-10-09T06:07:36.705 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.706 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.706 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.706 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:07:36.706 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.493230+0000", 2023-10-09T06:07:36.706 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:07:36.706 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.706 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.706 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:07:36.706 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.493234+0000", 2023-10-09T06:07:36.706 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9079999999999997e-06 2023-10-09T06:07:36.707 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.707 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.707 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:07:36.707 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.493241+0000", 2023-10-09T06:07:36.707 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3409999999999997e-06 2023-10-09T06:07:36.707 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.707 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.707 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:07:36.707 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.493247+0000", 2023-10-09T06:07:36.707 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3509999999999998e-06 2023-10-09T06:07:36.707 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.707 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.708 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.708 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.493318+0000", 2023-10-09T06:07:36.708 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1087999999999994e-05 2023-10-09T06:07:36.708 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.708 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.708 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:07:36.708 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.493327+0000", 2023-10-09T06:07:36.708 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9099999999999994e-06 2023-10-09T06:07:36.708 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.708 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.708 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.708 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.459415+0000", 2023-10-09T06:07:36.709 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96608790300000003 2023-10-09T06:07:36.709 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.709 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.709 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:07:36.709 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.459513+0000", 2023-10-09T06:07:36.709 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7750000000000004e-05 2023-10-09T06:07:36.709 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.709 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.709 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:07:36.709 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.459580+0000", 2023-10-09T06:07:36.710 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7197000000000001e-05 2023-10-09T06:07:36.710 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.710 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.710 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.710 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.463088+0000", 2023-10-09T06:07:36.710 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035077680000000001 2023-10-09T06:07:36.710 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.710 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.710 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.710 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.463136+0000", 2023-10-09T06:07:36.711 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8566999999999998e-05 2023-10-09T06:07:36.711 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.711 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.711 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:07:36.711 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.463148+0000", 2023-10-09T06:07:36.711 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1221e-05 2023-10-09T06:07:36.711 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.711 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.711 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:07:36.711 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.463189+0000", 2023-10-09T06:07:36.711 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0939000000000002e-05 2023-10-09T06:07:36.712 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.712 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.712 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:07:36.712 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.463223+0000", 2023-10-09T06:07:36.712 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3995999999999998e-05 2023-10-09T06:07:36.712 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.712 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:07:36.712 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.712 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.712 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.712 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:162070 8.1 8:8284fec7:::benchmark_data_smithi064_457003_object57552:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:07:36.713 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:52.505747+0000", 2023-10-09T06:07:36.713 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 104.142323976, 2023-10-09T06:07:36.713 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95761205500000002, 2023-10-09T06:07:36.713 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:07:36.713 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:36.713 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:07:36.713 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:07:36.713 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:36.713 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 162070 2023-10-09T06:07:36.713 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.713 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:07:36.713 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.714 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:07:36.714 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.505747+0000", 2023-10-09T06:07:36.714 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.714 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.714 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.714 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:07:36.714 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.505747+0000", 2023-10-09T06:07:36.714 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.714 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.714 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.714 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:07:36.715 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.505745+0000", 2023-10-09T06:07:36.715 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:36.715 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.715 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.715 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:07:36.715 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.505752+0000", 2023-10-09T06:07:36.715 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8059999999999997e-06 2023-10-09T06:07:36.715 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.715 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.715 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:07:36.715 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.505769+0000", 2023-10-09T06:07:36.715 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7124000000000001e-05 2023-10-09T06:07:36.716 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.716 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.716 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:07:36.716 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.505778+0000", 2023-10-09T06:07:36.716 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7790000000000001e-06 2023-10-09T06:07:36.716 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.716 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.716 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.716 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.505828+0000", 2023-10-09T06:07:36.716 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0229000000000003e-05 2023-10-09T06:07:36.716 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.717 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.717 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:07:36.717 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.505838+0000", 2023-10-09T06:07:36.717 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7489999999999993e-06 2023-10-09T06:07:36.717 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.717 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.717 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.717 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.461167+0000", 2023-10-09T06:07:36.717 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95532925800000001 2023-10-09T06:07:36.717 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.718 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.718 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:07:36.718 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.461291+0000", 2023-10-09T06:07:36.718 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012392999999999999 2023-10-09T06:07:36.718 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.718 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.718 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:07:36.718 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.461419+0000", 2023-10-09T06:07:36.718 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012794 2023-10-09T06:07:36.718 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.719 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.719 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:07:36.719 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.463231+0000", 2023-10-09T06:07:36.719 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001812094 2023-10-09T06:07:36.719 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.719 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.719 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.719 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.463260+0000", 2023-10-09T06:07:36.719 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9264999999999998e-05 2023-10-09T06:07:36.719 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.720 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.720 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.720 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.463290+0000", 2023-10-09T06:07:36.720 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9156999999999999e-05 2023-10-09T06:07:36.720 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.720 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.720 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:07:36.720 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.463329+0000", 2023-10-09T06:07:36.720 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9768000000000003e-05 2023-10-09T06:07:36.720 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.720 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.721 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:07:36.721 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.463359+0000", 2023-10-09T06:07:36.721 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9476e-05 2023-10-09T06:07:36.721 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.721 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:07:36.721 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.721 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.721 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.721 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:162076 8.1 8:8f73a7bc:::benchmark_data_smithi064_457003_object57558:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:07:36.721 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:52.513414+0000", 2023-10-09T06:07:36.721 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 104.134656709, 2023-10-09T06:07:36.721 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95656515200000003, 2023-10-09T06:07:36.722 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:07:36.722 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:36.722 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:07:36.722 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:07:36.722 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:36.722 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 162076 2023-10-09T06:07:36.722 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.722 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:07:36.722 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.722 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:07:36.723 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.513414+0000", 2023-10-09T06:07:36.723 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.723 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.723 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.723 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:07:36.723 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.513414+0000", 2023-10-09T06:07:36.723 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.723 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.723 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.723 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:07:36.723 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.513412+0000", 2023-10-09T06:07:36.723 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:36.724 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.724 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.724 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:07:36.724 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.513421+0000", 2023-10-09T06:07:36.724 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2930000000000001e-06 2023-10-09T06:07:36.724 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.724 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.724 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:07:36.724 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.513436+0000", 2023-10-09T06:07:36.724 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5336e-05 2023-10-09T06:07:36.724 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.724 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.725 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:07:36.725 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.513445+0000", 2023-10-09T06:07:36.725 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2289999999999992e-06 2023-10-09T06:07:36.725 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.725 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.725 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.725 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.513541+0000", 2023-10-09T06:07:36.725 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5920000000000003e-05 2023-10-09T06:07:36.725 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.725 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.725 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:07:36.725 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.513552+0000", 2023-10-09T06:07:36.726 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0917000000000001e-05 2023-10-09T06:07:36.726 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.726 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.726 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.726 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.463393+0000", 2023-10-09T06:07:36.726 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.94984060599999998 2023-10-09T06:07:36.726 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.726 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.726 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:07:36.726 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.463471+0000", 2023-10-09T06:07:36.727 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8708999999999995e-05 2023-10-09T06:07:36.727 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.727 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.727 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:07:36.727 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.463537+0000", 2023-10-09T06:07:36.727 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6043000000000004e-05 2023-10-09T06:07:36.727 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.727 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.727 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.727 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.469851+0000", 2023-10-09T06:07:36.727 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063139219999999996 2023-10-09T06:07:36.728 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.728 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.728 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:07:36.728 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.469860+0000", 2023-10-09T06:07:36.728 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9639999999999992e-06 2023-10-09T06:07:36.728 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.728 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.728 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.728 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.469904+0000", 2023-10-09T06:07:36.728 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3383000000000001e-05 2023-10-09T06:07:36.728 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.728 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.729 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:07:36.729 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.469941+0000", 2023-10-09T06:07:36.729 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6865e-05 2023-10-09T06:07:36.729 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.729 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.729 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:07:36.729 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.469979+0000", 2023-10-09T06:07:36.729 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8698000000000001e-05 2023-10-09T06:07:36.729 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.729 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:07:36.729 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.729 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.730 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.730 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:162100 8.1 8:801c8629:::benchmark_data_smithi064_457003_object57582:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:07:36.730 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:52.547508+0000", 2023-10-09T06:07:36.730 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 104.100562521, 2023-10-09T06:07:36.730 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.92733845999999998, 2023-10-09T06:07:36.730 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:07:36.730 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:36.730 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:07:36.730 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:07:36.730 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:36.730 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 162100 2023-10-09T06:07:36.730 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.731 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:07:36.731 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.731 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:07:36.731 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.547508+0000", 2023-10-09T06:07:36.731 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.731 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.731 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.731 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:07:36.731 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.547508+0000", 2023-10-09T06:07:36.731 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.731 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.731 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.732 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:07:36.732 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.547508+0000", 2023-10-09T06:07:36.732 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:07:36.732 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.732 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.732 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:07:36.732 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.547511+0000", 2023-10-09T06:07:36.732 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.304e-06 2023-10-09T06:07:36.732 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.732 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.732 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:07:36.732 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.547518+0000", 2023-10-09T06:07:36.733 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3200000000000002e-06 2023-10-09T06:07:36.733 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.733 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.733 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:07:36.733 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.547524+0000", 2023-10-09T06:07:36.733 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0109999999999997e-06 2023-10-09T06:07:36.733 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.733 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.733 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.733 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.547572+0000", 2023-10-09T06:07:36.733 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7444000000000002e-05 2023-10-09T06:07:36.733 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.733 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.734 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:07:36.734 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.547580+0000", 2023-10-09T06:07:36.734 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6479999999999992e-06 2023-10-09T06:07:36.734 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.734 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.734 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.734 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.465565+0000", 2023-10-09T06:07:36.734 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.91798429800000003 2023-10-09T06:07:36.734 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.734 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.734 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:07:36.735 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.465686+0000", 2023-10-09T06:07:36.735 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000121407 2023-10-09T06:07:36.735 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.735 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.735 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:07:36.735 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.465807+0000", 2023-10-09T06:07:36.735 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000121098 2023-10-09T06:07:36.735 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.735 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.735 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:07:36.735 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.469865+0000", 2023-10-09T06:07:36.735 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.004057962 2023-10-09T06:07:36.736 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.736 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.736 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.736 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.474715+0000", 2023-10-09T06:07:36.736 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048495819999999999 2023-10-09T06:07:36.736 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.736 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.736 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.736 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.474786+0000", 2023-10-09T06:07:36.736 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1254000000000006e-05 2023-10-09T06:07:36.736 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.736 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.737 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:07:36.737 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.474817+0000", 2023-10-09T06:07:36.737 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0738999999999998e-05 2023-10-09T06:07:36.737 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.737 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.737 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:07:36.737 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.474847+0000", 2023-10-09T06:07:36.737 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9909000000000001e-05 2023-10-09T06:07:36.737 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.737 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:07:36.737 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.737 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.737 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.738 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:162110 8.1 8:8e921651:::benchmark_data_smithi064_457003_object57592:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:07:36.738 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:52.573785+0000", 2023-10-09T06:07:36.738 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 104.07428530200001, 2023-10-09T06:07:36.738 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.905925435, 2023-10-09T06:07:36.738 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:07:36.738 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:36.738 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:07:36.738 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:07:36.738 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:36.738 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 162110 2023-10-09T06:07:36.738 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.738 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:07:36.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.739 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:07:36.739 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.573785+0000", 2023-10-09T06:07:36.739 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.739 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.739 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:07:36.739 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.573785+0000", 2023-10-09T06:07:36.739 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.739 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.740 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:07:36.740 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.573784+0000", 2023-10-09T06:07:36.740 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:07:36.740 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.740 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.740 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:07:36.740 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.573788+0000", 2023-10-09T06:07:36.740 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4400000000000001e-06 2023-10-09T06:07:36.740 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.741 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.741 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:07:36.741 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.573794+0000", 2023-10-09T06:07:36.741 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6019999999999996e-06 2023-10-09T06:07:36.741 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.741 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.741 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:07:36.741 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.573798+0000", 2023-10-09T06:07:36.741 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8840000000000004e-06 2023-10-09T06:07:36.741 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.741 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.741 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.742 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.573842+0000", 2023-10-09T06:07:36.742 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4132999999999999e-05 2023-10-09T06:07:36.742 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.742 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.742 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:07:36.742 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.573853+0000", 2023-10-09T06:07:36.742 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0348e-05 2023-10-09T06:07:36.742 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.742 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.742 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.742 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.467929+0000", 2023-10-09T06:07:36.743 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.894075807 2023-10-09T06:07:36.743 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.743 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.743 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:07:36.743 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.468058+0000", 2023-10-09T06:07:36.743 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000129386 2023-10-09T06:07:36.743 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.743 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.743 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:07:36.743 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.468187+0000", 2023-10-09T06:07:36.743 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012876000000000001 2023-10-09T06:07:36.744 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.744 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.744 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:07:36.744 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.472047+0000", 2023-10-09T06:07:36.744 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038605699999999998 2023-10-09T06:07:36.744 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.744 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.744 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.744 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.474909+0000", 2023-10-09T06:07:36.744 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028620780000000001 2023-10-09T06:07:36.744 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.745 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.745 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.745 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.479632+0000", 2023-10-09T06:07:36.745 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047231019999999999 2023-10-09T06:07:36.745 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.745 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.745 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:07:36.745 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.479685+0000", 2023-10-09T06:07:36.745 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2735000000000002e-05 2023-10-09T06:07:36.745 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.745 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.745 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:07:36.746 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.479711+0000", 2023-10-09T06:07:36.746 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5687e-05 2023-10-09T06:07:36.746 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.746 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:07:36.746 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.746 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.746 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.746 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:162114 8.1 8:8b02af32:::benchmark_data_smithi064_457003_object57596:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:07:36.746 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:52.583421+0000", 2023-10-09T06:07:36.746 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 104.064649448, 2023-10-09T06:07:36.746 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.89640450000000005, 2023-10-09T06:07:36.746 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:07:36.746 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:36.747 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:07:36.747 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:07:36.747 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:36.747 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 162114 2023-10-09T06:07:36.747 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.747 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:07:36.747 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.747 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:07:36.747 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.583421+0000", 2023-10-09T06:07:36.747 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.747 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.747 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.747 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:07:36.748 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.583421+0000", 2023-10-09T06:07:36.748 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.748 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:07:36.748 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.583420+0000", 2023-10-09T06:07:36.748 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:36.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.748 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:07:36.749 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.583427+0000", 2023-10-09T06:07:36.749 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4560000000000003e-06 2023-10-09T06:07:36.749 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.749 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.749 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:07:36.749 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.583444+0000", 2023-10-09T06:07:36.749 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6481999999999999e-05 2023-10-09T06:07:36.749 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.749 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.749 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:07:36.749 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.583452+0000", 2023-10-09T06:07:36.749 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2160000000000006e-06 2023-10-09T06:07:36.750 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.750 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.750 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.750 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.583504+0000", 2023-10-09T06:07:36.750 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2639000000000002e-05 2023-10-09T06:07:36.750 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.750 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.750 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:07:36.750 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.583514+0000", 2023-10-09T06:07:36.750 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8400000000000007e-06 2023-10-09T06:07:36.750 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.751 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.751 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.751 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.470007+0000", 2023-10-09T06:07:36.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88649222800000005 2023-10-09T06:07:36.751 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.751 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.751 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:07:36.751 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.470115+0000", 2023-10-09T06:07:36.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108137 2023-10-09T06:07:36.751 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.752 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.752 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:07:36.752 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.470228+0000", 2023-10-09T06:07:36.752 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000113377 2023-10-09T06:07:36.752 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.752 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.752 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:07:36.752 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.474723+0000", 2023-10-09T06:07:36.752 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044954269999999998 2023-10-09T06:07:36.752 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.753 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.474951+0000", 2023-10-09T06:07:36.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022724999999999999 2023-10-09T06:07:36.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.753 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.479759+0000", 2023-10-09T06:07:36.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.004808512 2023-10-09T06:07:36.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.753 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:07:36.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.479802+0000", 2023-10-09T06:07:36.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2945999999999998e-05 2023-10-09T06:07:36.754 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.754 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:07:36.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.479826+0000", 2023-10-09T06:07:36.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3668000000000001e-05 2023-10-09T06:07:36.754 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.754 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:07:36.754 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.754 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.755 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.755 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:162131 8.1 8:8403fa60:::benchmark_data_smithi064_457003_object57613:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:07:36.755 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:52.603173+0000", 2023-10-09T06:07:36.755 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 104.04489813799999, 2023-10-09T06:07:36.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.87681746800000004, 2023-10-09T06:07:36.755 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:07:36.755 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:36.755 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:07:36.755 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:07:36.755 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:36.755 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 162131 2023-10-09T06:07:36.755 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.756 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:07:36.756 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.756 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:07:36.756 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.603173+0000", 2023-10-09T06:07:36.756 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.756 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.756 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.756 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:07:36.756 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.603173+0000", 2023-10-09T06:07:36.756 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.756 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.757 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.757 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:07:36.757 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.603171+0000", 2023-10-09T06:07:36.757 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:36.757 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.757 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.757 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:07:36.757 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.603178+0000", 2023-10-09T06:07:36.757 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7479999999999993e-06 2023-10-09T06:07:36.757 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.757 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.758 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:07:36.758 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.603192+0000", 2023-10-09T06:07:36.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3485999999999999e-05 2023-10-09T06:07:36.758 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.758 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.758 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:07:36.758 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.603200+0000", 2023-10-09T06:07:36.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0800000000000006e-06 2023-10-09T06:07:36.758 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.758 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.758 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.759 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.603270+0000", 2023-10-09T06:07:36.759 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0234000000000006e-05 2023-10-09T06:07:36.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.759 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:07:36.759 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.603280+0000", 2023-10-09T06:07:36.759 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0128e-05 2023-10-09T06:07:36.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.759 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.759 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.472078+0000", 2023-10-09T06:07:36.760 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.86879760399999995 2023-10-09T06:07:36.760 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.760 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:07:36.760 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.472202+0000", 2023-10-09T06:07:36.760 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012362699999999999 2023-10-09T06:07:36.760 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.760 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:07:36.760 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.472336+0000", 2023-10-09T06:07:36.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013439999999999999 2023-10-09T06:07:36.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:07:36.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.474962+0000", 2023-10-09T06:07:36.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026261209999999999 2023-10-09T06:07:36.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.475001+0000", 2023-10-09T06:07:36.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8853000000000002e-05 2023-10-09T06:07:36.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.479907+0000", 2023-10-09T06:07:36.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049055399999999999 2023-10-09T06:07:36.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:07:36.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.479951+0000", 2023-10-09T06:07:36.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4700999999999998e-05 2023-10-09T06:07:36.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:07:36.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.479990+0000", 2023-10-09T06:07:36.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8859999999999997e-05 2023-10-09T06:07:36.763 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.763 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:07:36.763 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.764 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:162136 8.1 8:8865762b:::benchmark_data_smithi064_457003_object57618:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:07:36.764 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:52.606879+0000", 2023-10-09T06:07:36.764 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 104.04119207399999, 2023-10-09T06:07:36.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.87797659900000002, 2023-10-09T06:07:36.764 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:07:36.764 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:36.764 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:07:36.764 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:07:36.764 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:36.764 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 162136 2023-10-09T06:07:36.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.765 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:07:36.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:07:36.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.606879+0000", 2023-10-09T06:07:36.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:07:36.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.606879+0000", 2023-10-09T06:07:36.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:07:36.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.606876+0000", 2023-10-09T06:07:36.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T06:07:36.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:07:36.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.606888+0000", 2023-10-09T06:07:36.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1752e-05 2023-10-09T06:07:36.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:07:36.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.606915+0000", 2023-10-09T06:07:36.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6349000000000001e-05 2023-10-09T06:07:36.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:07:36.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.606927+0000", 2023-10-09T06:07:36.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2773e-05 2023-10-09T06:07:36.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.768 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.768 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.606990+0000", 2023-10-09T06:07:36.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2966000000000007e-05 2023-10-09T06:07:36.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.768 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:07:36.768 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.607004+0000", 2023-10-09T06:07:36.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3220999999999999e-05 2023-10-09T06:07:36.769 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.769 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.769 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.769 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.475026+0000", 2023-10-09T06:07:36.769 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.86802256099999997 2023-10-09T06:07:36.769 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.769 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.769 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:07:36.769 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.475118+0000", 2023-10-09T06:07:36.770 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2351999999999999e-05 2023-10-09T06:07:36.770 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.770 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:07:36.770 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.475250+0000", 2023-10-09T06:07:36.770 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013133099999999999 2023-10-09T06:07:36.770 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.770 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:07:36.770 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.479837+0000", 2023-10-09T06:07:36.770 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0045872100000000004 2023-10-09T06:07:36.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.771 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.479865+0000", 2023-10-09T06:07:36.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8095000000000001e-05 2023-10-09T06:07:36.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.771 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.484779+0000", 2023-10-09T06:07:36.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049136320000000002 2023-10-09T06:07:36.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:07:36.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.484816+0000", 2023-10-09T06:07:36.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6850000000000001e-05 2023-10-09T06:07:36.772 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:07:36.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.484855+0000", 2023-10-09T06:07:36.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9716999999999999e-05 2023-10-09T06:07:36.772 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.772 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:07:36.773 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.773 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:162168 8.1 8:8908c5b0:::benchmark_data_smithi064_457003_object57650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:07:36.773 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:52.676885+0000", 2023-10-09T06:07:36.773 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 103.971185506, 2023-10-09T06:07:36.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.813068016, 2023-10-09T06:07:36.773 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:07:36.773 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:36.773 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:07:36.774 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:07:36.774 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:36.774 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 162168 2023-10-09T06:07:36.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.774 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:07:36.774 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:07:36.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.676885+0000", 2023-10-09T06:07:36.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.774 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:07:36.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.676885+0000", 2023-10-09T06:07:36.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:07:36.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.676883+0000", 2023-10-09T06:07:36.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T06:07:36.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:07:36.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.676898+0000", 2023-10-09T06:07:36.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4671999999999999e-05 2023-10-09T06:07:36.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:07:36.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.676929+0000", 2023-10-09T06:07:36.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1375999999999999e-05 2023-10-09T06:07:36.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.777 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:07:36.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.676942+0000", 2023-10-09T06:07:36.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2649999999999999e-05 2023-10-09T06:07:36.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.777 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.677752+0000", 2023-10-09T06:07:36.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00081067699999999999 2023-10-09T06:07:36.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:07:36.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.677764+0000", 2023-10-09T06:07:36.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1755e-05 2023-10-09T06:07:36.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.477420+0000", 2023-10-09T06:07:36.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.79965610300000001 2023-10-09T06:07:36.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:07:36.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.477507+0000", 2023-10-09T06:07:36.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6831999999999998e-05 2023-10-09T06:07:36.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:07:36.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.477600+0000", 2023-10-09T06:07:36.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3043999999999999e-05 2023-10-09T06:07:36.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:07:36.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.480003+0000", 2023-10-09T06:07:36.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024032680000000001 2023-10-09T06:07:36.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.484918+0000", 2023-10-09T06:07:36.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049150039999999997 2023-10-09T06:07:36.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.489882+0000", 2023-10-09T06:07:36.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049636039999999999 2023-10-09T06:07:36.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:07:36.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.489912+0000", 2023-10-09T06:07:36.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0086e-05 2023-10-09T06:07:36.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:07:36.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.489953+0000", 2023-10-09T06:07:36.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1306000000000001e-05 2023-10-09T06:07:36.782 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.782 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:07:36.782 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.782 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.782 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:162174 8.1 8:893fbad4:::benchmark_data_smithi064_457003_object57656:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:07:36.782 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:52.688887+0000", 2023-10-09T06:07:36.782 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 103.95918344499999, 2023-10-09T06:07:36.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.80124467899999996, 2023-10-09T06:07:36.783 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:07:36.783 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:36.783 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:07:36.783 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:07:36.783 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:36.783 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 162174 2023-10-09T06:07:36.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.783 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:07:36.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.783 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:07:36.783 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.688887+0000", 2023-10-09T06:07:36.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.784 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:07:36.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.688887+0000", 2023-10-09T06:07:36.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.784 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:07:36.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.688886+0000", 2023-10-09T06:07:36.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:36.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.785 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:07:36.785 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.688893+0000", 2023-10-09T06:07:36.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6129999999999998e-06 2023-10-09T06:07:36.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.785 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:07:36.785 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.688906+0000", 2023-10-09T06:07:36.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2346e-05 2023-10-09T06:07:36.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:07:36.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.688915+0000", 2023-10-09T06:07:36.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6199999999999994e-06 2023-10-09T06:07:36.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.688973+0000", 2023-10-09T06:07:36.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7553000000000003e-05 2023-10-09T06:07:36.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:07:36.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.688983+0000", 2023-10-09T06:07:36.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7909999999999999e-06 2023-10-09T06:07:36.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.480041+0000", 2023-10-09T06:07:36.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.79105848999999995 2023-10-09T06:07:36.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:07:36.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.480151+0000", 2023-10-09T06:07:36.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110189 2023-10-09T06:07:36.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:07:36.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.480277+0000", 2023-10-09T06:07:36.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000125742 2023-10-09T06:07:36.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:07:36.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.484930+0000", 2023-10-09T06:07:36.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046535209999999999 2023-10-09T06:07:36.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.484970+0000", 2023-10-09T06:07:36.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9570000000000002e-05 2023-10-09T06:07:36.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.490079+0000", 2023-10-09T06:07:36.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051088490000000004 2023-10-09T06:07:36.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:07:36.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.490107+0000", 2023-10-09T06:07:36.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8433999999999999e-05 2023-10-09T06:07:36.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:07:36.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.490132+0000", 2023-10-09T06:07:36.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4677000000000001e-05 2023-10-09T06:07:36.791 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.791 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:07:36.791 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.791 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:162215 8.1 8:8db78b38:::benchmark_data_smithi064_457003_object57697:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:07:36.791 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:52.761265+0000", 2023-10-09T06:07:36.791 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 103.886806178, 2023-10-09T06:07:36.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.72899867200000001, 2023-10-09T06:07:36.791 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:07:36.791 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:36.792 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:07:36.792 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:07:36.792 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:36.792 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 162215 2023-10-09T06:07:36.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.792 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:07:36.792 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.792 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:07:36.792 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.761265+0000", 2023-10-09T06:07:36.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:07:36.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.761265+0000", 2023-10-09T06:07:36.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:07:36.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.761263+0000", 2023-10-09T06:07:36.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:07:36.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:07:36.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.761270+0000", 2023-10-09T06:07:36.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4150000000000004e-06 2023-10-09T06:07:36.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:07:36.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.761287+0000", 2023-10-09T06:07:36.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7493999999999999e-05 2023-10-09T06:07:36.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:07:36.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.761294+0000", 2023-10-09T06:07:36.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7789999999999998e-06 2023-10-09T06:07:36.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.761339+0000", 2023-10-09T06:07:36.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4814000000000002e-05 2023-10-09T06:07:36.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:07:36.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.761347+0000", 2023-10-09T06:07:36.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9289999999999998e-06 2023-10-09T06:07:36.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.482374+0000", 2023-10-09T06:07:36.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.72102741100000001 2023-10-09T06:07:36.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:07:36.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.482484+0000", 2023-10-09T06:07:36.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000109985 2023-10-09T06:07:36.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:07:36.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.482556+0000", 2023-10-09T06:07:36.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1958000000000005e-05 2023-10-09T06:07:36.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.799 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.799 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:07:36.799 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.487248+0000", 2023-10-09T06:07:36.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046913679999999996 2023-10-09T06:07:36.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.799 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.799 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.799 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.489844+0000", 2023-10-09T06:07:36.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002596292 2023-10-09T06:07:36.800 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.800 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.490204+0000", 2023-10-09T06:07:36.800 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00035965500000000001 2023-10-09T06:07:36.800 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:07:36.800 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.490244+0000", 2023-10-09T06:07:36.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0003999999999998e-05 2023-10-09T06:07:36.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:07:36.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.490263+0000", 2023-10-09T06:07:36.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9760000000000001e-05 2023-10-09T06:07:36.801 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.801 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:07:36.801 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.801 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:162231 8.1 8:8cc0cd38:::benchmark_data_smithi064_457003_object57713:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:07:36.802 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:52.789426+0000", 2023-10-09T06:07:36.802 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 103.858644829, 2023-10-09T06:07:36.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.70093691000000002, 2023-10-09T06:07:36.802 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:07:36.802 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:36.802 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:07:36.802 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:07:36.802 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:36.802 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 162231 2023-10-09T06:07:36.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.802 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:07:36.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.803 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:07:36.803 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.789426+0000", 2023-10-09T06:07:36.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.803 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:07:36.803 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.789426+0000", 2023-10-09T06:07:36.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:07:36.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.789424+0000", 2023-10-09T06:07:36.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:36.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:07:36.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.789431+0000", 2023-10-09T06:07:36.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.297e-06 2023-10-09T06:07:36.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:07:36.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.789445+0000", 2023-10-09T06:07:36.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3916e-05 2023-10-09T06:07:36.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:07:36.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.789451+0000", 2023-10-09T06:07:36.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3840000000000002e-06 2023-10-09T06:07:36.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.789491+0000", 2023-10-09T06:07:36.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0009999999999998e-05 2023-10-09T06:07:36.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:07:36.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.789499+0000", 2023-10-09T06:07:36.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4759999999999998e-06 2023-10-09T06:07:36.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.485003+0000", 2023-10-09T06:07:36.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.69550327999999995 2023-10-09T06:07:36.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:07:36.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.485120+0000", 2023-10-09T06:07:36.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011769600000000001 2023-10-09T06:07:36.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:07:36.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.485237+0000", 2023-10-09T06:07:36.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116437 2023-10-09T06:07:36.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.490030+0000", 2023-10-09T06:07:36.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047934600000000003 2023-10-09T06:07:36.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:07:36.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.490045+0000", 2023-10-09T06:07:36.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4165000000000001e-05 2023-10-09T06:07:36.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.490301+0000", 2023-10-09T06:07:36.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00025604000000000002 2023-10-09T06:07:36.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:07:36.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.490341+0000", 2023-10-09T06:07:36.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0754000000000003e-05 2023-10-09T06:07:36.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:07:36.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.490363+0000", 2023-10-09T06:07:36.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1563999999999999e-05 2023-10-09T06:07:36.810 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.810 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:07:36.810 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.810 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:162242 8.1 8:84b66939:::benchmark_data_smithi064_457003_object57724:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:07:36.810 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:52.813137+0000", 2023-10-09T06:07:36.810 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 103.834933466, 2023-10-09T06:07:36.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.677304935, 2023-10-09T06:07:36.811 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:07:36.811 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:36.811 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:07:36.811 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:07:36.811 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:36.811 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 162242 2023-10-09T06:07:36.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.811 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:07:36.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:07:36.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.813137+0000", 2023-10-09T06:07:36.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:07:36.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.813137+0000", 2023-10-09T06:07:36.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:07:36.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.813136+0000", 2023-10-09T06:07:36.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:36.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:07:36.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.813142+0000", 2023-10-09T06:07:36.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0929999999999999e-06 2023-10-09T06:07:36.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:07:36.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.813156+0000", 2023-10-09T06:07:36.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4221e-05 2023-10-09T06:07:36.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:07:36.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.813164+0000", 2023-10-09T06:07:36.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7549999999999994e-06 2023-10-09T06:07:36.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.813216+0000", 2023-10-09T06:07:36.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2500999999999998e-05 2023-10-09T06:07:36.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:07:36.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.813226+0000", 2023-10-09T06:07:36.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.978e-06 2023-10-09T06:07:36.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.487293+0000", 2023-10-09T06:07:36.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.67406692700000004 2023-10-09T06:07:36.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:07:36.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.487402+0000", 2023-10-09T06:07:36.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000109261 2023-10-09T06:07:36.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:07:36.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.487496+0000", 2023-10-09T06:07:36.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3195000000000005e-05 2023-10-09T06:07:36.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.490167+0000", 2023-10-09T06:07:36.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002671657 2023-10-09T06:07:36.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:07:36.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.490176+0000", 2023-10-09T06:07:36.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6249999999999996e-06 2023-10-09T06:07:36.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.490403+0000", 2023-10-09T06:07:36.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022667100000000001 2023-10-09T06:07:36.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:07:36.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.490425+0000", 2023-10-09T06:07:36.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2602000000000001e-05 2023-10-09T06:07:36.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:07:36.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.490442+0000", 2023-10-09T06:07:36.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7031e-05 2023-10-09T06:07:36.819 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.819 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:07:36.819 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.819 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:162251 8.1 8:88ef480e:::benchmark_data_smithi064_457003_object57733:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:07:36.819 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:52.835308+0000", 2023-10-09T06:07:36.819 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 103.81276313799999, 2023-10-09T06:07:36.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.66345648800000001, 2023-10-09T06:07:36.819 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:07:36.819 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:36.820 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:07:36.820 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:07:36.820 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:36.820 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 162251 2023-10-09T06:07:36.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.820 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:07:36.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:07:36.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.835308+0000", 2023-10-09T06:07:36.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:07:36.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.835308+0000", 2023-10-09T06:07:36.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:07:36.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.835306+0000", 2023-10-09T06:07:36.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:07:36.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:07:36.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.835312+0000", 2023-10-09T06:07:36.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8109999999999999e-06 2023-10-09T06:07:36.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:07:36.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.835328+0000", 2023-10-09T06:07:36.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6116999999999998e-05 2023-10-09T06:07:36.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:07:36.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.835336+0000", 2023-10-09T06:07:36.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6229999999999997e-06 2023-10-09T06:07:36.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.835384+0000", 2023-10-09T06:07:36.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7951999999999998e-05 2023-10-09T06:07:36.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:07:36.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.835395+0000", 2023-10-09T06:07:36.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1637000000000001e-05 2023-10-09T06:07:36.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.490477+0000", 2023-10-09T06:07:36.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.65508152500000005 2023-10-09T06:07:36.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:07:36.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.490562+0000", 2023-10-09T06:07:36.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5055999999999993e-05 2023-10-09T06:07:36.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:07:36.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.490631+0000", 2023-10-09T06:07:36.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9363999999999995e-05 2023-10-09T06:07:36.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.498561+0000", 2023-10-09T06:07:36.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0079292310000000001 2023-10-09T06:07:36.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.498596+0000", 2023-10-09T06:07:36.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5244999999999999e-05 2023-10-09T06:07:36.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:07:36.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.498677+0000", 2023-10-09T06:07:36.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0681999999999998e-05 2023-10-09T06:07:36.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:07:36.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.498729+0000", 2023-10-09T06:07:36.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2839e-05 2023-10-09T06:07:36.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:07:36.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.498764+0000", 2023-10-09T06:07:36.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4746000000000002e-05 2023-10-09T06:07:36.828 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.828 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:07:36.828 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.828 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:162260 8.1 8:8116d983:::benchmark_data_smithi064_457003_object57742:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:07:36.828 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:52.886803+0000", 2023-10-09T06:07:36.828 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 103.761267362, 2023-10-09T06:07:36.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.61208563400000005, 2023-10-09T06:07:36.828 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:07:36.829 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:07:36.829 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:07:36.829 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:07:36.829 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:07:36.829 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 162260 2023-10-09T06:07:36.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.829 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:07:36.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:07:36.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.886803+0000", 2023-10-09T06:07:36.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:07:36.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.886803+0000", 2023-10-09T06:07:36.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:07:36.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:07:36.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.886802+0000", 2023-10-09T06:07:36.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:07:36.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:07:36.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.886810+0000", 2023-10-09T06:07:36.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6079999999999996e-06 2023-10-09T06:07:36.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:07:36.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.886830+0000", 2023-10-09T06:07:36.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0058999999999999e-05 2023-10-09T06:07:36.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:07:36.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.886839+0000", 2023-10-09T06:07:36.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0410000000000003e-06 2023-10-09T06:07:36.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.886890+0000", 2023-10-09T06:07:36.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0702999999999999e-05 2023-10-09T06:07:36.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:07:36.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:52.886897+0000", 2023-10-09T06:07:36.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3080000000000002e-06 2023-10-09T06:07:36.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:07:36.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.493179+0000", 2023-10-09T06:07:36.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.60628179400000004 2023-10-09T06:07:36.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:07:36.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.493311+0000", 2023-10-09T06:07:36.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000132263 2023-10-09T06:07:36.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:07:36.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.493446+0000", 2023-10-09T06:07:36.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013456500000000001 2023-10-09T06:07:36.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.498664+0000", 2023-10-09T06:07:36.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0052180890000000004 2023-10-09T06:07:36.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:07:36.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.498807+0000", 2023-10-09T06:07:36.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014266999999999999 2023-10-09T06:07:36.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:07:36.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.498819+0000", 2023-10-09T06:07:36.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2598e-05 2023-10-09T06:07:36.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:07:36.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.498856+0000", 2023-10-09T06:07:36.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7089999999999999e-05 2023-10-09T06:07:36.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:07:36.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:07:36.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:07:36.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:53.498889+0000", 2023-10-09T06:07:36.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2583000000000003e-05 2023-10-09T06:07:36.837 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.837 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:07:36.837 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.837 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:07:36.837 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:07:36.837 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:07:36.837 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1] out_osds: [2] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 0, 4] 2023-10-09T06:07:36.837 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:07:36.838 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-10-09T06:07:36.838 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd in 2 2023-10-09T06:07:36.838 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:36.839 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 23 16 45612 45596 123.858 91.5625 0.00662889 0.00806893 2023-10-09T06:07:36.840 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[212750]: pgmap v1061: 119 pgs: 119 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T06:07:36.840 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:07:36.840 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:07:36.840 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2085408380' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:07:36.840 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[212750]: osdmap e431: 8 total, 8 up, 7 in 2023-10-09T06:07:36.840 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[212750]: failed to encode map e431 with expected crc 2023-10-09T06:07:36.840 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[212750]: failed to encode map e431 with expected crc 2023-10-09T06:07:36.840 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[212750]: failed to encode map e431 with expected crc 2023-10-09T06:07:36.840 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[212750]: failed to encode map e431 with expected crc 2023-10-09T06:07:36.840 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[212750]: failed to encode map e431 with expected crc 2023-10-09T06:07:36.840 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[212750]: failed to encode map e431 with expected crc 2023-10-09T06:07:36.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[212750]: failed to encode map e431 with expected crc 2023-10-09T06:07:36.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[212750]: failed to encode map e431 with expected crc 2023-10-09T06:07:36.841 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[220044]: pgmap v1061: 119 pgs: 119 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T06:07:36.841 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:07:36.841 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T06:07:36.842 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2085408380' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:07:36.842 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[220044]: osdmap e431: 8 total, 8 up, 7 in 2023-10-09T06:07:36.842 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[220044]: failed to encode map e431 with expected crc 2023-10-09T06:07:36.842 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[220044]: failed to encode map e431 with expected crc 2023-10-09T06:07:36.842 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[220044]: failed to encode map e431 with expected crc 2023-10-09T06:07:36.842 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[220044]: failed to encode map e431 with expected crc 2023-10-09T06:07:36.842 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[220044]: failed to encode map e431 with expected crc 2023-10-09T06:07:36.842 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[220044]: failed to encode map e431 with expected crc 2023-10-09T06:07:36.842 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[220044]: failed to encode map e431 with expected crc 2023-10-09T06:07:36.842 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:36 smithi064 ceph-mon[220044]: failed to encode map e431 with expected crc 2023-10-09T06:07:36.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:36 smithi156 ceph-mon[195101]: pgmap v1061: 119 pgs: 119 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T06:07:36.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:36 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:07:36.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:36 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T06:07:36.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:36 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2085408380' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:07:36.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:36 smithi156 ceph-mon[195101]: osdmap e431: 8 total, 8 up, 7 in 2023-10-09T06:07:36.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:36 smithi156 ceph-mon[195101]: failed to encode map e431 with expected crc 2023-10-09T06:07:36.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:36 smithi156 ceph-mon[195101]: failed to encode map e431 with expected crc 2023-10-09T06:07:36.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:36 smithi156 ceph-mon[195101]: failed to encode map e431 with expected crc 2023-10-09T06:07:36.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:36 smithi156 ceph-mon[195101]: failed to encode map e431 with expected crc 2023-10-09T06:07:36.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:36 smithi156 ceph-mon[195101]: failed to encode map e431 with expected crc 2023-10-09T06:07:36.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:36 smithi156 ceph-mon[195101]: failed to encode map e431 with expected crc 2023-10-09T06:07:36.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:36 smithi156 ceph-mon[195101]: failed to encode map e431 with expected crc 2023-10-09T06:07:36.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:36 smithi156 ceph-mon[195101]: failed to encode map e431 with expected crc 2023-10-09T06:07:36.909 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:37.468 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:37.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:37.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:37.578 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:37.630 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:37.699 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 24 16 46862 46846 121.937 78.125 0.00523914 0.0081952 2023-10-09T06:07:37.792 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:37.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:37.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:37.895 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:37.923 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:38.354 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:07:38.361 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:38.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:38.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:38.471 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:38.536 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:38.542 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:38.600 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:38.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:38.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:38.705 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 25 16 48133 48117 120.208 79.4375 0.00778068 0.00831373 2023-10-09T06:07:38.709 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:38.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:38 smithi064 ceph-mon[212750]: pgmap v1063: 119 pgs: 119 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-10-09T06:07:38.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:38 smithi064 ceph-mon[220044]: pgmap v1063: 119 pgs: 119 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-10-09T06:07:38.774 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:38.864 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:07:38.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:38 smithi156 ceph-mon[195101]: pgmap v1063: 119 pgs: 119 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-10-09T06:07:38.961 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:38.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:38.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:39.066 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:39.109 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:39.180 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:39.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:39.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:39.247 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:07:39.285 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:07:39.292 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:39.327 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:07:39.330 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:39.372 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:07:39.405 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:07:39.407 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:39.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:39.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:39.451 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:07:39.489 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:07:39.517 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:39.534 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:07:39.552 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:07:39.552 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:07:39.552 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:07:39.552 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:07:39.552 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:07:39.552 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:07:39.552 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:07:39.552 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:07:39.582 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:39.706 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 26 16 49365 49349 118.546 77 0.0290655 0.00842702 2023-10-09T06:07:39.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:39 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2632636364' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:39.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:39 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2632636364' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:39.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:39 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2632636364' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:40.353 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:40.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:40.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:40.461 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:40.520 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:40.705 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 27 16 50408 50392 116.571 65.1875 0.00227415 0.00855626 2023-10-09T06:07:40.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:40 smithi064 ceph-mon[212750]: pgmap v1064: 119 pgs: 119 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-10-09T06:07:40.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:40 smithi064 ceph-mon[220044]: pgmap v1064: 119 pgs: 119 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-10-09T06:07:40.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:40 smithi156 ceph-mon[195101]: pgmap v1064: 119 pgs: 119 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-10-09T06:07:40.894 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:41.104 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:41.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:41.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:41.215 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:41.261 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:41.331 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:41.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:41.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:41.439 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:41.505 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:41.583 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:41.706 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 28 16 51659 51643 115.2 78.1875 0.0154572 0.0086761 2023-10-09T06:07:41.710 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:41.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:41.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:41.821 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:41.889 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:41.982 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:41.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:41.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:42.089 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:42.146 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:42.246 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:42.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:42.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:42.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:42 smithi064 conmon[212726]: 2023-10-09T06:07:42.214+0000 7f8823f23700 -1 mon.a@0(leader).osd e431 definitely_dead 0 2023-10-09T06:07:42.354 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:42.402 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:42.443 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:42.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:42.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:42.550 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:42.570 INFO:teuthology.orchestra.run.smithi064.stderr:marked in osd.2. 2023-10-09T06:07:42.584 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:42.641 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:42.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:42.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:42.706 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 29 16 52694 52678 113.459 64.6875 0.00179386 0.00878016 2023-10-09T06:07:42.749 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:42.778 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:42.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:42 smithi156 ceph-mon[195101]: pgmap v1065: 119 pgs: 119 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T06:07:42.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:42 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3360368418' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T06:07:42.920 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:42 smithi064 ceph-mon[212750]: pgmap v1065: 119 pgs: 119 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T06:07:42.920 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:42 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3360368418' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T06:07:42.921 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:42 smithi064 ceph-mon[220044]: pgmap v1065: 119 pgs: 119 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T06:07:42.921 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:42 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3360368418' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T06:07:43.165 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:07:43.278 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:43.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:43.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:43.381 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:43.414 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:43.463 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:43.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:43.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:43.481 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-10-09T06:07:43.568 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:43.615 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:43.706 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 30 16 52694 52678 109.679 0 - 0.00878016 2023-10-09T06:07:43.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:43 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3360368418' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T06:07:43.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:43 smithi064 ceph-mon[212750]: osdmap e432: 8 total, 8 up, 8 in 2023-10-09T06:07:43.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:43 smithi064 ceph-mon[212750]: failed to encode map e432 with expected crc 2023-10-09T06:07:43.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:43 smithi064 ceph-mon[212750]: failed to encode map e432 with expected crc 2023-10-09T06:07:43.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:43 smithi064 ceph-mon[212750]: failed to encode map e432 with expected crc 2023-10-09T06:07:43.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:43 smithi064 ceph-mon[212750]: failed to encode map e432 with expected crc 2023-10-09T06:07:43.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:43 smithi064 ceph-mon[212750]: failed to encode map e432 with expected crc 2023-10-09T06:07:43.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:43 smithi064 ceph-mon[212750]: failed to encode map e432 with expected crc 2023-10-09T06:07:43.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:43 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:43.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:43 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:43.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:43 smithi064 ceph-mon[212750]: osdmap e433: 8 total, 8 up, 8 in 2023-10-09T06:07:43.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:43 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3360368418' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T06:07:43.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:43 smithi064 ceph-mon[220044]: osdmap e432: 8 total, 8 up, 8 in 2023-10-09T06:07:43.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:43 smithi064 ceph-mon[220044]: failed to encode map e432 with expected crc 2023-10-09T06:07:43.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:43 smithi064 ceph-mon[220044]: failed to encode map e432 with expected crc 2023-10-09T06:07:43.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:43 smithi064 ceph-mon[220044]: failed to encode map e432 with expected crc 2023-10-09T06:07:43.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:43 smithi064 ceph-mon[220044]: failed to encode map e432 with expected crc 2023-10-09T06:07:43.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:43 smithi064 ceph-mon[220044]: failed to encode map e432 with expected crc 2023-10-09T06:07:43.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:43 smithi064 ceph-mon[220044]: failed to encode map e432 with expected crc 2023-10-09T06:07:43.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:43 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:43.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:43 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:43.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:43 smithi064 ceph-mon[220044]: osdmap e433: 8 total, 8 up, 8 in 2023-10-09T06:07:43.813 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:43.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:43.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:43.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:43 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3360368418' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T06:07:43.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:43 smithi156 ceph-mon[195101]: osdmap e432: 8 total, 8 up, 8 in 2023-10-09T06:07:43.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:43 smithi156 ceph-mon[195101]: failed to encode map e432 with expected crc 2023-10-09T06:07:43.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:43 smithi156 ceph-mon[195101]: failed to encode map e432 with expected crc 2023-10-09T06:07:43.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:43 smithi156 ceph-mon[195101]: failed to encode map e432 with expected crc 2023-10-09T06:07:43.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:43 smithi156 ceph-mon[195101]: failed to encode map e432 with expected crc 2023-10-09T06:07:43.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:43 smithi156 ceph-mon[195101]: failed to encode map e432 with expected crc 2023-10-09T06:07:43.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:43 smithi156 ceph-mon[195101]: failed to encode map e432 with expected crc 2023-10-09T06:07:43.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:43 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:43.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:43 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:43.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:43 smithi156 ceph-mon[195101]: osdmap e433: 8 total, 8 up, 8 in 2023-10-09T06:07:43.917 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:43.944 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:43.987 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:43.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:43.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:44.093 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:44.124 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:44.272 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:44.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:44.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:44.375 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:07:44.409 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:44.436 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:44.465 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:07:44.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:44.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:07:44.485 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:07:44.485 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:07:44.570 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:44.600 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:44.647 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:44.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:44.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:44.706 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 31 16 52694 52678 106.143 0 - 0.00878016 2023-10-09T06:07:44.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:44 smithi064 ceph-mon[212750]: pgmap v1067: 119 pgs: 1 unknown, 118 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T06:07:44.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:44 smithi064 ceph-mon[212750]: failed to encode map e433 with expected crc 2023-10-09T06:07:44.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:44 smithi064 ceph-mon[212750]: failed to encode map e433 with expected crc 2023-10-09T06:07:44.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:44 smithi064 ceph-mon[212750]: failed to encode map e433 with expected crc 2023-10-09T06:07:44.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:44 smithi064 ceph-mon[212750]: failed to encode map e433 with expected crc 2023-10-09T06:07:44.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:44 smithi064 ceph-mon[212750]: failed to encode map e433 with expected crc 2023-10-09T06:07:44.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:44 smithi064 ceph-mon[212750]: failed to encode map e433 with expected crc 2023-10-09T06:07:44.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:44 smithi064 ceph-mon[212750]: osdmap e434: 8 total, 8 up, 8 in 2023-10-09T06:07:44.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:44 smithi064 ceph-mon[220044]: pgmap v1067: 119 pgs: 1 unknown, 118 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T06:07:44.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:44 smithi064 ceph-mon[220044]: failed to encode map e433 with expected crc 2023-10-09T06:07:44.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:44 smithi064 ceph-mon[220044]: failed to encode map e433 with expected crc 2023-10-09T06:07:44.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:44 smithi064 ceph-mon[220044]: failed to encode map e433 with expected crc 2023-10-09T06:07:44.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:44 smithi064 ceph-mon[220044]: failed to encode map e433 with expected crc 2023-10-09T06:07:44.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:44 smithi064 ceph-mon[220044]: failed to encode map e433 with expected crc 2023-10-09T06:07:44.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:44 smithi064 ceph-mon[220044]: failed to encode map e433 with expected crc 2023-10-09T06:07:44.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:44 smithi064 ceph-mon[220044]: osdmap e434: 8 total, 8 up, 8 in 2023-10-09T06:07:44.754 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:44.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:44 smithi156 ceph-mon[195101]: pgmap v1067: 119 pgs: 1 unknown, 118 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T06:07:44.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:44 smithi156 ceph-mon[195101]: failed to encode map e433 with expected crc 2023-10-09T06:07:44.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:44 smithi156 ceph-mon[195101]: failed to encode map e433 with expected crc 2023-10-09T06:07:44.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:44 smithi156 ceph-mon[195101]: failed to encode map e433 with expected crc 2023-10-09T06:07:44.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:44 smithi156 ceph-mon[195101]: failed to encode map e433 with expected crc 2023-10-09T06:07:44.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:44 smithi156 ceph-mon[195101]: failed to encode map e433 with expected crc 2023-10-09T06:07:44.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:44 smithi156 ceph-mon[195101]: failed to encode map e433 with expected crc 2023-10-09T06:07:44.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:44 smithi156 ceph-mon[195101]: osdmap e434: 8 total, 8 up, 8 in 2023-10-09T06:07:44.910 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:45.547 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:45.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:45.559 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:07:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:07:45] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.43.0" 2023-10-09T06:07:45.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:45.661 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:45.707 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 32 16 53462 53446 104.323 16 0.00450109 0.00957945 2023-10-09T06:07:45.711 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:07:45.741 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:45.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:45 smithi064 ceph-mon[212750]: failed to encode map e434 with expected crc 2023-10-09T06:07:45.834 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:45 smithi064 ceph-mon[212750]: failed to encode map e434 with expected crc 2023-10-09T06:07:45.834 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:45 smithi064 ceph-mon[212750]: failed to encode map e434 with expected crc 2023-10-09T06:07:45.834 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:45 smithi064 ceph-mon[212750]: failed to encode map e434 with expected crc 2023-10-09T06:07:45.834 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:45 smithi064 ceph-mon[212750]: failed to encode map e434 with expected crc 2023-10-09T06:07:45.834 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:45 smithi064 ceph-mon[212750]: osdmap e435: 8 total, 8 up, 8 in 2023-10-09T06:07:45.835 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:45 smithi064 ceph-mon[220044]: failed to encode map e434 with expected crc 2023-10-09T06:07:45.835 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:45 smithi064 ceph-mon[220044]: failed to encode map e434 with expected crc 2023-10-09T06:07:45.835 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:45 smithi064 ceph-mon[220044]: failed to encode map e434 with expected crc 2023-10-09T06:07:45.835 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:45 smithi064 ceph-mon[220044]: failed to encode map e434 with expected crc 2023-10-09T06:07:45.835 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:45 smithi064 ceph-mon[220044]: failed to encode map e434 with expected crc 2023-10-09T06:07:45.835 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:45 smithi064 ceph-mon[220044]: osdmap e435: 8 total, 8 up, 8 in 2023-10-09T06:07:45.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:45 smithi156 ceph-mon[195101]: failed to encode map e434 with expected crc 2023-10-09T06:07:45.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:45 smithi156 ceph-mon[195101]: failed to encode map e434 with expected crc 2023-10-09T06:07:45.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:45 smithi156 ceph-mon[195101]: failed to encode map e434 with expected crc 2023-10-09T06:07:45.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:45 smithi156 ceph-mon[195101]: failed to encode map e434 with expected crc 2023-10-09T06:07:45.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:45 smithi156 ceph-mon[195101]: failed to encode map e434 with expected crc 2023-10-09T06:07:45.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:45 smithi156 ceph-mon[195101]: osdmap e435: 8 total, 8 up, 8 in 2023-10-09T06:07:46.170 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:46.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:46.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:46.274 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:46.300 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:46.345 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:46.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:46.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:46.451 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:46.513 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:46.607 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:46 smithi156 ceph-mon[195101]: pgmap v1070: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 remapped+peering, 1 unknown, 115 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 3242/153660 objects misplaced (2.110%); 32 KiB/s, 0 objects/s recovering 2023-10-09T06:07:46.607 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:46 smithi156 ceph-mon[195101]: failed to encode map e435 with expected crc 2023-10-09T06:07:46.607 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:46 smithi156 ceph-mon[195101]: failed to encode map e435 with expected crc 2023-10-09T06:07:46.607 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:46 smithi156 ceph-mon[195101]: failed to encode map e435 with expected crc 2023-10-09T06:07:46.607 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:46 smithi156 ceph-mon[195101]: failed to encode map e435 with expected crc 2023-10-09T06:07:46.607 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:46 smithi156 ceph-mon[195101]: failed to encode map e435 with expected crc 2023-10-09T06:07:46.607 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:46 smithi156 ceph-mon[195101]: osdmap e436: 8 total, 8 up, 8 in 2023-10-09T06:07:46.709 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 33 16 54253 54237 102.656 49.4375 0.0191848 0.00973558 2023-10-09T06:07:46.767 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:46 smithi064 ceph-mon[220044]: pgmap v1070: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 remapped+peering, 1 unknown, 115 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 3242/153660 objects misplaced (2.110%); 32 KiB/s, 0 objects/s recovering 2023-10-09T06:07:46.767 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:46 smithi064 ceph-mon[220044]: failed to encode map e435 with expected crc 2023-10-09T06:07:46.767 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:46 smithi064 ceph-mon[220044]: failed to encode map e435 with expected crc 2023-10-09T06:07:46.768 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:46 smithi064 ceph-mon[220044]: failed to encode map e435 with expected crc 2023-10-09T06:07:46.768 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:46 smithi064 ceph-mon[220044]: failed to encode map e435 with expected crc 2023-10-09T06:07:46.768 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:46 smithi064 ceph-mon[220044]: failed to encode map e435 with expected crc 2023-10-09T06:07:46.768 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:46 smithi064 ceph-mon[220044]: osdmap e436: 8 total, 8 up, 8 in 2023-10-09T06:07:46.768 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:46 smithi064 ceph-mon[212750]: pgmap v1070: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 remapped+peering, 1 unknown, 115 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 3242/153660 objects misplaced (2.110%); 32 KiB/s, 0 objects/s recovering 2023-10-09T06:07:46.768 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:46 smithi064 ceph-mon[212750]: failed to encode map e435 with expected crc 2023-10-09T06:07:46.768 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:46 smithi064 ceph-mon[212750]: failed to encode map e435 with expected crc 2023-10-09T06:07:46.768 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:46 smithi064 ceph-mon[212750]: failed to encode map e435 with expected crc 2023-10-09T06:07:46.768 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:46 smithi064 ceph-mon[212750]: failed to encode map e435 with expected crc 2023-10-09T06:07:46.768 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:46 smithi064 ceph-mon[212750]: failed to encode map e435 with expected crc 2023-10-09T06:07:46.769 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:46 smithi064 ceph-mon[212750]: osdmap e436: 8 total, 8 up, 8 in 2023-10-09T06:07:46.781 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:46.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:46.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:46.881 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:46.893 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:46.924 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:46.985 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:46.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:46.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:47.093 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:47.195 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:47.381 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:47.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:47.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:47.495 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:47.526 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:47.595 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:47.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:47.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:47.618 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:07:47.666 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:07:47.685 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:07:47.703 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:47.704 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:07:47.710 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 34 16 55178 55162 101.337 57.8125 0.011799 0.00986163 2023-10-09T06:07:47.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:47 smithi064 ceph-mon[212750]: failed to encode map e436 with expected crc 2023-10-09T06:07:47.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:47 smithi064 ceph-mon[212750]: failed to encode map e436 with expected crc 2023-10-09T06:07:47.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:47 smithi064 ceph-mon[212750]: failed to encode map e436 with expected crc 2023-10-09T06:07:47.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:47 smithi064 ceph-mon[212750]: failed to encode map e436 with expected crc 2023-10-09T06:07:47.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:47 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/531627525' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:07:47.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:47 smithi064 ceph-mon[212750]: pgmap v1073: 119 pgs: 3 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 remapped+peering, 1 active+remapped, 109 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 7.0 KiB/s rd, 49 MiB/s wr, 792 op/s; 6759/159903 objects degraded (4.227%); 3242/159903 objects misplaced (2.027%); 125 KiB/s, 2 keys/s, 1 objects/s recovering 2023-10-09T06:07:47.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:47 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1713739416' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:47.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:47 smithi064 ceph-mon[220044]: failed to encode map e436 with expected crc 2023-10-09T06:07:47.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:47 smithi064 ceph-mon[220044]: failed to encode map e436 with expected crc 2023-10-09T06:07:47.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:47 smithi064 ceph-mon[220044]: failed to encode map e436 with expected crc 2023-10-09T06:07:47.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:47 smithi064 ceph-mon[220044]: failed to encode map e436 with expected crc 2023-10-09T06:07:47.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:47 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/531627525' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:07:47.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:47 smithi064 ceph-mon[220044]: pgmap v1073: 119 pgs: 3 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 remapped+peering, 1 active+remapped, 109 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 7.0 KiB/s rd, 49 MiB/s wr, 792 op/s; 6759/159903 objects degraded (4.227%); 3242/159903 objects misplaced (2.027%); 125 KiB/s, 2 keys/s, 1 objects/s recovering 2023-10-09T06:07:47.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:47 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1713739416' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:47.752 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:07:47.772 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:47.795 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:07:47.837 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:07:47.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:47 smithi156 ceph-mon[195101]: failed to encode map e436 with expected crc 2023-10-09T06:07:47.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:47 smithi156 ceph-mon[195101]: failed to encode map e436 with expected crc 2023-10-09T06:07:47.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:47 smithi156 ceph-mon[195101]: failed to encode map e436 with expected crc 2023-10-09T06:07:47.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:47 smithi156 ceph-mon[195101]: failed to encode map e436 with expected crc 2023-10-09T06:07:47.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:47 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/531627525' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:07:47.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:47 smithi156 ceph-mon[195101]: pgmap v1073: 119 pgs: 3 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 remapped+peering, 1 active+remapped, 109 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 7.0 KiB/s rd, 49 MiB/s wr, 792 op/s; 6759/159903 objects degraded (4.227%); 3242/159903 objects misplaced (2.027%); 125 KiB/s, 2 keys/s, 1 objects/s recovering 2023-10-09T06:07:47.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:47 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1713739416' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:47.876 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:07:47.917 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:07:47.927 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:07:47.927 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:07:47.927 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:07:47.927 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:07:47.928 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:07:47.928 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:07:47.928 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:07:47.928 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:07:48.482 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1, 2] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 0, 4] 2023-10-09T06:07:48.482 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:07:48.482 INFO:tasks.thrashosds.thrasher:Killing osd 0, live_osds are [1, 2, 3, 5, 6, 7, 0, 4] 2023-10-09T06:07:48.482 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-10-09T06:07:48.483 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.0 2023-10-09T06:07:48.709 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 35 16 56157 56141 100.191 61.1875 0.0107087 0.00997569 2023-10-09T06:07:48.969 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 06:07:48 smithi064 systemd[1]: Stopping Ceph osd.0 for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T06:07:48.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:48 smithi064 ceph-mon[212750]: Health check failed: Degraded data redundancy: 6759/159903 objects degraded (4.227%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:07:48.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:48 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:48.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:48 smithi064 ceph-mon[212750]: failed to encode map e437 with expected crc 2023-10-09T06:07:48.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:48 smithi064 ceph-mon[212750]: failed to encode map e437 with expected crc 2023-10-09T06:07:48.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:48 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/531627525' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:07:48.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:48 smithi064 ceph-mon[212750]: osdmap e437: 8 total, 8 up, 8 in 2023-10-09T06:07:48.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:48 smithi064 ceph-mon[212750]: failed to encode map e437 with expected crc 2023-10-09T06:07:48.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:48 smithi064 ceph-mon[220044]: Health check failed: Degraded data redundancy: 6759/159903 objects degraded (4.227%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:07:48.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:48 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:48.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:48 smithi064 ceph-mon[220044]: failed to encode map e437 with expected crc 2023-10-09T06:07:48.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:48 smithi064 ceph-mon[220044]: failed to encode map e437 with expected crc 2023-10-09T06:07:48.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:48 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/531627525' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:07:48.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:48 smithi064 ceph-mon[220044]: osdmap e437: 8 total, 8 up, 8 in 2023-10-09T06:07:48.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:48 smithi064 ceph-mon[220044]: failed to encode map e437 with expected crc 2023-10-09T06:07:49.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:48 smithi156 ceph-mon[195101]: Health check failed: Degraded data redundancy: 6759/159903 objects degraded (4.227%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:07:49.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:48 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:49.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:48 smithi156 ceph-mon[195101]: failed to encode map e437 with expected crc 2023-10-09T06:07:49.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:48 smithi156 ceph-mon[195101]: failed to encode map e437 with expected crc 2023-10-09T06:07:49.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:48 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/531627525' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:07:49.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:48 smithi156 ceph-mon[195101]: osdmap e437: 8 total, 8 up, 8 in 2023-10-09T06:07:49.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:48 smithi156 ceph-mon[195101]: failed to encode map e437 with expected crc 2023-10-09T06:07:49.618 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:49.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:49.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:49.709 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 36 16 57060 57044 98.9756 56.4375 0.0239976 0.0100987 2023-10-09T06:07:49.724 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:49.761 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:49.808 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:49.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:49.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:49.915 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:49.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:49 smithi064 ceph-mon[212750]: pgmap v1075: 119 pgs: 3 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 111 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 6.4 KiB/s rd, 44 MiB/s wr, 707 op/s; 6759/160581 objects degraded (4.209%); 3242/160581 objects misplaced (2.019%); 115 KiB/s, 2 keys/s, 1 objects/s recovering 2023-10-09T06:07:49.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:49 smithi064 ceph-mon[220044]: pgmap v1075: 119 pgs: 3 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 111 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 6.4 KiB/s rd, 44 MiB/s wr, 707 op/s; 6759/160581 objects degraded (4.209%); 3242/160581 objects misplaced (2.019%); 115 KiB/s, 2 keys/s, 1 objects/s recovering 2023-10-09T06:07:49.982 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:50.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:49 smithi156 ceph-mon[195101]: pgmap v1075: 119 pgs: 3 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 111 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 6.4 KiB/s rd, 44 MiB/s wr, 707 op/s; 6759/160581 objects degraded (4.209%); 3242/160581 objects misplaced (2.019%); 115 KiB/s, 2 keys/s, 1 objects/s recovering 2023-10-09T06:07:50.356 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:50.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:50.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:50.463 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:50.545 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:50.659 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 06:07:50 smithi064 conmon[297966]: 2023-10-09T06:07:50.514+0000 7fb961a83700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:07:50.659 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 06:07:50 smithi064 conmon[297966]: 2023-10-09T06:07:50.514+0000 7fb961a83700 -1 osd.0 437 *** Got signal Terminated *** 2023-10-09T06:07:50.659 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 06:07:50 smithi064 conmon[297966]: 2023-10-09T06:07:50.514+0000 7fb961a83700 -1 osd.0 437 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:07:50.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:50 smithi064 ceph-mon[212750]: osd.0 marked itself down and dead 2023-10-09T06:07:50.709 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 37 16 58040 58024 97.9564 61.25 0.0227135 0.0102021 2023-10-09T06:07:50.763 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:50.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:50.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:50.830 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:07:50.830 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T06:07:50.830 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T06:07:50.831 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T06:07:50.831 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:07:50.870 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:50.902 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:50.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:50 smithi064 ceph-mon[220044]: osd.0 marked itself down and dead 2023-10-09T06:07:51.058 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:51.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:51.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:51.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:50 smithi156 ceph-mon[195101]: osd.0 marked itself down and dead 2023-10-09T06:07:51.162 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:51.189 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:51.232 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:51.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:51.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:51.337 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:51.367 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:51.569 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:51.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:51.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:51.633 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:07:51.673 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:51.709 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 38 16 58095 58079 95.4702 3.4375 0.00279804 0.010208 2023-10-09T06:07:51.728 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:51.911 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 06:07:51 smithi064 bash[519824]: 866dc7ecde6084c1e76264c4c155a65c17c31a9cffdadde94c346398acca4379 2023-10-09T06:07:51.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:51 smithi064 ceph-mon[212750]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:07:51.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:51 smithi064 ceph-mon[212750]: osdmap e438: 8 total, 7 up, 8 in 2023-10-09T06:07:51.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:51 smithi064 ceph-mon[212750]: failed to encode map e438 with expected crc 2023-10-09T06:07:51.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:51 smithi064 ceph-mon[212750]: pgmap v1077: 119 pgs: 1 active+recovering+undersized+remapped, 8 stale+active+clean, 2 peering, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s; 6759/168057 objects degraded (4.022%); 3071/168057 objects misplaced (1.827%); 2.0 MiB/s, 1 keys/s, 32 objects/s recovering 2023-10-09T06:07:51.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:51 smithi064 ceph-mon[220044]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:07:51.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:51 smithi064 ceph-mon[220044]: osdmap e438: 8 total, 7 up, 8 in 2023-10-09T06:07:51.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:51 smithi064 ceph-mon[220044]: failed to encode map e438 with expected crc 2023-10-09T06:07:51.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:51 smithi064 ceph-mon[220044]: pgmap v1077: 119 pgs: 1 active+recovering+undersized+remapped, 8 stale+active+clean, 2 peering, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s; 6759/168057 objects degraded (4.022%); 3071/168057 objects misplaced (1.827%); 2.0 MiB/s, 1 keys/s, 32 objects/s recovering 2023-10-09T06:07:52.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:51 smithi156 ceph-mon[195101]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:07:52.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:51 smithi156 ceph-mon[195101]: osdmap e438: 8 total, 7 up, 8 in 2023-10-09T06:07:52.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:51 smithi156 ceph-mon[195101]: failed to encode map e438 with expected crc 2023-10-09T06:07:52.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:51 smithi156 ceph-mon[195101]: pgmap v1077: 119 pgs: 1 active+recovering+undersized+remapped, 8 stale+active+clean, 2 peering, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s; 6759/168057 objects degraded (4.022%); 3071/168057 objects misplaced (1.827%); 2.0 MiB/s, 1 keys/s, 32 objects/s recovering 2023-10-09T06:07:52.404 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:07:52.676 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:52.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:52.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:52.709 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 39 16 58951 58935 94.3944 53.5 0.00508152 0.010589 2023-10-09T06:07:52.785 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:52.875 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:52.953 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:52 smithi064 ceph-mon[220044]: failed to encode map e439 with expected crc 2023-10-09T06:07:52.953 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:52 smithi064 ceph-mon[220044]: osdmap e439: 8 total, 7 up, 8 in 2023-10-09T06:07:52.953 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:52 smithi064 ceph-mon[220044]: failed to encode map e439 with expected crc 2023-10-09T06:07:52.954 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:52 smithi064 ceph-mon[220044]: failed to encode map e439 with expected crc 2023-10-09T06:07:52.954 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:52 smithi064 ceph-mon[220044]: failed to encode map e439 with expected crc 2023-10-09T06:07:52.954 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:52 smithi064 ceph-mon[220044]: failed to encode map e439 with expected crc 2023-10-09T06:07:52.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:52 smithi064 ceph-mon[212750]: failed to encode map e439 with expected crc 2023-10-09T06:07:52.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:52 smithi064 ceph-mon[212750]: osdmap e439: 8 total, 7 up, 8 in 2023-10-09T06:07:52.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:52 smithi064 ceph-mon[212750]: failed to encode map e439 with expected crc 2023-10-09T06:07:52.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:52 smithi064 ceph-mon[212750]: failed to encode map e439 with expected crc 2023-10-09T06:07:52.955 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:52 smithi064 ceph-mon[212750]: failed to encode map e439 with expected crc 2023-10-09T06:07:52.955 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:52 smithi064 ceph-mon[212750]: failed to encode map e439 with expected crc 2023-10-09T06:07:52.955 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:07:53.059 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:53.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:53.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:53.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:52 smithi156 ceph-mon[195101]: failed to encode map e439 with expected crc 2023-10-09T06:07:53.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:52 smithi156 ceph-mon[195101]: osdmap e439: 8 total, 7 up, 8 in 2023-10-09T06:07:53.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:52 smithi156 ceph-mon[195101]: failed to encode map e439 with expected crc 2023-10-09T06:07:53.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:52 smithi156 ceph-mon[195101]: failed to encode map e439 with expected crc 2023-10-09T06:07:53.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:52 smithi156 ceph-mon[195101]: failed to encode map e439 with expected crc 2023-10-09T06:07:53.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:52 smithi156 ceph-mon[195101]: failed to encode map e439 with expected crc 2023-10-09T06:07:53.174 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:53.204 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:53.258 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:53.259 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:53.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:53.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:53.327 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:53.366 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:53.433 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:53.594 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:07:53.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:53.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:07:53.706 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:53.710 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-10-09T06:07:53.710326+0000 min lat: 0.00120925 max lat: 2.19099 avg lat: 0.0107077 2023-10-09T06:07:53.710 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:07:53.710 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 40 16 59796 59780 93.3552 52.8125 0.012725 0.0107077 2023-10-09T06:07:53.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:53 smithi064 ceph-mon[212750]: pgmap v1079: 119 pgs: 1 active+recovering+undersized+remapped, 8 stale+active+clean, 10 peering, 2 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 45 MiB/s wr, 727 op/s; 6759/173004 objects degraded (3.907%); 3071/173004 objects misplaced (1.775%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:07:53.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:53 smithi064 ceph-mon[220044]: pgmap v1079: 119 pgs: 1 active+recovering+undersized+remapped, 8 stale+active+clean, 10 peering, 2 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 45 MiB/s wr, 727 op/s; 6759/173004 objects degraded (3.907%); 3071/173004 objects misplaced (1.775%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:07:53.751 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:53.814 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:53.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:53.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:53.924 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:53.959 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:54.025 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:54.034 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:53 smithi156 ceph-mon[195101]: pgmap v1079: 119 pgs: 1 active+recovering+undersized+remapped, 8 stale+active+clean, 10 peering, 2 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 45 MiB/s wr, 727 op/s; 6759/173004 objects degraded (3.907%); 3071/173004 objects misplaced (1.775%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:07:54.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:54.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:54.135 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:54.235 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:54.710 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 41 16 60883 60867 92.7354 67.9375 0.00956837 0.0107785 2023-10-09T06:07:54.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:54 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 6759/173004 objects degraded (3.907%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:07:54.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:54 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 6759/173004 objects degraded (3.907%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:07:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:54 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 6759/173004 objects degraded (3.907%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:07:55.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:07:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:07:55] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.43.0" 2023-10-09T06:07:55.531 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:55.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:55.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:55.636 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:55.713 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 42 16 62010 61994 92.1967 70.4375 0.0202899 0.0108405 2023-10-09T06:07:55.716 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:55.856 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:55 smithi064 ceph-mon[220044]: pgmap v1080: 119 pgs: 1 active+recovering+undersized+remapped, 12 active+undersized, 5 stale+active+clean, 10 peering, 7 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 5.6 KiB/s rd, 48 MiB/s wr, 770 op/s; 14422/176682 objects degraded (8.163%); 2825/176682 objects misplaced (1.599%); 3.6 MiB/s, 55 objects/s recovering 2023-10-09T06:07:55.858 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:55 smithi064 ceph-mon[212750]: pgmap v1080: 119 pgs: 1 active+recovering+undersized+remapped, 12 active+undersized, 5 stale+active+clean, 10 peering, 7 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 5.6 KiB/s rd, 48 MiB/s wr, 770 op/s; 14422/176682 objects degraded (8.163%); 2825/176682 objects misplaced (1.599%); 3.6 MiB/s, 55 objects/s recovering 2023-10-09T06:07:55.893 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:56.002 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:56.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:56.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:56.109 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:07:56.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:55 smithi156 ceph-mon[195101]: pgmap v1080: 119 pgs: 1 active+recovering+undersized+remapped, 12 active+undersized, 5 stale+active+clean, 10 peering, 7 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 5.6 KiB/s rd, 48 MiB/s wr, 770 op/s; 14422/176682 objects degraded (8.163%); 2825/176682 objects misplaced (1.599%); 3.6 MiB/s, 55 objects/s recovering 2023-10-09T06:07:56.151 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:56.210 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:07:56.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:56.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:07:56.316 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:07:56.353 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:56.405 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:07:56.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:56.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:07:56.512 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:56.548 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:56.615 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:56.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:56.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:56.717 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 43 16 62940 62924 91.3972 58.125 0.0245317 0.0109349 2023-10-09T06:07:56.723 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:07:56.817 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:57.260 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:07:57.261 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:07:57.261 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:07:57.261 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:07:57.261 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:07:57.261 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:07:57.261 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:07:57.262 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-10-09T06:07:57.262 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:07:57.262 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:07:57.262 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:07:57.262 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:07:57.262 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:07:57.262 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:07:57.262 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:07:57.282 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:57 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3822930647' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:57.283 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:57 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3822930647' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:57.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:57 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3822930647' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:07:57.719 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 44 16 63827 63811 90.5752 55.4375 0.00591721 0.0110347 2023-10-09T06:07:58.071 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:07:58.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:58 smithi156 ceph-mon[195101]: pgmap v1081: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 5.1 KiB/s rd, 64 MiB/s wr, 1.02k op/s; 22260/185103 objects degraded (12.026%); 2825/185103 objects misplaced (1.526%); 3.3 MiB/s, 50 objects/s recovering 2023-10-09T06:07:58.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:58 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/555689461' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:58.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:58 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:58.390 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:58 smithi064 ceph-mon[212750]: pgmap v1081: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 5.1 KiB/s rd, 64 MiB/s wr, 1.02k op/s; 22260/185103 objects degraded (12.026%); 2825/185103 objects misplaced (1.526%); 3.3 MiB/s, 50 objects/s recovering 2023-10-09T06:07:58.390 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:58 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/555689461' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:58.390 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:58 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:58.391 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:58 smithi064 ceph-mon[220044]: pgmap v1081: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 5.1 KiB/s rd, 64 MiB/s wr, 1.02k op/s; 22260/185103 objects degraded (12.026%); 2825/185103 objects misplaced (1.526%); 3.3 MiB/s, 50 objects/s recovering 2023-10-09T06:07:58.391 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:58 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/555689461' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:58.391 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:58 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:07:58.400 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:07:58.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:58.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:07:58.415 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.0.service' 2023-10-09T06:07:58.507 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:07:58.570 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:58.576 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:07:58.576 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-10-09T06:07:58.719 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 45 16 64772 64756 89.8753 59.0625 0.0203889 0.0111207 2023-10-09T06:07:58.766 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:07:58.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:58.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:07:58.870 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:58.896 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:58.971 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:58.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:58.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:59.077 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:07:59.102 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:59.142 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:07:59.146 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:59 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:59.146 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:59 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:07:59.146 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:59 smithi156 ceph-mon[195101]: osdmap e440: 8 total, 7 up, 8 in 2023-10-09T06:07:59.146 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:59 smithi156 ceph-mon[195101]: failed to encode map e440 with expected crc 2023-10-09T06:07:59.146 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:59 smithi156 ceph-mon[195101]: failed to encode map e440 with expected crc 2023-10-09T06:07:59.146 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:59 smithi156 ceph-mon[195101]: failed to encode map e440 with expected crc 2023-10-09T06:07:59.146 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:59 smithi156 ceph-mon[195101]: failed to encode map e440 with expected crc 2023-10-09T06:07:59.146 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:59 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:59.147 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:07:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:59.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:59.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:07:59.248 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:07:59.295 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:59.372 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:07:59.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:07:59.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:07:59.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:59 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:59.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:59 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:07:59.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:59 smithi064 ceph-mon[212750]: osdmap e440: 8 total, 7 up, 8 in 2023-10-09T06:07:59.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:59 smithi064 ceph-mon[212750]: failed to encode map e440 with expected crc 2023-10-09T06:07:59.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:59 smithi064 ceph-mon[212750]: failed to encode map e440 with expected crc 2023-10-09T06:07:59.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:59 smithi064 ceph-mon[212750]: failed to encode map e440 with expected crc 2023-10-09T06:07:59.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:59 smithi064 ceph-mon[212750]: failed to encode map e440 with expected crc 2023-10-09T06:07:59.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:59 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:59.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:07:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:59.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:59 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:07:59.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:59 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:07:59.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:59 smithi064 ceph-mon[220044]: osdmap e440: 8 total, 7 up, 8 in 2023-10-09T06:07:59.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:59 smithi064 ceph-mon[220044]: failed to encode map e440 with expected crc 2023-10-09T06:07:59.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:59 smithi064 ceph-mon[220044]: failed to encode map e440 with expected crc 2023-10-09T06:07:59.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:59 smithi064 ceph-mon[220044]: failed to encode map e440 with expected crc 2023-10-09T06:07:59.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:59 smithi064 ceph-mon[220044]: failed to encode map e440 with expected crc 2023-10-09T06:07:59.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:59 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:07:59.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:07:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:59.479 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:07:59.541 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:07:59.720 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 46 16 65447 65431 88.8391 42.1875 0.0113097 0.0112515 2023-10-09T06:08:00.173 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:00.177 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T06:08:00.178 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T06:08:00.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:00.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:00.296 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:00.326 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:00 smithi156 ceph-mon[195101]: pgmap v1083: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 5.1 KiB/s rd, 47 MiB/s wr, 755 op/s; 22589/186090 objects degraded (12.139%); 2825/186090 objects misplaced (1.518%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T06:08:00.326 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:00 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 22589/186090 objects degraded (12.139%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:00.337 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:00.422 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:00.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:00.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:00.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:00 smithi064 ceph-mon[212750]: pgmap v1083: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 5.1 KiB/s rd, 47 MiB/s wr, 755 op/s; 22589/186090 objects degraded (12.139%); 2825/186090 objects misplaced (1.518%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T06:08:00.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:00 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 22589/186090 objects degraded (12.139%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:00.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:00 smithi064 ceph-mon[220044]: pgmap v1083: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 5.1 KiB/s rd, 47 MiB/s wr, 755 op/s; 22589/186090 objects degraded (12.139%); 2825/186090 objects misplaced (1.518%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T06:08:00.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:00 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 22589/186090 objects degraded (12.139%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:00.527 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:00.599 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:00.719 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 47 16 66491 66475 88.3374 65.25 0.0149264 0.0113148 2023-10-09T06:08:00.722 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:00.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:00.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:00.822 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:08:00.822 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:08:00.827 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:00.908 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:01.082 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:01.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:01.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:01.194 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:01.270 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:01.383 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:01.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:01.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:01.490 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:01.720 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 48 16 67551 67535 87.8774 66.25 0.0205354 0.0113745 2023-10-09T06:08:01.991 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:08:02.047 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:02.093 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:08:02.120 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:02.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:02.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:02.179 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:08:02.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:02 smithi064 ceph-mon[212750]: pgmap v1084: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 4.4 KiB/s rd, 47 MiB/s wr, 752 op/s; 22854/193767 objects degraded (11.795%); 2617/193767 objects misplaced (1.351%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T06:08:02.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:02 smithi064 ceph-mon[220044]: pgmap v1084: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 4.4 KiB/s rd, 47 MiB/s wr, 752 op/s; 22854/193767 objects degraded (11.795%); 2617/193767 objects misplaced (1.351%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T06:08:02.229 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:02.338 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:02.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:02 smithi156 ceph-mon[195101]: pgmap v1084: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 4.4 KiB/s rd, 47 MiB/s wr, 752 op/s; 22854/193767 objects degraded (11.795%); 2617/193767 objects misplaced (1.351%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T06:08:02.520 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:02.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:02.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:02.629 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:08:02.709 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:02.720 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 49 16 68458 68442 87.2404 56.6875 0.00906057 0.0114569 2023-10-09T06:08:02.764 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:08:02.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:02.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:08:02.827 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:02.868 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:02.885 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:02.902 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:02.968 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:02.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:02.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:03.070 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:03.079 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:03.148 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:03.346 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:03.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:03.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:03.461 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:03.495 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:03.560 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:03.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:03.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:03.577 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1, 2] out_osds: [] dead_osds: [0] live_osds: [1, 2, 3, 5, 6, 7, 4] 2023-10-09T06:08:03.577 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:08:03.578 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T06:08:03.578 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:08:03.667 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:03.696 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:03.726 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 50 16 69354 69338 86.6064 56 0.00983184 0.0115397 2023-10-09T06:08:03.755 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:03.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:03.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:03.863 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:03.897 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:03.957 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:03.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:03.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:04.064 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:04.129 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:04.726 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 51 16 70157 70141 85.8927 50.1875 0.00351455 0.0116374 2023-10-09T06:08:04.744 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:04 smithi064 ceph-mon[220044]: pgmap v1085: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 4.1 KiB/s rd, 54 MiB/s wr, 860 op/s; 23411/198750 objects degraded (11.779%); 2617/198750 objects misplaced (1.317%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:08:04.745 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:04 smithi064 ceph-mon[212750]: pgmap v1085: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 4.1 KiB/s rd, 54 MiB/s wr, 860 op/s; 23411/198750 objects degraded (11.779%); 2617/198750 objects misplaced (1.317%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:08:04.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:04 smithi156 ceph-mon[195101]: pgmap v1085: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 4.1 KiB/s rd, 54 MiB/s wr, 860 op/s; 23411/198750 objects degraded (11.779%); 2617/198750 objects misplaced (1.317%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:08:04.909 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:04.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:04.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:05.014 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:05.092 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:05.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:05 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 23947/202983 objects degraded (11.798%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:05.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:08:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:08:05] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.43.0" 2023-10-09T06:08:05.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:05 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 23947/202983 objects degraded (11.798%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:05.732 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 52 16 71036 71020 85.2881 54.9375 0.0126112 0.0117197 2023-10-09T06:08:05.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:05 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 23947/202983 objects degraded (11.798%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:05.978 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:05.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:05.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:06.091 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:08:06.137 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:06.207 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:08:06.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:06.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:08:06.313 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:06.396 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:06.541 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:06 smithi156 ceph-mon[195101]: pgmap v1086: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s; 23947/202983 objects degraded (11.798%); 2387/202983 objects misplaced (1.176%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:08:06.645 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:06.649 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:06 smithi064 ceph-mon[212750]: pgmap v1086: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s; 23947/202983 objects degraded (11.798%); 2387/202983 objects misplaced (1.176%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:08:06.649 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:06 smithi064 ceph-mon[220044]: pgmap v1086: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s; 23947/202983 objects degraded (11.798%); 2387/202983 objects misplaced (1.176%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:08:06.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:06.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:06.673 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:06.732 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 53 15 71989 71974 84.8042 59.625 0.0199885 0.0117851 2023-10-09T06:08:06.751 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:06.789 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:06.853 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:06.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:06.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:06.959 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:07.049 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:07.122 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:08:07.122 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:08:07.411 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:07.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:07.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:07.516 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:07.547 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:07.603 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:07.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:07.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:07.712 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:07.732 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 54 16 72861 72845 84.2422 54.4375 0.0140882 0.0118644 2023-10-09T06:08:07.787 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:08.225 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:08:08.252 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:08:08.290 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:08:08.329 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:08:08.369 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:08:08.411 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:08:08.451 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:08:08.452 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-10-09T06:08:08.452 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:08:08.452 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:08:08.452 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:08:08.452 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:08:08.452 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:08:08.452 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:08:08.452 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:08:08.567 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:08.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:08.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:08.607 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:08:08.619 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:08:08.679 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:08:08.713 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:08.732 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 55 15 73617 73602 83.5711 47.3125 0.0126797 0.011959 2023-10-09T06:08:08.772 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:08:08.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:08.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:08:08.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:08 smithi156 ceph-mon[195101]: pgmap v1087: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s; 24500/212331 objects degraded (11.539%); 2387/212331 objects misplaced (1.124%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:08:08.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:08 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1583053344' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:08:08.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:08 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:08:08.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:08 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2785557762' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:08.879 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:08:08.931 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:08.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:08 smithi064 ceph-mon[212750]: pgmap v1087: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s; 24500/212331 objects degraded (11.539%); 2387/212331 objects misplaced (1.124%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:08:08.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:08 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1583053344' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:08:08.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:08 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:08:08.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:08 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2785557762' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:08.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:08 smithi064 ceph-mon[220044]: pgmap v1087: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s; 24500/212331 objects degraded (11.539%); 2387/212331 objects misplaced (1.124%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:08:08.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:08 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1583053344' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:08:08.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:08 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:08:08.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:08 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2785557762' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:08.996 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:08:09.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:09.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:08:09.103 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:09.138 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:09.211 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:09.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:09.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:09.320 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:09.387 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:09.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:09 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:08:09.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:09 smithi064 ceph-mon[212750]: failed to encode map e441 with expected crc 2023-10-09T06:08:09.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:09 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:08:09.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:09 smithi064 ceph-mon[212750]: osdmap e441: 8 total, 7 up, 8 in 2023-10-09T06:08:09.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:09 smithi064 ceph-mon[212750]: failed to encode map e441 with expected crc 2023-10-09T06:08:09.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:09 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:08:09.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:09 smithi064 ceph-mon[220044]: failed to encode map e441 with expected crc 2023-10-09T06:08:09.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:09 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:08:09.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:09 smithi064 ceph-mon[220044]: osdmap e441: 8 total, 7 up, 8 in 2023-10-09T06:08:09.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:09 smithi064 ceph-mon[220044]: failed to encode map e441 with expected crc 2023-10-09T06:08:09.732 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 56 16 74467 74451 83.0265 53.0625 0.0251807 0.0120396 2023-10-09T06:08:09.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:09 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:08:09.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:09 smithi156 ceph-mon[195101]: failed to encode map e441 with expected crc 2023-10-09T06:08:09.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:09 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:08:09.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:09 smithi156 ceph-mon[195101]: osdmap e441: 8 total, 7 up, 8 in 2023-10-09T06:08:09.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:09 smithi156 ceph-mon[195101]: failed to encode map e441 with expected crc 2023-10-09T06:08:10.461 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:10.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:10.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:10.576 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:10.616 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:10.622 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:10 smithi064 ceph-mon[212750]: pgmap v1089: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 24759/213108 objects degraded (11.618%); 2387/213108 objects misplaced (1.120%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:08:10.622 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:10 smithi064 ceph-mon[220044]: pgmap v1089: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 24759/213108 objects degraded (11.618%); 2387/213108 objects misplaced (1.120%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:08:10.660 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T06:08:10.660 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T06:08:10.669 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:10.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:10.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:10.732 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 57 16 75348 75332 82.5362 55.0625 0.0116226 0.012109 2023-10-09T06:08:10.779 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:10.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:10 smithi156 ceph-mon[195101]: pgmap v1089: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 24759/213108 objects degraded (11.618%); 2387/213108 objects misplaced (1.120%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:08:10.881 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:11.009 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:11.041 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:11.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:11.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:11.146 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:11.183 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:11.260 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:11.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:11.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:11.369 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:11.475 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:11.506 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:08:11.517 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T06:08:11.669 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:11.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:11.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:11.734 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 58 16 76193 76177 82.0213 52.8125 0.019988 0.012186 2023-10-09T06:08:11.782 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:11.812 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:11.874 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:11.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:11.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:11.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:11 smithi064 ceph-mon[212750]: pgmap v1090: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s; 25007/220383 objects degraded (11.347%); 2158/220383 objects misplaced (0.979%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T06:08:11.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:11 smithi064 ceph-mon[212750]: from='client.46001 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:08:11.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:11 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 25007/220383 objects degraded (11.347%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:11.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:11 smithi064 ceph-mon[220044]: pgmap v1090: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s; 25007/220383 objects degraded (11.347%); 2158/220383 objects misplaced (0.979%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T06:08:11.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:11 smithi064 ceph-mon[220044]: from='client.46001 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:08:11.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:11 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 25007/220383 objects degraded (11.347%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:11.983 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:12.046 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:12.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:11 smithi156 ceph-mon[195101]: pgmap v1090: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s; 25007/220383 objects degraded (11.347%); 2158/220383 objects misplaced (0.979%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T06:08:12.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:11 smithi156 ceph-mon[195101]: from='client.46001 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:08:12.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:11 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 25007/220383 objects degraded (11.347%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:12.173 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:12.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:12.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:12.281 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:12.310 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:12.351 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:12.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:12.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:12.458 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:12.511 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:12.586 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:12.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:12.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:12.662 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:08:12.686 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:08:12.695 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:12.725 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:12.734 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 59 16 76986 76970 81.4714 49.5625 0.0160029 0.0122673 2023-10-09T06:08:12.782 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:12.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:12.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:12.887 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:12.913 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:12.982 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:12.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:12.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:13.090 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:13.162 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:13.287 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1090,"stamp":"2023-10-09T06:08:10.950777+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4813407829,"num_objects":73461,"num_object_clones":0,"num_object_copies":220383,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":25007,"num_objects_misplaced":2158,"num_objects_unfound":0,"num_objects_dirty":73461,"num_whiteouts":0,"num_read":37,"num_read_kb":69,"num_write":73516,"num_write_kb":4700180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1127,"num_bytes_recovered":71897537,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":78483,"ondisk_log_size":78483,"up":314,"acting":311,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19119104,"kb_used_data":13942400,"kb_used_omap":163,"kb_used_meta":5176476,"kb_avail":730842112,"statfs":{"total":767960285184,"available":748382322688,"internally_reserved":0,"allocated":14277017600,"data_stored":14262543037,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":167439,"internal_metadata":5300711921},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":581435392,"num_objects":8872,"num_object_clones":0,"num_object_copies":26616,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2153,"num_objects_misplaced":-459,"num_objects_unfound":0,"num_objects_dirty":8872,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8872,"num_write_kb":567808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":459,"num_bytes_recovered":30081024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.013686"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":718,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-09T06:07:51.699595+0000","last_change":"2023-10-09T06:07:46.599648+0000","last_active":"2023-10-09T06:07:51.699595+0000","last_peered":"2023-10-09T06:07:51.699595+0000","last_clean":"2023-10-09T06:07:51.699595+0000","last_became_active":"2023-10-09T06:07:46.599303+0000","last_became_peered":"2023-10-09T06:07:46.599303+0000","last_unstale":"2023-10-09T06:07:51.699595+0000","last_undegraded":"2023-10-09T06:07:51.699595+0000","last_fullsized":"2023-10-09T06:07:51.699595+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:05.412941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:05.412941+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:45.768743+0000","scrub_duration":0.0061014329999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":509,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.052230+0000","last_change":"2023-10-09T06:07:43.572773+0000","last_active":"2023-10-09T06:07:58.052230+0000","last_peered":"2023-10-09T06:07:58.052230+0000","last_clean":"2023-10-09T06:07:58.052230+0000","last_became_active":"2023-10-09T06:07:43.572299+0000","last_became_peered":"2023-10-09T06:07:43.572299+0000","last_unstale":"2023-10-09T06:07:58.052230+0000","last_undegraded":"2023-10-09T06:07:58.052230+0000","last_fullsized":"2023-10-09T06:07:58.052230+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:00:37.672435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":249,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.048514+0000","last_change":"2023-10-09T06:05:11.451972+0000","last_active":"2023-10-09T06:07:58.048514+0000","last_peered":"2023-10-09T06:07:58.048514+0000","last_clean":"2023-10-09T06:07:58.048514+0000","last_became_active":"2023-10-09T06:05:11.451448+0000","last_became_peered":"2023-10-09T06:05:11.451448+0000","last_unstale":"2023-10-09T06:07:58.048514+0000","last_undegraded":"2023-10-09T06:07:58.048514+0000","last_fullsized":"2023-10-09T06:07:58.048514+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:39.183621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":730,"reported_epoch":440,"state":"active+undersized","last_fresh":"2023-10-09T06:07:58.052647+0000","last_change":"2023-10-09T06:07:51.704972+0000","last_active":"2023-10-09T06:07:58.052647+0000","last_peered":"2023-10-09T06:07:58.052647+0000","last_clean":"2023-10-09T06:07:47.645886+0000","last_became_active":"2023-10-09T06:07:51.704972+0000","last_became_peered":"2023-10-09T06:07:51.704972+0000","last_unstale":"2023-10-09T06:07:58.052647+0000","last_undegraded":"2023-10-09T06:07:58.052647+0000","last_fullsized":"2023-10-09T06:07:51.697866+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:01.468346+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:01.468346+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:23:22.569733+0000","scrub_duration":0.0016258220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":120,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.050450+0000","last_change":"2023-10-09T06:01:54.762060+0000","last_active":"2023-10-09T06:07:58.050450+0000","last_peered":"2023-10-09T06:07:58.050450+0000","last_clean":"2023-10-09T06:07:58.050450+0000","last_became_active":"2023-10-09T06:01:54.761757+0000","last_became_peered":"2023-10-09T06:01:54.761757+0000","last_unstale":"2023-10-09T06:07:58.050450+0000","last_undegraded":"2023-10-09T06:07:58.050450+0000","last_fullsized":"2023-10-09T06:07:58.050450+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:40.294232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":407,"reported_epoch":439,"state":"active+undersized","last_fresh":"2023-10-09T06:07:51.711320+0000","last_change":"2023-10-09T06:07:51.711108+0000","last_active":"2023-10-09T06:07:51.711320+0000","last_peered":"2023-10-09T06:07:51.711320+0000","last_clean":"2023-10-09T06:07:47.641347+0000","last_became_active":"2023-10-09T06:07:51.711108+0000","last_became_peered":"2023-10-09T06:07:51.711108+0000","last_unstale":"2023-10-09T06:07:51.711320+0000","last_undegraded":"2023-10-09T06:07:51.711320+0000","last_fullsized":"2023-10-09T06:07:51.700763+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:42.556462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":685,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.051899+0000","last_change":"2023-10-09T06:02:43.293521+0000","last_active":"2023-10-09T06:07:58.051899+0000","last_peered":"2023-10-09T06:07:58.051899+0000","last_clean":"2023-10-09T06:07:58.051899+0000","last_became_active":"2023-10-09T06:01:57.803642+0000","last_became_peered":"2023-10-09T06:01:57.803642+0000","last_unstale":"2023-10-09T06:07:58.051899+0000","last_undegraded":"2023-10-09T06:07:58.051899+0000","last_fullsized":"2023-10-09T06:07:58.051899+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.293426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.293426+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:45.626761+0000","scrub_duration":0.001587946,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":222,"reported_epoch":440,"state":"active+undersized","last_fresh":"2023-10-09T06:07:58.045191+0000","last_change":"2023-10-09T06:07:51.721718+0000","last_active":"2023-10-09T06:07:58.045191+0000","last_peered":"2023-10-09T06:07:58.045191+0000","last_clean":"2023-10-09T06:07:47.627124+0000","last_became_active":"2023-10-09T06:07:51.721718+0000","last_became_peered":"2023-10-09T06:07:51.721718+0000","last_unstale":"2023-10-09T06:07:58.045191+0000","last_undegraded":"2023-10-09T06:07:58.045191+0000","last_fullsized":"2023-10-09T06:07:51.700492+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:30:49.288144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":477,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.063788+0000","last_change":"2023-10-09T06:01:54.757269+0000","last_active":"2023-10-09T06:07:58.063788+0000","last_peered":"2023-10-09T06:07:58.063788+0000","last_clean":"2023-10-09T06:07:58.063788+0000","last_became_active":"2023-10-09T06:01:54.756894+0000","last_became_peered":"2023-10-09T06:01:54.756894+0000","last_unstale":"2023-10-09T06:07:58.063788+0000","last_undegraded":"2023-10-09T06:07:58.063788+0000","last_fullsized":"2023-10-09T06:07:58.063788+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:45.212821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":759,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-09T06:07:51.700047+0000","last_change":"2023-10-09T06:01:57.804160+0000","last_active":"2023-10-09T06:07:51.700047+0000","last_peered":"2023-10-09T06:07:51.700047+0000","last_clean":"2023-10-09T06:07:51.700047+0000","last_became_active":"2023-10-09T06:01:57.803424+0000","last_became_peered":"2023-10-09T06:01:57.803424+0000","last_unstale":"2023-10-09T06:07:51.700047+0000","last_undegraded":"2023-10-09T06:07:51.700047+0000","last_fullsized":"2023-10-09T06:07:51.700047+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.384407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.384407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:42.857346+0000","scrub_duration":0.0040651960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":327,"reported_epoch":440,"state":"active+undersized","last_fresh":"2023-10-09T06:07:58.049538+0000","last_change":"2023-10-09T06:07:51.710005+0000","last_active":"2023-10-09T06:07:58.049538+0000","last_peered":"2023-10-09T06:07:58.049538+0000","last_clean":"2023-10-09T06:07:47.627648+0000","last_became_active":"2023-10-09T06:07:51.710005+0000","last_became_peered":"2023-10-09T06:07:51.710005+0000","last_unstale":"2023-10-09T06:07:58.049538+0000","last_undegraded":"2023-10-09T06:07:58.049538+0000","last_fullsized":"2023-10-09T06:07:51.703972+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:20.319164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":258,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.045269+0000","last_change":"2023-10-09T06:06:01.067544+0000","last_active":"2023-10-09T06:07:58.045269+0000","last_peered":"2023-10-09T06:07:58.045269+0000","last_clean":"2023-10-09T06:07:58.045269+0000","last_became_active":"2023-10-09T06:06:01.067188+0000","last_became_peered":"2023-10-09T06:06:01.067188+0000","last_unstale":"2023-10-09T06:07:58.045269+0000","last_undegraded":"2023-10-09T06:07:58.045269+0000","last_fullsized":"2023-10-09T06:07:58.045269+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:43.464186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":511,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-09T06:07:51.694723+0000","last_change":"2023-10-09T06:07:46.594860+0000","last_active":"2023-10-09T06:07:51.694723+0000","last_peered":"2023-10-09T06:07:51.694723+0000","last_clean":"2023-10-09T06:07:51.694723+0000","last_became_active":"2023-10-09T06:07:46.594424+0000","last_became_peered":"2023-10-09T06:07:46.594424+0000","last_unstale":"2023-10-09T06:07:51.694723+0000","last_undegraded":"2023-10-09T06:07:51.694723+0000","last_fullsized":"2023-10-09T06:07:51.694723+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:20.742561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:20.742561+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":489,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-09T06:07:51.700839+0000","last_change":"2023-10-09T06:01:54.766344+0000","last_active":"2023-10-09T06:07:51.700839+0000","last_peered":"2023-10-09T06:07:51.700839+0000","last_clean":"2023-10-09T06:07:51.700839+0000","last_became_active":"2023-10-09T06:01:54.766086+0000","last_became_peered":"2023-10-09T06:01:54.766086+0000","last_unstale":"2023-10-09T06:07:51.700839+0000","last_undegraded":"2023-10-09T06:07:51.700839+0000","last_fullsized":"2023-10-09T06:07:51.700839+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:47.163700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":389,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:59.847616+0000","last_change":"2023-10-09T05:53:59.185702+0000","last_active":"2023-10-09T06:07:59.847616+0000","last_peered":"2023-10-09T06:07:59.847616+0000","last_clean":"2023-10-09T06:07:59.847616+0000","last_became_active":"2023-10-09T05:53:59.185373+0000","last_became_peered":"2023-10-09T05:53:59.185373+0000","last_unstale":"2023-10-09T06:07:59.847616+0000","last_undegraded":"2023-10-09T06:07:59.847616+0000","last_fullsized":"2023-10-09T06:07:59.847616+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:50.756967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":865,"reported_epoch":440,"state":"active+undersized","last_fresh":"2023-10-09T06:07:58.063951+0000","last_change":"2023-10-09T06:07:51.708726+0000","last_active":"2023-10-09T06:07:58.063951+0000","last_peered":"2023-10-09T06:07:58.063951+0000","last_clean":"2023-10-09T06:07:47.656725+0000","last_became_active":"2023-10-09T06:07:51.708726+0000","last_became_peered":"2023-10-09T06:07:51.708726+0000","last_unstale":"2023-10-09T06:07:58.063951+0000","last_undegraded":"2023-10-09T06:07:58.063951+0000","last_fullsized":"2023-10-09T06:07:51.704451+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:48.374432+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:48.374432+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:55:02.953562+0000","scrub_duration":0.001678585,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":297,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-09T06:07:51.700642+0000","last_change":"2023-10-09T06:01:22.337444+0000","last_active":"2023-10-09T06:07:51.700642+0000","last_peered":"2023-10-09T06:07:51.700642+0000","last_clean":"2023-10-09T06:07:51.700642+0000","last_became_active":"2023-10-09T06:01:22.336348+0000","last_became_peered":"2023-10-09T06:01:22.336348+0000","last_unstale":"2023-10-09T06:07:51.700642+0000","last_undegraded":"2023-10-09T06:07:51.700642+0000","last_fullsized":"2023-10-09T06:07:51.700642+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:43:11.149649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":277,"reported_epoch":440,"state":"active+undersized","last_fresh":"2023-10-09T06:07:58.057936+0000","last_change":"2023-10-09T06:07:51.709718+0000","last_active":"2023-10-09T06:07:58.057936+0000","last_peered":"2023-10-09T06:07:58.057936+0000","last_clean":"2023-10-09T06:04:59.508425+0000","last_became_active":"2023-10-09T06:07:51.709718+0000","last_became_peered":"2023-10-09T06:07:51.709718+0000","last_unstale":"2023-10-09T06:07:58.057936+0000","last_undegraded":"2023-10-09T06:07:58.057936+0000","last_fullsized":"2023-10-09T06:07:51.702207+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:24.493938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":601,"reported_epoch":440,"state":"active+undersized","last_fresh":"2023-10-09T06:07:58.052568+0000","last_change":"2023-10-09T06:07:51.709216+0000","last_active":"2023-10-09T06:07:58.052568+0000","last_peered":"2023-10-09T06:07:58.052568+0000","last_clean":"2023-10-09T06:07:47.646442+0000","last_became_active":"2023-10-09T06:07:51.709216+0000","last_became_peered":"2023-10-09T06:07:51.709216+0000","last_unstale":"2023-10-09T06:07:58.052568+0000","last_undegraded":"2023-10-09T06:07:58.052568+0000","last_fullsized":"2023-10-09T06:07:51.698249+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.339121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.339121+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:20.507565+0000","scrub_duration":0.010474344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":435,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.052344+0000","last_change":"2023-10-09T06:01:22.335608+0000","last_active":"2023-10-09T06:07:58.052344+0000","last_peered":"2023-10-09T06:07:58.052344+0000","last_clean":"2023-10-09T06:07:58.052344+0000","last_became_active":"2023-10-09T06:01:22.335296+0000","last_became_peered":"2023-10-09T06:01:22.335296+0000","last_unstale":"2023-10-09T06:07:58.052344+0000","last_undegraded":"2023-10-09T06:07:58.052344+0000","last_fullsized":"2023-10-09T06:07:58.052344+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:21.843341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":453,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.052959+0000","last_change":"2023-10-09T06:01:54.764916+0000","last_active":"2023-10-09T06:07:58.052959+0000","last_peered":"2023-10-09T06:07:58.052959+0000","last_clean":"2023-10-09T06:07:58.052959+0000","last_became_active":"2023-10-09T06:01:54.764705+0000","last_became_peered":"2023-10-09T06:01:54.764705+0000","last_unstale":"2023-10-09T06:07:58.052959+0000","last_undegraded":"2023-10-09T06:07:58.052959+0000","last_fullsized":"2023-10-09T06:07:58.052959+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:12.527719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":640,"reported_epoch":440,"state":"active+undersized","last_fresh":"2023-10-09T06:07:58.045586+0000","last_change":"2023-10-09T06:07:51.721604+0000","last_active":"2023-10-09T06:07:58.045586+0000","last_peered":"2023-10-09T06:07:58.045586+0000","last_clean":"2023-10-09T06:07:47.629454+0000","last_became_active":"2023-10-09T06:07:51.721604+0000","last_became_peered":"2023-10-09T06:07:51.721604+0000","last_unstale":"2023-10-09T06:07:58.045586+0000","last_undegraded":"2023-10-09T06:07:58.045586+0000","last_fullsized":"2023-10-09T06:07:51.701437+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:42.900876+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:42.900876+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:19.634753+0000","scrub_duration":0.0014928649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":331,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-09T06:07:51.700592+0000","last_change":"2023-10-09T06:07:43.585040+0000","last_active":"2023-10-09T06:07:51.700592+0000","last_peered":"2023-10-09T06:07:51.700592+0000","last_clean":"2023-10-09T06:07:51.700592+0000","last_became_active":"2023-10-09T06:07:43.576991+0000","last_became_peered":"2023-10-09T06:07:43.576991+0000","last_unstale":"2023-10-09T06:07:51.700592+0000","last_undegraded":"2023-10-09T06:07:51.700592+0000","last_fullsized":"2023-10-09T06:07:51.700592+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:34.279182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":484,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:08:04.852265+0000","last_change":"2023-10-09T06:01:54.761558+0000","last_active":"2023-10-09T06:08:04.852265+0000","last_peered":"2023-10-09T06:08:04.852265+0000","last_clean":"2023-10-09T06:08:04.852265+0000","last_became_active":"2023-10-09T06:01:54.761251+0000","last_became_peered":"2023-10-09T06:01:54.761251+0000","last_unstale":"2023-10-09T06:08:04.852265+0000","last_undegraded":"2023-10-09T06:08:04.852265+0000","last_fullsized":"2023-10-09T06:08:04.852265+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:53.245267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":883,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-09T06:07:51.700033+0000","last_change":"2023-10-09T06:06:03.453188+0000","last_active":"2023-10-09T06:07:51.700033+0000","last_peered":"2023-10-09T06:07:51.700033+0000","last_clean":"2023-10-09T06:07:51.700033+0000","last_became_active":"2023-10-09T06:06:03.452853+0000","last_became_peered":"2023-10-09T06:06:03.452853+0000","last_unstale":"2023-10-09T06:07:51.700033+0000","last_undegraded":"2023-10-09T06:07:51.700033+0000","last_fullsized":"2023-10-09T06:07:51.700033+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:26:21.994828+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":372,"reported_epoch":439,"state":"active+undersized","last_fresh":"2023-10-09T06:07:51.715646+0000","last_change":"2023-10-09T06:07:51.715418+0000","last_active":"2023-10-09T06:07:51.715646+0000","last_peered":"2023-10-09T06:07:51.715646+0000","last_clean":"2023-10-09T06:07:47.641228+0000","last_became_active":"2023-10-09T06:07:51.715418+0000","last_became_peered":"2023-10-09T06:07:51.715418+0000","last_unstale":"2023-10-09T06:07:51.715646+0000","last_undegraded":"2023-10-09T06:07:51.715646+0000","last_fullsized":"2023-10-09T06:07:51.700165+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:19.287268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":373,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-09T06:07:51.700496+0000","last_change":"2023-10-09T06:06:01.071454+0000","last_active":"2023-10-09T06:07:51.700496+0000","last_peered":"2023-10-09T06:07:51.700496+0000","last_clean":"2023-10-09T06:07:51.700496+0000","last_became_active":"2023-10-09T06:06:01.071137+0000","last_became_peered":"2023-10-09T06:06:01.071137+0000","last_unstale":"2023-10-09T06:07:51.700496+0000","last_undegraded":"2023-10-09T06:07:51.700496+0000","last_fullsized":"2023-10-09T06:07:51.700496+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:58:52.761376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":644,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.051043+0000","last_change":"2023-10-09T06:07:06.561922+0000","last_active":"2023-10-09T06:07:58.051043+0000","last_peered":"2023-10-09T06:07:58.051043+0000","last_clean":"2023-10-09T06:07:58.051043+0000","last_became_active":"2023-10-09T06:07:06.561625+0000","last_became_peered":"2023-10-09T06:07:06.561625+0000","last_unstale":"2023-10-09T06:07:58.051043+0000","last_undegraded":"2023-10-09T06:07:58.051043+0000","last_fullsized":"2023-10-09T06:07:58.051043+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.316697+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.316697+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:08.265733+0000","scrub_duration":0.001666561,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":293,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-09T06:07:51.700106+0000","last_change":"2023-10-09T06:01:22.337171+0000","last_active":"2023-10-09T06:07:51.700106+0000","last_peered":"2023-10-09T06:07:51.700106+0000","last_clean":"2023-10-09T06:07:51.700106+0000","last_became_active":"2023-10-09T06:01:22.335438+0000","last_became_peered":"2023-10-09T06:01:22.335438+0000","last_unstale":"2023-10-09T06:07:51.700106+0000","last_undegraded":"2023-10-09T06:07:51.700106+0000","last_fullsized":"2023-10-09T06:07:51.700106+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:25.036448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":371,"reported_epoch":440,"state":"active+undersized","last_fresh":"2023-10-09T06:07:58.048876+0000","last_change":"2023-10-09T06:07:51.710706+0000","last_active":"2023-10-09T06:07:58.048876+0000","last_peered":"2023-10-09T06:07:58.048876+0000","last_clean":"2023-10-09T06:07:47.627324+0000","last_became_active":"2023-10-09T06:07:51.710706+0000","last_became_peered":"2023-10-09T06:07:51.710706+0000","last_unstale":"2023-10-09T06:07:58.048876+0000","last_undegraded":"2023-10-09T06:07:58.048876+0000","last_fullsized":"2023-10-09T06:07:51.705058+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:32.335550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":694,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.051377+0000","last_change":"2023-10-09T06:02:40.388302+0000","last_active":"2023-10-09T06:07:58.051377+0000","last_peered":"2023-10-09T06:07:58.051377+0000","last_clean":"2023-10-09T06:07:58.051377+0000","last_became_active":"2023-10-09T06:01:57.803733+0000","last_became_peered":"2023-10-09T06:01:57.803733+0000","last_unstale":"2023-10-09T06:07:58.051377+0000","last_undegraded":"2023-10-09T06:07:58.051377+0000","last_fullsized":"2023-10-09T06:07:58.051377+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:40.388125+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:40.388125+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:17.919838+0000","scrub_duration":0.0055576990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":453,"reported_epoch":440,"state":"active+undersized+degraded","last_fresh":"2023-10-09T06:07:58.062711+0000","last_change":"2023-10-09T06:07:51.707034+0000","last_active":"2023-10-09T06:07:58.062711+0000","last_peered":"2023-10-09T06:07:58.062711+0000","last_clean":"2023-10-09T06:07:47.656239+0000","last_became_active":"2023-10-09T06:07:51.707034+0000","last_became_peered":"2023-10-09T06:07:51.707034+0000","last_unstale":"2023-10-09T06:07:58.062711+0000","last_undegraded":"2023-10-09T06:07:51.704230+0000","last_fullsized":"2023-10-09T06:07:51.704124+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:24.002564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":234,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.049648+0000","last_change":"2023-10-09T06:07:02.023531+0000","last_active":"2023-10-09T06:07:58.049648+0000","last_peered":"2023-10-09T06:07:58.049648+0000","last_clean":"2023-10-09T06:07:58.049648+0000","last_became_active":"2023-10-09T06:07:02.023280+0000","last_became_peered":"2023-10-09T06:07:02.023280+0000","last_unstale":"2023-10-09T06:07:58.049648+0000","last_undegraded":"2023-10-09T06:07:58.049648+0000","last_fullsized":"2023-10-09T06:07:58.049648+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:53.202101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"440'4430","reported_seq":4459,"reported_epoch":440,"state":"active+undersized+degraded","last_fresh":"2023-10-09T06:08:06.005370+0000","last_change":"2023-10-09T06:07:51.708055+0000","last_active":"2023-10-09T06:08:06.005370+0000","last_peered":"2023-10-09T06:08:06.005370+0000","last_clean":"2023-10-09T06:07:50.629239+0000","last_became_active":"2023-10-09T06:07:51.708055+0000","last_became_peered":"2023-10-09T06:07:51.708055+0000","last_unstale":"2023-10-09T06:08:06.005370+0000","last_undegraded":"2023-10-09T06:07:51.700244+0000","last_fullsized":"2023-10-09T06:07:51.700136+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4430,"log_dups_size":0,"ondisk_log_size":4430,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:58.171315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290324480,"num_objects":4430,"num_object_clones":0,"num_object_copies":13290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4430,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4430,"num_write_kb":283520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":4430}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":816,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-09T06:07:51.700435+0000","last_change":"2023-10-09T06:01:57.804339+0000","last_active":"2023-10-09T06:07:51.700435+0000","last_peered":"2023-10-09T06:07:51.700435+0000","last_clean":"2023-10-09T06:07:51.700435+0000","last_became_active":"2023-10-09T06:01:57.803844+0000","last_became_peered":"2023-10-09T06:01:57.803844+0000","last_unstale":"2023-10-09T06:07:51.700435+0000","last_undegraded":"2023-10-09T06:07:51.700435+0000","last_fullsized":"2023-10-09T06:07:51.700435+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:53.469394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:53.469394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:51.125394+0000","scrub_duration":0.0059716659999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":304,"reported_epoch":439,"state":"active+undersized+degraded","last_fresh":"2023-10-09T06:07:51.702248+0000","last_change":"2023-10-09T06:07:51.702003+0000","last_active":"2023-10-09T06:07:51.702248+0000","last_peered":"2023-10-09T06:07:51.702248+0000","last_clean":"2023-10-09T06:07:47.641518+0000","last_became_active":"2023-10-09T06:07:51.702003+0000","last_became_peered":"2023-10-09T06:07:51.702003+0000","last_unstale":"2023-10-09T06:07:51.702248+0000","last_undegraded":"2023-10-09T06:07:51.699306+0000","last_fullsized":"2023-10-09T06:07:51.699191+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:39.807423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":356,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-09T06:07:51.699762+0000","last_change":"2023-10-09T06:06:01.074296+0000","last_active":"2023-10-09T06:07:51.699762+0000","last_peered":"2023-10-09T06:07:51.699762+0000","last_clean":"2023-10-09T06:07:51.699762+0000","last_became_active":"2023-10-09T06:06:01.073655+0000","last_became_peered":"2023-10-09T06:06:01.073655+0000","last_unstale":"2023-10-09T06:07:51.699762+0000","last_undegraded":"2023-10-09T06:07:51.699762+0000","last_fullsized":"2023-10-09T06:07:51.699762+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:20.319425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"441'4701","reported_seq":4726,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:08:10.538364+0000","last_change":"2023-10-09T06:07:07.580215+0000","last_active":"2023-10-09T06:08:10.538364+0000","last_peered":"2023-10-09T06:08:10.538364+0000","last_clean":"2023-10-09T06:08:10.538364+0000","last_became_active":"2023-10-09T06:07:07.579874+0000","last_became_peered":"2023-10-09T06:07:07.579874+0000","last_unstale":"2023-10-09T06:08:10.538364+0000","last_undegraded":"2023-10-09T06:08:10.538364+0000","last_fullsized":"2023-10-09T06:08:10.538364+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4701,"log_dups_size":0,"ondisk_log_size":4701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:04.426865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308084736,"num_objects":4701,"num_object_clones":0,"num_object_copies":14103,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4701,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4701,"num_write_kb":300864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":756,"reported_epoch":440,"state":"active+undersized+degraded","last_fresh":"2023-10-09T06:07:58.049615+0000","last_change":"2023-10-09T06:07:51.706761+0000","last_active":"2023-10-09T06:07:58.049615+0000","last_peered":"2023-10-09T06:07:58.049615+0000","last_clean":"2023-10-09T06:07:47.626849+0000","last_became_active":"2023-10-09T06:07:51.706761+0000","last_became_peered":"2023-10-09T06:07:51.706761+0000","last_unstale":"2023-10-09T06:07:58.049615+0000","last_undegraded":"2023-10-09T06:07:51.704410+0000","last_fullsized":"2023-10-09T06:07:51.704293+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:03.241335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:03.241335+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:22.443653+0000","scrub_duration":0.017780547000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"3,6","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":353,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-09T06:07:51.700928+0000","last_change":"2023-10-09T06:01:54.826540+0000","last_active":"2023-10-09T06:07:51.700928+0000","last_peered":"2023-10-09T06:07:51.700928+0000","last_clean":"2023-10-09T06:07:51.700928+0000","last_became_active":"2023-10-09T06:01:54.766205+0000","last_became_peered":"2023-10-09T06:01:54.766205+0000","last_unstale":"2023-10-09T06:07:51.700928+0000","last_undegraded":"2023-10-09T06:07:51.700928+0000","last_fullsized":"2023-10-09T06:07:51.700928+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:45.220336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":479,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.051558+0000","last_change":"2023-10-09T06:01:54.760367+0000","last_active":"2023-10-09T06:07:58.051558+0000","last_peered":"2023-10-09T06:07:58.051558+0000","last_clean":"2023-10-09T06:07:58.051558+0000","last_became_active":"2023-10-09T06:01:54.760087+0000","last_became_peered":"2023-10-09T06:01:54.760087+0000","last_unstale":"2023-10-09T06:07:58.051558+0000","last_undegraded":"2023-10-09T06:07:58.051558+0000","last_fullsized":"2023-10-09T06:07:58.051558+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:26.000104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"440'4431","reported_seq":4454,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:08:05.989779+0000","last_change":"2023-10-09T06:07:07.880402+0000","last_active":"2023-10-09T06:08:05.989779+0000","last_peered":"2023-10-09T06:08:05.989779+0000","last_clean":"2023-10-09T06:08:05.989779+0000","last_became_active":"2023-10-09T06:07:07.880006+0000","last_became_peered":"2023-10-09T06:07:07.880006+0000","last_unstale":"2023-10-09T06:08:05.989779+0000","last_undegraded":"2023-10-09T06:08:05.989779+0000","last_fullsized":"2023-10-09T06:08:05.989779+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4431,"log_dups_size":0,"ondisk_log_size":4431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:17.652201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290390016,"num_objects":4431,"num_object_clones":0,"num_object_copies":13293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4431,"num_write_kb":283584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":880,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.064122+0000","last_change":"2023-10-09T06:01:22.326209+0000","last_active":"2023-10-09T06:07:58.064122+0000","last_peered":"2023-10-09T06:07:58.064122+0000","last_clean":"2023-10-09T06:07:58.064122+0000","last_became_active":"2023-10-09T06:01:22.325866+0000","last_became_peered":"2023-10-09T06:01:22.325866+0000","last_unstale":"2023-10-09T06:07:58.064122+0000","last_undegraded":"2023-10-09T06:07:58.064122+0000","last_fullsized":"2023-10-09T06:07:58.064122+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:41.753506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:41.753506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:32.324674+0000","scrub_duration":0.0022121929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":292,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-09T06:07:51.700631+0000","last_change":"2023-10-09T06:06:01.078672+0000","last_active":"2023-10-09T06:07:51.700631+0000","last_peered":"2023-10-09T06:07:51.700631+0000","last_clean":"2023-10-09T06:07:51.700631+0000","last_became_active":"2023-10-09T06:06:01.072297+0000","last_became_peered":"2023-10-09T06:06:01.072297+0000","last_unstale":"2023-10-09T06:07:51.700631+0000","last_undegraded":"2023-10-09T06:07:51.700631+0000","last_fullsized":"2023-10-09T06:07:51.700631+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:35.905773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":425,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:08:08.573517+0000","last_change":"2023-10-09T06:06:01.077340+0000","last_active":"2023-10-09T06:08:08.573517+0000","last_peered":"2023-10-09T06:08:08.573517+0000","last_clean":"2023-10-09T06:08:08.573517+0000","last_became_active":"2023-10-09T06:06:01.076610+0000","last_became_peered":"2023-10-09T06:06:01.076610+0000","last_unstale":"2023-10-09T06:08:08.573517+0000","last_undegraded":"2023-10-09T06:08:08.573517+0000","last_fullsized":"2023-10-09T06:08:08.573517+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:45.158282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"441'4695","reported_seq":4725,"reported_epoch":441,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:08:10.535331+0000","last_change":"2023-10-09T06:07:44.729895+0000","last_active":"2023-10-09T06:08:10.535331+0000","last_peered":"2023-10-09T06:08:10.535331+0000","last_clean":"2023-10-09T06:07:42.388388+0000","last_became_active":"2023-10-09T06:07:44.720581+0000","last_became_peered":"2023-10-09T06:07:44.720581+0000","last_unstale":"2023-10-09T06:08:10.535331+0000","last_undegraded":"2023-10-09T06:07:44.592304+0000","last_fullsized":"2023-10-09T06:07:44.563310+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4695,"log_dups_size":0,"ondisk_log_size":4695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:29:15.772970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307691520,"num_objects":4695,"num_object_clones":0,"num_object_copies":14085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4695,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4695,"num_write_kb":300480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5","objects":1360},{"shards":"1,5,6","objects":3335}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":616,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.053401+0000","last_change":"2023-10-09T06:02:16.361201+0000","last_active":"2023-10-09T06:07:58.053401+0000","last_peered":"2023-10-09T06:07:58.053401+0000","last_clean":"2023-10-09T06:07:58.053401+0000","last_became_active":"2023-10-09T06:01:22.337294+0000","last_became_peered":"2023-10-09T06:01:22.337294+0000","last_unstale":"2023-10-09T06:07:58.053401+0000","last_undegraded":"2023-10-09T06:07:58.053401+0000","last_fullsized":"2023-10-09T06:07:58.053401+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:16.361097+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:16.361097+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:16:25.401939+0000","scrub_duration":0.028533749000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":207,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.049739+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T06:07:58.049739+0000","last_peered":"2023-10-09T06:07:58.049739+0000","last_clean":"2023-10-09T06:07:58.049739+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T06:07:58.049739+0000","last_undegraded":"2023-10-09T06:07:58.049739+0000","last_fullsized":"2023-10-09T06:07:58.049739+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":372,"reported_epoch":440,"state":"active+undersized","last_fresh":"2023-10-09T06:07:58.050721+0000","last_change":"2023-10-09T06:07:51.699039+0000","last_active":"2023-10-09T06:07:58.050721+0000","last_peered":"2023-10-09T06:07:58.050721+0000","last_clean":"2023-10-09T06:01:48.339734+0000","last_became_active":"2023-10-09T06:07:51.699039+0000","last_became_peered":"2023-10-09T06:07:51.699039+0000","last_unstale":"2023-10-09T06:07:58.050721+0000","last_undegraded":"2023-10-09T06:07:58.050721+0000","last_fullsized":"2023-10-09T06:07:51.696485+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:14.271591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"441'4656","reported_seq":4681,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:08:09.840891+0000","last_change":"2023-10-09T06:07:07.547742+0000","last_active":"2023-10-09T06:08:09.840891+0000","last_peered":"2023-10-09T06:08:09.840891+0000","last_clean":"2023-10-09T06:08:09.840891+0000","last_became_active":"2023-10-09T06:07:07.547431+0000","last_became_peered":"2023-10-09T06:07:07.547431+0000","last_unstale":"2023-10-09T06:08:09.840891+0000","last_undegraded":"2023-10-09T06:08:09.840891+0000","last_fullsized":"2023-10-09T06:08:09.840891+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4656,"log_dups_size":0,"ondisk_log_size":4656,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:35.316678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305135616,"num_objects":4656,"num_object_clones":0,"num_object_copies":13968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4656,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4656,"num_write_kb":297984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":923,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-09T06:07:51.700590+0000","last_change":"2023-10-09T06:07:46.595962+0000","last_active":"2023-10-09T06:07:51.700590+0000","last_peered":"2023-10-09T06:07:51.700590+0000","last_clean":"2023-10-09T06:07:51.700590+0000","last_became_active":"2023-10-09T06:07:46.595718+0000","last_became_peered":"2023-10-09T06:07:46.595718+0000","last_unstale":"2023-10-09T06:07:51.700590+0000","last_undegraded":"2023-10-09T06:07:51.700590+0000","last_fullsized":"2023-10-09T06:07:51.700590+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:55.437947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:55.437947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:45:57.993572+0000","scrub_duration":0.0091262979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":232,"reported_epoch":440,"state":"active+undersized+degraded","last_fresh":"2023-10-09T06:07:58.049165+0000","last_change":"2023-10-09T06:07:51.714397+0000","last_active":"2023-10-09T06:07:58.049165+0000","last_peered":"2023-10-09T06:07:58.049165+0000","last_clean":"2023-10-09T06:07:47.627601+0000","last_became_active":"2023-10-09T06:07:51.714397+0000","last_became_peered":"2023-10-09T06:07:51.714397+0000","last_unstale":"2023-10-09T06:07:58.049165+0000","last_undegraded":"2023-10-09T06:07:51.702756+0000","last_fullsized":"2023-10-09T06:07:51.702651+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:17:01.204610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":395,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.052266+0000","last_change":"2023-10-09T06:01:22.332033+0000","last_active":"2023-10-09T06:07:58.052266+0000","last_peered":"2023-10-09T06:07:58.052266+0000","last_clean":"2023-10-09T06:07:58.052266+0000","last_became_active":"2023-10-09T06:01:22.329825+0000","last_became_peered":"2023-10-09T06:01:22.329825+0000","last_unstale":"2023-10-09T06:07:58.052266+0000","last_undegraded":"2023-10-09T06:07:58.052266+0000","last_fullsized":"2023-10-09T06:07:58.052266+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:54.046681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"441'4865","reported_seq":4898,"reported_epoch":441,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:08:10.508468+0000","last_change":"2023-10-09T06:07:44.730713+0000","last_active":"2023-10-09T06:08:10.508468+0000","last_peered":"2023-10-09T06:08:10.508468+0000","last_clean":"2023-10-09T06:07:42.372390+0000","last_became_active":"2023-10-09T06:07:44.720879+0000","last_became_peered":"2023-10-09T06:07:44.720879+0000","last_unstale":"2023-10-09T06:08:10.508468+0000","last_undegraded":"2023-10-09T06:07:44.589117+0000","last_fullsized":"2023-10-09T06:07:44.561865+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4865,"log_dups_size":0,"ondisk_log_size":4865,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:21:16.721210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318832640,"num_objects":4865,"num_object_clones":0,"num_object_copies":14595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3424,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4865,"num_write_kb":311360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":1441}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":684,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.045052+0000","last_change":"2023-10-09T06:07:46.602409+0000","last_active":"2023-10-09T06:07:58.045052+0000","last_peered":"2023-10-09T06:07:58.045052+0000","last_clean":"2023-10-09T06:07:58.045052+0000","last_became_active":"2023-10-09T06:07:46.602045+0000","last_became_peered":"2023-10-09T06:07:46.602045+0000","last_unstale":"2023-10-09T06:07:58.045052+0000","last_undegraded":"2023-10-09T06:07:58.045052+0000","last_fullsized":"2023-10-09T06:07:58.045052+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.282393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.282393+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:35.114362+0000","scrub_duration":0.01402403,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":478,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.063019+0000","last_change":"2023-10-09T06:01:54.751188+0000","last_active":"2023-10-09T06:07:58.063019+0000","last_peered":"2023-10-09T06:07:58.063019+0000","last_clean":"2023-10-09T06:07:58.063019+0000","last_became_active":"2023-10-09T06:01:54.750870+0000","last_became_peered":"2023-10-09T06:01:54.750870+0000","last_unstale":"2023-10-09T06:07:58.063019+0000","last_undegraded":"2023-10-09T06:07:58.063019+0000","last_fullsized":"2023-10-09T06:07:58.063019+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:01.985974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":489,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.053452+0000","last_change":"2023-10-09T06:07:43.572173+0000","last_active":"2023-10-09T06:07:58.053452+0000","last_peered":"2023-10-09T06:07:58.053452+0000","last_clean":"2023-10-09T06:07:58.053452+0000","last_became_active":"2023-10-09T06:07:43.571690+0000","last_became_peered":"2023-10-09T06:07:43.571690+0000","last_unstale":"2023-10-09T06:07:58.053452+0000","last_undegraded":"2023-10-09T06:07:58.053452+0000","last_fullsized":"2023-10-09T06:07:58.053452+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:10.234125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"440'4517","reported_seq":4539,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:08:06.046188+0000","last_change":"2023-10-09T06:07:07.547801+0000","last_active":"2023-10-09T06:08:06.046188+0000","last_peered":"2023-10-09T06:08:06.046188+0000","last_clean":"2023-10-09T06:08:06.046188+0000","last_became_active":"2023-10-09T06:07:07.547533+0000","last_became_peered":"2023-10-09T06:07:07.547533+0000","last_unstale":"2023-10-09T06:08:06.046188+0000","last_undegraded":"2023-10-09T06:08:06.046188+0000","last_fullsized":"2023-10-09T06:08:06.046188+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4517,"log_dups_size":0,"ondisk_log_size":4517,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:43:16.000531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296026112,"num_objects":4517,"num_object_clones":0,"num_object_copies":13551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4517,"num_write_kb":289088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":729,"reported_epoch":440,"state":"active+undersized","last_fresh":"2023-10-09T06:07:58.051342+0000","last_change":"2023-10-09T06:07:51.708480+0000","last_active":"2023-10-09T06:07:58.051342+0000","last_peered":"2023-10-09T06:07:58.051342+0000","last_clean":"2023-10-09T06:07:47.635521+0000","last_became_active":"2023-10-09T06:07:51.708480+0000","last_became_peered":"2023-10-09T06:07:51.708480+0000","last_unstale":"2023-10-09T06:07:58.051342+0000","last_undegraded":"2023-10-09T06:07:58.051342+0000","last_fullsized":"2023-10-09T06:07:51.696895+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.373748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.373748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:04:09.876908+0000","scrub_duration":0.0024576450000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":431,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.051692+0000","last_change":"2023-10-09T06:07:43.574863+0000","last_active":"2023-10-09T06:07:58.051692+0000","last_peered":"2023-10-09T06:07:58.051692+0000","last_clean":"2023-10-09T06:07:58.051692+0000","last_became_active":"2023-10-09T06:07:43.574576+0000","last_became_peered":"2023-10-09T06:07:43.574576+0000","last_unstale":"2023-10-09T06:07:58.051692+0000","last_undegraded":"2023-10-09T06:07:58.051692+0000","last_fullsized":"2023-10-09T06:07:58.051692+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:16.166004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":462,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.053318+0000","last_change":"2023-10-09T06:06:01.068708+0000","last_active":"2023-10-09T06:07:58.053318+0000","last_peered":"2023-10-09T06:07:58.053318+0000","last_clean":"2023-10-09T06:07:58.053318+0000","last_became_active":"2023-10-09T06:06:01.068474+0000","last_became_peered":"2023-10-09T06:06:01.068474+0000","last_unstale":"2023-10-09T06:07:58.053318+0000","last_undegraded":"2023-10-09T06:07:58.053318+0000","last_fullsized":"2023-10-09T06:07:58.053318+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:31.749069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"441'4600","reported_seq":6807,"reported_epoch":441,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T06:08:09.870476+0000","last_change":"2023-10-09T06:07:44.746784+0000","last_active":"2023-10-09T06:08:09.870476+0000","last_peered":"2023-10-09T06:08:09.870476+0000","last_clean":"2023-10-09T06:07:42.380305+0000","last_became_active":"2023-10-09T06:07:44.713964+0000","last_became_peered":"2023-10-09T06:07:44.713964+0000","last_unstale":"2023-10-09T06:08:09.870476+0000","last_undegraded":"2023-10-09T06:08:09.870476+0000","last_fullsized":"2023-10-09T06:07:44.566632+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4600,"log_dups_size":0,"ondisk_log_size":4600,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:37.355319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301465600,"num_objects":4600,"num_object_clones":0,"num_object_copies":13800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2158,"num_objects_unfound":0,"num_objects_dirty":4600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4600,"num_write_kb":294400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1087,"num_bytes_recovered":71237632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3","objects":2442},{"shards":"1,3,7","objects":2158}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":634,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.062936+0000","last_change":"2023-10-09T06:07:46.586758+0000","last_active":"2023-10-09T06:07:58.062936+0000","last_peered":"2023-10-09T06:07:58.062936+0000","last_clean":"2023-10-09T06:07:58.062936+0000","last_became_active":"2023-10-09T06:07:46.586421+0000","last_became_peered":"2023-10-09T06:07:46.586421+0000","last_unstale":"2023-10-09T06:07:58.062936+0000","last_undegraded":"2023-10-09T06:07:58.062936+0000","last_fullsized":"2023-10-09T06:07:58.062936+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:52.129816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:52.129816+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:12.513805+0000","scrub_duration":0.0039938459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":231,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.045138+0000","last_change":"2023-10-09T06:01:54.761275+0000","last_active":"2023-10-09T06:07:58.045138+0000","last_peered":"2023-10-09T06:07:58.045138+0000","last_clean":"2023-10-09T06:07:58.045138+0000","last_became_active":"2023-10-09T06:01:54.760782+0000","last_became_peered":"2023-10-09T06:01:54.760782+0000","last_unstale":"2023-10-09T06:07:58.045138+0000","last_undegraded":"2023-10-09T06:07:58.045138+0000","last_fullsized":"2023-10-09T06:07:58.045138+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:28.119846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":120,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.051323+0000","last_change":"2023-10-09T06:01:54.761768+0000","last_active":"2023-10-09T06:07:58.051323+0000","last_peered":"2023-10-09T06:07:58.051323+0000","last_clean":"2023-10-09T06:07:58.051323+0000","last_became_active":"2023-10-09T06:01:54.761443+0000","last_became_peered":"2023-10-09T06:01:54.761443+0000","last_unstale":"2023-10-09T06:07:58.051323+0000","last_undegraded":"2023-10-09T06:07:58.051323+0000","last_fullsized":"2023-10-09T06:07:58.051323+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:51.555987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"441'4624","reported_seq":4649,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:08:10.501043+0000","last_change":"2023-10-09T06:07:07.880280+0000","last_active":"2023-10-09T06:08:10.501043+0000","last_peered":"2023-10-09T06:08:10.501043+0000","last_clean":"2023-10-09T06:08:10.501043+0000","last_became_active":"2023-10-09T06:07:07.879952+0000","last_became_peered":"2023-10-09T06:07:07.879952+0000","last_unstale":"2023-10-09T06:08:10.501043+0000","last_undegraded":"2023-10-09T06:08:10.501043+0000","last_fullsized":"2023-10-09T06:08:10.501043+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4624,"log_dups_size":0,"ondisk_log_size":4624,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:10.864472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303038464,"num_objects":4624,"num_object_clones":0,"num_object_copies":13872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4624,"num_write_kb":295936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":864,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.063868+0000","last_change":"2023-10-09T05:59:56.092654+0000","last_active":"2023-10-09T06:07:58.063868+0000","last_peered":"2023-10-09T06:07:58.063868+0000","last_clean":"2023-10-09T06:07:58.063868+0000","last_became_active":"2023-10-09T05:59:32.529949+0000","last_became_peered":"2023-10-09T05:59:32.529949+0000","last_unstale":"2023-10-09T06:07:58.063868+0000","last_undegraded":"2023-10-09T06:07:58.063868+0000","last_fullsized":"2023-10-09T06:07:58.063868+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.092586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.092586+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:15:13.234903+0000","scrub_duration":0.0013484219999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":156,"reported_epoch":441,"state":"active+undersized","last_fresh":"2023-10-09T06:08:08.571583+0000","last_change":"2023-10-09T06:07:51.703936+0000","last_active":"2023-10-09T06:08:08.571583+0000","last_peered":"2023-10-09T06:08:08.571583+0000","last_clean":"2023-10-09T06:07:47.635546+0000","last_became_active":"2023-10-09T06:07:51.703936+0000","last_became_peered":"2023-10-09T06:07:51.703936+0000","last_unstale":"2023-10-09T06:08:08.571583+0000","last_undegraded":"2023-10-09T06:08:08.571583+0000","last_fullsized":"2023-10-09T06:07:51.696607+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:28.960353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":226,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.048405+0000","last_change":"2023-10-09T06:07:43.577306+0000","last_active":"2023-10-09T06:07:58.048405+0000","last_peered":"2023-10-09T06:07:58.048405+0000","last_clean":"2023-10-09T06:07:58.048405+0000","last_became_active":"2023-10-09T06:07:43.576956+0000","last_became_peered":"2023-10-09T06:07:43.576956+0000","last_unstale":"2023-10-09T06:07:58.048405+0000","last_undegraded":"2023-10-09T06:07:58.048405+0000","last_fullsized":"2023-10-09T06:07:58.048405+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:21.020403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":136,"reported_epoch":441,"state":"active+undersized","last_fresh":"2023-10-09T06:08:08.573652+0000","last_change":"2023-10-09T06:07:51.698969+0000","last_active":"2023-10-09T06:08:08.573652+0000","last_peered":"2023-10-09T06:08:08.573652+0000","last_clean":"2023-10-09T06:07:47.638309+0000","last_became_active":"2023-10-09T06:07:51.698969+0000","last_became_peered":"2023-10-09T06:07:51.698969+0000","last_unstale":"2023-10-09T06:08:08.573652+0000","last_undegraded":"2023-10-09T06:08:08.573652+0000","last_fullsized":"2023-10-09T06:07:51.697103+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:24.008054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"440'4516","reported_seq":4546,"reported_epoch":440,"state":"active+undersized+degraded","last_fresh":"2023-10-09T06:08:06.041922+0000","last_change":"2023-10-09T06:07:51.705556+0000","last_active":"2023-10-09T06:08:06.041922+0000","last_peered":"2023-10-09T06:08:06.041922+0000","last_clean":"2023-10-09T06:07:50.661954+0000","last_became_active":"2023-10-09T06:07:51.705556+0000","last_became_peered":"2023-10-09T06:07:51.705556+0000","last_unstale":"2023-10-09T06:08:06.041922+0000","last_undegraded":"2023-10-09T06:07:51.700065+0000","last_fullsized":"2023-10-09T06:07:51.699931+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4516,"log_dups_size":0,"ondisk_log_size":4516,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:53:21.118229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295960576,"num_objects":4516,"num_object_clones":0,"num_object_copies":13548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4516,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4516,"num_write_kb":289024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":4516}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":574,"reported_epoch":440,"state":"active+undersized","last_fresh":"2023-10-09T06:07:58.058997+0000","last_change":"2023-10-09T06:07:51.709788+0000","last_active":"2023-10-09T06:07:58.058997+0000","last_peered":"2023-10-09T06:07:58.058997+0000","last_clean":"2023-10-09T05:56:42.247103+0000","last_became_active":"2023-10-09T06:07:51.709788+0000","last_became_peered":"2023-10-09T06:07:51.709788+0000","last_unstale":"2023-10-09T06:07:58.058997+0000","last_undegraded":"2023-10-09T06:07:58.058997+0000","last_fullsized":"2023-10-09T06:07:51.702197+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":121,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:08:08.573764+0000","last_change":"2023-10-09T06:01:54.754294+0000","last_active":"2023-10-09T06:08:08.573764+0000","last_peered":"2023-10-09T06:08:08.573764+0000","last_clean":"2023-10-09T06:08:08.573764+0000","last_became_active":"2023-10-09T06:01:54.754024+0000","last_became_peered":"2023-10-09T06:01:54.754024+0000","last_unstale":"2023-10-09T06:08:08.573764+0000","last_undegraded":"2023-10-09T06:08:08.573764+0000","last_fullsized":"2023-10-09T06:08:08.573764+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:19.746882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":411,"reported_epoch":440,"state":"active+undersized","last_fresh":"2023-10-09T06:07:58.052275+0000","last_change":"2023-10-09T06:07:51.705726+0000","last_active":"2023-10-09T06:07:58.052275+0000","last_peered":"2023-10-09T06:07:58.052275+0000","last_clean":"2023-10-09T06:07:47.645621+0000","last_became_active":"2023-10-09T06:07:51.705726+0000","last_became_peered":"2023-10-09T06:07:51.705726+0000","last_unstale":"2023-10-09T06:07:58.052275+0000","last_undegraded":"2023-10-09T06:07:58.052275+0000","last_fullsized":"2023-10-09T06:07:51.697389+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:23.793318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":264,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:08:08.571704+0000","last_change":"2023-10-09T06:07:43.580765+0000","last_active":"2023-10-09T06:08:08.571704+0000","last_peered":"2023-10-09T06:08:08.571704+0000","last_clean":"2023-10-09T06:08:08.571704+0000","last_became_active":"2023-10-09T06:07:43.580458+0000","last_became_peered":"2023-10-09T06:07:43.580458+0000","last_unstale":"2023-10-09T06:08:08.571704+0000","last_undegraded":"2023-10-09T06:08:08.571704+0000","last_fullsized":"2023-10-09T06:08:08.571704+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:55.161893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"441'4635","reported_seq":4659,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:08:10.542550+0000","last_change":"2023-10-09T06:07:07.546882+0000","last_active":"2023-10-09T06:08:10.542550+0000","last_peered":"2023-10-09T06:08:10.542550+0000","last_clean":"2023-10-09T06:08:10.542550+0000","last_became_active":"2023-10-09T06:07:07.546552+0000","last_became_peered":"2023-10-09T06:07:07.546552+0000","last_unstale":"2023-10-09T06:08:10.542550+0000","last_undegraded":"2023-10-09T06:08:10.542550+0000","last_fullsized":"2023-10-09T06:08:10.542550+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4635,"log_dups_size":0,"ondisk_log_size":4635,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:57.434915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303759360,"num_objects":4635,"num_object_clones":0,"num_object_copies":13905,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4635,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4635,"num_write_kb":296640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":771,"reported_epoch":439,"state":"active+undersized","last_fresh":"2023-10-09T06:07:51.705355+0000","last_change":"2023-10-09T06:07:51.705111+0000","last_active":"2023-10-09T06:07:51.705355+0000","last_peered":"2023-10-09T06:07:51.705355+0000","last_clean":"2023-10-09T06:07:47.641128+0000","last_became_active":"2023-10-09T06:07:51.705111+0000","last_became_peered":"2023-10-09T06:07:51.705111+0000","last_unstale":"2023-10-09T06:07:51.705355+0000","last_undegraded":"2023-10-09T06:07:51.705355+0000","last_fullsized":"2023-10-09T06:07:51.699095+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:55.897812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:55.897812+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:40.812052+0000","scrub_duration":0.0014511420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":209,"reported_epoch":440,"state":"active+undersized","last_fresh":"2023-10-09T06:07:58.045725+0000","last_change":"2023-10-09T06:07:51.715505+0000","last_active":"2023-10-09T06:07:58.045725+0000","last_peered":"2023-10-09T06:07:58.045725+0000","last_clean":"2023-10-09T06:07:47.629573+0000","last_became_active":"2023-10-09T06:07:51.715505+0000","last_became_peered":"2023-10-09T06:07:51.715505+0000","last_unstale":"2023-10-09T06:07:58.045725+0000","last_undegraded":"2023-10-09T06:07:58.045725+0000","last_fullsized":"2023-10-09T06:07:51.700869+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:11.904358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":190,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.053974+0000","last_change":"2023-10-09T06:07:43.571911+0000","last_active":"2023-10-09T06:07:58.053974+0000","last_peered":"2023-10-09T06:07:58.053974+0000","last_clean":"2023-10-09T06:07:58.053974+0000","last_became_active":"2023-10-09T06:07:43.571539+0000","last_became_peered":"2023-10-09T06:07:43.571539+0000","last_unstale":"2023-10-09T06:07:58.053974+0000","last_undegraded":"2023-10-09T06:07:58.053974+0000","last_fullsized":"2023-10-09T06:07:58.053974+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:26.591500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":126,"reported_epoch":440,"state":"active+undersized","last_fresh":"2023-10-09T06:07:58.051213+0000","last_change":"2023-10-09T06:07:51.708660+0000","last_active":"2023-10-09T06:07:58.051213+0000","last_peered":"2023-10-09T06:07:58.051213+0000","last_clean":"2023-10-09T06:07:47.642813+0000","last_became_active":"2023-10-09T06:07:51.708660+0000","last_became_peered":"2023-10-09T06:07:51.708660+0000","last_unstale":"2023-10-09T06:07:58.051213+0000","last_undegraded":"2023-10-09T06:07:58.051213+0000","last_fullsized":"2023-10-09T06:07:51.697721+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:20:31.014324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"441'4652","reported_seq":4683,"reported_epoch":441,"state":"active+undersized+degraded","last_fresh":"2023-10-09T06:08:09.853951+0000","last_change":"2023-10-09T06:07:51.710785+0000","last_active":"2023-10-09T06:08:09.853951+0000","last_peered":"2023-10-09T06:08:09.853951+0000","last_clean":"2023-10-09T06:07:50.696644+0000","last_became_active":"2023-10-09T06:07:51.710785+0000","last_became_peered":"2023-10-09T06:07:51.710785+0000","last_unstale":"2023-10-09T06:08:09.853951+0000","last_undegraded":"2023-10-09T06:07:51.700829+0000","last_fullsized":"2023-10-09T06:07:51.700712+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4652,"log_dups_size":0,"ondisk_log_size":4652,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:10.653965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304873472,"num_objects":4652,"num_object_clones":0,"num_object_copies":13956,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4652,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4652,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4652,"num_write_kb":297728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":4652}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":331,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-09T06:07:51.700720+0000","last_change":"2023-10-09T06:07:43.577756+0000","last_active":"2023-10-09T06:07:51.700720+0000","last_peered":"2023-10-09T06:07:51.700720+0000","last_clean":"2023-10-09T06:07:51.700720+0000","last_became_active":"2023-10-09T06:07:43.577430+0000","last_became_peered":"2023-10-09T06:07:43.577430+0000","last_unstale":"2023-10-09T06:07:51.700720+0000","last_undegraded":"2023-10-09T06:07:51.700720+0000","last_fullsized":"2023-10-09T06:07:51.700720+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:07.379946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":851,"reported_epoch":439,"state":"active+undersized","last_fresh":"2023-10-09T06:07:51.705848+0000","last_change":"2023-10-09T06:07:51.705658+0000","last_active":"2023-10-09T06:07:51.705848+0000","last_peered":"2023-10-09T06:07:51.705848+0000","last_clean":"2023-10-09T06:07:47.641104+0000","last_became_active":"2023-10-09T06:07:51.705658+0000","last_became_peered":"2023-10-09T06:07:51.705658+0000","last_unstale":"2023-10-09T06:07:51.705848+0000","last_undegraded":"2023-10-09T06:07:51.705848+0000","last_fullsized":"2023-10-09T06:07:51.700186+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:37.782173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:37.782173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:13.732682+0000","scrub_duration":0.001325908,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":208,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.047646+0000","last_change":"2023-10-09T05:57:23.479174+0000","last_active":"2023-10-09T06:07:58.047646+0000","last_peered":"2023-10-09T06:07:58.047646+0000","last_clean":"2023-10-09T06:07:58.047646+0000","last_became_active":"2023-10-09T05:57:23.478897+0000","last_became_peered":"2023-10-09T05:57:23.478897+0000","last_unstale":"2023-10-09T06:07:58.047646+0000","last_undegraded":"2023-10-09T06:07:58.047646+0000","last_fullsized":"2023-10-09T06:07:58.047646+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:00.160042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":351,"reported_epoch":440,"state":"active+undersized","last_fresh":"2023-10-09T06:07:58.053351+0000","last_change":"2023-10-09T06:07:51.701455+0000","last_active":"2023-10-09T06:07:58.053351+0000","last_peered":"2023-10-09T06:07:58.053351+0000","last_clean":"2023-10-09T06:07:47.645788+0000","last_became_active":"2023-10-09T06:07:51.701455+0000","last_became_peered":"2023-10-09T06:07:51.701455+0000","last_unstale":"2023-10-09T06:07:58.053351+0000","last_undegraded":"2023-10-09T06:07:58.053351+0000","last_fullsized":"2023-10-09T06:07:51.697157+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:50.734325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"441'4639","reported_seq":4670,"reported_epoch":441,"state":"active+undersized+degraded","last_fresh":"2023-10-09T06:08:08.694698+0000","last_change":"2023-10-09T06:07:51.706804+0000","last_active":"2023-10-09T06:08:08.694698+0000","last_peered":"2023-10-09T06:08:08.694698+0000","last_clean":"2023-10-09T06:07:50.696037+0000","last_became_active":"2023-10-09T06:07:51.706804+0000","last_became_peered":"2023-10-09T06:07:51.706804+0000","last_unstale":"2023-10-09T06:08:08.694698+0000","last_undegraded":"2023-10-09T06:07:51.703995+0000","last_fullsized":"2023-10-09T06:07:51.703886+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4639,"log_dups_size":0,"ondisk_log_size":4639,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:25.587346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304021504,"num_objects":4639,"num_object_clones":0,"num_object_copies":13917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4639,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4639,"num_write_kb":296896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"5,6","objects":4639}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":655,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.050958+0000","last_change":"2023-10-09T06:07:06.560255+0000","last_active":"2023-10-09T06:07:58.050958+0000","last_peered":"2023-10-09T06:07:58.050958+0000","last_clean":"2023-10-09T06:07:58.050958+0000","last_became_active":"2023-10-09T06:07:06.559524+0000","last_became_peered":"2023-10-09T06:07:06.559524+0000","last_unstale":"2023-10-09T06:07:58.050958+0000","last_undegraded":"2023-10-09T06:07:58.050958+0000","last_fullsized":"2023-10-09T06:07:58.050958+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:57.627296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:57.627296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:41.099391+0000","scrub_duration":0.001913166,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":323,"reported_epoch":439,"state":"active+undersized","last_fresh":"2023-10-09T06:07:51.704322+0000","last_change":"2023-10-09T06:07:51.704104+0000","last_active":"2023-10-09T06:07:51.704322+0000","last_peered":"2023-10-09T06:07:51.704322+0000","last_clean":"2023-10-09T06:07:47.640939+0000","last_became_active":"2023-10-09T06:07:51.704104+0000","last_became_peered":"2023-10-09T06:07:51.704104+0000","last_unstale":"2023-10-09T06:07:51.704322+0000","last_undegraded":"2023-10-09T06:07:51.704322+0000","last_fullsized":"2023-10-09T06:07:51.699516+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:07.052928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":279,"reported_epoch":440,"state":"active+undersized","last_fresh":"2023-10-09T06:07:58.051440+0000","last_change":"2023-10-09T06:07:51.710700+0000","last_active":"2023-10-09T06:07:58.051440+0000","last_peered":"2023-10-09T06:07:58.051440+0000","last_clean":"2023-10-09T06:00:51.356444+0000","last_became_active":"2023-10-09T06:07:51.710700+0000","last_became_peered":"2023-10-09T06:07:51.710700+0000","last_unstale":"2023-10-09T06:07:58.051440+0000","last_undegraded":"2023-10-09T06:07:58.051440+0000","last_fullsized":"2023-10-09T06:07:51.696429+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:21.980685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":520,"reported_epoch":440,"state":"active+undersized","last_fresh":"2023-10-09T06:07:58.052327+0000","last_change":"2023-10-09T06:07:51.702467+0000","last_active":"2023-10-09T06:07:58.052327+0000","last_peered":"2023-10-09T06:07:58.052327+0000","last_clean":"2023-10-09T06:07:35.533292+0000","last_became_active":"2023-10-09T06:07:51.702467+0000","last_became_peered":"2023-10-09T06:07:51.702467+0000","last_unstale":"2023-10-09T06:07:58.052327+0000","last_undegraded":"2023-10-09T06:07:58.052327+0000","last_fullsized":"2023-10-09T06:07:51.696276+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:08.479151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"440'4374","reported_seq":4397,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:08:06.039362+0000","last_change":"2023-10-09T06:07:07.579770+0000","last_active":"2023-10-09T06:08:06.039362+0000","last_peered":"2023-10-09T06:08:06.039362+0000","last_clean":"2023-10-09T06:08:06.039362+0000","last_became_active":"2023-10-09T06:07:07.579317+0000","last_became_peered":"2023-10-09T06:07:07.579317+0000","last_unstale":"2023-10-09T06:08:06.039362+0000","last_undegraded":"2023-10-09T06:08:06.039362+0000","last_fullsized":"2023-10-09T06:08:06.039362+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4374,"log_dups_size":0,"ondisk_log_size":4374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:30.621836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286654464,"num_objects":4374,"num_object_clones":0,"num_object_copies":13122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4374,"num_write_kb":279936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":861,"reported_epoch":440,"state":"active+undersized","last_fresh":"2023-10-09T06:07:58.063632+0000","last_change":"2023-10-09T06:07:51.706987+0000","last_active":"2023-10-09T06:07:58.063632+0000","last_peered":"2023-10-09T06:07:58.063632+0000","last_clean":"2023-10-09T06:07:47.656698+0000","last_became_active":"2023-10-09T06:07:51.706987+0000","last_became_peered":"2023-10-09T06:07:51.706987+0000","last_unstale":"2023-10-09T06:07:58.063632+0000","last_undegraded":"2023-10-09T06:07:58.063632+0000","last_fullsized":"2023-10-09T06:07:51.705453+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:54.074310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:54.074310+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:45.083735+0000","scrub_duration":0.011040648,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":298,"reported_epoch":439,"state":"active+undersized","last_fresh":"2023-10-09T06:07:51.703016+0000","last_change":"2023-10-09T06:07:51.702808+0000","last_active":"2023-10-09T06:07:51.703016+0000","last_peered":"2023-10-09T06:07:51.703016+0000","last_clean":"2023-10-09T06:07:47.641335+0000","last_became_active":"2023-10-09T06:07:51.702808+0000","last_became_peered":"2023-10-09T06:07:51.702808+0000","last_unstale":"2023-10-09T06:07:51.703016+0000","last_undegraded":"2023-10-09T06:07:51.703016+0000","last_fullsized":"2023-10-09T06:07:51.699160+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:01.489172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":139,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:08:08.571731+0000","last_change":"2023-10-09T06:07:43.578431+0000","last_active":"2023-10-09T06:08:08.571731+0000","last_peered":"2023-10-09T06:08:08.571731+0000","last_clean":"2023-10-09T06:08:08.571731+0000","last_became_active":"2023-10-09T06:07:43.578110+0000","last_became_peered":"2023-10-09T06:07:43.578110+0000","last_unstale":"2023-10-09T06:08:08.571731+0000","last_undegraded":"2023-10-09T06:08:08.571731+0000","last_fullsized":"2023-10-09T06:08:08.571731+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:18.556487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":219,"reported_epoch":439,"state":"active+undersized","last_fresh":"2023-10-09T06:07:51.706363+0000","last_change":"2023-10-09T06:07:51.706126+0000","last_active":"2023-10-09T06:07:51.706363+0000","last_peered":"2023-10-09T06:07:51.706363+0000","last_clean":"2023-10-09T06:07:47.641062+0000","last_became_active":"2023-10-09T06:07:51.706126+0000","last_became_peered":"2023-10-09T06:07:51.706126+0000","last_unstale":"2023-10-09T06:07:51.706363+0000","last_undegraded":"2023-10-09T06:07:51.706363+0000","last_fullsized":"2023-10-09T06:07:51.700276+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:41.030190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.d","version":"440'4475","reported_seq":4498,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:08:06.024743+0000","last_change":"2023-10-09T06:07:07.883000+0000","last_active":"2023-10-09T06:08:06.024743+0000","last_peered":"2023-10-09T06:08:06.024743+0000","last_clean":"2023-10-09T06:08:06.024743+0000","last_became_active":"2023-10-09T06:07:07.882612+0000","last_became_peered":"2023-10-09T06:07:07.882612+0000","last_unstale":"2023-10-09T06:08:06.024743+0000","last_undegraded":"2023-10-09T06:08:06.024743+0000","last_fullsized":"2023-10-09T06:08:06.024743+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4475,"log_dups_size":0,"ondisk_log_size":4475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:14:21.633273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293273600,"num_objects":4475,"num_object_clones":0,"num_object_copies":13425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4475,"num_write_kb":286400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":894,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.053566+0000","last_change":"2023-10-09T06:07:46.590898+0000","last_active":"2023-10-09T06:07:58.053566+0000","last_peered":"2023-10-09T06:07:58.053566+0000","last_clean":"2023-10-09T06:07:58.053566+0000","last_became_active":"2023-10-09T06:07:46.590596+0000","last_became_peered":"2023-10-09T06:07:46.590596+0000","last_unstale":"2023-10-09T06:07:58.053566+0000","last_undegraded":"2023-10-09T06:07:58.053566+0000","last_fullsized":"2023-10-09T06:07:58.053566+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:17.344518+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:17.344518+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:00.399548+0000","scrub_duration":0.0015914099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":457,"reported_epoch":440,"state":"active+undersized+degraded","last_fresh":"2023-10-09T06:07:58.049759+0000","last_change":"2023-10-09T06:07:51.710841+0000","last_active":"2023-10-09T06:07:58.049759+0000","last_peered":"2023-10-09T06:07:58.049759+0000","last_clean":"2023-10-09T06:07:47.627846+0000","last_became_active":"2023-10-09T06:07:51.710841+0000","last_became_peered":"2023-10-09T06:07:51.710841+0000","last_unstale":"2023-10-09T06:07:58.049759+0000","last_undegraded":"2023-10-09T06:07:51.705289+0000","last_fullsized":"2023-10-09T06:07:51.705193+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:41.829341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":236,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.052482+0000","last_change":"2023-10-09T06:07:06.560512+0000","last_active":"2023-10-09T06:07:58.052482+0000","last_peered":"2023-10-09T06:07:58.052482+0000","last_clean":"2023-10-09T06:07:58.052482+0000","last_became_active":"2023-10-09T06:07:06.560204+0000","last_became_peered":"2023-10-09T06:07:06.560204+0000","last_unstale":"2023-10-09T06:07:58.052482+0000","last_undegraded":"2023-10-09T06:07:58.052482+0000","last_fullsized":"2023-10-09T06:07:58.052482+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:19.102248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":214,"reported_epoch":440,"state":"active+undersized","last_fresh":"2023-10-09T06:07:58.045352+0000","last_change":"2023-10-09T06:07:51.710778+0000","last_active":"2023-10-09T06:07:58.045352+0000","last_peered":"2023-10-09T06:07:58.045352+0000","last_clean":"2023-10-09T06:07:47.627277+0000","last_became_active":"2023-10-09T06:07:51.710778+0000","last_became_peered":"2023-10-09T06:07:51.710778+0000","last_unstale":"2023-10-09T06:07:58.045352+0000","last_undegraded":"2023-10-09T06:07:58.045352+0000","last_fullsized":"2023-10-09T06:07:51.704350+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:56.304788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"441'4627","reported_seq":4652,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:08:09.785949+0000","last_change":"2023-10-09T06:07:07.580272+0000","last_active":"2023-10-09T06:08:09.785949+0000","last_peered":"2023-10-09T06:08:09.785949+0000","last_clean":"2023-10-09T06:08:09.785949+0000","last_became_active":"2023-10-09T06:07:07.579909+0000","last_became_peered":"2023-10-09T06:07:07.579909+0000","last_unstale":"2023-10-09T06:08:09.785949+0000","last_undegraded":"2023-10-09T06:08:09.785949+0000","last_fullsized":"2023-10-09T06:08:09.785949+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4627,"log_dups_size":0,"ondisk_log_size":4627,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:18.048371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303235072,"num_objects":4627,"num_object_clones":0,"num_object_copies":13881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4627,"num_write_kb":296128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":702,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.049477+0000","last_change":"2023-10-09T05:58:34.333129+0000","last_active":"2023-10-09T06:07:58.049477+0000","last_peered":"2023-10-09T06:07:58.049477+0000","last_clean":"2023-10-09T06:07:58.049477+0000","last_became_active":"2023-10-09T05:57:26.502911+0000","last_became_peered":"2023-10-09T05:57:26.502911+0000","last_unstale":"2023-10-09T06:07:58.049477+0000","last_undegraded":"2023-10-09T06:07:58.049477+0000","last_fullsized":"2023-10-09T06:07:58.049477+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:25.230712+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":465,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.064065+0000","last_change":"2023-10-09T06:07:43.573674+0000","last_active":"2023-10-09T06:07:58.064065+0000","last_peered":"2023-10-09T06:07:58.064065+0000","last_clean":"2023-10-09T06:07:58.064065+0000","last_became_active":"2023-10-09T06:07:43.573276+0000","last_became_peered":"2023-10-09T06:07:43.573276+0000","last_unstale":"2023-10-09T06:07:58.064065+0000","last_undegraded":"2023-10-09T06:07:58.064065+0000","last_fullsized":"2023-10-09T06:07:58.064065+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:17.556584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":401,"reported_epoch":440,"state":"active+undersized+degraded","last_fresh":"2023-10-09T06:07:58.052204+0000","last_change":"2023-10-09T06:07:51.702127+0000","last_active":"2023-10-09T06:07:58.052204+0000","last_peered":"2023-10-09T06:07:58.052204+0000","last_clean":"2023-10-09T06:01:48.340775+0000","last_became_active":"2023-10-09T06:07:51.702127+0000","last_became_peered":"2023-10-09T06:07:51.702127+0000","last_unstale":"2023-10-09T06:07:58.052204+0000","last_undegraded":"2023-10-09T06:07:51.698001+0000","last_fullsized":"2023-10-09T06:07:51.697901+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:52.260216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":387,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.051764+0000","last_change":"2023-10-09T06:01:22.334081+0000","last_active":"2023-10-09T06:07:58.051764+0000","last_peered":"2023-10-09T06:07:58.051764+0000","last_clean":"2023-10-09T06:07:58.051764+0000","last_became_active":"2023-10-09T06:01:22.333663+0000","last_became_peered":"2023-10-09T06:01:22.333663+0000","last_unstale":"2023-10-09T06:07:58.051764+0000","last_undegraded":"2023-10-09T06:07:58.051764+0000","last_fullsized":"2023-10-09T06:07:58.051764+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:55.809259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":675,"reported_epoch":440,"state":"active+undersized","last_fresh":"2023-10-09T06:07:58.045507+0000","last_change":"2023-10-09T06:07:51.705825+0000","last_active":"2023-10-09T06:07:58.045507+0000","last_peered":"2023-10-09T06:07:58.045507+0000","last_clean":"2023-10-09T06:07:47.629378+0000","last_became_active":"2023-10-09T06:07:51.705825+0000","last_became_peered":"2023-10-09T06:07:51.705825+0000","last_unstale":"2023-10-09T06:07:58.045507+0000","last_undegraded":"2023-10-09T06:07:58.045507+0000","last_fullsized":"2023-10-09T06:07:51.701768+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.181811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.181811+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:15.998745+0000","scrub_duration":0.011553313000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":35,"reported_epoch":440,"state":"active+undersized","last_fresh":"2023-10-09T06:07:58.052450+0000","last_change":"2023-10-09T06:07:51.701192+0000","last_active":"2023-10-09T06:07:58.052450+0000","last_peered":"2023-10-09T06:07:58.052450+0000","last_clean":"2023-10-09T05:40:11.603872+0000","last_became_active":"2023-10-09T06:07:51.701192+0000","last_became_peered":"2023-10-09T06:07:51.701192+0000","last_unstale":"2023-10-09T06:07:58.052450+0000","last_undegraded":"2023-10-09T06:07:58.052450+0000","last_fullsized":"2023-10-09T06:07:51.697698+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:45.907541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":140,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.053522+0000","last_change":"2023-10-09T06:07:43.580844+0000","last_active":"2023-10-09T06:07:58.053522+0000","last_peered":"2023-10-09T06:07:58.053522+0000","last_clean":"2023-10-09T06:07:58.053522+0000","last_became_active":"2023-10-09T06:07:43.580523+0000","last_became_peered":"2023-10-09T06:07:43.580523+0000","last_unstale":"2023-10-09T06:07:58.053522+0000","last_undegraded":"2023-10-09T06:07:58.053522+0000","last_fullsized":"2023-10-09T06:07:58.053522+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:55.942430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":787,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.048600+0000","last_change":"2023-10-09T06:00:04.215425+0000","last_active":"2023-10-09T06:07:58.048600+0000","last_peered":"2023-10-09T06:07:58.048600+0000","last_clean":"2023-10-09T06:07:58.048600+0000","last_became_active":"2023-10-09T05:59:40.326685+0000","last_became_peered":"2023-10-09T05:59:40.326685+0000","last_unstale":"2023-10-09T06:07:58.048600+0000","last_undegraded":"2023-10-09T06:07:58.048600+0000","last_fullsized":"2023-10-09T06:07:58.048600+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:04.215270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:04.215270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:03.963694+0000","scrub_duration":0.002850652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":212,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.052186+0000","last_change":"2023-10-09T06:01:54.824780+0000","last_active":"2023-10-09T06:07:58.052186+0000","last_peered":"2023-10-09T06:07:58.052186+0000","last_clean":"2023-10-09T06:07:58.052186+0000","last_became_active":"2023-10-09T06:01:54.757727+0000","last_became_peered":"2023-10-09T06:01:54.757727+0000","last_unstale":"2023-10-09T06:07:58.052186+0000","last_undegraded":"2023-10-09T06:07:58.052186+0000","last_fullsized":"2023-10-09T06:07:58.052186+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:44.132556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":459,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.052779+0000","last_change":"2023-10-09T06:01:54.764989+0000","last_active":"2023-10-09T06:07:58.052779+0000","last_peered":"2023-10-09T06:07:58.052779+0000","last_clean":"2023-10-09T06:07:58.052779+0000","last_became_active":"2023-10-09T06:01:54.764790+0000","last_became_peered":"2023-10-09T06:01:54.764790+0000","last_unstale":"2023-10-09T06:07:58.052779+0000","last_undegraded":"2023-10-09T06:07:58.052779+0000","last_fullsized":"2023-10-09T06:07:58.052779+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:14:07.208858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":198,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.053156+0000","last_change":"2023-10-09T06:01:22.342190+0000","last_active":"2023-10-09T06:07:58.053156+0000","last_peered":"2023-10-09T06:07:58.053156+0000","last_clean":"2023-10-09T06:07:58.053156+0000","last_became_active":"2023-10-09T06:01:22.341855+0000","last_became_peered":"2023-10-09T06:01:22.341855+0000","last_unstale":"2023-10-09T06:07:58.053156+0000","last_undegraded":"2023-10-09T06:07:58.053156+0000","last_fullsized":"2023-10-09T06:07:58.053156+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:01:21.322371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":446,"reported_epoch":440,"state":"active+undersized","last_fresh":"2023-10-09T06:07:58.052098+0000","last_change":"2023-10-09T06:07:51.719512+0000","last_active":"2023-10-09T06:07:58.052098+0000","last_peered":"2023-10-09T06:07:58.052098+0000","last_clean":"2023-10-09T06:07:47.646429+0000","last_became_active":"2023-10-09T06:07:51.719512+0000","last_became_peered":"2023-10-09T06:07:51.719512+0000","last_unstale":"2023-10-09T06:07:58.052098+0000","last_undegraded":"2023-10-09T06:07:58.052098+0000","last_fullsized":"2023-10-09T06:07:51.697099+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:13.633296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":706,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-09T06:07:51.699754+0000","last_change":"2023-10-09T06:01:57.801991+0000","last_active":"2023-10-09T06:07:51.699754+0000","last_peered":"2023-10-09T06:07:51.699754+0000","last_clean":"2023-10-09T06:07:51.699754+0000","last_became_active":"2023-10-09T06:01:57.801579+0000","last_became_peered":"2023-10-09T06:01:57.801579+0000","last_unstale":"2023-10-09T06:07:51.699754+0000","last_undegraded":"2023-10-09T06:07:51.699754+0000","last_fullsized":"2023-10-09T06:07:51.699754+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:00.456132+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:00.456132+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:48.548353+0000","scrub_duration":0.0014992199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":235,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.049009+0000","last_change":"2023-10-09T06:07:43.579543+0000","last_active":"2023-10-09T06:07:58.049009+0000","last_peered":"2023-10-09T06:07:58.049009+0000","last_clean":"2023-10-09T06:07:58.049009+0000","last_became_active":"2023-10-09T06:07:43.579184+0000","last_became_peered":"2023-10-09T06:07:43.579184+0000","last_unstale":"2023-10-09T06:07:58.049009+0000","last_undegraded":"2023-10-09T06:07:58.049009+0000","last_fullsized":"2023-10-09T06:07:58.049009+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:23.460469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":231,"reported_epoch":440,"state":"active+undersized","last_fresh":"2023-10-09T06:07:58.045800+0000","last_change":"2023-10-09T06:07:51.715145+0000","last_active":"2023-10-09T06:07:58.045800+0000","last_peered":"2023-10-09T06:07:58.045800+0000","last_clean":"2023-10-09T06:07:47.629643+0000","last_became_active":"2023-10-09T06:07:51.715145+0000","last_became_peered":"2023-10-09T06:07:51.715145+0000","last_unstale":"2023-10-09T06:07:58.045800+0000","last_undegraded":"2023-10-09T06:07:58.045800+0000","last_fullsized":"2023-10-09T06:07:51.700531+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:18.521772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":728,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.051045+0000","last_change":"2023-10-09T06:07:06.560275+0000","last_active":"2023-10-09T06:07:58.051045+0000","last_peered":"2023-10-09T06:07:58.051045+0000","last_clean":"2023-10-09T06:07:58.051045+0000","last_became_active":"2023-10-09T06:07:06.559953+0000","last_became_peered":"2023-10-09T06:07:06.559953+0000","last_unstale":"2023-10-09T06:07:58.051045+0000","last_undegraded":"2023-10-09T06:07:58.051045+0000","last_fullsized":"2023-10-09T06:07:58.051045+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:46.290510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:46.290510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:10.910416+0000","scrub_duration":0.002627819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":46,"reported_epoch":440,"state":"active+undersized+degraded","last_fresh":"2023-10-09T06:07:58.058523+0000","last_change":"2023-10-09T06:07:51.710347+0000","last_active":"2023-10-09T06:07:58.058523+0000","last_peered":"2023-10-09T06:07:58.058523+0000","last_clean":"2023-10-09T05:40:15.633679+0000","last_became_active":"2023-10-09T06:07:51.710347+0000","last_became_peered":"2023-10-09T06:07:51.710347+0000","last_unstale":"2023-10-09T06:07:58.058523+0000","last_undegraded":"2023-10-09T06:07:51.703005+0000","last_fullsized":"2023-10-09T06:07:51.702895+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:04.366013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":462,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:07:58.051300+0000","last_change":"2023-10-09T06:06:01.068828+0000","last_active":"2023-10-09T06:07:58.051300+0000","last_peered":"2023-10-09T06:07:58.051300+0000","last_clean":"2023-10-09T06:07:58.051300+0000","last_became_active":"2023-10-09T06:06:01.068491+0000","last_became_peered":"2023-10-09T06:06:01.068491+0000","last_unstale":"2023-10-09T06:07:58.051300+0000","last_undegraded":"2023-10-09T06:07:58.051300+0000","last_fullsized":"2023-10-09T06:07:58.051300+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:25.661796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":84,"acting":84,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":354,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10938,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":84,"acting":84,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":635093,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":37,"num_read_kb":69,"num_write":18,"num_write_kb":183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":659551,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":82,"acting":82,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":4812767232,"num_objects":73437,"num_object_clones":0,"num_object_copies":220311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24996,"num_objects_misplaced":2158,"num_objects_unfound":0,"num_objects_dirty":73437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":73437,"num_write_kb":4699968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1087,"num_bytes_recovered":71237632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14318305280,"data_stored":14318305280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":73437,"ondisk_log_size":73437,"up":44,"acting":41,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":20,"acting":20,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234341,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3933364,"kb_used_data":3036652,"kb_used_omap":0,"kb_used_meta":896704,"kb_avail":89811788,"statfs":{"total":95995035648,"available":91967270912,"internally_reserved":0,"allocated":3109531648,"data_stored":3108268842,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918224896},"hb_peers":[1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":2.7919999999999998,"15min":3.2309999999999999},"min":{"1min":0.22700000000000001,"5min":0.218,"15min":0.218},"max":{"1min":5.0229999999999997,"5min":99.459999999999994,"15min":165.792},"last":0.45900000000000002},{"interface":"front","average":{"1min":1.5800000000000001,"5min":3.0670000000000002,"15min":3.3700000000000001},"min":{"1min":0.26700000000000002,"5min":0.24399999999999999,"15min":0.21299999999999999},"max":{"1min":9.7149999999999999,"5min":100.096,"15min":166.00700000000001},"last":0.52400000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:07:50 2023","interfaces":[{"interface":"back","average":{"1min":2.145,"5min":4.0430000000000001,"15min":2.2559999999999998},"min":{"1min":0.27300000000000002,"5min":0.25,"15min":0.249},"max":{"1min":18.614999999999998,"5min":64.760000000000005,"15min":64.760000000000005},"last":2.98},{"interface":"front","average":{"1min":2.2309999999999999,"5min":4.0439999999999996,"15min":2.1309999999999998},"min":{"1min":0.32200000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":21.341999999999999,"5min":65.102000000000004,"15min":65.102000000000004},"last":1.306}]},{"osd":2,"last update":"Mon Oct 9 06:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.27,"5min":1.6020000000000001,"15min":2.9180000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.8319999999999999,"5min":16.878,"15min":127.86499999999999},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.3680000000000001,"5min":1.7969999999999999,"15min":2.8650000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.29599999999999999},"max":{"1min":5.1699999999999999,"5min":12.456,"15min":129.24000000000001},"last":0.36399999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:07:50 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":2.52,"15min":4.625},"min":{"1min":0.35099999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":22.013000000000002,"5min":36.402000000000001,"15min":164.16999999999999},"last":2.8570000000000002},{"interface":"front","average":{"1min":2.0310000000000001,"5min":2.601,"15min":4.6369999999999996},"min":{"1min":0.39800000000000002,"5min":0.34399999999999997,"15min":0.30299999999999999},"max":{"1min":19.393000000000001,"5min":36.872,"15min":164.40700000000001},"last":2.4649999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:07:25 2023","interfaces":[{"interface":"back","average":{"1min":1.1579999999999999,"5min":0.85099999999999998,"15min":0.81899999999999995},"min":{"1min":0.27200000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.9189999999999996,"5min":5.5570000000000004,"15min":5.5570000000000004},"last":0.23100000000000001},{"interface":"front","average":{"1min":1.161,"5min":0.86499999999999999,"15min":0.85799999999999998},"min":{"1min":0.36499999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.2000000000000002,"5min":5.2000000000000002,"15min":5.2000000000000002},"last":0.27700000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:07:38 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.95899999999999996,"15min":0.875},"min":{"1min":0.30499999999999999,"5min":0.247,"15min":0.247},"max":{"1min":2.25,"5min":9.7910000000000004,"15min":15.622},"last":0.29299999999999998},{"interface":"front","average":{"1min":1.054,"5min":1.083,"15min":0.86299999999999999},"min":{"1min":0.35199999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":5.1369999999999996,"5min":9.3409999999999993,"15min":9.3409999999999993},"last":0.24399999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:07:38 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":1.1080000000000001,"15min":0.92300000000000004},"min":{"1min":0.35599999999999998,"5min":0.28499999999999998,"15min":0.28000000000000003},"max":{"1min":2.3439999999999999,"5min":6.6219999999999999,"15min":12.727},"last":0.26600000000000001},{"interface":"front","average":{"1min":1.198,"5min":1.0169999999999999,"15min":0.871},"min":{"1min":0.377,"5min":0.33400000000000002,"15min":0.313},"max":{"1min":4.9509999999999996,"5min":5.5800000000000001,"15min":5.5800000000000001},"last":0.30599999999999999}]}]},{"osd":6,"up_from":46,"seq":197568495978,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2010528,"kb_used_data":1108888,"kb_used_omap":0,"kb_used_meta":901632,"kb_avail":91734624,"statfs":{"total":95995035648,"available":93936254976,"internally_reserved":0,"allocated":1135501312,"data_stored":1134258031,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923271168},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":5.782,"15min":3.6629999999999998},"min":{"1min":0.27100000000000002,"5min":0.24299999999999999,"15min":0.224},"max":{"1min":5.0289999999999999,"5min":225.422,"15min":225.422},"last":0.29399999999999998},{"interface":"front","average":{"1min":2.0329999999999999,"5min":6.1470000000000002,"15min":3.794},"min":{"1min":0.28899999999999998,"5min":0.23000000000000001,"15min":0.22800000000000001},"max":{"1min":15.311999999999999,"5min":225.255,"15min":225.255},"last":0.38800000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:08:06 2023","interfaces":[{"interface":"back","average":{"1min":9.0449999999999999,"5min":8.0600000000000005,"15min":3.411},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":91.188000000000002,"5min":109.101,"15min":109.101},"last":1.6910000000000001},{"interface":"front","average":{"1min":8.4209999999999994,"5min":7.8799999999999999,"15min":3.4529999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27000000000000002},"max":{"1min":91.372,"5min":109.071,"15min":109.071},"last":0.39400000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:07:13 2023","interfaces":[{"interface":"back","average":{"1min":2.3500000000000001,"5min":1.976,"15min":2.0790000000000002},"min":{"1min":0.28299999999999997,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":15.374000000000001,"5min":15.374000000000001,"15min":61.088999999999999},"last":1.9710000000000001},{"interface":"front","average":{"1min":1.6399999999999999,"5min":1.7210000000000001,"15min":2.016},"min":{"1min":0.32800000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":15.429,"5min":15.429,"15min":61.082000000000001},"last":2.694}]},{"osd":3,"last update":"Mon Oct 9 06:08:02 2023","interfaces":[{"interface":"back","average":{"1min":8.1620000000000008,"5min":7.9269999999999996,"15min":5.8769999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.29599999999999999},"max":{"1min":84.573999999999998,"5min":138.13300000000001,"15min":138.13300000000001},"last":2.4660000000000002},{"interface":"front","average":{"1min":8.6120000000000001,"5min":8.1080000000000005,"15min":5.8490000000000002},"min":{"1min":0.41299999999999998,"5min":0.36599999999999999,"15min":0.28499999999999998},"max":{"1min":84.495999999999995,"5min":137.215,"15min":137.215},"last":2.387}]},{"osd":4,"last update":"Mon Oct 9 06:07:34 2023","interfaces":[{"interface":"back","average":{"1min":8.5039999999999996,"5min":2.8170000000000002,"15min":1.8029999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.24199999999999999},"max":{"1min":151.35599999999999,"5min":151.35599999999999,"15min":151.35599999999999},"last":0.27100000000000002},{"interface":"front","average":{"1min":8.6010000000000009,"5min":3.0009999999999999,"15min":1.732},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":149.15799999999999,"5min":149.15799999999999,"15min":149.15799999999999},"last":0.35699999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:08:06 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":1.423,"15min":1.1719999999999999},"min":{"1min":0.315,"5min":0.30299999999999999,"15min":0.23899999999999999},"max":{"1min":2.1379999999999999,"5min":15.342000000000001,"15min":45.933999999999997},"last":0.315},{"interface":"front","average":{"1min":0.72899999999999998,"5min":1.4370000000000001,"15min":1.1830000000000001},"min":{"1min":0.39400000000000002,"5min":0.29199999999999998,"15min":0.255},"max":{"1min":1.7330000000000001,"5min":15.358000000000001,"15min":45.917999999999999},"last":1.7330000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:07:34 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.333,"15min":1.2569999999999999},"min":{"1min":0.34899999999999998,"5min":0.312,"15min":0.27000000000000002},"max":{"1min":3.4020000000000001,"5min":21.98,"15min":52.042999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.4119999999999999,"15min":1.242},"min":{"1min":0.34200000000000003,"5min":0.33300000000000002,"15min":0.32200000000000001},"max":{"1min":3.7029999999999998,"5min":18.241,"15min":51.259},"last":1.7609999999999999}]}]},{"osd":1,"up_from":89,"seq":382252089567,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3158100,"kb_used_data":2709772,"kb_used_omap":9,"kb_used_meta":448310,"kb_avail":90587052,"statfs":{"total":95995035648,"available":92761141248,"internally_reserved":0,"allocated":2774806528,"data_stored":2772288220,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":459070441},"hb_peers":[2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:25 2023","interfaces":[{"interface":"back","average":{"1min":15.635999999999999,"5min":7.8259999999999996,"15min":4.2140000000000004},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.191},"max":{"1min":154.066,"5min":154.066,"15min":154.066},"last":0.77300000000000002},{"interface":"front","average":{"1min":15.811999999999999,"5min":7.8650000000000002,"15min":4.3879999999999999},"min":{"1min":0.22900000000000001,"5min":0.218,"15min":0.21299999999999999},"max":{"1min":156.19900000000001,"5min":156.19900000000001,"15min":156.19900000000001},"last":0.67800000000000005}]},{"osd":2,"last update":"Mon Oct 9 06:07:33 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":1.4970000000000001,"15min":3.194},"min":{"1min":0.27200000000000002,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":4.7149999999999999,"5min":11.631,"15min":204.989},"last":2.9830000000000001},{"interface":"front","average":{"1min":0.99399999999999999,"5min":1.831,"15min":3.4820000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20300000000000001},"max":{"1min":6.7519999999999998,"5min":12.728,"15min":203.65899999999999},"last":3.085}]},{"osd":3,"last update":"Mon Oct 9 06:07:55 2023","interfaces":[{"interface":"back","average":{"1min":1.466,"5min":3.7050000000000001,"15min":4.8029999999999999},"min":{"1min":0.27100000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.7489999999999997,"5min":142.97,"15min":213.654},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.5029999999999999,"5min":3.706,"15min":4.8179999999999996},"min":{"1min":0.28699999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.5999999999999996,"5min":142.94800000000001,"15min":213.77199999999999},"last":0.52000000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.72,"15min":1.262},"min":{"1min":0.33200000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.7990000000000004,"5min":10.782,"15min":10.782},"last":2.617},{"interface":"front","average":{"1min":1.452,"5min":1.7090000000000001,"15min":1.6339999999999999},"min":{"1min":0.377,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.9130000000000003,"5min":8.875,"15min":9.8629999999999995},"last":3.1059999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:07:42 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.9179999999999999,"15min":1.5129999999999999},"min":{"1min":0.35499999999999998,"5min":0.34300000000000003,"15min":0.32800000000000001},"max":{"1min":4.0179999999999998,"5min":36.356999999999999,"15min":64.584999999999994},"last":2.6560000000000001},{"interface":"front","average":{"1min":1.1040000000000001,"5min":2.2639999999999998,"15min":1.7809999999999999},"min":{"1min":0.47699999999999998,"5min":0.35799999999999998,"15min":0.315},"max":{"1min":3.6110000000000002,"5min":36.268000000000001,"15min":64.150999999999996},"last":3.145}]},{"osd":6,"last update":"Mon Oct 9 06:07:42 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.675,"15min":1.2869999999999999},"min":{"1min":0.44500000000000001,"5min":0.33100000000000002,"15min":0.30099999999999999},"max":{"1min":3.254,"5min":10.81,"15min":10.81},"last":2.7440000000000002},{"interface":"front","average":{"1min":1.1020000000000001,"5min":1.9710000000000001,"15min":1.55},"min":{"1min":0.42099999999999999,"5min":0.39900000000000002,"15min":0.35499999999999998},"max":{"1min":3.6760000000000002,"5min":9.2949999999999999,"15min":9.2949999999999999},"last":3.5609999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:07:42 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":2.3439999999999999,"15min":2.1840000000000002},"min":{"1min":0.48199999999999998,"5min":0.39400000000000002,"15min":0.314},"max":{"1min":3.6469999999999998,"5min":37.963999999999999,"15min":108.553},"last":3.1749999999999998},{"interface":"front","average":{"1min":1.034,"5min":1.9219999999999999,"15min":1.903},"min":{"1min":0.46600000000000003,"5min":0.39300000000000002,"15min":0.38},"max":{"1min":3.3889999999999998,"5min":37.950000000000003,"15min":108.56999999999999},"last":2.71}]}]},{"osd":0,"up_from":129,"seq":554050781365,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1578356,"kb_used_data":1147052,"kb_used_omap":29,"kb_used_meta":431266,"kb_avail":92166796,"statfs":{"total":95995035648,"available":94378799104,"internally_reserved":0,"allocated":1174581248,"data_stored":1172301441,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30322,"internal_metadata":441616782},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:07:20 2023","interfaces":[{"interface":"back","average":{"1min":12.545,"5min":7.6920000000000002,"15min":3.2989999999999999},"min":{"1min":0.223,"5min":0.189,"15min":0.189},"max":{"1min":189.327,"5min":223.09100000000001,"15min":223.09100000000001},"last":4.1319999999999997},{"interface":"front","average":{"1min":12.852,"5min":8.3219999999999992,"15min":3.5299999999999998},"min":{"1min":0.29399999999999998,"5min":0.24399999999999999,"15min":0.24099999999999999},"max":{"1min":192.80500000000001,"5min":223.161,"15min":223.161},"last":0.40999999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:07:20 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":1.627,"15min":1.615},"min":{"1min":0.29399999999999998,"5min":0.245,"15min":0.23000000000000001},"max":{"1min":2.8279999999999998,"5min":16.152999999999999,"15min":52.405999999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.71599999999999997,"5min":1.5920000000000001,"15min":1.863},"min":{"1min":0.254,"5min":0.24399999999999999,"15min":0.219},"max":{"1min":3.3050000000000002,"5min":22.844000000000001,"15min":52.645000000000003},"last":2.4020000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:07:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":2.661,"15min":3.3119999999999998},"min":{"1min":0.28699999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":3.9630000000000001,"5min":70.662000000000006,"15min":134.185},"last":1.994},{"interface":"front","average":{"1min":0.98399999999999999,"5min":2.5939999999999999,"15min":3.3610000000000002},"min":{"1min":0.316,"5min":0.25,"15min":0.25},"max":{"1min":3.589,"5min":70.575000000000003,"15min":134.16800000000001},"last":4.8239999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:07:40 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":3.3410000000000002,"15min":2.395},"min":{"1min":0.33400000000000002,"5min":0.307,"15min":0.307},"max":{"1min":3.4180000000000001,"5min":118.82299999999999,"15min":118.82299999999999},"last":2.4870000000000001},{"interface":"front","average":{"1min":0.84599999999999997,"5min":3.202,"15min":1.998},"min":{"1min":0.41499999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":2.8660000000000001,"5min":118.89400000000001,"15min":118.89400000000001},"last":1.6259999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:07:20 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":2.21,"15min":2.2370000000000001},"min":{"1min":0.35599999999999998,"5min":0.35199999999999998,"15min":0.314},"max":{"1min":5.2640000000000002,"5min":70.638000000000005,"15min":100.25},"last":2.4500000000000002},{"interface":"front","average":{"1min":0.69499999999999995,"5min":1.8200000000000001,"15min":1.915},"min":{"1min":0.44,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.04,"5min":70.518000000000001,"15min":102.40300000000001},"last":1.601}]},{"osd":6,"last update":"Mon Oct 9 06:07:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":2.3980000000000001,"15min":1.7769999999999999},"min":{"1min":0.38500000000000001,"5min":0.34399999999999997,"15min":0.33800000000000002},"max":{"1min":12.967000000000001,"5min":70.745999999999995,"15min":70.745999999999995},"last":2.5219999999999998},{"interface":"front","average":{"1min":1.1799999999999999,"5min":1.9730000000000001,"15min":1.55},"min":{"1min":0.46300000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":5.1900000000000004,"5min":70.597999999999999,"15min":70.597999999999999},"last":1.5720000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:07:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":2.0950000000000002,"15min":2.117},"min":{"1min":0.38600000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":2.129,"5min":70.840000000000003,"15min":103.866},"last":1.6499999999999999},{"interface":"front","average":{"1min":0.88700000000000001,"5min":1.4450000000000001,"15min":1.879},"min":{"1min":0.42699999999999999,"5min":0.41599999999999998,"15min":0.378},"max":{"1min":2.8559999999999999,"5min":7.6609999999999996,"15min":103.848},"last":0.46800000000000003}]}]},{"osd":2,"up_from":94,"seq":403726926058,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":876956,"kb_used_data":301268,"kb_used_omap":81,"kb_used_meta":575598,"kb_avail":92868196,"statfs":{"total":95995035648,"available":95097032704,"internally_reserved":0,"allocated":308498432,"data_stored":306317973,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":589412570},"hb_peers":[1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:30 2023","interfaces":[{"interface":"back","average":{"1min":1.6779999999999999,"5min":4.4050000000000002,"15min":3.4809999999999999},"min":{"1min":0.23899999999999999,"5min":0.20699999999999999,"15min":0.17999999999999999},"max":{"1min":12.58,"5min":167.44499999999999,"15min":167.44499999999999},"last":1.294},{"interface":"front","average":{"1min":1.3089999999999999,"5min":4.3639999999999999,"15min":3.452},"min":{"1min":0.29799999999999999,"5min":0.218,"15min":0.215},"max":{"1min":10.497999999999999,"5min":167.566,"15min":167.566},"last":8.0380000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:07:17 2023","interfaces":[{"interface":"back","average":{"1min":2.4590000000000001,"5min":5.8869999999999996,"15min":2.9369999999999998},"min":{"1min":0.23799999999999999,"5min":0.22900000000000001,"15min":0.21099999999999999},"max":{"1min":19.286000000000001,"5min":222.59399999999999,"15min":222.59399999999999},"last":0.19900000000000001},{"interface":"front","average":{"1min":1.869,"5min":5.8550000000000004,"15min":2.9089999999999998},"min":{"1min":0.315,"5min":0.23999999999999999,"15min":0.223},"max":{"1min":16.312999999999999,"5min":224.619,"15min":224.619},"last":0.41199999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:07:49 2023","interfaces":[{"interface":"back","average":{"1min":1.474,"5min":3.524,"15min":3.3290000000000002},"min":{"1min":0.31,"5min":0.246,"15min":0.24099999999999999},"max":{"1min":5.5419999999999998,"5min":58.271999999999998,"15min":144.88200000000001},"last":1.2989999999999999},{"interface":"front","average":{"1min":1.3799999999999999,"5min":3.294,"15min":3.2989999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.25900000000000001},"max":{"1min":5.6790000000000003,"5min":58.201000000000001,"15min":140.79499999999999},"last":1.02}]},{"osd":4,"last update":"Mon Oct 9 06:07:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4550000000000001,"5min":1.694,"15min":1.7130000000000001},"min":{"1min":0.48099999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":10.712,"5min":10.718,"15min":10.718},"last":0.48099999999999998},{"interface":"front","average":{"1min":1.4810000000000001,"5min":1.8240000000000001,"15min":1.8759999999999999},"min":{"1min":0.37,"5min":0.32600000000000001,"15min":0.31},"max":{"1min":5.5,"5min":12.643000000000001,"15min":12.643000000000001},"last":0.40100000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:07:18 2023","interfaces":[{"interface":"back","average":{"1min":2.5960000000000001,"5min":1.673,"15min":1.556},"min":{"1min":0.379,"5min":0.32600000000000001,"15min":0.30599999999999999},"max":{"1min":28.48,"5min":28.48,"15min":28.48},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.7770000000000001,"5min":2.0270000000000001,"15min":1.625},"min":{"1min":0.38,"5min":0.34300000000000003,"15min":0.28899999999999998},"max":{"1min":30.318999999999999,"5min":30.318999999999999,"15min":30.318999999999999},"last":0.36599999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:07:18 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":1.8560000000000001,"15min":1.726},"min":{"1min":0.46300000000000002,"5min":0.314,"15min":0.314},"max":{"1min":10.763999999999999,"5min":15.164999999999999,"15min":30.420000000000002},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.401,"5min":1.768,"15min":1.6459999999999999},"min":{"1min":0.40300000000000002,"5min":0.36299999999999999,"15min":0.32200000000000001},"max":{"1min":5.6189999999999998,"5min":8.3900000000000006,"15min":30.713000000000001},"last":0.498}]},{"osd":7,"last update":"Mon Oct 9 06:07:18 2023","interfaces":[{"interface":"back","average":{"1min":1.488,"5min":1.536,"15min":2.0150000000000001},"min":{"1min":0.42899999999999999,"5min":0.35999999999999999,"15min":0.33200000000000002},"max":{"1min":5.6589999999999998,"5min":6.5140000000000002,"15min":55.052},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.4299999999999999,"5min":1.9490000000000001,"15min":1.889},"min":{"1min":0.436,"5min":0.38200000000000001,"15min":0.35599999999999998},"max":{"1min":5.609,"5min":12.682,"15min":40.113},"last":0.438}]}]},{"osd":3,"up_from":99,"seq":425201762532,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3369824,"kb_used_data":2655384,"kb_used_omap":7,"kb_used_meta":714424,"kb_avail":90375328,"statfs":{"total":95995035648,"available":92544335872,"internally_reserved":0,"allocated":2719113216,"data_stored":2716636856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":731570792},"hb_peers":[1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:33 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":5.343,"15min":3.331},"min":{"1min":0.223,"5min":0.20300000000000001,"15min":0.186},"max":{"1min":3.7240000000000002,"5min":131.363,"15min":131.363},"last":5.5119999999999996},{"interface":"front","average":{"1min":0.94899999999999995,"5min":4.8479999999999999,"15min":3.3530000000000002},"min":{"1min":0.253,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":2.6709999999999998,"5min":131.26300000000001,"15min":131.26300000000001},"last":1.9039999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:07:55 2023","interfaces":[{"interface":"back","average":{"1min":2.0979999999999999,"5min":4.4809999999999999,"15min":2.4670000000000001},"min":{"1min":0.255,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":11.936999999999999,"5min":207.75299999999999,"15min":207.75299999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":1.601,"5min":4.6210000000000004,"15min":2.8740000000000001},"min":{"1min":0.31900000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":6.9980000000000002,"5min":207.12200000000001,"15min":207.12200000000001},"last":0.60799999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:07:55 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":2.0619999999999998,"15min":2.2269999999999999},"min":{"1min":0.26100000000000001,"5min":0.223,"15min":0.223},"max":{"1min":2.855,"5min":41.945999999999998,"15min":107.01300000000001},"last":2.3769999999999998},{"interface":"front","average":{"1min":2.0800000000000001,"5min":2.5470000000000002,"15min":2.4940000000000002},"min":{"1min":0.31,"5min":0.28100000000000003,"15min":0.249},"max":{"1min":12.791,"5min":41.475999999999999,"15min":109.807},"last":0.58399999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2150000000000001,"5min":1.8420000000000001,"15min":5.3499999999999996},"min":{"1min":0.40100000000000002,"5min":0.32100000000000001,"15min":0.254},"max":{"1min":5.5,"5min":20.469000000000001,"15min":99.876000000000005},"last":5.2919999999999998},{"interface":"front","average":{"1min":0.86699999999999999,"5min":2.1219999999999999,"15min":5.1440000000000001},"min":{"1min":0.35799999999999998,"5min":0.316,"15min":0.316},"max":{"1min":2.956,"5min":20.454999999999998,"15min":99.914000000000001},"last":0.627}]},{"osd":5,"last update":"Mon Oct 9 06:07:55 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.5629999999999999,"15min":1.506},"min":{"1min":0.30599999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.5149999999999997,"5min":27.937000000000001,"15min":27.937000000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.0269999999999999,"5min":4.2919999999999998,"15min":2.4380000000000002},"min":{"1min":0.371,"5min":0.35599999999999998,"15min":0.28199999999999997},"max":{"1min":3.7000000000000002,"5min":235.56,"15min":235.56},"last":5.4080000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:07:55 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":5.2350000000000003,"15min":4.3319999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33100000000000002},"max":{"1min":4.4710000000000001,"5min":233.26599999999999,"15min":233.26599999999999},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.0249999999999999,"5min":5.5540000000000003,"15min":4.4580000000000002},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.318},"max":{"1min":2.9769999999999999,"5min":233.24100000000001,"15min":233.24100000000001},"last":5.3449999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:07:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":4.1310000000000002,"15min":3.9660000000000002},"min":{"1min":0.42299999999999999,"5min":0.39000000000000001,"15min":0.34200000000000003},"max":{"1min":3.5339999999999998,"5min":230.69399999999999,"15min":230.69399999999999},"last":5.3799999999999999},{"interface":"front","average":{"1min":1.149,"5min":4.1890000000000001,"15min":3.9369999999999998},"min":{"1min":0.45500000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":5.5739999999999998,"5min":230.67500000000001,"15min":230.67500000000001},"last":0.74099999999999999}]}]},{"osd":4,"up_from":322,"seq":1382979469395,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1218428,"kb_used_data":896052,"kb_used_omap":35,"kb_used_meta":322332,"kb_avail":92526724,"statfs":{"total":95995035648,"available":94747365376,"internally_reserved":0,"allocated":917557248,"data_stored":916294176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36552,"internal_metadata":330068280},"hb_peers":[1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:32 2023","interfaces":[{"interface":"back","average":{"1min":3.855,"5min":2.6339999999999999,"15min":1.722},"min":{"1min":0.23000000000000001,"5min":0.223,"15min":0.223},"max":{"1min":50.341999999999999,"5min":50.341999999999999,"15min":50.341999999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":3.7850000000000001,"5min":2.6600000000000001,"15min":2.1139999999999999},"min":{"1min":0.26700000000000002,"5min":0.247,"15min":0.247},"max":{"1min":50.454999999999998,"5min":50.454999999999998,"15min":50.454999999999998},"last":0.372}]},{"osd":1,"last update":"Mon Oct 9 06:07:32 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":3.2410000000000001,"15min":2.3279999999999998},"min":{"1min":0.27600000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.8319999999999999,"5min":78.891000000000005,"15min":78.891000000000005},"last":0.29399999999999998},{"interface":"front","average":{"1min":1.369,"5min":3.3500000000000001,"15min":2.1360000000000001},"min":{"1min":0.27600000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":6.734,"5min":78.921000000000006,"15min":78.921000000000006},"last":0.45600000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:07:32 2023","interfaces":[{"interface":"back","average":{"1min":1.4359999999999999,"5min":1.6259999999999999,"15min":1.798},"min":{"1min":0.41999999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.7060000000000004,"5min":7.9189999999999996,"15min":7.9189999999999996},"last":1.1659999999999999},{"interface":"front","average":{"1min":1.198,"5min":1.448,"15min":1.9790000000000001},"min":{"1min":0.33300000000000002,"5min":0.312,"15min":0.28199999999999997},"max":{"1min":6.6440000000000001,"5min":8.1069999999999993,"15min":11.381},"last":0.54900000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:07:32 2023","interfaces":[{"interface":"back","average":{"1min":1.958,"5min":2.9390000000000001,"15min":2.3839999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":9.8219999999999992,"5min":58.398000000000003,"15min":58.398000000000003},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.1870000000000001,"5min":2.3759999999999999,"15min":1.964},"min":{"1min":0.36399999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":6.7679999999999998,"5min":58.305999999999997,"15min":58.305999999999997},"last":1.1339999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:07:32 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.0880000000000001,"15min":0.97299999999999998},"min":{"1min":0.29499999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":6.5119999999999996,"5min":6.5119999999999996,"15min":6.5119999999999996},"last":0.308},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.238,"15min":1.4119999999999999},"min":{"1min":0.27900000000000003,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.2809999999999997,"5min":12.087,"15min":12.087},"last":0.32900000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:07:32 2023","interfaces":[{"interface":"back","average":{"1min":5.6920000000000002,"5min":2.2229999999999999,"15min":3.597},"min":{"1min":0.33100000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":95.492999999999995,"5min":95.492999999999995,"15min":95.492999999999995},"last":0.58099999999999996},{"interface":"front","average":{"1min":5.6619999999999999,"5min":2.3090000000000002,"15min":3.7909999999999999},"min":{"1min":0.36499999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":95.664000000000001,"5min":95.664000000000001,"15min":95.664000000000001},"last":0.36099999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:07:32 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":1.3029999999999999,"15min":4.7809999999999997},"min":{"1min":0.34000000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":7.056,"5min":8.5540000000000003,"15min":79.173000000000002},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.216,"5min":1.2889999999999999,"15min":3.9289999999999998},"min":{"1min":0.38300000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.681,"5min":6.681,"15min":78.894000000000005},"last":0.67500000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757615,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2973548,"kb_used_data":2087332,"kb_used_omap":0,"kb_used_meta":886208,"kb_avail":90771604,"statfs":{"total":95995035648,"available":92950122496,"internally_reserved":0,"allocated":2137427968,"data_stored":2136177498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":907476992},"hb_peers":[1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:37 2023","interfaces":[{"interface":"back","average":{"1min":1.1639999999999999,"5min":2.1619999999999999,"15min":1.704},"min":{"1min":0.26700000000000002,"5min":0.23499999999999999,"15min":0.22500000000000001},"max":{"1min":3.5139999999999998,"5min":69.088999999999999,"15min":69.088999999999999},"last":1.6899999999999999},{"interface":"front","average":{"1min":0.93700000000000006,"5min":2.2330000000000001,"15min":1.9259999999999999},"min":{"1min":0.25900000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.637,"5min":69.073999999999998,"15min":69.073999999999998},"last":1.704}]},{"osd":1,"last update":"Mon Oct 9 06:07:55 2023","interfaces":[{"interface":"back","average":{"1min":14.262,"5min":5.6539999999999999,"15min":3.488},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":132.93299999999999,"5min":132.93299999999999,"15min":132.93299999999999},"last":2.8159999999999998},{"interface":"front","average":{"1min":14.57,"5min":5.9569999999999999,"15min":3.3490000000000002},"min":{"1min":0.315,"5min":0.27600000000000002,"15min":0.26400000000000001},"max":{"1min":133.92099999999999,"5min":133.92099999999999,"15min":133.92099999999999},"last":3.3319999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:07:20 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":2.9529999999999998,"15min":3.3639999999999999},"min":{"1min":0.28499999999999998,"5min":0.254,"15min":0.254},"max":{"1min":9.0800000000000001,"5min":129.21199999999999,"15min":217.06399999999999},"last":2.7109999999999999},{"interface":"front","average":{"1min":1.956,"5min":3.2370000000000001,"15min":3.2919999999999998},"min":{"1min":0.43099999999999999,"5min":0.32300000000000001,"15min":0.27600000000000002},"max":{"1min":9.5310000000000006,"5min":129.042,"15min":217.018},"last":2.891}]},{"osd":3,"last update":"Mon Oct 9 06:07:55 2023","interfaces":[{"interface":"back","average":{"1min":12.901,"5min":5.7220000000000004,"15min":6.4610000000000003},"min":{"1min":0.39200000000000002,"5min":0.312,"15min":0.27500000000000002},"max":{"1min":111.596,"5min":111.596,"15min":211.40199999999999},"last":3.1789999999999998},{"interface":"front","average":{"1min":12.468999999999999,"5min":5.5090000000000003,"15min":6.4850000000000003},"min":{"1min":0.31900000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":111.646,"5min":111.646,"15min":211.578},"last":5.4039999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:07:37 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":1.107,"15min":0.97499999999999998},"min":{"1min":0.27900000000000003,"5min":0.27400000000000002,"15min":0.249},"max":{"1min":6.4489999999999998,"5min":8.1539999999999999,"15min":8.1539999999999999},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.284,"5min":1.0669999999999999,"15min":1.1020000000000001},"min":{"1min":0.312,"5min":0.27000000000000002,"15min":0.26000000000000001},"max":{"1min":8.7059999999999995,"5min":8.7059999999999995,"15min":8.7059999999999995},"last":0.32100000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:07:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.2849999999999999,"15min":1.419},"min":{"1min":0.35499999999999998,"5min":0.29099999999999998,"15min":0.251},"max":{"1min":5.2169999999999996,"5min":8.6809999999999992,"15min":126.89400000000001},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.3500000000000001,"15min":1.4430000000000001},"min":{"1min":0.32300000000000001,"5min":0.28000000000000003,"15min":0.26200000000000001},"max":{"1min":4.0970000000000004,"5min":9.9009999999999998,"15min":126.616},"last":0.30599999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:07:37 2023","interfaces":[{"interface":"back","average":{"1min":1.425,"5min":1.4299999999999999,"15min":2.0329999999999999},"min":{"1min":0.378,"5min":0.317,"15min":0.29999999999999999},"max":{"1min":8.2270000000000003,"5min":9.9380000000000006,"15min":133.27099999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.089,"5min":1.3380000000000001,"15min":1.8440000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.29799999999999999},"max":{"1min":3.742,"5min":9.9529999999999994,"15min":135.09200000000001},"last":0.38200000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1474,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4889,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1179123712,"data_stored":1179123712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2770206720,"data_stored":2770206720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":311623680,"data_stored":311623680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2739601408,"data_stored":2739601408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":916783104,"data_stored":916783104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2136408064,"data_stored":2136408064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1136656384,"data_stored":1136656384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3127902208,"data_stored":3127902208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:08:13.300 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T06:08:13.300 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 51 2023-10-09T06:08:13.313 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:13.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:13.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:13.437 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:13.512 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:13.740 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-10-09T06:08:13.735350+0000 min lat: 0.00120925 max lat: 2.19099 avg lat: 0.0123615 2023-10-09T06:08:13.741 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:08:13.741 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 60 16 77706 77690 80.8639 45 0.0163293 0.0123615 2023-10-09T06:08:13.806 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:13.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:13.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:13.914 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:08:13.935 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:13.936 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:13.946 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:14.003 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:08:14.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:14.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:08:14.086 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:08:14.086 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T06:08:14.086 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T06:08:14.087 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T06:08:14.087 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:08:14.109 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:14.153 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:14.215 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:14.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:14.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:14.323 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:14.378 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:14.450 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:14.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:14.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:14.556 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:08:14.600 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:14.685 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:08:14.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:14.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:08:14.703 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:14 smithi064 ceph-mon[220044]: pgmap v1091: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s; 25555/225357 objects degraded (11.340%); 2158/225357 objects misplaced (0.958%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T06:08:14.703 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:14.704 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:14 smithi064 ceph-mon[212750]: pgmap v1091: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s; 25555/225357 objects degraded (11.340%); 2158/225357 objects misplaced (0.958%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T06:08:14.704 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:14.734 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 61 16 78607 78591 80.4616 56.3125 0.00975756 0.0124234 2023-10-09T06:08:14.792 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:14.842 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:14 smithi156 ceph-mon[195101]: pgmap v1091: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s; 25555/225357 objects degraded (11.340%); 2158/225357 objects misplaced (0.958%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T06:08:14.842 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:14.848 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:14.911 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:14.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:14.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:15.020 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:08:15.061 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:15.103 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:08:15.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:15.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:08:15.211 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:15.244 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:15.308 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:15.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:15.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:15.415 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:15.451 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:15.549 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:15.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:15.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:15.567 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:08:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:08:15] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.43.0" 2023-10-09T06:08:15.656 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:15.707 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:15.734 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 62 16 79326 79310 79.8889 44.9375 0.00530281 0.0125057 2023-10-09T06:08:16.219 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:08:16.275 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:16.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:16.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:16.383 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:16.518 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:16 smithi064 ceph-mon[212750]: pgmap v1092: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s; 26094/229317 objects degraded (11.379%); 1903/229317 objects misplaced (0.830%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T06:08:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:16 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:08:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:16 smithi064 ceph-mon[220044]: pgmap v1092: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s; 26094/229317 objects degraded (11.379%); 1903/229317 objects misplaced (0.830%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T06:08:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:16 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:08:16.735 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 63 16 80118 80102 79.4068 49.5 0.00715275 0.0125879 2023-10-09T06:08:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:16 smithi156 ceph-mon[195101]: pgmap v1092: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s; 26094/229317 objects degraded (11.379%); 1903/229317 objects misplaced (0.830%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T06:08:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:16 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:08:16.920 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:16.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:16.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:17.032 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:17.052 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:17.158 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:17.273 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:17.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:17.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:17.382 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:17.462 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:17.483 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:17.572 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:17 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 26094/229317 objects degraded (11.379%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:17.572 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:17 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 26094/229317 objects degraded (11.379%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:17.660 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:08:17.693 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:17.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:17.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:17.704 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:08:17.735 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 64 16 80882 80866 78.9124 47.75 0.0129792 0.0126667 2023-10-09T06:08:17.752 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:08:17.791 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:08:17.804 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:17.843 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:08:17.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:17 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 26094/229317 objects degraded (11.379%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:17.881 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:17.883 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:08:17.921 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:08:17.921 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-10-09T06:08:17.922 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:08:17.922 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:08:17.922 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:08:17.922 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:08:17.922 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:08:17.922 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:08:17.922 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:08:18.735 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 65 16 81710 81694 78.4947 51.75 0.00823678 0.0127342 2023-10-09T06:08:18.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:18 smithi156 ceph-mon[195101]: pgmap v1093: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s; 26605/237558 objects degraded (11.199%); 1903/237558 objects misplaced (0.801%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T06:08:18.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:18 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3691100723' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:18.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:18 smithi064 ceph-mon[212750]: pgmap v1093: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s; 26605/237558 objects degraded (11.199%); 1903/237558 objects misplaced (0.801%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T06:08:18.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:18 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3691100723' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:18.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:18 smithi064 ceph-mon[220044]: pgmap v1093: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s; 26605/237558 objects degraded (11.199%); 1903/237558 objects misplaced (0.801%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T06:08:18.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:18 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3691100723' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:19.622 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T06:08:19.622 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T06:08:19.622 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:08:19.735 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 66 16 82401 82385 77.9601 43.1875 0.0148339 0.0128224 2023-10-09T06:08:19.771 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:19.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:19.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:19.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:19 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/761669689' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:08:19.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:19 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:08:19.885 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:08:19.959 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:19 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/761669689' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:08:19.959 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:19 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:08:19.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:19 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/761669689' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:08:19.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:19 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:08:19.960 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:20.034 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:08:20.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:20.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:08:20.138 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:20.174 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:20.217 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:20.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:20.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:20.325 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:20.359 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:20.434 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:20.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:20.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:20.546 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:08:20.588 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:20.648 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:08:20.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:20.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:08:20.735 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 67 16 83142 83126 77.488 46.3125 0.0135135 0.0128992 2023-10-09T06:08:20.754 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:08:20.791 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:20 smithi156 ceph-mon[195101]: pgmap v1094: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s; 26859/238320 objects degraded (11.270%); 1903/238320 objects misplaced (0.799%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T06:08:20.791 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:20 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:08:20.791 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:20 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:08:20.791 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:20 smithi156 ceph-mon[195101]: osdmap e442: 8 total, 7 up, 8 in 2023-10-09T06:08:20.791 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:20 smithi156 ceph-mon[195101]: failed to encode map e442 with expected crc 2023-10-09T06:08:20.792 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:20 smithi156 ceph-mon[195101]: failed to encode map e442 with expected crc 2023-10-09T06:08:20.792 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:20 smithi156 ceph-mon[195101]: failed to encode map e442 with expected crc 2023-10-09T06:08:20.792 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:20 smithi156 ceph-mon[195101]: failed to encode map e442 with expected crc 2023-10-09T06:08:20.792 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:20 smithi156 ceph-mon[195101]: failed to encode map e442 with expected crc 2023-10-09T06:08:20.792 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:20 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:08:20.792 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:20 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:08:20.794 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:20.871 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:08:20.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:20.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:08:20.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:20 smithi064 ceph-mon[212750]: pgmap v1094: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s; 26859/238320 objects degraded (11.270%); 1903/238320 objects misplaced (0.799%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T06:08:20.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:20 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:08:20.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:20 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:08:20.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:20 smithi064 ceph-mon[212750]: osdmap e442: 8 total, 7 up, 8 in 2023-10-09T06:08:20.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:20 smithi064 ceph-mon[212750]: failed to encode map e442 with expected crc 2023-10-09T06:08:20.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:20 smithi064 ceph-mon[212750]: failed to encode map e442 with expected crc 2023-10-09T06:08:20.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:20 smithi064 ceph-mon[212750]: failed to encode map e442 with expected crc 2023-10-09T06:08:20.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:20 smithi064 ceph-mon[212750]: failed to encode map e442 with expected crc 2023-10-09T06:08:20.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:20 smithi064 ceph-mon[212750]: failed to encode map e442 with expected crc 2023-10-09T06:08:20.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:20 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:08:20.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:20 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:08:20.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:20 smithi064 ceph-mon[220044]: pgmap v1094: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s; 26859/238320 objects degraded (11.270%); 1903/238320 objects misplaced (0.799%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T06:08:20.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:20 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:08:20.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:20 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:08:20.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:20 smithi064 ceph-mon[220044]: osdmap e442: 8 total, 7 up, 8 in 2023-10-09T06:08:20.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:20 smithi064 ceph-mon[220044]: failed to encode map e442 with expected crc 2023-10-09T06:08:20.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:20 smithi064 ceph-mon[220044]: failed to encode map e442 with expected crc 2023-10-09T06:08:20.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:20 smithi064 ceph-mon[220044]: failed to encode map e442 with expected crc 2023-10-09T06:08:20.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:20 smithi064 ceph-mon[220044]: failed to encode map e442 with expected crc 2023-10-09T06:08:20.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:20 smithi064 ceph-mon[220044]: failed to encode map e442 with expected crc 2023-10-09T06:08:20.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:20 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:08:20.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:20 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:08:20.975 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:21.027 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:21.107 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:21.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:21.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:21.214 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:21.528 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:21.623 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:08:21.739 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 68 16 84088 84072 77.2138 59.125 0.00648776 0.0129445 2023-10-09T06:08:22.003 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:22.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:22.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:22.112 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:22.239 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:22.402 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:22.414 INFO:teuthology.orchestra.run.smithi064.stderr:Error EINVAL: specified pg_num 51 < pg_num_max 32 2023-10-09T06:08:22.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:22.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:22.517 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:22.606 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:22.739 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 69 16 84786 84770 76.7272 43.625 0.00889726 0.0130282 2023-10-09T06:08:22.785 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:08:22.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:22 smithi064 ceph-mon[220044]: pgmap v1096: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 808 op/s; 27087/244659 objects degraded (11.071%); 1656/244659 objects misplaced (0.677%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T06:08:22.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:22 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1322313124' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-10-09T06:08:22.850 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:22 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 27087/244659 objects degraded (11.071%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:22.853 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:22 smithi064 ceph-mon[212750]: pgmap v1096: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 808 op/s; 27087/244659 objects degraded (11.071%); 1656/244659 objects misplaced (0.677%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T06:08:22.853 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:22 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1322313124' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-10-09T06:08:22.853 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:22 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 27087/244659 objects degraded (11.071%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:22.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:22 smithi156 ceph-mon[195101]: pgmap v1096: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 808 op/s; 27087/244659 objects degraded (11.071%); 1656/244659 objects misplaced (0.677%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T06:08:22.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:22 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1322313124' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-10-09T06:08:22.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:22 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 27087/244659 objects degraded (11.071%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:23.032 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:08:23.032 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:08:23.484 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:23.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:23.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:23.590 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:08:23.635 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:23.687 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:08:23.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:23.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:08:23.739 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 70 16 85501 85485 76.2698 44.6875 0.0773607 0.0130994 2023-10-09T06:08:23.794 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:23.820 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:23.871 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:23.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:23.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:23.987 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:24.001 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T06:08:24.079 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:24.275 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:24.303 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:24.310 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:08:24.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:24.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:24.413 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:24.431 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:24.496 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:24.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:24 smithi156 ceph-mon[195101]: pgmap v1097: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 789 op/s; 27568/249057 objects degraded (11.069%); 1656/249057 objects misplaced (0.665%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T06:08:24.651 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:24.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:24.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:24.740 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 71 16 86257 86241 75.8613 47.25 0.00485129 0.0131749 2023-10-09T06:08:24.765 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:24.778 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:08:24.800 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:24.836 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:24 smithi064 ceph-mon[220044]: pgmap v1097: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 789 op/s; 27568/249057 objects degraded (11.069%); 1656/249057 objects misplaced (0.665%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T06:08:24.843 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:24 smithi064 ceph-mon[212750]: pgmap v1097: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 789 op/s; 27568/249057 objects degraded (11.069%); 1656/249057 objects misplaced (0.665%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T06:08:24.908 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:24.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:24.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:25.015 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:25.079 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:25.660 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:08:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:08:25] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.43.0" 2023-10-09T06:08:25.742 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 72 16 87016 87000 75.4634 47.4375 0.0163491 0.0132461 2023-10-09T06:08:26.017 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:26.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:26.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:26.138 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:26.166 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:26.328 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:26.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:26.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:26.434 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:26.468 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:26.529 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:26.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:26.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:26.636 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:26.666 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:26.728 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:26.730 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:26 smithi156 ceph-mon[195101]: pgmap v1098: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 777 op/s; 28050/252651 objects degraded (11.102%); 1406/252651 objects misplaced (0.556%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:08:26.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:26.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:26.742 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 73 16 87794 87778 75.0959 48.625 0.0109297 0.0133099 2023-10-09T06:08:26.832 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:26.864 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:26 smithi064 ceph-mon[212750]: pgmap v1098: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 777 op/s; 28050/252651 objects degraded (11.102%); 1406/252651 objects misplaced (0.556%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:08:26.864 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:26 smithi064 ceph-mon[220044]: pgmap v1098: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 777 op/s; 28050/252651 objects degraded (11.102%); 1406/252651 objects misplaced (0.556%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:08:26.872 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:26.965 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:26.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:26.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:27.075 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:27.166 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:27.226 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:08:27.281 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:08:27.300 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:27.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:27.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:27.318 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:08:27.356 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:08:27.397 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:08:27.419 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:27.433 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:08:27.477 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:08:27.477 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-10-09T06:08:27.478 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:08:27.478 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:08:27.478 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:08:27.478 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:08:27.478 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:08:27.478 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:08:27.478 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:08:27.502 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:27.717 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:27.736 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:27 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3095415444' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:27.736 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:27 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 28050/252651 objects degraded (11.102%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:27.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:27.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:27.738 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:27 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3095415444' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:27.738 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:27 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 28050/252651 objects degraded (11.102%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:27.743 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 74 16 88582 88566 74.7468 49.25 0.0423833 0.0133712 2023-10-09T06:08:27.838 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:27.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:27 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3095415444' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:27.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:27 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 28050/252651 objects degraded (11.102%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:27.917 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:28.393 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:28.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:28.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:28.503 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:28.587 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:28.647 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:08:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:28 smithi064 ceph-mon[212750]: pgmap v1099: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s; 28517/260625 objects degraded (10.942%); 1406/260625 objects misplaced (0.539%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:08:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:28 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1534202868' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:08:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:28 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:08:28.688 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:28 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:08:28.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:28 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:28.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:28 smithi064 ceph-mon[220044]: pgmap v1099: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s; 28517/260625 objects degraded (10.942%); 1406/260625 objects misplaced (0.539%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:08:28.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:28 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1534202868' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:08:28.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:28 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:08:28.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:28 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:08:28.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:28 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:28.742 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 75 16 89306 89290 74.3537 45.25 0.0120022 0.0134418 2023-10-09T06:08:28.748 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T06:08:28.748 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T06:08:28.951 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:28 smithi156 ceph-mon[195101]: pgmap v1099: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s; 28517/260625 objects degraded (10.942%); 1406/260625 objects misplaced (0.539%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:08:28.951 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:28 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1534202868' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:08:28.951 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:28 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:08:28.951 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:28 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:08:28.951 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:28 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:28.966 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:28.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:28.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:29.003 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1, 2] out_osds: [] dead_osds: [0] live_osds: [1, 2, 3, 5, 6, 7, 4] 2023-10-09T06:08:29.003 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:08:29.003 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-10-09T06:08:29.003 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:08:29.003 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:08:29.004 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-10-09T06:08:29.078 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:29.117 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:29.161 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:29.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:29.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:29.206 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:08:29.206 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:08:29.206 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:29.207 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.35940.0:89602 9.6 e443/433 9:6b34b1f9:::benchmark_data_smithi064_511911_object89601:head v 443'5598, mlcod=0'0)", 2023-10-09T06:08:29.207 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:08:29.205203+0000", 2023-10-09T06:08:29.207 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0011179510000000001, 2023-10-09T06:08:29.207 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0011385029999999999, 2023-10-09T06:08:29.207 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:29.207 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:08:29.207 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:29.207 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:29.207 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:29.207 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:29.205203+0000", 2023-10-09T06:08:29.207 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:29.208 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:29.208 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:29.208 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:29.208 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:29.205203+0000", 2023-10-09T06:08:29.208 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:29.208 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:29.208 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:29.208 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:29.209 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:29.205202+0000", 2023-10-09T06:08:29.209 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:08:29.209 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:29.209 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:29.209 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:29.209 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:29.205262+0000", 2023-10-09T06:08:29.209 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9749e-05 2023-10-09T06:08:29.209 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:29.209 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:29.209 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:29.209 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:29.205274+0000", 2023-10-09T06:08:29.209 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2048000000000001e-05 2023-10-09T06:08:29.209 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:29.210 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:29.210 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:29.210 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:29.205278+0000", 2023-10-09T06:08:29.210 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3460000000000004e-06 2023-10-09T06:08:29.210 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:29.210 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:29.210 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:29.210 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:29.205313+0000", 2023-10-09T06:08:29.210 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.4785e-05 2023-10-09T06:08:29.210 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:29.210 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:29.211 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:29.211 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:29.205340+0000", 2023-10-09T06:08:29.211 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.6998000000000001e-05 2023-10-09T06:08:29.211 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:29.211 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:29.211 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:29.211 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:29.211 INFO:teuthology.orchestra.run.smithi156.stdout: ], 2023-10-09T06:08:29.211 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 1 2023-10-09T06:08:29.211 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:08:29.265 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:29.359 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:29.599 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:29.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:29.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:29.710 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:29.735 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:29.743 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 76 16 90019 90003 73.962 44.5625 0.0038864 0.0135048 2023-10-09T06:08:29.777 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:29.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:29.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:29.881 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:08:29.919 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:29 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:08:29.919 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:29 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:08:29.919 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:29 smithi156 ceph-mon[195101]: osdmap e443: 8 total, 7 up, 8 in 2023-10-09T06:08:29.919 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:29 smithi156 ceph-mon[195101]: failed to encode map e443 with expected crc 2023-10-09T06:08:29.920 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:29 smithi156 ceph-mon[195101]: failed to encode map e443 with expected crc 2023-10-09T06:08:29.920 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:29 smithi156 ceph-mon[195101]: failed to encode map e443 with expected crc 2023-10-09T06:08:29.920 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:29 smithi156 ceph-mon[195101]: failed to encode map e443 with expected crc 2023-10-09T06:08:29.920 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:29 smithi156 ceph-mon[195101]: pgmap v1101: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s; 28751/261327 objects degraded (11.002%); 1406/261327 objects misplaced (0.538%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:08:29.920 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:29.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:29 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:08:29.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:29 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:08:29.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:29 smithi064 ceph-mon[212750]: osdmap e443: 8 total, 7 up, 8 in 2023-10-09T06:08:29.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:29 smithi064 ceph-mon[212750]: failed to encode map e443 with expected crc 2023-10-09T06:08:29.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:29 smithi064 ceph-mon[212750]: failed to encode map e443 with expected crc 2023-10-09T06:08:29.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:29 smithi064 ceph-mon[212750]: failed to encode map e443 with expected crc 2023-10-09T06:08:29.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:29 smithi064 ceph-mon[212750]: failed to encode map e443 with expected crc 2023-10-09T06:08:29.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:29 smithi064 ceph-mon[212750]: pgmap v1101: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s; 28751/261327 objects degraded (11.002%); 1406/261327 objects misplaced (0.538%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:08:29.977 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:29 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:08:29.977 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:29 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:08:29.977 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:29 smithi064 ceph-mon[220044]: osdmap e443: 8 total, 7 up, 8 in 2023-10-09T06:08:29.977 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:29 smithi064 ceph-mon[220044]: failed to encode map e443 with expected crc 2023-10-09T06:08:29.977 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:29 smithi064 ceph-mon[220044]: failed to encode map e443 with expected crc 2023-10-09T06:08:29.977 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:29 smithi064 ceph-mon[220044]: failed to encode map e443 with expected crc 2023-10-09T06:08:29.977 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:29 smithi064 ceph-mon[220044]: failed to encode map e443 with expected crc 2023-10-09T06:08:29.977 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:29 smithi064 ceph-mon[220044]: pgmap v1101: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s; 28751/261327 objects degraded (11.002%); 1406/261327 objects misplaced (0.538%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:08:29.978 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:30.189 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:08:30.192 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:08:30.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:30.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:08:30.303 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:30.379 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:30.470 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:30.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:30.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:30.587 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:30.638 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:30.715 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:30.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:30.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:30.743 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 77 16 90817 90801 73.6493 49.875 0.0150144 0.0135727 2023-10-09T06:08:30.749 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:08:30.774 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:08:30.821 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:30.910 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:31.601 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:08:31.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:31 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:08:31.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:31 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:08:31.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:31 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:08:31.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:31 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:08:31.635 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:31 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:08:31.635 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:31 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:08:31.680 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:31.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:31.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:31.743 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 78 16 91386 91370 73.1613 35.5625 0.0103079 0.013638 2023-10-09T06:08:31.785 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:31.875 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:31.903 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:32.125 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:32.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:32.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:32.230 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:32.279 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:32.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:32 smithi156 ceph-mon[195101]: pgmap v1102: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 767 op/s; 28992/267702 objects degraded (10.830%); 1139/267702 objects misplaced (0.425%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:08:32.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:32 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:08:32.374 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:32 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:08:32.374 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:32 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:08:32.441 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:32.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:32.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:32.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:32 smithi064 ceph-mon[212750]: pgmap v1102: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 767 op/s; 28992/267702 objects degraded (10.830%); 1139/267702 objects misplaced (0.425%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:08:32.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:32 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:08:32.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:32 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:08:32.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:32 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:08:32.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:32 smithi064 ceph-mon[220044]: pgmap v1102: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 767 op/s; 28992/267702 objects degraded (10.830%); 1139/267702 objects misplaced (0.425%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:08:32.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:32 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:08:32.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:32 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:08:32.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:32 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:08:32.546 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:32.610 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:08:32.611 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "" 2023-10-09T06:08:32.611 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:08:32.614 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:32.709 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:32.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:32.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:32.722 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:08:32.722 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:08:32.723 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T06:08:32.723 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T06:08:32.723 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:08:32.745 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 79 16 92169 92153 72.8548 48.9375 0.028915 0.0137203 2023-10-09T06:08:32.813 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:32.895 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:33.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:33 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 28992/267702 objects degraded (10.830%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:33.691 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:08:33.715 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:33 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 28992/267702 objects degraded (10.830%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:33.716 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:33 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 28992/267702 objects degraded (10.830%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:33.743 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-10-09T06:08:33.743970+0000 min lat: 0.00120925 max lat: 2.19099 avg lat: 0.0137669 2023-10-09T06:08:33.743 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:08:33.743 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 80 16 93022 93006 72.6106 53.3125 0.00704982 0.0137669 2023-10-09T06:08:33.746 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:33.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:33.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:33.854 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:33.892 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:34.235 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:08:34.308 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:34.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:34.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:34.315 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:34.415 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:34.480 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:34.587 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:34.595 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:34 smithi064 ceph-mon[212750]: pgmap v1103: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 760 op/s; 29493/271902 objects degraded (10.847%); 1139/271902 objects misplaced (0.419%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:08:34.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:34 smithi064 ceph-mon[220044]: pgmap v1103: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 760 op/s; 29493/271902 objects degraded (10.847%); 1139/271902 objects misplaced (0.419%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:08:34.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:34.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:34.698 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:34.736 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:34.743 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 81 16 93825 93809 72.3338 50.1875 0.00799757 0.0138186 2023-10-09T06:08:34.857 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:34 smithi156 ceph-mon[195101]: pgmap v1103: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 760 op/s; 29493/271902 objects degraded (10.847%); 1139/271902 objects misplaced (0.419%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:08:34.876 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:34.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:34.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:34.933 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:08:34.966 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:08:34.986 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:35.007 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:08:35.044 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:35.051 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:08:35.096 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:08:35.118 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:35.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:35.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:35.133 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:08:35.172 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:08:35.172 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-10-09T06:08:35.172 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:08:35.172 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:08:35.173 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:08:35.173 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:08:35.173 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:08:35.173 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:08:35.173 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:08:35.221 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:35.273 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:35.473 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:08:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:08:35] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.43.0" 2023-10-09T06:08:35.536 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:35.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:35.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:35.640 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:35.669 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:35.743 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 82 16 94647 94631 72.0783 51.375 0.00330931 0.0138672 2023-10-09T06:08:35.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:35 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2994957660' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:35.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:35 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2994957660' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:35.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:35 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2994957660' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:36.091 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:36.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:36.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:36.186 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:08:36.187 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:08:36.196 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:36.223 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:36.699 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:36.701 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T06:08:36.702 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T06:08:36.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:36.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:36.743 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 83 16 95253 95237 71.6664 37.875 0.00414156 0.0139227 2023-10-09T06:08:36.811 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:36.851 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:36 smithi156 ceph-mon[195101]: pgmap v1104: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 29990/275673 objects degraded (10.879%); 859/275673 objects misplaced (0.312%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:08:36.855 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:36.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:36 smithi064 ceph-mon[212750]: pgmap v1104: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 29990/275673 objects degraded (10.879%); 859/275673 objects misplaced (0.312%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:08:36.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:36 smithi064 ceph-mon[220044]: pgmap v1104: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 29990/275673 objects degraded (10.879%); 859/275673 objects misplaced (0.312%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:08:37.119 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:08:37.127 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:37.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:37.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:37.233 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:37.308 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:37.415 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:37.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:37.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:37.521 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:37.565 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:37.654 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:37.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:37.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:37.675 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:08:37.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:37 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 29990/275673 objects degraded (10.879%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:37.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:37 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 29990/275673 objects degraded (10.879%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:37.743 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 84 16 96149 96133 71.48 56 0.0107275 0.0139854 2023-10-09T06:08:37.760 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:37.812 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:37.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:37 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 29990/275673 objects degraded (10.879%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:37.899 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:37.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:37.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:38.007 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:38.069 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:38.162 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:38.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:38.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:38.270 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:38.326 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:38.429 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:38.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:38.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:38.533 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:38 smithi156 ceph-mon[195101]: pgmap v1105: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 763 op/s; 30466/283539 objects degraded (10.745%); 859/283539 objects misplaced (0.303%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:08:38.533 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:38 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/68206285' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:08:38.539 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:38.580 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:38.588 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:08:38.618 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:38 smithi064 ceph-mon[212750]: pgmap v1105: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 763 op/s; 30466/283539 objects degraded (10.745%); 859/283539 objects misplaced (0.303%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:08:38.619 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:38 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/68206285' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:08:38.619 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:38 smithi064 ceph-mon[220044]: pgmap v1105: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 763 op/s; 30466/283539 objects degraded (10.745%); 859/283539 objects misplaced (0.303%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:08:38.619 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:38 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/68206285' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:08:38.703 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:08:38.743 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 85 16 97088 97072 71.3295 58.6875 0.0183596 0.0140141 2023-10-09T06:08:38.755 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:38.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:38.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:38.861 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:38.951 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:39.093 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:39.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:39.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:39.201 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:39.248 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:39.350 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1, 2] out_osds: [] dead_osds: [0] live_osds: [1, 2, 3, 5, 6, 7, 4] 2023-10-09T06:08:39.351 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:08:39.351 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_3 2023-10-09T06:08:39.351 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:08:39.354 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:39.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:39.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:39.462 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:39.490 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:39.683 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:39.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:39.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:39.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:39 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:08:39.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:39 smithi064 ceph-mon[212750]: failed to encode map e444 with expected crc 2023-10-09T06:08:39.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:39 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/68206285' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:08:39.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:39 smithi064 ceph-mon[212750]: osdmap e444: 8 total, 7 up, 8 in 2023-10-09T06:08:39.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:39 smithi064 ceph-mon[212750]: failed to encode map e444 with expected crc 2023-10-09T06:08:39.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:39 smithi064 ceph-mon[212750]: failed to encode map e444 with expected crc 2023-10-09T06:08:39.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:39 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:08:39.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:39 smithi064 ceph-mon[220044]: failed to encode map e444 with expected crc 2023-10-09T06:08:39.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:39 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/68206285' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:08:39.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:39 smithi064 ceph-mon[220044]: osdmap e444: 8 total, 7 up, 8 in 2023-10-09T06:08:39.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:39 smithi064 ceph-mon[220044]: failed to encode map e444 with expected crc 2023-10-09T06:08:39.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:39 smithi064 ceph-mon[220044]: failed to encode map e444 with expected crc 2023-10-09T06:08:39.743 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 86 16 97875 97859 71.0721 49.1875 0.0209269 0.0140642 2023-10-09T06:08:39.769 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:08:39.769 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:08:39.789 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:39.795 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:39 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:08:39.795 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:39 smithi156 ceph-mon[195101]: failed to encode map e444 with expected crc 2023-10-09T06:08:39.795 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:39 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/68206285' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:08:39.795 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:39 smithi156 ceph-mon[195101]: osdmap e444: 8 total, 7 up, 8 in 2023-10-09T06:08:39.795 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:39 smithi156 ceph-mon[195101]: failed to encode map e444 with expected crc 2023-10-09T06:08:39.795 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:39 smithi156 ceph-mon[195101]: failed to encode map e444 with expected crc 2023-10-09T06:08:39.832 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:40.101 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:40.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:40.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:40.140 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:40.194 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:40.207 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:40.285 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:40.377 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:40.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:40.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:40.481 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:40.516 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:40.744 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 87 16 98739 98723 70.8759 54 0.0233693 0.0141039 2023-10-09T06:08:40.771 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:08:40.772 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:40.772 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:40 smithi156 ceph-mon[195101]: pgmap v1107: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 763 op/s; 30706/284259 objects degraded (10.802%); 859/284259 objects misplaced (0.302%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:08:40.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:40.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:40.878 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:40.889 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:40 smithi064 ceph-mon[212750]: pgmap v1107: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 763 op/s; 30706/284259 objects degraded (10.802%); 859/284259 objects misplaced (0.302%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:08:40.889 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:40 smithi064 ceph-mon[220044]: pgmap v1107: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 763 op/s; 30706/284259 objects degraded (10.802%); 859/284259 objects misplaced (0.302%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:08:40.922 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:40.995 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:41.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:41.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:41.101 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:41.148 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:41.239 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:41.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:41.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:41.346 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:41.347 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:08:41.468 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:41.589 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:41.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:41.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:41.698 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:41.729 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:41.744 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 88 16 99655 99639 70.7211 57.25 0.00496962 0.0141303 2023-10-09T06:08:41.826 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:41.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:41.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:41.929 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:08:41.935 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:41.995 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:42.064 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:42.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:42.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:42.170 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:42.242 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:42.433 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:42.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:42.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:42.545 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:42.571 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:42 smithi064 ceph-mon[220044]: pgmap v1108: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 782 op/s; 30937/291201 objects degraded (10.624%); 446/291201 objects misplaced (0.153%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T06:08:42.572 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:42 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 30937/291201 objects degraded (10.624%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:42.581 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:42.750 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 89 16 100342 100326 70.4076 42.9375 0.0210443 0.0141977 2023-10-09T06:08:42.812 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:42.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:42.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:42.848 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:42 smithi156 ceph-mon[195101]: pgmap v1108: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 782 op/s; 30937/291201 objects degraded (10.624%); 446/291201 objects misplaced (0.153%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T06:08:42.848 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:42 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 30937/291201 objects degraded (10.624%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:42.857 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:42 smithi064 ceph-mon[212750]: pgmap v1108: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 782 op/s; 30937/291201 objects degraded (10.624%); 446/291201 objects misplaced (0.153%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T06:08:42.857 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:42 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 30937/291201 objects degraded (10.624%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:42.919 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:42.962 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:43.041 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:43.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:43.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:43.106 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:43.145 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:08:43.148 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:43.192 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:08:43.243 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:08:43.255 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:43.279 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:08:43.315 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:08:43.362 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:08:43.396 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:43.399 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:08:43.399 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-10-09T06:08:43.399 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:08:43.399 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:08:43.399 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:08:43.399 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:08:43.399 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:08:43.400 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:08:43.400 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:08:43.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:43.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:43.481 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:08:43.481 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:08:43.481 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T06:08:43.481 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T06:08:43.482 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:08:43.511 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:43.622 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:08:43.624 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:43.638 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T06:08:43.746 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total time run: 90.0101 2023-10-09T06:08:43.746 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total writes made: 101099 2023-10-09T06:08:43.746 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Write size: 65536 2023-10-09T06:08:43.746 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object size: 65536 2023-10-09T06:08:43.746 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Bandwidth (MB/sec): 70.1998 2023-10-09T06:08:43.746 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Bandwidth: 37.8838 2023-10-09T06:08:43.746 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max bandwidth (MB/sec): 195.25 2023-10-09T06:08:43.747 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min bandwidth (MB/sec): 0 2023-10-09T06:08:43.747 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average IOPS: 1123 2023-10-09T06:08:43.747 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev IOPS: 606.141 2023-10-09T06:08:43.747 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max IOPS: 3124 2023-10-09T06:08:43.747 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min IOPS: 0 2023-10-09T06:08:43.747 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average Latency(s): 0.0142407 2023-10-09T06:08:43.747 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Latency(s): 0.0336497 2023-10-09T06:08:43.747 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max latency(s): 2.19099 2023-10-09T06:08:43.747 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min latency(s): 0.00120925 2023-10-09T06:08:43.750 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T06:08:43.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:43 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4287586941' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:43.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:43 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:08:43.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:43 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:43.896 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:43 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4287586941' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:43.896 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:43 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:08:43.896 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:43 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:43.896 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:43 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4287586941' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:43.897 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:43 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:08:43.897 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:43 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:44.360 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:08:44.594 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:44 smithi156 ceph-mon[195101]: pgmap v1109: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 796 op/s; 31470/295806 objects degraded (10.639%); 446/295806 objects misplaced (0.151%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T06:08:44.594 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:44 smithi156 ceph-mon[195101]: from='client.46028 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:08:44.629 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:44.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:44.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:44.740 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:44.789 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:44.838 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:08:44.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:44 smithi064 ceph-mon[212750]: pgmap v1109: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 796 op/s; 31470/295806 objects degraded (10.639%); 446/295806 objects misplaced (0.151%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T06:08:44.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:44 smithi064 ceph-mon[212750]: from='client.46028 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:08:44.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:44 smithi064 ceph-mon[220044]: pgmap v1109: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 796 op/s; 31470/295806 objects degraded (10.639%); 446/295806 objects misplaced (0.151%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T06:08:44.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:44 smithi064 ceph-mon[220044]: from='client.46028 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:08:45.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:08:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:08:45] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.43.0" 2023-10-09T06:08:45.637 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:45.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:45.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:45.743 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:45.790 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:45.840 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T06:08:45.841 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T06:08:45.874 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1109,"stamp":"2023-10-09T06:08:42.975225+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6461048405,"num_objects":98602,"num_object_clones":0,"num_object_copies":295806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":31470,"num_objects_misplaced":446,"num_objects_unfound":0,"num_objects_dirty":98602,"num_whiteouts":0,"num_read":37,"num_read_kb":69,"num_write":98657,"num_write_kb":6309204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2839,"num_bytes_recovered":184095169,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":93124,"ondisk_log_size":93124,"up":314,"acting":311,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":23736764,"kb_used_data":18429628,"kb_used_omap":163,"kb_used_meta":5306908,"kb_avail":726224452,"statfs":{"total":767960285184,"available":743653838848,"internally_reserved":0,"allocated":18871939072,"data_stored":18857333226,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":167439,"internal_metadata":5434274289},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":522190848,"num_objects":7968,"num_object_clones":0,"num_object_copies":23904,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1977,"num_objects_misplaced":-693,"num_objects_unfound":0,"num_objects_dirty":7968,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7968,"num_write_kb":509952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":693,"num_bytes_recovered":45416448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.009345"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":722,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.635762+0000","last_change":"2023-10-09T06:07:46.599648+0000","last_active":"2023-10-09T06:08:28.635762+0000","last_peered":"2023-10-09T06:08:28.635762+0000","last_clean":"2023-10-09T06:08:28.635762+0000","last_became_active":"2023-10-09T06:07:46.599303+0000","last_became_peered":"2023-10-09T06:07:46.599303+0000","last_unstale":"2023-10-09T06:08:28.635762+0000","last_undegraded":"2023-10-09T06:08:28.635762+0000","last_fullsized":"2023-10-09T06:08:28.635762+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:05.412941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:05.412941+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:45.768743+0000","scrub_duration":0.0061014329999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":512,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.644424+0000","last_change":"2023-10-09T06:07:43.572773+0000","last_active":"2023-10-09T06:08:28.644424+0000","last_peered":"2023-10-09T06:08:28.644424+0000","last_clean":"2023-10-09T06:08:28.644424+0000","last_became_active":"2023-10-09T06:07:43.572299+0000","last_became_peered":"2023-10-09T06:07:43.572299+0000","last_unstale":"2023-10-09T06:08:28.644424+0000","last_undegraded":"2023-10-09T06:08:28.644424+0000","last_fullsized":"2023-10-09T06:08:28.644424+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:00:37.672435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":252,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.627724+0000","last_change":"2023-10-09T06:05:11.451972+0000","last_active":"2023-10-09T06:08:28.627724+0000","last_peered":"2023-10-09T06:08:28.627724+0000","last_clean":"2023-10-09T06:08:28.627724+0000","last_became_active":"2023-10-09T06:05:11.451448+0000","last_became_peered":"2023-10-09T06:05:11.451448+0000","last_unstale":"2023-10-09T06:08:28.627724+0000","last_undegraded":"2023-10-09T06:08:28.627724+0000","last_fullsized":"2023-10-09T06:08:28.627724+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:39.183621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":733,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.644764+0000","last_change":"2023-10-09T06:07:51.704972+0000","last_active":"2023-10-09T06:08:28.644764+0000","last_peered":"2023-10-09T06:08:28.644764+0000","last_clean":"2023-10-09T06:07:47.645886+0000","last_became_active":"2023-10-09T06:07:51.704972+0000","last_became_peered":"2023-10-09T06:07:51.704972+0000","last_unstale":"2023-10-09T06:08:28.644764+0000","last_undegraded":"2023-10-09T06:08:28.644764+0000","last_fullsized":"2023-10-09T06:07:51.697866+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:01.468346+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:01.468346+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:23:22.569733+0000","scrub_duration":0.0016258220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":123,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.633823+0000","last_change":"2023-10-09T06:01:54.762060+0000","last_active":"2023-10-09T06:08:28.633823+0000","last_peered":"2023-10-09T06:08:28.633823+0000","last_clean":"2023-10-09T06:08:28.633823+0000","last_became_active":"2023-10-09T06:01:54.761757+0000","last_became_peered":"2023-10-09T06:01:54.761757+0000","last_unstale":"2023-10-09T06:08:28.633823+0000","last_undegraded":"2023-10-09T06:08:28.633823+0000","last_fullsized":"2023-10-09T06:08:28.633823+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:40.294232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":411,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.639150+0000","last_change":"2023-10-09T06:07:51.711108+0000","last_active":"2023-10-09T06:08:28.639150+0000","last_peered":"2023-10-09T06:08:28.639150+0000","last_clean":"2023-10-09T06:07:47.641347+0000","last_became_active":"2023-10-09T06:07:51.711108+0000","last_became_peered":"2023-10-09T06:07:51.711108+0000","last_unstale":"2023-10-09T06:08:28.639150+0000","last_undegraded":"2023-10-09T06:08:28.639150+0000","last_fullsized":"2023-10-09T06:07:51.700763+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:42.556462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":688,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.643229+0000","last_change":"2023-10-09T06:02:43.293521+0000","last_active":"2023-10-09T06:08:28.643229+0000","last_peered":"2023-10-09T06:08:28.643229+0000","last_clean":"2023-10-09T06:08:28.643229+0000","last_became_active":"2023-10-09T06:01:57.803642+0000","last_became_peered":"2023-10-09T06:01:57.803642+0000","last_unstale":"2023-10-09T06:08:28.643229+0000","last_undegraded":"2023-10-09T06:08:28.643229+0000","last_fullsized":"2023-10-09T06:08:28.643229+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.293426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.293426+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:45.626761+0000","scrub_duration":0.001587946,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":225,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.626029+0000","last_change":"2023-10-09T06:07:51.721718+0000","last_active":"2023-10-09T06:08:28.626029+0000","last_peered":"2023-10-09T06:08:28.626029+0000","last_clean":"2023-10-09T06:07:47.627124+0000","last_became_active":"2023-10-09T06:07:51.721718+0000","last_became_peered":"2023-10-09T06:07:51.721718+0000","last_unstale":"2023-10-09T06:08:28.626029+0000","last_undegraded":"2023-10-09T06:08:28.626029+0000","last_fullsized":"2023-10-09T06:07:51.700492+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:30:49.288144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":480,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.634255+0000","last_change":"2023-10-09T06:01:54.757269+0000","last_active":"2023-10-09T06:08:28.634255+0000","last_peered":"2023-10-09T06:08:28.634255+0000","last_clean":"2023-10-09T06:08:28.634255+0000","last_became_active":"2023-10-09T06:01:54.756894+0000","last_became_peered":"2023-10-09T06:01:54.756894+0000","last_unstale":"2023-10-09T06:08:28.634255+0000","last_undegraded":"2023-10-09T06:08:28.634255+0000","last_fullsized":"2023-10-09T06:08:28.634255+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:45.212821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":763,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.636951+0000","last_change":"2023-10-09T06:01:57.804160+0000","last_active":"2023-10-09T06:08:28.636951+0000","last_peered":"2023-10-09T06:08:28.636951+0000","last_clean":"2023-10-09T06:08:28.636951+0000","last_became_active":"2023-10-09T06:01:57.803424+0000","last_became_peered":"2023-10-09T06:01:57.803424+0000","last_unstale":"2023-10-09T06:08:28.636951+0000","last_undegraded":"2023-10-09T06:08:28.636951+0000","last_fullsized":"2023-10-09T06:08:28.636951+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.384407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.384407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:42.857346+0000","scrub_duration":0.0040651960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":330,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.627969+0000","last_change":"2023-10-09T06:07:51.710005+0000","last_active":"2023-10-09T06:08:28.627969+0000","last_peered":"2023-10-09T06:08:28.627969+0000","last_clean":"2023-10-09T06:07:47.627648+0000","last_became_active":"2023-10-09T06:07:51.710005+0000","last_became_peered":"2023-10-09T06:07:51.710005+0000","last_unstale":"2023-10-09T06:08:28.627969+0000","last_undegraded":"2023-10-09T06:08:28.627969+0000","last_fullsized":"2023-10-09T06:07:51.703972+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:20.319164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":261,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.626162+0000","last_change":"2023-10-09T06:06:01.067544+0000","last_active":"2023-10-09T06:08:28.626162+0000","last_peered":"2023-10-09T06:08:28.626162+0000","last_clean":"2023-10-09T06:08:28.626162+0000","last_became_active":"2023-10-09T06:06:01.067188+0000","last_became_peered":"2023-10-09T06:06:01.067188+0000","last_unstale":"2023-10-09T06:08:28.626162+0000","last_undegraded":"2023-10-09T06:08:28.626162+0000","last_fullsized":"2023-10-09T06:08:28.626162+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:43.464186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":515,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.632847+0000","last_change":"2023-10-09T06:07:46.594860+0000","last_active":"2023-10-09T06:08:28.632847+0000","last_peered":"2023-10-09T06:08:28.632847+0000","last_clean":"2023-10-09T06:08:28.632847+0000","last_became_active":"2023-10-09T06:07:46.594424+0000","last_became_peered":"2023-10-09T06:07:46.594424+0000","last_unstale":"2023-10-09T06:08:28.632847+0000","last_undegraded":"2023-10-09T06:08:28.632847+0000","last_fullsized":"2023-10-09T06:08:28.632847+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:20.742561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:20.742561+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":493,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.641816+0000","last_change":"2023-10-09T06:01:54.766344+0000","last_active":"2023-10-09T06:08:28.641816+0000","last_peered":"2023-10-09T06:08:28.641816+0000","last_clean":"2023-10-09T06:08:28.641816+0000","last_became_active":"2023-10-09T06:01:54.766086+0000","last_became_peered":"2023-10-09T06:01:54.766086+0000","last_unstale":"2023-10-09T06:08:28.641816+0000","last_undegraded":"2023-10-09T06:08:28.641816+0000","last_fullsized":"2023-10-09T06:08:28.641816+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:47.163700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":392,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.626399+0000","last_change":"2023-10-09T05:53:59.185702+0000","last_active":"2023-10-09T06:08:28.626399+0000","last_peered":"2023-10-09T06:08:28.626399+0000","last_clean":"2023-10-09T06:08:28.626399+0000","last_became_active":"2023-10-09T05:53:59.185373+0000","last_became_peered":"2023-10-09T05:53:59.185373+0000","last_unstale":"2023-10-09T06:08:28.626399+0000","last_undegraded":"2023-10-09T06:08:28.626399+0000","last_fullsized":"2023-10-09T06:08:28.626399+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:50.756967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":868,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.634431+0000","last_change":"2023-10-09T06:07:51.708726+0000","last_active":"2023-10-09T06:08:28.634431+0000","last_peered":"2023-10-09T06:08:28.634431+0000","last_clean":"2023-10-09T06:07:47.656725+0000","last_became_active":"2023-10-09T06:07:51.708726+0000","last_became_peered":"2023-10-09T06:07:51.708726+0000","last_unstale":"2023-10-09T06:08:28.634431+0000","last_undegraded":"2023-10-09T06:08:28.634431+0000","last_fullsized":"2023-10-09T06:07:51.704451+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:48.374432+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:48.374432+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:55:02.953562+0000","scrub_duration":0.001678585,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":301,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.637549+0000","last_change":"2023-10-09T06:01:22.337444+0000","last_active":"2023-10-09T06:08:28.637549+0000","last_peered":"2023-10-09T06:08:28.637549+0000","last_clean":"2023-10-09T06:08:28.637549+0000","last_became_active":"2023-10-09T06:01:22.336348+0000","last_became_peered":"2023-10-09T06:01:22.336348+0000","last_unstale":"2023-10-09T06:08:28.637549+0000","last_undegraded":"2023-10-09T06:08:28.637549+0000","last_fullsized":"2023-10-09T06:08:28.637549+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:43:11.149649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":280,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.629510+0000","last_change":"2023-10-09T06:07:51.709718+0000","last_active":"2023-10-09T06:08:28.629510+0000","last_peered":"2023-10-09T06:08:28.629510+0000","last_clean":"2023-10-09T06:04:59.508425+0000","last_became_active":"2023-10-09T06:07:51.709718+0000","last_became_peered":"2023-10-09T06:07:51.709718+0000","last_unstale":"2023-10-09T06:08:28.629510+0000","last_undegraded":"2023-10-09T06:08:28.629510+0000","last_fullsized":"2023-10-09T06:07:51.702207+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:24.493938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":604,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.640527+0000","last_change":"2023-10-09T06:07:51.709216+0000","last_active":"2023-10-09T06:08:28.640527+0000","last_peered":"2023-10-09T06:08:28.640527+0000","last_clean":"2023-10-09T06:07:47.646442+0000","last_became_active":"2023-10-09T06:07:51.709216+0000","last_became_peered":"2023-10-09T06:07:51.709216+0000","last_unstale":"2023-10-09T06:08:28.640527+0000","last_undegraded":"2023-10-09T06:08:28.640527+0000","last_fullsized":"2023-10-09T06:07:51.698249+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.339121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.339121+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:20.507565+0000","scrub_duration":0.010474344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":438,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.645684+0000","last_change":"2023-10-09T06:01:22.335608+0000","last_active":"2023-10-09T06:08:28.645684+0000","last_peered":"2023-10-09T06:08:28.645684+0000","last_clean":"2023-10-09T06:08:28.645684+0000","last_became_active":"2023-10-09T06:01:22.335296+0000","last_became_peered":"2023-10-09T06:01:22.335296+0000","last_unstale":"2023-10-09T06:08:28.645684+0000","last_undegraded":"2023-10-09T06:08:28.645684+0000","last_fullsized":"2023-10-09T06:08:28.645684+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:21.843341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":456,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.644924+0000","last_change":"2023-10-09T06:01:54.764916+0000","last_active":"2023-10-09T06:08:28.644924+0000","last_peered":"2023-10-09T06:08:28.644924+0000","last_clean":"2023-10-09T06:08:28.644924+0000","last_became_active":"2023-10-09T06:01:54.764705+0000","last_became_peered":"2023-10-09T06:01:54.764705+0000","last_unstale":"2023-10-09T06:08:28.644924+0000","last_undegraded":"2023-10-09T06:08:28.644924+0000","last_fullsized":"2023-10-09T06:08:28.644924+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:12.527719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":643,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.625741+0000","last_change":"2023-10-09T06:07:51.721604+0000","last_active":"2023-10-09T06:08:28.625741+0000","last_peered":"2023-10-09T06:08:28.625741+0000","last_clean":"2023-10-09T06:07:47.629454+0000","last_became_active":"2023-10-09T06:07:51.721604+0000","last_became_peered":"2023-10-09T06:07:51.721604+0000","last_unstale":"2023-10-09T06:08:28.625741+0000","last_undegraded":"2023-10-09T06:08:28.625741+0000","last_fullsized":"2023-10-09T06:07:51.701437+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:42.900876+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:42.900876+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:19.634753+0000","scrub_duration":0.0014928649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":335,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.646107+0000","last_change":"2023-10-09T06:07:43.585040+0000","last_active":"2023-10-09T06:08:28.646107+0000","last_peered":"2023-10-09T06:08:28.646107+0000","last_clean":"2023-10-09T06:08:28.646107+0000","last_became_active":"2023-10-09T06:07:43.576991+0000","last_became_peered":"2023-10-09T06:07:43.576991+0000","last_unstale":"2023-10-09T06:08:28.646107+0000","last_undegraded":"2023-10-09T06:08:28.646107+0000","last_fullsized":"2023-10-09T06:08:28.646107+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:34.279182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":487,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.627759+0000","last_change":"2023-10-09T06:01:54.761558+0000","last_active":"2023-10-09T06:08:28.627759+0000","last_peered":"2023-10-09T06:08:28.627759+0000","last_clean":"2023-10-09T06:08:28.627759+0000","last_became_active":"2023-10-09T06:01:54.761251+0000","last_became_peered":"2023-10-09T06:01:54.761251+0000","last_unstale":"2023-10-09T06:08:28.627759+0000","last_undegraded":"2023-10-09T06:08:28.627759+0000","last_fullsized":"2023-10-09T06:08:28.627759+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:53.245267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":887,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.636157+0000","last_change":"2023-10-09T06:06:03.453188+0000","last_active":"2023-10-09T06:08:28.636157+0000","last_peered":"2023-10-09T06:08:28.636157+0000","last_clean":"2023-10-09T06:08:28.636157+0000","last_became_active":"2023-10-09T06:06:03.452853+0000","last_became_peered":"2023-10-09T06:06:03.452853+0000","last_unstale":"2023-10-09T06:08:28.636157+0000","last_undegraded":"2023-10-09T06:08:28.636157+0000","last_fullsized":"2023-10-09T06:08:28.636157+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:26:21.994828+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":376,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.636892+0000","last_change":"2023-10-09T06:07:51.715418+0000","last_active":"2023-10-09T06:08:28.636892+0000","last_peered":"2023-10-09T06:08:28.636892+0000","last_clean":"2023-10-09T06:07:47.641228+0000","last_became_active":"2023-10-09T06:07:51.715418+0000","last_became_peered":"2023-10-09T06:07:51.715418+0000","last_unstale":"2023-10-09T06:08:28.636892+0000","last_undegraded":"2023-10-09T06:08:28.636892+0000","last_fullsized":"2023-10-09T06:07:51.700165+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:19.287268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":377,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.637342+0000","last_change":"2023-10-09T06:06:01.071454+0000","last_active":"2023-10-09T06:08:28.637342+0000","last_peered":"2023-10-09T06:08:28.637342+0000","last_clean":"2023-10-09T06:08:28.637342+0000","last_became_active":"2023-10-09T06:06:01.071137+0000","last_became_peered":"2023-10-09T06:06:01.071137+0000","last_unstale":"2023-10-09T06:08:28.637342+0000","last_undegraded":"2023-10-09T06:08:28.637342+0000","last_fullsized":"2023-10-09T06:08:28.637342+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:58:52.761376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":647,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.634402+0000","last_change":"2023-10-09T06:07:06.561922+0000","last_active":"2023-10-09T06:08:28.634402+0000","last_peered":"2023-10-09T06:08:28.634402+0000","last_clean":"2023-10-09T06:08:28.634402+0000","last_became_active":"2023-10-09T06:07:06.561625+0000","last_became_peered":"2023-10-09T06:07:06.561625+0000","last_unstale":"2023-10-09T06:08:28.634402+0000","last_undegraded":"2023-10-09T06:08:28.634402+0000","last_fullsized":"2023-10-09T06:08:28.634402+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.316697+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.316697+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:08.265733+0000","scrub_duration":0.001666561,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":297,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.636094+0000","last_change":"2023-10-09T06:01:22.337171+0000","last_active":"2023-10-09T06:08:28.636094+0000","last_peered":"2023-10-09T06:08:28.636094+0000","last_clean":"2023-10-09T06:08:28.636094+0000","last_became_active":"2023-10-09T06:01:22.335438+0000","last_became_peered":"2023-10-09T06:01:22.335438+0000","last_unstale":"2023-10-09T06:08:28.636094+0000","last_undegraded":"2023-10-09T06:08:28.636094+0000","last_fullsized":"2023-10-09T06:08:28.636094+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:25.036448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":374,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.626281+0000","last_change":"2023-10-09T06:07:51.710706+0000","last_active":"2023-10-09T06:08:28.626281+0000","last_peered":"2023-10-09T06:08:28.626281+0000","last_clean":"2023-10-09T06:07:47.627324+0000","last_became_active":"2023-10-09T06:07:51.710706+0000","last_became_peered":"2023-10-09T06:07:51.710706+0000","last_unstale":"2023-10-09T06:08:28.626281+0000","last_undegraded":"2023-10-09T06:08:28.626281+0000","last_fullsized":"2023-10-09T06:07:51.705058+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:32.335550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":697,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.642624+0000","last_change":"2023-10-09T06:02:40.388302+0000","last_active":"2023-10-09T06:08:28.642624+0000","last_peered":"2023-10-09T06:08:28.642624+0000","last_clean":"2023-10-09T06:08:28.642624+0000","last_became_active":"2023-10-09T06:01:57.803733+0000","last_became_peered":"2023-10-09T06:01:57.803733+0000","last_unstale":"2023-10-09T06:08:28.642624+0000","last_undegraded":"2023-10-09T06:08:28.642624+0000","last_fullsized":"2023-10-09T06:08:28.642624+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:40.388125+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:40.388125+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:17.919838+0000","scrub_duration":0.0055576990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":456,"reported_epoch":443,"state":"active+undersized+degraded","last_fresh":"2023-10-09T06:08:28.633777+0000","last_change":"2023-10-09T06:07:51.707034+0000","last_active":"2023-10-09T06:08:28.633777+0000","last_peered":"2023-10-09T06:08:28.633777+0000","last_clean":"2023-10-09T06:07:47.656239+0000","last_became_active":"2023-10-09T06:07:51.707034+0000","last_became_peered":"2023-10-09T06:07:51.707034+0000","last_unstale":"2023-10-09T06:08:28.633777+0000","last_undegraded":"2023-10-09T06:07:51.704230+0000","last_fullsized":"2023-10-09T06:07:51.704124+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:24.002564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":237,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.628081+0000","last_change":"2023-10-09T06:07:02.023531+0000","last_active":"2023-10-09T06:08:28.628081+0000","last_peered":"2023-10-09T06:08:28.628081+0000","last_clean":"2023-10-09T06:08:28.628081+0000","last_became_active":"2023-10-09T06:07:02.023280+0000","last_became_peered":"2023-10-09T06:07:02.023280+0000","last_unstale":"2023-10-09T06:08:28.628081+0000","last_undegraded":"2023-10-09T06:08:28.628081+0000","last_fullsized":"2023-10-09T06:08:28.628081+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:53.202101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"444'6179","reported_seq":6212,"reported_epoch":444,"state":"active+undersized+degraded","last_fresh":"2023-10-09T06:08:41.078059+0000","last_change":"2023-10-09T06:07:51.708055+0000","last_active":"2023-10-09T06:08:41.078059+0000","last_peered":"2023-10-09T06:08:41.078059+0000","last_clean":"2023-10-09T06:07:50.629239+0000","last_became_active":"2023-10-09T06:07:51.708055+0000","last_became_peered":"2023-10-09T06:07:51.708055+0000","last_unstale":"2023-10-09T06:08:41.078059+0000","last_undegraded":"2023-10-09T06:07:51.700244+0000","last_fullsized":"2023-10-09T06:07:51.700136+0000","mapping_epoch":438,"log_start":"429'600","ondisk_log_start":"429'600","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":5579,"log_dups_size":0,"ondisk_log_size":5579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:58.171315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404946944,"num_objects":6179,"num_object_clones":0,"num_object_copies":18537,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6179,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6179,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6179,"num_write_kb":395456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":6179}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":820,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.637207+0000","last_change":"2023-10-09T06:01:57.804339+0000","last_active":"2023-10-09T06:08:28.637207+0000","last_peered":"2023-10-09T06:08:28.637207+0000","last_clean":"2023-10-09T06:08:28.637207+0000","last_became_active":"2023-10-09T06:01:57.803844+0000","last_became_peered":"2023-10-09T06:01:57.803844+0000","last_unstale":"2023-10-09T06:08:28.637207+0000","last_undegraded":"2023-10-09T06:08:28.637207+0000","last_fullsized":"2023-10-09T06:08:28.637207+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:53.469394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:53.469394+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:51.125394+0000","scrub_duration":0.0059716659999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":308,"reported_epoch":443,"state":"active+undersized+degraded","last_fresh":"2023-10-09T06:08:28.642541+0000","last_change":"2023-10-09T06:07:51.702003+0000","last_active":"2023-10-09T06:08:28.642541+0000","last_peered":"2023-10-09T06:08:28.642541+0000","last_clean":"2023-10-09T06:07:47.641518+0000","last_became_active":"2023-10-09T06:07:51.702003+0000","last_became_peered":"2023-10-09T06:07:51.702003+0000","last_unstale":"2023-10-09T06:08:28.642541+0000","last_undegraded":"2023-10-09T06:07:51.699306+0000","last_fullsized":"2023-10-09T06:07:51.699191+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:39.807423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":360,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.635902+0000","last_change":"2023-10-09T06:06:01.074296+0000","last_active":"2023-10-09T06:08:28.635902+0000","last_peered":"2023-10-09T06:08:28.635902+0000","last_clean":"2023-10-09T06:08:28.635902+0000","last_became_active":"2023-10-09T06:06:01.073655+0000","last_became_peered":"2023-10-09T06:06:01.073655+0000","last_unstale":"2023-10-09T06:08:28.635902+0000","last_undegraded":"2023-10-09T06:08:28.635902+0000","last_fullsized":"2023-10-09T06:08:28.635902+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:20.319425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"444'6184","reported_seq":6212,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T06:08:40.693970+0000","last_change":"2023-10-09T06:07:07.580215+0000","last_active":"2023-10-09T06:08:40.693970+0000","last_peered":"2023-10-09T06:08:40.693970+0000","last_clean":"2023-10-09T06:08:40.693970+0000","last_became_active":"2023-10-09T06:07:07.579874+0000","last_became_peered":"2023-10-09T06:07:07.579874+0000","last_unstale":"2023-10-09T06:08:40.693970+0000","last_undegraded":"2023-10-09T06:08:40.693970+0000","last_fullsized":"2023-10-09T06:08:40.693970+0000","mapping_epoch":425,"log_start":"429'800","ondisk_log_start":"429'800","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":5384,"log_dups_size":0,"ondisk_log_size":5384,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:04.426865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":405274624,"num_objects":6184,"num_object_clones":0,"num_object_copies":18552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6184,"num_write_kb":395776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":759,"reported_epoch":443,"state":"active+undersized+degraded","last_fresh":"2023-10-09T06:08:28.626120+0000","last_change":"2023-10-09T06:07:51.706761+0000","last_active":"2023-10-09T06:08:28.626120+0000","last_peered":"2023-10-09T06:08:28.626120+0000","last_clean":"2023-10-09T06:07:47.626849+0000","last_became_active":"2023-10-09T06:07:51.706761+0000","last_became_peered":"2023-10-09T06:07:51.706761+0000","last_unstale":"2023-10-09T06:08:28.626120+0000","last_undegraded":"2023-10-09T06:07:51.704410+0000","last_fullsized":"2023-10-09T06:07:51.704293+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:03.241335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:03.241335+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:22.443653+0000","scrub_duration":0.017780547000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"3,6","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":357,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.641956+0000","last_change":"2023-10-09T06:01:54.826540+0000","last_active":"2023-10-09T06:08:28.641956+0000","last_peered":"2023-10-09T06:08:28.641956+0000","last_clean":"2023-10-09T06:08:28.641956+0000","last_became_active":"2023-10-09T06:01:54.766205+0000","last_became_peered":"2023-10-09T06:01:54.766205+0000","last_unstale":"2023-10-09T06:08:28.641956+0000","last_undegraded":"2023-10-09T06:08:28.641956+0000","last_fullsized":"2023-10-09T06:08:28.641956+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:45.220336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":482,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.643728+0000","last_change":"2023-10-09T06:01:54.760367+0000","last_active":"2023-10-09T06:08:28.643728+0000","last_peered":"2023-10-09T06:08:28.643728+0000","last_clean":"2023-10-09T06:08:28.643728+0000","last_became_active":"2023-10-09T06:01:54.760087+0000","last_became_peered":"2023-10-09T06:01:54.760087+0000","last_unstale":"2023-10-09T06:08:28.643728+0000","last_undegraded":"2023-10-09T06:08:28.643728+0000","last_fullsized":"2023-10-09T06:08:28.643728+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:26.000104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"444'6164","reported_seq":6191,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T06:08:41.066061+0000","last_change":"2023-10-09T06:07:07.880402+0000","last_active":"2023-10-09T06:08:41.066061+0000","last_peered":"2023-10-09T06:08:41.066061+0000","last_clean":"2023-10-09T06:08:41.066061+0000","last_became_active":"2023-10-09T06:07:07.880006+0000","last_became_peered":"2023-10-09T06:07:07.880006+0000","last_unstale":"2023-10-09T06:08:41.066061+0000","last_undegraded":"2023-10-09T06:08:41.066061+0000","last_fullsized":"2023-10-09T06:08:41.066061+0000","mapping_epoch":425,"log_start":"429'600","ondisk_log_start":"429'600","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":5564,"log_dups_size":0,"ondisk_log_size":5564,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:17.652201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403963904,"num_objects":6164,"num_object_clones":0,"num_object_copies":18492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6164,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6164,"num_write_kb":394496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":883,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.634634+0000","last_change":"2023-10-09T06:01:22.326209+0000","last_active":"2023-10-09T06:08:28.634634+0000","last_peered":"2023-10-09T06:08:28.634634+0000","last_clean":"2023-10-09T06:08:28.634634+0000","last_became_active":"2023-10-09T06:01:22.325866+0000","last_became_peered":"2023-10-09T06:01:22.325866+0000","last_unstale":"2023-10-09T06:08:28.634634+0000","last_undegraded":"2023-10-09T06:08:28.634634+0000","last_fullsized":"2023-10-09T06:08:28.634634+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:41.753506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:41.753506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:32.324674+0000","scrub_duration":0.0022121929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":296,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.637633+0000","last_change":"2023-10-09T06:06:01.078672+0000","last_active":"2023-10-09T06:08:28.637633+0000","last_peered":"2023-10-09T06:08:28.637633+0000","last_clean":"2023-10-09T06:08:28.637633+0000","last_became_active":"2023-10-09T06:06:01.072297+0000","last_became_peered":"2023-10-09T06:06:01.072297+0000","last_unstale":"2023-10-09T06:08:28.637633+0000","last_undegraded":"2023-10-09T06:08:28.637633+0000","last_fullsized":"2023-10-09T06:08:28.637633+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:35.905773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":428,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.642303+0000","last_change":"2023-10-09T06:06:01.077340+0000","last_active":"2023-10-09T06:08:28.642303+0000","last_peered":"2023-10-09T06:08:28.642303+0000","last_clean":"2023-10-09T06:08:28.642303+0000","last_became_active":"2023-10-09T06:06:01.076610+0000","last_became_peered":"2023-10-09T06:06:01.076610+0000","last_unstale":"2023-10-09T06:08:28.642303+0000","last_undegraded":"2023-10-09T06:08:28.642303+0000","last_fullsized":"2023-10-09T06:08:28.642303+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:45.158282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"444'6132","reported_seq":6168,"reported_epoch":444,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:08:40.711351+0000","last_change":"2023-10-09T06:07:44.729895+0000","last_active":"2023-10-09T06:08:40.711351+0000","last_peered":"2023-10-09T06:08:40.711351+0000","last_clean":"2023-10-09T06:07:42.388388+0000","last_became_active":"2023-10-09T06:07:44.720581+0000","last_became_peered":"2023-10-09T06:07:44.720581+0000","last_unstale":"2023-10-09T06:08:40.711351+0000","last_undegraded":"2023-10-09T06:07:44.592304+0000","last_fullsized":"2023-10-09T06:07:44.563310+0000","mapping_epoch":433,"log_start":"429'700","ondisk_log_start":"429'700","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":5432,"log_dups_size":0,"ondisk_log_size":5432,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:29:15.772970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401866752,"num_objects":6132,"num_object_clones":0,"num_object_copies":18396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6132,"num_write_kb":392448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5","objects":2797},{"shards":"1,5,6","objects":3335}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":619,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.645648+0000","last_change":"2023-10-09T06:02:16.361201+0000","last_active":"2023-10-09T06:08:28.645648+0000","last_peered":"2023-10-09T06:08:28.645648+0000","last_clean":"2023-10-09T06:08:28.645648+0000","last_became_active":"2023-10-09T06:01:22.337294+0000","last_became_peered":"2023-10-09T06:01:22.337294+0000","last_unstale":"2023-10-09T06:08:28.645648+0000","last_undegraded":"2023-10-09T06:08:28.645648+0000","last_fullsized":"2023-10-09T06:08:28.645648+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:16.361097+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:16.361097+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:16:25.401939+0000","scrub_duration":0.028533749000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":211,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T06:08:38.554538+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T06:08:38.554538+0000","last_peered":"2023-10-09T06:08:38.554538+0000","last_clean":"2023-10-09T06:08:38.554538+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T06:08:38.554538+0000","last_undegraded":"2023-10-09T06:08:38.554538+0000","last_fullsized":"2023-10-09T06:08:38.554538+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":375,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.634029+0000","last_change":"2023-10-09T06:07:51.699039+0000","last_active":"2023-10-09T06:08:28.634029+0000","last_peered":"2023-10-09T06:08:28.634029+0000","last_clean":"2023-10-09T06:01:48.339734+0000","last_became_active":"2023-10-09T06:07:51.699039+0000","last_became_peered":"2023-10-09T06:07:51.699039+0000","last_unstale":"2023-10-09T06:08:28.634029+0000","last_undegraded":"2023-10-09T06:08:28.634029+0000","last_fullsized":"2023-10-09T06:07:51.696485+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:14.271591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"444'6110","reported_seq":6137,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T06:08:39.783016+0000","last_change":"2023-10-09T06:07:07.547742+0000","last_active":"2023-10-09T06:08:39.783016+0000","last_peered":"2023-10-09T06:08:39.783016+0000","last_clean":"2023-10-09T06:08:39.783016+0000","last_became_active":"2023-10-09T06:07:07.547431+0000","last_became_peered":"2023-10-09T06:07:07.547431+0000","last_unstale":"2023-10-09T06:08:39.783016+0000","last_undegraded":"2023-10-09T06:08:39.783016+0000","last_fullsized":"2023-10-09T06:08:39.783016+0000","mapping_epoch":425,"log_start":"429'800","ondisk_log_start":"429'800","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":5310,"log_dups_size":0,"ondisk_log_size":5310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:35.316678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400424960,"num_objects":6110,"num_object_clones":0,"num_object_copies":18330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6110,"num_write_kb":391040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":927,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.638267+0000","last_change":"2023-10-09T06:07:46.595962+0000","last_active":"2023-10-09T06:08:28.638267+0000","last_peered":"2023-10-09T06:08:28.638267+0000","last_clean":"2023-10-09T06:08:28.638267+0000","last_became_active":"2023-10-09T06:07:46.595718+0000","last_became_peered":"2023-10-09T06:07:46.595718+0000","last_unstale":"2023-10-09T06:08:28.638267+0000","last_undegraded":"2023-10-09T06:08:28.638267+0000","last_fullsized":"2023-10-09T06:08:28.638267+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:55.437947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:55.437947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:45:57.993572+0000","scrub_duration":0.0091262979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":236,"reported_epoch":444,"state":"active+undersized+degraded","last_fresh":"2023-10-09T06:08:38.552657+0000","last_change":"2023-10-09T06:07:51.714397+0000","last_active":"2023-10-09T06:08:38.552657+0000","last_peered":"2023-10-09T06:08:38.552657+0000","last_clean":"2023-10-09T06:07:47.627601+0000","last_became_active":"2023-10-09T06:07:51.714397+0000","last_became_peered":"2023-10-09T06:07:51.714397+0000","last_unstale":"2023-10-09T06:08:38.552657+0000","last_undegraded":"2023-10-09T06:07:51.702756+0000","last_fullsized":"2023-10-09T06:07:51.702651+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:17:01.204610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":398,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.645604+0000","last_change":"2023-10-09T06:01:22.332033+0000","last_active":"2023-10-09T06:08:28.645604+0000","last_peered":"2023-10-09T06:08:28.645604+0000","last_clean":"2023-10-09T06:08:28.645604+0000","last_became_active":"2023-10-09T06:01:22.329825+0000","last_became_peered":"2023-10-09T06:01:22.329825+0000","last_unstale":"2023-10-09T06:08:28.645604+0000","last_undegraded":"2023-10-09T06:08:28.645604+0000","last_fullsized":"2023-10-09T06:08:28.645604+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:54.046681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"444'6339","reported_seq":6375,"reported_epoch":444,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:08:40.680831+0000","last_change":"2023-10-09T06:07:44.730713+0000","last_active":"2023-10-09T06:08:40.680831+0000","last_peered":"2023-10-09T06:08:40.680831+0000","last_clean":"2023-10-09T06:07:42.372390+0000","last_became_active":"2023-10-09T06:07:44.720879+0000","last_became_peered":"2023-10-09T06:07:44.720879+0000","last_unstale":"2023-10-09T06:08:40.680831+0000","last_undegraded":"2023-10-09T06:07:44.589117+0000","last_fullsized":"2023-10-09T06:07:44.561865+0000","mapping_epoch":433,"log_start":"429'900","ondisk_log_start":"429'900","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":5439,"log_dups_size":0,"ondisk_log_size":5439,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:21:16.721210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":415432704,"num_objects":6339,"num_object_clones":0,"num_object_copies":19017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3424,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6339,"num_write_kb":405696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":2915}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":687,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.625771+0000","last_change":"2023-10-09T06:07:46.602409+0000","last_active":"2023-10-09T06:08:28.625771+0000","last_peered":"2023-10-09T06:08:28.625771+0000","last_clean":"2023-10-09T06:08:28.625771+0000","last_became_active":"2023-10-09T06:07:46.602045+0000","last_became_peered":"2023-10-09T06:07:46.602045+0000","last_unstale":"2023-10-09T06:08:28.625771+0000","last_undegraded":"2023-10-09T06:08:28.625771+0000","last_fullsized":"2023-10-09T06:08:28.625771+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.282393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.282393+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:35.114362+0000","scrub_duration":0.01402403,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":481,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.633180+0000","last_change":"2023-10-09T06:01:54.751188+0000","last_active":"2023-10-09T06:08:28.633180+0000","last_peered":"2023-10-09T06:08:28.633180+0000","last_clean":"2023-10-09T06:08:28.633180+0000","last_became_active":"2023-10-09T06:01:54.750870+0000","last_became_peered":"2023-10-09T06:01:54.750870+0000","last_unstale":"2023-10-09T06:08:28.633180+0000","last_undegraded":"2023-10-09T06:08:28.633180+0000","last_fullsized":"2023-10-09T06:08:28.633180+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:01.985974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":492,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.645240+0000","last_change":"2023-10-09T06:07:43.572173+0000","last_active":"2023-10-09T06:08:28.645240+0000","last_peered":"2023-10-09T06:08:28.645240+0000","last_clean":"2023-10-09T06:08:28.645240+0000","last_became_active":"2023-10-09T06:07:43.571690+0000","last_became_peered":"2023-10-09T06:07:43.571690+0000","last_unstale":"2023-10-09T06:08:28.645240+0000","last_undegraded":"2023-10-09T06:08:28.645240+0000","last_fullsized":"2023-10-09T06:08:28.645240+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:10.234125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"444'6311","reported_seq":6338,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T06:08:41.078044+0000","last_change":"2023-10-09T06:07:07.547801+0000","last_active":"2023-10-09T06:08:41.078044+0000","last_peered":"2023-10-09T06:08:41.078044+0000","last_clean":"2023-10-09T06:08:41.078044+0000","last_became_active":"2023-10-09T06:07:07.547533+0000","last_became_peered":"2023-10-09T06:07:07.547533+0000","last_unstale":"2023-10-09T06:08:41.078044+0000","last_undegraded":"2023-10-09T06:08:41.078044+0000","last_fullsized":"2023-10-09T06:08:41.078044+0000","mapping_epoch":425,"log_start":"429'700","ondisk_log_start":"429'700","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":5611,"log_dups_size":0,"ondisk_log_size":5611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:43:16.000531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413597696,"num_objects":6311,"num_object_clones":0,"num_object_copies":18933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6311,"num_write_kb":403904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":732,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.634207+0000","last_change":"2023-10-09T06:07:51.708480+0000","last_active":"2023-10-09T06:08:28.634207+0000","last_peered":"2023-10-09T06:08:28.634207+0000","last_clean":"2023-10-09T06:07:47.635521+0000","last_became_active":"2023-10-09T06:07:51.708480+0000","last_became_peered":"2023-10-09T06:07:51.708480+0000","last_unstale":"2023-10-09T06:08:28.634207+0000","last_undegraded":"2023-10-09T06:08:28.634207+0000","last_fullsized":"2023-10-09T06:07:51.696895+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.373748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.373748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:04:09.876908+0000","scrub_duration":0.0024576450000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":434,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.643861+0000","last_change":"2023-10-09T06:07:43.574863+0000","last_active":"2023-10-09T06:08:28.643861+0000","last_peered":"2023-10-09T06:08:28.643861+0000","last_clean":"2023-10-09T06:08:28.643861+0000","last_became_active":"2023-10-09T06:07:43.574576+0000","last_became_peered":"2023-10-09T06:07:43.574576+0000","last_unstale":"2023-10-09T06:08:28.643861+0000","last_undegraded":"2023-10-09T06:08:28.643861+0000","last_fullsized":"2023-10-09T06:08:28.643861+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:16.166004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":465,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.645596+0000","last_change":"2023-10-09T06:06:01.068708+0000","last_active":"2023-10-09T06:08:28.645596+0000","last_peered":"2023-10-09T06:08:28.645596+0000","last_clean":"2023-10-09T06:08:28.645596+0000","last_became_active":"2023-10-09T06:06:01.068474+0000","last_became_peered":"2023-10-09T06:06:01.068474+0000","last_unstale":"2023-10-09T06:08:28.645596+0000","last_undegraded":"2023-10-09T06:08:28.645596+0000","last_fullsized":"2023-10-09T06:08:28.645596+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:31.749069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"444'6135","reported_seq":11769,"reported_epoch":444,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T06:08:39.956913+0000","last_change":"2023-10-09T06:07:44.746784+0000","last_active":"2023-10-09T06:08:39.956913+0000","last_peered":"2023-10-09T06:08:39.956913+0000","last_clean":"2023-10-09T06:07:42.380305+0000","last_became_active":"2023-10-09T06:07:44.713964+0000","last_became_peered":"2023-10-09T06:07:44.713964+0000","last_unstale":"2023-10-09T06:08:39.956913+0000","last_undegraded":"2023-10-09T06:08:39.956913+0000","last_fullsized":"2023-10-09T06:07:44.566632+0000","mapping_epoch":433,"log_start":"429'800","ondisk_log_start":"429'800","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":5335,"log_dups_size":0,"ondisk_log_size":5335,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:37.355319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402063360,"num_objects":6135,"num_object_clones":0,"num_object_copies":18405,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":446,"num_objects_unfound":0,"num_objects_dirty":6135,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6135,"num_write_kb":392640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2799,"num_bytes_recovered":183435264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3","objects":5689},{"shards":"1,3,7","objects":446}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":637,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.633093+0000","last_change":"2023-10-09T06:07:46.586758+0000","last_active":"2023-10-09T06:08:28.633093+0000","last_peered":"2023-10-09T06:08:28.633093+0000","last_clean":"2023-10-09T06:08:28.633093+0000","last_became_active":"2023-10-09T06:07:46.586421+0000","last_became_peered":"2023-10-09T06:07:46.586421+0000","last_unstale":"2023-10-09T06:08:28.633093+0000","last_undegraded":"2023-10-09T06:08:28.633093+0000","last_fullsized":"2023-10-09T06:08:28.633093+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:52.129816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:52.129816+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:12.513805+0000","scrub_duration":0.0039938459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":235,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.627739+0000","last_change":"2023-10-09T06:01:54.761275+0000","last_active":"2023-10-09T06:08:28.627739+0000","last_peered":"2023-10-09T06:08:28.627739+0000","last_clean":"2023-10-09T06:08:28.627739+0000","last_became_active":"2023-10-09T06:01:54.760782+0000","last_became_peered":"2023-10-09T06:01:54.760782+0000","last_unstale":"2023-10-09T06:08:28.627739+0000","last_undegraded":"2023-10-09T06:08:28.627739+0000","last_fullsized":"2023-10-09T06:08:28.627739+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:28.119846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":124,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T06:08:38.543381+0000","last_change":"2023-10-09T06:01:54.761768+0000","last_active":"2023-10-09T06:08:38.543381+0000","last_peered":"2023-10-09T06:08:38.543381+0000","last_clean":"2023-10-09T06:08:38.543381+0000","last_became_active":"2023-10-09T06:01:54.761443+0000","last_became_peered":"2023-10-09T06:01:54.761443+0000","last_unstale":"2023-10-09T06:08:38.543381+0000","last_undegraded":"2023-10-09T06:08:38.543381+0000","last_fullsized":"2023-10-09T06:08:38.543381+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:51.555987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"444'6109","reported_seq":6137,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T06:08:40.686930+0000","last_change":"2023-10-09T06:07:07.880280+0000","last_active":"2023-10-09T06:08:40.686930+0000","last_peered":"2023-10-09T06:08:40.686930+0000","last_clean":"2023-10-09T06:08:40.686930+0000","last_became_active":"2023-10-09T06:07:07.879952+0000","last_became_peered":"2023-10-09T06:07:07.879952+0000","last_unstale":"2023-10-09T06:08:40.686930+0000","last_undegraded":"2023-10-09T06:08:40.686930+0000","last_fullsized":"2023-10-09T06:08:40.686930+0000","mapping_epoch":425,"log_start":"429'700","ondisk_log_start":"429'700","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":5409,"log_dups_size":0,"ondisk_log_size":5409,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:10.864472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400359424,"num_objects":6109,"num_object_clones":0,"num_object_copies":18327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6109,"num_write_kb":390976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":867,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.634333+0000","last_change":"2023-10-09T05:59:56.092654+0000","last_active":"2023-10-09T06:08:28.634333+0000","last_peered":"2023-10-09T06:08:28.634333+0000","last_clean":"2023-10-09T06:08:28.634333+0000","last_became_active":"2023-10-09T05:59:32.529949+0000","last_became_peered":"2023-10-09T05:59:32.529949+0000","last_unstale":"2023-10-09T06:08:28.634333+0000","last_undegraded":"2023-10-09T06:08:28.634333+0000","last_fullsized":"2023-10-09T06:08:28.634333+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.092586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.092586+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:15:13.234903+0000","scrub_duration":0.0013484219999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":159,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.634865+0000","last_change":"2023-10-09T06:07:51.703936+0000","last_active":"2023-10-09T06:08:28.634865+0000","last_peered":"2023-10-09T06:08:28.634865+0000","last_clean":"2023-10-09T06:07:47.635546+0000","last_became_active":"2023-10-09T06:07:51.703936+0000","last_became_peered":"2023-10-09T06:07:51.703936+0000","last_unstale":"2023-10-09T06:08:28.634865+0000","last_undegraded":"2023-10-09T06:08:28.634865+0000","last_fullsized":"2023-10-09T06:07:51.696607+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:28.960353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":229,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.626396+0000","last_change":"2023-10-09T06:07:43.577306+0000","last_active":"2023-10-09T06:08:28.626396+0000","last_peered":"2023-10-09T06:08:28.626396+0000","last_clean":"2023-10-09T06:08:28.626396+0000","last_became_active":"2023-10-09T06:07:43.576956+0000","last_became_peered":"2023-10-09T06:07:43.576956+0000","last_unstale":"2023-10-09T06:08:28.626396+0000","last_undegraded":"2023-10-09T06:08:28.626396+0000","last_fullsized":"2023-10-09T06:08:28.626396+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:21.020403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":139,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.642438+0000","last_change":"2023-10-09T06:07:51.698969+0000","last_active":"2023-10-09T06:08:28.642438+0000","last_peered":"2023-10-09T06:08:28.642438+0000","last_clean":"2023-10-09T06:07:47.638309+0000","last_became_active":"2023-10-09T06:07:51.698969+0000","last_became_peered":"2023-10-09T06:07:51.698969+0000","last_unstale":"2023-10-09T06:08:28.642438+0000","last_undegraded":"2023-10-09T06:08:28.642438+0000","last_fullsized":"2023-10-09T06:07:51.697103+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:24.008054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"444'6284","reported_seq":6318,"reported_epoch":444,"state":"active+undersized+degraded","last_fresh":"2023-10-09T06:08:41.047807+0000","last_change":"2023-10-09T06:07:51.705556+0000","last_active":"2023-10-09T06:08:41.047807+0000","last_peered":"2023-10-09T06:08:41.047807+0000","last_clean":"2023-10-09T06:07:50.661954+0000","last_became_active":"2023-10-09T06:07:51.705556+0000","last_became_peered":"2023-10-09T06:07:51.705556+0000","last_unstale":"2023-10-09T06:08:41.047807+0000","last_undegraded":"2023-10-09T06:07:51.700065+0000","last_fullsized":"2023-10-09T06:07:51.699931+0000","mapping_epoch":438,"log_start":"429'700","ondisk_log_start":"429'700","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":5584,"log_dups_size":0,"ondisk_log_size":5584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:53:21.118229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411828224,"num_objects":6284,"num_object_clones":0,"num_object_copies":18852,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6284,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6284,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6284,"num_write_kb":402176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":6284}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":577,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.629360+0000","last_change":"2023-10-09T06:07:51.709788+0000","last_active":"2023-10-09T06:08:28.629360+0000","last_peered":"2023-10-09T06:08:28.629360+0000","last_clean":"2023-10-09T05:56:42.247103+0000","last_became_active":"2023-10-09T06:07:51.709788+0000","last_became_peered":"2023-10-09T06:07:51.709788+0000","last_unstale":"2023-10-09T06:08:28.629360+0000","last_undegraded":"2023-10-09T06:08:28.629360+0000","last_fullsized":"2023-10-09T06:07:51.702197+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":124,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.642542+0000","last_change":"2023-10-09T06:01:54.754294+0000","last_active":"2023-10-09T06:08:28.642542+0000","last_peered":"2023-10-09T06:08:28.642542+0000","last_clean":"2023-10-09T06:08:28.642542+0000","last_became_active":"2023-10-09T06:01:54.754024+0000","last_became_peered":"2023-10-09T06:01:54.754024+0000","last_unstale":"2023-10-09T06:08:28.642542+0000","last_undegraded":"2023-10-09T06:08:28.642542+0000","last_fullsized":"2023-10-09T06:08:28.642542+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:19.746882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":414,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.644140+0000","last_change":"2023-10-09T06:07:51.705726+0000","last_active":"2023-10-09T06:08:28.644140+0000","last_peered":"2023-10-09T06:08:28.644140+0000","last_clean":"2023-10-09T06:07:47.645621+0000","last_became_active":"2023-10-09T06:07:51.705726+0000","last_became_peered":"2023-10-09T06:07:51.705726+0000","last_unstale":"2023-10-09T06:08:28.644140+0000","last_undegraded":"2023-10-09T06:08:28.644140+0000","last_fullsized":"2023-10-09T06:07:51.697389+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:23.793318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":267,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.639877+0000","last_change":"2023-10-09T06:07:43.580765+0000","last_active":"2023-10-09T06:08:28.639877+0000","last_peered":"2023-10-09T06:08:28.639877+0000","last_clean":"2023-10-09T06:08:28.639877+0000","last_became_active":"2023-10-09T06:07:43.580458+0000","last_became_peered":"2023-10-09T06:07:43.580458+0000","last_unstale":"2023-10-09T06:08:28.639877+0000","last_undegraded":"2023-10-09T06:08:28.639877+0000","last_fullsized":"2023-10-09T06:08:28.639877+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:55.161893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"444'6111","reported_seq":6139,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T06:08:40.654172+0000","last_change":"2023-10-09T06:07:07.546882+0000","last_active":"2023-10-09T06:08:40.654172+0000","last_peered":"2023-10-09T06:08:40.654172+0000","last_clean":"2023-10-09T06:08:40.654172+0000","last_became_active":"2023-10-09T06:07:07.546552+0000","last_became_peered":"2023-10-09T06:07:07.546552+0000","last_unstale":"2023-10-09T06:08:40.654172+0000","last_undegraded":"2023-10-09T06:08:40.654172+0000","last_fullsized":"2023-10-09T06:08:40.654172+0000","mapping_epoch":425,"log_start":"429'700","ondisk_log_start":"429'700","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":5411,"log_dups_size":0,"ondisk_log_size":5411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:57.434915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400490496,"num_objects":6111,"num_object_clones":0,"num_object_copies":18333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6111,"num_write_kb":391104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":775,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.636759+0000","last_change":"2023-10-09T06:07:51.705111+0000","last_active":"2023-10-09T06:08:28.636759+0000","last_peered":"2023-10-09T06:08:28.636759+0000","last_clean":"2023-10-09T06:07:47.641128+0000","last_became_active":"2023-10-09T06:07:51.705111+0000","last_became_peered":"2023-10-09T06:07:51.705111+0000","last_unstale":"2023-10-09T06:08:28.636759+0000","last_undegraded":"2023-10-09T06:08:28.636759+0000","last_fullsized":"2023-10-09T06:07:51.699095+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:55.897812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:55.897812+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:40.812052+0000","scrub_duration":0.0014511420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":212,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.625975+0000","last_change":"2023-10-09T06:07:51.715505+0000","last_active":"2023-10-09T06:08:28.625975+0000","last_peered":"2023-10-09T06:08:28.625975+0000","last_clean":"2023-10-09T06:07:47.629573+0000","last_became_active":"2023-10-09T06:07:51.715505+0000","last_became_peered":"2023-10-09T06:07:51.715505+0000","last_unstale":"2023-10-09T06:08:28.625975+0000","last_undegraded":"2023-10-09T06:08:28.625975+0000","last_fullsized":"2023-10-09T06:07:51.700869+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:11.904358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":193,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.636178+0000","last_change":"2023-10-09T06:07:43.571911+0000","last_active":"2023-10-09T06:08:28.636178+0000","last_peered":"2023-10-09T06:08:28.636178+0000","last_clean":"2023-10-09T06:08:28.636178+0000","last_became_active":"2023-10-09T06:07:43.571539+0000","last_became_peered":"2023-10-09T06:07:43.571539+0000","last_unstale":"2023-10-09T06:08:28.636178+0000","last_undegraded":"2023-10-09T06:08:28.636178+0000","last_fullsized":"2023-10-09T06:08:28.636178+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:26.591500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":130,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.639794+0000","last_change":"2023-10-09T06:07:51.708660+0000","last_active":"2023-10-09T06:08:28.639794+0000","last_peered":"2023-10-09T06:08:28.639794+0000","last_clean":"2023-10-09T06:07:47.642813+0000","last_became_active":"2023-10-09T06:07:51.708660+0000","last_became_peered":"2023-10-09T06:07:51.708660+0000","last_unstale":"2023-10-09T06:08:28.639794+0000","last_undegraded":"2023-10-09T06:08:28.639794+0000","last_fullsized":"2023-10-09T06:07:51.697721+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:20:31.014324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"444'6114","reported_seq":6148,"reported_epoch":444,"state":"active+undersized+degraded","last_fresh":"2023-10-09T06:08:39.903702+0000","last_change":"2023-10-09T06:07:51.710785+0000","last_active":"2023-10-09T06:08:39.903702+0000","last_peered":"2023-10-09T06:08:39.903702+0000","last_clean":"2023-10-09T06:07:50.696644+0000","last_became_active":"2023-10-09T06:07:51.710785+0000","last_became_peered":"2023-10-09T06:07:51.710785+0000","last_unstale":"2023-10-09T06:08:39.903702+0000","last_undegraded":"2023-10-09T06:07:51.700829+0000","last_fullsized":"2023-10-09T06:07:51.700712+0000","mapping_epoch":438,"log_start":"429'800","ondisk_log_start":"429'800","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":5314,"log_dups_size":0,"ondisk_log_size":5314,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:10.653965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400687104,"num_objects":6114,"num_object_clones":0,"num_object_copies":18342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6114,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6114,"num_write_kb":391296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":6114}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":335,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.639067+0000","last_change":"2023-10-09T06:07:43.577756+0000","last_active":"2023-10-09T06:08:28.639067+0000","last_peered":"2023-10-09T06:08:28.639067+0000","last_clean":"2023-10-09T06:08:28.639067+0000","last_became_active":"2023-10-09T06:07:43.577430+0000","last_became_peered":"2023-10-09T06:07:43.577430+0000","last_unstale":"2023-10-09T06:08:28.639067+0000","last_undegraded":"2023-10-09T06:08:28.639067+0000","last_fullsized":"2023-10-09T06:08:28.639067+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:07.379946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":855,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.636315+0000","last_change":"2023-10-09T06:07:51.705658+0000","last_active":"2023-10-09T06:08:28.636315+0000","last_peered":"2023-10-09T06:08:28.636315+0000","last_clean":"2023-10-09T06:07:47.641104+0000","last_became_active":"2023-10-09T06:07:51.705658+0000","last_became_peered":"2023-10-09T06:07:51.705658+0000","last_unstale":"2023-10-09T06:08:28.636315+0000","last_undegraded":"2023-10-09T06:08:28.636315+0000","last_fullsized":"2023-10-09T06:07:51.700186+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:37.782173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:37.782173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:13.732682+0000","scrub_duration":0.001325908,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":211,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.629610+0000","last_change":"2023-10-09T05:57:23.479174+0000","last_active":"2023-10-09T06:08:28.629610+0000","last_peered":"2023-10-09T06:08:28.629610+0000","last_clean":"2023-10-09T06:08:28.629610+0000","last_became_active":"2023-10-09T05:57:23.478897+0000","last_became_peered":"2023-10-09T05:57:23.478897+0000","last_unstale":"2023-10-09T06:08:28.629610+0000","last_undegraded":"2023-10-09T06:08:28.629610+0000","last_fullsized":"2023-10-09T06:08:28.629610+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:00.160042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":354,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.645120+0000","last_change":"2023-10-09T06:07:51.701455+0000","last_active":"2023-10-09T06:08:28.645120+0000","last_peered":"2023-10-09T06:08:28.645120+0000","last_clean":"2023-10-09T06:07:47.645788+0000","last_became_active":"2023-10-09T06:07:51.701455+0000","last_became_peered":"2023-10-09T06:07:51.701455+0000","last_unstale":"2023-10-09T06:08:28.645120+0000","last_undegraded":"2023-10-09T06:08:28.645120+0000","last_fullsized":"2023-10-09T06:07:51.697157+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:50.734325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"444'6123","reported_seq":6156,"reported_epoch":444,"state":"active+undersized+degraded","last_fresh":"2023-10-09T06:08:38.730058+0000","last_change":"2023-10-09T06:07:51.706804+0000","last_active":"2023-10-09T06:08:38.730058+0000","last_peered":"2023-10-09T06:08:38.730058+0000","last_clean":"2023-10-09T06:07:50.696037+0000","last_became_active":"2023-10-09T06:07:51.706804+0000","last_became_peered":"2023-10-09T06:07:51.706804+0000","last_unstale":"2023-10-09T06:08:38.730058+0000","last_undegraded":"2023-10-09T06:07:51.703995+0000","last_fullsized":"2023-10-09T06:07:51.703886+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":6123,"log_dups_size":0,"ondisk_log_size":6123,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:25.587346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401276928,"num_objects":6123,"num_object_clones":0,"num_object_copies":18369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6123,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6123,"num_write_kb":391872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"5,6","objects":6123}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":659,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.634327+0000","last_change":"2023-10-09T06:07:06.560255+0000","last_active":"2023-10-09T06:08:28.634327+0000","last_peered":"2023-10-09T06:08:28.634327+0000","last_clean":"2023-10-09T06:08:28.634327+0000","last_became_active":"2023-10-09T06:07:06.559524+0000","last_became_peered":"2023-10-09T06:07:06.559524+0000","last_unstale":"2023-10-09T06:08:28.634327+0000","last_undegraded":"2023-10-09T06:08:28.634327+0000","last_fullsized":"2023-10-09T06:08:28.634327+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:57.627296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:57.627296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:41.099391+0000","scrub_duration":0.001913166,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":327,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.636041+0000","last_change":"2023-10-09T06:07:51.704104+0000","last_active":"2023-10-09T06:08:28.636041+0000","last_peered":"2023-10-09T06:08:28.636041+0000","last_clean":"2023-10-09T06:07:47.640939+0000","last_became_active":"2023-10-09T06:07:51.704104+0000","last_became_peered":"2023-10-09T06:07:51.704104+0000","last_unstale":"2023-10-09T06:08:28.636041+0000","last_undegraded":"2023-10-09T06:08:28.636041+0000","last_fullsized":"2023-10-09T06:07:51.699516+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:07.052928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":282,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.643608+0000","last_change":"2023-10-09T06:07:51.710700+0000","last_active":"2023-10-09T06:08:28.643608+0000","last_peered":"2023-10-09T06:08:28.643608+0000","last_clean":"2023-10-09T06:00:51.356444+0000","last_became_active":"2023-10-09T06:07:51.710700+0000","last_became_peered":"2023-10-09T06:07:51.710700+0000","last_unstale":"2023-10-09T06:08:28.643608+0000","last_undegraded":"2023-10-09T06:08:28.643608+0000","last_fullsized":"2023-10-09T06:07:51.696429+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:21.980685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":523,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.640295+0000","last_change":"2023-10-09T06:07:51.702467+0000","last_active":"2023-10-09T06:08:28.640295+0000","last_peered":"2023-10-09T06:08:28.640295+0000","last_clean":"2023-10-09T06:07:35.533292+0000","last_became_active":"2023-10-09T06:07:51.702467+0000","last_became_peered":"2023-10-09T06:07:51.702467+0000","last_unstale":"2023-10-09T06:08:28.640295+0000","last_undegraded":"2023-10-09T06:08:28.640295+0000","last_fullsized":"2023-10-09T06:07:51.696276+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:08.479151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"444'6095","reported_seq":6122,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T06:08:41.048569+0000","last_change":"2023-10-09T06:07:07.579770+0000","last_active":"2023-10-09T06:08:41.048569+0000","last_peered":"2023-10-09T06:08:41.048569+0000","last_clean":"2023-10-09T06:08:41.048569+0000","last_became_active":"2023-10-09T06:07:07.579317+0000","last_became_peered":"2023-10-09T06:07:07.579317+0000","last_unstale":"2023-10-09T06:08:41.048569+0000","last_undegraded":"2023-10-09T06:08:41.048569+0000","last_fullsized":"2023-10-09T06:08:41.048569+0000","mapping_epoch":425,"log_start":"429'500","ondisk_log_start":"429'500","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":5595,"log_dups_size":0,"ondisk_log_size":5595,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:30.621836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399441920,"num_objects":6095,"num_object_clones":0,"num_object_copies":18285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6095,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6095,"num_write_kb":390080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":864,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.634094+0000","last_change":"2023-10-09T06:07:51.706987+0000","last_active":"2023-10-09T06:08:28.634094+0000","last_peered":"2023-10-09T06:08:28.634094+0000","last_clean":"2023-10-09T06:07:47.656698+0000","last_became_active":"2023-10-09T06:07:51.706987+0000","last_became_peered":"2023-10-09T06:07:51.706987+0000","last_unstale":"2023-10-09T06:08:28.634094+0000","last_undegraded":"2023-10-09T06:08:28.634094+0000","last_fullsized":"2023-10-09T06:07:51.705453+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:54.074310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:54.074310+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:45.083735+0000","scrub_duration":0.011040648,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":302,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.637426+0000","last_change":"2023-10-09T06:07:51.702808+0000","last_active":"2023-10-09T06:08:28.637426+0000","last_peered":"2023-10-09T06:08:28.637426+0000","last_clean":"2023-10-09T06:07:47.641335+0000","last_became_active":"2023-10-09T06:07:51.702808+0000","last_became_peered":"2023-10-09T06:07:51.702808+0000","last_unstale":"2023-10-09T06:08:28.637426+0000","last_undegraded":"2023-10-09T06:08:28.637426+0000","last_fullsized":"2023-10-09T06:07:51.699160+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:01.489172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":142,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.635020+0000","last_change":"2023-10-09T06:07:43.578431+0000","last_active":"2023-10-09T06:08:28.635020+0000","last_peered":"2023-10-09T06:08:28.635020+0000","last_clean":"2023-10-09T06:08:28.635020+0000","last_became_active":"2023-10-09T06:07:43.578110+0000","last_became_peered":"2023-10-09T06:07:43.578110+0000","last_unstale":"2023-10-09T06:08:28.635020+0000","last_undegraded":"2023-10-09T06:08:28.635020+0000","last_fullsized":"2023-10-09T06:08:28.635020+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:18.556487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":223,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.639161+0000","last_change":"2023-10-09T06:07:51.706126+0000","last_active":"2023-10-09T06:08:28.639161+0000","last_peered":"2023-10-09T06:08:28.639161+0000","last_clean":"2023-10-09T06:07:47.641062+0000","last_became_active":"2023-10-09T06:07:51.706126+0000","last_became_peered":"2023-10-09T06:07:51.706126+0000","last_unstale":"2023-10-09T06:08:28.639161+0000","last_undegraded":"2023-10-09T06:08:28.639161+0000","last_fullsized":"2023-10-09T06:07:51.700276+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:41.030190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.d","version":"444'6126","reported_seq":6153,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T06:08:41.013899+0000","last_change":"2023-10-09T06:07:07.883000+0000","last_active":"2023-10-09T06:08:41.013899+0000","last_peered":"2023-10-09T06:08:41.013899+0000","last_clean":"2023-10-09T06:08:41.013899+0000","last_became_active":"2023-10-09T06:07:07.882612+0000","last_became_peered":"2023-10-09T06:07:07.882612+0000","last_unstale":"2023-10-09T06:08:41.013899+0000","last_undegraded":"2023-10-09T06:08:41.013899+0000","last_fullsized":"2023-10-09T06:08:41.013899+0000","mapping_epoch":425,"log_start":"429'500","ondisk_log_start":"429'500","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":5626,"log_dups_size":0,"ondisk_log_size":5626,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:14:21.633273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401473536,"num_objects":6126,"num_object_clones":0,"num_object_copies":18378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6126,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6126,"num_write_kb":392064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":897,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.645328+0000","last_change":"2023-10-09T06:07:46.590898+0000","last_active":"2023-10-09T06:08:28.645328+0000","last_peered":"2023-10-09T06:08:28.645328+0000","last_clean":"2023-10-09T06:08:28.645328+0000","last_became_active":"2023-10-09T06:07:46.590596+0000","last_became_peered":"2023-10-09T06:07:46.590596+0000","last_unstale":"2023-10-09T06:08:28.645328+0000","last_undegraded":"2023-10-09T06:08:28.645328+0000","last_fullsized":"2023-10-09T06:08:28.645328+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:17.344518+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:17.344518+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:00.399548+0000","scrub_duration":0.0015914099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":460,"reported_epoch":443,"state":"active+undersized+degraded","last_fresh":"2023-10-09T06:08:28.628227+0000","last_change":"2023-10-09T06:07:51.710841+0000","last_active":"2023-10-09T06:08:28.628227+0000","last_peered":"2023-10-09T06:08:28.628227+0000","last_clean":"2023-10-09T06:07:47.627846+0000","last_became_active":"2023-10-09T06:07:51.710841+0000","last_became_peered":"2023-10-09T06:07:51.710841+0000","last_unstale":"2023-10-09T06:08:28.628227+0000","last_undegraded":"2023-10-09T06:07:51.705289+0000","last_fullsized":"2023-10-09T06:07:51.705193+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:41.829341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":240,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.640445+0000","last_change":"2023-10-09T06:07:06.560512+0000","last_active":"2023-10-09T06:08:28.640445+0000","last_peered":"2023-10-09T06:08:28.640445+0000","last_clean":"2023-10-09T06:08:28.640445+0000","last_became_active":"2023-10-09T06:07:06.560204+0000","last_became_peered":"2023-10-09T06:07:06.560204+0000","last_unstale":"2023-10-09T06:08:28.640445+0000","last_undegraded":"2023-10-09T06:08:28.640445+0000","last_fullsized":"2023-10-09T06:08:28.640445+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:19.102248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":217,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.626289+0000","last_change":"2023-10-09T06:07:51.710778+0000","last_active":"2023-10-09T06:08:28.626289+0000","last_peered":"2023-10-09T06:08:28.626289+0000","last_clean":"2023-10-09T06:07:47.627277+0000","last_became_active":"2023-10-09T06:07:51.710778+0000","last_became_peered":"2023-10-09T06:07:51.710778+0000","last_unstale":"2023-10-09T06:08:28.626289+0000","last_undegraded":"2023-10-09T06:08:28.626289+0000","last_fullsized":"2023-10-09T06:07:51.704350+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:56.304788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"444'6062","reported_seq":6090,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T06:08:39.926752+0000","last_change":"2023-10-09T06:07:07.580272+0000","last_active":"2023-10-09T06:08:39.926752+0000","last_peered":"2023-10-09T06:08:39.926752+0000","last_clean":"2023-10-09T06:08:39.926752+0000","last_became_active":"2023-10-09T06:07:07.579909+0000","last_became_peered":"2023-10-09T06:07:07.579909+0000","last_unstale":"2023-10-09T06:08:39.926752+0000","last_undegraded":"2023-10-09T06:08:39.926752+0000","last_fullsized":"2023-10-09T06:08:39.926752+0000","mapping_epoch":425,"log_start":"429'700","ondisk_log_start":"429'700","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":5362,"log_dups_size":0,"ondisk_log_size":5362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:18.048371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397279232,"num_objects":6062,"num_object_clones":0,"num_object_copies":18186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6062,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6062,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":705,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.625956+0000","last_change":"2023-10-09T05:58:34.333129+0000","last_active":"2023-10-09T06:08:28.625956+0000","last_peered":"2023-10-09T06:08:28.625956+0000","last_clean":"2023-10-09T06:08:28.625956+0000","last_became_active":"2023-10-09T05:57:26.502911+0000","last_became_peered":"2023-10-09T05:57:26.502911+0000","last_unstale":"2023-10-09T06:08:28.625956+0000","last_undegraded":"2023-10-09T06:08:28.625956+0000","last_fullsized":"2023-10-09T06:08:28.625956+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:25.230712+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":468,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.634552+0000","last_change":"2023-10-09T06:07:43.573674+0000","last_active":"2023-10-09T06:08:28.634552+0000","last_peered":"2023-10-09T06:08:28.634552+0000","last_clean":"2023-10-09T06:08:28.634552+0000","last_became_active":"2023-10-09T06:07:43.573276+0000","last_became_peered":"2023-10-09T06:07:43.573276+0000","last_unstale":"2023-10-09T06:08:28.634552+0000","last_undegraded":"2023-10-09T06:08:28.634552+0000","last_fullsized":"2023-10-09T06:08:28.634552+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:17.556584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":404,"reported_epoch":443,"state":"active+undersized+degraded","last_fresh":"2023-10-09T06:08:28.640171+0000","last_change":"2023-10-09T06:07:51.702127+0000","last_active":"2023-10-09T06:08:28.640171+0000","last_peered":"2023-10-09T06:08:28.640171+0000","last_clean":"2023-10-09T06:01:48.340775+0000","last_became_active":"2023-10-09T06:07:51.702127+0000","last_became_peered":"2023-10-09T06:07:51.702127+0000","last_unstale":"2023-10-09T06:08:28.640171+0000","last_undegraded":"2023-10-09T06:07:51.698001+0000","last_fullsized":"2023-10-09T06:07:51.697901+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:52.260216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":390,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.643932+0000","last_change":"2023-10-09T06:01:22.334081+0000","last_active":"2023-10-09T06:08:28.643932+0000","last_peered":"2023-10-09T06:08:28.643932+0000","last_clean":"2023-10-09T06:08:28.643932+0000","last_became_active":"2023-10-09T06:01:22.333663+0000","last_became_peered":"2023-10-09T06:01:22.333663+0000","last_unstale":"2023-10-09T06:08:28.643932+0000","last_undegraded":"2023-10-09T06:08:28.643932+0000","last_fullsized":"2023-10-09T06:08:28.643932+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:55.809259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":678,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.625705+0000","last_change":"2023-10-09T06:07:51.705825+0000","last_active":"2023-10-09T06:08:28.625705+0000","last_peered":"2023-10-09T06:08:28.625705+0000","last_clean":"2023-10-09T06:07:47.629378+0000","last_became_active":"2023-10-09T06:07:51.705825+0000","last_became_peered":"2023-10-09T06:07:51.705825+0000","last_unstale":"2023-10-09T06:08:28.625705+0000","last_undegraded":"2023-10-09T06:08:28.625705+0000","last_fullsized":"2023-10-09T06:07:51.701768+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.181811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.181811+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:15.998745+0000","scrub_duration":0.011553313000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":38,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.644614+0000","last_change":"2023-10-09T06:07:51.701192+0000","last_active":"2023-10-09T06:08:28.644614+0000","last_peered":"2023-10-09T06:08:28.644614+0000","last_clean":"2023-10-09T05:40:11.603872+0000","last_became_active":"2023-10-09T06:07:51.701192+0000","last_became_peered":"2023-10-09T06:07:51.701192+0000","last_unstale":"2023-10-09T06:08:28.644614+0000","last_undegraded":"2023-10-09T06:08:28.644614+0000","last_fullsized":"2023-10-09T06:07:51.697698+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:45.907541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":143,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.635792+0000","last_change":"2023-10-09T06:07:43.580844+0000","last_active":"2023-10-09T06:08:28.635792+0000","last_peered":"2023-10-09T06:08:28.635792+0000","last_clean":"2023-10-09T06:08:28.635792+0000","last_became_active":"2023-10-09T06:07:43.580523+0000","last_became_peered":"2023-10-09T06:07:43.580523+0000","last_unstale":"2023-10-09T06:08:28.635792+0000","last_undegraded":"2023-10-09T06:08:28.635792+0000","last_fullsized":"2023-10-09T06:08:28.635792+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:55.942430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":790,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.625992+0000","last_change":"2023-10-09T06:00:04.215425+0000","last_active":"2023-10-09T06:08:28.625992+0000","last_peered":"2023-10-09T06:08:28.625992+0000","last_clean":"2023-10-09T06:08:28.625992+0000","last_became_active":"2023-10-09T05:59:40.326685+0000","last_became_peered":"2023-10-09T05:59:40.326685+0000","last_unstale":"2023-10-09T06:08:28.625992+0000","last_undegraded":"2023-10-09T06:08:28.625992+0000","last_fullsized":"2023-10-09T06:08:28.625992+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:04.215270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:04.215270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:03.963694+0000","scrub_duration":0.002850652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":215,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.645523+0000","last_change":"2023-10-09T06:01:54.824780+0000","last_active":"2023-10-09T06:08:28.645523+0000","last_peered":"2023-10-09T06:08:28.645523+0000","last_clean":"2023-10-09T06:08:28.645523+0000","last_became_active":"2023-10-09T06:01:54.757727+0000","last_became_peered":"2023-10-09T06:01:54.757727+0000","last_unstale":"2023-10-09T06:08:28.645523+0000","last_undegraded":"2023-10-09T06:08:28.645523+0000","last_fullsized":"2023-10-09T06:08:28.645523+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:44.132556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":462,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.644781+0000","last_change":"2023-10-09T06:01:54.764989+0000","last_active":"2023-10-09T06:08:28.644781+0000","last_peered":"2023-10-09T06:08:28.644781+0000","last_clean":"2023-10-09T06:08:28.644781+0000","last_became_active":"2023-10-09T06:01:54.764790+0000","last_became_peered":"2023-10-09T06:01:54.764790+0000","last_unstale":"2023-10-09T06:08:28.644781+0000","last_undegraded":"2023-10-09T06:08:28.644781+0000","last_fullsized":"2023-10-09T06:08:28.644781+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:14:07.208858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":201,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.644933+0000","last_change":"2023-10-09T06:01:22.342190+0000","last_active":"2023-10-09T06:08:28.644933+0000","last_peered":"2023-10-09T06:08:28.644933+0000","last_clean":"2023-10-09T06:08:28.644933+0000","last_became_active":"2023-10-09T06:01:22.341855+0000","last_became_peered":"2023-10-09T06:01:22.341855+0000","last_unstale":"2023-10-09T06:08:28.644933+0000","last_undegraded":"2023-10-09T06:08:28.644933+0000","last_fullsized":"2023-10-09T06:08:28.644933+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:01:21.322371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":449,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.644300+0000","last_change":"2023-10-09T06:07:51.719512+0000","last_active":"2023-10-09T06:08:28.644300+0000","last_peered":"2023-10-09T06:08:28.644300+0000","last_clean":"2023-10-09T06:07:47.646429+0000","last_became_active":"2023-10-09T06:07:51.719512+0000","last_became_peered":"2023-10-09T06:07:51.719512+0000","last_unstale":"2023-10-09T06:08:28.644300+0000","last_undegraded":"2023-10-09T06:08:28.644300+0000","last_fullsized":"2023-10-09T06:07:51.697099+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:13.633296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":710,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.637070+0000","last_change":"2023-10-09T06:01:57.801991+0000","last_active":"2023-10-09T06:08:28.637070+0000","last_peered":"2023-10-09T06:08:28.637070+0000","last_clean":"2023-10-09T06:08:28.637070+0000","last_became_active":"2023-10-09T06:01:57.801579+0000","last_became_peered":"2023-10-09T06:01:57.801579+0000","last_unstale":"2023-10-09T06:08:28.637070+0000","last_undegraded":"2023-10-09T06:08:28.637070+0000","last_fullsized":"2023-10-09T06:08:28.637070+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:00.456132+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:00.456132+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:48.548353+0000","scrub_duration":0.0014992199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":238,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.626425+0000","last_change":"2023-10-09T06:07:43.579543+0000","last_active":"2023-10-09T06:08:28.626425+0000","last_peered":"2023-10-09T06:08:28.626425+0000","last_clean":"2023-10-09T06:08:28.626425+0000","last_became_active":"2023-10-09T06:07:43.579184+0000","last_became_peered":"2023-10-09T06:07:43.579184+0000","last_unstale":"2023-10-09T06:08:28.626425+0000","last_undegraded":"2023-10-09T06:08:28.626425+0000","last_fullsized":"2023-10-09T06:08:28.626425+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:23.460469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":234,"reported_epoch":443,"state":"active+undersized","last_fresh":"2023-10-09T06:08:28.626111+0000","last_change":"2023-10-09T06:07:51.715145+0000","last_active":"2023-10-09T06:08:28.626111+0000","last_peered":"2023-10-09T06:08:28.626111+0000","last_clean":"2023-10-09T06:07:47.629643+0000","last_became_active":"2023-10-09T06:07:51.715145+0000","last_became_peered":"2023-10-09T06:07:51.715145+0000","last_unstale":"2023-10-09T06:08:28.626111+0000","last_undegraded":"2023-10-09T06:08:28.626111+0000","last_fullsized":"2023-10-09T06:07:51.700531+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:18.521772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":731,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.638323+0000","last_change":"2023-10-09T06:07:06.560275+0000","last_active":"2023-10-09T06:08:28.638323+0000","last_peered":"2023-10-09T06:08:28.638323+0000","last_clean":"2023-10-09T06:08:28.638323+0000","last_became_active":"2023-10-09T06:07:06.559953+0000","last_became_peered":"2023-10-09T06:07:06.559953+0000","last_unstale":"2023-10-09T06:08:28.638323+0000","last_undegraded":"2023-10-09T06:08:28.638323+0000","last_fullsized":"2023-10-09T06:08:28.638323+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:46.290510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:46.290510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:10.910416+0000","scrub_duration":0.002627819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":49,"reported_epoch":443,"state":"active+undersized+degraded","last_fresh":"2023-10-09T06:08:28.630788+0000","last_change":"2023-10-09T06:07:51.710347+0000","last_active":"2023-10-09T06:08:28.630788+0000","last_peered":"2023-10-09T06:08:28.630788+0000","last_clean":"2023-10-09T05:40:15.633679+0000","last_became_active":"2023-10-09T06:07:51.710347+0000","last_became_peered":"2023-10-09T06:07:51.710347+0000","last_unstale":"2023-10-09T06:08:28.630788+0000","last_undegraded":"2023-10-09T06:07:51.703005+0000","last_fullsized":"2023-10-09T06:07:51.702895+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:04.366013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":465,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T06:08:28.643444+0000","last_change":"2023-10-09T06:06:01.068828+0000","last_active":"2023-10-09T06:08:28.643444+0000","last_peered":"2023-10-09T06:08:28.643444+0000","last_clean":"2023-10-09T06:08:28.643444+0000","last_became_active":"2023-10-09T06:06:01.068491+0000","last_became_peered":"2023-10-09T06:06:01.068491+0000","last_unstale":"2023-10-09T06:08:28.643444+0000","last_undegraded":"2023-10-09T06:08:28.643444+0000","last_fullsized":"2023-10-09T06:08:28.643444+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:25.661796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":84,"acting":84,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":354,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10938,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":84,"acting":84,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":635093,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":37,"num_read_kb":69,"num_write":18,"num_write_kb":183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":659551,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":82,"acting":82,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":6460407808,"num_objects":98578,"num_object_clones":0,"num_object_copies":295734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":31459,"num_objects_misplaced":446,"num_objects_unfound":0,"num_objects_dirty":98578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":98578,"num_write_kb":6308992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2799,"num_bytes_recovered":183435264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18901434368,"data_stored":18901434368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":88078,"ondisk_log_size":88078,"up":44,"acting":41,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":20,"acting":20,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234348,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5044964,"kb_used_data":4156444,"kb_used_omap":0,"kb_used_meta":888512,"kb_avail":88700188,"statfs":{"total":95995035648,"available":90828992512,"internally_reserved":0,"allocated":4256198656,"data_stored":4254919566,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":909836288},"hb_peers":[1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":2.7919999999999998,"15min":3.2309999999999999},"min":{"1min":0.22700000000000001,"5min":0.218,"15min":0.218},"max":{"1min":5.0229999999999997,"5min":99.459999999999994,"15min":165.792},"last":0.45900000000000002},{"interface":"front","average":{"1min":1.5800000000000001,"5min":3.0670000000000002,"15min":3.3700000000000001},"min":{"1min":0.26700000000000002,"5min":0.24399999999999999,"15min":0.21299999999999999},"max":{"1min":9.7149999999999999,"5min":100.096,"15min":166.00700000000001},"last":0.52400000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:07:50 2023","interfaces":[{"interface":"back","average":{"1min":2.145,"5min":4.0430000000000001,"15min":2.2559999999999998},"min":{"1min":0.27300000000000002,"5min":0.25,"15min":0.249},"max":{"1min":18.614999999999998,"5min":64.760000000000005,"15min":64.760000000000005},"last":0.55700000000000005},{"interface":"front","average":{"1min":2.2309999999999999,"5min":4.0439999999999996,"15min":2.1309999999999998},"min":{"1min":0.32200000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":21.341999999999999,"5min":65.102000000000004,"15min":65.102000000000004},"last":0.57599999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:08:19 2023","interfaces":[{"interface":"back","average":{"1min":1.8100000000000001,"5min":1.76,"15min":2.7290000000000001},"min":{"1min":0.32600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.5270000000000001,"5min":16.878,"15min":127.86499999999999},"last":1.833},{"interface":"front","average":{"1min":1.627,"5min":1.8799999999999999,"15min":2.6739999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.5860000000000003,"5min":12.456,"15min":129.24000000000001},"last":0.67800000000000005}]},{"osd":3,"last update":"Mon Oct 9 06:07:50 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":2.52,"15min":4.625},"min":{"1min":0.35099999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":22.013000000000002,"5min":36.402000000000001,"15min":164.16999999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":2.0310000000000001,"5min":2.601,"15min":4.6369999999999996},"min":{"1min":0.39800000000000002,"5min":0.34399999999999997,"15min":0.30299999999999999},"max":{"1min":19.393000000000001,"5min":36.872,"15min":164.40700000000001},"last":1.867}]},{"osd":4,"last update":"Mon Oct 9 06:08:29 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":0.90300000000000002,"15min":0.81999999999999995},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":3.0350000000000001,"5min":5.5570000000000004,"15min":5.5570000000000004},"last":1.8899999999999999},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.95499999999999996,"15min":0.86599999999999999},"min":{"1min":0.27700000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.8419999999999996,"5min":5.2000000000000002,"15min":5.2000000000000002},"last":0.61399999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:08:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2110000000000001,"5min":1.0740000000000001,"15min":0.91300000000000003},"min":{"1min":0.248,"5min":0.247,"15min":0.247},"max":{"1min":9.4199999999999999,"5min":9.7910000000000004,"15min":15.622},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.2869999999999999,"5min":1.226,"15min":0.90600000000000003},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.649,"5min":9.3409999999999993,"15min":9.3409999999999993},"last":0.53600000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:08:39 2023","interfaces":[{"interface":"back","average":{"1min":1.4570000000000001,"5min":1.2649999999999999,"15min":0.98299999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":9.4540000000000006,"5min":9.4540000000000006,"15min":12.727},"last":5.71},{"interface":"front","average":{"1min":1.482,"5min":1.1870000000000001,"15min":0.92900000000000005},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.476,"5min":5.5800000000000001,"15min":5.5800000000000001},"last":1.915}]}]},{"osd":6,"up_from":46,"seq":197568495984,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2291332,"kb_used_data":1389692,"kb_used_omap":0,"kb_used_meta":901632,"kb_avail":91453820,"statfs":{"total":95995035648,"available":93648711680,"internally_reserved":0,"allocated":1423044608,"data_stored":1421789114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923271168},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":5.782,"15min":3.6629999999999998},"min":{"1min":0.27100000000000002,"5min":0.24299999999999999,"15min":0.224},"max":{"1min":5.0289999999999999,"5min":225.422,"15min":225.422},"last":0.29399999999999998},{"interface":"front","average":{"1min":2.0329999999999999,"5min":6.1470000000000002,"15min":3.794},"min":{"1min":0.28899999999999998,"5min":0.23000000000000001,"15min":0.22800000000000001},"max":{"1min":15.311999999999999,"5min":225.255,"15min":225.255},"last":0.38800000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:08:06 2023","interfaces":[{"interface":"back","average":{"1min":9.0449999999999999,"5min":8.0600000000000005,"15min":3.411},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":91.188000000000002,"5min":109.101,"15min":109.101},"last":0.53200000000000003},{"interface":"front","average":{"1min":8.4209999999999994,"5min":7.8799999999999999,"15min":3.4529999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27000000000000002},"max":{"1min":91.372,"5min":109.071,"15min":109.071},"last":0.47799999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:08:14 2023","interfaces":[{"interface":"back","average":{"1min":1.764,"5min":2.0139999999999998,"15min":2.1040000000000001},"min":{"1min":0.39700000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.8869999999999996,"5min":15.374000000000001,"15min":61.088999999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":1.3799999999999999,"5min":1.835,"15min":2.0110000000000001},"min":{"1min":0.29199999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.4770000000000003,"5min":15.429,"15min":61.082000000000001},"last":0.61899999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:08:02 2023","interfaces":[{"interface":"back","average":{"1min":8.1620000000000008,"5min":7.9269999999999996,"15min":5.8769999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.29599999999999999},"max":{"1min":84.573999999999998,"5min":138.13300000000001,"15min":138.13300000000001},"last":0.67000000000000004},{"interface":"front","average":{"1min":8.6120000000000001,"5min":8.1080000000000005,"15min":5.8490000000000002},"min":{"1min":0.41299999999999998,"5min":0.36599999999999999,"15min":0.28499999999999998},"max":{"1min":84.495999999999995,"5min":137.215,"15min":137.215},"last":0.45600000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:08:35 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":2.8170000000000002,"15min":1.7549999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":2.3300000000000001,"5min":151.35599999999999,"15min":151.35599999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.76000000000000001,"5min":3.016,"15min":1.7090000000000001},"min":{"1min":0.315,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":5.6520000000000001,"5min":149.15799999999999,"15min":149.15799999999999},"last":0.60099999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:08:06 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":1.423,"15min":1.1719999999999999},"min":{"1min":0.315,"5min":0.30299999999999999,"15min":0.23899999999999999},"max":{"1min":2.1379999999999999,"5min":15.342000000000001,"15min":45.933999999999997},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.72899999999999998,"5min":1.4370000000000001,"15min":1.1830000000000001},"min":{"1min":0.39400000000000002,"5min":0.29199999999999998,"15min":0.255},"max":{"1min":1.7330000000000001,"5min":15.358000000000001,"15min":45.917999999999999},"last":0.63400000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:08:35 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":1.3939999999999999,"15min":1.284},"min":{"1min":0.377,"5min":0.312,"15min":0.27000000000000002},"max":{"1min":5.6929999999999996,"5min":21.98,"15min":52.042999999999999},"last":0.65200000000000002},{"interface":"front","average":{"1min":1.1259999999999999,"5min":1.4550000000000001,"15min":1.258},"min":{"1min":0.36199999999999999,"5min":0.33300000000000002,"15min":0.32200000000000001},"max":{"1min":5.2830000000000004,"5min":18.241,"15min":51.259},"last":0.56499999999999995}]}]},{"osd":1,"up_from":89,"seq":382252089573,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4021572,"kb_used_data":3551100,"kb_used_omap":9,"kb_used_meta":470454,"kb_avail":89723580,"statfs":{"total":95995035648,"available":91876945920,"internally_reserved":0,"allocated":3636326400,"data_stored":3633784584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":481745897},"hb_peers":[2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:25 2023","interfaces":[{"interface":"back","average":{"1min":15.635999999999999,"5min":7.8259999999999996,"15min":4.2140000000000004},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.191},"max":{"1min":154.066,"5min":154.066,"15min":154.066},"last":0.77300000000000002},{"interface":"front","average":{"1min":15.811999999999999,"5min":7.8650000000000002,"15min":4.3879999999999999},"min":{"1min":0.22900000000000001,"5min":0.218,"15min":0.21299999999999999},"max":{"1min":156.19900000000001,"5min":156.19900000000001,"15min":156.19900000000001},"last":0.67800000000000005}]},{"osd":2,"last update":"Mon Oct 9 06:08:35 2023","interfaces":[{"interface":"back","average":{"1min":2.6800000000000002,"5min":1.8979999999999999,"15min":3.3149999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.20799999999999999},"max":{"1min":17.681000000000001,"5min":17.681000000000001,"15min":204.989},"last":0.21099999999999999},{"interface":"front","average":{"1min":1.681,"5min":1.768,"15min":3.4169999999999998},"min":{"1min":0.24299999999999999,"5min":0.22700000000000001,"15min":0.20300000000000001},"max":{"1min":6.6360000000000001,"5min":10.699,"15min":203.65899999999999},"last":2.246}]},{"osd":3,"last update":"Mon Oct 9 06:07:55 2023","interfaces":[{"interface":"back","average":{"1min":1.466,"5min":3.7050000000000001,"15min":4.8029999999999999},"min":{"1min":0.27100000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.7489999999999997,"5min":142.97,"15min":213.654},"last":0.24199999999999999},{"interface":"front","average":{"1min":1.5029999999999999,"5min":3.706,"15min":4.8179999999999996},"min":{"1min":0.28699999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.5999999999999996,"5min":142.94800000000001,"15min":213.77199999999999},"last":0.28100000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:08:35 2023","interfaces":[{"interface":"back","average":{"1min":3.4500000000000002,"5min":2.1509999999999998,"15min":1.423},"min":{"1min":0.31,"5min":0.308,"15min":0.30399999999999999},"max":{"1min":35.366,"5min":35.366,"15min":35.366},"last":0.84599999999999997},{"interface":"front","average":{"1min":4.8879999999999999,"5min":2.5089999999999999,"15min":1.853},"min":{"1min":0.371,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":35.384999999999998,"5min":35.384999999999998,"15min":35.384999999999998},"last":1.4470000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:07:42 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.9179999999999999,"15min":1.5129999999999999},"min":{"1min":0.35499999999999998,"5min":0.34300000000000003,"15min":0.32800000000000001},"max":{"1min":4.0179999999999998,"5min":36.356999999999999,"15min":64.584999999999994},"last":0.89600000000000002},{"interface":"front","average":{"1min":1.1040000000000001,"5min":2.2639999999999998,"15min":1.7809999999999999},"min":{"1min":0.47699999999999998,"5min":0.35799999999999998,"15min":0.315},"max":{"1min":3.6110000000000002,"5min":36.268000000000001,"15min":64.150999999999996},"last":4.3419999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:07:42 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.675,"15min":1.2869999999999999},"min":{"1min":0.44500000000000001,"5min":0.33100000000000002,"15min":0.30099999999999999},"max":{"1min":3.254,"5min":10.81,"15min":10.81},"last":5.1390000000000002},{"interface":"front","average":{"1min":1.1020000000000001,"5min":1.9710000000000001,"15min":1.55},"min":{"1min":0.42099999999999999,"5min":0.39900000000000002,"15min":0.35499999999999998},"max":{"1min":3.6760000000000002,"5min":9.2949999999999999,"15min":9.2949999999999999},"last":4.1479999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:07:42 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":2.3439999999999999,"15min":2.1840000000000002},"min":{"1min":0.48199999999999998,"5min":0.39400000000000002,"15min":0.314},"max":{"1min":3.6469999999999998,"5min":37.963999999999999,"15min":108.553},"last":1.2030000000000001},{"interface":"front","average":{"1min":1.034,"5min":1.9219999999999999,"15min":1.903},"min":{"1min":0.46600000000000003,"5min":0.39300000000000002,"15min":0.38},"max":{"1min":3.3889999999999998,"5min":37.950000000000003,"15min":108.56999999999999},"last":1.1870000000000001}]}]},{"osd":0,"up_from":129,"seq":554050781365,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1578356,"kb_used_data":1147052,"kb_used_omap":29,"kb_used_meta":431266,"kb_avail":92166796,"statfs":{"total":95995035648,"available":94378799104,"internally_reserved":0,"allocated":1174581248,"data_stored":1172301441,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30322,"internal_metadata":441616782},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:07:20 2023","interfaces":[{"interface":"back","average":{"1min":12.545,"5min":7.6920000000000002,"15min":3.2989999999999999},"min":{"1min":0.223,"5min":0.189,"15min":0.189},"max":{"1min":189.327,"5min":223.09100000000001,"15min":223.09100000000001},"last":4.1319999999999997},{"interface":"front","average":{"1min":12.852,"5min":8.3219999999999992,"15min":3.5299999999999998},"min":{"1min":0.29399999999999998,"5min":0.24399999999999999,"15min":0.24099999999999999},"max":{"1min":192.80500000000001,"5min":223.161,"15min":223.161},"last":0.40999999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:07:20 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":1.627,"15min":1.615},"min":{"1min":0.29399999999999998,"5min":0.245,"15min":0.23000000000000001},"max":{"1min":2.8279999999999998,"5min":16.152999999999999,"15min":52.405999999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.71599999999999997,"5min":1.5920000000000001,"15min":1.863},"min":{"1min":0.254,"5min":0.24399999999999999,"15min":0.219},"max":{"1min":3.3050000000000002,"5min":22.844000000000001,"15min":52.645000000000003},"last":2.4020000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:07:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":2.661,"15min":3.3119999999999998},"min":{"1min":0.28699999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":3.9630000000000001,"5min":70.662000000000006,"15min":134.185},"last":1.994},{"interface":"front","average":{"1min":0.98399999999999999,"5min":2.5939999999999999,"15min":3.3610000000000002},"min":{"1min":0.316,"5min":0.25,"15min":0.25},"max":{"1min":3.589,"5min":70.575000000000003,"15min":134.16800000000001},"last":4.8239999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:07:40 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":3.3410000000000002,"15min":2.395},"min":{"1min":0.33400000000000002,"5min":0.307,"15min":0.307},"max":{"1min":3.4180000000000001,"5min":118.82299999999999,"15min":118.82299999999999},"last":2.4870000000000001},{"interface":"front","average":{"1min":0.84599999999999997,"5min":3.202,"15min":1.998},"min":{"1min":0.41499999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":2.8660000000000001,"5min":118.89400000000001,"15min":118.89400000000001},"last":1.6259999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:07:20 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":2.21,"15min":2.2370000000000001},"min":{"1min":0.35599999999999998,"5min":0.35199999999999998,"15min":0.314},"max":{"1min":5.2640000000000002,"5min":70.638000000000005,"15min":100.25},"last":2.4500000000000002},{"interface":"front","average":{"1min":0.69499999999999995,"5min":1.8200000000000001,"15min":1.915},"min":{"1min":0.44,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.04,"5min":70.518000000000001,"15min":102.40300000000001},"last":1.601}]},{"osd":6,"last update":"Mon Oct 9 06:07:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":2.3980000000000001,"15min":1.7769999999999999},"min":{"1min":0.38500000000000001,"5min":0.34399999999999997,"15min":0.33800000000000002},"max":{"1min":12.967000000000001,"5min":70.745999999999995,"15min":70.745999999999995},"last":2.5219999999999998},{"interface":"front","average":{"1min":1.1799999999999999,"5min":1.9730000000000001,"15min":1.55},"min":{"1min":0.46300000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":5.1900000000000004,"5min":70.597999999999999,"15min":70.597999999999999},"last":1.5720000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:07:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":2.0950000000000002,"15min":2.117},"min":{"1min":0.38600000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":2.129,"5min":70.840000000000003,"15min":103.866},"last":1.6499999999999999},{"interface":"front","average":{"1min":0.88700000000000001,"5min":1.4450000000000001,"15min":1.879},"min":{"1min":0.42699999999999999,"5min":0.41599999999999998,"15min":0.378},"max":{"1min":2.8559999999999999,"5min":7.6609999999999996,"15min":103.848},"last":0.46800000000000003}]}]},{"osd":2,"up_from":94,"seq":403726926065,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1424732,"kb_used_data":762900,"kb_used_omap":81,"kb_used_meta":661742,"kb_avail":92320420,"statfs":{"total":95995035648,"available":94536110080,"internally_reserved":0,"allocated":781209600,"data_stored":778997797,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":677624026},"hb_peers":[1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:30 2023","interfaces":[{"interface":"back","average":{"1min":1.6779999999999999,"5min":4.4050000000000002,"15min":3.4809999999999999},"min":{"1min":0.23899999999999999,"5min":0.20699999999999999,"15min":0.17999999999999999},"max":{"1min":12.58,"5min":167.44499999999999,"15min":167.44499999999999},"last":1.294},{"interface":"front","average":{"1min":1.3089999999999999,"5min":4.3639999999999999,"15min":3.452},"min":{"1min":0.29799999999999999,"5min":0.218,"15min":0.215},"max":{"1min":10.497999999999999,"5min":167.566,"15min":167.566},"last":8.0380000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:08:20 2023","interfaces":[{"interface":"back","average":{"1min":5.2039999999999997,"5min":6.7080000000000002,"15min":3.133},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":66.665999999999997,"5min":222.59399999999999,"15min":222.59399999999999},"last":4.4909999999999997},{"interface":"front","average":{"1min":6.5380000000000003,"5min":6.9980000000000002,"15min":3.2149999999999999},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":66.638999999999996,"5min":224.619,"15min":224.619},"last":3.145}]},{"osd":3,"last update":"Mon Oct 9 06:07:49 2023","interfaces":[{"interface":"back","average":{"1min":1.474,"5min":3.524,"15min":3.3290000000000002},"min":{"1min":0.31,"5min":0.246,"15min":0.24099999999999999},"max":{"1min":5.5419999999999998,"5min":58.271999999999998,"15min":144.88200000000001},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.3799999999999999,"5min":3.294,"15min":3.2989999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.25900000000000001},"max":{"1min":5.6790000000000003,"5min":58.201000000000001,"15min":140.79499999999999},"last":2.8559999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:08:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1930000000000001,"5min":1.8799999999999999,"15min":1.744},"min":{"1min":0.28899999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.2450000000000001,"5min":10.718,"15min":10.718},"last":1.806},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.764,"15min":1.8240000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.31},"max":{"1min":5.4299999999999997,"5min":12.643000000000001,"15min":12.643000000000001},"last":2.8860000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:08:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.728,"15min":1.5},"min":{"1min":0.45000000000000001,"5min":0.32600000000000001,"15min":0.30599999999999999},"max":{"1min":3.8719999999999999,"5min":28.48,"15min":28.48},"last":4.3840000000000003},{"interface":"front","average":{"1min":0.89300000000000002,"5min":2.0030000000000001,"15min":1.571},"min":{"1min":0.36599999999999999,"5min":0.34300000000000003,"15min":0.28899999999999998},"max":{"1min":4.4909999999999997,"5min":30.318999999999999,"15min":30.318999999999999},"last":3.351}]},{"osd":6,"last update":"Mon Oct 9 06:08:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":1.9370000000000001,"15min":1.6659999999999999},"min":{"1min":0.32000000000000001,"5min":0.314,"15min":0.314},"max":{"1min":7.3019999999999996,"5min":15.164999999999999,"15min":30.420000000000002},"last":2.3879999999999999},{"interface":"front","average":{"1min":0.91600000000000004,"5min":1.7829999999999999,"15min":1.6060000000000001},"min":{"1min":0.47599999999999998,"5min":0.36299999999999999,"15min":0.32200000000000001},"max":{"1min":3.8479999999999999,"5min":8.3900000000000006,"15min":30.713000000000001},"last":4.4269999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:08:20 2023","interfaces":[{"interface":"back","average":{"1min":1.407,"5min":1.6679999999999999,"15min":1.8859999999999999},"min":{"1min":0.434,"5min":0.35999999999999999,"15min":0.33200000000000002},"max":{"1min":4.758,"5min":6.5140000000000002,"15min":55.052},"last":4.4560000000000004},{"interface":"front","average":{"1min":0.82099999999999995,"5min":1.8999999999999999,"15min":1.738},"min":{"1min":0.438,"5min":0.38200000000000001,"15min":0.35599999999999998},"max":{"1min":2.855,"5min":12.682,"15min":33.098999999999997},"last":2.8170000000000002}]}]},{"osd":3,"up_from":99,"seq":425201762538,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4256784,"kb_used_data":3516104,"kb_used_omap":7,"kb_used_meta":740664,"kb_avail":89488368,"statfs":{"total":95995035648,"available":91636088832,"internally_reserved":0,"allocated":3600490496,"data_stored":3597990628,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":758440552},"hb_peers":[1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:33 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":5.343,"15min":3.331},"min":{"1min":0.223,"5min":0.20300000000000001,"15min":0.186},"max":{"1min":3.7240000000000002,"5min":131.363,"15min":131.363},"last":5.5119999999999996},{"interface":"front","average":{"1min":0.94899999999999995,"5min":4.8479999999999999,"15min":3.3530000000000002},"min":{"1min":0.253,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":2.6709999999999998,"5min":131.26300000000001,"15min":131.26300000000001},"last":1.9039999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:07:55 2023","interfaces":[{"interface":"back","average":{"1min":2.0979999999999999,"5min":4.4809999999999999,"15min":2.4670000000000001},"min":{"1min":0.255,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":11.936999999999999,"5min":207.75299999999999,"15min":207.75299999999999},"last":0.247},{"interface":"front","average":{"1min":1.601,"5min":4.6210000000000004,"15min":2.8740000000000001},"min":{"1min":0.31900000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":6.9980000000000002,"5min":207.12200000000001,"15min":207.12200000000001},"last":10.529}]},{"osd":2,"last update":"Mon Oct 9 06:07:55 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":2.0619999999999998,"15min":2.2269999999999999},"min":{"1min":0.26100000000000001,"5min":0.223,"15min":0.223},"max":{"1min":2.855,"5min":41.945999999999998,"15min":107.01300000000001},"last":6.3780000000000001},{"interface":"front","average":{"1min":2.0800000000000001,"5min":2.5470000000000002,"15min":2.4940000000000002},"min":{"1min":0.31,"5min":0.28100000000000003,"15min":0.249},"max":{"1min":12.791,"5min":41.475999999999999,"15min":109.807},"last":10.481999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:08:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3520000000000001,"5min":1.909,"15min":4.9660000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.254},"max":{"1min":6.0739999999999998,"5min":20.469000000000001,"15min":99.876000000000005},"last":0.32800000000000001},{"interface":"front","average":{"1min":1.5960000000000001,"5min":2.2570000000000001,"15min":4.806},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":9.9589999999999996,"5min":20.454999999999998,"15min":99.914000000000001},"last":9.9589999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:07:55 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.5629999999999999,"15min":1.506},"min":{"1min":0.30599999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.5149999999999997,"5min":27.937000000000001,"15min":27.937000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.0269999999999999,"5min":4.2919999999999998,"15min":2.4380000000000002},"min":{"1min":0.371,"5min":0.35599999999999998,"15min":0.28199999999999997},"max":{"1min":3.7000000000000002,"5min":235.56,"15min":235.56},"last":0.36099999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:07:55 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":5.2350000000000003,"15min":4.3319999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33100000000000002},"max":{"1min":4.4710000000000001,"5min":233.26599999999999,"15min":233.26599999999999},"last":2.165},{"interface":"front","average":{"1min":1.0249999999999999,"5min":5.5540000000000003,"15min":4.4580000000000002},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.318},"max":{"1min":2.9769999999999999,"5min":233.24100000000001,"15min":233.24100000000001},"last":2.1200000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:07:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":4.1310000000000002,"15min":3.9660000000000002},"min":{"1min":0.42299999999999999,"5min":0.39000000000000001,"15min":0.34200000000000003},"max":{"1min":3.5339999999999998,"5min":230.69399999999999,"15min":230.69399999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.149,"5min":4.1890000000000001,"15min":3.9369999999999998},"min":{"1min":0.45500000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":5.5739999999999998,"5min":230.67500000000001,"15min":230.67500000000001},"last":10.006}]}]},{"osd":4,"up_from":322,"seq":1382979469401,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1494368,"kb_used_data":1171992,"kb_used_omap":35,"kb_used_meta":322332,"kb_avail":92250784,"statfs":{"total":95995035648,"available":94464802816,"internally_reserved":0,"allocated":1200119808,"data_stored":1198844523,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36552,"internal_metadata":330068280},"hb_peers":[1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:32 2023","interfaces":[{"interface":"back","average":{"1min":3.855,"5min":2.6339999999999999,"15min":1.722},"min":{"1min":0.23000000000000001,"5min":0.223,"15min":0.223},"max":{"1min":50.341999999999999,"5min":50.341999999999999,"15min":50.341999999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":3.7850000000000001,"5min":2.6600000000000001,"15min":2.1139999999999999},"min":{"1min":0.26700000000000002,"5min":0.247,"15min":0.247},"max":{"1min":50.454999999999998,"5min":50.454999999999998,"15min":50.454999999999998},"last":0.372}]},{"osd":1,"last update":"Mon Oct 9 06:08:35 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":3.278,"15min":2.2799999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.5700000000000003,"5min":78.891000000000005,"15min":78.891000000000005},"last":0.28399999999999997},{"interface":"front","average":{"1min":1.444,"5min":3.4700000000000002,"15min":2.1299999999999999},"min":{"1min":0.28299999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.6950000000000003,"5min":78.921000000000006,"15min":78.921000000000006},"last":1.4430000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:08:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4219999999999999,"5min":1.7430000000000001,"15min":1.7669999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.4720000000000004,"5min":7.9189999999999996,"15min":7.9189999999999996},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.94399999999999995,"5min":1.4670000000000001,"15min":1.8919999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.28199999999999997},"max":{"1min":3.052,"5min":8.1069999999999993,"15min":11.381},"last":1.3560000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:08:35 2023","interfaces":[{"interface":"back","average":{"1min":1.726,"5min":3.0670000000000002,"15min":2.359},"min":{"1min":0.36799999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":5.7389999999999999,"5min":58.398000000000003,"15min":58.398000000000003},"last":5.7389999999999999},{"interface":"front","average":{"1min":1.6899999999999999,"5min":2.4710000000000001,"15min":1.96},"min":{"1min":0.38100000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":6.5190000000000001,"5min":58.305999999999997,"15min":58.305999999999997},"last":2.895}]},{"osd":5,"last update":"Mon Oct 9 06:08:35 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.1759999999999999,"15min":0.98199999999999998},"min":{"1min":0.27600000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":9.2690000000000001,"5min":9.2690000000000001,"15min":9.2690000000000001},"last":0.311},{"interface":"front","average":{"1min":0.57999999999999996,"5min":1.2529999999999999,"15min":1.351},"min":{"1min":0.29599999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.157,"5min":12.087,"15min":12.087},"last":0.437}]},{"osd":6,"last update":"Mon Oct 9 06:08:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":2.3260000000000001,"15min":3.3839999999999999},"min":{"1min":0.34200000000000003,"5min":0.31,"15min":0.30499999999999999},"max":{"1min":5.8010000000000002,"5min":95.492999999999995,"15min":95.492999999999995},"last":5.8010000000000002},{"interface":"front","average":{"1min":0.91400000000000003,"5min":2.3860000000000001,"15min":3.5499999999999998},"min":{"1min":0.35599999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.6280000000000001,"5min":95.664000000000001,"15min":95.664000000000001},"last":0.51000000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:08:35 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":1.431,"15min":4.4299999999999997},"min":{"1min":0.39300000000000002,"5min":0.34000000000000002,"15min":0.32500000000000001},"max":{"1min":4.1210000000000004,"5min":8.5540000000000003,"15min":79.173000000000002},"last":2.9540000000000002},{"interface":"front","average":{"1min":1.2090000000000001,"5min":1.4019999999999999,"15min":3.6589999999999998},"min":{"1min":0.36899999999999999,"5min":0.308,"15min":0.28499999999999998},"max":{"1min":9.1229999999999993,"5min":9.1229999999999993,"15min":78.894000000000005},"last":0.73799999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757621,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3624656,"kb_used_data":2734344,"kb_used_omap":0,"kb_used_meta":890304,"kb_avail":90120496,"statfs":{"total":95995035648,"available":92283387904,"internally_reserved":0,"allocated":2799968256,"data_stored":2798705573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911671296},"hb_peers":[1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:07:37 2023","interfaces":[{"interface":"back","average":{"1min":1.1639999999999999,"5min":2.1619999999999999,"15min":1.704},"min":{"1min":0.26700000000000002,"5min":0.23499999999999999,"15min":0.22500000000000001},"max":{"1min":3.5139999999999998,"5min":69.088999999999999,"15min":69.088999999999999},"last":1.6899999999999999},{"interface":"front","average":{"1min":0.93700000000000006,"5min":2.2330000000000001,"15min":1.9259999999999999},"min":{"1min":0.25900000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.637,"5min":69.073999999999998,"15min":69.073999999999998},"last":1.704}]},{"osd":1,"last update":"Mon Oct 9 06:07:55 2023","interfaces":[{"interface":"back","average":{"1min":14.262,"5min":5.6539999999999999,"15min":3.488},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":132.93299999999999,"5min":132.93299999999999,"15min":132.93299999999999},"last":0.22600000000000001},{"interface":"front","average":{"1min":14.57,"5min":5.9569999999999999,"15min":3.3490000000000002},"min":{"1min":0.315,"5min":0.27600000000000002,"15min":0.26400000000000001},"max":{"1min":133.92099999999999,"5min":133.92099999999999,"15min":133.92099999999999},"last":4.5670000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:08:22 2023","interfaces":[{"interface":"back","average":{"1min":1.677,"5min":1.7969999999999999,"15min":2.9489999999999998},"min":{"1min":0.29299999999999998,"5min":0.28100000000000003,"15min":0.254},"max":{"1min":5.0499999999999998,"5min":11.271000000000001,"15min":217.06399999999999},"last":0.27000000000000002},{"interface":"front","average":{"1min":1.556,"5min":2.0089999999999999,"15min":2.8660000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.27600000000000002},"max":{"1min":3.8940000000000001,"5min":11.318,"15min":217.018},"last":2.0569999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:07:55 2023","interfaces":[{"interface":"back","average":{"1min":12.901,"5min":5.7220000000000004,"15min":6.4610000000000003},"min":{"1min":0.39200000000000002,"5min":0.312,"15min":0.27500000000000002},"max":{"1min":111.596,"5min":111.596,"15min":211.40199999999999},"last":1.0629999999999999},{"interface":"front","average":{"1min":12.468999999999999,"5min":5.5090000000000003,"15min":6.4850000000000003},"min":{"1min":0.31900000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":111.646,"5min":111.646,"15min":211.578},"last":0.29599999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:08:38 2023","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":1.2370000000000001,"15min":0.98899999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":3.9849999999999999,"5min":8.1539999999999999,"15min":8.1539999999999999},"last":3.1899999999999999},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.0449999999999999,"15min":1.083},"min":{"1min":0.29899999999999999,"5min":0.28299999999999997,"15min":0.26000000000000001},"max":{"1min":4.1920000000000002,"5min":8.7059999999999995,"15min":8.7059999999999995},"last":1.097}]},{"osd":6,"last update":"Mon Oct 9 06:07:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.2849999999999999,"15min":1.419},"min":{"1min":0.35499999999999998,"5min":0.29099999999999998,"15min":0.251},"max":{"1min":5.2169999999999996,"5min":8.6809999999999992,"15min":126.89400000000001},"last":1.1739999999999999},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.3500000000000001,"15min":1.4430000000000001},"min":{"1min":0.32300000000000001,"5min":0.28000000000000003,"15min":0.26200000000000001},"max":{"1min":4.0970000000000004,"5min":9.9009999999999998,"15min":126.616},"last":0.50600000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:08:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0649999999999999,"5min":1.478,"15min":2.0089999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.29999999999999999},"max":{"1min":4.3680000000000003,"5min":9.9380000000000006,"15min":133.27099999999999},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.874,"5min":1.3420000000000001,"15min":1.861},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.016,"5min":9.9529999999999994,"15min":135.09200000000001},"last":2.016}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1474,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4889,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1179123712,"data_stored":1179123712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3638427648,"data_stored":3638427648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":787021824,"data_stored":787021824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3607298048,"data_stored":3607298048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1202192384,"data_stored":1202192384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2802515968,"data_stored":2802515968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1425342464,"data_stored":1425342464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4259512320,"data_stored":4259512320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:08:45.881 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_3 pgp_num 16 2023-10-09T06:08:45.882 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:45.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:45.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:45.987 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:46.017 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:46.078 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:46.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:46.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:46.185 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:46.215 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:46.274 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:46.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:46.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:46.378 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:46.380 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:46.415 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:46.509 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:46.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:46.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:46.612 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:08:46.676 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:08:46.677 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:08:46.677 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T06:08:46.677 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T06:08:46.677 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:08:46.677 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.678 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45263.0:188660 7.d 7:bca5ce24:::benchmark_data_smithi064_398483_object83307:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2023-10-09T06:08:46.678 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:01:53.772813+0000", 2023-10-09T06:08:46.678 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 412.88128193799997, 2023-10-09T06:08:46.678 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1706843, 2023-10-09T06:08:46.678 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:46.678 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:08:46.678 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:46.678 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45263", 2023-10-09T06:08:46.678 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:08:46.678 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188660 2023-10-09T06:08:46.678 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.679 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:46.679 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.679 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:46.679 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:53.772813+0000", 2023-10-09T06:08:46.679 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.679 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.679 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.679 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:46.679 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:53.772813+0000", 2023-10-09T06:08:46.679 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.679 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.680 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.680 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:46.680 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:53.772813+0000", 2023-10-09T06:08:46.680 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:08:46.680 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.680 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.680 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:46.680 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:53.772816+0000", 2023-10-09T06:08:46.680 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7620000000000001e-06 2023-10-09T06:08:46.680 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.680 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.680 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:46.680 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:53.772827+0000", 2023-10-09T06:08:46.681 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1021e-05 2023-10-09T06:08:46.681 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.681 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.681 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:46.681 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:53.772832+0000", 2023-10-09T06:08:46.681 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6670000000000004e-06 2023-10-09T06:08:46.681 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.681 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.681 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.681 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:53.772887+0000", 2023-10-09T06:08:46.681 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4669999999999997e-05 2023-10-09T06:08:46.682 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.682 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.682 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:08:46.682 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:53.772899+0000", 2023-10-09T06:08:46.682 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1822e-05 2023-10-09T06:08:46.682 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.682 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.682 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.682 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:54.737748+0000", 2023-10-09T06:08:46.682 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96484949200000003 2023-10-09T06:08:46.682 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.682 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.683 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:08:46.683 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:54.737761+0000", 2023-10-09T06:08:46.683 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2468e-05 2023-10-09T06:08:46.683 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.683 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.683 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.683 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.918226+0000", 2023-10-09T06:08:46.683 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.18046494 2023-10-09T06:08:46.683 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.683 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.683 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:46.683 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.918331+0000", 2023-10-09T06:08:46.684 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000105758 2023-10-09T06:08:46.684 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.684 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.684 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1,3", 2023-10-09T06:08:46.684 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.918412+0000", 2023-10-09T06:08:46.684 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0919000000000002e-05 2023-10-09T06:08:46.684 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.684 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.684 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:08:46.684 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.926663+0000", 2023-10-09T06:08:46.684 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0082510210000000007 2023-10-09T06:08:46.684 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.684 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.685 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.685 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.931510+0000", 2023-10-09T06:08:46.685 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0048469819999999997 2023-10-09T06:08:46.685 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.685 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.685 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.685 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.939599+0000", 2023-10-09T06:08:46.685 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0080890849999999993 2023-10-09T06:08:46.685 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.685 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.685 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.685 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.943343+0000", 2023-10-09T06:08:46.686 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0037436539999999999 2023-10-09T06:08:46.686 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.686 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.686 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:08:46.686 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.943388+0000", 2023-10-09T06:08:46.686 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5268000000000001e-05 2023-10-09T06:08:46.686 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.686 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.686 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:08:46.686 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.943498+0000", 2023-10-09T06:08:46.686 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000109561 2023-10-09T06:08:46.687 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.687 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:46.687 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.687 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.687 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.687 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45263.0:188665 7.d 7:bc9e132d:::benchmark_data_smithi064_398483_object83312:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2023-10-09T06:08:46.687 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:01:53.777873+0000", 2023-10-09T06:08:46.687 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 412.87622206499998, 2023-10-09T06:08:46.687 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1689668989999999, 2023-10-09T06:08:46.687 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:46.687 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:08:46.687 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:46.688 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45263", 2023-10-09T06:08:46.688 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:08:46.688 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188665 2023-10-09T06:08:46.688 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.688 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:46.688 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.688 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:46.894 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:53.777873+0000", 2023-10-09T06:08:46.894 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.894 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.894 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.895 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:46.895 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:53.777873+0000", 2023-10-09T06:08:46.895 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.895 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.895 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.895 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:46.895 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:53.777872+0000", 2023-10-09T06:08:46.895 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:08:46.895 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.895 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.896 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:46.896 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:53.777878+0000", 2023-10-09T06:08:46.896 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3740000000000003e-06 2023-10-09T06:08:46.896 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.896 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.896 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:46.896 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:53.777891+0000", 2023-10-09T06:08:46.896 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2927e-05 2023-10-09T06:08:46.896 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.896 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.896 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:46.896 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:53.777897+0000", 2023-10-09T06:08:46.897 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0279999999999997e-06 2023-10-09T06:08:46.897 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.897 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.897 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.898 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:53.777955+0000", 2023-10-09T06:08:46.899 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7306999999999999e-05 2023-10-09T06:08:46.899 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.899 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.899 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:08:46.899 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:53.777964+0000", 2023-10-09T06:08:46.899 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.4919999999999999e-06 2023-10-09T06:08:46.899 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.899 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.899 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.899 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:54.737784+0000", 2023-10-09T06:08:46.900 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.95981976400000002 2023-10-09T06:08:46.900 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.900 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.900 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:08:46.900 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:54.737800+0000", 2023-10-09T06:08:46.900 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5968999999999999e-05 2023-10-09T06:08:46.900 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.900 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.900 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.900 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.921069+0000", 2023-10-09T06:08:46.900 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.183269643 2023-10-09T06:08:46.900 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.901 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.901 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:46.901 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.921162+0000", 2023-10-09T06:08:46.901 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2125000000000003e-05 2023-10-09T06:08:46.901 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.901 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.901 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1,3", 2023-10-09T06:08:46.901 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.921240+0000", 2023-10-09T06:08:46.901 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8517999999999996e-05 2023-10-09T06:08:46.901 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.902 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.902 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:08:46.902 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.926669+0000", 2023-10-09T06:08:46.902 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0054289400000000002 2023-10-09T06:08:46.902 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.902 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.902 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.902 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.939633+0000", 2023-10-09T06:08:46.902 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012964204999999999 2023-10-09T06:08:46.902 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.902 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.903 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.903 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.943538+0000", 2023-10-09T06:08:46.903 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0039044639999999999 2023-10-09T06:08:46.903 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.903 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.903 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.903 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.946727+0000", 2023-10-09T06:08:46.903 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.003189374 2023-10-09T06:08:46.903 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.903 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.903 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:08:46.903 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.946763+0000", 2023-10-09T06:08:46.904 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6043e-05 2023-10-09T06:08:46.904 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.904 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.904 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:08:46.904 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.946840+0000", 2023-10-09T06:08:46.904 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7113999999999994e-05 2023-10-09T06:08:46.904 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.904 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:46.904 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.904 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.904 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.904 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45263.0:188667 7.d 7:b7f17857:::benchmark_data_smithi064_398483_object83314:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2023-10-09T06:08:46.905 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:01:53.779797+0000", 2023-10-09T06:08:46.905 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 412.87429807299998, 2023-10-09T06:08:46.905 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1672009000000001, 2023-10-09T06:08:46.905 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:46.905 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:08:46.905 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:46.905 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45263", 2023-10-09T06:08:46.905 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:08:46.905 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188667 2023-10-09T06:08:46.905 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.905 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:46.906 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.906 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:46.906 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:53.779797+0000", 2023-10-09T06:08:46.906 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.906 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.906 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.906 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:46.906 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:53.779797+0000", 2023-10-09T06:08:46.906 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.906 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.906 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.906 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:46.907 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:53.779796+0000", 2023-10-09T06:08:46.907 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:08:46.907 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.907 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.907 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:46.907 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:53.779805+0000", 2023-10-09T06:08:46.907 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.5159999999999994e-06 2023-10-09T06:08:46.907 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.907 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.907 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:46.907 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:53.779821+0000", 2023-10-09T06:08:46.907 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6049000000000002e-05 2023-10-09T06:08:46.908 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.908 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.908 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:46.908 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:53.779828+0000", 2023-10-09T06:08:46.908 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9879999999999998e-06 2023-10-09T06:08:46.908 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.908 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.908 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.908 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:53.779885+0000", 2023-10-09T06:08:46.908 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6606e-05 2023-10-09T06:08:46.908 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.908 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.909 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:08:46.909 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:53.779895+0000", 2023-10-09T06:08:46.909 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0721e-05 2023-10-09T06:08:46.909 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.909 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.909 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.909 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:54.737822+0000", 2023-10-09T06:08:46.909 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.95792699699999995 2023-10-09T06:08:46.909 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.909 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.909 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:08:46.909 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:54.737830+0000", 2023-10-09T06:08:46.910 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1860000000000003e-06 2023-10-09T06:08:46.910 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.910 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.910 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.910 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.923528+0000", 2023-10-09T06:08:46.910 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.185698149 2023-10-09T06:08:46.910 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.910 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.910 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:46.910 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.923667+0000", 2023-10-09T06:08:46.910 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013966900000000001 2023-10-09T06:08:46.910 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.911 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.911 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1,3", 2023-10-09T06:08:46.911 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.923807+0000", 2023-10-09T06:08:46.911 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000139462 2023-10-09T06:08:46.911 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.911 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.911 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:08:46.911 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.926690+0000", 2023-10-09T06:08:46.911 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002883217 2023-10-09T06:08:46.911 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.911 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.911 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.912 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.939658+0000", 2023-10-09T06:08:46.912 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012967944 2023-10-09T06:08:46.912 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.912 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.912 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.912 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.946869+0000", 2023-10-09T06:08:46.912 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0072111349999999996 2023-10-09T06:08:46.912 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.912 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.912 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.912 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.946896+0000", 2023-10-09T06:08:46.912 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.6641000000000001e-05 2023-10-09T06:08:46.913 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.913 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.913 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:08:46.913 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.946926+0000", 2023-10-09T06:08:46.913 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9819e-05 2023-10-09T06:08:46.913 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.913 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.913 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:08:46.913 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:01:55.946998+0000", 2023-10-09T06:08:46.913 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2681000000000004e-05 2023-10-09T06:08:46.913 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.913 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:46.914 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.914 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.914 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.914 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:188353 8.9 8:9c34743c:::benchmark_data_smithi064_457003_object83835:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2023-10-09T06:08:46.914 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:27.431298+0000", 2023-10-09T06:08:46.914 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 139.22279749800001, 2023-10-09T06:08:46.914 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0494895609999999, 2023-10-09T06:08:46.914 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:46.914 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:08:46.914 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:46.914 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:08:46.915 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:08:46.915 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188353 2023-10-09T06:08:46.915 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.915 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:46.915 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.915 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:46.915 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.431298+0000", 2023-10-09T06:08:46.915 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.915 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.915 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.915 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:46.915 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.431298+0000", 2023-10-09T06:08:46.916 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.916 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.916 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.916 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:46.916 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.431296+0000", 2023-10-09T06:08:46.916 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:08:46.916 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.916 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.916 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:46.916 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.431305+0000", 2023-10-09T06:08:46.916 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.2279999999999998e-06 2023-10-09T06:08:46.916 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.917 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.917 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:46.917 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.431317+0000", 2023-10-09T06:08:46.917 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2605e-05 2023-10-09T06:08:46.917 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.917 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.917 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:46.917 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.431323+0000", 2023-10-09T06:08:46.917 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.178e-06 2023-10-09T06:08:46.917 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.917 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.918 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.918 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.431394+0000", 2023-10-09T06:08:46.918 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0308000000000005e-05 2023-10-09T06:08:46.918 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.918 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.918 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:46.918 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.431517+0000", 2023-10-09T06:08:46.918 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012297 2023-10-09T06:08:46.918 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.918 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.918 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T06:08:46.919 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.431637+0000", 2023-10-09T06:08:46.919 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000120615 2023-10-09T06:08:46.919 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.919 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.919 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:08:46.919 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.435725+0000", 2023-10-09T06:08:46.919 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0040881099999999998 2023-10-09T06:08:46.919 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.919 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.919 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.919 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.442248+0000", 2023-10-09T06:08:46.920 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0065223720000000002 2023-10-09T06:08:46.920 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.920 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.920 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.920 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.446429+0000", 2023-10-09T06:08:46.920 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0041808890000000001 2023-10-09T06:08:46.920 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.920 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.920 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.920 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.451740+0000", 2023-10-09T06:08:46.920 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0053108440000000003 2023-10-09T06:08:46.920 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.921 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.921 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:08:46.921 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.451749+0000", 2023-10-09T06:08:46.921 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2609999999999999e-06 2023-10-09T06:08:46.921 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.921 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.921 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.921 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.480704+0000", 2023-10-09T06:08:46.921 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0289548829999999 2023-10-09T06:08:46.921 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.921 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.921 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:08:46.922 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.480787+0000", 2023-10-09T06:08:46.922 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.3778999999999999e-05 2023-10-09T06:08:46.922 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.922 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:46.922 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.922 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.922 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.922 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:188357 8.9 8:9381b48d:::benchmark_data_smithi064_457003_object83839:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2023-10-09T06:08:46.922 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:27.437756+0000", 2023-10-09T06:08:46.922 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 139.21633952299999, 2023-10-09T06:08:46.922 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0435701239999999, 2023-10-09T06:08:46.923 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:46.923 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:08:46.923 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:46.923 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:08:46.923 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:08:46.923 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188357 2023-10-09T06:08:46.923 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.923 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:46.923 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.923 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:46.923 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.437756+0000", 2023-10-09T06:08:46.923 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.924 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.924 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.924 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:46.924 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.437756+0000", 2023-10-09T06:08:46.924 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.924 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.924 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.924 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:46.924 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.437754+0000", 2023-10-09T06:08:46.924 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:08:46.924 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.924 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.925 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:46.925 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.437761+0000", 2023-10-09T06:08:46.925 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0820000000000001e-06 2023-10-09T06:08:46.925 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.925 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.925 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:46.925 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.437774+0000", 2023-10-09T06:08:46.925 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2945000000000001e-05 2023-10-09T06:08:46.925 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.925 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.925 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:46.926 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.437781+0000", 2023-10-09T06:08:46.926 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4659999999999996e-06 2023-10-09T06:08:46.926 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.926 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.926 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.926 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.437824+0000", 2023-10-09T06:08:46.926 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3556000000000001e-05 2023-10-09T06:08:46.926 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.926 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.926 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:46.926 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.438012+0000", 2023-10-09T06:08:46.926 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00018795899999999999 2023-10-09T06:08:46.927 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.927 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.927 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T06:08:46.927 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.438176+0000", 2023-10-09T06:08:46.927 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00016331200000000001 2023-10-09T06:08:46.927 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.927 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.927 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:08:46.927 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.442461+0000", 2023-10-09T06:08:46.927 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0042849079999999996 2023-10-09T06:08:46.927 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.927 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.928 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.928 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.446454+0000", 2023-10-09T06:08:46.928 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0039938040000000001 2023-10-09T06:08:46.928 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.928 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.928 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.928 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.448508+0000", 2023-10-09T06:08:46.928 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0020538090000000002 2023-10-09T06:08:46.928 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.928 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.928 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.928 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.452359+0000", 2023-10-09T06:08:46.929 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0038505330000000002 2023-10-09T06:08:46.929 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.929 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.929 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:08:46.929 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.452366+0000", 2023-10-09T06:08:46.929 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.17e-06 2023-10-09T06:08:46.929 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.929 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.929 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.929 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.481280+0000", 2023-10-09T06:08:46.929 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.028914012 2023-10-09T06:08:46.930 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.930 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.930 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:08:46.930 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.481326+0000", 2023-10-09T06:08:46.930 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6041999999999997e-05 2023-10-09T06:08:46.930 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.930 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:46.930 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.930 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.930 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.930 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:188369 8.9 8:90ddf148:::benchmark_data_smithi064_457003_object83851:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2023-10-09T06:08:46.930 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:27.448204+0000", 2023-10-09T06:08:46.931 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 139.205891724, 2023-10-09T06:08:46.931 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0446369950000001, 2023-10-09T06:08:46.931 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:46.931 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:08:46.931 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:46.931 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:08:46.931 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:08:46.931 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188369 2023-10-09T06:08:46.931 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.931 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:46.931 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.931 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:46.932 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.448204+0000", 2023-10-09T06:08:46.932 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.932 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.932 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.932 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:46.932 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.448204+0000", 2023-10-09T06:08:46.932 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.932 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.932 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.932 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:46.932 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.448202+0000", 2023-10-09T06:08:46.933 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T06:08:46.933 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.933 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.933 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:46.933 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.448209+0000", 2023-10-09T06:08:46.933 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9380000000000003e-06 2023-10-09T06:08:46.933 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.933 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.933 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:46.933 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.448221+0000", 2023-10-09T06:08:46.933 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1702000000000001e-05 2023-10-09T06:08:46.933 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.933 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.934 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:46.934 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.448227+0000", 2023-10-09T06:08:46.934 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0209999999999996e-06 2023-10-09T06:08:46.934 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.934 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.934 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.934 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.448281+0000", 2023-10-09T06:08:46.934 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4097000000000001e-05 2023-10-09T06:08:46.934 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.934 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.934 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op must wait for map", 2023-10-09T06:08:46.935 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.448287+0000", 2023-10-09T06:08:46.935 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3960000000000003e-06 2023-10-09T06:08:46.935 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.935 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.935 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.935 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.452389+0000", 2023-10-09T06:08:46.935 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0041019660000000003 2023-10-09T06:08:46.935 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.935 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.935 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:08:46.935 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.452394+0000", 2023-10-09T06:08:46.936 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4439999999999999e-06 2023-10-09T06:08:46.936 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.936 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.936 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.936 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.481372+0000", 2023-10-09T06:08:46.936 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0289773689999999 2023-10-09T06:08:46.936 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.936 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.936 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:46.936 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.481499+0000", 2023-10-09T06:08:46.936 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012766700000000001 2023-10-09T06:08:46.936 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.936 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.937 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:08:46.937 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.481637+0000", 2023-10-09T06:08:46.937 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000137214 2023-10-09T06:08:46.937 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.937 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.937 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.937 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.489010+0000", 2023-10-09T06:08:46.937 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0073732950000000002 2023-10-09T06:08:46.937 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.937 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.937 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.937 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.489162+0000", 2023-10-09T06:08:46.938 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015227599999999999 2023-10-09T06:08:46.938 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.938 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.938 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:08:46.938 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.492763+0000", 2023-10-09T06:08:46.938 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0036004140000000001 2023-10-09T06:08:46.938 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.938 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.938 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:08:46.938 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.492798+0000", 2023-10-09T06:08:46.938 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5027999999999997e-05 2023-10-09T06:08:46.939 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.939 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.939 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:08:46.939 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.492841+0000", 2023-10-09T06:08:46.939 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2972000000000001e-05 2023-10-09T06:08:46.939 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.939 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:46.939 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.939 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.939 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.939 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:188370 8.9 8:9fe10030:::benchmark_data_smithi064_457003_object83852:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2023-10-09T06:08:46.939 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:27.449292+0000", 2023-10-09T06:08:46.940 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 139.204803193, 2023-10-09T06:08:46.940 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0437020880000001, 2023-10-09T06:08:46.940 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:46.940 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:08:46.940 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:46.940 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:08:46.940 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:08:46.940 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188370 2023-10-09T06:08:46.940 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.940 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:46.940 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.940 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:46.941 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.449292+0000", 2023-10-09T06:08:46.941 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.941 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.941 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.941 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:46.941 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.449292+0000", 2023-10-09T06:08:46.941 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.941 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.941 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.941 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:46.941 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.449291+0000", 2023-10-09T06:08:46.941 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:08:46.942 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.942 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.942 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:46.942 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.449295+0000", 2023-10-09T06:08:46.942 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9099999999999998e-06 2023-10-09T06:08:46.942 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.942 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.942 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:46.942 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.449306+0000", 2023-10-09T06:08:46.942 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0444e-05 2023-10-09T06:08:46.942 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.943 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.943 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:46.943 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.449310+0000", 2023-10-09T06:08:46.943 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3200000000000001e-06 2023-10-09T06:08:46.943 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.943 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.943 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.943 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.453072+0000", 2023-10-09T06:08:46.943 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0037617789999999998 2023-10-09T06:08:46.943 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.943 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.943 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:08:46.944 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.453079+0000", 2023-10-09T06:08:46.944 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5079999999999999e-06 2023-10-09T06:08:46.944 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.944 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.944 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.944 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.485497+0000", 2023-10-09T06:08:46.944 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.032417825 2023-10-09T06:08:46.944 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.944 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.944 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:46.944 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.485616+0000", 2023-10-09T06:08:46.944 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011836999999999999 2023-10-09T06:08:46.945 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.945 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.945 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:08:46.945 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.485734+0000", 2023-10-09T06:08:46.945 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000118118 2023-10-09T06:08:46.945 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.945 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.945 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.945 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.492756+0000", 2023-10-09T06:08:46.945 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0070217819999999999 2023-10-09T06:08:46.945 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.945 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.946 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:08:46.946 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.492849+0000", 2023-10-09T06:08:46.946 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3039000000000001e-05 2023-10-09T06:08:46.946 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.946 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.946 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.946 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.492901+0000", 2023-10-09T06:08:46.946 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1978999999999998e-05 2023-10-09T06:08:46.946 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.946 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.946 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:08:46.946 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.492953+0000", 2023-10-09T06:08:46.947 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1996000000000002e-05 2023-10-09T06:08:46.947 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.947 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.947 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:08:46.947 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.492994+0000", 2023-10-09T06:08:46.947 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1740999999999999e-05 2023-10-09T06:08:46.947 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.947 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:46.947 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.947 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.947 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.947 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:188393 8.9 8:94976dfa:::benchmark_data_smithi064_457003_object83875:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2023-10-09T06:08:46.948 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:27.473154+0000", 2023-10-09T06:08:46.948 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 139.180941163, 2023-10-09T06:08:46.948 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0258810519999999, 2023-10-09T06:08:46.948 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:46.948 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:08:46.948 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:46.948 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:08:46.948 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:08:46.948 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188393 2023-10-09T06:08:46.948 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.948 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:46.948 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.949 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:46.949 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.473154+0000", 2023-10-09T06:08:46.949 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.949 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.949 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.949 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:46.949 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.473154+0000", 2023-10-09T06:08:46.949 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.949 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.949 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.949 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:46.949 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.473152+0000", 2023-10-09T06:08:46.949 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T06:08:46.950 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.950 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.950 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:46.950 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.473161+0000", 2023-10-09T06:08:46.950 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.2179999999999999e-06 2023-10-09T06:08:46.950 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.950 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.950 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:46.950 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.473179+0000", 2023-10-09T06:08:46.950 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7932999999999999e-05 2023-10-09T06:08:46.950 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.951 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.951 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:46.951 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.473187+0000", 2023-10-09T06:08:46.951 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0579999999999998e-06 2023-10-09T06:08:46.951 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.951 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.951 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.951 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.473235+0000", 2023-10-09T06:08:46.951 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8350000000000003e-05 2023-10-09T06:08:46.951 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.951 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.951 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:08:46.952 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.473244+0000", 2023-10-09T06:08:46.952 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0510000000000002e-06 2023-10-09T06:08:46.952 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.952 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.952 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.952 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.489188+0000", 2023-10-09T06:08:46.952 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.015943974 2023-10-09T06:08:46.952 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.952 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.952 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:46.952 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.489307+0000", 2023-10-09T06:08:46.952 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000119138 2023-10-09T06:08:46.953 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.953 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.953 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:08:46.953 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.489434+0000", 2023-10-09T06:08:46.953 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000126995 2023-10-09T06:08:46.953 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.953 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.953 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:08:46.953 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.496145+0000", 2023-10-09T06:08:46.953 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0067113260000000001 2023-10-09T06:08:46.953 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.953 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.954 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.954 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.498903+0000", 2023-10-09T06:08:46.954 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0027579330000000002 2023-10-09T06:08:46.954 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.954 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.954 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.954 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.498950+0000", 2023-10-09T06:08:46.954 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6959999999999998e-05 2023-10-09T06:08:46.954 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.954 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.954 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:08:46.955 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.498997+0000", 2023-10-09T06:08:46.955 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6675000000000003e-05 2023-10-09T06:08:46.955 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.955 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.955 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:08:46.955 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.499035+0000", 2023-10-09T06:08:46.955 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8235000000000002e-05 2023-10-09T06:08:46.955 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.955 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:46.955 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.955 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.956 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.956 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:188403 8.9 8:9096b6fa:::benchmark_data_smithi064_457003_object83885:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2023-10-09T06:08:46.956 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:27.481397+0000", 2023-10-09T06:08:46.956 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 139.172698556, 2023-10-09T06:08:46.956 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.035516009, 2023-10-09T06:08:46.956 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:46.956 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:08:46.956 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:46.956 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:08:46.956 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:08:46.956 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188403 2023-10-09T06:08:46.956 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.957 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:46.957 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.957 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:46.957 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.481397+0000", 2023-10-09T06:08:46.957 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.957 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.957 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.957 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:46.957 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.481397+0000", 2023-10-09T06:08:46.957 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.957 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.957 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.958 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:46.958 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.481396+0000", 2023-10-09T06:08:46.958 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:08:46.958 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.958 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.958 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:46.958 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.481404+0000", 2023-10-09T06:08:46.958 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.7919999999999998e-06 2023-10-09T06:08:46.958 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.958 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.958 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:46.959 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.481416+0000", 2023-10-09T06:08:46.959 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1484000000000001e-05 2023-10-09T06:08:46.959 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.959 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.959 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:46.959 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.481421+0000", 2023-10-09T06:08:46.959 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0980000000000001e-06 2023-10-09T06:08:46.959 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.959 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.959 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.959 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.481462+0000", 2023-10-09T06:08:46.959 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1106000000000003e-05 2023-10-09T06:08:46.960 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.960 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.960 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:08:46.960 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.481471+0000", 2023-10-09T06:08:46.960 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0059999999999998e-06 2023-10-09T06:08:46.960 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.960 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.960 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.960 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.493032+0000", 2023-10-09T06:08:46.960 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0115604680000001 2023-10-09T06:08:46.960 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.960 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.961 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:46.961 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.493102+0000", 2023-10-09T06:08:46.961 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0866999999999998e-05 2023-10-09T06:08:46.961 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.961 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.961 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:08:46.961 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.493173+0000", 2023-10-09T06:08:46.961 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0990999999999998e-05 2023-10-09T06:08:46.961 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.961 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.961 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:08:46.961 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.499044+0000", 2023-10-09T06:08:46.962 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.005870269 2023-10-09T06:08:46.962 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.962 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.962 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.962 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.504577+0000", 2023-10-09T06:08:46.962 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0055330509999999998 2023-10-09T06:08:46.962 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.962 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.962 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.962 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.516846+0000", 2023-10-09T06:08:46.962 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012268891000000001 2023-10-09T06:08:46.962 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.963 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.963 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:08:46.963 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.516887+0000", 2023-10-09T06:08:46.963 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1427000000000002e-05 2023-10-09T06:08:46.963 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.963 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.963 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:08:46.963 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.516913+0000", 2023-10-09T06:08:46.963 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.5761e-05 2023-10-09T06:08:46.963 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.963 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:46.963 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.964 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.964 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.964 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:188411 8.9 8:9b524035:::benchmark_data_smithi064_457003_object83893:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2023-10-09T06:08:46.964 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:27.488449+0000", 2023-10-09T06:08:46.964 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 139.16564636499999, 2023-10-09T06:08:46.964 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0412856909999999, 2023-10-09T06:08:46.964 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:46.964 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:08:46.964 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:46.964 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:08:46.964 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:08:46.964 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188411 2023-10-09T06:08:46.965 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.965 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:46.965 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.965 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:46.965 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.488449+0000", 2023-10-09T06:08:46.965 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.965 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.965 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.965 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:46.965 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.488449+0000", 2023-10-09T06:08:46.965 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.965 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.966 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.966 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:46.966 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.488448+0000", 2023-10-09T06:08:46.966 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:08:46.966 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.966 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.966 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:46.966 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.488454+0000", 2023-10-09T06:08:46.966 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9839999999999998e-06 2023-10-09T06:08:46.966 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.966 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.967 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:46.967 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.488469+0000", 2023-10-09T06:08:46.967 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4807000000000001e-05 2023-10-09T06:08:46.967 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.967 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.967 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:46.967 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.488475+0000", 2023-10-09T06:08:46.967 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3670000000000002e-06 2023-10-09T06:08:46.967 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.967 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.967 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.967 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.488537+0000", 2023-10-09T06:08:46.968 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2039999999999996e-05 2023-10-09T06:08:46.968 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.968 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.968 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:08:46.968 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.488544+0000", 2023-10-09T06:08:46.968 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4590000000000001e-06 2023-10-09T06:08:46.968 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.968 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.968 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.968 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.496186+0000", 2023-10-09T06:08:46.968 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0076412109999999 2023-10-09T06:08:46.968 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.496275+0000", 2023-10-09T06:08:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9154999999999996e-05 2023-10-09T06:08:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:08:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.496350+0000", 2023-10-09T06:08:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4803999999999997e-05 2023-10-09T06:08:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:08:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.501737+0000", 2023-10-09T06:08:46.970 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0053872169999999997 2023-10-09T06:08:46.970 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.970 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.970 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.970 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.504624+0000", 2023-10-09T06:08:46.970 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0028874769999999998 2023-10-09T06:08:46.970 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.970 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.970 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.970 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.529656+0000", 2023-10-09T06:08:46.970 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.025032085999999999 2023-10-09T06:08:46.971 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.971 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.971 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:08:46.971 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.529691+0000", 2023-10-09T06:08:46.971 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5105999999999999e-05 2023-10-09T06:08:46.971 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.971 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.971 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:08:46.971 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.529735+0000", 2023-10-09T06:08:46.971 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3306999999999997e-05 2023-10-09T06:08:46.971 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.971 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:46.972 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.972 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.972 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.972 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:188453 8.9 8:9cddc70a:::benchmark_data_smithi064_457003_object83935:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2023-10-09T06:08:46.972 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:27.528917+0000", 2023-10-09T06:08:46.972 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 139.12517887800001, 2023-10-09T06:08:46.972 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0009949220000001, 2023-10-09T06:08:46.972 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:46.972 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:08:46.972 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:46.972 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:08:46.972 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:08:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188453 2023-10-09T06:08:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.528917+0000", 2023-10-09T06:08:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.528917+0000", 2023-10-09T06:08:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.974 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.974 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:46.974 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.528915+0000", 2023-10-09T06:08:46.974 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:08:46.974 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.974 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.974 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:46.974 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.528921+0000", 2023-10-09T06:08:46.974 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4140000000000002e-06 2023-10-09T06:08:46.974 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.974 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.975 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:46.975 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.528933+0000", 2023-10-09T06:08:46.975 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2618e-05 2023-10-09T06:08:46.975 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.975 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.975 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:46.975 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.528938+0000", 2023-10-09T06:08:46.975 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6809999999999997e-06 2023-10-09T06:08:46.975 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.975 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.975 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.975 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.528989+0000", 2023-10-09T06:08:46.976 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1201e-05 2023-10-09T06:08:46.976 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.976 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.976 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:08:46.976 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.528999+0000", 2023-10-09T06:08:46.976 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.9089999999999995e-06 2023-10-09T06:08:46.976 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.976 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.976 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.976 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.499061+0000", 2023-10-09T06:08:46.976 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.97006157000000004 2023-10-09T06:08:46.976 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.976 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.977 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:46.977 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.499136+0000", 2023-10-09T06:08:46.977 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4838000000000004e-05 2023-10-09T06:08:46.977 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.977 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.977 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:08:46.977 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.499210+0000", 2023-10-09T06:08:46.977 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4647999999999994e-05 2023-10-09T06:08:46.977 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.977 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.977 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:08:46.977 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.510262+0000", 2023-10-09T06:08:46.978 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011051221999999999 2023-10-09T06:08:46.978 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.978 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.978 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.978 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.516809+0000", 2023-10-09T06:08:46.978 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.006547125 2023-10-09T06:08:46.978 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.978 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.978 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.978 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.529829+0000", 2023-10-09T06:08:46.978 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013020056 2023-10-09T06:08:46.978 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.979 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.979 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:08:46.979 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.529873+0000", 2023-10-09T06:08:46.979 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4036999999999999e-05 2023-10-09T06:08:46.979 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.979 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.979 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:08:46.979 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.529911+0000", 2023-10-09T06:08:46.979 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8702999999999999e-05 2023-10-09T06:08:46.979 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.979 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:46.979 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.980 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.980 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.980 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:188454 8.9 8:984ccfac:::benchmark_data_smithi064_457003_object83936:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2023-10-09T06:08:46.980 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:27.532247+0000", 2023-10-09T06:08:46.980 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 139.121848059, 2023-10-09T06:08:46.980 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.004500784, 2023-10-09T06:08:46.980 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:46.980 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:08:46.980 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:46.980 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:08:46.980 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:08:46.980 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188454 2023-10-09T06:08:46.981 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.981 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:46.981 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.981 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:46.981 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.532247+0000", 2023-10-09T06:08:46.981 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.981 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.981 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.981 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:46.981 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.532247+0000", 2023-10-09T06:08:46.981 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.981 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.982 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.982 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:46.982 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.532246+0000", 2023-10-09T06:08:46.982 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:08:46.982 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.982 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.982 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:46.982 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.532252+0000", 2023-10-09T06:08:46.982 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7440000000000004e-06 2023-10-09T06:08:46.982 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.982 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.983 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:46.983 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.532265+0000", 2023-10-09T06:08:46.983 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2741e-05 2023-10-09T06:08:46.983 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.983 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.983 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:46.983 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.532270+0000", 2023-10-09T06:08:46.983 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9080000000000003e-06 2023-10-09T06:08:46.983 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.983 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.983 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.983 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.532315+0000", 2023-10-09T06:08:46.984 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5488999999999998e-05 2023-10-09T06:08:46.984 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.984 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.984 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:08:46.984 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.532323+0000", 2023-10-09T06:08:46.984 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.413e-06 2023-10-09T06:08:46.984 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.984 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.984 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.984 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.501780+0000", 2023-10-09T06:08:46.984 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96945757300000002 2023-10-09T06:08:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.501860+0000", 2023-10-09T06:08:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0241999999999995e-05 2023-10-09T06:08:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:08:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.501950+0000", 2023-10-09T06:08:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9709000000000005e-05 2023-10-09T06:08:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:08:46.986 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.516761+0000", 2023-10-09T06:08:46.986 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.014810886000000001 2023-10-09T06:08:46.986 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.986 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.986 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.986 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.529956+0000", 2023-10-09T06:08:46.986 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013195098000000001 2023-10-09T06:08:46.986 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.986 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.986 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.986 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.536665+0000", 2023-10-09T06:08:46.987 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0067089239999999998 2023-10-09T06:08:46.987 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.987 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.987 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:08:46.987 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.536711+0000", 2023-10-09T06:08:46.987 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5556999999999998e-05 2023-10-09T06:08:46.987 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.987 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.987 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:08:46.987 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.536748+0000", 2023-10-09T06:08:46.987 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7622e-05 2023-10-09T06:08:46.987 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.988 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:46.988 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.988 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.988 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.988 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:188519 8.9 8:9265e9e6:::benchmark_data_smithi064_457003_object84001:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2023-10-09T06:08:46.988 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:27.599383+0000", 2023-10-09T06:08:46.988 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 139.05471199799999, 2023-10-09T06:08:46.988 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.94240030699999999, 2023-10-09T06:08:46.988 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:46.988 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:08:46.988 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:46.989 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:08:46.989 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:08:46.989 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188519 2023-10-09T06:08:46.989 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.989 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:46.989 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.989 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:46.989 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.599383+0000", 2023-10-09T06:08:46.990 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.990 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.990 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.990 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:46.990 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.599383+0000", 2023-10-09T06:08:46.990 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.990 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.990 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.990 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:46.991 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.599382+0000", 2023-10-09T06:08:46.991 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:08:46.991 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.991 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.991 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:46.991 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.599392+0000", 2023-10-09T06:08:46.991 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.6029999999999994e-06 2023-10-09T06:08:46.991 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.991 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.991 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:46.992 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.599408+0000", 2023-10-09T06:08:46.992 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6436e-05 2023-10-09T06:08:46.992 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.992 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.992 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:46.992 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.599414+0000", 2023-10-09T06:08:46.992 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0009999999999998e-06 2023-10-09T06:08:46.992 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.992 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.992 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.992 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.599467+0000", 2023-10-09T06:08:46.993 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2541999999999999e-05 2023-10-09T06:08:46.993 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.993 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.993 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:08:46.993 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.599477+0000", 2023-10-09T06:08:46.993 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.9920000000000002e-06 2023-10-09T06:08:46.993 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.993 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.993 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:46.993 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.504656+0000", 2023-10-09T06:08:46.994 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.90517970800000003 2023-10-09T06:08:46.994 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.994 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.994 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:46.994 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.504775+0000", 2023-10-09T06:08:46.994 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000119188 2023-10-09T06:08:46.994 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.994 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.994 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:08:46.995 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.504902+0000", 2023-10-09T06:08:46.995 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012631799999999999 2023-10-09T06:08:46.995 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.995 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.995 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:08:46.995 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.516767+0000", 2023-10-09T06:08:46.995 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011865031999999999 2023-10-09T06:08:46.995 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.995 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.995 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.996 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.536788+0000", 2023-10-09T06:08:46.996 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.020021329000000001 2023-10-09T06:08:46.996 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.996 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.996 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:46.996 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.541699+0000", 2023-10-09T06:08:46.996 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0049108559999999999 2023-10-09T06:08:46.996 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.996 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.996 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:08:46.997 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.541737+0000", 2023-10-09T06:08:46.997 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8359000000000003e-05 2023-10-09T06:08:46.997 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.997 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.997 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:08:46.997 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.541784+0000", 2023-10-09T06:08:46.997 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6403000000000003e-05 2023-10-09T06:08:46.997 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.997 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:46.997 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:46.998 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.998 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.998 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:188541 8.9 8:976c196f:::benchmark_data_smithi064_457003_object84023:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2023-10-09T06:08:46.998 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:27.623480+0000", 2023-10-09T06:08:46.998 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 139.03061590799999, 2023-10-09T06:08:46.998 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.92449378299999996, 2023-10-09T06:08:46.998 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:46.998 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:08:46.998 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:46.999 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:08:46.999 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:08:46.999 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188541 2023-10-09T06:08:46.999 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:46.999 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:46.999 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:46.999 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:46.999 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.623480+0000", 2023-10-09T06:08:46.999 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:46.999 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.000 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.000 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:47.000 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.623480+0000", 2023-10-09T06:08:47.000 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:47.000 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.000 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.000 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:47.000 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.623478+0000", 2023-10-09T06:08:47.000 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:08:47.001 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.001 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.001 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:47.001 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.623485+0000", 2023-10-09T06:08:47.001 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6660000000000002e-06 2023-10-09T06:08:47.001 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.001 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.001 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:47.001 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.623500+0000", 2023-10-09T06:08:47.002 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5361000000000001e-05 2023-10-09T06:08:47.002 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.002 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.002 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:47.002 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.623507+0000", 2023-10-09T06:08:47.002 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9110000000000004e-06 2023-10-09T06:08:47.002 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.002 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.002 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:47.002 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.623554+0000", 2023-10-09T06:08:47.003 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7104000000000002e-05 2023-10-09T06:08:47.003 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.003 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.003 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:08:47.003 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.623564+0000", 2023-10-09T06:08:47.003 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0152999999999999e-05 2023-10-09T06:08:47.003 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.003 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.003 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:47.003 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.507577+0000", 2023-10-09T06:08:47.004 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.88401231300000005 2023-10-09T06:08:47.004 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.004 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.004 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:47.004 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.507706+0000", 2023-10-09T06:08:47.004 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012916499999999999 2023-10-09T06:08:47.004 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.004 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.004 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:08:47.004 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.507786+0000", 2023-10-09T06:08:47.005 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0057999999999998e-05 2023-10-09T06:08:47.005 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.005 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.005 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:08:47.005 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.516927+0000", 2023-10-09T06:08:47.005 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0091417119999999998 2023-10-09T06:08:47.005 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.005 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.005 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:47.005 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.536813+0000", 2023-10-09T06:08:47.006 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.019885612 2023-10-09T06:08:47.006 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.006 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.006 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:47.006 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.547886+0000", 2023-10-09T06:08:47.006 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011072465 2023-10-09T06:08:47.006 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.006 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.006 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:08:47.006 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.547935+0000", 2023-10-09T06:08:47.007 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9446000000000001e-05 2023-10-09T06:08:47.007 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.007 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.007 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:08:47.007 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.547973+0000", 2023-10-09T06:08:47.007 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8278999999999997e-05 2023-10-09T06:08:47.007 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:47.007 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:47.007 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:47.007 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.007 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.008 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:188544 8.9 8:90ec74b5:::benchmark_data_smithi064_457003_object84026:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2023-10-09T06:08:47.008 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:27.625183+0000", 2023-10-09T06:08:47.008 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 139.028912392, 2023-10-09T06:08:47.008 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.92289775500000004, 2023-10-09T06:08:47.008 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:47.008 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:08:47.008 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:47.008 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:08:47.008 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:08:47.008 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188544 2023-10-09T06:08:47.009 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.009 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:47.009 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.009 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:47.009 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.625183+0000", 2023-10-09T06:08:47.009 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:47.009 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.009 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.009 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:47.009 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.625183+0000", 2023-10-09T06:08:47.010 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:47.010 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.010 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.010 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:47.010 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.625182+0000", 2023-10-09T06:08:47.010 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:08:47.010 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.010 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.010 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:47.010 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.625187+0000", 2023-10-09T06:08:47.011 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2009999999999998e-06 2023-10-09T06:08:47.011 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.011 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.011 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:47.011 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.625202+0000", 2023-10-09T06:08:47.011 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4395e-05 2023-10-09T06:08:47.011 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.011 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.011 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:47.011 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.625208+0000", 2023-10-09T06:08:47.011 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3130000000000004e-06 2023-10-09T06:08:47.012 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.012 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.012 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:47.012 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.625245+0000", 2023-10-09T06:08:47.012 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6535999999999997e-05 2023-10-09T06:08:47.012 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.012 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.012 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:08:47.012 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.625250+0000", 2023-10-09T06:08:47.012 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9320000000000004e-06 2023-10-09T06:08:47.013 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.013 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.013 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:47.013 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.511069+0000", 2023-10-09T06:08:47.013 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.88581980500000002 2023-10-09T06:08:47.013 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.013 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.013 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:47.013 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.511192+0000", 2023-10-09T06:08:47.013 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012233799999999999 2023-10-09T06:08:47.013 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.014 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.014 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:08:47.014 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.511328+0000", 2023-10-09T06:08:47.014 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000136627 2023-10-09T06:08:47.014 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.014 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.014 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:08:47.014 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.516934+0000", 2023-10-09T06:08:47.014 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0056052899999999998 2023-10-09T06:08:47.014 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.015 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.015 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:47.015 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.536841+0000", 2023-10-09T06:08:47.015 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.019907659000000001 2023-10-09T06:08:47.015 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.015 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.015 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:47.015 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.548005+0000", 2023-10-09T06:08:47.015 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011163632999999999 2023-10-09T06:08:47.015 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.015 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.016 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:08:47.016 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.548048+0000", 2023-10-09T06:08:47.016 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2670000000000003e-05 2023-10-09T06:08:47.016 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.016 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.016 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:08:47.016 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.548081+0000", 2023-10-09T06:08:47.016 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.3244999999999998e-05 2023-10-09T06:08:47.016 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:47.016 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:47.017 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:47.017 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.017 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.017 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:58055 9.b 9:d73c869a:::benchmark_data_smithi064_511911_object58054:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-10-09T06:08:47.017 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:07:50.737236+0000", 2023-10-09T06:08:47.017 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 55.916859399000003, 2023-10-09T06:08:47.017 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.98528114, 2023-10-09T06:08:47.017 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:47.017 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:08:47.018 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:47.018 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:08:47.018 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:08:47.018 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 58055 2023-10-09T06:08:47.018 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.018 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:47.018 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.018 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:47.018 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.737236+0000", 2023-10-09T06:08:47.019 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:47.019 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.019 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.019 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:47.019 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.737236+0000", 2023-10-09T06:08:47.019 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:47.019 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.019 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.019 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:47.019 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.737235+0000", 2023-10-09T06:08:47.020 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:08:47.020 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.020 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.020 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:47.020 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.737274+0000", 2023-10-09T06:08:47.020 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9276000000000001e-05 2023-10-09T06:08:47.020 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.020 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.020 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:47.020 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.737289+0000", 2023-10-09T06:08:47.021 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5199e-05 2023-10-09T06:08:47.021 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.021 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.021 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:47.021 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.737296+0000", 2023-10-09T06:08:47.021 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0530000000000001e-06 2023-10-09T06:08:47.021 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.021 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.021 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:47.021 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.737357+0000", 2023-10-09T06:08:47.021 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0908e-05 2023-10-09T06:08:47.022 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.022 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.022 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:08:47.022 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.737368+0000", 2023-10-09T06:08:47.022 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0958000000000001e-05 2023-10-09T06:08:47.022 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.022 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.022 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:47.022 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.710653+0000", 2023-10-09T06:08:47.022 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.97328509799999996 2023-10-09T06:08:47.023 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.023 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.023 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:47.023 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.710782+0000", 2023-10-09T06:08:47.023 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012853100000000001 2023-10-09T06:08:47.023 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.023 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.023 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5", 2023-10-09T06:08:47.023 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.710945+0000", 2023-10-09T06:08:47.023 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000163177 2023-10-09T06:08:47.024 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.024 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.024 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:47.024 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.722281+0000", 2023-10-09T06:08:47.024 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011335638 2023-10-09T06:08:47.024 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.024 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.024 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:08:47.024 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.722372+0000", 2023-10-09T06:08:47.024 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1652999999999998e-05 2023-10-09T06:08:47.024 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.025 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.025 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:08:47.025 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.722427+0000", 2023-10-09T06:08:47.025 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4744000000000003e-05 2023-10-09T06:08:47.025 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.025 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.025 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:08:47.025 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.722517+0000", 2023-10-09T06:08:47.025 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0185000000000005e-05 2023-10-09T06:08:47.025 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:47.026 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:47.026 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:47.026 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.026 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.026 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:58058 9.b 9:d65d9511:::benchmark_data_smithi064_511911_object58057:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-10-09T06:08:47.026 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:07:50.741059+0000", 2023-10-09T06:08:47.026 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 55.913036605999999, 2023-10-09T06:08:47.026 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.98159827200000005, 2023-10-09T06:08:47.026 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:47.026 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:08:47.027 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:47.027 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:08:47.027 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:08:47.027 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 58058 2023-10-09T06:08:47.027 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.027 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:47.027 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.027 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:47.027 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.741059+0000", 2023-10-09T06:08:47.027 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:47.028 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.028 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.028 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:47.028 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.741059+0000", 2023-10-09T06:08:47.028 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:47.028 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.028 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.028 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:47.028 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.741058+0000", 2023-10-09T06:08:47.028 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:08:47.028 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.029 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.029 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:47.029 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.741169+0000", 2023-10-09T06:08:47.029 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000110797 2023-10-09T06:08:47.029 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.029 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.029 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:47.029 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.741182+0000", 2023-10-09T06:08:47.029 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3769e-05 2023-10-09T06:08:47.029 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.030 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.030 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:47.030 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.741188+0000", 2023-10-09T06:08:47.030 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5119999999999999e-06 2023-10-09T06:08:47.030 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.030 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.030 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:47.030 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.741222+0000", 2023-10-09T06:08:47.030 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.3902000000000002e-05 2023-10-09T06:08:47.030 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.031 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.031 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:08:47.031 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.741227+0000", 2023-10-09T06:08:47.031 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3979999999999998e-06 2023-10-09T06:08:47.031 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.031 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.031 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:47.031 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.713357+0000", 2023-10-09T06:08:47.031 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.972130147 2023-10-09T06:08:47.031 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.032 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.032 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:47.032 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.713483+0000", 2023-10-09T06:08:47.032 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000125375 2023-10-09T06:08:47.032 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.032 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.032 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5", 2023-10-09T06:08:47.032 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.713641+0000", 2023-10-09T06:08:47.032 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0001584 2023-10-09T06:08:47.032 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.033 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.033 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:47.033 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.722365+0000", 2023-10-09T06:08:47.033 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0087238769999999997 2023-10-09T06:08:47.033 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.033 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.033 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:08:47.033 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.722534+0000", 2023-10-09T06:08:47.033 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00016915300000000001 2023-10-09T06:08:47.033 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.034 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.034 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:08:47.034 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.722578+0000", 2023-10-09T06:08:47.034 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4209000000000003e-05 2023-10-09T06:08:47.034 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.034 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.034 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:08:47.034 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.722657+0000", 2023-10-09T06:08:47.034 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8801000000000007e-05 2023-10-09T06:08:47.034 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:47.035 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:47.035 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:47.035 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.035 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.035 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:58075 9.b 9:d33de774:::benchmark_data_smithi064_511911_object58074:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-10-09T06:08:47.035 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:07:50.764393+0000", 2023-10-09T06:08:47.035 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 55.889702028999999, 2023-10-09T06:08:47.035 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96020978199999996, 2023-10-09T06:08:47.035 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:47.035 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:08:47.036 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:47.036 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:08:47.036 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:08:47.036 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 58075 2023-10-09T06:08:47.036 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.036 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:47.036 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.036 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:47.036 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.764393+0000", 2023-10-09T06:08:47.036 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:47.036 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.037 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.037 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:47.037 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.764393+0000", 2023-10-09T06:08:47.037 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:47.037 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.037 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.037 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:47.037 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.764392+0000", 2023-10-09T06:08:47.037 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:08:47.037 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.038 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.038 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:47.038 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.764486+0000", 2023-10-09T06:08:47.038 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.4242999999999999e-05 2023-10-09T06:08:47.038 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.038 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.038 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:47.038 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.764507+0000", 2023-10-09T06:08:47.038 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0353e-05 2023-10-09T06:08:47.038 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.039 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.039 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:47.039 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.764515+0000", 2023-10-09T06:08:47.039 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0239999999999997e-06 2023-10-09T06:08:47.039 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.039 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.039 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:47.039 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.764570+0000", 2023-10-09T06:08:47.039 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5151000000000001e-05 2023-10-09T06:08:47.039 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.040 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.040 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:08:47.040 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.764580+0000", 2023-10-09T06:08:47.040 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.9769999999999995e-06 2023-10-09T06:08:47.040 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.040 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.040 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:47.040 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.715693+0000", 2023-10-09T06:08:47.040 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.95111346500000005 2023-10-09T06:08:47.040 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.041 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.041 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:47.041 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.715812+0000", 2023-10-09T06:08:47.041 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000119128 2023-10-09T06:08:47.041 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.041 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.041 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5", 2023-10-09T06:08:47.041 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.715953+0000", 2023-10-09T06:08:47.041 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014025800000000001 2023-10-09T06:08:47.041 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.042 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.042 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:08:47.042 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.722663+0000", 2023-10-09T06:08:47.042 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0067107199999999999 2023-10-09T06:08:47.042 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.042 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.042 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:47.042 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.724457+0000", 2023-10-09T06:08:47.042 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001793988 2023-10-09T06:08:47.042 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.043 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.043 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:08:47.043 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.724513+0000", 2023-10-09T06:08:47.043 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5543e-05 2023-10-09T06:08:47.043 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.043 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.043 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:08:47.043 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.724603+0000", 2023-10-09T06:08:47.043 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0174999999999996e-05 2023-10-09T06:08:47.043 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:47.044 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:47.044 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:47.044 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.044 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.044 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:58079 9.b 9:d29a828a:::benchmark_data_smithi064_511911_object58078:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-10-09T06:08:47.044 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:07:50.767169+0000", 2023-10-09T06:08:47.044 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 55.886926066999997, 2023-10-09T06:08:47.044 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96095287900000004, 2023-10-09T06:08:47.044 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:47.044 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:08:47.045 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:47.045 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:08:47.045 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:08:47.045 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 58079 2023-10-09T06:08:47.045 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.045 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:47.045 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.045 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:47.045 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.767169+0000", 2023-10-09T06:08:47.045 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:47.046 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.046 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.046 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:47.046 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.767169+0000", 2023-10-09T06:08:47.046 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:47.046 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.046 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.046 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:47.046 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.767168+0000", 2023-10-09T06:08:47.047 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:08:47.047 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.047 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.047 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:47.047 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.767248+0000", 2023-10-09T06:08:47.047 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0154000000000005e-05 2023-10-09T06:08:47.047 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.047 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.047 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:47.047 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.767263+0000", 2023-10-09T06:08:47.047 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4696e-05 2023-10-09T06:08:47.048 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.048 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.048 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:47.048 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.767268+0000", 2023-10-09T06:08:47.048 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0989999999999998e-06 2023-10-09T06:08:47.048 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.048 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.048 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:47.048 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.767303+0000", 2023-10-09T06:08:47.048 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5082000000000001e-05 2023-10-09T06:08:47.049 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.049 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.049 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:08:47.049 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.767311+0000", 2023-10-09T06:08:47.049 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3270000000000003e-06 2023-10-09T06:08:47.049 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.049 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.049 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:47.049 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718011+0000", 2023-10-09T06:08:47.049 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.950700086 2023-10-09T06:08:47.050 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.050 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.050 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:47.050 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718138+0000", 2023-10-09T06:08:47.050 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000127296 2023-10-09T06:08:47.050 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.050 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.050 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5", 2023-10-09T06:08:47.050 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718281+0000", 2023-10-09T06:08:47.050 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014276299999999999 2023-10-09T06:08:47.051 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.051 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.051 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:08:47.051 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.722668+0000", 2023-10-09T06:08:47.051 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0043875570000000003 2023-10-09T06:08:47.051 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.051 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.051 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:47.051 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.727987+0000", 2023-10-09T06:08:47.052 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0053181729999999998 2023-10-09T06:08:47.052 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.052 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.052 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:08:47.052 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.728043+0000", 2023-10-09T06:08:47.052 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6505999999999997e-05 2023-10-09T06:08:47.052 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.052 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.052 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:08:47.053 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.728122+0000", 2023-10-09T06:08:47.053 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9190000000000006e-05 2023-10-09T06:08:47.053 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:47.053 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:47.053 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:47.053 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.053 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.053 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:58084 9.b 9:dd7d63dc:::benchmark_data_smithi064_511911_object58083:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-10-09T06:08:47.053 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:07:50.772540+0000", 2023-10-09T06:08:47.053 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 55.881555806000001, 2023-10-09T06:08:47.054 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.95571486000000005, 2023-10-09T06:08:47.054 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:47.054 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:08:47.054 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:47.054 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:08:47.054 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:08:47.054 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 58084 2023-10-09T06:08:47.054 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.054 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:47.054 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.055 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:47.055 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.772540+0000", 2023-10-09T06:08:47.055 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:47.055 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.055 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.055 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:47.055 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.772540+0000", 2023-10-09T06:08:47.055 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:47.055 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.055 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.056 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:47.056 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.772538+0000", 2023-10-09T06:08:47.056 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:08:47.056 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.056 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.056 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:47.056 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.772575+0000", 2023-10-09T06:08:47.056 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6492000000000002e-05 2023-10-09T06:08:47.056 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.056 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.057 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:47.057 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.772593+0000", 2023-10-09T06:08:47.057 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8938999999999999e-05 2023-10-09T06:08:47.057 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.057 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.057 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:47.057 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.772601+0000", 2023-10-09T06:08:47.057 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.486e-06 2023-10-09T06:08:47.057 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.057 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.058 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:47.058 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.772645+0000", 2023-10-09T06:08:47.058 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4379999999999999e-05 2023-10-09T06:08:47.058 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.058 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.058 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:08:47.058 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.772652+0000", 2023-10-09T06:08:47.058 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8499999999999996e-06 2023-10-09T06:08:47.058 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.058 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.059 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:47.059 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.720211+0000", 2023-10-09T06:08:47.059 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.94755932399999998 2023-10-09T06:08:47.059 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.059 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.059 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:47.059 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.720331+0000", 2023-10-09T06:08:47.059 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000119253 2023-10-09T06:08:47.059 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.059 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.059 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5", 2023-10-09T06:08:47.060 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.720440+0000", 2023-10-09T06:08:47.060 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000109214 2023-10-09T06:08:47.060 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.060 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.060 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:08:47.060 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.723074+0000", 2023-10-09T06:08:47.060 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0026341860000000002 2023-10-09T06:08:47.060 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.060 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.060 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:47.061 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.728174+0000", 2023-10-09T06:08:47.061 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0051001379999999997 2023-10-09T06:08:47.061 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.061 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.061 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:08:47.061 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.728209+0000", 2023-10-09T06:08:47.061 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.4615e-05 2023-10-09T06:08:47.061 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:47.061 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:47.061 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:08:47.061 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.728254+0000", 2023-10-09T06:08:47.062 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5562000000000002e-05 2023-10-09T06:08:47.062 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:47.062 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:47.062 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:47.062 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:47.062 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:47.062 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:08:47.063 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:46 smithi064 ceph-mon[212750]: pgmap v1110: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 792 op/s; 31840/298062 objects degraded (10.682%); 48/298062 objects misplaced (0.016%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T06:08:47.063 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:46 smithi064 ceph-mon[212750]: failed to encode map e445 with expected crc 2023-10-09T06:08:47.063 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:46 smithi064 ceph-mon[212750]: osdmap e445: 8 total, 7 up, 8 in 2023-10-09T06:08:47.063 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:46 smithi064 ceph-mon[212750]: failed to encode map e445 with expected crc 2023-10-09T06:08:47.064 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:46 smithi064 ceph-mon[212750]: failed to encode map e445 with expected crc 2023-10-09T06:08:47.064 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:46 smithi064 ceph-mon[212750]: failed to encode map e445 with expected crc 2023-10-09T06:08:47.064 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:46 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1350967313' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:08:47.064 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:46 smithi156 ceph-mon[195101]: pgmap v1110: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 792 op/s; 31840/298062 objects degraded (10.682%); 48/298062 objects misplaced (0.016%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T06:08:47.064 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:46 smithi156 ceph-mon[195101]: failed to encode map e445 with expected crc 2023-10-09T06:08:47.064 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:46 smithi156 ceph-mon[195101]: osdmap e445: 8 total, 7 up, 8 in 2023-10-09T06:08:47.064 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:46 smithi156 ceph-mon[195101]: failed to encode map e445 with expected crc 2023-10-09T06:08:47.064 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:46 smithi156 ceph-mon[195101]: failed to encode map e445 with expected crc 2023-10-09T06:08:47.065 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:46 smithi156 ceph-mon[195101]: failed to encode map e445 with expected crc 2023-10-09T06:08:47.065 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:46 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1350967313' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:08:47.065 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:46 smithi064 ceph-mon[220044]: pgmap v1110: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 11 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 792 op/s; 31840/298062 objects degraded (10.682%); 48/298062 objects misplaced (0.016%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T06:08:47.065 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:46 smithi064 ceph-mon[220044]: failed to encode map e445 with expected crc 2023-10-09T06:08:47.065 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:46 smithi064 ceph-mon[220044]: osdmap e445: 8 total, 7 up, 8 in 2023-10-09T06:08:47.065 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:46 smithi064 ceph-mon[220044]: failed to encode map e445 with expected crc 2023-10-09T06:08:47.065 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:46 smithi064 ceph-mon[220044]: failed to encode map e445 with expected crc 2023-10-09T06:08:47.065 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:46 smithi064 ceph-mon[220044]: failed to encode map e445 with expected crc 2023-10-09T06:08:47.065 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:46 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1350967313' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:08:47.117 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:47.407 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:08:47.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:47.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:08:47.483 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:08:47.514 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:47.622 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:47.733 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:47.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:47.750 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:47 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:08:47.750 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:47 smithi156 ceph-mon[195101]: failed to encode map e446 with expected crc 2023-10-09T06:08:47.750 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:47 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1350967313' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:08:47.750 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:47 smithi156 ceph-mon[195101]: osdmap e446: 8 total, 7 up, 8 in 2023-10-09T06:08:47.750 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:47 smithi156 ceph-mon[195101]: failed to encode map e446 with expected crc 2023-10-09T06:08:47.750 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:47 smithi156 ceph-mon[195101]: failed to encode map e446 with expected crc 2023-10-09T06:08:47.750 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:47 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 31840/298062 objects degraded (10.682%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:08:47.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:47.841 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:08:47.851 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:47.901 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:47 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:08:47.901 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:47 smithi064 ceph-mon[212750]: failed to encode map e446 with expected crc 2023-10-09T06:08:47.901 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:47 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1350967313' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:08:47.901 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:47 smithi064 ceph-mon[212750]: osdmap e446: 8 total, 7 up, 8 in 2023-10-09T06:08:47.902 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:47 smithi064 ceph-mon[212750]: failed to encode map e446 with expected crc 2023-10-09T06:08:47.902 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:47 smithi064 ceph-mon[212750]: failed to encode map e446 with expected crc 2023-10-09T06:08:47.902 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:47 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 31840/298062 objects degraded (10.682%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:08:47.902 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:47 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:08:47.902 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:47 smithi064 ceph-mon[220044]: failed to encode map e446 with expected crc 2023-10-09T06:08:47.902 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:47 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1350967313' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:08:47.902 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:47 smithi064 ceph-mon[220044]: osdmap e446: 8 total, 7 up, 8 in 2023-10-09T06:08:47.902 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:47 smithi064 ceph-mon[220044]: failed to encode map e446 with expected crc 2023-10-09T06:08:47.902 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:47 smithi064 ceph-mon[220044]: failed to encode map e446 with expected crc 2023-10-09T06:08:47.903 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:47 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 31840/298062 objects degraded (10.682%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:08:47.905 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:47.975 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:47.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:47.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:48.056 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:08:48.096 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:48.149 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:48.561 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:48.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:48.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:48.671 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:48.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:48 smithi064 ceph-mon[212750]: pgmap v1113: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 50 MiB/s wr, 977 op/s; 28626/299013 objects degraded (9.573%); 3269/299013 objects misplaced (1.093%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T06:08:48.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:48 smithi064 ceph-mon[220044]: pgmap v1113: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 50 MiB/s wr, 977 op/s; 28626/299013 objects degraded (9.573%); 3269/299013 objects misplaced (1.093%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T06:08:48.721 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:48.787 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:48.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:48.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:48.869 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:48.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:48 smithi156 ceph-mon[195101]: pgmap v1113: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 50 MiB/s wr, 977 op/s; 28626/299013 objects degraded (9.573%); 3269/299013 objects misplaced (1.093%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T06:08:48.898 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:48.923 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:48.964 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:48.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:48.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:49.067 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:08:49.107 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:49.407 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:08:49.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:49.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:08:49.513 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:49.612 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:49.766 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:49.774 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:49 smithi064 ceph-mon[212750]: pgmap v1114: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 32 MiB/s wr, 720 op/s; 28371/298248 objects degraded (9.513%); 3269/298248 objects misplaced (1.096%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:08:49.774 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:49 smithi064 ceph-mon[220044]: pgmap v1114: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 32 MiB/s wr, 720 op/s; 28371/298248 objects degraded (9.513%); 3269/298248 objects misplaced (1.096%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:08:49.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:49.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:49.876 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:49.936 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:49.983 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:08:49.983 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:08:49.983 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.983 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:105797 9.5 9:a7ece25e:::benchmark_data_smithi064_511911_object4695:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e446)", 2023-10-09T06:08:49.983 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:08:49.976786+0000", 2023-10-09T06:08:49.983 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0047853210000000004, 2023-10-09T06:08:49.983 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0048170879999999998, 2023-10-09T06:08:49.983 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:49.983 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:08:49.983 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:49.984 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:08:49.984 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:08:49.984 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 105797 2023-10-09T06:08:49.984 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.984 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:49.984 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.984 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:49.984 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.976786+0000", 2023-10-09T06:08:49.984 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:49.984 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.984 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.985 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:49.985 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.976786+0000", 2023-10-09T06:08:49.985 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:49.985 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.985 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.985 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:49.985 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.976785+0000", 2023-10-09T06:08:49.985 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:08:49.985 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.985 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.985 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:49.985 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.976789+0000", 2023-10-09T06:08:49.986 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5700000000000001e-06 2023-10-09T06:08:49.986 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.986 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.986 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:49.986 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.976798+0000", 2023-10-09T06:08:49.986 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1360000000000003e-06 2023-10-09T06:08:49.986 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.986 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.986 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:49.986 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.976804+0000", 2023-10-09T06:08:49.986 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4369999999999998e-06 2023-10-09T06:08:49.987 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.987 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.987 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:49.987 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.976936+0000", 2023-10-09T06:08:49.987 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013223099999999999 2023-10-09T06:08:49.987 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.987 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.987 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:49.987 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.977050+0000", 2023-10-09T06:08:49.987 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000114751 2023-10-09T06:08:49.987 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.987 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.988 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5", 2023-10-09T06:08:49.988 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.977159+0000", 2023-10-09T06:08:49.988 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000108077 2023-10-09T06:08:49.988 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:49.988 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:49.988 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:49.988 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.988 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.988 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.35940.0:105794 9.c e446/425 9:372bf0ea:::benchmark_data_smithi064_511911_object4692:head v 446'6521, mlcod=446'6519)", 2023-10-09T06:08:49.988 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:08:49.976848+0000", 2023-10-09T06:08:49.988 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0047235269999999999, 2023-10-09T06:08:49.988 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.004836524, 2023-10-09T06:08:49.989 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:49.989 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:08:49.989 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:49.989 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.989 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:49.989 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.976848+0000", 2023-10-09T06:08:49.989 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:49.989 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.989 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.989 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:49.989 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.976848+0000", 2023-10-09T06:08:49.989 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:49.990 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.990 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.990 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:49.990 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.976847+0000", 2023-10-09T06:08:49.990 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:08:49.990 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.990 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.990 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:49.990 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.976851+0000", 2023-10-09T06:08:49.990 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.579e-06 2023-10-09T06:08:49.991 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.991 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.991 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:49.991 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.976856+0000", 2023-10-09T06:08:49.991 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2700000000000004e-06 2023-10-09T06:08:49.991 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.991 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.991 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:49.991 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.976859+0000", 2023-10-09T06:08:49.991 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1690000000000001e-06 2023-10-09T06:08:49.991 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.991 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.992 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:49.992 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.977838+0000", 2023-10-09T06:08:49.992 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00097873899999999995 2023-10-09T06:08:49.992 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.992 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.992 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:49.992 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.977850+0000", 2023-10-09T06:08:49.992 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1907e-05 2023-10-09T06:08:49.992 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:49.992 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:49.992 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:49.992 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.993 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.993 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.35940.0:105796 9.1 e446/425 9:802674e4:::benchmark_data_smithi064_511911_object4694:head v 446'6576, mlcod=446'6574)", 2023-10-09T06:08:49.993 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:08:49.979622+0000", 2023-10-09T06:08:49.993 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0019494270000000001, 2023-10-09T06:08:49.993 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0021230659999999998, 2023-10-09T06:08:49.993 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:49.993 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:08:49.993 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:49.993 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.993 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:49.993 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.979622+0000", 2023-10-09T06:08:49.994 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:49.994 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.994 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.994 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:49.994 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.979622+0000", 2023-10-09T06:08:49.994 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:49.994 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.994 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.994 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:49.994 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.979621+0000", 2023-10-09T06:08:49.994 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:08:49.995 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.995 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.995 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:49.995 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.979633+0000", 2023-10-09T06:08:49.995 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1783999999999999e-05 2023-10-09T06:08:49.995 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.995 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.995 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:49.995 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.979644+0000", 2023-10-09T06:08:49.995 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1559999999999999e-05 2023-10-09T06:08:49.995 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.995 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.996 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:49.996 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.979651+0000", 2023-10-09T06:08:49.996 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.224e-06 2023-10-09T06:08:49.996 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.996 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.996 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:49.996 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.980507+0000", 2023-10-09T06:08:49.996 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00085603100000000004 2023-10-09T06:08:49.996 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.996 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.996 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:49.996 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.980524+0000", 2023-10-09T06:08:49.997 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7028999999999999e-05 2023-10-09T06:08:49.997 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:49.997 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:49.997 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:49.997 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.997 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.997 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:105800 9.e 9:77f5b648:::benchmark_data_smithi064_511911_object4698:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e446)", 2023-10-09T06:08:49.997 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:08:49.979816+0000", 2023-10-09T06:08:49.997 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0017553460000000001, 2023-10-09T06:08:49.997 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0019879110000000002, 2023-10-09T06:08:49.997 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:49.998 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:08:49.998 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:49.998 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:08:49.998 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:08:49.998 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 105800 2023-10-09T06:08:49.998 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.998 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:49.998 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.998 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:49.998 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.979816+0000", 2023-10-09T06:08:49.998 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:49.998 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.999 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.999 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:49.999 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.979816+0000", 2023-10-09T06:08:49.999 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:49.999 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.999 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.999 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:49.999 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.979815+0000", 2023-10-09T06:08:49.999 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:08:49.999 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:49.999 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:49.999 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:49.999 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.979821+0000", 2023-10-09T06:08:50.000 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8669999999999999e-06 2023-10-09T06:08:50.000 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.000 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.000 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:50.000 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.979831+0000", 2023-10-09T06:08:50.000 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0927000000000001e-05 2023-10-09T06:08:50.000 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.000 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.000 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:50.000 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.979837+0000", 2023-10-09T06:08:50.000 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6559999999999997e-06 2023-10-09T06:08:50.000 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.001 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.001 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:50.001 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.979886+0000", 2023-10-09T06:08:50.001 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9218999999999998e-05 2023-10-09T06:08:50.001 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.001 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.001 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:50.001 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.979996+0000", 2023-10-09T06:08:50.001 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000109305 2023-10-09T06:08:50.001 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.001 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.001 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:08:50.002 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.980124+0000", 2023-10-09T06:08:50.002 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012832100000000001 2023-10-09T06:08:50.002 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:50.002 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:50.002 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:50.002 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.002 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.002 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:105801 9.7 9:e7854de6:::benchmark_data_smithi064_511911_object4699:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e446)", 2023-10-09T06:08:50.002 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:08:49.979852+0000", 2023-10-09T06:08:50.002 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0017192430000000001, 2023-10-09T06:08:50.002 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0020318179999999999, 2023-10-09T06:08:50.003 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:50.003 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:08:50.003 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:50.003 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:08:50.003 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:08:50.003 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 105801 2023-10-09T06:08:50.003 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.003 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:50.003 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.003 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:50.003 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.979852+0000", 2023-10-09T06:08:50.003 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:50.004 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.004 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.004 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:50.004 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.979852+0000", 2023-10-09T06:08:50.004 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:50.004 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.004 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.004 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:50.004 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.979852+0000", 2023-10-09T06:08:50.004 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:08:50.004 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.004 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.004 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:50.005 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.979854+0000", 2023-10-09T06:08:50.005 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.5519999999999999e-06 2023-10-09T06:08:50.005 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.005 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.005 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:50.005 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.979860+0000", 2023-10-09T06:08:50.005 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3119999999999999e-06 2023-10-09T06:08:50.005 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.005 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.005 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:50.005 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.979864+0000", 2023-10-09T06:08:50.006 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.63e-06 2023-10-09T06:08:50.006 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.006 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.006 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:50.006 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.980126+0000", 2023-10-09T06:08:50.006 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00026163100000000002 2023-10-09T06:08:50.006 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.006 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.006 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:50.006 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.980238+0000", 2023-10-09T06:08:50.006 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000111956 2023-10-09T06:08:50.006 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.007 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.007 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:08:50.007 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.980311+0000", 2023-10-09T06:08:50.007 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2933000000000001e-05 2023-10-09T06:08:50.007 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:50.007 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:50.007 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:50.007 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.007 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.007 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:105785 9.7 9:eb4b186b:::benchmark_data_smithi064_511911_object4683:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e446)", 2023-10-09T06:08:50.007 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:08:49.963019+0000", 2023-10-09T06:08:50.007 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.018552388999999999, 2023-10-09T06:08:50.007 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018935771000000001, 2023-10-09T06:08:50.008 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:50.008 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:08:50.008 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:50.008 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:08:50.008 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:08:50.008 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 105785 2023-10-09T06:08:50.008 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.008 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:50.008 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.008 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:50.008 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.963019+0000", 2023-10-09T06:08:50.008 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:50.009 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.009 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.009 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:50.009 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.963019+0000", 2023-10-09T06:08:50.009 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:50.009 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.009 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.009 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:50.009 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.963018+0000", 2023-10-09T06:08:50.009 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:08:50.009 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.009 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.010 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:50.010 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.963025+0000", 2023-10-09T06:08:50.010 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0280000000000003e-06 2023-10-09T06:08:50.010 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.010 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.010 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:50.010 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.963032+0000", 2023-10-09T06:08:50.010 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.348e-06 2023-10-09T06:08:50.010 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.010 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.010 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:50.011 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.963036+0000", 2023-10-09T06:08:50.011 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9450000000000003e-06 2023-10-09T06:08:50.011 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.011 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.011 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:50.011 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.969193+0000", 2023-10-09T06:08:50.011 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0061569049999999998 2023-10-09T06:08:50.011 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.011 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.011 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:50.011 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.969306+0000", 2023-10-09T06:08:50.011 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000113811 2023-10-09T06:08:50.012 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.012 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.012 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:08:50.012 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.969416+0000", 2023-10-09T06:08:50.012 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010969200000000001 2023-10-09T06:08:50.012 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.012 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.012 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:08:50.012 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.977817+0000", 2023-10-09T06:08:50.012 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0084007130000000006 2023-10-09T06:08:50.013 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.013 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.013 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:50.013 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.980106+0000", 2023-10-09T06:08:50.013 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0022888869999999999 2023-10-09T06:08:50.013 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:50.013 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:50.013 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:50.013 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.013 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.013 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:105789 9.d 9:b39f30c8:::benchmark_data_smithi064_511911_object4687:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e446)", 2023-10-09T06:08:50.013 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:08:49.963082+0000", 2023-10-09T06:08:50.014 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.018489351000000001, 2023-10-09T06:08:50.014 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018966750000000001, 2023-10-09T06:08:50.014 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:50.014 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:08:50.014 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:50.014 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:08:50.014 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:08:50.014 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 105789 2023-10-09T06:08:50.014 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.015 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:50.015 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.015 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:50.015 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.963082+0000", 2023-10-09T06:08:50.015 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:50.015 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.015 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.015 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:50.015 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.963082+0000", 2023-10-09T06:08:50.016 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:50.016 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.016 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.016 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:50.016 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.963082+0000", 2023-10-09T06:08:50.016 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:08:50.016 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.016 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.016 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:50.016 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.963084+0000", 2023-10-09T06:08:50.016 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.204e-06 2023-10-09T06:08:50.016 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.017 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.017 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:50.017 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.963090+0000", 2023-10-09T06:08:50.017 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4320000000000004e-06 2023-10-09T06:08:50.017 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.017 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.017 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:50.017 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.963094+0000", 2023-10-09T06:08:50.017 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8840000000000004e-06 2023-10-09T06:08:50.017 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.017 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.018 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:50.018 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.969573+0000", 2023-10-09T06:08:50.018 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0064791470000000002 2023-10-09T06:08:50.018 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.018 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.018 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:50.018 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.969670+0000", 2023-10-09T06:08:50.018 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.6762999999999995e-05 2023-10-09T06:08:50.018 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.018 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.018 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:08:50.018 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.969771+0000", 2023-10-09T06:08:50.019 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000100715 2023-10-09T06:08:50.019 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.019 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.019 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:08:50.019 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.975334+0000", 2023-10-09T06:08:50.019 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.005563559 2023-10-09T06:08:50.019 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.019 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.019 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:08:50.019 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.975349+0000", 2023-10-09T06:08:50.019 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4859e-05 2023-10-09T06:08:50.019 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:50.020 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:50.020 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:50.020 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.020 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.020 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:105793 9.7 9:e2d594b1:::benchmark_data_smithi064_511911_object4691:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e446)", 2023-10-09T06:08:50.020 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:08:49.974781+0000", 2023-10-09T06:08:50.020 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0067900119999999998, 2023-10-09T06:08:50.020 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0073514460000000002, 2023-10-09T06:08:50.020 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:08:50.020 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:08:50.020 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:08:50.020 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:08:50.020 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:08:50.021 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 105793 2023-10-09T06:08:50.021 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.021 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:08:50.021 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.021 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:08:50.021 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.974781+0000", 2023-10-09T06:08:50.021 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:50.021 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.021 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.021 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:08:50.021 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.974781+0000", 2023-10-09T06:08:50.021 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:08:50.022 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.022 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.022 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:08:50.022 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.974780+0000", 2023-10-09T06:08:50.022 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:08:50.022 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.022 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.022 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:08:50.022 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.974786+0000", 2023-10-09T06:08:50.022 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0399999999999998e-06 2023-10-09T06:08:50.022 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.023 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.023 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:08:50.023 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.974799+0000", 2023-10-09T06:08:50.023 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3467e-05 2023-10-09T06:08:50.023 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.023 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.023 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:08:50.023 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.974805+0000", 2023-10-09T06:08:50.023 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7180000000000001e-06 2023-10-09T06:08:50.023 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.024 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.024 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:08:50.024 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.975003+0000", 2023-10-09T06:08:50.024 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00019802900000000001 2023-10-09T06:08:50.024 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.024 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.024 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:08:50.024 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.975133+0000", 2023-10-09T06:08:50.024 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000130333 2023-10-09T06:08:50.024 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:08:50.024 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:08:50.024 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:08:50.025 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:08:49.975256+0000", 2023-10-09T06:08:50.025 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000122743 2023-10-09T06:08:50.025 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:50.025 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:08:50.025 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:50.025 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:08:50.025 INFO:teuthology.orchestra.run.smithi156.stdout: ], 2023-10-09T06:08:50.025 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 8 2023-10-09T06:08:50.025 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:08:50.026 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:49 smithi156 ceph-mon[195101]: pgmap v1114: 119 pgs: 1 active+recovering+undersized+remapped, 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 32 MiB/s wr, 720 op/s; 28371/298248 objects degraded (9.513%); 3269/298248 objects misplaced (1.096%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:08:50.028 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:50.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:50.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:50.133 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:50.163 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:50.274 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:50.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:50.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:50.381 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:50.410 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:50.463 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:08:50.654 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:50.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:50.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:50.769 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:50.770 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 9 pgp_num to 16 2023-10-09T06:08:50.865 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:08:50.898 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:50.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:50 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3924246152' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:08:51.062 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:50 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3924246152' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:08:51.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:50 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3924246152' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:08:51.591 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:08:51.840 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:51.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:51.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:51.955 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:52.028 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:51 smithi064 ceph-mon[212750]: failed to encode map e447 with expected crc 2023-10-09T06:08:52.028 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:51 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3924246152' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T06:08:52.028 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:51 smithi064 ceph-mon[212750]: osdmap e447: 8 total, 7 up, 8 in 2023-10-09T06:08:52.028 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:51 smithi064 ceph-mon[212750]: failed to encode map e447 with expected crc 2023-10-09T06:08:52.028 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:51 smithi064 ceph-mon[212750]: failed to encode map e447 with expected crc 2023-10-09T06:08:52.028 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:51 smithi064 ceph-mon[212750]: pgmap v1116: 119 pgs: 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 74 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 170 B/s rd, 16 MiB/s wr, 830 op/s; 28160/292311 objects degraded (9.634%); 2966/292311 objects misplaced (1.015%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:08:52.028 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:51 smithi064 ceph-mon[220044]: failed to encode map e447 with expected crc 2023-10-09T06:08:52.028 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:51 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3924246152' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T06:08:52.029 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:51 smithi064 ceph-mon[220044]: osdmap e447: 8 total, 7 up, 8 in 2023-10-09T06:08:52.029 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:51 smithi064 ceph-mon[220044]: failed to encode map e447 with expected crc 2023-10-09T06:08:52.029 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:51 smithi064 ceph-mon[220044]: failed to encode map e447 with expected crc 2023-10-09T06:08:52.029 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:51 smithi064 ceph-mon[220044]: pgmap v1116: 119 pgs: 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 74 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 170 B/s rd, 16 MiB/s wr, 830 op/s; 28160/292311 objects degraded (9.634%); 2966/292311 objects misplaced (1.015%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:08:52.029 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:51 smithi156 ceph-mon[195101]: failed to encode map e447 with expected crc 2023-10-09T06:08:52.029 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:51 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3924246152' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T06:08:52.029 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:51 smithi156 ceph-mon[195101]: osdmap e447: 8 total, 7 up, 8 in 2023-10-09T06:08:52.030 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:51 smithi156 ceph-mon[195101]: failed to encode map e447 with expected crc 2023-10-09T06:08:52.030 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:51 smithi156 ceph-mon[195101]: failed to encode map e447 with expected crc 2023-10-09T06:08:52.030 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:51 smithi156 ceph-mon[195101]: pgmap v1116: 119 pgs: 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 74 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 170 B/s rd, 16 MiB/s wr, 830 op/s; 28160/292311 objects degraded (9.634%); 2966/292311 objects misplaced (1.015%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:08:52.045 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:52.208 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:52.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:52.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:52.252 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:52.318 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:52.332 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:08:52.377 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:08:52.407 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:52.421 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:08:52.463 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:08:52.510 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:08:52.550 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:08:52.574 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:52.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:52.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:52.613 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:08:52.613 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-10-09T06:08:52.613 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:08:52.613 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:08:52.613 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:08:52.614 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:08:52.614 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:08:52.614 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:08:52.614 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:08:52.685 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:52.755 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:52.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:52 smithi064 ceph-mon[212750]: failed to encode map e447 with expected crc 2023-10-09T06:08:52.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:52 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1759856454' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:52.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:52 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 28160/292311 objects degraded (9.634%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:08:52.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:52 smithi064 ceph-mon[220044]: failed to encode map e447 with expected crc 2023-10-09T06:08:52.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:52 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1759856454' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:52.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:52 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 28160/292311 objects degraded (9.634%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:08:53.011 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:52 smithi156 ceph-mon[195101]: failed to encode map e447 with expected crc 2023-10-09T06:08:53.012 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:52 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1759856454' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:08:53.012 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:52 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 28160/292311 objects degraded (9.634%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:08:53.321 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:08:53.321 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:08:53.868 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:53.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:53.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:53.973 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:54.017 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:53 smithi156 ceph-mon[195101]: pgmap v1117: 119 pgs: 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 74 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 139 B/s rd, 13 MiB/s wr, 858 op/s; 27735/288309 objects degraded (9.620%); 2966/288309 objects misplaced (1.029%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T06:08:54.022 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:54.198 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:54.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:54.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:54.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:53 smithi064 ceph-mon[212750]: pgmap v1117: 119 pgs: 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 74 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 139 B/s rd, 13 MiB/s wr, 858 op/s; 27735/288309 objects degraded (9.620%); 2966/288309 objects misplaced (1.029%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T06:08:54.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:53 smithi064 ceph-mon[220044]: pgmap v1117: 119 pgs: 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 74 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 139 B/s rd, 13 MiB/s wr, 858 op/s; 27735/288309 objects degraded (9.620%); 2966/288309 objects misplaced (1.029%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T06:08:54.247 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:08:54.301 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:54.353 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:54.448 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T06:08:54.448 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T06:08:54.635 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:54.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:54.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:54.740 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:08:54.802 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:54.805 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:08:54.895 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:08:54.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:54.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:08:55.009 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:55.085 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:55.240 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:55.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:55.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:55.348 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:08:55.426 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:08:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:08:55] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.43.0" 2023-10-09T06:08:55.429 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:55.531 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:08:55.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:55.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:08:55.651 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:55.756 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:55.872 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:55.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:55.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:55.983 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:56.045 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:56.045 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:56 smithi064 ceph-mon[220044]: pgmap v1118: 119 pgs: 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 74 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 27228/284532 objects degraded (9.569%); 2966/284532 objects misplaced (1.042%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T06:08:56.046 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:56 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1197731857' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:08:56.046 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:56 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:08:56.100 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:08:56.201 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:56.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:56.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:56.315 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:56.358 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:56 smithi156 ceph-mon[195101]: pgmap v1118: 119 pgs: 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 74 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 27228/284532 objects degraded (9.569%); 2966/284532 objects misplaced (1.042%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T06:08:56.358 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:56 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1197731857' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:08:56.358 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:56 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:08:56.366 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:56.449 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:08:56.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:56 smithi064 ceph-mon[212750]: pgmap v1118: 119 pgs: 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 74 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 27228/284532 objects degraded (9.569%); 2966/284532 objects misplaced (1.042%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T06:08:56.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:56 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1197731857' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:08:56.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:56 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:08:56.566 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:56.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:56.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:56.676 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:08:56.737 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:56.815 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:08:56.815 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:08:56.816 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:08:56.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:56.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:08:56.927 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:08:56.978 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:56.979 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1, 2] out_osds: [] dead_osds: [0] live_osds: [1, 2, 3, 5, 6, 7, 4] 2023-10-09T06:08:56.979 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:08:56.980 INFO:tasks.thrashosds.thrasher:Reweighting osd 7 to 0.45426207032249655 2023-10-09T06:08:56.980 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 0.45426207032249655 2023-10-09T06:08:57.086 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:08:57.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:57.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:08:57.190 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:57.287 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:57.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:57 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:08:57.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:57 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:08:57.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:57 smithi156 ceph-mon[195101]: osdmap e448: 8 total, 7 up, 8 in 2023-10-09T06:08:57.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:57 smithi156 ceph-mon[195101]: failed to encode map e448 with expected crc 2023-10-09T06:08:57.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:57 smithi156 ceph-mon[195101]: failed to encode map e448 with expected crc 2023-10-09T06:08:57.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:57 smithi156 ceph-mon[195101]: failed to encode map e448 with expected crc 2023-10-09T06:08:57.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:57 smithi156 ceph-mon[195101]: failed to encode map e448 with expected crc 2023-10-09T06:08:57.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:57 smithi156 ceph-mon[195101]: failed to encode map e448 with expected crc 2023-10-09T06:08:57.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:57 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:08:57.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:57 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:08:57.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:57 smithi064 ceph-mon[220044]: osdmap e448: 8 total, 7 up, 8 in 2023-10-09T06:08:57.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:57 smithi064 ceph-mon[220044]: failed to encode map e448 with expected crc 2023-10-09T06:08:57.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:57 smithi064 ceph-mon[220044]: failed to encode map e448 with expected crc 2023-10-09T06:08:57.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:57 smithi064 ceph-mon[220044]: failed to encode map e448 with expected crc 2023-10-09T06:08:57.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:57 smithi064 ceph-mon[220044]: failed to encode map e448 with expected crc 2023-10-09T06:08:57.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:57 smithi064 ceph-mon[220044]: failed to encode map e448 with expected crc 2023-10-09T06:08:57.476 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:57 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:08:57.476 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:57 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:08:57.476 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:57 smithi064 ceph-mon[212750]: osdmap e448: 8 total, 7 up, 8 in 2023-10-09T06:08:57.476 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:57 smithi064 ceph-mon[212750]: failed to encode map e448 with expected crc 2023-10-09T06:08:57.476 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:57 smithi064 ceph-mon[212750]: failed to encode map e448 with expected crc 2023-10-09T06:08:57.476 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:57 smithi064 ceph-mon[212750]: failed to encode map e448 with expected crc 2023-10-09T06:08:57.477 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:57 smithi064 ceph-mon[212750]: failed to encode map e448 with expected crc 2023-10-09T06:08:57.477 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:57 smithi064 ceph-mon[212750]: failed to encode map e448 with expected crc 2023-10-09T06:08:57.483 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:57.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:57.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:57.597 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:57.638 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:57.698 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:57.764 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:08:57.771 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:08:57.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:57.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:08:57.883 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:57.926 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:08:57.974 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:58.079 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:58.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:58.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:58.193 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:08:58.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:58 smithi064 ceph-mon[212750]: pgmap v1120: 119 pgs: 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 74 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 944 op/s; 26697/275580 objects degraded (9.688%); 2728/275580 objects misplaced (0.990%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T06:08:58.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:58 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 26697/275580 objects degraded (9.688%), 13 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-10-09T06:08:58.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:58 smithi064 ceph-mon[220044]: pgmap v1120: 119 pgs: 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 74 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 944 op/s; 26697/275580 objects degraded (9.688%); 2728/275580 objects misplaced (0.990%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T06:08:58.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:58 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 26697/275580 objects degraded (9.688%), 13 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-10-09T06:08:58.267 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:58.300 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:08:58.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:58 smithi156 ceph-mon[195101]: pgmap v1120: 119 pgs: 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 74 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 944 op/s; 26697/275580 objects degraded (9.688%); 2728/275580 objects misplaced (0.990%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T06:08:58.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:58 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 26697/275580 objects degraded (9.688%), 13 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-10-09T06:08:58.895 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:08:58.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:58.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:08:59.005 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:08:59.071 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:59.486 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:08:59.703 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:08:59.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:08:59.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:08:59.813 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:08:59.849 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:59 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:08:59.849 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:08:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:59.851 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:08:59.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:59 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:08:59.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:08:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:59.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:59 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:08:59.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:08:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:00.025 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:09:00.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:00.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:09:00.130 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:09:00.170 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:00.245 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:09:00.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:00.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:09:00.291 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:09:00.291 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:09:00.350 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:09:00.423 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:00.573 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:09:00.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:00.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:09:00.684 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:09:00.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:00 smithi064 ceph-mon[212750]: pgmap v1121: 119 pgs: 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 74 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 26384/274641 objects degraded (9.607%); 2728/274641 objects misplaced (0.993%); 3.4 MiB/s, 55 objects/s recovering 2023-10-09T06:09:00.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:00 smithi064 ceph-mon[220044]: pgmap v1121: 119 pgs: 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 74 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 26384/274641 objects degraded (9.607%); 2728/274641 objects misplaced (0.993%); 3.4 MiB/s, 55 objects/s recovering 2023-10-09T06:09:00.761 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:00.800 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:00.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:00 smithi156 ceph-mon[195101]: pgmap v1121: 119 pgs: 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 74 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 26384/274641 objects degraded (9.607%); 2728/274641 objects misplaced (0.993%); 3.4 MiB/s, 55 objects/s recovering 2023-10-09T06:09:00.917 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:09:00.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:00.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:09:00.937 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:09:00.993 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:09:01.028 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:09:01.051 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:09:01.051 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:09:01.058 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:01.081 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:09:01.129 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:09:01.169 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:09:01.169 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-10-09T06:09:01.169 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:09:01.169 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:09:01.170 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:09:01.170 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:09:01.170 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:09:01.170 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:09:01.170 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:09:01.186 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:09:01.190 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:09:01.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:01.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:09:01.296 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:09:01.380 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:01.547 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:09:01.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:01.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:09:01.659 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:09:01.698 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:09:01.733 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:01.823 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:09:01.828 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:01 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/134419966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:09:01.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:01.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:09:01.930 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:09:01.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:01 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/134419966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:09:01.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:01 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/134419966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:09:01.972 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:02.016 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:09:02.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:02.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:09:02.119 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:09:02.189 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:02.568 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.7 to 0.454262 (744a) 2023-10-09T06:09:02.626 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:09:02.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:02.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:09:02.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:02 smithi064 ceph-mon[212750]: pgmap v1122: 119 pgs: 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 74 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 26103/267258 objects degraded (9.767%); 2488/267258 objects misplaced (0.931%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T06:09:02.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:02 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1007431409' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 0.45426207032249655}]: dispatch 2023-10-09T06:09:02.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:02 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 26103/267258 objects degraded (9.767%), 13 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:02.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:02 smithi064 ceph-mon[220044]: pgmap v1122: 119 pgs: 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 74 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 26103/267258 objects degraded (9.767%); 2488/267258 objects misplaced (0.931%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T06:09:02.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:02 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1007431409' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 0.45426207032249655}]: dispatch 2023-10-09T06:09:02.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:02 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 26103/267258 objects degraded (9.767%), 13 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:02.729 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:09:02.761 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:02.779 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T06:09:02.780 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T06:09:02.815 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:09:02.816 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:02 smithi156 ceph-mon[195101]: pgmap v1122: 119 pgs: 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 74 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 26103/267258 objects degraded (9.767%); 2488/267258 objects misplaced (0.931%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T06:09:02.817 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:02 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1007431409' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 0.45426207032249655}]: dispatch 2023-10-09T06:09:02.817 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:02 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 26103/267258 objects degraded (9.767%), 13 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:02.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:02.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:09:02.918 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:09:02.945 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:03.138 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:09:03.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:03.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:09:03.246 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:09:03.276 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:03.576 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:03 smithi064 ceph-mon[220044]: failed to encode map e449 with expected crc 2023-10-09T06:09:03.576 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:03 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1007431409' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 0.45426207032249655}]': finished 2023-10-09T06:09:03.577 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:03 smithi064 ceph-mon[220044]: osdmap e449: 8 total, 7 up, 8 in 2023-10-09T06:09:03.577 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:03 smithi064 ceph-mon[220044]: failed to encode map e449 with expected crc 2023-10-09T06:09:03.577 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:03 smithi064 ceph-mon[220044]: failed to encode map e449 with expected crc 2023-10-09T06:09:03.593 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:09:03.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:03.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:09:03.618 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:09:03.618 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:09:03.618 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T06:09:03.618 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T06:09:03.618 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:09:03.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:03 smithi156 ceph-mon[195101]: failed to encode map e449 with expected crc 2023-10-09T06:09:03.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:03 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1007431409' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 0.45426207032249655}]': finished 2023-10-09T06:09:03.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:03 smithi156 ceph-mon[195101]: osdmap e449: 8 total, 7 up, 8 in 2023-10-09T06:09:03.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:03 smithi156 ceph-mon[195101]: failed to encode map e449 with expected crc 2023-10-09T06:09:03.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:03 smithi156 ceph-mon[195101]: failed to encode map e449 with expected crc 2023-10-09T06:09:03.699 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:09:03.736 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:03.846 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:09:03.847 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:03 smithi064 ceph-mon[212750]: failed to encode map e449 with expected crc 2023-10-09T06:09:03.847 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:03 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1007431409' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 0.45426207032249655}]': finished 2023-10-09T06:09:03.847 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:03 smithi064 ceph-mon[212750]: osdmap e449: 8 total, 7 up, 8 in 2023-10-09T06:09:03.847 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:03 smithi064 ceph-mon[212750]: failed to encode map e449 with expected crc 2023-10-09T06:09:03.847 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:03 smithi064 ceph-mon[212750]: failed to encode map e449 with expected crc 2023-10-09T06:09:03.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:03.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:09:03.949 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:09:03.982 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:04.026 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:09:04.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:04.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:09:04.129 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:09:04.156 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:04.408 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:09:04.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:04.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:09:04.471 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:09:04.512 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:09:04.544 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:04.613 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:09:04.650 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:09:04.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:04.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:09:04.758 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:09:04.781 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:09:04.835 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:04 smithi156 ceph-mon[195101]: pgmap v1124: 119 pgs: 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 74 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 25597/262518 objects degraded (9.751%); 2488/262518 objects misplaced (0.948%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:09:04.835 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:04 smithi156 ceph-mon[195101]: failed to encode map e450 with expected crc 2023-10-09T06:09:04.835 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:04 smithi156 ceph-mon[195101]: failed to encode map e450 with expected crc 2023-10-09T06:09:04.835 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:04 smithi156 ceph-mon[195101]: failed to encode map e450 with expected crc 2023-10-09T06:09:04.835 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:04 smithi156 ceph-mon[195101]: failed to encode map e450 with expected crc 2023-10-09T06:09:04.836 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:04 smithi156 ceph-mon[195101]: failed to encode map e450 with expected crc 2023-10-09T06:09:04.836 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:04 smithi156 ceph-mon[195101]: osdmap e450: 8 total, 7 up, 8 in 2023-10-09T06:09:04.836 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:04 smithi156 ceph-mon[195101]: failed to encode map e450 with expected crc 2023-10-09T06:09:04.836 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:04 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1214425865' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:09:04.836 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:04 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:09:04.840 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:04.843 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:04 smithi064 ceph-mon[212750]: pgmap v1124: 119 pgs: 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 74 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 25597/262518 objects degraded (9.751%); 2488/262518 objects misplaced (0.948%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:09:04.843 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:04 smithi064 ceph-mon[212750]: failed to encode map e450 with expected crc 2023-10-09T06:09:04.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:04 smithi064 ceph-mon[212750]: failed to encode map e450 with expected crc 2023-10-09T06:09:04.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:04 smithi064 ceph-mon[212750]: failed to encode map e450 with expected crc 2023-10-09T06:09:04.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:04 smithi064 ceph-mon[212750]: failed to encode map e450 with expected crc 2023-10-09T06:09:04.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:04 smithi064 ceph-mon[212750]: failed to encode map e450 with expected crc 2023-10-09T06:09:04.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:04 smithi064 ceph-mon[212750]: osdmap e450: 8 total, 7 up, 8 in 2023-10-09T06:09:04.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:04 smithi064 ceph-mon[212750]: failed to encode map e450 with expected crc 2023-10-09T06:09:04.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:04 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1214425865' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:09:04.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:04 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:09:04.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:04 smithi064 ceph-mon[220044]: pgmap v1124: 119 pgs: 32 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 74 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 25597/262518 objects degraded (9.751%); 2488/262518 objects misplaced (0.948%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:09:04.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:04 smithi064 ceph-mon[220044]: failed to encode map e450 with expected crc 2023-10-09T06:09:04.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:04 smithi064 ceph-mon[220044]: failed to encode map e450 with expected crc 2023-10-09T06:09:04.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:04 smithi064 ceph-mon[220044]: failed to encode map e450 with expected crc 2023-10-09T06:09:04.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:04 smithi064 ceph-mon[220044]: failed to encode map e450 with expected crc 2023-10-09T06:09:04.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:04 smithi064 ceph-mon[220044]: failed to encode map e450 with expected crc 2023-10-09T06:09:04.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:04 smithi064 ceph-mon[220044]: osdmap e450: 8 total, 7 up, 8 in 2023-10-09T06:09:04.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:04 smithi064 ceph-mon[220044]: failed to encode map e450 with expected crc 2023-10-09T06:09:04.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:04 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1214425865' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:09:04.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:04 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:09:04.917 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:09:04.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:04.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:09:05.001 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:09:05.035 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:09:05.134 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:05.262 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:09:05.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:05.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:09:05.371 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:09:05.430 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:05.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:09:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:09:05] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.43.0" 2023-10-09T06:09:05.584 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:09:05.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:05.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:09:05.693 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:09:05.769 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:05.851 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:09:05.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:05.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:09:05.856 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:05 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:09:05.856 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:05 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:09:05.857 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:05 smithi156 ceph-mon[195101]: osdmap e451: 8 total, 7 up, 8 in 2023-10-09T06:09:05.857 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:05 smithi156 ceph-mon[195101]: failed to encode map e451 with expected crc 2023-10-09T06:09:05.857 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:05 smithi156 ceph-mon[195101]: failed to encode map e451 with expected crc 2023-10-09T06:09:05.857 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:05 smithi156 ceph-mon[195101]: failed to encode map e451 with expected crc 2023-10-09T06:09:05.857 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:05 smithi156 ceph-mon[195101]: failed to encode map e451 with expected crc 2023-10-09T06:09:05.858 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:05 smithi156 ceph-mon[195101]: failed to encode map e451 with expected crc 2023-10-09T06:09:05.858 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:05 smithi156 ceph-mon[195101]: failed to encode map e451 with expected crc 2023-10-09T06:09:05.858 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:05 smithi156 ceph-mon[195101]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 448'7231 2023-10-09T06:09:05.858 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:05 smithi156 ceph-mon[195101]: 9.9 starting backfill to osd.3 from (0'0,0'0] MAX to 448'7364 2023-10-09T06:09:05.858 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:05 smithi156 ceph-mon[195101]: 9.7 starting backfill to osd.2 from (0'0,0'0] MAX to 448'7223 2023-10-09T06:09:05.859 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:05 smithi156 ceph-mon[195101]: failed to encode map e451 with expected crc 2023-10-09T06:09:05.859 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:05 smithi156 ceph-mon[195101]: 9.5 starting backfill to osd.1 from (0'0,0'0] MAX to 448'7237 2023-10-09T06:09:05.859 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:05 smithi156 ceph-mon[195101]: 9.1 starting backfill to osd.4 from (0'0,0'0] MAX to 448'7273 2023-10-09T06:09:05.859 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:05 smithi156 ceph-mon[195101]: 9.3 starting backfill to osd.5 from (0'0,0'0] MAX to 448'7467 2023-10-09T06:09:05.956 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:09:05.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:09:05.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:09:05.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[212750]: osdmap e451: 8 total, 7 up, 8 in 2023-10-09T06:09:05.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[212750]: failed to encode map e451 with expected crc 2023-10-09T06:09:05.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[212750]: failed to encode map e451 with expected crc 2023-10-09T06:09:05.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[212750]: failed to encode map e451 with expected crc 2023-10-09T06:09:05.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[212750]: failed to encode map e451 with expected crc 2023-10-09T06:09:05.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[212750]: failed to encode map e451 with expected crc 2023-10-09T06:09:05.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[212750]: failed to encode map e451 with expected crc 2023-10-09T06:09:05.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[212750]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 448'7231 2023-10-09T06:09:05.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[212750]: 9.9 starting backfill to osd.3 from (0'0,0'0] MAX to 448'7364 2023-10-09T06:09:05.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[212750]: 9.7 starting backfill to osd.2 from (0'0,0'0] MAX to 448'7223 2023-10-09T06:09:05.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[212750]: failed to encode map e451 with expected crc 2023-10-09T06:09:05.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[212750]: 9.5 starting backfill to osd.1 from (0'0,0'0] MAX to 448'7237 2023-10-09T06:09:05.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[212750]: 9.1 starting backfill to osd.4 from (0'0,0'0] MAX to 448'7273 2023-10-09T06:09:05.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[212750]: 9.3 starting backfill to osd.5 from (0'0,0'0] MAX to 448'7467 2023-10-09T06:09:05.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:09:05.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:09:05.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[220044]: osdmap e451: 8 total, 7 up, 8 in 2023-10-09T06:09:05.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[220044]: failed to encode map e451 with expected crc 2023-10-09T06:09:05.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[220044]: failed to encode map e451 with expected crc 2023-10-09T06:09:05.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[220044]: failed to encode map e451 with expected crc 2023-10-09T06:09:05.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[220044]: failed to encode map e451 with expected crc 2023-10-09T06:09:05.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[220044]: failed to encode map e451 with expected crc 2023-10-09T06:09:05.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[220044]: failed to encode map e451 with expected crc 2023-10-09T06:09:05.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[220044]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 448'7231 2023-10-09T06:09:05.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[220044]: 9.9 starting backfill to osd.3 from (0'0,0'0] MAX to 448'7364 2023-10-09T06:09:05.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[220044]: 9.7 starting backfill to osd.2 from (0'0,0'0] MAX to 448'7223 2023-10-09T06:09:05.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[220044]: failed to encode map e451 with expected crc 2023-10-09T06:09:05.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[220044]: 9.5 starting backfill to osd.1 from (0'0,0'0] MAX to 448'7237 2023-10-09T06:09:05.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[220044]: 9.1 starting backfill to osd.4 from (0'0,0'0] MAX to 448'7273 2023-10-09T06:09:05.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:05 smithi064 ceph-mon[220044]: 9.3 starting backfill to osd.5 from (0'0,0'0] MAX to 448'7467 2023-10-09T06:09:06.008 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:06.074 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:09:06.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:06.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:09:06.133 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:06.180 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:09:06.226 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:06.568 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:09:06.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:06.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:09:06.679 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:09:06.729 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:06.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:06 smithi064 ceph-mon[212750]: pgmap v1127: 119 pgs: 3 remapped+peering, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+undersized+degraded, 72 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 9.2 KiB/s rd, 24 KiB/s wr, 719 op/s; 25361/261813 objects degraded (9.687%); 7871/261813 objects misplaced (3.006%); 2.4 MiB/s, 36 objects/s recovering 2023-10-09T06:09:06.868 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:06 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:09:06.868 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:06 smithi064 ceph-mon[212750]: failed to encode map e452 with expected crc 2023-10-09T06:09:06.868 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:06 smithi064 ceph-mon[212750]: osdmap e452: 8 total, 7 up, 8 in 2023-10-09T06:09:06.868 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:06 smithi064 ceph-mon[212750]: failed to encode map e452 with expected crc 2023-10-09T06:09:06.868 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:06 smithi064 ceph-mon[212750]: failed to encode map e452 with expected crc 2023-10-09T06:09:06.868 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:06 smithi064 ceph-mon[212750]: failed to encode map e452 with expected crc 2023-10-09T06:09:06.868 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:06 smithi064 ceph-mon[220044]: pgmap v1127: 119 pgs: 3 remapped+peering, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+undersized+degraded, 72 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 9.2 KiB/s rd, 24 KiB/s wr, 719 op/s; 25361/261813 objects degraded (9.687%); 7871/261813 objects misplaced (3.006%); 2.4 MiB/s, 36 objects/s recovering 2023-10-09T06:09:06.869 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:06 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:09:06.869 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:06 smithi064 ceph-mon[220044]: failed to encode map e452 with expected crc 2023-10-09T06:09:06.869 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:06 smithi064 ceph-mon[220044]: osdmap e452: 8 total, 7 up, 8 in 2023-10-09T06:09:06.869 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:06 smithi064 ceph-mon[220044]: failed to encode map e452 with expected crc 2023-10-09T06:09:06.869 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:06 smithi064 ceph-mon[220044]: failed to encode map e452 with expected crc 2023-10-09T06:09:06.870 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:06 smithi064 ceph-mon[220044]: failed to encode map e452 with expected crc 2023-10-09T06:09:06.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:06 smithi156 ceph-mon[195101]: pgmap v1127: 119 pgs: 3 remapped+peering, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+undersized+degraded, 72 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 9.2 KiB/s rd, 24 KiB/s wr, 719 op/s; 25361/261813 objects degraded (9.687%); 7871/261813 objects misplaced (3.006%); 2.4 MiB/s, 36 objects/s recovering 2023-10-09T06:09:06.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:06 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:09:06.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:06 smithi156 ceph-mon[195101]: failed to encode map e452 with expected crc 2023-10-09T06:09:06.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:06 smithi156 ceph-mon[195101]: osdmap e452: 8 total, 7 up, 8 in 2023-10-09T06:09:06.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:06 smithi156 ceph-mon[195101]: failed to encode map e452 with expected crc 2023-10-09T06:09:06.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:06 smithi156 ceph-mon[195101]: failed to encode map e452 with expected crc 2023-10-09T06:09:06.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:06 smithi156 ceph-mon[195101]: failed to encode map e452 with expected crc 2023-10-09T06:09:06.882 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:09:06.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:06.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:09:06.991 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:09:07.045 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:07.147 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:09:07.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:07.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:09:07.215 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:09:07.215 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T06:09:07.215 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T06:09:07.215 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:09:07.215 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.215 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:134383 8.4 8:22d7e2f9:::benchmark_data_smithi064_457003_object29865:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2023-10-09T06:09:07.215 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:10.426661+0000", 2023-10-09T06:09:07.215 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 236.783928661, 2023-10-09T06:09:07.215 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1812402510000002, 2023-10-09T06:09:07.215 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:09:07.216 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:09:07.216 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:09:07.216 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:09:07.216 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:09:07.216 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 134383 2023-10-09T06:09:07.216 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.216 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:09:07.216 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.216 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:09:07.216 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.426661+0000", 2023-10-09T06:09:07.216 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.216 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.217 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.217 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:09:07.217 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.426661+0000", 2023-10-09T06:09:07.217 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.217 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.217 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.217 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:09:07.217 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.426660+0000", 2023-10-09T06:09:07.217 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:09:07.217 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.217 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.218 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:09:07.218 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.426664+0000", 2023-10-09T06:09:07.218 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3610000000000003e-06 2023-10-09T06:09:07.218 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.218 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.218 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:09:07.218 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.426676+0000", 2023-10-09T06:09:07.218 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1798e-05 2023-10-09T06:09:07.218 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.218 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.218 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:09:07.218 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.426680+0000", 2023-10-09T06:09:07.219 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2010000000000001e-06 2023-10-09T06:09:07.219 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.219 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.219 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.219 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.426731+0000", 2023-10-09T06:09:07.220 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0767999999999999e-05 2023-10-09T06:09:07.220 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.220 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.220 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:09:07.220 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.426864+0000", 2023-10-09T06:09:07.220 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013331000000000001 2023-10-09T06:09:07.220 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.221 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.221 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:09:07.221 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.427008+0000", 2023-10-09T06:09:07.221 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014342700000000001 2023-10-09T06:09:07.221 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.221 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.221 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.222 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.436794+0000", 2023-10-09T06:09:07.222 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0097867189999999993 2023-10-09T06:09:07.222 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.222 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.222 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.223 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.436805+0000", 2023-10-09T06:09:07.223 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0703999999999999e-05 2023-10-09T06:09:07.223 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.223 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.223 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.223 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:11.450433+0000", 2023-10-09T06:09:07.223 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.01362781 2023-10-09T06:09:07.224 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.224 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.224 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.224 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:11.450445+0000", 2023-10-09T06:09:07.224 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1657999999999999e-05 2023-10-09T06:09:07.224 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.224 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.224 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.224 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.607824+0000", 2023-10-09T06:09:07.225 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1573795309999999 2023-10-09T06:09:07.225 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.225 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.225 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:09:07.225 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.607901+0000", 2023-10-09T06:09:07.225 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6910999999999996e-05 2023-10-09T06:09:07.225 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.225 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:09:07.225 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.225 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.225 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.226 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:134384 8.4 8:2434c918:::benchmark_data_smithi064_457003_object29866:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2023-10-09T06:09:07.226 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:10.429188+0000", 2023-10-09T06:09:07.226 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 236.78140091200001, 2023-10-09T06:09:07.226 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1788163570000001, 2023-10-09T06:09:07.226 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:09:07.226 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:09:07.226 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:09:07.226 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:09:07.226 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:09:07.226 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 134384 2023-10-09T06:09:07.227 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.227 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:09:07.227 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.227 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:09:07.227 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.429188+0000", 2023-10-09T06:09:07.227 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.227 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.227 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.227 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:09:07.227 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.429188+0000", 2023-10-09T06:09:07.227 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.227 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.228 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.228 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:09:07.228 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.429187+0000", 2023-10-09T06:09:07.228 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:09:07.228 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.228 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.229 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:09:07.229 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.429193+0000", 2023-10-09T06:09:07.229 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.049e-06 2023-10-09T06:09:07.229 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.229 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.229 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:09:07.229 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.429207+0000", 2023-10-09T06:09:07.229 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4011e-05 2023-10-09T06:09:07.229 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.230 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.230 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:09:07.230 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.429216+0000", 2023-10-09T06:09:07.230 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8629999999999997e-06 2023-10-09T06:09:07.230 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.230 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.230 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.230 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.429661+0000", 2023-10-09T06:09:07.230 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00044492400000000002 2023-10-09T06:09:07.230 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.231 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.231 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:09:07.231 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.429777+0000", 2023-10-09T06:09:07.231 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000116107 2023-10-09T06:09:07.231 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.231 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.231 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:09:07.231 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.429856+0000", 2023-10-09T06:09:07.231 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8425999999999998e-05 2023-10-09T06:09:07.231 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.231 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.231 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.232 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.436827+0000", 2023-10-09T06:09:07.232 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0069719680000000003 2023-10-09T06:09:07.232 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.232 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.232 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.232 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.436833+0000", 2023-10-09T06:09:07.232 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6529999999999999e-06 2023-10-09T06:09:07.232 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.232 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.232 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.233 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:11.450477+0000", 2023-10-09T06:09:07.233 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0136439020000001 2023-10-09T06:09:07.233 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.233 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.233 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.233 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:11.450484+0000", 2023-10-09T06:09:07.233 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3440000000000003e-06 2023-10-09T06:09:07.233 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.233 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.233 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.233 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.607929+0000", 2023-10-09T06:09:07.233 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1574441870000001 2023-10-09T06:09:07.234 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.234 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.234 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:09:07.234 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.608005+0000", 2023-10-09T06:09:07.234 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6285000000000006e-05 2023-10-09T06:09:07.234 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.234 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:09:07.234 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.234 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.234 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.234 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:134394 8.4 8:2f57916d:::benchmark_data_smithi064_457003_object29876:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-10-09T06:09:07.235 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:10.577118+0000", 2023-10-09T06:09:07.235 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 236.63347109, 2023-10-09T06:09:07.235 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.121318697, 2023-10-09T06:09:07.235 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:09:07.235 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:09:07.235 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:09:07.235 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:09:07.235 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:09:07.235 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 134394 2023-10-09T06:09:07.235 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.235 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:09:07.235 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.236 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:09:07.236 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.577118+0000", 2023-10-09T06:09:07.236 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.236 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.236 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.236 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:09:07.237 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.577118+0000", 2023-10-09T06:09:07.237 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.237 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.237 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.237 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:09:07.238 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.577117+0000", 2023-10-09T06:09:07.238 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T06:09:07.238 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.238 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.238 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:09:07.238 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.577123+0000", 2023-10-09T06:09:07.238 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1929999999999998e-06 2023-10-09T06:09:07.238 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.238 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.239 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:09:07.239 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.577140+0000", 2023-10-09T06:09:07.239 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7090000000000001e-05 2023-10-09T06:09:07.239 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.241 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.241 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:09:07.241 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.577147+0000", 2023-10-09T06:09:07.242 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2280000000000001e-06 2023-10-09T06:09:07.242 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.242 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.242 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.242 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.577209+0000", 2023-10-09T06:09:07.242 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1564999999999998e-05 2023-10-09T06:09:07.243 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.243 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.243 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.243 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.577218+0000", 2023-10-09T06:09:07.243 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0529999999999996e-06 2023-10-09T06:09:07.243 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.243 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.244 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.244 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:11.450510+0000", 2023-10-09T06:09:07.244 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.87329198299999999 2023-10-09T06:09:07.244 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.244 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.244 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.245 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:11.452323+0000", 2023-10-09T06:09:07.245 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001812917 2023-10-09T06:09:07.245 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.245 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.245 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.245 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.608037+0000", 2023-10-09T06:09:07.246 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1557139999999999 2023-10-09T06:09:07.246 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.246 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.246 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:09:07.246 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.608112+0000", 2023-10-09T06:09:07.246 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5640000000000001e-05 2023-10-09T06:09:07.246 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.246 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.246 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:09:07.246 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.608577+0000", 2023-10-09T06:09:07.246 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00046510900000000001 2023-10-09T06:09:07.247 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.247 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.247 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:09:07.247 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.616852+0000", 2023-10-09T06:09:07.247 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0082743859999999999 2023-10-09T06:09:07.247 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.247 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.247 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.247 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.616922+0000", 2023-10-09T06:09:07.247 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0260999999999997e-05 2023-10-09T06:09:07.247 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.247 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.247 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.248 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.616961+0000", 2023-10-09T06:09:07.248 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9382999999999999e-05 2023-10-09T06:09:07.248 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.248 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.248 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.248 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.698270+0000", 2023-10-09T06:09:07.248 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.081308510000000001 2023-10-09T06:09:07.248 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.248 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.248 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:09:07.248 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.698319+0000", 2023-10-09T06:09:07.248 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9443000000000001e-05 2023-10-09T06:09:07.249 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.249 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.249 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:09:07.249 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.698437+0000", 2023-10-09T06:09:07.249 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011770800000000001 2023-10-09T06:09:07.249 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.249 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:09:07.249 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.249 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.249 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.249 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:134403 8.4 8:2a26dc4b:::benchmark_data_smithi064_457003_object29885:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-10-09T06:09:07.249 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:10.590324+0000", 2023-10-09T06:09:07.249 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 236.62026547599999, 2023-10-09T06:09:07.250 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1145394660000001, 2023-10-09T06:09:07.250 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:09:07.250 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:09:07.250 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:09:07.250 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:09:07.250 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:09:07.250 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 134403 2023-10-09T06:09:07.250 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.250 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:09:07.250 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.250 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:09:07.251 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590324+0000", 2023-10-09T06:09:07.251 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.251 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.251 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.251 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:09:07.251 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590324+0000", 2023-10-09T06:09:07.251 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.251 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.251 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.252 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:09:07.252 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590322+0000", 2023-10-09T06:09:07.252 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T06:09:07.252 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.252 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.252 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:09:07.252 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590331+0000", 2023-10-09T06:09:07.252 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2590000000000006e-06 2023-10-09T06:09:07.252 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.252 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.252 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:09:07.253 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590346+0000", 2023-10-09T06:09:07.253 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4663e-05 2023-10-09T06:09:07.253 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.253 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.253 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:09:07.253 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590354+0000", 2023-10-09T06:09:07.253 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6939999999999995e-06 2023-10-09T06:09:07.253 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.253 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.253 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.253 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590417+0000", 2023-10-09T06:09:07.254 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3152999999999997e-05 2023-10-09T06:09:07.254 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.254 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.254 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.254 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590430+0000", 2023-10-09T06:09:07.254 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.288e-05 2023-10-09T06:09:07.254 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.254 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.254 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.254 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:11.452416+0000", 2023-10-09T06:09:07.254 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.86198688400000001 2023-10-09T06:09:07.255 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.255 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.255 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.255 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:11.452427+0000", 2023-10-09T06:09:07.255 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0414999999999999e-05 2023-10-09T06:09:07.255 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.255 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.255 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.255 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.613291+0000", 2023-10-09T06:09:07.255 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.160863693 2023-10-09T06:09:07.255 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.256 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.256 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:09:07.256 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.613436+0000", 2023-10-09T06:09:07.256 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014545000000000001 2023-10-09T06:09:07.256 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.256 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.256 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:09:07.256 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.613593+0000", 2023-10-09T06:09:07.256 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015705600000000001 2023-10-09T06:09:07.256 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.256 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.257 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.257 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.620497+0000", 2023-10-09T06:09:07.257 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0069035349999999997 2023-10-09T06:09:07.257 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.257 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.257 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.257 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.620582+0000", 2023-10-09T06:09:07.257 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.5128000000000002e-05 2023-10-09T06:09:07.257 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.257 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.258 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:09:07.258 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.623088+0000", 2023-10-09T06:09:07.258 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0025060540000000002 2023-10-09T06:09:07.258 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.258 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.258 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.258 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.704684+0000", 2023-10-09T06:09:07.258 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.081596740000000001 2023-10-09T06:09:07.258 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.258 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.259 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:09:07.259 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.704743+0000", 2023-10-09T06:09:07.259 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8674000000000001e-05 2023-10-09T06:09:07.259 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.259 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.259 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:09:07.259 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.704863+0000", 2023-10-09T06:09:07.260 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000120219 2023-10-09T06:09:07.260 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.260 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:09:07.260 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.260 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.261 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.261 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149154 8.4 8:20798e00:::benchmark_data_smithi064_457003_object44636:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e391)", 2023-10-09T06:09:07.261 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.422228+0000", 2023-10-09T06:09:07.261 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 212.78836106200001, 2023-10-09T06:09:07.261 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.034537332, 2023-10-09T06:09:07.261 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:09:07.261 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:09:07.261 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:09:07.261 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:09:07.262 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:09:07.262 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149154 2023-10-09T06:09:07.262 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.262 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:09:07.262 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.262 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:09:07.262 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.422228+0000", 2023-10-09T06:09:07.262 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.263 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.263 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.263 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:09:07.263 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.422228+0000", 2023-10-09T06:09:07.263 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.263 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.263 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.263 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:09:07.264 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.422227+0000", 2023-10-09T06:09:07.264 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:09:07.264 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.264 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.264 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:09:07.264 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.422234+0000", 2023-10-09T06:09:07.264 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0060000000000003e-06 2023-10-09T06:09:07.265 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.265 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.265 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:09:07.265 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.422247+0000", 2023-10-09T06:09:07.265 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2954e-05 2023-10-09T06:09:07.265 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.265 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.265 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:09:07.265 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.422253+0000", 2023-10-09T06:09:07.266 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5860000000000001e-06 2023-10-09T06:09:07.266 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.266 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.266 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.266 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.422301+0000", 2023-10-09T06:09:07.266 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7951000000000002e-05 2023-10-09T06:09:07.267 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.267 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.267 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:09:07.267 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.422419+0000", 2023-10-09T06:09:07.267 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000118121 2023-10-09T06:09:07.267 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.267 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.267 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:09:07.267 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.422533+0000", 2023-10-09T06:09:07.267 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011380899999999999 2023-10-09T06:09:07.267 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.267 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.268 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.268 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.428222+0000", 2023-10-09T06:09:07.268 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0056886109999999997 2023-10-09T06:09:07.268 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.268 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.268 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.268 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.430128+0000", 2023-10-09T06:09:07.268 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0019062689999999999 2023-10-09T06:09:07.268 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.268 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.268 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:09:07.269 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.436483+0000", 2023-10-09T06:09:07.269 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0063544279999999996 2023-10-09T06:09:07.269 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.269 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.269 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.269 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.442211+0000", 2023-10-09T06:09:07.269 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0057284190000000002 2023-10-09T06:09:07.269 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.269 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.269 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.269 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.442220+0000", 2023-10-09T06:09:07.270 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3160000000000003e-06 2023-10-09T06:09:07.270 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.270 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.270 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.270 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.456686+0000", 2023-10-09T06:09:07.270 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.014465739 2023-10-09T06:09:07.270 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.270 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.270 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:09:07.270 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.456766+0000", 2023-10-09T06:09:07.270 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9617999999999996e-05 2023-10-09T06:09:07.271 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.271 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:09:07.271 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.271 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.271 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.271 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149161 8.4 8:2898f062:::benchmark_data_smithi064_457003_object44643:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:09:07.271 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.439170+0000", 2023-10-09T06:09:07.271 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 212.77141947699999, 2023-10-09T06:09:07.271 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.061805517, 2023-10-09T06:09:07.271 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:09:07.271 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:09:07.272 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:09:07.272 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:09:07.272 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:09:07.272 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149161 2023-10-09T06:09:07.272 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.272 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:09:07.272 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.272 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:09:07.272 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.439170+0000", 2023-10-09T06:09:07.272 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.273 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.273 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.273 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:09:07.273 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.439170+0000", 2023-10-09T06:09:07.273 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.273 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.273 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.273 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:09:07.273 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.439168+0000", 2023-10-09T06:09:07.273 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:09:07.274 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.274 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.274 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:09:07.274 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.439179+0000", 2023-10-09T06:09:07.274 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0719e-05 2023-10-09T06:09:07.274 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.274 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.274 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:09:07.274 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.439192+0000", 2023-10-09T06:09:07.274 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2734e-05 2023-10-09T06:09:07.275 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.275 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.275 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:09:07.275 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.439199+0000", 2023-10-09T06:09:07.275 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5570000000000001e-06 2023-10-09T06:09:07.275 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.275 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.275 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.275 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.442783+0000", 2023-10-09T06:09:07.275 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0035848360000000001 2023-10-09T06:09:07.275 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.276 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.276 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.276 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.442787+0000", 2023-10-09T06:09:07.276 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0080000000000004e-06 2023-10-09T06:09:07.276 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.276 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.276 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.276 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.456792+0000", 2023-10-09T06:09:07.276 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.014004393 2023-10-09T06:09:07.276 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.276 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.277 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:09:07.277 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.456908+0000", 2023-10-09T06:09:07.277 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000116222 2023-10-09T06:09:07.277 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.277 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.277 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:09:07.277 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.457059+0000", 2023-10-09T06:09:07.277 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000150543 2023-10-09T06:09:07.277 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.277 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.278 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:09:07.278 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.468559+0000", 2023-10-09T06:09:07.278 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011500836 2023-10-09T06:09:07.278 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.278 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.278 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.278 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.500849+0000", 2023-10-09T06:09:07.279 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.032290039999999999 2023-10-09T06:09:07.279 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.279 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.279 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.279 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.500886+0000", 2023-10-09T06:09:07.280 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6297999999999997e-05 2023-10-09T06:09:07.280 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.280 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.280 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:09:07.280 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.500935+0000", 2023-10-09T06:09:07.280 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9323999999999997e-05 2023-10-09T06:09:07.280 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.280 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.281 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:09:07.281 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.500975+0000", 2023-10-09T06:09:07.281 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0432999999999997e-05 2023-10-09T06:09:07.281 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.281 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:09:07.281 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.281 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.281 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.281 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149163 8.4 8:2aaf630e:::benchmark_data_smithi064_457003_object44645:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:09:07.281 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.439209+0000", 2023-10-09T06:09:07.281 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 212.77138063999999, 2023-10-09T06:09:07.282 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.061909999, 2023-10-09T06:09:07.282 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:09:07.282 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:09:07.282 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:09:07.282 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:09:07.282 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:09:07.282 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149163 2023-10-09T06:09:07.282 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.282 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:09:07.282 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.282 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:09:07.283 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.439209+0000", 2023-10-09T06:09:07.283 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.283 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.283 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.283 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:09:07.283 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.439209+0000", 2023-10-09T06:09:07.283 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.283 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.283 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.283 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:09:07.283 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.439208+0000", 2023-10-09T06:09:07.283 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:09:07.283 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.284 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.284 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:09:07.284 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.439211+0000", 2023-10-09T06:09:07.284 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7379999999999999e-06 2023-10-09T06:09:07.284 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.284 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.284 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:09:07.284 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.439222+0000", 2023-10-09T06:09:07.284 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0821e-05 2023-10-09T06:09:07.284 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.284 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.284 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:09:07.285 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.439226+0000", 2023-10-09T06:09:07.285 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7809999999999999e-06 2023-10-09T06:09:07.285 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.285 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.285 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.285 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.445449+0000", 2023-10-09T06:09:07.285 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0062232609999999999 2023-10-09T06:09:07.285 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.285 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.286 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.286 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.445458+0000", 2023-10-09T06:09:07.286 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6109999999999994e-06 2023-10-09T06:09:07.286 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.286 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.286 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.286 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.459701+0000", 2023-10-09T06:09:07.286 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0142438330000001 2023-10-09T06:09:07.286 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.286 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.286 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:09:07.287 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.459837+0000", 2023-10-09T06:09:07.287 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000135365 2023-10-09T06:09:07.287 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.287 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.287 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:09:07.287 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.459967+0000", 2023-10-09T06:09:07.287 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012998099999999999 2023-10-09T06:09:07.287 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.287 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.287 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:09:07.287 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.483196+0000", 2023-10-09T06:09:07.288 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.02322886 2023-10-09T06:09:07.288 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.288 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.288 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.288 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501029+0000", 2023-10-09T06:09:07.288 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.017833762999999999 2023-10-09T06:09:07.288 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.288 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.288 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.288 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501057+0000", 2023-10-09T06:09:07.288 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.8053999999999999e-05 2023-10-09T06:09:07.289 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.289 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.289 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:09:07.289 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501085+0000", 2023-10-09T06:09:07.289 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7274e-05 2023-10-09T06:09:07.289 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.289 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.289 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:09:07.289 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501119+0000", 2023-10-09T06:09:07.289 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.4059e-05 2023-10-09T06:09:07.289 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.289 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:09:07.290 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.290 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.290 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.290 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149178 8.4 8:2eb2dbb9:::benchmark_data_smithi064_457003_object44660:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:09:07.290 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.449151+0000", 2023-10-09T06:09:07.290 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 212.761438252, 2023-10-09T06:09:07.290 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0521200980000001, 2023-10-09T06:09:07.290 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:09:07.290 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:09:07.290 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:09:07.290 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:09:07.290 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:09:07.291 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149178 2023-10-09T06:09:07.291 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.291 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:09:07.291 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.291 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:09:07.291 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.449151+0000", 2023-10-09T06:09:07.291 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.291 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.291 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.291 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:09:07.292 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.449151+0000", 2023-10-09T06:09:07.292 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.292 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.292 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.292 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:09:07.292 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.449150+0000", 2023-10-09T06:09:07.292 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:09:07.292 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.292 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.292 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:09:07.292 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.449156+0000", 2023-10-09T06:09:07.293 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3999999999999997e-06 2023-10-09T06:09:07.293 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.293 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.293 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:09:07.293 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.449168+0000", 2023-10-09T06:09:07.293 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1606999999999999e-05 2023-10-09T06:09:07.293 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.293 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.293 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:09:07.293 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.449174+0000", 2023-10-09T06:09:07.293 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4470000000000003e-06 2023-10-09T06:09:07.293 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.293 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.294 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.294 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.449221+0000", 2023-10-09T06:09:07.294 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6761999999999997e-05 2023-10-09T06:09:07.294 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.294 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.294 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.294 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.449228+0000", 2023-10-09T06:09:07.294 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4900000000000003e-06 2023-10-09T06:09:07.294 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.294 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.294 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.295 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.462573+0000", 2023-10-09T06:09:07.295 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.013344947 2023-10-09T06:09:07.295 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.295 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.295 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:09:07.295 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.462663+0000", 2023-10-09T06:09:07.295 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9839999999999994e-05 2023-10-09T06:09:07.295 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.295 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.295 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:09:07.295 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.462739+0000", 2023-10-09T06:09:07.296 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5459999999999999e-05 2023-10-09T06:09:07.296 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.296 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.296 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:09:07.296 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.483201+0000", 2023-10-09T06:09:07.296 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.020462620000000001 2023-10-09T06:09:07.296 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.296 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.296 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.296 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501192+0000", 2023-10-09T06:09:07.296 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.017991333000000002 2023-10-09T06:09:07.297 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.297 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.297 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.297 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501220+0000", 2023-10-09T06:09:07.297 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7775000000000001e-05 2023-10-09T06:09:07.297 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.297 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.297 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:09:07.297 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501244+0000", 2023-10-09T06:09:07.297 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3858000000000001e-05 2023-10-09T06:09:07.297 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.297 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.298 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:09:07.298 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501271+0000", 2023-10-09T06:09:07.298 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7115e-05 2023-10-09T06:09:07.298 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.298 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:09:07.298 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.298 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.298 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.298 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149189 8.4 8:2633a930:::benchmark_data_smithi064_457003_object44671:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:09:07.298 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.470034+0000", 2023-10-09T06:09:07.299 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 212.740555223, 2023-10-09T06:09:07.299 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.03132147, 2023-10-09T06:09:07.299 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:09:07.299 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:09:07.299 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:09:07.299 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:09:07.299 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:09:07.299 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149189 2023-10-09T06:09:07.299 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.299 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:09:07.299 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.299 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:09:07.300 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470034+0000", 2023-10-09T06:09:07.300 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.300 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.300 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.300 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:09:07.300 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470034+0000", 2023-10-09T06:09:07.300 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.300 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.300 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.300 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:09:07.300 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470034+0000", 2023-10-09T06:09:07.300 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:09:07.301 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.301 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.301 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:09:07.301 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470037+0000", 2023-10-09T06:09:07.301 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7799999999999998e-06 2023-10-09T06:09:07.301 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.301 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.301 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:09:07.301 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470045+0000", 2023-10-09T06:09:07.301 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5689999999999999e-06 2023-10-09T06:09:07.301 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.302 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.302 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:09:07.302 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470050+0000", 2023-10-09T06:09:07.302 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0599999999999998e-06 2023-10-09T06:09:07.302 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.302 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.302 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.302 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470099+0000", 2023-10-09T06:09:07.302 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9329000000000002e-05 2023-10-09T06:09:07.302 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.302 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.303 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.303 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470108+0000", 2023-10-09T06:09:07.303 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.5180000000000004e-06 2023-10-09T06:09:07.303 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.303 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.303 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.303 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.465423+0000", 2023-10-09T06:09:07.303 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.995315328 2023-10-09T06:09:07.303 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.303 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.303 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:09:07.304 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.465549+0000", 2023-10-09T06:09:07.304 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012567899999999999 2023-10-09T06:09:07.304 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.304 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.304 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:09:07.304 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.465702+0000", 2023-10-09T06:09:07.304 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000153401 2023-10-09T06:09:07.304 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.304 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.304 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:09:07.304 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.485892+0000", 2023-10-09T06:09:07.304 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.020189628000000001 2023-10-09T06:09:07.305 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.305 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.305 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.305 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501297+0000", 2023-10-09T06:09:07.305 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015405106999999999 2023-10-09T06:09:07.305 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.305 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.305 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.305 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501314+0000", 2023-10-09T06:09:07.306 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6880000000000001e-05 2023-10-09T06:09:07.306 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.306 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.306 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:09:07.306 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501337+0000", 2023-10-09T06:09:07.306 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3289999999999999e-05 2023-10-09T06:09:07.306 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.306 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.306 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:09:07.306 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501356+0000", 2023-10-09T06:09:07.306 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8420999999999999e-05 2023-10-09T06:09:07.306 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.307 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:09:07.307 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.307 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.307 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.307 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149190 8.4 8:2b47ddf4:::benchmark_data_smithi064_457003_object44672:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:09:07.307 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.470066+0000", 2023-10-09T06:09:07.307 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 212.74052302699999, 2023-10-09T06:09:07.307 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.031372773, 2023-10-09T06:09:07.307 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:09:07.307 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:09:07.307 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:09:07.308 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:09:07.308 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:09:07.308 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149190 2023-10-09T06:09:07.308 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.308 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:09:07.308 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.308 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:09:07.308 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470066+0000", 2023-10-09T06:09:07.308 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.308 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.308 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.309 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:09:07.309 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470066+0000", 2023-10-09T06:09:07.309 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.309 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.309 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.309 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:09:07.309 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470066+0000", 2023-10-09T06:09:07.309 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:09:07.310 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.310 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.310 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:09:07.310 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470068+0000", 2023-10-09T06:09:07.310 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2129999999999998e-06 2023-10-09T06:09:07.310 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.310 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.310 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:09:07.310 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470073+0000", 2023-10-09T06:09:07.311 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0340000000000004e-06 2023-10-09T06:09:07.311 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.311 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.311 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:09:07.311 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470076+0000", 2023-10-09T06:09:07.311 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7779999999999999e-06 2023-10-09T06:09:07.311 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.311 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.311 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.312 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470127+0000", 2023-10-09T06:09:07.312 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1140000000000002e-05 2023-10-09T06:09:07.312 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.312 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.312 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.312 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.470132+0000", 2023-10-09T06:09:07.312 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.028e-06 2023-10-09T06:09:07.313 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.313 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.313 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.313 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.468594+0000", 2023-10-09T06:09:07.313 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99846229600000003 2023-10-09T06:09:07.313 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.313 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.313 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:09:07.314 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.468669+0000", 2023-10-09T06:09:07.314 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4534000000000002e-05 2023-10-09T06:09:07.314 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.314 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.314 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:09:07.314 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.468766+0000", 2023-10-09T06:09:07.314 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7283999999999999e-05 2023-10-09T06:09:07.315 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.315 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.315 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:09:07.315 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.485898+0000", 2023-10-09T06:09:07.315 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.017131825 2023-10-09T06:09:07.315 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.315 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.316 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.316 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501375+0000", 2023-10-09T06:09:07.316 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.01547684 2023-10-09T06:09:07.316 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.316 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.316 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.316 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501397+0000", 2023-10-09T06:09:07.316 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2228000000000001e-05 2023-10-09T06:09:07.317 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.317 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.317 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:09:07.317 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501421+0000", 2023-10-09T06:09:07.317 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3438000000000002e-05 2023-10-09T06:09:07.317 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.318 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.318 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:09:07.318 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501439+0000", 2023-10-09T06:09:07.318 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8499e-05 2023-10-09T06:09:07.318 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.318 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:09:07.319 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.319 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.319 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149203 8.4 8:2695d781:::benchmark_data_smithi064_457003_object44685:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:09:07.319 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.485058+0000", 2023-10-09T06:09:07.319 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 212.72553161299999, 2023-10-09T06:09:07.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.017548871, 2023-10-09T06:09:07.319 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:09:07.320 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:09:07.320 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:09:07.320 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:09:07.320 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:09:07.320 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149203 2023-10-09T06:09:07.320 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.320 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:09:07.321 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.321 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:09:07.321 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.485058+0000", 2023-10-09T06:09:07.321 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.322 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.322 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.322 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:09:07.322 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.485058+0000", 2023-10-09T06:09:07.322 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.322 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.323 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.323 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:09:07.323 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.485056+0000", 2023-10-09T06:09:07.323 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:09:07.323 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.323 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.323 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:09:07.323 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.485063+0000", 2023-10-09T06:09:07.324 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8580000000000002e-06 2023-10-09T06:09:07.324 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.324 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.324 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:09:07.324 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.485077+0000", 2023-10-09T06:09:07.324 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3822000000000001e-05 2023-10-09T06:09:07.324 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.324 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.324 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:09:07.324 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.485084+0000", 2023-10-09T06:09:07.325 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5849999999999996e-06 2023-10-09T06:09:07.325 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.325 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.325 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.325 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.485169+0000", 2023-10-09T06:09:07.325 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.5824999999999999e-05 2023-10-09T06:09:07.325 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.325 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.325 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.325 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.485178+0000", 2023-10-09T06:09:07.325 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1769999999999998e-06 2023-10-09T06:09:07.326 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.326 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.326 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.326 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.471219+0000", 2023-10-09T06:09:07.326 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.98604176700000001 2023-10-09T06:09:07.326 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.326 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.326 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:09:07.326 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.471328+0000", 2023-10-09T06:09:07.326 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000108862 2023-10-09T06:09:07.327 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.327 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.327 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:09:07.327 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.471447+0000", 2023-10-09T06:09:07.327 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000118905 2023-10-09T06:09:07.327 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.327 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.328 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:09:07.328 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.485903+0000", 2023-10-09T06:09:07.328 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.014455956000000001 2023-10-09T06:09:07.328 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.328 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.328 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.328 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501458+0000", 2023-10-09T06:09:07.328 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015554869000000001 2023-10-09T06:09:07.329 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.329 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.329 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.329 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.502526+0000", 2023-10-09T06:09:07.329 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0010681250000000001 2023-10-09T06:09:07.329 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.329 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.329 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:09:07.330 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.502573+0000", 2023-10-09T06:09:07.330 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6508999999999998e-05 2023-10-09T06:09:07.330 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.330 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.330 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:09:07.330 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.502607+0000", 2023-10-09T06:09:07.330 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.4057000000000003e-05 2023-10-09T06:09:07.330 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.330 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:09:07.330 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.330 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.330 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.331 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149220 8.4 8:2e455896:::benchmark_data_smithi064_457003_object44702:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:09:07.331 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.508273+0000", 2023-10-09T06:09:07.331 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 212.702316467, 2023-10-09T06:09:07.331 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.994475785, 2023-10-09T06:09:07.331 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:09:07.331 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:09:07.331 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:09:07.331 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:09:07.331 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:09:07.331 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149220 2023-10-09T06:09:07.331 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.331 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:09:07.332 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.332 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:09:07.332 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.508273+0000", 2023-10-09T06:09:07.332 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.332 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.332 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:09:07.332 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.508273+0000", 2023-10-09T06:09:07.332 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.332 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.333 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:09:07.333 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.508272+0000", 2023-10-09T06:09:07.333 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:09:07.333 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.333 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.333 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:09:07.333 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.508278+0000", 2023-10-09T06:09:07.333 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8749999999999997e-06 2023-10-09T06:09:07.333 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.333 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.333 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:09:07.334 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.508292+0000", 2023-10-09T06:09:07.334 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4306e-05 2023-10-09T06:09:07.334 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.334 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.334 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:09:07.334 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.508299+0000", 2023-10-09T06:09:07.334 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9199999999999998e-06 2023-10-09T06:09:07.334 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.334 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.334 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.334 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.508352+0000", 2023-10-09T06:09:07.335 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2942000000000002e-05 2023-10-09T06:09:07.335 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.335 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.335 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.335 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.508358+0000", 2023-10-09T06:09:07.335 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3960000000000003e-06 2023-10-09T06:09:07.335 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.335 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.335 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.335 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.474150+0000", 2023-10-09T06:09:07.335 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96579230199999999 2023-10-09T06:09:07.335 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.335 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.336 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:09:07.336 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.474258+0000", 2023-10-09T06:09:07.336 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000107396 2023-10-09T06:09:07.336 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.336 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.336 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:09:07.336 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.474372+0000", 2023-10-09T06:09:07.336 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000114195 2023-10-09T06:09:07.336 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.336 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.336 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:09:07.337 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.485909+0000", 2023-10-09T06:09:07.337 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.01153674 2023-10-09T06:09:07.337 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.337 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.337 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.337 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501474+0000", 2023-10-09T06:09:07.337 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015565422000000001 2023-10-09T06:09:07.337 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.337 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.337 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.338 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.502662+0000", 2023-10-09T06:09:07.338 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0011879200000000001 2023-10-09T06:09:07.338 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.338 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.338 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:09:07.338 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.502706+0000", 2023-10-09T06:09:07.338 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3810999999999998e-05 2023-10-09T06:09:07.338 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.338 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.338 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:09:07.338 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.502749+0000", 2023-10-09T06:09:07.339 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2738000000000003e-05 2023-10-09T06:09:07.339 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.339 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:09:07.339 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.339 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.339 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.340 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149247 8.4 8:2a12e90d:::benchmark_data_smithi064_457003_object44729:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:09:07.340 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.538893+0000", 2023-10-09T06:09:07.340 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 212.67169674199999, 2023-10-09T06:09:07.340 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96400028800000004, 2023-10-09T06:09:07.340 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:09:07.340 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:09:07.340 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:09:07.341 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:09:07.341 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:09:07.341 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149247 2023-10-09T06:09:07.341 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.341 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:09:07.341 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.341 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:09:07.341 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.538893+0000", 2023-10-09T06:09:07.341 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.341 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.341 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.341 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:09:07.341 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.538893+0000", 2023-10-09T06:09:07.342 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.342 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.342 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.342 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:09:07.342 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.538891+0000", 2023-10-09T06:09:07.342 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:09:07.342 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.342 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.342 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:09:07.342 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.538898+0000", 2023-10-09T06:09:07.342 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7630000000000003e-06 2023-10-09T06:09:07.343 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.343 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.343 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:09:07.343 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.538911+0000", 2023-10-09T06:09:07.343 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3047000000000001e-05 2023-10-09T06:09:07.343 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.343 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.343 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:09:07.343 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.538918+0000", 2023-10-09T06:09:07.343 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5520000000000001e-06 2023-10-09T06:09:07.343 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.343 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.344 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.344 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.538968+0000", 2023-10-09T06:09:07.344 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0691e-05 2023-10-09T06:09:07.344 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.344 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.344 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.344 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.538977+0000", 2023-10-09T06:09:07.344 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8079999999999994e-06 2023-10-09T06:09:07.344 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.344 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.344 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.344 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.477116+0000", 2023-10-09T06:09:07.344 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.93813863799999997 2023-10-09T06:09:07.345 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.345 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.345 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:09:07.345 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.477219+0000", 2023-10-09T06:09:07.345 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000103694 2023-10-09T06:09:07.345 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.345 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.345 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:09:07.345 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.477336+0000", 2023-10-09T06:09:07.346 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000116764 2023-10-09T06:09:07.346 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.346 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.346 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:09:07.346 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.485914+0000", 2023-10-09T06:09:07.346 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0085778710000000008 2023-10-09T06:09:07.346 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.346 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.346 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.347 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501493+0000", 2023-10-09T06:09:07.347 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015579434 2023-10-09T06:09:07.347 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.347 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.347 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.347 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.502811+0000", 2023-10-09T06:09:07.347 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0013172539999999999 2023-10-09T06:09:07.347 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.347 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.347 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:09:07.347 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.502859+0000", 2023-10-09T06:09:07.347 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8408000000000001e-05 2023-10-09T06:09:07.347 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.348 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.348 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:09:07.348 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.502893+0000", 2023-10-09T06:09:07.348 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.3868999999999997e-05 2023-10-09T06:09:07.348 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.348 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:09:07.348 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.348 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.348 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.348 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149253 8.4 8:270bc248:::benchmark_data_smithi064_457003_object44735:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:09:07.348 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.547086+0000", 2023-10-09T06:09:07.349 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 212.66350306000001, 2023-10-09T06:09:07.349 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.95591631499999996, 2023-10-09T06:09:07.349 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:09:07.349 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:09:07.349 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:09:07.349 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:09:07.349 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:09:07.349 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149253 2023-10-09T06:09:07.349 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.349 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:09:07.349 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.349 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:09:07.349 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547086+0000", 2023-10-09T06:09:07.350 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.350 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.350 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.350 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:09:07.350 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547086+0000", 2023-10-09T06:09:07.350 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.350 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.350 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.350 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:09:07.350 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547085+0000", 2023-10-09T06:09:07.350 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:09:07.350 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.351 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.351 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:09:07.351 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547091+0000", 2023-10-09T06:09:07.351 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5610000000000004e-06 2023-10-09T06:09:07.351 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.351 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.351 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:09:07.351 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547104+0000", 2023-10-09T06:09:07.351 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2527999999999999e-05 2023-10-09T06:09:07.351 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.351 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.351 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:09:07.352 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547109+0000", 2023-10-09T06:09:07.352 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4940000000000003e-06 2023-10-09T06:09:07.352 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.352 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.352 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.352 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547160+0000", 2023-10-09T06:09:07.352 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0955000000000003e-05 2023-10-09T06:09:07.352 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.352 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.352 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.352 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547167+0000", 2023-10-09T06:09:07.352 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4570000000000002e-06 2023-10-09T06:09:07.353 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.353 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.353 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.353 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.480627+0000", 2023-10-09T06:09:07.353 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.93346054199999995 2023-10-09T06:09:07.353 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.353 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.354 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:09:07.354 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.480719+0000", 2023-10-09T06:09:07.354 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2038000000000002e-05 2023-10-09T06:09:07.354 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.354 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.354 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:09:07.354 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.480800+0000", 2023-10-09T06:09:07.355 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0419000000000004e-05 2023-10-09T06:09:07.355 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.355 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.355 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:09:07.355 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.488792+0000", 2023-10-09T06:09:07.355 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0079922489999999999 2023-10-09T06:09:07.356 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.356 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.356 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.356 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501516+0000", 2023-10-09T06:09:07.356 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012724231000000001 2023-10-09T06:09:07.356 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.356 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.357 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.357 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.502932+0000", 2023-10-09T06:09:07.357 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001415302 2023-10-09T06:09:07.357 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.357 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.357 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:09:07.357 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.502969+0000", 2023-10-09T06:09:07.358 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7469000000000003e-05 2023-10-09T06:09:07.358 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.358 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.358 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:09:07.358 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.503003+0000", 2023-10-09T06:09:07.358 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.366e-05 2023-10-09T06:09:07.359 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.359 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:09:07.359 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.359 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.359 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.359 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149254 8.4 8:2ee8489f:::benchmark_data_smithi064_457003_object44736:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:09:07.359 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.547125+0000", 2023-10-09T06:09:07.360 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 212.66346486200001, 2023-10-09T06:09:07.360 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.95964453599999999, 2023-10-09T06:09:07.360 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:09:07.360 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:09:07.360 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:09:07.360 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:09:07.361 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:09:07.361 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149254 2023-10-09T06:09:07.361 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.361 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:09:07.361 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.361 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:09:07.362 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547125+0000", 2023-10-09T06:09:07.362 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.362 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.362 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.362 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:09:07.362 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547125+0000", 2023-10-09T06:09:07.363 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.363 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.363 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.363 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:09:07.363 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547124+0000", 2023-10-09T06:09:07.363 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:09:07.363 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.364 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.364 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:09:07.364 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547127+0000", 2023-10-09T06:09:07.364 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.117e-06 2023-10-09T06:09:07.364 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.364 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.365 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:09:07.365 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547134+0000", 2023-10-09T06:09:07.365 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3400000000000003e-06 2023-10-09T06:09:07.365 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.365 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.365 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:09:07.365 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547138+0000", 2023-10-09T06:09:07.366 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.002e-06 2023-10-09T06:09:07.366 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.366 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.366 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.366 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547190+0000", 2023-10-09T06:09:07.366 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2871000000000003e-05 2023-10-09T06:09:07.367 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.367 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.367 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.367 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.547199+0000", 2023-10-09T06:09:07.367 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0330000000000008e-06 2023-10-09T06:09:07.367 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.367 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.368 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.368 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.483229+0000", 2023-10-09T06:09:07.368 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.93603051800000003 2023-10-09T06:09:07.368 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.368 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.368 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:09:07.369 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.483338+0000", 2023-10-09T06:09:07.369 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000108942 2023-10-09T06:09:07.369 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.369 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.369 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:09:07.369 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.483456+0000", 2023-10-09T06:09:07.369 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000117689 2023-10-09T06:09:07.370 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.370 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.370 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:09:07.370 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.497626+0000", 2023-10-09T06:09:07.370 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.014170075000000001 2023-10-09T06:09:07.370 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.371 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.371 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.371 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501553+0000", 2023-10-09T06:09:07.371 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0039269229999999997 2023-10-09T06:09:07.371 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.371 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.372 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.372 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.506695+0000", 2023-10-09T06:09:07.372 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0051421440000000004 2023-10-09T06:09:07.372 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.372 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.372 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:09:07.373 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.506732+0000", 2023-10-09T06:09:07.373 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7499000000000001e-05 2023-10-09T06:09:07.373 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.373 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.373 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:09:07.373 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.506769+0000", 2023-10-09T06:09:07.374 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6773000000000002e-05 2023-10-09T06:09:07.374 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.374 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:09:07.374 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.374 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.374 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.375 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:149256 8.4 8:21764160:::benchmark_data_smithi064_457003_object44738:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T06:09:07.375 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:34.550527+0000", 2023-10-09T06:09:07.375 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 212.660062856, 2023-10-09T06:09:07.375 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.95636602400000004, 2023-10-09T06:09:07.375 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:09:07.375 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:09:07.376 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:09:07.376 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:09:07.376 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:09:07.376 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149256 2023-10-09T06:09:07.376 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.376 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:09:07.376 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.377 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:09:07.377 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.550527+0000", 2023-10-09T06:09:07.377 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.377 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.377 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.377 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:09:07.377 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.550527+0000", 2023-10-09T06:09:07.378 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.378 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.378 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.378 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:09:07.378 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.550525+0000", 2023-10-09T06:09:07.378 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:09:07.378 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.378 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.379 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:09:07.379 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.550533+0000", 2023-10-09T06:09:07.379 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8900000000000007e-06 2023-10-09T06:09:07.379 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.379 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.379 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:09:07.379 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.550546+0000", 2023-10-09T06:09:07.379 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2383e-05 2023-10-09T06:09:07.379 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.380 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.380 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:09:07.380 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.550551+0000", 2023-10-09T06:09:07.380 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2499999999999997e-06 2023-10-09T06:09:07.380 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.380 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.380 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.380 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.550589+0000", 2023-10-09T06:09:07.380 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7891e-05 2023-10-09T06:09:07.380 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.381 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.381 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.381 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:34.550595+0000", 2023-10-09T06:09:07.381 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7799999999999997e-06 2023-10-09T06:09:07.381 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.381 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.382 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.382 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.485946+0000", 2023-10-09T06:09:07.382 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.93535175100000001 2023-10-09T06:09:07.382 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.382 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.382 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:09:07.382 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.486048+0000", 2023-10-09T06:09:07.383 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000101838 2023-10-09T06:09:07.383 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.383 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.383 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:09:07.383 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.486176+0000", 2023-10-09T06:09:07.383 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012777900000000001 2023-10-09T06:09:07.384 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.384 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.384 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:09:07.384 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.500497+0000", 2023-10-09T06:09:07.384 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.014320502000000001 2023-10-09T06:09:07.384 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.384 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.385 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.385 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.501581+0000", 2023-10-09T06:09:07.385 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0010841869999999999 2023-10-09T06:09:07.385 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.385 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.385 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.385 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.506806+0000", 2023-10-09T06:09:07.386 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0052252360000000003 2023-10-09T06:09:07.386 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.386 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.386 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:09:07.386 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.506848+0000", 2023-10-09T06:09:07.386 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2057000000000001e-05 2023-10-09T06:09:07.386 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.387 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.387 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:09:07.387 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:35.506893+0000", 2023-10-09T06:09:07.387 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4580999999999999e-05 2023-10-09T06:09:07.387 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.387 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:09:07.387 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.388 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.388 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.388 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:116248 9.e 9:7ca2e1df:::benchmark_data_smithi064_511911_object15146:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e449)", 2023-10-09T06:09:07.388 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:02.760089+0000", 2023-10-09T06:09:07.388 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 4.450500334, 2023-10-09T06:09:07.388 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.865845524, 2023-10-09T06:09:07.388 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:09:07.389 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:09:07.389 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:09:07.389 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:09:07.389 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:09:07.389 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 116248 2023-10-09T06:09:07.389 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.389 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:09:07.389 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.390 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:09:07.390 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.760089+0000", 2023-10-09T06:09:07.390 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.390 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.390 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.390 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:09:07.390 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.760089+0000", 2023-10-09T06:09:07.390 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.390 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.391 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.391 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:09:07.391 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.760088+0000", 2023-10-09T06:09:07.391 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:09:07.391 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.391 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.391 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:09:07.391 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.760094+0000", 2023-10-09T06:09:07.391 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7959999999999998e-06 2023-10-09T06:09:07.391 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.391 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.391 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:09:07.392 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.760106+0000", 2023-10-09T06:09:07.392 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1379999999999999e-05 2023-10-09T06:09:07.392 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.392 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.392 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:09:07.392 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.760114+0000", 2023-10-09T06:09:07.392 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.4370000000000007e-06 2023-10-09T06:09:07.392 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.392 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.392 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.393 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.761388+0000", 2023-10-09T06:09:07.393 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0012735699999999999 2023-10-09T06:09:07.393 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.393 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.393 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.393 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.761399+0000", 2023-10-09T06:09:07.393 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0885e-05 2023-10-09T06:09:07.394 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.394 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.394 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.394 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.589288+0000", 2023-10-09T06:09:07.394 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.827889388 2023-10-09T06:09:07.394 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.394 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.394 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.394 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.589301+0000", 2023-10-09T06:09:07.394 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3095000000000001e-05 2023-10-09T06:09:07.394 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.395 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.395 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.395 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.625819+0000", 2023-10-09T06:09:07.395 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.036518066 2023-10-09T06:09:07.395 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.395 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.395 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:09:07.395 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.625935+0000", 2023-10-09T06:09:07.395 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011529199999999999 2023-10-09T06:09:07.395 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.395 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:09:07.396 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.396 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.396 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.396 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:116271 9.e 9:713dc955:::benchmark_data_smithi064_511911_object15169:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e449)", 2023-10-09T06:09:07.396 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:02.760121+0000", 2023-10-09T06:09:07.396 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 4.4504685349999997, 2023-10-09T06:09:07.396 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8662323810000001, 2023-10-09T06:09:07.396 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:09:07.396 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:09:07.396 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:09:07.396 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:09:07.396 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:09:07.397 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 116271 2023-10-09T06:09:07.397 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.397 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:09:07.397 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.397 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:09:07.397 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.760121+0000", 2023-10-09T06:09:07.397 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.397 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.397 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.397 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:09:07.397 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.760121+0000", 2023-10-09T06:09:07.398 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.398 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.398 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.398 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:09:07.398 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.760120+0000", 2023-10-09T06:09:07.398 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:09:07.398 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.399 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.399 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:09:07.399 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.760123+0000", 2023-10-09T06:09:07.399 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.6299999999999998e-06 2023-10-09T06:09:07.399 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.399 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.399 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:09:07.400 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.760129+0000", 2023-10-09T06:09:07.400 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.212e-06 2023-10-09T06:09:07.400 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.400 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.400 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:09:07.400 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.760133+0000", 2023-10-09T06:09:07.400 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9310000000000001e-06 2023-10-09T06:09:07.400 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.401 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.401 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.401 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.761435+0000", 2023-10-09T06:09:07.401 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0013021580000000001 2023-10-09T06:09:07.401 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.401 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.401 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.402 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.761444+0000", 2023-10-09T06:09:07.402 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.2099999999999993e-06 2023-10-09T06:09:07.402 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.402 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.402 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.402 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.589326+0000", 2023-10-09T06:09:07.402 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.82788211099999998 2023-10-09T06:09:07.403 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.403 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.403 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.403 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.589333+0000", 2023-10-09T06:09:07.403 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4680000000000003e-06 2023-10-09T06:09:07.403 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.403 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.403 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.403 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.626276+0000", 2023-10-09T06:09:07.403 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0369433379999999 2023-10-09T06:09:07.403 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.403 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.404 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:09:07.404 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.626353+0000", 2023-10-09T06:09:07.404 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6755000000000006e-05 2023-10-09T06:09:07.404 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.404 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:09:07.404 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.404 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.404 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.404 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:09:07.406 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:116291 9.e 9:7a5f1263:::benchmark_data_smithi064_511911_object15189:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e449)", 2023-10-09T06:09:07.406 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:02.760137+0000", 2023-10-09T06:09:07.406 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 4.4504521429999997, 2023-10-09T06:09:07.406 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.891237442, 2023-10-09T06:09:07.406 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:09:07.406 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:09:07.406 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:09:07.406 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:09:07.406 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:09:07.406 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 116291 2023-10-09T06:09:07.407 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.407 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:09:07.407 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.407 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:09:07.407 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.760137+0000", 2023-10-09T06:09:07.407 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.407 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.407 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.408 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:09:07.408 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.760137+0000", 2023-10-09T06:09:07.408 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.408 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.408 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.409 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:09:07.409 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.760137+0000", 2023-10-09T06:09:07.409 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:09:07.409 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.409 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.409 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:09:07.409 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.760139+0000", 2023-10-09T06:09:07.410 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2390000000000001e-06 2023-10-09T06:09:07.410 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.410 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.410 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:09:07.410 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.760145+0000", 2023-10-09T06:09:07.410 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4589999999999997e-06 2023-10-09T06:09:07.410 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.411 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.411 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:09:07.411 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.760148+0000", 2023-10-09T06:09:07.411 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5870000000000001e-06 2023-10-09T06:09:07.411 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.411 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.411 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.412 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.761468+0000", 2023-10-09T06:09:07.412 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001319473 2023-10-09T06:09:07.412 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.412 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.412 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.412 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.761474+0000", 2023-10-09T06:09:07.412 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9789999999999998e-06 2023-10-09T06:09:07.413 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.413 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.413 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.413 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.589356+0000", 2023-10-09T06:09:07.413 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.82788202 2023-10-09T06:09:07.413 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.413 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.413 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.414 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.589363+0000", 2023-10-09T06:09:07.414 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.973e-06 2023-10-09T06:09:07.414 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.414 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.414 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.414 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.626391+0000", 2023-10-09T06:09:07.414 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.037028439 2023-10-09T06:09:07.415 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.415 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.415 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for missing object", 2023-10-09T06:09:07.415 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.626524+0000", 2023-10-09T06:09:07.415 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000132395 2023-10-09T06:09:07.415 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.415 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.415 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.416 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.651273+0000", 2023-10-09T06:09:07.416 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.024749376999999999 2023-10-09T06:09:07.416 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.416 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.416 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:09:07.416 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.651375+0000", 2023-10-09T06:09:07.416 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000101801 2023-10-09T06:09:07.417 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.417 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:09:07.417 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.417 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.417 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.417 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:116299 9.e 9:7736abf9:::benchmark_data_smithi064_511911_object15197:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2023-10-09T06:09:07.417 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:02.790091+0000", 2023-10-09T06:09:07.418 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 4.4204979670000002, 2023-10-09T06:09:07.418 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8561298210000001, 2023-10-09T06:09:07.418 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:09:07.418 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:09:07.418 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:09:07.418 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:09:07.418 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:09:07.419 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 116299 2023-10-09T06:09:07.419 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.419 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:09:07.419 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.419 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:09:07.419 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.790091+0000", 2023-10-09T06:09:07.419 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.420 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.420 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.420 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:09:07.420 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.790091+0000", 2023-10-09T06:09:07.420 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:09:07.420 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.420 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.420 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:09:07.421 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.790090+0000", 2023-10-09T06:09:07.421 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:09:07.421 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.421 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.421 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:09:07.421 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.790097+0000", 2023-10-09T06:09:07.421 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2219999999999996e-06 2023-10-09T06:09:07.422 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.422 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.422 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:09:07.422 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.790113+0000", 2023-10-09T06:09:07.422 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5387e-05 2023-10-09T06:09:07.423 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.423 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.423 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:09:07.423 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.790120+0000", 2023-10-09T06:09:07.424 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3339999999999996e-06 2023-10-09T06:09:07.424 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.424 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.424 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.424 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.790161+0000", 2023-10-09T06:09:07.424 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0825999999999998e-05 2023-10-09T06:09:07.425 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.425 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.425 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.425 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.790172+0000", 2023-10-09T06:09:07.426 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1426999999999999e-05 2023-10-09T06:09:07.426 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.426 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.426 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.426 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.591648+0000", 2023-10-09T06:09:07.426 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.80147579000000002 2023-10-09T06:09:07.426 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.427 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.427 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:09:07.427 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.591657+0000", 2023-10-09T06:09:07.427 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.5960000000000004e-06 2023-10-09T06:09:07.427 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.427 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.428 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:09:07.428 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.626574+0000", 2023-10-09T06:09:07.428 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0349174240000001 2023-10-09T06:09:07.428 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.428 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.428 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:09:07.429 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.626718+0000", 2023-10-09T06:09:07.429 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014404000000000001 2023-10-09T06:09:07.429 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.429 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.429 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:09:07.429 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.626871+0000", 2023-10-09T06:09:07.429 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015270999999999999 2023-10-09T06:09:07.430 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.430 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.430 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.430 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.633109+0000", 2023-10-09T06:09:07.430 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0062385540000000003 2023-10-09T06:09:07.430 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.430 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.430 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:09:07.431 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.633129+0000", 2023-10-09T06:09:07.431 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0203999999999998e-05 2023-10-09T06:09:07.431 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.431 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.431 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:09:07.431 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.646043+0000", 2023-10-09T06:09:07.431 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012913678 2023-10-09T06:09:07.432 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.432 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.432 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:09:07.432 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.646109+0000", 2023-10-09T06:09:07.432 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5858999999999994e-05 2023-10-09T06:09:07.432 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:09:07.432 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:09:07.433 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:09:07.433 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.646221+0000", 2023-10-09T06:09:07.433 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011223200000000001 2023-10-09T06:09:07.433 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.433 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:09:07.433 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.433 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:09:07.434 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:09:07.434 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:09:07.452 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:07.520 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:09:07.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:07.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:09:07.626 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:09:07.671 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:07.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:07 smithi064 ceph-mon[212750]: failed to encode map e453 with expected crc 2023-10-09T06:09:07.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:07 smithi064 ceph-mon[212750]: osdmap e453: 8 total, 7 up, 8 in 2023-10-09T06:09:07.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:07 smithi064 ceph-mon[212750]: failed to encode map e453 with expected crc 2023-10-09T06:09:07.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:07 smithi064 ceph-mon[212750]: failed to encode map e453 with expected crc 2023-10-09T06:09:07.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:07 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 25361/261813 objects degraded (9.687%), 13 pgs degraded, 43 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:07.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:07 smithi064 ceph-mon[220044]: failed to encode map e453 with expected crc 2023-10-09T06:09:07.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:07 smithi064 ceph-mon[220044]: osdmap e453: 8 total, 7 up, 8 in 2023-10-09T06:09:07.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:07 smithi064 ceph-mon[220044]: failed to encode map e453 with expected crc 2023-10-09T06:09:07.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:07 smithi064 ceph-mon[220044]: failed to encode map e453 with expected crc 2023-10-09T06:09:07.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:07 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 25361/261813 objects degraded (9.687%), 13 pgs degraded, 43 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:07.765 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:09:07.767 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:07 smithi156 ceph-mon[195101]: failed to encode map e453 with expected crc 2023-10-09T06:09:07.767 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:07 smithi156 ceph-mon[195101]: osdmap e453: 8 total, 7 up, 8 in 2023-10-09T06:09:07.767 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:07 smithi156 ceph-mon[195101]: failed to encode map e453 with expected crc 2023-10-09T06:09:07.767 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:07 smithi156 ceph-mon[195101]: failed to encode map e453 with expected crc 2023-10-09T06:09:07.767 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:07 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 25361/261813 objects degraded (9.687%), 13 pgs degraded, 43 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:07.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:07.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:09:07.869 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:09:07.897 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:07.941 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:09:07.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:07.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:09:08.043 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:09:08.045 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:09:08.072 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:08.129 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:09:08.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:08.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:09:08.235 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:09:08.277 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:08.436 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:09:08.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:08.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:09:08.501 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:09:08.542 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:09:08.644 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:08.716 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:09:08.728 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1, 2] out_osds: [] dead_osds: [0] live_osds: [1, 2, 3, 5, 6, 7, 4] 2023-10-09T06:09:08.729 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:09:08.729 INFO:tasks.thrashosds.thrasher:Reviving osd 0 2023-10-09T06:09:08.729 INFO:tasks.cephadm.osd.0:Restarting osd.0 (starting--it wasn't running)... 2023-10-09T06:09:08.729 DEBUG:teuthology.orchestra.run.smithi064:osd.0> sudo journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.0.service 2023-10-09T06:09:08.754 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl start ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.0 2023-10-09T06:09:08.761 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:09:08.799 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:09:08.814 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:09:08.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:08.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:09:08.861 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:09:08.868 INFO:journalctl@ceph.osd.0.smithi064.stdout:-- Logs begin at Mon 2023-10-09 05:16:16 UTC. -- 2023-10-09T06:09:08.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:08 smithi156 ceph-mon[195101]: pgmap v1130: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 1 active+remapped+backfilling, 10 active+clean+remapped, 3 remapped+peering, 1 active+undersized+degraded+remapped+backfill_wait, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 7 active+undersized+degraded, 60 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 29 KiB/s rd, 78 KiB/s wr, 560 op/s; 24941/256062 objects degraded (9.740%); 39376/256062 objects misplaced (15.378%); 8.8 MiB/s, 138 objects/s recovering 2023-10-09T06:09:08.885 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:08.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:08 smithi064 ceph-mon[212750]: pgmap v1130: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 1 active+remapped+backfilling, 10 active+clean+remapped, 3 remapped+peering, 1 active+undersized+degraded+remapped+backfill_wait, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 7 active+undersized+degraded, 60 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 29 KiB/s rd, 78 KiB/s wr, 560 op/s; 24941/256062 objects degraded (9.740%); 39376/256062 objects misplaced (15.378%); 8.8 MiB/s, 138 objects/s recovering 2023-10-09T06:09:08.917 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:08 smithi064 ceph-mon[220044]: pgmap v1130: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 1 active+remapped+backfilling, 10 active+clean+remapped, 3 remapped+peering, 1 active+undersized+degraded+remapped+backfill_wait, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 7 active+undersized+degraded, 60 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 29 KiB/s rd, 78 KiB/s wr, 560 op/s; 24941/256062 objects degraded (9.740%); 39376/256062 objects misplaced (15.378%); 8.8 MiB/s, 138 objects/s recovering 2023-10-09T06:09:08.920 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:09:08.927 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:09:08.974 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:09:09.010 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:09:09.010 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-10-09T06:09:09.010 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:09:09.011 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:09:09.011 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:09:09.011 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:09:09.011 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:09:09.011 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:09:09.011 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:09:09.014 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:09.224 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 06:09:08 smithi064 systemd[1]: Starting Ceph osd.0 for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T06:09:09.337 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:09:09.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:09.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:09:09.451 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:09:09.531 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:09.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:09 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2991728455' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:09:09.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:09 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2991728455' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:09:10.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:09 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2991728455' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:09:10.586 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:09:10.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:10.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:09:10.695 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:09:10.729 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:10.772 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T06:09:10.772 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T06:09:10.797 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:09:10.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:10.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:09:10.904 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:09:10.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:10 smithi064 ceph-mon[212750]: pgmap v1131: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 1 active+remapped+backfilling, 11 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 7 active+undersized+degraded, 62 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 22 KiB/s rd, 57 KiB/s wr, 450 op/s; 24745/255474 objects degraded (9.686%); 39376/255474 objects misplaced (15.413%); 6.5 MiB/s, 102 objects/s recovering 2023-10-09T06:09:10.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:10 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T06:09:10.916 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:10 smithi064 ceph-mon[220044]: pgmap v1131: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 1 active+remapped+backfilling, 11 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 7 active+undersized+degraded, 62 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 22 KiB/s rd, 57 KiB/s wr, 450 op/s; 24745/255474 objects degraded (9.686%); 39376/255474 objects misplaced (15.413%); 6.5 MiB/s, 102 objects/s recovering 2023-10-09T06:09:10.916 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:10 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T06:09:10.935 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:10 smithi156 ceph-mon[195101]: pgmap v1131: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 1 active+remapped+backfilling, 11 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 7 active+undersized+degraded, 62 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 22 KiB/s rd, 57 KiB/s wr, 450 op/s; 24745/255474 objects degraded (9.686%); 39376/255474 objects misplaced (15.413%); 6.5 MiB/s, 102 objects/s recovering 2023-10-09T06:09:10.936 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:10 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T06:09:10.940 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:11.014 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:09:11.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:11.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:09:11.121 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:09:11.209 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:11.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:11 smithi064 ceph-mon[212750]: pgmap v1132: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 9.8 KiB/s rd, 26 KiB/s wr, 880 op/s; 23462/245247 objects degraded (9.567%); 37146/245247 objects misplaced (15.146%); 22 MiB/s, 345 objects/s recovering 2023-10-09T06:09:11.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:11 smithi064 ceph-mon[220044]: pgmap v1132: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 9.8 KiB/s rd, 26 KiB/s wr, 880 op/s; 23462/245247 objects degraded (9.567%); 37146/245247 objects misplaced (15.146%); 22 MiB/s, 345 objects/s recovering 2023-10-09T06:09:12.111 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:09:12.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:12.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:09:12.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:11 smithi156 ceph-mon[195101]: pgmap v1132: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 9.8 KiB/s rd, 26 KiB/s wr, 880 op/s; 23462/245247 objects degraded (9.567%); 37146/245247 objects misplaced (15.146%); 22 MiB/s, 345 objects/s recovering 2023-10-09T06:09:12.219 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:09:12.287 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:12.369 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:12.450 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:09:12.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:12.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:09:12.561 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:09:12.595 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:12.667 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:09:12.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:12.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:09:12.718 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:12 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 23462/245247 objects degraded (9.567%), 11 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:12.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:12 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 23462/245247 objects degraded (9.567%), 11 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:12.773 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:09:12.784 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:09:12.818 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:12.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:12 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 23462/245247 objects degraded (9.567%), 11 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:12.878 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:09:12.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:12.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:09:12.983 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:09:13.073 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:13.260 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 06:09:13 smithi064 bash[537673]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T06:09:13.261 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 06:09:13 smithi064 bash[537673]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T06:09:13.261 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 06:09:13 smithi064 bash[537673]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-10-09T06:09:13.261 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 06:09:13 smithi064 bash[537673]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-10-09T06:09:13.261 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 06:09:13 smithi064 bash[537673]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-10-09T06:09:13.261 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 06:09:13 smithi064 bash[537673]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-10-09T06:09:13.261 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 06:09:13 smithi064 bash[537673]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T06:09:13.261 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 06:09:13 smithi064 bash[537673]: --> ceph-volume lvm activate successful for osd ID: 0 2023-10-09T06:09:13.710 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:13 smithi064 ceph-mon[212750]: pgmap v1133: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7.9 KiB/s rd, 21 KiB/s wr, 723 op/s; 22729/244581 objects degraded (9.293%); 36924/244581 objects misplaced (15.097%); 21 MiB/s, 342 objects/s recovering 2023-10-09T06:09:13.711 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:13 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3601062431' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:09:13.711 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:13 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:09:13.711 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:13 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:13.711 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:13 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:13.729 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:09:13.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:13 smithi064 ceph-mon[220044]: pgmap v1133: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7.9 KiB/s rd, 21 KiB/s wr, 723 op/s; 22729/244581 objects degraded (9.293%); 36924/244581 objects misplaced (15.097%); 21 MiB/s, 342 objects/s recovering 2023-10-09T06:09:13.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:13 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3601062431' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:09:13.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:13 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:09:13.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:13 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:13.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:13 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:14.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:13 smithi156 ceph-mon[195101]: pgmap v1133: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7.9 KiB/s rd, 21 KiB/s wr, 723 op/s; 22729/244581 objects degraded (9.293%); 36924/244581 objects misplaced (15.097%); 21 MiB/s, 342 objects/s recovering 2023-10-09T06:09:14.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:13 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3601062431' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:09:14.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:13 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:09:14.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:13 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:14.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:13 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:14.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:14 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:09:14.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:14 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:09:14.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:14 smithi064 ceph-mon[212750]: osdmap e454: 8 total, 7 up, 8 in 2023-10-09T06:09:14.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:14 smithi064 ceph-mon[212750]: failed to encode map e454 with expected crc 2023-10-09T06:09:14.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:14 smithi064 ceph-mon[212750]: failed to encode map e454 with expected crc 2023-10-09T06:09:14.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:14 smithi064 ceph-mon[212750]: failed to encode map e454 with expected crc 2023-10-09T06:09:14.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:14 smithi064 ceph-mon[212750]: failed to encode map e454 with expected crc 2023-10-09T06:09:14.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:14 smithi064 ceph-mon[212750]: failed to encode map e454 with expected crc 2023-10-09T06:09:14.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:14 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:09:14.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:14 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:09:14.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:14 smithi064 ceph-mon[220044]: osdmap e454: 8 total, 7 up, 8 in 2023-10-09T06:09:14.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:14 smithi064 ceph-mon[220044]: failed to encode map e454 with expected crc 2023-10-09T06:09:14.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:14 smithi064 ceph-mon[220044]: failed to encode map e454 with expected crc 2023-10-09T06:09:14.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:14 smithi064 ceph-mon[220044]: failed to encode map e454 with expected crc 2023-10-09T06:09:14.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:14 smithi064 ceph-mon[220044]: failed to encode map e454 with expected crc 2023-10-09T06:09:14.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:14 smithi064 ceph-mon[220044]: failed to encode map e454 with expected crc 2023-10-09T06:09:15.021 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:09:15.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:15.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:09:15.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:14 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:09:15.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:14 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:09:15.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:14 smithi156 ceph-mon[195101]: osdmap e454: 8 total, 7 up, 8 in 2023-10-09T06:09:15.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:14 smithi156 ceph-mon[195101]: failed to encode map e454 with expected crc 2023-10-09T06:09:15.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:14 smithi156 ceph-mon[195101]: failed to encode map e454 with expected crc 2023-10-09T06:09:15.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:14 smithi156 ceph-mon[195101]: failed to encode map e454 with expected crc 2023-10-09T06:09:15.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:14 smithi156 ceph-mon[195101]: failed to encode map e454 with expected crc 2023-10-09T06:09:15.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:14 smithi156 ceph-mon[195101]: failed to encode map e454 with expected crc 2023-10-09T06:09:15.129 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:09:15.172 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:15.259 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:09:15.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:15.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:09:15.364 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:09:15.485 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:15.713 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:09:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:09:15] "GET /metrics HTTP/1.1" 200 36488 "" "Prometheus/2.43.0" 2023-10-09T06:09:15.715 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:15 smithi064 ceph-mon[220044]: pgmap v1135: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 21452/243159 objects degraded (8.822%); 36694/243159 objects misplaced (15.091%); 23 MiB/s, 364 objects/s recovering 2023-10-09T06:09:15.717 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:15 smithi064 ceph-mon[212750]: pgmap v1135: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 21452/243159 objects degraded (8.822%); 36694/243159 objects misplaced (15.091%); 23 MiB/s, 364 objects/s recovering 2023-10-09T06:09:16.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:15 smithi156 ceph-mon[195101]: pgmap v1135: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 21452/243159 objects degraded (8.822%); 36694/243159 objects misplaced (15.091%); 23 MiB/s, 364 objects/s recovering 2023-10-09T06:09:16.248 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:09:16.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:16.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:09:16.359 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:09:16.409 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:16.476 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:09:16.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:16.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:09:16.585 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:09:16.672 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:16.727 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:16.810 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:09:16.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:16.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:09:16.890 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:09:16.890 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:09:16.920 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:09:16.966 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:17.060 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:09:17.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:17.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:09:17.167 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:09:17.224 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:17.313 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:09:17.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:17.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:09:17.434 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:09:17.490 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:17.571 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:09:17.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:17.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:09:17.683 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:09:17.719 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:18.336 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:09:18.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:18 smithi156 ceph-mon[195101]: pgmap v1136: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 20433/233187 objects degraded (8.762%); 34521/233187 objects misplaced (14.804%); 28 MiB/s, 446 objects/s recovering 2023-10-09T06:09:18.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:18 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 20433/233187 objects degraded (8.762%), 11 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:18.396 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:18 smithi064 ceph-mon[220044]: pgmap v1136: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 20433/233187 objects degraded (8.762%); 34521/233187 objects misplaced (14.804%); 28 MiB/s, 446 objects/s recovering 2023-10-09T06:09:18.396 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:18 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 20433/233187 objects degraded (8.762%), 11 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:18.396 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:18 smithi064 ceph-mon[212750]: pgmap v1136: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 20433/233187 objects degraded (8.762%); 34521/233187 objects misplaced (14.804%); 28 MiB/s, 446 objects/s recovering 2023-10-09T06:09:18.396 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:18 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 20433/233187 objects degraded (8.762%), 11 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:18.555 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:09:18.607 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:09:18.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:18.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:09:18.726 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:09:18.834 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:18.844 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:09:19.026 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 06:09:18 smithi064 bash[538963]: 631dc6265ff12bb31f544d5b252dfc4348cbd3fe94f57fa1706642287acf3b11 2023-10-09T06:09:19.026 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 06:09:18 smithi064 systemd[1]: Started Ceph osd.0 for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T06:09:19.048 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:09:19.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:19.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:09:19.156 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:09:19.197 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:19.271 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:09:19.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:19.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:09:19.375 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:09:19.428 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:19.469 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 06:09:19 smithi064 conmon[539142]: 2023-10-09T06:09:19.122+0000 7f54ef67d640 -1 Falling back to public interface 2023-10-09T06:09:19.509 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:09:19.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:19.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:09:19.617 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:09:19.673 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:19.995 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:09:20.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:20.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:09:20.106 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:09:20.142 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:20.196 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:09:20.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:20.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:09:20.295 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:20.295 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:20.301 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:20.303 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:09:20.329 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:20 smithi156 ceph-mon[195101]: pgmap v1137: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 20179/232425 objects degraded (8.682%); 34521/232425 objects misplaced (14.853%); 28 MiB/s, 446 objects/s recovering 2023-10-09T06:09:20.333 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:20.397 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:09:20.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:20.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:09:20.474 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:20 smithi064 ceph-mon[212750]: pgmap v1137: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 20179/232425 objects degraded (8.682%); 34521/232425 objects misplaced (14.853%); 28 MiB/s, 446 objects/s recovering 2023-10-09T06:09:20.474 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:20 smithi064 ceph-mon[220044]: pgmap v1137: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 20179/232425 objects degraded (8.682%); 34521/232425 objects misplaced (14.853%); 28 MiB/s, 446 objects/s recovering 2023-10-09T06:09:20.502 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:09:20.573 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:21.072 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:09:21.140 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:09:21.162 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:09:21.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:21.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:09:21.185 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:09:21.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:21 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2389904736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:09:21.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:21 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2389904736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:09:21.251 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:09:21.285 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:09:21.291 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:09:21.348 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:09:21.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:21 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2389904736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:09:21.373 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:21.393 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:09:21.393 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-10-09T06:09:21.393 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:09:21.393 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:09:21.393 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:09:21.393 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:09:21.393 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:09:21.393 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:09:21.394 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:09:22.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:22 smithi156 ceph-mon[195101]: pgmap v1138: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 19050/222654 objects degraded (8.556%); 32273/222654 objects misplaced (14.495%); 28 MiB/s, 446 objects/s recovering 2023-10-09T06:09:22.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:22 smithi064 ceph-mon[220044]: pgmap v1138: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 19050/222654 objects degraded (8.556%); 32273/222654 objects misplaced (14.495%); 28 MiB/s, 446 objects/s recovering 2023-10-09T06:09:22.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:22 smithi064 ceph-mon[212750]: pgmap v1138: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 19050/222654 objects degraded (8.556%); 32273/222654 objects misplaced (14.495%); 28 MiB/s, 446 objects/s recovering 2023-10-09T06:09:23.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:23 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 19050/222654 objects degraded (8.556%), 11 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:23.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:23 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 19050/222654 objects degraded (8.556%), 11 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:23.343 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:09:23.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:23.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:09:23.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:23 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 19050/222654 objects degraded (8.556%), 11 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:23.449 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:09:23.541 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:23.612 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:09:23.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:23.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:09:23.718 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:09:23.761 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:23.826 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:09:23.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:23.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:09:23.931 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:09:23.965 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:24.054 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:09:24.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:24.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:09:24.164 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:09:24.190 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:24.231 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:09:24.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:24.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:09:24.338 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:09:24.381 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:24.455 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:09:24.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:24.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:09:24.561 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:09:24.645 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:24.719 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 06:09:24 smithi064 conmon[539142]: 2023-10-09T06:09:24.585+0000 7f54ef67d640 -1 osd.0 437 log_to_monitors true 2023-10-09T06:09:24.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:24 smithi064 ceph-mon[220044]: pgmap v1139: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 18376/221943 objects degraded (8.280%); 32036/221943 objects misplaced (14.434%); 28 MiB/s, 444 objects/s recovering 2023-10-09T06:09:24.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:24 smithi064 ceph-mon[212750]: pgmap v1139: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 18376/221943 objects degraded (8.280%); 32036/221943 objects misplaced (14.434%); 28 MiB/s, 444 objects/s recovering 2023-10-09T06:09:24.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:24 smithi156 ceph-mon[195101]: pgmap v1139: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 18376/221943 objects degraded (8.280%); 32036/221943 objects misplaced (14.434%); 28 MiB/s, 444 objects/s recovering 2023-10-09T06:09:25.327 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:09:25.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:25.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:09:25.429 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T06:09:25.430 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T06:09:25.445 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:09:25.486 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:25.533 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:09:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:09:25] "GET /metrics HTTP/1.1" 200 36488 "" "Prometheus/2.43.0" 2023-10-09T06:09:25.565 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:09:25.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:25.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:09:25.683 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:09:25.722 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:25.791 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:09:25.795 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:25 smithi156 ceph-mon[195101]: from='osd.0 [v2:172.21.15.64:6802/119405434,v1:172.21.15.64:6803/119405434]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T06:09:25.795 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:25 smithi156 ceph-mon[195101]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T06:09:25.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:25.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:09:25.808 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 06:09:25 smithi064 conmon[539142]: 2023-10-09T06:09:25.588+0000 7f54df2e6700 -1 osd.0 437 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T06:09:25.814 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:25 smithi064 ceph-mon[212750]: from='osd.0 [v2:172.21.15.64:6802/119405434,v1:172.21.15.64:6803/119405434]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T06:09:25.814 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:25 smithi064 ceph-mon[212750]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T06:09:25.814 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:25 smithi064 ceph-mon[220044]: from='osd.0 [v2:172.21.15.64:6802/119405434,v1:172.21.15.64:6803/119405434]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T06:09:25.814 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:25 smithi064 ceph-mon[220044]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T06:09:25.897 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:09:25.981 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:26.125 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:09:26.125 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T06:09:26.125 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T06:09:26.125 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T06:09:26.125 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:09:26.226 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:09:26.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:26.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:09:26.360 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:09:26.442 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:26.712 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:09:26.712 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T06:09:26.712 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2023-10-09T06:09:26.712 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:09:26.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:26 smithi156 ceph-mon[195101]: pgmap v1140: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 17415/220539 objects degraded (7.897%); 31803/220539 objects misplaced (14.421%); 28 MiB/s, 447 objects/s recovering 2023-10-09T06:09:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:26 smithi156 ceph-mon[195101]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T06:09:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:26 smithi156 ceph-mon[195101]: osdmap e455: 8 total, 7 up, 8 in 2023-10-09T06:09:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:26 smithi156 ceph-mon[195101]: from='osd.0 [v2:172.21.15.64:6802/119405434,v1:172.21.15.64:6803/119405434]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T06:09:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:26 smithi156 ceph-mon[195101]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T06:09:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:26 smithi156 ceph-mon[195101]: failed to encode map e455 with expected crc 2023-10-09T06:09:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:26 smithi156 ceph-mon[195101]: failed to encode map e455 with expected crc 2023-10-09T06:09:26.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:26 smithi156 ceph-mon[195101]: failed to encode map e455 with expected crc 2023-10-09T06:09:26.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:26 smithi156 ceph-mon[195101]: failed to encode map e455 with expected crc 2023-10-09T06:09:26.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:26 smithi064 ceph-mon[220044]: pgmap v1140: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 17415/220539 objects degraded (7.897%); 31803/220539 objects misplaced (14.421%); 28 MiB/s, 447 objects/s recovering 2023-10-09T06:09:26.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:26 smithi064 ceph-mon[220044]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T06:09:26.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:26 smithi064 ceph-mon[220044]: osdmap e455: 8 total, 7 up, 8 in 2023-10-09T06:09:26.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:26 smithi064 ceph-mon[220044]: from='osd.0 [v2:172.21.15.64:6802/119405434,v1:172.21.15.64:6803/119405434]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T06:09:26.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:26 smithi064 ceph-mon[220044]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T06:09:26.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:26 smithi064 ceph-mon[220044]: failed to encode map e455 with expected crc 2023-10-09T06:09:26.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:26 smithi064 ceph-mon[220044]: failed to encode map e455 with expected crc 2023-10-09T06:09:26.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:26 smithi064 ceph-mon[220044]: failed to encode map e455 with expected crc 2023-10-09T06:09:26.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:26 smithi064 ceph-mon[220044]: failed to encode map e455 with expected crc 2023-10-09T06:09:26.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:26 smithi064 ceph-mon[212750]: pgmap v1140: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 17415/220539 objects degraded (7.897%); 31803/220539 objects misplaced (14.421%); 28 MiB/s, 447 objects/s recovering 2023-10-09T06:09:26.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:26 smithi064 ceph-mon[212750]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T06:09:26.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:26 smithi064 ceph-mon[212750]: osdmap e455: 8 total, 7 up, 8 in 2023-10-09T06:09:26.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:26 smithi064 ceph-mon[212750]: from='osd.0 [v2:172.21.15.64:6802/119405434,v1:172.21.15.64:6803/119405434]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T06:09:26.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:26 smithi064 ceph-mon[212750]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-10-09T06:09:26.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:26 smithi064 ceph-mon[212750]: failed to encode map e455 with expected crc 2023-10-09T06:09:26.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:26 smithi064 ceph-mon[212750]: failed to encode map e455 with expected crc 2023-10-09T06:09:26.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:26 smithi064 ceph-mon[212750]: failed to encode map e455 with expected crc 2023-10-09T06:09:26.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:26 smithi064 ceph-mon[212750]: failed to encode map e455 with expected crc 2023-10-09T06:09:27.431 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:09:27.689 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:09:27.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:27 smithi064 ceph-mon[220044]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:09:27.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:27 smithi064 ceph-mon[220044]: osd.0 [v2:172.21.15.64:6802/119405434,v1:172.21.15.64:6803/119405434] boot 2023-10-09T06:09:27.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:27 smithi064 ceph-mon[220044]: osdmap e456: 8 total, 8 up, 8 in 2023-10-09T06:09:27.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:27 smithi064 ceph-mon[220044]: failed to encode map e456 with expected crc 2023-10-09T06:09:27.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:27 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:09:27.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:27 smithi064 ceph-mon[220044]: failed to encode map e456 with expected crc 2023-10-09T06:09:27.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:27 smithi064 ceph-mon[220044]: failed to encode map e456 with expected crc 2023-10-09T06:09:27.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:27 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3344706052' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:09:27.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:27 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 17415/220539 objects degraded (7.897%), 11 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:27.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:27 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:09:27.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:27 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3344706052' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:09:27.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:27 smithi064 ceph-mon[220044]: osdmap e457: 8 total, 8 up, 8 in 2023-10-09T06:09:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:27 smithi064 ceph-mon[212750]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:09:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:27 smithi064 ceph-mon[212750]: osd.0 [v2:172.21.15.64:6802/119405434,v1:172.21.15.64:6803/119405434] boot 2023-10-09T06:09:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:27 smithi064 ceph-mon[212750]: osdmap e456: 8 total, 8 up, 8 in 2023-10-09T06:09:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:27 smithi064 ceph-mon[212750]: failed to encode map e456 with expected crc 2023-10-09T06:09:27.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:27 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:09:27.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:27 smithi064 ceph-mon[212750]: failed to encode map e456 with expected crc 2023-10-09T06:09:27.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:27 smithi064 ceph-mon[212750]: failed to encode map e456 with expected crc 2023-10-09T06:09:27.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:27 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3344706052' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:09:27.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:27 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 17415/220539 objects degraded (7.897%), 11 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:27.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:27 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:09:27.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:27 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3344706052' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:09:27.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:27 smithi064 ceph-mon[212750]: osdmap e457: 8 total, 8 up, 8 in 2023-10-09T06:09:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:27 smithi156 ceph-mon[195101]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:09:27.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:27 smithi156 ceph-mon[195101]: osd.0 [v2:172.21.15.64:6802/119405434,v1:172.21.15.64:6803/119405434] boot 2023-10-09T06:09:27.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:27 smithi156 ceph-mon[195101]: osdmap e456: 8 total, 8 up, 8 in 2023-10-09T06:09:27.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:27 smithi156 ceph-mon[195101]: failed to encode map e456 with expected crc 2023-10-09T06:09:27.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:27 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:09:27.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:27 smithi156 ceph-mon[195101]: failed to encode map e456 with expected crc 2023-10-09T06:09:27.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:27 smithi156 ceph-mon[195101]: failed to encode map e456 with expected crc 2023-10-09T06:09:27.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:27 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3344706052' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:09:27.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:27 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 17415/220539 objects degraded (7.897%), 11 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:27.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:27 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:09:27.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:27 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3344706052' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:09:27.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:27 smithi156 ceph-mon[195101]: osdmap e457: 8 total, 8 up, 8 in 2023-10-09T06:09:28.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:28 smithi156 ceph-mon[195101]: pgmap v1143: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 16587/210810 objects degraded (7.868%); 29726/210810 objects misplaced (14.101%); 32 MiB/s, 504 objects/s recovering 2023-10-09T06:09:28.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:28 smithi156 ceph-mon[195101]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:28 smithi156 ceph-mon[195101]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:28 smithi156 ceph-mon[195101]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:28 smithi156 ceph-mon[195101]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:28 smithi156 ceph-mon[195101]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:28 smithi156 ceph-mon[195101]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:28 smithi156 ceph-mon[195101]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:28 smithi156 ceph-mon[195101]: 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 455'8157 2023-10-09T06:09:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:28 smithi156 ceph-mon[195101]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:28 smithi156 ceph-mon[195101]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:28 smithi156 ceph-mon[195101]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:28 smithi156 ceph-mon[195101]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:28 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:28.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:28 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:28.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:28 smithi156 ceph-mon[195101]: osdmap e458: 8 total, 8 up, 8 in 2023-10-09T06:09:28.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[220044]: pgmap v1143: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 16587/210810 objects degraded (7.868%); 29726/210810 objects misplaced (14.101%); 32 MiB/s, 504 objects/s recovering 2023-10-09T06:09:28.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[220044]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[220044]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[220044]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[220044]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[220044]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[220044]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[220044]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[220044]: 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 455'8157 2023-10-09T06:09:28.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[220044]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[220044]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[220044]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[220044]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:28.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:28.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[220044]: osdmap e458: 8 total, 8 up, 8 in 2023-10-09T06:09:28.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[212750]: pgmap v1143: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 66 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 16587/210810 objects degraded (7.868%); 29726/210810 objects misplaced (14.101%); 32 MiB/s, 504 objects/s recovering 2023-10-09T06:09:28.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[212750]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[212750]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[212750]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[212750]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[212750]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[212750]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[212750]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[212750]: 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 455'8157 2023-10-09T06:09:28.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[212750]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[212750]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[212750]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[212750]: failed to encode map e457 with expected crc 2023-10-09T06:09:28.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:28.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:28.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:28 smithi064 ceph-mon[212750]: osdmap e458: 8 total, 8 up, 8 in 2023-10-09T06:09:29.652 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:09:29.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:29.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:09:29.767 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:09:29.794 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:29.844 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:09:29.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:29.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:09:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:29 smithi156 ceph-mon[195101]: failed to encode map e458 with expected crc 2023-10-09T06:09:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:29 smithi156 ceph-mon[195101]: failed to encode map e458 with expected crc 2023-10-09T06:09:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:29 smithi156 ceph-mon[195101]: failed to encode map e458 with expected crc 2023-10-09T06:09:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:29 smithi156 ceph-mon[195101]: failed to encode map e458 with expected crc 2023-10-09T06:09:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:29 smithi156 ceph-mon[195101]: failed to encode map e458 with expected crc 2023-10-09T06:09:29.951 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:09:29.978 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:29.990 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:29 smithi064 ceph-mon[212750]: failed to encode map e458 with expected crc 2023-10-09T06:09:29.990 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:29 smithi064 ceph-mon[212750]: failed to encode map e458 with expected crc 2023-10-09T06:09:29.990 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:29 smithi064 ceph-mon[212750]: failed to encode map e458 with expected crc 2023-10-09T06:09:29.990 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:29 smithi064 ceph-mon[212750]: failed to encode map e458 with expected crc 2023-10-09T06:09:29.990 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:29 smithi064 ceph-mon[212750]: failed to encode map e458 with expected crc 2023-10-09T06:09:29.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:29 smithi064 ceph-mon[220044]: failed to encode map e458 with expected crc 2023-10-09T06:09:29.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:29 smithi064 ceph-mon[220044]: failed to encode map e458 with expected crc 2023-10-09T06:09:29.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:29 smithi064 ceph-mon[220044]: failed to encode map e458 with expected crc 2023-10-09T06:09:29.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:29 smithi064 ceph-mon[220044]: failed to encode map e458 with expected crc 2023-10-09T06:09:29.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:29 smithi064 ceph-mon[220044]: failed to encode map e458 with expected crc 2023-10-09T06:09:30.001 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:09:30.042 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:09:30.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:30.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:09:30.148 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:09:30.183 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:30.254 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:09:30.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:30.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:09:30.359 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:09:30.438 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:30.473 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:30.593 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:09:30.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:30.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:09:30.706 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:09:30.792 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:30.859 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:30 smithi064 ceph-mon[212750]: pgmap v1146: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+undersized+degraded, 69 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 16837/210396 objects degraded (8.003%); 29726/210396 objects misplaced (14.129%); 20 MiB/s, 313 objects/s recovering 2023-10-09T06:09:30.859 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:30 smithi064 ceph-mon[212750]: 1.d scrub starts 2023-10-09T06:09:30.859 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:30 smithi064 ceph-mon[220044]: pgmap v1146: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+undersized+degraded, 69 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 16837/210396 objects degraded (8.003%); 29726/210396 objects misplaced (14.129%); 20 MiB/s, 313 objects/s recovering 2023-10-09T06:09:30.859 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:30 smithi064 ceph-mon[220044]: 1.d scrub starts 2023-10-09T06:09:30.913 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:09:30.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:30.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:09:31.020 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:09:31.052 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:30 smithi156 ceph-mon[195101]: pgmap v1146: 119 pgs: 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+undersized+degraded, 69 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 16837/210396 objects degraded (8.003%); 29726/210396 objects misplaced (14.129%); 20 MiB/s, 313 objects/s recovering 2023-10-09T06:09:31.052 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:30 smithi156 ceph-mon[195101]: 1.d scrub starts 2023-10-09T06:09:31.053 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:31.108 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:31.124 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:09:31.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:31.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:09:31.228 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:09:31.266 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:31.364 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:09:31.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:31.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:09:31.469 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:09:31.507 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:31.554 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:09:31.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:31.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:09:31.660 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:09:31.688 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:31.746 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:09:31.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:31.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:09:31.864 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:09:31.965 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:32.245 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:09:32.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:32.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:09:32.297 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:09:32.364 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:09:32.400 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:32.480 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:09:32.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:32.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:09:32.585 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:09:32.684 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:32.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:32 smithi156 ceph-mon[195101]: pgmap v1147: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 105 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 16263/205329 objects degraded (7.920%); 25720/205329 objects misplaced (12.526%); 30 MiB/s, 490 objects/s recovering 2023-10-09T06:09:32.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:32 smithi156 ceph-mon[195101]: 1.2 scrub starts 2023-10-09T06:09:32.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:32 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:09:32.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:32 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 16263/205329 objects degraded (7.920%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:32.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:32 smithi064 ceph-mon[212750]: pgmap v1147: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 105 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 16263/205329 objects degraded (7.920%); 25720/205329 objects misplaced (12.526%); 30 MiB/s, 490 objects/s recovering 2023-10-09T06:09:32.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:32 smithi064 ceph-mon[212750]: 1.2 scrub starts 2023-10-09T06:09:32.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:32 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:09:32.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:32 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 16263/205329 objects degraded (7.920%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:32.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:32 smithi064 ceph-mon[220044]: pgmap v1147: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 105 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 16263/205329 objects degraded (7.920%); 25720/205329 objects misplaced (12.526%); 30 MiB/s, 490 objects/s recovering 2023-10-09T06:09:32.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:32 smithi064 ceph-mon[220044]: 1.2 scrub starts 2023-10-09T06:09:32.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:32 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:09:32.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:32 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 16263/205329 objects degraded (7.920%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:33.017 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:09:33.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:33.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:09:33.036 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:09:33.060 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:09:33.092 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:09:33.126 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:09:33.133 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:09:33.201 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:33.333 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:09:33.333 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:09:33.333 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:09:33.333 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:09:33.333 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:09:33.333 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:09:33.334 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:09:33.334 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:09:33.334 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:09:33.334 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:09:33.334 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:09:33.334 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:09:33.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:33 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/12908343' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:09:33.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:33 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/12908343' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:09:33.985 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:33 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/12908343' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:09:34.417 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:09:34.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:34.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:09:34.522 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:09:34.618 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:34.749 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:09:34.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:34.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:09:34.787 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:34 smithi064 ceph-mon[212750]: pgmap v1148: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 297 op/s; 16166/205026 objects degraded (7.885%); 25997/205026 objects misplaced (12.680%); 19 MiB/s, 314 objects/s recovering 2023-10-09T06:09:34.787 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:34 smithi064 ceph-mon[220044]: pgmap v1148: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 297 op/s; 16166/205026 objects degraded (7.885%); 25997/205026 objects misplaced (12.680%); 19 MiB/s, 314 objects/s recovering 2023-10-09T06:09:34.863 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:09:34.921 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:34 smithi156 ceph-mon[195101]: pgmap v1148: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 297 op/s; 16166/205026 objects degraded (7.885%); 25997/205026 objects misplaced (12.680%); 19 MiB/s, 314 objects/s recovering 2023-10-09T06:09:34.922 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:34.996 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:09:35.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:35.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:09:35.020 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1, 2] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 4, 0] 2023-10-09T06:09:35.020 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:09:35.020 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T06:09:35.020 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:09:35.020 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:09:35.021 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T06:09:35.024 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:35.105 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:09:35.164 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:35.176 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:09:35.176 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:09:35.273 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:09:35.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:35.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:09:35.380 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:09:35.465 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:09:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:09:35] "GET /metrics HTTP/1.1" 200 36330 "" "Prometheus/2.43.0" 2023-10-09T06:09:35.501 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:35.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:35 smithi064 ceph-mon[212750]: pgmap v1149: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 281 op/s; 16083/204024 objects degraded (7.883%); 25779/204024 objects misplaced (12.635%); 17 MiB/s, 279 objects/s recovering 2023-10-09T06:09:35.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:35 smithi064 ceph-mon[220044]: pgmap v1149: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 281 op/s; 16083/204024 objects degraded (7.883%); 25779/204024 objects misplaced (12.635%); 17 MiB/s, 279 objects/s recovering 2023-10-09T06:09:35.759 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:09:35.878 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:09:35.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:35.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:09:35.983 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:09:36.038 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:35 smithi156 ceph-mon[195101]: pgmap v1149: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 281 op/s; 16083/204024 objects degraded (7.883%); 25779/204024 objects misplaced (12.635%); 17 MiB/s, 279 objects/s recovering 2023-10-09T06:09:36.045 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:36.228 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:09:36.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:36.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:09:36.334 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:09:36.417 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:36.559 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:09:36.746 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:09:36.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:36.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:09:36.858 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:09:36.896 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:36.900 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:09:38.077 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:38.133 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:38.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:38 smithi064 ceph-mon[212750]: pgmap v1150: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 503 op/s; 15974/196452 objects degraded (8.131%); 24820/196452 objects misplaced (12.634%); 16 MiB/s, 262 objects/s recovering 2023-10-09T06:09:38.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:38 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 15974/196452 objects degraded (8.131%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:38.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:38 smithi064 ceph-mon[220044]: pgmap v1150: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 503 op/s; 15974/196452 objects degraded (8.131%); 24820/196452 objects misplaced (12.634%); 16 MiB/s, 262 objects/s recovering 2023-10-09T06:09:38.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:38 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 15974/196452 objects degraded (8.131%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:38.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:38 smithi156 ceph-mon[195101]: pgmap v1150: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 503 op/s; 15974/196452 objects degraded (8.131%); 24820/196452 objects misplaced (12.634%); 16 MiB/s, 262 objects/s recovering 2023-10-09T06:09:38.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:38 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 15974/196452 objects degraded (8.131%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:39.562 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:09:39.733 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:09:39.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:39.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:09:39.808 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:39 smithi156 ceph-mon[195101]: 1.d scrub starts 2023-10-09T06:09:39.809 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:39 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2619829274' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:09:39.809 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:39 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:09:39.838 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:09:39.901 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:39.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:39 smithi064 ceph-mon[220044]: 1.d scrub starts 2023-10-09T06:09:39.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:39 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2619829274' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:09:39.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:39 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:09:39.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:39 smithi064 ceph-mon[212750]: 1.d scrub starts 2023-10-09T06:09:39.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:39 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2619829274' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:09:39.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:39 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:09:40.134 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:09:40.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:40.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:09:40.246 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:09:40.297 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:40.494 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:09:40.494 INFO:teuthology.orchestra.run.smithi156.stdout: "success": "" 2023-10-09T06:09:40.494 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:09:40.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:40 smithi156 ceph-mon[195101]: pgmap v1151: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 15974/195918 objects degraded (8.153%); 24820/195918 objects misplaced (12.669%); 15 MiB/s, 237 objects/s recovering 2023-10-09T06:09:40.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:40 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:09:40.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:40 smithi156 ceph-mon[195101]: failed to encode map e459 with expected crc 2023-10-09T06:09:40.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:40 smithi156 ceph-mon[195101]: failed to encode map e459 with expected crc 2023-10-09T06:09:40.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:40 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:09:40.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:40 smithi156 ceph-mon[195101]: osdmap e459: 8 total, 8 up, 8 in 2023-10-09T06:09:40.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:40 smithi064 ceph-mon[212750]: pgmap v1151: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 15974/195918 objects degraded (8.153%); 24820/195918 objects misplaced (12.669%); 15 MiB/s, 237 objects/s recovering 2023-10-09T06:09:40.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:40 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:09:40.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:40 smithi064 ceph-mon[212750]: failed to encode map e459 with expected crc 2023-10-09T06:09:40.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:40 smithi064 ceph-mon[212750]: failed to encode map e459 with expected crc 2023-10-09T06:09:40.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:40 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:09:40.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:40 smithi064 ceph-mon[212750]: osdmap e459: 8 total, 8 up, 8 in 2023-10-09T06:09:40.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:40 smithi064 ceph-mon[220044]: pgmap v1151: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 15974/195918 objects degraded (8.153%); 24820/195918 objects misplaced (12.669%); 15 MiB/s, 237 objects/s recovering 2023-10-09T06:09:40.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:40 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:09:40.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:40 smithi064 ceph-mon[220044]: failed to encode map e459 with expected crc 2023-10-09T06:09:40.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:40 smithi064 ceph-mon[220044]: failed to encode map e459 with expected crc 2023-10-09T06:09:40.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:40 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:09:40.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:40 smithi064 ceph-mon[220044]: osdmap e459: 8 total, 8 up, 8 in 2023-10-09T06:09:41.237 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:09:41.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:41.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:09:41.352 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:09:41.441 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:41.618 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:09:41.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:41.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:09:41.734 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:09:41.790 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:09:41.838 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:09:41.854 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:41 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:41.854 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:41 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:41.854 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:41 smithi064 ceph-mon[212750]: pgmap v1153: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 15903/188124 objects degraded (8.453%); 23429/188124 objects misplaced (12.454%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T06:09:41.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:41 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:41.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:41 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:41.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:41 smithi064 ceph-mon[220044]: pgmap v1153: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 15903/188124 objects degraded (8.453%); 23429/188124 objects misplaced (12.454%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T06:09:41.860 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:41.911 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:09:41.957 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:09:41.972 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:09:41.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:41.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:09:41.998 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:41 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:41.998 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:41 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:41.998 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:41 smithi156 ceph-mon[195101]: pgmap v1153: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 15903/188124 objects degraded (8.453%); 23429/188124 objects misplaced (12.454%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T06:09:42.011 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:09:42.059 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:09:42.081 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:09:42.109 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:09:42.152 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:09:42.168 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:09:42.168 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:09:42.169 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:09:42.169 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:09:42.169 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:09:42.169 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:09:42.169 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:09:42.169 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:09:42.203 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:42.564 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:09:42.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:42.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:09:42.672 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:09:42.705 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:09:42.705 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:09:42.705 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:09:42.705 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35940.0:141084 9.0 e459/433 9:038a1570:::benchmark_data_smithi064_511911_object39982:head v 459'9123, mlcod=0'0)", 2023-10-09T06:09:42.705 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:42.693668+0000", 2023-10-09T06:09:42.706 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.003516113, 2023-10-09T06:09:42.706 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035504930000000001, 2023-10-09T06:09:42.706 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:09:42.706 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:09:42.706 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:09:42.706 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:09:42.706 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:09:42.706 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:42.693668+0000", 2023-10-09T06:09:42.706 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:09:42.706 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:09:42.707 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:09:42.707 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:09:42.707 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:42.693668+0000", 2023-10-09T06:09:42.707 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:09:42.707 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:09:42.707 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:09:42.707 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:09:42.707 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:42.693667+0000", 2023-10-09T06:09:42.707 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:09:42.707 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:09:42.707 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:09:42.707 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:09:42.708 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:42.693677+0000", 2023-10-09T06:09:42.708 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8789999999999998e-06 2023-10-09T06:09:42.708 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:09:42.708 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:09:42.708 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:09:42.708 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:42.693694+0000", 2023-10-09T06:09:42.708 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6495999999999999e-05 2023-10-09T06:09:42.708 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:09:42.708 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:09:42.708 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:09:42.708 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:42.693702+0000", 2023-10-09T06:09:42.708 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9039999999999992e-06 2023-10-09T06:09:42.709 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:09:42.709 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:09:42.709 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:09:42.709 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:42.693808+0000", 2023-10-09T06:09:42.709 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010603399999999999 2023-10-09T06:09:42.709 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:09:42.709 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:09:42.709 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:09:42.709 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:42.693842+0000", 2023-10-09T06:09:42.709 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4171000000000002e-05 2023-10-09T06:09:42.709 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:09:42.709 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:09:42.710 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:09:42.710 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:09:42.710 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-10-09T06:09:42.710 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 1 2023-10-09T06:09:42.710 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:09:42.710 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:42.785 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:09:42.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:42.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:09:42.824 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:42 smithi064 ceph-mon[212750]: 1.2 scrub starts 2023-10-09T06:09:42.824 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:42 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2024661764' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:09:42.824 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:42 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 15903/188124 objects degraded (8.453%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:42.826 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:42 smithi064 ceph-mon[220044]: 1.2 scrub starts 2023-10-09T06:09:42.827 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:42 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2024661764' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:09:42.827 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:42 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 15903/188124 objects degraded (8.453%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:42.892 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:09:42.975 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:43.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:42 smithi156 ceph-mon[195101]: 1.2 scrub starts 2023-10-09T06:09:43.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:42 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2024661764' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:09:43.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:42 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 15903/188124 objects degraded (8.453%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:43.346 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:09:43.920 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:09:43.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:43.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:09:44.034 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:09:44.062 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:44.748 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:09:44.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:44.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:09:44.794 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:44 smithi064 ceph-mon[212750]: pgmap v1154: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 15800/187542 objects degraded (8.425%); 23429/187542 objects misplaced (12.493%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T06:09:44.794 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:44 smithi064 ceph-mon[212750]: 1.d scrub starts 2023-10-09T06:09:44.794 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:44.794 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:44.794 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:44 smithi064 ceph-mon[220044]: pgmap v1154: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 15800/187542 objects degraded (8.425%); 23429/187542 objects misplaced (12.493%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T06:09:44.794 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:44 smithi064 ceph-mon[220044]: 1.d scrub starts 2023-10-09T06:09:44.795 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:44.795 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:44.855 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:09:44.861 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:09:44.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:44 smithi156 ceph-mon[195101]: pgmap v1154: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 15800/187542 objects degraded (8.425%); 23429/187542 objects misplaced (12.493%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T06:09:44.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:44 smithi156 ceph-mon[195101]: 1.d scrub starts 2023-10-09T06:09:44.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:44.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:44.899 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:44.984 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:09:44.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:44.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:09:45.003 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:45.095 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:09:45.128 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:45.189 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:09:45.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:45.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:09:45.298 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:09:45.365 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:45.523 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:45 smithi156 ceph-mon[195101]: 1.2 scrub starts 2023-10-09T06:09:45.614 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:45 smithi064 ceph-mon[212750]: 1.2 scrub starts 2023-10-09T06:09:45.614 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:09:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:09:45] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.43.0" 2023-10-09T06:09:45.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:45 smithi064 ceph-mon[220044]: 1.2 scrub starts 2023-10-09T06:09:45.615 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:09:45.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:45.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:09:45.723 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:09:45.755 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:45.835 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:09:45.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:45.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:09:45.946 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:09:46.045 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:46.192 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:09:46.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:46.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:09:46.310 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:09:46.422 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:46.622 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:09:46.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:46 smithi064 ceph-mon[212750]: pgmap v1155: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 15711/186468 objects degraded (8.426%); 23184/186468 objects misplaced (12.433%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T06:09:46.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:46 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:46.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:46 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:46.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:46 smithi064 ceph-mon[220044]: pgmap v1155: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 15711/186468 objects degraded (8.426%); 23184/186468 objects misplaced (12.433%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T06:09:46.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:46 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:46.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:46 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:46.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:46 smithi156 ceph-mon[195101]: pgmap v1155: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 15711/186468 objects degraded (8.426%); 23184/186468 objects misplaced (12.433%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T06:09:46.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:46 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:46.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:46 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:47.031 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1, 2] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 4, 0] 2023-10-09T06:09:47.031 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:09:47.032 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-10-09T06:09:47.032 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:09:47.032 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:09:47.032 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-10-09T06:09:47.033 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:09:47.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:47.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:09:47.145 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:09:47.174 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:47.224 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:09:47.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:47.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:09:47.297 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:47.329 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:09:47.380 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:47.469 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:47.483 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:09:47.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:47.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:09:47.593 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:09:47.637 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:47 smithi064 ceph-mon[212750]: 1.d scrub starts 2023-10-09T06:09:47.637 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:47 smithi064 ceph-mon[212750]: osdmap e460: 8 total, 8 up, 8 in 2023-10-09T06:09:47.637 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:47 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:09:47.637 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:47 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:09:47.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:47 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:47.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:47 smithi064 ceph-mon[220044]: 1.d scrub starts 2023-10-09T06:09:47.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:47 smithi064 ceph-mon[220044]: osdmap e460: 8 total, 8 up, 8 in 2023-10-09T06:09:47.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:47 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:09:47.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:47 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:09:47.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:47 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:47.641 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:47.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:47 smithi156 ceph-mon[195101]: 1.d scrub starts 2023-10-09T06:09:47.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:47 smithi156 ceph-mon[195101]: osdmap e460: 8 total, 8 up, 8 in 2023-10-09T06:09:47.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:47 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:09:47.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:47 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:09:47.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:47 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:47.872 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:09:47.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:47.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:09:47.977 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:09:48.059 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:48.162 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:48.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:48 smithi064 ceph-mon[212750]: failed to encode map e460 with expected crc 2023-10-09T06:09:48.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:48 smithi064 ceph-mon[212750]: failed to encode map e460 with expected crc 2023-10-09T06:09:48.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:48 smithi064 ceph-mon[212750]: pgmap v1157: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 3.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 14847/178608 objects degraded (8.313%); 20259/178608 objects misplaced (11.343%); 7.3 MiB/s, 143 objects/s recovering 2023-10-09T06:09:48.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:48 smithi064 ceph-mon[212750]: 1.2 scrub starts 2023-10-09T06:09:48.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:48 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 15711/186468 objects degraded (8.426%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:48.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:48 smithi064 ceph-mon[212750]: failed to encode map e461 with expected crc 2023-10-09T06:09:48.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:48 smithi064 ceph-mon[212750]: osdmap e461: 8 total, 8 up, 8 in 2023-10-09T06:09:48.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:48 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3756744514' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:09:48.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:48 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:09:48.840 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:09:48.850 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:48 smithi064 ceph-mon[220044]: failed to encode map e460 with expected crc 2023-10-09T06:09:48.850 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:48 smithi064 ceph-mon[220044]: failed to encode map e460 with expected crc 2023-10-09T06:09:48.850 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:48 smithi064 ceph-mon[220044]: pgmap v1157: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 3.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 14847/178608 objects degraded (8.313%); 20259/178608 objects misplaced (11.343%); 7.3 MiB/s, 143 objects/s recovering 2023-10-09T06:09:48.850 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:48 smithi064 ceph-mon[220044]: 1.2 scrub starts 2023-10-09T06:09:48.850 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:48 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 15711/186468 objects degraded (8.426%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:48.850 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:48 smithi064 ceph-mon[220044]: failed to encode map e461 with expected crc 2023-10-09T06:09:48.850 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:48 smithi064 ceph-mon[220044]: osdmap e461: 8 total, 8 up, 8 in 2023-10-09T06:09:48.851 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:48 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3756744514' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:09:48.851 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:48 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:09:48.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:48.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:09:48.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:48 smithi156 ceph-mon[195101]: failed to encode map e460 with expected crc 2023-10-09T06:09:48.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:48 smithi156 ceph-mon[195101]: failed to encode map e460 with expected crc 2023-10-09T06:09:48.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:48 smithi156 ceph-mon[195101]: pgmap v1157: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 3.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 14847/178608 objects degraded (8.313%); 20259/178608 objects misplaced (11.343%); 7.3 MiB/s, 143 objects/s recovering 2023-10-09T06:09:48.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:48 smithi156 ceph-mon[195101]: 1.2 scrub starts 2023-10-09T06:09:48.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:48 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 15711/186468 objects degraded (8.426%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:48.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:48 smithi156 ceph-mon[195101]: failed to encode map e461 with expected crc 2023-10-09T06:09:48.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:48 smithi156 ceph-mon[195101]: osdmap e461: 8 total, 8 up, 8 in 2023-10-09T06:09:48.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:48 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3756744514' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:09:48.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:48 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:09:48.952 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:09:49.035 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:09:49.043 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:49.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:49 smithi156 ceph-mon[195101]: failed to encode map e461 with expected crc 2023-10-09T06:09:49.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:49 smithi156 ceph-mon[195101]: failed to encode map e461 with expected crc 2023-10-09T06:09:49.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:49 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:09:49.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:49 smithi156 ceph-mon[195101]: failed to encode map e462 with expected crc 2023-10-09T06:09:49.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:49 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:09:49.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:49 smithi156 ceph-mon[195101]: osdmap e462: 8 total, 8 up, 8 in 2023-10-09T06:09:49.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:49 smithi064 ceph-mon[220044]: failed to encode map e461 with expected crc 2023-10-09T06:09:49.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:49 smithi064 ceph-mon[220044]: failed to encode map e461 with expected crc 2023-10-09T06:09:49.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:49 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:09:49.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:49 smithi064 ceph-mon[220044]: failed to encode map e462 with expected crc 2023-10-09T06:09:49.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:49 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:09:49.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:49 smithi064 ceph-mon[220044]: osdmap e462: 8 total, 8 up, 8 in 2023-10-09T06:09:49.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:49 smithi064 ceph-mon[212750]: failed to encode map e461 with expected crc 2023-10-09T06:09:49.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:49 smithi064 ceph-mon[212750]: failed to encode map e461 with expected crc 2023-10-09T06:09:49.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:49 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:09:49.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:49 smithi064 ceph-mon[212750]: failed to encode map e462 with expected crc 2023-10-09T06:09:49.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:49 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:09:49.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:49 smithi064 ceph-mon[212750]: osdmap e462: 8 total, 8 up, 8 in 2023-10-09T06:09:50.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:50 smithi064 ceph-mon[212750]: failed to encode map e462 with expected crc 2023-10-09T06:09:50.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:50 smithi064 ceph-mon[212750]: failed to encode map e462 with expected crc 2023-10-09T06:09:50.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:50 smithi064 ceph-mon[212750]: pgmap v1160: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 3.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 14847/178125 objects degraded (8.335%); 20259/178125 objects misplaced (11.373%); 4.0 MiB/s, 99 objects/s recovering 2023-10-09T06:09:50.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:50 smithi064 ceph-mon[220044]: failed to encode map e462 with expected crc 2023-10-09T06:09:50.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:50 smithi064 ceph-mon[220044]: failed to encode map e462 with expected crc 2023-10-09T06:09:50.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:50 smithi064 ceph-mon[220044]: pgmap v1160: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 3.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 14847/178125 objects degraded (8.335%); 20259/178125 objects misplaced (11.373%); 4.0 MiB/s, 99 objects/s recovering 2023-10-09T06:09:50.783 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:09:50.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:50.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:09:50.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:50 smithi156 ceph-mon[195101]: failed to encode map e462 with expected crc 2023-10-09T06:09:50.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:50 smithi156 ceph-mon[195101]: failed to encode map e462 with expected crc 2023-10-09T06:09:50.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:50 smithi156 ceph-mon[195101]: pgmap v1160: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 3.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 14847/178125 objects degraded (8.335%); 20259/178125 objects misplaced (11.373%); 4.0 MiB/s, 99 objects/s recovering 2023-10-09T06:09:50.896 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:09:50.934 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:51.014 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:09:51.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:51.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:09:51.121 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:09:51.245 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:51.454 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:09:51.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:51.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:09:51.559 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:09:51.603 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:51.671 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:09:51.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:51.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:09:51.778 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:09:51.853 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:52.067 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:09:52.067 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:09:52.067 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:09:52.068 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:09:52.068 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:09:52.097 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:09:52.132 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:09:52.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:52.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:09:52.165 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:09:52.241 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:09:52.267 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:52.316 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:09:52.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:52.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:09:52.334 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:09:52.334 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:09:52.334 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:09:52.334 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:09:52.334 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:09:52.334 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:09:52.334 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:09:52.334 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:09:52.335 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:09:52.423 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:09:52.451 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:52.520 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:09:52.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:52.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:09:52.626 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:09:52.711 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:52.875 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:52 smithi156 ceph-mon[195101]: pgmap v1161: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 855 op/s; 14089/171066 objects degraded (8.236%); 19699/171066 objects misplaced (11.515%); 6.5 MiB/s, 141 objects/s recovering 2023-10-09T06:09:52.875 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:52 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/103401317' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:09:52.876 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:52 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 14089/171066 objects degraded (8.236%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:09:52.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:52 smithi064 ceph-mon[220044]: pgmap v1161: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 855 op/s; 14089/171066 objects degraded (8.236%); 19699/171066 objects misplaced (11.515%); 6.5 MiB/s, 141 objects/s recovering 2023-10-09T06:09:52.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:52 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/103401317' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:09:52.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:52 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 14089/171066 objects degraded (8.236%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:09:52.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:52 smithi064 ceph-mon[212750]: pgmap v1161: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 855 op/s; 14089/171066 objects degraded (8.236%); 19699/171066 objects misplaced (11.515%); 6.5 MiB/s, 141 objects/s recovering 2023-10-09T06:09:52.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:52 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/103401317' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:09:52.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:52 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 14089/171066 objects degraded (8.236%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:09:52.986 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:09:52.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:52.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:09:53.093 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:09:53.108 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:09:53.161 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:53.956 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:09:53.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:53.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:09:54.005 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:09:54.005 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:09:54.063 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:09:54.150 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:54.577 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:54 smithi064 ceph-mon[212750]: pgmap v1162: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 442 op/s; 13986/170583 objects degraded (8.199%); 19505/170583 objects misplaced (11.434%); 7.0 MiB/s, 113 objects/s recovering 2023-10-09T06:09:54.577 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:54 smithi064 ceph-mon[220044]: pgmap v1162: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 442 op/s; 13986/170583 objects degraded (8.199%); 19505/170583 objects misplaced (11.434%); 7.0 MiB/s, 113 objects/s recovering 2023-10-09T06:09:54.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:54 smithi156 ceph-mon[195101]: pgmap v1162: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 442 op/s; 13986/170583 objects degraded (8.199%); 19505/170583 objects misplaced (11.434%); 7.0 MiB/s, 113 objects/s recovering 2023-10-09T06:09:55.107 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:09:55.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:55.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:09:55.212 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:09:55.253 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:55.259 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:09:55.317 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:09:55.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:55.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:09:55.413 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:09:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:09:55] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.43.0" 2023-10-09T06:09:55.423 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:09:55.541 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:55.579 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:55.696 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:09:55.697 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "" 2023-10-09T06:09:55.697 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:09:55.704 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:09:55.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:55.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:09:55.823 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:09:55.864 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:55.932 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:09:55.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:55.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:09:55.954 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:09:56.038 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:09:56.190 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:56.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:56 smithi156 ceph-mon[195101]: pgmap v1163: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 380 op/s; 13897/169461 objects degraded (8.201%); 19234/169461 objects misplaced (11.350%); 7.4 MiB/s, 119 objects/s recovering 2023-10-09T06:09:56.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:56 smithi064 ceph-mon[212750]: pgmap v1163: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 380 op/s; 13897/169461 objects degraded (8.201%); 19234/169461 objects misplaced (11.350%); 7.4 MiB/s, 119 objects/s recovering 2023-10-09T06:09:56.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:56 smithi064 ceph-mon[220044]: pgmap v1163: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 380 op/s; 13897/169461 objects degraded (8.201%); 19234/169461 objects misplaced (11.350%); 7.4 MiB/s, 119 objects/s recovering 2023-10-09T06:09:57.028 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:09:57.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:57.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:09:57.136 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:09:57.182 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:57.248 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:09:57.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:57.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:09:57.353 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:09:57.409 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:57.492 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:09:57.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:57.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:09:57.600 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:57 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 13897/169461 objects degraded (8.201%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:09:57.606 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:09:57.692 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:57.798 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:09:57.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:57.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:09:57.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:57 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 13897/169461 objects degraded (8.201%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:09:57.910 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:09:57.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:57 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 13897/169461 objects degraded (8.201%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:09:58.006 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:58.124 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:58.190 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:09:58.243 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:09:58.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:58.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:09:58.362 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:09:58.404 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:58.494 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:09:58.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:58.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:09:58.601 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:09:58.729 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:58.848 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:58 smithi064 ceph-mon[212750]: pgmap v1164: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 13777/161724 objects degraded (8.519%); 18100/161724 objects misplaced (11.192%); 9.1 MiB/s, 147 objects/s recovering 2023-10-09T06:09:58.848 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:58 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:58.848 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:58.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:58 smithi064 ceph-mon[220044]: pgmap v1164: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 13777/161724 objects degraded (8.519%); 18100/161724 objects misplaced (11.192%); 9.1 MiB/s, 147 objects/s recovering 2023-10-09T06:09:58.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:58 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:58.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:58 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:58.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:58 smithi156 ceph-mon[195101]: pgmap v1164: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 13777/161724 objects degraded (8.519%); 18100/161724 objects misplaced (11.192%); 9.1 MiB/s, 147 objects/s recovering 2023-10-09T06:09:58.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:58 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:09:58.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:58 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:59.447 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:09:59.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:59.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:09:59.553 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:09:59.583 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:59.636 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:09:59.724 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:09:59 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2072789808' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:09:59.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:09:59 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2072789808' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:09:59.724 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:09:59.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:59.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:09:59.832 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:09:59.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:09:59 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2072789808' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:09:59.874 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:09:59.943 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:09:59.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:09:59.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:10:00.052 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:10:00.122 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:00.836 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:10:00.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:00.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[212750]: pgmap v1165: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 13777/161157 objects degraded (8.549%); 18100/161157 objects misplaced (11.231%); 8.2 MiB/s, 132 objects/s recovering 2023-10-09T06:10:00.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:10:00.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2072789808' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:10:00.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[212750]: osdmap e463: 8 total, 8 up, 8 in 2023-10-09T06:10:00.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[212750]: failed to encode map e463 with expected crc 2023-10-09T06:10:00.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[212750]: failed to encode map e463 with expected crc 2023-10-09T06:10:00.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[212750]: failed to encode map e463 with expected crc 2023-10-09T06:10:00.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[212750]: failed to encode map e463 with expected crc 2023-10-09T06:10:00.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[212750]: failed to encode map e463 with expected crc 2023-10-09T06:10:00.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[212750]: failed to encode map e463 with expected crc 2023-10-09T06:10:00.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[212750]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 13777/161157 objects degraded (8.549%), 5 pgs degraded, 1 pg undersized 2023-10-09T06:10:00.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[212750]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-10-09T06:10:00.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[212750]: [WRN] PG_DEGRADED: Degraded data redundancy: 13777/161157 objects degraded (8.549%), 5 pgs degraded, 1 pg undersized 2023-10-09T06:10:00.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[212750]: pg 9.0 is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [1,6] 2023-10-09T06:10:00.843 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[212750]: pg 9.5 is active+recovering+undersized+degraded+remapped, acting [5,7] 2023-10-09T06:10:00.843 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[212750]: pg 9.8 is active+recovery_wait+undersized+degraded+remapped, acting [3,7] 2023-10-09T06:10:00.843 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[212750]: pg 9.9 is active+recovering+undersized+degraded+remapped, acting [7,5] 2023-10-09T06:10:00.843 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[212750]: pg 9.b is active+recovery_wait+undersized+degraded+remapped, acting [6,5] 2023-10-09T06:10:00.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:10:00.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[220044]: pgmap v1165: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 13777/161157 objects degraded (8.549%); 18100/161157 objects misplaced (11.231%); 8.2 MiB/s, 132 objects/s recovering 2023-10-09T06:10:00.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:10:00.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2072789808' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:10:00.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[220044]: osdmap e463: 8 total, 8 up, 8 in 2023-10-09T06:10:00.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[220044]: failed to encode map e463 with expected crc 2023-10-09T06:10:00.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[220044]: failed to encode map e463 with expected crc 2023-10-09T06:10:00.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[220044]: failed to encode map e463 with expected crc 2023-10-09T06:10:00.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[220044]: failed to encode map e463 with expected crc 2023-10-09T06:10:00.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[220044]: failed to encode map e463 with expected crc 2023-10-09T06:10:00.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[220044]: failed to encode map e463 with expected crc 2023-10-09T06:10:00.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[220044]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 13777/161157 objects degraded (8.549%), 5 pgs degraded, 1 pg undersized 2023-10-09T06:10:00.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[220044]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-10-09T06:10:00.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[220044]: [WRN] PG_DEGRADED: Degraded data redundancy: 13777/161157 objects degraded (8.549%), 5 pgs degraded, 1 pg undersized 2023-10-09T06:10:00.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[220044]: pg 9.0 is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [1,6] 2023-10-09T06:10:00.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[220044]: pg 9.5 is active+recovering+undersized+degraded+remapped, acting [5,7] 2023-10-09T06:10:00.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[220044]: pg 9.8 is active+recovery_wait+undersized+degraded+remapped, acting [3,7] 2023-10-09T06:10:00.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[220044]: pg 9.9 is active+recovering+undersized+degraded+remapped, acting [7,5] 2023-10-09T06:10:00.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:00 smithi064 ceph-mon[220044]: pg 9.b is active+recovery_wait+undersized+degraded+remapped, acting [6,5] 2023-10-09T06:10:00.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:00 smithi156 ceph-mon[195101]: pgmap v1165: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 13777/161157 objects degraded (8.549%); 18100/161157 objects misplaced (11.231%); 8.2 MiB/s, 132 objects/s recovering 2023-10-09T06:10:00.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:00 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:10:00.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:00 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2072789808' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:10:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:00 smithi156 ceph-mon[195101]: osdmap e463: 8 total, 8 up, 8 in 2023-10-09T06:10:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:00 smithi156 ceph-mon[195101]: failed to encode map e463 with expected crc 2023-10-09T06:10:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:00 smithi156 ceph-mon[195101]: failed to encode map e463 with expected crc 2023-10-09T06:10:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:00 smithi156 ceph-mon[195101]: failed to encode map e463 with expected crc 2023-10-09T06:10:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:00 smithi156 ceph-mon[195101]: failed to encode map e463 with expected crc 2023-10-09T06:10:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:00 smithi156 ceph-mon[195101]: failed to encode map e463 with expected crc 2023-10-09T06:10:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:00 smithi156 ceph-mon[195101]: failed to encode map e463 with expected crc 2023-10-09T06:10:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:00 smithi156 ceph-mon[195101]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 13777/161157 objects degraded (8.549%), 5 pgs degraded, 1 pg undersized 2023-10-09T06:10:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:00 smithi156 ceph-mon[195101]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-10-09T06:10:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:00 smithi156 ceph-mon[195101]: [WRN] PG_DEGRADED: Degraded data redundancy: 13777/161157 objects degraded (8.549%), 5 pgs degraded, 1 pg undersized 2023-10-09T06:10:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:00 smithi156 ceph-mon[195101]: pg 9.0 is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [1,6] 2023-10-09T06:10:00.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:00 smithi156 ceph-mon[195101]: pg 9.5 is active+recovering+undersized+degraded+remapped, acting [5,7] 2023-10-09T06:10:00.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:00 smithi156 ceph-mon[195101]: pg 9.8 is active+recovery_wait+undersized+degraded+remapped, acting [3,7] 2023-10-09T06:10:00.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:00 smithi156 ceph-mon[195101]: pg 9.9 is active+recovering+undersized+degraded+remapped, acting [7,5] 2023-10-09T06:10:00.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:00 smithi156 ceph-mon[195101]: pg 9.b is active+recovery_wait+undersized+degraded+remapped, acting [6,5] 2023-10-09T06:10:00.944 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:10:00.998 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:01.865 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:10:01.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:01.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:10:01.981 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:10:02.042 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1, 2] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 4, 0] 2023-10-09T06:10:02.042 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:10:02.042 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:10:02.043 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.000000 2023-10-09T06:10:02.043 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 0 2023-10-09T06:10:02.094 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:02.238 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:10:02.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:02.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:10:02.354 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:10:02.405 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:02.460 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:10:02.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:02.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:10:02.568 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:10:02.651 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:02.657 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:10:02.657 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:10:02.739 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:02 smithi156 ceph-mon[195101]: pgmap v1167: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 13683/153174 objects degraded (8.933%); 16847/153174 objects misplaced (10.999%); 8.1 MiB/s, 128 objects/s recovering 2023-10-09T06:10:02.739 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:02 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 13683/153174 objects degraded (8.933%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:10:02.955 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:10:02.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:02.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:10:02.960 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:02 smithi064 ceph-mon[220044]: pgmap v1167: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 13683/153174 objects degraded (8.933%); 16847/153174 objects misplaced (10.999%); 8.1 MiB/s, 128 objects/s recovering 2023-10-09T06:10:02.960 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:02 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 13683/153174 objects degraded (8.933%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:10:02.960 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:02 smithi064 ceph-mon[212750]: pgmap v1167: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 13683/153174 objects degraded (8.933%); 16847/153174 objects misplaced (10.999%); 8.1 MiB/s, 128 objects/s recovering 2023-10-09T06:10:02.960 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:02 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 13683/153174 objects degraded (8.933%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:10:03.061 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:10:03.100 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:03.184 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:10:03.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:03.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:10:03.299 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:10:03.346 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:03.391 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:10:03.391 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:10:03.391 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:10:03.391 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:10:03.391 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:10:03.391 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:10:03.391 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:10:03.419 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:10:03.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:03.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:10:03.541 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:10:03.571 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:03.580 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:10:03.580 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:10:03.580 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:10:03.580 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:10:03.580 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:10:03.580 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:10:03.580 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:10:03.580 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:10:03.581 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:10:03.619 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:10:03.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:03.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:10:03.723 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:10:03.767 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:03 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3539633745' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:10:03.767 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:03.850 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:10:03.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:03.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:10:03.933 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:10:03.957 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:10:03.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:03 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3539633745' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:10:03.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:03 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3539633745' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:10:03.996 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:04.065 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:10:04.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:04.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:10:04.157 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:10:04.172 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:10:04.204 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:04.267 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:10:04.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:04.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:10:04.377 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:10:04.405 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:04.450 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:10:04.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:04.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:10:04.556 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:10:04.606 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:04.654 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:04 smithi064 ceph-mon[212750]: pgmap v1168: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 13580/152541 objects degraded (8.903%); 16647/152541 objects misplaced (10.913%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T06:10:04.654 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:04 smithi064 ceph-mon[220044]: pgmap v1168: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 13580/152541 objects degraded (8.903%); 16647/152541 objects misplaced (10.913%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T06:10:04.673 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:10:04.676 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:04 smithi156 ceph-mon[195101]: pgmap v1168: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 13580/152541 objects degraded (8.903%); 16647/152541 objects misplaced (10.913%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T06:10:04.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:04.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:10:04.778 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:10:04.817 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:04.879 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:10:04.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:04.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:10:04.983 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:10:05.067 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:05.076 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:05.104 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:05.202 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:10:05.206 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:05.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:05.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:10:05.310 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:10:05.350 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:05.419 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:10:05.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:05.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:10:05.470 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:10:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:10:05] "GET /metrics HTTP/1.1" 200 36476 "" "Prometheus/2.43.0" 2023-10-09T06:10:05.527 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:10:05.615 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:06.655 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:06 smithi064 ceph-mon[212750]: pgmap v1169: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 13490/151434 objects degraded (8.908%); 16356/151434 objects misplaced (10.801%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T06:10:06.729 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:10:06.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:06.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:10:06.834 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:10:06.894 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:10:06.902 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:06.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:06 smithi064 ceph-mon[220044]: pgmap v1169: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 13490/151434 objects degraded (8.908%); 16356/151434 objects misplaced (10.801%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T06:10:07.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:06 smithi156 ceph-mon[195101]: pgmap v1169: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 13490/151434 objects degraded (8.908%); 16356/151434 objects misplaced (10.801%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T06:10:07.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:07 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 13490/151434 objects degraded (8.908%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:10:07.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:07 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 13490/151434 objects degraded (8.908%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:10:08.067 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:10:08.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:08.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:10:08.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:07 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 13490/151434 objects degraded (8.908%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:10:08.179 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:10:08.223 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:08.318 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:10:08.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:08.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:10:08.424 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:10:08.525 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:08.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:08.797 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:08 smithi064 ceph-mon[212750]: pgmap v1170: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 13243/144054 objects degraded (9.193%); 15350/144054 objects misplaced (10.656%); 7.6 MiB/s, 135 objects/s recovering 2023-10-09T06:10:08.797 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:08 smithi064 ceph-mon[220044]: pgmap v1170: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 13243/144054 objects degraded (9.193%); 15350/144054 objects misplaced (10.656%); 7.6 MiB/s, 135 objects/s recovering 2023-10-09T06:10:08.798 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:10:08.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:10:08.899 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:10:09.003 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:09.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:08 smithi156 ceph-mon[195101]: pgmap v1170: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 13243/144054 objects degraded (9.193%); 15350/144054 objects misplaced (10.656%); 7.6 MiB/s, 135 objects/s recovering 2023-10-09T06:10:09.149 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:10:09.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:09.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:10:09.263 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:10:09.312 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:09.385 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:10:09.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:09.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:10:09.495 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:10:09.622 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:09.767 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:09.809 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:10:09.809 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:10:09.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.810 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:156993 9.2 9:4f08760e:::benchmark_data_smithi064_511911_object55891:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e463)", 2023-10-09T06:10:09.810 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:10:09.750022+0000", 2023-10-09T06:10:09.810 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.058817446000000002, 2023-10-09T06:10:09.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.058844911999999999, 2023-10-09T06:10:09.810 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:09.810 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:10:09.810 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:09.810 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:10:09.810 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:09.810 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156993 2023-10-09T06:10:09.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.811 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:09.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:09.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.750022+0000", 2023-10-09T06:10:09.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:09.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:09.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.750022+0000", 2023-10-09T06:10:09.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:09.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:09.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.750021+0000", 2023-10-09T06:10:09.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:10:09.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:09.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.750026+0000", 2023-10-09T06:10:09.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4589999999999997e-06 2023-10-09T06:10:09.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:09.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.750036+0000", 2023-10-09T06:10:09.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0029e-05 2023-10-09T06:10:09.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:09.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.750043+0000", 2023-10-09T06:10:09.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6560000000000003e-06 2023-10-09T06:10:09.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:09.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.755680+0000", 2023-10-09T06:10:09.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056373919999999998 2023-10-09T06:10:09.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:10:09.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.755819+0000", 2023-10-09T06:10:09.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013871599999999999 2023-10-09T06:10:09.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:10:09.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.755938+0000", 2023-10-09T06:10:09.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011887099999999999 2023-10-09T06:10:09.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:10:09.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.762662+0000", 2023-10-09T06:10:09.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067240399999999997 2023-10-09T06:10:09.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:09.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.773679+0000", 2023-10-09T06:10:09.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011016966 2023-10-09T06:10:09.816 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:09.816 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:09.816 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:09.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.816 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:157000 9.3 9:c2afa865:::benchmark_data_smithi064_511911_object55898:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e463)", 2023-10-09T06:10:09.817 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:10:09.775796+0000", 2023-10-09T06:10:09.817 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.033043897000000003, 2023-10-09T06:10:09.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.033134853999999998, 2023-10-09T06:10:09.817 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:09.817 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:10:09.817 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:09.817 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:10:09.817 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:09.817 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 157000 2023-10-09T06:10:09.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.818 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:09.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:09.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.775796+0000", 2023-10-09T06:10:09.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:09.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:09.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.775796+0000", 2023-10-09T06:10:09.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:09.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:09.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.775794+0000", 2023-10-09T06:10:09.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:10:09.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:09.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.775804+0000", 2023-10-09T06:10:09.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0837e-05 2023-10-09T06:10:09.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:09.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.775823+0000", 2023-10-09T06:10:09.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8674000000000001e-05 2023-10-09T06:10:09.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:09.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.775834+0000", 2023-10-09T06:10:09.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0696999999999999e-05 2023-10-09T06:10:09.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:09.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.775948+0000", 2023-10-09T06:10:09.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000114011 2023-10-09T06:10:09.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:10:09.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.776121+0000", 2023-10-09T06:10:09.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017287499999999999 2023-10-09T06:10:09.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T06:10:09.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.776286+0000", 2023-10-09T06:10:09.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016509599999999999 2023-10-09T06:10:09.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:09.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.806808+0000", 2023-10-09T06:10:09.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.030522377 2023-10-09T06:10:09.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:09.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.806849+0000", 2023-10-09T06:10:09.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0482000000000003e-05 2023-10-09T06:10:09.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:10:09.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.806871+0000", 2023-10-09T06:10:09.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2408e-05 2023-10-09T06:10:09.824 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:09.824 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:09.824 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:09.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.824 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:157002 9.2 9:4dbd9a00:::benchmark_data_smithi064_511911_object55900:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e463)", 2023-10-09T06:10:09.824 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:10:09.775852+0000", 2023-10-09T06:10:09.824 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.032987463000000002, 2023-10-09T06:10:09.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.033134905999999999, 2023-10-09T06:10:09.825 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:09.825 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:10:09.825 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:09.825 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:10:09.825 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:09.825 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 157002 2023-10-09T06:10:09.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.826 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:09.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:09.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.775852+0000", 2023-10-09T06:10:09.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:09.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:09.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.775852+0000", 2023-10-09T06:10:09.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:09.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:09.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.775852+0000", 2023-10-09T06:10:09.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:10:09.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:09.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.775855+0000", 2023-10-09T06:10:09.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8689999999999997e-06 2023-10-09T06:10:09.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:09.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.775864+0000", 2023-10-09T06:10:09.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9709999999999993e-06 2023-10-09T06:10:09.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:09.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.775871+0000", 2023-10-09T06:10:09.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4810000000000003e-06 2023-10-09T06:10:09.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:09.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.776847+0000", 2023-10-09T06:10:09.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00097582300000000001 2023-10-09T06:10:09.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:10:09.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.776957+0000", 2023-10-09T06:10:09.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110572 2023-10-09T06:10:09.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:10:09.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.777061+0000", 2023-10-09T06:10:09.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000103304 2023-10-09T06:10:09.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:10:09.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.784605+0000", 2023-10-09T06:10:09.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0075444600000000002 2023-10-09T06:10:09.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:09.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.786589+0000", 2023-10-09T06:10:09.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019835109999999999 2023-10-09T06:10:09.831 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:09.831 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:09.831 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:09.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.831 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:157003 9.2 9:466305b3:::benchmark_data_smithi064_511911_object55901:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e463)", 2023-10-09T06:10:09.831 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:10:09.775877+0000", 2023-10-09T06:10:09.831 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.032962801, 2023-10-09T06:10:09.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.033162071000000001, 2023-10-09T06:10:09.831 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:09.832 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:10:09.832 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:09.832 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:10:09.832 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:09.832 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 157003 2023-10-09T06:10:09.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.832 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:09.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:09.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.775877+0000", 2023-10-09T06:10:09.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:09.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:09.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.775877+0000", 2023-10-09T06:10:09.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:09.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:09.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.775876+0000", 2023-10-09T06:10:09.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:10:09.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:09.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.775879+0000", 2023-10-09T06:10:09.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8310000000000001e-06 2023-10-09T06:10:09.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:09.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.775891+0000", 2023-10-09T06:10:09.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1729000000000001e-05 2023-10-09T06:10:09.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:09.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.775897+0000", 2023-10-09T06:10:09.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0179999999999998e-06 2023-10-09T06:10:09.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:09.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.780104+0000", 2023-10-09T06:10:09.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042074160000000003 2023-10-09T06:10:09.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:10:09.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.780237+0000", 2023-10-09T06:10:09.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013241500000000001 2023-10-09T06:10:09.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:10:09.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.780363+0000", 2023-10-09T06:10:09.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012617600000000001 2023-10-09T06:10:09.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:10:09.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.784612+0000", 2023-10-09T06:10:09.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042485470000000001 2023-10-09T06:10:09.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:09.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.786668+0000", 2023-10-09T06:10:09.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020569439999999998 2023-10-09T06:10:09.837 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:09.837 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:09.837 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:09.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.837 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:157009 9.7 9:ec57a7a6:::benchmark_data_smithi064_511911_object55907:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e463)", 2023-10-09T06:10:09.837 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:10:09.794871+0000", 2023-10-09T06:10:09.837 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.013968741999999999, 2023-10-09T06:10:09.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014225326, 2023-10-09T06:10:09.837 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:09.837 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:10:09.837 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:09.838 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:10:09.838 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:09.838 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 157009 2023-10-09T06:10:09.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.838 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:09.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:09.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.794871+0000", 2023-10-09T06:10:09.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:09.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:09.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.794871+0000", 2023-10-09T06:10:09.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:09.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:09.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.794869+0000", 2023-10-09T06:10:09.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T06:10:09.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:09.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.794880+0000", 2023-10-09T06:10:09.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1428e-05 2023-10-09T06:10:09.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:09.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.794902+0000", 2023-10-09T06:10:09.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1891000000000001e-05 2023-10-09T06:10:09.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:09.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.794914+0000", 2023-10-09T06:10:09.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1970000000000001e-05 2023-10-09T06:10:09.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:09.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.794987+0000", 2023-10-09T06:10:09.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2836000000000005e-05 2023-10-09T06:10:09.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:10:09.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.795140+0000", 2023-10-09T06:10:09.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015298500000000001 2023-10-09T06:10:09.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,5,7", 2023-10-09T06:10:09.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.795285+0000", 2023-10-09T06:10:09.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014492699999999999 2023-10-09T06:10:09.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:10:09.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.801839+0000", 2023-10-09T06:10:09.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065542480000000004 2023-10-09T06:10:09.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:09.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.801899+0000", 2023-10-09T06:10:09.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9840999999999998e-05 2023-10-09T06:10:09.843 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:09.843 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:09.843 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:09.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.843 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:157012 9.7 9:eaaff850:::benchmark_data_smithi064_511911_object55910:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e463)", 2023-10-09T06:10:09.843 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:10:09.794936+0000", 2023-10-09T06:10:09.843 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.013903442, 2023-10-09T06:10:09.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014210283000000001, 2023-10-09T06:10:09.843 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:09.843 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:10:09.843 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:09.844 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:10:09.844 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:09.844 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 157012 2023-10-09T06:10:09.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.844 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:09.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:09.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.794936+0000", 2023-10-09T06:10:09.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:09.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:09.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.794936+0000", 2023-10-09T06:10:09.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:09.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:09.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.794935+0000", 2023-10-09T06:10:09.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:10:09.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:09.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.794939+0000", 2023-10-09T06:10:09.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0450000000000002e-06 2023-10-09T06:10:09.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:09.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.794948+0000", 2023-10-09T06:10:09.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4719999999999995e-06 2023-10-09T06:10:09.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:09.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.794955+0000", 2023-10-09T06:10:09.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.861e-06 2023-10-09T06:10:09.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:09.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.798281+0000", 2023-10-09T06:10:09.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003326416 2023-10-09T06:10:09.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:10:09.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.798418+0000", 2023-10-09T06:10:09.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013698000000000001 2023-10-09T06:10:09.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,5,7", 2023-10-09T06:10:09.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.798565+0000", 2023-10-09T06:10:09.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014719900000000001 2023-10-09T06:10:09.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:10:09.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.801854+0000", 2023-10-09T06:10:09.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032883629999999999 2023-10-09T06:10:09.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:09.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:09.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:09.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:09.803990+0000", 2023-10-09T06:10:09.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0021366330000000002 2023-10-09T06:10:09.849 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:09.849 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:09.849 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:09.849 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:09.849 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-10-09T06:10:09.849 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 6 2023-10-09T06:10:09.849 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:10:09.850 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:10:09.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:09.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:10:09.955 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:10:10.046 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:10.221 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:10:10.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:10.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:10:10.336 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:10:10.414 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:10.546 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:10:10.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:10.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:10:10.657 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:10:10.691 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:10.729 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.2 primary-affinity to 0 (802) 2023-10-09T06:10:10.745 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:10:10.826 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:10:10.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:10.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:10:10.938 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:10:10.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:10 smithi064 ceph-mon[212750]: pgmap v1171: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 13243/143406 objects degraded (9.235%); 15350/143406 objects misplaced (10.704%); 7.6 MiB/s, 135 objects/s recovering 2023-10-09T06:10:10.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:10 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3870800276' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:10:10.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:10 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:10:10.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:10 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3145855440' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T06:10:10.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:10 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T06:10:10.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:10 smithi064 ceph-mon[220044]: pgmap v1171: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 13243/143406 objects degraded (9.235%); 15350/143406 objects misplaced (10.704%); 7.6 MiB/s, 135 objects/s recovering 2023-10-09T06:10:10.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:10 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3870800276' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:10:10.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:10 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:10:10.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:10 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3145855440' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T06:10:10.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:10 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T06:10:10.978 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:10 smithi156 ceph-mon[195101]: pgmap v1171: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 13243/143406 objects degraded (9.235%); 15350/143406 objects misplaced (10.704%); 7.6 MiB/s, 135 objects/s recovering 2023-10-09T06:10:10.979 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:10 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3870800276' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:10:10.979 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:10 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:10:10.979 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:10 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3145855440' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T06:10:10.979 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:10 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T06:10:10.981 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:11.075 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:10:11.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:11.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:10:11.187 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:10:11.277 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:12.039 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:11 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:10:12.040 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:11 smithi064 ceph-mon[220044]: failed to encode map e464 with expected crc 2023-10-09T06:10:12.040 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:11 smithi064 ceph-mon[220044]: failed to encode map e464 with expected crc 2023-10-09T06:10:12.040 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:11 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:10:12.040 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:11 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-10-09T06:10:12.040 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:11 smithi064 ceph-mon[220044]: osdmap e464: 8 total, 8 up, 8 in 2023-10-09T06:10:12.040 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:11 smithi064 ceph-mon[220044]: pgmap v1173: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 12957/135135 objects degraded (9.588%); 14241/135135 objects misplaced (10.538%); 6.6 MiB/s, 137 objects/s recovering 2023-10-09T06:10:12.040 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:11 smithi064 ceph-mon[220044]: osdmap e465: 8 total, 8 up, 8 in 2023-10-09T06:10:12.040 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:11 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:10:12.040 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:11 smithi064 ceph-mon[212750]: failed to encode map e464 with expected crc 2023-10-09T06:10:12.040 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:11 smithi064 ceph-mon[212750]: failed to encode map e464 with expected crc 2023-10-09T06:10:12.041 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:11 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:10:12.041 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:11 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-10-09T06:10:12.041 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:11 smithi064 ceph-mon[212750]: osdmap e464: 8 total, 8 up, 8 in 2023-10-09T06:10:12.041 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:11 smithi064 ceph-mon[212750]: pgmap v1173: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 12957/135135 objects degraded (9.588%); 14241/135135 objects misplaced (10.538%); 6.6 MiB/s, 137 objects/s recovering 2023-10-09T06:10:12.041 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:11 smithi064 ceph-mon[212750]: osdmap e465: 8 total, 8 up, 8 in 2023-10-09T06:10:12.065 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:10:12.068 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:10:12.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:12.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:10:12.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:11 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:10:12.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:11 smithi156 ceph-mon[195101]: failed to encode map e464 with expected crc 2023-10-09T06:10:12.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:11 smithi156 ceph-mon[195101]: failed to encode map e464 with expected crc 2023-10-09T06:10:12.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:11 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:10:12.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:11 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-10-09T06:10:12.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:11 smithi156 ceph-mon[195101]: osdmap e464: 8 total, 8 up, 8 in 2023-10-09T06:10:12.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:11 smithi156 ceph-mon[195101]: pgmap v1173: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 12957/135135 objects degraded (9.588%); 14241/135135 objects misplaced (10.538%); 6.6 MiB/s, 137 objects/s recovering 2023-10-09T06:10:12.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:11 smithi156 ceph-mon[195101]: osdmap e465: 8 total, 8 up, 8 in 2023-10-09T06:10:12.176 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:10:12.273 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:12.797 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:10:12.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:12.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:10:12.903 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:10:13.002 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:12 smithi064 ceph-mon[212750]: failed to encode map e465 with expected crc 2023-10-09T06:10:13.002 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:12 smithi064 ceph-mon[212750]: failed to encode map e465 with expected crc 2023-10-09T06:10:13.002 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:12 smithi064 ceph-mon[212750]: failed to encode map e465 with expected crc 2023-10-09T06:10:13.002 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:12 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 12957/135135 objects degraded (9.588%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:10:13.003 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:12 smithi064 ceph-mon[220044]: failed to encode map e465 with expected crc 2023-10-09T06:10:13.003 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:12 smithi064 ceph-mon[220044]: failed to encode map e465 with expected crc 2023-10-09T06:10:13.003 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:12 smithi064 ceph-mon[220044]: failed to encode map e465 with expected crc 2023-10-09T06:10:13.003 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:12 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 12957/135135 objects degraded (9.588%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:10:13.004 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:13.125 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:12 smithi156 ceph-mon[195101]: failed to encode map e465 with expected crc 2023-10-09T06:10:13.126 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:12 smithi156 ceph-mon[195101]: failed to encode map e465 with expected crc 2023-10-09T06:10:13.126 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:12 smithi156 ceph-mon[195101]: failed to encode map e465 with expected crc 2023-10-09T06:10:13.126 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:12 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 12957/135135 objects degraded (9.588%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:10:13.704 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:10:13.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:13.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:10:13.836 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:10:13.853 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:13.878 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:13.955 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:10:13.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:13.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:10:13.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:13 smithi064 ceph-mon[212750]: 1.d scrub starts 2023-10-09T06:10:13.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:13 smithi064 ceph-mon[212750]: pgmap v1175: 119 pgs: 1 peering, 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 12855/134481 objects degraded (9.559%); 14043/134481 objects misplaced (10.442%); 5.9 MiB/s, 135 objects/s recovering 2023-10-09T06:10:13.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:13 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:13.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:13 smithi064 ceph-mon[220044]: 1.d scrub starts 2023-10-09T06:10:13.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:13 smithi064 ceph-mon[220044]: pgmap v1175: 119 pgs: 1 peering, 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 12855/134481 objects degraded (9.559%); 14043/134481 objects misplaced (10.442%); 5.9 MiB/s, 135 objects/s recovering 2023-10-09T06:10:13.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:13 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:14.061 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:10:14.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:13 smithi156 ceph-mon[195101]: 1.d scrub starts 2023-10-09T06:10:14.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:13 smithi156 ceph-mon[195101]: pgmap v1175: 119 pgs: 1 peering, 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 12855/134481 objects degraded (9.559%); 14043/134481 objects misplaced (10.442%); 5.9 MiB/s, 135 objects/s recovering 2023-10-09T06:10:14.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:13 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:14.124 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:14.200 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:10:14.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:14.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:10:14.308 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:10:14.446 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:14.610 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:10:14.616 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:10:14.660 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:10:14.702 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:10:14.724 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:10:14.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:14.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:10:14.781 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:10:14.814 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:10:14.832 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:10:14.864 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:14.869 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:10:14.913 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:10:14.937 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:10:14.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:14.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:10:14.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:14 smithi064 ceph-mon[212750]: 1.2 scrub starts 2023-10-09T06:10:14.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:14 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:10:14.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:14 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2720627268' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:10:14.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:14 smithi064 ceph-mon[220044]: 1.2 scrub starts 2023-10-09T06:10:14.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:14 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:10:14.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:14 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2720627268' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:10:14.972 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:10:15.001 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:10:15.461 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:10:15.461 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:10:15.461 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:10:15.461 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:10:15.461 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:10:15.461 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:10:15.461 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:10:15.462 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:10:15.462 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:14 smithi156 ceph-mon[195101]: 1.2 scrub starts 2023-10-09T06:10:15.463 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:14 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:10:15.463 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:14 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2720627268' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:10:15.527 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:15.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:10:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:10:15] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.43.0" 2023-10-09T06:10:16.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:15 smithi156 ceph-mon[195101]: pgmap v1176: 119 pgs: 1 peering, 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 444 op/s; 12755/133368 objects degraded (9.564%); 13726/133368 objects misplaced (10.292%); 6.2 MiB/s, 124 objects/s recovering 2023-10-09T06:10:16.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:15 smithi064 ceph-mon[212750]: pgmap v1176: 119 pgs: 1 peering, 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 444 op/s; 12755/133368 objects degraded (9.564%); 13726/133368 objects misplaced (10.292%); 6.2 MiB/s, 124 objects/s recovering 2023-10-09T06:10:16.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:15 smithi064 ceph-mon[220044]: pgmap v1176: 119 pgs: 1 peering, 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 444 op/s; 12755/133368 objects degraded (9.564%); 13726/133368 objects misplaced (10.292%); 6.2 MiB/s, 124 objects/s recovering 2023-10-09T06:10:16.364 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:10:16.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:16.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:10:16.470 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:10:16.544 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:16.686 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:10:16.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:16.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:10:16.744 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:16.797 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:10:16.876 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:17.014 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:10:17.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:17.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:10:17.127 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:10:17.235 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:17.254 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:17 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:10:17.255 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:17 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:10:17.383 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:10:17.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:17.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:10:17.427 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:10:17.427 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:10:17.492 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:10:17.584 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:17.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:17 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:10:18.139 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:10:18.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:18.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:10:18.245 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:10:18.319 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:18.409 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:10:18.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:18.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:10:18.423 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:18 smithi064 ceph-mon[212750]: pgmap v1177: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 12373/125361 objects degraded (9.870%); 12674/125361 objects misplaced (10.110%); 8.1 MiB/s, 185 objects/s recovering 2023-10-09T06:10:18.423 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:18 smithi064 ceph-mon[212750]: 1.d scrub starts 2023-10-09T06:10:18.423 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:18 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 12373/125361 objects degraded (9.870%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:10:18.424 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:18 smithi064 ceph-mon[220044]: pgmap v1177: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 12373/125361 objects degraded (9.870%); 12674/125361 objects misplaced (10.110%); 8.1 MiB/s, 185 objects/s recovering 2023-10-09T06:10:18.424 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:18 smithi064 ceph-mon[220044]: 1.d scrub starts 2023-10-09T06:10:18.424 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:18 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 12373/125361 objects degraded (9.870%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:10:18.523 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:10:18.561 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:18 smithi156 ceph-mon[195101]: pgmap v1177: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 12373/125361 objects degraded (9.870%); 12674/125361 objects misplaced (10.110%); 8.1 MiB/s, 185 objects/s recovering 2023-10-09T06:10:18.561 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:18 smithi156 ceph-mon[195101]: 1.d scrub starts 2023-10-09T06:10:18.561 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:18 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 12373/125361 objects degraded (9.870%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:10:18.567 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:18.621 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:10:18.621 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1, 2] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 4, 0] 2023-10-09T06:10:18.621 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:10:18.621 INFO:tasks.thrashosds.thrasher:Reweighting osd 6 to 0.6106151152545612 2023-10-09T06:10:18.621 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 0.6106151152545612 2023-10-09T06:10:18.643 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:10:18.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:18.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:10:18.750 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:10:18.780 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:19.177 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:10:19.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:19.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:10:19.281 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:10:19.292 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:10:19.418 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:19.585 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:10:19.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:19.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:10:19.720 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:19.725 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:10:19.750 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:19.769 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:19.832 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:10:19.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:19.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:10:19.945 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:10:20.064 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:20.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:20 smithi064 ceph-mon[212750]: pgmap v1178: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 12373/124776 objects degraded (9.916%); 12674/124776 objects misplaced (10.157%); 5.6 MiB/s, 144 objects/s recovering 2023-10-09T06:10:20.197 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:10:20.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:20.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:10:20.302 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:10:20.419 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:20.501 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:20 smithi064 ceph-mon[220044]: pgmap v1178: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 12373/124776 objects degraded (9.916%); 12674/124776 objects misplaced (10.157%); 5.6 MiB/s, 144 objects/s recovering 2023-10-09T06:10:20.565 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:10:20.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:20.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:10:20.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:20 smithi156 ceph-mon[195101]: pgmap v1178: 119 pgs: 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 12373/124776 objects degraded (9.916%); 12674/124776 objects misplaced (10.157%); 5.6 MiB/s, 144 objects/s recovering 2023-10-09T06:10:20.680 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:10:20.771 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:21.234 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:10:21.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:21 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/758255630' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:10:21.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:21 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:10:21.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:21 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/758255630' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:10:21.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:21 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:10:21.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:21 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/758255630' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:10:21.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:21 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:10:22.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:22 smithi064 ceph-mon[220044]: pgmap v1179: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 12303/120069 objects degraded (10.247%); 11809/120069 objects misplaced (9.835%); 6.1 MiB/s, 135 objects/s recovering 2023-10-09T06:10:22.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:22 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:10:22.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:22 smithi064 ceph-mon[220044]: failed to encode map e466 with expected crc 2023-10-09T06:10:22.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:22 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:10:22.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:22 smithi064 ceph-mon[220044]: osdmap e466: 8 total, 8 up, 8 in 2023-10-09T06:10:22.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:22 smithi064 ceph-mon[220044]: failed to encode map e466 with expected crc 2023-10-09T06:10:22.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:22 smithi064 ceph-mon[220044]: 1.2 scrub starts 2023-10-09T06:10:22.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:22 smithi064 ceph-mon[212750]: pgmap v1179: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 12303/120069 objects degraded (10.247%); 11809/120069 objects misplaced (9.835%); 6.1 MiB/s, 135 objects/s recovering 2023-10-09T06:10:22.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:22 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:10:22.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:22 smithi064 ceph-mon[212750]: failed to encode map e466 with expected crc 2023-10-09T06:10:22.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:22 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:10:22.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:22 smithi064 ceph-mon[212750]: osdmap e466: 8 total, 8 up, 8 in 2023-10-09T06:10:22.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:22 smithi064 ceph-mon[212750]: failed to encode map e466 with expected crc 2023-10-09T06:10:22.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:22 smithi064 ceph-mon[212750]: 1.2 scrub starts 2023-10-09T06:10:22.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:22 smithi156 ceph-mon[195101]: pgmap v1179: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 12303/120069 objects degraded (10.247%); 11809/120069 objects misplaced (9.835%); 6.1 MiB/s, 135 objects/s recovering 2023-10-09T06:10:22.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:22 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:10:22.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:22 smithi156 ceph-mon[195101]: failed to encode map e466 with expected crc 2023-10-09T06:10:22.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:22 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:10:22.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:22 smithi156 ceph-mon[195101]: osdmap e466: 8 total, 8 up, 8 in 2023-10-09T06:10:22.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:22 smithi156 ceph-mon[195101]: failed to encode map e466 with expected crc 2023-10-09T06:10:22.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:22 smithi156 ceph-mon[195101]: 1.2 scrub starts 2023-10-09T06:10:22.956 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:10:22.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:22.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:10:23.069 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:10:23.106 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:23.159 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:10:23.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:23.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:10:23.264 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:10:23.303 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:23.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:23 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 12303/120069 objects degraded (10.247%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:10:23.375 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:10:23.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:23.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:10:23.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:23 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 12303/120069 objects degraded (10.247%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:10:23.475 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:23 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 12303/120069 objects degraded (10.247%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:10:23.486 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:10:23.590 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:23.748 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:23.918 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:10:23.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:23.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:10:24.038 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:10:24.154 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:24.355 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:10:24.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:24.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:10:24.379 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:10:24.431 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:10:24.466 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:10:24.476 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:10:24.528 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:10:24.589 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:10:24.598 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:24.629 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:10:24.679 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:10:24.721 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:10:24.730 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:10:24.731 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:10:24.731 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:10:24.731 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:10:24.731 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:10:24.731 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:10:24.731 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:10:24.731 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:10:24.837 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:10:24.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:24.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:10:24.848 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:24 smithi064 ceph-mon[212750]: pgmap v1181: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 11957/116250 objects degraded (10.286%); 11407/116250 objects misplaced (9.812%); 6.4 MiB/s, 161 objects/s recovering 2023-10-09T06:10:24.848 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:24 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3500137652' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:10:24.848 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:24 smithi064 ceph-mon[220044]: pgmap v1181: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 11957/116250 objects degraded (10.286%); 11407/116250 objects misplaced (9.812%); 6.4 MiB/s, 161 objects/s recovering 2023-10-09T06:10:24.848 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:24 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3500137652' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:10:24.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:24 smithi156 ceph-mon[195101]: pgmap v1181: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 11957/116250 objects degraded (10.286%); 11407/116250 objects misplaced (9.812%); 6.4 MiB/s, 161 objects/s recovering 2023-10-09T06:10:24.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:24 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3500137652' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:10:24.945 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:10:25.044 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:25.605 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:25 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/802047456' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.6106151152545612}]: dispatch 2023-10-09T06:10:25.606 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:10:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:10:25] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.43.0" 2023-10-09T06:10:25.622 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.6 to 0.610615 (9c51) 2023-10-09T06:10:25.688 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:10:25.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:25 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/802047456' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.6106151152545612}]: dispatch 2023-10-09T06:10:25.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:25 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.6106151152545612}]: dispatch 2023-10-09T06:10:25.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:25 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/802047456' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.6106151152545612}]: dispatch 2023-10-09T06:10:25.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:25 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.6106151152545612}]: dispatch 2023-10-09T06:10:25.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:25 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.6106151152545612}]: dispatch 2023-10-09T06:10:26.075 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:10:26.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:26.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:10:26.201 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:10:26.244 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:26.323 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:10:26.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:26.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:10:26.429 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:10:26.459 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:26.521 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:10:26.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:26.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:10:26.628 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:10:26.693 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:26.756 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:10:26.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:26.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:10:26.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:26 smithi156 ceph-mon[195101]: pgmap v1182: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 11957/115080 objects degraded (10.390%); 11407/115080 objects misplaced (9.912%); 4.4 MiB/s, 163 objects/s recovering 2023-10-09T06:10:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:26 smithi156 ceph-mon[195101]: 1.d deep-scrub starts 2023-10-09T06:10:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:26 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 0.6106151152545612}]': finished 2023-10-09T06:10:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:26 smithi156 ceph-mon[195101]: osdmap e467: 8 total, 8 up, 8 in 2023-10-09T06:10:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:26 smithi156 ceph-mon[195101]: failed to encode map e467 with expected crc 2023-10-09T06:10:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:26 smithi156 ceph-mon[195101]: failed to encode map e467 with expected crc 2023-10-09T06:10:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:26 smithi156 ceph-mon[195101]: failed to encode map e467 with expected crc 2023-10-09T06:10:26.871 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:10:26.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:26 smithi064 ceph-mon[212750]: pgmap v1182: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 11957/115080 objects degraded (10.390%); 11407/115080 objects misplaced (9.912%); 4.4 MiB/s, 163 objects/s recovering 2023-10-09T06:10:26.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:26 smithi064 ceph-mon[212750]: 1.d deep-scrub starts 2023-10-09T06:10:26.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:26 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 0.6106151152545612}]': finished 2023-10-09T06:10:26.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:26 smithi064 ceph-mon[212750]: osdmap e467: 8 total, 8 up, 8 in 2023-10-09T06:10:26.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:26 smithi064 ceph-mon[212750]: failed to encode map e467 with expected crc 2023-10-09T06:10:26.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:26 smithi064 ceph-mon[212750]: failed to encode map e467 with expected crc 2023-10-09T06:10:26.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:26 smithi064 ceph-mon[212750]: failed to encode map e467 with expected crc 2023-10-09T06:10:26.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:26 smithi064 ceph-mon[220044]: pgmap v1182: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 11957/115080 objects degraded (10.390%); 11407/115080 objects misplaced (9.912%); 4.4 MiB/s, 163 objects/s recovering 2023-10-09T06:10:26.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:26 smithi064 ceph-mon[220044]: 1.d deep-scrub starts 2023-10-09T06:10:26.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:26 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 0.6106151152545612}]': finished 2023-10-09T06:10:26.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:26 smithi064 ceph-mon[220044]: osdmap e467: 8 total, 8 up, 8 in 2023-10-09T06:10:26.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:26 smithi064 ceph-mon[220044]: failed to encode map e467 with expected crc 2023-10-09T06:10:26.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:26 smithi064 ceph-mon[220044]: failed to encode map e467 with expected crc 2023-10-09T06:10:26.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:26 smithi064 ceph-mon[220044]: failed to encode map e467 with expected crc 2023-10-09T06:10:26.925 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:27.485 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:10:27.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:27.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:10:27.590 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:10:27.682 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:27.707 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:27 smithi064 ceph-mon[212750]: 1.2 scrub starts 2023-10-09T06:10:27.707 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:27 smithi064 ceph-mon[212750]: failed to encode map e468 with expected crc 2023-10-09T06:10:27.707 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:27 smithi064 ceph-mon[212750]: osdmap e468: 8 total, 8 up, 8 in 2023-10-09T06:10:27.707 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:27 smithi064 ceph-mon[212750]: failed to encode map e468 with expected crc 2023-10-09T06:10:27.707 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:27 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 11957/115080 objects degraded (10.390%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:10:27.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:27 smithi064 ceph-mon[220044]: 1.2 scrub starts 2023-10-09T06:10:27.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:27 smithi064 ceph-mon[220044]: failed to encode map e468 with expected crc 2023-10-09T06:10:27.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:27 smithi064 ceph-mon[220044]: osdmap e468: 8 total, 8 up, 8 in 2023-10-09T06:10:27.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:27 smithi064 ceph-mon[220044]: failed to encode map e468 with expected crc 2023-10-09T06:10:27.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:27 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 11957/115080 objects degraded (10.390%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:10:27.912 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:27.951 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:10:27.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:27.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:10:28.060 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:10:28.063 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:10:28.063 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T06:10:28.063 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T06:10:28.063 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:10:28.063 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.063 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:188637 7.2 7:41277314:::benchmark_data_smithi064_398483_object83284:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T06:10:28.063 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:01:53.723003+0000", 2023-10-09T06:10:28.063 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 514.32957885400003, 2023-10-09T06:10:28.064 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2240001060000001, 2023-10-09T06:10:28.064 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:28.064 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:10:28.064 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:28.064 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:10:28.064 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:10:28.064 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 188637 2023-10-09T06:10:28.064 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.064 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:28.064 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.065 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:28.065 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.723003+0000", 2023-10-09T06:10:28.065 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.065 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.065 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.065 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:28.065 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.723003+0000", 2023-10-09T06:10:28.065 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.065 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.065 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.065 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:28.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.723002+0000", 2023-10-09T06:10:28.066 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:10:28.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:28.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.723006+0000", 2023-10-09T06:10:28.066 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1480000000000002e-06 2023-10-09T06:10:28.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:28.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.723012+0000", 2023-10-09T06:10:28.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8959999999999997e-06 2023-10-09T06:10:28.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:28.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.723018+0000", 2023-10-09T06:10:28.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3159999999999996e-06 2023-10-09T06:10:28.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.723303+0000", 2023-10-09T06:10:28.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00028491600000000002 2023-10-09T06:10:28.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.068 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:10:28.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.723425+0000", 2023-10-09T06:10:28.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000121991 2023-10-09T06:10:28.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.068 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,7", 2023-10-09T06:10:28.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.723532+0000", 2023-10-09T06:10:28.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010740599999999999 2023-10-09T06:10:28.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.069 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:10:28.069 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.727500+0000", 2023-10-09T06:10:28.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039675049999999996 2023-10-09T06:10:28.069 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.069 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.069 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.751543+0000", 2023-10-09T06:10:28.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.024043604 2023-10-09T06:10:28.069 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.751556+0000", 2023-10-09T06:10:28.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3142000000000001e-05 2023-10-09T06:10:28.070 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.070 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.746147+0000", 2023-10-09T06:10:28.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99459092900000001 2023-10-09T06:10:28.070 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.070 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.071 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.071 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.746164+0000", 2023-10-09T06:10:28.071 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6728999999999999e-05 2023-10-09T06:10:28.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.071 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.071 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.946783+0000", 2023-10-09T06:10:28.071 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2006188790000001 2023-10-09T06:10:28.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.072 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:10:28.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.947003+0000", 2023-10-09T06:10:28.072 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022009300000000001 2023-10-09T06:10:28.072 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.072 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:28.072 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.072 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.072 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.072 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:188647 7.2 7:43ded6de:::benchmark_data_smithi064_398483_object83294:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T06:10:28.072 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:01:53.733487+0000", 2023-10-09T06:10:28.072 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 514.31909460999998, 2023-10-09T06:10:28.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.244246736, 2023-10-09T06:10:28.073 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:28.073 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:28.073 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:28.073 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:10:28.073 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:10:28.073 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 188647 2023-10-09T06:10:28.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.073 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:28.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:28.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.733487+0000", 2023-10-09T06:10:28.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.074 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:28.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.733487+0000", 2023-10-09T06:10:28.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.074 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:28.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.733487+0000", 2023-10-09T06:10:28.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:10:28.075 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.075 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:28.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.733489+0000", 2023-10-09T06:10:28.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4380000000000002e-06 2023-10-09T06:10:28.075 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.075 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:28.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.733494+0000", 2023-10-09T06:10:28.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.04e-06 2023-10-09T06:10:28.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.076 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.076 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:28.076 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.733499+0000", 2023-10-09T06:10:28.076 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4100000000000001e-06 2023-10-09T06:10:28.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.076 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.076 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.076 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.753126+0000", 2023-10-09T06:10:28.076 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.019626964 2023-10-09T06:10:28.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.077 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.077 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.077 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.753140+0000", 2023-10-09T06:10:28.077 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3987e-05 2023-10-09T06:10:28.077 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.077 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.077 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.077 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.746684+0000", 2023-10-09T06:10:28.077 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99354400499999995 2023-10-09T06:10:28.077 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.077 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.077 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.078 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.746695+0000", 2023-10-09T06:10:28.078 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1477e-05 2023-10-09T06:10:28.078 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.078 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.078 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.078 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.947305+0000", 2023-10-09T06:10:28.078 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2006102190000001 2023-10-09T06:10:28.078 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.078 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.078 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:10:28.078 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.947448+0000", 2023-10-09T06:10:28.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014231000000000001 2023-10-09T06:10:28.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T06:10:28.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.947593+0000", 2023-10-09T06:10:28.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014539899999999999 2023-10-09T06:10:28.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.966520+0000", 2023-10-09T06:10:28.080 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018926897000000002 2023-10-09T06:10:28.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.966586+0000", 2023-10-09T06:10:28.080 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5819000000000001e-05 2023-10-09T06:10:28.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:10:28.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.966925+0000", 2023-10-09T06:10:28.080 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000338783 2023-10-09T06:10:28.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.977573+0000", 2023-10-09T06:10:28.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010648036 2023-10-09T06:10:28.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:10:28.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.977608+0000", 2023-10-09T06:10:28.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5738999999999998e-05 2023-10-09T06:10:28.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:10:28.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.977734+0000", 2023-10-09T06:10:28.082 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012555199999999999 2023-10-09T06:10:28.082 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.082 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:28.082 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.082 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.082 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.082 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:188663 7.2 7:43951539:::benchmark_data_smithi064_398483_object83310:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2023-10-09T06:10:28.082 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:01:53.773089+0000", 2023-10-09T06:10:28.082 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 514.27949235200003, 2023-10-09T06:10:28.082 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.204785733, 2023-10-09T06:10:28.082 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:28.083 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:28.083 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:28.083 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:10:28.083 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:10:28.083 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 188663 2023-10-09T06:10:28.083 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.083 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:28.083 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.083 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:28.083 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.773089+0000", 2023-10-09T06:10:28.083 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.083 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.083 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.084 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:28.084 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.773089+0000", 2023-10-09T06:10:28.084 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.084 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.084 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.084 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:28.084 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.773089+0000", 2023-10-09T06:10:28.084 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:10:28.084 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.084 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.084 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:28.084 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.773092+0000", 2023-10-09T06:10:28.084 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0520000000000002e-06 2023-10-09T06:10:28.085 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.085 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.085 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:28.085 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.773114+0000", 2023-10-09T06:10:28.085 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1593000000000001e-05 2023-10-09T06:10:28.085 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.085 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.085 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:28.085 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.773120+0000", 2023-10-09T06:10:28.085 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5350000000000001e-06 2023-10-09T06:10:28.085 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.085 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.086 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.086 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.773180+0000", 2023-10-09T06:10:28.086 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0223999999999998e-05 2023-10-09T06:10:28.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.086 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.086 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.773194+0000", 2023-10-09T06:10:28.086 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4059999999999999e-05 2023-10-09T06:10:28.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.086 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.086 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.746723+0000", 2023-10-09T06:10:28.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97352899999999998 2023-10-09T06:10:28.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.746733+0000", 2023-10-09T06:10:28.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6099999999999995e-06 2023-10-09T06:10:28.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.950663+0000", 2023-10-09T06:10:28.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.203930186 2023-10-09T06:10:28.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:10:28.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.950786+0000", 2023-10-09T06:10:28.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012234499999999999 2023-10-09T06:10:28.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T06:10:28.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.950923+0000", 2023-10-09T06:10:28.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013722800000000001 2023-10-09T06:10:28.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.966626+0000", 2023-10-09T06:10:28.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015702774999999999 2023-10-09T06:10:28.089 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.089 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.089 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.089 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.966658+0000", 2023-10-09T06:10:28.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2387e-05 2023-10-09T06:10:28.089 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.089 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.089 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:10:28.089 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.971199+0000", 2023-10-09T06:10:28.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0045408920000000004 2023-10-09T06:10:28.089 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.089 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.090 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.090 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.977796+0000", 2023-10-09T06:10:28.090 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065968099999999998 2023-10-09T06:10:28.090 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.090 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.090 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:10:28.090 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.977821+0000", 2023-10-09T06:10:28.090 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4983000000000001e-05 2023-10-09T06:10:28.090 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.090 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.090 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:10:28.090 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.977875+0000", 2023-10-09T06:10:28.091 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4512000000000003e-05 2023-10-09T06:10:28.091 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.091 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:28.091 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.091 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.091 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.091 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:188673 7.2 7:4798b52f:::benchmark_data_smithi064_398483_object83320:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2023-10-09T06:10:28.091 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:01:53.787980+0000", 2023-10-09T06:10:28.091 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 514.264601965, 2023-10-09T06:10:28.091 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.190044833, 2023-10-09T06:10:28.091 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:28.091 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:28.092 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:28.092 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:10:28.092 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:10:28.092 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 188673 2023-10-09T06:10:28.092 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.092 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:28.092 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.092 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:28.092 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.787980+0000", 2023-10-09T06:10:28.092 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.092 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.092 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.092 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:28.093 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.787980+0000", 2023-10-09T06:10:28.093 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.093 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.093 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.093 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:28.093 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.787979+0000", 2023-10-09T06:10:28.093 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:10:28.093 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.093 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.093 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:28.093 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.787984+0000", 2023-10-09T06:10:28.094 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3580000000000002e-06 2023-10-09T06:10:28.094 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.094 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.094 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:28.094 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.787996+0000", 2023-10-09T06:10:28.094 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1557e-05 2023-10-09T06:10:28.094 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.094 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.094 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:28.094 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.788001+0000", 2023-10-09T06:10:28.094 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6110000000000001e-06 2023-10-09T06:10:28.095 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.095 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.095 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.095 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.788065+0000", 2023-10-09T06:10:28.095 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3528999999999994e-05 2023-10-09T06:10:28.095 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.095 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.095 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.095 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:53.788078+0000", 2023-10-09T06:10:28.095 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3283e-05 2023-10-09T06:10:28.095 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.095 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.096 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.096 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.746837+0000", 2023-10-09T06:10:28.096 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95875847000000003 2023-10-09T06:10:28.096 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.096 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.096 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.096 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:54.746848+0000", 2023-10-09T06:10:28.096 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1361e-05 2023-10-09T06:10:28.096 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.096 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.096 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.963386+0000", 2023-10-09T06:10:28.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.216537725 2023-10-09T06:10:28.097 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.097 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.097 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:10:28.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.963469+0000", 2023-10-09T06:10:28.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3786e-05 2023-10-09T06:10:28.097 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.097 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.097 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T06:10:28.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.963552+0000", 2023-10-09T06:10:28.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2280000000000005e-05 2023-10-09T06:10:28.097 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.098 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.098 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.098 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.966912+0000", 2023-10-09T06:10:28.098 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033606030000000002 2023-10-09T06:10:28.098 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.098 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.098 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:10:28.098 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.971224+0000", 2023-10-09T06:10:28.098 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043120759999999998 2023-10-09T06:10:28.098 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.098 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.099 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.099 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.977775+0000", 2023-10-09T06:10:28.099 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065506870000000003 2023-10-09T06:10:28.099 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.099 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.099 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.099 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.977952+0000", 2023-10-09T06:10:28.099 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017698099999999999 2023-10-09T06:10:28.099 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.099 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.099 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:10:28.099 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.977973+0000", 2023-10-09T06:10:28.099 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0883999999999999e-05 2023-10-09T06:10:28.100 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.100 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.100 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:10:28.100 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:01:55.978025+0000", 2023-10-09T06:10:28.100 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1677e-05 2023-10-09T06:10:28.100 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.100 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:28.100 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.100 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.100 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.100 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:201276 7.2 7:46bc3faa:::benchmark_data_smithi064_398483_object95923:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T06:10:28.100 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:02:17.222371+0000", 2023-10-09T06:10:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 490.83021055699999, 2023-10-09T06:10:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2549091190000001, 2023-10-09T06:10:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:10:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:10:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:10:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 201276 2023-10-09T06:10:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.222371+0000", 2023-10-09T06:10:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.222371+0000", 2023-10-09T06:10:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.222369+0000", 2023-10-09T06:10:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:10:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.222378+0000", 2023-10-09T06:10:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3080000000000008e-06 2023-10-09T06:10:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.222393+0000", 2023-10-09T06:10:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5420000000000001e-05 2023-10-09T06:10:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.222406+0000", 2023-10-09T06:10:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3373000000000001e-05 2023-10-09T06:10:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.222579+0000", 2023-10-09T06:10:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017300700000000001 2023-10-09T06:10:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:10:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.222707+0000", 2023-10-09T06:10:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000127224 2023-10-09T06:10:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T06:10:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.222827+0000", 2023-10-09T06:10:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000120133 2023-10-09T06:10:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.407710+0000", 2023-10-09T06:10:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18488295399999999 2023-10-09T06:10:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.407718+0000", 2023-10-09T06:10:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9780000000000005e-06 2023-10-09T06:10:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:18.462485+0000", 2023-10-09T06:10:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.054767682 2023-10-09T06:10:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-10-09T06:10:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:18.462638+0000", 2023-10-09T06:10:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015246199999999999 2023-10-09T06:10:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:18.477220+0000", 2023-10-09T06:10:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014581999 2023-10-09T06:10:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:10:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:18.477280+0000", 2023-10-09T06:10:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0467999999999998e-05 2023-10-09T06:10:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45263.0:201279 7.2 7:46954836:::benchmark_data_smithi064_398483_object95926:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T06:10:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:02:17.222440+0000", 2023-10-09T06:10:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 490.83014221600001, 2023-10-09T06:10:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2615216570000001, 2023-10-09T06:10:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:10:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45263", 2023-10-09T06:10:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2417540799", 2023-10-09T06:10:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 201279 2023-10-09T06:10:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.222440+0000", 2023-10-09T06:10:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.222440+0000", 2023-10-09T06:10:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.222439+0000", 2023-10-09T06:10:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:10:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.222441+0000", 2023-10-09T06:10:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1560000000000002e-06 2023-10-09T06:10:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.222447+0000", 2023-10-09T06:10:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1730000000000003e-06 2023-10-09T06:10:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.222451+0000", 2023-10-09T06:10:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7620000000000003e-06 2023-10-09T06:10:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.226323+0000", 2023-10-09T06:10:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038718799999999999 2023-10-09T06:10:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:10:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.226438+0000", 2023-10-09T06:10:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000114333 2023-10-09T06:10:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,7", 2023-10-09T06:10:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.226553+0000", 2023-10-09T06:10:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011495200000000001 2023-10-09T06:10:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.407741+0000", 2023-10-09T06:10:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18118849400000001 2023-10-09T06:10:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:17.407747+0000", 2023-10-09T06:10:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.392e-06 2023-10-09T06:10:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:18.463108+0000", 2023-10-09T06:10:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0553603819999999 2023-10-09T06:10:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-10-09T06:10:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:18.463257+0000", 2023-10-09T06:10:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014875399999999999 2023-10-09T06:10:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:18.483867+0000", 2023-10-09T06:10:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020610441 2023-10-09T06:10:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:10:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:02:18.483961+0000", 2023-10-09T06:10:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4221000000000005e-05 2023-10-09T06:10:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:134367 8.f 8:f7ea56f9:::benchmark_data_smithi064_457003_object29849:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2023-10-09T06:10:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:10.416594+0000", 2023-10-09T06:10:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 317.63598765799998, 2023-10-09T06:10:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2992004480000001, 2023-10-09T06:10:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:10:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:10:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:10:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 134367 2023-10-09T06:10:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.416594+0000", 2023-10-09T06:10:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.416594+0000", 2023-10-09T06:10:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.416592+0000", 2023-10-09T06:10:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:10:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.416600+0000", 2023-10-09T06:10:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7970000000000001e-06 2023-10-09T06:10:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560025+0000", 2023-10-09T06:10:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14342429500000001 2023-10-09T06:10:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560035+0000", 2023-10-09T06:10:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0399e-05 2023-10-09T06:10:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.562476+0000", 2023-10-09T06:10:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024408189999999999 2023-10-09T06:10:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:10:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.562602+0000", 2023-10-09T06:10:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012572700000000001 2023-10-09T06:10:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:10:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.562735+0000", 2023-10-09T06:10:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013364599999999999 2023-10-09T06:10:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.573458+0000", 2023-10-09T06:10:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010723264 2023-10-09T06:10:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.573473+0000", 2023-10-09T06:10:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4623e-05 2023-10-09T06:10:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:11.446813+0000", 2023-10-09T06:10:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.87334020000000001 2023-10-09T06:10:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:11.446827+0000", 2023-10-09T06:10:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3597999999999999e-05 2023-10-09T06:10:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.582495+0000", 2023-10-09T06:10:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.135667666 2023-10-09T06:10:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-10-09T06:10:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.582634+0000", 2023-10-09T06:10:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000139698 2023-10-09T06:10:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.715717+0000", 2023-10-09T06:10:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.13308252800000001 2023-10-09T06:10:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:10:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.715795+0000", 2023-10-09T06:10:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7799000000000005e-05 2023-10-09T06:10:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:134376 8.f 8:f16b3848:::benchmark_data_smithi064_457003_object29858:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2023-10-09T06:10:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:10.560131+0000", 2023-10-09T06:10:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 317.49245116399999, 2023-10-09T06:10:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1496100760000001, 2023-10-09T06:10:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:10:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:10:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 134376 2023-10-09T06:10:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560131+0000", 2023-10-09T06:10:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560131+0000", 2023-10-09T06:10:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560130+0000", 2023-10-09T06:10:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:10:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560135+0000", 2023-10-09T06:10:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2499999999999997e-06 2023-10-09T06:10:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560145+0000", 2023-10-09T06:10:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0074e-05 2023-10-09T06:10:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560153+0000", 2023-10-09T06:10:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8529999999999992e-06 2023-10-09T06:10:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.573993+0000", 2023-10-09T06:10:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013840427000000001 2023-10-09T06:10:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.574002+0000", 2023-10-09T06:10:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9220000000000003e-06 2023-10-09T06:10:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:11.446946+0000", 2023-10-09T06:10:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.87294397499999998 2023-10-09T06:10:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:11.446956+0000", 2023-10-09T06:10:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8779999999999993e-06 2023-10-09T06:10:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.582901+0000", 2023-10-09T06:10:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.135944788 2023-10-09T06:10:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:10:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.583040+0000", 2023-10-09T06:10:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013896899999999999 2023-10-09T06:10:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,4", 2023-10-09T06:10:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.586304+0000", 2023-10-09T06:10:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032639660000000001 2023-10-09T06:10:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:10:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.600938+0000", 2023-10-09T06:10:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014634311000000001 2023-10-09T06:10:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.600984+0000", 2023-10-09T06:10:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5691e-05 2023-10-09T06:10:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.602931+0000", 2023-10-09T06:10:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019468339999999999 2023-10-09T06:10:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.709596+0000", 2023-10-09T06:10:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.106665146 2023-10-09T06:10:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:10:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.709659+0000", 2023-10-09T06:10:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2990000000000005e-05 2023-10-09T06:10:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:10:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.709741+0000", 2023-10-09T06:10:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1841999999999993e-05 2023-10-09T06:10:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:134381 8.f 8:f7feb3ed:::benchmark_data_smithi064_457003_object29863:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2023-10-09T06:10:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:10.560160+0000", 2023-10-09T06:10:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 317.49242189400002, 2023-10-09T06:10:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1498039279999999, 2023-10-09T06:10:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:10:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:10:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 134381 2023-10-09T06:10:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560160+0000", 2023-10-09T06:10:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560160+0000", 2023-10-09T06:10:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560159+0000", 2023-10-09T06:10:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:10:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560162+0000", 2023-10-09T06:10:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8509999999999999e-06 2023-10-09T06:10:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560169+0000", 2023-10-09T06:10:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.489e-06 2023-10-09T06:10:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560175+0000", 2023-10-09T06:10:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5579999999999997e-06 2023-10-09T06:10:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.574927+0000", 2023-10-09T06:10:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014751195 2023-10-09T06:10:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.574936+0000", 2023-10-09T06:10:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4769999999999992e-06 2023-10-09T06:10:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:11.446983+0000", 2023-10-09T06:10:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.87204703100000003 2023-10-09T06:10:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:11.446991+0000", 2023-10-09T06:10:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4200000000000007e-06 2023-10-09T06:10:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.593274+0000", 2023-10-09T06:10:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1462829640000001 2023-10-09T06:10:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:10:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.593416+0000", 2023-10-09T06:10:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000141827 2023-10-09T06:10:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,4", 2023-10-09T06:10:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.593560+0000", 2023-10-09T06:10:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014419699999999999 2023-10-09T06:10:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.601021+0000", 2023-10-09T06:10:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0074606000000000004 2023-10-09T06:10:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:10:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.602791+0000", 2023-10-09T06:10:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0017704789999999999 2023-10-09T06:10:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.616064+0000", 2023-10-09T06:10:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013272879 2023-10-09T06:10:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.709794+0000", 2023-10-09T06:10:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.093729268000000004 2023-10-09T06:10:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:10:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.709848+0000", 2023-10-09T06:10:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.427e-05 2023-10-09T06:10:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:10:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.709964+0000", 2023-10-09T06:10:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000115875 2023-10-09T06:10:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:134407 8.f 8:fbf853b5:::benchmark_data_smithi064_457003_object29889:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-10-09T06:10:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:10.594588+0000", 2023-10-09T06:10:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 317.45799371800001, 2023-10-09T06:10:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1155959489999998, 2023-10-09T06:10:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:10:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:10:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 134407 2023-10-09T06:10:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.594588+0000", 2023-10-09T06:10:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.594588+0000", 2023-10-09T06:10:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.594587+0000", 2023-10-09T06:10:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:10:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.594592+0000", 2023-10-09T06:10:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2689999999999999e-06 2023-10-09T06:10:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.594606+0000", 2023-10-09T06:10:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.38e-05 2023-10-09T06:10:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.594612+0000", 2023-10-09T06:10:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3600000000000004e-06 2023-10-09T06:10:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.594675+0000", 2023-10-09T06:10:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3595000000000004e-05 2023-10-09T06:10:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.594687+0000", 2023-10-09T06:10:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2289999999999999e-05 2023-10-09T06:10:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:11.447669+0000", 2023-10-09T06:10:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.85298183299999997 2023-10-09T06:10:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:11.447680+0000", 2023-10-09T06:10:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1025e-05 2023-10-09T06:10:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.597255+0000", 2023-10-09T06:10:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.149574305 2023-10-09T06:10:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:10:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.597398+0000", 2023-10-09T06:10:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000143036 2023-10-09T06:10:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,4", 2023-10-09T06:10:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.597558+0000", 2023-10-09T06:10:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016018900000000001 2023-10-09T06:10:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.601331+0000", 2023-10-09T06:10:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037728509999999998 2023-10-09T06:10:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:10:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.602800+0000", 2023-10-09T06:10:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014689779999999999 2023-10-09T06:10:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.616098+0000", 2023-10-09T06:10:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013298783 2023-10-09T06:10:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.710024+0000", 2023-10-09T06:10:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.093925355000000002 2023-10-09T06:10:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:10:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.710070+0000", 2023-10-09T06:10:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6561999999999999e-05 2023-10-09T06:10:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:10:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.710184+0000", 2023-10-09T06:10:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011362900000000001 2023-10-09T06:10:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116287 9.1 9:8655aad1:::benchmark_data_smithi064_511911_object15185:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-10-09T06:10:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.530424+0000", 2023-10-09T06:10:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 85.522157672000006, 2023-10-09T06:10:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.181524692, 2023-10-09T06:10:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:10:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:10:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116287 2023-10-09T06:10:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530424+0000", 2023-10-09T06:10:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530424+0000", 2023-10-09T06:10:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530422+0000", 2023-10-09T06:10:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:10:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530432+0000", 2023-10-09T06:10:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0105000000000001e-05 2023-10-09T06:10:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530454+0000", 2023-10-09T06:10:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1579000000000001e-05 2023-10-09T06:10:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530465+0000", 2023-10-09T06:10:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1044e-05 2023-10-09T06:10:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530519+0000", 2023-10-09T06:10:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4567000000000001e-05 2023-10-09T06:10:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:10:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530668+0000", 2023-10-09T06:10:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014871900000000001 2023-10-09T06:10:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:10:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530823+0000", 2023-10-09T06:10:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000155375 2023-10-09T06:10:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.533953+0000", 2023-10-09T06:10:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031297159999999998 2023-10-09T06:10:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:10:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.534034+0000", 2023-10-09T06:10:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0731999999999999e-05 2023-10-09T06:10:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.605818+0000", 2023-10-09T06:10:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.071784291 2023-10-09T06:10:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.605835+0000", 2023-10-09T06:10:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6589e-05 2023-10-09T06:10:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578572+0000", 2023-10-09T06:10:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97273759299999996 2023-10-09T06:10:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578584+0000", 2023-10-09T06:10:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1283e-05 2023-10-09T06:10:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.711858+0000", 2023-10-09T06:10:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1332740379999999 2023-10-09T06:10:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:10:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.711949+0000", 2023-10-09T06:10:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1187000000000006e-05 2023-10-09T06:10:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116293 9.1 9:8657b0ae:::benchmark_data_smithi064_511911_object15191:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-10-09T06:10:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.545024+0000", 2023-10-09T06:10:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 85.507557358, 2023-10-09T06:10:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1673195999999999, 2023-10-09T06:10:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:10:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:10:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116293 2023-10-09T06:10:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545024+0000", 2023-10-09T06:10:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545024+0000", 2023-10-09T06:10:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545023+0000", 2023-10-09T06:10:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:10:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545029+0000", 2023-10-09T06:10:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5910000000000003e-06 2023-10-09T06:10:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545037+0000", 2023-10-09T06:10:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.909e-06 2023-10-09T06:10:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545042+0000", 2023-10-09T06:10:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1030000000000001e-06 2023-10-09T06:10:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545077+0000", 2023-10-09T06:10:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4620999999999999e-05 2023-10-09T06:10:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:10:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545160+0000", 2023-10-09T06:10:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3925999999999996e-05 2023-10-09T06:10:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:10:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545236+0000", 2023-10-09T06:10:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5909999999999997e-05 2023-10-09T06:10:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:10:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.547494+0000", 2023-10-09T06:10:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0022575310000000001 2023-10-09T06:10:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.547584+0000", 2023-10-09T06:10:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0315999999999994e-05 2023-10-09T06:10:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.605910+0000", 2023-10-09T06:10:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.058325845000000001 2023-10-09T06:10:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.605922+0000", 2023-10-09T06:10:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1712999999999999e-05 2023-10-09T06:10:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578649+0000", 2023-10-09T06:10:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97272731099999998 2023-10-09T06:10:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578660+0000", 2023-10-09T06:10:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0560999999999999e-05 2023-10-09T06:10:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.712239+0000", 2023-10-09T06:10:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.133579602 2023-10-09T06:10:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:10:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.712344+0000", 2023-10-09T06:10:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000104746 2023-10-09T06:10:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116277 9.7 9:e3c0da0d:::benchmark_data_smithi064_511911_object15175:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e449)", 2023-10-09T06:10:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.606356+0000", 2023-10-09T06:10:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 85.446225599000002, 2023-10-09T06:10:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1339391660000002, 2023-10-09T06:10:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:10:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:10:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116277 2023-10-09T06:10:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606356+0000", 2023-10-09T06:10:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606356+0000", 2023-10-09T06:10:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606355+0000", 2023-10-09T06:10:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:10:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606362+0000", 2023-10-09T06:10:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8249999999999999e-06 2023-10-09T06:10:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606372+0000", 2023-10-09T06:10:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0946e-05 2023-10-09T06:10:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606382+0000", 2023-10-09T06:10:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4849999999999998e-06 2023-10-09T06:10:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606441+0000", 2023-10-09T06:10:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8758000000000002e-05 2023-10-09T06:10:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606452+0000", 2023-10-09T06:10:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1435e-05 2023-10-09T06:10:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578004+0000", 2023-10-09T06:10:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97155135599999998 2023-10-09T06:10:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578015+0000", 2023-10-09T06:10:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1185999999999999e-05 2023-10-09T06:10:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.737752+0000", 2023-10-09T06:10:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.159737572 2023-10-09T06:10:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:10:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.740295+0000", 2023-10-09T06:10:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025430359999999998 2023-10-09T06:10:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116283 9.3 9:c10cf455:::benchmark_data_smithi064_511911_object15181:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e449)", 2023-10-09T06:10:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.606404+0000", 2023-10-09T06:10:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 85.446177528000007, 2023-10-09T06:10:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.160898918, 2023-10-09T06:10:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:10:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:10:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116283 2023-10-09T06:10:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606404+0000", 2023-10-09T06:10:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606404+0000", 2023-10-09T06:10:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606404+0000", 2023-10-09T06:10:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:10:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606407+0000", 2023-10-09T06:10:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9179999999999998e-06 2023-10-09T06:10:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606415+0000", 2023-10-09T06:10:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3180000000000007e-06 2023-10-09T06:10:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606421+0000", 2023-10-09T06:10:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2430000000000002e-06 2023-10-09T06:10:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606613+0000", 2023-10-09T06:10:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019148599999999999 2023-10-09T06:10:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606636+0000", 2023-10-09T06:10:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3680999999999999e-05 2023-10-09T06:10:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.576977+0000", 2023-10-09T06:10:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97034085699999995 2023-10-09T06:10:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.577913+0000", 2023-10-09T06:10:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00093576100000000001 2023-10-09T06:10:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.723002+0000", 2023-10-09T06:10:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1450891699999999 2023-10-09T06:10:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-10-09T06:10:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.723112+0000", 2023-10-09T06:10:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110089 2023-10-09T06:10:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.767258+0000", 2023-10-09T06:10:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.044145982 2023-10-09T06:10:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:10:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.767303+0000", 2023-10-09T06:10:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4894000000000001e-05 2023-10-09T06:10:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116296 9.1 9:8540568d:::benchmark_data_smithi064_511911_object15194:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2023-10-09T06:10:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.773495+0000", 2023-10-09T06:10:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 85.279087090000004, 2023-10-09T06:10:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.958274855, 2023-10-09T06:10:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:10:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116296 2023-10-09T06:10:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.773495+0000", 2023-10-09T06:10:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.773495+0000", 2023-10-09T06:10:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.773494+0000", 2023-10-09T06:10:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:10:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.773499+0000", 2023-10-09T06:10:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6319999999999999e-06 2023-10-09T06:10:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.773512+0000", 2023-10-09T06:10:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3288000000000001e-05 2023-10-09T06:10:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.773519+0000", 2023-10-09T06:10:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9090000000000003e-06 2023-10-09T06:10:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.773579+0000", 2023-10-09T06:10:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9871999999999999e-05 2023-10-09T06:10:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.773591+0000", 2023-10-09T06:10:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2526e-05 2023-10-09T06:10:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578692+0000", 2023-10-09T06:10:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.80510072099999996 2023-10-09T06:10:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578700+0000", 2023-10-09T06:10:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3380000000000005e-06 2023-10-09T06:10:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.712529+0000", 2023-10-09T06:10:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1338282500000001 2023-10-09T06:10:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:10:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.712695+0000", 2023-10-09T06:10:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000166875 2023-10-09T06:10:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6,7", 2023-10-09T06:10:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.713276+0000", 2023-10-09T06:10:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00058104399999999996 2023-10-09T06:10:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.720154+0000", 2023-10-09T06:10:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0068775190000000003 2023-10-09T06:10:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:10:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.723009+0000", 2023-10-09T06:10:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002855486 2023-10-09T06:10:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.731564+0000", 2023-10-09T06:10:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.008555053 2023-10-09T06:10:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.731603+0000", 2023-10-09T06:10:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8117000000000001e-05 2023-10-09T06:10:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:10:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.731663+0000", 2023-10-09T06:10:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0297000000000002e-05 2023-10-09T06:10:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:10:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.731770+0000", 2023-10-09T06:10:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000106616 2023-10-09T06:10:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116300 9.1 9:882a33e5:::benchmark_data_smithi064_511911_object15198:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2023-10-09T06:10:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.790061+0000", 2023-10-09T06:10:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 85.262520264000003, 2023-10-09T06:10:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.951866648, 2023-10-09T06:10:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:10:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116300 2023-10-09T06:10:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.790061+0000", 2023-10-09T06:10:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.790061+0000", 2023-10-09T06:10:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.790060+0000", 2023-10-09T06:10:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:10:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.790066+0000", 2023-10-09T06:10:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.434e-06 2023-10-09T06:10:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.790076+0000", 2023-10-09T06:10:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0404000000000001e-05 2023-10-09T06:10:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.790082+0000", 2023-10-09T06:10:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.908e-06 2023-10-09T06:10:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.790236+0000", 2023-10-09T06:10:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015350499999999999 2023-10-09T06:10:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.790243+0000", 2023-10-09T06:10:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6000000000000001e-06 2023-10-09T06:10:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578724+0000", 2023-10-09T06:10:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.78848058899999995 2023-10-09T06:10:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578734+0000", 2023-10-09T06:10:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6549999999999999e-06 2023-10-09T06:10:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.720179+0000", 2023-10-09T06:10:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1414453629999999 2023-10-09T06:10:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:10:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.720309+0000", 2023-10-09T06:10:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013024900000000001 2023-10-09T06:10:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6,7", 2023-10-09T06:10:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.720444+0000", 2023-10-09T06:10:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013511799999999999 2023-10-09T06:10:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:10:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.731215+0000", 2023-10-09T06:10:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010770693 2023-10-09T06:10:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.731528+0000", 2023-10-09T06:10:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000313109 2023-10-09T06:10:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.734686+0000", 2023-10-09T06:10:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031578380000000001 2023-10-09T06:10:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.741743+0000", 2023-10-09T06:10:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0070569559999999996 2023-10-09T06:10:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:10:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.741803+0000", 2023-10-09T06:10:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0517999999999999e-05 2023-10-09T06:10:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:10:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.741928+0000", 2023-10-09T06:10:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012472300000000001 2023-10-09T06:10:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:132917 9.8 9:12f89378:::benchmark_data_smithi064_511911_object31815:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e456)", 2023-10-09T06:10:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:26.626655+0000", 2023-10-09T06:10:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 61.425926672999999, 2023-10-09T06:10:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1614917130000002, 2023-10-09T06:10:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:10:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132917 2023-10-09T06:10:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.626655+0000", 2023-10-09T06:10:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.626655+0000", 2023-10-09T06:10:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.626653+0000", 2023-10-09T06:10:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:10:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.626661+0000", 2023-10-09T06:10:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7069999999999992e-06 2023-10-09T06:10:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.626673+0000", 2023-10-09T06:10:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2517999999999999e-05 2023-10-09T06:10:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.626683+0000", 2023-10-09T06:10:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4490000000000005e-06 2023-10-09T06:10:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.627477+0000", 2023-10-09T06:10:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00079445099999999995 2023-10-09T06:10:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.627484+0000", 2023-10-09T06:10:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4520000000000002e-06 2023-10-09T06:10:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528505+0000", 2023-10-09T06:10:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90102110700000004 2023-10-09T06:10:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528519+0000", 2023-10-09T06:10:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3943e-05 2023-10-09T06:10:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.755152+0000", 2023-10-09T06:10:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2266332689999999 2023-10-09T06:10:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:10:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.755289+0000", 2023-10-09T06:10:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013710400000000001 2023-10-09T06:10:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:10:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.755435+0000", 2023-10-09T06:10:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000145496 2023-10-09T06:10:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:10:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.766695+0000", 2023-10-09T06:10:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011260564000000001 2023-10-09T06:10:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.769909+0000", 2023-10-09T06:10:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032138940000000001 2023-10-09T06:10:28.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.785944+0000", 2023-10-09T06:10:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016034757 2023-10-09T06:10:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:10:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.785992+0000", 2023-10-09T06:10:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8412999999999999e-05 2023-10-09T06:10:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:10:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.788147+0000", 2023-10-09T06:10:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0021544020000000001 2023-10-09T06:10:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:132920 9.8 9:10a1d199:::benchmark_data_smithi064_511911_object31818:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e456)", 2023-10-09T06:10:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:26.641712+0000", 2023-10-09T06:10:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 61.410869382000001, 2023-10-09T06:10:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.148552274, 2023-10-09T06:10:28.210 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:10:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132920 2023-10-09T06:10:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.641712+0000", 2023-10-09T06:10:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.641712+0000", 2023-10-09T06:10:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.641710+0000", 2023-10-09T06:10:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:10:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.641717+0000", 2023-10-09T06:10:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0049999999999998e-06 2023-10-09T06:10:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.641731+0000", 2023-10-09T06:10:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3648e-05 2023-10-09T06:10:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.641743+0000", 2023-10-09T06:10:28.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1839e-05 2023-10-09T06:10:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.641792+0000", 2023-10-09T06:10:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9376000000000003e-05 2023-10-09T06:10:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.641803+0000", 2023-10-09T06:10:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1141000000000001e-05 2023-10-09T06:10:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528543+0000", 2023-10-09T06:10:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88673970899999999 2023-10-09T06:10:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528551+0000", 2023-10-09T06:10:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9319999999999996e-06 2023-10-09T06:10:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.760814+0000", 2023-10-09T06:10:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2322625380000001 2023-10-09T06:10:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:10:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.760963+0000", 2023-10-09T06:10:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000149522 2023-10-09T06:10:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:10:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.761205+0000", 2023-10-09T06:10:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000241831 2023-10-09T06:10:28.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:10:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.769866+0000", 2023-10-09T06:10:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086611819999999999 2023-10-09T06:10:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.769951+0000", 2023-10-09T06:10:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5286999999999998e-05 2023-10-09T06:10:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.218 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.788276+0000", 2023-10-09T06:10:28.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018324944999999999 2023-10-09T06:10:28.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.218 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:10:28.218 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.788329+0000", 2023-10-09T06:10:28.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2846999999999997e-05 2023-10-09T06:10:28.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.218 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:10:28.218 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.790265+0000", 2023-10-09T06:10:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019354929999999999 2023-10-09T06:10:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:132924 9.8 9:100587d5:::benchmark_data_smithi064_511911_object31822:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e456)", 2023-10-09T06:10:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:26.652624+0000", 2023-10-09T06:10:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 61.399957641999997, 2023-10-09T06:10:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1523453090000002, 2023-10-09T06:10:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:28.219 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:10:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132924 2023-10-09T06:10:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652624+0000", 2023-10-09T06:10:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652624+0000", 2023-10-09T06:10:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652616+0000", 2023-10-09T06:10:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999924 2023-10-09T06:10:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652632+0000", 2023-10-09T06:10:28.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5653000000000001e-05 2023-10-09T06:10:28.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:28.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652647+0000", 2023-10-09T06:10:28.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5244e-05 2023-10-09T06:10:28.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:28.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652656+0000", 2023-10-09T06:10:28.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0380000000000005e-06 2023-10-09T06:10:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652718+0000", 2023-10-09T06:10:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1267999999999997e-05 2023-10-09T06:10:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652730+0000", 2023-10-09T06:10:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2069000000000001e-05 2023-10-09T06:10:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528571+0000", 2023-10-09T06:10:28.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.875841753 2023-10-09T06:10:28.224 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528579+0000", 2023-10-09T06:10:28.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6410000000000002e-06 2023-10-09T06:10:28.224 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.764042+0000", 2023-10-09T06:10:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2354630339999999 2023-10-09T06:10:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:10:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.764130+0000", 2023-10-09T06:10:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7968999999999998e-05 2023-10-09T06:10:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:10:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.764240+0000", 2023-10-09T06:10:28.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010986 2023-10-09T06:10:28.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:10:28.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.769918+0000", 2023-10-09T06:10:28.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056776170000000003 2023-10-09T06:10:28.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.769984+0000", 2023-10-09T06:10:28.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6703000000000002e-05 2023-10-09T06:10:28.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.802902+0000", 2023-10-09T06:10:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.032917469999999997 2023-10-09T06:10:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:10:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.802957+0000", 2023-10-09T06:10:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5047999999999999e-05 2023-10-09T06:10:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:10:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.804969+0000", 2023-10-09T06:10:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020126900000000001 2023-10-09T06:10:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:132927 9.8 9:1f462da4:::benchmark_data_smithi064_511911_object31825:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e456)", 2023-10-09T06:10:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:26.663633+0000", 2023-10-09T06:10:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 61.388948737, 2023-10-09T06:10:28.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1462853150000001, 2023-10-09T06:10:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:10:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:10:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:10:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132927 2023-10-09T06:10:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:10:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:10:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.663633+0000", 2023-10-09T06:10:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.229 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:10:28.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.663633+0000", 2023-10-09T06:10:28.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:10:28.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:10:28.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.663631+0000", 2023-10-09T06:10:28.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:10:28.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:10:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.663639+0000", 2023-10-09T06:10:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3089999999999998e-06 2023-10-09T06:10:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:10:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.663652+0000", 2023-10-09T06:10:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3485999999999999e-05 2023-10-09T06:10:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:10:28.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.663658+0000", 2023-10-09T06:10:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.874e-06 2023-10-09T06:10:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.663716+0000", 2023-10-09T06:10:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8115000000000002e-05 2023-10-09T06:10:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.663728+0000", 2023-10-09T06:10:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.168e-05 2023-10-09T06:10:28.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528601+0000", 2023-10-09T06:10:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.86487267700000003 2023-10-09T06:10:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:10:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528608+0000", 2023-10-09T06:10:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2509999999999997e-06 2023-10-09T06:10:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.233 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:10:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.766727+0000", 2023-10-09T06:10:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2381187899999999 2023-10-09T06:10:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:10:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.766857+0000", 2023-10-09T06:10:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013015999999999999 2023-10-09T06:10:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:10:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.767000+0000", 2023-10-09T06:10:28.234 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014345199999999999 2023-10-09T06:10:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.770019+0000", 2023-10-09T06:10:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030185609999999999 2023-10-09T06:10:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:10:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.785896+0000", 2023-10-09T06:10:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015877437000000001 2023-10-09T06:10:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.808044+0000", 2023-10-09T06:10:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.022147962 2023-10-09T06:10:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:10:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.808105+0000", 2023-10-09T06:10:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0856000000000001e-05 2023-10-09T06:10:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:10:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:10:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:10:28.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.809918+0000", 2023-10-09T06:10:28.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018133559999999999 2023-10-09T06:10:28.237 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.237 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:28.237 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.237 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:10:28.237 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:10:28.237 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:10:28.238 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:27 smithi156 ceph-mon[195101]: 1.2 scrub starts 2023-10-09T06:10:28.238 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:27 smithi156 ceph-mon[195101]: failed to encode map e468 with expected crc 2023-10-09T06:10:28.238 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:27 smithi156 ceph-mon[195101]: osdmap e468: 8 total, 8 up, 8 in 2023-10-09T06:10:28.238 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:27 smithi156 ceph-mon[195101]: failed to encode map e468 with expected crc 2023-10-09T06:10:28.239 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:27 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 11957/115080 objects degraded (10.390%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:10:28.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:28.239 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:28.240 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:10:28.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:10:28.340 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:10:28.396 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:28.460 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:10:28.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:28.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:10:28.567 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:10:28.662 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:28.751 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:10:28.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:28.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:10:28.858 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:10:28.889 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:28.944 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:10:28.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:28 smithi064 ceph-mon[212750]: pgmap v1185: 119 pgs: 2 peering, 1 active+recovering+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 107 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 11559/106824 objects degraded (10.821%); 10341/106824 objects misplaced (9.680%); 5.2 MiB/s, 218 objects/s recovering 2023-10-09T06:10:28.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:28 smithi064 ceph-mon[212750]: 1.d deep-scrub starts 2023-10-09T06:10:28.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:28 smithi064 ceph-mon[212750]: failed to encode map e469 with expected crc 2023-10-09T06:10:28.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:28 smithi064 ceph-mon[212750]: osdmap e469: 8 total, 8 up, 8 in 2023-10-09T06:10:28.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:28 smithi064 ceph-mon[212750]: failed to encode map e469 with expected crc 2023-10-09T06:10:28.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:28 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:10:28.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:28 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:28.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:28 smithi064 ceph-mon[220044]: pgmap v1185: 119 pgs: 2 peering, 1 active+recovering+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 107 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 11559/106824 objects degraded (10.821%); 10341/106824 objects misplaced (9.680%); 5.2 MiB/s, 218 objects/s recovering 2023-10-09T06:10:28.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:28 smithi064 ceph-mon[220044]: 1.d deep-scrub starts 2023-10-09T06:10:28.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:28 smithi064 ceph-mon[220044]: failed to encode map e469 with expected crc 2023-10-09T06:10:28.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:28 smithi064 ceph-mon[220044]: osdmap e469: 8 total, 8 up, 8 in 2023-10-09T06:10:28.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:28 smithi064 ceph-mon[220044]: failed to encode map e469 with expected crc 2023-10-09T06:10:28.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:28 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:10:28.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:28 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:28.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:28.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:10:29.050 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:10:29.086 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:28 smithi156 ceph-mon[195101]: pgmap v1185: 119 pgs: 2 peering, 1 active+recovering+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 107 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 11559/106824 objects degraded (10.821%); 10341/106824 objects misplaced (9.680%); 5.2 MiB/s, 218 objects/s recovering 2023-10-09T06:10:29.086 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:28 smithi156 ceph-mon[195101]: 1.d deep-scrub starts 2023-10-09T06:10:29.086 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:28 smithi156 ceph-mon[195101]: failed to encode map e469 with expected crc 2023-10-09T06:10:29.086 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:28 smithi156 ceph-mon[195101]: osdmap e469: 8 total, 8 up, 8 in 2023-10-09T06:10:29.086 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:28 smithi156 ceph-mon[195101]: failed to encode map e469 with expected crc 2023-10-09T06:10:29.086 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:28 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:10:29.087 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:28 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:29.090 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:29.168 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:10:29.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:29.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:10:29.275 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:10:29.354 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:29.468 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:10:29.502 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:10:29.650 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:10:29.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:29.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:10:29.764 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:10:29.811 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:29 smithi156 ceph-mon[195101]: failed to encode map e470 with expected crc 2023-10-09T06:10:29.812 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:29 smithi156 ceph-mon[195101]: osdmap e470: 8 total, 8 up, 8 in 2023-10-09T06:10:29.812 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:29 smithi156 ceph-mon[195101]: failed to encode map e470 with expected crc 2023-10-09T06:10:29.812 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:29 smithi156 ceph-mon[195101]: failed to encode map e470 with expected crc 2023-10-09T06:10:29.824 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:29.888 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:10:29.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:29.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:10:29.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:29 smithi064 ceph-mon[212750]: failed to encode map e470 with expected crc 2023-10-09T06:10:29.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:29 smithi064 ceph-mon[212750]: osdmap e470: 8 total, 8 up, 8 in 2023-10-09T06:10:29.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:29 smithi064 ceph-mon[212750]: failed to encode map e470 with expected crc 2023-10-09T06:10:29.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:29 smithi064 ceph-mon[212750]: failed to encode map e470 with expected crc 2023-10-09T06:10:29.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:29 smithi064 ceph-mon[220044]: failed to encode map e470 with expected crc 2023-10-09T06:10:29.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:29 smithi064 ceph-mon[220044]: osdmap e470: 8 total, 8 up, 8 in 2023-10-09T06:10:29.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:29 smithi064 ceph-mon[220044]: failed to encode map e470 with expected crc 2023-10-09T06:10:29.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:29 smithi064 ceph-mon[220044]: failed to encode map e470 with expected crc 2023-10-09T06:10:30.004 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:10:30.076 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:10:30.107 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:30.405 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:10:30.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:30.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:10:30.514 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:10:30.573 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:30.648 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:30.703 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:30 smithi064 ceph-mon[212750]: pgmap v1188: 119 pgs: 2 peering, 1 active+recovering+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 107 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 9440/106263 objects degraded (8.884%); 10341/106263 objects misplaced (9.732%); 5.3 MiB/s, 149 objects/s recovering 2023-10-09T06:10:30.703 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:30 smithi064 ceph-mon[212750]: 1.2 scrub starts 2023-10-09T06:10:30.703 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:30 smithi064 ceph-mon[212750]: failed to encode map e471 with expected crc 2023-10-09T06:10:30.703 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:30 smithi064 ceph-mon[212750]: failed to encode map e471 with expected crc 2023-10-09T06:10:30.704 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:30 smithi064 ceph-mon[212750]: osdmap e471: 8 total, 8 up, 8 in 2023-10-09T06:10:30.704 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:30 smithi064 ceph-mon[212750]: failed to encode map e471 with expected crc 2023-10-09T06:10:30.798 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:10:30.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:30.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:10:30.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:30 smithi156 ceph-mon[195101]: pgmap v1188: 119 pgs: 2 peering, 1 active+recovering+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 107 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 9440/106263 objects degraded (8.884%); 10341/106263 objects misplaced (9.732%); 5.3 MiB/s, 149 objects/s recovering 2023-10-09T06:10:30.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:30 smithi156 ceph-mon[195101]: 1.2 scrub starts 2023-10-09T06:10:30.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:30 smithi156 ceph-mon[195101]: failed to encode map e471 with expected crc 2023-10-09T06:10:30.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:30 smithi156 ceph-mon[195101]: failed to encode map e471 with expected crc 2023-10-09T06:10:30.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:30 smithi156 ceph-mon[195101]: osdmap e471: 8 total, 8 up, 8 in 2023-10-09T06:10:30.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:30 smithi156 ceph-mon[195101]: failed to encode map e471 with expected crc 2023-10-09T06:10:30.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:30 smithi156 ceph-mon[195101]: failed to encode map e471 with expected crc 2023-10-09T06:10:30.904 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:10:30.967 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:30 smithi064 ceph-mon[212750]: failed to encode map e471 with expected crc 2023-10-09T06:10:30.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:30 smithi064 ceph-mon[220044]: pgmap v1188: 119 pgs: 2 peering, 1 active+recovering+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 107 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 9440/106263 objects degraded (8.884%); 10341/106263 objects misplaced (9.732%); 5.3 MiB/s, 149 objects/s recovering 2023-10-09T06:10:30.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:30 smithi064 ceph-mon[220044]: 1.2 scrub starts 2023-10-09T06:10:30.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:30 smithi064 ceph-mon[220044]: failed to encode map e471 with expected crc 2023-10-09T06:10:30.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:30 smithi064 ceph-mon[220044]: failed to encode map e471 with expected crc 2023-10-09T06:10:30.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:30 smithi064 ceph-mon[220044]: osdmap e471: 8 total, 8 up, 8 in 2023-10-09T06:10:30.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:30 smithi064 ceph-mon[220044]: failed to encode map e471 with expected crc 2023-10-09T06:10:30.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:30 smithi064 ceph-mon[220044]: failed to encode map e471 with expected crc 2023-10-09T06:10:30.987 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:31.181 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:10:31.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:31.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:10:31.294 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:10:31.339 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:31.402 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:10:31.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:31.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:10:31.508 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:10:31.547 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:31.615 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:10:31.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:31.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:10:31.722 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:10:31.764 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:10:31.767 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:31.834 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:10:31.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:31.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:10:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:31 smithi156 ceph-mon[195101]: 1.d scrub starts 2023-10-09T06:10:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:31 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/995158857' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:10:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:31 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:10:31.940 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:10:32.019 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:31 smithi064 ceph-mon[212750]: 1.d scrub starts 2023-10-09T06:10:32.019 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:31 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/995158857' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:10:32.019 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:31 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:10:32.020 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:31 smithi064 ceph-mon[220044]: 1.d scrub starts 2023-10-09T06:10:32.020 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:31 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/995158857' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:10:32.020 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:31 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:10:32.027 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:32.597 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1, 2] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 4, 0] 2023-10-09T06:10:32.597 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:10:32.598 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:10:32.598 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.000000 2023-10-09T06:10:32.598 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 0 2023-10-09T06:10:32.845 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:10:32.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:32.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:10:32.958 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:10:32.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:32 smithi064 ceph-mon[220044]: pgmap v1190: 119 pgs: 1 peering, 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 4.5 KiB/s rd, 12 KiB/s wr, 398 op/s; 10127/101592 objects degraded (9.968%); 11007/101592 objects misplaced (10.835%); 1.3 MiB/s, 60 objects/s recovering 2023-10-09T06:10:32.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:32 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:10:32.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:32 smithi064 ceph-mon[220044]: failed to encode map e472 with expected crc 2023-10-09T06:10:32.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:32 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:10:32.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:32 smithi064 ceph-mon[220044]: osdmap e472: 8 total, 8 up, 8 in 2023-10-09T06:10:32.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:32 smithi064 ceph-mon[220044]: failed to encode map e472 with expected crc 2023-10-09T06:10:32.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:32 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 10127/101592 objects degraded (9.968%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:32.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:32 smithi064 ceph-mon[212750]: pgmap v1190: 119 pgs: 1 peering, 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 4.5 KiB/s rd, 12 KiB/s wr, 398 op/s; 10127/101592 objects degraded (9.968%); 11007/101592 objects misplaced (10.835%); 1.3 MiB/s, 60 objects/s recovering 2023-10-09T06:10:32.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:32 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:10:32.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:32 smithi064 ceph-mon[212750]: failed to encode map e472 with expected crc 2023-10-09T06:10:32.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:32 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:10:32.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:32 smithi064 ceph-mon[212750]: osdmap e472: 8 total, 8 up, 8 in 2023-10-09T06:10:32.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:32 smithi064 ceph-mon[212750]: failed to encode map e472 with expected crc 2023-10-09T06:10:32.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:32 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 10127/101592 objects degraded (9.968%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:33.021 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:33.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:32 smithi156 ceph-mon[195101]: pgmap v1190: 119 pgs: 1 peering, 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 4.5 KiB/s rd, 12 KiB/s wr, 398 op/s; 10127/101592 objects degraded (9.968%); 11007/101592 objects misplaced (10.835%); 1.3 MiB/s, 60 objects/s recovering 2023-10-09T06:10:33.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:32 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:10:33.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:32 smithi156 ceph-mon[195101]: failed to encode map e472 with expected crc 2023-10-09T06:10:33.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:32 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:10:33.124 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:32 smithi156 ceph-mon[195101]: osdmap e472: 8 total, 8 up, 8 in 2023-10-09T06:10:33.124 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:32 smithi156 ceph-mon[195101]: failed to encode map e472 with expected crc 2023-10-09T06:10:33.124 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:32 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 10127/101592 objects degraded (9.968%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:33.576 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:10:33.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:33.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:10:33.693 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:10:33.728 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:33.804 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:10:33.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:33.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:10:33.909 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:10:34.003 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:33 smithi064 ceph-mon[212750]: pgmap v1192: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 3.7 KiB/s rd, 10 KiB/s wr, 596 op/s; 10303/97239 objects degraded (10.596%); 12759/97239 objects misplaced (13.121%); 1.4 MiB/s, 124 objects/s recovering 2023-10-09T06:10:34.007 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:33 smithi064 ceph-mon[220044]: pgmap v1192: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 3.7 KiB/s rd, 10 KiB/s wr, 596 op/s; 10303/97239 objects degraded (10.596%); 12759/97239 objects misplaced (13.121%); 1.4 MiB/s, 124 objects/s recovering 2023-10-09T06:10:34.020 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:34.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:33 smithi156 ceph-mon[195101]: pgmap v1192: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 3.7 KiB/s rd, 10 KiB/s wr, 596 op/s; 10303/97239 objects degraded (10.596%); 12759/97239 objects misplaced (13.121%); 1.4 MiB/s, 124 objects/s recovering 2023-10-09T06:10:34.163 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:34.241 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:10:34.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:34.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:10:34.334 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:10:34.348 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:10:34.385 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:10:34.392 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:34.418 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:10:34.457 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:10:34.470 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:10:34.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:34.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:10:34.493 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:10:34.558 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:10:34.574 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:10:34.597 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:10:34.609 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:34.636 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:10:34.641 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:10:34.641 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:10:34.641 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:10:34.642 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:10:34.642 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:10:34.642 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:10:34.642 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:10:34.642 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:10:34.691 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:10:34.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:34.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:10:34.763 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:34 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1786207578' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:10:34.797 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:10:34.839 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:34.911 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:10:34.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:34.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:10:35.018 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:10:35.057 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:34 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1786207578' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:10:35.060 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:35.156 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:10:35.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:35.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:10:35.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:34 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1786207578' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:10:35.264 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:10:35.340 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:35.556 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:10:35.583 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:10:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:10:35] "GET /metrics HTTP/1.1" 200 36471 "" "Prometheus/2.43.0" 2023-10-09T06:10:35.964 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:35 smithi064 ceph-mon[212750]: pgmap v1193: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 3.1 KiB/s rd, 8.6 KiB/s wr, 568 op/s; 10009/95994 objects degraded (10.427%); 12759/95994 objects misplaced (13.291%); 3.3 MiB/s, 139 objects/s recovering 2023-10-09T06:10:35.964 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:35 smithi064 ceph-mon[220044]: pgmap v1193: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 3.1 KiB/s rd, 8.6 KiB/s wr, 568 op/s; 10009/95994 objects degraded (10.427%); 12759/95994 objects misplaced (13.291%); 3.3 MiB/s, 139 objects/s recovering 2023-10-09T06:10:36.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:35 smithi156 ceph-mon[195101]: pgmap v1193: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 3.1 KiB/s rd, 8.6 KiB/s wr, 568 op/s; 10009/95994 objects degraded (10.427%); 12759/95994 objects misplaced (13.291%); 3.3 MiB/s, 139 objects/s recovering 2023-10-09T06:10:36.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:36.606 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:10:36.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:10:36.707 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:10:36.739 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:36.805 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:10:36.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:36.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:10:36.908 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:10:36.978 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:37.002 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:37.135 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:10:37.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:37.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:10:37.248 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:10:37.288 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:37.358 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:10:37.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:37.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:10:37.462 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:10:37.499 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:37.566 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:10:37.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:37.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:10:37.670 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:10:37.717 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:37.894 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:10:37.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:37.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:10:38.004 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:10:38.081 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:38.213 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:38 smithi156 ceph-mon[195101]: pgmap v1194: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.5 KiB/s rd, 6.9 KiB/s wr, 807 op/s; 9605/86928 objects degraded (11.049%); 11849/86928 objects misplaced (13.631%); 2.0 MiB/s, 137 objects/s recovering 2023-10-09T06:10:38.213 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:38 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 9605/86928 objects degraded (11.049%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:38.213 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:38 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3234566926' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T06:10:38.247 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:10:38.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:38.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:10:38.351 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:10:38.384 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:38.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:38 smithi064 ceph-mon[212750]: pgmap v1194: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.5 KiB/s rd, 6.9 KiB/s wr, 807 op/s; 9605/86928 objects degraded (11.049%); 11849/86928 objects misplaced (13.631%); 2.0 MiB/s, 137 objects/s recovering 2023-10-09T06:10:38.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:38 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 9605/86928 objects degraded (11.049%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:38.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:38 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3234566926' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T06:10:38.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:38 smithi064 ceph-mon[220044]: pgmap v1194: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 2.5 KiB/s rd, 6.9 KiB/s wr, 807 op/s; 9605/86928 objects degraded (11.049%); 11849/86928 objects misplaced (13.631%); 2.0 MiB/s, 137 objects/s recovering 2023-10-09T06:10:38.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:38 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 9605/86928 objects degraded (11.049%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:38.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:38 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3234566926' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T06:10:38.488 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:10:38.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:38.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:10:38.592 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:10:38.614 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.2 primary-affinity to 0 (802) 2023-10-09T06:10:38.648 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:10:38.691 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:38.725 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:10:38.726 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:10:38.903 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:10:38.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:38.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:10:39.023 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:10:39.095 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:39.404 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:10:39.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:39.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:10:39.511 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:10:39.558 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:39.640 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:10:39.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:39.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:10:39.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:39 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3234566926' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-10-09T06:10:39.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:39 smithi064 ceph-mon[220044]: osdmap e473: 8 total, 8 up, 8 in 2023-10-09T06:10:39.687 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:39 smithi064 ceph-mon[220044]: failed to encode map e473 with expected crc 2023-10-09T06:10:39.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:39 smithi064 ceph-mon[220044]: failed to encode map e473 with expected crc 2023-10-09T06:10:39.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:39 smithi064 ceph-mon[220044]: failed to encode map e473 with expected crc 2023-10-09T06:10:39.688 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:39 smithi064 ceph-mon[220044]: failed to encode map e473 with expected crc 2023-10-09T06:10:39.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:39 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3234566926' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-10-09T06:10:39.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:39 smithi064 ceph-mon[212750]: osdmap e473: 8 total, 8 up, 8 in 2023-10-09T06:10:39.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:39 smithi064 ceph-mon[212750]: failed to encode map e473 with expected crc 2023-10-09T06:10:39.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:39 smithi064 ceph-mon[212750]: failed to encode map e473 with expected crc 2023-10-09T06:10:39.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:39 smithi064 ceph-mon[212750]: failed to encode map e473 with expected crc 2023-10-09T06:10:39.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:39 smithi064 ceph-mon[212750]: failed to encode map e473 with expected crc 2023-10-09T06:10:39.747 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:10:39.797 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:39.868 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:10:39.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:39.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:10:39.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:39 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3234566926' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-10-09T06:10:39.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:39 smithi156 ceph-mon[195101]: osdmap e473: 8 total, 8 up, 8 in 2023-10-09T06:10:39.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:39 smithi156 ceph-mon[195101]: failed to encode map e473 with expected crc 2023-10-09T06:10:39.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:39 smithi156 ceph-mon[195101]: failed to encode map e473 with expected crc 2023-10-09T06:10:39.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:39 smithi156 ceph-mon[195101]: failed to encode map e473 with expected crc 2023-10-09T06:10:39.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:39 smithi156 ceph-mon[195101]: failed to encode map e473 with expected crc 2023-10-09T06:10:39.971 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:10:40.005 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:40.054 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:40.058 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:10:40.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:40.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:10:40.161 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:10:40.242 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:40.384 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:10:40.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:40.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:10:40.482 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:10:40.493 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:10:40.574 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:40.634 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:10:40.863 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:40 smithi064 ceph-mon[212750]: pgmap v1196: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 9404/86268 objects degraded (10.901%); 11849/86268 objects misplaced (13.735%); 3.5 MiB/s, 140 objects/s recovering 2023-10-09T06:10:40.864 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:40 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4000708916' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:10:40.864 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:40 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:10:40.864 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:40 smithi064 ceph-mon[220044]: pgmap v1196: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 9404/86268 objects degraded (10.901%); 11849/86268 objects misplaced (13.735%); 3.5 MiB/s, 140 objects/s recovering 2023-10-09T06:10:40.865 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:40 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4000708916' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:10:40.865 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:40 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:10:40.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:40 smithi156 ceph-mon[195101]: pgmap v1196: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 9404/86268 objects degraded (10.901%); 11849/86268 objects misplaced (13.735%); 3.5 MiB/s, 140 objects/s recovering 2023-10-09T06:10:40.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:40 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4000708916' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:10:40.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:40 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:10:41.017 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:10:41.221 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:10:41.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:41.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:10:41.336 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:10:41.390 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:41.445 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:10:41.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:41.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:10:41.551 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:10:41.597 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:41.643 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:41 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:10:41.643 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:41 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:10:41.644 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:41 smithi064 ceph-mon[212750]: osdmap e474: 8 total, 8 up, 8 in 2023-10-09T06:10:41.644 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:41 smithi064 ceph-mon[212750]: failed to encode map e474 with expected crc 2023-10-09T06:10:41.644 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:41 smithi064 ceph-mon[212750]: failed to encode map e474 with expected crc 2023-10-09T06:10:41.644 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:41 smithi064 ceph-mon[212750]: failed to encode map e474 with expected crc 2023-10-09T06:10:41.644 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:41 smithi064 ceph-mon[212750]: failed to encode map e474 with expected crc 2023-10-09T06:10:41.644 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:41 smithi064 ceph-mon[212750]: failed to encode map e474 with expected crc 2023-10-09T06:10:41.729 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:10:41.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:41.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:10:41.835 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:10:41.868 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:41 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:10:41.868 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:41 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:10:41.868 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:41 smithi156 ceph-mon[195101]: osdmap e474: 8 total, 8 up, 8 in 2023-10-09T06:10:41.869 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:41 smithi156 ceph-mon[195101]: failed to encode map e474 with expected crc 2023-10-09T06:10:41.869 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:41 smithi156 ceph-mon[195101]: failed to encode map e474 with expected crc 2023-10-09T06:10:41.869 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:41 smithi156 ceph-mon[195101]: failed to encode map e474 with expected crc 2023-10-09T06:10:41.869 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:41 smithi156 ceph-mon[195101]: failed to encode map e474 with expected crc 2023-10-09T06:10:41.869 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:41 smithi156 ceph-mon[195101]: failed to encode map e474 with expected crc 2023-10-09T06:10:41.877 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:41.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:41 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:10:41.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:41 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:10:41.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:41 smithi064 ceph-mon[220044]: osdmap e474: 8 total, 8 up, 8 in 2023-10-09T06:10:41.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:41 smithi064 ceph-mon[220044]: failed to encode map e474 with expected crc 2023-10-09T06:10:41.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:41 smithi064 ceph-mon[220044]: failed to encode map e474 with expected crc 2023-10-09T06:10:41.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:41 smithi064 ceph-mon[220044]: failed to encode map e474 with expected crc 2023-10-09T06:10:41.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:41 smithi064 ceph-mon[220044]: failed to encode map e474 with expected crc 2023-10-09T06:10:41.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:41 smithi064 ceph-mon[220044]: failed to encode map e474 with expected crc 2023-10-09T06:10:41.988 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:10:41.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:41.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:10:42.094 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:10:42.122 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:42.377 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:10:42.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:42.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:10:42.484 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:10:42.521 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:42.602 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:10:42.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:42.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:10:42.712 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:10:42.764 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:42.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:42 smithi156 ceph-mon[195101]: pgmap v1198: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 9322/81234 objects degraded (11.475%); 11149/81234 objects misplaced (13.725%); 3.3 MiB/s, 89 objects/s recovering 2023-10-09T06:10:42.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:42 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 9322/81234 objects degraded (11.475%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:42.906 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:10:42.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:42.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:10:42.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:42 smithi064 ceph-mon[220044]: pgmap v1198: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 9322/81234 objects degraded (11.475%); 11149/81234 objects misplaced (13.725%); 3.3 MiB/s, 89 objects/s recovering 2023-10-09T06:10:42.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:42 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 9322/81234 objects degraded (11.475%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:42.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:42 smithi064 ceph-mon[212750]: pgmap v1198: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 9322/81234 objects degraded (11.475%); 11149/81234 objects misplaced (13.725%); 3.3 MiB/s, 89 objects/s recovering 2023-10-09T06:10:42.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:42 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 9322/81234 objects degraded (11.475%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:43.018 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:10:43.073 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:43.142 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:10:43.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:43.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:10:43.180 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:10:43.221 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:10:43.249 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:10:43.263 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:10:43.310 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:10:43.336 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:43.349 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:10:43.391 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:10:43.433 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:10:43.439 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:10:43.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:43.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:10:43.472 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:10:43.483 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:10:43.483 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:10:43.483 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:10:43.483 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:10:43.483 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:10:43.483 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:10:43.483 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:10:43.483 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:10:43.553 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:10:43.600 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:43.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:43 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/792142285' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:10:43.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:43 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:10:43.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:43 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:43.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:43 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/792142285' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:10:43.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:43 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:10:43.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:43 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:43.682 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:10:43.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:43.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:10:43.738 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:10:43.739 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:10:43.739 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T06:10:43.739 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T06:10:43.739 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:10:43.789 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:10:43.850 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:43 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/792142285' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:10:43.850 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:43 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:10:43.850 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:43 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:43.862 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:44.202 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:10:44.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:44.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:10:44.307 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:10:44.360 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:10:44.371 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:44.593 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1, 2] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 4, 0] 2023-10-09T06:10:44.593 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:10:44.593 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:10:44.593 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 0.000000 2023-10-09T06:10:44.594 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 0 2023-10-09T06:10:44.681 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:44 smithi064 ceph-mon[212750]: pgmap v1199: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 8881/76818 objects degraded (11.561%); 10937/76818 objects misplaced (14.238%); 1.6 MiB/s, 103 objects/s recovering 2023-10-09T06:10:44.681 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:44 smithi064 ceph-mon[220044]: pgmap v1199: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 8881/76818 objects degraded (11.561%); 10937/76818 objects misplaced (14.238%); 1.6 MiB/s, 103 objects/s recovering 2023-10-09T06:10:44.729 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:10:44.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:44.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:10:44.850 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:10:44.930 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:45.080 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:10:45.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:45.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:10:45.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:44 smithi156 ceph-mon[195101]: pgmap v1199: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 8881/76818 objects degraded (11.561%); 10937/76818 objects misplaced (14.238%); 1.6 MiB/s, 103 objects/s recovering 2023-10-09T06:10:45.199 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:10:45.205 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:10:45.264 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:45.288 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:45.401 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:10:45.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:45.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:10:45.504 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:45.515 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:10:45.596 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:10:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:10:45] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.43.0" 2023-10-09T06:10:45.605 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:45.725 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:10:45.806 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:10:45.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:45.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:10:45.918 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:10:45.981 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:46.629 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:10:46.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:46.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:10:46.750 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:10:46.775 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:10:46.793 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:46.842 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:10:46.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:46.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:10:46.945 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:10:46.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:46 smithi064 ceph-mon[212750]: pgmap v1200: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 8592/75573 objects degraded (11.369%); 10937/75573 objects misplaced (14.472%); 3.3 MiB/s, 94 objects/s recovering 2023-10-09T06:10:46.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:46 smithi064 ceph-mon[220044]: pgmap v1200: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 8592/75573 objects degraded (11.369%); 10937/75573 objects misplaced (14.472%); 3.3 MiB/s, 94 objects/s recovering 2023-10-09T06:10:46.970 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:46 smithi156 ceph-mon[195101]: pgmap v1200: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 8592/75573 objects degraded (11.369%); 10937/75573 objects misplaced (14.472%); 3.3 MiB/s, 94 objects/s recovering 2023-10-09T06:10:46.973 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:47.031 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:10:47.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:47.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:10:47.135 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:10:47.209 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:47.678 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:10:47.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:47.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:10:47.786 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:10:47.815 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:47 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:10:47.815 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:47 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 8592/75573 objects degraded (11.369%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:47.820 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:47.936 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:10:47.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:47.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:10:47.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:47 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:10:47.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:47 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 8592/75573 objects degraded (11.369%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:47.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:47 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:10:47.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:47 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 8592/75573 objects degraded (11.369%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:48.041 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:10:48.156 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:48.202 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:10:48.202 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T06:10:48.202 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T06:10:48.202 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:10:48.202 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.202 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:66801 8.8 e370/360 8:1e71bc11:::benchmark_data_smithi064_457003_object66800:head v 370'4119, mlcod=370'4117)", 2023-10-09T06:10:48.202 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:03:47.803790+0000", 2023-10-09T06:10:48.203 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 420.392580366, 2023-10-09T06:10:48.203 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.244972208, 2023-10-09T06:10:48.203 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:10:48.203 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:48.203 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:10:48.203 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.203 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:10:48.203 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.803790+0000", 2023-10-09T06:10:48.203 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.203 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.203 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.203 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:10:48.204 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.803790+0000", 2023-10-09T06:10:48.204 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.204 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.204 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.204 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:10:48.204 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.803789+0000", 2023-10-09T06:10:48.204 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:10:48.204 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.204 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.204 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:10:48.204 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.803850+0000", 2023-10-09T06:10:48.204 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0393000000000003e-05 2023-10-09T06:10:48.205 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.205 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.205 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:10:48.205 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.803861+0000", 2023-10-09T06:10:48.205 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0635999999999999e-05 2023-10-09T06:10:48.205 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.205 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.205 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:10:48.205 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.803866+0000", 2023-10-09T06:10:48.205 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5310000000000001e-06 2023-10-09T06:10:48.205 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.205 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.206 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.206 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.803906+0000", 2023-10-09T06:10:48.206 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0277000000000001e-05 2023-10-09T06:10:48.206 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.206 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.206 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:10:48.206 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.803924+0000", 2023-10-09T06:10:48.206 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8063e-05 2023-10-09T06:10:48.206 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.206 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.206 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:10:48.207 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.805945+0000", 2023-10-09T06:10:48.207 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0020206209999999998 2023-10-09T06:10:48.207 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.207 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.207 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:10:48.207 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048763+0000", 2023-10-09T06:10:48.207 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24281751800000001 2023-10-09T06:10:48.207 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.207 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:10:48.207 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.207 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.207 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.208 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:66805 8.d e370/360 8:beda68c4:::benchmark_data_smithi064_457003_object66804:head v 370'4056, mlcod=370'4054)", 2023-10-09T06:10:48.208 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:03:47.805362+0000", 2023-10-09T06:10:48.208 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 420.39100888000002, 2023-10-09T06:10:48.208 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24348254499999999, 2023-10-09T06:10:48.208 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:10:48.208 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:48.208 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:10:48.208 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.208 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:10:48.208 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.805362+0000", 2023-10-09T06:10:48.208 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.208 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.209 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.209 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:10:48.209 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.805362+0000", 2023-10-09T06:10:48.209 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.209 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.209 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.209 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:10:48.209 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.805361+0000", 2023-10-09T06:10:48.209 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:10:48.209 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.209 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.210 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:10:48.210 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.805385+0000", 2023-10-09T06:10:48.210 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4508999999999999e-05 2023-10-09T06:10:48.210 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.210 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.210 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:10:48.210 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.805396+0000", 2023-10-09T06:10:48.210 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0275000000000001e-05 2023-10-09T06:10:48.210 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.210 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.210 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:10:48.211 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.805402+0000", 2023-10-09T06:10:48.211 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1670000000000004e-06 2023-10-09T06:10:48.211 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.211 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.211 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.211 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.805446+0000", 2023-10-09T06:10:48.211 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4209999999999999e-05 2023-10-09T06:10:48.211 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.211 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.211 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:10:48.211 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.805464+0000", 2023-10-09T06:10:48.212 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7649e-05 2023-10-09T06:10:48.212 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.212 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.212 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:10:48.212 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048738+0000", 2023-10-09T06:10:48.212 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.243274142 2023-10-09T06:10:48.212 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.212 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.212 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:10:48.212 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048844+0000", 2023-10-09T06:10:48.212 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010647200000000001 2023-10-09T06:10:48.212 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.212 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:10:48.213 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.213 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.213 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.213 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:66804 8.a e370/360 8:503f273d:::benchmark_data_smithi064_457003_object66803:head v 370'4302, mlcod=370'4300)", 2023-10-09T06:10:48.213 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:03:47.807368+0000", 2023-10-09T06:10:48.213 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 420.38900280600001, 2023-10-09T06:10:48.213 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24623639999999999, 2023-10-09T06:10:48.213 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:10:48.213 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:48.213 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:10:48.213 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.214 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:10:48.214 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.807368+0000", 2023-10-09T06:10:48.214 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.214 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.214 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.214 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:10:48.214 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.807368+0000", 2023-10-09T06:10:48.214 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.214 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.214 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.214 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:10:48.214 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.807367+0000", 2023-10-09T06:10:48.215 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:10:48.215 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.215 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.215 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:10:48.215 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.807414+0000", 2023-10-09T06:10:48.215 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6770000000000001e-05 2023-10-09T06:10:48.215 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.215 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.215 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:10:48.215 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048692+0000", 2023-10-09T06:10:48.215 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24127759100000001 2023-10-09T06:10:48.215 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.216 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.216 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:10:48.216 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048718+0000", 2023-10-09T06:10:48.216 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.6829e-05 2023-10-09T06:10:48.216 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.216 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.216 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.216 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048837+0000", 2023-10-09T06:10:48.216 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011845099999999999 2023-10-09T06:10:48.216 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.216 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.216 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:10:48.217 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048858+0000", 2023-10-09T06:10:48.217 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1131000000000001e-05 2023-10-09T06:10:48.217 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.217 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.217 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:10:48.217 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.053554+0000", 2023-10-09T06:10:48.217 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0046963669999999999 2023-10-09T06:10:48.217 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.217 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.217 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:10:48.217 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.053604+0000", 2023-10-09T06:10:48.218 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9839000000000002e-05 2023-10-09T06:10:48.218 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.218 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:10:48.218 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.218 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.218 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.218 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:66806 8.d e370/360 8:ba263c5e:::benchmark_data_smithi064_457003_object66805:head v 370'4057, mlcod=370'4054)", 2023-10-09T06:10:48.218 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:03:47.808240+0000", 2023-10-09T06:10:48.218 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 420.388130483, 2023-10-09T06:10:48.218 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24444445000000001, 2023-10-09T06:10:48.218 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:10:48.219 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:48.219 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:10:48.219 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.219 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:10:48.219 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.808240+0000", 2023-10-09T06:10:48.219 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.219 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.219 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.219 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:10:48.219 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.808240+0000", 2023-10-09T06:10:48.219 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.219 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.220 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.220 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:10:48.220 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.808240+0000", 2023-10-09T06:10:48.220 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:10:48.220 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.220 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.220 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:10:48.220 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.808280+0000", 2023-10-09T06:10:48.220 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0431e-05 2023-10-09T06:10:48.220 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.220 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.220 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:10:48.221 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048727+0000", 2023-10-09T06:10:48.221 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.240446787 2023-10-09T06:10:48.221 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.221 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.221 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:10:48.221 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048747+0000", 2023-10-09T06:10:48.221 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0438999999999999e-05 2023-10-09T06:10:48.221 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.221 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.221 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.221 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048872+0000", 2023-10-09T06:10:48.222 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012439300000000001 2023-10-09T06:10:48.222 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.222 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.222 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:10:48.222 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048889+0000", 2023-10-09T06:10:48.222 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7540000000000001e-05 2023-10-09T06:10:48.222 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.222 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.222 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:10:48.222 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.052646+0000", 2023-10-09T06:10:48.222 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0037566460000000002 2023-10-09T06:10:48.222 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.223 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.223 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:10:48.223 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.052685+0000", 2023-10-09T06:10:48.223 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8875999999999998e-05 2023-10-09T06:10:48.223 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.223 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:10:48.223 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.223 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.223 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.223 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:66808 8.8 e370/360 8:173e3c76:::benchmark_data_smithi064_457003_object66807:head v 370'4120, mlcod=370'4117)", 2023-10-09T06:10:48.223 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:03:47.809035+0000", 2023-10-09T06:10:48.223 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 420.387335484, 2023-10-09T06:10:48.224 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24526658800000001, 2023-10-09T06:10:48.224 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:10:48.224 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:48.224 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:10:48.224 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.224 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:10:48.224 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.809035+0000", 2023-10-09T06:10:48.224 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.224 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.224 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.224 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:10:48.225 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.809035+0000", 2023-10-09T06:10:48.225 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.225 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.225 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.225 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:10:48.225 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.809035+0000", 2023-10-09T06:10:48.225 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:10:48.225 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.225 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.225 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:10:48.225 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:47.809092+0000", 2023-10-09T06:10:48.225 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7357e-05 2023-10-09T06:10:48.226 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.226 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.226 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:10:48.226 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048698+0000", 2023-10-09T06:10:48.226 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.239606072 2023-10-09T06:10:48.226 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.226 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.226 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:10:48.226 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048723+0000", 2023-10-09T06:10:48.226 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.5075999999999998e-05 2023-10-09T06:10:48.226 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.226 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.227 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.227 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048829+0000", 2023-10-09T06:10:48.227 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000105544 2023-10-09T06:10:48.227 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.227 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.227 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:10:48.227 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.048855+0000", 2023-10-09T06:10:48.227 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.6302999999999999e-05 2023-10-09T06:10:48.227 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.227 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.227 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:10:48.227 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.054261+0000", 2023-10-09T06:10:48.228 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0054058939999999996 2023-10-09T06:10:48.228 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.228 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.228 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:10:48.228 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:48.054302+0000", 2023-10-09T06:10:48.228 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1029999999999998e-05 2023-10-09T06:10:48.228 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.228 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:10:48.228 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.228 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.228 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.228 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:68174 8.0 e370/360 8:0d01f709:::benchmark_data_smithi064_457003_object68173:head v 370'4157, mlcod=370'4153)", 2023-10-09T06:10:48.229 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:03:49.347132+0000", 2023-10-09T06:10:48.229 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 418.84923905300002, 2023-10-09T06:10:48.229 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.23505520799999999, 2023-10-09T06:10:48.229 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:10:48.229 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:48.229 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:10:48.229 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.229 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:10:48.229 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.347132+0000", 2023-10-09T06:10:48.229 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.229 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.229 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.230 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:10:48.230 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.347132+0000", 2023-10-09T06:10:48.230 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.230 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.230 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.230 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:10:48.230 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.347130+0000", 2023-10-09T06:10:48.230 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:10:48.230 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.230 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.230 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:10:48.230 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.347209+0000", 2023-10-09T06:10:48.231 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9308999999999996e-05 2023-10-09T06:10:48.231 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.231 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.231 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:10:48.231 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.347230+0000", 2023-10-09T06:10:48.231 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0126999999999999e-05 2023-10-09T06:10:48.231 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.231 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.231 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:10:48.231 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.347239+0000", 2023-10-09T06:10:48.231 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.9599999999999995e-06 2023-10-09T06:10:48.231 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.232 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.232 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.232 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.572401+0000", 2023-10-09T06:10:48.232 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.22516175399999999 2023-10-09T06:10:48.232 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.232 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.232 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:10:48.232 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.572432+0000", 2023-10-09T06:10:48.232 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0445000000000001e-05 2023-10-09T06:10:48.232 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.232 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.232 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:10:48.233 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.582162+0000", 2023-10-09T06:10:48.233 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0097302889999999996 2023-10-09T06:10:48.233 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.233 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.233 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:10:48.233 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:03:49.582187+0000", 2023-10-09T06:10:48.233 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4878000000000001e-05 2023-10-09T06:10:48.233 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.233 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:10:48.233 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.233 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.233 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.234 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:113850 8.8 e378/360 8:1788d6a1:::benchmark_data_smithi064_457003_object9332:head v 378'7157, mlcod=378'7151)", 2023-10-09T06:10:48.234 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:04:44.127135+0000", 2023-10-09T06:10:48.234 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 364.06923545199999, 2023-10-09T06:10:48.234 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.23983844600000001, 2023-10-09T06:10:48.234 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:10:48.234 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:48.234 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:10:48.234 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.234 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:10:48.234 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:44.127135+0000", 2023-10-09T06:10:48.234 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.234 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.235 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.235 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:10:48.235 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:44.127135+0000", 2023-10-09T06:10:48.235 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.235 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.235 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.235 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:10:48.235 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:44.127135+0000", 2023-10-09T06:10:48.235 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:10:48.235 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.235 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.235 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:10:48.236 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:44.127144+0000", 2023-10-09T06:10:48.236 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.696e-06 2023-10-09T06:10:48.236 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.236 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.236 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:10:48.236 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:44.361103+0000", 2023-10-09T06:10:48.236 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.233958529 2023-10-09T06:10:48.236 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.236 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.236 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:10:48.236 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:44.361115+0000", 2023-10-09T06:10:48.236 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1528e-05 2023-10-09T06:10:48.237 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.237 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.237 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.237 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:44.361156+0000", 2023-10-09T06:10:48.237 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1081000000000002e-05 2023-10-09T06:10:48.237 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.237 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.237 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:10:48.237 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:44.361178+0000", 2023-10-09T06:10:48.237 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2473999999999999e-05 2023-10-09T06:10:48.237 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.238 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.238 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:10:48.238 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:44.366932+0000", 2023-10-09T06:10:48.238 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0057534700000000001 2023-10-09T06:10:48.238 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.238 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.238 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:10:48.238 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:44.366974+0000", 2023-10-09T06:10:48.238 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2103e-05 2023-10-09T06:10:48.238 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.238 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:10:48.238 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.239 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.239 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.239 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.45605.0:113849 8.3 e378/360 8:c60b32df:::benchmark_data_smithi064_457003_object9331:head v 378'7212, mlcod=378'7210)", 2023-10-09T06:10:48.239 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:04:44.127145+0000", 2023-10-09T06:10:48.239 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 364.06922577699999, 2023-10-09T06:10:48.239 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24029709299999999, 2023-10-09T06:10:48.239 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:10:48.239 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:48.239 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:10:48.239 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.239 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:10:48.240 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:44.127145+0000", 2023-10-09T06:10:48.240 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.240 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.240 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.240 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:10:48.240 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:44.127145+0000", 2023-10-09T06:10:48.240 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.240 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.240 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.240 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:10:48.240 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:44.127144+0000", 2023-10-09T06:10:48.240 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:10:48.240 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.241 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.241 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:10:48.241 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:44.127154+0000", 2023-10-09T06:10:48.241 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7720000000000006e-06 2023-10-09T06:10:48.241 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.241 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.241 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:10:48.241 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:44.364061+0000", 2023-10-09T06:10:48.241 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.23690719700000001 2023-10-09T06:10:48.241 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.241 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.242 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:10:48.242 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:44.364072+0000", 2023-10-09T06:10:48.242 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0312999999999999e-05 2023-10-09T06:10:48.242 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.242 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.242 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.242 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:44.364116+0000", 2023-10-09T06:10:48.242 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4103000000000001e-05 2023-10-09T06:10:48.242 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.242 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.242 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:10:48.242 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:44.364136+0000", 2023-10-09T06:10:48.243 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0761e-05 2023-10-09T06:10:48.243 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.243 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.243 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:10:48.243 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:44.367402+0000", 2023-10-09T06:10:48.243 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0032653729999999998 2023-10-09T06:10:48.243 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.243 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.243 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:10:48.243 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:04:44.367442+0000", 2023-10-09T06:10:48.243 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0204999999999999e-05 2023-10-09T06:10:48.243 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.244 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:10:48.244 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.244 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.244 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.244 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:118905 9.5 9:abe25aca:::benchmark_data_smithi064_511911_object17803:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T06:10:48.244 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:07.826739+0000", 2023-10-09T06:10:48.244 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 100.36963196400001, 2023-10-09T06:10:48.244 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.28890842500000002, 2023-10-09T06:10:48.244 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:10:48.244 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:48.244 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:10:48.244 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:10:48.245 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:48.245 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 118905 2023-10-09T06:10:48.245 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.245 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:10:48.245 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.245 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:10:48.245 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.826739+0000", 2023-10-09T06:10:48.245 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.245 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.245 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.245 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:10:48.245 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.826739+0000", 2023-10-09T06:10:48.246 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.246 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.246 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.246 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:10:48.246 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.826737+0000", 2023-10-09T06:10:48.246 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:10:48.246 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.246 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.246 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:10:48.246 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.826744+0000", 2023-10-09T06:10:48.246 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5339999999999996e-06 2023-10-09T06:10:48.246 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.247 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.247 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:10:48.247 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.826760+0000", 2023-10-09T06:10:48.247 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6149000000000001e-05 2023-10-09T06:10:48.247 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.247 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.247 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:10:48.247 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.826766+0000", 2023-10-09T06:10:48.247 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4559999999999997e-06 2023-10-09T06:10:48.247 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.247 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.247 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.248 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.826818+0000", 2023-10-09T06:10:48.248 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1634000000000001e-05 2023-10-09T06:10:48.248 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.248 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.248 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:10:48.248 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.826957+0000", 2023-10-09T06:10:48.248 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013895199999999999 2023-10-09T06:10:48.248 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.248 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.248 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:10:48.248 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.827119+0000", 2023-10-09T06:10:48.248 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000161979 2023-10-09T06:10:48.249 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.249 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.249 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:10:48.249 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.835830+0000", 2023-10-09T06:10:48.249 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0087111010000000006 2023-10-09T06:10:48.249 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.249 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.249 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:48.249 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.837632+0000", 2023-10-09T06:10:48.249 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001801506 2023-10-09T06:10:48.249 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.249 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.250 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:48.250 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.115484+0000", 2023-10-09T06:10:48.250 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.27785286599999998 2023-10-09T06:10:48.250 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.250 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.250 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:10:48.250 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.115545+0000", 2023-10-09T06:10:48.250 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0812e-05 2023-10-09T06:10:48.250 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.250 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.250 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:10:48.250 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.115647+0000", 2023-10-09T06:10:48.251 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010185500000000001 2023-10-09T06:10:48.251 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.251 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:10:48.251 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.251 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.251 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.251 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:118910 9.5 9:aa92e6b2:::benchmark_data_smithi064_511911_object17808:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T06:10:48.251 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:07.827816+0000", 2023-10-09T06:10:48.251 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 100.368554863, 2023-10-09T06:10:48.251 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.28799813499999999, 2023-10-09T06:10:48.251 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:10:48.251 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:48.252 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:10:48.252 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:10:48.252 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:48.252 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 118910 2023-10-09T06:10:48.252 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.252 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:10:48.252 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.252 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:10:48.252 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.827816+0000", 2023-10-09T06:10:48.252 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.252 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.252 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.253 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:10:48.253 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.827816+0000", 2023-10-09T06:10:48.253 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.253 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.253 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.253 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:10:48.253 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.827815+0000", 2023-10-09T06:10:48.253 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:10:48.253 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.253 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.253 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:10:48.253 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.827824+0000", 2023-10-09T06:10:48.254 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0729999999999994e-06 2023-10-09T06:10:48.254 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.254 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.254 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:10:48.254 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.827836+0000", 2023-10-09T06:10:48.254 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2374999999999999e-05 2023-10-09T06:10:48.254 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.254 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.254 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:10:48.254 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.827843+0000", 2023-10-09T06:10:48.254 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2079999999999997e-06 2023-10-09T06:10:48.255 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.255 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.255 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.255 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.830165+0000", 2023-10-09T06:10:48.255 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0023226179999999998 2023-10-09T06:10:48.255 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.255 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.255 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:10:48.255 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.830259+0000", 2023-10-09T06:10:48.255 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3519999999999999e-05 2023-10-09T06:10:48.255 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.255 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.256 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:10:48.256 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.830357+0000", 2023-10-09T06:10:48.256 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7931999999999997e-05 2023-10-09T06:10:48.256 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.256 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.256 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:10:48.256 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.835838+0000", 2023-10-09T06:10:48.256 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0054813279999999997 2023-10-09T06:10:48.256 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.257 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.257 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:48.257 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.837660+0000", 2023-10-09T06:10:48.257 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0018216009999999999 2023-10-09T06:10:48.257 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.257 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.257 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:48.257 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.115684+0000", 2023-10-09T06:10:48.257 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.27802399700000002 2023-10-09T06:10:48.257 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.257 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.257 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:10:48.258 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.115722+0000", 2023-10-09T06:10:48.258 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8550999999999997e-05 2023-10-09T06:10:48.258 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.258 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.258 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:10:48.258 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.115814+0000", 2023-10-09T06:10:48.258 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1831999999999998e-05 2023-10-09T06:10:48.258 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.258 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:10:48.258 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.258 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.259 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.259 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:118921 9.5 9:a45440d7:::benchmark_data_smithi064_511911_object17819:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T06:10:48.259 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:07.843763+0000", 2023-10-09T06:10:48.259 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 100.352607322, 2023-10-09T06:10:48.259 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.27222496800000001, 2023-10-09T06:10:48.259 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:10:48.259 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:48.259 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:10:48.259 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:10:48.259 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:48.259 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 118921 2023-10-09T06:10:48.259 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.260 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:10:48.260 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.260 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:10:48.260 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.843763+0000", 2023-10-09T06:10:48.260 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.260 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.260 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.260 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:10:48.260 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.843763+0000", 2023-10-09T06:10:48.260 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.260 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.260 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.261 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:10:48.261 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.843762+0000", 2023-10-09T06:10:48.261 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:10:48.261 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.261 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.261 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:10:48.261 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.843769+0000", 2023-10-09T06:10:48.261 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8059999999999997e-06 2023-10-09T06:10:48.261 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.261 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.261 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:10:48.261 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.843780+0000", 2023-10-09T06:10:48.262 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1321000000000001e-05 2023-10-09T06:10:48.262 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.262 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.262 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:10:48.262 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.843787+0000", 2023-10-09T06:10:48.262 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7839999999999998e-06 2023-10-09T06:10:48.262 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.262 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.262 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.262 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.843825+0000", 2023-10-09T06:10:48.262 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8098e-05 2023-10-09T06:10:48.263 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.263 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.263 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:10:48.263 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.843921+0000", 2023-10-09T06:10:48.263 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.6386999999999997e-05 2023-10-09T06:10:48.263 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.263 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.263 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:10:48.263 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.843999+0000", 2023-10-09T06:10:48.263 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7979000000000007e-05 2023-10-09T06:10:48.264 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.264 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.264 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:10:48.264 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.847809+0000", 2023-10-09T06:10:48.264 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0038100170000000002 2023-10-09T06:10:48.264 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.264 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.264 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:48.264 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.854122+0000", 2023-10-09T06:10:48.264 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0063132070000000004 2023-10-09T06:10:48.264 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.264 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.265 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:48.265 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.115847+0000", 2023-10-09T06:10:48.265 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.26172430899999999 2023-10-09T06:10:48.265 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.265 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.265 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:10:48.265 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.115895+0000", 2023-10-09T06:10:48.265 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8265e-05 2023-10-09T06:10:48.265 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.265 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.265 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:10:48.265 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.115988+0000", 2023-10-09T06:10:48.265 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3286000000000001e-05 2023-10-09T06:10:48.266 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.266 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:10:48.266 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.266 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.266 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.266 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:118928 9.5 9:a5a9efbb:::benchmark_data_smithi064_511911_object17826:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T06:10:48.266 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:07.850035+0000", 2023-10-09T06:10:48.266 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 100.34633571400001, 2023-10-09T06:10:48.266 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.26612815200000001, 2023-10-09T06:10:48.266 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:10:48.266 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:48.267 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:10:48.267 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:10:48.267 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:48.267 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 118928 2023-10-09T06:10:48.267 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.267 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:10:48.267 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.267 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:10:48.267 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.850035+0000", 2023-10-09T06:10:48.267 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.267 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.267 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.268 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:10:48.268 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.850035+0000", 2023-10-09T06:10:48.268 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.268 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.268 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.268 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:10:48.268 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.850034+0000", 2023-10-09T06:10:48.268 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:10:48.268 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.268 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.268 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:10:48.268 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.850041+0000", 2023-10-09T06:10:48.269 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0230000000000004e-06 2023-10-09T06:10:48.269 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.269 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.269 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:10:48.269 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.850052+0000", 2023-10-09T06:10:48.269 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1402e-05 2023-10-09T06:10:48.269 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.269 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.269 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:10:48.269 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.850057+0000", 2023-10-09T06:10:48.269 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.826e-06 2023-10-09T06:10:48.269 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.270 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.270 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.270 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.850099+0000", 2023-10-09T06:10:48.270 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1897999999999998e-05 2023-10-09T06:10:48.270 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.270 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.270 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:10:48.270 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.850235+0000", 2023-10-09T06:10:48.270 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013637400000000001 2023-10-09T06:10:48.270 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.270 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.271 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:10:48.271 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.850361+0000", 2023-10-09T06:10:48.271 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012588500000000001 2023-10-09T06:10:48.271 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.271 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.271 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:10:48.271 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.853287+0000", 2023-10-09T06:10:48.271 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0029253259999999998 2023-10-09T06:10:48.271 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.271 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.271 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:48.271 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.854302+0000", 2023-10-09T06:10:48.271 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001015192 2023-10-09T06:10:48.272 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.272 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.272 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:48.272 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.116034+0000", 2023-10-09T06:10:48.272 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.26173186300000001 2023-10-09T06:10:48.272 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.272 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.272 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:10:48.272 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.116072+0000", 2023-10-09T06:10:48.272 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8405999999999998e-05 2023-10-09T06:10:48.272 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.272 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.273 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:10:48.273 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.116163+0000", 2023-10-09T06:10:48.273 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1043000000000002e-05 2023-10-09T06:10:48.273 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.273 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:10:48.273 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.273 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.273 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.273 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:132908 9.5 9:a62fba3d:::benchmark_data_smithi064_511911_object31806:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e455)", 2023-10-09T06:10:48.273 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:26.572407+0000", 2023-10-09T06:10:48.273 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 81.623963619999998, 2023-10-09T06:10:48.273 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.177650452, 2023-10-09T06:10:48.274 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:10:48.274 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:10:48.274 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:10:48.274 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:10:48.274 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:48.274 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 132908 2023-10-09T06:10:48.274 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.274 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:10:48.274 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.274 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:10:48.274 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572407+0000", 2023-10-09T06:10:48.274 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.275 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.275 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.275 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:10:48.275 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572407+0000", 2023-10-09T06:10:48.275 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.275 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.275 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.275 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:10:48.275 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572406+0000", 2023-10-09T06:10:48.275 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:10:48.275 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.275 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.276 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:10:48.276 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572416+0000", 2023-10-09T06:10:48.276 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.9879999999999999e-06 2023-10-09T06:10:48.276 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.276 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.276 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:10:48.276 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572433+0000", 2023-10-09T06:10:48.276 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7169000000000001e-05 2023-10-09T06:10:48.276 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.276 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.276 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:10:48.276 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572441+0000", 2023-10-09T06:10:48.277 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1089999999999998e-06 2023-10-09T06:10:48.277 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.277 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.277 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.277 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572523+0000", 2023-10-09T06:10:48.277 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1679000000000002e-05 2023-10-09T06:10:48.277 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.277 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.277 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:10:48.277 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572680+0000", 2023-10-09T06:10:48.277 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015694099999999999 2023-10-09T06:10:48.277 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.277 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.278 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:10:48.278 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572798+0000", 2023-10-09T06:10:48.278 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000118408 2023-10-09T06:10:48.278 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.278 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.278 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:10:48.278 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.584244+0000", 2023-10-09T06:10:48.278 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.01144565 2023-10-09T06:10:48.278 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.278 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.278 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.279 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.586841+0000", 2023-10-09T06:10:48.279 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002596887 2023-10-09T06:10:48.279 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.279 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.279 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:10:48.279 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.586852+0000", 2023-10-09T06:10:48.279 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1617000000000001e-05 2023-10-09T06:10:48.279 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.279 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.279 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.279 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.730544+0000", 2023-10-09T06:10:48.280 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1436917819999999 2023-10-09T06:10:48.280 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.280 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.280 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for missing object", 2023-10-09T06:10:48.280 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.730669+0000", 2023-10-09T06:10:48.280 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012532400000000001 2023-10-09T06:10:48.280 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.280 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.280 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.280 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.749944+0000", 2023-10-09T06:10:48.280 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.019274291999999998 2023-10-09T06:10:48.280 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.281 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.281 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:10:48.281 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.750058+0000", 2023-10-09T06:10:48.281 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000113964 2023-10-09T06:10:48.281 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.281 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:10:48.281 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.281 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.281 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.281 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:132912 9.5 9:aec1ab85:::benchmark_data_smithi064_511911_object31810:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e455)", 2023-10-09T06:10:48.282 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:26.572458+0000", 2023-10-09T06:10:48.282 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 81.623912645000004, 2023-10-09T06:10:48.282 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.180201992, 2023-10-09T06:10:48.282 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:10:48.282 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:10:48.282 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:10:48.282 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:10:48.282 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:48.282 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 132912 2023-10-09T06:10:48.282 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.282 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:10:48.282 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.283 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:10:48.283 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572458+0000", 2023-10-09T06:10:48.283 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.283 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.283 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.283 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:10:48.283 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572458+0000", 2023-10-09T06:10:48.283 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.283 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.283 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.283 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:10:48.283 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572457+0000", 2023-10-09T06:10:48.284 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:10:48.284 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.284 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.284 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:10:48.284 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572462+0000", 2023-10-09T06:10:48.284 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4660000000000001e-06 2023-10-09T06:10:48.284 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.284 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.284 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:10:48.284 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572473+0000", 2023-10-09T06:10:48.284 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1507e-05 2023-10-09T06:10:48.285 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.285 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.285 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:10:48.285 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572479+0000", 2023-10-09T06:10:48.285 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.361e-06 2023-10-09T06:10:48.285 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.285 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.285 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.285 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.576291+0000", 2023-10-09T06:10:48.285 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0038126309999999999 2023-10-09T06:10:48.285 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.285 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.286 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:10:48.286 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.576420+0000", 2023-10-09T06:10:48.286 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012867799999999999 2023-10-09T06:10:48.286 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.286 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.286 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:10:48.286 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.576543+0000", 2023-10-09T06:10:48.286 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000122886 2023-10-09T06:10:48.286 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.286 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.286 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:10:48.286 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.584250+0000", 2023-10-09T06:10:48.287 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0077072110000000003 2023-10-09T06:10:48.287 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.287 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.287 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.287 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.586885+0000", 2023-10-09T06:10:48.287 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0026347440000000001 2023-10-09T06:10:48.287 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.287 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.287 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:10:48.287 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.586892+0000", 2023-10-09T06:10:48.287 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0829999999999998e-06 2023-10-09T06:10:48.287 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.288 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.288 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.288 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.730711+0000", 2023-10-09T06:10:48.288 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.143819385 2023-10-09T06:10:48.288 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.288 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.288 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for missing object", 2023-10-09T06:10:48.288 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.730800+0000", 2023-10-09T06:10:48.288 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8787000000000002e-05 2023-10-09T06:10:48.288 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.288 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.289 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.289 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.752573+0000", 2023-10-09T06:10:48.289 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.02177246 2023-10-09T06:10:48.289 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.289 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.289 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:10:48.289 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.752660+0000", 2023-10-09T06:10:48.289 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.7353000000000002e-05 2023-10-09T06:10:48.289 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.289 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:10:48.289 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.290 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.290 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.290 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:132915 9.5 9:abaae183:::benchmark_data_smithi064_511911_object31813:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e455)", 2023-10-09T06:10:48.290 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:26.574849+0000", 2023-10-09T06:10:48.290 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 81.621521293000001, 2023-10-09T06:10:48.290 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1802385870000001, 2023-10-09T06:10:48.290 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:10:48.290 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:10:48.290 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:10:48.290 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:10:48.290 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:48.290 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 132915 2023-10-09T06:10:48.291 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.291 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:10:48.291 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.291 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:10:48.291 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.574849+0000", 2023-10-09T06:10:48.291 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.291 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.291 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.291 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:10:48.291 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.574849+0000", 2023-10-09T06:10:48.291 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.291 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.292 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.292 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:10:48.292 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.574848+0000", 2023-10-09T06:10:48.292 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:10:48.292 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.292 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.292 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:10:48.292 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.574855+0000", 2023-10-09T06:10:48.292 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2520000000000002e-06 2023-10-09T06:10:48.292 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.292 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.292 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:10:48.293 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.574872+0000", 2023-10-09T06:10:48.293 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6239000000000002e-05 2023-10-09T06:10:48.293 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.293 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.293 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:10:48.293 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.574878+0000", 2023-10-09T06:10:48.293 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8600000000000004e-06 2023-10-09T06:10:48.293 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.293 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.293 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.293 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.580205+0000", 2023-10-09T06:10:48.293 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0053268100000000004 2023-10-09T06:10:48.294 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.294 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.294 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:10:48.294 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.580339+0000", 2023-10-09T06:10:48.294 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000134074 2023-10-09T06:10:48.294 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.294 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.294 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:10:48.294 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.580463+0000", 2023-10-09T06:10:48.294 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012364 2023-10-09T06:10:48.294 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.295 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.295 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.295 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.586919+0000", 2023-10-09T06:10:48.295 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0064555890000000003 2023-10-09T06:10:48.295 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.295 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.295 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:10:48.295 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.586925+0000", 2023-10-09T06:10:48.295 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.81e-06 2023-10-09T06:10:48.295 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.295 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.295 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.295 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.730833+0000", 2023-10-09T06:10:48.296 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.143908047 2023-10-09T06:10:48.296 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.296 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.296 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for missing object", 2023-10-09T06:10:48.296 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.730909+0000", 2023-10-09T06:10:48.296 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5944000000000003e-05 2023-10-09T06:10:48.296 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.296 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.296 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.296 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.755019+0000", 2023-10-09T06:10:48.296 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.024109756 2023-10-09T06:10:48.296 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.297 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.297 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:10:48.297 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.755088+0000", 2023-10-09T06:10:48.297 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8837000000000005e-05 2023-10-09T06:10:48.297 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.297 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:10:48.297 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.297 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.297 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.297 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:132921 9.5 9:a6236301:::benchmark_data_smithi064_511911_object31819:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e456)", 2023-10-09T06:10:48.297 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:26.641383+0000", 2023-10-09T06:10:48.297 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 81.554987725999993, 2023-10-09T06:10:48.298 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1059502830000001, 2023-10-09T06:10:48.298 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:10:48.298 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:48.298 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:10:48.298 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:10:48.298 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:48.298 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 132921 2023-10-09T06:10:48.298 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.298 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:10:48.298 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.298 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:10:48.299 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.641383+0000", 2023-10-09T06:10:48.299 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.299 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.299 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.299 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:10:48.299 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.641383+0000", 2023-10-09T06:10:48.299 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.299 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.299 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.299 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:10:48.299 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.641382+0000", 2023-10-09T06:10:48.299 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:10:48.300 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.300 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.300 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:10:48.300 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.641387+0000", 2023-10-09T06:10:48.300 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5899999999999998e-06 2023-10-09T06:10:48.300 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.300 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.300 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:10:48.300 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.641400+0000", 2023-10-09T06:10:48.300 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2187999999999999e-05 2023-10-09T06:10:48.300 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.301 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.301 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:10:48.301 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.641405+0000", 2023-10-09T06:10:48.301 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3419999999999998e-06 2023-10-09T06:10:48.301 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.301 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.301 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.301 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.641457+0000", 2023-10-09T06:10:48.301 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2496e-05 2023-10-09T06:10:48.301 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.301 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.301 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:10:48.302 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.641468+0000", 2023-10-09T06:10:48.302 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0890000000000001e-05 2023-10-09T06:10:48.302 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.302 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.302 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.302 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.730943+0000", 2023-10-09T06:10:48.302 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0894744140000001 2023-10-09T06:10:48.302 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.302 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.302 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:10:48.302 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.731044+0000", 2023-10-09T06:10:48.302 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010098999999999999 2023-10-09T06:10:48.303 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.303 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.303 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1,7", 2023-10-09T06:10:48.303 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.731138+0000", 2023-10-09T06:10:48.303 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.4436000000000002e-05 2023-10-09T06:10:48.303 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.303 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.303 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:48.303 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.735600+0000", 2023-10-09T06:10:48.303 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0044618449999999999 2023-10-09T06:10:48.303 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.303 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.304 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:48.304 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.735653+0000", 2023-10-09T06:10:48.304 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3016999999999997e-05 2023-10-09T06:10:48.304 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.304 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.304 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:10:48.304 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.735663+0000", 2023-10-09T06:10:48.304 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0472000000000001e-05 2023-10-09T06:10:48.304 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.304 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.304 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:48.304 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.745120+0000", 2023-10-09T06:10:48.305 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0094562810000000004 2023-10-09T06:10:48.305 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.305 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.305 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:10:48.305 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.745189+0000", 2023-10-09T06:10:48.305 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8912999999999996e-05 2023-10-09T06:10:48.305 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.305 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.305 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:10:48.305 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.747333+0000", 2023-10-09T06:10:48.305 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0021445589999999999 2023-10-09T06:10:48.305 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.306 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:10:48.306 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.306 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.306 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.306 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:134661 9.5 9:a335a6bc:::benchmark_data_smithi064_511911_object33559:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e458)", 2023-10-09T06:10:48.306 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:31.499352+0000", 2023-10-09T06:10:48.306 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 76.697018322999995, 2023-10-09T06:10:48.306 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.250994569, 2023-10-09T06:10:48.306 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:10:48.306 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:48.306 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:10:48.306 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:10:48.307 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:48.307 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 134661 2023-10-09T06:10:48.307 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.307 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:10:48.307 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.307 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:10:48.307 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.499352+0000", 2023-10-09T06:10:48.307 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.307 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.307 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.307 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:10:48.307 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.499352+0000", 2023-10-09T06:10:48.308 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.308 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.308 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.308 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:10:48.308 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.499351+0000", 2023-10-09T06:10:48.308 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:10:48.308 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.308 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.308 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:10:48.308 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.499358+0000", 2023-10-09T06:10:48.308 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0419999999999997e-06 2023-10-09T06:10:48.308 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.309 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.309 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:10:48.309 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.499370+0000", 2023-10-09T06:10:48.309 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1891e-05 2023-10-09T06:10:48.309 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.309 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.309 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:10:48.309 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.499376+0000", 2023-10-09T06:10:48.309 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6350000000000003e-06 2023-10-09T06:10:48.309 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.309 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.309 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.310 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.500390+0000", 2023-10-09T06:10:48.310 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0010140360000000001 2023-10-09T06:10:48.310 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.310 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.310 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:10:48.310 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.500496+0000", 2023-10-09T06:10:48.310 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000106449 2023-10-09T06:10:48.310 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.310 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.310 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1,7", 2023-10-09T06:10:48.310 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.500584+0000", 2023-10-09T06:10:48.310 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.81e-05 2023-10-09T06:10:48.311 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.311 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.311 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:10:48.311 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.512762+0000", 2023-10-09T06:10:48.311 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012178193 2023-10-09T06:10:48.311 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.311 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.311 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:48.311 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.512800+0000", 2023-10-09T06:10:48.311 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7969999999999997e-05 2023-10-09T06:10:48.311 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.311 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.312 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:48.312 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.512827+0000", 2023-10-09T06:10:48.312 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7047e-05 2023-10-09T06:10:48.312 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.312 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.312 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:48.312 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.748195+0000", 2023-10-09T06:10:48.312 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.23536779599999999 2023-10-09T06:10:48.312 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.312 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.312 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:10:48.312 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.748249+0000", 2023-10-09T06:10:48.312 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3430999999999997e-05 2023-10-09T06:10:48.313 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.313 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.313 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:10:48.313 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.750347+0000", 2023-10-09T06:10:48.313 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002098262 2023-10-09T06:10:48.313 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.313 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:10:48.313 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.313 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.313 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.313 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:153149 9.5 9:a6ab5184:::benchmark_data_smithi064_511911_object52047:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e463)", 2023-10-09T06:10:48.313 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:10:03.425900+0000", 2023-10-09T06:10:48.314 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 44.770470433, 2023-10-09T06:10:48.314 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.240294219, 2023-10-09T06:10:48.314 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:10:48.314 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:48.314 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:10:48.314 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:10:48.314 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:48.314 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 153149 2023-10-09T06:10:48.314 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.314 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:10:48.314 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.314 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:10:48.315 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.425900+0000", 2023-10-09T06:10:48.315 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.315 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.315 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.315 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:10:48.315 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.425900+0000", 2023-10-09T06:10:48.315 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.315 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.315 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.315 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:10:48.315 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.425900+0000", 2023-10-09T06:10:48.315 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:10:48.316 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.316 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.316 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:10:48.316 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.425904+0000", 2023-10-09T06:10:48.316 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0999999999999997e-06 2023-10-09T06:10:48.316 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.316 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.316 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:10:48.316 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.425916+0000", 2023-10-09T06:10:48.316 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2787e-05 2023-10-09T06:10:48.316 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.316 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.317 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:10:48.317 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.425922+0000", 2023-10-09T06:10:48.317 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5300000000000004e-06 2023-10-09T06:10:48.317 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.317 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.317 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.317 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.432133+0000", 2023-10-09T06:10:48.317 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0062113849999999998 2023-10-09T06:10:48.317 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.317 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.317 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:10:48.317 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.432299+0000", 2023-10-09T06:10:48.318 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000166014 2023-10-09T06:10:48.318 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.318 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.318 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1,7", 2023-10-09T06:10:48.318 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.432478+0000", 2023-10-09T06:10:48.318 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00017833899999999999 2023-10-09T06:10:48.318 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.318 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.318 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:10:48.318 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.436209+0000", 2023-10-09T06:10:48.318 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.003731079 2023-10-09T06:10:48.318 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.319 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.319 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:48.319 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.439000+0000", 2023-10-09T06:10:48.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0027908239999999999 2023-10-09T06:10:48.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.319 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.319 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:48.319 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.451004+0000", 2023-10-09T06:10:48.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012004795 2023-10-09T06:10:48.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.319 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.319 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:48.320 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.665717+0000", 2023-10-09T06:10:48.320 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.21471283899999999 2023-10-09T06:10:48.320 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.320 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.320 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:10:48.320 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.665754+0000", 2023-10-09T06:10:48.320 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7042999999999998e-05 2023-10-09T06:10:48.320 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.320 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.320 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:10:48.320 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.666194+0000", 2023-10-09T06:10:48.320 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000440133 2023-10-09T06:10:48.321 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.321 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:10:48.321 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.321 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.321 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.321 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:178997 9.5 9:a657cd64:::benchmark_data_smithi064_511911_object77895:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T06:10:48.321 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:10:44.053726+0000", 2023-10-09T06:10:48.321 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 4.142644948, 2023-10-09T06:10:48.321 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.27684753699999998, 2023-10-09T06:10:48.321 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:10:48.321 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:48.321 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:10:48.322 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:10:48.322 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:48.322 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 178997 2023-10-09T06:10:48.322 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.322 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:10:48.322 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.322 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:10:48.322 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.053726+0000", 2023-10-09T06:10:48.322 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.322 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.322 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.322 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:10:48.323 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.053726+0000", 2023-10-09T06:10:48.323 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.323 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.323 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.323 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:10:48.323 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.053724+0000", 2023-10-09T06:10:48.323 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:10:48.323 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.323 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.323 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:10:48.323 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.053734+0000", 2023-10-09T06:10:48.324 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2399999999999996e-06 2023-10-09T06:10:48.324 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.324 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.324 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:10:48.324 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.053745+0000", 2023-10-09T06:10:48.324 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1708e-05 2023-10-09T06:10:48.324 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.324 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.324 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:10:48.324 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.053751+0000", 2023-10-09T06:10:48.324 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9479999999999996e-06 2023-10-09T06:10:48.324 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.325 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.325 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.325 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.053791+0000", 2023-10-09T06:10:48.325 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9721000000000001e-05 2023-10-09T06:10:48.325 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.325 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.325 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:10:48.325 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.053933+0000", 2023-10-09T06:10:48.325 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014163500000000001 2023-10-09T06:10:48.325 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.325 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.325 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1,7", 2023-10-09T06:10:48.326 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.054086+0000", 2023-10-09T06:10:48.326 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015320700000000001 2023-10-09T06:10:48.326 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.326 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.326 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:10:48.326 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.058079+0000", 2023-10-09T06:10:48.326 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0039933429999999999 2023-10-09T06:10:48.326 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.326 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.326 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:48.326 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.061474+0000", 2023-10-09T06:10:48.327 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0033949919999999999 2023-10-09T06:10:48.327 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.327 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.327 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:48.327 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.067939+0000", 2023-10-09T06:10:48.327 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0064653269999999999 2023-10-09T06:10:48.327 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.327 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.327 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:48.327 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.330290+0000", 2023-10-09T06:10:48.327 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.26235035800000001 2023-10-09T06:10:48.327 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.328 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.328 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:10:48.328 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.330328+0000", 2023-10-09T06:10:48.328 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8448999999999997e-05 2023-10-09T06:10:48.328 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.328 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.328 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:10:48.328 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.330573+0000", 2023-10-09T06:10:48.328 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00024499500000000002 2023-10-09T06:10:48.328 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.328 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:10:48.328 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.329 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.329 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.329 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:179005 9.5 9:a7a3d65d:::benchmark_data_smithi064_511911_object77903:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T06:10:48.329 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:10:44.064308+0000", 2023-10-09T06:10:48.329 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 4.1320630769999998, 2023-10-09T06:10:48.329 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.26648006499999999, 2023-10-09T06:10:48.329 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:10:48.329 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:10:48.329 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:10:48.329 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:10:48.329 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:10:48.329 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 179005 2023-10-09T06:10:48.330 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.330 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:10:48.330 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.330 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:10:48.330 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.064308+0000", 2023-10-09T06:10:48.330 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.330 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.330 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.330 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:10:48.330 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.064308+0000", 2023-10-09T06:10:48.330 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:10:48.330 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.331 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.331 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:10:48.331 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.064306+0000", 2023-10-09T06:10:48.331 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:10:48.331 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.331 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.331 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:10:48.331 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.064312+0000", 2023-10-09T06:10:48.331 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.665e-06 2023-10-09T06:10:48.331 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.331 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.331 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:10:48.332 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.064325+0000", 2023-10-09T06:10:48.332 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3108000000000001e-05 2023-10-09T06:10:48.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.332 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.332 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:10:48.332 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.064334+0000", 2023-10-09T06:10:48.332 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2830000000000007e-06 2023-10-09T06:10:48.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.332 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.332 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:10:48.332 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.064382+0000", 2023-10-09T06:10:48.332 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7667000000000003e-05 2023-10-09T06:10:48.333 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.333 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.333 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:10:48.333 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.064488+0000", 2023-10-09T06:10:48.333 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010548999999999999 2023-10-09T06:10:48.333 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.333 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.333 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1,7", 2023-10-09T06:10:48.333 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.064585+0000", 2023-10-09T06:10:48.333 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7508000000000002e-05 2023-10-09T06:10:48.333 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.333 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.334 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:10:48.334 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.067958+0000", 2023-10-09T06:10:48.334 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.003372608 2023-10-09T06:10:48.334 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.334 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.334 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:48.334 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.068016+0000", 2023-10-09T06:10:48.334 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8072000000000002e-05 2023-10-09T06:10:48.334 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.334 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.334 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:48.334 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.068603+0000", 2023-10-09T06:10:48.335 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00058710100000000003 2023-10-09T06:10:48.335 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.335 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.335 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:10:48.335 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.330621+0000", 2023-10-09T06:10:48.335 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.26201773499999997 2023-10-09T06:10:48.335 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.335 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.335 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:10:48.335 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.330665+0000", 2023-10-09T06:10:48.335 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4579000000000001e-05 2023-10-09T06:10:48.336 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:10:48.336 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:10:48.336 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:10:48.336 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.330788+0000", 2023-10-09T06:10:48.336 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012247500000000001 2023-10-09T06:10:48.336 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.336 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:10:48.336 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.336 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:10:48.336 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:10:48.336 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:10:48.422 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:10:48.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:48.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:10:48.530 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:10:48.576 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:48.601 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:48.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:48 smithi156 ceph-mon[195101]: pgmap v1201: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 1.4 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 8091/67170 objects degraded (12.046%); 10114/67170 objects misplaced (15.057%); 3.1 MiB/s, 133 objects/s recovering 2023-10-09T06:10:48.922 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:48 smithi064 ceph-mon[212750]: pgmap v1201: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 1.4 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 8091/67170 objects degraded (12.046%); 10114/67170 objects misplaced (15.057%); 3.1 MiB/s, 133 objects/s recovering 2023-10-09T06:10:48.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:48 smithi064 ceph-mon[220044]: pgmap v1201: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 1.4 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 8091/67170 objects degraded (12.046%); 10114/67170 objects misplaced (15.057%); 3.1 MiB/s, 133 objects/s recovering 2023-10-09T06:10:49.327 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:10:49.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:49.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:10:49.432 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:10:49.473 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:10:49.496 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:49.699 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:10:49.714 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:49 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1626433371' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:10:49.714 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:49 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:10:49.714 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:49 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2439955021' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-09T06:10:49.714 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:49 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-09T06:10:49.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:49.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:10:49.739 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:10:49.747 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.6 primary-affinity to 0 (802) 2023-10-09T06:10:49.819 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:10:49.864 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:49.929 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:10:49.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:49.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:10:49.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:49 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1626433371' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:10:49.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:49 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:10:49.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:49 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2439955021' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-09T06:10:49.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:49 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-09T06:10:50.036 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:10:50.075 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:50.113 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:10:50.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:49 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1626433371' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:10:50.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:49 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:10:50.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:49 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2439955021' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-09T06:10:50.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:49 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-09T06:10:50.185 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:10:50.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:50.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:10:50.290 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:10:50.320 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:50.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:50 smithi064 ceph-mon[212750]: pgmap v1202: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 1.4 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 7839/66576 objects degraded (11.775%); 10114/66576 objects misplaced (15.192%); 3.0 MiB/s, 118 objects/s recovering 2023-10-09T06:10:50.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:50 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:10:50.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:50 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:10:50.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:50 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2023-10-09T06:10:50.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:50 smithi064 ceph-mon[212750]: osdmap e475: 8 total, 8 up, 8 in 2023-10-09T06:10:50.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:50 smithi064 ceph-mon[212750]: failed to encode map e475 with expected crc 2023-10-09T06:10:50.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:50 smithi064 ceph-mon[212750]: failed to encode map e475 with expected crc 2023-10-09T06:10:50.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:50 smithi064 ceph-mon[212750]: failed to encode map e475 with expected crc 2023-10-09T06:10:50.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:50 smithi064 ceph-mon[212750]: failed to encode map e475 with expected crc 2023-10-09T06:10:50.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:50 smithi064 ceph-mon[212750]: failed to encode map e475 with expected crc 2023-10-09T06:10:50.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:50 smithi064 ceph-mon[220044]: pgmap v1202: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 1.4 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 7839/66576 objects degraded (11.775%); 10114/66576 objects misplaced (15.192%); 3.0 MiB/s, 118 objects/s recovering 2023-10-09T06:10:50.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:50 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:10:50.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:50 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:10:50.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:50 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2023-10-09T06:10:50.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:50 smithi064 ceph-mon[220044]: osdmap e475: 8 total, 8 up, 8 in 2023-10-09T06:10:50.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:50 smithi064 ceph-mon[220044]: failed to encode map e475 with expected crc 2023-10-09T06:10:50.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:50 smithi064 ceph-mon[220044]: failed to encode map e475 with expected crc 2023-10-09T06:10:50.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:50 smithi064 ceph-mon[220044]: failed to encode map e475 with expected crc 2023-10-09T06:10:50.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:50 smithi064 ceph-mon[220044]: failed to encode map e475 with expected crc 2023-10-09T06:10:50.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:50 smithi064 ceph-mon[220044]: failed to encode map e475 with expected crc 2023-10-09T06:10:51.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:50 smithi156 ceph-mon[195101]: pgmap v1202: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 109 active+clean; 1.4 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 7839/66576 objects degraded (11.775%); 10114/66576 objects misplaced (15.192%); 3.0 MiB/s, 118 objects/s recovering 2023-10-09T06:10:51.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:50 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:10:51.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:50 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:10:51.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:50 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2023-10-09T06:10:51.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:50 smithi156 ceph-mon[195101]: osdmap e475: 8 total, 8 up, 8 in 2023-10-09T06:10:51.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:50 smithi156 ceph-mon[195101]: failed to encode map e475 with expected crc 2023-10-09T06:10:51.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:50 smithi156 ceph-mon[195101]: failed to encode map e475 with expected crc 2023-10-09T06:10:51.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:50 smithi156 ceph-mon[195101]: failed to encode map e475 with expected crc 2023-10-09T06:10:51.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:50 smithi156 ceph-mon[195101]: failed to encode map e475 with expected crc 2023-10-09T06:10:51.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:50 smithi156 ceph-mon[195101]: failed to encode map e475 with expected crc 2023-10-09T06:10:52.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:51 smithi156 ceph-mon[195101]: osdmap e476: 8 total, 8 up, 8 in 2023-10-09T06:10:52.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:51 smithi156 ceph-mon[195101]: failed to encode map e476 with expected crc 2023-10-09T06:10:52.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:51 smithi156 ceph-mon[195101]: failed to encode map e476 with expected crc 2023-10-09T06:10:52.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:51 smithi156 ceph-mon[195101]: failed to encode map e476 with expected crc 2023-10-09T06:10:52.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:51 smithi156 ceph-mon[195101]: failed to encode map e476 with expected crc 2023-10-09T06:10:52.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:51 smithi156 ceph-mon[195101]: failed to encode map e476 with expected crc 2023-10-09T06:10:52.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:51 smithi156 ceph-mon[195101]: failed to encode map e476 with expected crc 2023-10-09T06:10:52.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:51 smithi156 ceph-mon[195101]: failed to encode map e476 with expected crc 2023-10-09T06:10:52.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:51 smithi156 ceph-mon[195101]: failed to encode map e476 with expected crc 2023-10-09T06:10:52.181 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:51 smithi064 ceph-mon[212750]: osdmap e476: 8 total, 8 up, 8 in 2023-10-09T06:10:52.182 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:51 smithi064 ceph-mon[212750]: failed to encode map e476 with expected crc 2023-10-09T06:10:52.182 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:51 smithi064 ceph-mon[212750]: failed to encode map e476 with expected crc 2023-10-09T06:10:52.182 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:51 smithi064 ceph-mon[212750]: failed to encode map e476 with expected crc 2023-10-09T06:10:52.182 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:51 smithi064 ceph-mon[212750]: failed to encode map e476 with expected crc 2023-10-09T06:10:52.182 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:51 smithi064 ceph-mon[212750]: failed to encode map e476 with expected crc 2023-10-09T06:10:52.182 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:51 smithi064 ceph-mon[212750]: failed to encode map e476 with expected crc 2023-10-09T06:10:52.182 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:51 smithi064 ceph-mon[212750]: failed to encode map e476 with expected crc 2023-10-09T06:10:52.182 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:51 smithi064 ceph-mon[212750]: failed to encode map e476 with expected crc 2023-10-09T06:10:52.183 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:51 smithi064 ceph-mon[220044]: osdmap e476: 8 total, 8 up, 8 in 2023-10-09T06:10:52.183 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:51 smithi064 ceph-mon[220044]: failed to encode map e476 with expected crc 2023-10-09T06:10:52.183 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:51 smithi064 ceph-mon[220044]: failed to encode map e476 with expected crc 2023-10-09T06:10:52.183 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:51 smithi064 ceph-mon[220044]: failed to encode map e476 with expected crc 2023-10-09T06:10:52.183 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:51 smithi064 ceph-mon[220044]: failed to encode map e476 with expected crc 2023-10-09T06:10:52.183 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:51 smithi064 ceph-mon[220044]: failed to encode map e476 with expected crc 2023-10-09T06:10:52.183 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:51 smithi064 ceph-mon[220044]: failed to encode map e476 with expected crc 2023-10-09T06:10:52.183 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:51 smithi064 ceph-mon[220044]: failed to encode map e476 with expected crc 2023-10-09T06:10:52.183 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:51 smithi064 ceph-mon[220044]: failed to encode map e476 with expected crc 2023-10-09T06:10:52.313 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:10:52.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:52.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:10:52.418 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:10:52.480 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:52.672 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:10:52.672 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:10:52.672 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 0 2023-10-09T06:10:52.672 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:10:52.680 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:10:52.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:52.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:10:52.788 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:10:52.885 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:53.051 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:10:53.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:53.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:10:53.068 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:52 smithi064 ceph-mon[212750]: pgmap v1205: 119 pgs: 1 activating, 6 peering, 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 102 active+clean; 1.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 15 KiB/s rd, 41 KiB/s wr, 614 op/s; 5769/62358 objects degraded (9.251%); 9541/62358 objects misplaced (15.300%); 4.0 MiB/s, 109 objects/s recovering 2023-10-09T06:10:53.068 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:52 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-10-09T06:10:53.068 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:52 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 5769/62358 objects degraded (9.251%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:53.069 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:52 smithi064 ceph-mon[220044]: pgmap v1205: 119 pgs: 1 activating, 6 peering, 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 102 active+clean; 1.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 15 KiB/s rd, 41 KiB/s wr, 614 op/s; 5769/62358 objects degraded (9.251%); 9541/62358 objects misplaced (15.300%); 4.0 MiB/s, 109 objects/s recovering 2023-10-09T06:10:53.069 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:52 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-10-09T06:10:53.069 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:52 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 5769/62358 objects degraded (9.251%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:53.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:52 smithi156 ceph-mon[195101]: pgmap v1205: 119 pgs: 1 activating, 6 peering, 3 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 102 active+clean; 1.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 15 KiB/s rd, 41 KiB/s wr, 614 op/s; 5769/62358 objects degraded (9.251%); 9541/62358 objects misplaced (15.300%); 4.0 MiB/s, 109 objects/s recovering 2023-10-09T06:10:53.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:52 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-10-09T06:10:53.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:52 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 5769/62358 objects degraded (9.251%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:53.169 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:10:53.196 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:53.198 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:10:53.219 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:10:53.241 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:10:53.263 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:10:53.443 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:10:53.443 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:10:53.444 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:10:53.444 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:10:53.444 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:10:53.444 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:10:53.444 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:10:53.444 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:10:53.444 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:10:53.444 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:10:53.444 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:10:53.444 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:10:53.778 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:53 smithi064 ceph-mon[212750]: pgmap v1206: 119 pgs: 1 active+remapped+backfilling, 1 activating, 6 peering, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 103 active+clean; 1.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 693 op/s; 5365/59172 objects degraded (9.067%); 8764/59172 objects misplaced (14.811%); 7.0 MiB/s, 200 objects/s recovering 2023-10-09T06:10:53.778 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:53 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3405092304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:10:53.837 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:10:53.872 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:10:53.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:53.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:10:53.981 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:10:54.060 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:53 smithi064 ceph-mon[220044]: pgmap v1206: 119 pgs: 1 active+remapped+backfilling, 1 activating, 6 peering, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 103 active+clean; 1.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 693 op/s; 5365/59172 objects degraded (9.067%); 8764/59172 objects misplaced (14.811%); 7.0 MiB/s, 200 objects/s recovering 2023-10-09T06:10:54.060 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:53 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3405092304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:10:54.064 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:54.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:53 smithi156 ceph-mon[195101]: pgmap v1206: 119 pgs: 1 active+remapped+backfilling, 1 activating, 6 peering, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 103 active+clean; 1.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 693 op/s; 5365/59172 objects degraded (9.067%); 8764/59172 objects misplaced (14.811%); 7.0 MiB/s, 200 objects/s recovering 2023-10-09T06:10:54.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:53 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3405092304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:10:54.209 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:10:54.609 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:10:54.988 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:10:55.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:55.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:10:55.102 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:10:55.187 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:55.318 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:10:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:10:55] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.43.0" 2023-10-09T06:10:55.339 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:10:55.345 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:55.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:55.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:10:55.460 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:10:55.500 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:56.177 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:10:56.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:56.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:10:56.283 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:10:56.349 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:56.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:56 smithi064 ceph-mon[220044]: pgmap v1207: 119 pgs: 1 active+remapped+backfilling, 1 activating, 4 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 105 active+clean; 1.2 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 386 op/s; 7272/58155 objects degraded (12.505%); 8764/58155 objects misplaced (15.070%); 7.4 MiB/s, 160 objects/s recovering 2023-10-09T06:10:56.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:56 smithi064 ceph-mon[212750]: pgmap v1207: 119 pgs: 1 active+remapped+backfilling, 1 activating, 4 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 105 active+clean; 1.2 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 386 op/s; 7272/58155 objects degraded (12.505%); 8764/58155 objects misplaced (15.070%); 7.4 MiB/s, 160 objects/s recovering 2023-10-09T06:10:56.598 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:56 smithi156 ceph-mon[195101]: pgmap v1207: 119 pgs: 1 active+remapped+backfilling, 1 activating, 4 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 105 active+clean; 1.2 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 386 op/s; 7272/58155 objects degraded (12.505%); 8764/58155 objects misplaced (15.070%); 7.4 MiB/s, 160 objects/s recovering 2023-10-09T06:10:56.939 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:10:56.939 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:10:57.129 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:10:57.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:57 smithi156 ceph-mon[195101]: osdmap e477: 8 total, 8 up, 8 in 2023-10-09T06:10:57.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:57 smithi156 ceph-mon[195101]: failed to encode map e477 with expected crc 2023-10-09T06:10:57.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:57 smithi156 ceph-mon[195101]: failed to encode map e477 with expected crc 2023-10-09T06:10:57.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:57 smithi156 ceph-mon[195101]: failed to encode map e477 with expected crc 2023-10-09T06:10:57.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:57 smithi156 ceph-mon[195101]: failed to encode map e477 with expected crc 2023-10-09T06:10:57.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:57 smithi156 ceph-mon[195101]: failed to encode map e477 with expected crc 2023-10-09T06:10:57.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:57 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:10:57.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:57 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:10:57.401 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1, 2] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 4, 0] 2023-10-09T06:10:57.401 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:10:57.401 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T06:10:57.401 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:10:57.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:57 smithi064 ceph-mon[220044]: osdmap e477: 8 total, 8 up, 8 in 2023-10-09T06:10:57.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:57 smithi064 ceph-mon[220044]: failed to encode map e477 with expected crc 2023-10-09T06:10:57.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:57 smithi064 ceph-mon[220044]: failed to encode map e477 with expected crc 2023-10-09T06:10:57.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:57 smithi064 ceph-mon[220044]: failed to encode map e477 with expected crc 2023-10-09T06:10:57.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:57 smithi064 ceph-mon[220044]: failed to encode map e477 with expected crc 2023-10-09T06:10:57.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:57 smithi064 ceph-mon[220044]: failed to encode map e477 with expected crc 2023-10-09T06:10:57.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:57 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:10:57.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:57 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:10:57.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:57 smithi064 ceph-mon[212750]: osdmap e477: 8 total, 8 up, 8 in 2023-10-09T06:10:57.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:57 smithi064 ceph-mon[212750]: failed to encode map e477 with expected crc 2023-10-09T06:10:57.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:57 smithi064 ceph-mon[212750]: failed to encode map e477 with expected crc 2023-10-09T06:10:57.442 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:57 smithi064 ceph-mon[212750]: failed to encode map e477 with expected crc 2023-10-09T06:10:57.442 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:57 smithi064 ceph-mon[212750]: failed to encode map e477 with expected crc 2023-10-09T06:10:57.442 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:57 smithi064 ceph-mon[212750]: failed to encode map e477 with expected crc 2023-10-09T06:10:57.442 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:57 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:10:57.442 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:57 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:10:57.492 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:10:57.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:57.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:10:57.601 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:10:57.637 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:57.711 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:10:57.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:57.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:10:57.770 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:10:57.816 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:10:57.920 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:58.059 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:10:58.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:58.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:10:58.166 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:10:58.201 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:58 smithi064 ceph-mon[212750]: pgmap v1209: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 110 active+clean; 1.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 15 KiB/s rd, 40 KiB/s wr, 770 op/s; 6942/49869 objects degraded (13.920%); 7621/49869 objects misplaced (15.282%); 12 MiB/s, 235 objects/s recovering 2023-10-09T06:10:58.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:58 smithi064 ceph-mon[212750]: failed to encode map e478 with expected crc 2023-10-09T06:10:58.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:58 smithi064 ceph-mon[212750]: failed to encode map e478 with expected crc 2023-10-09T06:10:58.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:58 smithi064 ceph-mon[212750]: osdmap e478: 8 total, 8 up, 8 in 2023-10-09T06:10:58.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:58 smithi064 ceph-mon[212750]: failed to encode map e478 with expected crc 2023-10-09T06:10:58.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:58 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-10-09T06:10:58.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:58 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 6942/49869 objects degraded (13.920%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:58.202 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:58 smithi064 ceph-mon[220044]: pgmap v1209: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 110 active+clean; 1.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 15 KiB/s rd, 40 KiB/s wr, 770 op/s; 6942/49869 objects degraded (13.920%); 7621/49869 objects misplaced (15.282%); 12 MiB/s, 235 objects/s recovering 2023-10-09T06:10:58.202 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:58 smithi064 ceph-mon[220044]: failed to encode map e478 with expected crc 2023-10-09T06:10:58.202 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:58 smithi064 ceph-mon[220044]: failed to encode map e478 with expected crc 2023-10-09T06:10:58.203 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:58 smithi064 ceph-mon[220044]: osdmap e478: 8 total, 8 up, 8 in 2023-10-09T06:10:58.203 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:58 smithi064 ceph-mon[220044]: failed to encode map e478 with expected crc 2023-10-09T06:10:58.203 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:58 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-10-09T06:10:58.203 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:58 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 6942/49869 objects degraded (13.920%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:58.207 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:58.262 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:10:58.289 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:10:58.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:58.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:10:58.393 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:10:58.470 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:58.472 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:58.493 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:58 smithi156 ceph-mon[195101]: pgmap v1209: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 110 active+clean; 1.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 15 KiB/s rd, 40 KiB/s wr, 770 op/s; 6942/49869 objects degraded (13.920%); 7621/49869 objects misplaced (15.282%); 12 MiB/s, 235 objects/s recovering 2023-10-09T06:10:58.493 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:58 smithi156 ceph-mon[195101]: failed to encode map e478 with expected crc 2023-10-09T06:10:58.493 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:58 smithi156 ceph-mon[195101]: failed to encode map e478 with expected crc 2023-10-09T06:10:58.493 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:58 smithi156 ceph-mon[195101]: osdmap e478: 8 total, 8 up, 8 in 2023-10-09T06:10:58.493 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:58 smithi156 ceph-mon[195101]: failed to encode map e478 with expected crc 2023-10-09T06:10:58.493 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:58 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-10-09T06:10:58.493 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:58 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 6942/49869 objects degraded (13.920%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:58.591 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:10:58.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:58.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:10:58.697 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:10:58.711 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:10:58.746 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:58.860 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:10:58.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:58.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:10:58.967 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:10:58.995 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:59.037 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:10:59.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:10:59.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:10:59.140 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:10:59.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:59.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:10:59 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/139107541' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:10:59.208 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:59.208 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:10:59 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/139107541' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:10:59.235 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:10:59.252 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:10:59.376 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:59.376 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:10:59 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/139107541' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:11:00.083 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:11:00.084 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:11:00.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:00 smithi156 ceph-mon[195101]: pgmap v1211: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 110 active+clean; 1.0 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 6942/49335 objects degraded (14.071%); 7621/49335 objects misplaced (15.447%); 11 MiB/s, 212 objects/s recovering 2023-10-09T06:11:00.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:00 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:11:00.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:00 smithi156 ceph-mon[195101]: failed to encode map e479 with expected crc 2023-10-09T06:11:00.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:00 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/139107541' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:11:00.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:00 smithi156 ceph-mon[195101]: osdmap e479: 8 total, 8 up, 8 in 2023-10-09T06:11:00.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:00 smithi156 ceph-mon[195101]: failed to encode map e479 with expected crc 2023-10-09T06:11:00.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:00 smithi156 ceph-mon[195101]: failed to encode map e479 with expected crc 2023-10-09T06:11:00.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:00 smithi156 ceph-mon[195101]: failed to encode map e479 with expected crc 2023-10-09T06:11:00.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:00 smithi156 ceph-mon[195101]: 1.d scrub starts 2023-10-09T06:11:00.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:00 smithi156 ceph-mon[195101]: 1.d scrub ok 2023-10-09T06:11:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:00 smithi064 ceph-mon[212750]: pgmap v1211: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 110 active+clean; 1.0 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 6942/49335 objects degraded (14.071%); 7621/49335 objects misplaced (15.447%); 11 MiB/s, 212 objects/s recovering 2023-10-09T06:11:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:00 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:11:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:00 smithi064 ceph-mon[212750]: failed to encode map e479 with expected crc 2023-10-09T06:11:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:00 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/139107541' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:11:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:00 smithi064 ceph-mon[212750]: osdmap e479: 8 total, 8 up, 8 in 2023-10-09T06:11:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:00 smithi064 ceph-mon[212750]: failed to encode map e479 with expected crc 2023-10-09T06:11:00.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:00 smithi064 ceph-mon[212750]: failed to encode map e479 with expected crc 2023-10-09T06:11:00.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:00 smithi064 ceph-mon[212750]: failed to encode map e479 with expected crc 2023-10-09T06:11:00.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:00 smithi064 ceph-mon[212750]: 1.d scrub starts 2023-10-09T06:11:00.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:00 smithi064 ceph-mon[212750]: 1.d scrub ok 2023-10-09T06:11:00.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:00 smithi064 ceph-mon[220044]: pgmap v1211: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 110 active+clean; 1.0 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 6942/49335 objects degraded (14.071%); 7621/49335 objects misplaced (15.447%); 11 MiB/s, 212 objects/s recovering 2023-10-09T06:11:00.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:00 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:11:00.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:00 smithi064 ceph-mon[220044]: failed to encode map e479 with expected crc 2023-10-09T06:11:00.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:00 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/139107541' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:11:00.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:00 smithi064 ceph-mon[220044]: osdmap e479: 8 total, 8 up, 8 in 2023-10-09T06:11:00.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:00 smithi064 ceph-mon[220044]: failed to encode map e479 with expected crc 2023-10-09T06:11:00.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:00 smithi064 ceph-mon[220044]: failed to encode map e479 with expected crc 2023-10-09T06:11:00.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:00 smithi064 ceph-mon[220044]: failed to encode map e479 with expected crc 2023-10-09T06:11:00.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:00 smithi064 ceph-mon[220044]: 1.d scrub starts 2023-10-09T06:11:00.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:00 smithi064 ceph-mon[220044]: 1.d scrub ok 2023-10-09T06:11:00.965 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:11:01.091 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:11:01.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:01.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:11:01.201 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:01 smithi156 ceph-mon[195101]: 1.2 scrub starts 2023-10-09T06:11:01.201 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:01 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:11:01.201 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:01 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:11:01.204 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:11:01.241 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:01.311 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:11:01.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:01.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:11:01.421 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:11:01.465 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:01.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:01 smithi064 ceph-mon[220044]: 1.2 scrub starts 2023-10-09T06:11:01.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:01 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:11:01.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:01 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:11:01.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:01 smithi064 ceph-mon[212750]: 1.2 scrub starts 2023-10-09T06:11:01.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:01 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:11:01.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:01 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:11:01.497 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:11:01.544 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:11:01.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:01.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:11:01.655 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:11:01.714 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:01.800 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:11:01.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:01.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:11:01.904 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:11:01.993 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:02.203 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:02 smithi064 ceph-mon[220044]: pgmap v1213: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 110 active+clean; 939 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 6674/45123 objects degraded (14.791%); 7054/45123 objects misplaced (15.633%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T06:11:02.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:02 smithi064 ceph-mon[212750]: pgmap v1213: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 110 active+clean; 939 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 6674/45123 objects degraded (14.791%); 7054/45123 objects misplaced (15.633%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T06:11:02.253 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:11:02.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:02.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:11:02.358 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:11:02.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:02 smithi156 ceph-mon[195101]: pgmap v1213: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 110 active+clean; 939 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 6674/45123 objects degraded (14.791%); 7054/45123 objects misplaced (15.633%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T06:11:02.398 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:03.139 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:11:03.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:03.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:11:03.251 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:11:03.357 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:03.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:03 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:03.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:03 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:11:03.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:03 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:11:03.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:03 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 6674/45123 objects degraded (14.791%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:03.399 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:03 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:03.399 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:03 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:11:03.399 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:03 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:11:03.399 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:03 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 6674/45123 objects degraded (14.791%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:03.399 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:03 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:03.399 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:03 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:11:03.399 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:03 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:11:03.399 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:03 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 6674/45123 objects degraded (14.791%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:03.459 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:11:03.459 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:11:03.465 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:11:03.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:03.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:11:03.578 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:11:03.637 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:03.715 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:11:03.724 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:11:03.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:03.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:11:03.762 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:11:03.803 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:11:03.828 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:11:03.830 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:11:03.850 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:11:03.854 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:03.896 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:11:03.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:03.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:11:03.986 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:11:04.000 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:11:04.034 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:04.040 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:11:04.103 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:11:04.111 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:11:04.111 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:11:04.111 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:11:04.111 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:11:04.111 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:11:04.111 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:11:04.111 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:11:04.111 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:11:04.248 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:11:04.329 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:11:04.330 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:11:04.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:04.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:11:04.413 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:11:04.428 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T06:11:04.437 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:11:04.488 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:04.579 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:11:04.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:04.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:11:04.685 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:11:04.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:04 smithi064 ceph-mon[212750]: pgmap v1214: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 865 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 6593/41583 objects degraded (15.855%); 7054/41583 objects misplaced (16.964%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T06:11:04.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:04 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1222629962' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:11:04.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:04 smithi064 ceph-mon[220044]: pgmap v1214: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 865 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 6593/41583 objects degraded (15.855%); 7054/41583 objects misplaced (16.964%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T06:11:04.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:04 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1222629962' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:11:04.744 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:04.844 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:11:04.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:04 smithi156 ceph-mon[195101]: pgmap v1214: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 865 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 6593/41583 objects degraded (15.855%); 7054/41583 objects misplaced (16.964%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T06:11:04.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:04 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1222629962' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:11:04.884 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:11:04.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:04.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:11:04.987 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:11:05.185 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:05.358 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:11:05.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:05.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:11:05.462 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:11:05.503 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:05.564 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:11:05.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:05.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:11:05.606 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:11:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:11:05] "GET /metrics HTTP/1.1" 200 36469 "" "Prometheus/2.43.0" 2023-10-09T06:11:05.669 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:11:05.699 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:05.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:05 smithi156 ceph-mon[195101]: from='client.46181 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:11:05.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:05 smithi064 ceph-mon[212750]: from='client.46181 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:11:05.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:05 smithi064 ceph-mon[220044]: from='client.46181 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:11:06.395 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1214,"stamp":"2023-10-09T06:11:03.065936+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":907499582,"num_objects":13861,"num_object_clones":0,"num_object_copies":41583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6593,"num_objects_misplaced":7054,"num_objects_unfound":0,"num_objects_dirty":13861,"num_whiteouts":0,"num_read":123,"num_read_kb":228,"num_write":188482,"num_write_kb":6470969,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27010,"num_bytes_recovered":1529531966,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":93051,"ondisk_log_size":93051,"up":357,"acting":353,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":376,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8769160,"kb_used_data":3614728,"kb_used_omap":165,"kb_used_meta":5153882,"kb_avail":741192056,"statfs":{"total":767960285184,"available":758980665344,"internally_reserved":0,"allocated":3701481472,"data_stored":3685487643,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":169911,"internal_metadata":5277575241},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-276903017,"num_objects":-4225,"num_object_clones":0,"num_object_copies":-12675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-467,"num_objects_misplaced":-1100,"num_objects_unfound":0,"num_objects_dirty":-4225,"num_whiteouts":0,"num_read":-11,"num_read_kb":-21,"num_write":4220,"num_write_kb":-55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":701,"num_bytes_recovered":45900484,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.919128"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":757,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.201928+0000","last_change":"2023-10-09T06:07:46.599648+0000","last_active":"2023-10-09T06:10:57.201928+0000","last_peered":"2023-10-09T06:10:57.201928+0000","last_clean":"2023-10-09T06:10:57.201928+0000","last_became_active":"2023-10-09T06:07:46.599303+0000","last_became_peered":"2023-10-09T06:07:46.599303+0000","last_unstale":"2023-10-09T06:10:57.201928+0000","last_undegraded":"2023-10-09T06:10:57.201928+0000","last_fullsized":"2023-10-09T06:10:57.201928+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:05.412941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:05.412941+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:45.768743+0000","scrub_duration":0.0061014329999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":547,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.158992+0000","last_change":"2023-10-09T06:07:43.572773+0000","last_active":"2023-10-09T06:10:57.158992+0000","last_peered":"2023-10-09T06:10:57.158992+0000","last_clean":"2023-10-09T06:10:57.158992+0000","last_became_active":"2023-10-09T06:07:43.572299+0000","last_became_peered":"2023-10-09T06:07:43.572299+0000","last_unstale":"2023-10-09T06:10:57.158992+0000","last_undegraded":"2023-10-09T06:10:57.158992+0000","last_fullsized":"2023-10-09T06:10:57.158992+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:00:37.672435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":288,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.205987+0000","last_change":"2023-10-09T06:05:11.451972+0000","last_active":"2023-10-09T06:10:57.205987+0000","last_peered":"2023-10-09T06:10:57.205987+0000","last_clean":"2023-10-09T06:10:57.205987+0000","last_became_active":"2023-10-09T06:05:11.451448+0000","last_became_peered":"2023-10-09T06:05:11.451448+0000","last_unstale":"2023-10-09T06:10:57.205987+0000","last_undegraded":"2023-10-09T06:10:57.205987+0000","last_fullsized":"2023-10-09T06:10:57.205987+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:39.183621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":778,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.158467+0000","last_change":"2023-10-09T06:09:27.554162+0000","last_active":"2023-10-09T06:10:57.158467+0000","last_peered":"2023-10-09T06:10:57.158467+0000","last_clean":"2023-10-09T06:10:57.158467+0000","last_became_active":"2023-10-09T06:09:27.553659+0000","last_became_peered":"2023-10-09T06:09:27.553659+0000","last_unstale":"2023-10-09T06:10:57.158467+0000","last_undegraded":"2023-10-09T06:10:57.158467+0000","last_fullsized":"2023-10-09T06:10:57.158467+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:01.468346+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:01.468346+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:03.443511+0000","scrub_duration":0.0016258220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":159,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.161812+0000","last_change":"2023-10-09T06:01:54.762060+0000","last_active":"2023-10-09T06:10:57.161812+0000","last_peered":"2023-10-09T06:10:57.161812+0000","last_clean":"2023-10-09T06:10:57.161812+0000","last_became_active":"2023-10-09T06:01:54.761757+0000","last_became_peered":"2023-10-09T06:01:54.761757+0000","last_unstale":"2023-10-09T06:10:57.161812+0000","last_undegraded":"2023-10-09T06:10:57.161812+0000","last_fullsized":"2023-10-09T06:10:57.161812+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:40.294232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":452,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.205801+0000","last_change":"2023-10-09T06:09:28.544417+0000","last_active":"2023-10-09T06:10:57.205801+0000","last_peered":"2023-10-09T06:10:57.205801+0000","last_clean":"2023-10-09T06:10:57.205801+0000","last_became_active":"2023-10-09T06:09:28.543518+0000","last_became_peered":"2023-10-09T06:09:28.543518+0000","last_unstale":"2023-10-09T06:10:57.205801+0000","last_undegraded":"2023-10-09T06:10:57.205801+0000","last_fullsized":"2023-10-09T06:10:57.205801+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:20.008506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":723,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.158088+0000","last_change":"2023-10-09T06:02:43.293521+0000","last_active":"2023-10-09T06:10:57.158088+0000","last_peered":"2023-10-09T06:10:57.158088+0000","last_clean":"2023-10-09T06:10:57.158088+0000","last_became_active":"2023-10-09T06:01:57.803642+0000","last_became_peered":"2023-10-09T06:01:57.803642+0000","last_unstale":"2023-10-09T06:10:57.158088+0000","last_undegraded":"2023-10-09T06:10:57.158088+0000","last_fullsized":"2023-10-09T06:10:57.158088+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.293426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.293426+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:45.626761+0000","scrub_duration":0.001587946,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":270,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.206441+0000","last_change":"2023-10-09T06:09:27.564377+0000","last_active":"2023-10-09T06:10:57.206441+0000","last_peered":"2023-10-09T06:10:57.206441+0000","last_clean":"2023-10-09T06:10:57.206441+0000","last_became_active":"2023-10-09T06:09:27.563710+0000","last_became_peered":"2023-10-09T06:09:27.563710+0000","last_unstale":"2023-10-09T06:10:57.206441+0000","last_undegraded":"2023-10-09T06:10:57.206441+0000","last_fullsized":"2023-10-09T06:10:57.206441+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:48.168447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":376,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.161415+0000","last_change":"2023-10-09T06:10:50.741328+0000","last_active":"2023-10-09T06:10:57.161415+0000","last_peered":"2023-10-09T06:10:57.161415+0000","last_clean":"2023-10-09T06:10:57.161415+0000","last_became_active":"2023-10-09T06:10:50.741035+0000","last_became_peered":"2023-10-09T06:10:50.741035+0000","last_unstale":"2023-10-09T06:10:57.161415+0000","last_undegraded":"2023-10-09T06:10:57.161415+0000","last_fullsized":"2023-10-09T06:10:57.161415+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:54.243241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":686,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.162096+0000","last_change":"2023-10-09T06:09:06.649199+0000","last_active":"2023-10-09T06:10:57.162096+0000","last_peered":"2023-10-09T06:10:57.162096+0000","last_clean":"2023-10-09T06:10:57.162096+0000","last_became_active":"2023-10-09T06:09:06.648801+0000","last_became_peered":"2023-10-09T06:09:06.648801+0000","last_unstale":"2023-10-09T06:10:57.162096+0000","last_undegraded":"2023-10-09T06:10:57.162096+0000","last_fullsized":"2023-10-09T06:10:57.162096+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.384407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.384407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:48:23.220950+0000","scrub_duration":0.0040651960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":375,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.205686+0000","last_change":"2023-10-09T06:09:27.559756+0000","last_active":"2023-10-09T06:10:57.205686+0000","last_peered":"2023-10-09T06:10:57.205686+0000","last_clean":"2023-10-09T06:10:57.205686+0000","last_became_active":"2023-10-09T06:09:27.559443+0000","last_became_peered":"2023-10-09T06:09:27.559443+0000","last_unstale":"2023-10-09T06:10:57.205686+0000","last_undegraded":"2023-10-09T06:10:57.205686+0000","last_fullsized":"2023-10-09T06:10:57.205686+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:14.519398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":297,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.206501+0000","last_change":"2023-10-09T06:06:01.067544+0000","last_active":"2023-10-09T06:10:57.206501+0000","last_peered":"2023-10-09T06:10:57.206501+0000","last_clean":"2023-10-09T06:10:57.206501+0000","last_became_active":"2023-10-09T06:06:01.067188+0000","last_became_peered":"2023-10-09T06:06:01.067188+0000","last_unstale":"2023-10-09T06:10:57.206501+0000","last_undegraded":"2023-10-09T06:10:57.206501+0000","last_fullsized":"2023-10-09T06:10:57.206501+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:43.464186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":603,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.202262+0000","last_change":"2023-10-09T06:10:11.729656+0000","last_active":"2023-10-09T06:10:57.202262+0000","last_peered":"2023-10-09T06:10:57.202262+0000","last_clean":"2023-10-09T06:10:57.202262+0000","last_became_active":"2023-10-09T06:10:11.728960+0000","last_became_peered":"2023-10-09T06:10:11.728960+0000","last_unstale":"2023-10-09T06:10:57.202262+0000","last_undegraded":"2023-10-09T06:10:57.202262+0000","last_fullsized":"2023-10-09T06:10:57.202262+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:20.742561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:20.742561+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:45.543709+0000","scrub_duration":0.0027005039999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":415,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.162231+0000","last_change":"2023-10-09T06:09:03.594568+0000","last_active":"2023-10-09T06:10:57.162231+0000","last_peered":"2023-10-09T06:10:57.162231+0000","last_clean":"2023-10-09T06:10:57.162231+0000","last_became_active":"2023-10-09T06:09:03.594233+0000","last_became_peered":"2023-10-09T06:09:03.594233+0000","last_unstale":"2023-10-09T06:10:57.162231+0000","last_undegraded":"2023-10-09T06:10:57.162231+0000","last_fullsized":"2023-10-09T06:10:57.162231+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:56.654125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":437,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.204885+0000","last_change":"2023-10-09T06:09:03.597030+0000","last_active":"2023-10-09T06:10:57.204885+0000","last_peered":"2023-10-09T06:10:57.204885+0000","last_clean":"2023-10-09T06:10:57.204885+0000","last_became_active":"2023-10-09T06:09:03.596737+0000","last_became_peered":"2023-10-09T06:09:03.596737+0000","last_unstale":"2023-10-09T06:10:57.204885+0000","last_undegraded":"2023-10-09T06:10:57.204885+0000","last_fullsized":"2023-10-09T06:10:57.204885+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:40.495511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":644,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.159221+0000","last_change":"2023-10-09T06:10:50.742582+0000","last_active":"2023-10-09T06:10:57.159221+0000","last_peered":"2023-10-09T06:10:57.159221+0000","last_clean":"2023-10-09T06:10:57.159221+0000","last_became_active":"2023-10-09T06:10:50.742289+0000","last_became_peered":"2023-10-09T06:10:50.742289+0000","last_unstale":"2023-10-09T06:10:57.159221+0000","last_undegraded":"2023-10-09T06:10:57.159221+0000","last_fullsized":"2023-10-09T06:10:57.159221+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:48.374432+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:48.374432+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":94,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.204835+0000","last_change":"2023-10-09T06:09:28.558317+0000","last_active":"2023-10-09T06:10:57.204835+0000","last_peered":"2023-10-09T06:10:57.204835+0000","last_clean":"2023-10-09T06:10:57.204835+0000","last_became_active":"2023-10-09T06:09:28.543965+0000","last_became_peered":"2023-10-09T06:09:28.543965+0000","last_unstale":"2023-10-09T06:10:57.204835+0000","last_undegraded":"2023-10-09T06:10:57.204835+0000","last_fullsized":"2023-10-09T06:10:57.204835+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:27:29.745235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":309,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.215689+0000","last_change":"2023-10-09T06:09:27.553337+0000","last_active":"2023-10-09T06:10:57.215689+0000","last_peered":"2023-10-09T06:10:57.215689+0000","last_clean":"2023-10-09T06:10:57.215689+0000","last_became_active":"2023-10-09T06:09:27.551069+0000","last_became_peered":"2023-10-09T06:09:27.551069+0000","last_unstale":"2023-10-09T06:10:57.215689+0000","last_undegraded":"2023-10-09T06:10:57.215689+0000","last_fullsized":"2023-10-09T06:10:57.215689+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:47.664965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":649,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.162383+0000","last_change":"2023-10-09T06:09:27.574336+0000","last_active":"2023-10-09T06:10:57.162383+0000","last_peered":"2023-10-09T06:10:57.162383+0000","last_clean":"2023-10-09T06:10:57.162383+0000","last_became_active":"2023-10-09T06:09:27.573974+0000","last_became_peered":"2023-10-09T06:09:27.573974+0000","last_unstale":"2023-10-09T06:10:57.162383+0000","last_undegraded":"2023-10-09T06:10:57.162383+0000","last_fullsized":"2023-10-09T06:10:57.162383+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.339121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.339121+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:45.908579+0000","scrub_duration":0.010474344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":473,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.159068+0000","last_change":"2023-10-09T06:01:22.335608+0000","last_active":"2023-10-09T06:10:57.159068+0000","last_peered":"2023-10-09T06:10:57.159068+0000","last_clean":"2023-10-09T06:10:57.159068+0000","last_became_active":"2023-10-09T06:01:22.335296+0000","last_became_peered":"2023-10-09T06:01:22.335296+0000","last_unstale":"2023-10-09T06:10:57.159068+0000","last_undegraded":"2023-10-09T06:10:57.159068+0000","last_fullsized":"2023-10-09T06:10:57.159068+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:21.843341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":491,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.158734+0000","last_change":"2023-10-09T06:01:54.764916+0000","last_active":"2023-10-09T06:10:57.158734+0000","last_peered":"2023-10-09T06:10:57.158734+0000","last_clean":"2023-10-09T06:10:57.158734+0000","last_became_active":"2023-10-09T06:01:54.764705+0000","last_became_peered":"2023-10-09T06:01:54.764705+0000","last_unstale":"2023-10-09T06:10:57.158734+0000","last_undegraded":"2023-10-09T06:10:57.158734+0000","last_fullsized":"2023-10-09T06:10:57.158734+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:12.527719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":688,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.204235+0000","last_change":"2023-10-09T06:09:27.564817+0000","last_active":"2023-10-09T06:10:57.204235+0000","last_peered":"2023-10-09T06:10:57.204235+0000","last_clean":"2023-10-09T06:10:57.204235+0000","last_became_active":"2023-10-09T06:09:27.563589+0000","last_became_peered":"2023-10-09T06:09:27.563589+0000","last_unstale":"2023-10-09T06:10:57.204235+0000","last_undegraded":"2023-10-09T06:10:57.204235+0000","last_fullsized":"2023-10-09T06:10:57.204235+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:42.900876+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:42.900876+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:12.383955+0000","scrub_duration":0.0014928649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":370,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.202311+0000","last_change":"2023-10-09T06:07:43.585040+0000","last_active":"2023-10-09T06:10:57.202311+0000","last_peered":"2023-10-09T06:10:57.202311+0000","last_clean":"2023-10-09T06:10:57.202311+0000","last_became_active":"2023-10-09T06:07:43.576991+0000","last_became_peered":"2023-10-09T06:07:43.576991+0000","last_unstale":"2023-10-09T06:10:57.202311+0000","last_undegraded":"2023-10-09T06:10:57.202311+0000","last_fullsized":"2023-10-09T06:10:57.202311+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:34.279182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":523,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.204900+0000","last_change":"2023-10-09T06:01:54.761558+0000","last_active":"2023-10-09T06:10:57.204900+0000","last_peered":"2023-10-09T06:10:57.204900+0000","last_clean":"2023-10-09T06:10:57.204900+0000","last_became_active":"2023-10-09T06:01:54.761251+0000","last_became_peered":"2023-10-09T06:01:54.761251+0000","last_unstale":"2023-10-09T06:10:57.204900+0000","last_undegraded":"2023-10-09T06:10:57.204900+0000","last_fullsized":"2023-10-09T06:10:57.204900+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:53.245267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":922,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.202173+0000","last_change":"2023-10-09T06:06:03.453188+0000","last_active":"2023-10-09T06:10:57.202173+0000","last_peered":"2023-10-09T06:10:57.202173+0000","last_clean":"2023-10-09T06:10:57.202173+0000","last_became_active":"2023-10-09T06:06:03.452853+0000","last_became_peered":"2023-10-09T06:06:03.452853+0000","last_unstale":"2023-10-09T06:10:57.202173+0000","last_undegraded":"2023-10-09T06:10:57.202173+0000","last_fullsized":"2023-10-09T06:10:57.202173+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:26:21.994828+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":417,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.204298+0000","last_change":"2023-10-09T06:09:28.543938+0000","last_active":"2023-10-09T06:10:57.204298+0000","last_peered":"2023-10-09T06:10:57.204298+0000","last_clean":"2023-10-09T06:10:57.204298+0000","last_became_active":"2023-10-09T06:09:28.543628+0000","last_became_peered":"2023-10-09T06:09:28.543628+0000","last_unstale":"2023-10-09T06:10:57.204298+0000","last_undegraded":"2023-10-09T06:10:57.204298+0000","last_fullsized":"2023-10-09T06:10:57.204298+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:57:27.970416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":412,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.202916+0000","last_change":"2023-10-09T06:06:01.071454+0000","last_active":"2023-10-09T06:10:57.202916+0000","last_peered":"2023-10-09T06:10:57.202916+0000","last_clean":"2023-10-09T06:10:57.202916+0000","last_became_active":"2023-10-09T06:06:01.071137+0000","last_became_peered":"2023-10-09T06:06:01.071137+0000","last_unstale":"2023-10-09T06:10:57.202916+0000","last_undegraded":"2023-10-09T06:10:57.202916+0000","last_fullsized":"2023-10-09T06:10:57.202916+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:58:52.761376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":683,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.161665+0000","last_change":"2023-10-09T06:07:06.561922+0000","last_active":"2023-10-09T06:10:57.161665+0000","last_peered":"2023-10-09T06:10:57.161665+0000","last_clean":"2023-10-09T06:10:57.161665+0000","last_became_active":"2023-10-09T06:07:06.561625+0000","last_became_peered":"2023-10-09T06:07:06.561625+0000","last_unstale":"2023-10-09T06:10:57.161665+0000","last_undegraded":"2023-10-09T06:10:57.161665+0000","last_fullsized":"2023-10-09T06:10:57.161665+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.316697+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.316697+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:08.265733+0000","scrub_duration":0.001666561,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":332,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.204044+0000","last_change":"2023-10-09T06:01:22.337171+0000","last_active":"2023-10-09T06:10:57.204044+0000","last_peered":"2023-10-09T06:10:57.204044+0000","last_clean":"2023-10-09T06:10:57.204044+0000","last_became_active":"2023-10-09T06:01:22.335438+0000","last_became_peered":"2023-10-09T06:01:22.335438+0000","last_unstale":"2023-10-09T06:10:57.204044+0000","last_undegraded":"2023-10-09T06:10:57.204044+0000","last_fullsized":"2023-10-09T06:10:57.204044+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:25.036448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":430,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.205874+0000","last_change":"2023-10-09T06:09:28.544528+0000","last_active":"2023-10-09T06:10:57.205874+0000","last_peered":"2023-10-09T06:10:57.205874+0000","last_clean":"2023-10-09T06:10:57.205874+0000","last_became_active":"2023-10-09T06:09:28.544242+0000","last_became_peered":"2023-10-09T06:09:28.544242+0000","last_unstale":"2023-10-09T06:10:57.205874+0000","last_undegraded":"2023-10-09T06:10:57.205874+0000","last_fullsized":"2023-10-09T06:10:57.205874+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:29:07.672911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":733,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.162191+0000","last_change":"2023-10-09T06:02:40.388302+0000","last_active":"2023-10-09T06:10:57.162191+0000","last_peered":"2023-10-09T06:10:57.162191+0000","last_clean":"2023-10-09T06:10:57.162191+0000","last_became_active":"2023-10-09T06:01:57.803733+0000","last_became_peered":"2023-10-09T06:01:57.803733+0000","last_unstale":"2023-10-09T06:10:57.162191+0000","last_undegraded":"2023-10-09T06:10:57.162191+0000","last_fullsized":"2023-10-09T06:10:57.162191+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:40.388125+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:40.388125+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:17.919838+0000","scrub_duration":0.0055576990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":31,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.215413+0000","last_change":"2023-10-09T06:10:50.746147+0000","last_active":"2023-10-09T06:10:57.215413+0000","last_peered":"2023-10-09T06:10:57.215413+0000","last_clean":"2023-10-09T06:10:57.215413+0000","last_became_active":"2023-10-09T06:10:50.745733+0000","last_became_peered":"2023-10-09T06:10:50.745733+0000","last_unstale":"2023-10-09T06:10:57.215413+0000","last_undegraded":"2023-10-09T06:10:57.215413+0000","last_fullsized":"2023-10-09T06:10:57.215413+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:17.611365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":282,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.205764+0000","last_change":"2023-10-09T06:09:03.594501+0000","last_active":"2023-10-09T06:10:57.205764+0000","last_peered":"2023-10-09T06:10:57.205764+0000","last_clean":"2023-10-09T06:10:57.205764+0000","last_became_active":"2023-10-09T06:09:03.594181+0000","last_became_peered":"2023-10-09T06:09:03.594181+0000","last_unstale":"2023-10-09T06:10:57.205764+0000","last_undegraded":"2023-10-09T06:10:57.205764+0000","last_fullsized":"2023-10-09T06:10:57.205764+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:57.488346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"479'11700","reported_seq":25022,"reported_epoch":479,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:10:59.841476+0000","last_change":"2023-10-09T06:10:26.674294+0000","last_active":"2023-10-09T06:10:59.841476+0000","last_peered":"2023-10-09T06:10:59.841476+0000","last_clean":"2023-10-09T06:07:50.629239+0000","last_became_active":"2023-10-09T06:09:27.713078+0000","last_became_peered":"2023-10-09T06:09:27.713078+0000","last_unstale":"2023-10-09T06:10:59.841476+0000","last_undegraded":"2023-10-09T06:10:26.668222+0000","last_fullsized":"2023-10-09T06:09:27.541433+0000","mapping_epoch":456,"log_start":"441'4800","ondisk_log_start":"441'4800","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":6900,"log_dups_size":0,"ondisk_log_size":6900,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:47.782981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60424192,"num_objects":922,"num_object_clones":0,"num_object_copies":2766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":286,"num_objects_misplaced":1323,"num_objects_unfound":0,"num_objects_dirty":922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11700,"num_write_kb":403904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6615,"num_bytes_recovered":391774208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":922}],"blocked_by":[],"up_primary":1,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":719,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-10-09T06:10:59.422694+0000","last_change":"2023-10-09T06:10:59.422644+0000","last_active":"2023-10-09T06:10:59.422694+0000","last_peered":"2023-10-09T06:10:59.422694+0000","last_clean":"2023-10-09T06:10:59.422694+0000","last_became_active":"2023-10-09T06:09:06.649707+0000","last_became_peered":"2023-10-09T06:09:06.649707+0000","last_unstale":"2023-10-09T06:10:59.422694+0000","last_undegraded":"2023-10-09T06:10:59.422694+0000","last_fullsized":"2023-10-09T06:10:59.422694+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:10:59.422551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:10:59.422551+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:08.392769+0000","scrub_duration":0.0042232470000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":95,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.158470+0000","last_change":"2023-10-09T06:09:28.539123+0000","last_active":"2023-10-09T06:10:57.158470+0000","last_peered":"2023-10-09T06:10:57.158470+0000","last_clean":"2023-10-09T06:10:57.158470+0000","last_became_active":"2023-10-09T06:09:28.538726+0000","last_became_peered":"2023-10-09T06:09:28.538726+0000","last_unstale":"2023-10-09T06:10:57.158470+0000","last_undegraded":"2023-10-09T06:10:57.158470+0000","last_fullsized":"2023-10-09T06:10:57.158470+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:24.132933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":352,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.157911+0000","last_change":"2023-10-09T06:09:03.594770+0000","last_active":"2023-10-09T06:10:57.157911+0000","last_peered":"2023-10-09T06:10:57.157911+0000","last_clean":"2023-10-09T06:10:57.157911+0000","last_became_active":"2023-10-09T06:09:03.594371+0000","last_became_peered":"2023-10-09T06:09:03.594371+0000","last_unstale":"2023-10-09T06:10:57.157911+0000","last_undegraded":"2023-10-09T06:10:57.157911+0000","last_fullsized":"2023-10-09T06:10:57.157911+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:41.361335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.4","version":"479'11853","reported_seq":11917,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-10-09T06:11:01.235779+0000","last_change":"2023-10-09T06:07:07.580215+0000","last_active":"2023-10-09T06:11:01.235779+0000","last_peered":"2023-10-09T06:11:01.235779+0000","last_clean":"2023-10-09T06:11:01.235779+0000","last_became_active":"2023-10-09T06:07:07.579874+0000","last_became_peered":"2023-10-09T06:07:07.579874+0000","last_unstale":"2023-10-09T06:11:01.235779+0000","last_undegraded":"2023-10-09T06:11:01.235779+0000","last_fullsized":"2023-10-09T06:11:01.235779+0000","mapping_epoch":425,"log_start":"448'6944","ondisk_log_start":"448'6944","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4909,"log_dups_size":0,"ondisk_log_size":4909,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:04.426865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54722560,"num_objects":835,"num_object_clones":0,"num_object_copies":2505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11853,"num_write_kb":406016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":804,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.204644+0000","last_change":"2023-10-09T06:09:27.562925+0000","last_active":"2023-10-09T06:10:57.204644+0000","last_peered":"2023-10-09T06:10:57.204644+0000","last_clean":"2023-10-09T06:10:57.204644+0000","last_became_active":"2023-10-09T06:09:27.562532+0000","last_became_peered":"2023-10-09T06:09:27.562532+0000","last_unstale":"2023-10-09T06:10:57.204644+0000","last_undegraded":"2023-10-09T06:10:57.204644+0000","last_fullsized":"2023-10-09T06:10:57.204644+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:03.241335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:03.241335+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:29.706812+0000","scrub_duration":0.017780547000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":95,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.158569+0000","last_change":"2023-10-09T06:09:28.542485+0000","last_active":"2023-10-09T06:10:57.158569+0000","last_peered":"2023-10-09T06:10:57.158569+0000","last_clean":"2023-10-09T06:10:57.158569+0000","last_became_active":"2023-10-09T06:09:28.539178+0000","last_became_peered":"2023-10-09T06:09:28.539178+0000","last_unstale":"2023-10-09T06:10:57.158569+0000","last_undegraded":"2023-10-09T06:10:57.158569+0000","last_fullsized":"2023-10-09T06:10:57.158569+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:34.804101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":517,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.158184+0000","last_change":"2023-10-09T06:01:54.760367+0000","last_active":"2023-10-09T06:10:57.158184+0000","last_peered":"2023-10-09T06:10:57.158184+0000","last_clean":"2023-10-09T06:10:57.158184+0000","last_became_active":"2023-10-09T06:01:54.760087+0000","last_became_peered":"2023-10-09T06:01:54.760087+0000","last_unstale":"2023-10-09T06:10:57.158184+0000","last_undegraded":"2023-10-09T06:10:57.158184+0000","last_fullsized":"2023-10-09T06:10:57.158184+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:26.000104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"479'11720","reported_seq":11793,"reported_epoch":479,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:11:00.163672+0000","last_change":"2023-10-09T06:09:45.989399+0000","last_active":"2023-10-09T06:11:00.163672+0000","last_peered":"2023-10-09T06:11:00.163672+0000","last_clean":"2023-10-09T06:09:02.497781+0000","last_became_active":"2023-10-09T06:09:04.726124+0000","last_became_peered":"2023-10-09T06:09:04.726124+0000","last_unstale":"2023-10-09T06:11:00.163672+0000","last_undegraded":"2023-10-09T06:11:00.163672+0000","last_fullsized":"2023-10-09T06:11:00.163672+0000","mapping_epoch":450,"log_start":"447'6785","ondisk_log_start":"447'6785","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4935,"log_dups_size":0,"ondisk_log_size":4935,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:00.218495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58589184,"num_objects":894,"num_object_clones":0,"num_object_copies":2682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":894,"num_objects_unfound":0,"num_objects_dirty":894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11720,"num_write_kb":403648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":894}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":692,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.158553+0000","last_change":"2023-10-09T06:10:50.741131+0000","last_active":"2023-10-09T06:10:57.158553+0000","last_peered":"2023-10-09T06:10:57.158553+0000","last_clean":"2023-10-09T06:10:57.158553+0000","last_became_active":"2023-10-09T06:10:50.740762+0000","last_became_peered":"2023-10-09T06:10:50.740762+0000","last_unstale":"2023-10-09T06:10:57.158553+0000","last_undegraded":"2023-10-09T06:10:57.158553+0000","last_fullsized":"2023-10-09T06:10:57.158553+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:41.753506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:41.753506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":331,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.203119+0000","last_change":"2023-10-09T06:06:01.078672+0000","last_active":"2023-10-09T06:10:57.203119+0000","last_peered":"2023-10-09T06:10:57.203119+0000","last_clean":"2023-10-09T06:10:57.203119+0000","last_became_active":"2023-10-09T06:06:01.072297+0000","last_became_peered":"2023-10-09T06:06:01.072297+0000","last_unstale":"2023-10-09T06:10:57.203119+0000","last_undegraded":"2023-10-09T06:10:57.203119+0000","last_fullsized":"2023-10-09T06:10:57.203119+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:35.905773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":484,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.161810+0000","last_change":"2023-10-09T06:09:28.540287+0000","last_active":"2023-10-09T06:10:57.161810+0000","last_peered":"2023-10-09T06:10:57.161810+0000","last_clean":"2023-10-09T06:10:57.161810+0000","last_became_active":"2023-10-09T06:09:28.540012+0000","last_became_peered":"2023-10-09T06:09:28.540012+0000","last_unstale":"2023-10-09T06:10:57.161810+0000","last_undegraded":"2023-10-09T06:10:57.161810+0000","last_fullsized":"2023-10-09T06:10:57.161810+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:50.310629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"479'11732","reported_seq":18485,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-10-09T06:11:01.231286+0000","last_change":"2023-10-09T06:09:48.010236+0000","last_active":"2023-10-09T06:11:01.231286+0000","last_peered":"2023-10-09T06:11:01.231286+0000","last_clean":"2023-10-09T06:11:01.231286+0000","last_became_active":"2023-10-09T06:09:47.988218+0000","last_became_peered":"2023-10-09T06:09:47.988218+0000","last_unstale":"2023-10-09T06:11:01.231286+0000","last_undegraded":"2023-10-09T06:11:01.231286+0000","last_fullsized":"2023-10-09T06:11:01.231286+0000","mapping_epoch":460,"log_start":"447'6801","ondisk_log_start":"447'6801","created":425,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4931,"log_dups_size":0,"ondisk_log_size":4931,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:13.479541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52428800,"num_objects":800,"num_object_clones":0,"num_object_copies":2400,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11732,"num_write_kb":401024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3335,"num_bytes_recovered":203948032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":674,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.158802+0000","last_change":"2023-10-09T06:09:06.648659+0000","last_active":"2023-10-09T06:10:57.158802+0000","last_peered":"2023-10-09T06:10:57.158802+0000","last_clean":"2023-10-09T06:10:57.158802+0000","last_became_active":"2023-10-09T06:09:06.648332+0000","last_became_peered":"2023-10-09T06:09:06.648332+0000","last_unstale":"2023-10-09T06:10:57.158802+0000","last_undegraded":"2023-10-09T06:10:57.158802+0000","last_fullsized":"2023-10-09T06:10:57.158802+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:16.361097+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:16.361097+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:52:53.333570+0000","scrub_duration":0.028533749000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":247,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.204736+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T06:10:57.204736+0000","last_peered":"2023-10-09T06:10:57.204736+0000","last_clean":"2023-10-09T06:10:57.204736+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T06:10:57.204736+0000","last_undegraded":"2023-10-09T06:10:57.204736+0000","last_fullsized":"2023-10-09T06:10:57.204736+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":490,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.215749+0000","last_change":"2023-10-09T06:09:27.558353+0000","last_active":"2023-10-09T06:10:57.215749+0000","last_peered":"2023-10-09T06:10:57.215749+0000","last_clean":"2023-10-09T06:10:57.215749+0000","last_became_active":"2023-10-09T06:09:27.557233+0000","last_became_peered":"2023-10-09T06:09:27.557233+0000","last_unstale":"2023-10-09T06:10:57.215749+0000","last_undegraded":"2023-10-09T06:10:57.215749+0000","last_fullsized":"2023-10-09T06:10:57.215749+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:53.149350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"479'11723","reported_seq":19007,"reported_epoch":479,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:11:00.195010+0000","last_change":"2023-10-09T06:09:28.546151+0000","last_active":"2023-10-09T06:11:00.195010+0000","last_peered":"2023-10-09T06:11:00.195010+0000","last_clean":"2023-10-09T06:09:02.515225+0000","last_became_active":"2023-10-09T06:09:04.700996+0000","last_became_peered":"2023-10-09T06:09:04.700996+0000","last_unstale":"2023-10-09T06:11:00.195010+0000","last_undegraded":"2023-10-09T06:11:00.195010+0000","last_fullsized":"2023-10-09T06:11:00.195010+0000","mapping_epoch":450,"log_start":"447'6835","ondisk_log_start":"447'6835","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4888,"log_dups_size":0,"ondisk_log_size":4888,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:54.423826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56557568,"num_objects":863,"num_object_clones":0,"num_object_copies":2589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":186,"num_objects_unfound":0,"num_objects_dirty":863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11723,"num_write_kb":402752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3603,"num_bytes_recovered":236126208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,7],"avail_no_missing":["3","6","7"],"object_location_counts":[{"shards":"3,6,7","objects":863}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":962,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.204160+0000","last_change":"2023-10-09T06:07:46.595962+0000","last_active":"2023-10-09T06:10:57.204160+0000","last_peered":"2023-10-09T06:10:57.204160+0000","last_clean":"2023-10-09T06:10:57.204160+0000","last_became_active":"2023-10-09T06:07:46.595718+0000","last_became_peered":"2023-10-09T06:07:46.595718+0000","last_unstale":"2023-10-09T06:10:57.204160+0000","last_undegraded":"2023-10-09T06:10:57.204160+0000","last_fullsized":"2023-10-09T06:10:57.204160+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:55.437947+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:55.437947+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:45:57.993572+0000","scrub_duration":0.0091262979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":281,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.205543+0000","last_change":"2023-10-09T06:09:27.565666+0000","last_active":"2023-10-09T06:10:57.205543+0000","last_peered":"2023-10-09T06:10:57.205543+0000","last_clean":"2023-10-09T06:10:57.205543+0000","last_became_active":"2023-10-09T06:09:27.565324+0000","last_became_peered":"2023-10-09T06:09:27.565324+0000","last_unstale":"2023-10-09T06:10:57.205543+0000","last_undegraded":"2023-10-09T06:10:57.205543+0000","last_fullsized":"2023-10-09T06:10:57.205543+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:51.216947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":433,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.158971+0000","last_change":"2023-10-09T06:01:22.332033+0000","last_active":"2023-10-09T06:10:57.158971+0000","last_peered":"2023-10-09T06:10:57.158971+0000","last_clean":"2023-10-09T06:10:57.158971+0000","last_became_active":"2023-10-09T06:01:22.329825+0000","last_became_peered":"2023-10-09T06:01:22.329825+0000","last_unstale":"2023-10-09T06:10:57.158971+0000","last_undegraded":"2023-10-09T06:10:57.158971+0000","last_fullsized":"2023-10-09T06:10:57.158971+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:54.046681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"479'12108","reported_seq":19041,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-10-09T06:11:01.219181+0000","last_change":"2023-10-09T06:10:50.767465+0000","last_active":"2023-10-09T06:11:01.219181+0000","last_peered":"2023-10-09T06:11:01.219181+0000","last_clean":"2023-10-09T06:11:01.219181+0000","last_became_active":"2023-10-09T06:10:50.741045+0000","last_became_peered":"2023-10-09T06:10:50.741045+0000","last_unstale":"2023-10-09T06:11:01.219181+0000","last_undegraded":"2023-10-09T06:11:01.219181+0000","last_fullsized":"2023-10-09T06:11:01.219181+0000","mapping_epoch":475,"log_start":"448'7235","ondisk_log_start":"448'7235","created":425,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4873,"log_dups_size":0,"ondisk_log_size":4873,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:40:41.021292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57540608,"num_objects":878,"num_object_clones":0,"num_object_copies":2634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12108,"num_write_kb":415552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3424,"num_bytes_recovered":42795008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":723,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.206264+0000","last_change":"2023-10-09T06:07:46.602409+0000","last_active":"2023-10-09T06:10:57.206264+0000","last_peered":"2023-10-09T06:10:57.206264+0000","last_clean":"2023-10-09T06:10:57.206264+0000","last_became_active":"2023-10-09T06:07:46.602045+0000","last_became_peered":"2023-10-09T06:07:46.602045+0000","last_unstale":"2023-10-09T06:10:57.206264+0000","last_undegraded":"2023-10-09T06:10:57.206264+0000","last_fullsized":"2023-10-09T06:10:57.206264+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.282393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.282393+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:35.114362+0000","scrub_duration":0.01402403,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":378,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.158832+0000","last_change":"2023-10-09T06:10:50.741006+0000","last_active":"2023-10-09T06:10:57.158832+0000","last_peered":"2023-10-09T06:10:57.158832+0000","last_clean":"2023-10-09T06:10:57.158832+0000","last_became_active":"2023-10-09T06:10:50.740683+0000","last_became_peered":"2023-10-09T06:10:50.740683+0000","last_unstale":"2023-10-09T06:10:57.158832+0000","last_undegraded":"2023-10-09T06:10:57.158832+0000","last_fullsized":"2023-10-09T06:10:57.158832+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:15.973370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":527,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.158489+0000","last_change":"2023-10-09T06:07:43.572173+0000","last_active":"2023-10-09T06:10:57.158489+0000","last_peered":"2023-10-09T06:10:57.158489+0000","last_clean":"2023-10-09T06:10:57.158489+0000","last_became_active":"2023-10-09T06:07:43.571690+0000","last_became_peered":"2023-10-09T06:07:43.571690+0000","last_unstale":"2023-10-09T06:10:57.158489+0000","last_undegraded":"2023-10-09T06:10:57.158489+0000","last_fullsized":"2023-10-09T06:10:57.158489+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:10.234125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"479'12014","reported_seq":12091,"reported_epoch":479,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:11:00.148981+0000","last_change":"2023-10-09T06:09:04.778690+0000","last_active":"2023-10-09T06:11:00.148981+0000","last_peered":"2023-10-09T06:11:00.148981+0000","last_clean":"2023-10-09T06:09:02.515252+0000","last_became_active":"2023-10-09T06:09:04.710755+0000","last_became_peered":"2023-10-09T06:09:04.710755+0000","last_unstale":"2023-10-09T06:11:00.148981+0000","last_undegraded":"2023-10-09T06:11:00.148981+0000","last_fullsized":"2023-10-09T06:11:00.148981+0000","mapping_epoch":450,"log_start":"448'7129","ondisk_log_start":"448'7129","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4885,"log_dups_size":0,"ondisk_log_size":4885,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:30.606908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57409536,"num_objects":876,"num_object_clones":0,"num_object_copies":2628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":876,"num_objects_unfound":0,"num_objects_dirty":876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12014,"num_write_kb":412480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,7],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3,7","objects":876}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":778,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-10-09T06:10:59.230468+0000","last_change":"2023-10-09T06:09:27.548687+0000","last_active":"2023-10-09T06:10:59.230468+0000","last_peered":"2023-10-09T06:10:59.230468+0000","last_clean":"2023-10-09T06:10:59.230468+0000","last_became_active":"2023-10-09T06:09:27.548382+0000","last_became_peered":"2023-10-09T06:09:27.548382+0000","last_unstale":"2023-10-09T06:10:59.230468+0000","last_undegraded":"2023-10-09T06:10:59.230468+0000","last_fullsized":"2023-10-09T06:10:59.230468+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.373748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.373748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:33.228748+0000","scrub_duration":0.0024576450000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":469,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.158122+0000","last_change":"2023-10-09T06:07:43.574863+0000","last_active":"2023-10-09T06:10:57.158122+0000","last_peered":"2023-10-09T06:10:57.158122+0000","last_clean":"2023-10-09T06:10:57.158122+0000","last_became_active":"2023-10-09T06:07:43.574576+0000","last_became_peered":"2023-10-09T06:07:43.574576+0000","last_unstale":"2023-10-09T06:10:57.158122+0000","last_undegraded":"2023-10-09T06:10:57.158122+0000","last_fullsized":"2023-10-09T06:10:57.158122+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:16.166004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":500,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.158714+0000","last_change":"2023-10-09T06:06:01.068708+0000","last_active":"2023-10-09T06:10:57.158714+0000","last_peered":"2023-10-09T06:10:57.158714+0000","last_clean":"2023-10-09T06:10:57.158714+0000","last_became_active":"2023-10-09T06:06:01.068474+0000","last_became_peered":"2023-10-09T06:06:01.068474+0000","last_unstale":"2023-10-09T06:10:57.158714+0000","last_undegraded":"2023-10-09T06:10:57.158714+0000","last_fullsized":"2023-10-09T06:10:57.158714+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:31.749069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"479'11756","reported_seq":18329,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-10-09T06:11:00.174897+0000","last_change":"2023-10-09T06:08:46.636506+0000","last_active":"2023-10-09T06:11:00.174897+0000","last_peered":"2023-10-09T06:11:00.174897+0000","last_clean":"2023-10-09T06:11:00.174897+0000","last_became_active":"2023-10-09T06:08:46.628177+0000","last_became_peered":"2023-10-09T06:08:46.628177+0000","last_unstale":"2023-10-09T06:11:00.174897+0000","last_undegraded":"2023-10-09T06:11:00.174897+0000","last_fullsized":"2023-10-09T06:11:00.174897+0000","mapping_epoch":445,"log_start":"447'6866","ondisk_log_start":"447'6866","created":425,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4890,"log_dups_size":0,"ondisk_log_size":4890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:01.324221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61210624,"num_objects":934,"num_object_clones":0,"num_object_copies":2802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11756,"num_write_kb":406080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3245,"num_bytes_recovered":212664320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":554,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.158642+0000","last_change":"2023-10-09T06:10:50.742579+0000","last_active":"2023-10-09T06:10:57.158642+0000","last_peered":"2023-10-09T06:10:57.158642+0000","last_clean":"2023-10-09T06:10:57.158642+0000","last_became_active":"2023-10-09T06:10:50.742185+0000","last_became_peered":"2023-10-09T06:10:50.742185+0000","last_unstale":"2023-10-09T06:10:57.158642+0000","last_undegraded":"2023-10-09T06:10:57.158642+0000","last_fullsized":"2023-10-09T06:10:57.158642+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:52.129816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:52.129816+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":280,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.206377+0000","last_change":"2023-10-09T06:09:03.587524+0000","last_active":"2023-10-09T06:10:57.206377+0000","last_peered":"2023-10-09T06:10:57.206377+0000","last_clean":"2023-10-09T06:10:57.206377+0000","last_became_active":"2023-10-09T06:09:03.586906+0000","last_became_peered":"2023-10-09T06:09:03.586906+0000","last_unstale":"2023-10-09T06:10:57.206377+0000","last_undegraded":"2023-10-09T06:10:57.206377+0000","last_fullsized":"2023-10-09T06:10:57.206377+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:35:01.785360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":160,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.161590+0000","last_change":"2023-10-09T06:01:54.761768+0000","last_active":"2023-10-09T06:10:57.161590+0000","last_peered":"2023-10-09T06:10:57.161590+0000","last_clean":"2023-10-09T06:10:57.161590+0000","last_became_active":"2023-10-09T06:01:54.761443+0000","last_became_peered":"2023-10-09T06:01:54.761443+0000","last_unstale":"2023-10-09T06:10:57.161590+0000","last_undegraded":"2023-10-09T06:10:57.161590+0000","last_fullsized":"2023-10-09T06:10:57.161590+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:51.555987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"479'11691","reported_seq":11754,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-10-09T06:11:01.168610+0000","last_change":"2023-10-09T06:07:07.880280+0000","last_active":"2023-10-09T06:11:01.168610+0000","last_peered":"2023-10-09T06:11:01.168610+0000","last_clean":"2023-10-09T06:11:01.168610+0000","last_became_active":"2023-10-09T06:07:07.879952+0000","last_became_peered":"2023-10-09T06:07:07.879952+0000","last_unstale":"2023-10-09T06:11:01.168610+0000","last_undegraded":"2023-10-09T06:11:01.168610+0000","last_fullsized":"2023-10-09T06:11:01.168610+0000","mapping_epoch":425,"log_start":"447'6754","ondisk_log_start":"447'6754","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4937,"log_dups_size":0,"ondisk_log_size":4937,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:10.864472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56033280,"num_objects":855,"num_object_clones":0,"num_object_copies":2565,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":855,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11691,"num_write_kb":401472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":657,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.205593+0000","last_change":"2023-10-09T06:10:29.731684+0000","last_active":"2023-10-09T06:10:57.205593+0000","last_peered":"2023-10-09T06:10:57.205593+0000","last_clean":"2023-10-09T06:10:57.205593+0000","last_became_active":"2023-10-09T06:10:29.731319+0000","last_became_peered":"2023-10-09T06:10:29.731319+0000","last_unstale":"2023-10-09T06:10:57.205593+0000","last_undegraded":"2023-10-09T06:10:57.205593+0000","last_fullsized":"2023-10-09T06:10:57.205593+0000","mapping_epoch":470,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.092586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.092586+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":204,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.162053+0000","last_change":"2023-10-09T06:09:27.549207+0000","last_active":"2023-10-09T06:10:57.162053+0000","last_peered":"2023-10-09T06:10:57.162053+0000","last_clean":"2023-10-09T06:10:57.162053+0000","last_became_active":"2023-10-09T06:09:27.547145+0000","last_became_peered":"2023-10-09T06:09:27.547145+0000","last_unstale":"2023-10-09T06:10:57.162053+0000","last_undegraded":"2023-10-09T06:10:57.162053+0000","last_fullsized":"2023-10-09T06:10:57.162053+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:48.202707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":265,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.204677+0000","last_change":"2023-10-09T06:07:43.577306+0000","last_active":"2023-10-09T06:10:57.204677+0000","last_peered":"2023-10-09T06:10:57.204677+0000","last_clean":"2023-10-09T06:10:57.204677+0000","last_became_active":"2023-10-09T06:07:43.576956+0000","last_became_peered":"2023-10-09T06:07:43.576956+0000","last_unstale":"2023-10-09T06:10:57.204677+0000","last_undegraded":"2023-10-09T06:10:57.204677+0000","last_fullsized":"2023-10-09T06:10:57.204677+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:21.020403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":184,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.161954+0000","last_change":"2023-10-09T06:09:27.557353+0000","last_active":"2023-10-09T06:10:57.161954+0000","last_peered":"2023-10-09T06:10:57.161954+0000","last_clean":"2023-10-09T06:10:57.161954+0000","last_became_active":"2023-10-09T06:09:27.557042+0000","last_became_peered":"2023-10-09T06:09:27.557042+0000","last_unstale":"2023-10-09T06:10:57.161954+0000","last_undegraded":"2023-10-09T06:10:57.161954+0000","last_fullsized":"2023-10-09T06:10:57.161954+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:28.264019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"479'11966","reported_seq":20275,"reported_epoch":479,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:11:01.143082+0000","last_change":"2023-10-09T06:10:26.674387+0000","last_active":"2023-10-09T06:11:01.143082+0000","last_peered":"2023-10-09T06:11:01.143082+0000","last_clean":"2023-10-09T06:07:50.661954+0000","last_became_active":"2023-10-09T06:09:27.738698+0000","last_became_peered":"2023-10-09T06:09:27.738698+0000","last_unstale":"2023-10-09T06:11:01.143082+0000","last_undegraded":"2023-10-09T06:09:27.673640+0000","last_fullsized":"2023-10-09T06:09:27.533273+0000","mapping_epoch":456,"log_start":"440'4200","ondisk_log_start":"440'4200","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":7766,"log_dups_size":0,"ondisk_log_size":7766,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:19.516164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55967744,"num_objects":854,"num_object_clones":0,"num_object_copies":2562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":452,"num_objects_misplaced":2940,"num_objects_unfound":0,"num_objects_dirty":854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11966,"num_write_kb":410240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4106,"num_bytes_recovered":268894208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"3,5,7","objects":401},{"shards":"5,7","objects":-100}],"blocked_by":[],"up_primary":3,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":712,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.215965+0000","last_change":"2023-10-09T06:09:27.544555+0000","last_active":"2023-10-09T06:10:57.215965+0000","last_peered":"2023-10-09T06:10:57.215965+0000","last_clean":"2023-10-09T06:10:57.215965+0000","last_became_active":"2023-10-09T06:09:27.544222+0000","last_became_peered":"2023-10-09T06:09:27.544222+0000","last_unstale":"2023-10-09T06:10:57.215965+0000","last_undegraded":"2023-10-09T06:10:57.215965+0000","last_fullsized":"2023-10-09T06:10:57.215965+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:45.944253+0000","scrub_duration":0.0016660799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":169,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.162095+0000","last_change":"2023-10-09T06:10:26.655740+0000","last_active":"2023-10-09T06:10:57.162095+0000","last_peered":"2023-10-09T06:10:57.162095+0000","last_clean":"2023-10-09T06:10:57.162095+0000","last_became_active":"2023-10-09T06:10:26.649431+0000","last_became_peered":"2023-10-09T06:10:26.649431+0000","last_unstale":"2023-10-09T06:10:57.162095+0000","last_undegraded":"2023-10-09T06:10:57.162095+0000","last_fullsized":"2023-10-09T06:10:57.162095+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:36.123840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":459,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.158248+0000","last_change":"2023-10-09T06:09:27.553958+0000","last_active":"2023-10-09T06:10:57.158248+0000","last_peered":"2023-10-09T06:10:57.158248+0000","last_clean":"2023-10-09T06:10:57.158248+0000","last_became_active":"2023-10-09T06:09:27.553193+0000","last_became_peered":"2023-10-09T06:09:27.553193+0000","last_unstale":"2023-10-09T06:10:57.158248+0000","last_undegraded":"2023-10-09T06:10:57.158248+0000","last_fullsized":"2023-10-09T06:10:57.158248+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:09.754579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":303,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.162130+0000","last_change":"2023-10-09T06:07:43.580765+0000","last_active":"2023-10-09T06:10:57.162130+0000","last_peered":"2023-10-09T06:10:57.162130+0000","last_clean":"2023-10-09T06:10:57.162130+0000","last_became_active":"2023-10-09T06:07:43.580458+0000","last_became_peered":"2023-10-09T06:07:43.580458+0000","last_unstale":"2023-10-09T06:10:57.162130+0000","last_undegraded":"2023-10-09T06:10:57.162130+0000","last_fullsized":"2023-10-09T06:10:57.162130+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:55.161893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"479'11648","reported_seq":11712,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-10-09T06:11:01.231301+0000","last_change":"2023-10-09T06:07:07.546882+0000","last_active":"2023-10-09T06:11:01.231301+0000","last_peered":"2023-10-09T06:11:01.231301+0000","last_clean":"2023-10-09T06:11:01.231301+0000","last_became_active":"2023-10-09T06:07:07.546552+0000","last_became_peered":"2023-10-09T06:07:07.546552+0000","last_unstale":"2023-10-09T06:11:01.231301+0000","last_undegraded":"2023-10-09T06:11:01.231301+0000","last_fullsized":"2023-10-09T06:11:01.231301+0000","mapping_epoch":425,"log_start":"447'6748","ondisk_log_start":"447'6748","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4900,"log_dups_size":0,"ondisk_log_size":4900,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:57.434915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54919168,"num_objects":838,"num_object_clones":0,"num_object_copies":2514,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":838,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11648,"num_write_kb":399552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":559,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-10-09T06:11:00.443006+0000","last_change":"2023-10-09T06:11:00.443006+0000","last_active":"2023-10-09T06:11:00.443006+0000","last_peered":"2023-10-09T06:11:00.443006+0000","last_clean":"2023-10-09T06:11:00.443006+0000","last_became_active":"2023-10-09T06:09:28.538562+0000","last_became_peered":"2023-10-09T06:09:28.538562+0000","last_unstale":"2023-10-09T06:11:00.443006+0000","last_undegraded":"2023-10-09T06:11:00.443006+0000","last_fullsized":"2023-10-09T06:11:00.443006+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:55.897812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:55.897812+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":257,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.204380+0000","last_change":"2023-10-09T06:09:27.565004+0000","last_active":"2023-10-09T06:10:57.204380+0000","last_peered":"2023-10-09T06:10:57.204380+0000","last_clean":"2023-10-09T06:10:57.204380+0000","last_became_active":"2023-10-09T06:09:27.564073+0000","last_became_peered":"2023-10-09T06:09:27.564073+0000","last_unstale":"2023-10-09T06:10:57.204380+0000","last_undegraded":"2023-10-09T06:10:57.204380+0000","last_fullsized":"2023-10-09T06:10:57.204380+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:15.146042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":229,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.162538+0000","last_change":"2023-10-09T06:07:43.571911+0000","last_active":"2023-10-09T06:10:57.162538+0000","last_peered":"2023-10-09T06:10:57.162538+0000","last_clean":"2023-10-09T06:10:57.162538+0000","last_became_active":"2023-10-09T06:07:43.571539+0000","last_became_peered":"2023-10-09T06:07:43.571539+0000","last_unstale":"2023-10-09T06:10:57.162538+0000","last_undegraded":"2023-10-09T06:10:57.162538+0000","last_fullsized":"2023-10-09T06:10:57.162538+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:26.591500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":175,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.161519+0000","last_change":"2023-10-09T06:09:27.547953+0000","last_active":"2023-10-09T06:10:57.161519+0000","last_peered":"2023-10-09T06:10:57.161519+0000","last_clean":"2023-10-09T06:10:57.161519+0000","last_became_active":"2023-10-09T06:09:27.547624+0000","last_became_peered":"2023-10-09T06:09:27.547624+0000","last_unstale":"2023-10-09T06:10:57.161519+0000","last_undegraded":"2023-10-09T06:10:57.161519+0000","last_fullsized":"2023-10-09T06:10:57.161519+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:39.630486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"479'11734","reported_seq":12728,"reported_epoch":479,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T06:11:00.165116+0000","last_change":"2023-10-09T06:10:49.755996+0000","last_active":"2023-10-09T06:11:00.165116+0000","last_peered":"2023-10-09T06:11:00.165116+0000","last_clean":"2023-10-09T06:07:50.696644+0000","last_became_active":"2023-10-09T06:09:28.677635+0000","last_became_peered":"2023-10-09T06:09:28.677635+0000","last_unstale":"2023-10-09T06:11:00.165116+0000","last_undegraded":"2023-10-09T06:09:28.553195+0000","last_fullsized":"2023-10-09T06:09:28.523094+0000","mapping_epoch":457,"log_start":"447'6812","ondisk_log_start":"447'6812","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4922,"log_dups_size":0,"ondisk_log_size":4922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:43.601977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55181312,"num_objects":842,"num_object_clones":0,"num_object_copies":2526,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2163,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":842,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11734,"num_write_kb":402432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":457,"num_bytes_recovered":29949952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-1321}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":40,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.205616+0000","last_change":"2023-10-09T06:09:03.586696+0000","last_active":"2023-10-09T06:10:57.205616+0000","last_peered":"2023-10-09T06:10:57.205616+0000","last_clean":"2023-10-09T06:10:57.205616+0000","last_became_active":"2023-10-09T06:09:03.586343+0000","last_became_peered":"2023-10-09T06:09:03.586343+0000","last_unstale":"2023-10-09T06:10:57.205616+0000","last_undegraded":"2023-10-09T06:10:57.205616+0000","last_fullsized":"2023-10-09T06:10:57.205616+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:32.651148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":464,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.215306+0000","last_change":"2023-10-09T06:09:28.534433+0000","last_active":"2023-10-09T06:10:57.215306+0000","last_peered":"2023-10-09T06:10:57.215306+0000","last_clean":"2023-10-09T06:10:57.215306+0000","last_became_active":"2023-10-09T06:09:28.534059+0000","last_became_peered":"2023-10-09T06:09:28.534059+0000","last_unstale":"2023-10-09T06:10:57.215306+0000","last_undegraded":"2023-10-09T06:10:57.215306+0000","last_fullsized":"2023-10-09T06:10:57.215306+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:37.782173+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:37.782173+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":247,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.204384+0000","last_change":"2023-10-09T05:57:23.479174+0000","last_active":"2023-10-09T06:10:57.204384+0000","last_peered":"2023-10-09T06:10:57.204384+0000","last_clean":"2023-10-09T06:10:57.204384+0000","last_became_active":"2023-10-09T05:57:23.478897+0000","last_became_peered":"2023-10-09T05:57:23.478897+0000","last_unstale":"2023-10-09T06:10:57.204384+0000","last_undegraded":"2023-10-09T06:10:57.204384+0000","last_fullsized":"2023-10-09T06:10:57.204384+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:00.160042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":399,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.158411+0000","last_change":"2023-10-09T06:09:27.552948+0000","last_active":"2023-10-09T06:10:57.158411+0000","last_peered":"2023-10-09T06:10:57.158411+0000","last_clean":"2023-10-09T06:10:57.158411+0000","last_became_active":"2023-10-09T06:09:27.552680+0000","last_became_peered":"2023-10-09T06:09:27.552680+0000","last_unstale":"2023-10-09T06:10:57.158411+0000","last_undegraded":"2023-10-09T06:10:57.158411+0000","last_fullsized":"2023-10-09T06:10:57.158411+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:14:08.024300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"478'11791","reported_seq":13948,"reported_epoch":478,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:10:58.806780+0000","last_change":"2023-10-09T06:10:50.797343+0000","last_active":"2023-10-09T06:10:58.806780+0000","last_peered":"2023-10-09T06:10:58.806780+0000","last_clean":"2023-10-09T06:07:50.696037+0000","last_became_active":"2023-10-09T06:10:50.747035+0000","last_became_peered":"2023-10-09T06:10:50.747035+0000","last_unstale":"2023-10-09T06:10:58.806780+0000","last_undegraded":"2023-10-09T06:10:50.744226+0000","last_fullsized":"2023-10-09T06:10:50.739556+0000","mapping_epoch":475,"log_start":"431'2900","ondisk_log_start":"431'2900","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":8891,"log_dups_size":0,"ondisk_log_size":8891,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:21:37.136014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62324736,"num_objects":951,"num_object_clones":0,"num_object_copies":2853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3692,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11791,"num_write_kb":407744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1035,"num_bytes_recovered":67698688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"5,6","objects":-185}],"blocked_by":[],"up_primary":0,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":695,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.161574+0000","last_change":"2023-10-09T06:07:06.560255+0000","last_active":"2023-10-09T06:10:57.161574+0000","last_peered":"2023-10-09T06:10:57.161574+0000","last_clean":"2023-10-09T06:10:57.161574+0000","last_became_active":"2023-10-09T06:07:06.559524+0000","last_became_peered":"2023-10-09T06:07:06.559524+0000","last_unstale":"2023-10-09T06:10:57.161574+0000","last_undegraded":"2023-10-09T06:10:57.161574+0000","last_fullsized":"2023-10-09T06:10:57.161574+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:57.627296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:57.627296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:41.099391+0000","scrub_duration":0.001913166,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":372,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.202094+0000","last_change":"2023-10-09T06:09:27.544373+0000","last_active":"2023-10-09T06:10:57.202094+0000","last_peered":"2023-10-09T06:10:57.202094+0000","last_clean":"2023-10-09T06:10:57.202094+0000","last_became_active":"2023-10-09T06:09:27.544121+0000","last_became_peered":"2023-10-09T06:09:27.544121+0000","last_unstale":"2023-10-09T06:10:57.202094+0000","last_undegraded":"2023-10-09T06:10:57.202094+0000","last_fullsized":"2023-10-09T06:10:57.202094+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:40.429491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":396,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.215708+0000","last_change":"2023-10-09T06:09:27.569074+0000","last_active":"2023-10-09T06:10:57.215708+0000","last_peered":"2023-10-09T06:10:57.215708+0000","last_clean":"2023-10-09T06:10:57.215708+0000","last_became_active":"2023-10-09T06:09:27.568435+0000","last_became_peered":"2023-10-09T06:09:27.568435+0000","last_unstale":"2023-10-09T06:10:57.215708+0000","last_undegraded":"2023-10-09T06:10:57.215708+0000","last_fullsized":"2023-10-09T06:10:57.215708+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:33:09.130779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":553,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.216375+0000","last_change":"2023-10-09T06:09:27.543678+0000","last_active":"2023-10-09T06:10:57.216375+0000","last_peered":"2023-10-09T06:10:57.216375+0000","last_clean":"2023-10-09T06:10:57.216375+0000","last_became_active":"2023-10-09T06:09:27.543298+0000","last_became_peered":"2023-10-09T06:09:27.543298+0000","last_unstale":"2023-10-09T06:10:57.216375+0000","last_undegraded":"2023-10-09T06:10:57.216375+0000","last_fullsized":"2023-10-09T06:10:57.216375+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:26.297842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"479'11587","reported_seq":11737,"reported_epoch":479,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:11:00.637101+0000","last_change":"2023-10-09T06:09:27.745393+0000","last_active":"2023-10-09T06:11:00.637101+0000","last_peered":"2023-10-09T06:11:00.637101+0000","last_clean":"2023-10-09T06:09:26.540222+0000","last_became_active":"2023-10-09T06:09:27.646752+0000","last_became_peered":"2023-10-09T06:09:27.646752+0000","last_unstale":"2023-10-09T06:11:00.637101+0000","last_undegraded":"2023-10-09T06:11:00.637101+0000","last_fullsized":"2023-10-09T06:11:00.637101+0000","mapping_epoch":456,"log_start":"444'6000","ondisk_log_start":"444'6000","created":425,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":5587,"log_dups_size":0,"ondisk_log_size":5587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:23:17.448799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55246848,"num_objects":843,"num_object_clones":0,"num_object_copies":2529,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":835,"num_objects_unfound":0,"num_objects_dirty":843,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11587,"num_write_kb":397760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":1966080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,7],"avail_no_missing":["4","1","7"],"object_location_counts":[{"shards":"1,4,7","objects":843}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":631,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.206101+0000","last_change":"2023-10-09T06:10:50.744189+0000","last_active":"2023-10-09T06:10:57.206101+0000","last_peered":"2023-10-09T06:10:57.206101+0000","last_clean":"2023-10-09T06:10:57.206101+0000","last_became_active":"2023-10-09T06:10:50.743866+0000","last_became_peered":"2023-10-09T06:10:50.743866+0000","last_unstale":"2023-10-09T06:10:57.206101+0000","last_undegraded":"2023-10-09T06:10:57.206101+0000","last_fullsized":"2023-10-09T06:10:57.206101+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:54.074310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:54.074310+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":347,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.202980+0000","last_change":"2023-10-09T06:09:27.552738+0000","last_active":"2023-10-09T06:10:57.202980+0000","last_peered":"2023-10-09T06:10:57.202980+0000","last_clean":"2023-10-09T06:10:57.202980+0000","last_became_active":"2023-10-09T06:09:27.552471+0000","last_became_peered":"2023-10-09T06:09:27.552471+0000","last_unstale":"2023-10-09T06:10:57.202980+0000","last_undegraded":"2023-10-09T06:10:57.202980+0000","last_fullsized":"2023-10-09T06:10:57.202980+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:15.085267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":178,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.162211+0000","last_change":"2023-10-09T06:07:43.578431+0000","last_active":"2023-10-09T06:10:57.162211+0000","last_peered":"2023-10-09T06:10:57.162211+0000","last_clean":"2023-10-09T06:10:57.162211+0000","last_became_active":"2023-10-09T06:07:43.578110+0000","last_became_peered":"2023-10-09T06:07:43.578110+0000","last_unstale":"2023-10-09T06:10:57.162211+0000","last_undegraded":"2023-10-09T06:10:57.162211+0000","last_fullsized":"2023-10-09T06:10:57.162211+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:18.556487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":268,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.202170+0000","last_change":"2023-10-09T06:09:27.549869+0000","last_active":"2023-10-09T06:10:57.202170+0000","last_peered":"2023-10-09T06:10:57.202170+0000","last_clean":"2023-10-09T06:10:57.202170+0000","last_became_active":"2023-10-09T06:09:27.549551+0000","last_became_peered":"2023-10-09T06:09:27.549551+0000","last_unstale":"2023-10-09T06:10:57.202170+0000","last_undegraded":"2023-10-09T06:10:57.202170+0000","last_fullsized":"2023-10-09T06:10:57.202170+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:49.060007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.d","version":"479'11730","reported_seq":14048,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-10-09T06:11:01.218540+0000","last_change":"2023-10-09T06:10:57.189991+0000","last_active":"2023-10-09T06:11:01.218540+0000","last_peered":"2023-10-09T06:11:01.218540+0000","last_clean":"2023-10-09T06:11:01.218540+0000","last_became_active":"2023-10-09T06:10:57.163000+0000","last_became_peered":"2023-10-09T06:10:57.163000+0000","last_unstale":"2023-10-09T06:11:01.218540+0000","last_undegraded":"2023-10-09T06:11:01.218540+0000","last_fullsized":"2023-10-09T06:11:01.218540+0000","mapping_epoch":477,"log_start":"447'6835","ondisk_log_start":"447'6835","created":425,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4895,"log_dups_size":0,"ondisk_log_size":4895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:41:01.583844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51839000,"num_objects":792,"num_object_clones":0,"num_object_copies":2376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":792,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11730,"num_write_kb":400641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1114,"num_bytes_recovered":72744984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":932,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.158797+0000","last_change":"2023-10-09T06:07:46.590898+0000","last_active":"2023-10-09T06:10:57.158797+0000","last_peered":"2023-10-09T06:10:57.158797+0000","last_clean":"2023-10-09T06:10:57.158797+0000","last_became_active":"2023-10-09T06:07:46.590596+0000","last_became_peered":"2023-10-09T06:07:46.590596+0000","last_unstale":"2023-10-09T06:10:57.158797+0000","last_undegraded":"2023-10-09T06:10:57.158797+0000","last_fullsized":"2023-10-09T06:10:57.158797+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:17.344518+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:17.344518+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:00.399548+0000","scrub_duration":0.0015914099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":521,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-10-09T06:10:59.234791+0000","last_change":"2023-10-09T06:09:28.543659+0000","last_active":"2023-10-09T06:10:59.234791+0000","last_peered":"2023-10-09T06:10:59.234791+0000","last_clean":"2023-10-09T06:10:59.234791+0000","last_became_active":"2023-10-09T06:09:28.543310+0000","last_became_peered":"2023-10-09T06:09:28.543310+0000","last_unstale":"2023-10-09T06:10:59.234791+0000","last_undegraded":"2023-10-09T06:10:59.234791+0000","last_fullsized":"2023-10-09T06:10:59.234791+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:46.713995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":276,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.162305+0000","last_change":"2023-10-09T06:07:06.560512+0000","last_active":"2023-10-09T06:10:57.162305+0000","last_peered":"2023-10-09T06:10:57.162305+0000","last_clean":"2023-10-09T06:10:57.162305+0000","last_became_active":"2023-10-09T06:07:06.560204+0000","last_became_peered":"2023-10-09T06:07:06.560204+0000","last_unstale":"2023-10-09T06:10:57.162305+0000","last_undegraded":"2023-10-09T06:10:57.162305+0000","last_fullsized":"2023-10-09T06:10:57.162305+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:19.102248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":273,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.206564+0000","last_change":"2023-10-09T06:09:28.554885+0000","last_active":"2023-10-09T06:10:57.206564+0000","last_peered":"2023-10-09T06:10:57.206564+0000","last_clean":"2023-10-09T06:10:57.206564+0000","last_became_active":"2023-10-09T06:09:28.554556+0000","last_became_peered":"2023-10-09T06:09:28.554556+0000","last_unstale":"2023-10-09T06:10:57.206564+0000","last_undegraded":"2023-10-09T06:10:57.206564+0000","last_fullsized":"2023-10-09T06:10:57.206564+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:56.537731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"479'11610","reported_seq":11674,"reported_epoch":479,"state":"active+clean","last_fresh":"2023-10-09T06:11:00.112778+0000","last_change":"2023-10-09T06:07:07.580272+0000","last_active":"2023-10-09T06:11:00.112778+0000","last_peered":"2023-10-09T06:11:00.112778+0000","last_clean":"2023-10-09T06:11:00.112778+0000","last_became_active":"2023-10-09T06:07:07.579909+0000","last_became_peered":"2023-10-09T06:07:07.579909+0000","last_unstale":"2023-10-09T06:11:00.112778+0000","last_undegraded":"2023-10-09T06:11:00.112778+0000","last_fullsized":"2023-10-09T06:11:00.112778+0000","mapping_epoch":425,"log_start":"447'6714","ondisk_log_start":"447'6714","created":425,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:07:06.533690+0000","last_clean_scrub_stamp":"2023-10-09T06:07:06.533690+0000","objects_scrubbed":0,"log_size":4896,"log_dups_size":0,"ondisk_log_size":4896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:18.048371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56360960,"num_objects":860,"num_object_clones":0,"num_object_copies":2580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11610,"num_write_kb":399040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":758,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.204473+0000","last_change":"2023-10-09T06:09:06.649781+0000","last_active":"2023-10-09T06:10:57.204473+0000","last_peered":"2023-10-09T06:10:57.204473+0000","last_clean":"2023-10-09T06:10:57.204473+0000","last_became_active":"2023-10-09T06:09:06.649473+0000","last_became_peered":"2023-10-09T06:09:06.649473+0000","last_unstale":"2023-10-09T06:10:57.204473+0000","last_undegraded":"2023-10-09T06:10:57.204473+0000","last_fullsized":"2023-10-09T06:10:57.204473+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:56:17.325138+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":490,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.205240+0000","last_change":"2023-10-09T06:10:50.740875+0000","last_active":"2023-10-09T06:10:57.205240+0000","last_peered":"2023-10-09T06:10:57.205240+0000","last_clean":"2023-10-09T06:10:57.205240+0000","last_became_active":"2023-10-09T06:10:50.740544+0000","last_became_peered":"2023-10-09T06:10:50.740544+0000","last_unstale":"2023-10-09T06:10:57.205240+0000","last_undegraded":"2023-10-09T06:10:57.205240+0000","last_fullsized":"2023-10-09T06:10:57.205240+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:50:36.995570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":523,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.216671+0000","last_change":"2023-10-09T06:09:27.578756+0000","last_active":"2023-10-09T06:10:57.216671+0000","last_peered":"2023-10-09T06:10:57.216671+0000","last_clean":"2023-10-09T06:10:57.216671+0000","last_became_active":"2023-10-09T06:09:27.578384+0000","last_became_peered":"2023-10-09T06:09:27.578384+0000","last_unstale":"2023-10-09T06:10:57.216671+0000","last_undegraded":"2023-10-09T06:10:57.216671+0000","last_fullsized":"2023-10-09T06:10:57.216671+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:22.182630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":425,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.158260+0000","last_change":"2023-10-09T06:01:22.334081+0000","last_active":"2023-10-09T06:10:57.158260+0000","last_peered":"2023-10-09T06:10:57.158260+0000","last_clean":"2023-10-09T06:10:57.158260+0000","last_became_active":"2023-10-09T06:01:22.333663+0000","last_became_peered":"2023-10-09T06:01:22.333663+0000","last_unstale":"2023-10-09T06:10:57.158260+0000","last_undegraded":"2023-10-09T06:10:57.158260+0000","last_fullsized":"2023-10-09T06:10:57.158260+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:55.809259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":723,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.204138+0000","last_change":"2023-10-09T06:09:27.560157+0000","last_active":"2023-10-09T06:10:57.204138+0000","last_peered":"2023-10-09T06:10:57.204138+0000","last_clean":"2023-10-09T06:10:57.204138+0000","last_became_active":"2023-10-09T06:09:27.559466+0000","last_became_peered":"2023-10-09T06:09:27.559466+0000","last_unstale":"2023-10-09T06:10:57.204138+0000","last_undegraded":"2023-10-09T06:10:57.204138+0000","last_fullsized":"2023-10-09T06:10:57.204138+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.181811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.181811+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:53.431491+0000","scrub_duration":0.011553313000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":388,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.217673+0000","last_change":"2023-10-09T06:09:27.558538+0000","last_active":"2023-10-09T06:10:57.217673+0000","last_peered":"2023-10-09T06:10:57.217673+0000","last_clean":"2023-10-09T06:10:57.217673+0000","last_became_active":"2023-10-09T06:09:27.557685+0000","last_became_peered":"2023-10-09T06:09:27.557685+0000","last_unstale":"2023-10-09T06:10:57.217673+0000","last_undegraded":"2023-10-09T06:10:57.217673+0000","last_fullsized":"2023-10-09T06:10:57.217673+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:50.886632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":179,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.162285+0000","last_change":"2023-10-09T06:07:43.580844+0000","last_active":"2023-10-09T06:10:57.162285+0000","last_peered":"2023-10-09T06:10:57.162285+0000","last_clean":"2023-10-09T06:10:57.162285+0000","last_became_active":"2023-10-09T06:07:43.580523+0000","last_became_peered":"2023-10-09T06:07:43.580523+0000","last_unstale":"2023-10-09T06:10:57.162285+0000","last_undegraded":"2023-10-09T06:10:57.162285+0000","last_fullsized":"2023-10-09T06:10:57.162285+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:55.942430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":826,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.205094+0000","last_change":"2023-10-09T06:00:04.215425+0000","last_active":"2023-10-09T06:10:57.205094+0000","last_peered":"2023-10-09T06:10:57.205094+0000","last_clean":"2023-10-09T06:10:57.205094+0000","last_became_active":"2023-10-09T05:59:40.326685+0000","last_became_peered":"2023-10-09T05:59:40.326685+0000","last_unstale":"2023-10-09T06:10:57.205094+0000","last_undegraded":"2023-10-09T06:10:57.205094+0000","last_fullsized":"2023-10-09T06:10:57.205094+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:04.215270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:04.215270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:03.963694+0000","scrub_duration":0.002850652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":250,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.158242+0000","last_change":"2023-10-09T06:01:54.824780+0000","last_active":"2023-10-09T06:10:57.158242+0000","last_peered":"2023-10-09T06:10:57.158242+0000","last_clean":"2023-10-09T06:10:57.158242+0000","last_became_active":"2023-10-09T06:01:54.757727+0000","last_became_peered":"2023-10-09T06:01:54.757727+0000","last_unstale":"2023-10-09T06:10:57.158242+0000","last_undegraded":"2023-10-09T06:10:57.158242+0000","last_fullsized":"2023-10-09T06:10:57.158242+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:44.132556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":497,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.158547+0000","last_change":"2023-10-09T06:01:54.764989+0000","last_active":"2023-10-09T06:10:57.158547+0000","last_peered":"2023-10-09T06:10:57.158547+0000","last_clean":"2023-10-09T06:10:57.158547+0000","last_became_active":"2023-10-09T06:01:54.764790+0000","last_became_peered":"2023-10-09T06:01:54.764790+0000","last_unstale":"2023-10-09T06:10:57.158547+0000","last_undegraded":"2023-10-09T06:10:57.158547+0000","last_fullsized":"2023-10-09T06:10:57.158547+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:14:07.208858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":250,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.158072+0000","last_change":"2023-10-09T06:10:26.682338+0000","last_active":"2023-10-09T06:10:57.158072+0000","last_peered":"2023-10-09T06:10:57.158072+0000","last_clean":"2023-10-09T06:10:57.158072+0000","last_became_active":"2023-10-09T06:10:26.646411+0000","last_became_peered":"2023-10-09T06:10:26.646411+0000","last_unstale":"2023-10-09T06:10:57.158072+0000","last_undegraded":"2023-10-09T06:10:57.158072+0000","last_fullsized":"2023-10-09T06:10:57.158072+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:10.789202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":494,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.158904+0000","last_change":"2023-10-09T06:09:27.565198+0000","last_active":"2023-10-09T06:10:57.158904+0000","last_peered":"2023-10-09T06:10:57.158904+0000","last_clean":"2023-10-09T06:10:57.158904+0000","last_became_active":"2023-10-09T06:09:27.564909+0000","last_became_peered":"2023-10-09T06:09:27.564909+0000","last_unstale":"2023-10-09T06:10:57.158904+0000","last_undegraded":"2023-10-09T06:10:57.158904+0000","last_fullsized":"2023-10-09T06:10:57.158904+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:46.101014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":599,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.161848+0000","last_change":"2023-10-09T06:10:50.741337+0000","last_active":"2023-10-09T06:10:57.161848+0000","last_peered":"2023-10-09T06:10:57.161848+0000","last_clean":"2023-10-09T06:10:57.161848+0000","last_became_active":"2023-10-09T06:10:50.740982+0000","last_became_peered":"2023-10-09T06:10:50.740982+0000","last_unstale":"2023-10-09T06:10:57.161848+0000","last_undegraded":"2023-10-09T06:10:57.161848+0000","last_fullsized":"2023-10-09T06:10:57.161848+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:00.456132+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:00.456132+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:48:40.422236+0000","scrub_duration":0.0014992199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":274,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.205956+0000","last_change":"2023-10-09T06:07:43.579543+0000","last_active":"2023-10-09T06:10:57.205956+0000","last_peered":"2023-10-09T06:10:57.205956+0000","last_clean":"2023-10-09T06:10:57.205956+0000","last_became_active":"2023-10-09T06:07:43.579184+0000","last_became_peered":"2023-10-09T06:07:43.579184+0000","last_unstale":"2023-10-09T06:10:57.205956+0000","last_undegraded":"2023-10-09T06:10:57.205956+0000","last_fullsized":"2023-10-09T06:10:57.205956+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:23.460469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":279,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.204463+0000","last_change":"2023-10-09T06:09:27.565185+0000","last_active":"2023-10-09T06:10:57.204463+0000","last_peered":"2023-10-09T06:10:57.204463+0000","last_clean":"2023-10-09T06:10:57.204463+0000","last_became_active":"2023-10-09T06:09:27.564478+0000","last_became_peered":"2023-10-09T06:09:27.564478+0000","last_unstale":"2023-10-09T06:10:57.204463+0000","last_undegraded":"2023-10-09T06:10:57.204463+0000","last_fullsized":"2023-10-09T06:10:57.204463+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:04.622171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":767,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.161532+0000","last_change":"2023-10-09T06:07:06.560275+0000","last_active":"2023-10-09T06:10:57.161532+0000","last_peered":"2023-10-09T06:10:57.161532+0000","last_clean":"2023-10-09T06:10:57.161532+0000","last_became_active":"2023-10-09T06:07:06.559953+0000","last_became_peered":"2023-10-09T06:07:06.559953+0000","last_unstale":"2023-10-09T06:10:57.161532+0000","last_undegraded":"2023-10-09T06:10:57.161532+0000","last_fullsized":"2023-10-09T06:10:57.161532+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:46.290510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:46.290510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:10.910416+0000","scrub_duration":0.002627819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":469,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.215575+0000","last_change":"2023-10-09T06:09:27.556271+0000","last_active":"2023-10-09T06:10:57.215575+0000","last_peered":"2023-10-09T06:10:57.215575+0000","last_clean":"2023-10-09T06:10:57.215575+0000","last_became_active":"2023-10-09T06:09:27.555932+0000","last_became_peered":"2023-10-09T06:09:27.555932+0000","last_unstale":"2023-10-09T06:10:57.215575+0000","last_undegraded":"2023-10-09T06:10:57.215575+0000","last_fullsized":"2023-10-09T06:10:57.215575+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:33.888622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":500,"reported_epoch":478,"state":"active+clean","last_fresh":"2023-10-09T06:10:57.158119+0000","last_change":"2023-10-09T06:06:01.068828+0000","last_active":"2023-10-09T06:10:57.158119+0000","last_peered":"2023-10-09T06:10:57.158119+0000","last_clean":"2023-10-09T06:10:57.158119+0000","last_became_active":"2023-10-09T06:06:01.068491+0000","last_became_peered":"2023-10-09T06:06:01.068491+0000","last_unstale":"2023-10-09T06:10:57.158119+0000","last_undegraded":"2023-10-09T06:10:57.158119+0000","last_fullsized":"2023-10-09T06:10:57.158119+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:25.661796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":2128,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13410,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":737958,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":122,"num_read_kb":227,"num_write":58,"num_write_kb":603,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":968150,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":906756120,"num_objects":13837,"num_object_clones":0,"num_object_copies":41511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6593,"num_objects_misplaced":7054,"num_objects_unfound":0,"num_objects_dirty":13837,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":188363,"num_write_kb":6470337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26966,"num_bytes_recovered":1528561688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3596107776,"data_stored":3596091488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":88005,"ondisk_log_size":88005,"up":48,"acting":44,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234376,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1456760,"kb_used_data":564144,"kb_used_omap":0,"kb_used_meta":892608,"kb_avail":92288392,"statfs":{"total":95995035648,"available":94503313408,"internally_reserved":0,"allocated":577683456,"data_stored":576287405,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":914030592},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:10:28 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":3.5470000000000002,"5min":3.5470000000000002,"15min":3.5470000000000002},"last":0.26400000000000001},{"interface":"front","average":{"1min":1.145,"5min":1.145,"15min":1.145},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":4.1239999999999997,"5min":4.1239999999999997,"15min":4.1239999999999997},"last":0.36499999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:10:57 2023","interfaces":[{"interface":"back","average":{"1min":6.3479999999999999,"5min":4.1680000000000001,"15min":3.214},"min":{"1min":0.255,"5min":0.218,"15min":0.218},"max":{"1min":41.600999999999999,"5min":124.66200000000001,"15min":124.66200000000001},"last":0.73299999999999998},{"interface":"front","average":{"1min":7.2999999999999998,"5min":4.2149999999999999,"15min":3.1480000000000001},"min":{"1min":0.34100000000000003,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":41.720999999999997,"5min":125.023,"15min":125.023},"last":0.53700000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:10:24 2023","interfaces":[{"interface":"back","average":{"1min":1.4219999999999999,"5min":1.8600000000000001,"15min":2.8500000000000001},"min":{"1min":0.29799999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.1799999999999997,"5min":16.878,"15min":127.86499999999999},"last":0.436},{"interface":"front","average":{"1min":1.8480000000000001,"5min":1.6799999999999999,"15min":2.843},"min":{"1min":0.34300000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.002000000000001,"5min":10.002000000000001,"15min":129.24000000000001},"last":10.680999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:10:57 2023","interfaces":[{"interface":"back","average":{"1min":14.962,"5min":5.6920000000000002,"15min":6.1079999999999997},"min":{"1min":0.38600000000000001,"5min":0.32100000000000001,"15min":0.29899999999999999},"max":{"1min":116.68000000000001,"5min":116.68000000000001,"15min":164.16999999999999},"last":9.0440000000000005},{"interface":"front","average":{"1min":15.933999999999999,"5min":6.1420000000000003,"15min":6.2539999999999996},"min":{"1min":0.33300000000000002,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":116.797,"5min":116.797,"15min":164.40700000000001},"last":23.776}]},{"osd":4,"last update":"Mon Oct 9 06:10:34 2023","interfaces":[{"interface":"back","average":{"1min":1.22,"5min":1.274,"15min":0.92500000000000004},"min":{"1min":0.23999999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.2080000000000002,"5min":14.371,"15min":14.371},"last":0.29799999999999999},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.242,"15min":0.94299999999999995},"min":{"1min":0.28599999999999998,"5min":0.27700000000000002,"15min":0.27100000000000002},"max":{"1min":3.016,"5min":14.259,"15min":14.259},"last":0.42399999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:10:42 2023","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":1.048,"15min":0.97199999999999998},"min":{"1min":0.31,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.0899999999999999,"5min":9.7910000000000004,"15min":15.622},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.79900000000000004,"5min":1.111,"15min":0.95099999999999996},"min":{"1min":0.32300000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.8599999999999999,"5min":9.3409999999999993,"15min":9.3409999999999993},"last":0.45300000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:10:42 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":1.175,"15min":1.018},"min":{"1min":0.34399999999999997,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.1850000000000001,"5min":9.4540000000000006,"15min":12.727},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.71999999999999997,"5min":1.28,"15min":1.004},"min":{"1min":0.379,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.206,"5min":6.2329999999999997,"15min":6.2329999999999997},"last":0.40200000000000002}]}]},{"osd":6,"up_from":46,"seq":197568496012,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1295572,"kb_used_data":389836,"kb_used_omap":0,"kb_used_meta":905728,"kb_avail":92449580,"statfs":{"total":95995035648,"available":94668369920,"internally_reserved":0,"allocated":399192064,"data_stored":397823520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":927465472},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:10:27 2023","interfaces":[{"interface":"back","average":{"1min":2.2429999999999999,"5min":2.2429999999999999,"15min":2.2429999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":11.73,"5min":11.73,"15min":11.73},"last":0.629},{"interface":"front","average":{"1min":1.9890000000000001,"5min":1.9890000000000001,"15min":1.9890000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.8819999999999997,"5min":5.8819999999999997,"15min":5.8819999999999997},"last":0.62}]},{"osd":1,"last update":"Mon Oct 9 06:10:13 2023","interfaces":[{"interface":"back","average":{"1min":1.55,"5min":6.2809999999999997,"15min":3.5350000000000001},"min":{"1min":0.26200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.335,"5min":109.101,"15min":109.101},"last":3.3420000000000001},{"interface":"front","average":{"1min":2.5489999999999999,"5min":6.4089999999999998,"15min":3.6379999999999999},"min":{"1min":0.29799999999999999,"5min":0.27400000000000002,"15min":0.27000000000000002},"max":{"1min":10.26,"5min":109.071,"15min":109.071},"last":3.1549999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:10:17 2023","interfaces":[{"interface":"back","average":{"1min":2.073,"5min":2.0049999999999999,"15min":2.2400000000000002},"min":{"1min":0.313,"5min":0.27300000000000002,"15min":0.26200000000000001},"max":{"1min":5.9459999999999997,"5min":15.374000000000001,"15min":61.088999999999999},"last":5.4409999999999998},{"interface":"front","average":{"1min":2.6080000000000001,"5min":1.8700000000000001,"15min":2.2000000000000002},"min":{"1min":0.312,"5min":0.29199999999999998,"15min":0.28599999999999998},"max":{"1min":9.2639999999999993,"5min":15.429,"15min":61.082000000000001},"last":0.47099999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:10:08 2023","interfaces":[{"interface":"back","average":{"1min":1.8520000000000001,"5min":7.5570000000000004,"15min":6.0800000000000001},"min":{"1min":0.32700000000000001,"5min":0.318,"15min":0.29599999999999999},"max":{"1min":8.1669999999999998,"5min":138.13300000000001,"15min":138.13300000000001},"last":13.425000000000001},{"interface":"front","average":{"1min":1.7050000000000001,"5min":7.2779999999999996,"15min":5.9850000000000003},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.28499999999999998},"max":{"1min":9.2889999999999997,"5min":137.215,"15min":137.215},"last":0.379}]},{"osd":4,"last update":"Mon Oct 9 06:10:37 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":2.6480000000000001,"15min":1.738},"min":{"1min":0.28399999999999997,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":6.9299999999999997,"5min":151.35599999999999,"15min":151.35599999999999},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.97999999999999998,"5min":2.895,"15min":1.7190000000000001},"min":{"1min":0.249,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":4.5940000000000003,"5min":149.15799999999999,"15min":149.15799999999999},"last":0.35999999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:10:13 2023","interfaces":[{"interface":"back","average":{"1min":1.5529999999999999,"5min":1.409,"15min":1.294},"min":{"1min":0.34499999999999997,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":9.1959999999999997,"5min":15.342000000000001,"15min":45.933999999999997},"last":0.434},{"interface":"front","average":{"1min":1.165,"5min":1.3680000000000001,"15min":1.2869999999999999},"min":{"1min":0.318,"5min":0.28199999999999997,"15min":0.255},"max":{"1min":4.4660000000000002,"5min":15.358000000000001,"15min":45.917999999999999},"last":0.42099999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":1.4790000000000001,"15min":1.3879999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.27000000000000002},"max":{"1min":2.706,"5min":21.98,"15min":52.042999999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.85199999999999998,"5min":1.3700000000000001,"15min":1.3540000000000001},"min":{"1min":0.378,"5min":0.33300000000000002,"15min":0.32200000000000001},"max":{"1min":3.0150000000000001,"5min":18.241,"15min":51.259},"last":0.53700000000000003}]}]},{"osd":1,"up_from":89,"seq":382252089601,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1085776,"kb_used_data":533064,"kb_used_omap":9,"kb_used_meta":552502,"kb_avail":92659376,"statfs":{"total":95995035648,"available":94883201024,"internally_reserved":0,"allocated":545857536,"data_stored":543090141,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":565763049},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:10:30 2023","interfaces":[{"interface":"back","average":{"1min":2.2759999999999998,"5min":2.2759999999999998,"15min":2.2759999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":9.7379999999999995,"5min":9.7379999999999995,"15min":9.7379999999999995},"last":1.345},{"interface":"front","average":{"1min":2.3399999999999999,"5min":2.3399999999999999,"15min":2.3399999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":9.3390000000000004,"5min":9.3390000000000004,"15min":9.3390000000000004},"last":7.4089999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:10:37 2023","interfaces":[{"interface":"back","average":{"1min":1.9630000000000001,"5min":1.8480000000000001,"15min":3.4809999999999999},"min":{"1min":0.246,"5min":0.21099999999999999,"15min":0.20799999999999999},"max":{"1min":9.7729999999999997,"5min":17.681000000000001,"15min":204.989},"last":1.3720000000000001},{"interface":"front","average":{"1min":3.9900000000000002,"5min":2.145,"15min":3.6859999999999999},"min":{"1min":0.42099999999999999,"5min":0.22700000000000001,"15min":0.20300000000000001},"max":{"1min":11.929,"5min":13.539999999999999,"15min":203.65899999999999},"last":7.3799999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:09:58 2023","interfaces":[{"interface":"back","average":{"1min":2.6469999999999998,"5min":4.1310000000000002,"15min":4.5499999999999998},"min":{"1min":0.29099999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":9.8409999999999993,"5min":142.97,"15min":213.654},"last":1.3939999999999999},{"interface":"front","average":{"1min":2.8500000000000001,"5min":4.0110000000000001,"15min":4.508},"min":{"1min":0.32300000000000001,"5min":0.27600000000000002,"15min":0.27400000000000002},"max":{"1min":9.1799999999999997,"5min":142.94800000000001,"15min":213.77199999999999},"last":9.5350000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:10:37 2023","interfaces":[{"interface":"back","average":{"1min":1.7170000000000001,"5min":2.1269999999999998,"15min":1.542},"min":{"1min":0.34000000000000002,"5min":0.31,"15min":0.30399999999999999},"max":{"1min":9.2370000000000001,"5min":35.366,"15min":35.366},"last":7.492},{"interface":"front","average":{"1min":4.343,"5min":2.9260000000000002,"15min":2.0739999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":15.33,"5min":35.384999999999998,"15min":35.384999999999998},"last":0.38700000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:10:47 2023","interfaces":[{"interface":"back","average":{"1min":2.8359999999999999,"5min":1.8069999999999999,"15min":1.74},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.3109999999999999,"5min":10.750999999999999,"15min":64.584999999999994},"last":7.3499999999999996},{"interface":"front","average":{"1min":5.1859999999999999,"5min":2.9529999999999998,"15min":2.351},"min":{"1min":0.68600000000000005,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":17.603999999999999,"5min":17.603999999999999,"15min":64.150999999999996},"last":4.4210000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:10:47 2023","interfaces":[{"interface":"back","average":{"1min":3.2280000000000002,"5min":2.6970000000000001,"15min":1.7749999999999999},"min":{"1min":0.374,"5min":0.34300000000000003,"15min":0.30099999999999999},"max":{"1min":9.282,"5min":52.207000000000001,"15min":52.207000000000001},"last":7.3040000000000003},{"interface":"front","average":{"1min":4.3040000000000003,"5min":3.3610000000000002,"15min":2.266},"min":{"1min":0.436,"5min":0.39100000000000001,"15min":0.35499999999999998},"max":{"1min":15.384,"5min":51.212000000000003,"15min":51.212000000000003},"last":0.34999999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:10:47 2023","interfaces":[{"interface":"back","average":{"1min":5.0119999999999996,"5min":2.9620000000000002,"15min":2.762},"min":{"1min":0.51200000000000001,"5min":0.42999999999999999,"15min":0.314},"max":{"1min":15.284000000000001,"5min":15.284000000000001,"15min":108.553},"last":9.4009999999999998},{"interface":"front","average":{"1min":1.996,"5min":2.1269999999999998,"15min":2.258},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":6.4790000000000001,"5min":11.869,"15min":108.56999999999999},"last":7.7149999999999999}]}]},{"osd":0,"up_from":456,"seq":1958505086996,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":865952,"kb_used_data":738520,"kb_used_omap":32,"kb_used_meta":127391,"kb_avail":92879200,"statfs":{"total":95995035648,"available":95108300800,"internally_reserved":0,"allocated":756244480,"data_stored":753700822,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32794,"internal_metadata":130449382},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:10:31 2023","interfaces":[{"interface":"back","average":{"1min":11.134,"5min":11.134,"15min":11.134},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":166.833,"5min":166.833,"15min":166.833},"last":0.93100000000000005},{"interface":"front","average":{"1min":9.9920000000000009,"5min":9.9920000000000009,"15min":9.9920000000000009},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":166.81100000000001,"5min":166.81100000000001,"15min":166.81100000000001},"last":0.38}]},{"osd":2,"last update":"Mon Oct 9 06:10:31 2023","interfaces":[{"interface":"back","average":{"1min":4.0590000000000002,"5min":4.0590000000000002,"15min":4.0590000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":14.52,"5min":14.52,"15min":14.52},"last":0.36599999999999999},{"interface":"front","average":{"1min":3.903,"5min":3.903,"15min":3.903},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":20.152000000000001,"5min":20.152000000000001,"15min":20.152000000000001},"last":4.266}]},{"osd":3,"last update":"Mon Oct 9 06:10:26 2023","interfaces":[{"interface":"back","average":{"1min":15.923999999999999,"5min":15.923999999999999,"15min":15.923999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":203.59299999999999,"5min":203.59299999999999,"15min":203.59299999999999},"last":4.2199999999999998},{"interface":"front","average":{"1min":15.02,"5min":15.02,"15min":15.02},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":202.96600000000001,"5min":202.96600000000001,"15min":202.96600000000001},"last":4.0800000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:10:31 2023","interfaces":[{"interface":"back","average":{"1min":2.4089999999999998,"5min":2.4089999999999998,"15min":2.4089999999999998},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":10.19,"5min":10.19,"15min":10.19},"last":4.1950000000000003},{"interface":"front","average":{"1min":1.9930000000000001,"5min":1.9930000000000001,"15min":1.9930000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":8.8559999999999999,"5min":8.8559999999999999,"15min":8.8559999999999999},"last":0.55400000000000005}]},{"osd":5,"last update":"Mon Oct 9 06:10:31 2023","interfaces":[{"interface":"back","average":{"1min":2.1899999999999999,"5min":2.1899999999999999,"15min":2.1899999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":7.7350000000000003,"5min":7.7350000000000003,"15min":7.7350000000000003},"last":161.286},{"interface":"front","average":{"1min":2.4740000000000002,"5min":2.4740000000000002,"15min":2.4740000000000002},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":12.122,"5min":12.122,"15min":12.122},"last":161.31800000000001}]},{"osd":6,"last update":"Mon Oct 9 06:10:31 2023","interfaces":[{"interface":"back","average":{"1min":2.4769999999999999,"5min":2.4769999999999999,"15min":2.4769999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":12.244,"5min":12.244,"15min":12.244},"last":0.44500000000000001},{"interface":"front","average":{"1min":2.8380000000000001,"5min":2.8380000000000001,"15min":2.8380000000000001},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":10.233000000000001,"5min":10.233000000000001,"15min":10.233000000000001},"last":4.1550000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:10:31 2023","interfaces":[{"interface":"back","average":{"1min":2.5329999999999999,"5min":2.5329999999999999,"15min":2.5329999999999999},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":12.217000000000001,"5min":12.217000000000001,"15min":12.217000000000001},"last":0.41099999999999998},{"interface":"front","average":{"1min":2.7629999999999999,"5min":2.7629999999999999,"15min":2.7629999999999999},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":10.263,"5min":10.263,"15min":10.263},"last":4.2430000000000003}]}]},{"osd":2,"up_from":94,"seq":403726926093,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":846296,"kb_used_data":161616,"kb_used_omap":81,"kb_used_meta":684590,"kb_avail":92898856,"statfs":{"total":95995035648,"available":95128428544,"internally_reserved":0,"allocated":165494784,"data_stored":163073203,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":701020378},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:10:30 2023","interfaces":[{"interface":"back","average":{"1min":3.149,"5min":3.149,"15min":3.149},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":10.146000000000001,"5min":10.146000000000001,"15min":10.146000000000001},"last":0.311},{"interface":"front","average":{"1min":2.2429999999999999,"5min":2.2429999999999999,"15min":2.2429999999999999},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":6.7939999999999996,"5min":6.7939999999999996,"15min":6.7939999999999996},"last":3.2149999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:10:30 2023","interfaces":[{"interface":"back","average":{"1min":8.3960000000000008,"5min":4.6639999999999997,"15min":3.7610000000000001},"min":{"1min":0.318,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":64.475999999999999,"5min":66.665999999999997,"15min":222.59399999999999},"last":8.3420000000000005},{"interface":"front","average":{"1min":11.33,"5min":5.4909999999999997,"15min":4.1029999999999998},"min":{"1min":0.27600000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":61.838000000000001,"5min":66.638999999999996,"15min":224.619},"last":7.3620000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:10:55 2023","interfaces":[{"interface":"back","average":{"1min":12.388999999999999,"5min":5.4580000000000002,"15min":4.5060000000000002},"min":{"1min":0.28499999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":115.742,"5min":115.742,"15min":144.88200000000001},"last":0.35599999999999998},{"interface":"front","average":{"1min":12.180999999999999,"5min":5.3890000000000002,"15min":4.5149999999999997},"min":{"1min":0.312,"5min":0.28799999999999998,"15min":0.25900000000000001},"max":{"1min":115.55,"5min":115.55,"15min":140.79499999999999},"last":3.3399999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:10:37 2023","interfaces":[{"interface":"back","average":{"1min":9.6240000000000006,"5min":3.2490000000000001,"15min":2.2509999999999999},"min":{"1min":0.32500000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":107.51300000000001,"5min":107.51300000000001,"15min":107.51300000000001},"last":3.032},{"interface":"front","average":{"1min":6.8319999999999999,"5min":2.7210000000000001,"15min":2.1850000000000001},"min":{"1min":0.38700000000000001,"5min":0.32000000000000001,"15min":0.31},"max":{"1min":108.768,"5min":108.768,"15min":108.768},"last":1.944}]},{"osd":5,"last update":"Mon Oct 9 06:10:30 2023","interfaces":[{"interface":"back","average":{"1min":1.9430000000000001,"5min":1.7609999999999999,"15min":1.607},"min":{"1min":0.34599999999999997,"5min":0.32600000000000001,"15min":0.30599999999999999},"max":{"1min":7.6790000000000003,"5min":28.48,"15min":28.48},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.931,"5min":2.105,"15min":1.734},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.28899999999999998},"max":{"1min":9.8699999999999992,"5min":30.318999999999999,"15min":30.318999999999999},"last":1.75}]},{"osd":6,"last update":"Mon Oct 9 06:10:30 2023","interfaces":[{"interface":"back","average":{"1min":5.0259999999999998,"5min":2.2559999999999998,"15min":2.008},"min":{"1min":0.41199999999999998,"5min":0.314,"15min":0.314},"max":{"1min":14.962,"5min":14.962,"15min":30.420000000000002},"last":3.0760000000000001},{"interface":"front","average":{"1min":1.948,"5min":1.6859999999999999,"15min":1.768},"min":{"1min":0.378,"5min":0.378,"15min":0.32200000000000001},"max":{"1min":7.6459999999999999,"5min":7.6459999999999999,"15min":30.713000000000001},"last":0.432}]},{"osd":7,"last update":"Mon Oct 9 06:10:30 2023","interfaces":[{"interface":"back","average":{"1min":7.9770000000000003,"5min":2.992,"15min":2.4569999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":124.069,"5min":124.069,"15min":124.069},"last":0.50800000000000001},{"interface":"front","average":{"1min":7.4450000000000003,"5min":2.988,"15min":2.3010000000000002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":124.08799999999999,"5min":124.08799999999999,"15min":124.08799999999999},"last":1.667}]}]},{"osd":3,"up_from":99,"seq":425201762566,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1327132,"kb_used_data":548564,"kb_used_omap":7,"kb_used_meta":778424,"kb_avail":92418020,"statfs":{"total":95995035648,"available":94636052480,"internally_reserved":0,"allocated":561729536,"data_stored":559004826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":797106792},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:10:28 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":2.109,"15min":2.109},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.5579999999999998,"5min":5.5579999999999998,"15min":5.5579999999999998},"last":0.17299999999999999},{"interface":"front","average":{"1min":3.5259999999999998,"5min":3.5259999999999998,"15min":3.5259999999999998},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":14.321,"5min":14.321,"15min":14.321},"last":1.762}]},{"osd":1,"last update":"Mon Oct 9 06:09:59 2023","interfaces":[{"interface":"back","average":{"1min":3.2229999999999999,"5min":4.867,"15min":2.7440000000000002},"min":{"1min":0.222,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":22.579999999999998,"5min":207.75299999999999,"15min":207.75299999999999},"last":0.19800000000000001},{"interface":"front","average":{"1min":5.5960000000000001,"5min":5.5350000000000001,"15min":3.3969999999999998},"min":{"1min":0.32100000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":22.558,"5min":207.12200000000001,"15min":207.12200000000001},"last":5.548}]},{"osd":2,"last update":"Mon Oct 9 06:09:59 2023","interfaces":[{"interface":"back","average":{"1min":3.4329999999999998,"5min":2.0569999999999999,"15min":2.4929999999999999},"min":{"1min":0.30399999999999999,"5min":0.23200000000000001,"15min":0.223},"max":{"1min":14.157,"5min":14.157,"15min":107.01300000000001},"last":1.9119999999999999},{"interface":"front","average":{"1min":4.7480000000000002,"5min":2.7040000000000002,"15min":2.895},"min":{"1min":0.36499999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":11.561,"5min":14.739000000000001,"15min":109.807},"last":2.3300000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:10:39 2023","interfaces":[{"interface":"back","average":{"1min":2.875,"5min":1.847,"15min":4.2869999999999999},"min":{"1min":0.375,"5min":0.312,"15min":0.254},"max":{"1min":14.211,"5min":14.211,"15min":99.876000000000005},"last":1.827},{"interface":"front","average":{"1min":3.1699999999999999,"5min":2.3370000000000002,"15min":4.3209999999999997},"min":{"1min":0.33800000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":9.0519999999999996,"5min":10.255000000000001,"15min":99.914000000000001},"last":2.3719999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:09:59 2023","interfaces":[{"interface":"back","average":{"1min":1.292,"5min":1.429,"15min":1.627},"min":{"1min":0.48599999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":2.6739999999999999,"5min":8.7100000000000009,"15min":27.937000000000001},"last":0.24199999999999999},{"interface":"front","average":{"1min":2.5369999999999999,"5min":1.8819999999999999,"15min":2.6070000000000002},"min":{"1min":0.32500000000000001,"5min":0.28799999999999998,"15min":0.28199999999999997},"max":{"1min":14.241,"5min":14.241,"15min":235.56},"last":1.857}]},{"osd":6,"last update":"Mon Oct 9 06:09:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4870000000000001,"5min":1.474,"15min":4.4550000000000001},"min":{"1min":0.45500000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":2.8809999999999998,"5min":8.6809999999999992,"15min":233.26599999999999},"last":0.26100000000000001},{"interface":"front","average":{"1min":2.73,"5min":1.798,"15min":4.6449999999999996},"min":{"1min":0.39000000000000001,"5min":0.33600000000000002,"15min":0.318},"max":{"1min":14.292999999999999,"5min":14.292999999999999,"15min":233.24100000000001},"last":1.8859999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:09:59 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":2.5150000000000001,"15min":4.306},"min":{"1min":0.41199999999999998,"5min":0.35699999999999998,"15min":0.34200000000000003},"max":{"1min":14.266999999999999,"5min":51.323999999999998,"15min":230.69399999999999},"last":1.96},{"interface":"front","average":{"1min":3.7200000000000002,"5min":2.8330000000000002,"15min":4.4349999999999996},"min":{"1min":0.496,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":8.7799999999999994,"5min":51.468000000000004,"15min":230.67500000000001},"last":2.3969999999999998}]}]},{"osd":4,"up_from":322,"seq":1382979469429,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":586036,"kb_used_data":263660,"kb_used_omap":35,"kb_used_meta":322332,"kb_avail":93159116,"statfs":{"total":95995035648,"available":95394934784,"internally_reserved":0,"allocated":269987840,"data_stored":268595594,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36552,"internal_metadata":330068280},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:10:29 2023","interfaces":[{"interface":"back","average":{"1min":1.488,"5min":1.488,"15min":1.488},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":6.7409999999999997,"5min":6.7409999999999997,"15min":6.7409999999999997},"last":0.26600000000000001},{"interface":"front","average":{"1min":1.597,"5min":1.597,"15min":1.597},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":7.7999999999999998,"5min":7.7999999999999998,"15min":7.7999999999999998},"last":0.36899999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":1.921,"5min":1.4399999999999999,"15min":2.2210000000000001},"min":{"1min":0.32900000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":6.4160000000000004,"5min":8.4280000000000008,"15min":78.891000000000005},"last":0.441},{"interface":"front","average":{"1min":2.1600000000000001,"5min":1.8839999999999999,"15min":2.2069999999999999},"min":{"1min":0.317,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":6.2450000000000001,"5min":11.231999999999999,"15min":78.921000000000006},"last":3.0550000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":2.7770000000000001,"5min":1.966,"15min":1.8420000000000001},"min":{"1min":0.311,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":10.121,"5min":10.121,"15min":10.121},"last":1.708},{"interface":"front","average":{"1min":1.847,"5min":1.5309999999999999,"15min":1.8140000000000001},"min":{"1min":0.311,"5min":0.29399999999999998,"15min":0.28199999999999997},"max":{"1min":7.6669999999999998,"5min":7.6669999999999998,"15min":11.381},"last":0.40400000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":2.1440000000000001,"5min":2.1749999999999998,"15min":2.391},"min":{"1min":0.37,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":8.2780000000000005,"5min":10.217000000000001,"15min":58.398000000000003},"last":1.119},{"interface":"front","average":{"1min":2.0899999999999999,"5min":1.9670000000000001,"15min":2.0600000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":10.331,"5min":10.331,"15min":58.305999999999997},"last":0.34000000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":1.1459999999999999,"15min":1.0049999999999999},"min":{"1min":0.27500000000000002,"5min":0.216,"15min":0.216},"max":{"1min":7.5999999999999996,"5min":9.2690000000000001,"15min":9.2690000000000001},"last":0.307},{"interface":"front","average":{"1min":0.85199999999999998,"5min":0.92900000000000005,"15min":1.244},"min":{"1min":0.29499999999999998,"5min":0.245,"15min":0.245},"max":{"1min":2.4260000000000002,"5min":5.5830000000000002,"15min":12.087},"last":0.41199999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":1.3899999999999999,"5min":2.2240000000000002,"15min":2.9689999999999999},"min":{"1min":0.34100000000000003,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":7.641,"5min":95.492999999999995,"15min":95.492999999999995},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.099,"5min":2.1890000000000001,"15min":3.0880000000000001},"min":{"1min":0.33100000000000002,"5min":0.312,"15min":0.27900000000000003},"max":{"1min":3.6640000000000001,"5min":95.664000000000001,"15min":95.664000000000001},"last":0.38700000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:10:36 2023","interfaces":[{"interface":"back","average":{"1min":1.8460000000000001,"5min":1.4039999999999999,"15min":3.7349999999999999},"min":{"1min":0.35699999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":17.286000000000001,"5min":17.286000000000001,"15min":79.173000000000002},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.657,"5min":1.413,"15min":3.1600000000000001},"min":{"1min":0.33700000000000002,"5min":0.251,"15min":0.251},"max":{"1min":16.097999999999999,"5min":16.097999999999999,"15min":78.894000000000005},"last":0.55300000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757649,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1305636,"kb_used_data":415324,"kb_used_omap":0,"kb_used_meta":890304,"kb_avail":92439516,"statfs":{"total":95995035648,"available":94658064384,"internally_reserved":0,"allocated":425291776,"data_stored":423912132,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911671296},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:10:28 2023","interfaces":[{"interface":"back","average":{"1min":2.4020000000000001,"5min":2.4020000000000001,"15min":2.4020000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":23.238,"5min":23.238,"15min":23.238},"last":2.1080000000000001},{"interface":"front","average":{"1min":3.2240000000000002,"5min":3.2240000000000002,"15min":3.2240000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":23.225000000000001,"5min":23.225000000000001,"15min":23.225000000000001},"last":2.121}]},{"osd":1,"last update":"Mon Oct 9 06:10:58 2023","interfaces":[{"interface":"back","average":{"1min":12.914999999999999,"5min":7.8289999999999997,"15min":4.4189999999999996},"min":{"1min":0.33300000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":151.78200000000001,"5min":151.78200000000001,"15min":151.78200000000001},"last":3.5539999999999998},{"interface":"front","average":{"1min":12.388999999999999,"5min":8.1219999999999999,"15min":4.367},"min":{"1min":0.28899999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":152.251,"5min":152.251,"15min":152.251},"last":3.9630000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:10:24 2023","interfaces":[{"interface":"back","average":{"1min":1.9330000000000001,"5min":1.746,"15min":3.0950000000000002},"min":{"1min":0.31,"5min":0.27000000000000002,"15min":0.254},"max":{"1min":7.6890000000000001,"5min":9.0800000000000001,"15min":217.06399999999999},"last":0.69699999999999995},{"interface":"front","average":{"1min":2.0150000000000001,"5min":1.8939999999999999,"15min":3.0489999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28199999999999997},"max":{"1min":6.3639999999999999,"5min":9.5310000000000006,"15min":217.018},"last":3.2829999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:10:58 2023","interfaces":[{"interface":"back","average":{"1min":9.5609999999999999,"5min":7.7199999999999998,"15min":7.1529999999999996},"min":{"1min":0.312,"5min":0.29499999999999998,"15min":0.27500000000000002},"max":{"1min":111.072,"5min":143.31200000000001,"15min":211.40199999999999},"last":0.54000000000000004},{"interface":"front","average":{"1min":10.759,"5min":7.931,"15min":7.2439999999999998},"min":{"1min":0.34499999999999997,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":111.01300000000001,"5min":140.00899999999999,"15min":211.578},"last":0.42199999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:10:44 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":1.407,"15min":1.069},"min":{"1min":0.28999999999999998,"5min":0.245,"15min":0.245},"max":{"1min":7.4880000000000004,"5min":8.1539999999999999,"15min":8.1539999999999999},"last":0.40600000000000003},{"interface":"front","average":{"1min":1.3049999999999999,"5min":1.135,"15min":1.0800000000000001},"min":{"1min":0.29699999999999999,"5min":0.26300000000000001,"15min":0.26000000000000001},"max":{"1min":9.2370000000000001,"5min":9.2370000000000001,"15min":9.2370000000000001},"last":0.376}]},{"osd":6,"last update":"Mon Oct 9 06:10:58 2023","interfaces":[{"interface":"back","average":{"1min":1.5940000000000001,"5min":1.2709999999999999,"15min":1.5529999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.251},"max":{"1min":7.4980000000000002,"5min":8.6809999999999992,"15min":126.89400000000001},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.526,"5min":1.2250000000000001,"15min":1.554},"min":{"1min":0.308,"5min":0.27300000000000002,"15min":0.26200000000000001},"max":{"1min":7.7140000000000004,"5min":7.7140000000000004,"15min":126.616},"last":0.39200000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:10:44 2023","interfaces":[{"interface":"back","average":{"1min":1.579,"5min":2.8889999999999998,"15min":2.5859999999999999},"min":{"1min":0.36299999999999999,"5min":0.309,"15min":0.308},"max":{"1min":9.3179999999999996,"5min":157.471,"15min":157.471},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.94899999999999995,"5min":2.7530000000000001,"15min":2.431},"min":{"1min":0.32100000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.0960000000000001,"5min":157.51599999999999,"15min":157.51599999999999},"last":0.42999999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4889,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":749273088,"data_stored":749273088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":531435520,"data_stored":531431448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":151519232,"data_stored":151519232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":546504704,"data_stored":546504704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":263983104,"data_stored":263979032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":411897856,"data_stored":411893784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":386138112,"data_stored":386138112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":555356160,"data_stored":555352088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:11:06.403 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T06:11:06.404 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 61 2023-10-09T06:11:06.445 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:11:06.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:06.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:11:06.548 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:11:06.576 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:11:06.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:06 smithi064 ceph-mon[220044]: pgmap v1215: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 838 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 398 op/s; 6534/40278 objects degraded (16.222%); 7054/40278 objects misplaced (17.513%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T06:11:06.685 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:06.784 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:11:06.784 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:11:06.784 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T06:11:06.784 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T06:11:06.784 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:11:06.808 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:06 smithi156 ceph-mon[195101]: pgmap v1215: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 838 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 398 op/s; 6534/40278 objects degraded (16.222%); 7054/40278 objects misplaced (17.513%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T06:11:06.826 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:11:06.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:06.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:11:06.934 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:11:06.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:06 smithi064 ceph-mon[212750]: pgmap v1215: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 838 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 398 op/s; 6534/40278 objects degraded (16.222%); 7054/40278 objects misplaced (17.513%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T06:11:06.979 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:07.040 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:11:07.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:07.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:11:07.144 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:11:07.170 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:07.211 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:11:07.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:07.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:11:07.315 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:11:07.355 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:07.653 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:07 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 6534/40278 objects degraded (16.222%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:07.653 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:11:07.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:07.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:11:07.732 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:11:07.765 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:11:07.838 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:07.891 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:11:07.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:07.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:11:07.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:07 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 6534/40278 objects degraded (16.222%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:07.996 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:11:08.043 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:07 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 6534/40278 objects degraded (16.222%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:08.044 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:08.153 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:11:08.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:08.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:11:08.237 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:11:08.266 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:11:08.314 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:08.366 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:11:08.419 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:11:08.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:08.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:11:08.523 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:11:08.566 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:08.628 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:11:08.681 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:08 smithi064 ceph-mon[212750]: pgmap v1216: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 652 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 6209/31326 objects degraded (19.821%); 6409/31326 objects misplaced (20.459%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T06:11:08.682 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:11:08.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:08.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:11:08.787 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:11:08.851 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:08.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:08 smithi156 ceph-mon[195101]: pgmap v1216: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 652 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 6209/31326 objects degraded (19.821%); 6409/31326 objects misplaced (20.459%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T06:11:08.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:08 smithi064 ceph-mon[220044]: pgmap v1216: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 652 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 6209/31326 objects degraded (19.821%); 6409/31326 objects misplaced (20.459%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T06:11:09.036 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:11:09.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:09.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:11:09.140 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:11:09.227 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:09.465 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:11:09.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:09.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:11:09.579 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:11:09.623 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:09.697 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:11:09.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:09.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:11:09.804 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:11:09.843 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:09.936 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:11:09.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:09.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:11:10.046 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:11:10.104 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:10.154 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:11:10.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:10.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:11:10.213 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:11:10.213 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T06:11:10.213 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T06:11:10.213 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:11:10.213 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.213 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:134387 8.d 8:b264af51:::benchmark_data_smithi064_457003_object29869:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-10-09T06:11:10.213 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:10.564773+0000", 2023-10-09T06:11:10.213 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 359.64563693000002, 2023-10-09T06:11:10.214 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1306900139999998, 2023-10-09T06:11:10.214 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:11:10.214 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:11:10.214 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:11:10.214 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:11:10.214 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:11:10.214 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 134387 2023-10-09T06:11:10.214 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.214 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:11:10.214 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.214 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:11:10.215 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.564773+0000", 2023-10-09T06:11:10.215 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.215 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.215 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.215 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:11:10.215 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.564773+0000", 2023-10-09T06:11:10.215 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.215 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.215 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.215 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:11:10.215 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.564772+0000", 2023-10-09T06:11:10.215 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:11:10.216 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.216 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.216 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:11:10.216 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.564778+0000", 2023-10-09T06:11:10.216 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3600000000000004e-06 2023-10-09T06:11:10.216 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.216 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.216 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:11:10.216 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.564792+0000", 2023-10-09T06:11:10.216 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4109e-05 2023-10-09T06:11:10.216 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.216 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.217 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:11:10.217 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.564798+0000", 2023-10-09T06:11:10.217 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1829999999999999e-06 2023-10-09T06:11:10.217 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.217 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.217 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.217 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.564842+0000", 2023-10-09T06:11:10.217 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4147999999999998e-05 2023-10-09T06:11:10.217 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.217 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.217 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:11:10.217 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.564849+0000", 2023-10-09T06:11:10.218 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.652e-06 2023-10-09T06:11:10.218 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.218 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.218 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.218 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:11.450042+0000", 2023-10-09T06:11:10.218 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.88519274999999997 2023-10-09T06:11:10.218 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.218 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.218 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:11:10.218 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:11.450054+0000", 2023-10-09T06:11:10.218 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2559e-05 2023-10-09T06:11:10.218 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.219 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.219 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.219 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.582018+0000", 2023-10-09T06:11:10.219 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1319637280000001 2023-10-09T06:11:10.219 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.219 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.219 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:11:10.219 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.582181+0000", 2023-10-09T06:11:10.219 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00016312600000000001 2023-10-09T06:11:10.219 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.219 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.219 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:11:10.220 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.582742+0000", 2023-10-09T06:11:10.220 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00056150299999999998 2023-10-09T06:11:10.220 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.220 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.220 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:11:10.220 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.590865+0000", 2023-10-09T06:11:10.220 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0081227209999999994 2023-10-09T06:11:10.220 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.220 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.220 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.220 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.591171+0000", 2023-10-09T06:11:10.220 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00030548100000000003 2023-10-09T06:11:10.221 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.221 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.221 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.221 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.591371+0000", 2023-10-09T06:11:10.221 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00019998600000000001 2023-10-09T06:11:10.221 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.221 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.221 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.221 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.695356+0000", 2023-10-09T06:11:10.221 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.103985074 2023-10-09T06:11:10.221 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.221 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.222 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:11:10.222 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.695398+0000", 2023-10-09T06:11:10.222 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2641000000000001e-05 2023-10-09T06:11:10.222 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.222 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.222 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:11:10.222 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.695463+0000", 2023-10-09T06:11:10.222 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5043999999999996e-05 2023-10-09T06:11:10.222 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.222 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:10.222 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.222 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.223 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.223 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:134404 8.d 8:bd17c3be:::benchmark_data_smithi064_457003_object29886:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-10-09T06:11:10.223 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:10.590604+0000", 2023-10-09T06:11:10.223 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 359.61980599399999, 2023-10-09T06:11:10.223 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1059837670000001, 2023-10-09T06:11:10.223 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:11:10.223 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:11:10.223 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:11:10.223 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:11:10.223 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:11:10.223 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 134404 2023-10-09T06:11:10.223 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.224 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:11:10.224 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.224 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:11:10.224 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590604+0000", 2023-10-09T06:11:10.224 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.224 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.224 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.224 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:11:10.224 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590604+0000", 2023-10-09T06:11:10.224 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.224 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.224 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.225 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:11:10.225 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590603+0000", 2023-10-09T06:11:10.225 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:11:10.225 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.225 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.225 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:11:10.225 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590609+0000", 2023-10-09T06:11:10.225 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6620000000000002e-06 2023-10-09T06:11:10.225 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.225 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.225 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:11:10.225 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590621+0000", 2023-10-09T06:11:10.225 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.254e-05 2023-10-09T06:11:10.226 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.226 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.226 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:11:10.226 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590626+0000", 2023-10-09T06:11:10.226 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6840000000000004e-06 2023-10-09T06:11:10.226 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.226 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.226 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.226 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590680+0000", 2023-10-09T06:11:10.226 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4625e-05 2023-10-09T06:11:10.226 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.226 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.227 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:11:10.227 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:10.590691+0000", 2023-10-09T06:11:10.227 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0981999999999999e-05 2023-10-09T06:11:10.227 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.227 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.227 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.227 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:11.450082+0000", 2023-10-09T06:11:10.227 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.85939068100000005 2023-10-09T06:11:10.227 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.227 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.227 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:11:10.227 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:11.450091+0000", 2023-10-09T06:11:10.228 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.5590000000000006e-06 2023-10-09T06:11:10.228 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.228 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.228 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.228 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.587676+0000", 2023-10-09T06:11:10.228 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1375855880000001 2023-10-09T06:11:10.228 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.228 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.228 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:11:10.228 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.587819+0000", 2023-10-09T06:11:10.228 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014301599999999999 2023-10-09T06:11:10.228 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.229 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.229 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:11:10.229 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.587967+0000", 2023-10-09T06:11:10.229 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000148133 2023-10-09T06:11:10.229 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.229 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.229 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:11:10.229 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.591403+0000", 2023-10-09T06:11:10.229 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.003435235 2023-10-09T06:11:10.229 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.229 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.229 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.230 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.596927+0000", 2023-10-09T06:11:10.230 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0055241159999999999 2023-10-09T06:11:10.230 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.230 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.230 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.230 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.597280+0000", 2023-10-09T06:11:10.230 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00035289300000000001 2023-10-09T06:11:10.230 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.230 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.230 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.230 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.696456+0000", 2023-10-09T06:11:10.231 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.099176147000000006 2023-10-09T06:11:10.231 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.231 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.231 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:11:10.231 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.696499+0000", 2023-10-09T06:11:10.231 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3584999999999997e-05 2023-10-09T06:11:10.231 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.231 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.231 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:11:10.231 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:12.696588+0000", 2023-10-09T06:11:10.231 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8826000000000006e-05 2023-10-09T06:11:10.231 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.231 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:10.232 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.232 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.232 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.232 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:148095 8.d 8:bd028436:::benchmark_data_smithi064_457003_object43577:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e389)", 2023-10-09T06:11:10.232 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:31.553767+0000", 2023-10-09T06:11:10.232 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 338.656643239, 2023-10-09T06:11:10.232 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0552826390000001, 2023-10-09T06:11:10.232 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:11:10.232 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:11:10.232 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:11:10.232 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:11:10.232 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:11:10.233 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 148095 2023-10-09T06:11:10.233 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.233 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:11:10.233 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.233 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:11:10.233 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.553767+0000", 2023-10-09T06:11:10.233 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.233 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.233 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.233 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:11:10.233 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.553767+0000", 2023-10-09T06:11:10.233 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.234 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.234 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.234 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:11:10.234 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.553766+0000", 2023-10-09T06:11:10.234 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:11:10.234 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.234 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.234 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:11:10.234 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.553770+0000", 2023-10-09T06:11:10.235 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1250000000000003e-06 2023-10-09T06:11:10.235 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.235 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.235 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:11:10.235 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.553779+0000", 2023-10-09T06:11:10.235 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.5300000000000002e-06 2023-10-09T06:11:10.235 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.235 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.235 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:11:10.235 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.553785+0000", 2023-10-09T06:11:10.235 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6350000000000003e-06 2023-10-09T06:11:10.235 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.236 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.236 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.236 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.554078+0000", 2023-10-09T06:11:10.236 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00029307500000000001 2023-10-09T06:11:10.236 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.236 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.236 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:11:10.236 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.554086+0000", 2023-10-09T06:11:10.236 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.3580000000000003e-06 2023-10-09T06:11:10.236 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.236 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.236 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.237 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:32.420947+0000", 2023-10-09T06:11:10.237 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.86686022200000001 2023-10-09T06:11:10.237 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.237 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.237 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:11:10.237 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:32.420958+0000", 2023-10-09T06:11:10.237 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1759e-05 2023-10-09T06:11:10.237 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.237 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.237 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.237 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.556878+0000", 2023-10-09T06:11:10.237 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.135919782 2023-10-09T06:11:10.238 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.238 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.238 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:11:10.238 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.556994+0000", 2023-10-09T06:11:10.238 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011585499999999999 2023-10-09T06:11:10.238 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.238 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.238 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T06:11:10.238 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.557534+0000", 2023-10-09T06:11:10.238 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00054018100000000004 2023-10-09T06:11:10.238 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.238 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.239 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:11:10.239 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.567819+0000", 2023-10-09T06:11:10.239 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010284695999999999 2023-10-09T06:11:10.239 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.239 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.239 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.239 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.608747+0000", 2023-10-09T06:11:10.239 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.040927951999999997 2023-10-09T06:11:10.239 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.239 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.239 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.239 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.608813+0000", 2023-10-09T06:11:10.240 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5950000000000004e-05 2023-10-09T06:11:10.240 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.240 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.240 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.240 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.608861+0000", 2023-10-09T06:11:10.240 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8052e-05 2023-10-09T06:11:10.240 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.240 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.240 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:11:10.240 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.608911+0000", 2023-10-09T06:11:10.240 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0008e-05 2023-10-09T06:11:10.240 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.240 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.241 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:11:10.241 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.609050+0000", 2023-10-09T06:11:10.241 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013881700000000001 2023-10-09T06:11:10.241 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.241 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:10.241 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.241 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.241 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.241 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:148097 8.d 8:be110a7d:::benchmark_data_smithi064_457003_object43579:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e389)", 2023-10-09T06:11:10.241 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:31.553792+0000", 2023-10-09T06:11:10.241 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 338.656618299, 2023-10-09T06:11:10.241 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0555980979999999, 2023-10-09T06:11:10.242 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:11:10.242 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:11:10.242 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:11:10.242 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:11:10.242 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:11:10.242 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 148097 2023-10-09T06:11:10.242 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.242 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:11:10.242 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.242 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:11:10.242 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.553792+0000", 2023-10-09T06:11:10.242 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.243 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.243 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.243 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:11:10.243 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.553792+0000", 2023-10-09T06:11:10.243 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.243 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.243 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.243 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:11:10.243 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.553792+0000", 2023-10-09T06:11:10.243 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:11:10.243 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.244 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.244 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:11:10.244 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.553794+0000", 2023-10-09T06:11:10.244 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2819999999999999e-06 2023-10-09T06:11:10.244 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.244 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.244 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:11:10.244 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.553800+0000", 2023-10-09T06:11:10.244 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4689999999999996e-06 2023-10-09T06:11:10.244 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.244 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.244 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:11:10.245 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.553803+0000", 2023-10-09T06:11:10.245 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8099999999999999e-06 2023-10-09T06:11:10.245 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.245 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.245 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.245 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.554150+0000", 2023-10-09T06:11:10.245 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00034632100000000002 2023-10-09T06:11:10.245 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.245 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.245 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:11:10.245 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.554156+0000", 2023-10-09T06:11:10.245 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6200000000000001e-06 2023-10-09T06:11:10.246 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.246 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.246 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.246 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:32.420988+0000", 2023-10-09T06:11:10.246 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.866831295 2023-10-09T06:11:10.246 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.246 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.246 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:11:10.246 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:32.420995+0000", 2023-10-09T06:11:10.246 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6299999999999998e-06 2023-10-09T06:11:10.246 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.246 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.247 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.247 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.561631+0000", 2023-10-09T06:11:10.247 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1406362379999999 2023-10-09T06:11:10.247 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.247 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.247 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:11:10.247 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.561721+0000", 2023-10-09T06:11:10.247 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9339999999999995e-05 2023-10-09T06:11:10.247 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.247 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.247 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T06:11:10.247 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.561833+0000", 2023-10-09T06:11:10.248 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000112286 2023-10-09T06:11:10.248 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.248 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.248 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:11:10.248 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.571366+0000", 2023-10-09T06:11:10.248 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0095330120000000004 2023-10-09T06:11:10.248 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.248 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.248 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.248 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.609100+0000", 2023-10-09T06:11:10.248 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.037734151 2023-10-09T06:11:10.248 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.249 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.249 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.249 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.609154+0000", 2023-10-09T06:11:10.249 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3628999999999998e-05 2023-10-09T06:11:10.249 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.249 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.249 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.249 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.609217+0000", 2023-10-09T06:11:10.249 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3595000000000004e-05 2023-10-09T06:11:10.249 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.249 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.249 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:11:10.250 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.609272+0000", 2023-10-09T06:11:10.250 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4932000000000002e-05 2023-10-09T06:11:10.250 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.250 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.250 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:11:10.250 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.609390+0000", 2023-10-09T06:11:10.250 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000117782 2023-10-09T06:11:10.250 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.250 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:10.250 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.250 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.250 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.251 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:148135 8.d 8:b2daf003:::benchmark_data_smithi064_457003_object43617:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e389)", 2023-10-09T06:11:10.251 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:05:31.608060+0000", 2023-10-09T06:11:10.251 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 338.60235009899998, 2023-10-09T06:11:10.251 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0017507889999999, 2023-10-09T06:11:10.251 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:11:10.251 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:11:10.251 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:11:10.251 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:11:10.251 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:11:10.251 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 148135 2023-10-09T06:11:10.251 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.251 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:11:10.252 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.252 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:11:10.252 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.608060+0000", 2023-10-09T06:11:10.252 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.252 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.252 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.252 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:11:10.252 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.608060+0000", 2023-10-09T06:11:10.252 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.252 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.252 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.252 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:11:10.253 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.608059+0000", 2023-10-09T06:11:10.253 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:11:10.253 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.253 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.253 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:11:10.253 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.608067+0000", 2023-10-09T06:11:10.253 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5000000000000002e-06 2023-10-09T06:11:10.253 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.253 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.253 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:11:10.253 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.608083+0000", 2023-10-09T06:11:10.253 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6328e-05 2023-10-09T06:11:10.253 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.254 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.254 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:11:10.254 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.608090+0000", 2023-10-09T06:11:10.254 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3509999999999996e-06 2023-10-09T06:11:10.254 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.254 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.254 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.254 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.608152+0000", 2023-10-09T06:11:10.254 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1729000000000005e-05 2023-10-09T06:11:10.254 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.254 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.254 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:11:10.255 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:31.608162+0000", 2023-10-09T06:11:10.255 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0149999999999999e-05 2023-10-09T06:11:10.255 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.255 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.255 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.255 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:32.421010+0000", 2023-10-09T06:11:10.255 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.81284742899999995 2023-10-09T06:11:10.255 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.255 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.255 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:11:10.255 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:32.421014+0000", 2023-10-09T06:11:10.255 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8770000000000003e-06 2023-10-09T06:11:10.256 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.256 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.256 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.256 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.564679+0000", 2023-10-09T06:11:10.256 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.143665806 2023-10-09T06:11:10.256 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.256 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.256 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:11:10.256 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.564811+0000", 2023-10-09T06:11:10.256 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000131505 2023-10-09T06:11:10.256 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.256 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.257 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T06:11:10.257 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.564954+0000", 2023-10-09T06:11:10.257 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014360299999999999 2023-10-09T06:11:10.257 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.257 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.257 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:11:10.257 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.574596+0000", 2023-10-09T06:11:10.257 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0096416899999999996 2023-10-09T06:11:10.257 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.257 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.257 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.257 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.609438+0000", 2023-10-09T06:11:10.258 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.034841346000000002 2023-10-09T06:11:10.258 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.258 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.258 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.258 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.609592+0000", 2023-10-09T06:11:10.258 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000154716 2023-10-09T06:11:10.258 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.258 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.258 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.258 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.609634+0000", 2023-10-09T06:11:10.258 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2018999999999999e-05 2023-10-09T06:11:10.258 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.259 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.259 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:11:10.259 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.609683+0000", 2023-10-09T06:11:10.259 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8888999999999999e-05 2023-10-09T06:11:10.259 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.259 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.259 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:11:10.259 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:05:33.609811+0000", 2023-10-09T06:11:10.259 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012789999999999999 2023-10-09T06:11:10.259 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.259 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:10.259 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.260 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.260 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.260 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:58018 9.5 9:ad237675:::benchmark_data_smithi064_511911_object58017:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2023-10-09T06:11:10.260 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:07:50.684831+0000", 2023-10-09T06:11:10.260 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 199.52557925900001, 2023-10-09T06:11:10.260 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0269654070000001, 2023-10-09T06:11:10.260 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:11:10.260 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:11:10.260 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:11:10.260 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:11:10.260 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:11:10.261 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 58018 2023-10-09T06:11:10.261 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.261 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:11:10.261 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.261 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:11:10.261 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.684831+0000", 2023-10-09T06:11:10.261 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.261 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.261 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.261 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:11:10.261 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.684831+0000", 2023-10-09T06:11:10.261 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.261 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.262 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.262 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:11:10.262 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.684830+0000", 2023-10-09T06:11:10.262 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:11:10.262 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.262 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.262 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:11:10.262 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.684922+0000", 2023-10-09T06:11:10.262 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2584000000000007e-05 2023-10-09T06:11:10.262 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.262 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.262 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:11:10.263 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.684946+0000", 2023-10-09T06:11:10.263 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3587e-05 2023-10-09T06:11:10.263 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.263 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.263 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:11:10.263 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.684966+0000", 2023-10-09T06:11:10.263 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0222000000000001e-05 2023-10-09T06:11:10.263 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.263 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.263 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.263 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.685010+0000", 2023-10-09T06:11:10.263 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3668999999999998e-05 2023-10-09T06:11:10.264 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.264 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.264 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:11:10.264 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.685167+0000", 2023-10-09T06:11:10.264 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000157116 2023-10-09T06:11:10.264 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.264 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.264 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:11:10.264 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.685324+0000", 2023-10-09T06:11:10.264 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015695100000000001 2023-10-09T06:11:10.264 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.264 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.265 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:11:10.265 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.689172+0000", 2023-10-09T06:11:10.265 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0038483359999999999 2023-10-09T06:11:10.265 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.265 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.265 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.265 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.689351+0000", 2023-10-09T06:11:10.265 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00017877 2023-10-09T06:11:10.265 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.265 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.265 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.265 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.709096+0000", 2023-10-09T06:11:10.265 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.019744817000000001 2023-10-09T06:11:10.266 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.266 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.266 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:11:10.266 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.709105+0000", 2023-10-09T06:11:10.266 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9209999999999998e-06 2023-10-09T06:11:10.266 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.266 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.266 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.266 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.711724+0000", 2023-10-09T06:11:10.266 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0026194079999999 2023-10-09T06:11:10.266 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.266 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.267 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:11:10.267 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.711797+0000", 2023-10-09T06:11:10.267 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.237e-05 2023-10-09T06:11:10.267 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.267 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:10.267 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.267 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.267 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.267 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:58040 9.9 9:949f63c6:::benchmark_data_smithi064_511911_object58039:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2023-10-09T06:11:10.267 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:07:50.711513+0000", 2023-10-09T06:11:10.267 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 199.49889783200001, 2023-10-09T06:11:10.267 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0069264769999999, 2023-10-09T06:11:10.267 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:11:10.268 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:11:10.268 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:11:10.268 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:11:10.268 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:11:10.268 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 58040 2023-10-09T06:11:10.268 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.268 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:11:10.268 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.268 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:11:10.268 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.711513+0000", 2023-10-09T06:11:10.268 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.268 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.269 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.269 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:11:10.269 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.711513+0000", 2023-10-09T06:11:10.269 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.269 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.269 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.269 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:11:10.269 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.711512+0000", 2023-10-09T06:11:10.269 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:11:10.269 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.269 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.269 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:11:10.269 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.711542+0000", 2023-10-09T06:11:10.270 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9875000000000001e-05 2023-10-09T06:11:10.270 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.270 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.270 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:11:10.270 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.711548+0000", 2023-10-09T06:11:10.270 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.28e-06 2023-10-09T06:11:10.270 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.270 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.270 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:11:10.270 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.711552+0000", 2023-10-09T06:11:10.270 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6109999999999998e-06 2023-10-09T06:11:10.270 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.271 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.271 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.271 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.716882+0000", 2023-10-09T06:11:10.271 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0053301729999999997 2023-10-09T06:11:10.271 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.271 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.271 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:11:10.271 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.716889+0000", 2023-10-09T06:11:10.271 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7610000000000002e-06 2023-10-09T06:11:10.271 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.271 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.271 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.272 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.709263+0000", 2023-10-09T06:11:10.272 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99237410599999998 2023-10-09T06:11:10.272 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.272 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.272 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:11:10.272 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.709402+0000", 2023-10-09T06:11:10.272 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013855700000000001 2023-10-09T06:11:10.272 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.272 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.272 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5", 2023-10-09T06:11:10.272 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.709567+0000", 2023-10-09T06:11:10.272 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00016538100000000001 2023-10-09T06:11:10.272 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.273 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.273 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:11:10.273 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.716242+0000", 2023-10-09T06:11:10.273 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0066750159999999998 2023-10-09T06:11:10.273 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.273 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.273 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.273 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718290+0000", 2023-10-09T06:11:10.273 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0020477130000000001 2023-10-09T06:11:10.273 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.273 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.273 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:11:10.273 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718340+0000", 2023-10-09T06:11:10.274 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0121999999999999e-05 2023-10-09T06:11:10.274 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.274 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.274 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:11:10.274 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718439+0000", 2023-10-09T06:11:10.274 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.9265999999999995e-05 2023-10-09T06:11:10.274 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.274 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:10.274 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.274 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.274 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.274 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:58064 9.9 9:972c46fb:::benchmark_data_smithi064_511911_object58063:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-10-09T06:11:10.275 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:07:50.749569+0000", 2023-10-09T06:11:10.275 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 199.460841069, 2023-10-09T06:11:10.275 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96902807899999999, 2023-10-09T06:11:10.275 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:11:10.275 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:11:10.275 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:11:10.275 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:11:10.275 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:11:10.275 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 58064 2023-10-09T06:11:10.275 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.275 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:11:10.276 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.276 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:11:10.276 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.749569+0000", 2023-10-09T06:11:10.276 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.276 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.276 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.276 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:11:10.276 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.749569+0000", 2023-10-09T06:11:10.276 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.276 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.276 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.276 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:11:10.277 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.749568+0000", 2023-10-09T06:11:10.277 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:11:10.277 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.277 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.277 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:11:10.277 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.749620+0000", 2023-10-09T06:11:10.277 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1990000000000002e-05 2023-10-09T06:11:10.277 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.277 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.277 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:11:10.277 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.749631+0000", 2023-10-09T06:11:10.277 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0131e-05 2023-10-09T06:11:10.277 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.278 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.278 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:11:10.278 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.749635+0000", 2023-10-09T06:11:10.278 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9539999999999998e-06 2023-10-09T06:11:10.278 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.278 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.278 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.278 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.749677+0000", 2023-10-09T06:11:10.278 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2719000000000002e-05 2023-10-09T06:11:10.278 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.278 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.279 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:11:10.279 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.749686+0000", 2023-10-09T06:11:10.279 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1719999999999996e-06 2023-10-09T06:11:10.279 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.279 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.279 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.279 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.711769+0000", 2023-10-09T06:11:10.279 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96208275300000001 2023-10-09T06:11:10.279 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.279 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.279 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:11:10.279 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.711891+0000", 2023-10-09T06:11:10.280 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012202200000000001 2023-10-09T06:11:10.280 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.280 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.280 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5", 2023-10-09T06:11:10.280 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.712032+0000", 2023-10-09T06:11:10.280 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014087799999999999 2023-10-09T06:11:10.280 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.280 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.280 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:11:10.280 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718043+0000", 2023-10-09T06:11:10.280 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0060111720000000004 2023-10-09T06:11:10.280 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.280 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.281 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.281 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718475+0000", 2023-10-09T06:11:10.281 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00043195499999999998 2023-10-09T06:11:10.281 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.281 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.281 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:11:10.281 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718519+0000", 2023-10-09T06:11:10.281 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3587000000000001e-05 2023-10-09T06:11:10.281 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.281 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.281 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:11:10.281 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718597+0000", 2023-10-09T06:11:10.282 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8603000000000006e-05 2023-10-09T06:11:10.282 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.282 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:10.282 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.282 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.282 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.282 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:58070 9.9 9:97018e1f:::benchmark_data_smithi064_511911_object58069:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-10-09T06:11:10.282 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:07:50.758559+0000", 2023-10-09T06:11:10.282 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 199.45185157700001, 2023-10-09T06:11:10.282 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96067754699999997, 2023-10-09T06:11:10.282 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:11:10.283 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:11:10.283 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:11:10.283 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:11:10.283 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:11:10.283 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 58070 2023-10-09T06:11:10.283 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.283 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:11:10.283 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.283 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:11:10.283 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.758559+0000", 2023-10-09T06:11:10.283 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.283 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.283 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.284 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:11:10.284 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.758559+0000", 2023-10-09T06:11:10.284 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.284 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.284 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.284 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:11:10.284 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.758558+0000", 2023-10-09T06:11:10.284 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:11:10.284 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.284 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.284 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:11:10.285 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.758601+0000", 2023-10-09T06:11:10.285 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3180000000000003e-05 2023-10-09T06:11:10.285 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.285 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.285 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:11:10.285 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.758614+0000", 2023-10-09T06:11:10.285 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3309000000000001e-05 2023-10-09T06:11:10.285 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.285 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.285 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:11:10.285 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.758619+0000", 2023-10-09T06:11:10.285 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7700000000000001e-06 2023-10-09T06:11:10.286 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.286 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.286 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.286 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.758669+0000", 2023-10-09T06:11:10.286 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0027000000000001e-05 2023-10-09T06:11:10.286 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.286 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.286 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:11:10.286 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.758677+0000", 2023-10-09T06:11:10.286 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1389999999999995e-06 2023-10-09T06:11:10.286 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.287 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.287 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.287 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.714028+0000", 2023-10-09T06:11:10.287 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.95535058699999997 2023-10-09T06:11:10.287 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.287 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.287 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:11:10.287 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.714132+0000", 2023-10-09T06:11:10.287 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000104565 2023-10-09T06:11:10.287 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.287 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.287 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5", 2023-10-09T06:11:10.288 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.714264+0000", 2023-10-09T06:11:10.288 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000131798 2023-10-09T06:11:10.288 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.288 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.288 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.288 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.719035+0000", 2023-10-09T06:11:10.288 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0047712479999999996 2023-10-09T06:11:10.288 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.288 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.288 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:11:10.288 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.719088+0000", 2023-10-09T06:11:10.288 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2954000000000002e-05 2023-10-09T06:11:10.288 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.289 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.289 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:11:10.289 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.719156+0000", 2023-10-09T06:11:10.289 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7958000000000002e-05 2023-10-09T06:11:10.289 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.289 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.289 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:11:10.289 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.719236+0000", 2023-10-09T06:11:10.289 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9980000000000003e-05 2023-10-09T06:11:10.289 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.289 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:10.289 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.290 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.290 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.290 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:58081 9.5 9:a1616141:::benchmark_data_smithi064_511911_object58080:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-10-09T06:11:10.290 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:07:50.770142+0000", 2023-10-09T06:11:10.290 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 199.44026798799999, 2023-10-09T06:11:10.290 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.94865276799999998, 2023-10-09T06:11:10.290 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:11:10.290 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:11:10.290 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:11:10.290 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:11:10.290 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:11:10.291 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 58081 2023-10-09T06:11:10.291 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.291 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:11:10.291 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.291 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:11:10.291 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770142+0000", 2023-10-09T06:11:10.291 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.291 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.291 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.291 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:11:10.291 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770142+0000", 2023-10-09T06:11:10.291 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.292 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.292 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.292 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:11:10.292 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770141+0000", 2023-10-09T06:11:10.292 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:11:10.292 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.292 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.292 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:11:10.292 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770188+0000", 2023-10-09T06:11:10.292 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6885999999999998e-05 2023-10-09T06:11:10.292 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.292 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.293 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:11:10.293 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770199+0000", 2023-10-09T06:11:10.293 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.058e-05 2023-10-09T06:11:10.293 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.293 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.293 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:11:10.293 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770203+0000", 2023-10-09T06:11:10.293 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4139999999999996e-06 2023-10-09T06:11:10.293 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.293 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.293 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.294 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770301+0000", 2023-10-09T06:11:10.294 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7325999999999996e-05 2023-10-09T06:11:10.294 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.294 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.294 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:11:10.294 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770319+0000", 2023-10-09T06:11:10.294 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8601e-05 2023-10-09T06:11:10.294 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.294 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.294 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.294 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.711820+0000", 2023-10-09T06:11:10.294 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.94150076199999999 2023-10-09T06:11:10.295 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.295 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.295 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:11:10.295 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.711925+0000", 2023-10-09T06:11:10.295 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000105262 2023-10-09T06:11:10.295 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.295 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.295 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5", 2023-10-09T06:11:10.295 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.712064+0000", 2023-10-09T06:11:10.295 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013905699999999999 2023-10-09T06:11:10.295 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.295 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.296 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.296 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718649+0000", 2023-10-09T06:11:10.296 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0065846630000000001 2023-10-09T06:11:10.296 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.296 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.296 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:11:10.296 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718660+0000", 2023-10-09T06:11:10.296 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0929e-05 2023-10-09T06:11:10.296 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.296 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.296 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:11:10.297 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718711+0000", 2023-10-09T06:11:10.297 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0955000000000003e-05 2023-10-09T06:11:10.297 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.297 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.297 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:11:10.297 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718795+0000", 2023-10-09T06:11:10.297 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.4289000000000006e-05 2023-10-09T06:11:10.297 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.297 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:10.297 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.297 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.297 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.297 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:58083 9.9 9:99aef645:::benchmark_data_smithi064_511911_object58082:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-10-09T06:11:10.298 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:07:50.770277+0000", 2023-10-09T06:11:10.298 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 199.44013287999999, 2023-10-09T06:11:10.298 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.95279251799999998, 2023-10-09T06:11:10.298 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:11:10.298 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:11:10.298 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:11:10.298 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:11:10.298 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:11:10.298 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 58083 2023-10-09T06:11:10.298 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.298 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:11:10.299 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.299 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:11:10.299 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770277+0000", 2023-10-09T06:11:10.299 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.299 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.299 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.299 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:11:10.299 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770277+0000", 2023-10-09T06:11:10.299 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.299 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.299 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.299 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:11:10.300 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770277+0000", 2023-10-09T06:11:10.300 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:11:10.300 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.300 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.300 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:11:10.300 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770313+0000", 2023-10-09T06:11:10.300 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6062000000000001e-05 2023-10-09T06:11:10.300 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.300 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.300 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:11:10.300 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770323+0000", 2023-10-09T06:11:10.300 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0016e-05 2023-10-09T06:11:10.300 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.301 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.301 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:11:10.301 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770328+0000", 2023-10-09T06:11:10.301 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8339999999999998e-06 2023-10-09T06:11:10.301 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.301 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.301 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.301 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770379+0000", 2023-10-09T06:11:10.301 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1304999999999998e-05 2023-10-09T06:11:10.301 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.301 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.301 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:11:10.302 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770388+0000", 2023-10-09T06:11:10.302 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8079999999999994e-06 2023-10-09T06:11:10.302 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.302 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.302 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.302 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.716270+0000", 2023-10-09T06:11:10.302 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.94588199100000003 2023-10-09T06:11:10.302 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.302 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.302 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:11:10.302 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.716372+0000", 2023-10-09T06:11:10.303 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000101827 2023-10-09T06:11:10.303 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.303 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.303 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5", 2023-10-09T06:11:10.303 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.716452+0000", 2023-10-09T06:11:10.303 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0498000000000001e-05 2023-10-09T06:11:10.303 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.303 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.303 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:11:10.303 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.719244+0000", 2023-10-09T06:11:10.303 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0027915470000000001 2023-10-09T06:11:10.303 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.304 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.304 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.304 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.722915+0000", 2023-10-09T06:11:10.304 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0036714529999999999 2023-10-09T06:11:10.304 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.304 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.304 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:11:10.304 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.722964+0000", 2023-10-09T06:11:10.304 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8678999999999999e-05 2023-10-09T06:11:10.304 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.304 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.304 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:11:10.305 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.723070+0000", 2023-10-09T06:11:10.305 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000105936 2023-10-09T06:11:10.305 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.305 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:10.305 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.305 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.305 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.305 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:58085 9.5 9:a3465072:::benchmark_data_smithi064_511911_object58084:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-10-09T06:11:10.305 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:07:50.774234+0000", 2023-10-09T06:11:10.305 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 199.43617616899999, 2023-10-09T06:11:10.305 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.94506012299999997, 2023-10-09T06:11:10.305 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:11:10.306 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:11:10.306 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:11:10.306 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:11:10.306 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:11:10.306 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 58085 2023-10-09T06:11:10.306 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.306 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:11:10.306 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.306 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:11:10.306 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.774234+0000", 2023-10-09T06:11:10.306 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.307 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.307 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.307 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:11:10.307 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.774234+0000", 2023-10-09T06:11:10.307 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.307 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.307 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.307 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:11:10.307 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.774233+0000", 2023-10-09T06:11:10.307 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:11:10.307 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.308 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.308 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:11:10.308 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.774284+0000", 2023-10-09T06:11:10.308 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0781000000000001e-05 2023-10-09T06:11:10.308 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.308 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.308 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:11:10.308 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.774295+0000", 2023-10-09T06:11:10.308 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0556e-05 2023-10-09T06:11:10.308 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.308 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.308 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:11:10.309 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.774299+0000", 2023-10-09T06:11:10.309 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9400000000000004e-06 2023-10-09T06:11:10.309 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.309 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.309 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.309 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.774347+0000", 2023-10-09T06:11:10.309 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8619999999999999e-05 2023-10-09T06:11:10.309 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.309 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.309 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:11:10.309 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.774356+0000", 2023-10-09T06:11:10.309 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9719999999999998e-06 2023-10-09T06:11:10.310 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.310 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.310 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.310 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.714102+0000", 2023-10-09T06:11:10.310 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.93974562800000006 2023-10-09T06:11:10.310 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.310 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.310 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:11:10.310 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.714202+0000", 2023-10-09T06:11:10.310 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010035900000000001 2023-10-09T06:11:10.310 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.310 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.311 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5", 2023-10-09T06:11:10.311 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.714332+0000", 2023-10-09T06:11:10.311 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013015499999999999 2023-10-09T06:11:10.311 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.311 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.311 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.311 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.719172+0000", 2023-10-09T06:11:10.311 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00483936 2023-10-09T06:11:10.311 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.311 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.311 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:11:10.311 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.719178+0000", 2023-10-09T06:11:10.312 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1070000000000001e-06 2023-10-09T06:11:10.312 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.312 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.312 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:11:10.312 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.719218+0000", 2023-10-09T06:11:10.312 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0370000000000001e-05 2023-10-09T06:11:10.312 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.312 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.312 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:11:10.312 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.719294+0000", 2023-10-09T06:11:10.312 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6116000000000001e-05 2023-10-09T06:11:10.312 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.313 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:10.313 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.313 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.313 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.313 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:58095 9.5 9:aeb94bf3:::benchmark_data_smithi064_511911_object58094:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-10-09T06:11:10.313 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:07:50.797444+0000", 2023-10-09T06:11:10.313 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 199.41296626100001, 2023-10-09T06:11:10.313 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.92562260699999999, 2023-10-09T06:11:10.313 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:11:10.313 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:11:10.313 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:11:10.313 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:11:10.314 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:11:10.314 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 58095 2023-10-09T06:11:10.314 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.314 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:11:10.314 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.314 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:11:10.314 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.797444+0000", 2023-10-09T06:11:10.314 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.314 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.314 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.314 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:11:10.315 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.797444+0000", 2023-10-09T06:11:10.315 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.315 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.315 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.315 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:11:10.315 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.797443+0000", 2023-10-09T06:11:10.315 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:11:10.315 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.315 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.315 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:11:10.315 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.797507+0000", 2023-10-09T06:11:10.315 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3756000000000004e-05 2023-10-09T06:11:10.316 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.316 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.316 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:11:10.316 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.797520+0000", 2023-10-09T06:11:10.316 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2954e-05 2023-10-09T06:11:10.316 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.316 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.316 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:11:10.316 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.797539+0000", 2023-10-09T06:11:10.316 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9412999999999999e-05 2023-10-09T06:11:10.316 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.316 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.317 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.317 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.797571+0000", 2023-10-09T06:11:10.317 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1519000000000001e-05 2023-10-09T06:11:10.317 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.317 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.317 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:11:10.317 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.797576+0000", 2023-10-09T06:11:10.317 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4809999999999997e-06 2023-10-09T06:11:10.317 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.317 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.317 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.317 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.716291+0000", 2023-10-09T06:11:10.318 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.91871448200000005 2023-10-09T06:11:10.318 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.318 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.318 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:11:10.318 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.716389+0000", 2023-10-09T06:11:10.318 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.8221999999999996e-05 2023-10-09T06:11:10.318 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.318 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.318 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5", 2023-10-09T06:11:10.318 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.716518+0000", 2023-10-09T06:11:10.318 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012928899999999999 2023-10-09T06:11:10.318 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.319 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.319 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:11:10.319 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.719305+0000", 2023-10-09T06:11:10.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0027869650000000002 2023-10-09T06:11:10.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.319 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.319 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.319 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.722931+0000", 2023-10-09T06:11:10.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0036255459999999999 2023-10-09T06:11:10.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.319 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.319 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:11:10.320 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.722973+0000", 2023-10-09T06:11:10.320 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2101000000000002e-05 2023-10-09T06:11:10.320 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.320 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.320 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:11:10.320 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.723067+0000", 2023-10-09T06:11:10.320 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3758999999999995e-05 2023-10-09T06:11:10.320 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.320 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:10.320 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.320 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.320 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.321 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:116248 9.e 9:7ca2e1df:::benchmark_data_smithi064_511911_object15146:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-10-09T06:11:10.321 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:02.491125+0000", 2023-10-09T06:11:10.321 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 127.71928577, 2023-10-09T06:11:10.321 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.28000069399999999, 2023-10-09T06:11:10.321 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:11:10.321 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:11:10.321 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:11:10.321 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:11:10.321 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:11:10.321 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 116248 2023-10-09T06:11:10.321 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.321 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:11:10.322 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.322 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:11:10.322 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.491125+0000", 2023-10-09T06:11:10.322 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.322 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.322 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.322 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:11:10.322 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.491125+0000", 2023-10-09T06:11:10.322 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.322 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.322 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.322 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:11:10.323 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.491124+0000", 2023-10-09T06:11:10.323 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:11:10.323 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.323 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.323 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:11:10.323 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.491127+0000", 2023-10-09T06:11:10.323 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8419999999999998e-06 2023-10-09T06:11:10.323 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.323 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.323 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:11:10.323 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.491135+0000", 2023-10-09T06:11:10.323 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6830000000000008e-06 2023-10-09T06:11:10.324 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.324 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.324 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:11:10.324 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.491140+0000", 2023-10-09T06:11:10.324 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4449999999999998e-06 2023-10-09T06:11:10.324 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.324 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.324 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.324 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.491181+0000", 2023-10-09T06:11:10.324 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0951000000000001e-05 2023-10-09T06:11:10.324 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.324 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.325 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:11:10.325 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.491288+0000", 2023-10-09T06:11:10.325 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010794900000000001 2023-10-09T06:11:10.325 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.325 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.325 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:11:10.325 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.491386+0000", 2023-10-09T06:11:10.325 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7652999999999995e-05 2023-10-09T06:11:10.325 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.325 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.325 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:11:10.326 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.494465+0000", 2023-10-09T06:11:10.326 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0030789680000000001 2023-10-09T06:11:10.326 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.326 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.326 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.326 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.494565+0000", 2023-10-09T06:11:10.326 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.9776000000000002e-05 2023-10-09T06:11:10.326 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.326 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.326 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.326 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.771053+0000", 2023-10-09T06:11:10.326 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.276488124 2023-10-09T06:11:10.327 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.327 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.327 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:11:10.327 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.771091+0000", 2023-10-09T06:11:10.327 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8513999999999998e-05 2023-10-09T06:11:10.327 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.327 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.327 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:11:10.327 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.771125+0000", 2023-10-09T06:11:10.327 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.379e-05 2023-10-09T06:11:10.327 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.327 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:10.328 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.328 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.328 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.328 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:116286 9.9 9:91b2f3bd:::benchmark_data_smithi064_511911_object15184:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e450)", 2023-10-09T06:11:10.328 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:03.586689+0000", 2023-10-09T06:11:10.328 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 126.623721817, 2023-10-09T06:11:10.328 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1393943420000001, 2023-10-09T06:11:10.328 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:11:10.328 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:11:10.328 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:11:10.328 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:11:10.328 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:11:10.329 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 116286 2023-10-09T06:11:10.329 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.329 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:11:10.329 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.329 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:11:10.329 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586689+0000", 2023-10-09T06:11:10.329 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.329 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.329 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.329 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:11:10.329 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586689+0000", 2023-10-09T06:11:10.329 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.330 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.330 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.330 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:11:10.330 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586687+0000", 2023-10-09T06:11:10.330 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:11:10.330 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.330 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.330 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:11:10.330 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586695+0000", 2023-10-09T06:11:10.330 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0700000000000007e-06 2023-10-09T06:11:10.330 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.330 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.331 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:11:10.331 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586708+0000", 2023-10-09T06:11:10.331 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2928999999999999e-05 2023-10-09T06:11:10.331 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.331 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.331 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:11:10.331 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586715+0000", 2023-10-09T06:11:10.331 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1629999999999999e-06 2023-10-09T06:11:10.331 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.331 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.331 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.331 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.590775+0000", 2023-10-09T06:11:10.332 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0040602310000000001 2023-10-09T06:11:10.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.332 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.332 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:11:10.332 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.590785+0000", 2023-10-09T06:11:10.332 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2529999999999993e-06 2023-10-09T06:11:10.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.332 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.332 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.332 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.708849+0000", 2023-10-09T06:11:10.332 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1180647349999999 2023-10-09T06:11:10.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.333 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.333 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for missing object", 2023-10-09T06:11:10.333 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.708984+0000", 2023-10-09T06:11:10.333 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000134136 2023-10-09T06:11:10.333 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.333 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.333 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.333 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.725947+0000", 2023-10-09T06:11:10.333 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.016963676 2023-10-09T06:11:10.333 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.333 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.334 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:11:10.334 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.726083+0000", 2023-10-09T06:11:10.334 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000135699 2023-10-09T06:11:10.334 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.334 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:10.334 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.334 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.334 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.334 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:116302 9.9 9:97b6a430:::benchmark_data_smithi064_511911_object15200:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e450)", 2023-10-09T06:11:10.334 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:03.586724+0000", 2023-10-09T06:11:10.334 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 126.623686521, 2023-10-09T06:11:10.334 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.139970148, 2023-10-09T06:11:10.335 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:11:10.335 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:11:10.335 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:11:10.335 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:11:10.335 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:11:10.335 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 116302 2023-10-09T06:11:10.335 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.335 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:11:10.335 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.335 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:11:10.335 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586724+0000", 2023-10-09T06:11:10.336 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.336 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.336 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.336 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:11:10.336 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586724+0000", 2023-10-09T06:11:10.336 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.336 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.336 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.336 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:11:10.336 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586723+0000", 2023-10-09T06:11:10.336 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:11:10.336 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.337 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.337 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:11:10.337 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586726+0000", 2023-10-09T06:11:10.337 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.8909999999999999e-06 2023-10-09T06:11:10.337 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.337 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.337 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:11:10.337 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586733+0000", 2023-10-09T06:11:10.337 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.131e-06 2023-10-09T06:11:10.337 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.337 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.337 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:11:10.337 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586739+0000", 2023-10-09T06:11:10.338 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.485e-06 2023-10-09T06:11:10.338 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.338 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.338 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.338 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.599094+0000", 2023-10-09T06:11:10.338 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012355138 2023-10-09T06:11:10.338 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.338 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.338 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:11:10.338 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.599106+0000", 2023-10-09T06:11:10.338 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1565e-05 2023-10-09T06:11:10.339 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.339 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.339 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.339 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.709008+0000", 2023-10-09T06:11:10.339 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1099019999999999 2023-10-09T06:11:10.339 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.339 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.339 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:11:10.339 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.709136+0000", 2023-10-09T06:11:10.339 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012808900000000001 2023-10-09T06:11:10.339 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.339 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.340 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:11:10.340 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.709273+0000", 2023-10-09T06:11:10.340 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000137584 2023-10-09T06:11:10.340 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.340 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.340 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:11:10.340 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.725163+0000", 2023-10-09T06:11:10.340 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015889678000000001 2023-10-09T06:11:10.340 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.340 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.340 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.340 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.725700+0000", 2023-10-09T06:11:10.341 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00053697700000000001 2023-10-09T06:11:10.341 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.341 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.341 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.341 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.726552+0000", 2023-10-09T06:11:10.341 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000852399 2023-10-09T06:11:10.341 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.341 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.341 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:11:10.341 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.726596+0000", 2023-10-09T06:11:10.341 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3451999999999997e-05 2023-10-09T06:11:10.341 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.342 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.342 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:11:10.342 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.726694+0000", 2023-10-09T06:11:10.342 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.8176000000000003e-05 2023-10-09T06:11:10.342 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.342 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:10.342 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.342 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.342 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.342 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.35940.0:118905 9.5 e453/450 9:abe25aca:::benchmark_data_smithi064_511911_object17803:head v 453'7396, mlcod=453'7393)", 2023-10-09T06:11:10.342 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:07.834480+0000", 2023-10-09T06:11:10.343 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 122.37592997, 2023-10-09T06:11:10.343 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.27981055300000002, 2023-10-09T06:11:10.343 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:11:10.343 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:11:10.343 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:11:10.343 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.343 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:11:10.343 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.834480+0000", 2023-10-09T06:11:10.343 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.343 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.343 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.343 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:11:10.344 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.834480+0000", 2023-10-09T06:11:10.344 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.344 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.344 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.344 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:11:10.344 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.834480+0000", 2023-10-09T06:11:10.344 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:11:10.344 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.344 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.344 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:11:10.344 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.834487+0000", 2023-10-09T06:11:10.344 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2130000000000002e-06 2023-10-09T06:11:10.345 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.345 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.345 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:11:10.345 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.834502+0000", 2023-10-09T06:11:10.345 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4545999999999999e-05 2023-10-09T06:11:10.345 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.345 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.345 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:11:10.345 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.834507+0000", 2023-10-09T06:11:10.345 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7409999999999998e-06 2023-10-09T06:11:10.345 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.345 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.346 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.346 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.090816+0000", 2023-10-09T06:11:10.346 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.25630837400000001 2023-10-09T06:11:10.346 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.346 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.346 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:11:10.346 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.090855+0000", 2023-10-09T06:11:10.346 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9369999999999997e-05 2023-10-09T06:11:10.346 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.346 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.346 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:11:10.346 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.114210+0000", 2023-10-09T06:11:10.347 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.023355483999999999 2023-10-09T06:11:10.347 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.347 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.347 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:11:10.347 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.114291+0000", 2023-10-09T06:11:10.347 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0457999999999994e-05 2023-10-09T06:11:10.347 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.347 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:10.347 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.347 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.347 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.347 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.35940.0:118910 9.5 e453/450 9:aa92e6b2:::benchmark_data_smithi064_511911_object17808:head v 453'7397, mlcod=453'7393)", 2023-10-09T06:11:10.348 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:07.834529+0000", 2023-10-09T06:11:10.348 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 122.375881474, 2023-10-09T06:11:10.348 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.279820917, 2023-10-09T06:11:10.348 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:11:10.348 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:11:10.348 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:11:10.348 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.348 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:11:10.348 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.834529+0000", 2023-10-09T06:11:10.348 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.348 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.348 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.349 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:11:10.349 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.834529+0000", 2023-10-09T06:11:10.349 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.349 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.349 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.349 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:11:10.349 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.834529+0000", 2023-10-09T06:11:10.349 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:11:10.349 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.349 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.349 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:11:10.349 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.834532+0000", 2023-10-09T06:11:10.350 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.4489999999999999e-06 2023-10-09T06:11:10.350 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.350 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.350 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:11:10.350 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.834538+0000", 2023-10-09T06:11:10.350 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.66e-06 2023-10-09T06:11:10.350 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.350 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.350 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:11:10.350 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.834542+0000", 2023-10-09T06:11:10.350 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.031e-06 2023-10-09T06:11:10.351 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.351 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.351 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.351 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.093878+0000", 2023-10-09T06:11:10.351 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.25933609099999999 2023-10-09T06:11:10.351 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.351 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.351 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:11:10.351 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.093899+0000", 2023-10-09T06:11:10.351 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1423000000000001e-05 2023-10-09T06:11:10.351 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.351 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.352 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:11:10.352 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.114296+0000", 2023-10-09T06:11:10.352 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.020396766 2023-10-09T06:11:10.352 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.352 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.352 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:11:10.352 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.114350+0000", 2023-10-09T06:11:10.352 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3763e-05 2023-10-09T06:11:10.352 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.352 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:10.352 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.352 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.353 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.353 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:132900 9.9 9:9c50d9a2:::benchmark_data_smithi064_511911_object31798:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e455)", 2023-10-09T06:11:10.353 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:26.559784+0000", 2023-10-09T06:11:10.353 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 103.65062633300001, 2023-10-09T06:11:10.353 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2243415390000001, 2023-10-09T06:11:10.353 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:11:10.353 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:11:10.353 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:11:10.353 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:11:10.353 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:11:10.353 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 132900 2023-10-09T06:11:10.353 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.354 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:11:10.354 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.354 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:11:10.354 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.559784+0000", 2023-10-09T06:11:10.354 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.354 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.354 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.354 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:11:10.354 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.559784+0000", 2023-10-09T06:11:10.354 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.354 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.354 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.355 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:11:10.355 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.559783+0000", 2023-10-09T06:11:10.355 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:11:10.355 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.355 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.355 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:11:10.355 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.559789+0000", 2023-10-09T06:11:10.355 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5690000000000001e-06 2023-10-09T06:11:10.355 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.355 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.355 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:11:10.355 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.559800+0000", 2023-10-09T06:11:10.356 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0288e-05 2023-10-09T06:11:10.356 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.356 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.356 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:11:10.356 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.562277+0000", 2023-10-09T06:11:10.356 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002477103 2023-10-09T06:11:10.356 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.356 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.356 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.356 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.565865+0000", 2023-10-09T06:11:10.356 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0035886989999999999 2023-10-09T06:11:10.356 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.357 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.357 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:11:10.357 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.566006+0000", 2023-10-09T06:11:10.357 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014012900000000001 2023-10-09T06:11:10.357 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.357 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.357 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:11:10.357 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.566132+0000", 2023-10-09T06:11:10.357 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012682599999999999 2023-10-09T06:11:10.357 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.357 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.358 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.358 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572376+0000", 2023-10-09T06:11:10.358 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0062437980000000001 2023-10-09T06:11:10.358 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.358 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.358 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:11:10.358 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572387+0000", 2023-10-09T06:11:10.358 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0492999999999999e-05 2023-10-09T06:11:10.358 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.358 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.358 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.358 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.587508+0000", 2023-10-09T06:11:10.359 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015120971 2023-10-09T06:11:10.359 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.359 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.359 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:11:10.359 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.587518+0000", 2023-10-09T06:11:10.359 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0197000000000001e-05 2023-10-09T06:11:10.359 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.359 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.359 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.359 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.761187+0000", 2023-10-09T06:11:10.359 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.17366953 2023-10-09T06:11:10.359 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.360 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.360 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for missing object", 2023-10-09T06:11:10.360 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.761345+0000", 2023-10-09T06:11:10.360 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015735400000000001 2023-10-09T06:11:10.360 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.360 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.360 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.360 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.784048+0000", 2023-10-09T06:11:10.360 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.022703204000000001 2023-10-09T06:11:10.360 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.360 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.361 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:11:10.361 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.784126+0000", 2023-10-09T06:11:10.361 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7649000000000001e-05 2023-10-09T06:11:10.361 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.361 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:10.361 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.361 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.361 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.361 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:132902 9.9 9:90d35044:::benchmark_data_smithi064_511911_object31800:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e455)", 2023-10-09T06:11:10.361 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:26.562296+0000", 2023-10-09T06:11:10.361 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 103.64811433, 2023-10-09T06:11:10.361 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2265417860000001, 2023-10-09T06:11:10.362 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:11:10.362 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:11:10.362 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:11:10.362 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:11:10.362 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:11:10.362 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 132902 2023-10-09T06:11:10.362 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.362 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:11:10.362 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.362 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:11:10.362 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.562296+0000", 2023-10-09T06:11:10.362 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.363 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.363 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.363 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:11:10.363 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.562296+0000", 2023-10-09T06:11:10.363 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:10.363 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.363 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.363 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:11:10.363 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.562295+0000", 2023-10-09T06:11:10.363 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:11:10.363 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.363 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.365 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:11:10.365 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.562300+0000", 2023-10-09T06:11:10.365 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5339999999999999e-06 2023-10-09T06:11:10.365 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.365 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.365 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:11:10.365 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.562309+0000", 2023-10-09T06:11:10.365 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3470000000000005e-06 2023-10-09T06:11:10.366 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.366 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.366 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:11:10.366 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.562314+0000", 2023-10-09T06:11:10.366 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7299999999999996e-06 2023-10-09T06:11:10.366 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.366 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.366 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.366 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.569303+0000", 2023-10-09T06:11:10.366 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0069888600000000004 2023-10-09T06:11:10.366 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.366 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.367 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:11:10.367 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.569430+0000", 2023-10-09T06:11:10.367 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012732600000000001 2023-10-09T06:11:10.367 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.367 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.367 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:11:10.367 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.569519+0000", 2023-10-09T06:11:10.367 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8524999999999997e-05 2023-10-09T06:11:10.367 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.367 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.367 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:11:10.367 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.582104+0000", 2023-10-09T06:11:10.368 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012585202 2023-10-09T06:11:10.368 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.368 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.368 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:11:10.368 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.582122+0000", 2023-10-09T06:11:10.368 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8048999999999999e-05 2023-10-09T06:11:10.368 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.368 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.368 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.368 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.587546+0000", 2023-10-09T06:11:10.368 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0054235079999999996 2023-10-09T06:11:10.368 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.369 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.369 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:11:10.369 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.587553+0000", 2023-10-09T06:11:10.369 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.99e-06 2023-10-09T06:11:10.369 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.369 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.369 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.369 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.761400+0000", 2023-10-09T06:11:10.369 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.173847133 2023-10-09T06:11:10.369 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.369 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.369 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for missing object", 2023-10-09T06:11:10.370 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.761507+0000", 2023-10-09T06:11:10.370 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000106971 2023-10-09T06:11:10.370 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.370 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.370 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:10.370 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.788758+0000", 2023-10-09T06:11:10.370 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.027251068999999999 2023-10-09T06:11:10.370 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:10.370 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:10.370 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:11:10.370 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.788838+0000", 2023-10-09T06:11:10.371 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9989999999999998e-05 2023-10-09T06:11:10.371 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.371 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:10.371 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.371 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:10.371 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:10.371 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:11:10.371 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:11:10.434 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:10.691 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:11:10.717 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:10 smithi064 ceph-mon[212750]: pgmap v1217: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 637 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 6209/30627 objects degraded (20.273%); 6409/30627 objects misplaced (20.926%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T06:11:10.717 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:10 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/298144852' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:11:10.717 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:10 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:11:10.717 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:10 smithi064 ceph-mon[220044]: pgmap v1217: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 637 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 6209/30627 objects degraded (20.273%); 6409/30627 objects misplaced (20.926%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T06:11:10.717 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:10 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/298144852' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:11:10.717 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:10 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:11:11.125 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:10 smithi156 ceph-mon[195101]: pgmap v1217: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 637 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 6209/30627 objects degraded (20.273%); 6409/30627 objects misplaced (20.926%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T06:11:11.126 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:10 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/298144852' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:11:11.126 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:10 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:11:11.149 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:11:11.174 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:11:11.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:11.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:11:11.289 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:11:11.318 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:11.623 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:11:11.774 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:11:11.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:11.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:11:11.867 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:11 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:11:11.867 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:11 smithi064 ceph-mon[220044]: failed to encode map e480 with expected crc 2023-10-09T06:11:11.867 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:11 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:11:11.867 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:11 smithi064 ceph-mon[220044]: osdmap e480: 8 total, 8 up, 8 in 2023-10-09T06:11:11.867 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:11 smithi064 ceph-mon[220044]: failed to encode map e480 with expected crc 2023-10-09T06:11:11.867 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:11 smithi064 ceph-mon[220044]: failed to encode map e480 with expected crc 2023-10-09T06:11:11.867 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:11:11.868 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:11 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:11:11.868 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:11 smithi064 ceph-mon[212750]: failed to encode map e480 with expected crc 2023-10-09T06:11:11.868 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:11 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:11:11.868 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:11 smithi064 ceph-mon[212750]: osdmap e480: 8 total, 8 up, 8 in 2023-10-09T06:11:11.868 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:11 smithi064 ceph-mon[212750]: failed to encode map e480 with expected crc 2023-10-09T06:11:11.868 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:11 smithi064 ceph-mon[212750]: failed to encode map e480 with expected crc 2023-10-09T06:11:11.886 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:11:11.925 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:12.053 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:11 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:11:12.054 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:11 smithi156 ceph-mon[195101]: failed to encode map e480 with expected crc 2023-10-09T06:11:12.054 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:11 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:11:12.054 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:11 smithi156 ceph-mon[195101]: osdmap e480: 8 total, 8 up, 8 in 2023-10-09T06:11:12.054 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:11 smithi156 ceph-mon[195101]: failed to encode map e480 with expected crc 2023-10-09T06:11:12.054 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:11 smithi156 ceph-mon[195101]: failed to encode map e480 with expected crc 2023-10-09T06:11:12.659 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:11:12.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:12.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:11:12.765 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:11:12.798 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:12.816 INFO:teuthology.orchestra.run.smithi064.stderr:Error EINVAL: specified pg_num 61 < pg_num_max 32 2023-10-09T06:11:12.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:12 smithi064 ceph-mon[212750]: pgmap v1219: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 537 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 5928/25800 objects degraded (22.977%); 5743/25800 objects misplaced (22.260%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:11:12.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:12 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 5928/25800 objects degraded (22.977%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:12.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:12 smithi064 ceph-mon[220044]: pgmap v1219: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 537 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 5928/25800 objects degraded (22.977%); 5743/25800 objects misplaced (22.260%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:11:12.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:12 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 5928/25800 objects degraded (22.977%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:12.875 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:12 smithi156 ceph-mon[195101]: pgmap v1219: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 537 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 5928/25800 objects degraded (22.977%); 5743/25800 objects misplaced (22.260%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:11:12.875 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:12 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 5928/25800 objects degraded (22.977%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:13.072 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:11:13.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:13.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:11:13.179 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:11:13.210 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:13.251 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:11:13.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:13.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:11:13.355 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:11:13.383 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:13.479 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:11:13.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:13.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:11:13.583 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:11:13.609 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:13.695 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:11:13.695 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:11:13.695 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.696 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.35940.0:196824 9.1 e480/450 9:8899c7df:::benchmark_data_smithi064_511911_object95722:head v 480'12267, mlcod=480'12265)", 2023-10-09T06:11:13.696 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:11:13.689939+0000", 2023-10-09T06:11:13.696 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0050387929999999997, 2023-10-09T06:11:13.696 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0050736610000000001, 2023-10-09T06:11:13.696 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:11:13.696 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:11:13.696 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:11:13.696 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.696 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:11:13.696 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.689939+0000", 2023-10-09T06:11:13.696 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:13.696 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.697 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.697 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:11:13.697 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.689939+0000", 2023-10-09T06:11:13.697 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:13.697 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.697 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.697 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:11:13.697 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.689939+0000", 2023-10-09T06:11:13.697 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:11:13.697 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.697 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.697 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:11:13.698 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.689944+0000", 2023-10-09T06:11:13.698 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1089999999999997e-06 2023-10-09T06:11:13.698 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.698 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.698 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:11:13.698 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.689950+0000", 2023-10-09T06:11:13.698 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6930000000000004e-06 2023-10-09T06:11:13.698 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.698 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.698 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:11:13.698 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.689954+0000", 2023-10-09T06:11:13.698 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7900000000000001e-06 2023-10-09T06:11:13.699 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.699 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.699 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:13.699 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.689996+0000", 2023-10-09T06:11:13.699 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2611000000000002e-05 2023-10-09T06:11:13.699 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.699 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.699 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:11:13.699 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.690018+0000", 2023-10-09T06:11:13.699 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1335000000000002e-05 2023-10-09T06:11:13.699 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:13.699 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:13.700 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:13.700 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.700 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.700 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.35940.0:196826 9.1 e480/450 9:86cf2084:::benchmark_data_smithi064_511911_object95724:head v 480'12268, mlcod=480'12265)", 2023-10-09T06:11:13.700 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:11:13.692604+0000", 2023-10-09T06:11:13.700 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0023736819999999998, 2023-10-09T06:11:13.700 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0025040539999999999, 2023-10-09T06:11:13.700 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:11:13.700 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:11:13.700 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:11:13.700 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.700 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:11:13.701 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.692604+0000", 2023-10-09T06:11:13.701 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:13.701 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.701 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.701 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:11:13.701 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.692604+0000", 2023-10-09T06:11:13.701 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:13.701 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.701 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.701 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:11:13.701 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.692604+0000", 2023-10-09T06:11:13.701 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:11:13.701 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.702 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.702 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:11:13.702 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.692614+0000", 2023-10-09T06:11:13.702 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0648999999999999e-05 2023-10-09T06:11:13.702 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.702 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.702 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:11:13.702 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.692624+0000", 2023-10-09T06:11:13.702 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.8209999999999997e-06 2023-10-09T06:11:13.702 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.702 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.702 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:11:13.703 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.692630+0000", 2023-10-09T06:11:13.703 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1519999999999997e-06 2023-10-09T06:11:13.703 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.703 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.703 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:13.703 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.692687+0000", 2023-10-09T06:11:13.703 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6425999999999998e-05 2023-10-09T06:11:13.703 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.703 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.703 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:11:13.703 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.692706+0000", 2023-10-09T06:11:13.703 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9100999999999999e-05 2023-10-09T06:11:13.703 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:13.704 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:13.704 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:13.704 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.704 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.704 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.35940.0:196821 9.4 e480/425 9:25edef8f:::benchmark_data_smithi064_511911_object95719:head v 480'12333, mlcod=480'12331)", 2023-10-09T06:11:13.704 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:11:13.693714+0000", 2023-10-09T06:11:13.704 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0012638930000000001, 2023-10-09T06:11:13.704 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0014550679999999999, 2023-10-09T06:11:13.704 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:11:13.704 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:11:13.704 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:11:13.704 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.705 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:11:13.705 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.693714+0000", 2023-10-09T06:11:13.705 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:13.705 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.705 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.705 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:11:13.705 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.693714+0000", 2023-10-09T06:11:13.705 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:13.705 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.705 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.705 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:11:13.705 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.693713+0000", 2023-10-09T06:11:13.705 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:11:13.706 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.706 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.706 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:11:13.706 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.693726+0000", 2023-10-09T06:11:13.706 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2624e-05 2023-10-09T06:11:13.706 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.706 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.706 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:11:13.706 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.693741+0000", 2023-10-09T06:11:13.706 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5153999999999999e-05 2023-10-09T06:11:13.706 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.707 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.707 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:11:13.707 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.693747+0000", 2023-10-09T06:11:13.707 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0009999999999998e-06 2023-10-09T06:11:13.707 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.707 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.707 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:13.707 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.693813+0000", 2023-10-09T06:11:13.707 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5480000000000003e-05 2023-10-09T06:11:13.707 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.707 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.707 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:11:13.708 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.693833+0000", 2023-10-09T06:11:13.708 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0244000000000001e-05 2023-10-09T06:11:13.708 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:13.708 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:13.708 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:13.708 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.708 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.708 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.35940.0:196827 9.1 e480/450 9:8273fb17:::benchmark_data_smithi064_511911_object95725:head v 480'12269, mlcod=480'12265)", 2023-10-09T06:11:13.708 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:11:13.694372+0000", 2023-10-09T06:11:13.708 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.00060603100000000004, 2023-10-09T06:11:13.708 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00085635099999999999, 2023-10-09T06:11:13.709 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:11:13.709 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:11:13.709 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:11:13.709 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.709 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:11:13.709 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.694372+0000", 2023-10-09T06:11:13.709 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:13.709 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.709 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.709 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:11:13.709 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.694372+0000", 2023-10-09T06:11:13.709 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:11:13.710 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.710 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.710 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:11:13.710 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.694371+0000", 2023-10-09T06:11:13.710 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:11:13.710 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.710 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.710 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:11:13.710 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.694383+0000", 2023-10-09T06:11:13.710 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1827999999999999e-05 2023-10-09T06:11:13.710 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.711 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.711 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:11:13.711 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.694396+0000", 2023-10-09T06:11:13.711 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3237999999999999e-05 2023-10-09T06:11:13.711 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.711 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.711 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:11:13.711 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.694402+0000", 2023-10-09T06:11:13.711 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6289999999999998e-06 2023-10-09T06:11:13.711 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.711 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.712 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:11:13.712 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.694837+0000", 2023-10-09T06:11:13.712 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00043480599999999998 2023-10-09T06:11:13.712 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:11:13.712 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:11:13.712 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:11:13.712 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:13.694861+0000", 2023-10-09T06:11:13.712 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4133000000000001e-05 2023-10-09T06:11:13.712 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:13.712 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:11:13.712 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:13.712 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:11:13.713 INFO:teuthology.orchestra.run.smithi156.stdout: ], 2023-10-09T06:11:13.713 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 4 2023-10-09T06:11:13.713 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:11:13.713 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:11:13.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:13.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:11:13.815 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:11:13.873 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:13.877 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:13 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1175166969' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-10-09T06:11:13.877 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:13 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-10-09T06:11:13.877 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:13 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:13.903 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:11:13.934 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:11:13.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:13.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:11:13.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:13 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1175166969' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-10-09T06:11:13.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:13 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-10-09T06:11:13.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:13 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:13.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:13 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1175166969' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-10-09T06:11:13.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:13 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-10-09T06:11:13.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:13 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:14.038 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:11:14.098 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:14.318 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:11:14.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:14.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:11:14.427 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:11:14.456 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:14.462 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T06:11:14.526 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:11:14.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:14.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:11:14.536 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:11:14.634 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:11:14.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:14 smithi064 ceph-mon[220044]: pgmap v1220: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 454 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 5826/21810 objects degraded (26.713%); 5743/21810 objects misplaced (26.332%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:11:14.755 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:14.762 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:11:14.938 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:11:14.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:14.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:11:14.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:14 smithi064 ceph-mon[212750]: pgmap v1220: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 454 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 5826/21810 objects degraded (26.713%); 5743/21810 objects misplaced (26.332%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:11:15.047 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:11:15.086 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:11:15.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:14 smithi156 ceph-mon[195101]: pgmap v1220: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 454 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 5826/21810 objects degraded (26.713%); 5743/21810 objects misplaced (26.332%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:11:15.123 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:15.285 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:11:15.318 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:11:15.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:15.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:11:15.339 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:11:15.374 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:11:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:11:15] "GET /metrics HTTP/1.1" 200 36467 "" "Prometheus/2.43.0" 2023-10-09T06:11:15.382 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:11:15.423 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:11:15.434 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:11:15.460 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:11:15.504 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:15.510 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:11:15.562 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:11:15.607 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:11:15.630 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:11:15.630 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:11:15.630 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:11:15.630 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:11:15.630 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:11:15.630 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:11:15.631 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:11:15.631 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:11:15.825 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:15 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/991402663' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:11:15.826 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:15 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/991402663' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:11:16.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:15 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/991402663' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:11:16.595 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:11:16.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:16.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:11:16.706 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:11:16.760 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:16.886 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:11:16.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:16.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:11:16.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:16 smithi064 ceph-mon[212750]: pgmap v1221: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 429 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 5757/20640 objects degraded (27.892%); 5743/20640 objects misplaced (27.825%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:11:16.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:16 smithi064 ceph-mon[220044]: pgmap v1221: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 429 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 5757/20640 objects degraded (27.892%); 5743/20640 objects misplaced (27.825%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:11:16.990 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:11:17.014 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:11:17.014 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:11:17.038 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:16 smithi156 ceph-mon[195101]: pgmap v1221: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 429 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 5757/20640 objects degraded (27.892%); 5743/20640 objects misplaced (27.825%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:11:17.068 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:17.891 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:11:17.980 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:11:17.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:17.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:11:18.099 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:11:18.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:17 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 5757/20640 objects degraded (27.892%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:18.169 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:17 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 5757/20640 objects degraded (27.892%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:18.170 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:17 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 5757/20640 objects degraded (27.892%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:18.181 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:18.388 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:11:18.388 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:11:18.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:18.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:11:18.496 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:11:18.540 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:18.657 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:11:18.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:18.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:11:18.764 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:11:18.801 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:18 smithi156 ceph-mon[195101]: pgmap v1222: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 257 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 5445/12375 objects degraded (44.000%); 5131/12375 objects misplaced (41.463%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T06:11:18.812 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:19.025 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:11:19.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:19.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:11:19.129 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:11:19.201 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:18 smithi064 ceph-mon[220044]: pgmap v1222: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 257 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 5445/12375 objects degraded (44.000%); 5131/12375 objects misplaced (41.463%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T06:11:19.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:18 smithi064 ceph-mon[212750]: pgmap v1222: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 257 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 5445/12375 objects degraded (44.000%); 5131/12375 objects misplaced (41.463%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T06:11:19.207 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:19.333 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:11:19.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:19.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:11:19.446 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:11:19.463 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1, 2] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 4, 0] 2023-10-09T06:11:19.463 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:11:19.463 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T06:11:19.464 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-10-09T06:11:19.496 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:19.674 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:11:19.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:19.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:11:19.781 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:11:19.853 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:11:19.887 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:19.991 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:11:20.062 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:19 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4233212096' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:11:20.062 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:19 smithi064 ceph-mon[220044]: pgmap v1223: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 245 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 5445/11808 objects degraded (46.113%); 5131/11808 objects misplaced (43.454%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T06:11:20.063 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:19 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4233212096' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:11:20.063 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:19 smithi064 ceph-mon[212750]: pgmap v1223: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 245 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 5445/11808 objects degraded (46.113%); 5131/11808 objects misplaced (43.454%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T06:11:20.073 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:11:20.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:20.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:11:20.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:19 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4233212096' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:11:20.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:19 smithi156 ceph-mon[195101]: pgmap v1223: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 245 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 5445/11808 objects degraded (46.113%); 5131/11808 objects misplaced (43.454%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T06:11:20.182 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:11:20.218 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:11:20.219 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:11:20.230 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:20.920 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:11:20.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:20.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:11:21.029 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:11:21.057 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:20 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:11:21.057 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:20 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4233212096' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:11:21.058 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:20 smithi156 ceph-mon[195101]: osdmap e481: 8 total, 8 up, 8 in 2023-10-09T06:11:21.058 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:20 smithi156 ceph-mon[195101]: failed to encode map e481 with expected crc 2023-10-09T06:11:21.058 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:20 smithi156 ceph-mon[195101]: failed to encode map e481 with expected crc 2023-10-09T06:11:21.058 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:20 smithi156 ceph-mon[195101]: failed to encode map e481 with expected crc 2023-10-09T06:11:21.061 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:21.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:20 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:11:21.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:20 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4233212096' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:11:21.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:20 smithi064 ceph-mon[212750]: osdmap e481: 8 total, 8 up, 8 in 2023-10-09T06:11:21.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:20 smithi064 ceph-mon[212750]: failed to encode map e481 with expected crc 2023-10-09T06:11:21.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:20 smithi064 ceph-mon[212750]: failed to encode map e481 with expected crc 2023-10-09T06:11:21.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:20 smithi064 ceph-mon[212750]: failed to encode map e481 with expected crc 2023-10-09T06:11:21.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:20 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:11:21.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:20 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4233212096' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:11:21.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:20 smithi064 ceph-mon[220044]: osdmap e481: 8 total, 8 up, 8 in 2023-10-09T06:11:21.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:20 smithi064 ceph-mon[220044]: failed to encode map e481 with expected crc 2023-10-09T06:11:21.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:20 smithi064 ceph-mon[220044]: failed to encode map e481 with expected crc 2023-10-09T06:11:21.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:20 smithi064 ceph-mon[220044]: failed to encode map e481 with expected crc 2023-10-09T06:11:21.334 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:11:21.349 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:11:21.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:21.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:11:21.450 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:11:21.478 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:21.547 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:11:21.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:21.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:11:21.654 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:11:21.776 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:21.862 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:11:22.023 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:11:22.024 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Removed 101099 objects 2023-10-09T06:11:22.024 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Clean up completed and total clean up time :158.194 2023-10-09T06:11:22.025 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:21 smithi064 ceph-mon[212750]: failed to encode map e481 with expected crc 2023-10-09T06:11:22.025 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:21 smithi064 ceph-mon[212750]: failed to encode map e481 with expected crc 2023-10-09T06:11:22.025 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:21 smithi064 ceph-mon[212750]: pgmap v1225: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 139 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 5230/6693 objects degraded (78.141%); 4446/6693 objects misplaced (66.428%); 2.1 MiB/s, 43 objects/s recovering 2023-10-09T06:11:22.025 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:21 smithi064 ceph-mon[220044]: failed to encode map e481 with expected crc 2023-10-09T06:11:22.025 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:21 smithi064 ceph-mon[220044]: failed to encode map e481 with expected crc 2023-10-09T06:11:22.025 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:21 smithi064 ceph-mon[220044]: pgmap v1225: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 139 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 5230/6693 objects degraded (78.141%); 4446/6693 objects misplaced (66.428%); 2.1 MiB/s, 43 objects/s recovering 2023-10-09T06:11:22.076 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:11:22.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:21 smithi156 ceph-mon[195101]: failed to encode map e481 with expected crc 2023-10-09T06:11:22.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:21 smithi156 ceph-mon[195101]: failed to encode map e481 with expected crc 2023-10-09T06:11:22.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:21 smithi156 ceph-mon[195101]: pgmap v1225: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 139 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 5230/6693 objects degraded (78.141%); 4446/6693 objects misplaced (66.428%); 2.1 MiB/s, 43 objects/s recovering 2023-10-09T06:11:22.158 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:11:22.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:22.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:11:22.263 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:11:22.291 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:22.335 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:11:22.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:22.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:11:22.440 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:11:22.467 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:22.509 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:11:22.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:22.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:11:22.616 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:11:22.648 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:23.050 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:11:23.051 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:22 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 5230/6693 objects degraded (78.141%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:23.052 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:22 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 5230/6693 objects degraded (78.141%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:23.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:23.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:11:23.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:22 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 5230/6693 objects degraded (78.141%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:23.155 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:11:23.183 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:23.226 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:11:23.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:23.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:11:23.331 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:11:23.361 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:23.495 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-10-09T06:11:23.495 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-10-09T06:11:23.727 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:11:24.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:23 smithi156 ceph-mon[195101]: pgmap v1226: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 48 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 5133/2358 objects degraded (217.684%); 4446/2358 objects misplaced (188.550%); 2.1 MiB/s, 43 objects/s recovering 2023-10-09T06:11:24.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:23 smithi064 ceph-mon[212750]: pgmap v1226: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 48 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 5133/2358 objects degraded (217.684%); 4446/2358 objects misplaced (188.550%); 2.1 MiB/s, 43 objects/s recovering 2023-10-09T06:11:24.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:23 smithi064 ceph-mon[220044]: pgmap v1226: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 48 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 5133/2358 objects degraded (217.684%); 4446/2358 objects misplaced (188.550%); 2.1 MiB/s, 43 objects/s recovering 2023-10-09T06:11:25.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:11:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:11:25] "GET /metrics HTTP/1.1" 200 36467 "" "Prometheus/2.43.0" 2023-10-09T06:11:26.205 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:11:26.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:26.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:11:26.309 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:11:26.339 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:26.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:26 smithi064 ceph-mon[212750]: pgmap v1227: 119 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 35 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 5112/1734 objects degraded (294.810%); 4446/1734 objects misplaced (256.401%); 2.1 MiB/s, 43 objects/s recovering 2023-10-09T06:11:26.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:26 smithi064 ceph-mon[220044]: pgmap v1227: 119 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 35 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 5112/1734 objects degraded (294.810%); 4446/1734 objects misplaced (256.401%); 2.1 MiB/s, 43 objects/s recovering 2023-10-09T06:11:26.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:26 smithi156 ceph-mon[195101]: pgmap v1227: 119 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 35 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 5112/1734 objects degraded (294.810%); 4446/1734 objects misplaced (256.401%); 2.1 MiB/s, 43 objects/s recovering 2023-10-09T06:11:27.039 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:11:27.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:27.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:11:27.145 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:11:27.193 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:27.257 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:11:27.268 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:11:27.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:27.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:11:27.355 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:11:27.375 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:11:27.409 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:27.481 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:11:27.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:27.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:11:27.585 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:11:27.622 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:27.848 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:11:27.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:27.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:11:27.877 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:11:27.877 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:11:27.952 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:11:27.983 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:28.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:28 smithi064 ceph-mon[212750]: pgmap v1228: 119 pgs: 2 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 726 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 409 op/s; 5100/72 objects degraded (7083.333%); 4263/72 objects misplaced (5920.833%); 883 KiB/s, 81 objects/s recovering 2023-10-09T06:11:28.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:28 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 5100/72 objects degraded (7083.333%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:28.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:28 smithi064 ceph-mon[220044]: pgmap v1228: 119 pgs: 2 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 726 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 409 op/s; 5100/72 objects degraded (7083.333%); 4263/72 objects misplaced (5920.833%); 883 KiB/s, 81 objects/s recovering 2023-10-09T06:11:28.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:28 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 5100/72 objects degraded (7083.333%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:28.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:28 smithi156 ceph-mon[195101]: pgmap v1228: 119 pgs: 2 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 726 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 409 op/s; 5100/72 objects degraded (7083.333%); 4263/72 objects misplaced (5920.833%); 883 KiB/s, 81 objects/s recovering 2023-10-09T06:11:28.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:28 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 5100/72 objects degraded (7083.333%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:29.153 INFO:teuthology.orchestra.run.smithi064.stdout:moved 20 / 357 (5.60224%) 2023-10-09T06:11:29.153 INFO:teuthology.orchestra.run.smithi064.stdout:avg 44.625 2023-10-09T06:11:29.153 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 13.7562 -> 17.1751 (expected baseline 6.24875) 2023-10-09T06:11:29.154 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.7 with 22 -> 16 pgs (0.492997 -> 0.358543 * mean) 2023-10-09T06:11:29.154 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.1 with 62 -> 65 pgs (1.38936 -> 1.45658 * mean) 2023-10-09T06:11:29.154 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:11:29.154 INFO:teuthology.orchestra.run.smithi064.stdout:oload 110 2023-10-09T06:11:29.154 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 3 2023-10-09T06:11:29.154 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-10-09T06:11:29.154 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0081 2023-10-09T06:11:29.154 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.0089 2023-10-09T06:11:29.154 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 weight 0.6106 -> 0.4400 2023-10-09T06:11:29.154 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 weight 0.4543 -> 0.3595 2023-10-09T06:11:29.155 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 weight 0.7847 -> 0.6564 2023-10-09T06:11:29.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:29 smithi064 ceph-mon[212750]: from='client.36192 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:29.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'5':'43016','6':'28834','7':'23562'}"}]: dispatch 2023-10-09T06:11:29.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:29 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'5':'43016','6':'28834','7':'23562'}"}]: dispatch 2023-10-09T06:11:29.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:29.424 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:29 smithi064 ceph-mon[220044]: from='client.36192 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:29.424 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'5':'43016','6':'28834','7':'23562'}"}]: dispatch 2023-10-09T06:11:29.424 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:29 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'5':'43016','6':'28834','7':'23562'}"}]: dispatch 2023-10-09T06:11:29.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:29.538 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:11:29.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:29.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:11:29.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:29 smithi156 ceph-mon[195101]: from='client.36192 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:29.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'5':'43016','6':'28834','7':'23562'}"}]: dispatch 2023-10-09T06:11:29.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:29 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'5':'43016','6':'28834','7':'23562'}"}]: dispatch 2023-10-09T06:11:29.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:29.644 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:11:29.676 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:29.724 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:11:29.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:29.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:11:29.832 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:11:29.868 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:30.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:30 smithi064 ceph-mon[212750]: pgmap v1229: 119 pgs: 2 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 726 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 391 op/s; 5100/72 objects degraded (7083.333%); 4263/72 objects misplaced (5920.833%); 883 KiB/s, 81 objects/s recovering 2023-10-09T06:11:30.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:30 smithi064 ceph-mon[212750]: failed to encode map e482 with expected crc 2023-10-09T06:11:30.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:30 smithi064 ceph-mon[212750]: failed to encode map e482 with expected crc 2023-10-09T06:11:30.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:30 smithi064 ceph-mon[212750]: failed to encode map e482 with expected crc 2023-10-09T06:11:30.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:30 smithi064 ceph-mon[212750]: failed to encode map e482 with expected crc 2023-10-09T06:11:30.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:30 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'5':'43016','6':'28834','7':'23562'}"}]': finished 2023-10-09T06:11:30.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:30 smithi064 ceph-mon[212750]: osdmap e482: 8 total, 8 up, 8 in 2023-10-09T06:11:30.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:30 smithi064 ceph-mon[212750]: failed to encode map e482 with expected crc 2023-10-09T06:11:30.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:30 smithi064 ceph-mon[220044]: pgmap v1229: 119 pgs: 2 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 726 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 391 op/s; 5100/72 objects degraded (7083.333%); 4263/72 objects misplaced (5920.833%); 883 KiB/s, 81 objects/s recovering 2023-10-09T06:11:30.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:30 smithi064 ceph-mon[220044]: failed to encode map e482 with expected crc 2023-10-09T06:11:30.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:30 smithi064 ceph-mon[220044]: failed to encode map e482 with expected crc 2023-10-09T06:11:30.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:30 smithi064 ceph-mon[220044]: failed to encode map e482 with expected crc 2023-10-09T06:11:30.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:30 smithi064 ceph-mon[220044]: failed to encode map e482 with expected crc 2023-10-09T06:11:30.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:30 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'5':'43016','6':'28834','7':'23562'}"}]': finished 2023-10-09T06:11:30.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:30 smithi064 ceph-mon[220044]: osdmap e482: 8 total, 8 up, 8 in 2023-10-09T06:11:30.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:30 smithi064 ceph-mon[220044]: failed to encode map e482 with expected crc 2023-10-09T06:11:30.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:30 smithi156 ceph-mon[195101]: pgmap v1229: 119 pgs: 2 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 111 active+clean; 726 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 391 op/s; 5100/72 objects degraded (7083.333%); 4263/72 objects misplaced (5920.833%); 883 KiB/s, 81 objects/s recovering 2023-10-09T06:11:30.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:30 smithi156 ceph-mon[195101]: failed to encode map e482 with expected crc 2023-10-09T06:11:30.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:30 smithi156 ceph-mon[195101]: failed to encode map e482 with expected crc 2023-10-09T06:11:30.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:30 smithi156 ceph-mon[195101]: failed to encode map e482 with expected crc 2023-10-09T06:11:30.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:30 smithi156 ceph-mon[195101]: failed to encode map e482 with expected crc 2023-10-09T06:11:30.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:30 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'5':'43016','6':'28834','7':'23562'}"}]': finished 2023-10-09T06:11:30.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:30 smithi156 ceph-mon[195101]: osdmap e482: 8 total, 8 up, 8 in 2023-10-09T06:11:30.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:30 smithi156 ceph-mon[195101]: failed to encode map e482 with expected crc 2023-10-09T06:11:30.721 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:11:30.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:30.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:11:30.825 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:11:30.852 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:30.898 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:11:30.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:30.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:11:31.006 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:11:31.035 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:31.134 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:11:31.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:31.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:11:31.238 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:11:31.268 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:31.454 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:11:31.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:31 smithi064 ceph-mon[212750]: 9.1 starting backfill to osd.2 from (0'0,0'0] MAX to 481'12586 2023-10-09T06:11:31.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:31 smithi064 ceph-mon[212750]: osdmap e483: 8 total, 8 up, 8 in 2023-10-09T06:11:31.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:31 smithi064 ceph-mon[212750]: failed to encode map e483 with expected crc 2023-10-09T06:11:31.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:31 smithi064 ceph-mon[212750]: failed to encode map e483 with expected crc 2023-10-09T06:11:31.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:31 smithi064 ceph-mon[212750]: failed to encode map e483 with expected crc 2023-10-09T06:11:31.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:31 smithi064 ceph-mon[212750]: failed to encode map e483 with expected crc 2023-10-09T06:11:31.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:31 smithi064 ceph-mon[212750]: failed to encode map e483 with expected crc 2023-10-09T06:11:31.458 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:31 smithi064 ceph-mon[212750]: failed to encode map e483 with expected crc 2023-10-09T06:11:31.458 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:31 smithi064 ceph-mon[212750]: 9.1 continuing backfill to osd.4 from (454'7735,481'12586] 9:8c5a9b8b:::benchmark_data_smithi064_511911_object95062:head to 481'12586 2023-10-09T06:11:31.458 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:31 smithi064 ceph-mon[220044]: 9.1 starting backfill to osd.2 from (0'0,0'0] MAX to 481'12586 2023-10-09T06:11:31.458 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:31 smithi064 ceph-mon[220044]: osdmap e483: 8 total, 8 up, 8 in 2023-10-09T06:11:31.458 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:31 smithi064 ceph-mon[220044]: failed to encode map e483 with expected crc 2023-10-09T06:11:31.458 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:31 smithi064 ceph-mon[220044]: failed to encode map e483 with expected crc 2023-10-09T06:11:31.458 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:31 smithi064 ceph-mon[220044]: failed to encode map e483 with expected crc 2023-10-09T06:11:31.458 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:31 smithi064 ceph-mon[220044]: failed to encode map e483 with expected crc 2023-10-09T06:11:31.459 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:31 smithi064 ceph-mon[220044]: failed to encode map e483 with expected crc 2023-10-09T06:11:31.459 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:31 smithi064 ceph-mon[220044]: failed to encode map e483 with expected crc 2023-10-09T06:11:31.459 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:31 smithi064 ceph-mon[220044]: 9.1 continuing backfill to osd.4 from (454'7735,481'12586] 9:8c5a9b8b:::benchmark_data_smithi064_511911_object95062:head to 481'12586 2023-10-09T06:11:31.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:31.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:11:31.499 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:11:31.518 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:11:31.538 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:11:31.557 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:11:31.560 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:11:31.576 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:11:31.584 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:31 smithi156 ceph-mon[195101]: 9.1 starting backfill to osd.2 from (0'0,0'0] MAX to 481'12586 2023-10-09T06:11:31.584 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:31 smithi156 ceph-mon[195101]: osdmap e483: 8 total, 8 up, 8 in 2023-10-09T06:11:31.584 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:31 smithi156 ceph-mon[195101]: failed to encode map e483 with expected crc 2023-10-09T06:11:31.584 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:31 smithi156 ceph-mon[195101]: failed to encode map e483 with expected crc 2023-10-09T06:11:31.585 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:31 smithi156 ceph-mon[195101]: failed to encode map e483 with expected crc 2023-10-09T06:11:31.585 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:31 smithi156 ceph-mon[195101]: failed to encode map e483 with expected crc 2023-10-09T06:11:31.585 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:31 smithi156 ceph-mon[195101]: failed to encode map e483 with expected crc 2023-10-09T06:11:31.585 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:31 smithi156 ceph-mon[195101]: failed to encode map e483 with expected crc 2023-10-09T06:11:31.585 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:31 smithi156 ceph-mon[195101]: 9.1 continuing backfill to osd.4 from (454'7735,481'12586] 9:8c5a9b8b:::benchmark_data_smithi064_511911_object95062:head to 481'12586 2023-10-09T06:11:31.587 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:31.596 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:11:31.615 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:11:31.633 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:11:31.636 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:11:31.638 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:11:31.638 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:11:31.638 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:11:31.638 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:11:31.638 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:11:31.638 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:11:31.638 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:11:31.638 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:11:31.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:31.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:11:31.740 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:11:31.770 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:32.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:32 smithi064 ceph-mon[212750]: pgmap v1232: 119 pgs: 5 remapped+peering, 1 active+recovery_wait+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 107 active+clean; 752 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 99 op/s; 5488/72 objects degraded (7622.222%); 2940/72 objects misplaced (4083.333%); 10 KiB/s, 188 objects/s recovering 2023-10-09T06:11:32.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:32 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:11:32.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:32 smithi064 ceph-mon[212750]: failed to encode map e484 with expected crc 2023-10-09T06:11:32.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:32 smithi064 ceph-mon[212750]: failed to encode map e484 with expected crc 2023-10-09T06:11:32.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:32 smithi064 ceph-mon[212750]: failed to encode map e484 with expected crc 2023-10-09T06:11:32.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:32 smithi064 ceph-mon[212750]: osdmap e484: 8 total, 8 up, 8 in 2023-10-09T06:11:32.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:32 smithi064 ceph-mon[212750]: failed to encode map e484 with expected crc 2023-10-09T06:11:32.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:32 smithi064 ceph-mon[212750]: 9.5 continuing backfill to osd.1 from (443'5800,481'12622] 9:aa9971a7:::benchmark_data_smithi064_511911_object72370:head to 481'12622 2023-10-09T06:11:32.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:32 smithi064 ceph-mon[212750]: 9.5 starting backfill to osd.3 from (0'0,0'0] MAX to 481'12622 2023-10-09T06:11:32.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:32 smithi064 ceph-mon[212750]: failed to encode map e484 with expected crc 2023-10-09T06:11:32.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:32 smithi064 ceph-mon[212750]: failed to encode map e484 with expected crc 2023-10-09T06:11:32.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:32 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1287604865' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:11:32.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:32 smithi064 ceph-mon[220044]: pgmap v1232: 119 pgs: 5 remapped+peering, 1 active+recovery_wait+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 107 active+clean; 752 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 99 op/s; 5488/72 objects degraded (7622.222%); 2940/72 objects misplaced (4083.333%); 10 KiB/s, 188 objects/s recovering 2023-10-09T06:11:32.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:32 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:11:32.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:32 smithi064 ceph-mon[220044]: failed to encode map e484 with expected crc 2023-10-09T06:11:32.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:32 smithi064 ceph-mon[220044]: failed to encode map e484 with expected crc 2023-10-09T06:11:32.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:32 smithi064 ceph-mon[220044]: failed to encode map e484 with expected crc 2023-10-09T06:11:32.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:32 smithi064 ceph-mon[220044]: osdmap e484: 8 total, 8 up, 8 in 2023-10-09T06:11:32.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:32 smithi064 ceph-mon[220044]: failed to encode map e484 with expected crc 2023-10-09T06:11:32.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:32 smithi064 ceph-mon[220044]: 9.5 continuing backfill to osd.1 from (443'5800,481'12622] 9:aa9971a7:::benchmark_data_smithi064_511911_object72370:head to 481'12622 2023-10-09T06:11:32.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:32 smithi064 ceph-mon[220044]: 9.5 starting backfill to osd.3 from (0'0,0'0] MAX to 481'12622 2023-10-09T06:11:32.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:32 smithi064 ceph-mon[220044]: failed to encode map e484 with expected crc 2023-10-09T06:11:32.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:32 smithi064 ceph-mon[220044]: failed to encode map e484 with expected crc 2023-10-09T06:11:32.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:32 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1287604865' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:11:32.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:32 smithi156 ceph-mon[195101]: pgmap v1232: 119 pgs: 5 remapped+peering, 1 active+recovery_wait+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 107 active+clean; 752 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 99 op/s; 5488/72 objects degraded (7622.222%); 2940/72 objects misplaced (4083.333%); 10 KiB/s, 188 objects/s recovering 2023-10-09T06:11:32.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:32 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:11:32.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:32 smithi156 ceph-mon[195101]: failed to encode map e484 with expected crc 2023-10-09T06:11:32.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:32 smithi156 ceph-mon[195101]: failed to encode map e484 with expected crc 2023-10-09T06:11:32.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:32 smithi156 ceph-mon[195101]: failed to encode map e484 with expected crc 2023-10-09T06:11:32.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:32 smithi156 ceph-mon[195101]: osdmap e484: 8 total, 8 up, 8 in 2023-10-09T06:11:32.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:32 smithi156 ceph-mon[195101]: failed to encode map e484 with expected crc 2023-10-09T06:11:32.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:32 smithi156 ceph-mon[195101]: 9.5 continuing backfill to osd.1 from (443'5800,481'12622] 9:aa9971a7:::benchmark_data_smithi064_511911_object72370:head to 481'12622 2023-10-09T06:11:32.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:32 smithi156 ceph-mon[195101]: 9.5 starting backfill to osd.3 from (0'0,0'0] MAX to 481'12622 2023-10-09T06:11:32.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:32 smithi156 ceph-mon[195101]: failed to encode map e484 with expected crc 2023-10-09T06:11:32.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:32 smithi156 ceph-mon[195101]: failed to encode map e484 with expected crc 2023-10-09T06:11:32.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:32 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1287604865' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:11:33.182 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T06:11:33.183 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-10-09T06:11:33.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:33 smithi064 ceph-mon[212750]: failed to encode map e485 with expected crc 2023-10-09T06:11:33.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:33 smithi064 ceph-mon[212750]: failed to encode map e485 with expected crc 2023-10-09T06:11:33.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:33 smithi064 ceph-mon[212750]: osdmap e485: 8 total, 8 up, 8 in 2023-10-09T06:11:33.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:33 smithi064 ceph-mon[212750]: failed to encode map e485 with expected crc 2023-10-09T06:11:33.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:33 smithi064 ceph-mon[212750]: failed to encode map e485 with expected crc 2023-10-09T06:11:33.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:33 smithi064 ceph-mon[212750]: failed to encode map e485 with expected crc 2023-10-09T06:11:33.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:33 smithi064 ceph-mon[212750]: failed to encode map e485 with expected crc 2023-10-09T06:11:33.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:33 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 5488/72 objects degraded (7622.222%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:33.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:33 smithi064 ceph-mon[220044]: failed to encode map e485 with expected crc 2023-10-09T06:11:33.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:33 smithi064 ceph-mon[220044]: failed to encode map e485 with expected crc 2023-10-09T06:11:33.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:33 smithi064 ceph-mon[220044]: osdmap e485: 8 total, 8 up, 8 in 2023-10-09T06:11:33.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:33 smithi064 ceph-mon[220044]: failed to encode map e485 with expected crc 2023-10-09T06:11:33.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:33 smithi064 ceph-mon[220044]: failed to encode map e485 with expected crc 2023-10-09T06:11:33.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:33 smithi064 ceph-mon[220044]: failed to encode map e485 with expected crc 2023-10-09T06:11:33.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:33 smithi064 ceph-mon[220044]: failed to encode map e485 with expected crc 2023-10-09T06:11:33.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:33 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 5488/72 objects degraded (7622.222%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:33.541 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:11:33.586 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:11:33.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:33.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:11:33.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:33 smithi156 ceph-mon[195101]: failed to encode map e485 with expected crc 2023-10-09T06:11:33.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:33 smithi156 ceph-mon[195101]: failed to encode map e485 with expected crc 2023-10-09T06:11:33.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:33 smithi156 ceph-mon[195101]: osdmap e485: 8 total, 8 up, 8 in 2023-10-09T06:11:33.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:33 smithi156 ceph-mon[195101]: failed to encode map e485 with expected crc 2023-10-09T06:11:33.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:33 smithi156 ceph-mon[195101]: failed to encode map e485 with expected crc 2023-10-09T06:11:33.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:33 smithi156 ceph-mon[195101]: failed to encode map e485 with expected crc 2023-10-09T06:11:33.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:33 smithi156 ceph-mon[195101]: failed to encode map e485 with expected crc 2023-10-09T06:11:33.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:33 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 5488/72 objects degraded (7622.222%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:33.691 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:11:33.720 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:33.832 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:11:33.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:33.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:11:33.940 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:11:33.980 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:33.992 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:11:34.030 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:11:34.052 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:11:34.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:34.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:11:34.157 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:11:34.184 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:34.228 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:11:34.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:34.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:11:34.335 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:11:34.366 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:34.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:34 smithi064 ceph-mon[220044]: pgmap v1235: 119 pgs: 5 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 103 active+clean; 752 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 7 op/s; 6442/72 objects degraded (8947.222%); 954/72 objects misplaced (1325.000%); 21 KiB/s, 271 objects/s recovering 2023-10-09T06:11:34.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:34 smithi064 ceph-mon[220044]: failed to encode map e486 with expected crc 2023-10-09T06:11:34.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:34 smithi064 ceph-mon[220044]: failed to encode map e486 with expected crc 2023-10-09T06:11:34.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:34 smithi064 ceph-mon[220044]: osdmap e486: 8 total, 8 up, 8 in 2023-10-09T06:11:34.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:34 smithi064 ceph-mon[220044]: failed to encode map e486 with expected crc 2023-10-09T06:11:34.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:34 smithi064 ceph-mon[220044]: failed to encode map e486 with expected crc 2023-10-09T06:11:34.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:34 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1809555383' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:11:34.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:34 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:11:34.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:34 smithi064 ceph-mon[212750]: pgmap v1235: 119 pgs: 5 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 103 active+clean; 752 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 7 op/s; 6442/72 objects degraded (8947.222%); 954/72 objects misplaced (1325.000%); 21 KiB/s, 271 objects/s recovering 2023-10-09T06:11:34.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:34 smithi064 ceph-mon[212750]: failed to encode map e486 with expected crc 2023-10-09T06:11:34.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:34 smithi064 ceph-mon[212750]: failed to encode map e486 with expected crc 2023-10-09T06:11:34.442 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:34 smithi064 ceph-mon[212750]: osdmap e486: 8 total, 8 up, 8 in 2023-10-09T06:11:34.442 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:34 smithi064 ceph-mon[212750]: failed to encode map e486 with expected crc 2023-10-09T06:11:34.442 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:34 smithi064 ceph-mon[212750]: failed to encode map e486 with expected crc 2023-10-09T06:11:34.442 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:34 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1809555383' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:11:34.442 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:34 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:11:34.517 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:11:34.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:34.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:11:34.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:34 smithi156 ceph-mon[195101]: pgmap v1235: 119 pgs: 5 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 103 active+clean; 752 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 7 op/s; 6442/72 objects degraded (8947.222%); 954/72 objects misplaced (1325.000%); 21 KiB/s, 271 objects/s recovering 2023-10-09T06:11:34.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:34 smithi156 ceph-mon[195101]: failed to encode map e486 with expected crc 2023-10-09T06:11:34.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:34 smithi156 ceph-mon[195101]: failed to encode map e486 with expected crc 2023-10-09T06:11:34.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:34 smithi156 ceph-mon[195101]: osdmap e486: 8 total, 8 up, 8 in 2023-10-09T06:11:34.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:34 smithi156 ceph-mon[195101]: failed to encode map e486 with expected crc 2023-10-09T06:11:34.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:34 smithi156 ceph-mon[195101]: failed to encode map e486 with expected crc 2023-10-09T06:11:34.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:34 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1809555383' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:11:34.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:34 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:11:34.626 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:11:34.640 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:11:34.650 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_3' removed 2023-10-09T06:11:34.677 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:34.755 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:11:34.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:34.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:11:34.861 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:11:34.889 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:35.072 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:11:35.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:11:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:11:35] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.43.0" 2023-10-09T06:11:35.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:35 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4236122498' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:11:35.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:35 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:11:35.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:35 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:11:35.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:35 smithi064 ceph-mon[212750]: failed to encode map e487 with expected crc 2023-10-09T06:11:35.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:35 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:11:35.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:35 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:11:35.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:35 smithi064 ceph-mon[212750]: osdmap e487: 8 total, 8 up, 8 in 2023-10-09T06:11:35.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:35 smithi064 ceph-mon[212750]: failed to encode map e487 with expected crc 2023-10-09T06:11:35.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:35 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4236122498' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:11:35.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:35 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:11:35.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:35 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:11:35.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:35 smithi064 ceph-mon[220044]: failed to encode map e487 with expected crc 2023-10-09T06:11:35.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:35 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:11:35.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:35 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:11:35.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:35 smithi064 ceph-mon[220044]: osdmap e487: 8 total, 8 up, 8 in 2023-10-09T06:11:35.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:35 smithi064 ceph-mon[220044]: failed to encode map e487 with expected crc 2023-10-09T06:11:35.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:35 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4236122498' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:11:35.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:35 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:11:35.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:35 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:11:35.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:35 smithi156 ceph-mon[195101]: failed to encode map e487 with expected crc 2023-10-09T06:11:35.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:35 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:11:35.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:35 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:11:35.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:35 smithi156 ceph-mon[195101]: osdmap e487: 8 total, 8 up, 8 in 2023-10-09T06:11:35.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:35 smithi156 ceph-mon[195101]: failed to encode map e487 with expected crc 2023-10-09T06:11:35.645 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:11:35.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:35.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:11:35.753 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:11:35.782 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:36.108 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:11:36.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:36.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:11:36.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:11:36.241 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:36.286 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:11:36.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:36.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:11:36.390 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:11:36.421 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:36.423 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:36 smithi064 ceph-mon[220044]: failed to encode map e487 with expected crc 2023-10-09T06:11:36.423 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:36 smithi064 ceph-mon[220044]: failed to encode map e487 with expected crc 2023-10-09T06:11:36.423 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:36 smithi064 ceph-mon[220044]: pgmap v1238: 103 pgs: 4 remapped+peering, 4 active+undersized+remapped, 95 active+clean; 752 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:36.423 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:36 smithi064 ceph-mon[220044]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6442/72 objects degraded (8947.222%), 3 pgs degraded, 2 pgs undersized) 2023-10-09T06:11:36.423 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:36 smithi064 ceph-mon[220044]: osdmap e488: 8 total, 8 up, 8 in 2023-10-09T06:11:36.424 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:36 smithi064 ceph-mon[220044]: failed to encode map e488 with expected crc 2023-10-09T06:11:36.424 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:36 smithi064 ceph-mon[220044]: failed to encode map e488 with expected crc 2023-10-09T06:11:36.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:36 smithi064 ceph-mon[212750]: failed to encode map e487 with expected crc 2023-10-09T06:11:36.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:36 smithi064 ceph-mon[212750]: failed to encode map e487 with expected crc 2023-10-09T06:11:36.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:36 smithi064 ceph-mon[212750]: pgmap v1238: 103 pgs: 4 remapped+peering, 4 active+undersized+remapped, 95 active+clean; 752 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:36.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:36 smithi064 ceph-mon[212750]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6442/72 objects degraded (8947.222%), 3 pgs degraded, 2 pgs undersized) 2023-10-09T06:11:36.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:36 smithi064 ceph-mon[212750]: osdmap e488: 8 total, 8 up, 8 in 2023-10-09T06:11:36.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:36 smithi064 ceph-mon[212750]: failed to encode map e488 with expected crc 2023-10-09T06:11:36.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:36 smithi064 ceph-mon[212750]: failed to encode map e488 with expected crc 2023-10-09T06:11:36.470 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:11:36.474 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:36 smithi156 ceph-mon[195101]: failed to encode map e487 with expected crc 2023-10-09T06:11:36.474 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:36 smithi156 ceph-mon[195101]: failed to encode map e487 with expected crc 2023-10-09T06:11:36.474 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:36 smithi156 ceph-mon[195101]: pgmap v1238: 103 pgs: 4 remapped+peering, 4 active+undersized+remapped, 95 active+clean; 752 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:36.474 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:36 smithi156 ceph-mon[195101]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6442/72 objects degraded (8947.222%), 3 pgs degraded, 2 pgs undersized) 2023-10-09T06:11:36.474 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:36 smithi156 ceph-mon[195101]: osdmap e488: 8 total, 8 up, 8 in 2023-10-09T06:11:36.474 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:36 smithi156 ceph-mon[195101]: failed to encode map e488 with expected crc 2023-10-09T06:11:36.474 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:36 smithi156 ceph-mon[195101]: failed to encode map e488 with expected crc 2023-10-09T06:11:36.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:36.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:11:36.577 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:11:36.606 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:36.651 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:11:36.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:36.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:11:36.760 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:11:36.789 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:37.041 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:11:37.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:37.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:11:37.048 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T06:11:37.048 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T06:11:37.049 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-10-09T06:11:37.049 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-10-09T06:11:37.093 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:11:37.146 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:11:37.150 INFO:teuthology.orchestra.run.smithi064.stdout:moved 0 / 309 (0%) 2023-10-09T06:11:37.150 INFO:teuthology.orchestra.run.smithi064.stdout:avg 38.625 2023-10-09T06:11:37.151 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 14.9075 -> 14.9075 (expected baseline 5.81351) 2023-10-09T06:11:37.151 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.7 with 13 -> 13 pgs (0.33657 -> 0.33657 * mean) 2023-10-09T06:11:37.151 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.1 with 55 -> 55 pgs (1.42395 -> 1.42395 * mean) 2023-10-09T06:11:37.151 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:11:37.151 INFO:teuthology.orchestra.run.smithi064.stdout:oload 1000 2023-10-09T06:11:37.151 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 1 2023-10-09T06:11:37.151 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-10-09T06:11:37.151 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0078 2023-10-09T06:11:37.151 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.0785 2023-10-09T06:11:37.153 INFO:teuthology.orchestra.run.smithi064.stderr:no change 2023-10-09T06:11:37.176 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:37.222 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:11:37.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:37.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:11:37.329 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:11:37.358 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:37.832 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:11:37.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:37.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:11:37.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:37 smithi064 ceph-mon[212750]: 1.0 scrub starts 2023-10-09T06:11:37.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:37 smithi064 ceph-mon[212750]: 1.0 scrub ok 2023-10-09T06:11:37.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:37 smithi064 ceph-mon[212750]: failed to encode map e489 with expected crc 2023-10-09T06:11:37.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:37 smithi064 ceph-mon[212750]: osdmap e489: 8 total, 8 up, 8 in 2023-10-09T06:11:37.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:37 smithi064 ceph-mon[212750]: failed to encode map e489 with expected crc 2023-10-09T06:11:37.915 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:37 smithi064 ceph-mon[220044]: 1.0 scrub starts 2023-10-09T06:11:37.915 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:37 smithi064 ceph-mon[220044]: 1.0 scrub ok 2023-10-09T06:11:37.915 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:37 smithi064 ceph-mon[220044]: failed to encode map e489 with expected crc 2023-10-09T06:11:37.915 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:37 smithi064 ceph-mon[220044]: osdmap e489: 8 total, 8 up, 8 in 2023-10-09T06:11:37.915 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:37 smithi064 ceph-mon[220044]: failed to encode map e489 with expected crc 2023-10-09T06:11:37.916 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T06:11:37.916 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-10-09T06:11:37.937 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:11:37.966 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:37 smithi156 ceph-mon[195101]: 1.0 scrub starts 2023-10-09T06:11:37.966 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:37 smithi156 ceph-mon[195101]: 1.0 scrub ok 2023-10-09T06:11:37.966 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:37 smithi156 ceph-mon[195101]: failed to encode map e489 with expected crc 2023-10-09T06:11:37.966 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:37 smithi156 ceph-mon[195101]: osdmap e489: 8 total, 8 up, 8 in 2023-10-09T06:11:37.967 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:37 smithi156 ceph-mon[195101]: failed to encode map e489 with expected crc 2023-10-09T06:11:37.969 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:38.018 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:11:38.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:38.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:11:38.056 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:11:38.056 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T06:11:38.056 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T06:11:38.056 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T06:11:38.057 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:11:38.126 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:11:38.161 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:38.169 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:11:38.832 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:11:38.834 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:11:38.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:38.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:11:38.937 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:11:38.963 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:38 smithi064 ceph-mon[212750]: pgmap v1241: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 726 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:38.963 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:38 smithi064 ceph-mon[212750]: from='client.46232 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:38.963 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:38 smithi064 ceph-mon[212750]: failed to encode map e489 with expected crc 2023-10-09T06:11:38.963 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:38 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:11:38.963 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:38 smithi064 ceph-mon[212750]: failed to encode map e489 with expected crc 2023-10-09T06:11:38.964 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:38 smithi064 ceph-mon[220044]: pgmap v1241: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 726 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:38.964 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:38 smithi064 ceph-mon[220044]: from='client.46232 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:38.964 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:38 smithi064 ceph-mon[220044]: failed to encode map e489 with expected crc 2023-10-09T06:11:38.964 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:38 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:11:38.964 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:38 smithi064 ceph-mon[220044]: failed to encode map e489 with expected crc 2023-10-09T06:11:38.966 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:39.064 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:11:39.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:39.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:11:39.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:38 smithi156 ceph-mon[195101]: pgmap v1241: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 726 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:39.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:38 smithi156 ceph-mon[195101]: from='client.46232 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:39.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:38 smithi156 ceph-mon[195101]: failed to encode map e489 with expected crc 2023-10-09T06:11:39.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:38 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:11:39.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:38 smithi156 ceph-mon[195101]: failed to encode map e489 with expected crc 2023-10-09T06:11:39.168 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:11:39.200 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:39.249 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:11:39.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:39.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:11:39.357 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:11:39.385 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:39.431 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:11:39.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:39.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:11:39.459 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:11:39.481 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:11:39.511 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:11:39.535 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:11:39.536 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:11:39.562 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:39.604 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:11:39.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:39.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:11:39.709 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:11:39.712 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:39 smithi064 ceph-mon[212750]: 1.17 scrub starts 2023-10-09T06:11:39.713 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:39 smithi064 ceph-mon[212750]: 1.17 scrub ok 2023-10-09T06:11:39.713 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:39 smithi064 ceph-mon[212750]: 1.7 scrub starts 2023-10-09T06:11:39.713 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:39 smithi064 ceph-mon[212750]: 1.7 scrub ok 2023-10-09T06:11:39.713 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:39 smithi064 ceph-mon[220044]: 1.17 scrub starts 2023-10-09T06:11:39.713 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:39 smithi064 ceph-mon[220044]: 1.17 scrub ok 2023-10-09T06:11:39.713 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:39 smithi064 ceph-mon[220044]: 1.7 scrub starts 2023-10-09T06:11:39.713 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:39 smithi064 ceph-mon[220044]: 1.7 scrub ok 2023-10-09T06:11:39.742 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:39.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:39 smithi156 ceph-mon[195101]: 1.17 scrub starts 2023-10-09T06:11:39.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:39 smithi156 ceph-mon[195101]: 1.17 scrub ok 2023-10-09T06:11:39.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:39 smithi156 ceph-mon[195101]: 1.7 scrub starts 2023-10-09T06:11:39.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:39 smithi156 ceph-mon[195101]: 1.7 scrub ok 2023-10-09T06:11:40.033 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:11:40.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:40.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:11:40.136 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:11:40.166 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:40.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:40 smithi064 ceph-mon[220044]: pgmap v1242: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 726 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:40.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:40 smithi064 ceph-mon[220044]: 1.a scrub starts 2023-10-09T06:11:40.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:40 smithi064 ceph-mon[220044]: 1.a scrub ok 2023-10-09T06:11:40.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:40 smithi064 ceph-mon[212750]: pgmap v1242: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 726 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:40.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:40 smithi064 ceph-mon[212750]: 1.a scrub starts 2023-10-09T06:11:40.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:40 smithi064 ceph-mon[212750]: 1.a scrub ok 2023-10-09T06:11:41.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:40 smithi156 ceph-mon[195101]: pgmap v1242: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 726 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:41.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:40 smithi156 ceph-mon[195101]: 1.a scrub starts 2023-10-09T06:11:41.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:40 smithi156 ceph-mon[195101]: 1.a scrub ok 2023-10-09T06:11:41.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:41 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1720529010' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:11:41.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:41 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1720529010' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:11:41.769 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:11:41.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:41.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:11:41.873 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:11:41.904 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:42.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:41 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1720529010' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:11:42.218 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:11:42.218 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:11:42.218 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:11:42.219 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:11:42.219 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:11:42.219 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:11:42.219 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:11:42.219 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:11:42.219 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:11:42.219 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:11:42.219 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:11:42.219 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:11:42.219 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:11:42.220 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:11:42.220 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:11:42.220 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:11:42.562 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:11:42.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:42.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:11:42.666 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:11:42.697 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:42.746 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:11:42.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:42.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:11:42.854 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:11:42.881 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:42 smithi064 ceph-mon[212750]: pgmap v1243: 103 pgs: 103 active+clean; 674 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 13 KiB/s, 0 objects/s recovering 2023-10-09T06:11:42.881 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:42 smithi064 ceph-mon[220044]: pgmap v1243: 103 pgs: 103 active+clean; 674 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 13 KiB/s, 0 objects/s recovering 2023-10-09T06:11:42.884 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:43.092 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:11:43.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:43.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:11:43.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:42 smithi156 ceph-mon[195101]: pgmap v1243: 103 pgs: 103 active+clean; 674 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 13 KiB/s, 0 objects/s recovering 2023-10-09T06:11:43.196 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:11:43.227 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:43.277 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:11:43.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:43.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:11:43.385 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:11:43.413 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:43.457 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:11:43.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:43.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:11:43.563 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:11:43.595 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:43.659 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:11:43.660 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:11:43.682 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:43 smithi064 ceph-mon[212750]: 1.9 scrub starts 2023-10-09T06:11:43.682 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:43 smithi064 ceph-mon[212750]: 1.9 scrub ok 2023-10-09T06:11:43.682 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:43 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:11:43.682 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:43 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:43.682 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:43 smithi064 ceph-mon[220044]: 1.9 scrub starts 2023-10-09T06:11:43.683 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:43 smithi064 ceph-mon[220044]: 1.9 scrub ok 2023-10-09T06:11:43.683 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:43 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:11:43.683 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:43 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:43.996 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:11:44.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:44.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:11:44.101 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:11:44.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:43 smithi156 ceph-mon[195101]: 1.9 scrub starts 2023-10-09T06:11:44.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:43 smithi156 ceph-mon[195101]: 1.9 scrub ok 2023-10-09T06:11:44.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:43 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:11:44.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:43 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:44.135 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:44.237 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:11:44.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:44.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:11:44.255 INFO:teuthology.orchestra.run.smithi064.stdout:moved 0 / 309 (0%) 2023-10-09T06:11:44.255 INFO:teuthology.orchestra.run.smithi064.stdout:avg 38.625 2023-10-09T06:11:44.255 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 14.9075 -> 14.9075 (expected baseline 5.81351) 2023-10-09T06:11:44.255 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.7 with 13 -> 13 pgs (0.33657 -> 0.33657 * mean) 2023-10-09T06:11:44.255 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.1 with 55 -> 55 pgs (1.42395 -> 1.42395 * mean) 2023-10-09T06:11:44.256 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:11:44.256 INFO:teuthology.orchestra.run.smithi064.stdout:oload 1000 2023-10-09T06:11:44.256 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 1 2023-10-09T06:11:44.256 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-10-09T06:11:44.256 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0078 2023-10-09T06:11:44.256 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.0776 2023-10-09T06:11:44.257 INFO:teuthology.orchestra.run.smithi064.stderr:no change 2023-10-09T06:11:44.343 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:11:44.371 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:44.419 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:11:44.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:44.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:11:44.524 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:11:44.558 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:44.691 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_4' created 2023-10-09T06:11:44.691 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:11:44.874 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:44 smithi064 ceph-mon[212750]: pgmap v1244: 103 pgs: 103 active+clean; 674 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 11 KiB/s, 0 objects/s recovering 2023-10-09T06:11:44.874 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:44 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2116073449' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:11:44.875 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:44 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2271633589' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T06:11:44.875 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:44 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T06:11:44.875 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:44 smithi064 ceph-mon[220044]: pgmap v1244: 103 pgs: 103 active+clean; 674 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 11 KiB/s, 0 objects/s recovering 2023-10-09T06:11:44.875 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:44 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2116073449' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:11:44.875 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:44 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2271633589' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T06:11:44.876 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:44 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T06:11:44.892 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:11:44.949 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:11:44.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:44.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:11:45.053 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:11:45.081 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:45.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:44 smithi156 ceph-mon[195101]: pgmap v1244: 103 pgs: 103 active+clean; 674 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 11 KiB/s, 0 objects/s recovering 2023-10-09T06:11:45.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:44 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2116073449' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:11:45.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:44 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2271633589' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T06:11:45.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:44 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T06:11:45.224 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:11:45.437 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:11:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:11:45] "GET /metrics HTTP/1.1" 200 34092 "" "Prometheus/2.43.0" 2023-10-09T06:11:45.632 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:11:45.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:45.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:11:45.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:45 smithi064 ceph-mon[212750]: from='client.46256 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:45.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:45 smithi064 ceph-mon[212750]: 1.2 scrub starts 2023-10-09T06:11:45.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:45 smithi064 ceph-mon[212750]: 1.2 scrub ok 2023-10-09T06:11:45.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:45 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:11:45.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:45 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:11:45.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:45 smithi064 ceph-mon[212750]: failed to encode map e490 with expected crc 2023-10-09T06:11:45.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:45 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2116073449' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:11:45.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:45 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T06:11:45.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:45 smithi064 ceph-mon[212750]: osdmap e490: 8 total, 8 up, 8 in 2023-10-09T06:11:45.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:45 smithi064 ceph-mon[212750]: failed to encode map e490 with expected crc 2023-10-09T06:11:45.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:45 smithi064 ceph-mon[212750]: failed to encode map e490 with expected crc 2023-10-09T06:11:45.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:45 smithi064 ceph-mon[220044]: from='client.46256 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:45.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:45 smithi064 ceph-mon[220044]: 1.2 scrub starts 2023-10-09T06:11:45.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:45 smithi064 ceph-mon[220044]: 1.2 scrub ok 2023-10-09T06:11:45.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:45 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:11:45.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:45 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T06:11:45.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:45 smithi064 ceph-mon[220044]: failed to encode map e490 with expected crc 2023-10-09T06:11:45.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:45 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2116073449' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:11:45.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:45 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T06:11:45.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:45 smithi064 ceph-mon[220044]: osdmap e490: 8 total, 8 up, 8 in 2023-10-09T06:11:45.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:45 smithi064 ceph-mon[220044]: failed to encode map e490 with expected crc 2023-10-09T06:11:45.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:45 smithi064 ceph-mon[220044]: failed to encode map e490 with expected crc 2023-10-09T06:11:45.736 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:11:45.763 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:45.885 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-10-09T06:11:45.914 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:11:45.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:45.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:11:45.928 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T06:11:45.929 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-10-09T06:11:46.019 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:11:46.054 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:46.117 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:11:46.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:45 smithi156 ceph-mon[195101]: from='client.46256 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:46.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:45 smithi156 ceph-mon[195101]: 1.2 scrub starts 2023-10-09T06:11:46.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:45 smithi156 ceph-mon[195101]: 1.2 scrub ok 2023-10-09T06:11:46.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:45 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:11:46.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:45 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T06:11:46.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:45 smithi156 ceph-mon[195101]: failed to encode map e490 with expected crc 2023-10-09T06:11:46.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:45 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2116073449' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:11:46.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:45 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T06:11:46.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:45 smithi156 ceph-mon[195101]: osdmap e490: 8 total, 8 up, 8 in 2023-10-09T06:11:46.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:45 smithi156 ceph-mon[195101]: failed to encode map e490 with expected crc 2023-10-09T06:11:46.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:45 smithi156 ceph-mon[195101]: failed to encode map e490 with expected crc 2023-10-09T06:11:46.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:46.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:11:46.181 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:11:46.223 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:11:46.264 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:46.321 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:11:46.432 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:11:46.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:46.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:11:46.537 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:11:46.567 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:46.598 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:11:46.670 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:11:46.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:46.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:11:46.774 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:11:46.803 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:46.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:46 smithi064 ceph-mon[220044]: pgmap v1246: 119 pgs: 16 unknown, 103 active+clean; 674 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:46.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:46 smithi064 ceph-mon[220044]: failed to encode map e491 with expected crc 2023-10-09T06:11:46.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:46 smithi064 ceph-mon[220044]: osdmap e491: 8 total, 8 up, 8 in 2023-10-09T06:11:46.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:46 smithi064 ceph-mon[220044]: failed to encode map e491 with expected crc 2023-10-09T06:11:46.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:46 smithi064 ceph-mon[220044]: failed to encode map e491 with expected crc 2023-10-09T06:11:46.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:46 smithi064 ceph-mon[220044]: 1.6 scrub starts 2023-10-09T06:11:46.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:46 smithi064 ceph-mon[220044]: 1.6 scrub ok 2023-10-09T06:11:46.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:46 smithi064 ceph-mon[212750]: pgmap v1246: 119 pgs: 16 unknown, 103 active+clean; 674 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:46.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:46 smithi064 ceph-mon[212750]: failed to encode map e491 with expected crc 2023-10-09T06:11:46.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:46 smithi064 ceph-mon[212750]: osdmap e491: 8 total, 8 up, 8 in 2023-10-09T06:11:46.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:46 smithi064 ceph-mon[212750]: failed to encode map e491 with expected crc 2023-10-09T06:11:46.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:46 smithi064 ceph-mon[212750]: failed to encode map e491 with expected crc 2023-10-09T06:11:46.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:46 smithi064 ceph-mon[212750]: 1.6 scrub starts 2023-10-09T06:11:46.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:46 smithi064 ceph-mon[212750]: 1.6 scrub ok 2023-10-09T06:11:47.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:46 smithi156 ceph-mon[195101]: pgmap v1246: 119 pgs: 16 unknown, 103 active+clean; 674 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:47.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:46 smithi156 ceph-mon[195101]: failed to encode map e491 with expected crc 2023-10-09T06:11:47.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:46 smithi156 ceph-mon[195101]: osdmap e491: 8 total, 8 up, 8 in 2023-10-09T06:11:47.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:46 smithi156 ceph-mon[195101]: failed to encode map e491 with expected crc 2023-10-09T06:11:47.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:46 smithi156 ceph-mon[195101]: failed to encode map e491 with expected crc 2023-10-09T06:11:47.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:46 smithi156 ceph-mon[195101]: 1.6 scrub starts 2023-10-09T06:11:47.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:46 smithi156 ceph-mon[195101]: 1.6 scrub ok 2023-10-09T06:11:47.353 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:11:47.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:47.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:11:47.457 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:11:47.615 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:47.818 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:11:47.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:47.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:11:47.875 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:47 smithi064 ceph-mon[212750]: osdmap e492: 8 total, 8 up, 8 in 2023-10-09T06:11:47.876 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:47 smithi064 ceph-mon[212750]: failed to encode map e492 with expected crc 2023-10-09T06:11:47.876 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:47 smithi064 ceph-mon[212750]: failed to encode map e492 with expected crc 2023-10-09T06:11:47.876 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:47 smithi064 ceph-mon[220044]: osdmap e492: 8 total, 8 up, 8 in 2023-10-09T06:11:47.876 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:47 smithi064 ceph-mon[220044]: failed to encode map e492 with expected crc 2023-10-09T06:11:47.876 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:47 smithi064 ceph-mon[220044]: failed to encode map e492 with expected crc 2023-10-09T06:11:47.898 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:11:47.922 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:11:47.952 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:47 smithi156 ceph-mon[195101]: osdmap e492: 8 total, 8 up, 8 in 2023-10-09T06:11:47.952 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:47 smithi156 ceph-mon[195101]: failed to encode map e492 with expected crc 2023-10-09T06:11:47.953 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:47 smithi156 ceph-mon[195101]: failed to encode map e492 with expected crc 2023-10-09T06:11:47.955 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:48.005 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:11:48.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:48.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:11:48.111 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:11:48.134 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:11:48.139 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:48.183 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:11:48.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:48.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:11:48.262 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:11:48.286 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:11:48.289 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:11:48.315 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:48.324 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:11:48.346 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:11:48.357 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:11:48.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:48.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:11:48.368 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:11:48.392 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:11:48.411 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:11:48.430 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:11:48.431 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:11:48.431 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:11:48.431 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:11:48.432 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:11:48.432 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:11:48.432 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:11:48.432 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:11:48.432 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:11:48.461 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:11:48.487 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:48.528 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:11:48.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:48.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:11:48.636 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:11:48.662 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:48.704 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:11:48.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:48.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:11:48.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:48 smithi064 ceph-mon[212750]: pgmap v1249: 119 pgs: 6 creating+activating, 3 creating+peering, 110 active+clean; 621 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:48.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:48 smithi064 ceph-mon[212750]: failed to encode map e492 with expected crc 2023-10-09T06:11:48.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:48 smithi064 ceph-mon[212750]: failed to encode map e492 with expected crc 2023-10-09T06:11:48.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:48 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/102123869' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:11:48.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:48 smithi064 ceph-mon[220044]: pgmap v1249: 119 pgs: 6 creating+activating, 3 creating+peering, 110 active+clean; 621 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:48.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:48 smithi064 ceph-mon[220044]: failed to encode map e492 with expected crc 2023-10-09T06:11:48.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:48 smithi064 ceph-mon[220044]: failed to encode map e492 with expected crc 2023-10-09T06:11:48.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:48 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/102123869' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:11:48.810 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:11:48.819 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:11:48.819 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T06:11:48.820 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2023-10-09T06:11:48.820 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:11:48.837 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:48.879 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:11:48.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:48.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:11:48.987 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:11:49.018 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:49.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:48 smithi156 ceph-mon[195101]: pgmap v1249: 119 pgs: 6 creating+activating, 3 creating+peering, 110 active+clean; 621 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:49.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:48 smithi156 ceph-mon[195101]: failed to encode map e492 with expected crc 2023-10-09T06:11:49.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:48 smithi156 ceph-mon[195101]: failed to encode map e492 with expected crc 2023-10-09T06:11:49.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:48 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/102123869' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:11:49.815 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:11:49.881 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:49 smithi064 ceph-mon[212750]: 1.f scrub starts 2023-10-09T06:11:49.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:49 smithi064 ceph-mon[212750]: 1.f scrub ok 2023-10-09T06:11:49.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:49 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:11:49.882 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:49 smithi064 ceph-mon[220044]: 1.f scrub starts 2023-10-09T06:11:49.882 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:49 smithi064 ceph-mon[220044]: 1.f scrub ok 2023-10-09T06:11:49.882 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:49 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:11:50.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:49 smithi156 ceph-mon[195101]: 1.f scrub starts 2023-10-09T06:11:50.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:49 smithi156 ceph-mon[195101]: 1.f scrub ok 2023-10-09T06:11:50.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:49 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:11:50.130 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:11:50.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:50.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:11:50.234 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:11:50.265 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:50.317 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:11:50.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:50.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:11:50.421 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:11:50.453 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:50.501 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:11:50.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:50.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:11:50.608 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:11:50.636 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:50.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:50 smithi156 ceph-mon[195101]: pgmap v1250: 119 pgs: 6 creating+activating, 3 creating+peering, 110 active+clean; 621 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:50.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:50 smithi064 ceph-mon[212750]: pgmap v1250: 119 pgs: 6 creating+activating, 3 creating+peering, 110 active+clean; 621 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:50.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:50 smithi064 ceph-mon[220044]: pgmap v1250: 119 pgs: 6 creating+activating, 3 creating+peering, 110 active+clean; 621 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:51.020 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:11:51.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:51.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:11:51.061 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:11:51.125 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:11:51.129 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:11:51.157 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:51.205 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:11:51.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:51.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:11:51.314 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:11:51.344 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:51.593 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:11:51.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:51.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:11:51.698 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:11:51.719 INFO:teuthology.orchestra.run.smithi064.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-10-09T06:11:51.733 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:51.790 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:51 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/800119971' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:11:51.790 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:51 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:11:51.790 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:51 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/800119971' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:11:51.790 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:51 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:11:51.835 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:11:51.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:51.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:11:51.939 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:11:51.968 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:51 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/800119971' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:11:51.969 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:51 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:11:51.971 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:52.019 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:11:52.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:52.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:11:52.085 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:11:52.125 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:11:52.155 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:52.505 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:11:52.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:52.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:11:52.609 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:11:52.641 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:52.689 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:11:52.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:52.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:11:52.723 INFO:teuthology.orchestra.run.smithi064.stdout:moved 26 / 357 (7.28291%) 2023-10-09T06:11:52.723 INFO:teuthology.orchestra.run.smithi064.stdout:avg 44.625 2023-10-09T06:11:52.723 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 17.3345 -> 20.8743 (expected baseline 6.24875) 2023-10-09T06:11:52.723 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.7 with 14 -> 9 pgs (0.313726 -> 0.201681 * mean) 2023-10-09T06:11:52.723 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.4 with 64 -> 67 pgs (1.43417 -> 1.5014 * mean) 2023-10-09T06:11:52.723 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:11:52.723 INFO:teuthology.orchestra.run.smithi064.stdout:oload 110 2023-10-09T06:11:52.723 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 3 2023-10-09T06:11:52.723 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-10-09T06:11:52.724 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0076 2023-10-09T06:11:52.724 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.0084 2023-10-09T06:11:52.724 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 weight 0.4400 -> 0.3124 2023-10-09T06:11:52.724 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 weight 0.3595 -> 0.2769 2023-10-09T06:11:52.724 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 weight 0.6564 -> 0.5172 2023-10-09T06:11:52.724 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 weight 1.0000 -> 0.8745 2023-10-09T06:11:52.796 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:11:52.822 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:52 smithi064 ceph-mon[220044]: pgmap v1251: 119 pgs: 119 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:52.822 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:52 smithi064 ceph-mon[220044]: failed to encode map e493 with expected crc 2023-10-09T06:11:52.822 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:52 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:11:52.822 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:52 smithi064 ceph-mon[220044]: osdmap e493: 8 total, 8 up, 8 in 2023-10-09T06:11:52.822 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:52 smithi064 ceph-mon[220044]: from='client.36255 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:52.823 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:52 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'57311','5':'33893','6':'20474','7':'18148'}"}]: dispatch 2023-10-09T06:11:52.823 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:52 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'57311','5':'33893','6':'20474','7':'18148'}"}]: dispatch 2023-10-09T06:11:52.823 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:52 smithi064 ceph-mon[212750]: pgmap v1251: 119 pgs: 119 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:52.823 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:52 smithi064 ceph-mon[212750]: failed to encode map e493 with expected crc 2023-10-09T06:11:52.823 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:52 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:11:52.823 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:52 smithi064 ceph-mon[212750]: osdmap e493: 8 total, 8 up, 8 in 2023-10-09T06:11:52.823 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:52 smithi064 ceph-mon[212750]: from='client.36255 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:52.823 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:52 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'57311','5':'33893','6':'20474','7':'18148'}"}]: dispatch 2023-10-09T06:11:52.823 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:52 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'57311','5':'33893','6':'20474','7':'18148'}"}]: dispatch 2023-10-09T06:11:52.825 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:53.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:52 smithi156 ceph-mon[195101]: pgmap v1251: 119 pgs: 119 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:53.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:52 smithi156 ceph-mon[195101]: failed to encode map e493 with expected crc 2023-10-09T06:11:53.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:52 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:11:53.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:52 smithi156 ceph-mon[195101]: osdmap e493: 8 total, 8 up, 8 in 2023-10-09T06:11:53.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:52 smithi156 ceph-mon[195101]: from='client.36255 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:53.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:52 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'57311','5':'33893','6':'20474','7':'18148'}"}]: dispatch 2023-10-09T06:11:53.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:52 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'57311','5':'33893','6':'20474','7':'18148'}"}]: dispatch 2023-10-09T06:11:53.248 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:11:53.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:53.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:11:53.353 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:11:53.385 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:53.434 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:11:53.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:53.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:11:53.542 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:11:53.569 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:53.934 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T06:11:53.934 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 3.0 2023-10-09T06:11:53.979 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:11:53.982 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:53 smithi064 ceph-mon[212750]: failed to encode map e493 with expected crc 2023-10-09T06:11:53.982 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:53 smithi064 ceph-mon[212750]: failed to encode map e493 with expected crc 2023-10-09T06:11:53.982 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:53 smithi064 ceph-mon[212750]: failed to encode map e493 with expected crc 2023-10-09T06:11:53.982 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:53 smithi064 ceph-mon[212750]: failed to encode map e494 with expected crc 2023-10-09T06:11:53.982 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:53 smithi064 ceph-mon[212750]: failed to encode map e494 with expected crc 2023-10-09T06:11:53.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:53 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'3':'57311','5':'33893','6':'20474','7':'18148'}"}]': finished 2023-10-09T06:11:53.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:53 smithi064 ceph-mon[212750]: osdmap e494: 8 total, 8 up, 8 in 2023-10-09T06:11:53.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:53 smithi064 ceph-mon[212750]: failed to encode map e494 with expected crc 2023-10-09T06:11:53.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:53 smithi064 ceph-mon[212750]: failed to encode map e494 with expected crc 2023-10-09T06:11:53.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:53 smithi064 ceph-mon[212750]: failed to encode map e494 with expected crc 2023-10-09T06:11:53.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:53 smithi064 ceph-mon[212750]: failed to encode map e494 with expected crc 2023-10-09T06:11:53.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:53 smithi064 ceph-mon[212750]: failed to encode map e494 with expected crc 2023-10-09T06:11:53.983 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:53 smithi064 ceph-mon[220044]: failed to encode map e493 with expected crc 2023-10-09T06:11:53.983 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:53 smithi064 ceph-mon[220044]: failed to encode map e493 with expected crc 2023-10-09T06:11:53.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:53 smithi064 ceph-mon[220044]: failed to encode map e493 with expected crc 2023-10-09T06:11:53.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:53 smithi064 ceph-mon[220044]: failed to encode map e494 with expected crc 2023-10-09T06:11:53.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:53 smithi064 ceph-mon[220044]: failed to encode map e494 with expected crc 2023-10-09T06:11:53.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:53 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'3':'57311','5':'33893','6':'20474','7':'18148'}"}]': finished 2023-10-09T06:11:53.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:53 smithi064 ceph-mon[220044]: osdmap e494: 8 total, 8 up, 8 in 2023-10-09T06:11:53.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:53 smithi064 ceph-mon[220044]: failed to encode map e494 with expected crc 2023-10-09T06:11:53.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:53 smithi064 ceph-mon[220044]: failed to encode map e494 with expected crc 2023-10-09T06:11:53.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:53 smithi064 ceph-mon[220044]: failed to encode map e494 with expected crc 2023-10-09T06:11:53.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:53 smithi064 ceph-mon[220044]: failed to encode map e494 with expected crc 2023-10-09T06:11:53.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:53 smithi064 ceph-mon[220044]: failed to encode map e494 with expected crc 2023-10-09T06:11:53.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:53.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:11:54.086 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:11:54.110 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:53 smithi156 ceph-mon[195101]: failed to encode map e493 with expected crc 2023-10-09T06:11:54.111 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:53 smithi156 ceph-mon[195101]: failed to encode map e493 with expected crc 2023-10-09T06:11:54.111 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:53 smithi156 ceph-mon[195101]: failed to encode map e493 with expected crc 2023-10-09T06:11:54.111 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:53 smithi156 ceph-mon[195101]: failed to encode map e494 with expected crc 2023-10-09T06:11:54.111 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:53 smithi156 ceph-mon[195101]: failed to encode map e494 with expected crc 2023-10-09T06:11:54.111 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:53 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'3':'57311','5':'33893','6':'20474','7':'18148'}"}]': finished 2023-10-09T06:11:54.111 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:53 smithi156 ceph-mon[195101]: osdmap e494: 8 total, 8 up, 8 in 2023-10-09T06:11:54.111 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:53 smithi156 ceph-mon[195101]: failed to encode map e494 with expected crc 2023-10-09T06:11:54.111 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:53 smithi156 ceph-mon[195101]: failed to encode map e494 with expected crc 2023-10-09T06:11:54.112 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:53 smithi156 ceph-mon[195101]: failed to encode map e494 with expected crc 2023-10-09T06:11:54.112 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:53 smithi156 ceph-mon[195101]: failed to encode map e494 with expected crc 2023-10-09T06:11:54.112 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:53 smithi156 ceph-mon[195101]: failed to encode map e494 with expected crc 2023-10-09T06:11:54.113 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:54.155 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:11:54.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:54.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:11:54.250 DEBUG:teuthology.orchestra.run.smithi064:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-10-09T06:11:54.253 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T06:11:54.253 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T06:11:54.263 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:11:54.289 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:54.308 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:hints = 1 2023-10-09T06:11:54.308 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T06:11:54.308 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object prefix: benchmark_data_smithi064_573750 2023-10-09T06:11:54.308 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:11:54.309 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 0 0 0 0 0 0 - 0 2023-10-09T06:11:54.339 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:11:54.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:54.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:11:54.445 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:11:54.463 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:11:54.513 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:54.608 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:11:54.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:54.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:11:54.713 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:11:54.768 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:11:54.768 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T06:11:54.768 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T06:11:54.768 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T06:11:54.768 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:11:54.777 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:11:54.792 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:54.914 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:11:54.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:54.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:11:55.021 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:11:55.039 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:54 smithi064 ceph-mon[212750]: pgmap v1254: 119 pgs: 119 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:55.039 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:54 smithi064 ceph-mon[212750]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.039 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:54 smithi064 ceph-mon[212750]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.039 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:54 smithi064 ceph-mon[212750]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.039 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:54 smithi064 ceph-mon[212750]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.040 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:54 smithi064 ceph-mon[212750]: osdmap e495: 8 total, 8 up, 8 in 2023-10-09T06:11:55.040 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:54 smithi064 ceph-mon[212750]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.040 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:54 smithi064 ceph-mon[212750]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.040 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:54 smithi064 ceph-mon[212750]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.040 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:54 smithi064 ceph-mon[212750]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.040 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:54 smithi064 ceph-mon[212750]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.040 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:54 smithi064 ceph-mon[212750]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.040 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:54 smithi064 ceph-mon[220044]: pgmap v1254: 119 pgs: 119 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:55.040 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:54 smithi064 ceph-mon[220044]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.041 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:54 smithi064 ceph-mon[220044]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.041 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:54 smithi064 ceph-mon[220044]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.041 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:54 smithi064 ceph-mon[220044]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.041 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:54 smithi064 ceph-mon[220044]: osdmap e495: 8 total, 8 up, 8 in 2023-10-09T06:11:55.041 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:54 smithi064 ceph-mon[220044]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.041 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:54 smithi064 ceph-mon[220044]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.041 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:54 smithi064 ceph-mon[220044]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.041 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:54 smithi064 ceph-mon[220044]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.041 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:54 smithi064 ceph-mon[220044]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.041 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:54 smithi064 ceph-mon[220044]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.091 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:54 smithi156 ceph-mon[195101]: pgmap v1254: 119 pgs: 119 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:54 smithi156 ceph-mon[195101]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:54 smithi156 ceph-mon[195101]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:54 smithi156 ceph-mon[195101]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:54 smithi156 ceph-mon[195101]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:54 smithi156 ceph-mon[195101]: osdmap e495: 8 total, 8 up, 8 in 2023-10-09T06:11:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:54 smithi156 ceph-mon[195101]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:54 smithi156 ceph-mon[195101]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:54 smithi156 ceph-mon[195101]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:54 smithi156 ceph-mon[195101]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:54 smithi156 ceph-mon[195101]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:54 smithi156 ceph-mon[195101]: failed to encode map e495 with expected crc 2023-10-09T06:11:55.309 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 1 16 2638 2622 163.86 163.875 0.00701241 0.00609353 2023-10-09T06:11:55.497 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:11:55.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:55.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:11:55.606 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:11:55.631 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:11:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:11:55] "GET /metrics HTTP/1.1" 200 34092 "" "Prometheus/2.43.0" 2023-10-09T06:11:55.634 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:55.760 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:11:55.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:55 smithi064 ceph-mon[212750]: osdmap e496: 8 total, 8 up, 8 in 2023-10-09T06:11:55.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:55 smithi064 ceph-mon[212750]: failed to encode map e496 with expected crc 2023-10-09T06:11:55.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:55 smithi064 ceph-mon[212750]: failed to encode map e496 with expected crc 2023-10-09T06:11:55.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:55 smithi064 ceph-mon[212750]: failed to encode map e496 with expected crc 2023-10-09T06:11:55.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:55 smithi064 ceph-mon[212750]: failed to encode map e496 with expected crc 2023-10-09T06:11:55.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:55 smithi064 ceph-mon[212750]: failed to encode map e496 with expected crc 2023-10-09T06:11:55.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:55 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/404290797' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:11:55.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:55 smithi064 ceph-mon[220044]: osdmap e496: 8 total, 8 up, 8 in 2023-10-09T06:11:55.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:55 smithi064 ceph-mon[220044]: failed to encode map e496 with expected crc 2023-10-09T06:11:55.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:55 smithi064 ceph-mon[220044]: failed to encode map e496 with expected crc 2023-10-09T06:11:55.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:55 smithi064 ceph-mon[220044]: failed to encode map e496 with expected crc 2023-10-09T06:11:55.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:55 smithi064 ceph-mon[220044]: failed to encode map e496 with expected crc 2023-10-09T06:11:55.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:55 smithi064 ceph-mon[220044]: failed to encode map e496 with expected crc 2023-10-09T06:11:55.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:55 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/404290797' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:11:56.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:55 smithi156 ceph-mon[195101]: osdmap e496: 8 total, 8 up, 8 in 2023-10-09T06:11:56.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:55 smithi156 ceph-mon[195101]: failed to encode map e496 with expected crc 2023-10-09T06:11:56.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:55 smithi156 ceph-mon[195101]: failed to encode map e496 with expected crc 2023-10-09T06:11:56.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:55 smithi156 ceph-mon[195101]: failed to encode map e496 with expected crc 2023-10-09T06:11:56.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:55 smithi156 ceph-mon[195101]: failed to encode map e496 with expected crc 2023-10-09T06:11:56.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:55 smithi156 ceph-mon[195101]: failed to encode map e496 with expected crc 2023-10-09T06:11:56.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:55 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/404290797' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:11:56.309 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 2 16 4932 4916 153.553 143.375 0.00707261 0.00650169 2023-10-09T06:11:56.469 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:11:56.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:56.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:11:56.576 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:11:56.636 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:56.774 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:56 smithi064 ceph-mon[220044]: pgmap v1257: 119 pgs: 119 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:56.774 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:56 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:11:56.774 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:56 smithi064 ceph-mon[220044]: failed to encode map e497 with expected crc 2023-10-09T06:11:56.774 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:56 smithi064 ceph-mon[220044]: failed to encode map e497 with expected crc 2023-10-09T06:11:56.775 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:56 smithi064 ceph-mon[220044]: failed to encode map e497 with expected crc 2023-10-09T06:11:56.775 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:56 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/404290797' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:11:56.775 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:56 smithi064 ceph-mon[220044]: osdmap e497: 8 total, 8 up, 8 in 2023-10-09T06:11:56.775 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:56 smithi064 ceph-mon[220044]: failed to encode map e497 with expected crc 2023-10-09T06:11:56.775 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:56 smithi064 ceph-mon[220044]: failed to encode map e497 with expected crc 2023-10-09T06:11:56.775 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:56 smithi064 ceph-mon[220044]: failed to encode map e497 with expected crc 2023-10-09T06:11:57.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:56 smithi156 ceph-mon[195101]: pgmap v1257: 119 pgs: 119 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:57.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:56 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:11:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:56 smithi156 ceph-mon[195101]: failed to encode map e497 with expected crc 2023-10-09T06:11:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:56 smithi156 ceph-mon[195101]: failed to encode map e497 with expected crc 2023-10-09T06:11:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:56 smithi156 ceph-mon[195101]: failed to encode map e497 with expected crc 2023-10-09T06:11:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:56 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/404290797' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:11:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:56 smithi156 ceph-mon[195101]: osdmap e497: 8 total, 8 up, 8 in 2023-10-09T06:11:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:56 smithi156 ceph-mon[195101]: failed to encode map e497 with expected crc 2023-10-09T06:11:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:56 smithi156 ceph-mon[195101]: failed to encode map e497 with expected crc 2023-10-09T06:11:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:56 smithi156 ceph-mon[195101]: failed to encode map e497 with expected crc 2023-10-09T06:11:57.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:56 smithi064 ceph-mon[212750]: pgmap v1257: 119 pgs: 119 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:11:57.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:56 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:11:57.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:56 smithi064 ceph-mon[212750]: failed to encode map e497 with expected crc 2023-10-09T06:11:57.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:56 smithi064 ceph-mon[212750]: failed to encode map e497 with expected crc 2023-10-09T06:11:57.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:56 smithi064 ceph-mon[212750]: failed to encode map e497 with expected crc 2023-10-09T06:11:57.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:56 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/404290797' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:11:57.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:56 smithi064 ceph-mon[212750]: osdmap e497: 8 total, 8 up, 8 in 2023-10-09T06:11:57.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:56 smithi064 ceph-mon[212750]: failed to encode map e497 with expected crc 2023-10-09T06:11:57.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:56 smithi064 ceph-mon[212750]: failed to encode map e497 with expected crc 2023-10-09T06:11:57.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:56 smithi064 ceph-mon[212750]: failed to encode map e497 with expected crc 2023-10-09T06:11:57.310 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 3 16 8051 8035 167.293 194.938 0.00431188 0.005968 2023-10-09T06:11:57.606 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:11:57.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:57.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:11:57.711 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:11:57.737 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:57.787 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:11:57.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:57.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:11:57.894 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:11:57.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:57 smithi064 ceph-mon[220044]: osdmap e498: 8 total, 8 up, 8 in 2023-10-09T06:11:57.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:57 smithi064 ceph-mon[220044]: failed to encode map e498 with expected crc 2023-10-09T06:11:57.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:57 smithi064 ceph-mon[220044]: failed to encode map e498 with expected crc 2023-10-09T06:11:57.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:57 smithi064 ceph-mon[220044]: failed to encode map e498 with expected crc 2023-10-09T06:11:57.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:57 smithi064 ceph-mon[220044]: pgmap v1260: 119 pgs: 6 peering, 2 active+undersized+remapped, 111 active+clean; 231 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 5.0 KiB/s rd, 58 MiB/s wr, 926 op/s; 0 B/s, 2 keys/s, 0 objects/s recovering 2023-10-09T06:11:57.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:57 smithi064 ceph-mon[212750]: osdmap e498: 8 total, 8 up, 8 in 2023-10-09T06:11:57.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:57 smithi064 ceph-mon[212750]: failed to encode map e498 with expected crc 2023-10-09T06:11:57.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:57 smithi064 ceph-mon[212750]: failed to encode map e498 with expected crc 2023-10-09T06:11:57.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:57 smithi064 ceph-mon[212750]: failed to encode map e498 with expected crc 2023-10-09T06:11:57.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:57 smithi064 ceph-mon[212750]: pgmap v1260: 119 pgs: 6 peering, 2 active+undersized+remapped, 111 active+clean; 231 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 5.0 KiB/s rd, 58 MiB/s wr, 926 op/s; 0 B/s, 2 keys/s, 0 objects/s recovering 2023-10-09T06:11:57.951 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:58.016 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:11:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:57 smithi156 ceph-mon[195101]: osdmap e498: 8 total, 8 up, 8 in 2023-10-09T06:11:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:57 smithi156 ceph-mon[195101]: failed to encode map e498 with expected crc 2023-10-09T06:11:58.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:57 smithi156 ceph-mon[195101]: failed to encode map e498 with expected crc 2023-10-09T06:11:58.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:57 smithi156 ceph-mon[195101]: failed to encode map e498 with expected crc 2023-10-09T06:11:58.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:57 smithi156 ceph-mon[195101]: pgmap v1260: 119 pgs: 6 peering, 2 active+undersized+remapped, 111 active+clean; 231 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 5.0 KiB/s rd, 58 MiB/s wr, 926 op/s; 0 B/s, 2 keys/s, 0 objects/s recovering 2023-10-09T06:11:58.310 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 4 16 9863 9847 153.783 113.25 0.00235339 0.0063572 2023-10-09T06:11:58.341 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:11:58.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:58.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:11:58.449 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:11:58.509 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:58.739 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:11:58.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:58.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:11:58.845 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:11:58.876 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:58.936 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:11:58.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:58.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:11:59.040 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:11:59.073 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:58 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:11:59.073 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:58 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:11:59.074 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:11:58 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:59.077 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:59.144 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:11:59.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:59.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:11:59.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:58 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:11:59.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:58 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:11:59.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:11:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:59.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:58 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:11:59.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:58 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:11:59.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:11:58 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:59.251 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:11:59.312 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 5 16 12329 12313 153.813 154.125 0.00823473 0.00649255 2023-10-09T06:11:59.335 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:11:59.858 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:11:59.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:11:59.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:11:59.912 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:11:59.950 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:11:59.963 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:11:59.990 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:12:00.002 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:00.033 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:12:00.070 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:12:00.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:00.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:12:00.076 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:12:00.115 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:12:00.153 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:12:00.176 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:12:00.195 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:12:00.199 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:12:00.199 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:12:00.199 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:12:00.199 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:12:00.199 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:12:00.200 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:12:00.200 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:12:00.200 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:12:00.213 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:00.284 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:12:00.288 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:00 smithi156 ceph-mon[195101]: pgmap v1261: 119 pgs: 6 peering, 2 active+undersized+remapped, 111 active+clean; 231 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3.7 KiB/s rd, 43 MiB/s wr, 691 op/s; 0 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T06:12:00.288 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:00 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:00.288 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:00 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2508688167' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:12:00.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:00.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:12:00.313 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 6 16 14811 14795 154.006 155.125 0.00791256 0.00648683 2023-10-09T06:12:00.390 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:12:00.434 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:00 smithi064 ceph-mon[212750]: pgmap v1261: 119 pgs: 6 peering, 2 active+undersized+remapped, 111 active+clean; 231 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3.7 KiB/s rd, 43 MiB/s wr, 691 op/s; 0 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T06:12:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:00 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:00.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:00 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2508688167' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:12:00.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:00 smithi064 ceph-mon[220044]: pgmap v1261: 119 pgs: 6 peering, 2 active+undersized+remapped, 111 active+clean; 231 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3.7 KiB/s rd, 43 MiB/s wr, 691 op/s; 0 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T06:12:00.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:00 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:00.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:00 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2508688167' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:12:00.496 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:12:00.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:00.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:12:00.603 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:12:00.637 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:00.683 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:12:00.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:00.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:12:00.789 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:12:00.850 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:00.975 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:12:01.114 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:12:01.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:01.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:12:01.228 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:12:01.255 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:01.313 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 7 15 16481 16466 146.927 104.438 0.00707746 0.0067997 2023-10-09T06:12:01.974 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:02.073 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:12:02.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:02.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:12:02.189 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:12:02.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:02 smithi064 ceph-mon[220044]: pgmap v1262: 119 pgs: 5 peering, 114 active+clean; 841 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 3.2 KiB/s rd, 133 MiB/s wr, 2.12k op/s; 0 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T06:12:02.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:02 smithi064 ceph-mon[212750]: pgmap v1262: 119 pgs: 5 peering, 114 active+clean; 841 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 3.2 KiB/s rd, 133 MiB/s wr, 2.12k op/s; 0 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T06:12:02.225 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:02.237 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:02.286 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:12:02.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:02.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:12:02.314 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 8 16 18580 18564 144.927 131.125 0.00699252 0.00689317 2023-10-09T06:12:02.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:02 smithi156 ceph-mon[195101]: pgmap v1262: 119 pgs: 5 peering, 114 active+clean; 841 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 3.2 KiB/s rd, 133 MiB/s wr, 2.12k op/s; 0 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T06:12:02.394 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:12:02.432 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:02.489 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:12:02.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:02.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:12:02.595 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:12:02.663 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:03.051 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:12:03.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:03.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:12:03.166 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:12:03.217 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:03.314 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 9 16 20431 20415 141.679 115.688 0.00660541 0.00705257 2023-10-09T06:12:03.350 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:12:03.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:03.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:12:03.457 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:12:03.517 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:03.654 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:03 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:12:03.654 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:03 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:12:03.654 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:03 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'7':'19699'}"}]: dispatch 2023-10-09T06:12:03.654 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:03 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'7':'19699'}"}]: dispatch 2023-10-09T06:12:03.667 INFO:teuthology.orchestra.run.smithi064.stdout:moved 2 / 357 (0.560224%) 2023-10-09T06:12:03.667 INFO:teuthology.orchestra.run.smithi064.stdout:avg 44.625 2023-10-09T06:12:03.667 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 20.8743 -> 20.5179 (expected baseline 6.24875) 2023-10-09T06:12:03.667 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.7 with 9 -> 11 pgs (0.201681 -> 0.246499 * mean) 2023-10-09T06:12:03.667 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.4 with 67 -> 67 pgs (1.5014 -> 1.5014 * mean) 2023-10-09T06:12:03.667 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:12:03.668 INFO:teuthology.orchestra.run.smithi064.stdout:oload 1000 2023-10-09T06:12:03.668 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 3 2023-10-09T06:12:03.668 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-10-09T06:12:03.668 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0113 2023-10-09T06:12:03.668 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.1126 2023-10-09T06:12:03.668 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 weight 0.2769 -> 0.3006 2023-10-09T06:12:03.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:03 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'7':'19699'}"}]: dispatch 2023-10-09T06:12:03.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:03 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'7':'19699'}"}]: dispatch 2023-10-09T06:12:04.009 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:12:04.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:03 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:12:04.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:03 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'7':'19699'}"}]: dispatch 2023-10-09T06:12:04.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:03 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'7':'19699'}"}]: dispatch 2023-10-09T06:12:04.314 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 10 16 21631 21615 135.014 75 0.00323493 0.00740057 2023-10-09T06:12:04.587 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:12:04.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:04.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:12:04.693 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:12:04.764 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:04.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:04 smithi064 ceph-mon[220044]: pgmap v1263: 119 pgs: 119 active+clean; 975 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2.5 KiB/s rd, 122 MiB/s wr, 1.95k op/s; 0 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T06:12:04.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:04 smithi064 ceph-mon[220044]: from='client.56110 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:12:04.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:04 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:12:04.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:04 smithi064 ceph-mon[220044]: failed to encode map e499 with expected crc 2023-10-09T06:12:04.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:04 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'7':'19699'}"}]': finished 2023-10-09T06:12:04.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:04 smithi064 ceph-mon[220044]: osdmap e499: 8 total, 8 up, 8 in 2023-10-09T06:12:04.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:04 smithi064 ceph-mon[212750]: pgmap v1263: 119 pgs: 119 active+clean; 975 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2.5 KiB/s rd, 122 MiB/s wr, 1.95k op/s; 0 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T06:12:04.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:04 smithi064 ceph-mon[212750]: from='client.56110 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:12:04.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:04 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:12:04.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:04 smithi064 ceph-mon[212750]: failed to encode map e499 with expected crc 2023-10-09T06:12:04.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:04 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'7':'19699'}"}]': finished 2023-10-09T06:12:04.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:04 smithi064 ceph-mon[212750]: osdmap e499: 8 total, 8 up, 8 in 2023-10-09T06:12:05.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:04 smithi156 ceph-mon[195101]: pgmap v1263: 119 pgs: 119 active+clean; 975 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 2.5 KiB/s rd, 122 MiB/s wr, 1.95k op/s; 0 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T06:12:05.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:04 smithi156 ceph-mon[195101]: from='client.56110 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:12:05.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:04 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:12:05.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:04 smithi156 ceph-mon[195101]: failed to encode map e499 with expected crc 2023-10-09T06:12:05.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:04 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'7':'19699'}"}]': finished 2023-10-09T06:12:05.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:04 smithi156 ceph-mon[195101]: osdmap e499: 8 total, 8 up, 8 in 2023-10-09T06:12:05.315 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 11 16 23289 23273 132.161 103.625 0.00207814 0.0075425 2023-10-09T06:12:05.678 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:12:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:12:05] "GET /metrics HTTP/1.1" 200 36502 "" "Prometheus/2.43.0" 2023-10-09T06:12:05.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:05 smithi064 ceph-mon[212750]: failed to encode map e500 with expected crc 2023-10-09T06:12:05.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:05 smithi064 ceph-mon[212750]: osdmap e500: 8 total, 8 up, 8 in 2023-10-09T06:12:05.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:05 smithi064 ceph-mon[212750]: failed to encode map e500 with expected crc 2023-10-09T06:12:05.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:05 smithi064 ceph-mon[212750]: failed to encode map e500 with expected crc 2023-10-09T06:12:05.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:05 smithi064 ceph-mon[220044]: failed to encode map e500 with expected crc 2023-10-09T06:12:05.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:05 smithi064 ceph-mon[220044]: osdmap e500: 8 total, 8 up, 8 in 2023-10-09T06:12:05.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:05 smithi064 ceph-mon[220044]: failed to encode map e500 with expected crc 2023-10-09T06:12:05.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:05 smithi064 ceph-mon[220044]: failed to encode map e500 with expected crc 2023-10-09T06:12:06.100 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:12:06.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:06.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:12:06.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:05 smithi156 ceph-mon[195101]: failed to encode map e500 with expected crc 2023-10-09T06:12:06.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:05 smithi156 ceph-mon[195101]: osdmap e500: 8 total, 8 up, 8 in 2023-10-09T06:12:06.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:05 smithi156 ceph-mon[195101]: failed to encode map e500 with expected crc 2023-10-09T06:12:06.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:05 smithi156 ceph-mon[195101]: failed to encode map e500 with expected crc 2023-10-09T06:12:06.210 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:12:06.283 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:06.315 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 12 16 24847 24831 129.262 97.375 0.00701409 0.00772554 2023-10-09T06:12:06.508 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:12:06.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:06.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:12:06.538 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:06.621 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:12:06.686 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:06.936 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:06 smithi064 ceph-mon[212750]: pgmap v1266: 119 pgs: 119 active+clean; 975 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-09T06:12:06.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:06 smithi064 ceph-mon[212750]: failed to encode map e501 with expected crc 2023-10-09T06:12:06.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:06 smithi064 ceph-mon[212750]: osdmap e501: 8 total, 8 up, 8 in 2023-10-09T06:12:06.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:06 smithi064 ceph-mon[212750]: failed to encode map e501 with expected crc 2023-10-09T06:12:06.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:06 smithi064 ceph-mon[212750]: failed to encode map e501 with expected crc 2023-10-09T06:12:06.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:06 smithi064 ceph-mon[212750]: failed to encode map e501 with expected crc 2023-10-09T06:12:06.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:06 smithi064 ceph-mon[220044]: pgmap v1266: 119 pgs: 119 active+clean; 975 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-09T06:12:06.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:06 smithi064 ceph-mon[220044]: failed to encode map e501 with expected crc 2023-10-09T06:12:06.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:06 smithi064 ceph-mon[220044]: osdmap e501: 8 total, 8 up, 8 in 2023-10-09T06:12:06.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:06 smithi064 ceph-mon[220044]: failed to encode map e501 with expected crc 2023-10-09T06:12:06.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:06 smithi064 ceph-mon[220044]: failed to encode map e501 with expected crc 2023-10-09T06:12:06.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:06 smithi064 ceph-mon[220044]: failed to encode map e501 with expected crc 2023-10-09T06:12:06.953 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:12:06.953 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:12:07.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:06 smithi156 ceph-mon[195101]: pgmap v1266: 119 pgs: 119 active+clean; 975 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-09T06:12:07.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:06 smithi156 ceph-mon[195101]: failed to encode map e501 with expected crc 2023-10-09T06:12:07.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:06 smithi156 ceph-mon[195101]: osdmap e501: 8 total, 8 up, 8 in 2023-10-09T06:12:07.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:06 smithi156 ceph-mon[195101]: failed to encode map e501 with expected crc 2023-10-09T06:12:07.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:06 smithi156 ceph-mon[195101]: failed to encode map e501 with expected crc 2023-10-09T06:12:07.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:06 smithi156 ceph-mon[195101]: failed to encode map e501 with expected crc 2023-10-09T06:12:07.306 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:12:07.314 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 13 16 26523 26507 127.377 104.75 0.00886611 0.00784406 2023-10-09T06:12:07.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:07.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:12:07.418 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:12:07.469 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:07.548 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:12:07.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:07.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:12:07.657 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:12:07.719 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:07 smithi156 ceph-mon[195101]: failed to encode map e502 with expected crc 2023-10-09T06:12:07.719 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:07 smithi156 ceph-mon[195101]: failed to encode map e502 with expected crc 2023-10-09T06:12:07.719 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:07 smithi156 ceph-mon[195101]: osdmap e502: 8 total, 8 up, 8 in 2023-10-09T06:12:07.720 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:07 smithi156 ceph-mon[195101]: failed to encode map e502 with expected crc 2023-10-09T06:12:07.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:07 smithi064 ceph-mon[220044]: failed to encode map e502 with expected crc 2023-10-09T06:12:07.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:07 smithi064 ceph-mon[220044]: failed to encode map e502 with expected crc 2023-10-09T06:12:07.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:07 smithi064 ceph-mon[220044]: osdmap e502: 8 total, 8 up, 8 in 2023-10-09T06:12:07.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:07 smithi064 ceph-mon[220044]: failed to encode map e502 with expected crc 2023-10-09T06:12:07.728 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:08.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:07 smithi064 ceph-mon[212750]: failed to encode map e502 with expected crc 2023-10-09T06:12:08.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:07 smithi064 ceph-mon[212750]: failed to encode map e502 with expected crc 2023-10-09T06:12:08.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:07 smithi064 ceph-mon[212750]: osdmap e502: 8 total, 8 up, 8 in 2023-10-09T06:12:08.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:07 smithi064 ceph-mon[212750]: failed to encode map e502 with expected crc 2023-10-09T06:12:08.315 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 14 16 28374 28358 126.54 115.688 0.0111387 0.00789691 2023-10-09T06:12:08.643 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:12:08.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:08.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:12:08.749 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:12:08.776 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:08.831 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:12:08.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:08.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:12:08.937 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:12:08.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:08 smithi064 ceph-mon[212750]: pgmap v1269: 119 pgs: 1 active+undersized+remapped, 118 active+clean; 1.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2023-10-09T06:12:08.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:08 smithi064 ceph-mon[212750]: failed to encode map e503 with expected crc 2023-10-09T06:12:08.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:08 smithi064 ceph-mon[212750]: osdmap e503: 8 total, 8 up, 8 in 2023-10-09T06:12:08.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:08 smithi064 ceph-mon[212750]: failed to encode map e503 with expected crc 2023-10-09T06:12:08.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:08 smithi064 ceph-mon[212750]: failed to encode map e503 with expected crc 2023-10-09T06:12:08.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:08 smithi064 ceph-mon[212750]: failed to encode map e503 with expected crc 2023-10-09T06:12:08.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:08 smithi064 ceph-mon[212750]: failed to encode map e503 with expected crc 2023-10-09T06:12:08.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:08 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:08.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:08 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:08.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:08 smithi064 ceph-mon[220044]: pgmap v1269: 119 pgs: 1 active+undersized+remapped, 118 active+clean; 1.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2023-10-09T06:12:08.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:08 smithi064 ceph-mon[220044]: failed to encode map e503 with expected crc 2023-10-09T06:12:08.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:08 smithi064 ceph-mon[220044]: osdmap e503: 8 total, 8 up, 8 in 2023-10-09T06:12:08.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:08 smithi064 ceph-mon[220044]: failed to encode map e503 with expected crc 2023-10-09T06:12:08.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:08 smithi064 ceph-mon[220044]: failed to encode map e503 with expected crc 2023-10-09T06:12:08.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:08 smithi064 ceph-mon[220044]: failed to encode map e503 with expected crc 2023-10-09T06:12:08.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:08 smithi064 ceph-mon[220044]: failed to encode map e503 with expected crc 2023-10-09T06:12:08.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:08 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:08.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:08 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:09.010 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:09.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:08 smithi156 ceph-mon[195101]: pgmap v1269: 119 pgs: 1 active+undersized+remapped, 118 active+clean; 1.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2023-10-09T06:12:09.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:08 smithi156 ceph-mon[195101]: failed to encode map e503 with expected crc 2023-10-09T06:12:09.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:08 smithi156 ceph-mon[195101]: osdmap e503: 8 total, 8 up, 8 in 2023-10-09T06:12:09.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:08 smithi156 ceph-mon[195101]: failed to encode map e503 with expected crc 2023-10-09T06:12:09.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:08 smithi156 ceph-mon[195101]: failed to encode map e503 with expected crc 2023-10-09T06:12:09.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:08 smithi156 ceph-mon[195101]: failed to encode map e503 with expected crc 2023-10-09T06:12:09.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:08 smithi156 ceph-mon[195101]: failed to encode map e503 with expected crc 2023-10-09T06:12:09.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:08 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:09.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:08 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:09.315 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 15 16 29702 29686 123.638 83 0.00340074 0.00808015 2023-10-09T06:12:09.923 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:12:10.123 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:12:10.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:10.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:12:10.182 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:12:10.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:10 smithi064 ceph-mon[212750]: pgmap v1271: 119 pgs: 1 active+undersized+remapped, 118 active+clean; 1.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T06:12:10.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:10 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:10.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:10 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3494240956' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:12:10.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:10 smithi064 ceph-mon[220044]: pgmap v1271: 119 pgs: 1 active+undersized+remapped, 118 active+clean; 1.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T06:12:10.215 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:10 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:10.215 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:10 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3494240956' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:12:10.231 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:12:10.315 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 16 16 31105 31089 121.391 87.6875 0.0149032 0.00823271 2023-10-09T06:12:10.354 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:10.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:10 smithi156 ceph-mon[195101]: pgmap v1271: 119 pgs: 1 active+undersized+remapped, 118 active+clean; 1.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T06:12:10.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:10 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:10.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:10 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3494240956' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:12:10.631 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:12:10.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:10.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:12:10.740 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:12:10.813 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:11.147 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1, 2] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 4, 0] 2023-10-09T06:12:11.147 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:12:11.147 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T06:12:11.148 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:12:11.316 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 17 16 32366 32350 118.884 78.8125 0.0043633 0.00840598 2023-10-09T06:12:11.425 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:12:11.427 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:11 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:12:11.427 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:11 smithi064 ceph-mon[220044]: failed to encode map e504 with expected crc 2023-10-09T06:12:11.427 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:11 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3494240956' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:12:11.427 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:11 smithi064 ceph-mon[220044]: osdmap e504: 8 total, 8 up, 8 in 2023-10-09T06:12:11.427 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:11 smithi064 ceph-mon[220044]: failed to encode map e504 with expected crc 2023-10-09T06:12:11.427 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:11 smithi064 ceph-mon[220044]: failed to encode map e504 with expected crc 2023-10-09T06:12:11.430 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:11 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:12:11.430 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:11 smithi064 ceph-mon[212750]: failed to encode map e504 with expected crc 2023-10-09T06:12:11.430 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:11 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3494240956' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:12:11.430 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:11 smithi064 ceph-mon[212750]: osdmap e504: 8 total, 8 up, 8 in 2023-10-09T06:12:11.431 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:11 smithi064 ceph-mon[212750]: failed to encode map e504 with expected crc 2023-10-09T06:12:11.431 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:11 smithi064 ceph-mon[212750]: failed to encode map e504 with expected crc 2023-10-09T06:12:11.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:11.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:12:11.536 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:12:11.603 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:11.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:11 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:12:11.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:11 smithi156 ceph-mon[195101]: failed to encode map e504 with expected crc 2023-10-09T06:12:11.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:11 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3494240956' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:12:11.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:11 smithi156 ceph-mon[195101]: osdmap e504: 8 total, 8 up, 8 in 2023-10-09T06:12:11.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:11 smithi156 ceph-mon[195101]: failed to encode map e504 with expected crc 2023-10-09T06:12:11.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:11 smithi156 ceph-mon[195101]: failed to encode map e504 with expected crc 2023-10-09T06:12:11.726 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:11.742 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:12:11.748 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:12:11.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:11.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:12:11.786 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:12:11.835 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:12:11.851 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:12:11.879 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:12:11.881 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:11.921 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:12:11.938 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:12:11.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:11.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:12:11.959 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:12:11.999 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:11.999 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:12:12.040 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:12:12.043 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:12:12.044 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:12:12.044 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:12:12.044 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:12:12.044 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:12:12.044 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:12:12.044 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:12:12.044 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:12:12.044 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:12:12.118 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:12.316 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 18 16 33647 33631 116.727 80.0625 0.00812861 0.00856178 2023-10-09T06:12:12.361 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:12 smithi064 ceph-mon[220044]: pgmap v1273: 119 pgs: 119 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 157 MiB/s wr, 2.52k op/s 2023-10-09T06:12:12.361 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:12 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1240006748' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:12:12.361 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:12 smithi064 ceph-mon[212750]: pgmap v1273: 119 pgs: 119 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 157 MiB/s wr, 2.52k op/s 2023-10-09T06:12:12.361 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:12 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1240006748' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:12:12.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:12 smithi156 ceph-mon[195101]: pgmap v1273: 119 pgs: 119 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 157 MiB/s wr, 2.52k op/s 2023-10-09T06:12:12.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:12 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1240006748' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:12:13.230 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:12:13.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:13.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:12:13.316 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 19 16 35077 35061 115.287 89.375 0.00634945 0.00866841 2023-10-09T06:12:13.336 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:12:13.387 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:13.427 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:12:13.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:13.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:12:13.450 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:12:13.534 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:12:13.562 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:14.316 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-10-09T06:12:14.317234+0000 min lat: 0.0010971 max lat: 0.248233 avg lat: 0.00880639 2023-10-09T06:12:14.317 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:12:14.317 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 20 16 36345 36329 113.484 79.25 0.00926861 0.00880639 2023-10-09T06:12:14.382 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:12:14.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:14.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:12:14.494 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:12:14.533 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:14.595 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:12:14.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:14.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:12:14.702 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:12:14.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:14 smithi064 ceph-mon[212750]: pgmap v1274: 119 pgs: 119 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 77 MiB/s wr, 1.22k op/s 2023-10-09T06:12:14.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:14.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:14.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:14 smithi064 ceph-mon[220044]: pgmap v1274: 119 pgs: 119 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 77 MiB/s wr, 1.22k op/s 2023-10-09T06:12:14.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:14.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:14.765 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:14.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:14 smithi156 ceph-mon[195101]: pgmap v1274: 119 pgs: 119 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 77 MiB/s wr, 1.22k op/s 2023-10-09T06:12:14.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:14.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:15.284 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:12:15.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:15.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:12:15.316 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 21 16 37654 37638 111.976 81.8125 0.0155012 0.00892562 2023-10-09T06:12:15.389 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:12:15.467 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:15.477 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:15.612 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:12:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:12:15] "GET /metrics HTTP/1.1" 200 36516 "" "Prometheus/2.43.0" 2023-10-09T06:12:15.761 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:12:15.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:15.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:12:15.870 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:12:15.942 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:16.230 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:12:16.316 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:12:16.316 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 22 16 39037 39021 110.815 86.4375 0.0214575 0.00901759 2023-10-09T06:12:16.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:16.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:12:16.333 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:12:16.333 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T06:12:16.333 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2023-10-09T06:12:16.334 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:12:16.423 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:12:16.453 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:16.509 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:12:16.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:16.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:12:16.613 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:12:16.701 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:16.758 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:16 smithi064 ceph-mon[212750]: pgmap v1275: 119 pgs: 119 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s 2023-10-09T06:12:16.760 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:16 smithi064 ceph-mon[220044]: pgmap v1275: 119 pgs: 119 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s 2023-10-09T06:12:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:16 smithi156 ceph-mon[195101]: pgmap v1275: 119 pgs: 119 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s 2023-10-09T06:12:16.994 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:12:17.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:17.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:12:17.104 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:12:17.174 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:17.317 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 23 16 40401 40385 109.703 85.25 0.0120375 0.00911028 2023-10-09T06:12:17.433 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:12:17.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:17.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:12:17.544 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:12:17.609 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:17.702 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:12:17.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:17.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:12:17.809 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:12:17.839 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:17.884 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:12:17.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:17.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:12:17.988 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:12:18.042 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:18.317 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 24 16 41561 41545 108.153 72.5 0.00226624 0.00920711 2023-10-09T06:12:18.562 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:12:18.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:18.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:12:18.668 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:12:18.760 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:18.830 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:12:18.832 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T06:12:18.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:18 smithi064 ceph-mon[212750]: pgmap v1276: 119 pgs: 119 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2023-10-09T06:12:18.855 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:18 smithi064 ceph-mon[220044]: pgmap v1276: 119 pgs: 119 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2023-10-09T06:12:19.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:18 smithi156 ceph-mon[195101]: pgmap v1276: 119 pgs: 119 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2023-10-09T06:12:19.307 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:12:19.317 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 25 16 42665 42649 106.587 69 0.0145134 0.00937758 2023-10-09T06:12:19.371 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:12:19.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:19.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:12:19.479 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:12:19.603 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:20.318 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 26 16 43843 43827 105.319 73.625 0.020559 0.00948802 2023-10-09T06:12:20.319 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:12:20.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:20.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:12:20.338 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1276,"stamp":"2023-10-09T06:12:17.101565+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2509327369,"num_objects":38304,"num_object_clones":0,"num_object_copies":114912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":38304,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":38346,"num_write_kb":2450004,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":637482,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":43326,"ondisk_log_size":43326,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12653612,"kb_used_data":7283500,"kb_used_omap":165,"kb_used_meta":5369882,"kb_avail":737307604,"statfs":{"total":767960285184,"available":755002986496,"internally_reserved":0,"allocated":7458304000,"data_stored":7441674659,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":169911,"internal_metadata":5498759241},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,2,6,3,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":938868736,"num_objects":14326,"num_object_clones":0,"num_object_copies":42978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14326,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14326,"num_write_kb":916864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.380637"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":712,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.732774+0000","last_change":"2023-10-09T06:11:56.780847+0000","last_active":"2023-10-09T06:12:07.732774+0000","last_peered":"2023-10-09T06:12:07.732774+0000","last_clean":"2023-10-09T06:12:07.732774+0000","last_became_active":"2023-10-09T06:11:56.780327+0000","last_became_peered":"2023-10-09T06:11:56.780327+0000","last_unstale":"2023-10-09T06:12:07.732774+0000","last_undegraded":"2023-10-09T06:12:07.732774+0000","last_fullsized":"2023-10-09T06:12:07.732774+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:05.412941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:05.412941+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:16.535142+0000","scrub_duration":0.0061014329999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":572,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.733806+0000","last_change":"2023-10-09T06:07:43.572773+0000","last_active":"2023-10-09T06:12:07.733806+0000","last_peered":"2023-10-09T06:12:07.733806+0000","last_clean":"2023-10-09T06:12:07.733806+0000","last_became_active":"2023-10-09T06:07:43.572299+0000","last_became_peered":"2023-10-09T06:07:43.572299+0000","last_unstale":"2023-10-09T06:12:07.733806+0000","last_undegraded":"2023-10-09T06:12:07.733806+0000","last_fullsized":"2023-10-09T06:12:07.733806+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:00:37.672435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":313,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.729783+0000","last_change":"2023-10-09T06:05:11.451972+0000","last_active":"2023-10-09T06:12:07.729783+0000","last_peered":"2023-10-09T06:12:07.729783+0000","last_clean":"2023-10-09T06:12:07.729783+0000","last_became_active":"2023-10-09T06:05:11.451448+0000","last_became_peered":"2023-10-09T06:05:11.451448+0000","last_unstale":"2023-10-09T06:12:07.729783+0000","last_undegraded":"2023-10-09T06:12:07.729783+0000","last_fullsized":"2023-10-09T06:12:07.729783+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:39.183621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":803,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.734064+0000","last_change":"2023-10-09T06:09:27.554162+0000","last_active":"2023-10-09T06:12:07.734064+0000","last_peered":"2023-10-09T06:12:07.734064+0000","last_clean":"2023-10-09T06:12:07.734064+0000","last_became_active":"2023-10-09T06:09:27.553659+0000","last_became_peered":"2023-10-09T06:09:27.553659+0000","last_unstale":"2023-10-09T06:12:07.734064+0000","last_undegraded":"2023-10-09T06:12:07.734064+0000","last_fullsized":"2023-10-09T06:12:07.734064+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:01.468346+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:01.468346+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:03.443511+0000","scrub_duration":0.0016258220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":185,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.732209+0000","last_change":"2023-10-09T06:01:54.762060+0000","last_active":"2023-10-09T06:12:07.732209+0000","last_peered":"2023-10-09T06:12:07.732209+0000","last_clean":"2023-10-09T06:12:07.732209+0000","last_became_active":"2023-10-09T06:01:54.761757+0000","last_became_peered":"2023-10-09T06:01:54.761757+0000","last_unstale":"2023-10-09T06:12:07.732209+0000","last_undegraded":"2023-10-09T06:12:07.732209+0000","last_fullsized":"2023-10-09T06:12:07.732209+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:40.294232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":443,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.733833+0000","last_change":"2023-10-09T06:11:53.738199+0000","last_active":"2023-10-09T06:12:07.733833+0000","last_peered":"2023-10-09T06:12:07.733833+0000","last_clean":"2023-10-09T06:12:07.733833+0000","last_became_active":"2023-10-09T06:11:53.737857+0000","last_became_peered":"2023-10-09T06:11:53.737857+0000","last_unstale":"2023-10-09T06:12:07.733833+0000","last_undegraded":"2023-10-09T06:12:07.733833+0000","last_fullsized":"2023-10-09T06:12:07.733833+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:30.875503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":788,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.739417+0000","last_change":"2023-10-09T06:12:07.739121+0000","last_active":"2023-10-09T06:12:07.739417+0000","last_peered":"2023-10-09T06:12:07.739417+0000","last_clean":"2023-10-09T06:12:07.739417+0000","last_became_active":"2023-10-09T06:12:07.738739+0000","last_became_peered":"2023-10-09T06:12:07.738739+0000","last_unstale":"2023-10-09T06:12:07.739417+0000","last_undegraded":"2023-10-09T06:12:07.739417+0000","last_fullsized":"2023-10-09T06:12:07.739417+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.293426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.293426+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:19.815422+0000","scrub_duration":0.001587946,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":296,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.723054+0000","last_change":"2023-10-09T06:09:27.564377+0000","last_active":"2023-10-09T06:12:07.723054+0000","last_peered":"2023-10-09T06:12:07.723054+0000","last_clean":"2023-10-09T06:12:07.723054+0000","last_became_active":"2023-10-09T06:09:27.563710+0000","last_became_peered":"2023-10-09T06:09:27.563710+0000","last_unstale":"2023-10-09T06:12:07.723054+0000","last_undegraded":"2023-10-09T06:12:07.723054+0000","last_fullsized":"2023-10-09T06:12:07.723054+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:48.168447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":410,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.734612+0000","last_change":"2023-10-09T06:11:30.165482+0000","last_active":"2023-10-09T06:12:07.734612+0000","last_peered":"2023-10-09T06:12:07.734612+0000","last_clean":"2023-10-09T06:12:07.734612+0000","last_became_active":"2023-10-09T06:11:30.165161+0000","last_became_peered":"2023-10-09T06:11:30.165161+0000","last_unstale":"2023-10-09T06:12:07.734612+0000","last_undegraded":"2023-10-09T06:12:07.734612+0000","last_fullsized":"2023-10-09T06:12:07.734612+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:44.820752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":712,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.733975+0000","last_change":"2023-10-09T06:09:06.649199+0000","last_active":"2023-10-09T06:12:07.733975+0000","last_peered":"2023-10-09T06:12:07.733975+0000","last_clean":"2023-10-09T06:12:07.733975+0000","last_became_active":"2023-10-09T06:09:06.648801+0000","last_became_peered":"2023-10-09T06:09:06.648801+0000","last_unstale":"2023-10-09T06:12:07.733975+0000","last_undegraded":"2023-10-09T06:12:07.733975+0000","last_fullsized":"2023-10-09T06:12:07.733975+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.384407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.384407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:48:23.220950+0000","scrub_duration":0.0040651960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":401,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.727439+0000","last_change":"2023-10-09T06:09:27.559756+0000","last_active":"2023-10-09T06:12:07.727439+0000","last_peered":"2023-10-09T06:12:07.727439+0000","last_clean":"2023-10-09T06:12:07.727439+0000","last_became_active":"2023-10-09T06:09:27.559443+0000","last_became_peered":"2023-10-09T06:09:27.559443+0000","last_unstale":"2023-10-09T06:12:07.727439+0000","last_undegraded":"2023-10-09T06:12:07.727439+0000","last_fullsized":"2023-10-09T06:12:07.727439+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:14.519398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":323,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.723122+0000","last_change":"2023-10-09T06:06:01.067544+0000","last_active":"2023-10-09T06:12:07.723122+0000","last_peered":"2023-10-09T06:12:07.723122+0000","last_clean":"2023-10-09T06:12:07.723122+0000","last_became_active":"2023-10-09T06:06:01.067188+0000","last_became_peered":"2023-10-09T06:06:01.067188+0000","last_unstale":"2023-10-09T06:12:07.723122+0000","last_undegraded":"2023-10-09T06:12:07.723122+0000","last_fullsized":"2023-10-09T06:12:07.723122+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:43.464186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":628,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.737463+0000","last_change":"2023-10-09T06:10:11.729656+0000","last_active":"2023-10-09T06:12:07.737463+0000","last_peered":"2023-10-09T06:12:07.737463+0000","last_clean":"2023-10-09T06:12:07.737463+0000","last_became_active":"2023-10-09T06:10:11.728960+0000","last_became_peered":"2023-10-09T06:10:11.728960+0000","last_unstale":"2023-10-09T06:12:07.737463+0000","last_undegraded":"2023-10-09T06:12:07.737463+0000","last_fullsized":"2023-10-09T06:12:07.737463+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:20.742561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:20.742561+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:45.543709+0000","scrub_duration":0.0027005039999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":459,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.734228+0000","last_change":"2023-10-09T06:11:53.735093+0000","last_active":"2023-10-09T06:12:07.734228+0000","last_peered":"2023-10-09T06:12:07.734228+0000","last_clean":"2023-10-09T06:12:07.734228+0000","last_became_active":"2023-10-09T06:11:53.734629+0000","last_became_peered":"2023-10-09T06:11:53.734629+0000","last_unstale":"2023-10-09T06:12:07.734228+0000","last_undegraded":"2023-10-09T06:12:07.734228+0000","last_fullsized":"2023-10-09T06:12:07.734228+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:05.584805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":471,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.723821+0000","last_change":"2023-10-09T06:11:53.738132+0000","last_active":"2023-10-09T06:12:07.723821+0000","last_peered":"2023-10-09T06:12:07.723821+0000","last_clean":"2023-10-09T06:12:07.723821+0000","last_became_active":"2023-10-09T06:11:53.737798+0000","last_became_peered":"2023-10-09T06:11:53.737798+0000","last_unstale":"2023-10-09T06:12:07.723821+0000","last_undegraded":"2023-10-09T06:12:07.723821+0000","last_fullsized":"2023-10-09T06:12:07.723821+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:28.025838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":695,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.727272+0000","last_change":"2023-10-09T06:11:56.775270+0000","last_active":"2023-10-09T06:12:07.727272+0000","last_peered":"2023-10-09T06:12:07.727272+0000","last_clean":"2023-10-09T06:12:07.727272+0000","last_became_active":"2023-10-09T06:11:56.774867+0000","last_became_peered":"2023-10-09T06:11:56.774867+0000","last_unstale":"2023-10-09T06:12:07.727272+0000","last_undegraded":"2023-10-09T06:12:07.727272+0000","last_fullsized":"2023-10-09T06:12:07.727272+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:38.588641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:38.588641+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:42.249351+0000","scrub_duration":0.010116385,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":120,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.729697+0000","last_change":"2023-10-09T06:09:28.558317+0000","last_active":"2023-10-09T06:12:07.729697+0000","last_peered":"2023-10-09T06:12:07.729697+0000","last_clean":"2023-10-09T06:12:07.729697+0000","last_became_active":"2023-10-09T06:09:28.543965+0000","last_became_peered":"2023-10-09T06:09:28.543965+0000","last_unstale":"2023-10-09T06:12:07.729697+0000","last_undegraded":"2023-10-09T06:12:07.729697+0000","last_fullsized":"2023-10-09T06:12:07.729697+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:27:29.745235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":335,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.729671+0000","last_change":"2023-10-09T06:09:27.553337+0000","last_active":"2023-10-09T06:12:07.729671+0000","last_peered":"2023-10-09T06:12:07.729671+0000","last_clean":"2023-10-09T06:12:07.729671+0000","last_became_active":"2023-10-09T06:09:27.551069+0000","last_became_peered":"2023-10-09T06:09:27.551069+0000","last_unstale":"2023-10-09T06:12:07.729671+0000","last_undegraded":"2023-10-09T06:12:07.729671+0000","last_fullsized":"2023-10-09T06:12:07.729671+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:47.664965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":675,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.734371+0000","last_change":"2023-10-09T06:09:27.574336+0000","last_active":"2023-10-09T06:12:07.734371+0000","last_peered":"2023-10-09T06:12:07.734371+0000","last_clean":"2023-10-09T06:12:07.734371+0000","last_became_active":"2023-10-09T06:09:27.573974+0000","last_became_peered":"2023-10-09T06:09:27.573974+0000","last_unstale":"2023-10-09T06:12:07.734371+0000","last_undegraded":"2023-10-09T06:12:07.734371+0000","last_fullsized":"2023-10-09T06:12:07.734371+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.339121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.339121+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:45.908579+0000","scrub_duration":0.010474344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":518,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.732672+0000","last_change":"2023-10-09T06:11:30.181872+0000","last_active":"2023-10-09T06:12:07.732672+0000","last_peered":"2023-10-09T06:12:07.732672+0000","last_clean":"2023-10-09T06:12:07.732672+0000","last_became_active":"2023-10-09T06:11:30.167850+0000","last_became_peered":"2023-10-09T06:11:30.167850+0000","last_unstale":"2023-10-09T06:12:07.732672+0000","last_undegraded":"2023-10-09T06:12:07.732672+0000","last_fullsized":"2023-10-09T06:12:07.732672+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:08.406244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":516,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.734423+0000","last_change":"2023-10-09T06:01:54.764916+0000","last_active":"2023-10-09T06:12:07.734423+0000","last_peered":"2023-10-09T06:12:07.734423+0000","last_clean":"2023-10-09T06:12:07.734423+0000","last_became_active":"2023-10-09T06:01:54.764705+0000","last_became_peered":"2023-10-09T06:01:54.764705+0000","last_unstale":"2023-10-09T06:12:07.734423+0000","last_undegraded":"2023-10-09T06:12:07.734423+0000","last_fullsized":"2023-10-09T06:12:07.734423+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:12.527719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":714,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.723368+0000","last_change":"2023-10-09T06:09:27.564817+0000","last_active":"2023-10-09T06:12:07.723368+0000","last_peered":"2023-10-09T06:12:07.723368+0000","last_clean":"2023-10-09T06:12:07.723368+0000","last_became_active":"2023-10-09T06:09:27.563589+0000","last_became_peered":"2023-10-09T06:09:27.563589+0000","last_unstale":"2023-10-09T06:12:07.723368+0000","last_undegraded":"2023-10-09T06:12:07.723368+0000","last_fullsized":"2023-10-09T06:12:07.723368+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:42.900876+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:42.900876+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:12.383955+0000","scrub_duration":0.0014928649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":93,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.734351+0000","last_change":"2023-10-09T06:11:53.745233+0000","last_active":"2023-10-09T06:12:07.734351+0000","last_peered":"2023-10-09T06:12:07.734351+0000","last_clean":"2023-10-09T06:12:07.734351+0000","last_became_active":"2023-10-09T06:11:53.735973+0000","last_became_peered":"2023-10-09T06:11:53.735973+0000","last_unstale":"2023-10-09T06:12:07.734351+0000","last_undegraded":"2023-10-09T06:12:07.734351+0000","last_fullsized":"2023-10-09T06:12:07.734351+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:11.575293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":548,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.727325+0000","last_change":"2023-10-09T06:01:54.761558+0000","last_active":"2023-10-09T06:12:07.727325+0000","last_peered":"2023-10-09T06:12:07.727325+0000","last_clean":"2023-10-09T06:12:07.727325+0000","last_became_active":"2023-10-09T06:01:54.761251+0000","last_became_peered":"2023-10-09T06:01:54.761251+0000","last_unstale":"2023-10-09T06:12:07.727325+0000","last_undegraded":"2023-10-09T06:12:07.727325+0000","last_fullsized":"2023-10-09T06:12:07.727325+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:53.245267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":967,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.736083+0000","last_change":"2023-10-09T06:11:56.777063+0000","last_active":"2023-10-09T06:12:07.736083+0000","last_peered":"2023-10-09T06:12:07.736083+0000","last_clean":"2023-10-09T06:12:07.736083+0000","last_became_active":"2023-10-09T06:11:56.776702+0000","last_became_peered":"2023-10-09T06:11:56.776702+0000","last_unstale":"2023-10-09T06:12:07.736083+0000","last_undegraded":"2023-10-09T06:12:07.736083+0000","last_fullsized":"2023-10-09T06:12:07.736083+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:54.446900+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":443,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.723488+0000","last_change":"2023-10-09T06:09:28.543938+0000","last_active":"2023-10-09T06:12:07.723488+0000","last_peered":"2023-10-09T06:12:07.723488+0000","last_clean":"2023-10-09T06:12:07.723488+0000","last_became_active":"2023-10-09T06:09:28.543628+0000","last_became_peered":"2023-10-09T06:09:28.543628+0000","last_unstale":"2023-10-09T06:12:07.723488+0000","last_undegraded":"2023-10-09T06:12:07.723488+0000","last_fullsized":"2023-10-09T06:12:07.723488+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:57:27.970416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":348,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.729586+0000","last_change":"2023-10-09T06:11:53.740173+0000","last_active":"2023-10-09T06:12:07.729586+0000","last_peered":"2023-10-09T06:12:07.729586+0000","last_clean":"2023-10-09T06:12:07.729586+0000","last_became_active":"2023-10-09T06:11:53.739869+0000","last_became_peered":"2023-10-09T06:11:53.739869+0000","last_unstale":"2023-10-09T06:12:07.729586+0000","last_undegraded":"2023-10-09T06:12:07.729586+0000","last_fullsized":"2023-10-09T06:12:07.729586+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:16:43.276363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":726,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.733313+0000","last_change":"2023-10-09T06:11:56.780510+0000","last_active":"2023-10-09T06:12:07.733313+0000","last_peered":"2023-10-09T06:12:07.733313+0000","last_clean":"2023-10-09T06:12:07.733313+0000","last_became_active":"2023-10-09T06:11:56.780057+0000","last_became_peered":"2023-10-09T06:11:56.780057+0000","last_unstale":"2023-10-09T06:12:07.733313+0000","last_undegraded":"2023-10-09T06:12:07.733313+0000","last_fullsized":"2023-10-09T06:12:07.733313+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.316697+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.316697+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:07.866189+0000","scrub_duration":0.001666561,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":357,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.737416+0000","last_change":"2023-10-09T06:01:22.337171+0000","last_active":"2023-10-09T06:12:07.737416+0000","last_peered":"2023-10-09T06:12:07.737416+0000","last_clean":"2023-10-09T06:12:07.737416+0000","last_became_active":"2023-10-09T06:01:22.335438+0000","last_became_peered":"2023-10-09T06:01:22.335438+0000","last_unstale":"2023-10-09T06:12:07.737416+0000","last_undegraded":"2023-10-09T06:12:07.737416+0000","last_fullsized":"2023-10-09T06:12:07.737416+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:25.036448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":456,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.726399+0000","last_change":"2023-10-09T06:09:28.544528+0000","last_active":"2023-10-09T06:12:07.726399+0000","last_peered":"2023-10-09T06:12:07.726399+0000","last_clean":"2023-10-09T06:12:07.726399+0000","last_became_active":"2023-10-09T06:09:28.544242+0000","last_became_peered":"2023-10-09T06:09:28.544242+0000","last_unstale":"2023-10-09T06:12:07.726399+0000","last_undegraded":"2023-10-09T06:12:07.726399+0000","last_fullsized":"2023-10-09T06:12:07.726399+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:29:07.672911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":776,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.732732+0000","last_change":"2023-10-09T06:11:33.177363+0000","last_active":"2023-10-09T06:12:07.732732+0000","last_peered":"2023-10-09T06:12:07.732732+0000","last_clean":"2023-10-09T06:12:07.732732+0000","last_became_active":"2023-10-09T06:11:33.176993+0000","last_became_peered":"2023-10-09T06:11:33.176993+0000","last_unstale":"2023-10-09T06:12:07.732732+0000","last_undegraded":"2023-10-09T06:12:07.732732+0000","last_fullsized":"2023-10-09T06:12:07.732732+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:40.388125+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:40.388125+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:16.726353+0000","scrub_duration":0.0055576990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":56,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.728752+0000","last_change":"2023-10-09T06:10:50.746147+0000","last_active":"2023-10-09T06:12:07.728752+0000","last_peered":"2023-10-09T06:12:07.728752+0000","last_clean":"2023-10-09T06:12:07.728752+0000","last_became_active":"2023-10-09T06:10:50.745733+0000","last_became_peered":"2023-10-09T06:10:50.745733+0000","last_unstale":"2023-10-09T06:12:07.728752+0000","last_undegraded":"2023-10-09T06:12:07.728752+0000","last_fullsized":"2023-10-09T06:12:07.728752+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:17.611365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":265,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.731693+0000","last_change":"2023-10-09T06:11:53.744599+0000","last_active":"2023-10-09T06:12:07.731693+0000","last_peered":"2023-10-09T06:12:07.731693+0000","last_clean":"2023-10-09T06:12:07.731693+0000","last_became_active":"2023-10-09T06:11:53.744237+0000","last_became_peered":"2023-10-09T06:11:53.744237+0000","last_unstale":"2023-10-09T06:12:07.731693+0000","last_undegraded":"2023-10-09T06:12:07.731693+0000","last_fullsized":"2023-10-09T06:12:07.731693+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:36.115605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.6","version":"504'2476","reported_seq":2499,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T06:12:15.796435+0000","last_change":"2023-10-09T06:11:46.032384+0000","last_active":"2023-10-09T06:12:15.796435+0000","last_peered":"2023-10-09T06:12:15.796435+0000","last_clean":"2023-10-09T06:12:15.796435+0000","last_became_active":"2023-10-09T06:11:46.032091+0000","last_became_peered":"2023-10-09T06:11:46.032091+0000","last_unstale":"2023-10-09T06:12:15.796435+0000","last_undegraded":"2023-10-09T06:12:15.796435+0000","last_fullsized":"2023-10-09T06:12:15.796435+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":2476,"log_dups_size":0,"ondisk_log_size":2476,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:31.117384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162267136,"num_objects":2476,"num_object_clones":0,"num_object_copies":7428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2476,"num_write_kb":158464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":744,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.734278+0000","last_change":"2023-10-09T06:10:59.422644+0000","last_active":"2023-10-09T06:12:07.734278+0000","last_peered":"2023-10-09T06:12:07.734278+0000","last_clean":"2023-10-09T06:12:07.734278+0000","last_became_active":"2023-10-09T06:09:06.649707+0000","last_became_peered":"2023-10-09T06:09:06.649707+0000","last_unstale":"2023-10-09T06:12:07.734278+0000","last_undegraded":"2023-10-09T06:12:07.734278+0000","last_fullsized":"2023-10-09T06:12:07.734278+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:10:59.422551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:10:59.422551+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:08.392769+0000","scrub_duration":0.0042232470000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":120,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.732175+0000","last_change":"2023-10-09T06:09:28.539123+0000","last_active":"2023-10-09T06:12:07.732175+0000","last_peered":"2023-10-09T06:12:07.732175+0000","last_clean":"2023-10-09T06:12:07.732175+0000","last_became_active":"2023-10-09T06:09:28.538726+0000","last_became_peered":"2023-10-09T06:09:28.538726+0000","last_unstale":"2023-10-09T06:12:07.732175+0000","last_undegraded":"2023-10-09T06:12:07.732175+0000","last_fullsized":"2023-10-09T06:12:07.732175+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:24.132933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":377,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.732636+0000","last_change":"2023-10-09T06:09:03.594770+0000","last_active":"2023-10-09T06:12:07.732636+0000","last_peered":"2023-10-09T06:12:07.732636+0000","last_clean":"2023-10-09T06:12:07.732636+0000","last_became_active":"2023-10-09T06:09:03.594371+0000","last_became_peered":"2023-10-09T06:09:03.594371+0000","last_unstale":"2023-10-09T06:12:07.732636+0000","last_undegraded":"2023-10-09T06:12:07.732636+0000","last_fullsized":"2023-10-09T06:12:07.732636+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:41.361335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.7","version":"504'2362","reported_seq":2385,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T06:12:15.249280+0000","last_change":"2023-10-09T06:11:46.032166+0000","last_active":"2023-10-09T06:12:15.249280+0000","last_peered":"2023-10-09T06:12:15.249280+0000","last_clean":"2023-10-09T06:12:15.249280+0000","last_became_active":"2023-10-09T06:11:46.031915+0000","last_became_peered":"2023-10-09T06:11:46.031915+0000","last_unstale":"2023-10-09T06:12:15.249280+0000","last_undegraded":"2023-10-09T06:12:15.249280+0000","last_fullsized":"2023-10-09T06:12:15.249280+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":2362,"log_dups_size":0,"ondisk_log_size":2362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:15.690065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154796032,"num_objects":2362,"num_object_clones":0,"num_object_copies":7086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2362,"num_write_kb":151168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":851,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.727142+0000","last_change":"2023-10-09T06:11:36.648291+0000","last_active":"2023-10-09T06:12:07.727142+0000","last_peered":"2023-10-09T06:12:07.727142+0000","last_clean":"2023-10-09T06:12:07.727142+0000","last_became_active":"2023-10-09T06:11:36.647960+0000","last_became_peered":"2023-10-09T06:11:36.647960+0000","last_unstale":"2023-10-09T06:12:07.727142+0000","last_undegraded":"2023-10-09T06:12:07.727142+0000","last_fullsized":"2023-10-09T06:12:07.727142+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:03.241335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:03.241335+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:25.724953+0000","scrub_duration":0.017780547000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":120,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.732814+0000","last_change":"2023-10-09T06:09:28.542485+0000","last_active":"2023-10-09T06:12:07.732814+0000","last_peered":"2023-10-09T06:12:07.732814+0000","last_clean":"2023-10-09T06:12:07.732814+0000","last_became_active":"2023-10-09T06:09:28.539178+0000","last_became_peered":"2023-10-09T06:09:28.539178+0000","last_unstale":"2023-10-09T06:12:07.732814+0000","last_undegraded":"2023-10-09T06:12:07.732814+0000","last_fullsized":"2023-10-09T06:12:07.732814+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:34.804101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":542,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.732504+0000","last_change":"2023-10-09T06:01:54.760367+0000","last_active":"2023-10-09T06:12:07.732504+0000","last_peered":"2023-10-09T06:12:07.732504+0000","last_clean":"2023-10-09T06:12:07.732504+0000","last_became_active":"2023-10-09T06:01:54.760087+0000","last_became_peered":"2023-10-09T06:01:54.760087+0000","last_unstale":"2023-10-09T06:12:07.732504+0000","last_undegraded":"2023-10-09T06:12:07.732504+0000","last_fullsized":"2023-10-09T06:12:07.732504+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:26.000104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"504'2320","reported_seq":2342,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T06:12:15.698837+0000","last_change":"2023-10-09T06:11:45.696810+0000","last_active":"2023-10-09T06:12:15.698837+0000","last_peered":"2023-10-09T06:12:15.698837+0000","last_clean":"2023-10-09T06:12:15.698837+0000","last_became_active":"2023-10-09T06:11:45.696476+0000","last_became_peered":"2023-10-09T06:11:45.696476+0000","last_unstale":"2023-10-09T06:12:15.698837+0000","last_undegraded":"2023-10-09T06:12:15.698837+0000","last_fullsized":"2023-10-09T06:12:15.698837+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":2320,"log_dups_size":0,"ondisk_log_size":2320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:41:33.351567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152043520,"num_objects":2320,"num_object_clones":0,"num_object_copies":6960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2320,"num_write_kb":148480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":747,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.732263+0000","last_change":"2023-10-09T06:11:56.768849+0000","last_active":"2023-10-09T06:12:07.732263+0000","last_peered":"2023-10-09T06:12:07.732263+0000","last_clean":"2023-10-09T06:12:07.732263+0000","last_became_active":"2023-10-09T06:11:56.768516+0000","last_became_peered":"2023-10-09T06:11:56.768516+0000","last_unstale":"2023-10-09T06:12:07.732263+0000","last_undegraded":"2023-10-09T06:12:07.732263+0000","last_fullsized":"2023-10-09T06:12:07.732263+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:48.495739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:48.495739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:17.288016+0000","scrub_duration":0.0016345509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":279,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.729415+0000","last_change":"2023-10-09T06:11:30.183789+0000","last_active":"2023-10-09T06:12:07.729415+0000","last_peered":"2023-10-09T06:12:07.729415+0000","last_clean":"2023-10-09T06:12:07.729415+0000","last_became_active":"2023-10-09T06:11:30.170153+0000","last_became_peered":"2023-10-09T06:11:30.170153+0000","last_unstale":"2023-10-09T06:12:07.729415+0000","last_undegraded":"2023-10-09T06:12:07.729415+0000","last_fullsized":"2023-10-09T06:12:07.729415+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:32.285122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":509,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.732488+0000","last_change":"2023-10-09T06:09:28.540287+0000","last_active":"2023-10-09T06:12:07.732488+0000","last_peered":"2023-10-09T06:12:07.732488+0000","last_clean":"2023-10-09T06:12:07.732488+0000","last_became_active":"2023-10-09T06:09:28.540012+0000","last_became_peered":"2023-10-09T06:09:28.540012+0000","last_unstale":"2023-10-09T06:12:07.732488+0000","last_undegraded":"2023-10-09T06:12:07.732488+0000","last_fullsized":"2023-10-09T06:12:07.732488+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:50.310629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"504'2343","reported_seq":2366,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T06:12:15.795959+0000","last_change":"2023-10-09T06:11:46.031195+0000","last_active":"2023-10-09T06:12:15.795959+0000","last_peered":"2023-10-09T06:12:15.795959+0000","last_clean":"2023-10-09T06:12:15.795959+0000","last_became_active":"2023-10-09T06:11:46.030891+0000","last_became_peered":"2023-10-09T06:11:46.030891+0000","last_unstale":"2023-10-09T06:12:15.795959+0000","last_undegraded":"2023-10-09T06:12:15.795959+0000","last_fullsized":"2023-10-09T06:12:15.795959+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":2343,"log_dups_size":0,"ondisk_log_size":2343,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:25.788661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153550848,"num_objects":2343,"num_object_clones":0,"num_object_copies":7029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2343,"num_write_kb":149952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":699,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.732567+0000","last_change":"2023-10-09T06:09:06.648659+0000","last_active":"2023-10-09T06:12:07.732567+0000","last_peered":"2023-10-09T06:12:07.732567+0000","last_clean":"2023-10-09T06:12:07.732567+0000","last_became_active":"2023-10-09T06:09:06.648332+0000","last_became_peered":"2023-10-09T06:09:06.648332+0000","last_unstale":"2023-10-09T06:12:07.732567+0000","last_undegraded":"2023-10-09T06:12:07.732567+0000","last_fullsized":"2023-10-09T06:12:07.732567+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:16.361097+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:16.361097+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:52:53.333570+0000","scrub_duration":0.028533749000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":272,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.727213+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T06:12:07.727213+0000","last_peered":"2023-10-09T06:12:07.727213+0000","last_clean":"2023-10-09T06:12:07.727213+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T06:12:07.727213+0000","last_undegraded":"2023-10-09T06:12:07.727213+0000","last_fullsized":"2023-10-09T06:12:07.727213+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":516,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.728946+0000","last_change":"2023-10-09T06:09:27.558353+0000","last_active":"2023-10-09T06:12:07.728946+0000","last_peered":"2023-10-09T06:12:07.728946+0000","last_clean":"2023-10-09T06:12:07.728946+0000","last_became_active":"2023-10-09T06:09:27.557233+0000","last_became_peered":"2023-10-09T06:09:27.557233+0000","last_unstale":"2023-10-09T06:12:07.728946+0000","last_undegraded":"2023-10-09T06:12:07.728946+0000","last_fullsized":"2023-10-09T06:12:07.728946+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:53.149350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"504'2431","reported_seq":2454,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T06:12:16.309572+0000","last_change":"2023-10-09T06:11:45.694648+0000","last_active":"2023-10-09T06:12:16.309572+0000","last_peered":"2023-10-09T06:12:16.309572+0000","last_clean":"2023-10-09T06:12:16.309572+0000","last_became_active":"2023-10-09T06:11:45.694321+0000","last_became_peered":"2023-10-09T06:11:45.694321+0000","last_unstale":"2023-10-09T06:12:16.309572+0000","last_undegraded":"2023-10-09T06:12:16.309572+0000","last_fullsized":"2023-10-09T06:12:16.309572+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":2431,"log_dups_size":0,"ondisk_log_size":2431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:43.704525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159318016,"num_objects":2431,"num_object_clones":0,"num_object_copies":7293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2431,"num_write_kb":155584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":548,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.723746+0000","last_change":"2023-10-09T06:11:42.795888+0000","last_active":"2023-10-09T06:12:07.723746+0000","last_peered":"2023-10-09T06:12:07.723746+0000","last_clean":"2023-10-09T06:12:07.723746+0000","last_became_active":"2023-10-09T06:11:33.174664+0000","last_became_peered":"2023-10-09T06:11:33.174664+0000","last_unstale":"2023-10-09T06:12:07.723746+0000","last_undegraded":"2023-10-09T06:12:07.723746+0000","last_fullsized":"2023-10-09T06:12:07.723746+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:42.795781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:42.795781+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:54.467987+0000","scrub_duration":0.001491041,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":306,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.723720+0000","last_change":"2023-10-09T06:09:27.565666+0000","last_active":"2023-10-09T06:12:07.723720+0000","last_peered":"2023-10-09T06:12:07.723720+0000","last_clean":"2023-10-09T06:12:07.723720+0000","last_became_active":"2023-10-09T06:09:27.565324+0000","last_became_peered":"2023-10-09T06:09:27.565324+0000","last_unstale":"2023-10-09T06:12:07.723720+0000","last_undegraded":"2023-10-09T06:12:07.723720+0000","last_fullsized":"2023-10-09T06:12:07.723720+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:51.216947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":468,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.732647+0000","last_change":"2023-10-09T06:11:30.164883+0000","last_active":"2023-10-09T06:12:07.732647+0000","last_peered":"2023-10-09T06:12:07.732647+0000","last_clean":"2023-10-09T06:12:07.732647+0000","last_became_active":"2023-10-09T06:11:30.164361+0000","last_became_peered":"2023-10-09T06:11:30.164361+0000","last_unstale":"2023-10-09T06:12:07.732647+0000","last_undegraded":"2023-10-09T06:12:07.732647+0000","last_fullsized":"2023-10-09T06:12:07.732647+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:05.759535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"504'2370","reported_seq":2389,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T06:12:15.698526+0000","last_change":"2023-10-09T06:11:53.740707+0000","last_active":"2023-10-09T06:12:15.698526+0000","last_peered":"2023-10-09T06:12:15.698526+0000","last_clean":"2023-10-09T06:12:15.698526+0000","last_became_active":"2023-10-09T06:11:53.740371+0000","last_became_peered":"2023-10-09T06:11:53.740371+0000","last_unstale":"2023-10-09T06:12:15.698526+0000","last_undegraded":"2023-10-09T06:12:15.698526+0000","last_fullsized":"2023-10-09T06:12:15.698526+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":2370,"log_dups_size":0,"ondisk_log_size":2370,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:12.301519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155320320,"num_objects":2370,"num_object_clones":0,"num_object_copies":7110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2370,"num_write_kb":151680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":749,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.722842+0000","last_change":"2023-10-09T06:07:46.602409+0000","last_active":"2023-10-09T06:12:07.722842+0000","last_peered":"2023-10-09T06:12:07.722842+0000","last_clean":"2023-10-09T06:12:07.722842+0000","last_became_active":"2023-10-09T06:07:46.602045+0000","last_became_peered":"2023-10-09T06:07:46.602045+0000","last_unstale":"2023-10-09T06:12:07.722842+0000","last_undegraded":"2023-10-09T06:12:07.722842+0000","last_fullsized":"2023-10-09T06:12:07.722842+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.282393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.282393+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:35.114362+0000","scrub_duration":0.01402403,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":413,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.733590+0000","last_change":"2023-10-09T06:11:53.742245+0000","last_active":"2023-10-09T06:12:07.733590+0000","last_peered":"2023-10-09T06:12:07.733590+0000","last_clean":"2023-10-09T06:12:07.733590+0000","last_became_active":"2023-10-09T06:11:53.741901+0000","last_became_peered":"2023-10-09T06:11:53.741901+0000","last_unstale":"2023-10-09T06:12:07.733590+0000","last_undegraded":"2023-10-09T06:12:07.733590+0000","last_fullsized":"2023-10-09T06:12:07.733590+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:25.243667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":552,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.734083+0000","last_change":"2023-10-09T06:07:43.572173+0000","last_active":"2023-10-09T06:12:07.734083+0000","last_peered":"2023-10-09T06:12:07.734083+0000","last_clean":"2023-10-09T06:12:07.734083+0000","last_became_active":"2023-10-09T06:07:43.571690+0000","last_became_peered":"2023-10-09T06:07:43.571690+0000","last_unstale":"2023-10-09T06:12:07.734083+0000","last_undegraded":"2023-10-09T06:12:07.734083+0000","last_fullsized":"2023-10-09T06:12:07.734083+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:10.234125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"504'2500","reported_seq":2532,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T06:12:16.348483+0000","last_change":"2023-10-09T06:11:53.743291+0000","last_active":"2023-10-09T06:12:16.348483+0000","last_peered":"2023-10-09T06:12:16.348483+0000","last_clean":"2023-10-09T06:12:16.348483+0000","last_became_active":"2023-10-09T06:11:53.742997+0000","last_became_peered":"2023-10-09T06:11:53.742997+0000","last_unstale":"2023-10-09T06:12:16.348483+0000","last_undegraded":"2023-10-09T06:12:16.348483+0000","last_fullsized":"2023-10-09T06:12:16.348483+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":2500,"log_dups_size":0,"ondisk_log_size":2500,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:01.455755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840000,"num_objects":2500,"num_object_clones":0,"num_object_copies":7500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2500,"num_write_kb":160000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":820,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.733150+0000","last_change":"2023-10-09T06:11:33.177331+0000","last_active":"2023-10-09T06:12:07.733150+0000","last_peered":"2023-10-09T06:12:07.733150+0000","last_clean":"2023-10-09T06:12:07.733150+0000","last_became_active":"2023-10-09T06:11:33.176972+0000","last_became_peered":"2023-10-09T06:11:33.176972+0000","last_unstale":"2023-10-09T06:12:07.733150+0000","last_undegraded":"2023-10-09T06:12:07.733150+0000","last_fullsized":"2023-10-09T06:12:07.733150+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.373748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.373748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:31.674340+0000","scrub_duration":0.0024576450000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":494,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.732636+0000","last_change":"2023-10-09T06:07:43.574863+0000","last_active":"2023-10-09T06:12:07.732636+0000","last_peered":"2023-10-09T06:12:07.732636+0000","last_clean":"2023-10-09T06:12:07.732636+0000","last_became_active":"2023-10-09T06:07:43.574576+0000","last_became_peered":"2023-10-09T06:07:43.574576+0000","last_unstale":"2023-10-09T06:12:07.732636+0000","last_undegraded":"2023-10-09T06:12:07.732636+0000","last_fullsized":"2023-10-09T06:12:07.732636+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:16.166004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":525,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.732652+0000","last_change":"2023-10-09T06:06:01.068708+0000","last_active":"2023-10-09T06:12:07.732652+0000","last_peered":"2023-10-09T06:12:07.732652+0000","last_clean":"2023-10-09T06:12:07.732652+0000","last_became_active":"2023-10-09T06:06:01.068474+0000","last_became_peered":"2023-10-09T06:06:01.068474+0000","last_unstale":"2023-10-09T06:12:07.732652+0000","last_undegraded":"2023-10-09T06:12:07.732652+0000","last_fullsized":"2023-10-09T06:12:07.732652+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:31.749069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"504'2425","reported_seq":2447,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T06:12:15.736125+0000","last_change":"2023-10-09T06:11:45.695709+0000","last_active":"2023-10-09T06:12:15.736125+0000","last_peered":"2023-10-09T06:12:15.736125+0000","last_clean":"2023-10-09T06:12:15.736125+0000","last_became_active":"2023-10-09T06:11:45.695219+0000","last_became_peered":"2023-10-09T06:11:45.695219+0000","last_unstale":"2023-10-09T06:12:15.736125+0000","last_undegraded":"2023-10-09T06:12:15.736125+0000","last_fullsized":"2023-10-09T06:12:15.736125+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":2425,"log_dups_size":0,"ondisk_log_size":2425,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:26.533964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158924800,"num_objects":2425,"num_object_clones":0,"num_object_copies":7275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2425,"num_write_kb":155200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":632,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.733444+0000","last_change":"2023-10-09T06:11:56.767180+0000","last_active":"2023-10-09T06:12:07.733444+0000","last_peered":"2023-10-09T06:12:07.733444+0000","last_clean":"2023-10-09T06:12:07.733444+0000","last_became_active":"2023-10-09T06:11:56.766832+0000","last_became_peered":"2023-10-09T06:11:56.766832+0000","last_unstale":"2023-10-09T06:12:07.733444+0000","last_undegraded":"2023-10-09T06:12:07.733444+0000","last_fullsized":"2023-10-09T06:12:07.733444+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:39.633410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:39.633410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:31:46.365915+0000","scrub_duration":0.0039938459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":305,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.722982+0000","last_change":"2023-10-09T06:09:03.587524+0000","last_active":"2023-10-09T06:12:07.722982+0000","last_peered":"2023-10-09T06:12:07.722982+0000","last_clean":"2023-10-09T06:12:07.722982+0000","last_became_active":"2023-10-09T06:09:03.586906+0000","last_became_peered":"2023-10-09T06:09:03.586906+0000","last_unstale":"2023-10-09T06:12:07.722982+0000","last_undegraded":"2023-10-09T06:12:07.722982+0000","last_fullsized":"2023-10-09T06:12:07.722982+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:35:01.785360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":185,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.734778+0000","last_change":"2023-10-09T06:01:54.761768+0000","last_active":"2023-10-09T06:12:07.734778+0000","last_peered":"2023-10-09T06:12:07.734778+0000","last_clean":"2023-10-09T06:12:07.734778+0000","last_became_active":"2023-10-09T06:01:54.761443+0000","last_became_peered":"2023-10-09T06:01:54.761443+0000","last_unstale":"2023-10-09T06:12:07.734778+0000","last_undegraded":"2023-10-09T06:12:07.734778+0000","last_fullsized":"2023-10-09T06:12:07.734778+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:51.555987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"504'2369","reported_seq":2391,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T06:12:15.726275+0000","last_change":"2023-10-09T06:11:45.695985+0000","last_active":"2023-10-09T06:12:15.726275+0000","last_peered":"2023-10-09T06:12:15.726275+0000","last_clean":"2023-10-09T06:12:15.726275+0000","last_became_active":"2023-10-09T06:11:45.695457+0000","last_became_peered":"2023-10-09T06:11:45.695457+0000","last_unstale":"2023-10-09T06:12:15.726275+0000","last_undegraded":"2023-10-09T06:12:15.726275+0000","last_fullsized":"2023-10-09T06:12:15.726275+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":2369,"log_dups_size":0,"ondisk_log_size":2369,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:04.418441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155254784,"num_objects":2369,"num_object_clones":0,"num_object_copies":7107,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2369,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2369,"num_write_kb":151616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":651,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.734471+0000","last_change":"2023-10-09T06:11:56.774093+0000","last_active":"2023-10-09T06:12:07.734471+0000","last_peered":"2023-10-09T06:12:07.734471+0000","last_clean":"2023-10-09T06:12:07.734471+0000","last_became_active":"2023-10-09T06:11:56.773767+0000","last_became_peered":"2023-10-09T06:11:56.773767+0000","last_unstale":"2023-10-09T06:12:07.734471+0000","last_undegraded":"2023-10-09T06:12:07.734471+0000","last_fullsized":"2023-10-09T06:12:07.734471+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:38.767847+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:38.767847+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":247,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.733039+0000","last_change":"2023-10-09T06:12:04.680247+0000","last_active":"2023-10-09T06:12:07.733039+0000","last_peered":"2023-10-09T06:12:07.733039+0000","last_clean":"2023-10-09T06:12:07.733039+0000","last_became_active":"2023-10-09T06:12:04.679943+0000","last_became_peered":"2023-10-09T06:12:04.679943+0000","last_unstale":"2023-10-09T06:12:07.733039+0000","last_undegraded":"2023-10-09T06:12:07.733039+0000","last_fullsized":"2023-10-09T06:12:07.733039+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:56.692774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":291,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.729502+0000","last_change":"2023-10-09T06:07:43.577306+0000","last_active":"2023-10-09T06:12:07.729502+0000","last_peered":"2023-10-09T06:12:07.729502+0000","last_clean":"2023-10-09T06:12:07.729502+0000","last_became_active":"2023-10-09T06:07:43.576956+0000","last_became_peered":"2023-10-09T06:07:43.576956+0000","last_unstale":"2023-10-09T06:12:07.729502+0000","last_undegraded":"2023-10-09T06:12:07.729502+0000","last_fullsized":"2023-10-09T06:12:07.729502+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:21.020403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":209,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.732558+0000","last_change":"2023-10-09T06:09:27.557353+0000","last_active":"2023-10-09T06:12:07.732558+0000","last_peered":"2023-10-09T06:12:07.732558+0000","last_clean":"2023-10-09T06:12:07.732558+0000","last_became_active":"2023-10-09T06:09:27.557042+0000","last_became_peered":"2023-10-09T06:09:27.557042+0000","last_unstale":"2023-10-09T06:12:07.732558+0000","last_undegraded":"2023-10-09T06:12:07.732558+0000","last_fullsized":"2023-10-09T06:12:07.732558+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:28.264019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"504'2340","reported_seq":2363,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T06:12:15.783561+0000","last_change":"2023-10-09T06:11:46.029349+0000","last_active":"2023-10-09T06:12:15.783561+0000","last_peered":"2023-10-09T06:12:15.783561+0000","last_clean":"2023-10-09T06:12:15.783561+0000","last_became_active":"2023-10-09T06:11:46.029041+0000","last_became_peered":"2023-10-09T06:11:46.029041+0000","last_unstale":"2023-10-09T06:12:15.783561+0000","last_undegraded":"2023-10-09T06:12:15.783561+0000","last_fullsized":"2023-10-09T06:12:15.783561+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":2340,"log_dups_size":0,"ondisk_log_size":2340,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:07.662395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153354240,"num_objects":2340,"num_object_clones":0,"num_object_copies":7020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2340,"num_write_kb":149760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":738,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.728695+0000","last_change":"2023-10-09T06:09:27.544555+0000","last_active":"2023-10-09T06:12:07.728695+0000","last_peered":"2023-10-09T06:12:07.728695+0000","last_clean":"2023-10-09T06:12:07.728695+0000","last_became_active":"2023-10-09T06:09:27.544222+0000","last_became_peered":"2023-10-09T06:09:27.544222+0000","last_unstale":"2023-10-09T06:12:07.728695+0000","last_undegraded":"2023-10-09T06:12:07.728695+0000","last_fullsized":"2023-10-09T06:12:07.728695+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:45.944253+0000","scrub_duration":0.0016660799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":194,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.732651+0000","last_change":"2023-10-09T06:10:26.655740+0000","last_active":"2023-10-09T06:12:07.732651+0000","last_peered":"2023-10-09T06:12:07.732651+0000","last_clean":"2023-10-09T06:12:07.732651+0000","last_became_active":"2023-10-09T06:10:26.649431+0000","last_became_peered":"2023-10-09T06:10:26.649431+0000","last_unstale":"2023-10-09T06:12:07.732651+0000","last_undegraded":"2023-10-09T06:12:07.732651+0000","last_fullsized":"2023-10-09T06:12:07.732651+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:36.123840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":484,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.733721+0000","last_change":"2023-10-09T06:09:27.553958+0000","last_active":"2023-10-09T06:12:07.733721+0000","last_peered":"2023-10-09T06:12:07.733721+0000","last_clean":"2023-10-09T06:12:07.733721+0000","last_became_active":"2023-10-09T06:09:27.553193+0000","last_became_peered":"2023-10-09T06:09:27.553193+0000","last_unstale":"2023-10-09T06:12:07.733721+0000","last_undegraded":"2023-10-09T06:12:07.733721+0000","last_fullsized":"2023-10-09T06:12:07.733721+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:09.754579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":328,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.733118+0000","last_change":"2023-10-09T06:07:43.580765+0000","last_active":"2023-10-09T06:12:07.733118+0000","last_peered":"2023-10-09T06:12:07.733118+0000","last_clean":"2023-10-09T06:12:07.733118+0000","last_became_active":"2023-10-09T06:07:43.580458+0000","last_became_peered":"2023-10-09T06:07:43.580458+0000","last_unstale":"2023-10-09T06:12:07.733118+0000","last_undegraded":"2023-10-09T06:12:07.733118+0000","last_fullsized":"2023-10-09T06:12:07.733118+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:55.161893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"504'2386","reported_seq":2408,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T06:12:15.675872+0000","last_change":"2023-10-09T06:11:46.035099+0000","last_active":"2023-10-09T06:12:15.675872+0000","last_peered":"2023-10-09T06:12:15.675872+0000","last_clean":"2023-10-09T06:12:15.675872+0000","last_became_active":"2023-10-09T06:11:46.034721+0000","last_became_peered":"2023-10-09T06:11:46.034721+0000","last_unstale":"2023-10-09T06:12:15.675872+0000","last_undegraded":"2023-10-09T06:12:15.675872+0000","last_fullsized":"2023-10-09T06:12:15.675872+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":2386,"log_dups_size":0,"ondisk_log_size":2386,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:58:38.054044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156368896,"num_objects":2386,"num_object_clones":0,"num_object_copies":7158,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2386,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2386,"num_write_kb":152704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":594,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.733893+0000","last_change":"2023-10-09T06:11:44.483535+0000","last_active":"2023-10-09T06:12:07.733893+0000","last_peered":"2023-10-09T06:12:07.733893+0000","last_clean":"2023-10-09T06:12:07.733893+0000","last_became_active":"2023-10-09T06:09:28.538562+0000","last_became_peered":"2023-10-09T06:09:28.538562+0000","last_unstale":"2023-10-09T06:12:07.733893+0000","last_undegraded":"2023-10-09T06:12:07.733893+0000","last_fullsized":"2023-10-09T06:12:07.733893+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:44.483427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.483427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:21.958551+0000","scrub_duration":0.0019545769999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":283,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.723514+0000","last_change":"2023-10-09T06:09:27.565004+0000","last_active":"2023-10-09T06:12:07.723514+0000","last_peered":"2023-10-09T06:12:07.723514+0000","last_clean":"2023-10-09T06:12:07.723514+0000","last_became_active":"2023-10-09T06:09:27.564073+0000","last_became_peered":"2023-10-09T06:09:27.564073+0000","last_unstale":"2023-10-09T06:12:07.723514+0000","last_undegraded":"2023-10-09T06:12:07.723514+0000","last_fullsized":"2023-10-09T06:12:07.723514+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:15.146042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":264,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.733496+0000","last_change":"2023-10-09T06:11:30.168188+0000","last_active":"2023-10-09T06:12:07.733496+0000","last_peered":"2023-10-09T06:12:07.733496+0000","last_clean":"2023-10-09T06:12:07.733496+0000","last_became_active":"2023-10-09T06:11:30.167846+0000","last_became_peered":"2023-10-09T06:11:30.167846+0000","last_unstale":"2023-10-09T06:12:07.733496+0000","last_undegraded":"2023-10-09T06:12:07.733496+0000","last_fullsized":"2023-10-09T06:12:07.733496+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:00.415514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":200,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.734715+0000","last_change":"2023-10-09T06:09:27.547953+0000","last_active":"2023-10-09T06:12:07.734715+0000","last_peered":"2023-10-09T06:12:07.734715+0000","last_clean":"2023-10-09T06:12:07.734715+0000","last_became_active":"2023-10-09T06:09:27.547624+0000","last_became_peered":"2023-10-09T06:09:27.547624+0000","last_unstale":"2023-10-09T06:12:07.734715+0000","last_undegraded":"2023-10-09T06:12:07.734715+0000","last_fullsized":"2023-10-09T06:12:07.734715+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:39.630486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"504'2442","reported_seq":2465,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T06:12:16.371976+0000","last_change":"2023-10-09T06:11:45.694696+0000","last_active":"2023-10-09T06:12:16.371976+0000","last_peered":"2023-10-09T06:12:16.371976+0000","last_clean":"2023-10-09T06:12:16.371976+0000","last_became_active":"2023-10-09T06:11:45.694370+0000","last_became_peered":"2023-10-09T06:11:45.694370+0000","last_unstale":"2023-10-09T06:12:16.371976+0000","last_undegraded":"2023-10-09T06:12:16.371976+0000","last_fullsized":"2023-10-09T06:12:16.371976+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":2442,"log_dups_size":0,"ondisk_log_size":2442,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:12.551928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160038912,"num_objects":2442,"num_object_clones":0,"num_object_copies":7326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2442,"num_write_kb":156288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":66,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.723801+0000","last_change":"2023-10-09T06:09:03.586696+0000","last_active":"2023-10-09T06:12:07.723801+0000","last_peered":"2023-10-09T06:12:07.723801+0000","last_clean":"2023-10-09T06:12:07.723801+0000","last_became_active":"2023-10-09T06:09:03.586343+0000","last_became_peered":"2023-10-09T06:09:03.586343+0000","last_unstale":"2023-10-09T06:12:07.723801+0000","last_undegraded":"2023-10-09T06:12:07.723801+0000","last_fullsized":"2023-10-09T06:12:07.723801+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:32.651148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":519,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.728569+0000","last_change":"2023-10-09T06:11:56.778316+0000","last_active":"2023-10-09T06:12:07.728569+0000","last_peered":"2023-10-09T06:12:07.728569+0000","last_clean":"2023-10-09T06:12:07.728569+0000","last_became_active":"2023-10-09T06:11:56.777964+0000","last_became_peered":"2023-10-09T06:11:56.777964+0000","last_unstale":"2023-10-09T06:12:07.728569+0000","last_undegraded":"2023-10-09T06:12:07.728569+0000","last_fullsized":"2023-10-09T06:12:07.728569+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:35.783434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:35.783434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:22.139753+0000","scrub_duration":0.0043794799999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":282,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.723564+0000","last_change":"2023-10-09T06:11:53.737258+0000","last_active":"2023-10-09T06:12:07.723564+0000","last_peered":"2023-10-09T06:12:07.723564+0000","last_clean":"2023-10-09T06:12:07.723564+0000","last_became_active":"2023-10-09T06:11:53.736975+0000","last_became_peered":"2023-10-09T06:11:53.736975+0000","last_unstale":"2023-10-09T06:12:07.723564+0000","last_undegraded":"2023-10-09T06:12:07.723564+0000","last_fullsized":"2023-10-09T06:12:07.723564+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:54:57.287243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":424,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.734006+0000","last_change":"2023-10-09T06:09:27.552948+0000","last_active":"2023-10-09T06:12:07.734006+0000","last_peered":"2023-10-09T06:12:07.734006+0000","last_clean":"2023-10-09T06:12:07.734006+0000","last_became_active":"2023-10-09T06:09:27.552680+0000","last_became_peered":"2023-10-09T06:09:27.552680+0000","last_unstale":"2023-10-09T06:12:07.734006+0000","last_undegraded":"2023-10-09T06:12:07.734006+0000","last_fullsized":"2023-10-09T06:12:07.734006+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:14:08.024300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"504'2400","reported_seq":2423,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T06:12:15.795340+0000","last_change":"2023-10-09T06:11:46.031577+0000","last_active":"2023-10-09T06:12:15.795340+0000","last_peered":"2023-10-09T06:12:15.795340+0000","last_clean":"2023-10-09T06:12:15.795340+0000","last_became_active":"2023-10-09T06:11:46.031338+0000","last_became_peered":"2023-10-09T06:11:46.031338+0000","last_unstale":"2023-10-09T06:12:15.795340+0000","last_undegraded":"2023-10-09T06:12:15.795340+0000","last_fullsized":"2023-10-09T06:12:15.795340+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":2400,"log_dups_size":0,"ondisk_log_size":2400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:18.183021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157286400,"num_objects":2400,"num_object_clones":0,"num_object_copies":7200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2400,"num_write_kb":153600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":737,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.733213+0000","last_change":"2023-10-09T06:11:33.177378+0000","last_active":"2023-10-09T06:12:07.733213+0000","last_peered":"2023-10-09T06:12:07.733213+0000","last_clean":"2023-10-09T06:12:07.733213+0000","last_became_active":"2023-10-09T06:11:33.176968+0000","last_became_peered":"2023-10-09T06:11:33.176968+0000","last_unstale":"2023-10-09T06:12:07.733213+0000","last_undegraded":"2023-10-09T06:12:07.733213+0000","last_fullsized":"2023-10-09T06:12:07.733213+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:57.627296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:57.627296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:37.520663+0000","scrub_duration":0.001913166,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":397,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.735977+0000","last_change":"2023-10-09T06:09:27.544373+0000","last_active":"2023-10-09T06:12:07.735977+0000","last_peered":"2023-10-09T06:12:07.735977+0000","last_clean":"2023-10-09T06:12:07.735977+0000","last_became_active":"2023-10-09T06:09:27.544121+0000","last_became_peered":"2023-10-09T06:09:27.544121+0000","last_unstale":"2023-10-09T06:12:07.735977+0000","last_undegraded":"2023-10-09T06:12:07.735977+0000","last_fullsized":"2023-10-09T06:12:07.735977+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:40.429491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":431,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.728471+0000","last_change":"2023-10-09T06:11:53.738325+0000","last_active":"2023-10-09T06:12:07.728471+0000","last_peered":"2023-10-09T06:12:07.728471+0000","last_clean":"2023-10-09T06:12:07.728471+0000","last_became_active":"2023-10-09T06:11:53.737857+0000","last_became_peered":"2023-10-09T06:11:53.737857+0000","last_unstale":"2023-10-09T06:12:07.728471+0000","last_undegraded":"2023-10-09T06:12:07.728471+0000","last_fullsized":"2023-10-09T06:12:07.728471+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:07:07.909823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":579,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.728973+0000","last_change":"2023-10-09T06:09:27.543678+0000","last_active":"2023-10-09T06:12:07.728973+0000","last_peered":"2023-10-09T06:12:07.728973+0000","last_clean":"2023-10-09T06:12:07.728973+0000","last_became_active":"2023-10-09T06:09:27.543298+0000","last_became_peered":"2023-10-09T06:09:27.543298+0000","last_unstale":"2023-10-09T06:12:07.728973+0000","last_undegraded":"2023-10-09T06:12:07.728973+0000","last_fullsized":"2023-10-09T06:12:07.728973+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:26.297842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"504'2410","reported_seq":2432,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T06:12:15.724919+0000","last_change":"2023-10-09T06:11:45.696353+0000","last_active":"2023-10-09T06:12:15.724919+0000","last_peered":"2023-10-09T06:12:15.724919+0000","last_clean":"2023-10-09T06:12:15.724919+0000","last_became_active":"2023-10-09T06:11:45.696042+0000","last_became_peered":"2023-10-09T06:11:45.696042+0000","last_unstale":"2023-10-09T06:12:15.724919+0000","last_undegraded":"2023-10-09T06:12:15.724919+0000","last_fullsized":"2023-10-09T06:12:15.724919+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":2410,"log_dups_size":0,"ondisk_log_size":2410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:16.750138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157941760,"num_objects":2410,"num_object_clones":0,"num_object_copies":7230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2410,"num_write_kb":154240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":685,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.726400+0000","last_change":"2023-10-09T06:11:45.844988+0000","last_active":"2023-10-09T06:12:07.726400+0000","last_peered":"2023-10-09T06:12:07.726400+0000","last_clean":"2023-10-09T06:12:07.726400+0000","last_became_active":"2023-10-09T06:11:33.173357+0000","last_became_peered":"2023-10-09T06:11:33.173357+0000","last_unstale":"2023-10-09T06:12:07.726400+0000","last_undegraded":"2023-10-09T06:12:07.726400+0000","last_fullsized":"2023-10-09T06:12:07.726400+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:45.844892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:45.844892+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:09.608545+0000","scrub_duration":0.0017521489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":349,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.730052+0000","last_change":"2023-10-09T06:11:30.167040+0000","last_active":"2023-10-09T06:12:07.730052+0000","last_peered":"2023-10-09T06:12:07.730052+0000","last_clean":"2023-10-09T06:12:07.730052+0000","last_became_active":"2023-10-09T06:11:30.166678+0000","last_became_peered":"2023-10-09T06:11:30.166678+0000","last_unstale":"2023-10-09T06:12:07.730052+0000","last_undegraded":"2023-10-09T06:12:07.730052+0000","last_fullsized":"2023-10-09T06:12:07.730052+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:58.662377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":212,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.733208+0000","last_change":"2023-10-09T06:11:30.168040+0000","last_active":"2023-10-09T06:12:07.733208+0000","last_peered":"2023-10-09T06:12:07.733208+0000","last_clean":"2023-10-09T06:12:07.733208+0000","last_became_active":"2023-10-09T06:11:30.167694+0000","last_became_peered":"2023-10-09T06:11:30.167694+0000","last_unstale":"2023-10-09T06:12:07.733208+0000","last_undegraded":"2023-10-09T06:12:07.733208+0000","last_fullsized":"2023-10-09T06:12:07.733208+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:15.351478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":20,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.732875+0000","last_change":"2023-10-09T06:11:53.738315+0000","last_active":"2023-10-09T06:12:07.732875+0000","last_peered":"2023-10-09T06:12:07.732875+0000","last_clean":"2023-10-09T06:12:07.732875+0000","last_became_active":"2023-10-09T06:11:53.737981+0000","last_became_peered":"2023-10-09T06:11:53.737981+0000","last_unstale":"2023-10-09T06:12:07.732875+0000","last_undegraded":"2023-10-09T06:12:07.732875+0000","last_fullsized":"2023-10-09T06:12:07.732875+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:11.946659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.e","version":"504'2387","reported_seq":2409,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T06:12:15.714960+0000","last_change":"2023-10-09T06:11:46.031540+0000","last_active":"2023-10-09T06:12:15.714960+0000","last_peered":"2023-10-09T06:12:15.714960+0000","last_clean":"2023-10-09T06:12:15.714960+0000","last_became_active":"2023-10-09T06:11:46.031225+0000","last_became_peered":"2023-10-09T06:11:46.031225+0000","last_unstale":"2023-10-09T06:12:15.714960+0000","last_undegraded":"2023-10-09T06:12:15.714960+0000","last_fullsized":"2023-10-09T06:12:15.714960+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":2387,"log_dups_size":0,"ondisk_log_size":2387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:07.427097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156434432,"num_objects":2387,"num_object_clones":0,"num_object_copies":7161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2387,"num_write_kb":152768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":957,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.734201+0000","last_change":"2023-10-09T06:07:46.590898+0000","last_active":"2023-10-09T06:12:07.734201+0000","last_peered":"2023-10-09T06:12:07.734201+0000","last_clean":"2023-10-09T06:12:07.734201+0000","last_became_active":"2023-10-09T06:07:46.590596+0000","last_became_peered":"2023-10-09T06:07:46.590596+0000","last_unstale":"2023-10-09T06:12:07.734201+0000","last_undegraded":"2023-10-09T06:12:07.734201+0000","last_fullsized":"2023-10-09T06:12:07.734201+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:17.344518+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:17.344518+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:00.399548+0000","scrub_duration":0.0015914099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":546,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.727531+0000","last_change":"2023-10-09T06:09:28.543659+0000","last_active":"2023-10-09T06:12:07.727531+0000","last_peered":"2023-10-09T06:12:07.727531+0000","last_clean":"2023-10-09T06:12:07.727531+0000","last_became_active":"2023-10-09T06:09:28.543310+0000","last_became_peered":"2023-10-09T06:09:28.543310+0000","last_unstale":"2023-10-09T06:12:07.727531+0000","last_undegraded":"2023-10-09T06:12:07.727531+0000","last_fullsized":"2023-10-09T06:12:07.727531+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:46.713995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":310,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.734304+0000","last_change":"2023-10-09T06:11:53.741811+0000","last_active":"2023-10-09T06:12:07.734304+0000","last_peered":"2023-10-09T06:12:07.734304+0000","last_clean":"2023-10-09T06:12:07.734304+0000","last_became_active":"2023-10-09T06:11:53.741489+0000","last_became_peered":"2023-10-09T06:11:53.741489+0000","last_unstale":"2023-10-09T06:12:07.734304+0000","last_undegraded":"2023-10-09T06:12:07.734304+0000","last_fullsized":"2023-10-09T06:12:07.734304+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:55.391818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":299,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.723189+0000","last_change":"2023-10-09T06:09:28.554885+0000","last_active":"2023-10-09T06:12:07.723189+0000","last_peered":"2023-10-09T06:12:07.723189+0000","last_clean":"2023-10-09T06:12:07.723189+0000","last_became_active":"2023-10-09T06:09:28.554556+0000","last_became_peered":"2023-10-09T06:09:28.554556+0000","last_unstale":"2023-10-09T06:12:07.723189+0000","last_undegraded":"2023-10-09T06:12:07.723189+0000","last_fullsized":"2023-10-09T06:12:07.723189+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:56.537731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"504'2319","reported_seq":2340,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T06:12:15.249403+0000","last_change":"2023-10-09T06:11:46.031709+0000","last_active":"2023-10-09T06:12:15.249403+0000","last_peered":"2023-10-09T06:12:15.249403+0000","last_clean":"2023-10-09T06:12:15.249403+0000","last_became_active":"2023-10-09T06:11:46.031421+0000","last_became_peered":"2023-10-09T06:11:46.031421+0000","last_unstale":"2023-10-09T06:12:15.249403+0000","last_undegraded":"2023-10-09T06:12:15.249403+0000","last_fullsized":"2023-10-09T06:12:15.249403+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":2319,"log_dups_size":0,"ondisk_log_size":2319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:13.095174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151977984,"num_objects":2319,"num_object_clones":0,"num_object_copies":6957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2319,"num_write_kb":148416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":801,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.727062+0000","last_change":"2023-10-09T06:11:33.179099+0000","last_active":"2023-10-09T06:12:07.727062+0000","last_peered":"2023-10-09T06:12:07.727062+0000","last_clean":"2023-10-09T06:12:07.727062+0000","last_became_active":"2023-10-09T06:11:33.178747+0000","last_became_peered":"2023-10-09T06:11:33.178747+0000","last_unstale":"2023-10-09T06:12:07.727062+0000","last_undegraded":"2023-10-09T06:12:07.727062+0000","last_fullsized":"2023-10-09T06:12:07.727062+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:31.798204+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":515,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.727416+0000","last_change":"2023-10-09T06:10:50.740875+0000","last_active":"2023-10-09T06:12:07.727416+0000","last_peered":"2023-10-09T06:12:07.727416+0000","last_clean":"2023-10-09T06:12:07.727416+0000","last_became_active":"2023-10-09T06:10:50.740544+0000","last_became_peered":"2023-10-09T06:10:50.740544+0000","last_unstale":"2023-10-09T06:12:07.727416+0000","last_undegraded":"2023-10-09T06:12:07.727416+0000","last_fullsized":"2023-10-09T06:12:07.727416+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:50:36.995570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":549,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.729229+0000","last_change":"2023-10-09T06:09:27.578756+0000","last_active":"2023-10-09T06:12:07.729229+0000","last_peered":"2023-10-09T06:12:07.729229+0000","last_clean":"2023-10-09T06:12:07.729229+0000","last_became_active":"2023-10-09T06:09:27.578384+0000","last_became_peered":"2023-10-09T06:09:27.578384+0000","last_unstale":"2023-10-09T06:12:07.729229+0000","last_undegraded":"2023-10-09T06:12:07.729229+0000","last_fullsized":"2023-10-09T06:12:07.729229+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:22.182630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":450,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.732717+0000","last_change":"2023-10-09T06:01:22.334081+0000","last_active":"2023-10-09T06:12:07.732717+0000","last_peered":"2023-10-09T06:12:07.732717+0000","last_clean":"2023-10-09T06:12:07.732717+0000","last_became_active":"2023-10-09T06:01:22.333663+0000","last_became_peered":"2023-10-09T06:01:22.333663+0000","last_unstale":"2023-10-09T06:12:07.732717+0000","last_undegraded":"2023-10-09T06:12:07.732717+0000","last_fullsized":"2023-10-09T06:12:07.732717+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:55.809259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":749,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.723893+0000","last_change":"2023-10-09T06:09:27.560157+0000","last_active":"2023-10-09T06:12:07.723893+0000","last_peered":"2023-10-09T06:12:07.723893+0000","last_clean":"2023-10-09T06:12:07.723893+0000","last_became_active":"2023-10-09T06:09:27.559466+0000","last_became_peered":"2023-10-09T06:09:27.559466+0000","last_unstale":"2023-10-09T06:12:07.723893+0000","last_undegraded":"2023-10-09T06:12:07.723893+0000","last_fullsized":"2023-10-09T06:12:07.723893+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.181811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.181811+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:53.431491+0000","scrub_duration":0.011553313000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":423,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.729580+0000","last_change":"2023-10-09T06:11:30.166578+0000","last_active":"2023-10-09T06:12:07.729580+0000","last_peered":"2023-10-09T06:12:07.729580+0000","last_clean":"2023-10-09T06:12:07.729580+0000","last_became_active":"2023-10-09T06:11:30.166184+0000","last_became_peered":"2023-10-09T06:11:30.166184+0000","last_unstale":"2023-10-09T06:12:07.729580+0000","last_undegraded":"2023-10-09T06:12:07.729580+0000","last_fullsized":"2023-10-09T06:12:07.729580+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:59.189549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":205,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T06:12:10.155695+0000","last_change":"2023-10-09T06:07:43.580844+0000","last_active":"2023-10-09T06:12:10.155695+0000","last_peered":"2023-10-09T06:12:10.155695+0000","last_clean":"2023-10-09T06:12:10.155695+0000","last_became_active":"2023-10-09T06:07:43.580523+0000","last_became_peered":"2023-10-09T06:07:43.580523+0000","last_unstale":"2023-10-09T06:12:10.155695+0000","last_undegraded":"2023-10-09T06:12:10.155695+0000","last_fullsized":"2023-10-09T06:12:10.155695+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:55.942430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":852,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.726317+0000","last_change":"2023-10-09T06:00:04.215425+0000","last_active":"2023-10-09T06:12:07.726317+0000","last_peered":"2023-10-09T06:12:07.726317+0000","last_clean":"2023-10-09T06:12:07.726317+0000","last_became_active":"2023-10-09T05:59:40.326685+0000","last_became_peered":"2023-10-09T05:59:40.326685+0000","last_unstale":"2023-10-09T06:12:07.726317+0000","last_undegraded":"2023-10-09T06:12:07.726317+0000","last_fullsized":"2023-10-09T06:12:07.726317+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:04.215270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:04.215270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:03.963694+0000","scrub_duration":0.002850652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":275,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.732896+0000","last_change":"2023-10-09T06:01:54.824780+0000","last_active":"2023-10-09T06:12:07.732896+0000","last_peered":"2023-10-09T06:12:07.732896+0000","last_clean":"2023-10-09T06:12:07.732896+0000","last_became_active":"2023-10-09T06:01:54.757727+0000","last_became_peered":"2023-10-09T06:01:54.757727+0000","last_unstale":"2023-10-09T06:12:07.732896+0000","last_undegraded":"2023-10-09T06:12:07.732896+0000","last_fullsized":"2023-10-09T06:12:07.732896+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:44.132556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":522,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.734138+0000","last_change":"2023-10-09T06:01:54.764989+0000","last_active":"2023-10-09T06:12:07.734138+0000","last_peered":"2023-10-09T06:12:07.734138+0000","last_clean":"2023-10-09T06:12:07.734138+0000","last_became_active":"2023-10-09T06:01:54.764790+0000","last_became_peered":"2023-10-09T06:01:54.764790+0000","last_unstale":"2023-10-09T06:12:07.734138+0000","last_undegraded":"2023-10-09T06:12:07.734138+0000","last_fullsized":"2023-10-09T06:12:07.734138+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:14:07.208858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":275,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.732887+0000","last_change":"2023-10-09T06:10:26.682338+0000","last_active":"2023-10-09T06:12:07.732887+0000","last_peered":"2023-10-09T06:12:07.732887+0000","last_clean":"2023-10-09T06:12:07.732887+0000","last_became_active":"2023-10-09T06:10:26.646411+0000","last_became_peered":"2023-10-09T06:10:26.646411+0000","last_unstale":"2023-10-09T06:12:07.732887+0000","last_undegraded":"2023-10-09T06:12:07.732887+0000","last_fullsized":"2023-10-09T06:12:07.732887+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:10.789202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":519,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.733663+0000","last_change":"2023-10-09T06:09:27.565198+0000","last_active":"2023-10-09T06:12:07.733663+0000","last_peered":"2023-10-09T06:12:07.733663+0000","last_clean":"2023-10-09T06:12:07.733663+0000","last_became_active":"2023-10-09T06:09:27.564909+0000","last_became_peered":"2023-10-09T06:09:27.564909+0000","last_unstale":"2023-10-09T06:12:07.733663+0000","last_undegraded":"2023-10-09T06:12:07.733663+0000","last_fullsized":"2023-10-09T06:12:07.733663+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:46.101014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":641,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.732833+0000","last_change":"2023-10-09T06:11:33.181766+0000","last_active":"2023-10-09T06:12:07.732833+0000","last_peered":"2023-10-09T06:12:07.732833+0000","last_clean":"2023-10-09T06:12:07.732833+0000","last_became_active":"2023-10-09T06:11:33.181369+0000","last_became_peered":"2023-10-09T06:11:33.181369+0000","last_unstale":"2023-10-09T06:12:07.732833+0000","last_undegraded":"2023-10-09T06:12:07.732833+0000","last_fullsized":"2023-10-09T06:12:07.732833+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:00.456132+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:00.456132+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:26.717305+0000","scrub_duration":0.0014992199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":300,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.726502+0000","last_change":"2023-10-09T06:07:43.579543+0000","last_active":"2023-10-09T06:12:07.726502+0000","last_peered":"2023-10-09T06:12:07.726502+0000","last_clean":"2023-10-09T06:12:07.726502+0000","last_became_active":"2023-10-09T06:07:43.579184+0000","last_became_peered":"2023-10-09T06:07:43.579184+0000","last_unstale":"2023-10-09T06:12:07.726502+0000","last_undegraded":"2023-10-09T06:12:07.726502+0000","last_fullsized":"2023-10-09T06:12:07.726502+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:23.460469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":305,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.723588+0000","last_change":"2023-10-09T06:09:27.565185+0000","last_active":"2023-10-09T06:12:07.723588+0000","last_peered":"2023-10-09T06:12:07.723588+0000","last_clean":"2023-10-09T06:12:07.723588+0000","last_became_active":"2023-10-09T06:09:27.564478+0000","last_became_peered":"2023-10-09T06:09:27.564478+0000","last_unstale":"2023-10-09T06:12:07.723588+0000","last_undegraded":"2023-10-09T06:12:07.723588+0000","last_fullsized":"2023-10-09T06:12:07.723588+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:04.622171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":810,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T06:12:10.156047+0000","last_change":"2023-10-09T06:11:56.774098+0000","last_active":"2023-10-09T06:12:10.156047+0000","last_peered":"2023-10-09T06:12:10.156047+0000","last_clean":"2023-10-09T06:12:10.156047+0000","last_became_active":"2023-10-09T06:11:56.773819+0000","last_became_peered":"2023-10-09T06:11:56.773819+0000","last_unstale":"2023-10-09T06:12:10.156047+0000","last_undegraded":"2023-10-09T06:12:10.156047+0000","last_fullsized":"2023-10-09T06:12:10.156047+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:46.290510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:46.290510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:09.762736+0000","scrub_duration":0.002627819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":495,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.729291+0000","last_change":"2023-10-09T06:09:27.556271+0000","last_active":"2023-10-09T06:12:07.729291+0000","last_peered":"2023-10-09T06:12:07.729291+0000","last_clean":"2023-10-09T06:12:07.729291+0000","last_became_active":"2023-10-09T06:09:27.555932+0000","last_became_peered":"2023-10-09T06:09:27.555932+0000","last_unstale":"2023-10-09T06:12:07.729291+0000","last_undegraded":"2023-10-09T06:12:07.729291+0000","last_fullsized":"2023-10-09T06:12:07.729291+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:33.888622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":525,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T06:12:07.732794+0000","last_change":"2023-10-09T06:06:01.068828+0000","last_active":"2023-10-09T06:12:07.732794+0000","last_peered":"2023-10-09T06:12:07.732794+0000","last_clean":"2023-10-09T06:12:07.732794+0000","last_became_active":"2023-10-09T06:06:01.068491+0000","last_became_peered":"2023-10-09T06:06:01.068491+0000","last_unstale":"2023-10-09T06:12:07.732794+0000","last_undegraded":"2023-10-09T06:12:07.732794+0000","last_fullsized":"2023-10-09T06:12:07.732794+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:25.661796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2508718080,"num_objects":38280,"num_object_clones":0,"num_object_copies":114840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":38280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":38280,"num_write_kb":2449920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7371227136,"data_stored":7371227136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":38280,"ondisk_log_size":38280,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":46,"num_bytes_recovered":6862,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13410,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":603785,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":630620,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234391,"num_pgs":11,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1059268,"kb_used_data":158460,"kb_used_omap":0,"kb_used_meta":900800,"kb_avail":92685884,"statfs":{"total":95995035648,"available":94910345216,"internally_reserved":0,"allocated":162263040,"data_stored":160858017,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":922419200},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:11:29 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.1870000000000001,"15min":1.2170000000000001},"min":{"1min":0.26400000000000001,"5min":0.25,"15min":0.25},"max":{"1min":5.7469999999999999,"5min":5.7469999999999999,"15min":5.7469999999999999},"last":0.254},{"interface":"front","average":{"1min":1.5249999999999999,"5min":1.2210000000000001,"15min":1.1699999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.7850000000000001,"5min":7.7850000000000001,"15min":7.7850000000000001},"last":2.0529999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:11:57 2023","interfaces":[{"interface":"back","average":{"1min":1.597,"5min":4.0309999999999997,"15min":3.2320000000000002},"min":{"1min":0.315,"5min":0.218,"15min":0.218},"max":{"1min":6.0419999999999998,"5min":124.66200000000001,"15min":124.66200000000001},"last":0.81999999999999995},{"interface":"front","average":{"1min":0.94599999999999995,"5min":4.0540000000000003,"15min":3.1499999999999999},"min":{"1min":0.34999999999999998,"5min":0.26300000000000001,"15min":0.22700000000000001},"max":{"1min":4.6779999999999999,"5min":125.023,"15min":125.023},"last":1.871}]},{"osd":2,"last update":"Mon Oct 9 06:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.5049999999999999,"15min":2.8650000000000002},"min":{"1min":0.27400000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.4710000000000001,"5min":6.5270000000000001,"15min":127.86499999999999},"last":0.54600000000000004},{"interface":"front","average":{"1min":2.6259999999999999,"5min":1.8300000000000001,"15min":2.9510000000000001},"min":{"1min":0.30099999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.680999999999999,"5min":10.680999999999999,"15min":129.24000000000001},"last":0.46999999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:11:57 2023","interfaces":[{"interface":"back","average":{"1min":1.629,"5min":5.5730000000000004,"15min":5.9699999999999998},"min":{"1min":0.44700000000000001,"5min":0.32100000000000001,"15min":0.29899999999999999},"max":{"1min":8.6809999999999992,"5min":116.68000000000001,"15min":164.16999999999999},"last":3.4100000000000001},{"interface":"front","average":{"1min":1.5640000000000001,"5min":6.0099999999999998,"15min":6.1390000000000002},"min":{"1min":0.45200000000000001,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":4.6390000000000002,"5min":116.797,"15min":164.40700000000001},"last":3.6869999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:11:35 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":1.21,"15min":0.93100000000000005},"min":{"1min":0.29799999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":3.2210000000000001,"5min":14.371,"15min":14.371},"last":0.92600000000000005},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.22,"15min":0.95999999999999996},"min":{"1min":0.30199999999999999,"5min":0.27700000000000002,"15min":0.27100000000000002},"max":{"1min":6.8719999999999999,"5min":14.259,"15min":14.259},"last":0.499}]},{"osd":5,"last update":"Mon Oct 9 06:11:43 2023","interfaces":[{"interface":"back","average":{"1min":3.5299999999999998,"5min":1.448,"15min":1.1479999999999999},"min":{"1min":0.36499999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":43.826999999999998,"5min":43.826999999999998,"15min":43.826999999999998},"last":0.52900000000000003},{"interface":"front","average":{"1min":3.4390000000000001,"5min":1.53,"15min":1.1439999999999999},"min":{"1min":0.44800000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":43.880000000000003,"5min":43.880000000000003,"15min":43.880000000000003},"last":0.56200000000000006}]},{"osd":6,"last update":"Mon Oct 9 06:11:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":0.96299999999999997,"15min":1.028},"min":{"1min":0.314,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.8679999999999999,"5min":9.4540000000000006,"15min":12.727},"last":0.59299999999999997},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.226,"15min":1.0389999999999999},"min":{"1min":0.372,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.516,"5min":6.2329999999999997,"15min":6.2329999999999997},"last":0.432}]}]},{"osd":6,"up_from":46,"seq":197568496027,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1239312,"kb_used_data":337672,"kb_used_omap":0,"kb_used_meta":901632,"kb_avail":92505840,"statfs":{"total":95995035648,"available":94725980160,"internally_reserved":0,"allocated":345776128,"data_stored":344363356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923271168},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:11:28 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":2.0289999999999999,"15min":2.1709999999999998},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":8.3330000000000002,"5min":11.73,"15min":11.73},"last":191.50800000000001},{"interface":"front","average":{"1min":1.3009999999999999,"5min":1.851,"15min":1.9430000000000001},"min":{"1min":0.32500000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":8.3469999999999995,"5min":8.3469999999999995,"15min":8.3469999999999995},"last":191.52600000000001}]},{"osd":1,"last update":"Mon Oct 9 06:11:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6040000000000001,"5min":4.0339999999999998,"15min":3.5859999999999999},"min":{"1min":0.30599999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.1589999999999998,"5min":91.188000000000002,"15min":109.101},"last":0.501},{"interface":"front","average":{"1min":1.635,"5min":4.2199999999999998,"15min":3.6840000000000002},"min":{"1min":0.28499999999999998,"5min":0.27400000000000002,"15min":0.27000000000000002},"max":{"1min":10.27,"5min":91.372,"15min":109.071},"last":0.51800000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:11:23 2023","interfaces":[{"interface":"back","average":{"1min":1.532,"5min":1.8819999999999999,"15min":2.2730000000000001},"min":{"1min":0.27500000000000002,"5min":0.27300000000000002,"15min":0.26200000000000001},"max":{"1min":6.3109999999999999,"5min":15.374000000000001,"15min":61.088999999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.6379999999999999,"5min":1.855,"15min":2.2309999999999999},"min":{"1min":0.35399999999999998,"5min":0.29199999999999998,"15min":0.28599999999999998},"max":{"1min":10.093999999999999,"5min":15.429,"15min":61.082000000000001},"last":0.435}]},{"osd":3,"last update":"Mon Oct 9 06:12:10 2023","interfaces":[{"interface":"back","average":{"1min":6.032,"5min":5.117,"15min":5.9779999999999998},"min":{"1min":0.32600000000000001,"5min":0.318,"15min":0.29599999999999999},"max":{"1min":87.796000000000006,"5min":87.796000000000006,"15min":138.13300000000001},"last":0.55500000000000005},{"interface":"front","average":{"1min":5.7430000000000003,"5min":4.7679999999999998,"15min":5.9000000000000004},"min":{"1min":0.38700000000000001,"5min":0.33600000000000002,"15min":0.28499999999999998},"max":{"1min":87.319999999999993,"5min":87.319999999999993,"15min":137.215},"last":0.40799999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:11:41 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":2.4209999999999998,"15min":1.698},"min":{"1min":0.29699999999999999,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":3.302,"5min":151.35599999999999,"15min":151.35599999999999},"last":0.377},{"interface":"front","average":{"1min":0.78900000000000003,"5min":2.4980000000000002,"15min":1.698},"min":{"1min":0.34499999999999997,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":2.6080000000000001,"5min":149.15799999999999,"15min":149.15799999999999},"last":0.35499999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:11:17 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":1.246,"15min":1.3029999999999999},"min":{"1min":0.28000000000000003,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.9729999999999999,"5min":15.342000000000001,"15min":45.933999999999997},"last":0.34200000000000003},{"interface":"front","average":{"1min":1.083,"5min":1.234,"15min":1.296},"min":{"1min":0.32600000000000001,"5min":0.28199999999999997,"15min":0.255},"max":{"1min":3.8490000000000002,"5min":15.358000000000001,"15min":45.917999999999999},"last":0.58899999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:11:41 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":1.1240000000000001,"15min":1.3939999999999999},"min":{"1min":0.376,"5min":0.311,"15min":0.27000000000000002},"max":{"1min":2.633,"5min":5.6929999999999996,"15min":52.042999999999999},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.85199999999999998,"5min":1.093,"15min":1.3420000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.32200000000000001},"max":{"1min":3.4969999999999999,"5min":5.3710000000000004,"15min":51.259},"last":0.61599999999999999}]}]},{"osd":1,"up_from":89,"seq":382252089616,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1029588,"kb_used_data":477068,"kb_used_omap":9,"kb_used_meta":552502,"kb_avail":92715564,"statfs":{"total":95995035648,"available":94940737536,"internally_reserved":0,"allocated":488517632,"data_stored":485634458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":565763049},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:11:32 2023","interfaces":[{"interface":"back","average":{"1min":2.4260000000000002,"5min":2.306,"15min":2.286},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":9.8070000000000004,"5min":9.8070000000000004,"15min":9.8070000000000004},"last":1.8340000000000001},{"interface":"front","average":{"1min":3.3100000000000001,"5min":2.5339999999999998,"15min":2.4039999999999999},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":24.239999999999998,"5min":24.239999999999998,"15min":24.239999999999998},"last":5.7380000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:11:39 2023","interfaces":[{"interface":"back","average":{"1min":2.133,"5min":1.9930000000000001,"15min":3.569},"min":{"1min":0.254,"5min":0.21099999999999999,"15min":0.20799999999999999},"max":{"1min":15.522,"5min":17.681000000000001,"15min":204.989},"last":1.9079999999999999},{"interface":"front","average":{"1min":2.2919999999999998,"5min":2.3109999999999999,"15min":3.77},"min":{"1min":0.28199999999999997,"5min":0.22700000000000001,"15min":0.20300000000000001},"max":{"1min":8.8369999999999997,"5min":13.539999999999999,"15min":203.65899999999999},"last":1.242}]},{"osd":3,"last update":"Mon Oct 9 06:12:07 2023","interfaces":[{"interface":"back","average":{"1min":7.1100000000000003,"5min":3.8620000000000001,"15min":5.0780000000000003},"min":{"1min":0.28399999999999997,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":113.322,"5min":113.322,"15min":213.654},"last":1.8740000000000001},{"interface":"front","average":{"1min":7.0890000000000004,"5min":4.0449999999999999,"15min":5.0880000000000001},"min":{"1min":0.34200000000000003,"5min":0.27600000000000002,"15min":0.27400000000000002},"max":{"1min":113.38200000000001,"5min":113.38200000000001,"15min":213.77199999999999},"last":0.67500000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:11:39 2023","interfaces":[{"interface":"back","average":{"1min":1.883,"5min":2.073,"15min":1.599},"min":{"1min":0.33900000000000002,"5min":0.31,"15min":0.30399999999999999},"max":{"1min":8.7850000000000001,"5min":35.366,"15min":35.366},"last":0.71599999999999997},{"interface":"front","average":{"1min":1.883,"5min":2.944,"15min":2.093},"min":{"1min":0.36499999999999999,"5min":0.308,"15min":0.308},"max":{"1min":7.8499999999999996,"5min":35.384999999999998,"15min":35.384999999999998},"last":1.151}]},{"osd":5,"last update":"Mon Oct 9 06:11:53 2023","interfaces":[{"interface":"back","average":{"1min":1.3999999999999999,"5min":1.7390000000000001,"15min":1.5629999999999999},"min":{"1min":0.36699999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":8.7539999999999996,"5min":9.3109999999999999,"15min":36.356999999999999},"last":0.77900000000000003},{"interface":"front","average":{"1min":1.7649999999999999,"5min":2.8639999999999999,"15min":2.1709999999999998},"min":{"1min":0.41799999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.5439999999999996,"5min":17.603999999999999,"15min":36.268000000000001},"last":1.214}]},{"osd":6,"last update":"Mon Oct 9 06:11:53 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":2.5369999999999999,"15min":1.8069999999999999},"min":{"1min":0.39200000000000002,"5min":0.34300000000000003,"15min":0.30099999999999999},"max":{"1min":8.8019999999999996,"5min":52.207000000000001,"15min":52.207000000000001},"last":0.748},{"interface":"front","average":{"1min":1.3200000000000001,"5min":3.2189999999999999,"15min":2.2679999999999998},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":5.2809999999999997,"5min":51.212000000000003,"15min":51.212000000000003},"last":1.258}]},{"osd":7,"last update":"Mon Oct 9 06:11:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8899999999999999,"5min":2.923,"15min":2.5870000000000002},"min":{"1min":0.47499999999999998,"5min":0.436,"15min":0.314},"max":{"1min":9.4009999999999998,"5min":15.284000000000001,"15min":108.553},"last":1.2929999999999999},{"interface":"front","average":{"1min":1.8720000000000001,"5min":2.0720000000000001,"15min":2.121},"min":{"1min":0.49199999999999999,"5min":0.38,"15min":0.38},"max":{"1min":7.7149999999999999,"5min":9.9489999999999998,"15min":108.56999999999999},"last":2.0710000000000002}]}]},{"osd":0,"up_from":456,"seq":1958505087011,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1949052,"kb_used_data":1763380,"kb_used_omap":32,"kb_used_meta":185631,"kb_avail":91796100,"statfs":{"total":95995035648,"available":93999206400,"internally_reserved":0,"allocated":1805701120,"data_stored":1803018923,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32794,"internal_metadata":190087142},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:11:33 2023","interfaces":[{"interface":"back","average":{"1min":4.5919999999999996,"5min":9.8249999999999993,"15min":10.696999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":72.177999999999997,"5min":166.833,"15min":166.833},"last":2.5510000000000002},{"interface":"front","average":{"1min":6.2770000000000001,"5min":9.2490000000000006,"15min":9.7439999999999998},"min":{"1min":0.27000000000000002,"5min":0.245,"15min":0.245},"max":{"1min":70.709000000000003,"5min":166.81100000000001,"15min":166.81100000000001},"last":3.5470000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:11:33 2023","interfaces":[{"interface":"back","average":{"1min":2.1240000000000001,"5min":3.6720000000000002,"15min":3.9300000000000002},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":8.5039999999999996,"5min":14.52,"15min":14.52},"last":1.0269999999999999},{"interface":"front","average":{"1min":1.782,"5min":3.4780000000000002,"15min":3.7610000000000001},"min":{"1min":0.27300000000000002,"5min":0.245,"15min":0.245},"max":{"1min":6.6059999999999999,"5min":20.152000000000001,"15min":20.152000000000001},"last":2.6019999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:11:30 2023","interfaces":[{"interface":"back","average":{"1min":2.8740000000000001,"5min":13.314,"15min":15.054},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.7439999999999998,"5min":203.59299999999999,"15min":203.59299999999999},"last":2.669},{"interface":"front","average":{"1min":3.0459999999999998,"5min":12.625,"15min":14.221},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.8680000000000003,"5min":202.96600000000001,"15min":202.96600000000001},"last":1.105}]},{"osd":4,"last update":"Mon Oct 9 06:11:33 2023","interfaces":[{"interface":"back","average":{"1min":1.6539999999999999,"5min":2.258,"15min":2.3580000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":6.3559999999999999,"5min":10.19,"15min":10.19},"last":2.641},{"interface":"front","average":{"1min":2.673,"5min":2.129,"15min":2.0379999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":9.8770000000000007,"5min":9.8770000000000007,"15min":9.8770000000000007},"last":3.3999999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:11:33 2023","interfaces":[{"interface":"back","average":{"1min":9.9540000000000006,"5min":3.742,"15min":2.7069999999999999},"min":{"1min":0.32700000000000001,"5min":0.316,"15min":0.316},"max":{"1min":161.286,"5min":161.286,"15min":161.286},"last":3.4329999999999998},{"interface":"front","average":{"1min":9.6289999999999996,"5min":3.9049999999999998,"15min":2.9510000000000001},"min":{"1min":0.43099999999999999,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":161.31800000000001,"5min":161.31800000000001,"15min":161.31800000000001},"last":1.1799999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:11:33 2023","interfaces":[{"interface":"back","average":{"1min":2.5299999999999998,"5min":2.4870000000000001,"15min":2.48},"min":{"1min":0.438,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":8.4619999999999997,"5min":12.244,"15min":12.244},"last":1.1459999999999999},{"interface":"front","average":{"1min":1.5629999999999999,"5min":2.5830000000000002,"15min":2.7530000000000001},"min":{"1min":0.442,"5min":0.372,"15min":0.372},"max":{"1min":6.3310000000000004,"5min":10.233000000000001,"15min":10.233000000000001},"last":4.649}]},{"osd":7,"last update":"Mon Oct 9 06:11:33 2023","interfaces":[{"interface":"back","average":{"1min":8.9489999999999998,"5min":3.8159999999999998,"15min":2.96},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":123.685,"5min":123.685,"15min":123.685},"last":2.4729999999999999},{"interface":"front","average":{"1min":8.0670000000000002,"5min":3.823,"15min":3.1160000000000001},"min":{"1min":0.433,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":122.482,"5min":122.482,"15min":122.482},"last":2.9399999999999999}]}]},{"osd":2,"up_from":94,"seq":403726926107,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1671728,"kb_used_data":946856,"kb_used_omap":81,"kb_used_meta":724782,"kb_avail":92073424,"statfs":{"total":95995035648,"available":94283186176,"internally_reserved":0,"allocated":969580544,"data_stored":967027732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":742176986},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.9690000000000001,"5min":2.9129999999999998,"15min":3.0699999999999998},"min":{"1min":0.20200000000000001,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":16.263999999999999,"5min":16.263999999999999,"15min":16.263999999999999},"last":4.7939999999999996},{"interface":"front","average":{"1min":1.083,"5min":2.0110000000000001,"15min":2.165},"min":{"1min":0.216,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":4.4470000000000001,"5min":6.7939999999999996,"15min":6.7939999999999996},"last":3.294}]},{"osd":1,"last update":"Mon Oct 9 06:11:35 2023","interfaces":[{"interface":"back","average":{"1min":9.7479999999999993,"5min":5.7560000000000002,"15min":4.3579999999999997},"min":{"1min":0.25,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":160.45500000000001,"5min":160.45500000000001,"15min":222.59399999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":9.6310000000000002,"5min":6.5439999999999996,"15min":4.702},"min":{"1min":0.27200000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":163.697,"5min":163.697,"15min":224.619},"last":1.8899999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:11:56 2023","interfaces":[{"interface":"back","average":{"1min":1.752,"5min":4.5759999999999996,"15min":4.5640000000000001},"min":{"1min":0.28899999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":10.637,"5min":115.742,"15min":144.88200000000001},"last":3.911},{"interface":"front","average":{"1min":1.167,"5min":4.6079999999999997,"15min":4.5170000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":7.008,"5min":115.55,"15min":140.79499999999999},"last":0.60399999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:11:41 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":3.1549999999999998,"15min":2.2080000000000002},"min":{"1min":0.34699999999999998,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":4.9059999999999997,"5min":107.51300000000001,"15min":107.51300000000001},"last":1.8560000000000001},{"interface":"front","average":{"1min":1.107,"5min":2.5840000000000001,"15min":2.1320000000000001},"min":{"1min":0.34100000000000003,"5min":0.32000000000000001,"15min":0.31},"max":{"1min":4.3769999999999998,"5min":108.768,"15min":108.768},"last":0.68200000000000005}]},{"osd":5,"last update":"Mon Oct 9 06:11:35 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.6379999999999999,"15min":1.589},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.30599999999999999},"max":{"1min":3.2530000000000001,"5min":28.48,"15min":28.48},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.95599999999999996,"5min":1.8799999999999999,"15min":1.748},"min":{"1min":0.316,"5min":0.316,"15min":0.28899999999999998},"max":{"1min":2.3109999999999999,"5min":30.318999999999999,"15min":30.318999999999999},"last":0.629}]},{"osd":6,"last update":"Mon Oct 9 06:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":2.1230000000000002,"15min":2.0139999999999998},"min":{"1min":0.36399999999999999,"5min":0.32000000000000001,"15min":0.314},"max":{"1min":4.9349999999999996,"5min":14.962,"15min":30.420000000000002},"last":1.8160000000000001},{"interface":"front","average":{"1min":1.1379999999999999,"5min":1.474,"15min":1.79},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.32200000000000001},"max":{"1min":3.9260000000000002,"5min":7.6459999999999999,"15min":30.713000000000001},"last":0.752}]},{"osd":7,"last update":"Mon Oct 9 06:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":2.867,"15min":2.4220000000000002},"min":{"1min":0.41799999999999998,"5min":0.313,"15min":0.313},"max":{"1min":4.2530000000000001,"5min":124.069,"15min":124.069},"last":0.77700000000000002},{"interface":"front","average":{"1min":0.98899999999999999,"5min":2.7040000000000002,"15min":2.3130000000000002},"min":{"1min":0.36599999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":2.4129999999999998,"5min":124.08799999999999,"15min":124.08799999999999},"last":0.65300000000000002}]}]},{"osd":3,"up_from":99,"seq":425201762581,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1913384,"kb_used_data":1037856,"kb_used_omap":7,"kb_used_meta":875512,"kb_avail":91831768,"statfs":{"total":95995035648,"available":94035730432,"internally_reserved":0,"allocated":1062764544,"data_stored":1059916037,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":896524904},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,1,2,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":1.8169999999999999,"15min":2.0110000000000001},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":3.1819999999999999,"5min":5.5579999999999998,"15min":5.5579999999999998},"last":2.286},{"interface":"front","average":{"1min":3.109,"5min":3.4420000000000002,"15min":3.4980000000000002},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":26.795000000000002,"5min":26.795000000000002,"15min":26.795000000000002},"last":0.23200000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:12:04 2023","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":2.0169999999999999,"15min":2.7610000000000001},"min":{"1min":0.247,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":3.9900000000000002,"5min":22.579999999999998,"15min":207.75299999999999},"last":2.3490000000000002},{"interface":"front","average":{"1min":1.1719999999999999,"5min":3.3889999999999998,"15min":3.669},"min":{"1min":0.28100000000000003,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.8849999999999998,"5min":30.524000000000001,"15min":207.12200000000001},"last":1.24}]},{"osd":2,"last update":"Mon Oct 9 06:12:04 2023","interfaces":[{"interface":"back","average":{"1min":1.099,"5min":2.161,"15min":2.73},"min":{"1min":0.26900000000000002,"5min":0.23200000000000001,"15min":0.223},"max":{"1min":5.875,"5min":30.547999999999998,"15min":107.01300000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.2889999999999999,"5min":3.1230000000000002,"15min":3.1419999999999999},"min":{"1min":0.29799999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":6.008,"5min":25.984000000000002,"15min":109.807},"last":1.206}]},{"osd":4,"last update":"Mon Oct 9 06:11:43 2023","interfaces":[{"interface":"back","average":{"1min":2.6299999999999999,"5min":1.841,"15min":3.988},"min":{"1min":0.40600000000000003,"5min":0.312,"15min":0.254},"max":{"1min":26.864999999999998,"5min":26.864999999999998,"15min":99.876000000000005},"last":0.34899999999999998},{"interface":"front","average":{"1min":3.1509999999999998,"5min":2.3290000000000002,"15min":4.0880000000000001},"min":{"1min":0.39700000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":26.053999999999998,"5min":26.053999999999998,"15min":99.914000000000001},"last":1.2769999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:12:04 2023","interfaces":[{"interface":"back","average":{"1min":4.3499999999999996,"5min":2.7559999999999998,"15min":2.1259999999999999},"min":{"1min":0.40300000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":65.040999999999997,"5min":74.706000000000003,"15min":74.706000000000003},"last":2.4060000000000001},{"interface":"front","average":{"1min":4.5410000000000004,"5min":3.4449999999999998,"15min":3.3340000000000001},"min":{"1min":0.41199999999999998,"5min":0.28799999999999998,"15min":0.28199999999999997},"max":{"1min":63.851999999999997,"5min":74.685000000000002,"15min":235.56},"last":0.45100000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:12:04 2023","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":1.272,"15min":4.4349999999999996},"min":{"1min":0.46899999999999997,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.5960000000000001,"5min":8.6809999999999992,"15min":233.26599999999999},"last":2.3799999999999999},{"interface":"front","average":{"1min":1.359,"5min":2.0379999999999998,"15min":4.8680000000000003},"min":{"1min":0.435,"5min":0.33600000000000002,"15min":0.318},"max":{"1min":5.9420000000000002,"5min":26.890000000000001,"15min":233.24100000000001},"last":0.48099999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:12:04 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":3.4319999999999999,"15min":4.8470000000000004},"min":{"1min":0.46400000000000002,"5min":0.35699999999999998,"15min":0.34200000000000003},"max":{"1min":6.1280000000000001,"5min":81.811000000000007,"15min":230.69399999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":1.101,"5min":3.8530000000000002,"15min":4.9100000000000001},"min":{"1min":0.45500000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":5.9349999999999996,"5min":75.909999999999997,"15min":230.67500000000001},"last":1.3240000000000001}]}]},{"osd":4,"up_from":322,"seq":1382979469444,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2158820,"kb_used_data":1824156,"kb_used_omap":35,"kb_used_meta":334620,"kb_avail":91586332,"statfs":{"total":95995035648,"available":93784403968,"internally_reserved":0,"allocated":1867935744,"data_stored":1866503226,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36552,"internal_metadata":342651192},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:11:29 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":1.4670000000000001,"15min":1.4810000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":6.8470000000000004,"5min":6.8470000000000004,"15min":6.8470000000000004},"last":0.216},{"interface":"front","average":{"1min":1.6719999999999999,"5min":1.6120000000000001,"15min":1.6020000000000001},"min":{"1min":0.28899999999999998,"5min":0.247,"15min":0.247},"max":{"1min":7.2619999999999996,"5min":7.7999999999999998,"15min":7.7999999999999998},"last":0.85199999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:11:38 2023","interfaces":[{"interface":"back","average":{"1min":2.2690000000000001,"5min":1.4730000000000001,"15min":2.2469999999999999},"min":{"1min":0.32700000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.141,"5min":9.141,"15min":78.891000000000005},"last":0.432},{"interface":"front","average":{"1min":1.8140000000000001,"5min":1.766,"15min":2.226},"min":{"1min":0.28499999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":8.1859999999999999,"5min":10.257,"15min":78.921000000000006},"last":0.28799999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:11:38 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":1.8460000000000001,"15min":1.8149999999999999},"min":{"1min":0.29199999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.484,"5min":10.121,"15min":10.121},"last":0.32000000000000001},{"interface":"front","average":{"1min":2.2629999999999999,"5min":1.5429999999999999,"15min":1.8149999999999999},"min":{"1min":0.39200000000000002,"5min":0.29399999999999998,"15min":0.28199999999999997},"max":{"1min":7.5039999999999996,"5min":7.6669999999999998,"15min":11.381},"last":0.27900000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:11:38 2023","interfaces":[{"interface":"back","average":{"1min":2.4260000000000002,"5min":2.1629999999999998,"15min":2.4129999999999998},"min":{"1min":0.33300000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.2420000000000009,"5min":10.217000000000001,"15min":58.398000000000003},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.325,"5min":1.843,"15min":2.0310000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":5.0720000000000001,"5min":10.331,"15min":58.305999999999997},"last":0.69099999999999995}]},{"osd":5,"last update":"Mon Oct 9 06:11:38 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":1.123,"15min":1.0289999999999999},"min":{"1min":0.23499999999999999,"5min":0.216,"15min":0.216},"max":{"1min":7.3209999999999997,"5min":9.2690000000000001,"15min":9.2690000000000001},"last":0.24399999999999999},{"interface":"front","average":{"1min":1.5369999999999999,"5min":0.92100000000000004,"15min":1.2470000000000001},"min":{"1min":0.29299999999999998,"5min":0.245,"15min":0.245},"max":{"1min":9.1110000000000007,"5min":9.1110000000000007,"15min":12.087},"last":0.29899999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:11:38 2023","interfaces":[{"interface":"back","average":{"1min":1.575,"5min":2.1389999999999998,"15min":2.7879999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.3620000000000001,"5min":95.492999999999995,"15min":95.492999999999995},"last":0.38800000000000001},{"interface":"front","average":{"1min":2.0830000000000002,"5min":2.161,"15min":2.9249999999999998},"min":{"1min":0.32100000000000001,"5min":0.312,"15min":0.27900000000000003},"max":{"1min":9.3239999999999998,"5min":95.664000000000001,"15min":95.664000000000001},"last":0.40200000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:11:38 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":1.4059999999999999,"15min":3.4239999999999999},"min":{"1min":0.29699999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":9.2089999999999996,"5min":17.286000000000001,"15min":79.173000000000002},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.1859999999999999,"5min":1.323,"15min":2.8889999999999998},"min":{"1min":0.27800000000000002,"5min":0.251,"15min":0.251},"max":{"1min":5.2969999999999997,"5min":16.097999999999999,"15min":78.894000000000005},"last":0.49399999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757664,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1632460,"kb_used_data":738052,"kb_used_omap":0,"kb_used_meta":894400,"kb_avail":92112692,"statfs":{"total":95995035648,"available":94323396608,"internally_reserved":0,"allocated":755765248,"data_stored":754352910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915865600},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:11:28 2023","interfaces":[{"interface":"back","average":{"1min":1.524,"5min":2.226,"15min":2.343},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":8.077,"5min":23.238,"15min":23.238},"last":1.349},{"interface":"front","average":{"1min":1.357,"5min":2.8500000000000001,"15min":3.0990000000000002},"min":{"1min":0.30099999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.8330000000000002,"5min":23.225000000000001,"15min":23.225000000000001},"last":0.27600000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:11:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":7.8280000000000003,"15min":4.4039999999999999},"min":{"1min":0.28599999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":6.1950000000000003,"5min":151.78200000000001,"15min":151.78200000000001},"last":0.85299999999999998},{"interface":"front","average":{"1min":1.5720000000000001,"5min":8.0980000000000008,"15min":4.4249999999999998},"min":{"1min":0.34999999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":12.57,"5min":152.251,"15min":152.251},"last":2.903}]},{"osd":2,"last update":"Mon Oct 9 06:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":1.681,"15min":3.121},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.254},"max":{"1min":4.194,"5min":9.0800000000000001,"15min":217.06399999999999},"last":1.4950000000000001},{"interface":"front","average":{"1min":2.5790000000000002,"5min":2.032,"15min":3.181},"min":{"1min":0.33500000000000002,"5min":0.28599999999999998,"15min":0.28199999999999997},"max":{"1min":11.381,"5min":11.381,"15min":217.018},"last":1.4099999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:11:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":7.4370000000000003,"15min":7.1459999999999999},"min":{"1min":0.39500000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.923,"5min":143.31200000000001,"15min":211.40199999999999},"last":2.9849999999999999},{"interface":"front","average":{"1min":1.2609999999999999,"5min":7.7969999999999997,"15min":7.2240000000000002},"min":{"1min":0.35399999999999998,"5min":0.29599999999999999,"15min":0.29399999999999998},"max":{"1min":6.2679999999999998,"5min":140.00899999999999,"15min":211.578},"last":1.5249999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:11:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.268,"15min":1.077},"min":{"1min":0.29599999999999999,"5min":0.245,"15min":0.245},"max":{"1min":6.2759999999999998,"5min":7.5060000000000002,"15min":8.1539999999999999},"last":1.381},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.99299999999999999,"15min":1.0489999999999999},"min":{"1min":0.32000000000000001,"5min":0.26300000000000001,"15min":0.26000000000000001},"max":{"1min":2.8969999999999998,"5min":9.2370000000000001,"15min":9.2370000000000001},"last":0.79300000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:11:59 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":1.0720000000000001,"15min":1.548},"min":{"1min":0.36299999999999999,"5min":0.26200000000000001,"15min":0.251},"max":{"1min":2.1899999999999999,"5min":7.4980000000000002,"15min":126.89400000000001},"last":1.137},{"interface":"front","average":{"1min":0.57799999999999996,"5min":1.0549999999999999,"15min":1.546},"min":{"1min":0.316,"5min":0.27300000000000002,"15min":0.26700000000000002},"max":{"1min":1.4079999999999999,"5min":7.7140000000000004,"15min":126.616},"last":0.51800000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":2.6360000000000001,"15min":2.5880000000000001},"min":{"1min":0.34599999999999997,"5min":0.309,"15min":0.308},"max":{"1min":2.9849999999999999,"5min":157.471,"15min":157.471},"last":1.5469999999999999},{"interface":"front","average":{"1min":1.208,"5min":2.5489999999999999,"15min":2.472},"min":{"1min":0.36699999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.4260000000000002,"5min":157.51599999999999,"15min":157.51599999999999},"last":1.556}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4889,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1726808064,"data_stored":1726808064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":483196928,"data_stored":483196928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":995426304,"data_stored":995426304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1069285376,"data_stored":1069285376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1885143040,"data_stored":1885143040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":760020992,"data_stored":760020992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":289406976,"data_stored":289406976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":161939456,"data_stored":161939456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:12:20.346 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T06:12:20.347 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 71 2023-10-09T06:12:20.434 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:12:20.450 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:20 smithi064 ceph-mon[212750]: from='client.36285 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:12:20.450 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:20 smithi064 ceph-mon[212750]: pgmap v1277: 119 pgs: 119 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T06:12:20.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:20 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:20.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:20 smithi064 ceph-mon[220044]: from='client.36285 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:12:20.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:20 smithi064 ceph-mon[220044]: pgmap v1277: 119 pgs: 119 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T06:12:20.455 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:20 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:20.465 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:20 smithi156 ceph-mon[195101]: from='client.36285 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:12:20.466 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:20 smithi156 ceph-mon[195101]: pgmap v1277: 119 pgs: 119 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T06:12:20.466 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:20 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:20.471 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:20.522 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:12:20.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:20.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:12:20.591 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:20.633 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:12:20.667 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:20.720 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:12:20.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:20.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:12:20.825 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:12:20.856 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:20.904 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:12:20.907 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:20.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:20.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:12:21.009 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:12:21.043 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:21.107 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:12:21.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:21.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:12:21.208 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:12:21.213 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:12:21.240 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:21 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2755871954' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:12:21.241 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:21 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:12:21.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:21 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2755871954' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:12:21.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:21 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:12:21.299 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:21.317 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 27 16 44892 44876 103.847 65.5625 0.026295 0.00962417 2023-10-09T06:12:21.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:21 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2755871954' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:12:21.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:21 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:12:21.868 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:12:21.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:21.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:12:21.975 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:12:22.039 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:22.185 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:22 smithi064 ceph-mon[212750]: pgmap v1278: 119 pgs: 119 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T06:12:22.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:22 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:12:22.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:22 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:12:22.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:22 smithi064 ceph-mon[212750]: osdmap e505: 8 total, 8 up, 8 in 2023-10-09T06:12:22.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:22 smithi064 ceph-mon[212750]: failed to encode map e505 with expected crc 2023-10-09T06:12:22.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:22 smithi064 ceph-mon[212750]: failed to encode map e505 with expected crc 2023-10-09T06:12:22.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:22 smithi064 ceph-mon[212750]: failed to encode map e505 with expected crc 2023-10-09T06:12:22.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:22 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:22.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:22 smithi064 ceph-mon[212750]: failed to encode map e505 with expected crc 2023-10-09T06:12:22.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:22 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:22.317 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 28 16 46259 46243 103.189 85.4375 0.00833407 0.00968562 2023-10-09T06:12:22.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:22 smithi064 ceph-mon[220044]: pgmap v1278: 119 pgs: 119 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T06:12:22.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:22 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:12:22.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:22 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:12:22.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:22 smithi064 ceph-mon[220044]: osdmap e505: 8 total, 8 up, 8 in 2023-10-09T06:12:22.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:22 smithi064 ceph-mon[220044]: failed to encode map e505 with expected crc 2023-10-09T06:12:22.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:22 smithi064 ceph-mon[220044]: failed to encode map e505 with expected crc 2023-10-09T06:12:22.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:22 smithi064 ceph-mon[220044]: failed to encode map e505 with expected crc 2023-10-09T06:12:22.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:22 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:22.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:22 smithi064 ceph-mon[220044]: failed to encode map e505 with expected crc 2023-10-09T06:12:22.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:22 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:22.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:22 smithi156 ceph-mon[195101]: pgmap v1278: 119 pgs: 119 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T06:12:22.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:22 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:12:22.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:22 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:12:22.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:22 smithi156 ceph-mon[195101]: osdmap e505: 8 total, 8 up, 8 in 2023-10-09T06:12:22.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:22 smithi156 ceph-mon[195101]: failed to encode map e505 with expected crc 2023-10-09T06:12:22.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:22 smithi156 ceph-mon[195101]: failed to encode map e505 with expected crc 2023-10-09T06:12:22.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:22 smithi156 ceph-mon[195101]: failed to encode map e505 with expected crc 2023-10-09T06:12:22.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:22 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:22.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:22 smithi156 ceph-mon[195101]: failed to encode map e505 with expected crc 2023-10-09T06:12:22.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:22 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:22.641 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:23.132 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:12:23.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:23.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:12:23.239 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:12:23.278 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:23.320 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 29 16 47455 47439 102.203 74.75 0.0104415 0.00977869 2023-10-09T06:12:23.339 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:12:23.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:23.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:12:23.447 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:12:23.485 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:23.564 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:12:23.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:23.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:12:23.674 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:12:23.700 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:23.743 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:12:23.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:23.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:12:23.847 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:12:23.863 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:23 smithi064 ceph-mon[212750]: 1.7 scrub starts 2023-10-09T06:12:23.863 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:23 smithi064 ceph-mon[212750]: 1.7 scrub ok 2023-10-09T06:12:23.863 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:23 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:23.863 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:23 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:12:23.864 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:23 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:23.864 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:23 smithi064 ceph-mon[220044]: 1.7 scrub starts 2023-10-09T06:12:23.864 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:23 smithi064 ceph-mon[220044]: 1.7 scrub ok 2023-10-09T06:12:23.864 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:23 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:23.864 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:23 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:12:23.864 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:23 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:23.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:23 smithi156 ceph-mon[195101]: 1.7 scrub starts 2023-10-09T06:12:23.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:23 smithi156 ceph-mon[195101]: 1.7 scrub ok 2023-10-09T06:12:23.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:23 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:23.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:23 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:12:23.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:23 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:23.932 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:24.224 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:12:24.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:24.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:12:24.324 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 30 16 48569 48553 101.107 69.625 0.0139976 0.00988548 2023-10-09T06:12:24.346 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:12:24.385 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:24.465 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:12:24.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:24.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:12:24.574 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:12:24.605 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:24.671 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:12:24.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:24.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:12:24.686 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:24 smithi064 ceph-mon[212750]: pgmap v1280: 119 pgs: 119 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T06:12:24.686 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:24 smithi064 ceph-mon[220044]: pgmap v1280: 119 pgs: 119 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T06:12:24.778 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:12:24.866 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:25.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:24 smithi156 ceph-mon[195101]: pgmap v1280: 119 pgs: 119 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T06:12:25.322 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 31 16 49747 49731 100.221 73.625 0.00785201 0.00997199 2023-10-09T06:12:25.697 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:12:25.698 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:12:25.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:25.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:12:25.711 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:12:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:12:25] "GET /metrics HTTP/1.1" 200 36516 "" "Prometheus/2.43.0" 2023-10-09T06:12:25.739 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:12:25.779 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:12:25.812 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:12:25.820 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:12:25.866 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:12:25.903 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:12:25.905 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:12:25.911 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:25.946 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:12:25.986 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:12:25.997 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:12:25.997 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:12:25.997 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:12:25.997 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:12:25.997 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:12:25.997 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:12:25.997 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:12:25.997 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:12:26.324 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 32 16 50859 50843 99.2547 69.5 0.0093686 0.010069 2023-10-09T06:12:26.368 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:12:26.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:26.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:12:26.472 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:12:26.551 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:26.832 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:12:26.833 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:12:26.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:26 smithi064 ceph-mon[212750]: pgmap v1281: 119 pgs: 119 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T06:12:26.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:26 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4111044239' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:12:26.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:26 smithi064 ceph-mon[220044]: pgmap v1281: 119 pgs: 119 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T06:12:26.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:26 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4111044239' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:12:27.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:26 smithi156 ceph-mon[195101]: pgmap v1281: 119 pgs: 119 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T06:12:27.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:26 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4111044239' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:12:27.240 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:12:27.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:27.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:12:27.326 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 33 16 51795 51779 98.0151 58.5 0.0144595 0.0101968 2023-10-09T06:12:27.351 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:12:27.386 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:27.461 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:12:27.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:27.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:12:27.570 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:12:27.613 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:28.303 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:12:28.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:28.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:12:28.326 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 34 16 52933 52917 97.2244 71.125 0.0164374 0.0102807 2023-10-09T06:12:28.397 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:12:28.409 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:12:28.445 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:28.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:28.633 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:12:28.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:12:28.734 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:12:28.751 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:28.762 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:28 smithi156 ceph-mon[195101]: pgmap v1282: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-10-09T06:12:28.763 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:28 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:28.771 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:28.817 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:12:28.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:28.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:12:28.924 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:12:28.960 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:28.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:28 smithi064 ceph-mon[212750]: pgmap v1282: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-10-09T06:12:28.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:28 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:28.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:28 smithi064 ceph-mon[220044]: pgmap v1282: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-10-09T06:12:28.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:28 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:29.025 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:12:29.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:29.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:12:29.132 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:12:29.163 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:29.179 INFO:teuthology.orchestra.run.smithi064.stderr:Error EINVAL: specified pg_num 71 < pg_num_max 32 2023-10-09T06:12:29.223 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:12:29.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:29.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:12:29.329 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 35 16 54064 54048 96.4573 70.6875 0.0163751 0.0103624 2023-10-09T06:12:29.330 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:12:29.409 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:29.674 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:29 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3758927179' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-10-09T06:12:29.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:29 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3758927179' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-10-09T06:12:29.971 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:12:30.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:29 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3758927179' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-10-09T06:12:30.329 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 36 16 55193 55177 95.7382 70.5625 0.00946047 0.0104401 2023-10-09T06:12:30.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:30 smithi064 ceph-mon[220044]: pgmap v1283: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-10-09T06:12:30.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:30 smithi064 ceph-mon[212750]: pgmap v1283: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-10-09T06:12:30.986 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:12:30.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:30.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:12:31.057 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T06:12:31.093 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:12:31.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:30 smithi156 ceph-mon[195101]: pgmap v1283: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-10-09T06:12:31.128 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:31.196 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:12:31.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:31.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:12:31.304 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:12:31.321 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:31.329 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 37 16 56235 56219 94.911 65.125 0.0188081 0.0105307 2023-10-09T06:12:31.339 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:31.393 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:12:31.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:31.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:12:31.499 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:12:31.540 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:31.547 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:31.602 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:12:31.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:31.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:12:31.709 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:12:31.781 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:32.021 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:12:32.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:32.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:12:32.131 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:12:32.164 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:32.224 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:12:32.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:32.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:12:32.330 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 38 16 57388 57372 94.3098 72.0625 0.0110397 0.010598 2023-10-09T06:12:32.333 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:12:32.426 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:32.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:32 smithi064 ceph-mon[212750]: pgmap v1284: 119 pgs: 119 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:12:32.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:32 smithi064 ceph-mon[220044]: pgmap v1284: 119 pgs: 119 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:12:33.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:32 smithi156 ceph-mon[195101]: pgmap v1284: 119 pgs: 119 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:12:33.329 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 39 16 58350 58334 93.4336 60.125 0.0204565 0.0106978 2023-10-09T06:12:33.345 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:12:33.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:33.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:12:33.454 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:12:33.490 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:33.564 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:12:33.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:33.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:12:33.672 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:12:33.709 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:33.713 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:12:33.759 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:12:33.765 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:33 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/899497117' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:12:33.765 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:33 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:12:33.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:33.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:12:33.867 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:12:33.894 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:33 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/899497117' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:12:33.894 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:33 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:12:33.894 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:33 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/899497117' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:12:33.894 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:33 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:12:33.897 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:34.330 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-10-09T06:12:34.330533+0000 min lat: 0.0010971 max lat: 0.349458 avg lat: 0.0107955 2023-10-09T06:12:34.330 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:12:34.330 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 40 15 59303 59288 92.5886 59.625 0.013555 0.0107955 2023-10-09T06:12:34.361 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:12:34.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:34.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:12:34.471 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:12:34.501 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:34.569 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:12:34.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:34.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:12:34.676 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:12:34.704 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:34.746 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:12:34.750 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:34 smithi156 ceph-mon[195101]: pgmap v1285: 119 pgs: 119 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T06:12:34.750 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:34 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:12:34.750 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:34 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T06:12:34.750 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:34 smithi156 ceph-mon[195101]: failed to encode map e506 with expected crc 2023-10-09T06:12:34.750 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:34 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:12:34.750 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:34 smithi156 ceph-mon[195101]: osdmap e506: 8 total, 8 up, 8 in 2023-10-09T06:12:34.750 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:34 smithi156 ceph-mon[195101]: failed to encode map e506 with expected crc 2023-10-09T06:12:34.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:34.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:12:34.852 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:12:34.937 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:34.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:34 smithi064 ceph-mon[212750]: pgmap v1285: 119 pgs: 119 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T06:12:34.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:34 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:12:34.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:34 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:12:34.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:34 smithi064 ceph-mon[212750]: failed to encode map e506 with expected crc 2023-10-09T06:12:34.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:34 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:12:34.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:34 smithi064 ceph-mon[212750]: osdmap e506: 8 total, 8 up, 8 in 2023-10-09T06:12:34.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:34 smithi064 ceph-mon[212750]: failed to encode map e506 with expected crc 2023-10-09T06:12:34.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:34 smithi064 ceph-mon[220044]: pgmap v1285: 119 pgs: 119 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T06:12:34.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:34 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:12:34.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:34 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T06:12:34.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:34 smithi064 ceph-mon[220044]: failed to encode map e506 with expected crc 2023-10-09T06:12:34.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:34 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:12:34.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:34 smithi064 ceph-mon[220044]: osdmap e506: 8 total, 8 up, 8 in 2023-10-09T06:12:34.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:34 smithi064 ceph-mon[220044]: failed to encode map e506 with expected crc 2023-10-09T06:12:35.247 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:12:35.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:35.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:12:35.331 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 41 16 60309 60293 91.8603 62.8125 0.0184384 0.010881 2023-10-09T06:12:35.360 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:12:35.400 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:35.458 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:12:35.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:35.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:12:35.468 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:12:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:12:35] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.43.0" 2023-10-09T06:12:35.563 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:12:35.593 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:35.651 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:12:35.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:35.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:12:35.759 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:12:35.772 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:12:35.772 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:12:35.790 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:35.796 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:12:35.838 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:12:35.854 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:12:35.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:35.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:12:35.898 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:12:35.935 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:12:35.961 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:12:35.975 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:12:36.019 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:12:36.040 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:36.059 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1, 2] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 4, 0] 2023-10-09T06:12:36.059 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T06:12:36.062 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:12:36.100 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:12:36.106 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:12:36.106 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:12:36.106 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:12:36.106 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:12:36.106 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:12:36.106 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:12:36.106 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:12:36.106 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:12:36.168 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:12:36.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:36.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:12:36.276 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:12:36.332 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 42 16 61417 61401 91.32 69.25 0.0173699 0.0109454 2023-10-09T06:12:36.366 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:36.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:36 smithi064 ceph-mon[212750]: pgmap v1287: 119 pgs: 119 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T06:12:36.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:36 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2686526514' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:12:36.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:36 smithi064 ceph-mon[220044]: pgmap v1287: 119 pgs: 119 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T06:12:36.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:36 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2686526514' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:12:37.099 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:12:37.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:37.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:12:37.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:36 smithi156 ceph-mon[195101]: pgmap v1287: 119 pgs: 119 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T06:12:37.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:36 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2686526514' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:12:37.204 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:12:37.261 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:12:37.266 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:37.332 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 43 16 62546 62530 90.8374 70.5625 0.00523123 0.0110026 2023-10-09T06:12:37.430 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:12:37.449 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:12:37.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:37.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:12:37.556 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:12:37.684 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:37.774 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:37.852 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:12:37.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:37.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:12:37.967 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:12:38.039 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:38.211 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:12:38.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:38.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:12:38.315 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:12:38.332 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 44 16 63469 63453 90.0842 57.6875 0.0149404 0.0110956 2023-10-09T06:12:38.351 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:38.393 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:38.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:38 smithi064 ceph-mon[212750]: pgmap v1288: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T06:12:38.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:38 smithi064 ceph-mon[220044]: pgmap v1288: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T06:12:39.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:38 smithi156 ceph-mon[195101]: pgmap v1288: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T06:12:39.293 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:12:39.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:39.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:12:39.332 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 45 16 64419 64403 89.402 59.375 0.0171078 0.0111803 2023-10-09T06:12:39.400 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:12:39.495 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:39.497 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:12:39.563 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:39.647 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:12:39.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:39.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:12:39.778 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:12:39.857 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:40.203 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:12:40.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:40.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:12:40.329 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:12:40.333 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 46 16 65426 65410 88.8268 62.9375 0.00938715 0.0112526 2023-10-09T06:12:40.410 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:40.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:40 smithi064 ceph-mon[220044]: pgmap v1289: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T06:12:40.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:40 smithi064 ceph-mon[212750]: pgmap v1289: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T06:12:41.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:40 smithi156 ceph-mon[195101]: pgmap v1289: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T06:12:41.220 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:12:41.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:41.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:12:41.331 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:12:41.332 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 47 16 66599 66583 88.4967 73.3125 0.027997 0.0112931 2023-10-09T06:12:41.366 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:41.427 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:12:41.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:41.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:12:41.533 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:12:41.567 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:41.631 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:12:41.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:41.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:12:41.738 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:12:41.819 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:42.332 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 48 16 67641 67625 88.0099 65.125 0.0192402 0.0113573 2023-10-09T06:12:42.746 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:12:42.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:42.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:12:42.783 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T06:12:42.857 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:12:42.889 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:42.957 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:12:42.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:42.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:12:43.058 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:42 smithi064 ceph-mon[212750]: pgmap v1290: 119 pgs: 119 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T06:12:43.059 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:42 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/842251744' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:12:43.059 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:42 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:12:43.059 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:42 smithi064 ceph-mon[220044]: pgmap v1290: 119 pgs: 119 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T06:12:43.059 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:42 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/842251744' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:12:43.059 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:42 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:12:43.063 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:12:43.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:42 smithi156 ceph-mon[195101]: pgmap v1290: 119 pgs: 119 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T06:12:43.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:42 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/842251744' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:12:43.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:42 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:12:43.139 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:43.332 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 49 16 68730 68714 87.6028 68.0625 0.0119598 0.0114106 2023-10-09T06:12:43.348 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:12:43.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:43.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:12:43.455 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:12:43.481 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:43.540 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:12:43.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:43.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:12:43.647 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:12:43.681 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:43.760 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:12:43.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:43.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:12:43.869 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:12:43.889 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:43.933 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T06:12:43.968 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:44.053 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:43 smithi064 ceph-mon[212750]: failed to encode map e507 with expected crc 2023-10-09T06:12:44.053 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:43 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:12:44.053 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:43 smithi064 ceph-mon[212750]: osdmap e507: 8 total, 8 up, 8 in 2023-10-09T06:12:44.054 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:43 smithi064 ceph-mon[212750]: failed to encode map e507 with expected crc 2023-10-09T06:12:44.054 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:43 smithi064 ceph-mon[212750]: failed to encode map e507 with expected crc 2023-10-09T06:12:44.054 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:43 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:44.054 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:43 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:44.054 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:43 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1340700169' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:12:44.054 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:43 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:12:44.055 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:43 smithi064 ceph-mon[220044]: failed to encode map e507 with expected crc 2023-10-09T06:12:44.055 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:43 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:12:44.055 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:43 smithi064 ceph-mon[220044]: osdmap e507: 8 total, 8 up, 8 in 2023-10-09T06:12:44.055 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:43 smithi064 ceph-mon[220044]: failed to encode map e507 with expected crc 2023-10-09T06:12:44.055 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:43 smithi064 ceph-mon[220044]: failed to encode map e507 with expected crc 2023-10-09T06:12:44.055 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:43 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:44.055 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:43 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:44.055 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:43 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1340700169' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:12:44.055 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:43 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:12:44.064 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:12:44.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:44.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:12:44.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:43 smithi156 ceph-mon[195101]: failed to encode map e507 with expected crc 2023-10-09T06:12:44.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:43 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:12:44.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:43 smithi156 ceph-mon[195101]: osdmap e507: 8 total, 8 up, 8 in 2023-10-09T06:12:44.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:43 smithi156 ceph-mon[195101]: failed to encode map e507 with expected crc 2023-10-09T06:12:44.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:43 smithi156 ceph-mon[195101]: failed to encode map e507 with expected crc 2023-10-09T06:12:44.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:43 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:44.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:43 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:44.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:43 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1340700169' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:12:44.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:43 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:12:44.170 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:12:44.209 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:44.267 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:12:44.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:44.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:12:44.333 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 50 16 69597 69581 86.9347 54.1875 0.0178059 0.0114965 2023-10-09T06:12:44.376 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:12:44.465 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:12:44.465 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:12:44.465 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.465 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:69690 10.7 10:e460ab92:::benchmark_data_smithi064_573750_object69689:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e507)", 2023-10-09T06:12:44.465 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:12:44.438884+0000", 2023-10-09T06:12:44.465 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.024554106999999999, 2023-10-09T06:12:44.465 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.024588326000000001, 2023-10-09T06:12:44.466 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:12:44.466 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:12:44.466 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:12:44.466 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:12:44.466 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:12:44.466 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 69690 2023-10-09T06:12:44.466 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.466 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:12:44.466 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:12:44.466 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.438884+0000", 2023-10-09T06:12:44.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:12:44.467 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.467 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.467 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:12:44.467 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.438884+0000", 2023-10-09T06:12:44.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:12:44.467 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.467 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.467 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:12:44.467 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.438882+0000", 2023-10-09T06:12:44.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:12:44.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:12:44.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.438908+0000", 2023-10-09T06:12:44.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5956999999999999e-05 2023-10-09T06:12:44.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:12:44.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.438972+0000", 2023-10-09T06:12:44.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4082000000000001e-05 2023-10-09T06:12:44.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:12:44.469 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.438983+0000", 2023-10-09T06:12:44.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0423e-05 2023-10-09T06:12:44.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:12:44.469 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.439167+0000", 2023-10-09T06:12:44.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000184081 2023-10-09T06:12:44.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.470 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:12:44.470 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.439352+0000", 2023-10-09T06:12:44.470 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000185642 2023-10-09T06:12:44.470 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.470 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:12:44.470 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.439520+0000", 2023-10-09T06:12:44.470 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016775700000000001 2023-10-09T06:12:44.470 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.470 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:12:44.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.446582+0000", 2023-10-09T06:12:44.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.007061721 2023-10-09T06:12:44.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:12:44.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.448867+0000", 2023-10-09T06:12:44.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002285245 2023-10-09T06:12:44.471 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:12:44.471 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:12:44.471 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:12:44.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.472 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:69686 10.d e507/490 10:be81439a:::benchmark_data_smithi064_573750_object69685:head v 507'4341, mlcod=507'4339)", 2023-10-09T06:12:44.472 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:12:44.442290+0000", 2023-10-09T06:12:44.472 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.021147902, 2023-10-09T06:12:44.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021240451, 2023-10-09T06:12:44.472 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:12:44.472 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:12:44.472 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:12:44.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:12:44.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442290+0000", 2023-10-09T06:12:44.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:12:44.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.473 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:12:44.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442290+0000", 2023-10-09T06:12:44.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:12:44.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.473 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:12:44.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442289+0000", 2023-10-09T06:12:44.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:12:44.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:12:44.474 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442332+0000", 2023-10-09T06:12:44.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2724e-05 2023-10-09T06:12:44.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:12:44.474 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442344+0000", 2023-10-09T06:12:44.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2306e-05 2023-10-09T06:12:44.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:12:44.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442352+0000", 2023-10-09T06:12:44.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0239999999999997e-06 2023-10-09T06:12:44.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:12:44.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442704+0000", 2023-10-09T06:12:44.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00035142399999999997 2023-10-09T06:12:44.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:12:44.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442726+0000", 2023-10-09T06:12:44.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2076999999999999e-05 2023-10-09T06:12:44.476 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:12:44.476 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:12:44.476 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:12:44.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.476 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:69687 10.4 e507/490 10:2b86245c:::benchmark_data_smithi064_573750_object69686:head v 507'4289, mlcod=507'4287)", 2023-10-09T06:12:44.476 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:12:44.442370+0000", 2023-10-09T06:12:44.476 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.02106775, 2023-10-09T06:12:44.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021209645999999999, 2023-10-09T06:12:44.477 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:12:44.477 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:12:44.477 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:12:44.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:12:44.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442370+0000", 2023-10-09T06:12:44.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:12:44.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:12:44.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442370+0000", 2023-10-09T06:12:44.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:12:44.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:12:44.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442370+0000", 2023-10-09T06:12:44.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:12:44.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:12:44.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442407+0000", 2023-10-09T06:12:44.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6863000000000003e-05 2023-10-09T06:12:44.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:12:44.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442414+0000", 2023-10-09T06:12:44.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1579999999999999e-06 2023-10-09T06:12:44.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:12:44.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442419+0000", 2023-10-09T06:12:44.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5040000000000002e-06 2023-10-09T06:12:44.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:12:44.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.455236+0000", 2023-10-09T06:12:44.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012816398 2023-10-09T06:12:44.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:12:44.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.455264+0000", 2023-10-09T06:12:44.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8314e-05 2023-10-09T06:12:44.480 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:12:44.481 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:12:44.481 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:12:44.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.481 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:69693 10.9 e507/490 10:9c1d1b90:::benchmark_data_smithi064_573750_object69692:head v 507'4324, mlcod=507'4322)", 2023-10-09T06:12:44.481 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:12:44.442430+0000", 2023-10-09T06:12:44.481 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.021008158999999998, 2023-10-09T06:12:44.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021226368999999998, 2023-10-09T06:12:44.481 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:12:44.481 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:12:44.481 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:12:44.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.482 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:12:44.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442430+0000", 2023-10-09T06:12:44.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:12:44.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.482 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:12:44.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442430+0000", 2023-10-09T06:12:44.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:12:44.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:12:44.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442430+0000", 2023-10-09T06:12:44.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:12:44.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:12:44.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442461+0000", 2023-10-09T06:12:44.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1594000000000003e-05 2023-10-09T06:12:44.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:12:44.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442468+0000", 2023-10-09T06:12:44.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5200000000000003e-06 2023-10-09T06:12:44.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:12:44.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442473+0000", 2023-10-09T06:12:44.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1549999999999998e-06 2023-10-09T06:12:44.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:12:44.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442610+0000", 2023-10-09T06:12:44.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013687700000000001 2023-10-09T06:12:44.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:12:44.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442651+0000", 2023-10-09T06:12:44.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1693000000000002e-05 2023-10-09T06:12:44.485 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:12:44.485 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:12:44.485 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:12:44.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.486 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:69688 10.4 e507/490 10:2efa7590:::benchmark_data_smithi064_573750_object69687:head v 507'4290, mlcod=507'4287)", 2023-10-09T06:12:44.486 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:12:44.442487+0000", 2023-10-09T06:12:44.486 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.020950685, 2023-10-09T06:12:44.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021220180000000002, 2023-10-09T06:12:44.486 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:12:44.486 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:12:44.486 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:12:44.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:12:44.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442487+0000", 2023-10-09T06:12:44.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:12:44.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:12:44.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442487+0000", 2023-10-09T06:12:44.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:12:44.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:12:44.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442487+0000", 2023-10-09T06:12:44.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:12:44.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:12:44.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442518+0000", 2023-10-09T06:12:44.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1016000000000003e-05 2023-10-09T06:12:44.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:12:44.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442524+0000", 2023-10-09T06:12:44.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2140000000000001e-06 2023-10-09T06:12:44.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:12:44.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.442529+0000", 2023-10-09T06:12:44.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3479999999999997e-06 2023-10-09T06:12:44.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:12:44.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.457995+0000", 2023-10-09T06:12:44.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015466357 2023-10-09T06:12:44.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:12:44.490 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.458026+0000", 2023-10-09T06:12:44.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1228e-05 2023-10-09T06:12:44.490 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:12:44.490 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:12:44.490 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:12:44.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.490 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:69696 10.4 e507/490 10:20b34414:::benchmark_data_smithi064_573750_object69695:head v 507'4291, mlcod=507'4287)", 2023-10-09T06:12:44.490 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:12:44.445362+0000", 2023-10-09T06:12:44.491 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.018076360999999999, 2023-10-09T06:12:44.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018406601000000002, 2023-10-09T06:12:44.491 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:12:44.491 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:12:44.491 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:12:44.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:12:44.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.445362+0000", 2023-10-09T06:12:44.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:12:44.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:12:44.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.445362+0000", 2023-10-09T06:12:44.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:12:44.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:12:44.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.445361+0000", 2023-10-09T06:12:44.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:12:44.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:12:44.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.445397+0000", 2023-10-09T06:12:44.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6675999999999999e-05 2023-10-09T06:12:44.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:12:44.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.445410+0000", 2023-10-09T06:12:44.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2091e-05 2023-10-09T06:12:44.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:12:44.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.445417+0000", 2023-10-09T06:12:44.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3320000000000003e-06 2023-10-09T06:12:44.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:12:44.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.460910+0000", 2023-10-09T06:12:44.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015492858999999999 2023-10-09T06:12:44.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:12:44.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.460935+0000", 2023-10-09T06:12:44.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5673999999999998e-05 2023-10-09T06:12:44.495 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:12:44.495 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:12:44.495 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:12:44.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.495 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:69698 10.4 e507/490 10:2ee6d5bd:::benchmark_data_smithi064_573750_object69697:head v 507'4292, mlcod=507'4287)", 2023-10-09T06:12:44.495 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:12:44.452897+0000", 2023-10-09T06:12:44.495 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.01054097, 2023-10-09T06:12:44.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010937898999999999, 2023-10-09T06:12:44.495 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:12:44.495 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:12:44.496 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:12:44.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:12:44.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.452897+0000", 2023-10-09T06:12:44.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:12:44.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:12:44.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.452897+0000", 2023-10-09T06:12:44.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:12:44.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:12:44.497 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.452895+0000", 2023-10-09T06:12:44.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:12:44.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:12:44.497 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.452967+0000", 2023-10-09T06:12:44.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1614999999999998e-05 2023-10-09T06:12:44.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:12:44.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.452991+0000", 2023-10-09T06:12:44.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4155000000000002e-05 2023-10-09T06:12:44.498 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.498 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:12:44.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.453002+0000", 2023-10-09T06:12:44.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0716e-05 2023-10-09T06:12:44.498 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.498 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:12:44.499 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.463559+0000", 2023-10-09T06:12:44.499 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010557306000000001 2023-10-09T06:12:44.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.499 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:12:44.499 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.463581+0000", 2023-10-09T06:12:44.499 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1957999999999999e-05 2023-10-09T06:12:44.499 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:12:44.499 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:12:44.499 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:12:44.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.500 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.500 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:69701 10.9 e507/490 10:92d7ea18:::benchmark_data_smithi064_573750_object69700:head v 507'4325, mlcod=507'4322)", 2023-10-09T06:12:44.500 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:12:44.453012+0000", 2023-10-09T06:12:44.500 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.010426358, 2023-10-09T06:12:44.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010885173999999999, 2023-10-09T06:12:44.500 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:12:44.500 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:12:44.500 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:12:44.500 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.500 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:12:44.500 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.453012+0000", 2023-10-09T06:12:44.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:12:44.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:12:44.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.453012+0000", 2023-10-09T06:12:44.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:12:44.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:12:44.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.453011+0000", 2023-10-09T06:12:44.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:12:44.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:12:44.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.453063+0000", 2023-10-09T06:12:44.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1545999999999997e-05 2023-10-09T06:12:44.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:12:44.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.453075+0000", 2023-10-09T06:12:44.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2520999999999999e-05 2023-10-09T06:12:44.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:12:44.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.453081+0000", 2023-10-09T06:12:44.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8950000000000003e-06 2023-10-09T06:12:44.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:12:44.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.453130+0000", 2023-10-09T06:12:44.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9119000000000002e-05 2023-10-09T06:12:44.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:12:44.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.453152+0000", 2023-10-09T06:12:44.504 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.173e-05 2023-10-09T06:12:44.504 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:12:44.504 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:12:44.504 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:12:44.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.504 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:69704 10.4 e507/490)", 2023-10-09T06:12:44.504 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:12:44.458102+0000", 2023-10-09T06:12:44.504 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0053355310000000001, 2023-10-09T06:12:44.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058481690000000003, 2023-10-09T06:12:44.505 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:12:44.505 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-10-09T06:12:44.505 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:12:44.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:12:44.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.458102+0000", 2023-10-09T06:12:44.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:12:44.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:12:44.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.458102+0000", 2023-10-09T06:12:44.506 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:12:44.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:12:44.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.458101+0000", 2023-10-09T06:12:44.506 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:12:44.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:12:44.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.458155+0000", 2023-10-09T06:12:44.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4052999999999999e-05 2023-10-09T06:12:44.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.507 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.507 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:12:44.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.458171+0000", 2023-10-09T06:12:44.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5387999999999999e-05 2023-10-09T06:12:44.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.507 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.507 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:12:44.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.458217+0000", 2023-10-09T06:12:44.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6357999999999999e-05 2023-10-09T06:12:44.508 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:12:44.508 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:12:44.508 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:12:44.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.508 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:69705 10.c e507/490)", 2023-10-09T06:12:44.508 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:12:44.458233+0000", 2023-10-09T06:12:44.508 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0052052879999999998, 2023-10-09T06:12:44.508 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.005761314, 2023-10-09T06:12:44.508 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:12:44.508 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-10-09T06:12:44.508 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:12:44.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:12:44.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.458233+0000", 2023-10-09T06:12:44.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:12:44.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:12:44.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.458233+0000", 2023-10-09T06:12:44.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:12:44.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:12:44.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.458232+0000", 2023-10-09T06:12:44.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:12:44.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:12:44.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.458267+0000", 2023-10-09T06:12:44.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4789999999999997e-05 2023-10-09T06:12:44.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:12:44.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.458275+0000", 2023-10-09T06:12:44.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8320000000000006e-06 2023-10-09T06:12:44.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:12:44.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:12:44.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:12:44.511 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:44.458283+0000", 2023-10-09T06:12:44.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3590000000000008e-06 2023-10-09T06:12:44.511 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:12:44.511 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:12:44.511 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:12:44.511 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:12:44.511 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-10-09T06:12:44.511 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 10 2023-10-09T06:12:44.511 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:12:44.512 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:44.556 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:12:44.981 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:44 smithi064 ceph-mon[212750]: pgmap v1292: 119 pgs: 119 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T06:12:44.981 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:44 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:12:44.981 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:44 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:12:44.982 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:44 smithi064 ceph-mon[212750]: osdmap e508: 8 total, 8 up, 8 in 2023-10-09T06:12:44.982 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:44 smithi064 ceph-mon[220044]: pgmap v1292: 119 pgs: 119 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T06:12:44.982 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:44 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:12:44.982 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:44 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:12:44.982 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:44 smithi064 ceph-mon[220044]: osdmap e508: 8 total, 8 up, 8 in 2023-10-09T06:12:45.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:44 smithi156 ceph-mon[195101]: pgmap v1292: 119 pgs: 119 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T06:12:45.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:44 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:12:45.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:44 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:12:45.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:44 smithi156 ceph-mon[195101]: osdmap e508: 8 total, 8 up, 8 in 2023-10-09T06:12:45.305 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:12:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:12:45] "GET /metrics HTTP/1.1" 200 36506 "" "Prometheus/2.43.0" 2023-10-09T06:12:45.335 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 51 16 70642 70626 86.507 65.3125 0.010431 0.0115551 2023-10-09T06:12:45.606 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:12:45.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:45.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:12:45.721 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:12:45.755 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:45.820 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:12:45.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:45.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:12:45.926 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:12:45.962 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:45.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:45 smithi064 ceph-mon[220044]: failed to encode map e508 with expected crc 2023-10-09T06:12:45.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:45 smithi064 ceph-mon[220044]: failed to encode map e508 with expected crc 2023-10-09T06:12:45.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:45 smithi064 ceph-mon[220044]: pgmap v1294: 119 pgs: 119 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 40 MiB/s wr, 645 op/s 2023-10-09T06:12:45.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:45 smithi064 ceph-mon[212750]: failed to encode map e508 with expected crc 2023-10-09T06:12:45.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:45 smithi064 ceph-mon[212750]: failed to encode map e508 with expected crc 2023-10-09T06:12:45.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:45 smithi064 ceph-mon[212750]: pgmap v1294: 119 pgs: 119 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 40 MiB/s wr, 645 op/s 2023-10-09T06:12:46.031 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:12:46.035 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:45 smithi156 ceph-mon[195101]: failed to encode map e508 with expected crc 2023-10-09T06:12:46.035 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:45 smithi156 ceph-mon[195101]: failed to encode map e508 with expected crc 2023-10-09T06:12:46.036 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:45 smithi156 ceph-mon[195101]: pgmap v1294: 119 pgs: 119 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 40 MiB/s wr, 645 op/s 2023-10-09T06:12:46.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:46.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:12:46.138 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:12:46.174 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:46.240 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:12:46.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:46.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:12:46.335 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 52 16 71529 71513 85.9096 55.4375 0.00260142 0.0116325 2023-10-09T06:12:46.347 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:12:46.376 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:46.862 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:12:46.992 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:12:47.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:47.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:12:47.107 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:12:47.138 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:47.140 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:47.194 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:12:47.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:47.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:12:47.301 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:12:47.335 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 53 16 72369 72353 85.2784 52.5 0.0113531 0.0117196 2023-10-09T06:12:47.336 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:47.401 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:12:47.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:47.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:12:47.509 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:12:47.544 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:47.618 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:12:47.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:47.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:12:47.725 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:12:47.763 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:47.822 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:12:47.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:47.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:12:47.921 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:12:47.930 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:12:47.968 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:12:48.018 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:48.018 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:12:48.058 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:12:48.096 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:12:48.137 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:12:48.171 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:12:48.204 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:12:48.219 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:12:48.219 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:12:48.219 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:12:48.219 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:12:48.220 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:12:48.220 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:12:48.220 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:12:48.220 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:12:48.280 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:12:48.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:48.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:12:48.337 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 54 16 73316 73300 84.7936 59.1875 0.0326617 0.0117886 2023-10-09T06:12:48.385 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:12:48.427 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:48.489 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:12:48.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:48.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:12:48.596 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:12:48.623 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:48.664 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:12:48.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:48.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:12:48.769 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:12:48.805 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:48 smithi156 ceph-mon[195101]: pgmap v1295: 119 pgs: 119 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T06:12:48.805 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:48 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3245785683' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:12:48.810 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:48.876 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:12:48.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:48.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:12:48.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:48 smithi064 ceph-mon[220044]: pgmap v1295: 119 pgs: 119 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T06:12:49.080 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:48 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3245785683' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:12:49.080 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:12:49.081 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:12:49.082 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:48 smithi064 ceph-mon[212750]: pgmap v1295: 119 pgs: 119 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T06:12:49.082 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:48 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3245785683' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:12:49.115 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:49.184 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:12:49.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:49.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:12:49.295 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:12:49.334 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:49.339 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 55 16 74252 74236 84.3129 58.5 0.00883702 0.0118555 2023-10-09T06:12:49.402 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:12:49.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:49.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:12:49.510 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:12:49.546 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:49.590 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:49.602 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:12:49.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:49.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:12:49.709 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:12:49.740 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:49.807 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:12:49.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:49.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:12:49.916 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:12:49.996 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:50.273 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:12:50.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:50.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:12:50.338 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 56 16 75215 75199 83.8822 60.1875 0.00454749 0.0119161 2023-10-09T06:12:50.386 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:12:50.421 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:50.480 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:12:50.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:50.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:12:50.592 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:12:50.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:50 smithi064 ceph-mon[212750]: pgmap v1296: 119 pgs: 119 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s 2023-10-09T06:12:50.704 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:50.807 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:12:50.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:50.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:12:50.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:50 smithi156 ceph-mon[195101]: pgmap v1296: 119 pgs: 119 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s 2023-10-09T06:12:50.919 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:12:50.949 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:50.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:50 smithi064 ceph-mon[220044]: pgmap v1296: 119 pgs: 119 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s 2023-10-09T06:12:51.020 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:12:51.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:51.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:12:51.127 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:12:51.194 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:51.339 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 57 16 76212 76196 83.5033 62.3125 0.0118453 0.0119701 2023-10-09T06:12:51.401 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:51.413 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:12:51.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:51.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:12:51.522 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:12:51.554 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:51.619 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:12:51.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:51.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:12:51.694 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T06:12:51.696 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:12:51.726 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:12:51.758 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:51.815 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:12:51.817 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:51 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1833182211' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:12:51.818 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:51 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:12:51.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:51.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:12:51.920 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:12:51.947 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:51.965 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:51 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1833182211' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:12:51.965 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:51 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:12:51.966 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:51 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1833182211' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:12:51.966 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:51 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:12:51.992 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:12:51.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:51.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:12:52.097 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:12:52.125 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:52.167 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:12:52.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:52.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:12:52.273 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:12:52.299 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:52.339 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 58 16 76736 76720 82.6285 32.75 0.00334634 0.0119857 2023-10-09T06:12:52.342 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:12:52.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:52.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:12:52.450 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:12:52.477 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:52.520 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:12:52.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:52.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:12:52.625 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:12:52.671 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:52.726 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:12:52.726 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:12:52.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:52 smithi156 ceph-mon[195101]: pgmap v1297: 119 pgs: 119 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T06:12:52.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:52 smithi156 ceph-mon[195101]: failed to encode map e509 with expected crc 2023-10-09T06:12:52.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:52 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:12:52.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:52 smithi156 ceph-mon[195101]: osdmap e509: 8 total, 8 up, 8 in 2023-10-09T06:12:52.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:52 smithi156 ceph-mon[195101]: failed to encode map e509 with expected crc 2023-10-09T06:12:52.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:52 smithi156 ceph-mon[195101]: osdmap e510: 8 total, 8 up, 8 in 2023-10-09T06:12:52.961 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:52 smithi064 ceph-mon[212750]: pgmap v1297: 119 pgs: 119 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T06:12:52.961 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:52 smithi064 ceph-mon[212750]: failed to encode map e509 with expected crc 2023-10-09T06:12:52.961 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:52 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:12:52.962 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:52 smithi064 ceph-mon[212750]: osdmap e509: 8 total, 8 up, 8 in 2023-10-09T06:12:52.962 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:52 smithi064 ceph-mon[212750]: failed to encode map e509 with expected crc 2023-10-09T06:12:52.962 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:52 smithi064 ceph-mon[212750]: osdmap e510: 8 total, 8 up, 8 in 2023-10-09T06:12:52.962 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:52 smithi064 ceph-mon[220044]: pgmap v1297: 119 pgs: 119 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T06:12:52.962 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:52 smithi064 ceph-mon[220044]: failed to encode map e509 with expected crc 2023-10-09T06:12:52.962 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:52 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:12:52.962 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:52 smithi064 ceph-mon[220044]: osdmap e509: 8 total, 8 up, 8 in 2023-10-09T06:12:52.962 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:52 smithi064 ceph-mon[220044]: failed to encode map e509 with expected crc 2023-10-09T06:12:52.962 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:52 smithi064 ceph-mon[220044]: osdmap e510: 8 total, 8 up, 8 in 2023-10-09T06:12:53.339 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 59 16 76736 76720 81.2286 0 - 0.0119857 2023-10-09T06:12:53.358 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:12:53.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:53.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:12:53.462 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:12:53.491 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:53.719 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T06:12:53.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:53 smithi064 ceph-mon[212750]: failed to encode map e510 with expected crc 2023-10-09T06:12:53.960 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:53 smithi064 ceph-mon[212750]: failed to encode map e510 with expected crc 2023-10-09T06:12:53.960 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:53 smithi064 ceph-mon[212750]: failed to encode map e510 with expected crc 2023-10-09T06:12:53.960 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:53 smithi064 ceph-mon[212750]: osdmap e511: 8 total, 8 up, 8 in 2023-10-09T06:12:53.960 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:53 smithi064 ceph-mon[220044]: failed to encode map e510 with expected crc 2023-10-09T06:12:53.960 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:53 smithi064 ceph-mon[220044]: failed to encode map e510 with expected crc 2023-10-09T06:12:53.960 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:53 smithi064 ceph-mon[220044]: failed to encode map e510 with expected crc 2023-10-09T06:12:53.960 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:53 smithi064 ceph-mon[220044]: osdmap e511: 8 total, 8 up, 8 in 2023-10-09T06:12:53.964 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:12:53.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:53.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:12:54.077 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:12:54.084 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:12:54.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:53 smithi156 ceph-mon[195101]: failed to encode map e510 with expected crc 2023-10-09T06:12:54.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:53 smithi156 ceph-mon[195101]: failed to encode map e510 with expected crc 2023-10-09T06:12:54.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:53 smithi156 ceph-mon[195101]: failed to encode map e510 with expected crc 2023-10-09T06:12:54.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:53 smithi156 ceph-mon[195101]: osdmap e511: 8 total, 8 up, 8 in 2023-10-09T06:12:54.178 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:54.307 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:54.314 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:12:54.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:54.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:12:54.344 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-10-09T06:12:54.344871+0000 min lat: 0.0010971 max lat: 2.06707 avg lat: 0.0124372 2023-10-09T06:12:54.344 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:12:54.344 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 60 16 77207 77191 80.3598 14.7188 0.027424 0.0124372 2023-10-09T06:12:54.426 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:12:54.582 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:54.795 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:54 smithi064 ceph-mon[212750]: failed to encode map e510 with expected crc 2023-10-09T06:12:54.795 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:54 smithi064 ceph-mon[212750]: failed to encode map e510 with expected crc 2023-10-09T06:12:54.796 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:54 smithi064 ceph-mon[212750]: pgmap v1300: 119 pgs: 1 remapped+peering, 118 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T06:12:54.796 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:54 smithi064 ceph-mon[212750]: failed to encode map e511 with expected crc 2023-10-09T06:12:54.796 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:54 smithi064 ceph-mon[212750]: 10.3 starting backfill to osd.3 from (0'0,0'0] MAX to 508'4902 2023-10-09T06:12:54.796 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:54 smithi064 ceph-mon[212750]: failed to encode map e511 with expected crc 2023-10-09T06:12:54.796 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:54 smithi064 ceph-mon[212750]: failed to encode map e511 with expected crc 2023-10-09T06:12:54.796 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:54 smithi064 ceph-mon[212750]: osdmap e512: 8 total, 8 up, 8 in 2023-10-09T06:12:54.798 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:54 smithi064 ceph-mon[220044]: failed to encode map e510 with expected crc 2023-10-09T06:12:54.798 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:54 smithi064 ceph-mon[220044]: failed to encode map e510 with expected crc 2023-10-09T06:12:54.798 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:54 smithi064 ceph-mon[220044]: pgmap v1300: 119 pgs: 1 remapped+peering, 118 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T06:12:54.798 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:54 smithi064 ceph-mon[220044]: failed to encode map e511 with expected crc 2023-10-09T06:12:54.798 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:54 smithi064 ceph-mon[220044]: 10.3 starting backfill to osd.3 from (0'0,0'0] MAX to 508'4902 2023-10-09T06:12:54.798 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:54 smithi064 ceph-mon[220044]: failed to encode map e511 with expected crc 2023-10-09T06:12:54.798 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:54 smithi064 ceph-mon[220044]: failed to encode map e511 with expected crc 2023-10-09T06:12:54.798 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:54 smithi064 ceph-mon[220044]: osdmap e512: 8 total, 8 up, 8 in 2023-10-09T06:12:54.970 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:12:54.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:54.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:12:55.083 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:12:55.098 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:54 smithi156 ceph-mon[195101]: failed to encode map e510 with expected crc 2023-10-09T06:12:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:54 smithi156 ceph-mon[195101]: failed to encode map e510 with expected crc 2023-10-09T06:12:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:54 smithi156 ceph-mon[195101]: pgmap v1300: 119 pgs: 1 remapped+peering, 118 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T06:12:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:54 smithi156 ceph-mon[195101]: failed to encode map e511 with expected crc 2023-10-09T06:12:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:54 smithi156 ceph-mon[195101]: 10.3 starting backfill to osd.3 from (0'0,0'0] MAX to 508'4902 2023-10-09T06:12:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:54 smithi156 ceph-mon[195101]: failed to encode map e511 with expected crc 2023-10-09T06:12:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:54 smithi156 ceph-mon[195101]: failed to encode map e511 with expected crc 2023-10-09T06:12:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:54 smithi156 ceph-mon[195101]: osdmap e512: 8 total, 8 up, 8 in 2023-10-09T06:12:55.198 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:55.344 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 61 16 78004 77988 79.8592 49.8125 0.00800695 0.0125152 2023-10-09T06:12:55.354 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:12:55.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:55.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:12:55.461 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:12:55.487 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:55.532 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:12:55.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:55.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:12:55.636 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:12:55.666 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:12:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:12:55] "GET /metrics HTTP/1.1" 200 36506 "" "Prometheus/2.43.0" 2023-10-09T06:12:55.667 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:12:55.668 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:55.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:55 smithi156 ceph-mon[195101]: failed to encode map e512 with expected crc 2023-10-09T06:12:55.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:55 smithi156 ceph-mon[195101]: failed to encode map e512 with expected crc 2023-10-09T06:12:55.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:55 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/721137151' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:12:55.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:55 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:12:55.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:55 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:12:55.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:55 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:12:55.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:55 smithi156 ceph-mon[195101]: osdmap e513: 8 total, 8 up, 8 in 2023-10-09T06:12:55.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:55 smithi064 ceph-mon[212750]: failed to encode map e512 with expected crc 2023-10-09T06:12:55.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:55 smithi064 ceph-mon[212750]: failed to encode map e512 with expected crc 2023-10-09T06:12:55.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:55 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/721137151' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:12:55.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:55 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:12:55.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:55 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:12:55.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:55 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:12:55.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:55 smithi064 ceph-mon[212750]: osdmap e513: 8 total, 8 up, 8 in 2023-10-09T06:12:55.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:55 smithi064 ceph-mon[220044]: failed to encode map e512 with expected crc 2023-10-09T06:12:55.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:55 smithi064 ceph-mon[220044]: failed to encode map e512 with expected crc 2023-10-09T06:12:55.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:55 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/721137151' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:12:55.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:55 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:12:55.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:55 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:12:55.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:55 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:12:55.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:55 smithi064 ceph-mon[220044]: osdmap e513: 8 total, 8 up, 8 in 2023-10-09T06:12:56.003 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:12:56.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:56.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:12:56.117 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:12:56.155 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:56.220 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:12:56.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:56.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:12:56.331 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:12:56.344 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 62 16 78690 78674 79.2628 42.875 0.00302993 0.0126025 2023-10-09T06:12:56.357 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:56.399 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:12:56.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:56.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:12:56.506 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:12:56.531 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:56.585 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:12:56.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:56.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:12:56.691 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:12:56.725 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:56.783 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:12:56.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:56.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:12:56.891 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:12:56.923 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:56.934 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:56 smithi064 ceph-mon[212750]: pgmap v1303: 119 pgs: 1 remapped+peering, 118 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 14 MiB/s wr, 226 op/s 2023-10-09T06:12:56.935 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:56 smithi064 ceph-mon[212750]: failed to encode map e513 with expected crc 2023-10-09T06:12:56.935 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:56 smithi064 ceph-mon[212750]: failed to encode map e513 with expected crc 2023-10-09T06:12:56.935 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:56 smithi064 ceph-mon[212750]: failed to encode map e513 with expected crc 2023-10-09T06:12:56.935 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:56 smithi064 ceph-mon[212750]: failed to encode map e513 with expected crc 2023-10-09T06:12:56.935 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:56 smithi064 ceph-mon[212750]: failed to encode map e513 with expected crc 2023-10-09T06:12:56.936 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:56 smithi064 ceph-mon[220044]: pgmap v1303: 119 pgs: 1 remapped+peering, 118 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 14 MiB/s wr, 226 op/s 2023-10-09T06:12:56.936 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:56 smithi064 ceph-mon[220044]: failed to encode map e513 with expected crc 2023-10-09T06:12:56.936 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:56 smithi064 ceph-mon[220044]: failed to encode map e513 with expected crc 2023-10-09T06:12:56.936 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:56 smithi064 ceph-mon[220044]: failed to encode map e513 with expected crc 2023-10-09T06:12:56.936 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:56 smithi064 ceph-mon[220044]: failed to encode map e513 with expected crc 2023-10-09T06:12:56.936 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:56 smithi064 ceph-mon[220044]: failed to encode map e513 with expected crc 2023-10-09T06:12:56.969 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:56.977 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:12:56.980 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:56 smithi156 ceph-mon[195101]: pgmap v1303: 119 pgs: 1 remapped+peering, 118 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 14 MiB/s wr, 226 op/s 2023-10-09T06:12:56.980 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:56 smithi156 ceph-mon[195101]: failed to encode map e513 with expected crc 2023-10-09T06:12:56.981 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:56 smithi156 ceph-mon[195101]: failed to encode map e513 with expected crc 2023-10-09T06:12:56.981 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:56 smithi156 ceph-mon[195101]: failed to encode map e513 with expected crc 2023-10-09T06:12:56.981 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:56 smithi156 ceph-mon[195101]: failed to encode map e513 with expected crc 2023-10-09T06:12:56.981 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:56 smithi156 ceph-mon[195101]: failed to encode map e513 with expected crc 2023-10-09T06:12:56.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:56.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:12:57.083 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:12:57.115 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:57.181 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:12:57.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:57.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:12:57.287 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:12:57.344 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 63 16 79460 79444 78.7688 48.125 0.0196084 0.0126904 2023-10-09T06:12:57.368 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:58.344 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 64 16 80258 80242 78.3174 49.875 0.0101688 0.0127634 2023-10-09T06:12:58.530 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:12:58.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:58.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:12:58.648 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:12:58.743 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:58.757 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:12:58.795 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:12:58.843 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:12:58.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:58 smithi064 ceph-mon[220044]: pgmap v1305: 119 pgs: 1 peering, 1 remapped+peering, 117 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 35 MiB/s wr, 559 op/s 2023-10-09T06:12:58.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:58 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:58.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:58 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:58.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:12:58 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/995822137' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:12:58.858 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:58 smithi064 ceph-mon[212750]: pgmap v1305: 119 pgs: 1 peering, 1 remapped+peering, 117 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 35 MiB/s wr, 559 op/s 2023-10-09T06:12:58.858 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:58 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:58.858 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:58.858 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:12:58 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/995822137' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:12:58.876 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:12:58.911 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:12:58.912 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:12:58.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:58.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:12:58.969 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:12:58.999 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:12:59.005 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:12:59.018 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:12:59.020 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:12:59.020 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:12:59.020 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:12:59.021 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:12:59.021 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:12:59.021 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:12:59.021 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:12:59.021 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:12:59.021 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:12:59.049 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:12:59.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:58 smithi156 ceph-mon[195101]: pgmap v1305: 119 pgs: 1 peering, 1 remapped+peering, 117 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 35 MiB/s wr, 559 op/s 2023-10-09T06:12:59.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:58 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:12:59.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:58 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:59.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:12:58 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/995822137' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:12:59.344 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 65 16 80983 80967 77.8098 45.3125 0.0110124 0.0128469 2023-10-09T06:12:59.726 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:12:59.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:12:59.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:12:59.837 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:12:59.887 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:00.345 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 66 16 81761 81745 77.3677 48.625 0.0279154 0.0129195 2023-10-09T06:13:00.489 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:13:00.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:00.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:13:00.607 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:13:00.696 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:00.907 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:13:00.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:00.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:13:01.017 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:13:01.049 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:00 smithi156 ceph-mon[195101]: pgmap v1306: 119 pgs: 1 active+remapped+backfilling, 1 peering, 117 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 27 MiB/s wr, 436 op/s; 4755/235188 objects misplaced (2.022%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:13:01.052 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:01.089 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:13:01.108 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:13:01.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:01.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:13:01.218 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:13:01.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:00 smithi064 ceph-mon[212750]: pgmap v1306: 119 pgs: 1 active+remapped+backfilling, 1 peering, 117 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 27 MiB/s wr, 436 op/s; 4755/235188 objects misplaced (2.022%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:13:01.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:00 smithi064 ceph-mon[220044]: pgmap v1306: 119 pgs: 1 active+remapped+backfilling, 1 peering, 117 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 27 MiB/s wr, 436 op/s; 4755/235188 objects misplaced (2.022%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:13:01.247 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:13:01.248 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T06:13:01.248 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T06:13:01.248 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:13:01.248 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.248 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:134367 8.f 8:f7ea56f9:::benchmark_data_smithi064_457003_object29849:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2023-10-09T06:13:01.248 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:10.416594+0000", 2023-10-09T06:13:01.248 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 470.82648733100001, 2023-10-09T06:13:01.248 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2992004480000001, 2023-10-09T06:13:01.248 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:13:01.248 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:13:01.249 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:13:01.249 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:13:01.249 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:13:01.249 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 134367 2023-10-09T06:13:01.249 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.249 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:13:01.249 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.249 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:13:01.249 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.416594+0000", 2023-10-09T06:13:01.249 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.249 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.249 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.250 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:13:01.250 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.416594+0000", 2023-10-09T06:13:01.250 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.250 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.250 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.250 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:13:01.250 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.416592+0000", 2023-10-09T06:13:01.250 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:13:01.251 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.251 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.252 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:13:01.252 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.416600+0000", 2023-10-09T06:13:01.252 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7970000000000001e-06 2023-10-09T06:13:01.252 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.252 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.252 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:13:01.252 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560025+0000", 2023-10-09T06:13:01.252 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14342429500000001 2023-10-09T06:13:01.252 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.252 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.252 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:13:01.252 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560035+0000", 2023-10-09T06:13:01.253 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0399e-05 2023-10-09T06:13:01.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.253 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.253 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.562476+0000", 2023-10-09T06:13:01.253 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024408189999999999 2023-10-09T06:13:01.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.253 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:13:01.253 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.562602+0000", 2023-10-09T06:13:01.253 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012572700000000001 2023-10-09T06:13:01.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.254 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:13:01.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.562735+0000", 2023-10-09T06:13:01.254 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013364599999999999 2023-10-09T06:13:01.254 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.254 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.573458+0000", 2023-10-09T06:13:01.254 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010723264 2023-10-09T06:13:01.254 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.255 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.255 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.573473+0000", 2023-10-09T06:13:01.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4623e-05 2023-10-09T06:13:01.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.255 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.255 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:11.446813+0000", 2023-10-09T06:13:01.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.87334020000000001 2023-10-09T06:13:01.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.256 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.256 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:11.446827+0000", 2023-10-09T06:13:01.256 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3597999999999999e-05 2023-10-09T06:13:01.256 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.256 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.256 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.256 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.582495+0000", 2023-10-09T06:13:01.256 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.135667666 2023-10-09T06:13:01.256 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.256 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.256 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-10-09T06:13:01.256 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.582634+0000", 2023-10-09T06:13:01.257 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000139698 2023-10-09T06:13:01.257 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.257 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.257 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.257 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.715717+0000", 2023-10-09T06:13:01.257 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.13308252800000001 2023-10-09T06:13:01.257 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.257 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.257 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:13:01.257 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.715795+0000", 2023-10-09T06:13:01.257 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7799000000000005e-05 2023-10-09T06:13:01.257 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.258 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:13:01.258 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.258 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.258 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.258 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:134376 8.f 8:f16b3848:::benchmark_data_smithi064_457003_object29858:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2023-10-09T06:13:01.258 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:10.560131+0000", 2023-10-09T06:13:01.258 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 470.68295083700002, 2023-10-09T06:13:01.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1496100760000001, 2023-10-09T06:13:01.258 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:13:01.258 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:01.258 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:13:01.259 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:13:01.259 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:13:01.259 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 134376 2023-10-09T06:13:01.259 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.259 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:13:01.259 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.259 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:13:01.259 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560131+0000", 2023-10-09T06:13:01.259 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.259 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.260 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.260 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:13:01.260 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560131+0000", 2023-10-09T06:13:01.260 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.260 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.260 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.260 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:13:01.260 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560130+0000", 2023-10-09T06:13:01.260 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:13:01.260 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.260 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.260 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:13:01.260 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560135+0000", 2023-10-09T06:13:01.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2499999999999997e-06 2023-10-09T06:13:01.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.261 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.261 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:13:01.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560145+0000", 2023-10-09T06:13:01.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0074e-05 2023-10-09T06:13:01.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.261 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.261 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:13:01.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560153+0000", 2023-10-09T06:13:01.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8529999999999992e-06 2023-10-09T06:13:01.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.573993+0000", 2023-10-09T06:13:01.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013840427000000001 2023-10-09T06:13:01.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.574002+0000", 2023-10-09T06:13:01.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9220000000000003e-06 2023-10-09T06:13:01.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.263 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.263 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.263 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:11.446946+0000", 2023-10-09T06:13:01.263 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.87294397499999998 2023-10-09T06:13:01.263 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.263 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.263 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.263 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:11.446956+0000", 2023-10-09T06:13:01.263 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8779999999999993e-06 2023-10-09T06:13:01.264 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.264 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.264 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.264 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.582901+0000", 2023-10-09T06:13:01.264 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.135944788 2023-10-09T06:13:01.264 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.264 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.264 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:13:01.264 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.583040+0000", 2023-10-09T06:13:01.264 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013896899999999999 2023-10-09T06:13:01.264 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.264 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.265 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,4", 2023-10-09T06:13:01.265 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.586304+0000", 2023-10-09T06:13:01.265 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032639660000000001 2023-10-09T06:13:01.265 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.265 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.265 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:13:01.265 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.600938+0000", 2023-10-09T06:13:01.265 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014634311000000001 2023-10-09T06:13:01.265 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.265 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.265 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.600984+0000", 2023-10-09T06:13:01.266 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5691e-05 2023-10-09T06:13:01.266 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.266 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.266 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.602931+0000", 2023-10-09T06:13:01.266 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019468339999999999 2023-10-09T06:13:01.266 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.266 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.266 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.709596+0000", 2023-10-09T06:13:01.266 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.106665146 2023-10-09T06:13:01.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:13:01.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.709659+0000", 2023-10-09T06:13:01.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2990000000000005e-05 2023-10-09T06:13:01.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:13:01.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.709741+0000", 2023-10-09T06:13:01.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1841999999999993e-05 2023-10-09T06:13:01.267 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.267 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:13:01.268 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.268 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.268 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:134381 8.f 8:f7feb3ed:::benchmark_data_smithi064_457003_object29863:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2023-10-09T06:13:01.268 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:10.560160+0000", 2023-10-09T06:13:01.268 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 470.68292156699999, 2023-10-09T06:13:01.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1498039279999999, 2023-10-09T06:13:01.268 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:13:01.268 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:01.268 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:13:01.268 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:13:01.268 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:13:01.268 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 134381 2023-10-09T06:13:01.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.269 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:13:01.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:13:01.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560160+0000", 2023-10-09T06:13:01.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:13:01.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560160+0000", 2023-10-09T06:13:01.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:13:01.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560159+0000", 2023-10-09T06:13:01.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:13:01.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:13:01.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560162+0000", 2023-10-09T06:13:01.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8509999999999999e-06 2023-10-09T06:13:01.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.271 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:13:01.271 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560169+0000", 2023-10-09T06:13:01.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.489e-06 2023-10-09T06:13:01.271 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.271 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:13:01.271 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.560175+0000", 2023-10-09T06:13:01.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5579999999999997e-06 2023-10-09T06:13:01.271 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.271 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.271 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.574927+0000", 2023-10-09T06:13:01.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014751195 2023-10-09T06:13:01.272 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.272 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.272 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.272 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.574936+0000", 2023-10-09T06:13:01.272 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4769999999999992e-06 2023-10-09T06:13:01.272 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.272 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.272 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.272 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:11.446983+0000", 2023-10-09T06:13:01.272 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.87204703100000003 2023-10-09T06:13:01.272 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.272 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.273 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.273 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:11.446991+0000", 2023-10-09T06:13:01.273 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4200000000000007e-06 2023-10-09T06:13:01.273 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.273 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.273 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.273 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.593274+0000", 2023-10-09T06:13:01.273 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1462829640000001 2023-10-09T06:13:01.273 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.273 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.273 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:13:01.273 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.593416+0000", 2023-10-09T06:13:01.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000141827 2023-10-09T06:13:01.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.274 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,4", 2023-10-09T06:13:01.274 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.593560+0000", 2023-10-09T06:13:01.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014419699999999999 2023-10-09T06:13:01.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.274 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.274 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.601021+0000", 2023-10-09T06:13:01.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0074606000000000004 2023-10-09T06:13:01.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:13:01.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.602791+0000", 2023-10-09T06:13:01.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0017704789999999999 2023-10-09T06:13:01.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.616064+0000", 2023-10-09T06:13:01.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013272879 2023-10-09T06:13:01.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.276 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.709794+0000", 2023-10-09T06:13:01.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.093729268000000004 2023-10-09T06:13:01.276 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:13:01.276 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.709848+0000", 2023-10-09T06:13:01.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.427e-05 2023-10-09T06:13:01.276 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:13:01.276 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.709964+0000", 2023-10-09T06:13:01.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000115875 2023-10-09T06:13:01.277 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.277 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:13:01.277 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.277 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.45605.0:134407 8.f 8:fbf853b5:::benchmark_data_smithi064_457003_object29889:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-10-09T06:13:01.277 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:05:10.594588+0000", 2023-10-09T06:13:01.277 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 470.64849339099999, 2023-10-09T06:13:01.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1155959489999998, 2023-10-09T06:13:01.277 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:13:01.277 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:01.278 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:13:01.278 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.45605", 2023-10-09T06:13:01.278 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:13:01.278 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 134407 2023-10-09T06:13:01.278 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.278 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:13:01.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.278 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:13:01.278 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.594588+0000", 2023-10-09T06:13:01.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.278 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:13:01.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.594588+0000", 2023-10-09T06:13:01.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:13:01.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.594587+0000", 2023-10-09T06:13:01.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:13:01.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:13:01.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.594592+0000", 2023-10-09T06:13:01.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2689999999999999e-06 2023-10-09T06:13:01.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:13:01.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.594606+0000", 2023-10-09T06:13:01.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.38e-05 2023-10-09T06:13:01.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:13:01.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.594612+0000", 2023-10-09T06:13:01.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3600000000000004e-06 2023-10-09T06:13:01.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.281 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.594675+0000", 2023-10-09T06:13:01.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3595000000000004e-05 2023-10-09T06:13:01.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.281 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:10.594687+0000", 2023-10-09T06:13:01.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2289999999999999e-05 2023-10-09T06:13:01.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:11.447669+0000", 2023-10-09T06:13:01.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.85298183299999997 2023-10-09T06:13:01.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:11.447680+0000", 2023-10-09T06:13:01.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1025e-05 2023-10-09T06:13:01.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.597255+0000", 2023-10-09T06:13:01.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.149574305 2023-10-09T06:13:01.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:13:01.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.597398+0000", 2023-10-09T06:13:01.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000143036 2023-10-09T06:13:01.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,4", 2023-10-09T06:13:01.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.597558+0000", 2023-10-09T06:13:01.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016018900000000001 2023-10-09T06:13:01.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.601331+0000", 2023-10-09T06:13:01.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037728509999999998 2023-10-09T06:13:01.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:13:01.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.602800+0000", 2023-10-09T06:13:01.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014689779999999999 2023-10-09T06:13:01.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.616098+0000", 2023-10-09T06:13:01.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013298783 2023-10-09T06:13:01.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.710024+0000", 2023-10-09T06:13:01.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.093925355000000002 2023-10-09T06:13:01.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:13:01.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.710070+0000", 2023-10-09T06:13:01.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6561999999999999e-05 2023-10-09T06:13:01.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:13:01.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:05:12.710184+0000", 2023-10-09T06:13:01.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011362900000000001 2023-10-09T06:13:01.286 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.286 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:13:01.286 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.287 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116287 9.1 9:8655aad1:::benchmark_data_smithi064_511911_object15185:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-10-09T06:13:01.287 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.530424+0000", 2023-10-09T06:13:01.287 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 238.712657345, 2023-10-09T06:13:01.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.181524692, 2023-10-09T06:13:01.287 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:13:01.287 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:13:01.287 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:13:01.287 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:13:01.287 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:13:01.287 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116287 2023-10-09T06:13:01.288 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.288 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:13:01.288 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.288 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:13:01.288 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530424+0000", 2023-10-09T06:13:01.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.288 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.288 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.288 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:13:01.288 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530424+0000", 2023-10-09T06:13:01.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.288 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:13:01.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530422+0000", 2023-10-09T06:13:01.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:13:01.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:13:01.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530432+0000", 2023-10-09T06:13:01.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0105000000000001e-05 2023-10-09T06:13:01.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:13:01.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530454+0000", 2023-10-09T06:13:01.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1579000000000001e-05 2023-10-09T06:13:01.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:13:01.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530465+0000", 2023-10-09T06:13:01.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1044e-05 2023-10-09T06:13:01.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530519+0000", 2023-10-09T06:13:01.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4567000000000001e-05 2023-10-09T06:13:01.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.291 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:13:01.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530668+0000", 2023-10-09T06:13:01.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014871900000000001 2023-10-09T06:13:01.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:13:01.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530823+0000", 2023-10-09T06:13:01.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000155375 2023-10-09T06:13:01.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.533953+0000", 2023-10-09T06:13:01.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031297159999999998 2023-10-09T06:13:01.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:13:01.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.534034+0000", 2023-10-09T06:13:01.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0731999999999999e-05 2023-10-09T06:13:01.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.605818+0000", 2023-10-09T06:13:01.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.071784291 2023-10-09T06:13:01.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.605835+0000", 2023-10-09T06:13:01.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6589e-05 2023-10-09T06:13:01.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578572+0000", 2023-10-09T06:13:01.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97273759299999996 2023-10-09T06:13:01.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578584+0000", 2023-10-09T06:13:01.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1283e-05 2023-10-09T06:13:01.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.711858+0000", 2023-10-09T06:13:01.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1332740379999999 2023-10-09T06:13:01.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:13:01.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.711949+0000", 2023-10-09T06:13:01.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1187000000000006e-05 2023-10-09T06:13:01.296 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.296 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:13:01.296 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.296 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116293 9.1 9:8657b0ae:::benchmark_data_smithi064_511911_object15191:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-10-09T06:13:01.296 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.545024+0000", 2023-10-09T06:13:01.296 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 238.69805703099999, 2023-10-09T06:13:01.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1673195999999999, 2023-10-09T06:13:01.297 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:13:01.297 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:13:01.297 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:13:01.297 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:13:01.297 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:13:01.297 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116293 2023-10-09T06:13:01.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.297 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:13:01.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:13:01.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545024+0000", 2023-10-09T06:13:01.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:13:01.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545024+0000", 2023-10-09T06:13:01.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:13:01.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545023+0000", 2023-10-09T06:13:01.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:13:01.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:13:01.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545029+0000", 2023-10-09T06:13:01.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5910000000000003e-06 2023-10-09T06:13:01.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:13:01.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545037+0000", 2023-10-09T06:13:01.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.909e-06 2023-10-09T06:13:01.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:13:01.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545042+0000", 2023-10-09T06:13:01.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1030000000000001e-06 2023-10-09T06:13:01.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545077+0000", 2023-10-09T06:13:01.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4620999999999999e-05 2023-10-09T06:13:01.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:13:01.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545160+0000", 2023-10-09T06:13:01.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3925999999999996e-05 2023-10-09T06:13:01.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:13:01.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545236+0000", 2023-10-09T06:13:01.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5909999999999997e-05 2023-10-09T06:13:01.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:13:01.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.547494+0000", 2023-10-09T06:13:01.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0022575310000000001 2023-10-09T06:13:01.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.302 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.302 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.547584+0000", 2023-10-09T06:13:01.302 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0315999999999994e-05 2023-10-09T06:13:01.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.302 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.302 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.605910+0000", 2023-10-09T06:13:01.302 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.058325845000000001 2023-10-09T06:13:01.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.303 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.605922+0000", 2023-10-09T06:13:01.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1712999999999999e-05 2023-10-09T06:13:01.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.303 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578649+0000", 2023-10-09T06:13:01.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97272731099999998 2023-10-09T06:13:01.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.303 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578660+0000", 2023-10-09T06:13:01.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0560999999999999e-05 2023-10-09T06:13:01.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.712239+0000", 2023-10-09T06:13:01.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.133579602 2023-10-09T06:13:01.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:13:01.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.712344+0000", 2023-10-09T06:13:01.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000104746 2023-10-09T06:13:01.304 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.305 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:13:01.305 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.305 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116277 9.7 9:e3c0da0d:::benchmark_data_smithi064_511911_object15175:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e449)", 2023-10-09T06:13:01.305 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.606356+0000", 2023-10-09T06:13:01.305 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 238.63672527200001, 2023-10-09T06:13:01.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1339391660000002, 2023-10-09T06:13:01.305 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:13:01.305 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:13:01.305 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:13:01.305 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:13:01.306 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:13:01.306 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116277 2023-10-09T06:13:01.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.306 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:13:01.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:13:01.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606356+0000", 2023-10-09T06:13:01.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:13:01.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606356+0000", 2023-10-09T06:13:01.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:13:01.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606355+0000", 2023-10-09T06:13:01.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:13:01.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:13:01.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606362+0000", 2023-10-09T06:13:01.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8249999999999999e-06 2023-10-09T06:13:01.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:13:01.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606372+0000", 2023-10-09T06:13:01.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0946e-05 2023-10-09T06:13:01.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:13:01.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606382+0000", 2023-10-09T06:13:01.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4849999999999998e-06 2023-10-09T06:13:01.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606441+0000", 2023-10-09T06:13:01.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8758000000000002e-05 2023-10-09T06:13:01.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606452+0000", 2023-10-09T06:13:01.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1435e-05 2023-10-09T06:13:01.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578004+0000", 2023-10-09T06:13:01.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97155135599999998 2023-10-09T06:13:01.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578015+0000", 2023-10-09T06:13:01.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1185999999999999e-05 2023-10-09T06:13:01.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.737752+0000", 2023-10-09T06:13:01.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.159737572 2023-10-09T06:13:01.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:13:01.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.740295+0000", 2023-10-09T06:13:01.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025430359999999998 2023-10-09T06:13:01.311 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.311 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:13:01.311 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.311 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116283 9.3 9:c10cf455:::benchmark_data_smithi064_511911_object15181:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e449)", 2023-10-09T06:13:01.312 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.606404+0000", 2023-10-09T06:13:01.312 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 238.636677201, 2023-10-09T06:13:01.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.160898918, 2023-10-09T06:13:01.312 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:13:01.312 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:13:01.312 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:13:01.312 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:13:01.312 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:13:01.312 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116283 2023-10-09T06:13:01.312 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.312 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:13:01.312 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:13:01.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606404+0000", 2023-10-09T06:13:01.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:13:01.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606404+0000", 2023-10-09T06:13:01.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:13:01.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606404+0000", 2023-10-09T06:13:01.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:13:01.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:13:01.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606407+0000", 2023-10-09T06:13:01.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9179999999999998e-06 2023-10-09T06:13:01.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:13:01.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606415+0000", 2023-10-09T06:13:01.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3180000000000007e-06 2023-10-09T06:13:01.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.315 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:13:01.315 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606421+0000", 2023-10-09T06:13:01.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2430000000000002e-06 2023-10-09T06:13:01.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.315 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.315 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606613+0000", 2023-10-09T06:13:01.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019148599999999999 2023-10-09T06:13:01.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606636+0000", 2023-10-09T06:13:01.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3680999999999999e-05 2023-10-09T06:13:01.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.576977+0000", 2023-10-09T06:13:01.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97034085699999995 2023-10-09T06:13:01.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.577913+0000", 2023-10-09T06:13:01.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00093576100000000001 2023-10-09T06:13:01.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.317 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.723002+0000", 2023-10-09T06:13:01.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1450891699999999 2023-10-09T06:13:01.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-10-09T06:13:01.317 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.723112+0000", 2023-10-09T06:13:01.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110089 2023-10-09T06:13:01.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.318 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.318 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.767258+0000", 2023-10-09T06:13:01.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.044145982 2023-10-09T06:13:01.318 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.318 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.318 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:13:01.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.767303+0000", 2023-10-09T06:13:01.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4894000000000001e-05 2023-10-09T06:13:01.318 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.318 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:13:01.319 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.319 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116296 9.1 9:8540568d:::benchmark_data_smithi064_511911_object15194:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2023-10-09T06:13:01.319 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.773495+0000", 2023-10-09T06:13:01.319 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 238.469586763, 2023-10-09T06:13:01.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.958274855, 2023-10-09T06:13:01.319 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:13:01.319 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:01.319 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:13:01.319 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:13:01.319 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:13:01.320 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116296 2023-10-09T06:13:01.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.320 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:13:01.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:13:01.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.773495+0000", 2023-10-09T06:13:01.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:13:01.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.773495+0000", 2023-10-09T06:13:01.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:13:01.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.773494+0000", 2023-10-09T06:13:01.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:13:01.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:13:01.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.773499+0000", 2023-10-09T06:13:01.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6319999999999999e-06 2023-10-09T06:13:01.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:13:01.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.773512+0000", 2023-10-09T06:13:01.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3288000000000001e-05 2023-10-09T06:13:01.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:13:01.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.773519+0000", 2023-10-09T06:13:01.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9090000000000003e-06 2023-10-09T06:13:01.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.773579+0000", 2023-10-09T06:13:01.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9871999999999999e-05 2023-10-09T06:13:01.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.773591+0000", 2023-10-09T06:13:01.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2526e-05 2023-10-09T06:13:01.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578692+0000", 2023-10-09T06:13:01.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.80510072099999996 2023-10-09T06:13:01.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578700+0000", 2023-10-09T06:13:01.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3380000000000005e-06 2023-10-09T06:13:01.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.325 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.325 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.712529+0000", 2023-10-09T06:13:01.325 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1338282500000001 2023-10-09T06:13:01.325 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.325 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.325 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:13:01.325 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.712695+0000", 2023-10-09T06:13:01.325 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000166875 2023-10-09T06:13:01.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.326 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6,7", 2023-10-09T06:13:01.326 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.713276+0000", 2023-10-09T06:13:01.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00058104399999999996 2023-10-09T06:13:01.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.720154+0000", 2023-10-09T06:13:01.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0068775190000000003 2023-10-09T06:13:01.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:13:01.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.723009+0000", 2023-10-09T06:13:01.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002855486 2023-10-09T06:13:01.328 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.731564+0000", 2023-10-09T06:13:01.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.008555053 2023-10-09T06:13:01.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.329 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.731603+0000", 2023-10-09T06:13:01.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8117000000000001e-05 2023-10-09T06:13:01.330 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:13:01.330 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.731663+0000", 2023-10-09T06:13:01.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0297000000000002e-05 2023-10-09T06:13:01.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:13:01.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.731770+0000", 2023-10-09T06:13:01.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000106616 2023-10-09T06:13:01.331 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.331 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:13:01.331 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.332 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116300 9.1 9:882a33e5:::benchmark_data_smithi064_511911_object15198:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2023-10-09T06:13:01.332 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.790061+0000", 2023-10-09T06:13:01.332 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 238.45301993699999, 2023-10-09T06:13:01.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.951866648, 2023-10-09T06:13:01.333 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:13:01.333 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:01.333 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:13:01.333 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:13:01.333 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:13:01.334 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116300 2023-10-09T06:13:01.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.334 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:13:01.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.334 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:13:01.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.790061+0000", 2023-10-09T06:13:01.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:13:01.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.790061+0000", 2023-10-09T06:13:01.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:13:01.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.790060+0000", 2023-10-09T06:13:01.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:13:01.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:13:01.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.790066+0000", 2023-10-09T06:13:01.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.434e-06 2023-10-09T06:13:01.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:13:01.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.790076+0000", 2023-10-09T06:13:01.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0404000000000001e-05 2023-10-09T06:13:01.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:13:01.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.790082+0000", 2023-10-09T06:13:01.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.908e-06 2023-10-09T06:13:01.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.790236+0000", 2023-10-09T06:13:01.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015350499999999999 2023-10-09T06:13:01.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.338 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.338 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.790243+0000", 2023-10-09T06:13:01.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6000000000000001e-06 2023-10-09T06:13:01.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.338 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.338 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578724+0000", 2023-10-09T06:13:01.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.78848058899999995 2023-10-09T06:13:01.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578734+0000", 2023-10-09T06:13:01.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6549999999999999e-06 2023-10-09T06:13:01.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.720179+0000", 2023-10-09T06:13:01.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1414453629999999 2023-10-09T06:13:01.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:13:01.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.720309+0000", 2023-10-09T06:13:01.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013024900000000001 2023-10-09T06:13:01.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.340 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6,7", 2023-10-09T06:13:01.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.720444+0000", 2023-10-09T06:13:01.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013511799999999999 2023-10-09T06:13:01.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.340 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:13:01.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.731215+0000", 2023-10-09T06:13:01.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010770693 2023-10-09T06:13:01.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.731528+0000", 2023-10-09T06:13:01.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000313109 2023-10-09T06:13:01.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.734686+0000", 2023-10-09T06:13:01.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031578380000000001 2023-10-09T06:13:01.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.741743+0000", 2023-10-09T06:13:01.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0070569559999999996 2023-10-09T06:13:01.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.342 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:13:01.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.741803+0000", 2023-10-09T06:13:01.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0517999999999999e-05 2023-10-09T06:13:01.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.342 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:13:01.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.741928+0000", 2023-10-09T06:13:01.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012472300000000001 2023-10-09T06:13:01.342 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.343 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:13:01.343 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.343 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:132917 9.8 9:12f89378:::benchmark_data_smithi064_511911_object31815:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e456)", 2023-10-09T06:13:01.343 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:26.626655+0000", 2023-10-09T06:13:01.343 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 214.616426346, 2023-10-09T06:13:01.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1614917130000002, 2023-10-09T06:13:01.343 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:13:01.343 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:01.343 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:13:01.343 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:13:01.344 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:13:01.344 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132917 2023-10-09T06:13:01.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.344 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:13:01.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:13:01.344 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.626655+0000", 2023-10-09T06:13:01.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:13:01.344 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.626655+0000", 2023-10-09T06:13:01.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.345 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:13:01.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.626653+0000", 2023-10-09T06:13:01.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:13:01.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.345 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:13:01.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.626661+0000", 2023-10-09T06:13:01.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7069999999999992e-06 2023-10-09T06:13:01.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:13:01.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.626673+0000", 2023-10-09T06:13:01.346 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2517999999999999e-05 2023-10-09T06:13:01.346 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:13:01.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.626683+0000", 2023-10-09T06:13:01.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4490000000000005e-06 2023-10-09T06:13:01.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.347 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.627477+0000", 2023-10-09T06:13:01.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00079445099999999995 2023-10-09T06:13:01.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.347 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.627484+0000", 2023-10-09T06:13:01.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4520000000000002e-06 2023-10-09T06:13:01.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528505+0000", 2023-10-09T06:13:01.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90102110700000004 2023-10-09T06:13:01.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528519+0000", 2023-10-09T06:13:01.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3943e-05 2023-10-09T06:13:01.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.755152+0000", 2023-10-09T06:13:01.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2266332689999999 2023-10-09T06:13:01.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:13:01.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.755289+0000", 2023-10-09T06:13:01.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013710400000000001 2023-10-09T06:13:01.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:13:01.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.755435+0000", 2023-10-09T06:13:01.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000145496 2023-10-09T06:13:01.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:13:01.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.766695+0000", 2023-10-09T06:13:01.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011260564000000001 2023-10-09T06:13:01.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.769909+0000", 2023-10-09T06:13:01.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032138940000000001 2023-10-09T06:13:01.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.351 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.785944+0000", 2023-10-09T06:13:01.351 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016034757 2023-10-09T06:13:01.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:13:01.352 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.785992+0000", 2023-10-09T06:13:01.352 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8412999999999999e-05 2023-10-09T06:13:01.352 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.352 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.352 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:13:01.352 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.788147+0000", 2023-10-09T06:13:01.352 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0021544020000000001 2023-10-09T06:13:01.352 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.352 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:13:01.352 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.352 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.352 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.352 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:132920 9.8 9:10a1d199:::benchmark_data_smithi064_511911_object31818:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e456)", 2023-10-09T06:13:01.353 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:26.641712+0000", 2023-10-09T06:13:01.353 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 214.60136905499999, 2023-10-09T06:13:01.353 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.148552274, 2023-10-09T06:13:01.353 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:13:01.353 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:01.353 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:13:01.354 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:13:01.354 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:13:01.354 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132920 2023-10-09T06:13:01.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.354 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:13:01.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.355 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:13:01.355 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.641712+0000", 2023-10-09T06:13:01.355 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.355 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:13:01.355 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.641712+0000", 2023-10-09T06:13:01.355 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:13:01.356 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.641710+0000", 2023-10-09T06:13:01.356 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:13:01.356 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.356 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:13:01.356 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.641717+0000", 2023-10-09T06:13:01.356 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0049999999999998e-06 2023-10-09T06:13:01.356 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.356 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:13:01.356 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.641731+0000", 2023-10-09T06:13:01.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3648e-05 2023-10-09T06:13:01.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:13:01.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.641743+0000", 2023-10-09T06:13:01.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1839e-05 2023-10-09T06:13:01.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.641792+0000", 2023-10-09T06:13:01.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9376000000000003e-05 2023-10-09T06:13:01.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.358 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.641803+0000", 2023-10-09T06:13:01.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1141000000000001e-05 2023-10-09T06:13:01.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528543+0000", 2023-10-09T06:13:01.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88673970899999999 2023-10-09T06:13:01.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528551+0000", 2023-10-09T06:13:01.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9319999999999996e-06 2023-10-09T06:13:01.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.360 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.360 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.760814+0000", 2023-10-09T06:13:01.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2322625380000001 2023-10-09T06:13:01.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:13:01.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.760963+0000", 2023-10-09T06:13:01.361 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000149522 2023-10-09T06:13:01.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:13:01.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.761205+0000", 2023-10-09T06:13:01.361 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000241831 2023-10-09T06:13:01.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.362 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:13:01.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.769866+0000", 2023-10-09T06:13:01.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086611819999999999 2023-10-09T06:13:01.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.769951+0000", 2023-10-09T06:13:01.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5286999999999998e-05 2023-10-09T06:13:01.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.788276+0000", 2023-10-09T06:13:01.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018324944999999999 2023-10-09T06:13:01.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:13:01.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.788329+0000", 2023-10-09T06:13:01.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2846999999999997e-05 2023-10-09T06:13:01.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.364 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:13:01.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.790265+0000", 2023-10-09T06:13:01.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019354929999999999 2023-10-09T06:13:01.364 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.364 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:13:01.364 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.364 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:132924 9.8 9:100587d5:::benchmark_data_smithi064_511911_object31822:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e456)", 2023-10-09T06:13:01.365 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:26.652624+0000", 2023-10-09T06:13:01.365 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 214.59045731500001, 2023-10-09T06:13:01.365 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1523453090000002, 2023-10-09T06:13:01.365 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:13:01.365 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:01.365 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:13:01.365 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:13:01.365 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:13:01.365 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132924 2023-10-09T06:13:01.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.365 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:13:01.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:13:01.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652624+0000", 2023-10-09T06:13:01.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:13:01.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652624+0000", 2023-10-09T06:13:01.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:13:01.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652616+0000", 2023-10-09T06:13:01.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999924 2023-10-09T06:13:01.367 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:13:01.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652632+0000", 2023-10-09T06:13:01.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5653000000000001e-05 2023-10-09T06:13:01.367 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:13:01.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652647+0000", 2023-10-09T06:13:01.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5244e-05 2023-10-09T06:13:01.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.368 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:13:01.368 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652656+0000", 2023-10-09T06:13:01.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0380000000000005e-06 2023-10-09T06:13:01.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.368 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.368 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652718+0000", 2023-10-09T06:13:01.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1267999999999997e-05 2023-10-09T06:13:01.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.368 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652730+0000", 2023-10-09T06:13:01.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2069000000000001e-05 2023-10-09T06:13:01.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528571+0000", 2023-10-09T06:13:01.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.875841753 2023-10-09T06:13:01.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528579+0000", 2023-10-09T06:13:01.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6410000000000002e-06 2023-10-09T06:13:01.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.764042+0000", 2023-10-09T06:13:01.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2354630339999999 2023-10-09T06:13:01.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:13:01.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.764130+0000", 2023-10-09T06:13:01.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7968999999999998e-05 2023-10-09T06:13:01.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:13:01.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.764240+0000", 2023-10-09T06:13:01.371 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010986 2023-10-09T06:13:01.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:13:01.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.769918+0000", 2023-10-09T06:13:01.371 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056776170000000003 2023-10-09T06:13:01.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.769984+0000", 2023-10-09T06:13:01.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6703000000000002e-05 2023-10-09T06:13:01.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.802902+0000", 2023-10-09T06:13:01.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.032917469999999997 2023-10-09T06:13:01.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:13:01.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.802957+0000", 2023-10-09T06:13:01.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5047999999999999e-05 2023-10-09T06:13:01.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:13:01.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.804969+0000", 2023-10-09T06:13:01.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020126900000000001 2023-10-09T06:13:01.373 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.373 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:13:01.373 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.373 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:132927 9.8 9:1f462da4:::benchmark_data_smithi064_511911_object31825:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e456)", 2023-10-09T06:13:01.373 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:26.663633+0000", 2023-10-09T06:13:01.373 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 214.57944841, 2023-10-09T06:13:01.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1462853150000001, 2023-10-09T06:13:01.374 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:13:01.374 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:01.374 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:13:01.374 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:13:01.374 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:13:01.374 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132927 2023-10-09T06:13:01.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.374 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:13:01.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:13:01.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.663633+0000", 2023-10-09T06:13:01.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:13:01.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.663633+0000", 2023-10-09T06:13:01.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:13:01.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.663631+0000", 2023-10-09T06:13:01.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:13:01.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.376 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:13:01.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.663639+0000", 2023-10-09T06:13:01.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3089999999999998e-06 2023-10-09T06:13:01.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.376 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:13:01.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.663652+0000", 2023-10-09T06:13:01.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3485999999999999e-05 2023-10-09T06:13:01.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:13:01.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.663658+0000", 2023-10-09T06:13:01.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.874e-06 2023-10-09T06:13:01.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.663716+0000", 2023-10-09T06:13:01.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8115000000000002e-05 2023-10-09T06:13:01.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.663728+0000", 2023-10-09T06:13:01.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.168e-05 2023-10-09T06:13:01.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528601+0000", 2023-10-09T06:13:01.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.86487267700000003 2023-10-09T06:13:01.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:13:01.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528608+0000", 2023-10-09T06:13:01.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2509999999999997e-06 2023-10-09T06:13:01.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.766727+0000", 2023-10-09T06:13:01.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2381187899999999 2023-10-09T06:13:01.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:13:01.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.766857+0000", 2023-10-09T06:13:01.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013015999999999999 2023-10-09T06:13:01.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:13:01.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.767000+0000", 2023-10-09T06:13:01.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014345199999999999 2023-10-09T06:13:01.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.770019+0000", 2023-10-09T06:13:01.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030185609999999999 2023-10-09T06:13:01.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:13:01.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.785896+0000", 2023-10-09T06:13:01.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015877437000000001 2023-10-09T06:13:01.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.808044+0000", 2023-10-09T06:13:01.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.022147962 2023-10-09T06:13:01.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:13:01.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.808105+0000", 2023-10-09T06:13:01.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0856000000000001e-05 2023-10-09T06:13:01.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:13:01.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.809918+0000", 2023-10-09T06:13:01.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018133559999999999 2023-10-09T06:13:01.382 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.382 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:13:01.382 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.382 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:47835 10.7 10:e47dc5b5:::benchmark_data_smithi064_573750_object47834:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T06:13:01.382 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:12:23.572914+0000", 2023-10-09T06:13:01.382 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 37.670167253999999, 2023-10-09T06:13:01.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.214116626, 2023-10-09T06:13:01.383 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:13:01.383 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:01.383 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:13:01.383 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:13:01.383 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:13:01.383 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 47835 2023-10-09T06:13:01.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.383 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:13:01.383 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.383 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:13:01.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.572914+0000", 2023-10-09T06:13:01.384 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.384 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.384 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:13:01.384 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.572914+0000", 2023-10-09T06:13:01.384 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.384 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.384 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:13:01.384 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.572913+0000", 2023-10-09T06:13:01.384 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:13:01.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:13:01.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.572939+0000", 2023-10-09T06:13:01.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6465000000000002e-05 2023-10-09T06:13:01.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:13:01.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.572953+0000", 2023-10-09T06:13:01.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3512e-05 2023-10-09T06:13:01.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:13:01.386 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.572961+0000", 2023-10-09T06:13:01.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7700000000000007e-06 2023-10-09T06:13:01.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.386 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.573928+0000", 2023-10-09T06:13:01.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00096672999999999995 2023-10-09T06:13:01.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.387 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:13:01.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.574067+0000", 2023-10-09T06:13:01.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013872999999999999 2023-10-09T06:13:01.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.387 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:13:01.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.574204+0000", 2023-10-09T06:13:01.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013745000000000001 2023-10-09T06:13:01.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.388 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.578671+0000", 2023-10-09T06:13:01.388 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.004466345 2023-10-09T06:13:01.388 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:13:01.388 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.578727+0000", 2023-10-09T06:13:01.388 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6452000000000001e-05 2023-10-09T06:13:01.388 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.786924+0000", 2023-10-09T06:13:01.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20819700399999999 2023-10-09T06:13:01.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:13:01.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.786991+0000", 2023-10-09T06:13:01.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7402000000000003e-05 2023-10-09T06:13:01.390 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:13:01.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.787031+0000", 2023-10-09T06:13:01.390 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9382999999999999e-05 2023-10-09T06:13:01.390 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.390 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:13:01.390 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.390 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.390 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:47836 10.7 10:ee4aad89:::benchmark_data_smithi064_573750_object47835:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T06:13:01.391 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:12:23.572978+0000", 2023-10-09T06:13:01.391 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 37.670103683000001, 2023-10-09T06:13:01.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21427558399999999, 2023-10-09T06:13:01.391 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:13:01.391 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:01.391 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:13:01.391 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:13:01.392 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:13:01.392 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 47836 2023-10-09T06:13:01.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.392 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:13:01.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:13:01.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.572978+0000", 2023-10-09T06:13:01.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:13:01.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.572978+0000", 2023-10-09T06:13:01.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:13:01.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.572977+0000", 2023-10-09T06:13:01.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:13:01.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:13:01.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.573000+0000", 2023-10-09T06:13:01.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3175000000000001e-05 2023-10-09T06:13:01.394 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.394 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.394 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:13:01.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.573008+0000", 2023-10-09T06:13:01.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2020000000000004e-06 2023-10-09T06:13:01.394 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.394 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.394 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:13:01.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.573015+0000", 2023-10-09T06:13:01.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.083e-06 2023-10-09T06:13:01.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.395 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.575913+0000", 2023-10-09T06:13:01.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028982959999999999 2023-10-09T06:13:01.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.395 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:13:01.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.576075+0000", 2023-10-09T06:13:01.396 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016197999999999999 2023-10-09T06:13:01.396 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.396 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.396 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:13:01.396 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.576223+0000", 2023-10-09T06:13:01.396 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000148407 2023-10-09T06:13:01.396 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.396 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.396 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:13:01.396 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.578732+0000", 2023-10-09T06:13:01.396 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025086040000000002 2023-10-09T06:13:01.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.397 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.397 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.397 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.580409+0000", 2023-10-09T06:13:01.397 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001676762 2023-10-09T06:13:01.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.397 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.397 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.397 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.787173+0000", 2023-10-09T06:13:01.397 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.206764752 2023-10-09T06:13:01.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:13:01.398 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.787221+0000", 2023-10-09T06:13:01.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7253999999999999e-05 2023-10-09T06:13:01.398 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:13:01.398 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.787253+0000", 2023-10-09T06:13:01.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2749000000000001e-05 2023-10-09T06:13:01.398 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.398 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:13:01.399 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.399 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.46283.0:47835 10.7 e505/490 ondisk, result = 0)", 2023-10-09T06:13:01.399 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:12:23.580544+0000", 2023-10-09T06:13:01.399 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 37.662537829999998, 2023-10-09T06:13:01.399 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20649993899999999, 2023-10-09T06:13:01.399 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:13:01.399 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:13:01.399 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:13:01.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.400 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:13:01.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.580544+0000", 2023-10-09T06:13:01.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.400 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:13:01.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.580544+0000", 2023-10-09T06:13:01.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.400 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:13:01.401 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.580543+0000", 2023-10-09T06:13:01.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:13:01.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:13:01.401 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.580554+0000", 2023-10-09T06:13:01.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1382e-05 2023-10-09T06:13:01.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:13:01.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.580564+0000", 2023-10-09T06:13:01.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8879999999999992e-06 2023-10-09T06:13:01.402 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.402 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.402 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:13:01.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.580573+0000", 2023-10-09T06:13:01.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3100000000000006e-06 2023-10-09T06:13:01.402 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.402 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.402 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.580613+0000", 2023-10-09T06:13:01.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9641000000000002e-05 2023-10-09T06:13:01.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:13:01.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.580634+0000", 2023-10-09T06:13:01.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0939000000000001e-05 2023-10-09T06:13:01.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:13:01.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.787044+0000", 2023-10-09T06:13:01.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.206409606 2023-10-09T06:13:01.404 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.404 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:13:01.404 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.404 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.404 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.46283.0:47836 10.7 e505/490 ondisk, result = 0)", 2023-10-09T06:13:01.404 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:12:23.580588+0000", 2023-10-09T06:13:01.404 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 37.662493189000003, 2023-10-09T06:13:01.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20667137499999999, 2023-10-09T06:13:01.404 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:13:01.405 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:13:01.405 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:13:01.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:13:01.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.580588+0000", 2023-10-09T06:13:01.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:13:01.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.580588+0000", 2023-10-09T06:13:01.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.406 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:13:01.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.580588+0000", 2023-10-09T06:13:01.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:13:01.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.406 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:13:01.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.580591+0000", 2023-10-09T06:13:01.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9160000000000001e-06 2023-10-09T06:13:01.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:13:01.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.580598+0000", 2023-10-09T06:13:01.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9460000000000001e-06 2023-10-09T06:13:01.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:13:01.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.580603+0000", 2023-10-09T06:13:01.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8980000000000004e-06 2023-10-09T06:13:01.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.787160+0000", 2023-10-09T06:13:01.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.206556931 2023-10-09T06:13:01.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:13:01.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.787167+0000", 2023-10-09T06:13:01.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7279999999999995e-06 2023-10-09T06:13:01.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:13:01.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.787260+0000", 2023-10-09T06:13:01.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2315e-05 2023-10-09T06:13:01.409 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.409 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:13:01.409 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.409 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:47842 10.9 e505/490 10:91e926cd:::benchmark_data_smithi064_573750_object47841:head v 505'2950, mlcod=505'2947)", 2023-10-09T06:13:01.410 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:12:23.580649+0000", 2023-10-09T06:13:01.410 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 37.662432813000002, 2023-10-09T06:13:01.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20985614299999999, 2023-10-09T06:13:01.410 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:13:01.410 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:01.410 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:13:01.410 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:13:01.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.580649+0000", 2023-10-09T06:13:01.411 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.411 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:13:01.411 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.580649+0000", 2023-10-09T06:13:01.411 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.411 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:13:01.411 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.580648+0000", 2023-10-09T06:13:01.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:13:01.412 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.412 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.412 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:13:01.412 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.580685+0000", 2023-10-09T06:13:01.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7289999999999997e-05 2023-10-09T06:13:01.412 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.412 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.412 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:13:01.413 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.786940+0000", 2023-10-09T06:13:01.413 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.206254566 2023-10-09T06:13:01.413 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.413 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.413 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:13:01.413 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.786950+0000", 2023-10-09T06:13:01.413 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9110000000000005e-06 2023-10-09T06:13:01.413 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.413 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.413 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.413 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.787005+0000", 2023-10-09T06:13:01.413 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5485000000000002e-05 2023-10-09T06:13:01.414 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.414 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.414 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:13:01.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.787029+0000", 2023-10-09T06:13:01.414 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4057e-05 2023-10-09T06:13:01.414 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.418 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.418 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:13:01.418 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.790481+0000", 2023-10-09T06:13:01.418 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034515269999999998 2023-10-09T06:13:01.418 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.418 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.419 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:13:01.419 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.790505+0000", 2023-10-09T06:13:01.419 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3827000000000001e-05 2023-10-09T06:13:01.419 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.419 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:13:01.419 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.419 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.419 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.419 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:47852 10.7 10:e136bdbb:::benchmark_data_smithi064_573750_object47851:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T06:13:01.419 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:12:23.582771+0000", 2023-10-09T06:13:01.419 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 37.660310318000001, 2023-10-09T06:13:01.420 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21171788599999999, 2023-10-09T06:13:01.420 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:13:01.420 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:01.420 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:13:01.420 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:13:01.420 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:13:01.420 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 47852 2023-10-09T06:13:01.420 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.420 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:13:01.420 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.420 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:13:01.420 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.582771+0000", 2023-10-09T06:13:01.421 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:13:01.421 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.582771+0000", 2023-10-09T06:13:01.421 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:13:01.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:13:01.421 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.582769+0000", 2023-10-09T06:13:01.422 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:13:01.422 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.422 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.422 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:13:01.422 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.582793+0000", 2023-10-09T06:13:01.422 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4015e-05 2023-10-09T06:13:01.422 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.422 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.422 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:13:01.422 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.787055+0000", 2023-10-09T06:13:01.422 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20426161400000001 2023-10-09T06:13:01.423 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.423 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.423 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:13:01.423 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.787064+0000", 2023-10-09T06:13:01.423 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4059999999999994e-06 2023-10-09T06:13:01.423 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.423 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.423 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:13:01.423 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.787908+0000", 2023-10-09T06:13:01.423 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00084332600000000004 2023-10-09T06:13:01.423 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.424 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.424 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:13:01.424 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.788052+0000", 2023-10-09T06:13:01.424 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000144332 2023-10-09T06:13:01.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.424 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.424 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:13:01.424 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.788192+0000", 2023-10-09T06:13:01.424 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013966099999999999 2023-10-09T06:13:01.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.424 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.790291+0000", 2023-10-09T06:13:01.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020994619999999999 2023-10-09T06:13:01.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:13:01.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.791115+0000", 2023-10-09T06:13:01.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00082404899999999998 2023-10-09T06:13:01.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:01.426 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.794397+0000", 2023-10-09T06:13:01.426 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003282161 2023-10-09T06:13:01.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:13:01.426 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.794449+0000", 2023-10-09T06:13:01.426 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1607999999999998e-05 2023-10-09T06:13:01.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:13:01.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:13:01.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:13:01.426 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:23.794489+0000", 2023-10-09T06:13:01.426 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9904999999999998e-05 2023-10-09T06:13:01.427 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.427 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:13:01.427 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.427 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:13:01.427 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:13:01.427 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:13:01.428 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 67 16 82580 82564 76.9771 51.1875 0.00964789 0.0129864 2023-10-09T06:13:01.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:01.429 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:01.429 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:13:01.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:13:01.530 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:13:01.618 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:01.735 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:13:01.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:01.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:13:01.812 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T06:13:01.827 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:01 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3096043642' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:13:01.827 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:01 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3096043642' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:13:01.845 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:13:01.871 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:01.913 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:13:01.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:01.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:13:02.019 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:13:02.043 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:01 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3096043642' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:13:02.045 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:02.087 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:13:02.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:02.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:13:02.191 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:13:02.284 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:02.345 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 68 16 83247 83231 76.4583 41.6875 0.0232798 0.0130725 2023-10-09T06:13:02.537 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:13:02.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:02 smithi064 ceph-mon[212750]: pgmap v1307: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 740 op/s; 4755/243942 objects misplaced (1.949%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:13:02.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:02 smithi064 ceph-mon[212750]: failed to encode map e514 with expected crc 2023-10-09T06:13:02.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:02 smithi064 ceph-mon[212750]: failed to encode map e514 with expected crc 2023-10-09T06:13:02.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:02 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3096043642' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:13:02.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:02 smithi064 ceph-mon[212750]: osdmap e514: 8 total, 8 up, 8 in 2023-10-09T06:13:02.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:02 smithi064 ceph-mon[220044]: pgmap v1307: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 740 op/s; 4755/243942 objects misplaced (1.949%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:13:02.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:02 smithi064 ceph-mon[220044]: failed to encode map e514 with expected crc 2023-10-09T06:13:02.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:02 smithi064 ceph-mon[220044]: failed to encode map e514 with expected crc 2023-10-09T06:13:02.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:02 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3096043642' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:13:02.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:02 smithi064 ceph-mon[220044]: osdmap e514: 8 total, 8 up, 8 in 2023-10-09T06:13:03.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:02 smithi156 ceph-mon[195101]: pgmap v1307: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 740 op/s; 4755/243942 objects misplaced (1.949%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:13:03.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:02 smithi156 ceph-mon[195101]: failed to encode map e514 with expected crc 2023-10-09T06:13:03.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:02 smithi156 ceph-mon[195101]: failed to encode map e514 with expected crc 2023-10-09T06:13:03.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:02 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3096043642' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:13:03.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:02 smithi156 ceph-mon[195101]: osdmap e514: 8 total, 8 up, 8 in 2023-10-09T06:13:03.283 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:13:03.345 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 69 16 83966 83950 76.0016 44.9375 0.010354 0.0131525 2023-10-09T06:13:03.428 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:13:03.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:03.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:13:03.538 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:13:03.576 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:03.659 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:13:03.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:03.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:13:03.766 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:13:03.766 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:13:03.802 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:04.045 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T06:13:04.200 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:13:04.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:04.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:13:04.315 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:13:04.345 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 70 16 84738 84722 75.6053 48.25 0.0176735 0.0132221 2023-10-09T06:13:04.471 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:04.524 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:13:04.616 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:13:04.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:04.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:13:04.722 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:13:04.761 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:04.823 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:13:04.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:04.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:13:04.926 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:04 smithi064 ceph-mon[212750]: pgmap v1309: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 818 op/s; 4572/246996 objects misplaced (1.851%); 5.6 MiB/s, 90 objects/s recovering 2023-10-09T06:13:04.927 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:13:04.928 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:04 smithi064 ceph-mon[220044]: pgmap v1309: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 818 op/s; 4572/246996 objects misplaced (1.851%); 5.6 MiB/s, 90 objects/s recovering 2023-10-09T06:13:05.033 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:05.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:04 smithi156 ceph-mon[195101]: pgmap v1309: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 818 op/s; 4572/246996 objects misplaced (1.851%); 5.6 MiB/s, 90 objects/s recovering 2023-10-09T06:13:05.345 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 71 16 85581 85565 75.282 52.6875 0.0112269 0.0132788 2023-10-09T06:13:05.363 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:13:05.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:05.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:13:05.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:13:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:13:05] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.43.0" 2023-10-09T06:13:05.469 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:13:05.504 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:05.561 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:13:05.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:05.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:13:05.665 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:13:05.692 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:05.796 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:13:05.969 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:13:05.972 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:05 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3457128528' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:13:05.973 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:05 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:13:05.973 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 06:13:05 smithi156 conmon[175419]: logger=cleanup t=2023-10-09T06:13:05.699653475Z level=info msg="Completed cleanup jobs" duration=740.829863ms 2023-10-09T06:13:05.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:05.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:13:06.075 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:13:06.123 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:06.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:05 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3457128528' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:13:06.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:05 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:13:06.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:05 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3457128528' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:13:06.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:05 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:13:06.233 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:13:06.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:06.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:13:06.338 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:13:06.345 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 72 16 86115 86099 74.7001 33.375 0.0124327 0.0133798 2023-10-09T06:13:06.366 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:06.610 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:13:06.610 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:13:06.610 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:06.610 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.46283.0:86233 10.e e515/490 10:7364a390:::benchmark_data_smithi064_573750_object86232:head v 515'5453, mlcod=515'5451)", 2023-10-09T06:13:06.610 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:06.604591+0000", 2023-10-09T06:13:06.610 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0025978070000000002, 2023-10-09T06:13:06.610 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0026110019999999999, 2023-10-09T06:13:06.610 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:06.611 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:06.611 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:06.611 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:06.611 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:06.611 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:06.604591+0000", 2023-10-09T06:13:06.611 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:06.611 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:06.611 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:06.611 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:06.611 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:06.604591+0000", 2023-10-09T06:13:06.611 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:06.611 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:06.612 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:06.612 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:06.612 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:06.604590+0000", 2023-10-09T06:13:06.612 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:13:06.612 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:06.612 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:06.612 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:06.612 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:06.604614+0000", 2023-10-09T06:13:06.612 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.349e-05 2023-10-09T06:13:06.612 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:06.612 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:06.612 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:06.613 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:06.604630+0000", 2023-10-09T06:13:06.613 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6759e-05 2023-10-09T06:13:06.613 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:06.613 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:06.613 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:06.613 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:06.604636+0000", 2023-10-09T06:13:06.613 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7760000000000003e-06 2023-10-09T06:13:06.613 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:06.613 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:06.613 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:06.613 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:06.604677+0000", 2023-10-09T06:13:06.613 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0673999999999997e-05 2023-10-09T06:13:06.614 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:06.614 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:06.614 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:06.614 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:06.604699+0000", 2023-10-09T06:13:06.614 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2416999999999999e-05 2023-10-09T06:13:06.614 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:06.614 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:06.614 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:13:06.614 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:06.607165+0000", 2023-10-09T06:13:06.614 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002465152 2023-10-09T06:13:06.614 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:06.614 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:06.615 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:13:06.615 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:06.607202+0000", 2023-10-09T06:13:06.615 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7623000000000002e-05 2023-10-09T06:13:06.615 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:06.615 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:06.615 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:06.615 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:06.615 INFO:teuthology.orchestra.run.smithi156.stdout: ], 2023-10-09T06:13:06.615 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 1 2023-10-09T06:13:06.615 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:13:06.855 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:13:06.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:06.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:13:06.960 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:13:06.973 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:13:06.990 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:06 smithi156 ceph-mon[195101]: pgmap v1310: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 41 MiB/s wr, 652 op/s; 4572/246996 objects misplaced (1.851%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T06:13:06.990 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:06 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:13:06.990 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:06 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:13:06.991 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:06 smithi156 ceph-mon[195101]: osdmap e515: 8 total, 8 up, 8 in 2023-10-09T06:13:06.991 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:06 smithi156 ceph-mon[195101]: failed to encode map e515 with expected crc 2023-10-09T06:13:06.994 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:07.050 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:13:07.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:07.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:13:07.154 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:13:07.194 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:07.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:06 smithi064 ceph-mon[212750]: pgmap v1310: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 41 MiB/s wr, 652 op/s; 4572/246996 objects misplaced (1.851%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T06:13:07.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:06 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:13:07.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:06 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:13:07.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:06 smithi064 ceph-mon[212750]: osdmap e515: 8 total, 8 up, 8 in 2023-10-09T06:13:07.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:06 smithi064 ceph-mon[212750]: failed to encode map e515 with expected crc 2023-10-09T06:13:07.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:06 smithi064 ceph-mon[220044]: pgmap v1310: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 41 MiB/s wr, 652 op/s; 4572/246996 objects misplaced (1.851%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T06:13:07.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:06 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:13:07.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:06 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:13:07.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:06 smithi064 ceph-mon[220044]: osdmap e515: 8 total, 8 up, 8 in 2023-10-09T06:13:07.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:06 smithi064 ceph-mon[220044]: failed to encode map e515 with expected crc 2023-10-09T06:13:07.259 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:13:07.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:07.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:13:07.346 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 73 16 86864 86848 74.3182 46.8125 0.0171979 0.0134499 2023-10-09T06:13:07.364 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:13:07.405 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:07.524 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:13:07.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:07.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:13:07.592 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:13:07.628 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:13:07.714 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:07.746 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:13:07.794 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:13:07.805 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:07 smithi156 ceph-mon[195101]: failed to encode map e515 with expected crc 2023-10-09T06:13:07.805 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:07 smithi156 ceph-mon[195101]: failed to encode map e515 with expected crc 2023-10-09T06:13:07.805 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:07 smithi156 ceph-mon[195101]: failed to encode map e515 with expected crc 2023-10-09T06:13:07.805 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:07 smithi156 ceph-mon[195101]: pgmap v1312: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s; 4572/256932 objects misplaced (1.779%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:13:07.805 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:07 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/711943364' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:13:07.853 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:13:07.893 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:07 smithi064 ceph-mon[212750]: failed to encode map e515 with expected crc 2023-10-09T06:13:07.893 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:07 smithi064 ceph-mon[212750]: failed to encode map e515 with expected crc 2023-10-09T06:13:07.893 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:07 smithi064 ceph-mon[212750]: failed to encode map e515 with expected crc 2023-10-09T06:13:07.893 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:07 smithi064 ceph-mon[212750]: pgmap v1312: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s; 4572/256932 objects misplaced (1.779%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:13:07.894 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:07 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/711943364' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:13:07.894 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:07 smithi064 ceph-mon[220044]: failed to encode map e515 with expected crc 2023-10-09T06:13:07.894 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:07 smithi064 ceph-mon[220044]: failed to encode map e515 with expected crc 2023-10-09T06:13:07.894 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:07 smithi064 ceph-mon[220044]: failed to encode map e515 with expected crc 2023-10-09T06:13:07.894 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:07 smithi064 ceph-mon[220044]: pgmap v1312: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s; 4572/256932 objects misplaced (1.779%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:13:07.894 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:07 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/711943364' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:13:07.896 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:13:07.935 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:13:07.943 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:13:07.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:07.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:13:07.974 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:13:08.014 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:13:08.052 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:13:08.060 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:13:08.191 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:13:08.192 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:13:08.192 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:13:08.193 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:13:08.193 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:13:08.193 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:13:08.194 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:13:08.194 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:13:08.194 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:13:08.194 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:08.224 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:13:08.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:08.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:13:08.336 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:13:08.349 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 74 16 87761 87745 74.0681 56.0625 0.0103459 0.0134962 2023-10-09T06:13:08.422 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:09.002 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:13:09.349 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 75 16 88376 88360 73.5932 38.4375 0.00914926 0.0135819 2023-10-09T06:13:09.861 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:13:09.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:09.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:13:09.905 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:13:09.905 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:13:09.970 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:13:10.058 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:10.165 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:13:10.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:10.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:13:10.200 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:10 smithi064 ceph-mon[212750]: pgmap v1313: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 36 MiB/s wr, 570 op/s; 4237/257640 objects misplaced (1.645%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T06:13:10.282 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:13:10.347 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:10.349 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 76 16 89100 89084 73.2204 45.25 0.0324576 0.0136521 2023-10-09T06:13:10.383 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:13:10.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:10 smithi064 ceph-mon[220044]: pgmap v1313: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 36 MiB/s wr, 570 op/s; 4237/257640 objects misplaced (1.645%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T06:13:10.477 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:13:10.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:10.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:13:10.594 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:13:10.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:10 smithi156 ceph-mon[195101]: pgmap v1313: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 36 MiB/s wr, 570 op/s; 4237/257640 objects misplaced (1.645%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T06:13:10.663 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:10.713 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:13:10.783 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:13:10.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:10.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:13:10.888 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:13:10.928 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:11.017 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:13:11.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:11.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:13:11.122 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:13:11.190 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:13:11.229 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:11 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/45895010' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:13:11.229 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:11 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:13:11.229 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:11 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/45895010' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:13:11.229 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:11 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:13:11.230 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:11.237 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T06:13:11.320 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:13:11.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:11.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:13:11.349 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 77 16 89934 89918 72.9465 52.125 0.00372868 0.0136958 2023-10-09T06:13:11.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:11 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/45895010' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:13:11.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:11 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:13:11.425 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:13:11.454 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:11.497 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:13:11.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:11.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:13:11.601 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:13:11.631 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:11.942 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:13:12.350 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 78 16 89944 89928 72.0197 0.625 0.00340622 0.0136947 2023-10-09T06:13:12.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:12 smithi064 ceph-mon[220044]: pgmap v1314: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 44 MiB/s wr, 705 op/s; 4237/266145 objects misplaced (1.592%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T06:13:12.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:12 smithi064 ceph-mon[220044]: failed to encode map e516 with expected crc 2023-10-09T06:13:12.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:12 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:13:12.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:12 smithi064 ceph-mon[220044]: osdmap e516: 8 total, 8 up, 8 in 2023-10-09T06:13:12.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:12 smithi064 ceph-mon[220044]: failed to encode map e516 with expected crc 2023-10-09T06:13:12.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:12 smithi064 ceph-mon[220044]: failed to encode map e516 with expected crc 2023-10-09T06:13:12.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:12 smithi064 ceph-mon[220044]: failed to encode map e516 with expected crc 2023-10-09T06:13:12.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:12 smithi064 ceph-mon[220044]: failed to encode map e516 with expected crc 2023-10-09T06:13:12.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:12 smithi064 ceph-mon[220044]: failed to encode map e516 with expected crc 2023-10-09T06:13:12.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:12 smithi064 ceph-mon[212750]: pgmap v1314: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 44 MiB/s wr, 705 op/s; 4237/266145 objects misplaced (1.592%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T06:13:12.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:12 smithi064 ceph-mon[212750]: failed to encode map e516 with expected crc 2023-10-09T06:13:12.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:12 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:13:12.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:12 smithi064 ceph-mon[212750]: osdmap e516: 8 total, 8 up, 8 in 2023-10-09T06:13:12.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:12 smithi064 ceph-mon[212750]: failed to encode map e516 with expected crc 2023-10-09T06:13:12.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:12 smithi064 ceph-mon[212750]: failed to encode map e516 with expected crc 2023-10-09T06:13:12.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:12 smithi064 ceph-mon[212750]: failed to encode map e516 with expected crc 2023-10-09T06:13:12.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:12 smithi064 ceph-mon[212750]: failed to encode map e516 with expected crc 2023-10-09T06:13:12.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:12 smithi064 ceph-mon[212750]: failed to encode map e516 with expected crc 2023-10-09T06:13:12.593 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:12 smithi156 ceph-mon[195101]: pgmap v1314: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 44 MiB/s wr, 705 op/s; 4237/266145 objects misplaced (1.592%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T06:13:12.593 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:12 smithi156 ceph-mon[195101]: failed to encode map e516 with expected crc 2023-10-09T06:13:12.594 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:12 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:13:12.594 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:12 smithi156 ceph-mon[195101]: osdmap e516: 8 total, 8 up, 8 in 2023-10-09T06:13:12.594 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:12 smithi156 ceph-mon[195101]: failed to encode map e516 with expected crc 2023-10-09T06:13:12.594 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:12 smithi156 ceph-mon[195101]: failed to encode map e516 with expected crc 2023-10-09T06:13:12.594 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:12 smithi156 ceph-mon[195101]: failed to encode map e516 with expected crc 2023-10-09T06:13:12.594 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:12 smithi156 ceph-mon[195101]: failed to encode map e516 with expected crc 2023-10-09T06:13:12.594 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:12 smithi156 ceph-mon[195101]: failed to encode map e516 with expected crc 2023-10-09T06:13:12.818 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:13:12.818 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:13:12.975 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T06:13:13.026 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:13:13.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:13.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:13:13.107 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:13:13.132 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:13:13.168 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:13.239 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:13:13.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:13.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:13:13.351 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 79 16 89959 89943 71.1194 0.9375 0.00929807 0.0138751 2023-10-09T06:13:13.359 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:13:13.407 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:13.445 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:13:13.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:13 smithi064 ceph-mon[220044]: failed to encode map e517 with expected crc 2023-10-09T06:13:13.447 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:13 smithi064 ceph-mon[220044]: failed to encode map e517 with expected crc 2023-10-09T06:13:13.447 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:13 smithi064 ceph-mon[220044]: osdmap e517: 8 total, 8 up, 8 in 2023-10-09T06:13:13.447 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:13 smithi064 ceph-mon[220044]: failed to encode map e517 with expected crc 2023-10-09T06:13:13.447 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:13 smithi064 ceph-mon[220044]: failed to encode map e517 with expected crc 2023-10-09T06:13:13.452 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:13 smithi064 ceph-mon[212750]: failed to encode map e517 with expected crc 2023-10-09T06:13:13.452 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:13 smithi064 ceph-mon[212750]: failed to encode map e517 with expected crc 2023-10-09T06:13:13.453 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:13 smithi064 ceph-mon[212750]: osdmap e517: 8 total, 8 up, 8 in 2023-10-09T06:13:13.453 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:13 smithi064 ceph-mon[212750]: failed to encode map e517 with expected crc 2023-10-09T06:13:13.453 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:13 smithi064 ceph-mon[212750]: failed to encode map e517 with expected crc 2023-10-09T06:13:13.480 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:13:13.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:13.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:13:13.589 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:13:13.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:13 smithi156 ceph-mon[195101]: failed to encode map e517 with expected crc 2023-10-09T06:13:13.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:13 smithi156 ceph-mon[195101]: failed to encode map e517 with expected crc 2023-10-09T06:13:13.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:13 smithi156 ceph-mon[195101]: osdmap e517: 8 total, 8 up, 8 in 2023-10-09T06:13:13.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:13 smithi156 ceph-mon[195101]: failed to encode map e517 with expected crc 2023-10-09T06:13:13.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:13 smithi156 ceph-mon[195101]: failed to encode map e517 with expected crc 2023-10-09T06:13:13.695 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:13.903 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:13:13.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:13.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:13:13.967 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:13:14.014 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:13:14.109 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:14.232 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:13:14.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:14.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:13:14.271 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:13:14.339 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:13:14.351 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-10-09T06:13:14.351801+0000 min lat: 0.0010971 max lat: 2.12813 avg lat: 0.0141049 2023-10-09T06:13:14.351 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:13:14.351 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 80 16 90707 90691 70.8148 46.75 0.00564992 0.0141049 2023-10-09T06:13:14.372 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:14.423 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:13:14.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:14.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:13:14.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:14 smithi064 ceph-mon[220044]: pgmap v1317: 119 pgs: 1 active+remapped+backfilling, 6 peering, 112 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 733 op/s; 3097/268692 objects misplaced (1.153%); 15 MiB/s, 240 objects/s recovering 2023-10-09T06:13:14.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:14 smithi064 ceph-mon[220044]: failed to encode map e518 with expected crc 2023-10-09T06:13:14.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:14 smithi064 ceph-mon[220044]: osdmap e518: 8 total, 8 up, 8 in 2023-10-09T06:13:14.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:14 smithi064 ceph-mon[220044]: failed to encode map e518 with expected crc 2023-10-09T06:13:14.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:14 smithi064 ceph-mon[220044]: failed to encode map e518 with expected crc 2023-10-09T06:13:14.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:14 smithi064 ceph-mon[220044]: failed to encode map e518 with expected crc 2023-10-09T06:13:14.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:14 smithi064 ceph-mon[220044]: failed to encode map e518 with expected crc 2023-10-09T06:13:14.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:14 smithi064 ceph-mon[220044]: 10.9 starting backfill to osd.5 from (0'0,0'0] MAX to 515'5566 2023-10-09T06:13:14.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:14 smithi064 ceph-mon[220044]: 10.7 starting backfill to osd.5 from (0'0,0'0] MAX to 515'5637 2023-10-09T06:13:14.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:14.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:14 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3317354917' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:13:14.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:14 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:13:14.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:14 smithi064 ceph-mon[212750]: pgmap v1317: 119 pgs: 1 active+remapped+backfilling, 6 peering, 112 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 733 op/s; 3097/268692 objects misplaced (1.153%); 15 MiB/s, 240 objects/s recovering 2023-10-09T06:13:14.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:14 smithi064 ceph-mon[212750]: failed to encode map e518 with expected crc 2023-10-09T06:13:14.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:14 smithi064 ceph-mon[212750]: osdmap e518: 8 total, 8 up, 8 in 2023-10-09T06:13:14.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:14 smithi064 ceph-mon[212750]: failed to encode map e518 with expected crc 2023-10-09T06:13:14.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:14 smithi064 ceph-mon[212750]: failed to encode map e518 with expected crc 2023-10-09T06:13:14.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:14 smithi064 ceph-mon[212750]: failed to encode map e518 with expected crc 2023-10-09T06:13:14.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:14 smithi064 ceph-mon[212750]: failed to encode map e518 with expected crc 2023-10-09T06:13:14.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:14 smithi064 ceph-mon[212750]: 10.9 starting backfill to osd.5 from (0'0,0'0] MAX to 515'5566 2023-10-09T06:13:14.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:14 smithi064 ceph-mon[212750]: 10.7 starting backfill to osd.5 from (0'0,0'0] MAX to 515'5637 2023-10-09T06:13:14.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:14.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:14 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3317354917' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:13:14.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:14 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:13:14.525 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:14 smithi156 ceph-mon[195101]: pgmap v1317: 119 pgs: 1 active+remapped+backfilling, 6 peering, 112 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 733 op/s; 3097/268692 objects misplaced (1.153%); 15 MiB/s, 240 objects/s recovering 2023-10-09T06:13:14.525 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:14 smithi156 ceph-mon[195101]: failed to encode map e518 with expected crc 2023-10-09T06:13:14.525 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:14 smithi156 ceph-mon[195101]: osdmap e518: 8 total, 8 up, 8 in 2023-10-09T06:13:14.525 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:14 smithi156 ceph-mon[195101]: failed to encode map e518 with expected crc 2023-10-09T06:13:14.526 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:14 smithi156 ceph-mon[195101]: failed to encode map e518 with expected crc 2023-10-09T06:13:14.526 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:14 smithi156 ceph-mon[195101]: failed to encode map e518 with expected crc 2023-10-09T06:13:14.526 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:14 smithi156 ceph-mon[195101]: failed to encode map e518 with expected crc 2023-10-09T06:13:14.526 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:14 smithi156 ceph-mon[195101]: 10.9 starting backfill to osd.5 from (0'0,0'0] MAX to 515'5566 2023-10-09T06:13:14.526 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:14 smithi156 ceph-mon[195101]: 10.7 starting backfill to osd.5 from (0'0,0'0] MAX to 515'5637 2023-10-09T06:13:14.526 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:14.527 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:14 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3317354917' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:13:14.527 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:14 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:13:14.527 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:13:14.579 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:14.669 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:13:14.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:14.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:13:14.775 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:13:14.830 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:14.900 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:13:14.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:14.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:13:15.006 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:13:15.032 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:15.075 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:13:15.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:15.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:13:15.178 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:13:15.279 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:15.351 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 81 16 91400 91384 70.4754 43.3125 0.0142957 0.0141703 2023-10-09T06:13:15.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:13:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:13:15] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.43.0" 2023-10-09T06:13:15.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:15 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:13:15.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:15 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T06:13:15.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:15 smithi064 ceph-mon[220044]: failed to encode map e519 with expected crc 2023-10-09T06:13:15.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:15 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:13:15.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:15 smithi064 ceph-mon[220044]: osdmap e519: 8 total, 8 up, 8 in 2023-10-09T06:13:15.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:15 smithi064 ceph-mon[220044]: failed to encode map e519 with expected crc 2023-10-09T06:13:15.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:15 smithi064 ceph-mon[220044]: failed to encode map e519 with expected crc 2023-10-09T06:13:15.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:15 smithi064 ceph-mon[220044]: failed to encode map e519 with expected crc 2023-10-09T06:13:15.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:15 smithi064 ceph-mon[220044]: failed to encode map e519 with expected crc 2023-10-09T06:13:15.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:15 smithi064 ceph-mon[220044]: osdmap e520: 8 total, 8 up, 8 in 2023-10-09T06:13:15.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:15 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:13:15.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:15 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:13:15.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:15 smithi064 ceph-mon[212750]: failed to encode map e519 with expected crc 2023-10-09T06:13:15.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:15 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:13:15.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:15 smithi064 ceph-mon[212750]: osdmap e519: 8 total, 8 up, 8 in 2023-10-09T06:13:15.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:15 smithi064 ceph-mon[212750]: failed to encode map e519 with expected crc 2023-10-09T06:13:15.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:15 smithi064 ceph-mon[212750]: failed to encode map e519 with expected crc 2023-10-09T06:13:15.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:15 smithi064 ceph-mon[212750]: failed to encode map e519 with expected crc 2023-10-09T06:13:15.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:15 smithi064 ceph-mon[212750]: failed to encode map e519 with expected crc 2023-10-09T06:13:15.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:15 smithi064 ceph-mon[212750]: osdmap e520: 8 total, 8 up, 8 in 2023-10-09T06:13:15.544 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:15 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:13:15.544 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:15 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T06:13:15.544 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:15 smithi156 ceph-mon[195101]: failed to encode map e519 with expected crc 2023-10-09T06:13:15.544 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:15 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:13:15.544 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:15 smithi156 ceph-mon[195101]: osdmap e519: 8 total, 8 up, 8 in 2023-10-09T06:13:15.544 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:15 smithi156 ceph-mon[195101]: failed to encode map e519 with expected crc 2023-10-09T06:13:15.544 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:15 smithi156 ceph-mon[195101]: failed to encode map e519 with expected crc 2023-10-09T06:13:15.545 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:15 smithi156 ceph-mon[195101]: failed to encode map e519 with expected crc 2023-10-09T06:13:15.545 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:15 smithi156 ceph-mon[195101]: failed to encode map e519 with expected crc 2023-10-09T06:13:15.545 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:15 smithi156 ceph-mon[195101]: osdmap e520: 8 total, 8 up, 8 in 2023-10-09T06:13:15.552 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:13:15.738 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:13:15.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:15.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:13:15.845 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:13:15.887 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:16.061 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:13:16.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:16.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:13:16.140 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:13:16.140 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:13:16.166 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:13:16.213 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:16.353 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 82 16 92221 92205 70.24 51.3125 0.0172295 0.0142313 2023-10-09T06:13:16.386 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:13:16.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:16.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:13:16.393 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:13:16.430 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:13:16.477 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:13:16.490 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:13:16.521 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:13:16.525 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:16 smithi156 ceph-mon[195101]: pgmap v1320: 119 pgs: 1 active+remapped+backfilling, 6 peering, 112 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 13 MiB/s wr, 212 op/s; 3097/268692 objects misplaced (1.153%); 19 MiB/s, 311 objects/s recovering 2023-10-09T06:13:16.525 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:16 smithi156 ceph-mon[195101]: failed to encode map e520 with expected crc 2023-10-09T06:13:16.525 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:16 smithi156 ceph-mon[195101]: failed to encode map e520 with expected crc 2023-10-09T06:13:16.526 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:16 smithi156 ceph-mon[195101]: failed to encode map e520 with expected crc 2023-10-09T06:13:16.528 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:16.570 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:13:16.591 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:13:16.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:16.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:13:16.607 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:13:16.654 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:13:16.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:16 smithi064 ceph-mon[212750]: pgmap v1320: 119 pgs: 1 active+remapped+backfilling, 6 peering, 112 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 13 MiB/s wr, 212 op/s; 3097/268692 objects misplaced (1.153%); 19 MiB/s, 311 objects/s recovering 2023-10-09T06:13:16.692 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:16 smithi064 ceph-mon[212750]: failed to encode map e520 with expected crc 2023-10-09T06:13:16.692 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:16 smithi064 ceph-mon[212750]: failed to encode map e520 with expected crc 2023-10-09T06:13:16.692 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:16 smithi064 ceph-mon[212750]: failed to encode map e520 with expected crc 2023-10-09T06:13:16.692 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:13:16.694 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:16 smithi064 ceph-mon[220044]: pgmap v1320: 119 pgs: 1 active+remapped+backfilling, 6 peering, 112 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 13 MiB/s wr, 212 op/s; 3097/268692 objects misplaced (1.153%); 19 MiB/s, 311 objects/s recovering 2023-10-09T06:13:16.694 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:16 smithi064 ceph-mon[220044]: failed to encode map e520 with expected crc 2023-10-09T06:13:16.694 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:16 smithi064 ceph-mon[220044]: failed to encode map e520 with expected crc 2023-10-09T06:13:16.694 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:16 smithi064 ceph-mon[220044]: failed to encode map e520 with expected crc 2023-10-09T06:13:16.694 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:13:16.695 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:13:16.695 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:13:16.695 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:13:16.695 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:13:16.695 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:13:16.695 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:13:16.695 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:13:16.698 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:13:16.724 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:16.777 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:13:16.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:16.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:13:16.881 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:13:16.971 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:17.226 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:13:17.353 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 83 16 93021 93005 69.9962 50 0.00349508 0.0142718 2023-10-09T06:13:17.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:17 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3287945550' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:13:17.673 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:13:17.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:17 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3287945550' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:13:17.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:17 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3287945550' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:13:17.887 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:13:17.952 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:13:17.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:17.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:13:18.063 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:13:18.097 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:18.263 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:13:18.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:18.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:13:18.354 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 84 16 93677 93661 69.6507 41 0.0193699 0.0143526 2023-10-09T06:13:18.367 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:13:18.394 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:18.557 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:13:18.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:18.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:13:18.662 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:13:18.733 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:18.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:18 smithi156 ceph-mon[195101]: pgmap v1322: 119 pgs: 1 active+remapped+backfill_wait, 3 activating, 2 active+remapped+backfilling, 113 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 29 MiB/s wr, 466 op/s; 13771/275103 objects misplaced (5.006%); 25 MiB/s, 401 objects/s recovering 2023-10-09T06:13:18.898 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:13:18.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:18.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:13:18.976 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:18 smithi064 ceph-mon[212750]: pgmap v1322: 119 pgs: 1 active+remapped+backfill_wait, 3 activating, 2 active+remapped+backfilling, 113 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 29 MiB/s wr, 466 op/s; 13771/275103 objects misplaced (5.006%); 25 MiB/s, 401 objects/s recovering 2023-10-09T06:13:18.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:18 smithi064 ceph-mon[220044]: pgmap v1322: 119 pgs: 1 active+remapped+backfill_wait, 3 activating, 2 active+remapped+backfilling, 113 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 29 MiB/s wr, 466 op/s; 13771/275103 objects misplaced (5.006%); 25 MiB/s, 401 objects/s recovering 2023-10-09T06:13:19.015 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:13:19.067 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:19.143 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:13:19.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:19.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:13:19.255 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:13:19.259 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:13:19.290 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:19.354 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 85 16 94345 94329 69.3225 41.75 0.0274643 0.0144195 2023-10-09T06:13:19.365 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:13:19.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:19.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:13:19.473 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:13:19.514 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:19.591 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:13:19.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:19.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:13:19.614 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:13:19.614 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:13:19.614 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T06:13:19.614 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T06:13:19.615 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:13:19.696 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:13:19.785 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:19.891 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:13:19.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:19.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:13:19.995 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:13:20.034 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:20.181 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:13:20.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:20.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:13:20.290 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:13:20.354 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 86 16 95000 94984 68.9926 40.9375 0.0114017 0.0144894 2023-10-09T06:13:20.447 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:13:20.542 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:20.588 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:13:20.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:20.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:13:20.678 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:13:20.691 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:13:20.697 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T06:13:20.725 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:20.771 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:13:20.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:20.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:13:20.875 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:13:20.884 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:13:20.909 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:20.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:20 smithi064 ceph-mon[212750]: pgmap v1323: 119 pgs: 1 active+remapped+backfill_wait, 3 activating, 2 active+remapped+backfilling, 113 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 25 MiB/s wr, 398 op/s; 13308/275862 objects misplaced (4.824%); 14 MiB/s, 218 objects/s recovering 2023-10-09T06:13:20.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:20 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1667260041' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:13:20.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:20 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:13:20.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:20 smithi064 ceph-mon[220044]: pgmap v1323: 119 pgs: 1 active+remapped+backfill_wait, 3 activating, 2 active+remapped+backfilling, 113 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 25 MiB/s wr, 398 op/s; 13308/275862 objects misplaced (4.824%); 14 MiB/s, 218 objects/s recovering 2023-10-09T06:13:20.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:20 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1667260041' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:13:20.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:20 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:13:20.973 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:13:20.976 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:20 smithi156 ceph-mon[195101]: pgmap v1323: 119 pgs: 1 active+remapped+backfill_wait, 3 activating, 2 active+remapped+backfilling, 113 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 25 MiB/s wr, 398 op/s; 13308/275862 objects misplaced (4.824%); 14 MiB/s, 218 objects/s recovering 2023-10-09T06:13:20.977 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:20 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1667260041' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:13:20.977 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:20 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:13:20.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:20.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:13:21.079 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:13:21.108 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:21.165 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:13:21.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:21.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:13:21.269 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:13:21.299 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:21.354 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 87 16 95216 95200 68.355 13.5 0.00330668 0.0145253 2023-10-09T06:13:21.942 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:13:21.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:21.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:13:21.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:21 smithi064 ceph-mon[212750]: failed to encode map e521 with expected crc 2023-10-09T06:13:21.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:21 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:13:21.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:21 smithi064 ceph-mon[212750]: osdmap e521: 8 total, 8 up, 8 in 2023-10-09T06:13:21.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:21 smithi064 ceph-mon[212750]: failed to encode map e521 with expected crc 2023-10-09T06:13:21.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:21 smithi064 ceph-mon[212750]: failed to encode map e521 with expected crc 2023-10-09T06:13:21.953 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:21 smithi064 ceph-mon[220044]: failed to encode map e521 with expected crc 2023-10-09T06:13:21.953 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:21 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:13:21.953 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:21 smithi064 ceph-mon[220044]: osdmap e521: 8 total, 8 up, 8 in 2023-10-09T06:13:21.953 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:21 smithi064 ceph-mon[220044]: failed to encode map e521 with expected crc 2023-10-09T06:13:21.954 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:21 smithi064 ceph-mon[220044]: failed to encode map e521 with expected crc 2023-10-09T06:13:22.026 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T06:13:22.047 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:13:22.106 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:22.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:21 smithi156 ceph-mon[195101]: failed to encode map e521 with expected crc 2023-10-09T06:13:22.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:21 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:13:22.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:21 smithi156 ceph-mon[195101]: osdmap e521: 8 total, 8 up, 8 in 2023-10-09T06:13:22.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:21 smithi156 ceph-mon[195101]: failed to encode map e521 with expected crc 2023-10-09T06:13:22.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:21 smithi156 ceph-mon[195101]: failed to encode map e521 with expected crc 2023-10-09T06:13:22.182 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:13:22.270 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:13:22.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:22.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:13:22.354 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 88 16 95224 95208 67.5842 0.5 0.00371695 0.0145368 2023-10-09T06:13:22.377 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:13:22.409 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:22.616 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:13:22.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:22.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:13:22.721 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:13:22.761 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:22.862 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:22 smithi064 ceph-mon[220044]: pgmap v1325: 119 pgs: 13 peering, 3 active+remapped+backfilling, 103 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 45 MiB/s wr, 724 op/s; 12924/283662 objects misplaced (4.556%); 19 MiB/s, 304 objects/s recovering 2023-10-09T06:13:22.862 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:22 smithi064 ceph-mon[220044]: failed to encode map e522 with expected crc 2023-10-09T06:13:22.863 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:22 smithi064 ceph-mon[220044]: failed to encode map e522 with expected crc 2023-10-09T06:13:22.863 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:22 smithi064 ceph-mon[220044]: failed to encode map e522 with expected crc 2023-10-09T06:13:22.863 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:22 smithi064 ceph-mon[220044]: failed to encode map e522 with expected crc 2023-10-09T06:13:22.863 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:22 smithi064 ceph-mon[220044]: osdmap e522: 8 total, 8 up, 8 in 2023-10-09T06:13:22.863 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:22 smithi064 ceph-mon[220044]: failed to encode map e522 with expected crc 2023-10-09T06:13:22.863 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:22 smithi064 ceph-mon[220044]: failed to encode map e522 with expected crc 2023-10-09T06:13:22.864 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:22 smithi064 ceph-mon[220044]: failed to encode map e522 with expected crc 2023-10-09T06:13:22.864 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:22 smithi064 ceph-mon[220044]: 10.7 continuing backfill to osd.5 from (498'1193,520'5980] 10:e23103f1:::benchmark_data_smithi064_573750_object60581:head to 520'5980 2023-10-09T06:13:22.864 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:22 smithi064 ceph-mon[220044]: 10.7 starting backfill to osd.6 from (0'0,0'0] MAX to 520'5980 2023-10-09T06:13:22.864 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:22 smithi064 ceph-mon[212750]: pgmap v1325: 119 pgs: 13 peering, 3 active+remapped+backfilling, 103 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 45 MiB/s wr, 724 op/s; 12924/283662 objects misplaced (4.556%); 19 MiB/s, 304 objects/s recovering 2023-10-09T06:13:22.864 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:22 smithi064 ceph-mon[212750]: failed to encode map e522 with expected crc 2023-10-09T06:13:22.864 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:22 smithi064 ceph-mon[212750]: failed to encode map e522 with expected crc 2023-10-09T06:13:22.864 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:22 smithi064 ceph-mon[212750]: failed to encode map e522 with expected crc 2023-10-09T06:13:22.864 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:22 smithi064 ceph-mon[212750]: failed to encode map e522 with expected crc 2023-10-09T06:13:22.864 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:22 smithi064 ceph-mon[212750]: osdmap e522: 8 total, 8 up, 8 in 2023-10-09T06:13:22.865 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:22 smithi064 ceph-mon[212750]: failed to encode map e522 with expected crc 2023-10-09T06:13:22.865 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:22 smithi064 ceph-mon[212750]: failed to encode map e522 with expected crc 2023-10-09T06:13:22.865 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:22 smithi064 ceph-mon[212750]: failed to encode map e522 with expected crc 2023-10-09T06:13:22.865 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:22 smithi064 ceph-mon[212750]: 10.7 continuing backfill to osd.5 from (498'1193,520'5980] 10:e23103f1:::benchmark_data_smithi064_573750_object60581:head to 520'5980 2023-10-09T06:13:22.865 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:22 smithi064 ceph-mon[212750]: 10.7 starting backfill to osd.6 from (0'0,0'0] MAX to 520'5980 2023-10-09T06:13:22.904 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:13:22.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:22.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:13:22.972 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:13:22.972 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T06:13:22.973 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T06:13:22.973 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:13:22.973 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.973 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:188353 8.9 8:9c34743c:::benchmark_data_smithi064_457003_object83835:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2023-10-09T06:13:22.973 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:27.431298+0000", 2023-10-09T06:13:22.973 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 415.536718114, 2023-10-09T06:13:22.973 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0494895609999999, 2023-10-09T06:13:22.973 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:22.974 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:13:22.974 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:13:22.974 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:13:22.974 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:13:22.974 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188353 2023-10-09T06:13:22.974 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.974 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:22.974 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.974 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:22.974 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.431298+0000", 2023-10-09T06:13:22.974 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:22.975 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.975 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.975 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:22.975 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.431298+0000", 2023-10-09T06:13:22.975 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:22.975 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.975 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.975 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:22.975 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.431296+0000", 2023-10-09T06:13:22.975 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:13:22.975 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.975 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.976 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:22.976 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.431305+0000", 2023-10-09T06:13:22.976 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.2279999999999998e-06 2023-10-09T06:13:22.976 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.976 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.976 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:22.976 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.431317+0000", 2023-10-09T06:13:22.976 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2605e-05 2023-10-09T06:13:22.976 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.976 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.976 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:22.976 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.431323+0000", 2023-10-09T06:13:22.977 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.178e-06 2023-10-09T06:13:22.977 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.977 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.977 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:22.977 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.431394+0000", 2023-10-09T06:13:22.977 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0308000000000005e-05 2023-10-09T06:13:22.977 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.977 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.977 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:22.977 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.431517+0000", 2023-10-09T06:13:22.977 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012297 2023-10-09T06:13:22.977 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.978 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.978 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T06:13:22.978 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.431637+0000", 2023-10-09T06:13:22.978 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000120615 2023-10-09T06:13:22.978 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.978 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.978 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:13:22.978 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.435725+0000", 2023-10-09T06:13:22.978 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0040881099999999998 2023-10-09T06:13:22.978 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.979 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.979 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:22.979 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.442248+0000", 2023-10-09T06:13:22.979 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0065223720000000002 2023-10-09T06:13:22.979 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.979 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.979 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:22.979 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.446429+0000", 2023-10-09T06:13:22.979 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0041808890000000001 2023-10-09T06:13:22.979 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.979 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.980 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:22.980 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.451740+0000", 2023-10-09T06:13:22.980 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0053108440000000003 2023-10-09T06:13:22.980 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.980 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.980 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:13:22.980 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.451749+0000", 2023-10-09T06:13:22.980 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2609999999999999e-06 2023-10-09T06:13:22.980 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.980 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.980 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:22.980 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.480704+0000", 2023-10-09T06:13:22.981 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0289548829999999 2023-10-09T06:13:22.981 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.981 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.981 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:13:22.981 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.480787+0000", 2023-10-09T06:13:22.981 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.3778999999999999e-05 2023-10-09T06:13:22.981 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:22.981 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:22.981 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:22.981 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.981 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.981 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:188357 8.9 8:9381b48d:::benchmark_data_smithi064_457003_object83839:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2023-10-09T06:13:22.982 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:27.437756+0000", 2023-10-09T06:13:22.982 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 415.53026013900001, 2023-10-09T06:13:22.982 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0435701239999999, 2023-10-09T06:13:22.982 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:22.982 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:13:22.982 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:13:22.982 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:13:22.982 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:13:22.982 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188357 2023-10-09T06:13:22.982 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.982 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:22.983 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.983 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:22.983 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.437756+0000", 2023-10-09T06:13:22.983 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:22.983 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.983 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.983 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:22.983 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.437756+0000", 2023-10-09T06:13:22.983 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:22.983 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.983 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.983 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:22.984 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.437754+0000", 2023-10-09T06:13:22.984 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:13:22.984 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.984 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.984 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:22.984 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.437761+0000", 2023-10-09T06:13:22.984 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0820000000000001e-06 2023-10-09T06:13:22.984 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.984 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.984 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:22.984 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.437774+0000", 2023-10-09T06:13:22.984 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2945000000000001e-05 2023-10-09T06:13:22.985 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.985 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.985 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:22.985 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.437781+0000", 2023-10-09T06:13:22.985 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4659999999999996e-06 2023-10-09T06:13:22.985 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.985 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.985 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:22.985 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.437824+0000", 2023-10-09T06:13:22.985 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3556000000000001e-05 2023-10-09T06:13:22.985 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.985 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.985 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:22.986 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.438012+0000", 2023-10-09T06:13:22.986 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00018795899999999999 2023-10-09T06:13:22.986 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.986 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.986 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T06:13:22.986 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.438176+0000", 2023-10-09T06:13:22.986 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00016331200000000001 2023-10-09T06:13:22.986 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.986 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.986 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:13:22.986 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.442461+0000", 2023-10-09T06:13:22.987 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0042849079999999996 2023-10-09T06:13:22.987 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.987 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.987 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:22.987 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.446454+0000", 2023-10-09T06:13:22.987 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0039938040000000001 2023-10-09T06:13:22.987 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.987 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.987 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:22.987 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.448508+0000", 2023-10-09T06:13:22.987 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0020538090000000002 2023-10-09T06:13:22.987 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.988 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.988 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:22.988 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.452359+0000", 2023-10-09T06:13:22.988 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0038505330000000002 2023-10-09T06:13:22.988 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.988 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.988 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:13:22.988 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.452366+0000", 2023-10-09T06:13:22.988 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.17e-06 2023-10-09T06:13:22.988 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.988 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.988 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:22.988 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.481280+0000", 2023-10-09T06:13:22.989 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.028914012 2023-10-09T06:13:22.989 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.989 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.989 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:13:22.989 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.481326+0000", 2023-10-09T06:13:22.989 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6041999999999997e-05 2023-10-09T06:13:22.989 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:22.989 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:22.989 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:22.989 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.989 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.989 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:188369 8.9 8:90ddf148:::benchmark_data_smithi064_457003_object83851:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2023-10-09T06:13:22.990 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:27.448204+0000", 2023-10-09T06:13:22.990 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 415.51981233999999, 2023-10-09T06:13:22.990 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0446369950000001, 2023-10-09T06:13:22.990 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:22.990 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:22.990 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:13:22.990 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:13:22.990 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:13:22.990 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188369 2023-10-09T06:13:22.990 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.990 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:22.991 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.991 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:22.991 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.448204+0000", 2023-10-09T06:13:22.991 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:22.991 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.991 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.991 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:22.991 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.448204+0000", 2023-10-09T06:13:22.991 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:22.991 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.991 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.991 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:22.992 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.448202+0000", 2023-10-09T06:13:22.992 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T06:13:22.992 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.992 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.992 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:22.992 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.448209+0000", 2023-10-09T06:13:22.992 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9380000000000003e-06 2023-10-09T06:13:22.992 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.993 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.993 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:22.993 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.448221+0000", 2023-10-09T06:13:22.993 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1702000000000001e-05 2023-10-09T06:13:22.993 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.993 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.993 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:22.993 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.448227+0000", 2023-10-09T06:13:22.993 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0209999999999996e-06 2023-10-09T06:13:22.994 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.994 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.994 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:22.994 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.448281+0000", 2023-10-09T06:13:22.994 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4097000000000001e-05 2023-10-09T06:13:22.994 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.994 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.995 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op must wait for map", 2023-10-09T06:13:22.995 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.448287+0000", 2023-10-09T06:13:22.995 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3960000000000003e-06 2023-10-09T06:13:22.995 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.995 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.995 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:22.995 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.452389+0000", 2023-10-09T06:13:22.995 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0041019660000000003 2023-10-09T06:13:22.996 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.996 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.996 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:13:22.996 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.452394+0000", 2023-10-09T06:13:22.996 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4439999999999999e-06 2023-10-09T06:13:22.996 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.996 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.996 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:22.997 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.481372+0000", 2023-10-09T06:13:22.997 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0289773689999999 2023-10-09T06:13:22.997 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.997 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.997 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:22.997 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.481499+0000", 2023-10-09T06:13:22.997 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012766700000000001 2023-10-09T06:13:22.997 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.997 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.997 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:13:22.997 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.481637+0000", 2023-10-09T06:13:22.998 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000137214 2023-10-09T06:13:22.998 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.998 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.998 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:22.998 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.489010+0000", 2023-10-09T06:13:22.998 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0073732950000000002 2023-10-09T06:13:22.998 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.999 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.999 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:22.999 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.489162+0000", 2023-10-09T06:13:22.999 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015227599999999999 2023-10-09T06:13:22.999 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:22.999 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:22.999 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:13:22.999 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.492763+0000", 2023-10-09T06:13:23.000 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0036004140000000001 2023-10-09T06:13:23.000 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.000 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.000 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:13:23.000 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.492798+0000", 2023-10-09T06:13:23.000 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5027999999999997e-05 2023-10-09T06:13:23.000 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.001 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.001 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:13:23.001 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.492841+0000", 2023-10-09T06:13:23.001 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2972000000000001e-05 2023-10-09T06:13:23.001 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.001 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:23.001 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.001 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.002 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.002 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:188370 8.9 8:9fe10030:::benchmark_data_smithi064_457003_object83852:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2023-10-09T06:13:23.002 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:27.449292+0000", 2023-10-09T06:13:23.002 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 415.51872380899999, 2023-10-09T06:13:23.002 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0437020880000001, 2023-10-09T06:13:23.002 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:23.002 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:23.003 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:13:23.003 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:13:23.003 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:13:23.003 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188370 2023-10-09T06:13:23.003 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.003 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:23.003 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.004 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:23.004 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.449292+0000", 2023-10-09T06:13:23.004 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.004 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.004 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.004 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:23.004 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.449292+0000", 2023-10-09T06:13:23.004 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.005 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.005 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.005 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:23.005 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.449291+0000", 2023-10-09T06:13:23.005 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:13:23.005 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.005 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.005 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:23.006 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.449295+0000", 2023-10-09T06:13:23.006 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9099999999999998e-06 2023-10-09T06:13:23.006 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.006 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.006 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:23.006 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.449306+0000", 2023-10-09T06:13:23.006 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0444e-05 2023-10-09T06:13:23.007 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.007 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.007 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:23.007 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.449310+0000", 2023-10-09T06:13:23.007 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3200000000000001e-06 2023-10-09T06:13:23.007 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.007 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.008 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.008 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.453072+0000", 2023-10-09T06:13:23.008 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0037617789999999998 2023-10-09T06:13:23.008 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.008 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.008 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:13:23.008 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.453079+0000", 2023-10-09T06:13:23.008 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5079999999999999e-06 2023-10-09T06:13:23.008 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.008 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.009 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.009 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.485497+0000", 2023-10-09T06:13:23.009 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.032417825 2023-10-09T06:13:23.009 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.009 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.009 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:23.009 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.485616+0000", 2023-10-09T06:13:23.009 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011836999999999999 2023-10-09T06:13:23.009 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.010 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.010 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:13:23.010 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.485734+0000", 2023-10-09T06:13:23.010 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000118118 2023-10-09T06:13:23.010 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.010 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.010 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:23.010 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.492756+0000", 2023-10-09T06:13:23.010 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0070217819999999999 2023-10-09T06:13:23.010 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.010 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.011 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:13:23.011 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.492849+0000", 2023-10-09T06:13:23.011 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3039000000000001e-05 2023-10-09T06:13:23.011 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.011 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.011 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:23.011 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.492901+0000", 2023-10-09T06:13:23.011 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1978999999999998e-05 2023-10-09T06:13:23.011 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.012 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.012 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:13:23.012 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.492953+0000", 2023-10-09T06:13:23.012 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1996000000000002e-05 2023-10-09T06:13:23.012 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.012 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.013 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:13:23.013 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.492994+0000", 2023-10-09T06:13:23.013 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1740999999999999e-05 2023-10-09T06:13:23.013 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.013 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:23.014 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.014 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.014 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.014 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:188393 8.9 8:94976dfa:::benchmark_data_smithi064_457003_object83875:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2023-10-09T06:13:23.014 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:27.473154+0000", 2023-10-09T06:13:23.014 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 415.49486177900002, 2023-10-09T06:13:23.014 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0258810519999999, 2023-10-09T06:13:23.015 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:23.015 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:23.015 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:13:23.015 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:13:23.015 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:13:23.015 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188393 2023-10-09T06:13:23.015 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.015 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:23.016 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.016 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:23.016 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.473154+0000", 2023-10-09T06:13:23.016 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.016 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.016 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.016 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:23.017 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.473154+0000", 2023-10-09T06:13:23.017 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.017 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.017 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.017 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:23.017 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.473152+0000", 2023-10-09T06:13:23.017 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T06:13:23.018 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.018 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.018 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:23.018 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.473161+0000", 2023-10-09T06:13:23.018 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.2179999999999999e-06 2023-10-09T06:13:23.018 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.018 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.018 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:23.018 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.473179+0000", 2023-10-09T06:13:23.019 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7932999999999999e-05 2023-10-09T06:13:23.019 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.019 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.019 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:23.019 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.473187+0000", 2023-10-09T06:13:23.019 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0579999999999998e-06 2023-10-09T06:13:23.019 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.019 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.019 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.020 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.473235+0000", 2023-10-09T06:13:23.020 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8350000000000003e-05 2023-10-09T06:13:23.020 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.020 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.020 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:13:23.020 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.473244+0000", 2023-10-09T06:13:23.020 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0510000000000002e-06 2023-10-09T06:13:23.020 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.020 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.020 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.020 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.489188+0000", 2023-10-09T06:13:23.020 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.015943974 2023-10-09T06:13:23.020 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.021 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.021 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:23.021 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.489307+0000", 2023-10-09T06:13:23.021 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000119138 2023-10-09T06:13:23.021 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.021 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.021 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:13:23.021 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.489434+0000", 2023-10-09T06:13:23.021 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000126995 2023-10-09T06:13:23.021 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.021 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.021 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:13:23.022 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.496145+0000", 2023-10-09T06:13:23.022 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0067113260000000001 2023-10-09T06:13:23.022 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.022 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.022 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:23.022 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.498903+0000", 2023-10-09T06:13:23.022 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0027579330000000002 2023-10-09T06:13:23.022 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.022 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.022 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:23.022 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.498950+0000", 2023-10-09T06:13:23.023 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6959999999999998e-05 2023-10-09T06:13:23.023 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.023 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.023 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:13:23.023 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.498997+0000", 2023-10-09T06:13:23.023 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6675000000000003e-05 2023-10-09T06:13:23.023 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.023 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.023 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:13:23.023 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.499035+0000", 2023-10-09T06:13:23.024 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8235000000000002e-05 2023-10-09T06:13:23.024 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.024 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:23.024 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.024 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.024 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.024 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:188403 8.9 8:9096b6fa:::benchmark_data_smithi064_457003_object83885:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2023-10-09T06:13:23.024 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:27.481397+0000", 2023-10-09T06:13:23.024 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 415.48661917200002, 2023-10-09T06:13:23.024 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.035516009, 2023-10-09T06:13:23.025 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:23.025 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:23.025 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:13:23.025 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:13:23.025 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:13:23.025 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188403 2023-10-09T06:13:23.025 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.025 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:23.025 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.025 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:23.025 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.481397+0000", 2023-10-09T06:13:23.025 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.025 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.026 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.026 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:23.026 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.481397+0000", 2023-10-09T06:13:23.026 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.026 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.026 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.026 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:23.026 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.481396+0000", 2023-10-09T06:13:23.026 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:13:23.026 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.026 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.027 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:23.027 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.481404+0000", 2023-10-09T06:13:23.027 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.7919999999999998e-06 2023-10-09T06:13:23.027 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.027 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.027 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:23.027 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.481416+0000", 2023-10-09T06:13:23.027 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1484000000000001e-05 2023-10-09T06:13:23.027 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.027 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.027 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:23.027 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.481421+0000", 2023-10-09T06:13:23.028 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0980000000000001e-06 2023-10-09T06:13:23.028 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.028 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.028 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.028 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.481462+0000", 2023-10-09T06:13:23.028 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1106000000000003e-05 2023-10-09T06:13:23.028 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.028 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.028 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:13:23.028 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.481471+0000", 2023-10-09T06:13:23.028 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0059999999999998e-06 2023-10-09T06:13:23.028 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.028 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.029 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.029 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.493032+0000", 2023-10-09T06:13:23.029 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0115604680000001 2023-10-09T06:13:23.029 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.029 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.029 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:23.029 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.493102+0000", 2023-10-09T06:13:23.029 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0866999999999998e-05 2023-10-09T06:13:23.029 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.029 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.029 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:13:23.030 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.493173+0000", 2023-10-09T06:13:23.030 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0990999999999998e-05 2023-10-09T06:13:23.030 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.030 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.030 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:13:23.030 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.499044+0000", 2023-10-09T06:13:23.030 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.005870269 2023-10-09T06:13:23.030 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.030 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.030 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:23.030 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.504577+0000", 2023-10-09T06:13:23.031 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0055330509999999998 2023-10-09T06:13:23.031 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.031 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.031 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:23.031 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.516846+0000", 2023-10-09T06:13:23.031 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012268891000000001 2023-10-09T06:13:23.031 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.031 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.031 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:13:23.031 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.516887+0000", 2023-10-09T06:13:23.031 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1427000000000002e-05 2023-10-09T06:13:23.032 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.032 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.032 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:13:23.032 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.516913+0000", 2023-10-09T06:13:23.032 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.5761e-05 2023-10-09T06:13:23.032 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.032 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:23.032 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.032 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.032 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.032 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:188411 8.9 8:9b524035:::benchmark_data_smithi064_457003_object83893:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2023-10-09T06:13:23.032 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:27.488449+0000", 2023-10-09T06:13:23.032 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 415.479566981, 2023-10-09T06:13:23.033 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0412856909999999, 2023-10-09T06:13:23.033 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:23.033 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:23.033 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:13:23.033 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:13:23.033 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:13:23.033 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188411 2023-10-09T06:13:23.033 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.033 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:23.033 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.033 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:23.033 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.488449+0000", 2023-10-09T06:13:23.034 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.034 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.034 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.034 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:23.034 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.488449+0000", 2023-10-09T06:13:23.034 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.034 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.034 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.034 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:23.034 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.488448+0000", 2023-10-09T06:13:23.034 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:13:23.035 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.035 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.035 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:23.035 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.488454+0000", 2023-10-09T06:13:23.035 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9839999999999998e-06 2023-10-09T06:13:23.035 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.035 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.035 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:23.035 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.488469+0000", 2023-10-09T06:13:23.035 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4807000000000001e-05 2023-10-09T06:13:23.035 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.035 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.036 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:23.036 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.488475+0000", 2023-10-09T06:13:23.036 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3670000000000002e-06 2023-10-09T06:13:23.036 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.036 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.036 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.036 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.488537+0000", 2023-10-09T06:13:23.036 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2039999999999996e-05 2023-10-09T06:13:23.036 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.037 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.037 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:13:23.037 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.488544+0000", 2023-10-09T06:13:23.037 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4590000000000001e-06 2023-10-09T06:13:23.037 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.037 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.037 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.037 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.496186+0000", 2023-10-09T06:13:23.037 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0076412109999999 2023-10-09T06:13:23.037 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.037 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.037 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:23.038 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.496275+0000", 2023-10-09T06:13:23.038 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9154999999999996e-05 2023-10-09T06:13:23.038 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.038 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.038 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:13:23.038 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.496350+0000", 2023-10-09T06:13:23.038 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4803999999999997e-05 2023-10-09T06:13:23.038 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.038 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.038 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:13:23.038 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.501737+0000", 2023-10-09T06:13:23.039 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0053872169999999997 2023-10-09T06:13:23.039 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.039 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.039 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:23.039 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.504624+0000", 2023-10-09T06:13:23.039 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0028874769999999998 2023-10-09T06:13:23.039 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.039 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.039 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:23.039 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.529656+0000", 2023-10-09T06:13:23.039 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.025032085999999999 2023-10-09T06:13:23.039 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.040 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.040 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:13:23.040 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.529691+0000", 2023-10-09T06:13:23.040 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5105999999999999e-05 2023-10-09T06:13:23.040 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.040 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.040 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:13:23.040 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.529735+0000", 2023-10-09T06:13:23.040 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3306999999999997e-05 2023-10-09T06:13:23.040 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.040 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:23.040 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.041 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.041 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.041 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.45605.0:188454 8.9 8:984ccfac:::benchmark_data_smithi064_457003_object83936:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2023-10-09T06:13:23.041 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:06:27.532247+0000", 2023-10-09T06:13:23.041 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 415.43576867500002, 2023-10-09T06:13:23.041 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.004500784, 2023-10-09T06:13:23.041 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:23.041 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:23.041 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:13:23.041 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.45605", 2023-10-09T06:13:23.041 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/4076964113", 2023-10-09T06:13:23.041 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 188454 2023-10-09T06:13:23.042 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.042 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:23.042 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.042 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:23.042 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.532247+0000", 2023-10-09T06:13:23.042 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.042 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.042 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.042 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:23.042 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.532247+0000", 2023-10-09T06:13:23.042 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.043 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.043 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.043 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:23.043 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.532246+0000", 2023-10-09T06:13:23.043 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:13:23.043 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.043 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.043 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:23.043 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.532252+0000", 2023-10-09T06:13:23.043 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7440000000000004e-06 2023-10-09T06:13:23.043 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.043 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.043 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:23.044 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.532265+0000", 2023-10-09T06:13:23.044 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2741e-05 2023-10-09T06:13:23.044 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.044 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.044 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:23.044 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.532270+0000", 2023-10-09T06:13:23.044 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9080000000000003e-06 2023-10-09T06:13:23.044 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.044 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.044 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.044 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.532315+0000", 2023-10-09T06:13:23.044 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5488999999999998e-05 2023-10-09T06:13:23.045 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.045 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.045 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:13:23.045 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:27.532323+0000", 2023-10-09T06:13:23.045 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.413e-06 2023-10-09T06:13:23.045 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.045 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.045 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.045 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.501780+0000", 2023-10-09T06:13:23.045 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96945757300000002 2023-10-09T06:13:23.046 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.046 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.046 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:23.046 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.501860+0000", 2023-10-09T06:13:23.046 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0241999999999995e-05 2023-10-09T06:13:23.046 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.046 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.046 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:13:23.046 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.501950+0000", 2023-10-09T06:13:23.047 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9709000000000005e-05 2023-10-09T06:13:23.047 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.047 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.047 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:13:23.047 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.516761+0000", 2023-10-09T06:13:23.047 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.014810886000000001 2023-10-09T06:13:23.047 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.047 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.047 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:23.047 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.529956+0000", 2023-10-09T06:13:23.047 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013195098000000001 2023-10-09T06:13:23.047 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.048 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.048 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:23.048 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.536665+0000", 2023-10-09T06:13:23.048 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0067089239999999998 2023-10-09T06:13:23.048 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.048 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.048 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:13:23.048 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.536711+0000", 2023-10-09T06:13:23.048 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5556999999999998e-05 2023-10-09T06:13:23.048 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.048 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.048 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:13:23.049 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:06:28.536748+0000", 2023-10-09T06:13:23.049 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7622e-05 2023-10-09T06:13:23.049 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.049 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:23.049 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.049 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.049 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.049 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:132905 9.b 9:dd1a3f80:::benchmark_data_smithi064_511911_object31803:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e455)", 2023-10-09T06:13:23.049 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:26.571593+0000", 2023-10-09T06:13:23.049 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 236.39642327600001, 2023-10-09T06:13:23.050 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.150692598, 2023-10-09T06:13:23.050 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:23.050 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:13:23.050 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:13:23.050 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:13:23.050 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:13:23.050 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 132905 2023-10-09T06:13:23.050 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.050 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:23.050 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.050 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:23.051 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.571593+0000", 2023-10-09T06:13:23.051 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.051 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.051 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.051 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:23.051 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.571593+0000", 2023-10-09T06:13:23.051 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.051 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.051 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.051 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:23.051 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.571591+0000", 2023-10-09T06:13:23.051 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T06:13:23.052 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.052 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.052 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:23.052 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.571602+0000", 2023-10-09T06:13:23.052 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0804e-05 2023-10-09T06:13:23.052 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.052 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.052 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:23.052 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.571620+0000", 2023-10-09T06:13:23.052 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7983999999999999e-05 2023-10-09T06:13:23.052 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.053 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.053 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:23.053 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.571628+0000", 2023-10-09T06:13:23.053 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.909e-06 2023-10-09T06:13:23.053 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.053 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.053 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.053 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572040+0000", 2023-10-09T06:13:23.053 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00041241599999999998 2023-10-09T06:13:23.053 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.053 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.053 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:23.054 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572178+0000", 2023-10-09T06:13:23.054 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000137589 2023-10-09T06:13:23.054 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.054 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.054 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5", 2023-10-09T06:13:23.054 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572302+0000", 2023-10-09T06:13:23.054 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000124597 2023-10-09T06:13:23.055 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.055 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.055 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:13:23.055 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.584526+0000", 2023-10-09T06:13:23.055 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012223982 2023-10-09T06:13:23.055 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.055 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.056 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.056 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.593255+0000", 2023-10-09T06:13:23.056 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0087291759999999999 2023-10-09T06:13:23.056 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.056 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.056 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:13:23.057 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.593267+0000", 2023-10-09T06:13:23.057 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1508999999999999e-05 2023-10-09T06:13:23.057 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.057 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.057 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.058 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.548189+0000", 2023-10-09T06:13:23.058 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.95492167299999997 2023-10-09T06:13:23.058 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.058 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.058 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:13:23.058 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.548200+0000", 2023-10-09T06:13:23.059 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1698e-05 2023-10-09T06:13:23.059 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.059 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.059 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.059 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:28.701140+0000", 2023-10-09T06:13:23.059 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1529395469999999 2023-10-09T06:13:23.059 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.060 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.060 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for missing object", 2023-10-09T06:13:23.060 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:28.701266+0000", 2023-10-09T06:13:23.060 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012610400000000001 2023-10-09T06:13:23.060 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.060 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.061 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.061 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:28.722202+0000", 2023-10-09T06:13:23.061 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.020936481999999999 2023-10-09T06:13:23.061 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.061 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.061 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:13:23.062 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:28.722285+0000", 2023-10-09T06:13:23.062 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.2952000000000001e-05 2023-10-09T06:13:23.062 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.062 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:23.062 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.062 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.063 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.063 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:132907 9.b 9:dbad7df3:::benchmark_data_smithi064_511911_object31805:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e455)", 2023-10-09T06:13:23.063 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:26.571642+0000", 2023-10-09T06:13:23.063 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 236.396373711, 2023-10-09T06:13:23.063 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1540348549999999, 2023-10-09T06:13:23.063 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:23.063 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:13:23.063 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:13:23.063 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:13:23.064 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:13:23.064 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 132907 2023-10-09T06:13:23.064 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.064 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:23.064 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.064 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:23.064 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.571642+0000", 2023-10-09T06:13:23.064 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.064 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.064 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.064 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:23.064 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.571642+0000", 2023-10-09T06:13:23.065 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.065 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.065 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.065 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:23.065 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.571642+0000", 2023-10-09T06:13:23.065 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:13:23.065 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.065 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.065 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:23.065 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.571645+0000", 2023-10-09T06:13:23.065 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.903e-06 2023-10-09T06:13:23.065 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.066 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.066 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:23.066 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.571651+0000", 2023-10-09T06:13:23.066 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.959e-06 2023-10-09T06:13:23.066 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.066 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.066 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:23.066 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.571655+0000", 2023-10-09T06:13:23.066 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8759999999999998e-06 2023-10-09T06:13:23.066 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.066 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.067 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.067 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.576676+0000", 2023-10-09T06:13:23.067 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.005021378 2023-10-09T06:13:23.067 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.067 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.067 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:23.067 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.576799+0000", 2023-10-09T06:13:23.067 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012273899999999999 2023-10-09T06:13:23.067 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.067 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.067 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5", 2023-10-09T06:13:23.068 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.576912+0000", 2023-10-09T06:13:23.068 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000113755 2023-10-09T06:13:23.068 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.068 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.068 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:13:23.068 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.584534+0000", 2023-10-09T06:13:23.068 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0076210460000000002 2023-10-09T06:13:23.068 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.068 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.068 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.068 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.593408+0000", 2023-10-09T06:13:23.069 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0088745109999999999 2023-10-09T06:13:23.069 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.069 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.069 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:13:23.069 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.593418+0000", 2023-10-09T06:13:23.069 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.8689999999999999e-06 2023-10-09T06:13:23.069 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.069 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.069 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.069 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.548227+0000", 2023-10-09T06:13:23.069 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.95480958299999996 2023-10-09T06:13:23.069 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.070 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.070 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:13:23.070 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.548234+0000", 2023-10-09T06:13:23.070 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9460000000000001e-06 2023-10-09T06:13:23.070 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.070 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.070 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.070 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:28.702287+0000", 2023-10-09T06:13:23.070 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1540522229999999 2023-10-09T06:13:23.070 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.071 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.071 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for missing object", 2023-10-09T06:13:23.071 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:28.702363+0000", 2023-10-09T06:13:23.071 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6505e-05 2023-10-09T06:13:23.071 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.071 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.071 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.071 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:28.725566+0000", 2023-10-09T06:13:23.071 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.023202725 2023-10-09T06:13:23.071 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.071 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.072 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:13:23.072 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:28.725677+0000", 2023-10-09T06:13:23.072 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000111301 2023-10-09T06:13:23.072 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.072 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:23.072 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.072 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.073 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.073 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:132918 9.b 9:ddad097d:::benchmark_data_smithi064_511911_object31816:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e456)", 2023-10-09T06:13:23.073 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:26.631527+0000", 2023-10-09T06:13:23.073 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 236.336489266, 2023-10-09T06:13:23.073 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0917619850000002, 2023-10-09T06:13:23.073 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:23.073 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:23.073 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:13:23.073 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:13:23.073 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:13:23.073 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 132918 2023-10-09T06:13:23.074 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.074 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:23.074 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.074 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:23.074 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.631527+0000", 2023-10-09T06:13:23.074 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.074 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.074 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.074 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:23.074 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.631527+0000", 2023-10-09T06:13:23.074 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.074 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.075 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.075 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:23.075 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.631526+0000", 2023-10-09T06:13:23.075 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:13:23.075 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.075 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.075 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:23.075 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.631531+0000", 2023-10-09T06:13:23.075 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7729999999999996e-06 2023-10-09T06:13:23.075 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.075 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.075 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:23.076 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.631547+0000", 2023-10-09T06:13:23.076 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5903e-05 2023-10-09T06:13:23.076 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.076 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.076 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:23.076 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.631553+0000", 2023-10-09T06:13:23.076 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9270000000000002e-06 2023-10-09T06:13:23.076 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.076 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.076 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.076 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.631605+0000", 2023-10-09T06:13:23.076 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1641000000000002e-05 2023-10-09T06:13:23.077 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.077 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.077 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:13:23.077 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.631616+0000", 2023-10-09T06:13:23.077 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1022000000000001e-05 2023-10-09T06:13:23.077 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.077 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.077 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.077 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.548254+0000", 2023-10-09T06:13:23.077 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.916638481 2023-10-09T06:13:23.077 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.077 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.078 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:13:23.078 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.548258+0000", 2023-10-09T06:13:23.078 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.002e-06 2023-10-09T06:13:23.078 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.078 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.078 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.078 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:28.702409+0000", 2023-10-09T06:13:23.078 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1541502720000001 2023-10-09T06:13:23.078 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.078 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.079 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:23.079 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:28.702508+0000", 2023-10-09T06:13:23.079 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.9251000000000002e-05 2023-10-09T06:13:23.079 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.079 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.079 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:13:23.079 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:28.702596+0000", 2023-10-09T06:13:23.079 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.7676000000000006e-05 2023-10-09T06:13:23.079 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.079 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.080 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:23.080 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:28.710893+0000", 2023-10-09T06:13:23.080 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0082968790000000001 2023-10-09T06:13:23.080 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.080 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.080 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:13:23.080 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:28.715243+0000", 2023-10-09T06:13:23.080 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0043502300000000001 2023-10-09T06:13:23.080 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.080 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.081 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:23.081 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:28.722314+0000", 2023-10-09T06:13:23.081 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0070711350000000001 2023-10-09T06:13:23.081 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.081 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.081 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:13:23.081 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:28.722346+0000", 2023-10-09T06:13:23.081 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.2308000000000003e-05 2023-10-09T06:13:23.081 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.081 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.081 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:13:23.082 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:28.723289+0000", 2023-10-09T06:13:23.082 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00094254099999999997 2023-10-09T06:13:23.082 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.082 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:23.082 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.082 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.082 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.082 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:132922 9.b 9:d1a81dc6:::benchmark_data_smithi064_511911_object31820:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e456)", 2023-10-09T06:13:23.082 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:26.645122+0000", 2023-10-09T06:13:23.083 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 236.32289414300001, 2023-10-09T06:13:23.083 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0790063299999999, 2023-10-09T06:13:23.083 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:23.083 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:23.083 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:13:23.083 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:13:23.083 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:13:23.084 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 132922 2023-10-09T06:13:23.084 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.084 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:23.084 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.084 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:23.084 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.645122+0000", 2023-10-09T06:13:23.084 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.085 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.085 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.085 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:23.085 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.645122+0000", 2023-10-09T06:13:23.085 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.085 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.086 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.086 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:23.086 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.645121+0000", 2023-10-09T06:13:23.086 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:13:23.086 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.086 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.087 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:23.087 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.645130+0000", 2023-10-09T06:13:23.087 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1470000000000007e-06 2023-10-09T06:13:23.087 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.087 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.087 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:23.088 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.645145+0000", 2023-10-09T06:13:23.088 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5688e-05 2023-10-09T06:13:23.088 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.088 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.088 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:23.088 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.645151+0000", 2023-10-09T06:13:23.088 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.947e-06 2023-10-09T06:13:23.089 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.089 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.089 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.089 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.645213+0000", 2023-10-09T06:13:23.089 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1291000000000007e-05 2023-10-09T06:13:23.089 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.090 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.090 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:13:23.090 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.645222+0000", 2023-10-09T06:13:23.090 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7289999999999995e-06 2023-10-09T06:13:23.090 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.090 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.091 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.091 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.548281+0000", 2023-10-09T06:13:23.091 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.90305871400000004 2023-10-09T06:13:23.091 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.091 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.092 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:13:23.092 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.548288+0000", 2023-10-09T06:13:23.092 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1080000000000004e-06 2023-10-09T06:13:23.092 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.092 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.092 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.092 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:28.706184+0000", 2023-10-09T06:13:23.092 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.157896048 2023-10-09T06:13:23.093 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.093 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.093 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:23.093 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:28.706311+0000", 2023-10-09T06:13:23.093 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000126785 2023-10-09T06:13:23.093 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.093 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.093 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:13:23.093 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:28.706408+0000", 2023-10-09T06:13:23.093 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.6873000000000006e-05 2023-10-09T06:13:23.093 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.094 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.094 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:13:23.094 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:28.715254+0000", 2023-10-09T06:13:23.094 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0088463940000000005 2023-10-09T06:13:23.094 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.094 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.094 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:23.094 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:28.719236+0000", 2023-10-09T06:13:23.094 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0039821869999999999 2023-10-09T06:13:23.094 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.094 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.095 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:23.095 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:28.723400+0000", 2023-10-09T06:13:23.095 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0041633929999999996 2023-10-09T06:13:23.095 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.095 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.095 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:13:23.095 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:28.723436+0000", 2023-10-09T06:13:23.095 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5877000000000002e-05 2023-10-09T06:13:23.095 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.095 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.095 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:13:23.095 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:28.724128+0000", 2023-10-09T06:13:23.096 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00069251600000000001 2023-10-09T06:13:23.096 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.096 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:23.096 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.096 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.096 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.096 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:182287 9.b 9:ddfaf3ee:::benchmark_data_smithi064_511911_object81185:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T06:13:23.096 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:10:49.739834+0000", 2023-10-09T06:13:23.096 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 153.22818208999999, 2023-10-09T06:13:23.096 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0692159409999999, 2023-10-09T06:13:23.096 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:23.096 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:23.097 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:13:23.097 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:13:23.097 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:13:23.097 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 182287 2023-10-09T06:13:23.097 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.097 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:23.097 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.097 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:23.097 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.739834+0000", 2023-10-09T06:13:23.097 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.097 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.097 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.098 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:23.098 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.739834+0000", 2023-10-09T06:13:23.098 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.098 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.098 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.098 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:23.098 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.739833+0000", 2023-10-09T06:13:23.098 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:13:23.098 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.098 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.098 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:23.099 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.739838+0000", 2023-10-09T06:13:23.099 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1120000000000004e-06 2023-10-09T06:13:23.099 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.099 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.099 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:23.099 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.739847+0000", 2023-10-09T06:13:23.099 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1349999999999998e-06 2023-10-09T06:13:23.099 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.099 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.099 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:23.099 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.739853+0000", 2023-10-09T06:13:23.100 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1639999999999998e-06 2023-10-09T06:13:23.100 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.100 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.100 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.100 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.743061+0000", 2023-10-09T06:13:23.100 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0032083229999999999 2023-10-09T06:13:23.100 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.100 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.100 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:13:23.100 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.743073+0000", 2023-10-09T06:13:23.100 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2092e-05 2023-10-09T06:13:23.100 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.101 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.101 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.101 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.791644+0000", 2023-10-09T06:13:23.101 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0485706990000001 2023-10-09T06:13:23.101 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.101 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.101 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:23.101 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.791803+0000", 2023-10-09T06:13:23.101 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015846600000000001 2023-10-09T06:13:23.101 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.101 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.102 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:13:23.102 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.791940+0000", 2023-10-09T06:13:23.102 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013713000000000001 2023-10-09T06:13:23.102 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.102 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.102 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:13:23.102 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.803203+0000", 2023-10-09T06:13:23.102 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011263451000000001 2023-10-09T06:13:23.102 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.102 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.102 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:23.103 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.807914+0000", 2023-10-09T06:13:23.103 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0047103630000000004 2023-10-09T06:13:23.103 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.103 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.103 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:23.103 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.807982+0000", 2023-10-09T06:13:23.103 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8454999999999994e-05 2023-10-09T06:13:23.103 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.103 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.103 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:13:23.103 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.808045+0000", 2023-10-09T06:13:23.105 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2638000000000006e-05 2023-10-09T06:13:23.105 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.105 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.105 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:13:23.105 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.809050+0000", 2023-10-09T06:13:23.105 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0010052349999999999 2023-10-09T06:13:23.105 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.105 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:23.105 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.106 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.106 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.106 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:182292 9.b 9:dc1a8787:::benchmark_data_smithi064_511911_object81190:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T06:13:23.106 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:10:49.753105+0000", 2023-10-09T06:13:23.106 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 153.21491112000001, 2023-10-09T06:13:23.106 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.072077706, 2023-10-09T06:13:23.106 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:23.106 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:23.106 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:13:23.106 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:13:23.106 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:13:23.107 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 182292 2023-10-09T06:13:23.107 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.107 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:23.107 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.107 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:23.107 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.753105+0000", 2023-10-09T06:13:23.107 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.107 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.107 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.107 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:23.107 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.753105+0000", 2023-10-09T06:13:23.107 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.108 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.108 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.108 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:23.108 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.753103+0000", 2023-10-09T06:13:23.108 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:13:23.108 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.108 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.108 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:23.108 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.753110+0000", 2023-10-09T06:13:23.108 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6379999999999998e-06 2023-10-09T06:13:23.108 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.109 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.109 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:23.109 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.753123+0000", 2023-10-09T06:13:23.109 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2986999999999999e-05 2023-10-09T06:13:23.109 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.109 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.109 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:23.109 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.753130+0000", 2023-10-09T06:13:23.109 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0899999999999999e-06 2023-10-09T06:13:23.109 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.109 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.109 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.110 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.753584+0000", 2023-10-09T06:13:23.110 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00045353800000000002 2023-10-09T06:13:23.110 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.110 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.110 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:13:23.110 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.753594+0000", 2023-10-09T06:13:23.110 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0025e-05 2023-10-09T06:13:23.110 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.110 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.110 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.110 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.798330+0000", 2023-10-09T06:13:23.111 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.044735881 2023-10-09T06:13:23.111 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.111 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.111 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:23.111 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.798467+0000", 2023-10-09T06:13:23.111 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000137341 2023-10-09T06:13:23.111 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.111 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.111 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:13:23.111 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.798603+0000", 2023-10-09T06:13:23.111 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013654499999999999 2023-10-09T06:13:23.112 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.112 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.112 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:13:23.112 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.809190+0000", 2023-10-09T06:13:23.112 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0105863 2023-10-09T06:13:23.112 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.112 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.112 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:23.112 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.809228+0000", 2023-10-09T06:13:23.112 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8352000000000001e-05 2023-10-09T06:13:23.112 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.112 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.112 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:23.113 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.824086+0000", 2023-10-09T06:13:23.113 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.014858356 2023-10-09T06:13:23.113 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.113 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.113 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:13:23.113 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.824141+0000", 2023-10-09T06:13:23.113 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4466999999999999e-05 2023-10-09T06:13:23.113 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.113 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.113 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:13:23.113 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.825183+0000", 2023-10-09T06:13:23.113 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0010417 2023-10-09T06:13:23.114 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.114 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:23.114 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.114 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.114 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.114 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:182294 9.b 9:d6e67af7:::benchmark_data_smithi064_511911_object81192:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T06:13:23.114 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:10:49.757688+0000", 2023-10-09T06:13:23.114 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 153.21032772000001, 2023-10-09T06:13:23.114 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.068185304, 2023-10-09T06:13:23.114 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:23.114 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:23.115 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:13:23.115 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:13:23.115 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:13:23.115 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 182294 2023-10-09T06:13:23.115 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.115 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:23.115 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.115 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:23.115 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.757688+0000", 2023-10-09T06:13:23.115 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.115 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.115 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.115 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:23.116 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.757688+0000", 2023-10-09T06:13:23.116 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.116 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.116 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.116 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:23.116 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.757687+0000", 2023-10-09T06:13:23.116 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T06:13:23.116 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.116 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.116 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:23.116 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.757698+0000", 2023-10-09T06:13:23.116 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1630999999999999e-05 2023-10-09T06:13:23.117 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.117 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.117 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:23.117 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.757717+0000", 2023-10-09T06:13:23.117 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8512000000000002e-05 2023-10-09T06:13:23.117 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.117 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.117 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:23.117 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.757724+0000", 2023-10-09T06:13:23.117 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0140000000000001e-06 2023-10-09T06:13:23.117 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.118 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.118 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.118 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.757787+0000", 2023-10-09T06:13:23.118 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3429000000000005e-05 2023-10-09T06:13:23.118 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.118 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.118 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:13:23.118 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.757800+0000", 2023-10-09T06:13:23.118 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2876e-05 2023-10-09T06:13:23.118 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.118 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.119 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.119 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.803261+0000", 2023-10-09T06:13:23.119 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0454610019999999 2023-10-09T06:13:23.119 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.119 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.119 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:23.119 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.803436+0000", 2023-10-09T06:13:23.119 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000174768 2023-10-09T06:13:23.119 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.119 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.119 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:13:23.119 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.803598+0000", 2023-10-09T06:13:23.120 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000162452 2023-10-09T06:13:23.120 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.120 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.120 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:23.120 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.809288+0000", 2023-10-09T06:13:23.120 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.005689805 2023-10-09T06:13:23.120 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.120 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.120 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:13:23.120 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.809301+0000", 2023-10-09T06:13:23.121 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3229e-05 2023-10-09T06:13:23.121 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.121 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.121 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:23.121 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.825263+0000", 2023-10-09T06:13:23.121 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015961921 2023-10-09T06:13:23.121 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.121 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.121 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:13:23.121 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.825310+0000", 2023-10-09T06:13:23.121 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6968000000000002e-05 2023-10-09T06:13:23.121 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.122 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.122 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:13:23.122 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.825874+0000", 2023-10-09T06:13:23.122 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00056341400000000004 2023-10-09T06:13:23.122 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.122 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:23.122 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.122 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.122 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.122 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:182320 9.b 9:d7a47e95:::benchmark_data_smithi064_511911_object81218:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T06:13:23.122 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:10:49.824803+0000", 2023-10-09T06:13:23.123 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 153.143212871, 2023-10-09T06:13:23.123 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.001582857, 2023-10-09T06:13:23.123 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:23.123 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:23.123 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:13:23.123 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:13:23.123 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:13:23.123 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 182320 2023-10-09T06:13:23.123 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.123 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:23.123 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.123 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:23.124 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.824803+0000", 2023-10-09T06:13:23.124 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.124 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.124 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.124 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:23.124 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.824803+0000", 2023-10-09T06:13:23.124 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.124 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.124 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.124 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:23.125 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.824802+0000", 2023-10-09T06:13:23.125 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:13:23.125 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.125 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.125 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:23.125 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.824810+0000", 2023-10-09T06:13:23.125 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.2539999999999992e-06 2023-10-09T06:13:23.125 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.125 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.125 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:23.126 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.824832+0000", 2023-10-09T06:13:23.126 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1909e-05 2023-10-09T06:13:23.126 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.126 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.126 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:23.126 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.824839+0000", 2023-10-09T06:13:23.126 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4819999999999997e-06 2023-10-09T06:13:23.126 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.127 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.127 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.127 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.824891+0000", 2023-10-09T06:13:23.127 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2064999999999997e-05 2023-10-09T06:13:23.127 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.128 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.128 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:13:23.128 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.824904+0000", 2023-10-09T06:13:23.128 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2578999999999999e-05 2023-10-09T06:13:23.128 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.128 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.129 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.129 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.809326+0000", 2023-10-09T06:13:23.129 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.98442220000000002 2023-10-09T06:13:23.129 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.129 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.129 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:23.130 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.809438+0000", 2023-10-09T06:13:23.130 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000111531 2023-10-09T06:13:23.130 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.130 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.130 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:13:23.130 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.809544+0000", 2023-10-09T06:13:23.130 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000105988 2023-10-09T06:13:23.131 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.131 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.131 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:23.131 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.824045+0000", 2023-10-09T06:13:23.131 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.014501328000000001 2023-10-09T06:13:23.131 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.131 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.132 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:13:23.132 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.825199+0000", 2023-10-09T06:13:23.132 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0011542740000000001 2023-10-09T06:13:23.132 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.132 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.132 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:23.132 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.825941+0000", 2023-10-09T06:13:23.132 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00074225400000000003 2023-10-09T06:13:23.133 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.133 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.133 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:13:23.133 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.825973+0000", 2023-10-09T06:13:23.133 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1418000000000003e-05 2023-10-09T06:13:23.134 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.134 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.134 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:13:23.134 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.826386+0000", 2023-10-09T06:13:23.134 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00041313299999999999 2023-10-09T06:13:23.134 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.134 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:23.134 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.134 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.134 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.134 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:182329 9.b 9:ddc708a0:::benchmark_data_smithi064_511911_object81227:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T06:13:23.135 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:10:49.834744+0000", 2023-10-09T06:13:23.135 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 153.13327229000001, 2023-10-09T06:13:23.135 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0013349039999999, 2023-10-09T06:13:23.135 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:23.135 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:23.135 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:13:23.135 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:13:23.135 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:13:23.135 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 182329 2023-10-09T06:13:23.135 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.135 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:23.135 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.136 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:23.136 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.834744+0000", 2023-10-09T06:13:23.136 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.136 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.136 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.136 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:23.136 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.834744+0000", 2023-10-09T06:13:23.136 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.136 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.136 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.136 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:23.136 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.834742+0000", 2023-10-09T06:13:23.137 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:13:23.137 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.137 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.137 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:23.137 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.834753+0000", 2023-10-09T06:13:23.137 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0342e-05 2023-10-09T06:13:23.137 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.137 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.137 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:23.137 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.834774+0000", 2023-10-09T06:13:23.137 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0925999999999999e-05 2023-10-09T06:13:23.138 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.138 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.138 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:23.138 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.834782+0000", 2023-10-09T06:13:23.138 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0930000000000003e-06 2023-10-09T06:13:23.138 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.138 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.138 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.138 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.834853+0000", 2023-10-09T06:13:23.138 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1280999999999997e-05 2023-10-09T06:13:23.138 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.139 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.139 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:13:23.139 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.834866+0000", 2023-10-09T06:13:23.139 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3122000000000001e-05 2023-10-09T06:13:23.139 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.139 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.139 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.139 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.814103+0000", 2023-10-09T06:13:23.139 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.97923662199999995 2023-10-09T06:13:23.139 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.139 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.139 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:23.140 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.814260+0000", 2023-10-09T06:13:23.140 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015774800000000001 2023-10-09T06:13:23.140 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.140 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.140 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:13:23.140 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.814363+0000", 2023-10-09T06:13:23.140 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000102413 2023-10-09T06:13:23.140 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.140 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.140 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:13:23.140 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.825207+0000", 2023-10-09T06:13:23.140 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010844457 2023-10-09T06:13:23.141 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.141 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.141 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:23.141 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.825238+0000", 2023-10-09T06:13:23.141 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0802000000000001e-05 2023-10-09T06:13:23.141 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.141 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.141 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:23.141 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.834895+0000", 2023-10-09T06:13:23.141 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.009657367 2023-10-09T06:13:23.141 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.141 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.142 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:13:23.142 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.834957+0000", 2023-10-09T06:13:23.142 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1943e-05 2023-10-09T06:13:23.142 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.142 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.142 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:13:23.142 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.836079+0000", 2023-10-09T06:13:23.142 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0011211979999999999 2023-10-09T06:13:23.142 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.142 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:23.142 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.143 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.143 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.143 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.46283.0:1027 10.f e496/490 10:fd783c6b:::benchmark_data_smithi064_573750_object1026:head v 496'64, mlcod=495'61)", 2023-10-09T06:13:23.143 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:11:54.781184+0000", 2023-10-09T06:13:23.143 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 88.186831917000006, 2023-10-09T06:13:23.143 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013828508, 2023-10-09T06:13:23.143 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:23.143 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:23.143 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:23.143 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.143 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:23.143 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:54.781184+0000", 2023-10-09T06:13:23.144 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.144 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.144 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.144 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:23.144 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:54.781184+0000", 2023-10-09T06:13:23.144 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.144 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.144 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.144 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:23.144 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:54.781183+0000", 2023-10-09T06:13:23.144 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:13:23.144 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.144 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.145 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:23.145 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:54.781232+0000", 2023-10-09T06:13:23.145 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8646000000000001e-05 2023-10-09T06:13:23.145 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.145 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.145 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:23.145 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:54.781244+0000", 2023-10-09T06:13:23.145 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2328e-05 2023-10-09T06:13:23.145 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.145 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.146 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:23.146 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:54.781249+0000", 2023-10-09T06:13:23.146 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9790000000000001e-06 2023-10-09T06:13:23.146 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.146 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.146 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.146 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:54.781290+0000", 2023-10-09T06:13:23.146 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0300999999999999e-05 2023-10-09T06:13:23.146 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.146 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.147 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:23.147 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:54.781306+0000", 2023-10-09T06:13:23.147 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6322000000000001e-05 2023-10-09T06:13:23.147 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.147 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.147 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:13:23.147 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:54.794970+0000", 2023-10-09T06:13:23.147 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013663999 2023-10-09T06:13:23.147 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.148 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.148 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:13:23.148 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:54.795013+0000", 2023-10-09T06:13:23.148 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2632000000000001e-05 2023-10-09T06:13:23.148 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.148 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:23.148 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.148 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.148 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.148 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.46283.0:1029 10.a e496/490 10:584e5114:::benchmark_data_smithi064_573750_object1028:head v 496'77, mlcod=495'75)", 2023-10-09T06:13:23.149 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:11:54.781442+0000", 2023-10-09T06:13:23.149 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 88.186574254999996, 2023-10-09T06:13:23.149 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.014059468, 2023-10-09T06:13:23.149 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:23.149 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:23.149 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:23.149 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.149 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:23.149 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:54.781442+0000", 2023-10-09T06:13:23.149 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.149 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.149 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.150 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:23.150 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:54.781442+0000", 2023-10-09T06:13:23.150 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.150 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.150 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.150 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:23.150 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:54.781441+0000", 2023-10-09T06:13:23.150 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:13:23.150 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.150 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.150 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:23.151 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:54.781517+0000", 2023-10-09T06:13:23.151 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5092999999999994e-05 2023-10-09T06:13:23.151 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.151 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.151 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:23.151 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:54.781531+0000", 2023-10-09T06:13:23.151 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4204e-05 2023-10-09T06:13:23.151 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.151 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.151 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:23.151 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:54.781536+0000", 2023-10-09T06:13:23.151 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7289999999999997e-06 2023-10-09T06:13:23.152 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.152 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.152 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.152 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:54.781574+0000", 2023-10-09T06:13:23.152 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7243000000000002e-05 2023-10-09T06:13:23.152 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.152 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.152 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:23.152 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:54.781598+0000", 2023-10-09T06:13:23.152 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4261e-05 2023-10-09T06:13:23.152 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.152 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.153 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:13:23.153 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:54.795476+0000", 2023-10-09T06:13:23.153 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013878021000000001 2023-10-09T06:13:23.153 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.153 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.153 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:13:23.153 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:11:54.795501+0000", 2023-10-09T06:13:23.153 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.5264000000000001e-05 2023-10-09T06:13:23.153 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.153 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:23.153 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.153 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.154 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.154 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.46283.0:67331 10.a e506/490 10:5eda3918:::benchmark_data_smithi064_573750_object67330:head v 506'4163, mlcod=506'4161)", 2023-10-09T06:13:23.154 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:12:42.074177+0000", 2023-10-09T06:13:23.154 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 40.893838885999998, 2023-10-09T06:13:23.154 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015137797999999999, 2023-10-09T06:13:23.154 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:23.154 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:13:23.154 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:23.154 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.154 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:23.154 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:12:42.074177+0000", 2023-10-09T06:13:23.155 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.155 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.155 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.155 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:23.155 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:12:42.074177+0000", 2023-10-09T06:13:23.155 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:23.155 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.155 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.156 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:23.156 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:12:42.074176+0000", 2023-10-09T06:13:23.156 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:13:23.156 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.156 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.156 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:23.156 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:12:42.074288+0000", 2023-10-09T06:13:23.156 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000112114 2023-10-09T06:13:23.157 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.157 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.157 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:23.157 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:12:42.074303+0000", 2023-10-09T06:13:23.157 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4761e-05 2023-10-09T06:13:23.157 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.157 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.157 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:23.157 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:12:42.074310+0000", 2023-10-09T06:13:23.157 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3339999999999998e-06 2023-10-09T06:13:23.158 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.158 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.158 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:23.158 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:12:42.074364+0000", 2023-10-09T06:13:23.158 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4280999999999997e-05 2023-10-09T06:13:23.158 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.158 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.158 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:23.158 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:12:42.074386+0000", 2023-10-09T06:13:23.158 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2541e-05 2023-10-09T06:13:23.159 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.159 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.159 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:13:23.159 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:12:42.089264+0000", 2023-10-09T06:13:23.159 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.014877998 2023-10-09T06:13:23.159 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:23.159 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:23.159 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:13:23.159 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:12:42.089315+0000", 2023-10-09T06:13:23.159 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0549e-05 2023-10-09T06:13:23.159 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.159 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:23.160 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.160 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:23.160 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:23.160 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:13:23.160 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:13:23.161 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:22 smithi156 ceph-mon[195101]: pgmap v1325: 119 pgs: 13 peering, 3 active+remapped+backfilling, 103 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 45 MiB/s wr, 724 op/s; 12924/283662 objects misplaced (4.556%); 19 MiB/s, 304 objects/s recovering 2023-10-09T06:13:23.161 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:22 smithi156 ceph-mon[195101]: failed to encode map e522 with expected crc 2023-10-09T06:13:23.161 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:22 smithi156 ceph-mon[195101]: failed to encode map e522 with expected crc 2023-10-09T06:13:23.161 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:22 smithi156 ceph-mon[195101]: failed to encode map e522 with expected crc 2023-10-09T06:13:23.161 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:22 smithi156 ceph-mon[195101]: failed to encode map e522 with expected crc 2023-10-09T06:13:23.161 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:22 smithi156 ceph-mon[195101]: osdmap e522: 8 total, 8 up, 8 in 2023-10-09T06:13:23.161 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:22 smithi156 ceph-mon[195101]: failed to encode map e522 with expected crc 2023-10-09T06:13:23.162 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:22 smithi156 ceph-mon[195101]: failed to encode map e522 with expected crc 2023-10-09T06:13:23.162 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:22 smithi156 ceph-mon[195101]: failed to encode map e522 with expected crc 2023-10-09T06:13:23.162 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:22 smithi156 ceph-mon[195101]: 10.7 continuing backfill to osd.5 from (498'1193,520'5980] 10:e23103f1:::benchmark_data_smithi064_573750_object60581:head to 520'5980 2023-10-09T06:13:23.162 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:22 smithi156 ceph-mon[195101]: 10.7 starting backfill to osd.6 from (0'0,0'0] MAX to 520'5980 2023-10-09T06:13:23.214 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:23.309 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:13:23.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:23.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:13:23.357 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 89 16 95621 95605 67.1021 24.8125 0.0251646 0.0148971 2023-10-09T06:13:23.411 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:13:23.486 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:23.575 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:13:23.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:23.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:13:23.679 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:13:23.710 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:23.729 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:13:23.749 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[212750]: failed to encode map e523 with expected crc 2023-10-09T06:13:23.749 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[212750]: osdmap e523: 8 total, 8 up, 8 in 2023-10-09T06:13:23.749 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[212750]: failed to encode map e523 with expected crc 2023-10-09T06:13:23.749 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[212750]: failed to encode map e523 with expected crc 2023-10-09T06:13:23.749 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[212750]: failed to encode map e523 with expected crc 2023-10-09T06:13:23.749 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[212750]: failed to encode map e523 with expected crc 2023-10-09T06:13:23.749 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[212750]: failed to encode map e523 with expected crc 2023-10-09T06:13:23.749 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[212750]: failed to encode map e523 with expected crc 2023-10-09T06:13:23.749 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[212750]: 10.0 starting backfill to osd.6 from (0'0,0'0] MAX to 520'6055 2023-10-09T06:13:23.750 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[212750]: 10.b starting backfill to osd.6 from (0'0,0'0] MAX to 520'5898 2023-10-09T06:13:23.750 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[212750]: failed to encode map e523 with expected crc 2023-10-09T06:13:23.750 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[212750]: 10.d starting backfill to osd.6 from (0'0,0'0] MAX to 520'5926 2023-10-09T06:13:23.750 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[212750]: 10.e starting backfill to osd.6 from (0'0,0'0] MAX to 520'6029 2023-10-09T06:13:23.750 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:13:23.750 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3763682496' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:13:23.750 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:13:23.750 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[220044]: failed to encode map e523 with expected crc 2023-10-09T06:13:23.751 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[220044]: osdmap e523: 8 total, 8 up, 8 in 2023-10-09T06:13:23.751 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[220044]: failed to encode map e523 with expected crc 2023-10-09T06:13:23.751 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[220044]: failed to encode map e523 with expected crc 2023-10-09T06:13:23.751 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[220044]: failed to encode map e523 with expected crc 2023-10-09T06:13:23.751 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[220044]: failed to encode map e523 with expected crc 2023-10-09T06:13:23.751 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[220044]: failed to encode map e523 with expected crc 2023-10-09T06:13:23.751 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[220044]: failed to encode map e523 with expected crc 2023-10-09T06:13:23.752 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[220044]: 10.0 starting backfill to osd.6 from (0'0,0'0] MAX to 520'6055 2023-10-09T06:13:23.752 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[220044]: 10.b starting backfill to osd.6 from (0'0,0'0] MAX to 520'5898 2023-10-09T06:13:23.752 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[220044]: failed to encode map e523 with expected crc 2023-10-09T06:13:23.752 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[220044]: 10.d starting backfill to osd.6 from (0'0,0'0] MAX to 520'5926 2023-10-09T06:13:23.752 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[220044]: 10.e starting backfill to osd.6 from (0'0,0'0] MAX to 520'6029 2023-10-09T06:13:23.752 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:13:23.752 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3763682496' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:13:23.753 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:23 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:13:24.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:23 smithi156 ceph-mon[195101]: failed to encode map e523 with expected crc 2023-10-09T06:13:24.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:23 smithi156 ceph-mon[195101]: osdmap e523: 8 total, 8 up, 8 in 2023-10-09T06:13:24.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:23 smithi156 ceph-mon[195101]: failed to encode map e523 with expected crc 2023-10-09T06:13:24.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:23 smithi156 ceph-mon[195101]: failed to encode map e523 with expected crc 2023-10-09T06:13:24.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:23 smithi156 ceph-mon[195101]: failed to encode map e523 with expected crc 2023-10-09T06:13:24.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:23 smithi156 ceph-mon[195101]: failed to encode map e523 with expected crc 2023-10-09T06:13:24.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:23 smithi156 ceph-mon[195101]: failed to encode map e523 with expected crc 2023-10-09T06:13:24.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:23 smithi156 ceph-mon[195101]: failed to encode map e523 with expected crc 2023-10-09T06:13:24.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:23 smithi156 ceph-mon[195101]: 10.0 starting backfill to osd.6 from (0'0,0'0] MAX to 520'6055 2023-10-09T06:13:24.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:23 smithi156 ceph-mon[195101]: 10.b starting backfill to osd.6 from (0'0,0'0] MAX to 520'5898 2023-10-09T06:13:24.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:23 smithi156 ceph-mon[195101]: failed to encode map e523 with expected crc 2023-10-09T06:13:24.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:23 smithi156 ceph-mon[195101]: 10.d starting backfill to osd.6 from (0'0,0'0] MAX to 520'5926 2023-10-09T06:13:24.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:23 smithi156 ceph-mon[195101]: 10.e starting backfill to osd.6 from (0'0,0'0] MAX to 520'6029 2023-10-09T06:13:24.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:23 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:13:24.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:23 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3763682496' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:13:24.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:23 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:13:24.207 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:13:24.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:24.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:13:24.311 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:13:24.346 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:13:24.356 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total time run: 90.0362 2023-10-09T06:13:24.357 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total writes made: 96212 2023-10-09T06:13:24.357 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Write size: 65536 2023-10-09T06:13:24.357 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object size: 65536 2023-10-09T06:13:24.357 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Bandwidth (MB/sec): 66.787 2023-10-09T06:13:24.357 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Bandwidth: 34.4712 2023-10-09T06:13:24.357 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max bandwidth (MB/sec): 194.938 2023-10-09T06:13:24.357 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min bandwidth (MB/sec): 0 2023-10-09T06:13:24.357 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average IOPS: 1068 2023-10-09T06:13:24.357 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev IOPS: 551.548 2023-10-09T06:13:24.357 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max IOPS: 3119 2023-10-09T06:13:24.357 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min IOPS: 0 2023-10-09T06:13:24.358 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average Latency(s): 0.0149672 2023-10-09T06:13:24.358 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Latency(s): 0.0486894 2023-10-09T06:13:24.358 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max latency(s): 2.32389 2023-10-09T06:13:24.358 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min latency(s): 0.0010971 2023-10-09T06:13:24.361 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T06:13:24.385 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:24.974 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:13:24.980 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:24 smithi064 ceph-mon[220044]: pgmap v1328: 119 pgs: 1 unknown, 23 peering, 3 active+remapped+backfilling, 92 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 37 MiB/s wr, 584 op/s; 11629/285633 objects misplaced (4.071%); 28 MiB/s, 452 objects/s recovering 2023-10-09T06:13:24.980 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:24 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:13:24.980 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:24 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:13:24.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:24 smithi064 ceph-mon[220044]: osdmap e524: 8 total, 8 up, 8 in 2023-10-09T06:13:24.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:24 smithi064 ceph-mon[220044]: failed to encode map e524 with expected crc 2023-10-09T06:13:24.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:24 smithi064 ceph-mon[220044]: failed to encode map e524 with expected crc 2023-10-09T06:13:24.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:24 smithi064 ceph-mon[220044]: failed to encode map e524 with expected crc 2023-10-09T06:13:24.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:24 smithi064 ceph-mon[220044]: failed to encode map e524 with expected crc 2023-10-09T06:13:24.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:24 smithi064 ceph-mon[220044]: failed to encode map e524 with expected crc 2023-10-09T06:13:24.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:24 smithi064 ceph-mon[220044]: failed to encode map e524 with expected crc 2023-10-09T06:13:24.981 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:24 smithi064 ceph-mon[220044]: failed to encode map e524 with expected crc 2023-10-09T06:13:24.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:24.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:13:24.985 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:24 smithi064 ceph-mon[212750]: pgmap v1328: 119 pgs: 1 unknown, 23 peering, 3 active+remapped+backfilling, 92 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 37 MiB/s wr, 584 op/s; 11629/285633 objects misplaced (4.071%); 28 MiB/s, 452 objects/s recovering 2023-10-09T06:13:24.985 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:24 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:13:24.985 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:24 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:13:24.985 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:24 smithi064 ceph-mon[212750]: osdmap e524: 8 total, 8 up, 8 in 2023-10-09T06:13:24.985 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:24 smithi064 ceph-mon[212750]: failed to encode map e524 with expected crc 2023-10-09T06:13:24.985 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:24 smithi064 ceph-mon[212750]: failed to encode map e524 with expected crc 2023-10-09T06:13:24.986 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:24 smithi064 ceph-mon[212750]: failed to encode map e524 with expected crc 2023-10-09T06:13:24.986 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:24 smithi064 ceph-mon[212750]: failed to encode map e524 with expected crc 2023-10-09T06:13:24.986 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:24 smithi064 ceph-mon[212750]: failed to encode map e524 with expected crc 2023-10-09T06:13:24.986 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:24 smithi064 ceph-mon[212750]: failed to encode map e524 with expected crc 2023-10-09T06:13:24.986 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:24 smithi064 ceph-mon[212750]: failed to encode map e524 with expected crc 2023-10-09T06:13:25.060 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:13:25.067 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:13:25.086 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:13:25.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:24 smithi156 ceph-mon[195101]: pgmap v1328: 119 pgs: 1 unknown, 23 peering, 3 active+remapped+backfilling, 92 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 37 MiB/s wr, 584 op/s; 11629/285633 objects misplaced (4.071%); 28 MiB/s, 452 objects/s recovering 2023-10-09T06:13:25.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:24 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:13:25.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:24 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:13:25.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:24 smithi156 ceph-mon[195101]: osdmap e524: 8 total, 8 up, 8 in 2023-10-09T06:13:25.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:24 smithi156 ceph-mon[195101]: failed to encode map e524 with expected crc 2023-10-09T06:13:25.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:24 smithi156 ceph-mon[195101]: failed to encode map e524 with expected crc 2023-10-09T06:13:25.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:24 smithi156 ceph-mon[195101]: failed to encode map e524 with expected crc 2023-10-09T06:13:25.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:24 smithi156 ceph-mon[195101]: failed to encode map e524 with expected crc 2023-10-09T06:13:25.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:24 smithi156 ceph-mon[195101]: failed to encode map e524 with expected crc 2023-10-09T06:13:25.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:24 smithi156 ceph-mon[195101]: failed to encode map e524 with expected crc 2023-10-09T06:13:25.124 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:24 smithi156 ceph-mon[195101]: failed to encode map e524 with expected crc 2023-10-09T06:13:25.178 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:25.481 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:13:25.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:25.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:13:25.585 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:13:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:13:25] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.43.0" 2023-10-09T06:13:25.592 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:13:25.640 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:25.758 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:25 smithi156 ceph-mon[195101]: osdmap e525: 8 total, 8 up, 8 in 2023-10-09T06:13:25.759 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:25 smithi156 ceph-mon[195101]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.759 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:25 smithi156 ceph-mon[195101]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.759 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:25 smithi156 ceph-mon[195101]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.759 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:25 smithi156 ceph-mon[195101]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.759 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:25 smithi156 ceph-mon[195101]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.759 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:25 smithi156 ceph-mon[195101]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.759 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:25 smithi156 ceph-mon[195101]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.759 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:25 smithi156 ceph-mon[195101]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.759 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:25 smithi156 ceph-mon[195101]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.759 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:25 smithi156 ceph-mon[195101]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.760 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:25 smithi156 ceph-mon[195101]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.760 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:25 smithi156 ceph-mon[195101]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.817 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:13:25.860 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:13:25.884 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:13:25.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:25.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:13:25.894 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:13:25.941 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:13:25.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[220044]: osdmap e525: 8 total, 8 up, 8 in 2023-10-09T06:13:25.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[220044]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[220044]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[220044]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[220044]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[220044]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[220044]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[220044]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[220044]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[220044]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[220044]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[220044]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[220044]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[212750]: osdmap e525: 8 total, 8 up, 8 in 2023-10-09T06:13:25.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[212750]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[212750]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[212750]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[212750]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[212750]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[212750]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[212750]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[212750]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[212750]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[212750]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[212750]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:25 smithi064 ceph-mon[212750]: failed to encode map e525 with expected crc 2023-10-09T06:13:25.985 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:13:25.988 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:13:26.030 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:13:26.069 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:13:26.082 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:26.124 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:13:26.140 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:13:26.140 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:13:26.140 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:13:26.140 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:13:26.140 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:13:26.140 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:13:26.140 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:13:26.140 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:13:27.096 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:13:27.129 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:26 smithi156 ceph-mon[195101]: pgmap v1331: 119 pgs: 1 unknown, 23 peering, 3 active+remapped+backfilling, 92 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 10 MiB/s wr, 164 op/s; 11629/285633 objects misplaced (4.071%); 21 MiB/s, 336 objects/s recovering 2023-10-09T06:13:27.129 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:26 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/849658308' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:13:27.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:26 smithi064 ceph-mon[220044]: pgmap v1331: 119 pgs: 1 unknown, 23 peering, 3 active+remapped+backfilling, 92 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 10 MiB/s wr, 164 op/s; 11629/285633 objects misplaced (4.071%); 21 MiB/s, 336 objects/s recovering 2023-10-09T06:13:27.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:26 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/849658308' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:13:27.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:26 smithi064 ceph-mon[212750]: pgmap v1331: 119 pgs: 1 unknown, 23 peering, 3 active+remapped+backfilling, 92 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 10 MiB/s wr, 164 op/s; 11629/285633 objects misplaced (4.071%); 21 MiB/s, 336 objects/s recovering 2023-10-09T06:13:27.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:26 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/849658308' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:13:27.245 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:13:27.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:27.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:13:27.359 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:13:27.438 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:27.765 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:13:27.765 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:13:28.020 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:13:28.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:28.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:13:28.125 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:13:28.219 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:28.411 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:13:28.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:28.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:13:28.516 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:13:28.566 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:28.771 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:13:28.989 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:13:28.992 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:13:28.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:28.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:13:29.031 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:28 smithi156 ceph-mon[195101]: pgmap v1332: 119 pgs: 5 active+remapped+backfill_wait, 2 active+remapped+backfilling, 112 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 187 B/s rd, 12 MiB/s wr, 403 op/s; 38940/285192 objects misplaced (13.654%); 45 MiB/s, 1 keys/s, 720 objects/s recovering 2023-10-09T06:13:29.031 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:28 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:29.098 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:13:29.163 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:29.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:28 smithi064 ceph-mon[220044]: pgmap v1332: 119 pgs: 5 active+remapped+backfill_wait, 2 active+remapped+backfilling, 112 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 187 B/s rd, 12 MiB/s wr, 403 op/s; 38940/285192 objects misplaced (13.654%); 45 MiB/s, 1 keys/s, 720 objects/s recovering 2023-10-09T06:13:29.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:28 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:29.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:28 smithi064 ceph-mon[212750]: pgmap v1332: 119 pgs: 5 active+remapped+backfill_wait, 2 active+remapped+backfilling, 112 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 187 B/s rd, 12 MiB/s wr, 403 op/s; 38940/285192 objects misplaced (13.654%); 45 MiB/s, 1 keys/s, 720 objects/s recovering 2023-10-09T06:13:29.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:28 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:29.359 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:13:29.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:29.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:13:29.464 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:13:29.549 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:29.716 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:13:29.808 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T06:13:29.836 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:13:29.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:29.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:13:29.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:29 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/229874378' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:13:29.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:29 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:13:29.845 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:29 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/229874378' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:13:29.846 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:29 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:13:29.945 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:13:29.983 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:30.081 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:29 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/229874378' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:13:30.082 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:29 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:13:30.152 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:13:31.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:30 smithi156 ceph-mon[195101]: pgmap v1333: 119 pgs: 5 active+remapped+backfill_wait, 2 active+remapped+backfilling, 112 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 158 B/s rd, 11 MiB/s wr, 376 op/s; 38430/284865 objects misplaced (13.491%); 43 MiB/s, 1 keys/s, 685 objects/s recovering 2023-10-09T06:13:31.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:30 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:13:31.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:30 smithi156 ceph-mon[195101]: osdmap e526: 8 total, 8 up, 8 in 2023-10-09T06:13:31.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:30 smithi156 ceph-mon[195101]: failed to encode map e526 with expected crc 2023-10-09T06:13:31.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:30 smithi156 ceph-mon[195101]: failed to encode map e526 with expected crc 2023-10-09T06:13:31.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:30 smithi156 ceph-mon[195101]: osdmap e527: 8 total, 8 up, 8 in 2023-10-09T06:13:31.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:30 smithi064 ceph-mon[212750]: pgmap v1333: 119 pgs: 5 active+remapped+backfill_wait, 2 active+remapped+backfilling, 112 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 158 B/s rd, 11 MiB/s wr, 376 op/s; 38430/284865 objects misplaced (13.491%); 43 MiB/s, 1 keys/s, 685 objects/s recovering 2023-10-09T06:13:31.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:30 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:13:31.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:30 smithi064 ceph-mon[212750]: osdmap e526: 8 total, 8 up, 8 in 2023-10-09T06:13:31.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:30 smithi064 ceph-mon[212750]: failed to encode map e526 with expected crc 2023-10-09T06:13:31.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:30 smithi064 ceph-mon[212750]: failed to encode map e526 with expected crc 2023-10-09T06:13:31.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:30 smithi064 ceph-mon[212750]: osdmap e527: 8 total, 8 up, 8 in 2023-10-09T06:13:31.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:30 smithi064 ceph-mon[220044]: pgmap v1333: 119 pgs: 5 active+remapped+backfill_wait, 2 active+remapped+backfilling, 112 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 158 B/s rd, 11 MiB/s wr, 376 op/s; 38430/284865 objects misplaced (13.491%); 43 MiB/s, 1 keys/s, 685 objects/s recovering 2023-10-09T06:13:31.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:30 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:13:31.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:30 smithi064 ceph-mon[220044]: osdmap e526: 8 total, 8 up, 8 in 2023-10-09T06:13:31.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:30 smithi064 ceph-mon[220044]: failed to encode map e526 with expected crc 2023-10-09T06:13:31.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:30 smithi064 ceph-mon[220044]: failed to encode map e526 with expected crc 2023-10-09T06:13:31.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:30 smithi064 ceph-mon[220044]: osdmap e527: 8 total, 8 up, 8 in 2023-10-09T06:13:31.903 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T06:13:31.908 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:13:31.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:31.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:13:31.974 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:13:31.975 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:13:32.003 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:31 smithi156 ceph-mon[195101]: failed to encode map e527 with expected crc 2023-10-09T06:13:32.003 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:31 smithi156 ceph-mon[195101]: failed to encode map e527 with expected crc 2023-10-09T06:13:32.003 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:31 smithi156 ceph-mon[195101]: failed to encode map e527 with expected crc 2023-10-09T06:13:32.003 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:31 smithi156 ceph-mon[195101]: failed to encode map e527 with expected crc 2023-10-09T06:13:32.003 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:31 smithi156 ceph-mon[195101]: 10.9 continuing backfill to osd.5 from (498'1100,525'6208] 10:97b96e2c:::benchmark_data_smithi064_573750_object83508:head to 525'6208 2023-10-09T06:13:32.003 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:31 smithi156 ceph-mon[195101]: 10.9 starting backfill to osd.7 from (0'0,0'0] MAX to 525'6208 2023-10-09T06:13:32.004 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:31 smithi156 ceph-mon[195101]: failed to encode map e527 with expected crc 2023-10-09T06:13:32.004 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:31 smithi156 ceph-mon[195101]: failed to encode map e527 with expected crc 2023-10-09T06:13:32.004 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:31 smithi156 ceph-mon[195101]: failed to encode map e527 with expected crc 2023-10-09T06:13:32.004 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:31 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:13:32.004 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:31 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:13:32.004 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:31 smithi156 ceph-mon[195101]: osdmap e528: 8 total, 8 up, 8 in 2023-10-09T06:13:32.013 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:13:32.120 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:31 smithi064 ceph-mon[220044]: failed to encode map e527 with expected crc 2023-10-09T06:13:32.120 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:31 smithi064 ceph-mon[220044]: failed to encode map e527 with expected crc 2023-10-09T06:13:32.120 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:31 smithi064 ceph-mon[220044]: failed to encode map e527 with expected crc 2023-10-09T06:13:32.120 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:31 smithi064 ceph-mon[220044]: failed to encode map e527 with expected crc 2023-10-09T06:13:32.120 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:31 smithi064 ceph-mon[220044]: 10.9 continuing backfill to osd.5 from (498'1100,525'6208] 10:97b96e2c:::benchmark_data_smithi064_573750_object83508:head to 525'6208 2023-10-09T06:13:32.120 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:31 smithi064 ceph-mon[220044]: 10.9 starting backfill to osd.7 from (0'0,0'0] MAX to 525'6208 2023-10-09T06:13:32.120 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:31 smithi064 ceph-mon[220044]: failed to encode map e527 with expected crc 2023-10-09T06:13:32.120 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:31 smithi064 ceph-mon[220044]: failed to encode map e527 with expected crc 2023-10-09T06:13:32.120 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:31 smithi064 ceph-mon[220044]: failed to encode map e527 with expected crc 2023-10-09T06:13:32.121 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:31 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:13:32.121 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:31 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:13:32.121 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:31 smithi064 ceph-mon[220044]: osdmap e528: 8 total, 8 up, 8 in 2023-10-09T06:13:32.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:31 smithi064 ceph-mon[212750]: failed to encode map e527 with expected crc 2023-10-09T06:13:32.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:31 smithi064 ceph-mon[212750]: failed to encode map e527 with expected crc 2023-10-09T06:13:32.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:31 smithi064 ceph-mon[212750]: failed to encode map e527 with expected crc 2023-10-09T06:13:32.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:31 smithi064 ceph-mon[212750]: failed to encode map e527 with expected crc 2023-10-09T06:13:32.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:31 smithi064 ceph-mon[212750]: 10.9 continuing backfill to osd.5 from (498'1100,525'6208] 10:97b96e2c:::benchmark_data_smithi064_573750_object83508:head to 525'6208 2023-10-09T06:13:32.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:31 smithi064 ceph-mon[212750]: 10.9 starting backfill to osd.7 from (0'0,0'0] MAX to 525'6208 2023-10-09T06:13:32.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:31 smithi064 ceph-mon[212750]: failed to encode map e527 with expected crc 2023-10-09T06:13:32.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:31 smithi064 ceph-mon[212750]: failed to encode map e527 with expected crc 2023-10-09T06:13:32.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:31 smithi064 ceph-mon[212750]: failed to encode map e527 with expected crc 2023-10-09T06:13:32.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:31 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:13:32.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:31 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:13:32.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:31 smithi064 ceph-mon[212750]: osdmap e528: 8 total, 8 up, 8 in 2023-10-09T06:13:32.125 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:32.272 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:13:32.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:32.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:13:32.382 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:13:32.425 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:32.468 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:13:32.486 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:13:32.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:32.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:13:32.590 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:13:32.626 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:32.639 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:13:32.694 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:13:32.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:32.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:13:32.798 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:13:32.838 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:32.940 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:13:32.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:32.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:13:33.004 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:32 smithi064 ceph-mon[212750]: pgmap v1336: 119 pgs: 5 remapped+peering, 10 peering, 4 active+remapped+backfill_wait, 2 active+remapped+backfilling, 98 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 6.5 KiB/s rd, 10 MiB/s wr, 694 op/s; 34680/278505 objects misplaced (12.452%); 30 MiB/s, 1 keys/s, 478 objects/s recovering 2023-10-09T06:13:33.004 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:32 smithi064 ceph-mon[212750]: failed to encode map e528 with expected crc 2023-10-09T06:13:33.004 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:32 smithi064 ceph-mon[212750]: failed to encode map e528 with expected crc 2023-10-09T06:13:33.004 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:32 smithi064 ceph-mon[212750]: failed to encode map e528 with expected crc 2023-10-09T06:13:33.004 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:32 smithi064 ceph-mon[212750]: failed to encode map e528 with expected crc 2023-10-09T06:13:33.004 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:32 smithi064 ceph-mon[212750]: 10.8 starting backfill to osd.7 from (0'0,0'0] MAX to 525'6196 2023-10-09T06:13:33.004 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:32 smithi064 ceph-mon[212750]: failed to encode map e528 with expected crc 2023-10-09T06:13:33.005 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:32 smithi064 ceph-mon[212750]: failed to encode map e528 with expected crc 2023-10-09T06:13:33.005 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:32 smithi064 ceph-mon[212750]: 10.c starting backfill to osd.7 from (0'0,0'0] MAX to 525'6338 2023-10-09T06:13:33.005 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:32 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:13:33.005 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:32 smithi064 ceph-mon[212750]: osdmap e529: 8 total, 8 up, 8 in 2023-10-09T06:13:33.005 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:32 smithi064 ceph-mon[220044]: pgmap v1336: 119 pgs: 5 remapped+peering, 10 peering, 4 active+remapped+backfill_wait, 2 active+remapped+backfilling, 98 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 6.5 KiB/s rd, 10 MiB/s wr, 694 op/s; 34680/278505 objects misplaced (12.452%); 30 MiB/s, 1 keys/s, 478 objects/s recovering 2023-10-09T06:13:33.005 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:32 smithi064 ceph-mon[220044]: failed to encode map e528 with expected crc 2023-10-09T06:13:33.005 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:32 smithi064 ceph-mon[220044]: failed to encode map e528 with expected crc 2023-10-09T06:13:33.005 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:32 smithi064 ceph-mon[220044]: failed to encode map e528 with expected crc 2023-10-09T06:13:33.006 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:32 smithi064 ceph-mon[220044]: failed to encode map e528 with expected crc 2023-10-09T06:13:33.006 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:32 smithi064 ceph-mon[220044]: 10.8 starting backfill to osd.7 from (0'0,0'0] MAX to 525'6196 2023-10-09T06:13:33.006 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:32 smithi064 ceph-mon[220044]: failed to encode map e528 with expected crc 2023-10-09T06:13:33.006 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:32 smithi064 ceph-mon[220044]: failed to encode map e528 with expected crc 2023-10-09T06:13:33.006 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:32 smithi064 ceph-mon[220044]: 10.c starting backfill to osd.7 from (0'0,0'0] MAX to 525'6338 2023-10-09T06:13:33.006 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:32 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:13:33.007 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:32 smithi064 ceph-mon[220044]: osdmap e529: 8 total, 8 up, 8 in 2023-10-09T06:13:33.044 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:13:33.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:32 smithi156 ceph-mon[195101]: pgmap v1336: 119 pgs: 5 remapped+peering, 10 peering, 4 active+remapped+backfill_wait, 2 active+remapped+backfilling, 98 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 6.5 KiB/s rd, 10 MiB/s wr, 694 op/s; 34680/278505 objects misplaced (12.452%); 30 MiB/s, 1 keys/s, 478 objects/s recovering 2023-10-09T06:13:33.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:32 smithi156 ceph-mon[195101]: failed to encode map e528 with expected crc 2023-10-09T06:13:33.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:32 smithi156 ceph-mon[195101]: failed to encode map e528 with expected crc 2023-10-09T06:13:33.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:32 smithi156 ceph-mon[195101]: failed to encode map e528 with expected crc 2023-10-09T06:13:33.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:32 smithi156 ceph-mon[195101]: failed to encode map e528 with expected crc 2023-10-09T06:13:33.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:32 smithi156 ceph-mon[195101]: 10.8 starting backfill to osd.7 from (0'0,0'0] MAX to 525'6196 2023-10-09T06:13:33.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:32 smithi156 ceph-mon[195101]: failed to encode map e528 with expected crc 2023-10-09T06:13:33.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:32 smithi156 ceph-mon[195101]: failed to encode map e528 with expected crc 2023-10-09T06:13:33.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:32 smithi156 ceph-mon[195101]: 10.c starting backfill to osd.7 from (0'0,0'0] MAX to 525'6338 2023-10-09T06:13:33.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:32 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:13:33.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:32 smithi156 ceph-mon[195101]: osdmap e529: 8 total, 8 up, 8 in 2023-10-09T06:13:33.130 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:33.689 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:13:33.918 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:13:33.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:33 smithi064 ceph-mon[212750]: failed to encode map e529 with expected crc 2023-10-09T06:13:33.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:33 smithi064 ceph-mon[212750]: failed to encode map e529 with expected crc 2023-10-09T06:13:33.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:33 smithi064 ceph-mon[212750]: failed to encode map e529 with expected crc 2023-10-09T06:13:33.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:33 smithi064 ceph-mon[212750]: failed to encode map e529 with expected crc 2023-10-09T06:13:33.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:33 smithi064 ceph-mon[212750]: failed to encode map e529 with expected crc 2023-10-09T06:13:33.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:33 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1222825399' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:13:33.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:33 smithi064 ceph-mon[212750]: pgmap v1339: 119 pgs: 9 remapped+peering, 10 peering, 5 active+remapped+backfill_wait, 1 active+remapped+backfilling, 94 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 657 op/s; 33347/277068 objects misplaced (12.036%); 16 MiB/s, 260 objects/s recovering 2023-10-09T06:13:33.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:33 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:13:33.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:33 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1222825399' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:13:33.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:33 smithi064 ceph-mon[212750]: osdmap e530: 8 total, 8 up, 8 in 2023-10-09T06:13:33.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:33 smithi064 ceph-mon[212750]: failed to encode map e530 with expected crc 2023-10-09T06:13:33.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:33 smithi064 ceph-mon[212750]: failed to encode map e530 with expected crc 2023-10-09T06:13:33.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:33 smithi064 ceph-mon[220044]: failed to encode map e529 with expected crc 2023-10-09T06:13:33.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:33 smithi064 ceph-mon[220044]: failed to encode map e529 with expected crc 2023-10-09T06:13:33.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:33 smithi064 ceph-mon[220044]: failed to encode map e529 with expected crc 2023-10-09T06:13:33.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:33 smithi064 ceph-mon[220044]: failed to encode map e529 with expected crc 2023-10-09T06:13:33.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:33 smithi064 ceph-mon[220044]: failed to encode map e529 with expected crc 2023-10-09T06:13:33.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:33 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1222825399' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:13:33.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:33 smithi064 ceph-mon[220044]: pgmap v1339: 119 pgs: 9 remapped+peering, 10 peering, 5 active+remapped+backfill_wait, 1 active+remapped+backfilling, 94 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 657 op/s; 33347/277068 objects misplaced (12.036%); 16 MiB/s, 260 objects/s recovering 2023-10-09T06:13:33.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:33 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:13:33.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:33 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1222825399' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:13:33.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:33 smithi064 ceph-mon[220044]: osdmap e530: 8 total, 8 up, 8 in 2023-10-09T06:13:33.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:33 smithi064 ceph-mon[220044]: failed to encode map e530 with expected crc 2023-10-09T06:13:33.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:33 smithi064 ceph-mon[220044]: failed to encode map e530 with expected crc 2023-10-09T06:13:34.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:33 smithi156 ceph-mon[195101]: failed to encode map e529 with expected crc 2023-10-09T06:13:34.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:33 smithi156 ceph-mon[195101]: failed to encode map e529 with expected crc 2023-10-09T06:13:34.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:33 smithi156 ceph-mon[195101]: failed to encode map e529 with expected crc 2023-10-09T06:13:34.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:33 smithi156 ceph-mon[195101]: failed to encode map e529 with expected crc 2023-10-09T06:13:34.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:33 smithi156 ceph-mon[195101]: failed to encode map e529 with expected crc 2023-10-09T06:13:34.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:33 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1222825399' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:13:34.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:33 smithi156 ceph-mon[195101]: pgmap v1339: 119 pgs: 9 remapped+peering, 10 peering, 5 active+remapped+backfill_wait, 1 active+remapped+backfilling, 94 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 657 op/s; 33347/277068 objects misplaced (12.036%); 16 MiB/s, 260 objects/s recovering 2023-10-09T06:13:34.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:33 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:13:34.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:33 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1222825399' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:13:34.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:33 smithi156 ceph-mon[195101]: osdmap e530: 8 total, 8 up, 8 in 2023-10-09T06:13:34.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:33 smithi156 ceph-mon[195101]: failed to encode map e530 with expected crc 2023-10-09T06:13:34.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:33 smithi156 ceph-mon[195101]: failed to encode map e530 with expected crc 2023-10-09T06:13:34.426 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:13:35.005 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:34 smithi156 ceph-mon[195101]: failed to encode map e530 with expected crc 2023-10-09T06:13:35.005 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:34 smithi156 ceph-mon[195101]: failed to encode map e530 with expected crc 2023-10-09T06:13:35.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:34 smithi064 ceph-mon[212750]: failed to encode map e530 with expected crc 2023-10-09T06:13:35.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:34 smithi064 ceph-mon[212750]: failed to encode map e530 with expected crc 2023-10-09T06:13:35.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:34 smithi064 ceph-mon[220044]: failed to encode map e530 with expected crc 2023-10-09T06:13:35.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:34 smithi064 ceph-mon[220044]: failed to encode map e530 with expected crc 2023-10-09T06:13:35.601 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:13:35.603 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:13:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:13:35] "GET /metrics HTTP/1.1" 200 36494 "" "Prometheus/2.43.0" 2023-10-09T06:13:35.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:35.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:13:35.705 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:13:35.735 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:35.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:35 smithi064 ceph-mon[212750]: pgmap v1341: 119 pgs: 9 remapped+peering, 10 peering, 5 active+remapped+backfill_wait, 1 active+remapped+backfilling, 94 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 9.4 KiB/s rd, 25 KiB/s wr, 303 op/s; 33347/277068 objects misplaced (12.036%); 15 MiB/s, 238 objects/s recovering 2023-10-09T06:13:35.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:35 smithi064 ceph-mon[220044]: pgmap v1341: 119 pgs: 9 remapped+peering, 10 peering, 5 active+remapped+backfill_wait, 1 active+remapped+backfilling, 94 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 9.4 KiB/s rd, 25 KiB/s wr, 303 op/s; 33347/277068 objects misplaced (12.036%); 15 MiB/s, 238 objects/s recovering 2023-10-09T06:13:36.087 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:13:36.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:36.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:13:36.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:35 smithi156 ceph-mon[195101]: pgmap v1341: 119 pgs: 9 remapped+peering, 10 peering, 5 active+remapped+backfill_wait, 1 active+remapped+backfilling, 94 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 9.4 KiB/s rd, 25 KiB/s wr, 303 op/s; 33347/277068 objects misplaced (12.036%); 15 MiB/s, 238 objects/s recovering 2023-10-09T06:13:36.203 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:13:36.267 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:36.405 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:13:36.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:36.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:13:36.523 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:13:36.569 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:36.677 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:13:36.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:36.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:13:36.786 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:13:36.908 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:36.948 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:13:36.948 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:13:37.249 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:13:37.256 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:13:37.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:37.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:13:37.314 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:13:37.360 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:13:37.372 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:13:37.410 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:37.417 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:13:37.469 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:13:37.476 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:13:37.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:37.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:13:37.510 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:13:37.530 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:13:37.580 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:13:37.604 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:37 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2692063413' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:13:37.604 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:37 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T06:13:37.606 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:37.649 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:13:37.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:37.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:13:37.654 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:13:37.655 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:13:37.655 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:13:37.655 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:13:37.655 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:13:37.655 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:13:37.655 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:13:37.656 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:13:37.656 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:13:37.685 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:37 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2692063413' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:13:37.686 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:37 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T06:13:37.686 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:37 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2692063413' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:13:37.686 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:37 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T06:13:37.752 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:13:37.830 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:38.103 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:13:38.131 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:13:38.307 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T06:13:38.375 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:38 smithi156 ceph-mon[195101]: pgmap v1342: 119 pgs: 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 5.5 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 47930/269232 objects misplaced (17.802%); 56 MiB/s, 892 objects/s recovering 2023-10-09T06:13:38.375 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:38 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4200494617' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:13:38.375 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:38 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:13:38.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:38 smithi064 ceph-mon[212750]: pgmap v1342: 119 pgs: 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 5.5 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 47930/269232 objects misplaced (17.802%); 56 MiB/s, 892 objects/s recovering 2023-10-09T06:13:38.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:38 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4200494617' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:13:38.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:38 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:13:38.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:38 smithi064 ceph-mon[220044]: pgmap v1342: 119 pgs: 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 5.5 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 47930/269232 objects misplaced (17.802%); 56 MiB/s, 892 objects/s recovering 2023-10-09T06:13:38.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:38 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4200494617' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:13:38.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:38 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:13:38.744 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:13:39.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:39 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:13:39.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:39 smithi156 ceph-mon[195101]: osdmap e531: 8 total, 8 up, 8 in 2023-10-09T06:13:39.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:39 smithi156 ceph-mon[195101]: failed to encode map e531 with expected crc 2023-10-09T06:13:39.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:39 smithi156 ceph-mon[195101]: failed to encode map e531 with expected crc 2023-10-09T06:13:39.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:39 smithi156 ceph-mon[195101]: failed to encode map e531 with expected crc 2023-10-09T06:13:39.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:39 smithi156 ceph-mon[195101]: failed to encode map e531 with expected crc 2023-10-09T06:13:39.666 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:13:39.671 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:39 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:13:39.671 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:39 smithi064 ceph-mon[212750]: osdmap e531: 8 total, 8 up, 8 in 2023-10-09T06:13:39.671 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:39 smithi064 ceph-mon[212750]: failed to encode map e531 with expected crc 2023-10-09T06:13:39.671 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:39 smithi064 ceph-mon[212750]: failed to encode map e531 with expected crc 2023-10-09T06:13:39.671 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:39 smithi064 ceph-mon[212750]: failed to encode map e531 with expected crc 2023-10-09T06:13:39.672 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:39 smithi064 ceph-mon[212750]: failed to encode map e531 with expected crc 2023-10-09T06:13:39.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:39 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:13:39.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:39 smithi064 ceph-mon[220044]: osdmap e531: 8 total, 8 up, 8 in 2023-10-09T06:13:39.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:39 smithi064 ceph-mon[220044]: failed to encode map e531 with expected crc 2023-10-09T06:13:39.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:39 smithi064 ceph-mon[220044]: failed to encode map e531 with expected crc 2023-10-09T06:13:39.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:39 smithi064 ceph-mon[220044]: failed to encode map e531 with expected crc 2023-10-09T06:13:39.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:39 smithi064 ceph-mon[220044]: failed to encode map e531 with expected crc 2023-10-09T06:13:39.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:39.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:13:39.774 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:13:39.833 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:39.890 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:13:39.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:39.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:13:39.999 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:13:40.054 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:40.119 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T06:13:40.269 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:13:40.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:40.270 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:13:40.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:13:40.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:40 smithi064 ceph-mon[212750]: pgmap v1344: 119 pgs: 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 5.5 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 403 op/s; 47930/269232 objects misplaced (17.802%); 42 MiB/s, 665 objects/s recovering 2023-10-09T06:13:40.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:40 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:13:40.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:40 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:13:40.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:40 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:13:40.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:40 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:13:40.335 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:40 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:13:40.335 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:40 smithi064 ceph-mon[220044]: pgmap v1344: 119 pgs: 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 5.5 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 403 op/s; 47930/269232 objects misplaced (17.802%); 42 MiB/s, 665 objects/s recovering 2023-10-09T06:13:40.335 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:40 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:13:40.335 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:40 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:13:40.335 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:40 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:13:40.335 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:40 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:13:40.336 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:40 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:13:40.371 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:13:40.421 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:40 smithi156 ceph-mon[195101]: pgmap v1344: 119 pgs: 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 5.5 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 403 op/s; 47930/269232 objects misplaced (17.802%); 42 MiB/s, 665 objects/s recovering 2023-10-09T06:13:40.421 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:40 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:13:40.421 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:40 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:13:40.421 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:40 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:13:40.421 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:40 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:13:40.421 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:40 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:13:40.471 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:40.681 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:13:40.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:40.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:13:40.791 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:13:40.868 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:40.967 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:13:40.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:40.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:13:41.072 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:13:41.119 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:13:41.119 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:13:41.120 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.120 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.46283.0:106299 10.b e531/522 10:d96a1615:::benchmark_data_smithi064_573750_object10084:head v 531'6585, mlcod=531'6583)", 2023-10-09T06:13:41.120 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:41.111618+0000", 2023-10-09T06:13:41.120 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0051154920000000001, 2023-10-09T06:13:41.120 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0051455900000000002, 2023-10-09T06:13:41.120 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:41.120 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:13:41.120 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:41.120 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.120 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:41.121 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.111618+0000", 2023-10-09T06:13:41.121 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:41.121 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.121 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.121 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:41.121 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.111618+0000", 2023-10-09T06:13:41.121 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:41.121 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.121 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.121 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:41.121 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.111618+0000", 2023-10-09T06:13:41.121 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:13:41.122 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.122 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.122 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:41.122 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.111625+0000", 2023-10-09T06:13:41.122 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9519999999999997e-06 2023-10-09T06:13:41.122 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.122 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.122 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:41.122 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.111632+0000", 2023-10-09T06:13:41.122 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4479999999999997e-06 2023-10-09T06:13:41.123 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.123 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.123 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:41.123 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.111636+0000", 2023-10-09T06:13:41.123 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.4249999999999998e-06 2023-10-09T06:13:41.123 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.123 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.123 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:41.123 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.111676+0000", 2023-10-09T06:13:41.123 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0493e-05 2023-10-09T06:13:41.123 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.124 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.124 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:41.124 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.111700+0000", 2023-10-09T06:13:41.124 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4094999999999999e-05 2023-10-09T06:13:41.124 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:41.124 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:41.124 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:41.124 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.124 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.124 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.46283.0:106296 10.2 e531/490 10:455c2135:::benchmark_data_smithi064_573750_object10081:head v 531'6628, mlcod=531'6626)", 2023-10-09T06:13:41.124 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:41.111648+0000", 2023-10-09T06:13:41.125 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0050863030000000004, 2023-10-09T06:13:41.125 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0051829520000000002, 2023-10-09T06:13:41.125 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:41.125 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:13:41.125 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:41.125 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.125 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:41.125 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.111648+0000", 2023-10-09T06:13:41.125 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:41.125 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.125 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.126 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:41.126 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.111648+0000", 2023-10-09T06:13:41.126 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:41.126 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.126 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.126 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:41.126 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.111647+0000", 2023-10-09T06:13:41.126 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967296 2023-10-09T06:13:41.126 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.126 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.126 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:41.127 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.111651+0000", 2023-10-09T06:13:41.127 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8460000000000001e-06 2023-10-09T06:13:41.127 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.127 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.127 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:41.127 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.111657+0000", 2023-10-09T06:13:41.127 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3619999999999997e-06 2023-10-09T06:13:41.127 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.127 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.127 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:41.127 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.111660+0000", 2023-10-09T06:13:41.128 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.613e-06 2023-10-09T06:13:41.128 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.128 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.128 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:41.128 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.111701+0000", 2023-10-09T06:13:41.128 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0636999999999997e-05 2023-10-09T06:13:41.128 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.128 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.128 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:41.128 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.111719+0000", 2023-10-09T06:13:41.128 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8111e-05 2023-10-09T06:13:41.129 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:41.129 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:41.129 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:41.129 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.129 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.129 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.46283.0:106300 10.b e531/522 10:dccfaf58:::benchmark_data_smithi064_573750_object10085:head v 531'6586, mlcod=531'6583)", 2023-10-09T06:13:41.129 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:41.111672+0000", 2023-10-09T06:13:41.129 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0050622929999999998, 2023-10-09T06:13:41.129 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.005219436, 2023-10-09T06:13:41.129 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:41.129 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:13:41.129 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:41.130 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.130 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:41.130 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.111672+0000", 2023-10-09T06:13:41.130 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:41.130 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.130 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.130 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:41.130 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.111672+0000", 2023-10-09T06:13:41.130 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:41.130 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.131 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.131 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:41.131 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.111671+0000", 2023-10-09T06:13:41.131 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:13:41.131 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.131 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.131 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:41.131 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.111675+0000", 2023-10-09T06:13:41.131 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8720000000000004e-06 2023-10-09T06:13:41.131 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.131 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.131 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:41.132 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.111681+0000", 2023-10-09T06:13:41.132 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7409999999999998e-06 2023-10-09T06:13:41.132 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.132 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.132 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:41.132 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.111685+0000", 2023-10-09T06:13:41.132 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1779999999999996e-06 2023-10-09T06:13:41.132 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.132 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.132 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:41.132 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.114421+0000", 2023-10-09T06:13:41.132 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0027359110000000002 2023-10-09T06:13:41.133 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.133 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.133 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:41.133 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.114443+0000", 2023-10-09T06:13:41.133 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.209e-05 2023-10-09T06:13:41.133 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:41.133 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:41.133 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:41.133 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.133 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.133 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.46283.0:106303 10.2 e531/490 10:4638c773:::benchmark_data_smithi064_573750_object10088:head v 531'6629, mlcod=531'6626)", 2023-10-09T06:13:41.134 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:41.112594+0000", 2023-10-09T06:13:41.134 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0041401670000000002, 2023-10-09T06:13:41.134 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0043556200000000002, 2023-10-09T06:13:41.134 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:41.134 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:13:41.134 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:41.134 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.134 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:41.134 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.112594+0000", 2023-10-09T06:13:41.134 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:41.134 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.134 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.135 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:41.135 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.112594+0000", 2023-10-09T06:13:41.135 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:41.135 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.135 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.135 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:41.135 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.112593+0000", 2023-10-09T06:13:41.135 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:13:41.135 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.135 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.135 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:41.135 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.112598+0000", 2023-10-09T06:13:41.136 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2460000000000003e-06 2023-10-09T06:13:41.136 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.136 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.136 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:41.136 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.112606+0000", 2023-10-09T06:13:41.136 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4000000000000003e-06 2023-10-09T06:13:41.136 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.136 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.136 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:41.136 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.112611+0000", 2023-10-09T06:13:41.136 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1000000000000003e-06 2023-10-09T06:13:41.137 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.137 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.137 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:41.137 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.114420+0000", 2023-10-09T06:13:41.137 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0018094960000000001 2023-10-09T06:13:41.137 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.137 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.137 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:41.137 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.114441+0000", 2023-10-09T06:13:41.137 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0069999999999999e-05 2023-10-09T06:13:41.137 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:41.138 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:41.138 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:41.138 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.138 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.138 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.46283.0:106306 10.6 e531/490 10:65716a64:::benchmark_data_smithi064_573750_object10091:head v 531'6629, mlcod=531'6627)", 2023-10-09T06:13:41.138 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:41.113398+0000", 2023-10-09T06:13:41.138 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0033359259999999999, 2023-10-09T06:13:41.138 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0036108239999999999, 2023-10-09T06:13:41.138 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:41.138 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:13:41.139 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:41.139 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.139 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:41.139 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.113398+0000", 2023-10-09T06:13:41.139 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:41.139 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.139 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.139 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:41.139 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.113398+0000", 2023-10-09T06:13:41.139 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:41.139 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.140 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.140 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:41.140 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.113397+0000", 2023-10-09T06:13:41.140 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:13:41.140 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.140 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.140 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:41.140 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.113403+0000", 2023-10-09T06:13:41.140 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4909999999999996e-06 2023-10-09T06:13:41.140 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.140 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.140 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:41.141 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.113413+0000", 2023-10-09T06:13:41.141 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.005e-05 2023-10-09T06:13:41.141 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.141 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.141 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:41.141 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.113417+0000", 2023-10-09T06:13:41.141 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3209999999999998e-06 2023-10-09T06:13:41.141 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.141 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.141 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:41.141 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.113458+0000", 2023-10-09T06:13:41.142 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0389000000000002e-05 2023-10-09T06:13:41.142 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.142 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.142 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:41.142 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.113478+0000", 2023-10-09T06:13:41.142 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0188999999999999e-05 2023-10-09T06:13:41.142 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:41.142 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:41.142 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:41.142 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.142 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.143 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop_reply(client.46283.0:106304 10.9 e531/526)", 2023-10-09T06:13:41.143 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:41.115480+0000", 2023-10-09T06:13:41.143 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0012538530000000001, 2023-10-09T06:13:41.143 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001591601, 2023-10-09T06:13:41.143 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:41.143 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "queued for pg", 2023-10-09T06:13:41.143 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:41.143 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.143 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:41.143 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.115480+0000", 2023-10-09T06:13:41.143 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:41.143 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.144 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.144 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:41.144 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.115480+0000", 2023-10-09T06:13:41.144 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:41.144 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.144 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.144 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:41.144 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.115479+0000", 2023-10-09T06:13:41.144 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:13:41.144 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.144 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.145 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:41.145 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.115485+0000", 2023-10-09T06:13:41.145 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5199999999999997e-06 2023-10-09T06:13:41.145 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.145 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.145 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:41.145 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.115494+0000", 2023-10-09T06:13:41.145 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1239999999999994e-06 2023-10-09T06:13:41.145 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.145 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.145 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:41.146 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.115499+0000", 2023-10-09T06:13:41.146 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2129999999999999e-06 2023-10-09T06:13:41.146 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:41.146 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:41.146 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:41.146 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.146 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.146 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:106308 10.d 10:bf3b4b8c:::benchmark_data_smithi064_573750_object10093:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e531)", 2023-10-09T06:13:41.146 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:41.115524+0000", 2023-10-09T06:13:41.146 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0012095610000000001, 2023-10-09T06:13:41.146 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0015980689999999999, 2023-10-09T06:13:41.147 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:41.147 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:13:41.147 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:13:41.147 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:13:41.147 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:13:41.147 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 106308 2023-10-09T06:13:41.147 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.147 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:41.147 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.147 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:41.148 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.115524+0000", 2023-10-09T06:13:41.148 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:41.148 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.148 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.148 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:41.148 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.115524+0000", 2023-10-09T06:13:41.148 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:41.148 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.148 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.148 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:41.148 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.115523+0000", 2023-10-09T06:13:41.148 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:13:41.149 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.149 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.149 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:41.149 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.115531+0000", 2023-10-09T06:13:41.149 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.3070000000000003e-06 2023-10-09T06:13:41.149 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.149 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.149 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:41.149 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.115540+0000", 2023-10-09T06:13:41.149 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1109999999999997e-06 2023-10-09T06:13:41.149 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.150 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.150 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:41.150 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.115545+0000", 2023-10-09T06:13:41.150 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.87e-06 2023-10-09T06:13:41.150 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.150 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.150 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:41.150 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.115654+0000", 2023-10-09T06:13:41.150 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000109134 2023-10-09T06:13:41.150 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.150 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.151 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:41.151 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.115793+0000", 2023-10-09T06:13:41.151 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013855900000000001 2023-10-09T06:13:41.151 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.151 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.151 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T06:13:41.151 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.115933+0000", 2023-10-09T06:13:41.151 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000139784 2023-10-09T06:13:41.151 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:41.151 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:41.151 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:41.151 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.152 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.152 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop_reply(client.46283.0:106304 10.9 e531/526)", 2023-10-09T06:13:41.152 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:41.115573+0000", 2023-10-09T06:13:41.152 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.001160966, 2023-10-09T06:13:41.152 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001618511, 2023-10-09T06:13:41.152 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:41.152 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "queued for pg", 2023-10-09T06:13:41.152 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:41.152 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.152 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:41.152 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.115573+0000", 2023-10-09T06:13:41.152 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:41.153 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.153 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.153 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:41.153 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.115573+0000", 2023-10-09T06:13:41.153 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:41.153 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.153 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.153 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:41.153 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.115573+0000", 2023-10-09T06:13:41.153 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:13:41.153 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.153 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.154 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:41.154 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.115575+0000", 2023-10-09T06:13:41.154 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.8540000000000001e-06 2023-10-09T06:13:41.154 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.154 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.154 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:41.154 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.115584+0000", 2023-10-09T06:13:41.154 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.755e-06 2023-10-09T06:13:41.154 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.154 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.154 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:41.155 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.115588+0000", 2023-10-09T06:13:41.155 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.126e-06 2023-10-09T06:13:41.155 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:41.155 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:41.155 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:41.155 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.155 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.155 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:106292 10.d 10:b285ca87:::benchmark_data_smithi064_573750_object10077:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e531)", 2023-10-09T06:13:41.155 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:41.094562+0000", 2023-10-09T06:13:41.155 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.022172197000000001, 2023-10-09T06:13:41.155 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.022683572999999999, 2023-10-09T06:13:41.156 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:41.156 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:13:41.156 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:13:41.156 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:13:41.156 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:13:41.156 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 106292 2023-10-09T06:13:41.156 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.156 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:41.156 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.156 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:41.156 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.094562+0000", 2023-10-09T06:13:41.156 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:41.156 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.157 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.157 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:41.157 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.094562+0000", 2023-10-09T06:13:41.157 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:41.157 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.157 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.157 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:41.157 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.094561+0000", 2023-10-09T06:13:41.157 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:13:41.157 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.157 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.157 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:41.157 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.094564+0000", 2023-10-09T06:13:41.158 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.576e-06 2023-10-09T06:13:41.158 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.158 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.158 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:41.158 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.094569+0000", 2023-10-09T06:13:41.158 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4870000000000002e-06 2023-10-09T06:13:41.158 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.158 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.158 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:41.158 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.094573+0000", 2023-10-09T06:13:41.158 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.4970000000000001e-06 2023-10-09T06:13:41.159 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.159 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.159 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:41.159 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.097793+0000", 2023-10-09T06:13:41.159 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.003219998 2023-10-09T06:13:41.159 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.159 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.159 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:41.159 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.097916+0000", 2023-10-09T06:13:41.159 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012332399999999999 2023-10-09T06:13:41.159 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.160 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.160 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T06:13:41.160 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.098051+0000", 2023-10-09T06:13:41.160 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013520900000000001 2023-10-09T06:13:41.160 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.160 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.160 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:13:41.160 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.101501+0000", 2023-10-09T06:13:41.160 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0034495960000000001 2023-10-09T06:13:41.160 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.160 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.160 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:41.161 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.105693+0000", 2023-10-09T06:13:41.161 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0041919119999999999 2023-10-09T06:13:41.161 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.161 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.161 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:41.161 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.113372+0000", 2023-10-09T06:13:41.161 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0076791710000000003 2023-10-09T06:13:41.161 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:41.161 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:41.161 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:41.161 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.162 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.162 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:106304 10.9 10:968c1e94:::benchmark_data_smithi064_573750_object10089:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e531)", 2023-10-09T06:13:41.162 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:41.106051+0000", 2023-10-09T06:13:41.162 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.010682861, 2023-10-09T06:13:41.162 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011283694, 2023-10-09T06:13:41.162 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:13:41.162 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:13:41.162 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:13:41.162 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:13:41.162 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:13:41.162 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 106304 2023-10-09T06:13:41.163 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.163 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:13:41.163 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.163 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:13:41.163 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.106051+0000", 2023-10-09T06:13:41.163 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:41.163 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.163 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.163 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:13:41.163 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.106051+0000", 2023-10-09T06:13:41.163 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:13:41.163 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.164 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.164 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:13:41.164 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.106050+0000", 2023-10-09T06:13:41.164 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:13:41.164 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.164 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.164 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:13:41.164 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.106056+0000", 2023-10-09T06:13:41.164 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2600000000000002e-06 2023-10-09T06:13:41.164 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.164 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.165 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:13:41.165 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.106071+0000", 2023-10-09T06:13:41.165 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5099999999999999e-05 2023-10-09T06:13:41.165 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.165 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.165 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:13:41.165 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.106077+0000", 2023-10-09T06:13:41.165 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.049e-06 2023-10-09T06:13:41.165 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.165 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.165 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:13:41.165 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.106130+0000", 2023-10-09T06:13:41.166 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3329999999999999e-05 2023-10-09T06:13:41.166 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.166 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.166 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:13:41.166 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.106278+0000", 2023-10-09T06:13:41.166 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014764300000000001 2023-10-09T06:13:41.166 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.166 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.167 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,3,5,7", 2023-10-09T06:13:41.167 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.106368+0000", 2023-10-09T06:13:41.167 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0095999999999999e-05 2023-10-09T06:13:41.167 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.167 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.167 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:13:41.167 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.109381+0000", 2023-10-09T06:13:41.167 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0030126599999999999 2023-10-09T06:13:41.167 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.167 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.167 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:41.167 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.109662+0000", 2023-10-09T06:13:41.168 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00028113500000000001 2023-10-09T06:13:41.168 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:13:41.168 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:13:41.168 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:13:41.168 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:41.109741+0000", 2023-10-09T06:13:41.168 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9221000000000006e-05 2023-10-09T06:13:41.168 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:41.168 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:13:41.168 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:41.168 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:13:41.168 INFO:teuthology.orchestra.run.smithi156.stdout: ], 2023-10-09T06:13:41.169 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 10 2023-10-09T06:13:41.169 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:13:41.169 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:41.869 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:13:41.902 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:13:42.010 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:13:42.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:42.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:13:42.122 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:13:42.148 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:42.326 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:42 smithi156 ceph-mon[195101]: pgmap v1345: 119 pgs: 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 44592/260907 objects misplaced (17.091%); 48 MiB/s, 774 objects/s recovering 2023-10-09T06:13:42.427 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:13:42.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:42.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:13:42.475 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:42 smithi064 ceph-mon[212750]: pgmap v1345: 119 pgs: 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 44592/260907 objects misplaced (17.091%); 48 MiB/s, 774 objects/s recovering 2023-10-09T06:13:42.475 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:42 smithi064 ceph-mon[220044]: pgmap v1345: 119 pgs: 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 44592/260907 objects misplaced (17.091%); 48 MiB/s, 774 objects/s recovering 2023-10-09T06:13:42.523 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:13:42.532 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:13:42.637 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:42.774 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:13:42.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:42.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:13:42.882 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:13:42.962 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:43.069 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:13:43.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:43.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:13:43.178 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:13:43.181 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:13:43.261 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:43.362 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:13:43.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:43.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:13:43.477 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:13:43.534 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:43.607 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:13:43.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:43.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:13:43.713 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:13:43.714 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:43 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:43.800 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:43.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:43 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:43.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:43 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:44.142 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:13:44.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:44.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:13:44.238 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:13:44.258 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:13:44.308 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:44.351 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:13:44.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:44.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:13:44.454 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:13:44.479 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:44.529 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:13:44.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:44.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:13:44.638 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:13:44.690 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:44.759 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:13:44.964 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:13:44.975 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:44 smithi156 ceph-mon[195101]: pgmap v1346: 119 pgs: 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 44228/258939 objects misplaced (17.080%); 41 MiB/s, 654 objects/s recovering 2023-10-09T06:13:44.975 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:44 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/64209105' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:13:44.975 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:44 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:13:44.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:44.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:13:44.990 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:44 smithi064 ceph-mon[212750]: pgmap v1346: 119 pgs: 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 44228/258939 objects misplaced (17.080%); 41 MiB/s, 654 objects/s recovering 2023-10-09T06:13:44.991 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:44 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/64209105' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:13:44.991 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:44 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:13:44.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:44 smithi064 ceph-mon[220044]: pgmap v1346: 119 pgs: 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 44228/258939 objects misplaced (17.080%); 41 MiB/s, 654 objects/s recovering 2023-10-09T06:13:44.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:44 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/64209105' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:13:44.991 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:44 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:13:45.077 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:13:45.125 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:45.236 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:13:45.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:45.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:13:45.340 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:13:45.405 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:45.504 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:13:45.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:45.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:13:45.608 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:13:45.665 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:13:45.665 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:13:45.665 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T06:13:45.665 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T06:13:45.666 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:13:45.700 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:45.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:13:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:13:45] "GET /metrics HTTP/1.1" 200 36494 "" "Prometheus/2.43.0" 2023-10-09T06:13:45.795 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T06:13:46.125 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:45 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:13:46.125 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:45 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:13:46.125 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:45 smithi156 ceph-mon[195101]: osdmap e532: 8 total, 8 up, 8 in 2023-10-09T06:13:46.126 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:45 smithi156 ceph-mon[195101]: failed to encode map e532 with expected crc 2023-10-09T06:13:46.126 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:45 smithi156 ceph-mon[195101]: failed to encode map e532 with expected crc 2023-10-09T06:13:46.126 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:45 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4133632805' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:13:46.126 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:45 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:13:46.131 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:45 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:13:46.131 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:45 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:13:46.132 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:45 smithi064 ceph-mon[212750]: osdmap e532: 8 total, 8 up, 8 in 2023-10-09T06:13:46.132 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:45 smithi064 ceph-mon[212750]: failed to encode map e532 with expected crc 2023-10-09T06:13:46.132 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:45 smithi064 ceph-mon[212750]: failed to encode map e532 with expected crc 2023-10-09T06:13:46.132 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:45 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4133632805' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:13:46.132 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:45 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:13:46.132 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:45 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:13:46.132 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:45 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:13:46.133 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:45 smithi064 ceph-mon[220044]: osdmap e532: 8 total, 8 up, 8 in 2023-10-09T06:13:46.133 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:45 smithi064 ceph-mon[220044]: failed to encode map e532 with expected crc 2023-10-09T06:13:46.133 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:45 smithi064 ceph-mon[220044]: failed to encode map e532 with expected crc 2023-10-09T06:13:46.133 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:45 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4133632805' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:13:46.133 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:45 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:13:46.342 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:13:46.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:46.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:13:46.445 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:13:46.474 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:46.552 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:13:46.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:46.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:13:46.662 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:13:46.692 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:46.876 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:46 smithi156 ceph-mon[195101]: pgmap v1348: 119 pgs: 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 428 op/s; 44228/258939 objects misplaced (17.080%); 15 MiB/s, 235 objects/s recovering 2023-10-09T06:13:46.876 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:46 smithi156 ceph-mon[195101]: failed to encode map e533 with expected crc 2023-10-09T06:13:46.876 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:46 smithi156 ceph-mon[195101]: failed to encode map e533 with expected crc 2023-10-09T06:13:46.876 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:46 smithi156 ceph-mon[195101]: failed to encode map e533 with expected crc 2023-10-09T06:13:46.877 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:46 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:13:46.877 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:46 smithi156 ceph-mon[195101]: osdmap e533: 8 total, 8 up, 8 in 2023-10-09T06:13:46.877 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:46 smithi156 ceph-mon[195101]: failed to encode map e533 with expected crc 2023-10-09T06:13:46.877 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:46 smithi156 ceph-mon[195101]: failed to encode map e533 with expected crc 2023-10-09T06:13:46.898 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:13:46.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:46.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:13:46.904 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:13:46.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:46 smithi064 ceph-mon[212750]: pgmap v1348: 119 pgs: 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 428 op/s; 44228/258939 objects misplaced (17.080%); 15 MiB/s, 235 objects/s recovering 2023-10-09T06:13:46.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:46 smithi064 ceph-mon[212750]: failed to encode map e533 with expected crc 2023-10-09T06:13:46.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:46 smithi064 ceph-mon[212750]: failed to encode map e533 with expected crc 2023-10-09T06:13:46.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:46 smithi064 ceph-mon[212750]: failed to encode map e533 with expected crc 2023-10-09T06:13:46.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:46 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:13:46.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:46 smithi064 ceph-mon[212750]: osdmap e533: 8 total, 8 up, 8 in 2023-10-09T06:13:46.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:46 smithi064 ceph-mon[212750]: failed to encode map e533 with expected crc 2023-10-09T06:13:46.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:46 smithi064 ceph-mon[212750]: failed to encode map e533 with expected crc 2023-10-09T06:13:46.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:46 smithi064 ceph-mon[220044]: pgmap v1348: 119 pgs: 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 428 op/s; 44228/258939 objects misplaced (17.080%); 15 MiB/s, 235 objects/s recovering 2023-10-09T06:13:46.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:46 smithi064 ceph-mon[220044]: failed to encode map e533 with expected crc 2023-10-09T06:13:46.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:46 smithi064 ceph-mon[220044]: failed to encode map e533 with expected crc 2023-10-09T06:13:46.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:46 smithi064 ceph-mon[220044]: failed to encode map e533 with expected crc 2023-10-09T06:13:46.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:46 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:13:46.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:46 smithi064 ceph-mon[220044]: osdmap e533: 8 total, 8 up, 8 in 2023-10-09T06:13:46.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:46 smithi064 ceph-mon[220044]: failed to encode map e533 with expected crc 2023-10-09T06:13:46.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:46 smithi064 ceph-mon[220044]: failed to encode map e533 with expected crc 2023-10-09T06:13:47.005 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:13:47.078 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:47.191 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:13:47.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:47.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:13:47.235 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T06:13:47.235 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:13:47.317 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:13:47.402 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:47.578 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:13:47.596 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:13:47.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:47.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:13:47.710 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:13:47.764 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:47.898 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:13:47.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:47.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:13:48.004 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:13:48.005 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:13:48.040 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:13:48.053 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:48.085 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:13:48.133 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:13:48.175 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:13:48.211 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:13:48.250 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:13:48.292 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:13:48.334 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:13:48.358 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:13:48.358 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:13:48.358 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:13:48.358 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:13:48.358 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:13:48.358 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:13:48.358 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:13:48.358 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:13:48.593 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:13:48.604 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:13:48.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:48.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:13:48.706 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:13:48.762 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:48.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:48 smithi156 ceph-mon[195101]: pgmap v1350: 119 pgs: 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 5.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 40789/248520 objects misplaced (16.413%); 28 MiB/s, 447 objects/s recovering 2023-10-09T06:13:48.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:48 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1721527396' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:13:48.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:48 smithi064 ceph-mon[212750]: pgmap v1350: 119 pgs: 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 5.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 40789/248520 objects misplaced (16.413%); 28 MiB/s, 447 objects/s recovering 2023-10-09T06:13:48.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:48 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1721527396' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:13:48.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:48 smithi064 ceph-mon[220044]: pgmap v1350: 119 pgs: 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 5.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 40789/248520 objects misplaced (16.413%); 28 MiB/s, 447 objects/s recovering 2023-10-09T06:13:48.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:48 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1721527396' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:13:48.994 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:13:48.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:48.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:13:49.098 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:13:49.123 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:49.382 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:13:49.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:49.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:13:49.489 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:13:49.553 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:49.708 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:13:49.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:49.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:13:49.812 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:13:49.858 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:50.180 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:13:50.180 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:13:50.198 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:13:50.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:50.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:13:50.320 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:13:50.400 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:50.511 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:13:50.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:50.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:13:50.540 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:13:50.616 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:13:50.693 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:50.905 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:13:50.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:50.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:13:50.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:50 smithi064 ceph-mon[212750]: pgmap v1351: 119 pgs: 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 5.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 40789/248520 objects misplaced (16.413%); 13 MiB/s, 211 objects/s recovering 2023-10-09T06:13:50.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:50 smithi064 ceph-mon[220044]: pgmap v1351: 119 pgs: 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 5.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 40789/248520 objects misplaced (16.413%); 13 MiB/s, 211 objects/s recovering 2023-10-09T06:13:51.010 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:13:51.084 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:51.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:50 smithi156 ceph-mon[195101]: pgmap v1351: 119 pgs: 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 5.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 40789/248520 objects misplaced (16.413%); 13 MiB/s, 211 objects/s recovering 2023-10-09T06:13:51.253 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:13:51.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:51.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:13:51.358 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:13:51.381 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:13:51.384 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T06:13:51.393 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:51.448 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:13:51.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:51.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:13:51.474 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:13:51.552 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:13:51.583 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:51.636 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:13:51.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:51.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:13:51.743 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:13:51.774 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:51.830 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:13:51.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:51.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:13:51.937 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:13:51.983 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:52.080 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:13:52.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:52.096 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:51 smithi156 ceph-mon[195101]: osdmap e534: 8 total, 8 up, 8 in 2023-10-09T06:13:52.096 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:51 smithi156 ceph-mon[195101]: failed to encode map e534 with expected crc 2023-10-09T06:13:52.096 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:51 smithi156 ceph-mon[195101]: failed to encode map e534 with expected crc 2023-10-09T06:13:52.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:13:52.198 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:13:52.209 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:13:52.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:51 smithi064 ceph-mon[220044]: osdmap e534: 8 total, 8 up, 8 in 2023-10-09T06:13:52.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:51 smithi064 ceph-mon[220044]: failed to encode map e534 with expected crc 2023-10-09T06:13:52.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:51 smithi064 ceph-mon[220044]: failed to encode map e534 with expected crc 2023-10-09T06:13:52.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:51 smithi064 ceph-mon[212750]: osdmap e534: 8 total, 8 up, 8 in 2023-10-09T06:13:52.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:51 smithi064 ceph-mon[212750]: failed to encode map e534 with expected crc 2023-10-09T06:13:52.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:51 smithi064 ceph-mon[212750]: failed to encode map e534 with expected crc 2023-10-09T06:13:52.246 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:52.347 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:13:52.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:52.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:13:52.455 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:13:52.504 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:52.932 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:13:52.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:52.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:13:53.031 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1353,"stamp":"2023-10-09T06:13:51.159393+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5248549003,"num_objects":80102,"num_object_clones":0,"num_object_copies":240306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":37572,"num_objects_unfound":0,"num_objects_dirty":80102,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":112420,"num_write_kb":6157708,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19936,"num_bytes_recovered":1303998107,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":98641,"ondisk_log_size":98641,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":368,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22146340,"kb_used_data":16594212,"kb_used_omap":165,"kb_used_meta":5551770,"kb_avail":727814876,"statfs":{"total":767960285184,"available":745282433024,"internally_reserved":0,"allocated":16992473088,"data_stored":16974874705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":169911,"internal_metadata":5685012553},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,0,2,7,1,0,0,0,3],"upper_bound":512},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-407044096,"num_objects":-6211,"num_object_clones":0,"num_object_copies":-18633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-6656,"num_objects_unfound":0,"num_objects_dirty":-6211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6211,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3634,"num_bytes_recovered":238157824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.433502"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":772,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.744334+0000","last_change":"2023-10-09T06:13:33.699246+0000","last_active":"2023-10-09T06:13:44.744334+0000","last_peered":"2023-10-09T06:13:44.744334+0000","last_clean":"2023-10-09T06:13:44.744334+0000","last_became_active":"2023-10-09T06:13:33.698733+0000","last_became_peered":"2023-10-09T06:13:33.698733+0000","last_unstale":"2023-10-09T06:13:44.744334+0000","last_undegraded":"2023-10-09T06:13:44.744334+0000","last_fullsized":"2023-10-09T06:13:44.744334+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:05.412941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:05.412941+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:57.044673+0000","scrub_duration":0.0061014329999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":602,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.780220+0000","last_change":"2023-10-09T06:07:43.572773+0000","last_active":"2023-10-09T06:13:45.780220+0000","last_peered":"2023-10-09T06:13:45.780220+0000","last_clean":"2023-10-09T06:13:45.780220+0000","last_became_active":"2023-10-09T06:07:43.572299+0000","last_became_peered":"2023-10-09T06:07:43.572299+0000","last_unstale":"2023-10-09T06:13:45.780220+0000","last_undegraded":"2023-10-09T06:13:45.780220+0000","last_fullsized":"2023-10-09T06:13:45.780220+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:00:37.672435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":343,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.740333+0000","last_change":"2023-10-09T06:05:11.451972+0000","last_active":"2023-10-09T06:13:44.740333+0000","last_peered":"2023-10-09T06:13:44.740333+0000","last_clean":"2023-10-09T06:13:44.740333+0000","last_became_active":"2023-10-09T06:05:11.451448+0000","last_became_peered":"2023-10-09T06:05:11.451448+0000","last_unstale":"2023-10-09T06:13:44.740333+0000","last_undegraded":"2023-10-09T06:13:44.740333+0000","last_fullsized":"2023-10-09T06:13:44.740333+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:39.183621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":833,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.778319+0000","last_change":"2023-10-09T06:09:27.554162+0000","last_active":"2023-10-09T06:13:45.778319+0000","last_peered":"2023-10-09T06:13:45.778319+0000","last_clean":"2023-10-09T06:13:45.778319+0000","last_became_active":"2023-10-09T06:09:27.553659+0000","last_became_peered":"2023-10-09T06:09:27.553659+0000","last_unstale":"2023-10-09T06:13:45.778319+0000","last_undegraded":"2023-10-09T06:13:45.778319+0000","last_fullsized":"2023-10-09T06:13:45.778319+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:01.468346+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:01.468346+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:03.443511+0000","scrub_duration":0.0016258220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":214,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.748593+0000","last_change":"2023-10-09T06:01:54.762060+0000","last_active":"2023-10-09T06:13:44.748593+0000","last_peered":"2023-10-09T06:13:44.748593+0000","last_clean":"2023-10-09T06:13:44.748593+0000","last_became_active":"2023-10-09T06:01:54.761757+0000","last_became_peered":"2023-10-09T06:01:54.761757+0000","last_unstale":"2023-10-09T06:13:44.748593+0000","last_undegraded":"2023-10-09T06:13:44.748593+0000","last_fullsized":"2023-10-09T06:13:44.748593+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:40.294232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":16,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.744829+0000","last_change":"2023-10-09T06:13:30.789834+0000","last_active":"2023-10-09T06:13:44.744829+0000","last_peered":"2023-10-09T06:13:44.744829+0000","last_clean":"2023-10-09T06:13:44.744829+0000","last_became_active":"2023-10-09T06:13:30.789517+0000","last_became_peered":"2023-10-09T06:13:30.789517+0000","last_unstale":"2023-10-09T06:13:44.744829+0000","last_undegraded":"2023-10-09T06:13:44.744829+0000","last_fullsized":"2023-10-09T06:13:44.744829+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:45.467922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":818,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.778757+0000","last_change":"2023-10-09T06:12:07.739121+0000","last_active":"2023-10-09T06:13:45.778757+0000","last_peered":"2023-10-09T06:13:45.778757+0000","last_clean":"2023-10-09T06:13:45.778757+0000","last_became_active":"2023-10-09T06:12:07.738739+0000","last_became_peered":"2023-10-09T06:12:07.738739+0000","last_unstale":"2023-10-09T06:13:45.778757+0000","last_undegraded":"2023-10-09T06:13:45.778757+0000","last_fullsized":"2023-10-09T06:13:45.778757+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.293426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.293426+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:19.815422+0000","scrub_duration":0.001587946,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":325,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.726860+0000","last_change":"2023-10-09T06:09:27.564377+0000","last_active":"2023-10-09T06:13:44.726860+0000","last_peered":"2023-10-09T06:13:44.726860+0000","last_clean":"2023-10-09T06:13:44.726860+0000","last_became_active":"2023-10-09T06:09:27.563710+0000","last_became_peered":"2023-10-09T06:09:27.563710+0000","last_unstale":"2023-10-09T06:13:44.726860+0000","last_undegraded":"2023-10-09T06:13:44.726860+0000","last_fullsized":"2023-10-09T06:13:44.726860+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:48.168447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":449,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.751762+0000","last_change":"2023-10-09T06:13:21.727677+0000","last_active":"2023-10-09T06:13:44.751762+0000","last_peered":"2023-10-09T06:13:44.751762+0000","last_clean":"2023-10-09T06:13:44.751762+0000","last_became_active":"2023-10-09T06:13:21.727321+0000","last_became_peered":"2023-10-09T06:13:21.727321+0000","last_unstale":"2023-10-09T06:13:44.751762+0000","last_undegraded":"2023-10-09T06:13:44.751762+0000","last_fullsized":"2023-10-09T06:13:44.751762+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:10.146447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":751,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.744277+0000","last_change":"2023-10-09T06:13:33.701687+0000","last_active":"2023-10-09T06:13:44.744277+0000","last_peered":"2023-10-09T06:13:44.744277+0000","last_clean":"2023-10-09T06:13:44.744277+0000","last_became_active":"2023-10-09T06:13:33.701233+0000","last_became_peered":"2023-10-09T06:13:33.701233+0000","last_unstale":"2023-10-09T06:13:44.744277+0000","last_undegraded":"2023-10-09T06:13:44.744277+0000","last_fullsized":"2023-10-09T06:13:44.744277+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.384407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.384407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:04.887826+0000","scrub_duration":0.0040651960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":431,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.782464+0000","last_change":"2023-10-09T06:09:27.559756+0000","last_active":"2023-10-09T06:13:45.782464+0000","last_peered":"2023-10-09T06:13:45.782464+0000","last_clean":"2023-10-09T06:13:45.782464+0000","last_became_active":"2023-10-09T06:09:27.559443+0000","last_became_peered":"2023-10-09T06:09:27.559443+0000","last_unstale":"2023-10-09T06:13:45.782464+0000","last_undegraded":"2023-10-09T06:13:45.782464+0000","last_fullsized":"2023-10-09T06:13:45.782464+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:14.519398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":362,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.338400+0000","last_change":"2023-10-09T06:13:21.737897+0000","last_active":"2023-10-09T06:13:45.338400+0000","last_peered":"2023-10-09T06:13:45.338400+0000","last_clean":"2023-10-09T06:13:45.338400+0000","last_became_active":"2023-10-09T06:13:21.737564+0000","last_became_peered":"2023-10-09T06:13:21.737564+0000","last_unstale":"2023-10-09T06:13:45.338400+0000","last_undegraded":"2023-10-09T06:13:45.338400+0000","last_fullsized":"2023-10-09T06:13:45.338400+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:34.863320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":657,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.745441+0000","last_change":"2023-10-09T06:10:11.729656+0000","last_active":"2023-10-09T06:13:44.745441+0000","last_peered":"2023-10-09T06:13:44.745441+0000","last_clean":"2023-10-09T06:13:44.745441+0000","last_became_active":"2023-10-09T06:10:11.728960+0000","last_became_peered":"2023-10-09T06:10:11.728960+0000","last_unstale":"2023-10-09T06:13:44.745441+0000","last_undegraded":"2023-10-09T06:13:44.745441+0000","last_fullsized":"2023-10-09T06:13:44.745441+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:20.742561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:20.742561+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:45.543709+0000","scrub_duration":0.0027005039999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":16,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.744767+0000","last_change":"2023-10-09T06:13:30.802749+0000","last_active":"2023-10-09T06:13:44.744767+0000","last_peered":"2023-10-09T06:13:44.744767+0000","last_clean":"2023-10-09T06:13:44.744767+0000","last_became_active":"2023-10-09T06:13:30.802449+0000","last_became_peered":"2023-10-09T06:13:30.802449+0000","last_unstale":"2023-10-09T06:13:44.744767+0000","last_undegraded":"2023-10-09T06:13:44.744767+0000","last_fullsized":"2023-10-09T06:13:44.744767+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:29.017181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":519,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.727919+0000","last_change":"2023-10-09T06:13:30.792445+0000","last_active":"2023-10-09T06:13:44.727919+0000","last_peered":"2023-10-09T06:13:44.727919+0000","last_clean":"2023-10-09T06:13:44.727919+0000","last_became_active":"2023-10-09T06:13:30.792116+0000","last_became_peered":"2023-10-09T06:13:30.792116+0000","last_unstale":"2023-10-09T06:13:44.727919+0000","last_undegraded":"2023-10-09T06:13:44.727919+0000","last_fullsized":"2023-10-09T06:13:44.727919+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:20:53.506293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":744,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.746478+0000","last_change":"2023-10-09T06:13:24.760843+0000","last_active":"2023-10-09T06:13:44.746478+0000","last_peered":"2023-10-09T06:13:44.746478+0000","last_clean":"2023-10-09T06:13:44.746478+0000","last_became_active":"2023-10-09T06:13:24.760333+0000","last_became_peered":"2023-10-09T06:13:24.760333+0000","last_unstale":"2023-10-09T06:13:44.746478+0000","last_undegraded":"2023-10-09T06:13:44.746478+0000","last_fullsized":"2023-10-09T06:13:44.746478+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:38.588641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:38.588641+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:30.660400+0000","scrub_duration":0.010116385,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":22,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.744197+0000","last_change":"2023-10-09T06:13:30.826588+0000","last_active":"2023-10-09T06:13:44.744197+0000","last_peered":"2023-10-09T06:13:44.744197+0000","last_clean":"2023-10-09T06:13:44.744197+0000","last_became_active":"2023-10-09T06:13:30.799199+0000","last_became_peered":"2023-10-09T06:13:30.799199+0000","last_unstale":"2023-10-09T06:13:44.744197+0000","last_undegraded":"2023-10-09T06:13:44.744197+0000","last_fullsized":"2023-10-09T06:13:44.744197+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:38.703864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":314,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.746551+0000","last_change":"2023-10-09T06:13:21.739214+0000","last_active":"2023-10-09T06:13:44.746551+0000","last_peered":"2023-10-09T06:13:44.746551+0000","last_clean":"2023-10-09T06:13:44.746551+0000","last_became_active":"2023-10-09T06:13:21.738951+0000","last_became_peered":"2023-10-09T06:13:21.738951+0000","last_unstale":"2023-10-09T06:13:44.746551+0000","last_undegraded":"2023-10-09T06:13:44.746551+0000","last_fullsized":"2023-10-09T06:13:44.746551+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:17.477241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":704,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.751672+0000","last_change":"2023-10-09T06:09:27.574336+0000","last_active":"2023-10-09T06:13:44.751672+0000","last_peered":"2023-10-09T06:13:44.751672+0000","last_clean":"2023-10-09T06:13:44.751672+0000","last_became_active":"2023-10-09T06:09:27.573974+0000","last_became_peered":"2023-10-09T06:09:27.573974+0000","last_unstale":"2023-10-09T06:13:44.751672+0000","last_undegraded":"2023-10-09T06:13:44.751672+0000","last_fullsized":"2023-10-09T06:13:44.751672+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.339121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.339121+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:45.908579+0000","scrub_duration":0.010474344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":568,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.779652+0000","last_change":"2023-10-09T06:13:30.812598+0000","last_active":"2023-10-09T06:13:45.779652+0000","last_peered":"2023-10-09T06:13:45.779652+0000","last_clean":"2023-10-09T06:13:45.779652+0000","last_became_active":"2023-10-09T06:13:30.791460+0000","last_became_peered":"2023-10-09T06:13:30.791460+0000","last_unstale":"2023-10-09T06:13:45.779652+0000","last_undegraded":"2023-10-09T06:13:45.779652+0000","last_fullsized":"2023-10-09T06:13:45.779652+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:28.046730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":546,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.778592+0000","last_change":"2023-10-09T06:01:54.764916+0000","last_active":"2023-10-09T06:13:45.778592+0000","last_peered":"2023-10-09T06:13:45.778592+0000","last_clean":"2023-10-09T06:13:45.778592+0000","last_became_active":"2023-10-09T06:01:54.764705+0000","last_became_peered":"2023-10-09T06:01:54.764705+0000","last_unstale":"2023-10-09T06:13:45.778592+0000","last_undegraded":"2023-10-09T06:13:45.778592+0000","last_fullsized":"2023-10-09T06:13:45.778592+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:12.527719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":761,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.727557+0000","last_change":"2023-10-09T06:13:15.425525+0000","last_active":"2023-10-09T06:13:44.727557+0000","last_peered":"2023-10-09T06:13:44.727557+0000","last_clean":"2023-10-09T06:13:44.727557+0000","last_became_active":"2023-10-09T06:13:15.425086+0000","last_became_peered":"2023-10-09T06:13:15.425086+0000","last_unstale":"2023-10-09T06:13:44.727557+0000","last_undegraded":"2023-10-09T06:13:44.727557+0000","last_fullsized":"2023-10-09T06:13:44.727557+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:42.900876+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:42.900876+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:46.042715+0000","scrub_duration":0.0014928649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":22,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.744101+0000","last_change":"2023-10-09T06:13:30.811420+0000","last_active":"2023-10-09T06:13:44.744101+0000","last_peered":"2023-10-09T06:13:44.744101+0000","last_clean":"2023-10-09T06:13:44.744101+0000","last_became_active":"2023-10-09T06:13:30.792835+0000","last_became_peered":"2023-10-09T06:13:30.792835+0000","last_unstale":"2023-10-09T06:13:44.744101+0000","last_undegraded":"2023-10-09T06:13:44.744101+0000","last_fullsized":"2023-10-09T06:13:44.744101+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:38.962645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":578,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.733666+0000","last_change":"2023-10-09T06:01:54.761558+0000","last_active":"2023-10-09T06:13:44.733666+0000","last_peered":"2023-10-09T06:13:44.733666+0000","last_clean":"2023-10-09T06:13:44.733666+0000","last_became_active":"2023-10-09T06:01:54.761251+0000","last_became_peered":"2023-10-09T06:01:54.761251+0000","last_unstale":"2023-10-09T06:13:44.733666+0000","last_undegraded":"2023-10-09T06:13:44.733666+0000","last_fullsized":"2023-10-09T06:13:44.733666+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:53.245267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":1036,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.744465+0000","last_change":"2023-10-09T06:13:24.754018+0000","last_active":"2023-10-09T06:13:44.744465+0000","last_peered":"2023-10-09T06:13:44.744465+0000","last_clean":"2023-10-09T06:13:44.744465+0000","last_became_active":"2023-10-09T06:13:24.753712+0000","last_became_peered":"2023-10-09T06:13:24.753712+0000","last_unstale":"2023-10-09T06:13:44.744465+0000","last_undegraded":"2023-10-09T06:13:44.744465+0000","last_fullsized":"2023-10-09T06:13:44.744465+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:18.490833+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":16,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.744033+0000","last_change":"2023-10-09T06:13:30.788160+0000","last_active":"2023-10-09T06:13:44.744033+0000","last_peered":"2023-10-09T06:13:44.744033+0000","last_clean":"2023-10-09T06:13:44.744033+0000","last_became_active":"2023-10-09T06:13:30.787724+0000","last_became_peered":"2023-10-09T06:13:30.787724+0000","last_unstale":"2023-10-09T06:13:44.744033+0000","last_undegraded":"2023-10-09T06:13:44.744033+0000","last_fullsized":"2023-10-09T06:13:44.744033+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:38.376770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":16,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.744428+0000","last_change":"2023-10-09T06:13:30.789225+0000","last_active":"2023-10-09T06:13:44.744428+0000","last_peered":"2023-10-09T06:13:44.744428+0000","last_clean":"2023-10-09T06:13:44.744428+0000","last_became_active":"2023-10-09T06:13:30.788869+0000","last_became_peered":"2023-10-09T06:13:30.788869+0000","last_unstale":"2023-10-09T06:13:44.744428+0000","last_undegraded":"2023-10-09T06:13:44.744428+0000","last_fullsized":"2023-10-09T06:13:44.744428+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:19.519966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":773,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.758483+0000","last_change":"2023-10-09T06:12:55.750923+0000","last_active":"2023-10-09T06:13:44.758483+0000","last_peered":"2023-10-09T06:13:44.758483+0000","last_clean":"2023-10-09T06:13:44.758483+0000","last_became_active":"2023-10-09T06:12:55.750521+0000","last_became_peered":"2023-10-09T06:12:55.750521+0000","last_unstale":"2023-10-09T06:13:44.758483+0000","last_undegraded":"2023-10-09T06:13:44.758483+0000","last_fullsized":"2023-10-09T06:13:44.758483+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.316697+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.316697+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:23.591439+0000","scrub_duration":0.001666561,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":386,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.745257+0000","last_change":"2023-10-09T06:01:22.337171+0000","last_active":"2023-10-09T06:13:44.745257+0000","last_peered":"2023-10-09T06:13:44.745257+0000","last_clean":"2023-10-09T06:13:44.745257+0000","last_became_active":"2023-10-09T06:01:22.335438+0000","last_became_peered":"2023-10-09T06:01:22.335438+0000","last_unstale":"2023-10-09T06:13:44.745257+0000","last_undegraded":"2023-10-09T06:13:44.745257+0000","last_fullsized":"2023-10-09T06:13:44.745257+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:25.036448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":495,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.781063+0000","last_change":"2023-10-09T06:13:30.788284+0000","last_active":"2023-10-09T06:13:45.781063+0000","last_peered":"2023-10-09T06:13:45.781063+0000","last_clean":"2023-10-09T06:13:45.781063+0000","last_became_active":"2023-10-09T06:13:30.787933+0000","last_became_peered":"2023-10-09T06:13:30.787933+0000","last_unstale":"2023-10-09T06:13:45.781063+0000","last_undegraded":"2023-10-09T06:13:45.781063+0000","last_fullsized":"2023-10-09T06:13:45.781063+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:10.735438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":823,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.750695+0000","last_change":"2023-10-09T06:13:24.753908+0000","last_active":"2023-10-09T06:13:44.750695+0000","last_peered":"2023-10-09T06:13:44.750695+0000","last_clean":"2023-10-09T06:13:44.750695+0000","last_became_active":"2023-10-09T06:13:24.753566+0000","last_became_peered":"2023-10-09T06:13:24.753566+0000","last_unstale":"2023-10-09T06:13:44.750695+0000","last_undegraded":"2023-10-09T06:13:44.750695+0000","last_fullsized":"2023-10-09T06:13:44.750695+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:40.388125+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:40.388125+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:35.101978+0000","scrub_duration":0.0055576990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":106,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.775417+0000","last_change":"2023-10-09T06:13:12.243605+0000","last_active":"2023-10-09T06:13:45.775417+0000","last_peered":"2023-10-09T06:13:45.775417+0000","last_clean":"2023-10-09T06:13:45.775417+0000","last_became_active":"2023-10-09T06:13:12.237061+0000","last_became_peered":"2023-10-09T06:13:12.237061+0000","last_unstale":"2023-10-09T06:13:45.775417+0000","last_undegraded":"2023-10-09T06:13:45.775417+0000","last_fullsized":"2023-10-09T06:13:45.775417+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:29:03.666031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":43,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.742575+0000","last_change":"2023-10-09T06:13:30.788733+0000","last_active":"2023-10-09T06:13:44.742575+0000","last_peered":"2023-10-09T06:13:44.742575+0000","last_clean":"2023-10-09T06:13:44.742575+0000","last_became_active":"2023-10-09T06:13:30.788353+0000","last_became_peered":"2023-10-09T06:13:30.788353+0000","last_unstale":"2023-10-09T06:13:44.742575+0000","last_undegraded":"2023-10-09T06:13:44.742575+0000","last_fullsized":"2023-10-09T06:13:44.742575+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:06.308426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"534'7055","reported_seq":7108,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.028677+0000","last_change":"2023-10-09T06:11:46.032384+0000","last_active":"2023-10-09T06:13:51.028677+0000","last_peered":"2023-10-09T06:13:51.028677+0000","last_clean":"2023-10-09T06:13:51.028677+0000","last_became_active":"2023-10-09T06:11:46.032091+0000","last_became_peered":"2023-10-09T06:11:46.032091+0000","last_unstale":"2023-10-09T06:13:51.028677+0000","last_undegraded":"2023-10-09T06:13:51.028677+0000","last_fullsized":"2023-10-09T06:13:51.028677+0000","mapping_epoch":490,"log_start":"498'900","ondisk_log_start":"498'900","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6155,"log_dups_size":0,"ondisk_log_size":6155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:31.117384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327483392,"num_objects":4997,"num_object_clones":0,"num_object_copies":14991,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4997,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7055,"num_write_kb":385664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":714,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.744502+0000","last_change":"2023-10-09T06:13:33.701521+0000","last_active":"2023-10-09T06:13:44.744502+0000","last_peered":"2023-10-09T06:13:44.744502+0000","last_clean":"2023-10-09T06:13:44.744502+0000","last_became_active":"2023-10-09T06:13:33.700870+0000","last_became_peered":"2023-10-09T06:13:33.700870+0000","last_unstale":"2023-10-09T06:13:44.744502+0000","last_undegraded":"2023-10-09T06:13:44.744502+0000","last_fullsized":"2023-10-09T06:13:44.744502+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:10:59.422551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:10:59.422551+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:54.402494+0000","scrub_duration":0.0059716659999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":20,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.745595+0000","last_change":"2023-10-09T06:13:30.822903+0000","last_active":"2023-10-09T06:13:44.745595+0000","last_peered":"2023-10-09T06:13:44.745595+0000","last_clean":"2023-10-09T06:13:44.745595+0000","last_became_active":"2023-10-09T06:13:30.799013+0000","last_became_peered":"2023-10-09T06:13:30.799013+0000","last_unstale":"2023-10-09T06:13:44.745595+0000","last_undegraded":"2023-10-09T06:13:44.745595+0000","last_fullsized":"2023-10-09T06:13:44.745595+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:44.271821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":16,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.744291+0000","last_change":"2023-10-09T06:13:30.793531+0000","last_active":"2023-10-09T06:13:44.744291+0000","last_peered":"2023-10-09T06:13:44.744291+0000","last_clean":"2023-10-09T06:13:44.744291+0000","last_became_active":"2023-10-09T06:13:30.792689+0000","last_became_peered":"2023-10-09T06:13:30.792689+0000","last_unstale":"2023-10-09T06:13:44.744291+0000","last_undegraded":"2023-10-09T06:13:44.744291+0000","last_fullsized":"2023-10-09T06:13:44.744291+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:02.559186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"533'7035","reported_seq":8715,"reported_epoch":533,"state":"active+remapped","last_fresh":"2023-10-09T06:13:50.319572+0000","last_change":"2023-10-09T06:13:30.811777+0000","last_active":"2023-10-09T06:13:50.319572+0000","last_peered":"2023-10-09T06:13:50.319572+0000","last_clean":"2023-10-09T06:13:11.194737+0000","last_became_active":"2023-10-09T06:13:21.782116+0000","last_became_peered":"2023-10-09T06:13:21.782116+0000","last_unstale":"2023-10-09T06:13:50.319572+0000","last_undegraded":"2023-10-09T06:13:50.319572+0000","last_fullsized":"2023-10-09T06:13:50.319572+0000","mapping_epoch":521,"log_start":"498'1193","ondisk_log_start":"498'1193","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5842,"log_dups_size":0,"ondisk_log_size":5842,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:00.656766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330760192,"num_objects":5047,"num_object_clones":0,"num_object_copies":15141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":9414,"num_objects_unfound":0,"num_objects_dirty":5047,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7035,"num_write_kb":386624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":801,"num_bytes_recovered":52494336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,2,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":5047}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":902,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.733443+0000","last_change":"2023-10-09T06:13:24.753866+0000","last_active":"2023-10-09T06:13:44.733443+0000","last_peered":"2023-10-09T06:13:44.733443+0000","last_clean":"2023-10-09T06:13:44.733443+0000","last_became_active":"2023-10-09T06:13:24.753432+0000","last_became_peered":"2023-10-09T06:13:24.753432+0000","last_unstale":"2023-10-09T06:13:44.733443+0000","last_undegraded":"2023-10-09T06:13:44.733443+0000","last_fullsized":"2023-10-09T06:13:44.733443+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:03.241335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:03.241335+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:19.854449+0000","scrub_duration":0.017780547000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":20,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.744697+0000","last_change":"2023-10-09T06:13:30.829763+0000","last_active":"2023-10-09T06:13:44.744697+0000","last_peered":"2023-10-09T06:13:44.744697+0000","last_clean":"2023-10-09T06:13:44.744697+0000","last_became_active":"2023-10-09T06:13:30.802486+0000","last_became_peered":"2023-10-09T06:13:30.802486+0000","last_unstale":"2023-10-09T06:13:44.744697+0000","last_undegraded":"2023-10-09T06:13:44.744697+0000","last_fullsized":"2023-10-09T06:13:44.744697+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:05.468205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":572,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.778506+0000","last_change":"2023-10-09T06:01:54.760367+0000","last_active":"2023-10-09T06:13:45.778506+0000","last_peered":"2023-10-09T06:13:45.778506+0000","last_clean":"2023-10-09T06:13:45.778506+0000","last_became_active":"2023-10-09T06:01:54.760087+0000","last_became_peered":"2023-10-09T06:01:54.760087+0000","last_unstale":"2023-10-09T06:13:45.778506+0000","last_undegraded":"2023-10-09T06:13:45.778506+0000","last_fullsized":"2023-10-09T06:13:45.778506+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:26.000104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"534'7056","reported_seq":7108,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:13:50.836322+0000","last_change":"2023-10-09T06:11:45.696810+0000","last_active":"2023-10-09T06:13:50.836322+0000","last_peered":"2023-10-09T06:13:50.836322+0000","last_clean":"2023-10-09T06:13:50.836322+0000","last_became_active":"2023-10-09T06:11:45.696476+0000","last_became_peered":"2023-10-09T06:11:45.696476+0000","last_unstale":"2023-10-09T06:13:50.836322+0000","last_undegraded":"2023-10-09T06:13:50.836322+0000","last_fullsized":"2023-10-09T06:13:50.836322+0000","mapping_epoch":490,"log_start":"502'1600","ondisk_log_start":"502'1600","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5456,"log_dups_size":0,"ondisk_log_size":5456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:41:33.351567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319553536,"num_objects":4876,"num_object_clones":0,"num_object_copies":14628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7056,"num_write_kb":381824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":797,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.778414+0000","last_change":"2023-10-09T06:13:24.768958+0000","last_active":"2023-10-09T06:13:45.778414+0000","last_peered":"2023-10-09T06:13:45.778414+0000","last_clean":"2023-10-09T06:13:45.778414+0000","last_became_active":"2023-10-09T06:13:24.768643+0000","last_became_peered":"2023-10-09T06:13:24.768643+0000","last_unstale":"2023-10-09T06:13:45.778414+0000","last_undegraded":"2023-10-09T06:13:45.778414+0000","last_fullsized":"2023-10-09T06:13:45.778414+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:48.495739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:48.495739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:18:50.654354+0000","scrub_duration":0.0016345509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":20,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.744356+0000","last_change":"2023-10-09T06:13:30.814434+0000","last_active":"2023-10-09T06:13:44.744356+0000","last_peered":"2023-10-09T06:13:44.744356+0000","last_clean":"2023-10-09T06:13:44.744356+0000","last_became_active":"2023-10-09T06:13:30.793119+0000","last_became_peered":"2023-10-09T06:13:30.793119+0000","last_unstale":"2023-10-09T06:13:44.744356+0000","last_undegraded":"2023-10-09T06:13:44.744356+0000","last_fullsized":"2023-10-09T06:13:44.744356+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:34:31.724006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":557,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.750348+0000","last_change":"2023-10-09T06:13:30.792490+0000","last_active":"2023-10-09T06:13:44.750348+0000","last_peered":"2023-10-09T06:13:44.750348+0000","last_clean":"2023-10-09T06:13:44.750348+0000","last_became_active":"2023-10-09T06:13:30.792209+0000","last_became_peered":"2023-10-09T06:13:30.792209+0000","last_unstale":"2023-10-09T06:13:44.750348+0000","last_undegraded":"2023-10-09T06:13:44.750348+0000","last_fullsized":"2023-10-09T06:13:44.750348+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:14.208042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"534'6950","reported_seq":7003,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.011738+0000","last_change":"2023-10-09T06:11:46.031195+0000","last_active":"2023-10-09T06:13:51.011738+0000","last_peered":"2023-10-09T06:13:51.011738+0000","last_clean":"2023-10-09T06:13:51.011738+0000","last_became_active":"2023-10-09T06:11:46.030891+0000","last_became_peered":"2023-10-09T06:11:46.030891+0000","last_unstale":"2023-10-09T06:13:51.011738+0000","last_undegraded":"2023-10-09T06:13:51.011738+0000","last_fullsized":"2023-10-09T06:13:51.011738+0000","mapping_epoch":490,"log_start":"498'800","ondisk_log_start":"498'800","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6150,"log_dups_size":0,"ondisk_log_size":6150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:25.788661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325451776,"num_objects":4966,"num_object_clones":0,"num_object_copies":14898,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4966,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6950,"num_write_kb":381312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":749,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.778888+0000","last_change":"2023-10-09T06:13:33.708291+0000","last_active":"2023-10-09T06:13:45.778888+0000","last_peered":"2023-10-09T06:13:45.778888+0000","last_clean":"2023-10-09T06:13:45.778888+0000","last_became_active":"2023-10-09T06:13:33.707930+0000","last_became_peered":"2023-10-09T06:13:33.707930+0000","last_unstale":"2023-10-09T06:13:45.778888+0000","last_undegraded":"2023-10-09T06:13:45.778888+0000","last_fullsized":"2023-10-09T06:13:45.778888+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:16.361097+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:16.361097+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:18.111404+0000","scrub_duration":0.028533749000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":302,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.733530+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T06:13:44.733530+0000","last_peered":"2023-10-09T06:13:44.733530+0000","last_clean":"2023-10-09T06:13:44.733530+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T06:13:44.733530+0000","last_undegraded":"2023-10-09T06:13:44.733530+0000","last_fullsized":"2023-10-09T06:13:44.733530+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":546,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.772563+0000","last_change":"2023-10-09T06:09:27.558353+0000","last_active":"2023-10-09T06:13:45.772563+0000","last_peered":"2023-10-09T06:13:45.772563+0000","last_clean":"2023-10-09T06:13:45.772563+0000","last_became_active":"2023-10-09T06:09:27.557233+0000","last_became_peered":"2023-10-09T06:09:27.557233+0000","last_unstale":"2023-10-09T06:13:45.772563+0000","last_undegraded":"2023-10-09T06:13:45.772563+0000","last_fullsized":"2023-10-09T06:13:45.772563+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:53.149350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"533'6845","reported_seq":6897,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:46.451889+0000","last_change":"2023-10-09T06:11:45.694648+0000","last_active":"2023-10-09T06:13:46.451889+0000","last_peered":"2023-10-09T06:13:46.451889+0000","last_clean":"2023-10-09T06:13:46.451889+0000","last_became_active":"2023-10-09T06:11:45.694321+0000","last_became_peered":"2023-10-09T06:11:45.694321+0000","last_unstale":"2023-10-09T06:13:46.451889+0000","last_undegraded":"2023-10-09T06:13:46.451889+0000","last_fullsized":"2023-10-09T06:13:46.451889+0000","mapping_epoch":490,"log_start":"498'700","ondisk_log_start":"498'700","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6145,"log_dups_size":0,"ondisk_log_size":6145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:43.704525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332726272,"num_objects":5077,"num_object_clones":0,"num_object_copies":15231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6845,"num_write_kb":381504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":919,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.745168+0000","last_change":"2023-10-09T06:13:33.716191+0000","last_active":"2023-10-09T06:13:44.745168+0000","last_peered":"2023-10-09T06:13:44.745168+0000","last_clean":"2023-10-09T06:13:44.745168+0000","last_became_active":"2023-10-09T06:13:33.715831+0000","last_became_peered":"2023-10-09T06:13:33.715831+0000","last_unstale":"2023-10-09T06:13:44.745168+0000","last_undegraded":"2023-10-09T06:13:44.745168+0000","last_fullsized":"2023-10-09T06:13:44.745168+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:42.795781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:42.795781+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:45.712738+0000","scrub_duration":0.0091262979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":336,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.728082+0000","last_change":"2023-10-09T06:09:27.565666+0000","last_active":"2023-10-09T06:13:44.728082+0000","last_peered":"2023-10-09T06:13:44.728082+0000","last_clean":"2023-10-09T06:13:44.728082+0000","last_became_active":"2023-10-09T06:09:27.565324+0000","last_became_peered":"2023-10-09T06:09:27.565324+0000","last_unstale":"2023-10-09T06:13:44.728082+0000","last_undegraded":"2023-10-09T06:13:44.728082+0000","last_fullsized":"2023-10-09T06:13:44.728082+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:51.216947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":508,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.779725+0000","last_change":"2023-10-09T06:13:21.727745+0000","last_active":"2023-10-09T06:13:45.779725+0000","last_peered":"2023-10-09T06:13:45.779725+0000","last_clean":"2023-10-09T06:13:45.779725+0000","last_became_active":"2023-10-09T06:13:21.727404+0000","last_became_peered":"2023-10-09T06:13:21.727404+0000","last_unstale":"2023-10-09T06:13:45.779725+0000","last_undegraded":"2023-10-09T06:13:45.779725+0000","last_fullsized":"2023-10-09T06:13:45.779725+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:24.931626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"533'7136","reported_seq":18682,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:50.219010+0000","last_change":"2023-10-09T06:13:32.697002+0000","last_active":"2023-10-09T06:13:50.219010+0000","last_peered":"2023-10-09T06:13:50.219010+0000","last_clean":"2023-10-09T06:13:50.219010+0000","last_became_active":"2023-10-09T06:13:32.683104+0000","last_became_peered":"2023-10-09T06:13:32.683104+0000","last_unstale":"2023-10-09T06:13:50.219010+0000","last_undegraded":"2023-10-09T06:13:50.219010+0000","last_fullsized":"2023-10-09T06:13:50.219010+0000","mapping_epoch":528,"log_start":"498'1235","ondisk_log_start":"498'1235","created":490,"last_epoch_clean":529,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5901,"log_dups_size":0,"ondisk_log_size":5901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:19.426571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336723968,"num_objects":5138,"num_object_clones":0,"num_object_copies":15414,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5138,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7136,"num_write_kb":392768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5732,"num_bytes_recovered":375652352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":795,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.726691+0000","last_change":"2023-10-09T06:13:15.442141+0000","last_active":"2023-10-09T06:13:44.726691+0000","last_peered":"2023-10-09T06:13:44.726691+0000","last_clean":"2023-10-09T06:13:44.726691+0000","last_became_active":"2023-10-09T06:13:15.441716+0000","last_became_peered":"2023-10-09T06:13:15.441716+0000","last_unstale":"2023-10-09T06:13:44.726691+0000","last_undegraded":"2023-10-09T06:13:44.726691+0000","last_fullsized":"2023-10-09T06:13:44.726691+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.282393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.282393+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:16.798440+0000","scrub_duration":0.01402403,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":453,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.778887+0000","last_change":"2023-10-09T06:13:21.738218+0000","last_active":"2023-10-09T06:13:45.778887+0000","last_peered":"2023-10-09T06:13:45.778887+0000","last_clean":"2023-10-09T06:13:45.778887+0000","last_became_active":"2023-10-09T06:13:21.737969+0000","last_became_peered":"2023-10-09T06:13:21.737969+0000","last_unstale":"2023-10-09T06:13:45.778887+0000","last_undegraded":"2023-10-09T06:13:45.778887+0000","last_fullsized":"2023-10-09T06:13:45.778887+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:38.620281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":582,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.779207+0000","last_change":"2023-10-09T06:07:43.572173+0000","last_active":"2023-10-09T06:13:45.779207+0000","last_peered":"2023-10-09T06:13:45.779207+0000","last_clean":"2023-10-09T06:13:45.779207+0000","last_became_active":"2023-10-09T06:07:43.571690+0000","last_became_peered":"2023-10-09T06:07:43.571690+0000","last_unstale":"2023-10-09T06:13:45.779207+0000","last_undegraded":"2023-10-09T06:13:45.779207+0000","last_fullsized":"2023-10-09T06:13:45.779207+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:10.234125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"533'7005","reported_seq":7077,"reported_epoch":533,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:13:46.545844+0000","last_change":"2023-10-09T06:13:22.855496+0000","last_active":"2023-10-09T06:13:46.545844+0000","last_peered":"2023-10-09T06:13:46.545844+0000","last_clean":"2023-10-09T06:13:20.539588+0000","last_became_active":"2023-10-09T06:13:22.846120+0000","last_became_peered":"2023-10-09T06:13:22.846120+0000","last_unstale":"2023-10-09T06:13:46.545844+0000","last_undegraded":"2023-10-09T06:13:46.545844+0000","last_fullsized":"2023-10-09T06:13:46.545844+0000","mapping_epoch":522,"log_start":"498'900","ondisk_log_start":"498'900","created":490,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6105,"log_dups_size":0,"ondisk_log_size":6105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:55.561925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342425600,"num_objects":5225,"num_object_clones":0,"num_object_copies":15675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5225,"num_objects_unfound":0,"num_objects_dirty":5225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7005,"num_write_kb":391360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,4,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":5225}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":867,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.758130+0000","last_change":"2023-10-09T06:13:15.428701+0000","last_active":"2023-10-09T06:13:44.758130+0000","last_peered":"2023-10-09T06:13:44.758130+0000","last_clean":"2023-10-09T06:13:44.758130+0000","last_became_active":"2023-10-09T06:13:15.428343+0000","last_became_peered":"2023-10-09T06:13:15.428343+0000","last_unstale":"2023-10-09T06:13:44.758130+0000","last_undegraded":"2023-10-09T06:13:44.758130+0000","last_fullsized":"2023-10-09T06:13:44.758130+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.373748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.373748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:55:34.235074+0000","scrub_duration":0.0024576450000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":524,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.778641+0000","last_change":"2023-10-09T06:07:43.574863+0000","last_active":"2023-10-09T06:13:45.778641+0000","last_peered":"2023-10-09T06:13:45.778641+0000","last_clean":"2023-10-09T06:13:45.778641+0000","last_became_active":"2023-10-09T06:07:43.574576+0000","last_became_peered":"2023-10-09T06:07:43.574576+0000","last_unstale":"2023-10-09T06:13:45.778641+0000","last_undegraded":"2023-10-09T06:13:45.778641+0000","last_fullsized":"2023-10-09T06:13:45.778641+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:16.166004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":565,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.778956+0000","last_change":"2023-10-09T06:13:21.727814+0000","last_active":"2023-10-09T06:13:45.778956+0000","last_peered":"2023-10-09T06:13:45.778956+0000","last_clean":"2023-10-09T06:13:45.778956+0000","last_became_active":"2023-10-09T06:13:21.727401+0000","last_became_peered":"2023-10-09T06:13:21.727401+0000","last_unstale":"2023-10-09T06:13:45.778956+0000","last_undegraded":"2023-10-09T06:13:45.778956+0000","last_fullsized":"2023-10-09T06:13:45.778956+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:53:37.968028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"533'7151","reported_seq":7203,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:50.789980+0000","last_change":"2023-10-09T06:11:45.695709+0000","last_active":"2023-10-09T06:13:50.789980+0000","last_peered":"2023-10-09T06:13:50.789980+0000","last_clean":"2023-10-09T06:13:50.789980+0000","last_became_active":"2023-10-09T06:11:45.695219+0000","last_became_peered":"2023-10-09T06:11:45.695219+0000","last_unstale":"2023-10-09T06:13:50.789980+0000","last_undegraded":"2023-10-09T06:13:50.789980+0000","last_fullsized":"2023-10-09T06:13:50.789980+0000","mapping_epoch":490,"log_start":"502'1700","ondisk_log_start":"502'1700","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5451,"log_dups_size":0,"ondisk_log_size":5451,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:26.533964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325124096,"num_objects":4961,"num_object_clones":0,"num_object_copies":14883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7151,"num_write_kb":387584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":611,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.746808+0000","last_change":"2023-10-09T06:13:24.753519+0000","last_active":"2023-10-09T06:13:44.746808+0000","last_peered":"2023-10-09T06:13:44.746808+0000","last_clean":"2023-10-09T06:13:44.746808+0000","last_became_active":"2023-10-09T06:13:24.753185+0000","last_became_peered":"2023-10-09T06:13:24.753185+0000","last_unstale":"2023-10-09T06:13:44.746808+0000","last_undegraded":"2023-10-09T06:13:44.746808+0000","last_fullsized":"2023-10-09T06:13:44.746808+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:39.633410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:39.633410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:56.871057+0000","scrub_duration":0.001547624,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":344,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.726787+0000","last_change":"2023-10-09T06:13:30.781849+0000","last_active":"2023-10-09T06:13:44.726787+0000","last_peered":"2023-10-09T06:13:44.726787+0000","last_clean":"2023-10-09T06:13:44.726787+0000","last_became_active":"2023-10-09T06:13:30.781084+0000","last_became_peered":"2023-10-09T06:13:30.781084+0000","last_unstale":"2023-10-09T06:13:44.726787+0000","last_undegraded":"2023-10-09T06:13:44.726787+0000","last_fullsized":"2023-10-09T06:13:44.726787+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:02.199298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":215,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.751910+0000","last_change":"2023-10-09T06:01:54.761768+0000","last_active":"2023-10-09T06:13:44.751910+0000","last_peered":"2023-10-09T06:13:44.751910+0000","last_clean":"2023-10-09T06:13:44.751910+0000","last_became_active":"2023-10-09T06:01:54.761443+0000","last_became_peered":"2023-10-09T06:01:54.761443+0000","last_unstale":"2023-10-09T06:13:44.751910+0000","last_undegraded":"2023-10-09T06:13:44.751910+0000","last_fullsized":"2023-10-09T06:13:44.751910+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:51.555987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"534'7118","reported_seq":7176,"reported_epoch":534,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:13:51.010303+0000","last_change":"2023-10-09T06:13:31.904728+0000","last_active":"2023-10-09T06:13:51.010303+0000","last_peered":"2023-10-09T06:13:51.010303+0000","last_clean":"2023-10-09T06:13:29.715670+0000","last_became_active":"2023-10-09T06:13:31.891405+0000","last_became_peered":"2023-10-09T06:13:31.891405+0000","last_unstale":"2023-10-09T06:13:51.010303+0000","last_undegraded":"2023-10-09T06:13:51.010303+0000","last_fullsized":"2023-10-09T06:13:51.010303+0000","mapping_epoch":527,"log_start":"498'1200","ondisk_log_start":"498'1200","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5918,"log_dups_size":0,"ondisk_log_size":5918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:13:59.527962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333447168,"num_objects":5088,"num_object_clones":0,"num_object_copies":15264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5088,"num_objects_unfound":0,"num_objects_dirty":5088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7118,"num_write_kb":390592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[0,4,3],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,3,4","objects":5088}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":735,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.742677+0000","last_change":"2023-10-09T06:13:33.707885+0000","last_active":"2023-10-09T06:13:44.742677+0000","last_peered":"2023-10-09T06:13:44.742677+0000","last_clean":"2023-10-09T06:13:44.742677+0000","last_became_active":"2023-10-09T06:13:33.707474+0000","last_became_peered":"2023-10-09T06:13:33.707474+0000","last_unstale":"2023-10-09T06:13:44.742677+0000","last_undegraded":"2023-10-09T06:13:44.742677+0000","last_fullsized":"2023-10-09T06:13:44.742677+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:12:22.289803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:12:22.289803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:27.842817+0000","scrub_duration":0.0076565050000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":277,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.750648+0000","last_change":"2023-10-09T06:12:04.680247+0000","last_active":"2023-10-09T06:13:44.750648+0000","last_peered":"2023-10-09T06:13:44.750648+0000","last_clean":"2023-10-09T06:13:44.750648+0000","last_became_active":"2023-10-09T06:12:04.679943+0000","last_became_peered":"2023-10-09T06:12:04.679943+0000","last_unstale":"2023-10-09T06:13:44.750648+0000","last_undegraded":"2023-10-09T06:13:44.750648+0000","last_fullsized":"2023-10-09T06:13:44.750648+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:56.692774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":320,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.740139+0000","last_change":"2023-10-09T06:07:43.577306+0000","last_active":"2023-10-09T06:13:44.740139+0000","last_peered":"2023-10-09T06:13:44.740139+0000","last_clean":"2023-10-09T06:13:44.740139+0000","last_became_active":"2023-10-09T06:07:43.576956+0000","last_became_peered":"2023-10-09T06:07:43.576956+0000","last_unstale":"2023-10-09T06:13:44.740139+0000","last_undegraded":"2023-10-09T06:13:44.740139+0000","last_fullsized":"2023-10-09T06:13:44.740139+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:21.020403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":248,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.750494+0000","last_change":"2023-10-09T06:13:12.234849+0000","last_active":"2023-10-09T06:13:44.750494+0000","last_peered":"2023-10-09T06:13:44.750494+0000","last_clean":"2023-10-09T06:13:44.750494+0000","last_became_active":"2023-10-09T06:13:12.234284+0000","last_became_peered":"2023-10-09T06:13:12.234284+0000","last_unstale":"2023-10-09T06:13:44.750494+0000","last_undegraded":"2023-10-09T06:13:44.750494+0000","last_fullsized":"2023-10-09T06:13:44.750494+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:40:43.809773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"534'6966","reported_seq":7018,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.013170+0000","last_change":"2023-10-09T06:11:46.029349+0000","last_active":"2023-10-09T06:13:51.013170+0000","last_peered":"2023-10-09T06:13:51.013170+0000","last_clean":"2023-10-09T06:13:51.013170+0000","last_became_active":"2023-10-09T06:11:46.029041+0000","last_became_peered":"2023-10-09T06:11:46.029041+0000","last_unstale":"2023-10-09T06:13:51.013170+0000","last_undegraded":"2023-10-09T06:13:51.013170+0000","last_fullsized":"2023-10-09T06:13:51.013170+0000","mapping_epoch":490,"log_start":"498'800","ondisk_log_start":"498'800","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6166,"log_dups_size":0,"ondisk_log_size":6166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:07.662395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321519616,"num_objects":4906,"num_object_clones":0,"num_object_copies":14718,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4906,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6966,"num_write_kb":379904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":769,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.772205+0000","last_change":"2023-10-09T06:09:27.544555+0000","last_active":"2023-10-09T06:13:45.772205+0000","last_peered":"2023-10-09T06:13:45.772205+0000","last_clean":"2023-10-09T06:13:45.772205+0000","last_became_active":"2023-10-09T06:09:27.544222+0000","last_became_peered":"2023-10-09T06:09:27.544222+0000","last_unstale":"2023-10-09T06:13:45.772205+0000","last_undegraded":"2023-10-09T06:13:45.772205+0000","last_fullsized":"2023-10-09T06:13:45.772205+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:45.944253+0000","scrub_duration":0.0016660799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":233,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.750584+0000","last_change":"2023-10-09T06:13:21.738040+0000","last_active":"2023-10-09T06:13:44.750584+0000","last_peered":"2023-10-09T06:13:44.750584+0000","last_clean":"2023-10-09T06:13:44.750584+0000","last_became_active":"2023-10-09T06:13:21.737717+0000","last_became_peered":"2023-10-09T06:13:21.737717+0000","last_unstale":"2023-10-09T06:13:44.750584+0000","last_undegraded":"2023-10-09T06:13:44.750584+0000","last_fullsized":"2023-10-09T06:13:44.750584+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:39.451628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":514,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.778502+0000","last_change":"2023-10-09T06:09:27.553958+0000","last_active":"2023-10-09T06:13:45.778502+0000","last_peered":"2023-10-09T06:13:45.778502+0000","last_clean":"2023-10-09T06:13:45.778502+0000","last_became_active":"2023-10-09T06:09:27.553193+0000","last_became_peered":"2023-10-09T06:09:27.553193+0000","last_unstale":"2023-10-09T06:13:45.778502+0000","last_undegraded":"2023-10-09T06:13:45.778502+0000","last_fullsized":"2023-10-09T06:13:45.778502+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:09.754579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":358,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.750724+0000","last_change":"2023-10-09T06:07:43.580765+0000","last_active":"2023-10-09T06:13:44.750724+0000","last_peered":"2023-10-09T06:13:44.750724+0000","last_clean":"2023-10-09T06:13:44.750724+0000","last_became_active":"2023-10-09T06:07:43.580458+0000","last_became_peered":"2023-10-09T06:07:43.580458+0000","last_unstale":"2023-10-09T06:13:44.750724+0000","last_undegraded":"2023-10-09T06:13:44.750724+0000","last_fullsized":"2023-10-09T06:13:44.750724+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:55.161893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"533'7006","reported_seq":14503,"reported_epoch":533,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:13:50.754298+0000","last_change":"2023-10-09T06:13:32.706549+0000","last_active":"2023-10-09T06:13:50.754298+0000","last_peered":"2023-10-09T06:13:50.754298+0000","last_clean":"2023-10-09T06:13:11.209726+0000","last_became_active":"2023-10-09T06:13:30.857105+0000","last_became_peered":"2023-10-09T06:13:30.857105+0000","last_unstale":"2023-10-09T06:13:50.754298+0000","last_undegraded":"2023-10-09T06:13:50.754298+0000","last_fullsized":"2023-10-09T06:13:50.754298+0000","mapping_epoch":526,"log_start":"501'1600","ondisk_log_start":"501'1600","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5406,"log_dups_size":0,"ondisk_log_size":5406,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:30.233175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320733184,"num_objects":4894,"num_object_clones":0,"num_object_copies":14682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6674,"num_objects_unfound":0,"num_objects_dirty":4894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7006,"num_write_kb":380800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3703,"num_bytes_recovered":242679808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[4,2,3],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"2,3,4","objects":4894}],"blocked_by":[],"up_primary":7,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":578,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.743952+0000","last_change":"2023-10-09T06:13:33.705726+0000","last_active":"2023-10-09T06:13:44.743952+0000","last_peered":"2023-10-09T06:13:44.743952+0000","last_clean":"2023-10-09T06:13:44.743952+0000","last_became_active":"2023-10-09T06:13:33.705351+0000","last_became_peered":"2023-10-09T06:13:33.705351+0000","last_unstale":"2023-10-09T06:13:44.743952+0000","last_undegraded":"2023-10-09T06:13:44.743952+0000","last_fullsized":"2023-10-09T06:13:44.743952+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:44.483427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.483427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":313,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.727737+0000","last_change":"2023-10-09T06:09:27.565004+0000","last_active":"2023-10-09T06:13:44.727737+0000","last_peered":"2023-10-09T06:13:44.727737+0000","last_clean":"2023-10-09T06:13:44.727737+0000","last_became_active":"2023-10-09T06:09:27.564073+0000","last_became_peered":"2023-10-09T06:09:27.564073+0000","last_unstale":"2023-10-09T06:13:44.727737+0000","last_undegraded":"2023-10-09T06:13:44.727737+0000","last_fullsized":"2023-10-09T06:13:44.727737+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:15.146042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":302,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.748960+0000","last_change":"2023-10-09T06:13:21.728291+0000","last_active":"2023-10-09T06:13:44.748960+0000","last_peered":"2023-10-09T06:13:44.748960+0000","last_clean":"2023-10-09T06:13:44.748960+0000","last_became_active":"2023-10-09T06:13:21.727926+0000","last_became_peered":"2023-10-09T06:13:21.727926+0000","last_unstale":"2023-10-09T06:13:44.748960+0000","last_undegraded":"2023-10-09T06:13:44.748960+0000","last_fullsized":"2023-10-09T06:13:44.748960+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:35.292785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":230,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.751835+0000","last_change":"2023-10-09T06:09:27.547953+0000","last_active":"2023-10-09T06:13:44.751835+0000","last_peered":"2023-10-09T06:13:44.751835+0000","last_clean":"2023-10-09T06:13:44.751835+0000","last_became_active":"2023-10-09T06:09:27.547624+0000","last_became_peered":"2023-10-09T06:09:27.547624+0000","last_unstale":"2023-10-09T06:13:44.751835+0000","last_undegraded":"2023-10-09T06:13:44.751835+0000","last_fullsized":"2023-10-09T06:13:44.751835+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:39.630486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"533'6819","reported_seq":9987,"reported_epoch":533,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:13:46.532668+0000","last_change":"2023-10-09T06:13:30.783853+0000","last_active":"2023-10-09T06:13:46.532668+0000","last_peered":"2023-10-09T06:13:46.532668+0000","last_clean":"2023-10-09T06:13:20.543162+0000","last_became_active":"2023-10-09T06:13:22.836318+0000","last_became_peered":"2023-10-09T06:13:22.836318+0000","last_unstale":"2023-10-09T06:13:46.532668+0000","last_undegraded":"2023-10-09T06:13:46.532668+0000","last_fullsized":"2023-10-09T06:13:46.532668+0000","mapping_epoch":522,"log_start":"498'700","ondisk_log_start":"498'700","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6119,"log_dups_size":0,"ondisk_log_size":6119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:40.353464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335216640,"num_objects":5115,"num_object_clones":0,"num_object_copies":15345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3746,"num_objects_unfound":0,"num_objects_dirty":5115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6819,"num_write_kb":381888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1552,"num_bytes_recovered":101711872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,0,4],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":5115}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.744178+0000","last_change":"2023-10-09T06:13:30.791685+0000","last_active":"2023-10-09T06:13:44.744178+0000","last_peered":"2023-10-09T06:13:44.744178+0000","last_clean":"2023-10-09T06:13:44.744178+0000","last_became_active":"2023-10-09T06:13:30.791255+0000","last_became_peered":"2023-10-09T06:13:30.791255+0000","last_unstale":"2023-10-09T06:13:44.744178+0000","last_undegraded":"2023-10-09T06:13:44.744178+0000","last_fullsized":"2023-10-09T06:13:44.744178+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:24.896244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":576,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.744245+0000","last_change":"2023-10-09T06:13:33.704835+0000","last_active":"2023-10-09T06:13:44.744245+0000","last_peered":"2023-10-09T06:13:44.744245+0000","last_clean":"2023-10-09T06:13:44.744245+0000","last_became_active":"2023-10-09T06:13:33.704466+0000","last_became_peered":"2023-10-09T06:13:33.704466+0000","last_unstale":"2023-10-09T06:13:44.744245+0000","last_undegraded":"2023-10-09T06:13:44.744245+0000","last_fullsized":"2023-10-09T06:13:44.744245+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:35.783434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:35.783434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:38.721974+0000","scrub_duration":0.0043794799999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":320,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.727348+0000","last_change":"2023-10-09T06:13:12.238280+0000","last_active":"2023-10-09T06:13:44.727348+0000","last_peered":"2023-10-09T06:13:44.727348+0000","last_clean":"2023-10-09T06:13:44.727348+0000","last_became_active":"2023-10-09T06:13:12.237986+0000","last_became_peered":"2023-10-09T06:13:12.237986+0000","last_unstale":"2023-10-09T06:13:44.727348+0000","last_undegraded":"2023-10-09T06:13:44.727348+0000","last_fullsized":"2023-10-09T06:13:44.727348+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:58.520176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":464,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.779121+0000","last_change":"2023-10-09T06:13:21.733940+0000","last_active":"2023-10-09T06:13:45.779121+0000","last_peered":"2023-10-09T06:13:45.779121+0000","last_clean":"2023-10-09T06:13:45.779121+0000","last_became_active":"2023-10-09T06:13:21.733628+0000","last_became_peered":"2023-10-09T06:13:21.733628+0000","last_unstale":"2023-10-09T06:13:45.779121+0000","last_undegraded":"2023-10-09T06:13:45.779121+0000","last_fullsized":"2023-10-09T06:13:45.779121+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:13.348443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"534'7007","reported_seq":11796,"reported_epoch":534,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T06:13:51.047733+0000","last_change":"2023-10-09T06:13:32.707214+0000","last_active":"2023-10-09T06:13:51.047733+0000","last_peered":"2023-10-09T06:13:51.047733+0000","last_clean":"2023-10-09T06:13:29.756900+0000","last_became_active":"2023-10-09T06:13:31.891262+0000","last_became_peered":"2023-10-09T06:13:31.891262+0000","last_unstale":"2023-10-09T06:13:51.047733+0000","last_undegraded":"2023-10-09T06:13:51.047733+0000","last_fullsized":"2023-10-09T06:13:51.047733+0000","mapping_epoch":527,"log_start":"498'900","ondisk_log_start":"498'900","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6107,"log_dups_size":0,"ondisk_log_size":6107,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321585152,"num_objects":4907,"num_object_clones":0,"num_object_copies":14721,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2726,"num_objects_unfound":0,"num_objects_dirty":4907,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7007,"num_write_kb":381248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2364,"num_bytes_recovered":154927104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[0,2,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":4907}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":801,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.758210+0000","last_change":"2023-10-09T06:13:24.753842+0000","last_active":"2023-10-09T06:13:44.758210+0000","last_peered":"2023-10-09T06:13:44.758210+0000","last_clean":"2023-10-09T06:13:44.758210+0000","last_became_active":"2023-10-09T06:13:24.753485+0000","last_became_peered":"2023-10-09T06:13:24.753485+0000","last_unstale":"2023-10-09T06:13:44.758210+0000","last_undegraded":"2023-10-09T06:13:44.758210+0000","last_fullsized":"2023-10-09T06:13:44.758210+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:57.627296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:57.627296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:56.656218+0000","scrub_duration":0.001913166,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":426,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.744421+0000","last_change":"2023-10-09T06:09:27.544373+0000","last_active":"2023-10-09T06:13:44.744421+0000","last_peered":"2023-10-09T06:13:44.744421+0000","last_clean":"2023-10-09T06:13:44.744421+0000","last_became_active":"2023-10-09T06:09:27.544121+0000","last_became_peered":"2023-10-09T06:09:27.544121+0000","last_unstale":"2023-10-09T06:13:44.744421+0000","last_undegraded":"2023-10-09T06:13:44.744421+0000","last_fullsized":"2023-10-09T06:13:44.744421+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:40.429491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":471,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.772018+0000","last_change":"2023-10-09T06:12:52.670573+0000","last_active":"2023-10-09T06:13:45.772018+0000","last_peered":"2023-10-09T06:13:45.772018+0000","last_clean":"2023-10-09T06:13:45.772018+0000","last_became_active":"2023-10-09T06:12:52.670242+0000","last_became_peered":"2023-10-09T06:12:52.670242+0000","last_unstale":"2023-10-09T06:13:45.772018+0000","last_undegraded":"2023-10-09T06:13:45.772018+0000","last_fullsized":"2023-10-09T06:13:45.772018+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:25.237580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":610,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:46.048163+0000","last_change":"2023-10-09T06:09:27.543678+0000","last_active":"2023-10-09T06:13:46.048163+0000","last_peered":"2023-10-09T06:13:46.048163+0000","last_clean":"2023-10-09T06:13:46.048163+0000","last_became_active":"2023-10-09T06:09:27.543298+0000","last_became_peered":"2023-10-09T06:09:27.543298+0000","last_unstale":"2023-10-09T06:13:46.048163+0000","last_undegraded":"2023-10-09T06:13:46.048163+0000","last_fullsized":"2023-10-09T06:13:46.048163+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:26.297842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"534'7067","reported_seq":7567,"reported_epoch":534,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T06:13:50.879560+0000","last_change":"2023-10-09T06:13:50.058863+0000","last_active":"2023-10-09T06:13:50.879560+0000","last_peered":"2023-10-09T06:13:50.879560+0000","last_clean":"2023-10-09T06:13:20.517658+0000","last_became_active":"2023-10-09T06:13:22.825586+0000","last_became_peered":"2023-10-09T06:13:22.825586+0000","last_unstale":"2023-10-09T06:13:50.879560+0000","last_undegraded":"2023-10-09T06:13:50.879560+0000","last_fullsized":"2023-10-09T06:13:50.879560+0000","mapping_epoch":522,"log_start":"502'1700","ondisk_log_start":"502'1700","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5367,"log_dups_size":0,"ondisk_log_size":5367,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:09.871700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322175000,"num_objects":4917,"num_object_clones":0,"num_object_copies":14751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4699,"num_objects_unfound":0,"num_objects_dirty":4917,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7067,"num_write_kb":383425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":219,"num_bytes_recovered":14352384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,0,3],"avail_no_missing":["4","0","3"],"object_location_counts":[{"shards":"0,3,4","objects":4917}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":732,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.740095+0000","last_change":"2023-10-09T06:13:24.756288+0000","last_active":"2023-10-09T06:13:44.740095+0000","last_peered":"2023-10-09T06:13:44.740095+0000","last_clean":"2023-10-09T06:13:44.740095+0000","last_became_active":"2023-10-09T06:13:24.755945+0000","last_became_peered":"2023-10-09T06:13:24.755945+0000","last_unstale":"2023-10-09T06:13:44.740095+0000","last_undegraded":"2023-10-09T06:13:44.740095+0000","last_fullsized":"2023-10-09T06:13:44.740095+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:45.844892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:45.844892+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:44.107376+0000","scrub_duration":0.0017521489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.744105+0000","last_change":"2023-10-09T06:13:30.799535+0000","last_active":"2023-10-09T06:13:44.744105+0000","last_peered":"2023-10-09T06:13:44.744105+0000","last_clean":"2023-10-09T06:13:44.744105+0000","last_became_active":"2023-10-09T06:13:30.799227+0000","last_became_peered":"2023-10-09T06:13:30.799227+0000","last_unstale":"2023-10-09T06:13:44.744105+0000","last_undegraded":"2023-10-09T06:13:44.744105+0000","last_fullsized":"2023-10-09T06:13:44.744105+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:14.741742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":251,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.750801+0000","last_change":"2023-10-09T06:13:12.237622+0000","last_active":"2023-10-09T06:13:44.750801+0000","last_peered":"2023-10-09T06:13:44.750801+0000","last_clean":"2023-10-09T06:13:44.750801+0000","last_became_active":"2023-10-09T06:13:12.237287+0000","last_became_peered":"2023-10-09T06:13:12.237287+0000","last_unstale":"2023-10-09T06:13:44.750801+0000","last_undegraded":"2023-10-09T06:13:44.750801+0000","last_fullsized":"2023-10-09T06:13:44.750801+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:01.942898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":16,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.744004+0000","last_change":"2023-10-09T06:13:30.788382+0000","last_active":"2023-10-09T06:13:44.744004+0000","last_peered":"2023-10-09T06:13:44.744004+0000","last_clean":"2023-10-09T06:13:44.744004+0000","last_became_active":"2023-10-09T06:13:30.788060+0000","last_became_peered":"2023-10-09T06:13:30.788060+0000","last_unstale":"2023-10-09T06:13:44.744004+0000","last_undegraded":"2023-10-09T06:13:44.744004+0000","last_fullsized":"2023-10-09T06:13:44.744004+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:36:58.984799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.e","version":"533'7143","reported_seq":18219,"reported_epoch":534,"state":"peering","last_fresh":"2023-10-09T06:13:50.815491+0000","last_change":"2023-10-09T06:13:50.816254+0000","last_active":"2023-10-09T06:13:50.815453+0000","last_peered":"2023-10-09T06:13:50.803889+0000","last_clean":"2023-10-09T06:13:50.803889+0000","last_became_active":"2023-10-09T06:13:22.814519+0000","last_became_peered":"2023-10-09T06:13:22.814519+0000","last_unstale":"2023-10-09T06:13:50.815491+0000","last_undegraded":"2023-10-09T06:13:50.815491+0000","last_fullsized":"2023-10-09T06:13:50.815491+0000","mapping_epoch":534,"log_start":"502'1700","ondisk_log_start":"502'1700","created":490,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5443,"log_dups_size":0,"ondisk_log_size":5443,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:09:04.062598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329842688,"num_objects":5033,"num_object_clones":0,"num_object_copies":15099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5033,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7143,"num_write_kb":389632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5503,"num_bytes_recovered":360644608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,5,6],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":1007,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.779341+0000","last_change":"2023-10-09T06:13:24.762291+0000","last_active":"2023-10-09T06:13:45.779341+0000","last_peered":"2023-10-09T06:13:45.779341+0000","last_clean":"2023-10-09T06:13:45.779341+0000","last_became_active":"2023-10-09T06:13:24.761918+0000","last_became_peered":"2023-10-09T06:13:24.761918+0000","last_unstale":"2023-10-09T06:13:45.779341+0000","last_undegraded":"2023-10-09T06:13:45.779341+0000","last_fullsized":"2023-10-09T06:13:45.779341+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:17.344518+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:17.344518+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:58.227603+0000","scrub_duration":0.0015914099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":589,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.743188+0000","last_change":"2023-10-09T06:13:30.798394+0000","last_active":"2023-10-09T06:13:44.743188+0000","last_peered":"2023-10-09T06:13:44.743188+0000","last_clean":"2023-10-09T06:13:44.743188+0000","last_became_active":"2023-10-09T06:13:30.790913+0000","last_became_peered":"2023-10-09T06:13:30.790913+0000","last_unstale":"2023-10-09T06:13:44.743188+0000","last_undegraded":"2023-10-09T06:13:44.743188+0000","last_fullsized":"2023-10-09T06:13:44.743188+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:26.508690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":349,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.751582+0000","last_change":"2023-10-09T06:13:21.728009+0000","last_active":"2023-10-09T06:13:44.751582+0000","last_peered":"2023-10-09T06:13:44.751582+0000","last_clean":"2023-10-09T06:13:44.751582+0000","last_became_active":"2023-10-09T06:13:21.727683+0000","last_became_peered":"2023-10-09T06:13:21.727683+0000","last_unstale":"2023-10-09T06:13:44.751582+0000","last_undegraded":"2023-10-09T06:13:44.751582+0000","last_fullsized":"2023-10-09T06:13:44.751582+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:13.074232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":338,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.727000+0000","last_change":"2023-10-09T06:13:30.782038+0000","last_active":"2023-10-09T06:13:44.727000+0000","last_peered":"2023-10-09T06:13:44.727000+0000","last_clean":"2023-10-09T06:13:44.727000+0000","last_became_active":"2023-10-09T06:13:30.781500+0000","last_became_peered":"2023-10-09T06:13:30.781500+0000","last_unstale":"2023-10-09T06:13:44.727000+0000","last_undegraded":"2023-10-09T06:13:44.727000+0000","last_fullsized":"2023-10-09T06:13:44.727000+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:43.663265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"533'6989","reported_seq":7041,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:50.298846+0000","last_change":"2023-10-09T06:11:46.031709+0000","last_active":"2023-10-09T06:13:50.298846+0000","last_peered":"2023-10-09T06:13:50.298846+0000","last_clean":"2023-10-09T06:13:50.298846+0000","last_became_active":"2023-10-09T06:11:46.031421+0000","last_became_peered":"2023-10-09T06:11:46.031421+0000","last_unstale":"2023-10-09T06:13:50.298846+0000","last_undegraded":"2023-10-09T06:13:50.298846+0000","last_fullsized":"2023-10-09T06:13:50.298846+0000","mapping_epoch":490,"log_start":"498'1125","ondisk_log_start":"498'1125","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5864,"log_dups_size":0,"ondisk_log_size":5864,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:13.095174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323158016,"num_objects":4931,"num_object_clones":0,"num_object_copies":14793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6989,"num_write_kb":381440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":864,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.733350+0000","last_change":"2023-10-09T06:13:33.708100+0000","last_active":"2023-10-09T06:13:44.733350+0000","last_peered":"2023-10-09T06:13:44.733350+0000","last_clean":"2023-10-09T06:13:44.733350+0000","last_became_active":"2023-10-09T06:13:33.707809+0000","last_became_peered":"2023-10-09T06:13:33.707809+0000","last_unstale":"2023-10-09T06:13:44.733350+0000","last_undegraded":"2023-10-09T06:13:44.733350+0000","last_fullsized":"2023-10-09T06:13:44.733350+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:43:26.848778+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.745726+0000","last_change":"2023-10-09T06:13:30.799948+0000","last_active":"2023-10-09T06:13:44.745726+0000","last_peered":"2023-10-09T06:13:44.745726+0000","last_clean":"2023-10-09T06:13:44.745726+0000","last_became_active":"2023-10-09T06:13:30.799690+0000","last_became_peered":"2023-10-09T06:13:30.799690+0000","last_unstale":"2023-10-09T06:13:44.745726+0000","last_undegraded":"2023-10-09T06:13:44.745726+0000","last_fullsized":"2023-10-09T06:13:44.745726+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:43.893184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":580,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:46.048204+0000","last_change":"2023-10-09T06:09:27.578756+0000","last_active":"2023-10-09T06:13:46.048204+0000","last_peered":"2023-10-09T06:13:46.048204+0000","last_clean":"2023-10-09T06:13:46.048204+0000","last_became_active":"2023-10-09T06:09:27.578384+0000","last_became_peered":"2023-10-09T06:09:27.578384+0000","last_unstale":"2023-10-09T06:13:46.048204+0000","last_undegraded":"2023-10-09T06:13:46.048204+0000","last_fullsized":"2023-10-09T06:13:46.048204+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:22.182630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":480,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.778712+0000","last_change":"2023-10-09T06:01:22.334081+0000","last_active":"2023-10-09T06:13:45.778712+0000","last_peered":"2023-10-09T06:13:45.778712+0000","last_clean":"2023-10-09T06:13:45.778712+0000","last_became_active":"2023-10-09T06:01:22.333663+0000","last_became_peered":"2023-10-09T06:01:22.333663+0000","last_unstale":"2023-10-09T06:13:45.778712+0000","last_undegraded":"2023-10-09T06:13:45.778712+0000","last_fullsized":"2023-10-09T06:13:45.778712+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:55.809259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":778,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.727458+0000","last_change":"2023-10-09T06:09:27.560157+0000","last_active":"2023-10-09T06:13:44.727458+0000","last_peered":"2023-10-09T06:13:44.727458+0000","last_clean":"2023-10-09T06:13:44.727458+0000","last_became_active":"2023-10-09T06:09:27.559466+0000","last_became_peered":"2023-10-09T06:09:27.559466+0000","last_unstale":"2023-10-09T06:13:44.727458+0000","last_undegraded":"2023-10-09T06:13:44.727458+0000","last_fullsized":"2023-10-09T06:13:44.727458+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.181811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.181811+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:53.431491+0000","scrub_duration":0.011553313000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":462,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.772778+0000","last_change":"2023-10-09T06:13:30.791343+0000","last_active":"2023-10-09T06:13:45.772778+0000","last_peered":"2023-10-09T06:13:45.772778+0000","last_clean":"2023-10-09T06:13:45.772778+0000","last_became_active":"2023-10-09T06:13:30.790945+0000","last_became_peered":"2023-10-09T06:13:30.790945+0000","last_unstale":"2023-10-09T06:13:45.772778+0000","last_undegraded":"2023-10-09T06:13:45.772778+0000","last_fullsized":"2023-10-09T06:13:45.772778+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:14.208391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":234,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.750876+0000","last_change":"2023-10-09T06:07:43.580844+0000","last_active":"2023-10-09T06:13:44.750876+0000","last_peered":"2023-10-09T06:13:44.750876+0000","last_clean":"2023-10-09T06:13:44.750876+0000","last_became_active":"2023-10-09T06:07:43.580523+0000","last_became_peered":"2023-10-09T06:07:43.580523+0000","last_unstale":"2023-10-09T06:13:44.750876+0000","last_undegraded":"2023-10-09T06:13:44.750876+0000","last_fullsized":"2023-10-09T06:13:44.750876+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:55.942430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":882,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.338547+0000","last_change":"2023-10-09T06:00:04.215425+0000","last_active":"2023-10-09T06:13:45.338547+0000","last_peered":"2023-10-09T06:13:45.338547+0000","last_clean":"2023-10-09T06:13:45.338547+0000","last_became_active":"2023-10-09T05:59:40.326685+0000","last_became_peered":"2023-10-09T05:59:40.326685+0000","last_unstale":"2023-10-09T06:13:45.338547+0000","last_undegraded":"2023-10-09T06:13:45.338547+0000","last_fullsized":"2023-10-09T06:13:45.338547+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:04.215270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:04.215270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:03.963694+0000","scrub_duration":0.002850652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":305,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.779889+0000","last_change":"2023-10-09T06:01:54.824780+0000","last_active":"2023-10-09T06:13:45.779889+0000","last_peered":"2023-10-09T06:13:45.779889+0000","last_clean":"2023-10-09T06:13:45.779889+0000","last_became_active":"2023-10-09T06:01:54.757727+0000","last_became_peered":"2023-10-09T06:01:54.757727+0000","last_unstale":"2023-10-09T06:13:45.779889+0000","last_undegraded":"2023-10-09T06:13:45.779889+0000","last_fullsized":"2023-10-09T06:13:45.779889+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:44.132556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":552,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.778395+0000","last_change":"2023-10-09T06:01:54.764989+0000","last_active":"2023-10-09T06:13:45.778395+0000","last_peered":"2023-10-09T06:13:45.778395+0000","last_clean":"2023-10-09T06:13:45.778395+0000","last_became_active":"2023-10-09T06:01:54.764790+0000","last_became_peered":"2023-10-09T06:01:54.764790+0000","last_unstale":"2023-10-09T06:13:45.778395+0000","last_undegraded":"2023-10-09T06:13:45.778395+0000","last_fullsized":"2023-10-09T06:13:45.778395+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:14:07.208858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":319,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.778839+0000","last_change":"2023-10-09T06:13:21.739730+0000","last_active":"2023-10-09T06:13:45.778839+0000","last_peered":"2023-10-09T06:13:45.778839+0000","last_clean":"2023-10-09T06:13:45.778839+0000","last_became_active":"2023-10-09T06:13:21.727326+0000","last_became_peered":"2023-10-09T06:13:21.727326+0000","last_unstale":"2023-10-09T06:13:45.778839+0000","last_undegraded":"2023-10-09T06:13:45.778839+0000","last_fullsized":"2023-10-09T06:13:45.778839+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:51.344571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":549,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.778956+0000","last_change":"2023-10-09T06:09:27.565198+0000","last_active":"2023-10-09T06:13:45.778956+0000","last_peered":"2023-10-09T06:13:45.778956+0000","last_clean":"2023-10-09T06:13:45.778956+0000","last_became_active":"2023-10-09T06:09:27.564909+0000","last_became_peered":"2023-10-09T06:09:27.564909+0000","last_unstale":"2023-10-09T06:13:45.778956+0000","last_undegraded":"2023-10-09T06:13:45.778956+0000","last_fullsized":"2023-10-09T06:13:45.778956+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:46.101014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":698,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.745143+0000","last_change":"2023-10-09T06:13:33.699235+0000","last_active":"2023-10-09T06:13:44.745143+0000","last_peered":"2023-10-09T06:13:44.745143+0000","last_clean":"2023-10-09T06:13:44.745143+0000","last_became_active":"2023-10-09T06:13:33.698886+0000","last_became_peered":"2023-10-09T06:13:33.698886+0000","last_unstale":"2023-10-09T06:13:44.745143+0000","last_undegraded":"2023-10-09T06:13:44.745143+0000","last_fullsized":"2023-10-09T06:13:44.745143+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:00.456132+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:00.456132+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:59.421706+0000","scrub_duration":0.0014992199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":339,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.740260+0000","last_change":"2023-10-09T06:13:21.740383+0000","last_active":"2023-10-09T06:13:44.740260+0000","last_peered":"2023-10-09T06:13:44.740260+0000","last_clean":"2023-10-09T06:13:44.740260+0000","last_became_active":"2023-10-09T06:13:21.740023+0000","last_became_peered":"2023-10-09T06:13:21.740023+0000","last_unstale":"2023-10-09T06:13:44.740260+0000","last_undegraded":"2023-10-09T06:13:44.740260+0000","last_fullsized":"2023-10-09T06:13:44.740260+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:27.380765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":334,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.727794+0000","last_change":"2023-10-09T06:09:27.565185+0000","last_active":"2023-10-09T06:13:44.727794+0000","last_peered":"2023-10-09T06:13:44.727794+0000","last_clean":"2023-10-09T06:13:44.727794+0000","last_became_active":"2023-10-09T06:09:27.564478+0000","last_became_peered":"2023-10-09T06:09:27.564478+0000","last_unstale":"2023-10-09T06:13:44.727794+0000","last_undegraded":"2023-10-09T06:13:44.727794+0000","last_fullsized":"2023-10-09T06:13:44.727794+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:04.622171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":856,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T06:13:44.750984+0000","last_change":"2023-10-09T06:12:55.753877+0000","last_active":"2023-10-09T06:13:44.750984+0000","last_peered":"2023-10-09T06:13:44.750984+0000","last_clean":"2023-10-09T06:13:44.750984+0000","last_became_active":"2023-10-09T06:12:55.753512+0000","last_became_peered":"2023-10-09T06:12:55.753512+0000","last_unstale":"2023-10-09T06:13:44.750984+0000","last_undegraded":"2023-10-09T06:13:44.750984+0000","last_fullsized":"2023-10-09T06:13:44.750984+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:46.290510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:46.290510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:30.368031+0000","scrub_duration":0.002627819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":526,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.771792+0000","last_change":"2023-10-09T06:09:27.556271+0000","last_active":"2023-10-09T06:13:45.771792+0000","last_peered":"2023-10-09T06:13:45.771792+0000","last_clean":"2023-10-09T06:13:45.771792+0000","last_became_active":"2023-10-09T06:09:27.555932+0000","last_became_peered":"2023-10-09T06:09:27.555932+0000","last_unstale":"2023-10-09T06:13:45.771792+0000","last_undegraded":"2023-10-09T06:13:45.771792+0000","last_fullsized":"2023-10-09T06:13:45.771792+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:33.888622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":565,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:13:45.777955+0000","last_change":"2023-10-09T06:13:21.727720+0000","last_active":"2023-10-09T06:13:45.777955+0000","last_peered":"2023-10-09T06:13:45.777955+0000","last_clean":"2023-10-09T06:13:45.777955+0000","last_became_active":"2023-10-09T06:13:21.727339+0000","last_became_peered":"2023-10-09T06:13:21.727339+0000","last_unstale":"2023-10-09T06:13:45.777955+0000","last_undegraded":"2023-10-09T06:13:45.777955+0000","last_fullsized":"2023-10-09T06:13:45.777955+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:25.723752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5247926296,"num_objects":80078,"num_object_clones":0,"num_object_copies":240234,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":37572,"num_objects_unfound":0,"num_objects_dirty":80078,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":112348,"num_write_kb":6157569,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19874,"num_bytes_recovered":1302462464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16871862272,"data_stored":16871850056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":93595,"ondisk_log_size":93595,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":11714,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13410,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":617203,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1523929,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234409,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1395596,"kb_used_data":498884,"kb_used_omap":0,"kb_used_meta":896704,"kb_avail":92349556,"statfs":{"total":95995035648,"available":94565945344,"internally_reserved":0,"allocated":510857216,"data_stored":509354883,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918224896},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:31 2023","interfaces":[{"interface":"back","average":{"1min":8.7140000000000004,"5min":2.887,"15min":1.784},"min":{"1min":0.26900000000000002,"5min":0.25,"15min":0.25},"max":{"1min":57.435000000000002,"5min":57.435000000000002,"15min":57.435000000000002},"last":0.65100000000000002},{"interface":"front","average":{"1min":8.8059999999999992,"5min":3.0590000000000002,"15min":1.7829999999999999},"min":{"1min":0.318,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":57.472999999999999,"5min":57.472999999999999,"15min":57.472999999999999},"last":0.51600000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:13:01 2023","interfaces":[{"interface":"back","average":{"1min":3.407,"5min":4.2830000000000004,"15min":3.1709999999999998},"min":{"1min":0.29099999999999998,"5min":0.218,"15min":0.218},"max":{"1min":58.040999999999997,"5min":124.66200000000001,"15min":124.66200000000001},"last":1.3360000000000001},{"interface":"front","average":{"1min":3.7109999999999999,"5min":4.3499999999999996,"15min":3.1720000000000002},"min":{"1min":0.39700000000000002,"5min":0.26300000000000001,"15min":0.22700000000000001},"max":{"1min":58.786999999999999,"5min":125.023,"15min":125.023},"last":0.53600000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:13:31 2023","interfaces":[{"interface":"back","average":{"1min":3.456,"5min":1.788,"15min":2.222},"min":{"1min":0.316,"5min":0.27400000000000002,"15min":0.24299999999999999},"max":{"1min":47.503,"5min":47.503,"15min":127.86499999999999},"last":2.0630000000000002},{"interface":"front","average":{"1min":2.9900000000000002,"5min":2.0110000000000001,"15min":2.2250000000000001},"min":{"1min":0.318,"5min":0.29599999999999999,"15min":0.27000000000000002},"max":{"1min":47.545000000000002,"5min":47.545000000000002,"15min":129.24000000000001},"last":1.875}]},{"osd":3,"last update":"Mon Oct 9 06:13:01 2023","interfaces":[{"interface":"back","average":{"1min":5.2770000000000001,"5min":6.2149999999999999,"15min":5.8540000000000001},"min":{"1min":0.35199999999999998,"5min":0.32100000000000001,"15min":0.29899999999999999},"max":{"1min":58.448999999999998,"5min":116.68000000000001,"15min":164.16999999999999},"last":2.2050000000000001},{"interface":"front","average":{"1min":5.0229999999999997,"5min":6.6079999999999997,"15min":6.0330000000000004},"min":{"1min":0.34300000000000003,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":58.945,"5min":116.797,"15min":164.40700000000001},"last":7.0919999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":1.1220000000000001,"15min":0.92600000000000005},"min":{"1min":0.27100000000000002,"5min":0.23999999999999999,"15min":0.23100000000000001},"max":{"1min":3.4180000000000001,"5min":14.371,"15min":14.371},"last":0.89700000000000002},{"interface":"front","average":{"1min":0.51300000000000001,"5min":1.002,"15min":0.91500000000000004},"min":{"1min":0.30199999999999999,"5min":0.28599999999999998,"15min":0.27100000000000002},"max":{"1min":1.3220000000000001,"5min":14.259,"15min":14.259},"last":1.508}]},{"osd":5,"last update":"Mon Oct 9 06:12:46 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":1.458,"15min":1.073},"min":{"1min":0.33000000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.3500000000000001,"5min":43.826999999999998,"15min":43.826999999999998},"last":1.2509999999999999},{"interface":"front","average":{"1min":0.55300000000000005,"5min":1.4299999999999999,"15min":1.1339999999999999},"min":{"1min":0.38,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.88500000000000001,"5min":43.880000000000003,"15min":43.880000000000003},"last":0.85399999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:12:46 2023","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":0.97599999999999998,"15min":0.98899999999999999},"min":{"1min":0.36099999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.3100000000000001,"5min":9.4540000000000006,"15min":9.4540000000000006},"last":1.304},{"interface":"front","average":{"1min":0.64200000000000002,"5min":1.115,"15min":1.0229999999999999},"min":{"1min":0.32000000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.667,"5min":6.2329999999999997,"15min":6.2329999999999997},"last":0.93300000000000005}]}]},{"osd":6,"up_from":46,"seq":197568496046,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1926660,"kb_used_data":1025020,"kb_used_omap":0,"kb_used_meta":901632,"kb_avail":91818492,"statfs":{"total":95995035648,"available":94022135808,"internally_reserved":0,"allocated":1049620480,"data_stored":1048133778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923271168},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:28 2023","interfaces":[{"interface":"back","average":{"1min":18.359000000000002,"5min":7.04,"15min":3.8420000000000001},"min":{"1min":0.246,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":181.209,"5min":191.50800000000001,"15min":191.50800000000001},"last":0.32800000000000001},{"interface":"front","average":{"1min":18.960999999999999,"5min":7.0990000000000002,"15min":3.6920000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":181.233,"5min":191.52600000000001,"15min":191.52600000000001},"last":1.0449999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:13:23 2023","interfaces":[{"interface":"back","average":{"1min":26.061,"5min":6.2930000000000001,"15min":5.1870000000000003},"min":{"1min":0.27000000000000002,"5min":0.25,"15min":0.23200000000000001},"max":{"1min":328.04399999999998,"5min":328.04399999999998,"15min":328.04399999999998},"last":0.36899999999999999},{"interface":"front","average":{"1min":26.131,"5min":6.4470000000000001,"15min":5.2800000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":328.10000000000002,"5min":328.10000000000002,"15min":328.10000000000002},"last":8.7140000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:13:28 2023","interfaces":[{"interface":"back","average":{"1min":18.006,"5min":4.798,"15min":3.0049999999999999},"min":{"1min":0.28000000000000003,"5min":0.27300000000000002,"15min":0.26200000000000001},"max":{"1min":163.04900000000001,"5min":163.04900000000001,"15min":163.04900000000001},"last":1.0740000000000001},{"interface":"front","average":{"1min":17.959,"5min":4.9500000000000002,"15min":2.96},"min":{"1min":0.31900000000000001,"5min":0.29399999999999998,"15min":0.28599999999999998},"max":{"1min":162.93000000000001,"5min":162.93000000000001,"15min":162.93000000000001},"last":0.47999999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:13:11 2023","interfaces":[{"interface":"back","average":{"1min":17.170000000000002,"5min":6.9180000000000001,"15min":6.9450000000000003},"min":{"1min":0.318,"5min":0.318,"15min":0.307},"max":{"1min":250.744,"5min":250.744,"15min":250.744},"last":5.2670000000000003},{"interface":"front","average":{"1min":16.963999999999999,"5min":6.4390000000000001,"15min":6.8479999999999999},"min":{"1min":0.34499999999999997,"5min":0.33600000000000002,"15min":0.314},"max":{"1min":250.83799999999999,"5min":250.83799999999999,"15min":250.83799999999999},"last":0.53300000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:13:47 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":1.0740000000000001,"15min":1.681},"min":{"1min":0.28100000000000003,"5min":0.24399999999999999,"15min":0.24199999999999999},"max":{"1min":9.0869999999999997,"5min":9.0869999999999997,"15min":151.35599999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.97,"5min":1.175,"15min":1.7350000000000001},"min":{"1min":0.33500000000000002,"5min":0.249,"15min":0.23999999999999999},"max":{"1min":7.8849999999999998,"5min":7.8849999999999998,"15min":149.15799999999999},"last":0.97699999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:13:23 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.95599999999999996,"15min":1.0629999999999999},"min":{"1min":0.29699999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.3799999999999999,"5min":9.1959999999999997,"15min":15.342000000000001},"last":0.505},{"interface":"front","average":{"1min":0.90000000000000002,"5min":1.1100000000000001,"15min":1.131},"min":{"1min":0.312,"5min":0.28199999999999997,"15min":0.255},"max":{"1min":2.762,"5min":8.9809999999999999,"15min":15.358000000000001},"last":0.44600000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:13:47 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":1.105,"15min":1.2070000000000001},"min":{"1min":0.372,"5min":0.30299999999999999,"15min":0.27000000000000002},"max":{"1min":5.819,"5min":5.819,"15min":21.98},"last":1.0149999999999999},{"interface":"front","average":{"1min":0.79400000000000004,"5min":1.002,"15min":1.1559999999999999},"min":{"1min":0.39300000000000002,"5min":0.34200000000000003,"15min":0.33100000000000002},"max":{"1min":2.8570000000000002,"5min":5.3710000000000004,"15min":18.241},"last":0.41499999999999998}]}]},{"osd":1,"up_from":89,"seq":382252089634,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1579236,"kb_used_data":996764,"kb_used_omap":9,"kb_used_meta":582454,"kb_avail":92165916,"statfs":{"total":95995035648,"available":94377897984,"internally_reserved":0,"allocated":1020686336,"data_stored":1017645799,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":596433897},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,3],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:35 2023","interfaces":[{"interface":"back","average":{"1min":1.7809999999999999,"5min":1.946,"15min":2.1659999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.613,"5min":10.613,"15min":10.613},"last":0.371},{"interface":"front","average":{"1min":2.79,"5min":2.391,"15min":2.3570000000000002},"min":{"1min":0.254,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":13.121,"5min":24.239999999999998,"15min":24.239999999999998},"last":0.26500000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:13:44 2023","interfaces":[{"interface":"back","average":{"1min":2.7469999999999999,"5min":1.9850000000000001,"15min":2.4660000000000002},"min":{"1min":0.27800000000000002,"5min":0.21099999999999999,"15min":0.20799999999999999},"max":{"1min":12.818,"5min":15.522,"15min":129.053},"last":0.314},{"interface":"front","average":{"1min":2.3540000000000001,"5min":2.4900000000000002,"15min":2.6629999999999998},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.20300000000000001},"max":{"1min":8.9949999999999992,"5min":13.539999999999999,"15min":129.077},"last":0.33100000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:13:11 2023","interfaces":[{"interface":"back","average":{"1min":4.851,"5min":4.5389999999999997,"15min":4.75},"min":{"1min":0.26800000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":51.069000000000003,"5min":113.322,"15min":213.654},"last":0.55400000000000005},{"interface":"front","average":{"1min":4.8879999999999999,"5min":4.7220000000000004,"15min":4.7939999999999996},"min":{"1min":0.30299999999999999,"5min":0.27600000000000002,"15min":0.27400000000000002},"max":{"1min":53.148000000000003,"5min":113.38200000000001,"15min":213.77199999999999},"last":0.57499999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:13:44 2023","interfaces":[{"interface":"back","average":{"1min":9.1400000000000006,"5min":3.1629999999999998,"15min":2.133},"min":{"1min":0.40400000000000003,"5min":0.311,"15min":0.30399999999999999},"max":{"1min":93.747,"5min":93.747,"15min":93.747},"last":54.417000000000002},{"interface":"front","average":{"1min":9.8000000000000007,"5min":3.8359999999999999,"15min":2.6000000000000001},"min":{"1min":0.32800000000000001,"5min":0.308,"15min":0.308},"max":{"1min":97.852000000000004,"5min":97.852000000000004,"15min":97.852000000000004},"last":56.387}]},{"osd":5,"last update":"Mon Oct 9 06:12:56 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":1.752,"15min":1.579},"min":{"1min":0.38400000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.1820000000000004,"5min":9.3109999999999999,"15min":36.356999999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.9550000000000001,"5min":3.0339999999999998,"15min":2.2429999999999999},"min":{"1min":0.34399999999999997,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.6799999999999997,"5min":17.603999999999999,"15min":36.268000000000001},"last":0.39600000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:12:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1479999999999999,"5min":2.544,"15min":1.825},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.30099999999999999},"max":{"1min":6.218,"5min":52.207000000000001,"15min":52.207000000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.8240000000000001,"5min":3.3639999999999999,"15min":2.3250000000000002},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":8.7959999999999994,"5min":51.212000000000003,"15min":51.212000000000003},"last":0.42599999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:12:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8160000000000001,"5min":3.0600000000000001,"15min":2.6349999999999998},"min":{"1min":0.374,"5min":0.374,"15min":0.314},"max":{"1min":8.7699999999999996,"5min":15.284000000000001,"15min":108.553},"last":23.754999999999999},{"interface":"front","average":{"1min":0.95899999999999996,"5min":2.0569999999999999,"15min":2.113},"min":{"1min":0.41799999999999998,"5min":0.38,"15min":0.38},"max":{"1min":2.7799999999999998,"5min":9.9489999999999998,"15min":108.56999999999999},"last":23.788}]}]},{"osd":0,"up_from":456,"seq":1958505087030,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3763340,"kb_used_data":3509444,"kb_used_omap":32,"kb_used_meta":253855,"kb_avail":89981812,"statfs":{"total":95995035648,"available":92141375488,"internally_reserved":0,"allocated":3593670656,"data_stored":3590831986,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32794,"internal_metadata":259948518},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":5.7939999999999996,"15min":9.3539999999999992},"min":{"1min":0.23999999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":5.157,"5min":166.833,"15min":166.833},"last":0.501},{"interface":"front","average":{"1min":3.29,"5min":6.1779999999999999,"15min":8.7200000000000006},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":39.424999999999997,"5min":166.81100000000001,"15min":166.81100000000001},"last":0.51700000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":3.0449999999999999,"5min":2.9129999999999998,"15min":3.677},"min":{"1min":0.29999999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":21.648,"5min":21.648,"15min":21.648},"last":0.68799999999999994},{"interface":"front","average":{"1min":2.1800000000000002,"5min":2.6600000000000001,"15min":3.488},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":11.890000000000001,"5min":20.152000000000001,"15min":20.152000000000001},"last":0.57699999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:13:32 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":7.532,"15min":13.125999999999999},"min":{"1min":0.26500000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.9029999999999996,"5min":203.59299999999999,"15min":203.59299999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":3.4969999999999999,"5min":7.657,"15min":12.565},"min":{"1min":0.35299999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":18.192,"5min":202.96600000000001,"15min":202.96600000000001},"last":0.43099999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":2.444,"5min":2.0659999999999998,"15min":2.294},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":23.515999999999998,"5min":23.515999999999998,"15min":23.515999999999998},"last":0.871},{"interface":"front","average":{"1min":4.2149999999999999,"5min":2.5169999999999999,"15min":2.1669999999999998},"min":{"1min":0.35799999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":39.302,"5min":39.302,"15min":39.302},"last":0.88100000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":3.1899999999999999,"5min":3.7200000000000002,"15min":2.7000000000000002},"min":{"1min":0.32200000000000001,"5min":0.316,"15min":0.316},"max":{"1min":39.369999999999997,"5min":161.286,"15min":161.286},"last":2.52},{"interface":"front","average":{"1min":2.1110000000000002,"5min":3.532,"15min":2.8260000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":13.731999999999999,"5min":161.31800000000001,"15min":161.31800000000001},"last":2.577}]},{"osd":6,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":2.1190000000000002,"15min":2.3570000000000002},"min":{"1min":0.38900000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":13.795,"5min":13.795,"15min":13.795},"last":1.036},{"interface":"front","average":{"1min":1.3759999999999999,"5min":1.931,"15min":2.5350000000000001},"min":{"1min":0.44700000000000001,"5min":0.372,"15min":0.372},"max":{"1min":5.274,"5min":10.233000000000001,"15min":10.233000000000001},"last":2.895}]},{"osd":7,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":2.423,"5min":3.5070000000000001,"15min":2.8570000000000002},"min":{"1min":0.44900000000000001,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":13.821,"5min":123.685,"15min":123.685},"last":0.96899999999999997},{"interface":"front","average":{"1min":1.5049999999999999,"5min":3.214,"15min":2.9129999999999998},"min":{"1min":0.44,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":5.3200000000000003,"5min":122.482,"15min":122.482},"last":0.97899999999999998}]}]},{"osd":2,"up_from":94,"seq":403726926126,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3047736,"kb_used_data":2294960,"kb_used_omap":81,"kb_used_meta":752686,"kb_avail":90697416,"statfs":{"total":95995035648,"available":92874153984,"internally_reserved":0,"allocated":2350039040,"data_stored":2347333309,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":770750682},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":7.7000000000000002,"5min":3.5070000000000001,"15min":3.2679999999999998},"min":{"1min":0.221,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":56.213000000000001,"5min":56.213000000000001,"15min":56.213000000000001},"last":1.2350000000000001},{"interface":"front","average":{"1min":6.4610000000000003,"5min":2.8140000000000001,"15min":2.4329999999999998},"min":{"1min":0.221,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":50.764000000000003,"5min":50.764000000000003,"15min":50.764000000000003},"last":3.3109999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":5.0819999999999999,"15min":4.4909999999999997},"min":{"1min":0.27800000000000002,"5min":0.248,"15min":0.19400000000000001},"max":{"1min":4.258,"5min":160.45500000000001,"15min":222.59399999999999},"last":3.3370000000000002},{"interface":"front","average":{"1min":2.1379999999999999,"5min":6.0540000000000003,"15min":4.9480000000000004},"min":{"1min":0.27600000000000002,"5min":0.27200000000000002,"15min":0.20200000000000001},"max":{"1min":10.345000000000001,"5min":163.697,"15min":224.619},"last":0.313}]},{"osd":3,"last update":"Mon Oct 9 06:12:57 2023","interfaces":[{"interface":"back","average":{"1min":4.274,"5min":5.1360000000000001,"15min":4.3710000000000004},"min":{"1min":0.317,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":29.399999999999999,"5min":115.742,"15min":144.88200000000001},"last":2.4089999999999998},{"interface":"front","average":{"1min":3.9990000000000001,"5min":5.1310000000000002,"15min":4.3259999999999996},"min":{"1min":0.46100000000000002,"5min":0.252,"15min":0.252},"max":{"1min":29.648,"5min":115.55,"15min":140.79499999999999},"last":3.355}]},{"osd":4,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":3.1659999999999999,"15min":2.226},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.28299999999999997},"max":{"1min":8.9550000000000001,"5min":107.51300000000001,"15min":107.51300000000001},"last":4.2830000000000004},{"interface":"front","average":{"1min":1.081,"5min":2.6040000000000001,"15min":2.0579999999999998},"min":{"1min":0.36599999999999999,"5min":0.34100000000000003,"15min":0.31},"max":{"1min":3.4039999999999999,"5min":108.768,"15min":108.768},"last":3.4039999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":1.73,"5min":1.502,"15min":1.5549999999999999},"min":{"1min":0.376,"5min":0.34399999999999997,"15min":0.32600000000000001},"max":{"1min":11.015000000000001,"5min":11.015000000000001,"15min":28.48},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.2430000000000001,"5min":1.7150000000000001,"15min":1.728},"min":{"1min":0.379,"5min":0.316,"15min":0.28899999999999998},"max":{"1min":6.9180000000000001,"5min":9.8699999999999992,"15min":30.318999999999999},"last":3.3730000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":1.603,"5min":2.218,"15min":2.085},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.314},"max":{"1min":10.301,"5min":14.962,"15min":30.420000000000002},"last":2.2730000000000001},{"interface":"front","average":{"1min":1.8540000000000001,"5min":1.621,"15min":1.7749999999999999},"min":{"1min":0.432,"5min":0.36099999999999999,"15min":0.32200000000000001},"max":{"1min":11.103,"5min":11.103,"15min":30.713000000000001},"last":0.48299999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":1.8300000000000001,"5min":2.9089999999999998,"15min":2.2349999999999999},"min":{"1min":0.40799999999999997,"5min":0.313,"15min":0.313},"max":{"1min":11.065,"5min":124.069,"15min":124.069},"last":3.0219999999999998},{"interface":"front","average":{"1min":1.0449999999999999,"5min":2.8119999999999998,"15min":2.258},"min":{"1min":0.376,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.4210000000000003,"5min":124.08799999999999,"15min":124.08799999999999},"last":3.3889999999999998}]}]},{"osd":3,"up_from":99,"seq":425201762600,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3528204,"kb_used_data":2584580,"kb_used_omap":7,"kb_used_meta":943608,"kb_avail":90216948,"statfs":{"total":95995035648,"available":92382154752,"internally_reserved":0,"allocated":2646609920,"data_stored":2643585850,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":966255208},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:36 2023","interfaces":[{"interface":"back","average":{"1min":3.5409999999999999,"5min":1.9590000000000001,"15min":2.0590000000000002},"min":{"1min":0.222,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":30.841000000000001,"5min":30.841000000000001,"15min":30.841000000000001},"last":1.9159999999999999},{"interface":"front","average":{"1min":3.4769999999999999,"5min":2.8650000000000002,"15min":3.3050000000000002},"min":{"1min":0.27900000000000003,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":28.652000000000001,"5min":28.652000000000001,"15min":28.652000000000001},"last":1.099}]},{"osd":1,"last update":"Mon Oct 9 06:13:08 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":1.853,"15min":2.7530000000000001},"min":{"1min":0.24299999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":5.5990000000000002,"5min":22.579999999999998,"15min":207.75299999999999},"last":2.427},{"interface":"front","average":{"1min":1.794,"5min":3.4279999999999999,"15min":3.7429999999999999},"min":{"1min":0.27900000000000003,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.29,"5min":30.524000000000001,"15min":207.12200000000001},"last":2.3490000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:13:08 2023","interfaces":[{"interface":"back","average":{"1min":1.6779999999999999,"5min":2.3100000000000001,"15min":2.7919999999999998},"min":{"1min":0.28799999999999998,"5min":0.23200000000000001,"15min":0.223},"max":{"1min":5.9850000000000003,"5min":30.547999999999998,"15min":107.01300000000001},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.962,"5min":3.0990000000000002,"15min":3.1880000000000002},"min":{"1min":0.25600000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":12.864000000000001,"5min":25.984000000000002,"15min":109.807},"last":1.26}]},{"osd":4,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":6.8099999999999996,"5min":2.8759999999999999,"15min":3.5569999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.254},"max":{"1min":95.739000000000004,"5min":95.739000000000004,"15min":99.876000000000005},"last":0.38400000000000001},{"interface":"front","average":{"1min":6.8259999999999996,"5min":3.399,"15min":3.7210000000000001},"min":{"1min":0.373,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":97.941999999999993,"5min":97.941999999999993,"15min":99.914000000000001},"last":1.319}]},{"osd":5,"last update":"Mon Oct 9 06:13:08 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":2.827,"15min":2.105},"min":{"1min":0.441,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.4480000000000004,"5min":74.706000000000003,"15min":74.706000000000003},"last":1.575},{"interface":"front","average":{"1min":1.169,"5min":3.4740000000000002,"15min":3.286},"min":{"1min":0.375,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.915,"5min":74.685000000000002,"15min":235.56},"last":0.35999999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:13:08 2023","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":1.3560000000000001,"15min":4.2690000000000001},"min":{"1min":0.45500000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.484,"5min":8.6809999999999992,"15min":233.26599999999999},"last":1.6519999999999999},{"interface":"front","average":{"1min":1.165,"5min":2.0659999999999998,"15min":4.7089999999999996},"min":{"1min":0.432,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.8819999999999997,"5min":26.890000000000001,"15min":233.24100000000001},"last":0.42499999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:13:08 2023","interfaces":[{"interface":"back","average":{"1min":1.339,"5min":3.4870000000000001,"15min":4.7990000000000004},"min":{"1min":0.40699999999999997,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":5.077,"5min":81.811000000000007,"15min":230.69399999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.4590000000000001,"5min":3.915,"15min":4.9359999999999999},"min":{"1min":0.44500000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":4.673,"5min":75.909999999999997,"15min":230.67500000000001},"last":1.2949999999999999}]}]},{"osd":4,"up_from":322,"seq":1382979469463,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4202200,"kb_used_data":3871504,"kb_used_omap":35,"kb_used_meta":330524,"kb_avail":89542952,"statfs":{"total":95995035648,"available":91691982848,"internally_reserved":0,"allocated":3964420096,"data_stored":3962913602,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36552,"internal_metadata":338456888},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:31 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.3500000000000001,"15min":1.4419999999999999},"min":{"1min":0.222,"5min":0.216,"15min":0.216},"max":{"1min":4.6840000000000002,"5min":6.8470000000000004,"15min":6.8470000000000004},"last":0.28499999999999998},{"interface":"front","average":{"1min":1.504,"5min":1.4159999999999999,"15min":1.536},"min":{"1min":0.26000000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":8.7699999999999996,"5min":8.7699999999999996,"15min":8.7699999999999996},"last":0.27100000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":1.778,"5min":1.883,"15min":2.2839999999999998},"min":{"1min":0.28499999999999998,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.3449999999999998,"5min":18.884,"15min":78.891000000000005},"last":2.306},{"interface":"front","average":{"1min":1.4390000000000001,"5min":1.8300000000000001,"15min":2.2309999999999999},"min":{"1min":0.25600000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.946999999999999,"5min":18.617999999999999,"15min":78.921000000000006},"last":0.47299999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":14.574999999999999,"5min":4.4219999999999997,"15min":2.613},"min":{"1min":0.32500000000000001,"5min":0.29199999999999998,"15min":0.27200000000000002},"max":{"1min":245.166,"5min":245.166,"15min":245.166},"last":0.50700000000000001},{"interface":"front","average":{"1min":14.550000000000001,"5min":4.2469999999999999,"15min":2.5600000000000001},"min":{"1min":0.38500000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":245.119,"5min":245.119,"15min":245.119},"last":0.38500000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":24.687999999999999,"5min":6.6550000000000002,"15min":3.875},"min":{"1min":0.438,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":246.39400000000001,"5min":246.39400000000001,"15min":246.39400000000001},"last":2.4470000000000001},{"interface":"front","average":{"1min":23.437999999999999,"5min":6.21,"15min":3.444},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":246.99000000000001,"5min":246.99000000000001,"15min":246.99000000000001},"last":2.202}]},{"osd":5,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.93799999999999994,"15min":0.98999999999999999},"min":{"1min":0.27900000000000003,"5min":0.216,"15min":0.216},"max":{"1min":2.5600000000000001,"5min":7.5999999999999996,"15min":9.2690000000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":1.1719999999999999,"5min":0.94099999999999995,"15min":1.165},"min":{"1min":0.33700000000000002,"5min":0.245,"15min":0.245},"max":{"1min":5.3479999999999999,"5min":9.1110000000000007,"15min":12.087},"last":5.0880000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":1.036,"15min":2.3010000000000002},"min":{"1min":0.32100000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.4279999999999999,"5min":7.641,"15min":95.492999999999995},"last":0.42099999999999999},{"interface":"front","average":{"1min":1.1759999999999999,"5min":1.1990000000000001,"15min":2.4390000000000001},"min":{"1min":0.372,"5min":0.312,"15min":0.27900000000000003},"max":{"1min":5.5469999999999997,"5min":9.3239999999999998,"15min":95.664000000000001},"last":2.399}]},{"osd":7,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":1.3069999999999999,"15min":2.6920000000000002},"min":{"1min":0.38,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":8.1229999999999993,"5min":17.286000000000001,"15min":79.173000000000002},"last":8.1229999999999993},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.151,"15min":2.294},"min":{"1min":0.33000000000000002,"5min":0.251,"15min":0.251},"max":{"1min":5.1779999999999999,"5min":16.097999999999999,"15min":78.894000000000005},"last":0.54100000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757683,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2703368,"kb_used_data":1813056,"kb_used_omap":0,"kb_used_meta":890304,"kb_avail":91041784,"statfs":{"total":95995035648,"available":93226786816,"internally_reserved":0,"allocated":1856569344,"data_stored":1855075498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911671296},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:33 2023","interfaces":[{"interface":"back","average":{"1min":8.1929999999999996,"5min":3.508,"15min":2.77},"min":{"1min":0.24399999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":115.92400000000001,"5min":115.92400000000001,"15min":115.92400000000001},"last":3.6850000000000001},{"interface":"front","average":{"1min":8.4220000000000006,"5min":3.855,"15min":3.4340000000000002},"min":{"1min":0.29299999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":115.904,"5min":115.904,"15min":115.904},"last":12.045}]},{"osd":1,"last update":"Mon Oct 9 06:13:03 2023","interfaces":[{"interface":"back","average":{"1min":2.1899999999999999,"5min":5.4130000000000003,"15min":4.4619999999999997},"min":{"1min":0.314,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":18.812000000000001,"5min":151.78200000000001,"15min":151.78200000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":2.3090000000000002,"5min":5.6459999999999999,"15min":4.5119999999999996},"min":{"1min":0.309,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":19.376999999999999,"5min":152.251,"15min":152.251},"last":2.4169999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:13:27 2023","interfaces":[{"interface":"back","average":{"1min":9.2119999999999997,"5min":3.1019999999999999,"15min":2.9119999999999999},"min":{"1min":0.29799999999999999,"5min":0.26600000000000001,"15min":0.254},"max":{"1min":117.97199999999999,"5min":117.97199999999999,"15min":129.21199999999999},"last":0.33600000000000002},{"interface":"front","average":{"1min":9.6690000000000005,"5min":3.4500000000000002,"15min":2.9870000000000001},"min":{"1min":0.30599999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":118.52,"5min":118.52,"15min":129.042},"last":2.5350000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:13:03 2023","interfaces":[{"interface":"back","average":{"1min":2.7530000000000001,"5min":5.407,"15min":6.9669999999999996},"min":{"1min":0.33900000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":39.435000000000002,"5min":143.31200000000001,"15min":211.40199999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":3.109,"5min":5.9249999999999998,"15min":7.0350000000000001},"min":{"1min":0.38400000000000001,"5min":0.29599999999999999,"15min":0.29399999999999998},"max":{"1min":39.450000000000003,"5min":140.00899999999999,"15min":211.578},"last":1.98}]},{"osd":4,"last update":"Mon Oct 9 06:12:50 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":1.161,"15min":1.0589999999999999},"min":{"1min":0.246,"5min":0.245,"15min":0.245},"max":{"1min":3.25,"5min":7.5060000000000002,"15min":8.1539999999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.86299999999999999,"15min":1.016},"min":{"1min":0.316,"5min":0.26300000000000001,"15min":0.26000000000000001},"max":{"1min":1.597,"5min":9.2370000000000001,"15min":9.2370000000000001},"last":0.312}]},{"osd":6,"last update":"Mon Oct 9 06:13:03 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.96799999999999997,"15min":1.5},"min":{"1min":0.27500000000000002,"5min":0.26200000000000001,"15min":0.251},"max":{"1min":1.137,"5min":7.4980000000000002,"15min":126.89400000000001},"last":0.34499999999999997},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.0669999999999999,"15min":1.5569999999999999},"min":{"1min":0.33000000000000002,"5min":0.27300000000000002,"15min":0.26700000000000002},"max":{"1min":5.4279999999999999,"5min":7.7140000000000004,"15min":126.616},"last":2.04}]},{"osd":7,"last update":"Mon Oct 9 06:12:50 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":2.5430000000000001,"15min":2.5760000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.71,"5min":157.471,"15min":157.471},"last":2.016},{"interface":"front","average":{"1min":0.69399999999999995,"5min":2.4700000000000002,"15min":2.4399999999999999},"min":{"1min":0.34000000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.2890000000000001,"5min":157.51599999999999,"15min":157.51599999999999},"last":0.40899999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4889,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3570405376,"data_stored":3570401304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1010302976,"data_stored":1010302976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2336489472,"data_stored":2336489472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2620329984,"data_stored":2620325912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3932360704,"data_stored":3932356632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1844772864,"data_stored":1844772864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1050279936,"data_stored":1050279936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":506920960,"data_stored":506920960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:13:53.039 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:13:53.042 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:13:53.084 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:53.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:52 smithi064 ceph-mon[212750]: pgmap v1353: 119 pgs: 1 peering, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 965 op/s; 37572/240306 objects misplaced (15.635%); 35 MiB/s, 564 objects/s recovering 2023-10-09T06:13:53.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:52 smithi064 ceph-mon[212750]: from='client.36384 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:13:53.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:52 smithi064 ceph-mon[212750]: failed to encode map e535 with expected crc 2023-10-09T06:13:53.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:52 smithi064 ceph-mon[212750]: osdmap e535: 8 total, 8 up, 8 in 2023-10-09T06:13:53.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:52 smithi064 ceph-mon[212750]: failed to encode map e535 with expected crc 2023-10-09T06:13:53.123 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:52 smithi064 ceph-mon[220044]: pgmap v1353: 119 pgs: 1 peering, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 965 op/s; 37572/240306 objects misplaced (15.635%); 35 MiB/s, 564 objects/s recovering 2023-10-09T06:13:53.123 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:52 smithi064 ceph-mon[220044]: from='client.36384 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:13:53.123 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:52 smithi064 ceph-mon[220044]: failed to encode map e535 with expected crc 2023-10-09T06:13:53.123 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:52 smithi064 ceph-mon[220044]: osdmap e535: 8 total, 8 up, 8 in 2023-10-09T06:13:53.123 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:52 smithi064 ceph-mon[220044]: failed to encode map e535 with expected crc 2023-10-09T06:13:53.124 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:52 smithi156 ceph-mon[195101]: pgmap v1353: 119 pgs: 1 peering, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 965 op/s; 37572/240306 objects misplaced (15.635%); 35 MiB/s, 564 objects/s recovering 2023-10-09T06:13:53.124 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:52 smithi156 ceph-mon[195101]: from='client.36384 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:13:53.124 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:52 smithi156 ceph-mon[195101]: failed to encode map e535 with expected crc 2023-10-09T06:13:53.124 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:52 smithi156 ceph-mon[195101]: osdmap e535: 8 total, 8 up, 8 in 2023-10-09T06:13:53.124 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:52 smithi156 ceph-mon[195101]: failed to encode map e535 with expected crc 2023-10-09T06:13:53.156 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:13:53.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:53.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:13:53.185 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:13:53.237 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:13:53.266 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:13:53.408 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:53.676 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:13:53.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:53.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:13:53.782 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:13:53.831 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:53.924 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:53 smithi064 ceph-mon[220044]: pgmap v1355: 119 pgs: 1 peering, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 37241/238608 objects misplaced (15.608%); 31 MiB/s, 491 objects/s recovering 2023-10-09T06:13:53.924 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:53 smithi064 ceph-mon[212750]: pgmap v1355: 119 pgs: 1 peering, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 37241/238608 objects misplaced (15.608%); 31 MiB/s, 491 objects/s recovering 2023-10-09T06:13:53.934 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:13:53.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:53.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:13:54.049 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:13:54.095 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:53 smithi156 ceph-mon[195101]: pgmap v1355: 119 pgs: 1 peering, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 37241/238608 objects misplaced (15.608%); 31 MiB/s, 491 objects/s recovering 2023-10-09T06:13:54.104 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:54.177 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:13:54.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:54.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:13:54.286 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:13:54.361 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:54.656 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:13:54.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:54.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:13:54.761 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:13:54.790 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:54.840 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:13:54.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:54.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:13:54.890 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:13:54.946 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:13:54.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:54 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2241659280' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:13:54.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:54 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:13:54.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:54 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2241659280' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:13:54.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:54 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:13:54.976 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:55.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:54 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2241659280' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:13:55.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:54 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:13:55.305 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:13:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:13:55] "GET /metrics HTTP/1.1" 200 36494 "" "Prometheus/2.43.0" 2023-10-09T06:13:55.966 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:13:55.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:55.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:13:56.072 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:13:56.097 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:55 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:13:56.097 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:55 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T06:13:56.097 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:55 smithi156 ceph-mon[195101]: failed to encode map e536 with expected crc 2023-10-09T06:13:56.097 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:55 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:13:56.097 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:55 smithi156 ceph-mon[195101]: osdmap e536: 8 total, 8 up, 8 in 2023-10-09T06:13:56.098 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:55 smithi156 ceph-mon[195101]: failed to encode map e536 with expected crc 2023-10-09T06:13:56.098 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:55 smithi156 ceph-mon[195101]: pgmap v1357: 119 pgs: 1 peering, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 37241/238608 objects misplaced (15.608%); 20 MiB/s, 323 objects/s recovering 2023-10-09T06:13:56.100 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:56.158 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:13:56.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:56.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:13:56.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:55 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:13:56.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:55 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:13:56.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:55 smithi064 ceph-mon[212750]: failed to encode map e536 with expected crc 2023-10-09T06:13:56.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:55 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:13:56.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:55 smithi064 ceph-mon[212750]: osdmap e536: 8 total, 8 up, 8 in 2023-10-09T06:13:56.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:55 smithi064 ceph-mon[212750]: failed to encode map e536 with expected crc 2023-10-09T06:13:56.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:55 smithi064 ceph-mon[212750]: pgmap v1357: 119 pgs: 1 peering, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 37241/238608 objects misplaced (15.608%); 20 MiB/s, 323 objects/s recovering 2023-10-09T06:13:56.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:55 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:13:56.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:55 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T06:13:56.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:55 smithi064 ceph-mon[220044]: failed to encode map e536 with expected crc 2023-10-09T06:13:56.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:55 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:13:56.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:55 smithi064 ceph-mon[220044]: osdmap e536: 8 total, 8 up, 8 in 2023-10-09T06:13:56.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:55 smithi064 ceph-mon[220044]: failed to encode map e536 with expected crc 2023-10-09T06:13:56.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:55 smithi064 ceph-mon[220044]: pgmap v1357: 119 pgs: 1 peering, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 1 active+remapped, 111 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 37241/238608 objects misplaced (15.608%); 20 MiB/s, 323 objects/s recovering 2023-10-09T06:13:56.263 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:13:56.325 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:56.734 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:13:56.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:56.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:13:56.844 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:13:56.856 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:13:56.888 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:56.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:56 smithi064 ceph-mon[212750]: failed to encode map e536 with expected crc 2023-10-09T06:13:56.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:56 smithi064 ceph-mon[212750]: failed to encode map e536 with expected crc 2023-10-09T06:13:56.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:56 smithi064 ceph-mon[220044]: failed to encode map e536 with expected crc 2023-10-09T06:13:56.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:56 smithi064 ceph-mon[220044]: failed to encode map e536 with expected crc 2023-10-09T06:13:56.990 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:13:56.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:56.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:13:57.097 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:13:57.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:56 smithi156 ceph-mon[195101]: failed to encode map e536 with expected crc 2023-10-09T06:13:57.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:56 smithi156 ceph-mon[195101]: failed to encode map e536 with expected crc 2023-10-09T06:13:57.184 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:57.391 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:13:57.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:57.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:13:57.479 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:13:57.480 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:13:57.495 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:13:57.577 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:57.648 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:13:57.695 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:13:57.730 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:13:57.780 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:13:57.820 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:13:57.858 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:13:57.904 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:13:57.926 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:57 smithi064 ceph-mon[212750]: pgmap v1358: 119 pgs: 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 1 active+remapped, 112 active+clean; 4.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 32856/230085 objects misplaced (14.280%); 50 MiB/s, 793 objects/s recovering 2023-10-09T06:13:57.927 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:57 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2112123406' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:13:57.927 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:57 smithi064 ceph-mon[220044]: pgmap v1358: 119 pgs: 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 1 active+remapped, 112 active+clean; 4.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 32856/230085 objects misplaced (14.280%); 50 MiB/s, 793 objects/s recovering 2023-10-09T06:13:57.927 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:57 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2112123406' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:13:57.948 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:13:57.950 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:13:57.950 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:13:57.950 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:13:57.950 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:13:57.950 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:13:57.950 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:13:57.950 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:13:57.950 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:13:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:57 smithi156 ceph-mon[195101]: pgmap v1358: 119 pgs: 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 1 active+remapped, 112 active+clean; 4.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 32856/230085 objects misplaced (14.280%); 50 MiB/s, 793 objects/s recovering 2023-10-09T06:13:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:57 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2112123406' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:13:58.549 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:13:58.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:58.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:13:58.657 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:13:58.697 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:13:58.702 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:58.791 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:13:58.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:58.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:13:58.903 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:13:58.956 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:59.042 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:13:59.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:13:59.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:13:59.155 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:13:59.252 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:13:59.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:59 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:13:59.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:13:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:59.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:59 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:13:59.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:13:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:59.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:59 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:13:59.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:13:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:00.244 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:14:00.262 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:14:00.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:00.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:14:00.371 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:14:00.423 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:00.506 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:14:00.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:00.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:14:00.548 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:14:00.615 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:14:00.661 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:00.753 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:14:00.769 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:00 smithi156 ceph-mon[195101]: pgmap v1359: 119 pgs: 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 1 active+remapped, 112 active+clean; 4.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 425 op/s; 32856/230085 objects misplaced (14.280%); 24 MiB/s, 387 objects/s recovering 2023-10-09T06:14:00.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:00.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:14:00.819 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:00 smithi064 ceph-mon[212750]: pgmap v1359: 119 pgs: 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 1 active+remapped, 112 active+clean; 4.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 425 op/s; 32856/230085 objects misplaced (14.280%); 24 MiB/s, 387 objects/s recovering 2023-10-09T06:14:00.824 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:00 smithi064 ceph-mon[220044]: pgmap v1359: 119 pgs: 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 1 active+remapped, 112 active+clean; 4.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 425 op/s; 32856/230085 objects misplaced (14.280%); 24 MiB/s, 387 objects/s recovering 2023-10-09T06:14:00.870 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:00.919 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:01.010 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:01.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:01.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:01.119 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:14:01.157 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:01.242 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:14:01.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:01.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:14:01.348 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:14:01.461 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:01.575 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:14:01.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:01.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:14:01.650 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:14:01.652 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T06:14:01.683 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:14:01.711 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:01.754 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:14:01.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:01.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:14:01.857 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:14:01.923 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:02.208 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:14:02.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:02.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:14:02.289 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:14:02.312 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:14:02.360 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:02.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:02 smithi156 ceph-mon[195101]: pgmap v1360: 119 pgs: 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 1 active+remapped, 112 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 29975/225606 objects misplaced (13.286%); 35 MiB/s, 560 objects/s recovering 2023-10-09T06:14:02.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:02 smithi156 ceph-mon[195101]: failed to encode map e537 with expected crc 2023-10-09T06:14:02.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:02 smithi156 ceph-mon[195101]: failed to encode map e537 with expected crc 2023-10-09T06:14:02.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:02 smithi156 ceph-mon[195101]: osdmap e537: 8 total, 8 up, 8 in 2023-10-09T06:14:02.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:02 smithi156 ceph-mon[195101]: from='client.56245 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:14:02.898 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:02 smithi064 ceph-mon[212750]: pgmap v1360: 119 pgs: 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 1 active+remapped, 112 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 29975/225606 objects misplaced (13.286%); 35 MiB/s, 560 objects/s recovering 2023-10-09T06:14:02.898 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:02 smithi064 ceph-mon[212750]: failed to encode map e537 with expected crc 2023-10-09T06:14:02.898 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:02 smithi064 ceph-mon[212750]: failed to encode map e537 with expected crc 2023-10-09T06:14:02.898 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:02 smithi064 ceph-mon[212750]: osdmap e537: 8 total, 8 up, 8 in 2023-10-09T06:14:02.898 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:02 smithi064 ceph-mon[212750]: from='client.56245 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:14:02.898 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:14:02.901 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:02 smithi064 ceph-mon[220044]: pgmap v1360: 119 pgs: 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 1 active+remapped, 112 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 29975/225606 objects misplaced (13.286%); 35 MiB/s, 560 objects/s recovering 2023-10-09T06:14:02.901 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:02 smithi064 ceph-mon[220044]: failed to encode map e537 with expected crc 2023-10-09T06:14:02.901 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:02 smithi064 ceph-mon[220044]: failed to encode map e537 with expected crc 2023-10-09T06:14:02.901 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:02 smithi064 ceph-mon[220044]: osdmap e537: 8 total, 8 up, 8 in 2023-10-09T06:14:02.901 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:02 smithi064 ceph-mon[220044]: from='client.56245 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:14:02.927 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:14:02.933 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1361,"stamp":"2023-10-09T06:14:01.584302+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4827218059,"num_objects":73673,"num_object_clones":0,"num_object_copies":221019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":28743,"num_objects_unfound":0,"num_objects_dirty":73673,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":118849,"num_write_kb":6157708,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25820,"num_bytes_recovered":1689546419,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":101670,"ondisk_log_size":101670,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":368,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21311164,"kb_used_data":15729916,"kb_used_omap":165,"kb_used_meta":5580442,"kb_avail":728650052,"statfs":{"total":767960285184,"available":746137653248,"internally_reserved":0,"allocated":16107433984,"data_stored":16089695877,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":169911,"internal_metadata":5714372681},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-384237568,"num_objects":-5863,"num_object_clones":0,"num_object_copies":-17589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-8498,"num_objects_unfound":0,"num_objects_dirty":-5863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5863,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5884,"num_bytes_recovered":385548312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.423594"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":775,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.845169+0000","last_change":"2023-10-09T06:13:33.699246+0000","last_active":"2023-10-09T06:13:51.845169+0000","last_peered":"2023-10-09T06:13:51.845169+0000","last_clean":"2023-10-09T06:13:51.845169+0000","last_became_active":"2023-10-09T06:13:33.698733+0000","last_became_peered":"2023-10-09T06:13:33.698733+0000","last_unstale":"2023-10-09T06:13:51.845169+0000","last_undegraded":"2023-10-09T06:13:51.845169+0000","last_fullsized":"2023-10-09T06:13:51.845169+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:05.412941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:05.412941+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:57.044673+0000","scrub_duration":0.0061014329999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":604,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.813684+0000","last_change":"2023-10-09T06:07:43.572773+0000","last_active":"2023-10-09T06:13:51.813684+0000","last_peered":"2023-10-09T06:13:51.813684+0000","last_clean":"2023-10-09T06:13:51.813684+0000","last_became_active":"2023-10-09T06:07:43.572299+0000","last_became_peered":"2023-10-09T06:07:43.572299+0000","last_unstale":"2023-10-09T06:13:51.813684+0000","last_undegraded":"2023-10-09T06:13:51.813684+0000","last_fullsized":"2023-10-09T06:13:51.813684+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:00:37.672435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":346,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.809690+0000","last_change":"2023-10-09T06:05:11.451972+0000","last_active":"2023-10-09T06:13:51.809690+0000","last_peered":"2023-10-09T06:13:51.809690+0000","last_clean":"2023-10-09T06:13:51.809690+0000","last_became_active":"2023-10-09T06:05:11.451448+0000","last_became_peered":"2023-10-09T06:05:11.451448+0000","last_unstale":"2023-10-09T06:13:51.809690+0000","last_undegraded":"2023-10-09T06:13:51.809690+0000","last_fullsized":"2023-10-09T06:13:51.809690+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:39.183621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":835,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.815437+0000","last_change":"2023-10-09T06:09:27.554162+0000","last_active":"2023-10-09T06:13:51.815437+0000","last_peered":"2023-10-09T06:13:51.815437+0000","last_clean":"2023-10-09T06:13:51.815437+0000","last_became_active":"2023-10-09T06:09:27.553659+0000","last_became_peered":"2023-10-09T06:09:27.553659+0000","last_unstale":"2023-10-09T06:13:51.815437+0000","last_undegraded":"2023-10-09T06:13:51.815437+0000","last_fullsized":"2023-10-09T06:13:51.815437+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:01.468346+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:01.468346+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:03.443511+0000","scrub_duration":0.0016258220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":218,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:13:54.890485+0000","last_change":"2023-10-09T06:01:54.762060+0000","last_active":"2023-10-09T06:13:54.890485+0000","last_peered":"2023-10-09T06:13:54.890485+0000","last_clean":"2023-10-09T06:13:54.890485+0000","last_became_active":"2023-10-09T06:01:54.761757+0000","last_became_peered":"2023-10-09T06:01:54.761757+0000","last_unstale":"2023-10-09T06:13:54.890485+0000","last_undegraded":"2023-10-09T06:13:54.890485+0000","last_fullsized":"2023-10-09T06:13:54.890485+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:40.294232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":19,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.846201+0000","last_change":"2023-10-09T06:13:30.789834+0000","last_active":"2023-10-09T06:13:51.846201+0000","last_peered":"2023-10-09T06:13:51.846201+0000","last_clean":"2023-10-09T06:13:51.846201+0000","last_became_active":"2023-10-09T06:13:30.789517+0000","last_became_peered":"2023-10-09T06:13:30.789517+0000","last_unstale":"2023-10-09T06:13:51.846201+0000","last_undegraded":"2023-10-09T06:13:51.846201+0000","last_fullsized":"2023-10-09T06:13:51.846201+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:45.467922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":820,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.813067+0000","last_change":"2023-10-09T06:12:07.739121+0000","last_active":"2023-10-09T06:13:51.813067+0000","last_peered":"2023-10-09T06:13:51.813067+0000","last_clean":"2023-10-09T06:13:51.813067+0000","last_became_active":"2023-10-09T06:12:07.738739+0000","last_became_peered":"2023-10-09T06:12:07.738739+0000","last_unstale":"2023-10-09T06:13:51.813067+0000","last_undegraded":"2023-10-09T06:13:51.813067+0000","last_fullsized":"2023-10-09T06:13:51.813067+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.293426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.293426+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:19.815422+0000","scrub_duration":0.001587946,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":328,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.807906+0000","last_change":"2023-10-09T06:09:27.564377+0000","last_active":"2023-10-09T06:13:51.807906+0000","last_peered":"2023-10-09T06:13:51.807906+0000","last_clean":"2023-10-09T06:13:51.807906+0000","last_became_active":"2023-10-09T06:09:27.563710+0000","last_became_peered":"2023-10-09T06:09:27.563710+0000","last_unstale":"2023-10-09T06:13:51.807906+0000","last_undegraded":"2023-10-09T06:13:51.807906+0000","last_fullsized":"2023-10-09T06:13:51.807906+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:48.168447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":452,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.809225+0000","last_change":"2023-10-09T06:13:21.727677+0000","last_active":"2023-10-09T06:13:51.809225+0000","last_peered":"2023-10-09T06:13:51.809225+0000","last_clean":"2023-10-09T06:13:51.809225+0000","last_became_active":"2023-10-09T06:13:21.727321+0000","last_became_peered":"2023-10-09T06:13:21.727321+0000","last_unstale":"2023-10-09T06:13:51.809225+0000","last_undegraded":"2023-10-09T06:13:51.809225+0000","last_fullsized":"2023-10-09T06:13:51.809225+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:10.146447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":754,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.845651+0000","last_change":"2023-10-09T06:13:33.701687+0000","last_active":"2023-10-09T06:13:51.845651+0000","last_peered":"2023-10-09T06:13:51.845651+0000","last_clean":"2023-10-09T06:13:51.845651+0000","last_became_active":"2023-10-09T06:13:33.701233+0000","last_became_peered":"2023-10-09T06:13:33.701233+0000","last_unstale":"2023-10-09T06:13:51.845651+0000","last_undegraded":"2023-10-09T06:13:51.845651+0000","last_fullsized":"2023-10-09T06:13:51.845651+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.384407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.384407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:04.887826+0000","scrub_duration":0.0040651960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":434,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.808582+0000","last_change":"2023-10-09T06:09:27.559756+0000","last_active":"2023-10-09T06:13:51.808582+0000","last_peered":"2023-10-09T06:13:51.808582+0000","last_clean":"2023-10-09T06:13:51.808582+0000","last_became_active":"2023-10-09T06:09:27.559443+0000","last_became_peered":"2023-10-09T06:09:27.559443+0000","last_unstale":"2023-10-09T06:13:51.808582+0000","last_undegraded":"2023-10-09T06:13:51.808582+0000","last_fullsized":"2023-10-09T06:13:51.808582+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:14.519398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":365,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.807997+0000","last_change":"2023-10-09T06:13:21.737897+0000","last_active":"2023-10-09T06:13:51.807997+0000","last_peered":"2023-10-09T06:13:51.807997+0000","last_clean":"2023-10-09T06:13:51.807997+0000","last_became_active":"2023-10-09T06:13:21.737564+0000","last_became_peered":"2023-10-09T06:13:21.737564+0000","last_unstale":"2023-10-09T06:13:51.807997+0000","last_undegraded":"2023-10-09T06:13:51.807997+0000","last_fullsized":"2023-10-09T06:13:51.807997+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:34.863320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":660,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.845851+0000","last_change":"2023-10-09T06:10:11.729656+0000","last_active":"2023-10-09T06:13:51.845851+0000","last_peered":"2023-10-09T06:13:51.845851+0000","last_clean":"2023-10-09T06:13:51.845851+0000","last_became_active":"2023-10-09T06:10:11.728960+0000","last_became_peered":"2023-10-09T06:10:11.728960+0000","last_unstale":"2023-10-09T06:13:51.845851+0000","last_undegraded":"2023-10-09T06:13:51.845851+0000","last_fullsized":"2023-10-09T06:13:51.845851+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:20.742561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:20.742561+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:45.543709+0000","scrub_duration":0.0027005039999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":19,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.846139+0000","last_change":"2023-10-09T06:13:30.802749+0000","last_active":"2023-10-09T06:13:51.846139+0000","last_peered":"2023-10-09T06:13:51.846139+0000","last_clean":"2023-10-09T06:13:51.846139+0000","last_became_active":"2023-10-09T06:13:30.802449+0000","last_became_peered":"2023-10-09T06:13:30.802449+0000","last_unstale":"2023-10-09T06:13:51.846139+0000","last_undegraded":"2023-10-09T06:13:51.846139+0000","last_fullsized":"2023-10-09T06:13:51.846139+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:29.017181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":522,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.809139+0000","last_change":"2023-10-09T06:13:30.792445+0000","last_active":"2023-10-09T06:13:51.809139+0000","last_peered":"2023-10-09T06:13:51.809139+0000","last_clean":"2023-10-09T06:13:51.809139+0000","last_became_active":"2023-10-09T06:13:30.792116+0000","last_became_peered":"2023-10-09T06:13:30.792116+0000","last_unstale":"2023-10-09T06:13:51.809139+0000","last_undegraded":"2023-10-09T06:13:51.809139+0000","last_fullsized":"2023-10-09T06:13:51.809139+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:20:53.506293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":747,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.812521+0000","last_change":"2023-10-09T06:13:24.760843+0000","last_active":"2023-10-09T06:13:51.812521+0000","last_peered":"2023-10-09T06:13:51.812521+0000","last_clean":"2023-10-09T06:13:51.812521+0000","last_became_active":"2023-10-09T06:13:24.760333+0000","last_became_peered":"2023-10-09T06:13:24.760333+0000","last_unstale":"2023-10-09T06:13:51.812521+0000","last_undegraded":"2023-10-09T06:13:51.812521+0000","last_fullsized":"2023-10-09T06:13:51.812521+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:38.588641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:38.588641+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:30.660400+0000","scrub_duration":0.010116385,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":25,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.845577+0000","last_change":"2023-10-09T06:13:30.826588+0000","last_active":"2023-10-09T06:13:51.845577+0000","last_peered":"2023-10-09T06:13:51.845577+0000","last_clean":"2023-10-09T06:13:51.845577+0000","last_became_active":"2023-10-09T06:13:30.799199+0000","last_became_peered":"2023-10-09T06:13:30.799199+0000","last_unstale":"2023-10-09T06:13:51.845577+0000","last_undegraded":"2023-10-09T06:13:51.845577+0000","last_fullsized":"2023-10-09T06:13:51.845577+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:38.703864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":317,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.811946+0000","last_change":"2023-10-09T06:13:21.739214+0000","last_active":"2023-10-09T06:13:51.811946+0000","last_peered":"2023-10-09T06:13:51.811946+0000","last_clean":"2023-10-09T06:13:51.811946+0000","last_became_active":"2023-10-09T06:13:21.738951+0000","last_became_peered":"2023-10-09T06:13:21.738951+0000","last_unstale":"2023-10-09T06:13:51.811946+0000","last_undegraded":"2023-10-09T06:13:51.811946+0000","last_fullsized":"2023-10-09T06:13:51.811946+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:17.477241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":707,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.809823+0000","last_change":"2023-10-09T06:09:27.574336+0000","last_active":"2023-10-09T06:13:51.809823+0000","last_peered":"2023-10-09T06:13:51.809823+0000","last_clean":"2023-10-09T06:13:51.809823+0000","last_became_active":"2023-10-09T06:09:27.573974+0000","last_became_peered":"2023-10-09T06:09:27.573974+0000","last_unstale":"2023-10-09T06:13:51.809823+0000","last_undegraded":"2023-10-09T06:13:51.809823+0000","last_fullsized":"2023-10-09T06:13:51.809823+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.339121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.339121+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:45.908579+0000","scrub_duration":0.010474344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":570,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.813246+0000","last_change":"2023-10-09T06:13:30.812598+0000","last_active":"2023-10-09T06:13:51.813246+0000","last_peered":"2023-10-09T06:13:51.813246+0000","last_clean":"2023-10-09T06:13:51.813246+0000","last_became_active":"2023-10-09T06:13:30.791460+0000","last_became_peered":"2023-10-09T06:13:30.791460+0000","last_unstale":"2023-10-09T06:13:51.813246+0000","last_undegraded":"2023-10-09T06:13:51.813246+0000","last_fullsized":"2023-10-09T06:13:51.813246+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:28.046730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":548,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.815773+0000","last_change":"2023-10-09T06:01:54.764916+0000","last_active":"2023-10-09T06:13:51.815773+0000","last_peered":"2023-10-09T06:13:51.815773+0000","last_clean":"2023-10-09T06:13:51.815773+0000","last_became_active":"2023-10-09T06:01:54.764705+0000","last_became_peered":"2023-10-09T06:01:54.764705+0000","last_unstale":"2023-10-09T06:13:51.815773+0000","last_undegraded":"2023-10-09T06:13:51.815773+0000","last_fullsized":"2023-10-09T06:13:51.815773+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:12.527719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":764,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.808758+0000","last_change":"2023-10-09T06:13:15.425525+0000","last_active":"2023-10-09T06:13:51.808758+0000","last_peered":"2023-10-09T06:13:51.808758+0000","last_clean":"2023-10-09T06:13:51.808758+0000","last_became_active":"2023-10-09T06:13:15.425086+0000","last_became_peered":"2023-10-09T06:13:15.425086+0000","last_unstale":"2023-10-09T06:13:51.808758+0000","last_undegraded":"2023-10-09T06:13:51.808758+0000","last_fullsized":"2023-10-09T06:13:51.808758+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:42.900876+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:42.900876+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:46.042715+0000","scrub_duration":0.0014928649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":25,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.845317+0000","last_change":"2023-10-09T06:13:30.811420+0000","last_active":"2023-10-09T06:13:51.845317+0000","last_peered":"2023-10-09T06:13:51.845317+0000","last_clean":"2023-10-09T06:13:51.845317+0000","last_became_active":"2023-10-09T06:13:30.792835+0000","last_became_peered":"2023-10-09T06:13:30.792835+0000","last_unstale":"2023-10-09T06:13:51.845317+0000","last_undegraded":"2023-10-09T06:13:51.845317+0000","last_fullsized":"2023-10-09T06:13:51.845317+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:38.962645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":581,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.809529+0000","last_change":"2023-10-09T06:01:54.761558+0000","last_active":"2023-10-09T06:13:51.809529+0000","last_peered":"2023-10-09T06:13:51.809529+0000","last_clean":"2023-10-09T06:13:51.809529+0000","last_became_active":"2023-10-09T06:01:54.761251+0000","last_became_peered":"2023-10-09T06:01:54.761251+0000","last_unstale":"2023-10-09T06:13:51.809529+0000","last_undegraded":"2023-10-09T06:13:51.809529+0000","last_fullsized":"2023-10-09T06:13:51.809529+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:53.245267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":1039,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.845390+0000","last_change":"2023-10-09T06:13:24.754018+0000","last_active":"2023-10-09T06:13:51.845390+0000","last_peered":"2023-10-09T06:13:51.845390+0000","last_clean":"2023-10-09T06:13:51.845390+0000","last_became_active":"2023-10-09T06:13:24.753712+0000","last_became_peered":"2023-10-09T06:13:24.753712+0000","last_unstale":"2023-10-09T06:13:51.845390+0000","last_undegraded":"2023-10-09T06:13:51.845390+0000","last_fullsized":"2023-10-09T06:13:51.845390+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:18.490833+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":19,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.845282+0000","last_change":"2023-10-09T06:13:30.788160+0000","last_active":"2023-10-09T06:13:51.845282+0000","last_peered":"2023-10-09T06:13:51.845282+0000","last_clean":"2023-10-09T06:13:51.845282+0000","last_became_active":"2023-10-09T06:13:30.787724+0000","last_became_peered":"2023-10-09T06:13:30.787724+0000","last_unstale":"2023-10-09T06:13:51.845282+0000","last_undegraded":"2023-10-09T06:13:51.845282+0000","last_fullsized":"2023-10-09T06:13:51.845282+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:38.376770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":19,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.845796+0000","last_change":"2023-10-09T06:13:30.789225+0000","last_active":"2023-10-09T06:13:51.845796+0000","last_peered":"2023-10-09T06:13:51.845796+0000","last_clean":"2023-10-09T06:13:51.845796+0000","last_became_active":"2023-10-09T06:13:30.788869+0000","last_became_peered":"2023-10-09T06:13:30.788869+0000","last_unstale":"2023-10-09T06:13:51.845796+0000","last_undegraded":"2023-10-09T06:13:51.845796+0000","last_fullsized":"2023-10-09T06:13:51.845796+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:19.519966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":776,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.810859+0000","last_change":"2023-10-09T06:12:55.750923+0000","last_active":"2023-10-09T06:13:51.810859+0000","last_peered":"2023-10-09T06:13:51.810859+0000","last_clean":"2023-10-09T06:13:51.810859+0000","last_became_active":"2023-10-09T06:12:55.750521+0000","last_became_peered":"2023-10-09T06:12:55.750521+0000","last_unstale":"2023-10-09T06:13:51.810859+0000","last_undegraded":"2023-10-09T06:13:51.810859+0000","last_fullsized":"2023-10-09T06:13:51.810859+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.316697+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.316697+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:23.591439+0000","scrub_duration":0.001666561,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":389,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.847185+0000","last_change":"2023-10-09T06:01:22.337171+0000","last_active":"2023-10-09T06:13:51.847185+0000","last_peered":"2023-10-09T06:13:51.847185+0000","last_clean":"2023-10-09T06:13:51.847185+0000","last_became_active":"2023-10-09T06:01:22.335438+0000","last_became_peered":"2023-10-09T06:01:22.335438+0000","last_unstale":"2023-10-09T06:13:51.847185+0000","last_undegraded":"2023-10-09T06:13:51.847185+0000","last_fullsized":"2023-10-09T06:13:51.847185+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:25.036448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":498,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.808507+0000","last_change":"2023-10-09T06:13:30.788284+0000","last_active":"2023-10-09T06:13:51.808507+0000","last_peered":"2023-10-09T06:13:51.808507+0000","last_clean":"2023-10-09T06:13:51.808507+0000","last_became_active":"2023-10-09T06:13:30.787933+0000","last_became_peered":"2023-10-09T06:13:30.787933+0000","last_unstale":"2023-10-09T06:13:51.808507+0000","last_undegraded":"2023-10-09T06:13:51.808507+0000","last_fullsized":"2023-10-09T06:13:51.808507+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:10.735438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":826,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.810458+0000","last_change":"2023-10-09T06:13:24.753908+0000","last_active":"2023-10-09T06:13:51.810458+0000","last_peered":"2023-10-09T06:13:51.810458+0000","last_clean":"2023-10-09T06:13:51.810458+0000","last_became_active":"2023-10-09T06:13:24.753566+0000","last_became_peered":"2023-10-09T06:13:24.753566+0000","last_unstale":"2023-10-09T06:13:51.810458+0000","last_undegraded":"2023-10-09T06:13:51.810458+0000","last_fullsized":"2023-10-09T06:13:51.810458+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:40.388125+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:40.388125+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:35.101978+0000","scrub_duration":0.0055576990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":108,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.845134+0000","last_change":"2023-10-09T06:13:12.243605+0000","last_active":"2023-10-09T06:13:51.845134+0000","last_peered":"2023-10-09T06:13:51.845134+0000","last_clean":"2023-10-09T06:13:51.845134+0000","last_became_active":"2023-10-09T06:13:12.237061+0000","last_became_peered":"2023-10-09T06:13:12.237061+0000","last_unstale":"2023-10-09T06:13:51.845134+0000","last_undegraded":"2023-10-09T06:13:51.845134+0000","last_fullsized":"2023-10-09T06:13:51.845134+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:29:03.666031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":46,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.808165+0000","last_change":"2023-10-09T06:13:30.788733+0000","last_active":"2023-10-09T06:13:51.808165+0000","last_peered":"2023-10-09T06:13:51.808165+0000","last_clean":"2023-10-09T06:13:51.808165+0000","last_became_active":"2023-10-09T06:13:30.788353+0000","last_became_peered":"2023-10-09T06:13:30.788353+0000","last_unstale":"2023-10-09T06:13:51.808165+0000","last_undegraded":"2023-10-09T06:13:51.808165+0000","last_fullsized":"2023-10-09T06:13:51.808165+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:06.308426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"536'7455","reported_seq":7511,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:14:01.147167+0000","last_change":"2023-10-09T06:11:46.032384+0000","last_active":"2023-10-09T06:14:01.147167+0000","last_peered":"2023-10-09T06:14:01.147167+0000","last_clean":"2023-10-09T06:14:01.147167+0000","last_became_active":"2023-10-09T06:11:46.032091+0000","last_became_peered":"2023-10-09T06:11:46.032091+0000","last_unstale":"2023-10-09T06:14:01.147167+0000","last_undegraded":"2023-10-09T06:14:01.147167+0000","last_fullsized":"2023-10-09T06:14:01.147167+0000","mapping_epoch":490,"log_start":"498'1000","ondisk_log_start":"498'1000","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6455,"log_dups_size":0,"ondisk_log_size":6455,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:31.117384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301268992,"num_objects":4597,"num_object_clones":0,"num_object_copies":13791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7455,"num_write_kb":385664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":717,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.845868+0000","last_change":"2023-10-09T06:13:33.701521+0000","last_active":"2023-10-09T06:13:51.845868+0000","last_peered":"2023-10-09T06:13:51.845868+0000","last_clean":"2023-10-09T06:13:51.845868+0000","last_became_active":"2023-10-09T06:13:33.700870+0000","last_became_peered":"2023-10-09T06:13:33.700870+0000","last_unstale":"2023-10-09T06:13:51.845868+0000","last_undegraded":"2023-10-09T06:13:51.845868+0000","last_fullsized":"2023-10-09T06:13:51.845868+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:10:59.422551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:10:59.422551+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:54.402494+0000","scrub_duration":0.0059716659999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":23,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.847193+0000","last_change":"2023-10-09T06:13:30.822903+0000","last_active":"2023-10-09T06:13:51.847193+0000","last_peered":"2023-10-09T06:13:51.847193+0000","last_clean":"2023-10-09T06:13:51.847193+0000","last_became_active":"2023-10-09T06:13:30.799013+0000","last_became_peered":"2023-10-09T06:13:30.799013+0000","last_unstale":"2023-10-09T06:13:51.847193+0000","last_undegraded":"2023-10-09T06:13:51.847193+0000","last_fullsized":"2023-10-09T06:13:51.847193+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:44.271821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":19,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.845084+0000","last_change":"2023-10-09T06:13:30.793531+0000","last_active":"2023-10-09T06:13:51.845084+0000","last_peered":"2023-10-09T06:13:51.845084+0000","last_clean":"2023-10-09T06:13:51.845084+0000","last_became_active":"2023-10-09T06:13:30.792689+0000","last_became_peered":"2023-10-09T06:13:30.792689+0000","last_unstale":"2023-10-09T06:13:51.845084+0000","last_undegraded":"2023-10-09T06:13:51.845084+0000","last_fullsized":"2023-10-09T06:13:51.845084+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:02.559186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"536'7455","reported_seq":9139,"reported_epoch":536,"state":"active+remapped","last_fresh":"2023-10-09T06:14:00.325846+0000","last_change":"2023-10-09T06:13:30.811777+0000","last_active":"2023-10-09T06:14:00.325846+0000","last_peered":"2023-10-09T06:14:00.325846+0000","last_clean":"2023-10-09T06:13:11.194737+0000","last_became_active":"2023-10-09T06:13:21.782116+0000","last_became_peered":"2023-10-09T06:13:21.782116+0000","last_unstale":"2023-10-09T06:14:00.325846+0000","last_undegraded":"2023-10-09T06:14:00.325846+0000","last_fullsized":"2023-10-09T06:14:00.325846+0000","mapping_epoch":521,"log_start":"500'1493","ondisk_log_start":"500'1493","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5962,"log_dups_size":0,"ondisk_log_size":5962,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:00.656766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303235072,"num_objects":4627,"num_object_clones":0,"num_object_copies":13881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":8626,"num_objects_unfound":0,"num_objects_dirty":4627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7455,"num_write_kb":386624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":801,"num_bytes_recovered":52494336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,2,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":4627}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":905,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.809312+0000","last_change":"2023-10-09T06:13:24.753866+0000","last_active":"2023-10-09T06:13:51.809312+0000","last_peered":"2023-10-09T06:13:51.809312+0000","last_clean":"2023-10-09T06:13:51.809312+0000","last_became_active":"2023-10-09T06:13:24.753432+0000","last_became_peered":"2023-10-09T06:13:24.753432+0000","last_unstale":"2023-10-09T06:13:51.809312+0000","last_undegraded":"2023-10-09T06:13:51.809312+0000","last_fullsized":"2023-10-09T06:13:51.809312+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:03.241335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:03.241335+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:19.854449+0000","scrub_duration":0.017780547000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":23,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.846070+0000","last_change":"2023-10-09T06:13:30.829763+0000","last_active":"2023-10-09T06:13:51.846070+0000","last_peered":"2023-10-09T06:13:51.846070+0000","last_clean":"2023-10-09T06:13:51.846070+0000","last_became_active":"2023-10-09T06:13:30.802486+0000","last_became_peered":"2023-10-09T06:13:30.802486+0000","last_unstale":"2023-10-09T06:13:51.846070+0000","last_undegraded":"2023-10-09T06:13:51.846070+0000","last_fullsized":"2023-10-09T06:13:51.846070+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:05.468205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":574,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.813363+0000","last_change":"2023-10-09T06:01:54.760367+0000","last_active":"2023-10-09T06:13:51.813363+0000","last_peered":"2023-10-09T06:13:51.813363+0000","last_clean":"2023-10-09T06:13:51.813363+0000","last_became_active":"2023-10-09T06:01:54.760087+0000","last_became_peered":"2023-10-09T06:01:54.760087+0000","last_unstale":"2023-10-09T06:13:51.813363+0000","last_undegraded":"2023-10-09T06:13:51.813363+0000","last_fullsized":"2023-10-09T06:13:51.813363+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:26.000104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"536'7384","reported_seq":7438,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:14:00.835032+0000","last_change":"2023-10-09T06:11:45.696810+0000","last_active":"2023-10-09T06:14:00.835032+0000","last_peered":"2023-10-09T06:14:00.835032+0000","last_clean":"2023-10-09T06:14:00.835032+0000","last_became_active":"2023-10-09T06:11:45.696476+0000","last_became_peered":"2023-10-09T06:11:45.696476+0000","last_unstale":"2023-10-09T06:14:00.835032+0000","last_undegraded":"2023-10-09T06:14:00.835032+0000","last_fullsized":"2023-10-09T06:14:00.835032+0000","mapping_epoch":490,"log_start":"504'2000","ondisk_log_start":"504'2000","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5384,"log_dups_size":0,"ondisk_log_size":5384,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:41:33.351567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298057728,"num_objects":4548,"num_object_clones":0,"num_object_copies":13644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7384,"num_write_kb":381824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":799,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.813706+0000","last_change":"2023-10-09T06:13:24.768958+0000","last_active":"2023-10-09T06:13:51.813706+0000","last_peered":"2023-10-09T06:13:51.813706+0000","last_clean":"2023-10-09T06:13:51.813706+0000","last_became_active":"2023-10-09T06:13:24.768643+0000","last_became_peered":"2023-10-09T06:13:24.768643+0000","last_unstale":"2023-10-09T06:13:51.813706+0000","last_undegraded":"2023-10-09T06:13:51.813706+0000","last_fullsized":"2023-10-09T06:13:51.813706+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:48.495739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:48.495739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:18:50.654354+0000","scrub_duration":0.0016345509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":23,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.845727+0000","last_change":"2023-10-09T06:13:30.814434+0000","last_active":"2023-10-09T06:13:51.845727+0000","last_peered":"2023-10-09T06:13:51.845727+0000","last_clean":"2023-10-09T06:13:51.845727+0000","last_became_active":"2023-10-09T06:13:30.793119+0000","last_became_peered":"2023-10-09T06:13:30.793119+0000","last_unstale":"2023-10-09T06:13:51.845727+0000","last_undegraded":"2023-10-09T06:13:51.845727+0000","last_fullsized":"2023-10-09T06:13:51.845727+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:34:31.724006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":560,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.810140+0000","last_change":"2023-10-09T06:13:30.792490+0000","last_active":"2023-10-09T06:13:51.810140+0000","last_peered":"2023-10-09T06:13:51.810140+0000","last_clean":"2023-10-09T06:13:51.810140+0000","last_became_active":"2023-10-09T06:13:30.792209+0000","last_became_peered":"2023-10-09T06:13:30.792209+0000","last_unstale":"2023-10-09T06:13:51.810140+0000","last_undegraded":"2023-10-09T06:13:51.810140+0000","last_fullsized":"2023-10-09T06:13:51.810140+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:14.208042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"536'7345","reported_seq":7401,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:14:01.131533+0000","last_change":"2023-10-09T06:11:46.031195+0000","last_active":"2023-10-09T06:14:01.131533+0000","last_peered":"2023-10-09T06:14:01.131533+0000","last_clean":"2023-10-09T06:14:01.131533+0000","last_became_active":"2023-10-09T06:11:46.030891+0000","last_became_peered":"2023-10-09T06:11:46.030891+0000","last_unstale":"2023-10-09T06:14:01.131533+0000","last_undegraded":"2023-10-09T06:14:01.131533+0000","last_fullsized":"2023-10-09T06:14:01.131533+0000","mapping_epoch":490,"log_start":"498'900","ondisk_log_start":"498'900","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6445,"log_dups_size":0,"ondisk_log_size":6445,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:25.788661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299565056,"num_objects":4571,"num_object_clones":0,"num_object_copies":13713,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4571,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7345,"num_write_kb":381312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":751,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.813891+0000","last_change":"2023-10-09T06:13:33.708291+0000","last_active":"2023-10-09T06:13:51.813891+0000","last_peered":"2023-10-09T06:13:51.813891+0000","last_clean":"2023-10-09T06:13:51.813891+0000","last_became_active":"2023-10-09T06:13:33.707930+0000","last_became_peered":"2023-10-09T06:13:33.707930+0000","last_unstale":"2023-10-09T06:13:51.813891+0000","last_undegraded":"2023-10-09T06:13:51.813891+0000","last_fullsized":"2023-10-09T06:13:51.813891+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:16.361097+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:16.361097+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:18.111404+0000","scrub_duration":0.028533749000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":305,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.809398+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T06:13:51.809398+0000","last_peered":"2023-10-09T06:13:51.809398+0000","last_clean":"2023-10-09T06:13:51.809398+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T06:13:51.809398+0000","last_undegraded":"2023-10-09T06:13:51.809398+0000","last_fullsized":"2023-10-09T06:13:51.809398+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":549,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:13:54.991504+0000","last_change":"2023-10-09T06:09:27.558353+0000","last_active":"2023-10-09T06:13:54.991504+0000","last_peered":"2023-10-09T06:13:54.991504+0000","last_clean":"2023-10-09T06:13:54.991504+0000","last_became_active":"2023-10-09T06:09:27.557233+0000","last_became_peered":"2023-10-09T06:09:27.557233+0000","last_unstale":"2023-10-09T06:13:54.991504+0000","last_undegraded":"2023-10-09T06:13:54.991504+0000","last_fullsized":"2023-10-09T06:13:54.991504+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:53.149350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"536'7394","reported_seq":7450,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:14:01.477907+0000","last_change":"2023-10-09T06:11:45.694648+0000","last_active":"2023-10-09T06:14:01.477907+0000","last_peered":"2023-10-09T06:14:01.477907+0000","last_clean":"2023-10-09T06:14:01.477907+0000","last_became_active":"2023-10-09T06:11:45.694321+0000","last_became_peered":"2023-10-09T06:11:45.694321+0000","last_unstale":"2023-10-09T06:14:01.477907+0000","last_undegraded":"2023-10-09T06:14:01.477907+0000","last_fullsized":"2023-10-09T06:14:01.477907+0000","mapping_epoch":490,"log_start":"498'800","ondisk_log_start":"498'800","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6594,"log_dups_size":0,"ondisk_log_size":6594,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:43.704525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296747008,"num_objects":4528,"num_object_clones":0,"num_object_copies":13584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7394,"num_write_kb":381504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":922,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.847079+0000","last_change":"2023-10-09T06:13:33.716191+0000","last_active":"2023-10-09T06:13:51.847079+0000","last_peered":"2023-10-09T06:13:51.847079+0000","last_clean":"2023-10-09T06:13:51.847079+0000","last_became_active":"2023-10-09T06:13:33.715831+0000","last_became_peered":"2023-10-09T06:13:33.715831+0000","last_unstale":"2023-10-09T06:13:51.847079+0000","last_undegraded":"2023-10-09T06:13:51.847079+0000","last_fullsized":"2023-10-09T06:13:51.847079+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:42.795781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:42.795781+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:45.712738+0000","scrub_duration":0.0091262979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":339,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.808862+0000","last_change":"2023-10-09T06:09:27.565666+0000","last_active":"2023-10-09T06:13:51.808862+0000","last_peered":"2023-10-09T06:13:51.808862+0000","last_clean":"2023-10-09T06:13:51.808862+0000","last_became_active":"2023-10-09T06:09:27.565324+0000","last_became_peered":"2023-10-09T06:09:27.565324+0000","last_unstale":"2023-10-09T06:13:51.808862+0000","last_undegraded":"2023-10-09T06:13:51.808862+0000","last_fullsized":"2023-10-09T06:13:51.808862+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:51.216947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":510,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.813372+0000","last_change":"2023-10-09T06:13:21.727745+0000","last_active":"2023-10-09T06:13:51.813372+0000","last_peered":"2023-10-09T06:13:51.813372+0000","last_clean":"2023-10-09T06:13:51.813372+0000","last_became_active":"2023-10-09T06:13:21.727404+0000","last_became_peered":"2023-10-09T06:13:21.727404+0000","last_unstale":"2023-10-09T06:13:51.813372+0000","last_undegraded":"2023-10-09T06:13:51.813372+0000","last_fullsized":"2023-10-09T06:13:51.813372+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:24.931626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"536'7505","reported_seq":19054,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:14:00.271037+0000","last_change":"2023-10-09T06:13:32.697002+0000","last_active":"2023-10-09T06:14:00.271037+0000","last_peered":"2023-10-09T06:14:00.271037+0000","last_clean":"2023-10-09T06:14:00.271037+0000","last_became_active":"2023-10-09T06:13:32.683104+0000","last_became_peered":"2023-10-09T06:13:32.683104+0000","last_unstale":"2023-10-09T06:14:00.271037+0000","last_undegraded":"2023-10-09T06:14:00.271037+0000","last_fullsized":"2023-10-09T06:14:00.271037+0000","mapping_epoch":528,"log_start":"501'1535","ondisk_log_start":"501'1535","created":490,"last_epoch_clean":529,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5970,"log_dups_size":0,"ondisk_log_size":5970,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:19.426571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312541184,"num_objects":4769,"num_object_clones":0,"num_object_copies":14307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7505,"num_write_kb":392768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5732,"num_bytes_recovered":375652352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":798,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.807709+0000","last_change":"2023-10-09T06:13:15.442141+0000","last_active":"2023-10-09T06:13:51.807709+0000","last_peered":"2023-10-09T06:13:51.807709+0000","last_clean":"2023-10-09T06:13:51.807709+0000","last_became_active":"2023-10-09T06:13:15.441716+0000","last_became_peered":"2023-10-09T06:13:15.441716+0000","last_unstale":"2023-10-09T06:13:51.807709+0000","last_undegraded":"2023-10-09T06:13:51.807709+0000","last_fullsized":"2023-10-09T06:13:51.807709+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.282393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.282393+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:16.798440+0000","scrub_duration":0.01402403,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":455,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.813436+0000","last_change":"2023-10-09T06:13:21.738218+0000","last_active":"2023-10-09T06:13:51.813436+0000","last_peered":"2023-10-09T06:13:51.813436+0000","last_clean":"2023-10-09T06:13:51.813436+0000","last_became_active":"2023-10-09T06:13:21.737969+0000","last_became_peered":"2023-10-09T06:13:21.737969+0000","last_unstale":"2023-10-09T06:13:51.813436+0000","last_undegraded":"2023-10-09T06:13:51.813436+0000","last_fullsized":"2023-10-09T06:13:51.813436+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:38.620281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":584,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.814003+0000","last_change":"2023-10-09T06:07:43.572173+0000","last_active":"2023-10-09T06:13:51.814003+0000","last_peered":"2023-10-09T06:13:51.814003+0000","last_clean":"2023-10-09T06:13:51.814003+0000","last_became_active":"2023-10-09T06:07:43.571690+0000","last_became_peered":"2023-10-09T06:07:43.571690+0000","last_unstale":"2023-10-09T06:13:51.814003+0000","last_undegraded":"2023-10-09T06:13:51.814003+0000","last_fullsized":"2023-10-09T06:13:51.814003+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:10.234125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"536'7591","reported_seq":7697,"reported_epoch":536,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T06:14:01.567190+0000","last_change":"2023-10-09T06:14:01.451081+0000","last_active":"2023-10-09T06:14:01.567190+0000","last_peered":"2023-10-09T06:14:01.567190+0000","last_clean":"2023-10-09T06:13:20.539588+0000","last_became_active":"2023-10-09T06:13:22.846120+0000","last_became_peered":"2023-10-09T06:13:22.846120+0000","last_unstale":"2023-10-09T06:14:01.567190+0000","last_undegraded":"2023-10-09T06:14:01.567190+0000","last_fullsized":"2023-10-09T06:14:01.567190+0000","mapping_epoch":522,"log_start":"498'1000","ondisk_log_start":"498'1000","created":490,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6591,"log_dups_size":0,"ondisk_log_size":6591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:55.561925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304021504,"num_objects":4639,"num_object_clones":0,"num_object_copies":13917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4624,"num_objects_unfound":0,"num_objects_dirty":4639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7591,"num_write_kb":391360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":1048576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,4,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":4639}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":870,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.810606+0000","last_change":"2023-10-09T06:13:15.428701+0000","last_active":"2023-10-09T06:13:51.810606+0000","last_peered":"2023-10-09T06:13:51.810606+0000","last_clean":"2023-10-09T06:13:51.810606+0000","last_became_active":"2023-10-09T06:13:15.428343+0000","last_became_peered":"2023-10-09T06:13:15.428343+0000","last_unstale":"2023-10-09T06:13:51.810606+0000","last_undegraded":"2023-10-09T06:13:51.810606+0000","last_fullsized":"2023-10-09T06:13:51.810606+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.373748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.373748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:55:34.235074+0000","scrub_duration":0.0024576450000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":526,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.813453+0000","last_change":"2023-10-09T06:07:43.574863+0000","last_active":"2023-10-09T06:13:51.813453+0000","last_peered":"2023-10-09T06:13:51.813453+0000","last_clean":"2023-10-09T06:13:51.813453+0000","last_became_active":"2023-10-09T06:07:43.574576+0000","last_became_peered":"2023-10-09T06:07:43.574576+0000","last_unstale":"2023-10-09T06:13:51.813453+0000","last_undegraded":"2023-10-09T06:13:51.813453+0000","last_fullsized":"2023-10-09T06:13:51.813453+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:16.166004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":567,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.813960+0000","last_change":"2023-10-09T06:13:21.727814+0000","last_active":"2023-10-09T06:13:51.813960+0000","last_peered":"2023-10-09T06:13:51.813960+0000","last_clean":"2023-10-09T06:13:51.813960+0000","last_became_active":"2023-10-09T06:13:21.727401+0000","last_became_peered":"2023-10-09T06:13:21.727401+0000","last_unstale":"2023-10-09T06:13:51.813960+0000","last_undegraded":"2023-10-09T06:13:51.813960+0000","last_fullsized":"2023-10-09T06:13:51.813960+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:53:37.968028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"536'7504","reported_seq":7558,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:14:00.834084+0000","last_change":"2023-10-09T06:11:45.695709+0000","last_active":"2023-10-09T06:14:00.834084+0000","last_peered":"2023-10-09T06:14:00.834084+0000","last_clean":"2023-10-09T06:14:00.834084+0000","last_became_active":"2023-10-09T06:11:45.695219+0000","last_became_peered":"2023-10-09T06:11:45.695219+0000","last_unstale":"2023-10-09T06:14:00.834084+0000","last_undegraded":"2023-10-09T06:14:00.834084+0000","last_fullsized":"2023-10-09T06:14:00.834084+0000","mapping_epoch":490,"log_start":"504'2100","ondisk_log_start":"504'2100","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5404,"log_dups_size":0,"ondisk_log_size":5404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:26.533964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301989888,"num_objects":4608,"num_object_clones":0,"num_object_copies":13824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7504,"num_write_kb":387584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":614,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.812118+0000","last_change":"2023-10-09T06:13:24.753519+0000","last_active":"2023-10-09T06:13:51.812118+0000","last_peered":"2023-10-09T06:13:51.812118+0000","last_clean":"2023-10-09T06:13:51.812118+0000","last_became_active":"2023-10-09T06:13:24.753185+0000","last_became_peered":"2023-10-09T06:13:24.753185+0000","last_unstale":"2023-10-09T06:13:51.812118+0000","last_undegraded":"2023-10-09T06:13:51.812118+0000","last_fullsized":"2023-10-09T06:13:51.812118+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:39.633410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:39.633410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:56.871057+0000","scrub_duration":0.001547624,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":347,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.807819+0000","last_change":"2023-10-09T06:13:30.781849+0000","last_active":"2023-10-09T06:13:51.807819+0000","last_peered":"2023-10-09T06:13:51.807819+0000","last_clean":"2023-10-09T06:13:51.807819+0000","last_became_active":"2023-10-09T06:13:30.781084+0000","last_became_peered":"2023-10-09T06:13:30.781084+0000","last_unstale":"2023-10-09T06:13:51.807819+0000","last_undegraded":"2023-10-09T06:13:51.807819+0000","last_fullsized":"2023-10-09T06:13:51.807819+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:02.199298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":218,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.809366+0000","last_change":"2023-10-09T06:01:54.761768+0000","last_active":"2023-10-09T06:13:51.809366+0000","last_peered":"2023-10-09T06:13:51.809366+0000","last_clean":"2023-10-09T06:13:51.809366+0000","last_became_active":"2023-10-09T06:01:54.761443+0000","last_became_peered":"2023-10-09T06:01:54.761443+0000","last_unstale":"2023-10-09T06:13:51.809366+0000","last_undegraded":"2023-10-09T06:13:51.809366+0000","last_fullsized":"2023-10-09T06:13:51.809366+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:51.555987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"536'7489","reported_seq":7549,"reported_epoch":536,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:14:01.056657+0000","last_change":"2023-10-09T06:13:31.904728+0000","last_active":"2023-10-09T06:14:01.056657+0000","last_peered":"2023-10-09T06:14:01.056657+0000","last_clean":"2023-10-09T06:13:29.715670+0000","last_became_active":"2023-10-09T06:13:31.891405+0000","last_became_peered":"2023-10-09T06:13:31.891405+0000","last_unstale":"2023-10-09T06:14:01.056657+0000","last_undegraded":"2023-10-09T06:14:01.056657+0000","last_fullsized":"2023-10-09T06:14:01.056657+0000","mapping_epoch":527,"log_start":"498'1200","ondisk_log_start":"498'1200","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6289,"log_dups_size":0,"ondisk_log_size":6289,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:13:59.527962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309133312,"num_objects":4717,"num_object_clones":0,"num_object_copies":14151,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4717,"num_objects_unfound":0,"num_objects_dirty":4717,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7489,"num_write_kb":390592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[0,4,3],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,3,4","objects":4717}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":738,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.808253+0000","last_change":"2023-10-09T06:13:33.707885+0000","last_active":"2023-10-09T06:13:51.808253+0000","last_peered":"2023-10-09T06:13:51.808253+0000","last_clean":"2023-10-09T06:13:51.808253+0000","last_became_active":"2023-10-09T06:13:33.707474+0000","last_became_peered":"2023-10-09T06:13:33.707474+0000","last_unstale":"2023-10-09T06:13:51.808253+0000","last_undegraded":"2023-10-09T06:13:51.808253+0000","last_fullsized":"2023-10-09T06:13:51.808253+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:12:22.289803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:12:22.289803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:27.842817+0000","scrub_duration":0.0076565050000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":280,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.808897+0000","last_change":"2023-10-09T06:12:04.680247+0000","last_active":"2023-10-09T06:13:51.808897+0000","last_peered":"2023-10-09T06:13:51.808897+0000","last_clean":"2023-10-09T06:13:51.808897+0000","last_became_active":"2023-10-09T06:12:04.679943+0000","last_became_peered":"2023-10-09T06:12:04.679943+0000","last_unstale":"2023-10-09T06:13:51.808897+0000","last_undegraded":"2023-10-09T06:13:51.808897+0000","last_fullsized":"2023-10-09T06:13:51.808897+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:56.692774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":323,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.809530+0000","last_change":"2023-10-09T06:07:43.577306+0000","last_active":"2023-10-09T06:13:51.809530+0000","last_peered":"2023-10-09T06:13:51.809530+0000","last_clean":"2023-10-09T06:13:51.809530+0000","last_became_active":"2023-10-09T06:07:43.576956+0000","last_became_peered":"2023-10-09T06:07:43.576956+0000","last_unstale":"2023-10-09T06:13:51.809530+0000","last_undegraded":"2023-10-09T06:13:51.809530+0000","last_fullsized":"2023-10-09T06:13:51.809530+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:21.020403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":251,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.810273+0000","last_change":"2023-10-09T06:13:12.234849+0000","last_active":"2023-10-09T06:13:51.810273+0000","last_peered":"2023-10-09T06:13:51.810273+0000","last_clean":"2023-10-09T06:13:51.810273+0000","last_became_active":"2023-10-09T06:13:12.234284+0000","last_became_peered":"2023-10-09T06:13:12.234284+0000","last_unstale":"2023-10-09T06:13:51.810273+0000","last_undegraded":"2023-10-09T06:13:51.810273+0000","last_fullsized":"2023-10-09T06:13:51.810273+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:40:43.809773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"536'7308","reported_seq":7364,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:14:01.108354+0000","last_change":"2023-10-09T06:11:46.029349+0000","last_active":"2023-10-09T06:14:01.108354+0000","last_peered":"2023-10-09T06:14:01.108354+0000","last_clean":"2023-10-09T06:14:01.108354+0000","last_became_active":"2023-10-09T06:11:46.029041+0000","last_became_peered":"2023-10-09T06:11:46.029041+0000","last_unstale":"2023-10-09T06:14:01.108354+0000","last_undegraded":"2023-10-09T06:14:01.108354+0000","last_fullsized":"2023-10-09T06:14:01.108354+0000","mapping_epoch":490,"log_start":"498'900","ondisk_log_start":"498'900","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6408,"log_dups_size":0,"ondisk_log_size":6408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:07.662395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299106304,"num_objects":4564,"num_object_clones":0,"num_object_copies":13692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7308,"num_write_kb":379904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":771,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.844083+0000","last_change":"2023-10-09T06:09:27.544555+0000","last_active":"2023-10-09T06:13:51.844083+0000","last_peered":"2023-10-09T06:13:51.844083+0000","last_clean":"2023-10-09T06:13:51.844083+0000","last_became_active":"2023-10-09T06:09:27.544222+0000","last_became_peered":"2023-10-09T06:09:27.544222+0000","last_unstale":"2023-10-09T06:13:51.844083+0000","last_undegraded":"2023-10-09T06:13:51.844083+0000","last_fullsized":"2023-10-09T06:13:51.844083+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:45.944253+0000","scrub_duration":0.0016660799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":236,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.810351+0000","last_change":"2023-10-09T06:13:21.738040+0000","last_active":"2023-10-09T06:13:51.810351+0000","last_peered":"2023-10-09T06:13:51.810351+0000","last_clean":"2023-10-09T06:13:51.810351+0000","last_became_active":"2023-10-09T06:13:21.737717+0000","last_became_peered":"2023-10-09T06:13:21.737717+0000","last_unstale":"2023-10-09T06:13:51.810351+0000","last_undegraded":"2023-10-09T06:13:51.810351+0000","last_fullsized":"2023-10-09T06:13:51.810351+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:39.451628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":516,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.815116+0000","last_change":"2023-10-09T06:09:27.553958+0000","last_active":"2023-10-09T06:13:51.815116+0000","last_peered":"2023-10-09T06:13:51.815116+0000","last_clean":"2023-10-09T06:13:51.815116+0000","last_became_active":"2023-10-09T06:09:27.553193+0000","last_became_peered":"2023-10-09T06:09:27.553193+0000","last_unstale":"2023-10-09T06:13:51.815116+0000","last_undegraded":"2023-10-09T06:13:51.815116+0000","last_fullsized":"2023-10-09T06:13:51.815116+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:09.754579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":361,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.808969+0000","last_change":"2023-10-09T06:07:43.580765+0000","last_active":"2023-10-09T06:13:51.808969+0000","last_peered":"2023-10-09T06:13:51.808969+0000","last_clean":"2023-10-09T06:13:51.808969+0000","last_became_active":"2023-10-09T06:07:43.580458+0000","last_became_peered":"2023-10-09T06:07:43.580458+0000","last_unstale":"2023-10-09T06:13:51.808969+0000","last_undegraded":"2023-10-09T06:13:51.808969+0000","last_fullsized":"2023-10-09T06:13:51.808969+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:55.161893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"536'7383","reported_seq":14882,"reported_epoch":536,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:14:00.831561+0000","last_change":"2023-10-09T06:13:32.706549+0000","last_active":"2023-10-09T06:14:00.831561+0000","last_peered":"2023-10-09T06:14:00.831561+0000","last_clean":"2023-10-09T06:13:11.209726+0000","last_became_active":"2023-10-09T06:13:30.857105+0000","last_became_peered":"2023-10-09T06:13:30.857105+0000","last_unstale":"2023-10-09T06:14:00.831561+0000","last_undegraded":"2023-10-09T06:14:00.831561+0000","last_fullsized":"2023-10-09T06:14:00.831561+0000","mapping_epoch":526,"log_start":"504'2000","ondisk_log_start":"504'2000","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5383,"log_dups_size":0,"ondisk_log_size":5383,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:30.233175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296026112,"num_objects":4517,"num_object_clones":0,"num_object_copies":13551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6151,"num_objects_unfound":0,"num_objects_dirty":4517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7383,"num_write_kb":380800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3703,"num_bytes_recovered":242679808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[4,2,3],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"2,3,4","objects":4517}],"blocked_by":[],"up_primary":7,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":581,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.845360+0000","last_change":"2023-10-09T06:13:33.705726+0000","last_active":"2023-10-09T06:13:51.845360+0000","last_peered":"2023-10-09T06:13:51.845360+0000","last_clean":"2023-10-09T06:13:51.845360+0000","last_became_active":"2023-10-09T06:13:33.705351+0000","last_became_peered":"2023-10-09T06:13:33.705351+0000","last_unstale":"2023-10-09T06:13:51.845360+0000","last_undegraded":"2023-10-09T06:13:51.845360+0000","last_fullsized":"2023-10-09T06:13:51.845360+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:44.483427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.483427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":316,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.808891+0000","last_change":"2023-10-09T06:09:27.565004+0000","last_active":"2023-10-09T06:13:51.808891+0000","last_peered":"2023-10-09T06:13:51.808891+0000","last_clean":"2023-10-09T06:13:51.808891+0000","last_became_active":"2023-10-09T06:09:27.564073+0000","last_became_peered":"2023-10-09T06:09:27.564073+0000","last_unstale":"2023-10-09T06:13:51.808891+0000","last_undegraded":"2023-10-09T06:13:51.808891+0000","last_fullsized":"2023-10-09T06:13:51.808891+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:15.146042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":306,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:13:54.891308+0000","last_change":"2023-10-09T06:13:21.728291+0000","last_active":"2023-10-09T06:13:54.891308+0000","last_peered":"2023-10-09T06:13:54.891308+0000","last_clean":"2023-10-09T06:13:54.891308+0000","last_became_active":"2023-10-09T06:13:21.727926+0000","last_became_peered":"2023-10-09T06:13:21.727926+0000","last_unstale":"2023-10-09T06:13:54.891308+0000","last_undegraded":"2023-10-09T06:13:54.891308+0000","last_fullsized":"2023-10-09T06:13:54.891308+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:35.292785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":233,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.809296+0000","last_change":"2023-10-09T06:09:27.547953+0000","last_active":"2023-10-09T06:13:51.809296+0000","last_peered":"2023-10-09T06:13:51.809296+0000","last_clean":"2023-10-09T06:13:51.809296+0000","last_became_active":"2023-10-09T06:09:27.547624+0000","last_became_peered":"2023-10-09T06:09:27.547624+0000","last_unstale":"2023-10-09T06:13:51.809296+0000","last_undegraded":"2023-10-09T06:13:51.809296+0000","last_fullsized":"2023-10-09T06:13:51.809296+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:39.630486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"536'7366","reported_seq":10539,"reported_epoch":536,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:14:01.553946+0000","last_change":"2023-10-09T06:13:30.783853+0000","last_active":"2023-10-09T06:14:01.553946+0000","last_peered":"2023-10-09T06:14:01.553946+0000","last_clean":"2023-10-09T06:13:20.543162+0000","last_became_active":"2023-10-09T06:13:22.836318+0000","last_became_peered":"2023-10-09T06:13:22.836318+0000","last_unstale":"2023-10-09T06:14:01.553946+0000","last_undegraded":"2023-10-09T06:14:01.553946+0000","last_fullsized":"2023-10-09T06:14:01.553946+0000","mapping_epoch":522,"log_start":"498'800","ondisk_log_start":"498'800","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6566,"log_dups_size":0,"ondisk_log_size":6566,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:40.353464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299368448,"num_objects":4568,"num_object_clones":0,"num_object_copies":13704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3342,"num_objects_unfound":0,"num_objects_dirty":4568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7366,"num_write_kb":381888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1552,"num_bytes_recovered":101711872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,0,4],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":4568}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":19,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.845389+0000","last_change":"2023-10-09T06:13:30.791685+0000","last_active":"2023-10-09T06:13:51.845389+0000","last_peered":"2023-10-09T06:13:51.845389+0000","last_clean":"2023-10-09T06:13:51.845389+0000","last_became_active":"2023-10-09T06:13:30.791255+0000","last_became_peered":"2023-10-09T06:13:30.791255+0000","last_unstale":"2023-10-09T06:13:51.845389+0000","last_undegraded":"2023-10-09T06:13:51.845389+0000","last_fullsized":"2023-10-09T06:13:51.845389+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:24.896244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":579,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.845076+0000","last_change":"2023-10-09T06:13:33.704835+0000","last_active":"2023-10-09T06:13:51.845076+0000","last_peered":"2023-10-09T06:13:51.845076+0000","last_clean":"2023-10-09T06:13:51.845076+0000","last_became_active":"2023-10-09T06:13:33.704466+0000","last_became_peered":"2023-10-09T06:13:33.704466+0000","last_unstale":"2023-10-09T06:13:51.845076+0000","last_undegraded":"2023-10-09T06:13:51.845076+0000","last_fullsized":"2023-10-09T06:13:51.845076+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:35.783434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:35.783434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:38.721974+0000","scrub_duration":0.0043794799999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":323,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.808119+0000","last_change":"2023-10-09T06:13:12.238280+0000","last_active":"2023-10-09T06:13:51.808119+0000","last_peered":"2023-10-09T06:13:51.808119+0000","last_clean":"2023-10-09T06:13:51.808119+0000","last_became_active":"2023-10-09T06:13:12.237986+0000","last_became_peered":"2023-10-09T06:13:12.237986+0000","last_unstale":"2023-10-09T06:13:51.808119+0000","last_undegraded":"2023-10-09T06:13:51.808119+0000","last_fullsized":"2023-10-09T06:13:51.808119+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:58.520176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":466,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.813930+0000","last_change":"2023-10-09T06:13:21.733940+0000","last_active":"2023-10-09T06:13:51.813930+0000","last_peered":"2023-10-09T06:13:51.813930+0000","last_clean":"2023-10-09T06:13:51.813930+0000","last_became_active":"2023-10-09T06:13:21.733628+0000","last_became_peered":"2023-10-09T06:13:21.733628+0000","last_unstale":"2023-10-09T06:13:51.813930+0000","last_undegraded":"2023-10-09T06:13:51.813930+0000","last_fullsized":"2023-10-09T06:13:51.813930+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:13.348443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"536'7367","reported_seq":15173,"reported_epoch":536,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T06:14:01.156195+0000","last_change":"2023-10-09T06:13:32.707214+0000","last_active":"2023-10-09T06:14:01.156195+0000","last_peered":"2023-10-09T06:14:01.156195+0000","last_clean":"2023-10-09T06:13:29.756900+0000","last_became_active":"2023-10-09T06:13:31.891262+0000","last_became_peered":"2023-10-09T06:13:31.891262+0000","last_unstale":"2023-10-09T06:14:01.156195+0000","last_undegraded":"2023-10-09T06:14:01.156195+0000","last_fullsized":"2023-10-09T06:14:01.156195+0000","mapping_epoch":527,"log_start":"498'900","ondisk_log_start":"498'900","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6467,"log_dups_size":0,"ondisk_log_size":6467,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297992192,"num_objects":4547,"num_object_clones":0,"num_object_copies":13641,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1107,"num_objects_unfound":0,"num_objects_dirty":4547,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7367,"num_write_kb":381248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3871,"num_bytes_recovered":253689856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[0,2,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":4547}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":804,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.810704+0000","last_change":"2023-10-09T06:13:24.753842+0000","last_active":"2023-10-09T06:13:51.810704+0000","last_peered":"2023-10-09T06:13:51.810704+0000","last_clean":"2023-10-09T06:13:51.810704+0000","last_became_active":"2023-10-09T06:13:24.753485+0000","last_became_peered":"2023-10-09T06:13:24.753485+0000","last_unstale":"2023-10-09T06:13:51.810704+0000","last_undegraded":"2023-10-09T06:13:51.810704+0000","last_fullsized":"2023-10-09T06:13:51.810704+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:57.627296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:57.627296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:56.656218+0000","scrub_duration":0.001913166,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":429,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.845310+0000","last_change":"2023-10-09T06:09:27.544373+0000","last_active":"2023-10-09T06:13:51.845310+0000","last_peered":"2023-10-09T06:13:51.845310+0000","last_clean":"2023-10-09T06:13:51.845310+0000","last_became_active":"2023-10-09T06:09:27.544121+0000","last_became_peered":"2023-10-09T06:09:27.544121+0000","last_unstale":"2023-10-09T06:13:51.845310+0000","last_undegraded":"2023-10-09T06:13:51.845310+0000","last_fullsized":"2023-10-09T06:13:51.845310+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:40.429491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":473,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.843893+0000","last_change":"2023-10-09T06:12:52.670573+0000","last_active":"2023-10-09T06:13:51.843893+0000","last_peered":"2023-10-09T06:13:51.843893+0000","last_clean":"2023-10-09T06:13:51.843893+0000","last_became_active":"2023-10-09T06:12:52.670242+0000","last_became_peered":"2023-10-09T06:12:52.670242+0000","last_unstale":"2023-10-09T06:13:51.843893+0000","last_undegraded":"2023-10-09T06:13:51.843893+0000","last_fullsized":"2023-10-09T06:13:51.843893+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:25.237580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":612,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.845086+0000","last_change":"2023-10-09T06:09:27.543678+0000","last_active":"2023-10-09T06:13:51.845086+0000","last_peered":"2023-10-09T06:13:51.845086+0000","last_clean":"2023-10-09T06:13:51.845086+0000","last_became_active":"2023-10-09T06:09:27.543298+0000","last_became_peered":"2023-10-09T06:09:27.543298+0000","last_unstale":"2023-10-09T06:13:51.845086+0000","last_undegraded":"2023-10-09T06:13:51.845086+0000","last_fullsized":"2023-10-09T06:13:51.845086+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:26.297842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"536'7402","reported_seq":16625,"reported_epoch":536,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T06:14:00.877869+0000","last_change":"2023-10-09T06:13:50.058863+0000","last_active":"2023-10-09T06:14:00.877869+0000","last_peered":"2023-10-09T06:14:00.877869+0000","last_clean":"2023-10-09T06:13:20.517658+0000","last_became_active":"2023-10-09T06:13:22.825586+0000","last_became_peered":"2023-10-09T06:13:22.825586+0000","last_unstale":"2023-10-09T06:14:00.877869+0000","last_undegraded":"2023-10-09T06:14:00.877869+0000","last_fullsized":"2023-10-09T06:14:00.877869+0000","mapping_epoch":522,"log_start":"504'2000","ondisk_log_start":"504'2000","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5402,"log_dups_size":0,"ondisk_log_size":5402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:09.871700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300220440,"num_objects":4582,"num_object_clones":0,"num_object_copies":13746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":176,"num_objects_unfound":0,"num_objects_dirty":4582,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7402,"num_write_kb":383425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4580,"num_bytes_recovered":300089368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,0,3],"avail_no_missing":["4","0","3"],"object_location_counts":[{"shards":"0,3,4","objects":4582}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":735,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.808443+0000","last_change":"2023-10-09T06:13:24.756288+0000","last_active":"2023-10-09T06:13:51.808443+0000","last_peered":"2023-10-09T06:13:51.808443+0000","last_clean":"2023-10-09T06:13:51.808443+0000","last_became_active":"2023-10-09T06:13:24.755945+0000","last_became_peered":"2023-10-09T06:13:24.755945+0000","last_unstale":"2023-10-09T06:13:51.808443+0000","last_undegraded":"2023-10-09T06:13:51.808443+0000","last_fullsized":"2023-10-09T06:13:51.808443+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:45.844892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:45.844892+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:44.107376+0000","scrub_duration":0.0017521489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.845501+0000","last_change":"2023-10-09T06:13:30.799535+0000","last_active":"2023-10-09T06:13:51.845501+0000","last_peered":"2023-10-09T06:13:51.845501+0000","last_clean":"2023-10-09T06:13:51.845501+0000","last_became_active":"2023-10-09T06:13:30.799227+0000","last_became_peered":"2023-10-09T06:13:30.799227+0000","last_unstale":"2023-10-09T06:13:51.845501+0000","last_undegraded":"2023-10-09T06:13:51.845501+0000","last_fullsized":"2023-10-09T06:13:51.845501+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:14.741742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":254,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.809051+0000","last_change":"2023-10-09T06:13:12.237622+0000","last_active":"2023-10-09T06:13:51.809051+0000","last_peered":"2023-10-09T06:13:51.809051+0000","last_clean":"2023-10-09T06:13:51.809051+0000","last_became_active":"2023-10-09T06:13:12.237287+0000","last_became_peered":"2023-10-09T06:13:12.237287+0000","last_unstale":"2023-10-09T06:13:51.809051+0000","last_undegraded":"2023-10-09T06:13:51.809051+0000","last_fullsized":"2023-10-09T06:13:51.809051+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:01.942898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":19,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.845233+0000","last_change":"2023-10-09T06:13:30.788382+0000","last_active":"2023-10-09T06:13:51.845233+0000","last_peered":"2023-10-09T06:13:51.845233+0000","last_clean":"2023-10-09T06:13:51.845233+0000","last_became_active":"2023-10-09T06:13:30.788060+0000","last_became_peered":"2023-10-09T06:13:30.788060+0000","last_unstale":"2023-10-09T06:13:51.845233+0000","last_undegraded":"2023-10-09T06:13:51.845233+0000","last_fullsized":"2023-10-09T06:13:51.845233+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:36:58.984799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.e","version":"536'7501","reported_seq":18587,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:14:00.849453+0000","last_change":"2023-10-09T06:13:51.831598+0000","last_active":"2023-10-09T06:14:00.849453+0000","last_peered":"2023-10-09T06:14:00.849453+0000","last_clean":"2023-10-09T06:14:00.849453+0000","last_became_active":"2023-10-09T06:13:51.816987+0000","last_became_peered":"2023-10-09T06:13:51.816987+0000","last_unstale":"2023-10-09T06:14:00.849453+0000","last_undegraded":"2023-10-09T06:14:00.849453+0000","last_fullsized":"2023-10-09T06:14:00.849453+0000","mapping_epoch":534,"log_start":"504'2100","ondisk_log_start":"504'2100","created":490,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5401,"log_dups_size":0,"ondisk_log_size":5401,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:52.460066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306380800,"num_objects":4675,"num_object_clones":0,"num_object_copies":14025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7501,"num_write_kb":389632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5503,"num_bytes_recovered":360644608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":1009,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.814152+0000","last_change":"2023-10-09T06:13:24.762291+0000","last_active":"2023-10-09T06:13:51.814152+0000","last_peered":"2023-10-09T06:13:51.814152+0000","last_clean":"2023-10-09T06:13:51.814152+0000","last_became_active":"2023-10-09T06:13:24.761918+0000","last_became_peered":"2023-10-09T06:13:24.761918+0000","last_unstale":"2023-10-09T06:13:51.814152+0000","last_undegraded":"2023-10-09T06:13:51.814152+0000","last_fullsized":"2023-10-09T06:13:51.814152+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:17.344518+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:17.344518+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:58.227603+0000","scrub_duration":0.0015914099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":592,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.808751+0000","last_change":"2023-10-09T06:13:30.798394+0000","last_active":"2023-10-09T06:13:51.808751+0000","last_peered":"2023-10-09T06:13:51.808751+0000","last_clean":"2023-10-09T06:13:51.808751+0000","last_became_active":"2023-10-09T06:13:30.790913+0000","last_became_peered":"2023-10-09T06:13:30.790913+0000","last_unstale":"2023-10-09T06:13:51.808751+0000","last_undegraded":"2023-10-09T06:13:51.808751+0000","last_fullsized":"2023-10-09T06:13:51.808751+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:26.508690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":352,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.809741+0000","last_change":"2023-10-09T06:13:21.728009+0000","last_active":"2023-10-09T06:13:51.809741+0000","last_peered":"2023-10-09T06:13:51.809741+0000","last_clean":"2023-10-09T06:13:51.809741+0000","last_became_active":"2023-10-09T06:13:21.727683+0000","last_became_peered":"2023-10-09T06:13:21.727683+0000","last_unstale":"2023-10-09T06:13:51.809741+0000","last_undegraded":"2023-10-09T06:13:51.809741+0000","last_fullsized":"2023-10-09T06:13:51.809741+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:13.074232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":341,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.808075+0000","last_change":"2023-10-09T06:13:30.782038+0000","last_active":"2023-10-09T06:13:51.808075+0000","last_peered":"2023-10-09T06:13:51.808075+0000","last_clean":"2023-10-09T06:13:51.808075+0000","last_became_active":"2023-10-09T06:13:30.781500+0000","last_became_peered":"2023-10-09T06:13:30.781500+0000","last_unstale":"2023-10-09T06:13:51.808075+0000","last_undegraded":"2023-10-09T06:13:51.808075+0000","last_fullsized":"2023-10-09T06:13:51.808075+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:43.663265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"536'7328","reported_seq":7384,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:14:00.209512+0000","last_change":"2023-10-09T06:11:46.031709+0000","last_active":"2023-10-09T06:14:00.209512+0000","last_peered":"2023-10-09T06:14:00.209512+0000","last_clean":"2023-10-09T06:14:00.209512+0000","last_became_active":"2023-10-09T06:11:46.031421+0000","last_became_peered":"2023-10-09T06:11:46.031421+0000","last_unstale":"2023-10-09T06:14:00.209512+0000","last_undegraded":"2023-10-09T06:14:00.209512+0000","last_fullsized":"2023-10-09T06:14:00.209512+0000","mapping_epoch":490,"log_start":"500'1425","ondisk_log_start":"500'1425","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5903,"log_dups_size":0,"ondisk_log_size":5903,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:13.095174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300941312,"num_objects":4592,"num_object_clones":0,"num_object_copies":13776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7328,"num_write_kb":381440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":867,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.809226+0000","last_change":"2023-10-09T06:13:33.708100+0000","last_active":"2023-10-09T06:13:51.809226+0000","last_peered":"2023-10-09T06:13:51.809226+0000","last_clean":"2023-10-09T06:13:51.809226+0000","last_became_active":"2023-10-09T06:13:33.707809+0000","last_became_peered":"2023-10-09T06:13:33.707809+0000","last_unstale":"2023-10-09T06:13:51.809226+0000","last_undegraded":"2023-10-09T06:13:51.809226+0000","last_fullsized":"2023-10-09T06:13:51.809226+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:43:26.848778+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.847298+0000","last_change":"2023-10-09T06:13:30.799948+0000","last_active":"2023-10-09T06:13:51.847298+0000","last_peered":"2023-10-09T06:13:51.847298+0000","last_clean":"2023-10-09T06:13:51.847298+0000","last_became_active":"2023-10-09T06:13:30.799690+0000","last_became_peered":"2023-10-09T06:13:30.799690+0000","last_unstale":"2023-10-09T06:13:51.847298+0000","last_undegraded":"2023-10-09T06:13:51.847298+0000","last_fullsized":"2023-10-09T06:13:51.847298+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:43.893184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":582,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.845312+0000","last_change":"2023-10-09T06:09:27.578756+0000","last_active":"2023-10-09T06:13:51.845312+0000","last_peered":"2023-10-09T06:13:51.845312+0000","last_clean":"2023-10-09T06:13:51.845312+0000","last_became_active":"2023-10-09T06:09:27.578384+0000","last_became_peered":"2023-10-09T06:09:27.578384+0000","last_unstale":"2023-10-09T06:13:51.845312+0000","last_undegraded":"2023-10-09T06:13:51.845312+0000","last_fullsized":"2023-10-09T06:13:51.845312+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:22.182630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":482,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.813470+0000","last_change":"2023-10-09T06:01:22.334081+0000","last_active":"2023-10-09T06:13:51.813470+0000","last_peered":"2023-10-09T06:13:51.813470+0000","last_clean":"2023-10-09T06:13:51.813470+0000","last_became_active":"2023-10-09T06:01:22.333663+0000","last_became_peered":"2023-10-09T06:01:22.333663+0000","last_unstale":"2023-10-09T06:13:51.813470+0000","last_undegraded":"2023-10-09T06:13:51.813470+0000","last_fullsized":"2023-10-09T06:13:51.813470+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:55.809259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":781,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.808679+0000","last_change":"2023-10-09T06:09:27.560157+0000","last_active":"2023-10-09T06:13:51.808679+0000","last_peered":"2023-10-09T06:13:51.808679+0000","last_clean":"2023-10-09T06:13:51.808679+0000","last_became_active":"2023-10-09T06:09:27.559466+0000","last_became_peered":"2023-10-09T06:09:27.559466+0000","last_unstale":"2023-10-09T06:13:51.808679+0000","last_undegraded":"2023-10-09T06:13:51.808679+0000","last_fullsized":"2023-10-09T06:13:51.808679+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.181811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.181811+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:53.431491+0000","scrub_duration":0.011553313000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":464,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.846140+0000","last_change":"2023-10-09T06:13:30.791343+0000","last_active":"2023-10-09T06:13:51.846140+0000","last_peered":"2023-10-09T06:13:51.846140+0000","last_clean":"2023-10-09T06:13:51.846140+0000","last_became_active":"2023-10-09T06:13:30.790945+0000","last_became_peered":"2023-10-09T06:13:30.790945+0000","last_unstale":"2023-10-09T06:13:51.846140+0000","last_undegraded":"2023-10-09T06:13:51.846140+0000","last_fullsized":"2023-10-09T06:13:51.846140+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:14.208391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":237,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.809127+0000","last_change":"2023-10-09T06:07:43.580844+0000","last_active":"2023-10-09T06:13:51.809127+0000","last_peered":"2023-10-09T06:13:51.809127+0000","last_clean":"2023-10-09T06:13:51.809127+0000","last_became_active":"2023-10-09T06:07:43.580523+0000","last_became_peered":"2023-10-09T06:07:43.580523+0000","last_unstale":"2023-10-09T06:13:51.809127+0000","last_undegraded":"2023-10-09T06:13:51.809127+0000","last_fullsized":"2023-10-09T06:13:51.809127+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:55.942430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":885,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.808281+0000","last_change":"2023-10-09T06:00:04.215425+0000","last_active":"2023-10-09T06:13:51.808281+0000","last_peered":"2023-10-09T06:13:51.808281+0000","last_clean":"2023-10-09T06:13:51.808281+0000","last_became_active":"2023-10-09T05:59:40.326685+0000","last_became_peered":"2023-10-09T05:59:40.326685+0000","last_unstale":"2023-10-09T06:13:51.808281+0000","last_undegraded":"2023-10-09T06:13:51.808281+0000","last_fullsized":"2023-10-09T06:13:51.808281+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:04.215270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:04.215270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:03.963694+0000","scrub_duration":0.002850652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":307,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.813486+0000","last_change":"2023-10-09T06:01:54.824780+0000","last_active":"2023-10-09T06:13:51.813486+0000","last_peered":"2023-10-09T06:13:51.813486+0000","last_clean":"2023-10-09T06:13:51.813486+0000","last_became_active":"2023-10-09T06:01:54.757727+0000","last_became_peered":"2023-10-09T06:01:54.757727+0000","last_unstale":"2023-10-09T06:13:51.813486+0000","last_undegraded":"2023-10-09T06:13:51.813486+0000","last_fullsized":"2023-10-09T06:13:51.813486+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:44.132556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":554,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.815512+0000","last_change":"2023-10-09T06:01:54.764989+0000","last_active":"2023-10-09T06:13:51.815512+0000","last_peered":"2023-10-09T06:13:51.815512+0000","last_clean":"2023-10-09T06:13:51.815512+0000","last_became_active":"2023-10-09T06:01:54.764790+0000","last_became_peered":"2023-10-09T06:01:54.764790+0000","last_unstale":"2023-10-09T06:13:51.815512+0000","last_undegraded":"2023-10-09T06:13:51.815512+0000","last_fullsized":"2023-10-09T06:13:51.815512+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:14:07.208858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":321,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.813717+0000","last_change":"2023-10-09T06:13:21.739730+0000","last_active":"2023-10-09T06:13:51.813717+0000","last_peered":"2023-10-09T06:13:51.813717+0000","last_clean":"2023-10-09T06:13:51.813717+0000","last_became_active":"2023-10-09T06:13:21.727326+0000","last_became_peered":"2023-10-09T06:13:21.727326+0000","last_unstale":"2023-10-09T06:13:51.813717+0000","last_undegraded":"2023-10-09T06:13:51.813717+0000","last_fullsized":"2023-10-09T06:13:51.813717+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:51.344571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":551,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.813522+0000","last_change":"2023-10-09T06:09:27.565198+0000","last_active":"2023-10-09T06:13:51.813522+0000","last_peered":"2023-10-09T06:13:51.813522+0000","last_clean":"2023-10-09T06:13:51.813522+0000","last_became_active":"2023-10-09T06:09:27.564909+0000","last_became_peered":"2023-10-09T06:09:27.564909+0000","last_unstale":"2023-10-09T06:13:51.813522+0000","last_undegraded":"2023-10-09T06:13:51.813522+0000","last_fullsized":"2023-10-09T06:13:51.813522+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:46.101014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":701,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.845541+0000","last_change":"2023-10-09T06:13:33.699235+0000","last_active":"2023-10-09T06:13:51.845541+0000","last_peered":"2023-10-09T06:13:51.845541+0000","last_clean":"2023-10-09T06:13:51.845541+0000","last_became_active":"2023-10-09T06:13:33.698886+0000","last_became_peered":"2023-10-09T06:13:33.698886+0000","last_unstale":"2023-10-09T06:13:51.845541+0000","last_undegraded":"2023-10-09T06:13:51.845541+0000","last_fullsized":"2023-10-09T06:13:51.845541+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:00.456132+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:00.456132+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:59.421706+0000","scrub_duration":0.0014992199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":342,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.808543+0000","last_change":"2023-10-09T06:13:21.740383+0000","last_active":"2023-10-09T06:13:51.808543+0000","last_peered":"2023-10-09T06:13:51.808543+0000","last_clean":"2023-10-09T06:13:51.808543+0000","last_became_active":"2023-10-09T06:13:21.740023+0000","last_became_peered":"2023-10-09T06:13:21.740023+0000","last_unstale":"2023-10-09T06:13:51.808543+0000","last_undegraded":"2023-10-09T06:13:51.808543+0000","last_fullsized":"2023-10-09T06:13:51.808543+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:27.380765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":337,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.808964+0000","last_change":"2023-10-09T06:09:27.565185+0000","last_active":"2023-10-09T06:13:51.808964+0000","last_peered":"2023-10-09T06:13:51.808964+0000","last_clean":"2023-10-09T06:13:51.808964+0000","last_became_active":"2023-10-09T06:09:27.564478+0000","last_became_peered":"2023-10-09T06:09:27.564478+0000","last_unstale":"2023-10-09T06:13:51.808964+0000","last_undegraded":"2023-10-09T06:13:51.808964+0000","last_fullsized":"2023-10-09T06:13:51.808964+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:04.622171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":859,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.808380+0000","last_change":"2023-10-09T06:12:55.753877+0000","last_active":"2023-10-09T06:13:51.808380+0000","last_peered":"2023-10-09T06:13:51.808380+0000","last_clean":"2023-10-09T06:13:51.808380+0000","last_became_active":"2023-10-09T06:12:55.753512+0000","last_became_peered":"2023-10-09T06:12:55.753512+0000","last_unstale":"2023-10-09T06:13:51.808380+0000","last_undegraded":"2023-10-09T06:13:51.808380+0000","last_fullsized":"2023-10-09T06:13:51.808380+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:46.290510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:46.290510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:30.368031+0000","scrub_duration":0.002627819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":528,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.844855+0000","last_change":"2023-10-09T06:09:27.556271+0000","last_active":"2023-10-09T06:13:51.844855+0000","last_peered":"2023-10-09T06:13:51.844855+0000","last_clean":"2023-10-09T06:13:51.844855+0000","last_became_active":"2023-10-09T06:09:27.555932+0000","last_became_peered":"2023-10-09T06:09:27.555932+0000","last_unstale":"2023-10-09T06:13:51.844855+0000","last_undegraded":"2023-10-09T06:13:51.844855+0000","last_fullsized":"2023-10-09T06:13:51.844855+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:33.888622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":567,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:13:51.812944+0000","last_change":"2023-10-09T06:13:21.727720+0000","last_active":"2023-10-09T06:13:51.812944+0000","last_peered":"2023-10-09T06:13:51.812944+0000","last_clean":"2023-10-09T06:13:51.812944+0000","last_became_active":"2023-10-09T06:13:21.727339+0000","last_became_peered":"2023-10-09T06:13:21.727339+0000","last_unstale":"2023-10-09T06:13:51.812944+0000","last_undegraded":"2023-10-09T06:13:51.812944+0000","last_fullsized":"2023-10-09T06:13:51.812944+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:25.723752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4826595352,"num_objects":73649,"num_object_clones":0,"num_object_copies":220947,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":28743,"num_objects_unfound":0,"num_objects_dirty":73649,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":118777,"num_write_kb":6157569,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25758,"num_bytes_recovered":1688010776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15985491968,"data_stored":15985475680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96624,"ondisk_log_size":96624,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":11714,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13410,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":617203,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1523929,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234412,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1455548,"kb_used_data":558836,"kb_used_omap":0,"kb_used_meta":896704,"kb_avail":92289604,"statfs":{"total":95995035648,"available":94504554496,"internally_reserved":0,"allocated":572248064,"data_stored":570730056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918224896},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:31 2023","interfaces":[{"interface":"back","average":{"1min":8.7140000000000004,"5min":2.887,"15min":1.784},"min":{"1min":0.26900000000000002,"5min":0.25,"15min":0.25},"max":{"1min":57.435000000000002,"5min":57.435000000000002,"15min":57.435000000000002},"last":3.335},{"interface":"front","average":{"1min":8.8059999999999992,"5min":3.0590000000000002,"15min":1.7829999999999999},"min":{"1min":0.318,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":57.472999999999999,"5min":57.472999999999999,"15min":57.472999999999999},"last":3.3700000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:13:01 2023","interfaces":[{"interface":"back","average":{"1min":3.407,"5min":4.2830000000000004,"15min":3.1709999999999998},"min":{"1min":0.29099999999999998,"5min":0.218,"15min":0.218},"max":{"1min":58.040999999999997,"5min":124.66200000000001,"15min":124.66200000000001},"last":2.677},{"interface":"front","average":{"1min":3.7109999999999999,"5min":4.3499999999999996,"15min":3.1720000000000002},"min":{"1min":0.39700000000000002,"5min":0.26300000000000001,"15min":0.22700000000000001},"max":{"1min":58.786999999999999,"5min":125.023,"15min":125.023},"last":3.5070000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:13:31 2023","interfaces":[{"interface":"back","average":{"1min":3.456,"5min":1.788,"15min":2.222},"min":{"1min":0.316,"5min":0.27400000000000002,"15min":0.24299999999999999},"max":{"1min":47.503,"5min":47.503,"15min":127.86499999999999},"last":0.56699999999999995},{"interface":"front","average":{"1min":2.9900000000000002,"5min":2.0110000000000001,"15min":2.2250000000000001},"min":{"1min":0.318,"5min":0.29599999999999999,"15min":0.27000000000000002},"max":{"1min":47.545000000000002,"5min":47.545000000000002,"15min":129.24000000000001},"last":0.373}]},{"osd":3,"last update":"Mon Oct 9 06:13:01 2023","interfaces":[{"interface":"back","average":{"1min":5.2770000000000001,"5min":6.2149999999999999,"15min":5.8540000000000001},"min":{"1min":0.35199999999999998,"5min":0.32100000000000001,"15min":0.29899999999999999},"max":{"1min":58.448999999999998,"5min":116.68000000000001,"15min":164.16999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":5.0229999999999997,"5min":6.6079999999999997,"15min":6.0330000000000004},"min":{"1min":0.34300000000000003,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":58.945,"5min":116.797,"15min":164.40700000000001},"last":3.7490000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":1.1220000000000001,"15min":0.92600000000000005},"min":{"1min":0.27100000000000002,"5min":0.23999999999999999,"15min":0.23100000000000001},"max":{"1min":3.4180000000000001,"5min":14.371,"15min":14.371},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.51300000000000001,"5min":1.002,"15min":0.91500000000000004},"min":{"1min":0.30199999999999999,"5min":0.28599999999999998,"15min":0.27100000000000002},"max":{"1min":1.3220000000000001,"5min":14.259,"15min":14.259},"last":0.442}]},{"osd":5,"last update":"Mon Oct 9 06:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":1.381,"15min":1.0669999999999999},"min":{"1min":0.317,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.6589999999999998,"5min":43.826999999999998,"15min":43.826999999999998},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.53800000000000003,"5min":1.28,"15min":1.119},"min":{"1min":0.39200000000000002,"5min":0.27300000000000002,"15min":0.24399999999999999},"max":{"1min":0.85399999999999998,"5min":43.880000000000003,"15min":43.880000000000003},"last":0.54600000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.85599999999999998,"15min":0.97999999999999998},"min":{"1min":0.33600000000000002,"5min":0.28000000000000003,"15min":0.26600000000000001},"max":{"1min":2.9209999999999998,"5min":4.6779999999999999,"15min":9.4540000000000006},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.94099999999999995,"15min":1.0089999999999999},"min":{"1min":0.35299999999999998,"5min":0.307,"15min":0.30599999999999999},"max":{"1min":2.3940000000000001,"5min":6.2329999999999997,"15min":6.2329999999999997},"last":0.60999999999999999}]}]},{"osd":6,"up_from":46,"seq":197568496049,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2107436,"kb_used_data":1213924,"kb_used_omap":0,"kb_used_meta":893440,"kb_avail":91637716,"statfs":{"total":95995035648,"available":93837021184,"internally_reserved":0,"allocated":1243058176,"data_stored":1241555621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":914882560},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:28 2023","interfaces":[{"interface":"back","average":{"1min":18.359000000000002,"5min":7.04,"15min":3.8420000000000001},"min":{"1min":0.246,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":181.209,"5min":191.50800000000001,"15min":191.50800000000001},"last":2.024},{"interface":"front","average":{"1min":18.960999999999999,"5min":7.0990000000000002,"15min":3.6920000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":181.233,"5min":191.52600000000001,"15min":191.52600000000001},"last":2.0379999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:13:23 2023","interfaces":[{"interface":"back","average":{"1min":26.061,"5min":6.2930000000000001,"15min":5.1870000000000003},"min":{"1min":0.27000000000000002,"5min":0.25,"15min":0.23200000000000001},"max":{"1min":328.04399999999998,"5min":328.04399999999998,"15min":328.04399999999998},"last":0.34000000000000002},{"interface":"front","average":{"1min":26.131,"5min":6.4470000000000001,"15min":5.2800000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":328.10000000000002,"5min":328.10000000000002,"15min":328.10000000000002},"last":0.92400000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:13:28 2023","interfaces":[{"interface":"back","average":{"1min":18.006,"5min":4.798,"15min":3.0049999999999999},"min":{"1min":0.28000000000000003,"5min":0.27300000000000002,"15min":0.26200000000000001},"max":{"1min":163.04900000000001,"5min":163.04900000000001,"15min":163.04900000000001},"last":1.726},{"interface":"front","average":{"1min":17.959,"5min":4.9500000000000002,"15min":2.96},"min":{"1min":0.31900000000000001,"5min":0.29399999999999998,"15min":0.28599999999999998},"max":{"1min":162.93000000000001,"5min":162.93000000000001,"15min":162.93000000000001},"last":0.433}]},{"osd":3,"last update":"Mon Oct 9 06:13:11 2023","interfaces":[{"interface":"back","average":{"1min":17.170000000000002,"5min":6.9180000000000001,"15min":6.9450000000000003},"min":{"1min":0.318,"5min":0.318,"15min":0.307},"max":{"1min":250.744,"5min":250.744,"15min":250.744},"last":0.63300000000000001},{"interface":"front","average":{"1min":16.963999999999999,"5min":6.4390000000000001,"15min":6.8479999999999999},"min":{"1min":0.34499999999999997,"5min":0.33600000000000002,"15min":0.314},"max":{"1min":250.83799999999999,"5min":250.83799999999999,"15min":250.83799999999999},"last":0.498}]},{"osd":4,"last update":"Mon Oct 9 06:13:47 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":1.0740000000000001,"15min":1.681},"min":{"1min":0.28100000000000003,"5min":0.24399999999999999,"15min":0.24199999999999999},"max":{"1min":9.0869999999999997,"5min":9.0869999999999997,"15min":151.35599999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.97,"5min":1.175,"15min":1.7350000000000001},"min":{"1min":0.33500000000000002,"5min":0.249,"15min":0.23999999999999999},"max":{"1min":7.8849999999999998,"5min":7.8849999999999998,"15min":149.15799999999999},"last":0.65900000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:13:23 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.95599999999999996,"15min":1.0629999999999999},"min":{"1min":0.29699999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.3799999999999999,"5min":9.1959999999999997,"15min":15.342000000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.90000000000000002,"5min":1.1100000000000001,"15min":1.131},"min":{"1min":0.312,"5min":0.28199999999999997,"15min":0.255},"max":{"1min":2.762,"5min":8.9809999999999999,"15min":15.358000000000001},"last":0.56200000000000006}]},{"osd":7,"last update":"Mon Oct 9 06:13:47 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":1.105,"15min":1.2070000000000001},"min":{"1min":0.372,"5min":0.30299999999999999,"15min":0.27000000000000002},"max":{"1min":5.819,"5min":5.819,"15min":21.98},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.79400000000000004,"5min":1.002,"15min":1.1559999999999999},"min":{"1min":0.39300000000000002,"5min":0.34200000000000003,"15min":0.33100000000000002},"max":{"1min":2.8570000000000002,"5min":5.3710000000000004,"15min":18.241},"last":0.60099999999999998}]}]},{"osd":1,"up_from":89,"seq":382252089637,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1476948,"kb_used_data":890380,"kb_used_omap":9,"kb_used_meta":586550,"kb_avail":92268204,"statfs":{"total":95995035648,"available":94482640896,"internally_reserved":0,"allocated":911749120,"data_stored":908685544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":600628201},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:35 2023","interfaces":[{"interface":"back","average":{"1min":1.7809999999999999,"5min":1.946,"15min":2.1659999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.613,"5min":10.613,"15min":10.613},"last":5.0990000000000002},{"interface":"front","average":{"1min":2.79,"5min":2.391,"15min":2.3570000000000002},"min":{"1min":0.254,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":13.121,"5min":24.239999999999998,"15min":24.239999999999998},"last":0.42799999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:13:44 2023","interfaces":[{"interface":"back","average":{"1min":2.7469999999999999,"5min":1.9850000000000001,"15min":2.4660000000000002},"min":{"1min":0.27800000000000002,"5min":0.21099999999999999,"15min":0.20799999999999999},"max":{"1min":12.818,"5min":15.522,"15min":129.053},"last":4.1239999999999997},{"interface":"front","average":{"1min":2.3540000000000001,"5min":2.4900000000000002,"15min":2.6629999999999998},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.20300000000000001},"max":{"1min":8.9949999999999992,"5min":13.539999999999999,"15min":129.077},"last":12.369999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:13:11 2023","interfaces":[{"interface":"back","average":{"1min":4.851,"5min":4.5389999999999997,"15min":4.75},"min":{"1min":0.26800000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":51.069000000000003,"5min":113.322,"15min":213.654},"last":3.7469999999999999},{"interface":"front","average":{"1min":4.8879999999999999,"5min":4.7220000000000004,"15min":4.7939999999999996},"min":{"1min":0.30299999999999999,"5min":0.27600000000000002,"15min":0.27400000000000002},"max":{"1min":53.148000000000003,"5min":113.38200000000001,"15min":213.77199999999999},"last":3.6429999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:13:44 2023","interfaces":[{"interface":"back","average":{"1min":9.1400000000000006,"5min":3.1629999999999998,"15min":2.133},"min":{"1min":0.40400000000000003,"5min":0.311,"15min":0.30399999999999999},"max":{"1min":93.747,"5min":93.747,"15min":93.747},"last":0.32900000000000001},{"interface":"front","average":{"1min":9.8000000000000007,"5min":3.8359999999999999,"15min":2.6000000000000001},"min":{"1min":0.32800000000000001,"5min":0.308,"15min":0.308},"max":{"1min":97.852000000000004,"5min":97.852000000000004,"15min":97.852000000000004},"last":3.399}]},{"osd":5,"last update":"Mon Oct 9 06:13:57 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.6220000000000001,"15min":1.5680000000000001},"min":{"1min":0.32200000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.4850000000000003,"5min":9.3109999999999999,"15min":36.356999999999999},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.383,"5min":2.5870000000000002,"15min":2.27},"min":{"1min":0.34599999999999997,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.2619999999999996,"5min":17.603999999999999,"15min":36.268000000000001},"last":3.4969999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:13:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":1.7410000000000001,"15min":1.7789999999999999},"min":{"1min":0.38500000000000001,"5min":0.33700000000000002,"15min":0.318},"max":{"1min":5.5170000000000003,"5min":9.282,"15min":52.207000000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.901,"5min":2.4180000000000001,"15min":2.3559999999999999},"min":{"1min":0.38500000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":8.7159999999999993,"5min":15.384,"15min":51.212000000000003},"last":3.4580000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:13:57 2023","interfaces":[{"interface":"back","average":{"1min":5.7400000000000002,"5min":3.3849999999999998,"15min":2.9329999999999998},"min":{"1min":0.39800000000000002,"5min":0.374,"15min":0.374},"max":{"1min":56.881,"5min":56.881,"15min":108.553},"last":3.5310000000000001},{"interface":"front","average":{"1min":5.7210000000000001,"5min":2.6360000000000001,"15min":2.3980000000000001},"min":{"1min":0.46000000000000002,"5min":0.38,"15min":0.38},"max":{"1min":56.941000000000003,"5min":56.941000000000003,"15min":108.56999999999999},"last":3.7130000000000001}]}]},{"osd":0,"up_from":456,"seq":1958505087032,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3524472,"kb_used_data":3257968,"kb_used_omap":32,"kb_used_meta":266143,"kb_avail":90220680,"statfs":{"total":95995035648,"available":92385976320,"internally_reserved":0,"allocated":3336159232,"data_stored":3333301288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32794,"internal_metadata":272531430},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":5.7939999999999996,"15min":9.3539999999999992},"min":{"1min":0.23999999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":5.157,"5min":166.833,"15min":166.833},"last":3.4630000000000001},{"interface":"front","average":{"1min":3.29,"5min":6.1779999999999999,"15min":8.7200000000000006},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":39.424999999999997,"5min":166.81100000000001,"15min":166.81100000000001},"last":5.0869999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":3.0449999999999999,"5min":2.9129999999999998,"15min":3.677},"min":{"1min":0.29999999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":21.648,"5min":21.648,"15min":21.648},"last":0.60799999999999998},{"interface":"front","average":{"1min":2.1800000000000002,"5min":2.6600000000000001,"15min":3.488},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":11.890000000000001,"5min":20.152000000000001,"15min":20.152000000000001},"last":4.1360000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:13:32 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":7.532,"15min":13.125999999999999},"min":{"1min":0.26500000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.9029999999999996,"5min":203.59299999999999,"15min":203.59299999999999},"last":5.9809999999999999},{"interface":"front","average":{"1min":3.4969999999999999,"5min":7.657,"15min":12.565},"min":{"1min":0.35299999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":18.192,"5min":202.96600000000001,"15min":202.96600000000001},"last":5.0279999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":2.444,"5min":2.0659999999999998,"15min":2.294},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":23.515999999999998,"5min":23.515999999999998,"15min":23.515999999999998},"last":4.0419999999999998},{"interface":"front","average":{"1min":4.2149999999999999,"5min":2.5169999999999999,"15min":2.1669999999999998},"min":{"1min":0.35799999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":39.302,"5min":39.302,"15min":39.302},"last":4.952}]},{"osd":5,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":3.1899999999999999,"5min":3.7200000000000002,"15min":2.7000000000000002},"min":{"1min":0.32200000000000001,"5min":0.316,"15min":0.316},"max":{"1min":39.369999999999997,"5min":161.286,"15min":161.286},"last":5.1340000000000003},{"interface":"front","average":{"1min":2.1110000000000002,"5min":3.532,"15min":2.8260000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":13.731999999999999,"5min":161.31800000000001,"15min":161.31800000000001},"last":0.58599999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":2.1190000000000002,"15min":2.3570000000000002},"min":{"1min":0.38900000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":13.795,"5min":13.795,"15min":13.795},"last":0.55500000000000005},{"interface":"front","average":{"1min":1.3759999999999999,"5min":1.931,"15min":2.5350000000000001},"min":{"1min":0.44700000000000001,"5min":0.372,"15min":0.372},"max":{"1min":5.274,"5min":10.233000000000001,"15min":10.233000000000001},"last":4.101}]},{"osd":7,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":2.423,"5min":3.5070000000000001,"15min":2.8570000000000002},"min":{"1min":0.44900000000000001,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":13.821,"5min":123.685,"15min":123.685},"last":2.9929999999999999},{"interface":"front","average":{"1min":1.5049999999999999,"5min":3.214,"15min":2.9129999999999998},"min":{"1min":0.44,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":5.3200000000000003,"5min":122.482,"15min":122.482},"last":4.0709999999999997}]}]},{"osd":2,"up_from":94,"seq":403726926128,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2881124,"kb_used_data":2120156,"kb_used_omap":81,"kb_used_meta":760878,"kb_avail":90864028,"statfs":{"total":95995035648,"available":93044764672,"internally_reserved":0,"allocated":2171039744,"data_stored":2168314731,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":779139290},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":7.7000000000000002,"5min":3.5070000000000001,"15min":3.2679999999999998},"min":{"1min":0.221,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":56.213000000000001,"5min":56.213000000000001,"15min":56.213000000000001},"last":6.2190000000000003},{"interface":"front","average":{"1min":6.4610000000000003,"5min":2.8140000000000001,"15min":2.4329999999999998},"min":{"1min":0.221,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":50.764000000000003,"5min":50.764000000000003,"15min":50.764000000000003},"last":8.7170000000000005}]},{"osd":1,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":5.0819999999999999,"15min":4.4909999999999997},"min":{"1min":0.27800000000000002,"5min":0.248,"15min":0.19400000000000001},"max":{"1min":4.258,"5min":160.45500000000001,"15min":222.59399999999999},"last":0.752},{"interface":"front","average":{"1min":2.1379999999999999,"5min":6.0540000000000003,"15min":4.9480000000000004},"min":{"1min":0.27600000000000002,"5min":0.27200000000000002,"15min":0.20200000000000001},"max":{"1min":10.345000000000001,"5min":163.697,"15min":224.619},"last":3.766}]},{"osd":3,"last update":"Mon Oct 9 06:12:57 2023","interfaces":[{"interface":"back","average":{"1min":4.274,"5min":5.1360000000000001,"15min":4.3710000000000004},"min":{"1min":0.317,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":29.399999999999999,"5min":115.742,"15min":144.88200000000001},"last":0.997},{"interface":"front","average":{"1min":3.9990000000000001,"5min":5.1310000000000002,"15min":4.3259999999999996},"min":{"1min":0.46100000000000002,"5min":0.252,"15min":0.252},"max":{"1min":29.648,"5min":115.55,"15min":140.79499999999999},"last":6.3049999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":3.1659999999999999,"15min":2.226},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.28299999999999997},"max":{"1min":8.9550000000000001,"5min":107.51300000000001,"15min":107.51300000000001},"last":2.9079999999999999},{"interface":"front","average":{"1min":1.081,"5min":2.6040000000000001,"15min":2.0579999999999998},"min":{"1min":0.36599999999999999,"5min":0.34100000000000003,"15min":0.31},"max":{"1min":3.4039999999999999,"5min":108.768,"15min":108.768},"last":0.78600000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":1.73,"5min":1.502,"15min":1.5549999999999999},"min":{"1min":0.376,"5min":0.34399999999999997,"15min":0.32600000000000001},"max":{"1min":11.015000000000001,"5min":11.015000000000001,"15min":28.48},"last":0.84099999999999997},{"interface":"front","average":{"1min":1.2430000000000001,"5min":1.7150000000000001,"15min":1.728},"min":{"1min":0.379,"5min":0.316,"15min":0.28899999999999998},"max":{"1min":6.9180000000000001,"5min":9.8699999999999992,"15min":30.318999999999999},"last":3.4510000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":1.603,"5min":2.218,"15min":2.085},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.314},"max":{"1min":10.301,"5min":14.962,"15min":30.420000000000002},"last":2.9609999999999999},{"interface":"front","average":{"1min":1.8540000000000001,"5min":1.621,"15min":1.7749999999999999},"min":{"1min":0.432,"5min":0.36099999999999999,"15min":0.32200000000000001},"max":{"1min":11.103,"5min":11.103,"15min":30.713000000000001},"last":0.81899999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":1.8300000000000001,"5min":2.9089999999999998,"15min":2.2349999999999999},"min":{"1min":0.40799999999999997,"5min":0.313,"15min":0.313},"max":{"1min":11.065,"5min":124.069,"15min":124.069},"last":0.86099999999999999},{"interface":"front","average":{"1min":1.0449999999999999,"5min":2.8119999999999998,"15min":2.258},"min":{"1min":0.376,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.4210000000000003,"5min":124.08799999999999,"15min":124.08799999999999},"last":3.4769999999999999}]}]},{"osd":3,"up_from":99,"seq":425201762602,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3346300,"kb_used_data":2394484,"kb_used_omap":7,"kb_used_meta":951800,"kb_avail":90398852,"statfs":{"total":95995035648,"available":92568424448,"internally_reserved":0,"allocated":2451951616,"data_stored":2448904507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":974643816},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:36 2023","interfaces":[{"interface":"back","average":{"1min":3.5409999999999999,"5min":1.9590000000000001,"15min":2.0590000000000002},"min":{"1min":0.222,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":30.841000000000001,"5min":30.841000000000001,"15min":30.841000000000001},"last":220.45400000000001},{"interface":"front","average":{"1min":3.4769999999999999,"5min":2.8650000000000002,"15min":3.3050000000000002},"min":{"1min":0.27900000000000003,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":28.652000000000001,"5min":28.652000000000001,"15min":28.652000000000001},"last":216.44200000000001}]},{"osd":1,"last update":"Mon Oct 9 06:13:08 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":1.853,"15min":2.7530000000000001},"min":{"1min":0.24299999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":5.5990000000000002,"5min":22.579999999999998,"15min":207.75299999999999},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.794,"5min":3.4279999999999999,"15min":3.7429999999999999},"min":{"1min":0.27900000000000003,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.29,"5min":30.524000000000001,"15min":207.12200000000001},"last":3.1829999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:13:08 2023","interfaces":[{"interface":"back","average":{"1min":1.6779999999999999,"5min":2.3100000000000001,"15min":2.7919999999999998},"min":{"1min":0.28799999999999998,"5min":0.23200000000000001,"15min":0.223},"max":{"1min":5.9850000000000003,"5min":30.547999999999998,"15min":107.01300000000001},"last":0.373},{"interface":"front","average":{"1min":1.962,"5min":3.0990000000000002,"15min":3.1880000000000002},"min":{"1min":0.25600000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":12.864000000000001,"5min":25.984000000000002,"15min":109.807},"last":3.222}]},{"osd":4,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":6.8099999999999996,"5min":2.8759999999999999,"15min":3.5569999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.254},"max":{"1min":95.739000000000004,"5min":95.739000000000004,"15min":99.876000000000005},"last":0.40799999999999997},{"interface":"front","average":{"1min":6.8259999999999996,"5min":3.399,"15min":3.7210000000000001},"min":{"1min":0.373,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":97.941999999999993,"5min":97.941999999999993,"15min":99.914000000000001},"last":3.3029999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:13:08 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":2.827,"15min":2.105},"min":{"1min":0.441,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.4480000000000004,"5min":74.706000000000003,"15min":74.706000000000003},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.169,"5min":3.4740000000000002,"15min":3.286},"min":{"1min":0.375,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.915,"5min":74.685000000000002,"15min":235.56},"last":0.56799999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:13:08 2023","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":1.3560000000000001,"15min":4.2690000000000001},"min":{"1min":0.45500000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.484,"5min":8.6809999999999992,"15min":233.26599999999999},"last":2.8559999999999999},{"interface":"front","average":{"1min":1.165,"5min":2.0659999999999998,"15min":4.7089999999999996},"min":{"1min":0.432,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.8819999999999997,"5min":26.890000000000001,"15min":233.24100000000001},"last":2.8380000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:13:08 2023","interfaces":[{"interface":"back","average":{"1min":1.339,"5min":3.4870000000000001,"15min":4.7990000000000004},"min":{"1min":0.40699999999999997,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":5.077,"5min":81.811000000000007,"15min":230.69399999999999},"last":4.3330000000000002},{"interface":"front","average":{"1min":1.4590000000000001,"5min":3.915,"15min":4.9359999999999999},"min":{"1min":0.44500000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":4.673,"5min":75.909999999999997,"15min":230.67500000000001},"last":3.2679999999999998}]}]},{"osd":4,"up_from":322,"seq":1382979469465,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3920700,"kb_used_data":3594036,"kb_used_omap":35,"kb_used_meta":326428,"kb_avail":89824452,"statfs":{"total":95995035648,"available":91980238848,"internally_reserved":0,"allocated":3680292864,"data_stored":3678774589,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36552,"internal_metadata":334262584},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:31 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.3500000000000001,"15min":1.4419999999999999},"min":{"1min":0.222,"5min":0.216,"15min":0.216},"max":{"1min":4.6840000000000002,"5min":6.8470000000000004,"15min":6.8470000000000004},"last":1.552},{"interface":"front","average":{"1min":1.504,"5min":1.4159999999999999,"15min":1.536},"min":{"1min":0.26000000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":8.7699999999999996,"5min":8.7699999999999996,"15min":8.7699999999999996},"last":0.307}]},{"osd":1,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":1.778,"5min":1.883,"15min":2.2839999999999998},"min":{"1min":0.28499999999999998,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.3449999999999998,"5min":18.884,"15min":78.891000000000005},"last":1.728},{"interface":"front","average":{"1min":1.4390000000000001,"5min":1.8300000000000001,"15min":2.2309999999999999},"min":{"1min":0.25600000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.946999999999999,"5min":18.617999999999999,"15min":78.921000000000006},"last":0.46000000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":14.574999999999999,"5min":4.4219999999999997,"15min":2.613},"min":{"1min":0.32500000000000001,"5min":0.29199999999999998,"15min":0.27200000000000002},"max":{"1min":245.166,"5min":245.166,"15min":245.166},"last":1.4079999999999999},{"interface":"front","average":{"1min":14.550000000000001,"5min":4.2469999999999999,"15min":2.5600000000000001},"min":{"1min":0.38500000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":245.119,"5min":245.119,"15min":245.119},"last":2.335}]},{"osd":3,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":24.687999999999999,"5min":6.6550000000000002,"15min":3.875},"min":{"1min":0.438,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":246.39400000000001,"5min":246.39400000000001,"15min":246.39400000000001},"last":6.4320000000000004},{"interface":"front","average":{"1min":23.437999999999999,"5min":6.21,"15min":3.444},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":246.99000000000001,"5min":246.99000000000001,"15min":246.99000000000001},"last":0.40899999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.93799999999999994,"15min":0.98999999999999999},"min":{"1min":0.27900000000000003,"5min":0.216,"15min":0.216},"max":{"1min":2.5600000000000001,"5min":7.5999999999999996,"15min":9.2690000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.1719999999999999,"5min":0.94099999999999995,"15min":1.165},"min":{"1min":0.33700000000000002,"5min":0.245,"15min":0.245},"max":{"1min":5.3479999999999999,"5min":9.1110000000000007,"15min":12.087},"last":1.6910000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":1.036,"15min":2.3010000000000002},"min":{"1min":0.32100000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.4279999999999999,"5min":7.641,"15min":95.492999999999995},"last":0.373},{"interface":"front","average":{"1min":1.1759999999999999,"5min":1.1990000000000001,"15min":2.4390000000000001},"min":{"1min":0.372,"5min":0.312,"15min":0.27900000000000003},"max":{"1min":5.5469999999999997,"5min":9.3239999999999998,"15min":95.664000000000001},"last":0.59499999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":1.3069999999999999,"15min":2.6920000000000002},"min":{"1min":0.38,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":8.1229999999999993,"5min":17.286000000000001,"15min":79.173000000000002},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.151,"15min":2.294},"min":{"1min":0.33000000000000002,"5min":0.251,"15min":0.251},"max":{"1min":5.1779999999999999,"5min":16.097999999999999,"15min":78.894000000000005},"last":0.434}]}]},{"osd":5,"up_from":38,"seq":163208757685,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2598636,"kb_used_data":1700132,"kb_used_omap":0,"kb_used_meta":898496,"kb_avail":91146516,"statfs":{"total":95995035648,"available":93334032384,"internally_reserved":0,"allocated":1740935168,"data_stored":1739429541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920059904},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:33 2023","interfaces":[{"interface":"back","average":{"1min":8.1929999999999996,"5min":3.508,"15min":2.77},"min":{"1min":0.24399999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":115.92400000000001,"5min":115.92400000000001,"15min":115.92400000000001},"last":18.347999999999999},{"interface":"front","average":{"1min":8.4220000000000006,"5min":3.855,"15min":3.4340000000000002},"min":{"1min":0.29299999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":115.904,"5min":115.904,"15min":115.904},"last":18.364999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:13:03 2023","interfaces":[{"interface":"back","average":{"1min":2.1899999999999999,"5min":5.4130000000000003,"15min":4.4619999999999997},"min":{"1min":0.314,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":18.812000000000001,"5min":151.78200000000001,"15min":151.78200000000001},"last":0.33200000000000002},{"interface":"front","average":{"1min":2.3090000000000002,"5min":5.6459999999999999,"15min":4.5119999999999996},"min":{"1min":0.309,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":19.376999999999999,"5min":152.251,"15min":152.251},"last":0.44900000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:13:27 2023","interfaces":[{"interface":"back","average":{"1min":9.2119999999999997,"5min":3.1019999999999999,"15min":2.9119999999999999},"min":{"1min":0.29799999999999999,"5min":0.26600000000000001,"15min":0.254},"max":{"1min":117.97199999999999,"5min":117.97199999999999,"15min":129.21199999999999},"last":0.373},{"interface":"front","average":{"1min":9.6690000000000005,"5min":3.4500000000000002,"15min":2.9870000000000001},"min":{"1min":0.30599999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":118.52,"5min":118.52,"15min":129.042},"last":0.38900000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:13:03 2023","interfaces":[{"interface":"back","average":{"1min":2.7530000000000001,"5min":5.407,"15min":6.9669999999999996},"min":{"1min":0.33900000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":39.435000000000002,"5min":143.31200000000001,"15min":211.40199999999999},"last":32.701000000000001},{"interface":"front","average":{"1min":3.109,"5min":5.9249999999999998,"15min":7.0350000000000001},"min":{"1min":0.38400000000000001,"5min":0.29599999999999999,"15min":0.29399999999999998},"max":{"1min":39.450000000000003,"5min":140.00899999999999,"15min":211.578},"last":32.729999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:13:52 2023","interfaces":[{"interface":"back","average":{"1min":4.5039999999999996,"5min":1.8360000000000001,"15min":1.298},"min":{"1min":0.26500000000000001,"5min":0.246,"15min":0.245},"max":{"1min":68.001999999999995,"5min":68.001999999999995,"15min":68.001999999999995},"last":0.29099999999999998},{"interface":"front","average":{"1min":4.2880000000000003,"5min":1.5529999999999999,"15min":1.228},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26000000000000001},"max":{"1min":61.743000000000002,"5min":61.743000000000002,"15min":61.743000000000002},"last":0.35999999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:13:03 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.96799999999999997,"15min":1.5},"min":{"1min":0.27500000000000002,"5min":0.26200000000000001,"15min":0.251},"max":{"1min":1.137,"5min":7.4980000000000002,"15min":126.89400000000001},"last":0.32000000000000001},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.0669999999999999,"15min":1.5569999999999999},"min":{"1min":0.33000000000000002,"5min":0.27300000000000002,"15min":0.26700000000000002},"max":{"1min":5.4279999999999999,"5min":7.7140000000000004,"15min":126.616},"last":0.53700000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:13:52 2023","interfaces":[{"interface":"back","average":{"1min":4.5670000000000002,"5min":3.2440000000000002,"15min":2.7999999999999998},"min":{"1min":0.40000000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":64.766000000000005,"5min":157.471,"15min":157.471},"last":0.41699999999999998},{"interface":"front","average":{"1min":4.7830000000000004,"5min":3.2519999999999998,"15min":2.7189999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.29599999999999999},"max":{"1min":67.956999999999994,"5min":157.51599999999999,"15min":157.51599999999999},"last":0.34799999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4889,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3308195840,"data_stored":3308191768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":900005888,"data_stored":900005888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2161508352,"data_stored":2161508352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2428112896,"data_stored":2428108824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3651735552,"data_stored":3651731480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1724841984,"data_stored":1724841984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1242107904,"data_stored":1242103832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":568983552,"data_stored":568983552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:14:02.942 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T06:14:02.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:02.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:14:03.045 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:14:03.156 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:03.334 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:14:03.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:03.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:14:03.444 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:03.486 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:03.553 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:03.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:03.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:03.641 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:14:03.665 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:14:03.705 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:03.782 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:14:03.786 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:03 smithi156 ceph-mon[195101]: osdmap e538: 8 total, 8 up, 8 in 2023-10-09T06:14:03.786 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:03 smithi156 ceph-mon[195101]: failed to encode map e538 with expected crc 2023-10-09T06:14:03.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:03.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:14:03.888 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:14:03.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:03 smithi064 ceph-mon[212750]: osdmap e538: 8 total, 8 up, 8 in 2023-10-09T06:14:03.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:03 smithi064 ceph-mon[212750]: failed to encode map e538 with expected crc 2023-10-09T06:14:03.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:03 smithi064 ceph-mon[220044]: osdmap e538: 8 total, 8 up, 8 in 2023-10-09T06:14:03.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:03 smithi064 ceph-mon[220044]: failed to encode map e538 with expected crc 2023-10-09T06:14:04.010 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:04.154 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:14:04.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:04.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:14:04.264 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:14:04.293 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:04.345 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:14:04.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:04.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:14:04.449 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:04.501 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:04.606 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:04.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:04.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:04.713 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:04 smithi064 ceph-mon[212750]: pgmap v1363: 119 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 1 active+remapped, 112 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 28743/221019 objects misplaced (13.005%); 46 MiB/s, 735 objects/s recovering 2023-10-09T06:14:04.713 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:04 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3647244924' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:14:04.727 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:14:04.736 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:14:04.761 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:04 smithi156 ceph-mon[195101]: pgmap v1363: 119 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 1 active+remapped, 112 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 28743/221019 objects misplaced (13.005%); 46 MiB/s, 735 objects/s recovering 2023-10-09T06:14:04.761 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:04 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3647244924' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:14:04.763 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:04.847 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:14:04.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:04.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:14:04.957 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:14:04.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:04 smithi064 ceph-mon[220044]: pgmap v1363: 119 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 1 active+remapped, 112 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 28743/221019 objects misplaced (13.005%); 46 MiB/s, 735 objects/s recovering 2023-10-09T06:14:04.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:04 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3647244924' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:14:05.081 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:05.303 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:14:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:14:05] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.43.0" 2023-10-09T06:14:05.837 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:14:05.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:05.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:14:05.942 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:14:05.956 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:05 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:14:05.957 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:05 smithi064 ceph-mon[212750]: failed to encode map e539 with expected crc 2023-10-09T06:14:05.957 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:05 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3647244924' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:14:05.957 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:05 smithi064 ceph-mon[212750]: osdmap e539: 8 total, 8 up, 8 in 2023-10-09T06:14:05.957 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:05 smithi064 ceph-mon[212750]: failed to encode map e539 with expected crc 2023-10-09T06:14:05.957 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:05 smithi064 ceph-mon[212750]: failed to encode map e539 with expected crc 2023-10-09T06:14:05.957 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:05 smithi064 ceph-mon[212750]: failed to encode map e539 with expected crc 2023-10-09T06:14:05.957 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:05 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:14:05.957 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:05 smithi064 ceph-mon[220044]: failed to encode map e539 with expected crc 2023-10-09T06:14:05.958 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:05 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3647244924' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:14:05.958 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:05 smithi064 ceph-mon[220044]: osdmap e539: 8 total, 8 up, 8 in 2023-10-09T06:14:05.958 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:05 smithi064 ceph-mon[220044]: failed to encode map e539 with expected crc 2023-10-09T06:14:05.958 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:05 smithi064 ceph-mon[220044]: failed to encode map e539 with expected crc 2023-10-09T06:14:05.958 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:05 smithi064 ceph-mon[220044]: failed to encode map e539 with expected crc 2023-10-09T06:14:06.031 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:14:06.085 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:06.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:05 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:14:06.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:05 smithi156 ceph-mon[195101]: failed to encode map e539 with expected crc 2023-10-09T06:14:06.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:05 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3647244924' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:14:06.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:05 smithi156 ceph-mon[195101]: osdmap e539: 8 total, 8 up, 8 in 2023-10-09T06:14:06.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:05 smithi156 ceph-mon[195101]: failed to encode map e539 with expected crc 2023-10-09T06:14:06.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:05 smithi156 ceph-mon[195101]: failed to encode map e539 with expected crc 2023-10-09T06:14:06.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:05 smithi156 ceph-mon[195101]: failed to encode map e539 with expected crc 2023-10-09T06:14:06.355 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:14:06.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:06.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:14:06.467 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:14:06.543 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:06.930 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:06 smithi064 ceph-mon[212750]: pgmap v1365: 119 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 1 active+remapped, 112 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 503 op/s; 28743/221019 objects misplaced (13.005%); 29 MiB/s, 463 objects/s recovering 2023-10-09T06:14:06.930 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:06 smithi064 ceph-mon[220044]: pgmap v1365: 119 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 1 active+remapped, 112 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 503 op/s; 28743/221019 objects misplaced (13.005%); 29 MiB/s, 463 objects/s recovering 2023-10-09T06:14:07.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:06 smithi156 ceph-mon[195101]: pgmap v1365: 119 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 1 active+remapped, 112 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 503 op/s; 28743/221019 objects misplaced (13.005%); 29 MiB/s, 463 objects/s recovering 2023-10-09T06:14:07.238 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:14:07.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:07.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:14:07.347 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:14:07.415 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:07.545 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:14:07.545 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T06:14:07.545 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T06:14:07.545 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T06:14:07.545 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:14:07.679 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:14:07.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:07.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:14:07.786 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:14:07.861 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:07.994 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:14:08.884 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:14:08.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:08.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:14:08.990 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:14:09.029 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:08 smithi064 ceph-mon[212750]: pgmap v1366: 119 pgs: 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 113 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 25820/212010 objects misplaced (12.179%); 25 MiB/s, 402 objects/s recovering 2023-10-09T06:14:09.030 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:08 smithi064 ceph-mon[220044]: pgmap v1366: 119 pgs: 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 113 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 25820/212010 objects misplaced (12.179%); 25 MiB/s, 402 objects/s recovering 2023-10-09T06:14:09.037 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:09.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:08 smithi156 ceph-mon[195101]: pgmap v1366: 119 pgs: 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 113 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 25820/212010 objects misplaced (12.179%); 25 MiB/s, 402 objects/s recovering 2023-10-09T06:14:09.274 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:14:09.275 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:14:09.275 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:14:09.275 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:14:09.275 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:14:09.275 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:14:09.275 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:14:09.275 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:14:09.276 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:14:09.276 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:14:09.276 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:14:09.276 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:14:09.276 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:14:09.276 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:14:09.276 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:14:09.276 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:14:09.584 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:14:09.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:09.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:14:09.687 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:09.717 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:09.746 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:14:09.768 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:09.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:09.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:09.874 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:14:09.893 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:09 smithi064 ceph-mon[212750]: failed to encode map e540 with expected crc 2023-10-09T06:14:09.894 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:09 smithi064 ceph-mon[212750]: osdmap e540: 8 total, 8 up, 8 in 2023-10-09T06:14:09.894 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:09 smithi064 ceph-mon[212750]: failed to encode map e540 with expected crc 2023-10-09T06:14:09.894 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:09 smithi064 ceph-mon[212750]: failed to encode map e540 with expected crc 2023-10-09T06:14:09.894 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:09 smithi064 ceph-mon[212750]: failed to encode map e540 with expected crc 2023-10-09T06:14:09.894 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:09 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3169581295' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:14:09.894 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:09 smithi064 ceph-mon[220044]: failed to encode map e540 with expected crc 2023-10-09T06:14:09.894 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:09 smithi064 ceph-mon[220044]: osdmap e540: 8 total, 8 up, 8 in 2023-10-09T06:14:09.894 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:09 smithi064 ceph-mon[220044]: failed to encode map e540 with expected crc 2023-10-09T06:14:09.894 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:09 smithi064 ceph-mon[220044]: failed to encode map e540 with expected crc 2023-10-09T06:14:09.895 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:09 smithi064 ceph-mon[220044]: failed to encode map e540 with expected crc 2023-10-09T06:14:09.895 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:09 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3169581295' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:14:09.995 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:10.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:09 smithi156 ceph-mon[195101]: failed to encode map e540 with expected crc 2023-10-09T06:14:10.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:09 smithi156 ceph-mon[195101]: osdmap e540: 8 total, 8 up, 8 in 2023-10-09T06:14:10.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:09 smithi156 ceph-mon[195101]: failed to encode map e540 with expected crc 2023-10-09T06:14:10.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:09 smithi156 ceph-mon[195101]: failed to encode map e540 with expected crc 2023-10-09T06:14:10.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:09 smithi156 ceph-mon[195101]: failed to encode map e540 with expected crc 2023-10-09T06:14:10.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:09 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3169581295' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:14:10.457 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:14:10.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:10.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:14:10.565 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:14:10.609 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:10.663 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:14:10.704 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:14:10.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:10.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:14:10.814 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:14:10.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:10 smithi156 ceph-mon[195101]: pgmap v1368: 119 pgs: 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 113 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 25820/212010 objects misplaced (12.179%); 17 MiB/s, 270 objects/s recovering 2023-10-09T06:14:10.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:10 smithi156 ceph-mon[195101]: failed to encode map e541 with expected crc 2023-10-09T06:14:10.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:10 smithi156 ceph-mon[195101]: osdmap e541: 8 total, 8 up, 8 in 2023-10-09T06:14:10.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:10 smithi156 ceph-mon[195101]: failed to encode map e541 with expected crc 2023-10-09T06:14:10.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:10 smithi156 ceph-mon[195101]: failed to encode map e541 with expected crc 2023-10-09T06:14:10.892 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:11.201 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:14:11.210 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:10 smithi064 ceph-mon[220044]: pgmap v1368: 119 pgs: 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 113 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 25820/212010 objects misplaced (12.179%); 17 MiB/s, 270 objects/s recovering 2023-10-09T06:14:11.210 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:10 smithi064 ceph-mon[220044]: failed to encode map e541 with expected crc 2023-10-09T06:14:11.210 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:10 smithi064 ceph-mon[220044]: osdmap e541: 8 total, 8 up, 8 in 2023-10-09T06:14:11.210 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:10 smithi064 ceph-mon[220044]: failed to encode map e541 with expected crc 2023-10-09T06:14:11.210 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:10 smithi064 ceph-mon[220044]: failed to encode map e541 with expected crc 2023-10-09T06:14:11.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:11.215 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:10 smithi064 ceph-mon[212750]: pgmap v1368: 119 pgs: 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 113 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 25820/212010 objects misplaced (12.179%); 17 MiB/s, 270 objects/s recovering 2023-10-09T06:14:11.215 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:10 smithi064 ceph-mon[212750]: failed to encode map e541 with expected crc 2023-10-09T06:14:11.215 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:10 smithi064 ceph-mon[212750]: osdmap e541: 8 total, 8 up, 8 in 2023-10-09T06:14:11.215 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:10 smithi064 ceph-mon[212750]: failed to encode map e541 with expected crc 2023-10-09T06:14:11.215 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:10 smithi064 ceph-mon[212750]: failed to encode map e541 with expected crc 2023-10-09T06:14:11.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:14:11.272 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:14:11.317 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:14:11.365 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:11.438 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:14:11.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:11.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:14:11.547 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:14:11.672 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:11.897 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:14:11.897 INFO:teuthology.orchestra.run.smithi064.stdout:{"fsid":"44c77acc-6665-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1713,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":541,"num_osds":8,"num_up_osds":8,"osd_up_since":1696831766,"num_in_osds":8,"osd_in_since":1696831662,"num_remapped_pgs":5},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":113},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+remapped+backfilling","count":2},{"state_name":"peering","count":1}],"num_pgs":119,"num_pools":5,"num_objects":68973,"data_bytes":4519198859,"bytes_used":20880019456,"bytes_avail":747080265728,"bytes_total":767960285184,"inactive_pgs_ratio":0.0084033617749810219,"misplaced_objects":23809,"misplaced_total":206919,"misplaced_ratio":0.11506434885148295,"recovering_objects_per_sec":491,"recovering_bytes_per_sec":32189805,"recovering_keys_per_sec":0,"num_objects_recovered":3175,"num_bytes_recovered":208076800,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":727},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi064.gmrwsy","status":"up:active","gid":24551}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"}},"servicemap":{"epoch":26,"modified":"2023-10-09T06:10:27.041106+0000","services":{}},"progress_events":{"28a7925c-af7b-4940-a0f9-d3ed10a1f964":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"ca7f429f-bb9e-4fb1-944c-f3f1e3d51bf1":{"message":"Global Recovery Event (75s)\n [==========================..] (remaining: 3s)","progress":0.94957983493804932,"add_to_ceph_s":true}}} 2023-10-09T06:14:11.994 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:14:12.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:12.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:14:12.104 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:14:12.195 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:12.555 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:14:12.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:12 smithi064 ceph-mon[212750]: pgmap v1370: 119 pgs: 1 peering, 2 active+remapped+backfilling, 3 active+remapped+backfill_wait, 113 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 23809/206919 objects misplaced (11.506%); 31 MiB/s, 491 objects/s recovering 2023-10-09T06:14:12.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:12 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/117087433' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:14:12.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:12 smithi064 ceph-mon[220044]: pgmap v1370: 119 pgs: 1 peering, 2 active+remapped+backfilling, 3 active+remapped+backfill_wait, 113 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 23809/206919 objects misplaced (11.506%); 31 MiB/s, 491 objects/s recovering 2023-10-09T06:14:12.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:12 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/117087433' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:14:13.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:12 smithi156 ceph-mon[195101]: pgmap v1370: 119 pgs: 1 peering, 2 active+remapped+backfilling, 3 active+remapped+backfill_wait, 113 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 23809/206919 objects misplaced (11.506%); 31 MiB/s, 491 objects/s recovering 2023-10-09T06:14:13.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:12 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/117087433' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:14:13.143 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:14:13.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:13.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:14:13.254 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:13.286 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:13.343 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:13.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:13.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:13.453 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:14:13.526 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:13.929 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T06:14:13.929 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:14:14.022 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:14:14.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:14.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:14:14.136 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:14:14.204 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:14.415 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:14:14.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:14.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:14:14.519 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:14.572 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:14.660 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:14.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:14.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:14.767 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:14:14.813 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:14 smithi156 ceph-mon[195101]: pgmap v1371: 119 pgs: 1 peering, 2 active+remapped+backfilling, 3 active+remapped+backfill_wait, 113 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 22428/202677 objects misplaced (11.066%); 32 MiB/s, 509 objects/s recovering 2023-10-09T06:14:14.813 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:14:14.813 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:14.824 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:14.921 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:14 smithi064 ceph-mon[220044]: pgmap v1371: 119 pgs: 1 peering, 2 active+remapped+backfilling, 3 active+remapped+backfill_wait, 113 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 22428/202677 objects misplaced (11.066%); 32 MiB/s, 509 objects/s recovering 2023-10-09T06:14:14.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:14:14.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:14.922 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:14 smithi064 ceph-mon[212750]: pgmap v1371: 119 pgs: 1 peering, 2 active+remapped+backfilling, 3 active+remapped+backfill_wait, 113 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 22428/202677 objects misplaced (11.066%); 32 MiB/s, 509 objects/s recovering 2023-10-09T06:14:14.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:14:14.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:14.923 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:14:14.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:14.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:14:15.030 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:14:15.072 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:15.083 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:14:15.143 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:14:15.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:15.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:14:15.154 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:14:15.251 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:14:15.296 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:15.390 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:14:15.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:15.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:14:15.505 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:14:15.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:15 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2278033814' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:14:15.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:15 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:14:15.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:15 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2278033814' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:14:15.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:15 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:14:15.569 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:14:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:14:15] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.43.0" 2023-10-09T06:14:15.584 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:14:15.639 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:15 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2278033814' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:14:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:15 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:14:15.976 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:14:15.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:15.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:14:16.090 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:16.133 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:16.222 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:16.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:16.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:16.334 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:14:16.383 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:16.518 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:14:16.518 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:14:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:16 smithi064 ceph-mon[212750]: pgmap v1372: 119 pgs: 2 active+remapped+backfilling, 3 active+remapped+backfill_wait, 114 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 416 op/s; 22428/202014 objects misplaced (11.102%); 18 MiB/s, 287 objects/s recovering 2023-10-09T06:14:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:16 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:14:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:16 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:14:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:16 smithi064 ceph-mon[212750]: osdmap e542: 8 total, 8 up, 8 in 2023-10-09T06:14:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:16 smithi064 ceph-mon[212750]: failed to encode map e542 with expected crc 2023-10-09T06:14:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:16 smithi064 ceph-mon[212750]: failed to encode map e542 with expected crc 2023-10-09T06:14:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:16 smithi064 ceph-mon[212750]: failed to encode map e542 with expected crc 2023-10-09T06:14:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:16 smithi064 ceph-mon[212750]: failed to encode map e542 with expected crc 2023-10-09T06:14:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:16 smithi064 ceph-mon[220044]: pgmap v1372: 119 pgs: 2 active+remapped+backfilling, 3 active+remapped+backfill_wait, 114 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 416 op/s; 22428/202014 objects misplaced (11.102%); 18 MiB/s, 287 objects/s recovering 2023-10-09T06:14:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:16 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:14:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:16 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:14:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:16 smithi064 ceph-mon[220044]: osdmap e542: 8 total, 8 up, 8 in 2023-10-09T06:14:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:16 smithi064 ceph-mon[220044]: failed to encode map e542 with expected crc 2023-10-09T06:14:16.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:16 smithi064 ceph-mon[220044]: failed to encode map e542 with expected crc 2023-10-09T06:14:16.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:16 smithi064 ceph-mon[220044]: failed to encode map e542 with expected crc 2023-10-09T06:14:16.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:16 smithi064 ceph-mon[220044]: failed to encode map e542 with expected crc 2023-10-09T06:14:16.772 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:14:16.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:16.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:14:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:16 smithi156 ceph-mon[195101]: pgmap v1372: 119 pgs: 2 active+remapped+backfilling, 3 active+remapped+backfill_wait, 114 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 416 op/s; 22428/202014 objects misplaced (11.102%); 18 MiB/s, 287 objects/s recovering 2023-10-09T06:14:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:16 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:14:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:16 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:14:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:16 smithi156 ceph-mon[195101]: osdmap e542: 8 total, 8 up, 8 in 2023-10-09T06:14:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:16 smithi156 ceph-mon[195101]: failed to encode map e542 with expected crc 2023-10-09T06:14:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:16 smithi156 ceph-mon[195101]: failed to encode map e542 with expected crc 2023-10-09T06:14:16.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:16 smithi156 ceph-mon[195101]: failed to encode map e542 with expected crc 2023-10-09T06:14:16.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:16 smithi156 ceph-mon[195101]: failed to encode map e542 with expected crc 2023-10-09T06:14:16.883 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:14:16.955 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:17.062 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:14:17.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:17.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:14:17.171 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:17.203 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:17.262 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:17.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:17.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:17.368 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:14:17.432 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:17.977 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:14:17.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:17.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:14:18.083 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:14:18.163 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:18.377 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:14:18.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:18.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:14:18.483 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:14:18.543 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:18.645 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:14:18.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:18.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:14:18.754 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:14:18.808 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:18.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:18 smithi156 ceph-mon[195101]: pgmap v1374: 119 pgs: 2 active+remapped+backfilling, 3 active+remapped+backfill_wait, 114 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 18220/192492 objects misplaced (9.465%); 34 MiB/s, 540 objects/s recovering 2023-10-09T06:14:18.976 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:18 smithi064 ceph-mon[212750]: pgmap v1374: 119 pgs: 2 active+remapped+backfilling, 3 active+remapped+backfill_wait, 114 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 18220/192492 objects misplaced (9.465%); 34 MiB/s, 540 objects/s recovering 2023-10-09T06:14:18.976 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:18 smithi064 ceph-mon[220044]: pgmap v1374: 119 pgs: 2 active+remapped+backfilling, 3 active+remapped+backfill_wait, 114 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 18220/192492 objects misplaced (9.465%); 34 MiB/s, 540 objects/s recovering 2023-10-09T06:14:19.161 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:14:19.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:19.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:14:19.274 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:19.311 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:14:19.329 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:19.411 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:19.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:19.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:19.522 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:14:19.637 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:19.962 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:14:19.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:19.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:14:20.069 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:14:20.074 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:14:20.117 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:20.158 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:14:20.174 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:14:20.179 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:14:20.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:20.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:14:20.196 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:14:20.236 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:14:20.285 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:14:20.300 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:14:20.335 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:14:20.382 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:14:20.388 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:20.440 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:14:20.515 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:14:20.533 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:14:20.541 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:14:20.541 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:14:20.541 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:14:20.541 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:14:20.541 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:14:20.541 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:14:20.541 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:14:20.541 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:14:20.557 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T06:14:20.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:20 smithi064 ceph-mon[212750]: pgmap v1375: 119 pgs: 2 active+remapped+backfilling, 3 active+remapped+backfill_wait, 114 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 18220/192492 objects misplaced (9.465%); 27 MiB/s, 426 objects/s recovering 2023-10-09T06:14:20.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:20 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3674094647' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:14:20.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:20 smithi064 ceph-mon[220044]: pgmap v1375: 119 pgs: 2 active+remapped+backfilling, 3 active+remapped+backfill_wait, 114 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 18220/192492 objects misplaced (9.465%); 27 MiB/s, 426 objects/s recovering 2023-10-09T06:14:20.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:20 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3674094647' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:14:21.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:20 smithi156 ceph-mon[195101]: pgmap v1375: 119 pgs: 2 active+remapped+backfilling, 3 active+remapped+backfill_wait, 114 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 18220/192492 objects misplaced (9.465%); 27 MiB/s, 426 objects/s recovering 2023-10-09T06:14:21.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:20 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3674094647' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:14:21.355 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:14:21.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:21.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:14:21.460 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:14:21.494 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:22.124 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:21 smithi156 ceph-mon[195101]: from='client.46457 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:14:22.124 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:21 smithi156 ceph-mon[195101]: failed to encode map e543 with expected crc 2023-10-09T06:14:22.124 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:21 smithi156 ceph-mon[195101]: failed to encode map e543 with expected crc 2023-10-09T06:14:22.124 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:21 smithi156 ceph-mon[195101]: osdmap e543: 8 total, 8 up, 8 in 2023-10-09T06:14:22.125 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:21 smithi156 ceph-mon[195101]: failed to encode map e543 with expected crc 2023-10-09T06:14:22.203 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:21 smithi064 ceph-mon[220044]: from='client.46457 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:14:22.203 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:21 smithi064 ceph-mon[220044]: failed to encode map e543 with expected crc 2023-10-09T06:14:22.204 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:21 smithi064 ceph-mon[220044]: failed to encode map e543 with expected crc 2023-10-09T06:14:22.204 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:21 smithi064 ceph-mon[220044]: osdmap e543: 8 total, 8 up, 8 in 2023-10-09T06:14:22.204 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:21 smithi064 ceph-mon[220044]: failed to encode map e543 with expected crc 2023-10-09T06:14:22.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:21 smithi064 ceph-mon[212750]: from='client.46457 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:14:22.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:21 smithi064 ceph-mon[212750]: failed to encode map e543 with expected crc 2023-10-09T06:14:22.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:21 smithi064 ceph-mon[212750]: failed to encode map e543 with expected crc 2023-10-09T06:14:22.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:21 smithi064 ceph-mon[212750]: osdmap e543: 8 total, 8 up, 8 in 2023-10-09T06:14:22.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:21 smithi064 ceph-mon[212750]: failed to encode map e543 with expected crc 2023-10-09T06:14:22.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:22 smithi064 ceph-mon[212750]: pgmap v1377: 119 pgs: 2 active+remapped+backfilling, 3 active+remapped+backfill_wait, 114 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 17816/187194 objects misplaced (9.517%); 16 MiB/s, 252 objects/s recovering 2023-10-09T06:14:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:22 smithi064 ceph-mon[212750]: failed to encode map e544 with expected crc 2023-10-09T06:14:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:22 smithi064 ceph-mon[212750]: failed to encode map e544 with expected crc 2023-10-09T06:14:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:22 smithi064 ceph-mon[212750]: osdmap e544: 8 total, 8 up, 8 in 2023-10-09T06:14:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:22 smithi064 ceph-mon[212750]: failed to encode map e544 with expected crc 2023-10-09T06:14:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:22 smithi064 ceph-mon[212750]: failed to encode map e544 with expected crc 2023-10-09T06:14:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:22 smithi064 ceph-mon[212750]: failed to encode map e544 with expected crc 2023-10-09T06:14:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:22 smithi064 ceph-mon[212750]: failed to encode map e544 with expected crc 2023-10-09T06:14:22.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:22 smithi064 ceph-mon[220044]: pgmap v1377: 119 pgs: 2 active+remapped+backfilling, 3 active+remapped+backfill_wait, 114 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 17816/187194 objects misplaced (9.517%); 16 MiB/s, 252 objects/s recovering 2023-10-09T06:14:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:22 smithi064 ceph-mon[220044]: failed to encode map e544 with expected crc 2023-10-09T06:14:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:22 smithi064 ceph-mon[220044]: failed to encode map e544 with expected crc 2023-10-09T06:14:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:22 smithi064 ceph-mon[220044]: osdmap e544: 8 total, 8 up, 8 in 2023-10-09T06:14:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:22 smithi064 ceph-mon[220044]: failed to encode map e544 with expected crc 2023-10-09T06:14:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:22 smithi064 ceph-mon[220044]: failed to encode map e544 with expected crc 2023-10-09T06:14:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:22 smithi064 ceph-mon[220044]: failed to encode map e544 with expected crc 2023-10-09T06:14:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:22 smithi064 ceph-mon[220044]: failed to encode map e544 with expected crc 2023-10-09T06:14:23.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:22 smithi156 ceph-mon[195101]: pgmap v1377: 119 pgs: 2 active+remapped+backfilling, 3 active+remapped+backfill_wait, 114 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 17816/187194 objects misplaced (9.517%); 16 MiB/s, 252 objects/s recovering 2023-10-09T06:14:23.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:22 smithi156 ceph-mon[195101]: failed to encode map e544 with expected crc 2023-10-09T06:14:23.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:22 smithi156 ceph-mon[195101]: failed to encode map e544 with expected crc 2023-10-09T06:14:23.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:22 smithi156 ceph-mon[195101]: osdmap e544: 8 total, 8 up, 8 in 2023-10-09T06:14:23.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:22 smithi156 ceph-mon[195101]: failed to encode map e544 with expected crc 2023-10-09T06:14:23.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:22 smithi156 ceph-mon[195101]: failed to encode map e544 with expected crc 2023-10-09T06:14:23.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:22 smithi156 ceph-mon[195101]: failed to encode map e544 with expected crc 2023-10-09T06:14:23.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:22 smithi156 ceph-mon[195101]: failed to encode map e544 with expected crc 2023-10-09T06:14:23.319 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:14:23.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:23.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:14:23.428 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:14:23.455 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:23.496 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:14:23.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:23.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:14:23.601 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:14:23.650 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:23.672 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1375,"stamp":"2023-10-09T06:14:19.170575+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4204036235,"num_objects":64164,"num_object_clones":0,"num_object_copies":192492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":18220,"num_objects_unfound":0,"num_objects_dirty":64164,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":128358,"num_write_kb":6157708,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31919,"num_bytes_recovered":2089250483,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":101779,"ondisk_log_size":101779,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":368,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19673004,"kb_used_data":14096172,"kb_used_omap":165,"kb_used_meta":5576346,"kb_avail":730288212,"statfs":{"total":767960285184,"available":747815129088,"internally_reserved":0,"allocated":14434480128,"data_stored":14416522553,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":169911,"internal_metadata":5710178377},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-417529856,"num_objects":-6371,"num_object_clones":0,"num_object_copies":-19113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-7275,"num_objects_unfound":0,"num_objects_dirty":-6371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6371,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4004,"num_bytes_recovered":262406144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.391373"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":780,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.789064+0000","last_change":"2023-10-09T06:13:33.699246+0000","last_active":"2023-10-09T06:14:08.789064+0000","last_peered":"2023-10-09T06:14:08.789064+0000","last_clean":"2023-10-09T06:14:08.789064+0000","last_became_active":"2023-10-09T06:13:33.698733+0000","last_became_peered":"2023-10-09T06:13:33.698733+0000","last_unstale":"2023-10-09T06:14:08.789064+0000","last_undegraded":"2023-10-09T06:14:08.789064+0000","last_fullsized":"2023-10-09T06:14:08.789064+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:05.412941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:05.412941+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:57.044673+0000","scrub_duration":0.0061014329999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":610,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.789033+0000","last_change":"2023-10-09T06:07:43.572773+0000","last_active":"2023-10-09T06:14:09.789033+0000","last_peered":"2023-10-09T06:14:09.789033+0000","last_clean":"2023-10-09T06:14:09.789033+0000","last_became_active":"2023-10-09T06:07:43.572299+0000","last_became_peered":"2023-10-09T06:07:43.572299+0000","last_unstale":"2023-10-09T06:14:09.789033+0000","last_undegraded":"2023-10-09T06:14:09.789033+0000","last_fullsized":"2023-10-09T06:14:09.789033+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:00:37.672435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":351,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.796263+0000","last_change":"2023-10-09T06:05:11.451972+0000","last_active":"2023-10-09T06:14:08.796263+0000","last_peered":"2023-10-09T06:14:08.796263+0000","last_clean":"2023-10-09T06:14:08.796263+0000","last_became_active":"2023-10-09T06:05:11.451448+0000","last_became_peered":"2023-10-09T06:05:11.451448+0000","last_unstale":"2023-10-09T06:14:08.796263+0000","last_undegraded":"2023-10-09T06:14:08.796263+0000","last_fullsized":"2023-10-09T06:14:08.796263+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:39.183621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":841,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.789413+0000","last_change":"2023-10-09T06:09:27.554162+0000","last_active":"2023-10-09T06:14:09.789413+0000","last_peered":"2023-10-09T06:14:09.789413+0000","last_clean":"2023-10-09T06:14:09.789413+0000","last_became_active":"2023-10-09T06:09:27.553659+0000","last_became_peered":"2023-10-09T06:09:27.553659+0000","last_unstale":"2023-10-09T06:14:09.789413+0000","last_undegraded":"2023-10-09T06:14:09.789413+0000","last_fullsized":"2023-10-09T06:14:09.789413+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:01.468346+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:01.468346+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:03.443511+0000","scrub_duration":0.0016258220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":224,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.809662+0000","last_change":"2023-10-09T06:01:54.762060+0000","last_active":"2023-10-09T06:14:09.809662+0000","last_peered":"2023-10-09T06:14:09.809662+0000","last_clean":"2023-10-09T06:14:09.809662+0000","last_became_active":"2023-10-09T06:01:54.761757+0000","last_became_peered":"2023-10-09T06:01:54.761757+0000","last_unstale":"2023-10-09T06:14:09.809662+0000","last_undegraded":"2023-10-09T06:14:09.809662+0000","last_fullsized":"2023-10-09T06:14:09.809662+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:40.294232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":24,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.786777+0000","last_change":"2023-10-09T06:13:30.789834+0000","last_active":"2023-10-09T06:14:08.786777+0000","last_peered":"2023-10-09T06:14:08.786777+0000","last_clean":"2023-10-09T06:14:08.786777+0000","last_became_active":"2023-10-09T06:13:30.789517+0000","last_became_peered":"2023-10-09T06:13:30.789517+0000","last_unstale":"2023-10-09T06:14:08.786777+0000","last_undegraded":"2023-10-09T06:14:08.786777+0000","last_fullsized":"2023-10-09T06:14:08.786777+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:45.467922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":826,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.788571+0000","last_change":"2023-10-09T06:12:07.739121+0000","last_active":"2023-10-09T06:14:09.788571+0000","last_peered":"2023-10-09T06:14:09.788571+0000","last_clean":"2023-10-09T06:14:09.788571+0000","last_became_active":"2023-10-09T06:12:07.738739+0000","last_became_peered":"2023-10-09T06:12:07.738739+0000","last_unstale":"2023-10-09T06:14:09.788571+0000","last_undegraded":"2023-10-09T06:14:09.788571+0000","last_fullsized":"2023-10-09T06:14:09.788571+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.293426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.293426+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:19.815422+0000","scrub_duration":0.001587946,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":333,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.795538+0000","last_change":"2023-10-09T06:09:27.564377+0000","last_active":"2023-10-09T06:14:08.795538+0000","last_peered":"2023-10-09T06:14:08.795538+0000","last_clean":"2023-10-09T06:14:08.795538+0000","last_became_active":"2023-10-09T06:09:27.563710+0000","last_became_peered":"2023-10-09T06:09:27.563710+0000","last_unstale":"2023-10-09T06:14:08.795538+0000","last_undegraded":"2023-10-09T06:14:08.795538+0000","last_fullsized":"2023-10-09T06:14:08.795538+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:48.168447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":458,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.812008+0000","last_change":"2023-10-09T06:13:21.727677+0000","last_active":"2023-10-09T06:14:09.812008+0000","last_peered":"2023-10-09T06:14:09.812008+0000","last_clean":"2023-10-09T06:14:09.812008+0000","last_became_active":"2023-10-09T06:13:21.727321+0000","last_became_peered":"2023-10-09T06:13:21.727321+0000","last_unstale":"2023-10-09T06:14:09.812008+0000","last_undegraded":"2023-10-09T06:14:09.812008+0000","last_fullsized":"2023-10-09T06:14:09.812008+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:10.146447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":759,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.787015+0000","last_change":"2023-10-09T06:13:33.701687+0000","last_active":"2023-10-09T06:14:08.787015+0000","last_peered":"2023-10-09T06:14:08.787015+0000","last_clean":"2023-10-09T06:14:08.787015+0000","last_became_active":"2023-10-09T06:13:33.701233+0000","last_became_peered":"2023-10-09T06:13:33.701233+0000","last_unstale":"2023-10-09T06:14:08.787015+0000","last_undegraded":"2023-10-09T06:14:08.787015+0000","last_fullsized":"2023-10-09T06:14:08.787015+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.384407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.384407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:04.887826+0000","scrub_duration":0.0040651960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":439,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.797120+0000","last_change":"2023-10-09T06:09:27.559756+0000","last_active":"2023-10-09T06:14:08.797120+0000","last_peered":"2023-10-09T06:14:08.797120+0000","last_clean":"2023-10-09T06:14:08.797120+0000","last_became_active":"2023-10-09T06:09:27.559443+0000","last_became_peered":"2023-10-09T06:09:27.559443+0000","last_unstale":"2023-10-09T06:14:08.797120+0000","last_undegraded":"2023-10-09T06:14:08.797120+0000","last_fullsized":"2023-10-09T06:14:08.797120+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:14.519398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":370,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.795694+0000","last_change":"2023-10-09T06:13:21.737897+0000","last_active":"2023-10-09T06:14:08.795694+0000","last_peered":"2023-10-09T06:14:08.795694+0000","last_clean":"2023-10-09T06:14:08.795694+0000","last_became_active":"2023-10-09T06:13:21.737564+0000","last_became_peered":"2023-10-09T06:13:21.737564+0000","last_unstale":"2023-10-09T06:14:08.795694+0000","last_undegraded":"2023-10-09T06:14:08.795694+0000","last_fullsized":"2023-10-09T06:14:08.795694+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:34.863320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":665,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.788207+0000","last_change":"2023-10-09T06:10:11.729656+0000","last_active":"2023-10-09T06:14:08.788207+0000","last_peered":"2023-10-09T06:14:08.788207+0000","last_clean":"2023-10-09T06:14:08.788207+0000","last_became_active":"2023-10-09T06:10:11.728960+0000","last_became_peered":"2023-10-09T06:10:11.728960+0000","last_unstale":"2023-10-09T06:14:08.788207+0000","last_undegraded":"2023-10-09T06:14:08.788207+0000","last_fullsized":"2023-10-09T06:14:08.788207+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:20.742561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:20.742561+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:45.543709+0000","scrub_duration":0.0027005039999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":24,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.786688+0000","last_change":"2023-10-09T06:13:30.802749+0000","last_active":"2023-10-09T06:14:08.786688+0000","last_peered":"2023-10-09T06:14:08.786688+0000","last_clean":"2023-10-09T06:14:08.786688+0000","last_became_active":"2023-10-09T06:13:30.802449+0000","last_became_peered":"2023-10-09T06:13:30.802449+0000","last_unstale":"2023-10-09T06:14:08.786688+0000","last_undegraded":"2023-10-09T06:14:08.786688+0000","last_fullsized":"2023-10-09T06:14:08.786688+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:29.017181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":527,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.795916+0000","last_change":"2023-10-09T06:13:30.792445+0000","last_active":"2023-10-09T06:14:08.795916+0000","last_peered":"2023-10-09T06:14:08.795916+0000","last_clean":"2023-10-09T06:14:08.795916+0000","last_became_active":"2023-10-09T06:13:30.792116+0000","last_became_peered":"2023-10-09T06:13:30.792116+0000","last_unstale":"2023-10-09T06:14:08.795916+0000","last_undegraded":"2023-10-09T06:14:08.795916+0000","last_fullsized":"2023-10-09T06:14:08.795916+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:20:53.506293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":752,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.784761+0000","last_change":"2023-10-09T06:13:24.760843+0000","last_active":"2023-10-09T06:14:08.784761+0000","last_peered":"2023-10-09T06:14:08.784761+0000","last_clean":"2023-10-09T06:14:08.784761+0000","last_became_active":"2023-10-09T06:13:24.760333+0000","last_became_peered":"2023-10-09T06:13:24.760333+0000","last_unstale":"2023-10-09T06:14:08.784761+0000","last_undegraded":"2023-10-09T06:14:08.784761+0000","last_fullsized":"2023-10-09T06:14:08.784761+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:38.588641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:38.588641+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:30.660400+0000","scrub_duration":0.010116385,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":30,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.786931+0000","last_change":"2023-10-09T06:13:30.826588+0000","last_active":"2023-10-09T06:14:08.786931+0000","last_peered":"2023-10-09T06:14:08.786931+0000","last_clean":"2023-10-09T06:14:08.786931+0000","last_became_active":"2023-10-09T06:13:30.799199+0000","last_became_peered":"2023-10-09T06:13:30.799199+0000","last_unstale":"2023-10-09T06:14:08.786931+0000","last_undegraded":"2023-10-09T06:14:08.786931+0000","last_fullsized":"2023-10-09T06:14:08.786931+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:38.703864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":322,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.784072+0000","last_change":"2023-10-09T06:13:21.739214+0000","last_active":"2023-10-09T06:14:08.784072+0000","last_peered":"2023-10-09T06:14:08.784072+0000","last_clean":"2023-10-09T06:14:08.784072+0000","last_became_active":"2023-10-09T06:13:21.738951+0000","last_became_peered":"2023-10-09T06:13:21.738951+0000","last_unstale":"2023-10-09T06:14:08.784072+0000","last_undegraded":"2023-10-09T06:14:08.784072+0000","last_fullsized":"2023-10-09T06:14:08.784072+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:17.477241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":714,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.812311+0000","last_change":"2023-10-09T06:09:27.574336+0000","last_active":"2023-10-09T06:14:09.812311+0000","last_peered":"2023-10-09T06:14:09.812311+0000","last_clean":"2023-10-09T06:14:09.812311+0000","last_became_active":"2023-10-09T06:09:27.573974+0000","last_became_peered":"2023-10-09T06:09:27.573974+0000","last_unstale":"2023-10-09T06:14:09.812311+0000","last_undegraded":"2023-10-09T06:14:09.812311+0000","last_fullsized":"2023-10-09T06:14:09.812311+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.339121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.339121+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:45.908579+0000","scrub_duration":0.010474344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":576,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.789237+0000","last_change":"2023-10-09T06:13:30.812598+0000","last_active":"2023-10-09T06:14:09.789237+0000","last_peered":"2023-10-09T06:14:09.789237+0000","last_clean":"2023-10-09T06:14:09.789237+0000","last_became_active":"2023-10-09T06:13:30.791460+0000","last_became_peered":"2023-10-09T06:13:30.791460+0000","last_unstale":"2023-10-09T06:14:09.789237+0000","last_undegraded":"2023-10-09T06:14:09.789237+0000","last_fullsized":"2023-10-09T06:14:09.789237+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:28.046730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":554,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.789749+0000","last_change":"2023-10-09T06:01:54.764916+0000","last_active":"2023-10-09T06:14:09.789749+0000","last_peered":"2023-10-09T06:14:09.789749+0000","last_clean":"2023-10-09T06:14:09.789749+0000","last_became_active":"2023-10-09T06:01:54.764705+0000","last_became_peered":"2023-10-09T06:01:54.764705+0000","last_unstale":"2023-10-09T06:14:09.789749+0000","last_undegraded":"2023-10-09T06:14:09.789749+0000","last_fullsized":"2023-10-09T06:14:09.789749+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:12.527719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":769,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.795593+0000","last_change":"2023-10-09T06:13:15.425525+0000","last_active":"2023-10-09T06:14:08.795593+0000","last_peered":"2023-10-09T06:14:08.795593+0000","last_clean":"2023-10-09T06:14:08.795593+0000","last_became_active":"2023-10-09T06:13:15.425086+0000","last_became_peered":"2023-10-09T06:13:15.425086+0000","last_unstale":"2023-10-09T06:14:08.795593+0000","last_undegraded":"2023-10-09T06:14:08.795593+0000","last_fullsized":"2023-10-09T06:14:08.795593+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:42.900876+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:42.900876+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:46.042715+0000","scrub_duration":0.0014928649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":30,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.786700+0000","last_change":"2023-10-09T06:13:30.811420+0000","last_active":"2023-10-09T06:14:08.786700+0000","last_peered":"2023-10-09T06:14:08.786700+0000","last_clean":"2023-10-09T06:14:08.786700+0000","last_became_active":"2023-10-09T06:13:30.792835+0000","last_became_peered":"2023-10-09T06:13:30.792835+0000","last_unstale":"2023-10-09T06:14:08.786700+0000","last_undegraded":"2023-10-09T06:14:08.786700+0000","last_fullsized":"2023-10-09T06:14:08.786700+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:38.962645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":586,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.812690+0000","last_change":"2023-10-09T06:01:54.761558+0000","last_active":"2023-10-09T06:14:08.812690+0000","last_peered":"2023-10-09T06:14:08.812690+0000","last_clean":"2023-10-09T06:14:08.812690+0000","last_became_active":"2023-10-09T06:01:54.761251+0000","last_became_peered":"2023-10-09T06:01:54.761251+0000","last_unstale":"2023-10-09T06:14:08.812690+0000","last_undegraded":"2023-10-09T06:14:08.812690+0000","last_fullsized":"2023-10-09T06:14:08.812690+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:53.245267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":1044,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.789243+0000","last_change":"2023-10-09T06:13:24.754018+0000","last_active":"2023-10-09T06:14:08.789243+0000","last_peered":"2023-10-09T06:14:08.789243+0000","last_clean":"2023-10-09T06:14:08.789243+0000","last_became_active":"2023-10-09T06:13:24.753712+0000","last_became_peered":"2023-10-09T06:13:24.753712+0000","last_unstale":"2023-10-09T06:14:08.789243+0000","last_undegraded":"2023-10-09T06:14:08.789243+0000","last_fullsized":"2023-10-09T06:14:08.789243+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:18.490833+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":24,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.786356+0000","last_change":"2023-10-09T06:13:30.788160+0000","last_active":"2023-10-09T06:14:08.786356+0000","last_peered":"2023-10-09T06:14:08.786356+0000","last_clean":"2023-10-09T06:14:08.786356+0000","last_became_active":"2023-10-09T06:13:30.787724+0000","last_became_peered":"2023-10-09T06:13:30.787724+0000","last_unstale":"2023-10-09T06:14:08.786356+0000","last_undegraded":"2023-10-09T06:14:08.786356+0000","last_fullsized":"2023-10-09T06:14:08.786356+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:38.376770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":24,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.787187+0000","last_change":"2023-10-09T06:13:30.789225+0000","last_active":"2023-10-09T06:14:08.787187+0000","last_peered":"2023-10-09T06:14:08.787187+0000","last_clean":"2023-10-09T06:14:08.787187+0000","last_became_active":"2023-10-09T06:13:30.788869+0000","last_became_peered":"2023-10-09T06:13:30.788869+0000","last_unstale":"2023-10-09T06:14:08.787187+0000","last_undegraded":"2023-10-09T06:14:08.787187+0000","last_fullsized":"2023-10-09T06:14:08.787187+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:19.519966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":782,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.811100+0000","last_change":"2023-10-09T06:12:55.750923+0000","last_active":"2023-10-09T06:14:09.811100+0000","last_peered":"2023-10-09T06:14:09.811100+0000","last_clean":"2023-10-09T06:14:09.811100+0000","last_became_active":"2023-10-09T06:12:55.750521+0000","last_became_peered":"2023-10-09T06:12:55.750521+0000","last_unstale":"2023-10-09T06:14:09.811100+0000","last_undegraded":"2023-10-09T06:14:09.811100+0000","last_fullsized":"2023-10-09T06:14:09.811100+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.316697+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.316697+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:23.591439+0000","scrub_duration":0.001666561,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":394,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.786403+0000","last_change":"2023-10-09T06:01:22.337171+0000","last_active":"2023-10-09T06:14:08.786403+0000","last_peered":"2023-10-09T06:14:08.786403+0000","last_clean":"2023-10-09T06:14:08.786403+0000","last_became_active":"2023-10-09T06:01:22.335438+0000","last_became_peered":"2023-10-09T06:01:22.335438+0000","last_unstale":"2023-10-09T06:14:08.786403+0000","last_undegraded":"2023-10-09T06:14:08.786403+0000","last_fullsized":"2023-10-09T06:14:08.786403+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:25.036448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":503,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.796337+0000","last_change":"2023-10-09T06:13:30.788284+0000","last_active":"2023-10-09T06:14:08.796337+0000","last_peered":"2023-10-09T06:14:08.796337+0000","last_clean":"2023-10-09T06:14:08.796337+0000","last_became_active":"2023-10-09T06:13:30.787933+0000","last_became_peered":"2023-10-09T06:13:30.787933+0000","last_unstale":"2023-10-09T06:14:08.796337+0000","last_undegraded":"2023-10-09T06:14:08.796337+0000","last_fullsized":"2023-10-09T06:14:08.796337+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:10.735438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":832,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.810161+0000","last_change":"2023-10-09T06:13:24.753908+0000","last_active":"2023-10-09T06:14:09.810161+0000","last_peered":"2023-10-09T06:14:09.810161+0000","last_clean":"2023-10-09T06:14:09.810161+0000","last_became_active":"2023-10-09T06:13:24.753566+0000","last_became_peered":"2023-10-09T06:13:24.753566+0000","last_unstale":"2023-10-09T06:14:09.810161+0000","last_undegraded":"2023-10-09T06:14:09.810161+0000","last_fullsized":"2023-10-09T06:14:09.810161+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:40.388125+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:40.388125+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:35.101978+0000","scrub_duration":0.0055576990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":114,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.783359+0000","last_change":"2023-10-09T06:13:12.243605+0000","last_active":"2023-10-09T06:14:09.783359+0000","last_peered":"2023-10-09T06:14:09.783359+0000","last_clean":"2023-10-09T06:14:09.783359+0000","last_became_active":"2023-10-09T06:13:12.237061+0000","last_became_peered":"2023-10-09T06:13:12.237061+0000","last_unstale":"2023-10-09T06:14:09.783359+0000","last_undegraded":"2023-10-09T06:14:09.783359+0000","last_fullsized":"2023-10-09T06:14:09.783359+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:29:03.666031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":51,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.796703+0000","last_change":"2023-10-09T06:13:30.788733+0000","last_active":"2023-10-09T06:14:08.796703+0000","last_peered":"2023-10-09T06:14:08.796703+0000","last_clean":"2023-10-09T06:14:08.796703+0000","last_became_active":"2023-10-09T06:13:30.788353+0000","last_became_peered":"2023-10-09T06:13:30.788353+0000","last_unstale":"2023-10-09T06:14:08.796703+0000","last_undegraded":"2023-10-09T06:14:08.796703+0000","last_fullsized":"2023-10-09T06:14:08.796703+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:06.308426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"542'8094","reported_seq":8156,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T06:14:16.157563+0000","last_change":"2023-10-09T06:11:46.032384+0000","last_active":"2023-10-09T06:14:16.157563+0000","last_peered":"2023-10-09T06:14:16.157563+0000","last_clean":"2023-10-09T06:14:16.157563+0000","last_became_active":"2023-10-09T06:11:46.032091+0000","last_became_peered":"2023-10-09T06:11:46.032091+0000","last_unstale":"2023-10-09T06:14:16.157563+0000","last_undegraded":"2023-10-09T06:14:16.157563+0000","last_fullsized":"2023-10-09T06:14:16.157563+0000","mapping_epoch":490,"log_start":"502'1700","ondisk_log_start":"502'1700","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6394,"log_dups_size":0,"ondisk_log_size":6394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:31.117384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259391488,"num_objects":3958,"num_object_clones":0,"num_object_copies":11874,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3958,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8094,"num_write_kb":385664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":722,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.787266+0000","last_change":"2023-10-09T06:13:33.701521+0000","last_active":"2023-10-09T06:14:08.787266+0000","last_peered":"2023-10-09T06:14:08.787266+0000","last_clean":"2023-10-09T06:14:08.787266+0000","last_became_active":"2023-10-09T06:13:33.700870+0000","last_became_peered":"2023-10-09T06:13:33.700870+0000","last_unstale":"2023-10-09T06:14:08.787266+0000","last_undegraded":"2023-10-09T06:14:08.787266+0000","last_fullsized":"2023-10-09T06:14:08.787266+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:10:59.422551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:10:59.422551+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:54.402494+0000","scrub_duration":0.0059716659999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":28,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.787538+0000","last_change":"2023-10-09T06:13:30.822903+0000","last_active":"2023-10-09T06:14:08.787538+0000","last_peered":"2023-10-09T06:14:08.787538+0000","last_clean":"2023-10-09T06:14:08.787538+0000","last_became_active":"2023-10-09T06:13:30.799013+0000","last_became_peered":"2023-10-09T06:13:30.799013+0000","last_unstale":"2023-10-09T06:14:08.787538+0000","last_undegraded":"2023-10-09T06:14:08.787538+0000","last_fullsized":"2023-10-09T06:14:08.787538+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:44.271821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":24,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.788975+0000","last_change":"2023-10-09T06:13:30.793531+0000","last_active":"2023-10-09T06:14:08.788975+0000","last_peered":"2023-10-09T06:14:08.788975+0000","last_clean":"2023-10-09T06:14:08.788975+0000","last_became_active":"2023-10-09T06:13:30.792689+0000","last_became_peered":"2023-10-09T06:13:30.792689+0000","last_unstale":"2023-10-09T06:14:08.788975+0000","last_undegraded":"2023-10-09T06:14:08.788975+0000","last_fullsized":"2023-10-09T06:14:08.788975+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:02.559186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"541'8057","reported_seq":9747,"reported_epoch":541,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:14:15.345018+0000","last_change":"2023-10-09T06:14:00.813784+0000","last_active":"2023-10-09T06:14:15.345018+0000","last_peered":"2023-10-09T06:14:15.345018+0000","last_clean":"2023-10-09T06:13:11.194737+0000","last_became_active":"2023-10-09T06:13:21.782116+0000","last_became_peered":"2023-10-09T06:13:21.782116+0000","last_unstale":"2023-10-09T06:14:15.345018+0000","last_undegraded":"2023-10-09T06:14:15.345018+0000","last_fullsized":"2023-10-09T06:14:15.345018+0000","mapping_epoch":521,"log_start":"504'2093","ondisk_log_start":"504'2093","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5964,"log_dups_size":0,"ondisk_log_size":5964,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:00.656766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263782400,"num_objects":4025,"num_object_clones":0,"num_object_copies":12075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":7517,"num_objects_unfound":0,"num_objects_dirty":4025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8057,"num_write_kb":386624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":801,"num_bytes_recovered":52494336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,2,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":4025}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":910,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.798086+0000","last_change":"2023-10-09T06:13:24.753866+0000","last_active":"2023-10-09T06:14:08.798086+0000","last_peered":"2023-10-09T06:14:08.798086+0000","last_clean":"2023-10-09T06:14:08.798086+0000","last_became_active":"2023-10-09T06:13:24.753432+0000","last_became_peered":"2023-10-09T06:13:24.753432+0000","last_unstale":"2023-10-09T06:14:08.798086+0000","last_undegraded":"2023-10-09T06:14:08.798086+0000","last_fullsized":"2023-10-09T06:14:08.798086+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:03.241335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:03.241335+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:19.854449+0000","scrub_duration":0.017780547000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":28,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.786602+0000","last_change":"2023-10-09T06:13:30.829763+0000","last_active":"2023-10-09T06:14:08.786602+0000","last_peered":"2023-10-09T06:14:08.786602+0000","last_clean":"2023-10-09T06:14:08.786602+0000","last_became_active":"2023-10-09T06:13:30.802486+0000","last_became_peered":"2023-10-09T06:13:30.802486+0000","last_unstale":"2023-10-09T06:14:08.786602+0000","last_undegraded":"2023-10-09T06:14:08.786602+0000","last_fullsized":"2023-10-09T06:14:08.786602+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:05.468205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":580,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.789114+0000","last_change":"2023-10-09T06:01:54.760367+0000","last_active":"2023-10-09T06:14:09.789114+0000","last_peered":"2023-10-09T06:14:09.789114+0000","last_clean":"2023-10-09T06:14:09.789114+0000","last_became_active":"2023-10-09T06:01:54.760087+0000","last_became_peered":"2023-10-09T06:01:54.760087+0000","last_unstale":"2023-10-09T06:14:09.789114+0000","last_undegraded":"2023-10-09T06:14:09.789114+0000","last_fullsized":"2023-10-09T06:14:09.789114+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:26.000104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"542'7938","reported_seq":7998,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T06:14:15.820129+0000","last_change":"2023-10-09T06:11:45.696810+0000","last_active":"2023-10-09T06:14:15.820129+0000","last_peered":"2023-10-09T06:14:15.820129+0000","last_clean":"2023-10-09T06:14:15.820129+0000","last_became_active":"2023-10-09T06:11:45.696476+0000","last_became_peered":"2023-10-09T06:11:45.696476+0000","last_unstale":"2023-10-09T06:14:15.820129+0000","last_undegraded":"2023-10-09T06:14:15.820129+0000","last_fullsized":"2023-10-09T06:14:15.820129+0000","mapping_epoch":490,"log_start":"504'2500","ondisk_log_start":"504'2500","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5438,"log_dups_size":0,"ondisk_log_size":5438,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:41:33.351567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261750784,"num_objects":3994,"num_object_clones":0,"num_object_copies":11982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3994,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7938,"num_write_kb":381824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":805,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.789735+0000","last_change":"2023-10-09T06:13:24.768958+0000","last_active":"2023-10-09T06:14:09.789735+0000","last_peered":"2023-10-09T06:14:09.789735+0000","last_clean":"2023-10-09T06:14:09.789735+0000","last_became_active":"2023-10-09T06:13:24.768643+0000","last_became_peered":"2023-10-09T06:13:24.768643+0000","last_unstale":"2023-10-09T06:14:09.789735+0000","last_undegraded":"2023-10-09T06:14:09.789735+0000","last_fullsized":"2023-10-09T06:14:09.789735+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:48.495739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:48.495739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:18:50.654354+0000","scrub_duration":0.0016345509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":28,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.787113+0000","last_change":"2023-10-09T06:13:30.814434+0000","last_active":"2023-10-09T06:14:08.787113+0000","last_peered":"2023-10-09T06:14:08.787113+0000","last_clean":"2023-10-09T06:14:08.787113+0000","last_became_active":"2023-10-09T06:13:30.793119+0000","last_became_peered":"2023-10-09T06:13:30.793119+0000","last_unstale":"2023-10-09T06:14:08.787113+0000","last_undegraded":"2023-10-09T06:14:08.787113+0000","last_fullsized":"2023-10-09T06:14:08.787113+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:34:31.724006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":566,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.809877+0000","last_change":"2023-10-09T06:13:30.792490+0000","last_active":"2023-10-09T06:14:09.809877+0000","last_peered":"2023-10-09T06:14:09.809877+0000","last_clean":"2023-10-09T06:14:09.809877+0000","last_became_active":"2023-10-09T06:13:30.792209+0000","last_became_peered":"2023-10-09T06:13:30.792209+0000","last_unstale":"2023-10-09T06:14:09.809877+0000","last_undegraded":"2023-10-09T06:14:09.809877+0000","last_fullsized":"2023-10-09T06:14:09.809877+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:14.208042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"542'7930","reported_seq":7992,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T06:14:16.151320+0000","last_change":"2023-10-09T06:11:46.031195+0000","last_active":"2023-10-09T06:14:16.151320+0000","last_peered":"2023-10-09T06:14:16.151320+0000","last_clean":"2023-10-09T06:14:16.151320+0000","last_became_active":"2023-10-09T06:11:46.030891+0000","last_became_peered":"2023-10-09T06:11:46.030891+0000","last_unstale":"2023-10-09T06:14:16.151320+0000","last_undegraded":"2023-10-09T06:14:16.151320+0000","last_fullsized":"2023-10-09T06:14:16.151320+0000","mapping_epoch":490,"log_start":"501'1500","ondisk_log_start":"501'1500","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6430,"log_dups_size":0,"ondisk_log_size":6430,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:25.788661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261226496,"num_objects":3986,"num_object_clones":0,"num_object_copies":11958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7930,"num_write_kb":381312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":757,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.789449+0000","last_change":"2023-10-09T06:13:33.708291+0000","last_active":"2023-10-09T06:14:09.789449+0000","last_peered":"2023-10-09T06:14:09.789449+0000","last_clean":"2023-10-09T06:14:09.789449+0000","last_became_active":"2023-10-09T06:13:33.707930+0000","last_became_peered":"2023-10-09T06:13:33.707930+0000","last_unstale":"2023-10-09T06:14:09.789449+0000","last_undegraded":"2023-10-09T06:14:09.789449+0000","last_fullsized":"2023-10-09T06:14:09.789449+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:16.361097+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:16.361097+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:18.111404+0000","scrub_duration":0.028533749000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":310,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.798096+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T06:14:08.798096+0000","last_peered":"2023-10-09T06:14:08.798096+0000","last_clean":"2023-10-09T06:14:08.798096+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T06:14:08.798096+0000","last_undegraded":"2023-10-09T06:14:08.798096+0000","last_fullsized":"2023-10-09T06:14:08.798096+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":555,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.783482+0000","last_change":"2023-10-09T06:09:27.558353+0000","last_active":"2023-10-09T06:14:09.783482+0000","last_peered":"2023-10-09T06:14:09.783482+0000","last_clean":"2023-10-09T06:14:09.783482+0000","last_became_active":"2023-10-09T06:09:27.557233+0000","last_became_peered":"2023-10-09T06:09:27.557233+0000","last_unstale":"2023-10-09T06:14:09.783482+0000","last_undegraded":"2023-10-09T06:14:09.783482+0000","last_fullsized":"2023-10-09T06:14:09.783482+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:53.149350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"542'8008","reported_seq":8070,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T06:14:16.568031+0000","last_change":"2023-10-09T06:11:45.694648+0000","last_active":"2023-10-09T06:14:16.568031+0000","last_peered":"2023-10-09T06:14:16.568031+0000","last_clean":"2023-10-09T06:14:16.568031+0000","last_became_active":"2023-10-09T06:11:45.694321+0000","last_became_peered":"2023-10-09T06:11:45.694321+0000","last_unstale":"2023-10-09T06:14:16.568031+0000","last_undegraded":"2023-10-09T06:14:16.568031+0000","last_fullsized":"2023-10-09T06:14:16.568031+0000","mapping_epoch":490,"log_start":"499'1400","ondisk_log_start":"499'1400","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6608,"log_dups_size":0,"ondisk_log_size":6608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:43.704525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256507904,"num_objects":3914,"num_object_clones":0,"num_object_copies":11742,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3914,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8008,"num_write_kb":381504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":927,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.786304+0000","last_change":"2023-10-09T06:13:33.716191+0000","last_active":"2023-10-09T06:14:08.786304+0000","last_peered":"2023-10-09T06:14:08.786304+0000","last_clean":"2023-10-09T06:14:08.786304+0000","last_became_active":"2023-10-09T06:13:33.715831+0000","last_became_peered":"2023-10-09T06:13:33.715831+0000","last_unstale":"2023-10-09T06:14:08.786304+0000","last_undegraded":"2023-10-09T06:14:08.786304+0000","last_fullsized":"2023-10-09T06:14:08.786304+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:42.795781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:42.795781+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:45.712738+0000","scrub_duration":0.0091262979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":344,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.796019+0000","last_change":"2023-10-09T06:09:27.565666+0000","last_active":"2023-10-09T06:14:08.796019+0000","last_peered":"2023-10-09T06:14:08.796019+0000","last_clean":"2023-10-09T06:14:08.796019+0000","last_became_active":"2023-10-09T06:09:27.565324+0000","last_became_peered":"2023-10-09T06:09:27.565324+0000","last_unstale":"2023-10-09T06:14:08.796019+0000","last_undegraded":"2023-10-09T06:14:08.796019+0000","last_fullsized":"2023-10-09T06:14:08.796019+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:51.216947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":516,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.789371+0000","last_change":"2023-10-09T06:13:21.727745+0000","last_active":"2023-10-09T06:14:09.789371+0000","last_peered":"2023-10-09T06:14:09.789371+0000","last_clean":"2023-10-09T06:14:09.789371+0000","last_became_active":"2023-10-09T06:13:21.727404+0000","last_became_peered":"2023-10-09T06:13:21.727404+0000","last_unstale":"2023-10-09T06:14:09.789371+0000","last_undegraded":"2023-10-09T06:14:09.789371+0000","last_fullsized":"2023-10-09T06:14:09.789371+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:24.931626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"541'8108","reported_seq":19662,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:15.346585+0000","last_change":"2023-10-09T06:13:32.697002+0000","last_active":"2023-10-09T06:14:15.346585+0000","last_peered":"2023-10-09T06:14:15.346585+0000","last_clean":"2023-10-09T06:14:15.346585+0000","last_became_active":"2023-10-09T06:13:32.683104+0000","last_became_peered":"2023-10-09T06:13:32.683104+0000","last_unstale":"2023-10-09T06:14:15.346585+0000","last_undegraded":"2023-10-09T06:14:15.346585+0000","last_fullsized":"2023-10-09T06:14:15.346585+0000","mapping_epoch":528,"log_start":"504'2135","ondisk_log_start":"504'2135","created":490,"last_epoch_clean":529,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5973,"log_dups_size":0,"ondisk_log_size":5973,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:19.426571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273022976,"num_objects":4166,"num_object_clones":0,"num_object_copies":12498,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4166,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8108,"num_write_kb":392768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5732,"num_bytes_recovered":375652352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":804,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.779863+0000","last_change":"2023-10-09T06:13:15.442141+0000","last_active":"2023-10-09T06:14:09.779863+0000","last_peered":"2023-10-09T06:14:09.779863+0000","last_clean":"2023-10-09T06:14:09.779863+0000","last_became_active":"2023-10-09T06:13:15.441716+0000","last_became_peered":"2023-10-09T06:13:15.441716+0000","last_unstale":"2023-10-09T06:14:09.779863+0000","last_undegraded":"2023-10-09T06:14:09.779863+0000","last_fullsized":"2023-10-09T06:14:09.779863+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.282393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.282393+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:16.798440+0000","scrub_duration":0.01402403,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":461,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.788926+0000","last_change":"2023-10-09T06:13:21.738218+0000","last_active":"2023-10-09T06:14:09.788926+0000","last_peered":"2023-10-09T06:14:09.788926+0000","last_clean":"2023-10-09T06:14:09.788926+0000","last_became_active":"2023-10-09T06:13:21.737969+0000","last_became_peered":"2023-10-09T06:13:21.737969+0000","last_unstale":"2023-10-09T06:14:09.788926+0000","last_undegraded":"2023-10-09T06:14:09.788926+0000","last_fullsized":"2023-10-09T06:14:09.788926+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:38.620281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":590,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.789671+0000","last_change":"2023-10-09T06:07:43.572173+0000","last_active":"2023-10-09T06:14:09.789671+0000","last_peered":"2023-10-09T06:14:09.789671+0000","last_clean":"2023-10-09T06:14:09.789671+0000","last_became_active":"2023-10-09T06:07:43.571690+0000","last_became_peered":"2023-10-09T06:07:43.571690+0000","last_unstale":"2023-10-09T06:14:09.789671+0000","last_undegraded":"2023-10-09T06:14:09.789671+0000","last_fullsized":"2023-10-09T06:14:09.789671+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:10.234125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"542'8195","reported_seq":13115,"reported_epoch":542,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T06:14:16.589952+0000","last_change":"2023-10-09T06:14:01.451081+0000","last_active":"2023-10-09T06:14:16.589952+0000","last_peered":"2023-10-09T06:14:16.589952+0000","last_clean":"2023-10-09T06:13:20.539588+0000","last_became_active":"2023-10-09T06:13:22.846120+0000","last_became_peered":"2023-10-09T06:13:22.846120+0000","last_unstale":"2023-10-09T06:14:16.589952+0000","last_undegraded":"2023-10-09T06:14:16.589952+0000","last_fullsized":"2023-10-09T06:14:16.589952+0000","mapping_epoch":522,"log_start":"501'1600","ondisk_log_start":"501'1600","created":490,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6595,"log_dups_size":0,"ondisk_log_size":6595,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:55.561925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264437760,"num_objects":4035,"num_object_clones":0,"num_object_copies":12105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1815,"num_objects_unfound":0,"num_objects_dirty":4035,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8195,"num_write_kb":391360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2419,"num_bytes_recovered":158531584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,4,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":4035}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":876,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.810408+0000","last_change":"2023-10-09T06:13:15.428701+0000","last_active":"2023-10-09T06:14:09.810408+0000","last_peered":"2023-10-09T06:14:09.810408+0000","last_clean":"2023-10-09T06:14:09.810408+0000","last_became_active":"2023-10-09T06:13:15.428343+0000","last_became_peered":"2023-10-09T06:13:15.428343+0000","last_unstale":"2023-10-09T06:14:09.810408+0000","last_undegraded":"2023-10-09T06:14:09.810408+0000","last_fullsized":"2023-10-09T06:14:09.810408+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.373748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.373748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:55:34.235074+0000","scrub_duration":0.0024576450000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":532,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.789219+0000","last_change":"2023-10-09T06:07:43.574863+0000","last_active":"2023-10-09T06:14:09.789219+0000","last_peered":"2023-10-09T06:14:09.789219+0000","last_clean":"2023-10-09T06:14:09.789219+0000","last_became_active":"2023-10-09T06:07:43.574576+0000","last_became_peered":"2023-10-09T06:07:43.574576+0000","last_unstale":"2023-10-09T06:14:09.789219+0000","last_undegraded":"2023-10-09T06:14:09.789219+0000","last_fullsized":"2023-10-09T06:14:09.789219+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:16.166004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":573,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.789586+0000","last_change":"2023-10-09T06:13:21.727814+0000","last_active":"2023-10-09T06:14:09.789586+0000","last_peered":"2023-10-09T06:14:09.789586+0000","last_clean":"2023-10-09T06:14:09.789586+0000","last_became_active":"2023-10-09T06:13:21.727401+0000","last_became_peered":"2023-10-09T06:13:21.727401+0000","last_unstale":"2023-10-09T06:14:09.789586+0000","last_undegraded":"2023-10-09T06:14:09.789586+0000","last_fullsized":"2023-10-09T06:14:09.789586+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:53:37.968028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"542'8082","reported_seq":8142,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T06:14:15.864399+0000","last_change":"2023-10-09T06:11:45.695709+0000","last_active":"2023-10-09T06:14:15.864399+0000","last_peered":"2023-10-09T06:14:15.864399+0000","last_clean":"2023-10-09T06:14:15.864399+0000","last_became_active":"2023-10-09T06:11:45.695219+0000","last_became_peered":"2023-10-09T06:11:45.695219+0000","last_unstale":"2023-10-09T06:14:15.864399+0000","last_undegraded":"2023-10-09T06:14:15.864399+0000","last_fullsized":"2023-10-09T06:14:15.864399+0000","mapping_epoch":490,"log_start":"504'2700","ondisk_log_start":"504'2700","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5382,"log_dups_size":0,"ondisk_log_size":5382,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:26.533964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264110080,"num_objects":4030,"num_object_clones":0,"num_object_copies":12090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8082,"num_write_kb":387584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":619,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.784233+0000","last_change":"2023-10-09T06:13:24.753519+0000","last_active":"2023-10-09T06:14:08.784233+0000","last_peered":"2023-10-09T06:14:08.784233+0000","last_clean":"2023-10-09T06:14:08.784233+0000","last_became_active":"2023-10-09T06:13:24.753185+0000","last_became_peered":"2023-10-09T06:13:24.753185+0000","last_unstale":"2023-10-09T06:14:08.784233+0000","last_undegraded":"2023-10-09T06:14:08.784233+0000","last_fullsized":"2023-10-09T06:14:08.784233+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:39.633410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:39.633410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:56.871057+0000","scrub_duration":0.001547624,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":352,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.795488+0000","last_change":"2023-10-09T06:13:30.781849+0000","last_active":"2023-10-09T06:14:08.795488+0000","last_peered":"2023-10-09T06:14:08.795488+0000","last_clean":"2023-10-09T06:14:08.795488+0000","last_became_active":"2023-10-09T06:13:30.781084+0000","last_became_peered":"2023-10-09T06:13:30.781084+0000","last_unstale":"2023-10-09T06:14:08.795488+0000","last_undegraded":"2023-10-09T06:14:08.795488+0000","last_fullsized":"2023-10-09T06:14:08.795488+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:02.199298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":224,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.812138+0000","last_change":"2023-10-09T06:01:54.761768+0000","last_active":"2023-10-09T06:14:09.812138+0000","last_peered":"2023-10-09T06:14:09.812138+0000","last_clean":"2023-10-09T06:14:09.812138+0000","last_became_active":"2023-10-09T06:01:54.761443+0000","last_became_peered":"2023-10-09T06:01:54.761443+0000","last_unstale":"2023-10-09T06:14:09.812138+0000","last_undegraded":"2023-10-09T06:14:09.812138+0000","last_fullsized":"2023-10-09T06:14:09.812138+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:51.555987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"542'8071","reported_seq":8138,"reported_epoch":542,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:14:16.160183+0000","last_change":"2023-10-09T06:13:31.904728+0000","last_active":"2023-10-09T06:14:16.160183+0000","last_peered":"2023-10-09T06:14:16.160183+0000","last_clean":"2023-10-09T06:13:29.715670+0000","last_became_active":"2023-10-09T06:13:31.891405+0000","last_became_peered":"2023-10-09T06:13:31.891405+0000","last_unstale":"2023-10-09T06:14:16.160183+0000","last_undegraded":"2023-10-09T06:14:16.160183+0000","last_fullsized":"2023-10-09T06:14:16.160183+0000","mapping_epoch":527,"log_start":"502'1600","ondisk_log_start":"502'1600","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6471,"log_dups_size":0,"ondisk_log_size":6471,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:13:59.527962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270991360,"num_objects":4135,"num_object_clones":0,"num_object_copies":12405,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4135,"num_objects_unfound":0,"num_objects_dirty":4135,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8071,"num_write_kb":390592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[0,4,3],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,3,4","objects":4135}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":743,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.796779+0000","last_change":"2023-10-09T06:13:33.707885+0000","last_active":"2023-10-09T06:14:08.796779+0000","last_peered":"2023-10-09T06:14:08.796779+0000","last_clean":"2023-10-09T06:14:08.796779+0000","last_became_active":"2023-10-09T06:13:33.707474+0000","last_became_peered":"2023-10-09T06:13:33.707474+0000","last_unstale":"2023-10-09T06:14:08.796779+0000","last_undegraded":"2023-10-09T06:14:08.796779+0000","last_fullsized":"2023-10-09T06:14:08.796779+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:12:22.289803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:12:22.289803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:27.842817+0000","scrub_duration":0.0076565050000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":286,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.809963+0000","last_change":"2023-10-09T06:12:04.680247+0000","last_active":"2023-10-09T06:14:09.809963+0000","last_peered":"2023-10-09T06:14:09.809963+0000","last_clean":"2023-10-09T06:14:09.809963+0000","last_became_active":"2023-10-09T06:12:04.679943+0000","last_became_peered":"2023-10-09T06:12:04.679943+0000","last_unstale":"2023-10-09T06:14:09.809963+0000","last_undegraded":"2023-10-09T06:14:09.809963+0000","last_fullsized":"2023-10-09T06:14:09.809963+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:56.692774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":329,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.780389+0000","last_change":"2023-10-09T06:07:43.577306+0000","last_active":"2023-10-09T06:14:09.780389+0000","last_peered":"2023-10-09T06:14:09.780389+0000","last_clean":"2023-10-09T06:14:09.780389+0000","last_became_active":"2023-10-09T06:07:43.576956+0000","last_became_peered":"2023-10-09T06:07:43.576956+0000","last_unstale":"2023-10-09T06:14:09.780389+0000","last_undegraded":"2023-10-09T06:14:09.780389+0000","last_fullsized":"2023-10-09T06:14:09.780389+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:21.020403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":257,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.810008+0000","last_change":"2023-10-09T06:13:12.234849+0000","last_active":"2023-10-09T06:14:09.810008+0000","last_peered":"2023-10-09T06:14:09.810008+0000","last_clean":"2023-10-09T06:14:09.810008+0000","last_became_active":"2023-10-09T06:13:12.234284+0000","last_became_peered":"2023-10-09T06:13:12.234284+0000","last_unstale":"2023-10-09T06:14:09.810008+0000","last_undegraded":"2023-10-09T06:14:09.810008+0000","last_fullsized":"2023-10-09T06:14:09.810008+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:40:43.809773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"542'7907","reported_seq":7969,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T06:14:16.157044+0000","last_change":"2023-10-09T06:11:46.029349+0000","last_active":"2023-10-09T06:14:16.157044+0000","last_peered":"2023-10-09T06:14:16.157044+0000","last_clean":"2023-10-09T06:14:16.157044+0000","last_became_active":"2023-10-09T06:11:46.029041+0000","last_became_peered":"2023-10-09T06:11:46.029041+0000","last_unstale":"2023-10-09T06:14:16.157044+0000","last_undegraded":"2023-10-09T06:14:16.157044+0000","last_fullsized":"2023-10-09T06:14:16.157044+0000","mapping_epoch":490,"log_start":"501'1500","ondisk_log_start":"501'1500","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6407,"log_dups_size":0,"ondisk_log_size":6407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:07.662395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259850240,"num_objects":3965,"num_object_clones":0,"num_object_copies":11895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3965,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7907,"num_write_kb":379904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":777,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.783824+0000","last_change":"2023-10-09T06:09:27.544555+0000","last_active":"2023-10-09T06:14:09.783824+0000","last_peered":"2023-10-09T06:14:09.783824+0000","last_clean":"2023-10-09T06:14:09.783824+0000","last_became_active":"2023-10-09T06:09:27.544222+0000","last_became_peered":"2023-10-09T06:09:27.544222+0000","last_unstale":"2023-10-09T06:14:09.783824+0000","last_undegraded":"2023-10-09T06:14:09.783824+0000","last_fullsized":"2023-10-09T06:14:09.783824+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:45.944253+0000","scrub_duration":0.0016660799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":242,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.810082+0000","last_change":"2023-10-09T06:13:21.738040+0000","last_active":"2023-10-09T06:14:09.810082+0000","last_peered":"2023-10-09T06:14:09.810082+0000","last_clean":"2023-10-09T06:14:09.810082+0000","last_became_active":"2023-10-09T06:13:21.737717+0000","last_became_peered":"2023-10-09T06:13:21.737717+0000","last_unstale":"2023-10-09T06:14:09.810082+0000","last_undegraded":"2023-10-09T06:14:09.810082+0000","last_fullsized":"2023-10-09T06:14:09.810082+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:39.451628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":522,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.788996+0000","last_change":"2023-10-09T06:09:27.553958+0000","last_active":"2023-10-09T06:14:09.788996+0000","last_peered":"2023-10-09T06:14:09.788996+0000","last_clean":"2023-10-09T06:14:09.788996+0000","last_became_active":"2023-10-09T06:09:27.553193+0000","last_became_peered":"2023-10-09T06:09:27.553193+0000","last_unstale":"2023-10-09T06:14:09.788996+0000","last_undegraded":"2023-10-09T06:14:09.788996+0000","last_fullsized":"2023-10-09T06:14:09.788996+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:09.754579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":367,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.810085+0000","last_change":"2023-10-09T06:07:43.580765+0000","last_active":"2023-10-09T06:14:09.810085+0000","last_peered":"2023-10-09T06:14:09.810085+0000","last_clean":"2023-10-09T06:14:09.810085+0000","last_became_active":"2023-10-09T06:07:43.580458+0000","last_became_peered":"2023-10-09T06:07:43.580458+0000","last_unstale":"2023-10-09T06:14:09.810085+0000","last_undegraded":"2023-10-09T06:14:09.810085+0000","last_fullsized":"2023-10-09T06:14:09.810085+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:55.161893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"542'7960","reported_seq":21521,"reported_epoch":542,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T06:14:15.884958+0000","last_change":"2023-10-09T06:14:08.672062+0000","last_active":"2023-10-09T06:14:15.884958+0000","last_peered":"2023-10-09T06:14:15.884958+0000","last_clean":"2023-10-09T06:13:11.209726+0000","last_became_active":"2023-10-09T06:13:30.857105+0000","last_became_peered":"2023-10-09T06:13:30.857105+0000","last_unstale":"2023-10-09T06:14:15.884958+0000","last_undegraded":"2023-10-09T06:14:15.884958+0000","last_fullsized":"2023-10-09T06:14:15.884958+0000","mapping_epoch":526,"log_start":"504'2600","ondisk_log_start":"504'2600","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5360,"log_dups_size":0,"ondisk_log_size":5360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:30.233175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258211840,"num_objects":3940,"num_object_clones":0,"num_object_copies":11820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1878,"num_objects_unfound":0,"num_objects_dirty":3940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7960,"num_write_kb":380800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6159,"num_bytes_recovered":403636224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[4,2,3],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"2,3,4","objects":3940}],"blocked_by":[],"up_primary":7,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":586,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.786453+0000","last_change":"2023-10-09T06:13:33.705726+0000","last_active":"2023-10-09T06:14:08.786453+0000","last_peered":"2023-10-09T06:14:08.786453+0000","last_clean":"2023-10-09T06:14:08.786453+0000","last_became_active":"2023-10-09T06:13:33.705351+0000","last_became_peered":"2023-10-09T06:13:33.705351+0000","last_unstale":"2023-10-09T06:14:08.786453+0000","last_undegraded":"2023-10-09T06:14:08.786453+0000","last_fullsized":"2023-10-09T06:14:08.786453+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:44.483427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.483427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":321,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.795762+0000","last_change":"2023-10-09T06:09:27.565004+0000","last_active":"2023-10-09T06:14:08.795762+0000","last_peered":"2023-10-09T06:14:08.795762+0000","last_clean":"2023-10-09T06:14:08.795762+0000","last_became_active":"2023-10-09T06:09:27.564073+0000","last_became_peered":"2023-10-09T06:09:27.564073+0000","last_unstale":"2023-10-09T06:14:08.795762+0000","last_undegraded":"2023-10-09T06:14:08.795762+0000","last_fullsized":"2023-10-09T06:14:08.795762+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:15.146042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":312,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.809878+0000","last_change":"2023-10-09T06:13:21.728291+0000","last_active":"2023-10-09T06:14:09.809878+0000","last_peered":"2023-10-09T06:14:09.809878+0000","last_clean":"2023-10-09T06:14:09.809878+0000","last_became_active":"2023-10-09T06:13:21.727926+0000","last_became_peered":"2023-10-09T06:13:21.727926+0000","last_unstale":"2023-10-09T06:14:09.809878+0000","last_undegraded":"2023-10-09T06:14:09.809878+0000","last_fullsized":"2023-10-09T06:14:09.809878+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:35.292785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":239,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.812116+0000","last_change":"2023-10-09T06:09:27.547953+0000","last_active":"2023-10-09T06:14:09.812116+0000","last_peered":"2023-10-09T06:14:09.812116+0000","last_clean":"2023-10-09T06:14:09.812116+0000","last_became_active":"2023-10-09T06:09:27.547624+0000","last_became_peered":"2023-10-09T06:09:27.547624+0000","last_unstale":"2023-10-09T06:14:09.812116+0000","last_undegraded":"2023-10-09T06:14:09.812116+0000","last_fullsized":"2023-10-09T06:14:09.812116+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:39.630486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"542'7999","reported_seq":11179,"reported_epoch":542,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:14:16.572534+0000","last_change":"2023-10-09T06:13:30.783853+0000","last_active":"2023-10-09T06:14:16.572534+0000","last_peered":"2023-10-09T06:14:16.572534+0000","last_clean":"2023-10-09T06:13:20.543162+0000","last_became_active":"2023-10-09T06:13:22.836318+0000","last_became_peered":"2023-10-09T06:13:22.836318+0000","last_unstale":"2023-10-09T06:14:16.572534+0000","last_undegraded":"2023-10-09T06:14:16.572534+0000","last_fullsized":"2023-10-09T06:14:16.572534+0000","mapping_epoch":522,"log_start":"500'1400","ondisk_log_start":"500'1400","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6599,"log_dups_size":0,"ondisk_log_size":6599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:40.353464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257884160,"num_objects":3935,"num_object_clones":0,"num_object_copies":11805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2875,"num_objects_unfound":0,"num_objects_dirty":3935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7999,"num_write_kb":381888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1552,"num_bytes_recovered":101711872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,0,4],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":3935}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":24,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.786765+0000","last_change":"2023-10-09T06:13:30.791685+0000","last_active":"2023-10-09T06:14:08.786765+0000","last_peered":"2023-10-09T06:14:08.786765+0000","last_clean":"2023-10-09T06:14:08.786765+0000","last_became_active":"2023-10-09T06:13:30.791255+0000","last_became_peered":"2023-10-09T06:13:30.791255+0000","last_unstale":"2023-10-09T06:14:08.786765+0000","last_undegraded":"2023-10-09T06:14:08.786765+0000","last_fullsized":"2023-10-09T06:14:08.786765+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:24.896244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":584,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.788894+0000","last_change":"2023-10-09T06:13:33.704835+0000","last_active":"2023-10-09T06:14:08.788894+0000","last_peered":"2023-10-09T06:14:08.788894+0000","last_clean":"2023-10-09T06:14:08.788894+0000","last_became_active":"2023-10-09T06:13:33.704466+0000","last_became_peered":"2023-10-09T06:13:33.704466+0000","last_unstale":"2023-10-09T06:14:08.788894+0000","last_undegraded":"2023-10-09T06:14:08.788894+0000","last_fullsized":"2023-10-09T06:14:08.788894+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:35.783434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:35.783434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:38.721974+0000","scrub_duration":0.0043794799999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":329,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.795738+0000","last_change":"2023-10-09T06:13:12.238280+0000","last_active":"2023-10-09T06:14:08.795738+0000","last_peered":"2023-10-09T06:14:08.795738+0000","last_clean":"2023-10-09T06:14:08.795738+0000","last_became_active":"2023-10-09T06:13:12.237986+0000","last_became_peered":"2023-10-09T06:13:12.237986+0000","last_unstale":"2023-10-09T06:14:08.795738+0000","last_undegraded":"2023-10-09T06:14:08.795738+0000","last_fullsized":"2023-10-09T06:14:08.795738+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:58.520176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":472,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.789592+0000","last_change":"2023-10-09T06:13:21.733940+0000","last_active":"2023-10-09T06:14:09.789592+0000","last_peered":"2023-10-09T06:14:09.789592+0000","last_clean":"2023-10-09T06:14:09.789592+0000","last_became_active":"2023-10-09T06:13:21.733628+0000","last_became_peered":"2023-10-09T06:13:21.733628+0000","last_unstale":"2023-10-09T06:14:09.789592+0000","last_undegraded":"2023-10-09T06:14:09.789592+0000","last_fullsized":"2023-10-09T06:14:09.789592+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:13.348443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"541'7923","reported_seq":17872,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:14.248953+0000","last_change":"2023-10-09T06:14:09.843785+0000","last_active":"2023-10-09T06:14:14.248953+0000","last_peered":"2023-10-09T06:14:14.248953+0000","last_clean":"2023-10-09T06:14:14.248953+0000","last_became_active":"2023-10-09T06:14:09.789864+0000","last_became_peered":"2023-10-09T06:14:09.789864+0000","last_unstale":"2023-10-09T06:14:14.248953+0000","last_undegraded":"2023-10-09T06:14:14.248953+0000","last_fullsized":"2023-10-09T06:14:14.248953+0000","mapping_epoch":540,"log_start":"501'1500","ondisk_log_start":"501'1500","created":490,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6423,"log_dups_size":0,"ondisk_log_size":6423,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:03.191322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261554176,"num_objects":3991,"num_object_clones":0,"num_object_copies":11973,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3991,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7923,"num_write_kb":381248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4936,"num_bytes_recovered":323485696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":810,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.810847+0000","last_change":"2023-10-09T06:13:24.753842+0000","last_active":"2023-10-09T06:14:09.810847+0000","last_peered":"2023-10-09T06:14:09.810847+0000","last_clean":"2023-10-09T06:14:09.810847+0000","last_became_active":"2023-10-09T06:13:24.753485+0000","last_became_peered":"2023-10-09T06:13:24.753485+0000","last_unstale":"2023-10-09T06:14:09.810847+0000","last_undegraded":"2023-10-09T06:14:09.810847+0000","last_fullsized":"2023-10-09T06:14:09.810847+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:57.627296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:57.627296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:56.656218+0000","scrub_duration":0.001913166,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":434,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.789157+0000","last_change":"2023-10-09T06:09:27.544373+0000","last_active":"2023-10-09T06:14:08.789157+0000","last_peered":"2023-10-09T06:14:08.789157+0000","last_clean":"2023-10-09T06:14:08.789157+0000","last_became_active":"2023-10-09T06:09:27.544121+0000","last_became_peered":"2023-10-09T06:09:27.544121+0000","last_unstale":"2023-10-09T06:14:08.789157+0000","last_undegraded":"2023-10-09T06:14:08.789157+0000","last_fullsized":"2023-10-09T06:14:08.789157+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:40.429491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":479,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.783569+0000","last_change":"2023-10-09T06:12:52.670573+0000","last_active":"2023-10-09T06:14:09.783569+0000","last_peered":"2023-10-09T06:14:09.783569+0000","last_clean":"2023-10-09T06:14:09.783569+0000","last_became_active":"2023-10-09T06:12:52.670242+0000","last_became_peered":"2023-10-09T06:12:52.670242+0000","last_unstale":"2023-10-09T06:14:09.783569+0000","last_undegraded":"2023-10-09T06:14:09.783569+0000","last_fullsized":"2023-10-09T06:14:09.783569+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:25.237580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":618,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.783900+0000","last_change":"2023-10-09T06:09:27.543678+0000","last_active":"2023-10-09T06:14:09.783900+0000","last_peered":"2023-10-09T06:14:09.783900+0000","last_clean":"2023-10-09T06:14:09.783900+0000","last_became_active":"2023-10-09T06:09:27.543298+0000","last_became_peered":"2023-10-09T06:09:27.543298+0000","last_unstale":"2023-10-09T06:14:09.783900+0000","last_undegraded":"2023-10-09T06:14:09.783900+0000","last_fullsized":"2023-10-09T06:14:09.783900+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:26.297842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"542'8028","reported_seq":17621,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T06:14:15.817318+0000","last_change":"2023-10-09T06:14:02.643285+0000","last_active":"2023-10-09T06:14:15.817318+0000","last_peered":"2023-10-09T06:14:15.817318+0000","last_clean":"2023-10-09T06:14:15.817318+0000","last_became_active":"2023-10-09T06:14:02.599489+0000","last_became_peered":"2023-10-09T06:14:02.599489+0000","last_unstale":"2023-10-09T06:14:15.817318+0000","last_undegraded":"2023-10-09T06:14:15.817318+0000","last_fullsized":"2023-10-09T06:14:15.817318+0000","mapping_epoch":537,"log_start":"504'2600","ondisk_log_start":"504'2600","created":490,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5428,"log_dups_size":0,"ondisk_log_size":5428,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:10.022074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259194904,"num_objects":3956,"num_object_clones":0,"num_object_copies":11868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3956,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8028,"num_write_kb":383425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4755,"num_bytes_recovered":311558168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":740,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.796257+0000","last_change":"2023-10-09T06:13:24.756288+0000","last_active":"2023-10-09T06:14:08.796257+0000","last_peered":"2023-10-09T06:14:08.796257+0000","last_clean":"2023-10-09T06:14:08.796257+0000","last_became_active":"2023-10-09T06:13:24.755945+0000","last_became_peered":"2023-10-09T06:13:24.755945+0000","last_unstale":"2023-10-09T06:14:08.796257+0000","last_undegraded":"2023-10-09T06:14:08.796257+0000","last_fullsized":"2023-10-09T06:14:08.796257+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:45.844892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:45.844892+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:44.107376+0000","scrub_duration":0.0017521489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":24,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.786841+0000","last_change":"2023-10-09T06:13:30.799535+0000","last_active":"2023-10-09T06:14:08.786841+0000","last_peered":"2023-10-09T06:14:08.786841+0000","last_clean":"2023-10-09T06:14:08.786841+0000","last_became_active":"2023-10-09T06:13:30.799227+0000","last_became_peered":"2023-10-09T06:13:30.799227+0000","last_unstale":"2023-10-09T06:14:08.786841+0000","last_undegraded":"2023-10-09T06:14:08.786841+0000","last_fullsized":"2023-10-09T06:14:08.786841+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:14.741742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":260,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.810166+0000","last_change":"2023-10-09T06:13:12.237622+0000","last_active":"2023-10-09T06:14:09.810166+0000","last_peered":"2023-10-09T06:14:09.810166+0000","last_clean":"2023-10-09T06:14:09.810166+0000","last_became_active":"2023-10-09T06:13:12.237287+0000","last_became_peered":"2023-10-09T06:13:12.237287+0000","last_unstale":"2023-10-09T06:14:09.810166+0000","last_undegraded":"2023-10-09T06:14:09.810166+0000","last_fullsized":"2023-10-09T06:14:09.810166+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:01.942898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":24,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.786607+0000","last_change":"2023-10-09T06:13:30.788382+0000","last_active":"2023-10-09T06:14:08.786607+0000","last_peered":"2023-10-09T06:14:08.786607+0000","last_clean":"2023-10-09T06:14:08.786607+0000","last_became_active":"2023-10-09T06:13:30.788060+0000","last_became_peered":"2023-10-09T06:13:30.788060+0000","last_unstale":"2023-10-09T06:14:08.786607+0000","last_undegraded":"2023-10-09T06:14:08.786607+0000","last_fullsized":"2023-10-09T06:14:08.786607+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:36:58.984799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.e","version":"542'8076","reported_seq":19168,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-09T06:14:15.868462+0000","last_change":"2023-10-09T06:13:51.831598+0000","last_active":"2023-10-09T06:14:15.868462+0000","last_peered":"2023-10-09T06:14:15.868462+0000","last_clean":"2023-10-09T06:14:15.868462+0000","last_became_active":"2023-10-09T06:13:51.816987+0000","last_became_peered":"2023-10-09T06:13:51.816987+0000","last_unstale":"2023-10-09T06:14:15.868462+0000","last_undegraded":"2023-10-09T06:14:15.868462+0000","last_fullsized":"2023-10-09T06:14:15.868462+0000","mapping_epoch":534,"log_start":"504'2700","ondisk_log_start":"504'2700","created":490,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5376,"log_dups_size":0,"ondisk_log_size":5376,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:52.460066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268697600,"num_objects":4100,"num_object_clones":0,"num_object_copies":12300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8076,"num_write_kb":389632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5503,"num_bytes_recovered":360644608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":1015,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.789827+0000","last_change":"2023-10-09T06:13:24.762291+0000","last_active":"2023-10-09T06:14:09.789827+0000","last_peered":"2023-10-09T06:14:09.789827+0000","last_clean":"2023-10-09T06:14:09.789827+0000","last_became_active":"2023-10-09T06:13:24.761918+0000","last_became_peered":"2023-10-09T06:13:24.761918+0000","last_unstale":"2023-10-09T06:14:09.789827+0000","last_undegraded":"2023-10-09T06:14:09.789827+0000","last_fullsized":"2023-10-09T06:14:09.789827+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:17.344518+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:17.344518+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:58.227603+0000","scrub_duration":0.0015914099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":597,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.797197+0000","last_change":"2023-10-09T06:13:30.798394+0000","last_active":"2023-10-09T06:14:08.797197+0000","last_peered":"2023-10-09T06:14:08.797197+0000","last_clean":"2023-10-09T06:14:08.797197+0000","last_became_active":"2023-10-09T06:13:30.790913+0000","last_became_peered":"2023-10-09T06:13:30.790913+0000","last_unstale":"2023-10-09T06:14:08.797197+0000","last_undegraded":"2023-10-09T06:14:08.797197+0000","last_fullsized":"2023-10-09T06:14:08.797197+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:26.508690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":358,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.812222+0000","last_change":"2023-10-09T06:13:21.728009+0000","last_active":"2023-10-09T06:14:09.812222+0000","last_peered":"2023-10-09T06:14:09.812222+0000","last_clean":"2023-10-09T06:14:09.812222+0000","last_became_active":"2023-10-09T06:13:21.727683+0000","last_became_peered":"2023-10-09T06:13:21.727683+0000","last_unstale":"2023-10-09T06:14:09.812222+0000","last_undegraded":"2023-10-09T06:14:09.812222+0000","last_fullsized":"2023-10-09T06:14:09.812222+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:13.074232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":346,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.795777+0000","last_change":"2023-10-09T06:13:30.782038+0000","last_active":"2023-10-09T06:14:08.795777+0000","last_peered":"2023-10-09T06:14:08.795777+0000","last_clean":"2023-10-09T06:14:08.795777+0000","last_became_active":"2023-10-09T06:13:30.781500+0000","last_became_peered":"2023-10-09T06:13:30.781500+0000","last_unstale":"2023-10-09T06:14:08.795777+0000","last_undegraded":"2023-10-09T06:14:08.795777+0000","last_fullsized":"2023-10-09T06:14:08.795777+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:43.663265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"541'7910","reported_seq":7971,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:15.303193+0000","last_change":"2023-10-09T06:11:46.031709+0000","last_active":"2023-10-09T06:14:15.303193+0000","last_peered":"2023-10-09T06:14:15.303193+0000","last_clean":"2023-10-09T06:14:15.303193+0000","last_became_active":"2023-10-09T06:11:46.031421+0000","last_became_peered":"2023-10-09T06:11:46.031421+0000","last_unstale":"2023-10-09T06:14:15.303193+0000","last_undegraded":"2023-10-09T06:14:15.303193+0000","last_fullsized":"2023-10-09T06:14:15.303193+0000","mapping_epoch":490,"log_start":"504'2025","ondisk_log_start":"504'2025","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5885,"log_dups_size":0,"ondisk_log_size":5885,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:13.095174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262799360,"num_objects":4010,"num_object_clones":0,"num_object_copies":12030,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4010,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7910,"num_write_kb":381440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":873,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.780658+0000","last_change":"2023-10-09T06:13:33.708100+0000","last_active":"2023-10-09T06:14:09.780658+0000","last_peered":"2023-10-09T06:14:09.780658+0000","last_clean":"2023-10-09T06:14:09.780658+0000","last_became_active":"2023-10-09T06:13:33.707809+0000","last_became_peered":"2023-10-09T06:13:33.707809+0000","last_unstale":"2023-10-09T06:14:09.780658+0000","last_undegraded":"2023-10-09T06:14:09.780658+0000","last_fullsized":"2023-10-09T06:14:09.780658+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:43:26.848778+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":24,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.787671+0000","last_change":"2023-10-09T06:13:30.799948+0000","last_active":"2023-10-09T06:14:08.787671+0000","last_peered":"2023-10-09T06:14:08.787671+0000","last_clean":"2023-10-09T06:14:08.787671+0000","last_became_active":"2023-10-09T06:13:30.799690+0000","last_became_peered":"2023-10-09T06:13:30.799690+0000","last_unstale":"2023-10-09T06:14:08.787671+0000","last_undegraded":"2023-10-09T06:14:08.787671+0000","last_fullsized":"2023-10-09T06:14:08.787671+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:43.893184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":588,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.784140+0000","last_change":"2023-10-09T06:09:27.578756+0000","last_active":"2023-10-09T06:14:09.784140+0000","last_peered":"2023-10-09T06:14:09.784140+0000","last_clean":"2023-10-09T06:14:09.784140+0000","last_became_active":"2023-10-09T06:09:27.578384+0000","last_became_peered":"2023-10-09T06:09:27.578384+0000","last_unstale":"2023-10-09T06:14:09.784140+0000","last_undegraded":"2023-10-09T06:14:09.784140+0000","last_fullsized":"2023-10-09T06:14:09.784140+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:22.182630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":488,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.789273+0000","last_change":"2023-10-09T06:01:22.334081+0000","last_active":"2023-10-09T06:14:09.789273+0000","last_peered":"2023-10-09T06:14:09.789273+0000","last_clean":"2023-10-09T06:14:09.789273+0000","last_became_active":"2023-10-09T06:01:22.333663+0000","last_became_peered":"2023-10-09T06:01:22.333663+0000","last_unstale":"2023-10-09T06:14:09.789273+0000","last_undegraded":"2023-10-09T06:14:09.789273+0000","last_fullsized":"2023-10-09T06:14:09.789273+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:55.809259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":787,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.780481+0000","last_change":"2023-10-09T06:09:27.560157+0000","last_active":"2023-10-09T06:14:09.780481+0000","last_peered":"2023-10-09T06:14:09.780481+0000","last_clean":"2023-10-09T06:14:09.780481+0000","last_became_active":"2023-10-09T06:09:27.559466+0000","last_became_peered":"2023-10-09T06:09:27.559466+0000","last_unstale":"2023-10-09T06:14:09.780481+0000","last_undegraded":"2023-10-09T06:14:09.780481+0000","last_fullsized":"2023-10-09T06:14:09.780481+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.181811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.181811+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:53.431491+0000","scrub_duration":0.011553313000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":470,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.783733+0000","last_change":"2023-10-09T06:13:30.791343+0000","last_active":"2023-10-09T06:14:09.783733+0000","last_peered":"2023-10-09T06:14:09.783733+0000","last_clean":"2023-10-09T06:14:09.783733+0000","last_became_active":"2023-10-09T06:13:30.790945+0000","last_became_peered":"2023-10-09T06:13:30.790945+0000","last_unstale":"2023-10-09T06:14:09.783733+0000","last_undegraded":"2023-10-09T06:14:09.783733+0000","last_fullsized":"2023-10-09T06:14:09.783733+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:14.208391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":243,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.810240+0000","last_change":"2023-10-09T06:07:43.580844+0000","last_active":"2023-10-09T06:14:09.810240+0000","last_peered":"2023-10-09T06:14:09.810240+0000","last_clean":"2023-10-09T06:14:09.810240+0000","last_became_active":"2023-10-09T06:07:43.580523+0000","last_became_peered":"2023-10-09T06:07:43.580523+0000","last_unstale":"2023-10-09T06:14:09.810240+0000","last_undegraded":"2023-10-09T06:14:09.810240+0000","last_fullsized":"2023-10-09T06:14:09.810240+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:55.942430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":890,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.796108+0000","last_change":"2023-10-09T06:00:04.215425+0000","last_active":"2023-10-09T06:14:08.796108+0000","last_peered":"2023-10-09T06:14:08.796108+0000","last_clean":"2023-10-09T06:14:08.796108+0000","last_became_active":"2023-10-09T05:59:40.326685+0000","last_became_peered":"2023-10-09T05:59:40.326685+0000","last_unstale":"2023-10-09T06:14:08.796108+0000","last_undegraded":"2023-10-09T06:14:08.796108+0000","last_fullsized":"2023-10-09T06:14:08.796108+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:04.215270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:04.215270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:03.963694+0000","scrub_duration":0.002850652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":313,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.789510+0000","last_change":"2023-10-09T06:01:54.824780+0000","last_active":"2023-10-09T06:14:09.789510+0000","last_peered":"2023-10-09T06:14:09.789510+0000","last_clean":"2023-10-09T06:14:09.789510+0000","last_became_active":"2023-10-09T06:01:54.757727+0000","last_became_peered":"2023-10-09T06:01:54.757727+0000","last_unstale":"2023-10-09T06:14:09.789510+0000","last_undegraded":"2023-10-09T06:14:09.789510+0000","last_fullsized":"2023-10-09T06:14:09.789510+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:44.132556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":560,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.789495+0000","last_change":"2023-10-09T06:01:54.764989+0000","last_active":"2023-10-09T06:14:09.789495+0000","last_peered":"2023-10-09T06:14:09.789495+0000","last_clean":"2023-10-09T06:14:09.789495+0000","last_became_active":"2023-10-09T06:01:54.764790+0000","last_became_peered":"2023-10-09T06:01:54.764790+0000","last_unstale":"2023-10-09T06:14:09.789495+0000","last_undegraded":"2023-10-09T06:14:09.789495+0000","last_fullsized":"2023-10-09T06:14:09.789495+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:14:07.208858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":327,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.789274+0000","last_change":"2023-10-09T06:13:21.739730+0000","last_active":"2023-10-09T06:14:09.789274+0000","last_peered":"2023-10-09T06:14:09.789274+0000","last_clean":"2023-10-09T06:14:09.789274+0000","last_became_active":"2023-10-09T06:13:21.727326+0000","last_became_peered":"2023-10-09T06:13:21.727326+0000","last_unstale":"2023-10-09T06:14:09.789274+0000","last_undegraded":"2023-10-09T06:14:09.789274+0000","last_fullsized":"2023-10-09T06:14:09.789274+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:51.344571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":557,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.788933+0000","last_change":"2023-10-09T06:09:27.565198+0000","last_active":"2023-10-09T06:14:09.788933+0000","last_peered":"2023-10-09T06:14:09.788933+0000","last_clean":"2023-10-09T06:14:09.788933+0000","last_became_active":"2023-10-09T06:09:27.564909+0000","last_became_peered":"2023-10-09T06:09:27.564909+0000","last_unstale":"2023-10-09T06:14:09.788933+0000","last_undegraded":"2023-10-09T06:14:09.788933+0000","last_fullsized":"2023-10-09T06:14:09.788933+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:46.101014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":706,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.787889+0000","last_change":"2023-10-09T06:13:33.699235+0000","last_active":"2023-10-09T06:14:08.787889+0000","last_peered":"2023-10-09T06:14:08.787889+0000","last_clean":"2023-10-09T06:14:08.787889+0000","last_became_active":"2023-10-09T06:13:33.698886+0000","last_became_peered":"2023-10-09T06:13:33.698886+0000","last_unstale":"2023-10-09T06:14:08.787889+0000","last_undegraded":"2023-10-09T06:14:08.787889+0000","last_fullsized":"2023-10-09T06:14:08.787889+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:00.456132+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:00.456132+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:59.421706+0000","scrub_duration":0.0014992199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":347,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.796420+0000","last_change":"2023-10-09T06:13:21.740383+0000","last_active":"2023-10-09T06:14:08.796420+0000","last_peered":"2023-10-09T06:14:08.796420+0000","last_clean":"2023-10-09T06:14:08.796420+0000","last_became_active":"2023-10-09T06:13:21.740023+0000","last_became_peered":"2023-10-09T06:13:21.740023+0000","last_unstale":"2023-10-09T06:14:08.796420+0000","last_undegraded":"2023-10-09T06:14:08.796420+0000","last_fullsized":"2023-10-09T06:14:08.796420+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:27.380765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":342,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:14:08.795815+0000","last_change":"2023-10-09T06:09:27.565185+0000","last_active":"2023-10-09T06:14:08.795815+0000","last_peered":"2023-10-09T06:14:08.795815+0000","last_clean":"2023-10-09T06:14:08.795815+0000","last_became_active":"2023-10-09T06:09:27.564478+0000","last_became_peered":"2023-10-09T06:09:27.564478+0000","last_unstale":"2023-10-09T06:14:08.795815+0000","last_undegraded":"2023-10-09T06:14:08.795815+0000","last_fullsized":"2023-10-09T06:14:08.795815+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:04.622171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":865,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.810342+0000","last_change":"2023-10-09T06:12:55.753877+0000","last_active":"2023-10-09T06:14:09.810342+0000","last_peered":"2023-10-09T06:14:09.810342+0000","last_clean":"2023-10-09T06:14:09.810342+0000","last_became_active":"2023-10-09T06:12:55.753512+0000","last_became_peered":"2023-10-09T06:12:55.753512+0000","last_unstale":"2023-10-09T06:14:09.810342+0000","last_undegraded":"2023-10-09T06:14:09.810342+0000","last_fullsized":"2023-10-09T06:14:09.810342+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:46.290510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:46.290510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:30.368031+0000","scrub_duration":0.002627819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":534,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.784094+0000","last_change":"2023-10-09T06:09:27.556271+0000","last_active":"2023-10-09T06:14:09.784094+0000","last_peered":"2023-10-09T06:14:09.784094+0000","last_clean":"2023-10-09T06:14:09.784094+0000","last_became_active":"2023-10-09T06:09:27.555932+0000","last_became_peered":"2023-10-09T06:09:27.555932+0000","last_unstale":"2023-10-09T06:14:09.784094+0000","last_undegraded":"2023-10-09T06:14:09.784094+0000","last_fullsized":"2023-10-09T06:14:09.784094+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:33.888622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":573,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-09T06:14:09.788510+0000","last_change":"2023-10-09T06:13:21.727720+0000","last_active":"2023-10-09T06:14:09.788510+0000","last_peered":"2023-10-09T06:14:09.788510+0000","last_clean":"2023-10-09T06:14:09.788510+0000","last_became_active":"2023-10-09T06:13:21.727339+0000","last_became_peered":"2023-10-09T06:13:21.727339+0000","last_unstale":"2023-10-09T06:14:09.788510+0000","last_undegraded":"2023-10-09T06:14:09.788510+0000","last_fullsized":"2023-10-09T06:14:09.788510+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:25.723752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4203413528,"num_objects":64140,"num_object_clones":0,"num_object_copies":192420,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":18220,"num_objects_unfound":0,"num_objects_dirty":64140,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":128286,"num_write_kb":6157569,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31857,"num_bytes_recovered":2087714840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14310457344,"data_stored":14310441056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96733,"ondisk_log_size":96733,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":11714,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13410,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":617203,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1523929,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234415,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1584184,"kb_used_data":687472,"kb_used_omap":0,"kb_used_meta":896704,"kb_avail":92160968,"statfs":{"total":95995035648,"available":94372831232,"internally_reserved":0,"allocated":703971328,"data_stored":702433454,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918224896},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:31 2023","interfaces":[{"interface":"back","average":{"1min":8.7140000000000004,"5min":2.887,"15min":1.784},"min":{"1min":0.26900000000000002,"5min":0.25,"15min":0.25},"max":{"1min":57.435000000000002,"5min":57.435000000000002,"15min":57.435000000000002},"last":3.8570000000000002},{"interface":"front","average":{"1min":8.8059999999999992,"5min":3.0590000000000002,"15min":1.7829999999999999},"min":{"1min":0.318,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":57.472999999999999,"5min":57.472999999999999,"15min":57.472999999999999},"last":3.871}]},{"osd":1,"last update":"Mon Oct 9 06:14:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":4.141,"15min":3.2189999999999999},"min":{"1min":0.27200000000000002,"5min":0.255,"15min":0.218},"max":{"1min":5.9580000000000002,"5min":124.66200000000001,"15min":124.66200000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.216,"5min":4.1829999999999998,"15min":3.2069999999999999},"min":{"1min":0.33900000000000002,"5min":0.27100000000000002,"15min":0.22700000000000001},"max":{"1min":5.8650000000000002,"5min":125.023,"15min":125.023},"last":2.0070000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:13:31 2023","interfaces":[{"interface":"back","average":{"1min":3.456,"5min":1.788,"15min":2.222},"min":{"1min":0.316,"5min":0.27400000000000002,"15min":0.24299999999999999},"max":{"1min":47.503,"5min":47.503,"15min":127.86499999999999},"last":0.77300000000000002},{"interface":"front","average":{"1min":2.9900000000000002,"5min":2.0110000000000001,"15min":2.2250000000000001},"min":{"1min":0.318,"5min":0.29599999999999999,"15min":0.27000000000000002},"max":{"1min":47.545000000000002,"5min":47.545000000000002,"15min":129.24000000000001},"last":0.41099999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:14:05 2023","interfaces":[{"interface":"back","average":{"1min":10.307,"5min":7.9900000000000002,"15min":6.4820000000000002},"min":{"1min":0.35499999999999998,"5min":0.32100000000000001,"15min":0.29899999999999999},"max":{"1min":67.852000000000004,"5min":116.68000000000001,"15min":164.16999999999999},"last":1.0600000000000001},{"interface":"front","average":{"1min":10.298,"5min":8.2260000000000009,"15min":6.6589999999999998},"min":{"1min":0.36699999999999999,"5min":0.33300000000000002,"15min":0.32100000000000001},"max":{"1min":67.881,"5min":116.797,"15min":164.40700000000001},"last":0.50600000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":1.1220000000000001,"15min":0.92600000000000005},"min":{"1min":0.27100000000000002,"5min":0.23999999999999999,"15min":0.23100000000000001},"max":{"1min":3.4180000000000001,"5min":14.371,"15min":14.371},"last":0.317},{"interface":"front","average":{"1min":0.51300000000000001,"5min":1.002,"15min":0.91500000000000004},"min":{"1min":0.30199999999999999,"5min":0.28599999999999998,"15min":0.27100000000000002},"max":{"1min":1.3220000000000001,"5min":14.259,"15min":14.259},"last":0.47499999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":1.381,"15min":1.0669999999999999},"min":{"1min":0.317,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.6589999999999998,"5min":43.826999999999998,"15min":43.826999999999998},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.53800000000000003,"5min":1.28,"15min":1.119},"min":{"1min":0.39200000000000002,"5min":0.27300000000000002,"15min":0.24399999999999999},"max":{"1min":0.85399999999999998,"5min":43.880000000000003,"15min":43.880000000000003},"last":0.44800000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.85599999999999998,"15min":0.97999999999999998},"min":{"1min":0.33600000000000002,"5min":0.28000000000000003,"15min":0.26600000000000001},"max":{"1min":2.9209999999999998,"5min":4.6779999999999999,"15min":9.4540000000000006},"last":0.81100000000000005},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.94099999999999995,"15min":1.0089999999999999},"min":{"1min":0.35299999999999998,"5min":0.307,"15min":0.30599999999999999},"max":{"1min":2.3940000000000001,"5min":6.2329999999999997,"15min":6.2329999999999997},"last":0.72699999999999998}]}]},{"osd":6,"up_from":46,"seq":197568496052,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2144552,"kb_used_data":1246944,"kb_used_omap":0,"kb_used_meta":897536,"kb_avail":91600600,"statfs":{"total":95995035648,"available":93799014400,"internally_reserved":0,"allocated":1276870656,"data_stored":1275348235,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919076864},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:28 2023","interfaces":[{"interface":"back","average":{"1min":18.359000000000002,"5min":7.04,"15min":3.8420000000000001},"min":{"1min":0.246,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":181.209,"5min":191.50800000000001,"15min":191.50800000000001},"last":0.25700000000000001},{"interface":"front","average":{"1min":18.960999999999999,"5min":7.0990000000000002,"15min":3.6920000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":181.233,"5min":191.52600000000001,"15min":191.52600000000001},"last":0.38900000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:13:23 2023","interfaces":[{"interface":"back","average":{"1min":26.061,"5min":6.2930000000000001,"15min":5.1870000000000003},"min":{"1min":0.27000000000000002,"5min":0.25,"15min":0.23200000000000001},"max":{"1min":328.04399999999998,"5min":328.04399999999998,"15min":328.04399999999998},"last":0.35399999999999998},{"interface":"front","average":{"1min":26.131,"5min":6.4470000000000001,"15min":5.2800000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":328.10000000000002,"5min":328.10000000000002,"15min":328.10000000000002},"last":0.30599999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:13:28 2023","interfaces":[{"interface":"back","average":{"1min":18.006,"5min":4.798,"15min":3.0049999999999999},"min":{"1min":0.28000000000000003,"5min":0.27300000000000002,"15min":0.26200000000000001},"max":{"1min":163.04900000000001,"5min":163.04900000000001,"15min":163.04900000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":17.959,"5min":4.9500000000000002,"15min":2.96},"min":{"1min":0.31900000000000001,"5min":0.29399999999999998,"15min":0.28599999999999998},"max":{"1min":162.93000000000001,"5min":162.93000000000001,"15min":162.93000000000001},"last":0.41099999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:14:12 2023","interfaces":[{"interface":"back","average":{"1min":11.064,"5min":8.5009999999999994,"15min":7.375},"min":{"1min":0.33000000000000002,"5min":0.318,"15min":0.307},"max":{"1min":81.317999999999998,"5min":250.744,"15min":250.744},"last":54.834000000000003},{"interface":"front","average":{"1min":9.3520000000000003,"5min":7.9539999999999997,"15min":7.1600000000000001},"min":{"1min":0.374,"5min":0.33600000000000002,"15min":0.314},"max":{"1min":81.233000000000004,"5min":250.83799999999999,"15min":250.83799999999999},"last":54.892000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:13:47 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":1.0740000000000001,"15min":1.681},"min":{"1min":0.28100000000000003,"5min":0.24399999999999999,"15min":0.24199999999999999},"max":{"1min":9.0869999999999997,"5min":9.0869999999999997,"15min":151.35599999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":1.97,"5min":1.175,"15min":1.7350000000000001},"min":{"1min":0.33500000000000002,"5min":0.249,"15min":0.23999999999999999},"max":{"1min":7.8849999999999998,"5min":7.8849999999999998,"15min":149.15799999999999},"last":0.48699999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:13:23 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.95599999999999996,"15min":1.0629999999999999},"min":{"1min":0.29699999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.3799999999999999,"5min":9.1959999999999997,"15min":15.342000000000001},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.90000000000000002,"5min":1.1100000000000001,"15min":1.131},"min":{"1min":0.312,"5min":0.28199999999999997,"15min":0.255},"max":{"1min":2.762,"5min":8.9809999999999999,"15min":15.358000000000001},"last":0.43099999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:13:47 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":1.105,"15min":1.2070000000000001},"min":{"1min":0.372,"5min":0.30299999999999999,"15min":0.27000000000000002},"max":{"1min":5.819,"5min":5.819,"15min":21.98},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.79400000000000004,"5min":1.002,"15min":1.1559999999999999},"min":{"1min":0.39300000000000002,"5min":0.34200000000000003,"15min":0.33100000000000002},"max":{"1min":2.8570000000000002,"5min":5.3710000000000004,"15min":18.241},"last":0.73199999999999998}]}]},{"osd":1,"up_from":89,"seq":382252089640,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1351852,"kb_used_data":773476,"kb_used_omap":9,"kb_used_meta":578358,"kb_avail":92393300,"statfs":{"total":95995035648,"available":94610739200,"internally_reserved":0,"allocated":792039424,"data_stored":788936995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":592239593},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:35 2023","interfaces":[{"interface":"back","average":{"1min":1.7809999999999999,"5min":1.946,"15min":2.1659999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.613,"5min":10.613,"15min":10.613},"last":2.8809999999999998},{"interface":"front","average":{"1min":2.79,"5min":2.391,"15min":2.3570000000000002},"min":{"1min":0.254,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":13.121,"5min":24.239999999999998,"15min":24.239999999999998},"last":4.2480000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:13:44 2023","interfaces":[{"interface":"back","average":{"1min":2.7469999999999999,"5min":1.9850000000000001,"15min":2.4660000000000002},"min":{"1min":0.27800000000000002,"5min":0.21099999999999999,"15min":0.20799999999999999},"max":{"1min":12.818,"5min":15.522,"15min":129.053},"last":1.0149999999999999},{"interface":"front","average":{"1min":2.3540000000000001,"5min":2.4900000000000002,"15min":2.6629999999999998},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.20300000000000001},"max":{"1min":8.9949999999999992,"5min":13.539999999999999,"15min":129.077},"last":55.694000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:14:13 2023","interfaces":[{"interface":"back","average":{"1min":5.1669999999999998,"5min":4.484,"15min":5.0309999999999997},"min":{"1min":0.28000000000000003,"5min":0.26800000000000002,"15min":0.24199999999999999},"max":{"1min":48.177999999999997,"5min":113.322,"15min":213.654},"last":3.6880000000000002},{"interface":"front","average":{"1min":4.3879999999999999,"5min":4.6349999999999998,"15min":5.0250000000000004},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.27400000000000002},"max":{"1min":48.216999999999999,"5min":113.38200000000001,"15min":213.77199999999999},"last":3.6429999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:13:44 2023","interfaces":[{"interface":"back","average":{"1min":9.1400000000000006,"5min":3.1629999999999998,"15min":2.133},"min":{"1min":0.40400000000000003,"5min":0.311,"15min":0.30399999999999999},"max":{"1min":93.747,"5min":93.747,"15min":93.747},"last":0.499},{"interface":"front","average":{"1min":9.8000000000000007,"5min":3.8359999999999999,"15min":2.6000000000000001},"min":{"1min":0.32800000000000001,"5min":0.308,"15min":0.308},"max":{"1min":97.852000000000004,"5min":97.852000000000004,"15min":97.852000000000004},"last":1.3540000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:13:57 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.6220000000000001,"15min":1.5680000000000001},"min":{"1min":0.32200000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.4850000000000003,"5min":9.3109999999999999,"15min":36.356999999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":1.383,"5min":2.5870000000000002,"15min":2.27},"min":{"1min":0.34599999999999997,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.2619999999999996,"5min":17.603999999999999,"15min":36.268000000000001},"last":1.4550000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:13:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":1.7410000000000001,"15min":1.7789999999999999},"min":{"1min":0.38500000000000001,"5min":0.33700000000000002,"15min":0.318},"max":{"1min":5.5170000000000003,"5min":9.282,"15min":52.207000000000001},"last":0.53000000000000003},{"interface":"front","average":{"1min":1.901,"5min":2.4180000000000001,"15min":2.3559999999999999},"min":{"1min":0.38500000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":8.7159999999999993,"5min":15.384,"15min":51.212000000000003},"last":1.4279999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:13:57 2023","interfaces":[{"interface":"back","average":{"1min":5.7400000000000002,"5min":3.3849999999999998,"15min":2.9329999999999998},"min":{"1min":0.39800000000000002,"5min":0.374,"15min":0.374},"max":{"1min":56.881,"5min":56.881,"15min":108.553},"last":1.3959999999999999},{"interface":"front","average":{"1min":5.7210000000000001,"5min":2.6360000000000001,"15min":2.3980000000000001},"min":{"1min":0.46000000000000002,"5min":0.38,"15min":0.38},"max":{"1min":56.941000000000003,"5min":56.941000000000003,"15min":108.56999999999999},"last":0.97599999999999998}]}]},{"osd":0,"up_from":456,"seq":1958505087035,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3087232,"kb_used_data":2833208,"kb_used_omap":32,"kb_used_meta":253855,"kb_avail":90657920,"statfs":{"total":95995035648,"available":92833710080,"internally_reserved":0,"allocated":2901204992,"data_stored":2898315924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32794,"internal_metadata":259948518},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":5.7939999999999996,"15min":9.3539999999999992},"min":{"1min":0.23999999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":5.157,"5min":166.833,"15min":166.833},"last":1.9910000000000001},{"interface":"front","average":{"1min":3.29,"5min":6.1779999999999999,"15min":8.7200000000000006},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":39.424999999999997,"5min":166.81100000000001,"15min":166.81100000000001},"last":2.0310000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":3.0449999999999999,"5min":2.9129999999999998,"15min":3.677},"min":{"1min":0.29999999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":21.648,"5min":21.648,"15min":21.648},"last":0.438},{"interface":"front","average":{"1min":2.1800000000000002,"5min":2.6600000000000001,"15min":3.488},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":11.890000000000001,"5min":20.152000000000001,"15min":20.152000000000001},"last":2.0920000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:13:32 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":7.532,"15min":13.125999999999999},"min":{"1min":0.26500000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.9029999999999996,"5min":203.59299999999999,"15min":203.59299999999999},"last":2.1339999999999999},{"interface":"front","average":{"1min":3.4969999999999999,"5min":7.657,"15min":12.565},"min":{"1min":0.35299999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":18.192,"5min":202.96600000000001,"15min":202.96600000000001},"last":0.46899999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":2.444,"5min":2.0659999999999998,"15min":2.294},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":23.515999999999998,"5min":23.515999999999998,"15min":23.515999999999998},"last":2.1669999999999998},{"interface":"front","average":{"1min":4.2149999999999999,"5min":2.5169999999999999,"15min":2.1669999999999998},"min":{"1min":0.35799999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":39.302,"5min":39.302,"15min":39.302},"last":2.069}]},{"osd":5,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":3.1899999999999999,"5min":3.7200000000000002,"15min":2.7000000000000002},"min":{"1min":0.32200000000000001,"5min":0.316,"15min":0.316},"max":{"1min":39.369999999999997,"5min":161.286,"15min":161.286},"last":3.4769999999999999},{"interface":"front","average":{"1min":2.1110000000000002,"5min":3.532,"15min":2.8260000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":13.731999999999999,"5min":161.31800000000001,"15min":161.31800000000001},"last":0.51300000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":2.1190000000000002,"15min":2.3570000000000002},"min":{"1min":0.38900000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":13.795,"5min":13.795,"15min":13.795},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.3759999999999999,"5min":1.931,"15min":2.5350000000000001},"min":{"1min":0.44700000000000001,"5min":0.372,"15min":0.372},"max":{"1min":5.274,"5min":10.233000000000001,"15min":10.233000000000001},"last":2.2360000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":2.423,"5min":3.5070000000000001,"15min":2.8570000000000002},"min":{"1min":0.44900000000000001,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":13.821,"5min":123.685,"15min":123.685},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.5049999999999999,"5min":3.214,"15min":2.9129999999999998},"min":{"1min":0.44,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":5.3200000000000003,"5min":122.482,"15min":122.482},"last":2.2010000000000001}]}]},{"osd":2,"up_from":94,"seq":403726926132,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2529532,"kb_used_data":1764468,"kb_used_omap":81,"kb_used_meta":764974,"kb_avail":91215620,"statfs":{"total":95995035648,"available":93404794880,"internally_reserved":0,"allocated":1806815232,"data_stored":1804051337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":783333594},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":7.7000000000000002,"5min":3.5070000000000001,"15min":3.2679999999999998},"min":{"1min":0.221,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":56.213000000000001,"5min":56.213000000000001,"15min":56.213000000000001},"last":5.9690000000000003},{"interface":"front","average":{"1min":6.4610000000000003,"5min":2.8140000000000001,"15min":2.4329999999999998},"min":{"1min":0.221,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":50.764000000000003,"5min":50.764000000000003,"15min":50.764000000000003},"last":0.77200000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":5.0819999999999999,"15min":4.4909999999999997},"min":{"1min":0.27800000000000002,"5min":0.248,"15min":0.19400000000000001},"max":{"1min":4.258,"5min":160.45500000000001,"15min":222.59399999999999},"last":0.59799999999999998},{"interface":"front","average":{"1min":2.1379999999999999,"5min":6.0540000000000003,"15min":4.9480000000000004},"min":{"1min":0.27600000000000002,"5min":0.27200000000000002,"15min":0.20200000000000001},"max":{"1min":10.345000000000001,"5min":163.697,"15min":224.619},"last":10.129}]},{"osd":3,"last update":"Mon Oct 9 06:13:58 2023","interfaces":[{"interface":"back","average":{"1min":12.108000000000001,"5min":7.2910000000000004,"15min":5.101},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23000000000000001},"max":{"1min":122.205,"5min":122.205,"15min":144.88200000000001},"last":6.165},{"interface":"front","average":{"1min":11.619,"5min":6.952,"15min":5.0369999999999999},"min":{"1min":0.30199999999999999,"5min":0.252,"15min":0.252},"max":{"1min":122.45,"5min":122.45,"15min":140.79499999999999},"last":0.45900000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":3.1659999999999999,"15min":2.226},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.28299999999999997},"max":{"1min":8.9550000000000001,"5min":107.51300000000001,"15min":107.51300000000001},"last":8.0009999999999994},{"interface":"front","average":{"1min":1.081,"5min":2.6040000000000001,"15min":2.0579999999999998},"min":{"1min":0.36599999999999999,"5min":0.34100000000000003,"15min":0.31},"max":{"1min":3.4039999999999999,"5min":108.768,"15min":108.768},"last":0.42599999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":1.73,"5min":1.502,"15min":1.5549999999999999},"min":{"1min":0.376,"5min":0.34399999999999997,"15min":0.32600000000000001},"max":{"1min":11.015000000000001,"5min":11.015000000000001,"15min":28.48},"last":11.108000000000001},{"interface":"front","average":{"1min":1.2430000000000001,"5min":1.7150000000000001,"15min":1.728},"min":{"1min":0.379,"5min":0.316,"15min":0.28899999999999998},"max":{"1min":6.9180000000000001,"5min":9.8699999999999992,"15min":30.318999999999999},"last":0.63200000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":1.603,"5min":2.218,"15min":2.085},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.314},"max":{"1min":10.301,"5min":14.962,"15min":30.420000000000002},"last":10.09},{"interface":"front","average":{"1min":1.8540000000000001,"5min":1.621,"15min":1.7749999999999999},"min":{"1min":0.432,"5min":0.36099999999999999,"15min":0.32200000000000001},"max":{"1min":11.103,"5min":11.103,"15min":30.713000000000001},"last":6.3940000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":1.8300000000000001,"5min":2.9089999999999998,"15min":2.2349999999999999},"min":{"1min":0.40799999999999997,"5min":0.313,"15min":0.313},"max":{"1min":11.065,"5min":124.069,"15min":124.069},"last":11.147},{"interface":"front","average":{"1min":1.0449999999999999,"5min":2.8119999999999998,"15min":2.258},"min":{"1min":0.376,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.4210000000000003,"5min":124.08799999999999,"15min":124.08799999999999},"last":1.444}]}]},{"osd":3,"up_from":99,"seq":425201762605,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3061700,"kb_used_data":2109884,"kb_used_omap":7,"kb_used_meta":951800,"kb_avail":90683452,"statfs":{"total":95995035648,"available":92859854848,"internally_reserved":0,"allocated":2160521216,"data_stored":2157442962,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":974643816},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:36 2023","interfaces":[{"interface":"back","average":{"1min":3.5409999999999999,"5min":1.9590000000000001,"15min":2.0590000000000002},"min":{"1min":0.222,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":30.841000000000001,"5min":30.841000000000001,"15min":30.841000000000001},"last":92.039000000000001},{"interface":"front","average":{"1min":3.4769999999999999,"5min":2.8650000000000002,"15min":3.3050000000000002},"min":{"1min":0.27900000000000003,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":28.652000000000001,"5min":28.652000000000001,"15min":28.652000000000001},"last":91.843999999999994}]},{"osd":1,"last update":"Mon Oct 9 06:14:10 2023","interfaces":[{"interface":"back","average":{"1min":4.1269999999999998,"5min":2.218,"15min":2.9510000000000001},"min":{"1min":0.253,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":21.934000000000001,"5min":22.579999999999998,"15min":207.75299999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":4.3049999999999997,"5min":3.54,"15min":3.9649999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.23200000000000001},"max":{"1min":21.809000000000001,"5min":30.524000000000001,"15min":207.12200000000001},"last":0.46400000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:14:10 2023","interfaces":[{"interface":"back","average":{"1min":2.3889999999999998,"5min":2.4609999999999999,"15min":2.6549999999999998},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.223},"max":{"1min":12.561,"5min":30.547999999999998,"15min":107.01300000000001},"last":0.26000000000000001},{"interface":"front","average":{"1min":2.6640000000000001,"5min":3.0899999999999999,"15min":3.0699999999999998},"min":{"1min":0.313,"5min":0.25600000000000001,"15min":0.23899999999999999},"max":{"1min":11.749000000000001,"5min":25.984000000000002,"15min":109.807},"last":0.314}]},{"osd":4,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":6.8099999999999996,"5min":2.8759999999999999,"15min":3.5569999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.254},"max":{"1min":95.739000000000004,"5min":95.739000000000004,"15min":99.876000000000005},"last":0.29699999999999999},{"interface":"front","average":{"1min":6.8259999999999996,"5min":3.399,"15min":3.7210000000000001},"min":{"1min":0.373,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":97.941999999999993,"5min":97.941999999999993,"15min":99.914000000000001},"last":0.35499999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:14:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2989999999999999,"5min":2.7200000000000002,"15min":2.0649999999999999},"min":{"1min":0.40500000000000003,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.3970000000000002,"5min":74.706000000000003,"15min":74.706000000000003},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.202,"5min":3.4420000000000002,"15min":3.2759999999999998},"min":{"1min":0.32800000000000001,"5min":0.32500000000000001,"15min":0.28799999999999998},"max":{"1min":4.0449999999999999,"5min":74.685000000000002,"15min":235.56},"last":0.33900000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:14:10 2023","interfaces":[{"interface":"back","average":{"1min":1.734,"5min":1.351,"15min":4.3070000000000004},"min":{"1min":0.46500000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.4210000000000003,"5min":6.8739999999999997,"15min":233.26599999999999},"last":0.48599999999999999},{"interface":"front","average":{"1min":2.266,"5min":2.2069999999999999,"15min":4.7619999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.33600000000000002},"max":{"1min":12.471,"5min":26.890000000000001,"15min":233.24100000000001},"last":0.40000000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:14:10 2023","interfaces":[{"interface":"back","average":{"1min":7.8600000000000003,"5min":4.2569999999999997,"15min":5.2229999999999999},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.35699999999999998},"max":{"1min":94.400000000000006,"5min":94.400000000000006,"15min":230.69399999999999},"last":0.36799999999999999},{"interface":"front","average":{"1min":8.6210000000000004,"5min":4.5999999999999996,"15min":5.4000000000000004},"min":{"1min":0.52600000000000002,"5min":0.44500000000000001,"15min":0.32600000000000001},"max":{"1min":94.195999999999998,"5min":94.195999999999998,"15min":230.67500000000001},"last":0.41699999999999998}]}]},{"osd":4,"up_from":322,"seq":1382979469468,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3480120,"kb_used_data":3145456,"kb_used_omap":35,"kb_used_meta":334620,"kb_avail":90265032,"statfs":{"total":95995035648,"available":92431392768,"internally_reserved":0,"allocated":3220946944,"data_stored":3219408803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36552,"internal_metadata":342651192},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:31 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.3500000000000001,"15min":1.4419999999999999},"min":{"1min":0.222,"5min":0.216,"15min":0.216},"max":{"1min":4.6840000000000002,"5min":6.8470000000000004,"15min":6.8470000000000004},"last":0.254},{"interface":"front","average":{"1min":1.504,"5min":1.4159999999999999,"15min":1.536},"min":{"1min":0.26000000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":8.7699999999999996,"5min":8.7699999999999996,"15min":8.7699999999999996},"last":0.36399999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":1.778,"5min":1.883,"15min":2.2839999999999998},"min":{"1min":0.28499999999999998,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.3449999999999998,"5min":18.884,"15min":78.891000000000005},"last":2.1680000000000001},{"interface":"front","average":{"1min":1.4390000000000001,"5min":1.8300000000000001,"15min":2.2309999999999999},"min":{"1min":0.25600000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.946999999999999,"5min":18.617999999999999,"15min":78.921000000000006},"last":1.798}]},{"osd":2,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":14.574999999999999,"5min":4.4219999999999997,"15min":2.613},"min":{"1min":0.32500000000000001,"5min":0.29199999999999998,"15min":0.27200000000000002},"max":{"1min":245.166,"5min":245.166,"15min":245.166},"last":5.2350000000000003},{"interface":"front","average":{"1min":14.550000000000001,"5min":4.2469999999999999,"15min":2.5600000000000001},"min":{"1min":0.38500000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":245.119,"5min":245.119,"15min":245.119},"last":1.46}]},{"osd":3,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":24.687999999999999,"5min":6.6550000000000002,"15min":3.875},"min":{"1min":0.438,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":246.39400000000001,"5min":246.39400000000001,"15min":246.39400000000001},"last":0.437},{"interface":"front","average":{"1min":23.437999999999999,"5min":6.21,"15min":3.444},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":246.99000000000001,"5min":246.99000000000001,"15min":246.99000000000001},"last":5.3810000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.93799999999999994,"15min":0.98999999999999999},"min":{"1min":0.27900000000000003,"5min":0.216,"15min":0.216},"max":{"1min":2.5600000000000001,"5min":7.5999999999999996,"15min":9.2690000000000001},"last":0.28899999999999998},{"interface":"front","average":{"1min":1.1719999999999999,"5min":0.94099999999999995,"15min":1.165},"min":{"1min":0.33700000000000002,"5min":0.245,"15min":0.245},"max":{"1min":5.3479999999999999,"5min":9.1110000000000007,"15min":12.087},"last":0.34399999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":1.036,"15min":2.3010000000000002},"min":{"1min":0.32100000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.4279999999999999,"5min":7.641,"15min":95.492999999999995},"last":0.32200000000000001},{"interface":"front","average":{"1min":1.1759999999999999,"5min":1.1990000000000001,"15min":2.4390000000000001},"min":{"1min":0.372,"5min":0.312,"15min":0.27900000000000003},"max":{"1min":5.5469999999999997,"5min":9.3239999999999998,"15min":95.664000000000001},"last":0.39700000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":1.3069999999999999,"15min":2.6920000000000002},"min":{"1min":0.38,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":8.1229999999999993,"5min":17.286000000000001,"15min":79.173000000000002},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.151,"15min":2.294},"min":{"1min":0.33000000000000002,"5min":0.251,"15min":0.251},"max":{"1min":5.1779999999999999,"5min":16.097999999999999,"15min":78.894000000000005},"last":0.38100000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757688,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2433832,"kb_used_data":1535264,"kb_used_omap":0,"kb_used_meta":898496,"kb_avail":91311320,"statfs":{"total":95995035648,"available":93502791680,"internally_reserved":0,"allocated":1572110336,"data_stored":1570584843,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920059904},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:33 2023","interfaces":[{"interface":"back","average":{"1min":8.1929999999999996,"5min":3.508,"15min":2.77},"min":{"1min":0.24399999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":115.92400000000001,"5min":115.92400000000001,"15min":115.92400000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":8.4220000000000006,"5min":3.855,"15min":3.4340000000000002},"min":{"1min":0.29299999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":115.904,"5min":115.904,"15min":115.904},"last":0.41699999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:14:03 2023","interfaces":[{"interface":"back","average":{"1min":1.3560000000000001,"5min":4.6040000000000001,"15min":4.4950000000000001},"min":{"1min":0.26500000000000001,"5min":0.248,"15min":0.22600000000000001},"max":{"1min":5.4880000000000004,"5min":151.78200000000001,"15min":151.78200000000001},"last":0.83699999999999997},{"interface":"front","average":{"1min":1.9990000000000001,"5min":4.6900000000000004,"15min":4.6059999999999999},"min":{"1min":0.32300000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":9.7750000000000004,"5min":152.251,"15min":152.251},"last":0.504}]},{"osd":2,"last update":"Mon Oct 9 06:13:27 2023","interfaces":[{"interface":"back","average":{"1min":9.2119999999999997,"5min":3.1019999999999999,"15min":2.9119999999999999},"min":{"1min":0.29799999999999999,"5min":0.26600000000000001,"15min":0.254},"max":{"1min":117.97199999999999,"5min":117.97199999999999,"15min":129.21199999999999},"last":0.86199999999999999},{"interface":"front","average":{"1min":9.6690000000000005,"5min":3.4500000000000002,"15min":2.9870000000000001},"min":{"1min":0.30599999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":118.52,"5min":118.52,"15min":129.042},"last":0.36399999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:14:03 2023","interfaces":[{"interface":"back","average":{"1min":9.4779999999999998,"5min":6.5380000000000003,"15min":6.9169999999999998},"min":{"1min":0.316,"5min":0.30299999999999999,"15min":0.29499999999999998},"max":{"1min":93.338999999999999,"5min":143.31200000000001,"15min":143.31200000000001},"last":1.2949999999999999},{"interface":"front","average":{"1min":9.6940000000000008,"5min":7.1029999999999998,"15min":6.9989999999999997},"min":{"1min":0.40300000000000002,"5min":0.34499999999999997,"15min":0.29399999999999998},"max":{"1min":93.435000000000002,"5min":140.00899999999999,"15min":140.00899999999999},"last":0.88600000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:13:52 2023","interfaces":[{"interface":"back","average":{"1min":4.5039999999999996,"5min":1.8360000000000001,"15min":1.298},"min":{"1min":0.26500000000000001,"5min":0.246,"15min":0.245},"max":{"1min":68.001999999999995,"5min":68.001999999999995,"15min":68.001999999999995},"last":0.32500000000000001},{"interface":"front","average":{"1min":4.2880000000000003,"5min":1.5529999999999999,"15min":1.228},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26000000000000001},"max":{"1min":61.743000000000002,"5min":61.743000000000002,"15min":61.743000000000002},"last":0.379}]},{"osd":6,"last update":"Mon Oct 9 06:14:03 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":1.0149999999999999,"15min":1.48},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.251},"max":{"1min":2.3439999999999999,"5min":7.4980000000000002,"15min":126.89400000000001},"last":0.442},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.099,"15min":1.5620000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.26700000000000002},"max":{"1min":2.4340000000000002,"5min":7.7140000000000004,"15min":126.616},"last":0.75900000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:13:52 2023","interfaces":[{"interface":"back","average":{"1min":4.5670000000000002,"5min":3.2440000000000002,"15min":2.7999999999999998},"min":{"1min":0.40000000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":64.766000000000005,"5min":157.471,"15min":157.471},"last":0.81000000000000005},{"interface":"front","average":{"1min":4.7830000000000004,"5min":3.2519999999999998,"15min":2.7189999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.29599999999999999},"max":{"1min":67.956999999999994,"5min":157.51599999999999,"15min":157.51599999999999},"last":0.58399999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4889,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2872840192,"data_stored":2872836120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":778764288,"data_stored":778764288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1793785856,"data_stored":1793785856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2132283392,"data_stored":2132279320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3192459264,"data_stored":3192455192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1566441472,"data_stored":1566441472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1269436416,"data_stored":1269432344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":704446464,"data_stored":704446464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:14:23.751 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:14:23.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:23.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:14:23.861 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:14:23.907 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:23.978 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:14:23.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:23.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:14:24.084 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:24.085 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:14:24.131 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:24.200 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:24.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:24.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:24.308 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:14:24.407 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:24.553 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:14:24.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:24.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:14:24.666 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:14:24.727 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:14:24.760 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:14:24.760 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:14:24.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.760 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.46283.0:134386 10.0 e544/522)", 2023-10-09T06:14:24.760 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:24.742918+0000", 2023-10-09T06:14:24.760 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.00031459000000000001, 2023-10-09T06:14:24.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00034035599999999999, 2023-10-09T06:14:24.761 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:14:24.761 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-10-09T06:14:24.761 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:14:24.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:14:24.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.742918+0000", 2023-10-09T06:14:24.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:14:24.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:14:24.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.742918+0000", 2023-10-09T06:14:24.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:14:24.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:14:24.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.742918+0000", 2023-10-09T06:14:24.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:14:24.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:14:24.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.742920+0000", 2023-10-09T06:14:24.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3499999999999999e-06 2023-10-09T06:14:24.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:14:24.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.742926+0000", 2023-10-09T06:14:24.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2870000000000004e-06 2023-10-09T06:14:24.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:14:24.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.742930+0000", 2023-10-09T06:14:24.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.172e-06 2023-10-09T06:14:24.763 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:14:24.763 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:14:24.764 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:14:24.764 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.764 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPushReply(10.0 544/522 [PushReplyOp(10:0e65fbb1:::benchmark_data_smithi064_573750_object42664:head)])", 2023-10-09T06:14:24.764 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:24.742949+0000", 2023-10-09T06:14:24.764 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.00028414800000000002, 2023-10-09T06:14:24.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00036643599999999999, 2023-10-09T06:14:24.764 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:14:24.764 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-10-09T06:14:24.764 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:14:24.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:14:24.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.742949+0000", 2023-10-09T06:14:24.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:14:24.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:14:24.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.742949+0000", 2023-10-09T06:14:24.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:14:24.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:14:24.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.742948+0000", 2023-10-09T06:14:24.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:14:24.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:14:24.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.742956+0000", 2023-10-09T06:14:24.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2010000000000002e-06 2023-10-09T06:14:24.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:14:24.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.742964+0000", 2023-10-09T06:14:24.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.816e-06 2023-10-09T06:14:24.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:14:24.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.742970+0000", 2023-10-09T06:14:24.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6919999999999999e-06 2023-10-09T06:14:24.767 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:14:24.767 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:14:24.767 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:14:24.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.767 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.46283.0:134386 10.0 e544/522)", 2023-10-09T06:14:24.767 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:24.742976+0000", 2023-10-09T06:14:24.768 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.000257347, 2023-10-09T06:14:24.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00039057000000000002, 2023-10-09T06:14:24.768 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:14:24.768 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-10-09T06:14:24.768 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:14:24.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.768 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:14:24.768 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.742976+0000", 2023-10-09T06:14:24.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:14:24.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.768 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:14:24.769 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.742976+0000", 2023-10-09T06:14:24.769 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:14:24.769 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.769 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.769 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:14:24.769 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.742975+0000", 2023-10-09T06:14:24.769 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967296 2023-10-09T06:14:24.769 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.769 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.769 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:14:24.769 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.742977+0000", 2023-10-09T06:14:24.769 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0789999999999999e-06 2023-10-09T06:14:24.769 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.770 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:14:24.770 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.742982+0000", 2023-10-09T06:14:24.770 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8940000000000001e-06 2023-10-09T06:14:24.770 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.770 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:14:24.770 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.742986+0000", 2023-10-09T06:14:24.770 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9790000000000004e-06 2023-10-09T06:14:24.770 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:14:24.770 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:14:24.771 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:14:24.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.771 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.771 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:134385 10.b 10:dc99be0a:::benchmark_data_smithi064_573750_object38170:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-10-09T06:14:24.771 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:24.732497+0000", 2023-10-09T06:14:24.771 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.010735586, 2023-10-09T06:14:24.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010927052, 2023-10-09T06:14:24.771 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:14:24.771 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:14:24.771 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:14:24.771 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:14:24.771 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:14:24.772 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 134385 2023-10-09T06:14:24.772 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.772 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:14:24.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:14:24.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.732497+0000", 2023-10-09T06:14:24.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:14:24.772 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:14:24.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.732497+0000", 2023-10-09T06:14:24.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:14:24.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.773 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:14:24.773 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.732496+0000", 2023-10-09T06:14:24.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:14:24.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.773 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:14:24.773 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.732502+0000", 2023-10-09T06:14:24.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0140000000000004e-06 2023-10-09T06:14:24.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:14:24.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.732513+0000", 2023-10-09T06:14:24.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1055e-05 2023-10-09T06:14:24.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.774 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:14:24.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.732520+0000", 2023-10-09T06:14:24.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2969999999999998e-06 2023-10-09T06:14:24.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.774 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:14:24.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.732609+0000", 2023-10-09T06:14:24.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8739000000000005e-05 2023-10-09T06:14:24.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:14:24.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.732738+0000", 2023-10-09T06:14:24.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012925299999999999 2023-10-09T06:14:24.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,6", 2023-10-09T06:14:24.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.732880+0000", 2023-10-09T06:14:24.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014133 2023-10-09T06:14:24.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:14:24.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.736613+0000", 2023-10-09T06:14:24.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037327739999999999 2023-10-09T06:14:24.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:14:24.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.742979+0000", 2023-10-09T06:14:24.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063665220000000003 2023-10-09T06:14:24.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:14:24.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.743058+0000", 2023-10-09T06:14:24.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8653999999999996e-05 2023-10-09T06:14:24.777 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:14:24.777 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:14:24.777 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:14:24.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.777 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:134386 10.0 10:0bb68580:::benchmark_data_smithi064_573750_object38171:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-10-09T06:14:24.777 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:24.732534+0000", 2023-10-09T06:14:24.777 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.010698696000000001, 2023-10-09T06:14:24.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010991522, 2023-10-09T06:14:24.777 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:14:24.778 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:14:24.778 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:14:24.778 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:14:24.778 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:14:24.778 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 134386 2023-10-09T06:14:24.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.778 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:14:24.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:14:24.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.732534+0000", 2023-10-09T06:14:24.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:14:24.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:14:24.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.732534+0000", 2023-10-09T06:14:24.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:14:24.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:14:24.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.732534+0000", 2023-10-09T06:14:24.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:14:24.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:14:24.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.732537+0000", 2023-10-09T06:14:24.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7250000000000002e-06 2023-10-09T06:14:24.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:14:24.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.732542+0000", 2023-10-09T06:14:24.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9739999999999999e-06 2023-10-09T06:14:24.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:14:24.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.732547+0000", 2023-10-09T06:14:24.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9910000000000002e-06 2023-10-09T06:14:24.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:14:24.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.737023+0000", 2023-10-09T06:14:24.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044753869999999999 2023-10-09T06:14:24.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:14:24.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.737159+0000", 2023-10-09T06:14:24.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013597699999999999 2023-10-09T06:14:24.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,6", 2023-10-09T06:14:24.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.737261+0000", 2023-10-09T06:14:24.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010171900000000001 2023-10-09T06:14:24.782 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:14:24.782 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:14:24.782 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:14:24.782 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.782 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:134395 10.b 10:df1519c8:::benchmark_data_smithi064_573750_object38180:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-10-09T06:14:24.782 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:24.738853+0000", 2023-10-09T06:14:24.782 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0043803369999999998, 2023-10-09T06:14:24.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047537400000000002, 2023-10-09T06:14:24.783 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:14:24.783 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:14:24.783 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:14:24.783 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:14:24.783 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:14:24.783 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 134395 2023-10-09T06:14:24.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.783 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:14:24.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.783 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:14:24.783 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.738853+0000", 2023-10-09T06:14:24.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:14:24.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.784 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:14:24.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.738853+0000", 2023-10-09T06:14:24.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:14:24.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.784 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:14:24.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.738851+0000", 2023-10-09T06:14:24.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:14:24.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.785 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:14:24.785 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.738858+0000", 2023-10-09T06:14:24.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5290000000000003e-06 2023-10-09T06:14:24.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.785 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:14:24.785 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.738874+0000", 2023-10-09T06:14:24.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5248e-05 2023-10-09T06:14:24.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.785 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:14:24.785 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.738884+0000", 2023-10-09T06:14:24.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9729999999999992e-06 2023-10-09T06:14:24.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:14:24.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.738938+0000", 2023-10-09T06:14:24.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4333000000000003e-05 2023-10-09T06:14:24.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:14:24.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.739087+0000", 2023-10-09T06:14:24.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014888 2023-10-09T06:14:24.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:24.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:24.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,6", 2023-10-09T06:14:24.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:24.739239+0000", 2023-10-09T06:14:24.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015255100000000001 2023-10-09T06:14:24.787 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:14:24.787 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:14:24.787 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:14:24.787 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:14:24.787 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-10-09T06:14:24.787 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 6 2023-10-09T06:14:24.787 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:14:24.797 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:24.823 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:24 smithi064 ceph-mon[212750]: pgmap v1379: 119 pgs: 3 active+remapped+backfilling, 2 active+remapped+backfill_wait, 114 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 16310/182643 objects misplaced (8.930%); 25 MiB/s, 403 objects/s recovering 2023-10-09T06:14:24.823 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:24 smithi064 ceph-mon[220044]: pgmap v1379: 119 pgs: 3 active+remapped+backfilling, 2 active+remapped+backfill_wait, 114 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 16310/182643 objects misplaced (8.930%); 25 MiB/s, 403 objects/s recovering 2023-10-09T06:14:25.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:24 smithi156 ceph-mon[195101]: pgmap v1379: 119 pgs: 3 active+remapped+backfilling, 2 active+remapped+backfill_wait, 114 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 16310/182643 objects misplaced (8.930%); 25 MiB/s, 403 objects/s recovering 2023-10-09T06:14:25.346 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:14:25.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:25.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:14:25.452 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:14:25.519 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:14:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:14:25] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.43.0" 2023-10-09T06:14:25.533 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:26.321 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:14:26.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:26.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:14:26.434 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:14:26.493 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:26.576 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:14:26.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:26.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:14:26.655 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:14:26.682 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:14:26.683 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:14:26.772 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:26 smithi156 ceph-mon[195101]: pgmap v1380: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 478 op/s; 14432/181005 objects misplaced (7.973%); 15 MiB/s, 246 objects/s recovering 2023-10-09T06:14:26.870 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:14:26.896 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:26 smithi064 ceph-mon[212750]: pgmap v1380: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 478 op/s; 14432/181005 objects misplaced (7.973%); 15 MiB/s, 246 objects/s recovering 2023-10-09T06:14:26.897 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:26 smithi064 ceph-mon[220044]: pgmap v1380: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 478 op/s; 14432/181005 objects misplaced (7.973%); 15 MiB/s, 246 objects/s recovering 2023-10-09T06:14:26.956 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:27.222 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:14:27.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:27.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:14:27.331 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:14:27.424 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:27.751 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:14:27.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:27.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:14:27.857 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:14:27.890 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:27.951 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:14:27.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:27.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:14:28.057 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:28.085 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:28.131 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:28.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:28.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:28.236 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:14:28.264 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:28.310 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:14:28.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:28.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:14:28.415 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:14:28.443 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:28.490 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:14:28.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:28.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:14:28.595 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:14:28.629 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:28.726 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:14:28.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:28 smithi064 ceph-mon[212750]: pgmap v1381: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 12118/171630 objects misplaced (7.061%); 27 MiB/s, 436 objects/s recovering 2023-10-09T06:14:28.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:28 smithi064 ceph-mon[212750]: osdmap e545: 8 total, 8 up, 8 in 2023-10-09T06:14:28.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:28 smithi064 ceph-mon[212750]: failed to encode map e545 with expected crc 2023-10-09T06:14:28.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:28 smithi064 ceph-mon[212750]: failed to encode map e545 with expected crc 2023-10-09T06:14:28.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:28 smithi064 ceph-mon[212750]: failed to encode map e545 with expected crc 2023-10-09T06:14:28.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:28 smithi064 ceph-mon[212750]: failed to encode map e545 with expected crc 2023-10-09T06:14:28.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:28 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1843467424' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:14:28.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:28 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:28.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:28 smithi064 ceph-mon[220044]: pgmap v1381: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 12118/171630 objects misplaced (7.061%); 27 MiB/s, 436 objects/s recovering 2023-10-09T06:14:28.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:28 smithi064 ceph-mon[220044]: osdmap e545: 8 total, 8 up, 8 in 2023-10-09T06:14:28.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:28 smithi064 ceph-mon[220044]: failed to encode map e545 with expected crc 2023-10-09T06:14:28.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:28 smithi064 ceph-mon[220044]: failed to encode map e545 with expected crc 2023-10-09T06:14:28.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:28 smithi064 ceph-mon[220044]: failed to encode map e545 with expected crc 2023-10-09T06:14:28.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:28 smithi064 ceph-mon[220044]: failed to encode map e545 with expected crc 2023-10-09T06:14:28.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:28 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1843467424' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:14:28.975 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:28 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:28.981 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:14:28.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:28.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:14:29.085 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:29.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:28 smithi156 ceph-mon[195101]: pgmap v1381: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 12118/171630 objects misplaced (7.061%); 27 MiB/s, 436 objects/s recovering 2023-10-09T06:14:29.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:28 smithi156 ceph-mon[195101]: osdmap e545: 8 total, 8 up, 8 in 2023-10-09T06:14:29.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:28 smithi156 ceph-mon[195101]: failed to encode map e545 with expected crc 2023-10-09T06:14:29.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:28 smithi156 ceph-mon[195101]: failed to encode map e545 with expected crc 2023-10-09T06:14:29.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:28 smithi156 ceph-mon[195101]: failed to encode map e545 with expected crc 2023-10-09T06:14:29.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:28 smithi156 ceph-mon[195101]: failed to encode map e545 with expected crc 2023-10-09T06:14:29.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:28 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1843467424' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:14:29.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:28 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:29.134 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:29.208 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:29.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:29.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:29.313 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:14:29.378 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:29.795 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:14:29.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:29.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:14:29.894 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:29 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:14:29.894 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:29 smithi064 ceph-mon[212750]: failed to encode map e546 with expected crc 2023-10-09T06:14:29.895 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:29 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1843467424' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:14:29.895 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:29 smithi064 ceph-mon[212750]: osdmap e546: 8 total, 8 up, 8 in 2023-10-09T06:14:29.895 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:29 smithi064 ceph-mon[212750]: failed to encode map e546 with expected crc 2023-10-09T06:14:29.895 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:29 smithi064 ceph-mon[212750]: failed to encode map e546 with expected crc 2023-10-09T06:14:29.895 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:29 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:14:29.896 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:29 smithi064 ceph-mon[220044]: failed to encode map e546 with expected crc 2023-10-09T06:14:29.896 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:29 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1843467424' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:14:29.896 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:29 smithi064 ceph-mon[220044]: osdmap e546: 8 total, 8 up, 8 in 2023-10-09T06:14:29.896 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:29 smithi064 ceph-mon[220044]: failed to encode map e546 with expected crc 2023-10-09T06:14:29.896 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:29 smithi064 ceph-mon[220044]: failed to encode map e546 with expected crc 2023-10-09T06:14:29.904 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:14:29.910 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:14:30.032 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:14:30.050 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:30.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:29 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:14:30.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:29 smithi156 ceph-mon[195101]: failed to encode map e546 with expected crc 2023-10-09T06:14:30.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:29 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1843467424' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:14:30.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:29 smithi156 ceph-mon[195101]: osdmap e546: 8 total, 8 up, 8 in 2023-10-09T06:14:30.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:29 smithi156 ceph-mon[195101]: failed to encode map e546 with expected crc 2023-10-09T06:14:30.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:29 smithi156 ceph-mon[195101]: failed to encode map e546 with expected crc 2023-10-09T06:14:30.186 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:14:30.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:30.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:14:30.294 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:14:30.350 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:30.422 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:14:30.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:30.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:14:30.528 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:14:30.618 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:14:30.622 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:30.685 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:14:30.716 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:14:30.747 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:14:30.756 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:14:30.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:30.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:14:30.796 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:14:30.837 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:14:30.863 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:14:30.877 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:14:30.934 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:14:30.949 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:30 smithi064 ceph-mon[212750]: pgmap v1384: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 12118/171630 objects misplaced (7.061%); 20 MiB/s, 326 objects/s recovering 2023-10-09T06:14:30.949 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:30 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4285430912' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:14:30.949 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:30 smithi064 ceph-mon[220044]: pgmap v1384: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 12118/171630 objects misplaced (7.061%); 20 MiB/s, 326 objects/s recovering 2023-10-09T06:14:30.950 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:30 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4285430912' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:14:30.959 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:14:30.959 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:14:30.959 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:14:30.959 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:14:30.960 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:14:30.960 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:14:30.960 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:14:30.960 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:14:30.977 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:31.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:30 smithi156 ceph-mon[195101]: pgmap v1384: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 12118/171630 objects misplaced (7.061%); 20 MiB/s, 326 objects/s recovering 2023-10-09T06:14:31.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:30 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4285430912' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:14:31.898 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:14:32.250 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:14:32.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:32.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:14:32.360 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:14:32.406 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:33.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:32 smithi156 ceph-mon[195101]: pgmap v1385: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 11775/166557 objects misplaced (7.070%); 19 MiB/s, 303 objects/s recovering 2023-10-09T06:14:33.190 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:14:33.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:32 smithi064 ceph-mon[212750]: pgmap v1385: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 11775/166557 objects misplaced (7.070%); 19 MiB/s, 303 objects/s recovering 2023-10-09T06:14:33.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:32 smithi064 ceph-mon[220044]: pgmap v1385: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 11775/166557 objects misplaced (7.070%); 19 MiB/s, 303 objects/s recovering 2023-10-09T06:14:33.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:33.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:14:33.294 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:14:33.341 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:33.416 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:14:33.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:33.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:14:33.527 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:14:33.569 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:33.637 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:14:33.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:33.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:14:33.744 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:14:33.820 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:33.944 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:14:33.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:33.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:14:34.051 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:14:34.156 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:34.290 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:14:34.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:34.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:14:34.399 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:14:34.433 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:34.480 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:14:34.501 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:14:34.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:34.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:14:34.557 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:14:34.557 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:14:34.610 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:14:34.643 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:34.678 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:14:34.698 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T06:14:34.723 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:14:34.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:34.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:14:34.829 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:34.885 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:34.896 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:34 smithi064 ceph-mon[212750]: pgmap v1386: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 116 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 9480/162213 objects misplaced (5.844%); 28 MiB/s, 445 objects/s recovering 2023-10-09T06:14:34.900 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:34 smithi064 ceph-mon[220044]: pgmap v1386: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 116 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 9480/162213 objects misplaced (5.844%); 28 MiB/s, 445 objects/s recovering 2023-10-09T06:14:34.986 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:34.992 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:34 smithi156 ceph-mon[195101]: pgmap v1386: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 116 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 9480/162213 objects misplaced (5.844%); 28 MiB/s, 445 objects/s recovering 2023-10-09T06:14:34.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:34.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:35.095 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:14:35.192 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:35.472 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:14:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:14:35] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.43.0" 2023-10-09T06:14:35.567 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:14:35.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:35.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:14:35.610 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:14:35.678 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:35.723 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:35.786 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:35.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:35.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:35.893 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:14:35.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:35 smithi064 ceph-mon[220044]: from='client.46469 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:14:35.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:35 smithi064 ceph-mon[212750]: from='client.46469 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:14:35.989 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:36.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:35 smithi156 ceph-mon[195101]: from='client.46469 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:14:36.729 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:14:36.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:36.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:14:36.772 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:14:36.833 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1386,"stamp":"2023-10-09T06:14:33.188478+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3542581387,"num_objects":54071,"num_object_clones":0,"num_object_copies":162213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":9480,"num_objects_unfound":0,"num_objects_dirty":54071,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":138451,"num_write_kb":6157708,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37464,"num_bytes_recovered":2452647603,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102072,"ondisk_log_size":102072,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":368,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17982984,"kb_used_data":12400520,"kb_used_omap":165,"kb_used_meta":5582106,"kb_avail":731978232,"statfs":{"total":767960285184,"available":749545709568,"internally_reserved":0,"allocated":12698132480,"data_stored":12679988425,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":169911,"internal_metadata":5716076617},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,0,3,1,1,0,0,4,12],"upper_bound":1024},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-410517504,"num_objects":-6264,"num_object_clones":0,"num_object_copies":-18792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-4952,"num_objects_unfound":0,"num_objects_dirty":-6264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6264,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3572,"num_bytes_recovered":234094592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.012802"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":785,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.714113+0000","last_change":"2023-10-09T06:13:33.699246+0000","last_active":"2023-10-09T06:14:27.714113+0000","last_peered":"2023-10-09T06:14:27.714113+0000","last_clean":"2023-10-09T06:14:27.714113+0000","last_became_active":"2023-10-09T06:13:33.698733+0000","last_became_peered":"2023-10-09T06:13:33.698733+0000","last_unstale":"2023-10-09T06:14:27.714113+0000","last_undegraded":"2023-10-09T06:14:27.714113+0000","last_fullsized":"2023-10-09T06:14:27.714113+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:05.412941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:05.412941+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:57.044673+0000","scrub_duration":0.0061014329999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":614,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.717797+0000","last_change":"2023-10-09T06:07:43.572773+0000","last_active":"2023-10-09T06:14:27.717797+0000","last_peered":"2023-10-09T06:14:27.717797+0000","last_clean":"2023-10-09T06:14:27.717797+0000","last_became_active":"2023-10-09T06:07:43.572299+0000","last_became_peered":"2023-10-09T06:07:43.572299+0000","last_unstale":"2023-10-09T06:14:27.717797+0000","last_undegraded":"2023-10-09T06:14:27.717797+0000","last_fullsized":"2023-10-09T06:14:27.717797+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:00:37.672435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":356,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.725259+0000","last_change":"2023-10-09T06:05:11.451972+0000","last_active":"2023-10-09T06:14:27.725259+0000","last_peered":"2023-10-09T06:14:27.725259+0000","last_clean":"2023-10-09T06:14:27.725259+0000","last_became_active":"2023-10-09T06:05:11.451448+0000","last_became_peered":"2023-10-09T06:05:11.451448+0000","last_unstale":"2023-10-09T06:14:27.725259+0000","last_undegraded":"2023-10-09T06:14:27.725259+0000","last_fullsized":"2023-10-09T06:14:27.725259+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:39.183621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":845,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.718155+0000","last_change":"2023-10-09T06:09:27.554162+0000","last_active":"2023-10-09T06:14:27.718155+0000","last_peered":"2023-10-09T06:14:27.718155+0000","last_clean":"2023-10-09T06:14:27.718155+0000","last_became_active":"2023-10-09T06:09:27.553659+0000","last_became_peered":"2023-10-09T06:09:27.553659+0000","last_unstale":"2023-10-09T06:14:27.718155+0000","last_undegraded":"2023-10-09T06:14:27.718155+0000","last_fullsized":"2023-10-09T06:14:27.718155+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:01.468346+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:01.468346+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:03.443511+0000","scrub_duration":0.0016258220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":228,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.710862+0000","last_change":"2023-10-09T06:01:54.762060+0000","last_active":"2023-10-09T06:14:27.710862+0000","last_peered":"2023-10-09T06:14:27.710862+0000","last_clean":"2023-10-09T06:14:27.710862+0000","last_became_active":"2023-10-09T06:01:54.761757+0000","last_became_peered":"2023-10-09T06:01:54.761757+0000","last_unstale":"2023-10-09T06:14:27.710862+0000","last_undegraded":"2023-10-09T06:14:27.710862+0000","last_fullsized":"2023-10-09T06:14:27.710862+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:40.294232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":29,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.714570+0000","last_change":"2023-10-09T06:13:30.789834+0000","last_active":"2023-10-09T06:14:27.714570+0000","last_peered":"2023-10-09T06:14:27.714570+0000","last_clean":"2023-10-09T06:14:27.714570+0000","last_became_active":"2023-10-09T06:13:30.789517+0000","last_became_peered":"2023-10-09T06:13:30.789517+0000","last_unstale":"2023-10-09T06:14:27.714570+0000","last_undegraded":"2023-10-09T06:14:27.714570+0000","last_fullsized":"2023-10-09T06:14:27.714570+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:45.467922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":830,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.717528+0000","last_change":"2023-10-09T06:12:07.739121+0000","last_active":"2023-10-09T06:14:27.717528+0000","last_peered":"2023-10-09T06:14:27.717528+0000","last_clean":"2023-10-09T06:14:27.717528+0000","last_became_active":"2023-10-09T06:12:07.738739+0000","last_became_peered":"2023-10-09T06:12:07.738739+0000","last_unstale":"2023-10-09T06:14:27.717528+0000","last_undegraded":"2023-10-09T06:14:27.717528+0000","last_fullsized":"2023-10-09T06:14:27.717528+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.293426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.293426+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:19.815422+0000","scrub_duration":0.001587946,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":339,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.720660+0000","last_change":"2023-10-09T06:09:27.564377+0000","last_active":"2023-10-09T06:14:27.720660+0000","last_peered":"2023-10-09T06:14:27.720660+0000","last_clean":"2023-10-09T06:14:27.720660+0000","last_became_active":"2023-10-09T06:09:27.563710+0000","last_became_peered":"2023-10-09T06:09:27.563710+0000","last_unstale":"2023-10-09T06:14:27.720660+0000","last_undegraded":"2023-10-09T06:14:27.720660+0000","last_fullsized":"2023-10-09T06:14:27.720660+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:48.168447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":462,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.712882+0000","last_change":"2023-10-09T06:13:21.727677+0000","last_active":"2023-10-09T06:14:27.712882+0000","last_peered":"2023-10-09T06:14:27.712882+0000","last_clean":"2023-10-09T06:14:27.712882+0000","last_became_active":"2023-10-09T06:13:21.727321+0000","last_became_peered":"2023-10-09T06:13:21.727321+0000","last_unstale":"2023-10-09T06:14:27.712882+0000","last_undegraded":"2023-10-09T06:14:27.712882+0000","last_fullsized":"2023-10-09T06:14:27.712882+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:10.146447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":764,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.714325+0000","last_change":"2023-10-09T06:13:33.701687+0000","last_active":"2023-10-09T06:14:27.714325+0000","last_peered":"2023-10-09T06:14:27.714325+0000","last_clean":"2023-10-09T06:14:27.714325+0000","last_became_active":"2023-10-09T06:13:33.701233+0000","last_became_peered":"2023-10-09T06:13:33.701233+0000","last_unstale":"2023-10-09T06:14:27.714325+0000","last_undegraded":"2023-10-09T06:14:27.714325+0000","last_fullsized":"2023-10-09T06:14:27.714325+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.384407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.384407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:04.887826+0000","scrub_duration":0.0040651960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":444,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.728036+0000","last_change":"2023-10-09T06:09:27.559756+0000","last_active":"2023-10-09T06:14:27.728036+0000","last_peered":"2023-10-09T06:14:27.728036+0000","last_clean":"2023-10-09T06:14:27.728036+0000","last_became_active":"2023-10-09T06:09:27.559443+0000","last_became_peered":"2023-10-09T06:09:27.559443+0000","last_unstale":"2023-10-09T06:14:27.728036+0000","last_undegraded":"2023-10-09T06:14:27.728036+0000","last_fullsized":"2023-10-09T06:14:27.728036+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:14.519398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":375,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.720737+0000","last_change":"2023-10-09T06:13:21.737897+0000","last_active":"2023-10-09T06:14:27.720737+0000","last_peered":"2023-10-09T06:14:27.720737+0000","last_clean":"2023-10-09T06:14:27.720737+0000","last_became_active":"2023-10-09T06:13:21.737564+0000","last_became_peered":"2023-10-09T06:13:21.737564+0000","last_unstale":"2023-10-09T06:14:27.720737+0000","last_undegraded":"2023-10-09T06:14:27.720737+0000","last_fullsized":"2023-10-09T06:14:27.720737+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:34.863320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":670,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.714482+0000","last_change":"2023-10-09T06:10:11.729656+0000","last_active":"2023-10-09T06:14:27.714482+0000","last_peered":"2023-10-09T06:14:27.714482+0000","last_clean":"2023-10-09T06:14:27.714482+0000","last_became_active":"2023-10-09T06:10:11.728960+0000","last_became_peered":"2023-10-09T06:10:11.728960+0000","last_unstale":"2023-10-09T06:14:27.714482+0000","last_undegraded":"2023-10-09T06:14:27.714482+0000","last_fullsized":"2023-10-09T06:14:27.714482+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:20.742561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:20.742561+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:45.543709+0000","scrub_duration":0.0027005039999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":29,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.714490+0000","last_change":"2023-10-09T06:13:30.802749+0000","last_active":"2023-10-09T06:14:27.714490+0000","last_peered":"2023-10-09T06:14:27.714490+0000","last_clean":"2023-10-09T06:14:27.714490+0000","last_became_active":"2023-10-09T06:13:30.802449+0000","last_became_peered":"2023-10-09T06:13:30.802449+0000","last_unstale":"2023-10-09T06:14:27.714490+0000","last_undegraded":"2023-10-09T06:14:27.714490+0000","last_fullsized":"2023-10-09T06:14:27.714490+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:29.017181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":532,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.721540+0000","last_change":"2023-10-09T06:13:30.792445+0000","last_active":"2023-10-09T06:14:27.721540+0000","last_peered":"2023-10-09T06:14:27.721540+0000","last_clean":"2023-10-09T06:14:27.721540+0000","last_became_active":"2023-10-09T06:13:30.792116+0000","last_became_peered":"2023-10-09T06:13:30.792116+0000","last_unstale":"2023-10-09T06:14:27.721540+0000","last_undegraded":"2023-10-09T06:14:27.721540+0000","last_fullsized":"2023-10-09T06:14:27.721540+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:20:53.506293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":757,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.724894+0000","last_change":"2023-10-09T06:13:24.760843+0000","last_active":"2023-10-09T06:14:27.724894+0000","last_peered":"2023-10-09T06:14:27.724894+0000","last_clean":"2023-10-09T06:14:27.724894+0000","last_became_active":"2023-10-09T06:13:24.760333+0000","last_became_peered":"2023-10-09T06:13:24.760333+0000","last_unstale":"2023-10-09T06:14:27.724894+0000","last_undegraded":"2023-10-09T06:14:27.724894+0000","last_fullsized":"2023-10-09T06:14:27.724894+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:38.588641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:38.588641+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:30.660400+0000","scrub_duration":0.010116385,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":35,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.714285+0000","last_change":"2023-10-09T06:13:30.826588+0000","last_active":"2023-10-09T06:14:27.714285+0000","last_peered":"2023-10-09T06:14:27.714285+0000","last_clean":"2023-10-09T06:14:27.714285+0000","last_became_active":"2023-10-09T06:13:30.799199+0000","last_became_peered":"2023-10-09T06:13:30.799199+0000","last_unstale":"2023-10-09T06:14:27.714285+0000","last_undegraded":"2023-10-09T06:14:27.714285+0000","last_fullsized":"2023-10-09T06:14:27.714285+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:38.703864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":327,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.724441+0000","last_change":"2023-10-09T06:13:21.739214+0000","last_active":"2023-10-09T06:14:27.724441+0000","last_peered":"2023-10-09T06:14:27.724441+0000","last_clean":"2023-10-09T06:14:27.724441+0000","last_became_active":"2023-10-09T06:13:21.738951+0000","last_became_peered":"2023-10-09T06:13:21.738951+0000","last_unstale":"2023-10-09T06:14:27.724441+0000","last_undegraded":"2023-10-09T06:14:27.724441+0000","last_fullsized":"2023-10-09T06:14:27.724441+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:17.477241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":718,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.712823+0000","last_change":"2023-10-09T06:09:27.574336+0000","last_active":"2023-10-09T06:14:27.712823+0000","last_peered":"2023-10-09T06:14:27.712823+0000","last_clean":"2023-10-09T06:14:27.712823+0000","last_became_active":"2023-10-09T06:09:27.573974+0000","last_became_peered":"2023-10-09T06:09:27.573974+0000","last_unstale":"2023-10-09T06:14:27.712823+0000","last_undegraded":"2023-10-09T06:14:27.712823+0000","last_fullsized":"2023-10-09T06:14:27.712823+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.339121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.339121+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:45.908579+0000","scrub_duration":0.010474344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":580,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.717713+0000","last_change":"2023-10-09T06:13:30.812598+0000","last_active":"2023-10-09T06:14:27.717713+0000","last_peered":"2023-10-09T06:14:27.717713+0000","last_clean":"2023-10-09T06:14:27.717713+0000","last_became_active":"2023-10-09T06:13:30.791460+0000","last_became_peered":"2023-10-09T06:13:30.791460+0000","last_unstale":"2023-10-09T06:14:27.717713+0000","last_undegraded":"2023-10-09T06:14:27.717713+0000","last_fullsized":"2023-10-09T06:14:27.717713+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:28.046730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":558,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.718541+0000","last_change":"2023-10-09T06:01:54.764916+0000","last_active":"2023-10-09T06:14:27.718541+0000","last_peered":"2023-10-09T06:14:27.718541+0000","last_clean":"2023-10-09T06:14:27.718541+0000","last_became_active":"2023-10-09T06:01:54.764705+0000","last_became_peered":"2023-10-09T06:01:54.764705+0000","last_unstale":"2023-10-09T06:14:27.718541+0000","last_undegraded":"2023-10-09T06:14:27.718541+0000","last_fullsized":"2023-10-09T06:14:27.718541+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:12.527719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":774,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.720635+0000","last_change":"2023-10-09T06:13:15.425525+0000","last_active":"2023-10-09T06:14:27.720635+0000","last_peered":"2023-10-09T06:14:27.720635+0000","last_clean":"2023-10-09T06:14:27.720635+0000","last_became_active":"2023-10-09T06:13:15.425086+0000","last_became_peered":"2023-10-09T06:13:15.425086+0000","last_unstale":"2023-10-09T06:14:27.720635+0000","last_undegraded":"2023-10-09T06:14:27.720635+0000","last_fullsized":"2023-10-09T06:14:27.720635+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:42.900876+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:42.900876+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:46.042715+0000","scrub_duration":0.0014928649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":35,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.714148+0000","last_change":"2023-10-09T06:13:30.811420+0000","last_active":"2023-10-09T06:14:27.714148+0000","last_peered":"2023-10-09T06:14:27.714148+0000","last_clean":"2023-10-09T06:14:27.714148+0000","last_became_active":"2023-10-09T06:13:30.792835+0000","last_became_peered":"2023-10-09T06:13:30.792835+0000","last_unstale":"2023-10-09T06:14:27.714148+0000","last_undegraded":"2023-10-09T06:14:27.714148+0000","last_fullsized":"2023-10-09T06:14:27.714148+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:38.962645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":591,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.724159+0000","last_change":"2023-10-09T06:01:54.761558+0000","last_active":"2023-10-09T06:14:27.724159+0000","last_peered":"2023-10-09T06:14:27.724159+0000","last_clean":"2023-10-09T06:14:27.724159+0000","last_became_active":"2023-10-09T06:01:54.761251+0000","last_became_peered":"2023-10-09T06:01:54.761251+0000","last_unstale":"2023-10-09T06:14:27.724159+0000","last_undegraded":"2023-10-09T06:14:27.724159+0000","last_fullsized":"2023-10-09T06:14:27.724159+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:53.245267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":1049,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.714098+0000","last_change":"2023-10-09T06:13:24.754018+0000","last_active":"2023-10-09T06:14:27.714098+0000","last_peered":"2023-10-09T06:14:27.714098+0000","last_clean":"2023-10-09T06:14:27.714098+0000","last_became_active":"2023-10-09T06:13:24.753712+0000","last_became_peered":"2023-10-09T06:13:24.753712+0000","last_unstale":"2023-10-09T06:14:27.714098+0000","last_undegraded":"2023-10-09T06:14:27.714098+0000","last_fullsized":"2023-10-09T06:14:27.714098+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:18.490833+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":29,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.713944+0000","last_change":"2023-10-09T06:13:30.788160+0000","last_active":"2023-10-09T06:14:27.713944+0000","last_peered":"2023-10-09T06:14:27.713944+0000","last_clean":"2023-10-09T06:14:27.713944+0000","last_became_active":"2023-10-09T06:13:30.787724+0000","last_became_peered":"2023-10-09T06:13:30.787724+0000","last_unstale":"2023-10-09T06:14:27.713944+0000","last_undegraded":"2023-10-09T06:14:27.713944+0000","last_fullsized":"2023-10-09T06:14:27.713944+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:38.376770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":29,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.714572+0000","last_change":"2023-10-09T06:13:30.789225+0000","last_active":"2023-10-09T06:14:27.714572+0000","last_peered":"2023-10-09T06:14:27.714572+0000","last_clean":"2023-10-09T06:14:27.714572+0000","last_became_active":"2023-10-09T06:13:30.788869+0000","last_became_peered":"2023-10-09T06:13:30.788869+0000","last_unstale":"2023-10-09T06:14:27.714572+0000","last_undegraded":"2023-10-09T06:14:27.714572+0000","last_fullsized":"2023-10-09T06:14:27.714572+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:19.519966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":786,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.711847+0000","last_change":"2023-10-09T06:12:55.750923+0000","last_active":"2023-10-09T06:14:27.711847+0000","last_peered":"2023-10-09T06:14:27.711847+0000","last_clean":"2023-10-09T06:14:27.711847+0000","last_became_active":"2023-10-09T06:12:55.750521+0000","last_became_peered":"2023-10-09T06:12:55.750521+0000","last_unstale":"2023-10-09T06:14:27.711847+0000","last_undegraded":"2023-10-09T06:14:27.711847+0000","last_fullsized":"2023-10-09T06:14:27.711847+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.316697+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.316697+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:23.591439+0000","scrub_duration":0.001666561,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":399,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.714537+0000","last_change":"2023-10-09T06:01:22.337171+0000","last_active":"2023-10-09T06:14:27.714537+0000","last_peered":"2023-10-09T06:14:27.714537+0000","last_clean":"2023-10-09T06:14:27.714537+0000","last_became_active":"2023-10-09T06:01:22.335438+0000","last_became_peered":"2023-10-09T06:01:22.335438+0000","last_unstale":"2023-10-09T06:14:27.714537+0000","last_undegraded":"2023-10-09T06:14:27.714537+0000","last_fullsized":"2023-10-09T06:14:27.714537+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:25.036448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":508,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.725114+0000","last_change":"2023-10-09T06:13:30.788284+0000","last_active":"2023-10-09T06:14:27.725114+0000","last_peered":"2023-10-09T06:14:27.725114+0000","last_clean":"2023-10-09T06:14:27.725114+0000","last_became_active":"2023-10-09T06:13:30.787933+0000","last_became_peered":"2023-10-09T06:13:30.787933+0000","last_unstale":"2023-10-09T06:14:27.725114+0000","last_undegraded":"2023-10-09T06:14:27.725114+0000","last_fullsized":"2023-10-09T06:14:27.725114+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:10.735438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":836,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.711533+0000","last_change":"2023-10-09T06:13:24.753908+0000","last_active":"2023-10-09T06:14:27.711533+0000","last_peered":"2023-10-09T06:14:27.711533+0000","last_clean":"2023-10-09T06:14:27.711533+0000","last_became_active":"2023-10-09T06:13:24.753566+0000","last_became_peered":"2023-10-09T06:13:24.753566+0000","last_unstale":"2023-10-09T06:14:27.711533+0000","last_undegraded":"2023-10-09T06:14:27.711533+0000","last_fullsized":"2023-10-09T06:14:27.711533+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:40.388125+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:40.388125+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:35.101978+0000","scrub_duration":0.0055576990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":118,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.718940+0000","last_change":"2023-10-09T06:13:12.243605+0000","last_active":"2023-10-09T06:14:27.718940+0000","last_peered":"2023-10-09T06:14:27.718940+0000","last_clean":"2023-10-09T06:14:27.718940+0000","last_became_active":"2023-10-09T06:13:12.237061+0000","last_became_peered":"2023-10-09T06:13:12.237061+0000","last_unstale":"2023-10-09T06:14:27.718940+0000","last_undegraded":"2023-10-09T06:14:27.718940+0000","last_fullsized":"2023-10-09T06:14:27.718940+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:29:03.666031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":56,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.727744+0000","last_change":"2023-10-09T06:13:30.788733+0000","last_active":"2023-10-09T06:14:27.727744+0000","last_peered":"2023-10-09T06:14:27.727744+0000","last_clean":"2023-10-09T06:14:27.727744+0000","last_became_active":"2023-10-09T06:13:30.788353+0000","last_became_peered":"2023-10-09T06:13:30.788353+0000","last_unstale":"2023-10-09T06:14:27.727744+0000","last_undegraded":"2023-10-09T06:14:27.727744+0000","last_fullsized":"2023-10-09T06:14:27.727744+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:06.308426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"546'8747","reported_seq":8813,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T06:14:31.179256+0000","last_change":"2023-10-09T06:11:46.032384+0000","last_active":"2023-10-09T06:14:31.179256+0000","last_peered":"2023-10-09T06:14:31.179256+0000","last_clean":"2023-10-09T06:14:31.179256+0000","last_became_active":"2023-10-09T06:11:46.032091+0000","last_became_peered":"2023-10-09T06:11:46.032091+0000","last_unstale":"2023-10-09T06:14:31.179256+0000","last_undegraded":"2023-10-09T06:14:31.179256+0000","last_fullsized":"2023-10-09T06:14:31.179256+0000","mapping_epoch":490,"log_start":"504'2300","ondisk_log_start":"504'2300","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6447,"log_dups_size":0,"ondisk_log_size":6447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:31.117384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216596480,"num_objects":3305,"num_object_clones":0,"num_object_copies":9915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8747,"num_write_kb":385664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":727,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.714648+0000","last_change":"2023-10-09T06:13:33.701521+0000","last_active":"2023-10-09T06:14:27.714648+0000","last_peered":"2023-10-09T06:14:27.714648+0000","last_clean":"2023-10-09T06:14:27.714648+0000","last_became_active":"2023-10-09T06:13:33.700870+0000","last_became_peered":"2023-10-09T06:13:33.700870+0000","last_unstale":"2023-10-09T06:14:27.714648+0000","last_undegraded":"2023-10-09T06:14:27.714648+0000","last_fullsized":"2023-10-09T06:14:27.714648+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:10:59.422551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:10:59.422551+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:54.402494+0000","scrub_duration":0.0059716659999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":33,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.714516+0000","last_change":"2023-10-09T06:13:30.822903+0000","last_active":"2023-10-09T06:14:27.714516+0000","last_peered":"2023-10-09T06:14:27.714516+0000","last_clean":"2023-10-09T06:14:27.714516+0000","last_became_active":"2023-10-09T06:13:30.799013+0000","last_became_peered":"2023-10-09T06:13:30.799013+0000","last_unstale":"2023-10-09T06:14:27.714516+0000","last_undegraded":"2023-10-09T06:14:27.714516+0000","last_fullsized":"2023-10-09T06:14:27.714516+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:44.271821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":29,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.713843+0000","last_change":"2023-10-09T06:13:30.793531+0000","last_active":"2023-10-09T06:14:27.713843+0000","last_peered":"2023-10-09T06:14:27.713843+0000","last_clean":"2023-10-09T06:14:27.713843+0000","last_became_active":"2023-10-09T06:13:30.792689+0000","last_became_peered":"2023-10-09T06:13:30.792689+0000","last_unstale":"2023-10-09T06:14:27.713843+0000","last_undegraded":"2023-10-09T06:14:27.713843+0000","last_fullsized":"2023-10-09T06:14:27.713843+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:02.559186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"546'8674","reported_seq":10370,"reported_epoch":546,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:14:30.350749+0000","last_change":"2023-10-09T06:14:00.813784+0000","last_active":"2023-10-09T06:14:30.350749+0000","last_peered":"2023-10-09T06:14:30.350749+0000","last_clean":"2023-10-09T06:13:11.194737+0000","last_became_active":"2023-10-09T06:13:21.782116+0000","last_became_peered":"2023-10-09T06:13:21.782116+0000","last_unstale":"2023-10-09T06:14:30.350749+0000","last_undegraded":"2023-10-09T06:14:30.350749+0000","last_fullsized":"2023-10-09T06:14:30.350749+0000","mapping_epoch":521,"log_start":"504'2693","ondisk_log_start":"504'2693","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5981,"log_dups_size":0,"ondisk_log_size":5981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:00.656766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223346688,"num_objects":3408,"num_object_clones":0,"num_object_copies":10224,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6380,"num_objects_unfound":0,"num_objects_dirty":3408,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8674,"num_write_kb":386624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":801,"num_bytes_recovered":52494336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,2,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":3408}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":915,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.724075+0000","last_change":"2023-10-09T06:13:24.753866+0000","last_active":"2023-10-09T06:14:27.724075+0000","last_peered":"2023-10-09T06:14:27.724075+0000","last_clean":"2023-10-09T06:14:27.724075+0000","last_became_active":"2023-10-09T06:13:24.753432+0000","last_became_peered":"2023-10-09T06:13:24.753432+0000","last_unstale":"2023-10-09T06:14:27.724075+0000","last_undegraded":"2023-10-09T06:14:27.724075+0000","last_fullsized":"2023-10-09T06:14:27.724075+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:03.241335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:03.241335+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:19.854449+0000","scrub_duration":0.017780547000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":33,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.714405+0000","last_change":"2023-10-09T06:13:30.829763+0000","last_active":"2023-10-09T06:14:27.714405+0000","last_peered":"2023-10-09T06:14:27.714405+0000","last_clean":"2023-10-09T06:14:27.714405+0000","last_became_active":"2023-10-09T06:13:30.802486+0000","last_became_peered":"2023-10-09T06:13:30.802486+0000","last_unstale":"2023-10-09T06:14:27.714405+0000","last_undegraded":"2023-10-09T06:14:27.714405+0000","last_fullsized":"2023-10-09T06:14:27.714405+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:05.468205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":584,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.717617+0000","last_change":"2023-10-09T06:01:54.760367+0000","last_active":"2023-10-09T06:14:27.717617+0000","last_peered":"2023-10-09T06:14:27.717617+0000","last_clean":"2023-10-09T06:14:27.717617+0000","last_became_active":"2023-10-09T06:01:54.760087+0000","last_became_peered":"2023-10-09T06:01:54.760087+0000","last_unstale":"2023-10-09T06:14:27.717617+0000","last_undegraded":"2023-10-09T06:14:27.717617+0000","last_fullsized":"2023-10-09T06:14:27.717617+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:26.000104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"546'8543","reported_seq":8607,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T06:14:30.833331+0000","last_change":"2023-10-09T06:11:45.696810+0000","last_active":"2023-10-09T06:14:30.833331+0000","last_peered":"2023-10-09T06:14:30.833331+0000","last_clean":"2023-10-09T06:14:30.833331+0000","last_became_active":"2023-10-09T06:11:45.696476+0000","last_became_peered":"2023-10-09T06:11:45.696476+0000","last_unstale":"2023-10-09T06:14:30.833331+0000","last_undegraded":"2023-10-09T06:14:30.833331+0000","last_fullsized":"2023-10-09T06:14:30.833331+0000","mapping_epoch":490,"log_start":"505'3100","ondisk_log_start":"505'3100","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5443,"log_dups_size":0,"ondisk_log_size":5443,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:41:33.351567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222101504,"num_objects":3389,"num_object_clones":0,"num_object_copies":10167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8543,"num_write_kb":381824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":809,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.718212+0000","last_change":"2023-10-09T06:13:24.768958+0000","last_active":"2023-10-09T06:14:27.718212+0000","last_peered":"2023-10-09T06:14:27.718212+0000","last_clean":"2023-10-09T06:14:27.718212+0000","last_became_active":"2023-10-09T06:13:24.768643+0000","last_became_peered":"2023-10-09T06:13:24.768643+0000","last_unstale":"2023-10-09T06:14:27.718212+0000","last_undegraded":"2023-10-09T06:14:27.718212+0000","last_fullsized":"2023-10-09T06:14:27.718212+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:48.495739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:48.495739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:18:50.654354+0000","scrub_duration":0.0016345509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":33,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.714486+0000","last_change":"2023-10-09T06:13:30.814434+0000","last_active":"2023-10-09T06:14:27.714486+0000","last_peered":"2023-10-09T06:14:27.714486+0000","last_clean":"2023-10-09T06:14:27.714486+0000","last_became_active":"2023-10-09T06:13:30.793119+0000","last_became_peered":"2023-10-09T06:13:30.793119+0000","last_unstale":"2023-10-09T06:14:27.714486+0000","last_undegraded":"2023-10-09T06:14:27.714486+0000","last_fullsized":"2023-10-09T06:14:27.714486+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:34:31.724006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":570,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.711180+0000","last_change":"2023-10-09T06:13:30.792490+0000","last_active":"2023-10-09T06:14:27.711180+0000","last_peered":"2023-10-09T06:14:27.711180+0000","last_clean":"2023-10-09T06:14:27.711180+0000","last_became_active":"2023-10-09T06:13:30.792209+0000","last_became_peered":"2023-10-09T06:13:30.792209+0000","last_unstale":"2023-10-09T06:14:27.711180+0000","last_undegraded":"2023-10-09T06:14:27.711180+0000","last_fullsized":"2023-10-09T06:14:27.711180+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:14.208042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"546'8598","reported_seq":8664,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T06:14:31.262805+0000","last_change":"2023-10-09T06:11:46.031195+0000","last_active":"2023-10-09T06:14:31.262805+0000","last_peered":"2023-10-09T06:14:31.262805+0000","last_clean":"2023-10-09T06:14:31.262805+0000","last_became_active":"2023-10-09T06:11:46.030891+0000","last_became_peered":"2023-10-09T06:11:46.030891+0000","last_unstale":"2023-10-09T06:14:31.262805+0000","last_undegraded":"2023-10-09T06:14:31.262805+0000","last_fullsized":"2023-10-09T06:14:31.262805+0000","mapping_epoch":490,"log_start":"504'2200","ondisk_log_start":"504'2200","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6398,"log_dups_size":0,"ondisk_log_size":6398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:25.788661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217448448,"num_objects":3318,"num_object_clones":0,"num_object_copies":9954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8598,"num_write_kb":381312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":761,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.718138+0000","last_change":"2023-10-09T06:13:33.708291+0000","last_active":"2023-10-09T06:14:27.718138+0000","last_peered":"2023-10-09T06:14:27.718138+0000","last_clean":"2023-10-09T06:14:27.718138+0000","last_became_active":"2023-10-09T06:13:33.707930+0000","last_became_peered":"2023-10-09T06:13:33.707930+0000","last_unstale":"2023-10-09T06:14:27.718138+0000","last_undegraded":"2023-10-09T06:14:27.718138+0000","last_fullsized":"2023-10-09T06:14:27.718138+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:16.361097+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:16.361097+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:18.111404+0000","scrub_duration":0.028533749000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":315,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.724073+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T06:14:27.724073+0000","last_peered":"2023-10-09T06:14:27.724073+0000","last_clean":"2023-10-09T06:14:27.724073+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T06:14:27.724073+0000","last_undegraded":"2023-10-09T06:14:27.724073+0000","last_fullsized":"2023-10-09T06:14:27.724073+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":559,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.718833+0000","last_change":"2023-10-09T06:09:27.558353+0000","last_active":"2023-10-09T06:14:27.718833+0000","last_peered":"2023-10-09T06:14:27.718833+0000","last_clean":"2023-10-09T06:14:27.718833+0000","last_became_active":"2023-10-09T06:09:27.557233+0000","last_became_peered":"2023-10-09T06:09:27.557233+0000","last_unstale":"2023-10-09T06:14:27.718833+0000","last_undegraded":"2023-10-09T06:14:27.718833+0000","last_fullsized":"2023-10-09T06:14:27.718833+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:53.149350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"546'8628","reported_seq":8694,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T06:14:31.583252+0000","last_change":"2023-10-09T06:11:45.694648+0000","last_active":"2023-10-09T06:14:31.583252+0000","last_peered":"2023-10-09T06:14:31.583252+0000","last_clean":"2023-10-09T06:14:31.583252+0000","last_became_active":"2023-10-09T06:11:45.694321+0000","last_became_peered":"2023-10-09T06:11:45.694321+0000","last_unstale":"2023-10-09T06:14:31.583252+0000","last_undegraded":"2023-10-09T06:14:31.583252+0000","last_fullsized":"2023-10-09T06:14:31.583252+0000","mapping_epoch":490,"log_start":"504'2100","ondisk_log_start":"504'2100","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6528,"log_dups_size":0,"ondisk_log_size":6528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:43.704525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215875584,"num_objects":3294,"num_object_clones":0,"num_object_copies":9882,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3294,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8628,"num_write_kb":381504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":932,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.714009+0000","last_change":"2023-10-09T06:13:33.716191+0000","last_active":"2023-10-09T06:14:27.714009+0000","last_peered":"2023-10-09T06:14:27.714009+0000","last_clean":"2023-10-09T06:14:27.714009+0000","last_became_active":"2023-10-09T06:13:33.715831+0000","last_became_peered":"2023-10-09T06:13:33.715831+0000","last_unstale":"2023-10-09T06:14:27.714009+0000","last_undegraded":"2023-10-09T06:14:27.714009+0000","last_fullsized":"2023-10-09T06:14:27.714009+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:42.795781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:42.795781+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:45.712738+0000","scrub_duration":0.0091262979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":349,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.722948+0000","last_change":"2023-10-09T06:09:27.565666+0000","last_active":"2023-10-09T06:14:27.722948+0000","last_peered":"2023-10-09T06:14:27.722948+0000","last_clean":"2023-10-09T06:14:27.722948+0000","last_became_active":"2023-10-09T06:09:27.565324+0000","last_became_peered":"2023-10-09T06:09:27.565324+0000","last_unstale":"2023-10-09T06:14:27.722948+0000","last_undegraded":"2023-10-09T06:14:27.722948+0000","last_fullsized":"2023-10-09T06:14:27.722948+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:51.216947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":520,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.717725+0000","last_change":"2023-10-09T06:13:21.727745+0000","last_active":"2023-10-09T06:14:27.717725+0000","last_peered":"2023-10-09T06:14:27.717725+0000","last_clean":"2023-10-09T06:14:27.717725+0000","last_became_active":"2023-10-09T06:13:21.727404+0000","last_became_peered":"2023-10-09T06:13:21.727404+0000","last_unstale":"2023-10-09T06:14:27.717725+0000","last_undegraded":"2023-10-09T06:14:27.717725+0000","last_fullsized":"2023-10-09T06:14:27.717725+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:24.931626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"546'8747","reported_seq":20306,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T06:14:30.366660+0000","last_change":"2023-10-09T06:13:32.697002+0000","last_active":"2023-10-09T06:14:30.366660+0000","last_peered":"2023-10-09T06:14:30.366660+0000","last_clean":"2023-10-09T06:14:30.366660+0000","last_became_active":"2023-10-09T06:13:32.683104+0000","last_became_peered":"2023-10-09T06:13:32.683104+0000","last_unstale":"2023-10-09T06:14:30.366660+0000","last_undegraded":"2023-10-09T06:14:30.366660+0000","last_fullsized":"2023-10-09T06:14:30.366660+0000","mapping_epoch":528,"log_start":"505'2835","ondisk_log_start":"505'2835","created":490,"last_epoch_clean":529,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5912,"log_dups_size":0,"ondisk_log_size":5912,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:19.426571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231145472,"num_objects":3527,"num_object_clones":0,"num_object_copies":10581,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3527,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8747,"num_write_kb":392768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5732,"num_bytes_recovered":375652352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":809,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.720471+0000","last_change":"2023-10-09T06:13:15.442141+0000","last_active":"2023-10-09T06:14:27.720471+0000","last_peered":"2023-10-09T06:14:27.720471+0000","last_clean":"2023-10-09T06:14:27.720471+0000","last_became_active":"2023-10-09T06:13:15.441716+0000","last_became_peered":"2023-10-09T06:13:15.441716+0000","last_unstale":"2023-10-09T06:14:27.720471+0000","last_undegraded":"2023-10-09T06:14:27.720471+0000","last_fullsized":"2023-10-09T06:14:27.720471+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.282393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.282393+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:16.798440+0000","scrub_duration":0.01402403,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":465,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.717637+0000","last_change":"2023-10-09T06:13:21.738218+0000","last_active":"2023-10-09T06:14:27.717637+0000","last_peered":"2023-10-09T06:14:27.717637+0000","last_clean":"2023-10-09T06:14:27.717637+0000","last_became_active":"2023-10-09T06:13:21.737969+0000","last_became_peered":"2023-10-09T06:13:21.737969+0000","last_unstale":"2023-10-09T06:14:27.717637+0000","last_undegraded":"2023-10-09T06:14:27.717637+0000","last_fullsized":"2023-10-09T06:14:27.717637+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:38.620281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":594,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.717949+0000","last_change":"2023-10-09T06:07:43.572173+0000","last_active":"2023-10-09T06:14:27.717949+0000","last_peered":"2023-10-09T06:14:27.717949+0000","last_clean":"2023-10-09T06:14:27.717949+0000","last_became_active":"2023-10-09T06:07:43.571690+0000","last_became_peered":"2023-10-09T06:07:43.571690+0000","last_unstale":"2023-10-09T06:14:27.717949+0000","last_undegraded":"2023-10-09T06:14:27.717949+0000","last_fullsized":"2023-10-09T06:14:27.717949+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:10.234125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"546'8852","reported_seq":17221,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T06:14:31.577166+0000","last_change":"2023-10-09T06:14:28.746278+0000","last_active":"2023-10-09T06:14:31.577166+0000","last_peered":"2023-10-09T06:14:31.577166+0000","last_clean":"2023-10-09T06:14:31.577166+0000","last_became_active":"2023-10-09T06:14:28.728616+0000","last_became_peered":"2023-10-09T06:14:28.728616+0000","last_unstale":"2023-10-09T06:14:31.577166+0000","last_undegraded":"2023-10-09T06:14:31.577166+0000","last_fullsized":"2023-10-09T06:14:31.577166+0000","mapping_epoch":545,"log_start":"504'2300","ondisk_log_start":"504'2300","created":490,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6552,"log_dups_size":0,"ondisk_log_size":6552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:32.106540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221380608,"num_objects":3378,"num_object_clones":0,"num_object_copies":10134,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3378,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8852,"num_write_kb":391360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4137,"num_bytes_recovered":271122432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":880,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.711443+0000","last_change":"2023-10-09T06:13:15.428701+0000","last_active":"2023-10-09T06:14:27.711443+0000","last_peered":"2023-10-09T06:14:27.711443+0000","last_clean":"2023-10-09T06:14:27.711443+0000","last_became_active":"2023-10-09T06:13:15.428343+0000","last_became_peered":"2023-10-09T06:13:15.428343+0000","last_unstale":"2023-10-09T06:14:27.711443+0000","last_undegraded":"2023-10-09T06:14:27.711443+0000","last_fullsized":"2023-10-09T06:14:27.711443+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.373748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.373748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:55:34.235074+0000","scrub_duration":0.0024576450000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":536,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.717797+0000","last_change":"2023-10-09T06:07:43.574863+0000","last_active":"2023-10-09T06:14:27.717797+0000","last_peered":"2023-10-09T06:14:27.717797+0000","last_clean":"2023-10-09T06:14:27.717797+0000","last_became_active":"2023-10-09T06:07:43.574576+0000","last_became_peered":"2023-10-09T06:07:43.574576+0000","last_unstale":"2023-10-09T06:14:27.717797+0000","last_undegraded":"2023-10-09T06:14:27.717797+0000","last_fullsized":"2023-10-09T06:14:27.717797+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:16.166004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":577,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.718227+0000","last_change":"2023-10-09T06:13:21.727814+0000","last_active":"2023-10-09T06:14:27.718227+0000","last_peered":"2023-10-09T06:14:27.718227+0000","last_clean":"2023-10-09T06:14:27.718227+0000","last_became_active":"2023-10-09T06:13:21.727401+0000","last_became_peered":"2023-10-09T06:13:21.727401+0000","last_unstale":"2023-10-09T06:14:27.718227+0000","last_undegraded":"2023-10-09T06:14:27.718227+0000","last_fullsized":"2023-10-09T06:14:27.718227+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:53:37.968028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"546'8745","reported_seq":8809,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T06:14:30.865603+0000","last_change":"2023-10-09T06:11:45.695709+0000","last_active":"2023-10-09T06:14:30.865603+0000","last_peered":"2023-10-09T06:14:30.865603+0000","last_clean":"2023-10-09T06:14:30.865603+0000","last_became_active":"2023-10-09T06:11:45.695219+0000","last_became_peered":"2023-10-09T06:11:45.695219+0000","last_unstale":"2023-10-09T06:14:30.865603+0000","last_undegraded":"2023-10-09T06:14:30.865603+0000","last_fullsized":"2023-10-09T06:14:30.865603+0000","mapping_epoch":490,"log_start":"505'3300","ondisk_log_start":"505'3300","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5445,"log_dups_size":0,"ondisk_log_size":5445,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:26.533964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220659712,"num_objects":3367,"num_object_clones":0,"num_object_copies":10101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8745,"num_write_kb":387584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":624,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.724614+0000","last_change":"2023-10-09T06:13:24.753519+0000","last_active":"2023-10-09T06:14:27.724614+0000","last_peered":"2023-10-09T06:14:27.724614+0000","last_clean":"2023-10-09T06:14:27.724614+0000","last_became_active":"2023-10-09T06:13:24.753185+0000","last_became_peered":"2023-10-09T06:13:24.753185+0000","last_unstale":"2023-10-09T06:14:27.724614+0000","last_undegraded":"2023-10-09T06:14:27.724614+0000","last_fullsized":"2023-10-09T06:14:27.724614+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:39.633410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:39.633410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:56.871057+0000","scrub_duration":0.001547624,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":357,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.720564+0000","last_change":"2023-10-09T06:13:30.781849+0000","last_active":"2023-10-09T06:14:27.720564+0000","last_peered":"2023-10-09T06:14:27.720564+0000","last_clean":"2023-10-09T06:14:27.720564+0000","last_became_active":"2023-10-09T06:13:30.781084+0000","last_became_peered":"2023-10-09T06:13:30.781084+0000","last_unstale":"2023-10-09T06:14:27.720564+0000","last_undegraded":"2023-10-09T06:14:27.720564+0000","last_fullsized":"2023-10-09T06:14:27.720564+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:02.199298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":228,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.713021+0000","last_change":"2023-10-09T06:01:54.761768+0000","last_active":"2023-10-09T06:14:27.713021+0000","last_peered":"2023-10-09T06:14:27.713021+0000","last_clean":"2023-10-09T06:14:27.713021+0000","last_became_active":"2023-10-09T06:01:54.761443+0000","last_became_peered":"2023-10-09T06:01:54.761443+0000","last_unstale":"2023-10-09T06:14:27.713021+0000","last_undegraded":"2023-10-09T06:14:27.713021+0000","last_fullsized":"2023-10-09T06:14:27.713021+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:51.555987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"546'8774","reported_seq":12755,"reported_epoch":546,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T06:14:31.276987+0000","last_change":"2023-10-09T06:14:28.756902+0000","last_active":"2023-10-09T06:14:31.276987+0000","last_peered":"2023-10-09T06:14:31.276987+0000","last_clean":"2023-10-09T06:13:29.715670+0000","last_became_active":"2023-10-09T06:13:31.891405+0000","last_became_peered":"2023-10-09T06:13:31.891405+0000","last_unstale":"2023-10-09T06:14:31.276987+0000","last_undegraded":"2023-10-09T06:14:31.276987+0000","last_fullsized":"2023-10-09T06:14:31.276987+0000","mapping_epoch":527,"log_start":"504'2300","ondisk_log_start":"504'2300","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6474,"log_dups_size":0,"ondisk_log_size":6474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:13:59.527962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224919552,"num_objects":3432,"num_object_clones":0,"num_object_copies":10296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1614,"num_objects_unfound":0,"num_objects_dirty":3432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8774,"num_write_kb":390592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1953,"num_bytes_recovered":127991808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[0,4,3],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,3,4","objects":3432}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":748,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.727805+0000","last_change":"2023-10-09T06:13:33.707885+0000","last_active":"2023-10-09T06:14:27.727805+0000","last_peered":"2023-10-09T06:14:27.727805+0000","last_clean":"2023-10-09T06:14:27.727805+0000","last_became_active":"2023-10-09T06:13:33.707474+0000","last_became_peered":"2023-10-09T06:13:33.707474+0000","last_unstale":"2023-10-09T06:14:27.727805+0000","last_undegraded":"2023-10-09T06:14:27.727805+0000","last_fullsized":"2023-10-09T06:14:27.727805+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:12:22.289803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:12:22.289803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:27.842817+0000","scrub_duration":0.0076565050000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":291,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.711795+0000","last_change":"2023-10-09T06:12:04.680247+0000","last_active":"2023-10-09T06:14:27.711795+0000","last_peered":"2023-10-09T06:14:27.711795+0000","last_clean":"2023-10-09T06:14:27.711795+0000","last_became_active":"2023-10-09T06:12:04.679943+0000","last_became_peered":"2023-10-09T06:12:04.679943+0000","last_unstale":"2023-10-09T06:14:27.711795+0000","last_undegraded":"2023-10-09T06:14:27.711795+0000","last_fullsized":"2023-10-09T06:14:27.711795+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:56.692774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":334,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.725052+0000","last_change":"2023-10-09T06:07:43.577306+0000","last_active":"2023-10-09T06:14:27.725052+0000","last_peered":"2023-10-09T06:14:27.725052+0000","last_clean":"2023-10-09T06:14:27.725052+0000","last_became_active":"2023-10-09T06:07:43.576956+0000","last_became_peered":"2023-10-09T06:07:43.576956+0000","last_unstale":"2023-10-09T06:14:27.725052+0000","last_undegraded":"2023-10-09T06:14:27.725052+0000","last_fullsized":"2023-10-09T06:14:27.725052+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:21.020403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":262,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.711333+0000","last_change":"2023-10-09T06:13:12.234849+0000","last_active":"2023-10-09T06:14:27.711333+0000","last_peered":"2023-10-09T06:14:27.711333+0000","last_clean":"2023-10-09T06:14:27.711333+0000","last_became_active":"2023-10-09T06:13:12.234284+0000","last_became_peered":"2023-10-09T06:13:12.234284+0000","last_unstale":"2023-10-09T06:14:27.711333+0000","last_undegraded":"2023-10-09T06:14:27.711333+0000","last_fullsized":"2023-10-09T06:14:27.711333+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:40:43.809773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"546'8547","reported_seq":8613,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T06:14:31.171941+0000","last_change":"2023-10-09T06:11:46.029349+0000","last_active":"2023-10-09T06:14:31.171941+0000","last_peered":"2023-10-09T06:14:31.171941+0000","last_clean":"2023-10-09T06:14:31.171941+0000","last_became_active":"2023-10-09T06:11:46.029041+0000","last_became_peered":"2023-10-09T06:11:46.029041+0000","last_unstale":"2023-10-09T06:14:31.171941+0000","last_undegraded":"2023-10-09T06:14:31.171941+0000","last_fullsized":"2023-10-09T06:14:31.171941+0000","mapping_epoch":490,"log_start":"504'2100","ondisk_log_start":"504'2100","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6447,"log_dups_size":0,"ondisk_log_size":6447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:07.662395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217907200,"num_objects":3325,"num_object_clones":0,"num_object_copies":9975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8547,"num_write_kb":379904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":781,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.720141+0000","last_change":"2023-10-09T06:09:27.544555+0000","last_active":"2023-10-09T06:14:27.720141+0000","last_peered":"2023-10-09T06:14:27.720141+0000","last_clean":"2023-10-09T06:14:27.720141+0000","last_became_active":"2023-10-09T06:09:27.544222+0000","last_became_peered":"2023-10-09T06:09:27.544222+0000","last_unstale":"2023-10-09T06:14:27.720141+0000","last_undegraded":"2023-10-09T06:14:27.720141+0000","last_fullsized":"2023-10-09T06:14:27.720141+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:45.944253+0000","scrub_duration":0.0016660799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":247,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.711415+0000","last_change":"2023-10-09T06:13:21.738040+0000","last_active":"2023-10-09T06:14:27.711415+0000","last_peered":"2023-10-09T06:14:27.711415+0000","last_clean":"2023-10-09T06:14:27.711415+0000","last_became_active":"2023-10-09T06:13:21.737717+0000","last_became_peered":"2023-10-09T06:13:21.737717+0000","last_unstale":"2023-10-09T06:14:27.711415+0000","last_undegraded":"2023-10-09T06:14:27.711415+0000","last_fullsized":"2023-10-09T06:14:27.711415+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:39.451628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":526,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.717635+0000","last_change":"2023-10-09T06:09:27.553958+0000","last_active":"2023-10-09T06:14:27.717635+0000","last_peered":"2023-10-09T06:14:27.717635+0000","last_clean":"2023-10-09T06:14:27.717635+0000","last_became_active":"2023-10-09T06:09:27.553193+0000","last_became_peered":"2023-10-09T06:09:27.553193+0000","last_unstale":"2023-10-09T06:14:27.717635+0000","last_undegraded":"2023-10-09T06:14:27.717635+0000","last_fullsized":"2023-10-09T06:14:27.717635+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:09.754579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":372,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.711878+0000","last_change":"2023-10-09T06:07:43.580765+0000","last_active":"2023-10-09T06:14:27.711878+0000","last_peered":"2023-10-09T06:14:27.711878+0000","last_clean":"2023-10-09T06:14:27.711878+0000","last_became_active":"2023-10-09T06:07:43.580458+0000","last_became_peered":"2023-10-09T06:07:43.580458+0000","last_unstale":"2023-10-09T06:14:27.711878+0000","last_undegraded":"2023-10-09T06:14:27.711878+0000","last_fullsized":"2023-10-09T06:14:27.711878+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:55.161893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"546'8498","reported_seq":24818,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T06:14:29.322013+0000","last_change":"2023-10-09T06:14:21.793267+0000","last_active":"2023-10-09T06:14:29.322013+0000","last_peered":"2023-10-09T06:14:29.322013+0000","last_clean":"2023-10-09T06:14:29.322013+0000","last_became_active":"2023-10-09T06:14:21.734665+0000","last_became_peered":"2023-10-09T06:14:21.734665+0000","last_unstale":"2023-10-09T06:14:29.322013+0000","last_undegraded":"2023-10-09T06:14:29.322013+0000","last_fullsized":"2023-10-09T06:14:29.322013+0000","mapping_epoch":543,"log_start":"505'2800","ondisk_log_start":"505'2800","created":490,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5698,"log_dups_size":0,"ondisk_log_size":5698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:01.941433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222953472,"num_objects":3402,"num_object_clones":0,"num_object_copies":10206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8498,"num_write_kb":380800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7073,"num_bytes_recovered":463536128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":591,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.714044+0000","last_change":"2023-10-09T06:13:33.705726+0000","last_active":"2023-10-09T06:14:27.714044+0000","last_peered":"2023-10-09T06:14:27.714044+0000","last_clean":"2023-10-09T06:14:27.714044+0000","last_became_active":"2023-10-09T06:13:33.705351+0000","last_became_peered":"2023-10-09T06:13:33.705351+0000","last_unstale":"2023-10-09T06:14:27.714044+0000","last_undegraded":"2023-10-09T06:14:27.714044+0000","last_fullsized":"2023-10-09T06:14:27.714044+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:44.483427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.483427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":326,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.721275+0000","last_change":"2023-10-09T06:09:27.565004+0000","last_active":"2023-10-09T06:14:27.721275+0000","last_peered":"2023-10-09T06:14:27.721275+0000","last_clean":"2023-10-09T06:14:27.721275+0000","last_became_active":"2023-10-09T06:09:27.564073+0000","last_became_peered":"2023-10-09T06:09:27.564073+0000","last_unstale":"2023-10-09T06:14:27.721275+0000","last_undegraded":"2023-10-09T06:14:27.721275+0000","last_fullsized":"2023-10-09T06:14:27.721275+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:15.146042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":316,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.711981+0000","last_change":"2023-10-09T06:13:21.728291+0000","last_active":"2023-10-09T06:14:27.711981+0000","last_peered":"2023-10-09T06:14:27.711981+0000","last_clean":"2023-10-09T06:14:27.711981+0000","last_became_active":"2023-10-09T06:13:21.727926+0000","last_became_peered":"2023-10-09T06:13:21.727926+0000","last_unstale":"2023-10-09T06:14:27.711981+0000","last_undegraded":"2023-10-09T06:14:27.711981+0000","last_fullsized":"2023-10-09T06:14:27.711981+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:35.292785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":243,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.712956+0000","last_change":"2023-10-09T06:09:27.547953+0000","last_active":"2023-10-09T06:14:27.712956+0000","last_peered":"2023-10-09T06:14:27.712956+0000","last_clean":"2023-10-09T06:14:27.712956+0000","last_became_active":"2023-10-09T06:09:27.547624+0000","last_became_peered":"2023-10-09T06:09:27.547624+0000","last_unstale":"2023-10-09T06:14:27.712956+0000","last_undegraded":"2023-10-09T06:14:27.712956+0000","last_fullsized":"2023-10-09T06:14:27.712956+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:39.630486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"546'8632","reported_seq":13737,"reported_epoch":546,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T06:14:31.594054+0000","last_change":"2023-10-09T06:14:27.376332+0000","last_active":"2023-10-09T06:14:31.594054+0000","last_peered":"2023-10-09T06:14:31.594054+0000","last_clean":"2023-10-09T06:13:20.543162+0000","last_became_active":"2023-10-09T06:13:22.836318+0000","last_became_peered":"2023-10-09T06:13:22.836318+0000","last_unstale":"2023-10-09T06:14:31.594054+0000","last_undegraded":"2023-10-09T06:14:31.594054+0000","last_fullsized":"2023-10-09T06:14:31.594054+0000","mapping_epoch":522,"log_start":"504'2100","ondisk_log_start":"504'2100","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6532,"log_dups_size":0,"ondisk_log_size":6532,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:40.353464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216399872,"num_objects":3302,"num_object_clones":0,"num_object_copies":9906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1486,"num_objects_unfound":0,"num_objects_dirty":3302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8632,"num_write_kb":381888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2512,"num_bytes_recovered":164626432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,0,4],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":3302}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":29,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.714180+0000","last_change":"2023-10-09T06:13:30.791685+0000","last_active":"2023-10-09T06:14:27.714180+0000","last_peered":"2023-10-09T06:14:27.714180+0000","last_clean":"2023-10-09T06:14:27.714180+0000","last_became_active":"2023-10-09T06:13:30.791255+0000","last_became_peered":"2023-10-09T06:13:30.791255+0000","last_unstale":"2023-10-09T06:14:27.714180+0000","last_undegraded":"2023-10-09T06:14:27.714180+0000","last_fullsized":"2023-10-09T06:14:27.714180+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:24.896244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":589,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.713843+0000","last_change":"2023-10-09T06:13:33.704835+0000","last_active":"2023-10-09T06:14:27.713843+0000","last_peered":"2023-10-09T06:14:27.713843+0000","last_clean":"2023-10-09T06:14:27.713843+0000","last_became_active":"2023-10-09T06:13:33.704466+0000","last_became_peered":"2023-10-09T06:13:33.704466+0000","last_unstale":"2023-10-09T06:14:27.713843+0000","last_undegraded":"2023-10-09T06:14:27.713843+0000","last_fullsized":"2023-10-09T06:14:27.713843+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:35.783434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:35.783434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:38.721974+0000","scrub_duration":0.0043794799999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":334,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.721118+0000","last_change":"2023-10-09T06:13:12.238280+0000","last_active":"2023-10-09T06:14:27.721118+0000","last_peered":"2023-10-09T06:14:27.721118+0000","last_clean":"2023-10-09T06:14:27.721118+0000","last_became_active":"2023-10-09T06:13:12.237986+0000","last_became_peered":"2023-10-09T06:13:12.237986+0000","last_unstale":"2023-10-09T06:14:27.721118+0000","last_undegraded":"2023-10-09T06:14:27.721118+0000","last_fullsized":"2023-10-09T06:14:27.721118+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:58.520176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":476,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.717923+0000","last_change":"2023-10-09T06:13:21.733940+0000","last_active":"2023-10-09T06:14:27.717923+0000","last_peered":"2023-10-09T06:14:27.717923+0000","last_clean":"2023-10-09T06:14:27.717923+0000","last_became_active":"2023-10-09T06:13:21.733628+0000","last_became_peered":"2023-10-09T06:13:21.733628+0000","last_unstale":"2023-10-09T06:14:27.717923+0000","last_undegraded":"2023-10-09T06:14:27.717923+0000","last_fullsized":"2023-10-09T06:14:27.717923+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:13.348443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"546'8524","reported_seq":18481,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T06:14:29.303945+0000","last_change":"2023-10-09T06:14:09.843785+0000","last_active":"2023-10-09T06:14:29.303945+0000","last_peered":"2023-10-09T06:14:29.303945+0000","last_clean":"2023-10-09T06:14:29.303945+0000","last_became_active":"2023-10-09T06:14:09.789864+0000","last_became_peered":"2023-10-09T06:14:09.789864+0000","last_unstale":"2023-10-09T06:14:29.303945+0000","last_undegraded":"2023-10-09T06:14:29.303945+0000","last_fullsized":"2023-10-09T06:14:29.303945+0000","mapping_epoch":540,"log_start":"504'2100","ondisk_log_start":"504'2100","created":490,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6424,"log_dups_size":0,"ondisk_log_size":6424,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:03.191322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222167040,"num_objects":3390,"num_object_clones":0,"num_object_copies":10170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3390,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8524,"num_write_kb":381248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4936,"num_bytes_recovered":323485696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":815,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T06:14:28.724242+0000","last_change":"2023-10-09T06:13:24.753842+0000","last_active":"2023-10-09T06:14:28.724242+0000","last_peered":"2023-10-09T06:14:28.724242+0000","last_clean":"2023-10-09T06:14:28.724242+0000","last_became_active":"2023-10-09T06:13:24.753485+0000","last_became_peered":"2023-10-09T06:13:24.753485+0000","last_unstale":"2023-10-09T06:14:28.724242+0000","last_undegraded":"2023-10-09T06:14:28.724242+0000","last_fullsized":"2023-10-09T06:14:28.724242+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:57.627296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:57.627296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:56.656218+0000","scrub_duration":0.001913166,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":439,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.714034+0000","last_change":"2023-10-09T06:09:27.544373+0000","last_active":"2023-10-09T06:14:27.714034+0000","last_peered":"2023-10-09T06:14:27.714034+0000","last_clean":"2023-10-09T06:14:27.714034+0000","last_became_active":"2023-10-09T06:09:27.544121+0000","last_became_peered":"2023-10-09T06:09:27.544121+0000","last_unstale":"2023-10-09T06:14:27.714034+0000","last_undegraded":"2023-10-09T06:14:27.714034+0000","last_fullsized":"2023-10-09T06:14:27.714034+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:40.429491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":483,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.719927+0000","last_change":"2023-10-09T06:12:52.670573+0000","last_active":"2023-10-09T06:14:27.719927+0000","last_peered":"2023-10-09T06:14:27.719927+0000","last_clean":"2023-10-09T06:14:27.719927+0000","last_became_active":"2023-10-09T06:12:52.670242+0000","last_became_peered":"2023-10-09T06:12:52.670242+0000","last_unstale":"2023-10-09T06:14:27.719927+0000","last_undegraded":"2023-10-09T06:14:27.719927+0000","last_fullsized":"2023-10-09T06:14:27.719927+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:25.237580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":622,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.720333+0000","last_change":"2023-10-09T06:09:27.543678+0000","last_active":"2023-10-09T06:14:27.720333+0000","last_peered":"2023-10-09T06:14:27.720333+0000","last_clean":"2023-10-09T06:14:27.720333+0000","last_became_active":"2023-10-09T06:09:27.543298+0000","last_became_peered":"2023-10-09T06:09:27.543298+0000","last_unstale":"2023-10-09T06:14:27.720333+0000","last_undegraded":"2023-10-09T06:14:27.720333+0000","last_fullsized":"2023-10-09T06:14:27.720333+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:26.297842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"546'8629","reported_seq":18226,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T06:14:30.864209+0000","last_change":"2023-10-09T06:14:02.643285+0000","last_active":"2023-10-09T06:14:30.864209+0000","last_peered":"2023-10-09T06:14:30.864209+0000","last_clean":"2023-10-09T06:14:30.864209+0000","last_became_active":"2023-10-09T06:14:02.599489+0000","last_became_peered":"2023-10-09T06:14:02.599489+0000","last_unstale":"2023-10-09T06:14:30.864209+0000","last_undegraded":"2023-10-09T06:14:30.864209+0000","last_fullsized":"2023-10-09T06:14:30.864209+0000","mapping_epoch":537,"log_start":"505'3200","ondisk_log_start":"505'3200","created":490,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5429,"log_dups_size":0,"ondisk_log_size":5429,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:10.022074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219807768,"num_objects":3355,"num_object_clones":0,"num_object_copies":10065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3355,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8629,"num_write_kb":383425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4755,"num_bytes_recovered":311558168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":745,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.725026+0000","last_change":"2023-10-09T06:13:24.756288+0000","last_active":"2023-10-09T06:14:27.725026+0000","last_peered":"2023-10-09T06:14:27.725026+0000","last_clean":"2023-10-09T06:14:27.725026+0000","last_became_active":"2023-10-09T06:13:24.755945+0000","last_became_peered":"2023-10-09T06:13:24.755945+0000","last_unstale":"2023-10-09T06:14:27.725026+0000","last_undegraded":"2023-10-09T06:14:27.725026+0000","last_fullsized":"2023-10-09T06:14:27.725026+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:45.844892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:45.844892+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:44.107376+0000","scrub_duration":0.0017521489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":29,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.714228+0000","last_change":"2023-10-09T06:13:30.799535+0000","last_active":"2023-10-09T06:14:27.714228+0000","last_peered":"2023-10-09T06:14:27.714228+0000","last_clean":"2023-10-09T06:14:27.714228+0000","last_became_active":"2023-10-09T06:13:30.799227+0000","last_became_peered":"2023-10-09T06:13:30.799227+0000","last_unstale":"2023-10-09T06:14:27.714228+0000","last_undegraded":"2023-10-09T06:14:27.714228+0000","last_fullsized":"2023-10-09T06:14:27.714228+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:14.741742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":265,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.711970+0000","last_change":"2023-10-09T06:13:12.237622+0000","last_active":"2023-10-09T06:14:27.711970+0000","last_peered":"2023-10-09T06:14:27.711970+0000","last_clean":"2023-10-09T06:14:27.711970+0000","last_became_active":"2023-10-09T06:13:12.237287+0000","last_became_peered":"2023-10-09T06:13:12.237287+0000","last_unstale":"2023-10-09T06:14:27.711970+0000","last_undegraded":"2023-10-09T06:14:27.711970+0000","last_fullsized":"2023-10-09T06:14:27.711970+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:01.942898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":29,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.714119+0000","last_change":"2023-10-09T06:13:30.788382+0000","last_active":"2023-10-09T06:14:27.714119+0000","last_peered":"2023-10-09T06:14:27.714119+0000","last_clean":"2023-10-09T06:14:27.714119+0000","last_became_active":"2023-10-09T06:13:30.788060+0000","last_became_peered":"2023-10-09T06:13:30.788060+0000","last_unstale":"2023-10-09T06:14:27.714119+0000","last_undegraded":"2023-10-09T06:14:27.714119+0000","last_fullsized":"2023-10-09T06:14:27.714119+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:36:58.984799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.e","version":"546'8725","reported_seq":19821,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T06:14:30.821151+0000","last_change":"2023-10-09T06:13:51.831598+0000","last_active":"2023-10-09T06:14:30.821151+0000","last_peered":"2023-10-09T06:14:30.821151+0000","last_clean":"2023-10-09T06:14:30.821151+0000","last_became_active":"2023-10-09T06:13:51.816987+0000","last_became_peered":"2023-10-09T06:13:51.816987+0000","last_unstale":"2023-10-09T06:14:30.821151+0000","last_undegraded":"2023-10-09T06:14:30.821151+0000","last_fullsized":"2023-10-09T06:14:30.821151+0000","mapping_epoch":534,"log_start":"505'3300","ondisk_log_start":"505'3300","created":490,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5425,"log_dups_size":0,"ondisk_log_size":5425,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:52.460066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226164736,"num_objects":3451,"num_object_clones":0,"num_object_copies":10353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8725,"num_write_kb":389632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5503,"num_bytes_recovered":360644608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":1019,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.718050+0000","last_change":"2023-10-09T06:13:24.762291+0000","last_active":"2023-10-09T06:14:27.718050+0000","last_peered":"2023-10-09T06:14:27.718050+0000","last_clean":"2023-10-09T06:14:27.718050+0000","last_became_active":"2023-10-09T06:13:24.761918+0000","last_became_peered":"2023-10-09T06:13:24.761918+0000","last_unstale":"2023-10-09T06:14:27.718050+0000","last_undegraded":"2023-10-09T06:14:27.718050+0000","last_fullsized":"2023-10-09T06:14:27.718050+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:17.344518+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:17.344518+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:58.227603+0000","scrub_duration":0.0015914099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":602,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.728121+0000","last_change":"2023-10-09T06:13:30.798394+0000","last_active":"2023-10-09T06:14:27.728121+0000","last_peered":"2023-10-09T06:14:27.728121+0000","last_clean":"2023-10-09T06:14:27.728121+0000","last_became_active":"2023-10-09T06:13:30.790913+0000","last_became_peered":"2023-10-09T06:13:30.790913+0000","last_unstale":"2023-10-09T06:14:27.728121+0000","last_undegraded":"2023-10-09T06:14:27.728121+0000","last_fullsized":"2023-10-09T06:14:27.728121+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:26.508690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":362,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.712757+0000","last_change":"2023-10-09T06:13:21.728009+0000","last_active":"2023-10-09T06:14:27.712757+0000","last_peered":"2023-10-09T06:14:27.712757+0000","last_clean":"2023-10-09T06:14:27.712757+0000","last_became_active":"2023-10-09T06:13:21.727683+0000","last_became_peered":"2023-10-09T06:13:21.727683+0000","last_unstale":"2023-10-09T06:14:27.712757+0000","last_undegraded":"2023-10-09T06:14:27.712757+0000","last_fullsized":"2023-10-09T06:14:27.712757+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:13.074232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":351,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.720810+0000","last_change":"2023-10-09T06:13:30.782038+0000","last_active":"2023-10-09T06:14:27.720810+0000","last_peered":"2023-10-09T06:14:27.720810+0000","last_clean":"2023-10-09T06:14:27.720810+0000","last_became_active":"2023-10-09T06:13:30.781500+0000","last_became_peered":"2023-10-09T06:13:30.781500+0000","last_unstale":"2023-10-09T06:14:27.720810+0000","last_undegraded":"2023-10-09T06:14:27.720810+0000","last_fullsized":"2023-10-09T06:14:27.720810+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:43.663265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"546'8516","reported_seq":8582,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-09T06:14:30.317252+0000","last_change":"2023-10-09T06:11:46.031709+0000","last_active":"2023-10-09T06:14:30.317252+0000","last_peered":"2023-10-09T06:14:30.317252+0000","last_clean":"2023-10-09T06:14:30.317252+0000","last_became_active":"2023-10-09T06:11:46.031421+0000","last_became_peered":"2023-10-09T06:11:46.031421+0000","last_unstale":"2023-10-09T06:14:30.317252+0000","last_undegraded":"2023-10-09T06:14:30.317252+0000","last_fullsized":"2023-10-09T06:14:30.317252+0000","mapping_epoch":490,"log_start":"504'2625","ondisk_log_start":"504'2625","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5891,"log_dups_size":0,"ondisk_log_size":5891,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:13.095174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223084544,"num_objects":3404,"num_object_clones":0,"num_object_copies":10212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8516,"num_write_kb":381440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":878,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.723959+0000","last_change":"2023-10-09T06:13:33.708100+0000","last_active":"2023-10-09T06:14:27.723959+0000","last_peered":"2023-10-09T06:14:27.723959+0000","last_clean":"2023-10-09T06:14:27.723959+0000","last_became_active":"2023-10-09T06:13:33.707809+0000","last_became_peered":"2023-10-09T06:13:33.707809+0000","last_unstale":"2023-10-09T06:14:27.723959+0000","last_undegraded":"2023-10-09T06:14:27.723959+0000","last_fullsized":"2023-10-09T06:14:27.723959+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:43:26.848778+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":29,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.714644+0000","last_change":"2023-10-09T06:13:30.799948+0000","last_active":"2023-10-09T06:14:27.714644+0000","last_peered":"2023-10-09T06:14:27.714644+0000","last_clean":"2023-10-09T06:14:27.714644+0000","last_became_active":"2023-10-09T06:13:30.799690+0000","last_became_peered":"2023-10-09T06:13:30.799690+0000","last_unstale":"2023-10-09T06:14:27.714644+0000","last_undegraded":"2023-10-09T06:14:27.714644+0000","last_fullsized":"2023-10-09T06:14:27.714644+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:43.893184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":592,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.720588+0000","last_change":"2023-10-09T06:09:27.578756+0000","last_active":"2023-10-09T06:14:27.720588+0000","last_peered":"2023-10-09T06:14:27.720588+0000","last_clean":"2023-10-09T06:14:27.720588+0000","last_became_active":"2023-10-09T06:09:27.578384+0000","last_became_peered":"2023-10-09T06:09:27.578384+0000","last_unstale":"2023-10-09T06:14:27.720588+0000","last_undegraded":"2023-10-09T06:14:27.720588+0000","last_fullsized":"2023-10-09T06:14:27.720588+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:22.182630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":492,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.717875+0000","last_change":"2023-10-09T06:01:22.334081+0000","last_active":"2023-10-09T06:14:27.717875+0000","last_peered":"2023-10-09T06:14:27.717875+0000","last_clean":"2023-10-09T06:14:27.717875+0000","last_became_active":"2023-10-09T06:01:22.333663+0000","last_became_peered":"2023-10-09T06:01:22.333663+0000","last_unstale":"2023-10-09T06:14:27.717875+0000","last_undegraded":"2023-10-09T06:14:27.717875+0000","last_fullsized":"2023-10-09T06:14:27.717875+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:55.809259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":792,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.720541+0000","last_change":"2023-10-09T06:09:27.560157+0000","last_active":"2023-10-09T06:14:27.720541+0000","last_peered":"2023-10-09T06:14:27.720541+0000","last_clean":"2023-10-09T06:14:27.720541+0000","last_became_active":"2023-10-09T06:09:27.559466+0000","last_became_peered":"2023-10-09T06:09:27.559466+0000","last_unstale":"2023-10-09T06:14:27.720541+0000","last_undegraded":"2023-10-09T06:14:27.720541+0000","last_fullsized":"2023-10-09T06:14:27.720541+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.181811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.181811+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:53.431491+0000","scrub_duration":0.011553313000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":475,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.719041+0000","last_change":"2023-10-09T06:13:30.791343+0000","last_active":"2023-10-09T06:14:27.719041+0000","last_peered":"2023-10-09T06:14:27.719041+0000","last_clean":"2023-10-09T06:14:27.719041+0000","last_became_active":"2023-10-09T06:13:30.790945+0000","last_became_peered":"2023-10-09T06:13:30.790945+0000","last_unstale":"2023-10-09T06:14:27.719041+0000","last_undegraded":"2023-10-09T06:14:27.719041+0000","last_fullsized":"2023-10-09T06:14:27.719041+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:14.208391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":247,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.712052+0000","last_change":"2023-10-09T06:07:43.580844+0000","last_active":"2023-10-09T06:14:27.712052+0000","last_peered":"2023-10-09T06:14:27.712052+0000","last_clean":"2023-10-09T06:14:27.712052+0000","last_became_active":"2023-10-09T06:07:43.580523+0000","last_became_peered":"2023-10-09T06:07:43.580523+0000","last_unstale":"2023-10-09T06:14:27.712052+0000","last_undegraded":"2023-10-09T06:14:27.712052+0000","last_fullsized":"2023-10-09T06:14:27.712052+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:55.942430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":895,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.725050+0000","last_change":"2023-10-09T06:00:04.215425+0000","last_active":"2023-10-09T06:14:27.725050+0000","last_peered":"2023-10-09T06:14:27.725050+0000","last_clean":"2023-10-09T06:14:27.725050+0000","last_became_active":"2023-10-09T05:59:40.326685+0000","last_became_peered":"2023-10-09T05:59:40.326685+0000","last_unstale":"2023-10-09T06:14:27.725050+0000","last_undegraded":"2023-10-09T06:14:27.725050+0000","last_fullsized":"2023-10-09T06:14:27.725050+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:04.215270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:04.215270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:03.963694+0000","scrub_duration":0.002850652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":317,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.717989+0000","last_change":"2023-10-09T06:01:54.824780+0000","last_active":"2023-10-09T06:14:27.717989+0000","last_peered":"2023-10-09T06:14:27.717989+0000","last_clean":"2023-10-09T06:14:27.717989+0000","last_became_active":"2023-10-09T06:01:54.757727+0000","last_became_peered":"2023-10-09T06:01:54.757727+0000","last_unstale":"2023-10-09T06:14:27.717989+0000","last_undegraded":"2023-10-09T06:14:27.717989+0000","last_fullsized":"2023-10-09T06:14:27.717989+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:44.132556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":564,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.718249+0000","last_change":"2023-10-09T06:01:54.764989+0000","last_active":"2023-10-09T06:14:27.718249+0000","last_peered":"2023-10-09T06:14:27.718249+0000","last_clean":"2023-10-09T06:14:27.718249+0000","last_became_active":"2023-10-09T06:01:54.764790+0000","last_became_peered":"2023-10-09T06:01:54.764790+0000","last_unstale":"2023-10-09T06:14:27.718249+0000","last_undegraded":"2023-10-09T06:14:27.718249+0000","last_fullsized":"2023-10-09T06:14:27.718249+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:14:07.208858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":331,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.717762+0000","last_change":"2023-10-09T06:13:21.739730+0000","last_active":"2023-10-09T06:14:27.717762+0000","last_peered":"2023-10-09T06:14:27.717762+0000","last_clean":"2023-10-09T06:14:27.717762+0000","last_became_active":"2023-10-09T06:13:21.727326+0000","last_became_peered":"2023-10-09T06:13:21.727326+0000","last_unstale":"2023-10-09T06:14:27.717762+0000","last_undegraded":"2023-10-09T06:14:27.717762+0000","last_fullsized":"2023-10-09T06:14:27.717762+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:51.344571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":561,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.717689+0000","last_change":"2023-10-09T06:09:27.565198+0000","last_active":"2023-10-09T06:14:27.717689+0000","last_peered":"2023-10-09T06:14:27.717689+0000","last_clean":"2023-10-09T06:14:27.717689+0000","last_became_active":"2023-10-09T06:09:27.564909+0000","last_became_peered":"2023-10-09T06:09:27.564909+0000","last_unstale":"2023-10-09T06:14:27.717689+0000","last_undegraded":"2023-10-09T06:14:27.717689+0000","last_fullsized":"2023-10-09T06:14:27.717689+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:46.101014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":711,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.714035+0000","last_change":"2023-10-09T06:13:33.699235+0000","last_active":"2023-10-09T06:14:27.714035+0000","last_peered":"2023-10-09T06:14:27.714035+0000","last_clean":"2023-10-09T06:14:27.714035+0000","last_became_active":"2023-10-09T06:13:33.698886+0000","last_became_peered":"2023-10-09T06:13:33.698886+0000","last_unstale":"2023-10-09T06:14:27.714035+0000","last_undegraded":"2023-10-09T06:14:27.714035+0000","last_fullsized":"2023-10-09T06:14:27.714035+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:00.456132+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:00.456132+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:59.421706+0000","scrub_duration":0.0014992199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":352,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.725150+0000","last_change":"2023-10-09T06:13:21.740383+0000","last_active":"2023-10-09T06:14:27.725150+0000","last_peered":"2023-10-09T06:14:27.725150+0000","last_clean":"2023-10-09T06:14:27.725150+0000","last_became_active":"2023-10-09T06:13:21.740023+0000","last_became_peered":"2023-10-09T06:13:21.740023+0000","last_unstale":"2023-10-09T06:14:27.725150+0000","last_undegraded":"2023-10-09T06:14:27.725150+0000","last_fullsized":"2023-10-09T06:14:27.725150+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:27.380765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":348,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.721350+0000","last_change":"2023-10-09T06:09:27.565185+0000","last_active":"2023-10-09T06:14:27.721350+0000","last_peered":"2023-10-09T06:14:27.721350+0000","last_clean":"2023-10-09T06:14:27.721350+0000","last_became_active":"2023-10-09T06:09:27.564478+0000","last_became_peered":"2023-10-09T06:09:27.564478+0000","last_unstale":"2023-10-09T06:14:27.721350+0000","last_undegraded":"2023-10-09T06:14:27.721350+0000","last_fullsized":"2023-10-09T06:14:27.721350+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:04.622171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":869,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.712150+0000","last_change":"2023-10-09T06:12:55.753877+0000","last_active":"2023-10-09T06:14:27.712150+0000","last_peered":"2023-10-09T06:14:27.712150+0000","last_clean":"2023-10-09T06:14:27.712150+0000","last_became_active":"2023-10-09T06:12:55.753512+0000","last_became_peered":"2023-10-09T06:12:55.753512+0000","last_unstale":"2023-10-09T06:14:27.712150+0000","last_undegraded":"2023-10-09T06:14:27.712150+0000","last_fullsized":"2023-10-09T06:14:27.712150+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:46.290510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:46.290510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:30.368031+0000","scrub_duration":0.002627819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":538,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.718530+0000","last_change":"2023-10-09T06:09:27.556271+0000","last_active":"2023-10-09T06:14:27.718530+0000","last_peered":"2023-10-09T06:14:27.718530+0000","last_clean":"2023-10-09T06:14:27.718530+0000","last_became_active":"2023-10-09T06:09:27.555932+0000","last_became_peered":"2023-10-09T06:09:27.555932+0000","last_unstale":"2023-10-09T06:14:27.718530+0000","last_undegraded":"2023-10-09T06:14:27.718530+0000","last_fullsized":"2023-10-09T06:14:27.718530+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:33.888622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":577,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-09T06:14:27.718106+0000","last_change":"2023-10-09T06:13:21.727720+0000","last_active":"2023-10-09T06:14:27.718106+0000","last_peered":"2023-10-09T06:14:27.718106+0000","last_clean":"2023-10-09T06:14:27.718106+0000","last_became_active":"2023-10-09T06:13:21.727339+0000","last_became_peered":"2023-10-09T06:13:21.727339+0000","last_unstale":"2023-10-09T06:14:27.718106+0000","last_undegraded":"2023-10-09T06:14:27.718106+0000","last_fullsized":"2023-10-09T06:14:27.718106+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:25.723752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3541958680,"num_objects":54047,"num_object_clones":0,"num_object_copies":162141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":9480,"num_objects_unfound":0,"num_objects_dirty":54047,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":138379,"num_write_kb":6157569,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37402,"num_bytes_recovered":2451111960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12552847360,"data_stored":12552831072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97026,"ondisk_log_size":97026,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":11714,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13410,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":617203,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1523929,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234418,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1650792,"kb_used_data":754080,"kb_used_omap":0,"kb_used_meta":896704,"kb_avail":92094360,"statfs":{"total":95995035648,"available":94304624640,"internally_reserved":0,"allocated":772177920,"data_stored":770623942,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918224896},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:31 2023","interfaces":[{"interface":"back","average":{"1min":8.7140000000000004,"5min":2.887,"15min":1.784},"min":{"1min":0.26900000000000002,"5min":0.25,"15min":0.25},"max":{"1min":57.435000000000002,"5min":57.435000000000002,"15min":57.435000000000002},"last":0.19500000000000001},{"interface":"front","average":{"1min":8.8059999999999992,"5min":3.0590000000000002,"15min":1.7829999999999999},"min":{"1min":0.318,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":57.472999999999999,"5min":57.472999999999999,"15min":57.472999999999999},"last":0.25600000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:14:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":4.141,"15min":3.2189999999999999},"min":{"1min":0.27200000000000002,"5min":0.255,"15min":0.218},"max":{"1min":5.9580000000000002,"5min":124.66200000000001,"15min":124.66200000000001},"last":0.41299999999999998},{"interface":"front","average":{"1min":1.216,"5min":4.1829999999999998,"15min":3.2069999999999999},"min":{"1min":0.33900000000000002,"5min":0.27100000000000002,"15min":0.22700000000000001},"max":{"1min":5.8650000000000002,"5min":125.023,"15min":125.023},"last":0.38500000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:13:31 2023","interfaces":[{"interface":"back","average":{"1min":3.456,"5min":1.788,"15min":2.222},"min":{"1min":0.316,"5min":0.27400000000000002,"15min":0.24299999999999999},"max":{"1min":47.503,"5min":47.503,"15min":127.86499999999999},"last":0.32000000000000001},{"interface":"front","average":{"1min":2.9900000000000002,"5min":2.0110000000000001,"15min":2.2250000000000001},"min":{"1min":0.318,"5min":0.29599999999999999,"15min":0.27000000000000002},"max":{"1min":47.545000000000002,"5min":47.545000000000002,"15min":129.24000000000001},"last":0.30599999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:14:05 2023","interfaces":[{"interface":"back","average":{"1min":10.307,"5min":7.9900000000000002,"15min":6.4820000000000002},"min":{"1min":0.35499999999999998,"5min":0.32100000000000001,"15min":0.29899999999999999},"max":{"1min":67.852000000000004,"5min":116.68000000000001,"15min":164.16999999999999},"last":0.33800000000000002},{"interface":"front","average":{"1min":10.298,"5min":8.2260000000000009,"15min":6.6589999999999998},"min":{"1min":0.36699999999999999,"5min":0.33300000000000002,"15min":0.32100000000000001},"max":{"1min":67.881,"5min":116.797,"15min":164.40700000000001},"last":0.34599999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":1.1220000000000001,"15min":0.92600000000000005},"min":{"1min":0.27100000000000002,"5min":0.23999999999999999,"15min":0.23100000000000001},"max":{"1min":3.4180000000000001,"5min":14.371,"15min":14.371},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.51300000000000001,"5min":1.002,"15min":0.91500000000000004},"min":{"1min":0.30199999999999999,"5min":0.28599999999999998,"15min":0.27100000000000002},"max":{"1min":1.3220000000000001,"5min":14.259,"15min":14.259},"last":0.40200000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":1.381,"15min":1.0669999999999999},"min":{"1min":0.317,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.6589999999999998,"5min":43.826999999999998,"15min":43.826999999999998},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":1.28,"15min":1.119},"min":{"1min":0.39200000000000002,"5min":0.27300000000000002,"15min":0.24399999999999999},"max":{"1min":0.85399999999999998,"5min":43.880000000000003,"15min":43.880000000000003},"last":0.47899999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.85599999999999998,"15min":0.97999999999999998},"min":{"1min":0.33600000000000002,"5min":0.28000000000000003,"15min":0.26600000000000001},"max":{"1min":2.9209999999999998,"5min":4.6779999999999999,"15min":9.4540000000000006},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.94099999999999995,"15min":1.0089999999999999},"min":{"1min":0.35299999999999998,"5min":0.307,"15min":0.30599999999999999},"max":{"1min":2.3940000000000001,"5min":6.2329999999999997,"15min":6.2329999999999997},"last":0.373}]}]},{"osd":6,"up_from":46,"seq":197568496055,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2120920,"kb_used_data":1219280,"kb_used_omap":0,"kb_used_meta":901632,"kb_avail":91624232,"statfs":{"total":95995035648,"available":93823213568,"internally_reserved":0,"allocated":1248542720,"data_stored":1247004195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923271168},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:28 2023","interfaces":[{"interface":"back","average":{"1min":18.359000000000002,"5min":7.04,"15min":3.8420000000000001},"min":{"1min":0.246,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":181.209,"5min":191.50800000000001,"15min":191.50800000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":18.960999999999999,"5min":7.0990000000000002,"15min":3.6920000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":181.233,"5min":191.52600000000001,"15min":191.52600000000001},"last":0.46000000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:14:28 2023","interfaces":[{"interface":"back","average":{"1min":13.445,"5min":8.6509999999999998,"15min":6.0129999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.23200000000000001},"max":{"1min":193.845,"5min":328.04399999999998,"15min":328.04399999999998},"last":0.32600000000000001},{"interface":"front","average":{"1min":13.298,"5min":8.8420000000000005,"15min":6.1219999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":193.779,"5min":328.10000000000002,"15min":328.10000000000002},"last":0.28999999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:13:28 2023","interfaces":[{"interface":"back","average":{"1min":18.006,"5min":4.798,"15min":3.0049999999999999},"min":{"1min":0.28000000000000003,"5min":0.27300000000000002,"15min":0.26200000000000001},"max":{"1min":163.04900000000001,"5min":163.04900000000001,"15min":163.04900000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":17.959,"5min":4.9500000000000002,"15min":2.96},"min":{"1min":0.31900000000000001,"5min":0.29399999999999998,"15min":0.28599999999999998},"max":{"1min":162.93000000000001,"5min":162.93000000000001,"15min":162.93000000000001},"last":0.44500000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:14:12 2023","interfaces":[{"interface":"back","average":{"1min":11.064,"5min":8.5009999999999994,"15min":7.375},"min":{"1min":0.33000000000000002,"5min":0.318,"15min":0.307},"max":{"1min":81.317999999999998,"5min":250.744,"15min":250.744},"last":0.371},{"interface":"front","average":{"1min":9.3520000000000003,"5min":7.9539999999999997,"15min":7.1600000000000001},"min":{"1min":0.374,"5min":0.33600000000000002,"15min":0.314},"max":{"1min":81.233000000000004,"5min":250.83799999999999,"15min":250.83799999999999},"last":0.41799999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:13:47 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":1.0740000000000001,"15min":1.681},"min":{"1min":0.28100000000000003,"5min":0.24399999999999999,"15min":0.24199999999999999},"max":{"1min":9.0869999999999997,"5min":9.0869999999999997,"15min":151.35599999999999},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.97,"5min":1.175,"15min":1.7350000000000001},"min":{"1min":0.33500000000000002,"5min":0.249,"15min":0.23999999999999999},"max":{"1min":7.8849999999999998,"5min":7.8849999999999998,"15min":149.15799999999999},"last":0.316}]},{"osd":5,"last update":"Mon Oct 9 06:14:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":0.94299999999999995,"15min":1.1040000000000001},"min":{"1min":0.315,"5min":0.28000000000000003,"15min":0.23000000000000001},"max":{"1min":8.3829999999999991,"5min":9.1959999999999997,"15min":15.342000000000001},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.95699999999999996,"5min":1.03,"15min":1.1579999999999999},"min":{"1min":0.316,"5min":0.28699999999999998,"15min":0.255},"max":{"1min":2.8450000000000002,"5min":8.9809999999999999,"15min":15.358000000000001},"last":0.42799999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:13:47 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":1.105,"15min":1.2070000000000001},"min":{"1min":0.372,"5min":0.30299999999999999,"15min":0.27000000000000002},"max":{"1min":5.819,"5min":5.819,"15min":21.98},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.79400000000000004,"5min":1.002,"15min":1.1559999999999999},"min":{"1min":0.39300000000000002,"5min":0.34200000000000003,"15min":0.33100000000000002},"max":{"1min":2.8570000000000002,"5min":5.3710000000000004,"15min":18.241},"last":0.54900000000000004}]}]},{"osd":1,"up_from":89,"seq":382252089643,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1239660,"kb_used_data":653092,"kb_used_omap":9,"kb_used_meta":586550,"kb_avail":92505492,"statfs":{"total":95995035648,"available":94725623808,"internally_reserved":0,"allocated":668766208,"data_stored":665632623,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":600628201},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,0,0,0,4,12],"upper_bound":1024},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:35 2023","interfaces":[{"interface":"back","average":{"1min":1.7809999999999999,"5min":1.946,"15min":2.1659999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.613,"5min":10.613,"15min":10.613},"last":0.318},{"interface":"front","average":{"1min":2.79,"5min":2.391,"15min":2.3570000000000002},"min":{"1min":0.254,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":13.121,"5min":24.239999999999998,"15min":24.239999999999998},"last":0.41299999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:13:44 2023","interfaces":[{"interface":"back","average":{"1min":2.7469999999999999,"5min":1.9850000000000001,"15min":2.4660000000000002},"min":{"1min":0.27800000000000002,"5min":0.21099999999999999,"15min":0.20799999999999999},"max":{"1min":12.818,"5min":15.522,"15min":129.053},"last":0.39500000000000002},{"interface":"front","average":{"1min":2.3540000000000001,"5min":2.4900000000000002,"15min":2.6629999999999998},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.20300000000000001},"max":{"1min":8.9949999999999992,"5min":13.539999999999999,"15min":129.077},"last":0.35999999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:14:13 2023","interfaces":[{"interface":"back","average":{"1min":5.1669999999999998,"5min":4.484,"15min":5.0309999999999997},"min":{"1min":0.28000000000000003,"5min":0.26800000000000002,"15min":0.24199999999999999},"max":{"1min":48.177999999999997,"5min":113.322,"15min":213.654},"last":0.53300000000000003},{"interface":"front","average":{"1min":4.3879999999999999,"5min":4.6349999999999998,"15min":5.0250000000000004},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.27400000000000002},"max":{"1min":48.216999999999999,"5min":113.38200000000001,"15min":213.77199999999999},"last":0.63900000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:13:44 2023","interfaces":[{"interface":"back","average":{"1min":9.1400000000000006,"5min":3.1629999999999998,"15min":2.133},"min":{"1min":0.40400000000000003,"5min":0.311,"15min":0.30399999999999999},"max":{"1min":93.747,"5min":93.747,"15min":93.747},"last":0.55400000000000005},{"interface":"front","average":{"1min":9.8000000000000007,"5min":3.8359999999999999,"15min":2.6000000000000001},"min":{"1min":0.32800000000000001,"5min":0.308,"15min":0.308},"max":{"1min":97.852000000000004,"5min":97.852000000000004,"15min":97.852000000000004},"last":0.495}]},{"osd":5,"last update":"Mon Oct 9 06:13:57 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.6220000000000001,"15min":1.5680000000000001},"min":{"1min":0.32200000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.4850000000000003,"5min":9.3109999999999999,"15min":36.356999999999999},"last":0.69399999999999995},{"interface":"front","average":{"1min":1.383,"5min":2.5870000000000002,"15min":2.27},"min":{"1min":0.34599999999999997,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.2619999999999996,"5min":17.603999999999999,"15min":36.268000000000001},"last":0.65400000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:13:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":1.7410000000000001,"15min":1.7789999999999999},"min":{"1min":0.38500000000000001,"5min":0.33700000000000002,"15min":0.318},"max":{"1min":5.5170000000000003,"5min":9.282,"15min":52.207000000000001},"last":0.67100000000000004},{"interface":"front","average":{"1min":1.901,"5min":2.4180000000000001,"15min":2.3559999999999999},"min":{"1min":0.38500000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":8.7159999999999993,"5min":15.384,"15min":51.212000000000003},"last":0.45400000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:13:57 2023","interfaces":[{"interface":"back","average":{"1min":5.7400000000000002,"5min":3.3849999999999998,"15min":2.9329999999999998},"min":{"1min":0.39800000000000002,"5min":0.374,"15min":0.374},"max":{"1min":56.881,"5min":56.881,"15min":108.553},"last":0.59599999999999997},{"interface":"front","average":{"1min":5.7210000000000001,"5min":2.6360000000000001,"15min":2.3980000000000001},"min":{"1min":0.46000000000000002,"5min":0.38,"15min":0.38},"max":{"1min":56.941000000000003,"5min":56.941000000000003,"15min":108.56999999999999},"last":0.61899999999999999}]}]},{"osd":0,"up_from":456,"seq":1958505087038,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2641668,"kb_used_data":2383676,"kb_used_omap":32,"kb_used_meta":257951,"kb_avail":91103484,"statfs":{"total":95995035648,"available":93289967616,"internally_reserved":0,"allocated":2440884224,"data_stored":2437967471,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32794,"internal_metadata":264142822},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":5.7939999999999996,"15min":9.3539999999999992},"min":{"1min":0.23999999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":5.157,"5min":166.833,"15min":166.833},"last":0.377},{"interface":"front","average":{"1min":3.29,"5min":6.1779999999999999,"15min":8.7200000000000006},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":39.424999999999997,"5min":166.81100000000001,"15min":166.81100000000001},"last":0.48599999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":3.0449999999999999,"5min":2.9129999999999998,"15min":3.677},"min":{"1min":0.29999999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":21.648,"5min":21.648,"15min":21.648},"last":1.76},{"interface":"front","average":{"1min":2.1800000000000002,"5min":2.6600000000000001,"15min":3.488},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":11.890000000000001,"5min":20.152000000000001,"15min":20.152000000000001},"last":1.048}]},{"osd":3,"last update":"Mon Oct 9 06:13:32 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":7.532,"15min":13.125999999999999},"min":{"1min":0.26500000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.9029999999999996,"5min":203.59299999999999,"15min":203.59299999999999},"last":1.1220000000000001},{"interface":"front","average":{"1min":3.4969999999999999,"5min":7.657,"15min":12.565},"min":{"1min":0.35299999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":18.192,"5min":202.96600000000001,"15min":202.96600000000001},"last":1.617}]},{"osd":4,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":2.444,"5min":2.0659999999999998,"15min":2.294},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":23.515999999999998,"5min":23.515999999999998,"15min":23.515999999999998},"last":0.40699999999999997},{"interface":"front","average":{"1min":4.2149999999999999,"5min":2.5169999999999999,"15min":2.1669999999999998},"min":{"1min":0.35799999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":39.302,"5min":39.302,"15min":39.302},"last":0.54300000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":3.1899999999999999,"5min":3.7200000000000002,"15min":2.7000000000000002},"min":{"1min":0.32200000000000001,"5min":0.316,"15min":0.316},"max":{"1min":39.369999999999997,"5min":161.286,"15min":161.286},"last":0.51900000000000002},{"interface":"front","average":{"1min":2.1110000000000002,"5min":3.532,"15min":2.8260000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":13.731999999999999,"5min":161.31800000000001,"15min":161.31800000000001},"last":1.583}]},{"osd":6,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":2.1190000000000002,"15min":2.3570000000000002},"min":{"1min":0.38900000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":13.795,"5min":13.795,"15min":13.795},"last":1.544},{"interface":"front","average":{"1min":1.3759999999999999,"5min":1.931,"15min":2.5350000000000001},"min":{"1min":0.44700000000000001,"5min":0.372,"15min":0.372},"max":{"1min":5.274,"5min":10.233000000000001,"15min":10.233000000000001},"last":0.623}]},{"osd":7,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":2.423,"5min":3.5070000000000001,"15min":2.8570000000000002},"min":{"1min":0.44900000000000001,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":13.821,"5min":123.685,"15min":123.685},"last":1.502},{"interface":"front","average":{"1min":1.5049999999999999,"5min":3.214,"15min":2.9129999999999998},"min":{"1min":0.44,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":5.3200000000000003,"5min":122.482,"15min":122.482},"last":0.65300000000000002}]}]},{"osd":2,"up_from":94,"seq":403726926135,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2248560,"kb_used_data":1487528,"kb_used_omap":81,"kb_used_meta":760878,"kb_avail":91496592,"statfs":{"total":95995035648,"available":93692510208,"internally_reserved":0,"allocated":1523228672,"data_stored":1520444850,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":779139290},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":7.7000000000000002,"5min":3.5070000000000001,"15min":3.2679999999999998},"min":{"1min":0.221,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":56.213000000000001,"5min":56.213000000000001,"15min":56.213000000000001},"last":83.652000000000001},{"interface":"front","average":{"1min":6.4610000000000003,"5min":2.8140000000000001,"15min":2.4329999999999998},"min":{"1min":0.221,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":50.764000000000003,"5min":50.764000000000003,"15min":50.764000000000003},"last":77.578999999999994}]},{"osd":1,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":5.0819999999999999,"15min":4.4909999999999997},"min":{"1min":0.27800000000000002,"5min":0.248,"15min":0.19400000000000001},"max":{"1min":4.258,"5min":160.45500000000001,"15min":222.59399999999999},"last":78.921999999999997},{"interface":"front","average":{"1min":2.1379999999999999,"5min":6.0540000000000003,"15min":4.9480000000000004},"min":{"1min":0.27600000000000002,"5min":0.27200000000000002,"15min":0.20200000000000001},"max":{"1min":10.345000000000001,"5min":163.697,"15min":224.619},"last":81.292000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:13:58 2023","interfaces":[{"interface":"back","average":{"1min":12.108000000000001,"5min":7.2910000000000004,"15min":5.101},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23000000000000001},"max":{"1min":122.205,"5min":122.205,"15min":144.88200000000001},"last":78.385000000000005},{"interface":"front","average":{"1min":11.619,"5min":6.952,"15min":5.0369999999999999},"min":{"1min":0.30199999999999999,"5min":0.252,"15min":0.252},"max":{"1min":122.45,"5min":122.45,"15min":140.79499999999999},"last":78.436000000000007}]},{"osd":4,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":3.1659999999999999,"15min":2.226},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.28299999999999997},"max":{"1min":8.9550000000000001,"5min":107.51300000000001,"15min":107.51300000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.081,"5min":2.6040000000000001,"15min":2.0579999999999998},"min":{"1min":0.36599999999999999,"5min":0.34100000000000003,"15min":0.31},"max":{"1min":3.4039999999999999,"5min":108.768,"15min":108.768},"last":0.40600000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":1.73,"5min":1.502,"15min":1.5549999999999999},"min":{"1min":0.376,"5min":0.34399999999999997,"15min":0.32600000000000001},"max":{"1min":11.015000000000001,"5min":11.015000000000001,"15min":28.48},"last":0.32800000000000001},{"interface":"front","average":{"1min":1.2430000000000001,"5min":1.7150000000000001,"15min":1.728},"min":{"1min":0.379,"5min":0.316,"15min":0.28899999999999998},"max":{"1min":6.9180000000000001,"5min":9.8699999999999992,"15min":30.318999999999999},"last":0.439}]},{"osd":6,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":1.603,"5min":2.218,"15min":2.085},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.314},"max":{"1min":10.301,"5min":14.962,"15min":30.420000000000002},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.8540000000000001,"5min":1.621,"15min":1.7749999999999999},"min":{"1min":0.432,"5min":0.36099999999999999,"15min":0.32200000000000001},"max":{"1min":11.103,"5min":11.103,"15min":30.713000000000001},"last":0.35799999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:13:39 2023","interfaces":[{"interface":"back","average":{"1min":1.8300000000000001,"5min":2.9089999999999998,"15min":2.2349999999999999},"min":{"1min":0.40799999999999997,"5min":0.313,"15min":0.313},"max":{"1min":11.065,"5min":124.069,"15min":124.069},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.0449999999999999,"5min":2.8119999999999998,"15min":2.258},"min":{"1min":0.376,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.4210000000000003,"5min":124.08799999999999,"15min":124.08799999999999},"last":0.372}]}]},{"osd":3,"up_from":99,"seq":425201762608,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2771536,"kb_used_data":1805704,"kb_used_omap":7,"kb_used_meta":965752,"kb_avail":90973616,"statfs":{"total":95995035648,"available":93156982784,"internally_reserved":0,"allocated":1849040896,"data_stored":1845927464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":988930664},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:36 2023","interfaces":[{"interface":"back","average":{"1min":3.5409999999999999,"5min":1.9590000000000001,"15min":2.0590000000000002},"min":{"1min":0.222,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":30.841000000000001,"5min":30.841000000000001,"15min":30.841000000000001},"last":0.25700000000000001},{"interface":"front","average":{"1min":3.4769999999999999,"5min":2.8650000000000002,"15min":3.3050000000000002},"min":{"1min":0.27900000000000003,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":28.652000000000001,"5min":28.652000000000001,"15min":28.652000000000001},"last":0.54500000000000004}]},{"osd":1,"last update":"Mon Oct 9 06:14:10 2023","interfaces":[{"interface":"back","average":{"1min":4.1269999999999998,"5min":2.218,"15min":2.9510000000000001},"min":{"1min":0.253,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":21.934000000000001,"5min":22.579999999999998,"15min":207.75299999999999},"last":0.28799999999999998},{"interface":"front","average":{"1min":4.3049999999999997,"5min":3.54,"15min":3.9649999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.23200000000000001},"max":{"1min":21.809000000000001,"5min":30.524000000000001,"15min":207.12200000000001},"last":0.39600000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:14:10 2023","interfaces":[{"interface":"back","average":{"1min":2.3889999999999998,"5min":2.4609999999999999,"15min":2.6549999999999998},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.223},"max":{"1min":12.561,"5min":30.547999999999998,"15min":107.01300000000001},"last":0.499},{"interface":"front","average":{"1min":2.6640000000000001,"5min":3.0899999999999999,"15min":3.0699999999999998},"min":{"1min":0.313,"5min":0.25600000000000001,"15min":0.23899999999999999},"max":{"1min":11.749000000000001,"5min":25.984000000000002,"15min":109.807},"last":0.63900000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":6.8099999999999996,"5min":2.8759999999999999,"15min":3.5569999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.254},"max":{"1min":95.739000000000004,"5min":95.739000000000004,"15min":99.876000000000005},"last":0.35599999999999998},{"interface":"front","average":{"1min":6.8259999999999996,"5min":3.399,"15min":3.7210000000000001},"min":{"1min":0.373,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":97.941999999999993,"5min":97.941999999999993,"15min":99.914000000000001},"last":0.51000000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:14:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2989999999999999,"5min":2.7200000000000002,"15min":2.0649999999999999},"min":{"1min":0.40500000000000003,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.3970000000000002,"5min":74.706000000000003,"15min":74.706000000000003},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.202,"5min":3.4420000000000002,"15min":3.2759999999999998},"min":{"1min":0.32800000000000001,"5min":0.32500000000000001,"15min":0.28799999999999998},"max":{"1min":4.0449999999999999,"5min":74.685000000000002,"15min":235.56},"last":0.38800000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:14:10 2023","interfaces":[{"interface":"back","average":{"1min":1.734,"5min":1.351,"15min":4.3070000000000004},"min":{"1min":0.46500000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.4210000000000003,"5min":6.8739999999999997,"15min":233.26599999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":2.266,"5min":2.2069999999999999,"15min":4.7619999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.33600000000000002},"max":{"1min":12.471,"5min":26.890000000000001,"15min":233.24100000000001},"last":0.46200000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:14:10 2023","interfaces":[{"interface":"back","average":{"1min":7.8600000000000003,"5min":4.2569999999999997,"15min":5.2229999999999999},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.35699999999999998},"max":{"1min":94.400000000000006,"5min":94.400000000000006,"15min":230.69399999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":8.6210000000000004,"5min":4.5999999999999996,"15min":5.4000000000000004},"min":{"1min":0.52600000000000002,"5min":0.44500000000000001,"15min":0.32600000000000001},"max":{"1min":94.195999999999998,"5min":94.195999999999998,"15min":230.67500000000001},"last":0.47299999999999998}]}]},{"osd":4,"up_from":322,"seq":1382979469471,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3014452,"kb_used_data":2687980,"kb_used_omap":35,"kb_used_meta":326428,"kb_avail":90730700,"statfs":{"total":95995035648,"available":92908236800,"internally_reserved":0,"allocated":2752491520,"data_stored":2750933216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36552,"internal_metadata":334262584},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:31 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.3500000000000001,"15min":1.4419999999999999},"min":{"1min":0.222,"5min":0.216,"15min":0.216},"max":{"1min":4.6840000000000002,"5min":6.8470000000000004,"15min":6.8470000000000004},"last":1.359},{"interface":"front","average":{"1min":1.504,"5min":1.4159999999999999,"15min":1.536},"min":{"1min":0.26000000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":8.7699999999999996,"5min":8.7699999999999996,"15min":8.7699999999999996},"last":1.246}]},{"osd":1,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":1.778,"5min":1.883,"15min":2.2839999999999998},"min":{"1min":0.28499999999999998,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.3449999999999998,"5min":18.884,"15min":78.891000000000005},"last":0.29799999999999999},{"interface":"front","average":{"1min":1.4390000000000001,"5min":1.8300000000000001,"15min":2.2309999999999999},"min":{"1min":0.25600000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.946999999999999,"5min":18.617999999999999,"15min":78.921000000000006},"last":0.69199999999999995}]},{"osd":2,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":14.574999999999999,"5min":4.4219999999999997,"15min":2.613},"min":{"1min":0.32500000000000001,"5min":0.29199999999999998,"15min":0.27200000000000002},"max":{"1min":245.166,"5min":245.166,"15min":245.166},"last":0.41699999999999998},{"interface":"front","average":{"1min":14.550000000000001,"5min":4.2469999999999999,"15min":2.5600000000000001},"min":{"1min":0.38500000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":245.119,"5min":245.119,"15min":245.119},"last":0.93600000000000005}]},{"osd":3,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":24.687999999999999,"5min":6.6550000000000002,"15min":3.875},"min":{"1min":0.438,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":246.39400000000001,"5min":246.39400000000001,"15min":246.39400000000001},"last":6.6630000000000003},{"interface":"front","average":{"1min":23.437999999999999,"5min":6.21,"15min":3.444},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":246.99000000000001,"5min":246.99000000000001,"15min":246.99000000000001},"last":1.165}]},{"osd":5,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.93799999999999994,"15min":0.98999999999999999},"min":{"1min":0.27900000000000003,"5min":0.216,"15min":0.216},"max":{"1min":2.5600000000000001,"5min":7.5999999999999996,"15min":9.2690000000000001},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.1719999999999999,"5min":0.94099999999999995,"15min":1.165},"min":{"1min":0.33700000000000002,"5min":0.245,"15min":0.245},"max":{"1min":5.3479999999999999,"5min":9.1110000000000007,"15min":12.087},"last":0.34200000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":1.036,"15min":2.3010000000000002},"min":{"1min":0.32100000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.4279999999999999,"5min":7.641,"15min":95.492999999999995},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.1759999999999999,"5min":1.1990000000000001,"15min":2.4390000000000001},"min":{"1min":0.372,"5min":0.312,"15min":0.27900000000000003},"max":{"1min":5.5469999999999997,"5min":9.3239999999999998,"15min":95.664000000000001},"last":0.36499999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":1.3069999999999999,"15min":2.6920000000000002},"min":{"1min":0.38,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":8.1229999999999993,"5min":17.286000000000001,"15min":79.173000000000002},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.151,"15min":2.294},"min":{"1min":0.33000000000000002,"5min":0.251,"15min":0.251},"max":{"1min":5.1779999999999999,"5min":16.097999999999999,"15min":78.894000000000005},"last":0.72499999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757691,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2295396,"kb_used_data":1409180,"kb_used_omap":0,"kb_used_meta":886208,"kb_avail":91449756,"statfs":{"total":95995035648,"available":93644550144,"internally_reserved":0,"allocated":1443000320,"data_stored":1441454664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":907476992},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:13:33 2023","interfaces":[{"interface":"back","average":{"1min":8.1929999999999996,"5min":3.508,"15min":2.77},"min":{"1min":0.24399999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":115.92400000000001,"5min":115.92400000000001,"15min":115.92400000000001},"last":0.29099999999999998},{"interface":"front","average":{"1min":8.4220000000000006,"5min":3.855,"15min":3.4340000000000002},"min":{"1min":0.29299999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":115.904,"5min":115.904,"15min":115.904},"last":0.33700000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:14:03 2023","interfaces":[{"interface":"back","average":{"1min":1.3560000000000001,"5min":4.6040000000000001,"15min":4.4950000000000001},"min":{"1min":0.26500000000000001,"5min":0.248,"15min":0.22600000000000001},"max":{"1min":5.4880000000000004,"5min":151.78200000000001,"15min":151.78200000000001},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.9990000000000001,"5min":4.6900000000000004,"15min":4.6059999999999999},"min":{"1min":0.32300000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":9.7750000000000004,"5min":152.251,"15min":152.251},"last":0.36899999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:14:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":2.9390000000000001,"15min":2.9529999999999998},"min":{"1min":0.33600000000000002,"5min":0.26600000000000001,"15min":0.254},"max":{"1min":2.5009999999999999,"5min":117.97199999999999,"15min":129.21199999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.96799999999999997,"5min":3.2330000000000001,"15min":3.0129999999999999},"min":{"1min":0.35599999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.6160000000000001,"5min":118.52,"15min":129.042},"last":0.44500000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:14:03 2023","interfaces":[{"interface":"back","average":{"1min":9.4779999999999998,"5min":6.5380000000000003,"15min":6.9169999999999998},"min":{"1min":0.316,"5min":0.30299999999999999,"15min":0.29499999999999998},"max":{"1min":93.338999999999999,"5min":143.31200000000001,"15min":143.31200000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":9.6940000000000008,"5min":7.1029999999999998,"15min":6.9989999999999997},"min":{"1min":0.40300000000000002,"5min":0.34499999999999997,"15min":0.29399999999999998},"max":{"1min":93.435000000000002,"5min":140.00899999999999,"15min":140.00899999999999},"last":0.52700000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:13:52 2023","interfaces":[{"interface":"back","average":{"1min":4.5039999999999996,"5min":1.8360000000000001,"15min":1.298},"min":{"1min":0.26500000000000001,"5min":0.246,"15min":0.245},"max":{"1min":68.001999999999995,"5min":68.001999999999995,"15min":68.001999999999995},"last":0.40600000000000003},{"interface":"front","average":{"1min":4.2880000000000003,"5min":1.5529999999999999,"15min":1.228},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26000000000000001},"max":{"1min":61.743000000000002,"5min":61.743000000000002,"15min":61.743000000000002},"last":0.46100000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:14:03 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":1.0149999999999999,"15min":1.48},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.251},"max":{"1min":2.3439999999999999,"5min":7.4980000000000002,"15min":126.89400000000001},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.099,"15min":1.5620000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.26700000000000002},"max":{"1min":2.4340000000000002,"5min":7.7140000000000004,"15min":126.616},"last":0.55500000000000005}]},{"osd":7,"last update":"Mon Oct 9 06:13:52 2023","interfaces":[{"interface":"back","average":{"1min":4.5670000000000002,"5min":3.2440000000000002,"15min":2.7999999999999998},"min":{"1min":0.40000000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":64.766000000000005,"5min":157.471,"15min":157.471},"last":0.48499999999999999},{"interface":"front","average":{"1min":4.7830000000000004,"5min":3.2519999999999998,"15min":2.7189999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.29599999999999999},"max":{"1min":67.956999999999994,"5min":157.51599999999999,"15min":157.51599999999999},"last":0.501}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4889,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2413170688,"data_stored":2413166616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":653656064,"data_stored":653656064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1510408192,"data_stored":1510408192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1819480064,"data_stored":1819475992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2711687168,"data_stored":2711683096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1426456576,"data_stored":1426456576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1238503424,"data_stored":1238499352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":779485184,"data_stored":779485184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:14:36.842 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T06:14:36.843 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:14:36.961 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:36.995 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:36 smithi064 ceph-mon[212750]: pgmap v1387: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 116 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 441 op/s; 9480/161010 objects misplaced (5.888%); 16 MiB/s, 256 objects/s recovering 2023-10-09T06:14:36.996 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:36 smithi064 ceph-mon[220044]: pgmap v1387: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 116 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 441 op/s; 9480/161010 objects misplaced (5.888%); 16 MiB/s, 256 objects/s recovering 2023-10-09T06:14:37.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:36 smithi156 ceph-mon[195101]: pgmap v1387: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 116 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 441 op/s; 9480/161010 objects misplaced (5.888%); 16 MiB/s, 256 objects/s recovering 2023-10-09T06:14:37.156 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:14:37.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:37.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:14:37.267 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:37.300 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:37.363 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:37.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:37.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:37.475 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:14:37.560 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:38.322 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:14:38.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:38.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:14:38.430 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:14:38.485 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:14:38.510 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:38.661 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:14:38.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:38.670 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:14:38.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:14:38.746 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:14:38.771 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:14:38.814 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:38.905 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:14:38.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:38.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:14:38.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:38 smithi064 ceph-mon[212750]: pgmap v1388: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 7703/151347 objects misplaced (5.090%); 21 MiB/s, 340 objects/s recovering 2023-10-09T06:14:38.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:38 smithi064 ceph-mon[220044]: pgmap v1388: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 7703/151347 objects misplaced (5.090%); 21 MiB/s, 340 objects/s recovering 2023-10-09T06:14:39.016 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:14:39.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:38 smithi156 ceph-mon[195101]: pgmap v1388: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 7703/151347 objects misplaced (5.090%); 21 MiB/s, 340 objects/s recovering 2023-10-09T06:14:39.131 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:39.706 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:14:39.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:39.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:14:39.819 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:14:39.859 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:39.934 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:14:39.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:39.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:14:40.047 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:14:40.128 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:40.214 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:14:40.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:40.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:14:40.320 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:14:40.376 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:40.497 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:14:40.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:40.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:14:40.604 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:14:40.636 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:40.820 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:14:40.978 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:40 smithi064 ceph-mon[212750]: pgmap v1389: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 7703/151347 objects misplaced (5.090%); 19 MiB/s, 308 objects/s recovering 2023-10-09T06:14:40.978 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:40 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3581825603' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:14:40.978 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:40 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:14:40.978 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:40 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:14:40.978 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:40 smithi064 ceph-mon[220044]: pgmap v1389: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 7703/151347 objects misplaced (5.090%); 19 MiB/s, 308 objects/s recovering 2023-10-09T06:14:40.978 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:40 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3581825603' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:14:40.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:40 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:14:40.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:40 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:14:41.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:40 smithi156 ceph-mon[195101]: pgmap v1389: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 7703/151347 objects misplaced (5.090%); 19 MiB/s, 308 objects/s recovering 2023-10-09T06:14:41.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:40 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3581825603' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:14:41.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:40 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:14:41.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:40 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:14:41.236 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:14:41.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:41.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:14:41.343 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:14:41.382 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:41.441 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:14:41.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:41.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:14:41.545 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:41.573 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:41.668 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:41.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:41.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:41.775 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:14:41.855 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:41.958 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:14:41.964 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:41 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:14:41.964 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:41 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T06:14:41.964 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:41 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:14:41.964 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:41 smithi156 ceph-mon[195101]: osdmap e547: 8 total, 8 up, 8 in 2023-10-09T06:14:41.965 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:41 smithi156 ceph-mon[195101]: failed to encode map e547 with expected crc 2023-10-09T06:14:41.965 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:41 smithi156 ceph-mon[195101]: failed to encode map e547 with expected crc 2023-10-09T06:14:41.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:41.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:14:42.067 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:14:42.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:41 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:14:42.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:41 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:14:42.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:41 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:14:42.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:41 smithi064 ceph-mon[212750]: osdmap e547: 8 total, 8 up, 8 in 2023-10-09T06:14:42.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:41 smithi064 ceph-mon[212750]: failed to encode map e547 with expected crc 2023-10-09T06:14:42.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:41 smithi064 ceph-mon[212750]: failed to encode map e547 with expected crc 2023-10-09T06:14:42.177 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:41 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:14:42.177 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:41 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T06:14:42.178 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:41 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:14:42.178 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:41 smithi064 ceph-mon[220044]: osdmap e547: 8 total, 8 up, 8 in 2023-10-09T06:14:42.178 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:41 smithi064 ceph-mon[220044]: failed to encode map e547 with expected crc 2023-10-09T06:14:42.178 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:41 smithi064 ceph-mon[220044]: failed to encode map e547 with expected crc 2023-10-09T06:14:42.277 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:42.740 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:14:42.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:42.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:14:42.844 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:14:42.872 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:43.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:42 smithi156 ceph-mon[195101]: pgmap v1391: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 7301/145548 objects misplaced (5.016%); 20 MiB/s, 322 objects/s recovering 2023-10-09T06:14:43.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:42 smithi156 ceph-mon[195101]: osdmap e548: 8 total, 8 up, 8 in 2023-10-09T06:14:43.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:42 smithi064 ceph-mon[212750]: pgmap v1391: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 7301/145548 objects misplaced (5.016%); 20 MiB/s, 322 objects/s recovering 2023-10-09T06:14:43.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:42 smithi064 ceph-mon[212750]: osdmap e548: 8 total, 8 up, 8 in 2023-10-09T06:14:43.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:42 smithi064 ceph-mon[220044]: pgmap v1391: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 7301/145548 objects misplaced (5.016%); 20 MiB/s, 322 objects/s recovering 2023-10-09T06:14:43.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:42 smithi064 ceph-mon[220044]: osdmap e548: 8 total, 8 up, 8 in 2023-10-09T06:14:43.495 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:14:43.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:43.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:14:43.600 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:14:43.631 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:43.867 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:14:43.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:43.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:14:43.979 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:44.024 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:44.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:43 smithi156 ceph-mon[195101]: failed to encode map e548 with expected crc 2023-10-09T06:14:44.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:43 smithi156 ceph-mon[195101]: failed to encode map e548 with expected crc 2023-10-09T06:14:44.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:43 smithi156 ceph-mon[195101]: failed to encode map e548 with expected crc 2023-10-09T06:14:44.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:43 smithi156 ceph-mon[195101]: failed to encode map e548 with expected crc 2023-10-09T06:14:44.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:43 smithi156 ceph-mon[195101]: failed to encode map e548 with expected crc 2023-10-09T06:14:44.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:43 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:14:44.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:43 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:44.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:43 smithi156 ceph-mon[195101]: osdmap e549: 8 total, 8 up, 8 in 2023-10-09T06:14:44.144 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:43 smithi064 ceph-mon[212750]: failed to encode map e548 with expected crc 2023-10-09T06:14:44.144 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:43 smithi064 ceph-mon[212750]: failed to encode map e548 with expected crc 2023-10-09T06:14:44.144 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:43 smithi064 ceph-mon[212750]: failed to encode map e548 with expected crc 2023-10-09T06:14:44.145 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:43 smithi064 ceph-mon[212750]: failed to encode map e548 with expected crc 2023-10-09T06:14:44.145 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:43 smithi064 ceph-mon[212750]: failed to encode map e548 with expected crc 2023-10-09T06:14:44.145 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:43 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:14:44.145 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:43 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:44.145 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:43 smithi064 ceph-mon[212750]: osdmap e549: 8 total, 8 up, 8 in 2023-10-09T06:14:44.145 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:43 smithi064 ceph-mon[220044]: failed to encode map e548 with expected crc 2023-10-09T06:14:44.145 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:43 smithi064 ceph-mon[220044]: failed to encode map e548 with expected crc 2023-10-09T06:14:44.145 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:43 smithi064 ceph-mon[220044]: failed to encode map e548 with expected crc 2023-10-09T06:14:44.146 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:43 smithi064 ceph-mon[220044]: failed to encode map e548 with expected crc 2023-10-09T06:14:44.146 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:43 smithi064 ceph-mon[220044]: failed to encode map e548 with expected crc 2023-10-09T06:14:44.146 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:43 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:14:44.146 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:43 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:44.146 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:43 smithi064 ceph-mon[220044]: osdmap e549: 8 total, 8 up, 8 in 2023-10-09T06:14:44.179 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:14:44.179 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:14:44.197 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:44.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:44.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:44.229 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:14:44.268 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:14:44.303 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:14:44.303 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:14:44.334 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:14:44.348 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:44.365 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:14:44.400 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:14:44.430 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:14:44.458 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:14:44.460 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:14:44.460 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:14:44.460 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:14:44.460 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:14:44.460 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:14:44.460 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:14:44.460 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:14:44.460 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:14:44.878 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:14:44.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:44.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:14:44.983 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:14:45.045 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:45.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:44 smithi156 ceph-mon[195101]: pgmap v1393: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 5869/140820 objects misplaced (4.168%); 20 MiB/s, 315 objects/s recovering 2023-10-09T06:14:45.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:44 smithi156 ceph-mon[195101]: failed to encode map e549 with expected crc 2023-10-09T06:14:45.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:44 smithi156 ceph-mon[195101]: failed to encode map e549 with expected crc 2023-10-09T06:14:45.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:44 smithi156 ceph-mon[195101]: failed to encode map e549 with expected crc 2023-10-09T06:14:45.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:44 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/189390835' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:14:45.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:14:45.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:14:45.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:44 smithi156 ceph-mon[195101]: osdmap e550: 8 total, 8 up, 8 in 2023-10-09T06:14:45.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:44 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1635533795' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:14:45.157 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:14:45.157 INFO:teuthology.orchestra.run.smithi064.stdout:{"fsid":"44c77acc-6665-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1746,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":550,"num_osds":8,"num_up_osds":8,"osd_up_since":1696831766,"num_in_osds":8,"osd_in_since":1696831662,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":116},{"state_name":"active+remapped+backfilling","count":2},{"state_name":"active+remapped+backfill_wait","count":1}],"num_pgs":119,"num_pools":5,"num_objects":46940,"data_bytes":3075244171,"bytes_used":17041551360,"bytes_avail":750918733824,"bytes_total":767960285184,"misplaced_objects":5869,"misplaced_total":140820,"misplaced_ratio":0.041677318562704159,"recovering_objects_per_sec":315,"recovering_bytes_per_sec":20705379,"recovering_keys_per_sec":0,"num_objects_recovered":2530,"num_bytes_recovered":165806080,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":840},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi064.gmrwsy","status":"up:active","gid":24551}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"}},"servicemap":{"epoch":26,"modified":"2023-10-09T06:10:27.041106+0000","services":{}},"progress_events":{"28a7925c-af7b-4940-a0f9-d3ed10a1f964":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"ca7f429f-bb9e-4fb1-944c-f3f1e3d51bf1":{"message":"Global Recovery Event (105s)\n [===========================.] (remaining: 2s)","progress":0.97478991746902466,"add_to_ceph_s":true}}} 2023-10-09T06:14:45.165 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:14:45.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:45.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:14:45.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:44 smithi064 ceph-mon[212750]: pgmap v1393: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 5869/140820 objects misplaced (4.168%); 20 MiB/s, 315 objects/s recovering 2023-10-09T06:14:45.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:44 smithi064 ceph-mon[212750]: failed to encode map e549 with expected crc 2023-10-09T06:14:45.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:44 smithi064 ceph-mon[212750]: failed to encode map e549 with expected crc 2023-10-09T06:14:45.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:44 smithi064 ceph-mon[212750]: failed to encode map e549 with expected crc 2023-10-09T06:14:45.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:44 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/189390835' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:14:45.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:14:45.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:14:45.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:44 smithi064 ceph-mon[212750]: osdmap e550: 8 total, 8 up, 8 in 2023-10-09T06:14:45.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:44 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1635533795' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:14:45.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:44 smithi064 ceph-mon[220044]: pgmap v1393: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 5869/140820 objects misplaced (4.168%); 20 MiB/s, 315 objects/s recovering 2023-10-09T06:14:45.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:44 smithi064 ceph-mon[220044]: failed to encode map e549 with expected crc 2023-10-09T06:14:45.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:44 smithi064 ceph-mon[220044]: failed to encode map e549 with expected crc 2023-10-09T06:14:45.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:44 smithi064 ceph-mon[220044]: failed to encode map e549 with expected crc 2023-10-09T06:14:45.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:44 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/189390835' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:14:45.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:14:45.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:14:45.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:44 smithi064 ceph-mon[220044]: osdmap e550: 8 total, 8 up, 8 in 2023-10-09T06:14:45.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:44 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1635533795' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:14:45.272 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:14:45.300 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:45.392 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:14:45.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:45.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:14:45.493 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:14:45.500 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:14:45.554 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:45.555 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:14:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:14:45] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.43.0" 2023-10-09T06:14:45.622 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:14:45.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:45.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:14:45.730 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:14:45.840 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:46.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:45 smithi156 ceph-mon[195101]: failed to encode map e550 with expected crc 2023-10-09T06:14:46.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:45 smithi156 ceph-mon[195101]: failed to encode map e550 with expected crc 2023-10-09T06:14:46.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:45 smithi156 ceph-mon[195101]: failed to encode map e550 with expected crc 2023-10-09T06:14:46.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:45 smithi156 ceph-mon[195101]: failed to encode map e550 with expected crc 2023-10-09T06:14:46.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:45 smithi064 ceph-mon[220044]: failed to encode map e550 with expected crc 2023-10-09T06:14:46.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:45 smithi064 ceph-mon[220044]: failed to encode map e550 with expected crc 2023-10-09T06:14:46.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:45 smithi064 ceph-mon[220044]: failed to encode map e550 with expected crc 2023-10-09T06:14:46.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:45 smithi064 ceph-mon[220044]: failed to encode map e550 with expected crc 2023-10-09T06:14:46.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:45 smithi064 ceph-mon[212750]: failed to encode map e550 with expected crc 2023-10-09T06:14:46.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:45 smithi064 ceph-mon[212750]: failed to encode map e550 with expected crc 2023-10-09T06:14:46.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:45 smithi064 ceph-mon[212750]: failed to encode map e550 with expected crc 2023-10-09T06:14:46.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:45 smithi064 ceph-mon[212750]: failed to encode map e550 with expected crc 2023-10-09T06:14:46.698 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:14:46.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:46.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:14:46.808 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:46.855 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:46.933 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:46.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:46.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:46.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:46 smithi064 ceph-mon[212750]: pgmap v1396: 119 pgs: 1 peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 116 active+clean; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 5553/139464 objects misplaced (3.982%); 24 MiB/s, 377 objects/s recovering 2023-10-09T06:14:46.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:46 smithi064 ceph-mon[220044]: pgmap v1396: 119 pgs: 1 peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 116 active+clean; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 5553/139464 objects misplaced (3.982%); 24 MiB/s, 377 objects/s recovering 2023-10-09T06:14:47.040 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:14:47.084 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:47.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:46 smithi156 ceph-mon[195101]: pgmap v1396: 119 pgs: 1 peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 116 active+clean; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 5553/139464 objects misplaced (3.982%); 24 MiB/s, 377 objects/s recovering 2023-10-09T06:14:47.186 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:14:47.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:47.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:14:47.225 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:14:47.294 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:14:47.321 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:47.363 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:14:47.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:47.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:14:47.461 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T06:14:47.461 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:14:47.472 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:14:47.502 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:47.542 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:14:47.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:47.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:14:47.616 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:14:47.648 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:14:47.684 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:47.727 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:14:47.756 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:14:47.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:47.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:14:47.862 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:47.910 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:47.984 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:47.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:47.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:48.092 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:14:48.131 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:48.214 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:14:48.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:48.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:14:48.321 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:14:48.372 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:48.452 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:14:48.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:48.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:14:48.564 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:14:48.635 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:48.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:48 smithi156 ceph-mon[195101]: pgmap v1397: 119 pgs: 1 peering, 1 active+remapped+backfilling, 117 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 3940/132900 objects misplaced (2.965%); 26 MiB/s, 419 objects/s recovering 2023-10-09T06:14:48.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:48 smithi156 ceph-mon[195101]: 1.2 deep-scrub starts 2023-10-09T06:14:48.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:48 smithi156 ceph-mon[195101]: 1.2 deep-scrub ok 2023-10-09T06:14:48.894 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:48 smithi064 ceph-mon[212750]: pgmap v1397: 119 pgs: 1 peering, 1 active+remapped+backfilling, 117 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 3940/132900 objects misplaced (2.965%); 26 MiB/s, 419 objects/s recovering 2023-10-09T06:14:48.895 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:48 smithi064 ceph-mon[212750]: 1.2 deep-scrub starts 2023-10-09T06:14:48.895 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:48 smithi064 ceph-mon[212750]: 1.2 deep-scrub ok 2023-10-09T06:14:48.895 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:48 smithi064 ceph-mon[220044]: pgmap v1397: 119 pgs: 1 peering, 1 active+remapped+backfilling, 117 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 3940/132900 objects misplaced (2.965%); 26 MiB/s, 419 objects/s recovering 2023-10-09T06:14:48.895 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:48 smithi064 ceph-mon[220044]: 1.2 deep-scrub starts 2023-10-09T06:14:48.895 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:48 smithi064 ceph-mon[220044]: 1.2 deep-scrub ok 2023-10-09T06:14:48.910 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:14:48.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:48.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:14:49.018 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:14:49.091 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:49.237 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:14:49.653 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:14:49.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:49.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:14:49.758 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:14:49.793 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:49.885 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:14:49.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:49.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:14:49.956 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:14:49.994 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:14:50.075 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:50.324 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:14:50.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:50.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:14:50.440 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:50.498 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:50.583 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:14:50.584 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:50.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:50.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:50.694 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:14:50.801 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:50.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:50 smithi156 ceph-mon[195101]: pgmap v1398: 119 pgs: 1 peering, 1 active+remapped+backfilling, 117 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 407 op/s; 3940/132900 objects misplaced (2.965%); 11 MiB/s, 179 objects/s recovering 2023-10-09T06:14:50.933 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:50 smithi064 ceph-mon[212750]: pgmap v1398: 119 pgs: 1 peering, 1 active+remapped+backfilling, 117 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 407 op/s; 3940/132900 objects misplaced (2.965%); 11 MiB/s, 179 objects/s recovering 2023-10-09T06:14:50.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:50 smithi064 ceph-mon[220044]: pgmap v1398: 119 pgs: 1 peering, 1 active+remapped+backfilling, 117 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 407 op/s; 3940/132900 objects misplaced (2.965%); 11 MiB/s, 179 objects/s recovering 2023-10-09T06:14:51.021 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:14:51.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:51.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:14:51.130 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:51.172 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:51.281 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:51.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:51.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:51.387 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:14:51.428 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:51.487 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:14:51.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:51.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:14:51.526 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:14:51.526 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:14:51.527 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:51.527 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:152388 10.e e550/534 10:7b3b8f7e:::benchmark_data_smithi064_573750_object56173:head v 550'9645, mlcod=550'9642)", 2023-10-09T06:14:51.527 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:51.504215+0000", 2023-10-09T06:14:51.527 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.021083128999999999, 2023-10-09T06:14:51.527 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021109816, 2023-10-09T06:14:51.527 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:14:51.527 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:14:51.527 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:14:51.527 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:51.528 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:14:51.528 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:51.504215+0000", 2023-10-09T06:14:51.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:14:51.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:51.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:51.528 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:14:51.528 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:51.504215+0000", 2023-10-09T06:14:51.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:14:51.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:51.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:51.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:14:51.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:51.504214+0000", 2023-10-09T06:14:51.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:14:51.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:51.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:51.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:14:51.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:51.504223+0000", 2023-10-09T06:14:51.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1600000000000004e-06 2023-10-09T06:14:51.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:51.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:51.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:14:51.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:51.504234+0000", 2023-10-09T06:14:51.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1701e-05 2023-10-09T06:14:51.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:51.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:51.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:14:51.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:51.504242+0000", 2023-10-09T06:14:51.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6040000000000004e-06 2023-10-09T06:14:51.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:51.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:51.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:14:51.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:51.504465+0000", 2023-10-09T06:14:51.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000222671 2023-10-09T06:14:51.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:51.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:51.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:14:51.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:51.504487+0000", 2023-10-09T06:14:51.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2572e-05 2023-10-09T06:14:51.531 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:14:51.531 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:14:51.532 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:14:51.532 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:51.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:51.532 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:152391 10.9 e550/543 10:9831e361:::benchmark_data_smithi064_573750_object56176:head v 550'9443, mlcod=550'9441)", 2023-10-09T06:14:51.532 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:51.507717+0000", 2023-10-09T06:14:51.532 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.017580268999999999, 2023-10-09T06:14:51.532 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01764872, 2023-10-09T06:14:51.532 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:14:51.532 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:14:51.532 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:14:51.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:51.532 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:14:51.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:51.507717+0000", 2023-10-09T06:14:51.533 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:14:51.533 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:51.533 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:51.533 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:14:51.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:51.507717+0000", 2023-10-09T06:14:51.533 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:14:51.533 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:51.533 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:51.533 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:14:51.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:51.507717+0000", 2023-10-09T06:14:51.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:14:51.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:51.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:51.534 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:14:51.534 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:51.507721+0000", 2023-10-09T06:14:51.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8410000000000002e-06 2023-10-09T06:14:51.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:51.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:51.534 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:14:51.534 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:51.507726+0000", 2023-10-09T06:14:51.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.31e-06 2023-10-09T06:14:51.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:51.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:51.535 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:14:51.535 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:51.507731+0000", 2023-10-09T06:14:51.535 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2699999999999998e-06 2023-10-09T06:14:51.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:51.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:51.535 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:14:51.535 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:51.511493+0000", 2023-10-09T06:14:51.535 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037623399999999999 2023-10-09T06:14:51.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:14:51.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:14:51.536 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:14:51.536 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:51.511603+0000", 2023-10-09T06:14:51.536 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010972999999999999 2023-10-09T06:14:51.536 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:14:51.536 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:14:51.536 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:14:51.536 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:14:51.536 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-10-09T06:14:51.536 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 2 2023-10-09T06:14:51.536 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:14:51.599 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:14:51.658 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:51.744 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:51 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3987176578' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:14:51.780 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:14:52.097 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:51 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3987176578' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:14:52.098 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:51 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3987176578' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:14:52.718 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:14:52.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:52.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:14:52.822 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:14:52.853 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:52.977 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:14:52.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:52.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:14:53.089 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:53.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:52 smithi156 ceph-mon[195101]: pgmap v1399: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 2736/125982 objects misplaced (2.172%); 13 MiB/s, 205 objects/s recovering 2023-10-09T06:14:53.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:52 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:14:53.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:52 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3987176578' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:14:53.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:52 smithi156 ceph-mon[195101]: osdmap e551: 8 total, 8 up, 8 in 2023-10-09T06:14:53.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:52 smithi156 ceph-mon[195101]: failed to encode map e551 with expected crc 2023-10-09T06:14:53.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:52 smithi156 ceph-mon[195101]: failed to encode map e551 with expected crc 2023-10-09T06:14:53.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:52 smithi156 ceph-mon[195101]: failed to encode map e551 with expected crc 2023-10-09T06:14:53.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:52 smithi156 ceph-mon[195101]: failed to encode map e551 with expected crc 2023-10-09T06:14:53.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:52 smithi156 ceph-mon[195101]: failed to encode map e551 with expected crc 2023-10-09T06:14:53.146 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:53.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:52 smithi064 ceph-mon[220044]: pgmap v1399: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 2736/125982 objects misplaced (2.172%); 13 MiB/s, 205 objects/s recovering 2023-10-09T06:14:53.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:52 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:14:53.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:52 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3987176578' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:14:53.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:52 smithi064 ceph-mon[220044]: osdmap e551: 8 total, 8 up, 8 in 2023-10-09T06:14:53.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:52 smithi064 ceph-mon[220044]: failed to encode map e551 with expected crc 2023-10-09T06:14:53.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:52 smithi064 ceph-mon[220044]: failed to encode map e551 with expected crc 2023-10-09T06:14:53.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:52 smithi064 ceph-mon[220044]: failed to encode map e551 with expected crc 2023-10-09T06:14:53.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:52 smithi064 ceph-mon[220044]: failed to encode map e551 with expected crc 2023-10-09T06:14:53.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:52 smithi064 ceph-mon[220044]: failed to encode map e551 with expected crc 2023-10-09T06:14:53.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:52 smithi064 ceph-mon[212750]: pgmap v1399: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 2736/125982 objects misplaced (2.172%); 13 MiB/s, 205 objects/s recovering 2023-10-09T06:14:53.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:52 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:14:53.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:52 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3987176578' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:14:53.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:52 smithi064 ceph-mon[212750]: osdmap e551: 8 total, 8 up, 8 in 2023-10-09T06:14:53.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:52 smithi064 ceph-mon[212750]: failed to encode map e551 with expected crc 2023-10-09T06:14:53.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:52 smithi064 ceph-mon[212750]: failed to encode map e551 with expected crc 2023-10-09T06:14:53.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:52 smithi064 ceph-mon[212750]: failed to encode map e551 with expected crc 2023-10-09T06:14:53.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:52 smithi064 ceph-mon[212750]: failed to encode map e551 with expected crc 2023-10-09T06:14:53.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:52 smithi064 ceph-mon[212750]: failed to encode map e551 with expected crc 2023-10-09T06:14:53.229 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:53.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:53.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:53.344 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:53.403 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:53.502 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:53.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:53.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:53.618 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:14:53.688 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:54.981 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:14:55.056 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:54 smithi064 ceph-mon[212750]: pgmap v1401: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 2736/122016 objects misplaced (2.242%); 9.8 MiB/s, 157 objects/s recovering 2023-10-09T06:14:55.056 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:54 smithi064 ceph-mon[220044]: pgmap v1401: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 2736/122016 objects misplaced (2.242%); 9.8 MiB/s, 157 objects/s recovering 2023-10-09T06:14:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:54 smithi156 ceph-mon[195101]: pgmap v1401: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 2736/122016 objects misplaced (2.242%); 9.8 MiB/s, 157 objects/s recovering 2023-10-09T06:14:55.247 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:14:55.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:55.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:14:55.320 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:14:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:14:55] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.43.0" 2023-10-09T06:14:55.377 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:14:55.409 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:55.471 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:14:55.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:55.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:14:55.579 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:14:55.627 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:55.708 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:14:55.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:55.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:14:55.814 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:14:55.914 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:56.120 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:14:56.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:56.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:14:56.232 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:14:56.274 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:56.340 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:14:56.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:56.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:14:56.449 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:56.455 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:14:56.478 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:56.539 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:56.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:56.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:56.630 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:14:56.647 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:14:56.651 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:14:56.678 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T06:14:56.689 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:56.694 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:14:56.746 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:14:56.777 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:14:56.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:56.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:14:56.799 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:14:56.834 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:14:56.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:56 smithi156 ceph-mon[195101]: pgmap v1402: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 2736/119793 objects misplaced (2.284%); 8.3 MiB/s, 133 objects/s recovering 2023-10-09T06:14:56.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:56 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:14:56.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:56 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:14:56.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:56 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/594140460' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:14:56.877 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:14:56.883 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:14:56.912 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:56.929 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:14:56.963 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:56 smithi064 ceph-mon[220044]: pgmap v1402: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 2736/119793 objects misplaced (2.284%); 8.3 MiB/s, 133 objects/s recovering 2023-10-09T06:14:56.963 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:56 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:14:56.964 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:56 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:14:56.964 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:56 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/594140460' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:14:56.964 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:56 smithi064 ceph-mon[212750]: pgmap v1402: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 2736/119793 objects misplaced (2.284%); 8.3 MiB/s, 133 objects/s recovering 2023-10-09T06:14:56.964 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:56 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:14:56.965 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:56 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:14:56.965 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:56 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/594140460' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:14:56.988 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:14:56.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:56.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:14:57.005 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:14:57.055 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:14:57.055 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:14:57.055 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:14:57.055 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:14:57.055 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:14:57.056 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:14:57.056 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:14:57.056 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:14:57.095 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:14:57.142 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:57.220 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:14:57.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:57.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:14:57.318 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:14:57.328 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:14:57.371 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:57.445 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:14:57.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:57.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:14:57.553 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:14:57.594 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:57.684 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:14:57.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:57.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:14:57.795 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:14:57.838 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:57.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:57 smithi156 ceph-mon[195101]: from='client.36447 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:14:57.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:57 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:14:57.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:57 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:14:57.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:57 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:14:57.920 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:14:57.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:57.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:14:58.037 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:14:58.063 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:58.104 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:14:58.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:58.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:14:58.211 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:14:58.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:57 smithi064 ceph-mon[212750]: from='client.36447 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:14:58.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:57 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:14:58.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:57 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:14:58.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:57 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:14:58.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:57 smithi064 ceph-mon[220044]: from='client.36447 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:14:58.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:57 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:14:58.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:57 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:14:58.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:57 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:14:58.242 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:58.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:58 smithi064 ceph-mon[212750]: pgmap v1403: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 1468/110382 objects misplaced (1.330%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T06:14:58.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:58 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:14:58.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:14:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:58.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:58 smithi064 ceph-mon[220044]: pgmap v1403: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 1468/110382 objects misplaced (1.330%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T06:14:58.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:58 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:14:58.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:14:58 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:59.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:58 smithi156 ceph-mon[195101]: pgmap v1403: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 1468/110382 objects misplaced (1.330%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T06:14:59.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:58 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:14:59.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:14:58 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:59.267 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1402,"stamp":"2023-10-09T06:14:55.208686+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2615875512,"num_objects":39931,"num_object_clones":0,"num_object_copies":119793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2736,"num_objects_unfound":0,"num_objects_dirty":39931,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":152580,"num_write_kb":6157598,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41640,"num_bytes_recovered":2726245434,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102912,"ondisk_log_size":102912,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":368,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15237620,"kb_used_data":9602548,"kb_used_omap":165,"kb_used_meta":5634586,"kb_avail":734723596,"statfs":{"total":767960285184,"available":752356962304,"internally_reserved":0,"allocated":9833009152,"data_stored":9814742489,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":169911,"internal_metadata":5769816137},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,5,1,5,6,0,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-429746387,"num_objects":-6557,"num_object_clones":0,"num_object_copies":-19671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-2817,"num_objects_unfound":0,"num_objects_dirty":-6557,"num_whiteouts":0,"num_read":-22,"num_read_kb":-41,"num_write":6546,"num_write_kb":-110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1333,"num_bytes_recovered":87278983,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.009257"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":790,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.734533+0000","last_change":"2023-10-09T06:13:33.699246+0000","last_active":"2023-10-09T06:14:44.734533+0000","last_peered":"2023-10-09T06:14:44.734533+0000","last_clean":"2023-10-09T06:14:44.734533+0000","last_became_active":"2023-10-09T06:13:33.698733+0000","last_became_peered":"2023-10-09T06:13:33.698733+0000","last_unstale":"2023-10-09T06:14:44.734533+0000","last_undegraded":"2023-10-09T06:14:44.734533+0000","last_fullsized":"2023-10-09T06:14:44.734533+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:05.412941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:05.412941+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:57.044673+0000","scrub_duration":0.0061014329999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":618,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.742366+0000","last_change":"2023-10-09T06:07:43.572773+0000","last_active":"2023-10-09T06:14:43.742366+0000","last_peered":"2023-10-09T06:14:43.742366+0000","last_clean":"2023-10-09T06:14:43.742366+0000","last_became_active":"2023-10-09T06:07:43.572299+0000","last_became_peered":"2023-10-09T06:07:43.572299+0000","last_unstale":"2023-10-09T06:14:43.742366+0000","last_undegraded":"2023-10-09T06:14:43.742366+0000","last_fullsized":"2023-10-09T06:14:43.742366+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:00:37.672435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":360,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.738102+0000","last_change":"2023-10-09T06:05:11.451972+0000","last_active":"2023-10-09T06:14:43.738102+0000","last_peered":"2023-10-09T06:14:43.738102+0000","last_clean":"2023-10-09T06:14:43.738102+0000","last_became_active":"2023-10-09T06:05:11.451448+0000","last_became_peered":"2023-10-09T06:05:11.451448+0000","last_unstale":"2023-10-09T06:14:43.738102+0000","last_undegraded":"2023-10-09T06:14:43.738102+0000","last_fullsized":"2023-10-09T06:14:43.738102+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:39.183621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":849,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.742556+0000","last_change":"2023-10-09T06:09:27.554162+0000","last_active":"2023-10-09T06:14:43.742556+0000","last_peered":"2023-10-09T06:14:43.742556+0000","last_clean":"2023-10-09T06:14:43.742556+0000","last_became_active":"2023-10-09T06:09:27.553659+0000","last_became_peered":"2023-10-09T06:09:27.553659+0000","last_unstale":"2023-10-09T06:14:43.742556+0000","last_undegraded":"2023-10-09T06:14:43.742556+0000","last_fullsized":"2023-10-09T06:14:43.742556+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:01.468346+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:01.468346+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:03.443511+0000","scrub_duration":0.0016258220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":232,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.732105+0000","last_change":"2023-10-09T06:01:54.762060+0000","last_active":"2023-10-09T06:14:43.732105+0000","last_peered":"2023-10-09T06:14:43.732105+0000","last_clean":"2023-10-09T06:14:43.732105+0000","last_became_active":"2023-10-09T06:01:54.761757+0000","last_became_peered":"2023-10-09T06:01:54.761757+0000","last_unstale":"2023-10-09T06:14:43.732105+0000","last_undegraded":"2023-10-09T06:14:43.732105+0000","last_fullsized":"2023-10-09T06:14:43.732105+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:40.294232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":34,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.735552+0000","last_change":"2023-10-09T06:13:30.789834+0000","last_active":"2023-10-09T06:14:44.735552+0000","last_peered":"2023-10-09T06:14:44.735552+0000","last_clean":"2023-10-09T06:14:44.735552+0000","last_became_active":"2023-10-09T06:13:30.789517+0000","last_became_peered":"2023-10-09T06:13:30.789517+0000","last_unstale":"2023-10-09T06:14:44.735552+0000","last_undegraded":"2023-10-09T06:14:44.735552+0000","last_fullsized":"2023-10-09T06:14:44.735552+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:45.467922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":834,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.742178+0000","last_change":"2023-10-09T06:12:07.739121+0000","last_active":"2023-10-09T06:14:43.742178+0000","last_peered":"2023-10-09T06:14:43.742178+0000","last_clean":"2023-10-09T06:14:43.742178+0000","last_became_active":"2023-10-09T06:12:07.738739+0000","last_became_peered":"2023-10-09T06:12:07.738739+0000","last_unstale":"2023-10-09T06:14:43.742178+0000","last_undegraded":"2023-10-09T06:14:43.742178+0000","last_fullsized":"2023-10-09T06:14:43.742178+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.293426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.293426+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:19.815422+0000","scrub_duration":0.001587946,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":343,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.735506+0000","last_change":"2023-10-09T06:09:27.564377+0000","last_active":"2023-10-09T06:14:43.735506+0000","last_peered":"2023-10-09T06:14:43.735506+0000","last_clean":"2023-10-09T06:14:43.735506+0000","last_became_active":"2023-10-09T06:09:27.563710+0000","last_became_peered":"2023-10-09T06:09:27.563710+0000","last_unstale":"2023-10-09T06:14:43.735506+0000","last_undegraded":"2023-10-09T06:14:43.735506+0000","last_fullsized":"2023-10-09T06:14:43.735506+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:48.168447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":466,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.732722+0000","last_change":"2023-10-09T06:13:21.727677+0000","last_active":"2023-10-09T06:14:43.732722+0000","last_peered":"2023-10-09T06:14:43.732722+0000","last_clean":"2023-10-09T06:14:43.732722+0000","last_became_active":"2023-10-09T06:13:21.727321+0000","last_became_peered":"2023-10-09T06:13:21.727321+0000","last_unstale":"2023-10-09T06:14:43.732722+0000","last_undegraded":"2023-10-09T06:14:43.732722+0000","last_fullsized":"2023-10-09T06:14:43.732722+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:10.146447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":769,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.735292+0000","last_change":"2023-10-09T06:13:33.701687+0000","last_active":"2023-10-09T06:14:44.735292+0000","last_peered":"2023-10-09T06:14:44.735292+0000","last_clean":"2023-10-09T06:14:44.735292+0000","last_became_active":"2023-10-09T06:13:33.701233+0000","last_became_peered":"2023-10-09T06:13:33.701233+0000","last_unstale":"2023-10-09T06:14:44.735292+0000","last_undegraded":"2023-10-09T06:14:44.735292+0000","last_fullsized":"2023-10-09T06:14:44.735292+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.384407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.384407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:04.887826+0000","scrub_duration":0.0040651960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":448,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.738290+0000","last_change":"2023-10-09T06:09:27.559756+0000","last_active":"2023-10-09T06:14:43.738290+0000","last_peered":"2023-10-09T06:14:43.738290+0000","last_clean":"2023-10-09T06:14:43.738290+0000","last_became_active":"2023-10-09T06:09:27.559443+0000","last_became_peered":"2023-10-09T06:09:27.559443+0000","last_unstale":"2023-10-09T06:14:43.738290+0000","last_undegraded":"2023-10-09T06:14:43.738290+0000","last_fullsized":"2023-10-09T06:14:43.738290+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:14.519398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":379,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.735885+0000","last_change":"2023-10-09T06:13:21.737897+0000","last_active":"2023-10-09T06:14:43.735885+0000","last_peered":"2023-10-09T06:14:43.735885+0000","last_clean":"2023-10-09T06:14:43.735885+0000","last_became_active":"2023-10-09T06:13:21.737564+0000","last_became_peered":"2023-10-09T06:13:21.737564+0000","last_unstale":"2023-10-09T06:14:43.735885+0000","last_undegraded":"2023-10-09T06:14:43.735885+0000","last_fullsized":"2023-10-09T06:14:43.735885+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:34.863320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":675,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.735298+0000","last_change":"2023-10-09T06:10:11.729656+0000","last_active":"2023-10-09T06:14:44.735298+0000","last_peered":"2023-10-09T06:14:44.735298+0000","last_clean":"2023-10-09T06:14:44.735298+0000","last_became_active":"2023-10-09T06:10:11.728960+0000","last_became_peered":"2023-10-09T06:10:11.728960+0000","last_unstale":"2023-10-09T06:14:44.735298+0000","last_undegraded":"2023-10-09T06:14:44.735298+0000","last_fullsized":"2023-10-09T06:14:44.735298+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:20.742561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:20.742561+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:45.543709+0000","scrub_duration":0.0027005039999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":34,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.735480+0000","last_change":"2023-10-09T06:13:30.802749+0000","last_active":"2023-10-09T06:14:44.735480+0000","last_peered":"2023-10-09T06:14:44.735480+0000","last_clean":"2023-10-09T06:14:44.735480+0000","last_became_active":"2023-10-09T06:13:30.802449+0000","last_became_peered":"2023-10-09T06:13:30.802449+0000","last_unstale":"2023-10-09T06:14:44.735480+0000","last_undegraded":"2023-10-09T06:14:44.735480+0000","last_fullsized":"2023-10-09T06:14:44.735480+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:29.017181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":536,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.735775+0000","last_change":"2023-10-09T06:13:30.792445+0000","last_active":"2023-10-09T06:14:43.735775+0000","last_peered":"2023-10-09T06:14:43.735775+0000","last_clean":"2023-10-09T06:14:43.735775+0000","last_became_active":"2023-10-09T06:13:30.792116+0000","last_became_peered":"2023-10-09T06:13:30.792116+0000","last_unstale":"2023-10-09T06:14:43.735775+0000","last_undegraded":"2023-10-09T06:14:43.735775+0000","last_fullsized":"2023-10-09T06:14:43.735775+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:20:53.506293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":762,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.736198+0000","last_change":"2023-10-09T06:13:24.760843+0000","last_active":"2023-10-09T06:14:44.736198+0000","last_peered":"2023-10-09T06:14:44.736198+0000","last_clean":"2023-10-09T06:14:44.736198+0000","last_became_active":"2023-10-09T06:13:24.760333+0000","last_became_peered":"2023-10-09T06:13:24.760333+0000","last_unstale":"2023-10-09T06:14:44.736198+0000","last_undegraded":"2023-10-09T06:14:44.736198+0000","last_fullsized":"2023-10-09T06:14:44.736198+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:38.588641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:38.588641+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:30.660400+0000","scrub_duration":0.010116385,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":40,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.735237+0000","last_change":"2023-10-09T06:13:30.826588+0000","last_active":"2023-10-09T06:14:44.735237+0000","last_peered":"2023-10-09T06:14:44.735237+0000","last_clean":"2023-10-09T06:14:44.735237+0000","last_became_active":"2023-10-09T06:13:30.799199+0000","last_became_peered":"2023-10-09T06:13:30.799199+0000","last_unstale":"2023-10-09T06:14:44.735237+0000","last_undegraded":"2023-10-09T06:14:44.735237+0000","last_fullsized":"2023-10-09T06:14:44.735237+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:38.703864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":332,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.735841+0000","last_change":"2023-10-09T06:13:21.739214+0000","last_active":"2023-10-09T06:14:44.735841+0000","last_peered":"2023-10-09T06:14:44.735841+0000","last_clean":"2023-10-09T06:14:44.735841+0000","last_became_active":"2023-10-09T06:13:21.738951+0000","last_became_peered":"2023-10-09T06:13:21.738951+0000","last_unstale":"2023-10-09T06:14:44.735841+0000","last_undegraded":"2023-10-09T06:14:44.735841+0000","last_fullsized":"2023-10-09T06:14:44.735841+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:17.477241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":722,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.732757+0000","last_change":"2023-10-09T06:09:27.574336+0000","last_active":"2023-10-09T06:14:43.732757+0000","last_peered":"2023-10-09T06:14:43.732757+0000","last_clean":"2023-10-09T06:14:43.732757+0000","last_became_active":"2023-10-09T06:09:27.573974+0000","last_became_peered":"2023-10-09T06:09:27.573974+0000","last_unstale":"2023-10-09T06:14:43.732757+0000","last_undegraded":"2023-10-09T06:14:43.732757+0000","last_fullsized":"2023-10-09T06:14:43.732757+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.339121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.339121+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:45.908579+0000","scrub_duration":0.010474344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":584,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.742722+0000","last_change":"2023-10-09T06:13:30.812598+0000","last_active":"2023-10-09T06:14:43.742722+0000","last_peered":"2023-10-09T06:14:43.742722+0000","last_clean":"2023-10-09T06:14:43.742722+0000","last_became_active":"2023-10-09T06:13:30.791460+0000","last_became_peered":"2023-10-09T06:13:30.791460+0000","last_unstale":"2023-10-09T06:14:43.742722+0000","last_undegraded":"2023-10-09T06:14:43.742722+0000","last_fullsized":"2023-10-09T06:14:43.742722+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:28.046730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":562,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.743086+0000","last_change":"2023-10-09T06:01:54.764916+0000","last_active":"2023-10-09T06:14:43.743086+0000","last_peered":"2023-10-09T06:14:43.743086+0000","last_clean":"2023-10-09T06:14:43.743086+0000","last_became_active":"2023-10-09T06:01:54.764705+0000","last_became_peered":"2023-10-09T06:01:54.764705+0000","last_unstale":"2023-10-09T06:14:43.743086+0000","last_undegraded":"2023-10-09T06:14:43.743086+0000","last_fullsized":"2023-10-09T06:14:43.743086+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:12.527719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":778,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.735573+0000","last_change":"2023-10-09T06:13:15.425525+0000","last_active":"2023-10-09T06:14:43.735573+0000","last_peered":"2023-10-09T06:14:43.735573+0000","last_clean":"2023-10-09T06:14:43.735573+0000","last_became_active":"2023-10-09T06:13:15.425086+0000","last_became_peered":"2023-10-09T06:13:15.425086+0000","last_unstale":"2023-10-09T06:14:43.735573+0000","last_undegraded":"2023-10-09T06:14:43.735573+0000","last_fullsized":"2023-10-09T06:14:43.735573+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:42.900876+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:42.900876+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:46.042715+0000","scrub_duration":0.0014928649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":40,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.735019+0000","last_change":"2023-10-09T06:13:30.811420+0000","last_active":"2023-10-09T06:14:44.735019+0000","last_peered":"2023-10-09T06:14:44.735019+0000","last_clean":"2023-10-09T06:14:44.735019+0000","last_became_active":"2023-10-09T06:13:30.792835+0000","last_became_peered":"2023-10-09T06:13:30.792835+0000","last_unstale":"2023-10-09T06:14:44.735019+0000","last_undegraded":"2023-10-09T06:14:44.735019+0000","last_fullsized":"2023-10-09T06:14:44.735019+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:38.962645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":595,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.738292+0000","last_change":"2023-10-09T06:01:54.761558+0000","last_active":"2023-10-09T06:14:43.738292+0000","last_peered":"2023-10-09T06:14:43.738292+0000","last_clean":"2023-10-09T06:14:43.738292+0000","last_became_active":"2023-10-09T06:01:54.761251+0000","last_became_peered":"2023-10-09T06:01:54.761251+0000","last_unstale":"2023-10-09T06:14:43.738292+0000","last_undegraded":"2023-10-09T06:14:43.738292+0000","last_fullsized":"2023-10-09T06:14:43.738292+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:53.245267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":1054,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.734742+0000","last_change":"2023-10-09T06:13:24.754018+0000","last_active":"2023-10-09T06:14:44.734742+0000","last_peered":"2023-10-09T06:14:44.734742+0000","last_clean":"2023-10-09T06:14:44.734742+0000","last_became_active":"2023-10-09T06:13:24.753712+0000","last_became_peered":"2023-10-09T06:13:24.753712+0000","last_unstale":"2023-10-09T06:14:44.734742+0000","last_undegraded":"2023-10-09T06:14:44.734742+0000","last_fullsized":"2023-10-09T06:14:44.734742+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:18.490833+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":34,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.734719+0000","last_change":"2023-10-09T06:13:30.788160+0000","last_active":"2023-10-09T06:14:44.734719+0000","last_peered":"2023-10-09T06:14:44.734719+0000","last_clean":"2023-10-09T06:14:44.734719+0000","last_became_active":"2023-10-09T06:13:30.787724+0000","last_became_peered":"2023-10-09T06:13:30.787724+0000","last_unstale":"2023-10-09T06:14:44.734719+0000","last_undegraded":"2023-10-09T06:14:44.734719+0000","last_fullsized":"2023-10-09T06:14:44.734719+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:38.376770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":34,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.735372+0000","last_change":"2023-10-09T06:13:30.789225+0000","last_active":"2023-10-09T06:14:44.735372+0000","last_peered":"2023-10-09T06:14:44.735372+0000","last_clean":"2023-10-09T06:14:44.735372+0000","last_became_active":"2023-10-09T06:13:30.788869+0000","last_became_peered":"2023-10-09T06:13:30.788869+0000","last_unstale":"2023-10-09T06:14:44.735372+0000","last_undegraded":"2023-10-09T06:14:44.735372+0000","last_fullsized":"2023-10-09T06:14:44.735372+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:19.519966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":790,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.733015+0000","last_change":"2023-10-09T06:12:55.750923+0000","last_active":"2023-10-09T06:14:43.733015+0000","last_peered":"2023-10-09T06:14:43.733015+0000","last_clean":"2023-10-09T06:14:43.733015+0000","last_became_active":"2023-10-09T06:12:55.750521+0000","last_became_peered":"2023-10-09T06:12:55.750521+0000","last_unstale":"2023-10-09T06:14:43.733015+0000","last_undegraded":"2023-10-09T06:14:43.733015+0000","last_fullsized":"2023-10-09T06:14:43.733015+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.316697+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.316697+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:23.591439+0000","scrub_duration":0.001666561,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":404,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.734753+0000","last_change":"2023-10-09T06:01:22.337171+0000","last_active":"2023-10-09T06:14:44.734753+0000","last_peered":"2023-10-09T06:14:44.734753+0000","last_clean":"2023-10-09T06:14:44.734753+0000","last_became_active":"2023-10-09T06:01:22.335438+0000","last_became_peered":"2023-10-09T06:01:22.335438+0000","last_unstale":"2023-10-09T06:14:44.734753+0000","last_undegraded":"2023-10-09T06:14:44.734753+0000","last_fullsized":"2023-10-09T06:14:44.734753+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:25.036448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":512,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.738272+0000","last_change":"2023-10-09T06:13:30.788284+0000","last_active":"2023-10-09T06:14:43.738272+0000","last_peered":"2023-10-09T06:14:43.738272+0000","last_clean":"2023-10-09T06:14:43.738272+0000","last_became_active":"2023-10-09T06:13:30.787933+0000","last_became_peered":"2023-10-09T06:13:30.787933+0000","last_unstale":"2023-10-09T06:14:43.738272+0000","last_undegraded":"2023-10-09T06:14:43.738272+0000","last_fullsized":"2023-10-09T06:14:43.738272+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:10.735438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":840,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.732858+0000","last_change":"2023-10-09T06:13:24.753908+0000","last_active":"2023-10-09T06:14:43.732858+0000","last_peered":"2023-10-09T06:14:43.732858+0000","last_clean":"2023-10-09T06:14:43.732858+0000","last_became_active":"2023-10-09T06:13:24.753566+0000","last_became_peered":"2023-10-09T06:13:24.753566+0000","last_unstale":"2023-10-09T06:14:43.732858+0000","last_undegraded":"2023-10-09T06:14:43.732858+0000","last_fullsized":"2023-10-09T06:14:43.732858+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:40.388125+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:40.388125+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:35.101978+0000","scrub_duration":0.0055576990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":122,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.733829+0000","last_change":"2023-10-09T06:13:12.243605+0000","last_active":"2023-10-09T06:14:43.733829+0000","last_peered":"2023-10-09T06:14:43.733829+0000","last_clean":"2023-10-09T06:14:43.733829+0000","last_became_active":"2023-10-09T06:13:12.237061+0000","last_became_peered":"2023-10-09T06:13:12.237061+0000","last_unstale":"2023-10-09T06:14:43.733829+0000","last_undegraded":"2023-10-09T06:14:43.733829+0000","last_fullsized":"2023-10-09T06:14:43.733829+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:29:03.666031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":60,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.738106+0000","last_change":"2023-10-09T06:13:30.788733+0000","last_active":"2023-10-09T06:14:43.738106+0000","last_peered":"2023-10-09T06:14:43.738106+0000","last_clean":"2023-10-09T06:14:43.738106+0000","last_became_active":"2023-10-09T06:13:30.788353+0000","last_became_peered":"2023-10-09T06:13:30.788353+0000","last_unstale":"2023-10-09T06:14:43.738106+0000","last_undegraded":"2023-10-09T06:14:43.738106+0000","last_fullsized":"2023-10-09T06:14:43.738106+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:06.308426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"550'9538","reported_seq":9608,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:51.255698+0000","last_change":"2023-10-09T06:11:46.032384+0000","last_active":"2023-10-09T06:14:51.255698+0000","last_peered":"2023-10-09T06:14:51.255698+0000","last_clean":"2023-10-09T06:14:51.255698+0000","last_became_active":"2023-10-09T06:11:46.032091+0000","last_became_peered":"2023-10-09T06:11:46.032091+0000","last_unstale":"2023-10-09T06:14:51.255698+0000","last_undegraded":"2023-10-09T06:14:51.255698+0000","last_fullsized":"2023-10-09T06:14:51.255698+0000","mapping_epoch":490,"log_start":"505'3100","ondisk_log_start":"505'3100","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6438,"log_dups_size":0,"ondisk_log_size":6438,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:31.117384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164757504,"num_objects":2514,"num_object_clones":0,"num_object_copies":7542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9538,"num_write_kb":385664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":732,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.735371+0000","last_change":"2023-10-09T06:13:33.701521+0000","last_active":"2023-10-09T06:14:44.735371+0000","last_peered":"2023-10-09T06:14:44.735371+0000","last_clean":"2023-10-09T06:14:44.735371+0000","last_became_active":"2023-10-09T06:13:33.700870+0000","last_became_peered":"2023-10-09T06:13:33.700870+0000","last_unstale":"2023-10-09T06:14:44.735371+0000","last_undegraded":"2023-10-09T06:14:44.735371+0000","last_fullsized":"2023-10-09T06:14:44.735371+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:10:59.422551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:10:59.422551+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:54.402494+0000","scrub_duration":0.0059716659999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":38,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.735592+0000","last_change":"2023-10-09T06:13:30.822903+0000","last_active":"2023-10-09T06:14:44.735592+0000","last_peered":"2023-10-09T06:14:44.735592+0000","last_clean":"2023-10-09T06:14:44.735592+0000","last_became_active":"2023-10-09T06:13:30.799013+0000","last_became_peered":"2023-10-09T06:13:30.799013+0000","last_unstale":"2023-10-09T06:14:44.735592+0000","last_undegraded":"2023-10-09T06:14:44.735592+0000","last_fullsized":"2023-10-09T06:14:44.735592+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:44.271821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":34,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.734406+0000","last_change":"2023-10-09T06:13:30.793531+0000","last_active":"2023-10-09T06:14:44.734406+0000","last_peered":"2023-10-09T06:14:44.734406+0000","last_clean":"2023-10-09T06:14:44.734406+0000","last_became_active":"2023-10-09T06:13:30.792689+0000","last_became_peered":"2023-10-09T06:13:30.792689+0000","last_unstale":"2023-10-09T06:14:44.734406+0000","last_undegraded":"2023-10-09T06:14:44.734406+0000","last_fullsized":"2023-10-09T06:14:44.734406+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:02.559186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"550'9513","reported_seq":14768,"reported_epoch":550,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T06:14:50.411311+0000","last_change":"2023-10-09T06:14:41.857773+0000","last_active":"2023-10-09T06:14:50.411311+0000","last_peered":"2023-10-09T06:14:50.411311+0000","last_clean":"2023-10-09T06:13:11.194737+0000","last_became_active":"2023-10-09T06:13:21.782116+0000","last_became_peered":"2023-10-09T06:13:21.782116+0000","last_unstale":"2023-10-09T06:14:50.411311+0000","last_undegraded":"2023-10-09T06:14:50.411311+0000","last_fullsized":"2023-10-09T06:14:50.411311+0000","mapping_epoch":521,"log_start":"505'3593","ondisk_log_start":"505'3593","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5920,"log_dups_size":0,"ondisk_log_size":5920,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:00.656766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168361984,"num_objects":2569,"num_object_clones":0,"num_object_copies":7707,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2736,"num_objects_unfound":0,"num_objects_dirty":2569,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9513,"num_write_kb":386624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2113,"num_bytes_recovered":138477568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,2,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":2569}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":919,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.738024+0000","last_change":"2023-10-09T06:13:24.753866+0000","last_active":"2023-10-09T06:14:43.738024+0000","last_peered":"2023-10-09T06:14:43.738024+0000","last_clean":"2023-10-09T06:14:43.738024+0000","last_became_active":"2023-10-09T06:13:24.753432+0000","last_became_peered":"2023-10-09T06:13:24.753432+0000","last_unstale":"2023-10-09T06:14:43.738024+0000","last_undegraded":"2023-10-09T06:14:43.738024+0000","last_fullsized":"2023-10-09T06:14:43.738024+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:03.241335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:03.241335+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:19.854449+0000","scrub_duration":0.017780547000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":38,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.735405+0000","last_change":"2023-10-09T06:13:30.829763+0000","last_active":"2023-10-09T06:14:44.735405+0000","last_peered":"2023-10-09T06:14:44.735405+0000","last_clean":"2023-10-09T06:14:44.735405+0000","last_became_active":"2023-10-09T06:13:30.802486+0000","last_became_peered":"2023-10-09T06:13:30.802486+0000","last_unstale":"2023-10-09T06:14:44.735405+0000","last_undegraded":"2023-10-09T06:14:44.735405+0000","last_fullsized":"2023-10-09T06:14:44.735405+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:05.468205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":588,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.742201+0000","last_change":"2023-10-09T06:01:54.760367+0000","last_active":"2023-10-09T06:14:43.742201+0000","last_peered":"2023-10-09T06:14:43.742201+0000","last_clean":"2023-10-09T06:14:43.742201+0000","last_became_active":"2023-10-09T06:01:54.760087+0000","last_became_peered":"2023-10-09T06:01:54.760087+0000","last_unstale":"2023-10-09T06:14:43.742201+0000","last_undegraded":"2023-10-09T06:14:43.742201+0000","last_fullsized":"2023-10-09T06:14:43.742201+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:26.000104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"550'9341","reported_seq":9409,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:50.881894+0000","last_change":"2023-10-09T06:11:45.696810+0000","last_active":"2023-10-09T06:14:50.881894+0000","last_peered":"2023-10-09T06:14:50.881894+0000","last_clean":"2023-10-09T06:14:50.881894+0000","last_became_active":"2023-10-09T06:11:45.696476+0000","last_became_peered":"2023-10-09T06:11:45.696476+0000","last_unstale":"2023-10-09T06:14:50.881894+0000","last_undegraded":"2023-10-09T06:14:50.881894+0000","last_fullsized":"2023-10-09T06:14:50.881894+0000","mapping_epoch":490,"log_start":"506'3900","ondisk_log_start":"506'3900","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5441,"log_dups_size":0,"ondisk_log_size":5441,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:41:33.351567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169803776,"num_objects":2591,"num_object_clones":0,"num_object_copies":7773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9341,"num_write_kb":381824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":813,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.742990+0000","last_change":"2023-10-09T06:13:24.768958+0000","last_active":"2023-10-09T06:14:43.742990+0000","last_peered":"2023-10-09T06:14:43.742990+0000","last_clean":"2023-10-09T06:14:43.742990+0000","last_became_active":"2023-10-09T06:13:24.768643+0000","last_became_peered":"2023-10-09T06:13:24.768643+0000","last_unstale":"2023-10-09T06:14:43.742990+0000","last_undegraded":"2023-10-09T06:14:43.742990+0000","last_fullsized":"2023-10-09T06:14:43.742990+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:48.495739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:48.495739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:18:50.654354+0000","scrub_duration":0.0016345509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":38,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.735322+0000","last_change":"2023-10-09T06:13:30.814434+0000","last_active":"2023-10-09T06:14:44.735322+0000","last_peered":"2023-10-09T06:14:44.735322+0000","last_clean":"2023-10-09T06:14:44.735322+0000","last_became_active":"2023-10-09T06:13:30.793119+0000","last_became_peered":"2023-10-09T06:13:30.793119+0000","last_unstale":"2023-10-09T06:14:44.735322+0000","last_undegraded":"2023-10-09T06:14:44.735322+0000","last_fullsized":"2023-10-09T06:14:44.735322+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:34:31.724006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":575,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.732550+0000","last_change":"2023-10-09T06:13:30.792490+0000","last_active":"2023-10-09T06:14:43.732550+0000","last_peered":"2023-10-09T06:14:43.732550+0000","last_clean":"2023-10-09T06:14:43.732550+0000","last_became_active":"2023-10-09T06:13:30.792209+0000","last_became_peered":"2023-10-09T06:13:30.792209+0000","last_unstale":"2023-10-09T06:14:43.732550+0000","last_undegraded":"2023-10-09T06:14:43.732550+0000","last_fullsized":"2023-10-09T06:14:43.732550+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:14.208042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"550'9455","reported_seq":9525,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:51.285533+0000","last_change":"2023-10-09T06:11:46.031195+0000","last_active":"2023-10-09T06:14:51.285533+0000","last_peered":"2023-10-09T06:14:51.285533+0000","last_clean":"2023-10-09T06:14:51.285533+0000","last_became_active":"2023-10-09T06:11:46.030891+0000","last_became_peered":"2023-10-09T06:11:46.030891+0000","last_unstale":"2023-10-09T06:14:51.285533+0000","last_undegraded":"2023-10-09T06:14:51.285533+0000","last_fullsized":"2023-10-09T06:14:51.285533+0000","mapping_epoch":490,"log_start":"505'3000","ondisk_log_start":"505'3000","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6455,"log_dups_size":0,"ondisk_log_size":6455,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:25.788661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161284096,"num_objects":2461,"num_object_clones":0,"num_object_copies":7383,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2461,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9455,"num_write_kb":381312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":765,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.742665+0000","last_change":"2023-10-09T06:13:33.708291+0000","last_active":"2023-10-09T06:14:43.742665+0000","last_peered":"2023-10-09T06:14:43.742665+0000","last_clean":"2023-10-09T06:14:43.742665+0000","last_became_active":"2023-10-09T06:13:33.707930+0000","last_became_peered":"2023-10-09T06:13:33.707930+0000","last_unstale":"2023-10-09T06:14:43.742665+0000","last_undegraded":"2023-10-09T06:14:43.742665+0000","last_fullsized":"2023-10-09T06:14:43.742665+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:16.361097+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:16.361097+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:18.111404+0000","scrub_duration":0.028533749000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":319,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.738121+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T06:14:43.738121+0000","last_peered":"2023-10-09T06:14:43.738121+0000","last_clean":"2023-10-09T06:14:43.738121+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T06:14:43.738121+0000","last_undegraded":"2023-10-09T06:14:43.738121+0000","last_fullsized":"2023-10-09T06:14:43.738121+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":563,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.734024+0000","last_change":"2023-10-09T06:09:27.558353+0000","last_active":"2023-10-09T06:14:43.734024+0000","last_peered":"2023-10-09T06:14:43.734024+0000","last_clean":"2023-10-09T06:14:43.734024+0000","last_became_active":"2023-10-09T06:09:27.557233+0000","last_became_peered":"2023-10-09T06:09:27.557233+0000","last_unstale":"2023-10-09T06:14:43.734024+0000","last_undegraded":"2023-10-09T06:14:43.734024+0000","last_fullsized":"2023-10-09T06:14:43.734024+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:53.149350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"550'9517","reported_seq":9586,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:51.648945+0000","last_change":"2023-10-09T06:11:45.694648+0000","last_active":"2023-10-09T06:14:51.648945+0000","last_peered":"2023-10-09T06:14:51.648945+0000","last_clean":"2023-10-09T06:14:51.648945+0000","last_became_active":"2023-10-09T06:11:45.694321+0000","last_became_peered":"2023-10-09T06:11:45.694321+0000","last_unstale":"2023-10-09T06:14:51.648945+0000","last_undegraded":"2023-10-09T06:14:51.648945+0000","last_fullsized":"2023-10-09T06:14:51.648945+0000","mapping_epoch":490,"log_start":"505'2900","ondisk_log_start":"505'2900","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6617,"log_dups_size":0,"ondisk_log_size":6617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:43.704525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157614080,"num_objects":2405,"num_object_clones":0,"num_object_copies":7215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2405,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9517,"num_write_kb":381504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":937,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.734635+0000","last_change":"2023-10-09T06:13:33.716191+0000","last_active":"2023-10-09T06:14:44.734635+0000","last_peered":"2023-10-09T06:14:44.734635+0000","last_clean":"2023-10-09T06:14:44.734635+0000","last_became_active":"2023-10-09T06:13:33.715831+0000","last_became_peered":"2023-10-09T06:13:33.715831+0000","last_unstale":"2023-10-09T06:14:44.734635+0000","last_undegraded":"2023-10-09T06:14:44.734635+0000","last_fullsized":"2023-10-09T06:14:44.734635+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:42.795781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:42.795781+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:45.712738+0000","scrub_duration":0.0091262979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":353,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.737777+0000","last_change":"2023-10-09T06:09:27.565666+0000","last_active":"2023-10-09T06:14:43.737777+0000","last_peered":"2023-10-09T06:14:43.737777+0000","last_clean":"2023-10-09T06:14:43.737777+0000","last_became_active":"2023-10-09T06:09:27.565324+0000","last_became_peered":"2023-10-09T06:09:27.565324+0000","last_unstale":"2023-10-09T06:14:43.737777+0000","last_undegraded":"2023-10-09T06:14:43.737777+0000","last_fullsized":"2023-10-09T06:14:43.737777+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:51.216947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":524,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.743060+0000","last_change":"2023-10-09T06:13:21.727745+0000","last_active":"2023-10-09T06:14:43.743060+0000","last_peered":"2023-10-09T06:14:43.743060+0000","last_clean":"2023-10-09T06:14:43.743060+0000","last_became_active":"2023-10-09T06:13:21.727404+0000","last_became_peered":"2023-10-09T06:13:21.727404+0000","last_unstale":"2023-10-09T06:14:43.743060+0000","last_undegraded":"2023-10-09T06:14:43.743060+0000","last_fullsized":"2023-10-09T06:14:43.743060+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:24.931626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"550'9636","reported_seq":21199,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:50.396550+0000","last_change":"2023-10-09T06:13:32.697002+0000","last_active":"2023-10-09T06:14:50.396550+0000","last_peered":"2023-10-09T06:14:50.396550+0000","last_clean":"2023-10-09T06:14:50.396550+0000","last_became_active":"2023-10-09T06:13:32.683104+0000","last_became_peered":"2023-10-09T06:13:32.683104+0000","last_unstale":"2023-10-09T06:14:50.396550+0000","last_undegraded":"2023-10-09T06:14:50.396550+0000","last_fullsized":"2023-10-09T06:14:50.396550+0000","mapping_epoch":528,"log_start":"505'3735","ondisk_log_start":"505'3735","created":490,"last_epoch_clean":529,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5901,"log_dups_size":0,"ondisk_log_size":5901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:19.426571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172883968,"num_objects":2638,"num_object_clones":0,"num_object_copies":7914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9636,"num_write_kb":392768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5732,"num_bytes_recovered":375652352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":813,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.735410+0000","last_change":"2023-10-09T06:13:15.442141+0000","last_active":"2023-10-09T06:14:43.735410+0000","last_peered":"2023-10-09T06:14:43.735410+0000","last_clean":"2023-10-09T06:14:43.735410+0000","last_became_active":"2023-10-09T06:13:15.441716+0000","last_became_peered":"2023-10-09T06:13:15.441716+0000","last_unstale":"2023-10-09T06:14:43.735410+0000","last_undegraded":"2023-10-09T06:14:43.735410+0000","last_fullsized":"2023-10-09T06:14:43.735410+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.282393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.282393+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:16.798440+0000","scrub_duration":0.01402403,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":469,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.742241+0000","last_change":"2023-10-09T06:13:21.738218+0000","last_active":"2023-10-09T06:14:43.742241+0000","last_peered":"2023-10-09T06:14:43.742241+0000","last_clean":"2023-10-09T06:14:43.742241+0000","last_became_active":"2023-10-09T06:13:21.737969+0000","last_became_peered":"2023-10-09T06:13:21.737969+0000","last_unstale":"2023-10-09T06:14:43.742241+0000","last_undegraded":"2023-10-09T06:14:43.742241+0000","last_fullsized":"2023-10-09T06:14:43.742241+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:38.620281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":598,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.742603+0000","last_change":"2023-10-09T06:07:43.572173+0000","last_active":"2023-10-09T06:14:43.742603+0000","last_peered":"2023-10-09T06:14:43.742603+0000","last_clean":"2023-10-09T06:14:43.742603+0000","last_became_active":"2023-10-09T06:07:43.571690+0000","last_became_peered":"2023-10-09T06:07:43.571690+0000","last_unstale":"2023-10-09T06:14:43.742603+0000","last_undegraded":"2023-10-09T06:14:43.742603+0000","last_fullsized":"2023-10-09T06:14:43.742603+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:10.234125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"550'9673","reported_seq":18047,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:51.652802+0000","last_change":"2023-10-09T06:14:28.746278+0000","last_active":"2023-10-09T06:14:51.652802+0000","last_peered":"2023-10-09T06:14:51.652802+0000","last_clean":"2023-10-09T06:14:51.652802+0000","last_became_active":"2023-10-09T06:14:28.728616+0000","last_became_peered":"2023-10-09T06:14:28.728616+0000","last_unstale":"2023-10-09T06:14:51.652802+0000","last_undegraded":"2023-10-09T06:14:51.652802+0000","last_fullsized":"2023-10-09T06:14:51.652802+0000","mapping_epoch":545,"log_start":"505'3100","ondisk_log_start":"505'3100","created":490,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6573,"log_dups_size":0,"ondisk_log_size":6573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:32.106540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167575552,"num_objects":2557,"num_object_clones":0,"num_object_copies":7671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2557,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9673,"num_write_kb":391360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4137,"num_bytes_recovered":271122432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":884,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.732319+0000","last_change":"2023-10-09T06:13:15.428701+0000","last_active":"2023-10-09T06:14:43.732319+0000","last_peered":"2023-10-09T06:14:43.732319+0000","last_clean":"2023-10-09T06:14:43.732319+0000","last_became_active":"2023-10-09T06:13:15.428343+0000","last_became_peered":"2023-10-09T06:13:15.428343+0000","last_unstale":"2023-10-09T06:14:43.732319+0000","last_undegraded":"2023-10-09T06:14:43.732319+0000","last_fullsized":"2023-10-09T06:14:43.732319+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.373748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.373748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:55:34.235074+0000","scrub_duration":0.0024576450000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":540,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.742313+0000","last_change":"2023-10-09T06:07:43.574863+0000","last_active":"2023-10-09T06:14:43.742313+0000","last_peered":"2023-10-09T06:14:43.742313+0000","last_clean":"2023-10-09T06:14:43.742313+0000","last_became_active":"2023-10-09T06:07:43.574576+0000","last_became_peered":"2023-10-09T06:07:43.574576+0000","last_unstale":"2023-10-09T06:14:43.742313+0000","last_undegraded":"2023-10-09T06:14:43.742313+0000","last_fullsized":"2023-10-09T06:14:43.742313+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:16.166004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":581,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.742693+0000","last_change":"2023-10-09T06:13:21.727814+0000","last_active":"2023-10-09T06:14:43.742693+0000","last_peered":"2023-10-09T06:14:43.742693+0000","last_clean":"2023-10-09T06:14:43.742693+0000","last_became_active":"2023-10-09T06:13:21.727401+0000","last_became_peered":"2023-10-09T06:13:21.727401+0000","last_unstale":"2023-10-09T06:14:43.742693+0000","last_undegraded":"2023-10-09T06:14:43.742693+0000","last_fullsized":"2023-10-09T06:14:43.742693+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:53:37.968028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"550'9575","reported_seq":9643,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:50.823085+0000","last_change":"2023-10-09T06:11:45.695709+0000","last_active":"2023-10-09T06:14:50.823085+0000","last_peered":"2023-10-09T06:14:50.823085+0000","last_clean":"2023-10-09T06:14:50.823085+0000","last_became_active":"2023-10-09T06:11:45.695219+0000","last_became_peered":"2023-10-09T06:11:45.695219+0000","last_unstale":"2023-10-09T06:14:50.823085+0000","last_undegraded":"2023-10-09T06:14:50.823085+0000","last_fullsized":"2023-10-09T06:14:50.823085+0000","mapping_epoch":490,"log_start":"506'4200","ondisk_log_start":"506'4200","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5375,"log_dups_size":0,"ondisk_log_size":5375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:26.533964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166264832,"num_objects":2537,"num_object_clones":0,"num_object_copies":7611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9575,"num_write_kb":387584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":629,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.736079+0000","last_change":"2023-10-09T06:13:24.753519+0000","last_active":"2023-10-09T06:14:44.736079+0000","last_peered":"2023-10-09T06:14:44.736079+0000","last_clean":"2023-10-09T06:14:44.736079+0000","last_became_active":"2023-10-09T06:13:24.753185+0000","last_became_peered":"2023-10-09T06:13:24.753185+0000","last_unstale":"2023-10-09T06:14:44.736079+0000","last_undegraded":"2023-10-09T06:14:44.736079+0000","last_fullsized":"2023-10-09T06:14:44.736079+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:39.633410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:39.633410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:56.871057+0000","scrub_duration":0.001547624,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":362,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.737692+0000","last_change":"2023-10-09T06:13:30.781849+0000","last_active":"2023-10-09T06:14:44.737692+0000","last_peered":"2023-10-09T06:14:44.737692+0000","last_clean":"2023-10-09T06:14:44.737692+0000","last_became_active":"2023-10-09T06:13:30.781084+0000","last_became_peered":"2023-10-09T06:13:30.781084+0000","last_unstale":"2023-10-09T06:14:44.737692+0000","last_undegraded":"2023-10-09T06:14:44.737692+0000","last_fullsized":"2023-10-09T06:14:44.737692+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:02.199298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":232,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.732915+0000","last_change":"2023-10-09T06:01:54.761768+0000","last_active":"2023-10-09T06:14:43.732915+0000","last_peered":"2023-10-09T06:14:43.732915+0000","last_clean":"2023-10-09T06:14:43.732915+0000","last_became_active":"2023-10-09T06:01:54.761443+0000","last_became_peered":"2023-10-09T06:01:54.761443+0000","last_unstale":"2023-10-09T06:14:43.732915+0000","last_undegraded":"2023-10-09T06:14:43.732915+0000","last_fullsized":"2023-10-09T06:14:43.732915+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:51.555987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"551'9764","reported_seq":16701,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-09T06:14:54.326159+0000","last_change":"2023-10-09T06:14:44.803724+0000","last_active":"2023-10-09T06:14:54.326159+0000","last_peered":"2023-10-09T06:14:54.326159+0000","last_clean":"2023-10-09T06:14:54.326159+0000","last_became_active":"2023-10-09T06:14:44.737126+0000","last_became_peered":"2023-10-09T06:14:44.737126+0000","last_unstale":"2023-10-09T06:14:54.326159+0000","last_undegraded":"2023-10-09T06:14:54.326159+0000","last_fullsized":"2023-10-09T06:14:54.326159+0000","mapping_epoch":549,"log_start":"505'3300","ondisk_log_start":"505'3300","created":490,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6464,"log_dups_size":0,"ondisk_log_size":6464,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:14.630144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160038912,"num_objects":2442,"num_object_clones":0,"num_object_copies":7326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9764,"num_write_kb":390592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3423,"num_bytes_recovered":224329728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":752,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.738172+0000","last_change":"2023-10-09T06:13:33.707885+0000","last_active":"2023-10-09T06:14:43.738172+0000","last_peered":"2023-10-09T06:14:43.738172+0000","last_clean":"2023-10-09T06:14:43.738172+0000","last_became_active":"2023-10-09T06:13:33.707474+0000","last_became_peered":"2023-10-09T06:13:33.707474+0000","last_unstale":"2023-10-09T06:14:43.738172+0000","last_undegraded":"2023-10-09T06:14:43.738172+0000","last_fullsized":"2023-10-09T06:14:43.738172+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:12:22.289803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:12:22.289803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:27.842817+0000","scrub_duration":0.0076565050000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":295,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.732445+0000","last_change":"2023-10-09T06:12:04.680247+0000","last_active":"2023-10-09T06:14:43.732445+0000","last_peered":"2023-10-09T06:14:43.732445+0000","last_clean":"2023-10-09T06:14:43.732445+0000","last_became_active":"2023-10-09T06:12:04.679943+0000","last_became_peered":"2023-10-09T06:12:04.679943+0000","last_unstale":"2023-10-09T06:14:43.732445+0000","last_undegraded":"2023-10-09T06:14:43.732445+0000","last_fullsized":"2023-10-09T06:14:43.732445+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:56.692774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":338,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.738015+0000","last_change":"2023-10-09T06:07:43.577306+0000","last_active":"2023-10-09T06:14:43.738015+0000","last_peered":"2023-10-09T06:14:43.738015+0000","last_clean":"2023-10-09T06:14:43.738015+0000","last_became_active":"2023-10-09T06:07:43.576956+0000","last_became_peered":"2023-10-09T06:07:43.576956+0000","last_unstale":"2023-10-09T06:14:43.738015+0000","last_undegraded":"2023-10-09T06:14:43.738015+0000","last_fullsized":"2023-10-09T06:14:43.738015+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:21.020403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":266,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.732697+0000","last_change":"2023-10-09T06:13:12.234849+0000","last_active":"2023-10-09T06:14:43.732697+0000","last_peered":"2023-10-09T06:14:43.732697+0000","last_clean":"2023-10-09T06:14:43.732697+0000","last_became_active":"2023-10-09T06:13:12.234284+0000","last_became_peered":"2023-10-09T06:13:12.234284+0000","last_unstale":"2023-10-09T06:14:43.732697+0000","last_undegraded":"2023-10-09T06:14:43.732697+0000","last_fullsized":"2023-10-09T06:14:43.732697+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:40:43.809773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"550'9397","reported_seq":9467,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:51.275391+0000","last_change":"2023-10-09T06:11:46.029349+0000","last_active":"2023-10-09T06:14:51.275391+0000","last_peered":"2023-10-09T06:14:51.275391+0000","last_clean":"2023-10-09T06:14:51.275391+0000","last_became_active":"2023-10-09T06:11:46.029041+0000","last_became_peered":"2023-10-09T06:11:46.029041+0000","last_unstale":"2023-10-09T06:14:51.275391+0000","last_undegraded":"2023-10-09T06:14:51.275391+0000","last_fullsized":"2023-10-09T06:14:51.275391+0000","mapping_epoch":490,"log_start":"505'3000","ondisk_log_start":"505'3000","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6397,"log_dups_size":0,"ondisk_log_size":6397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:07.662395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162201600,"num_objects":2475,"num_object_clones":0,"num_object_copies":7425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9397,"num_write_kb":379904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":785,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.734310+0000","last_change":"2023-10-09T06:09:27.544555+0000","last_active":"2023-10-09T06:14:43.734310+0000","last_peered":"2023-10-09T06:14:43.734310+0000","last_clean":"2023-10-09T06:14:43.734310+0000","last_became_active":"2023-10-09T06:09:27.544222+0000","last_became_peered":"2023-10-09T06:09:27.544222+0000","last_unstale":"2023-10-09T06:14:43.734310+0000","last_undegraded":"2023-10-09T06:14:43.734310+0000","last_fullsized":"2023-10-09T06:14:43.734310+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:45.944253+0000","scrub_duration":0.0016660799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":251,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.732770+0000","last_change":"2023-10-09T06:13:21.738040+0000","last_active":"2023-10-09T06:14:43.732770+0000","last_peered":"2023-10-09T06:14:43.732770+0000","last_clean":"2023-10-09T06:14:43.732770+0000","last_became_active":"2023-10-09T06:13:21.737717+0000","last_became_peered":"2023-10-09T06:13:21.737717+0000","last_unstale":"2023-10-09T06:14:43.732770+0000","last_undegraded":"2023-10-09T06:14:43.732770+0000","last_fullsized":"2023-10-09T06:14:43.732770+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:39.451628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":530,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.743214+0000","last_change":"2023-10-09T06:09:27.553958+0000","last_active":"2023-10-09T06:14:43.743214+0000","last_peered":"2023-10-09T06:14:43.743214+0000","last_clean":"2023-10-09T06:14:43.743214+0000","last_became_active":"2023-10-09T06:09:27.553193+0000","last_became_peered":"2023-10-09T06:09:27.553193+0000","last_unstale":"2023-10-09T06:14:43.743214+0000","last_undegraded":"2023-10-09T06:14:43.743214+0000","last_fullsized":"2023-10-09T06:14:43.743214+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:09.754579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":376,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.732520+0000","last_change":"2023-10-09T06:07:43.580765+0000","last_active":"2023-10-09T06:14:43.732520+0000","last_peered":"2023-10-09T06:14:43.732520+0000","last_clean":"2023-10-09T06:14:43.732520+0000","last_became_active":"2023-10-09T06:07:43.580458+0000","last_became_peered":"2023-10-09T06:07:43.580458+0000","last_unstale":"2023-10-09T06:14:43.732520+0000","last_undegraded":"2023-10-09T06:14:43.732520+0000","last_fullsized":"2023-10-09T06:14:43.732520+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:55.161893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"551'9581","reported_seq":25906,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-09T06:14:54.315352+0000","last_change":"2023-10-09T06:14:21.793267+0000","last_active":"2023-10-09T06:14:54.315352+0000","last_peered":"2023-10-09T06:14:54.315352+0000","last_clean":"2023-10-09T06:14:54.315352+0000","last_became_active":"2023-10-09T06:14:21.734665+0000","last_became_peered":"2023-10-09T06:14:21.734665+0000","last_unstale":"2023-10-09T06:14:54.315352+0000","last_undegraded":"2023-10-09T06:14:54.315352+0000","last_fullsized":"2023-10-09T06:14:54.315352+0000","mapping_epoch":543,"log_start":"505'3100","ondisk_log_start":"505'3100","created":490,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6481,"log_dups_size":0,"ondisk_log_size":6481,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:01.941433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151977984,"num_objects":2319,"num_object_clones":0,"num_object_copies":6957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9581,"num_write_kb":380800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7073,"num_bytes_recovered":463536128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":606,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:47.339332+0000","last_change":"2023-10-09T06:14:47.339271+0000","last_active":"2023-10-09T06:14:47.339332+0000","last_peered":"2023-10-09T06:14:47.339332+0000","last_clean":"2023-10-09T06:14:47.339332+0000","last_became_active":"2023-10-09T06:13:33.705351+0000","last_became_peered":"2023-10-09T06:13:33.705351+0000","last_unstale":"2023-10-09T06:14:47.339332+0000","last_undegraded":"2023-10-09T06:14:47.339332+0000","last_fullsized":"2023-10-09T06:14:47.339332+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:14:47.339163+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T06:14:47.339163+0000","last_clean_scrub_stamp":"2023-10-09T06:14:47.339163+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:34.887758+0000","scrub_duration":0.0016607729999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":330,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.735667+0000","last_change":"2023-10-09T06:09:27.565004+0000","last_active":"2023-10-09T06:14:43.735667+0000","last_peered":"2023-10-09T06:14:43.735667+0000","last_clean":"2023-10-09T06:14:43.735667+0000","last_became_active":"2023-10-09T06:09:27.564073+0000","last_became_peered":"2023-10-09T06:09:27.564073+0000","last_unstale":"2023-10-09T06:14:43.735667+0000","last_undegraded":"2023-10-09T06:14:43.735667+0000","last_fullsized":"2023-10-09T06:14:43.735667+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:15.146042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":320,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.732305+0000","last_change":"2023-10-09T06:13:21.728291+0000","last_active":"2023-10-09T06:14:43.732305+0000","last_peered":"2023-10-09T06:14:43.732305+0000","last_clean":"2023-10-09T06:14:43.732305+0000","last_became_active":"2023-10-09T06:13:21.727926+0000","last_became_peered":"2023-10-09T06:13:21.727926+0000","last_unstale":"2023-10-09T06:14:43.732305+0000","last_undegraded":"2023-10-09T06:14:43.732305+0000","last_fullsized":"2023-10-09T06:14:43.732305+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:35.292785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":247,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.732807+0000","last_change":"2023-10-09T06:09:27.547953+0000","last_active":"2023-10-09T06:14:43.732807+0000","last_peered":"2023-10-09T06:14:43.732807+0000","last_clean":"2023-10-09T06:14:43.732807+0000","last_became_active":"2023-10-09T06:09:27.547624+0000","last_became_peered":"2023-10-09T06:09:27.547624+0000","last_unstale":"2023-10-09T06:14:43.732807+0000","last_undegraded":"2023-10-09T06:14:43.732807+0000","last_fullsized":"2023-10-09T06:14:43.732807+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:39.630486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"550'9481","reported_seq":17387,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:51.517581+0000","last_change":"2023-10-09T06:14:43.759714+0000","last_active":"2023-10-09T06:14:51.517581+0000","last_peered":"2023-10-09T06:14:51.517581+0000","last_clean":"2023-10-09T06:14:51.517581+0000","last_became_active":"2023-10-09T06:14:43.736592+0000","last_became_peered":"2023-10-09T06:14:43.736592+0000","last_unstale":"2023-10-09T06:14:51.517581+0000","last_undegraded":"2023-10-09T06:14:51.517581+0000","last_fullsized":"2023-10-09T06:14:51.517581+0000","mapping_epoch":548,"log_start":"505'2900","ondisk_log_start":"505'2900","created":490,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6581,"log_dups_size":0,"ondisk_log_size":6581,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:35.196970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160759808,"num_objects":2453,"num_object_clones":0,"num_object_copies":7359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2453,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9481,"num_write_kb":381888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3906,"num_bytes_recovered":255983616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":34,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.735105+0000","last_change":"2023-10-09T06:13:30.791685+0000","last_active":"2023-10-09T06:14:44.735105+0000","last_peered":"2023-10-09T06:14:44.735105+0000","last_clean":"2023-10-09T06:14:44.735105+0000","last_became_active":"2023-10-09T06:13:30.791255+0000","last_became_peered":"2023-10-09T06:13:30.791255+0000","last_unstale":"2023-10-09T06:14:44.735105+0000","last_undegraded":"2023-10-09T06:14:44.735105+0000","last_fullsized":"2023-10-09T06:14:44.735105+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:24.896244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":594,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.734320+0000","last_change":"2023-10-09T06:13:33.704835+0000","last_active":"2023-10-09T06:14:44.734320+0000","last_peered":"2023-10-09T06:14:44.734320+0000","last_clean":"2023-10-09T06:14:44.734320+0000","last_became_active":"2023-10-09T06:13:33.704466+0000","last_became_peered":"2023-10-09T06:13:33.704466+0000","last_unstale":"2023-10-09T06:14:44.734320+0000","last_undegraded":"2023-10-09T06:14:44.734320+0000","last_fullsized":"2023-10-09T06:14:44.734320+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:35.783434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:35.783434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:38.721974+0000","scrub_duration":0.0043794799999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":338,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.735684+0000","last_change":"2023-10-09T06:13:12.238280+0000","last_active":"2023-10-09T06:14:43.735684+0000","last_peered":"2023-10-09T06:14:43.735684+0000","last_clean":"2023-10-09T06:14:43.735684+0000","last_became_active":"2023-10-09T06:13:12.237986+0000","last_became_peered":"2023-10-09T06:13:12.237986+0000","last_unstale":"2023-10-09T06:14:43.735684+0000","last_undegraded":"2023-10-09T06:14:43.735684+0000","last_fullsized":"2023-10-09T06:14:43.735684+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:58.520176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":480,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.742551+0000","last_change":"2023-10-09T06:13:21.733940+0000","last_active":"2023-10-09T06:14:43.742551+0000","last_peered":"2023-10-09T06:14:43.742551+0000","last_clean":"2023-10-09T06:14:43.742551+0000","last_became_active":"2023-10-09T06:13:21.733628+0000","last_became_peered":"2023-10-09T06:13:21.733628+0000","last_unstale":"2023-10-09T06:14:43.742551+0000","last_undegraded":"2023-10-09T06:14:43.742551+0000","last_fullsized":"2023-10-09T06:14:43.742551+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:13.348443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"551'9575","reported_seq":19537,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-09T06:14:54.336386+0000","last_change":"2023-10-09T06:14:09.843785+0000","last_active":"2023-10-09T06:14:54.336386+0000","last_peered":"2023-10-09T06:14:54.336386+0000","last_clean":"2023-10-09T06:14:54.336386+0000","last_became_active":"2023-10-09T06:14:09.789864+0000","last_became_peered":"2023-10-09T06:14:09.789864+0000","last_unstale":"2023-10-09T06:14:54.336386+0000","last_undegraded":"2023-10-09T06:14:54.336386+0000","last_fullsized":"2023-10-09T06:14:54.336386+0000","mapping_epoch":540,"log_start":"505'3100","ondisk_log_start":"505'3100","created":490,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6475,"log_dups_size":0,"ondisk_log_size":6475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:03.191322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153288704,"num_objects":2339,"num_object_clones":0,"num_object_copies":7017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9575,"num_write_kb":381248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4936,"num_bytes_recovered":323485696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":819,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.732408+0000","last_change":"2023-10-09T06:13:24.753842+0000","last_active":"2023-10-09T06:14:43.732408+0000","last_peered":"2023-10-09T06:14:43.732408+0000","last_clean":"2023-10-09T06:14:43.732408+0000","last_became_active":"2023-10-09T06:13:24.753485+0000","last_became_peered":"2023-10-09T06:13:24.753485+0000","last_unstale":"2023-10-09T06:14:43.732408+0000","last_undegraded":"2023-10-09T06:14:43.732408+0000","last_fullsized":"2023-10-09T06:14:43.732408+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:57.627296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:57.627296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:56.656218+0000","scrub_duration":0.001913166,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":444,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.734668+0000","last_change":"2023-10-09T06:09:27.544373+0000","last_active":"2023-10-09T06:14:44.734668+0000","last_peered":"2023-10-09T06:14:44.734668+0000","last_clean":"2023-10-09T06:14:44.734668+0000","last_became_active":"2023-10-09T06:09:27.544121+0000","last_became_peered":"2023-10-09T06:09:27.544121+0000","last_unstale":"2023-10-09T06:14:44.734668+0000","last_undegraded":"2023-10-09T06:14:44.734668+0000","last_fullsized":"2023-10-09T06:14:44.734668+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:40.429491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":487,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.734104+0000","last_change":"2023-10-09T06:12:52.670573+0000","last_active":"2023-10-09T06:14:43.734104+0000","last_peered":"2023-10-09T06:14:43.734104+0000","last_clean":"2023-10-09T06:14:43.734104+0000","last_became_active":"2023-10-09T06:12:52.670242+0000","last_became_peered":"2023-10-09T06:12:52.670242+0000","last_unstale":"2023-10-09T06:14:43.734104+0000","last_undegraded":"2023-10-09T06:14:43.734104+0000","last_fullsized":"2023-10-09T06:14:43.734104+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:25.237580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":626,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.734291+0000","last_change":"2023-10-09T06:09:27.543678+0000","last_active":"2023-10-09T06:14:43.734291+0000","last_peered":"2023-10-09T06:14:43.734291+0000","last_clean":"2023-10-09T06:14:43.734291+0000","last_became_active":"2023-10-09T06:09:27.543298+0000","last_became_peered":"2023-10-09T06:09:27.543298+0000","last_unstale":"2023-10-09T06:14:43.734291+0000","last_undegraded":"2023-10-09T06:14:43.734291+0000","last_fullsized":"2023-10-09T06:14:43.734291+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:26.297842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"550'9479","reported_seq":19080,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:50.889174+0000","last_change":"2023-10-09T06:14:02.643285+0000","last_active":"2023-10-09T06:14:50.889174+0000","last_peered":"2023-10-09T06:14:50.889174+0000","last_clean":"2023-10-09T06:14:50.889174+0000","last_became_active":"2023-10-09T06:14:02.599489+0000","last_became_peered":"2023-10-09T06:14:02.599489+0000","last_unstale":"2023-10-09T06:14:50.889174+0000","last_undegraded":"2023-10-09T06:14:50.889174+0000","last_fullsized":"2023-10-09T06:14:50.889174+0000","mapping_epoch":537,"log_start":"506'4100","ondisk_log_start":"506'4100","created":490,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5379,"log_dups_size":0,"ondisk_log_size":5379,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:10.022074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164102168,"num_objects":2505,"num_object_clones":0,"num_object_copies":7515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2505,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9479,"num_write_kb":383425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4755,"num_bytes_recovered":311558168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":749,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.738172+0000","last_change":"2023-10-09T06:13:24.756288+0000","last_active":"2023-10-09T06:14:43.738172+0000","last_peered":"2023-10-09T06:14:43.738172+0000","last_clean":"2023-10-09T06:14:43.738172+0000","last_became_active":"2023-10-09T06:13:24.755945+0000","last_became_peered":"2023-10-09T06:13:24.755945+0000","last_unstale":"2023-10-09T06:14:43.738172+0000","last_undegraded":"2023-10-09T06:14:43.738172+0000","last_fullsized":"2023-10-09T06:14:43.738172+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:45.844892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:45.844892+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:44.107376+0000","scrub_duration":0.0017521489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":34,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.735200+0000","last_change":"2023-10-09T06:13:30.799535+0000","last_active":"2023-10-09T06:14:44.735200+0000","last_peered":"2023-10-09T06:14:44.735200+0000","last_clean":"2023-10-09T06:14:44.735200+0000","last_became_active":"2023-10-09T06:13:30.799227+0000","last_became_peered":"2023-10-09T06:13:30.799227+0000","last_unstale":"2023-10-09T06:14:44.735200+0000","last_undegraded":"2023-10-09T06:14:44.735200+0000","last_fullsized":"2023-10-09T06:14:44.735200+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:14.741742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":269,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.732602+0000","last_change":"2023-10-09T06:13:12.237622+0000","last_active":"2023-10-09T06:14:43.732602+0000","last_peered":"2023-10-09T06:14:43.732602+0000","last_clean":"2023-10-09T06:14:43.732602+0000","last_became_active":"2023-10-09T06:13:12.237287+0000","last_became_peered":"2023-10-09T06:13:12.237287+0000","last_unstale":"2023-10-09T06:14:43.732602+0000","last_undegraded":"2023-10-09T06:14:43.732602+0000","last_fullsized":"2023-10-09T06:14:43.732602+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:01.942898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":34,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.734933+0000","last_change":"2023-10-09T06:13:30.788382+0000","last_active":"2023-10-09T06:14:44.734933+0000","last_peered":"2023-10-09T06:14:44.734933+0000","last_clean":"2023-10-09T06:14:44.734933+0000","last_became_active":"2023-10-09T06:13:30.788060+0000","last_became_peered":"2023-10-09T06:13:30.788060+0000","last_unstale":"2023-10-09T06:14:44.734933+0000","last_undegraded":"2023-10-09T06:14:44.734933+0000","last_fullsized":"2023-10-09T06:14:44.734933+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:36:58.984799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.e","version":"550'9612","reported_seq":20712,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:50.873876+0000","last_change":"2023-10-09T06:13:51.831598+0000","last_active":"2023-10-09T06:14:50.873876+0000","last_peered":"2023-10-09T06:14:50.873876+0000","last_clean":"2023-10-09T06:14:50.873876+0000","last_became_active":"2023-10-09T06:13:51.816987+0000","last_became_peered":"2023-10-09T06:13:51.816987+0000","last_unstale":"2023-10-09T06:14:50.873876+0000","last_undegraded":"2023-10-09T06:14:50.873876+0000","last_fullsized":"2023-10-09T06:14:50.873876+0000","mapping_epoch":534,"log_start":"506'4200","ondisk_log_start":"506'4200","created":490,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5412,"log_dups_size":0,"ondisk_log_size":5412,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:52.460066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168034304,"num_objects":2564,"num_object_clones":0,"num_object_copies":7692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9612,"num_write_kb":389632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5503,"num_bytes_recovered":360644608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":1023,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.742688+0000","last_change":"2023-10-09T06:13:24.762291+0000","last_active":"2023-10-09T06:14:43.742688+0000","last_peered":"2023-10-09T06:14:43.742688+0000","last_clean":"2023-10-09T06:14:43.742688+0000","last_became_active":"2023-10-09T06:13:24.761918+0000","last_became_peered":"2023-10-09T06:13:24.761918+0000","last_unstale":"2023-10-09T06:14:43.742688+0000","last_undegraded":"2023-10-09T06:14:43.742688+0000","last_fullsized":"2023-10-09T06:14:43.742688+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:17.344518+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:17.344518+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:58.227603+0000","scrub_duration":0.0015914099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":606,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.738395+0000","last_change":"2023-10-09T06:13:30.798394+0000","last_active":"2023-10-09T06:14:43.738395+0000","last_peered":"2023-10-09T06:14:43.738395+0000","last_clean":"2023-10-09T06:14:43.738395+0000","last_became_active":"2023-10-09T06:13:30.790913+0000","last_became_peered":"2023-10-09T06:13:30.790913+0000","last_unstale":"2023-10-09T06:14:43.738395+0000","last_undegraded":"2023-10-09T06:14:43.738395+0000","last_fullsized":"2023-10-09T06:14:43.738395+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:26.508690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":366,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.732750+0000","last_change":"2023-10-09T06:13:21.728009+0000","last_active":"2023-10-09T06:14:43.732750+0000","last_peered":"2023-10-09T06:14:43.732750+0000","last_clean":"2023-10-09T06:14:43.732750+0000","last_became_active":"2023-10-09T06:13:21.727683+0000","last_became_peered":"2023-10-09T06:13:21.727683+0000","last_unstale":"2023-10-09T06:14:43.732750+0000","last_undegraded":"2023-10-09T06:14:43.732750+0000","last_fullsized":"2023-10-09T06:14:43.732750+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:13.074232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":355,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.735915+0000","last_change":"2023-10-09T06:13:30.782038+0000","last_active":"2023-10-09T06:14:43.735915+0000","last_peered":"2023-10-09T06:14:43.735915+0000","last_clean":"2023-10-09T06:14:43.735915+0000","last_became_active":"2023-10-09T06:13:30.781500+0000","last_became_peered":"2023-10-09T06:13:30.781500+0000","last_unstale":"2023-10-09T06:14:43.735915+0000","last_undegraded":"2023-10-09T06:14:43.735915+0000","last_fullsized":"2023-10-09T06:14:43.735915+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:43.663265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"550'9382","reported_seq":9452,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:50.397130+0000","last_change":"2023-10-09T06:11:46.031709+0000","last_active":"2023-10-09T06:14:50.397130+0000","last_peered":"2023-10-09T06:14:50.397130+0000","last_clean":"2023-10-09T06:14:50.397130+0000","last_became_active":"2023-10-09T06:11:46.031421+0000","last_became_peered":"2023-10-09T06:11:46.031421+0000","last_unstale":"2023-10-09T06:14:50.397130+0000","last_undegraded":"2023-10-09T06:14:50.397130+0000","last_fullsized":"2023-10-09T06:14:50.397130+0000","mapping_epoch":490,"log_start":"505'3425","ondisk_log_start":"505'3425","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5957,"log_dups_size":0,"ondisk_log_size":5957,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:13.095174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166330368,"num_objects":2538,"num_object_clones":0,"num_object_copies":7614,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2538,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9382,"num_write_kb":381440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":882,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.737912+0000","last_change":"2023-10-09T06:13:33.708100+0000","last_active":"2023-10-09T06:14:43.737912+0000","last_peered":"2023-10-09T06:14:43.737912+0000","last_clean":"2023-10-09T06:14:43.737912+0000","last_became_active":"2023-10-09T06:13:33.707809+0000","last_became_peered":"2023-10-09T06:13:33.707809+0000","last_unstale":"2023-10-09T06:14:43.737912+0000","last_undegraded":"2023-10-09T06:14:43.737912+0000","last_fullsized":"2023-10-09T06:14:43.737912+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:43:26.848778+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":34,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.735728+0000","last_change":"2023-10-09T06:13:30.799948+0000","last_active":"2023-10-09T06:14:44.735728+0000","last_peered":"2023-10-09T06:14:44.735728+0000","last_clean":"2023-10-09T06:14:44.735728+0000","last_became_active":"2023-10-09T06:13:30.799690+0000","last_became_peered":"2023-10-09T06:13:30.799690+0000","last_unstale":"2023-10-09T06:14:44.735728+0000","last_undegraded":"2023-10-09T06:14:44.735728+0000","last_fullsized":"2023-10-09T06:14:44.735728+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:43.893184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":596,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.734569+0000","last_change":"2023-10-09T06:09:27.578756+0000","last_active":"2023-10-09T06:14:43.734569+0000","last_peered":"2023-10-09T06:14:43.734569+0000","last_clean":"2023-10-09T06:14:43.734569+0000","last_became_active":"2023-10-09T06:09:27.578384+0000","last_became_peered":"2023-10-09T06:09:27.578384+0000","last_unstale":"2023-10-09T06:14:43.734569+0000","last_undegraded":"2023-10-09T06:14:43.734569+0000","last_fullsized":"2023-10-09T06:14:43.734569+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:22.182630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":496,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.742304+0000","last_change":"2023-10-09T06:01:22.334081+0000","last_active":"2023-10-09T06:14:43.742304+0000","last_peered":"2023-10-09T06:14:43.742304+0000","last_clean":"2023-10-09T06:14:43.742304+0000","last_became_active":"2023-10-09T06:01:22.333663+0000","last_became_peered":"2023-10-09T06:01:22.333663+0000","last_unstale":"2023-10-09T06:14:43.742304+0000","last_undegraded":"2023-10-09T06:14:43.742304+0000","last_fullsized":"2023-10-09T06:14:43.742304+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:55.809259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":796,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.735515+0000","last_change":"2023-10-09T06:09:27.560157+0000","last_active":"2023-10-09T06:14:43.735515+0000","last_peered":"2023-10-09T06:14:43.735515+0000","last_clean":"2023-10-09T06:14:43.735515+0000","last_became_active":"2023-10-09T06:09:27.559466+0000","last_became_peered":"2023-10-09T06:09:27.559466+0000","last_unstale":"2023-10-09T06:14:43.735515+0000","last_undegraded":"2023-10-09T06:14:43.735515+0000","last_fullsized":"2023-10-09T06:14:43.735515+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.181811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.181811+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:53.431491+0000","scrub_duration":0.011553313000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":479,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.734241+0000","last_change":"2023-10-09T06:13:30.791343+0000","last_active":"2023-10-09T06:14:43.734241+0000","last_peered":"2023-10-09T06:14:43.734241+0000","last_clean":"2023-10-09T06:14:43.734241+0000","last_became_active":"2023-10-09T06:13:30.790945+0000","last_became_peered":"2023-10-09T06:13:30.790945+0000","last_unstale":"2023-10-09T06:14:43.734241+0000","last_undegraded":"2023-10-09T06:14:43.734241+0000","last_fullsized":"2023-10-09T06:14:43.734241+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:14.208391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":252,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.732703+0000","last_change":"2023-10-09T06:07:43.580844+0000","last_active":"2023-10-09T06:14:43.732703+0000","last_peered":"2023-10-09T06:14:43.732703+0000","last_clean":"2023-10-09T06:14:43.732703+0000","last_became_active":"2023-10-09T06:07:43.580523+0000","last_became_peered":"2023-10-09T06:07:43.580523+0000","last_unstale":"2023-10-09T06:14:43.732703+0000","last_undegraded":"2023-10-09T06:14:43.732703+0000","last_fullsized":"2023-10-09T06:14:43.732703+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:55.942430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":899,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.738016+0000","last_change":"2023-10-09T06:00:04.215425+0000","last_active":"2023-10-09T06:14:43.738016+0000","last_peered":"2023-10-09T06:14:43.738016+0000","last_clean":"2023-10-09T06:14:43.738016+0000","last_became_active":"2023-10-09T05:59:40.326685+0000","last_became_peered":"2023-10-09T05:59:40.326685+0000","last_unstale":"2023-10-09T06:14:43.738016+0000","last_undegraded":"2023-10-09T06:14:43.738016+0000","last_fullsized":"2023-10-09T06:14:43.738016+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:04.215270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:04.215270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:03.963694+0000","scrub_duration":0.002850652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":321,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.743238+0000","last_change":"2023-10-09T06:01:54.824780+0000","last_active":"2023-10-09T06:14:43.743238+0000","last_peered":"2023-10-09T06:14:43.743238+0000","last_clean":"2023-10-09T06:14:43.743238+0000","last_became_active":"2023-10-09T06:01:54.757727+0000","last_became_peered":"2023-10-09T06:01:54.757727+0000","last_unstale":"2023-10-09T06:14:43.743238+0000","last_undegraded":"2023-10-09T06:14:43.743238+0000","last_fullsized":"2023-10-09T06:14:43.743238+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:44.132556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":568,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.742634+0000","last_change":"2023-10-09T06:01:54.764989+0000","last_active":"2023-10-09T06:14:43.742634+0000","last_peered":"2023-10-09T06:14:43.742634+0000","last_clean":"2023-10-09T06:14:43.742634+0000","last_became_active":"2023-10-09T06:01:54.764790+0000","last_became_peered":"2023-10-09T06:01:54.764790+0000","last_unstale":"2023-10-09T06:14:43.742634+0000","last_undegraded":"2023-10-09T06:14:43.742634+0000","last_fullsized":"2023-10-09T06:14:43.742634+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:14:07.208858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":335,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.742428+0000","last_change":"2023-10-09T06:13:21.739730+0000","last_active":"2023-10-09T06:14:43.742428+0000","last_peered":"2023-10-09T06:14:43.742428+0000","last_clean":"2023-10-09T06:14:43.742428+0000","last_became_active":"2023-10-09T06:13:21.727326+0000","last_became_peered":"2023-10-09T06:13:21.727326+0000","last_unstale":"2023-10-09T06:14:43.742428+0000","last_undegraded":"2023-10-09T06:14:43.742428+0000","last_fullsized":"2023-10-09T06:14:43.742428+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:51.344571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":565,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.742271+0000","last_change":"2023-10-09T06:09:27.565198+0000","last_active":"2023-10-09T06:14:43.742271+0000","last_peered":"2023-10-09T06:14:43.742271+0000","last_clean":"2023-10-09T06:14:43.742271+0000","last_became_active":"2023-10-09T06:09:27.564909+0000","last_became_peered":"2023-10-09T06:09:27.564909+0000","last_unstale":"2023-10-09T06:14:43.742271+0000","last_undegraded":"2023-10-09T06:14:43.742271+0000","last_fullsized":"2023-10-09T06:14:43.742271+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:46.101014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":716,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-09T06:14:44.735152+0000","last_change":"2023-10-09T06:13:33.699235+0000","last_active":"2023-10-09T06:14:44.735152+0000","last_peered":"2023-10-09T06:14:44.735152+0000","last_clean":"2023-10-09T06:14:44.735152+0000","last_became_active":"2023-10-09T06:13:33.698886+0000","last_became_peered":"2023-10-09T06:13:33.698886+0000","last_unstale":"2023-10-09T06:14:44.735152+0000","last_undegraded":"2023-10-09T06:14:44.735152+0000","last_fullsized":"2023-10-09T06:14:44.735152+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:00.456132+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:00.456132+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:59.421706+0000","scrub_duration":0.0014992199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":356,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.738268+0000","last_change":"2023-10-09T06:13:21.740383+0000","last_active":"2023-10-09T06:14:43.738268+0000","last_peered":"2023-10-09T06:14:43.738268+0000","last_clean":"2023-10-09T06:14:43.738268+0000","last_became_active":"2023-10-09T06:13:21.740023+0000","last_became_peered":"2023-10-09T06:13:21.740023+0000","last_unstale":"2023-10-09T06:14:43.738268+0000","last_undegraded":"2023-10-09T06:14:43.738268+0000","last_fullsized":"2023-10-09T06:14:43.738268+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:27.380765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":352,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.735685+0000","last_change":"2023-10-09T06:09:27.565185+0000","last_active":"2023-10-09T06:14:43.735685+0000","last_peered":"2023-10-09T06:14:43.735685+0000","last_clean":"2023-10-09T06:14:43.735685+0000","last_became_active":"2023-10-09T06:09:27.564478+0000","last_became_peered":"2023-10-09T06:09:27.564478+0000","last_unstale":"2023-10-09T06:14:43.735685+0000","last_undegraded":"2023-10-09T06:14:43.735685+0000","last_fullsized":"2023-10-09T06:14:43.735685+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:04.622171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":874,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.732821+0000","last_change":"2023-10-09T06:12:55.753877+0000","last_active":"2023-10-09T06:14:43.732821+0000","last_peered":"2023-10-09T06:14:43.732821+0000","last_clean":"2023-10-09T06:14:43.732821+0000","last_became_active":"2023-10-09T06:12:55.753512+0000","last_became_peered":"2023-10-09T06:12:55.753512+0000","last_unstale":"2023-10-09T06:14:43.732821+0000","last_undegraded":"2023-10-09T06:14:43.732821+0000","last_fullsized":"2023-10-09T06:14:43.732821+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:46.290510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:46.290510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:30.368031+0000","scrub_duration":0.002627819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":542,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.734411+0000","last_change":"2023-10-09T06:09:27.556271+0000","last_active":"2023-10-09T06:14:43.734411+0000","last_peered":"2023-10-09T06:14:43.734411+0000","last_clean":"2023-10-09T06:14:43.734411+0000","last_became_active":"2023-10-09T06:09:27.555932+0000","last_became_peered":"2023-10-09T06:09:27.555932+0000","last_unstale":"2023-10-09T06:14:43.734411+0000","last_undegraded":"2023-10-09T06:14:43.734411+0000","last_fullsized":"2023-10-09T06:14:43.734411+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:33.888622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":581,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-09T06:14:43.742045+0000","last_change":"2023-10-09T06:13:21.727720+0000","last_active":"2023-10-09T06:14:43.742045+0000","last_peered":"2023-10-09T06:14:43.742045+0000","last_clean":"2023-10-09T06:14:43.742045+0000","last_became_active":"2023-10-09T06:13:21.727339+0000","last_became_peered":"2023-10-09T06:13:21.727339+0000","last_unstale":"2023-10-09T06:14:43.742045+0000","last_undegraded":"2023-10-09T06:14:43.742045+0000","last_fullsized":"2023-10-09T06:14:43.742045+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:25.723752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2615279640,"num_objects":39907,"num_object_clones":0,"num_object_copies":119721,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2736,"num_objects_unfound":0,"num_objects_dirty":39907,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":152519,"num_write_kb":6157569,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41578,"num_bytes_recovered":2724790296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9682632704,"data_stored":9682616416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97866,"ondisk_log_size":97866,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":11714,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13410,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1443424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234423,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1504868,"kb_used_data":612124,"kb_used_omap":0,"kb_used_meta":892608,"kb_avail":92240284,"statfs":{"total":95995035648,"available":94454050816,"internally_reserved":0,"allocated":626814976,"data_stored":625248701,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":914030592},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:14:32 2023","interfaces":[{"interface":"back","average":{"1min":14.369,"5min":5.5149999999999997,"15min":2.6600000000000001},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":126.167,"5min":126.167,"15min":126.167},"last":5.9189999999999996},{"interface":"front","average":{"1min":14.465,"5min":5.7229999999999999,"15min":2.6709999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":126.139,"5min":126.139,"15min":126.139},"last":5.9400000000000004}]},{"osd":1,"last update":"Mon Oct 9 06:14:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":4.141,"15min":3.2189999999999999},"min":{"1min":0.27200000000000002,"5min":0.255,"15min":0.218},"max":{"1min":5.9580000000000002,"5min":124.66200000000001,"15min":124.66200000000001},"last":0.81100000000000005},{"interface":"front","average":{"1min":1.216,"5min":4.1829999999999998,"15min":3.2069999999999999},"min":{"1min":0.33900000000000002,"5min":0.27100000000000002,"15min":0.22700000000000001},"max":{"1min":5.8650000000000002,"5min":125.023,"15min":125.023},"last":6.9660000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:14:32 2023","interfaces":[{"interface":"back","average":{"1min":3.766,"5min":2.1549999999999998,"15min":2.403},"min":{"1min":0.29899999999999999,"5min":0.27400000000000002,"15min":0.24299999999999999},"max":{"1min":28.475999999999999,"5min":47.503,"15min":127.86499999999999},"last":0.65500000000000003},{"interface":"front","average":{"1min":3.4929999999999999,"5min":2.3719999999999999,"15min":2.391},"min":{"1min":0.30599999999999999,"5min":0.30099999999999999,"15min":0.27000000000000002},"max":{"1min":28.925000000000001,"5min":47.545000000000002,"15min":129.24000000000001},"last":0.40699999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:14:05 2023","interfaces":[{"interface":"back","average":{"1min":10.307,"5min":7.9900000000000002,"15min":6.4820000000000002},"min":{"1min":0.35499999999999998,"5min":0.32100000000000001,"15min":0.29899999999999999},"max":{"1min":67.852000000000004,"5min":116.68000000000001,"15min":164.16999999999999},"last":0.77200000000000002},{"interface":"front","average":{"1min":10.298,"5min":8.2260000000000009,"15min":6.6589999999999998},"min":{"1min":0.36699999999999999,"5min":0.33300000000000002,"15min":0.32100000000000001},"max":{"1min":67.881,"5min":116.797,"15min":164.40700000000001},"last":5.0179999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:14:41 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":0.98499999999999999,"15min":0.95799999999999996},"min":{"1min":0.27900000000000003,"5min":0.23999999999999999,"15min":0.23100000000000001},"max":{"1min":6.9939999999999998,"5min":6.9939999999999998,"15min":14.371},"last":0.32600000000000001},{"interface":"front","average":{"1min":1.3520000000000001,"5min":0.90100000000000002,"15min":0.94799999999999995},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":7.9459999999999997,"5min":7.9459999999999997,"15min":14.259},"last":0.34100000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":1.3420000000000001,"15min":1.083},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.24399999999999999},"max":{"1min":2.4700000000000002,"5min":43.826999999999998,"15min":43.826999999999998},"last":0.75},{"interface":"front","average":{"1min":1.038,"5min":1.2729999999999999,"15min":1.139},"min":{"1min":0.34899999999999998,"5min":0.32300000000000001,"15min":0.24399999999999999},"max":{"1min":5.9569999999999999,"5min":43.880000000000003,"15min":43.880000000000003},"last":0.379}]},{"osd":6,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":0.78600000000000003,"15min":0.97999999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.26600000000000001},"max":{"1min":2.468,"5min":3.8679999999999999,"15min":9.4540000000000006},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.90200000000000002,"5min":0.80600000000000005,"15min":1.018},"min":{"1min":0.33600000000000002,"5min":0.32000000000000001,"15min":0.30599999999999999},"max":{"1min":3.3279999999999998,"5min":4.516,"15min":6.2329999999999997},"last":1.6359999999999999}]}]},{"osd":6,"up_from":46,"seq":197568496060,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1922772,"kb_used_data":1021132,"kb_used_omap":0,"kb_used_meta":901632,"kb_avail":91822380,"statfs":{"total":95995035648,"available":94026117120,"internally_reserved":0,"allocated":1045639168,"data_stored":1044088346,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923271168},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:14:29 2023","interfaces":[{"interface":"back","average":{"1min":11.574,"5min":8.9060000000000006,"15min":4.4640000000000004},"min":{"1min":0.25,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":92.424999999999997,"5min":191.50800000000001,"15min":191.50800000000001},"last":0.30499999999999999},{"interface":"front","average":{"1min":11.849,"5min":9.0709999999999997,"15min":4.3490000000000002},"min":{"1min":0.30399999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":92.396000000000001,"5min":191.52600000000001,"15min":191.52600000000001},"last":4.0640000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:14:28 2023","interfaces":[{"interface":"back","average":{"1min":13.445,"5min":8.6509999999999998,"15min":6.0129999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.23200000000000001},"max":{"1min":193.845,"5min":328.04399999999998,"15min":328.04399999999998},"last":0.35599999999999998},{"interface":"front","average":{"1min":13.298,"5min":8.8420000000000005,"15min":6.1219999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":193.779,"5min":328.10000000000002,"15min":328.10000000000002},"last":0.68600000000000005}]},{"osd":2,"last update":"Mon Oct 9 06:14:29 2023","interfaces":[{"interface":"back","average":{"1min":1.5600000000000001,"5min":4.7709999999999999,"15min":2.944},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.7549999999999999,"5min":163.04900000000001,"15min":163.04900000000001},"last":2.948},{"interface":"front","average":{"1min":1.3759999999999999,"5min":4.8239999999999998,"15min":2.879},"min":{"1min":0.32100000000000001,"5min":0.312,"15min":0.28599999999999998},"max":{"1min":5.9820000000000002,"5min":162.93000000000001,"15min":162.93000000000001},"last":0.71799999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:14:12 2023","interfaces":[{"interface":"back","average":{"1min":11.064,"5min":8.5009999999999994,"15min":7.375},"min":{"1min":0.33000000000000002,"5min":0.318,"15min":0.307},"max":{"1min":81.317999999999998,"5min":250.744,"15min":250.744},"last":0.38300000000000001},{"interface":"front","average":{"1min":9.3520000000000003,"5min":7.9539999999999997,"15min":7.1600000000000001},"min":{"1min":0.374,"5min":0.33600000000000002,"15min":0.314},"max":{"1min":81.233000000000004,"5min":250.83799999999999,"15min":250.83799999999999},"last":1.4690000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":1.018,"15min":1.6519999999999999},"min":{"1min":0.28899999999999998,"5min":0.24399999999999999,"15min":0.24199999999999999},"max":{"1min":5.4880000000000004,"5min":9.0869999999999997,"15min":151.35599999999999},"last":1.105},{"interface":"front","average":{"1min":1.091,"5min":1.121,"15min":1.7350000000000001},"min":{"1min":0.307,"5min":0.249,"15min":0.23999999999999999},"max":{"1min":5.077,"5min":7.8849999999999998,"15min":149.15799999999999},"last":3.1699999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:14:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":0.94299999999999995,"15min":1.1040000000000001},"min":{"1min":0.315,"5min":0.28000000000000003,"15min":0.23000000000000001},"max":{"1min":8.3829999999999991,"5min":9.1959999999999997,"15min":15.342000000000001},"last":0.73999999999999999},{"interface":"front","average":{"1min":0.95699999999999996,"5min":1.03,"15min":1.1579999999999999},"min":{"1min":0.316,"5min":0.28699999999999998,"15min":0.255},"max":{"1min":2.8450000000000002,"5min":8.9809999999999999,"15min":15.358000000000001},"last":1.6799999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:14:48 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":0.96999999999999997,"15min":1.238},"min":{"1min":0.33700000000000002,"5min":0.30299999999999999,"15min":0.27000000000000002},"max":{"1min":5.8380000000000001,"5min":5.8380000000000001,"15min":21.98},"last":4.9059999999999997},{"interface":"front","average":{"1min":1.01,"5min":0.86399999999999999,"15min":1.1739999999999999},"min":{"1min":0.34999999999999998,"5min":0.34200000000000003,"15min":0.33100000000000002},"max":{"1min":3.714,"5min":3.714,"15min":18.241},"last":0.61499999999999999}]}]},{"osd":1,"up_from":89,"seq":382252089647,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1063588,"kb_used_data":485148,"kb_used_omap":9,"kb_used_meta":578358,"kb_avail":92681564,"statfs":{"total":95995035648,"available":94905921536,"internally_reserved":0,"allocated":496791552,"data_stored":493634772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":592239593},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:14:39 2023","interfaces":[{"interface":"back","average":{"1min":4.2060000000000004,"5min":2.3319999999999999,"15min":2.294},"min":{"1min":0.221,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":35.939999999999998,"5min":35.939999999999998,"15min":35.939999999999998},"last":55.652000000000001},{"interface":"front","average":{"1min":4.3760000000000003,"5min":2.798,"15min":2.492},"min":{"1min":0.21199999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":35.326000000000001,"5min":35.326000000000001,"15min":35.326000000000001},"last":54.527999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:14:45 2023","interfaces":[{"interface":"back","average":{"1min":1.2849999999999999,"5min":1.7929999999999999,"15min":2.4630000000000001},"min":{"1min":0.29899999999999999,"5min":0.217,"15min":0.20799999999999999},"max":{"1min":4.8929999999999998,"5min":15.522,"15min":129.053},"last":0.29299999999999998},{"interface":"front","average":{"1min":4.8550000000000004,"5min":2.9409999999999998,"15min":2.9350000000000001},"min":{"1min":0.32400000000000001,"5min":0.22600000000000001,"15min":0.222},"max":{"1min":55.694000000000003,"5min":55.694000000000003,"15min":129.077},"last":0.27400000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:14:13 2023","interfaces":[{"interface":"back","average":{"1min":5.1669999999999998,"5min":4.484,"15min":5.0309999999999997},"min":{"1min":0.28000000000000003,"5min":0.26800000000000002,"15min":0.24199999999999999},"max":{"1min":48.177999999999997,"5min":113.322,"15min":213.654},"last":0.38100000000000001},{"interface":"front","average":{"1min":4.3879999999999999,"5min":4.6349999999999998,"15min":5.0250000000000004},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.27400000000000002},"max":{"1min":48.216999999999999,"5min":113.38200000000001,"15min":213.77199999999999},"last":0.45300000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:14:45 2023","interfaces":[{"interface":"back","average":{"1min":3.7559999999999998,"5min":3.4860000000000002,"15min":2.3149999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":54.018000000000001,"5min":93.747,"15min":93.747},"last":0.49199999999999999},{"interface":"front","average":{"1min":5.0860000000000003,"5min":4.4219999999999997,"15min":2.8319999999999999},"min":{"1min":0.35799999999999998,"5min":0.308,"15min":0.308},"max":{"1min":54.280000000000001,"5min":97.852000000000004,"15min":97.852000000000004},"last":0.36499999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:13:57 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.6220000000000001,"15min":1.5680000000000001},"min":{"1min":0.32200000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.4850000000000003,"5min":9.3109999999999999,"15min":36.356999999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":1.383,"5min":2.5870000000000002,"15min":2.27},"min":{"1min":0.34599999999999997,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.2619999999999996,"5min":17.603999999999999,"15min":36.268000000000001},"last":0.40999999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:13:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":1.7410000000000001,"15min":1.7789999999999999},"min":{"1min":0.38500000000000001,"5min":0.33700000000000002,"15min":0.318},"max":{"1min":5.5170000000000003,"5min":9.282,"15min":52.207000000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.901,"5min":2.4180000000000001,"15min":2.3559999999999999},"min":{"1min":0.38500000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":8.7159999999999993,"5min":15.384,"15min":51.212000000000003},"last":0.44600000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:13:57 2023","interfaces":[{"interface":"back","average":{"1min":5.7400000000000002,"5min":3.3849999999999998,"15min":2.9329999999999998},"min":{"1min":0.39800000000000002,"5min":0.374,"15min":0.374},"max":{"1min":56.881,"5min":56.881,"15min":108.553},"last":0.47699999999999998},{"interface":"front","average":{"1min":5.7210000000000001,"5min":2.6360000000000001,"15min":2.3980000000000001},"min":{"1min":0.46000000000000002,"5min":0.38,"15min":0.38},"max":{"1min":56.941000000000003,"5min":56.941000000000003,"15min":108.56999999999999},"last":0.54100000000000004}]}]},{"osd":0,"up_from":456,"seq":1958505087042,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2087548,"kb_used_data":1799156,"kb_used_omap":32,"kb_used_meta":288287,"kb_avail":91657604,"statfs":{"total":95995035648,"available":93857386496,"internally_reserved":0,"allocated":1842335744,"data_stored":1839395796,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32794,"internal_metadata":295206886},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:14:40 2023","interfaces":[{"interface":"back","average":{"1min":15.137,"5min":6.5940000000000003,"15min":9.6199999999999992},"min":{"1min":0.24099999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":102.81999999999999,"5min":166.833,"15min":166.833},"last":0.41099999999999998},{"interface":"front","average":{"1min":14.031000000000001,"5min":6.9859999999999998,"15min":8.9900000000000002},"min":{"1min":0.28699999999999998,"5min":0.223,"15min":0.223},"max":{"1min":103.108,"5min":166.81100000000001,"15min":166.81100000000001},"last":0.46600000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:14:40 2023","interfaces":[{"interface":"back","average":{"1min":2.7400000000000002,"5min":2.649,"15min":3.589},"min":{"1min":0.30599999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":12.413,"5min":21.648,"15min":21.648},"last":3.3279999999999998},{"interface":"front","average":{"1min":2.21,"5min":2.3210000000000002,"15min":3.375},"min":{"1min":0.25700000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":10.119,"5min":20.152000000000001,"15min":20.152000000000001},"last":2.6499999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:14:32 2023","interfaces":[{"interface":"back","average":{"1min":3.2850000000000001,"5min":5.0039999999999996,"15min":12.284000000000001},"min":{"1min":0.38700000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":10.079000000000001,"5min":203.59299999999999,"15min":203.59299999999999},"last":0.83399999999999996},{"interface":"front","average":{"1min":3.2440000000000002,"5min":5.3019999999999996,"15min":11.779999999999999},"min":{"1min":0.30299999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":12.191000000000001,"5min":202.96600000000001,"15min":202.96600000000001},"last":3.363}]},{"osd":4,"last update":"Mon Oct 9 06:14:40 2023","interfaces":[{"interface":"back","average":{"1min":3.9289999999999998,"5min":2.3700000000000001,"15min":2.3959999999999999},"min":{"1min":0.39400000000000002,"5min":0.317,"15min":0.317},"max":{"1min":21.994,"5min":23.515999999999998,"15min":23.515999999999998},"last":0.38100000000000001},{"interface":"front","average":{"1min":3.3319999999999999,"5min":2.7839999999999998,"15min":2.2559999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":21.972999999999999,"5min":39.302,"15min":39.302},"last":0.51600000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:14:40 2023","interfaces":[{"interface":"back","average":{"1min":2.6179999999999999,"5min":3.8050000000000002,"15min":2.7280000000000002},"min":{"1min":0.32700000000000001,"5min":0.316,"15min":0.316},"max":{"1min":8.2949999999999999,"5min":161.286,"15min":161.286},"last":0.495},{"interface":"front","average":{"1min":2.8010000000000002,"5min":3.597,"15min":2.8479999999999999},"min":{"1min":0.40000000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":12.038,"5min":161.31800000000001,"15min":161.31800000000001},"last":3.2229999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:14:40 2023","interfaces":[{"interface":"back","average":{"1min":4.657,"5min":2.5550000000000002,"15min":2.5030000000000001},"min":{"1min":0.43099999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":31.648,"5min":31.648,"15min":31.648},"last":3.2709999999999999},{"interface":"front","average":{"1min":3.5910000000000002,"5min":2.081,"15min":2.585},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":24.43,"5min":24.43,"15min":24.43},"last":0.58299999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:14:40 2023","interfaces":[{"interface":"back","average":{"1min":2.859,"5min":3.5720000000000001,"15min":2.879},"min":{"1min":0.53700000000000003,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":12.066000000000001,"5min":123.685,"15min":123.685},"last":3.302},{"interface":"front","average":{"1min":2.4089999999999998,"5min":3.1429999999999998,"15min":2.8889999999999998},"min":{"1min":0.55900000000000005,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":10.054,"5min":122.482,"15min":122.482},"last":0.60699999999999998}]}]},{"osd":2,"up_from":94,"seq":403726926139,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1887336,"kb_used_data":1104224,"kb_used_omap":81,"kb_used_meta":783022,"kb_avail":91857816,"statfs":{"total":95995035648,"available":94062403584,"internally_reserved":0,"allocated":1130725376,"data_stored":1127925815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":801814746},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:14:41 2023","interfaces":[{"interface":"back","average":{"1min":7.6920000000000002,"5min":4.415,"15min":3.5710000000000002},"min":{"1min":0.35999999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":83.652000000000001,"5min":83.652000000000001,"15min":83.652000000000001},"last":0.89300000000000002},{"interface":"front","average":{"1min":7.3620000000000001,"5min":3.8380000000000001,"15min":2.774},"min":{"1min":0.35399999999999998,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":77.578999999999994,"5min":77.578999999999994,"15min":77.578999999999994},"last":3.2850000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:14:41 2023","interfaces":[{"interface":"back","average":{"1min":7.0499999999999998,"5min":5.8970000000000002,"15min":4.9119999999999999},"min":{"1min":0.27300000000000002,"5min":0.248,"15min":0.19400000000000001},"max":{"1min":78.921999999999997,"5min":160.45500000000001,"15min":222.59399999999999},"last":0.78500000000000003},{"interface":"front","average":{"1min":8.5739999999999998,"5min":7.0979999999999999,"15min":5.4329999999999998},"min":{"1min":0.313,"5min":0.27200000000000002,"15min":0.20200000000000001},"max":{"1min":81.292000000000002,"5min":163.697,"15min":224.619},"last":0.315}]},{"osd":3,"last update":"Mon Oct 9 06:13:58 2023","interfaces":[{"interface":"back","average":{"1min":12.108000000000001,"5min":7.2910000000000004,"15min":5.101},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23000000000000001},"max":{"1min":122.205,"5min":122.205,"15min":144.88200000000001},"last":5.8099999999999996},{"interface":"front","average":{"1min":11.619,"5min":6.952,"15min":5.0369999999999999},"min":{"1min":0.30199999999999999,"5min":0.252,"15min":0.252},"max":{"1min":122.45,"5min":122.45,"15min":140.79499999999999},"last":0.68300000000000005}]},{"osd":4,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":2.8210000000000002,"5min":3.4449999999999998,"15min":2.2989999999999999},"min":{"1min":0.33600000000000002,"5min":0.32200000000000001,"15min":0.28299999999999997},"max":{"1min":9.3849999999999998,"5min":107.51300000000001,"15min":107.51300000000001},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.645,"5min":2.4580000000000002,"15min":2.0409999999999999},"min":{"1min":0.36299999999999999,"5min":0.34100000000000003,"15min":0.31},"max":{"1min":4.6589999999999998,"5min":108.768,"15min":108.768},"last":0.73599999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:14:41 2023","interfaces":[{"interface":"back","average":{"1min":1.788,"5min":1.528,"15min":1.5569999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32600000000000001},"max":{"1min":11.108000000000001,"5min":11.108000000000001,"15min":28.48},"last":0.622},{"interface":"front","average":{"1min":1.8500000000000001,"5min":1.516,"15min":1.79},"min":{"1min":0.40200000000000002,"5min":0.316,"15min":0.28899999999999998},"max":{"1min":4.6890000000000001,"5min":9.8699999999999992,"15min":30.318999999999999},"last":0.71099999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:14:41 2023","interfaces":[{"interface":"back","average":{"1min":2.7890000000000001,"5min":2.407,"15min":2.1850000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.314},"max":{"1min":10.09,"5min":14.962,"15min":30.420000000000002},"last":0.52200000000000002},{"interface":"front","average":{"1min":2.1739999999999999,"5min":1.6619999999999999,"15min":1.7969999999999999},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.32200000000000001},"max":{"1min":6.3940000000000001,"5min":11.103,"15min":30.713000000000001},"last":0.58299999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:14:41 2023","interfaces":[{"interface":"back","average":{"1min":2.2469999999999999,"5min":2.919,"15min":2.2959999999999998},"min":{"1min":0.39400000000000002,"5min":0.313,"15min":0.313},"max":{"1min":11.147,"5min":124.069,"15min":124.069},"last":0.64900000000000002},{"interface":"front","average":{"1min":2.3660000000000001,"5min":2.7170000000000001,"15min":2.3450000000000002},"min":{"1min":0.372,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.6769999999999996,"5min":124.08799999999999,"15min":124.08799999999999},"last":2.2759999999999998}]}]},{"osd":3,"up_from":99,"seq":425201762612,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2371972,"kb_used_data":1406204,"kb_used_omap":7,"kb_used_meta":965752,"kb_avail":91373180,"statfs":{"total":95995035648,"available":93566136320,"internally_reserved":0,"allocated":1439952896,"data_stored":1436820026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":988930664},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:14:36 2023","interfaces":[{"interface":"back","average":{"1min":20.771999999999998,"5min":5.6909999999999998,"15min":3.3029999999999999},"min":{"1min":0.19600000000000001,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":220.45400000000001,"5min":220.45400000000001,"15min":220.45400000000001},"last":1.8560000000000001},{"interface":"front","average":{"1min":21.698,"5min":6.5,"15min":4.5170000000000003},"min":{"1min":0.317,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":216.44200000000001,"5min":216.44200000000001,"15min":216.44200000000001},"last":1.756}]},{"osd":1,"last update":"Mon Oct 9 06:14:10 2023","interfaces":[{"interface":"back","average":{"1min":4.1269999999999998,"5min":2.218,"15min":2.9510000000000001},"min":{"1min":0.253,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":21.934000000000001,"5min":22.579999999999998,"15min":207.75299999999999},"last":1.8100000000000001},{"interface":"front","average":{"1min":4.3049999999999997,"5min":3.54,"15min":3.9649999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.23200000000000001},"max":{"1min":21.809000000000001,"5min":30.524000000000001,"15min":207.12200000000001},"last":9.6020000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:14:10 2023","interfaces":[{"interface":"back","average":{"1min":2.3889999999999998,"5min":2.4609999999999999,"15min":2.6549999999999998},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.223},"max":{"1min":12.561,"5min":30.547999999999998,"15min":107.01300000000001},"last":0.93799999999999994},{"interface":"front","average":{"1min":2.6640000000000001,"5min":3.0899999999999999,"15min":3.0699999999999998},"min":{"1min":0.313,"5min":0.25600000000000001,"15min":0.23899999999999999},"max":{"1min":11.749000000000001,"5min":25.984000000000002,"15min":109.807},"last":8.9049999999999994}]},{"osd":4,"last update":"Mon Oct 9 06:13:46 2023","interfaces":[{"interface":"back","average":{"1min":6.8099999999999996,"5min":2.8759999999999999,"15min":3.5569999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.254},"max":{"1min":95.739000000000004,"5min":95.739000000000004,"15min":99.876000000000005},"last":0.439},{"interface":"front","average":{"1min":6.8259999999999996,"5min":3.399,"15min":3.7210000000000001},"min":{"1min":0.373,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":97.941999999999993,"5min":97.941999999999993,"15min":99.914000000000001},"last":1.883}]},{"osd":5,"last update":"Mon Oct 9 06:14:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2989999999999999,"5min":2.7200000000000002,"15min":2.0649999999999999},"min":{"1min":0.40500000000000003,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.3970000000000002,"5min":74.706000000000003,"15min":74.706000000000003},"last":0.61199999999999999},{"interface":"front","average":{"1min":1.202,"5min":3.4420000000000002,"15min":3.2759999999999998},"min":{"1min":0.32800000000000001,"5min":0.32500000000000001,"15min":0.28799999999999998},"max":{"1min":4.0449999999999999,"5min":74.685000000000002,"15min":235.56},"last":0.48399999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:14:10 2023","interfaces":[{"interface":"back","average":{"1min":1.734,"5min":1.351,"15min":4.3070000000000004},"min":{"1min":0.46500000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.4210000000000003,"5min":6.8739999999999997,"15min":233.26599999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":2.266,"5min":2.2069999999999999,"15min":4.7619999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.33600000000000002},"max":{"1min":12.471,"5min":26.890000000000001,"15min":233.24100000000001},"last":0.51500000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:14:10 2023","interfaces":[{"interface":"back","average":{"1min":7.8600000000000003,"5min":4.2569999999999997,"15min":5.2229999999999999},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.35699999999999998},"max":{"1min":94.400000000000006,"5min":94.400000000000006,"15min":230.69399999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":8.6210000000000004,"5min":4.5999999999999996,"15min":5.4000000000000004},"min":{"1min":0.52600000000000002,"5min":0.44500000000000001,"15min":0.32600000000000001},"max":{"1min":94.195999999999998,"5min":94.195999999999998,"15min":230.67500000000001},"last":1.9159999999999999}]}]},{"osd":4,"up_from":322,"seq":1382979469475,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2397532,"kb_used_data":2075156,"kb_used_omap":35,"kb_used_meta":322332,"kb_avail":91347620,"statfs":{"total":95995035648,"available":93539962880,"internally_reserved":0,"allocated":2124959744,"data_stored":2123393231,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36552,"internal_metadata":330068280},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,4,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:14:32 2023","interfaces":[{"interface":"back","average":{"1min":4.5990000000000002,"5min":1.972,"15min":1.649},"min":{"1min":0.254,"5min":0.216,"15min":0.216},"max":{"1min":23.206,"5min":23.206,"15min":23.206},"last":0.36299999999999999},{"interface":"front","average":{"1min":3.323,"5min":1.7609999999999999,"15min":1.651},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":22.902999999999999,"5min":22.902999999999999,"15min":22.902999999999999},"last":2.3540000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":1.77,"5min":2.0489999999999999,"15min":2.2770000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.20999999999999999},"max":{"1min":4.8360000000000003,"5min":18.884,"15min":78.891000000000005},"last":0.24099999999999999},{"interface":"front","average":{"1min":1.905,"5min":1.802,"15min":2.2559999999999998},"min":{"1min":0.32200000000000001,"5min":0.25600000000000001,"15min":0.22800000000000001},"max":{"1min":10.039,"5min":18.617999999999999,"15min":78.921000000000006},"last":0.39200000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":2.9929999999999999,"5min":4.5960000000000001,"15min":2.6869999999999998},"min":{"1min":0.39200000000000002,"5min":0.29199999999999998,"15min":0.27200000000000002},"max":{"1min":10.084,"5min":245.166,"15min":245.166},"last":1.8440000000000001},{"interface":"front","average":{"1min":1.845,"5min":4.3230000000000004,"15min":2.5329999999999999},"min":{"1min":0.28699999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.2080000000000002,"5min":245.119,"15min":245.119},"last":1.415}]},{"osd":3,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":3.601,"5min":6.8630000000000004,"15min":3.9740000000000002},"min":{"1min":0.34599999999999997,"5min":0.29399999999999998,"15min":0.26800000000000002},"max":{"1min":20.100999999999999,"5min":246.39400000000001,"15min":246.39400000000001},"last":2.3769999999999998},{"interface":"front","average":{"1min":3.7240000000000002,"5min":6.3700000000000001,"15min":3.5750000000000002},"min":{"1min":0.40899999999999997,"5min":0.314,"15min":0.314},"max":{"1min":19.984000000000002,"5min":246.99000000000001,"15min":246.99000000000001},"last":1.7290000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":6.9269999999999996,"5min":2.0880000000000001,"15min":1.3899999999999999},"min":{"1min":0.28899999999999998,"5min":0.23499999999999999,"15min":0.216},"max":{"1min":99.405000000000001,"5min":99.405000000000001,"15min":99.405000000000001},"last":1.339},{"interface":"front","average":{"1min":7.2169999999999996,"5min":2.274,"15min":1.546},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.245},"max":{"1min":99.206000000000003,"5min":99.206000000000003,"15min":99.206000000000003},"last":0.26700000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.0569999999999999,"15min":2.0859999999999999},"min":{"1min":0.32200000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.9900000000000002,"5min":7.641,"15min":95.492999999999995},"last":1.375},{"interface":"front","average":{"1min":1.7070000000000001,"5min":1.3300000000000001,"15min":2.25},"min":{"1min":0.33200000000000002,"5min":0.32100000000000001,"15min":0.27900000000000003},"max":{"1min":9.2110000000000003,"5min":9.3239999999999998,"15min":95.664000000000001},"last":0.33200000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":6.9320000000000004,"5min":2.5390000000000001,"15min":2.7200000000000002},"min":{"1min":0.38800000000000001,"5min":0.29699999999999999,"15min":0.29499999999999998},"max":{"1min":95.167000000000002,"5min":95.167000000000002,"15min":95.167000000000002},"last":1.286},{"interface":"front","average":{"1min":6.8949999999999996,"5min":2.2599999999999998,"15min":2.403},"min":{"1min":0.34499999999999997,"5min":0.27800000000000002,"15min":0.251},"max":{"1min":94.811999999999998,"5min":94.811999999999998,"15min":94.811999999999998},"last":0.51300000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757696,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2002004,"kb_used_data":1099404,"kb_used_omap":0,"kb_used_meta":902592,"kb_avail":91743148,"statfs":{"total":95995035648,"available":93944983552,"internally_reserved":0,"allocated":1125789696,"data_stored":1124235802,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924254208},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:14:34 2023","interfaces":[{"interface":"back","average":{"1min":2.6160000000000001,"5min":3.5510000000000002,"15min":2.7850000000000001},"min":{"1min":0.26700000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":18.347999999999999,"5min":115.92400000000001,"15min":115.92400000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":3.3650000000000002,"5min":3.8839999999999999,"15min":3.444},"min":{"1min":0.314,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":18.364999999999998,"5min":115.904,"15min":115.904},"last":0.52500000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:14:03 2023","interfaces":[{"interface":"back","average":{"1min":1.3560000000000001,"5min":4.6040000000000001,"15min":4.4950000000000001},"min":{"1min":0.26500000000000001,"5min":0.248,"15min":0.22600000000000001},"max":{"1min":5.4880000000000004,"5min":151.78200000000001,"15min":151.78200000000001},"last":0.498},{"interface":"front","average":{"1min":1.9990000000000001,"5min":4.6900000000000004,"15min":4.6059999999999999},"min":{"1min":0.32300000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":9.7750000000000004,"5min":152.251,"15min":152.251},"last":0.48099999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:14:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":2.9390000000000001,"15min":2.9529999999999998},"min":{"1min":0.33600000000000002,"5min":0.26600000000000001,"15min":0.254},"max":{"1min":2.5009999999999999,"5min":117.97199999999999,"15min":129.21199999999999},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.96799999999999997,"5min":3.2330000000000001,"15min":3.0129999999999999},"min":{"1min":0.35599999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.6160000000000001,"5min":118.52,"15min":129.042},"last":0.54300000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:14:03 2023","interfaces":[{"interface":"back","average":{"1min":9.4779999999999998,"5min":6.5380000000000003,"15min":6.9169999999999998},"min":{"1min":0.316,"5min":0.30299999999999999,"15min":0.29499999999999998},"max":{"1min":93.338999999999999,"5min":143.31200000000001,"15min":143.31200000000001},"last":19.231000000000002},{"interface":"front","average":{"1min":9.6940000000000008,"5min":7.1029999999999998,"15min":6.9989999999999997},"min":{"1min":0.40300000000000002,"5min":0.34499999999999997,"15min":0.29399999999999998},"max":{"1min":93.435000000000002,"5min":140.00899999999999,"15min":140.00899999999999},"last":20.035}]},{"osd":4,"last update":"Mon Oct 9 06:14:52 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":1.6279999999999999,"15min":1.278},"min":{"1min":0.28000000000000003,"5min":0.246,"15min":0.245},"max":{"1min":2.097,"5min":68.001999999999995,"15min":68.001999999999995},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.88300000000000001,"5min":1.5529999999999999,"15min":1.212},"min":{"1min":0.33900000000000002,"5min":0.26100000000000001,"15min":0.26000000000000001},"max":{"1min":5.7729999999999997,"5min":61.743000000000002,"15min":61.743000000000002},"last":0.436}]},{"osd":6,"last update":"Mon Oct 9 06:14:03 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":1.0149999999999999,"15min":1.48},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.251},"max":{"1min":2.3439999999999999,"5min":7.4980000000000002,"15min":126.89400000000001},"last":0.378},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.099,"15min":1.5620000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.26700000000000002},"max":{"1min":2.4340000000000002,"5min":7.7140000000000004,"15min":126.616},"last":0.40300000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:14:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3480000000000001,"5min":1.8380000000000001,"15min":2.456},"min":{"1min":0.36299999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.8289999999999997,"5min":64.766000000000005,"15min":157.471},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.85699999999999998,"5min":1.698,"15min":2.3410000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.3380000000000001,"5min":67.956999999999994,"15min":157.51599999999999},"last":0.41899999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4889,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1824133120,"data_stored":1824129048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":485883904,"data_stored":485883904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1114767360,"data_stored":1114767360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1411846144,"data_stored":1411842072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2085294080,"data_stored":2085290008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1111752704,"data_stored":1111752704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1029115904,"data_stored":1029111832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":619839488,"data_stored":619839488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:14:59.285 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:14:59.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:59.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:14:59.394 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:14:59.450 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:59.495 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:14:59.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:59.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:14:59.602 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:14:59.656 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:14:59.753 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:14:59.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:14:59.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:14:59.864 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:14:59.954 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:00.113 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:15:00.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:00.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:15:00.224 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:15:00.287 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:00.332 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:00.407 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:15:00.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:00.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:15:00.519 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:15:00.550 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:15:00.550 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:15:00.560 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:00.647 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:15:00.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:00.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:15:00.756 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:15:00.812 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:00.845 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:00 smithi064 ceph-mon[212750]: pgmap v1404: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 1468/110382 objects misplaced (1.330%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T06:15:00.845 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:00 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T06:15:00.845 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:00 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T06:15:00.847 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:00 smithi064 ceph-mon[220044]: pgmap v1404: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 1468/110382 objects misplaced (1.330%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T06:15:00.848 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:00 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T06:15:00.848 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:00 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T06:15:00.883 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:15:00.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:00.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:15:00.997 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:15:01.055 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:01.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:00 smithi156 ceph-mon[195101]: pgmap v1404: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 1468/110382 objects misplaced (1.330%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T06:15:01.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:00 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T06:15:01.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:00 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T06:15:01.142 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:15:01.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:01.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:15:01.250 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:15:01.310 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:01.461 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:15:01.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:01.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:15:01.565 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:15:01.618 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:01.710 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:15:01.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:01.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:15:01.823 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:15:01.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:01 smithi156 ceph-mon[195101]: failed to encode map e552 with expected crc 2023-10-09T06:15:01.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:01 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-10-09T06:15:01.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:01 smithi156 ceph-mon[195101]: osdmap e552: 8 total, 8 up, 8 in 2023-10-09T06:15:01.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:01 smithi156 ceph-mon[195101]: failed to encode map e552 with expected crc 2023-10-09T06:15:01.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:01 smithi156 ceph-mon[195101]: failed to encode map e552 with expected crc 2023-10-09T06:15:01.879 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:01.970 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:15:01.976 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:15:01.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:01.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:15:02.084 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:15:02.123 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:02.166 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:15:02.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:02.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:15:02.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:01 smithi064 ceph-mon[212750]: failed to encode map e552 with expected crc 2023-10-09T06:15:02.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:01 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-10-09T06:15:02.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:01 smithi064 ceph-mon[212750]: osdmap e552: 8 total, 8 up, 8 in 2023-10-09T06:15:02.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:01 smithi064 ceph-mon[212750]: failed to encode map e552 with expected crc 2023-10-09T06:15:02.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:01 smithi064 ceph-mon[212750]: failed to encode map e552 with expected crc 2023-10-09T06:15:02.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:01 smithi064 ceph-mon[220044]: failed to encode map e552 with expected crc 2023-10-09T06:15:02.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:01 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-10-09T06:15:02.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:01 smithi064 ceph-mon[220044]: osdmap e552: 8 total, 8 up, 8 in 2023-10-09T06:15:02.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:01 smithi064 ceph-mon[220044]: failed to encode map e552 with expected crc 2023-10-09T06:15:02.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:01 smithi064 ceph-mon[220044]: failed to encode map e552 with expected crc 2023-10-09T06:15:02.274 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:15:02.279 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:15:02.323 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:02.365 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:15:02.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:02.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:15:02.474 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:15:02.565 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:15:02.604 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:02.863 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:15:02.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:02.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:15:02.972 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:15:03.048 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:03.078 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:03.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:02 smithi156 ceph-mon[195101]: pgmap v1406: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 388/103398 objects misplaced (0.375%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T06:15:03.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:02 smithi156 ceph-mon[195101]: osdmap e553: 8 total, 8 up, 8 in 2023-10-09T06:15:03.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:02 smithi156 ceph-mon[195101]: failed to encode map e553 with expected crc 2023-10-09T06:15:03.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:02 smithi156 ceph-mon[195101]: failed to encode map e553 with expected crc 2023-10-09T06:15:03.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:02 smithi156 ceph-mon[195101]: failed to encode map e553 with expected crc 2023-10-09T06:15:03.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:02 smithi156 ceph-mon[195101]: failed to encode map e553 with expected crc 2023-10-09T06:15:03.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:02 smithi156 ceph-mon[195101]: failed to encode map e553 with expected crc 2023-10-09T06:15:03.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:02 smithi156 ceph-mon[195101]: failed to encode map e553 with expected crc 2023-10-09T06:15:03.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:02 smithi156 ceph-mon[195101]: failed to encode map e553 with expected crc 2023-10-09T06:15:03.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:02 smithi156 ceph-mon[195101]: osdmap e554: 8 total, 8 up, 8 in 2023-10-09T06:15:03.131 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:03.208 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:15:03.214 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:02 smithi064 ceph-mon[212750]: pgmap v1406: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 388/103398 objects misplaced (0.375%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T06:15:03.214 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:02 smithi064 ceph-mon[212750]: osdmap e553: 8 total, 8 up, 8 in 2023-10-09T06:15:03.214 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:02 smithi064 ceph-mon[212750]: failed to encode map e553 with expected crc 2023-10-09T06:15:03.214 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:02 smithi064 ceph-mon[212750]: failed to encode map e553 with expected crc 2023-10-09T06:15:03.214 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:02 smithi064 ceph-mon[212750]: failed to encode map e553 with expected crc 2023-10-09T06:15:03.214 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:02 smithi064 ceph-mon[212750]: failed to encode map e553 with expected crc 2023-10-09T06:15:03.214 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:02 smithi064 ceph-mon[212750]: failed to encode map e553 with expected crc 2023-10-09T06:15:03.215 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:02 smithi064 ceph-mon[212750]: failed to encode map e553 with expected crc 2023-10-09T06:15:03.215 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:02 smithi064 ceph-mon[212750]: failed to encode map e553 with expected crc 2023-10-09T06:15:03.215 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:02 smithi064 ceph-mon[212750]: osdmap e554: 8 total, 8 up, 8 in 2023-10-09T06:15:03.215 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:02 smithi064 ceph-mon[220044]: pgmap v1406: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 388/103398 objects misplaced (0.375%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T06:15:03.215 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:02 smithi064 ceph-mon[220044]: osdmap e553: 8 total, 8 up, 8 in 2023-10-09T06:15:03.215 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:02 smithi064 ceph-mon[220044]: failed to encode map e553 with expected crc 2023-10-09T06:15:03.215 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:02 smithi064 ceph-mon[220044]: failed to encode map e553 with expected crc 2023-10-09T06:15:03.215 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:02 smithi064 ceph-mon[220044]: failed to encode map e553 with expected crc 2023-10-09T06:15:03.215 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:02 smithi064 ceph-mon[220044]: failed to encode map e553 with expected crc 2023-10-09T06:15:03.216 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:02 smithi064 ceph-mon[220044]: failed to encode map e553 with expected crc 2023-10-09T06:15:03.216 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:02 smithi064 ceph-mon[220044]: failed to encode map e553 with expected crc 2023-10-09T06:15:03.216 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:02 smithi064 ceph-mon[220044]: failed to encode map e553 with expected crc 2023-10-09T06:15:03.216 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:02 smithi064 ceph-mon[220044]: osdmap e554: 8 total, 8 up, 8 in 2023-10-09T06:15:03.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:03.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:15:03.317 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:15:03.351 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:03.564 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:15:03.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:03.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:15:03.668 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:15:03.696 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:03.740 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:15:03.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:03.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:15:03.781 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:03.848 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:15:03.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:03 smithi064 ceph-mon[212750]: failed to encode map e554 with expected crc 2023-10-09T06:15:03.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:03 smithi064 ceph-mon[212750]: failed to encode map e554 with expected crc 2023-10-09T06:15:03.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:03 smithi064 ceph-mon[212750]: failed to encode map e554 with expected crc 2023-10-09T06:15:03.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:03 smithi064 ceph-mon[212750]: pgmap v1409: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 388/98916 objects misplaced (0.392%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T06:15:03.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:03 smithi064 ceph-mon[212750]: osdmap e555: 8 total, 8 up, 8 in 2023-10-09T06:15:03.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:03 smithi064 ceph-mon[212750]: failed to encode map e555 with expected crc 2023-10-09T06:15:03.934 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:04.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:03 smithi156 ceph-mon[195101]: failed to encode map e554 with expected crc 2023-10-09T06:15:04.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:03 smithi156 ceph-mon[195101]: failed to encode map e554 with expected crc 2023-10-09T06:15:04.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:03 smithi156 ceph-mon[195101]: failed to encode map e554 with expected crc 2023-10-09T06:15:04.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:03 smithi156 ceph-mon[195101]: pgmap v1409: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 388/98916 objects misplaced (0.392%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T06:15:04.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:03 smithi156 ceph-mon[195101]: osdmap e555: 8 total, 8 up, 8 in 2023-10-09T06:15:04.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:03 smithi156 ceph-mon[195101]: failed to encode map e555 with expected crc 2023-10-09T06:15:04.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:03 smithi064 ceph-mon[220044]: failed to encode map e554 with expected crc 2023-10-09T06:15:04.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:03 smithi064 ceph-mon[220044]: failed to encode map e554 with expected crc 2023-10-09T06:15:04.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:03 smithi064 ceph-mon[220044]: failed to encode map e554 with expected crc 2023-10-09T06:15:04.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:03 smithi064 ceph-mon[220044]: pgmap v1409: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 388/98916 objects misplaced (0.392%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T06:15:04.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:03 smithi064 ceph-mon[220044]: osdmap e555: 8 total, 8 up, 8 in 2023-10-09T06:15:04.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:03 smithi064 ceph-mon[220044]: failed to encode map e555 with expected crc 2023-10-09T06:15:04.894 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:15:05.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:04 smithi064 ceph-mon[220044]: failed to encode map e555 with expected crc 2023-10-09T06:15:05.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:04 smithi064 ceph-mon[220044]: failed to encode map e555 with expected crc 2023-10-09T06:15:05.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:04 smithi064 ceph-mon[220044]: failed to encode map e555 with expected crc 2023-10-09T06:15:05.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:04 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/433580146' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:15:05.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:04 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:15:05.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:04 smithi064 ceph-mon[212750]: failed to encode map e555 with expected crc 2023-10-09T06:15:05.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:04 smithi064 ceph-mon[212750]: failed to encode map e555 with expected crc 2023-10-09T06:15:05.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:04 smithi064 ceph-mon[212750]: failed to encode map e555 with expected crc 2023-10-09T06:15:05.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:04 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/433580146' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:15:05.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:04 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:15:05.260 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:15:05.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:05.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:15:05.371 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:15:05.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:04 smithi156 ceph-mon[195101]: failed to encode map e555 with expected crc 2023-10-09T06:15:05.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:04 smithi156 ceph-mon[195101]: failed to encode map e555 with expected crc 2023-10-09T06:15:05.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:04 smithi156 ceph-mon[195101]: failed to encode map e555 with expected crc 2023-10-09T06:15:05.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:04 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/433580146' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:15:05.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:04 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:15:05.441 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:05.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:15:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:15:05] "GET /metrics HTTP/1.1" 200 36471 "" "Prometheus/2.43.0" 2023-10-09T06:15:06.058 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:15:06.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:06.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:15:06.126 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:15:06.126 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:15:06.126 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[212750]: osdmap e556: 8 total, 8 up, 8 in 2023-10-09T06:15:06.126 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[212750]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.126 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[212750]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.127 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[212750]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.127 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[212750]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.127 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[212750]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.127 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[212750]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.127 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[212750]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.127 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[212750]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.127 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[212750]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.127 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[212750]: pgmap v1412: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 388/97056 objects misplaced (0.400%) 2023-10-09T06:15:06.127 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:15:06.127 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:15:06.128 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[220044]: osdmap e556: 8 total, 8 up, 8 in 2023-10-09T06:15:06.128 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[220044]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.128 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[220044]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.128 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[220044]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.128 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[220044]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.128 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[220044]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.128 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[220044]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.128 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[220044]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.128 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[220044]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.128 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[220044]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.128 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:05 smithi064 ceph-mon[220044]: pgmap v1412: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 388/97056 objects misplaced (0.400%) 2023-10-09T06:15:06.173 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:15:06.262 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:06.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:05 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:15:06.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:05 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:15:06.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:05 smithi156 ceph-mon[195101]: osdmap e556: 8 total, 8 up, 8 in 2023-10-09T06:15:06.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:05 smithi156 ceph-mon[195101]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:05 smithi156 ceph-mon[195101]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:05 smithi156 ceph-mon[195101]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:05 smithi156 ceph-mon[195101]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:05 smithi156 ceph-mon[195101]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:05 smithi156 ceph-mon[195101]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:05 smithi156 ceph-mon[195101]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:05 smithi156 ceph-mon[195101]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:05 smithi156 ceph-mon[195101]: failed to encode map e556 with expected crc 2023-10-09T06:15:06.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:05 smithi156 ceph-mon[195101]: pgmap v1412: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 388/97056 objects misplaced (0.400%) 2023-10-09T06:15:06.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:06 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3179960867' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:15:07.074 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:15:07.075 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:15:07.075 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:15:07.075 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:15:07.075 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:15:07.110 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:15:07.143 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:15:07.176 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:15:07.187 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:15:07.187 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:15:07.187 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:15:07.187 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:15:07.188 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:15:07.188 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:15:07.188 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:15:07.188 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:15:07.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:06 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3179960867' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:15:07.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:06 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3179960867' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:15:07.784 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:15:07.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:07.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:15:07.887 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:15:07.890 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:15:07.919 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T06:15:07.937 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:07.999 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:15:08.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:08.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:15:08.106 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:15:08.139 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:15:08.152 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:08.239 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:15:08.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:08.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:15:08.360 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:15:08.460 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:08.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:08 smithi064 ceph-mon[220044]: pgmap v1413: 119 pgs: 119 active+clean; 1.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 2.2 MiB/s, 35 objects/s recovering 2023-10-09T06:15:08.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:08 smithi064 ceph-mon[212750]: pgmap v1413: 119 pgs: 119 active+clean; 1.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 2.2 MiB/s, 35 objects/s recovering 2023-10-09T06:15:09.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:08 smithi156 ceph-mon[195101]: pgmap v1413: 119 pgs: 119 active+clean; 1.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 2.2 MiB/s, 35 objects/s recovering 2023-10-09T06:15:09.177 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:15:09.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:09.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:15:09.263 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1413,"stamp":"2023-10-09T06:15:07.212437+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1931286456,"num_objects":29485,"num_object_clones":0,"num_object_copies":88455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29485,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":163026,"num_write_kb":6157598,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42856,"num_bytes_recovered":2805937210,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":103058,"ondisk_log_size":103058,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":367,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13226708,"kb_used_data":7576084,"kb_used_omap":165,"kb_used_meta":5650202,"kb_avail":736734508,"statfs":{"total":767960285184,"available":754416136192,"internally_reserved":0,"allocated":7757910016,"data_stored":7739575550,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":169911,"internal_metadata":5785806921},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-228524032,"num_objects":-3487,"num_object_clones":0,"num_object_copies":-10461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-388,"num_objects_unfound":0,"num_objects_dirty":-3487,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3487,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":191,"num_bytes_recovered":12517376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.353174"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":794,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.739037+0000","last_change":"2023-10-09T06:13:33.699246+0000","last_active":"2023-10-09T06:15:02.739037+0000","last_peered":"2023-10-09T06:15:02.739037+0000","last_clean":"2023-10-09T06:15:02.739037+0000","last_became_active":"2023-10-09T06:13:33.698733+0000","last_became_peered":"2023-10-09T06:13:33.698733+0000","last_unstale":"2023-10-09T06:15:02.739037+0000","last_undegraded":"2023-10-09T06:15:02.739037+0000","last_fullsized":"2023-10-09T06:15:02.739037+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:05.412941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:05.412941+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:57.044673+0000","scrub_duration":0.0061014329999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":624,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.776427+0000","last_change":"2023-10-09T06:07:43.572773+0000","last_active":"2023-10-09T06:15:03.776427+0000","last_peered":"2023-10-09T06:15:03.776427+0000","last_clean":"2023-10-09T06:15:03.776427+0000","last_became_active":"2023-10-09T06:07:43.572299+0000","last_became_peered":"2023-10-09T06:07:43.572299+0000","last_unstale":"2023-10-09T06:15:03.776427+0000","last_undegraded":"2023-10-09T06:15:03.776427+0000","last_fullsized":"2023-10-09T06:15:03.776427+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:00:37.672435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":367,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.728824+0000","last_change":"2023-10-09T06:05:11.451972+0000","last_active":"2023-10-09T06:15:03.728824+0000","last_peered":"2023-10-09T06:15:03.728824+0000","last_clean":"2023-10-09T06:15:03.728824+0000","last_became_active":"2023-10-09T06:05:11.451448+0000","last_became_peered":"2023-10-09T06:05:11.451448+0000","last_unstale":"2023-10-09T06:15:03.728824+0000","last_undegraded":"2023-10-09T06:15:03.728824+0000","last_fullsized":"2023-10-09T06:15:03.728824+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:39.183621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":855,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.776268+0000","last_change":"2023-10-09T06:09:27.554162+0000","last_active":"2023-10-09T06:15:03.776268+0000","last_peered":"2023-10-09T06:15:03.776268+0000","last_clean":"2023-10-09T06:15:03.776268+0000","last_became_active":"2023-10-09T06:09:27.553659+0000","last_became_peered":"2023-10-09T06:09:27.553659+0000","last_unstale":"2023-10-09T06:15:03.776268+0000","last_undegraded":"2023-10-09T06:15:03.776268+0000","last_fullsized":"2023-10-09T06:15:03.776268+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:01.468346+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:01.468346+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:03.443511+0000","scrub_duration":0.0016258220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":238,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.731166+0000","last_change":"2023-10-09T06:01:54.762060+0000","last_active":"2023-10-09T06:15:03.731166+0000","last_peered":"2023-10-09T06:15:03.731166+0000","last_clean":"2023-10-09T06:15:03.731166+0000","last_became_active":"2023-10-09T06:01:54.761757+0000","last_became_peered":"2023-10-09T06:01:54.761757+0000","last_unstale":"2023-10-09T06:15:03.731166+0000","last_undegraded":"2023-10-09T06:15:03.731166+0000","last_fullsized":"2023-10-09T06:15:03.731166+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:40.294232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":38,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.741902+0000","last_change":"2023-10-09T06:13:30.789834+0000","last_active":"2023-10-09T06:15:02.741902+0000","last_peered":"2023-10-09T06:15:02.741902+0000","last_clean":"2023-10-09T06:15:02.741902+0000","last_became_active":"2023-10-09T06:13:30.789517+0000","last_became_peered":"2023-10-09T06:13:30.789517+0000","last_unstale":"2023-10-09T06:15:02.741902+0000","last_undegraded":"2023-10-09T06:15:02.741902+0000","last_fullsized":"2023-10-09T06:15:02.741902+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:45.467922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":840,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.775555+0000","last_change":"2023-10-09T06:12:07.739121+0000","last_active":"2023-10-09T06:15:03.775555+0000","last_peered":"2023-10-09T06:15:03.775555+0000","last_clean":"2023-10-09T06:15:03.775555+0000","last_became_active":"2023-10-09T06:12:07.738739+0000","last_became_peered":"2023-10-09T06:12:07.738739+0000","last_unstale":"2023-10-09T06:15:03.775555+0000","last_undegraded":"2023-10-09T06:15:03.775555+0000","last_fullsized":"2023-10-09T06:15:03.775555+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.293426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.293426+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:19.815422+0000","scrub_duration":0.001587946,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":349,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.726889+0000","last_change":"2023-10-09T06:09:27.564377+0000","last_active":"2023-10-09T06:15:03.726889+0000","last_peered":"2023-10-09T06:15:03.726889+0000","last_clean":"2023-10-09T06:15:03.726889+0000","last_became_active":"2023-10-09T06:09:27.563710+0000","last_became_peered":"2023-10-09T06:09:27.563710+0000","last_unstale":"2023-10-09T06:15:03.726889+0000","last_undegraded":"2023-10-09T06:15:03.726889+0000","last_fullsized":"2023-10-09T06:15:03.726889+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:48.168447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":444,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.738982+0000","last_change":"2023-10-09T06:15:01.885861+0000","last_active":"2023-10-09T06:15:02.738982+0000","last_peered":"2023-10-09T06:15:02.738982+0000","last_clean":"2023-10-09T06:15:02.738982+0000","last_became_active":"2023-10-09T06:15:01.885560+0000","last_became_peered":"2023-10-09T06:15:01.885560+0000","last_unstale":"2023-10-09T06:15:02.738982+0000","last_undegraded":"2023-10-09T06:15:02.738982+0000","last_fullsized":"2023-10-09T06:15:02.738982+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:44.842197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":773,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.741222+0000","last_change":"2023-10-09T06:13:33.701687+0000","last_active":"2023-10-09T06:15:02.741222+0000","last_peered":"2023-10-09T06:15:02.741222+0000","last_clean":"2023-10-09T06:15:02.741222+0000","last_became_active":"2023-10-09T06:13:33.701233+0000","last_became_peered":"2023-10-09T06:13:33.701233+0000","last_unstale":"2023-10-09T06:15:02.741222+0000","last_undegraded":"2023-10-09T06:15:02.741222+0000","last_fullsized":"2023-10-09T06:15:02.741222+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.384407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.384407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:04.887826+0000","scrub_duration":0.0040651960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":454,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.731235+0000","last_change":"2023-10-09T06:09:27.559756+0000","last_active":"2023-10-09T06:15:03.731235+0000","last_peered":"2023-10-09T06:15:03.731235+0000","last_clean":"2023-10-09T06:15:03.731235+0000","last_became_active":"2023-10-09T06:09:27.559443+0000","last_became_peered":"2023-10-09T06:09:27.559443+0000","last_unstale":"2023-10-09T06:15:03.731235+0000","last_undegraded":"2023-10-09T06:15:03.731235+0000","last_fullsized":"2023-10-09T06:15:03.731235+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:14.519398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":385,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.726942+0000","last_change":"2023-10-09T06:13:21.737897+0000","last_active":"2023-10-09T06:15:03.726942+0000","last_peered":"2023-10-09T06:15:03.726942+0000","last_clean":"2023-10-09T06:15:03.726942+0000","last_became_active":"2023-10-09T06:13:21.737564+0000","last_became_peered":"2023-10-09T06:13:21.737564+0000","last_unstale":"2023-10-09T06:15:03.726942+0000","last_undegraded":"2023-10-09T06:15:03.726942+0000","last_fullsized":"2023-10-09T06:15:03.726942+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:34.863320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":679,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.742243+0000","last_change":"2023-10-09T06:10:11.729656+0000","last_active":"2023-10-09T06:15:02.742243+0000","last_peered":"2023-10-09T06:15:02.742243+0000","last_clean":"2023-10-09T06:15:02.742243+0000","last_became_active":"2023-10-09T06:10:11.728960+0000","last_became_peered":"2023-10-09T06:10:11.728960+0000","last_unstale":"2023-10-09T06:15:02.742243+0000","last_undegraded":"2023-10-09T06:15:02.742243+0000","last_fullsized":"2023-10-09T06:15:02.742243+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:20.742561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:20.742561+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:45.543709+0000","scrub_duration":0.0027005039999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":38,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.741814+0000","last_change":"2023-10-09T06:13:30.802749+0000","last_active":"2023-10-09T06:15:02.741814+0000","last_peered":"2023-10-09T06:15:02.741814+0000","last_clean":"2023-10-09T06:15:02.741814+0000","last_became_active":"2023-10-09T06:13:30.802449+0000","last_became_peered":"2023-10-09T06:13:30.802449+0000","last_unstale":"2023-10-09T06:15:02.741814+0000","last_undegraded":"2023-10-09T06:15:02.741814+0000","last_fullsized":"2023-10-09T06:15:02.741814+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:29.017181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":543,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.727501+0000","last_change":"2023-10-09T06:13:30.792445+0000","last_active":"2023-10-09T06:15:03.727501+0000","last_peered":"2023-10-09T06:15:03.727501+0000","last_clean":"2023-10-09T06:15:03.727501+0000","last_became_active":"2023-10-09T06:13:30.792116+0000","last_became_peered":"2023-10-09T06:13:30.792116+0000","last_unstale":"2023-10-09T06:15:03.727501+0000","last_undegraded":"2023-10-09T06:15:03.727501+0000","last_fullsized":"2023-10-09T06:15:03.727501+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:20:53.506293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":766,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.720854+0000","last_change":"2023-10-09T06:13:24.760843+0000","last_active":"2023-10-09T06:15:02.720854+0000","last_peered":"2023-10-09T06:15:02.720854+0000","last_clean":"2023-10-09T06:15:02.720854+0000","last_became_active":"2023-10-09T06:13:24.760333+0000","last_became_peered":"2023-10-09T06:13:24.760333+0000","last_unstale":"2023-10-09T06:15:02.720854+0000","last_undegraded":"2023-10-09T06:15:02.720854+0000","last_fullsized":"2023-10-09T06:15:02.720854+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:38.588641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:38.588641+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:30.660400+0000","scrub_duration":0.010116385,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":44,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.741126+0000","last_change":"2023-10-09T06:13:30.826588+0000","last_active":"2023-10-09T06:15:02.741126+0000","last_peered":"2023-10-09T06:15:02.741126+0000","last_clean":"2023-10-09T06:15:02.741126+0000","last_became_active":"2023-10-09T06:13:30.799199+0000","last_became_peered":"2023-10-09T06:13:30.799199+0000","last_unstale":"2023-10-09T06:15:02.741126+0000","last_undegraded":"2023-10-09T06:15:02.741126+0000","last_fullsized":"2023-10-09T06:15:02.741126+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:38.703864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":336,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.720062+0000","last_change":"2023-10-09T06:13:21.739214+0000","last_active":"2023-10-09T06:15:02.720062+0000","last_peered":"2023-10-09T06:15:02.720062+0000","last_clean":"2023-10-09T06:15:02.720062+0000","last_became_active":"2023-10-09T06:13:21.738951+0000","last_became_peered":"2023-10-09T06:13:21.738951+0000","last_unstale":"2023-10-09T06:15:02.720062+0000","last_undegraded":"2023-10-09T06:15:02.720062+0000","last_fullsized":"2023-10-09T06:15:02.720062+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:17.477241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":728,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.732587+0000","last_change":"2023-10-09T06:09:27.574336+0000","last_active":"2023-10-09T06:15:03.732587+0000","last_peered":"2023-10-09T06:15:03.732587+0000","last_clean":"2023-10-09T06:15:03.732587+0000","last_became_active":"2023-10-09T06:09:27.573974+0000","last_became_peered":"2023-10-09T06:09:27.573974+0000","last_unstale":"2023-10-09T06:15:03.732587+0000","last_undegraded":"2023-10-09T06:15:03.732587+0000","last_fullsized":"2023-10-09T06:15:03.732587+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.339121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.339121+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:45.908579+0000","scrub_duration":0.010474344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":590,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.776018+0000","last_change":"2023-10-09T06:13:30.812598+0000","last_active":"2023-10-09T06:15:03.776018+0000","last_peered":"2023-10-09T06:15:03.776018+0000","last_clean":"2023-10-09T06:15:03.776018+0000","last_became_active":"2023-10-09T06:13:30.791460+0000","last_became_peered":"2023-10-09T06:13:30.791460+0000","last_unstale":"2023-10-09T06:15:03.776018+0000","last_undegraded":"2023-10-09T06:15:03.776018+0000","last_fullsized":"2023-10-09T06:15:03.776018+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:28.046730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":568,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.776581+0000","last_change":"2023-10-09T06:01:54.764916+0000","last_active":"2023-10-09T06:15:03.776581+0000","last_peered":"2023-10-09T06:15:03.776581+0000","last_clean":"2023-10-09T06:15:03.776581+0000","last_became_active":"2023-10-09T06:01:54.764705+0000","last_became_peered":"2023-10-09T06:01:54.764705+0000","last_unstale":"2023-10-09T06:15:03.776581+0000","last_undegraded":"2023-10-09T06:15:03.776581+0000","last_fullsized":"2023-10-09T06:15:03.776581+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:12.527719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":784,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.727218+0000","last_change":"2023-10-09T06:13:15.425525+0000","last_active":"2023-10-09T06:15:03.727218+0000","last_peered":"2023-10-09T06:15:03.727218+0000","last_clean":"2023-10-09T06:15:03.727218+0000","last_became_active":"2023-10-09T06:13:15.425086+0000","last_became_peered":"2023-10-09T06:13:15.425086+0000","last_unstale":"2023-10-09T06:15:03.727218+0000","last_undegraded":"2023-10-09T06:15:03.727218+0000","last_fullsized":"2023-10-09T06:15:03.727218+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:42.900876+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:42.900876+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:46.042715+0000","scrub_duration":0.0014928649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":44,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.740861+0000","last_change":"2023-10-09T06:13:30.811420+0000","last_active":"2023-10-09T06:15:02.740861+0000","last_peered":"2023-10-09T06:15:02.740861+0000","last_clean":"2023-10-09T06:15:02.740861+0000","last_became_active":"2023-10-09T06:13:30.792835+0000","last_became_peered":"2023-10-09T06:13:30.792835+0000","last_unstale":"2023-10-09T06:15:02.740861+0000","last_undegraded":"2023-10-09T06:15:02.740861+0000","last_fullsized":"2023-10-09T06:15:02.740861+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:38.962645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":602,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T06:15:04.877800+0000","last_change":"2023-10-09T06:01:54.761558+0000","last_active":"2023-10-09T06:15:04.877800+0000","last_peered":"2023-10-09T06:15:04.877800+0000","last_clean":"2023-10-09T06:15:04.877800+0000","last_became_active":"2023-10-09T06:01:54.761251+0000","last_became_peered":"2023-10-09T06:01:54.761251+0000","last_unstale":"2023-10-09T06:15:04.877800+0000","last_undegraded":"2023-10-09T06:15:04.877800+0000","last_fullsized":"2023-10-09T06:15:04.877800+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:53.245267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":1058,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.739281+0000","last_change":"2023-10-09T06:13:24.754018+0000","last_active":"2023-10-09T06:15:02.739281+0000","last_peered":"2023-10-09T06:15:02.739281+0000","last_clean":"2023-10-09T06:15:02.739281+0000","last_became_active":"2023-10-09T06:13:24.753712+0000","last_became_peered":"2023-10-09T06:13:24.753712+0000","last_unstale":"2023-10-09T06:15:02.739281+0000","last_undegraded":"2023-10-09T06:15:02.739281+0000","last_fullsized":"2023-10-09T06:15:02.739281+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:18.490833+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":38,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.740474+0000","last_change":"2023-10-09T06:13:30.788160+0000","last_active":"2023-10-09T06:15:02.740474+0000","last_peered":"2023-10-09T06:15:02.740474+0000","last_clean":"2023-10-09T06:15:02.740474+0000","last_became_active":"2023-10-09T06:13:30.787724+0000","last_became_peered":"2023-10-09T06:13:30.787724+0000","last_unstale":"2023-10-09T06:15:02.740474+0000","last_undegraded":"2023-10-09T06:15:02.740474+0000","last_fullsized":"2023-10-09T06:15:02.740474+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:38.376770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":38,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.741418+0000","last_change":"2023-10-09T06:13:30.789225+0000","last_active":"2023-10-09T06:15:02.741418+0000","last_peered":"2023-10-09T06:15:02.741418+0000","last_clean":"2023-10-09T06:15:02.741418+0000","last_became_active":"2023-10-09T06:13:30.788869+0000","last_became_peered":"2023-10-09T06:13:30.788869+0000","last_unstale":"2023-10-09T06:15:02.741418+0000","last_undegraded":"2023-10-09T06:15:02.741418+0000","last_fullsized":"2023-10-09T06:15:02.741418+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:19.519966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":796,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.731908+0000","last_change":"2023-10-09T06:12:55.750923+0000","last_active":"2023-10-09T06:15:03.731908+0000","last_peered":"2023-10-09T06:15:03.731908+0000","last_clean":"2023-10-09T06:15:03.731908+0000","last_became_active":"2023-10-09T06:12:55.750521+0000","last_became_peered":"2023-10-09T06:12:55.750521+0000","last_unstale":"2023-10-09T06:15:03.731908+0000","last_undegraded":"2023-10-09T06:15:03.731908+0000","last_fullsized":"2023-10-09T06:15:03.731908+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.316697+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.316697+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:23.591439+0000","scrub_duration":0.001666561,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":408,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.740513+0000","last_change":"2023-10-09T06:01:22.337171+0000","last_active":"2023-10-09T06:15:02.740513+0000","last_peered":"2023-10-09T06:15:02.740513+0000","last_clean":"2023-10-09T06:15:02.740513+0000","last_became_active":"2023-10-09T06:01:22.335438+0000","last_became_peered":"2023-10-09T06:01:22.335438+0000","last_unstale":"2023-10-09T06:15:02.740513+0000","last_undegraded":"2023-10-09T06:15:02.740513+0000","last_fullsized":"2023-10-09T06:15:02.740513+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:25.036448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":518,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.729961+0000","last_change":"2023-10-09T06:13:30.788284+0000","last_active":"2023-10-09T06:15:03.729961+0000","last_peered":"2023-10-09T06:15:03.729961+0000","last_clean":"2023-10-09T06:15:03.729961+0000","last_became_active":"2023-10-09T06:13:30.787933+0000","last_became_peered":"2023-10-09T06:13:30.787933+0000","last_unstale":"2023-10-09T06:15:03.729961+0000","last_undegraded":"2023-10-09T06:15:03.729961+0000","last_fullsized":"2023-10-09T06:15:03.729961+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:10.735438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":846,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.731496+0000","last_change":"2023-10-09T06:13:24.753908+0000","last_active":"2023-10-09T06:15:03.731496+0000","last_peered":"2023-10-09T06:15:03.731496+0000","last_clean":"2023-10-09T06:15:03.731496+0000","last_became_active":"2023-10-09T06:13:24.753566+0000","last_became_peered":"2023-10-09T06:13:24.753566+0000","last_unstale":"2023-10-09T06:15:03.731496+0000","last_undegraded":"2023-10-09T06:15:03.731496+0000","last_fullsized":"2023-10-09T06:15:03.731496+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:40.388125+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:40.388125+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:35.101978+0000","scrub_duration":0.0055576990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":128,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.767734+0000","last_change":"2023-10-09T06:13:12.243605+0000","last_active":"2023-10-09T06:15:03.767734+0000","last_peered":"2023-10-09T06:15:03.767734+0000","last_clean":"2023-10-09T06:15:03.767734+0000","last_became_active":"2023-10-09T06:13:12.237061+0000","last_became_peered":"2023-10-09T06:13:12.237061+0000","last_unstale":"2023-10-09T06:15:03.767734+0000","last_undegraded":"2023-10-09T06:15:03.767734+0000","last_fullsized":"2023-10-09T06:15:03.767734+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:29:03.666031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":66,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.730438+0000","last_change":"2023-10-09T06:13:30.788733+0000","last_active":"2023-10-09T06:15:03.730438+0000","last_peered":"2023-10-09T06:15:03.730438+0000","last_clean":"2023-10-09T06:15:03.730438+0000","last_became_active":"2023-10-09T06:13:30.788353+0000","last_became_peered":"2023-10-09T06:13:30.788353+0000","last_unstale":"2023-10-09T06:15:03.730438+0000","last_undegraded":"2023-10-09T06:15:03.730438+0000","last_fullsized":"2023-10-09T06:15:03.730438+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:06.308426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"556'10221","reported_seq":10297,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T06:15:06.345513+0000","last_change":"2023-10-09T06:11:46.032384+0000","last_active":"2023-10-09T06:15:06.345513+0000","last_peered":"2023-10-09T06:15:06.345513+0000","last_clean":"2023-10-09T06:15:06.345513+0000","last_became_active":"2023-10-09T06:11:46.032091+0000","last_became_peered":"2023-10-09T06:11:46.032091+0000","last_unstale":"2023-10-09T06:15:06.345513+0000","last_undegraded":"2023-10-09T06:15:06.345513+0000","last_fullsized":"2023-10-09T06:15:06.345513+0000","mapping_epoch":490,"log_start":"506'3800","ondisk_log_start":"506'3800","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6421,"log_dups_size":0,"ondisk_log_size":6421,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:31.117384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119996416,"num_objects":1831,"num_object_clones":0,"num_object_copies":5493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10221,"num_write_kb":385664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":736,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.741501+0000","last_change":"2023-10-09T06:13:33.701521+0000","last_active":"2023-10-09T06:15:02.741501+0000","last_peered":"2023-10-09T06:15:02.741501+0000","last_clean":"2023-10-09T06:15:02.741501+0000","last_became_active":"2023-10-09T06:13:33.700870+0000","last_became_peered":"2023-10-09T06:13:33.700870+0000","last_unstale":"2023-10-09T06:15:02.741501+0000","last_undegraded":"2023-10-09T06:15:02.741501+0000","last_fullsized":"2023-10-09T06:15:02.741501+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:10:59.422551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:10:59.422551+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:54.402494+0000","scrub_duration":0.0059716659999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":42,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.740895+0000","last_change":"2023-10-09T06:13:30.822903+0000","last_active":"2023-10-09T06:15:02.740895+0000","last_peered":"2023-10-09T06:15:02.740895+0000","last_clean":"2023-10-09T06:15:02.740895+0000","last_became_active":"2023-10-09T06:13:30.799013+0000","last_became_peered":"2023-10-09T06:13:30.799013+0000","last_unstale":"2023-10-09T06:15:02.740895+0000","last_undegraded":"2023-10-09T06:15:02.740895+0000","last_fullsized":"2023-10-09T06:15:02.740895+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:44.271821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":38,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.738961+0000","last_change":"2023-10-09T06:13:30.793531+0000","last_active":"2023-10-09T06:15:02.738961+0000","last_peered":"2023-10-09T06:15:02.738961+0000","last_clean":"2023-10-09T06:15:02.738961+0000","last_became_active":"2023-10-09T06:13:30.792689+0000","last_became_peered":"2023-10-09T06:13:30.792689+0000","last_unstale":"2023-10-09T06:15:02.738961+0000","last_undegraded":"2023-10-09T06:15:02.738961+0000","last_fullsized":"2023-10-09T06:15:02.738961+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:02.559186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"556'10212","reported_seq":19135,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T06:15:05.469831+0000","last_change":"2023-10-09T06:15:03.759950+0000","last_active":"2023-10-09T06:15:05.469831+0000","last_peered":"2023-10-09T06:15:05.469831+0000","last_clean":"2023-10-09T06:15:05.469831+0000","last_became_active":"2023-10-09T06:15:03.735000+0000","last_became_peered":"2023-10-09T06:15:03.735000+0000","last_unstale":"2023-10-09T06:15:05.469831+0000","last_undegraded":"2023-10-09T06:15:05.469831+0000","last_fullsized":"2023-10-09T06:15:05.469831+0000","mapping_epoch":554,"log_start":"507'4293","ondisk_log_start":"507'4293","created":490,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5919,"log_dups_size":0,"ondisk_log_size":5919,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:01.420748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122552320,"num_objects":1870,"num_object_clones":0,"num_object_copies":5610,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1870,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10212,"num_write_kb":386624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3329,"num_bytes_recovered":218169344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":925,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.727510+0000","last_change":"2023-10-09T06:13:24.753866+0000","last_active":"2023-10-09T06:15:03.727510+0000","last_peered":"2023-10-09T06:15:03.727510+0000","last_clean":"2023-10-09T06:15:03.727510+0000","last_became_active":"2023-10-09T06:13:24.753432+0000","last_became_peered":"2023-10-09T06:13:24.753432+0000","last_unstale":"2023-10-09T06:15:03.727510+0000","last_undegraded":"2023-10-09T06:15:03.727510+0000","last_fullsized":"2023-10-09T06:15:03.727510+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:03.241335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:03.241335+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:19.854449+0000","scrub_duration":0.017780547000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":42,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.741721+0000","last_change":"2023-10-09T06:13:30.829763+0000","last_active":"2023-10-09T06:15:02.741721+0000","last_peered":"2023-10-09T06:15:02.741721+0000","last_clean":"2023-10-09T06:15:02.741721+0000","last_became_active":"2023-10-09T06:13:30.802486+0000","last_became_peered":"2023-10-09T06:13:30.802486+0000","last_unstale":"2023-10-09T06:15:02.741721+0000","last_undegraded":"2023-10-09T06:15:02.741721+0000","last_fullsized":"2023-10-09T06:15:02.741721+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:05.468205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":594,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.775768+0000","last_change":"2023-10-09T06:01:54.760367+0000","last_active":"2023-10-09T06:15:03.775768+0000","last_peered":"2023-10-09T06:15:03.775768+0000","last_clean":"2023-10-09T06:15:03.775768+0000","last_became_active":"2023-10-09T06:01:54.760087+0000","last_became_peered":"2023-10-09T06:01:54.760087+0000","last_unstale":"2023-10-09T06:15:03.775768+0000","last_undegraded":"2023-10-09T06:15:03.775768+0000","last_fullsized":"2023-10-09T06:15:03.775768+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:26.000104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"556'10079","reported_seq":10153,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T06:15:05.880335+0000","last_change":"2023-10-09T06:11:45.696810+0000","last_active":"2023-10-09T06:15:05.880335+0000","last_peered":"2023-10-09T06:15:05.880335+0000","last_clean":"2023-10-09T06:15:05.880335+0000","last_became_active":"2023-10-09T06:11:45.696476+0000","last_became_peered":"2023-10-09T06:11:45.696476+0000","last_unstale":"2023-10-09T06:15:05.880335+0000","last_undegraded":"2023-10-09T06:15:05.880335+0000","last_fullsized":"2023-10-09T06:15:05.880335+0000","mapping_epoch":490,"log_start":"508'4700","ondisk_log_start":"508'4700","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5379,"log_dups_size":0,"ondisk_log_size":5379,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:41:33.351567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121438208,"num_objects":1853,"num_object_clones":0,"num_object_copies":5559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10079,"num_write_kb":381824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":819,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.777102+0000","last_change":"2023-10-09T06:13:24.768958+0000","last_active":"2023-10-09T06:15:03.777102+0000","last_peered":"2023-10-09T06:15:03.777102+0000","last_clean":"2023-10-09T06:15:03.777102+0000","last_became_active":"2023-10-09T06:13:24.768643+0000","last_became_peered":"2023-10-09T06:13:24.768643+0000","last_unstale":"2023-10-09T06:15:03.777102+0000","last_undegraded":"2023-10-09T06:15:03.777102+0000","last_fullsized":"2023-10-09T06:15:03.777102+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:48.495739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:48.495739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:18:50.654354+0000","scrub_duration":0.0016345509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":42,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.741343+0000","last_change":"2023-10-09T06:13:30.814434+0000","last_active":"2023-10-09T06:15:02.741343+0000","last_peered":"2023-10-09T06:15:02.741343+0000","last_clean":"2023-10-09T06:15:02.741343+0000","last_became_active":"2023-10-09T06:13:30.793119+0000","last_became_peered":"2023-10-09T06:13:30.793119+0000","last_unstale":"2023-10-09T06:15:02.741343+0000","last_undegraded":"2023-10-09T06:15:02.741343+0000","last_fullsized":"2023-10-09T06:15:02.741343+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:34:31.724006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":581,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.731350+0000","last_change":"2023-10-09T06:13:30.792490+0000","last_active":"2023-10-09T06:15:03.731350+0000","last_peered":"2023-10-09T06:15:03.731350+0000","last_clean":"2023-10-09T06:15:03.731350+0000","last_became_active":"2023-10-09T06:13:30.792209+0000","last_became_peered":"2023-10-09T06:13:30.792209+0000","last_unstale":"2023-10-09T06:15:03.731350+0000","last_undegraded":"2023-10-09T06:15:03.731350+0000","last_fullsized":"2023-10-09T06:15:03.731350+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:14.208042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"556'10146","reported_seq":10221,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T06:15:06.423986+0000","last_change":"2023-10-09T06:11:46.031195+0000","last_active":"2023-10-09T06:15:06.423986+0000","last_peered":"2023-10-09T06:15:06.423986+0000","last_clean":"2023-10-09T06:15:06.423986+0000","last_became_active":"2023-10-09T06:11:46.030891+0000","last_became_peered":"2023-10-09T06:11:46.030891+0000","last_unstale":"2023-10-09T06:15:06.423986+0000","last_undegraded":"2023-10-09T06:15:06.423986+0000","last_fullsized":"2023-10-09T06:15:06.423986+0000","mapping_epoch":490,"log_start":"506'3700","ondisk_log_start":"506'3700","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6446,"log_dups_size":0,"ondisk_log_size":6446,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:25.788661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115998720,"num_objects":1770,"num_object_clones":0,"num_object_copies":5310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10146,"num_write_kb":381312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":771,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.777415+0000","last_change":"2023-10-09T06:13:33.708291+0000","last_active":"2023-10-09T06:15:03.777415+0000","last_peered":"2023-10-09T06:15:03.777415+0000","last_clean":"2023-10-09T06:15:03.777415+0000","last_became_active":"2023-10-09T06:13:33.707930+0000","last_became_peered":"2023-10-09T06:13:33.707930+0000","last_unstale":"2023-10-09T06:15:03.777415+0000","last_undegraded":"2023-10-09T06:15:03.777415+0000","last_fullsized":"2023-10-09T06:15:03.777415+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:16.361097+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:16.361097+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:18.111404+0000","scrub_duration":0.028533749000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":325,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.727560+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T06:15:03.727560+0000","last_peered":"2023-10-09T06:15:03.727560+0000","last_clean":"2023-10-09T06:15:03.727560+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T06:15:03.727560+0000","last_undegraded":"2023-10-09T06:15:03.727560+0000","last_fullsized":"2023-10-09T06:15:03.727560+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":569,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.768176+0000","last_change":"2023-10-09T06:09:27.558353+0000","last_active":"2023-10-09T06:15:03.768176+0000","last_peered":"2023-10-09T06:15:03.768176+0000","last_clean":"2023-10-09T06:15:03.768176+0000","last_became_active":"2023-10-09T06:09:27.557233+0000","last_became_peered":"2023-10-09T06:09:27.557233+0000","last_unstale":"2023-10-09T06:15:03.768176+0000","last_undegraded":"2023-10-09T06:15:03.768176+0000","last_fullsized":"2023-10-09T06:15:03.768176+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:53.149350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"556'10241","reported_seq":10317,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T06:15:06.659189+0000","last_change":"2023-10-09T06:11:45.694648+0000","last_active":"2023-10-09T06:15:06.659189+0000","last_peered":"2023-10-09T06:15:06.659189+0000","last_clean":"2023-10-09T06:15:06.659189+0000","last_became_active":"2023-10-09T06:11:45.694321+0000","last_became_peered":"2023-10-09T06:11:45.694321+0000","last_unstale":"2023-10-09T06:15:06.659189+0000","last_undegraded":"2023-10-09T06:15:06.659189+0000","last_fullsized":"2023-10-09T06:15:06.659189+0000","mapping_epoch":490,"log_start":"505'3500","ondisk_log_start":"505'3500","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6741,"log_dups_size":0,"ondisk_log_size":6741,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:43.704525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110166016,"num_objects":1681,"num_object_clones":0,"num_object_copies":5043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10241,"num_write_kb":381504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":941,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.740332+0000","last_change":"2023-10-09T06:13:33.716191+0000","last_active":"2023-10-09T06:15:02.740332+0000","last_peered":"2023-10-09T06:15:02.740332+0000","last_clean":"2023-10-09T06:15:02.740332+0000","last_became_active":"2023-10-09T06:13:33.715831+0000","last_became_peered":"2023-10-09T06:13:33.715831+0000","last_unstale":"2023-10-09T06:15:02.740332+0000","last_undegraded":"2023-10-09T06:15:02.740332+0000","last_fullsized":"2023-10-09T06:15:02.740332+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:42.795781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:42.795781+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:45.712738+0000","scrub_duration":0.0091262979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":359,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.730364+0000","last_change":"2023-10-09T06:09:27.565666+0000","last_active":"2023-10-09T06:15:03.730364+0000","last_peered":"2023-10-09T06:15:03.730364+0000","last_clean":"2023-10-09T06:15:03.730364+0000","last_became_active":"2023-10-09T06:09:27.565324+0000","last_became_peered":"2023-10-09T06:09:27.565324+0000","last_unstale":"2023-10-09T06:15:03.730364+0000","last_undegraded":"2023-10-09T06:15:03.730364+0000","last_fullsized":"2023-10-09T06:15:03.730364+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:51.216947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":530,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.776101+0000","last_change":"2023-10-09T06:13:21.727745+0000","last_active":"2023-10-09T06:15:03.776101+0000","last_peered":"2023-10-09T06:15:03.776101+0000","last_clean":"2023-10-09T06:15:03.776101+0000","last_became_active":"2023-10-09T06:13:21.727404+0000","last_became_peered":"2023-10-09T06:13:21.727404+0000","last_unstale":"2023-10-09T06:15:03.776101+0000","last_undegraded":"2023-10-09T06:15:03.776101+0000","last_fullsized":"2023-10-09T06:15:03.776101+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:24.931626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"556'10382","reported_seq":21951,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T06:15:05.409755+0000","last_change":"2023-10-09T06:13:32.697002+0000","last_active":"2023-10-09T06:15:05.409755+0000","last_peered":"2023-10-09T06:15:05.409755+0000","last_clean":"2023-10-09T06:15:05.409755+0000","last_became_active":"2023-10-09T06:13:32.683104+0000","last_became_peered":"2023-10-09T06:13:32.683104+0000","last_unstale":"2023-10-09T06:15:05.409755+0000","last_undegraded":"2023-10-09T06:15:05.409755+0000","last_fullsized":"2023-10-09T06:15:05.409755+0000","mapping_epoch":528,"log_start":"507'4435","ondisk_log_start":"507'4435","created":490,"last_epoch_clean":529,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5947,"log_dups_size":0,"ondisk_log_size":5947,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:19.426571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123994112,"num_objects":1892,"num_object_clones":0,"num_object_copies":5676,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1892,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10382,"num_write_kb":392768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5732,"num_bytes_recovered":375652352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":819,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.726774+0000","last_change":"2023-10-09T06:13:15.442141+0000","last_active":"2023-10-09T06:15:03.726774+0000","last_peered":"2023-10-09T06:15:03.726774+0000","last_clean":"2023-10-09T06:15:03.726774+0000","last_became_active":"2023-10-09T06:13:15.441716+0000","last_became_peered":"2023-10-09T06:13:15.441716+0000","last_unstale":"2023-10-09T06:15:03.726774+0000","last_undegraded":"2023-10-09T06:15:03.726774+0000","last_fullsized":"2023-10-09T06:15:03.726774+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.282393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.282393+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:16.798440+0000","scrub_duration":0.01402403,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":475,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.775690+0000","last_change":"2023-10-09T06:13:21.738218+0000","last_active":"2023-10-09T06:15:03.775690+0000","last_peered":"2023-10-09T06:15:03.775690+0000","last_clean":"2023-10-09T06:15:03.775690+0000","last_became_active":"2023-10-09T06:13:21.737969+0000","last_became_peered":"2023-10-09T06:13:21.737969+0000","last_unstale":"2023-10-09T06:15:03.775690+0000","last_undegraded":"2023-10-09T06:15:03.775690+0000","last_fullsized":"2023-10-09T06:15:03.775690+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:38.620281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":604,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.776881+0000","last_change":"2023-10-09T06:07:43.572173+0000","last_active":"2023-10-09T06:15:03.776881+0000","last_peered":"2023-10-09T06:15:03.776881+0000","last_clean":"2023-10-09T06:15:03.776881+0000","last_became_active":"2023-10-09T06:07:43.571690+0000","last_became_peered":"2023-10-09T06:07:43.571690+0000","last_unstale":"2023-10-09T06:15:03.776881+0000","last_undegraded":"2023-10-09T06:15:03.776881+0000","last_fullsized":"2023-10-09T06:15:03.776881+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:10.234125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"556'10400","reported_seq":18780,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T06:15:06.693061+0000","last_change":"2023-10-09T06:14:28.746278+0000","last_active":"2023-10-09T06:15:06.693061+0000","last_peered":"2023-10-09T06:15:06.693061+0000","last_clean":"2023-10-09T06:15:06.693061+0000","last_became_active":"2023-10-09T06:14:28.728616+0000","last_became_peered":"2023-10-09T06:14:28.728616+0000","last_unstale":"2023-10-09T06:15:06.693061+0000","last_undegraded":"2023-10-09T06:15:06.693061+0000","last_fullsized":"2023-10-09T06:15:06.693061+0000","mapping_epoch":545,"log_start":"505'3700","ondisk_log_start":"505'3700","created":490,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6700,"log_dups_size":0,"ondisk_log_size":6700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:32.106540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119930880,"num_objects":1830,"num_object_clones":0,"num_object_copies":5490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10400,"num_write_kb":391360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4137,"num_bytes_recovered":271122432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":890,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.731664+0000","last_change":"2023-10-09T06:13:15.428701+0000","last_active":"2023-10-09T06:15:03.731664+0000","last_peered":"2023-10-09T06:15:03.731664+0000","last_clean":"2023-10-09T06:15:03.731664+0000","last_became_active":"2023-10-09T06:13:15.428343+0000","last_became_peered":"2023-10-09T06:13:15.428343+0000","last_unstale":"2023-10-09T06:15:03.731664+0000","last_undegraded":"2023-10-09T06:15:03.731664+0000","last_fullsized":"2023-10-09T06:15:03.731664+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.373748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.373748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:55:34.235074+0000","scrub_duration":0.0024576450000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":546,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.775839+0000","last_change":"2023-10-09T06:07:43.574863+0000","last_active":"2023-10-09T06:15:03.775839+0000","last_peered":"2023-10-09T06:15:03.775839+0000","last_clean":"2023-10-09T06:15:03.775839+0000","last_became_active":"2023-10-09T06:07:43.574576+0000","last_became_peered":"2023-10-09T06:07:43.574576+0000","last_unstale":"2023-10-09T06:15:03.775839+0000","last_undegraded":"2023-10-09T06:15:03.775839+0000","last_fullsized":"2023-10-09T06:15:03.775839+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:16.166004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":587,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.777481+0000","last_change":"2023-10-09T06:13:21.727814+0000","last_active":"2023-10-09T06:15:03.777481+0000","last_peered":"2023-10-09T06:15:03.777481+0000","last_clean":"2023-10-09T06:15:03.777481+0000","last_became_active":"2023-10-09T06:13:21.727401+0000","last_became_peered":"2023-10-09T06:13:21.727401+0000","last_unstale":"2023-10-09T06:15:03.777481+0000","last_undegraded":"2023-10-09T06:15:03.777481+0000","last_fullsized":"2023-10-09T06:15:03.777481+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:53:37.968028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"556'10272","reported_seq":10346,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T06:15:05.864306+0000","last_change":"2023-10-09T06:11:45.695709+0000","last_active":"2023-10-09T06:15:05.864306+0000","last_peered":"2023-10-09T06:15:05.864306+0000","last_clean":"2023-10-09T06:15:05.864306+0000","last_became_active":"2023-10-09T06:11:45.695219+0000","last_became_peered":"2023-10-09T06:11:45.695219+0000","last_unstale":"2023-10-09T06:15:05.864306+0000","last_undegraded":"2023-10-09T06:15:05.864306+0000","last_fullsized":"2023-10-09T06:15:05.864306+0000","mapping_epoch":490,"log_start":"513'4900","ondisk_log_start":"513'4900","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5372,"log_dups_size":0,"ondisk_log_size":5372,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:26.533964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120586240,"num_objects":1840,"num_object_clones":0,"num_object_copies":5520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10272,"num_write_kb":387584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":633,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.722599+0000","last_change":"2023-10-09T06:13:24.753519+0000","last_active":"2023-10-09T06:15:02.722599+0000","last_peered":"2023-10-09T06:15:02.722599+0000","last_clean":"2023-10-09T06:15:02.722599+0000","last_became_active":"2023-10-09T06:13:24.753185+0000","last_became_peered":"2023-10-09T06:13:24.753185+0000","last_unstale":"2023-10-09T06:15:02.722599+0000","last_undegraded":"2023-10-09T06:15:02.722599+0000","last_fullsized":"2023-10-09T06:15:02.722599+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:39.633410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:39.633410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:56.871057+0000","scrub_duration":0.001547624,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":368,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.726842+0000","last_change":"2023-10-09T06:13:30.781849+0000","last_active":"2023-10-09T06:15:03.726842+0000","last_peered":"2023-10-09T06:15:03.726842+0000","last_clean":"2023-10-09T06:15:03.726842+0000","last_became_active":"2023-10-09T06:13:30.781084+0000","last_became_peered":"2023-10-09T06:13:30.781084+0000","last_unstale":"2023-10-09T06:15:03.726842+0000","last_undegraded":"2023-10-09T06:15:03.726842+0000","last_fullsized":"2023-10-09T06:15:03.726842+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:02.199298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":238,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.732731+0000","last_change":"2023-10-09T06:01:54.761768+0000","last_active":"2023-10-09T06:15:03.732731+0000","last_peered":"2023-10-09T06:15:03.732731+0000","last_clean":"2023-10-09T06:15:03.732731+0000","last_became_active":"2023-10-09T06:01:54.761443+0000","last_became_peered":"2023-10-09T06:01:54.761443+0000","last_unstale":"2023-10-09T06:15:03.732731+0000","last_undegraded":"2023-10-09T06:15:03.732731+0000","last_fullsized":"2023-10-09T06:15:03.732731+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:51.555987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"555'10216","reported_seq":17159,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:04.174415+0000","last_change":"2023-10-09T06:14:44.803724+0000","last_active":"2023-10-09T06:15:04.174415+0000","last_peered":"2023-10-09T06:15:04.174415+0000","last_clean":"2023-10-09T06:15:04.174415+0000","last_became_active":"2023-10-09T06:14:44.737126+0000","last_became_peered":"2023-10-09T06:14:44.737126+0000","last_unstale":"2023-10-09T06:15:04.174415+0000","last_undegraded":"2023-10-09T06:15:04.174415+0000","last_fullsized":"2023-10-09T06:15:04.174415+0000","mapping_epoch":549,"log_start":"506'3800","ondisk_log_start":"506'3800","created":490,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6416,"log_dups_size":0,"ondisk_log_size":6416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:14.630144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130416640,"num_objects":1990,"num_object_clones":0,"num_object_copies":5970,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1990,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10216,"num_write_kb":390592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3423,"num_bytes_recovered":224329728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":759,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T06:15:04.878052+0000","last_change":"2023-10-09T06:13:33.707885+0000","last_active":"2023-10-09T06:15:04.878052+0000","last_peered":"2023-10-09T06:15:04.878052+0000","last_clean":"2023-10-09T06:15:04.878052+0000","last_became_active":"2023-10-09T06:13:33.707474+0000","last_became_peered":"2023-10-09T06:13:33.707474+0000","last_unstale":"2023-10-09T06:15:04.878052+0000","last_undegraded":"2023-10-09T06:15:04.878052+0000","last_fullsized":"2023-10-09T06:15:04.878052+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:12:22.289803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:12:22.289803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:27.842817+0000","scrub_duration":0.0076565050000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":301,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.731892+0000","last_change":"2023-10-09T06:12:04.680247+0000","last_active":"2023-10-09T06:15:03.731892+0000","last_peered":"2023-10-09T06:15:03.731892+0000","last_clean":"2023-10-09T06:15:03.731892+0000","last_became_active":"2023-10-09T06:12:04.679943+0000","last_became_peered":"2023-10-09T06:12:04.679943+0000","last_unstale":"2023-10-09T06:15:03.731892+0000","last_undegraded":"2023-10-09T06:15:03.731892+0000","last_fullsized":"2023-10-09T06:15:03.731892+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:56.692774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":344,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.728606+0000","last_change":"2023-10-09T06:07:43.577306+0000","last_active":"2023-10-09T06:15:03.728606+0000","last_peered":"2023-10-09T06:15:03.728606+0000","last_clean":"2023-10-09T06:15:03.728606+0000","last_became_active":"2023-10-09T06:07:43.576956+0000","last_became_peered":"2023-10-09T06:07:43.576956+0000","last_unstale":"2023-10-09T06:15:03.728606+0000","last_undegraded":"2023-10-09T06:15:03.728606+0000","last_fullsized":"2023-10-09T06:15:03.728606+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:21.020403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":272,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.731429+0000","last_change":"2023-10-09T06:13:12.234849+0000","last_active":"2023-10-09T06:15:03.731429+0000","last_peered":"2023-10-09T06:15:03.731429+0000","last_clean":"2023-10-09T06:15:03.731429+0000","last_became_active":"2023-10-09T06:13:12.234284+0000","last_became_peered":"2023-10-09T06:13:12.234284+0000","last_unstale":"2023-10-09T06:15:03.731429+0000","last_undegraded":"2023-10-09T06:15:03.731429+0000","last_fullsized":"2023-10-09T06:15:03.731429+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:40:43.809773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"556'10091","reported_seq":10167,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T06:15:06.388850+0000","last_change":"2023-10-09T06:11:46.029349+0000","last_active":"2023-10-09T06:15:06.388850+0000","last_peered":"2023-10-09T06:15:06.388850+0000","last_clean":"2023-10-09T06:15:06.388850+0000","last_became_active":"2023-10-09T06:11:46.029041+0000","last_became_peered":"2023-10-09T06:11:46.029041+0000","last_unstale":"2023-10-09T06:15:06.388850+0000","last_undegraded":"2023-10-09T06:15:06.388850+0000","last_fullsized":"2023-10-09T06:15:06.388850+0000","mapping_epoch":490,"log_start":"506'3700","ondisk_log_start":"506'3700","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6391,"log_dups_size":0,"ondisk_log_size":6391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:07.662395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116719616,"num_objects":1781,"num_object_clones":0,"num_object_copies":5343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10091,"num_write_kb":379904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":791,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.767922+0000","last_change":"2023-10-09T06:09:27.544555+0000","last_active":"2023-10-09T06:15:03.767922+0000","last_peered":"2023-10-09T06:15:03.767922+0000","last_clean":"2023-10-09T06:15:03.767922+0000","last_became_active":"2023-10-09T06:09:27.544222+0000","last_became_peered":"2023-10-09T06:09:27.544222+0000","last_unstale":"2023-10-09T06:15:03.767922+0000","last_undegraded":"2023-10-09T06:15:03.767922+0000","last_fullsized":"2023-10-09T06:15:03.767922+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:45.944253+0000","scrub_duration":0.0016660799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":257,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.731449+0000","last_change":"2023-10-09T06:13:21.738040+0000","last_active":"2023-10-09T06:15:03.731449+0000","last_peered":"2023-10-09T06:15:03.731449+0000","last_clean":"2023-10-09T06:15:03.731449+0000","last_became_active":"2023-10-09T06:13:21.737717+0000","last_became_peered":"2023-10-09T06:13:21.737717+0000","last_unstale":"2023-10-09T06:15:03.731449+0000","last_undegraded":"2023-10-09T06:15:03.731449+0000","last_fullsized":"2023-10-09T06:15:03.731449+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:39.451628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":536,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.775559+0000","last_change":"2023-10-09T06:09:27.553958+0000","last_active":"2023-10-09T06:15:03.775559+0000","last_peered":"2023-10-09T06:15:03.775559+0000","last_clean":"2023-10-09T06:15:03.775559+0000","last_became_active":"2023-10-09T06:09:27.553193+0000","last_became_peered":"2023-10-09T06:09:27.553193+0000","last_unstale":"2023-10-09T06:15:03.775559+0000","last_undegraded":"2023-10-09T06:15:03.775559+0000","last_fullsized":"2023-10-09T06:15:03.775559+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:09.754579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":382,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.731970+0000","last_change":"2023-10-09T06:07:43.580765+0000","last_active":"2023-10-09T06:15:03.731970+0000","last_peered":"2023-10-09T06:15:03.731970+0000","last_clean":"2023-10-09T06:15:03.731970+0000","last_became_active":"2023-10-09T06:07:43.580458+0000","last_became_peered":"2023-10-09T06:07:43.580458+0000","last_unstale":"2023-10-09T06:15:03.731970+0000","last_undegraded":"2023-10-09T06:15:03.731970+0000","last_fullsized":"2023-10-09T06:15:03.731970+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:55.161893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"555'10018","reported_seq":26347,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:04.143149+0000","last_change":"2023-10-09T06:14:21.793267+0000","last_active":"2023-10-09T06:15:04.143149+0000","last_peered":"2023-10-09T06:15:04.143149+0000","last_clean":"2023-10-09T06:15:04.143149+0000","last_became_active":"2023-10-09T06:14:21.734665+0000","last_became_peered":"2023-10-09T06:14:21.734665+0000","last_unstale":"2023-10-09T06:15:04.143149+0000","last_undegraded":"2023-10-09T06:15:04.143149+0000","last_fullsized":"2023-10-09T06:15:04.143149+0000","mapping_epoch":543,"log_start":"505'3600","ondisk_log_start":"505'3600","created":490,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6418,"log_dups_size":0,"ondisk_log_size":6418,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:01.941433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123338752,"num_objects":1882,"num_object_clones":0,"num_object_copies":5646,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1882,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10018,"num_write_kb":380800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7073,"num_bytes_recovered":463536128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":610,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.740561+0000","last_change":"2023-10-09T06:14:47.339271+0000","last_active":"2023-10-09T06:15:02.740561+0000","last_peered":"2023-10-09T06:15:02.740561+0000","last_clean":"2023-10-09T06:15:02.740561+0000","last_became_active":"2023-10-09T06:13:33.705351+0000","last_became_peered":"2023-10-09T06:13:33.705351+0000","last_unstale":"2023-10-09T06:15:02.740561+0000","last_undegraded":"2023-10-09T06:15:02.740561+0000","last_fullsized":"2023-10-09T06:15:02.740561+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:14:47.339163+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T06:14:47.339163+0000","last_clean_scrub_stamp":"2023-10-09T06:14:47.339163+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:34.887758+0000","scrub_duration":0.0016607729999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":336,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.727674+0000","last_change":"2023-10-09T06:09:27.565004+0000","last_active":"2023-10-09T06:15:03.727674+0000","last_peered":"2023-10-09T06:15:03.727674+0000","last_clean":"2023-10-09T06:15:03.727674+0000","last_became_active":"2023-10-09T06:09:27.564073+0000","last_became_peered":"2023-10-09T06:09:27.564073+0000","last_unstale":"2023-10-09T06:15:03.727674+0000","last_undegraded":"2023-10-09T06:15:03.727674+0000","last_fullsized":"2023-10-09T06:15:03.727674+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:15.146042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":326,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.733976+0000","last_change":"2023-10-09T06:13:21.728291+0000","last_active":"2023-10-09T06:15:03.733976+0000","last_peered":"2023-10-09T06:15:03.733976+0000","last_clean":"2023-10-09T06:15:03.733976+0000","last_became_active":"2023-10-09T06:13:21.727926+0000","last_became_peered":"2023-10-09T06:13:21.727926+0000","last_unstale":"2023-10-09T06:15:03.733976+0000","last_undegraded":"2023-10-09T06:15:03.733976+0000","last_fullsized":"2023-10-09T06:15:03.733976+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:35.292785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":254,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.732676+0000","last_change":"2023-10-09T06:09:27.547953+0000","last_active":"2023-10-09T06:15:03.732676+0000","last_peered":"2023-10-09T06:15:03.732676+0000","last_clean":"2023-10-09T06:15:03.732676+0000","last_became_active":"2023-10-09T06:09:27.547624+0000","last_became_peered":"2023-10-09T06:09:27.547624+0000","last_unstale":"2023-10-09T06:15:03.732676+0000","last_undegraded":"2023-10-09T06:15:03.732676+0000","last_fullsized":"2023-10-09T06:15:03.732676+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:39.630486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"556'10180","reported_seq":18092,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T06:15:06.649754+0000","last_change":"2023-10-09T06:14:43.759714+0000","last_active":"2023-10-09T06:15:06.649754+0000","last_peered":"2023-10-09T06:15:06.649754+0000","last_clean":"2023-10-09T06:15:06.649754+0000","last_became_active":"2023-10-09T06:14:43.736592+0000","last_became_peered":"2023-10-09T06:14:43.736592+0000","last_unstale":"2023-10-09T06:15:06.649754+0000","last_undegraded":"2023-10-09T06:15:06.649754+0000","last_fullsized":"2023-10-09T06:15:06.649754+0000","mapping_epoch":548,"log_start":"505'3500","ondisk_log_start":"505'3500","created":490,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6680,"log_dups_size":0,"ondisk_log_size":6680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:35.196970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114950144,"num_objects":1754,"num_object_clones":0,"num_object_copies":5262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10180,"num_write_kb":381888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3906,"num_bytes_recovered":255983616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":38,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.740940+0000","last_change":"2023-10-09T06:13:30.791685+0000","last_active":"2023-10-09T06:15:02.740940+0000","last_peered":"2023-10-09T06:15:02.740940+0000","last_clean":"2023-10-09T06:15:02.740940+0000","last_became_active":"2023-10-09T06:13:30.791255+0000","last_became_peered":"2023-10-09T06:13:30.791255+0000","last_unstale":"2023-10-09T06:15:02.740940+0000","last_undegraded":"2023-10-09T06:15:02.740940+0000","last_fullsized":"2023-10-09T06:15:02.740940+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:24.896244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":598,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.738887+0000","last_change":"2023-10-09T06:13:33.704835+0000","last_active":"2023-10-09T06:15:02.738887+0000","last_peered":"2023-10-09T06:15:02.738887+0000","last_clean":"2023-10-09T06:15:02.738887+0000","last_became_active":"2023-10-09T06:13:33.704466+0000","last_became_peered":"2023-10-09T06:13:33.704466+0000","last_unstale":"2023-10-09T06:15:02.738887+0000","last_undegraded":"2023-10-09T06:15:02.738887+0000","last_fullsized":"2023-10-09T06:15:02.738887+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:35.783434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:35.783434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:38.721974+0000","scrub_duration":0.0043794799999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":344,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.727819+0000","last_change":"2023-10-09T06:13:12.238280+0000","last_active":"2023-10-09T06:15:03.727819+0000","last_peered":"2023-10-09T06:15:03.727819+0000","last_clean":"2023-10-09T06:15:03.727819+0000","last_became_active":"2023-10-09T06:13:12.237986+0000","last_became_peered":"2023-10-09T06:13:12.237986+0000","last_unstale":"2023-10-09T06:15:03.727819+0000","last_undegraded":"2023-10-09T06:15:03.727819+0000","last_fullsized":"2023-10-09T06:15:03.727819+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:58.520176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":486,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.776810+0000","last_change":"2023-10-09T06:13:21.733940+0000","last_active":"2023-10-09T06:15:03.776810+0000","last_peered":"2023-10-09T06:15:03.776810+0000","last_clean":"2023-10-09T06:15:03.776810+0000","last_became_active":"2023-10-09T06:13:21.733628+0000","last_became_peered":"2023-10-09T06:13:21.733628+0000","last_unstale":"2023-10-09T06:15:03.776810+0000","last_undegraded":"2023-10-09T06:15:03.776810+0000","last_fullsized":"2023-10-09T06:15:03.776810+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:13.348443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"555'10001","reported_seq":19967,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:04.154913+0000","last_change":"2023-10-09T06:14:09.843785+0000","last_active":"2023-10-09T06:15:04.154913+0000","last_peered":"2023-10-09T06:15:04.154913+0000","last_clean":"2023-10-09T06:15:04.154913+0000","last_became_active":"2023-10-09T06:14:09.789864+0000","last_became_peered":"2023-10-09T06:14:09.789864+0000","last_unstale":"2023-10-09T06:15:04.154913+0000","last_undegraded":"2023-10-09T06:15:04.154913+0000","last_fullsized":"2023-10-09T06:15:04.154913+0000","mapping_epoch":540,"log_start":"505'3600","ondisk_log_start":"505'3600","created":490,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6401,"log_dups_size":0,"ondisk_log_size":6401,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:03.191322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125370368,"num_objects":1913,"num_object_clones":0,"num_object_copies":5739,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1913,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10001,"num_write_kb":381248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4936,"num_bytes_recovered":323485696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":825,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.731742+0000","last_change":"2023-10-09T06:13:24.753842+0000","last_active":"2023-10-09T06:15:03.731742+0000","last_peered":"2023-10-09T06:15:03.731742+0000","last_clean":"2023-10-09T06:15:03.731742+0000","last_became_active":"2023-10-09T06:13:24.753485+0000","last_became_peered":"2023-10-09T06:13:24.753485+0000","last_unstale":"2023-10-09T06:15:03.731742+0000","last_undegraded":"2023-10-09T06:15:03.731742+0000","last_fullsized":"2023-10-09T06:15:03.731742+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:57.627296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:57.627296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:56.656218+0000","scrub_duration":0.001913166,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":448,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.739199+0000","last_change":"2023-10-09T06:09:27.544373+0000","last_active":"2023-10-09T06:15:02.739199+0000","last_peered":"2023-10-09T06:15:02.739199+0000","last_clean":"2023-10-09T06:15:02.739199+0000","last_became_active":"2023-10-09T06:09:27.544121+0000","last_became_peered":"2023-10-09T06:09:27.544121+0000","last_unstale":"2023-10-09T06:15:02.739199+0000","last_undegraded":"2023-10-09T06:15:02.739199+0000","last_fullsized":"2023-10-09T06:15:02.739199+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:40.429491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":493,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.767790+0000","last_change":"2023-10-09T06:12:52.670573+0000","last_active":"2023-10-09T06:15:03.767790+0000","last_peered":"2023-10-09T06:15:03.767790+0000","last_clean":"2023-10-09T06:15:03.767790+0000","last_became_active":"2023-10-09T06:12:52.670242+0000","last_became_peered":"2023-10-09T06:12:52.670242+0000","last_unstale":"2023-10-09T06:15:03.767790+0000","last_undegraded":"2023-10-09T06:15:03.767790+0000","last_fullsized":"2023-10-09T06:15:03.767790+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:25.237580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":632,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.768256+0000","last_change":"2023-10-09T06:09:27.543678+0000","last_active":"2023-10-09T06:15:03.768256+0000","last_peered":"2023-10-09T06:15:03.768256+0000","last_clean":"2023-10-09T06:15:03.768256+0000","last_became_active":"2023-10-09T06:09:27.543298+0000","last_became_peered":"2023-10-09T06:09:27.543298+0000","last_unstale":"2023-10-09T06:15:03.768256+0000","last_undegraded":"2023-10-09T06:15:03.768256+0000","last_fullsized":"2023-10-09T06:15:03.768256+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:26.297842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"556'10139","reported_seq":19746,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T06:15:05.887807+0000","last_change":"2023-10-09T06:14:02.643285+0000","last_active":"2023-10-09T06:15:05.887807+0000","last_peered":"2023-10-09T06:15:05.887807+0000","last_clean":"2023-10-09T06:15:05.887807+0000","last_became_active":"2023-10-09T06:14:02.599489+0000","last_became_peered":"2023-10-09T06:14:02.599489+0000","last_unstale":"2023-10-09T06:15:05.887807+0000","last_undegraded":"2023-10-09T06:15:05.887807+0000","last_fullsized":"2023-10-09T06:15:05.887807+0000","mapping_epoch":537,"log_start":"508'4700","ondisk_log_start":"508'4700","created":490,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5439,"log_dups_size":0,"ondisk_log_size":5439,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:10.022074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120848408,"num_objects":1845,"num_object_clones":0,"num_object_copies":5535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1845,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10139,"num_write_kb":383425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4755,"num_bytes_recovered":311558168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":755,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.729882+0000","last_change":"2023-10-09T06:13:24.756288+0000","last_active":"2023-10-09T06:15:03.729882+0000","last_peered":"2023-10-09T06:15:03.729882+0000","last_clean":"2023-10-09T06:15:03.729882+0000","last_became_active":"2023-10-09T06:13:24.755945+0000","last_became_peered":"2023-10-09T06:13:24.755945+0000","last_unstale":"2023-10-09T06:15:03.729882+0000","last_undegraded":"2023-10-09T06:15:03.729882+0000","last_fullsized":"2023-10-09T06:15:03.729882+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:45.844892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:45.844892+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:44.107376+0000","scrub_duration":0.0017521489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":38,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.741036+0000","last_change":"2023-10-09T06:13:30.799535+0000","last_active":"2023-10-09T06:15:02.741036+0000","last_peered":"2023-10-09T06:15:02.741036+0000","last_clean":"2023-10-09T06:15:02.741036+0000","last_became_active":"2023-10-09T06:13:30.799227+0000","last_became_peered":"2023-10-09T06:13:30.799227+0000","last_unstale":"2023-10-09T06:15:02.741036+0000","last_undegraded":"2023-10-09T06:15:02.741036+0000","last_fullsized":"2023-10-09T06:15:02.741036+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:14.741742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":275,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.732051+0000","last_change":"2023-10-09T06:13:12.237622+0000","last_active":"2023-10-09T06:15:03.732051+0000","last_peered":"2023-10-09T06:15:03.732051+0000","last_clean":"2023-10-09T06:15:03.732051+0000","last_became_active":"2023-10-09T06:13:12.237287+0000","last_became_peered":"2023-10-09T06:13:12.237287+0000","last_unstale":"2023-10-09T06:15:03.732051+0000","last_undegraded":"2023-10-09T06:15:03.732051+0000","last_fullsized":"2023-10-09T06:15:03.732051+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:01.942898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.740772+0000","last_change":"2023-10-09T06:13:30.788382+0000","last_active":"2023-10-09T06:15:02.740772+0000","last_peered":"2023-10-09T06:15:02.740772+0000","last_clean":"2023-10-09T06:15:02.740772+0000","last_became_active":"2023-10-09T06:13:30.788060+0000","last_became_peered":"2023-10-09T06:13:30.788060+0000","last_unstale":"2023-10-09T06:15:02.740772+0000","last_undegraded":"2023-10-09T06:15:02.740772+0000","last_fullsized":"2023-10-09T06:15:02.740772+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:36:58.984799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.e","version":"556'10287","reported_seq":21393,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T06:15:05.895887+0000","last_change":"2023-10-09T06:13:51.831598+0000","last_active":"2023-10-09T06:15:05.895887+0000","last_peered":"2023-10-09T06:15:05.895887+0000","last_clean":"2023-10-09T06:15:05.895887+0000","last_became_active":"2023-10-09T06:13:51.816987+0000","last_became_peered":"2023-10-09T06:13:51.816987+0000","last_unstale":"2023-10-09T06:15:05.895887+0000","last_undegraded":"2023-10-09T06:15:05.895887+0000","last_fullsized":"2023-10-09T06:15:05.895887+0000","mapping_epoch":534,"log_start":"513'4900","ondisk_log_start":"513'4900","created":490,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5387,"log_dups_size":0,"ondisk_log_size":5387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:52.460066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123797504,"num_objects":1889,"num_object_clones":0,"num_object_copies":5667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10287,"num_write_kb":389632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5503,"num_bytes_recovered":360644608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":1029,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.777004+0000","last_change":"2023-10-09T06:13:24.762291+0000","last_active":"2023-10-09T06:15:03.777004+0000","last_peered":"2023-10-09T06:15:03.777004+0000","last_clean":"2023-10-09T06:15:03.777004+0000","last_became_active":"2023-10-09T06:13:24.761918+0000","last_became_peered":"2023-10-09T06:13:24.761918+0000","last_unstale":"2023-10-09T06:15:03.777004+0000","last_undegraded":"2023-10-09T06:15:03.777004+0000","last_fullsized":"2023-10-09T06:15:03.777004+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:17.344518+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:17.344518+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:58.227603+0000","scrub_duration":0.0015914099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":612,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.731396+0000","last_change":"2023-10-09T06:13:30.798394+0000","last_active":"2023-10-09T06:15:03.731396+0000","last_peered":"2023-10-09T06:15:03.731396+0000","last_clean":"2023-10-09T06:15:03.731396+0000","last_became_active":"2023-10-09T06:13:30.790913+0000","last_became_peered":"2023-10-09T06:13:30.790913+0000","last_unstale":"2023-10-09T06:15:03.731396+0000","last_undegraded":"2023-10-09T06:15:03.731396+0000","last_fullsized":"2023-10-09T06:15:03.731396+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:26.508690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":373,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.732542+0000","last_change":"2023-10-09T06:13:21.728009+0000","last_active":"2023-10-09T06:15:03.732542+0000","last_peered":"2023-10-09T06:15:03.732542+0000","last_clean":"2023-10-09T06:15:03.732542+0000","last_became_active":"2023-10-09T06:13:21.727683+0000","last_became_peered":"2023-10-09T06:13:21.727683+0000","last_unstale":"2023-10-09T06:15:03.732542+0000","last_undegraded":"2023-10-09T06:15:03.732542+0000","last_fullsized":"2023-10-09T06:15:03.732542+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:13.074232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":361,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.727056+0000","last_change":"2023-10-09T06:13:30.782038+0000","last_active":"2023-10-09T06:15:03.727056+0000","last_peered":"2023-10-09T06:15:03.727056+0000","last_clean":"2023-10-09T06:15:03.727056+0000","last_became_active":"2023-10-09T06:13:30.781500+0000","last_became_peered":"2023-10-09T06:13:30.781500+0000","last_unstale":"2023-10-09T06:15:03.727056+0000","last_undegraded":"2023-10-09T06:15:03.727056+0000","last_fullsized":"2023-10-09T06:15:03.727056+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:43.663265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"556'10080","reported_seq":10156,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T06:15:05.477718+0000","last_change":"2023-10-09T06:11:46.031709+0000","last_active":"2023-10-09T06:15:05.477718+0000","last_peered":"2023-10-09T06:15:05.477718+0000","last_clean":"2023-10-09T06:15:05.477718+0000","last_became_active":"2023-10-09T06:11:46.031421+0000","last_became_peered":"2023-10-09T06:11:46.031421+0000","last_unstale":"2023-10-09T06:15:05.477718+0000","last_undegraded":"2023-10-09T06:15:05.477718+0000","last_fullsized":"2023-10-09T06:15:05.477718+0000","mapping_epoch":490,"log_start":"506'4125","ondisk_log_start":"506'4125","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5955,"log_dups_size":0,"ondisk_log_size":5955,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:13.095174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120586240,"num_objects":1840,"num_object_clones":0,"num_object_copies":5520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10080,"num_write_kb":381440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":888,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.727463+0000","last_change":"2023-10-09T06:13:33.708100+0000","last_active":"2023-10-09T06:15:03.727463+0000","last_peered":"2023-10-09T06:15:03.727463+0000","last_clean":"2023-10-09T06:15:03.727463+0000","last_became_active":"2023-10-09T06:13:33.707809+0000","last_became_peered":"2023-10-09T06:13:33.707809+0000","last_unstale":"2023-10-09T06:15:03.727463+0000","last_undegraded":"2023-10-09T06:15:03.727463+0000","last_fullsized":"2023-10-09T06:15:03.727463+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:43:26.848778+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":38,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.740970+0000","last_change":"2023-10-09T06:13:30.799948+0000","last_active":"2023-10-09T06:15:02.740970+0000","last_peered":"2023-10-09T06:15:02.740970+0000","last_clean":"2023-10-09T06:15:02.740970+0000","last_became_active":"2023-10-09T06:13:30.799690+0000","last_became_peered":"2023-10-09T06:13:30.799690+0000","last_unstale":"2023-10-09T06:15:02.740970+0000","last_undegraded":"2023-10-09T06:15:02.740970+0000","last_fullsized":"2023-10-09T06:15:02.740970+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:43.893184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":602,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.772320+0000","last_change":"2023-10-09T06:09:27.578756+0000","last_active":"2023-10-09T06:15:03.772320+0000","last_peered":"2023-10-09T06:15:03.772320+0000","last_clean":"2023-10-09T06:15:03.772320+0000","last_became_active":"2023-10-09T06:09:27.578384+0000","last_became_peered":"2023-10-09T06:09:27.578384+0000","last_unstale":"2023-10-09T06:15:03.772320+0000","last_undegraded":"2023-10-09T06:15:03.772320+0000","last_fullsized":"2023-10-09T06:15:03.772320+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:22.182630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":502,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.775870+0000","last_change":"2023-10-09T06:01:22.334081+0000","last_active":"2023-10-09T06:15:03.775870+0000","last_peered":"2023-10-09T06:15:03.775870+0000","last_clean":"2023-10-09T06:15:03.775870+0000","last_became_active":"2023-10-09T06:01:22.333663+0000","last_became_peered":"2023-10-09T06:01:22.333663+0000","last_unstale":"2023-10-09T06:15:03.775870+0000","last_undegraded":"2023-10-09T06:15:03.775870+0000","last_fullsized":"2023-10-09T06:15:03.775870+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:55.809259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":802,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.727166+0000","last_change":"2023-10-09T06:09:27.560157+0000","last_active":"2023-10-09T06:15:03.727166+0000","last_peered":"2023-10-09T06:15:03.727166+0000","last_clean":"2023-10-09T06:15:03.727166+0000","last_became_active":"2023-10-09T06:09:27.559466+0000","last_became_peered":"2023-10-09T06:09:27.559466+0000","last_unstale":"2023-10-09T06:15:03.727166+0000","last_undegraded":"2023-10-09T06:15:03.727166+0000","last_fullsized":"2023-10-09T06:15:03.727166+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.181811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.181811+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:53.431491+0000","scrub_duration":0.011553313000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":485,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.768389+0000","last_change":"2023-10-09T06:13:30.791343+0000","last_active":"2023-10-09T06:15:03.768389+0000","last_peered":"2023-10-09T06:15:03.768389+0000","last_clean":"2023-10-09T06:15:03.768389+0000","last_became_active":"2023-10-09T06:13:30.790945+0000","last_became_peered":"2023-10-09T06:13:30.790945+0000","last_unstale":"2023-10-09T06:15:03.768389+0000","last_undegraded":"2023-10-09T06:15:03.768389+0000","last_fullsized":"2023-10-09T06:15:03.768389+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:14.208391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":258,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.732125+0000","last_change":"2023-10-09T06:07:43.580844+0000","last_active":"2023-10-09T06:15:03.732125+0000","last_peered":"2023-10-09T06:15:03.732125+0000","last_clean":"2023-10-09T06:15:03.732125+0000","last_became_active":"2023-10-09T06:07:43.580523+0000","last_became_peered":"2023-10-09T06:07:43.580523+0000","last_unstale":"2023-10-09T06:15:03.732125+0000","last_undegraded":"2023-10-09T06:15:03.732125+0000","last_fullsized":"2023-10-09T06:15:03.732125+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:55.942430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":905,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.729736+0000","last_change":"2023-10-09T06:00:04.215425+0000","last_active":"2023-10-09T06:15:03.729736+0000","last_peered":"2023-10-09T06:15:03.729736+0000","last_clean":"2023-10-09T06:15:03.729736+0000","last_became_active":"2023-10-09T05:59:40.326685+0000","last_became_peered":"2023-10-09T05:59:40.326685+0000","last_unstale":"2023-10-09T06:15:03.729736+0000","last_undegraded":"2023-10-09T06:15:03.729736+0000","last_fullsized":"2023-10-09T06:15:03.729736+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:04.215270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:04.215270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:03.963694+0000","scrub_duration":0.002850652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":327,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.776206+0000","last_change":"2023-10-09T06:01:54.824780+0000","last_active":"2023-10-09T06:15:03.776206+0000","last_peered":"2023-10-09T06:15:03.776206+0000","last_clean":"2023-10-09T06:15:03.776206+0000","last_became_active":"2023-10-09T06:01:54.757727+0000","last_became_peered":"2023-10-09T06:01:54.757727+0000","last_unstale":"2023-10-09T06:15:03.776206+0000","last_undegraded":"2023-10-09T06:15:03.776206+0000","last_fullsized":"2023-10-09T06:15:03.776206+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:44.132556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":574,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.776336+0000","last_change":"2023-10-09T06:01:54.764989+0000","last_active":"2023-10-09T06:15:03.776336+0000","last_peered":"2023-10-09T06:15:03.776336+0000","last_clean":"2023-10-09T06:15:03.776336+0000","last_became_active":"2023-10-09T06:01:54.764790+0000","last_became_peered":"2023-10-09T06:01:54.764790+0000","last_unstale":"2023-10-09T06:15:03.776336+0000","last_undegraded":"2023-10-09T06:15:03.776336+0000","last_fullsized":"2023-10-09T06:15:03.776336+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:14:07.208858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":341,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.776533+0000","last_change":"2023-10-09T06:13:21.739730+0000","last_active":"2023-10-09T06:15:03.776533+0000","last_peered":"2023-10-09T06:15:03.776533+0000","last_clean":"2023-10-09T06:15:03.776533+0000","last_became_active":"2023-10-09T06:13:21.727326+0000","last_became_peered":"2023-10-09T06:13:21.727326+0000","last_unstale":"2023-10-09T06:15:03.776533+0000","last_undegraded":"2023-10-09T06:15:03.776533+0000","last_fullsized":"2023-10-09T06:15:03.776533+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:51.344571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":571,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.775763+0000","last_change":"2023-10-09T06:09:27.565198+0000","last_active":"2023-10-09T06:15:03.775763+0000","last_peered":"2023-10-09T06:15:03.775763+0000","last_clean":"2023-10-09T06:15:03.775763+0000","last_became_active":"2023-10-09T06:09:27.564909+0000","last_became_peered":"2023-10-09T06:09:27.564909+0000","last_unstale":"2023-10-09T06:15:03.775763+0000","last_undegraded":"2023-10-09T06:15:03.775763+0000","last_fullsized":"2023-10-09T06:15:03.775763+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:46.101014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":720,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T06:15:02.742053+0000","last_change":"2023-10-09T06:13:33.699235+0000","last_active":"2023-10-09T06:15:02.742053+0000","last_peered":"2023-10-09T06:15:02.742053+0000","last_clean":"2023-10-09T06:15:02.742053+0000","last_became_active":"2023-10-09T06:13:33.698886+0000","last_became_peered":"2023-10-09T06:13:33.698886+0000","last_unstale":"2023-10-09T06:15:02.742053+0000","last_undegraded":"2023-10-09T06:15:02.742053+0000","last_fullsized":"2023-10-09T06:15:02.742053+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:00.456132+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:00.456132+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:59.421706+0000","scrub_duration":0.0014992199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":362,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.730045+0000","last_change":"2023-10-09T06:13:21.740383+0000","last_active":"2023-10-09T06:15:03.730045+0000","last_peered":"2023-10-09T06:15:03.730045+0000","last_clean":"2023-10-09T06:15:03.730045+0000","last_became_active":"2023-10-09T06:13:21.740023+0000","last_became_peered":"2023-10-09T06:13:21.740023+0000","last_unstale":"2023-10-09T06:15:03.730045+0000","last_undegraded":"2023-10-09T06:15:03.730045+0000","last_fullsized":"2023-10-09T06:15:03.730045+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:27.380765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":358,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.727323+0000","last_change":"2023-10-09T06:09:27.565185+0000","last_active":"2023-10-09T06:15:03.727323+0000","last_peered":"2023-10-09T06:15:03.727323+0000","last_clean":"2023-10-09T06:15:03.727323+0000","last_became_active":"2023-10-09T06:09:27.564478+0000","last_became_peered":"2023-10-09T06:09:27.564478+0000","last_unstale":"2023-10-09T06:15:03.727323+0000","last_undegraded":"2023-10-09T06:15:03.727323+0000","last_fullsized":"2023-10-09T06:15:03.727323+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:04.622171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":880,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.731944+0000","last_change":"2023-10-09T06:12:55.753877+0000","last_active":"2023-10-09T06:15:03.731944+0000","last_peered":"2023-10-09T06:15:03.731944+0000","last_clean":"2023-10-09T06:15:03.731944+0000","last_became_active":"2023-10-09T06:12:55.753512+0000","last_became_peered":"2023-10-09T06:12:55.753512+0000","last_unstale":"2023-10-09T06:15:03.731944+0000","last_undegraded":"2023-10-09T06:15:03.731944+0000","last_fullsized":"2023-10-09T06:15:03.731944+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:46.290510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:46.290510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:30.368031+0000","scrub_duration":0.002627819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":548,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.768304+0000","last_change":"2023-10-09T06:09:27.556271+0000","last_active":"2023-10-09T06:15:03.768304+0000","last_peered":"2023-10-09T06:15:03.768304+0000","last_clean":"2023-10-09T06:15:03.768304+0000","last_became_active":"2023-10-09T06:09:27.555932+0000","last_became_peered":"2023-10-09T06:09:27.555932+0000","last_unstale":"2023-10-09T06:15:03.768304+0000","last_undegraded":"2023-10-09T06:15:03.768304+0000","last_fullsized":"2023-10-09T06:15:03.768304+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:33.888622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":587,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T06:15:03.775728+0000","last_change":"2023-10-09T06:13:21.727720+0000","last_active":"2023-10-09T06:15:03.775728+0000","last_peered":"2023-10-09T06:15:03.775728+0000","last_clean":"2023-10-09T06:15:03.775728+0000","last_became_active":"2023-10-09T06:13:21.727339+0000","last_became_peered":"2023-10-09T06:13:21.727339+0000","last_unstale":"2023-10-09T06:15:03.775728+0000","last_undegraded":"2023-10-09T06:15:03.775728+0000","last_fullsized":"2023-10-09T06:15:03.775728+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:25.723752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1930690584,"num_objects":29461,"num_object_clones":0,"num_object_copies":88383,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29461,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":162965,"num_write_kb":6157569,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42794,"num_bytes_recovered":2804482072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7629651968,"data_stored":7629635680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":98012,"ondisk_log_size":98012,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":11714,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13410,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1443424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234425,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1389372,"kb_used_data":504948,"kb_used_omap":0,"kb_used_meta":884416,"kb_avail":92355780,"statfs":{"total":95995035648,"available":94572318720,"internally_reserved":0,"allocated":517066752,"data_stored":515500444,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":905641984},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:14:32 2023","interfaces":[{"interface":"back","average":{"1min":14.369,"5min":5.5149999999999997,"15min":2.6600000000000001},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":126.167,"5min":126.167,"15min":126.167},"last":3.1030000000000002},{"interface":"front","average":{"1min":14.465,"5min":5.7229999999999999,"15min":2.6709999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":126.139,"5min":126.139,"15min":126.139},"last":6.1470000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:14:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":4.141,"15min":3.2189999999999999},"min":{"1min":0.27200000000000002,"5min":0.255,"15min":0.218},"max":{"1min":5.9580000000000002,"5min":124.66200000000001,"15min":124.66200000000001},"last":6.1219999999999999},{"interface":"front","average":{"1min":1.216,"5min":4.1829999999999998,"15min":3.2069999999999999},"min":{"1min":0.33900000000000002,"5min":0.27100000000000002,"15min":0.22700000000000001},"max":{"1min":5.8650000000000002,"5min":125.023,"15min":125.023},"last":2.3839999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:14:32 2023","interfaces":[{"interface":"back","average":{"1min":3.766,"5min":2.1549999999999998,"15min":2.403},"min":{"1min":0.29899999999999999,"5min":0.27400000000000002,"15min":0.24299999999999999},"max":{"1min":28.475999999999999,"5min":47.503,"15min":127.86499999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":3.4929999999999999,"5min":2.3719999999999999,"15min":2.391},"min":{"1min":0.30599999999999999,"5min":0.30099999999999999,"15min":0.27000000000000002},"max":{"1min":28.925000000000001,"5min":47.545000000000002,"15min":129.24000000000001},"last":2.3650000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:14:05 2023","interfaces":[{"interface":"back","average":{"1min":10.307,"5min":7.9900000000000002,"15min":6.4820000000000002},"min":{"1min":0.35499999999999998,"5min":0.32100000000000001,"15min":0.29899999999999999},"max":{"1min":67.852000000000004,"5min":116.68000000000001,"15min":164.16999999999999},"last":6.0720000000000001},{"interface":"front","average":{"1min":10.298,"5min":8.2260000000000009,"15min":6.6589999999999998},"min":{"1min":0.36699999999999999,"5min":0.33300000000000002,"15min":0.32100000000000001},"max":{"1min":67.881,"5min":116.797,"15min":164.40700000000001},"last":3.133}]},{"osd":4,"last update":"Mon Oct 9 06:14:41 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":0.98499999999999999,"15min":0.95799999999999996},"min":{"1min":0.27900000000000003,"5min":0.23999999999999999,"15min":0.23100000000000001},"max":{"1min":6.9939999999999998,"5min":6.9939999999999998,"15min":14.371},"last":1.875},{"interface":"front","average":{"1min":1.3520000000000001,"5min":0.90100000000000002,"15min":0.94799999999999995},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":7.9459999999999997,"5min":7.9459999999999997,"15min":14.259},"last":2.2919999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":1.3420000000000001,"15min":1.083},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.24399999999999999},"max":{"1min":2.4700000000000002,"5min":43.826999999999998,"15min":43.826999999999998},"last":6.04},{"interface":"front","average":{"1min":1.038,"5min":1.2729999999999999,"15min":1.139},"min":{"1min":0.34899999999999998,"5min":0.32300000000000001,"15min":0.24399999999999999},"max":{"1min":5.9569999999999999,"5min":43.880000000000003,"15min":43.880000000000003},"last":2.3439999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":0.78600000000000003,"15min":0.97999999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.26600000000000001},"max":{"1min":2.468,"5min":3.8679999999999999,"15min":9.4540000000000006},"last":6.0949999999999998},{"interface":"front","average":{"1min":0.90200000000000002,"5min":0.80600000000000005,"15min":1.018},"min":{"1min":0.33600000000000002,"5min":0.32000000000000001,"15min":0.30599999999999999},"max":{"1min":3.3279999999999998,"5min":4.516,"15min":6.2329999999999997},"last":0.40400000000000003}]}]},{"osd":6,"up_from":46,"seq":197568496062,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1780012,"kb_used_data":878372,"kb_used_omap":0,"kb_used_meta":901632,"kb_avail":91965140,"statfs":{"total":95995035648,"available":94172303360,"internally_reserved":0,"allocated":899452928,"data_stored":897902073,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923271168},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:14:29 2023","interfaces":[{"interface":"back","average":{"1min":11.574,"5min":8.9060000000000006,"15min":4.4640000000000004},"min":{"1min":0.25,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":92.424999999999997,"5min":191.50800000000001,"15min":191.50800000000001},"last":7.3710000000000004},{"interface":"front","average":{"1min":11.849,"5min":9.0709999999999997,"15min":4.3490000000000002},"min":{"1min":0.30399999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":92.396000000000001,"5min":191.52600000000001,"15min":191.52600000000001},"last":7.3559999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:14:28 2023","interfaces":[{"interface":"back","average":{"1min":13.445,"5min":8.6509999999999998,"15min":6.0129999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.23200000000000001},"max":{"1min":193.845,"5min":328.04399999999998,"15min":328.04399999999998},"last":2.0979999999999999},{"interface":"front","average":{"1min":13.298,"5min":8.8420000000000005,"15min":6.1219999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":193.779,"5min":328.10000000000002,"15min":328.10000000000002},"last":4.4299999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:14:29 2023","interfaces":[{"interface":"back","average":{"1min":1.5600000000000001,"5min":4.7709999999999999,"15min":2.944},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.7549999999999999,"5min":163.04900000000001,"15min":163.04900000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.3759999999999999,"5min":4.8239999999999998,"15min":2.879},"min":{"1min":0.32100000000000001,"5min":0.312,"15min":0.28599999999999998},"max":{"1min":5.9820000000000002,"5min":162.93000000000001,"15min":162.93000000000001},"last":4.6280000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:14:12 2023","interfaces":[{"interface":"back","average":{"1min":11.064,"5min":8.5009999999999994,"15min":7.375},"min":{"1min":0.33000000000000002,"5min":0.318,"15min":0.307},"max":{"1min":81.317999999999998,"5min":250.744,"15min":250.744},"last":2.2490000000000001},{"interface":"front","average":{"1min":9.3520000000000003,"5min":7.9539999999999997,"15min":7.1600000000000001},"min":{"1min":0.374,"5min":0.33600000000000002,"15min":0.314},"max":{"1min":81.233000000000004,"5min":250.83799999999999,"15min":250.83799999999999},"last":0.72099999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":1.018,"15min":1.6519999999999999},"min":{"1min":0.28899999999999998,"5min":0.24399999999999999,"15min":0.24199999999999999},"max":{"1min":5.4880000000000004,"5min":9.0869999999999997,"15min":151.35599999999999},"last":0.66200000000000003},{"interface":"front","average":{"1min":1.091,"5min":1.121,"15min":1.7350000000000001},"min":{"1min":0.307,"5min":0.249,"15min":0.23999999999999999},"max":{"1min":5.077,"5min":7.8849999999999998,"15min":149.15799999999999},"last":3.1619999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:14:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":0.94299999999999995,"15min":1.1040000000000001},"min":{"1min":0.315,"5min":0.28000000000000003,"15min":0.23000000000000001},"max":{"1min":8.3829999999999991,"5min":9.1959999999999997,"15min":15.342000000000001},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.95699999999999996,"5min":1.03,"15min":1.1579999999999999},"min":{"1min":0.316,"5min":0.28699999999999998,"15min":0.255},"max":{"1min":2.8450000000000002,"5min":8.9809999999999999,"15min":15.358000000000001},"last":0.753}]},{"osd":7,"last update":"Mon Oct 9 06:14:48 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":0.96999999999999997,"15min":1.238},"min":{"1min":0.33700000000000002,"5min":0.30299999999999999,"15min":0.27000000000000002},"max":{"1min":5.8380000000000001,"5min":5.8380000000000001,"15min":21.98},"last":0.79200000000000004},{"interface":"front","average":{"1min":1.01,"5min":0.86399999999999999,"15min":1.1739999999999999},"min":{"1min":0.34999999999999998,"5min":0.34200000000000003,"15min":0.33100000000000002},"max":{"1min":3.714,"5min":3.714,"15min":18.241},"last":0.57299999999999995}]}]},{"osd":1,"up_from":89,"seq":382252089650,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":973100,"kb_used_data":350372,"kb_used_omap":9,"kb_used_meta":622646,"kb_avail":92772052,"statfs":{"total":95995035648,"available":94998581248,"internally_reserved":0,"allocated":358780928,"data_stored":355601620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":637590505},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:14:39 2023","interfaces":[{"interface":"back","average":{"1min":4.2060000000000004,"5min":2.3319999999999999,"15min":2.294},"min":{"1min":0.221,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":35.939999999999998,"5min":35.939999999999998,"15min":35.939999999999998},"last":1.2789999999999999},{"interface":"front","average":{"1min":4.3760000000000003,"5min":2.798,"15min":2.492},"min":{"1min":0.21199999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":35.326000000000001,"5min":35.326000000000001,"15min":35.326000000000001},"last":0.56000000000000005}]},{"osd":2,"last update":"Mon Oct 9 06:14:45 2023","interfaces":[{"interface":"back","average":{"1min":1.2849999999999999,"5min":1.7929999999999999,"15min":2.4630000000000001},"min":{"1min":0.29899999999999999,"5min":0.217,"15min":0.20799999999999999},"max":{"1min":4.8929999999999998,"5min":15.522,"15min":129.053},"last":0.29499999999999998},{"interface":"front","average":{"1min":4.8550000000000004,"5min":2.9409999999999998,"15min":2.9350000000000001},"min":{"1min":0.32400000000000001,"5min":0.22600000000000001,"15min":0.222},"max":{"1min":55.694000000000003,"5min":55.694000000000003,"15min":129.077},"last":0.34599999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:14:13 2023","interfaces":[{"interface":"back","average":{"1min":5.1669999999999998,"5min":4.484,"15min":5.0309999999999997},"min":{"1min":0.28000000000000003,"5min":0.26800000000000002,"15min":0.24199999999999999},"max":{"1min":48.177999999999997,"5min":113.322,"15min":213.654},"last":0.33400000000000002},{"interface":"front","average":{"1min":4.3879999999999999,"5min":4.6349999999999998,"15min":5.0250000000000004},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.27400000000000002},"max":{"1min":48.216999999999999,"5min":113.38200000000001,"15min":213.77199999999999},"last":0.27600000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:14:45 2023","interfaces":[{"interface":"back","average":{"1min":3.7559999999999998,"5min":3.4860000000000002,"15min":2.3149999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":54.018000000000001,"5min":93.747,"15min":93.747},"last":0.317},{"interface":"front","average":{"1min":5.0860000000000003,"5min":4.4219999999999997,"15min":2.8319999999999999},"min":{"1min":0.35799999999999998,"5min":0.308,"15min":0.308},"max":{"1min":54.280000000000001,"5min":97.852000000000004,"15min":97.852000000000004},"last":0.35899999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.4730000000000001,"15min":1.5820000000000001},"min":{"1min":0.312,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.492,"5min":9.3109999999999999,"15min":36.356999999999999},"last":0.376},{"interface":"front","average":{"1min":2.2639999999999998,"5min":2.5099999999999998,"15min":2.319},"min":{"1min":0.318,"5min":0.318,"15min":0.29799999999999999},"max":{"1min":14.648,"5min":17.603999999999999,"15min":36.268000000000001},"last":0.41499999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":1.6399999999999999,"15min":1.8180000000000001},"min":{"1min":0.36699999999999999,"5min":0.33700000000000002,"15min":0.318},"max":{"1min":5.2050000000000001,"5min":9.282,"15min":52.207000000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":2.0579999999999998,"5min":2.2810000000000001,"15min":2.375},"min":{"1min":0.36199999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":14.619999999999999,"5min":15.384,"15min":51.212000000000003},"last":0.45300000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:14:58 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":3.3919999999999999,"15min":2.984},"min":{"1min":0.39400000000000002,"5min":0.374,"15min":0.374},"max":{"1min":14.571,"5min":56.881,"15min":108.553},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.4019999999999999,"5min":2.3900000000000001,"15min":2.4009999999999998},"min":{"1min":0.42999999999999999,"5min":0.38,"15min":0.38},"max":{"1min":5.9420000000000002,"5min":56.941000000000003,"15min":108.56999999999999},"last":0.63}]}]},{"osd":0,"up_from":456,"seq":1958505087045,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1596416,"kb_used_data":1320312,"kb_used_omap":32,"kb_used_meta":275999,"kb_avail":92148736,"statfs":{"total":95995035648,"available":94360305664,"internally_reserved":0,"allocated":1351999488,"data_stored":1349040719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32794,"internal_metadata":282623974},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:14:40 2023","interfaces":[{"interface":"back","average":{"1min":15.137,"5min":6.5940000000000003,"15min":9.6199999999999992},"min":{"1min":0.24099999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":102.81999999999999,"5min":166.833,"15min":166.833},"last":10.124000000000001},{"interface":"front","average":{"1min":14.031000000000001,"5min":6.9859999999999998,"15min":8.9900000000000002},"min":{"1min":0.28699999999999998,"5min":0.223,"15min":0.223},"max":{"1min":103.108,"5min":166.81100000000001,"15min":166.81100000000001},"last":2.9780000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:14:40 2023","interfaces":[{"interface":"back","average":{"1min":2.7400000000000002,"5min":2.649,"15min":3.589},"min":{"1min":0.30599999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":12.413,"5min":21.648,"15min":21.648},"last":1.1899999999999999},{"interface":"front","average":{"1min":2.21,"5min":2.3210000000000002,"15min":3.375},"min":{"1min":0.25700000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":10.119,"5min":20.152000000000001,"15min":20.152000000000001},"last":10.154999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:14:32 2023","interfaces":[{"interface":"back","average":{"1min":3.2850000000000001,"5min":5.0039999999999996,"15min":12.284000000000001},"min":{"1min":0.38700000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":10.079000000000001,"5min":203.59299999999999,"15min":203.59299999999999},"last":10.217000000000001},{"interface":"front","average":{"1min":3.2440000000000002,"5min":5.3019999999999996,"15min":11.779999999999999},"min":{"1min":0.30299999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":12.191000000000001,"5min":202.96600000000001,"15min":202.96600000000001},"last":5.9509999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:14:40 2023","interfaces":[{"interface":"back","average":{"1min":3.9289999999999998,"5min":2.3700000000000001,"15min":2.3959999999999999},"min":{"1min":0.39400000000000002,"5min":0.317,"15min":0.317},"max":{"1min":21.994,"5min":23.515999999999998,"15min":23.515999999999998},"last":10.242000000000001},{"interface":"front","average":{"1min":3.3319999999999999,"5min":2.7839999999999998,"15min":2.2559999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":21.972999999999999,"5min":39.302,"15min":39.302},"last":2.661}]},{"osd":5,"last update":"Mon Oct 9 06:14:40 2023","interfaces":[{"interface":"back","average":{"1min":2.6179999999999999,"5min":3.8050000000000002,"15min":2.7280000000000002},"min":{"1min":0.32700000000000001,"5min":0.316,"15min":0.316},"max":{"1min":8.2949999999999999,"5min":161.286,"15min":161.286},"last":4.6280000000000001},{"interface":"front","average":{"1min":2.8010000000000002,"5min":3.597,"15min":2.8479999999999999},"min":{"1min":0.40000000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":12.038,"5min":161.31800000000001,"15min":161.31800000000001},"last":0.38600000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:14:40 2023","interfaces":[{"interface":"back","average":{"1min":4.657,"5min":2.5550000000000002,"15min":2.5030000000000001},"min":{"1min":0.43099999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":31.648,"5min":31.648,"15min":31.648},"last":0.34699999999999998},{"interface":"front","average":{"1min":3.5910000000000002,"5min":2.081,"15min":2.585},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":24.43,"5min":24.43,"15min":24.43},"last":10.051}]},{"osd":7,"last update":"Mon Oct 9 06:14:40 2023","interfaces":[{"interface":"back","average":{"1min":2.859,"5min":3.5720000000000001,"15min":2.879},"min":{"1min":0.53700000000000003,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":12.066000000000001,"5min":123.685,"15min":123.685},"last":0.40999999999999998},{"interface":"front","average":{"1min":2.4089999999999998,"5min":3.1429999999999998,"15min":2.8889999999999998},"min":{"1min":0.55900000000000005,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":10.054,"5min":122.482,"15min":122.482},"last":10.093}]}]},{"osd":2,"up_from":94,"seq":403726926141,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1667720,"kb_used_data":888640,"kb_used_omap":81,"kb_used_meta":778926,"kb_avail":92077432,"statfs":{"total":95995035648,"available":94287290368,"internally_reserved":0,"allocated":909967360,"data_stored":907160331,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":797620442},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:14:41 2023","interfaces":[{"interface":"back","average":{"1min":7.6920000000000002,"5min":4.415,"15min":3.5710000000000002},"min":{"1min":0.35999999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":83.652000000000001,"5min":83.652000000000001,"15min":83.652000000000001},"last":0.36799999999999999},{"interface":"front","average":{"1min":7.3620000000000001,"5min":3.8380000000000001,"15min":2.774},"min":{"1min":0.35399999999999998,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":77.578999999999994,"5min":77.578999999999994,"15min":77.578999999999994},"last":0.88700000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:14:41 2023","interfaces":[{"interface":"back","average":{"1min":7.0499999999999998,"5min":5.8970000000000002,"15min":4.9119999999999999},"min":{"1min":0.27300000000000002,"5min":0.248,"15min":0.19400000000000001},"max":{"1min":78.921999999999997,"5min":160.45500000000001,"15min":222.59399999999999},"last":0.77900000000000003},{"interface":"front","average":{"1min":8.5739999999999998,"5min":7.0979999999999999,"15min":5.4329999999999998},"min":{"1min":0.313,"5min":0.27200000000000002,"15min":0.20200000000000001},"max":{"1min":81.292000000000002,"5min":163.697,"15min":224.619},"last":0.38400000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:13:58 2023","interfaces":[{"interface":"back","average":{"1min":12.108000000000001,"5min":7.2910000000000004,"15min":5.101},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23000000000000001},"max":{"1min":122.205,"5min":122.205,"15min":144.88200000000001},"last":0.68600000000000005},{"interface":"front","average":{"1min":11.619,"5min":6.952,"15min":5.0369999999999999},"min":{"1min":0.30199999999999999,"5min":0.252,"15min":0.252},"max":{"1min":122.45,"5min":122.45,"15min":140.79499999999999},"last":0.59599999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":2.8210000000000002,"5min":3.4449999999999998,"15min":2.2989999999999999},"min":{"1min":0.33600000000000002,"5min":0.32200000000000001,"15min":0.28299999999999997},"max":{"1min":9.3849999999999998,"5min":107.51300000000001,"15min":107.51300000000001},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.645,"5min":2.4580000000000002,"15min":2.0409999999999999},"min":{"1min":0.36299999999999999,"5min":0.34100000000000003,"15min":0.31},"max":{"1min":4.6589999999999998,"5min":108.768,"15min":108.768},"last":0.48899999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:14:41 2023","interfaces":[{"interface":"back","average":{"1min":1.788,"5min":1.528,"15min":1.5569999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32600000000000001},"max":{"1min":11.108000000000001,"5min":11.108000000000001,"15min":28.48},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.8500000000000001,"5min":1.516,"15min":1.79},"min":{"1min":0.40200000000000002,"5min":0.316,"15min":0.28899999999999998},"max":{"1min":4.6890000000000001,"5min":9.8699999999999992,"15min":30.318999999999999},"last":0.56299999999999994}]},{"osd":6,"last update":"Mon Oct 9 06:14:41 2023","interfaces":[{"interface":"back","average":{"1min":2.7890000000000001,"5min":2.407,"15min":2.1850000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.314},"max":{"1min":10.09,"5min":14.962,"15min":30.420000000000002},"last":0.47499999999999998},{"interface":"front","average":{"1min":2.1739999999999999,"5min":1.6619999999999999,"15min":1.7969999999999999},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.32200000000000001},"max":{"1min":6.3940000000000001,"5min":11.103,"15min":30.713000000000001},"last":0.64700000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:14:41 2023","interfaces":[{"interface":"back","average":{"1min":2.2469999999999999,"5min":2.919,"15min":2.2959999999999998},"min":{"1min":0.39400000000000002,"5min":0.313,"15min":0.313},"max":{"1min":11.147,"5min":124.069,"15min":124.069},"last":0.57799999999999996},{"interface":"front","average":{"1min":2.3660000000000001,"5min":2.7170000000000001,"15min":2.3450000000000002},"min":{"1min":0.372,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.6769999999999996,"5min":124.08799999999999,"15min":124.08799999999999},"last":0.52900000000000003}]}]},{"osd":3,"up_from":99,"seq":425201762615,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2056320,"kb_used_data":1094648,"kb_used_omap":7,"kb_used_meta":961656,"kb_avail":91688832,"statfs":{"total":95995035648,"available":93889363968,"internally_reserved":0,"allocated":1120919552,"data_stored":1117767923,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":984736360},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:14:36 2023","interfaces":[{"interface":"back","average":{"1min":20.771999999999998,"5min":5.6909999999999998,"15min":3.3029999999999999},"min":{"1min":0.19600000000000001,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":220.45400000000001,"5min":220.45400000000001,"15min":220.45400000000001},"last":4.8049999999999997},{"interface":"front","average":{"1min":21.698,"5min":6.5,"15min":4.5170000000000003},"min":{"1min":0.317,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":216.44200000000001,"5min":216.44200000000001,"15min":216.44200000000001},"last":12.954000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:14:10 2023","interfaces":[{"interface":"back","average":{"1min":4.1269999999999998,"5min":2.218,"15min":2.9510000000000001},"min":{"1min":0.253,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":21.934000000000001,"5min":22.579999999999998,"15min":207.75299999999999},"last":4.8300000000000001},{"interface":"front","average":{"1min":4.3049999999999997,"5min":3.54,"15min":3.9649999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.23200000000000001},"max":{"1min":21.809000000000001,"5min":30.524000000000001,"15min":207.12200000000001},"last":3.7480000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:14:10 2023","interfaces":[{"interface":"back","average":{"1min":2.3889999999999998,"5min":2.4609999999999999,"15min":2.6549999999999998},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.223},"max":{"1min":12.561,"5min":30.547999999999998,"15min":107.01300000000001},"last":4.9790000000000001},{"interface":"front","average":{"1min":2.6640000000000001,"5min":3.0899999999999999,"15min":3.0699999999999998},"min":{"1min":0.313,"5min":0.25600000000000001,"15min":0.23899999999999999},"max":{"1min":11.749000000000001,"5min":25.984000000000002,"15min":109.807},"last":3.6819999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:14:50 2023","interfaces":[{"interface":"back","average":{"1min":1.7929999999999999,"5min":3.008,"15min":3.2029999999999998},"min":{"1min":0.29699999999999999,"5min":0.28199999999999997,"15min":0.254},"max":{"1min":12.523999999999999,"5min":95.739000000000004,"15min":99.876000000000005},"last":4.1539999999999999},{"interface":"front","average":{"1min":2.351,"5min":3.2970000000000002,"15min":3.4340000000000002},"min":{"1min":0.35299999999999998,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":11.787000000000001,"5min":97.941999999999993,"15min":99.914000000000001},"last":3.6349999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:14:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2989999999999999,"5min":2.7200000000000002,"15min":2.0649999999999999},"min":{"1min":0.40500000000000003,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.3970000000000002,"5min":74.706000000000003,"15min":74.706000000000003},"last":4.8799999999999999},{"interface":"front","average":{"1min":1.202,"5min":3.4420000000000002,"15min":3.2759999999999998},"min":{"1min":0.32800000000000001,"5min":0.32500000000000001,"15min":0.28799999999999998},"max":{"1min":4.0449999999999999,"5min":74.685000000000002,"15min":235.56},"last":4.1970000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:14:10 2023","interfaces":[{"interface":"back","average":{"1min":1.734,"5min":1.351,"15min":4.3070000000000004},"min":{"1min":0.46500000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.4210000000000003,"5min":6.8739999999999997,"15min":233.26599999999999},"last":4.9000000000000004},{"interface":"front","average":{"1min":2.266,"5min":2.2069999999999999,"15min":4.7619999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.33600000000000002},"max":{"1min":12.471,"5min":26.890000000000001,"15min":233.24100000000001},"last":4.2240000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:14:10 2023","interfaces":[{"interface":"back","average":{"1min":7.8600000000000003,"5min":4.2569999999999997,"15min":5.2229999999999999},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.35699999999999998},"max":{"1min":94.400000000000006,"5min":94.400000000000006,"15min":230.69399999999999},"last":4.343},{"interface":"front","average":{"1min":8.6210000000000004,"5min":4.5999999999999996,"15min":5.4000000000000004},"min":{"1min":0.52600000000000002,"5min":0.44500000000000001,"15min":0.32600000000000001},"max":{"1min":94.195999999999998,"5min":94.195999999999998,"15min":230.67500000000001},"last":0.77900000000000003}]}]},{"osd":4,"up_from":322,"seq":1382979469478,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1886028,"kb_used_data":1559556,"kb_used_omap":35,"kb_used_meta":326428,"kb_avail":91859124,"statfs":{"total":95995035648,"available":94063742976,"internally_reserved":0,"allocated":1596985344,"data_stored":1595418703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36552,"internal_metadata":334262584},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:14:32 2023","interfaces":[{"interface":"back","average":{"1min":4.5990000000000002,"5min":1.972,"15min":1.649},"min":{"1min":0.254,"5min":0.216,"15min":0.216},"max":{"1min":23.206,"5min":23.206,"15min":23.206},"last":81.066000000000003},{"interface":"front","average":{"1min":3.323,"5min":1.7609999999999999,"15min":1.651},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":22.902999999999999,"5min":22.902999999999999,"15min":22.902999999999999},"last":81.603999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":1.77,"5min":2.0489999999999999,"15min":2.2770000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.20999999999999999},"max":{"1min":4.8360000000000003,"5min":18.884,"15min":78.891000000000005},"last":81.275999999999996},{"interface":"front","average":{"1min":1.905,"5min":1.802,"15min":2.2559999999999998},"min":{"1min":0.32200000000000001,"5min":0.25600000000000001,"15min":0.22800000000000001},"max":{"1min":10.039,"5min":18.617999999999999,"15min":78.921000000000006},"last":81.323999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":2.9929999999999999,"5min":4.5960000000000001,"15min":2.6869999999999998},"min":{"1min":0.39200000000000002,"5min":0.29199999999999998,"15min":0.27200000000000002},"max":{"1min":10.084,"5min":245.166,"15min":245.166},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.845,"5min":4.3230000000000004,"15min":2.5329999999999999},"min":{"1min":0.28699999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.2080000000000002,"5min":245.119,"15min":245.119},"last":0.432}]},{"osd":3,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":3.601,"5min":6.8630000000000004,"15min":3.9740000000000002},"min":{"1min":0.34599999999999997,"5min":0.29399999999999998,"15min":0.26800000000000002},"max":{"1min":20.100999999999999,"5min":246.39400000000001,"15min":246.39400000000001},"last":83.881},{"interface":"front","average":{"1min":3.7240000000000002,"5min":6.3700000000000001,"15min":3.5750000000000002},"min":{"1min":0.40899999999999997,"5min":0.314,"15min":0.314},"max":{"1min":19.984000000000002,"5min":246.99000000000001,"15min":246.99000000000001},"last":87.558999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":6.9269999999999996,"5min":2.0880000000000001,"15min":1.3899999999999999},"min":{"1min":0.28899999999999998,"5min":0.23499999999999999,"15min":0.216},"max":{"1min":99.405000000000001,"5min":99.405000000000001,"15min":99.405000000000001},"last":0.316},{"interface":"front","average":{"1min":7.2169999999999996,"5min":2.274,"15min":1.546},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.245},"max":{"1min":99.206000000000003,"5min":99.206000000000003,"15min":99.206000000000003},"last":0.35899999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.0569999999999999,"15min":2.0859999999999999},"min":{"1min":0.32200000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.9900000000000002,"5min":7.641,"15min":95.492999999999995},"last":0.379},{"interface":"front","average":{"1min":1.7070000000000001,"5min":1.3300000000000001,"15min":2.25},"min":{"1min":0.33200000000000002,"5min":0.32100000000000001,"15min":0.27900000000000003},"max":{"1min":9.2110000000000003,"5min":9.3239999999999998,"15min":95.664000000000001},"last":0.40000000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":6.9320000000000004,"5min":2.5390000000000001,"15min":2.7200000000000002},"min":{"1min":0.38800000000000001,"5min":0.29699999999999999,"15min":0.29499999999999998},"max":{"1min":95.167000000000002,"5min":95.167000000000002,"15min":95.167000000000002},"last":0.438},{"interface":"front","average":{"1min":6.8949999999999996,"5min":2.2599999999999998,"15min":2.403},"min":{"1min":0.34499999999999997,"5min":0.27800000000000002,"15min":0.251},"max":{"1min":94.811999999999998,"5min":94.811999999999998,"15min":94.811999999999998},"last":0.34599999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757698,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1877740,"kb_used_data":979236,"kb_used_omap":0,"kb_used_meta":898496,"kb_avail":91867412,"statfs":{"total":95995035648,"available":94072229888,"internally_reserved":0,"allocated":1002737664,"data_stored":1001183737,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920059904},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:14:34 2023","interfaces":[{"interface":"back","average":{"1min":2.6160000000000001,"5min":3.5510000000000002,"15min":2.7850000000000001},"min":{"1min":0.26700000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":18.347999999999999,"5min":115.92400000000001,"15min":115.92400000000001},"last":0.26600000000000001},{"interface":"front","average":{"1min":3.3650000000000002,"5min":3.8839999999999999,"15min":3.444},"min":{"1min":0.314,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":18.364999999999998,"5min":115.904,"15min":115.904},"last":0.36399999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":3.734,"15min":4.4800000000000004},"min":{"1min":0.253,"5min":0.253,"15min":0.22600000000000001},"max":{"1min":4.0369999999999999,"5min":151.78200000000001,"15min":151.78200000000001},"last":2.29},{"interface":"front","average":{"1min":1.097,"5min":3.8730000000000002,"15min":4.6319999999999997},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.3319999999999999,"5min":152.251,"15min":152.251},"last":2.3660000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:14:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":2.9390000000000001,"15min":2.9529999999999998},"min":{"1min":0.33600000000000002,"5min":0.26600000000000001,"15min":0.254},"max":{"1min":2.5009999999999999,"5min":117.97199999999999,"15min":129.21199999999999},"last":4.1470000000000002},{"interface":"front","average":{"1min":0.96799999999999997,"5min":3.2330000000000001,"15min":3.0129999999999999},"min":{"1min":0.35599999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.6160000000000001,"5min":118.52,"15min":129.042},"last":2.2069999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:15:04 2023","interfaces":[{"interface":"back","average":{"1min":3.2639999999999998,"5min":5.2110000000000003,"15min":7.0629999999999997},"min":{"1min":0.38700000000000001,"5min":0.312,"15min":0.29499999999999998},"max":{"1min":19.231000000000002,"5min":111.072,"15min":143.31200000000001},"last":0.95599999999999996},{"interface":"front","average":{"1min":3.2879999999999998,"5min":5.6219999999999999,"15min":7.1369999999999996},"min":{"1min":0.36299999999999999,"5min":0.34499999999999997,"15min":0.29399999999999998},"max":{"1min":20.035,"5min":111.01300000000001,"15min":140.00899999999999},"last":3.0830000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:14:52 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":1.6279999999999999,"15min":1.278},"min":{"1min":0.28000000000000003,"5min":0.246,"15min":0.245},"max":{"1min":2.097,"5min":68.001999999999995,"15min":68.001999999999995},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.88300000000000001,"5min":1.5529999999999999,"15min":1.212},"min":{"1min":0.33900000000000002,"5min":0.26100000000000001,"15min":0.26000000000000001},"max":{"1min":5.7729999999999997,"5min":61.743000000000002,"15min":61.743000000000002},"last":0.42599999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:15:04 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.91000000000000003,"15min":1.0940000000000001},"min":{"1min":0.314,"5min":0.26200000000000001,"15min":0.251},"max":{"1min":4.492,"5min":7.4980000000000002,"15min":9.0909999999999993},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.014,"15min":1.1639999999999999},"min":{"1min":0.34599999999999997,"5min":0.29099999999999998,"15min":0.26700000000000002},"max":{"1min":3.0379999999999998,"5min":7.7140000000000004,"15min":9.9009999999999998},"last":0.59399999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:14:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3480000000000001,"5min":1.8380000000000001,"15min":2.456},"min":{"1min":0.36299999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.8289999999999997,"5min":64.766000000000005,"15min":157.471},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.85699999999999998,"5min":1.698,"15min":2.3410000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.3380000000000001,"5min":67.956999999999994,"15min":157.51599999999999},"last":1.202}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4889,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1338380288,"data_stored":1338376216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":344915968,"data_stored":344915968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":896139264,"data_stored":896139264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1090064384,"data_stored":1090060312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1589776384,"data_stored":1589772312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":983629824,"data_stored":983629824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":881725440,"data_stored":881721368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":505020416,"data_stored":505020416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:15:09.272 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T06:15:09.272 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:15:09.273 INFO:tasks.thrashosds.thrasher:Reweighting osd 2 to 0.5496555576698482 2023-10-09T06:15:09.273 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 0.5496555576698482 2023-10-09T06:15:09.289 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:15:09.322 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:09.391 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:15:09.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:09.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:15:09.498 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:15:09.538 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:15:09.538 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T06:15:09.538 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T06:15:09.538 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:15:09.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.538 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:81471 10.3 10:cc6caecd:::benchmark_data_smithi064_573750_object81470:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-10-09T06:15:09.539 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:12:59.854905+0000", 2023-10-09T06:15:09.539 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 129.66615894700001, 2023-10-09T06:15:09.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.214265604, 2023-10-09T06:15:09.539 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:09.539 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:09.539 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:09.539 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:09.539 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:09.539 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 81471 2023-10-09T06:15:09.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.539 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.854905+0000", 2023-10-09T06:15:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.854905+0000", 2023-10-09T06:15:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.540 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.854903+0000", 2023-10-09T06:15:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T06:15:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.854933+0000", 2023-10-09T06:15:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0031000000000001e-05 2023-10-09T06:15:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.854950+0000", 2023-10-09T06:15:09.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6651e-05 2023-10-09T06:15:09.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.542 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:09.542 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.854978+0000", 2023-10-09T06:15:09.542 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7837000000000001e-05 2023-10-09T06:15:09.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.542 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:09.542 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.859094+0000", 2023-10-09T06:15:09.542 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041166450000000004 2023-10-09T06:15:09.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.859292+0000", 2023-10-09T06:15:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019750099999999999 2023-10-09T06:15:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:15:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.859461+0000", 2023-10-09T06:15:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000168717 2023-10-09T06:15:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.543 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.862286+0000", 2023-10-09T06:15:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028254970000000002 2023-10-09T06:15:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.862633+0000", 2023-10-09T06:15:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00034674599999999999 2023-10-09T06:15:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.864600+0000", 2023-10-09T06:15:09.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019670030000000002 2023-10-09T06:15:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:00.068977+0000", 2023-10-09T06:15:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.204377264 2023-10-09T06:15:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:00.069046+0000", 2023-10-09T06:15:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8472000000000004e-05 2023-10-09T06:15:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.545 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:09.546 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:00.069171+0000", 2023-10-09T06:15:09.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012542099999999999 2023-10-09T06:15:09.546 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.546 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:09.546 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.546 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:83019 10.3 10:ce4365a3:::benchmark_data_smithi064_573750_object83018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e514)", 2023-10-09T06:15:09.546 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.816350+0000", 2023-10-09T06:15:09.546 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 127.70471475399999, 2023-10-09T06:15:09.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27937314000000002, 2023-10-09T06:15:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 83019 2023-10-09T06:15:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.816350+0000", 2023-10-09T06:15:09.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.816350+0000", 2023-10-09T06:15:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.816349+0000", 2023-10-09T06:15:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:15:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.548 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.816373+0000", 2023-10-09T06:15:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3547999999999999e-05 2023-10-09T06:15:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.816379+0000", 2023-10-09T06:15:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7440000000000001e-06 2023-10-09T06:15:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:09.549 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.816385+0000", 2023-10-09T06:15:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2959999999999998e-06 2023-10-09T06:15:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.820672+0000", 2023-10-09T06:15:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.004287155 2023-10-09T06:15:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.820810+0000", 2023-10-09T06:15:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000137787 2023-10-09T06:15:09.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:15:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.820957+0000", 2023-10-09T06:15:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014771599999999999 2023-10-09T06:15:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.827525+0000", 2023-10-09T06:15:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065673570000000002 2023-10-09T06:15:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.827554+0000", 2023-10-09T06:15:09.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9244e-05 2023-10-09T06:15:09.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832230+0000", 2023-10-09T06:15:09.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046755659999999999 2023-10-09T06:15:09.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:09.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832551+0000", 2023-10-09T06:15:09.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00032125600000000002 2023-10-09T06:15:09.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:09.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832599+0000", 2023-10-09T06:15:09.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8233999999999999e-05 2023-10-09T06:15:09.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:09.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095723+0000", 2023-10-09T06:15:09.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26312367800000003 2023-10-09T06:15:09.553 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.553 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:09.554 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.554 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:83027 10.3 10:ce4b8369:::benchmark_data_smithi064_573750_object83026:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e514)", 2023-10-09T06:15:09.554 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.823657+0000", 2023-10-09T06:15:09.554 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 127.69740701000001, 2023-10-09T06:15:09.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27225728399999999, 2023-10-09T06:15:09.554 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:09.554 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:09.554 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:09.554 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 83027 2023-10-09T06:15:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.823657+0000", 2023-10-09T06:15:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:09.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.823657+0000", 2023-10-09T06:15:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.823656+0000", 2023-10-09T06:15:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:15:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.823681+0000", 2023-10-09T06:15:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4947999999999999e-05 2023-10-09T06:15:09.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.823690+0000", 2023-10-09T06:15:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0059999999999998e-06 2023-10-09T06:15:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.823696+0000", 2023-10-09T06:15:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1600000000000003e-06 2023-10-09T06:15:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.557 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824488+0000", 2023-10-09T06:15:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00079232200000000003 2023-10-09T06:15:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824646+0000", 2023-10-09T06:15:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015738899999999999 2023-10-09T06:15:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:15:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824798+0000", 2023-10-09T06:15:09.558 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015239999999999999 2023-10-09T06:15:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832263+0000", 2023-10-09T06:15:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0074651350000000003 2023-10-09T06:15:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832298+0000", 2023-10-09T06:15:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4591999999999997e-05 2023-10-09T06:15:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832327+0000", 2023-10-09T06:15:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8731999999999999e-05 2023-10-09T06:15:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095736+0000", 2023-10-09T06:15:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26340953700000003 2023-10-09T06:15:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095794+0000", 2023-10-09T06:15:09.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7886000000000001e-05 2023-10-09T06:15:09.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.561 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.561 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:09.561 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095915+0000", 2023-10-09T06:15:09.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012050200000000001 2023-10-09T06:15:09.561 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.561 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:09.561 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.561 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.561 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:83026 10.7 e514/490 10:e8d46d43:::benchmark_data_smithi064_573750_object83025:head v 514'5204, mlcod=514'5202)", 2023-10-09T06:15:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.824237+0000", 2023-10-09T06:15:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 127.69682752200001, 2023-10-09T06:15:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27149754799999998, 2023-10-09T06:15:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824237+0000", 2023-10-09T06:15:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.562 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824237+0000", 2023-10-09T06:15:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824236+0000", 2023-10-09T06:15:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:15:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:09.563 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824266+0000", 2023-10-09T06:15:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9991000000000001e-05 2023-10-09T06:15:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824275+0000", 2023-10-09T06:15:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7860000000000002e-06 2023-10-09T06:15:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824281+0000", 2023-10-09T06:15:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9020000000000004e-06 2023-10-09T06:15:09.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.826984+0000", 2023-10-09T06:15:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027032750000000002 2023-10-09T06:15:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.827009+0000", 2023-10-09T06:15:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4987e-05 2023-10-09T06:15:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:09.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095693+0000", 2023-10-09T06:15:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.268683799 2023-10-09T06:15:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095734+0000", 2023-10-09T06:15:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1600000000000002e-05 2023-10-09T06:15:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.566 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:83030 10.7 e514/490 10:e508533f:::benchmark_data_smithi064_573750_object83029:head v 514'5205, mlcod=514'5202)", 2023-10-09T06:15:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.831823+0000", 2023-10-09T06:15:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 127.689241094, 2023-10-09T06:15:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26393689799999998, 2023-10-09T06:15:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.831823+0000", 2023-10-09T06:15:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.831823+0000", 2023-10-09T06:15:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.831822+0000", 2023-10-09T06:15:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:15:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:09.568 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.831849+0000", 2023-10-09T06:15:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7291e-05 2023-10-09T06:15:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.831862+0000", 2023-10-09T06:15:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2138e-05 2023-10-09T06:15:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.831870+0000", 2023-10-09T06:15:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4020000000000002e-06 2023-10-09T06:15:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.570 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:09.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.831944+0000", 2023-10-09T06:15:09.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3606e-05 2023-10-09T06:15:09.570 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.570 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.570 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:09.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.831966+0000", 2023-10-09T06:15:09.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2504999999999999e-05 2023-10-09T06:15:09.570 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.571 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.571 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:09.571 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095739+0000", 2023-10-09T06:15:09.571 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.263772805 2023-10-09T06:15:09.571 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.571 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.571 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:09.571 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095760+0000", 2023-10-09T06:15:09.571 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1154000000000001e-05 2023-10-09T06:15:09.571 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.571 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:09.571 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:83038 10.9 e514/490 10:99b1b726:::benchmark_data_smithi064_573750_object83037:head v 514'5168, mlcod=513'5166)", 2023-10-09T06:15:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.834529+0000", 2023-10-09T06:15:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 127.686534936, 2023-10-09T06:15:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26413685199999998, 2023-10-09T06:15:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:09.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834529+0000", 2023-10-09T06:15:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834529+0000", 2023-10-09T06:15:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834529+0000", 2023-10-09T06:15:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:15:09.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834561+0000", 2023-10-09T06:15:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2317999999999998e-05 2023-10-09T06:15:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095659+0000", 2023-10-09T06:15:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26109802799999998 2023-10-09T06:15:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.574 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095671+0000", 2023-10-09T06:15:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2126e-05 2023-10-09T06:15:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095734+0000", 2023-10-09T06:15:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3114000000000006e-05 2023-10-09T06:15:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095761+0000", 2023-10-09T06:15:09.575 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7072000000000001e-05 2023-10-09T06:15:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.098631+0000", 2023-10-09T06:15:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002870147 2023-10-09T06:15:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.098666+0000", 2023-10-09T06:15:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4882999999999998e-05 2023-10-09T06:15:09.576 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPushReply(10.3 514/510 [PushReplyOp(10:c32ef31e:::benchmark_data_smithi064_573750_object32521:head)])", 2023-10-09T06:15:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:05.610248+0000", 2023-10-09T06:15:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 123.91081656, 2023-10-09T06:15:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29548134799999998, 2023-10-09T06:15:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:15:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:09.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.610248+0000", 2023-10-09T06:15:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.610248+0000", 2023-10-09T06:15:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:09.578 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.610247+0000", 2023-10-09T06:15:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:15:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.610251+0000", 2023-10-09T06:15:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2650000000000001e-06 2023-10-09T06:15:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.610258+0000", 2023-10-09T06:15:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.695e-06 2023-10-09T06:15:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.610274+0000", 2023-10-09T06:15:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5656000000000001e-05 2023-10-09T06:15:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.610296+0000", 2023-10-09T06:15:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1787999999999999e-05 2023-10-09T06:15:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:09.580 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.905729+0000", 2023-10-09T06:15:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29543320099999998 2023-10-09T06:15:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:85642 10.f e514/490 10:f2b65d04:::benchmark_data_smithi064_573750_object85641:head v 514'5293, mlcod=514'5291)", 2023-10-09T06:15:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:05.624785+0000", 2023-10-09T06:15:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 123.896279389, 2023-10-09T06:15:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29269213900000002, 2023-10-09T06:15:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:09.581 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:09.582 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:09.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.582 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:09.582 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.624785+0000", 2023-10-09T06:15:09.582 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.582 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:09.582 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.624785+0000", 2023-10-09T06:15:09.582 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.624784+0000", 2023-10-09T06:15:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:15:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.624814+0000", 2023-10-09T06:15:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0809999999999998e-05 2023-10-09T06:15:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.905640+0000", 2023-10-09T06:15:09.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28082586199999998 2023-10-09T06:15:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.905655+0000", 2023-10-09T06:15:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5089000000000001e-05 2023-10-09T06:15:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.906522+0000", 2023-10-09T06:15:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00086632699999999996 2023-10-09T06:15:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.584 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.906542+0000", 2023-10-09T06:15:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0621000000000001e-05 2023-10-09T06:15:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.917427+0000", 2023-10-09T06:15:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010885212 2023-10-09T06:15:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:09.585 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.917477+0000", 2023-10-09T06:15:09.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9546999999999998e-05 2023-10-09T06:15:09.586 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.586 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:09.586 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.586 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.586 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.586 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:86106 10.3 10:c9bcf3d5:::benchmark_data_smithi064_573750_object86105:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e515)", 2023-10-09T06:15:09.586 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:06.327720+0000", 2023-10-09T06:15:09.586 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 123.193344381, 2023-10-09T06:15:09.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18239918699999999, 2023-10-09T06:15:09.586 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 86106 2023-10-09T06:15:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.327720+0000", 2023-10-09T06:15:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.327720+0000", 2023-10-09T06:15:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.327718+0000", 2023-10-09T06:15:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:15:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.327749+0000", 2023-10-09T06:15:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0487000000000001e-05 2023-10-09T06:15:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.327763+0000", 2023-10-09T06:15:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4093e-05 2023-10-09T06:15:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.327772+0000", 2023-10-09T06:15:09.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1929999999999999e-06 2023-10-09T06:15:09.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.590 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:09.590 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.327970+0000", 2023-10-09T06:15:09.590 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019824600000000001 2023-10-09T06:15:09.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.590 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:09.590 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.328115+0000", 2023-10-09T06:15:09.590 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000144511 2023-10-09T06:15:09.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:15:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.328262+0000", 2023-10-09T06:15:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014681199999999999 2023-10-09T06:15:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.332137+0000", 2023-10-09T06:15:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038754219999999999 2023-10-09T06:15:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.591 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.332173+0000", 2023-10-09T06:15:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5808e-05 2023-10-09T06:15:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.334060+0000", 2023-10-09T06:15:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018872299999999999 2023-10-09T06:15:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.509949+0000", 2023-10-09T06:15:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.17588947499999999 2023-10-09T06:15:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.592 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.510005+0000", 2023-10-09T06:15:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5696999999999999e-05 2023-10-09T06:15:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.510119+0000", 2023-10-09T06:15:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000113927 2023-10-09T06:15:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.593 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:86114 10.3 10:ce2d7be7:::benchmark_data_smithi064_573750_object86113:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e515)", 2023-10-09T06:15:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:06.335134+0000", 2023-10-09T06:15:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 123.185929832, 2023-10-09T06:15:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18034898099999999, 2023-10-09T06:15:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 86114 2023-10-09T06:15:09.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.335134+0000", 2023-10-09T06:15:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.335134+0000", 2023-10-09T06:15:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.595 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.335133+0000", 2023-10-09T06:15:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:15:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.335155+0000", 2023-10-09T06:15:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1685e-05 2023-10-09T06:15:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.335164+0000", 2023-10-09T06:15:09.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5359999999999998e-06 2023-10-09T06:15:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.335170+0000", 2023-10-09T06:15:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9390000000000002e-06 2023-10-09T06:15:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.335383+0000", 2023-10-09T06:15:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00021336900000000001 2023-10-09T06:15:09.597 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.506890+0000", 2023-10-09T06:15:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.17150653399999999 2023-10-09T06:15:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:15:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.507101+0000", 2023-10-09T06:15:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00021099199999999999 2023-10-09T06:15:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.598 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.511984+0000", 2023-10-09T06:15:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048832479999999998 2023-10-09T06:15:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.512022+0000", 2023-10-09T06:15:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7540000000000003e-05 2023-10-09T06:15:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.512315+0000", 2023-10-09T06:15:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000293481 2023-10-09T06:15:09.599 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.515299+0000", 2023-10-09T06:15:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029837219999999999 2023-10-09T06:15:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.515362+0000", 2023-10-09T06:15:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2644999999999994e-05 2023-10-09T06:15:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.600 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.515483+0000", 2023-10-09T06:15:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012182500000000001 2023-10-09T06:15:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:92993 10.3 10:cbd0b40f:::benchmark_data_smithi064_573750_object92992:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-10-09T06:15:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:17.279671+0000", 2023-10-09T06:15:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 112.241392795, 2023-10-09T06:15:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20967518099999999, 2023-10-09T06:15:09.601 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 92993 2023-10-09T06:15:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.279671+0000", 2023-10-09T06:15:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.602 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.279671+0000", 2023-10-09T06:15:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.279670+0000", 2023-10-09T06:15:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:15:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.603 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.279696+0000", 2023-10-09T06:15:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6265999999999999e-05 2023-10-09T06:15:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.279708+0000", 2023-10-09T06:15:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1521e-05 2023-10-09T06:15:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.279716+0000", 2023-10-09T06:15:09.604 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0399999999999993e-06 2023-10-09T06:15:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.280201+0000", 2023-10-09T06:15:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00048531699999999997 2023-10-09T06:15:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.280360+0000", 2023-10-09T06:15:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015919400000000001 2023-10-09T06:15:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:15:09.606 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.280515+0000", 2023-10-09T06:15:09.606 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015432400000000001 2023-10-09T06:15:09.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.606 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.287237+0000", 2023-10-09T06:15:09.606 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067219150000000002 2023-10-09T06:15:09.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.287420+0000", 2023-10-09T06:15:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018320200000000001 2023-10-09T06:15:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.289348+0000", 2023-10-09T06:15:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001928206 2023-10-09T06:15:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.489167+0000", 2023-10-09T06:15:09.607 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.19981912800000001 2023-10-09T06:15:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.489228+0000", 2023-10-09T06:15:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0606999999999998e-05 2023-10-09T06:15:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.489347+0000", 2023-10-09T06:15:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118967 2023-10-09T06:15:09.608 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:93019 10.3 10:c9254536:::benchmark_data_smithi064_573750_object93018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-10-09T06:15:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:17.310056+0000", 2023-10-09T06:15:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 112.21100866099999, 2023-10-09T06:15:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.183025682, 2023-10-09T06:15:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:09.609 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:09.610 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:09.610 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 93019 2023-10-09T06:15:09.610 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.610 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:09.610 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.610 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:09.610 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310056+0000", 2023-10-09T06:15:09.610 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.610 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.610 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.610 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:09.611 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310056+0000", 2023-10-09T06:15:09.611 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.611 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:09.611 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310054+0000", 2023-10-09T06:15:09.611 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:15:09.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.611 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:09.611 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310075+0000", 2023-10-09T06:15:09.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0990999999999999e-05 2023-10-09T06:15:09.612 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:09.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310090+0000", 2023-10-09T06:15:09.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4449e-05 2023-10-09T06:15:09.612 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:09.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310112+0000", 2023-10-09T06:15:09.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2952999999999999e-05 2023-10-09T06:15:09.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:09.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310155+0000", 2023-10-09T06:15:09.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2534000000000003e-05 2023-10-09T06:15:09.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:09.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310286+0000", 2023-10-09T06:15:09.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013121699999999999 2023-10-09T06:15:09.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:15:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310404+0000", 2023-10-09T06:15:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011796899999999999 2023-10-09T06:15:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.312787+0000", 2023-10-09T06:15:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0023830919999999998 2023-10-09T06:15:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.313853+0000", 2023-10-09T06:15:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001066132 2023-10-09T06:15:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.313932+0000", 2023-10-09T06:15:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8236000000000001e-05 2023-10-09T06:15:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.492876+0000", 2023-10-09T06:15:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.17894394499999999 2023-10-09T06:15:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.492941+0000", 2023-10-09T06:15:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5642999999999995e-05 2023-10-09T06:15:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.493081+0000", 2023-10-09T06:15:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014008799999999999 2023-10-09T06:15:09.616 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:94743 10.3 10:c7ba2f96:::benchmark_data_smithi064_573750_object94742:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-10-09T06:15:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:19.875917+0000", 2023-10-09T06:15:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 109.645147134, 2023-10-09T06:15:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.22279960200000001, 2023-10-09T06:15:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:09.617 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 94743 2023-10-09T06:15:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.875917+0000", 2023-10-09T06:15:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.618 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.875917+0000", 2023-10-09T06:15:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.875916+0000", 2023-10-09T06:15:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:15:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.875940+0000", 2023-10-09T06:15:09.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4561000000000001e-05 2023-10-09T06:15:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.875953+0000", 2023-10-09T06:15:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3276e-05 2023-10-09T06:15:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.875962+0000", 2023-10-09T06:15:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1340000000000004e-06 2023-10-09T06:15:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.620 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.876240+0000", 2023-10-09T06:15:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00027892999999999999 2023-10-09T06:15:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.876436+0000", 2023-10-09T06:15:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000195317 2023-10-09T06:15:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:15:09.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.876653+0000", 2023-10-09T06:15:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000216918 2023-10-09T06:15:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.881334+0000", 2023-10-09T06:15:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046809870000000002 2023-10-09T06:15:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.881349+0000", 2023-10-09T06:15:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5311999999999998e-05 2023-10-09T06:15:09.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.884746+0000", 2023-10-09T06:15:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033974000000000001 2023-10-09T06:15:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.098510+0000", 2023-10-09T06:15:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.213763589 2023-10-09T06:15:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.623 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.098578+0000", 2023-10-09T06:15:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8112000000000001e-05 2023-10-09T06:15:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.098717+0000", 2023-10-09T06:15:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013864199999999999 2023-10-09T06:15:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.624 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:95775 10.3 10:c599e099:::benchmark_data_smithi064_573750_object95774:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-10-09T06:15:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:23.554330+0000", 2023-10-09T06:15:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 105.966734708, 2023-10-09T06:15:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26772326899999999, 2023-10-09T06:15:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 95775 2023-10-09T06:15:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.625 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:09.626 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.626 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:09.626 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.554330+0000", 2023-10-09T06:15:09.626 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.626 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.626 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.626 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:09.626 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.554330+0000", 2023-10-09T06:15:09.626 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.626 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.626 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.554328+0000", 2023-10-09T06:15:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:15:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.554352+0000", 2023-10-09T06:15:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4216999999999999e-05 2023-10-09T06:15:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:09.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.554365+0000", 2023-10-09T06:15:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2947e-05 2023-10-09T06:15:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.554373+0000", 2023-10-09T06:15:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2120000000000003e-06 2023-10-09T06:15:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.557893+0000", 2023-10-09T06:15:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035198059999999999 2023-10-09T06:15:09.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.558059+0000", 2023-10-09T06:15:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000165971 2023-10-09T06:15:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:15:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.558259+0000", 2023-10-09T06:15:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020019300000000001 2023-10-09T06:15:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.561632+0000", 2023-10-09T06:15:09.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033720709999999999 2023-10-09T06:15:09.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:09.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.561647+0000", 2023-10-09T06:15:09.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5965e-05 2023-10-09T06:15:09.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821847+0000", 2023-10-09T06:15:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26019937599999998 2023-10-09T06:15:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821877+0000", 2023-10-09T06:15:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9989e-05 2023-10-09T06:15:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821916+0000", 2023-10-09T06:15:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8781e-05 2023-10-09T06:15:09.631 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.822053+0000", 2023-10-09T06:15:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013723500000000001 2023-10-09T06:15:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPushReply(10.3 523/510 [PushReplyOp(10:cc80a1b0:::benchmark_data_smithi064_573750_object75768:head)])", 2023-10-09T06:15:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:23.562483+0000", 2023-10-09T06:15:09.632 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 105.958581708, 2023-10-09T06:15:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25932817600000002, 2023-10-09T06:15:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:15:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562483+0000", 2023-10-09T06:15:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:09.633 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562483+0000", 2023-10-09T06:15:09.634 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.634 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.634 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:09.634 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562482+0000", 2023-10-09T06:15:09.634 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:15:09.634 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.634 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:09.634 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562488+0000", 2023-10-09T06:15:09.634 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6699999999999997e-06 2023-10-09T06:15:09.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.635 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:09.635 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562501+0000", 2023-10-09T06:15:09.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3005e-05 2023-10-09T06:15:09.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.635 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:09.635 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562511+0000", 2023-10-09T06:15:09.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6679999999999996e-06 2023-10-09T06:15:09.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.636 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:09.636 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562563+0000", 2023-10-09T06:15:09.636 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1659000000000001e-05 2023-10-09T06:15:09.636 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.636 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:09.636 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821811+0000", 2023-10-09T06:15:09.636 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25924797700000002 2023-10-09T06:15:09.636 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.636 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:09.636 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.636 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.637 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.46283.0:95775 10.3 e523/510 ondisk, result = 0)", 2023-10-09T06:15:09.637 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:23.562660+0000", 2023-10-09T06:15:09.637 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 105.95840400199999, 2023-10-09T06:15:09.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25919107400000002, 2023-10-09T06:15:09.637 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:09.637 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:15:09.637 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:09.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.637 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:09.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562660+0000", 2023-10-09T06:15:09.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.638 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.638 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:09.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562660+0000", 2023-10-09T06:15:09.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.638 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.638 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:09.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562660+0000", 2023-10-09T06:15:09.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:15:09.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.639 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:09.639 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562665+0000", 2023-10-09T06:15:09.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.682e-06 2023-10-09T06:15:09.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.639 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:09.639 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562675+0000", 2023-10-09T06:15:09.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7510000000000003e-06 2023-10-09T06:15:09.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.640 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:09.640 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562682+0000", 2023-10-09T06:15:09.640 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.391e-06 2023-10-09T06:15:09.640 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.640 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:09.640 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821834+0000", 2023-10-09T06:15:09.640 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25915209099999997 2023-10-09T06:15:09.640 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.640 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:09.640 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821842+0000", 2023-10-09T06:15:09.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4719999999999998e-06 2023-10-09T06:15:09.641 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.641 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.641 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:09.641 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821851+0000", 2023-10-09T06:15:09.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4269999999999997e-06 2023-10-09T06:15:09.641 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.641 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:09.641 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.641 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.641 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.641 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.46283.0:95775 10.3 e523/510 ondisk, result = 0)", 2023-10-09T06:15:09.642 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:23.563416+0000", 2023-10-09T06:15:09.642 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 105.95764808200001, 2023-10-09T06:15:09.642 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25864464399999998, 2023-10-09T06:15:09.642 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:09.642 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:15:09.642 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:09.642 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.642 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:09.642 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.563416+0000", 2023-10-09T06:15:09.642 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.642 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:09.643 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.563416+0000", 2023-10-09T06:15:09.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:09.643 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.563416+0000", 2023-10-09T06:15:09.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:15:09.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:09.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.563421+0000", 2023-10-09T06:15:09.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1250000000000001e-06 2023-10-09T06:15:09.644 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:09.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821790+0000", 2023-10-09T06:15:09.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25836899800000002 2023-10-09T06:15:09.644 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:09.645 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821803+0000", 2023-10-09T06:15:09.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.326e-05 2023-10-09T06:15:09.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:09.645 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821867+0000", 2023-10-09T06:15:09.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3613000000000002e-05 2023-10-09T06:15:09.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:09.645 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821873+0000", 2023-10-09T06:15:09.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2090000000000002e-06 2023-10-09T06:15:09.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.646 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:09.646 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.822061+0000", 2023-10-09T06:15:09.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018808799999999999 2023-10-09T06:15:09.646 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.646 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:09.646 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.646 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:117026 10.f e536/490 10:f52e132e:::benchmark_data_smithi064_573750_object20811:head v 536'7258, mlcod=536'7256)", 2023-10-09T06:15:09.646 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:57.908348+0000", 2023-10-09T06:15:09.647 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 71.612716754000004, 2023-10-09T06:15:09.647 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.213648951, 2023-10-09T06:15:09.647 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:09.647 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:09.647 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:09.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:09.647 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908348+0000", 2023-10-09T06:15:09.647 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.647 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:09.647 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908348+0000", 2023-10-09T06:15:09.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.648 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.648 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.648 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:09.648 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908347+0000", 2023-10-09T06:15:09.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:15:09.648 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.648 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.648 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:09.648 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908353+0000", 2023-10-09T06:15:09.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7190000000000004e-06 2023-10-09T06:15:09.648 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.649 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.649 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:09.649 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908365+0000", 2023-10-09T06:15:09.649 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1433999999999999e-05 2023-10-09T06:15:09.649 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.649 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.649 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:09.649 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908372+0000", 2023-10-09T06:15:09.649 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4540000000000001e-06 2023-10-09T06:15:09.649 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.649 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.649 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:09.650 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908494+0000", 2023-10-09T06:15:09.650 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000121434 2023-10-09T06:15:09.650 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.650 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.650 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:09.650 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908513+0000", 2023-10-09T06:15:09.650 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9556999999999999e-05 2023-10-09T06:15:09.650 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.650 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.650 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:09.650 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.121931+0000", 2023-10-09T06:15:09.651 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21341768699999999 2023-10-09T06:15:09.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.651 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.651 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:09.651 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.121996+0000", 2023-10-09T06:15:09.651 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5518000000000005e-05 2023-10-09T06:15:09.651 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.651 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:09.651 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.652 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:117014 10.5 e536/490 10:a26da6a0:::benchmark_data_smithi064_573750_object20799:head v 536'7222, mlcod=536'7220)", 2023-10-09T06:15:09.652 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:57.908415+0000", 2023-10-09T06:15:09.652 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 71.612649374, 2023-10-09T06:15:09.652 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21368757199999999, 2023-10-09T06:15:09.652 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:09.652 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:09.652 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:09.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.652 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:09.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908415+0000", 2023-10-09T06:15:09.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.653 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.653 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:09.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908415+0000", 2023-10-09T06:15:09.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.653 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.653 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:09.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908415+0000", 2023-10-09T06:15:09.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:15:09.654 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.654 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.654 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:09.654 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908419+0000", 2023-10-09T06:15:09.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6480000000000002e-06 2023-10-09T06:15:09.654 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.654 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.654 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:09.654 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908427+0000", 2023-10-09T06:15:09.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6469999999999998e-06 2023-10-09T06:15:09.655 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.655 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.655 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:09.655 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908432+0000", 2023-10-09T06:15:09.655 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0239999999999997e-06 2023-10-09T06:15:09.655 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.655 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.655 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:09.655 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908476+0000", 2023-10-09T06:15:09.655 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4288e-05 2023-10-09T06:15:09.655 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.656 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:09.656 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908497+0000", 2023-10-09T06:15:09.656 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1121999999999999e-05 2023-10-09T06:15:09.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.656 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:09.656 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.122071+0000", 2023-10-09T06:15:09.656 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21357372499999999 2023-10-09T06:15:09.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:09.657 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.122102+0000", 2023-10-09T06:15:09.657 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1529000000000003e-05 2023-10-09T06:15:09.657 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.657 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:09.657 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.657 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.657 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.657 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:117020 10.5 e536/490 10:a87652b9:::benchmark_data_smithi064_573750_object20805:head v 536'7223, mlcod=536'7220)", 2023-10-09T06:15:09.657 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:57.910671+0000", 2023-10-09T06:15:09.657 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 71.610393646000006, 2023-10-09T06:15:09.657 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.22155350400000001, 2023-10-09T06:15:09.658 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:09.658 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:09.658 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:09.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.658 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:09.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.910671+0000", 2023-10-09T06:15:09.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.658 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:09.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.910671+0000", 2023-10-09T06:15:09.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:09.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:09.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.910670+0000", 2023-10-09T06:15:09.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:15:09.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:09.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.910676+0000", 2023-10-09T06:15:09.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9460000000000003e-06 2023-10-09T06:15:09.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.660 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:09.660 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.121961+0000", 2023-10-09T06:15:09.660 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21128570499999999 2023-10-09T06:15:09.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.660 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:09.660 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.121974+0000", 2023-10-09T06:15:09.660 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2244e-05 2023-10-09T06:15:09.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.660 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:09.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.122133+0000", 2023-10-09T06:15:09.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015961999999999999 2023-10-09T06:15:09.661 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.661 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.661 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:09.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.122152+0000", 2023-10-09T06:15:09.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9131e-05 2023-10-09T06:15:09.661 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.661 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.661 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:09.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.132185+0000", 2023-10-09T06:15:09.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010032961 2023-10-09T06:15:09.662 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:09.662 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:09.662 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:09.662 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.132224+0000", 2023-10-09T06:15:09.662 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8819999999999998e-05 2023-10-09T06:15:09.662 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.662 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:09.662 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.662 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:09.662 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:09.662 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:15:09.663 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:09.663 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:09.790 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:15:09.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:09.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:15:09.901 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:15:09.958 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:09 smithi064 ceph-mon[220044]: from='client.36465 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:15:09.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:09 smithi064 ceph-mon[212750]: from='client.36465 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:15:09.977 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:10.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:09 smithi156 ceph-mon[195101]: from='client.36465 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:15:10.470 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:15:10.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:10.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:15:10.578 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:15:10.604 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:15:10.694 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:10.850 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:15:10.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:10.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:15:10.962 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:15:11.008 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:10 smithi064 ceph-mon[220044]: pgmap v1414: 119 pgs: 119 active+clean; 1.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 1.8 MiB/s, 29 objects/s recovering 2023-10-09T06:15:11.008 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:10 smithi064 ceph-mon[212750]: pgmap v1414: 119 pgs: 119 active+clean; 1.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 1.8 MiB/s, 29 objects/s recovering 2023-10-09T06:15:11.014 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:10 smithi156 ceph-mon[195101]: pgmap v1414: 119 pgs: 119 active+clean; 1.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 1.8 MiB/s, 29 objects/s recovering 2023-10-09T06:15:11.019 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:11.024 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:11.085 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:15:11.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:11.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:15:11.198 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:15:11.199 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:15:11.283 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:11.726 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:15:11.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:11.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:15:11.834 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:15:11.878 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:11.945 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:15:11.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:11.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:15:12.052 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:15:12.087 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:12.156 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:15:12.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:12.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:15:12.246 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:12.263 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:15:12.360 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:12.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:12 smithi064 ceph-mon[212750]: pgmap v1415: 119 pgs: 119 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 1.5 MiB/s, 23 objects/s recovering 2023-10-09T06:15:12.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:12 smithi064 ceph-mon[220044]: pgmap v1415: 119 pgs: 119 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 1.5 MiB/s, 23 objects/s recovering 2023-10-09T06:15:12.984 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:15:12.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:12.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:15:13.090 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:15:13.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:12 smithi156 ceph-mon[195101]: pgmap v1415: 119 pgs: 119 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 1.5 MiB/s, 23 objects/s recovering 2023-10-09T06:15:13.145 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:13.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:13.461 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:15:13.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:15:13.494 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:13.562 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:15:13.635 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:13.790 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:15:13.894 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:13 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:13.894 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:13 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2327336198' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:15:13.894 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:13 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:13.895 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:13 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2327336198' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:15:14.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:13 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:14.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:13 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2327336198' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:15:14.975 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:15:14.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:14.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:15:15.094 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:15:15.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:14 smithi156 ceph-mon[195101]: pgmap v1416: 119 pgs: 119 active+clean; 1.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:15:15.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:14 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:15:15.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:14 smithi156 ceph-mon[195101]: failed to encode map e557 with expected crc 2023-10-09T06:15:15.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:14 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2327336198' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:15:15.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:14 smithi156 ceph-mon[195101]: osdmap e557: 8 total, 8 up, 8 in 2023-10-09T06:15:15.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:14 smithi156 ceph-mon[195101]: failed to encode map e557 with expected crc 2023-10-09T06:15:15.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:14 smithi156 ceph-mon[195101]: failed to encode map e557 with expected crc 2023-10-09T06:15:15.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:15.135 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:15.189 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:15:15.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:15.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:15:15.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:14 smithi064 ceph-mon[212750]: pgmap v1416: 119 pgs: 119 active+clean; 1.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:15:15.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:14 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:15:15.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:14 smithi064 ceph-mon[212750]: failed to encode map e557 with expected crc 2023-10-09T06:15:15.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:14 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2327336198' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:15:15.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:14 smithi064 ceph-mon[212750]: osdmap e557: 8 total, 8 up, 8 in 2023-10-09T06:15:15.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:14 smithi064 ceph-mon[212750]: failed to encode map e557 with expected crc 2023-10-09T06:15:15.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:14 smithi064 ceph-mon[212750]: failed to encode map e557 with expected crc 2023-10-09T06:15:15.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:15.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:14 smithi064 ceph-mon[220044]: pgmap v1416: 119 pgs: 119 active+clean; 1.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:15:15.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:14 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:15:15.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:14 smithi064 ceph-mon[220044]: failed to encode map e557 with expected crc 2023-10-09T06:15:15.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:14 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2327336198' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:15:15.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:14 smithi064 ceph-mon[220044]: osdmap e557: 8 total, 8 up, 8 in 2023-10-09T06:15:15.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:14 smithi064 ceph-mon[220044]: failed to encode map e557 with expected crc 2023-10-09T06:15:15.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:14 smithi064 ceph-mon[220044]: failed to encode map e557 with expected crc 2023-10-09T06:15:15.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:15.296 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:15:15.352 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:15.442 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:15:15.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:15.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:15:15.555 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:15:15.596 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:15.662 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:15:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:15:15] "GET /metrics HTTP/1.1" 200 36463 "" "Prometheus/2.43.0" 2023-10-09T06:15:15.673 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:15:15.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:15.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:15:15.784 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:15:15.803 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:15 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1850466273' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.5496555576698482}]: dispatch 2023-10-09T06:15:15.803 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:15 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.5496555576698482}]: dispatch 2023-10-09T06:15:15.831 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.2 to 0.549656 (8cb6) 2023-10-09T06:15:15.838 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:15.893 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:15:15.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:15.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:15:15.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:15 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1850466273' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.5496555576698482}]: dispatch 2023-10-09T06:15:15.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:15 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.5496555576698482}]: dispatch 2023-10-09T06:15:15.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:15 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1850466273' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.5496555576698482}]: dispatch 2023-10-09T06:15:15.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:15 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.5496555576698482}]: dispatch 2023-10-09T06:15:16.001 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:15:16.028 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:16.069 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:15:16.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:16.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:15:16.176 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:15:16.202 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:16.244 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:15:16.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:16.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:15:16.351 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:15:16.377 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:16.418 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:15:16.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:16.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:15:16.523 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:15:16.550 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:16.592 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:15:16.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:16.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:15:16.699 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:15:16.725 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:16.766 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:15:16.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:16.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:15:16.871 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:15:16.897 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:16.938 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:15:16.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:16.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:15:17.043 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:15:17.082 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:16 smithi064 ceph-mon[212750]: pgmap v1418: 119 pgs: 119 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 1.2 MiB/s, 19 objects/s recovering 2023-10-09T06:15:17.082 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:16 smithi064 ceph-mon[212750]: failed to encode map e558 with expected crc 2023-10-09T06:15:17.082 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:16 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.5496555576698482}]': finished 2023-10-09T06:15:17.082 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:16 smithi064 ceph-mon[212750]: osdmap e558: 8 total, 8 up, 8 in 2023-10-09T06:15:17.083 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:16 smithi064 ceph-mon[212750]: failed to encode map e558 with expected crc 2023-10-09T06:15:17.083 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:16 smithi064 ceph-mon[212750]: failed to encode map e558 with expected crc 2023-10-09T06:15:17.083 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:16 smithi064 ceph-mon[212750]: failed to encode map e558 with expected crc 2023-10-09T06:15:17.083 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:16 smithi064 ceph-mon[220044]: pgmap v1418: 119 pgs: 119 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 1.2 MiB/s, 19 objects/s recovering 2023-10-09T06:15:17.083 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:16 smithi064 ceph-mon[220044]: failed to encode map e558 with expected crc 2023-10-09T06:15:17.083 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:16 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.5496555576698482}]': finished 2023-10-09T06:15:17.083 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:16 smithi064 ceph-mon[220044]: osdmap e558: 8 total, 8 up, 8 in 2023-10-09T06:15:17.083 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:16 smithi064 ceph-mon[220044]: failed to encode map e558 with expected crc 2023-10-09T06:15:17.083 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:16 smithi064 ceph-mon[220044]: failed to encode map e558 with expected crc 2023-10-09T06:15:17.084 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:16 smithi064 ceph-mon[220044]: failed to encode map e558 with expected crc 2023-10-09T06:15:17.086 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:17.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:16 smithi156 ceph-mon[195101]: pgmap v1418: 119 pgs: 119 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 1.2 MiB/s, 19 objects/s recovering 2023-10-09T06:15:17.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:16 smithi156 ceph-mon[195101]: failed to encode map e558 with expected crc 2023-10-09T06:15:17.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:16 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.5496555576698482}]': finished 2023-10-09T06:15:17.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:16 smithi156 ceph-mon[195101]: osdmap e558: 8 total, 8 up, 8 in 2023-10-09T06:15:17.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:16 smithi156 ceph-mon[195101]: failed to encode map e558 with expected crc 2023-10-09T06:15:17.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:16 smithi156 ceph-mon[195101]: failed to encode map e558 with expected crc 2023-10-09T06:15:17.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:16 smithi156 ceph-mon[195101]: failed to encode map e558 with expected crc 2023-10-09T06:15:17.190 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:15:17.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:17.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:15:17.294 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:15:17.323 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:17.425 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:15:17.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:17.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:15:17.530 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:15:17.561 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:17.612 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:15:17.612 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:15:17.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.612 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:171685 10.3 10.65494ad3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:15:17.612 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:15:15.816204+0000", 2023-10-09T06:15:17.613 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 1.795029744, 2023-10-09T06:15:17.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7950568599999999, 2023-10-09T06:15:17.613 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:17.613 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "delayed", 2023-10-09T06:15:17.613 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:17.613 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:17.613 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:17.613 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171685 2023-10-09T06:15:17.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.613 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:17.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:17.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816204+0000", 2023-10-09T06:15:17.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:17.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:17.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816204+0000", 2023-10-09T06:15:17.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:17.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:17.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816203+0000", 2023-10-09T06:15:17.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:15:17.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.615 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:17.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816207+0000", 2023-10-09T06:15:17.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3000000000000003e-06 2023-10-09T06:15:17.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.615 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:17.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816220+0000", 2023-10-09T06:15:17.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3208e-05 2023-10-09T06:15:17.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:17.616 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816228+0000", 2023-10-09T06:15:17.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6869999999999994e-06 2023-10-09T06:15:17.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:17.616 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816276+0000", 2023-10-09T06:15:17.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7632000000000001e-05 2023-10-09T06:15:17.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.617 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.617 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:17.617 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816287+0000", 2023-10-09T06:15:17.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0908e-05 2023-10-09T06:15:17.617 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.617 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.617 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:17.617 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794918+0000", 2023-10-09T06:15:17.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97863145399999996 2023-10-09T06:15:17.617 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.617 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.618 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:17.618 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794928+0000", 2023-10-09T06:15:17.618 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.005e-05 2023-10-09T06:15:17.618 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:17.618 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:17.618 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:17.618 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.618 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.618 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:171712 10.3 10.bfde07d3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:15:17.618 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:15:15.859776+0000", 2023-10-09T06:15:17.618 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 1.7514568640000001, 2023-10-09T06:15:17.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7515426459999999, 2023-10-09T06:15:17.619 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:17.619 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "delayed", 2023-10-09T06:15:17.619 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:17.619 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:17.619 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:17.619 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171712 2023-10-09T06:15:17.619 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.619 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:17.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.619 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:17.619 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.859776+0000", 2023-10-09T06:15:17.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:17.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.620 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:17.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.859776+0000", 2023-10-09T06:15:17.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:17.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.620 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:17.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.859775+0000", 2023-10-09T06:15:17.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:15:17.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:17.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.859781+0000", 2023-10-09T06:15:17.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.733e-06 2023-10-09T06:15:17.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:17.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.859794+0000", 2023-10-09T06:15:17.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3067999999999999e-05 2023-10-09T06:15:17.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:17.622 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.859803+0000", 2023-10-09T06:15:17.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5690000000000005e-06 2023-10-09T06:15:17.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:17.622 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.859863+0000", 2023-10-09T06:15:17.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0359999999999998e-05 2023-10-09T06:15:17.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:17.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.859876+0000", 2023-10-09T06:15:17.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2473999999999999e-05 2023-10-09T06:15:17.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.623 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:17.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.795237+0000", 2023-10-09T06:15:17.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93536119399999995 2023-10-09T06:15:17.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.623 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:17.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.795247+0000", 2023-10-09T06:15:17.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0268999999999999e-05 2023-10-09T06:15:17.624 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:17.624 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:17.624 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:17.624 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.624 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.624 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:171672 10.3 10:c82752d7:::benchmark_data_smithi064_573750_object75457:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e557)", 2023-10-09T06:15:17.624 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:15:15.799808+0000", 2023-10-09T06:15:17.624 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 1.8114256439999998, 2023-10-09T06:15:17.624 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.81156315, 2023-10-09T06:15:17.624 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:17.624 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "delayed", 2023-10-09T06:15:17.624 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:17.625 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:17.625 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:17.625 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171672 2023-10-09T06:15:17.625 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.625 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:17.625 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.625 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:17.625 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799808+0000", 2023-10-09T06:15:17.625 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:17.625 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.625 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.626 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:17.626 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799808+0000", 2023-10-09T06:15:17.626 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:17.626 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.626 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.626 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:17.626 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799805+0000", 2023-10-09T06:15:17.626 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:15:17.626 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.626 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.626 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:17.626 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799813+0000", 2023-10-09T06:15:17.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9149999999999996e-06 2023-10-09T06:15:17.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:17.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799826+0000", 2023-10-09T06:15:17.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2824e-05 2023-10-09T06:15:17.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:17.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799834+0000", 2023-10-09T06:15:17.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8990000000000001e-06 2023-10-09T06:15:17.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.628 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:17.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.800015+0000", 2023-10-09T06:15:17.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018057300000000001 2023-10-09T06:15:17.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.628 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:17.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.800155+0000", 2023-10-09T06:15:17.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014034099999999999 2023-10-09T06:15:17.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:15:17.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.800283+0000", 2023-10-09T06:15:17.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000127945 2023-10-09T06:15:17.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:17.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.808806+0000", 2023-10-09T06:15:17.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085225639999999998 2023-10-09T06:15:17.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:17.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812281+0000", 2023-10-09T06:15:17.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003475653 2023-10-09T06:15:17.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:17.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812302+0000", 2023-10-09T06:15:17.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0441e-05 2023-10-09T06:15:17.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:17.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794653+0000", 2023-10-09T06:15:17.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98235096899999996 2023-10-09T06:15:17.631 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.631 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.631 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:17.631 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794672+0000", 2023-10-09T06:15:17.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9063e-05 2023-10-09T06:15:17.631 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:17.631 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:17.631 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:17.631 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.631 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.631 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:171676 10.3 10:c58e27ea:::benchmark_data_smithi064_573750_object75461:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e557)", 2023-10-09T06:15:17.632 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:15:15.799859+0000", 2023-10-09T06:15:17.632 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 1.8113741669999999, 2023-10-09T06:15:17.632 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8115740869999999, 2023-10-09T06:15:17.632 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:17.632 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "delayed", 2023-10-09T06:15:17.632 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:17.632 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:17.632 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:17.632 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171676 2023-10-09T06:15:17.632 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.632 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:17.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.633 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:17.633 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799859+0000", 2023-10-09T06:15:17.633 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:17.633 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.633 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.633 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:17.633 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799859+0000", 2023-10-09T06:15:17.633 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:17.633 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.633 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.633 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:17.634 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799859+0000", 2023-10-09T06:15:17.634 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:15:17.634 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.634 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:17.634 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799862+0000", 2023-10-09T06:15:17.634 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0750000000000002e-06 2023-10-09T06:15:17.634 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.634 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:17.634 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799875+0000", 2023-10-09T06:15:17.634 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.31e-05 2023-10-09T06:15:17.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.635 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:17.635 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799881+0000", 2023-10-09T06:15:17.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.801e-06 2023-10-09T06:15:17.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.635 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:17.635 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.803918+0000", 2023-10-09T06:15:17.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040373400000000004 2023-10-09T06:15:17.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.636 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:17.636 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.804065+0000", 2023-10-09T06:15:17.636 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014711600000000001 2023-10-09T06:15:17.636 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.636 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:15:17.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.804246+0000", 2023-10-09T06:15:17.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018100599999999999 2023-10-09T06:15:17.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.637 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:17.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812631+0000", 2023-10-09T06:15:17.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0083849679999999996 2023-10-09T06:15:17.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.637 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:17.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812642+0000", 2023-10-09T06:15:17.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0681e-05 2023-10-09T06:15:17.638 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.638 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:17.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794709+0000", 2023-10-09T06:15:17.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98206730200000003 2023-10-09T06:15:17.638 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.638 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:17.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794722+0000", 2023-10-09T06:15:17.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2621e-05 2023-10-09T06:15:17.638 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:17.639 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:17.639 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:17.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.639 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:171677 10.3 10.80488933 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:15:17.639 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:15:15.812441+0000", 2023-10-09T06:15:17.639 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 1.798792723, 2023-10-09T06:15:17.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7990482079999999, 2023-10-09T06:15:17.639 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:17.639 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "delayed", 2023-10-09T06:15:17.639 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:17.640 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:17.640 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:17.640 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171677 2023-10-09T06:15:17.640 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.640 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:17.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.640 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:17.640 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812441+0000", 2023-10-09T06:15:17.640 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:17.640 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.640 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:17.641 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812441+0000", 2023-10-09T06:15:17.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:17.641 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.641 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.641 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:17.641 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812439+0000", 2023-10-09T06:15:17.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:15:17.641 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.641 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.641 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:17.641 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812444+0000", 2023-10-09T06:15:17.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1150000000000002e-06 2023-10-09T06:15:17.642 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.642 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.642 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:17.642 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812455+0000", 2023-10-09T06:15:17.642 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1742999999999999e-05 2023-10-09T06:15:17.642 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.642 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.642 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:17.642 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812464+0000", 2023-10-09T06:15:17.642 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4910000000000005e-06 2023-10-09T06:15:17.642 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:17.643 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812964+0000", 2023-10-09T06:15:17.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00049951399999999995 2023-10-09T06:15:17.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:17.643 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812975+0000", 2023-10-09T06:15:17.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1063999999999999e-05 2023-10-09T06:15:17.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:17.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794751+0000", 2023-10-09T06:15:17.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.981776023 2023-10-09T06:15:17.644 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:17.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794760+0000", 2023-10-09T06:15:17.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9590000000000001e-06 2023-10-09T06:15:17.644 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:17.644 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:17.644 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:17.644 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.644 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:171684 10.3 10.9ab226f3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:15:17.645 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:15:15.815765+0000", 2023-10-09T06:15:17.645 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 1.795467946, 2023-10-09T06:15:17.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7957688790000002, 2023-10-09T06:15:17.645 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:17.645 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "delayed", 2023-10-09T06:15:17.645 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:17.645 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:17.645 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:17.645 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171684 2023-10-09T06:15:17.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.645 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:17.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.646 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:17.646 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815765+0000", 2023-10-09T06:15:17.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:17.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.646 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:17.646 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815765+0000", 2023-10-09T06:15:17.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:17.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:17.647 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815764+0000", 2023-10-09T06:15:17.647 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:15:17.647 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:17.647 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815771+0000", 2023-10-09T06:15:17.647 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3080000000000002e-06 2023-10-09T06:15:17.647 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:17.647 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815785+0000", 2023-10-09T06:15:17.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3903000000000001e-05 2023-10-09T06:15:17.648 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.648 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.648 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:17.648 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815795+0000", 2023-10-09T06:15:17.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8209999999999997e-06 2023-10-09T06:15:17.648 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.648 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.648 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:17.648 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815830+0000", 2023-10-09T06:15:17.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4987999999999998e-05 2023-10-09T06:15:17.648 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.649 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.649 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:17.649 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815839+0000", 2023-10-09T06:15:17.649 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5189999999999998e-06 2023-10-09T06:15:17.649 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.649 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.649 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:17.649 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794786+0000", 2023-10-09T06:15:17.649 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97894675900000006 2023-10-09T06:15:17.649 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:17.649 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:17.649 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:17.650 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794794+0000", 2023-10-09T06:15:17.650 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5590000000000006e-06 2023-10-09T06:15:17.650 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:17.650 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:17.650 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:17.650 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:17.650 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-10-09T06:15:17.650 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 6 2023-10-09T06:15:17.650 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:15:17.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:17.651 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:15:17.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:15:17.693 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:15:17.752 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:15:17.785 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:17.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:17 smithi064 ceph-mon[212750]: failed to encode map e559 with expected crc 2023-10-09T06:15:17.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:17 smithi064 ceph-mon[212750]: osdmap e559: 8 total, 8 up, 8 in 2023-10-09T06:15:17.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:17 smithi064 ceph-mon[212750]: failed to encode map e559 with expected crc 2023-10-09T06:15:17.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:17 smithi064 ceph-mon[212750]: failed to encode map e559 with expected crc 2023-10-09T06:15:17.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:17 smithi064 ceph-mon[212750]: failed to encode map e559 with expected crc 2023-10-09T06:15:17.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:17 smithi064 ceph-mon[212750]: failed to encode map e559 with expected crc 2023-10-09T06:15:17.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:17 smithi064 ceph-mon[212750]: failed to encode map e559 with expected crc 2023-10-09T06:15:17.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:17 smithi064 ceph-mon[212750]: failed to encode map e559 with expected crc 2023-10-09T06:15:17.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:17 smithi064 ceph-mon[212750]: failed to encode map e559 with expected crc 2023-10-09T06:15:17.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:17 smithi064 ceph-mon[212750]: osdmap e560: 8 total, 8 up, 8 in 2023-10-09T06:15:17.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:17 smithi064 ceph-mon[220044]: failed to encode map e559 with expected crc 2023-10-09T06:15:17.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:17 smithi064 ceph-mon[220044]: osdmap e559: 8 total, 8 up, 8 in 2023-10-09T06:15:17.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:17 smithi064 ceph-mon[220044]: failed to encode map e559 with expected crc 2023-10-09T06:15:17.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:17 smithi064 ceph-mon[220044]: failed to encode map e559 with expected crc 2023-10-09T06:15:17.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:17 smithi064 ceph-mon[220044]: failed to encode map e559 with expected crc 2023-10-09T06:15:17.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:17 smithi064 ceph-mon[220044]: failed to encode map e559 with expected crc 2023-10-09T06:15:17.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:17 smithi064 ceph-mon[220044]: failed to encode map e559 with expected crc 2023-10-09T06:15:17.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:17 smithi064 ceph-mon[220044]: failed to encode map e559 with expected crc 2023-10-09T06:15:17.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:17 smithi064 ceph-mon[220044]: failed to encode map e559 with expected crc 2023-10-09T06:15:17.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:17 smithi064 ceph-mon[220044]: osdmap e560: 8 total, 8 up, 8 in 2023-10-09T06:15:18.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:17 smithi156 ceph-mon[195101]: failed to encode map e559 with expected crc 2023-10-09T06:15:18.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:17 smithi156 ceph-mon[195101]: osdmap e559: 8 total, 8 up, 8 in 2023-10-09T06:15:18.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:17 smithi156 ceph-mon[195101]: failed to encode map e559 with expected crc 2023-10-09T06:15:18.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:17 smithi156 ceph-mon[195101]: failed to encode map e559 with expected crc 2023-10-09T06:15:18.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:17 smithi156 ceph-mon[195101]: failed to encode map e559 with expected crc 2023-10-09T06:15:18.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:17 smithi156 ceph-mon[195101]: failed to encode map e559 with expected crc 2023-10-09T06:15:18.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:17 smithi156 ceph-mon[195101]: failed to encode map e559 with expected crc 2023-10-09T06:15:18.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:17 smithi156 ceph-mon[195101]: failed to encode map e559 with expected crc 2023-10-09T06:15:18.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:17 smithi156 ceph-mon[195101]: failed to encode map e559 with expected crc 2023-10-09T06:15:18.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:17 smithi156 ceph-mon[195101]: osdmap e560: 8 total, 8 up, 8 in 2023-10-09T06:15:18.379 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:15:18.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:18.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:15:18.446 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:15:18.533 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:15:18.534 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:15:18.534 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:15:18.535 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:15:18.552 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:15:18.595 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:15:18.633 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:18.661 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:15:18.702 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:15:18.743 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:15:18.761 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:15:18.761 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:15:18.761 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:15:18.761 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:15:18.761 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:15:18.761 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:15:18.761 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:15:18.761 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:15:18.804 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:18.805 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:15:18.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:18.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:15:18.922 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:15:19.096 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:19.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:18 smithi156 ceph-mon[195101]: pgmap v1421: 119 pgs: 7 peering, 112 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 956 op/s 2023-10-09T06:15:19.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:18 smithi156 ceph-mon[195101]: failed to encode map e560 with expected crc 2023-10-09T06:15:19.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:18 smithi156 ceph-mon[195101]: failed to encode map e560 with expected crc 2023-10-09T06:15:19.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:18 smithi156 ceph-mon[195101]: failed to encode map e560 with expected crc 2023-10-09T06:15:19.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:18 smithi156 ceph-mon[195101]: 10.3 starting backfill to osd.4 from (497'400,526'6404] MIN to 557'10966 2023-10-09T06:15:19.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:18 smithi156 ceph-mon[195101]: 10.e starting backfill to osd.5 from (502'1700,533'7143] MIN to 557'10833 2023-10-09T06:15:19.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:18 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/865255826' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:15:19.198 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:15:19.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:19.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:15:19.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:18 smithi064 ceph-mon[220044]: pgmap v1421: 119 pgs: 7 peering, 112 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 956 op/s 2023-10-09T06:15:19.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:18 smithi064 ceph-mon[220044]: failed to encode map e560 with expected crc 2023-10-09T06:15:19.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:18 smithi064 ceph-mon[220044]: failed to encode map e560 with expected crc 2023-10-09T06:15:19.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:18 smithi064 ceph-mon[220044]: failed to encode map e560 with expected crc 2023-10-09T06:15:19.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:18 smithi064 ceph-mon[220044]: 10.3 starting backfill to osd.4 from (497'400,526'6404] MIN to 557'10966 2023-10-09T06:15:19.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:18 smithi064 ceph-mon[220044]: 10.e starting backfill to osd.5 from (502'1700,533'7143] MIN to 557'10833 2023-10-09T06:15:19.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:18 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/865255826' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:15:19.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:18 smithi064 ceph-mon[212750]: pgmap v1421: 119 pgs: 7 peering, 112 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 956 op/s 2023-10-09T06:15:19.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:18 smithi064 ceph-mon[212750]: failed to encode map e560 with expected crc 2023-10-09T06:15:19.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:18 smithi064 ceph-mon[212750]: failed to encode map e560 with expected crc 2023-10-09T06:15:19.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:18 smithi064 ceph-mon[212750]: failed to encode map e560 with expected crc 2023-10-09T06:15:19.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:18 smithi064 ceph-mon[212750]: 10.3 starting backfill to osd.4 from (497'400,526'6404] MIN to 557'10966 2023-10-09T06:15:19.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:18 smithi064 ceph-mon[212750]: 10.e starting backfill to osd.5 from (502'1700,533'7143] MIN to 557'10833 2023-10-09T06:15:19.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:18 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/865255826' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:15:19.309 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:15:19.380 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:19.771 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:15:19.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:19.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:15:19.879 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:15:19.928 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:20.031 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:15:20.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:20.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:15:20.046 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:19 smithi064 ceph-mon[220044]: osdmap e561: 8 total, 8 up, 8 in 2023-10-09T06:15:20.046 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:19 smithi064 ceph-mon[220044]: failed to encode map e561 with expected crc 2023-10-09T06:15:20.046 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:19 smithi064 ceph-mon[220044]: failed to encode map e561 with expected crc 2023-10-09T06:15:20.046 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:19 smithi064 ceph-mon[220044]: failed to encode map e561 with expected crc 2023-10-09T06:15:20.048 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:19 smithi064 ceph-mon[212750]: osdmap e561: 8 total, 8 up, 8 in 2023-10-09T06:15:20.048 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:19 smithi064 ceph-mon[212750]: failed to encode map e561 with expected crc 2023-10-09T06:15:20.049 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:19 smithi064 ceph-mon[212750]: failed to encode map e561 with expected crc 2023-10-09T06:15:20.049 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:19 smithi064 ceph-mon[212750]: failed to encode map e561 with expected crc 2023-10-09T06:15:20.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:19 smithi156 ceph-mon[195101]: osdmap e561: 8 total, 8 up, 8 in 2023-10-09T06:15:20.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:19 smithi156 ceph-mon[195101]: failed to encode map e561 with expected crc 2023-10-09T06:15:20.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:19 smithi156 ceph-mon[195101]: failed to encode map e561 with expected crc 2023-10-09T06:15:20.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:19 smithi156 ceph-mon[195101]: failed to encode map e561 with expected crc 2023-10-09T06:15:20.145 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:15:20.219 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:20.491 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:15:20.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:20.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:15:20.602 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:15:20.630 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:20.675 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:15:20.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:20.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:15:20.779 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:15:20.837 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:20.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:20 smithi064 ceph-mon[212750]: pgmap v1424: 119 pgs: 7 peering, 112 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2023-10-09T06:15:20.856 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:20 smithi064 ceph-mon[212750]: failed to encode map e562 with expected crc 2023-10-09T06:15:20.856 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:20 smithi064 ceph-mon[212750]: failed to encode map e562 with expected crc 2023-10-09T06:15:20.856 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:20 smithi064 ceph-mon[212750]: osdmap e562: 8 total, 8 up, 8 in 2023-10-09T06:15:20.909 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:15:20.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:20.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:15:21.019 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:15:21.079 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:21.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:20 smithi156 ceph-mon[195101]: pgmap v1424: 119 pgs: 7 peering, 112 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2023-10-09T06:15:21.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:20 smithi156 ceph-mon[195101]: failed to encode map e562 with expected crc 2023-10-09T06:15:21.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:20 smithi156 ceph-mon[195101]: failed to encode map e562 with expected crc 2023-10-09T06:15:21.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:20 smithi156 ceph-mon[195101]: osdmap e562: 8 total, 8 up, 8 in 2023-10-09T06:15:21.122 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:20 smithi064 ceph-mon[220044]: pgmap v1424: 119 pgs: 7 peering, 112 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2023-10-09T06:15:21.122 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:20 smithi064 ceph-mon[220044]: failed to encode map e562 with expected crc 2023-10-09T06:15:21.122 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:20 smithi064 ceph-mon[220044]: failed to encode map e562 with expected crc 2023-10-09T06:15:21.122 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:20 smithi064 ceph-mon[220044]: osdmap e562: 8 total, 8 up, 8 in 2023-10-09T06:15:21.133 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:15:21.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:21.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:15:21.175 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:21.240 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:15:21.284 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:21.346 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:15:21.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:21.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:15:21.454 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:15:21.490 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:21.704 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:15:21.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:21.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:15:21.816 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:15:21.893 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:21.898 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:15:22.006 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1, 2] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 4, 0] 2023-10-09T06:15:22.006 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:15:22.006 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T06:15:22.007 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:15:22.084 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:21 smithi064 ceph-mon[212750]: failed to encode map e562 with expected crc 2023-10-09T06:15:22.084 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:21 smithi064 ceph-mon[220044]: failed to encode map e562 with expected crc 2023-10-09T06:15:22.127 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:15:22.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:22.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:15:22.231 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:15:22.254 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:21 smithi156 ceph-mon[195101]: failed to encode map e562 with expected crc 2023-10-09T06:15:22.257 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:22.299 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:15:22.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:22.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:15:22.405 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:15:22.432 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:22.842 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:15:22.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:22.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:15:22.917 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:15:22.946 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:15:22.972 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:23.013 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:15:23.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:23.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:15:23.119 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:15:23.147 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:22 smithi064 ceph-mon[220044]: pgmap v1426: 119 pgs: 2 active+remapped+backfilling, 6 peering, 111 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 363 op/s; 1486/58407 objects misplaced (2.544%); 13 MiB/s, 211 objects/s recovering 2023-10-09T06:15:23.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:22 smithi064 ceph-mon[220044]: failed to encode map e563 with expected crc 2023-10-09T06:15:23.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:22 smithi064 ceph-mon[220044]: failed to encode map e563 with expected crc 2023-10-09T06:15:23.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:22 smithi064 ceph-mon[220044]: failed to encode map e563 with expected crc 2023-10-09T06:15:23.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:22 smithi064 ceph-mon[220044]: failed to encode map e563 with expected crc 2023-10-09T06:15:23.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:22 smithi064 ceph-mon[220044]: osdmap e563: 8 total, 8 up, 8 in 2023-10-09T06:15:23.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:22 smithi064 ceph-mon[220044]: failed to encode map e563 with expected crc 2023-10-09T06:15:23.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:22 smithi064 ceph-mon[220044]: failed to encode map e563 with expected crc 2023-10-09T06:15:23.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:22 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/668629087' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:15:23.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:22 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:15:23.149 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:22 smithi064 ceph-mon[212750]: pgmap v1426: 119 pgs: 2 active+remapped+backfilling, 6 peering, 111 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 363 op/s; 1486/58407 objects misplaced (2.544%); 13 MiB/s, 211 objects/s recovering 2023-10-09T06:15:23.149 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:22 smithi064 ceph-mon[212750]: failed to encode map e563 with expected crc 2023-10-09T06:15:23.149 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:22 smithi064 ceph-mon[212750]: failed to encode map e563 with expected crc 2023-10-09T06:15:23.149 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:22 smithi064 ceph-mon[212750]: failed to encode map e563 with expected crc 2023-10-09T06:15:23.149 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:22 smithi064 ceph-mon[212750]: failed to encode map e563 with expected crc 2023-10-09T06:15:23.149 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:22 smithi064 ceph-mon[212750]: osdmap e563: 8 total, 8 up, 8 in 2023-10-09T06:15:23.149 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:22 smithi064 ceph-mon[212750]: failed to encode map e563 with expected crc 2023-10-09T06:15:23.149 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:22 smithi064 ceph-mon[212750]: failed to encode map e563 with expected crc 2023-10-09T06:15:23.149 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:22 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/668629087' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:15:23.149 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:22 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:15:23.150 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:23.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:22 smithi156 ceph-mon[195101]: pgmap v1426: 119 pgs: 2 active+remapped+backfilling, 6 peering, 111 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 363 op/s; 1486/58407 objects misplaced (2.544%); 13 MiB/s, 211 objects/s recovering 2023-10-09T06:15:23.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:22 smithi156 ceph-mon[195101]: failed to encode map e563 with expected crc 2023-10-09T06:15:23.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:22 smithi156 ceph-mon[195101]: failed to encode map e563 with expected crc 2023-10-09T06:15:23.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:22 smithi156 ceph-mon[195101]: failed to encode map e563 with expected crc 2023-10-09T06:15:23.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:22 smithi156 ceph-mon[195101]: failed to encode map e563 with expected crc 2023-10-09T06:15:23.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:22 smithi156 ceph-mon[195101]: osdmap e563: 8 total, 8 up, 8 in 2023-10-09T06:15:23.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:22 smithi156 ceph-mon[195101]: failed to encode map e563 with expected crc 2023-10-09T06:15:23.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:22 smithi156 ceph-mon[195101]: failed to encode map e563 with expected crc 2023-10-09T06:15:23.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:22 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/668629087' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:15:23.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:22 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:15:23.927 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:15:23.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:23.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:15:24.031 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:15:24.074 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:24.142 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:15:24.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:24.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:15:24.152 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:23 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:15:24.152 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:23 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T06:15:24.152 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:23 smithi156 ceph-mon[195101]: failed to encode map e564 with expected crc 2023-10-09T06:15:24.152 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:23 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:15:24.153 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:23 smithi156 ceph-mon[195101]: osdmap e564: 8 total, 8 up, 8 in 2023-10-09T06:15:24.153 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:23 smithi156 ceph-mon[195101]: failed to encode map e564 with expected crc 2023-10-09T06:15:24.153 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:23 smithi156 ceph-mon[195101]: failed to encode map e564 with expected crc 2023-10-09T06:15:24.153 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:23 smithi156 ceph-mon[195101]: failed to encode map e564 with expected crc 2023-10-09T06:15:24.153 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:23 smithi156 ceph-mon[195101]: failed to encode map e564 with expected crc 2023-10-09T06:15:24.153 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:23 smithi156 ceph-mon[195101]: pgmap v1429: 119 pgs: 2 active+remapped+backfilling, 2 peering, 115 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 1486/55497 objects misplaced (2.678%); 13 MiB/s, 213 objects/s recovering 2023-10-09T06:15:24.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:23 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:15:24.172 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:23 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:15:24.172 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:23 smithi064 ceph-mon[212750]: failed to encode map e564 with expected crc 2023-10-09T06:15:24.172 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:23 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:15:24.172 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:23 smithi064 ceph-mon[212750]: osdmap e564: 8 total, 8 up, 8 in 2023-10-09T06:15:24.172 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:23 smithi064 ceph-mon[212750]: failed to encode map e564 with expected crc 2023-10-09T06:15:24.172 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:23 smithi064 ceph-mon[212750]: failed to encode map e564 with expected crc 2023-10-09T06:15:24.172 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:23 smithi064 ceph-mon[212750]: failed to encode map e564 with expected crc 2023-10-09T06:15:24.173 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:23 smithi064 ceph-mon[212750]: failed to encode map e564 with expected crc 2023-10-09T06:15:24.173 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:23 smithi064 ceph-mon[212750]: pgmap v1429: 119 pgs: 2 active+remapped+backfilling, 2 peering, 115 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 1486/55497 objects misplaced (2.678%); 13 MiB/s, 213 objects/s recovering 2023-10-09T06:15:24.173 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:23 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:15:24.173 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:23 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T06:15:24.173 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:23 smithi064 ceph-mon[220044]: failed to encode map e564 with expected crc 2023-10-09T06:15:24.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:23 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:15:24.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:23 smithi064 ceph-mon[220044]: osdmap e564: 8 total, 8 up, 8 in 2023-10-09T06:15:24.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:23 smithi064 ceph-mon[220044]: failed to encode map e564 with expected crc 2023-10-09T06:15:24.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:23 smithi064 ceph-mon[220044]: failed to encode map e564 with expected crc 2023-10-09T06:15:24.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:23 smithi064 ceph-mon[220044]: failed to encode map e564 with expected crc 2023-10-09T06:15:24.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:23 smithi064 ceph-mon[220044]: failed to encode map e564 with expected crc 2023-10-09T06:15:24.175 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:23 smithi064 ceph-mon[220044]: pgmap v1429: 119 pgs: 2 active+remapped+backfilling, 2 peering, 115 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 1486/55497 objects misplaced (2.678%); 13 MiB/s, 213 objects/s recovering 2023-10-09T06:15:24.206 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:24.250 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:15:24.308 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:24.362 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:24.392 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:15:24.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:24.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:15:24.506 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:15:24.524 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:15:24.524 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:15:24.547 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:24.639 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:15:24.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:24.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:15:24.749 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:15:24.796 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:24.918 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:15:24.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:24.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:15:25.027 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:15:25.072 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:24 smithi156 ceph-mon[195101]: failed to encode map e564 with expected crc 2023-10-09T06:15:25.072 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:24 smithi156 ceph-mon[195101]: failed to encode map e565 with expected crc 2023-10-09T06:15:25.072 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:24 smithi156 ceph-mon[195101]: failed to encode map e565 with expected crc 2023-10-09T06:15:25.072 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:24 smithi156 ceph-mon[195101]: osdmap e565: 8 total, 8 up, 8 in 2023-10-09T06:15:25.072 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:24 smithi156 ceph-mon[195101]: failed to encode map e565 with expected crc 2023-10-09T06:15:25.078 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:25.181 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:15:25.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:25.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:15:25.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:24 smithi064 ceph-mon[212750]: failed to encode map e564 with expected crc 2023-10-09T06:15:25.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:24 smithi064 ceph-mon[212750]: failed to encode map e565 with expected crc 2023-10-09T06:15:25.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:24 smithi064 ceph-mon[212750]: failed to encode map e565 with expected crc 2023-10-09T06:15:25.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:24 smithi064 ceph-mon[212750]: osdmap e565: 8 total, 8 up, 8 in 2023-10-09T06:15:25.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:24 smithi064 ceph-mon[212750]: failed to encode map e565 with expected crc 2023-10-09T06:15:25.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:24 smithi064 ceph-mon[220044]: failed to encode map e564 with expected crc 2023-10-09T06:15:25.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:24 smithi064 ceph-mon[220044]: failed to encode map e565 with expected crc 2023-10-09T06:15:25.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:24 smithi064 ceph-mon[220044]: failed to encode map e565 with expected crc 2023-10-09T06:15:25.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:24 smithi064 ceph-mon[220044]: osdmap e565: 8 total, 8 up, 8 in 2023-10-09T06:15:25.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:24 smithi064 ceph-mon[220044]: failed to encode map e565 with expected crc 2023-10-09T06:15:25.290 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:15:25.342 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:25.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:15:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:15:25] "GET /metrics HTTP/1.1" 200 36463 "" "Prometheus/2.43.0" 2023-10-09T06:15:26.079 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:25 smithi064 ceph-mon[212750]: pgmap v1431: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 481 op/s; 1486/53970 objects misplaced (2.753%); 11 MiB/s, 174 objects/s recovering 2023-10-09T06:15:26.079 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:25 smithi064 ceph-mon[220044]: pgmap v1431: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 481 op/s; 1486/53970 objects misplaced (2.753%); 11 MiB/s, 174 objects/s recovering 2023-10-09T06:15:26.083 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:15:26.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:25 smithi156 ceph-mon[195101]: pgmap v1431: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 481 op/s; 1486/53970 objects misplaced (2.753%); 11 MiB/s, 174 objects/s recovering 2023-10-09T06:15:26.490 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:15:26.933 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:15:26.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:26.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:15:27.036 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:15:27.123 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:27.295 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:15:27.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:27.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:15:27.402 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:15:27.447 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:27.516 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:15:27.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:27.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:15:27.623 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:15:27.682 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:27.690 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:27.761 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:15:27.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:27.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:15:27.799 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:27.868 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:15:27.889 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:15:27.940 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:15:27.957 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:27.986 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:15:28.042 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:15:28.086 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:15:28.130 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:15:28.176 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:15:28.218 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:15:28.232 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:15:28.232 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:15:28.232 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:15:28.232 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:15:28.233 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:15:28.233 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:15:28.233 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:15:28.233 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:15:28.238 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:15:28.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:28.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:15:28.348 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:15:28.406 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:28.639 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:15:28.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:28.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:15:28.745 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:15:28.810 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:28.849 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:15:28.854 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T06:15:28.899 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:15:28.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:28.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:15:29.023 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:15:29.072 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:28 smithi064 ceph-mon[212750]: pgmap v1432: 119 pgs: 119 active+clean; 973 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 3.5 MiB/s, 57 objects/s recovering 2023-10-09T06:15:29.072 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:28 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2963330530' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:15:29.072 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:28 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:29.073 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:28 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:29.073 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:28 smithi064 ceph-mon[220044]: pgmap v1432: 119 pgs: 119 active+clean; 973 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 3.5 MiB/s, 57 objects/s recovering 2023-10-09T06:15:29.073 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:28 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2963330530' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:15:29.073 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:28 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:29.073 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:28 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:29.082 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:29.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:28 smithi156 ceph-mon[195101]: pgmap v1432: 119 pgs: 119 active+clean; 973 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 3.5 MiB/s, 57 objects/s recovering 2023-10-09T06:15:29.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:28 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2963330530' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:15:29.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:28 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:29.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:28 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:29.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:29 smithi064 ceph-mon[212750]: from='client.36483 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:15:29.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:29 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:29.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:29 smithi064 ceph-mon[220044]: from='client.36483 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:15:29.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:29 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:30.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:29 smithi156 ceph-mon[195101]: from='client.36483 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:15:30.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:29 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:30.442 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:15:30.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:30.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:15:30.547 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:15:30.630 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:30.853 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1432,"stamp":"2023-10-09T06:15:27.219287+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1019746232,"num_objects":15576,"num_object_clones":0,"num_object_copies":46728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15576,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":176935,"num_write_kb":6157598,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44136,"num_bytes_recovered":2889627780,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100669,"ondisk_log_size":100669,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":369,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10331948,"kb_used_data":4356268,"kb_used_omap":165,"kb_used_meta":5974682,"kb_avail":739629268,"statfs":{"total":767960285184,"available":757380370432,"internally_reserved":0,"allocated":4460818432,"data_stored":4442244869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":169911,"internal_metadata":6118074441},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-255131648,"num_objects":-3893,"num_object_clones":0,"num_object_copies":-11679,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1486,"num_objects_unfound":0,"num_objects_dirty":-3893,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3893,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":344,"num_bytes_recovered":22282794,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001865"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":828,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.062518+0000","last_change":"2023-10-09T06:15:19.880315+0000","last_active":"2023-10-09T06:15:23.062518+0000","last_peered":"2023-10-09T06:15:23.062518+0000","last_clean":"2023-10-09T06:15:23.062518+0000","last_became_active":"2023-10-09T06:15:19.879857+0000","last_became_peered":"2023-10-09T06:15:19.879857+0000","last_unstale":"2023-10-09T06:15:23.062518+0000","last_undegraded":"2023-10-09T06:15:23.062518+0000","last_fullsized":"2023-10-09T06:15:23.062518+0000","mapping_epoch":561,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":562,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:05.412941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:05.412941+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:54.322404+0000","scrub_duration":0.0061014329999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":633,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.916816+0000","last_change":"2023-10-09T06:07:43.572773+0000","last_active":"2023-10-09T06:15:22.916816+0000","last_peered":"2023-10-09T06:15:22.916816+0000","last_clean":"2023-10-09T06:15:22.916816+0000","last_became_active":"2023-10-09T06:07:43.572299+0000","last_became_peered":"2023-10-09T06:07:43.572299+0000","last_unstale":"2023-10-09T06:15:22.916816+0000","last_undegraded":"2023-10-09T06:15:22.916816+0000","last_fullsized":"2023-10-09T06:15:22.916816+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:00:37.672435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":376,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.910291+0000","last_change":"2023-10-09T06:05:11.451972+0000","last_active":"2023-10-09T06:15:22.910291+0000","last_peered":"2023-10-09T06:15:22.910291+0000","last_clean":"2023-10-09T06:15:22.910291+0000","last_became_active":"2023-10-09T06:05:11.451448+0000","last_became_peered":"2023-10-09T06:05:11.451448+0000","last_unstale":"2023-10-09T06:15:22.910291+0000","last_undegraded":"2023-10-09T06:15:22.910291+0000","last_fullsized":"2023-10-09T06:15:22.910291+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:39.183621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":864,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.917249+0000","last_change":"2023-10-09T06:09:27.554162+0000","last_active":"2023-10-09T06:15:22.917249+0000","last_peered":"2023-10-09T06:15:22.917249+0000","last_clean":"2023-10-09T06:15:22.917249+0000","last_became_active":"2023-10-09T06:09:27.553659+0000","last_became_peered":"2023-10-09T06:09:27.553659+0000","last_unstale":"2023-10-09T06:15:22.917249+0000","last_undegraded":"2023-10-09T06:15:22.917249+0000","last_fullsized":"2023-10-09T06:15:22.917249+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:01.468346+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:01.468346+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:03.443511+0000","scrub_duration":0.0016258220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":247,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.911001+0000","last_change":"2023-10-09T06:01:54.762060+0000","last_active":"2023-10-09T06:15:22.911001+0000","last_peered":"2023-10-09T06:15:22.911001+0000","last_clean":"2023-10-09T06:15:22.911001+0000","last_became_active":"2023-10-09T06:01:54.761757+0000","last_became_peered":"2023-10-09T06:01:54.761757+0000","last_unstale":"2023-10-09T06:15:22.911001+0000","last_undegraded":"2023-10-09T06:15:22.911001+0000","last_fullsized":"2023-10-09T06:15:22.911001+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:40.294232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":48,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.063019+0000","last_change":"2023-10-09T06:13:30.789834+0000","last_active":"2023-10-09T06:15:23.063019+0000","last_peered":"2023-10-09T06:15:23.063019+0000","last_clean":"2023-10-09T06:15:23.063019+0000","last_became_active":"2023-10-09T06:13:30.789517+0000","last_became_peered":"2023-10-09T06:13:30.789517+0000","last_unstale":"2023-10-09T06:15:23.063019+0000","last_undegraded":"2023-10-09T06:15:23.063019+0000","last_fullsized":"2023-10-09T06:15:23.063019+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:45.467922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":849,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.916600+0000","last_change":"2023-10-09T06:12:07.739121+0000","last_active":"2023-10-09T06:15:22.916600+0000","last_peered":"2023-10-09T06:15:22.916600+0000","last_clean":"2023-10-09T06:15:22.916600+0000","last_became_active":"2023-10-09T06:12:07.738739+0000","last_became_peered":"2023-10-09T06:12:07.738739+0000","last_unstale":"2023-10-09T06:15:22.916600+0000","last_undegraded":"2023-10-09T06:15:22.916600+0000","last_fullsized":"2023-10-09T06:15:22.916600+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.293426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.293426+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:19.815422+0000","scrub_duration":0.001587946,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":358,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.909311+0000","last_change":"2023-10-09T06:09:27.564377+0000","last_active":"2023-10-09T06:15:22.909311+0000","last_peered":"2023-10-09T06:15:22.909311+0000","last_clean":"2023-10-09T06:15:22.909311+0000","last_became_active":"2023-10-09T06:09:27.563710+0000","last_became_peered":"2023-10-09T06:09:27.563710+0000","last_unstale":"2023-10-09T06:15:22.909311+0000","last_undegraded":"2023-10-09T06:15:22.909311+0000","last_fullsized":"2023-10-09T06:15:22.909311+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:48.168447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":454,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.393025+0000","last_change":"2023-10-09T06:15:01.885861+0000","last_active":"2023-10-09T06:15:23.393025+0000","last_peered":"2023-10-09T06:15:23.393025+0000","last_clean":"2023-10-09T06:15:23.393025+0000","last_became_active":"2023-10-09T06:15:01.885560+0000","last_became_peered":"2023-10-09T06:15:01.885560+0000","last_unstale":"2023-10-09T06:15:23.393025+0000","last_undegraded":"2023-10-09T06:15:23.393025+0000","last_fullsized":"2023-10-09T06:15:23.393025+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:44.842197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":783,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.062893+0000","last_change":"2023-10-09T06:13:33.701687+0000","last_active":"2023-10-09T06:15:23.062893+0000","last_peered":"2023-10-09T06:15:23.062893+0000","last_clean":"2023-10-09T06:15:23.062893+0000","last_became_active":"2023-10-09T06:13:33.701233+0000","last_became_peered":"2023-10-09T06:13:33.701233+0000","last_unstale":"2023-10-09T06:15:23.062893+0000","last_undegraded":"2023-10-09T06:15:23.062893+0000","last_fullsized":"2023-10-09T06:15:23.062893+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.384407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.384407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:04.887826+0000","scrub_duration":0.0040651960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":463,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.912086+0000","last_change":"2023-10-09T06:09:27.559756+0000","last_active":"2023-10-09T06:15:22.912086+0000","last_peered":"2023-10-09T06:15:22.912086+0000","last_clean":"2023-10-09T06:15:22.912086+0000","last_became_active":"2023-10-09T06:09:27.559443+0000","last_became_peered":"2023-10-09T06:09:27.559443+0000","last_unstale":"2023-10-09T06:15:22.912086+0000","last_undegraded":"2023-10-09T06:15:22.912086+0000","last_fullsized":"2023-10-09T06:15:22.912086+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:14.519398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":394,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.909383+0000","last_change":"2023-10-09T06:13:21.737897+0000","last_active":"2023-10-09T06:15:22.909383+0000","last_peered":"2023-10-09T06:15:22.909383+0000","last_clean":"2023-10-09T06:15:22.909383+0000","last_became_active":"2023-10-09T06:13:21.737564+0000","last_became_peered":"2023-10-09T06:13:21.737564+0000","last_unstale":"2023-10-09T06:15:22.909383+0000","last_undegraded":"2023-10-09T06:15:22.909383+0000","last_fullsized":"2023-10-09T06:15:22.909383+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:34.863320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":689,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.062835+0000","last_change":"2023-10-09T06:10:11.729656+0000","last_active":"2023-10-09T06:15:23.062835+0000","last_peered":"2023-10-09T06:15:23.062835+0000","last_clean":"2023-10-09T06:15:23.062835+0000","last_became_active":"2023-10-09T06:10:11.728960+0000","last_became_peered":"2023-10-09T06:10:11.728960+0000","last_unstale":"2023-10-09T06:15:23.062835+0000","last_undegraded":"2023-10-09T06:15:23.062835+0000","last_fullsized":"2023-10-09T06:15:23.062835+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:20.742561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:20.742561+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:45.543709+0000","scrub_duration":0.0027005039999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":48,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.062940+0000","last_change":"2023-10-09T06:13:30.802749+0000","last_active":"2023-10-09T06:15:23.062940+0000","last_peered":"2023-10-09T06:15:23.062940+0000","last_clean":"2023-10-09T06:15:23.062940+0000","last_became_active":"2023-10-09T06:13:30.802449+0000","last_became_peered":"2023-10-09T06:13:30.802449+0000","last_unstale":"2023-10-09T06:15:23.062940+0000","last_undegraded":"2023-10-09T06:15:23.062940+0000","last_fullsized":"2023-10-09T06:15:23.062940+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:29.017181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":552,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.909999+0000","last_change":"2023-10-09T06:13:30.792445+0000","last_active":"2023-10-09T06:15:22.909999+0000","last_peered":"2023-10-09T06:15:22.909999+0000","last_clean":"2023-10-09T06:15:22.909999+0000","last_became_active":"2023-10-09T06:13:30.792116+0000","last_became_peered":"2023-10-09T06:13:30.792116+0000","last_unstale":"2023-10-09T06:15:22.909999+0000","last_undegraded":"2023-10-09T06:15:22.909999+0000","last_fullsized":"2023-10-09T06:15:22.909999+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:20:53.506293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":776,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.913953+0000","last_change":"2023-10-09T06:13:24.760843+0000","last_active":"2023-10-09T06:15:22.913953+0000","last_peered":"2023-10-09T06:15:22.913953+0000","last_clean":"2023-10-09T06:15:22.913953+0000","last_became_active":"2023-10-09T06:13:24.760333+0000","last_became_peered":"2023-10-09T06:13:24.760333+0000","last_unstale":"2023-10-09T06:15:22.913953+0000","last_undegraded":"2023-10-09T06:15:22.913953+0000","last_fullsized":"2023-10-09T06:15:22.913953+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:38.588641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:38.588641+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:30.660400+0000","scrub_duration":0.010116385,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":54,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.062792+0000","last_change":"2023-10-09T06:13:30.826588+0000","last_active":"2023-10-09T06:15:23.062792+0000","last_peered":"2023-10-09T06:15:23.062792+0000","last_clean":"2023-10-09T06:15:23.062792+0000","last_became_active":"2023-10-09T06:13:30.799199+0000","last_became_peered":"2023-10-09T06:13:30.799199+0000","last_unstale":"2023-10-09T06:15:23.062792+0000","last_undegraded":"2023-10-09T06:15:23.062792+0000","last_fullsized":"2023-10-09T06:15:23.062792+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:38.703864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":346,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.913595+0000","last_change":"2023-10-09T06:13:21.739214+0000","last_active":"2023-10-09T06:15:22.913595+0000","last_peered":"2023-10-09T06:15:22.913595+0000","last_clean":"2023-10-09T06:15:22.913595+0000","last_became_active":"2023-10-09T06:13:21.738951+0000","last_became_peered":"2023-10-09T06:13:21.738951+0000","last_unstale":"2023-10-09T06:15:22.913595+0000","last_undegraded":"2023-10-09T06:15:22.913595+0000","last_fullsized":"2023-10-09T06:15:22.913595+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:17.477241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":737,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.911575+0000","last_change":"2023-10-09T06:09:27.574336+0000","last_active":"2023-10-09T06:15:22.911575+0000","last_peered":"2023-10-09T06:15:22.911575+0000","last_clean":"2023-10-09T06:15:22.911575+0000","last_became_active":"2023-10-09T06:09:27.573974+0000","last_became_peered":"2023-10-09T06:09:27.573974+0000","last_unstale":"2023-10-09T06:15:22.911575+0000","last_undegraded":"2023-10-09T06:15:22.911575+0000","last_fullsized":"2023-10-09T06:15:22.911575+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.339121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.339121+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:45.908579+0000","scrub_duration":0.010474344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":599,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.917062+0000","last_change":"2023-10-09T06:13:30.812598+0000","last_active":"2023-10-09T06:15:22.917062+0000","last_peered":"2023-10-09T06:15:22.917062+0000","last_clean":"2023-10-09T06:15:22.917062+0000","last_became_active":"2023-10-09T06:13:30.791460+0000","last_became_peered":"2023-10-09T06:13:30.791460+0000","last_unstale":"2023-10-09T06:15:22.917062+0000","last_undegraded":"2023-10-09T06:15:22.917062+0000","last_fullsized":"2023-10-09T06:15:22.917062+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:28.046730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":577,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.917411+0000","last_change":"2023-10-09T06:01:54.764916+0000","last_active":"2023-10-09T06:15:22.917411+0000","last_peered":"2023-10-09T06:15:22.917411+0000","last_clean":"2023-10-09T06:15:22.917411+0000","last_became_active":"2023-10-09T06:01:54.764705+0000","last_became_peered":"2023-10-09T06:01:54.764705+0000","last_unstale":"2023-10-09T06:15:22.917411+0000","last_undegraded":"2023-10-09T06:15:22.917411+0000","last_fullsized":"2023-10-09T06:15:22.917411+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:12.527719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":793,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.909484+0000","last_change":"2023-10-09T06:13:15.425525+0000","last_active":"2023-10-09T06:15:22.909484+0000","last_peered":"2023-10-09T06:15:22.909484+0000","last_clean":"2023-10-09T06:15:22.909484+0000","last_became_active":"2023-10-09T06:13:15.425086+0000","last_became_peered":"2023-10-09T06:13:15.425086+0000","last_unstale":"2023-10-09T06:15:22.909484+0000","last_undegraded":"2023-10-09T06:15:22.909484+0000","last_fullsized":"2023-10-09T06:15:22.909484+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:42.900876+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:42.900876+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:46.042715+0000","scrub_duration":0.0014928649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":54,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.062669+0000","last_change":"2023-10-09T06:13:30.811420+0000","last_active":"2023-10-09T06:15:23.062669+0000","last_peered":"2023-10-09T06:15:23.062669+0000","last_clean":"2023-10-09T06:15:23.062669+0000","last_became_active":"2023-10-09T06:13:30.792835+0000","last_became_peered":"2023-10-09T06:13:30.792835+0000","last_unstale":"2023-10-09T06:15:23.062669+0000","last_undegraded":"2023-10-09T06:15:23.062669+0000","last_fullsized":"2023-10-09T06:15:23.062669+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:38.962645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":611,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.910383+0000","last_change":"2023-10-09T06:01:54.761558+0000","last_active":"2023-10-09T06:15:22.910383+0000","last_peered":"2023-10-09T06:15:22.910383+0000","last_clean":"2023-10-09T06:15:22.910383+0000","last_became_active":"2023-10-09T06:01:54.761251+0000","last_became_peered":"2023-10-09T06:01:54.761251+0000","last_unstale":"2023-10-09T06:15:22.910383+0000","last_undegraded":"2023-10-09T06:15:22.910383+0000","last_fullsized":"2023-10-09T06:15:22.910383+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:53.245267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":1068,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.062639+0000","last_change":"2023-10-09T06:13:24.754018+0000","last_active":"2023-10-09T06:15:23.062639+0000","last_peered":"2023-10-09T06:15:23.062639+0000","last_clean":"2023-10-09T06:15:23.062639+0000","last_became_active":"2023-10-09T06:13:24.753712+0000","last_became_peered":"2023-10-09T06:13:24.753712+0000","last_unstale":"2023-10-09T06:15:23.062639+0000","last_undegraded":"2023-10-09T06:15:23.062639+0000","last_fullsized":"2023-10-09T06:15:23.062639+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:18.490833+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":48,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.062494+0000","last_change":"2023-10-09T06:13:30.788160+0000","last_active":"2023-10-09T06:15:23.062494+0000","last_peered":"2023-10-09T06:15:23.062494+0000","last_clean":"2023-10-09T06:15:23.062494+0000","last_became_active":"2023-10-09T06:13:30.787724+0000","last_became_peered":"2023-10-09T06:13:30.787724+0000","last_unstale":"2023-10-09T06:15:23.062494+0000","last_undegraded":"2023-10-09T06:15:23.062494+0000","last_fullsized":"2023-10-09T06:15:23.062494+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:38.376770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":48,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.063002+0000","last_change":"2023-10-09T06:13:30.789225+0000","last_active":"2023-10-09T06:15:23.063002+0000","last_peered":"2023-10-09T06:15:23.063002+0000","last_clean":"2023-10-09T06:15:23.063002+0000","last_became_active":"2023-10-09T06:13:30.788869+0000","last_became_peered":"2023-10-09T06:13:30.788869+0000","last_unstale":"2023-10-09T06:15:23.063002+0000","last_undegraded":"2023-10-09T06:15:23.063002+0000","last_fullsized":"2023-10-09T06:15:23.063002+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:19.519966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":805,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.911986+0000","last_change":"2023-10-09T06:12:55.750923+0000","last_active":"2023-10-09T06:15:22.911986+0000","last_peered":"2023-10-09T06:15:22.911986+0000","last_clean":"2023-10-09T06:15:22.911986+0000","last_became_active":"2023-10-09T06:12:55.750521+0000","last_became_peered":"2023-10-09T06:12:55.750521+0000","last_unstale":"2023-10-09T06:15:22.911986+0000","last_undegraded":"2023-10-09T06:15:22.911986+0000","last_fullsized":"2023-10-09T06:15:22.911986+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.316697+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.316697+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:23.591439+0000","scrub_duration":0.001666561,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":418,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.062573+0000","last_change":"2023-10-09T06:01:22.337171+0000","last_active":"2023-10-09T06:15:23.062573+0000","last_peered":"2023-10-09T06:15:23.062573+0000","last_clean":"2023-10-09T06:15:23.062573+0000","last_became_active":"2023-10-09T06:01:22.335438+0000","last_became_peered":"2023-10-09T06:01:22.335438+0000","last_unstale":"2023-10-09T06:15:23.062573+0000","last_undegraded":"2023-10-09T06:15:23.062573+0000","last_fullsized":"2023-10-09T06:15:23.062573+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:25.036448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":527,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.909815+0000","last_change":"2023-10-09T06:13:30.788284+0000","last_active":"2023-10-09T06:15:22.909815+0000","last_peered":"2023-10-09T06:15:22.909815+0000","last_clean":"2023-10-09T06:15:22.909815+0000","last_became_active":"2023-10-09T06:13:30.787933+0000","last_became_peered":"2023-10-09T06:13:30.787933+0000","last_unstale":"2023-10-09T06:15:22.909815+0000","last_undegraded":"2023-10-09T06:15:22.909815+0000","last_fullsized":"2023-10-09T06:15:22.909815+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:10.735438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":855,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.912181+0000","last_change":"2023-10-09T06:13:24.753908+0000","last_active":"2023-10-09T06:15:22.912181+0000","last_peered":"2023-10-09T06:15:22.912181+0000","last_clean":"2023-10-09T06:15:22.912181+0000","last_became_active":"2023-10-09T06:13:24.753566+0000","last_became_peered":"2023-10-09T06:13:24.753566+0000","last_unstale":"2023-10-09T06:15:22.912181+0000","last_undegraded":"2023-10-09T06:15:22.912181+0000","last_fullsized":"2023-10-09T06:15:22.912181+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:40.388125+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:40.388125+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:35.101978+0000","scrub_duration":0.0055576990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":137,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.390766+0000","last_change":"2023-10-09T06:13:12.243605+0000","last_active":"2023-10-09T06:15:23.390766+0000","last_peered":"2023-10-09T06:15:23.390766+0000","last_clean":"2023-10-09T06:15:23.390766+0000","last_became_active":"2023-10-09T06:13:12.237061+0000","last_became_peered":"2023-10-09T06:13:12.237061+0000","last_unstale":"2023-10-09T06:15:23.390766+0000","last_undegraded":"2023-10-09T06:15:23.390766+0000","last_fullsized":"2023-10-09T06:15:23.390766+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:29:03.666031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":76,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.910736+0000","last_change":"2023-10-09T06:13:30.788733+0000","last_active":"2023-10-09T06:15:22.910736+0000","last_peered":"2023-10-09T06:15:22.910736+0000","last_clean":"2023-10-09T06:15:22.910736+0000","last_became_active":"2023-10-09T06:13:30.788353+0000","last_became_peered":"2023-10-09T06:13:30.788353+0000","last_unstale":"2023-10-09T06:15:22.910736+0000","last_undegraded":"2023-10-09T06:15:22.910736+0000","last_fullsized":"2023-10-09T06:15:22.910736+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:06.308426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"565'11093","reported_seq":11178,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:26.403891+0000","last_change":"2023-10-09T06:11:46.032384+0000","last_active":"2023-10-09T06:15:26.403891+0000","last_peered":"2023-10-09T06:15:26.403891+0000","last_clean":"2023-10-09T06:15:26.403891+0000","last_became_active":"2023-10-09T06:11:46.032091+0000","last_became_peered":"2023-10-09T06:11:46.032091+0000","last_unstale":"2023-10-09T06:15:26.403891+0000","last_undegraded":"2023-10-09T06:15:26.403891+0000","last_fullsized":"2023-10-09T06:15:26.403891+0000","mapping_epoch":490,"log_start":"513'4923","ondisk_log_start":"513'4923","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6170,"log_dups_size":0,"ondisk_log_size":6170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:31.117384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62849024,"num_objects":959,"num_object_clones":0,"num_object_copies":2877,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":959,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11093,"num_write_kb":385664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":746,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.063015+0000","last_change":"2023-10-09T06:13:33.701521+0000","last_active":"2023-10-09T06:15:23.063015+0000","last_peered":"2023-10-09T06:15:23.063015+0000","last_clean":"2023-10-09T06:15:23.063015+0000","last_became_active":"2023-10-09T06:13:33.700870+0000","last_became_peered":"2023-10-09T06:13:33.700870+0000","last_unstale":"2023-10-09T06:15:23.063015+0000","last_undegraded":"2023-10-09T06:15:23.063015+0000","last_fullsized":"2023-10-09T06:15:23.063015+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:10:59.422551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:10:59.422551+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:54.402494+0000","scrub_duration":0.0059716659999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":52,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.062963+0000","last_change":"2023-10-09T06:13:30.822903+0000","last_active":"2023-10-09T06:15:23.062963+0000","last_peered":"2023-10-09T06:15:23.062963+0000","last_clean":"2023-10-09T06:15:23.062963+0000","last_became_active":"2023-10-09T06:13:30.799013+0000","last_became_peered":"2023-10-09T06:13:30.799013+0000","last_unstale":"2023-10-09T06:15:23.062963+0000","last_undegraded":"2023-10-09T06:15:23.062963+0000","last_fullsized":"2023-10-09T06:15:23.062963+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:44.271821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":48,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.062425+0000","last_change":"2023-10-09T06:13:30.793531+0000","last_active":"2023-10-09T06:15:23.062425+0000","last_peered":"2023-10-09T06:15:23.062425+0000","last_clean":"2023-10-09T06:15:23.062425+0000","last_became_active":"2023-10-09T06:13:30.792689+0000","last_became_peered":"2023-10-09T06:13:30.792689+0000","last_unstale":"2023-10-09T06:15:23.062425+0000","last_undegraded":"2023-10-09T06:15:23.062425+0000","last_fullsized":"2023-10-09T06:15:23.062425+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:02.559186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"565'11080","reported_seq":20012,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:25.458441+0000","last_change":"2023-10-09T06:15:03.759950+0000","last_active":"2023-10-09T06:15:25.458441+0000","last_peered":"2023-10-09T06:15:25.458441+0000","last_clean":"2023-10-09T06:15:25.458441+0000","last_became_active":"2023-10-09T06:15:03.735000+0000","last_became_peered":"2023-10-09T06:15:03.735000+0000","last_unstale":"2023-10-09T06:15:25.458441+0000","last_undegraded":"2023-10-09T06:15:25.458441+0000","last_fullsized":"2023-10-09T06:15:25.458441+0000","mapping_epoch":554,"log_start":"513'5193","ondisk_log_start":"513'5193","created":490,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5887,"log_dups_size":0,"ondisk_log_size":5887,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:01.420748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65667072,"num_objects":1002,"num_object_clones":0,"num_object_copies":3006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11080,"num_write_kb":386624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3329,"num_bytes_recovered":218169344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":934,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.910249+0000","last_change":"2023-10-09T06:13:24.753866+0000","last_active":"2023-10-09T06:15:22.910249+0000","last_peered":"2023-10-09T06:15:22.910249+0000","last_clean":"2023-10-09T06:15:22.910249+0000","last_became_active":"2023-10-09T06:13:24.753432+0000","last_became_peered":"2023-10-09T06:13:24.753432+0000","last_unstale":"2023-10-09T06:15:22.910249+0000","last_undegraded":"2023-10-09T06:15:22.910249+0000","last_fullsized":"2023-10-09T06:15:22.910249+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:03.241335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:03.241335+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:19.854449+0000","scrub_duration":0.017780547000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":52,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.062935+0000","last_change":"2023-10-09T06:13:30.829763+0000","last_active":"2023-10-09T06:15:23.062935+0000","last_peered":"2023-10-09T06:15:23.062935+0000","last_clean":"2023-10-09T06:15:23.062935+0000","last_became_active":"2023-10-09T06:13:30.802486+0000","last_became_peered":"2023-10-09T06:13:30.802486+0000","last_unstale":"2023-10-09T06:15:23.062935+0000","last_undegraded":"2023-10-09T06:15:23.062935+0000","last_fullsized":"2023-10-09T06:15:23.062935+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:05.468205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":603,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.916740+0000","last_change":"2023-10-09T06:01:54.760367+0000","last_active":"2023-10-09T06:15:22.916740+0000","last_peered":"2023-10-09T06:15:22.916740+0000","last_clean":"2023-10-09T06:15:22.916740+0000","last_became_active":"2023-10-09T06:01:54.760087+0000","last_became_peered":"2023-10-09T06:01:54.760087+0000","last_unstale":"2023-10-09T06:15:22.916740+0000","last_undegraded":"2023-10-09T06:15:22.916740+0000","last_fullsized":"2023-10-09T06:15:22.916740+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:26.000104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"565'10950","reported_seq":11033,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:25.962150+0000","last_change":"2023-10-09T06:11:45.696810+0000","last_active":"2023-10-09T06:15:25.962150+0000","last_peered":"2023-10-09T06:15:25.962150+0000","last_clean":"2023-10-09T06:15:25.962150+0000","last_became_active":"2023-10-09T06:11:45.696476+0000","last_became_peered":"2023-10-09T06:11:45.696476+0000","last_unstale":"2023-10-09T06:15:25.962150+0000","last_undegraded":"2023-10-09T06:15:25.962150+0000","last_fullsized":"2023-10-09T06:15:25.962150+0000","mapping_epoch":490,"log_start":"515'5500","ondisk_log_start":"515'5500","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5450,"log_dups_size":0,"ondisk_log_size":5450,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:41:33.351567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64356352,"num_objects":982,"num_object_clones":0,"num_object_copies":2946,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":982,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10950,"num_write_kb":381824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":828,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.917212+0000","last_change":"2023-10-09T06:13:24.768958+0000","last_active":"2023-10-09T06:15:22.917212+0000","last_peered":"2023-10-09T06:15:22.917212+0000","last_clean":"2023-10-09T06:15:22.917212+0000","last_became_active":"2023-10-09T06:13:24.768643+0000","last_became_peered":"2023-10-09T06:13:24.768643+0000","last_unstale":"2023-10-09T06:15:22.917212+0000","last_undegraded":"2023-10-09T06:15:22.917212+0000","last_fullsized":"2023-10-09T06:15:22.917212+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:48.495739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:48.495739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:18:50.654354+0000","scrub_duration":0.0016345509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":52,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.062909+0000","last_change":"2023-10-09T06:13:30.814434+0000","last_active":"2023-10-09T06:15:23.062909+0000","last_peered":"2023-10-09T06:15:23.062909+0000","last_clean":"2023-10-09T06:15:23.062909+0000","last_became_active":"2023-10-09T06:13:30.793119+0000","last_became_peered":"2023-10-09T06:13:30.793119+0000","last_unstale":"2023-10-09T06:15:23.062909+0000","last_undegraded":"2023-10-09T06:15:23.062909+0000","last_fullsized":"2023-10-09T06:15:23.062909+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:34:31.724006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":590,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.911762+0000","last_change":"2023-10-09T06:13:30.792490+0000","last_active":"2023-10-09T06:15:22.911762+0000","last_peered":"2023-10-09T06:15:22.911762+0000","last_clean":"2023-10-09T06:15:22.911762+0000","last_became_active":"2023-10-09T06:13:30.792209+0000","last_became_peered":"2023-10-09T06:13:30.792209+0000","last_unstale":"2023-10-09T06:15:22.911762+0000","last_undegraded":"2023-10-09T06:15:22.911762+0000","last_fullsized":"2023-10-09T06:15:22.911762+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:14.208042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"565'10994","reported_seq":11079,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:26.439538+0000","last_change":"2023-10-09T06:11:46.031195+0000","last_active":"2023-10-09T06:15:26.439538+0000","last_peered":"2023-10-09T06:15:26.439538+0000","last_clean":"2023-10-09T06:15:26.439538+0000","last_became_active":"2023-10-09T06:11:46.030891+0000","last_became_peered":"2023-10-09T06:11:46.030891+0000","last_unstale":"2023-10-09T06:15:26.439538+0000","last_undegraded":"2023-10-09T06:15:26.439538+0000","last_fullsized":"2023-10-09T06:15:26.439538+0000","mapping_epoch":490,"log_start":"512'4836","ondisk_log_start":"512'4836","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6158,"log_dups_size":0,"ondisk_log_size":6158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:25.788661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60424192,"num_objects":922,"num_object_clones":0,"num_object_copies":2766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10994,"num_write_kb":381312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":780,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.925927+0000","last_change":"2023-10-09T06:13:33.708291+0000","last_active":"2023-10-09T06:15:22.925927+0000","last_peered":"2023-10-09T06:15:22.925927+0000","last_clean":"2023-10-09T06:15:22.925927+0000","last_became_active":"2023-10-09T06:13:33.707930+0000","last_became_peered":"2023-10-09T06:13:33.707930+0000","last_unstale":"2023-10-09T06:15:22.925927+0000","last_undegraded":"2023-10-09T06:15:22.925927+0000","last_fullsized":"2023-10-09T06:15:22.925927+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:16.361097+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:16.361097+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:18.111404+0000","scrub_duration":0.028533749000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":335,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.913475+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T06:15:23.913475+0000","last_peered":"2023-10-09T06:15:23.913475+0000","last_clean":"2023-10-09T06:15:23.913475+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T06:15:23.913475+0000","last_undegraded":"2023-10-09T06:15:23.913475+0000","last_fullsized":"2023-10-09T06:15:23.913475+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":578,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.391100+0000","last_change":"2023-10-09T06:09:27.558353+0000","last_active":"2023-10-09T06:15:23.391100+0000","last_peered":"2023-10-09T06:15:23.391100+0000","last_clean":"2023-10-09T06:15:23.391100+0000","last_became_active":"2023-10-09T06:09:27.557233+0000","last_became_peered":"2023-10-09T06:09:27.557233+0000","last_unstale":"2023-10-09T06:15:23.391100+0000","last_undegraded":"2023-10-09T06:15:23.391100+0000","last_fullsized":"2023-10-09T06:15:23.391100+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:53.149350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"565'11060","reported_seq":11145,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:26.665028+0000","last_change":"2023-10-09T06:11:45.694648+0000","last_active":"2023-10-09T06:15:26.665028+0000","last_peered":"2023-10-09T06:15:26.665028+0000","last_clean":"2023-10-09T06:15:26.665028+0000","last_became_active":"2023-10-09T06:11:45.694321+0000","last_became_peered":"2023-10-09T06:11:45.694321+0000","last_unstale":"2023-10-09T06:15:26.665028+0000","last_undegraded":"2023-10-09T06:15:26.665028+0000","last_fullsized":"2023-10-09T06:15:26.665028+0000","mapping_epoch":490,"log_start":"508'4732","ondisk_log_start":"508'4732","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6328,"log_dups_size":0,"ondisk_log_size":6328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:43.704525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56492032,"num_objects":862,"num_object_clones":0,"num_object_copies":2586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11060,"num_write_kb":381504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":971,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.062437+0000","last_change":"2023-10-09T06:15:19.874375+0000","last_active":"2023-10-09T06:15:23.062437+0000","last_peered":"2023-10-09T06:15:23.062437+0000","last_clean":"2023-10-09T06:15:23.062437+0000","last_became_active":"2023-10-09T06:15:19.873993+0000","last_became_peered":"2023-10-09T06:15:19.873993+0000","last_unstale":"2023-10-09T06:15:23.062437+0000","last_undegraded":"2023-10-09T06:15:23.062437+0000","last_fullsized":"2023-10-09T06:15:23.062437+0000","mapping_epoch":561,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":562,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:42.795781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:42.795781+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:43.523891+0000","scrub_duration":0.0091262979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":369,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912930+0000","last_change":"2023-10-09T06:09:27.565666+0000","last_active":"2023-10-09T06:15:23.912930+0000","last_peered":"2023-10-09T06:15:23.912930+0000","last_clean":"2023-10-09T06:15:23.912930+0000","last_became_active":"2023-10-09T06:09:27.565324+0000","last_became_peered":"2023-10-09T06:09:27.565324+0000","last_unstale":"2023-10-09T06:15:23.912930+0000","last_undegraded":"2023-10-09T06:15:23.912930+0000","last_fullsized":"2023-10-09T06:15:23.912930+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:51.216947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":539,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.917093+0000","last_change":"2023-10-09T06:13:21.727745+0000","last_active":"2023-10-09T06:15:22.917093+0000","last_peered":"2023-10-09T06:15:22.917093+0000","last_clean":"2023-10-09T06:15:22.917093+0000","last_became_active":"2023-10-09T06:13:21.727404+0000","last_became_peered":"2023-10-09T06:13:21.727404+0000","last_unstale":"2023-10-09T06:15:22.917093+0000","last_undegraded":"2023-10-09T06:15:22.917093+0000","last_fullsized":"2023-10-09T06:15:22.917093+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:24.931626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"565'11244","reported_seq":24097,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:25.564085+0000","last_change":"2023-10-09T06:15:23.952835+0000","last_active":"2023-10-09T06:15:25.564085+0000","last_peered":"2023-10-09T06:15:25.564085+0000","last_clean":"2023-10-09T06:15:25.564085+0000","last_became_active":"2023-10-09T06:15:23.914520+0000","last_became_peered":"2023-10-09T06:15:23.914520+0000","last_unstale":"2023-10-09T06:15:25.564085+0000","last_undegraded":"2023-10-09T06:15:25.564085+0000","last_fullsized":"2023-10-09T06:15:25.564085+0000","mapping_epoch":564,"log_start":"514'5335","ondisk_log_start":"514'5335","created":490,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5909,"log_dups_size":0,"ondisk_log_size":5909,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:55.053464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67502080,"num_objects":1030,"num_object_clones":0,"num_object_copies":3090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11244,"num_write_kb":392768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6357,"num_bytes_recovered":416481280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":828,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.909195+0000","last_change":"2023-10-09T06:13:15.442141+0000","last_active":"2023-10-09T06:15:22.909195+0000","last_peered":"2023-10-09T06:15:22.909195+0000","last_clean":"2023-10-09T06:15:22.909195+0000","last_became_active":"2023-10-09T06:13:15.441716+0000","last_became_peered":"2023-10-09T06:13:15.441716+0000","last_unstale":"2023-10-09T06:15:22.909195+0000","last_undegraded":"2023-10-09T06:15:22.909195+0000","last_fullsized":"2023-10-09T06:15:22.909195+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.282393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.282393+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:16.798440+0000","scrub_duration":0.01402403,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":484,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.916699+0000","last_change":"2023-10-09T06:13:21.738218+0000","last_active":"2023-10-09T06:15:22.916699+0000","last_peered":"2023-10-09T06:15:22.916699+0000","last_clean":"2023-10-09T06:15:22.916699+0000","last_became_active":"2023-10-09T06:13:21.737969+0000","last_became_peered":"2023-10-09T06:13:21.737969+0000","last_unstale":"2023-10-09T06:15:22.916699+0000","last_undegraded":"2023-10-09T06:15:22.916699+0000","last_fullsized":"2023-10-09T06:15:22.916699+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:38.620281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":623,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.917056+0000","last_change":"2023-10-09T06:15:16.810530+0000","last_active":"2023-10-09T06:15:22.917056+0000","last_peered":"2023-10-09T06:15:22.917056+0000","last_clean":"2023-10-09T06:15:22.917056+0000","last_became_active":"2023-10-09T06:15:16.810238+0000","last_became_peered":"2023-10-09T06:15:16.810238+0000","last_unstale":"2023-10-09T06:15:22.917056+0000","last_undegraded":"2023-10-09T06:15:22.917056+0000","last_fullsized":"2023-10-09T06:15:22.917056+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:03.744638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"565'11319","reported_seq":19708,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:26.699778+0000","last_change":"2023-10-09T06:14:28.746278+0000","last_active":"2023-10-09T06:15:26.699778+0000","last_peered":"2023-10-09T06:15:26.699778+0000","last_clean":"2023-10-09T06:15:26.699778+0000","last_became_active":"2023-10-09T06:14:28.728616+0000","last_became_peered":"2023-10-09T06:14:28.728616+0000","last_unstale":"2023-10-09T06:15:26.699778+0000","last_undegraded":"2023-10-09T06:15:26.699778+0000","last_fullsized":"2023-10-09T06:15:26.699778+0000","mapping_epoch":545,"log_start":"513'5055","ondisk_log_start":"513'5055","created":490,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6264,"log_dups_size":0,"ondisk_log_size":6264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:32.106540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59703296,"num_objects":911,"num_object_clones":0,"num_object_copies":2733,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":911,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11319,"num_write_kb":391360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4137,"num_bytes_recovered":271122432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":899,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.911883+0000","last_change":"2023-10-09T06:13:15.428701+0000","last_active":"2023-10-09T06:15:22.911883+0000","last_peered":"2023-10-09T06:15:22.911883+0000","last_clean":"2023-10-09T06:15:22.911883+0000","last_became_active":"2023-10-09T06:13:15.428343+0000","last_became_peered":"2023-10-09T06:13:15.428343+0000","last_unstale":"2023-10-09T06:15:22.911883+0000","last_undegraded":"2023-10-09T06:15:22.911883+0000","last_fullsized":"2023-10-09T06:15:22.911883+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.373748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.373748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:55:34.235074+0000","scrub_duration":0.0024576450000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":555,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.917067+0000","last_change":"2023-10-09T06:07:43.574863+0000","last_active":"2023-10-09T06:15:22.917067+0000","last_peered":"2023-10-09T06:15:22.917067+0000","last_clean":"2023-10-09T06:15:22.917067+0000","last_became_active":"2023-10-09T06:07:43.574576+0000","last_became_peered":"2023-10-09T06:07:43.574576+0000","last_unstale":"2023-10-09T06:15:22.917067+0000","last_undegraded":"2023-10-09T06:15:22.917067+0000","last_fullsized":"2023-10-09T06:15:22.917067+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:16.166004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":596,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.925951+0000","last_change":"2023-10-09T06:13:21.727814+0000","last_active":"2023-10-09T06:15:22.925951+0000","last_peered":"2023-10-09T06:15:22.925951+0000","last_clean":"2023-10-09T06:15:22.925951+0000","last_became_active":"2023-10-09T06:13:21.727401+0000","last_became_peered":"2023-10-09T06:13:21.727401+0000","last_unstale":"2023-10-09T06:15:22.925951+0000","last_undegraded":"2023-10-09T06:15:22.925951+0000","last_fullsized":"2023-10-09T06:15:22.925951+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:53:37.968028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"565'11117","reported_seq":11199,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:25.969171+0000","last_change":"2023-10-09T06:11:45.695709+0000","last_active":"2023-10-09T06:15:25.969171+0000","last_peered":"2023-10-09T06:15:25.969171+0000","last_clean":"2023-10-09T06:15:25.969171+0000","last_became_active":"2023-10-09T06:11:45.695219+0000","last_became_peered":"2023-10-09T06:11:45.695219+0000","last_unstale":"2023-10-09T06:15:25.969171+0000","last_undegraded":"2023-10-09T06:15:25.969171+0000","last_fullsized":"2023-10-09T06:15:25.969171+0000","mapping_epoch":490,"log_start":"519'5700","ondisk_log_start":"519'5700","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5417,"log_dups_size":0,"ondisk_log_size":5417,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:26.533964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65208320,"num_objects":995,"num_object_clones":0,"num_object_copies":2985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11117,"num_write_kb":387584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":643,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.913743+0000","last_change":"2023-10-09T06:13:24.753519+0000","last_active":"2023-10-09T06:15:22.913743+0000","last_peered":"2023-10-09T06:15:22.913743+0000","last_clean":"2023-10-09T06:15:22.913743+0000","last_became_active":"2023-10-09T06:13:24.753185+0000","last_became_peered":"2023-10-09T06:13:24.753185+0000","last_unstale":"2023-10-09T06:15:22.913743+0000","last_undegraded":"2023-10-09T06:15:22.913743+0000","last_fullsized":"2023-10-09T06:15:22.913743+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:39.633410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:39.633410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:56.871057+0000","scrub_duration":0.001547624,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":377,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.909290+0000","last_change":"2023-10-09T06:13:30.781849+0000","last_active":"2023-10-09T06:15:22.909290+0000","last_peered":"2023-10-09T06:15:22.909290+0000","last_clean":"2023-10-09T06:15:22.909290+0000","last_became_active":"2023-10-09T06:13:30.781084+0000","last_became_peered":"2023-10-09T06:13:30.781084+0000","last_unstale":"2023-10-09T06:15:22.909290+0000","last_undegraded":"2023-10-09T06:15:22.909290+0000","last_fullsized":"2023-10-09T06:15:22.909290+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:02.199298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":248,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.911563+0000","last_change":"2023-10-09T06:01:54.761768+0000","last_active":"2023-10-09T06:15:23.911563+0000","last_peered":"2023-10-09T06:15:23.911563+0000","last_clean":"2023-10-09T06:15:23.911563+0000","last_became_active":"2023-10-09T06:01:54.761443+0000","last_became_peered":"2023-10-09T06:01:54.761443+0000","last_unstale":"2023-10-09T06:15:23.911563+0000","last_undegraded":"2023-10-09T06:15:23.911563+0000","last_fullsized":"2023-10-09T06:15:23.911563+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:51.555987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"565'11124","reported_seq":18077,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:24.343510+0000","last_change":"2023-10-09T06:14:44.803724+0000","last_active":"2023-10-09T06:15:24.343510+0000","last_peered":"2023-10-09T06:15:24.343510+0000","last_clean":"2023-10-09T06:15:24.343510+0000","last_became_active":"2023-10-09T06:14:44.737126+0000","last_became_peered":"2023-10-09T06:14:44.737126+0000","last_unstale":"2023-10-09T06:15:24.343510+0000","last_undegraded":"2023-10-09T06:15:24.343510+0000","last_fullsized":"2023-10-09T06:15:24.343510+0000","mapping_epoch":549,"log_start":"508'4815","ondisk_log_start":"508'4815","created":490,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6309,"log_dups_size":0,"ondisk_log_size":6309,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:14.630144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70909952,"num_objects":1082,"num_object_clones":0,"num_object_copies":3246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1082,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11124,"num_write_kb":390592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3423,"num_bytes_recovered":224329728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":768,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.910750+0000","last_change":"2023-10-09T06:13:33.707885+0000","last_active":"2023-10-09T06:15:22.910750+0000","last_peered":"2023-10-09T06:15:22.910750+0000","last_clean":"2023-10-09T06:15:22.910750+0000","last_became_active":"2023-10-09T06:13:33.707474+0000","last_became_peered":"2023-10-09T06:13:33.707474+0000","last_unstale":"2023-10-09T06:15:22.910750+0000","last_undegraded":"2023-10-09T06:15:22.910750+0000","last_fullsized":"2023-10-09T06:15:22.910750+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:12:22.289803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:12:22.289803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:27.842817+0000","scrub_duration":0.0076565050000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":310,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.911180+0000","last_change":"2023-10-09T06:12:04.680247+0000","last_active":"2023-10-09T06:15:22.911180+0000","last_peered":"2023-10-09T06:15:22.911180+0000","last_clean":"2023-10-09T06:15:22.911180+0000","last_became_active":"2023-10-09T06:12:04.679943+0000","last_became_peered":"2023-10-09T06:12:04.679943+0000","last_unstale":"2023-10-09T06:15:22.911180+0000","last_undegraded":"2023-10-09T06:15:22.911180+0000","last_fullsized":"2023-10-09T06:15:22.911180+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:56.692774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":353,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.909789+0000","last_change":"2023-10-09T06:07:43.577306+0000","last_active":"2023-10-09T06:15:22.909789+0000","last_peered":"2023-10-09T06:15:22.909789+0000","last_clean":"2023-10-09T06:15:22.909789+0000","last_became_active":"2023-10-09T06:07:43.576956+0000","last_became_peered":"2023-10-09T06:07:43.576956+0000","last_unstale":"2023-10-09T06:15:22.909789+0000","last_undegraded":"2023-10-09T06:15:22.909789+0000","last_fullsized":"2023-10-09T06:15:22.909789+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:21.020403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":281,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.912026+0000","last_change":"2023-10-09T06:13:12.234849+0000","last_active":"2023-10-09T06:15:22.912026+0000","last_peered":"2023-10-09T06:15:22.912026+0000","last_clean":"2023-10-09T06:15:22.912026+0000","last_became_active":"2023-10-09T06:13:12.234284+0000","last_became_peered":"2023-10-09T06:13:12.234284+0000","last_unstale":"2023-10-09T06:15:22.912026+0000","last_undegraded":"2023-10-09T06:15:22.912026+0000","last_fullsized":"2023-10-09T06:15:22.912026+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:40:43.809773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"565'10941","reported_seq":11026,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:26.450334+0000","last_change":"2023-10-09T06:11:46.029349+0000","last_active":"2023-10-09T06:15:26.450334+0000","last_peered":"2023-10-09T06:15:26.450334+0000","last_clean":"2023-10-09T06:15:26.450334+0000","last_became_active":"2023-10-09T06:11:46.029041+0000","last_became_peered":"2023-10-09T06:11:46.029041+0000","last_unstale":"2023-10-09T06:15:26.450334+0000","last_undegraded":"2023-10-09T06:15:26.450334+0000","last_fullsized":"2023-10-09T06:15:26.450334+0000","mapping_epoch":490,"log_start":"512'4800","ondisk_log_start":"512'4800","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6141,"log_dups_size":0,"ondisk_log_size":6141,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:07.662395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61014016,"num_objects":931,"num_object_clones":0,"num_object_copies":2793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10941,"num_write_kb":379904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":800,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.390909+0000","last_change":"2023-10-09T06:09:27.544555+0000","last_active":"2023-10-09T06:15:23.390909+0000","last_peered":"2023-10-09T06:15:23.390909+0000","last_clean":"2023-10-09T06:15:23.390909+0000","last_became_active":"2023-10-09T06:09:27.544222+0000","last_became_peered":"2023-10-09T06:09:27.544222+0000","last_unstale":"2023-10-09T06:15:23.390909+0000","last_undegraded":"2023-10-09T06:15:23.390909+0000","last_fullsized":"2023-10-09T06:15:23.390909+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:45.944253+0000","scrub_duration":0.0016660799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":266,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.912103+0000","last_change":"2023-10-09T06:13:21.738040+0000","last_active":"2023-10-09T06:15:22.912103+0000","last_peered":"2023-10-09T06:15:22.912103+0000","last_clean":"2023-10-09T06:15:22.912103+0000","last_became_active":"2023-10-09T06:13:21.737717+0000","last_became_peered":"2023-10-09T06:13:21.737717+0000","last_unstale":"2023-10-09T06:15:22.912103+0000","last_undegraded":"2023-10-09T06:15:22.912103+0000","last_fullsized":"2023-10-09T06:15:22.912103+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:39.451628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":545,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.916706+0000","last_change":"2023-10-09T06:09:27.553958+0000","last_active":"2023-10-09T06:15:22.916706+0000","last_peered":"2023-10-09T06:15:22.916706+0000","last_clean":"2023-10-09T06:15:22.916706+0000","last_became_active":"2023-10-09T06:09:27.553193+0000","last_became_peered":"2023-10-09T06:09:27.553193+0000","last_unstale":"2023-10-09T06:15:22.916706+0000","last_undegraded":"2023-10-09T06:15:22.916706+0000","last_fullsized":"2023-10-09T06:15:22.916706+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:09.754579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":391,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.911261+0000","last_change":"2023-10-09T06:07:43.580765+0000","last_active":"2023-10-09T06:15:22.911261+0000","last_peered":"2023-10-09T06:15:22.911261+0000","last_clean":"2023-10-09T06:15:22.911261+0000","last_became_active":"2023-10-09T06:07:43.580458+0000","last_became_peered":"2023-10-09T06:07:43.580458+0000","last_unstale":"2023-10-09T06:15:22.911261+0000","last_undegraded":"2023-10-09T06:15:22.911261+0000","last_fullsized":"2023-10-09T06:15:22.911261+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:55.161893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"565'10905","reported_seq":27244,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:24.342917+0000","last_change":"2023-10-09T06:14:21.793267+0000","last_active":"2023-10-09T06:15:24.342917+0000","last_peered":"2023-10-09T06:15:24.342917+0000","last_clean":"2023-10-09T06:15:24.342917+0000","last_became_active":"2023-10-09T06:14:21.734665+0000","last_became_peered":"2023-10-09T06:14:21.734665+0000","last_unstale":"2023-10-09T06:15:24.342917+0000","last_undegraded":"2023-10-09T06:15:24.342917+0000","last_fullsized":"2023-10-09T06:15:24.342917+0000","mapping_epoch":543,"log_start":"508'4616","ondisk_log_start":"508'4616","created":490,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6289,"log_dups_size":0,"ondisk_log_size":6289,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:01.941433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65208320,"num_objects":995,"num_object_clones":0,"num_object_copies":2985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10905,"num_write_kb":380800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7073,"num_bytes_recovered":463536128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":620,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.062536+0000","last_change":"2023-10-09T06:14:47.339271+0000","last_active":"2023-10-09T06:15:23.062536+0000","last_peered":"2023-10-09T06:15:23.062536+0000","last_clean":"2023-10-09T06:15:23.062536+0000","last_became_active":"2023-10-09T06:13:33.705351+0000","last_became_peered":"2023-10-09T06:13:33.705351+0000","last_unstale":"2023-10-09T06:15:23.062536+0000","last_undegraded":"2023-10-09T06:15:23.062536+0000","last_fullsized":"2023-10-09T06:15:23.062536+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:14:47.339163+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T06:14:47.339163+0000","last_clean_scrub_stamp":"2023-10-09T06:14:47.339163+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:34.887758+0000","scrub_duration":0.0016607729999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":354,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.909692+0000","last_change":"2023-10-09T06:15:16.802788+0000","last_active":"2023-10-09T06:15:22.909692+0000","last_peered":"2023-10-09T06:15:22.909692+0000","last_clean":"2023-10-09T06:15:22.909692+0000","last_became_active":"2023-10-09T06:15:16.802449+0000","last_became_peered":"2023-10-09T06:15:16.802449+0000","last_unstale":"2023-10-09T06:15:22.909692+0000","last_undegraded":"2023-10-09T06:15:22.909692+0000","last_fullsized":"2023-10-09T06:15:22.909692+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:55:30.488174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":335,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.911955+0000","last_change":"2023-10-09T06:13:21.728291+0000","last_active":"2023-10-09T06:15:22.911955+0000","last_peered":"2023-10-09T06:15:22.911955+0000","last_clean":"2023-10-09T06:15:22.911955+0000","last_became_active":"2023-10-09T06:13:21.727926+0000","last_became_peered":"2023-10-09T06:13:21.727926+0000","last_unstale":"2023-10-09T06:15:22.911955+0000","last_undegraded":"2023-10-09T06:15:22.911955+0000","last_fullsized":"2023-10-09T06:15:22.911955+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:35.292785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":263,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.911402+0000","last_change":"2023-10-09T06:09:27.547953+0000","last_active":"2023-10-09T06:15:22.911402+0000","last_peered":"2023-10-09T06:15:22.911402+0000","last_clean":"2023-10-09T06:15:22.911402+0000","last_became_active":"2023-10-09T06:09:27.547624+0000","last_became_peered":"2023-10-09T06:09:27.547624+0000","last_unstale":"2023-10-09T06:15:22.911402+0000","last_undegraded":"2023-10-09T06:15:22.911402+0000","last_fullsized":"2023-10-09T06:15:22.911402+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:39.630486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"565'11015","reported_seq":18936,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:26.652186+0000","last_change":"2023-10-09T06:14:43.759714+0000","last_active":"2023-10-09T06:15:26.652186+0000","last_peered":"2023-10-09T06:15:26.652186+0000","last_clean":"2023-10-09T06:15:26.652186+0000","last_became_active":"2023-10-09T06:14:43.736592+0000","last_became_peered":"2023-10-09T06:14:43.736592+0000","last_unstale":"2023-10-09T06:15:26.652186+0000","last_undegraded":"2023-10-09T06:15:26.652186+0000","last_fullsized":"2023-10-09T06:15:26.652186+0000","mapping_epoch":548,"log_start":"508'4710","ondisk_log_start":"508'4710","created":490,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6305,"log_dups_size":0,"ondisk_log_size":6305,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:35.196970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60227584,"num_objects":919,"num_object_clones":0,"num_object_copies":2757,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":919,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11015,"num_write_kb":381888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3906,"num_bytes_recovered":255983616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":48,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.062728+0000","last_change":"2023-10-09T06:13:30.791685+0000","last_active":"2023-10-09T06:15:23.062728+0000","last_peered":"2023-10-09T06:15:23.062728+0000","last_clean":"2023-10-09T06:15:23.062728+0000","last_became_active":"2023-10-09T06:13:30.791255+0000","last_became_peered":"2023-10-09T06:13:30.791255+0000","last_unstale":"2023-10-09T06:15:23.062728+0000","last_undegraded":"2023-10-09T06:15:23.062728+0000","last_fullsized":"2023-10-09T06:15:23.062728+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:24.896244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":608,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.062402+0000","last_change":"2023-10-09T06:13:33.704835+0000","last_active":"2023-10-09T06:15:23.062402+0000","last_peered":"2023-10-09T06:15:23.062402+0000","last_clean":"2023-10-09T06:15:23.062402+0000","last_became_active":"2023-10-09T06:13:33.704466+0000","last_became_peered":"2023-10-09T06:13:33.704466+0000","last_unstale":"2023-10-09T06:15:23.062402+0000","last_undegraded":"2023-10-09T06:15:23.062402+0000","last_fullsized":"2023-10-09T06:15:23.062402+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:35.783434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:35.783434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:38.721974+0000","scrub_duration":0.0043794799999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":353,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.909636+0000","last_change":"2023-10-09T06:13:12.238280+0000","last_active":"2023-10-09T06:15:22.909636+0000","last_peered":"2023-10-09T06:15:22.909636+0000","last_clean":"2023-10-09T06:15:22.909636+0000","last_became_active":"2023-10-09T06:13:12.237986+0000","last_became_peered":"2023-10-09T06:13:12.237986+0000","last_unstale":"2023-10-09T06:15:22.909636+0000","last_undegraded":"2023-10-09T06:15:22.909636+0000","last_fullsized":"2023-10-09T06:15:22.909636+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:58.520176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":495,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.917327+0000","last_change":"2023-10-09T06:13:21.733940+0000","last_active":"2023-10-09T06:15:22.917327+0000","last_peered":"2023-10-09T06:15:22.917327+0000","last_clean":"2023-10-09T06:15:22.917327+0000","last_became_active":"2023-10-09T06:13:21.733628+0000","last_became_peered":"2023-10-09T06:13:21.733628+0000","last_unstale":"2023-10-09T06:15:22.917327+0000","last_undegraded":"2023-10-09T06:15:22.917327+0000","last_fullsized":"2023-10-09T06:15:22.917327+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:13.348443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"565'10898","reported_seq":20874,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:24.335960+0000","last_change":"2023-10-09T06:14:09.843785+0000","last_active":"2023-10-09T06:15:24.335960+0000","last_peered":"2023-10-09T06:15:24.335960+0000","last_clean":"2023-10-09T06:15:24.335960+0000","last_became_active":"2023-10-09T06:14:09.789864+0000","last_became_peered":"2023-10-09T06:14:09.789864+0000","last_unstale":"2023-10-09T06:15:24.335960+0000","last_undegraded":"2023-10-09T06:15:24.335960+0000","last_fullsized":"2023-10-09T06:15:24.335960+0000","mapping_epoch":540,"log_start":"508'4611","ondisk_log_start":"508'4611","created":490,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6287,"log_dups_size":0,"ondisk_log_size":6287,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:03.191322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66584576,"num_objects":1016,"num_object_clones":0,"num_object_copies":3048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10898,"num_write_kb":381248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4936,"num_bytes_recovered":323485696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":834,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.911879+0000","last_change":"2023-10-09T06:13:24.753842+0000","last_active":"2023-10-09T06:15:22.911879+0000","last_peered":"2023-10-09T06:15:22.911879+0000","last_clean":"2023-10-09T06:15:22.911879+0000","last_became_active":"2023-10-09T06:13:24.753485+0000","last_became_peered":"2023-10-09T06:13:24.753485+0000","last_unstale":"2023-10-09T06:15:22.911879+0000","last_undegraded":"2023-10-09T06:15:22.911879+0000","last_fullsized":"2023-10-09T06:15:22.911879+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:57.627296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:57.627296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:56.656218+0000","scrub_duration":0.001913166,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":458,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.062618+0000","last_change":"2023-10-09T06:09:27.544373+0000","last_active":"2023-10-09T06:15:23.062618+0000","last_peered":"2023-10-09T06:15:23.062618+0000","last_clean":"2023-10-09T06:15:23.062618+0000","last_became_active":"2023-10-09T06:09:27.544121+0000","last_became_peered":"2023-10-09T06:09:27.544121+0000","last_unstale":"2023-10-09T06:15:23.062618+0000","last_undegraded":"2023-10-09T06:15:23.062618+0000","last_fullsized":"2023-10-09T06:15:23.062618+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:40.429491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":502,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.390715+0000","last_change":"2023-10-09T06:12:52.670573+0000","last_active":"2023-10-09T06:15:23.390715+0000","last_peered":"2023-10-09T06:15:23.390715+0000","last_clean":"2023-10-09T06:15:23.390715+0000","last_became_active":"2023-10-09T06:12:52.670242+0000","last_became_peered":"2023-10-09T06:12:52.670242+0000","last_unstale":"2023-10-09T06:15:23.390715+0000","last_undegraded":"2023-10-09T06:15:23.390715+0000","last_fullsized":"2023-10-09T06:15:23.390715+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:25.237580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":650,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.391339+0000","last_change":"2023-10-09T06:15:16.805514+0000","last_active":"2023-10-09T06:15:23.391339+0000","last_peered":"2023-10-09T06:15:23.391339+0000","last_clean":"2023-10-09T06:15:23.391339+0000","last_became_active":"2023-10-09T06:15:16.805160+0000","last_became_peered":"2023-10-09T06:15:16.805160+0000","last_unstale":"2023-10-09T06:15:23.391339+0000","last_undegraded":"2023-10-09T06:15:23.391339+0000","last_fullsized":"2023-10-09T06:15:23.391339+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:30.153699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"565'11027","reported_seq":20642,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:25.978468+0000","last_change":"2023-10-09T06:14:02.643285+0000","last_active":"2023-10-09T06:15:25.978468+0000","last_peered":"2023-10-09T06:15:25.978468+0000","last_clean":"2023-10-09T06:15:25.978468+0000","last_became_active":"2023-10-09T06:14:02.599489+0000","last_became_peered":"2023-10-09T06:14:02.599489+0000","last_unstale":"2023-10-09T06:15:25.978468+0000","last_undegraded":"2023-10-09T06:15:25.978468+0000","last_fullsized":"2023-10-09T06:15:25.978468+0000","mapping_epoch":537,"log_start":"516'5600","ondisk_log_start":"516'5600","created":490,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5427,"log_dups_size":0,"ondisk_log_size":5427,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:10.022074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62652440,"num_objects":957,"num_object_clones":0,"num_object_copies":2871,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":957,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11027,"num_write_kb":383425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4755,"num_bytes_recovered":311558168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":765,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.909725+0000","last_change":"2023-10-09T06:13:24.756288+0000","last_active":"2023-10-09T06:15:22.909725+0000","last_peered":"2023-10-09T06:15:22.909725+0000","last_clean":"2023-10-09T06:15:22.909725+0000","last_became_active":"2023-10-09T06:13:24.755945+0000","last_became_peered":"2023-10-09T06:13:24.755945+0000","last_unstale":"2023-10-09T06:15:22.909725+0000","last_undegraded":"2023-10-09T06:15:22.909725+0000","last_fullsized":"2023-10-09T06:15:22.909725+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:45.844892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:45.844892+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:44.107376+0000","scrub_duration":0.0017521489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":48,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.062783+0000","last_change":"2023-10-09T06:13:30.799535+0000","last_active":"2023-10-09T06:15:23.062783+0000","last_peered":"2023-10-09T06:15:23.062783+0000","last_clean":"2023-10-09T06:15:23.062783+0000","last_became_active":"2023-10-09T06:13:30.799227+0000","last_became_peered":"2023-10-09T06:13:30.799227+0000","last_unstale":"2023-10-09T06:15:23.062783+0000","last_undegraded":"2023-10-09T06:15:23.062783+0000","last_fullsized":"2023-10-09T06:15:23.062783+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:14.741742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":293,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.911348+0000","last_change":"2023-10-09T06:15:16.810272+0000","last_active":"2023-10-09T06:15:22.911348+0000","last_peered":"2023-10-09T06:15:22.911348+0000","last_clean":"2023-10-09T06:15:22.911348+0000","last_became_active":"2023-10-09T06:15:16.804086+0000","last_became_peered":"2023-10-09T06:15:16.804086+0000","last_unstale":"2023-10-09T06:15:22.911348+0000","last_undegraded":"2023-10-09T06:15:22.911348+0000","last_fullsized":"2023-10-09T06:15:22.911348+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:45.156184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":48,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.062628+0000","last_change":"2023-10-09T06:13:30.788382+0000","last_active":"2023-10-09T06:15:23.062628+0000","last_peered":"2023-10-09T06:15:23.062628+0000","last_clean":"2023-10-09T06:15:23.062628+0000","last_became_active":"2023-10-09T06:13:30.788060+0000","last_became_peered":"2023-10-09T06:13:30.788060+0000","last_unstale":"2023-10-09T06:15:23.062628+0000","last_undegraded":"2023-10-09T06:15:23.062628+0000","last_fullsized":"2023-10-09T06:15:23.062628+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:36:58.984799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.e","version":"565'11177","reported_seq":23627,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:25.948093+0000","last_change":"2023-10-09T06:15:23.486583+0000","last_active":"2023-10-09T06:15:25.948093+0000","last_peered":"2023-10-09T06:15:25.948093+0000","last_clean":"2023-10-09T06:15:25.948093+0000","last_became_active":"2023-10-09T06:15:23.398455+0000","last_became_peered":"2023-10-09T06:15:23.398455+0000","last_unstale":"2023-10-09T06:15:25.948093+0000","last_undegraded":"2023-10-09T06:15:25.948093+0000","last_fullsized":"2023-10-09T06:15:25.948093+0000","mapping_epoch":563,"log_start":"520'5800","ondisk_log_start":"520'5800","created":490,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5377,"log_dups_size":0,"ondisk_log_size":5377,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:19.708343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65470464,"num_objects":999,"num_object_clones":0,"num_object_copies":2997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11177,"num_write_kb":389632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6155,"num_bytes_recovered":403243008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":1058,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.917210+0000","last_change":"2023-10-09T06:15:19.869123+0000","last_active":"2023-10-09T06:15:22.917210+0000","last_peered":"2023-10-09T06:15:22.917210+0000","last_clean":"2023-10-09T06:15:22.917210+0000","last_became_active":"2023-10-09T06:15:19.868795+0000","last_became_peered":"2023-10-09T06:15:19.868795+0000","last_unstale":"2023-10-09T06:15:22.917210+0000","last_undegraded":"2023-10-09T06:15:22.917210+0000","last_fullsized":"2023-10-09T06:15:22.917210+0000","mapping_epoch":561,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":562,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:17.344518+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:17.344518+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:34.303488+0000","scrub_duration":0.0015914099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":622,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.912281+0000","last_change":"2023-10-09T06:13:30.798394+0000","last_active":"2023-10-09T06:15:22.912281+0000","last_peered":"2023-10-09T06:15:22.912281+0000","last_clean":"2023-10-09T06:15:22.912281+0000","last_became_active":"2023-10-09T06:13:30.790913+0000","last_became_peered":"2023-10-09T06:13:30.790913+0000","last_unstale":"2023-10-09T06:15:22.912281+0000","last_undegraded":"2023-10-09T06:15:22.912281+0000","last_fullsized":"2023-10-09T06:15:22.912281+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:26.508690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":382,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.911483+0000","last_change":"2023-10-09T06:13:21.728009+0000","last_active":"2023-10-09T06:15:22.911483+0000","last_peered":"2023-10-09T06:15:22.911483+0000","last_clean":"2023-10-09T06:15:22.911483+0000","last_became_active":"2023-10-09T06:13:21.727683+0000","last_became_peered":"2023-10-09T06:13:21.727683+0000","last_unstale":"2023-10-09T06:15:22.911483+0000","last_undegraded":"2023-10-09T06:15:22.911483+0000","last_fullsized":"2023-10-09T06:15:22.911483+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:13.074232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":370,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.909404+0000","last_change":"2023-10-09T06:13:30.782038+0000","last_active":"2023-10-09T06:15:22.909404+0000","last_peered":"2023-10-09T06:15:22.909404+0000","last_clean":"2023-10-09T06:15:22.909404+0000","last_became_active":"2023-10-09T06:13:30.781500+0000","last_became_peered":"2023-10-09T06:13:30.781500+0000","last_unstale":"2023-10-09T06:15:22.909404+0000","last_undegraded":"2023-10-09T06:15:22.909404+0000","last_fullsized":"2023-10-09T06:15:22.909404+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:43.663265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"565'10930","reported_seq":11015,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:25.462036+0000","last_change":"2023-10-09T06:11:46.031709+0000","last_active":"2023-10-09T06:15:25.462036+0000","last_peered":"2023-10-09T06:15:25.462036+0000","last_clean":"2023-10-09T06:15:25.462036+0000","last_became_active":"2023-10-09T06:11:46.031421+0000","last_became_peered":"2023-10-09T06:11:46.031421+0000","last_unstale":"2023-10-09T06:15:25.462036+0000","last_undegraded":"2023-10-09T06:15:25.462036+0000","last_fullsized":"2023-10-09T06:15:25.462036+0000","mapping_epoch":490,"log_start":"513'5025","ondisk_log_start":"513'5025","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5905,"log_dups_size":0,"ondisk_log_size":5905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:13.095174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64880640,"num_objects":990,"num_object_clones":0,"num_object_copies":2970,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":990,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10930,"num_write_kb":381440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":897,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.910227+0000","last_change":"2023-10-09T06:13:33.708100+0000","last_active":"2023-10-09T06:15:22.910227+0000","last_peered":"2023-10-09T06:15:22.910227+0000","last_clean":"2023-10-09T06:15:22.910227+0000","last_became_active":"2023-10-09T06:13:33.707809+0000","last_became_peered":"2023-10-09T06:13:33.707809+0000","last_unstale":"2023-10-09T06:15:22.910227+0000","last_undegraded":"2023-10-09T06:15:22.910227+0000","last_fullsized":"2023-10-09T06:15:22.910227+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:43:26.848778+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":48,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.063015+0000","last_change":"2023-10-09T06:13:30.799948+0000","last_active":"2023-10-09T06:15:23.063015+0000","last_peered":"2023-10-09T06:15:23.063015+0000","last_clean":"2023-10-09T06:15:23.063015+0000","last_became_active":"2023-10-09T06:13:30.799690+0000","last_became_peered":"2023-10-09T06:13:30.799690+0000","last_unstale":"2023-10-09T06:15:23.063015+0000","last_undegraded":"2023-10-09T06:15:23.063015+0000","last_fullsized":"2023-10-09T06:15:23.063015+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:43.893184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":611,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.391489+0000","last_change":"2023-10-09T06:09:27.578756+0000","last_active":"2023-10-09T06:15:23.391489+0000","last_peered":"2023-10-09T06:15:23.391489+0000","last_clean":"2023-10-09T06:15:23.391489+0000","last_became_active":"2023-10-09T06:09:27.578384+0000","last_became_peered":"2023-10-09T06:09:27.578384+0000","last_unstale":"2023-10-09T06:15:23.391489+0000","last_undegraded":"2023-10-09T06:15:23.391489+0000","last_fullsized":"2023-10-09T06:15:23.391489+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:22.182630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":511,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.917164+0000","last_change":"2023-10-09T06:01:22.334081+0000","last_active":"2023-10-09T06:15:22.917164+0000","last_peered":"2023-10-09T06:15:22.917164+0000","last_clean":"2023-10-09T06:15:22.917164+0000","last_became_active":"2023-10-09T06:01:22.333663+0000","last_became_peered":"2023-10-09T06:01:22.333663+0000","last_unstale":"2023-10-09T06:15:22.917164+0000","last_undegraded":"2023-10-09T06:15:22.917164+0000","last_fullsized":"2023-10-09T06:15:22.917164+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:55.809259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":811,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.909387+0000","last_change":"2023-10-09T06:09:27.560157+0000","last_active":"2023-10-09T06:15:22.909387+0000","last_peered":"2023-10-09T06:15:22.909387+0000","last_clean":"2023-10-09T06:15:22.909387+0000","last_became_active":"2023-10-09T06:09:27.559466+0000","last_became_peered":"2023-10-09T06:09:27.559466+0000","last_unstale":"2023-10-09T06:15:22.909387+0000","last_undegraded":"2023-10-09T06:15:22.909387+0000","last_fullsized":"2023-10-09T06:15:22.909387+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.181811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.181811+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:53.431491+0000","scrub_duration":0.011553313000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":494,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.390522+0000","last_change":"2023-10-09T06:13:30.791343+0000","last_active":"2023-10-09T06:15:23.390522+0000","last_peered":"2023-10-09T06:15:23.390522+0000","last_clean":"2023-10-09T06:15:23.390522+0000","last_became_active":"2023-10-09T06:13:30.790945+0000","last_became_peered":"2023-10-09T06:13:30.790945+0000","last_unstale":"2023-10-09T06:15:23.390522+0000","last_undegraded":"2023-10-09T06:15:23.390522+0000","last_fullsized":"2023-10-09T06:15:23.390522+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:14.208391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":276,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.911436+0000","last_change":"2023-10-09T06:15:16.803457+0000","last_active":"2023-10-09T06:15:22.911436+0000","last_peered":"2023-10-09T06:15:22.911436+0000","last_clean":"2023-10-09T06:15:22.911436+0000","last_became_active":"2023-10-09T06:15:16.803097+0000","last_became_peered":"2023-10-09T06:15:16.803097+0000","last_unstale":"2023-10-09T06:15:22.911436+0000","last_undegraded":"2023-10-09T06:15:22.911436+0000","last_fullsized":"2023-10-09T06:15:22.911436+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:45.511763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":914,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.909547+0000","last_change":"2023-10-09T06:00:04.215425+0000","last_active":"2023-10-09T06:15:22.909547+0000","last_peered":"2023-10-09T06:15:22.909547+0000","last_clean":"2023-10-09T06:15:22.909547+0000","last_became_active":"2023-10-09T05:59:40.326685+0000","last_became_peered":"2023-10-09T05:59:40.326685+0000","last_unstale":"2023-10-09T06:15:22.909547+0000","last_undegraded":"2023-10-09T06:15:22.909547+0000","last_fullsized":"2023-10-09T06:15:22.909547+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:04.215270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:04.215270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:03.963694+0000","scrub_duration":0.002850652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":336,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.917191+0000","last_change":"2023-10-09T06:01:54.824780+0000","last_active":"2023-10-09T06:15:22.917191+0000","last_peered":"2023-10-09T06:15:22.917191+0000","last_clean":"2023-10-09T06:15:22.917191+0000","last_became_active":"2023-10-09T06:01:54.757727+0000","last_became_peered":"2023-10-09T06:01:54.757727+0000","last_unstale":"2023-10-09T06:15:22.917191+0000","last_undegraded":"2023-10-09T06:15:22.917191+0000","last_fullsized":"2023-10-09T06:15:22.917191+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:44.132556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":583,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.917286+0000","last_change":"2023-10-09T06:01:54.764989+0000","last_active":"2023-10-09T06:15:22.917286+0000","last_peered":"2023-10-09T06:15:22.917286+0000","last_clean":"2023-10-09T06:15:22.917286+0000","last_became_active":"2023-10-09T06:01:54.764790+0000","last_became_peered":"2023-10-09T06:01:54.764790+0000","last_unstale":"2023-10-09T06:15:22.917286+0000","last_undegraded":"2023-10-09T06:15:22.917286+0000","last_fullsized":"2023-10-09T06:15:22.917286+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:14:07.208858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":350,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.916846+0000","last_change":"2023-10-09T06:13:21.739730+0000","last_active":"2023-10-09T06:15:22.916846+0000","last_peered":"2023-10-09T06:15:22.916846+0000","last_clean":"2023-10-09T06:15:22.916846+0000","last_became_active":"2023-10-09T06:13:21.727326+0000","last_became_peered":"2023-10-09T06:13:21.727326+0000","last_unstale":"2023-10-09T06:15:22.916846+0000","last_undegraded":"2023-10-09T06:15:22.916846+0000","last_fullsized":"2023-10-09T06:15:22.916846+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:51.344571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":580,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.916700+0000","last_change":"2023-10-09T06:09:27.565198+0000","last_active":"2023-10-09T06:15:22.916700+0000","last_peered":"2023-10-09T06:15:22.916700+0000","last_clean":"2023-10-09T06:15:22.916700+0000","last_became_active":"2023-10-09T06:09:27.564909+0000","last_became_peered":"2023-10-09T06:09:27.564909+0000","last_unstale":"2023-10-09T06:15:22.916700+0000","last_undegraded":"2023-10-09T06:15:22.916700+0000","last_fullsized":"2023-10-09T06:15:22.916700+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:46.101014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":730,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.062656+0000","last_change":"2023-10-09T06:13:33.699235+0000","last_active":"2023-10-09T06:15:23.062656+0000","last_peered":"2023-10-09T06:15:23.062656+0000","last_clean":"2023-10-09T06:15:23.062656+0000","last_became_active":"2023-10-09T06:13:33.698886+0000","last_became_peered":"2023-10-09T06:13:33.698886+0000","last_unstale":"2023-10-09T06:15:23.062656+0000","last_undegraded":"2023-10-09T06:15:23.062656+0000","last_fullsized":"2023-10-09T06:15:23.062656+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:00.456132+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:00.456132+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:59.421706+0000","scrub_duration":0.0014992199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":380,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.909913+0000","last_change":"2023-10-09T06:15:16.804361+0000","last_active":"2023-10-09T06:15:22.909913+0000","last_peered":"2023-10-09T06:15:22.909913+0000","last_clean":"2023-10-09T06:15:22.909913+0000","last_became_active":"2023-10-09T06:15:16.803977+0000","last_became_peered":"2023-10-09T06:15:16.803977+0000","last_unstale":"2023-10-09T06:15:22.909913+0000","last_undegraded":"2023-10-09T06:15:22.909913+0000","last_fullsized":"2023-10-09T06:15:22.909913+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:14.141738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":367,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.909779+0000","last_change":"2023-10-09T06:09:27.565185+0000","last_active":"2023-10-09T06:15:22.909779+0000","last_peered":"2023-10-09T06:15:22.909779+0000","last_clean":"2023-10-09T06:15:22.909779+0000","last_became_active":"2023-10-09T06:09:27.564478+0000","last_became_peered":"2023-10-09T06:09:27.564478+0000","last_unstale":"2023-10-09T06:15:22.909779+0000","last_undegraded":"2023-10-09T06:15:22.909779+0000","last_fullsized":"2023-10-09T06:15:22.909779+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:04.622171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":889,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.911062+0000","last_change":"2023-10-09T06:12:55.753877+0000","last_active":"2023-10-09T06:15:22.911062+0000","last_peered":"2023-10-09T06:15:22.911062+0000","last_clean":"2023-10-09T06:15:22.911062+0000","last_became_active":"2023-10-09T06:12:55.753512+0000","last_became_peered":"2023-10-09T06:12:55.753512+0000","last_unstale":"2023-10-09T06:15:22.911062+0000","last_undegraded":"2023-10-09T06:15:22.911062+0000","last_fullsized":"2023-10-09T06:15:22.911062+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:46.290510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:46.290510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:30.368031+0000","scrub_duration":0.002627819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":572,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.391481+0000","last_change":"2023-10-09T06:15:16.814498+0000","last_active":"2023-10-09T06:15:23.391481+0000","last_peered":"2023-10-09T06:15:23.391481+0000","last_clean":"2023-10-09T06:15:23.391481+0000","last_became_active":"2023-10-09T06:15:16.807217+0000","last_became_peered":"2023-10-09T06:15:16.807217+0000","last_unstale":"2023-10-09T06:15:23.391481+0000","last_undegraded":"2023-10-09T06:15:23.391481+0000","last_fullsized":"2023-10-09T06:15:23.391481+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:50.886043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":596,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T06:15:22.916568+0000","last_change":"2023-10-09T06:13:21.727720+0000","last_active":"2023-10-09T06:15:22.916568+0000","last_peered":"2023-10-09T06:15:22.916568+0000","last_clean":"2023-10-09T06:15:22.916568+0000","last_became_active":"2023-10-09T06:13:21.727339+0000","last_became_peered":"2023-10-09T06:13:21.727339+0000","last_unstale":"2023-10-09T06:15:22.916568+0000","last_undegraded":"2023-10-09T06:15:22.916568+0000","last_fullsized":"2023-10-09T06:15:22.916568+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:25.723752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1019150360,"num_objects":15552,"num_object_clones":0,"num_object_copies":46656,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15552,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":176874,"num_write_kb":6157569,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44071,"num_bytes_recovered":2887909400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4300943360,"data_stored":4300931144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95623,"ondisk_log_size":95623,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":59,"num_bytes_recovered":12268,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13410,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1706112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234429,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1181020,"kb_used_data":284308,"kb_used_omap":0,"kb_used_meta":896704,"kb_avail":92564132,"statfs":{"total":95995035648,"available":94785671168,"internally_reserved":0,"allocated":291131392,"data_stored":289548841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918224896},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:14:32 2023","interfaces":[{"interface":"back","average":{"1min":14.369,"5min":5.5149999999999997,"15min":2.6600000000000001},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":126.167,"5min":126.167,"15min":126.167},"last":72.067999999999998},{"interface":"front","average":{"1min":14.465,"5min":5.7229999999999999,"15min":2.6709999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":126.139,"5min":126.139,"15min":126.139},"last":72.140000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:15:07 2023","interfaces":[{"interface":"back","average":{"1min":12.173,"5min":4.9900000000000002,"15min":3.9649999999999999},"min":{"1min":0.35499999999999998,"5min":0.255,"15min":0.218},"max":{"1min":108.113,"5min":108.113,"15min":124.66200000000001},"last":73.069999999999993},{"interface":"front","average":{"1min":10.855,"5min":4.8049999999999997,"15min":3.863},"min":{"1min":0.34699999999999998,"5min":0.33900000000000002,"15min":0.22700000000000001},"max":{"1min":108.503,"5min":108.503,"15min":125.023},"last":75.390000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:14:32 2023","interfaces":[{"interface":"back","average":{"1min":3.766,"5min":2.1549999999999998,"15min":2.403},"min":{"1min":0.29899999999999999,"5min":0.27400000000000002,"15min":0.24299999999999999},"max":{"1min":28.475999999999999,"5min":47.503,"15min":127.86499999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":3.4929999999999999,"5min":2.3719999999999999,"15min":2.391},"min":{"1min":0.30599999999999999,"5min":0.30099999999999999,"15min":0.27000000000000002},"max":{"1min":28.925000000000001,"5min":47.545000000000002,"15min":129.24000000000001},"last":0.48799999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:15:07 2023","interfaces":[{"interface":"back","average":{"1min":6.7400000000000002,"5min":7.7830000000000004,"15min":6.0800000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.29899999999999999},"max":{"1min":47.875,"5min":116.68000000000001,"15min":164.16999999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":5.391,"5min":7.6420000000000003,"15min":6.1580000000000004},"min":{"1min":0.34599999999999997,"5min":0.33300000000000002,"15min":0.32100000000000001},"max":{"1min":48.036999999999999,"5min":116.797,"15min":164.40700000000001},"last":0.50800000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:14:41 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":0.98499999999999999,"15min":0.95799999999999996},"min":{"1min":0.27900000000000003,"5min":0.23999999999999999,"15min":0.23100000000000001},"max":{"1min":6.9939999999999998,"5min":6.9939999999999998,"15min":14.371},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.3520000000000001,"5min":0.90100000000000002,"15min":0.94799999999999995},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":7.9459999999999997,"5min":7.9459999999999997,"15min":14.259},"last":0.38700000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":1.3420000000000001,"15min":1.083},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.24399999999999999},"max":{"1min":2.4700000000000002,"5min":43.826999999999998,"15min":43.826999999999998},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.038,"5min":1.2729999999999999,"15min":1.139},"min":{"1min":0.34899999999999998,"5min":0.32300000000000001,"15min":0.24399999999999999},"max":{"1min":5.9569999999999999,"5min":43.880000000000003,"15min":43.880000000000003},"last":0.432}]},{"osd":6,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":0.78600000000000003,"15min":0.97999999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.26600000000000001},"max":{"1min":2.468,"5min":3.8679999999999999,"15min":9.4540000000000006},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.90200000000000002,"5min":0.80600000000000005,"15min":1.018},"min":{"1min":0.33600000000000002,"5min":0.32000000000000001,"15min":0.30599999999999999},"max":{"1min":3.3279999999999998,"5min":4.516,"15min":6.2329999999999997},"last":0.35499999999999998}]}]},{"osd":6,"up_from":46,"seq":197568496066,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1388880,"kb_used_data":495432,"kb_used_omap":0,"kb_used_meta":893440,"kb_avail":92356272,"statfs":{"total":95995035648,"available":94572822528,"internally_reserved":0,"allocated":507322368,"data_stored":505751728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":914882560},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:14:29 2023","interfaces":[{"interface":"back","average":{"1min":11.574,"5min":8.9060000000000006,"15min":4.4640000000000004},"min":{"1min":0.25,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":92.424999999999997,"5min":191.50800000000001,"15min":191.50800000000001},"last":4.4489999999999998},{"interface":"front","average":{"1min":11.849,"5min":9.0709999999999997,"15min":4.3490000000000002},"min":{"1min":0.30399999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":92.396000000000001,"5min":191.52600000000001,"15min":191.52600000000001},"last":4.4630000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:14:28 2023","interfaces":[{"interface":"back","average":{"1min":13.445,"5min":8.6509999999999998,"15min":6.0129999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.23200000000000001},"max":{"1min":193.845,"5min":328.04399999999998,"15min":328.04399999999998},"last":8.0299999999999994},{"interface":"front","average":{"1min":13.298,"5min":8.8420000000000005,"15min":6.1219999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":193.779,"5min":328.10000000000002,"15min":328.10000000000002},"last":0.58099999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:14:29 2023","interfaces":[{"interface":"back","average":{"1min":1.5600000000000001,"5min":4.7709999999999999,"15min":2.944},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.7549999999999999,"5min":163.04900000000001,"15min":163.04900000000001},"last":2.0459999999999998},{"interface":"front","average":{"1min":1.3759999999999999,"5min":4.8239999999999998,"15min":2.879},"min":{"1min":0.32100000000000001,"5min":0.312,"15min":0.28599999999999998},"max":{"1min":5.9820000000000002,"5min":162.93000000000001,"15min":162.93000000000001},"last":4.1120000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:15:15 2023","interfaces":[{"interface":"back","average":{"1min":5.2990000000000004,"5min":9.1899999999999995,"15min":7.5970000000000004},"min":{"1min":0.316,"5min":0.316,"15min":0.307},"max":{"1min":48.134999999999998,"5min":250.744,"15min":250.744},"last":0.34000000000000002},{"interface":"front","average":{"1min":5.4299999999999997,"5min":8.6989999999999998,"15min":7.3659999999999997},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":48.097000000000001,"5min":250.83799999999999,"15min":250.83799999999999},"last":0.41499999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":1.018,"15min":1.6519999999999999},"min":{"1min":0.28899999999999998,"5min":0.24399999999999999,"15min":0.24199999999999999},"max":{"1min":5.4880000000000004,"5min":9.0869999999999997,"15min":151.35599999999999},"last":0.26500000000000001},{"interface":"front","average":{"1min":1.091,"5min":1.121,"15min":1.7350000000000001},"min":{"1min":0.307,"5min":0.249,"15min":0.23999999999999999},"max":{"1min":5.077,"5min":7.8849999999999998,"15min":149.15799999999999},"last":0.28599999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:14:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":0.94299999999999995,"15min":1.1040000000000001},"min":{"1min":0.315,"5min":0.28000000000000003,"15min":0.23000000000000001},"max":{"1min":8.3829999999999991,"5min":9.1959999999999997,"15min":15.342000000000001},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.95699999999999996,"5min":1.03,"15min":1.1579999999999999},"min":{"1min":0.316,"5min":0.28699999999999998,"15min":0.255},"max":{"1min":2.8450000000000002,"5min":8.9809999999999999,"15min":15.358000000000001},"last":0.36899999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:14:48 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":0.96999999999999997,"15min":1.238},"min":{"1min":0.33700000000000002,"5min":0.30299999999999999,"15min":0.27000000000000002},"max":{"1min":5.8380000000000001,"5min":5.8380000000000001,"15min":21.98},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.01,"5min":0.86399999999999999,"15min":1.1739999999999999},"min":{"1min":0.34999999999999998,"5min":0.34200000000000003,"15min":0.33100000000000002},"max":{"1min":3.714,"5min":3.714,"15min":18.241},"last":0.40500000000000003}]}]},{"osd":1,"up_from":89,"seq":382252089654,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":833260,"kb_used_data":182692,"kb_used_omap":9,"kb_used_meta":650550,"kb_avail":92911892,"statfs":{"total":95995035648,"available":95141777408,"internally_reserved":0,"allocated":187076608,"data_stored":183844560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":666164201},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:14:39 2023","interfaces":[{"interface":"back","average":{"1min":4.2060000000000004,"5min":2.3319999999999999,"15min":2.294},"min":{"1min":0.221,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":35.939999999999998,"5min":35.939999999999998,"15min":35.939999999999998},"last":11.102},{"interface":"front","average":{"1min":4.3760000000000003,"5min":2.798,"15min":2.492},"min":{"1min":0.21199999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":35.326000000000001,"5min":35.326000000000001,"15min":35.326000000000001},"last":0.40300000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:14:45 2023","interfaces":[{"interface":"back","average":{"1min":1.2849999999999999,"5min":1.7929999999999999,"15min":2.4630000000000001},"min":{"1min":0.29899999999999999,"5min":0.217,"15min":0.20799999999999999},"max":{"1min":4.8929999999999998,"5min":15.522,"15min":129.053},"last":8.5969999999999995},{"interface":"front","average":{"1min":4.8550000000000004,"5min":2.9409999999999998,"15min":2.9350000000000001},"min":{"1min":0.32400000000000001,"5min":0.22600000000000001,"15min":0.222},"max":{"1min":55.694000000000003,"5min":55.694000000000003,"15min":129.077},"last":0.45900000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:15:13 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":4.3899999999999997,"15min":4.2939999999999996},"min":{"1min":0.318,"5min":0.26800000000000002,"15min":0.24199999999999999},"max":{"1min":12.961,"5min":113.322,"15min":142.97},"last":5.3310000000000004},{"interface":"front","average":{"1min":1.712,"5min":4.407,"15min":4.2590000000000003},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":4.5679999999999996,"5min":113.38200000000001,"15min":142.94800000000001},"last":6.1820000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:14:45 2023","interfaces":[{"interface":"back","average":{"1min":3.7559999999999998,"5min":3.4860000000000002,"15min":2.3149999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":54.018000000000001,"5min":93.747,"15min":93.747},"last":0.63100000000000001},{"interface":"front","average":{"1min":5.0860000000000003,"5min":4.4219999999999997,"15min":2.8319999999999999},"min":{"1min":0.35799999999999998,"5min":0.308,"15min":0.308},"max":{"1min":54.280000000000001,"5min":97.852000000000004,"15min":97.852000000000004},"last":0.71799999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.4730000000000001,"15min":1.5820000000000001},"min":{"1min":0.312,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.492,"5min":9.3109999999999999,"15min":36.356999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":2.2639999999999998,"5min":2.5099999999999998,"15min":2.319},"min":{"1min":0.318,"5min":0.318,"15min":0.29799999999999999},"max":{"1min":14.648,"5min":17.603999999999999,"15min":36.268000000000001},"last":0.39000000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":1.6399999999999999,"15min":1.8180000000000001},"min":{"1min":0.36699999999999999,"5min":0.33700000000000002,"15min":0.318},"max":{"1min":5.2050000000000001,"5min":9.282,"15min":52.207000000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":2.0579999999999998,"5min":2.2810000000000001,"15min":2.375},"min":{"1min":0.36199999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":14.619999999999999,"5min":15.384,"15min":51.212000000000003},"last":0.46999999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:14:58 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":3.3919999999999999,"15min":2.984},"min":{"1min":0.39400000000000002,"5min":0.374,"15min":0.374},"max":{"1min":14.571,"5min":56.881,"15min":108.553},"last":0.435},{"interface":"front","average":{"1min":1.4019999999999999,"5min":2.3900000000000001,"15min":2.4009999999999998},"min":{"1min":0.42999999999999999,"5min":0.38,"15min":0.38},"max":{"1min":5.9420000000000002,"5min":56.941000000000003,"15min":108.56999999999999},"last":4.6550000000000002}]}]},{"osd":0,"up_from":456,"seq":1958505087049,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1043508,"kb_used_data":759084,"kb_used_omap":32,"kb_used_meta":284191,"kb_avail":92701644,"statfs":{"total":95995035648,"available":94926483456,"internally_reserved":0,"allocated":777302016,"data_stored":774301499,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32794,"internal_metadata":291012582},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:14:40 2023","interfaces":[{"interface":"back","average":{"1min":15.137,"5min":6.5940000000000003,"15min":9.6199999999999992},"min":{"1min":0.24099999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":102.81999999999999,"5min":166.833,"15min":166.833},"last":1.048},{"interface":"front","average":{"1min":14.031000000000001,"5min":6.9859999999999998,"15min":8.9900000000000002},"min":{"1min":0.28699999999999998,"5min":0.223,"15min":0.223},"max":{"1min":103.108,"5min":166.81100000000001,"15min":166.81100000000001},"last":0.19700000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:14:40 2023","interfaces":[{"interface":"back","average":{"1min":2.7400000000000002,"5min":2.649,"15min":3.589},"min":{"1min":0.30599999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":12.413,"5min":21.648,"15min":21.648},"last":2.0710000000000002},{"interface":"front","average":{"1min":2.21,"5min":2.3210000000000002,"15min":3.375},"min":{"1min":0.25700000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":10.119,"5min":20.152000000000001,"15min":20.152000000000001},"last":1.121}]},{"osd":3,"last update":"Mon Oct 9 06:14:32 2023","interfaces":[{"interface":"back","average":{"1min":3.2850000000000001,"5min":5.0039999999999996,"15min":12.284000000000001},"min":{"1min":0.38700000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":10.079000000000001,"5min":203.59299999999999,"15min":203.59299999999999},"last":1.077},{"interface":"front","average":{"1min":3.2440000000000002,"5min":5.3019999999999996,"15min":11.779999999999999},"min":{"1min":0.30299999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":12.191000000000001,"5min":202.96600000000001,"15min":202.96600000000001},"last":2.105}]},{"osd":4,"last update":"Mon Oct 9 06:14:40 2023","interfaces":[{"interface":"back","average":{"1min":3.9289999999999998,"5min":2.3700000000000001,"15min":2.3959999999999999},"min":{"1min":0.39400000000000002,"5min":0.317,"15min":0.317},"max":{"1min":21.994,"5min":23.515999999999998,"15min":23.515999999999998},"last":1.103},{"interface":"front","average":{"1min":3.3319999999999999,"5min":2.7839999999999998,"15min":2.2559999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":21.972999999999999,"5min":39.302,"15min":39.302},"last":0.254}]},{"osd":5,"last update":"Mon Oct 9 06:14:40 2023","interfaces":[{"interface":"back","average":{"1min":2.6179999999999999,"5min":3.8050000000000002,"15min":2.7280000000000002},"min":{"1min":0.32700000000000001,"5min":0.316,"15min":0.316},"max":{"1min":8.2949999999999999,"5min":161.286,"15min":161.286},"last":0.29499999999999998},{"interface":"front","average":{"1min":2.8010000000000002,"5min":3.597,"15min":2.8479999999999999},"min":{"1min":0.40000000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":12.038,"5min":161.31800000000001,"15min":161.31800000000001},"last":2.0049999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:14:40 2023","interfaces":[{"interface":"back","average":{"1min":4.657,"5min":2.5550000000000002,"15min":2.5030000000000001},"min":{"1min":0.43099999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":31.648,"5min":31.648,"15min":31.648},"last":2.4910000000000001},{"interface":"front","average":{"1min":3.5910000000000002,"5min":2.081,"15min":2.585},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":24.43,"5min":24.43,"15min":24.43},"last":1.1479999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:14:40 2023","interfaces":[{"interface":"back","average":{"1min":2.859,"5min":3.5720000000000001,"15min":2.879},"min":{"1min":0.53700000000000003,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":12.066000000000001,"5min":123.685,"15min":123.685},"last":2.0449999999999999},{"interface":"front","average":{"1min":2.4089999999999998,"5min":3.1429999999999998,"15min":2.8889999999999998},"min":{"1min":0.55900000000000005,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":10.054,"5min":122.482,"15min":122.482},"last":1.1339999999999999}]}]},{"osd":2,"up_from":94,"seq":403726926145,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1299824,"kb_used_data":516712,"kb_used_omap":81,"kb_used_meta":783022,"kb_avail":92445328,"statfs":{"total":95995035648,"available":94664015872,"internally_reserved":0,"allocated":529113088,"data_stored":526275096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":801814746},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:14:41 2023","interfaces":[{"interface":"back","average":{"1min":7.6920000000000002,"5min":4.415,"15min":3.5710000000000002},"min":{"1min":0.35999999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":83.652000000000001,"5min":83.652000000000001,"15min":83.652000000000001},"last":0.92200000000000004},{"interface":"front","average":{"1min":7.3620000000000001,"5min":3.8380000000000001,"15min":2.774},"min":{"1min":0.35399999999999998,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":77.578999999999994,"5min":77.578999999999994,"15min":77.578999999999994},"last":0.72199999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:14:41 2023","interfaces":[{"interface":"back","average":{"1min":7.0499999999999998,"5min":5.8970000000000002,"15min":4.9119999999999999},"min":{"1min":0.27300000000000002,"5min":0.248,"15min":0.19400000000000001},"max":{"1min":78.921999999999997,"5min":160.45500000000001,"15min":222.59399999999999},"last":5.5670000000000002},{"interface":"front","average":{"1min":8.5739999999999998,"5min":7.0979999999999999,"15min":5.4329999999999998},"min":{"1min":0.313,"5min":0.27200000000000002,"15min":0.20200000000000001},"max":{"1min":81.292000000000002,"5min":163.697,"15min":224.619},"last":0.24099999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:15:04 2023","interfaces":[{"interface":"back","average":{"1min":10.073,"5min":8.1189999999999998,"15min":5.0279999999999996},"min":{"1min":0.57099999999999995,"5min":0.23100000000000001,"15min":0.23000000000000001},"max":{"1min":78.385000000000005,"5min":122.205,"15min":122.205},"last":5.4870000000000001},{"interface":"front","average":{"1min":9.5399999999999991,"5min":7.7009999999999996,"15min":4.944},"min":{"1min":0.28599999999999998,"5min":0.252,"15min":0.252},"max":{"1min":78.436000000000007,"5min":122.45,"15min":122.45},"last":5.532}]},{"osd":4,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":2.8210000000000002,"5min":3.4449999999999998,"15min":2.2989999999999999},"min":{"1min":0.33600000000000002,"5min":0.32200000000000001,"15min":0.28299999999999997},"max":{"1min":9.3849999999999998,"5min":107.51300000000001,"15min":107.51300000000001},"last":0.28999999999999998},{"interface":"front","average":{"1min":1.645,"5min":2.4580000000000002,"15min":2.0409999999999999},"min":{"1min":0.36299999999999999,"5min":0.34100000000000003,"15min":0.31},"max":{"1min":4.6589999999999998,"5min":108.768,"15min":108.768},"last":0.77900000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:14:41 2023","interfaces":[{"interface":"back","average":{"1min":1.788,"5min":1.528,"15min":1.5569999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32600000000000001},"max":{"1min":11.108000000000001,"5min":11.108000000000001,"15min":28.48},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.8500000000000001,"5min":1.516,"15min":1.79},"min":{"1min":0.40200000000000002,"5min":0.316,"15min":0.28899999999999998},"max":{"1min":4.6890000000000001,"5min":9.8699999999999992,"15min":30.318999999999999},"last":0.82099999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:14:41 2023","interfaces":[{"interface":"back","average":{"1min":2.7890000000000001,"5min":2.407,"15min":2.1850000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.314},"max":{"1min":10.09,"5min":14.962,"15min":30.420000000000002},"last":0.35799999999999998},{"interface":"front","average":{"1min":2.1739999999999999,"5min":1.6619999999999999,"15min":1.7969999999999999},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.32200000000000001},"max":{"1min":6.3940000000000001,"5min":11.103,"15min":30.713000000000001},"last":0.41299999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:14:41 2023","interfaces":[{"interface":"back","average":{"1min":2.2469999999999999,"5min":2.919,"15min":2.2959999999999998},"min":{"1min":0.39400000000000002,"5min":0.313,"15min":0.313},"max":{"1min":11.147,"5min":124.069,"15min":124.069},"last":0.43099999999999999},{"interface":"front","average":{"1min":2.3660000000000001,"5min":2.7170000000000001,"15min":2.3450000000000002},"min":{"1min":0.372,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.6769999999999996,"5min":124.08799999999999,"15min":124.08799999999999},"last":0.85099999999999998}]}]},{"osd":3,"up_from":99,"seq":425201762619,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1659904,"kb_used_data":698168,"kb_used_omap":7,"kb_used_meta":961656,"kb_avail":92085248,"statfs":{"total":95995035648,"available":94295293952,"internally_reserved":0,"allocated":714924032,"data_stored":711727238,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":984736360},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:14:36 2023","interfaces":[{"interface":"back","average":{"1min":20.771999999999998,"5min":5.6909999999999998,"15min":3.3029999999999999},"min":{"1min":0.19600000000000001,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":220.45400000000001,"5min":220.45400000000001,"15min":220.45400000000001},"last":9.0340000000000007},{"interface":"front","average":{"1min":21.698,"5min":6.5,"15min":4.5170000000000003},"min":{"1min":0.317,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":216.44200000000001,"5min":216.44200000000001,"15min":216.44200000000001},"last":7.1520000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:15:10 2023","interfaces":[{"interface":"back","average":{"1min":3.6549999999999998,"5min":2.3039999999999998,"15min":3.1070000000000002},"min":{"1min":0.20300000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":30.632000000000001,"5min":30.632000000000001,"15min":207.75299999999999},"last":9.0630000000000006},{"interface":"front","average":{"1min":4.157,"5min":3.2519999999999998,"15min":4.1429999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.23200000000000001},"max":{"1min":30.465,"5min":30.524000000000001,"15min":207.12200000000001},"last":1.3660000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:15:10 2023","interfaces":[{"interface":"back","average":{"1min":2.3900000000000001,"5min":2.2530000000000001,"15min":2.6560000000000001},"min":{"1min":0.26000000000000001,"5min":0.23400000000000001,"15min":0.223},"max":{"1min":5.6189999999999998,"5min":30.547999999999998,"15min":107.01300000000001},"last":2.0880000000000001},{"interface":"front","average":{"1min":2.891,"5min":2.7189999999999999,"15min":3.1320000000000001},"min":{"1min":0.29699999999999999,"5min":0.25600000000000001,"15min":0.23899999999999999},"max":{"1min":11.542,"5min":25.984000000000002,"15min":109.807},"last":7.665}]},{"osd":4,"last update":"Mon Oct 9 06:14:50 2023","interfaces":[{"interface":"back","average":{"1min":1.7929999999999999,"5min":3.008,"15min":3.2029999999999998},"min":{"1min":0.29699999999999999,"5min":0.28199999999999997,"15min":0.254},"max":{"1min":12.523999999999999,"5min":95.739000000000004,"15min":99.876000000000005},"last":2.0510000000000002},{"interface":"front","average":{"1min":2.351,"5min":3.2970000000000002,"15min":3.4340000000000002},"min":{"1min":0.35299999999999998,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":11.787000000000001,"5min":97.941999999999993,"15min":99.914000000000001},"last":1.405}]},{"osd":5,"last update":"Mon Oct 9 06:15:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8939999999999999,"5min":2.8410000000000002,"15min":2.1019999999999999},"min":{"1min":0.32300000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.8799999999999999,"5min":74.706000000000003,"15min":74.706000000000003},"last":8.9600000000000009},{"interface":"front","average":{"1min":1.9610000000000001,"5min":3.327,"15min":3.2719999999999998},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.28799999999999998},"max":{"1min":6.5529999999999999,"5min":74.685000000000002,"15min":235.56},"last":2.109}]},{"osd":6,"last update":"Mon Oct 9 06:15:10 2023","interfaces":[{"interface":"back","average":{"1min":1.774,"5min":1.4079999999999999,"15min":3.8399999999999999},"min":{"1min":0.35299999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":7.0430000000000001,"5min":7.0430000000000001,"15min":233.26599999999999},"last":9.0009999999999994},{"interface":"front","average":{"1min":2.8290000000000002,"5min":2.226,"15min":4.3419999999999996},"min":{"1min":0.40000000000000002,"5min":0.35999999999999999,"15min":0.33600000000000002},"max":{"1min":8.3360000000000003,"5min":26.890000000000001,"15min":233.24100000000001},"last":2.1309999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:15:10 2023","interfaces":[{"interface":"back","average":{"1min":2.5209999999999999,"5min":4.2599999999999998,"15min":4.6769999999999996},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.35699999999999998},"max":{"1min":7.0060000000000002,"5min":94.400000000000006,"15min":230.69399999999999},"last":2.153},{"interface":"front","average":{"1min":2.0859999999999999,"5min":4.274,"15min":4.8449999999999998},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.32600000000000001},"max":{"1min":11.311,"5min":94.195999999999998,"15min":230.67500000000001},"last":1.448}]}]},{"osd":4,"up_from":322,"seq":1382979469482,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1481764,"kb_used_data":866460,"kb_used_omap":35,"kb_used_meta":614812,"kb_avail":92263388,"statfs":{"total":95995035648,"available":94477709312,"internally_reserved":0,"allocated":887255040,"data_stored":885672189,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36552,"internal_metadata":629567800},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:14:32 2023","interfaces":[{"interface":"back","average":{"1min":4.5990000000000002,"5min":1.972,"15min":1.649},"min":{"1min":0.254,"5min":0.216,"15min":0.216},"max":{"1min":23.206,"5min":23.206,"15min":23.206},"last":0.38500000000000001},{"interface":"front","average":{"1min":3.323,"5min":1.7609999999999999,"15min":1.651},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":22.902999999999999,"5min":22.902999999999999,"15min":22.902999999999999},"last":0.32500000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":1.77,"5min":2.0489999999999999,"15min":2.2770000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.20999999999999999},"max":{"1min":4.8360000000000003,"5min":18.884,"15min":78.891000000000005},"last":0.84099999999999997},{"interface":"front","average":{"1min":1.905,"5min":1.802,"15min":2.2559999999999998},"min":{"1min":0.32200000000000001,"5min":0.25600000000000001,"15min":0.22800000000000001},"max":{"1min":10.039,"5min":18.617999999999999,"15min":78.921000000000006},"last":0.34200000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":2.9929999999999999,"5min":4.5960000000000001,"15min":2.6869999999999998},"min":{"1min":0.39200000000000002,"5min":0.29199999999999998,"15min":0.27200000000000002},"max":{"1min":10.084,"5min":245.166,"15min":245.166},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.845,"5min":4.3230000000000004,"15min":2.5329999999999999},"min":{"1min":0.28699999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.2080000000000002,"5min":245.119,"15min":245.119},"last":0.47899999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":3.601,"5min":6.8630000000000004,"15min":3.9740000000000002},"min":{"1min":0.34599999999999997,"5min":0.29399999999999998,"15min":0.26800000000000002},"max":{"1min":20.100999999999999,"5min":246.39400000000001,"15min":246.39400000000001},"last":0.86599999999999999},{"interface":"front","average":{"1min":3.7240000000000002,"5min":6.3700000000000001,"15min":3.5750000000000002},"min":{"1min":0.40899999999999997,"5min":0.314,"15min":0.314},"max":{"1min":19.984000000000002,"5min":246.99000000000001,"15min":246.99000000000001},"last":0.439}]},{"osd":5,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":6.9269999999999996,"5min":2.0880000000000001,"15min":1.3899999999999999},"min":{"1min":0.28899999999999998,"5min":0.23499999999999999,"15min":0.216},"max":{"1min":99.405000000000001,"5min":99.405000000000001,"15min":99.405000000000001},"last":4.9649999999999999},{"interface":"front","average":{"1min":7.2169999999999996,"5min":2.274,"15min":1.546},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.245},"max":{"1min":99.206000000000003,"5min":99.206000000000003,"15min":99.206000000000003},"last":0.81200000000000006}]},{"osd":6,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.0569999999999999,"15min":2.0859999999999999},"min":{"1min":0.32200000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.9900000000000002,"5min":7.641,"15min":95.492999999999995},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.7070000000000001,"5min":1.3300000000000001,"15min":2.25},"min":{"1min":0.33200000000000002,"5min":0.32100000000000001,"15min":0.27900000000000003},"max":{"1min":9.2110000000000003,"5min":9.3239999999999998,"15min":95.664000000000001},"last":5.1539999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":6.9320000000000004,"5min":2.5390000000000001,"15min":2.7200000000000002},"min":{"1min":0.38800000000000001,"5min":0.29699999999999999,"15min":0.29499999999999998},"max":{"1min":95.167000000000002,"5min":95.167000000000002,"15min":95.167000000000002},"last":0.89800000000000002},{"interface":"front","average":{"1min":6.8949999999999996,"5min":2.2599999999999998,"15min":2.403},"min":{"1min":0.34499999999999997,"5min":0.27800000000000002,"15min":0.251},"max":{"1min":94.811999999999998,"5min":94.811999999999998,"15min":94.811999999999998},"last":3.4369999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757702,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1443788,"kb_used_data":553412,"kb_used_omap":0,"kb_used_meta":890304,"kb_avail":92301364,"statfs":{"total":95995035648,"available":94516596736,"internally_reserved":0,"allocated":566693888,"data_stored":565123718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911671296},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:14:34 2023","interfaces":[{"interface":"back","average":{"1min":2.6160000000000001,"5min":3.5510000000000002,"15min":2.7850000000000001},"min":{"1min":0.26700000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":18.347999999999999,"5min":115.92400000000001,"15min":115.92400000000001},"last":14.207000000000001},{"interface":"front","average":{"1min":3.3650000000000002,"5min":3.8839999999999999,"15min":3.444},"min":{"1min":0.314,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":18.364999999999998,"5min":115.904,"15min":115.904},"last":5.2830000000000004}]},{"osd":1,"last update":"Mon Oct 9 06:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":3.734,"15min":4.4800000000000004},"min":{"1min":0.253,"5min":0.253,"15min":0.22600000000000001},"max":{"1min":4.0369999999999999,"5min":151.78200000000001,"15min":151.78200000000001},"last":0.30099999999999999},{"interface":"front","average":{"1min":1.097,"5min":3.8730000000000002,"15min":4.6319999999999997},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.3319999999999999,"5min":152.251,"15min":152.251},"last":0.40300000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:14:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":2.9390000000000001,"15min":2.9529999999999998},"min":{"1min":0.33600000000000002,"5min":0.26600000000000001,"15min":0.254},"max":{"1min":2.5009999999999999,"5min":117.97199999999999,"15min":129.21199999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.96799999999999997,"5min":3.2330000000000001,"15min":3.0129999999999999},"min":{"1min":0.35599999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.6160000000000001,"5min":118.52,"15min":129.042},"last":0.53800000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:15:04 2023","interfaces":[{"interface":"back","average":{"1min":3.2639999999999998,"5min":5.2110000000000003,"15min":7.0629999999999997},"min":{"1min":0.38700000000000001,"5min":0.312,"15min":0.29499999999999998},"max":{"1min":19.231000000000002,"5min":111.072,"15min":143.31200000000001},"last":0.44},{"interface":"front","average":{"1min":3.2879999999999998,"5min":5.6219999999999999,"15min":7.1369999999999996},"min":{"1min":0.36299999999999999,"5min":0.34499999999999997,"15min":0.29399999999999998},"max":{"1min":20.035,"5min":111.01300000000001,"15min":140.00899999999999},"last":0.34000000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:14:52 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":1.6279999999999999,"15min":1.278},"min":{"1min":0.28000000000000003,"5min":0.246,"15min":0.245},"max":{"1min":2.097,"5min":68.001999999999995,"15min":68.001999999999995},"last":1.03},{"interface":"front","average":{"1min":0.88300000000000001,"5min":1.5529999999999999,"15min":1.212},"min":{"1min":0.33900000000000002,"5min":0.26100000000000001,"15min":0.26000000000000001},"max":{"1min":5.7729999999999997,"5min":61.743000000000002,"15min":61.743000000000002},"last":0.38300000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:15:04 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.91000000000000003,"15min":1.0940000000000001},"min":{"1min":0.314,"5min":0.26200000000000001,"15min":0.251},"max":{"1min":4.492,"5min":7.4980000000000002,"15min":9.0909999999999993},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.014,"15min":1.1639999999999999},"min":{"1min":0.34599999999999997,"5min":0.29099999999999998,"15min":0.26700000000000002},"max":{"1min":3.0379999999999998,"5min":7.7140000000000004,"15min":9.9009999999999998},"last":0.373}]},{"osd":7,"last update":"Mon Oct 9 06:14:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3480000000000001,"5min":1.8380000000000001,"15min":2.456},"min":{"1min":0.36299999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.8289999999999997,"5min":64.766000000000005,"15min":157.471},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.85699999999999998,"5min":1.698,"15min":2.3410000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.3380000000000001,"5min":67.956999999999994,"15min":157.51599999999999},"last":0.57299999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4889,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":732172288,"data_stored":732168216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":176291840,"data_stored":176291840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":519766016,"data_stored":519766016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":689700864,"data_stored":689700864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":874582016,"data_stored":874577944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":546504704,"data_stored":546504704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":487067648,"data_stored":487063576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":274857984,"data_stored":274857984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:15:30.864 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T06:15:30.864 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 81 2023-10-09T06:15:31.058 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:30 smithi064 ceph-mon[212750]: pgmap v1433: 119 pgs: 119 active+clean; 973 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 399 op/s; 2.9 MiB/s, 46 objects/s recovering 2023-10-09T06:15:31.063 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:30 smithi064 ceph-mon[220044]: pgmap v1433: 119 pgs: 119 active+clean; 973 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 399 op/s; 2.9 MiB/s, 46 objects/s recovering 2023-10-09T06:15:31.104 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:15:31.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:31.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:15:31.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:30 smithi156 ceph-mon[195101]: pgmap v1433: 119 pgs: 119 active+clean; 973 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 399 op/s; 2.9 MiB/s, 46 objects/s recovering 2023-10-09T06:15:31.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:15:31.275 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:31.557 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:15:31.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:31.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:15:31.666 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:15:31.767 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:31.894 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:15:31.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:31.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:15:32.003 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:15:32.059 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:32.156 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:15:32.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:32.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:15:32.185 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:32.262 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:15:32.346 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:32.574 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:15:32.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:32.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:15:32.682 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:15:32.788 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:32.804 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:15:32.811 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:15:32.840 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:15:32.840 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T06:15:32.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T06:15:32.840 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:15:32.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.841 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116287 9.1 9:8655aad1:::benchmark_data_smithi064_511911_object15185:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-10-09T06:15:32.841 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.530424+0000", 2023-10-09T06:15:32.841 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 390.30355000399999, 2023-10-09T06:15:32.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.181524692, 2023-10-09T06:15:32.841 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:32.841 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:15:32.841 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:32.841 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:15:32.841 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:15:32.841 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116287 2023-10-09T06:15:32.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.842 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:32.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:32.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530424+0000", 2023-10-09T06:15:32.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:32.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530424+0000", 2023-10-09T06:15:32.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:32.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530422+0000", 2023-10-09T06:15:32.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:15:32.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:32.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530432+0000", 2023-10-09T06:15:32.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0105000000000001e-05 2023-10-09T06:15:32.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:32.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530454+0000", 2023-10-09T06:15:32.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1579000000000001e-05 2023-10-09T06:15:32.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:32.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530465+0000", 2023-10-09T06:15:32.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1044e-05 2023-10-09T06:15:32.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530519+0000", 2023-10-09T06:15:32.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4567000000000001e-05 2023-10-09T06:15:32.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:32.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530668+0000", 2023-10-09T06:15:32.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014871900000000001 2023-10-09T06:15:32.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:15:32.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530823+0000", 2023-10-09T06:15:32.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000155375 2023-10-09T06:15:32.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.533953+0000", 2023-10-09T06:15:32.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031297159999999998 2023-10-09T06:15:32.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:32.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.534034+0000", 2023-10-09T06:15:32.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0731999999999999e-05 2023-10-09T06:15:32.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.605818+0000", 2023-10-09T06:15:32.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.071784291 2023-10-09T06:15:32.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.605835+0000", 2023-10-09T06:15:32.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6589e-05 2023-10-09T06:15:32.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578572+0000", 2023-10-09T06:15:32.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97273759299999996 2023-10-09T06:15:32.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578584+0000", 2023-10-09T06:15:32.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1283e-05 2023-10-09T06:15:32.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.711858+0000", 2023-10-09T06:15:32.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1332740379999999 2023-10-09T06:15:32.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:32.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.711949+0000", 2023-10-09T06:15:32.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1187000000000006e-05 2023-10-09T06:15:32.849 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.849 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:32.849 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.850 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116293 9.1 9:8657b0ae:::benchmark_data_smithi064_511911_object15191:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-10-09T06:15:32.850 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.545024+0000", 2023-10-09T06:15:32.850 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 390.28894968999998, 2023-10-09T06:15:32.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1673195999999999, 2023-10-09T06:15:32.850 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:32.850 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:15:32.850 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:32.850 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:15:32.850 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:15:32.850 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116293 2023-10-09T06:15:32.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.851 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:32.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:32.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545024+0000", 2023-10-09T06:15:32.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:32.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545024+0000", 2023-10-09T06:15:32.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:32.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545023+0000", 2023-10-09T06:15:32.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:15:32.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:32.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545029+0000", 2023-10-09T06:15:32.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5910000000000003e-06 2023-10-09T06:15:32.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:32.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545037+0000", 2023-10-09T06:15:32.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.909e-06 2023-10-09T06:15:32.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:32.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545042+0000", 2023-10-09T06:15:32.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1030000000000001e-06 2023-10-09T06:15:32.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545077+0000", 2023-10-09T06:15:32.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4620999999999999e-05 2023-10-09T06:15:32.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:32.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545160+0000", 2023-10-09T06:15:32.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3925999999999996e-05 2023-10-09T06:15:32.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:15:32.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545236+0000", 2023-10-09T06:15:32.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5909999999999997e-05 2023-10-09T06:15:32.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.855 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:32.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.547494+0000", 2023-10-09T06:15:32.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0022575310000000001 2023-10-09T06:15:32.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.855 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.547584+0000", 2023-10-09T06:15:32.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0315999999999994e-05 2023-10-09T06:15:32.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.605910+0000", 2023-10-09T06:15:32.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.058325845000000001 2023-10-09T06:15:32.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.605922+0000", 2023-10-09T06:15:32.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1712999999999999e-05 2023-10-09T06:15:32.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578649+0000", 2023-10-09T06:15:32.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97272731099999998 2023-10-09T06:15:32.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578660+0000", 2023-10-09T06:15:32.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0560999999999999e-05 2023-10-09T06:15:32.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.858 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.712239+0000", 2023-10-09T06:15:32.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.133579602 2023-10-09T06:15:32.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.858 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:32.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.712344+0000", 2023-10-09T06:15:32.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000104746 2023-10-09T06:15:32.859 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.859 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:32.859 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.859 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116277 9.7 9:e3c0da0d:::benchmark_data_smithi064_511911_object15175:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e449)", 2023-10-09T06:15:32.859 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.606356+0000", 2023-10-09T06:15:32.859 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 390.227617931, 2023-10-09T06:15:32.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1339391660000002, 2023-10-09T06:15:32.860 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:32.860 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:15:32.860 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:32.860 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:15:32.860 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:15:32.860 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116277 2023-10-09T06:15:32.860 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.860 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:32.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:32.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606356+0000", 2023-10-09T06:15:32.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:32.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606356+0000", 2023-10-09T06:15:32.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:32.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606355+0000", 2023-10-09T06:15:32.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:15:32.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:32.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606362+0000", 2023-10-09T06:15:32.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8249999999999999e-06 2023-10-09T06:15:32.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:32.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606372+0000", 2023-10-09T06:15:32.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0946e-05 2023-10-09T06:15:32.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:32.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606382+0000", 2023-10-09T06:15:32.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4849999999999998e-06 2023-10-09T06:15:32.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.864 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606441+0000", 2023-10-09T06:15:32.864 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8758000000000002e-05 2023-10-09T06:15:32.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.864 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.864 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606452+0000", 2023-10-09T06:15:32.864 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1435e-05 2023-10-09T06:15:32.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578004+0000", 2023-10-09T06:15:32.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97155135599999998 2023-10-09T06:15:32.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578015+0000", 2023-10-09T06:15:32.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1185999999999999e-05 2023-10-09T06:15:32.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.737752+0000", 2023-10-09T06:15:32.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.159737572 2023-10-09T06:15:32.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:32.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.740295+0000", 2023-10-09T06:15:32.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025430359999999998 2023-10-09T06:15:32.867 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.867 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:32.867 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.867 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116283 9.3 9:c10cf455:::benchmark_data_smithi064_511911_object15181:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e449)", 2023-10-09T06:15:32.867 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.606404+0000", 2023-10-09T06:15:32.867 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 390.22756986000002, 2023-10-09T06:15:32.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.160898918, 2023-10-09T06:15:32.867 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:32.867 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:15:32.867 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:32.868 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:15:32.868 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:15:32.868 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116283 2023-10-09T06:15:32.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.868 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:32.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:32.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606404+0000", 2023-10-09T06:15:32.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:32.869 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606404+0000", 2023-10-09T06:15:32.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.869 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:32.869 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606404+0000", 2023-10-09T06:15:32.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:15:32.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.869 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:32.869 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606407+0000", 2023-10-09T06:15:32.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9179999999999998e-06 2023-10-09T06:15:32.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:32.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606415+0000", 2023-10-09T06:15:32.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3180000000000007e-06 2023-10-09T06:15:32.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:32.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606421+0000", 2023-10-09T06:15:32.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2430000000000002e-06 2023-10-09T06:15:32.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606613+0000", 2023-10-09T06:15:32.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019148599999999999 2023-10-09T06:15:32.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606636+0000", 2023-10-09T06:15:32.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3680999999999999e-05 2023-10-09T06:15:32.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.576977+0000", 2023-10-09T06:15:32.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97034085699999995 2023-10-09T06:15:32.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.577913+0000", 2023-10-09T06:15:32.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00093576100000000001 2023-10-09T06:15:32.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.723002+0000", 2023-10-09T06:15:32.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1450891699999999 2023-10-09T06:15:32.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.873 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-10-09T06:15:32.873 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.723112+0000", 2023-10-09T06:15:32.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110089 2023-10-09T06:15:32.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.873 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.873 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.767258+0000", 2023-10-09T06:15:32.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.044145982 2023-10-09T06:15:32.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.874 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:32.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.767303+0000", 2023-10-09T06:15:32.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4894000000000001e-05 2023-10-09T06:15:32.874 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.874 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:32.874 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.874 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.874 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116296 9.1 9:8540568d:::benchmark_data_smithi064_511911_object15194:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2023-10-09T06:15:32.874 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.773495+0000", 2023-10-09T06:15:32.874 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 390.06047942200001, 2023-10-09T06:15:32.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.958274855, 2023-10-09T06:15:32.875 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:32.875 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:32.875 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:32.875 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:15:32.875 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:15:32.875 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116296 2023-10-09T06:15:32.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.875 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:32.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:32.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.773495+0000", 2023-10-09T06:15:32.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:32.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.773495+0000", 2023-10-09T06:15:32.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:32.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.773494+0000", 2023-10-09T06:15:32.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:15:32.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:32.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.773499+0000", 2023-10-09T06:15:32.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6319999999999999e-06 2023-10-09T06:15:32.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:32.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.773512+0000", 2023-10-09T06:15:32.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3288000000000001e-05 2023-10-09T06:15:32.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:32.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.773519+0000", 2023-10-09T06:15:32.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9090000000000003e-06 2023-10-09T06:15:32.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.773579+0000", 2023-10-09T06:15:32.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9871999999999999e-05 2023-10-09T06:15:32.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.773591+0000", 2023-10-09T06:15:32.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2526e-05 2023-10-09T06:15:32.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578692+0000", 2023-10-09T06:15:32.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.80510072099999996 2023-10-09T06:15:32.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578700+0000", 2023-10-09T06:15:32.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3380000000000005e-06 2023-10-09T06:15:32.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.712529+0000", 2023-10-09T06:15:32.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1338282500000001 2023-10-09T06:15:32.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:32.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.712695+0000", 2023-10-09T06:15:32.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000166875 2023-10-09T06:15:32.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6,7", 2023-10-09T06:15:32.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.713276+0000", 2023-10-09T06:15:32.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00058104399999999996 2023-10-09T06:15:32.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.720154+0000", 2023-10-09T06:15:32.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0068775190000000003 2023-10-09T06:15:32.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:32.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.723009+0000", 2023-10-09T06:15:32.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002855486 2023-10-09T06:15:32.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.731564+0000", 2023-10-09T06:15:32.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.008555053 2023-10-09T06:15:32.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.731603+0000", 2023-10-09T06:15:32.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8117000000000001e-05 2023-10-09T06:15:32.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:32.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.731663+0000", 2023-10-09T06:15:32.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0297000000000002e-05 2023-10-09T06:15:32.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:32.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.731770+0000", 2023-10-09T06:15:32.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000106616 2023-10-09T06:15:32.884 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.885 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:32.885 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.885 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116300 9.1 9:882a33e5:::benchmark_data_smithi064_511911_object15198:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2023-10-09T06:15:32.885 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.790061+0000", 2023-10-09T06:15:32.885 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 390.04391259599998, 2023-10-09T06:15:32.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.951866648, 2023-10-09T06:15:32.885 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:32.885 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:32.885 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:32.886 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:15:32.886 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:15:32.886 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116300 2023-10-09T06:15:32.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.886 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:32.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:32.886 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.790061+0000", 2023-10-09T06:15:32.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.887 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:32.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.790061+0000", 2023-10-09T06:15:32.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.887 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:32.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.790060+0000", 2023-10-09T06:15:32.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:15:32.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:32.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.790066+0000", 2023-10-09T06:15:32.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.434e-06 2023-10-09T06:15:32.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:32.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.790076+0000", 2023-10-09T06:15:32.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0404000000000001e-05 2023-10-09T06:15:32.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:32.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.790082+0000", 2023-10-09T06:15:32.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.908e-06 2023-10-09T06:15:32.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.790236+0000", 2023-10-09T06:15:32.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015350499999999999 2023-10-09T06:15:32.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.790243+0000", 2023-10-09T06:15:32.890 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6000000000000001e-06 2023-10-09T06:15:32.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578724+0000", 2023-10-09T06:15:32.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.78848058899999995 2023-10-09T06:15:32.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578734+0000", 2023-10-09T06:15:32.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6549999999999999e-06 2023-10-09T06:15:32.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.720179+0000", 2023-10-09T06:15:32.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1414453629999999 2023-10-09T06:15:32.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:32.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.720309+0000", 2023-10-09T06:15:32.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013024900000000001 2023-10-09T06:15:32.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6,7", 2023-10-09T06:15:32.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.720444+0000", 2023-10-09T06:15:32.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013511799999999999 2023-10-09T06:15:32.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:32.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.731215+0000", 2023-10-09T06:15:32.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010770693 2023-10-09T06:15:32.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.731528+0000", 2023-10-09T06:15:32.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000313109 2023-10-09T06:15:32.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.734686+0000", 2023-10-09T06:15:32.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031578380000000001 2023-10-09T06:15:32.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.741743+0000", 2023-10-09T06:15:32.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0070569559999999996 2023-10-09T06:15:32.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:32.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.741803+0000", 2023-10-09T06:15:32.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0517999999999999e-05 2023-10-09T06:15:32.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:32.896 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.741928+0000", 2023-10-09T06:15:32.896 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012472300000000001 2023-10-09T06:15:32.896 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.896 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:32.896 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.896 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:132917 9.8 9:12f89378:::benchmark_data_smithi064_511911_object31815:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e456)", 2023-10-09T06:15:32.896 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:26.626655+0000", 2023-10-09T06:15:32.896 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 366.20731900499999, 2023-10-09T06:15:32.896 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1614917130000002, 2023-10-09T06:15:32.896 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:32.897 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:32.897 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:32.897 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:15:32.897 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:15:32.897 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132917 2023-10-09T06:15:32.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.897 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:32.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:32.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.626655+0000", 2023-10-09T06:15:32.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.898 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:32.898 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.626655+0000", 2023-10-09T06:15:32.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.898 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:32.898 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.626653+0000", 2023-10-09T06:15:32.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:15:32.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:32.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.626661+0000", 2023-10-09T06:15:32.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7069999999999992e-06 2023-10-09T06:15:32.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:32.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.626673+0000", 2023-10-09T06:15:32.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2517999999999999e-05 2023-10-09T06:15:32.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:32.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.626683+0000", 2023-10-09T06:15:32.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4490000000000005e-06 2023-10-09T06:15:32.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.627477+0000", 2023-10-09T06:15:32.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00079445099999999995 2023-10-09T06:15:32.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.627484+0000", 2023-10-09T06:15:32.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4520000000000002e-06 2023-10-09T06:15:32.901 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.901 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.901 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528505+0000", 2023-10-09T06:15:32.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90102110700000004 2023-10-09T06:15:32.901 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.901 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.901 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528519+0000", 2023-10-09T06:15:32.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3943e-05 2023-10-09T06:15:32.901 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.755152+0000", 2023-10-09T06:15:32.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2266332689999999 2023-10-09T06:15:32.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:32.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.755289+0000", 2023-10-09T06:15:32.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013710400000000001 2023-10-09T06:15:32.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:15:32.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.755435+0000", 2023-10-09T06:15:32.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000145496 2023-10-09T06:15:32.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:32.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.766695+0000", 2023-10-09T06:15:32.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011260564000000001 2023-10-09T06:15:32.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.769909+0000", 2023-10-09T06:15:32.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032138940000000001 2023-10-09T06:15:32.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.904 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.904 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.785944+0000", 2023-10-09T06:15:32.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016034757 2023-10-09T06:15:32.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.904 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:32.904 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.785992+0000", 2023-10-09T06:15:32.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8412999999999999e-05 2023-10-09T06:15:32.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.905 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:32.905 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.788147+0000", 2023-10-09T06:15:32.905 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0021544020000000001 2023-10-09T06:15:32.905 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.905 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:32.905 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.905 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.905 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.905 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:132920 9.8 9:10a1d199:::benchmark_data_smithi064_511911_object31818:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e456)", 2023-10-09T06:15:32.905 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:26.641712+0000", 2023-10-09T06:15:32.905 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 366.19226171399998, 2023-10-09T06:15:32.905 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.148552274, 2023-10-09T06:15:32.906 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:32.906 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:32.906 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:32.906 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:15:32.906 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:15:32.906 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132920 2023-10-09T06:15:32.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.906 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:32.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.906 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:32.906 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.641712+0000", 2023-10-09T06:15:32.906 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.907 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.907 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:32.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.641712+0000", 2023-10-09T06:15:32.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.907 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.907 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:32.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.641710+0000", 2023-10-09T06:15:32.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:15:32.907 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.908 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:32.908 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.641717+0000", 2023-10-09T06:15:32.908 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0049999999999998e-06 2023-10-09T06:15:32.908 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.908 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.908 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:32.908 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.641731+0000", 2023-10-09T06:15:32.908 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3648e-05 2023-10-09T06:15:32.908 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.908 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.908 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:32.908 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.641743+0000", 2023-10-09T06:15:32.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1839e-05 2023-10-09T06:15:32.909 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.909 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.909 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.909 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.641792+0000", 2023-10-09T06:15:32.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9376000000000003e-05 2023-10-09T06:15:32.909 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.909 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.909 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.909 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.641803+0000", 2023-10-09T06:15:32.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1141000000000001e-05 2023-10-09T06:15:32.909 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.910 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.910 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528543+0000", 2023-10-09T06:15:32.910 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88673970899999999 2023-10-09T06:15:32.910 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.910 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.910 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528551+0000", 2023-10-09T06:15:32.910 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9319999999999996e-06 2023-10-09T06:15:32.910 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.911 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.911 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.760814+0000", 2023-10-09T06:15:32.911 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2322625380000001 2023-10-09T06:15:32.911 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.911 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.911 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:32.911 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.760963+0000", 2023-10-09T06:15:32.911 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000149522 2023-10-09T06:15:32.911 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.911 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.911 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:15:32.912 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.761205+0000", 2023-10-09T06:15:32.912 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000241831 2023-10-09T06:15:32.912 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.912 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.912 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:32.912 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.769866+0000", 2023-10-09T06:15:32.912 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086611819999999999 2023-10-09T06:15:32.912 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.912 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.912 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.912 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.769951+0000", 2023-10-09T06:15:32.912 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5286999999999998e-05 2023-10-09T06:15:32.913 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.913 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.913 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.913 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.788276+0000", 2023-10-09T06:15:32.913 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018324944999999999 2023-10-09T06:15:32.913 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.913 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.913 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:32.913 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.788329+0000", 2023-10-09T06:15:32.913 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2846999999999997e-05 2023-10-09T06:15:32.913 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.914 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:32.914 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.790265+0000", 2023-10-09T06:15:32.914 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019354929999999999 2023-10-09T06:15:32.914 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.914 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:32.914 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.914 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.914 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:132924 9.8 9:100587d5:::benchmark_data_smithi064_511911_object31822:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e456)", 2023-10-09T06:15:32.914 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:26.652624+0000", 2023-10-09T06:15:32.915 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 366.181349974, 2023-10-09T06:15:32.915 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1523453090000002, 2023-10-09T06:15:32.915 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:32.915 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:32.915 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:32.915 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:15:32.915 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:15:32.915 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132924 2023-10-09T06:15:32.915 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.915 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:32.915 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.916 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:32.916 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652624+0000", 2023-10-09T06:15:32.916 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.916 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.916 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.916 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:32.916 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652624+0000", 2023-10-09T06:15:32.916 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.916 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.916 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.916 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:32.917 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652616+0000", 2023-10-09T06:15:32.917 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999924 2023-10-09T06:15:32.917 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.917 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.917 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:32.917 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652632+0000", 2023-10-09T06:15:32.917 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5653000000000001e-05 2023-10-09T06:15:32.917 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.917 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.917 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:32.917 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652647+0000", 2023-10-09T06:15:32.917 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5244e-05 2023-10-09T06:15:32.918 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.918 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.918 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:32.918 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652656+0000", 2023-10-09T06:15:32.918 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0380000000000005e-06 2023-10-09T06:15:32.918 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.918 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.918 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.918 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652718+0000", 2023-10-09T06:15:32.918 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1267999999999997e-05 2023-10-09T06:15:32.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.919 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.919 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652730+0000", 2023-10-09T06:15:32.919 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2069000000000001e-05 2023-10-09T06:15:32.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.919 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.919 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528571+0000", 2023-10-09T06:15:32.919 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.875841753 2023-10-09T06:15:32.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.920 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.920 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528579+0000", 2023-10-09T06:15:32.920 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6410000000000002e-06 2023-10-09T06:15:32.920 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.920 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.920 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.920 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.764042+0000", 2023-10-09T06:15:32.920 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2354630339999999 2023-10-09T06:15:32.920 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.920 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.920 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:32.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.764130+0000", 2023-10-09T06:15:32.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7968999999999998e-05 2023-10-09T06:15:32.921 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.921 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.921 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:15:32.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.764240+0000", 2023-10-09T06:15:32.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010986 2023-10-09T06:15:32.921 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.921 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.921 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:32.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.769918+0000", 2023-10-09T06:15:32.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056776170000000003 2023-10-09T06:15:32.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.922 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.922 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.769984+0000", 2023-10-09T06:15:32.922 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6703000000000002e-05 2023-10-09T06:15:32.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.922 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.922 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.802902+0000", 2023-10-09T06:15:32.922 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.032917469999999997 2023-10-09T06:15:32.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.923 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:32.923 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.802957+0000", 2023-10-09T06:15:32.923 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5047999999999999e-05 2023-10-09T06:15:32.923 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.923 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.923 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:32.923 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.804969+0000", 2023-10-09T06:15:32.923 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020126900000000001 2023-10-09T06:15:32.923 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.923 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:32.923 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.924 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.924 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.924 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:132927 9.8 9:1f462da4:::benchmark_data_smithi064_511911_object31825:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e456)", 2023-10-09T06:15:32.924 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:26.663633+0000", 2023-10-09T06:15:32.924 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 366.17034106900002, 2023-10-09T06:15:32.924 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1462853150000001, 2023-10-09T06:15:32.924 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:32.925 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:32.925 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:32.925 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:15:32.925 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:15:32.925 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132927 2023-10-09T06:15:32.925 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.925 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:32.925 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.926 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:32.926 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.663633+0000", 2023-10-09T06:15:32.926 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.926 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.926 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.926 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:32.926 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.663633+0000", 2023-10-09T06:15:32.926 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.926 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.926 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.926 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:32.927 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.663631+0000", 2023-10-09T06:15:32.927 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:15:32.927 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.927 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.927 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:32.927 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.663639+0000", 2023-10-09T06:15:32.927 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3089999999999998e-06 2023-10-09T06:15:32.927 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.927 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.927 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:32.927 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.663652+0000", 2023-10-09T06:15:32.928 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3485999999999999e-05 2023-10-09T06:15:32.928 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.928 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.928 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:32.928 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.663658+0000", 2023-10-09T06:15:32.928 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.874e-06 2023-10-09T06:15:32.928 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.928 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.928 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.928 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.663716+0000", 2023-10-09T06:15:32.928 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8115000000000002e-05 2023-10-09T06:15:32.928 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.929 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.929 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.929 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.663728+0000", 2023-10-09T06:15:32.929 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.168e-05 2023-10-09T06:15:32.929 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.929 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.929 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.929 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528601+0000", 2023-10-09T06:15:32.929 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.86487267700000003 2023-10-09T06:15:32.929 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.929 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.930 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.930 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528608+0000", 2023-10-09T06:15:32.930 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2509999999999997e-06 2023-10-09T06:15:32.930 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.930 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.930 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.930 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.766727+0000", 2023-10-09T06:15:32.930 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2381187899999999 2023-10-09T06:15:32.930 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.930 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.930 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:32.930 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.766857+0000", 2023-10-09T06:15:32.931 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013015999999999999 2023-10-09T06:15:32.931 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.931 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.931 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:15:32.931 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.767000+0000", 2023-10-09T06:15:32.931 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014345199999999999 2023-10-09T06:15:32.931 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.931 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.931 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.931 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.770019+0000", 2023-10-09T06:15:32.931 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030185609999999999 2023-10-09T06:15:32.931 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.932 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.932 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:32.932 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.785896+0000", 2023-10-09T06:15:32.932 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015877437000000001 2023-10-09T06:15:32.932 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.932 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.932 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.932 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.808044+0000", 2023-10-09T06:15:32.932 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.022147962 2023-10-09T06:15:32.932 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.932 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.932 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:32.932 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.808105+0000", 2023-10-09T06:15:32.933 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0856000000000001e-05 2023-10-09T06:15:32.933 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.933 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.933 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:32.933 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.809918+0000", 2023-10-09T06:15:32.933 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018133559999999999 2023-10-09T06:15:32.933 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.933 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:32.933 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.933 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.933 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.933 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:95179 10.7 10:eeefb019:::benchmark_data_smithi064_573750_object95178:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e521)", 2023-10-09T06:15:32.934 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:20.802346+0000", 2023-10-09T06:15:32.934 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 132.03162800600001, 2023-10-09T06:15:32.934 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.175117368, 2023-10-09T06:15:32.934 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:32.934 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:32.934 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:32.934 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:32.934 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:32.934 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 95179 2023-10-09T06:15:32.934 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.935 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:32.935 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:32.935 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.802346+0000", 2023-10-09T06:15:32.935 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.935 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.935 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:32.935 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.802346+0000", 2023-10-09T06:15:32.935 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.935 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.935 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:32.936 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.802344+0000", 2023-10-09T06:15:32.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:15:32.936 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.936 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.936 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:32.936 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.802363+0000", 2023-10-09T06:15:32.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8389999999999998e-05 2023-10-09T06:15:32.936 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.936 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.936 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:32.936 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.802372+0000", 2023-10-09T06:15:32.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3200000000000006e-06 2023-10-09T06:15:32.937 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.937 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:32.937 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.802378+0000", 2023-10-09T06:15:32.937 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0549999999999996e-06 2023-10-09T06:15:32.937 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.937 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.937 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.804722+0000", 2023-10-09T06:15:32.937 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0023433960000000002 2023-10-09T06:15:32.937 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.938 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.938 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.804730+0000", 2023-10-09T06:15:32.938 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9160000000000007e-06 2023-10-09T06:15:32.938 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.938 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.938 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.938 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.800510+0000", 2023-10-09T06:15:32.938 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99577908400000004 2023-10-09T06:15:32.938 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.938 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.938 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:32.938 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.800681+0000", 2023-10-09T06:15:32.939 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000171888 2023-10-09T06:15:32.939 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.939 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.939 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4,5,6", 2023-10-09T06:15:32.939 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.800845+0000", 2023-10-09T06:15:32.939 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016401600000000001 2023-10-09T06:15:32.939 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.939 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.939 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.939 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.804037+0000", 2023-10-09T06:15:32.939 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031911029999999998 2023-10-09T06:15:32.939 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.940 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.940 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.804099+0000", 2023-10-09T06:15:32.940 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2243999999999996e-05 2023-10-09T06:15:32.940 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.940 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:32.940 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.804113+0000", 2023-10-09T06:15:32.940 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3835000000000001e-05 2023-10-09T06:15:32.940 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.941 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.941 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.804137+0000", 2023-10-09T06:15:32.941 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4646000000000001e-05 2023-10-09T06:15:32.941 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.941 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.941 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.941 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.977248+0000", 2023-10-09T06:15:32.941 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.173110507 2023-10-09T06:15:32.941 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.941 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.941 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:32.942 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.977319+0000", 2023-10-09T06:15:32.942 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1324999999999999e-05 2023-10-09T06:15:32.942 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.942 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.942 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:32.942 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.977463+0000", 2023-10-09T06:15:32.942 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014437299999999999 2023-10-09T06:15:32.942 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.942 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:32.942 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.942 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.943 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.943 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:149297 10.7 10:e62c45a3:::benchmark_data_smithi064_573750_object53082:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e550)", 2023-10-09T06:15:32.943 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:47.201724+0000", 2023-10-09T06:15:32.943 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 45.632249913000003, 2023-10-09T06:15:32.943 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.49615405899999998, 2023-10-09T06:15:32.943 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:32.943 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:32.943 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:32.943 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:32.943 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:32.943 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149297 2023-10-09T06:15:32.943 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.944 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:32.944 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.944 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:32.944 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.201724+0000", 2023-10-09T06:15:32.944 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.944 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.944 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.944 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:32.944 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.201724+0000", 2023-10-09T06:15:32.944 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.944 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.944 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.945 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:32.945 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.201722+0000", 2023-10-09T06:15:32.945 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T06:15:32.945 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.945 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.945 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:32.945 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.201733+0000", 2023-10-09T06:15:32.945 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1395e-05 2023-10-09T06:15:32.945 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.945 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.946 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:32.946 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.201760+0000", 2023-10-09T06:15:32.946 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6696999999999999e-05 2023-10-09T06:15:32.946 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.946 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.946 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:32.946 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.201774+0000", 2023-10-09T06:15:32.946 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4558e-05 2023-10-09T06:15:32.946 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.947 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.947 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.947 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.201850+0000", 2023-10-09T06:15:32.947 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5852000000000005e-05 2023-10-09T06:15:32.947 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.947 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.947 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:32.947 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.202048+0000", 2023-10-09T06:15:32.947 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019787599999999999 2023-10-09T06:15:32.947 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.948 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.948 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4,5,6", 2023-10-09T06:15:32.948 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.202193+0000", 2023-10-09T06:15:32.948 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000144513 2023-10-09T06:15:32.948 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.948 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.948 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:32.948 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.206293+0000", 2023-10-09T06:15:32.948 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040998440000000001 2023-10-09T06:15:32.948 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.948 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.948 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.949 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.209231+0000", 2023-10-09T06:15:32.949 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029379580000000001 2023-10-09T06:15:32.949 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.949 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.949 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.949 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.210159+0000", 2023-10-09T06:15:32.949 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00092885299999999999 2023-10-09T06:15:32.949 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.949 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.949 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.949 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.217290+0000", 2023-10-09T06:15:32.950 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0071305159999999999 2023-10-09T06:15:32.950 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.950 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.950 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.950 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.697679+0000", 2023-10-09T06:15:32.950 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.48038878800000001 2023-10-09T06:15:32.950 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.950 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.950 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:32.950 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.697743+0000", 2023-10-09T06:15:32.950 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4542000000000006e-05 2023-10-09T06:15:32.951 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.951 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.951 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:32.951 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.697878+0000", 2023-10-09T06:15:32.951 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000135007 2023-10-09T06:15:32.951 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.951 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:32.951 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.951 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.951 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.952 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:149310 10.7 10:e74ed477:::benchmark_data_smithi064_573750_object53095:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e550)", 2023-10-09T06:15:32.952 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:47.225189+0000", 2023-10-09T06:15:32.952 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 45.608784864999997, 2023-10-09T06:15:32.952 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.47305104399999998, 2023-10-09T06:15:32.952 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:32.952 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:32.952 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:32.952 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:32.952 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:32.953 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149310 2023-10-09T06:15:32.953 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.953 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:32.953 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.953 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:32.953 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.225189+0000", 2023-10-09T06:15:32.953 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.953 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.953 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.953 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:32.954 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.225189+0000", 2023-10-09T06:15:32.954 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.954 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.954 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.954 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:32.954 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.225188+0000", 2023-10-09T06:15:32.954 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:15:32.954 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.955 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.955 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:32.955 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.225194+0000", 2023-10-09T06:15:32.955 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4249999999999997e-06 2023-10-09T06:15:32.955 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.955 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.955 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:32.955 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.225202+0000", 2023-10-09T06:15:32.955 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0779999999999996e-06 2023-10-09T06:15:32.955 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.956 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.956 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:32.956 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.225207+0000", 2023-10-09T06:15:32.956 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1699999999999996e-06 2023-10-09T06:15:32.956 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.956 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.956 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.956 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.225242+0000", 2023-10-09T06:15:32.956 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5080999999999998e-05 2023-10-09T06:15:32.957 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.957 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.957 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:32.957 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.225357+0000", 2023-10-09T06:15:32.957 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000114417 2023-10-09T06:15:32.957 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.957 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.957 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4,5,6", 2023-10-09T06:15:32.957 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.225503+0000", 2023-10-09T06:15:32.958 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000146458 2023-10-09T06:15:32.958 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.958 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.958 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.958 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.228978+0000", 2023-10-09T06:15:32.958 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034748679999999999 2023-10-09T06:15:32.958 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.958 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.958 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.959 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.231905+0000", 2023-10-09T06:15:32.959 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029272299999999998 2023-10-09T06:15:32.959 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.959 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.959 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:32.959 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.232046+0000", 2023-10-09T06:15:32.959 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014093600000000001 2023-10-09T06:15:32.959 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.959 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.959 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.959 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.235824+0000", 2023-10-09T06:15:32.960 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037774660000000002 2023-10-09T06:15:32.960 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.960 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.960 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.960 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.698037+0000", 2023-10-09T06:15:32.960 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.46221365199999997 2023-10-09T06:15:32.960 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.960 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.960 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:32.960 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.698096+0000", 2023-10-09T06:15:32.961 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8851999999999998e-05 2023-10-09T06:15:32.961 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.961 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.961 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:32.961 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.698240+0000", 2023-10-09T06:15:32.961 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000144256 2023-10-09T06:15:32.961 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.961 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:32.962 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.962 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.962 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.962 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:153133 10.f 10:fe4e4f6c:::benchmark_data_smithi064_573750_object56918:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-10-09T06:15:32.962 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:52.524591+0000", 2023-10-09T06:15:32.962 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 40.309382865000003, 2023-10-09T06:15:32.962 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30689329500000001, 2023-10-09T06:15:32.962 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:32.963 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:32.963 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:32.963 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:32.963 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:32.963 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 153133 2023-10-09T06:15:32.963 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.963 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:32.963 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.963 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:32.964 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:52.524591+0000", 2023-10-09T06:15:32.964 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.964 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.964 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.964 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:32.964 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:52.524591+0000", 2023-10-09T06:15:32.964 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.964 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.964 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.965 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:32.965 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:52.524591+0000", 2023-10-09T06:15:32.965 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:15:32.965 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.965 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.965 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:32.965 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:52.524593+0000", 2023-10-09T06:15:32.965 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6120000000000001e-06 2023-10-09T06:15:32.966 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.966 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.966 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:32.966 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:52.524599+0000", 2023-10-09T06:15:32.966 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8159999999999999e-06 2023-10-09T06:15:32.966 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.966 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.966 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:32.967 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:52.524604+0000", 2023-10-09T06:15:32.967 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.152e-06 2023-10-09T06:15:32.967 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.967 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.967 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.967 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:52.733953+0000", 2023-10-09T06:15:32.967 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20934868300000001 2023-10-09T06:15:32.967 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.967 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.967 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:32.967 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:52.734070+0000", 2023-10-09T06:15:32.968 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116834 2023-10-09T06:15:32.968 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.968 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.968 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:15:32.968 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:52.734182+0000", 2023-10-09T06:15:32.968 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000111796 2023-10-09T06:15:32.968 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.968 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.968 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:32.968 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:52.831112+0000", 2023-10-09T06:15:32.968 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.096930106000000002 2023-10-09T06:15:32.968 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.968 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.969 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.969 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:52.831244+0000", 2023-10-09T06:15:32.969 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013178500000000001 2023-10-09T06:15:32.969 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.969 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.969 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.969 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:52.831420+0000", 2023-10-09T06:15:32.969 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000175957 2023-10-09T06:15:32.969 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.969 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.969 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:32.969 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:52.831454+0000", 2023-10-09T06:15:32.970 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4551999999999997e-05 2023-10-09T06:15:32.970 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.970 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.970 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:32.970 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:52.831485+0000", 2023-10-09T06:15:32.970 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.04e-05 2023-10-09T06:15:32.970 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.970 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:32.970 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.970 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.970 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.971 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:171672 10.3 10:c82752d7:::benchmark_data_smithi064_573750_object75457:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e557)", 2023-10-09T06:15:32.971 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:15:15.799808+0000", 2023-10-09T06:15:32.971 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 17.034166482, 2023-10-09T06:15:32.971 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.162648323, 2023-10-09T06:15:32.971 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:32.971 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:15:32.971 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:32.971 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:32.971 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:32.971 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171672 2023-10-09T06:15:32.971 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.972 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:32.972 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.972 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:32.972 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799808+0000", 2023-10-09T06:15:32.972 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.972 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.972 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.972 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:32.972 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799808+0000", 2023-10-09T06:15:32.972 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.972 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.972 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.973 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:32.973 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799805+0000", 2023-10-09T06:15:32.973 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:15:32.973 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.973 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.973 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:32.973 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799813+0000", 2023-10-09T06:15:32.973 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9149999999999996e-06 2023-10-09T06:15:32.973 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.973 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.973 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:32.973 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799826+0000", 2023-10-09T06:15:32.973 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2824e-05 2023-10-09T06:15:32.974 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.974 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.974 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:32.974 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799834+0000", 2023-10-09T06:15:32.974 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8990000000000001e-06 2023-10-09T06:15:32.974 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.974 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.974 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.974 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.800015+0000", 2023-10-09T06:15:32.974 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018057300000000001 2023-10-09T06:15:32.974 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.974 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.975 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:32.975 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.800155+0000", 2023-10-09T06:15:32.975 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014034099999999999 2023-10-09T06:15:32.975 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.975 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.975 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:15:32.975 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.800283+0000", 2023-10-09T06:15:32.975 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000127945 2023-10-09T06:15:32.975 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.975 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.975 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:32.975 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.808806+0000", 2023-10-09T06:15:32.976 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085225639999999998 2023-10-09T06:15:32.976 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.976 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.976 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.976 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812281+0000", 2023-10-09T06:15:32.976 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003475653 2023-10-09T06:15:32.976 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.976 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.976 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.976 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812302+0000", 2023-10-09T06:15:32.976 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0441e-05 2023-10-09T06:15:32.976 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.977 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.977 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.977 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794653+0000", 2023-10-09T06:15:32.977 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98235096899999996 2023-10-09T06:15:32.977 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.977 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.977 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.977 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794672+0000", 2023-10-09T06:15:32.977 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9063e-05 2023-10-09T06:15:32.977 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.977 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.977 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.978 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.950344+0000", 2023-10-09T06:15:32.978 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.155671983 2023-10-09T06:15:32.978 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.978 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.978 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-10-09T06:15:32.978 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.950466+0000", 2023-10-09T06:15:32.978 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000122123 2023-10-09T06:15:32.978 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.978 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.978 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.978 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.962321+0000", 2023-10-09T06:15:32.978 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01185547 2023-10-09T06:15:32.979 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.979 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.979 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:32.979 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.962456+0000", 2023-10-09T06:15:32.979 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013469099999999999 2023-10-09T06:15:32.979 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.979 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:32.979 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.979 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.979 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.979 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:171676 10.3 10:c58e27ea:::benchmark_data_smithi064_573750_object75461:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e557)", 2023-10-09T06:15:32.979 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:15:15.799859+0000", 2023-10-09T06:15:32.980 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 17.034115005, 2023-10-09T06:15:32.980 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1745875799999999, 2023-10-09T06:15:32.980 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:32.980 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:15:32.980 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:32.980 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:32.980 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:32.980 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171676 2023-10-09T06:15:32.980 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.980 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:32.980 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.980 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:32.981 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799859+0000", 2023-10-09T06:15:32.981 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.981 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.981 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.981 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:32.981 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799859+0000", 2023-10-09T06:15:32.981 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.981 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.981 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.981 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:32.981 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799859+0000", 2023-10-09T06:15:32.981 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:15:32.982 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.982 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.982 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:32.982 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799862+0000", 2023-10-09T06:15:32.982 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0750000000000002e-06 2023-10-09T06:15:32.982 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.982 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.982 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:32.982 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799875+0000", 2023-10-09T06:15:32.982 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.31e-05 2023-10-09T06:15:32.982 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.982 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.983 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:32.983 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799881+0000", 2023-10-09T06:15:32.983 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.801e-06 2023-10-09T06:15:32.983 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.983 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.983 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.983 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.803918+0000", 2023-10-09T06:15:32.983 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040373400000000004 2023-10-09T06:15:32.983 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.983 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.983 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:32.983 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.804065+0000", 2023-10-09T06:15:32.984 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014711600000000001 2023-10-09T06:15:32.984 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.984 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.984 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:15:32.984 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.804246+0000", 2023-10-09T06:15:32.984 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018100599999999999 2023-10-09T06:15:32.984 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.984 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.984 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.984 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812631+0000", 2023-10-09T06:15:32.984 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0083849679999999996 2023-10-09T06:15:32.985 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.985 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.985 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.985 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812642+0000", 2023-10-09T06:15:32.985 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0681e-05 2023-10-09T06:15:32.985 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.985 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.985 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.985 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794709+0000", 2023-10-09T06:15:32.985 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98206730200000003 2023-10-09T06:15:32.985 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.985 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.986 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.986 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794722+0000", 2023-10-09T06:15:32.986 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2621e-05 2023-10-09T06:15:32.986 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.986 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.986 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.986 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.950773+0000", 2023-10-09T06:15:32.986 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.156051175 2023-10-09T06:15:32.986 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.986 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.986 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-10-09T06:15:32.987 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.950916+0000", 2023-10-09T06:15:32.987 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000142726 2023-10-09T06:15:32.987 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.987 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.987 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.987 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974353+0000", 2023-10-09T06:15:32.987 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023437021999999998 2023-10-09T06:15:32.987 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.987 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.987 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:32.987 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974447+0000", 2023-10-09T06:15:32.987 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4092000000000006e-05 2023-10-09T06:15:32.987 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.988 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:32.988 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.988 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.988 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.988 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:171677 10.3 10:cc911201:::benchmark_data_smithi064_573750_object75462:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:15:32.988 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:15:15.812441+0000", 2023-10-09T06:15:32.988 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 17.021533560999998, 2023-10-09T06:15:32.988 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.162245993, 2023-10-09T06:15:32.988 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:32.988 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:32.988 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:32.989 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:32.989 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:32.989 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171677 2023-10-09T06:15:32.989 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.989 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:32.989 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.989 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:32.989 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812441+0000", 2023-10-09T06:15:32.989 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.989 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.989 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.989 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:32.989 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812441+0000", 2023-10-09T06:15:32.990 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:32.990 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.990 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.990 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:32.990 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812439+0000", 2023-10-09T06:15:32.990 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:15:32.990 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.990 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.990 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:32.990 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812444+0000", 2023-10-09T06:15:32.990 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1150000000000002e-06 2023-10-09T06:15:32.991 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.991 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.991 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:32.991 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812455+0000", 2023-10-09T06:15:32.991 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1742999999999999e-05 2023-10-09T06:15:32.991 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.991 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.991 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:32.991 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812464+0000", 2023-10-09T06:15:32.991 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4910000000000005e-06 2023-10-09T06:15:32.991 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.991 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.992 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.992 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812964+0000", 2023-10-09T06:15:32.992 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00049951399999999995 2023-10-09T06:15:32.992 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.992 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.992 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.992 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812975+0000", 2023-10-09T06:15:32.992 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1063999999999999e-05 2023-10-09T06:15:32.992 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.992 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.992 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.992 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794751+0000", 2023-10-09T06:15:32.993 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.981776023 2023-10-09T06:15:32.993 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.993 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.993 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:32.993 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794760+0000", 2023-10-09T06:15:32.993 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9590000000000001e-06 2023-10-09T06:15:32.993 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.993 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.993 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:32.993 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.950951+0000", 2023-10-09T06:15:32.993 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.156191089 2023-10-09T06:15:32.993 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.994 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.994 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:32.994 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.951073+0000", 2023-10-09T06:15:32.994 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012265999999999999 2023-10-09T06:15:32.994 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.994 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.994 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T06:15:32.994 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.951219+0000", 2023-10-09T06:15:32.994 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014580999999999999 2023-10-09T06:15:32.994 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.994 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.995 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:32.995 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.961907+0000", 2023-10-09T06:15:32.995 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010687962000000001 2023-10-09T06:15:32.995 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.995 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.995 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.995 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.962502+0000", 2023-10-09T06:15:32.995 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000594646 2023-10-09T06:15:32.995 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.995 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.995 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.996 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.973943+0000", 2023-10-09T06:15:32.996 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011441019 2023-10-09T06:15:32.996 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.996 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.996 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:32.996 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974516+0000", 2023-10-09T06:15:32.996 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00057290800000000001 2023-10-09T06:15:32.996 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.997 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.997 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:32.997 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974566+0000", 2023-10-09T06:15:32.997 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0080999999999997e-05 2023-10-09T06:15:32.997 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.997 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.997 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:32.997 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974687+0000", 2023-10-09T06:15:32.998 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000120795 2023-10-09T06:15:32.998 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.998 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:32.998 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:32.998 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.998 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.998 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:171684 10.3 10:cf644d59:::benchmark_data_smithi064_573750_object75469:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:15:32.998 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:15:15.815765+0000", 2023-10-09T06:15:32.998 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 17.018208783999999, 2023-10-09T06:15:32.999 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1591680040000001, 2023-10-09T06:15:32.999 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:32.999 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:32.999 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:32.999 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:32.999 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:32.999 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171684 2023-10-09T06:15:32.999 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:32.999 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:32.999 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:32.999 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:32.999 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815765+0000", 2023-10-09T06:15:33.000 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:33.000 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.000 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.000 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:33.000 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815765+0000", 2023-10-09T06:15:33.000 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:33.000 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.000 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.000 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:33.000 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815764+0000", 2023-10-09T06:15:33.000 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:15:33.000 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.001 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.001 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:33.001 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815771+0000", 2023-10-09T06:15:33.001 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3080000000000002e-06 2023-10-09T06:15:33.001 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.001 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.001 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:33.001 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815785+0000", 2023-10-09T06:15:33.001 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3903000000000001e-05 2023-10-09T06:15:33.001 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.001 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.002 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:33.002 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815795+0000", 2023-10-09T06:15:33.002 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8209999999999997e-06 2023-10-09T06:15:33.002 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.002 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.002 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:33.002 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815830+0000", 2023-10-09T06:15:33.002 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4987999999999998e-05 2023-10-09T06:15:33.002 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.002 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.003 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:33.003 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815839+0000", 2023-10-09T06:15:33.003 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5189999999999998e-06 2023-10-09T06:15:33.003 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.003 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.003 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:33.003 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794786+0000", 2023-10-09T06:15:33.003 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97894675900000006 2023-10-09T06:15:33.003 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.003 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.003 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:33.004 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794794+0000", 2023-10-09T06:15:33.004 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5590000000000006e-06 2023-10-09T06:15:33.004 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.004 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.004 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:33.004 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.954768+0000", 2023-10-09T06:15:33.004 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.159973972 2023-10-09T06:15:33.004 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.004 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.004 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:33.004 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.954905+0000", 2023-10-09T06:15:33.004 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013690299999999999 2023-10-09T06:15:33.005 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.005 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.005 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T06:15:33.005 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.955044+0000", 2023-10-09T06:15:33.005 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013847499999999999 2023-10-09T06:15:33.005 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.005 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.005 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:33.005 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.962281+0000", 2023-10-09T06:15:33.005 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0072369289999999996 2023-10-09T06:15:33.005 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.005 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.006 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:33.006 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.962546+0000", 2023-10-09T06:15:33.006 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00026517000000000001 2023-10-09T06:15:33.006 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.006 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.006 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:33.006 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974728+0000", 2023-10-09T06:15:33.006 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012181772 2023-10-09T06:15:33.006 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.006 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.006 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:33.006 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974764+0000", 2023-10-09T06:15:33.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6820000000000003e-05 2023-10-09T06:15:33.007 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.007 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.007 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:33.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974817+0000", 2023-10-09T06:15:33.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2811e-05 2023-10-09T06:15:33.007 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.007 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.007 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:33.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974933+0000", 2023-10-09T06:15:33.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116087 2023-10-09T06:15:33.008 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:33.008 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:33.008 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:33.008 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.008 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.008 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:171685 10.3 10:cb5292a6:::benchmark_data_smithi064_573750_object75470:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:15:33.008 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:15:15.816204+0000", 2023-10-09T06:15:33.008 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 17.017770582000001, 2023-10-09T06:15:33.008 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1602268950000001, 2023-10-09T06:15:33.008 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:33.008 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:33.008 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:33.009 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:33.009 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:33.009 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171685 2023-10-09T06:15:33.009 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.009 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:33.009 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.009 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:33.009 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816204+0000", 2023-10-09T06:15:33.009 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:33.009 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.009 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.009 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:33.010 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816204+0000", 2023-10-09T06:15:33.010 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:33.010 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.010 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.010 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:33.010 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816203+0000", 2023-10-09T06:15:33.010 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:15:33.010 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.010 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.010 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:33.010 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816207+0000", 2023-10-09T06:15:33.011 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3000000000000003e-06 2023-10-09T06:15:33.011 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.011 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.011 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:33.011 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816220+0000", 2023-10-09T06:15:33.011 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3208e-05 2023-10-09T06:15:33.011 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.011 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.011 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:33.011 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816228+0000", 2023-10-09T06:15:33.011 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6869999999999994e-06 2023-10-09T06:15:33.011 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.012 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.012 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:33.012 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816276+0000", 2023-10-09T06:15:33.012 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7632000000000001e-05 2023-10-09T06:15:33.012 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.012 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.012 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:33.012 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816287+0000", 2023-10-09T06:15:33.012 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0908e-05 2023-10-09T06:15:33.012 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.012 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.013 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:33.013 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794918+0000", 2023-10-09T06:15:33.013 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97863145399999996 2023-10-09T06:15:33.013 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.013 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.013 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:33.013 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794928+0000", 2023-10-09T06:15:33.013 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.005e-05 2023-10-09T06:15:33.013 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.013 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.013 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:33.013 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.958279+0000", 2023-10-09T06:15:33.014 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.163350514 2023-10-09T06:15:33.014 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.014 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.014 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:33.014 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.958431+0000", 2023-10-09T06:15:33.014 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000151946 2023-10-09T06:15:33.014 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.014 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.014 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T06:15:33.014 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.958587+0000", 2023-10-09T06:15:33.014 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015680800000000001 2023-10-09T06:15:33.015 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.015 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.015 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:33.015 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.962454+0000", 2023-10-09T06:15:33.015 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038670100000000002 2023-10-09T06:15:33.015 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.015 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.015 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:33.015 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974038+0000", 2023-10-09T06:15:33.016 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01158314 2023-10-09T06:15:33.016 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.016 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:33.016 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974974+0000", 2023-10-09T06:15:33.016 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00093642900000000002 2023-10-09T06:15:33.016 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.016 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:33.017 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.976244+0000", 2023-10-09T06:15:33.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001270265 2023-10-09T06:15:33.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.017 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.017 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:33.017 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.976304+0000", 2023-10-09T06:15:33.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9621999999999999e-05 2023-10-09T06:15:33.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.017 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.017 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:33.018 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.976430+0000", 2023-10-09T06:15:33.018 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012658100000000001 2023-10-09T06:15:33.018 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:33.018 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:33.018 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:33.018 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.018 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:171712 10.3 10:cbe07bfd:::benchmark_data_smithi064_573750_object75497:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:15:33.018 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:15:15.859776+0000", 2023-10-09T06:15:33.019 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 16.974197702000001, 2023-10-09T06:15:33.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1168703689999999, 2023-10-09T06:15:33.019 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:15:33.019 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:33.019 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:15:33.019 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:15:33.019 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:15:33.019 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171712 2023-10-09T06:15:33.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.019 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:15:33.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:15:33.020 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.859776+0000", 2023-10-09T06:15:33.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:33.020 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:15:33.020 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.859776+0000", 2023-10-09T06:15:33.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:15:33.020 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:15:33.020 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.859775+0000", 2023-10-09T06:15:33.021 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:15:33.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.021 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.021 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:15:33.021 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.859781+0000", 2023-10-09T06:15:33.021 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.733e-06 2023-10-09T06:15:33.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.021 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.021 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:15:33.021 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.859794+0000", 2023-10-09T06:15:33.021 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3067999999999999e-05 2023-10-09T06:15:33.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.022 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.022 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:15:33.022 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.859803+0000", 2023-10-09T06:15:33.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5690000000000005e-06 2023-10-09T06:15:33.022 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.022 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.022 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:33.022 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.859863+0000", 2023-10-09T06:15:33.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0359999999999998e-05 2023-10-09T06:15:33.022 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.022 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:33.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.859876+0000", 2023-10-09T06:15:33.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2473999999999999e-05 2023-10-09T06:15:33.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:33.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.795237+0000", 2023-10-09T06:15:33.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93536119399999995 2023-10-09T06:15:33.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.024 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:15:33.024 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.795247+0000", 2023-10-09T06:15:33.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0268999999999999e-05 2023-10-09T06:15:33.024 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.024 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.024 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:15:33.024 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.962570+0000", 2023-10-09T06:15:33.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1673227690000001 2023-10-09T06:15:33.024 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:15:33.025 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.962691+0000", 2023-10-09T06:15:33.025 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000120765 2023-10-09T06:15:33.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T06:15:33.025 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.962775+0000", 2023-10-09T06:15:33.025 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4136999999999998e-05 2023-10-09T06:15:33.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.026 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:33.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974484+0000", 2023-10-09T06:15:33.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011709166 2023-10-09T06:15:33.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.026 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:33.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.975011+0000", 2023-10-09T06:15:33.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00052740899999999995 2023-10-09T06:15:33.027 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:15:33.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.975127+0000", 2023-10-09T06:15:33.027 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011598 2023-10-09T06:15:33.027 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:33.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.976493+0000", 2023-10-09T06:15:33.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013652670000000001 2023-10-09T06:15:33.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:15:33.028 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.976560+0000", 2023-10-09T06:15:33.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7445000000000002e-05 2023-10-09T06:15:33.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:15:33.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:15:33.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:15:33.029 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.976647+0000", 2023-10-09T06:15:33.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6798000000000005e-05 2023-10-09T06:15:33.029 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:33.029 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:33.029 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:33.029 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:15:33.029 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:15:33.029 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:15:33.030 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:32 smithi064 ceph-mon[220044]: pgmap v1434: 119 pgs: 119 active+clean; 797 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:15:33.030 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:32 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4199881415' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:15:33.030 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:32 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:15:33.030 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:32 smithi064 ceph-mon[212750]: pgmap v1434: 119 pgs: 119 active+clean; 797 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:15:33.031 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:32 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4199881415' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:15:33.031 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:32 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:15:33.125 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:32 smithi156 ceph-mon[195101]: pgmap v1434: 119 pgs: 119 active+clean; 797 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:15:33.125 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:32 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4199881415' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:15:33.125 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:32 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:15:33.584 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:15:33.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:33.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:15:33.693 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:15:33.720 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:33.922 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:33 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:15:33.922 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:33 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:15:33.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:33 smithi064 ceph-mon[212750]: osdmap e566: 8 total, 8 up, 8 in 2023-10-09T06:15:33.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:33 smithi064 ceph-mon[212750]: failed to encode map e566 with expected crc 2023-10-09T06:15:33.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:33 smithi064 ceph-mon[212750]: failed to encode map e566 with expected crc 2023-10-09T06:15:33.923 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:33 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:15:33.923 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:33 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:15:33.923 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:33 smithi064 ceph-mon[220044]: osdmap e566: 8 total, 8 up, 8 in 2023-10-09T06:15:33.923 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:33 smithi064 ceph-mon[220044]: failed to encode map e566 with expected crc 2023-10-09T06:15:33.923 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:33 smithi064 ceph-mon[220044]: failed to encode map e566 with expected crc 2023-10-09T06:15:34.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:33 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:15:34.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:33 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:15:34.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:33 smithi156 ceph-mon[195101]: osdmap e566: 8 total, 8 up, 8 in 2023-10-09T06:15:34.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:33 smithi156 ceph-mon[195101]: failed to encode map e566 with expected crc 2023-10-09T06:15:34.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:33 smithi156 ceph-mon[195101]: failed to encode map e566 with expected crc 2023-10-09T06:15:34.808 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:15:34.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:34.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:15:34.846 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:15:34.931 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:15:35.020 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:35.120 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:35.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:34 smithi156 ceph-mon[195101]: pgmap v1436: 119 pgs: 119 active+clean; 692 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:15:35.173 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:15:35.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:35.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:15:35.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:34 smithi064 ceph-mon[212750]: pgmap v1436: 119 pgs: 119 active+clean; 692 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:15:35.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:34 smithi064 ceph-mon[220044]: pgmap v1436: 119 pgs: 119 active+clean; 692 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:15:35.299 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:15:35.353 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:35.456 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:15:35.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:35.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:15:35.523 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:15:35.566 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:15:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:15:35] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.43.0" 2023-10-09T06:15:35.572 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:15:35.650 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:35.901 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:15:35.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:35.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:15:36.014 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:15:36.087 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:36.089 INFO:teuthology.orchestra.run.smithi064.stderr:Error EINVAL: specified pg_num 81 < pg_num_max 32 2023-10-09T06:15:36.233 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:15:36.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:36.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:15:36.343 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:15:36.375 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:36.477 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:15:36.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:36.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:15:36.581 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:15:36.651 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:36.763 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:15:36.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:36.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:15:36.873 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:15:36.922 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:15:36.931 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:37.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:36 smithi156 ceph-mon[195101]: pgmap v1437: 119 pgs: 119 active+clean; 641 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 2.1 MiB/s, 34 objects/s recovering 2023-10-09T06:15:37.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:36 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/772682748' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "81"}]: dispatch 2023-10-09T06:15:37.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:36 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "81"}]: dispatch 2023-10-09T06:15:37.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:36 smithi064 ceph-mon[212750]: pgmap v1437: 119 pgs: 119 active+clean; 641 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 2.1 MiB/s, 34 objects/s recovering 2023-10-09T06:15:37.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:36 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/772682748' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "81"}]: dispatch 2023-10-09T06:15:37.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:36 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "81"}]: dispatch 2023-10-09T06:15:37.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:36 smithi064 ceph-mon[220044]: pgmap v1437: 119 pgs: 119 active+clean; 641 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 2.1 MiB/s, 34 objects/s recovering 2023-10-09T06:15:37.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:36 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/772682748' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "81"}]: dispatch 2023-10-09T06:15:37.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:36 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "81"}]: dispatch 2023-10-09T06:15:37.251 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:15:37.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:37.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:15:37.356 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:15:37.431 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:37.629 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:15:37.640 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T06:15:37.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:37.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:15:37.744 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:15:37.789 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:37.794 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:37.875 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:15:37.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:37.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:15:37.988 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:15:38.069 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:38.263 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:15:38.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:38.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:15:38.291 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:15:38.326 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:15:38.371 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:15:38.372 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:15:38.415 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:15:38.418 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:38.461 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:15:38.502 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:15:38.536 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:15:38.540 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:15:38.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:38.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:15:38.578 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:15:38.580 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:15:38.580 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:15:38.580 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:15:38.580 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:15:38.580 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:15:38.580 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:15:38.581 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:15:38.581 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:15:38.642 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:15:38.709 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:38.825 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:38 smithi156 ceph-mon[195101]: pgmap v1438: 119 pgs: 119 active+clean; 419 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2023-10-09T06:15:38.825 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:38 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2225095251' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:15:38.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:38 smithi064 ceph-mon[212750]: pgmap v1438: 119 pgs: 119 active+clean; 419 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2023-10-09T06:15:38.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:38 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2225095251' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:15:38.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:38 smithi064 ceph-mon[220044]: pgmap v1438: 119 pgs: 119 active+clean; 419 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2023-10-09T06:15:38.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:38 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2225095251' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:15:39.986 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:15:39.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:39.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:15:40.096 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:15:40.170 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:40.318 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:15:40.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:40.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:15:40.426 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:15:40.527 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:40.653 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:15:40.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:40.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:15:40.764 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:15:40.827 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:40.829 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:40 smithi156 ceph-mon[195101]: pgmap v1439: 119 pgs: 119 active+clean; 419 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2023-10-09T06:15:40.910 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:15:40.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:40.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:15:40.938 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:15:40.938 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:15:41.014 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:15:41.089 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:41.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:40 smithi064 ceph-mon[212750]: pgmap v1439: 119 pgs: 119 active+clean; 419 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2023-10-09T06:15:41.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:40 smithi064 ceph-mon[220044]: pgmap v1439: 119 pgs: 119 active+clean; 419 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2023-10-09T06:15:41.255 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:15:41.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:41.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:15:41.365 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:15:41.412 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:41.456 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:15:41.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:41.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:15:41.559 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:15:41.587 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:41.647 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:15:41.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:41.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:15:41.756 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:15:41.818 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:41.836 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:41 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3493657204' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:15:41.836 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:41 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:15:41.837 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:41 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3493657204' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:15:41.837 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:41 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:15:41.858 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:15:42.069 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:41 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3493657204' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:15:42.069 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:41 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:15:42.097 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:15:42.098 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:15:42.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:42.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:15:42.139 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:15:42.209 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:15:42.279 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:42.387 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:15:42.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:42.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:15:42.499 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:15:42.526 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:42.566 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:15:42.641 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1, 2] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 4, 0] 2023-10-09T06:15:42.641 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:15:42.642 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-10-09T06:15:42.642 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:15:42.840 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:42 smithi064 ceph-mon[212750]: pgmap v1440: 119 pgs: 119 active+clean; 238 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-10-09T06:15:42.840 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:42 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:15:42.840 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:42 smithi064 ceph-mon[212750]: failed to encode map e567 with expected crc 2023-10-09T06:15:42.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:42 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:15:42.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:42 smithi064 ceph-mon[212750]: osdmap e567: 8 total, 8 up, 8 in 2023-10-09T06:15:42.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:42 smithi064 ceph-mon[212750]: failed to encode map e567 with expected crc 2023-10-09T06:15:42.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:42 smithi064 ceph-mon[212750]: failed to encode map e567 with expected crc 2023-10-09T06:15:42.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:42 smithi064 ceph-mon[212750]: failed to encode map e567 with expected crc 2023-10-09T06:15:42.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:42 smithi064 ceph-mon[212750]: failed to encode map e567 with expected crc 2023-10-09T06:15:42.913 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:15:42.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:42.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:15:43.033 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:15:43.086 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:42 smithi156 ceph-mon[195101]: pgmap v1440: 119 pgs: 119 active+clean; 238 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-10-09T06:15:43.086 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:42 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:15:43.086 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:42 smithi156 ceph-mon[195101]: failed to encode map e567 with expected crc 2023-10-09T06:15:43.086 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:42 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:15:43.087 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:42 smithi156 ceph-mon[195101]: osdmap e567: 8 total, 8 up, 8 in 2023-10-09T06:15:43.087 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:42 smithi156 ceph-mon[195101]: failed to encode map e567 with expected crc 2023-10-09T06:15:43.087 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:42 smithi156 ceph-mon[195101]: failed to encode map e567 with expected crc 2023-10-09T06:15:43.087 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:42 smithi156 ceph-mon[195101]: failed to encode map e567 with expected crc 2023-10-09T06:15:43.087 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:42 smithi156 ceph-mon[195101]: failed to encode map e567 with expected crc 2023-10-09T06:15:43.104 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:43.218 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:43.218 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:15:43.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:43.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:15:43.230 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:42 smithi064 ceph-mon[220044]: pgmap v1440: 119 pgs: 119 active+clean; 238 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-10-09T06:15:43.230 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:42 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:15:43.230 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:42 smithi064 ceph-mon[220044]: failed to encode map e567 with expected crc 2023-10-09T06:15:43.230 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:42 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:15:43.230 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:42 smithi064 ceph-mon[220044]: osdmap e567: 8 total, 8 up, 8 in 2023-10-09T06:15:43.230 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:42 smithi064 ceph-mon[220044]: failed to encode map e567 with expected crc 2023-10-09T06:15:43.230 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:42 smithi064 ceph-mon[220044]: failed to encode map e567 with expected crc 2023-10-09T06:15:43.230 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:42 smithi064 ceph-mon[220044]: failed to encode map e567 with expected crc 2023-10-09T06:15:43.231 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:42 smithi064 ceph-mon[220044]: failed to encode map e567 with expected crc 2023-10-09T06:15:43.324 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:15:43.361 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:43.375 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:43.404 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:15:43.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:43.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:15:43.508 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:15:43.540 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:43.597 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Removed 96212 objects 2023-10-09T06:15:43.598 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Clean up completed and total clean up time :139.236 2023-10-09T06:15:43.664 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:15:43.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:43.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:15:43.769 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:15:43.797 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:44.329 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:15:44.329 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:15:44.651 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:15:44.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:44.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:15:44.755 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:15:44.792 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:44 smithi064 ceph-mon[212750]: pgmap v1442: 119 pgs: 119 active+clean; 131 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 897 op/s 2023-10-09T06:15:44.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:44.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:44.842 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:44 smithi064 ceph-mon[220044]: pgmap v1442: 119 pgs: 119 active+clean; 131 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 897 op/s 2023-10-09T06:15:44.842 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:44.842 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:44.843 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:44.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:44 smithi156 ceph-mon[195101]: pgmap v1442: 119 pgs: 119 active+clean; 131 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 897 op/s 2023-10-09T06:15:44.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:44.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:44.896 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:15:44.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:44.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:15:45.000 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:15:45.028 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:45.060 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:15:45.075 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:15:45.175 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:15:45.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:45.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:15:45.279 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:15:45.307 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:45.353 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:15:45.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:45.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:15:45.433 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:15:45.458 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:15:45.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:15:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:15:45] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.43.0" 2023-10-09T06:15:45.492 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:45.739 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:15:45.741 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T06:15:45.789 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:45.794 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:15:45.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:45.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:15:45.898 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:15:45.927 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:45.971 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:15:45.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:45.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:15:45.975 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:15:45.994 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:15:46.014 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:15:46.033 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:15:46.052 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:15:46.071 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:15:46.074 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:15:46.092 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:15:46.104 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:46.115 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:15:46.116 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:15:46.116 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:15:46.116 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:15:46.117 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:15:46.117 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:15:46.117 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:15:46.117 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:15:46.117 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:15:46.254 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:15:46.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:46.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:15:46.358 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:15:46.385 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:46.479 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:15:46.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:46.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:15:46.588 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:15:46.616 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:46.715 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:46 smithi156 ceph-mon[195101]: pgmap v1443: 119 pgs: 119 active+clean; 90 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 881 op/s 2023-10-09T06:15:46.715 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:46 smithi156 ceph-mon[195101]: from='client.36510 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:15:46.715 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:46 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3746478213' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:15:46.764 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:15:46.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:46.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:15:46.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:46 smithi064 ceph-mon[212750]: pgmap v1443: 119 pgs: 119 active+clean; 90 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 881 op/s 2023-10-09T06:15:46.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:46 smithi064 ceph-mon[212750]: from='client.36510 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:15:46.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:46 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3746478213' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:15:46.855 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:46 smithi064 ceph-mon[220044]: pgmap v1443: 119 pgs: 119 active+clean; 90 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 881 op/s 2023-10-09T06:15:46.855 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:46 smithi064 ceph-mon[220044]: from='client.36510 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:15:46.855 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:46 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3746478213' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:15:46.868 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:15:46.900 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:46.915 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:15:46.915 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T06:15:46.915 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T06:15:46.916 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:15:46.916 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.916 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:118905 9.5 9:abe25aca:::benchmark_data_smithi064_511911_object17803:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T06:15:46.916 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:07.826739+0000", 2023-10-09T06:15:46.916 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 399.08551569500003, 2023-10-09T06:15:46.916 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.28890842500000002, 2023-10-09T06:15:46.916 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:15:46.916 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:46.917 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:15:46.917 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:15:46.917 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:15:46.917 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 118905 2023-10-09T06:15:46.917 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.917 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:15:46.917 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.917 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:15:46.918 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.826739+0000", 2023-10-09T06:15:46.918 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:46.918 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.918 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.918 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:15:46.918 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.826739+0000", 2023-10-09T06:15:46.918 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:46.918 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.919 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.919 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:15:46.919 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.826737+0000", 2023-10-09T06:15:46.919 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:15:46.919 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.919 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.919 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:15:46.919 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.826744+0000", 2023-10-09T06:15:46.920 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5339999999999996e-06 2023-10-09T06:15:46.920 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.920 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.920 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:15:46.920 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.826760+0000", 2023-10-09T06:15:46.920 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6149000000000001e-05 2023-10-09T06:15:46.920 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.921 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.921 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:15:46.921 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.826766+0000", 2023-10-09T06:15:46.921 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4559999999999997e-06 2023-10-09T06:15:46.921 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.921 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.921 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:46.921 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.826818+0000", 2023-10-09T06:15:46.921 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1634000000000001e-05 2023-10-09T06:15:46.922 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.922 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.922 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:15:46.922 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.826957+0000", 2023-10-09T06:15:46.922 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013895199999999999 2023-10-09T06:15:46.922 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.923 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.923 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:15:46.923 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.827119+0000", 2023-10-09T06:15:46.923 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000161979 2023-10-09T06:15:46.923 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.923 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.923 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:15:46.923 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.835830+0000", 2023-10-09T06:15:46.923 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0087111010000000006 2023-10-09T06:15:46.924 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.924 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.924 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:46.924 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.837632+0000", 2023-10-09T06:15:46.924 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001801506 2023-10-09T06:15:46.924 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.924 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.924 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:46.924 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.115484+0000", 2023-10-09T06:15:46.924 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.27785286599999998 2023-10-09T06:15:46.925 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.925 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.925 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:15:46.925 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.115545+0000", 2023-10-09T06:15:46.925 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0812e-05 2023-10-09T06:15:46.925 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.925 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.925 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:15:46.925 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.115647+0000", 2023-10-09T06:15:46.925 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010185500000000001 2023-10-09T06:15:46.925 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:46.925 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:15:46.926 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:46.926 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.926 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.926 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:118910 9.5 9:aa92e6b2:::benchmark_data_smithi064_511911_object17808:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T06:15:46.926 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:07.827816+0000", 2023-10-09T06:15:46.926 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 399.08443859400001, 2023-10-09T06:15:46.926 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.28799813499999999, 2023-10-09T06:15:46.926 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:15:46.926 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:46.926 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:15:46.926 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:15:46.927 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:15:46.927 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 118910 2023-10-09T06:15:46.927 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.927 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:15:46.927 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.927 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:15:46.927 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.827816+0000", 2023-10-09T06:15:46.927 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:46.927 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.927 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.927 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:15:46.928 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.827816+0000", 2023-10-09T06:15:46.928 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:46.928 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.928 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.928 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:15:46.928 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.827815+0000", 2023-10-09T06:15:46.928 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:15:46.928 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.928 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.928 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:15:46.928 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.827824+0000", 2023-10-09T06:15:46.929 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0729999999999994e-06 2023-10-09T06:15:46.929 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.929 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.929 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:15:46.929 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.827836+0000", 2023-10-09T06:15:46.929 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2374999999999999e-05 2023-10-09T06:15:46.929 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.929 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.929 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:15:46.929 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.827843+0000", 2023-10-09T06:15:46.929 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2079999999999997e-06 2023-10-09T06:15:46.930 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.930 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.930 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:46.930 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.830165+0000", 2023-10-09T06:15:46.930 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0023226179999999998 2023-10-09T06:15:46.930 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.930 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.930 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:15:46.930 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.830259+0000", 2023-10-09T06:15:46.930 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3519999999999999e-05 2023-10-09T06:15:46.930 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.931 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.931 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:15:46.931 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.830357+0000", 2023-10-09T06:15:46.931 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7931999999999997e-05 2023-10-09T06:15:46.931 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.931 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.931 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:15:46.931 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.835838+0000", 2023-10-09T06:15:46.931 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0054813279999999997 2023-10-09T06:15:46.931 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.931 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.932 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:46.932 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.837660+0000", 2023-10-09T06:15:46.932 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0018216009999999999 2023-10-09T06:15:46.932 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.932 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.932 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:46.932 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.115684+0000", 2023-10-09T06:15:46.932 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.27802399700000002 2023-10-09T06:15:46.932 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.933 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.933 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:15:46.933 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.115722+0000", 2023-10-09T06:15:46.934 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8550999999999997e-05 2023-10-09T06:15:46.934 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.934 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.934 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:15:46.934 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.115814+0000", 2023-10-09T06:15:46.934 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1831999999999998e-05 2023-10-09T06:15:46.934 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:46.934 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:15:46.934 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:46.934 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.935 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.935 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:118921 9.5 9:a45440d7:::benchmark_data_smithi064_511911_object17819:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T06:15:46.935 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:07.843763+0000", 2023-10-09T06:15:46.935 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 399.068491053, 2023-10-09T06:15:46.935 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.27222496800000001, 2023-10-09T06:15:46.935 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:15:46.935 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:46.936 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:15:46.936 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:15:46.936 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:15:46.936 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 118921 2023-10-09T06:15:46.936 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.936 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:15:46.936 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.936 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:15:46.936 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.843763+0000", 2023-10-09T06:15:46.937 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:46.937 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.937 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.937 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:15:46.937 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.843763+0000", 2023-10-09T06:15:46.937 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:46.937 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.937 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.937 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:15:46.938 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.843762+0000", 2023-10-09T06:15:46.938 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:15:46.938 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.938 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.939 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:15:46.939 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.843769+0000", 2023-10-09T06:15:46.939 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8059999999999997e-06 2023-10-09T06:15:46.939 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.939 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.939 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:15:46.940 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.843780+0000", 2023-10-09T06:15:46.940 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1321000000000001e-05 2023-10-09T06:15:46.940 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.940 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.940 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:15:46.940 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.843787+0000", 2023-10-09T06:15:46.940 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7839999999999998e-06 2023-10-09T06:15:46.940 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.940 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.940 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:46.940 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.843825+0000", 2023-10-09T06:15:46.940 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8098e-05 2023-10-09T06:15:46.941 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.941 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.941 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:15:46.941 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.843921+0000", 2023-10-09T06:15:46.941 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.6386999999999997e-05 2023-10-09T06:15:46.941 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.941 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.941 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:15:46.941 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.843999+0000", 2023-10-09T06:15:46.941 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7979000000000007e-05 2023-10-09T06:15:46.941 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.941 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.942 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:15:46.942 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.847809+0000", 2023-10-09T06:15:46.942 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0038100170000000002 2023-10-09T06:15:46.942 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.942 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.942 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:46.942 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.854122+0000", 2023-10-09T06:15:46.943 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0063132070000000004 2023-10-09T06:15:46.943 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.943 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.943 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:46.943 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.115847+0000", 2023-10-09T06:15:46.943 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.26172430899999999 2023-10-09T06:15:46.943 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.943 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.943 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:15:46.943 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.115895+0000", 2023-10-09T06:15:46.943 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8265e-05 2023-10-09T06:15:46.944 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.944 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.944 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:15:46.944 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.115988+0000", 2023-10-09T06:15:46.944 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3286000000000001e-05 2023-10-09T06:15:46.944 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:46.944 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:15:46.944 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:46.944 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.944 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.944 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:118928 9.5 9:a5a9efbb:::benchmark_data_smithi064_511911_object17826:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T06:15:46.944 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:07.850035+0000", 2023-10-09T06:15:46.945 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 399.06221944499998, 2023-10-09T06:15:46.945 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.26612815200000001, 2023-10-09T06:15:46.945 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:15:46.945 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:46.945 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:15:46.945 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:15:46.945 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:15:46.945 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 118928 2023-10-09T06:15:46.945 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.945 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:15:46.945 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.945 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:15:46.946 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.850035+0000", 2023-10-09T06:15:46.946 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:46.946 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.946 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.946 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:15:46.946 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.850035+0000", 2023-10-09T06:15:46.946 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:46.946 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.946 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.946 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:15:46.946 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.850034+0000", 2023-10-09T06:15:46.946 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:15:46.947 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.947 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.947 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:15:46.947 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.850041+0000", 2023-10-09T06:15:46.947 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0230000000000004e-06 2023-10-09T06:15:46.947 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.947 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.949 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:15:46.949 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.850052+0000", 2023-10-09T06:15:46.949 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1402e-05 2023-10-09T06:15:46.949 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.949 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.950 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:15:46.950 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.850057+0000", 2023-10-09T06:15:46.950 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.826e-06 2023-10-09T06:15:46.950 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.950 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.950 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:46.950 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.850099+0000", 2023-10-09T06:15:46.950 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1897999999999998e-05 2023-10-09T06:15:46.950 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.950 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.950 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:15:46.950 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.850235+0000", 2023-10-09T06:15:46.951 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013637400000000001 2023-10-09T06:15:46.951 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.951 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.951 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:15:46.951 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.850361+0000", 2023-10-09T06:15:46.951 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012588500000000001 2023-10-09T06:15:46.951 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.951 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.951 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:15:46.951 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.853287+0000", 2023-10-09T06:15:46.951 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0029253259999999998 2023-10-09T06:15:46.951 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.952 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.952 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:46.952 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:07.854302+0000", 2023-10-09T06:15:46.952 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001015192 2023-10-09T06:15:46.952 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.952 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.952 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:46.953 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.116034+0000", 2023-10-09T06:15:46.953 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.26173186300000001 2023-10-09T06:15:46.953 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.953 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.953 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:15:46.953 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.116072+0000", 2023-10-09T06:15:46.953 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8405999999999998e-05 2023-10-09T06:15:46.953 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.953 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.953 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:15:46.953 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:08.116163+0000", 2023-10-09T06:15:46.953 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1043000000000002e-05 2023-10-09T06:15:46.954 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:46.954 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:15:46.954 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:46.954 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.954 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.954 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:132908 9.5 9:a62fba3d:::benchmark_data_smithi064_511911_object31806:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e455)", 2023-10-09T06:15:46.954 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:26.572407+0000", 2023-10-09T06:15:46.954 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 380.339847351, 2023-10-09T06:15:46.954 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.177650452, 2023-10-09T06:15:46.954 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:15:46.954 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:15:46.954 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:15:46.955 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:15:46.955 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:15:46.955 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 132908 2023-10-09T06:15:46.955 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.955 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:15:46.955 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.955 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:15:46.955 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572407+0000", 2023-10-09T06:15:46.955 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:46.955 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.955 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.955 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:15:46.956 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572407+0000", 2023-10-09T06:15:46.956 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:46.956 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.956 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.956 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:15:46.956 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572406+0000", 2023-10-09T06:15:46.956 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:15:46.956 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.956 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.956 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:15:46.956 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572416+0000", 2023-10-09T06:15:46.956 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.9879999999999999e-06 2023-10-09T06:15:46.957 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.957 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.957 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:15:46.957 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572433+0000", 2023-10-09T06:15:46.957 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7169000000000001e-05 2023-10-09T06:15:46.957 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.957 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.957 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:15:46.957 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572441+0000", 2023-10-09T06:15:46.957 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1089999999999998e-06 2023-10-09T06:15:46.957 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.957 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.958 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:46.958 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572523+0000", 2023-10-09T06:15:46.958 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1679000000000002e-05 2023-10-09T06:15:46.958 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.958 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.958 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:15:46.958 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572680+0000", 2023-10-09T06:15:46.958 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015694099999999999 2023-10-09T06:15:46.958 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.958 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.958 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:15:46.959 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572798+0000", 2023-10-09T06:15:46.959 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000118408 2023-10-09T06:15:46.959 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.959 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.959 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:15:46.959 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.584244+0000", 2023-10-09T06:15:46.959 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.01144565 2023-10-09T06:15:46.959 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.959 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.959 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:46.959 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.586841+0000", 2023-10-09T06:15:46.960 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002596887 2023-10-09T06:15:46.960 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.960 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.960 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:15:46.960 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.586852+0000", 2023-10-09T06:15:46.960 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1617000000000001e-05 2023-10-09T06:15:46.960 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.960 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.960 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:46.960 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.730544+0000", 2023-10-09T06:15:46.961 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1436917819999999 2023-10-09T06:15:46.961 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.961 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.961 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for missing object", 2023-10-09T06:15:46.961 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.730669+0000", 2023-10-09T06:15:46.961 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012532400000000001 2023-10-09T06:15:46.961 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.961 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.961 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:46.961 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.749944+0000", 2023-10-09T06:15:46.961 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.019274291999999998 2023-10-09T06:15:46.962 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.962 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.962 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:15:46.962 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.750058+0000", 2023-10-09T06:15:46.962 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000113964 2023-10-09T06:15:46.962 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:46.962 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:15:46.962 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:46.962 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.962 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.962 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:132912 9.5 9:aec1ab85:::benchmark_data_smithi064_511911_object31810:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e455)", 2023-10-09T06:15:46.963 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:26.572458+0000", 2023-10-09T06:15:46.963 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 380.33979637599998, 2023-10-09T06:15:46.963 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.180201992, 2023-10-09T06:15:46.963 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:15:46.963 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:15:46.963 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:15:46.963 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:15:46.963 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:15:46.963 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 132912 2023-10-09T06:15:46.963 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.963 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:15:46.963 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.964 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:15:46.964 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572458+0000", 2023-10-09T06:15:46.964 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:46.964 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.964 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.964 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:15:46.964 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572458+0000", 2023-10-09T06:15:46.964 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:46.964 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.964 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.964 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:15:46.965 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572457+0000", 2023-10-09T06:15:46.965 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:15:46.965 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.965 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.965 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:15:46.965 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572462+0000", 2023-10-09T06:15:46.965 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4660000000000001e-06 2023-10-09T06:15:46.965 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.965 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.965 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:15:46.965 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572473+0000", 2023-10-09T06:15:46.965 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1507e-05 2023-10-09T06:15:46.966 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.966 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.966 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:15:46.966 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572479+0000", 2023-10-09T06:15:46.966 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.361e-06 2023-10-09T06:15:46.966 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.966 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.966 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:46.966 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.576291+0000", 2023-10-09T06:15:46.966 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0038126309999999999 2023-10-09T06:15:46.966 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.966 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.967 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:15:46.967 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.576420+0000", 2023-10-09T06:15:46.967 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012867799999999999 2023-10-09T06:15:46.967 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.967 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.967 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:15:46.967 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.576543+0000", 2023-10-09T06:15:46.967 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000122886 2023-10-09T06:15:46.967 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.967 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.967 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:15:46.967 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.584250+0000", 2023-10-09T06:15:46.968 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0077072110000000003 2023-10-09T06:15:46.968 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.968 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.968 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:46.968 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.586885+0000", 2023-10-09T06:15:46.968 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0026347440000000001 2023-10-09T06:15:46.968 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.968 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.968 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:15:46.968 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.586892+0000", 2023-10-09T06:15:46.968 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0829999999999998e-06 2023-10-09T06:15:46.968 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.730711+0000", 2023-10-09T06:15:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.143819385 2023-10-09T06:15:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for missing object", 2023-10-09T06:15:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.730800+0000", 2023-10-09T06:15:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8787000000000002e-05 2023-10-09T06:15:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.969 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:46.970 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.752573+0000", 2023-10-09T06:15:46.970 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.02177246 2023-10-09T06:15:46.970 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.970 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.970 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:15:46.970 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.752660+0000", 2023-10-09T06:15:46.970 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.7353000000000002e-05 2023-10-09T06:15:46.970 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:46.970 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:15:46.970 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:46.970 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.971 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.971 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:132915 9.5 9:abaae183:::benchmark_data_smithi064_511911_object31813:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e455)", 2023-10-09T06:15:46.971 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:26.574849+0000", 2023-10-09T06:15:46.971 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 380.33740502400002, 2023-10-09T06:15:46.971 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1802385870000001, 2023-10-09T06:15:46.971 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:15:46.971 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:15:46.971 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:15:46.971 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:15:46.971 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:15:46.971 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 132915 2023-10-09T06:15:46.971 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.972 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:15:46.972 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.972 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:15:46.972 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.574849+0000", 2023-10-09T06:15:46.972 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:46.972 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.972 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.972 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:15:46.972 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.574849+0000", 2023-10-09T06:15:46.972 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:46.972 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.972 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:15:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.574848+0000", 2023-10-09T06:15:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:15:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:15:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.574855+0000", 2023-10-09T06:15:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2520000000000002e-06 2023-10-09T06:15:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:15:46.973 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.574872+0000", 2023-10-09T06:15:46.974 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6239000000000002e-05 2023-10-09T06:15:46.974 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.974 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.974 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:15:46.974 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.574878+0000", 2023-10-09T06:15:46.974 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8600000000000004e-06 2023-10-09T06:15:46.974 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.974 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.974 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:46.974 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.580205+0000", 2023-10-09T06:15:46.974 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0053268100000000004 2023-10-09T06:15:46.975 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.975 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.975 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:15:46.975 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.580339+0000", 2023-10-09T06:15:46.975 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000134074 2023-10-09T06:15:46.975 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.975 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.975 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:15:46.975 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.580463+0000", 2023-10-09T06:15:46.975 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012364 2023-10-09T06:15:46.975 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.977 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.977 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:46.977 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.586919+0000", 2023-10-09T06:15:46.977 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0064555890000000003 2023-10-09T06:15:46.977 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.977 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.977 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:15:46.977 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.586925+0000", 2023-10-09T06:15:46.978 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.81e-06 2023-10-09T06:15:46.978 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.978 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.978 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:46.978 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.730833+0000", 2023-10-09T06:15:46.978 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.143908047 2023-10-09T06:15:46.978 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.978 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.978 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for missing object", 2023-10-09T06:15:46.978 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.730909+0000", 2023-10-09T06:15:46.978 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5944000000000003e-05 2023-10-09T06:15:46.978 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.979 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.979 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:46.979 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.755019+0000", 2023-10-09T06:15:46.979 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.024109756 2023-10-09T06:15:46.979 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.979 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.979 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:15:46.979 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.755088+0000", 2023-10-09T06:15:46.979 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8837000000000005e-05 2023-10-09T06:15:46.979 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:46.979 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:15:46.979 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:46.980 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.980 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.980 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:132921 9.5 9:a6236301:::benchmark_data_smithi064_511911_object31819:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e456)", 2023-10-09T06:15:46.980 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:26.641383+0000", 2023-10-09T06:15:46.980 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 380.270871457, 2023-10-09T06:15:46.980 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1059502830000001, 2023-10-09T06:15:46.980 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:15:46.980 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:46.980 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:15:46.980 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:15:46.980 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:15:46.980 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 132921 2023-10-09T06:15:46.981 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.981 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:15:46.981 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.981 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:15:46.981 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.641383+0000", 2023-10-09T06:15:46.981 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:46.981 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.981 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.981 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:15:46.981 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.641383+0000", 2023-10-09T06:15:46.981 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:46.981 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.982 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.982 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:15:46.982 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.641382+0000", 2023-10-09T06:15:46.982 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:15:46.982 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.982 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.982 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:15:46.982 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.641387+0000", 2023-10-09T06:15:46.982 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5899999999999998e-06 2023-10-09T06:15:46.982 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.982 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.982 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:15:46.983 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.641400+0000", 2023-10-09T06:15:46.983 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2187999999999999e-05 2023-10-09T06:15:46.983 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.983 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.983 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:15:46.983 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.641405+0000", 2023-10-09T06:15:46.983 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3419999999999998e-06 2023-10-09T06:15:46.983 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.983 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.983 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:46.983 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.641457+0000", 2023-10-09T06:15:46.983 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2496e-05 2023-10-09T06:15:46.984 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.984 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.984 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:15:46.984 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.641468+0000", 2023-10-09T06:15:46.984 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0890000000000001e-05 2023-10-09T06:15:46.984 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.984 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.984 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:46.984 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.730943+0000", 2023-10-09T06:15:46.984 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0894744140000001 2023-10-09T06:15:46.984 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.984 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:15:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.731044+0000", 2023-10-09T06:15:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010098999999999999 2023-10-09T06:15:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1,7", 2023-10-09T06:15:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.731138+0000", 2023-10-09T06:15:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.4436000000000002e-05 2023-10-09T06:15:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:46.985 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.735600+0000", 2023-10-09T06:15:46.986 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0044618449999999999 2023-10-09T06:15:46.986 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.986 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.986 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:46.986 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.735653+0000", 2023-10-09T06:15:46.986 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3016999999999997e-05 2023-10-09T06:15:46.986 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.986 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.986 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:15:46.986 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.735663+0000", 2023-10-09T06:15:46.986 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0472000000000001e-05 2023-10-09T06:15:46.986 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.987 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.987 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:46.987 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.745120+0000", 2023-10-09T06:15:46.987 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0094562810000000004 2023-10-09T06:15:46.987 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.987 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.987 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:15:46.987 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.745189+0000", 2023-10-09T06:15:46.987 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8912999999999996e-05 2023-10-09T06:15:46.987 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.987 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.987 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:15:46.988 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.747333+0000", 2023-10-09T06:15:46.988 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0021445589999999999 2023-10-09T06:15:46.988 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:46.988 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:15:46.988 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:46.988 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.988 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.988 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:134661 9.5 9:a335a6bc:::benchmark_data_smithi064_511911_object33559:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e458)", 2023-10-09T06:15:46.988 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:31.499352+0000", 2023-10-09T06:15:46.988 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 375.41290205400003, 2023-10-09T06:15:46.988 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.250994569, 2023-10-09T06:15:46.988 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:15:46.989 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:46.989 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:15:46.989 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:15:46.989 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:15:46.989 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 134661 2023-10-09T06:15:46.989 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.989 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:15:46.989 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.989 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:15:46.989 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.499352+0000", 2023-10-09T06:15:46.989 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:46.989 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.990 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.990 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:15:46.990 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.499352+0000", 2023-10-09T06:15:46.990 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:46.990 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.990 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.990 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:15:46.990 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.499351+0000", 2023-10-09T06:15:46.990 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:15:46.990 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.990 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.990 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:15:46.991 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.499358+0000", 2023-10-09T06:15:46.991 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0419999999999997e-06 2023-10-09T06:15:46.991 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.991 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.991 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:15:46.991 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.499370+0000", 2023-10-09T06:15:46.991 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1891e-05 2023-10-09T06:15:46.991 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.991 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.991 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:15:46.991 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.499376+0000", 2023-10-09T06:15:46.991 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6350000000000003e-06 2023-10-09T06:15:46.992 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.992 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.992 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:46.992 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.500390+0000", 2023-10-09T06:15:46.992 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0010140360000000001 2023-10-09T06:15:46.992 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.992 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.992 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:15:46.992 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.500496+0000", 2023-10-09T06:15:46.992 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000106449 2023-10-09T06:15:46.992 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.992 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.993 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1,7", 2023-10-09T06:15:46.993 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.500584+0000", 2023-10-09T06:15:46.993 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.81e-05 2023-10-09T06:15:46.993 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.993 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.993 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:15:46.993 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.512762+0000", 2023-10-09T06:15:46.993 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012178193 2023-10-09T06:15:46.993 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.993 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.993 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:46.993 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.512800+0000", 2023-10-09T06:15:46.994 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7969999999999997e-05 2023-10-09T06:15:46.994 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.994 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.994 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:46.994 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.512827+0000", 2023-10-09T06:15:46.994 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7047e-05 2023-10-09T06:15:46.994 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.994 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.994 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:46.994 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.748195+0000", 2023-10-09T06:15:46.994 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.23536779599999999 2023-10-09T06:15:46.995 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.995 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.995 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:15:46.995 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.748249+0000", 2023-10-09T06:15:46.995 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3430999999999997e-05 2023-10-09T06:15:46.995 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.995 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.995 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:15:46.995 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:31.750347+0000", 2023-10-09T06:15:46.995 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002098262 2023-10-09T06:15:46.995 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:46.995 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:15:46.995 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:46.996 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.996 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.996 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:153149 9.5 9:a6ab5184:::benchmark_data_smithi064_511911_object52047:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e463)", 2023-10-09T06:15:46.996 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:10:03.425900+0000", 2023-10-09T06:15:46.996 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 343.48635416399998, 2023-10-09T06:15:46.996 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.240294219, 2023-10-09T06:15:46.996 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:15:46.996 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:46.996 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:15:46.996 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:15:46.996 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:15:46.996 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 153149 2023-10-09T06:15:46.997 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.997 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:15:46.997 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.997 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:15:46.997 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.425900+0000", 2023-10-09T06:15:46.997 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:46.997 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.997 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.997 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:15:46.997 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.425900+0000", 2023-10-09T06:15:46.997 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:46.998 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.998 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.998 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:15:46.998 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.425900+0000", 2023-10-09T06:15:46.998 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:15:46.998 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.998 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.998 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:15:46.998 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.425904+0000", 2023-10-09T06:15:46.998 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0999999999999997e-06 2023-10-09T06:15:46.998 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.998 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.999 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:15:46.999 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.425916+0000", 2023-10-09T06:15:46.999 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2787e-05 2023-10-09T06:15:46.999 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.999 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.999 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:15:46.999 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.425922+0000", 2023-10-09T06:15:46.999 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5300000000000004e-06 2023-10-09T06:15:46.999 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:46.999 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:46.999 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:47.000 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.432133+0000", 2023-10-09T06:15:47.000 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0062113849999999998 2023-10-09T06:15:47.000 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.000 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.000 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:15:47.000 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.432299+0000", 2023-10-09T06:15:47.000 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000166014 2023-10-09T06:15:47.000 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.000 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.000 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1,7", 2023-10-09T06:15:47.000 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.432478+0000", 2023-10-09T06:15:47.000 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00017833899999999999 2023-10-09T06:15:47.000 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.001 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.001 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:15:47.001 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.436209+0000", 2023-10-09T06:15:47.001 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.003731079 2023-10-09T06:15:47.001 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.001 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.001 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:47.001 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.439000+0000", 2023-10-09T06:15:47.001 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0027908239999999999 2023-10-09T06:15:47.001 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.001 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.001 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:47.002 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.451004+0000", 2023-10-09T06:15:47.002 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012004795 2023-10-09T06:15:47.002 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.002 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.002 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:47.002 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.665717+0000", 2023-10-09T06:15:47.002 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.21471283899999999 2023-10-09T06:15:47.002 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.002 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.002 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:15:47.002 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.665754+0000", 2023-10-09T06:15:47.003 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7042999999999998e-05 2023-10-09T06:15:47.003 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.003 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.003 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:15:47.003 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:03.666194+0000", 2023-10-09T06:15:47.003 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000440133 2023-10-09T06:15:47.003 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:47.003 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:15:47.003 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:47.003 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.003 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.003 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:178997 9.5 9:a657cd64:::benchmark_data_smithi064_511911_object77895:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T06:15:47.004 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:10:44.053726+0000", 2023-10-09T06:15:47.004 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 302.85852867900002, 2023-10-09T06:15:47.004 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.27684753699999998, 2023-10-09T06:15:47.004 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:15:47.004 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:47.004 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:15:47.004 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:15:47.004 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:15:47.004 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 178997 2023-10-09T06:15:47.004 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.004 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:15:47.004 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.005 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:15:47.005 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.053726+0000", 2023-10-09T06:15:47.005 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:47.005 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.005 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.005 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:15:47.005 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.053726+0000", 2023-10-09T06:15:47.005 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:47.005 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.005 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.005 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:15:47.005 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.053724+0000", 2023-10-09T06:15:47.006 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:15:47.006 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.006 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.006 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:15:47.006 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.053734+0000", 2023-10-09T06:15:47.006 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2399999999999996e-06 2023-10-09T06:15:47.006 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.006 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.006 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:15:47.006 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.053745+0000", 2023-10-09T06:15:47.006 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1708e-05 2023-10-09T06:15:47.006 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.007 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.007 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:15:47.007 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.053751+0000", 2023-10-09T06:15:47.007 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9479999999999996e-06 2023-10-09T06:15:47.007 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.007 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.007 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:47.007 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.053791+0000", 2023-10-09T06:15:47.007 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9721000000000001e-05 2023-10-09T06:15:47.007 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.007 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.008 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:15:47.008 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.053933+0000", 2023-10-09T06:15:47.008 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014163500000000001 2023-10-09T06:15:47.008 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.008 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.008 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1,7", 2023-10-09T06:15:47.008 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.054086+0000", 2023-10-09T06:15:47.008 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015320700000000001 2023-10-09T06:15:47.008 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.008 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.008 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:15:47.008 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.058079+0000", 2023-10-09T06:15:47.009 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0039933429999999999 2023-10-09T06:15:47.009 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.009 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.009 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:47.009 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.061474+0000", 2023-10-09T06:15:47.009 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0033949919999999999 2023-10-09T06:15:47.009 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.009 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.009 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:47.009 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.067939+0000", 2023-10-09T06:15:47.009 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0064653269999999999 2023-10-09T06:15:47.009 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.010 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.010 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:47.010 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.330290+0000", 2023-10-09T06:15:47.010 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.26235035800000001 2023-10-09T06:15:47.010 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.010 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.010 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:15:47.010 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.330328+0000", 2023-10-09T06:15:47.010 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8448999999999997e-05 2023-10-09T06:15:47.010 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.010 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.010 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:15:47.011 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.330573+0000", 2023-10-09T06:15:47.011 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00024499500000000002 2023-10-09T06:15:47.011 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:47.011 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:15:47.011 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:47.011 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.011 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.011 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:179005 9.5 9:a7a3d65d:::benchmark_data_smithi064_511911_object77903:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T06:15:47.011 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:10:44.064308+0000", 2023-10-09T06:15:47.011 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 302.84794680800002, 2023-10-09T06:15:47.011 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.26648006499999999, 2023-10-09T06:15:47.012 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:15:47.012 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:47.012 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:15:47.012 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:15:47.012 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:15:47.012 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 179005 2023-10-09T06:15:47.012 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.012 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:15:47.012 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.012 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:15:47.012 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.064308+0000", 2023-10-09T06:15:47.012 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:47.013 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.013 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.013 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:15:47.013 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.064308+0000", 2023-10-09T06:15:47.013 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:47.013 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.013 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.013 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:15:47.013 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.064306+0000", 2023-10-09T06:15:47.013 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:15:47.013 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.013 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.014 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:15:47.014 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.064312+0000", 2023-10-09T06:15:47.014 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.665e-06 2023-10-09T06:15:47.014 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.014 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.014 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:15:47.014 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.064325+0000", 2023-10-09T06:15:47.014 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3108000000000001e-05 2023-10-09T06:15:47.014 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.014 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.014 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:15:47.015 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.064334+0000", 2023-10-09T06:15:47.015 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2830000000000007e-06 2023-10-09T06:15:47.015 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.015 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.015 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:47.015 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.064382+0000", 2023-10-09T06:15:47.015 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7667000000000003e-05 2023-10-09T06:15:47.015 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.015 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.015 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:15:47.015 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.064488+0000", 2023-10-09T06:15:47.015 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010548999999999999 2023-10-09T06:15:47.016 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.016 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.016 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1,7", 2023-10-09T06:15:47.016 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.064585+0000", 2023-10-09T06:15:47.016 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7508000000000002e-05 2023-10-09T06:15:47.016 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.016 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.016 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:15:47.016 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.067958+0000", 2023-10-09T06:15:47.016 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.003372608 2023-10-09T06:15:47.016 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.016 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.017 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:47.017 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.068016+0000", 2023-10-09T06:15:47.017 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8072000000000002e-05 2023-10-09T06:15:47.017 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.017 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.017 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:47.017 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.068603+0000", 2023-10-09T06:15:47.017 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00058710100000000003 2023-10-09T06:15:47.017 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.017 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.017 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:15:47.018 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.330621+0000", 2023-10-09T06:15:47.018 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.26201773499999997 2023-10-09T06:15:47.018 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.018 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.018 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:15:47.018 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.330665+0000", 2023-10-09T06:15:47.018 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4579000000000001e-05 2023-10-09T06:15:47.018 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.018 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.018 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:15:47.018 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:44.330788+0000", 2023-10-09T06:15:47.018 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012247500000000001 2023-10-09T06:15:47.019 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:47.019 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:15:47.019 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:47.019 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.019 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.019 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.46283.0:147581 10.a e550/490 10:5b7006ee:::benchmark_data_smithi064_573750_object51366:head v 550'9116, mlcod=550'9112)", 2023-10-09T06:15:47.019 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:14:45.042721+0000", 2023-10-09T06:15:47.019 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 61.869533683999997, 2023-10-09T06:15:47.019 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.029104343000000001, 2023-10-09T06:15:47.019 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:15:47.019 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:47.019 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:15:47.020 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.020 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:15:47.020 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.042721+0000", 2023-10-09T06:15:47.020 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:47.020 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.020 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.020 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:15:47.020 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.042721+0000", 2023-10-09T06:15:47.020 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:47.020 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.020 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.020 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:15:47.021 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.042720+0000", 2023-10-09T06:15:47.021 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:15:47.021 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.021 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.021 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:15:47.021 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.042727+0000", 2023-10-09T06:15:47.021 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.17e-06 2023-10-09T06:15:47.021 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.021 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.021 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:15:47.021 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.042738+0000", 2023-10-09T06:15:47.021 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0834e-05 2023-10-09T06:15:47.022 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.022 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.022 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:15:47.022 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.042743+0000", 2023-10-09T06:15:47.022 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4009999999999996e-06 2023-10-09T06:15:47.022 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.022 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.022 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:47.022 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.050199+0000", 2023-10-09T06:15:47.022 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0074555719999999997 2023-10-09T06:15:47.022 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.022 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.023 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:15:47.023 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.050219+0000", 2023-10-09T06:15:47.023 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0092e-05 2023-10-09T06:15:47.023 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.023 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.023 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:15:47.023 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.071786+0000", 2023-10-09T06:15:47.023 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.021566674000000001 2023-10-09T06:15:47.023 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.023 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.023 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:15:47.024 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.071825+0000", 2023-10-09T06:15:47.024 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9507e-05 2023-10-09T06:15:47.024 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:47.024 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:15:47.024 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:47.024 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.024 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.024 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.46283.0:147592 10.a e550/490 10:5e5d8ae2:::benchmark_data_smithi064_573750_object51377:head v 550'9117, mlcod=550'9113)", 2023-10-09T06:15:47.024 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:14:45.044043+0000", 2023-10-09T06:15:47.024 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 61.868210941000001, 2023-10-09T06:15:47.024 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.027886195999999999, 2023-10-09T06:15:47.024 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:15:47.025 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:47.025 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:15:47.025 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.025 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:15:47.025 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.044043+0000", 2023-10-09T06:15:47.025 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:47.025 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.025 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.025 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:15:47.025 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.044043+0000", 2023-10-09T06:15:47.025 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:47.025 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.026 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.026 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:15:47.026 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.044043+0000", 2023-10-09T06:15:47.026 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:15:47.026 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.026 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.026 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:15:47.026 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.044049+0000", 2023-10-09T06:15:47.026 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1310000000000003e-06 2023-10-09T06:15:47.026 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.026 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.027 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:15:47.027 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.044058+0000", 2023-10-09T06:15:47.027 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.431e-06 2023-10-09T06:15:47.027 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.027 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.027 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:15:47.027 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.044063+0000", 2023-10-09T06:15:47.027 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0499999999999999e-06 2023-10-09T06:15:47.027 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.027 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.027 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:47.028 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.053799+0000", 2023-10-09T06:15:47.028 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0097355700000000007 2023-10-09T06:15:47.028 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.028 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.028 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:15:47.028 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.053816+0000", 2023-10-09T06:15:47.028 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6983e-05 2023-10-09T06:15:47.028 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.028 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.028 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:15:47.028 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.071902+0000", 2023-10-09T06:15:47.028 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018085793999999999 2023-10-09T06:15:47.029 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.029 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.029 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:15:47.029 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.071930+0000", 2023-10-09T06:15:47.029 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7858999999999999e-05 2023-10-09T06:15:47.029 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:47.029 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:15:47.029 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:47.029 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.029 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.029 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.46283.0:147594 10.a e550/490 10:531548d7:::benchmark_data_smithi064_573750_object51379:head v 550'9118, mlcod=550'9113)", 2023-10-09T06:15:47.030 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:14:45.051175+0000", 2023-10-09T06:15:47.030 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 61.861079740999998, 2023-10-09T06:15:47.030 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.025487552, 2023-10-09T06:15:47.030 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:15:47.030 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:47.030 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:15:47.030 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.030 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:15:47.030 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.051175+0000", 2023-10-09T06:15:47.030 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:47.030 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.030 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.031 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:15:47.031 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.051175+0000", 2023-10-09T06:15:47.031 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:47.031 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.031 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.031 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:15:47.031 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.051173+0000", 2023-10-09T06:15:47.031 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:15:47.031 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.031 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.031 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:15:47.031 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.051186+0000", 2023-10-09T06:15:47.032 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2459e-05 2023-10-09T06:15:47.032 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.032 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.032 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:15:47.032 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.051198+0000", 2023-10-09T06:15:47.032 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2503999999999999e-05 2023-10-09T06:15:47.032 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.032 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.032 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:15:47.032 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.051205+0000", 2023-10-09T06:15:47.032 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1919999999999999e-06 2023-10-09T06:15:47.032 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.033 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.033 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:47.033 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.073077+0000", 2023-10-09T06:15:47.033 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.021871775 2023-10-09T06:15:47.033 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.033 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.033 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:15:47.033 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.073101+0000", 2023-10-09T06:15:47.033 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3947e-05 2023-10-09T06:15:47.033 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.033 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.034 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:15:47.034 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.076623+0000", 2023-10-09T06:15:47.034 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0035219679999999999 2023-10-09T06:15:47.034 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.034 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.034 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:15:47.034 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:45.076662+0000", 2023-10-09T06:15:47.034 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9076999999999999e-05 2023-10-09T06:15:47.034 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:47.034 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:15:47.034 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:47.035 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.035 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.035 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.46283.0:149276 10.9 e550/543 10:950b7898:::benchmark_data_smithi064_573750_object53061:head v 550'9228, mlcod=550'9226)", 2023-10-09T06:15:47.035 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:14:47.177782+0000", 2023-10-09T06:15:47.035 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 59.73447204, 2023-10-09T06:15:47.035 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.53700456699999999, 2023-10-09T06:15:47.035 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:15:47.035 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:47.035 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:15:47.035 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.035 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:15:47.035 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:47.177782+0000", 2023-10-09T06:15:47.036 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:47.036 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.036 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.036 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:15:47.036 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:47.177782+0000", 2023-10-09T06:15:47.036 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:47.036 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.036 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.036 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:15:47.036 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:47.177782+0000", 2023-10-09T06:15:47.036 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:15:47.037 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.037 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.037 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:15:47.037 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:47.177790+0000", 2023-10-09T06:15:47.037 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.2209999999999997e-06 2023-10-09T06:15:47.037 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.037 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.037 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:15:47.037 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:47.691166+0000", 2023-10-09T06:15:47.037 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.51337620500000003 2023-10-09T06:15:47.037 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.038 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.038 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:15:47.038 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:47.691176+0000", 2023-10-09T06:15:47.038 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.8220000000000002e-06 2023-10-09T06:15:47.038 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.038 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.038 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:47.038 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:47.691227+0000", 2023-10-09T06:15:47.038 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1277999999999999e-05 2023-10-09T06:15:47.038 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.038 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.039 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:15:47.039 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:47.691251+0000", 2023-10-09T06:15:47.039 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4091e-05 2023-10-09T06:15:47.039 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.039 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.039 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:15:47.039 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:47.714754+0000", 2023-10-09T06:15:47.039 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.023502632999999998 2023-10-09T06:15:47.039 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.039 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.039 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:15:47.040 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:47.714787+0000", 2023-10-09T06:15:47.040 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.3179000000000002e-05 2023-10-09T06:15:47.040 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:47.040 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:15:47.040 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:47.040 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.040 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.040 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.46283.0:149281 10.6 e550/490 10:61516434:::benchmark_data_smithi064_573750_object53066:head v 550'9356, mlcod=550'9353)", 2023-10-09T06:15:47.040 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:14:47.180212+0000", 2023-10-09T06:15:47.040 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 59.732042184000001, 2023-10-09T06:15:47.040 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.520549127, 2023-10-09T06:15:47.041 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:15:47.041 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:47.041 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:15:47.041 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.041 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:15:47.041 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:47.180212+0000", 2023-10-09T06:15:47.041 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:47.041 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.041 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.041 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:15:47.041 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:47.180212+0000", 2023-10-09T06:15:47.041 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:47.042 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.042 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.042 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:15:47.042 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:47.180211+0000", 2023-10-09T06:15:47.042 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:15:47.042 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.042 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.042 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:15:47.042 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:47.180223+0000", 2023-10-09T06:15:47.042 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2201999999999999e-05 2023-10-09T06:15:47.042 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.043 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.043 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:15:47.043 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:47.691159+0000", 2023-10-09T06:15:47.043 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.51093552099999995 2023-10-09T06:15:47.043 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.043 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.043 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:15:47.043 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:47.691177+0000", 2023-10-09T06:15:47.043 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7768e-05 2023-10-09T06:15:47.043 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.043 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.043 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:47.044 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:47.691231+0000", 2023-10-09T06:15:47.044 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4224999999999997e-05 2023-10-09T06:15:47.044 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.044 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.044 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:15:47.044 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:47.691251+0000", 2023-10-09T06:15:47.044 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0085000000000002e-05 2023-10-09T06:15:47.044 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.044 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.044 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:15:47.044 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:47.700736+0000", 2023-10-09T06:15:47.044 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0094851180000000007 2023-10-09T06:15:47.045 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.045 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.045 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:15:47.045 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:47.700761+0000", 2023-10-09T06:15:47.045 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.5273e-05 2023-10-09T06:15:47.045 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:47.045 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:15:47.045 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:47.045 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.045 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.045 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.46283.0:152161 10.7 e550/521 10:e6ee6858:::benchmark_data_smithi064_573750_object55946:head v 550'9557, mlcod=550'9554)", 2023-10-09T06:15:47.045 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:14:51.185146+0000", 2023-10-09T06:15:47.046 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 55.727108631999997, 2023-10-09T06:15:47.046 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.024201397999999999, 2023-10-09T06:15:47.046 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:15:47.046 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:47.046 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:15:47.046 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.046 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:15:47.046 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:51.185146+0000", 2023-10-09T06:15:47.046 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:47.046 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.046 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.047 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:15:47.047 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:51.185146+0000", 2023-10-09T06:15:47.047 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:47.047 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.047 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.047 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:15:47.047 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:51.185145+0000", 2023-10-09T06:15:47.047 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:15:47.047 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.047 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.047 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:15:47.047 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:51.185154+0000", 2023-10-09T06:15:47.048 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.6010000000000001e-06 2023-10-09T06:15:47.048 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.048 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.048 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:15:47.048 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:51.185171+0000", 2023-10-09T06:15:47.048 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6178999999999999e-05 2023-10-09T06:15:47.048 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.048 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.048 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:15:47.048 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:51.185177+0000", 2023-10-09T06:15:47.048 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1199999999999999e-06 2023-10-09T06:15:47.048 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.049 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.049 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:47.049 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:51.185236+0000", 2023-10-09T06:15:47.049 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.965e-05 2023-10-09T06:15:47.049 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.049 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.049 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:15:47.049 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:51.185260+0000", 2023-10-09T06:15:47.049 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3850999999999999e-05 2023-10-09T06:15:47.049 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.049 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.049 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:15:47.050 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:51.209290+0000", 2023-10-09T06:15:47.050 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.024029978 2023-10-09T06:15:47.050 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.050 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.050 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:15:47.050 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:51.209347+0000", 2023-10-09T06:15:47.050 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6910000000000002e-05 2023-10-09T06:15:47.050 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:47.050 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:15:47.050 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:47.050 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.051 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.051 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.46283.0:185946 10.3 e566/564 10:c1240338:::benchmark_data_smithi064_573750_object89731:head v 566'11838, mlcod=566'11834)", 2023-10-09T06:15:47.051 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:36.061175+0000", 2023-10-09T06:15:47.051 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 10.851079722, 2023-10-09T06:15:47.051 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.025016601999999999, 2023-10-09T06:15:47.051 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:15:47.051 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:47.051 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:15:47.051 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.051 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:15:47.051 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:36.061175+0000", 2023-10-09T06:15:47.051 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:47.052 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.052 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.052 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:15:47.052 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:36.061175+0000", 2023-10-09T06:15:47.052 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:47.052 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.052 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.052 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:15:47.052 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:36.061174+0000", 2023-10-09T06:15:47.052 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:15:47.052 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.052 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.053 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:15:47.053 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:36.061185+0000", 2023-10-09T06:15:47.053 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0560000000000001e-05 2023-10-09T06:15:47.053 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.053 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.053 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:15:47.053 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:36.061200+0000", 2023-10-09T06:15:47.053 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5146e-05 2023-10-09T06:15:47.053 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.053 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.053 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:15:47.053 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:36.061206+0000", 2023-10-09T06:15:47.054 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8390000000000003e-06 2023-10-09T06:15:47.054 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.054 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.054 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:47.054 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:36.066035+0000", 2023-10-09T06:15:47.054 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0048291280000000002 2023-10-09T06:15:47.054 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.054 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.054 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:15:47.054 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:36.066058+0000", 2023-10-09T06:15:47.054 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3308000000000001e-05 2023-10-09T06:15:47.054 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.055 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.055 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:15:47.055 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:36.086153+0000", 2023-10-09T06:15:47.055 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.02009505 2023-10-09T06:15:47.055 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.055 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.055 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:15:47.055 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:36.086191+0000", 2023-10-09T06:15:47.055 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7818999999999998e-05 2023-10-09T06:15:47.055 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:47.055 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:15:47.055 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:47.056 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.056 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.056 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.46283.0:185956 10.9 e566/543 10:9e489629:::benchmark_data_smithi064_573750_object89741:head v 566'11509, mlcod=566'11507)", 2023-10-09T06:15:47.056 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:36.062689+0000", 2023-10-09T06:15:47.056 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 10.84956526, 2023-10-09T06:15:47.056 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.026172503999999999, 2023-10-09T06:15:47.056 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:15:47.056 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:15:47.056 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:15:47.056 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.057 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:15:47.057 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:36.062689+0000", 2023-10-09T06:15:47.057 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:47.057 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.057 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.057 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:15:47.057 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:36.062689+0000", 2023-10-09T06:15:47.057 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:15:47.057 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.057 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.057 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:15:47.057 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:36.062688+0000", 2023-10-09T06:15:47.058 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:15:47.058 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.058 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.058 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:15:47.058 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:36.062697+0000", 2023-10-09T06:15:47.058 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.4789999999999996e-06 2023-10-09T06:15:47.058 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.058 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.058 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:15:47.058 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:36.062713+0000", 2023-10-09T06:15:47.058 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6059e-05 2023-10-09T06:15:47.059 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.059 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.059 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:15:47.059 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:36.062719+0000", 2023-10-09T06:15:47.059 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1009999999999997e-06 2023-10-09T06:15:47.059 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.059 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.059 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:15:47.059 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:36.063287+0000", 2023-10-09T06:15:47.059 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00056819100000000001 2023-10-09T06:15:47.059 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.059 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.059 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:15:47.060 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:36.063310+0000", 2023-10-09T06:15:47.060 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3164999999999999e-05 2023-10-09T06:15:47.060 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.060 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.060 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:15:47.060 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:36.088831+0000", 2023-10-09T06:15:47.060 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.025520622 2023-10-09T06:15:47.060 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:15:47.060 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:15:47.060 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:15:47.060 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:36.088862+0000", 2023-10-09T06:15:47.060 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0757999999999999e-05 2023-10-09T06:15:47.061 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:47.061 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:15:47.061 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:47.061 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:15:47.061 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:15:47.061 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:15:47.435 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1443,"stamp":"2023-10-09T06:15:45.233244+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":94705592,"num_objects":1461,"num_object_clones":0,"num_object_copies":4383,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1461,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":191050,"num_write_kb":6157598,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44136,"num_bytes_recovered":2889627780,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100684,"ondisk_log_size":100684,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":368,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7125628,"kb_used_data":1103932,"kb_used_omap":165,"kb_used_meta":6021402,"kb_avail":742835588,"statfs":{"total":767960285184,"available":760663642112,"internally_reserved":0,"allocated":1130426368,"data_stored":1111845145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":169911,"internal_metadata":6165915721},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-577896448,"num_objects":-8818,"num_object_clones":0,"num_object_copies":-26454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8818,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002738"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":830,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.795824+0000","last_change":"2023-10-09T06:15:19.880315+0000","last_active":"2023-10-09T06:15:32.795824+0000","last_peered":"2023-10-09T06:15:32.795824+0000","last_clean":"2023-10-09T06:15:32.795824+0000","last_became_active":"2023-10-09T06:15:19.879857+0000","last_became_peered":"2023-10-09T06:15:19.879857+0000","last_unstale":"2023-10-09T06:15:32.795824+0000","last_undegraded":"2023-10-09T06:15:32.795824+0000","last_fullsized":"2023-10-09T06:15:32.795824+0000","mapping_epoch":561,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":562,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:05.412941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:05.412941+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:54.322404+0000","scrub_duration":0.0061014329999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":634,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912388+0000","last_change":"2023-10-09T06:07:43.572773+0000","last_active":"2023-10-09T06:15:23.912388+0000","last_peered":"2023-10-09T06:15:23.912388+0000","last_clean":"2023-10-09T06:15:23.912388+0000","last_became_active":"2023-10-09T06:07:43.572299+0000","last_became_peered":"2023-10-09T06:07:43.572299+0000","last_unstale":"2023-10-09T06:15:23.912388+0000","last_undegraded":"2023-10-09T06:15:23.912388+0000","last_fullsized":"2023-10-09T06:15:23.912388+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:00:37.672435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":377,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.913601+0000","last_change":"2023-10-09T06:05:11.451972+0000","last_active":"2023-10-09T06:15:23.913601+0000","last_peered":"2023-10-09T06:15:23.913601+0000","last_clean":"2023-10-09T06:15:23.913601+0000","last_became_active":"2023-10-09T06:05:11.451448+0000","last_became_peered":"2023-10-09T06:05:11.451448+0000","last_unstale":"2023-10-09T06:15:23.913601+0000","last_undegraded":"2023-10-09T06:15:23.913601+0000","last_fullsized":"2023-10-09T06:15:23.913601+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:39.183621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":865,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912615+0000","last_change":"2023-10-09T06:09:27.554162+0000","last_active":"2023-10-09T06:15:23.912615+0000","last_peered":"2023-10-09T06:15:23.912615+0000","last_clean":"2023-10-09T06:15:23.912615+0000","last_became_active":"2023-10-09T06:09:27.553659+0000","last_became_peered":"2023-10-09T06:09:27.553659+0000","last_unstale":"2023-10-09T06:15:23.912615+0000","last_undegraded":"2023-10-09T06:15:23.912615+0000","last_fullsized":"2023-10-09T06:15:23.912615+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:01.468346+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:01.468346+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:03.443511+0000","scrub_duration":0.0016258220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":248,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.910302+0000","last_change":"2023-10-09T06:01:54.762060+0000","last_active":"2023-10-09T06:15:23.910302+0000","last_peered":"2023-10-09T06:15:23.910302+0000","last_clean":"2023-10-09T06:15:23.910302+0000","last_became_active":"2023-10-09T06:01:54.761757+0000","last_became_peered":"2023-10-09T06:01:54.761757+0000","last_unstale":"2023-10-09T06:15:23.910302+0000","last_undegraded":"2023-10-09T06:15:23.910302+0000","last_fullsized":"2023-10-09T06:15:23.910302+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:40.294232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":50,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.797681+0000","last_change":"2023-10-09T06:13:30.789834+0000","last_active":"2023-10-09T06:15:32.797681+0000","last_peered":"2023-10-09T06:15:32.797681+0000","last_clean":"2023-10-09T06:15:32.797681+0000","last_became_active":"2023-10-09T06:13:30.789517+0000","last_became_peered":"2023-10-09T06:13:30.789517+0000","last_unstale":"2023-10-09T06:15:32.797681+0000","last_undegraded":"2023-10-09T06:15:32.797681+0000","last_fullsized":"2023-10-09T06:15:32.797681+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:45.467922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":850,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912191+0000","last_change":"2023-10-09T06:12:07.739121+0000","last_active":"2023-10-09T06:15:23.912191+0000","last_peered":"2023-10-09T06:15:23.912191+0000","last_clean":"2023-10-09T06:15:23.912191+0000","last_became_active":"2023-10-09T06:12:07.738739+0000","last_became_peered":"2023-10-09T06:12:07.738739+0000","last_unstale":"2023-10-09T06:15:23.912191+0000","last_undegraded":"2023-10-09T06:15:23.912191+0000","last_fullsized":"2023-10-09T06:15:23.912191+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.293426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.293426+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:19.815422+0000","scrub_duration":0.001587946,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":359,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912573+0000","last_change":"2023-10-09T06:09:27.564377+0000","last_active":"2023-10-09T06:15:23.912573+0000","last_peered":"2023-10-09T06:15:23.912573+0000","last_clean":"2023-10-09T06:15:23.912573+0000","last_became_active":"2023-10-09T06:09:27.563710+0000","last_became_peered":"2023-10-09T06:09:27.563710+0000","last_unstale":"2023-10-09T06:15:23.912573+0000","last_undegraded":"2023-10-09T06:15:23.912573+0000","last_fullsized":"2023-10-09T06:15:23.912573+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:48.168447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":456,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.799589+0000","last_change":"2023-10-09T06:15:01.885861+0000","last_active":"2023-10-09T06:15:32.799589+0000","last_peered":"2023-10-09T06:15:32.799589+0000","last_clean":"2023-10-09T06:15:32.799589+0000","last_became_active":"2023-10-09T06:15:01.885560+0000","last_became_peered":"2023-10-09T06:15:01.885560+0000","last_unstale":"2023-10-09T06:15:32.799589+0000","last_undegraded":"2023-10-09T06:15:32.799589+0000","last_fullsized":"2023-10-09T06:15:32.799589+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:44.842197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":785,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.797518+0000","last_change":"2023-10-09T06:13:33.701687+0000","last_active":"2023-10-09T06:15:32.797518+0000","last_peered":"2023-10-09T06:15:32.797518+0000","last_clean":"2023-10-09T06:15:32.797518+0000","last_became_active":"2023-10-09T06:13:33.701233+0000","last_became_peered":"2023-10-09T06:13:33.701233+0000","last_unstale":"2023-10-09T06:15:32.797518+0000","last_undegraded":"2023-10-09T06:15:32.797518+0000","last_fullsized":"2023-10-09T06:15:32.797518+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.384407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.384407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:04.887826+0000","scrub_duration":0.0040651960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":464,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.913821+0000","last_change":"2023-10-09T06:09:27.559756+0000","last_active":"2023-10-09T06:15:23.913821+0000","last_peered":"2023-10-09T06:15:23.913821+0000","last_clean":"2023-10-09T06:15:23.913821+0000","last_became_active":"2023-10-09T06:09:27.559443+0000","last_became_peered":"2023-10-09T06:09:27.559443+0000","last_unstale":"2023-10-09T06:15:23.913821+0000","last_undegraded":"2023-10-09T06:15:23.913821+0000","last_fullsized":"2023-10-09T06:15:23.913821+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:14.519398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":395,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912572+0000","last_change":"2023-10-09T06:13:21.737897+0000","last_active":"2023-10-09T06:15:23.912572+0000","last_peered":"2023-10-09T06:15:23.912572+0000","last_clean":"2023-10-09T06:15:23.912572+0000","last_became_active":"2023-10-09T06:13:21.737564+0000","last_became_peered":"2023-10-09T06:13:21.737564+0000","last_unstale":"2023-10-09T06:15:23.912572+0000","last_undegraded":"2023-10-09T06:15:23.912572+0000","last_fullsized":"2023-10-09T06:15:23.912572+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:34.863320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":691,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.798173+0000","last_change":"2023-10-09T06:10:11.729656+0000","last_active":"2023-10-09T06:15:32.798173+0000","last_peered":"2023-10-09T06:15:32.798173+0000","last_clean":"2023-10-09T06:15:32.798173+0000","last_became_active":"2023-10-09T06:10:11.728960+0000","last_became_peered":"2023-10-09T06:10:11.728960+0000","last_unstale":"2023-10-09T06:15:32.798173+0000","last_undegraded":"2023-10-09T06:15:32.798173+0000","last_fullsized":"2023-10-09T06:15:32.798173+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:20.742561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:20.742561+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:45.543709+0000","scrub_duration":0.0027005039999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":50,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.797602+0000","last_change":"2023-10-09T06:13:30.802749+0000","last_active":"2023-10-09T06:15:32.797602+0000","last_peered":"2023-10-09T06:15:32.797602+0000","last_clean":"2023-10-09T06:15:32.797602+0000","last_became_active":"2023-10-09T06:13:30.802449+0000","last_became_peered":"2023-10-09T06:13:30.802449+0000","last_unstale":"2023-10-09T06:15:32.797602+0000","last_undegraded":"2023-10-09T06:15:32.797602+0000","last_fullsized":"2023-10-09T06:15:32.797602+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:29.017181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":553,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912847+0000","last_change":"2023-10-09T06:13:30.792445+0000","last_active":"2023-10-09T06:15:23.912847+0000","last_peered":"2023-10-09T06:15:23.912847+0000","last_clean":"2023-10-09T06:15:23.912847+0000","last_became_active":"2023-10-09T06:13:30.792116+0000","last_became_peered":"2023-10-09T06:13:30.792116+0000","last_unstale":"2023-10-09T06:15:23.912847+0000","last_undegraded":"2023-10-09T06:15:23.912847+0000","last_fullsized":"2023-10-09T06:15:23.912847+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:20:53.506293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":778,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.800728+0000","last_change":"2023-10-09T06:13:24.760843+0000","last_active":"2023-10-09T06:15:32.800728+0000","last_peered":"2023-10-09T06:15:32.800728+0000","last_clean":"2023-10-09T06:15:32.800728+0000","last_became_active":"2023-10-09T06:13:24.760333+0000","last_became_peered":"2023-10-09T06:13:24.760333+0000","last_unstale":"2023-10-09T06:15:32.800728+0000","last_undegraded":"2023-10-09T06:15:32.800728+0000","last_fullsized":"2023-10-09T06:15:32.800728+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:38.588641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:38.588641+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:30.660400+0000","scrub_duration":0.010116385,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":56,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.797445+0000","last_change":"2023-10-09T06:13:30.826588+0000","last_active":"2023-10-09T06:15:32.797445+0000","last_peered":"2023-10-09T06:15:32.797445+0000","last_clean":"2023-10-09T06:15:32.797445+0000","last_became_active":"2023-10-09T06:13:30.799199+0000","last_became_peered":"2023-10-09T06:13:30.799199+0000","last_unstale":"2023-10-09T06:15:32.797445+0000","last_undegraded":"2023-10-09T06:15:32.797445+0000","last_fullsized":"2023-10-09T06:15:32.797445+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:38.703864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":348,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.800036+0000","last_change":"2023-10-09T06:13:21.739214+0000","last_active":"2023-10-09T06:15:32.800036+0000","last_peered":"2023-10-09T06:15:32.800036+0000","last_clean":"2023-10-09T06:15:32.800036+0000","last_became_active":"2023-10-09T06:13:21.738951+0000","last_became_peered":"2023-10-09T06:13:21.738951+0000","last_unstale":"2023-10-09T06:15:32.800036+0000","last_undegraded":"2023-10-09T06:15:32.800036+0000","last_fullsized":"2023-10-09T06:15:32.800036+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:17.477241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":738,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.911668+0000","last_change":"2023-10-09T06:09:27.574336+0000","last_active":"2023-10-09T06:15:23.911668+0000","last_peered":"2023-10-09T06:15:23.911668+0000","last_clean":"2023-10-09T06:15:23.911668+0000","last_became_active":"2023-10-09T06:09:27.573974+0000","last_became_peered":"2023-10-09T06:09:27.573974+0000","last_unstale":"2023-10-09T06:15:23.911668+0000","last_undegraded":"2023-10-09T06:15:23.911668+0000","last_fullsized":"2023-10-09T06:15:23.911668+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.339121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.339121+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:45.908579+0000","scrub_duration":0.010474344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":600,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912503+0000","last_change":"2023-10-09T06:13:30.812598+0000","last_active":"2023-10-09T06:15:23.912503+0000","last_peered":"2023-10-09T06:15:23.912503+0000","last_clean":"2023-10-09T06:15:23.912503+0000","last_became_active":"2023-10-09T06:13:30.791460+0000","last_became_peered":"2023-10-09T06:13:30.791460+0000","last_unstale":"2023-10-09T06:15:23.912503+0000","last_undegraded":"2023-10-09T06:15:23.912503+0000","last_fullsized":"2023-10-09T06:15:23.912503+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:28.046730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":578,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912929+0000","last_change":"2023-10-09T06:01:54.764916+0000","last_active":"2023-10-09T06:15:23.912929+0000","last_peered":"2023-10-09T06:15:23.912929+0000","last_clean":"2023-10-09T06:15:23.912929+0000","last_became_active":"2023-10-09T06:01:54.764705+0000","last_became_peered":"2023-10-09T06:01:54.764705+0000","last_unstale":"2023-10-09T06:15:23.912929+0000","last_undegraded":"2023-10-09T06:15:23.912929+0000","last_fullsized":"2023-10-09T06:15:23.912929+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:12.527719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":794,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912559+0000","last_change":"2023-10-09T06:13:15.425525+0000","last_active":"2023-10-09T06:15:23.912559+0000","last_peered":"2023-10-09T06:15:23.912559+0000","last_clean":"2023-10-09T06:15:23.912559+0000","last_became_active":"2023-10-09T06:13:15.425086+0000","last_became_peered":"2023-10-09T06:13:15.425086+0000","last_unstale":"2023-10-09T06:15:23.912559+0000","last_undegraded":"2023-10-09T06:15:23.912559+0000","last_fullsized":"2023-10-09T06:15:23.912559+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:42.900876+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:42.900876+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:46.042715+0000","scrub_duration":0.0014928649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":56,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.796080+0000","last_change":"2023-10-09T06:13:30.811420+0000","last_active":"2023-10-09T06:15:32.796080+0000","last_peered":"2023-10-09T06:15:32.796080+0000","last_clean":"2023-10-09T06:15:32.796080+0000","last_became_active":"2023-10-09T06:13:30.792835+0000","last_became_peered":"2023-10-09T06:13:30.792835+0000","last_unstale":"2023-10-09T06:15:32.796080+0000","last_undegraded":"2023-10-09T06:15:32.796080+0000","last_fullsized":"2023-10-09T06:15:32.796080+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:38.962645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":612,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.913531+0000","last_change":"2023-10-09T06:01:54.761558+0000","last_active":"2023-10-09T06:15:23.913531+0000","last_peered":"2023-10-09T06:15:23.913531+0000","last_clean":"2023-10-09T06:15:23.913531+0000","last_became_active":"2023-10-09T06:01:54.761251+0000","last_became_peered":"2023-10-09T06:01:54.761251+0000","last_unstale":"2023-10-09T06:15:23.913531+0000","last_undegraded":"2023-10-09T06:15:23.913531+0000","last_fullsized":"2023-10-09T06:15:23.913531+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:53.245267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":1070,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.796056+0000","last_change":"2023-10-09T06:13:24.754018+0000","last_active":"2023-10-09T06:15:32.796056+0000","last_peered":"2023-10-09T06:15:32.796056+0000","last_clean":"2023-10-09T06:15:32.796056+0000","last_became_active":"2023-10-09T06:13:24.753712+0000","last_became_peered":"2023-10-09T06:13:24.753712+0000","last_unstale":"2023-10-09T06:15:32.796056+0000","last_undegraded":"2023-10-09T06:15:32.796056+0000","last_fullsized":"2023-10-09T06:15:32.796056+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:18.490833+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":50,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.795719+0000","last_change":"2023-10-09T06:13:30.788160+0000","last_active":"2023-10-09T06:15:32.795719+0000","last_peered":"2023-10-09T06:15:32.795719+0000","last_clean":"2023-10-09T06:15:32.795719+0000","last_became_active":"2023-10-09T06:13:30.787724+0000","last_became_peered":"2023-10-09T06:13:30.787724+0000","last_unstale":"2023-10-09T06:15:32.795719+0000","last_undegraded":"2023-10-09T06:15:32.795719+0000","last_fullsized":"2023-10-09T06:15:32.795719+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:38.376770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":50,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.797608+0000","last_change":"2023-10-09T06:13:30.789225+0000","last_active":"2023-10-09T06:15:32.797608+0000","last_peered":"2023-10-09T06:15:32.797608+0000","last_clean":"2023-10-09T06:15:32.797608+0000","last_became_active":"2023-10-09T06:13:30.788869+0000","last_became_peered":"2023-10-09T06:13:30.788869+0000","last_unstale":"2023-10-09T06:15:32.797608+0000","last_undegraded":"2023-10-09T06:15:32.797608+0000","last_fullsized":"2023-10-09T06:15:32.797608+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:19.519966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":806,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.911877+0000","last_change":"2023-10-09T06:12:55.750923+0000","last_active":"2023-10-09T06:15:23.911877+0000","last_peered":"2023-10-09T06:15:23.911877+0000","last_clean":"2023-10-09T06:15:23.911877+0000","last_became_active":"2023-10-09T06:12:55.750521+0000","last_became_peered":"2023-10-09T06:12:55.750521+0000","last_unstale":"2023-10-09T06:15:23.911877+0000","last_undegraded":"2023-10-09T06:15:23.911877+0000","last_fullsized":"2023-10-09T06:15:23.911877+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.316697+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.316697+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:23.591439+0000","scrub_duration":0.001666561,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":420,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.796453+0000","last_change":"2023-10-09T06:01:22.337171+0000","last_active":"2023-10-09T06:15:32.796453+0000","last_peered":"2023-10-09T06:15:32.796453+0000","last_clean":"2023-10-09T06:15:32.796453+0000","last_became_active":"2023-10-09T06:01:22.335438+0000","last_became_peered":"2023-10-09T06:01:22.335438+0000","last_unstale":"2023-10-09T06:15:32.796453+0000","last_undegraded":"2023-10-09T06:15:32.796453+0000","last_fullsized":"2023-10-09T06:15:32.796453+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:25.036448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":528,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.913650+0000","last_change":"2023-10-09T06:13:30.788284+0000","last_active":"2023-10-09T06:15:23.913650+0000","last_peered":"2023-10-09T06:15:23.913650+0000","last_clean":"2023-10-09T06:15:23.913650+0000","last_became_active":"2023-10-09T06:13:30.787933+0000","last_became_peered":"2023-10-09T06:13:30.787933+0000","last_unstale":"2023-10-09T06:15:23.913650+0000","last_undegraded":"2023-10-09T06:15:23.913650+0000","last_fullsized":"2023-10-09T06:15:23.913650+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:10.735438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":856,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.910664+0000","last_change":"2023-10-09T06:13:24.753908+0000","last_active":"2023-10-09T06:15:23.910664+0000","last_peered":"2023-10-09T06:15:23.910664+0000","last_clean":"2023-10-09T06:15:23.910664+0000","last_became_active":"2023-10-09T06:13:24.753566+0000","last_became_peered":"2023-10-09T06:13:24.753566+0000","last_unstale":"2023-10-09T06:15:23.910664+0000","last_undegraded":"2023-10-09T06:15:23.910664+0000","last_fullsized":"2023-10-09T06:15:23.910664+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:40.388125+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:40.388125+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:35.101978+0000","scrub_duration":0.0055576990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":139,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:31.457596+0000","last_change":"2023-10-09T06:13:12.243605+0000","last_active":"2023-10-09T06:15:31.457596+0000","last_peered":"2023-10-09T06:15:31.457596+0000","last_clean":"2023-10-09T06:15:31.457596+0000","last_became_active":"2023-10-09T06:13:12.237061+0000","last_became_peered":"2023-10-09T06:13:12.237061+0000","last_unstale":"2023-10-09T06:15:31.457596+0000","last_undegraded":"2023-10-09T06:15:31.457596+0000","last_fullsized":"2023-10-09T06:15:31.457596+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:29:03.666031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":77,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.913710+0000","last_change":"2023-10-09T06:13:30.788733+0000","last_active":"2023-10-09T06:15:23.913710+0000","last_peered":"2023-10-09T06:15:23.913710+0000","last_clean":"2023-10-09T06:15:23.913710+0000","last_became_active":"2023-10-09T06:13:30.788353+0000","last_became_peered":"2023-10-09T06:13:30.788353+0000","last_unstale":"2023-10-09T06:15:23.913710+0000","last_undegraded":"2023-10-09T06:15:23.913710+0000","last_fullsized":"2023-10-09T06:15:23.913710+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:06.308426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"566'11958","reported_seq":12044,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:41.388009+0000","last_change":"2023-10-09T06:11:46.032384+0000","last_active":"2023-10-09T06:15:41.388009+0000","last_peered":"2023-10-09T06:15:41.388009+0000","last_clean":"2023-10-09T06:15:41.388009+0000","last_became_active":"2023-10-09T06:11:46.032091+0000","last_became_peered":"2023-10-09T06:11:46.032091+0000","last_unstale":"2023-10-09T06:15:41.388009+0000","last_undegraded":"2023-10-09T06:15:41.388009+0000","last_fullsized":"2023-10-09T06:15:41.388009+0000","mapping_epoch":490,"log_start":"520'5823","ondisk_log_start":"520'5823","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6135,"log_dups_size":0,"ondisk_log_size":6135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:31.117384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6160384,"num_objects":94,"num_object_clones":0,"num_object_copies":282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":94,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11958,"num_write_kb":385664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":748,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.797669+0000","last_change":"2023-10-09T06:13:33.701521+0000","last_active":"2023-10-09T06:15:32.797669+0000","last_peered":"2023-10-09T06:15:32.797669+0000","last_clean":"2023-10-09T06:15:32.797669+0000","last_became_active":"2023-10-09T06:13:33.700870+0000","last_became_peered":"2023-10-09T06:13:33.700870+0000","last_unstale":"2023-10-09T06:15:32.797669+0000","last_undegraded":"2023-10-09T06:15:32.797669+0000","last_fullsized":"2023-10-09T06:15:32.797669+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:10:59.422551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:10:59.422551+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:54.402494+0000","scrub_duration":0.0059716659999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":54,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.797688+0000","last_change":"2023-10-09T06:13:30.822903+0000","last_active":"2023-10-09T06:15:32.797688+0000","last_peered":"2023-10-09T06:15:32.797688+0000","last_clean":"2023-10-09T06:15:32.797688+0000","last_became_active":"2023-10-09T06:13:30.799013+0000","last_became_peered":"2023-10-09T06:13:30.799013+0000","last_unstale":"2023-10-09T06:15:32.797688+0000","last_undegraded":"2023-10-09T06:15:32.797688+0000","last_fullsized":"2023-10-09T06:15:32.797688+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:44.271821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":50,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.795751+0000","last_change":"2023-10-09T06:13:30.793531+0000","last_active":"2023-10-09T06:15:32.795751+0000","last_peered":"2023-10-09T06:15:32.795751+0000","last_clean":"2023-10-09T06:15:32.795751+0000","last_became_active":"2023-10-09T06:13:30.792689+0000","last_became_peered":"2023-10-09T06:13:30.792689+0000","last_unstale":"2023-10-09T06:15:32.795751+0000","last_undegraded":"2023-10-09T06:15:32.795751+0000","last_fullsized":"2023-10-09T06:15:32.795751+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:02.559186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"566'11924","reported_seq":20857,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:40.553603+0000","last_change":"2023-10-09T06:15:03.759950+0000","last_active":"2023-10-09T06:15:40.553603+0000","last_peered":"2023-10-09T06:15:40.553603+0000","last_clean":"2023-10-09T06:15:40.553603+0000","last_became_active":"2023-10-09T06:15:03.735000+0000","last_became_peered":"2023-10-09T06:15:03.735000+0000","last_unstale":"2023-10-09T06:15:40.553603+0000","last_undegraded":"2023-10-09T06:15:40.553603+0000","last_fullsized":"2023-10-09T06:15:40.553603+0000","mapping_epoch":554,"log_start":"523'5993","ondisk_log_start":"523'5993","created":490,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5931,"log_dups_size":0,"ondisk_log_size":5931,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:01.420748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10354688,"num_objects":158,"num_object_clones":0,"num_object_copies":474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":158,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11924,"num_write_kb":386624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3329,"num_bytes_recovered":218169344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":936,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.797782+0000","last_change":"2023-10-09T06:13:24.753866+0000","last_active":"2023-10-09T06:15:32.797782+0000","last_peered":"2023-10-09T06:15:32.797782+0000","last_clean":"2023-10-09T06:15:32.797782+0000","last_became_active":"2023-10-09T06:13:24.753432+0000","last_became_peered":"2023-10-09T06:13:24.753432+0000","last_unstale":"2023-10-09T06:15:32.797782+0000","last_undegraded":"2023-10-09T06:15:32.797782+0000","last_fullsized":"2023-10-09T06:15:32.797782+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:03.241335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:03.241335+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:19.854449+0000","scrub_duration":0.017780547000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":54,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.797522+0000","last_change":"2023-10-09T06:13:30.829763+0000","last_active":"2023-10-09T06:15:32.797522+0000","last_peered":"2023-10-09T06:15:32.797522+0000","last_clean":"2023-10-09T06:15:32.797522+0000","last_became_active":"2023-10-09T06:13:30.802486+0000","last_became_peered":"2023-10-09T06:13:30.802486+0000","last_unstale":"2023-10-09T06:15:32.797522+0000","last_undegraded":"2023-10-09T06:15:32.797522+0000","last_fullsized":"2023-10-09T06:15:32.797522+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:05.468205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":604,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912224+0000","last_change":"2023-10-09T06:01:54.760367+0000","last_active":"2023-10-09T06:15:23.912224+0000","last_peered":"2023-10-09T06:15:23.912224+0000","last_clean":"2023-10-09T06:15:23.912224+0000","last_became_active":"2023-10-09T06:01:54.760087+0000","last_became_peered":"2023-10-09T06:01:54.760087+0000","last_unstale":"2023-10-09T06:15:23.912224+0000","last_undegraded":"2023-10-09T06:15:23.912224+0000","last_fullsized":"2023-10-09T06:15:23.912224+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:26.000104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"566'11794","reported_seq":11876,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:40.986688+0000","last_change":"2023-10-09T06:11:45.696810+0000","last_active":"2023-10-09T06:15:40.986688+0000","last_peered":"2023-10-09T06:15:40.986688+0000","last_clean":"2023-10-09T06:15:40.986688+0000","last_became_active":"2023-10-09T06:11:45.696476+0000","last_became_peered":"2023-10-09T06:11:45.696476+0000","last_unstale":"2023-10-09T06:15:40.986688+0000","last_undegraded":"2023-10-09T06:15:40.986688+0000","last_fullsized":"2023-10-09T06:15:40.986688+0000","mapping_epoch":490,"log_start":"530'6400","ondisk_log_start":"530'6400","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5394,"log_dups_size":0,"ondisk_log_size":5394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:41:33.351567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9043968,"num_objects":138,"num_object_clones":0,"num_object_copies":414,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":138,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11794,"num_write_kb":381824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":829,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.913143+0000","last_change":"2023-10-09T06:13:24.768958+0000","last_active":"2023-10-09T06:15:23.913143+0000","last_peered":"2023-10-09T06:15:23.913143+0000","last_clean":"2023-10-09T06:15:23.913143+0000","last_became_active":"2023-10-09T06:13:24.768643+0000","last_became_peered":"2023-10-09T06:13:24.768643+0000","last_unstale":"2023-10-09T06:15:23.913143+0000","last_undegraded":"2023-10-09T06:15:23.913143+0000","last_fullsized":"2023-10-09T06:15:23.913143+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:48.495739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:48.495739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:18:50.654354+0000","scrub_duration":0.0016345509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":54,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.797527+0000","last_change":"2023-10-09T06:13:30.814434+0000","last_active":"2023-10-09T06:15:32.797527+0000","last_peered":"2023-10-09T06:15:32.797527+0000","last_clean":"2023-10-09T06:15:32.797527+0000","last_became_active":"2023-10-09T06:13:30.793119+0000","last_became_peered":"2023-10-09T06:13:30.793119+0000","last_unstale":"2023-10-09T06:15:32.797527+0000","last_undegraded":"2023-10-09T06:15:32.797527+0000","last_fullsized":"2023-10-09T06:15:32.797527+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:34:31.724006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":591,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.910484+0000","last_change":"2023-10-09T06:13:30.792490+0000","last_active":"2023-10-09T06:15:23.910484+0000","last_peered":"2023-10-09T06:15:23.910484+0000","last_clean":"2023-10-09T06:15:23.910484+0000","last_became_active":"2023-10-09T06:13:30.792209+0000","last_became_peered":"2023-10-09T06:13:30.792209+0000","last_unstale":"2023-10-09T06:15:23.910484+0000","last_undegraded":"2023-10-09T06:15:23.910484+0000","last_fullsized":"2023-10-09T06:15:23.910484+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:14.208042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"566'11823","reported_seq":11909,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:41.419287+0000","last_change":"2023-10-09T06:11:46.031195+0000","last_active":"2023-10-09T06:15:41.419287+0000","last_peered":"2023-10-09T06:15:41.419287+0000","last_clean":"2023-10-09T06:15:41.419287+0000","last_became_active":"2023-10-09T06:11:46.030891+0000","last_became_peered":"2023-10-09T06:11:46.030891+0000","last_unstale":"2023-10-09T06:15:41.419287+0000","last_undegraded":"2023-10-09T06:15:41.419287+0000","last_fullsized":"2023-10-09T06:15:41.419287+0000","mapping_epoch":490,"log_start":"519'5636","ondisk_log_start":"519'5636","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6187,"log_dups_size":0,"ondisk_log_size":6187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:25.788661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6094848,"num_objects":93,"num_object_clones":0,"num_object_copies":279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":93,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11823,"num_write_kb":381312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":781,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.913054+0000","last_change":"2023-10-09T06:13:33.708291+0000","last_active":"2023-10-09T06:15:23.913054+0000","last_peered":"2023-10-09T06:15:23.913054+0000","last_clean":"2023-10-09T06:15:23.913054+0000","last_became_active":"2023-10-09T06:13:33.707930+0000","last_became_peered":"2023-10-09T06:13:33.707930+0000","last_unstale":"2023-10-09T06:15:23.913054+0000","last_undegraded":"2023-10-09T06:15:23.913054+0000","last_fullsized":"2023-10-09T06:15:23.913054+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:16.361097+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:16.361097+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:18.111404+0000","scrub_duration":0.028533749000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":336,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:25.565736+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T06:15:25.565736+0000","last_peered":"2023-10-09T06:15:25.565736+0000","last_clean":"2023-10-09T06:15:25.565736+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T06:15:25.565736+0000","last_undegraded":"2023-10-09T06:15:25.565736+0000","last_fullsized":"2023-10-09T06:15:25.565736+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":579,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.915860+0000","last_change":"2023-10-09T06:09:27.558353+0000","last_active":"2023-10-09T06:15:23.915860+0000","last_peered":"2023-10-09T06:15:23.915860+0000","last_clean":"2023-10-09T06:15:23.915860+0000","last_became_active":"2023-10-09T06:09:27.557233+0000","last_became_peered":"2023-10-09T06:09:27.557233+0000","last_unstale":"2023-10-09T06:15:23.915860+0000","last_undegraded":"2023-10-09T06:15:23.915860+0000","last_fullsized":"2023-10-09T06:15:23.915860+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:53.149350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"566'11831","reported_seq":11917,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:41.644009+0000","last_change":"2023-10-09T06:11:45.694648+0000","last_active":"2023-10-09T06:15:41.644009+0000","last_peered":"2023-10-09T06:15:41.644009+0000","last_clean":"2023-10-09T06:15:41.644009+0000","last_became_active":"2023-10-09T06:11:45.694321+0000","last_became_peered":"2023-10-09T06:11:45.694321+0000","last_unstale":"2023-10-09T06:15:41.644009+0000","last_undegraded":"2023-10-09T06:15:41.644009+0000","last_fullsized":"2023-10-09T06:15:41.644009+0000","mapping_epoch":490,"log_start":"515'5532","ondisk_log_start":"515'5532","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6299,"log_dups_size":0,"ondisk_log_size":6299,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:43.704525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5963776,"num_objects":91,"num_object_clones":0,"num_object_copies":273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":91,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11831,"num_write_kb":381504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":973,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.796319+0000","last_change":"2023-10-09T06:15:19.874375+0000","last_active":"2023-10-09T06:15:32.796319+0000","last_peered":"2023-10-09T06:15:32.796319+0000","last_clean":"2023-10-09T06:15:32.796319+0000","last_became_active":"2023-10-09T06:15:19.873993+0000","last_became_peered":"2023-10-09T06:15:19.873993+0000","last_unstale":"2023-10-09T06:15:32.796319+0000","last_undegraded":"2023-10-09T06:15:32.796319+0000","last_fullsized":"2023-10-09T06:15:32.796319+0000","mapping_epoch":561,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":562,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:42.795781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:42.795781+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:43.523891+0000","scrub_duration":0.0091262979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":370,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:25.565673+0000","last_change":"2023-10-09T06:09:27.565666+0000","last_active":"2023-10-09T06:15:25.565673+0000","last_peered":"2023-10-09T06:15:25.565673+0000","last_clean":"2023-10-09T06:15:25.565673+0000","last_became_active":"2023-10-09T06:09:27.565324+0000","last_became_peered":"2023-10-09T06:09:27.565324+0000","last_unstale":"2023-10-09T06:15:25.565673+0000","last_undegraded":"2023-10-09T06:15:25.565673+0000","last_fullsized":"2023-10-09T06:15:25.565673+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:51.216947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":540,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912578+0000","last_change":"2023-10-09T06:13:21.727745+0000","last_active":"2023-10-09T06:15:23.912578+0000","last_peered":"2023-10-09T06:15:23.912578+0000","last_clean":"2023-10-09T06:15:23.912578+0000","last_became_active":"2023-10-09T06:13:21.727404+0000","last_became_peered":"2023-10-09T06:13:21.727404+0000","last_unstale":"2023-10-09T06:15:23.912578+0000","last_undegraded":"2023-10-09T06:15:23.912578+0000","last_fullsized":"2023-10-09T06:15:23.912578+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:24.931626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"566'12125","reported_seq":24978,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:40.556469+0000","last_change":"2023-10-09T06:15:23.952835+0000","last_active":"2023-10-09T06:15:40.556469+0000","last_peered":"2023-10-09T06:15:40.556469+0000","last_clean":"2023-10-09T06:15:40.556469+0000","last_became_active":"2023-10-09T06:15:23.914520+0000","last_became_peered":"2023-10-09T06:15:23.914520+0000","last_unstale":"2023-10-09T06:15:40.556469+0000","last_undegraded":"2023-10-09T06:15:40.556469+0000","last_fullsized":"2023-10-09T06:15:40.556469+0000","mapping_epoch":564,"log_start":"525'6235","ondisk_log_start":"525'6235","created":490,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5890,"log_dups_size":0,"ondisk_log_size":5890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:55.053464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9764864,"num_objects":149,"num_object_clones":0,"num_object_copies":447,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":149,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12125,"num_write_kb":392768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6357,"num_bytes_recovered":416481280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":829,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912451+0000","last_change":"2023-10-09T06:13:15.442141+0000","last_active":"2023-10-09T06:15:23.912451+0000","last_peered":"2023-10-09T06:15:23.912451+0000","last_clean":"2023-10-09T06:15:23.912451+0000","last_became_active":"2023-10-09T06:13:15.441716+0000","last_became_peered":"2023-10-09T06:13:15.441716+0000","last_unstale":"2023-10-09T06:15:23.912451+0000","last_undegraded":"2023-10-09T06:15:23.912451+0000","last_fullsized":"2023-10-09T06:15:23.912451+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.282393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.282393+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:16.798440+0000","scrub_duration":0.01402403,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":485,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912284+0000","last_change":"2023-10-09T06:13:21.738218+0000","last_active":"2023-10-09T06:15:23.912284+0000","last_peered":"2023-10-09T06:15:23.912284+0000","last_clean":"2023-10-09T06:15:23.912284+0000","last_became_active":"2023-10-09T06:13:21.737969+0000","last_became_peered":"2023-10-09T06:13:21.737969+0000","last_unstale":"2023-10-09T06:15:23.912284+0000","last_undegraded":"2023-10-09T06:15:23.912284+0000","last_fullsized":"2023-10-09T06:15:23.912284+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:38.620281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":624,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912819+0000","last_change":"2023-10-09T06:15:16.810530+0000","last_active":"2023-10-09T06:15:23.912819+0000","last_peered":"2023-10-09T06:15:23.912819+0000","last_clean":"2023-10-09T06:15:23.912819+0000","last_became_active":"2023-10-09T06:15:16.810238+0000","last_became_peered":"2023-10-09T06:15:16.810238+0000","last_unstale":"2023-10-09T06:15:23.912819+0000","last_undegraded":"2023-10-09T06:15:23.912819+0000","last_fullsized":"2023-10-09T06:15:23.912819+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:03.744638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"566'12151","reported_seq":20541,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:41.646447+0000","last_change":"2023-10-09T06:14:28.746278+0000","last_active":"2023-10-09T06:15:41.646447+0000","last_peered":"2023-10-09T06:15:41.646447+0000","last_clean":"2023-10-09T06:15:41.646447+0000","last_became_active":"2023-10-09T06:14:28.728616+0000","last_became_peered":"2023-10-09T06:14:28.728616+0000","last_unstale":"2023-10-09T06:15:41.646447+0000","last_undegraded":"2023-10-09T06:15:41.646447+0000","last_fullsized":"2023-10-09T06:15:41.646447+0000","mapping_epoch":545,"log_start":"520'5855","ondisk_log_start":"520'5855","created":490,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6296,"log_dups_size":0,"ondisk_log_size":6296,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:32.106540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5177344,"num_objects":79,"num_object_clones":0,"num_object_copies":237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":79,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12151,"num_write_kb":391360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4137,"num_bytes_recovered":271122432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":901,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:31.704955+0000","last_change":"2023-10-09T06:13:15.428701+0000","last_active":"2023-10-09T06:15:31.704955+0000","last_peered":"2023-10-09T06:15:31.704955+0000","last_clean":"2023-10-09T06:15:31.704955+0000","last_became_active":"2023-10-09T06:13:15.428343+0000","last_became_peered":"2023-10-09T06:13:15.428343+0000","last_unstale":"2023-10-09T06:15:31.704955+0000","last_undegraded":"2023-10-09T06:15:31.704955+0000","last_fullsized":"2023-10-09T06:15:31.704955+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.373748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.373748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:55:34.235074+0000","scrub_duration":0.0024576450000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":556,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912308+0000","last_change":"2023-10-09T06:07:43.574863+0000","last_active":"2023-10-09T06:15:23.912308+0000","last_peered":"2023-10-09T06:15:23.912308+0000","last_clean":"2023-10-09T06:15:23.912308+0000","last_became_active":"2023-10-09T06:07:43.574576+0000","last_became_peered":"2023-10-09T06:07:43.574576+0000","last_unstale":"2023-10-09T06:15:23.912308+0000","last_undegraded":"2023-10-09T06:15:23.912308+0000","last_fullsized":"2023-10-09T06:15:23.912308+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:16.166004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":597,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.913081+0000","last_change":"2023-10-09T06:13:21.727814+0000","last_active":"2023-10-09T06:15:23.913081+0000","last_peered":"2023-10-09T06:15:23.913081+0000","last_clean":"2023-10-09T06:15:23.913081+0000","last_became_active":"2023-10-09T06:13:21.727401+0000","last_became_peered":"2023-10-09T06:13:21.727401+0000","last_unstale":"2023-10-09T06:15:23.913081+0000","last_undegraded":"2023-10-09T06:15:23.913081+0000","last_fullsized":"2023-10-09T06:15:23.913081+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:53:37.968028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"566'12005","reported_seq":12089,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:40.970846+0000","last_change":"2023-10-09T06:11:45.695709+0000","last_active":"2023-10-09T06:15:40.970846+0000","last_peered":"2023-10-09T06:15:40.970846+0000","last_clean":"2023-10-09T06:15:40.970846+0000","last_became_active":"2023-10-09T06:11:45.695219+0000","last_became_peered":"2023-10-09T06:11:45.695219+0000","last_unstale":"2023-10-09T06:15:40.970846+0000","last_undegraded":"2023-10-09T06:15:40.970846+0000","last_fullsized":"2023-10-09T06:15:40.970846+0000","mapping_epoch":490,"log_start":"530'6600","ondisk_log_start":"530'6600","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5405,"log_dups_size":0,"ondisk_log_size":5405,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:26.533964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7012352,"num_objects":107,"num_object_clones":0,"num_object_copies":321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12005,"num_write_kb":387584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":645,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.800199+0000","last_change":"2023-10-09T06:13:24.753519+0000","last_active":"2023-10-09T06:15:32.800199+0000","last_peered":"2023-10-09T06:15:32.800199+0000","last_clean":"2023-10-09T06:15:32.800199+0000","last_became_active":"2023-10-09T06:13:24.753185+0000","last_became_peered":"2023-10-09T06:13:24.753185+0000","last_unstale":"2023-10-09T06:15:32.800199+0000","last_undegraded":"2023-10-09T06:15:32.800199+0000","last_fullsized":"2023-10-09T06:15:32.800199+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:39.633410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:39.633410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:56.871057+0000","scrub_duration":0.001547624,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":378,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912475+0000","last_change":"2023-10-09T06:13:30.781849+0000","last_active":"2023-10-09T06:15:23.912475+0000","last_peered":"2023-10-09T06:15:23.912475+0000","last_clean":"2023-10-09T06:15:23.912475+0000","last_became_active":"2023-10-09T06:13:30.781084+0000","last_became_peered":"2023-10-09T06:13:30.781084+0000","last_unstale":"2023-10-09T06:15:23.912475+0000","last_undegraded":"2023-10-09T06:15:23.912475+0000","last_fullsized":"2023-10-09T06:15:23.912475+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:02.199298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":249,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:26.703516+0000","last_change":"2023-10-09T06:01:54.761768+0000","last_active":"2023-10-09T06:15:26.703516+0000","last_peered":"2023-10-09T06:15:26.703516+0000","last_clean":"2023-10-09T06:15:26.703516+0000","last_became_active":"2023-10-09T06:01:54.761443+0000","last_became_peered":"2023-10-09T06:01:54.761443+0000","last_unstale":"2023-10-09T06:15:26.703516+0000","last_undegraded":"2023-10-09T06:15:26.703516+0000","last_fullsized":"2023-10-09T06:15:26.703516+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:51.555987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"567'12206","reported_seq":19161,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T06:15:43.578051+0000","last_change":"2023-10-09T06:14:44.803724+0000","last_active":"2023-10-09T06:15:43.578051+0000","last_peered":"2023-10-09T06:15:43.578051+0000","last_clean":"2023-10-09T06:15:43.578051+0000","last_became_active":"2023-10-09T06:14:44.737126+0000","last_became_peered":"2023-10-09T06:14:44.737126+0000","last_unstale":"2023-10-09T06:15:43.578051+0000","last_undegraded":"2023-10-09T06:15:43.578051+0000","last_fullsized":"2023-10-09T06:15:43.578051+0000","mapping_epoch":549,"log_start":"520'5915","ondisk_log_start":"520'5915","created":490,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6291,"log_dups_size":0,"ondisk_log_size":6291,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:14.630144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12206,"num_write_kb":390592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3423,"num_bytes_recovered":224329728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":769,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.913739+0000","last_change":"2023-10-09T06:13:33.707885+0000","last_active":"2023-10-09T06:15:23.913739+0000","last_peered":"2023-10-09T06:15:23.913739+0000","last_clean":"2023-10-09T06:15:23.913739+0000","last_became_active":"2023-10-09T06:13:33.707474+0000","last_became_peered":"2023-10-09T06:13:33.707474+0000","last_unstale":"2023-10-09T06:15:23.913739+0000","last_undegraded":"2023-10-09T06:15:23.913739+0000","last_fullsized":"2023-10-09T06:15:23.913739+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:12:22.289803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:12:22.289803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:27.842817+0000","scrub_duration":0.0076565050000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":311,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.910670+0000","last_change":"2023-10-09T06:12:04.680247+0000","last_active":"2023-10-09T06:15:23.910670+0000","last_peered":"2023-10-09T06:15:23.910670+0000","last_clean":"2023-10-09T06:15:23.910670+0000","last_became_active":"2023-10-09T06:12:04.679943+0000","last_became_peered":"2023-10-09T06:12:04.679943+0000","last_unstale":"2023-10-09T06:15:23.910670+0000","last_undegraded":"2023-10-09T06:15:23.910670+0000","last_fullsized":"2023-10-09T06:15:23.910670+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:56.692774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":354,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.913507+0000","last_change":"2023-10-09T06:07:43.577306+0000","last_active":"2023-10-09T06:15:23.913507+0000","last_peered":"2023-10-09T06:15:23.913507+0000","last_clean":"2023-10-09T06:15:23.913507+0000","last_became_active":"2023-10-09T06:07:43.576956+0000","last_became_peered":"2023-10-09T06:07:43.576956+0000","last_unstale":"2023-10-09T06:15:23.913507+0000","last_undegraded":"2023-10-09T06:15:23.913507+0000","last_fullsized":"2023-10-09T06:15:23.913507+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:21.020403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":282,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.910564+0000","last_change":"2023-10-09T06:13:12.234849+0000","last_active":"2023-10-09T06:15:23.910564+0000","last_peered":"2023-10-09T06:15:23.910564+0000","last_clean":"2023-10-09T06:15:23.910564+0000","last_became_active":"2023-10-09T06:13:12.234284+0000","last_became_peered":"2023-10-09T06:13:12.234284+0000","last_unstale":"2023-10-09T06:15:23.910564+0000","last_undegraded":"2023-10-09T06:15:23.910564+0000","last_fullsized":"2023-10-09T06:15:23.910564+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:40:43.809773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"566'11789","reported_seq":11875,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:41.406330+0000","last_change":"2023-10-09T06:11:46.029349+0000","last_active":"2023-10-09T06:15:41.406330+0000","last_peered":"2023-10-09T06:15:41.406330+0000","last_clean":"2023-10-09T06:15:41.406330+0000","last_became_active":"2023-10-09T06:11:46.029041+0000","last_became_peered":"2023-10-09T06:11:46.029041+0000","last_unstale":"2023-10-09T06:15:41.406330+0000","last_undegraded":"2023-10-09T06:15:41.406330+0000","last_fullsized":"2023-10-09T06:15:41.406330+0000","mapping_epoch":490,"log_start":"519'5600","ondisk_log_start":"519'5600","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6189,"log_dups_size":0,"ondisk_log_size":6189,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:07.662395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5439488,"num_objects":83,"num_object_clones":0,"num_object_copies":249,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":83,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11789,"num_write_kb":379904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":802,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.803821+0000","last_change":"2023-10-09T06:09:27.544555+0000","last_active":"2023-10-09T06:15:32.803821+0000","last_peered":"2023-10-09T06:15:32.803821+0000","last_clean":"2023-10-09T06:15:32.803821+0000","last_became_active":"2023-10-09T06:09:27.544222+0000","last_became_peered":"2023-10-09T06:09:27.544222+0000","last_unstale":"2023-10-09T06:15:32.803821+0000","last_undegraded":"2023-10-09T06:15:32.803821+0000","last_fullsized":"2023-10-09T06:15:32.803821+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:45.944253+0000","scrub_duration":0.0016660799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":267,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.910606+0000","last_change":"2023-10-09T06:13:21.738040+0000","last_active":"2023-10-09T06:15:23.910606+0000","last_peered":"2023-10-09T06:15:23.910606+0000","last_clean":"2023-10-09T06:15:23.910606+0000","last_became_active":"2023-10-09T06:13:21.737717+0000","last_became_peered":"2023-10-09T06:13:21.737717+0000","last_unstale":"2023-10-09T06:15:23.910606+0000","last_undegraded":"2023-10-09T06:15:23.910606+0000","last_fullsized":"2023-10-09T06:15:23.910606+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:39.451628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":546,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912432+0000","last_change":"2023-10-09T06:09:27.553958+0000","last_active":"2023-10-09T06:15:23.912432+0000","last_peered":"2023-10-09T06:15:23.912432+0000","last_clean":"2023-10-09T06:15:23.912432+0000","last_became_active":"2023-10-09T06:09:27.553193+0000","last_became_peered":"2023-10-09T06:09:27.553193+0000","last_unstale":"2023-10-09T06:15:23.912432+0000","last_undegraded":"2023-10-09T06:15:23.912432+0000","last_fullsized":"2023-10-09T06:15:23.912432+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:09.754579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":392,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.910748+0000","last_change":"2023-10-09T06:07:43.580765+0000","last_active":"2023-10-09T06:15:23.910748+0000","last_peered":"2023-10-09T06:15:23.910748+0000","last_clean":"2023-10-09T06:15:23.910748+0000","last_became_active":"2023-10-09T06:07:43.580458+0000","last_became_peered":"2023-10-09T06:07:43.580458+0000","last_unstale":"2023-10-09T06:15:23.910748+0000","last_undegraded":"2023-10-09T06:15:23.910748+0000","last_fullsized":"2023-10-09T06:15:23.910748+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:55.161893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"567'11900","reported_seq":28241,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T06:15:43.593159+0000","last_change":"2023-10-09T06:14:21.793267+0000","last_active":"2023-10-09T06:15:43.593159+0000","last_peered":"2023-10-09T06:15:43.593159+0000","last_clean":"2023-10-09T06:15:43.593159+0000","last_became_active":"2023-10-09T06:14:21.734665+0000","last_became_peered":"2023-10-09T06:14:21.734665+0000","last_unstale":"2023-10-09T06:15:43.593159+0000","last_undegraded":"2023-10-09T06:15:43.593159+0000","last_fullsized":"2023-10-09T06:15:43.593159+0000","mapping_epoch":543,"log_start":"519'5616","ondisk_log_start":"519'5616","created":490,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6284,"log_dups_size":0,"ondisk_log_size":6284,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:01.941433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11900,"num_write_kb":380800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7073,"num_bytes_recovered":463536128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":622,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.795797+0000","last_change":"2023-10-09T06:14:47.339271+0000","last_active":"2023-10-09T06:15:32.795797+0000","last_peered":"2023-10-09T06:15:32.795797+0000","last_clean":"2023-10-09T06:15:32.795797+0000","last_became_active":"2023-10-09T06:13:33.705351+0000","last_became_peered":"2023-10-09T06:13:33.705351+0000","last_unstale":"2023-10-09T06:15:32.795797+0000","last_undegraded":"2023-10-09T06:15:32.795797+0000","last_fullsized":"2023-10-09T06:15:32.795797+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:14:47.339163+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T06:14:47.339163+0000","last_clean_scrub_stamp":"2023-10-09T06:14:47.339163+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:34.887758+0000","scrub_duration":0.0016607729999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":356,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.798437+0000","last_change":"2023-10-09T06:15:16.802788+0000","last_active":"2023-10-09T06:15:32.798437+0000","last_peered":"2023-10-09T06:15:32.798437+0000","last_clean":"2023-10-09T06:15:32.798437+0000","last_became_active":"2023-10-09T06:15:16.802449+0000","last_became_peered":"2023-10-09T06:15:16.802449+0000","last_unstale":"2023-10-09T06:15:32.798437+0000","last_undegraded":"2023-10-09T06:15:32.798437+0000","last_fullsized":"2023-10-09T06:15:32.798437+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:55:30.488174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":336,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.910431+0000","last_change":"2023-10-09T06:13:21.728291+0000","last_active":"2023-10-09T06:15:23.910431+0000","last_peered":"2023-10-09T06:15:23.910431+0000","last_clean":"2023-10-09T06:15:23.910431+0000","last_became_active":"2023-10-09T06:13:21.727926+0000","last_became_peered":"2023-10-09T06:13:21.727926+0000","last_unstale":"2023-10-09T06:15:23.910431+0000","last_undegraded":"2023-10-09T06:15:23.910431+0000","last_fullsized":"2023-10-09T06:15:23.910431+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:35.292785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":264,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.911532+0000","last_change":"2023-10-09T06:09:27.547953+0000","last_active":"2023-10-09T06:15:23.911532+0000","last_peered":"2023-10-09T06:15:23.911532+0000","last_clean":"2023-10-09T06:15:23.911532+0000","last_became_active":"2023-10-09T06:09:27.547624+0000","last_became_peered":"2023-10-09T06:09:27.547624+0000","last_unstale":"2023-10-09T06:15:23.911532+0000","last_undegraded":"2023-10-09T06:15:23.911532+0000","last_fullsized":"2023-10-09T06:15:23.911532+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:39.630486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"566'11842","reported_seq":19764,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:41.694858+0000","last_change":"2023-10-09T06:14:43.759714+0000","last_active":"2023-10-09T06:15:41.694858+0000","last_peered":"2023-10-09T06:15:41.694858+0000","last_clean":"2023-10-09T06:15:41.694858+0000","last_became_active":"2023-10-09T06:14:43.736592+0000","last_became_peered":"2023-10-09T06:14:43.736592+0000","last_unstale":"2023-10-09T06:15:41.694858+0000","last_undegraded":"2023-10-09T06:15:41.694858+0000","last_fullsized":"2023-10-09T06:15:41.694858+0000","mapping_epoch":548,"log_start":"515'5510","ondisk_log_start":"515'5510","created":490,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6332,"log_dups_size":0,"ondisk_log_size":6332,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:35.196970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6029312,"num_objects":92,"num_object_clones":0,"num_object_copies":276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":92,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11842,"num_write_kb":381888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3906,"num_bytes_recovered":255983616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":50,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.796155+0000","last_change":"2023-10-09T06:13:30.791685+0000","last_active":"2023-10-09T06:15:32.796155+0000","last_peered":"2023-10-09T06:15:32.796155+0000","last_clean":"2023-10-09T06:15:32.796155+0000","last_became_active":"2023-10-09T06:13:30.791255+0000","last_became_peered":"2023-10-09T06:13:30.791255+0000","last_unstale":"2023-10-09T06:15:32.796155+0000","last_undegraded":"2023-10-09T06:15:32.796155+0000","last_fullsized":"2023-10-09T06:15:32.796155+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:24.896244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":610,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.795675+0000","last_change":"2023-10-09T06:13:33.704835+0000","last_active":"2023-10-09T06:15:32.795675+0000","last_peered":"2023-10-09T06:15:32.795675+0000","last_clean":"2023-10-09T06:15:32.795675+0000","last_became_active":"2023-10-09T06:13:33.704466+0000","last_became_peered":"2023-10-09T06:13:33.704466+0000","last_unstale":"2023-10-09T06:15:32.795675+0000","last_undegraded":"2023-10-09T06:15:32.795675+0000","last_fullsized":"2023-10-09T06:15:32.795675+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:35.783434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:35.783434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:38.721974+0000","scrub_duration":0.0043794799999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":354,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912740+0000","last_change":"2023-10-09T06:13:12.238280+0000","last_active":"2023-10-09T06:15:23.912740+0000","last_peered":"2023-10-09T06:15:23.912740+0000","last_clean":"2023-10-09T06:15:23.912740+0000","last_became_active":"2023-10-09T06:13:12.237986+0000","last_became_peered":"2023-10-09T06:13:12.237986+0000","last_unstale":"2023-10-09T06:15:23.912740+0000","last_undegraded":"2023-10-09T06:15:23.912740+0000","last_fullsized":"2023-10-09T06:15:23.912740+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:58.520176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":496,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912762+0000","last_change":"2023-10-09T06:13:21.733940+0000","last_active":"2023-10-09T06:15:23.912762+0000","last_peered":"2023-10-09T06:15:23.912762+0000","last_clean":"2023-10-09T06:15:23.912762+0000","last_became_active":"2023-10-09T06:13:21.733628+0000","last_became_peered":"2023-10-09T06:13:21.733628+0000","last_unstale":"2023-10-09T06:15:23.912762+0000","last_undegraded":"2023-10-09T06:15:23.912762+0000","last_fullsized":"2023-10-09T06:15:23.912762+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:13.348443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"567'11914","reported_seq":21892,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T06:15:43.546004+0000","last_change":"2023-10-09T06:14:09.843785+0000","last_active":"2023-10-09T06:15:43.546004+0000","last_peered":"2023-10-09T06:15:43.546004+0000","last_clean":"2023-10-09T06:15:43.546004+0000","last_became_active":"2023-10-09T06:14:09.789864+0000","last_became_peered":"2023-10-09T06:14:09.789864+0000","last_unstale":"2023-10-09T06:15:43.546004+0000","last_undegraded":"2023-10-09T06:15:43.546004+0000","last_fullsized":"2023-10-09T06:15:43.546004+0000","mapping_epoch":540,"log_start":"518'5611","ondisk_log_start":"518'5611","created":490,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":6303,"log_dups_size":0,"ondisk_log_size":6303,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:03.191322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11914,"num_write_kb":381248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4936,"num_bytes_recovered":323485696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":835,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.911712+0000","last_change":"2023-10-09T06:13:24.753842+0000","last_active":"2023-10-09T06:15:23.911712+0000","last_peered":"2023-10-09T06:15:23.911712+0000","last_clean":"2023-10-09T06:15:23.911712+0000","last_became_active":"2023-10-09T06:13:24.753485+0000","last_became_peered":"2023-10-09T06:13:24.753485+0000","last_unstale":"2023-10-09T06:15:23.911712+0000","last_undegraded":"2023-10-09T06:15:23.911712+0000","last_fullsized":"2023-10-09T06:15:23.911712+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:57.627296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:57.627296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:56.656218+0000","scrub_duration":0.001913166,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":460,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.795971+0000","last_change":"2023-10-09T06:09:27.544373+0000","last_active":"2023-10-09T06:15:32.795971+0000","last_peered":"2023-10-09T06:15:32.795971+0000","last_clean":"2023-10-09T06:15:32.795971+0000","last_became_active":"2023-10-09T06:09:27.544121+0000","last_became_peered":"2023-10-09T06:09:27.544121+0000","last_unstale":"2023-10-09T06:15:32.795971+0000","last_undegraded":"2023-10-09T06:15:32.795971+0000","last_fullsized":"2023-10-09T06:15:32.795971+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:40.429491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":504,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.803615+0000","last_change":"2023-10-09T06:12:52.670573+0000","last_active":"2023-10-09T06:15:32.803615+0000","last_peered":"2023-10-09T06:15:32.803615+0000","last_clean":"2023-10-09T06:15:32.803615+0000","last_became_active":"2023-10-09T06:12:52.670242+0000","last_became_peered":"2023-10-09T06:12:52.670242+0000","last_unstale":"2023-10-09T06:15:32.803615+0000","last_undegraded":"2023-10-09T06:15:32.803615+0000","last_fullsized":"2023-10-09T06:15:32.803615+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:25.237580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":651,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.915870+0000","last_change":"2023-10-09T06:15:16.805514+0000","last_active":"2023-10-09T06:15:23.915870+0000","last_peered":"2023-10-09T06:15:23.915870+0000","last_clean":"2023-10-09T06:15:23.915870+0000","last_became_active":"2023-10-09T06:15:16.805160+0000","last_became_peered":"2023-10-09T06:15:16.805160+0000","last_unstale":"2023-10-09T06:15:23.915870+0000","last_undegraded":"2023-10-09T06:15:23.915870+0000","last_fullsized":"2023-10-09T06:15:23.915870+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:30.153699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"566'11867","reported_seq":21483,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:40.916041+0000","last_change":"2023-10-09T06:14:02.643285+0000","last_active":"2023-10-09T06:15:40.916041+0000","last_peered":"2023-10-09T06:15:40.916041+0000","last_clean":"2023-10-09T06:15:40.916041+0000","last_became_active":"2023-10-09T06:14:02.599489+0000","last_became_peered":"2023-10-09T06:14:02.599489+0000","last_unstale":"2023-10-09T06:15:40.916041+0000","last_undegraded":"2023-10-09T06:15:40.916041+0000","last_fullsized":"2023-10-09T06:15:40.916041+0000","mapping_epoch":537,"log_start":"531'6500","ondisk_log_start":"531'6500","created":490,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5367,"log_dups_size":0,"ondisk_log_size":5367,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:10.022074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7602200,"num_objects":117,"num_object_clones":0,"num_object_copies":351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":117,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11867,"num_write_kb":383425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4755,"num_bytes_recovered":311558168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":766,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.913638+0000","last_change":"2023-10-09T06:13:24.756288+0000","last_active":"2023-10-09T06:15:23.913638+0000","last_peered":"2023-10-09T06:15:23.913638+0000","last_clean":"2023-10-09T06:15:23.913638+0000","last_became_active":"2023-10-09T06:13:24.755945+0000","last_became_peered":"2023-10-09T06:13:24.755945+0000","last_unstale":"2023-10-09T06:15:23.913638+0000","last_undegraded":"2023-10-09T06:15:23.913638+0000","last_fullsized":"2023-10-09T06:15:23.913638+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:45.844892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:45.844892+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:44.107376+0000","scrub_duration":0.0017521489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":50,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.797353+0000","last_change":"2023-10-09T06:13:30.799535+0000","last_active":"2023-10-09T06:15:32.797353+0000","last_peered":"2023-10-09T06:15:32.797353+0000","last_clean":"2023-10-09T06:15:32.797353+0000","last_became_active":"2023-10-09T06:13:30.799227+0000","last_became_peered":"2023-10-09T06:13:30.799227+0000","last_unstale":"2023-10-09T06:15:32.797353+0000","last_undegraded":"2023-10-09T06:15:32.797353+0000","last_fullsized":"2023-10-09T06:15:32.797353+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:14.741742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":294,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.910830+0000","last_change":"2023-10-09T06:15:16.810272+0000","last_active":"2023-10-09T06:15:23.910830+0000","last_peered":"2023-10-09T06:15:23.910830+0000","last_clean":"2023-10-09T06:15:23.910830+0000","last_became_active":"2023-10-09T06:15:16.804086+0000","last_became_peered":"2023-10-09T06:15:16.804086+0000","last_unstale":"2023-10-09T06:15:23.910830+0000","last_undegraded":"2023-10-09T06:15:23.910830+0000","last_fullsized":"2023-10-09T06:15:23.910830+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:45.156184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":50,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.795999+0000","last_change":"2023-10-09T06:13:30.788382+0000","last_active":"2023-10-09T06:15:32.795999+0000","last_peered":"2023-10-09T06:15:32.795999+0000","last_clean":"2023-10-09T06:15:32.795999+0000","last_became_active":"2023-10-09T06:13:30.788060+0000","last_became_peered":"2023-10-09T06:13:30.788060+0000","last_unstale":"2023-10-09T06:15:32.795999+0000","last_undegraded":"2023-10-09T06:15:32.795999+0000","last_fullsized":"2023-10-09T06:15:32.795999+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:36:58.984799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.e","version":"566'12069","reported_seq":24519,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:40.976832+0000","last_change":"2023-10-09T06:15:23.486583+0000","last_active":"2023-10-09T06:15:40.976832+0000","last_peered":"2023-10-09T06:15:40.976832+0000","last_clean":"2023-10-09T06:15:40.976832+0000","last_became_active":"2023-10-09T06:15:23.398455+0000","last_became_peered":"2023-10-09T06:15:23.398455+0000","last_unstale":"2023-10-09T06:15:40.976832+0000","last_undegraded":"2023-10-09T06:15:40.976832+0000","last_fullsized":"2023-10-09T06:15:40.976832+0000","mapping_epoch":563,"log_start":"531'6700","ondisk_log_start":"531'6700","created":490,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5369,"log_dups_size":0,"ondisk_log_size":5369,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:19.708343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7012352,"num_objects":107,"num_object_clones":0,"num_object_copies":321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12069,"num_write_kb":389632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6155,"num_bytes_recovered":403243008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":1059,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912993+0000","last_change":"2023-10-09T06:15:19.869123+0000","last_active":"2023-10-09T06:15:23.912993+0000","last_peered":"2023-10-09T06:15:23.912993+0000","last_clean":"2023-10-09T06:15:23.912993+0000","last_became_active":"2023-10-09T06:15:19.868795+0000","last_became_peered":"2023-10-09T06:15:19.868795+0000","last_unstale":"2023-10-09T06:15:23.912993+0000","last_undegraded":"2023-10-09T06:15:23.912993+0000","last_fullsized":"2023-10-09T06:15:23.912993+0000","mapping_epoch":561,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":562,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:17.344518+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:17.344518+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:34.303488+0000","scrub_duration":0.0015914099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":623,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.913936+0000","last_change":"2023-10-09T06:13:30.798394+0000","last_active":"2023-10-09T06:15:23.913936+0000","last_peered":"2023-10-09T06:15:23.913936+0000","last_clean":"2023-10-09T06:15:23.913936+0000","last_became_active":"2023-10-09T06:13:30.790913+0000","last_became_peered":"2023-10-09T06:13:30.790913+0000","last_unstale":"2023-10-09T06:15:23.913936+0000","last_undegraded":"2023-10-09T06:15:23.913936+0000","last_fullsized":"2023-10-09T06:15:23.913936+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:26.508690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":383,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.911594+0000","last_change":"2023-10-09T06:13:21.728009+0000","last_active":"2023-10-09T06:15:23.911594+0000","last_peered":"2023-10-09T06:15:23.911594+0000","last_clean":"2023-10-09T06:15:23.911594+0000","last_became_active":"2023-10-09T06:13:21.727683+0000","last_became_peered":"2023-10-09T06:13:21.727683+0000","last_unstale":"2023-10-09T06:15:23.911594+0000","last_undegraded":"2023-10-09T06:15:23.911594+0000","last_fullsized":"2023-10-09T06:15:23.911594+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:13.074232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":372,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:30.574313+0000","last_change":"2023-10-09T06:13:30.782038+0000","last_active":"2023-10-09T06:15:30.574313+0000","last_peered":"2023-10-09T06:15:30.574313+0000","last_clean":"2023-10-09T06:15:30.574313+0000","last_became_active":"2023-10-09T06:13:30.781500+0000","last_became_peered":"2023-10-09T06:13:30.781500+0000","last_unstale":"2023-10-09T06:15:30.574313+0000","last_undegraded":"2023-10-09T06:15:30.574313+0000","last_fullsized":"2023-10-09T06:15:30.574313+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:43.663265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"566'11791","reported_seq":11877,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:40.534846+0000","last_change":"2023-10-09T06:11:46.031709+0000","last_active":"2023-10-09T06:15:40.534846+0000","last_peered":"2023-10-09T06:15:40.534846+0000","last_clean":"2023-10-09T06:15:40.534846+0000","last_became_active":"2023-10-09T06:11:46.031421+0000","last_became_peered":"2023-10-09T06:11:46.031421+0000","last_unstale":"2023-10-09T06:15:40.534846+0000","last_undegraded":"2023-10-09T06:15:40.534846+0000","last_fullsized":"2023-10-09T06:15:40.534846+0000","mapping_epoch":490,"log_start":"520'5825","ondisk_log_start":"520'5825","created":490,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:11:44.668553+0000","last_clean_scrub_stamp":"2023-10-09T06:11:44.668553+0000","objects_scrubbed":0,"log_size":5966,"log_dups_size":0,"ondisk_log_size":5966,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:13.095174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8454144,"num_objects":129,"num_object_clones":0,"num_object_copies":387,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":129,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11791,"num_write_kb":381440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":898,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.913370+0000","last_change":"2023-10-09T06:13:33.708100+0000","last_active":"2023-10-09T06:15:23.913370+0000","last_peered":"2023-10-09T06:15:23.913370+0000","last_clean":"2023-10-09T06:15:23.913370+0000","last_became_active":"2023-10-09T06:13:33.707809+0000","last_became_peered":"2023-10-09T06:13:33.707809+0000","last_unstale":"2023-10-09T06:15:23.913370+0000","last_undegraded":"2023-10-09T06:15:23.913370+0000","last_fullsized":"2023-10-09T06:15:23.913370+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:43:26.848778+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":50,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.797807+0000","last_change":"2023-10-09T06:13:30.799948+0000","last_active":"2023-10-09T06:15:32.797807+0000","last_peered":"2023-10-09T06:15:32.797807+0000","last_clean":"2023-10-09T06:15:32.797807+0000","last_became_active":"2023-10-09T06:13:30.799690+0000","last_became_peered":"2023-10-09T06:13:30.799690+0000","last_unstale":"2023-10-09T06:15:32.797807+0000","last_undegraded":"2023-10-09T06:15:32.797807+0000","last_fullsized":"2023-10-09T06:15:32.797807+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:43.893184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":612,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.916028+0000","last_change":"2023-10-09T06:09:27.578756+0000","last_active":"2023-10-09T06:15:23.916028+0000","last_peered":"2023-10-09T06:15:23.916028+0000","last_clean":"2023-10-09T06:15:23.916028+0000","last_became_active":"2023-10-09T06:09:27.578384+0000","last_became_peered":"2023-10-09T06:09:27.578384+0000","last_unstale":"2023-10-09T06:15:23.916028+0000","last_undegraded":"2023-10-09T06:15:23.916028+0000","last_fullsized":"2023-10-09T06:15:23.916028+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:22.182630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":512,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912330+0000","last_change":"2023-10-09T06:01:22.334081+0000","last_active":"2023-10-09T06:15:23.912330+0000","last_peered":"2023-10-09T06:15:23.912330+0000","last_clean":"2023-10-09T06:15:23.912330+0000","last_became_active":"2023-10-09T06:01:22.333663+0000","last_became_peered":"2023-10-09T06:01:22.333663+0000","last_unstale":"2023-10-09T06:15:23.912330+0000","last_undegraded":"2023-10-09T06:15:23.912330+0000","last_fullsized":"2023-10-09T06:15:23.912330+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:55.809259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":812,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912548+0000","last_change":"2023-10-09T06:09:27.560157+0000","last_active":"2023-10-09T06:15:23.912548+0000","last_peered":"2023-10-09T06:15:23.912548+0000","last_clean":"2023-10-09T06:15:23.912548+0000","last_became_active":"2023-10-09T06:09:27.559466+0000","last_became_peered":"2023-10-09T06:09:27.559466+0000","last_unstale":"2023-10-09T06:15:23.912548+0000","last_undegraded":"2023-10-09T06:15:23.912548+0000","last_fullsized":"2023-10-09T06:15:23.912548+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.181811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.181811+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:53.431491+0000","scrub_duration":0.011553313000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":495,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.916070+0000","last_change":"2023-10-09T06:13:30.791343+0000","last_active":"2023-10-09T06:15:23.916070+0000","last_peered":"2023-10-09T06:15:23.916070+0000","last_clean":"2023-10-09T06:15:23.916070+0000","last_became_active":"2023-10-09T06:13:30.790945+0000","last_became_peered":"2023-10-09T06:13:30.790945+0000","last_unstale":"2023-10-09T06:15:23.916070+0000","last_undegraded":"2023-10-09T06:15:23.916070+0000","last_fullsized":"2023-10-09T06:15:23.916070+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:14.208391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":277,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.910906+0000","last_change":"2023-10-09T06:15:16.803457+0000","last_active":"2023-10-09T06:15:23.910906+0000","last_peered":"2023-10-09T06:15:23.910906+0000","last_clean":"2023-10-09T06:15:23.910906+0000","last_became_active":"2023-10-09T06:15:16.803097+0000","last_became_peered":"2023-10-09T06:15:16.803097+0000","last_unstale":"2023-10-09T06:15:23.910906+0000","last_undegraded":"2023-10-09T06:15:23.910906+0000","last_fullsized":"2023-10-09T06:15:23.910906+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:45.511763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":915,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.913507+0000","last_change":"2023-10-09T06:00:04.215425+0000","last_active":"2023-10-09T06:15:23.913507+0000","last_peered":"2023-10-09T06:15:23.913507+0000","last_clean":"2023-10-09T06:15:23.913507+0000","last_became_active":"2023-10-09T05:59:40.326685+0000","last_became_peered":"2023-10-09T05:59:40.326685+0000","last_unstale":"2023-10-09T06:15:23.913507+0000","last_undegraded":"2023-10-09T06:15:23.913507+0000","last_fullsized":"2023-10-09T06:15:23.913507+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:04.215270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:04.215270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:03.963694+0000","scrub_duration":0.002850652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":337,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912742+0000","last_change":"2023-10-09T06:01:54.824780+0000","last_active":"2023-10-09T06:15:23.912742+0000","last_peered":"2023-10-09T06:15:23.912742+0000","last_clean":"2023-10-09T06:15:23.912742+0000","last_became_active":"2023-10-09T06:01:54.757727+0000","last_became_peered":"2023-10-09T06:01:54.757727+0000","last_unstale":"2023-10-09T06:15:23.912742+0000","last_undegraded":"2023-10-09T06:15:23.912742+0000","last_fullsized":"2023-10-09T06:15:23.912742+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:44.132556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":584,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912704+0000","last_change":"2023-10-09T06:01:54.764989+0000","last_active":"2023-10-09T06:15:23.912704+0000","last_peered":"2023-10-09T06:15:23.912704+0000","last_clean":"2023-10-09T06:15:23.912704+0000","last_became_active":"2023-10-09T06:01:54.764790+0000","last_became_peered":"2023-10-09T06:01:54.764790+0000","last_unstale":"2023-10-09T06:15:23.912704+0000","last_undegraded":"2023-10-09T06:15:23.912704+0000","last_fullsized":"2023-10-09T06:15:23.912704+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:14:07.208858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":351,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912437+0000","last_change":"2023-10-09T06:13:21.739730+0000","last_active":"2023-10-09T06:15:23.912437+0000","last_peered":"2023-10-09T06:15:23.912437+0000","last_clean":"2023-10-09T06:15:23.912437+0000","last_became_active":"2023-10-09T06:13:21.727326+0000","last_became_peered":"2023-10-09T06:13:21.727326+0000","last_unstale":"2023-10-09T06:15:23.912437+0000","last_undegraded":"2023-10-09T06:15:23.912437+0000","last_fullsized":"2023-10-09T06:15:23.912437+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:51.344571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":581,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912286+0000","last_change":"2023-10-09T06:09:27.565198+0000","last_active":"2023-10-09T06:15:23.912286+0000","last_peered":"2023-10-09T06:15:23.912286+0000","last_clean":"2023-10-09T06:15:23.912286+0000","last_became_active":"2023-10-09T06:09:27.564909+0000","last_became_peered":"2023-10-09T06:09:27.564909+0000","last_unstale":"2023-10-09T06:15:23.912286+0000","last_undegraded":"2023-10-09T06:15:23.912286+0000","last_fullsized":"2023-10-09T06:15:23.912286+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:46.101014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":732,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.797963+0000","last_change":"2023-10-09T06:13:33.699235+0000","last_active":"2023-10-09T06:15:32.797963+0000","last_peered":"2023-10-09T06:15:32.797963+0000","last_clean":"2023-10-09T06:15:32.797963+0000","last_became_active":"2023-10-09T06:13:33.698886+0000","last_became_peered":"2023-10-09T06:13:33.698886+0000","last_unstale":"2023-10-09T06:15:32.797963+0000","last_undegraded":"2023-10-09T06:15:32.797963+0000","last_fullsized":"2023-10-09T06:15:32.797963+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:00.456132+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:00.456132+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:59.421706+0000","scrub_duration":0.0014992199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":382,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:30.574484+0000","last_change":"2023-10-09T06:15:16.804361+0000","last_active":"2023-10-09T06:15:30.574484+0000","last_peered":"2023-10-09T06:15:30.574484+0000","last_clean":"2023-10-09T06:15:30.574484+0000","last_became_active":"2023-10-09T06:15:16.803977+0000","last_became_peered":"2023-10-09T06:15:16.803977+0000","last_unstale":"2023-10-09T06:15:30.574484+0000","last_undegraded":"2023-10-09T06:15:30.574484+0000","last_fullsized":"2023-10-09T06:15:30.574484+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:14.141738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":368,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912720+0000","last_change":"2023-10-09T06:09:27.565185+0000","last_active":"2023-10-09T06:15:23.912720+0000","last_peered":"2023-10-09T06:15:23.912720+0000","last_clean":"2023-10-09T06:15:23.912720+0000","last_became_active":"2023-10-09T06:09:27.564478+0000","last_became_peered":"2023-10-09T06:09:27.564478+0000","last_unstale":"2023-10-09T06:15:23.912720+0000","last_undegraded":"2023-10-09T06:15:23.912720+0000","last_fullsized":"2023-10-09T06:15:23.912720+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:04.622171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":890,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.911004+0000","last_change":"2023-10-09T06:12:55.753877+0000","last_active":"2023-10-09T06:15:23.911004+0000","last_peered":"2023-10-09T06:15:23.911004+0000","last_clean":"2023-10-09T06:15:23.911004+0000","last_became_active":"2023-10-09T06:12:55.753512+0000","last_became_peered":"2023-10-09T06:12:55.753512+0000","last_unstale":"2023-10-09T06:15:23.911004+0000","last_undegraded":"2023-10-09T06:15:23.911004+0000","last_fullsized":"2023-10-09T06:15:23.911004+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:46.290510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:46.290510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:30.368031+0000","scrub_duration":0.002627819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":574,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:15:32.810218+0000","last_change":"2023-10-09T06:15:16.814498+0000","last_active":"2023-10-09T06:15:32.810218+0000","last_peered":"2023-10-09T06:15:32.810218+0000","last_clean":"2023-10-09T06:15:32.810218+0000","last_became_active":"2023-10-09T06:15:16.807217+0000","last_became_peered":"2023-10-09T06:15:16.807217+0000","last_unstale":"2023-10-09T06:15:32.810218+0000","last_undegraded":"2023-10-09T06:15:32.810218+0000","last_fullsized":"2023-10-09T06:15:32.810218+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:50.886043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":597,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:15:23.912082+0000","last_change":"2023-10-09T06:13:21.727720+0000","last_active":"2023-10-09T06:15:23.912082+0000","last_peered":"2023-10-09T06:15:23.912082+0000","last_clean":"2023-10-09T06:15:23.912082+0000","last_became_active":"2023-10-09T06:13:21.727339+0000","last_became_peered":"2023-10-09T06:13:21.727339+0000","last_unstale":"2023-10-09T06:15:23.912082+0000","last_undegraded":"2023-10-09T06:15:23.912082+0000","last_fullsized":"2023-10-09T06:15:23.912082+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:25.723752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":94109720,"num_objects":1437,"num_object_clones":0,"num_object_copies":4311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1437,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":190989,"num_write_kb":6157569,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44071,"num_bytes_recovered":2887909400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":992092160,"data_stored":992084016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95638,"ondisk_log_size":95638,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":59,"num_bytes_recovered":12268,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13410,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1706112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234433,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903540,"kb_used_data":6828,"kb_used_omap":0,"kb_used_meta":896704,"kb_avail":92841612,"statfs":{"total":95995035648,"available":95069810688,"internally_reserved":0,"allocated":6991872,"data_stored":5409222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918224896},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:15:33 2023","interfaces":[{"interface":"back","average":{"1min":5.5789999999999997,"5min":6.3840000000000003,"15min":2.9500000000000002},"min":{"1min":0.246,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":72.067999999999998,"5min":126.167,"15min":126.167},"last":0.58299999999999996},{"interface":"front","average":{"1min":5.7869999999999999,"5min":6.6509999999999998,"15min":2.98},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":72.140000000000001,"5min":126.139,"15min":126.139},"last":0.54300000000000004}]},{"osd":1,"last update":"Mon Oct 9 06:15:07 2023","interfaces":[{"interface":"back","average":{"1min":12.173,"5min":4.9900000000000002,"15min":3.9649999999999999},"min":{"1min":0.35499999999999998,"5min":0.255,"15min":0.218},"max":{"1min":108.113,"5min":108.113,"15min":124.66200000000001},"last":0.56799999999999995},{"interface":"front","average":{"1min":10.855,"5min":4.8049999999999997,"15min":3.863},"min":{"1min":0.34699999999999998,"5min":0.33900000000000002,"15min":0.22700000000000001},"max":{"1min":108.503,"5min":108.503,"15min":125.023},"last":0.34999999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:15:33 2023","interfaces":[{"interface":"back","average":{"1min":2.528,"5min":2.3759999999999999,"15min":2.3559999999999999},"min":{"1min":0.32700000000000001,"5min":0.27400000000000002,"15min":0.24299999999999999},"max":{"1min":9.3900000000000006,"5min":47.503,"15min":127.86499999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.573,"5min":2.3170000000000002,"15min":2.3780000000000001},"min":{"1min":0.33300000000000002,"5min":0.30099999999999999,"15min":0.27000000000000002},"max":{"1min":6.2729999999999997,"5min":47.545000000000002,"15min":129.24000000000001},"last":0.375}]},{"osd":3,"last update":"Mon Oct 9 06:15:07 2023","interfaces":[{"interface":"back","average":{"1min":6.7400000000000002,"5min":7.7830000000000004,"15min":6.0800000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.29899999999999999},"max":{"1min":47.875,"5min":116.68000000000001,"15min":164.16999999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":5.391,"5min":7.6420000000000003,"15min":6.1580000000000004},"min":{"1min":0.34599999999999997,"5min":0.33300000000000002,"15min":0.32100000000000001},"max":{"1min":48.036999999999999,"5min":116.797,"15min":164.40700000000001},"last":0.53000000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:14:41 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":0.98499999999999999,"15min":0.95799999999999996},"min":{"1min":0.27900000000000003,"5min":0.23999999999999999,"15min":0.23100000000000001},"max":{"1min":6.9939999999999998,"5min":6.9939999999999998,"15min":14.371},"last":10.257},{"interface":"front","average":{"1min":1.3520000000000001,"5min":0.90100000000000002,"15min":0.94799999999999995},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":7.9459999999999997,"5min":7.9459999999999997,"15min":14.259},"last":12.583}]},{"osd":5,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":1.3420000000000001,"15min":1.083},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.24399999999999999},"max":{"1min":2.4700000000000002,"5min":43.826999999999998,"15min":43.826999999999998},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.038,"5min":1.2729999999999999,"15min":1.139},"min":{"1min":0.34899999999999998,"5min":0.32300000000000001,"15min":0.24399999999999999},"max":{"1min":5.9569999999999999,"5min":43.880000000000003,"15min":43.880000000000003},"last":0.31900000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":0.78600000000000003,"15min":0.97999999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.26600000000000001},"max":{"1min":2.468,"5min":3.8679999999999999,"15min":9.4540000000000006},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.90200000000000002,"5min":0.80600000000000005,"15min":1.018},"min":{"1min":0.33600000000000002,"5min":0.32000000000000001,"15min":0.30599999999999999},"max":{"1min":3.3279999999999998,"5min":4.516,"15min":6.2329999999999997},"last":0.55600000000000005}]}]},{"osd":6,"up_from":46,"seq":197568496070,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":907812,"kb_used_data":6172,"kb_used_omap":0,"kb_used_meta":901632,"kb_avail":92837340,"statfs":{"total":95995035648,"available":95065436160,"internally_reserved":0,"allocated":6320128,"data_stored":4753461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923271168},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:15:30 2023","interfaces":[{"interface":"back","average":{"1min":5.9260000000000002,"5min":9.6430000000000007,"15min":4.7089999999999996},"min":{"1min":0.30499999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":77.995000000000005,"5min":191.50800000000001,"15min":191.50800000000001},"last":2.1019999999999999},{"interface":"front","average":{"1min":6.4699999999999998,"5min":9.9670000000000005,"15min":4.6479999999999997},"min":{"1min":0.35199999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":77.983999999999995,"5min":191.52600000000001,"15min":191.52600000000001},"last":2.1339999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:15:30 2023","interfaces":[{"interface":"back","average":{"1min":4.1219999999999999,"5min":9.1660000000000004,"15min":6.2199999999999998},"min":{"1min":0.27200000000000002,"5min":0.254,"15min":0.23200000000000001},"max":{"1min":43.857999999999997,"5min":328.04399999999998,"15min":328.04399999999998},"last":1.0820000000000001},{"interface":"front","average":{"1min":1.254,"5min":8.5830000000000002,"15min":6.0949999999999998},"min":{"1min":0.26700000000000002,"5min":0.253,"15min":0.253},"max":{"1min":4.4299999999999997,"5min":328.10000000000002,"15min":328.10000000000002},"last":2.2770000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:15:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3240000000000001,"5min":4.6219999999999999,"15min":2.8940000000000001},"min":{"1min":0.314,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":7.6319999999999997,"5min":163.04900000000001,"15min":163.04900000000001},"last":0.59799999999999998},{"interface":"front","average":{"1min":2.1440000000000001,"5min":4.7309999999999999,"15min":2.8900000000000001},"min":{"1min":0.35799999999999998,"5min":0.31900000000000001,"15min":0.28599999999999998},"max":{"1min":9.7279999999999998,"5min":162.93000000000001,"15min":162.93000000000001},"last":0.46899999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:15:15 2023","interfaces":[{"interface":"back","average":{"1min":5.2990000000000004,"5min":9.1899999999999995,"15min":7.5970000000000004},"min":{"1min":0.316,"5min":0.316,"15min":0.307},"max":{"1min":48.134999999999998,"5min":250.744,"15min":250.744},"last":2.2440000000000002},{"interface":"front","average":{"1min":5.4299999999999997,"5min":8.6989999999999998,"15min":7.3659999999999997},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":48.097000000000001,"5min":250.83799999999999,"15min":250.83799999999999},"last":0.49399999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":1.018,"15min":1.6519999999999999},"min":{"1min":0.28899999999999998,"5min":0.24399999999999999,"15min":0.24199999999999999},"max":{"1min":5.4880000000000004,"5min":9.0869999999999997,"15min":151.35599999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.091,"5min":1.121,"15min":1.7350000000000001},"min":{"1min":0.307,"5min":0.249,"15min":0.23999999999999999},"max":{"1min":5.077,"5min":7.8849999999999998,"15min":149.15799999999999},"last":1.9850000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:15:30 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.751,"15min":1.0780000000000001},"min":{"1min":0.30399999999999999,"5min":0.28000000000000003,"15min":0.23000000000000001},"max":{"1min":2.1920000000000002,"5min":8.3829999999999991,"15min":15.342000000000001},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.6060000000000001,"5min":1.1180000000000001,"15min":1.1739999999999999},"min":{"1min":0.33000000000000002,"5min":0.28699999999999998,"15min":0.255},"max":{"1min":8.7289999999999992,"5min":8.9809999999999999,"15min":15.358000000000001},"last":0.63900000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:14:48 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":0.96999999999999997,"15min":1.238},"min":{"1min":0.33700000000000002,"5min":0.30299999999999999,"15min":0.27000000000000002},"max":{"1min":5.8380000000000001,"5min":5.8380000000000001,"15min":21.98},"last":3.8029999999999999},{"interface":"front","average":{"1min":1.01,"5min":0.86399999999999999,"15min":1.1739999999999999},"min":{"1min":0.34999999999999998,"5min":0.34200000000000003,"15min":0.33100000000000002},"max":{"1min":3.714,"5min":3.714,"15min":18.241},"last":3.7719999999999998}]}]},{"osd":1,"up_from":89,"seq":382252089657,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":687800,"kb_used_data":24944,"kb_used_omap":9,"kb_used_meta":662838,"kb_avail":93057352,"statfs":{"total":95995035648,"available":95290728448,"internally_reserved":0,"allocated":25542656,"data_stored":22306839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":678747113},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:14:39 2023","interfaces":[{"interface":"back","average":{"1min":4.2060000000000004,"5min":2.3319999999999999,"15min":2.294},"min":{"1min":0.221,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":35.939999999999998,"5min":35.939999999999998,"15min":35.939999999999998},"last":2.089},{"interface":"front","average":{"1min":4.3760000000000003,"5min":2.798,"15min":2.492},"min":{"1min":0.21199999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":35.326000000000001,"5min":35.326000000000001,"15min":35.326000000000001},"last":1.9419999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:14:45 2023","interfaces":[{"interface":"back","average":{"1min":1.2849999999999999,"5min":1.7929999999999999,"15min":2.4630000000000001},"min":{"1min":0.29899999999999999,"5min":0.217,"15min":0.20799999999999999},"max":{"1min":4.8929999999999998,"5min":15.522,"15min":129.053},"last":1.9610000000000001},{"interface":"front","average":{"1min":4.8550000000000004,"5min":2.9409999999999998,"15min":2.9350000000000001},"min":{"1min":0.32400000000000001,"5min":0.22600000000000001,"15min":0.222},"max":{"1min":55.694000000000003,"5min":55.694000000000003,"15min":129.077},"last":2.1349999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:15:13 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":4.3899999999999997,"15min":4.2939999999999996},"min":{"1min":0.318,"5min":0.26800000000000002,"15min":0.24199999999999999},"max":{"1min":12.961,"5min":113.322,"15min":142.97},"last":2.1269999999999998},{"interface":"front","average":{"1min":1.712,"5min":4.407,"15min":4.2590000000000003},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":4.5679999999999996,"5min":113.38200000000001,"15min":142.94800000000001},"last":2.1640000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:14:45 2023","interfaces":[{"interface":"back","average":{"1min":3.7559999999999998,"5min":3.4860000000000002,"15min":2.3149999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":54.018000000000001,"5min":93.747,"15min":93.747},"last":2.0390000000000001},{"interface":"front","average":{"1min":5.0860000000000003,"5min":4.4219999999999997,"15min":2.8319999999999999},"min":{"1min":0.35799999999999998,"5min":0.308,"15min":0.308},"max":{"1min":54.280000000000001,"5min":97.852000000000004,"15min":97.852000000000004},"last":0.437}]},{"osd":5,"last update":"Mon Oct 9 06:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.4730000000000001,"15min":1.5820000000000001},"min":{"1min":0.312,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.492,"5min":9.3109999999999999,"15min":36.356999999999999},"last":2.1040000000000001},{"interface":"front","average":{"1min":2.2639999999999998,"5min":2.5099999999999998,"15min":2.319},"min":{"1min":0.318,"5min":0.318,"15min":0.29799999999999999},"max":{"1min":14.648,"5min":17.603999999999999,"15min":36.268000000000001},"last":1.9750000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":1.6399999999999999,"15min":1.8180000000000001},"min":{"1min":0.36699999999999999,"5min":0.33700000000000002,"15min":0.318},"max":{"1min":5.2050000000000001,"5min":9.282,"15min":52.207000000000001},"last":2.0019999999999998},{"interface":"front","average":{"1min":2.0579999999999998,"5min":2.2810000000000001,"15min":2.375},"min":{"1min":0.36199999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":14.619999999999999,"5min":15.384,"15min":51.212000000000003},"last":2.153}]},{"osd":7,"last update":"Mon Oct 9 06:14:58 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":3.3919999999999999,"15min":2.984},"min":{"1min":0.39400000000000002,"5min":0.374,"15min":0.374},"max":{"1min":14.571,"5min":56.881,"15min":108.553},"last":2.1880000000000002},{"interface":"front","average":{"1min":1.4019999999999999,"5min":2.3900000000000001,"15min":2.4009999999999998},"min":{"1min":0.42999999999999999,"5min":0.38,"15min":0.38},"max":{"1min":5.9420000000000002,"5min":56.941000000000003,"15min":108.56999999999999},"last":2.0110000000000001}]}]},{"osd":0,"up_from":456,"seq":1958505087052,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":492348,"kb_used_data":194100,"kb_used_omap":32,"kb_used_meta":298143,"kb_avail":93252804,"statfs":{"total":95995035648,"available":95490871296,"internally_reserved":0,"allocated":198758400,"data_stored":195757850,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32794,"internal_metadata":305299430},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:14:40 2023","interfaces":[{"interface":"back","average":{"1min":15.137,"5min":6.5940000000000003,"15min":9.6199999999999992},"min":{"1min":0.24099999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":102.81999999999999,"5min":166.833,"15min":166.833},"last":0.34699999999999998},{"interface":"front","average":{"1min":14.031000000000001,"5min":6.9859999999999998,"15min":8.9900000000000002},"min":{"1min":0.28699999999999998,"5min":0.223,"15min":0.223},"max":{"1min":103.108,"5min":166.81100000000001,"15min":166.81100000000001},"last":0.32400000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:14:40 2023","interfaces":[{"interface":"back","average":{"1min":2.7400000000000002,"5min":2.649,"15min":3.589},"min":{"1min":0.30599999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":12.413,"5min":21.648,"15min":21.648},"last":0.38200000000000001},{"interface":"front","average":{"1min":2.21,"5min":2.3210000000000002,"15min":3.375},"min":{"1min":0.25700000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":10.119,"5min":20.152000000000001,"15min":20.152000000000001},"last":0.40899999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:15:36 2023","interfaces":[{"interface":"back","average":{"1min":2.6819999999999999,"5min":2.3559999999999999,"15min":11.401},"min":{"1min":0.40000000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":10.217000000000001,"5min":10.217000000000001,"15min":203.59299999999999},"last":1.724},{"interface":"front","average":{"1min":2.5739999999999998,"5min":2.8130000000000002,"15min":10.951000000000001},"min":{"1min":0.41299999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":9.5109999999999992,"5min":18.192,"15min":202.96600000000001},"last":1.7410000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:14:40 2023","interfaces":[{"interface":"back","average":{"1min":3.9289999999999998,"5min":2.3700000000000001,"15min":2.3959999999999999},"min":{"1min":0.39400000000000002,"5min":0.317,"15min":0.317},"max":{"1min":21.994,"5min":23.515999999999998,"15min":23.515999999999998},"last":0.52200000000000002},{"interface":"front","average":{"1min":3.3319999999999999,"5min":2.7839999999999998,"15min":2.2559999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":21.972999999999999,"5min":39.302,"15min":39.302},"last":0.42599999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:14:40 2023","interfaces":[{"interface":"back","average":{"1min":2.6179999999999999,"5min":3.8050000000000002,"15min":2.7280000000000002},"min":{"1min":0.32700000000000001,"5min":0.316,"15min":0.316},"max":{"1min":8.2949999999999999,"5min":161.286,"15min":161.286},"last":2.6970000000000001},{"interface":"front","average":{"1min":2.8010000000000002,"5min":3.597,"15min":2.8479999999999999},"min":{"1min":0.40000000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":12.038,"5min":161.31800000000001,"15min":161.31800000000001},"last":0.59499999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:14:40 2023","interfaces":[{"interface":"back","average":{"1min":4.657,"5min":2.5550000000000002,"15min":2.5030000000000001},"min":{"1min":0.43099999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":31.648,"5min":31.648,"15min":31.648},"last":0.502},{"interface":"front","average":{"1min":3.5910000000000002,"5min":2.081,"15min":2.585},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":24.43,"5min":24.43,"15min":24.43},"last":3.649}]},{"osd":7,"last update":"Mon Oct 9 06:14:40 2023","interfaces":[{"interface":"back","average":{"1min":2.859,"5min":3.5720000000000001,"15min":2.879},"min":{"1min":0.53700000000000003,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":12.066000000000001,"5min":123.685,"15min":123.685},"last":0.55800000000000005},{"interface":"front","average":{"1min":2.4089999999999998,"5min":3.1429999999999998,"15min":2.8889999999999998},"min":{"1min":0.55900000000000005,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":10.054,"5min":122.482,"15min":122.482},"last":0.628}]}]},{"osd":2,"up_from":94,"seq":403726926149,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":924824,"kb_used_data":141712,"kb_used_omap":81,"kb_used_meta":783022,"kb_avail":92820328,"statfs":{"total":95995035648,"available":95048015872,"internally_reserved":0,"allocated":145113088,"data_stored":142267529,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":801814746},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:14:41 2023","interfaces":[{"interface":"back","average":{"1min":7.6920000000000002,"5min":4.415,"15min":3.5710000000000002},"min":{"1min":0.35999999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":83.652000000000001,"5min":83.652000000000001,"15min":83.652000000000001},"last":0.436},{"interface":"front","average":{"1min":7.3620000000000001,"5min":3.8380000000000001,"15min":2.774},"min":{"1min":0.35399999999999998,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":77.578999999999994,"5min":77.578999999999994,"15min":77.578999999999994},"last":0.376}]},{"osd":1,"last update":"Mon Oct 9 06:14:41 2023","interfaces":[{"interface":"back","average":{"1min":7.0499999999999998,"5min":5.8970000000000002,"15min":4.9119999999999999},"min":{"1min":0.27300000000000002,"5min":0.248,"15min":0.19400000000000001},"max":{"1min":78.921999999999997,"5min":160.45500000000001,"15min":222.59399999999999},"last":0.93999999999999995},{"interface":"front","average":{"1min":8.5739999999999998,"5min":7.0979999999999999,"15min":5.4329999999999998},"min":{"1min":0.313,"5min":0.27200000000000002,"15min":0.20200000000000001},"max":{"1min":81.292000000000002,"5min":163.697,"15min":224.619},"last":3.5289999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:15:04 2023","interfaces":[{"interface":"back","average":{"1min":10.073,"5min":8.1189999999999998,"15min":5.0279999999999996},"min":{"1min":0.57099999999999995,"5min":0.23100000000000001,"15min":0.23000000000000001},"max":{"1min":78.385000000000005,"5min":122.205,"15min":122.205},"last":1.8320000000000001},{"interface":"front","average":{"1min":9.5399999999999991,"5min":7.7009999999999996,"15min":4.944},"min":{"1min":0.28599999999999998,"5min":0.252,"15min":0.252},"max":{"1min":78.436000000000007,"5min":122.45,"15min":122.45},"last":0.97599999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":2.8210000000000002,"5min":3.4449999999999998,"15min":2.2989999999999999},"min":{"1min":0.33600000000000002,"5min":0.32200000000000001,"15min":0.28299999999999997},"max":{"1min":9.3849999999999998,"5min":107.51300000000001,"15min":107.51300000000001},"last":8.5},{"interface":"front","average":{"1min":1.645,"5min":2.4580000000000002,"15min":2.0409999999999999},"min":{"1min":0.36299999999999999,"5min":0.34100000000000003,"15min":0.31},"max":{"1min":4.6589999999999998,"5min":108.768,"15min":108.768},"last":9.3759999999999994}]},{"osd":5,"last update":"Mon Oct 9 06:14:41 2023","interfaces":[{"interface":"back","average":{"1min":1.788,"5min":1.528,"15min":1.5569999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32600000000000001},"max":{"1min":11.108000000000001,"5min":11.108000000000001,"15min":28.48},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.8500000000000001,"5min":1.516,"15min":1.79},"min":{"1min":0.40200000000000002,"5min":0.316,"15min":0.28899999999999998},"max":{"1min":4.6890000000000001,"5min":9.8699999999999992,"15min":30.318999999999999},"last":0.42199999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:14:41 2023","interfaces":[{"interface":"back","average":{"1min":2.7890000000000001,"5min":2.407,"15min":2.1850000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.314},"max":{"1min":10.09,"5min":14.962,"15min":30.420000000000002},"last":0.45600000000000002},{"interface":"front","average":{"1min":2.1739999999999999,"5min":1.6619999999999999,"15min":1.7969999999999999},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.32200000000000001},"max":{"1min":6.3940000000000001,"5min":11.103,"15min":30.713000000000001},"last":1.5900000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:14:41 2023","interfaces":[{"interface":"back","average":{"1min":2.2469999999999999,"5min":2.919,"15min":2.2959999999999998},"min":{"1min":0.39400000000000002,"5min":0.313,"15min":0.313},"max":{"1min":11.147,"5min":124.069,"15min":124.069},"last":1.768},{"interface":"front","average":{"1min":2.3660000000000001,"5min":2.7170000000000001,"15min":2.3450000000000002},"min":{"1min":0.372,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.6769999999999996,"5min":124.08799999999999,"15min":124.08799999999999},"last":1.369}]}]},{"osd":3,"up_from":99,"seq":425201762622,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1302280,"kb_used_data":340608,"kb_used_omap":7,"kb_used_meta":961656,"kb_avail":92442872,"statfs":{"total":95995035648,"available":94661500928,"internally_reserved":0,"allocated":348782592,"data_stored":345585765,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":984736360},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:15:36 2023","interfaces":[{"interface":"back","average":{"1min":2.3849999999999998,"5min":5.7469999999999999,"15min":3.3210000000000002},"min":{"1min":0.186,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":9.0340000000000007,"5min":220.45400000000001,"15min":220.45400000000001},"last":0.313},{"interface":"front","average":{"1min":5.5250000000000004,"5min":6.899,"15min":4.6500000000000004},"min":{"1min":0.29799999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":49.292000000000002,"5min":216.44200000000001,"15min":216.44200000000001},"last":0.59899999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:15:10 2023","interfaces":[{"interface":"back","average":{"1min":3.6549999999999998,"5min":2.3039999999999998,"15min":3.1070000000000002},"min":{"1min":0.20300000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":30.632000000000001,"5min":30.632000000000001,"15min":207.75299999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":4.157,"5min":3.2519999999999998,"15min":4.1429999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.23200000000000001},"max":{"1min":30.465,"5min":30.524000000000001,"15min":207.12200000000001},"last":0.439}]},{"osd":2,"last update":"Mon Oct 9 06:15:10 2023","interfaces":[{"interface":"back","average":{"1min":2.3900000000000001,"5min":2.2530000000000001,"15min":2.6560000000000001},"min":{"1min":0.26000000000000001,"5min":0.23400000000000001,"15min":0.223},"max":{"1min":5.6189999999999998,"5min":30.547999999999998,"15min":107.01300000000001},"last":0.39400000000000002},{"interface":"front","average":{"1min":2.891,"5min":2.7189999999999999,"15min":3.1320000000000001},"min":{"1min":0.29699999999999999,"5min":0.25600000000000001,"15min":0.23899999999999999},"max":{"1min":11.542,"5min":25.984000000000002,"15min":109.807},"last":2.4710000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:14:50 2023","interfaces":[{"interface":"back","average":{"1min":1.7929999999999999,"5min":3.008,"15min":3.2029999999999998},"min":{"1min":0.29699999999999999,"5min":0.28199999999999997,"15min":0.254},"max":{"1min":12.523999999999999,"5min":95.739000000000004,"15min":99.876000000000005},"last":0.46100000000000002},{"interface":"front","average":{"1min":2.351,"5min":3.2970000000000002,"15min":3.4340000000000002},"min":{"1min":0.35299999999999998,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":11.787000000000001,"5min":97.941999999999993,"15min":99.914000000000001},"last":0.48499999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:15:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8939999999999999,"5min":2.8410000000000002,"15min":2.1019999999999999},"min":{"1min":0.32300000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.8799999999999999,"5min":74.706000000000003,"15min":74.706000000000003},"last":0.749},{"interface":"front","average":{"1min":1.9610000000000001,"5min":3.327,"15min":3.2719999999999998},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.28799999999999998},"max":{"1min":6.5529999999999999,"5min":74.685000000000002,"15min":235.56},"last":0.42599999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:15:10 2023","interfaces":[{"interface":"back","average":{"1min":1.774,"5min":1.4079999999999999,"15min":3.8399999999999999},"min":{"1min":0.35299999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":7.0430000000000001,"5min":7.0430000000000001,"15min":233.26599999999999},"last":1.806},{"interface":"front","average":{"1min":2.8290000000000002,"5min":2.226,"15min":4.3419999999999996},"min":{"1min":0.40000000000000002,"5min":0.35999999999999999,"15min":0.33600000000000002},"max":{"1min":8.3360000000000003,"5min":26.890000000000001,"15min":233.24100000000001},"last":1.792}]},{"osd":7,"last update":"Mon Oct 9 06:15:10 2023","interfaces":[{"interface":"back","average":{"1min":2.5209999999999999,"5min":4.2599999999999998,"15min":4.6769999999999996},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.35699999999999998},"max":{"1min":7.0060000000000002,"5min":94.400000000000006,"15min":230.69399999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":2.0859999999999999,"5min":4.274,"15min":4.8449999999999998},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.32600000000000001},"max":{"1min":11.311,"5min":94.195999999999998,"15min":230.67500000000001},"last":0.71199999999999997}]}]},{"osd":4,"up_from":322,"seq":1382979469485,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":917356,"kb_used_data":294308,"kb_used_omap":35,"kb_used_meta":623004,"kb_avail":92827796,"statfs":{"total":95995035648,"available":95055663104,"internally_reserved":0,"allocated":301371392,"data_stored":299788508,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36552,"internal_metadata":637956408},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:15:32 2023","interfaces":[{"interface":"back","average":{"1min":6.8639999999999999,"5min":3.048,"15min":2.008},"min":{"1min":0.29399999999999998,"5min":0.216,"15min":0.216},"max":{"1min":81.066000000000003,"5min":81.066000000000003,"15min":81.066000000000003},"last":1.321},{"interface":"front","average":{"1min":7.008,"5min":2.843,"15min":2.012},"min":{"1min":0.28100000000000003,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":81.603999999999999,"5min":81.603999999999999,"15min":81.603999999999999},"last":0.76700000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":1.77,"5min":2.0489999999999999,"15min":2.2770000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.20999999999999999},"max":{"1min":4.8360000000000003,"5min":18.884,"15min":78.891000000000005},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.905,"5min":1.802,"15min":2.2559999999999998},"min":{"1min":0.32200000000000001,"5min":0.25600000000000001,"15min":0.22800000000000001},"max":{"1min":10.039,"5min":18.617999999999999,"15min":78.921000000000006},"last":1.3460000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":2.9929999999999999,"5min":4.5960000000000001,"15min":2.6869999999999998},"min":{"1min":0.39200000000000002,"5min":0.29199999999999998,"15min":0.27200000000000002},"max":{"1min":10.084,"5min":245.166,"15min":245.166},"last":1.4770000000000001},{"interface":"front","average":{"1min":1.845,"5min":4.3230000000000004,"15min":2.5329999999999999},"min":{"1min":0.28699999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.2080000000000002,"5min":245.119,"15min":245.119},"last":0.501}]},{"osd":3,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":3.601,"5min":6.8630000000000004,"15min":3.9740000000000002},"min":{"1min":0.34599999999999997,"5min":0.29399999999999998,"15min":0.26800000000000002},"max":{"1min":20.100999999999999,"5min":246.39400000000001,"15min":246.39400000000001},"last":1.2709999999999999},{"interface":"front","average":{"1min":3.7240000000000002,"5min":6.3700000000000001,"15min":3.5750000000000002},"min":{"1min":0.40899999999999997,"5min":0.314,"15min":0.314},"max":{"1min":19.984000000000002,"5min":246.99000000000001,"15min":246.99000000000001},"last":9.2970000000000006}]},{"osd":5,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":6.9269999999999996,"5min":2.0880000000000001,"15min":1.3899999999999999},"min":{"1min":0.28899999999999998,"5min":0.23499999999999999,"15min":0.216},"max":{"1min":99.405000000000001,"5min":99.405000000000001,"15min":99.405000000000001},"last":0.45000000000000001},{"interface":"front","average":{"1min":7.2169999999999996,"5min":2.274,"15min":1.546},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.245},"max":{"1min":99.206000000000003,"5min":99.206000000000003,"15min":99.206000000000003},"last":0.51700000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.0569999999999999,"15min":2.0859999999999999},"min":{"1min":0.32200000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.9900000000000002,"5min":7.641,"15min":95.492999999999995},"last":0.872},{"interface":"front","average":{"1min":1.7070000000000001,"5min":1.3300000000000001,"15min":2.25},"min":{"1min":0.33200000000000002,"5min":0.32100000000000001,"15min":0.27900000000000003},"max":{"1min":9.2110000000000003,"5min":9.3239999999999998,"15min":95.664000000000001},"last":0.59199999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:14:49 2023","interfaces":[{"interface":"back","average":{"1min":6.9320000000000004,"5min":2.5390000000000001,"15min":2.7200000000000002},"min":{"1min":0.38800000000000001,"5min":0.29699999999999999,"15min":0.29499999999999998},"max":{"1min":95.167000000000002,"5min":95.167000000000002,"15min":95.167000000000002},"last":0.61899999999999999},{"interface":"front","average":{"1min":6.8949999999999996,"5min":2.2599999999999998,"15min":2.403},"min":{"1min":0.34499999999999997,"5min":0.27800000000000002,"15min":0.251},"max":{"1min":94.811999999999998,"5min":94.811999999999998,"15min":94.811999999999998},"last":1.375}]}]},{"osd":5,"up_from":38,"seq":163208757706,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":989668,"kb_used_data":95260,"kb_used_omap":0,"kb_used_meta":894400,"kb_avail":92755484,"statfs":{"total":95995035648,"available":94981615616,"internally_reserved":0,"allocated":97546240,"data_stored":95975971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915865600},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:15:36 2023","interfaces":[{"interface":"back","average":{"1min":6.0659999999999998,"5min":4.2839999999999998,"15min":3.0289999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":63.948999999999998,"5min":115.92400000000001,"15min":115.92400000000001},"last":0.29099999999999998},{"interface":"front","average":{"1min":5.7629999999999999,"5min":4.391,"15min":3.613},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":64.102000000000004,"5min":115.904,"15min":115.904},"last":0.34100000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":3.734,"15min":4.4800000000000004},"min":{"1min":0.253,"5min":0.253,"15min":0.22600000000000001},"max":{"1min":4.0369999999999999,"5min":151.78200000000001,"15min":151.78200000000001},"last":0.27200000000000002},{"interface":"front","average":{"1min":1.097,"5min":3.8730000000000002,"15min":4.6319999999999997},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.3319999999999999,"5min":152.251,"15min":152.251},"last":0.40500000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:15:29 2023","interfaces":[{"interface":"back","average":{"1min":2.1400000000000001,"5min":2.98,"15min":2.8260000000000001},"min":{"1min":0.28599999999999998,"5min":0.26600000000000001,"15min":0.254},"max":{"1min":6.8700000000000001,"5min":117.97199999999999,"15min":129.21199999999999},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.2050000000000001,"5min":3.0710000000000002,"15min":2.8999999999999999},"min":{"1min":0.27400000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.8690000000000002,"5min":118.52,"15min":129.042},"last":0.373}]},{"osd":3,"last update":"Mon Oct 9 06:15:04 2023","interfaces":[{"interface":"back","average":{"1min":3.2639999999999998,"5min":5.2110000000000003,"15min":7.0629999999999997},"min":{"1min":0.38700000000000001,"5min":0.312,"15min":0.29499999999999998},"max":{"1min":19.231000000000002,"5min":111.072,"15min":143.31200000000001},"last":0.92600000000000005},{"interface":"front","average":{"1min":3.2879999999999998,"5min":5.6219999999999999,"15min":7.1369999999999996},"min":{"1min":0.36299999999999999,"5min":0.34499999999999997,"15min":0.29399999999999998},"max":{"1min":20.035,"5min":111.01300000000001,"15min":140.00899999999999},"last":0.33500000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:14:52 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":1.6279999999999999,"15min":1.278},"min":{"1min":0.28000000000000003,"5min":0.246,"15min":0.245},"max":{"1min":2.097,"5min":68.001999999999995,"15min":68.001999999999995},"last":193.25200000000001},{"interface":"front","average":{"1min":0.88300000000000001,"5min":1.5529999999999999,"15min":1.212},"min":{"1min":0.33900000000000002,"5min":0.26100000000000001,"15min":0.26000000000000001},"max":{"1min":5.7729999999999997,"5min":61.743000000000002,"15min":61.743000000000002},"last":190.06800000000001}]},{"osd":6,"last update":"Mon Oct 9 06:15:04 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.91000000000000003,"15min":1.0940000000000001},"min":{"1min":0.314,"5min":0.26200000000000001,"15min":0.251},"max":{"1min":4.492,"5min":7.4980000000000002,"15min":9.0909999999999993},"last":0.317},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.014,"15min":1.1639999999999999},"min":{"1min":0.34599999999999997,"5min":0.29099999999999998,"15min":0.26700000000000002},"max":{"1min":3.0379999999999998,"5min":7.7140000000000004,"15min":9.9009999999999998},"last":0.35499999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:14:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3480000000000001,"5min":1.8380000000000001,"15min":2.456},"min":{"1min":0.36299999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.8289999999999997,"5min":64.766000000000005,"15min":157.471},"last":180.696},{"interface":"front","average":{"1min":0.85699999999999998,"5min":1.698,"15min":2.3410000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.3380000000000001,"5min":67.956999999999994,"15min":157.51599999999999},"last":180.74000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4889,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":152637440,"data_stored":152633368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":17170432,"data_stored":17170432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":126418944,"data_stored":126418944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":323223552,"data_stored":323223552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":283316224,"data_stored":283312152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":89325568,"data_stored":89325568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:15:47.444 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_4 pgp_num 16 2023-10-09T06:15:47.534 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:15:47.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:47.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:15:47.639 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:15:47.670 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:47.686 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:15:47.715 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:15:47.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:47.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:15:47.820 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:15:47.848 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:48.060 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:15:48.063 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:15:48.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:48.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:15:48.168 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:15:48.194 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:48.202 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:48.288 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:15:48.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:48.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:15:48.394 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:15:48.428 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:48.578 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:15:48.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:48.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:15:48.681 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:15:48.708 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:48.750 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:15:48.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:48.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:15:48.855 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:15:48.886 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:48.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:48 smithi064 ceph-mon[212750]: pgmap v1444: 119 pgs: 119 active+clean; 582 KiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-10-09T06:15:48.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:48 smithi064 ceph-mon[220044]: pgmap v1444: 119 pgs: 119 active+clean; 582 KiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-10-09T06:15:49.056 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:48 smithi156 ceph-mon[195101]: pgmap v1444: 119 pgs: 119 active+clean; 582 KiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-10-09T06:15:49.087 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:15:49.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:49.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:15:49.191 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:15:49.219 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:49.315 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:15:49.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:49.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:15:49.419 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:15:49.448 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:49.467 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:15:49.488 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:15:49.488 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:15:49.488 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 0 2023-10-09T06:15:49.488 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:15:49.518 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:15:49.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:49.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:15:49.622 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:15:49.650 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:49.747 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:15:49.747 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:15:49.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:49.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:15:49.851 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:15:49.888 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:50.036 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:15:50.038 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:49 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2577984082' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:15:50.038 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:49 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2577984082' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:15:50.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:50.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:15:50.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:49 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2577984082' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:15:50.140 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:15:50.168 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:50.175 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:15:50.211 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:15:50.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:50.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:15:50.318 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:15:50.346 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:50.390 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:15:50.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:50.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:15:50.497 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:15:50.525 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:50.571 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:15:50.621 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:15:50.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:50.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:15:50.687 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:50.727 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:15:50.755 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:50.758 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 10 pgp_num to 16 2023-10-09T06:15:50.784 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:50 smithi156 ceph-mon[195101]: pgmap v1445: 119 pgs: 119 active+clean; 582 KiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-10-09T06:15:50.785 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:50 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:15:50.785 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:50 smithi156 ceph-mon[195101]: failed to encode map e568 with expected crc 2023-10-09T06:15:50.785 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:50 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2577984082' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:15:50.785 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:50 smithi156 ceph-mon[195101]: osdmap e568: 8 total, 8 up, 8 in 2023-10-09T06:15:50.785 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:50 smithi156 ceph-mon[195101]: failed to encode map e568 with expected crc 2023-10-09T06:15:50.785 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:50 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2904363219' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:15:50.807 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:15:50.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:50.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:15:50.910 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:15:50.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:50 smithi064 ceph-mon[212750]: pgmap v1445: 119 pgs: 119 active+clean; 582 KiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-10-09T06:15:50.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:50 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:15:50.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:50 smithi064 ceph-mon[212750]: failed to encode map e568 with expected crc 2023-10-09T06:15:50.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:50 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2577984082' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:15:50.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:50 smithi064 ceph-mon[212750]: osdmap e568: 8 total, 8 up, 8 in 2023-10-09T06:15:50.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:50 smithi064 ceph-mon[212750]: failed to encode map e568 with expected crc 2023-10-09T06:15:50.937 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:50 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2904363219' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:15:50.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:50 smithi064 ceph-mon[220044]: pgmap v1445: 119 pgs: 119 active+clean; 582 KiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-10-09T06:15:50.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:50 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:15:50.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:50 smithi064 ceph-mon[220044]: failed to encode map e568 with expected crc 2023-10-09T06:15:50.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:50 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2577984082' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:15:50.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:50 smithi064 ceph-mon[220044]: osdmap e568: 8 total, 8 up, 8 in 2023-10-09T06:15:50.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:50 smithi064 ceph-mon[220044]: failed to encode map e568 with expected crc 2023-10-09T06:15:50.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:50 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2904363219' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:15:50.939 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:51.040 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:15:51.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:51.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:15:51.144 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:15:51.173 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:51.584 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-10-09T06:15:51.584 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-10-09T06:15:51.587 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:15:51.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:51.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:15:51.693 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:15:51.721 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:51.817 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:51 smithi156 ceph-mon[195101]: failed to encode map e568 with expected crc 2023-10-09T06:15:51.817 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:51 smithi156 ceph-mon[195101]: failed to encode map e568 with expected crc 2023-10-09T06:15:51.817 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:51 smithi156 ceph-mon[195101]: failed to encode map e569 with expected crc 2023-10-09T06:15:51.817 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:51 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2904363219' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T06:15:51.817 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:51 smithi156 ceph-mon[195101]: osdmap e569: 8 total, 8 up, 8 in 2023-10-09T06:15:51.828 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:51 smithi064 ceph-mon[212750]: failed to encode map e568 with expected crc 2023-10-09T06:15:51.829 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:51 smithi064 ceph-mon[212750]: failed to encode map e568 with expected crc 2023-10-09T06:15:51.829 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:51 smithi064 ceph-mon[212750]: failed to encode map e569 with expected crc 2023-10-09T06:15:51.829 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:51 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2904363219' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T06:15:51.829 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:51 smithi064 ceph-mon[212750]: osdmap e569: 8 total, 8 up, 8 in 2023-10-09T06:15:51.829 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:51 smithi064 ceph-mon[220044]: failed to encode map e568 with expected crc 2023-10-09T06:15:51.829 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:51 smithi064 ceph-mon[220044]: failed to encode map e568 with expected crc 2023-10-09T06:15:51.829 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:51 smithi064 ceph-mon[220044]: failed to encode map e569 with expected crc 2023-10-09T06:15:51.829 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:51 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2904363219' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T06:15:51.829 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:51 smithi064 ceph-mon[220044]: osdmap e569: 8 total, 8 up, 8 in 2023-10-09T06:15:51.871 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:15:51.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:51.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:15:51.975 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:15:52.002 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:52.028 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:15:52.029 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:15:52.029 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T06:15:52.029 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T06:15:52.029 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:15:52.247 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:15:52.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:52.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:15:52.350 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:15:52.381 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:52.381 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:52.431 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:15:52.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:52.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:15:52.535 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:15:52.565 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:52.612 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:15:52.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:52.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:15:52.652 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:15:52.710 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:15:52.716 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:15:52.780 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:52.827 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:15:52.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:52.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:15:52.851 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:15:52.874 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:15:52.887 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:52 smithi064 ceph-mon[220044]: pgmap v1448: 119 pgs: 119 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 261 op/s 2023-10-09T06:15:52.887 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:52 smithi064 ceph-mon[220044]: failed to encode map e569 with expected crc 2023-10-09T06:15:52.887 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:52 smithi064 ceph-mon[220044]: failed to encode map e569 with expected crc 2023-10-09T06:15:52.888 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:52 smithi064 ceph-mon[212750]: pgmap v1448: 119 pgs: 119 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 261 op/s 2023-10-09T06:15:52.888 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:52 smithi064 ceph-mon[212750]: failed to encode map e569 with expected crc 2023-10-09T06:15:52.888 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:52 smithi064 ceph-mon[212750]: failed to encode map e569 with expected crc 2023-10-09T06:15:52.898 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:15:52.922 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:15:52.932 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:15:52.948 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:15:52.956 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:52 smithi156 ceph-mon[195101]: pgmap v1448: 119 pgs: 119 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 261 op/s 2023-10-09T06:15:52.956 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:52 smithi156 ceph-mon[195101]: failed to encode map e569 with expected crc 2023-10-09T06:15:52.956 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:52 smithi156 ceph-mon[195101]: failed to encode map e569 with expected crc 2023-10-09T06:15:52.959 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:52.968 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:15:52.987 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:15:53.006 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:15:53.008 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:15:53.008 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:15:53.008 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:15:53.008 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:15:53.008 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:15:53.008 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:15:53.008 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:15:53.008 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:15:53.054 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:15:53.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:53.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:15:53.074 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:15:53.167 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:15:53.198 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:53.296 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:15:53.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:53.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:15:53.400 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:15:53.427 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:53.521 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:15:53.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:53.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:15:53.625 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:15:53.657 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:53.700 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:15:53.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:53.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:15:53.791 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:53.804 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:15:53.830 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:53.872 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:15:53.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:53.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:15:53.976 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:15:54.004 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:54.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:53 smithi156 ceph-mon[195101]: failed to encode map e569 with expected crc 2023-10-09T06:15:54.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:53 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4276588651' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:15:54.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:53 smithi064 ceph-mon[212750]: failed to encode map e569 with expected crc 2023-10-09T06:15:54.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:53 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4276588651' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:15:54.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:53 smithi064 ceph-mon[220044]: failed to encode map e569 with expected crc 2023-10-09T06:15:54.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:53 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4276588651' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:15:54.271 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:15:54.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:54.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:15:54.375 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:15:54.404 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:54.463 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:15:54.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:54.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:15:54.568 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:15:54.585 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:15:54.585 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:15:54.599 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:54.748 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:15:54.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:54.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:15:54.851 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:15:54.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:54 smithi156 ceph-mon[195101]: pgmap v1449: 119 pgs: 119 active+clean; 582 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 179 op/s 2023-10-09T06:15:54.878 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:54 smithi064 ceph-mon[212750]: pgmap v1449: 119 pgs: 119 active+clean; 582 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 179 op/s 2023-10-09T06:15:54.879 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:54 smithi064 ceph-mon[220044]: pgmap v1449: 119 pgs: 119 active+clean; 582 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 179 op/s 2023-10-09T06:15:54.882 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:54.936 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:15:54.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:54.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:15:55.040 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:15:55.068 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:55.114 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:15:55.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:55.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:15:55.219 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:15:55.250 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:15:55.252 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:55.297 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:15:55.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:55.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:15:55.402 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:15:55.429 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:55.476 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:15:55.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:55.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:15:55.582 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:15:55.605 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:15:55.609 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:55.717 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:15:55.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:15:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:15:55] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.43.0" 2023-10-09T06:15:55.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:55.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:15:55.775 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_4' removed 2023-10-09T06:15:55.820 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:15:55.821 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:15:55.866 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:55 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2620742930' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:15:55.879 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:55.967 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:15:55.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:55.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:15:56.077 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:15:56.080 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:55 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2620742930' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:15:56.080 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:55 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2620742930' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:15:56.110 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:56.481 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:15:56.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:56.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:15:56.565 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T06:15:56.566 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T06:15:56.566 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-10-09T06:15:56.566 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-10-09T06:15:56.585 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1, 2] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 4, 0] 2023-10-09T06:15:56.585 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:15:56.585 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [0, 3, 4, 5, 6, 7, 1, 2] 2023-10-09T06:15:56.585 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd out 2 2023-10-09T06:15:56.586 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:15:56.614 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:56.664 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:56.738 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:15:56.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:56.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:15:56.780 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:15:56.844 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:15:56.869 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:56 smithi156 ceph-mon[195101]: pgmap v1450: 119 pgs: 119 active+clean; 582 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:15:56.869 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:56 smithi156 ceph-mon[195101]: failed to encode map e570 with expected crc 2023-10-09T06:15:56.869 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:56 smithi156 ceph-mon[195101]: failed to encode map e570 with expected crc 2023-10-09T06:15:56.869 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:56 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2620742930' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:15:56.869 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:56 smithi156 ceph-mon[195101]: osdmap e570: 8 total, 8 up, 8 in 2023-10-09T06:15:56.869 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:56 smithi156 ceph-mon[195101]: failed to encode map e570 with expected crc 2023-10-09T06:15:56.869 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:56 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/415016262' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:15:56.872 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:57.018 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:15:57.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:57.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:15:57.121 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:15:57.129 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:56 smithi064 ceph-mon[212750]: pgmap v1450: 119 pgs: 119 active+clean; 582 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:15:57.129 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:56 smithi064 ceph-mon[212750]: failed to encode map e570 with expected crc 2023-10-09T06:15:57.129 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:56 smithi064 ceph-mon[212750]: failed to encode map e570 with expected crc 2023-10-09T06:15:57.129 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:56 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2620742930' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:15:57.129 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:56 smithi064 ceph-mon[212750]: osdmap e570: 8 total, 8 up, 8 in 2023-10-09T06:15:57.129 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:56 smithi064 ceph-mon[212750]: failed to encode map e570 with expected crc 2023-10-09T06:15:57.129 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:56 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/415016262' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:15:57.129 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:56 smithi064 ceph-mon[220044]: pgmap v1450: 119 pgs: 119 active+clean; 582 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:15:57.130 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:56 smithi064 ceph-mon[220044]: failed to encode map e570 with expected crc 2023-10-09T06:15:57.130 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:56 smithi064 ceph-mon[220044]: failed to encode map e570 with expected crc 2023-10-09T06:15:57.130 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:56 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2620742930' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:15:57.130 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:56 smithi064 ceph-mon[220044]: osdmap e570: 8 total, 8 up, 8 in 2023-10-09T06:15:57.130 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:56 smithi064 ceph-mon[220044]: failed to encode map e570 with expected crc 2023-10-09T06:15:57.130 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:56 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/415016262' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:15:57.152 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:57.197 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:15:57.198 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:15:57.969 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:15:57.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:57 smithi064 ceph-mon[212750]: failed to encode map e570 with expected crc 2023-10-09T06:15:57.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:57 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:15:57.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:57 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:15:57.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:57 smithi064 ceph-mon[212750]: failed to encode map e571 with expected crc 2023-10-09T06:15:57.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:57 smithi064 ceph-mon[212750]: failed to encode map e571 with expected crc 2023-10-09T06:15:57.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:57 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/415016262' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:15:57.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:57 smithi064 ceph-mon[212750]: osdmap e571: 8 total, 8 up, 8 in 2023-10-09T06:15:57.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:57 smithi064 ceph-mon[212750]: failed to encode map e571 with expected crc 2023-10-09T06:15:57.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:57 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:15:57.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:57 smithi064 ceph-mon[220044]: failed to encode map e570 with expected crc 2023-10-09T06:15:57.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:57 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:15:57.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:57 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T06:15:57.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:57 smithi064 ceph-mon[220044]: failed to encode map e571 with expected crc 2023-10-09T06:15:57.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:57 smithi064 ceph-mon[220044]: failed to encode map e571 with expected crc 2023-10-09T06:15:57.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:57 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/415016262' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:15:57.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:57 smithi064 ceph-mon[220044]: osdmap e571: 8 total, 8 up, 8 in 2023-10-09T06:15:57.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:57 smithi064 ceph-mon[220044]: failed to encode map e571 with expected crc 2023-10-09T06:15:57.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:57 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:15:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:57 smithi156 ceph-mon[195101]: failed to encode map e570 with expected crc 2023-10-09T06:15:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:57 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:15:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:57 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T06:15:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:57 smithi156 ceph-mon[195101]: failed to encode map e571 with expected crc 2023-10-09T06:15:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:57 smithi156 ceph-mon[195101]: failed to encode map e571 with expected crc 2023-10-09T06:15:58.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:57 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/415016262' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:15:58.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:57 smithi156 ceph-mon[195101]: osdmap e571: 8 total, 8 up, 8 in 2023-10-09T06:15:58.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:57 smithi156 ceph-mon[195101]: failed to encode map e571 with expected crc 2023-10-09T06:15:58.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:57 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:15:58.132 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:15:58.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:58.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:15:58.237 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:15:58.267 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:58.310 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:58.312 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:15:58.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:58.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:15:58.334 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:15:58.367 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:58.418 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:15:58.444 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:58.487 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:15:58.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:58.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:15:58.591 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:15:58.622 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:58.778 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:15:58.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:58.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:15:58.801 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:15:58.830 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:15:58.849 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:15:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:58 smithi156 ceph-mon[195101]: failed to encode map e571 with expected crc 2023-10-09T06:15:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:58 smithi156 ceph-mon[195101]: pgmap v1453: 103 pgs: 103 active+clean; 582 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:15:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:58 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:58 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:58 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:58 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:58 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:58 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:58.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:15:58 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3519837070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:15:58.872 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:15:58.886 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:15:58.891 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:15:58.914 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:15:58.918 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:15:58.936 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:15:58.960 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:15:58.962 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:15:58.962 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:15:58.962 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:15:58.962 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:15:58.962 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:15:58.962 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:15:58.962 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:15:58.962 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:15:59.185 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:58 smithi064 ceph-mon[212750]: failed to encode map e571 with expected crc 2023-10-09T06:15:59.185 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:58 smithi064 ceph-mon[212750]: pgmap v1453: 103 pgs: 103 active+clean; 582 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:15:59.185 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:58 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:59.185 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:58 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:59.185 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:58 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:59.185 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:58 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:59.185 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:58 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:59.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:59.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:15:58 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3519837070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:15:59.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:58 smithi064 ceph-mon[220044]: failed to encode map e571 with expected crc 2023-10-09T06:15:59.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:58 smithi064 ceph-mon[220044]: pgmap v1453: 103 pgs: 103 active+clean; 582 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:15:59.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:58 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:59.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:58 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:59.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:58 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:59.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:58 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:59.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:58 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:15:59.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:58 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:59.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:15:58 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3519837070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:15:59.892 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:15:59.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:15:59.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:15:59.997 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:16:00.030 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:00.035 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:16:00.036 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:16:00.036 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T06:16:00.036 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T06:16:00.036 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:16:00.128 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:16:00.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:00.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:16:00.158 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:16:00.232 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:16:00.272 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:00.593 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:16:00.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:00.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:16:00.696 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:16:00.734 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:00.793 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:16:00.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:00.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:16:00.850 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:00 smithi064 ceph-mon[212750]: pgmap v1454: 103 pgs: 103 active+clean; 582 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:16:00.850 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:00 smithi064 ceph-mon[220044]: pgmap v1454: 103 pgs: 103 active+clean; 582 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:16:00.898 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:16:00.922 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:16:00.928 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:00.978 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:16:00.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:00.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:16:01.085 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:16:01.110 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:00 smithi156 ceph-mon[195101]: pgmap v1454: 103 pgs: 103 active+clean; 582 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:16:01.113 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:01.159 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:16:01.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:01.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:16:01.264 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:16:01.294 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:01.336 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:16:01.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:01.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:16:01.442 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:16:01.443 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:16:01.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:01 smithi064 conmon[212726]: 2023-10-09T06:16:01.378+0000 7f8823f23700 -1 mon.a@0(leader).osd e571 definitely_dead 0 2023-10-09T06:16:01.472 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:01.517 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:01.566 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:16:01.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:01.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:16:01.671 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:01.702 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:01.747 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:01.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:01.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:01.851 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:16:01.879 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:01.966 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:16:02.230 INFO:teuthology.orchestra.run.smithi064.stderr:marked out osd.2. 2023-10-09T06:16:02.234 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_5' created 2023-10-09T06:16:02.263 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:02 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/354855071' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T06:16:02.263 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:02 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T06:16:02.263 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:02 smithi064 ceph-mon[212750]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T06:16:02.263 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:02 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/354855071' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T06:16:02.263 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:02 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T06:16:02.263 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:02 smithi064 ceph-mon[220044]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T06:16:02.580 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:16:02.582 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:02 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/354855071' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T06:16:02.582 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:02 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T06:16:02.582 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:02 smithi156 ceph-mon[195101]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T06:16:02.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:02.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:16:02.684 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:02.711 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:02.826 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:02.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:02.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:02.932 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:16:02.962 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:03.417 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:16:03.418 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T06:16:03.418 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T06:16:03.418 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:16:03.418 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.418 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:58018 9.5 9:ad237675:::benchmark_data_smithi064_511911_object58017:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2023-10-09T06:16:03.418 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:07:50.684831+0000", 2023-10-09T06:16:03.418 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 492.729653208, 2023-10-09T06:16:03.418 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0269654070000001, 2023-10-09T06:16:03.418 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:16:03.419 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:16:03.419 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:16:03.419 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:16:03.419 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:16:03.419 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 58018 2023-10-09T06:16:03.419 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.419 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:16:03.419 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.419 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:16:03.419 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.684831+0000", 2023-10-09T06:16:03.419 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.420 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.420 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.420 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:16:03.420 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.684831+0000", 2023-10-09T06:16:03.420 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.420 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.420 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.420 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:16:03.420 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.684830+0000", 2023-10-09T06:16:03.420 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:16:03.420 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.421 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.421 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:16:03.421 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.684922+0000", 2023-10-09T06:16:03.421 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2584000000000007e-05 2023-10-09T06:16:03.421 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.421 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.421 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:16:03.421 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.684946+0000", 2023-10-09T06:16:03.421 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3587e-05 2023-10-09T06:16:03.422 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.422 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.422 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:16:03.422 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.684966+0000", 2023-10-09T06:16:03.422 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0222000000000001e-05 2023-10-09T06:16:03.422 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.422 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.422 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.422 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.685010+0000", 2023-10-09T06:16:03.422 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3668999999999998e-05 2023-10-09T06:16:03.423 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.423 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.423 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:16:03.423 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.685167+0000", 2023-10-09T06:16:03.423 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000157116 2023-10-09T06:16:03.423 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.423 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.423 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:16:03.423 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.685324+0000", 2023-10-09T06:16:03.423 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015695100000000001 2023-10-09T06:16:03.423 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.423 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.424 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:16:03.424 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.689172+0000", 2023-10-09T06:16:03.424 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0038483359999999999 2023-10-09T06:16:03.424 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.424 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.424 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.424 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.689351+0000", 2023-10-09T06:16:03.424 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00017877 2023-10-09T06:16:03.424 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.424 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.424 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.424 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.709096+0000", 2023-10-09T06:16:03.425 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.019744817000000001 2023-10-09T06:16:03.425 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.425 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.425 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:16:03.425 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.709105+0000", 2023-10-09T06:16:03.425 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9209999999999998e-06 2023-10-09T06:16:03.425 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.425 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.425 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.425 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.711724+0000", 2023-10-09T06:16:03.425 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0026194079999999 2023-10-09T06:16:03.425 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.426 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.426 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:16:03.426 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.711797+0000", 2023-10-09T06:16:03.426 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.237e-05 2023-10-09T06:16:03.426 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.426 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:16:03.426 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.426 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.426 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.426 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:58040 9.9 9:949f63c6:::benchmark_data_smithi064_511911_object58039:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2023-10-09T06:16:03.426 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:07:50.711513+0000", 2023-10-09T06:16:03.426 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 492.70297178099997, 2023-10-09T06:16:03.427 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0069264769999999, 2023-10-09T06:16:03.427 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:16:03.427 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:16:03.427 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:16:03.427 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:16:03.427 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:16:03.427 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 58040 2023-10-09T06:16:03.427 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.427 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:16:03.427 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.427 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:16:03.427 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.711513+0000", 2023-10-09T06:16:03.428 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.428 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.428 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.428 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:16:03.428 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.711513+0000", 2023-10-09T06:16:03.428 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.428 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.428 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.428 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:16:03.428 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.711512+0000", 2023-10-09T06:16:03.428 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:16:03.428 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.429 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.429 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:16:03.429 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.711542+0000", 2023-10-09T06:16:03.429 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9875000000000001e-05 2023-10-09T06:16:03.429 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.429 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.429 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:16:03.429 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.711548+0000", 2023-10-09T06:16:03.429 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.28e-06 2023-10-09T06:16:03.429 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.429 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.429 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:16:03.430 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.711552+0000", 2023-10-09T06:16:03.430 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6109999999999998e-06 2023-10-09T06:16:03.430 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.430 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.430 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.430 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.716882+0000", 2023-10-09T06:16:03.430 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0053301729999999997 2023-10-09T06:16:03.430 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.430 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.430 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:16:03.430 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.716889+0000", 2023-10-09T06:16:03.430 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7610000000000002e-06 2023-10-09T06:16:03.431 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.431 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.431 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.431 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.709263+0000", 2023-10-09T06:16:03.431 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99237410599999998 2023-10-09T06:16:03.431 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.431 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.431 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:16:03.431 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.709402+0000", 2023-10-09T06:16:03.431 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013855700000000001 2023-10-09T06:16:03.431 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.431 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.432 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5", 2023-10-09T06:16:03.432 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.709567+0000", 2023-10-09T06:16:03.432 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00016538100000000001 2023-10-09T06:16:03.432 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.432 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.432 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:16:03.432 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.716242+0000", 2023-10-09T06:16:03.432 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0066750159999999998 2023-10-09T06:16:03.432 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.432 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.432 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.432 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718290+0000", 2023-10-09T06:16:03.433 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0020477130000000001 2023-10-09T06:16:03.433 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.433 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.433 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:16:03.433 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718340+0000", 2023-10-09T06:16:03.433 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0121999999999999e-05 2023-10-09T06:16:03.433 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.433 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.433 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:16:03.433 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718439+0000", 2023-10-09T06:16:03.433 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.9265999999999995e-05 2023-10-09T06:16:03.433 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.434 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:16:03.434 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.434 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.434 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.434 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:58064 9.9 9:972c46fb:::benchmark_data_smithi064_511911_object58063:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-10-09T06:16:03.434 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:07:50.749569+0000", 2023-10-09T06:16:03.434 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 492.66491501799999, 2023-10-09T06:16:03.434 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96902807899999999, 2023-10-09T06:16:03.434 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:16:03.434 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:16:03.434 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:16:03.435 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:16:03.435 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:16:03.435 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 58064 2023-10-09T06:16:03.435 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.435 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:16:03.435 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.435 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:16:03.435 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.749569+0000", 2023-10-09T06:16:03.435 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.435 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.435 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.436 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:16:03.436 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.749569+0000", 2023-10-09T06:16:03.436 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.436 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.436 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.436 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:16:03.436 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.749568+0000", 2023-10-09T06:16:03.436 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:16:03.436 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.436 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.436 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:16:03.436 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.749620+0000", 2023-10-09T06:16:03.437 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1990000000000002e-05 2023-10-09T06:16:03.437 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.437 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.437 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:16:03.437 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.749631+0000", 2023-10-09T06:16:03.437 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0131e-05 2023-10-09T06:16:03.437 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.437 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.437 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:16:03.437 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.749635+0000", 2023-10-09T06:16:03.437 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9539999999999998e-06 2023-10-09T06:16:03.437 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.438 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.438 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.438 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.749677+0000", 2023-10-09T06:16:03.438 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2719000000000002e-05 2023-10-09T06:16:03.438 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.438 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.438 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:16:03.438 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.749686+0000", 2023-10-09T06:16:03.438 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1719999999999996e-06 2023-10-09T06:16:03.438 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.438 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.438 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.439 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.711769+0000", 2023-10-09T06:16:03.439 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96208275300000001 2023-10-09T06:16:03.439 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.439 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.439 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:16:03.439 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.711891+0000", 2023-10-09T06:16:03.439 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012202200000000001 2023-10-09T06:16:03.439 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.439 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.439 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5", 2023-10-09T06:16:03.439 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.712032+0000", 2023-10-09T06:16:03.439 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014087799999999999 2023-10-09T06:16:03.440 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.440 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.440 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:16:03.440 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718043+0000", 2023-10-09T06:16:03.440 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0060111720000000004 2023-10-09T06:16:03.440 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.440 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.440 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.440 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718475+0000", 2023-10-09T06:16:03.440 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00043195499999999998 2023-10-09T06:16:03.440 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.441 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.441 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:16:03.441 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718519+0000", 2023-10-09T06:16:03.441 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3587000000000001e-05 2023-10-09T06:16:03.441 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.441 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.441 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:16:03.441 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718597+0000", 2023-10-09T06:16:03.441 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8603000000000006e-05 2023-10-09T06:16:03.441 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.441 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:16:03.441 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.442 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.442 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.442 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:58070 9.9 9:97018e1f:::benchmark_data_smithi064_511911_object58069:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-10-09T06:16:03.442 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:07:50.758559+0000", 2023-10-09T06:16:03.442 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 492.65592552599998, 2023-10-09T06:16:03.442 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96067754699999997, 2023-10-09T06:16:03.442 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:16:03.442 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:16:03.442 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:16:03.442 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:16:03.442 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:16:03.443 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 58070 2023-10-09T06:16:03.443 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.443 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:16:03.443 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.443 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:16:03.443 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.758559+0000", 2023-10-09T06:16:03.443 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.443 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.443 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.443 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:16:03.443 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.758559+0000", 2023-10-09T06:16:03.443 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.443 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.444 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.444 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:16:03.444 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.758558+0000", 2023-10-09T06:16:03.444 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:16:03.444 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.444 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.444 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:16:03.444 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.758601+0000", 2023-10-09T06:16:03.444 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3180000000000003e-05 2023-10-09T06:16:03.444 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.444 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.444 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:16:03.445 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.758614+0000", 2023-10-09T06:16:03.445 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3309000000000001e-05 2023-10-09T06:16:03.445 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.445 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.445 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:16:03.445 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.758619+0000", 2023-10-09T06:16:03.445 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7700000000000001e-06 2023-10-09T06:16:03.445 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.445 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.445 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.445 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.758669+0000", 2023-10-09T06:16:03.445 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0027000000000001e-05 2023-10-09T06:16:03.446 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.446 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.446 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:16:03.446 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.758677+0000", 2023-10-09T06:16:03.446 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1389999999999995e-06 2023-10-09T06:16:03.446 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.446 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.446 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.446 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.714028+0000", 2023-10-09T06:16:03.446 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.95535058699999997 2023-10-09T06:16:03.446 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.446 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.447 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:16:03.447 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.714132+0000", 2023-10-09T06:16:03.447 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000104565 2023-10-09T06:16:03.447 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.447 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.447 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5", 2023-10-09T06:16:03.447 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.714264+0000", 2023-10-09T06:16:03.447 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000131798 2023-10-09T06:16:03.447 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.447 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.447 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.447 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.719035+0000", 2023-10-09T06:16:03.448 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0047712479999999996 2023-10-09T06:16:03.448 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.448 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.448 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:16:03.448 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.719088+0000", 2023-10-09T06:16:03.448 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2954000000000002e-05 2023-10-09T06:16:03.448 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.448 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.448 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:16:03.448 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.719156+0000", 2023-10-09T06:16:03.448 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7958000000000002e-05 2023-10-09T06:16:03.448 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.449 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.449 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:16:03.449 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.719236+0000", 2023-10-09T06:16:03.449 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9980000000000003e-05 2023-10-09T06:16:03.449 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.449 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:16:03.449 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.449 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.449 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.449 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:58081 9.5 9:a1616141:::benchmark_data_smithi064_511911_object58080:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-10-09T06:16:03.449 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:07:50.770142+0000", 2023-10-09T06:16:03.449 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 492.64434193699998, 2023-10-09T06:16:03.450 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.94865276799999998, 2023-10-09T06:16:03.450 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:16:03.450 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:16:03.450 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:16:03.450 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:16:03.450 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:16:03.450 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 58081 2023-10-09T06:16:03.450 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.450 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:16:03.450 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.450 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:16:03.451 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770142+0000", 2023-10-09T06:16:03.451 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.451 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.451 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.451 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:16:03.451 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770142+0000", 2023-10-09T06:16:03.451 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.451 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.451 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.451 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:16:03.451 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770141+0000", 2023-10-09T06:16:03.451 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:16:03.452 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.452 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.452 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:16:03.452 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770188+0000", 2023-10-09T06:16:03.452 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6885999999999998e-05 2023-10-09T06:16:03.452 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.452 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.452 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:16:03.452 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770199+0000", 2023-10-09T06:16:03.452 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.058e-05 2023-10-09T06:16:03.452 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.452 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.453 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:16:03.453 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770203+0000", 2023-10-09T06:16:03.453 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4139999999999996e-06 2023-10-09T06:16:03.453 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.453 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.453 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.453 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770301+0000", 2023-10-09T06:16:03.453 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7325999999999996e-05 2023-10-09T06:16:03.453 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.453 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.453 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:16:03.453 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770319+0000", 2023-10-09T06:16:03.454 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8601e-05 2023-10-09T06:16:03.454 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.454 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.454 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.454 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.711820+0000", 2023-10-09T06:16:03.454 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.94150076199999999 2023-10-09T06:16:03.454 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.454 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.454 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:16:03.454 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.711925+0000", 2023-10-09T06:16:03.454 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000105262 2023-10-09T06:16:03.455 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.455 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.455 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5", 2023-10-09T06:16:03.455 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.712064+0000", 2023-10-09T06:16:03.455 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013905699999999999 2023-10-09T06:16:03.455 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.455 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.455 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.455 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718649+0000", 2023-10-09T06:16:03.455 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0065846630000000001 2023-10-09T06:16:03.455 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.455 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.456 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:16:03.456 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718660+0000", 2023-10-09T06:16:03.456 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0929e-05 2023-10-09T06:16:03.456 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.456 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.456 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:16:03.456 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718711+0000", 2023-10-09T06:16:03.456 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0955000000000003e-05 2023-10-09T06:16:03.456 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.456 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.456 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:16:03.456 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.718795+0000", 2023-10-09T06:16:03.457 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.4289000000000006e-05 2023-10-09T06:16:03.457 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.457 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:16:03.457 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.457 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.457 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.457 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:58083 9.9 9:99aef645:::benchmark_data_smithi064_511911_object58082:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-10-09T06:16:03.457 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:07:50.770277+0000", 2023-10-09T06:16:03.457 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 492.64420682899998, 2023-10-09T06:16:03.457 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.95279251799999998, 2023-10-09T06:16:03.457 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:16:03.457 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:16:03.458 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:16:03.458 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:16:03.458 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:16:03.458 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 58083 2023-10-09T06:16:03.458 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.458 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:16:03.458 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.458 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:16:03.458 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770277+0000", 2023-10-09T06:16:03.458 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.458 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.458 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.459 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:16:03.459 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770277+0000", 2023-10-09T06:16:03.459 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.459 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.459 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.459 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:16:03.459 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770277+0000", 2023-10-09T06:16:03.459 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:16:03.459 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.459 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.459 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:16:03.460 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770313+0000", 2023-10-09T06:16:03.460 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6062000000000001e-05 2023-10-09T06:16:03.460 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.460 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.460 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:16:03.460 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770323+0000", 2023-10-09T06:16:03.460 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0016e-05 2023-10-09T06:16:03.460 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.460 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.460 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:16:03.460 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770328+0000", 2023-10-09T06:16:03.460 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8339999999999998e-06 2023-10-09T06:16:03.461 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.461 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.461 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.461 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770379+0000", 2023-10-09T06:16:03.461 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1304999999999998e-05 2023-10-09T06:16:03.461 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.461 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.461 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:16:03.461 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:50.770388+0000", 2023-10-09T06:16:03.461 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8079999999999994e-06 2023-10-09T06:16:03.461 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.461 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.461 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.462 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.716270+0000", 2023-10-09T06:16:03.462 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.94588199100000003 2023-10-09T06:16:03.462 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.462 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.462 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:16:03.462 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.716372+0000", 2023-10-09T06:16:03.462 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000101827 2023-10-09T06:16:03.462 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.462 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.462 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5", 2023-10-09T06:16:03.462 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.716452+0000", 2023-10-09T06:16:03.463 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0498000000000001e-05 2023-10-09T06:16:03.463 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.463 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.463 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:16:03.463 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.719244+0000", 2023-10-09T06:16:03.463 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0027915470000000001 2023-10-09T06:16:03.463 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.463 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.463 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.463 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.722915+0000", 2023-10-09T06:16:03.463 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0036714529999999999 2023-10-09T06:16:03.463 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.463 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.464 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:16:03.464 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.722964+0000", 2023-10-09T06:16:03.464 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8678999999999999e-05 2023-10-09T06:16:03.464 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.464 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.464 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:16:03.464 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:07:51.723070+0000", 2023-10-09T06:16:03.464 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000105936 2023-10-09T06:16:03.464 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.464 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:16:03.464 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.464 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.465 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.465 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:116286 9.9 9:91b2f3bd:::benchmark_data_smithi064_511911_object15184:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e450)", 2023-10-09T06:16:03.465 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:03.586689+0000", 2023-10-09T06:16:03.465 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 419.82779576600001, 2023-10-09T06:16:03.465 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1393943420000001, 2023-10-09T06:16:03.465 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:16:03.465 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:16:03.465 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:16:03.465 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:16:03.465 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:16:03.465 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 116286 2023-10-09T06:16:03.466 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.466 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:16:03.466 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.466 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:16:03.466 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586689+0000", 2023-10-09T06:16:03.466 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.466 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.466 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.466 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:16:03.466 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586689+0000", 2023-10-09T06:16:03.466 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.467 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.467 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.467 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:16:03.467 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586687+0000", 2023-10-09T06:16:03.467 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:16:03.467 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.467 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.467 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:16:03.467 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586695+0000", 2023-10-09T06:16:03.467 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0700000000000007e-06 2023-10-09T06:16:03.467 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.467 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.468 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:16:03.468 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586708+0000", 2023-10-09T06:16:03.468 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2928999999999999e-05 2023-10-09T06:16:03.468 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.468 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.468 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:16:03.468 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586715+0000", 2023-10-09T06:16:03.468 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1629999999999999e-06 2023-10-09T06:16:03.468 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.468 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.468 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.468 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.590775+0000", 2023-10-09T06:16:03.469 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0040602310000000001 2023-10-09T06:16:03.469 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.469 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.469 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:16:03.469 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.590785+0000", 2023-10-09T06:16:03.469 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2529999999999993e-06 2023-10-09T06:16:03.469 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.469 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.469 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.469 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.708849+0000", 2023-10-09T06:16:03.469 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1180647349999999 2023-10-09T06:16:03.469 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.470 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.470 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for missing object", 2023-10-09T06:16:03.470 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.708984+0000", 2023-10-09T06:16:03.470 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000134136 2023-10-09T06:16:03.470 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.470 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.470 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.470 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.725947+0000", 2023-10-09T06:16:03.470 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.016963676 2023-10-09T06:16:03.470 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.470 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.471 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:16:03.471 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.726083+0000", 2023-10-09T06:16:03.471 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000135699 2023-10-09T06:16:03.471 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.471 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:16:03.471 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.471 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.471 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.471 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:116302 9.9 9:97b6a430:::benchmark_data_smithi064_511911_object15200:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e450)", 2023-10-09T06:16:03.471 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:03.586724+0000", 2023-10-09T06:16:03.471 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 419.82776046999999, 2023-10-09T06:16:03.471 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.139970148, 2023-10-09T06:16:03.472 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:16:03.472 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:16:03.472 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:16:03.472 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:16:03.472 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:16:03.472 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 116302 2023-10-09T06:16:03.472 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.472 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:16:03.472 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.472 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:16:03.472 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586724+0000", 2023-10-09T06:16:03.472 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.473 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.473 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.473 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:16:03.473 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586724+0000", 2023-10-09T06:16:03.473 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.473 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.473 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.473 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:16:03.473 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586723+0000", 2023-10-09T06:16:03.473 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:16:03.473 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.473 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.474 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:16:03.474 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586726+0000", 2023-10-09T06:16:03.474 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.8909999999999999e-06 2023-10-09T06:16:03.474 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.474 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.474 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:16:03.474 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586733+0000", 2023-10-09T06:16:03.474 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.131e-06 2023-10-09T06:16:03.474 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.474 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.474 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:16:03.474 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586739+0000", 2023-10-09T06:16:03.475 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.485e-06 2023-10-09T06:16:03.475 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.475 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.475 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.475 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.599094+0000", 2023-10-09T06:16:03.475 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012355138 2023-10-09T06:16:03.475 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.475 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.475 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:16:03.475 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.599106+0000", 2023-10-09T06:16:03.475 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1565e-05 2023-10-09T06:16:03.475 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.476 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.476 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.476 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.709008+0000", 2023-10-09T06:16:03.476 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1099019999999999 2023-10-09T06:16:03.476 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.476 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.476 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:16:03.476 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.709136+0000", 2023-10-09T06:16:03.476 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012808900000000001 2023-10-09T06:16:03.476 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.476 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.476 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:16:03.477 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.709273+0000", 2023-10-09T06:16:03.477 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000137584 2023-10-09T06:16:03.477 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.477 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.477 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:16:03.477 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.725163+0000", 2023-10-09T06:16:03.477 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015889678000000001 2023-10-09T06:16:03.477 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.477 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.477 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.477 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.725700+0000", 2023-10-09T06:16:03.477 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00053697700000000001 2023-10-09T06:16:03.478 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.478 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.478 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.478 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.726552+0000", 2023-10-09T06:16:03.478 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000852399 2023-10-09T06:16:03.478 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.478 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.478 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:16:03.478 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.726596+0000", 2023-10-09T06:16:03.478 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3451999999999997e-05 2023-10-09T06:16:03.478 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.479 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.479 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:16:03.479 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.726694+0000", 2023-10-09T06:16:03.479 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.8176000000000003e-05 2023-10-09T06:16:03.479 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.479 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:16:03.479 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.479 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.479 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.479 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:132900 9.9 9:9c50d9a2:::benchmark_data_smithi064_511911_object31798:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e455)", 2023-10-09T06:16:03.479 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:26.559784+0000", 2023-10-09T06:16:03.479 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 396.85470028200001, 2023-10-09T06:16:03.480 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2243415390000001, 2023-10-09T06:16:03.480 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:16:03.480 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:16:03.480 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:16:03.480 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:16:03.480 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:16:03.480 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 132900 2023-10-09T06:16:03.480 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.480 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:16:03.480 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.480 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:16:03.480 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.559784+0000", 2023-10-09T06:16:03.481 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.481 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.481 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.481 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:16:03.481 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.559784+0000", 2023-10-09T06:16:03.481 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.481 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.481 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.481 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:16:03.481 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.559783+0000", 2023-10-09T06:16:03.481 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:16:03.481 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.482 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.482 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:16:03.482 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.559789+0000", 2023-10-09T06:16:03.482 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5690000000000001e-06 2023-10-09T06:16:03.482 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.482 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.482 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:16:03.482 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.559800+0000", 2023-10-09T06:16:03.482 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0288e-05 2023-10-09T06:16:03.482 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.482 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.482 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:16:03.483 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.562277+0000", 2023-10-09T06:16:03.483 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002477103 2023-10-09T06:16:03.483 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.483 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.483 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.483 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.565865+0000", 2023-10-09T06:16:03.483 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0035886989999999999 2023-10-09T06:16:03.483 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.483 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.483 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:16:03.483 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.566006+0000", 2023-10-09T06:16:03.484 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014012900000000001 2023-10-09T06:16:03.484 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.484 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.484 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:16:03.484 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.566132+0000", 2023-10-09T06:16:03.484 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012682599999999999 2023-10-09T06:16:03.484 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.484 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.484 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.484 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572376+0000", 2023-10-09T06:16:03.484 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0062437980000000001 2023-10-09T06:16:03.484 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.485 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.485 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:16:03.485 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572387+0000", 2023-10-09T06:16:03.485 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0492999999999999e-05 2023-10-09T06:16:03.485 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.485 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.485 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.485 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.587508+0000", 2023-10-09T06:16:03.485 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015120971 2023-10-09T06:16:03.485 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.485 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.485 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:16:03.486 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.587518+0000", 2023-10-09T06:16:03.486 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0197000000000001e-05 2023-10-09T06:16:03.486 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.486 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.486 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.486 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.761187+0000", 2023-10-09T06:16:03.486 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.17366953 2023-10-09T06:16:03.486 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.486 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.486 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for missing object", 2023-10-09T06:16:03.486 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.761345+0000", 2023-10-09T06:16:03.486 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015735400000000001 2023-10-09T06:16:03.487 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.487 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.487 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.487 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.784048+0000", 2023-10-09T06:16:03.487 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.022703204000000001 2023-10-09T06:16:03.487 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.487 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.487 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:16:03.487 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.784126+0000", 2023-10-09T06:16:03.487 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7649000000000001e-05 2023-10-09T06:16:03.487 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.487 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:16:03.488 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.488 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.488 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.488 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:132902 9.9 9:90d35044:::benchmark_data_smithi064_511911_object31800:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e455)", 2023-10-09T06:16:03.488 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:26.562296+0000", 2023-10-09T06:16:03.488 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 396.85218827900002, 2023-10-09T06:16:03.488 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2265417860000001, 2023-10-09T06:16:03.488 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:16:03.488 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:16:03.488 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:16:03.488 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:16:03.488 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:16:03.489 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 132902 2023-10-09T06:16:03.489 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.489 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:16:03.489 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.489 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:16:03.489 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.562296+0000", 2023-10-09T06:16:03.489 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.489 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.489 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.489 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:16:03.489 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.562296+0000", 2023-10-09T06:16:03.490 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.490 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.490 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.490 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:16:03.490 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.562295+0000", 2023-10-09T06:16:03.490 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:16:03.490 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.490 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.490 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:16:03.490 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.562300+0000", 2023-10-09T06:16:03.490 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5339999999999999e-06 2023-10-09T06:16:03.490 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.491 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.491 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:16:03.491 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.562309+0000", 2023-10-09T06:16:03.491 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3470000000000005e-06 2023-10-09T06:16:03.491 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.491 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.491 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:16:03.491 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.562314+0000", 2023-10-09T06:16:03.491 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7299999999999996e-06 2023-10-09T06:16:03.491 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.491 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.492 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.492 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.569303+0000", 2023-10-09T06:16:03.492 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0069888600000000004 2023-10-09T06:16:03.492 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.492 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.492 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:16:03.492 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.569430+0000", 2023-10-09T06:16:03.492 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012732600000000001 2023-10-09T06:16:03.492 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.492 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.492 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:16:03.492 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.569519+0000", 2023-10-09T06:16:03.493 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8524999999999997e-05 2023-10-09T06:16:03.493 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.493 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.493 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.493 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.582104+0000", 2023-10-09T06:16:03.493 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012585202 2023-10-09T06:16:03.493 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.493 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.493 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:16:03.493 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.582122+0000", 2023-10-09T06:16:03.493 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8048999999999999e-05 2023-10-09T06:16:03.494 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.494 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.494 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.494 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.587546+0000", 2023-10-09T06:16:03.494 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0054235079999999996 2023-10-09T06:16:03.494 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.494 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.494 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:16:03.494 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.587553+0000", 2023-10-09T06:16:03.494 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.99e-06 2023-10-09T06:16:03.494 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.494 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.495 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.495 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.761400+0000", 2023-10-09T06:16:03.495 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.173847133 2023-10-09T06:16:03.495 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.495 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.495 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for missing object", 2023-10-09T06:16:03.495 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.761507+0000", 2023-10-09T06:16:03.495 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000106971 2023-10-09T06:16:03.495 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.495 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.495 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.495 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.788758+0000", 2023-10-09T06:16:03.496 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.027251068999999999 2023-10-09T06:16:03.496 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.496 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.496 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:16:03.496 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.788838+0000", 2023-10-09T06:16:03.496 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9989999999999998e-05 2023-10-09T06:16:03.496 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.496 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:16:03.496 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.496 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.496 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.496 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:131778 10.9 10:909f8957:::benchmark_data_smithi064_573750_object35563:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-10-09T06:16:03.497 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:14:20.794429+0000", 2023-10-09T06:16:03.497 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 102.620055572, 2023-10-09T06:16:03.497 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99906589199999996, 2023-10-09T06:16:03.497 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:16:03.497 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:16:03.497 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:16:03.497 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:16:03.497 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:16:03.497 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 131778 2023-10-09T06:16:03.497 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.497 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:16:03.497 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.498 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:16:03.498 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.794429+0000", 2023-10-09T06:16:03.498 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.498 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.498 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.498 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:16:03.498 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.794429+0000", 2023-10-09T06:16:03.498 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.498 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.498 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.498 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:16:03.499 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.794427+0000", 2023-10-09T06:16:03.499 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:16:03.499 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.499 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.499 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:16:03.499 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.794434+0000", 2023-10-09T06:16:03.499 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.934e-06 2023-10-09T06:16:03.499 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.499 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.499 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:16:03.499 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.794452+0000", 2023-10-09T06:16:03.499 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7730000000000001e-05 2023-10-09T06:16:03.500 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.500 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.500 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:16:03.500 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.794460+0000", 2023-10-09T06:16:03.500 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6469999999999998e-06 2023-10-09T06:16:03.500 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.500 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.500 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.500 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.794511+0000", 2023-10-09T06:16:03.500 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1496999999999998e-05 2023-10-09T06:16:03.500 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.500 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.501 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:16:03.501 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.794520+0000", 2023-10-09T06:16:03.501 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8130000000000002e-06 2023-10-09T06:16:03.501 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.501 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.501 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.501 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.759431+0000", 2023-10-09T06:16:03.501 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96491159000000004 2023-10-09T06:16:03.501 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.501 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.501 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:16:03.501 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.759582+0000", 2023-10-09T06:16:03.502 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015018799999999999 2023-10-09T06:16:03.502 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.502 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.502 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:16:03.502 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.759718+0000", 2023-10-09T06:16:03.502 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000136608 2023-10-09T06:16:03.502 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.502 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.502 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:16:03.502 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.775116+0000", 2023-10-09T06:16:03.502 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015397622 2023-10-09T06:16:03.502 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.503 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.503 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.503 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793364+0000", 2023-10-09T06:16:03.503 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018247780000000002 2023-10-09T06:16:03.503 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.503 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.503 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.503 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793393+0000", 2023-10-09T06:16:03.503 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9689999999999999e-05 2023-10-09T06:16:03.503 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.503 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.503 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:16:03.504 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793447+0000", 2023-10-09T06:16:03.504 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3127999999999997e-05 2023-10-09T06:16:03.504 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.504 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.504 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:16:03.504 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793495+0000", 2023-10-09T06:16:03.504 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8112999999999998e-05 2023-10-09T06:16:03.504 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.504 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:16:03.504 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.504 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.504 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.505 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:131788 10.9 10:9b77a1ac:::benchmark_data_smithi064_573750_object35573:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-10-09T06:16:03.505 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:14:20.803992+0000", 2023-10-09T06:16:03.505 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 102.610492663, 2023-10-09T06:16:03.505 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.98965816100000004, 2023-10-09T06:16:03.505 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:16:03.505 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:16:03.505 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:16:03.505 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:16:03.505 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:16:03.505 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 131788 2023-10-09T06:16:03.505 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.505 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:16:03.505 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.506 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:16:03.506 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.803992+0000", 2023-10-09T06:16:03.506 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.506 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.506 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.506 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:16:03.506 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.803992+0000", 2023-10-09T06:16:03.506 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.506 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.506 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.506 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:16:03.507 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.803990+0000", 2023-10-09T06:16:03.507 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:16:03.507 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.507 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.507 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:16:03.507 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.803997+0000", 2023-10-09T06:16:03.507 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2729999999999997e-06 2023-10-09T06:16:03.507 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.507 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.507 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:16:03.507 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804012+0000", 2023-10-09T06:16:03.507 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4501000000000001e-05 2023-10-09T06:16:03.508 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.508 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.508 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:16:03.508 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804018+0000", 2023-10-09T06:16:03.508 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1940000000000003e-06 2023-10-09T06:16:03.508 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.508 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.508 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.508 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804073+0000", 2023-10-09T06:16:03.508 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5361000000000001e-05 2023-10-09T06:16:03.508 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.508 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.508 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:16:03.509 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804081+0000", 2023-10-09T06:16:03.509 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5190000000000003e-06 2023-10-09T06:16:03.509 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.509 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.509 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.509 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.762019+0000", 2023-10-09T06:16:03.509 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.95793824800000005 2023-10-09T06:16:03.509 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.509 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.509 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:16:03.509 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.762126+0000", 2023-10-09T06:16:03.509 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000107262 2023-10-09T06:16:03.510 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.510 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.510 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:16:03.510 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.762202+0000", 2023-10-09T06:16:03.510 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5018000000000006e-05 2023-10-09T06:16:03.510 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.510 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.510 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:16:03.510 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.775126+0000", 2023-10-09T06:16:03.510 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012924542000000001 2023-10-09T06:16:03.510 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.511 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.511 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.511 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793550+0000", 2023-10-09T06:16:03.511 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018423946 2023-10-09T06:16:03.511 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.511 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.511 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.511 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793588+0000", 2023-10-09T06:16:03.511 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7679000000000003e-05 2023-10-09T06:16:03.511 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.511 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.511 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:16:03.512 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793627+0000", 2023-10-09T06:16:03.512 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9410999999999999e-05 2023-10-09T06:16:03.512 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.512 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.512 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:16:03.512 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793650+0000", 2023-10-09T06:16:03.512 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2719999999999999e-05 2023-10-09T06:16:03.512 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.512 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:16:03.512 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.512 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.512 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.513 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:131789 10.9 10:95bec615:::benchmark_data_smithi064_573750_object35574:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-10-09T06:16:03.513 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:14:20.804037+0000", 2023-10-09T06:16:03.513 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 102.610446888, 2023-10-09T06:16:03.513 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.98989388300000003, 2023-10-09T06:16:03.513 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:16:03.513 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:16:03.513 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:16:03.513 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:16:03.513 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:16:03.513 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 131789 2023-10-09T06:16:03.513 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.514 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:16:03.514 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.514 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:16:03.514 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804037+0000", 2023-10-09T06:16:03.514 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.514 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.514 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.514 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:16:03.514 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804037+0000", 2023-10-09T06:16:03.514 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.514 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.514 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.515 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:16:03.515 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804037+0000", 2023-10-09T06:16:03.515 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:16:03.515 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.515 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.515 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:16:03.515 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804040+0000", 2023-10-09T06:16:03.515 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9730000000000002e-06 2023-10-09T06:16:03.515 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.515 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.515 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:16:03.515 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804046+0000", 2023-10-09T06:16:03.516 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3249999999999996e-06 2023-10-09T06:16:03.516 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.516 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.516 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:16:03.516 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804050+0000", 2023-10-09T06:16:03.516 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2239999999999997e-06 2023-10-09T06:16:03.516 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.516 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.516 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.516 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804140+0000", 2023-10-09T06:16:03.516 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9087999999999998e-05 2023-10-09T06:16:03.516 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.517 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.517 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:16:03.517 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804145+0000", 2023-10-09T06:16:03.517 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0889999999999999e-06 2023-10-09T06:16:03.517 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.517 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.517 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.517 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.764349+0000", 2023-10-09T06:16:03.517 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96020412399999999 2023-10-09T06:16:03.517 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.517 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.517 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:16:03.518 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.764467+0000", 2023-10-09T06:16:03.518 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011781300000000001 2023-10-09T06:16:03.518 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.518 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.518 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:16:03.518 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.764590+0000", 2023-10-09T06:16:03.518 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012318400000000001 2023-10-09T06:16:03.518 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.518 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.518 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:16:03.518 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.775135+0000", 2023-10-09T06:16:03.518 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010544992 2023-10-09T06:16:03.519 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.519 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.519 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.519 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793727+0000", 2023-10-09T06:16:03.519 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018592457 2023-10-09T06:16:03.519 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.519 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.519 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.519 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793848+0000", 2023-10-09T06:16:03.519 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000120865 2023-10-09T06:16:03.519 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.519 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.520 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:16:03.520 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793888+0000", 2023-10-09T06:16:03.520 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0070000000000001e-05 2023-10-09T06:16:03.520 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.520 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.520 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:16:03.520 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793931+0000", 2023-10-09T06:16:03.520 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3118000000000003e-05 2023-10-09T06:16:03.520 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.520 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:16:03.520 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.520 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.521 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.521 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:131809 10.9 10:93420ba2:::benchmark_data_smithi064_573750_object35594:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-10-09T06:16:03.521 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:14:20.820581+0000", 2023-10-09T06:16:03.521 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 102.593903622, 2023-10-09T06:16:03.521 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.97345599199999999, 2023-10-09T06:16:03.521 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:16:03.521 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:16:03.521 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:16:03.521 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:16:03.521 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:16:03.521 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 131809 2023-10-09T06:16:03.521 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.522 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:16:03.522 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.522 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:16:03.522 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.820581+0000", 2023-10-09T06:16:03.522 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.522 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.522 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.522 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:16:03.522 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.820581+0000", 2023-10-09T06:16:03.522 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.522 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.523 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.523 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:16:03.523 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.820579+0000", 2023-10-09T06:16:03.523 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:16:03.523 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.523 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.523 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:16:03.523 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.820586+0000", 2023-10-09T06:16:03.523 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6000000000000003e-06 2023-10-09T06:16:03.523 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.523 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.523 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:16:03.524 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.820603+0000", 2023-10-09T06:16:03.524 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7329999999999998e-05 2023-10-09T06:16:03.524 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.524 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.524 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:16:03.524 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.820610+0000", 2023-10-09T06:16:03.524 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2079999999999997e-06 2023-10-09T06:16:03.524 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.524 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.524 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.524 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.820665+0000", 2023-10-09T06:16:03.524 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5900999999999999e-05 2023-10-09T06:16:03.525 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.525 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.525 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:16:03.525 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.820673+0000", 2023-10-09T06:16:03.525 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7780000000000007e-06 2023-10-09T06:16:03.525 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.525 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.525 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.525 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.766668+0000", 2023-10-09T06:16:03.525 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.94599452299999998 2023-10-09T06:16:03.525 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.525 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.526 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:16:03.526 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.766781+0000", 2023-10-09T06:16:03.526 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011279499999999999 2023-10-09T06:16:03.526 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.526 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.526 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:16:03.526 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.766894+0000", 2023-10-09T06:16:03.526 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000113927 2023-10-09T06:16:03.526 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.526 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.526 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:16:03.526 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.775144+0000", 2023-10-09T06:16:03.527 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.008249269 2023-10-09T06:16:03.527 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.527 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.527 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.527 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793750+0000", 2023-10-09T06:16:03.527 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018606527000000001 2023-10-09T06:16:03.527 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.527 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.527 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.527 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793979+0000", 2023-10-09T06:16:03.527 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000229175 2023-10-09T06:16:03.527 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.528 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.528 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:16:03.528 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.794008+0000", 2023-10-09T06:16:03.528 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.8907e-05 2023-10-09T06:16:03.528 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.528 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.528 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:16:03.528 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.794037+0000", 2023-10-09T06:16:03.528 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.8339000000000001e-05 2023-10-09T06:16:03.528 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.528 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:16:03.528 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.529 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.529 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.529 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:131814 10.9 10:98c8ec43:::benchmark_data_smithi064_573750_object35599:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-10-09T06:16:03.529 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:14:20.832064+0000", 2023-10-09T06:16:03.529 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 102.58242042000001, 2023-10-09T06:16:03.529 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96215577299999999, 2023-10-09T06:16:03.529 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:16:03.529 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:16:03.529 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:16:03.529 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:16:03.529 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:16:03.529 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 131814 2023-10-09T06:16:03.530 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.530 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:16:03.530 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.530 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:16:03.530 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.832064+0000", 2023-10-09T06:16:03.530 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.530 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.530 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.530 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:16:03.530 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.832064+0000", 2023-10-09T06:16:03.530 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.531 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.531 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.531 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:16:03.531 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.832063+0000", 2023-10-09T06:16:03.531 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:16:03.531 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.531 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.531 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:16:03.531 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.832073+0000", 2023-10-09T06:16:03.531 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0720000000000001e-05 2023-10-09T06:16:03.531 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.531 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.532 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:16:03.532 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.832085+0000", 2023-10-09T06:16:03.532 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2157000000000001e-05 2023-10-09T06:16:03.532 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.532 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.532 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:16:03.532 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.832091+0000", 2023-10-09T06:16:03.532 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4519999999999996e-06 2023-10-09T06:16:03.532 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.532 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.532 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.532 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.832140+0000", 2023-10-09T06:16:03.533 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9036999999999998e-05 2023-10-09T06:16:03.533 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.533 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.533 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:16:03.533 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.832146+0000", 2023-10-09T06:16:03.533 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4880000000000004e-06 2023-10-09T06:16:03.533 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.533 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.533 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.533 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.769195+0000", 2023-10-09T06:16:03.533 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.93704818499999998 2023-10-09T06:16:03.533 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.534 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.534 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:16:03.534 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.769310+0000", 2023-10-09T06:16:03.534 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000115648 2023-10-09T06:16:03.534 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.534 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.534 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:16:03.534 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.769403+0000", 2023-10-09T06:16:03.534 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2818999999999993e-05 2023-10-09T06:16:03.534 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.534 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.534 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:16:03.535 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.781216+0000", 2023-10-09T06:16:03.535 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011812977000000001 2023-10-09T06:16:03.535 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.535 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.535 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.535 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.794083+0000", 2023-10-09T06:16:03.535 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012867286 2023-10-09T06:16:03.535 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.535 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.535 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.535 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.794159+0000", 2023-10-09T06:16:03.535 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5951999999999994e-05 2023-10-09T06:16:03.536 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.536 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.536 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:16:03.536 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.794186+0000", 2023-10-09T06:16:03.536 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.6866999999999999e-05 2023-10-09T06:16:03.536 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.536 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.536 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:16:03.536 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.794220+0000", 2023-10-09T06:16:03.536 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.3445000000000003e-05 2023-10-09T06:16:03.536 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.536 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:16:03.537 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.537 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.537 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.537 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:184014 10.c 10:372be510:::benchmark_data_smithi064_573750_object87799:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e566)", 2023-10-09T06:16:03.537 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:33.643832+0000", 2023-10-09T06:16:03.537 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 29.770652029000001, 2023-10-09T06:16:03.537 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.17411793, 2023-10-09T06:16:03.537 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:16:03.537 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:16:03.537 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:16:03.537 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:16:03.537 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:16:03.538 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 184014 2023-10-09T06:16:03.538 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.538 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:16:03.538 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.538 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:16:03.538 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.643832+0000", 2023-10-09T06:16:03.538 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.538 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.538 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.538 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:16:03.538 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.643832+0000", 2023-10-09T06:16:03.538 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.539 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.539 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.539 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:16:03.539 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.643832+0000", 2023-10-09T06:16:03.539 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:16:03.539 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.539 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.539 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:16:03.539 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.643835+0000", 2023-10-09T06:16:03.539 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.8320000000000002e-06 2023-10-09T06:16:03.539 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.539 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.540 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:16:03.540 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.643840+0000", 2023-10-09T06:16:03.540 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2469999999999999e-06 2023-10-09T06:16:03.540 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.540 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.540 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:16:03.540 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.643843+0000", 2023-10-09T06:16:03.540 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.49e-06 2023-10-09T06:16:03.540 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.540 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.540 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.540 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.643890+0000", 2023-10-09T06:16:03.541 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6186000000000001e-05 2023-10-09T06:16:03.541 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.541 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.541 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:16:03.541 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.644005+0000", 2023-10-09T06:16:03.541 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000115052 2023-10-09T06:16:03.541 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.541 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.541 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:16:03.541 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.644134+0000", 2023-10-09T06:16:03.541 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000129278 2023-10-09T06:16:03.541 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.542 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.542 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:16:03.542 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.653635+0000", 2023-10-09T06:16:03.542 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0095014450000000007 2023-10-09T06:16:03.542 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.542 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.542 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.542 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.661328+0000", 2023-10-09T06:16:03.542 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.007692267 2023-10-09T06:16:03.542 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.542 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.542 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.543 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.817848+0000", 2023-10-09T06:16:03.543 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.15652070700000001 2023-10-09T06:16:03.543 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.543 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.543 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:16:03.543 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.817903+0000", 2023-10-09T06:16:03.543 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4518000000000002e-05 2023-10-09T06:16:03.543 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.543 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.543 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:16:03.543 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.817950+0000", 2023-10-09T06:16:03.543 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7280999999999997e-05 2023-10-09T06:16:03.544 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.544 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:16:03.544 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.544 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.544 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.544 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:184018 10.c 10:36215808:::benchmark_data_smithi064_573750_object87803:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e566)", 2023-10-09T06:16:03.544 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:33.644223+0000", 2023-10-09T06:16:03.544 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 29.770261581, 2023-10-09T06:16:03.544 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.17929303799999999, 2023-10-09T06:16:03.544 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:16:03.544 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:16:03.544 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:16:03.544 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:16:03.545 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:16:03.545 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 184018 2023-10-09T06:16:03.545 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.545 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:16:03.545 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.545 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:16:03.545 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.644223+0000", 2023-10-09T06:16:03.545 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.545 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.545 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.545 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:16:03.545 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.644223+0000", 2023-10-09T06:16:03.546 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.546 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.546 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.546 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:16:03.546 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.644222+0000", 2023-10-09T06:16:03.546 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:16:03.546 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.546 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.546 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:16:03.546 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.644227+0000", 2023-10-09T06:16:03.546 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3059999999999997e-06 2023-10-09T06:16:03.547 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.547 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.547 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:16:03.547 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.644239+0000", 2023-10-09T06:16:03.547 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1942e-05 2023-10-09T06:16:03.547 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.547 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.547 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:16:03.547 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.644255+0000", 2023-10-09T06:16:03.547 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5699999999999999e-05 2023-10-09T06:16:03.547 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.547 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.548 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.548 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.647123+0000", 2023-10-09T06:16:03.548 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0028679980000000001 2023-10-09T06:16:03.548 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.548 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.548 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:16:03.548 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.647229+0000", 2023-10-09T06:16:03.548 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000105962 2023-10-09T06:16:03.548 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.548 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.548 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:16:03.548 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.647336+0000", 2023-10-09T06:16:03.549 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000107475 2023-10-09T06:16:03.549 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.549 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.549 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:16:03.549 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.656972+0000", 2023-10-09T06:16:03.549 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0096362840000000002 2023-10-09T06:16:03.549 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.549 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.549 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.549 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.661356+0000", 2023-10-09T06:16:03.549 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.004383128 2023-10-09T06:16:03.549 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.550 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.550 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.550 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.823421+0000", 2023-10-09T06:16:03.550 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.162065511 2023-10-09T06:16:03.550 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.550 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.550 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:16:03.550 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.823472+0000", 2023-10-09T06:16:03.550 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0600999999999999e-05 2023-10-09T06:16:03.550 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.550 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.550 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:16:03.551 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.823516+0000", 2023-10-09T06:16:03.551 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4045000000000003e-05 2023-10-09T06:16:03.551 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.551 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:16:03.551 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.551 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.551 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.551 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:184022 10.c 10:325bdf6b:::benchmark_data_smithi064_573750_object87807:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e566)", 2023-10-09T06:16:03.551 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:33.648936+0000", 2023-10-09T06:16:03.551 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 29.765548766999999, 2023-10-09T06:16:03.551 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.177772335, 2023-10-09T06:16:03.551 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:16:03.552 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:16:03.552 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:16:03.552 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:16:03.552 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:16:03.552 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 184022 2023-10-09T06:16:03.552 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.552 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:16:03.552 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.552 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:16:03.552 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.648936+0000", 2023-10-09T06:16:03.552 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.552 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.553 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.553 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:16:03.553 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.648936+0000", 2023-10-09T06:16:03.553 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.553 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.553 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.553 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:16:03.553 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.648934+0000", 2023-10-09T06:16:03.553 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:16:03.553 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.553 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.553 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:16:03.554 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.648941+0000", 2023-10-09T06:16:03.554 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0960000000000003e-06 2023-10-09T06:16:03.554 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.554 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.554 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:16:03.554 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.648957+0000", 2023-10-09T06:16:03.554 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.571e-05 2023-10-09T06:16:03.554 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.554 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.554 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:16:03.554 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.648964+0000", 2023-10-09T06:16:03.554 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8059999999999997e-06 2023-10-09T06:16:03.555 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.555 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.555 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.555 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.650381+0000", 2023-10-09T06:16:03.555 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001417019 2023-10-09T06:16:03.555 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.555 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.555 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:16:03.555 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.650483+0000", 2023-10-09T06:16:03.555 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000101753 2023-10-09T06:16:03.555 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.555 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.556 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:16:03.556 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.650586+0000", 2023-10-09T06:16:03.556 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000102965 2023-10-09T06:16:03.556 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.556 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.556 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:16:03.556 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.656980+0000", 2023-10-09T06:16:03.556 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0063948299999999998 2023-10-09T06:16:03.556 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.556 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.556 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.556 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.662728+0000", 2023-10-09T06:16:03.557 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.005748136 2023-10-09T06:16:03.557 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.557 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.557 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.557 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.826621+0000", 2023-10-09T06:16:03.557 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.16389221800000001 2023-10-09T06:16:03.557 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.557 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.557 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:16:03.557 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.826669+0000", 2023-10-09T06:16:03.557 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8338999999999999e-05 2023-10-09T06:16:03.557 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.558 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.558 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:16:03.558 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.826708+0000", 2023-10-09T06:16:03.558 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8838000000000003e-05 2023-10-09T06:16:03.558 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.558 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:16:03.558 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.558 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.558 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.558 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:184025 10.8 10:11619914:::benchmark_data_smithi064_573750_object87810:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e566)", 2023-10-09T06:16:03.558 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:33.649996+0000", 2023-10-09T06:16:03.558 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 29.764488440000001, 2023-10-09T06:16:03.559 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.16799005, 2023-10-09T06:16:03.559 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:16:03.559 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:16:03.559 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:16:03.559 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:16:03.559 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:16:03.559 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 184025 2023-10-09T06:16:03.559 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.559 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:16:03.559 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.559 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:16:03.559 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.649996+0000", 2023-10-09T06:16:03.560 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.560 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.560 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.560 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:16:03.560 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.649996+0000", 2023-10-09T06:16:03.560 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.560 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.560 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.560 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:16:03.560 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.649995+0000", 2023-10-09T06:16:03.560 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:16:03.560 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.561 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.561 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:16:03.561 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.650000+0000", 2023-10-09T06:16:03.561 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0719999999999999e-06 2023-10-09T06:16:03.561 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.561 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.561 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:16:03.561 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.650014+0000", 2023-10-09T06:16:03.561 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3558e-05 2023-10-09T06:16:03.561 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.561 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.561 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:16:03.562 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.650029+0000", 2023-10-09T06:16:03.562 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5498999999999998e-05 2023-10-09T06:16:03.562 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.562 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.562 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.562 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.650089+0000", 2023-10-09T06:16:03.562 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9809000000000003e-05 2023-10-09T06:16:03.562 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.562 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.562 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:16:03.562 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.650189+0000", 2023-10-09T06:16:03.562 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010042 2023-10-09T06:16:03.563 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.563 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.563 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:16:03.563 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.650287+0000", 2023-10-09T06:16:03.563 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7986999999999995e-05 2023-10-09T06:16:03.563 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.563 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.563 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:16:03.563 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.661109+0000", 2023-10-09T06:16:03.563 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010821848 2023-10-09T06:16:03.563 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.563 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.564 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.564 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.661413+0000", 2023-10-09T06:16:03.564 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00030341 2023-10-09T06:16:03.564 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.564 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.564 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.564 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.817903+0000", 2023-10-09T06:16:03.564 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.15649028200000001 2023-10-09T06:16:03.564 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.564 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.564 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:16:03.564 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.817948+0000", 2023-10-09T06:16:03.565 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4969999999999998e-05 2023-10-09T06:16:03.565 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.565 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.565 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:16:03.565 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.817986+0000", 2023-10-09T06:16:03.565 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8006999999999997e-05 2023-10-09T06:16:03.565 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.565 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:16:03.565 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.565 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.565 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.565 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:184027 10.c 10:3e9985c1:::benchmark_data_smithi064_573750_object87812:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e566)", 2023-10-09T06:16:03.566 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:33.653517+0000", 2023-10-09T06:16:03.566 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 29.760967790999999, 2023-10-09T06:16:03.566 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.17329388000000001, 2023-10-09T06:16:03.566 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:16:03.566 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:16:03.566 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:16:03.566 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:16:03.566 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:16:03.566 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 184027 2023-10-09T06:16:03.566 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.566 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:16:03.566 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.567 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:16:03.567 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.653517+0000", 2023-10-09T06:16:03.567 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.567 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.567 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.567 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:16:03.567 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.653517+0000", 2023-10-09T06:16:03.567 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:16:03.567 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.567 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.567 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:16:03.568 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.653515+0000", 2023-10-09T06:16:03.568 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:16:03.568 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.568 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.568 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:16:03.568 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.653521+0000", 2023-10-09T06:16:03.568 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1199999999999999e-06 2023-10-09T06:16:03.568 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.568 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.568 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:16:03.568 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.653532+0000", 2023-10-09T06:16:03.568 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1148999999999999e-05 2023-10-09T06:16:03.569 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.569 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.569 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:16:03.569 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.653538+0000", 2023-10-09T06:16:03.569 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6720000000000001e-06 2023-10-09T06:16:03.569 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.569 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.569 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:16:03.569 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.653670+0000", 2023-10-09T06:16:03.569 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013164200000000001 2023-10-09T06:16:03.569 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.569 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.570 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:16:03.570 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.653776+0000", 2023-10-09T06:16:03.570 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000106729 2023-10-09T06:16:03.570 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.570 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.570 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:16:03.570 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.653881+0000", 2023-10-09T06:16:03.570 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000104949 2023-10-09T06:16:03.570 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.570 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.570 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:16:03.571 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.657087+0000", 2023-10-09T06:16:03.571 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0032051580000000001 2023-10-09T06:16:03.571 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.571 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.571 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.571 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.663253+0000", 2023-10-09T06:16:03.571 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0061667140000000002 2023-10-09T06:16:03.571 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.571 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.571 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:03.571 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.826742+0000", 2023-10-09T06:16:03.571 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.16348903200000001 2023-10-09T06:16:03.572 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.572 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.572 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:16:03.572 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.826778+0000", 2023-10-09T06:16:03.572 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5256000000000003e-05 2023-10-09T06:16:03.572 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:16:03.572 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:16:03.572 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:16:03.572 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:33.826810+0000", 2023-10-09T06:16:03.572 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.2805000000000002e-05 2023-10-09T06:16:03.572 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.572 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:16:03.573 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.573 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:16:03.573 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:16:03.573 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:16:03.573 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:03 smithi064 ceph-mon[212750]: pgmap v1455: 103 pgs: 103 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:16:03.574 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:03 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1950319679' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T06:16:03.574 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:03 smithi064 ceph-mon[212750]: failed to encode map e572 with expected crc 2023-10-09T06:16:03.574 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:03 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T06:16:03.574 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:03 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1950319679' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T06:16:03.574 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:03 smithi064 ceph-mon[212750]: osdmap e572: 8 total, 8 up, 7 in 2023-10-09T06:16:03.574 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:03 smithi064 ceph-mon[212750]: failed to encode map e572 with expected crc 2023-10-09T06:16:03.574 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:03 smithi064 ceph-mon[212750]: failed to encode map e573 with expected crc 2023-10-09T06:16:03.574 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:03 smithi064 ceph-mon[212750]: osdmap e573: 8 total, 8 up, 7 in 2023-10-09T06:16:03.574 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:03 smithi064 ceph-mon[212750]: failed to encode map e573 with expected crc 2023-10-09T06:16:03.574 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:03 smithi064 ceph-mon[212750]: failed to encode map e573 with expected crc 2023-10-09T06:16:03.575 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:03 smithi064 ceph-mon[212750]: failed to encode map e573 with expected crc 2023-10-09T06:16:03.577 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:03 smithi064 ceph-mon[220044]: pgmap v1455: 103 pgs: 103 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:16:03.577 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:03 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1950319679' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T06:16:03.577 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:03 smithi064 ceph-mon[220044]: failed to encode map e572 with expected crc 2023-10-09T06:16:03.578 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:03 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T06:16:03.578 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:03 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1950319679' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T06:16:03.578 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:03 smithi064 ceph-mon[220044]: osdmap e572: 8 total, 8 up, 7 in 2023-10-09T06:16:03.578 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:03 smithi064 ceph-mon[220044]: failed to encode map e572 with expected crc 2023-10-09T06:16:03.578 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:03 smithi064 ceph-mon[220044]: failed to encode map e573 with expected crc 2023-10-09T06:16:03.578 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:03 smithi064 ceph-mon[220044]: osdmap e573: 8 total, 8 up, 7 in 2023-10-09T06:16:03.578 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:03 smithi064 ceph-mon[220044]: failed to encode map e573 with expected crc 2023-10-09T06:16:03.578 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:03 smithi064 ceph-mon[220044]: failed to encode map e573 with expected crc 2023-10-09T06:16:03.578 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:03 smithi064 ceph-mon[220044]: failed to encode map e573 with expected crc 2023-10-09T06:16:03.597 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:03 smithi156 ceph-mon[195101]: pgmap v1455: 103 pgs: 103 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:16:03.598 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:03 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1950319679' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T06:16:03.598 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:03 smithi156 ceph-mon[195101]: failed to encode map e572 with expected crc 2023-10-09T06:16:03.598 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:03 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T06:16:03.598 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:03 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1950319679' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T06:16:03.598 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:03 smithi156 ceph-mon[195101]: osdmap e572: 8 total, 8 up, 7 in 2023-10-09T06:16:03.598 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:03 smithi156 ceph-mon[195101]: failed to encode map e572 with expected crc 2023-10-09T06:16:03.598 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:03 smithi156 ceph-mon[195101]: failed to encode map e573 with expected crc 2023-10-09T06:16:03.598 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:03 smithi156 ceph-mon[195101]: osdmap e573: 8 total, 8 up, 7 in 2023-10-09T06:16:03.598 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:03 smithi156 ceph-mon[195101]: failed to encode map e573 with expected crc 2023-10-09T06:16:03.598 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:03 smithi156 ceph-mon[195101]: failed to encode map e573 with expected crc 2023-10-09T06:16:03.599 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:03 smithi156 ceph-mon[195101]: failed to encode map e573 with expected crc 2023-10-09T06:16:03.650 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:16:03.652 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-10-09T06:16:03.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:03.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:16:03.755 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:03.785 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:04.046 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:04.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:04.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:04.104 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:16:04.151 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:16:04.177 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:04.227 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:16:04.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:04.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:16:04.247 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:04.332 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:16:04.363 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:04.408 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:04.466 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:16:04.573 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:16:04.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:04.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:16:04.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:04 smithi064 ceph-mon[212750]: failed to encode map e573 with expected crc 2023-10-09T06:16:04.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:04 smithi064 ceph-mon[212750]: pgmap v1458: 119 pgs: 1 remapped+peering, 16 unknown, 102 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T06:16:04.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:04 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:04.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:04 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:04.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:04 smithi064 ceph-mon[212750]: failed to encode map e574 with expected crc 2023-10-09T06:16:04.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:04 smithi064 ceph-mon[212750]: osdmap e574: 8 total, 8 up, 7 in 2023-10-09T06:16:04.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:04 smithi064 ceph-mon[212750]: failed to encode map e574 with expected crc 2023-10-09T06:16:04.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:04 smithi064 ceph-mon[212750]: failed to encode map e574 with expected crc 2023-10-09T06:16:04.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:04 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:16:04.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:04 smithi064 ceph-mon[220044]: failed to encode map e573 with expected crc 2023-10-09T06:16:04.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:04 smithi064 ceph-mon[220044]: pgmap v1458: 119 pgs: 1 remapped+peering, 16 unknown, 102 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T06:16:04.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:04 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:04.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:04 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:04.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:04 smithi064 ceph-mon[220044]: failed to encode map e574 with expected crc 2023-10-09T06:16:04.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:04 smithi064 ceph-mon[220044]: osdmap e574: 8 total, 8 up, 7 in 2023-10-09T06:16:04.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:04 smithi064 ceph-mon[220044]: failed to encode map e574 with expected crc 2023-10-09T06:16:04.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:04 smithi064 ceph-mon[220044]: failed to encode map e574 with expected crc 2023-10-09T06:16:04.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:04 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:16:04.681 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:16:04.714 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:04.744 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:04 smithi156 ceph-mon[195101]: failed to encode map e573 with expected crc 2023-10-09T06:16:04.744 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:04 smithi156 ceph-mon[195101]: pgmap v1458: 119 pgs: 1 remapped+peering, 16 unknown, 102 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2023-10-09T06:16:04.744 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:04 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:04.745 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:04 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:04.745 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:04 smithi156 ceph-mon[195101]: failed to encode map e574 with expected crc 2023-10-09T06:16:04.745 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:04 smithi156 ceph-mon[195101]: osdmap e574: 8 total, 8 up, 7 in 2023-10-09T06:16:04.745 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:04 smithi156 ceph-mon[195101]: failed to encode map e574 with expected crc 2023-10-09T06:16:04.745 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:04 smithi156 ceph-mon[195101]: failed to encode map e574 with expected crc 2023-10-09T06:16:04.745 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:04 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:16:04.754 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:16:04.766 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:16:04.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:04.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:16:04.870 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:04.899 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:05.049 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:05.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:05.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:05.152 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:16:05.179 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:05.222 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:16:05.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:05.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:16:05.325 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:05.351 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:05.446 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:16:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:16:05] "GET /metrics HTTP/1.1" 200 34050 "" "Prometheus/2.43.0" 2023-10-09T06:16:05.610 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:05.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:05.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:05.614 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:05 smithi156 ceph-mon[195101]: failed to encode map e574 with expected crc 2023-10-09T06:16:05.614 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:05 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:05.614 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:05 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:05.614 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:05 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/482489413' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:16:05.615 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:05 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:16:05.615 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:05 smithi156 ceph-mon[195101]: failed to encode map e575 with expected crc 2023-10-09T06:16:05.615 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:05 smithi156 ceph-mon[195101]: failed to encode map e575 with expected crc 2023-10-09T06:16:05.615 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:05 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/482489413' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:16:05.615 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:05 smithi156 ceph-mon[195101]: osdmap e575: 8 total, 8 up, 7 in 2023-10-09T06:16:05.615 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:05 smithi156 ceph-mon[195101]: failed to encode map e575 with expected crc 2023-10-09T06:16:05.615 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:05 smithi156 ceph-mon[195101]: failed to encode map e575 with expected crc 2023-10-09T06:16:05.615 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:05 smithi156 ceph-mon[195101]: failed to encode map e575 with expected crc 2023-10-09T06:16:05.715 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:16:05.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:05 smithi064 ceph-mon[220044]: failed to encode map e574 with expected crc 2023-10-09T06:16:05.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:05 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:05.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:05 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:05.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:05 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/482489413' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:16:05.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:05 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:16:05.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:05 smithi064 ceph-mon[220044]: failed to encode map e575 with expected crc 2023-10-09T06:16:05.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:05 smithi064 ceph-mon[220044]: failed to encode map e575 with expected crc 2023-10-09T06:16:05.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:05 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/482489413' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:16:05.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:05 smithi064 ceph-mon[220044]: osdmap e575: 8 total, 8 up, 7 in 2023-10-09T06:16:05.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:05 smithi064 ceph-mon[220044]: failed to encode map e575 with expected crc 2023-10-09T06:16:05.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:05 smithi064 ceph-mon[220044]: failed to encode map e575 with expected crc 2023-10-09T06:16:05.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:05 smithi064 ceph-mon[220044]: failed to encode map e575 with expected crc 2023-10-09T06:16:05.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:05 smithi064 ceph-mon[212750]: failed to encode map e574 with expected crc 2023-10-09T06:16:05.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:05 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:05.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:05 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:05.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:05 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/482489413' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:16:05.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:05 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:16:05.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:05 smithi064 ceph-mon[212750]: failed to encode map e575 with expected crc 2023-10-09T06:16:05.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:05 smithi064 ceph-mon[212750]: failed to encode map e575 with expected crc 2023-10-09T06:16:05.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:05 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/482489413' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:16:05.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:05 smithi064 ceph-mon[212750]: osdmap e575: 8 total, 8 up, 7 in 2023-10-09T06:16:05.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:05 smithi064 ceph-mon[212750]: failed to encode map e575 with expected crc 2023-10-09T06:16:05.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:05 smithi064 ceph-mon[212750]: failed to encode map e575 with expected crc 2023-10-09T06:16:05.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:05 smithi064 ceph-mon[212750]: failed to encode map e575 with expected crc 2023-10-09T06:16:05.743 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:05.794 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:16:05.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:05.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:16:05.900 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:16:05.930 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:06.079 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:16:06.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:06.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:16:06.183 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:06.213 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:06.221 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:16:06.221 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:16:06.221 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 0 2023-10-09T06:16:06.221 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:16:06.359 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:06.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:06.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:06.463 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:06.490 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:06.686 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:06.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:06.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:06.789 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:16:06.819 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:06.971 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:16:06.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:06.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:16:07.007 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:06 smithi156 ceph-mon[195101]: pgmap v1461: 119 pgs: 1 active+undersized+remapped, 1 remapped+peering, 12 unknown, 105 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 8 B/s, 2 keys/s, 0 objects/s recovering 2023-10-09T06:16:07.007 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:06 smithi156 ceph-mon[195101]: failed to encode map e576 with expected crc 2023-10-09T06:16:07.007 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:06 smithi156 ceph-mon[195101]: failed to encode map e576 with expected crc 2023-10-09T06:16:07.008 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:06 smithi156 ceph-mon[195101]: failed to encode map e576 with expected crc 2023-10-09T06:16:07.008 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:06 smithi156 ceph-mon[195101]: osdmap e576: 8 total, 8 up, 7 in 2023-10-09T06:16:07.008 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:06 smithi156 ceph-mon[195101]: failed to encode map e576 with expected crc 2023-10-09T06:16:07.008 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:06 smithi156 ceph-mon[195101]: failed to encode map e576 with expected crc 2023-10-09T06:16:07.008 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:06 smithi156 ceph-mon[195101]: failed to encode map e576 with expected crc 2023-10-09T06:16:07.079 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:16:07.108 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:06 smithi064 ceph-mon[220044]: pgmap v1461: 119 pgs: 1 active+undersized+remapped, 1 remapped+peering, 12 unknown, 105 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 8 B/s, 2 keys/s, 0 objects/s recovering 2023-10-09T06:16:07.108 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:06 smithi064 ceph-mon[220044]: failed to encode map e576 with expected crc 2023-10-09T06:16:07.108 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:06 smithi064 ceph-mon[220044]: failed to encode map e576 with expected crc 2023-10-09T06:16:07.108 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:06 smithi064 ceph-mon[220044]: failed to encode map e576 with expected crc 2023-10-09T06:16:07.108 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:06 smithi064 ceph-mon[220044]: osdmap e576: 8 total, 8 up, 7 in 2023-10-09T06:16:07.108 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:06 smithi064 ceph-mon[220044]: failed to encode map e576 with expected crc 2023-10-09T06:16:07.109 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:06 smithi064 ceph-mon[220044]: failed to encode map e576 with expected crc 2023-10-09T06:16:07.109 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:06 smithi064 ceph-mon[220044]: failed to encode map e576 with expected crc 2023-10-09T06:16:07.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:06 smithi064 ceph-mon[212750]: pgmap v1461: 119 pgs: 1 active+undersized+remapped, 1 remapped+peering, 12 unknown, 105 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 8 B/s, 2 keys/s, 0 objects/s recovering 2023-10-09T06:16:07.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:06 smithi064 ceph-mon[212750]: failed to encode map e576 with expected crc 2023-10-09T06:16:07.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:06 smithi064 ceph-mon[212750]: failed to encode map e576 with expected crc 2023-10-09T06:16:07.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:06 smithi064 ceph-mon[212750]: failed to encode map e576 with expected crc 2023-10-09T06:16:07.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:06 smithi064 ceph-mon[212750]: osdmap e576: 8 total, 8 up, 7 in 2023-10-09T06:16:07.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:06 smithi064 ceph-mon[212750]: failed to encode map e576 with expected crc 2023-10-09T06:16:07.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:06 smithi064 ceph-mon[212750]: failed to encode map e576 with expected crc 2023-10-09T06:16:07.110 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:06 smithi064 ceph-mon[212750]: failed to encode map e576 with expected crc 2023-10-09T06:16:07.112 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:07.276 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:16:07.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:07.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:16:07.379 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:16:07.408 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:07.477 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:16:07.537 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:16:07.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:07.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:16:07.634 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:16:07.640 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:16:07.653 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:16:07.670 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:07.671 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:16:07.691 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:16:07.710 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:16:07.729 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:16:07.749 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:16:07.769 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:16:07.770 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:16:07.770 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:16:07.770 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:16:07.770 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:16:07.771 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:16:07.771 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:16:07.771 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:16:07.771 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:16:07.819 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:16:07.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:07.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:16:07.918 INFO:teuthology.orchestra.run.smithi064.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-10-09T06:16:07.923 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:16:07.938 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:16:07.954 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:07.998 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:16:08.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:07 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:08.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:07 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:08.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:07 smithi156 ceph-mon[195101]: pgmap v1463: 119 pgs: 1 peering, 1 active+undersized+remapped, 117 active+clean; 582 KiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 7 B/s, 2 keys/s, 0 objects/s recovering 2023-10-09T06:16:08.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:07 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3150154343' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:16:08.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:07 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/846006262' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:16:08.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:07 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:16:08.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:07 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:08.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:07 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:08.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:07 smithi064 ceph-mon[212750]: pgmap v1463: 119 pgs: 1 peering, 1 active+undersized+remapped, 117 active+clean; 582 KiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 7 B/s, 2 keys/s, 0 objects/s recovering 2023-10-09T06:16:08.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:07 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3150154343' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:16:08.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:07 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/846006262' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:16:08.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:07 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:16:08.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:07 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:08.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:07 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:08.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:07 smithi064 ceph-mon[220044]: pgmap v1463: 119 pgs: 1 peering, 1 active+undersized+remapped, 117 active+clean; 582 KiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 7 B/s, 2 keys/s, 0 objects/s recovering 2023-10-09T06:16:08.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:07 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3150154343' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:16:08.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:07 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/846006262' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:16:08.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:07 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:16:08.554 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:16:08.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:08.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:16:08.658 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:16:08.688 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:08.836 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:16:08.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:08.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:16:08.903 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 6, 7, 1] out_osds: [2] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 4, 0] 2023-10-09T06:16:08.903 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T06:16:08.940 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:16:08.978 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:09.035 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:16:09.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:09.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:16:09.081 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:09.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:08 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:16:09.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:08 smithi156 ceph-mon[195101]: failed to encode map e577 with expected crc 2023-10-09T06:16:09.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:08 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:16:09.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:08 smithi156 ceph-mon[195101]: osdmap e577: 8 total, 8 up, 7 in 2023-10-09T06:16:09.142 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:09.170 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:09.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:08 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:16:09.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:08 smithi064 ceph-mon[212750]: failed to encode map e577 with expected crc 2023-10-09T06:16:09.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:08 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:16:09.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:08 smithi064 ceph-mon[212750]: osdmap e577: 8 total, 8 up, 7 in 2023-10-09T06:16:09.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:08 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:16:09.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:08 smithi064 ceph-mon[220044]: failed to encode map e577 with expected crc 2023-10-09T06:16:09.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:08 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:16:09.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:08 smithi064 ceph-mon[220044]: osdmap e577: 8 total, 8 up, 7 in 2023-10-09T06:16:09.419 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:09.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:09.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:09.523 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:16:09.552 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:09.845 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:16:09.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:09.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:16:09.950 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:16:09.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:09 smithi064 ceph-mon[220044]: failed to encode map e577 with expected crc 2023-10-09T06:16:09.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:09 smithi064 ceph-mon[220044]: pgmap v1465: 119 pgs: 1 peering, 1 active+undersized+remapped, 117 active+clean; 582 KiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 6 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T06:16:09.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:09 smithi064 ceph-mon[212750]: failed to encode map e577 with expected crc 2023-10-09T06:16:09.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:09 smithi064 ceph-mon[212750]: pgmap v1465: 119 pgs: 1 peering, 1 active+undersized+remapped, 117 active+clean; 582 KiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 6 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T06:16:09.980 DEBUG:teuthology.orchestra.run.smithi064:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-10-09T06:16:09.980 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:09.982 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T06:16:09.982 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T06:16:10.036 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:hints = 1 2023-10-09T06:16:10.036 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T06:16:10.036 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object prefix: benchmark_data_smithi064_631903 2023-10-09T06:16:10.036 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:16:10.036 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 0 0 0 0 0 0 - 0 2023-10-09T06:16:10.106 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:09 smithi156 ceph-mon[195101]: failed to encode map e577 with expected crc 2023-10-09T06:16:10.106 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:09 smithi156 ceph-mon[195101]: pgmap v1465: 119 pgs: 1 peering, 1 active+undersized+remapped, 117 active+clean; 582 KiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 6 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T06:16:10.119 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:10.143 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:16:10.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:10.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:16:10.250 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:16:10.321 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:10.396 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:16:10.396 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:16:10.427 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:16:10.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:10.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:16:10.535 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:16:10.584 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:10.661 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:16:10.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:10.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:16:10.767 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:16:10.793 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:10.836 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:16:10.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:10.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:16:10.939 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:16:10.965 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:11.006 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:16:11.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:11.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:16:11.019 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:16:11.036 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 1 16 2336 2320 144.989 145 0.00185166 0.00487137 2023-10-09T06:16:11.110 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:16:11.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:10 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:11.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:10 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:11.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:10 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:16:11.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:10 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:16:11.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:10 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:11.214 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:10 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:11.214 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:10 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:11.215 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:10 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:16:11.215 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:10 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:16:11.215 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:10 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:11.215 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:10 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:11.215 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:10 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:11.215 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:10 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:16:11.215 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:10 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:16:11.215 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:10 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:11.219 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:11.331 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:16:11.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:11.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:16:11.437 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:11.454 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:16:11.487 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:11.555 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:11.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:11.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:11.664 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:16:11.713 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:11.811 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:16:11.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:11.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:16:11.919 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:16:11.977 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:16:11.992 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:12.036 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 2 16 5313 5297 165.518 186.062 0.0132977 0.00602515 2023-10-09T06:16:12.077 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:16:12.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:11 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4283979159' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:16:12.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:11 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:16:12.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:11 smithi156 ceph-mon[195101]: pgmap v1466: 119 pgs: 119 active+clean; 61 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 9.3 MiB/s wr, 149 op/s 2023-10-09T06:16:12.214 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:11 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4283979159' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:16:12.214 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:11 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:16:12.214 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:11 smithi064 ceph-mon[212750]: pgmap v1466: 119 pgs: 119 active+clean; 61 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 9.3 MiB/s wr, 149 op/s 2023-10-09T06:16:12.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:11 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4283979159' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:16:12.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:11 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:16:12.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:11 smithi064 ceph-mon[220044]: pgmap v1466: 119 pgs: 119 active+clean; 61 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 9.3 MiB/s wr, 149 op/s 2023-10-09T06:16:12.251 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:16:12.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:12.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:16:12.359 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:12.430 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:12.572 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:12.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:12.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:12.683 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:16:12.737 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:12.820 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:16:12.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:12.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:16:12.925 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:12.972 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:13.036 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 3 16 8504 8488 176.819 199.438 0.00980204 0.00564901 2023-10-09T06:16:13.054 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:13.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:13.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:13.159 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:16:13.200 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:13.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:12 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:16:13.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:12 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:16:13.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:12 smithi064 ceph-mon[212750]: osdmap e578: 8 total, 8 up, 7 in 2023-10-09T06:16:13.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:12 smithi064 ceph-mon[212750]: failed to encode map e578 with expected crc 2023-10-09T06:16:13.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:12 smithi064 ceph-mon[212750]: failed to encode map e578 with expected crc 2023-10-09T06:16:13.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:12 smithi064 ceph-mon[212750]: failed to encode map e578 with expected crc 2023-10-09T06:16:13.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:12 smithi064 ceph-mon[212750]: failed to encode map e578 with expected crc 2023-10-09T06:16:13.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:12 smithi064 ceph-mon[212750]: failed to encode map e578 with expected crc 2023-10-09T06:16:13.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:12 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:16:13.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:12 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:16:13.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:12 smithi064 ceph-mon[220044]: osdmap e578: 8 total, 8 up, 7 in 2023-10-09T06:16:13.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:12 smithi064 ceph-mon[220044]: failed to encode map e578 with expected crc 2023-10-09T06:16:13.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:12 smithi064 ceph-mon[220044]: failed to encode map e578 with expected crc 2023-10-09T06:16:13.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:12 smithi064 ceph-mon[220044]: failed to encode map e578 with expected crc 2023-10-09T06:16:13.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:12 smithi064 ceph-mon[220044]: failed to encode map e578 with expected crc 2023-10-09T06:16:13.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:12 smithi064 ceph-mon[220044]: failed to encode map e578 with expected crc 2023-10-09T06:16:13.266 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:16:13.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:13.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:16:13.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:12 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:16:13.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:12 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:16:13.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:12 smithi156 ceph-mon[195101]: osdmap e578: 8 total, 8 up, 7 in 2023-10-09T06:16:13.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:12 smithi156 ceph-mon[195101]: failed to encode map e578 with expected crc 2023-10-09T06:16:13.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:12 smithi156 ceph-mon[195101]: failed to encode map e578 with expected crc 2023-10-09T06:16:13.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:12 smithi156 ceph-mon[195101]: failed to encode map e578 with expected crc 2023-10-09T06:16:13.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:12 smithi156 ceph-mon[195101]: failed to encode map e578 with expected crc 2023-10-09T06:16:13.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:12 smithi156 ceph-mon[195101]: failed to encode map e578 with expected crc 2023-10-09T06:16:13.372 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:16:13.425 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:13.751 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:13.763 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:16:13.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:13.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:16:13.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:16:13.928 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:14.037 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 4 16 11201 11185 174.745 168.562 0.00527051 0.00571606 2023-10-09T06:16:14.135 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:16:14.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:14.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:16:14.241 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:16:14.300 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:14.443 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:16:14.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:14.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:16:14.544 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:14 smithi156 ceph-mon[195101]: pgmap v1468: 119 pgs: 119 active+clean; 129 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 17 MiB/s wr, 273 op/s 2023-10-09T06:16:14.544 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:14.544 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:14.544 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:14.547 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:16:14.607 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:14.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:14 smithi064 ceph-mon[212750]: pgmap v1468: 119 pgs: 119 active+clean; 129 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 17 MiB/s wr, 273 op/s 2023-10-09T06:16:14.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:14.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:14.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:14.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:14 smithi064 ceph-mon[220044]: pgmap v1468: 119 pgs: 119 active+clean; 129 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 17 MiB/s wr, 273 op/s 2023-10-09T06:16:14.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:14.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:14.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:14.718 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:16:14.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:14.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:16:14.823 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:16:14.876 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:14.949 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:16:14.950 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:16:14.966 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:16:14.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:14.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:16:15.037 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 5 16 13823 13807 172.568 163.875 0.00725711 0.00578889 2023-10-09T06:16:15.072 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:16:15.121 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:15.270 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:16:15.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:15.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:16:15.377 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:16:15.432 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:15.530 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:16:15.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:15.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:16:15.545 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:15 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4013453723' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:16:15.545 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:15 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:16:15.562 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T06:16:15.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:15 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4013453723' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:16:15.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:15 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:16:15.592 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:15 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4013453723' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:16:15.592 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:15 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:16:15.592 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:16:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:16:15] "GET /metrics HTTP/1.1" 200 36436 "" "Prometheus/2.43.0" 2023-10-09T06:16:15.636 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:15.684 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:15.746 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:15.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:15.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:15.777 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:16:15.849 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:16:15.892 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:15.957 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:16:15.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:15.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:16:16.037 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 6 16 16169 16153 168.242 146.625 0.00193231 0.00585057 2023-10-09T06:16:16.073 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:16:16.140 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:16.482 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:16:16.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:16.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:16:16.566 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:16:16.587 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:16:16.613 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:16.656 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:16:16.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:16.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:16:16.762 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:16:16.790 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:16 smithi064 ceph-mon[212750]: pgmap v1469: 119 pgs: 119 active+clean; 323 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 40 MiB/s wr, 643 op/s 2023-10-09T06:16:16.791 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:16 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:16:16.791 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:16 smithi064 ceph-mon[212750]: osdmap e579: 8 total, 8 up, 7 in 2023-10-09T06:16:16.791 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:16 smithi064 ceph-mon[212750]: failed to encode map e579 with expected crc 2023-10-09T06:16:16.791 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:16 smithi064 ceph-mon[212750]: failed to encode map e579 with expected crc 2023-10-09T06:16:16.791 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:16 smithi064 ceph-mon[212750]: failed to encode map e579 with expected crc 2023-10-09T06:16:16.791 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:16 smithi064 ceph-mon[212750]: failed to encode map e579 with expected crc 2023-10-09T06:16:16.791 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:16 smithi064 ceph-mon[212750]: failed to encode map e579 with expected crc 2023-10-09T06:16:16.791 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:16 smithi064 ceph-mon[212750]: failed to encode map e579 with expected crc 2023-10-09T06:16:16.792 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:16 smithi064 ceph-mon[220044]: pgmap v1469: 119 pgs: 119 active+clean; 323 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 40 MiB/s wr, 643 op/s 2023-10-09T06:16:16.792 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:16 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:16:16.792 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:16 smithi064 ceph-mon[220044]: osdmap e579: 8 total, 8 up, 7 in 2023-10-09T06:16:16.792 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:16 smithi064 ceph-mon[220044]: failed to encode map e579 with expected crc 2023-10-09T06:16:16.792 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:16 smithi064 ceph-mon[220044]: failed to encode map e579 with expected crc 2023-10-09T06:16:16.792 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:16 smithi064 ceph-mon[220044]: failed to encode map e579 with expected crc 2023-10-09T06:16:16.792 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:16 smithi064 ceph-mon[220044]: failed to encode map e579 with expected crc 2023-10-09T06:16:16.792 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:16 smithi064 ceph-mon[220044]: failed to encode map e579 with expected crc 2023-10-09T06:16:16.792 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:16 smithi064 ceph-mon[220044]: failed to encode map e579 with expected crc 2023-10-09T06:16:16.795 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:16 smithi156 ceph-mon[195101]: pgmap v1469: 119 pgs: 119 active+clean; 323 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 40 MiB/s wr, 643 op/s 2023-10-09T06:16:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:16 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:16:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:16 smithi156 ceph-mon[195101]: osdmap e579: 8 total, 8 up, 7 in 2023-10-09T06:16:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:16 smithi156 ceph-mon[195101]: failed to encode map e579 with expected crc 2023-10-09T06:16:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:16 smithi156 ceph-mon[195101]: failed to encode map e579 with expected crc 2023-10-09T06:16:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:16 smithi156 ceph-mon[195101]: failed to encode map e579 with expected crc 2023-10-09T06:16:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:16 smithi156 ceph-mon[195101]: failed to encode map e579 with expected crc 2023-10-09T06:16:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:16 smithi156 ceph-mon[195101]: failed to encode map e579 with expected crc 2023-10-09T06:16:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:16 smithi156 ceph-mon[195101]: failed to encode map e579 with expected crc 2023-10-09T06:16:17.037 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 7 16 17964 17948 160.233 112.188 0.0111158 0.00622635 2023-10-09T06:16:17.147 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:16:17.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:17.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:16:17.253 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:16:17.266 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T06:16:17.317 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:17.494 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:16:17.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:17.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:16:17.600 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:16:17.618 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:17.633 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:17.635 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:17.695 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:16:17.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:17.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:16:17.792 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:16:17.859 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:16:17.861 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:16:17.871 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:16:17.902 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:17.914 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:16:17.952 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:16:18.083 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:16:18.083 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:16:18.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:18.084 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 8 16 19881 19865 155.164 119.812 0.00962281 0.00643895 2023-10-09T06:16:18.084 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:16:18.084 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:16:18.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:16:18.085 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:16:18.085 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:16:18.085 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:16:18.085 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:16:18.085 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:16:18.085 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:16:18.085 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:16:18.085 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:16:18.185 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:16:18.243 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:19.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 9 16 22255 22239 154.408 148.375 0.00571815 0.00647088 2023-10-09T06:16:19.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:18 smithi156 ceph-mon[195101]: pgmap v1471: 119 pgs: 119 active+clean; 948 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T06:16:19.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:18 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1468822598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:16:19.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:18 smithi064 ceph-mon[212750]: pgmap v1471: 119 pgs: 119 active+clean; 948 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T06:16:19.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:18 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1468822598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:16:19.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:18 smithi064 ceph-mon[220044]: pgmap v1471: 119 pgs: 119 active+clean; 948 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T06:16:19.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:18 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1468822598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:16:20.007 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:16:20.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:20.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:16:20.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 10 16 24520 24504 153.122 141.562 0.00794702 0.00652617 2023-10-09T06:16:20.128 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:16:20.205 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:21.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 11 16 26318 26302 149.417 112.375 0.00552213 0.00668694 2023-10-09T06:16:21.062 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:20 smithi064 ceph-mon[220044]: pgmap v1472: 119 pgs: 119 active+clean; 948 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-09T06:16:21.062 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:20 smithi064 ceph-mon[212750]: pgmap v1472: 119 pgs: 119 active+clean; 948 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-09T06:16:21.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:20 smithi156 ceph-mon[195101]: pgmap v1472: 119 pgs: 119 active+clean; 948 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-09T06:16:21.188 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:16:21.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:21.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:16:21.300 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:16:21.364 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:21.864 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:16:21.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:21.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:16:21.969 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:21.972 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:16:22.019 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:22.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 12 16 28079 28063 146.137 110.062 0.0106829 0.00683632 2023-10-09T06:16:22.099 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:16:22.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:22.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:16:22.206 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:22.257 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:22.328 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:22.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:22.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:22.436 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:16:22.476 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:22.558 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:16:22.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:22.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:16:22.666 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:16:22.706 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:22.780 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:16:22.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:22.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:16:22.886 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:22.938 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:22.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:22 smithi064 ceph-mon[212750]: pgmap v1473: 119 pgs: 119 active+clean; 1.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 140 MiB/s wr, 2.24k op/s 2023-10-09T06:16:22.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:22 smithi064 ceph-mon[220044]: pgmap v1473: 119 pgs: 119 active+clean; 1.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 140 MiB/s wr, 2.24k op/s 2023-10-09T06:16:23.029 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:23.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:23.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:23.037 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:22 smithi156 ceph-mon[195101]: pgmap v1473: 119 pgs: 119 active+clean; 1.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 140 MiB/s wr, 2.24k op/s 2023-10-09T06:16:23.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 13 16 29935 29919 143.814 116 0.00342023 0.00694945 2023-10-09T06:16:23.100 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:16:23.138 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:16:23.210 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:23.526 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:16:23.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:23.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:16:23.633 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:16:23.659 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:23.701 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:16:23.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:23.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:16:23.792 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:16:23.807 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:16:23.826 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:23 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1850124204' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:16:23.826 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:23 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1850124204' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:16:23.864 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:24.039 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 14 16 31307 31291 139.666 85.75 0.0151563 0.0071544 2023-10-09T06:16:24.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:23 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1850124204' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:16:24.162 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:16:24.163 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:16:24.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.163 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36591.0:31430 11.f e580/572 11:f9ffb9f3:::benchmark_data_smithi064_631903_object31429:head v 580'2034, mlcod=580'2031)", 2023-10-09T06:16:24.163 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:16:24.159859+0000", 2023-10-09T06:16:24.163 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0022432720000000001, 2023-10-09T06:16:24.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002274107, 2023-10-09T06:16:24.163 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:16:24.163 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:16:24.163 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:16:24.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:16:24.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.159859+0000", 2023-10-09T06:16:24.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:16:24.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:16:24.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.159859+0000", 2023-10-09T06:16:24.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:16:24.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:16:24.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.159849+0000", 2023-10-09T06:16:24.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999905 2023-10-09T06:16:24.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:16:24.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.159901+0000", 2023-10-09T06:16:24.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1762999999999999e-05 2023-10-09T06:16:24.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:16:24.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.159923+0000", 2023-10-09T06:16:24.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2433e-05 2023-10-09T06:16:24.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:16:24.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.159933+0000", 2023-10-09T06:16:24.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0168e-05 2023-10-09T06:16:24.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:16:24.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.161996+0000", 2023-10-09T06:16:24.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020629580000000002 2023-10-09T06:16:24.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.167 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:16:24.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.162022+0000", 2023-10-09T06:16:24.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5826e-05 2023-10-09T06:16:24.167 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:24.167 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:16:24.167 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:24.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.167 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36591.0:31432 11.9 e580/572 11:9d771959:::benchmark_data_smithi064_631903_object31431:head v 580'1947, mlcod=580'1944)", 2023-10-09T06:16:24.167 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:16:24.160572+0000", 2023-10-09T06:16:24.167 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0015300229999999999, 2023-10-09T06:16:24.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016326719999999999, 2023-10-09T06:16:24.168 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:16:24.168 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:16:24.168 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:16:24.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:16:24.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.160572+0000", 2023-10-09T06:16:24.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:16:24.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:16:24.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.160572+0000", 2023-10-09T06:16:24.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:16:24.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:16:24.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.160571+0000", 2023-10-09T06:16:24.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:16:24.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:16:24.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.160603+0000", 2023-10-09T06:16:24.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2295999999999997e-05 2023-10-09T06:16:24.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:16:24.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.160649+0000", 2023-10-09T06:16:24.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6196000000000003e-05 2023-10-09T06:16:24.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:16:24.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.160659+0000", 2023-10-09T06:16:24.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7680000000000003e-06 2023-10-09T06:16:24.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:16:24.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.160706+0000", 2023-10-09T06:16:24.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7172999999999997e-05 2023-10-09T06:16:24.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:16:24.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.160732+0000", 2023-10-09T06:16:24.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5230000000000001e-05 2023-10-09T06:16:24.171 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:24.171 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:16:24.171 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:24.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.172 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36591.0:31424 11.6 e580/572 11:67b6a5d3:::benchmark_data_smithi064_631903_object31423:head v 580'1943, mlcod=580'1941)", 2023-10-09T06:16:24.172 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:16:24.160615+0000", 2023-10-09T06:16:24.172 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0014864749999999999, 2023-10-09T06:16:24.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016517179999999999, 2023-10-09T06:16:24.172 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:16:24.172 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:16:24.172 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:16:24.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:16:24.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.160615+0000", 2023-10-09T06:16:24.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:16:24.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:16:24.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.160615+0000", 2023-10-09T06:16:24.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:16:24.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:16:24.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.160614+0000", 2023-10-09T06:16:24.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:16:24.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:16:24.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.160651+0000", 2023-10-09T06:16:24.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6981999999999999e-05 2023-10-09T06:16:24.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:16:24.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.160665+0000", 2023-10-09T06:16:24.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3507e-05 2023-10-09T06:16:24.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:16:24.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.160673+0000", 2023-10-09T06:16:24.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6209999999999993e-06 2023-10-09T06:16:24.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:16:24.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.160754+0000", 2023-10-09T06:16:24.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0443000000000002e-05 2023-10-09T06:16:24.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:16:24.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.160780+0000", 2023-10-09T06:16:24.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5709999999999999e-05 2023-10-09T06:16:24.175 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:24.175 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:16:24.176 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:24.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.176 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36591.0:31427 11.1 e580/572 11:8b70bcc6:::benchmark_data_smithi064_631903_object31426:head v 580'1913, mlcod=580'1911)", 2023-10-09T06:16:24.178 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:16:24.160678+0000", 2023-10-09T06:16:24.178 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.001423854, 2023-10-09T06:16:24.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016516969999999999, 2023-10-09T06:16:24.178 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:16:24.178 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:16:24.178 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:16:24.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:16:24.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.160678+0000", 2023-10-09T06:16:24.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:16:24.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:16:24.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.160678+0000", 2023-10-09T06:16:24.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:16:24.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:16:24.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.160678+0000", 2023-10-09T06:16:24.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:16:24.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:16:24.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.160704+0000", 2023-10-09T06:16:24.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6784e-05 2023-10-09T06:16:24.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:16:24.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.160713+0000", 2023-10-09T06:16:24.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6649999999999992e-06 2023-10-09T06:16:24.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:16:24.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.160719+0000", 2023-10-09T06:16:24.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0850000000000002e-06 2023-10-09T06:16:24.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:16:24.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.160788+0000", 2023-10-09T06:16:24.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8640999999999996e-05 2023-10-09T06:16:24.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:24.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:24.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:16:24.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:24.160809+0000", 2023-10-09T06:16:24.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1733e-05 2023-10-09T06:16:24.182 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:24.182 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:16:24.182 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:24.182 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:24.182 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-10-09T06:16:24.182 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 4 2023-10-09T06:16:24.182 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:16:24.389 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:16:24.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:24.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:16:24.501 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:16:24.555 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:24.648 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:16:24.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:24.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:16:24.757 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:16:24.783 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:24.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:24 smithi156 ceph-mon[195101]: pgmap v1474: 119 pgs: 119 active+clean; 1.5 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 146 MiB/s wr, 2.33k op/s 2023-10-09T06:16:24.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:24 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:16:24.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:24 smithi156 ceph-mon[195101]: failed to encode map e580 with expected crc 2023-10-09T06:16:24.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:24 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1850124204' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:16:24.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:24 smithi156 ceph-mon[195101]: osdmap e580: 8 total, 8 up, 7 in 2023-10-09T06:16:24.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:24 smithi156 ceph-mon[195101]: failed to encode map e580 with expected crc 2023-10-09T06:16:24.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:24 smithi156 ceph-mon[195101]: failed to encode map e580 with expected crc 2023-10-09T06:16:24.875 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:16:24.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:24.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:16:24.981 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:16:25.015 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:25.039 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 15 16 32956 32940 137.226 103.062 0.00374064 0.00726029 2023-10-09T06:16:25.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:24 smithi064 ceph-mon[212750]: pgmap v1474: 119 pgs: 119 active+clean; 1.5 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 146 MiB/s wr, 2.33k op/s 2023-10-09T06:16:25.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:24 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:16:25.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:24 smithi064 ceph-mon[212750]: failed to encode map e580 with expected crc 2023-10-09T06:16:25.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:24 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1850124204' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:16:25.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:24 smithi064 ceph-mon[212750]: osdmap e580: 8 total, 8 up, 7 in 2023-10-09T06:16:25.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:24 smithi064 ceph-mon[212750]: failed to encode map e580 with expected crc 2023-10-09T06:16:25.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:24 smithi064 ceph-mon[212750]: failed to encode map e580 with expected crc 2023-10-09T06:16:25.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:24 smithi064 ceph-mon[220044]: pgmap v1474: 119 pgs: 119 active+clean; 1.5 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 146 MiB/s wr, 2.33k op/s 2023-10-09T06:16:25.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:24 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:16:25.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:24 smithi064 ceph-mon[220044]: failed to encode map e580 with expected crc 2023-10-09T06:16:25.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:24 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1850124204' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:16:25.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:24 smithi064 ceph-mon[220044]: osdmap e580: 8 total, 8 up, 7 in 2023-10-09T06:16:25.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:24 smithi064 ceph-mon[220044]: failed to encode map e580 with expected crc 2023-10-09T06:16:25.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:24 smithi064 ceph-mon[220044]: failed to encode map e580 with expected crc 2023-10-09T06:16:25.720 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:16:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:16:25] "GET /metrics HTTP/1.1" 200 36436 "" "Prometheus/2.43.0" 2023-10-09T06:16:26.045 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 16 16 34614 34598 135.122 103.625 0.00607792 0.00739591 2023-10-09T06:16:26.193 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:16:26.201 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:16:26.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:26.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:16:26.310 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:16:26.375 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:26.385 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:26.598 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:16:26.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:26.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:16:26.704 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:26.746 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:26.825 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:26.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:26.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:26.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:26 smithi064 ceph-mon[220044]: pgmap v1476: 119 pgs: 119 active+clean; 1.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 145 MiB/s wr, 2.32k op/s 2023-10-09T06:16:26.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:26 smithi064 ceph-mon[212750]: pgmap v1476: 119 pgs: 119 active+clean; 1.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 145 MiB/s wr, 2.32k op/s 2023-10-09T06:16:26.933 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:16:27.004 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:27.039 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 17 16 36135 36119 132.766 95.0625 0.00928698 0.0075258 2023-10-09T06:16:27.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:26 smithi156 ceph-mon[195101]: pgmap v1476: 119 pgs: 119 active+clean; 1.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 145 MiB/s wr, 2.32k op/s 2023-10-09T06:16:27.384 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:16:27.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:27.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:16:27.491 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:16:27.552 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:27.638 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:16:27.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:27.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:16:27.743 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:16:27.769 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:27.808 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T06:16:27.813 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:16:27.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:27.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:16:27.891 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:16:27.919 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:16:27.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:27 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1687873247' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:16:27.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:27 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:16:27.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:27 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1687873247' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:16:27.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:27 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:16:27.962 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:28.043 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 18 16 37544 37528 130.282 88.0625 0.0114743 0.00766898 2023-10-09T06:16:28.043 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:16:28.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:28.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:16:28.051 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:27 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1687873247' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:16:28.051 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:27 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:16:28.152 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:16:28.223 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:28.937 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:16:28.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:28.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:16:29.015 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:29.022 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T06:16:29.043 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 19 16 39069 39053 128.42 95.3125 0.0218183 0.00778151 2023-10-09T06:16:29.044 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:16:29.113 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:28 smithi064 ceph-mon[212750]: pgmap v1477: 119 pgs: 119 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-10-09T06:16:29.113 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:28 smithi064 ceph-mon[212750]: failed to encode map e581 with expected crc 2023-10-09T06:16:29.113 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:28 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:16:29.113 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:28 smithi064 ceph-mon[212750]: osdmap e581: 8 total, 8 up, 7 in 2023-10-09T06:16:29.113 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:28 smithi064 ceph-mon[212750]: failed to encode map e581 with expected crc 2023-10-09T06:16:29.113 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:28 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:29.113 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:28 smithi064 ceph-mon[220044]: pgmap v1477: 119 pgs: 119 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-10-09T06:16:29.114 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:28 smithi064 ceph-mon[220044]: failed to encode map e581 with expected crc 2023-10-09T06:16:29.114 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:28 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:16:29.114 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:28 smithi064 ceph-mon[220044]: osdmap e581: 8 total, 8 up, 7 in 2023-10-09T06:16:29.114 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:28 smithi064 ceph-mon[220044]: failed to encode map e581 with expected crc 2023-10-09T06:16:29.114 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:28 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:29.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:28 smithi156 ceph-mon[195101]: pgmap v1477: 119 pgs: 119 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-10-09T06:16:29.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:28 smithi156 ceph-mon[195101]: failed to encode map e581 with expected crc 2023-10-09T06:16:29.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:28 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:16:29.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:28 smithi156 ceph-mon[195101]: osdmap e581: 8 total, 8 up, 7 in 2023-10-09T06:16:29.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:28 smithi156 ceph-mon[195101]: failed to encode map e581 with expected crc 2023-10-09T06:16:29.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:28 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:29.135 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:29.236 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:29.476 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:16:29.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:29.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:16:29.585 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:16:29.658 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:29.798 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:16:29.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:29.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:16:29.906 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:16:29.948 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:16:29.982 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:29.997 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:16:30.040 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:16:30.043 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-10-09T06:16:30.043752+0000 min lat: 0.00113085 max lat: 0.307063 avg lat: 0.00792042 2023-10-09T06:16:30.043 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:16:30.043 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 20 16 40399 40383 126.155 83.125 0.0210764 0.00792042 2023-10-09T06:16:30.075 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:29 smithi064 ceph-mon[212750]: failed to encode map e581 with expected crc 2023-10-09T06:16:30.075 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:29 smithi064 ceph-mon[212750]: failed to encode map e581 with expected crc 2023-10-09T06:16:30.076 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:29 smithi064 ceph-mon[220044]: failed to encode map e581 with expected crc 2023-10-09T06:16:30.076 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:29 smithi064 ceph-mon[220044]: failed to encode map e581 with expected crc 2023-10-09T06:16:30.078 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:16:30.114 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:16:30.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:29 smithi156 ceph-mon[195101]: failed to encode map e581 with expected crc 2023-10-09T06:16:30.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:29 smithi156 ceph-mon[195101]: failed to encode map e581 with expected crc 2023-10-09T06:16:30.151 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:16:30.189 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:16:30.224 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:16:30.236 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:16:30.237 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:16:30.237 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:16:30.237 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:16:30.237 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:16:30.237 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:16:30.237 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:16:30.237 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:16:31.043 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 21 16 41883 41867 124.564 92.75 0.00196593 0.00799516 2023-10-09T06:16:31.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:30 smithi156 ceph-mon[195101]: pgmap v1479: 119 pgs: 119 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 87 MiB/s wr, 1.38k op/s 2023-10-09T06:16:31.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:30 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3388128646' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:16:31.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:30 smithi064 ceph-mon[212750]: pgmap v1479: 119 pgs: 119 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 87 MiB/s wr, 1.38k op/s 2023-10-09T06:16:31.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:30 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3388128646' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:16:31.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:30 smithi064 ceph-mon[220044]: pgmap v1479: 119 pgs: 119 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 87 MiB/s wr, 1.38k op/s 2023-10-09T06:16:31.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:30 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3388128646' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:16:31.311 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:16:31.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:31.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:16:31.418 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:16:31.453 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:31.535 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:16:31.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:31.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:16:31.643 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:31.705 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:31.758 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:31.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:31.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:31.863 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:16:31.908 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:31.995 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:16:32.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:32.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:16:32.043 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 22 16 43294 43278 122.91 88.1875 0.00857067 0.00813157 2023-10-09T06:16:32.103 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:16:32.169 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:33.043 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 23 16 44747 44731 121.515 90.8125 0.0118095 0.00822494 2023-10-09T06:16:33.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:32 smithi064 ceph-mon[212750]: pgmap v1480: 119 pgs: 119 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-10-09T06:16:33.121 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:32 smithi064 ceph-mon[220044]: pgmap v1480: 119 pgs: 119 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-10-09T06:16:33.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:32 smithi156 ceph-mon[195101]: pgmap v1480: 119 pgs: 119 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-10-09T06:16:33.318 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:33.346 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:16:33.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:33.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:16:33.454 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:16:33.498 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:33.567 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:16:33.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:33.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:16:33.680 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:16:33.728 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:33.789 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:16:33.789 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:16:33.801 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:16:33.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:33.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:16:33.909 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:16:33.979 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:34.043 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 24 16 46211 46195 120.264 91.5 0.0192247 0.00830779 2023-10-09T06:16:34.273 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:16:34.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:34.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:16:34.386 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:34.442 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:34.525 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:34.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:34.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:34.630 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:16:34.686 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:34.829 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:16:34.944 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:16:34.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:34 smithi064 ceph-mon[212750]: pgmap v1481: 119 pgs: 119 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T06:16:34.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:34 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/949426411' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:16:34.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:34 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:16:34.948 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:34 smithi064 ceph-mon[220044]: pgmap v1481: 119 pgs: 119 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T06:16:34.948 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:34 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/949426411' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:16:34.948 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:34 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:16:34.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:34.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:16:35.043 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 25 16 47607 47591 118.943 87.25 0.00713391 0.0084024 2023-10-09T06:16:35.051 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:35.109 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:34 smithi156 ceph-mon[195101]: pgmap v1481: 119 pgs: 119 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T06:16:35.109 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:34 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/949426411' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:16:35.109 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:34 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:16:35.115 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:35.123 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:16:35.205 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:35.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:35.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:35.313 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:16:35.377 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:35.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:16:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:16:35] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.43.0" 2023-10-09T06:16:36.014 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:16:36.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:36.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:16:36.043 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 26 16 49019 49003 117.763 88.25 0.00712083 0.00848609 2023-10-09T06:16:36.078 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:35 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:16:36.078 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:35 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:16:36.078 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:35 smithi064 ceph-mon[212750]: failed to encode map e582 with expected crc 2023-10-09T06:16:36.078 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:35 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:16:36.078 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:35 smithi064 ceph-mon[212750]: osdmap e582: 8 total, 8 up, 7 in 2023-10-09T06:16:36.078 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:35 smithi064 ceph-mon[212750]: failed to encode map e582 with expected crc 2023-10-09T06:16:36.078 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:35 smithi064 ceph-mon[212750]: failed to encode map e582 with expected crc 2023-10-09T06:16:36.079 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:35 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:16:36.079 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:35 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T06:16:36.079 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:35 smithi064 ceph-mon[220044]: failed to encode map e582 with expected crc 2023-10-09T06:16:36.079 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:35 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:16:36.079 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:35 smithi064 ceph-mon[220044]: osdmap e582: 8 total, 8 up, 7 in 2023-10-09T06:16:36.079 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:35 smithi064 ceph-mon[220044]: failed to encode map e582 with expected crc 2023-10-09T06:16:36.079 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:35 smithi064 ceph-mon[220044]: failed to encode map e582 with expected crc 2023-10-09T06:16:36.112 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:16:36.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:35 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:16:36.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:35 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T06:16:36.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:35 smithi156 ceph-mon[195101]: failed to encode map e582 with expected crc 2023-10-09T06:16:36.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:35 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:16:36.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:35 smithi156 ceph-mon[195101]: osdmap e582: 8 total, 8 up, 7 in 2023-10-09T06:16:36.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:35 smithi156 ceph-mon[195101]: failed to encode map e582 with expected crc 2023-10-09T06:16:36.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:35 smithi156 ceph-mon[195101]: failed to encode map e582 with expected crc 2023-10-09T06:16:36.124 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:16:36.158 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:36.216 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:16:36.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:36.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:16:36.325 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:16:36.352 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:36.716 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:16:36.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:36.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:16:36.822 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:16:36.892 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:37.005 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:16:37.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:37.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:16:37.009 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:36 smithi064 ceph-mon[220044]: failed to encode map e582 with expected crc 2023-10-09T06:16:37.009 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:36 smithi064 ceph-mon[220044]: failed to encode map e582 with expected crc 2023-10-09T06:16:37.010 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:36 smithi064 ceph-mon[220044]: pgmap v1483: 119 pgs: 119 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T06:16:37.010 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:36 smithi064 ceph-mon[212750]: failed to encode map e582 with expected crc 2023-10-09T06:16:37.010 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:36 smithi064 ceph-mon[212750]: failed to encode map e582 with expected crc 2023-10-09T06:16:37.010 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:36 smithi064 ceph-mon[212750]: pgmap v1483: 119 pgs: 119 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T06:16:37.043 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 27 16 50108 50092 115.922 68.0625 0.00546006 0.00861934 2023-10-09T06:16:37.110 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:16:37.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:36 smithi156 ceph-mon[195101]: failed to encode map e582 with expected crc 2023-10-09T06:16:37.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:36 smithi156 ceph-mon[195101]: failed to encode map e582 with expected crc 2023-10-09T06:16:37.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:36 smithi156 ceph-mon[195101]: pgmap v1483: 119 pgs: 119 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T06:16:37.166 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:37.193 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:37.304 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:16:37.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:37.309 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:37.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:16:37.411 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:16:37.449 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:37.520 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:16:37.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:37.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:16:37.628 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:16:37.695 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:38.044 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 28 16 51337 51321 114.525 76.8125 0.0176768 0.00872693 2023-10-09T06:16:38.062 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:16:38.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:38.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:16:38.169 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:16:38.229 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:38.301 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:16:38.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:38.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:16:38.407 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:16:38.468 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:38.762 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T06:16:38.904 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:16:39.044 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 29 16 52594 52578 113.286 78.5625 0.00821628 0.00882292 2023-10-09T06:16:39.084 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:38 smithi064 ceph-mon[212750]: pgmap v1484: 119 pgs: 119 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T06:16:39.084 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:38 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2973992148' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:16:39.084 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:38 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:16:39.084 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:38 smithi064 ceph-mon[220044]: pgmap v1484: 119 pgs: 119 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T06:16:39.085 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:38 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2973992148' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:16:39.085 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:38 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:16:39.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:38 smithi156 ceph-mon[195101]: pgmap v1484: 119 pgs: 119 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T06:16:39.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:38 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2973992148' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:16:39.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:38 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:16:39.123 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:16:39.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:39.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:16:39.232 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:39.259 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:39.302 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:39.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:39.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:39.411 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:16:39.474 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:40.006 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:16:40.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:40.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:16:40.012 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:39 smithi064 ceph-mon[212750]: failed to encode map e583 with expected crc 2023-10-09T06:16:40.012 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:39 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:16:40.012 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:39 smithi064 ceph-mon[212750]: osdmap e583: 8 total, 8 up, 7 in 2023-10-09T06:16:40.012 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:39 smithi064 ceph-mon[220044]: failed to encode map e583 with expected crc 2023-10-09T06:16:40.013 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:39 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:16:40.013 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:39 smithi064 ceph-mon[220044]: osdmap e583: 8 total, 8 up, 7 in 2023-10-09T06:16:40.044 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 30 16 53466 53450 111.326 54.5 0.00853953 0.00897712 2023-10-09T06:16:40.113 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:16:40.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:39 smithi156 ceph-mon[195101]: failed to encode map e583 with expected crc 2023-10-09T06:16:40.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:39 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:16:40.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:39 smithi156 ceph-mon[195101]: osdmap e583: 8 total, 8 up, 7 in 2023-10-09T06:16:40.172 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:40.277 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:16:40.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:40.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:16:40.372 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T06:16:40.387 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:16:40.484 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:40.630 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:40.769 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:16:40.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:40.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:16:40.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:16:40.953 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:41.045 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 31 16 54667 54651 110.153 75.0625 0.00854541 0.00907399 2023-10-09T06:16:41.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:40 smithi156 ceph-mon[195101]: pgmap v1486: 119 pgs: 119 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T06:16:41.135 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:16:41.135 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:16:41.136 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:40 smithi064 ceph-mon[220044]: pgmap v1486: 119 pgs: 119 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T06:16:41.139 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:40 smithi064 ceph-mon[212750]: pgmap v1486: 119 pgs: 119 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T06:16:41.317 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:16:41.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:41.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:16:41.424 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:16:41.480 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:41.569 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:16:41.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:41.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:16:41.677 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:41.722 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:41.845 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:41.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:41.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:41.956 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:16:41.994 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:42.045 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 32 16 55969 55953 109.254 81.375 0.00314679 0.00913636 2023-10-09T06:16:42.045 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:16:42.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:42.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:16:42.132 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:16:42.150 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:16:42.167 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:16:42.217 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:16:42.220 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:42.253 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:16:42.291 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:16:42.328 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:16:42.360 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:16:42.379 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:16:42.380 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:16:42.380 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:16:42.380 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:16:42.380 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:16:42.380 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:16:42.381 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:16:42.381 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:16:42.381 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:16:43.045 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 33 16 57145 57129 108.17 73.5 0.00926532 0.009219 2023-10-09T06:16:43.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:42 smithi156 ceph-mon[195101]: pgmap v1487: 119 pgs: 119 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-10-09T06:16:43.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:42 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/176342190' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:16:43.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:42 smithi064 ceph-mon[212750]: pgmap v1487: 119 pgs: 119 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-10-09T06:16:43.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:42 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/176342190' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:16:43.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:42 smithi064 ceph-mon[220044]: pgmap v1487: 119 pgs: 119 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-10-09T06:16:43.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:42 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/176342190' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:16:43.522 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:16:43.619 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:16:43.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:43.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:16:43.729 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:43.755 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:43.797 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:43.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:43.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:43.903 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:16:43.975 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:44.045 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 34 16 58159 58143 106.853 63.375 0.0179458 0.00935337 2023-10-09T06:16:44.079 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:16:44.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:44.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:16:44.190 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:16:44.260 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:44.278 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:44.387 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:16:44.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:44.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:16:44.494 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:16:44.506 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:44.572 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:44.820 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:44 smithi064 ceph-mon[212750]: pgmap v1488: 119 pgs: 119 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-10-09T06:16:44.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:44.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:44.824 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:44 smithi064 ceph-mon[220044]: pgmap v1488: 119 pgs: 119 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-10-09T06:16:44.824 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:44.824 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:44.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:44 smithi156 ceph-mon[195101]: pgmap v1488: 119 pgs: 119 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-10-09T06:16:44.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:44.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:44.909 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:16:44.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:44.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:16:45.018 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:16:45.045 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 35 16 59420 59404 106.052 78.8125 0.0102432 0.00942446 2023-10-09T06:16:45.072 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:45.162 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:16:45.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:45.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:16:45.269 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:45.324 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:45.408 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:45.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:45.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:45.471 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:16:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:16:45] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.43.0" 2023-10-09T06:16:45.519 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:16:45.545 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:45.570 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:16:45.587 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:16:45.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:45.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:16:45.693 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:16:45.838 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:46.045 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 36 16 60496 60480 104.974 67.25 0.0182934 0.00952142 2023-10-09T06:16:46.184 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:16:46.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:46.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:16:46.294 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:16:46.337 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:46.405 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:16:46.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:46.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:16:46.515 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:16:46.576 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:16:46.594 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:46.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:46 smithi156 ceph-mon[195101]: pgmap v1489: 119 pgs: 119 active+clean; 3.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T06:16:46.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:46 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2838908927' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:16:46.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:46 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:16:46.800 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:16:46.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:46.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:16:46.808 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:46 smithi064 ceph-mon[220044]: pgmap v1489: 119 pgs: 119 active+clean; 3.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T06:16:46.809 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:46 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2838908927' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:16:46.809 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:46 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:16:46.809 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:46 smithi064 ceph-mon[212750]: pgmap v1489: 119 pgs: 119 active+clean; 3.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T06:16:46.809 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:46 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2838908927' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:16:46.810 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:46 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:16:46.909 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:46.959 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:47.040 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:47.047 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 37 16 61673 61657 104.121 73.5625 0.0151726 0.00959897 2023-10-09T06:16:47.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:47.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:47.157 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:16:47.231 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:47.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:47 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:16:47.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:47 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:16:47.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:47 smithi156 ceph-mon[195101]: failed to encode map e584 with expected crc 2023-10-09T06:16:47.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:47 smithi156 ceph-mon[195101]: failed to encode map e584 with expected crc 2023-10-09T06:16:47.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:47 smithi156 ceph-mon[195101]: osdmap e584: 8 total, 8 up, 7 in 2023-10-09T06:16:47.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:47 smithi156 ceph-mon[195101]: failed to encode map e584 with expected crc 2023-10-09T06:16:47.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:47 smithi156 ceph-mon[195101]: failed to encode map e584 with expected crc 2023-10-09T06:16:47.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:47 smithi156 ceph-mon[195101]: failed to encode map e584 with expected crc 2023-10-09T06:16:47.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:47 smithi156 ceph-mon[195101]: failed to encode map e584 with expected crc 2023-10-09T06:16:47.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:47 smithi156 ceph-mon[195101]: failed to encode map e584 with expected crc 2023-10-09T06:16:47.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:47 smithi156 ceph-mon[195101]: failed to encode map e584 with expected crc 2023-10-09T06:16:47.957 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:47 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:16:47.957 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:47 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:16:47.957 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:47 smithi064 ceph-mon[220044]: failed to encode map e584 with expected crc 2023-10-09T06:16:47.957 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:47 smithi064 ceph-mon[220044]: failed to encode map e584 with expected crc 2023-10-09T06:16:47.957 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:47 smithi064 ceph-mon[220044]: osdmap e584: 8 total, 8 up, 7 in 2023-10-09T06:16:47.957 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:47 smithi064 ceph-mon[220044]: failed to encode map e584 with expected crc 2023-10-09T06:16:47.957 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:47 smithi064 ceph-mon[220044]: failed to encode map e584 with expected crc 2023-10-09T06:16:47.958 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:47 smithi064 ceph-mon[220044]: failed to encode map e584 with expected crc 2023-10-09T06:16:47.958 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:47 smithi064 ceph-mon[220044]: failed to encode map e584 with expected crc 2023-10-09T06:16:47.958 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:47 smithi064 ceph-mon[220044]: failed to encode map e584 with expected crc 2023-10-09T06:16:47.958 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:47 smithi064 ceph-mon[220044]: failed to encode map e584 with expected crc 2023-10-09T06:16:47.958 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:47 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:16:47.958 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:47 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:16:47.958 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:47 smithi064 ceph-mon[212750]: failed to encode map e584 with expected crc 2023-10-09T06:16:47.958 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:47 smithi064 ceph-mon[212750]: failed to encode map e584 with expected crc 2023-10-09T06:16:47.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:47 smithi064 ceph-mon[212750]: osdmap e584: 8 total, 8 up, 7 in 2023-10-09T06:16:47.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:47 smithi064 ceph-mon[212750]: failed to encode map e584 with expected crc 2023-10-09T06:16:47.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:47 smithi064 ceph-mon[212750]: failed to encode map e584 with expected crc 2023-10-09T06:16:47.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:47 smithi064 ceph-mon[212750]: failed to encode map e584 with expected crc 2023-10-09T06:16:47.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:47 smithi064 ceph-mon[212750]: failed to encode map e584 with expected crc 2023-10-09T06:16:47.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:47 smithi064 ceph-mon[212750]: failed to encode map e584 with expected crc 2023-10-09T06:16:47.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:47 smithi064 ceph-mon[212750]: failed to encode map e584 with expected crc 2023-10-09T06:16:48.047 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 38 16 62775 62759 103.193 68.875 0.00667071 0.00966457 2023-10-09T06:16:48.403 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:16:48.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:48.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:16:48.508 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:48.545 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:48.553 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:48.630 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:48.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:48.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:48.737 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:48.777 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:48.865 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:48.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:48 smithi156 ceph-mon[195101]: pgmap v1491: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-10-09T06:16:48.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:48.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:48.972 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:16:49.053 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 39 16 63864 63848 102.289 68.0625 0.00999179 0.00977051 2023-10-09T06:16:49.056 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:48 smithi064 ceph-mon[212750]: pgmap v1491: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-10-09T06:16:49.057 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:48 smithi064 ceph-mon[220044]: pgmap v1491: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-10-09T06:16:49.067 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:49.161 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:16:49.161 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:16:49.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.161 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:63977 11.7 11:e7ec01da:::benchmark_data_smithi064_631903_object63976:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-10-09T06:16:49.161 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:16:49.151667+0000", 2023-10-09T06:16:49.162 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0087949780000000002, 2023-10-09T06:16:49.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0088315670000000002, 2023-10-09T06:16:49.162 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:16:49.162 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:16:49.162 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:16:49.162 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:16:49.162 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:16:49.162 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 63977 2023-10-09T06:16:49.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.162 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:16:49.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:16:49.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.151667+0000", 2023-10-09T06:16:49.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:16:49.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:16:49.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.151667+0000", 2023-10-09T06:16:49.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:16:49.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:16:49.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.151665+0000", 2023-10-09T06:16:49.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:16:49.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:16:49.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.151710+0000", 2023-10-09T06:16:49.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4895000000000003e-05 2023-10-09T06:16:49.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:16:49.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.151722+0000", 2023-10-09T06:16:49.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1378e-05 2023-10-09T06:16:49.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:16:49.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.151730+0000", 2023-10-09T06:16:49.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9179999999999994e-06 2023-10-09T06:16:49.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:16:49.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.151794+0000", 2023-10-09T06:16:49.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4049999999999998e-05 2023-10-09T06:16:49.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:16:49.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.151947+0000", 2023-10-09T06:16:49.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000153485 2023-10-09T06:16:49.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:16:49.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.152096+0000", 2023-10-09T06:16:49.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014878800000000001 2023-10-09T06:16:49.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.167 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:49.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.157152+0000", 2023-10-09T06:16:49.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050557700000000002 2023-10-09T06:16:49.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.167 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:16:49.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.157209+0000", 2023-10-09T06:16:49.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6867000000000003e-05 2023-10-09T06:16:49.167 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:49.167 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:16:49.167 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:49.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.168 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:63982 11.7 11:ee2b4d52:::benchmark_data_smithi064_631903_object63981:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-10-09T06:16:49.168 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:16:49.151746+0000", 2023-10-09T06:16:49.168 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0087163840000000006, 2023-10-09T06:16:49.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0088541290000000005, 2023-10-09T06:16:49.168 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:16:49.168 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:16:49.168 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:16:49.168 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:16:49.168 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:16:49.168 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 63982 2023-10-09T06:16:49.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.169 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:16:49.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:16:49.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.151746+0000", 2023-10-09T06:16:49.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:16:49.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:16:49.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.151746+0000", 2023-10-09T06:16:49.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:16:49.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:16:49.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.151745+0000", 2023-10-09T06:16:49.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:16:49.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:16:49.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.151764+0000", 2023-10-09T06:16:49.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9483e-05 2023-10-09T06:16:49.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:16:49.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.151771+0000", 2023-10-09T06:16:49.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5370000000000003e-06 2023-10-09T06:16:49.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:16:49.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.151796+0000", 2023-10-09T06:16:49.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4902e-05 2023-10-09T06:16:49.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:16:49.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.154373+0000", 2023-10-09T06:16:49.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025769450000000002 2023-10-09T06:16:49.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:16:49.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.154536+0000", 2023-10-09T06:16:49.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000163311 2023-10-09T06:16:49.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:16:49.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.154736+0000", 2023-10-09T06:16:49.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020020500000000001 2023-10-09T06:16:49.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:16:49.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.157214+0000", 2023-10-09T06:16:49.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002477707 2023-10-09T06:16:49.173 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:49.173 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:16:49.173 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:49.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.173 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:63983 11.6 11:659fea28:::benchmark_data_smithi064_631903_object63982:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-10-09T06:16:49.174 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:16:49.151814+0000", 2023-10-09T06:16:49.174 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0086480059999999997, 2023-10-09T06:16:49.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0088673680000000005, 2023-10-09T06:16:49.174 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:16:49.174 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:16:49.174 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:16:49.174 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:16:49.174 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:16:49.174 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 63983 2023-10-09T06:16:49.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.174 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:16:49.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:16:49.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.151814+0000", 2023-10-09T06:16:49.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:16:49.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:16:49.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.151814+0000", 2023-10-09T06:16:49.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:16:49.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:16:49.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.151813+0000", 2023-10-09T06:16:49.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:16:49.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:16:49.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.151865+0000", 2023-10-09T06:16:49.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1255000000000003e-05 2023-10-09T06:16:49.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:16:49.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.151874+0000", 2023-10-09T06:16:49.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2779999999999999e-06 2023-10-09T06:16:49.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:16:49.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.151881+0000", 2023-10-09T06:16:49.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2339999999999997e-06 2023-10-09T06:16:49.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:16:49.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.151933+0000", 2023-10-09T06:16:49.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1338999999999998e-05 2023-10-09T06:16:49.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:16:49.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.152058+0000", 2023-10-09T06:16:49.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012512600000000001 2023-10-09T06:16:49.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:16:49.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.152191+0000", 2023-10-09T06:16:49.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000133275 2023-10-09T06:16:49.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:16:49.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.155098+0000", 2023-10-09T06:16:49.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029071549999999998 2023-10-09T06:16:49.180 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:49.180 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:16:49.180 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:49.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.180 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:63986 11.c 11:361e5f54:::benchmark_data_smithi064_631903_object63985:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-10-09T06:16:49.180 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:16:49.158758+0000", 2023-10-09T06:16:49.180 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.00170403, 2023-10-09T06:16:49.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020022949999999999, 2023-10-09T06:16:49.181 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:16:49.181 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:16:49.181 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:16:49.181 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:16:49.181 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:16:49.181 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 63986 2023-10-09T06:16:49.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.181 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:16:49.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:16:49.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.158758+0000", 2023-10-09T06:16:49.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:16:49.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:16:49.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.158758+0000", 2023-10-09T06:16:49.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:16:49.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:16:49.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.158756+0000", 2023-10-09T06:16:49.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:16:49.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:16:49.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.158787+0000", 2023-10-09T06:16:49.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1257000000000002e-05 2023-10-09T06:16:49.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:16:49.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.158806+0000", 2023-10-09T06:16:49.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9049e-05 2023-10-09T06:16:49.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:16:49.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.158816+0000", 2023-10-09T06:16:49.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9920000000000002e-06 2023-10-09T06:16:49.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:16:49.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.159005+0000", 2023-10-09T06:16:49.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000188767 2023-10-09T06:16:49.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:16:49.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.159171+0000", 2023-10-09T06:16:49.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016566899999999999 2023-10-09T06:16:49.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,7", 2023-10-09T06:16:49.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.159333+0000", 2023-10-09T06:16:49.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016178399999999999 2023-10-09T06:16:49.186 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:49.186 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:16:49.186 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:49.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.186 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:63987 11.c 11.8e851b7c (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-10-09T06:16:49.186 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:16:49.158848+0000", 2023-10-09T06:16:49.186 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0016142509999999999, 2023-10-09T06:16:49.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019592640000000001, 2023-10-09T06:16:49.186 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:16:49.187 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-10-09T06:16:49.187 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:16:49.187 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:16:49.187 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:16:49.187 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 63987 2023-10-09T06:16:49.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.187 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:16:49.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:16:49.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.158848+0000", 2023-10-09T06:16:49.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:16:49.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:16:49.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.158848+0000", 2023-10-09T06:16:49.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:16:49.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:16:49.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.158847+0000", 2023-10-09T06:16:49.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:16:49.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:16:49.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.158869+0000", 2023-10-09T06:16:49.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1730999999999999e-05 2023-10-09T06:16:49.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:16:49.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.158880+0000", 2023-10-09T06:16:49.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0825000000000001e-05 2023-10-09T06:16:49.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:16:49.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.158886+0000", 2023-10-09T06:16:49.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7089999999999996e-06 2023-10-09T06:16:49.190 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:49.190 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:16:49.190 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:49.190 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.190 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:63989 11.7 11:eeed452c:::benchmark_data_smithi064_631903_object63988:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-10-09T06:16:49.190 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:16:49.158893+0000", 2023-10-09T06:16:49.190 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0015690140000000001, 2023-10-09T06:16:49.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019504380000000001, 2023-10-09T06:16:49.191 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:16:49.191 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:16:49.191 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:16:49.191 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:16:49.191 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:16:49.191 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 63989 2023-10-09T06:16:49.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.191 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:16:49.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:16:49.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.158893+0000", 2023-10-09T06:16:49.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:16:49.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:16:49.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.158893+0000", 2023-10-09T06:16:49.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:16:49.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:16:49.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.158892+0000", 2023-10-09T06:16:49.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:16:49.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:16:49.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.158941+0000", 2023-10-09T06:16:49.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8312000000000001e-05 2023-10-09T06:16:49.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:16:49.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.158949+0000", 2023-10-09T06:16:49.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6119999999999999e-06 2023-10-09T06:16:49.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:16:49.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.158956+0000", 2023-10-09T06:16:49.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7089999999999996e-06 2023-10-09T06:16:49.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:16:49.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.159008+0000", 2023-10-09T06:16:49.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1755999999999997e-05 2023-10-09T06:16:49.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:16:49.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.159170+0000", 2023-10-09T06:16:49.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016224800000000001 2023-10-09T06:16:49.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:16:49.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.159332+0000", 2023-10-09T06:16:49.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016246000000000001 2023-10-09T06:16:49.195 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:49.195 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:16:49.195 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:49.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.196 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36591.0:63990 11.4 e584/572)", 2023-10-09T06:16:49.196 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:16:49.160046+0000", 2023-10-09T06:16:49.196 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.00041603300000000001, 2023-10-09T06:16:49.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00083890199999999999, 2023-10-09T06:16:49.196 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:16:49.196 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-10-09T06:16:49.196 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:16:49.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:16:49.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.160046+0000", 2023-10-09T06:16:49.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:16:49.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:16:49.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.160046+0000", 2023-10-09T06:16:49.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:16:49.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:16:49.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.160045+0000", 2023-10-09T06:16:49.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:16:49.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:16:49.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.160099+0000", 2023-10-09T06:16:49.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4369000000000001e-05 2023-10-09T06:16:49.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:16:49.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.160110+0000", 2023-10-09T06:16:49.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1048e-05 2023-10-09T06:16:49.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:16:49.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.160118+0000", 2023-10-09T06:16:49.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.289e-06 2023-10-09T06:16:49.199 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:49.199 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:16:49.199 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:49.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.199 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:63973 11.c 11:30a60f66:::benchmark_data_smithi064_631903_object63972:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-10-09T06:16:49.199 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:16:49.148235+0000", 2023-10-09T06:16:49.199 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.012226805, 2023-10-09T06:16:49.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012681733000000001, 2023-10-09T06:16:49.200 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:16:49.200 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:16:49.200 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:16:49.200 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:16:49.200 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:16:49.200 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 63973 2023-10-09T06:16:49.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.200 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:16:49.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:16:49.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.148235+0000", 2023-10-09T06:16:49.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:16:49.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:16:49.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.148235+0000", 2023-10-09T06:16:49.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:16:49.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:16:49.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.148234+0000", 2023-10-09T06:16:49.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:16:49.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:16:49.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.148257+0000", 2023-10-09T06:16:49.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.332e-05 2023-10-09T06:16:49.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:16:49.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.148270+0000", 2023-10-09T06:16:49.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2955999999999999e-05 2023-10-09T06:16:49.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:16:49.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.148278+0000", 2023-10-09T06:16:49.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7789999999999996e-06 2023-10-09T06:16:49.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:16:49.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.148375+0000", 2023-10-09T06:16:49.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6339e-05 2023-10-09T06:16:49.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:16:49.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.148510+0000", 2023-10-09T06:16:49.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000135153 2023-10-09T06:16:49.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,7", 2023-10-09T06:16:49.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.148664+0000", 2023-10-09T06:16:49.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000153886 2023-10-09T06:16:49.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:16:49.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.151331+0000", 2023-10-09T06:16:49.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026670370000000001 2023-10-09T06:16:49.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:16:49.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:16:49.205 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:16:49.205 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.154988+0000", 2023-10-09T06:16:49.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0036578499999999998 2023-10-09T06:16:49.205 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:49.205 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:16:49.205 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:49.205 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:16:49.205 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-10-09T06:16:49.205 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 8 2023-10-09T06:16:49.205 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:16:49.804 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T06:16:49.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:49 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4264917097' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:16:49.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:49 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:16:49.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:49 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4264917097' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:16:49.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:49 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:16:50.048 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-10-09T06:16:50.049311+0000 min lat: 0.00113085 max lat: 0.307063 avg lat: 0.0098725 2023-10-09T06:16:50.049 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:16:50.049 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 40 16 64827 64811 101.236 60.1875 0.0190652 0.0098725 2023-10-09T06:16:50.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:49 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4264917097' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:16:50.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:49 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:16:50.285 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:16:50.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:50.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:16:50.393 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:50.419 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:50.494 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:50.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:50.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:50.579 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:16:50.604 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:16:50.654 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:50.721 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:16:50.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:50.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:16:50.826 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:16:50.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:50 smithi064 ceph-mon[220044]: pgmap v1492: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-10-09T06:16:50.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:50 smithi064 ceph-mon[220044]: failed to encode map e585 with expected crc 2023-10-09T06:16:50.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:50 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:16:50.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:50 smithi064 ceph-mon[220044]: osdmap e585: 8 total, 8 up, 7 in 2023-10-09T06:16:50.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:50 smithi064 ceph-mon[212750]: pgmap v1492: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-10-09T06:16:50.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:50 smithi064 ceph-mon[212750]: failed to encode map e585 with expected crc 2023-10-09T06:16:50.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:50 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:16:50.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:50 smithi064 ceph-mon[212750]: osdmap e585: 8 total, 8 up, 7 in 2023-10-09T06:16:50.917 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:51.049 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 41 16 65846 65830 100.321 63.6875 0.0126656 0.00996273 2023-10-09T06:16:51.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:50 smithi156 ceph-mon[195101]: pgmap v1492: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-10-09T06:16:51.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:50 smithi156 ceph-mon[195101]: failed to encode map e585 with expected crc 2023-10-09T06:16:51.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:50 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:16:51.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:50 smithi156 ceph-mon[195101]: osdmap e585: 8 total, 8 up, 7 in 2023-10-09T06:16:51.217 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:16:51.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:51.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:16:51.271 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T06:16:51.299 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:16:51.324 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:16:51.358 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:51.435 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:16:51.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:51.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:16:51.544 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:16:51.628 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:51.664 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:51.900 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:16:51.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:51.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:16:52.015 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:16:52.049 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 42 16 66859 66843 99.4392 63.3125 0.0151946 0.0100513 2023-10-09T06:16:52.088 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:52.321 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:16:52.333 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:16:52.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:52.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:16:52.362 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:16:52.395 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:16:52.429 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:16:52.440 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:16:52.467 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:16:52.469 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:52.484 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:52.524 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:16:52.560 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:16:52.563 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:16:52.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:52.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:16:52.613 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:16:52.621 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:16:52.621 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:16:52.621 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:16:52.621 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:16:52.621 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:16:52.622 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:16:52.622 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:16:52.622 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:16:52.672 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:16:52.689 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:52.718 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:52.776 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:52 smithi156 ceph-mon[195101]: pgmap v1494: 119 pgs: 119 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T06:16:52.776 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:52 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/432008632' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:16:52.793 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:16:52.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:52.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:16:52.903 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:16:52.971 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:53.013 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:16:53.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:53.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:16:53.049 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 43 16 67893 67877 98.6296 64.625 0.00900589 0.0101158 2023-10-09T06:16:53.118 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:16:53.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:52 smithi064 ceph-mon[212750]: pgmap v1494: 119 pgs: 119 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T06:16:53.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:52 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/432008632' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:16:53.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:52 smithi064 ceph-mon[220044]: pgmap v1494: 119 pgs: 119 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T06:16:53.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:52 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/432008632' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:16:53.161 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:53.671 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:16:53.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:53.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:16:53.777 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:53.804 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:53.844 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:53.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:53.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:53.952 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:16:53.999 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:54.049 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 44 16 68722 68706 97.5657 51.8125 0.00927294 0.0102435 2023-10-09T06:16:54.293 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:16:54.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:54.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:16:54.403 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:16:54.472 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:54.830 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:54 smithi064 ceph-mon[212750]: pgmap v1495: 119 pgs: 119 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T06:16:54.831 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:54 smithi064 ceph-mon[220044]: pgmap v1495: 119 pgs: 119 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T06:16:55.049 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 45 16 69899 69883 97.0322 73.5625 0.0064948 0.0103009 2023-10-09T06:16:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:54 smithi156 ceph-mon[195101]: pgmap v1495: 119 pgs: 119 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T06:16:55.194 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:16:55.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:55.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:16:55.305 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:16:55.347 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:55.391 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:16:55.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:55.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:16:55.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:16:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:16:55] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.43.0" 2023-10-09T06:16:55.499 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:16:55.579 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:55.831 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:16:55.862 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:55 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/511179039' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:16:55.863 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:55 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/511179039' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:16:56.050 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 46 16 70912 70896 96.2992 63.3125 0.0131542 0.0103793 2023-10-09T06:16:56.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:55 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/511179039' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:16:56.152 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:16:56.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:56.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:16:56.259 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:16:56.311 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:56.380 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:16:56.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:56.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:16:56.497 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:16:56.555 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:56.635 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:16:56.796 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:56 smithi064 ceph-mon[212750]: pgmap v1496: 119 pgs: 119 active+clean; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 58 MiB/s wr, 934 op/s 2023-10-09T06:16:56.797 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:56 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:16:56.797 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:56 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/511179039' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:16:56.797 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:56 smithi064 ceph-mon[212750]: osdmap e586: 8 total, 8 up, 7 in 2023-10-09T06:16:56.798 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:56 smithi064 ceph-mon[212750]: failed to encode map e586 with expected crc 2023-10-09T06:16:56.798 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:56 smithi064 ceph-mon[212750]: failed to encode map e586 with expected crc 2023-10-09T06:16:56.798 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:56 smithi064 ceph-mon[212750]: failed to encode map e586 with expected crc 2023-10-09T06:16:56.798 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:56 smithi064 ceph-mon[212750]: failed to encode map e586 with expected crc 2023-10-09T06:16:57.049 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 47 16 71916 71900 95.5854 62.75 0.0132563 0.0104568 2023-10-09T06:16:57.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:56 smithi156 ceph-mon[195101]: pgmap v1496: 119 pgs: 119 active+clean; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 58 MiB/s wr, 934 op/s 2023-10-09T06:16:57.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:56 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:16:57.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:56 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/511179039' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:16:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:56 smithi156 ceph-mon[195101]: osdmap e586: 8 total, 8 up, 7 in 2023-10-09T06:16:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:56 smithi156 ceph-mon[195101]: failed to encode map e586 with expected crc 2023-10-09T06:16:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:56 smithi156 ceph-mon[195101]: failed to encode map e586 with expected crc 2023-10-09T06:16:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:56 smithi156 ceph-mon[195101]: failed to encode map e586 with expected crc 2023-10-09T06:16:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:56 smithi156 ceph-mon[195101]: failed to encode map e586 with expected crc 2023-10-09T06:16:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:56 smithi156 ceph-mon[195101]: failed to encode map e586 with expected crc 2023-10-09T06:16:57.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:56 smithi064 ceph-mon[220044]: pgmap v1496: 119 pgs: 119 active+clean; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 58 MiB/s wr, 934 op/s 2023-10-09T06:16:57.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:56 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:16:57.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:56 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/511179039' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:16:57.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:56 smithi064 ceph-mon[220044]: osdmap e586: 8 total, 8 up, 7 in 2023-10-09T06:16:57.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:56 smithi064 ceph-mon[220044]: failed to encode map e586 with expected crc 2023-10-09T06:16:57.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:56 smithi064 ceph-mon[220044]: failed to encode map e586 with expected crc 2023-10-09T06:16:57.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:56 smithi064 ceph-mon[220044]: failed to encode map e586 with expected crc 2023-10-09T06:16:57.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:56 smithi064 ceph-mon[220044]: failed to encode map e586 with expected crc 2023-10-09T06:16:57.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:56 smithi064 ceph-mon[220044]: failed to encode map e586 with expected crc 2023-10-09T06:16:57.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:56 smithi064 ceph-mon[212750]: failed to encode map e586 with expected crc 2023-10-09T06:16:57.596 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:16:57.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:57.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:16:57.711 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:16:57.812 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:57.925 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:16:57.925 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:16:57.960 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:16:57.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:57.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:16:57.998 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:58.050 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 48 16 72989 72973 94.9912 67.0625 0.0125989 0.010521 2023-10-09T06:16:58.074 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:16:58.116 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:58.200 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:16:58.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:58.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:16:58.304 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:16:58.412 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:58.527 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:16:58.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:58.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:16:58.635 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:16:58.663 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:58.708 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:16:58.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:58.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:16:58.814 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:16:58.862 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:58.946 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:16:58.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:58.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:16:58.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:58 smithi064 ceph-mon[212750]: pgmap v1498: 119 pgs: 119 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-09T06:16:58.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:58 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:58.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:58.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:16:58 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/267436855' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:16:58.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:58 smithi064 ceph-mon[220044]: pgmap v1498: 119 pgs: 119 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-09T06:16:58.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:58 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:58.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:58 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:58.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:16:58 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/267436855' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:16:59.050 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 49 16 73821 73805 94.1139 52 0.00968273 0.0106203 2023-10-09T06:16:59.054 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:16:59.109 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:59.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:58 smithi156 ceph-mon[195101]: pgmap v1498: 119 pgs: 119 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-09T06:16:59.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:58 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:16:59.124 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:58 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:59.124 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:16:58 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/267436855' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:16:59.291 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:16:59.317 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:16:59.467 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:16:59.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:59.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:16:59.578 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:16:59.587 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T06:16:59.610 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:16:59.666 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:16:59.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:16:59.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:16:59.770 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:16:59.798 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:00.051 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 50 16 74660 74644 93.278 52.4375 0.016801 0.0107154 2023-10-09T06:17:00.239 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:17:00.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:00.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:00.344 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:00.369 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:00.416 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:17:00.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:00.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:00.527 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:00.597 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:00.624 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:00 smithi156 ceph-mon[195101]: pgmap v1499: 119 pgs: 119 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 60 MiB/s wr, 957 op/s 2023-10-09T06:17:00.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:00 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/267436855' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:17:00.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:00 smithi156 ceph-mon[195101]: osdmap e587: 8 total, 8 up, 7 in 2023-10-09T06:17:00.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:00 smithi156 ceph-mon[195101]: failed to encode map e587 with expected crc 2023-10-09T06:17:00.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:00 smithi156 ceph-mon[195101]: failed to encode map e587 with expected crc 2023-10-09T06:17:00.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:00 smithi156 ceph-mon[195101]: failed to encode map e587 with expected crc 2023-10-09T06:17:00.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:00 smithi156 ceph-mon[195101]: failed to encode map e587 with expected crc 2023-10-09T06:17:00.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:00 smithi156 ceph-mon[195101]: failed to encode map e587 with expected crc 2023-10-09T06:17:00.827 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:00 smithi064 ceph-mon[212750]: pgmap v1499: 119 pgs: 119 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 60 MiB/s wr, 957 op/s 2023-10-09T06:17:00.827 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:00 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/267436855' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:17:00.827 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:00 smithi064 ceph-mon[212750]: osdmap e587: 8 total, 8 up, 7 in 2023-10-09T06:17:00.827 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:00 smithi064 ceph-mon[212750]: failed to encode map e587 with expected crc 2023-10-09T06:17:00.827 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:00 smithi064 ceph-mon[212750]: failed to encode map e587 with expected crc 2023-10-09T06:17:00.827 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:00 smithi064 ceph-mon[212750]: failed to encode map e587 with expected crc 2023-10-09T06:17:00.827 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:00 smithi064 ceph-mon[212750]: failed to encode map e587 with expected crc 2023-10-09T06:17:00.827 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:00 smithi064 ceph-mon[212750]: failed to encode map e587 with expected crc 2023-10-09T06:17:00.828 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:00 smithi064 ceph-mon[220044]: pgmap v1499: 119 pgs: 119 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 60 MiB/s wr, 957 op/s 2023-10-09T06:17:00.828 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:00 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/267436855' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:17:00.828 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:00 smithi064 ceph-mon[220044]: osdmap e587: 8 total, 8 up, 7 in 2023-10-09T06:17:00.828 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:00 smithi064 ceph-mon[220044]: failed to encode map e587 with expected crc 2023-10-09T06:17:00.828 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:00 smithi064 ceph-mon[220044]: failed to encode map e587 with expected crc 2023-10-09T06:17:00.828 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:00 smithi064 ceph-mon[220044]: failed to encode map e587 with expected crc 2023-10-09T06:17:00.828 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:00 smithi064 ceph-mon[220044]: failed to encode map e587 with expected crc 2023-10-09T06:17:00.828 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:00 smithi064 ceph-mon[220044]: failed to encode map e587 with expected crc 2023-10-09T06:17:00.899 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:17:00.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:00.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:00.928 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T06:17:01.009 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:01.051 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 51 16 75657 75641 92.6708 62.3125 0.0344505 0.0107844 2023-10-09T06:17:01.108 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:01.137 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:01.195 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:01.308 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:17:01.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:01.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:01.416 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:01.472 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:01.545 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:17:01.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:01.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:01.655 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:01.696 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:01.786 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:17:01.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:01.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:01.897 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:01.911 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:17:01.959 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:17:01.966 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:02.003 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:17:02.052 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 52 16 76638 76622 92.0665 61.3125 0.0175199 0.0108565 2023-10-09T06:17:02.140 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:17:02.140 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:17:02.140 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:17:02.140 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:17:02.140 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:17:02.141 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:17:02.141 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:17:02.141 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:17:02.141 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:17:02.141 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:17:02.141 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:17:02.141 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:17:02.141 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:17:02.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:02 smithi156 ceph-mon[195101]: pgmap v1501: 119 pgs: 119 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T06:17:02.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:02 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2897630808' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:17:02.887 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:02 smithi064 ceph-mon[220044]: pgmap v1501: 119 pgs: 119 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T06:17:02.887 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:02 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2897630808' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:17:02.888 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:02 smithi064 ceph-mon[212750]: pgmap v1501: 119 pgs: 119 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T06:17:02.888 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:02 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2897630808' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:17:03.052 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 53 16 77529 77513 91.3802 55.6875 0.00710216 0.0109377 2023-10-09T06:17:03.853 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:17:03.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:03.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:03.957 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:03.999 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:04.041 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:17:04.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:04.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:04.052 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 54 16 78497 78481 90.8083 60.5 0.0088774 0.0109955 2023-10-09T06:17:04.146 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:04.185 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:04.535 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:04.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:04.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:04.641 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:04.648 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:04.683 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:04.754 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:17:04.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:04.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:04.860 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:04.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:04 smithi064 ceph-mon[220044]: pgmap v1502: 119 pgs: 119 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T06:17:04.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:04 smithi064 ceph-mon[212750]: pgmap v1502: 119 pgs: 119 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T06:17:04.914 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:05.052 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 55 16 79469 79453 90.2618 60.75 0.0083884 0.0110753 2023-10-09T06:17:05.069 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:17:05.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:05.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:05.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:04 smithi156 ceph-mon[195101]: pgmap v1502: 119 pgs: 119 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T06:17:05.177 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:05.239 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:05.308 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:17:05.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:05.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:05.414 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:05.451 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:05.458 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:17:05.458 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T06:17:05.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T06:17:05.458 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:17:05.458 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.458 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116275 9.d 9:bc074772:::benchmark_data_smithi064_511911_object15173:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e449)", 2023-10-09T06:17:05.458 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.678371+0000", 2023-10-09T06:17:05.458 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 482.77528202100001, 2023-10-09T06:17:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0175197690000002, 2023-10-09T06:17:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:17:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:17:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:17:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116275 2023-10-09T06:17:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.459 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.678371+0000", 2023-10-09T06:17:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.678371+0000", 2023-10-09T06:17:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.678369+0000", 2023-10-09T06:17:05.460 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:17:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.678374+0000", 2023-10-09T06:17:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4739999999999996e-06 2023-10-09T06:17:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.678384+0000", 2023-10-09T06:17:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0081000000000001e-05 2023-10-09T06:17:05.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.678396+0000", 2023-10-09T06:17:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1113e-05 2023-10-09T06:17:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.679003+0000", 2023-10-09T06:17:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00060782100000000003 2023-10-09T06:17:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.462 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.679017+0000", 2023-10-09T06:17:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3404000000000001e-05 2023-10-09T06:17:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.582513+0000", 2023-10-09T06:17:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90349592599999995 2023-10-09T06:17:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.582521+0000", 2023-10-09T06:17:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0029999999999994e-06 2023-10-09T06:17:05.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.695824+0000", 2023-10-09T06:17:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1133034020000001 2023-10-09T06:17:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.695890+0000", 2023-10-09T06:17:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6169999999999998e-05 2023-10-09T06:17:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:05.464 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116279 9.d 9:b25d4338:::benchmark_data_smithi064_511911_object15177:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e449)", 2023-10-09T06:17:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.678408+0000", 2023-10-09T06:17:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 482.77524500099997, 2023-10-09T06:17:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0596024009999998, 2023-10-09T06:17:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:17:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:17:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:17:05.465 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116279 2023-10-09T06:17:05.466 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.466 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:05.466 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:05.466 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.678408+0000", 2023-10-09T06:17:05.466 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.466 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.466 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:05.466 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.678408+0000", 2023-10-09T06:17:05.466 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.466 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.678407+0000", 2023-10-09T06:17:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:17:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.678410+0000", 2023-10-09T06:17:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4169999999999999e-06 2023-10-09T06:17:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.467 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.678415+0000", 2023-10-09T06:17:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4099999999999999e-06 2023-10-09T06:17:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.678419+0000", 2023-10-09T06:17:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4569999999999998e-06 2023-10-09T06:17:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.679041+0000", 2023-10-09T06:17:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00062169999999999999 2023-10-09T06:17:05.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.679051+0000", 2023-10-09T06:17:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6590000000000002e-06 2023-10-09T06:17:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.582535+0000", 2023-10-09T06:17:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90348404500000001 2023-10-09T06:17:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.582539+0000", 2023-10-09T06:17:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2570000000000001e-06 2023-10-09T06:17:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.695903+0000", 2023-10-09T06:17:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1133641329999999 2023-10-09T06:17:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-10-09T06:17:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.695984+0000", 2023-10-09T06:17:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0378999999999997e-05 2023-10-09T06:17:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.737951+0000", 2023-10-09T06:17:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.041967036999999999 2023-10-09T06:17:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.738010+0000", 2023-10-09T06:17:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9227000000000001e-05 2023-10-09T06:17:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116280 9.d 9:bd7f7e10:::benchmark_data_smithi064_511911_object15178:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e449)", 2023-10-09T06:17:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.678424+0000", 2023-10-09T06:17:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 482.77522882699998, 2023-10-09T06:17:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0599763680000001, 2023-10-09T06:17:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:17:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:17:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:17:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116280 2023-10-09T06:17:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.678424+0000", 2023-10-09T06:17:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.678424+0000", 2023-10-09T06:17:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.678423+0000", 2023-10-09T06:17:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:17:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.678425+0000", 2023-10-09T06:17:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0470000000000001e-06 2023-10-09T06:17:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.678432+0000", 2023-10-09T06:17:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5899999999999996e-06 2023-10-09T06:17:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.678438+0000", 2023-10-09T06:17:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6840000000000001e-06 2023-10-09T06:17:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.679088+0000", 2023-10-09T06:17:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00064999999999999997 2023-10-09T06:17:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.679096+0000", 2023-10-09T06:17:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1189999999999997e-06 2023-10-09T06:17:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.582552+0000", 2023-10-09T06:17:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90345589900000001 2023-10-09T06:17:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.582556+0000", 2023-10-09T06:17:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1690000000000002e-06 2023-10-09T06:17:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.696008+0000", 2023-10-09T06:17:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.113452071 2023-10-09T06:17:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-10-09T06:17:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.696080+0000", 2023-10-09T06:17:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2311000000000006e-05 2023-10-09T06:17:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.738336+0000", 2023-10-09T06:17:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.042255270999999997 2023-10-09T06:17:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.738400+0000", 2023-10-09T06:17:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4493999999999996e-05 2023-10-09T06:17:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116290 9.d 9:b869698c:::benchmark_data_smithi064_511911_object15188:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e449)", 2023-10-09T06:17:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.678442+0000", 2023-10-09T06:17:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 482.77521093299998, 2023-10-09T06:17:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.060317672, 2023-10-09T06:17:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:17:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:17:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:17:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116290 2023-10-09T06:17:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.678442+0000", 2023-10-09T06:17:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.678442+0000", 2023-10-09T06:17:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.678441+0000", 2023-10-09T06:17:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:17:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.678443+0000", 2023-10-09T06:17:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1299999999999999e-06 2023-10-09T06:17:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.678450+0000", 2023-10-09T06:17:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.404e-06 2023-10-09T06:17:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.678454+0000", 2023-10-09T06:17:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3000000000000003e-06 2023-10-09T06:17:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.679120+0000", 2023-10-09T06:17:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00066596799999999996 2023-10-09T06:17:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.679130+0000", 2023-10-09T06:17:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0336000000000001e-05 2023-10-09T06:17:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.584198+0000", 2023-10-09T06:17:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.905067493 2023-10-09T06:17:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.584210+0000", 2023-10-09T06:17:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2014000000000001e-05 2023-10-09T06:17:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.696107+0000", 2023-10-09T06:17:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.111896974 2023-10-09T06:17:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-10-09T06:17:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.696178+0000", 2023-10-09T06:17:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1205999999999996e-05 2023-10-09T06:17:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.738692+0000", 2023-10-09T06:17:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.042513399 2023-10-09T06:17:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.738759+0000", 2023-10-09T06:17:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7726999999999997e-05 2023-10-09T06:17:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:143501 9.6 9:6b716293:::benchmark_data_smithi064_511911_object42399:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-10-09T06:17:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:46.958883+0000", 2023-10-09T06:17:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 438.49477000899998, 2023-10-09T06:17:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.051109029, 2023-10-09T06:17:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:17:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:17:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:17:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 143501 2023-10-09T06:17:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.958883+0000", 2023-10-09T06:17:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.958883+0000", 2023-10-09T06:17:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.958882+0000", 2023-10-09T06:17:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:17:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.958885+0000", 2023-10-09T06:17:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6820000000000001e-06 2023-10-09T06:17:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.958892+0000", 2023-10-09T06:17:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3840000000000002e-06 2023-10-09T06:17:05.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:05.490 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.958896+0000", 2023-10-09T06:17:05.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.036e-06 2023-10-09T06:17:05.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.490 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.960976+0000", 2023-10-09T06:17:05.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020806700000000002 2023-10-09T06:17:05.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:05.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.961101+0000", 2023-10-09T06:17:05.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012489599999999999 2023-10-09T06:17:05.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:17:05.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.961219+0000", 2023-10-09T06:17:05.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000117402 2023-10-09T06:17:05.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.967846+0000", 2023-10-09T06:17:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0066270870000000003 2023-10-09T06:17:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.971423+0000", 2023-10-09T06:17:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035769899999999999 2023-10-09T06:17:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.985555+0000", 2023-10-09T06:17:05.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01413207 2023-10-09T06:17:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.985571+0000", 2023-10-09T06:17:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5868000000000001e-05 2023-10-09T06:17:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:48.009912+0000", 2023-10-09T06:17:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.024341443 2023-10-09T06:17:05.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:48.009992+0000", 2023-10-09T06:17:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9462000000000006e-05 2023-10-09T06:17:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:143502 9.6 9:62e58163:::benchmark_data_smithi064_511911_object42400:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-10-09T06:17:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:46.958901+0000", 2023-10-09T06:17:05.494 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 438.49475191900001, 2023-10-09T06:17:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.051503482, 2023-10-09T06:17:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:17:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:17:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:17:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 143502 2023-10-09T06:17:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:05.495 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.958901+0000", 2023-10-09T06:17:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.958901+0000", 2023-10-09T06:17:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.958900+0000", 2023-10-09T06:17:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:17:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.958902+0000", 2023-10-09T06:17:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7519999999999999e-06 2023-10-09T06:17:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.958907+0000", 2023-10-09T06:17:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8389999999999997e-06 2023-10-09T06:17:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.958911+0000", 2023-10-09T06:17:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0099999999999997e-06 2023-10-09T06:17:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.964731+0000", 2023-10-09T06:17:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058204779999999996 2023-10-09T06:17:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.964888+0000", 2023-10-09T06:17:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015670500000000001 2023-10-09T06:17:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:17:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.965038+0000", 2023-10-09T06:17:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015025800000000001 2023-10-09T06:17:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.971444+0000", 2023-10-09T06:17:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0064053349999999998 2023-10-09T06:17:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.987909+0000", 2023-10-09T06:17:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016464909999999999 2023-10-09T06:17:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.987922+0000", 2023-10-09T06:17:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2977e-05 2023-10-09T06:17:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:48.010335+0000", 2023-10-09T06:17:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.022413292 2023-10-09T06:17:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:48.010404+0000", 2023-10-09T06:17:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9220000000000005e-05 2023-10-09T06:17:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:143506 9.6 9:6e6537e4:::benchmark_data_smithi064_511911_object42404:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-10-09T06:17:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:46.965316+0000", 2023-10-09T06:17:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 438.488336506, 2023-10-09T06:17:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.045164429, 2023-10-09T06:17:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:17:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:17:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:17:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 143506 2023-10-09T06:17:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.965316+0000", 2023-10-09T06:17:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.965316+0000", 2023-10-09T06:17:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.965315+0000", 2023-10-09T06:17:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:17:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.965319+0000", 2023-10-09T06:17:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6109999999999998e-06 2023-10-09T06:17:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.965327+0000", 2023-10-09T06:17:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9079999999999995e-06 2023-10-09T06:17:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.965333+0000", 2023-10-09T06:17:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4010000000000002e-06 2023-10-09T06:17:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.967879+0000", 2023-10-09T06:17:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025457129999999998 2023-10-09T06:17:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.968026+0000", 2023-10-09T06:17:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000146689 2023-10-09T06:17:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:17:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.968161+0000", 2023-10-09T06:17:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000135616 2023-10-09T06:17:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.971449+0000", 2023-10-09T06:17:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003287824 2023-10-09T06:17:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.987952+0000", 2023-10-09T06:17:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016502594999999998 2023-10-09T06:17:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:46.987961+0000", 2023-10-09T06:17:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4749999999999999e-06 2023-10-09T06:17:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:48.010430+0000", 2023-10-09T06:17:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0224686780000001 2023-10-09T06:17:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:48.010480+0000", 2023-10-09T06:17:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0445000000000002e-05 2023-10-09T06:17:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:182253 9.0 9:082e4929:::benchmark_data_smithi064_511911_object81151:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T06:17:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:10:49.687732+0000", 2023-10-09T06:17:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 375.76592041200001, 2023-10-09T06:17:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.079514278, 2023-10-09T06:17:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:17:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:17:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:17:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 182253 2023-10-09T06:17:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:49.687732+0000", 2023-10-09T06:17:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:49.687732+0000", 2023-10-09T06:17:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:49.687731+0000", 2023-10-09T06:17:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:17:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:49.687736+0000", 2023-10-09T06:17:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9420000000000003e-06 2023-10-09T06:17:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:49.687745+0000", 2023-10-09T06:17:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5070000000000006e-06 2023-10-09T06:17:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:49.687751+0000", 2023-10-09T06:17:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1419999999999998e-06 2023-10-09T06:17:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:49.687807+0000", 2023-10-09T06:17:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5399000000000003e-05 2023-10-09T06:17:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:49.687979+0000", 2023-10-09T06:17:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017229699999999999 2023-10-09T06:17:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:17:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:49.688142+0000", 2023-10-09T06:17:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016299500000000001 2023-10-09T06:17:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:49.691150+0000", 2023-10-09T06:17:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030079030000000001 2023-10-09T06:17:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:49.701629+0000", 2023-10-09T06:17:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01047931 2023-10-09T06:17:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:49.741720+0000", 2023-10-09T06:17:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.040091162999999999 2023-10-09T06:17:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:49.741731+0000", 2023-10-09T06:17:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0805000000000001e-05 2023-10-09T06:17:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:50.767201+0000", 2023-10-09T06:17:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0254695810000001 2023-10-09T06:17:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:10:50.767246+0000", 2023-10-09T06:17:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5521e-05 2023-10-09T06:17:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:95180 10.b 10:dd6284c2:::benchmark_data_smithi064_573750_object95179:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e521)", 2023-10-09T06:17:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:20.909957+0000", 2023-10-09T06:17:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 224.54369599399999, 2023-10-09T06:17:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.947192375, 2023-10-09T06:17:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:17:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:17:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 95180 2023-10-09T06:17:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.909957+0000", 2023-10-09T06:17:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.909957+0000", 2023-10-09T06:17:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.909955+0000", 2023-10-09T06:17:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:17:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910013+0000", 2023-10-09T06:17:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7899000000000002e-05 2023-10-09T06:17:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910023+0000", 2023-10-09T06:17:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7739999999999999e-06 2023-10-09T06:17:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910031+0000", 2023-10-09T06:17:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3299999999999999e-06 2023-10-09T06:17:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.914656+0000", 2023-10-09T06:17:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046253429999999996 2023-10-09T06:17:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.914667+0000", 2023-10-09T06:17:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0550000000000001e-05 2023-10-09T06:17:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.708405+0000", 2023-10-09T06:17:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.79373826700000005 2023-10-09T06:17:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.708416+0000", 2023-10-09T06:17:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0716999999999999e-05 2023-10-09T06:17:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.841829+0000", 2023-10-09T06:17:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.133413338 2023-10-09T06:17:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.842001+0000", 2023-10-09T06:17:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000171738 2023-10-09T06:17:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,6", 2023-10-09T06:17:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.842131+0000", 2023-10-09T06:17:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000130011 2023-10-09T06:17:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.845569+0000", 2023-10-09T06:17:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034383439999999999 2023-10-09T06:17:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.846409+0000", 2023-10-09T06:17:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00083989199999999996 2023-10-09T06:17:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.847475+0000", 2023-10-09T06:17:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001065727 2023-10-09T06:17:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.856993+0000", 2023-10-09T06:17:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0095179770000000004 2023-10-09T06:17:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.857042+0000", 2023-10-09T06:17:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8761000000000003e-05 2023-10-09T06:17:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.857149+0000", 2023-10-09T06:17:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010726 2023-10-09T06:17:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:95187 10.0 10:0c9801d5:::benchmark_data_smithi064_573750_object95186:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e521)", 2023-10-09T06:17:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:20.910037+0000", 2023-10-09T06:17:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 224.54361521300001, 2023-10-09T06:17:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9651730569999999, 2023-10-09T06:17:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:17:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:17:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 95187 2023-10-09T06:17:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910037+0000", 2023-10-09T06:17:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910037+0000", 2023-10-09T06:17:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910037+0000", 2023-10-09T06:17:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:17:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910083+0000", 2023-10-09T06:17:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6375999999999998e-05 2023-10-09T06:17:05.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:05.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910091+0000", 2023-10-09T06:17:05.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2160000000000006e-06 2023-10-09T06:17:05.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:05.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910098+0000", 2023-10-09T06:17:05.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.494e-06 2023-10-09T06:17:05.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.915482+0000", 2023-10-09T06:17:05.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053844110000000004 2023-10-09T06:17:05.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.915491+0000", 2023-10-09T06:17:05.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7260000000000008e-06 2023-10-09T06:17:05.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.532 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.532 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.706280+0000", 2023-10-09T06:17:05.532 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.79078928900000001 2023-10-09T06:17:05.532 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.532 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.532 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.706293+0000", 2023-10-09T06:17:05.532 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2764e-05 2023-10-09T06:17:05.532 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.532 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.852061+0000", 2023-10-09T06:17:05.533 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.145767838 2023-10-09T06:17:05.533 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.533 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.533 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:05.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.852193+0000", 2023-10-09T06:17:05.533 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013227700000000001 2023-10-09T06:17:05.533 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.533 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.533 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,6", 2023-10-09T06:17:05.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.852336+0000", 2023-10-09T06:17:05.533 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014296400000000001 2023-10-09T06:17:05.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.534 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:05.534 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.866052+0000", 2023-10-09T06:17:05.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013715597 2023-10-09T06:17:05.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.534 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.534 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.869816+0000", 2023-10-09T06:17:05.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003763775 2023-10-09T06:17:05.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.535 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.535 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.869866+0000", 2023-10-09T06:17:05.535 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0725e-05 2023-10-09T06:17:05.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.535 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.535 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.875094+0000", 2023-10-09T06:17:05.535 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0052276029999999999 2023-10-09T06:17:05.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.535 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:05.536 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.875143+0000", 2023-10-09T06:17:05.536 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9153999999999997e-05 2023-10-09T06:17:05.536 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.536 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.536 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:05.536 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.875210+0000", 2023-10-09T06:17:05.536 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7352000000000002e-05 2023-10-09T06:17:05.536 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.537 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:05.537 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.537 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.537 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.537 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:95188 10.0 10:02867f4b:::benchmark_data_smithi064_573750_object95187:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e521)", 2023-10-09T06:17:05.537 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:20.910103+0000", 2023-10-09T06:17:05.537 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 224.54354910999999, 2023-10-09T06:17:05.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.966474829, 2023-10-09T06:17:05.537 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:05.537 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:05.538 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:05.538 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:17:05.538 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:17:05.538 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 95188 2023-10-09T06:17:05.538 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.538 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:05.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.538 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:05.538 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910103+0000", 2023-10-09T06:17:05.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.539 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.539 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:05.539 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910103+0000", 2023-10-09T06:17:05.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.539 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.539 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:05.539 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910103+0000", 2023-10-09T06:17:05.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:17:05.540 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.540 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.540 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:05.540 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910149+0000", 2023-10-09T06:17:05.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5704999999999997e-05 2023-10-09T06:17:05.540 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.540 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.540 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:05.540 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910156+0000", 2023-10-09T06:17:05.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4399999999999999e-06 2023-10-09T06:17:05.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.541 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.541 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:05.541 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910162+0000", 2023-10-09T06:17:05.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.156e-06 2023-10-09T06:17:05.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.541 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.541 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.541 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.915511+0000", 2023-10-09T06:17:05.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053488800000000003 2023-10-09T06:17:05.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.542 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.542 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.915518+0000", 2023-10-09T06:17:05.542 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0350000000000004e-06 2023-10-09T06:17:05.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.542 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.543 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.706332+0000", 2023-10-09T06:17:05.543 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.79081385800000004 2023-10-09T06:17:05.543 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.543 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.543 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.543 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.706341+0000", 2023-10-09T06:17:05.543 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.02e-06 2023-10-09T06:17:05.543 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.543 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.543 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.543 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.855606+0000", 2023-10-09T06:17:05.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.149264721 2023-10-09T06:17:05.544 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.544 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.544 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:05.544 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.855749+0000", 2023-10-09T06:17:05.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014338600000000001 2023-10-09T06:17:05.544 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.544 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.545 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,6", 2023-10-09T06:17:05.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.855904+0000", 2023-10-09T06:17:05.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000154988 2023-10-09T06:17:05.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.545 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:05.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.866060+0000", 2023-10-09T06:17:05.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010156072 2023-10-09T06:17:05.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.546 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.546 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.869906+0000", 2023-10-09T06:17:05.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038456649999999998 2023-10-09T06:17:05.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.546 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.547 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.869946+0000", 2023-10-09T06:17:05.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0096999999999999e-05 2023-10-09T06:17:05.547 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.547 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.547 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.547 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.876416+0000", 2023-10-09T06:17:05.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0064703460000000001 2023-10-09T06:17:05.547 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.547 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.547 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:05.547 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.876471+0000", 2023-10-09T06:17:05.548 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4540000000000003e-05 2023-10-09T06:17:05.548 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.548 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.548 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:05.548 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.876578+0000", 2023-10-09T06:17:05.548 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000107477 2023-10-09T06:17:05.548 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.548 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:05.548 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.548 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.549 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:95189 10.0 10:029c3de1:::benchmark_data_smithi064_573750_object95188:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e521)", 2023-10-09T06:17:05.549 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:20.910168+0000", 2023-10-09T06:17:05.549 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 224.543485011, 2023-10-09T06:17:05.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9667635670000001, 2023-10-09T06:17:05.549 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:05.549 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:05.549 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:05.549 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:17:05.549 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:17:05.549 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 95189 2023-10-09T06:17:05.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.550 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:05.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:05.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910168+0000", 2023-10-09T06:17:05.550 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:05.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910168+0000", 2023-10-09T06:17:05.550 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:05.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910167+0000", 2023-10-09T06:17:05.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:17:05.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:05.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910213+0000", 2023-10-09T06:17:05.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5887999999999998e-05 2023-10-09T06:17:05.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:05.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910221+0000", 2023-10-09T06:17:05.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3189999999999995e-06 2023-10-09T06:17:05.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:05.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910229+0000", 2023-10-09T06:17:05.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7370000000000006e-06 2023-10-09T06:17:05.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.915537+0000", 2023-10-09T06:17:05.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.005308442 2023-10-09T06:17:05.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.915544+0000", 2023-10-09T06:17:05.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5180000000000001e-06 2023-10-09T06:17:05.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.706486+0000", 2023-10-09T06:17:05.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.79094211699999994 2023-10-09T06:17:05.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.706495+0000", 2023-10-09T06:17:05.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3740000000000004e-06 2023-10-09T06:17:05.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.859126+0000", 2023-10-09T06:17:05.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1526308730000001 2023-10-09T06:17:05.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:05.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.859307+0000", 2023-10-09T06:17:05.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000180693 2023-10-09T06:17:05.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,6", 2023-10-09T06:17:05.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.859479+0000", 2023-10-09T06:17:05.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017154599999999999 2023-10-09T06:17:05.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:05.556 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.869771+0000", 2023-10-09T06:17:05.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010292088 2023-10-09T06:17:05.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.556 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.869985+0000", 2023-10-09T06:17:05.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00021465399999999999 2023-10-09T06:17:05.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.557 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.870027+0000", 2023-10-09T06:17:05.557 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2157999999999999e-05 2023-10-09T06:17:05.557 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.557 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.557 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.557 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.876615+0000", 2023-10-09T06:17:05.557 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065879279999999998 2023-10-09T06:17:05.557 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.557 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.557 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:05.557 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.876698+0000", 2023-10-09T06:17:05.557 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2490999999999993e-05 2023-10-09T06:17:05.558 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.558 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.558 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:05.558 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.876931+0000", 2023-10-09T06:17:05.558 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00023325599999999999 2023-10-09T06:17:05.558 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.558 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:05.558 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.558 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.558 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.558 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:95194 10.0 10:09ebd0a1:::benchmark_data_smithi064_573750_object95193:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e521)", 2023-10-09T06:17:05.558 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:20.910235+0000", 2023-10-09T06:17:05.559 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 224.54341725, 2023-10-09T06:17:05.559 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9669212979999999, 2023-10-09T06:17:05.559 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:05.559 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:05.559 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:05.559 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:17:05.559 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:17:05.559 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 95194 2023-10-09T06:17:05.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.559 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:05.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:05.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910235+0000", 2023-10-09T06:17:05.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:05.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910235+0000", 2023-10-09T06:17:05.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:05.561 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910235+0000", 2023-10-09T06:17:05.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:17:05.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.561 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.561 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:05.561 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910285+0000", 2023-10-09T06:17:05.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0624000000000002e-05 2023-10-09T06:17:05.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.561 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.561 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:05.561 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910295+0000", 2023-10-09T06:17:05.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5200000000000003e-06 2023-10-09T06:17:05.562 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.562 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.562 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:05.562 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910302+0000", 2023-10-09T06:17:05.562 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5800000000000003e-06 2023-10-09T06:17:05.562 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.562 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.562 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.562 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.915563+0000", 2023-10-09T06:17:05.562 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0052606229999999999 2023-10-09T06:17:05.562 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.563 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.915570+0000", 2023-10-09T06:17:05.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6250000000000001e-06 2023-10-09T06:17:05.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.563 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.706519+0000", 2023-10-09T06:17:05.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.79094940499999999 2023-10-09T06:17:05.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.706528+0000", 2023-10-09T06:17:05.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7579999999999998e-06 2023-10-09T06:17:05.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.862573+0000", 2023-10-09T06:17:05.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1560447110000001 2023-10-09T06:17:05.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:05.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.862770+0000", 2023-10-09T06:17:05.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019734200000000001 2023-10-09T06:17:05.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,6", 2023-10-09T06:17:05.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.862983+0000", 2023-10-09T06:17:05.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00021307699999999999 2023-10-09T06:17:05.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:05.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.869826+0000", 2023-10-09T06:17:05.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0068432709999999997 2023-10-09T06:17:05.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.566 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.566 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.874683+0000", 2023-10-09T06:17:05.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048568780000000002 2023-10-09T06:17:05.566 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.566 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.566 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.874719+0000", 2023-10-09T06:17:05.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5805e-05 2023-10-09T06:17:05.566 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.566 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.566 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.877016+0000", 2023-10-09T06:17:05.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0022973009999999999 2023-10-09T06:17:05.567 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.567 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:05.567 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.877069+0000", 2023-10-09T06:17:05.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3176e-05 2023-10-09T06:17:05.567 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.567 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:05.567 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.877157+0000", 2023-10-09T06:17:05.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7130000000000001e-05 2023-10-09T06:17:05.567 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.568 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:05.568 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.568 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.568 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:136492 10.0 10:060f4938:::benchmark_data_smithi064_573750_object40277:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-10-09T06:17:05.568 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:27.688648+0000", 2023-10-09T06:17:05.568 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 157.765004809, 2023-10-09T06:17:05.568 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.057387955, 2023-10-09T06:17:05.568 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:05.568 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:17:05.568 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:05.568 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:17:05.569 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:17:05.569 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 136492 2023-10-09T06:17:05.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.569 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:05.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:05.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.688648+0000", 2023-10-09T06:17:05.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:05.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.688648+0000", 2023-10-09T06:17:05.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.570 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.570 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.570 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:05.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.688647+0000", 2023-10-09T06:17:05.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:17:05.570 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.570 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.570 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:05.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.688650+0000", 2023-10-09T06:17:05.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0589999999999998e-06 2023-10-09T06:17:05.570 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.570 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.571 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:05.571 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.688659+0000", 2023-10-09T06:17:05.571 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3089999999999996e-06 2023-10-09T06:17:05.571 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.571 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.571 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:05.571 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.688664+0000", 2023-10-09T06:17:05.571 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9980000000000003e-06 2023-10-09T06:17:05.571 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.571 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.571 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.571 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.692528+0000", 2023-10-09T06:17:05.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003864335 2023-10-09T06:17:05.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.572 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.572 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:05.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.692650+0000", 2023-10-09T06:17:05.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000122373 2023-10-09T06:17:05.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.572 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.572 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,6", 2023-10-09T06:17:05.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.692765+0000", 2023-10-09T06:17:05.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011428499999999999 2023-10-09T06:17:05.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:05.573 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.702209+0000", 2023-10-09T06:17:05.573 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0094443300000000008 2023-10-09T06:17:05.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.573 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.707279+0000", 2023-10-09T06:17:05.573 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050699530000000003 2023-10-09T06:17:05.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.713389+0000", 2023-10-09T06:17:05.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061103319999999996 2023-10-09T06:17:05.574 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.574 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.574 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.713405+0000", 2023-10-09T06:17:05.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6257e-05 2023-10-09T06:17:05.574 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.574 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.574 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:28.745946+0000", 2023-10-09T06:17:05.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0325401729999999 2023-10-09T06:17:05.575 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.575 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.575 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:05.575 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:28.746036+0000", 2023-10-09T06:17:05.575 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0019999999999995e-05 2023-10-09T06:17:05.575 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.575 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:05.575 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.575 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.575 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.575 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:136497 10.0 10:0824707d:::benchmark_data_smithi064_573750_object40282:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-10-09T06:17:05.576 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:27.698007+0000", 2023-10-09T06:17:05.576 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 157.755645636, 2023-10-09T06:17:05.576 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.048447959, 2023-10-09T06:17:05.576 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:05.576 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:17:05.576 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:05.576 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:17:05.576 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:17:05.576 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 136497 2023-10-09T06:17:05.576 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.577 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:05.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:05.577 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.698007+0000", 2023-10-09T06:17:05.577 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:05.577 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.698007+0000", 2023-10-09T06:17:05.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.578 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.578 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:05.578 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.698005+0000", 2023-10-09T06:17:05.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:17:05.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.578 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.578 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:05.578 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.698010+0000", 2023-10-09T06:17:05.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1949999999999996e-06 2023-10-09T06:17:05.579 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.579 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:05.579 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.698018+0000", 2023-10-09T06:17:05.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5750000000000001e-06 2023-10-09T06:17:05.579 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.579 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:05.579 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.698024+0000", 2023-10-09T06:17:05.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6930000000000004e-06 2023-10-09T06:17:05.579 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.580 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.580 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.698195+0000", 2023-10-09T06:17:05.580 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000171399 2023-10-09T06:17:05.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.580 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.580 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:05.580 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.698314+0000", 2023-10-09T06:17:05.580 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118952 2023-10-09T06:17:05.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.580 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.580 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,6", 2023-10-09T06:17:05.580 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.698432+0000", 2023-10-09T06:17:05.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011773999999999999 2023-10-09T06:17:05.581 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.581 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:05.581 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.704743+0000", 2023-10-09T06:17:05.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063113170000000003 2023-10-09T06:17:05.581 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.581 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.581 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.709914+0000", 2023-10-09T06:17:05.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051711350000000003 2023-10-09T06:17:05.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.582 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.582 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.715484+0000", 2023-10-09T06:17:05.582 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0055690749999999997 2023-10-09T06:17:05.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.582 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.582 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.715498+0000", 2023-10-09T06:17:05.582 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4932e-05 2023-10-09T06:17:05.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.583 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:28.746379+0000", 2023-10-09T06:17:05.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.030880166 2023-10-09T06:17:05.583 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.583 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.583 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:05.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:28.746455+0000", 2023-10-09T06:17:05.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6191000000000003e-05 2023-10-09T06:17:05.583 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.583 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:05.583 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.584 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.584 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.584 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:136509 10.0 10:0e6bfcd4:::benchmark_data_smithi064_573750_object40294:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e545)", 2023-10-09T06:17:05.584 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:27.725325+0000", 2023-10-09T06:17:05.584 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 157.728327213, 2023-10-09T06:17:05.584 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.031877465, 2023-10-09T06:17:05.584 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:05.584 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:05.584 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:05.584 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:17:05.584 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:17:05.584 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 136509 2023-10-09T06:17:05.584 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.585 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:05.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.585 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:05.585 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.725325+0000", 2023-10-09T06:17:05.585 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.585 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.585 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:05.585 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.725325+0000", 2023-10-09T06:17:05.585 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.585 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.586 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.586 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:05.586 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.725324+0000", 2023-10-09T06:17:05.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:17:05.586 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.586 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.586 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:05.586 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.725330+0000", 2023-10-09T06:17:05.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4370000000000004e-06 2023-10-09T06:17:05.586 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.586 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:05.587 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.725342+0000", 2023-10-09T06:17:05.587 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2388999999999999e-05 2023-10-09T06:17:05.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:05.587 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.725350+0000", 2023-10-09T06:17:05.587 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.481e-06 2023-10-09T06:17:05.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.587 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.725413+0000", 2023-10-09T06:17:05.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3338999999999998e-05 2023-10-09T06:17:05.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:05.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:27.725425+0000", 2023-10-09T06:17:05.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1521e-05 2023-10-09T06:17:05.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:28.746498+0000", 2023-10-09T06:17:05.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0210735580000001 2023-10-09T06:17:05.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:05.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:28.746636+0000", 2023-10-09T06:17:05.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000137727 2023-10-09T06:17:05.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:17:05.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:28.746739+0000", 2023-10-09T06:17:05.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010265200000000001 2023-10-09T06:17:05.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.590 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.590 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:28.753492+0000", 2023-10-09T06:17:05.590 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067535750000000004 2023-10-09T06:17:05.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.590 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.590 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:28.753532+0000", 2023-10-09T06:17:05.590 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9440000000000002e-05 2023-10-09T06:17:05.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.590 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:05.591 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:28.757120+0000", 2023-10-09T06:17:05.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035885819999999999 2023-10-09T06:17:05.591 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.591 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.591 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:05.591 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:28.757163+0000", 2023-10-09T06:17:05.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2308999999999998e-05 2023-10-09T06:17:05.591 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.591 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.591 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:05.591 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:28.757203+0000", 2023-10-09T06:17:05.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0227e-05 2023-10-09T06:17:05.592 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.592 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:05.592 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.592 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.592 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.592 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:57135 11.0 11:04dc85fd:::benchmark_data_smithi064_631903_object57134:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e583)", 2023-10-09T06:17:05.592 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:16:42.963168+0000", 2023-10-09T06:17:05.592 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 22.490484348999999, 2023-10-09T06:17:05.592 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.212676905, 2023-10-09T06:17:05.592 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:05.592 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:05.593 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:05.593 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:17:05.593 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:17:05.593 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 57135 2023-10-09T06:17:05.593 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.593 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:05.593 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:05.593 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:42.963168+0000", 2023-10-09T06:17:05.593 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.593 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.593 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.594 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:05.594 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:42.963168+0000", 2023-10-09T06:17:05.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.594 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.594 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:05.594 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:42.963167+0000", 2023-10-09T06:17:05.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:17:05.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.594 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.594 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:05.595 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:42.963190+0000", 2023-10-09T06:17:05.595 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2730000000000001e-05 2023-10-09T06:17:05.595 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.595 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.595 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:05.595 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:42.963199+0000", 2023-10-09T06:17:05.595 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2799999999999992e-06 2023-10-09T06:17:05.595 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.595 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.595 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:05.595 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:42.963204+0000", 2023-10-09T06:17:05.595 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0860000000000001e-06 2023-10-09T06:17:05.596 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.596 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.596 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:42.963248+0000", 2023-10-09T06:17:05.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3859999999999997e-05 2023-10-09T06:17:05.596 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.596 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.596 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:05.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:42.963381+0000", 2023-10-09T06:17:05.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013257899999999999 2023-10-09T06:17:05.596 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.596 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.597 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:17:05.597 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:42.963507+0000", 2023-10-09T06:17:05.597 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012589799999999999 2023-10-09T06:17:05.597 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.597 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.597 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:05.597 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:42.965302+0000", 2023-10-09T06:17:05.597 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001795669 2023-10-09T06:17:05.597 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.597 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.597 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.597 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:43.173045+0000", 2023-10-09T06:17:05.597 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20774216400000001 2023-10-09T06:17:05.598 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.598 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.598 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.598 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:43.175764+0000", 2023-10-09T06:17:05.598 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027199099999999999 2023-10-09T06:17:05.598 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.598 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.598 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:05.598 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:43.175809+0000", 2023-10-09T06:17:05.598 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4081e-05 2023-10-09T06:17:05.598 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.599 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:05.599 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:43.175845+0000", 2023-10-09T06:17:05.599 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6528e-05 2023-10-09T06:17:05.599 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.599 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:05.599 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.599 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.599 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:64679 11.0 11:07c9f2af:::benchmark_data_smithi064_631903_object64678:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-10-09T06:17:05.599 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:16:49.680661+0000", 2023-10-09T06:17:05.599 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 15.772991383000001, 2023-10-09T06:17:05.600 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28262058800000001, 2023-10-09T06:17:05.600 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:05.600 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:05.600 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:05.600 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:17:05.600 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:17:05.600 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 64679 2023-10-09T06:17:05.600 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.600 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:05.600 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.600 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:05.600 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.680661+0000", 2023-10-09T06:17:05.601 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.601 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.601 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.601 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:05.601 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.680661+0000", 2023-10-09T06:17:05.601 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.601 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.601 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.601 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:05.601 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.680660+0000", 2023-10-09T06:17:05.601 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:17:05.601 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.602 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.602 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:05.602 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.680681+0000", 2023-10-09T06:17:05.602 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0327e-05 2023-10-09T06:17:05.602 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.602 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.602 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:05.602 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.680691+0000", 2023-10-09T06:17:05.602 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0217999999999999e-05 2023-10-09T06:17:05.602 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.602 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.603 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:05.603 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.680698+0000", 2023-10-09T06:17:05.603 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4030000000000001e-06 2023-10-09T06:17:05.603 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.603 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.603 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.683324+0000", 2023-10-09T06:17:05.603 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026253359999999998 2023-10-09T06:17:05.603 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.603 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:05.603 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.683409+0000", 2023-10-09T06:17:05.604 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5612000000000006e-05 2023-10-09T06:17:05.604 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.604 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.604 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:17:05.604 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.683533+0000", 2023-10-09T06:17:05.604 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000123151 2023-10-09T06:17:05.604 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.604 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.604 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:05.604 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.806974+0000", 2023-10-09T06:17:05.604 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.12344195600000001 2023-10-09T06:17:05.604 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.807304+0000", 2023-10-09T06:17:05.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00032903400000000002 2023-10-09T06:17:05.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.963195+0000", 2023-10-09T06:17:05.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15589098200000001 2023-10-09T06:17:05.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:05.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.963241+0000", 2023-10-09T06:17:05.606 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6431000000000003e-05 2023-10-09T06:17:05.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:05.606 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.963282+0000", 2023-10-09T06:17:05.606 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0785000000000003e-05 2023-10-09T06:17:05.606 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.606 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:05.606 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.607 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:64685 11.7 11:ef8a947c:::benchmark_data_smithi064_631903_object64684:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-10-09T06:17:05.607 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:16:49.683662+0000", 2023-10-09T06:17:05.607 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 15.769990592999999, 2023-10-09T06:17:05.607 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28063891899999999, 2023-10-09T06:17:05.607 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:05.607 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:05.607 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:05.607 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:17:05.607 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:17:05.607 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 64685 2023-10-09T06:17:05.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.607 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:05.608 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.608 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:05.608 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.683662+0000", 2023-10-09T06:17:05.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.608 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.608 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.608 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:05.608 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.683662+0000", 2023-10-09T06:17:05.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.608 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.608 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.609 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:05.609 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.683660+0000", 2023-10-09T06:17:05.609 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:17:05.609 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.609 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.609 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:05.609 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.683684+0000", 2023-10-09T06:17:05.609 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3722000000000001e-05 2023-10-09T06:17:05.609 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.609 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.609 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:05.609 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.683696+0000", 2023-10-09T06:17:05.610 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2e-05 2023-10-09T06:17:05.610 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.610 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.610 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:05.610 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.683704+0000", 2023-10-09T06:17:05.610 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0790000000000001e-06 2023-10-09T06:17:05.610 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.610 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.610 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.610 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.804591+0000", 2023-10-09T06:17:05.610 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.120886352 2023-10-09T06:17:05.610 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.611 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:05.611 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.804746+0000", 2023-10-09T06:17:05.611 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000155296 2023-10-09T06:17:05.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.611 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:17:05.611 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.804885+0000", 2023-10-09T06:17:05.611 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000138665 2023-10-09T06:17:05.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.611 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:05.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.963926+0000", 2023-10-09T06:17:05.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15904149300000001 2023-10-09T06:17:05.612 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.963964+0000", 2023-10-09T06:17:05.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7450000000000002e-05 2023-10-09T06:17:05.612 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.964211+0000", 2023-10-09T06:17:05.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00024763999999999998 2023-10-09T06:17:05.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:05.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.964268+0000", 2023-10-09T06:17:05.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6338999999999997e-05 2023-10-09T06:17:05.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:05.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.964301+0000", 2023-10-09T06:17:05.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3324000000000001e-05 2023-10-09T06:17:05.613 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.614 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:05.614 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.614 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:64687 11.0 11:07120844:::benchmark_data_smithi064_631903_object64686:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-10-09T06:17:05.614 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:16:49.683713+0000", 2023-10-09T06:17:05.614 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 15.769940006000001, 2023-10-09T06:17:05.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.279678608, 2023-10-09T06:17:05.614 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:05.614 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:05.614 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:05.615 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:17:05.615 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:17:05.615 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 64687 2023-10-09T06:17:05.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.615 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:05.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.615 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:05.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.683713+0000", 2023-10-09T06:17:05.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:05.616 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.683713+0000", 2023-10-09T06:17:05.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:05.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:05.616 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.683712+0000", 2023-10-09T06:17:05.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:17:05.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:05.616 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.683732+0000", 2023-10-09T06:17:05.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0234999999999999e-05 2023-10-09T06:17:05.617 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.617 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.617 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:05.617 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.683739+0000", 2023-10-09T06:17:05.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8800000000000002e-06 2023-10-09T06:17:05.617 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.617 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.617 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:05.617 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.683744+0000", 2023-10-09T06:17:05.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2970000000000003e-06 2023-10-09T06:17:05.617 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.618 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.618 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:05.618 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.685361+0000", 2023-10-09T06:17:05.618 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016163939999999999 2023-10-09T06:17:05.618 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.618 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.618 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:05.618 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.804060+0000", 2023-10-09T06:17:05.618 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.11869895699999999 2023-10-09T06:17:05.618 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.618 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.618 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:17:05.619 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.804480+0000", 2023-10-09T06:17:05.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000420295 2023-10-09T06:17:05.619 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.619 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:05.619 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.807635+0000", 2023-10-09T06:17:05.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031546759999999999 2023-10-09T06:17:05.619 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.619 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.619 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.963057+0000", 2023-10-09T06:17:05.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15542235800000001 2023-10-09T06:17:05.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.620 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:05.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.963320+0000", 2023-10-09T06:17:05.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000263062 2023-10-09T06:17:05.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.620 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:05.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.963358+0000", 2023-10-09T06:17:05.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8068999999999998e-05 2023-10-09T06:17:05.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:05.620 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:05.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:05.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:16:49.963391+0000", 2023-10-09T06:17:05.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2898999999999998e-05 2023-10-09T06:17:05.621 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.621 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:05.621 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.621 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:05.621 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:05.621 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:17:05.622 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:17:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:17:05] "GET /metrics HTTP/1.1" 200 36435 "" "Prometheus/2.43.0" 2023-10-09T06:17:05.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:05.623 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:17:05.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:05.724 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:05.785 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:05.870 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:17:05.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:05.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:05.979 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:06.003 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:17:06.023 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:06.052 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 56 16 80484 80468 89.7828 63.4375 0.0732722 0.0111239 2023-10-09T06:17:06.073 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:17:06.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:06.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:06.182 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:06.278 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:06.811 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:17:07.052 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 57 16 81355 81339 89.1627 54.4375 0.0185491 0.0112032 2023-10-09T06:17:07.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:06 smithi156 ceph-mon[195101]: pgmap v1503: 119 pgs: 119 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:17:07.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:06 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1476721985' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:17:07.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:06 smithi064 ceph-mon[212750]: pgmap v1503: 119 pgs: 119 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:17:07.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:06 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1476721985' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:17:07.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:06 smithi064 ceph-mon[220044]: pgmap v1503: 119 pgs: 119 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:17:07.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:06 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1476721985' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:17:07.600 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:07.609 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:17:07.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:07.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:07.717 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:07.772 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:07.840 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:17:07.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:07.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:07.954 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:07.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:07 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:17:07.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:07 smithi064 ceph-mon[212750]: failed to encode map e588 with expected crc 2023-10-09T06:17:07.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:07 smithi064 ceph-mon[212750]: failed to encode map e588 with expected crc 2023-10-09T06:17:07.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:07 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1476721985' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:17:07.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:07 smithi064 ceph-mon[212750]: osdmap e588: 8 total, 8 up, 7 in 2023-10-09T06:17:07.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:07 smithi064 ceph-mon[212750]: failed to encode map e588 with expected crc 2023-10-09T06:17:07.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:07 smithi064 ceph-mon[212750]: failed to encode map e588 with expected crc 2023-10-09T06:17:07.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:07 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:17:07.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:07 smithi064 ceph-mon[220044]: failed to encode map e588 with expected crc 2023-10-09T06:17:07.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:07 smithi064 ceph-mon[220044]: failed to encode map e588 with expected crc 2023-10-09T06:17:07.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:07 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1476721985' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:17:07.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:07 smithi064 ceph-mon[220044]: osdmap e588: 8 total, 8 up, 7 in 2023-10-09T06:17:07.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:07 smithi064 ceph-mon[220044]: failed to encode map e588 with expected crc 2023-10-09T06:17:07.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:07 smithi064 ceph-mon[220044]: failed to encode map e588 with expected crc 2023-10-09T06:17:07.983 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:08.052 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 58 16 82280 82264 88.6222 57.8125 0.0155136 0.0112786 2023-10-09T06:17:08.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:07 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:17:08.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:07 smithi156 ceph-mon[195101]: failed to encode map e588 with expected crc 2023-10-09T06:17:08.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:07 smithi156 ceph-mon[195101]: failed to encode map e588 with expected crc 2023-10-09T06:17:08.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:07 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1476721985' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:17:08.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:07 smithi156 ceph-mon[195101]: osdmap e588: 8 total, 8 up, 7 in 2023-10-09T06:17:08.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:07 smithi156 ceph-mon[195101]: failed to encode map e588 with expected crc 2023-10-09T06:17:08.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:07 smithi156 ceph-mon[195101]: failed to encode map e588 with expected crc 2023-10-09T06:17:08.304 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:08.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:08.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:08.411 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:08.492 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:08.701 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:08.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:08.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:08.805 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:08.825 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:08.884 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:08 smithi064 ceph-mon[212750]: pgmap v1505: 119 pgs: 119 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T06:17:08.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:08 smithi064 ceph-mon[220044]: pgmap v1505: 119 pgs: 119 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T06:17:08.892 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:09.021 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:09.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:09.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:09.058 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 59 16 83265 83249 88.1635 61.5625 0.00724055 0.0113376 2023-10-09T06:17:09.071 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:09.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:08 smithi156 ceph-mon[195101]: pgmap v1505: 119 pgs: 119 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T06:17:09.132 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:09.212 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:09.307 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:09.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:09.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:09.420 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:09.464 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:09.554 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:17:09.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:09.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:09.664 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:09.733 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:09.816 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:17:09.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:09.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:09.845 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T06:17:09.928 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:09.963 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:10.020 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:17:10.025 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:09 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2145998720' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:17:10.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:10.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:10.052 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-10-09T06:17:10.053321+0000 min lat: 0.00113085 max lat: 0.307063 avg lat: 0.0113991 2023-10-09T06:17:10.052 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:17:10.053 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 60 16 84219 84203 87.6879 59.625 0.00857437 0.0113991 2023-10-09T06:17:10.127 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:10.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:09 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2145998720' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:17:10.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:09 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2145998720' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:17:10.229 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:10.436 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:17:10.819 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:10 smithi064 ceph-mon[220044]: pgmap v1506: 119 pgs: 119 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 54 MiB/s wr, 857 op/s 2023-10-09T06:17:10.819 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:10 smithi064 ceph-mon[220044]: failed to encode map e589 with expected crc 2023-10-09T06:17:10.819 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:10 smithi064 ceph-mon[220044]: failed to encode map e589 with expected crc 2023-10-09T06:17:10.819 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:10 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2145998720' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:17:10.819 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:10 smithi064 ceph-mon[220044]: osdmap e589: 8 total, 8 up, 7 in 2023-10-09T06:17:10.819 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:10 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:17:11.053 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 61 16 84938 84922 86.987 44.9375 0.0118758 0.011488 2023-10-09T06:17:11.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:10 smithi156 ceph-mon[195101]: pgmap v1506: 119 pgs: 119 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 54 MiB/s wr, 857 op/s 2023-10-09T06:17:11.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:10 smithi156 ceph-mon[195101]: failed to encode map e589 with expected crc 2023-10-09T06:17:11.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:10 smithi156 ceph-mon[195101]: failed to encode map e589 with expected crc 2023-10-09T06:17:11.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:10 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2145998720' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:17:11.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:10 smithi156 ceph-mon[195101]: osdmap e589: 8 total, 8 up, 7 in 2023-10-09T06:17:11.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:10 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:17:11.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:10 smithi064 ceph-mon[212750]: pgmap v1506: 119 pgs: 119 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 54 MiB/s wr, 857 op/s 2023-10-09T06:17:11.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:10 smithi064 ceph-mon[212750]: failed to encode map e589 with expected crc 2023-10-09T06:17:11.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:10 smithi064 ceph-mon[212750]: failed to encode map e589 with expected crc 2023-10-09T06:17:11.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:10 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2145998720' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:17:11.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:10 smithi064 ceph-mon[212750]: osdmap e589: 8 total, 8 up, 7 in 2023-10-09T06:17:11.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:10 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:17:11.708 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:17:11.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:11.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:11.811 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:11.855 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:11.941 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:17:11.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:11.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:12.050 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:12.054 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 62 16 85735 85719 86.3871 49.8125 0.0177333 0.0115702 2023-10-09T06:17:12.107 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T06:17:12.156 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:12.338 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:17:12.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:12.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:12.450 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:12.514 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:12.603 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:17:12.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:12.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:12.626 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:12.711 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:12.778 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:12.806 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:17:12.806 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T06:17:12.806 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2023-10-09T06:17:12.806 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:17:12.855 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:17:12.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:12.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:12.947 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:17:12.964 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:12.989 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:17:13.039 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:17:13.047 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:13.053 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 63 16 86553 86537 85.8274 51.125 0.00799293 0.0116458 2023-10-09T06:17:13.077 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:17:13.118 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:17:13.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:12 smithi156 ceph-mon[195101]: pgmap v1508: 119 pgs: 119 active+clean; 5.1 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T06:17:13.160 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:17:13.199 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:17:13.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:12 smithi064 ceph-mon[212750]: pgmap v1508: 119 pgs: 119 active+clean; 5.1 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T06:17:13.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:12 smithi064 ceph-mon[220044]: pgmap v1508: 119 pgs: 119 active+clean; 5.1 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T06:17:13.237 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:17:13.246 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:17:13.246 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:17:13.246 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:17:13.247 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:17:13.247 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:17:13.247 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:17:13.247 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:17:13.247 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:17:13.894 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:13.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:13.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:14.008 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:14.053 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 64 16 87300 87284 85.2159 46.6875 0.0157284 0.0117303 2023-10-09T06:17:14.065 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:14.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:13 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2232011870' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:17:14.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:13 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:14.133 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:17:14.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:14.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:14.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:13 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2232011870' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:17:14.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:13 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:14.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:13 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2232011870' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:17:14.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:13 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:14.238 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:14.307 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:14.658 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:14.701 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:17:14.820 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:17:14.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:14.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:14.868 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:14 smithi064 ceph-mon[212750]: pgmap v1509: 119 pgs: 119 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T06:17:14.928 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:14.995 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:15.053 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 65 16 88216 88200 84.7856 57.25 0.0184806 0.0117892 2023-10-09T06:17:15.095 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:17:15.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:15.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:15.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:14 smithi156 ceph-mon[195101]: pgmap v1509: 119 pgs: 119 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T06:17:15.202 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:15.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:14 smithi064 ceph-mon[220044]: pgmap v1509: 119 pgs: 119 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T06:17:15.238 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:15.331 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:17:15.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:15.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:15.443 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:15.511 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:15.657 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:17:15.661 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:17:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:17:15] "GET /metrics HTTP/1.1" 200 36444 "" "Prometheus/2.43.0" 2023-10-09T06:17:15.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:15.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:15.764 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:15.816 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:15.887 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:17:15.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:15.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:15.995 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:16.040 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:16.055 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 66 16 89054 89038 84.2926 52.375 0.0240769 0.0118579 2023-10-09T06:17:16.123 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:17:16.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:16.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:16.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:16 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:17:16.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:16 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:17:16.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:16 smithi064 ceph-mon[212750]: pgmap v1510: 119 pgs: 119 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 50 MiB/s wr, 795 op/s 2023-10-09T06:17:16.229 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:16.281 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:16.360 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:17:16.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:16.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:16.466 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:16.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:16 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:17:16.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:16 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:17:16.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:16 smithi064 ceph-mon[220044]: pgmap v1510: 119 pgs: 119 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 50 MiB/s wr, 795 op/s 2023-10-09T06:17:16.521 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:16 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:17:16.521 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:16 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:17:16.521 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:16 smithi156 ceph-mon[195101]: pgmap v1510: 119 pgs: 119 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 50 MiB/s wr, 795 op/s 2023-10-09T06:17:16.524 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:16.620 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:17:16.634 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:17:16.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:16.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:16.740 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:16.804 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:17.057 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 67 16 89839 89823 83.7645 49.0625 0.0247988 0.0119333 2023-10-09T06:17:17.370 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:17:17.380 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:17:17.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:17.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:17.489 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:17.529 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:17.546 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:17.604 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:17:17.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:17.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:17.608 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:17 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3611975292' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:17:17.608 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:17 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:17:17.709 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:17.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:17 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3611975292' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:17:17.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:17 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:17:17.725 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:17 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3611975292' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:17:17.725 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:17 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:17:17.790 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:18.057 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 68 16 90757 90741 83.3759 57.375 0.0107234 0.0119887 2023-10-09T06:17:18.503 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:17:18.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:18.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:18.607 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:18.618 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:18 smithi064 ceph-mon[220044]: pgmap v1511: 119 pgs: 119 active+clean; 5.4 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T06:17:18.618 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:18 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:17:18.618 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:18 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T06:17:18.618 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:18 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:17:18.618 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:18 smithi064 ceph-mon[220044]: osdmap e590: 8 total, 8 up, 7 in 2023-10-09T06:17:18.618 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:18 smithi064 ceph-mon[220044]: failed to encode map e590 with expected crc 2023-10-09T06:17:18.619 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:18 smithi064 ceph-mon[220044]: failed to encode map e590 with expected crc 2023-10-09T06:17:18.619 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:18 smithi064 ceph-mon[220044]: failed to encode map e590 with expected crc 2023-10-09T06:17:18.619 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:18 smithi064 ceph-mon[212750]: pgmap v1511: 119 pgs: 119 active+clean; 5.4 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T06:17:18.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:18 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:17:18.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:18 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:17:18.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:18 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:17:18.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:18 smithi064 ceph-mon[212750]: osdmap e590: 8 total, 8 up, 7 in 2023-10-09T06:17:18.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:18 smithi064 ceph-mon[212750]: failed to encode map e590 with expected crc 2023-10-09T06:17:18.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:18 smithi064 ceph-mon[212750]: failed to encode map e590 with expected crc 2023-10-09T06:17:18.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:18 smithi064 ceph-mon[212750]: failed to encode map e590 with expected crc 2023-10-09T06:17:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:18 smithi156 ceph-mon[195101]: pgmap v1511: 119 pgs: 119 active+clean; 5.4 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T06:17:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:18 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:17:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:18 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T06:17:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:18 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:17:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:18 smithi156 ceph-mon[195101]: osdmap e590: 8 total, 8 up, 7 in 2023-10-09T06:17:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:18 smithi156 ceph-mon[195101]: failed to encode map e590 with expected crc 2023-10-09T06:17:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:18 smithi156 ceph-mon[195101]: failed to encode map e590 with expected crc 2023-10-09T06:17:18.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:18 smithi156 ceph-mon[195101]: failed to encode map e590 with expected crc 2023-10-09T06:17:18.683 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:18.854 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:19.057 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 69 16 91699 91683 83.0208 58.875 0.0059162 0.0120384 2023-10-09T06:17:19.135 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:19.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:19.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:19.245 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:19.312 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:19.968 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:17:19.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:19.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:20.057 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 70 16 92527 92511 82.5741 51.75 0.0322258 0.0121061 2023-10-09T06:17:20.080 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:20.198 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:20.393 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T06:17:20.492 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:17:20.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:20.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:20.602 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:20.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:20 smithi156 ceph-mon[195101]: pgmap v1513: 119 pgs: 119 active+clean; 5.4 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 49 MiB/s wr, 786 op/s 2023-10-09T06:17:20.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:20 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/531735610' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:17:20.663 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:20.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:20 smithi064 ceph-mon[212750]: pgmap v1513: 119 pgs: 119 active+clean; 5.4 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 49 MiB/s wr, 786 op/s 2023-10-09T06:17:20.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:20 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/531735610' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:17:20.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:20 smithi064 ceph-mon[220044]: pgmap v1513: 119 pgs: 119 active+clean; 5.4 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 49 MiB/s wr, 786 op/s 2023-10-09T06:17:20.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:20 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/531735610' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:17:20.762 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:17:20.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:20.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:20.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:20.889 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:17:20.909 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:21.057 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 71 16 93357 93341 82.1417 51.875 0.025102 0.0121698 2023-10-09T06:17:21.328 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:17:21.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:21.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:21.439 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:21.475 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:21.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:21 smithi064 ceph-mon[212750]: failed to encode map e591 with expected crc 2023-10-09T06:17:21.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:21 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/531735610' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:17:21.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:21 smithi064 ceph-mon[212750]: osdmap e591: 8 total, 8 up, 7 in 2023-10-09T06:17:21.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:21 smithi064 ceph-mon[212750]: failed to encode map e591 with expected crc 2023-10-09T06:17:21.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:21 smithi064 ceph-mon[212750]: failed to encode map e591 with expected crc 2023-10-09T06:17:21.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:21 smithi064 ceph-mon[212750]: failed to encode map e591 with expected crc 2023-10-09T06:17:21.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:21 smithi064 ceph-mon[220044]: failed to encode map e591 with expected crc 2023-10-09T06:17:21.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:21 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/531735610' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:17:21.725 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:21 smithi064 ceph-mon[220044]: osdmap e591: 8 total, 8 up, 7 in 2023-10-09T06:17:21.725 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:21 smithi064 ceph-mon[220044]: failed to encode map e591 with expected crc 2023-10-09T06:17:21.725 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:21 smithi064 ceph-mon[220044]: failed to encode map e591 with expected crc 2023-10-09T06:17:21.725 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:21 smithi064 ceph-mon[220044]: failed to encode map e591 with expected crc 2023-10-09T06:17:21.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:21 smithi156 ceph-mon[195101]: failed to encode map e591 with expected crc 2023-10-09T06:17:21.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:21 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/531735610' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:17:21.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:21 smithi156 ceph-mon[195101]: osdmap e591: 8 total, 8 up, 7 in 2023-10-09T06:17:21.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:21 smithi156 ceph-mon[195101]: failed to encode map e591 with expected crc 2023-10-09T06:17:21.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:21 smithi156 ceph-mon[195101]: failed to encode map e591 with expected crc 2023-10-09T06:17:21.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:21 smithi156 ceph-mon[195101]: failed to encode map e591 with expected crc 2023-10-09T06:17:21.975 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:21.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:21.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:22.058 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 72 16 94125 94109 81.6676 48 0.0107296 0.0122404 2023-10-09T06:17:22.084 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:22.159 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:22.230 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:17:22.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:22.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:22.255 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T06:17:22.256 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:17:22.339 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:22.380 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:22.451 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:17:22.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:22.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:22.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:22 smithi064 ceph-mon[220044]: pgmap v1515: 119 pgs: 119 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s 2023-10-09T06:17:22.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:22 smithi064 ceph-mon[212750]: pgmap v1515: 119 pgs: 119 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s 2023-10-09T06:17:22.562 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:22.594 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:22.672 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:17:22.683 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:22 smithi156 ceph-mon[195101]: pgmap v1515: 119 pgs: 119 active+clean; 5.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s 2023-10-09T06:17:22.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:22.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:22.738 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:22.784 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:22.792 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:17:22.792 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:17:22.927 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:17:22.928 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:22.974 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:17:23.011 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:17:23.058 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 73 16 94895 94879 81.2082 48.125 0.0157436 0.0123083 2023-10-09T06:17:23.059 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:17:23.102 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:17:23.138 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:17:23.172 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:17:23.212 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:17:23.218 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:17:23.218 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:17:23.218 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:17:23.218 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:17:23.218 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:17:23.218 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:17:23.219 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:17:23.219 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:17:23.384 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:23.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:23.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:23.491 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:23.529 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:23.812 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:23 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1800083356' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:17:23.812 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:23 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1800083356' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:17:24.058 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 74 16 95807 95791 80.881 57 0.00508786 0.0123585 2023-10-09T06:17:24.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:23 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1800083356' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:17:24.792 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:24.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:24.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:24.900 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:24.958 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:17:24.977 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:25.022 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:24 smithi064 ceph-mon[220044]: pgmap v1516: 119 pgs: 119 active+clean; 5.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 58 MiB/s wr, 920 op/s 2023-10-09T06:17:25.025 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:24 smithi064 ceph-mon[212750]: pgmap v1516: 119 pgs: 119 active+clean; 5.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 58 MiB/s wr, 920 op/s 2023-10-09T06:17:25.058 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 75 16 96748 96732 80.5868 58.8125 0.0102109 0.012404 2023-10-09T06:17:25.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:24 smithi156 ceph-mon[195101]: pgmap v1516: 119 pgs: 119 active+clean; 5.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 58 MiB/s wr, 920 op/s 2023-10-09T06:17:25.181 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:25.325 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:17:25.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:25.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:25.439 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:25.480 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:25.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:17:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:17:25] "GET /metrics HTTP/1.1" 200 36444 "" "Prometheus/2.43.0" 2023-10-09T06:17:25.940 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:25.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:25.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:26.052 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:26.058 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 76 16 97480 97464 80.1284 45.75 0.0116648 0.0124748 2023-10-09T06:17:26.151 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:26.502 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:26.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:26.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:26.507 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:26.608 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:26.676 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:26.786 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:17:26.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:26.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:26.821 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:26 smithi064 ceph-mon[220044]: pgmap v1517: 119 pgs: 119 active+clean; 5.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 42 MiB/s wr, 673 op/s 2023-10-09T06:17:26.901 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:26.934 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:27.023 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:17:27.058 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 77 16 98248 98232 79.7112 48 0.0103472 0.012537 2023-10-09T06:17:27.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:26 smithi156 ceph-mon[195101]: pgmap v1517: 119 pgs: 119 active+clean; 5.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 42 MiB/s wr, 673 op/s 2023-10-09T06:17:27.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:26 smithi064 ceph-mon[212750]: pgmap v1517: 119 pgs: 119 active+clean; 5.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 42 MiB/s wr, 673 op/s 2023-10-09T06:17:27.712 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:17:27.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:27.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:27.841 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:27.965 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:28.058 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 78 16 99081 99065 79.3567 52.0625 0.0294274 0.0125963 2023-10-09T06:17:28.143 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:17:28.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:28.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:28.248 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:28.360 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:28.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:28 smithi064 ceph-mon[220044]: pgmap v1518: 119 pgs: 119 active+clean; 5.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 53 MiB/s wr, 851 op/s 2023-10-09T06:17:28.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:28 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:17:28.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:28 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:17:28.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:28 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1821551084' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:17:28.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:28 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:17:28.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:28 smithi064 ceph-mon[212750]: pgmap v1518: 119 pgs: 119 active+clean; 5.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 53 MiB/s wr, 851 op/s 2023-10-09T06:17:28.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:28 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:17:28.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:28 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:17:28.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:28 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1821551084' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:17:28.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:28 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:17:28.867 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:17:28.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:28 smithi156 ceph-mon[195101]: pgmap v1518: 119 pgs: 119 active+clean; 5.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 53 MiB/s wr, 851 op/s 2023-10-09T06:17:28.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:28 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:17:28.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:28 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:17:28.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:28 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1821551084' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:17:28.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:28 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:17:29.059 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 79 16 99852 99836 78.9622 48.1875 0.00860854 0.0126599 2023-10-09T06:17:29.144 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:29.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:29.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:29.249 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:29.260 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:17:29.275 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T06:17:29.318 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:29.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:17:29.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:17:29.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:29 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:17:29.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:29 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:17:29.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:29.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:29 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:17:29.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:29 smithi064 ceph-mon[212750]: failed to encode map e592 with expected crc 2023-10-09T06:17:29.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:29 smithi064 ceph-mon[212750]: failed to encode map e592 with expected crc 2023-10-09T06:17:29.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:29 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:17:29.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:29 smithi064 ceph-mon[212750]: osdmap e592: 8 total, 8 up, 7 in 2023-10-09T06:17:29.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:17:29.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:17:29.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:29 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:17:29.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:29 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:17:29.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:29.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:29 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:17:29.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:29 smithi064 ceph-mon[220044]: failed to encode map e592 with expected crc 2023-10-09T06:17:29.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:29 smithi064 ceph-mon[220044]: failed to encode map e592 with expected crc 2023-10-09T06:17:29.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:29 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:17:29.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:29 smithi064 ceph-mon[220044]: osdmap e592: 8 total, 8 up, 7 in 2023-10-09T06:17:29.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:17:29.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:17:29.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:29 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:17:29.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:29 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:17:29.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:29.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:29 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:17:29.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:29 smithi156 ceph-mon[195101]: failed to encode map e592 with expected crc 2023-10-09T06:17:29.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:29 smithi156 ceph-mon[195101]: failed to encode map e592 with expected crc 2023-10-09T06:17:29.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:29 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:17:29.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:29 smithi156 ceph-mon[195101]: osdmap e592: 8 total, 8 up, 7 in 2023-10-09T06:17:30.059 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-10-09T06:17:30.059389+0000 min lat: 0.00113085 max lat: 0.307063 avg lat: 0.0127091 2023-10-09T06:17:30.059 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:17:30.060 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 80 16 100693 100677 78.6321 52.5625 0.0101973 0.0127091 2023-10-09T06:17:30.467 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:17:30.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:30.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:30.539 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:30.581 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:30.604 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1519,"stamp":"2023-10-09T06:17:28.840543+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6371940256,"num_objects":97243,"num_object_clones":0,"num_object_copies":291729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":97243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":97280,"num_write_kb":6222045,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":1718414,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96065,"ondisk_log_size":96065,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":24033768,"kb_used_data":18596912,"kb_used_omap":84,"kb_used_meta":5436651,"kb_avail":632182296,"statfs":{"total":671965249536,"available":647354671104,"internally_reserved":0,"allocated":19043237888,"data_stored":19027155961,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":86161,"internal_metadata":5567131503},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,4,5,1,2,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":553582592,"num_objects":8447,"num_object_clones":0,"num_object_copies":25341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8447,"num_write_kb":540608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.572691"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":854,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.367332+0000","last_change":"2023-10-09T06:15:19.880315+0000","last_active":"2023-10-09T06:17:17.367332+0000","last_peered":"2023-10-09T06:17:17.367332+0000","last_clean":"2023-10-09T06:17:17.367332+0000","last_became_active":"2023-10-09T06:15:19.879857+0000","last_became_peered":"2023-10-09T06:15:19.879857+0000","last_unstale":"2023-10-09T06:17:17.367332+0000","last_undegraded":"2023-10-09T06:17:17.367332+0000","last_fullsized":"2023-10-09T06:17:17.367332+0000","mapping_epoch":561,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":562,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:05.412941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:05.412941+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:54.322404+0000","scrub_duration":0.0061014329999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":669,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.359179+0000","last_change":"2023-10-09T06:16:02.748447+0000","last_active":"2023-10-09T06:17:17.359179+0000","last_peered":"2023-10-09T06:17:17.359179+0000","last_clean":"2023-10-09T06:17:17.359179+0000","last_became_active":"2023-10-09T06:16:02.747672+0000","last_became_peered":"2023-10-09T06:16:02.747672+0000","last_unstale":"2023-10-09T06:17:17.359179+0000","last_undegraded":"2023-10-09T06:17:17.359179+0000","last_fullsized":"2023-10-09T06:17:17.359179+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:31.326263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":403,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.394460+0000","last_change":"2023-10-09T06:05:11.451972+0000","last_active":"2023-10-09T06:17:20.394460+0000","last_peered":"2023-10-09T06:17:20.394460+0000","last_clean":"2023-10-09T06:17:20.394460+0000","last_became_active":"2023-10-09T06:05:11.451448+0000","last_became_peered":"2023-10-09T06:05:11.451448+0000","last_unstale":"2023-10-09T06:17:20.394460+0000","last_undegraded":"2023-10-09T06:17:20.394460+0000","last_fullsized":"2023-10-09T06:17:20.394460+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:39.183621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":890,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.359111+0000","last_change":"2023-10-09T06:09:27.554162+0000","last_active":"2023-10-09T06:17:17.359111+0000","last_peered":"2023-10-09T06:17:17.359111+0000","last_clean":"2023-10-09T06:17:17.359111+0000","last_became_active":"2023-10-09T06:09:27.553659+0000","last_became_peered":"2023-10-09T06:09:27.553659+0000","last_unstale":"2023-10-09T06:17:17.359111+0000","last_undegraded":"2023-10-09T06:17:17.359111+0000","last_fullsized":"2023-10-09T06:17:17.359111+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:01.468346+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:01.468346+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:03.443511+0000","scrub_duration":0.0016258220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":274,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.354959+0000","last_change":"2023-10-09T06:01:54.762060+0000","last_active":"2023-10-09T06:17:17.354959+0000","last_peered":"2023-10-09T06:17:17.354959+0000","last_clean":"2023-10-09T06:17:17.354959+0000","last_became_active":"2023-10-09T06:01:54.761757+0000","last_became_peered":"2023-10-09T06:01:54.761757+0000","last_unstale":"2023-10-09T06:17:17.354959+0000","last_undegraded":"2023-10-09T06:17:17.354959+0000","last_fullsized":"2023-10-09T06:17:17.354959+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:40.294232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":74,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.369750+0000","last_change":"2023-10-09T06:13:30.789834+0000","last_active":"2023-10-09T06:17:17.369750+0000","last_peered":"2023-10-09T06:17:17.369750+0000","last_clean":"2023-10-09T06:17:17.369750+0000","last_became_active":"2023-10-09T06:13:30.789517+0000","last_became_peered":"2023-10-09T06:13:30.789517+0000","last_unstale":"2023-10-09T06:17:17.369750+0000","last_undegraded":"2023-10-09T06:17:17.369750+0000","last_fullsized":"2023-10-09T06:17:17.369750+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:45.467922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":875,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.358863+0000","last_change":"2023-10-09T06:12:07.739121+0000","last_active":"2023-10-09T06:17:17.358863+0000","last_peered":"2023-10-09T06:17:17.358863+0000","last_clean":"2023-10-09T06:17:17.358863+0000","last_became_active":"2023-10-09T06:12:07.738739+0000","last_became_peered":"2023-10-09T06:12:07.738739+0000","last_unstale":"2023-10-09T06:17:17.358863+0000","last_undegraded":"2023-10-09T06:17:17.358863+0000","last_fullsized":"2023-10-09T06:17:17.358863+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.293426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.293426+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:19.815422+0000","scrub_duration":0.001587946,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":385,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.360970+0000","last_change":"2023-10-09T06:09:27.564377+0000","last_active":"2023-10-09T06:17:17.360970+0000","last_peered":"2023-10-09T06:17:17.360970+0000","last_clean":"2023-10-09T06:17:17.360970+0000","last_became_active":"2023-10-09T06:09:27.563710+0000","last_became_peered":"2023-10-09T06:09:27.563710+0000","last_unstale":"2023-10-09T06:17:17.360970+0000","last_undegraded":"2023-10-09T06:17:17.360970+0000","last_fullsized":"2023-10-09T06:17:17.360970+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:48.168447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":29,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.362641+0000","last_change":"2023-10-09T06:16:02.754245+0000","last_active":"2023-10-09T06:17:17.362641+0000","last_peered":"2023-10-09T06:17:17.362641+0000","last_clean":"2023-10-09T06:17:17.362641+0000","last_became_active":"2023-10-09T06:16:02.753886+0000","last_became_peered":"2023-10-09T06:16:02.753886+0000","last_unstale":"2023-10-09T06:17:17.362641+0000","last_undegraded":"2023-10-09T06:17:17.362641+0000","last_fullsized":"2023-10-09T06:17:17.362641+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:32.708190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":809,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.368057+0000","last_change":"2023-10-09T06:13:33.701687+0000","last_active":"2023-10-09T06:17:17.368057+0000","last_peered":"2023-10-09T06:17:17.368057+0000","last_clean":"2023-10-09T06:17:17.368057+0000","last_became_active":"2023-10-09T06:13:33.701233+0000","last_became_peered":"2023-10-09T06:13:33.701233+0000","last_unstale":"2023-10-09T06:17:17.368057+0000","last_undegraded":"2023-10-09T06:17:17.368057+0000","last_fullsized":"2023-10-09T06:17:17.368057+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.384407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.384407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:04.887826+0000","scrub_duration":0.0040651960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":490,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.361907+0000","last_change":"2023-10-09T06:09:27.559756+0000","last_active":"2023-10-09T06:17:17.361907+0000","last_peered":"2023-10-09T06:17:17.361907+0000","last_clean":"2023-10-09T06:17:17.361907+0000","last_became_active":"2023-10-09T06:09:27.559443+0000","last_became_peered":"2023-10-09T06:09:27.559443+0000","last_unstale":"2023-10-09T06:17:17.361907+0000","last_undegraded":"2023-10-09T06:17:17.361907+0000","last_fullsized":"2023-10-09T06:17:17.361907+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:14.519398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":421,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.361121+0000","last_change":"2023-10-09T06:13:21.737897+0000","last_active":"2023-10-09T06:17:17.361121+0000","last_peered":"2023-10-09T06:17:17.361121+0000","last_clean":"2023-10-09T06:17:17.361121+0000","last_became_active":"2023-10-09T06:13:21.737564+0000","last_became_peered":"2023-10-09T06:13:21.737564+0000","last_unstale":"2023-10-09T06:17:17.361121+0000","last_undegraded":"2023-10-09T06:17:17.361121+0000","last_fullsized":"2023-10-09T06:17:17.361121+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:34.863320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":735,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.375281+0000","last_change":"2023-10-09T06:16:05.761750+0000","last_active":"2023-10-09T06:17:17.375281+0000","last_peered":"2023-10-09T06:17:17.375281+0000","last_clean":"2023-10-09T06:17:17.375281+0000","last_became_active":"2023-10-09T06:16:05.761436+0000","last_became_peered":"2023-10-09T06:16:05.761436+0000","last_unstale":"2023-10-09T06:17:17.375281+0000","last_undegraded":"2023-10-09T06:17:17.375281+0000","last_fullsized":"2023-10-09T06:17:17.375281+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:20.742561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:20.742561+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:13.021950+0000","scrub_duration":0.0027005039999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":74,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.369714+0000","last_change":"2023-10-09T06:13:30.802749+0000","last_active":"2023-10-09T06:17:17.369714+0000","last_peered":"2023-10-09T06:17:17.369714+0000","last_clean":"2023-10-09T06:17:17.369714+0000","last_became_active":"2023-10-09T06:13:30.802449+0000","last_became_peered":"2023-10-09T06:13:30.802449+0000","last_unstale":"2023-10-09T06:17:17.369714+0000","last_undegraded":"2023-10-09T06:17:17.369714+0000","last_fullsized":"2023-10-09T06:17:17.369714+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:29.017181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":579,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.393021+0000","last_change":"2023-10-09T06:13:30.792445+0000","last_active":"2023-10-09T06:17:20.393021+0000","last_peered":"2023-10-09T06:17:20.393021+0000","last_clean":"2023-10-09T06:17:20.393021+0000","last_became_active":"2023-10-09T06:13:30.792116+0000","last_became_peered":"2023-10-09T06:13:30.792116+0000","last_unstale":"2023-10-09T06:17:20.393021+0000","last_undegraded":"2023-10-09T06:17:20.393021+0000","last_fullsized":"2023-10-09T06:17:20.393021+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:20:53.506293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":802,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.366688+0000","last_change":"2023-10-09T06:13:24.760843+0000","last_active":"2023-10-09T06:17:17.366688+0000","last_peered":"2023-10-09T06:17:17.366688+0000","last_clean":"2023-10-09T06:17:17.366688+0000","last_became_active":"2023-10-09T06:13:24.760333+0000","last_became_peered":"2023-10-09T06:13:24.760333+0000","last_unstale":"2023-10-09T06:17:17.366688+0000","last_undegraded":"2023-10-09T06:17:17.366688+0000","last_fullsized":"2023-10-09T06:17:17.366688+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:38.588641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:38.588641+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:30.660400+0000","scrub_duration":0.010116385,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":80,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.367961+0000","last_change":"2023-10-09T06:13:30.826588+0000","last_active":"2023-10-09T06:17:17.367961+0000","last_peered":"2023-10-09T06:17:17.367961+0000","last_clean":"2023-10-09T06:17:17.367961+0000","last_became_active":"2023-10-09T06:13:30.799199+0000","last_became_peered":"2023-10-09T06:13:30.799199+0000","last_unstale":"2023-10-09T06:17:17.367961+0000","last_undegraded":"2023-10-09T06:17:17.367961+0000","last_fullsized":"2023-10-09T06:17:17.367961+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:38.703864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":29,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.366736+0000","last_change":"2023-10-09T06:16:02.759286+0000","last_active":"2023-10-09T06:17:17.366736+0000","last_peered":"2023-10-09T06:17:17.366736+0000","last_clean":"2023-10-09T06:17:17.366736+0000","last_became_active":"2023-10-09T06:16:02.758956+0000","last_became_peered":"2023-10-09T06:16:02.758956+0000","last_unstale":"2023-10-09T06:17:17.366736+0000","last_undegraded":"2023-10-09T06:17:17.366736+0000","last_fullsized":"2023-10-09T06:17:17.366736+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:48.403200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":764,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.357580+0000","last_change":"2023-10-09T06:09:27.574336+0000","last_active":"2023-10-09T06:17:17.357580+0000","last_peered":"2023-10-09T06:17:17.357580+0000","last_clean":"2023-10-09T06:17:17.357580+0000","last_became_active":"2023-10-09T06:09:27.573974+0000","last_became_peered":"2023-10-09T06:09:27.573974+0000","last_unstale":"2023-10-09T06:17:17.357580+0000","last_undegraded":"2023-10-09T06:17:17.357580+0000","last_fullsized":"2023-10-09T06:17:17.357580+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.339121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.339121+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:45.908579+0000","scrub_duration":0.010474344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":625,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.359518+0000","last_change":"2023-10-09T06:13:30.812598+0000","last_active":"2023-10-09T06:17:17.359518+0000","last_peered":"2023-10-09T06:17:17.359518+0000","last_clean":"2023-10-09T06:17:17.359518+0000","last_became_active":"2023-10-09T06:13:30.791460+0000","last_became_peered":"2023-10-09T06:13:30.791460+0000","last_unstale":"2023-10-09T06:17:17.359518+0000","last_undegraded":"2023-10-09T06:17:17.359518+0000","last_fullsized":"2023-10-09T06:17:17.359518+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:28.046730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":603,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.359266+0000","last_change":"2023-10-09T06:01:54.764916+0000","last_active":"2023-10-09T06:17:17.359266+0000","last_peered":"2023-10-09T06:17:17.359266+0000","last_clean":"2023-10-09T06:17:17.359266+0000","last_became_active":"2023-10-09T06:01:54.764705+0000","last_became_peered":"2023-10-09T06:01:54.764705+0000","last_unstale":"2023-10-09T06:17:17.359266+0000","last_undegraded":"2023-10-09T06:17:17.359266+0000","last_fullsized":"2023-10-09T06:17:17.359266+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:12.527719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":820,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.367337+0000","last_change":"2023-10-09T06:13:15.425525+0000","last_active":"2023-10-09T06:17:17.367337+0000","last_peered":"2023-10-09T06:17:17.367337+0000","last_clean":"2023-10-09T06:17:17.367337+0000","last_became_active":"2023-10-09T06:13:15.425086+0000","last_became_peered":"2023-10-09T06:13:15.425086+0000","last_unstale":"2023-10-09T06:17:17.367337+0000","last_undegraded":"2023-10-09T06:17:17.367337+0000","last_fullsized":"2023-10-09T06:17:17.367337+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:42.900876+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:42.900876+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:46.042715+0000","scrub_duration":0.0014928649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":96,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.371293+0000","last_change":"2023-10-09T06:16:02.766996+0000","last_active":"2023-10-09T06:17:17.371293+0000","last_peered":"2023-10-09T06:17:17.371293+0000","last_clean":"2023-10-09T06:17:17.371293+0000","last_became_active":"2023-10-09T06:16:02.760831+0000","last_became_peered":"2023-10-09T06:16:02.760831+0000","last_unstale":"2023-10-09T06:17:17.371293+0000","last_undegraded":"2023-10-09T06:17:17.371293+0000","last_fullsized":"2023-10-09T06:17:17.371293+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:05.917714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":638,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.398015+0000","last_change":"2023-10-09T06:01:54.761558+0000","last_active":"2023-10-09T06:17:20.398015+0000","last_peered":"2023-10-09T06:17:20.398015+0000","last_clean":"2023-10-09T06:17:20.398015+0000","last_became_active":"2023-10-09T06:01:54.761251+0000","last_became_peered":"2023-10-09T06:01:54.761251+0000","last_unstale":"2023-10-09T06:17:20.398015+0000","last_undegraded":"2023-10-09T06:17:20.398015+0000","last_fullsized":"2023-10-09T06:17:20.398015+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:53.245267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":1094,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.367476+0000","last_change":"2023-10-09T06:13:24.754018+0000","last_active":"2023-10-09T06:17:17.367476+0000","last_peered":"2023-10-09T06:17:17.367476+0000","last_clean":"2023-10-09T06:17:17.367476+0000","last_became_active":"2023-10-09T06:13:24.753712+0000","last_became_peered":"2023-10-09T06:13:24.753712+0000","last_unstale":"2023-10-09T06:17:17.367476+0000","last_undegraded":"2023-10-09T06:17:17.367476+0000","last_fullsized":"2023-10-09T06:17:17.367476+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:18.490833+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":74,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.367716+0000","last_change":"2023-10-09T06:13:30.788160+0000","last_active":"2023-10-09T06:17:17.367716+0000","last_peered":"2023-10-09T06:17:17.367716+0000","last_clean":"2023-10-09T06:17:17.367716+0000","last_became_active":"2023-10-09T06:13:30.787724+0000","last_became_peered":"2023-10-09T06:13:30.787724+0000","last_unstale":"2023-10-09T06:17:17.367716+0000","last_undegraded":"2023-10-09T06:17:17.367716+0000","last_fullsized":"2023-10-09T06:17:17.367716+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:38.376770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":74,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.371584+0000","last_change":"2023-10-09T06:13:30.789225+0000","last_active":"2023-10-09T06:17:17.371584+0000","last_peered":"2023-10-09T06:17:17.371584+0000","last_clean":"2023-10-09T06:17:17.371584+0000","last_became_active":"2023-10-09T06:13:30.788869+0000","last_became_peered":"2023-10-09T06:13:30.788869+0000","last_unstale":"2023-10-09T06:17:17.371584+0000","last_undegraded":"2023-10-09T06:17:17.371584+0000","last_fullsized":"2023-10-09T06:17:17.371584+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:19.519966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":832,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.355375+0000","last_change":"2023-10-09T06:12:55.750923+0000","last_active":"2023-10-09T06:17:17.355375+0000","last_peered":"2023-10-09T06:17:17.355375+0000","last_clean":"2023-10-09T06:17:17.355375+0000","last_became_active":"2023-10-09T06:12:55.750521+0000","last_became_peered":"2023-10-09T06:12:55.750521+0000","last_unstale":"2023-10-09T06:17:17.355375+0000","last_undegraded":"2023-10-09T06:17:17.355375+0000","last_fullsized":"2023-10-09T06:17:17.355375+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.316697+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.316697+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:23.591439+0000","scrub_duration":0.001666561,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":444,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.367441+0000","last_change":"2023-10-09T06:01:22.337171+0000","last_active":"2023-10-09T06:17:17.367441+0000","last_peered":"2023-10-09T06:17:17.367441+0000","last_clean":"2023-10-09T06:17:17.367441+0000","last_became_active":"2023-10-09T06:01:22.335438+0000","last_became_peered":"2023-10-09T06:01:22.335438+0000","last_unstale":"2023-10-09T06:17:17.367441+0000","last_undegraded":"2023-10-09T06:17:17.367441+0000","last_fullsized":"2023-10-09T06:17:17.367441+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:25.036448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":554,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.361498+0000","last_change":"2023-10-09T06:13:30.788284+0000","last_active":"2023-10-09T06:17:17.361498+0000","last_peered":"2023-10-09T06:17:17.361498+0000","last_clean":"2023-10-09T06:17:17.361498+0000","last_became_active":"2023-10-09T06:13:30.787933+0000","last_became_peered":"2023-10-09T06:13:30.787933+0000","last_unstale":"2023-10-09T06:17:17.361498+0000","last_undegraded":"2023-10-09T06:17:17.361498+0000","last_fullsized":"2023-10-09T06:17:17.361498+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:10.735438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":882,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.355381+0000","last_change":"2023-10-09T06:13:24.753908+0000","last_active":"2023-10-09T06:17:17.355381+0000","last_peered":"2023-10-09T06:17:17.355381+0000","last_clean":"2023-10-09T06:17:17.355381+0000","last_became_active":"2023-10-09T06:13:24.753566+0000","last_became_peered":"2023-10-09T06:13:24.753566+0000","last_unstale":"2023-10-09T06:17:17.355381+0000","last_undegraded":"2023-10-09T06:17:17.355381+0000","last_fullsized":"2023-10-09T06:17:17.355381+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:40.388125+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:40.388125+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:35.101978+0000","scrub_duration":0.0055576990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":164,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.367006+0000","last_change":"2023-10-09T06:13:12.243605+0000","last_active":"2023-10-09T06:17:17.367006+0000","last_peered":"2023-10-09T06:17:17.367006+0000","last_clean":"2023-10-09T06:17:17.367006+0000","last_became_active":"2023-10-09T06:13:12.237061+0000","last_became_peered":"2023-10-09T06:13:12.237061+0000","last_unstale":"2023-10-09T06:17:17.367006+0000","last_undegraded":"2023-10-09T06:17:17.367006+0000","last_fullsized":"2023-10-09T06:17:17.367006+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:29:03.666031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":102,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.361763+0000","last_change":"2023-10-09T06:13:30.788733+0000","last_active":"2023-10-09T06:17:17.361763+0000","last_peered":"2023-10-09T06:17:17.361763+0000","last_clean":"2023-10-09T06:17:17.361763+0000","last_became_active":"2023-10-09T06:13:30.788353+0000","last_became_peered":"2023-10-09T06:13:30.788353+0000","last_unstale":"2023-10-09T06:17:17.361763+0000","last_undegraded":"2023-10-09T06:17:17.361763+0000","last_fullsized":"2023-10-09T06:17:17.361763+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:06.308426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"591'6136","reported_seq":6164,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:26.811715+0000","last_change":"2023-10-09T06:16:02.762978+0000","last_active":"2023-10-09T06:17:26.811715+0000","last_peered":"2023-10-09T06:17:26.811715+0000","last_clean":"2023-10-09T06:17:26.811715+0000","last_became_active":"2023-10-09T06:16:02.762657+0000","last_became_peered":"2023-10-09T06:16:02.762657+0000","last_unstale":"2023-10-09T06:17:26.811715+0000","last_undegraded":"2023-10-09T06:17:26.811715+0000","last_fullsized":"2023-10-09T06:17:26.811715+0000","mapping_epoch":572,"log_start":"578'900","ondisk_log_start":"578'900","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5236,"log_dups_size":0,"ondisk_log_size":5236,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:11:46.434393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402128896,"num_objects":6136,"num_object_clones":0,"num_object_copies":18408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6136,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6136,"num_write_kb":392704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":772,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.371634+0000","last_change":"2023-10-09T06:13:33.701521+0000","last_active":"2023-10-09T06:17:17.371634+0000","last_peered":"2023-10-09T06:17:17.371634+0000","last_clean":"2023-10-09T06:17:17.371634+0000","last_became_active":"2023-10-09T06:13:33.700870+0000","last_became_peered":"2023-10-09T06:13:33.700870+0000","last_unstale":"2023-10-09T06:17:17.371634+0000","last_undegraded":"2023-10-09T06:17:17.371634+0000","last_fullsized":"2023-10-09T06:17:17.371634+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:10:59.422551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:10:59.422551+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:54.402494+0000","scrub_duration":0.0059716659999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":78,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.373408+0000","last_change":"2023-10-09T06:13:30.822903+0000","last_active":"2023-10-09T06:17:17.373408+0000","last_peered":"2023-10-09T06:17:17.373408+0000","last_clean":"2023-10-09T06:17:17.373408+0000","last_became_active":"2023-10-09T06:13:30.799013+0000","last_became_peered":"2023-10-09T06:13:30.799013+0000","last_unstale":"2023-10-09T06:17:17.373408+0000","last_undegraded":"2023-10-09T06:17:17.373408+0000","last_fullsized":"2023-10-09T06:17:17.373408+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:44.271821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":74,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.367249+0000","last_change":"2023-10-09T06:13:30.793531+0000","last_active":"2023-10-09T06:17:17.367249+0000","last_peered":"2023-10-09T06:17:17.367249+0000","last_clean":"2023-10-09T06:17:17.367249+0000","last_became_active":"2023-10-09T06:13:30.792689+0000","last_became_peered":"2023-10-09T06:13:30.792689+0000","last_unstale":"2023-10-09T06:17:17.367249+0000","last_undegraded":"2023-10-09T06:17:17.367249+0000","last_fullsized":"2023-10-09T06:17:17.367249+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:02.559186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.6","version":"591'6082","reported_seq":6110,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:26.857241+0000","last_change":"2023-10-09T06:16:02.756362+0000","last_active":"2023-10-09T06:17:26.857241+0000","last_peered":"2023-10-09T06:17:26.857241+0000","last_clean":"2023-10-09T06:17:26.857241+0000","last_became_active":"2023-10-09T06:16:02.756033+0000","last_became_peered":"2023-10-09T06:16:02.756033+0000","last_unstale":"2023-10-09T06:17:26.857241+0000","last_undegraded":"2023-10-09T06:17:26.857241+0000","last_fullsized":"2023-10-09T06:17:26.857241+0000","mapping_epoch":572,"log_start":"578'900","ondisk_log_start":"578'900","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5182,"log_dups_size":0,"ondisk_log_size":5182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:07.492688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398589952,"num_objects":6082,"num_object_clones":0,"num_object_copies":18246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6082,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6082,"num_write_kb":389248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":961,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.363983+0000","last_change":"2023-10-09T06:13:24.753866+0000","last_active":"2023-10-09T06:17:17.363983+0000","last_peered":"2023-10-09T06:17:17.363983+0000","last_clean":"2023-10-09T06:17:17.363983+0000","last_became_active":"2023-10-09T06:13:24.753432+0000","last_became_peered":"2023-10-09T06:13:24.753432+0000","last_unstale":"2023-10-09T06:17:17.363983+0000","last_undegraded":"2023-10-09T06:17:17.363983+0000","last_fullsized":"2023-10-09T06:17:17.363983+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:03.241335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:03.241335+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:19.854449+0000","scrub_duration":0.017780547000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":78,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.369676+0000","last_change":"2023-10-09T06:13:30.829763+0000","last_active":"2023-10-09T06:17:17.369676+0000","last_peered":"2023-10-09T06:17:17.369676+0000","last_clean":"2023-10-09T06:17:17.369676+0000","last_became_active":"2023-10-09T06:13:30.802486+0000","last_became_peered":"2023-10-09T06:13:30.802486+0000","last_unstale":"2023-10-09T06:17:17.369676+0000","last_undegraded":"2023-10-09T06:17:17.369676+0000","last_fullsized":"2023-10-09T06:17:17.369676+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:05.468205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":629,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.359057+0000","last_change":"2023-10-09T06:01:54.760367+0000","last_active":"2023-10-09T06:17:17.359057+0000","last_peered":"2023-10-09T06:17:17.359057+0000","last_clean":"2023-10-09T06:17:17.359057+0000","last_became_active":"2023-10-09T06:01:54.760087+0000","last_became_peered":"2023-10-09T06:01:54.760087+0000","last_unstale":"2023-10-09T06:17:17.359057+0000","last_undegraded":"2023-10-09T06:17:17.359057+0000","last_fullsized":"2023-10-09T06:17:17.359057+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:26.000104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"591'6014","reported_seq":6041,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:24.377134+0000","last_change":"2023-10-09T06:16:02.761005+0000","last_active":"2023-10-09T06:17:24.377134+0000","last_peered":"2023-10-09T06:17:24.377134+0000","last_clean":"2023-10-09T06:17:24.377134+0000","last_became_active":"2023-10-09T06:16:02.760718+0000","last_became_peered":"2023-10-09T06:16:02.760718+0000","last_unstale":"2023-10-09T06:17:24.377134+0000","last_undegraded":"2023-10-09T06:17:24.377134+0000","last_fullsized":"2023-10-09T06:17:24.377134+0000","mapping_epoch":572,"log_start":"577'200","ondisk_log_start":"577'200","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5814,"log_dups_size":0,"ondisk_log_size":5814,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:32.418781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394133504,"num_objects":6014,"num_object_clones":0,"num_object_copies":18042,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6014,"num_write_kb":384896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":854,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.359665+0000","last_change":"2023-10-09T06:13:24.768958+0000","last_active":"2023-10-09T06:17:17.359665+0000","last_peered":"2023-10-09T06:17:17.359665+0000","last_clean":"2023-10-09T06:17:17.359665+0000","last_became_active":"2023-10-09T06:13:24.768643+0000","last_became_peered":"2023-10-09T06:13:24.768643+0000","last_unstale":"2023-10-09T06:17:17.359665+0000","last_undegraded":"2023-10-09T06:17:17.359665+0000","last_fullsized":"2023-10-09T06:17:17.359665+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:48.495739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:48.495739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:18:50.654354+0000","scrub_duration":0.0016345509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":78,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.371523+0000","last_change":"2023-10-09T06:13:30.814434+0000","last_active":"2023-10-09T06:17:17.371523+0000","last_peered":"2023-10-09T06:17:17.371523+0000","last_clean":"2023-10-09T06:17:17.371523+0000","last_became_active":"2023-10-09T06:13:30.793119+0000","last_became_peered":"2023-10-09T06:13:30.793119+0000","last_unstale":"2023-10-09T06:17:17.371523+0000","last_undegraded":"2023-10-09T06:17:17.371523+0000","last_fullsized":"2023-10-09T06:17:17.371523+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:34:31.724006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":617,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.354881+0000","last_change":"2023-10-09T06:13:30.792490+0000","last_active":"2023-10-09T06:17:17.354881+0000","last_peered":"2023-10-09T06:17:17.354881+0000","last_clean":"2023-10-09T06:17:17.354881+0000","last_became_active":"2023-10-09T06:13:30.792209+0000","last_became_peered":"2023-10-09T06:13:30.792209+0000","last_unstale":"2023-10-09T06:17:17.354881+0000","last_undegraded":"2023-10-09T06:17:17.354881+0000","last_fullsized":"2023-10-09T06:17:17.354881+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:14.208042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.4","version":"591'5962","reported_seq":5989,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:24.364568+0000","last_change":"2023-10-09T06:16:02.759679+0000","last_active":"2023-10-09T06:17:24.364568+0000","last_peered":"2023-10-09T06:17:24.364568+0000","last_clean":"2023-10-09T06:17:24.364568+0000","last_became_active":"2023-10-09T06:16:02.759382+0000","last_became_peered":"2023-10-09T06:16:02.759382+0000","last_unstale":"2023-10-09T06:17:24.364568+0000","last_undegraded":"2023-10-09T06:17:24.364568+0000","last_fullsized":"2023-10-09T06:17:24.364568+0000","mapping_epoch":572,"log_start":"577'100","ondisk_log_start":"577'100","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5862,"log_dups_size":0,"ondisk_log_size":5862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:04:34.950818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390725632,"num_objects":5962,"num_object_clones":0,"num_object_copies":17886,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5962,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5962,"num_write_kb":381568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":806,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.359817+0000","last_change":"2023-10-09T06:13:33.708291+0000","last_active":"2023-10-09T06:17:17.359817+0000","last_peered":"2023-10-09T06:17:17.359817+0000","last_clean":"2023-10-09T06:17:17.359817+0000","last_became_active":"2023-10-09T06:13:33.707930+0000","last_became_peered":"2023-10-09T06:13:33.707930+0000","last_unstale":"2023-10-09T06:17:17.359817+0000","last_undegraded":"2023-10-09T06:17:17.359817+0000","last_fullsized":"2023-10-09T06:17:17.359817+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:16.361097+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:16.361097+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:18.111404+0000","scrub_duration":0.028533749000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":361,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.362761+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T06:17:17.362761+0000","last_peered":"2023-10-09T06:17:17.362761+0000","last_clean":"2023-10-09T06:17:17.362761+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T06:17:17.362761+0000","last_undegraded":"2023-10-09T06:17:17.362761+0000","last_fullsized":"2023-10-09T06:17:17.362761+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":605,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.367236+0000","last_change":"2023-10-09T06:09:27.558353+0000","last_active":"2023-10-09T06:17:17.367236+0000","last_peered":"2023-10-09T06:17:17.367236+0000","last_clean":"2023-10-09T06:17:17.367236+0000","last_became_active":"2023-10-09T06:09:27.557233+0000","last_became_peered":"2023-10-09T06:09:27.557233+0000","last_unstale":"2023-10-09T06:17:17.367236+0000","last_undegraded":"2023-10-09T06:17:17.367236+0000","last_fullsized":"2023-10-09T06:17:17.367236+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:53.149350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"591'6144","reported_seq":6171,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:24.381479+0000","last_change":"2023-10-09T06:16:02.761692+0000","last_active":"2023-10-09T06:17:24.381479+0000","last_peered":"2023-10-09T06:17:24.381479+0000","last_clean":"2023-10-09T06:17:24.381479+0000","last_became_active":"2023-10-09T06:16:02.761411+0000","last_became_peered":"2023-10-09T06:16:02.761411+0000","last_unstale":"2023-10-09T06:17:24.381479+0000","last_undegraded":"2023-10-09T06:17:24.381479+0000","last_fullsized":"2023-10-09T06:17:24.381479+0000","mapping_epoch":572,"log_start":"577'300","ondisk_log_start":"577'300","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5844,"log_dups_size":0,"ondisk_log_size":5844,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:55.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402653184,"num_objects":6144,"num_object_clones":0,"num_object_copies":18432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6144,"num_write_kb":393216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":997,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.367500+0000","last_change":"2023-10-09T06:15:19.874375+0000","last_active":"2023-10-09T06:17:17.367500+0000","last_peered":"2023-10-09T06:17:17.367500+0000","last_clean":"2023-10-09T06:17:17.367500+0000","last_became_active":"2023-10-09T06:15:19.873993+0000","last_became_peered":"2023-10-09T06:15:19.873993+0000","last_unstale":"2023-10-09T06:17:17.367500+0000","last_undegraded":"2023-10-09T06:17:17.367500+0000","last_fullsized":"2023-10-09T06:17:17.367500+0000","mapping_epoch":561,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":562,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:42.795781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:42.795781+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:43.523891+0000","scrub_duration":0.0091262979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":395,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.361408+0000","last_change":"2023-10-09T06:09:27.565666+0000","last_active":"2023-10-09T06:17:17.361408+0000","last_peered":"2023-10-09T06:17:17.361408+0000","last_clean":"2023-10-09T06:17:17.361408+0000","last_became_active":"2023-10-09T06:09:27.565324+0000","last_became_peered":"2023-10-09T06:09:27.565324+0000","last_unstale":"2023-10-09T06:17:17.361408+0000","last_undegraded":"2023-10-09T06:17:17.361408+0000","last_fullsized":"2023-10-09T06:17:17.361408+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:51.216947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":565,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.359597+0000","last_change":"2023-10-09T06:13:21.727745+0000","last_active":"2023-10-09T06:17:17.359597+0000","last_peered":"2023-10-09T06:17:17.359597+0000","last_clean":"2023-10-09T06:17:17.359597+0000","last_became_active":"2023-10-09T06:13:21.727404+0000","last_became_peered":"2023-10-09T06:13:21.727404+0000","last_unstale":"2023-10-09T06:17:17.359597+0000","last_undegraded":"2023-10-09T06:17:17.359597+0000","last_fullsized":"2023-10-09T06:17:17.359597+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:24.931626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"591'6018","reported_seq":6046,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:25.829252+0000","last_change":"2023-10-09T06:16:02.747886+0000","last_active":"2023-10-09T06:17:25.829252+0000","last_peered":"2023-10-09T06:17:25.829252+0000","last_clean":"2023-10-09T06:17:25.829252+0000","last_became_active":"2023-10-09T06:16:02.747569+0000","last_became_peered":"2023-10-09T06:16:02.747569+0000","last_unstale":"2023-10-09T06:17:25.829252+0000","last_undegraded":"2023-10-09T06:17:25.829252+0000","last_fullsized":"2023-10-09T06:17:25.829252+0000","mapping_epoch":572,"log_start":"577'200","ondisk_log_start":"577'200","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5818,"log_dups_size":0,"ondisk_log_size":5818,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:18.720375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394395648,"num_objects":6018,"num_object_clones":0,"num_object_copies":18054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6018,"num_write_kb":385152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":872,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.360728+0000","last_change":"2023-10-09T06:16:05.756855+0000","last_active":"2023-10-09T06:17:17.360728+0000","last_peered":"2023-10-09T06:17:17.360728+0000","last_clean":"2023-10-09T06:17:17.360728+0000","last_became_active":"2023-10-09T06:16:05.756518+0000","last_became_peered":"2023-10-09T06:16:05.756518+0000","last_unstale":"2023-10-09T06:17:17.360728+0000","last_undegraded":"2023-10-09T06:17:17.360728+0000","last_fullsized":"2023-10-09T06:17:17.360728+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.282393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.282393+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:24:20.507377+0000","scrub_duration":0.01402403,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":510,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.359005+0000","last_change":"2023-10-09T06:13:21.738218+0000","last_active":"2023-10-09T06:17:17.359005+0000","last_peered":"2023-10-09T06:17:17.359005+0000","last_clean":"2023-10-09T06:17:17.359005+0000","last_became_active":"2023-10-09T06:13:21.737969+0000","last_became_peered":"2023-10-09T06:13:21.737969+0000","last_unstale":"2023-10-09T06:17:17.359005+0000","last_undegraded":"2023-10-09T06:17:17.359005+0000","last_fullsized":"2023-10-09T06:17:17.359005+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:38.620281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":649,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.362245+0000","last_change":"2023-10-09T06:15:16.810530+0000","last_active":"2023-10-09T06:17:17.362245+0000","last_peered":"2023-10-09T06:17:17.362245+0000","last_clean":"2023-10-09T06:17:17.362245+0000","last_became_active":"2023-10-09T06:15:16.810238+0000","last_became_peered":"2023-10-09T06:15:16.810238+0000","last_unstale":"2023-10-09T06:17:17.362245+0000","last_undegraded":"2023-10-09T06:17:17.362245+0000","last_fullsized":"2023-10-09T06:17:17.362245+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:03.744638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"591'5949","reported_seq":5977,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:25.836400+0000","last_change":"2023-10-09T06:16:02.753013+0000","last_active":"2023-10-09T06:17:25.836400+0000","last_peered":"2023-10-09T06:17:25.836400+0000","last_clean":"2023-10-09T06:17:25.836400+0000","last_became_active":"2023-10-09T06:16:02.752442+0000","last_became_peered":"2023-10-09T06:16:02.752442+0000","last_unstale":"2023-10-09T06:17:25.836400+0000","last_undegraded":"2023-10-09T06:17:25.836400+0000","last_fullsized":"2023-10-09T06:17:25.836400+0000","mapping_epoch":572,"log_start":"577'100","ondisk_log_start":"577'100","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5849,"log_dups_size":0,"ondisk_log_size":5849,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:16.543432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389873664,"num_objects":5949,"num_object_clones":0,"num_object_copies":17847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5949,"num_write_kb":380736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":926,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.355181+0000","last_change":"2023-10-09T06:13:15.428701+0000","last_active":"2023-10-09T06:17:17.355181+0000","last_peered":"2023-10-09T06:17:17.355181+0000","last_clean":"2023-10-09T06:17:17.355181+0000","last_became_active":"2023-10-09T06:13:15.428343+0000","last_became_peered":"2023-10-09T06:13:15.428343+0000","last_unstale":"2023-10-09T06:17:17.355181+0000","last_undegraded":"2023-10-09T06:17:17.355181+0000","last_fullsized":"2023-10-09T06:17:17.355181+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.373748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.373748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:55:34.235074+0000","scrub_duration":0.0024576450000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":591,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.359192+0000","last_change":"2023-10-09T06:16:02.744252+0000","last_active":"2023-10-09T06:17:17.359192+0000","last_peered":"2023-10-09T06:17:17.359192+0000","last_clean":"2023-10-09T06:17:17.359192+0000","last_became_active":"2023-10-09T06:16:02.743947+0000","last_became_peered":"2023-10-09T06:16:02.743947+0000","last_unstale":"2023-10-09T06:17:17.359192+0000","last_undegraded":"2023-10-09T06:17:17.359192+0000","last_fullsized":"2023-10-09T06:17:17.359192+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:52.293310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":622,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.359888+0000","last_change":"2023-10-09T06:13:21.727814+0000","last_active":"2023-10-09T06:17:17.359888+0000","last_peered":"2023-10-09T06:17:17.359888+0000","last_clean":"2023-10-09T06:17:17.359888+0000","last_became_active":"2023-10-09T06:13:21.727401+0000","last_became_peered":"2023-10-09T06:13:21.727401+0000","last_unstale":"2023-10-09T06:17:17.359888+0000","last_undegraded":"2023-10-09T06:17:17.359888+0000","last_fullsized":"2023-10-09T06:17:17.359888+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:53:37.968028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"591'6290","reported_seq":6317,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:26.883630+0000","last_change":"2023-10-09T06:16:02.744178+0000","last_active":"2023-10-09T06:17:26.883630+0000","last_peered":"2023-10-09T06:17:26.883630+0000","last_clean":"2023-10-09T06:17:26.883630+0000","last_became_active":"2023-10-09T06:16:02.743835+0000","last_became_peered":"2023-10-09T06:16:02.743835+0000","last_unstale":"2023-10-09T06:17:26.883630+0000","last_undegraded":"2023-10-09T06:17:26.883630+0000","last_fullsized":"2023-10-09T06:17:26.883630+0000","mapping_epoch":572,"log_start":"579'1100","ondisk_log_start":"579'1100","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5190,"log_dups_size":0,"ondisk_log_size":5190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:02.861316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":412221440,"num_objects":6290,"num_object_clones":0,"num_object_copies":18870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6290,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6290,"num_write_kb":402560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":650,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.359318+0000","last_change":"2023-10-09T06:16:05.760797+0000","last_active":"2023-10-09T06:17:17.359318+0000","last_peered":"2023-10-09T06:17:17.359318+0000","last_clean":"2023-10-09T06:17:17.359318+0000","last_became_active":"2023-10-09T06:16:05.760421+0000","last_became_peered":"2023-10-09T06:16:05.760421+0000","last_unstale":"2023-10-09T06:17:17.359318+0000","last_undegraded":"2023-10-09T06:17:17.359318+0000","last_fullsized":"2023-10-09T06:17:17.359318+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:39.633410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:39.633410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:02.235216+0000","scrub_duration":0.0039938459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":404,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.360879+0000","last_change":"2023-10-09T06:13:30.781849+0000","last_active":"2023-10-09T06:17:17.360879+0000","last_peered":"2023-10-09T06:17:17.360879+0000","last_clean":"2023-10-09T06:17:17.360879+0000","last_became_active":"2023-10-09T06:13:30.781084+0000","last_became_peered":"2023-10-09T06:13:30.781084+0000","last_unstale":"2023-10-09T06:17:17.360879+0000","last_undegraded":"2023-10-09T06:17:17.360879+0000","last_fullsized":"2023-10-09T06:17:17.360879+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:02.199298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":274,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.362810+0000","last_change":"2023-10-09T06:01:54.761768+0000","last_active":"2023-10-09T06:17:17.362810+0000","last_peered":"2023-10-09T06:17:17.362810+0000","last_clean":"2023-10-09T06:17:17.362810+0000","last_became_active":"2023-10-09T06:01:54.761443+0000","last_became_peered":"2023-10-09T06:01:54.761443+0000","last_unstale":"2023-10-09T06:17:17.362810+0000","last_undegraded":"2023-10-09T06:17:17.362810+0000","last_fullsized":"2023-10-09T06:17:17.362810+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:51.555987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"591'6038","reported_seq":6065,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:26.009852+0000","last_change":"2023-10-09T06:16:02.756342+0000","last_active":"2023-10-09T06:17:26.009852+0000","last_peered":"2023-10-09T06:17:26.009852+0000","last_clean":"2023-10-09T06:17:26.009852+0000","last_became_active":"2023-10-09T06:16:02.756000+0000","last_became_peered":"2023-10-09T06:16:02.756000+0000","last_unstale":"2023-10-09T06:17:26.009852+0000","last_undegraded":"2023-10-09T06:17:26.009852+0000","last_fullsized":"2023-10-09T06:17:26.009852+0000","mapping_epoch":572,"log_start":"577'100","ondisk_log_start":"577'100","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5938,"log_dups_size":0,"ondisk_log_size":5938,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:55.773353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":395706368,"num_objects":6038,"num_object_clones":0,"num_object_copies":18114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6038,"num_write_kb":386432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":795,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.361835+0000","last_change":"2023-10-09T06:13:33.707885+0000","last_active":"2023-10-09T06:17:17.361835+0000","last_peered":"2023-10-09T06:17:17.361835+0000","last_clean":"2023-10-09T06:17:17.361835+0000","last_became_active":"2023-10-09T06:13:33.707474+0000","last_became_peered":"2023-10-09T06:13:33.707474+0000","last_unstale":"2023-10-09T06:17:17.361835+0000","last_undegraded":"2023-10-09T06:17:17.361835+0000","last_fullsized":"2023-10-09T06:17:17.361835+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:12:22.289803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:12:22.289803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:27.842817+0000","scrub_duration":0.0076565050000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":337,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.357258+0000","last_change":"2023-10-09T06:12:04.680247+0000","last_active":"2023-10-09T06:17:17.357258+0000","last_peered":"2023-10-09T06:17:17.357258+0000","last_clean":"2023-10-09T06:17:17.357258+0000","last_became_active":"2023-10-09T06:12:04.679943+0000","last_became_peered":"2023-10-09T06:12:04.679943+0000","last_unstale":"2023-10-09T06:17:17.357258+0000","last_undegraded":"2023-10-09T06:17:17.357258+0000","last_fullsized":"2023-10-09T06:17:17.357258+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:56.692774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":389,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.360960+0000","last_change":"2023-10-09T06:16:02.759206+0000","last_active":"2023-10-09T06:17:17.360960+0000","last_peered":"2023-10-09T06:17:17.360960+0000","last_clean":"2023-10-09T06:17:17.360960+0000","last_became_active":"2023-10-09T06:16:02.758888+0000","last_became_peered":"2023-10-09T06:16:02.758888+0000","last_unstale":"2023-10-09T06:17:17.360960+0000","last_undegraded":"2023-10-09T06:17:17.360960+0000","last_fullsized":"2023-10-09T06:17:17.360960+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:33.728805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":308,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.355065+0000","last_change":"2023-10-09T06:13:12.234849+0000","last_active":"2023-10-09T06:17:17.355065+0000","last_peered":"2023-10-09T06:17:17.355065+0000","last_clean":"2023-10-09T06:17:17.355065+0000","last_became_active":"2023-10-09T06:13:12.234284+0000","last_became_peered":"2023-10-09T06:13:12.234284+0000","last_unstale":"2023-10-09T06:17:17.355065+0000","last_undegraded":"2023-10-09T06:17:17.355065+0000","last_fullsized":"2023-10-09T06:17:17.355065+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:40:43.809773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"591'5949","reported_seq":5977,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:25.832196+0000","last_change":"2023-10-09T06:16:02.749724+0000","last_active":"2023-10-09T06:17:25.832196+0000","last_peered":"2023-10-09T06:17:25.832196+0000","last_clean":"2023-10-09T06:17:25.832196+0000","last_became_active":"2023-10-09T06:16:02.749368+0000","last_became_peered":"2023-10-09T06:16:02.749368+0000","last_unstale":"2023-10-09T06:17:25.832196+0000","last_undegraded":"2023-10-09T06:17:25.832196+0000","last_fullsized":"2023-10-09T06:17:25.832196+0000","mapping_epoch":572,"log_start":"577'100","ondisk_log_start":"577'100","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5849,"log_dups_size":0,"ondisk_log_size":5849,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:24.189151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389873664,"num_objects":5949,"num_object_clones":0,"num_object_copies":17847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5949,"num_write_kb":380736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":844,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.368489+0000","last_change":"2023-10-09T06:16:05.759429+0000","last_active":"2023-10-09T06:17:17.368489+0000","last_peered":"2023-10-09T06:17:17.368489+0000","last_clean":"2023-10-09T06:17:17.368489+0000","last_became_active":"2023-10-09T06:16:05.759155+0000","last_became_peered":"2023-10-09T06:16:05.759155+0000","last_unstale":"2023-10-09T06:17:17.368489+0000","last_undegraded":"2023-10-09T06:17:17.368489+0000","last_fullsized":"2023-10-09T06:17:17.368489+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:37.915576+0000","scrub_duration":0.0016660799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":293,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.357184+0000","last_change":"2023-10-09T06:13:21.738040+0000","last_active":"2023-10-09T06:17:17.357184+0000","last_peered":"2023-10-09T06:17:17.357184+0000","last_clean":"2023-10-09T06:17:17.357184+0000","last_became_active":"2023-10-09T06:13:21.737717+0000","last_became_peered":"2023-10-09T06:13:21.737717+0000","last_unstale":"2023-10-09T06:17:17.357184+0000","last_undegraded":"2023-10-09T06:17:17.357184+0000","last_fullsized":"2023-10-09T06:17:17.357184+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:39.451628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":571,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.358926+0000","last_change":"2023-10-09T06:09:27.553958+0000","last_active":"2023-10-09T06:17:17.358926+0000","last_peered":"2023-10-09T06:17:17.358926+0000","last_clean":"2023-10-09T06:17:17.358926+0000","last_became_active":"2023-10-09T06:09:27.553193+0000","last_became_peered":"2023-10-09T06:09:27.553193+0000","last_unstale":"2023-10-09T06:17:17.358926+0000","last_undegraded":"2023-10-09T06:17:17.358926+0000","last_fullsized":"2023-10-09T06:17:17.358926+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:09.754579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":427,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.357338+0000","last_change":"2023-10-09T06:16:02.753637+0000","last_active":"2023-10-09T06:17:17.357338+0000","last_peered":"2023-10-09T06:17:17.357338+0000","last_clean":"2023-10-09T06:17:17.357338+0000","last_became_active":"2023-10-09T06:16:02.753254+0000","last_became_peered":"2023-10-09T06:16:02.753254+0000","last_unstale":"2023-10-09T06:17:17.357338+0000","last_undegraded":"2023-10-09T06:17:17.357338+0000","last_fullsized":"2023-10-09T06:17:17.357338+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:18.543079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"591'6120","reported_seq":6148,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:25.800887+0000","last_change":"2023-10-09T06:16:02.759668+0000","last_active":"2023-10-09T06:17:25.800887+0000","last_peered":"2023-10-09T06:17:25.800887+0000","last_clean":"2023-10-09T06:17:25.800887+0000","last_became_active":"2023-10-09T06:16:02.758840+0000","last_became_peered":"2023-10-09T06:16:02.758840+0000","last_unstale":"2023-10-09T06:17:25.800887+0000","last_undegraded":"2023-10-09T06:17:25.800887+0000","last_fullsized":"2023-10-09T06:17:25.800887+0000","mapping_epoch":572,"log_start":"577'300","ondisk_log_start":"577'300","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5820,"log_dups_size":0,"ondisk_log_size":5820,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:28.523305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401080320,"num_objects":6120,"num_object_clones":0,"num_object_copies":18360,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6120,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6120,"num_write_kb":391680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":646,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.367809+0000","last_change":"2023-10-09T06:14:47.339271+0000","last_active":"2023-10-09T06:17:17.367809+0000","last_peered":"2023-10-09T06:17:17.367809+0000","last_clean":"2023-10-09T06:17:17.367809+0000","last_became_active":"2023-10-09T06:13:33.705351+0000","last_became_peered":"2023-10-09T06:13:33.705351+0000","last_unstale":"2023-10-09T06:17:17.367809+0000","last_undegraded":"2023-10-09T06:17:17.367809+0000","last_fullsized":"2023-10-09T06:17:17.367809+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:14:47.339163+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T06:14:47.339163+0000","last_clean_scrub_stamp":"2023-10-09T06:14:47.339163+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:34.887758+0000","scrub_duration":0.0016607729999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":381,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.367357+0000","last_change":"2023-10-09T06:15:16.802788+0000","last_active":"2023-10-09T06:17:17.367357+0000","last_peered":"2023-10-09T06:17:17.367357+0000","last_clean":"2023-10-09T06:17:17.367357+0000","last_became_active":"2023-10-09T06:15:16.802449+0000","last_became_peered":"2023-10-09T06:15:16.802449+0000","last_unstale":"2023-10-09T06:17:17.367357+0000","last_undegraded":"2023-10-09T06:17:17.367357+0000","last_fullsized":"2023-10-09T06:17:17.367357+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:55:30.488174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":371,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.355211+0000","last_change":"2023-10-09T06:16:02.752290+0000","last_active":"2023-10-09T06:17:17.355211+0000","last_peered":"2023-10-09T06:17:17.355211+0000","last_clean":"2023-10-09T06:17:17.355211+0000","last_became_active":"2023-10-09T06:16:02.751942+0000","last_became_peered":"2023-10-09T06:16:02.751942+0000","last_unstale":"2023-10-09T06:17:17.355211+0000","last_undegraded":"2023-10-09T06:17:17.355211+0000","last_fullsized":"2023-10-09T06:17:17.355211+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:45:56.100957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":290,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.362727+0000","last_change":"2023-10-09T06:09:27.547953+0000","last_active":"2023-10-09T06:17:17.362727+0000","last_peered":"2023-10-09T06:17:17.362727+0000","last_clean":"2023-10-09T06:17:17.362727+0000","last_became_active":"2023-10-09T06:09:27.547624+0000","last_became_peered":"2023-10-09T06:09:27.547624+0000","last_unstale":"2023-10-09T06:17:17.362727+0000","last_undegraded":"2023-10-09T06:17:17.362727+0000","last_fullsized":"2023-10-09T06:17:17.362727+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:39.630486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"591'6028","reported_seq":6055,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:25.844732+0000","last_change":"2023-10-09T06:16:02.759576+0000","last_active":"2023-10-09T06:17:25.844732+0000","last_peered":"2023-10-09T06:17:25.844732+0000","last_clean":"2023-10-09T06:17:25.844732+0000","last_became_active":"2023-10-09T06:16:02.759323+0000","last_became_peered":"2023-10-09T06:16:02.759323+0000","last_unstale":"2023-10-09T06:17:25.844732+0000","last_undegraded":"2023-10-09T06:17:25.844732+0000","last_fullsized":"2023-10-09T06:17:25.844732+0000","mapping_epoch":572,"log_start":"577'200","ondisk_log_start":"577'200","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5828,"log_dups_size":0,"ondisk_log_size":5828,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:55.127489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":395051008,"num_objects":6028,"num_object_clones":0,"num_object_copies":18084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6028,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6028,"num_write_kb":385792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":84,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.371376+0000","last_change":"2023-10-09T06:16:02.761560+0000","last_active":"2023-10-09T06:17:17.371376+0000","last_peered":"2023-10-09T06:17:17.371376+0000","last_clean":"2023-10-09T06:17:17.371376+0000","last_became_active":"2023-10-09T06:16:02.761061+0000","last_became_peered":"2023-10-09T06:16:02.761061+0000","last_unstale":"2023-10-09T06:17:17.371376+0000","last_undegraded":"2023-10-09T06:17:17.371376+0000","last_fullsized":"2023-10-09T06:17:17.371376+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:33.297395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":634,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.367174+0000","last_change":"2023-10-09T06:13:33.704835+0000","last_active":"2023-10-09T06:17:17.367174+0000","last_peered":"2023-10-09T06:17:17.367174+0000","last_clean":"2023-10-09T06:17:17.367174+0000","last_became_active":"2023-10-09T06:13:33.704466+0000","last_became_peered":"2023-10-09T06:13:33.704466+0000","last_unstale":"2023-10-09T06:17:17.367174+0000","last_undegraded":"2023-10-09T06:17:17.367174+0000","last_fullsized":"2023-10-09T06:17:17.367174+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:35.783434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:35.783434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:38.721974+0000","scrub_duration":0.0043794799999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":380,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.364635+0000","last_change":"2023-10-09T06:13:12.238280+0000","last_active":"2023-10-09T06:17:17.364635+0000","last_peered":"2023-10-09T06:17:17.364635+0000","last_clean":"2023-10-09T06:17:17.364635+0000","last_became_active":"2023-10-09T06:13:12.237986+0000","last_became_peered":"2023-10-09T06:13:12.237986+0000","last_unstale":"2023-10-09T06:17:17.364635+0000","last_undegraded":"2023-10-09T06:17:17.364635+0000","last_fullsized":"2023-10-09T06:17:17.364635+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:58.520176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":521,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.362171+0000","last_change":"2023-10-09T06:13:21.733940+0000","last_active":"2023-10-09T06:17:17.362171+0000","last_peered":"2023-10-09T06:17:17.362171+0000","last_clean":"2023-10-09T06:17:17.362171+0000","last_became_active":"2023-10-09T06:13:21.733628+0000","last_became_peered":"2023-10-09T06:13:21.733628+0000","last_unstale":"2023-10-09T06:17:17.362171+0000","last_undegraded":"2023-10-09T06:17:17.362171+0000","last_fullsized":"2023-10-09T06:17:17.362171+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:13.348443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"591'6073","reported_seq":6101,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:25.812426+0000","last_change":"2023-10-09T06:16:02.752768+0000","last_active":"2023-10-09T06:17:25.812426+0000","last_peered":"2023-10-09T06:17:25.812426+0000","last_clean":"2023-10-09T06:17:25.812426+0000","last_became_active":"2023-10-09T06:16:02.752026+0000","last_became_peered":"2023-10-09T06:16:02.752026+0000","last_unstale":"2023-10-09T06:17:25.812426+0000","last_undegraded":"2023-10-09T06:17:25.812426+0000","last_fullsized":"2023-10-09T06:17:25.812426+0000","mapping_epoch":572,"log_start":"577'300","ondisk_log_start":"577'300","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5773,"log_dups_size":0,"ondisk_log_size":5773,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:30.737816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398000128,"num_objects":6073,"num_object_clones":0,"num_object_copies":18219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6073,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6073,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":861,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.355281+0000","last_change":"2023-10-09T06:13:24.753842+0000","last_active":"2023-10-09T06:17:17.355281+0000","last_peered":"2023-10-09T06:17:17.355281+0000","last_clean":"2023-10-09T06:17:17.355281+0000","last_became_active":"2023-10-09T06:13:24.753485+0000","last_became_peered":"2023-10-09T06:13:24.753485+0000","last_unstale":"2023-10-09T06:17:17.355281+0000","last_undegraded":"2023-10-09T06:17:17.355281+0000","last_fullsized":"2023-10-09T06:17:17.355281+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:57.627296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:57.627296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:56.656218+0000","scrub_duration":0.001913166,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":484,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.367402+0000","last_change":"2023-10-09T06:09:27.544373+0000","last_active":"2023-10-09T06:17:17.367402+0000","last_peered":"2023-10-09T06:17:17.367402+0000","last_clean":"2023-10-09T06:17:17.367402+0000","last_became_active":"2023-10-09T06:09:27.544121+0000","last_became_peered":"2023-10-09T06:09:27.544121+0000","last_unstale":"2023-10-09T06:17:17.367402+0000","last_undegraded":"2023-10-09T06:17:17.367402+0000","last_fullsized":"2023-10-09T06:17:17.367402+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:40.429491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":529,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.368366+0000","last_change":"2023-10-09T06:12:52.670573+0000","last_active":"2023-10-09T06:17:17.368366+0000","last_peered":"2023-10-09T06:17:17.368366+0000","last_clean":"2023-10-09T06:17:17.368366+0000","last_became_active":"2023-10-09T06:12:52.670242+0000","last_became_peered":"2023-10-09T06:12:52.670242+0000","last_unstale":"2023-10-09T06:17:17.368366+0000","last_undegraded":"2023-10-09T06:17:17.368366+0000","last_fullsized":"2023-10-09T06:17:17.368366+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:25.237580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":677,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.366327+0000","last_change":"2023-10-09T06:15:16.805514+0000","last_active":"2023-10-09T06:17:17.366327+0000","last_peered":"2023-10-09T06:17:17.366327+0000","last_clean":"2023-10-09T06:17:17.366327+0000","last_became_active":"2023-10-09T06:15:16.805160+0000","last_became_peered":"2023-10-09T06:15:16.805160+0000","last_unstale":"2023-10-09T06:17:17.366327+0000","last_undegraded":"2023-10-09T06:17:17.366327+0000","last_fullsized":"2023-10-09T06:17:17.366327+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:30.153699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"591'6157","reported_seq":6185,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:26.882139+0000","last_change":"2023-10-09T06:16:02.755231+0000","last_active":"2023-10-09T06:17:26.882139+0000","last_peered":"2023-10-09T06:17:26.882139+0000","last_clean":"2023-10-09T06:17:26.882139+0000","last_became_active":"2023-10-09T06:16:02.754877+0000","last_became_peered":"2023-10-09T06:16:02.754877+0000","last_unstale":"2023-10-09T06:17:26.882139+0000","last_undegraded":"2023-10-09T06:17:26.882139+0000","last_fullsized":"2023-10-09T06:17:26.882139+0000","mapping_epoch":572,"log_start":"578'900","ondisk_log_start":"578'900","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5257,"log_dups_size":0,"ondisk_log_size":5257,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:16:25.397496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403505152,"num_objects":6157,"num_object_clones":0,"num_object_copies":18471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6157,"num_write_kb":394048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":792,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.394572+0000","last_change":"2023-10-09T06:13:24.756288+0000","last_active":"2023-10-09T06:17:20.394572+0000","last_peered":"2023-10-09T06:17:20.394572+0000","last_clean":"2023-10-09T06:17:20.394572+0000","last_became_active":"2023-10-09T06:13:24.755945+0000","last_became_peered":"2023-10-09T06:13:24.755945+0000","last_unstale":"2023-10-09T06:17:20.394572+0000","last_undegraded":"2023-10-09T06:17:20.394572+0000","last_fullsized":"2023-10-09T06:17:20.394572+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:45.844892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:45.844892+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:44.107376+0000","scrub_duration":0.0017521489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":74,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.367854+0000","last_change":"2023-10-09T06:13:30.799535+0000","last_active":"2023-10-09T06:17:17.367854+0000","last_peered":"2023-10-09T06:17:17.367854+0000","last_clean":"2023-10-09T06:17:17.367854+0000","last_became_active":"2023-10-09T06:13:30.799227+0000","last_became_peered":"2023-10-09T06:13:30.799227+0000","last_unstale":"2023-10-09T06:17:17.367854+0000","last_undegraded":"2023-10-09T06:17:17.367854+0000","last_fullsized":"2023-10-09T06:17:17.367854+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:14.741742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":320,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.357469+0000","last_change":"2023-10-09T06:15:16.810272+0000","last_active":"2023-10-09T06:17:17.357469+0000","last_peered":"2023-10-09T06:17:17.357469+0000","last_clean":"2023-10-09T06:17:17.357469+0000","last_became_active":"2023-10-09T06:15:16.804086+0000","last_became_peered":"2023-10-09T06:15:16.804086+0000","last_unstale":"2023-10-09T06:17:17.357469+0000","last_undegraded":"2023-10-09T06:17:17.357469+0000","last_fullsized":"2023-10-09T06:17:17.357469+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:45.156184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":74,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.371209+0000","last_change":"2023-10-09T06:13:30.788382+0000","last_active":"2023-10-09T06:17:17.371209+0000","last_peered":"2023-10-09T06:17:17.371209+0000","last_clean":"2023-10-09T06:17:17.371209+0000","last_became_active":"2023-10-09T06:13:30.788060+0000","last_became_peered":"2023-10-09T06:13:30.788060+0000","last_unstale":"2023-10-09T06:17:17.371209+0000","last_undegraded":"2023-10-09T06:17:17.371209+0000","last_fullsized":"2023-10-09T06:17:17.371209+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:36:58.984799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.f","version":"591'6054","reported_seq":6081,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:24.369819+0000","last_change":"2023-10-09T06:16:02.755513+0000","last_active":"2023-10-09T06:17:24.369819+0000","last_peered":"2023-10-09T06:17:24.369819+0000","last_clean":"2023-10-09T06:17:24.369819+0000","last_became_active":"2023-10-09T06:16:02.755072+0000","last_became_peered":"2023-10-09T06:16:02.755072+0000","last_unstale":"2023-10-09T06:17:24.369819+0000","last_undegraded":"2023-10-09T06:17:24.369819+0000","last_fullsized":"2023-10-09T06:17:24.369819+0000","mapping_epoch":572,"log_start":"577'200","ondisk_log_start":"577'200","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5854,"log_dups_size":0,"ondisk_log_size":5854,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:11.106599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396754944,"num_objects":6054,"num_object_clones":0,"num_object_copies":18162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6054,"num_write_kb":387456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":1084,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.362378+0000","last_change":"2023-10-09T06:15:19.869123+0000","last_active":"2023-10-09T06:17:17.362378+0000","last_peered":"2023-10-09T06:17:17.362378+0000","last_clean":"2023-10-09T06:17:17.362378+0000","last_became_active":"2023-10-09T06:15:19.868795+0000","last_became_peered":"2023-10-09T06:15:19.868795+0000","last_unstale":"2023-10-09T06:17:17.362378+0000","last_undegraded":"2023-10-09T06:17:17.362378+0000","last_fullsized":"2023-10-09T06:17:17.362378+0000","mapping_epoch":561,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":562,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:17.344518+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:17.344518+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:34.303488+0000","scrub_duration":0.0015914099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":648,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.362028+0000","last_change":"2023-10-09T06:13:30.798394+0000","last_active":"2023-10-09T06:17:17.362028+0000","last_peered":"2023-10-09T06:17:17.362028+0000","last_clean":"2023-10-09T06:17:17.362028+0000","last_became_active":"2023-10-09T06:13:30.790913+0000","last_became_peered":"2023-10-09T06:13:30.790913+0000","last_unstale":"2023-10-09T06:17:17.362028+0000","last_undegraded":"2023-10-09T06:17:17.362028+0000","last_fullsized":"2023-10-09T06:17:17.362028+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:26.508690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":409,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.357556+0000","last_change":"2023-10-09T06:13:21.728009+0000","last_active":"2023-10-09T06:17:17.357556+0000","last_peered":"2023-10-09T06:17:17.357556+0000","last_clean":"2023-10-09T06:17:17.357556+0000","last_became_active":"2023-10-09T06:13:21.727683+0000","last_became_peered":"2023-10-09T06:13:21.727683+0000","last_unstale":"2023-10-09T06:17:17.357556+0000","last_undegraded":"2023-10-09T06:17:17.357556+0000","last_fullsized":"2023-10-09T06:17:17.357556+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:13.074232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":397,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.361190+0000","last_change":"2023-10-09T06:13:30.782038+0000","last_active":"2023-10-09T06:17:17.361190+0000","last_peered":"2023-10-09T06:17:17.361190+0000","last_clean":"2023-10-09T06:17:17.361190+0000","last_became_active":"2023-10-09T06:13:30.781500+0000","last_became_peered":"2023-10-09T06:13:30.781500+0000","last_unstale":"2023-10-09T06:17:17.361190+0000","last_undegraded":"2023-10-09T06:17:17.361190+0000","last_fullsized":"2023-10-09T06:17:17.361190+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:43.663265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"591'6205","reported_seq":6232,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:26.067527+0000","last_change":"2023-10-09T06:16:02.755624+0000","last_active":"2023-10-09T06:17:26.067527+0000","last_peered":"2023-10-09T06:17:26.067527+0000","last_clean":"2023-10-09T06:17:26.067527+0000","last_became_active":"2023-10-09T06:16:02.755333+0000","last_became_peered":"2023-10-09T06:16:02.755333+0000","last_unstale":"2023-10-09T06:17:26.067527+0000","last_undegraded":"2023-10-09T06:17:26.067527+0000","last_fullsized":"2023-10-09T06:17:26.067527+0000","mapping_epoch":572,"log_start":"577'300","ondisk_log_start":"577'300","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5905,"log_dups_size":0,"ondisk_log_size":5905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:59.742519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":406650880,"num_objects":6205,"num_object_clones":0,"num_object_copies":18615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6205,"num_write_kb":397120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":924,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.362552+0000","last_change":"2023-10-09T06:13:33.708100+0000","last_active":"2023-10-09T06:17:17.362552+0000","last_peered":"2023-10-09T06:17:17.362552+0000","last_clean":"2023-10-09T06:17:17.362552+0000","last_became_active":"2023-10-09T06:13:33.707809+0000","last_became_peered":"2023-10-09T06:13:33.707809+0000","last_unstale":"2023-10-09T06:17:17.362552+0000","last_undegraded":"2023-10-09T06:17:17.362552+0000","last_fullsized":"2023-10-09T06:17:17.362552+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:43:26.848778+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":84,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.373537+0000","last_change":"2023-10-09T06:16:02.760976+0000","last_active":"2023-10-09T06:17:17.373537+0000","last_peered":"2023-10-09T06:17:17.373537+0000","last_clean":"2023-10-09T06:17:17.373537+0000","last_became_active":"2023-10-09T06:16:02.760646+0000","last_became_peered":"2023-10-09T06:16:02.760646+0000","last_unstale":"2023-10-09T06:17:17.373537+0000","last_undegraded":"2023-10-09T06:17:17.373537+0000","last_fullsized":"2023-10-09T06:17:17.373537+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:03.779978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":638,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.366569+0000","last_change":"2023-10-09T06:09:27.578756+0000","last_active":"2023-10-09T06:17:17.366569+0000","last_peered":"2023-10-09T06:17:17.366569+0000","last_clean":"2023-10-09T06:17:17.366569+0000","last_became_active":"2023-10-09T06:09:27.578384+0000","last_became_peered":"2023-10-09T06:09:27.578384+0000","last_unstale":"2023-10-09T06:17:17.366569+0000","last_undegraded":"2023-10-09T06:17:17.366569+0000","last_fullsized":"2023-10-09T06:17:17.366569+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:22.182630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":537,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.359264+0000","last_change":"2023-10-09T06:01:22.334081+0000","last_active":"2023-10-09T06:17:17.359264+0000","last_peered":"2023-10-09T06:17:17.359264+0000","last_clean":"2023-10-09T06:17:17.359264+0000","last_became_active":"2023-10-09T06:01:22.333663+0000","last_became_peered":"2023-10-09T06:01:22.333663+0000","last_unstale":"2023-10-09T06:17:17.359264+0000","last_undegraded":"2023-10-09T06:17:17.359264+0000","last_fullsized":"2023-10-09T06:17:17.359264+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:55.809259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":838,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.367261+0000","last_change":"2023-10-09T06:09:27.560157+0000","last_active":"2023-10-09T06:17:17.367261+0000","last_peered":"2023-10-09T06:17:17.367261+0000","last_clean":"2023-10-09T06:17:17.367261+0000","last_became_active":"2023-10-09T06:09:27.559466+0000","last_became_peered":"2023-10-09T06:09:27.559466+0000","last_unstale":"2023-10-09T06:17:17.367261+0000","last_undegraded":"2023-10-09T06:17:17.367261+0000","last_fullsized":"2023-10-09T06:17:17.367261+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.181811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.181811+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:53.431491+0000","scrub_duration":0.011553313000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":521,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.367438+0000","last_change":"2023-10-09T06:13:30.791343+0000","last_active":"2023-10-09T06:17:17.367438+0000","last_peered":"2023-10-09T06:17:17.367438+0000","last_clean":"2023-10-09T06:17:17.367438+0000","last_became_active":"2023-10-09T06:13:30.790945+0000","last_became_peered":"2023-10-09T06:13:30.790945+0000","last_unstale":"2023-10-09T06:17:17.367438+0000","last_undegraded":"2023-10-09T06:17:17.367438+0000","last_fullsized":"2023-10-09T06:17:17.367438+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:14.208391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":303,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.357554+0000","last_change":"2023-10-09T06:15:16.803457+0000","last_active":"2023-10-09T06:17:17.357554+0000","last_peered":"2023-10-09T06:17:17.357554+0000","last_clean":"2023-10-09T06:17:17.357554+0000","last_became_active":"2023-10-09T06:15:16.803097+0000","last_became_peered":"2023-10-09T06:15:16.803097+0000","last_unstale":"2023-10-09T06:17:17.357554+0000","last_undegraded":"2023-10-09T06:17:17.357554+0000","last_fullsized":"2023-10-09T06:17:17.357554+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:45.511763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":941,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.361287+0000","last_change":"2023-10-09T06:00:04.215425+0000","last_active":"2023-10-09T06:17:17.361287+0000","last_peered":"2023-10-09T06:17:17.361287+0000","last_clean":"2023-10-09T06:17:17.361287+0000","last_became_active":"2023-10-09T05:59:40.326685+0000","last_became_peered":"2023-10-09T05:59:40.326685+0000","last_unstale":"2023-10-09T06:17:17.361287+0000","last_undegraded":"2023-10-09T06:17:17.361287+0000","last_fullsized":"2023-10-09T06:17:17.361287+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:04.215270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:04.215270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:03.963694+0000","scrub_duration":0.002850652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":362,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.359684+0000","last_change":"2023-10-09T06:01:54.824780+0000","last_active":"2023-10-09T06:17:17.359684+0000","last_peered":"2023-10-09T06:17:17.359684+0000","last_clean":"2023-10-09T06:17:17.359684+0000","last_became_active":"2023-10-09T06:01:54.757727+0000","last_became_peered":"2023-10-09T06:01:54.757727+0000","last_unstale":"2023-10-09T06:17:17.359684+0000","last_undegraded":"2023-10-09T06:17:17.359684+0000","last_fullsized":"2023-10-09T06:17:17.359684+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:44.132556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":609,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.359098+0000","last_change":"2023-10-09T06:01:54.764989+0000","last_active":"2023-10-09T06:17:17.359098+0000","last_peered":"2023-10-09T06:17:17.359098+0000","last_clean":"2023-10-09T06:17:17.359098+0000","last_became_active":"2023-10-09T06:01:54.764790+0000","last_became_peered":"2023-10-09T06:01:54.764790+0000","last_unstale":"2023-10-09T06:17:17.359098+0000","last_undegraded":"2023-10-09T06:17:17.359098+0000","last_fullsized":"2023-10-09T06:17:17.359098+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:14:07.208858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":376,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.361982+0000","last_change":"2023-10-09T06:13:21.739730+0000","last_active":"2023-10-09T06:17:17.361982+0000","last_peered":"2023-10-09T06:17:17.361982+0000","last_clean":"2023-10-09T06:17:17.361982+0000","last_became_active":"2023-10-09T06:13:21.727326+0000","last_became_peered":"2023-10-09T06:13:21.727326+0000","last_unstale":"2023-10-09T06:17:17.361982+0000","last_undegraded":"2023-10-09T06:17:17.361982+0000","last_fullsized":"2023-10-09T06:17:17.361982+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:51.344571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":606,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.359090+0000","last_change":"2023-10-09T06:09:27.565198+0000","last_active":"2023-10-09T06:17:17.359090+0000","last_peered":"2023-10-09T06:17:17.359090+0000","last_clean":"2023-10-09T06:17:17.359090+0000","last_became_active":"2023-10-09T06:09:27.564909+0000","last_became_peered":"2023-10-09T06:09:27.564909+0000","last_unstale":"2023-10-09T06:17:17.359090+0000","last_undegraded":"2023-10-09T06:17:17.359090+0000","last_fullsized":"2023-10-09T06:17:17.359090+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:46.101014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":756,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.375052+0000","last_change":"2023-10-09T06:13:33.699235+0000","last_active":"2023-10-09T06:17:17.375052+0000","last_peered":"2023-10-09T06:17:17.375052+0000","last_clean":"2023-10-09T06:17:17.375052+0000","last_became_active":"2023-10-09T06:13:33.698886+0000","last_became_peered":"2023-10-09T06:13:33.698886+0000","last_unstale":"2023-10-09T06:17:17.375052+0000","last_undegraded":"2023-10-09T06:17:17.375052+0000","last_fullsized":"2023-10-09T06:17:17.375052+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:00.456132+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:00.456132+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:59.421706+0000","scrub_duration":0.0014992199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":407,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.361576+0000","last_change":"2023-10-09T06:15:16.804361+0000","last_active":"2023-10-09T06:17:17.361576+0000","last_peered":"2023-10-09T06:17:17.361576+0000","last_clean":"2023-10-09T06:17:17.361576+0000","last_became_active":"2023-10-09T06:15:16.803977+0000","last_became_peered":"2023-10-09T06:15:16.803977+0000","last_unstale":"2023-10-09T06:17:17.361576+0000","last_undegraded":"2023-10-09T06:17:17.361576+0000","last_fullsized":"2023-10-09T06:17:17.361576+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:14.141738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":403,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.367445+0000","last_change":"2023-10-09T06:16:02.759834+0000","last_active":"2023-10-09T06:17:17.367445+0000","last_peered":"2023-10-09T06:17:17.367445+0000","last_clean":"2023-10-09T06:17:17.367445+0000","last_became_active":"2023-10-09T06:16:02.759505+0000","last_became_peered":"2023-10-09T06:16:02.759505+0000","last_unstale":"2023-10-09T06:17:17.367445+0000","last_undegraded":"2023-10-09T06:17:17.367445+0000","last_fullsized":"2023-10-09T06:17:17.367445+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:46.034378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":916,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.357143+0000","last_change":"2023-10-09T06:12:55.753877+0000","last_active":"2023-10-09T06:17:17.357143+0000","last_peered":"2023-10-09T06:17:17.357143+0000","last_clean":"2023-10-09T06:17:17.357143+0000","last_became_active":"2023-10-09T06:12:55.753512+0000","last_became_peered":"2023-10-09T06:12:55.753512+0000","last_unstale":"2023-10-09T06:17:17.357143+0000","last_undegraded":"2023-10-09T06:17:17.357143+0000","last_fullsized":"2023-10-09T06:17:17.357143+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:46.290510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:46.290510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:30.368031+0000","scrub_duration":0.002627819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":599,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.366539+0000","last_change":"2023-10-09T06:15:16.814498+0000","last_active":"2023-10-09T06:17:17.366539+0000","last_peered":"2023-10-09T06:17:17.366539+0000","last_clean":"2023-10-09T06:17:17.366539+0000","last_became_active":"2023-10-09T06:15:16.807217+0000","last_became_peered":"2023-10-09T06:15:16.807217+0000","last_unstale":"2023-10-09T06:17:17.366539+0000","last_undegraded":"2023-10-09T06:17:17.366539+0000","last_fullsized":"2023-10-09T06:17:17.366539+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:50.886043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":622,"reported_epoch":590,"state":"active+clean","last_fresh":"2023-10-09T06:17:17.358789+0000","last_change":"2023-10-09T06:13:21.727720+0000","last_active":"2023-10-09T06:17:17.358789+0000","last_peered":"2023-10-09T06:17:17.358789+0000","last_clean":"2023-10-09T06:17:17.358789+0000","last_became_active":"2023-10-09T06:13:21.727339+0000","last_became_peered":"2023-10-09T06:13:21.727339+0000","last_unstale":"2023-10-09T06:17:17.358789+0000","last_undegraded":"2023-10-09T06:17:17.358789+0000","last_fullsized":"2023-10-09T06:17:17.358789+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:25.723752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":6371344384,"num_objects":97219,"num_object_clones":0,"num_object_copies":291657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":97219,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":97219,"num_write_kb":6222016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19083821056,"data_stored":19083821056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91019,"ondisk_log_size":91019,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":61,"num_bytes_recovered":12302,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13410,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1706112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234453,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3972800,"kb_used_data":3084280,"kb_used_omap":0,"kb_used_meta":888512,"kb_avail":89772352,"statfs":{"total":95995035648,"available":91926888448,"internally_reserved":0,"allocated":3158302720,"data_stored":3156706240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":909836288},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:33 2023","interfaces":[{"interface":"back","average":{"1min":10.606999999999999,"5min":8.3040000000000003,"15min":3.5750000000000002},"min":{"1min":0.24399999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":166.94,"5min":166.94,"15min":166.94},"last":0.23899999999999999},{"interface":"front","average":{"1min":10.544,"5min":8.4550000000000001,"15min":3.6070000000000002},"min":{"1min":0.29299999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":166.923,"5min":166.923,"15min":166.923},"last":2.6400000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:17:08 2023","interfaces":[{"interface":"back","average":{"1min":1.9910000000000001,"5min":4.6840000000000002,"15min":4.226},"min":{"1min":0.255,"5min":0.252,"15min":0.218},"max":{"1min":10.122,"5min":108.113,"15min":124.66200000000001},"last":2.6709999999999998},{"interface":"front","average":{"1min":1.734,"5min":4.4950000000000001,"15min":4.1699999999999999},"min":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.22700000000000001},"max":{"1min":6.016,"5min":108.503,"15min":125.023},"last":3.6000000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:16:33 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":2.347,"15min":1.8400000000000001},"min":{"1min":0.34599999999999997,"5min":0.29899999999999999,"15min":0.24299999999999999},"max":{"1min":5.5449999999999999,"5min":47.503,"15min":47.503},"last":3.452},{"interface":"front","average":{"1min":0.70399999999999996,"5min":1.9330000000000001,"15min":1.863},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.27000000000000002},"max":{"1min":4.3479999999999999,"5min":47.545000000000002,"15min":47.545000000000002},"last":2.2389999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:17:08 2023","interfaces":[{"interface":"back","average":{"1min":2.0720000000000001,"5min":5.1239999999999997,"15min":4.351},"min":{"1min":0.37,"5min":0.33800000000000002,"15min":0.29899999999999999},"max":{"1min":6.6070000000000002,"5min":67.852000000000004,"15min":116.68000000000001},"last":2.3170000000000002},{"interface":"front","average":{"1min":1.462,"5min":4.6929999999999996,"15min":4.4119999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.32100000000000001},"max":{"1min":5.5129999999999999,"5min":67.881,"15min":116.797},"last":0.35899999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:16:48 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":1.0529999999999999,"15min":1.014},"min":{"1min":0.29099999999999998,"5min":0.24299999999999999,"15min":0.23100000000000001},"max":{"1min":5.4029999999999996,"5min":10.257,"15min":14.371},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.83399999999999996,"5min":0.98499999999999999,"15min":1.0029999999999999},"min":{"1min":0.27800000000000002,"5min":0.246,"15min":0.246},"max":{"1min":4.5599999999999996,"5min":12.583,"15min":14.259},"last":2.2789999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:16:57 2023","interfaces":[{"interface":"back","average":{"1min":10.836,"5min":2.9039999999999999,"15min":1.782},"min":{"1min":0.29899999999999999,"5min":0.27100000000000002,"15min":0.24399999999999999},"max":{"1min":216.036,"5min":216.036,"15min":216.036},"last":2.5830000000000002},{"interface":"front","average":{"1min":10.542999999999999,"5min":2.8580000000000001,"15min":1.829},"min":{"1min":0.32600000000000001,"5min":0.27800000000000002,"15min":0.24399999999999999},"max":{"1min":215.80600000000001,"5min":215.80600000000001,"15min":215.80600000000001},"last":2.3530000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:16:57 2023","interfaces":[{"interface":"back","average":{"1min":11.539999999999999,"5min":3.2120000000000002,"15min":1.784},"min":{"1min":0.34200000000000003,"5min":0.308,"15min":0.26600000000000001},"max":{"1min":233.56700000000001,"5min":233.56700000000001,"15min":233.56700000000001},"last":2.6150000000000002},{"interface":"front","average":{"1min":11.452,"5min":2.968,"15min":1.7230000000000001},"min":{"1min":0.34100000000000003,"5min":0.32000000000000001,"15min":0.30599999999999999},"max":{"1min":232.477,"5min":232.477,"15min":232.477},"last":0.44500000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496090,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3946740,"kb_used_data":3049196,"kb_used_omap":0,"kb_used_meta":897536,"kb_avail":89798412,"statfs":{"total":95995035648,"available":91953573888,"internally_reserved":0,"allocated":3122376704,"data_stored":3120792145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919076864},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:31 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":9.5510000000000002,"15min":4.6079999999999997},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.657,"5min":191.50800000000001,"15min":191.50800000000001},"last":2.7799999999999998},{"interface":"front","average":{"1min":1.2030000000000001,"5min":9.9480000000000004,"15min":4.5960000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.0940000000000003,"5min":191.52600000000001,"15min":191.52600000000001},"last":2.7890000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:16:31 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":9.0489999999999995,"15min":6.1719999999999997},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23200000000000001},"max":{"1min":4.4000000000000004,"5min":328.04399999999998,"15min":328.04399999999998},"last":5.6539999999999999},{"interface":"front","average":{"1min":0.86099999999999999,"5min":8.4280000000000008,"15min":5.9489999999999998},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":2.7679999999999998,"5min":328.10000000000002,"15min":328.10000000000002},"last":0.44600000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:16:31 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":4.5199999999999996,"15min":2.8570000000000002},"min":{"1min":0.28100000000000003,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.2640000000000002,"5min":163.04900000000001,"15min":163.04900000000001},"last":3.6960000000000002},{"interface":"front","average":{"1min":0.74299999999999999,"5min":4.5519999999999996,"15min":2.8170000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28599999999999998},"max":{"1min":3.8330000000000002,"5min":162.93000000000001,"15min":162.93000000000001},"last":3.617}]},{"osd":3,"last update":"Mon Oct 9 06:17:19 2023","interfaces":[{"interface":"back","average":{"1min":1.3959999999999999,"5min":7.2939999999999996,"15min":6.3019999999999996},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.024,"5min":250.744,"15min":250.744},"last":1.4179999999999999},{"interface":"front","average":{"1min":2.4780000000000002,"5min":7.0830000000000002,"15min":6.1580000000000004},"min":{"1min":0.45500000000000002,"5min":0.309,"15min":0.309},"max":{"1min":7.8600000000000003,"5min":250.83799999999999,"15min":250.83799999999999},"last":2.9489999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:16:52 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.91200000000000003,"15min":1.5700000000000001},"min":{"1min":0.251,"5min":0.24399999999999999,"15min":0.24199999999999999},"max":{"1min":0.97999999999999998,"5min":9.0869999999999997,"15min":151.35599999999999},"last":0.38700000000000001},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.3089999999999999,"15min":1.7689999999999999},"min":{"1min":0.315,"5min":0.25900000000000001,"15min":0.23999999999999999},"max":{"1min":6.9889999999999999,"5min":8.1780000000000008,"15min":149.15799999999999},"last":0.35299999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:16:31 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.77400000000000002,"15min":1.0720000000000001},"min":{"1min":0.30299999999999999,"5min":0.29699999999999999,"15min":0.23000000000000001},"max":{"1min":5.9210000000000003,"5min":8.3829999999999991,"15min":15.342000000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.86599999999999999,"5min":1.0740000000000001,"15min":1.1859999999999999},"min":{"1min":0.33300000000000002,"5min":0.28699999999999998,"15min":0.255},"max":{"1min":4.3579999999999997,"5min":8.9809999999999999,"15min":15.358000000000001},"last":0.498}]},{"osd":7,"last update":"Mon Oct 9 06:16:52 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":1.0980000000000001,"15min":1.2190000000000001},"min":{"1min":0.37,"5min":0.30299999999999999,"15min":0.29399999999999998},"max":{"1min":1.841,"5min":5.8380000000000001,"15min":21.98},"last":0.435},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.94699999999999995,"15min":1.1930000000000001},"min":{"1min":0.37,"5min":0.34999999999999998,"15min":0.33100000000000002},"max":{"1min":1.536,"5min":5.6769999999999996,"15min":18.241},"last":0.46100000000000002}]}]},{"osd":1,"up_from":89,"seq":382252089678,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4691360,"kb_used_data":3942360,"kb_used_omap":9,"kb_used_meta":748982,"kb_avail":89053792,"statfs":{"total":95995035648,"available":91191083008,"internally_reserved":0,"allocated":4036976640,"data_stored":4033650029,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":766958569},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:45 2023","interfaces":[{"interface":"back","average":{"1min":6.3220000000000001,"5min":3.831,"15min":2.8039999999999998},"min":{"1min":0.214,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":118.17,"5min":118.17,"15min":118.17},"last":1.518},{"interface":"front","average":{"1min":6.532,"5min":4.1890000000000001,"15min":3.0209999999999999},"min":{"1min":0.27600000000000002,"5min":0.20699999999999999,"15min":0.20200000000000001},"max":{"1min":117.634,"5min":117.634,"15min":117.634},"last":3.589}]},{"osd":2,"last update":"Mon Oct 9 06:16:48 2023","interfaces":[{"interface":"back","average":{"1min":0.81100000000000005,"5min":1.4610000000000001,"15min":1.6639999999999999},"min":{"1min":0.251,"5min":0.217,"15min":0.20799999999999999},"max":{"1min":4.2009999999999996,"5min":12.818,"15min":17.681000000000001},"last":2.3460000000000001},{"interface":"front","average":{"1min":0.59299999999999997,"5min":2.0569999999999999,"15min":2.0699999999999998},"min":{"1min":0.23499999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":3.7669999999999999,"5min":55.694000000000003,"15min":55.694000000000003},"last":0.92300000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:17:19 2023","interfaces":[{"interface":"back","average":{"1min":12.894,"5min":5.2130000000000001,"15min":4.6660000000000004},"min":{"1min":0.373,"5min":0.26800000000000002,"15min":0.24199999999999999},"max":{"1min":95.489999999999995,"5min":95.489999999999995,"15min":142.97},"last":1.587},{"interface":"front","average":{"1min":12.641,"5min":4.9199999999999999,"15min":4.6230000000000002},"min":{"1min":0.45100000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":95.436999999999998,"5min":95.436999999999998,"15min":142.94800000000001},"last":0.88700000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:16:48 2023","interfaces":[{"interface":"back","average":{"1min":4.2009999999999996,"5min":3.9319999999999999,"15min":2.5659999999999998},"min":{"1min":0.309,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":66.084000000000003,"5min":93.747,"15min":93.747},"last":0.99199999999999999},{"interface":"front","average":{"1min":4.6150000000000002,"5min":4.3719999999999999,"15min":3.0179999999999998},"min":{"1min":0.34399999999999997,"5min":0.32800000000000001,"15min":0.308},"max":{"1min":66.399000000000001,"5min":97.852000000000004,"15min":97.852000000000004},"last":0.63400000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:17:03 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":1.081,"15min":1.5840000000000001},"min":{"1min":0.378,"5min":0.312,"15min":0.30499999999999999},"max":{"1min":3.9849999999999999,"5min":6.1820000000000004,"15min":36.356999999999999},"last":0.95499999999999996},{"interface":"front","average":{"1min":1.262,"5min":1.6759999999999999,"15min":2.3149999999999999},"min":{"1min":0.39000000000000001,"5min":0.318,"15min":0.29799999999999999},"max":{"1min":4.0819999999999999,"5min":14.648,"15min":36.268000000000001},"last":0.69399999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:17:03 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":1.159,"15min":1.804},"min":{"1min":0.34599999999999997,"5min":0.33700000000000002,"15min":0.33100000000000002},"max":{"1min":3.9039999999999999,"5min":6.218,"15min":52.207000000000001},"last":1.024},{"interface":"front","average":{"1min":1.6699999999999999,"5min":1.7470000000000001,"15min":2.379},"min":{"1min":0.38500000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":4.8289999999999997,"5min":14.619999999999999,"15min":51.212000000000003},"last":0.72699999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:17:03 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":2.5379999999999998,"15min":2.6379999999999999},"min":{"1min":0.42899999999999999,"5min":0.374,"15min":0.374},"max":{"1min":4.0190000000000001,"5min":56.881,"15min":56.881},"last":0.66800000000000004},{"interface":"front","average":{"1min":1.0649999999999999,"5min":2.129,"15min":2.093},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.38},"max":{"1min":4.5469999999999997,"5min":56.941000000000003,"15min":56.941000000000003},"last":1.724}]}]},{"osd":0,"up_from":456,"seq":1958505087073,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2701676,"kb_used_data":2321444,"kb_used_omap":32,"kb_used_meta":380191,"kb_avail":91043476,"statfs":{"total":95995035648,"available":93228519424,"internally_reserved":0,"allocated":2377158656,"data_stored":2374059913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32794,"internal_metadata":389316582},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:16:45 2023","interfaces":[{"interface":"back","average":{"1min":1.335,"5min":4.2400000000000002,"15min":8.3989999999999991},"min":{"1min":0.22800000000000001,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":5.109,"5min":102.81999999999999,"15min":166.833},"last":4.694},{"interface":"front","average":{"1min":1.665,"5min":4.5529999999999999,"15min":7.931},"min":{"1min":0.28499999999999998,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":7.7469999999999999,"5min":103.108,"15min":166.81100000000001},"last":4.6760000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:16:45 2023","interfaces":[{"interface":"back","average":{"1min":1.2010000000000001,"5min":2.0179999999999998,"15min":3.2490000000000001},"min":{"1min":0.25700000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":5.2930000000000001,"5min":21.648,"15min":21.648},"last":1.1930000000000001},{"interface":"front","average":{"1min":1.05,"5min":1.931,"15min":3.1040000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.7330000000000005,"5min":11.890000000000001,"15min":20.152000000000001},"last":0.35599999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:16:37 2023","interfaces":[{"interface":"back","average":{"1min":1.218,"5min":2.0249999999999999,"15min":10.420999999999999},"min":{"1min":0.27400000000000002,"5min":0.26500000000000001,"15min":0.25600000000000001},"max":{"1min":4.7110000000000003,"5min":10.217000000000001,"15min":203.59299999999999},"last":1.1020000000000001},{"interface":"front","average":{"1min":1.244,"5min":2.452,"15min":10.032},"min":{"1min":0.29499999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":4.8339999999999996,"5min":18.192,"15min":202.96600000000001},"last":0.76600000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:16:45 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":2.069,"15min":2.2450000000000001},"min":{"1min":0.32500000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.7569999999999997,"5min":23.515999999999998,"15min":23.515999999999998},"last":1.069},{"interface":"front","average":{"1min":1.0189999999999999,"5min":2.423,"15min":2.181},"min":{"1min":0.39800000000000002,"5min":0.254,"15min":0.254},"max":{"1min":4.6260000000000003,"5min":39.302,"15min":39.302},"last":1.2969999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:16:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":2.0699999999999998,"15min":2.6669999999999998},"min":{"1min":0.42199999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":5.3920000000000003,"5min":39.369999999999997,"15min":161.286},"last":4.2460000000000004},{"interface":"front","average":{"1min":1.375,"5min":1.8069999999999999,"15min":2.7280000000000002},"min":{"1min":0.34899999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.8170000000000002,"5min":13.731999999999999,"15min":161.31800000000001},"last":2.2480000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:16:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":2.2069999999999999,"15min":2.3900000000000001},"min":{"1min":0.40999999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.7199999999999998,"5min":31.648,"15min":31.648},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.84699999999999998,"5min":1.7909999999999999,"15min":2.4039999999999999},"min":{"1min":0.36399999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":5.4320000000000004,"5min":24.43,"15min":24.43},"last":1.179}]},{"osd":7,"last update":"Mon Oct 9 06:16:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3819999999999999,"5min":1.9350000000000001,"15min":2.7610000000000001},"min":{"1min":0.42899999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":4.7839999999999998,"5min":13.821,"15min":123.685},"last":0.89700000000000002},{"interface":"front","average":{"1min":0.97099999999999997,"5min":1.5980000000000001,"15min":2.7280000000000002},"min":{"1min":0.47499999999999998,"5min":0.40999999999999998,"15min":0.40200000000000002},"max":{"1min":4.8010000000000002,"5min":10.093,"15min":122.482},"last":1.1419999999999999}]}]},{"osd":2,"up_from":0,"seq":403726926170,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":99,"seq":425201762643,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3722300,"kb_used_data":2710004,"kb_used_omap":7,"kb_used_meta":1012216,"kb_avail":90022852,"statfs":{"total":95995035648,"available":92183400448,"internally_reserved":0,"allocated":2775044096,"data_stored":2771745336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1036509800},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:37 2023","interfaces":[{"interface":"back","average":{"1min":3.6379999999999999,"5min":6.3440000000000003,"15min":3.423},"min":{"1min":0.20999999999999999,"5min":0.186,"15min":0.17299999999999999},"max":{"1min":50.491,"5min":220.45400000000001,"15min":220.45400000000001},"last":0.60699999999999998},{"interface":"front","average":{"1min":3.952,"5min":7.0679999999999996,"15min":4.6790000000000003},"min":{"1min":0.25600000000000001,"5min":0.23200000000000001,"15min":0.19600000000000001},"max":{"1min":50.173000000000002,"5min":216.44200000000001,"15min":216.44200000000001},"last":0.378}]},{"osd":1,"last update":"Mon Oct 9 06:17:12 2023","interfaces":[{"interface":"back","average":{"1min":3.0800000000000001,"5min":2.645,"15min":3.0720000000000001},"min":{"1min":0.223,"5min":0.20300000000000001,"15min":0.19800000000000001},"max":{"1min":9.1460000000000008,"5min":30.632000000000001,"15min":207.75299999999999},"last":0.65400000000000003},{"interface":"front","average":{"1min":3.169,"5min":3.407,"15min":3.8700000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":13.682,"5min":49.335999999999999,"15min":207.12200000000001},"last":4.4089999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:17:12 2023","interfaces":[{"interface":"back","average":{"1min":1.5029999999999999,"5min":1.8100000000000001,"15min":2.0859999999999999},"min":{"1min":0.23699999999999999,"5min":0.23400000000000001,"15min":0.223},"max":{"1min":5.0960000000000001,"5min":12.561,"15min":41.945999999999998},"last":0.46500000000000002},{"interface":"front","average":{"1min":2.4769999999999999,"5min":2.2679999999999998,"15min":2.653},"min":{"1min":0.30599999999999999,"5min":0.25600000000000001,"15min":0.23899999999999999},"max":{"1min":10.624000000000001,"5min":12.864000000000001,"15min":41.475999999999999},"last":4.3730000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:16:58 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":2.5329999999999999,"15min":2.4649999999999999},"min":{"1min":0.318,"5min":0.28199999999999997,"15min":0.254},"max":{"1min":4.1020000000000003,"5min":95.739000000000004,"15min":99.876000000000005},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.964,"5min":2.7930000000000001,"15min":2.8010000000000002},"min":{"1min":0.39200000000000002,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":7.8330000000000002,"5min":97.941999999999993,"15min":99.914000000000001},"last":4.4749999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:17:12 2023","interfaces":[{"interface":"back","average":{"1min":1.7330000000000001,"5min":2.0310000000000001,"15min":2.1480000000000001},"min":{"1min":0.39800000000000002,"5min":0.32300000000000001,"15min":0.24199999999999999},"max":{"1min":7.9409999999999998,"5min":49.363,"15min":74.706000000000003},"last":4.2380000000000004},{"interface":"front","average":{"1min":2.1059999999999999,"5min":2.0750000000000002,"15min":3.3540000000000001},"min":{"1min":0.35299999999999998,"5min":0.32800000000000001,"15min":0.28799999999999998},"max":{"1min":9.3070000000000004,"5min":49.392000000000003,"15min":235.56},"last":2.7770000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:17:12 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":2.101,"15min":2.923},"min":{"1min":0.35899999999999999,"5min":0.35299999999999998,"15min":0.26100000000000001},"max":{"1min":7.9820000000000002,"5min":49.526000000000003,"15min":233.26599999999999},"last":0.63700000000000001},{"interface":"front","average":{"1min":2.0150000000000001,"5min":2.3980000000000001,"15min":3.4089999999999998},"min":{"1min":0.46300000000000002,"5min":0.35999999999999999,"15min":0.33600000000000002},"max":{"1min":7.3490000000000002,"5min":49.429000000000002,"15min":233.24100000000001},"last":0.501}]},{"osd":7,"last update":"Mon Oct 9 06:17:12 2023","interfaces":[{"interface":"back","average":{"1min":2.0219999999999998,"5min":2.9870000000000001,"15min":3.6059999999999999},"min":{"1min":0.374,"5min":0.36799999999999999,"15min":0.35699999999999998},"max":{"1min":5.0709999999999997,"5min":94.400000000000006,"15min":230.69399999999999},"last":1.0600000000000001},{"interface":"front","average":{"1min":2.02,"5min":3.7149999999999999,"15min":3.9700000000000002},"min":{"1min":0.42199999999999999,"5min":0.41699999999999998,"15min":0.32600000000000001},"max":{"1min":7.8730000000000002,"5min":94.195999999999998,"15min":230.67500000000001},"last":4.444}]}]},{"osd":4,"up_from":322,"seq":1382979469506,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2576764,"kb_used_data":1953716,"kb_used_omap":35,"kb_used_meta":623004,"kb_avail":91168388,"statfs":{"total":95995035648,"available":93356429312,"internally_reserved":0,"allocated":2000605184,"data_stored":1999012509,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36552,"internal_metadata":637956408},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:34 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":2.9540000000000002,"15min":1.9690000000000001},"min":{"1min":0.26700000000000002,"5min":0.216,"15min":0.216},"max":{"1min":4.3840000000000003,"5min":81.066000000000003,"15min":81.066000000000003},"last":0.98399999999999999},{"interface":"front","average":{"1min":1.0429999999999999,"5min":2.7170000000000001,"15min":1.9750000000000001},"min":{"1min":0.36399999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.6440000000000001,"5min":81.603999999999999,"15min":81.603999999999999},"last":1.367}]},{"osd":1,"last update":"Mon Oct 9 06:16:54 2023","interfaces":[{"interface":"back","average":{"1min":7.8280000000000003,"5min":4.0410000000000004,"15min":2.9710000000000001},"min":{"1min":0.28399999999999997,"5min":0.24099999999999999,"15min":0.20999999999999999},"max":{"1min":101.562,"5min":101.562,"15min":101.562},"last":1.345},{"interface":"front","average":{"1min":8.1669999999999998,"5min":3.8929999999999998,"15min":3.0139999999999998},"min":{"1min":0.29999999999999999,"5min":0.25600000000000001,"15min":0.22800000000000001},"max":{"1min":101.5,"5min":101.5,"15min":101.5},"last":0.877}]},{"osd":2,"last update":"Mon Oct 9 06:16:54 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":4.2699999999999996,"15min":2.6099999999999999},"min":{"1min":0.32400000000000001,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":3.157,"5min":245.166,"15min":245.166},"last":1.236},{"interface":"front","average":{"1min":1.3480000000000001,"5min":4.1769999999999996,"15min":2.4590000000000001},"min":{"1min":0.33600000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.758,"5min":245.119,"15min":245.119},"last":1.3959999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:16:54 2023","interfaces":[{"interface":"back","average":{"1min":11.574999999999999,"5min":9.7050000000000001,"15min":4.9450000000000003},"min":{"1min":0.38900000000000001,"5min":0.29399999999999998,"15min":0.26800000000000002},"max":{"1min":197.31899999999999,"5min":246.39400000000001,"15min":246.39400000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":11.532,"5min":9.4459999999999997,"15min":4.593},"min":{"1min":0.39800000000000002,"5min":0.314,"15min":0.314},"max":{"1min":198.565,"5min":246.99000000000001,"15min":246.99000000000001},"last":0.96199999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:16:54 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":2.0449999999999999,"15min":1.405},"min":{"1min":0.249,"5min":0.24399999999999999,"15min":0.216},"max":{"1min":3.6030000000000002,"5min":99.405000000000001,"15min":99.405000000000001},"last":0.37},{"interface":"front","average":{"1min":0.71699999999999997,"5min":2.262,"15min":1.5009999999999999},"min":{"1min":0.31,"5min":0.26700000000000002,"15min":0.245},"max":{"1min":2.0529999999999999,"5min":99.206000000000003,"15min":99.206000000000003},"last":0.311}]},{"osd":6,"last update":"Mon Oct 9 06:16:54 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.92900000000000005,"15min":1.6699999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":3.6829999999999998,"5min":7.8140000000000001,"15min":95.492999999999995},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.58899999999999997,"5min":1.179,"15min":1.8080000000000001},"min":{"1min":0.373,"5min":0.33200000000000002,"15min":0.27900000000000003},"max":{"1min":1.956,"5min":9.2110000000000003,"15min":95.664000000000001},"last":0.26800000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:16:54 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":2.2759999999999998,"15min":2.0089999999999999},"min":{"1min":0.35999999999999999,"5min":0.33600000000000002,"15min":0.29499999999999998},"max":{"1min":2.3199999999999998,"5min":95.167000000000002,"15min":95.167000000000002},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.79700000000000004,"5min":2.206,"15min":1.875},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.251},"max":{"1min":3.1190000000000002,"5min":94.811999999999998,"15min":94.811999999999998},"last":0.34799999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757726,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2422128,"kb_used_data":1535912,"kb_used_omap":0,"kb_used_meta":886208,"kb_avail":91323024,"statfs":{"total":95995035648,"available":93514776576,"internally_reserved":0,"allocated":1572773888,"data_stored":1571189789,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":907476992},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:36 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":4.1020000000000003,"15min":2.9100000000000001},"min":{"1min":0.26200000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":2.016,"5min":115.92400000000001,"15min":115.92400000000001},"last":2.7909999999999999},{"interface":"front","average":{"1min":0.72599999999999998,"5min":4.2649999999999997,"15min":3.4460000000000002},"min":{"1min":0.34100000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.0630000000000002,"5min":115.904,"15min":115.904},"last":2.25}]},{"osd":1,"last update":"Mon Oct 9 06:17:08 2023","interfaces":[{"interface":"back","average":{"1min":1.839,"5min":1.6259999999999999,"15min":4.4900000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.22600000000000001},"max":{"1min":5.7430000000000003,"5min":18.812000000000001,"15min":151.78200000000001},"last":4.0979999999999999},{"interface":"front","average":{"1min":1.1930000000000001,"5min":1.5049999999999999,"15min":4.6390000000000002},"min":{"1min":0.38800000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":2.7480000000000002,"5min":19.376999999999999,"15min":152.251},"last":2.7559999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:16:31 2023","interfaces":[{"interface":"back","average":{"1min":1.3700000000000001,"5min":2.9750000000000001,"15min":2.7850000000000001},"min":{"1min":0.30499999999999999,"5min":0.28499999999999998,"15min":0.254},"max":{"1min":11.654999999999999,"5min":117.97199999999999,"15min":129.21199999999999},"last":4.1230000000000002},{"interface":"front","average":{"1min":1.002,"5min":2.7549999999999999,"15min":2.871},"min":{"1min":0.35799999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.6619999999999999,"5min":118.52,"15min":129.042},"last":2.8290000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:17:08 2023","interfaces":[{"interface":"back","average":{"1min":1.7569999999999999,"5min":4.0789999999999997,"15min":5.4829999999999997},"min":{"1min":0.318,"5min":0.316,"15min":0.29499999999999998},"max":{"1min":10.193,"5min":93.338999999999999,"15min":143.31200000000001},"last":2.2839999999999998},{"interface":"front","average":{"1min":2.6480000000000001,"5min":4.6059999999999999,"15min":5.71},"min":{"1min":0.40100000000000002,"5min":0.33500000000000002,"15min":0.29399999999999998},"max":{"1min":10.265000000000001,"5min":93.435000000000002,"15min":140.00899999999999},"last":4.0609999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:16:56 2023","interfaces":[{"interface":"back","average":{"1min":1.097,"5min":3.7000000000000002,"15min":2.008},"min":{"1min":0.32600000000000001,"5min":0.246,"15min":0.245},"max":{"1min":4.9039999999999999,"5min":193.25200000000001,"15min":193.25200000000001},"last":2.9350000000000001},{"interface":"front","average":{"1min":1.014,"5min":3.5510000000000002,"15min":1.859},"min":{"1min":0.30299999999999999,"5min":0.26100000000000001,"15min":0.26000000000000001},"max":{"1min":2.9140000000000001,"5min":190.06800000000001,"15min":190.06800000000001},"last":2.3100000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:17:08 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":0.90900000000000003,"15min":1.1240000000000001},"min":{"1min":0.28100000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.2199999999999998,"5min":6.2199999999999998,"15min":9.0909999999999993},"last":5.5499999999999998},{"interface":"front","average":{"1min":1.194,"5min":1.1830000000000001,"15min":1.2210000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27300000000000002},"max":{"1min":5.7670000000000003,"5min":11.577999999999999,"15min":11.577999999999999},"last":8.7210000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:16:56 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":3.8490000000000002,"15min":2.6360000000000001},"min":{"1min":0.36799999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.9550000000000001,"5min":180.696,"15min":180.696},"last":4.2229999999999999},{"interface":"front","average":{"1min":0.80500000000000005,"5min":3.4660000000000002,"15min":2.4620000000000002},"min":{"1min":0.35399999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.2549999999999999,"5min":180.74000000000001,"15min":180.74000000000001},"last":5.6879999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4889,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2380988416,"data_stored":2380988416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4036231168,"data_stored":4036231168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2776825856,"data_stored":2776825856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1998323712,"data_stored":1998323712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1574961152,"data_stored":1574961152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3144155136,"data_stored":3144155136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3172335616,"data_stored":3172335616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:17:30.614 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:17:30.640 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:30.713 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:17:30.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:30.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:30.718 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:30 smithi156 ceph-mon[195101]: failed to encode map e592 with expected crc 2023-10-09T06:17:30.718 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:30 smithi156 ceph-mon[195101]: from='client.36693 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:17:30.718 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:30 smithi156 ceph-mon[195101]: pgmap v1520: 119 pgs: 119 active+clean; 5.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 52 MiB/s wr, 825 op/s 2023-10-09T06:17:30.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:30 smithi064 ceph-mon[220044]: failed to encode map e592 with expected crc 2023-10-09T06:17:30.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:30 smithi064 ceph-mon[220044]: from='client.36693 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:17:30.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:30 smithi064 ceph-mon[220044]: pgmap v1520: 119 pgs: 119 active+clean; 5.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 52 MiB/s wr, 825 op/s 2023-10-09T06:17:30.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:30 smithi064 ceph-mon[212750]: failed to encode map e592 with expected crc 2023-10-09T06:17:30.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:30 smithi064 ceph-mon[212750]: from='client.36693 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:17:30.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:30 smithi064 ceph-mon[212750]: pgmap v1520: 119 pgs: 119 active+clean; 5.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 52 MiB/s wr, 825 op/s 2023-10-09T06:17:30.818 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:30.869 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:30.939 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:17:30.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:30.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:31.055 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:31.059 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 81 16 101519 101503 78.2986 51.625 0.0226983 0.0127665 2023-10-09T06:17:31.064 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:17:31.064 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T06:17:31.065 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T06:17:31.065 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:17:31.065 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.065 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:81471 10.3 10:cc6caecd:::benchmark_data_smithi064_573750_object81470:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-10-09T06:17:31.065 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:12:59.854905+0000", 2023-10-09T06:17:31.065 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 271.20541866399998, 2023-10-09T06:17:31.065 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.214265604, 2023-10-09T06:17:31.065 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:31.065 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:31.065 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:31.066 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:17:31.066 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:17:31.066 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 81471 2023-10-09T06:17:31.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.066 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:31.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:31.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.854905+0000", 2023-10-09T06:17:31.066 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:31.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.854905+0000", 2023-10-09T06:17:31.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:31.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.854903+0000", 2023-10-09T06:17:31.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T06:17:31.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:31.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.854933+0000", 2023-10-09T06:17:31.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0031000000000001e-05 2023-10-09T06:17:31.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.068 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:31.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.854950+0000", 2023-10-09T06:17:31.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6651e-05 2023-10-09T06:17:31.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.068 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:31.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.854978+0000", 2023-10-09T06:17:31.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7837000000000001e-05 2023-10-09T06:17:31.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.069 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:31.069 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.859094+0000", 2023-10-09T06:17:31.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041166450000000004 2023-10-09T06:17:31.069 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.069 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:31.069 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.859292+0000", 2023-10-09T06:17:31.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019750099999999999 2023-10-09T06:17:31.069 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.069 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:17:31.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.859461+0000", 2023-10-09T06:17:31.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000168717 2023-10-09T06:17:31.070 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.070 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.862286+0000", 2023-10-09T06:17:31.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028254970000000002 2023-10-09T06:17:31.070 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.070 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:31.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.862633+0000", 2023-10-09T06:17:31.071 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00034674599999999999 2023-10-09T06:17:31.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.071 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.071 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.864600+0000", 2023-10-09T06:17:31.071 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019670030000000002 2023-10-09T06:17:31.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.071 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.071 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:00.068977+0000", 2023-10-09T06:17:31.071 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.204377264 2023-10-09T06:17:31.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.072 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.072 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:31.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:00.069046+0000", 2023-10-09T06:17:31.072 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8472000000000004e-05 2023-10-09T06:17:31.072 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.072 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.072 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:31.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:00.069171+0000", 2023-10-09T06:17:31.072 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012542099999999999 2023-10-09T06:17:31.072 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.072 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:31.072 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.073 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:83019 10.3 10:ce4365a3:::benchmark_data_smithi064_573750_object83018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e514)", 2023-10-09T06:17:31.073 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.816350+0000", 2023-10-09T06:17:31.073 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 269.243974471, 2023-10-09T06:17:31.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27937314000000002, 2023-10-09T06:17:31.073 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:31.073 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:31.073 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:31.073 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:17:31.073 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:17:31.073 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 83019 2023-10-09T06:17:31.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.074 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:31.074 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:31.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.816350+0000", 2023-10-09T06:17:31.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.074 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:31.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.816350+0000", 2023-10-09T06:17:31.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.075 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.075 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:31.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.816349+0000", 2023-10-09T06:17:31.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:17:31.075 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.075 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:31.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.816373+0000", 2023-10-09T06:17:31.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3547999999999999e-05 2023-10-09T06:17:31.075 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.076 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:31.076 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.816379+0000", 2023-10-09T06:17:31.076 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7440000000000001e-06 2023-10-09T06:17:31.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.076 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.076 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:31.076 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.816385+0000", 2023-10-09T06:17:31.076 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2959999999999998e-06 2023-10-09T06:17:31.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.076 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.076 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:31.076 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.820672+0000", 2023-10-09T06:17:31.077 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.004287155 2023-10-09T06:17:31.077 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.077 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.077 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:31.077 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.820810+0000", 2023-10-09T06:17:31.077 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000137787 2023-10-09T06:17:31.077 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.077 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.077 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:17:31.077 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.820957+0000", 2023-10-09T06:17:31.077 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014771599999999999 2023-10-09T06:17:31.077 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.078 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.078 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.078 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.827525+0000", 2023-10-09T06:17:31.078 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065673570000000002 2023-10-09T06:17:31.078 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.078 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.078 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.078 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.827554+0000", 2023-10-09T06:17:31.078 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9244e-05 2023-10-09T06:17:31.078 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.078 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.078 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832230+0000", 2023-10-09T06:17:31.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046755659999999999 2023-10-09T06:17:31.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:31.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832551+0000", 2023-10-09T06:17:31.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00032125600000000002 2023-10-09T06:17:31.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:31.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832599+0000", 2023-10-09T06:17:31.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8233999999999999e-05 2023-10-09T06:17:31.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:31.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095723+0000", 2023-10-09T06:17:31.080 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26312367800000003 2023-10-09T06:17:31.080 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.080 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:31.080 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.080 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:83027 10.3 10:ce4b8369:::benchmark_data_smithi064_573750_object83026:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e514)", 2023-10-09T06:17:31.081 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.823657+0000", 2023-10-09T06:17:31.081 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 269.236666727, 2023-10-09T06:17:31.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27225728399999999, 2023-10-09T06:17:31.081 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:31.081 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:31.081 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:31.081 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:17:31.081 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:17:31.081 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 83027 2023-10-09T06:17:31.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.081 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:31.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.082 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:31.082 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.823657+0000", 2023-10-09T06:17:31.082 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.082 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.082 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.082 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:31.082 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.823657+0000", 2023-10-09T06:17:31.082 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.082 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.082 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.082 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:31.082 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.823656+0000", 2023-10-09T06:17:31.083 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:17:31.083 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.083 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.083 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:31.083 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.823681+0000", 2023-10-09T06:17:31.083 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4947999999999999e-05 2023-10-09T06:17:31.083 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.083 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.083 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:31.083 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.823690+0000", 2023-10-09T06:17:31.083 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0059999999999998e-06 2023-10-09T06:17:31.083 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.084 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.084 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:31.084 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.823696+0000", 2023-10-09T06:17:31.085 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1600000000000003e-06 2023-10-09T06:17:31.085 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.085 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.085 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:31.085 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824488+0000", 2023-10-09T06:17:31.085 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00079232200000000003 2023-10-09T06:17:31.085 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.085 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.085 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:31.085 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824646+0000", 2023-10-09T06:17:31.085 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015738899999999999 2023-10-09T06:17:31.085 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.086 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:17:31.086 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824798+0000", 2023-10-09T06:17:31.086 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015239999999999999 2023-10-09T06:17:31.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.086 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.086 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832263+0000", 2023-10-09T06:17:31.086 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0074651350000000003 2023-10-09T06:17:31.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.086 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832298+0000", 2023-10-09T06:17:31.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4591999999999997e-05 2023-10-09T06:17:31.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832327+0000", 2023-10-09T06:17:31.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8731999999999999e-05 2023-10-09T06:17:31.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:31.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095736+0000", 2023-10-09T06:17:31.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26340953700000003 2023-10-09T06:17:31.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:31.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095794+0000", 2023-10-09T06:17:31.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7886000000000001e-05 2023-10-09T06:17:31.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:31.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095915+0000", 2023-10-09T06:17:31.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012050200000000001 2023-10-09T06:17:31.088 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.088 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:31.089 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.089 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.089 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.089 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:83026 10.7 e514/490 10:e8d46d43:::benchmark_data_smithi064_573750_object83025:head v 514'5204, mlcod=514'5202)", 2023-10-09T06:17:31.089 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.824237+0000", 2023-10-09T06:17:31.089 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 269.23608723900003, 2023-10-09T06:17:31.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27149754799999998, 2023-10-09T06:17:31.089 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:31.089 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:31.089 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:31.089 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.089 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:31.089 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824237+0000", 2023-10-09T06:17:31.090 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.090 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.090 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.090 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:31.090 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824237+0000", 2023-10-09T06:17:31.090 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.090 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.090 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.090 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:31.090 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824236+0000", 2023-10-09T06:17:31.090 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:17:31.091 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.091 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.091 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:31.091 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824266+0000", 2023-10-09T06:17:31.091 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9991000000000001e-05 2023-10-09T06:17:31.091 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.091 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.091 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:31.091 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824275+0000", 2023-10-09T06:17:31.091 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7860000000000002e-06 2023-10-09T06:17:31.091 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.091 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.092 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:31.092 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824281+0000", 2023-10-09T06:17:31.092 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9020000000000004e-06 2023-10-09T06:17:31.092 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.092 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.092 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:31.092 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.826984+0000", 2023-10-09T06:17:31.092 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027032750000000002 2023-10-09T06:17:31.092 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.093 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.093 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:31.093 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.827009+0000", 2023-10-09T06:17:31.093 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4987e-05 2023-10-09T06:17:31.093 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.093 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.093 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:31.093 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095693+0000", 2023-10-09T06:17:31.093 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.268683799 2023-10-09T06:17:31.093 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.093 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.094 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:31.094 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095734+0000", 2023-10-09T06:17:31.094 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1600000000000002e-05 2023-10-09T06:17:31.094 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.094 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:31.094 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.094 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.094 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.094 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:83030 10.7 e514/490 10:e508533f:::benchmark_data_smithi064_573750_object83029:head v 514'5205, mlcod=514'5202)", 2023-10-09T06:17:31.094 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.831823+0000", 2023-10-09T06:17:31.094 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 269.228500811, 2023-10-09T06:17:31.094 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26393689799999998, 2023-10-09T06:17:31.095 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:31.095 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:31.095 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:31.095 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.095 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:31.095 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.831823+0000", 2023-10-09T06:17:31.095 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.095 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.095 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.095 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:31.095 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.831823+0000", 2023-10-09T06:17:31.095 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.096 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.096 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.096 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:31.096 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.831822+0000", 2023-10-09T06:17:31.096 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:17:31.096 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.096 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.096 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:31.096 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.831849+0000", 2023-10-09T06:17:31.096 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7291e-05 2023-10-09T06:17:31.096 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.096 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.097 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:31.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.831862+0000", 2023-10-09T06:17:31.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2138e-05 2023-10-09T06:17:31.097 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.097 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.097 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:31.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.831870+0000", 2023-10-09T06:17:31.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4020000000000002e-06 2023-10-09T06:17:31.097 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.097 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.097 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:31.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.831944+0000", 2023-10-09T06:17:31.098 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3606e-05 2023-10-09T06:17:31.098 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.098 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.098 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:31.098 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.831966+0000", 2023-10-09T06:17:31.098 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2504999999999999e-05 2023-10-09T06:17:31.098 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.098 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.098 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:31.098 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095739+0000", 2023-10-09T06:17:31.098 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.263772805 2023-10-09T06:17:31.099 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.099 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.099 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:31.099 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095760+0000", 2023-10-09T06:17:31.099 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1154000000000001e-05 2023-10-09T06:17:31.099 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.099 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:31.099 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.099 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.099 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.099 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:83038 10.9 e514/490 10:99b1b726:::benchmark_data_smithi064_573750_object83037:head v 514'5168, mlcod=513'5166)", 2023-10-09T06:17:31.099 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.834529+0000", 2023-10-09T06:17:31.099 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 269.22579465299998, 2023-10-09T06:17:31.100 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26413685199999998, 2023-10-09T06:17:31.100 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:31.100 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:31.100 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:31.100 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.100 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:31.100 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834529+0000", 2023-10-09T06:17:31.100 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.100 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.100 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.100 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:31.100 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834529+0000", 2023-10-09T06:17:31.101 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.101 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.101 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.101 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:31.101 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834529+0000", 2023-10-09T06:17:31.101 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:17:31.101 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.101 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.101 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:31.101 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834561+0000", 2023-10-09T06:17:31.101 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2317999999999998e-05 2023-10-09T06:17:31.101 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.102 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.102 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:31.102 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095659+0000", 2023-10-09T06:17:31.102 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26109802799999998 2023-10-09T06:17:31.102 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.102 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.102 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:31.102 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095671+0000", 2023-10-09T06:17:31.102 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2126e-05 2023-10-09T06:17:31.102 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.102 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.102 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:31.103 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095734+0000", 2023-10-09T06:17:31.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3114000000000006e-05 2023-10-09T06:17:31.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.103 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:31.103 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095761+0000", 2023-10-09T06:17:31.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7072000000000001e-05 2023-10-09T06:17:31.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.103 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:31.103 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.098631+0000", 2023-10-09T06:17:31.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002870147 2023-10-09T06:17:31.104 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.104 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.104 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:31.104 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.098666+0000", 2023-10-09T06:17:31.104 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4882999999999998e-05 2023-10-09T06:17:31.104 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.104 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:31.104 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.104 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.104 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.104 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPushReply(10.3 514/510 [PushReplyOp(10:c32ef31e:::benchmark_data_smithi064_573750_object32521:head)])", 2023-10-09T06:17:31.104 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:05.610248+0000", 2023-10-09T06:17:31.105 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 265.45007627699999, 2023-10-09T06:17:31.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29548134799999998, 2023-10-09T06:17:31.105 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:31.105 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:17:31.105 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:31.105 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.105 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:31.105 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.610248+0000", 2023-10-09T06:17:31.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.105 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.105 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.105 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:31.106 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.610248+0000", 2023-10-09T06:17:31.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:31.106 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.610247+0000", 2023-10-09T06:17:31.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:17:31.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:31.106 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.610251+0000", 2023-10-09T06:17:31.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2650000000000001e-06 2023-10-09T06:17:31.107 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.107 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:31.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.610258+0000", 2023-10-09T06:17:31.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.695e-06 2023-10-09T06:17:31.107 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.107 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:31.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.610274+0000", 2023-10-09T06:17:31.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5656000000000001e-05 2023-10-09T06:17:31.107 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.108 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:31.108 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.610296+0000", 2023-10-09T06:17:31.108 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1787999999999999e-05 2023-10-09T06:17:31.108 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.108 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.108 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:31.108 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.905729+0000", 2023-10-09T06:17:31.108 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29543320099999998 2023-10-09T06:17:31.108 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.108 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:31.108 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.109 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.109 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.109 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:85642 10.f e514/490 10:f2b65d04:::benchmark_data_smithi064_573750_object85641:head v 514'5293, mlcod=514'5291)", 2023-10-09T06:17:31.109 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:05.624785+0000", 2023-10-09T06:17:31.109 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 265.43553910600002, 2023-10-09T06:17:31.109 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29269213900000002, 2023-10-09T06:17:31.109 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:31.109 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:31.109 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:31.109 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.109 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:31.109 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.624785+0000", 2023-10-09T06:17:31.110 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.110 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:31.110 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.624785+0000", 2023-10-09T06:17:31.110 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.110 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:31.110 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.624784+0000", 2023-10-09T06:17:31.110 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:17:31.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.111 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:31.111 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.624814+0000", 2023-10-09T06:17:31.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0809999999999998e-05 2023-10-09T06:17:31.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.111 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:31.111 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.905640+0000", 2023-10-09T06:17:31.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28082586199999998 2023-10-09T06:17:31.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.112 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:31.112 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.905655+0000", 2023-10-09T06:17:31.112 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5089000000000001e-05 2023-10-09T06:17:31.112 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.114 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.114 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:31.115 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.906522+0000", 2023-10-09T06:17:31.115 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00086632699999999996 2023-10-09T06:17:31.115 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.115 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.115 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:31.115 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.906542+0000", 2023-10-09T06:17:31.115 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0621000000000001e-05 2023-10-09T06:17:31.115 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.115 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.116 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:31.116 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.917427+0000", 2023-10-09T06:17:31.116 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010885212 2023-10-09T06:17:31.116 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.116 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.116 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:31.116 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.917477+0000", 2023-10-09T06:17:31.116 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9546999999999998e-05 2023-10-09T06:17:31.116 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.117 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:31.117 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.117 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.118 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:86106 10.3 10:c9bcf3d5:::benchmark_data_smithi064_573750_object86105:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e515)", 2023-10-09T06:17:31.118 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:06.327720+0000", 2023-10-09T06:17:31.118 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 264.73260409800002, 2023-10-09T06:17:31.118 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18239918699999999, 2023-10-09T06:17:31.118 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:31.119 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:31.119 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:31.119 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:17:31.119 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:17:31.119 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 86106 2023-10-09T06:17:31.119 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.119 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:31.119 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.119 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:31.119 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.327720+0000", 2023-10-09T06:17:31.120 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.120 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:31.120 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.327720+0000", 2023-10-09T06:17:31.120 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.120 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:31.120 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.327718+0000", 2023-10-09T06:17:31.121 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:17:31.121 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.121 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:31.121 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.327749+0000", 2023-10-09T06:17:31.121 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0487000000000001e-05 2023-10-09T06:17:31.121 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.121 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:31.121 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.327763+0000", 2023-10-09T06:17:31.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4093e-05 2023-10-09T06:17:31.122 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.122 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.122 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:31.122 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.327772+0000", 2023-10-09T06:17:31.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1929999999999999e-06 2023-10-09T06:17:31.122 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.122 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.122 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:31.122 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.327970+0000", 2023-10-09T06:17:31.123 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019824600000000001 2023-10-09T06:17:31.123 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.123 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.123 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:31.123 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.328115+0000", 2023-10-09T06:17:31.123 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000144511 2023-10-09T06:17:31.123 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.123 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.123 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:17:31.123 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.328262+0000", 2023-10-09T06:17:31.123 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014681199999999999 2023-10-09T06:17:31.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:31.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.332137+0000", 2023-10-09T06:17:31.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038754219999999999 2023-10-09T06:17:31.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.332173+0000", 2023-10-09T06:17:31.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5808e-05 2023-10-09T06:17:31.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.334060+0000", 2023-10-09T06:17:31.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018872299999999999 2023-10-09T06:17:31.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.509949+0000", 2023-10-09T06:17:31.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.17588947499999999 2023-10-09T06:17:31.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:31.126 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.510005+0000", 2023-10-09T06:17:31.126 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5696999999999999e-05 2023-10-09T06:17:31.126 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.126 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.126 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:31.126 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.510119+0000", 2023-10-09T06:17:31.126 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000113927 2023-10-09T06:17:31.126 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.126 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:31.126 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.126 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.127 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.127 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:86114 10.3 10:ce2d7be7:::benchmark_data_smithi064_573750_object86113:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e515)", 2023-10-09T06:17:31.127 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:06.335134+0000", 2023-10-09T06:17:31.127 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 264.72518954899999, 2023-10-09T06:17:31.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18034898099999999, 2023-10-09T06:17:31.127 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:31.127 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:31.127 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:31.127 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:17:31.127 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:17:31.127 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 86114 2023-10-09T06:17:31.127 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.128 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:31.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.128 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:31.128 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.335134+0000", 2023-10-09T06:17:31.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.128 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:31.128 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.335134+0000", 2023-10-09T06:17:31.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:31.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.335133+0000", 2023-10-09T06:17:31.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:17:31.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:31.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.335155+0000", 2023-10-09T06:17:31.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1685e-05 2023-10-09T06:17:31.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:31.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.335164+0000", 2023-10-09T06:17:31.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5359999999999998e-06 2023-10-09T06:17:31.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:31.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.335170+0000", 2023-10-09T06:17:31.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9390000000000002e-06 2023-10-09T06:17:31.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:31.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.335383+0000", 2023-10-09T06:17:31.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00021336900000000001 2023-10-09T06:17:31.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:31.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.506890+0000", 2023-10-09T06:17:31.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.17150653399999999 2023-10-09T06:17:31.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:17:31.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.507101+0000", 2023-10-09T06:17:31.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00021099199999999999 2023-10-09T06:17:31.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:31.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.511984+0000", 2023-10-09T06:17:31.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048832479999999998 2023-10-09T06:17:31.132 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.512022+0000", 2023-10-09T06:17:31.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7540000000000003e-05 2023-10-09T06:17:31.132 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.512315+0000", 2023-10-09T06:17:31.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000293481 2023-10-09T06:17:31.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.133 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.515299+0000", 2023-10-09T06:17:31.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029837219999999999 2023-10-09T06:17:31.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.133 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:31.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.515362+0000", 2023-10-09T06:17:31.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2644999999999994e-05 2023-10-09T06:17:31.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:31.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.515483+0000", 2023-10-09T06:17:31.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012182500000000001 2023-10-09T06:17:31.134 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.134 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:31.134 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.134 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:92993 10.3 10:cbd0b40f:::benchmark_data_smithi064_573750_object92992:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-10-09T06:17:31.134 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:17.279671+0000", 2023-10-09T06:17:31.134 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 253.78065251199999, 2023-10-09T06:17:31.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20967518099999999, 2023-10-09T06:17:31.135 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:31.135 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:31.135 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:31.135 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:17:31.135 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:17:31.135 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 92993 2023-10-09T06:17:31.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.135 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:31.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:31.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.279671+0000", 2023-10-09T06:17:31.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:31.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.279671+0000", 2023-10-09T06:17:31.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:31.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.279670+0000", 2023-10-09T06:17:31.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:17:31.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:31.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.279696+0000", 2023-10-09T06:17:31.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6265999999999999e-05 2023-10-09T06:17:31.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:31.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.279708+0000", 2023-10-09T06:17:31.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1521e-05 2023-10-09T06:17:31.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:31.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.279716+0000", 2023-10-09T06:17:31.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0399999999999993e-06 2023-10-09T06:17:31.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:31.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.280201+0000", 2023-10-09T06:17:31.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00048531699999999997 2023-10-09T06:17:31.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:31.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.280360+0000", 2023-10-09T06:17:31.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015919400000000001 2023-10-09T06:17:31.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:17:31.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.280515+0000", 2023-10-09T06:17:31.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015432400000000001 2023-10-09T06:17:31.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.287237+0000", 2023-10-09T06:17:31.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067219150000000002 2023-10-09T06:17:31.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.287420+0000", 2023-10-09T06:17:31.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018320200000000001 2023-10-09T06:17:31.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:31.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.289348+0000", 2023-10-09T06:17:31.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001928206 2023-10-09T06:17:31.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.489167+0000", 2023-10-09T06:17:31.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.19981912800000001 2023-10-09T06:17:31.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:31.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.489228+0000", 2023-10-09T06:17:31.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0606999999999998e-05 2023-10-09T06:17:31.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:31.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.489347+0000", 2023-10-09T06:17:31.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118967 2023-10-09T06:17:31.142 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.142 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:31.142 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.142 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.142 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:93019 10.3 10:c9254536:::benchmark_data_smithi064_573750_object93018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-10-09T06:17:31.142 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:17.310056+0000", 2023-10-09T06:17:31.142 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 253.75026837799999, 2023-10-09T06:17:31.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.183025682, 2023-10-09T06:17:31.142 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:31.142 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:31.142 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:31.143 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:17:31.143 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:17:31.143 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 93019 2023-10-09T06:17:31.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.143 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:31.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.143 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:31.143 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310056+0000", 2023-10-09T06:17:31.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.143 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:31.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310056+0000", 2023-10-09T06:17:31.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:31.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310054+0000", 2023-10-09T06:17:31.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:17:31.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:31.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310075+0000", 2023-10-09T06:17:31.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0990999999999999e-05 2023-10-09T06:17:31.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:31.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310090+0000", 2023-10-09T06:17:31.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4449e-05 2023-10-09T06:17:31.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:31.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310112+0000", 2023-10-09T06:17:31.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2952999999999999e-05 2023-10-09T06:17:31.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:31.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310155+0000", 2023-10-09T06:17:31.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2534000000000003e-05 2023-10-09T06:17:31.146 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.146 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.146 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:31.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310286+0000", 2023-10-09T06:17:31.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013121699999999999 2023-10-09T06:17:31.146 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.146 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.146 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:17:31.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310404+0000", 2023-10-09T06:17:31.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011796899999999999 2023-10-09T06:17:31.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:31.147 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.312787+0000", 2023-10-09T06:17:31.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0023830919999999998 2023-10-09T06:17:31.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.147 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.313853+0000", 2023-10-09T06:17:31.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001066132 2023-10-09T06:17:31.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.313932+0000", 2023-10-09T06:17:31.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8236000000000001e-05 2023-10-09T06:17:31.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.492876+0000", 2023-10-09T06:17:31.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.17894394499999999 2023-10-09T06:17:31.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:31.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.492941+0000", 2023-10-09T06:17:31.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5642999999999995e-05 2023-10-09T06:17:31.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.149 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:31.149 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.493081+0000", 2023-10-09T06:17:31.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014008799999999999 2023-10-09T06:17:31.149 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.149 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:31.149 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.149 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:94743 10.3 10:c7ba2f96:::benchmark_data_smithi064_573750_object94742:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-10-09T06:17:31.149 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:19.875917+0000", 2023-10-09T06:17:31.149 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 251.18440685100001, 2023-10-09T06:17:31.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.22279960200000001, 2023-10-09T06:17:31.150 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:31.150 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:31.150 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:31.150 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:17:31.150 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:17:31.150 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 94743 2023-10-09T06:17:31.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.150 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:31.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:31.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.875917+0000", 2023-10-09T06:17:31.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:31.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.875917+0000", 2023-10-09T06:17:31.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:31.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.875916+0000", 2023-10-09T06:17:31.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:17:31.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:31.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.875940+0000", 2023-10-09T06:17:31.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4561000000000001e-05 2023-10-09T06:17:31.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:31.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.875953+0000", 2023-10-09T06:17:31.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3276e-05 2023-10-09T06:17:31.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.153 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:31.153 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.875962+0000", 2023-10-09T06:17:31.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1340000000000004e-06 2023-10-09T06:17:31.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.153 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.153 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:31.153 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.876240+0000", 2023-10-09T06:17:31.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00027892999999999999 2023-10-09T06:17:31.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.153 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.153 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:31.153 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.876436+0000", 2023-10-09T06:17:31.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000195317 2023-10-09T06:17:31.154 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:17:31.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.876653+0000", 2023-10-09T06:17:31.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000216918 2023-10-09T06:17:31.154 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.881334+0000", 2023-10-09T06:17:31.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046809870000000002 2023-10-09T06:17:31.154 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:31.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.881349+0000", 2023-10-09T06:17:31.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5311999999999998e-05 2023-10-09T06:17:31.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.884746+0000", 2023-10-09T06:17:31.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033974000000000001 2023-10-09T06:17:31.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.098510+0000", 2023-10-09T06:17:31.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.213763589 2023-10-09T06:17:31.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:31.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.098578+0000", 2023-10-09T06:17:31.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8112000000000001e-05 2023-10-09T06:17:31.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:31.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.098717+0000", 2023-10-09T06:17:31.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013864199999999999 2023-10-09T06:17:31.157 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.157 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:31.157 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.157 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.157 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:95775 10.3 10:c599e099:::benchmark_data_smithi064_573750_object95774:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-10-09T06:17:31.157 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:23.554330+0000", 2023-10-09T06:17:31.157 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 247.50599442500001, 2023-10-09T06:17:31.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26772326899999999, 2023-10-09T06:17:31.157 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:31.157 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:31.157 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:31.157 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:17:31.158 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:17:31.158 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 95775 2023-10-09T06:17:31.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.158 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:31.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:31.158 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.554330+0000", 2023-10-09T06:17:31.158 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:31.158 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.554330+0000", 2023-10-09T06:17:31.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.159 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.159 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:31.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.554328+0000", 2023-10-09T06:17:31.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:17:31.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.159 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.159 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:31.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.554352+0000", 2023-10-09T06:17:31.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4216999999999999e-05 2023-10-09T06:17:31.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:31.160 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.554365+0000", 2023-10-09T06:17:31.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2947e-05 2023-10-09T06:17:31.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:31.160 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.554373+0000", 2023-10-09T06:17:31.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2120000000000003e-06 2023-10-09T06:17:31.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:31.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.557893+0000", 2023-10-09T06:17:31.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035198059999999999 2023-10-09T06:17:31.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.161 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:31.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.558059+0000", 2023-10-09T06:17:31.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000165971 2023-10-09T06:17:31.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.161 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:17:31.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.558259+0000", 2023-10-09T06:17:31.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020019300000000001 2023-10-09T06:17:31.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.162 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.561632+0000", 2023-10-09T06:17:31.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033720709999999999 2023-10-09T06:17:31.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:31.162 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.561647+0000", 2023-10-09T06:17:31.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5965e-05 2023-10-09T06:17:31.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821847+0000", 2023-10-09T06:17:31.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26019937599999998 2023-10-09T06:17:31.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:31.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821877+0000", 2023-10-09T06:17:31.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9989e-05 2023-10-09T06:17:31.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:31.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821916+0000", 2023-10-09T06:17:31.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8781e-05 2023-10-09T06:17:31.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:31.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.822053+0000", 2023-10-09T06:17:31.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013723500000000001 2023-10-09T06:17:31.164 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.164 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:31.164 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.164 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPushReply(10.3 523/510 [PushReplyOp(10:cc80a1b0:::benchmark_data_smithi064_573750_object75768:head)])", 2023-10-09T06:17:31.164 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:23.562483+0000", 2023-10-09T06:17:31.165 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 247.49784142499999, 2023-10-09T06:17:31.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25932817600000002, 2023-10-09T06:17:31.165 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:31.165 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:17:31.165 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:31.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:31.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562483+0000", 2023-10-09T06:17:31.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:31.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562483+0000", 2023-10-09T06:17:31.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:31.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562482+0000", 2023-10-09T06:17:31.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:17:31.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:31.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562488+0000", 2023-10-09T06:17:31.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6699999999999997e-06 2023-10-09T06:17:31.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.167 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:31.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562501+0000", 2023-10-09T06:17:31.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3005e-05 2023-10-09T06:17:31.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.167 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:31.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562511+0000", 2023-10-09T06:17:31.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6679999999999996e-06 2023-10-09T06:17:31.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:31.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562563+0000", 2023-10-09T06:17:31.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1659000000000001e-05 2023-10-09T06:17:31.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:31.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821811+0000", 2023-10-09T06:17:31.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25924797700000002 2023-10-09T06:17:31.168 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.168 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:31.168 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.169 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.46283.0:95775 10.3 e523/510 ondisk, result = 0)", 2023-10-09T06:17:31.169 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:23.562660+0000", 2023-10-09T06:17:31.169 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 247.497663719, 2023-10-09T06:17:31.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25919107400000002, 2023-10-09T06:17:31.169 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:31.169 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:17:31.169 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:31.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:31.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562660+0000", 2023-10-09T06:17:31.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:31.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562660+0000", 2023-10-09T06:17:31.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:31.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562660+0000", 2023-10-09T06:17:31.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:17:31.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:31.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562665+0000", 2023-10-09T06:17:31.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.682e-06 2023-10-09T06:17:31.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:31.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562675+0000", 2023-10-09T06:17:31.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7510000000000003e-06 2023-10-09T06:17:31.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:31.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562682+0000", 2023-10-09T06:17:31.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.391e-06 2023-10-09T06:17:31.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:31.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821834+0000", 2023-10-09T06:17:31.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25915209099999997 2023-10-09T06:17:31.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:31.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821842+0000", 2023-10-09T06:17:31.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4719999999999998e-06 2023-10-09T06:17:31.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:31.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821851+0000", 2023-10-09T06:17:31.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4269999999999997e-06 2023-10-09T06:17:31.173 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.173 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:31.173 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.173 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.46283.0:95775 10.3 e523/510 ondisk, result = 0)", 2023-10-09T06:17:31.173 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:23.563416+0000", 2023-10-09T06:17:31.174 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 247.49690779900001, 2023-10-09T06:17:31.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25864464399999998, 2023-10-09T06:17:31.174 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:31.174 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:17:31.174 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:31.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:31.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.563416+0000", 2023-10-09T06:17:31.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:31.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.563416+0000", 2023-10-09T06:17:31.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:31.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.563416+0000", 2023-10-09T06:17:31.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:17:31.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:31.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.563421+0000", 2023-10-09T06:17:31.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1250000000000001e-06 2023-10-09T06:17:31.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:31.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821790+0000", 2023-10-09T06:17:31.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25836899800000002 2023-10-09T06:17:31.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:31.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821803+0000", 2023-10-09T06:17:31.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.326e-05 2023-10-09T06:17:31.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:31.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821867+0000", 2023-10-09T06:17:31.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3613000000000002e-05 2023-10-09T06:17:31.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:31.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821873+0000", 2023-10-09T06:17:31.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2090000000000002e-06 2023-10-09T06:17:31.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:31.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.822061+0000", 2023-10-09T06:17:31.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018808799999999999 2023-10-09T06:17:31.178 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.178 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:31.178 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.178 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:117026 10.f e536/490 10:f52e132e:::benchmark_data_smithi064_573750_object20811:head v 536'7258, mlcod=536'7256)", 2023-10-09T06:17:31.178 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:57.908348+0000", 2023-10-09T06:17:31.178 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 213.15197647100001, 2023-10-09T06:17:31.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.213648951, 2023-10-09T06:17:31.178 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:31.178 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:31.179 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:31.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:31.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908348+0000", 2023-10-09T06:17:31.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:31.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908348+0000", 2023-10-09T06:17:31.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:31.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908347+0000", 2023-10-09T06:17:31.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:17:31.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:31.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908353+0000", 2023-10-09T06:17:31.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7190000000000004e-06 2023-10-09T06:17:31.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:31.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908365+0000", 2023-10-09T06:17:31.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1433999999999999e-05 2023-10-09T06:17:31.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:31.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908372+0000", 2023-10-09T06:17:31.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4540000000000001e-06 2023-10-09T06:17:31.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:31.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908494+0000", 2023-10-09T06:17:31.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000121434 2023-10-09T06:17:31.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:31.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908513+0000", 2023-10-09T06:17:31.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9556999999999999e-05 2023-10-09T06:17:31.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:31.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.121931+0000", 2023-10-09T06:17:31.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21341768699999999 2023-10-09T06:17:31.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:31.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.121996+0000", 2023-10-09T06:17:31.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5518000000000005e-05 2023-10-09T06:17:31.183 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.183 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:31.183 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.183 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:117014 10.5 e536/490 10:a26da6a0:::benchmark_data_smithi064_573750_object20799:head v 536'7222, mlcod=536'7220)", 2023-10-09T06:17:31.183 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:57.908415+0000", 2023-10-09T06:17:31.183 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 213.15190909099999, 2023-10-09T06:17:31.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21368757199999999, 2023-10-09T06:17:31.184 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:31.184 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:31.184 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:31.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:31.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908415+0000", 2023-10-09T06:17:31.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:31.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908415+0000", 2023-10-09T06:17:31.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:31.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908415+0000", 2023-10-09T06:17:31.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:17:31.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:31.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908419+0000", 2023-10-09T06:17:31.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6480000000000002e-06 2023-10-09T06:17:31.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:31.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908427+0000", 2023-10-09T06:17:31.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6469999999999998e-06 2023-10-09T06:17:31.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:31.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908432+0000", 2023-10-09T06:17:31.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0239999999999997e-06 2023-10-09T06:17:31.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:31.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908476+0000", 2023-10-09T06:17:31.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4288e-05 2023-10-09T06:17:31.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:31.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908497+0000", 2023-10-09T06:17:31.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1121999999999999e-05 2023-10-09T06:17:31.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:31.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.122071+0000", 2023-10-09T06:17:31.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21357372499999999 2023-10-09T06:17:31.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:31.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.122102+0000", 2023-10-09T06:17:31.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1529000000000003e-05 2023-10-09T06:17:31.188 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.188 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:31.188 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.188 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:117020 10.5 e536/490 10:a87652b9:::benchmark_data_smithi064_573750_object20805:head v 536'7223, mlcod=536'7220)", 2023-10-09T06:17:31.188 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:57.910671+0000", 2023-10-09T06:17:31.188 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 213.149653363, 2023-10-09T06:17:31.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.22155350400000001, 2023-10-09T06:17:31.189 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:31.189 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:31.189 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:31.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:31.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.910671+0000", 2023-10-09T06:17:31.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:31.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.910671+0000", 2023-10-09T06:17:31.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:31.190 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:31.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.910670+0000", 2023-10-09T06:17:31.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:17:31.190 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:31.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.910676+0000", 2023-10-09T06:17:31.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9460000000000003e-06 2023-10-09T06:17:31.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:31.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.121961+0000", 2023-10-09T06:17:31.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21128570499999999 2023-10-09T06:17:31.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:31.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.121974+0000", 2023-10-09T06:17:31.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2244e-05 2023-10-09T06:17:31.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:31.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.122133+0000", 2023-10-09T06:17:31.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015961999999999999 2023-10-09T06:17:31.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:31.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.122152+0000", 2023-10-09T06:17:31.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9131e-05 2023-10-09T06:17:31.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:31.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.132185+0000", 2023-10-09T06:17:31.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010032961 2023-10-09T06:17:31.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:31.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:31.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:31.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.132224+0000", 2023-10-09T06:17:31.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8819999999999998e-05 2023-10-09T06:17:31.193 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.193 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:31.193 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.193 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:31.193 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:31.193 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:17:31.194 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:31.204 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:17:31.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:31.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:31.321 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:31.392 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:31.472 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:17:31.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:31.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:31.581 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:31.651 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:31.797 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:17:31.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:31.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:31.901 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:31.960 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:32.061 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 82 16 102341 102325 77.9685 51.375 0.0118904 0.0128213 2023-10-09T06:17:32.279 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:17:32.327 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:17:32.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:32.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:32.433 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:32.459 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:32.500 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:17:32.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:32.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:32.530 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:17:32.595 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:32 smithi064 ceph-mon[212750]: pgmap v1521: 119 pgs: 119 active+clean; 6.1 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 51 MiB/s wr, 820 op/s 2023-10-09T06:17:32.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:32 smithi064 ceph-mon[220044]: pgmap v1521: 119 pgs: 119 active+clean; 6.1 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 51 MiB/s wr, 820 op/s 2023-10-09T06:17:32.605 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:32.691 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:32.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:32 smithi156 ceph-mon[195101]: pgmap v1521: 119 pgs: 119 active+clean; 6.1 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 51 MiB/s wr, 820 op/s 2023-10-09T06:17:32.924 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:17:32.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:32.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:33.052 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:33.064 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 83 16 103120 103104 77.6157 48.6875 0.014689 0.0128783 2023-10-09T06:17:33.086 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:33.117 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:33.147 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:33.246 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:17:33.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:33.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:33.355 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:33.394 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:33.494 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:17:33.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:33.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:33.612 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:33.676 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:33.745 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:17:33.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:33.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:33.787 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:17:33.829 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:17:33.855 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:33.868 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:17:33.903 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:17:33.942 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:17:33.954 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:33 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/940807936' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:17:33.954 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:33 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/940807936' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:17:33.956 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:33.983 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:17:34.021 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:17:34.061 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 84 16 103989 103973 77.3383 54.3125 0.00917599 0.0129258 2023-10-09T06:17:34.066 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:17:34.085 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:17:34.085 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:17:34.085 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:17:34.085 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:17:34.085 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:17:34.085 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:17:34.085 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:17:34.085 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:17:34.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:33 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/940807936' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:17:34.306 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:34.517 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:17:34.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:34.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:34.621 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:34.690 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:34.807 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:34 smithi064 ceph-mon[220044]: pgmap v1522: 119 pgs: 119 active+clean; 6.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 51 MiB/s wr, 822 op/s 2023-10-09T06:17:35.061 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 85 16 104633 104617 76.902 40.25 0.0176406 0.0129989 2023-10-09T06:17:35.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:34 smithi156 ceph-mon[195101]: pgmap v1522: 119 pgs: 119 active+clean; 6.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 51 MiB/s wr, 822 op/s 2023-10-09T06:17:35.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:34 smithi064 ceph-mon[212750]: pgmap v1522: 119 pgs: 119 active+clean; 6.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 51 MiB/s wr, 822 op/s 2023-10-09T06:17:35.554 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:17:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:17:35] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.43.0" 2023-10-09T06:17:36.061 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 86 16 105533 105517 76.6618 56.25 0.0136592 0.0130397 2023-10-09T06:17:36.299 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:17:36.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:36.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:36.414 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:36.494 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:36.903 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:17:36.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:36.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:37.011 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:37.038 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:37.061 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 87 15 106370 106355 76.3827 52.375 0.017316 0.0130863 2023-10-09T06:17:37.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:36 smithi156 ceph-mon[195101]: pgmap v1523: 119 pgs: 119 active+clean; 6.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 51 MiB/s wr, 814 op/s 2023-10-09T06:17:37.143 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:17:37.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:37.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:37.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:36 smithi064 ceph-mon[212750]: pgmap v1523: 119 pgs: 119 active+clean; 6.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 51 MiB/s wr, 814 op/s 2023-10-09T06:17:37.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:36 smithi064 ceph-mon[220044]: pgmap v1523: 119 pgs: 119 active+clean; 6.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 51 MiB/s wr, 814 op/s 2023-10-09T06:17:37.258 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:37.325 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:37.832 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:17:37.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:37.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:37.941 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:38.013 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:17:38.013 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:17:38.052 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:38.061 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 88 16 107303 107287 76.1766 58.25 0.0249853 0.0131226 2023-10-09T06:17:38.230 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:17:38.231 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:17:38.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:38.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:38.240 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T06:17:38.260 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:17:38.338 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:38.384 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:38.463 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:17:38.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:38.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:38.582 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:38.653 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:38.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:38 smithi064 ceph-mon[212750]: pgmap v1524: 119 pgs: 119 active+clean; 6.4 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 50 MiB/s wr, 804 op/s 2023-10-09T06:17:38.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:38 smithi064 ceph-mon[220044]: pgmap v1524: 119 pgs: 119 active+clean; 6.4 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 50 MiB/s wr, 804 op/s 2023-10-09T06:17:39.061 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 89 16 108144 108128 75.9113 52.5625 0.0229846 0.0131677 2023-10-09T06:17:39.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:38 smithi156 ceph-mon[195101]: pgmap v1524: 119 pgs: 119 active+clean; 6.4 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 50 MiB/s wr, 804 op/s 2023-10-09T06:17:39.712 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1524,"stamp":"2023-10-09T06:17:37.273211+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6899439520,"num_objects":105292,"num_object_clones":0,"num_object_copies":315876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":105292,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":105329,"num_write_kb":6737181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":1718414,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95914,"ondisk_log_size":95914,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":25651692,"kb_used_data":20180468,"kb_used_omap":84,"kb_used_meta":5471083,"kb_avail":630564372,"statfs":{"total":671965249536,"available":645697916928,"internally_reserved":0,"allocated":20664799232,"data_stored":20648705576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":86161,"internal_metadata":5602389871},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,1,1,0,0,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":527499264,"num_objects":8049,"num_object_clones":0,"num_object_copies":24147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8049,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8049,"num_write_kb":515136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003255"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":855,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.390012+0000","last_change":"2023-10-09T06:15:19.880315+0000","last_active":"2023-10-09T06:17:20.390012+0000","last_peered":"2023-10-09T06:17:20.390012+0000","last_clean":"2023-10-09T06:17:20.390012+0000","last_became_active":"2023-10-09T06:15:19.879857+0000","last_became_peered":"2023-10-09T06:15:19.879857+0000","last_unstale":"2023-10-09T06:17:20.390012+0000","last_undegraded":"2023-10-09T06:17:20.390012+0000","last_fullsized":"2023-10-09T06:17:20.390012+0000","mapping_epoch":561,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":562,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:05.412941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:05.412941+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:54.322404+0000","scrub_duration":0.0061014329999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":670,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.403531+0000","last_change":"2023-10-09T06:16:02.748447+0000","last_active":"2023-10-09T06:17:20.403531+0000","last_peered":"2023-10-09T06:17:20.403531+0000","last_clean":"2023-10-09T06:17:20.403531+0000","last_became_active":"2023-10-09T06:16:02.747672+0000","last_became_peered":"2023-10-09T06:16:02.747672+0000","last_unstale":"2023-10-09T06:17:20.403531+0000","last_undegraded":"2023-10-09T06:17:20.403531+0000","last_fullsized":"2023-10-09T06:17:20.403531+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:31.326263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":404,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:25.856731+0000","last_change":"2023-10-09T06:05:11.451972+0000","last_active":"2023-10-09T06:17:25.856731+0000","last_peered":"2023-10-09T06:17:25.856731+0000","last_clean":"2023-10-09T06:17:25.856731+0000","last_became_active":"2023-10-09T06:05:11.451448+0000","last_became_peered":"2023-10-09T06:05:11.451448+0000","last_unstale":"2023-10-09T06:17:25.856731+0000","last_undegraded":"2023-10-09T06:17:25.856731+0000","last_fullsized":"2023-10-09T06:17:25.856731+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:39.183621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":891,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.403784+0000","last_change":"2023-10-09T06:09:27.554162+0000","last_active":"2023-10-09T06:17:20.403784+0000","last_peered":"2023-10-09T06:17:20.403784+0000","last_clean":"2023-10-09T06:17:20.403784+0000","last_became_active":"2023-10-09T06:09:27.553659+0000","last_became_peered":"2023-10-09T06:09:27.553659+0000","last_unstale":"2023-10-09T06:17:20.403784+0000","last_undegraded":"2023-10-09T06:17:20.403784+0000","last_fullsized":"2023-10-09T06:17:20.403784+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:01.468346+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:01.468346+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:03.443511+0000","scrub_duration":0.0016258220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":275,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.398683+0000","last_change":"2023-10-09T06:01:54.762060+0000","last_active":"2023-10-09T06:17:20.398683+0000","last_peered":"2023-10-09T06:17:20.398683+0000","last_clean":"2023-10-09T06:17:20.398683+0000","last_became_active":"2023-10-09T06:01:54.761757+0000","last_became_peered":"2023-10-09T06:01:54.761757+0000","last_unstale":"2023-10-09T06:17:20.398683+0000","last_undegraded":"2023-10-09T06:17:20.398683+0000","last_fullsized":"2023-10-09T06:17:20.398683+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:40.294232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":75,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.390461+0000","last_change":"2023-10-09T06:13:30.789834+0000","last_active":"2023-10-09T06:17:20.390461+0000","last_peered":"2023-10-09T06:17:20.390461+0000","last_clean":"2023-10-09T06:17:20.390461+0000","last_became_active":"2023-10-09T06:13:30.789517+0000","last_became_peered":"2023-10-09T06:13:30.789517+0000","last_unstale":"2023-10-09T06:17:20.390461+0000","last_undegraded":"2023-10-09T06:17:20.390461+0000","last_fullsized":"2023-10-09T06:17:20.390461+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:45.467922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":876,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.403339+0000","last_change":"2023-10-09T06:12:07.739121+0000","last_active":"2023-10-09T06:17:20.403339+0000","last_peered":"2023-10-09T06:17:20.403339+0000","last_clean":"2023-10-09T06:17:20.403339+0000","last_became_active":"2023-10-09T06:12:07.738739+0000","last_became_peered":"2023-10-09T06:12:07.738739+0000","last_unstale":"2023-10-09T06:17:20.403339+0000","last_undegraded":"2023-10-09T06:17:20.403339+0000","last_fullsized":"2023-10-09T06:17:20.403339+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.293426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.293426+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:19.815422+0000","scrub_duration":0.001587946,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":386,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.388759+0000","last_change":"2023-10-09T06:09:27.564377+0000","last_active":"2023-10-09T06:17:20.388759+0000","last_peered":"2023-10-09T06:17:20.388759+0000","last_clean":"2023-10-09T06:17:20.388759+0000","last_became_active":"2023-10-09T06:09:27.563710+0000","last_became_peered":"2023-10-09T06:09:27.563710+0000","last_unstale":"2023-10-09T06:17:20.388759+0000","last_undegraded":"2023-10-09T06:17:20.388759+0000","last_fullsized":"2023-10-09T06:17:20.388759+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:48.168447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":30,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.407663+0000","last_change":"2023-10-09T06:16:02.754245+0000","last_active":"2023-10-09T06:17:20.407663+0000","last_peered":"2023-10-09T06:17:20.407663+0000","last_clean":"2023-10-09T06:17:20.407663+0000","last_became_active":"2023-10-09T06:16:02.753886+0000","last_became_peered":"2023-10-09T06:16:02.753886+0000","last_unstale":"2023-10-09T06:17:20.407663+0000","last_undegraded":"2023-10-09T06:17:20.407663+0000","last_fullsized":"2023-10-09T06:17:20.407663+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:32.708190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":810,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.391049+0000","last_change":"2023-10-09T06:13:33.701687+0000","last_active":"2023-10-09T06:17:20.391049+0000","last_peered":"2023-10-09T06:17:20.391049+0000","last_clean":"2023-10-09T06:17:20.391049+0000","last_became_active":"2023-10-09T06:13:33.701233+0000","last_became_peered":"2023-10-09T06:13:33.701233+0000","last_unstale":"2023-10-09T06:17:20.391049+0000","last_undegraded":"2023-10-09T06:17:20.391049+0000","last_fullsized":"2023-10-09T06:17:20.391049+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.384407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.384407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:04.887826+0000","scrub_duration":0.0040651960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":491,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.394791+0000","last_change":"2023-10-09T06:09:27.559756+0000","last_active":"2023-10-09T06:17:20.394791+0000","last_peered":"2023-10-09T06:17:20.394791+0000","last_clean":"2023-10-09T06:17:20.394791+0000","last_became_active":"2023-10-09T06:09:27.559443+0000","last_became_peered":"2023-10-09T06:09:27.559443+0000","last_unstale":"2023-10-09T06:17:20.394791+0000","last_undegraded":"2023-10-09T06:17:20.394791+0000","last_fullsized":"2023-10-09T06:17:20.394791+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:14.519398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":422,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.388903+0000","last_change":"2023-10-09T06:13:21.737897+0000","last_active":"2023-10-09T06:17:20.388903+0000","last_peered":"2023-10-09T06:17:20.388903+0000","last_clean":"2023-10-09T06:17:20.388903+0000","last_became_active":"2023-10-09T06:13:21.737564+0000","last_became_peered":"2023-10-09T06:13:21.737564+0000","last_unstale":"2023-10-09T06:17:20.388903+0000","last_undegraded":"2023-10-09T06:17:20.388903+0000","last_fullsized":"2023-10-09T06:17:20.388903+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:34.863320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":736,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.390909+0000","last_change":"2023-10-09T06:16:05.761750+0000","last_active":"2023-10-09T06:17:20.390909+0000","last_peered":"2023-10-09T06:17:20.390909+0000","last_clean":"2023-10-09T06:17:20.390909+0000","last_became_active":"2023-10-09T06:16:05.761436+0000","last_became_peered":"2023-10-09T06:16:05.761436+0000","last_unstale":"2023-10-09T06:17:20.390909+0000","last_undegraded":"2023-10-09T06:17:20.390909+0000","last_fullsized":"2023-10-09T06:17:20.390909+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:20.742561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:20.742561+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:13.021950+0000","scrub_duration":0.0027005039999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":75,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.390373+0000","last_change":"2023-10-09T06:13:30.802749+0000","last_active":"2023-10-09T06:17:20.390373+0000","last_peered":"2023-10-09T06:17:20.390373+0000","last_clean":"2023-10-09T06:17:20.390373+0000","last_became_active":"2023-10-09T06:13:30.802449+0000","last_became_peered":"2023-10-09T06:13:30.802449+0000","last_unstale":"2023-10-09T06:17:20.390373+0000","last_undegraded":"2023-10-09T06:17:20.390373+0000","last_fullsized":"2023-10-09T06:17:20.390373+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:29.017181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":580,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.855797+0000","last_change":"2023-10-09T06:13:30.792445+0000","last_active":"2023-10-09T06:17:20.855797+0000","last_peered":"2023-10-09T06:17:20.855797+0000","last_clean":"2023-10-09T06:17:20.855797+0000","last_became_active":"2023-10-09T06:13:30.792116+0000","last_became_peered":"2023-10-09T06:13:30.792116+0000","last_unstale":"2023-10-09T06:17:20.855797+0000","last_undegraded":"2023-10-09T06:17:20.855797+0000","last_fullsized":"2023-10-09T06:17:20.855797+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:20:53.506293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":803,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.381368+0000","last_change":"2023-10-09T06:13:24.760843+0000","last_active":"2023-10-09T06:17:20.381368+0000","last_peered":"2023-10-09T06:17:20.381368+0000","last_clean":"2023-10-09T06:17:20.381368+0000","last_became_active":"2023-10-09T06:13:24.760333+0000","last_became_peered":"2023-10-09T06:13:24.760333+0000","last_unstale":"2023-10-09T06:17:20.381368+0000","last_undegraded":"2023-10-09T06:17:20.381368+0000","last_fullsized":"2023-10-09T06:17:20.381368+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:38.588641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:38.588641+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:30.660400+0000","scrub_duration":0.010116385,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":81,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.390965+0000","last_change":"2023-10-09T06:13:30.826588+0000","last_active":"2023-10-09T06:17:20.390965+0000","last_peered":"2023-10-09T06:17:20.390965+0000","last_clean":"2023-10-09T06:17:20.390965+0000","last_became_active":"2023-10-09T06:13:30.799199+0000","last_became_peered":"2023-10-09T06:13:30.799199+0000","last_unstale":"2023-10-09T06:17:20.390965+0000","last_undegraded":"2023-10-09T06:17:20.390965+0000","last_fullsized":"2023-10-09T06:17:20.390965+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:38.703864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":30,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.380633+0000","last_change":"2023-10-09T06:16:02.759286+0000","last_active":"2023-10-09T06:17:20.380633+0000","last_peered":"2023-10-09T06:17:20.380633+0000","last_clean":"2023-10-09T06:17:20.380633+0000","last_became_active":"2023-10-09T06:16:02.758956+0000","last_became_peered":"2023-10-09T06:16:02.758956+0000","last_unstale":"2023-10-09T06:17:20.380633+0000","last_undegraded":"2023-10-09T06:17:20.380633+0000","last_fullsized":"2023-10-09T06:17:20.380633+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:48.403200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":765,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.400761+0000","last_change":"2023-10-09T06:09:27.574336+0000","last_active":"2023-10-09T06:17:20.400761+0000","last_peered":"2023-10-09T06:17:20.400761+0000","last_clean":"2023-10-09T06:17:20.400761+0000","last_became_active":"2023-10-09T06:09:27.573974+0000","last_became_peered":"2023-10-09T06:09:27.573974+0000","last_unstale":"2023-10-09T06:17:20.400761+0000","last_undegraded":"2023-10-09T06:17:20.400761+0000","last_fullsized":"2023-10-09T06:17:20.400761+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.339121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.339121+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:45.908579+0000","scrub_duration":0.010474344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":626,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.403576+0000","last_change":"2023-10-09T06:13:30.812598+0000","last_active":"2023-10-09T06:17:20.403576+0000","last_peered":"2023-10-09T06:17:20.403576+0000","last_clean":"2023-10-09T06:17:20.403576+0000","last_became_active":"2023-10-09T06:13:30.791460+0000","last_became_peered":"2023-10-09T06:13:30.791460+0000","last_unstale":"2023-10-09T06:17:20.403576+0000","last_undegraded":"2023-10-09T06:17:20.403576+0000","last_fullsized":"2023-10-09T06:17:20.403576+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:28.046730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":604,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.403831+0000","last_change":"2023-10-09T06:01:54.764916+0000","last_active":"2023-10-09T06:17:20.403831+0000","last_peered":"2023-10-09T06:17:20.403831+0000","last_clean":"2023-10-09T06:17:20.403831+0000","last_became_active":"2023-10-09T06:01:54.764705+0000","last_became_peered":"2023-10-09T06:01:54.764705+0000","last_unstale":"2023-10-09T06:17:20.403831+0000","last_undegraded":"2023-10-09T06:17:20.403831+0000","last_fullsized":"2023-10-09T06:17:20.403831+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:12.527719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":821,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.394839+0000","last_change":"2023-10-09T06:13:15.425525+0000","last_active":"2023-10-09T06:17:20.394839+0000","last_peered":"2023-10-09T06:17:20.394839+0000","last_clean":"2023-10-09T06:17:20.394839+0000","last_became_active":"2023-10-09T06:13:15.425086+0000","last_became_peered":"2023-10-09T06:13:15.425086+0000","last_unstale":"2023-10-09T06:17:20.394839+0000","last_undegraded":"2023-10-09T06:17:20.394839+0000","last_fullsized":"2023-10-09T06:17:20.394839+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:42.900876+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:42.900876+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:46.042715+0000","scrub_duration":0.0014928649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":97,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.391086+0000","last_change":"2023-10-09T06:16:02.766996+0000","last_active":"2023-10-09T06:17:20.391086+0000","last_peered":"2023-10-09T06:17:20.391086+0000","last_clean":"2023-10-09T06:17:20.391086+0000","last_became_active":"2023-10-09T06:16:02.760831+0000","last_became_peered":"2023-10-09T06:16:02.760831+0000","last_unstale":"2023-10-09T06:17:20.391086+0000","last_undegraded":"2023-10-09T06:17:20.391086+0000","last_fullsized":"2023-10-09T06:17:20.391086+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:05.917714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":639,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:25.856690+0000","last_change":"2023-10-09T06:01:54.761558+0000","last_active":"2023-10-09T06:17:25.856690+0000","last_peered":"2023-10-09T06:17:25.856690+0000","last_clean":"2023-10-09T06:17:25.856690+0000","last_became_active":"2023-10-09T06:01:54.761251+0000","last_became_peered":"2023-10-09T06:01:54.761251+0000","last_unstale":"2023-10-09T06:17:25.856690+0000","last_undegraded":"2023-10-09T06:17:25.856690+0000","last_fullsized":"2023-10-09T06:17:25.856690+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:53.245267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":1095,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.390106+0000","last_change":"2023-10-09T06:13:24.754018+0000","last_active":"2023-10-09T06:17:20.390106+0000","last_peered":"2023-10-09T06:17:20.390106+0000","last_clean":"2023-10-09T06:17:20.390106+0000","last_became_active":"2023-10-09T06:13:24.753712+0000","last_became_peered":"2023-10-09T06:13:24.753712+0000","last_unstale":"2023-10-09T06:17:20.390106+0000","last_undegraded":"2023-10-09T06:17:20.390106+0000","last_fullsized":"2023-10-09T06:17:20.390106+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:18.490833+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":75,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.390520+0000","last_change":"2023-10-09T06:13:30.788160+0000","last_active":"2023-10-09T06:17:20.390520+0000","last_peered":"2023-10-09T06:17:20.390520+0000","last_clean":"2023-10-09T06:17:20.390520+0000","last_became_active":"2023-10-09T06:13:30.787724+0000","last_became_peered":"2023-10-09T06:13:30.787724+0000","last_unstale":"2023-10-09T06:17:20.390520+0000","last_undegraded":"2023-10-09T06:17:20.390520+0000","last_fullsized":"2023-10-09T06:17:20.390520+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:38.376770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":75,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.391270+0000","last_change":"2023-10-09T06:13:30.789225+0000","last_active":"2023-10-09T06:17:20.391270+0000","last_peered":"2023-10-09T06:17:20.391270+0000","last_clean":"2023-10-09T06:17:20.391270+0000","last_became_active":"2023-10-09T06:13:30.788869+0000","last_became_peered":"2023-10-09T06:13:30.788869+0000","last_unstale":"2023-10-09T06:17:20.391270+0000","last_undegraded":"2023-10-09T06:17:20.391270+0000","last_fullsized":"2023-10-09T06:17:20.391270+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:19.519966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":833,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.400322+0000","last_change":"2023-10-09T06:12:55.750923+0000","last_active":"2023-10-09T06:17:20.400322+0000","last_peered":"2023-10-09T06:17:20.400322+0000","last_clean":"2023-10-09T06:17:20.400322+0000","last_became_active":"2023-10-09T06:12:55.750521+0000","last_became_peered":"2023-10-09T06:12:55.750521+0000","last_unstale":"2023-10-09T06:17:20.400322+0000","last_undegraded":"2023-10-09T06:17:20.400322+0000","last_fullsized":"2023-10-09T06:17:20.400322+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.316697+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.316697+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:23.591439+0000","scrub_duration":0.001666561,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":445,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.390324+0000","last_change":"2023-10-09T06:01:22.337171+0000","last_active":"2023-10-09T06:17:20.390324+0000","last_peered":"2023-10-09T06:17:20.390324+0000","last_clean":"2023-10-09T06:17:20.390324+0000","last_became_active":"2023-10-09T06:01:22.335438+0000","last_became_peered":"2023-10-09T06:01:22.335438+0000","last_unstale":"2023-10-09T06:17:20.390324+0000","last_undegraded":"2023-10-09T06:17:20.390324+0000","last_fullsized":"2023-10-09T06:17:20.390324+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:25.036448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":555,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.394575+0000","last_change":"2023-10-09T06:13:30.788284+0000","last_active":"2023-10-09T06:17:20.394575+0000","last_peered":"2023-10-09T06:17:20.394575+0000","last_clean":"2023-10-09T06:17:20.394575+0000","last_became_active":"2023-10-09T06:13:30.787933+0000","last_became_peered":"2023-10-09T06:13:30.787933+0000","last_unstale":"2023-10-09T06:17:20.394575+0000","last_undegraded":"2023-10-09T06:17:20.394575+0000","last_fullsized":"2023-10-09T06:17:20.394575+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:10.735438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":883,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.399185+0000","last_change":"2023-10-09T06:13:24.753908+0000","last_active":"2023-10-09T06:17:20.399185+0000","last_peered":"2023-10-09T06:17:20.399185+0000","last_clean":"2023-10-09T06:17:20.399185+0000","last_became_active":"2023-10-09T06:13:24.753566+0000","last_became_peered":"2023-10-09T06:13:24.753566+0000","last_unstale":"2023-10-09T06:17:20.399185+0000","last_undegraded":"2023-10-09T06:17:20.399185+0000","last_fullsized":"2023-10-09T06:17:20.399185+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:40.388125+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:40.388125+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:35.101978+0000","scrub_duration":0.0055576990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":165,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.380651+0000","last_change":"2023-10-09T06:13:12.243605+0000","last_active":"2023-10-09T06:17:20.380651+0000","last_peered":"2023-10-09T06:17:20.380651+0000","last_clean":"2023-10-09T06:17:20.380651+0000","last_became_active":"2023-10-09T06:13:12.237061+0000","last_became_peered":"2023-10-09T06:13:12.237061+0000","last_unstale":"2023-10-09T06:17:20.380651+0000","last_undegraded":"2023-10-09T06:17:20.380651+0000","last_fullsized":"2023-10-09T06:17:20.380651+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:29:03.666031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":103,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.394626+0000","last_change":"2023-10-09T06:13:30.788733+0000","last_active":"2023-10-09T06:17:20.394626+0000","last_peered":"2023-10-09T06:17:20.394626+0000","last_clean":"2023-10-09T06:17:20.394626+0000","last_became_active":"2023-10-09T06:13:30.788353+0000","last_became_peered":"2023-10-09T06:13:30.788353+0000","last_unstale":"2023-10-09T06:17:20.394626+0000","last_undegraded":"2023-10-09T06:17:20.394626+0000","last_fullsized":"2023-10-09T06:17:20.394626+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:06.308426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"592'6640","reported_seq":6669,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-10-09T06:17:36.892226+0000","last_change":"2023-10-09T06:16:02.762978+0000","last_active":"2023-10-09T06:17:36.892226+0000","last_peered":"2023-10-09T06:17:36.892226+0000","last_clean":"2023-10-09T06:17:36.892226+0000","last_became_active":"2023-10-09T06:16:02.762657+0000","last_became_peered":"2023-10-09T06:16:02.762657+0000","last_unstale":"2023-10-09T06:17:36.892226+0000","last_undegraded":"2023-10-09T06:17:36.892226+0000","last_fullsized":"2023-10-09T06:17:36.892226+0000","mapping_epoch":572,"log_start":"579'1400","ondisk_log_start":"579'1400","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5240,"log_dups_size":0,"ondisk_log_size":5240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:11:46.434393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":435159040,"num_objects":6640,"num_object_clones":0,"num_object_copies":19920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6640,"num_write_kb":424960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":773,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.391340+0000","last_change":"2023-10-09T06:13:33.701521+0000","last_active":"2023-10-09T06:17:20.391340+0000","last_peered":"2023-10-09T06:17:20.391340+0000","last_clean":"2023-10-09T06:17:20.391340+0000","last_became_active":"2023-10-09T06:13:33.700870+0000","last_became_peered":"2023-10-09T06:13:33.700870+0000","last_unstale":"2023-10-09T06:17:20.391340+0000","last_undegraded":"2023-10-09T06:17:20.391340+0000","last_fullsized":"2023-10-09T06:17:20.391340+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:10:59.422551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:10:59.422551+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:54.402494+0000","scrub_duration":0.0059716659999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":79,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.390836+0000","last_change":"2023-10-09T06:13:30.822903+0000","last_active":"2023-10-09T06:17:20.390836+0000","last_peered":"2023-10-09T06:17:20.390836+0000","last_clean":"2023-10-09T06:17:20.390836+0000","last_became_active":"2023-10-09T06:13:30.799013+0000","last_became_peered":"2023-10-09T06:13:30.799013+0000","last_unstale":"2023-10-09T06:17:20.390836+0000","last_undegraded":"2023-10-09T06:17:20.390836+0000","last_fullsized":"2023-10-09T06:17:20.390836+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:44.271821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":75,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.389967+0000","last_change":"2023-10-09T06:13:30.793531+0000","last_active":"2023-10-09T06:17:20.389967+0000","last_peered":"2023-10-09T06:17:20.389967+0000","last_clean":"2023-10-09T06:17:20.389967+0000","last_became_active":"2023-10-09T06:13:30.792689+0000","last_became_peered":"2023-10-09T06:13:30.792689+0000","last_unstale":"2023-10-09T06:17:20.389967+0000","last_undegraded":"2023-10-09T06:17:20.389967+0000","last_fullsized":"2023-10-09T06:17:20.389967+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:02.559186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.6","version":"592'6606","reported_seq":6635,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-10-09T06:17:37.050653+0000","last_change":"2023-10-09T06:16:02.756362+0000","last_active":"2023-10-09T06:17:37.050653+0000","last_peered":"2023-10-09T06:17:37.050653+0000","last_clean":"2023-10-09T06:17:37.050653+0000","last_became_active":"2023-10-09T06:16:02.756033+0000","last_became_peered":"2023-10-09T06:16:02.756033+0000","last_unstale":"2023-10-09T06:17:37.050653+0000","last_undegraded":"2023-10-09T06:17:37.050653+0000","last_fullsized":"2023-10-09T06:17:37.050653+0000","mapping_epoch":572,"log_start":"579'1400","ondisk_log_start":"579'1400","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5206,"log_dups_size":0,"ondisk_log_size":5206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:07.492688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":432930816,"num_objects":6606,"num_object_clones":0,"num_object_copies":19818,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6606,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6606,"num_write_kb":422784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":962,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.391036+0000","last_change":"2023-10-09T06:13:24.753866+0000","last_active":"2023-10-09T06:17:20.391036+0000","last_peered":"2023-10-09T06:17:20.391036+0000","last_clean":"2023-10-09T06:17:20.391036+0000","last_became_active":"2023-10-09T06:13:24.753432+0000","last_became_peered":"2023-10-09T06:13:24.753432+0000","last_unstale":"2023-10-09T06:17:20.391036+0000","last_undegraded":"2023-10-09T06:17:20.391036+0000","last_fullsized":"2023-10-09T06:17:20.391036+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:03.241335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:03.241335+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:19.854449+0000","scrub_duration":0.017780547000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":79,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.390287+0000","last_change":"2023-10-09T06:13:30.829763+0000","last_active":"2023-10-09T06:17:20.390287+0000","last_peered":"2023-10-09T06:17:20.390287+0000","last_clean":"2023-10-09T06:17:20.390287+0000","last_became_active":"2023-10-09T06:13:30.802486+0000","last_became_peered":"2023-10-09T06:13:30.802486+0000","last_unstale":"2023-10-09T06:17:20.390287+0000","last_undegraded":"2023-10-09T06:17:20.390287+0000","last_fullsized":"2023-10-09T06:17:20.390287+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:05.468205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":630,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.403307+0000","last_change":"2023-10-09T06:01:54.760367+0000","last_active":"2023-10-09T06:17:20.403307+0000","last_peered":"2023-10-09T06:17:20.403307+0000","last_clean":"2023-10-09T06:17:20.403307+0000","last_became_active":"2023-10-09T06:01:54.760087+0000","last_became_peered":"2023-10-09T06:01:54.760087+0000","last_unstale":"2023-10-09T06:17:20.403307+0000","last_undegraded":"2023-10-09T06:17:20.403307+0000","last_fullsized":"2023-10-09T06:17:20.403307+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:26.000104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"592'6524","reported_seq":6552,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-10-09T06:17:34.328894+0000","last_change":"2023-10-09T06:16:02.761005+0000","last_active":"2023-10-09T06:17:34.328894+0000","last_peered":"2023-10-09T06:17:34.328894+0000","last_clean":"2023-10-09T06:17:34.328894+0000","last_became_active":"2023-10-09T06:16:02.760718+0000","last_became_peered":"2023-10-09T06:16:02.760718+0000","last_unstale":"2023-10-09T06:17:34.328894+0000","last_undegraded":"2023-10-09T06:17:34.328894+0000","last_fullsized":"2023-10-09T06:17:34.328894+0000","mapping_epoch":572,"log_start":"578'700","ondisk_log_start":"578'700","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5824,"log_dups_size":0,"ondisk_log_size":5824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:32.418781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":427556864,"num_objects":6524,"num_object_clones":0,"num_object_copies":19572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6524,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6524,"num_write_kb":417536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":855,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.403969+0000","last_change":"2023-10-09T06:13:24.768958+0000","last_active":"2023-10-09T06:17:20.403969+0000","last_peered":"2023-10-09T06:17:20.403969+0000","last_clean":"2023-10-09T06:17:20.403969+0000","last_became_active":"2023-10-09T06:13:24.768643+0000","last_became_peered":"2023-10-09T06:13:24.768643+0000","last_unstale":"2023-10-09T06:17:20.403969+0000","last_undegraded":"2023-10-09T06:17:20.403969+0000","last_fullsized":"2023-10-09T06:17:20.403969+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:48.495739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:48.495739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:18:50.654354+0000","scrub_duration":0.0016345509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":79,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.391203+0000","last_change":"2023-10-09T06:13:30.814434+0000","last_active":"2023-10-09T06:17:20.391203+0000","last_peered":"2023-10-09T06:17:20.391203+0000","last_clean":"2023-10-09T06:17:20.391203+0000","last_became_active":"2023-10-09T06:13:30.793119+0000","last_became_peered":"2023-10-09T06:13:30.793119+0000","last_unstale":"2023-10-09T06:17:20.391203+0000","last_undegraded":"2023-10-09T06:17:20.391203+0000","last_fullsized":"2023-10-09T06:17:20.391203+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:34:31.724006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":618,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.398902+0000","last_change":"2023-10-09T06:13:30.792490+0000","last_active":"2023-10-09T06:17:20.398902+0000","last_peered":"2023-10-09T06:17:20.398902+0000","last_clean":"2023-10-09T06:17:20.398902+0000","last_became_active":"2023-10-09T06:13:30.792209+0000","last_became_peered":"2023-10-09T06:13:30.792209+0000","last_unstale":"2023-10-09T06:17:20.398902+0000","last_undegraded":"2023-10-09T06:17:20.398902+0000","last_fullsized":"2023-10-09T06:17:20.398902+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:14.208042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.4","version":"592'6482","reported_seq":6510,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-10-09T06:17:34.317764+0000","last_change":"2023-10-09T06:16:02.759679+0000","last_active":"2023-10-09T06:17:34.317764+0000","last_peered":"2023-10-09T06:17:34.317764+0000","last_clean":"2023-10-09T06:17:34.317764+0000","last_became_active":"2023-10-09T06:16:02.759382+0000","last_became_peered":"2023-10-09T06:16:02.759382+0000","last_unstale":"2023-10-09T06:17:34.317764+0000","last_undegraded":"2023-10-09T06:17:34.317764+0000","last_fullsized":"2023-10-09T06:17:34.317764+0000","mapping_epoch":572,"log_start":"578'700","ondisk_log_start":"578'700","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5782,"log_dups_size":0,"ondisk_log_size":5782,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:04:34.950818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":424804352,"num_objects":6482,"num_object_clones":0,"num_object_copies":19446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6482,"num_write_kb":414848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":807,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.403600+0000","last_change":"2023-10-09T06:13:33.708291+0000","last_active":"2023-10-09T06:17:20.403600+0000","last_peered":"2023-10-09T06:17:20.403600+0000","last_clean":"2023-10-09T06:17:20.403600+0000","last_became_active":"2023-10-09T06:13:33.707930+0000","last_became_peered":"2023-10-09T06:13:33.707930+0000","last_unstale":"2023-10-09T06:17:20.403600+0000","last_undegraded":"2023-10-09T06:17:20.403600+0000","last_fullsized":"2023-10-09T06:17:20.403600+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:16.361097+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:16.361097+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:18.111404+0000","scrub_duration":0.028533749000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":362,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.391160+0000","last_change":"2023-10-09T05:57:23.487373+0000","last_active":"2023-10-09T06:17:20.391160+0000","last_peered":"2023-10-09T06:17:20.391160+0000","last_clean":"2023-10-09T06:17:20.391160+0000","last_became_active":"2023-10-09T05:57:23.486074+0000","last_became_peered":"2023-10-09T05:57:23.486074+0000","last_unstale":"2023-10-09T06:17:20.391160+0000","last_undegraded":"2023-10-09T06:17:20.391160+0000","last_fullsized":"2023-10-09T06:17:20.391160+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:58.194233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":606,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.384472+0000","last_change":"2023-10-09T06:09:27.558353+0000","last_active":"2023-10-09T06:17:20.384472+0000","last_peered":"2023-10-09T06:17:20.384472+0000","last_clean":"2023-10-09T06:17:20.384472+0000","last_became_active":"2023-10-09T06:09:27.557233+0000","last_became_peered":"2023-10-09T06:09:27.557233+0000","last_unstale":"2023-10-09T06:17:20.384472+0000","last_undegraded":"2023-10-09T06:17:20.384472+0000","last_fullsized":"2023-10-09T06:17:20.384472+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:53.149350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"592'6674","reported_seq":6702,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-10-09T06:17:34.354364+0000","last_change":"2023-10-09T06:16:02.761692+0000","last_active":"2023-10-09T06:17:34.354364+0000","last_peered":"2023-10-09T06:17:34.354364+0000","last_clean":"2023-10-09T06:17:34.354364+0000","last_became_active":"2023-10-09T06:16:02.761411+0000","last_became_peered":"2023-10-09T06:16:02.761411+0000","last_unstale":"2023-10-09T06:17:34.354364+0000","last_undegraded":"2023-10-09T06:17:34.354364+0000","last_fullsized":"2023-10-09T06:17:34.354364+0000","mapping_epoch":572,"log_start":"578'900","ondisk_log_start":"578'900","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5774,"log_dups_size":0,"ondisk_log_size":5774,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:55.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":437387264,"num_objects":6674,"num_object_clones":0,"num_object_copies":20022,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6674,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6674,"num_write_kb":427136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":998,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.390186+0000","last_change":"2023-10-09T06:15:19.874375+0000","last_active":"2023-10-09T06:17:20.390186+0000","last_peered":"2023-10-09T06:17:20.390186+0000","last_clean":"2023-10-09T06:17:20.390186+0000","last_became_active":"2023-10-09T06:15:19.873993+0000","last_became_peered":"2023-10-09T06:15:19.873993+0000","last_unstale":"2023-10-09T06:17:20.390186+0000","last_undegraded":"2023-10-09T06:17:20.390186+0000","last_fullsized":"2023-10-09T06:17:20.390186+0000","mapping_epoch":561,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":562,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:42.795781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:42.795781+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:43.523891+0000","scrub_duration":0.0091262979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":396,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.391125+0000","last_change":"2023-10-09T06:09:27.565666+0000","last_active":"2023-10-09T06:17:20.391125+0000","last_peered":"2023-10-09T06:17:20.391125+0000","last_clean":"2023-10-09T06:17:20.391125+0000","last_became_active":"2023-10-09T06:09:27.565324+0000","last_became_peered":"2023-10-09T06:09:27.565324+0000","last_unstale":"2023-10-09T06:17:20.391125+0000","last_undegraded":"2023-10-09T06:17:20.391125+0000","last_fullsized":"2023-10-09T06:17:20.391125+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:51.216947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":566,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.403674+0000","last_change":"2023-10-09T06:13:21.727745+0000","last_active":"2023-10-09T06:17:20.403674+0000","last_peered":"2023-10-09T06:17:20.403674+0000","last_clean":"2023-10-09T06:17:20.403674+0000","last_became_active":"2023-10-09T06:13:21.727404+0000","last_became_peered":"2023-10-09T06:13:21.727404+0000","last_unstale":"2023-10-09T06:17:20.403674+0000","last_undegraded":"2023-10-09T06:17:20.403674+0000","last_fullsized":"2023-10-09T06:17:20.403674+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:24.931626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"592'6567","reported_seq":6596,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-10-09T06:17:35.855562+0000","last_change":"2023-10-09T06:16:02.747886+0000","last_active":"2023-10-09T06:17:35.855562+0000","last_peered":"2023-10-09T06:17:35.855562+0000","last_clean":"2023-10-09T06:17:35.855562+0000","last_became_active":"2023-10-09T06:16:02.747569+0000","last_became_peered":"2023-10-09T06:16:02.747569+0000","last_unstale":"2023-10-09T06:17:35.855562+0000","last_undegraded":"2023-10-09T06:17:35.855562+0000","last_fullsized":"2023-10-09T06:17:35.855562+0000","mapping_epoch":572,"log_start":"578'700","ondisk_log_start":"578'700","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5867,"log_dups_size":0,"ondisk_log_size":5867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:18.720375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":430374912,"num_objects":6567,"num_object_clones":0,"num_object_copies":19701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6567,"num_write_kb":420288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":873,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.388531+0000","last_change":"2023-10-09T06:16:05.756855+0000","last_active":"2023-10-09T06:17:20.388531+0000","last_peered":"2023-10-09T06:17:20.388531+0000","last_clean":"2023-10-09T06:17:20.388531+0000","last_became_active":"2023-10-09T06:16:05.756518+0000","last_became_peered":"2023-10-09T06:16:05.756518+0000","last_unstale":"2023-10-09T06:17:20.388531+0000","last_undegraded":"2023-10-09T06:17:20.388531+0000","last_fullsized":"2023-10-09T06:17:20.388531+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.282393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.282393+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:24:20.507377+0000","scrub_duration":0.01402403,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":511,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.403431+0000","last_change":"2023-10-09T06:13:21.738218+0000","last_active":"2023-10-09T06:17:20.403431+0000","last_peered":"2023-10-09T06:17:20.403431+0000","last_clean":"2023-10-09T06:17:20.403431+0000","last_became_active":"2023-10-09T06:13:21.737969+0000","last_became_peered":"2023-10-09T06:13:21.737969+0000","last_unstale":"2023-10-09T06:17:20.403431+0000","last_undegraded":"2023-10-09T06:17:20.403431+0000","last_fullsized":"2023-10-09T06:17:20.403431+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:38.620281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":650,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.403972+0000","last_change":"2023-10-09T06:15:16.810530+0000","last_active":"2023-10-09T06:17:20.403972+0000","last_peered":"2023-10-09T06:17:20.403972+0000","last_clean":"2023-10-09T06:17:20.403972+0000","last_became_active":"2023-10-09T06:15:16.810238+0000","last_became_peered":"2023-10-09T06:15:16.810238+0000","last_unstale":"2023-10-09T06:17:20.403972+0000","last_undegraded":"2023-10-09T06:17:20.403972+0000","last_fullsized":"2023-10-09T06:17:20.403972+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:03.744638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"592'6438","reported_seq":6467,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-10-09T06:17:35.726265+0000","last_change":"2023-10-09T06:16:02.753013+0000","last_active":"2023-10-09T06:17:35.726265+0000","last_peered":"2023-10-09T06:17:35.726265+0000","last_clean":"2023-10-09T06:17:35.726265+0000","last_became_active":"2023-10-09T06:16:02.752442+0000","last_became_peered":"2023-10-09T06:16:02.752442+0000","last_unstale":"2023-10-09T06:17:35.726265+0000","last_undegraded":"2023-10-09T06:17:35.726265+0000","last_fullsized":"2023-10-09T06:17:35.726265+0000","mapping_epoch":572,"log_start":"578'600","ondisk_log_start":"578'600","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5838,"log_dups_size":0,"ondisk_log_size":5838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:16.543432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":421920768,"num_objects":6438,"num_object_clones":0,"num_object_copies":19314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6438,"num_write_kb":412032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":927,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.400156+0000","last_change":"2023-10-09T06:13:15.428701+0000","last_active":"2023-10-09T06:17:20.400156+0000","last_peered":"2023-10-09T06:17:20.400156+0000","last_clean":"2023-10-09T06:17:20.400156+0000","last_became_active":"2023-10-09T06:13:15.428343+0000","last_became_peered":"2023-10-09T06:13:15.428343+0000","last_unstale":"2023-10-09T06:17:20.400156+0000","last_undegraded":"2023-10-09T06:17:20.400156+0000","last_fullsized":"2023-10-09T06:17:20.400156+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.373748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.373748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:55:34.235074+0000","scrub_duration":0.0024576450000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":592,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.403406+0000","last_change":"2023-10-09T06:16:02.744252+0000","last_active":"2023-10-09T06:17:20.403406+0000","last_peered":"2023-10-09T06:17:20.403406+0000","last_clean":"2023-10-09T06:17:20.403406+0000","last_became_active":"2023-10-09T06:16:02.743947+0000","last_became_peered":"2023-10-09T06:16:02.743947+0000","last_unstale":"2023-10-09T06:17:20.403406+0000","last_undegraded":"2023-10-09T06:17:20.403406+0000","last_fullsized":"2023-10-09T06:17:20.403406+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:52.293310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":623,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.403666+0000","last_change":"2023-10-09T06:13:21.727814+0000","last_active":"2023-10-09T06:17:20.403666+0000","last_peered":"2023-10-09T06:17:20.403666+0000","last_clean":"2023-10-09T06:17:20.403666+0000","last_became_active":"2023-10-09T06:13:21.727401+0000","last_became_peered":"2023-10-09T06:13:21.727401+0000","last_unstale":"2023-10-09T06:17:20.403666+0000","last_undegraded":"2023-10-09T06:17:20.403666+0000","last_fullsized":"2023-10-09T06:17:20.403666+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:53:37.968028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"592'6795","reported_seq":6824,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-10-09T06:17:36.888964+0000","last_change":"2023-10-09T06:16:02.744178+0000","last_active":"2023-10-09T06:17:36.888964+0000","last_peered":"2023-10-09T06:17:36.888964+0000","last_clean":"2023-10-09T06:17:36.888964+0000","last_became_active":"2023-10-09T06:16:02.743835+0000","last_became_peered":"2023-10-09T06:16:02.743835+0000","last_unstale":"2023-10-09T06:17:36.888964+0000","last_undegraded":"2023-10-09T06:17:36.888964+0000","last_fullsized":"2023-10-09T06:17:36.888964+0000","mapping_epoch":572,"log_start":"579'1600","ondisk_log_start":"579'1600","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5195,"log_dups_size":0,"ondisk_log_size":5195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:02.861316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":445317120,"num_objects":6795,"num_object_clones":0,"num_object_copies":20385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6795,"num_write_kb":434880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":651,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.403284+0000","last_change":"2023-10-09T06:16:05.760797+0000","last_active":"2023-10-09T06:17:20.403284+0000","last_peered":"2023-10-09T06:17:20.403284+0000","last_clean":"2023-10-09T06:17:20.403284+0000","last_became_active":"2023-10-09T06:16:05.760421+0000","last_became_peered":"2023-10-09T06:16:05.760421+0000","last_unstale":"2023-10-09T06:17:20.403284+0000","last_undegraded":"2023-10-09T06:17:20.403284+0000","last_fullsized":"2023-10-09T06:17:20.403284+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:39.633410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:39.633410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:02.235216+0000","scrub_duration":0.0039938459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":405,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.388687+0000","last_change":"2023-10-09T06:13:30.781849+0000","last_active":"2023-10-09T06:17:20.388687+0000","last_peered":"2023-10-09T06:17:20.388687+0000","last_clean":"2023-10-09T06:17:20.388687+0000","last_became_active":"2023-10-09T06:13:30.781084+0000","last_became_peered":"2023-10-09T06:13:30.781084+0000","last_unstale":"2023-10-09T06:17:20.388687+0000","last_undegraded":"2023-10-09T06:17:20.388687+0000","last_fullsized":"2023-10-09T06:17:20.388687+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:02.199298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":275,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.407888+0000","last_change":"2023-10-09T06:01:54.761768+0000","last_active":"2023-10-09T06:17:20.407888+0000","last_peered":"2023-10-09T06:17:20.407888+0000","last_clean":"2023-10-09T06:17:20.407888+0000","last_became_active":"2023-10-09T06:01:54.761443+0000","last_became_peered":"2023-10-09T06:01:54.761443+0000","last_unstale":"2023-10-09T06:17:20.407888+0000","last_undegraded":"2023-10-09T06:17:20.407888+0000","last_fullsized":"2023-10-09T06:17:20.407888+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:51.555987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"592'6502","reported_seq":6529,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-10-09T06:17:36.057960+0000","last_change":"2023-10-09T06:16:02.756342+0000","last_active":"2023-10-09T06:17:36.057960+0000","last_peered":"2023-10-09T06:17:36.057960+0000","last_clean":"2023-10-09T06:17:36.057960+0000","last_became_active":"2023-10-09T06:16:02.756000+0000","last_became_peered":"2023-10-09T06:16:02.756000+0000","last_unstale":"2023-10-09T06:17:36.057960+0000","last_undegraded":"2023-10-09T06:17:36.057960+0000","last_fullsized":"2023-10-09T06:17:36.057960+0000","mapping_epoch":572,"log_start":"578'600","ondisk_log_start":"578'600","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5902,"log_dups_size":0,"ondisk_log_size":5902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:55.773353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":426115072,"num_objects":6502,"num_object_clones":0,"num_object_copies":19506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6502,"num_write_kb":416128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":796,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.394718+0000","last_change":"2023-10-09T06:13:33.707885+0000","last_active":"2023-10-09T06:17:20.394718+0000","last_peered":"2023-10-09T06:17:20.394718+0000","last_clean":"2023-10-09T06:17:20.394718+0000","last_became_active":"2023-10-09T06:13:33.707474+0000","last_became_peered":"2023-10-09T06:13:33.707474+0000","last_unstale":"2023-10-09T06:17:20.394718+0000","last_undegraded":"2023-10-09T06:17:20.394718+0000","last_fullsized":"2023-10-09T06:17:20.394718+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:12:22.289803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:12:22.289803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:27.842817+0000","scrub_duration":0.0076565050000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":338,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.399289+0000","last_change":"2023-10-09T06:12:04.680247+0000","last_active":"2023-10-09T06:17:20.399289+0000","last_peered":"2023-10-09T06:17:20.399289+0000","last_clean":"2023-10-09T06:17:20.399289+0000","last_became_active":"2023-10-09T06:12:04.679943+0000","last_became_peered":"2023-10-09T06:12:04.679943+0000","last_unstale":"2023-10-09T06:17:20.399289+0000","last_undegraded":"2023-10-09T06:17:20.399289+0000","last_fullsized":"2023-10-09T06:17:20.399289+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:56.692774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":390,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.391137+0000","last_change":"2023-10-09T06:16:02.759206+0000","last_active":"2023-10-09T06:17:20.391137+0000","last_peered":"2023-10-09T06:17:20.391137+0000","last_clean":"2023-10-09T06:17:20.391137+0000","last_became_active":"2023-10-09T06:16:02.758888+0000","last_became_peered":"2023-10-09T06:16:02.758888+0000","last_unstale":"2023-10-09T06:17:20.391137+0000","last_undegraded":"2023-10-09T06:17:20.391137+0000","last_fullsized":"2023-10-09T06:17:20.391137+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:33.728805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":309,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.399032+0000","last_change":"2023-10-09T06:13:12.234849+0000","last_active":"2023-10-09T06:17:20.399032+0000","last_peered":"2023-10-09T06:17:20.399032+0000","last_clean":"2023-10-09T06:17:20.399032+0000","last_became_active":"2023-10-09T06:13:12.234284+0000","last_became_peered":"2023-10-09T06:13:12.234284+0000","last_unstale":"2023-10-09T06:17:20.399032+0000","last_undegraded":"2023-10-09T06:17:20.399032+0000","last_fullsized":"2023-10-09T06:17:20.399032+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:40:43.809773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"592'6440","reported_seq":6469,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-10-09T06:17:35.758025+0000","last_change":"2023-10-09T06:16:02.749724+0000","last_active":"2023-10-09T06:17:35.758025+0000","last_peered":"2023-10-09T06:17:35.758025+0000","last_clean":"2023-10-09T06:17:35.758025+0000","last_became_active":"2023-10-09T06:16:02.749368+0000","last_became_peered":"2023-10-09T06:16:02.749368+0000","last_unstale":"2023-10-09T06:17:35.758025+0000","last_undegraded":"2023-10-09T06:17:35.758025+0000","last_fullsized":"2023-10-09T06:17:35.758025+0000","mapping_epoch":572,"log_start":"578'600","ondisk_log_start":"578'600","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5840,"log_dups_size":0,"ondisk_log_size":5840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:24.189151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":422051840,"num_objects":6440,"num_object_clones":0,"num_object_copies":19320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6440,"num_write_kb":412160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":845,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.383184+0000","last_change":"2023-10-09T06:16:05.759429+0000","last_active":"2023-10-09T06:17:20.383184+0000","last_peered":"2023-10-09T06:17:20.383184+0000","last_clean":"2023-10-09T06:17:20.383184+0000","last_became_active":"2023-10-09T06:16:05.759155+0000","last_became_peered":"2023-10-09T06:16:05.759155+0000","last_unstale":"2023-10-09T06:17:20.383184+0000","last_undegraded":"2023-10-09T06:17:20.383184+0000","last_fullsized":"2023-10-09T06:17:20.383184+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:37.915576+0000","scrub_duration":0.0016660799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":294,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.399104+0000","last_change":"2023-10-09T06:13:21.738040+0000","last_active":"2023-10-09T06:17:20.399104+0000","last_peered":"2023-10-09T06:17:20.399104+0000","last_clean":"2023-10-09T06:17:20.399104+0000","last_became_active":"2023-10-09T06:13:21.737717+0000","last_became_peered":"2023-10-09T06:13:21.737717+0000","last_unstale":"2023-10-09T06:17:20.399104+0000","last_undegraded":"2023-10-09T06:17:20.399104+0000","last_fullsized":"2023-10-09T06:17:20.399104+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:39.451628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":572,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.403376+0000","last_change":"2023-10-09T06:09:27.553958+0000","last_active":"2023-10-09T06:17:20.403376+0000","last_peered":"2023-10-09T06:17:20.403376+0000","last_clean":"2023-10-09T06:17:20.403376+0000","last_became_active":"2023-10-09T06:09:27.553193+0000","last_became_peered":"2023-10-09T06:09:27.553193+0000","last_unstale":"2023-10-09T06:17:20.403376+0000","last_undegraded":"2023-10-09T06:17:20.403376+0000","last_fullsized":"2023-10-09T06:17:20.403376+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:09.754579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":428,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.399370+0000","last_change":"2023-10-09T06:16:02.753637+0000","last_active":"2023-10-09T06:17:20.399370+0000","last_peered":"2023-10-09T06:17:20.399370+0000","last_clean":"2023-10-09T06:17:20.399370+0000","last_became_active":"2023-10-09T06:16:02.753254+0000","last_became_peered":"2023-10-09T06:16:02.753254+0000","last_unstale":"2023-10-09T06:17:20.399370+0000","last_undegraded":"2023-10-09T06:17:20.399370+0000","last_fullsized":"2023-10-09T06:17:20.399370+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:18.543079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"592'6641","reported_seq":6670,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-10-09T06:17:35.757856+0000","last_change":"2023-10-09T06:16:02.759668+0000","last_active":"2023-10-09T06:17:35.757856+0000","last_peered":"2023-10-09T06:17:35.757856+0000","last_clean":"2023-10-09T06:17:35.757856+0000","last_became_active":"2023-10-09T06:16:02.758840+0000","last_became_peered":"2023-10-09T06:16:02.758840+0000","last_unstale":"2023-10-09T06:17:35.757856+0000","last_undegraded":"2023-10-09T06:17:35.757856+0000","last_fullsized":"2023-10-09T06:17:35.757856+0000","mapping_epoch":572,"log_start":"578'800","ondisk_log_start":"578'800","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5841,"log_dups_size":0,"ondisk_log_size":5841,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:28.523305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":435224576,"num_objects":6641,"num_object_clones":0,"num_object_copies":19923,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6641,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6641,"num_write_kb":425024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":647,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.390666+0000","last_change":"2023-10-09T06:14:47.339271+0000","last_active":"2023-10-09T06:17:20.390666+0000","last_peered":"2023-10-09T06:17:20.390666+0000","last_clean":"2023-10-09T06:17:20.390666+0000","last_became_active":"2023-10-09T06:13:33.705351+0000","last_became_peered":"2023-10-09T06:13:33.705351+0000","last_unstale":"2023-10-09T06:17:20.390666+0000","last_undegraded":"2023-10-09T06:17:20.390666+0000","last_fullsized":"2023-10-09T06:17:20.390666+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:14:47.339163+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T06:14:47.339163+0000","last_clean_scrub_stamp":"2023-10-09T06:14:47.339163+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:34.887758+0000","scrub_duration":0.0016607729999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":382,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.392768+0000","last_change":"2023-10-09T06:15:16.802788+0000","last_active":"2023-10-09T06:17:20.392768+0000","last_peered":"2023-10-09T06:17:20.392768+0000","last_clean":"2023-10-09T06:17:20.392768+0000","last_became_active":"2023-10-09T06:15:16.802449+0000","last_became_peered":"2023-10-09T06:15:16.802449+0000","last_unstale":"2023-10-09T06:17:20.392768+0000","last_undegraded":"2023-10-09T06:17:20.392768+0000","last_fullsized":"2023-10-09T06:17:20.392768+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:55:30.488174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":372,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.398885+0000","last_change":"2023-10-09T06:16:02.752290+0000","last_active":"2023-10-09T06:17:20.398885+0000","last_peered":"2023-10-09T06:17:20.398885+0000","last_clean":"2023-10-09T06:17:20.398885+0000","last_became_active":"2023-10-09T06:16:02.751942+0000","last_became_peered":"2023-10-09T06:16:02.751942+0000","last_unstale":"2023-10-09T06:17:20.398885+0000","last_undegraded":"2023-10-09T06:17:20.398885+0000","last_fullsized":"2023-10-09T06:17:20.398885+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:45:56.100957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":291,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.407820+0000","last_change":"2023-10-09T06:09:27.547953+0000","last_active":"2023-10-09T06:17:20.407820+0000","last_peered":"2023-10-09T06:17:20.407820+0000","last_clean":"2023-10-09T06:17:20.407820+0000","last_became_active":"2023-10-09T06:09:27.547624+0000","last_became_peered":"2023-10-09T06:09:27.547624+0000","last_unstale":"2023-10-09T06:17:20.407820+0000","last_undegraded":"2023-10-09T06:17:20.407820+0000","last_fullsized":"2023-10-09T06:17:20.407820+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:39.630486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"592'6503","reported_seq":6532,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-10-09T06:17:35.855437+0000","last_change":"2023-10-09T06:16:02.759576+0000","last_active":"2023-10-09T06:17:35.855437+0000","last_peered":"2023-10-09T06:17:35.855437+0000","last_clean":"2023-10-09T06:17:35.855437+0000","last_became_active":"2023-10-09T06:16:02.759323+0000","last_became_peered":"2023-10-09T06:16:02.759323+0000","last_unstale":"2023-10-09T06:17:35.855437+0000","last_undegraded":"2023-10-09T06:17:35.855437+0000","last_fullsized":"2023-10-09T06:17:35.855437+0000","mapping_epoch":572,"log_start":"578'700","ondisk_log_start":"578'700","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5803,"log_dups_size":0,"ondisk_log_size":5803,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:55.127489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":426180608,"num_objects":6503,"num_object_clones":0,"num_object_copies":19509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6503,"num_write_kb":416192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":85,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.391158+0000","last_change":"2023-10-09T06:16:02.761560+0000","last_active":"2023-10-09T06:17:20.391158+0000","last_peered":"2023-10-09T06:17:20.391158+0000","last_clean":"2023-10-09T06:17:20.391158+0000","last_became_active":"2023-10-09T06:16:02.761061+0000","last_became_peered":"2023-10-09T06:16:02.761061+0000","last_unstale":"2023-10-09T06:17:20.391158+0000","last_undegraded":"2023-10-09T06:17:20.391158+0000","last_fullsized":"2023-10-09T06:17:20.391158+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:33.297395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":635,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.389923+0000","last_change":"2023-10-09T06:13:33.704835+0000","last_active":"2023-10-09T06:17:20.389923+0000","last_peered":"2023-10-09T06:17:20.389923+0000","last_clean":"2023-10-09T06:17:20.389923+0000","last_became_active":"2023-10-09T06:13:33.704466+0000","last_became_peered":"2023-10-09T06:13:33.704466+0000","last_unstale":"2023-10-09T06:17:20.389923+0000","last_undegraded":"2023-10-09T06:17:20.389923+0000","last_fullsized":"2023-10-09T06:17:20.389923+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:35.783434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:35.783434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:38.721974+0000","scrub_duration":0.0043794799999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":381,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.393992+0000","last_change":"2023-10-09T06:13:12.238280+0000","last_active":"2023-10-09T06:17:20.393992+0000","last_peered":"2023-10-09T06:17:20.393992+0000","last_clean":"2023-10-09T06:17:20.393992+0000","last_became_active":"2023-10-09T06:13:12.237986+0000","last_became_peered":"2023-10-09T06:13:12.237986+0000","last_unstale":"2023-10-09T06:17:20.393992+0000","last_undegraded":"2023-10-09T06:17:20.393992+0000","last_fullsized":"2023-10-09T06:17:20.393992+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:58.520176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":522,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.403899+0000","last_change":"2023-10-09T06:13:21.733940+0000","last_active":"2023-10-09T06:17:20.403899+0000","last_peered":"2023-10-09T06:17:20.403899+0000","last_clean":"2023-10-09T06:17:20.403899+0000","last_became_active":"2023-10-09T06:13:21.733628+0000","last_became_peered":"2023-10-09T06:13:21.733628+0000","last_unstale":"2023-10-09T06:17:20.403899+0000","last_undegraded":"2023-10-09T06:17:20.403899+0000","last_fullsized":"2023-10-09T06:17:20.403899+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:13.348443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"592'6530","reported_seq":6559,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-10-09T06:17:35.743767+0000","last_change":"2023-10-09T06:16:02.752768+0000","last_active":"2023-10-09T06:17:35.743767+0000","last_peered":"2023-10-09T06:17:35.743767+0000","last_clean":"2023-10-09T06:17:35.743767+0000","last_became_active":"2023-10-09T06:16:02.752026+0000","last_became_peered":"2023-10-09T06:16:02.752026+0000","last_unstale":"2023-10-09T06:17:35.743767+0000","last_undegraded":"2023-10-09T06:17:35.743767+0000","last_fullsized":"2023-10-09T06:17:35.743767+0000","mapping_epoch":572,"log_start":"578'700","ondisk_log_start":"578'700","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5830,"log_dups_size":0,"ondisk_log_size":5830,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:30.737816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":427950080,"num_objects":6530,"num_object_clones":0,"num_object_copies":19590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6530,"num_write_kb":417920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":862,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.400249+0000","last_change":"2023-10-09T06:13:24.753842+0000","last_active":"2023-10-09T06:17:20.400249+0000","last_peered":"2023-10-09T06:17:20.400249+0000","last_clean":"2023-10-09T06:17:20.400249+0000","last_became_active":"2023-10-09T06:13:24.753485+0000","last_became_peered":"2023-10-09T06:13:24.753485+0000","last_unstale":"2023-10-09T06:17:20.400249+0000","last_undegraded":"2023-10-09T06:17:20.400249+0000","last_fullsized":"2023-10-09T06:17:20.400249+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:57.627296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:57.627296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:56.656218+0000","scrub_duration":0.001913166,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":485,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.390046+0000","last_change":"2023-10-09T06:09:27.544373+0000","last_active":"2023-10-09T06:17:20.390046+0000","last_peered":"2023-10-09T06:17:20.390046+0000","last_clean":"2023-10-09T06:17:20.390046+0000","last_became_active":"2023-10-09T06:09:27.544121+0000","last_became_peered":"2023-10-09T06:09:27.544121+0000","last_unstale":"2023-10-09T06:17:20.390046+0000","last_undegraded":"2023-10-09T06:17:20.390046+0000","last_fullsized":"2023-10-09T06:17:20.390046+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:40.429491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":530,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.383004+0000","last_change":"2023-10-09T06:12:52.670573+0000","last_active":"2023-10-09T06:17:20.383004+0000","last_peered":"2023-10-09T06:17:20.383004+0000","last_clean":"2023-10-09T06:17:20.383004+0000","last_became_active":"2023-10-09T06:12:52.670242+0000","last_became_peered":"2023-10-09T06:12:52.670242+0000","last_unstale":"2023-10-09T06:17:20.383004+0000","last_undegraded":"2023-10-09T06:17:20.383004+0000","last_fullsized":"2023-10-09T06:17:20.383004+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:25.237580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":678,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.381094+0000","last_change":"2023-10-09T06:15:16.805514+0000","last_active":"2023-10-09T06:17:20.381094+0000","last_peered":"2023-10-09T06:17:20.381094+0000","last_clean":"2023-10-09T06:17:20.381094+0000","last_became_active":"2023-10-09T06:15:16.805160+0000","last_became_peered":"2023-10-09T06:15:16.805160+0000","last_unstale":"2023-10-09T06:17:20.381094+0000","last_undegraded":"2023-10-09T06:17:20.381094+0000","last_fullsized":"2023-10-09T06:17:20.381094+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:30.153699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"592'6674","reported_seq":6702,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-10-09T06:17:37.051805+0000","last_change":"2023-10-09T06:16:02.755231+0000","last_active":"2023-10-09T06:17:37.051805+0000","last_peered":"2023-10-09T06:17:37.051805+0000","last_clean":"2023-10-09T06:17:37.051805+0000","last_became_active":"2023-10-09T06:16:02.754877+0000","last_became_peered":"2023-10-09T06:16:02.754877+0000","last_unstale":"2023-10-09T06:17:37.051805+0000","last_undegraded":"2023-10-09T06:17:37.051805+0000","last_fullsized":"2023-10-09T06:17:37.051805+0000","mapping_epoch":572,"log_start":"579'1500","ondisk_log_start":"579'1500","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5174,"log_dups_size":0,"ondisk_log_size":5174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:16:25.397496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":437387264,"num_objects":6674,"num_object_clones":0,"num_object_copies":20022,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6674,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6674,"num_write_kb":427136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":793,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.855933+0000","last_change":"2023-10-09T06:13:24.756288+0000","last_active":"2023-10-09T06:17:20.855933+0000","last_peered":"2023-10-09T06:17:20.855933+0000","last_clean":"2023-10-09T06:17:20.855933+0000","last_became_active":"2023-10-09T06:13:24.755945+0000","last_became_peered":"2023-10-09T06:13:24.755945+0000","last_unstale":"2023-10-09T06:17:20.855933+0000","last_undegraded":"2023-10-09T06:17:20.855933+0000","last_fullsized":"2023-10-09T06:17:20.855933+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:45.844892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:45.844892+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:44.107376+0000","scrub_duration":0.0017521489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":75,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.390878+0000","last_change":"2023-10-09T06:13:30.799535+0000","last_active":"2023-10-09T06:17:20.390878+0000","last_peered":"2023-10-09T06:17:20.390878+0000","last_clean":"2023-10-09T06:17:20.390878+0000","last_became_active":"2023-10-09T06:13:30.799227+0000","last_became_peered":"2023-10-09T06:13:30.799227+0000","last_unstale":"2023-10-09T06:17:20.390878+0000","last_undegraded":"2023-10-09T06:17:20.390878+0000","last_fullsized":"2023-10-09T06:17:20.390878+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:14.741742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":321,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.404551+0000","last_change":"2023-10-09T06:15:16.810272+0000","last_active":"2023-10-09T06:17:20.404551+0000","last_peered":"2023-10-09T06:17:20.404551+0000","last_clean":"2023-10-09T06:17:20.404551+0000","last_became_active":"2023-10-09T06:15:16.804086+0000","last_became_peered":"2023-10-09T06:15:16.804086+0000","last_unstale":"2023-10-09T06:17:20.404551+0000","last_undegraded":"2023-10-09T06:17:20.404551+0000","last_fullsized":"2023-10-09T06:17:20.404551+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:45.156184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":75,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.390997+0000","last_change":"2023-10-09T06:13:30.788382+0000","last_active":"2023-10-09T06:17:20.390997+0000","last_peered":"2023-10-09T06:17:20.390997+0000","last_clean":"2023-10-09T06:17:20.390997+0000","last_became_active":"2023-10-09T06:13:30.788060+0000","last_became_peered":"2023-10-09T06:13:30.788060+0000","last_unstale":"2023-10-09T06:17:20.390997+0000","last_undegraded":"2023-10-09T06:17:20.390997+0000","last_fullsized":"2023-10-09T06:17:20.390997+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:36:58.984799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.f","version":"592'6541","reported_seq":6569,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-10-09T06:17:34.368220+0000","last_change":"2023-10-09T06:16:02.755513+0000","last_active":"2023-10-09T06:17:34.368220+0000","last_peered":"2023-10-09T06:17:34.368220+0000","last_clean":"2023-10-09T06:17:34.368220+0000","last_became_active":"2023-10-09T06:16:02.755072+0000","last_became_peered":"2023-10-09T06:16:02.755072+0000","last_unstale":"2023-10-09T06:17:34.368220+0000","last_undegraded":"2023-10-09T06:17:34.368220+0000","last_fullsized":"2023-10-09T06:17:34.368220+0000","mapping_epoch":572,"log_start":"578'700","ondisk_log_start":"578'700","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5841,"log_dups_size":0,"ondisk_log_size":5841,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:11.106599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":428670976,"num_objects":6541,"num_object_clones":0,"num_object_copies":19623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6541,"num_write_kb":418624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":1085,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.404100+0000","last_change":"2023-10-09T06:15:19.869123+0000","last_active":"2023-10-09T06:17:20.404100+0000","last_peered":"2023-10-09T06:17:20.404100+0000","last_clean":"2023-10-09T06:17:20.404100+0000","last_became_active":"2023-10-09T06:15:19.868795+0000","last_became_peered":"2023-10-09T06:15:19.868795+0000","last_unstale":"2023-10-09T06:17:20.404100+0000","last_undegraded":"2023-10-09T06:17:20.404100+0000","last_fullsized":"2023-10-09T06:17:20.404100+0000","mapping_epoch":561,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":562,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:17.344518+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:17.344518+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:34.303488+0000","scrub_duration":0.0015914099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":650,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-10-09T06:17:28.844457+0000","last_change":"2023-10-09T06:13:30.798394+0000","last_active":"2023-10-09T06:17:28.844457+0000","last_peered":"2023-10-09T06:17:28.844457+0000","last_clean":"2023-10-09T06:17:28.844457+0000","last_became_active":"2023-10-09T06:13:30.790913+0000","last_became_peered":"2023-10-09T06:13:30.790913+0000","last_unstale":"2023-10-09T06:17:28.844457+0000","last_undegraded":"2023-10-09T06:17:28.844457+0000","last_fullsized":"2023-10-09T06:17:28.844457+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:26.508690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":410,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.400686+0000","last_change":"2023-10-09T06:13:21.728009+0000","last_active":"2023-10-09T06:17:20.400686+0000","last_peered":"2023-10-09T06:17:20.400686+0000","last_clean":"2023-10-09T06:17:20.400686+0000","last_became_active":"2023-10-09T06:13:21.727683+0000","last_became_peered":"2023-10-09T06:13:21.727683+0000","last_unstale":"2023-10-09T06:17:20.400686+0000","last_undegraded":"2023-10-09T06:17:20.400686+0000","last_fullsized":"2023-10-09T06:17:20.400686+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:13.074232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":399,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-10-09T06:17:28.841472+0000","last_change":"2023-10-09T06:13:30.782038+0000","last_active":"2023-10-09T06:17:28.841472+0000","last_peered":"2023-10-09T06:17:28.841472+0000","last_clean":"2023-10-09T06:17:28.841472+0000","last_became_active":"2023-10-09T06:13:30.781500+0000","last_became_peered":"2023-10-09T06:13:30.781500+0000","last_unstale":"2023-10-09T06:17:28.841472+0000","last_undegraded":"2023-10-09T06:17:28.841472+0000","last_fullsized":"2023-10-09T06:17:28.841472+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:43.663265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"592'6711","reported_seq":6739,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-10-09T06:17:36.043269+0000","last_change":"2023-10-09T06:16:02.755624+0000","last_active":"2023-10-09T06:17:36.043269+0000","last_peered":"2023-10-09T06:17:36.043269+0000","last_clean":"2023-10-09T06:17:36.043269+0000","last_became_active":"2023-10-09T06:16:02.755333+0000","last_became_peered":"2023-10-09T06:16:02.755333+0000","last_unstale":"2023-10-09T06:17:36.043269+0000","last_undegraded":"2023-10-09T06:17:36.043269+0000","last_fullsized":"2023-10-09T06:17:36.043269+0000","mapping_epoch":572,"log_start":"578'800","ondisk_log_start":"578'800","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5911,"log_dups_size":0,"ondisk_log_size":5911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:59.742519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":439812096,"num_objects":6711,"num_object_clones":0,"num_object_copies":20133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6711,"num_write_kb":429504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":925,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.391028+0000","last_change":"2023-10-09T06:13:33.708100+0000","last_active":"2023-10-09T06:17:20.391028+0000","last_peered":"2023-10-09T06:17:20.391028+0000","last_clean":"2023-10-09T06:17:20.391028+0000","last_became_active":"2023-10-09T06:13:33.707809+0000","last_became_peered":"2023-10-09T06:13:33.707809+0000","last_unstale":"2023-10-09T06:17:20.391028+0000","last_undegraded":"2023-10-09T06:17:20.391028+0000","last_fullsized":"2023-10-09T06:17:20.391028+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:43:26.848778+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":85,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.390966+0000","last_change":"2023-10-09T06:16:02.760976+0000","last_active":"2023-10-09T06:17:20.390966+0000","last_peered":"2023-10-09T06:17:20.390966+0000","last_clean":"2023-10-09T06:17:20.390966+0000","last_became_active":"2023-10-09T06:16:02.760646+0000","last_became_peered":"2023-10-09T06:16:02.760646+0000","last_unstale":"2023-10-09T06:17:20.390966+0000","last_undegraded":"2023-10-09T06:17:20.390966+0000","last_fullsized":"2023-10-09T06:17:20.390966+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:03.779978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":639,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.381343+0000","last_change":"2023-10-09T06:09:27.578756+0000","last_active":"2023-10-09T06:17:20.381343+0000","last_peered":"2023-10-09T06:17:20.381343+0000","last_clean":"2023-10-09T06:17:20.381343+0000","last_became_active":"2023-10-09T06:09:27.578384+0000","last_became_peered":"2023-10-09T06:09:27.578384+0000","last_unstale":"2023-10-09T06:17:20.381343+0000","last_undegraded":"2023-10-09T06:17:20.381343+0000","last_fullsized":"2023-10-09T06:17:20.381343+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:22.182630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":538,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.403430+0000","last_change":"2023-10-09T06:01:22.334081+0000","last_active":"2023-10-09T06:17:20.403430+0000","last_peered":"2023-10-09T06:17:20.403430+0000","last_clean":"2023-10-09T06:17:20.403430+0000","last_became_active":"2023-10-09T06:01:22.333663+0000","last_became_peered":"2023-10-09T06:01:22.333663+0000","last_unstale":"2023-10-09T06:17:20.403430+0000","last_undegraded":"2023-10-09T06:17:20.403430+0000","last_fullsized":"2023-10-09T06:17:20.403430+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:55.809259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":839,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.391129+0000","last_change":"2023-10-09T06:09:27.560157+0000","last_active":"2023-10-09T06:17:20.391129+0000","last_peered":"2023-10-09T06:17:20.391129+0000","last_clean":"2023-10-09T06:17:20.391129+0000","last_became_active":"2023-10-09T06:09:27.559466+0000","last_became_peered":"2023-10-09T06:09:27.559466+0000","last_unstale":"2023-10-09T06:17:20.391129+0000","last_undegraded":"2023-10-09T06:17:20.391129+0000","last_fullsized":"2023-10-09T06:17:20.391129+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.181811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.181811+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:53.431491+0000","scrub_duration":0.011553313000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":522,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.384742+0000","last_change":"2023-10-09T06:13:30.791343+0000","last_active":"2023-10-09T06:17:20.384742+0000","last_peered":"2023-10-09T06:17:20.384742+0000","last_clean":"2023-10-09T06:17:20.384742+0000","last_became_active":"2023-10-09T06:13:30.790945+0000","last_became_peered":"2023-10-09T06:13:30.790945+0000","last_unstale":"2023-10-09T06:17:20.384742+0000","last_undegraded":"2023-10-09T06:17:20.384742+0000","last_fullsized":"2023-10-09T06:17:20.384742+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:14.208391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":304,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.404602+0000","last_change":"2023-10-09T06:15:16.803457+0000","last_active":"2023-10-09T06:17:20.404602+0000","last_peered":"2023-10-09T06:17:20.404602+0000","last_clean":"2023-10-09T06:17:20.404602+0000","last_became_active":"2023-10-09T06:15:16.803097+0000","last_became_peered":"2023-10-09T06:15:16.803097+0000","last_unstale":"2023-10-09T06:17:20.404602+0000","last_undegraded":"2023-10-09T06:17:20.404602+0000","last_fullsized":"2023-10-09T06:17:20.404602+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:45.511763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":942,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.394470+0000","last_change":"2023-10-09T06:00:04.215425+0000","last_active":"2023-10-09T06:17:20.394470+0000","last_peered":"2023-10-09T06:17:20.394470+0000","last_clean":"2023-10-09T06:17:20.394470+0000","last_became_active":"2023-10-09T05:59:40.326685+0000","last_became_peered":"2023-10-09T05:59:40.326685+0000","last_unstale":"2023-10-09T06:17:20.394470+0000","last_undegraded":"2023-10-09T06:17:20.394470+0000","last_fullsized":"2023-10-09T06:17:20.394470+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:04.215270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:04.215270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:03.963694+0000","scrub_duration":0.002850652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":363,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.403736+0000","last_change":"2023-10-09T06:01:54.824780+0000","last_active":"2023-10-09T06:17:20.403736+0000","last_peered":"2023-10-09T06:17:20.403736+0000","last_clean":"2023-10-09T06:17:20.403736+0000","last_became_active":"2023-10-09T06:01:54.757727+0000","last_became_peered":"2023-10-09T06:01:54.757727+0000","last_unstale":"2023-10-09T06:17:20.403736+0000","last_undegraded":"2023-10-09T06:17:20.403736+0000","last_fullsized":"2023-10-09T06:17:20.403736+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:44.132556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":610,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.403553+0000","last_change":"2023-10-09T06:01:54.764989+0000","last_active":"2023-10-09T06:17:20.403553+0000","last_peered":"2023-10-09T06:17:20.403553+0000","last_clean":"2023-10-09T06:17:20.403553+0000","last_became_active":"2023-10-09T06:01:54.764790+0000","last_became_peered":"2023-10-09T06:01:54.764790+0000","last_unstale":"2023-10-09T06:17:20.403553+0000","last_undegraded":"2023-10-09T06:17:20.403553+0000","last_fullsized":"2023-10-09T06:17:20.403553+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:14:07.208858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":377,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.403640+0000","last_change":"2023-10-09T06:13:21.739730+0000","last_active":"2023-10-09T06:17:20.403640+0000","last_peered":"2023-10-09T06:17:20.403640+0000","last_clean":"2023-10-09T06:17:20.403640+0000","last_became_active":"2023-10-09T06:13:21.727326+0000","last_became_peered":"2023-10-09T06:13:21.727326+0000","last_unstale":"2023-10-09T06:17:20.403640+0000","last_undegraded":"2023-10-09T06:17:20.403640+0000","last_fullsized":"2023-10-09T06:17:20.403640+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:51.344571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":607,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.403486+0000","last_change":"2023-10-09T06:09:27.565198+0000","last_active":"2023-10-09T06:17:20.403486+0000","last_peered":"2023-10-09T06:17:20.403486+0000","last_clean":"2023-10-09T06:17:20.403486+0000","last_became_active":"2023-10-09T06:09:27.564909+0000","last_became_peered":"2023-10-09T06:09:27.564909+0000","last_unstale":"2023-10-09T06:17:20.403486+0000","last_undegraded":"2023-10-09T06:17:20.403486+0000","last_fullsized":"2023-10-09T06:17:20.403486+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:46.101014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":757,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.390643+0000","last_change":"2023-10-09T06:13:33.699235+0000","last_active":"2023-10-09T06:17:20.390643+0000","last_peered":"2023-10-09T06:17:20.390643+0000","last_clean":"2023-10-09T06:17:20.390643+0000","last_became_active":"2023-10-09T06:13:33.698886+0000","last_became_peered":"2023-10-09T06:13:33.698886+0000","last_unstale":"2023-10-09T06:17:20.390643+0000","last_undegraded":"2023-10-09T06:17:20.390643+0000","last_fullsized":"2023-10-09T06:17:20.390643+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:00.456132+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:00.456132+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:59.421706+0000","scrub_duration":0.0014992199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":409,"reported_epoch":592,"state":"active+clean","last_fresh":"2023-10-09T06:17:28.843115+0000","last_change":"2023-10-09T06:15:16.804361+0000","last_active":"2023-10-09T06:17:28.843115+0000","last_peered":"2023-10-09T06:17:28.843115+0000","last_clean":"2023-10-09T06:17:28.843115+0000","last_became_active":"2023-10-09T06:15:16.803977+0000","last_became_peered":"2023-10-09T06:15:16.803977+0000","last_unstale":"2023-10-09T06:17:28.843115+0000","last_undegraded":"2023-10-09T06:17:28.843115+0000","last_fullsized":"2023-10-09T06:17:28.843115+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:14.141738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":404,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.392851+0000","last_change":"2023-10-09T06:16:02.759834+0000","last_active":"2023-10-09T06:17:20.392851+0000","last_peered":"2023-10-09T06:17:20.392851+0000","last_clean":"2023-10-09T06:17:20.392851+0000","last_became_active":"2023-10-09T06:16:02.759505+0000","last_became_peered":"2023-10-09T06:16:02.759505+0000","last_unstale":"2023-10-09T06:17:20.392851+0000","last_undegraded":"2023-10-09T06:17:20.392851+0000","last_fullsized":"2023-10-09T06:17:20.392851+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:46.034378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":917,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.399346+0000","last_change":"2023-10-09T06:12:55.753877+0000","last_active":"2023-10-09T06:17:20.399346+0000","last_peered":"2023-10-09T06:17:20.399346+0000","last_clean":"2023-10-09T06:17:20.399346+0000","last_became_active":"2023-10-09T06:12:55.753512+0000","last_became_peered":"2023-10-09T06:12:55.753512+0000","last_unstale":"2023-10-09T06:17:20.399346+0000","last_undegraded":"2023-10-09T06:17:20.399346+0000","last_fullsized":"2023-10-09T06:17:20.399346+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:46.290510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:46.290510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:30.368031+0000","scrub_duration":0.002627819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":600,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.381297+0000","last_change":"2023-10-09T06:15:16.814498+0000","last_active":"2023-10-09T06:17:20.381297+0000","last_peered":"2023-10-09T06:17:20.381297+0000","last_clean":"2023-10-09T06:17:20.381297+0000","last_became_active":"2023-10-09T06:15:16.807217+0000","last_became_peered":"2023-10-09T06:15:16.807217+0000","last_unstale":"2023-10-09T06:17:20.381297+0000","last_undegraded":"2023-10-09T06:17:20.381297+0000","last_fullsized":"2023-10-09T06:17:20.381297+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:50.886043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":623,"reported_epoch":591,"state":"active+clean","last_fresh":"2023-10-09T06:17:20.403211+0000","last_change":"2023-10-09T06:13:21.727720+0000","last_active":"2023-10-09T06:17:20.403211+0000","last_peered":"2023-10-09T06:17:20.403211+0000","last_clean":"2023-10-09T06:17:20.403211+0000","last_became_active":"2023-10-09T06:13:21.727339+0000","last_became_peered":"2023-10-09T06:13:21.727339+0000","last_unstale":"2023-10-09T06:17:20.403211+0000","last_undegraded":"2023-10-09T06:17:20.403211+0000","last_fullsized":"2023-10-09T06:17:20.403211+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:25.723752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":6898843648,"num_objects":105268,"num_object_clones":0,"num_object_copies":315804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":105268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":105268,"num_write_kb":6737152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20670971904,"data_stored":20670971904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90868,"ondisk_log_size":90868,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":61,"num_bytes_recovered":12302,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13410,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1706112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234455,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4241288,"kb_used_data":3352768,"kb_used_omap":0,"kb_used_meta":888512,"kb_avail":89503864,"statfs":{"total":95995035648,"available":91651956736,"internally_reserved":0,"allocated":3433234432,"data_stored":3431637890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":909836288},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:17:34 2023","interfaces":[{"interface":"back","average":{"1min":4.774,"5min":8.8079999999999998,"15min":3.8109999999999999},"min":{"1min":0.23899999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":83.269000000000005,"5min":166.94,"15min":166.94},"last":2.0139999999999998},{"interface":"front","average":{"1min":5.6420000000000003,"5min":9.048,"15min":3.907},"min":{"1min":0.33100000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":83.253,"5min":166.923,"15min":166.923},"last":0.41699999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:17:08 2023","interfaces":[{"interface":"back","average":{"1min":1.9910000000000001,"5min":4.6840000000000002,"15min":4.226},"min":{"1min":0.255,"5min":0.252,"15min":0.218},"max":{"1min":10.122,"5min":108.113,"15min":124.66200000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.734,"5min":4.4950000000000001,"15min":4.1699999999999999},"min":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.22700000000000001},"max":{"1min":6.016,"5min":108.503,"15min":125.023},"last":0.33700000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:17:34 2023","interfaces":[{"interface":"back","average":{"1min":1.552,"5min":2.4500000000000002,"15min":1.837},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.24299999999999999},"max":{"1min":3.452,"5min":47.503,"15min":47.503},"last":2.0430000000000001},{"interface":"front","average":{"1min":2.3199999999999998,"5min":2.2160000000000002,"15min":1.917},"min":{"1min":0.32400000000000001,"5min":0.29899999999999999,"15min":0.27000000000000002},"max":{"1min":7.3230000000000004,"5min":47.545000000000002,"15min":47.545000000000002},"last":0.37}]},{"osd":3,"last update":"Mon Oct 9 06:17:08 2023","interfaces":[{"interface":"back","average":{"1min":2.0720000000000001,"5min":5.1239999999999997,"15min":4.351},"min":{"1min":0.37,"5min":0.33800000000000002,"15min":0.29899999999999999},"max":{"1min":6.6070000000000002,"5min":67.852000000000004,"15min":116.68000000000001},"last":47.110999999999997},{"interface":"front","average":{"1min":1.462,"5min":4.6929999999999996,"15min":4.4119999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.32100000000000001},"max":{"1min":5.5129999999999999,"5min":67.881,"15min":116.797},"last":47.563000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:16:48 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":1.0529999999999999,"15min":1.014},"min":{"1min":0.29099999999999998,"5min":0.24299999999999999,"15min":0.23100000000000001},"max":{"1min":5.4029999999999996,"5min":10.257,"15min":14.371},"last":2.0640000000000001},{"interface":"front","average":{"1min":0.83399999999999996,"5min":0.98499999999999999,"15min":1.0029999999999999},"min":{"1min":0.27800000000000002,"5min":0.246,"15min":0.246},"max":{"1min":4.5599999999999996,"5min":12.583,"15min":14.259},"last":3.198}]},{"osd":5,"last update":"Mon Oct 9 06:16:57 2023","interfaces":[{"interface":"back","average":{"1min":10.836,"5min":2.9039999999999999,"15min":1.782},"min":{"1min":0.29899999999999999,"5min":0.27100000000000002,"15min":0.24399999999999999},"max":{"1min":216.036,"5min":216.036,"15min":216.036},"last":0.442},{"interface":"front","average":{"1min":10.542999999999999,"5min":2.8580000000000001,"15min":1.829},"min":{"1min":0.32600000000000001,"5min":0.27800000000000002,"15min":0.24399999999999999},"max":{"1min":215.80600000000001,"5min":215.80600000000001,"15min":215.80600000000001},"last":0.40100000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:16:57 2023","interfaces":[{"interface":"back","average":{"1min":11.539999999999999,"5min":3.2120000000000002,"15min":1.784},"min":{"1min":0.34200000000000003,"5min":0.308,"15min":0.26600000000000001},"max":{"1min":233.56700000000001,"5min":233.56700000000001,"15min":233.56700000000001},"last":0.57399999999999995},{"interface":"front","average":{"1min":11.452,"5min":2.968,"15min":1.7230000000000001},"min":{"1min":0.34100000000000003,"5min":0.32000000000000001,"15min":0.30599999999999999},"max":{"1min":232.477,"5min":232.477,"15min":232.477},"last":3.2989999999999999}]}]},{"osd":6,"up_from":46,"seq":197568496092,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4226492,"kb_used_data":3328948,"kb_used_omap":0,"kb_used_meta":897536,"kb_avail":89518660,"statfs":{"total":95995035648,"available":91667107840,"internally_reserved":0,"allocated":3408842752,"data_stored":3407258131,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919076864},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:17:32 2023","interfaces":[{"interface":"back","average":{"1min":1.599,"5min":7.6340000000000003,"15min":4.5650000000000004},"min":{"1min":0.29299999999999998,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":4.3250000000000002,"5min":181.209,"15min":191.50800000000001},"last":0.41799999999999998},{"interface":"front","average":{"1min":2.052,"5min":8.1069999999999993,"15min":4.5999999999999996},"min":{"1min":0.30299999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.359,"5min":181.233,"15min":191.52600000000001},"last":3.4420000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:17:32 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":9.2739999999999991,"15min":6.2300000000000004},"min":{"1min":0.28499999999999998,"5min":0.23799999999999999,"15min":0.23200000000000001},"max":{"1min":5.6539999999999999,"5min":328.04399999999998,"15min":328.04399999999998},"last":0.28799999999999998},{"interface":"front","average":{"1min":1.27,"5min":8.5619999999999994,"15min":5.9450000000000003},"min":{"1min":0.25800000000000001,"5min":0.253,"15min":0.253},"max":{"1min":3.8290000000000002,"5min":328.10000000000002,"15min":328.10000000000002},"last":0.44900000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:17:32 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":4.6950000000000003,"15min":2.7400000000000002},"min":{"1min":0.35399999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.3109999999999999,"5min":163.04900000000001,"15min":163.04900000000001},"last":3.504},{"interface":"front","average":{"1min":1.2150000000000001,"5min":4.6870000000000003,"15min":2.681},"min":{"1min":0.318,"5min":0.29299999999999998,"15min":0.28599999999999998},"max":{"1min":3.8620000000000001,"5min":162.93000000000001,"15min":162.93000000000001},"last":0.48299999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:17:19 2023","interfaces":[{"interface":"back","average":{"1min":1.3959999999999999,"5min":7.2939999999999996,"15min":6.3019999999999996},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.024,"5min":250.744,"15min":250.744},"last":0.46600000000000003},{"interface":"front","average":{"1min":2.4780000000000002,"5min":7.0830000000000002,"15min":6.1580000000000004},"min":{"1min":0.45500000000000002,"5min":0.309,"15min":0.309},"max":{"1min":7.8600000000000003,"5min":250.83799999999999,"15min":250.83799999999999},"last":0.50700000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:16:52 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.91200000000000003,"15min":1.5700000000000001},"min":{"1min":0.251,"5min":0.24399999999999999,"15min":0.24199999999999999},"max":{"1min":0.97999999999999998,"5min":9.0869999999999997,"15min":151.35599999999999},"last":5.2350000000000003},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.3089999999999999,"15min":1.7689999999999999},"min":{"1min":0.315,"5min":0.25900000000000001,"15min":0.23999999999999999},"max":{"1min":6.9889999999999999,"5min":8.1780000000000008,"15min":149.15799999999999},"last":3.5870000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:17:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":0.91700000000000004,"15min":1.0860000000000001},"min":{"1min":0.38200000000000001,"5min":0.29699999999999999,"15min":0.23000000000000001},"max":{"1min":7.4429999999999996,"5min":8.3829999999999991,"15min":15.342000000000001},"last":2.806},{"interface":"front","average":{"1min":1.3440000000000001,"5min":1.1339999999999999,"15min":1.214},"min":{"1min":0.33000000000000002,"5min":0.312,"15min":0.27700000000000002},"max":{"1min":7.1769999999999996,"5min":8.7289999999999992,"15min":15.358000000000001},"last":0.49199999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:16:52 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":1.0980000000000001,"15min":1.2190000000000001},"min":{"1min":0.37,"5min":0.30299999999999999,"15min":0.29399999999999998},"max":{"1min":1.841,"5min":5.8380000000000001,"15min":21.98},"last":3.5369999999999999},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.94699999999999995,"15min":1.1930000000000001},"min":{"1min":0.37,"5min":0.34999999999999998,"15min":0.33100000000000002},"max":{"1min":1.536,"5min":5.6769999999999996,"15min":18.241},"last":0.52400000000000002}]}]},{"osd":1,"up_from":89,"seq":382252089680,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5018988,"kb_used_data":4274084,"kb_used_omap":9,"kb_used_meta":744886,"kb_avail":88726164,"statfs":{"total":95995035648,"available":90855591936,"internally_reserved":0,"allocated":4376662016,"data_stored":4373331578,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":762764265},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:45 2023","interfaces":[{"interface":"back","average":{"1min":6.3220000000000001,"5min":3.831,"15min":2.8039999999999998},"min":{"1min":0.214,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":118.17,"5min":118.17,"15min":118.17},"last":0.66900000000000004},{"interface":"front","average":{"1min":6.532,"5min":4.1890000000000001,"15min":3.0209999999999999},"min":{"1min":0.27600000000000002,"5min":0.20699999999999999,"15min":0.20200000000000001},"max":{"1min":117.634,"5min":117.634,"15min":117.634},"last":4.1120000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:16:48 2023","interfaces":[{"interface":"back","average":{"1min":0.81100000000000005,"5min":1.4610000000000001,"15min":1.6639999999999999},"min":{"1min":0.251,"5min":0.217,"15min":0.20799999999999999},"max":{"1min":4.2009999999999996,"5min":12.818,"15min":17.681000000000001},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.59299999999999997,"5min":2.0569999999999999,"15min":2.0699999999999998},"min":{"1min":0.23499999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":3.7669999999999999,"5min":55.694000000000003,"15min":55.694000000000003},"last":3.8999999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:17:19 2023","interfaces":[{"interface":"back","average":{"1min":12.894,"5min":5.2130000000000001,"15min":4.6660000000000004},"min":{"1min":0.373,"5min":0.26800000000000002,"15min":0.24199999999999999},"max":{"1min":95.489999999999995,"5min":95.489999999999995,"15min":142.97},"last":5.5199999999999996},{"interface":"front","average":{"1min":12.641,"5min":4.9199999999999999,"15min":4.6230000000000002},"min":{"1min":0.45100000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":95.436999999999998,"5min":95.436999999999998,"15min":142.94800000000001},"last":4.1509999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:16:48 2023","interfaces":[{"interface":"back","average":{"1min":4.2009999999999996,"5min":3.9319999999999999,"15min":2.5659999999999998},"min":{"1min":0.309,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":66.084000000000003,"5min":93.747,"15min":93.747},"last":3.8570000000000002},{"interface":"front","average":{"1min":4.6150000000000002,"5min":4.3719999999999999,"15min":3.0179999999999998},"min":{"1min":0.34399999999999997,"5min":0.32800000000000001,"15min":0.308},"max":{"1min":66.399000000000001,"5min":97.852000000000004,"15min":97.852000000000004},"last":0.63}]},{"osd":5,"last update":"Mon Oct 9 06:17:03 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":1.081,"15min":1.5840000000000001},"min":{"1min":0.378,"5min":0.312,"15min":0.30499999999999999},"max":{"1min":3.9849999999999999,"5min":6.1820000000000004,"15min":36.356999999999999},"last":3.9630000000000001},{"interface":"front","average":{"1min":1.262,"5min":1.6759999999999999,"15min":2.3149999999999999},"min":{"1min":0.39000000000000001,"5min":0.318,"15min":0.29799999999999999},"max":{"1min":4.0819999999999999,"5min":14.648,"15min":36.268000000000001},"last":0.59699999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:17:03 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":1.159,"15min":1.804},"min":{"1min":0.34599999999999997,"5min":0.33700000000000002,"15min":0.33100000000000002},"max":{"1min":3.9039999999999999,"5min":6.218,"15min":52.207000000000001},"last":3.9300000000000002},{"interface":"front","average":{"1min":1.6699999999999999,"5min":1.7470000000000001,"15min":2.379},"min":{"1min":0.38500000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":4.8289999999999997,"5min":14.619999999999999,"15min":51.212000000000003},"last":0.56399999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:17:03 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":2.5379999999999998,"15min":2.6379999999999999},"min":{"1min":0.42899999999999999,"5min":0.374,"15min":0.374},"max":{"1min":4.0190000000000001,"5min":56.881,"15min":56.881},"last":0.65600000000000003},{"interface":"front","average":{"1min":1.0649999999999999,"5min":2.129,"15min":2.093},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.38},"max":{"1min":4.5469999999999997,"5min":56.941000000000003,"15min":56.941000000000003},"last":0.89000000000000001}]}]},{"osd":0,"up_from":456,"seq":1958505087075,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2906808,"kb_used_data":2514288,"kb_used_omap":32,"kb_used_meta":392479,"kb_avail":90838344,"statfs":{"total":95995035648,"available":93018464256,"internally_reserved":0,"allocated":2574630912,"data_stored":2571528342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32794,"internal_metadata":401899494},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:16:45 2023","interfaces":[{"interface":"back","average":{"1min":1.335,"5min":4.2400000000000002,"15min":8.3989999999999991},"min":{"1min":0.22800000000000001,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":5.109,"5min":102.81999999999999,"15min":166.833},"last":0.26200000000000001},{"interface":"front","average":{"1min":1.665,"5min":4.5529999999999999,"15min":7.931},"min":{"1min":0.28499999999999998,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":7.7469999999999999,"5min":103.108,"15min":166.81100000000001},"last":0.249}]},{"osd":2,"last update":"Mon Oct 9 06:16:45 2023","interfaces":[{"interface":"back","average":{"1min":1.2010000000000001,"5min":2.0179999999999998,"15min":3.2490000000000001},"min":{"1min":0.25700000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":5.2930000000000001,"5min":21.648,"15min":21.648},"last":1.895},{"interface":"front","average":{"1min":1.05,"5min":1.931,"15min":3.1040000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.7330000000000005,"5min":11.890000000000001,"15min":20.152000000000001},"last":0.47899999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:16:37 2023","interfaces":[{"interface":"back","average":{"1min":1.218,"5min":2.0249999999999999,"15min":10.420999999999999},"min":{"1min":0.27400000000000002,"5min":0.26500000000000001,"15min":0.25600000000000001},"max":{"1min":4.7110000000000003,"5min":10.217000000000001,"15min":203.59299999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":1.244,"5min":2.452,"15min":10.032},"min":{"1min":0.29499999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":4.8339999999999996,"5min":18.192,"15min":202.96600000000001},"last":1.9399999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:16:45 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":2.069,"15min":2.2450000000000001},"min":{"1min":0.32500000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.7569999999999997,"5min":23.515999999999998,"15min":23.515999999999998},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.0189999999999999,"5min":2.423,"15min":2.181},"min":{"1min":0.39800000000000002,"5min":0.254,"15min":0.254},"max":{"1min":4.6260000000000003,"5min":39.302,"15min":39.302},"last":0.36499999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:16:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":2.0699999999999998,"15min":2.6669999999999998},"min":{"1min":0.42199999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":5.3920000000000003,"5min":39.369999999999997,"15min":161.286},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.375,"5min":1.8069999999999999,"15min":2.7280000000000002},"min":{"1min":0.34899999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.8170000000000002,"5min":13.731999999999999,"15min":161.31800000000001},"last":1.966}]},{"osd":6,"last update":"Mon Oct 9 06:16:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":2.2069999999999999,"15min":2.3900000000000001},"min":{"1min":0.40999999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.7199999999999998,"5min":31.648,"15min":31.648},"last":4.8209999999999997},{"interface":"front","average":{"1min":0.84699999999999998,"5min":1.7909999999999999,"15min":2.4039999999999999},"min":{"1min":0.36399999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":5.4320000000000004,"5min":24.43,"15min":24.43},"last":0.56200000000000006}]},{"osd":7,"last update":"Mon Oct 9 06:16:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3819999999999999,"5min":1.9350000000000001,"15min":2.7610000000000001},"min":{"1min":0.42899999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":4.7839999999999998,"5min":13.821,"15min":123.685},"last":1.998},{"interface":"front","average":{"1min":0.97099999999999997,"5min":1.5980000000000001,"15min":2.7280000000000002},"min":{"1min":0.47499999999999998,"5min":0.40999999999999998,"15min":0.40200000000000002},"max":{"1min":4.8010000000000002,"5min":10.093,"15min":122.482},"last":0.58599999999999997}]}]},{"osd":2,"up_from":0,"seq":403726926171,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":99,"seq":425201762645,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3970632,"kb_used_data":2932160,"kb_used_omap":7,"kb_used_meta":1038456,"kb_avail":89774520,"statfs":{"total":95995035648,"available":91929108480,"internally_reserved":0,"allocated":3002531840,"data_stored":2999229253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1063379560},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:37 2023","interfaces":[{"interface":"back","average":{"1min":3.6379999999999999,"5min":6.3440000000000003,"15min":3.423},"min":{"1min":0.20999999999999999,"5min":0.186,"15min":0.17299999999999999},"max":{"1min":50.491,"5min":220.45400000000001,"15min":220.45400000000001},"last":4.9939999999999998},{"interface":"front","average":{"1min":3.952,"5min":7.0679999999999996,"15min":4.6790000000000003},"min":{"1min":0.25600000000000001,"5min":0.23200000000000001,"15min":0.19600000000000001},"max":{"1min":50.173000000000002,"5min":216.44200000000001,"15min":216.44200000000001},"last":1.1699999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:17:12 2023","interfaces":[{"interface":"back","average":{"1min":3.0800000000000001,"5min":2.645,"15min":3.0720000000000001},"min":{"1min":0.223,"5min":0.20300000000000001,"15min":0.19800000000000001},"max":{"1min":9.1460000000000008,"5min":30.632000000000001,"15min":207.75299999999999},"last":5.0709999999999997},{"interface":"front","average":{"1min":3.169,"5min":3.407,"15min":3.8700000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":13.682,"5min":49.335999999999999,"15min":207.12200000000001},"last":1.5760000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:17:12 2023","interfaces":[{"interface":"back","average":{"1min":1.5029999999999999,"5min":1.8100000000000001,"15min":2.0859999999999999},"min":{"1min":0.23699999999999999,"5min":0.23400000000000001,"15min":0.223},"max":{"1min":5.0960000000000001,"5min":12.561,"15min":41.945999999999998},"last":0.624},{"interface":"front","average":{"1min":2.4769999999999999,"5min":2.2679999999999998,"15min":2.653},"min":{"1min":0.30599999999999999,"5min":0.25600000000000001,"15min":0.23899999999999999},"max":{"1min":10.624000000000001,"5min":12.864000000000001,"15min":41.475999999999999},"last":4.4630000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:16:58 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":2.5329999999999999,"15min":2.4649999999999999},"min":{"1min":0.318,"5min":0.28199999999999997,"15min":0.254},"max":{"1min":4.1020000000000003,"5min":95.739000000000004,"15min":99.876000000000005},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.964,"5min":2.7930000000000001,"15min":2.8010000000000002},"min":{"1min":0.39200000000000002,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":7.8330000000000002,"5min":97.941999999999993,"15min":99.914000000000001},"last":1.6100000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:17:12 2023","interfaces":[{"interface":"back","average":{"1min":1.7330000000000001,"5min":2.0310000000000001,"15min":2.1480000000000001},"min":{"1min":0.39800000000000002,"5min":0.32300000000000001,"15min":0.24199999999999999},"max":{"1min":7.9409999999999998,"5min":49.363,"15min":74.706000000000003},"last":5.0170000000000003},{"interface":"front","average":{"1min":2.1059999999999999,"5min":2.0750000000000002,"15min":3.3540000000000001},"min":{"1min":0.35299999999999998,"5min":0.32800000000000001,"15min":0.28799999999999998},"max":{"1min":9.3070000000000004,"5min":49.392000000000003,"15min":235.56},"last":0.56100000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:17:12 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":2.101,"15min":2.923},"min":{"1min":0.35899999999999999,"5min":0.35299999999999998,"15min":0.26100000000000001},"max":{"1min":7.9820000000000002,"5min":49.526000000000003,"15min":233.26599999999999},"last":4.9589999999999996},{"interface":"front","average":{"1min":2.0150000000000001,"5min":2.3980000000000001,"15min":3.4089999999999998},"min":{"1min":0.46300000000000002,"5min":0.35999999999999999,"15min":0.33600000000000002},"max":{"1min":7.3490000000000002,"5min":49.429000000000002,"15min":233.24100000000001},"last":0.60399999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:17:12 2023","interfaces":[{"interface":"back","average":{"1min":2.0219999999999998,"5min":2.9870000000000001,"15min":3.6059999999999999},"min":{"1min":0.374,"5min":0.36799999999999999,"15min":0.35699999999999998},"max":{"1min":5.0709999999999997,"5min":94.400000000000006,"15min":230.69399999999999},"last":0.58299999999999996},{"interface":"front","average":{"1min":2.02,"5min":3.7149999999999999,"15min":3.9700000000000002},"min":{"1min":0.42199999999999999,"5min":0.41699999999999998,"15min":0.32600000000000001},"max":{"1min":7.8730000000000002,"5min":94.195999999999998,"15min":230.67500000000001},"last":4.9349999999999996}]}]},{"osd":4,"up_from":322,"seq":1382979469508,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2727236,"kb_used_data":2112380,"kb_used_omap":35,"kb_used_meta":614812,"kb_avail":91017916,"statfs":{"total":95995035648,"available":93202345984,"internally_reserved":0,"allocated":2163077120,"data_stored":2161484383,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36552,"internal_metadata":629567800},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:34 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":2.9540000000000002,"15min":1.9690000000000001},"min":{"1min":0.26700000000000002,"5min":0.216,"15min":0.216},"max":{"1min":4.3840000000000003,"5min":81.066000000000003,"15min":81.066000000000003},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.0429999999999999,"5min":2.7170000000000001,"15min":1.9750000000000001},"min":{"1min":0.36399999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.6440000000000001,"5min":81.603999999999999,"15min":81.603999999999999},"last":0.40699999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:16:54 2023","interfaces":[{"interface":"back","average":{"1min":7.8280000000000003,"5min":4.0410000000000004,"15min":2.9710000000000001},"min":{"1min":0.28399999999999997,"5min":0.24099999999999999,"15min":0.20999999999999999},"max":{"1min":101.562,"5min":101.562,"15min":101.562},"last":9.6359999999999992},{"interface":"front","average":{"1min":8.1669999999999998,"5min":3.8929999999999998,"15min":3.0139999999999998},"min":{"1min":0.29999999999999999,"5min":0.25600000000000001,"15min":0.22800000000000001},"max":{"1min":101.5,"5min":101.5,"15min":101.5},"last":9.5380000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:16:54 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":4.2699999999999996,"15min":2.6099999999999999},"min":{"1min":0.32400000000000001,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":3.157,"5min":245.166,"15min":245.166},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.3480000000000001,"5min":4.1769999999999996,"15min":2.4590000000000001},"min":{"1min":0.33600000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.758,"5min":245.119,"15min":245.119},"last":0.371}]},{"osd":3,"last update":"Mon Oct 9 06:16:54 2023","interfaces":[{"interface":"back","average":{"1min":11.574999999999999,"5min":9.7050000000000001,"15min":4.9450000000000003},"min":{"1min":0.38900000000000001,"5min":0.29399999999999998,"15min":0.26800000000000002},"max":{"1min":197.31899999999999,"5min":246.39400000000001,"15min":246.39400000000001},"last":13.516},{"interface":"front","average":{"1min":11.532,"5min":9.4459999999999997,"15min":4.593},"min":{"1min":0.39800000000000002,"5min":0.314,"15min":0.314},"max":{"1min":198.565,"5min":246.99000000000001,"15min":246.99000000000001},"last":13.664999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:16:54 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":2.0449999999999999,"15min":1.405},"min":{"1min":0.249,"5min":0.24399999999999999,"15min":0.216},"max":{"1min":3.6030000000000002,"5min":99.405000000000001,"15min":99.405000000000001},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.71699999999999997,"5min":2.262,"15min":1.5009999999999999},"min":{"1min":0.31,"5min":0.26700000000000002,"15min":0.245},"max":{"1min":2.0529999999999999,"5min":99.206000000000003,"15min":99.206000000000003},"last":0.38300000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:16:54 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.92900000000000005,"15min":1.6699999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":3.6829999999999998,"5min":7.8140000000000001,"15min":95.492999999999995},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.58899999999999997,"5min":1.179,"15min":1.8080000000000001},"min":{"1min":0.373,"5min":0.33200000000000002,"15min":0.27900000000000003},"max":{"1min":1.956,"5min":9.2110000000000003,"15min":95.664000000000001},"last":0.46800000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:16:54 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":2.2759999999999998,"15min":2.0089999999999999},"min":{"1min":0.35999999999999999,"5min":0.33600000000000002,"15min":0.29499999999999998},"max":{"1min":2.3199999999999998,"5min":95.167000000000002,"15min":95.167000000000002},"last":0.436},{"interface":"front","average":{"1min":0.79700000000000004,"5min":2.206,"15min":1.875},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.251},"max":{"1min":3.1190000000000002,"5min":94.811999999999998,"15min":94.811999999999998},"last":0.47899999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757728,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2560248,"kb_used_data":1665840,"kb_used_omap":0,"kb_used_meta":894400,"kb_avail":91184904,"statfs":{"total":95995035648,"available":93373341696,"internally_reserved":0,"allocated":1705820160,"data_stored":1704235999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915865600},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:36 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":4.1020000000000003,"15min":2.9100000000000001},"min":{"1min":0.26200000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":2.016,"5min":115.92400000000001,"15min":115.92400000000001},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.72599999999999998,"5min":4.2649999999999997,"15min":3.4460000000000002},"min":{"1min":0.34100000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.0630000000000002,"5min":115.904,"15min":115.904},"last":0.68300000000000005}]},{"osd":1,"last update":"Mon Oct 9 06:17:08 2023","interfaces":[{"interface":"back","average":{"1min":1.839,"5min":1.6259999999999999,"15min":4.4900000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.22600000000000001},"max":{"1min":5.7430000000000003,"5min":18.812000000000001,"15min":151.78200000000001},"last":4.0640000000000001},{"interface":"front","average":{"1min":1.1930000000000001,"5min":1.5049999999999999,"15min":4.6390000000000002},"min":{"1min":0.38800000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":2.7480000000000002,"5min":19.376999999999999,"15min":152.251},"last":0.44800000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:17:32 2023","interfaces":[{"interface":"back","average":{"1min":2.5680000000000001,"5min":3.2799999999999998,"15min":2.609},"min":{"1min":0.29599999999999999,"5min":0.28599999999999998,"15min":0.254},"max":{"1min":8.6219999999999999,"5min":117.97199999999999,"15min":129.21199999999999},"last":4.9130000000000003},{"interface":"front","average":{"1min":2.2000000000000002,"5min":3.008,"15min":2.6909999999999998},"min":{"1min":0.317,"5min":0.27400000000000002,"15min":0.26300000000000001},"max":{"1min":7.117,"5min":118.52,"15min":129.042},"last":0.47299999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:17:08 2023","interfaces":[{"interface":"back","average":{"1min":1.7569999999999999,"5min":4.0789999999999997,"15min":5.4829999999999997},"min":{"1min":0.318,"5min":0.316,"15min":0.29499999999999998},"max":{"1min":10.193,"5min":93.338999999999999,"15min":143.31200000000001},"last":0.65500000000000003},{"interface":"front","average":{"1min":2.6480000000000001,"5min":4.6059999999999999,"15min":5.71},"min":{"1min":0.40100000000000002,"5min":0.33500000000000002,"15min":0.29399999999999998},"max":{"1min":10.265000000000001,"5min":93.435000000000002,"15min":140.00899999999999},"last":1.6850000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:16:56 2023","interfaces":[{"interface":"back","average":{"1min":1.097,"5min":3.7000000000000002,"15min":2.008},"min":{"1min":0.32600000000000001,"5min":0.246,"15min":0.245},"max":{"1min":4.9039999999999999,"5min":193.25200000000001,"15min":193.25200000000001},"last":0.39200000000000002},{"interface":"front","average":{"1min":1.014,"5min":3.5510000000000002,"15min":1.859},"min":{"1min":0.30299999999999999,"5min":0.26100000000000001,"15min":0.26000000000000001},"max":{"1min":2.9140000000000001,"5min":190.06800000000001,"15min":190.06800000000001},"last":0.53900000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:17:08 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":0.90900000000000003,"15min":1.1240000000000001},"min":{"1min":0.28100000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.2199999999999998,"5min":6.2199999999999998,"15min":9.0909999999999993},"last":0.62},{"interface":"front","average":{"1min":1.194,"5min":1.1830000000000001,"15min":1.2210000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27300000000000002},"max":{"1min":5.7670000000000003,"5min":11.577999999999999,"15min":11.577999999999999},"last":1.6200000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:16:56 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":3.8490000000000002,"15min":2.6360000000000001},"min":{"1min":0.36799999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.9550000000000001,"5min":180.696,"15min":180.696},"last":1.6579999999999999},{"interface":"front","average":{"1min":0.80500000000000005,"5min":3.4660000000000002,"15min":2.4620000000000002},"min":{"1min":0.35399999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.2549999999999999,"5min":180.74000000000001,"15min":180.74000000000001},"last":0.64500000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4889,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2579955712,"data_stored":2579955712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4369285120,"data_stored":4369285120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3005022208,"data_stored":3005022208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2161639424,"data_stored":2161639424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1707147264,"data_stored":1707147264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3406692352,"data_stored":3406692352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3441229824,"data_stored":3441229824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:17:39.721 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T06:17:39.722 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:17:39.722 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 3, 4, 5, 6, 7, 1] 2023-10-09T06:17:39.722 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd out 6 2023-10-09T06:17:39.805 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:17:39.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:39.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:39.847 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:17:39.879 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:17:39.926 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:39.941 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:39 smithi064 ceph-mon[220044]: from='client.36705 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:17:39.941 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:39 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3425232024' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:17:39.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:39 smithi064 ceph-mon[212750]: from='client.36705 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:17:39.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:39 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3425232024' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:17:39.958 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:39.970 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:40.031 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:17:40.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:40.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:40.061 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total time run: 90.0114 2023-10-09T06:17:40.061 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total writes made: 108911 2023-10-09T06:17:40.062 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Write size: 65536 2023-10-09T06:17:40.062 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object size: 65536 2023-10-09T06:17:40.062 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Bandwidth (MB/sec): 75.6231 2023-10-09T06:17:40.062 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Bandwidth: 33.4225 2023-10-09T06:17:40.062 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max bandwidth (MB/sec): 199.438 2023-10-09T06:17:40.062 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min bandwidth (MB/sec): 40.25 2023-10-09T06:17:40.062 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average IOPS: 1209 2023-10-09T06:17:40.062 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev IOPS: 534.761 2023-10-09T06:17:40.062 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max IOPS: 3191 2023-10-09T06:17:40.063 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min IOPS: 644 2023-10-09T06:17:40.063 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average Latency(s): 0.0132196 2023-10-09T06:17:40.063 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Latency(s): 0.0158962 2023-10-09T06:17:40.063 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max latency(s): 0.307063 2023-10-09T06:17:40.063 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min latency(s): 0.00113085 2023-10-09T06:17:40.066 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T06:17:40.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:39 smithi156 ceph-mon[195101]: from='client.36705 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:17:40.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:39 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3425232024' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:17:40.137 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:40.206 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:40.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:40.677 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:17:40.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:40.778 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:40.853 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:40.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:40 smithi064 ceph-mon[212750]: pgmap v1525: 119 pgs: 119 active+clean; 6.4 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 48 MiB/s wr, 771 op/s 2023-10-09T06:17:40.943 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:40 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:17:40.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:40 smithi064 ceph-mon[212750]: failed to encode map e593 with expected crc 2023-10-09T06:17:40.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:40 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3425232024' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:17:40.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:40 smithi064 ceph-mon[212750]: osdmap e593: 8 total, 8 up, 7 in 2023-10-09T06:17:40.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:40 smithi064 ceph-mon[212750]: failed to encode map e593 with expected crc 2023-10-09T06:17:40.944 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:40 smithi064 ceph-mon[212750]: failed to encode map e593 with expected crc 2023-10-09T06:17:40.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:40 smithi064 ceph-mon[220044]: pgmap v1525: 119 pgs: 119 active+clean; 6.4 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 48 MiB/s wr, 771 op/s 2023-10-09T06:17:40.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:40 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:17:40.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:40 smithi064 ceph-mon[220044]: failed to encode map e593 with expected crc 2023-10-09T06:17:40.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:40 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3425232024' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:17:40.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:40 smithi064 ceph-mon[220044]: osdmap e593: 8 total, 8 up, 7 in 2023-10-09T06:17:40.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:40 smithi064 ceph-mon[220044]: failed to encode map e593 with expected crc 2023-10-09T06:17:40.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:40 smithi064 ceph-mon[220044]: failed to encode map e593 with expected crc 2023-10-09T06:17:40.969 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:17:40.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:40.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:41.082 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:41.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:40 smithi156 ceph-mon[195101]: pgmap v1525: 119 pgs: 119 active+clean; 6.4 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 48 MiB/s wr, 771 op/s 2023-10-09T06:17:41.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:40 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:17:41.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:40 smithi156 ceph-mon[195101]: failed to encode map e593 with expected crc 2023-10-09T06:17:41.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:40 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3425232024' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:17:41.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:40 smithi156 ceph-mon[195101]: osdmap e593: 8 total, 8 up, 7 in 2023-10-09T06:17:41.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:40 smithi156 ceph-mon[195101]: failed to encode map e593 with expected crc 2023-10-09T06:17:41.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:40 smithi156 ceph-mon[195101]: failed to encode map e593 with expected crc 2023-10-09T06:17:41.149 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:41.406 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:41.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:41.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:41.514 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:41.578 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:41.673 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:17:41.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:41.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:41.786 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:41.843 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:41.935 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:17:41.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:41.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:41.951 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:42.042 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:42.056 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:42.133 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:42.361 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:17:42.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:42.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:42.470 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:42.522 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:42.604 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:17:42.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:42.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:42.719 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:42.768 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:42.807 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:17:42.845 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:17:42.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:42.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:42.954 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:43.076 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:43.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:42 smithi156 ceph-mon[195101]: pgmap v1527: 119 pgs: 119 active+clean; 6.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 49 MiB/s wr, 823 op/s 2023-10-09T06:17:43.207 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:42 smithi064 ceph-mon[212750]: pgmap v1527: 119 pgs: 119 active+clean; 6.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 49 MiB/s wr, 823 op/s 2023-10-09T06:17:43.207 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:42 smithi064 ceph-mon[220044]: pgmap v1527: 119 pgs: 119 active+clean; 6.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 49 MiB/s wr, 823 op/s 2023-10-09T06:17:43.734 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:17:43.773 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:17:43.906 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:17:43.934 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:17:43.969 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:17:44.008 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:17:44.112 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:17:44.112 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:17:44.113 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:17:44.113 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:17:44.113 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:17:44.113 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:17:44.113 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:17:44.113 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:17:44.113 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:17:44.113 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:17:44.558 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:44.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:44.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:44.664 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:44.717 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:44.718 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:44 smithi064 ceph-mon[220044]: pgmap v1528: 119 pgs: 119 active+clean; 6.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 47 MiB/s wr, 824 op/s 2023-10-09T06:17:44.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:17:44.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:44.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:44 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4064081307' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:17:44.724 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:44 smithi064 ceph-mon[212750]: pgmap v1528: 119 pgs: 119 active+clean; 6.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 47 MiB/s wr, 824 op/s 2023-10-09T06:17:44.724 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:17:44.724 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:44.724 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:44 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4064081307' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:17:44.801 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:17:44.813 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:44 smithi156 ceph-mon[195101]: pgmap v1528: 119 pgs: 119 active+clean; 6.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 47 MiB/s wr, 824 op/s 2023-10-09T06:17:44.813 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:17:44.813 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:44.813 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:44 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4064081307' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:17:44.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:44.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:44.915 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:44.966 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:45.159 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:17:45.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:45.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:45.281 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:45.304 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:17:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:17:45] "GET /metrics HTTP/1.1" 200 36451 "" "Prometheus/2.43.0" 2023-10-09T06:17:45.326 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:45.328 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:45.405 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:17:45.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:45.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:45.496 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:17:45.496 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T06:17:45.496 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T06:17:45.496 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T06:17:45.496 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:17:45.518 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:45.569 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:45.672 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:17:45.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:45.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:45.780 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:45.848 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:45.929 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:17:45.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:45.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:46.039 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:46.075 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:46.150 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:17:46.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:46.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:46.259 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:46.289 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:46.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:46 smithi064 conmon[212726]: 2023-10-09T06:17:46.216+0000 7f8823f23700 -1 mon.a@0(leader).osd e593 definitely_dead 0 2023-10-09T06:17:46.595 INFO:teuthology.orchestra.run.smithi064.stderr:marked out osd.6. 2023-10-09T06:17:46.705 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:17:46.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:46.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:46.809 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:46.831 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:17:46.838 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:46 smithi064 ceph-mon[220044]: pgmap v1529: 119 pgs: 119 active+clean; 6.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 41 MiB/s wr, 838 op/s 2023-10-09T06:17:46.839 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:46 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4163938945' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T06:17:46.839 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:46 smithi064 ceph-mon[220044]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T06:17:46.839 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:46 smithi064 ceph-mon[212750]: pgmap v1529: 119 pgs: 119 active+clean; 6.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 41 MiB/s wr, 838 op/s 2023-10-09T06:17:46.839 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:46 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4163938945' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T06:17:46.839 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:46 smithi064 ceph-mon[212750]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T06:17:46.842 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:46.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:46 smithi156 ceph-mon[195101]: pgmap v1529: 119 pgs: 119 active+clean; 6.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 41 MiB/s wr, 838 op/s 2023-10-09T06:17:46.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:46 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4163938945' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T06:17:46.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:46 smithi156 ceph-mon[195101]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T06:17:46.994 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:46.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:46.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:47.069 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:17:47.097 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:47.143 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:47.584 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:47 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4163938945' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T06:17:47.585 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:47 smithi156 ceph-mon[195101]: osdmap e594: 8 total, 8 up, 6 in 2023-10-09T06:17:47.585 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:47 smithi156 ceph-mon[195101]: failed to encode map e594 with expected crc 2023-10-09T06:17:47.585 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:47 smithi156 ceph-mon[195101]: failed to encode map e594 with expected crc 2023-10-09T06:17:47.585 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:47 smithi156 ceph-mon[195101]: failed to encode map e594 with expected crc 2023-10-09T06:17:47.585 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:47 smithi156 ceph-mon[195101]: failed to encode map e594 with expected crc 2023-10-09T06:17:47.585 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:47 smithi156 ceph-mon[195101]: failed to encode map e594 with expected crc 2023-10-09T06:17:47.585 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:47 smithi156 ceph-mon[195101]: failed to encode map e594 with expected crc 2023-10-09T06:17:47.585 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:47 smithi156 ceph-mon[195101]: failed to encode map e594 with expected crc 2023-10-09T06:17:47.586 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:47 smithi156 ceph-mon[195101]: failed to encode map e594 with expected crc 2023-10-09T06:17:47.657 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:17:47.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:47.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:47.762 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:47.788 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:47.833 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:17:47.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:47.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:47.940 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:47.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:47 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4163938945' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T06:17:47.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:47 smithi064 ceph-mon[212750]: osdmap e594: 8 total, 8 up, 6 in 2023-10-09T06:17:47.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:47 smithi064 ceph-mon[212750]: failed to encode map e594 with expected crc 2023-10-09T06:17:47.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:47 smithi064 ceph-mon[212750]: failed to encode map e594 with expected crc 2023-10-09T06:17:47.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:47 smithi064 ceph-mon[212750]: failed to encode map e594 with expected crc 2023-10-09T06:17:47.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:47 smithi064 ceph-mon[212750]: failed to encode map e594 with expected crc 2023-10-09T06:17:47.960 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:47 smithi064 ceph-mon[212750]: failed to encode map e594 with expected crc 2023-10-09T06:17:47.960 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:47 smithi064 ceph-mon[212750]: failed to encode map e594 with expected crc 2023-10-09T06:17:47.960 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:47 smithi064 ceph-mon[212750]: failed to encode map e594 with expected crc 2023-10-09T06:17:47.960 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:47 smithi064 ceph-mon[212750]: failed to encode map e594 with expected crc 2023-10-09T06:17:47.960 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:47 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4163938945' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T06:17:47.960 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:47 smithi064 ceph-mon[220044]: osdmap e594: 8 total, 8 up, 6 in 2023-10-09T06:17:47.960 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:47 smithi064 ceph-mon[220044]: failed to encode map e594 with expected crc 2023-10-09T06:17:47.961 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:47 smithi064 ceph-mon[220044]: failed to encode map e594 with expected crc 2023-10-09T06:17:47.961 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:47 smithi064 ceph-mon[220044]: failed to encode map e594 with expected crc 2023-10-09T06:17:47.961 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:47 smithi064 ceph-mon[220044]: failed to encode map e594 with expected crc 2023-10-09T06:17:47.961 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:47 smithi064 ceph-mon[220044]: failed to encode map e594 with expected crc 2023-10-09T06:17:47.961 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:47 smithi064 ceph-mon[220044]: failed to encode map e594 with expected crc 2023-10-09T06:17:47.961 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:47 smithi064 ceph-mon[220044]: failed to encode map e594 with expected crc 2023-10-09T06:17:47.961 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:47 smithi064 ceph-mon[220044]: failed to encode map e594 with expected crc 2023-10-09T06:17:47.966 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:48.009 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:17:48.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:48.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:48.119 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:48.145 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:48.187 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:17:48.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:48.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:48.294 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:48.307 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:48.334 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:48.489 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:48.493 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:48.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:48.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:48.595 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:48.654 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:48.759 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:48 smithi064 ceph-mon[220044]: pgmap v1531: 119 pgs: 9 peering, 110 active+clean; 6.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 127 B/s rd, 28 MiB/s wr, 1.09k op/s 2023-10-09T06:17:48.759 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:48 smithi064 ceph-mon[220044]: failed to encode map e595 with expected crc 2023-10-09T06:17:48.759 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:48 smithi064 ceph-mon[220044]: osdmap e595: 8 total, 8 up, 6 in 2023-10-09T06:17:48.760 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:48 smithi064 ceph-mon[220044]: failed to encode map e595 with expected crc 2023-10-09T06:17:48.760 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:48 smithi064 ceph-mon[220044]: failed to encode map e595 with expected crc 2023-10-09T06:17:48.760 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:48 smithi064 ceph-mon[220044]: failed to encode map e595 with expected crc 2023-10-09T06:17:48.760 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:48 smithi064 ceph-mon[220044]: failed to encode map e595 with expected crc 2023-10-09T06:17:48.760 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:48 smithi064 ceph-mon[220044]: failed to encode map e595 with expected crc 2023-10-09T06:17:48.760 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:48 smithi064 ceph-mon[220044]: failed to encode map e595 with expected crc 2023-10-09T06:17:48.761 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:48 smithi064 ceph-mon[212750]: pgmap v1531: 119 pgs: 9 peering, 110 active+clean; 6.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 127 B/s rd, 28 MiB/s wr, 1.09k op/s 2023-10-09T06:17:48.761 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:48 smithi064 ceph-mon[212750]: failed to encode map e595 with expected crc 2023-10-09T06:17:48.761 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:48 smithi064 ceph-mon[212750]: osdmap e595: 8 total, 8 up, 6 in 2023-10-09T06:17:48.761 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:48 smithi064 ceph-mon[212750]: failed to encode map e595 with expected crc 2023-10-09T06:17:48.761 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:48 smithi064 ceph-mon[212750]: failed to encode map e595 with expected crc 2023-10-09T06:17:48.761 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:48 smithi064 ceph-mon[212750]: failed to encode map e595 with expected crc 2023-10-09T06:17:48.761 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:48 smithi064 ceph-mon[212750]: failed to encode map e595 with expected crc 2023-10-09T06:17:48.761 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:48 smithi064 ceph-mon[212750]: failed to encode map e595 with expected crc 2023-10-09T06:17:48.762 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:48 smithi064 ceph-mon[212750]: failed to encode map e595 with expected crc 2023-10-09T06:17:48.868 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:17:48.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:48 smithi156 ceph-mon[195101]: pgmap v1531: 119 pgs: 9 peering, 110 active+clean; 6.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 127 B/s rd, 28 MiB/s wr, 1.09k op/s 2023-10-09T06:17:48.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:48 smithi156 ceph-mon[195101]: failed to encode map e595 with expected crc 2023-10-09T06:17:48.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:48 smithi156 ceph-mon[195101]: osdmap e595: 8 total, 8 up, 6 in 2023-10-09T06:17:48.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:48 smithi156 ceph-mon[195101]: failed to encode map e595 with expected crc 2023-10-09T06:17:48.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:48 smithi156 ceph-mon[195101]: failed to encode map e595 with expected crc 2023-10-09T06:17:48.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:48 smithi156 ceph-mon[195101]: failed to encode map e595 with expected crc 2023-10-09T06:17:48.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:48 smithi156 ceph-mon[195101]: failed to encode map e595 with expected crc 2023-10-09T06:17:48.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:48 smithi156 ceph-mon[195101]: failed to encode map e595 with expected crc 2023-10-09T06:17:48.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:48 smithi156 ceph-mon[195101]: failed to encode map e595 with expected crc 2023-10-09T06:17:48.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:48.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:48.986 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:49.028 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:49.503 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:17:49.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:49.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:49.610 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:49.611 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:17:49.646 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:49.727 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:17:49.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:49.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:49.841 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:49.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:49 smithi156 ceph-mon[195101]: 11.1 starting backfill to osd.4 from (0'0,0'0] MAX to 593'7029 2023-10-09T06:17:49.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:49 smithi156 ceph-mon[195101]: 11.8 starting backfill to osd.0 from (0'0,0'0] MAX to 593'7257 2023-10-09T06:17:49.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:49 smithi156 ceph-mon[195101]: osdmap e596: 8 total, 8 up, 6 in 2023-10-09T06:17:49.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:49 smithi156 ceph-mon[195101]: failed to encode map e596 with expected crc 2023-10-09T06:17:49.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:49 smithi156 ceph-mon[195101]: failed to encode map e596 with expected crc 2023-10-09T06:17:49.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:49 smithi156 ceph-mon[195101]: failed to encode map e596 with expected crc 2023-10-09T06:17:49.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:49 smithi156 ceph-mon[195101]: 11.a starting backfill to osd.1 from (0'0,0'0] MAX to 593'7084 2023-10-09T06:17:49.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:49 smithi156 ceph-mon[195101]: failed to encode map e596 with expected crc 2023-10-09T06:17:49.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:49 smithi156 ceph-mon[195101]: 11.9 starting backfill to osd.1 from (0'0,0'0] MAX to 593'7114 2023-10-09T06:17:49.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:49 smithi156 ceph-mon[195101]: 11.d starting backfill to osd.5 from (0'0,0'0] MAX to 593'7100 2023-10-09T06:17:49.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:49 smithi156 ceph-mon[195101]: failed to encode map e596 with expected crc 2023-10-09T06:17:49.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:49 smithi156 ceph-mon[195101]: failed to encode map e596 with expected crc 2023-10-09T06:17:49.874 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:49 smithi156 ceph-mon[195101]: 11.4 starting backfill to osd.3 from (0'0,0'0] MAX to 593'7134 2023-10-09T06:17:49.875 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:49 smithi156 ceph-mon[195101]: 11.5 starting backfill to osd.3 from (0'0,0'0] MAX to 593'7222 2023-10-09T06:17:49.875 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:49 smithi156 ceph-mon[195101]: 11.3 starting backfill to osd.5 from (0'0,0'0] MAX to 593'7336 2023-10-09T06:17:49.875 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:49 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2681864431' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:17:49.904 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:49.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[220044]: 11.1 starting backfill to osd.4 from (0'0,0'0] MAX to 593'7029 2023-10-09T06:17:49.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[220044]: 11.8 starting backfill to osd.0 from (0'0,0'0] MAX to 593'7257 2023-10-09T06:17:49.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[220044]: osdmap e596: 8 total, 8 up, 6 in 2023-10-09T06:17:49.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[220044]: failed to encode map e596 with expected crc 2023-10-09T06:17:49.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[220044]: failed to encode map e596 with expected crc 2023-10-09T06:17:49.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[220044]: failed to encode map e596 with expected crc 2023-10-09T06:17:49.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[220044]: 11.a starting backfill to osd.1 from (0'0,0'0] MAX to 593'7084 2023-10-09T06:17:49.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[220044]: failed to encode map e596 with expected crc 2023-10-09T06:17:49.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[220044]: 11.9 starting backfill to osd.1 from (0'0,0'0] MAX to 593'7114 2023-10-09T06:17:49.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[220044]: 11.d starting backfill to osd.5 from (0'0,0'0] MAX to 593'7100 2023-10-09T06:17:49.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[220044]: failed to encode map e596 with expected crc 2023-10-09T06:17:49.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[220044]: failed to encode map e596 with expected crc 2023-10-09T06:17:49.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[220044]: 11.4 starting backfill to osd.3 from (0'0,0'0] MAX to 593'7134 2023-10-09T06:17:49.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[220044]: 11.5 starting backfill to osd.3 from (0'0,0'0] MAX to 593'7222 2023-10-09T06:17:49.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[220044]: 11.3 starting backfill to osd.5 from (0'0,0'0] MAX to 593'7336 2023-10-09T06:17:49.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2681864431' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:17:49.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[212750]: 11.1 starting backfill to osd.4 from (0'0,0'0] MAX to 593'7029 2023-10-09T06:17:49.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[212750]: 11.8 starting backfill to osd.0 from (0'0,0'0] MAX to 593'7257 2023-10-09T06:17:49.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[212750]: osdmap e596: 8 total, 8 up, 6 in 2023-10-09T06:17:49.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[212750]: failed to encode map e596 with expected crc 2023-10-09T06:17:49.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[212750]: failed to encode map e596 with expected crc 2023-10-09T06:17:49.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[212750]: failed to encode map e596 with expected crc 2023-10-09T06:17:49.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[212750]: 11.a starting backfill to osd.1 from (0'0,0'0] MAX to 593'7084 2023-10-09T06:17:49.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[212750]: failed to encode map e596 with expected crc 2023-10-09T06:17:49.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[212750]: 11.9 starting backfill to osd.1 from (0'0,0'0] MAX to 593'7114 2023-10-09T06:17:49.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[212750]: 11.d starting backfill to osd.5 from (0'0,0'0] MAX to 593'7100 2023-10-09T06:17:49.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[212750]: failed to encode map e596 with expected crc 2023-10-09T06:17:49.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[212750]: failed to encode map e596 with expected crc 2023-10-09T06:17:49.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[212750]: 11.4 starting backfill to osd.3 from (0'0,0'0] MAX to 593'7134 2023-10-09T06:17:49.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[212750]: 11.5 starting backfill to osd.3 from (0'0,0'0] MAX to 593'7222 2023-10-09T06:17:49.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[212750]: 11.3 starting backfill to osd.5 from (0'0,0'0] MAX to 593'7336 2023-10-09T06:17:49.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:49 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2681864431' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:17:49.995 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:17:50.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:50.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:50.110 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:50.181 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:50.248 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:17:50.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:50.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:50.354 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:50.383 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:50.432 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:17:50.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:50.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:50.538 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:50.611 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:50.743 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:17:50.747 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:50 smithi156 ceph-mon[195101]: pgmap v1534: 119 pgs: 9 peering, 110 active+clean; 6.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 1.5 MiB/s wr, 745 op/s 2023-10-09T06:17:50.747 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:50 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:17:50.747 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:50 smithi156 ceph-mon[195101]: failed to encode map e597 with expected crc 2023-10-09T06:17:50.747 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:50 smithi156 ceph-mon[195101]: failed to encode map e597 with expected crc 2023-10-09T06:17:50.747 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:50 smithi156 ceph-mon[195101]: failed to encode map e597 with expected crc 2023-10-09T06:17:50.747 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:50 smithi156 ceph-mon[195101]: failed to encode map e597 with expected crc 2023-10-09T06:17:50.747 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:50 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2681864431' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:17:50.747 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:50 smithi156 ceph-mon[195101]: osdmap e597: 8 total, 8 up, 6 in 2023-10-09T06:17:50.748 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:50 smithi156 ceph-mon[195101]: failed to encode map e597 with expected crc 2023-10-09T06:17:50.748 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:50 smithi156 ceph-mon[195101]: failed to encode map e597 with expected crc 2023-10-09T06:17:50.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:50.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:50.849 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:50.875 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:50 smithi064 ceph-mon[212750]: pgmap v1534: 119 pgs: 9 peering, 110 active+clean; 6.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 1.5 MiB/s wr, 745 op/s 2023-10-09T06:17:50.876 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:50 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:17:50.876 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:50 smithi064 ceph-mon[212750]: failed to encode map e597 with expected crc 2023-10-09T06:17:50.876 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:50 smithi064 ceph-mon[212750]: failed to encode map e597 with expected crc 2023-10-09T06:17:50.876 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:50 smithi064 ceph-mon[212750]: failed to encode map e597 with expected crc 2023-10-09T06:17:50.876 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:50 smithi064 ceph-mon[212750]: failed to encode map e597 with expected crc 2023-10-09T06:17:50.876 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:50 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2681864431' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:17:50.876 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:50 smithi064 ceph-mon[212750]: osdmap e597: 8 total, 8 up, 6 in 2023-10-09T06:17:50.876 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:50 smithi064 ceph-mon[212750]: failed to encode map e597 with expected crc 2023-10-09T06:17:50.876 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:50 smithi064 ceph-mon[212750]: failed to encode map e597 with expected crc 2023-10-09T06:17:50.877 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:50 smithi064 ceph-mon[220044]: pgmap v1534: 119 pgs: 9 peering, 110 active+clean; 6.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 1.5 MiB/s wr, 745 op/s 2023-10-09T06:17:50.877 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:50 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:17:50.877 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:50 smithi064 ceph-mon[220044]: failed to encode map e597 with expected crc 2023-10-09T06:17:50.877 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:50 smithi064 ceph-mon[220044]: failed to encode map e597 with expected crc 2023-10-09T06:17:50.877 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:50 smithi064 ceph-mon[220044]: failed to encode map e597 with expected crc 2023-10-09T06:17:50.877 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:50 smithi064 ceph-mon[220044]: failed to encode map e597 with expected crc 2023-10-09T06:17:50.877 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:50 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2681864431' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:17:50.877 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:50 smithi064 ceph-mon[220044]: osdmap e597: 8 total, 8 up, 6 in 2023-10-09T06:17:50.877 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:50 smithi064 ceph-mon[220044]: failed to encode map e597 with expected crc 2023-10-09T06:17:50.877 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:50 smithi064 ceph-mon[220044]: failed to encode map e597 with expected crc 2023-10-09T06:17:50.878 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:51.502 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:17:51.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:51.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:51.607 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:51.658 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:51.763 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:17:51.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:51.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:51.786 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:17:51.786 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T06:17:51.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T06:17:51.787 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:17:51.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.787 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116287 9.1 9:8655aad1:::benchmark_data_smithi064_511911_object15185:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-10-09T06:17:51.787 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.530424+0000", 2023-10-09T06:17:51.787 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 529.25222935600004, 2023-10-09T06:17:51.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.181524692, 2023-10-09T06:17:51.788 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:51.788 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:17:51.788 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:51.788 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:17:51.788 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:17:51.788 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116287 2023-10-09T06:17:51.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.788 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:51.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:51.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530424+0000", 2023-10-09T06:17:51.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:51.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530424+0000", 2023-10-09T06:17:51.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:51.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530422+0000", 2023-10-09T06:17:51.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:17:51.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:51.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530432+0000", 2023-10-09T06:17:51.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0105000000000001e-05 2023-10-09T06:17:51.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:51.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530454+0000", 2023-10-09T06:17:51.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1579000000000001e-05 2023-10-09T06:17:51.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:51.792 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530465+0000", 2023-10-09T06:17:51.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1044e-05 2023-10-09T06:17:51.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.792 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.792 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.792 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530519+0000", 2023-10-09T06:17:51.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4567000000000001e-05 2023-10-09T06:17:51.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:51.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530668+0000", 2023-10-09T06:17:51.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014871900000000001 2023-10-09T06:17:51.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:17:51.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.530823+0000", 2023-10-09T06:17:51.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000155375 2023-10-09T06:17:51.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.533953+0000", 2023-10-09T06:17:51.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031297159999999998 2023-10-09T06:17:51.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:51.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.534034+0000", 2023-10-09T06:17:51.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0731999999999999e-05 2023-10-09T06:17:51.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.605818+0000", 2023-10-09T06:17:51.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.071784291 2023-10-09T06:17:51.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.605835+0000", 2023-10-09T06:17:51.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6589e-05 2023-10-09T06:17:51.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578572+0000", 2023-10-09T06:17:51.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97273759299999996 2023-10-09T06:17:51.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578584+0000", 2023-10-09T06:17:51.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1283e-05 2023-10-09T06:17:51.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.711858+0000", 2023-10-09T06:17:51.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1332740379999999 2023-10-09T06:17:51.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:51.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.711949+0000", 2023-10-09T06:17:51.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1187000000000006e-05 2023-10-09T06:17:51.798 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.798 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:51.798 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.799 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116293 9.1 9:8657b0ae:::benchmark_data_smithi064_511911_object15191:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-10-09T06:17:51.799 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.545024+0000", 2023-10-09T06:17:51.799 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 529.23762904199998, 2023-10-09T06:17:51.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1673195999999999, 2023-10-09T06:17:51.799 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:51.799 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:17:51.799 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:51.799 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:17:51.799 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:17:51.800 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116293 2023-10-09T06:17:51.800 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.800 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:51.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:51.800 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545024+0000", 2023-10-09T06:17:51.800 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.800 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:51.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545024+0000", 2023-10-09T06:17:51.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:51.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545023+0000", 2023-10-09T06:17:51.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:17:51.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:51.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545029+0000", 2023-10-09T06:17:51.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5910000000000003e-06 2023-10-09T06:17:51.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:51.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545037+0000", 2023-10-09T06:17:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.909e-06 2023-10-09T06:17:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545042+0000", 2023-10-09T06:17:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1030000000000001e-06 2023-10-09T06:17:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545077+0000", 2023-10-09T06:17:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4620999999999999e-05 2023-10-09T06:17:51.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:51.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545160+0000", 2023-10-09T06:17:51.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3925999999999996e-05 2023-10-09T06:17:51.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:17:51.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.545236+0000", 2023-10-09T06:17:51.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5909999999999997e-05 2023-10-09T06:17:51.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.547494+0000", 2023-10-09T06:17:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0022575310000000001 2023-10-09T06:17:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.547584+0000", 2023-10-09T06:17:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0315999999999994e-05 2023-10-09T06:17:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.605910+0000", 2023-10-09T06:17:51.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.058325845000000001 2023-10-09T06:17:51.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.605922+0000", 2023-10-09T06:17:51.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1712999999999999e-05 2023-10-09T06:17:51.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578649+0000", 2023-10-09T06:17:51.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97272731099999998 2023-10-09T06:17:51.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.578660+0000", 2023-10-09T06:17:51.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0560999999999999e-05 2023-10-09T06:17:51.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.712239+0000", 2023-10-09T06:17:51.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.133579602 2023-10-09T06:17:51.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.712344+0000", 2023-10-09T06:17:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000104746 2023-10-09T06:17:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:116283 9.3 9:c10cf455:::benchmark_data_smithi064_511911_object15181:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e449)", 2023-10-09T06:17:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:02.606404+0000", 2023-10-09T06:17:51.808 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 529.17624921200002, 2023-10-09T06:17:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.160898918, 2023-10-09T06:17:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:17:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:17:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:17:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 116283 2023-10-09T06:17:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:51.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606404+0000", 2023-10-09T06:17:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606404+0000", 2023-10-09T06:17:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606404+0000", 2023-10-09T06:17:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:17:51.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:51.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606407+0000", 2023-10-09T06:17:51.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9179999999999998e-06 2023-10-09T06:17:51.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:51.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606415+0000", 2023-10-09T06:17:51.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3180000000000007e-06 2023-10-09T06:17:51.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:51.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606421+0000", 2023-10-09T06:17:51.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2430000000000002e-06 2023-10-09T06:17:51.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606613+0000", 2023-10-09T06:17:51.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019148599999999999 2023-10-09T06:17:51.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:02.606636+0000", 2023-10-09T06:17:51.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3680999999999999e-05 2023-10-09T06:17:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.576977+0000", 2023-10-09T06:17:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97034085699999995 2023-10-09T06:17:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:03.577913+0000", 2023-10-09T06:17:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00093576100000000001 2023-10-09T06:17:51.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.723002+0000", 2023-10-09T06:17:51.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1450891699999999 2023-10-09T06:17:51.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-10-09T06:17:51.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.723112+0000", 2023-10-09T06:17:51.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110089 2023-10-09T06:17:51.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.767258+0000", 2023-10-09T06:17:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.044145982 2023-10-09T06:17:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:04.767303+0000", 2023-10-09T06:17:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4894000000000001e-05 2023-10-09T06:17:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.816 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:132917 9.8 9:12f89378:::benchmark_data_smithi064_511911_object31815:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e456)", 2023-10-09T06:17:51.816 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:26.626655+0000", 2023-10-09T06:17:51.816 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 505.15599835699999, 2023-10-09T06:17:51.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1614917130000002, 2023-10-09T06:17:51.816 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:51.816 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:51.816 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:51.816 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:17:51.816 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:17:51.816 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132917 2023-10-09T06:17:51.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.817 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:51.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:51.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.626655+0000", 2023-10-09T06:17:51.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:51.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.626655+0000", 2023-10-09T06:17:51.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:51.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.626653+0000", 2023-10-09T06:17:51.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:17:51.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:51.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.626661+0000", 2023-10-09T06:17:51.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7069999999999992e-06 2023-10-09T06:17:51.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:51.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.626673+0000", 2023-10-09T06:17:51.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2517999999999999e-05 2023-10-09T06:17:51.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:51.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.626683+0000", 2023-10-09T06:17:51.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4490000000000005e-06 2023-10-09T06:17:51.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.627477+0000", 2023-10-09T06:17:51.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00079445099999999995 2023-10-09T06:17:51.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.627484+0000", 2023-10-09T06:17:51.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4520000000000002e-06 2023-10-09T06:17:51.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528505+0000", 2023-10-09T06:17:51.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90102110700000004 2023-10-09T06:17:51.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528519+0000", 2023-10-09T06:17:51.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3943e-05 2023-10-09T06:17:51.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.755152+0000", 2023-10-09T06:17:51.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2266332689999999 2023-10-09T06:17:51.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:51.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.755289+0000", 2023-10-09T06:17:51.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013710400000000001 2023-10-09T06:17:51.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:17:51.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.755435+0000", 2023-10-09T06:17:51.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000145496 2023-10-09T06:17:51.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:51.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.766695+0000", 2023-10-09T06:17:51.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011260564000000001 2023-10-09T06:17:51.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.769909+0000", 2023-10-09T06:17:51.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032138940000000001 2023-10-09T06:17:51.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.785944+0000", 2023-10-09T06:17:51.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016034757 2023-10-09T06:17:51.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:51.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.785992+0000", 2023-10-09T06:17:51.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8412999999999999e-05 2023-10-09T06:17:51.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:51.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.788147+0000", 2023-10-09T06:17:51.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0021544020000000001 2023-10-09T06:17:51.826 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.827 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:51.827 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.827 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35940.0:132924 9.8 9:100587d5:::benchmark_data_smithi064_511911_object31822:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e456)", 2023-10-09T06:17:51.827 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:09:26.652624+0000", 2023-10-09T06:17:51.827 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 505.130029326, 2023-10-09T06:17:51.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1523453090000002, 2023-10-09T06:17:51.827 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:51.827 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:51.828 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:51.828 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35940", 2023-10-09T06:17:51.828 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:17:51.828 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 132924 2023-10-09T06:17:51.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.828 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:51.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:51.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652624+0000", 2023-10-09T06:17:51.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:51.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652624+0000", 2023-10-09T06:17:51.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:51.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652616+0000", 2023-10-09T06:17:51.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999924 2023-10-09T06:17:51.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:51.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652632+0000", 2023-10-09T06:17:51.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5653000000000001e-05 2023-10-09T06:17:51.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:51.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652647+0000", 2023-10-09T06:17:51.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5244e-05 2023-10-09T06:17:51.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:51.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652656+0000", 2023-10-09T06:17:51.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0380000000000005e-06 2023-10-09T06:17:51.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652718+0000", 2023-10-09T06:17:51.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1267999999999997e-05 2023-10-09T06:17:51.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:26.652730+0000", 2023-10-09T06:17:51.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2069000000000001e-05 2023-10-09T06:17:51.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528571+0000", 2023-10-09T06:17:51.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.875841753 2023-10-09T06:17:51.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:27.528579+0000", 2023-10-09T06:17:51.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6410000000000002e-06 2023-10-09T06:17:51.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.764042+0000", 2023-10-09T06:17:51.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2354630339999999 2023-10-09T06:17:51.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:51.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.764130+0000", 2023-10-09T06:17:51.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7968999999999998e-05 2023-10-09T06:17:51.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:17:51.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.764240+0000", 2023-10-09T06:17:51.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010986 2023-10-09T06:17:51.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:51.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.769918+0000", 2023-10-09T06:17:51.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056776170000000003 2023-10-09T06:17:51.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.769984+0000", 2023-10-09T06:17:51.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6703000000000002e-05 2023-10-09T06:17:51.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.802902+0000", 2023-10-09T06:17:51.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.032917469999999997 2023-10-09T06:17:51.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:51.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.802957+0000", 2023-10-09T06:17:51.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5047999999999999e-05 2023-10-09T06:17:51.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:51.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:09:28.804969+0000", 2023-10-09T06:17:51.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020126900000000001 2023-10-09T06:17:51.837 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.837 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:51.837 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.837 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:171672 10.3 10:c82752d7:::benchmark_data_smithi064_573750_object75457:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e557)", 2023-10-09T06:17:51.837 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:15:15.799808+0000", 2023-10-09T06:17:51.837 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 155.98284583399999, 2023-10-09T06:17:51.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.162648323, 2023-10-09T06:17:51.837 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:51.838 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:17:51.838 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:51.838 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:17:51.838 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:17:51.838 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171672 2023-10-09T06:17:51.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.838 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:51.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:51.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799808+0000", 2023-10-09T06:17:51.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799808+0000", 2023-10-09T06:17:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799805+0000", 2023-10-09T06:17:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:17:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:51.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799813+0000", 2023-10-09T06:17:51.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9149999999999996e-06 2023-10-09T06:17:51.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:51.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799826+0000", 2023-10-09T06:17:51.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2824e-05 2023-10-09T06:17:51.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:51.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799834+0000", 2023-10-09T06:17:51.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8990000000000001e-06 2023-10-09T06:17:51.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.800015+0000", 2023-10-09T06:17:51.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018057300000000001 2023-10-09T06:17:51.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:51.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.800155+0000", 2023-10-09T06:17:51.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014034099999999999 2023-10-09T06:17:51.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:17:51.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.800283+0000", 2023-10-09T06:17:51.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000127945 2023-10-09T06:17:51.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:51.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.808806+0000", 2023-10-09T06:17:51.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085225639999999998 2023-10-09T06:17:51.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812281+0000", 2023-10-09T06:17:51.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003475653 2023-10-09T06:17:51.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812302+0000", 2023-10-09T06:17:51.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0441e-05 2023-10-09T06:17:51.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794653+0000", 2023-10-09T06:17:51.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98235096899999996 2023-10-09T06:17:51.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794672+0000", 2023-10-09T06:17:51.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9063e-05 2023-10-09T06:17:51.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.950344+0000", 2023-10-09T06:17:51.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.155671983 2023-10-09T06:17:51.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-10-09T06:17:51.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.950466+0000", 2023-10-09T06:17:51.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000122123 2023-10-09T06:17:51.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.962321+0000", 2023-10-09T06:17:51.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01185547 2023-10-09T06:17:51.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:51.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.962456+0000", 2023-10-09T06:17:51.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013469099999999999 2023-10-09T06:17:51.846 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.846 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:51.846 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.846 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:171676 10.3 10:c58e27ea:::benchmark_data_smithi064_573750_object75461:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e557)", 2023-10-09T06:17:51.846 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:15:15.799859+0000", 2023-10-09T06:17:51.846 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 155.98279435699999, 2023-10-09T06:17:51.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1745875799999999, 2023-10-09T06:17:51.847 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:51.847 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:17:51.847 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:51.847 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:17:51.847 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:17:51.847 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171676 2023-10-09T06:17:51.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.847 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:51.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:51.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799859+0000", 2023-10-09T06:17:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799859+0000", 2023-10-09T06:17:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799859+0000", 2023-10-09T06:17:51.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:17:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799862+0000", 2023-10-09T06:17:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0750000000000002e-06 2023-10-09T06:17:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799875+0000", 2023-10-09T06:17:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.31e-05 2023-10-09T06:17:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:51.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799881+0000", 2023-10-09T06:17:51.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.801e-06 2023-10-09T06:17:51.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.803918+0000", 2023-10-09T06:17:51.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040373400000000004 2023-10-09T06:17:51.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.804065+0000", 2023-10-09T06:17:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014711600000000001 2023-10-09T06:17:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:17:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.804246+0000", 2023-10-09T06:17:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018100599999999999 2023-10-09T06:17:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812631+0000", 2023-10-09T06:17:51.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0083849679999999996 2023-10-09T06:17:51.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812642+0000", 2023-10-09T06:17:51.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0681e-05 2023-10-09T06:17:51.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794709+0000", 2023-10-09T06:17:51.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98206730200000003 2023-10-09T06:17:51.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794722+0000", 2023-10-09T06:17:51.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2621e-05 2023-10-09T06:17:51.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.950773+0000", 2023-10-09T06:17:51.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.156051175 2023-10-09T06:17:51.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-10-09T06:17:51.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.950916+0000", 2023-10-09T06:17:51.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000142726 2023-10-09T06:17:51.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974353+0000", 2023-10-09T06:17:51.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023437021999999998 2023-10-09T06:17:51.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:51.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974447+0000", 2023-10-09T06:17:51.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4092000000000006e-05 2023-10-09T06:17:51.855 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.855 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:51.855 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.855 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:171677 10.3 10:cc911201:::benchmark_data_smithi064_573750_object75462:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:17:51.855 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:15:15.812441+0000", 2023-10-09T06:17:51.855 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 155.97021291300001, 2023-10-09T06:17:51.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.162245993, 2023-10-09T06:17:51.855 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:51.856 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:51.856 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:51.856 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:17:51.856 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:17:51.856 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171677 2023-10-09T06:17:51.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.856 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:51.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:51.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812441+0000", 2023-10-09T06:17:51.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:51.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812441+0000", 2023-10-09T06:17:51.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:51.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812439+0000", 2023-10-09T06:17:51.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:17:51.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.858 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:51.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812444+0000", 2023-10-09T06:17:51.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1150000000000002e-06 2023-10-09T06:17:51.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.858 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:51.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812455+0000", 2023-10-09T06:17:51.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1742999999999999e-05 2023-10-09T06:17:51.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.859 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:51.859 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812464+0000", 2023-10-09T06:17:51.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4910000000000005e-06 2023-10-09T06:17:51.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.859 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.859 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812964+0000", 2023-10-09T06:17:51.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00049951399999999995 2023-10-09T06:17:51.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.860 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812975+0000", 2023-10-09T06:17:51.860 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1063999999999999e-05 2023-10-09T06:17:51.860 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.860 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794751+0000", 2023-10-09T06:17:51.860 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.981776023 2023-10-09T06:17:51.860 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794760+0000", 2023-10-09T06:17:51.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9590000000000001e-06 2023-10-09T06:17:51.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.950951+0000", 2023-10-09T06:17:51.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.156191089 2023-10-09T06:17:51.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:51.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.951073+0000", 2023-10-09T06:17:51.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012265999999999999 2023-10-09T06:17:51.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T06:17:51.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.951219+0000", 2023-10-09T06:17:51.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014580999999999999 2023-10-09T06:17:51.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:51.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.961907+0000", 2023-10-09T06:17:51.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010687962000000001 2023-10-09T06:17:51.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.962502+0000", 2023-10-09T06:17:51.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000594646 2023-10-09T06:17:51.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.973943+0000", 2023-10-09T06:17:51.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011441019 2023-10-09T06:17:51.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.864 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.864 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974516+0000", 2023-10-09T06:17:51.864 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00057290800000000001 2023-10-09T06:17:51.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.864 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:51.864 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974566+0000", 2023-10-09T06:17:51.864 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0080999999999997e-05 2023-10-09T06:17:51.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.864 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:51.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974687+0000", 2023-10-09T06:17:51.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000120795 2023-10-09T06:17:51.865 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.865 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:51.865 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.865 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:171684 10.3 10:cf644d59:::benchmark_data_smithi064_573750_object75469:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:17:51.865 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:15:15.815765+0000", 2023-10-09T06:17:51.865 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 155.96688813599999, 2023-10-09T06:17:51.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1591680040000001, 2023-10-09T06:17:51.865 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:51.866 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:51.866 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:51.866 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:17:51.866 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:17:51.866 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171684 2023-10-09T06:17:51.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.866 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:51.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:51.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815765+0000", 2023-10-09T06:17:51.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:51.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815765+0000", 2023-10-09T06:17:51.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:51.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815764+0000", 2023-10-09T06:17:51.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:17:51.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:51.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815771+0000", 2023-10-09T06:17:51.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3080000000000002e-06 2023-10-09T06:17:51.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:51.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815785+0000", 2023-10-09T06:17:51.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3903000000000001e-05 2023-10-09T06:17:51.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:51.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815795+0000", 2023-10-09T06:17:51.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8209999999999997e-06 2023-10-09T06:17:51.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.869 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.869 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815830+0000", 2023-10-09T06:17:51.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4987999999999998e-05 2023-10-09T06:17:51.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.869 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.869 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815839+0000", 2023-10-09T06:17:51.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5189999999999998e-06 2023-10-09T06:17:51.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794786+0000", 2023-10-09T06:17:51.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97894675900000006 2023-10-09T06:17:51.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794794+0000", 2023-10-09T06:17:51.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5590000000000006e-06 2023-10-09T06:17:51.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.954768+0000", 2023-10-09T06:17:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.159973972 2023-10-09T06:17:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.954905+0000", 2023-10-09T06:17:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013690299999999999 2023-10-09T06:17:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T06:17:51.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.955044+0000", 2023-10-09T06:17:51.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013847499999999999 2023-10-09T06:17:51.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:51.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.962281+0000", 2023-10-09T06:17:51.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0072369289999999996 2023-10-09T06:17:51.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.962546+0000", 2023-10-09T06:17:51.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00026517000000000001 2023-10-09T06:17:51.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.873 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.873 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974728+0000", 2023-10-09T06:17:51.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012181772 2023-10-09T06:17:51.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.873 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.873 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974764+0000", 2023-10-09T06:17:51.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6820000000000003e-05 2023-10-09T06:17:51.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.874 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:51.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974817+0000", 2023-10-09T06:17:51.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2811e-05 2023-10-09T06:17:51.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.874 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.874 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:51.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974933+0000", 2023-10-09T06:17:51.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116087 2023-10-09T06:17:51.874 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.874 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:51.875 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.875 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:171685 10.3 10:cb5292a6:::benchmark_data_smithi064_573750_object75470:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:17:51.875 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:15:15.816204+0000", 2023-10-09T06:17:51.875 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 155.966449934, 2023-10-09T06:17:51.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1602268950000001, 2023-10-09T06:17:51.875 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:51.875 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:51.875 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:51.875 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:17:51.876 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:17:51.876 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171685 2023-10-09T06:17:51.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.876 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:51.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:51.876 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816204+0000", 2023-10-09T06:17:51.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816204+0000", 2023-10-09T06:17:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816203+0000", 2023-10-09T06:17:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:17:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816207+0000", 2023-10-09T06:17:51.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3000000000000003e-06 2023-10-09T06:17:51.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:51.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816220+0000", 2023-10-09T06:17:51.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3208e-05 2023-10-09T06:17:51.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:51.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816228+0000", 2023-10-09T06:17:51.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6869999999999994e-06 2023-10-09T06:17:51.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816276+0000", 2023-10-09T06:17:51.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7632000000000001e-05 2023-10-09T06:17:51.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816287+0000", 2023-10-09T06:17:51.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0908e-05 2023-10-09T06:17:51.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794918+0000", 2023-10-09T06:17:51.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97863145399999996 2023-10-09T06:17:51.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794928+0000", 2023-10-09T06:17:51.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.005e-05 2023-10-09T06:17:51.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.958279+0000", 2023-10-09T06:17:51.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.163350514 2023-10-09T06:17:51.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:51.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.958431+0000", 2023-10-09T06:17:51.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000151946 2023-10-09T06:17:51.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T06:17:51.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.958587+0000", 2023-10-09T06:17:51.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015680800000000001 2023-10-09T06:17:51.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:51.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.962454+0000", 2023-10-09T06:17:51.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038670100000000002 2023-10-09T06:17:51.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974038+0000", 2023-10-09T06:17:51.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01158314 2023-10-09T06:17:51.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974974+0000", 2023-10-09T06:17:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00093642900000000002 2023-10-09T06:17:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.976244+0000", 2023-10-09T06:17:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001270265 2023-10-09T06:17:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.976304+0000", 2023-10-09T06:17:51.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9621999999999999e-05 2023-10-09T06:17:51.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:51.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.976430+0000", 2023-10-09T06:17:51.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012658100000000001 2023-10-09T06:17:51.884 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.884 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:51.884 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.884 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:114773 11.a 11:59d5ab77:::benchmark_data_smithi064_631903_object5859:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-10-09T06:17:51.884 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:17:46.566276+0000", 2023-10-09T06:17:51.885 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 5.2163770920000001, 2023-10-09T06:17:51.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2362202039999999, 2023-10-09T06:17:51.885 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:51.885 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:17:51.885 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:51.885 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:17:51.885 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:17:51.885 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 114773 2023-10-09T06:17:51.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.885 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:51.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:51.886 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566276+0000", 2023-10-09T06:17:51.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:51.886 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566276+0000", 2023-10-09T06:17:51.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566275+0000", 2023-10-09T06:17:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:17:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566281+0000", 2023-10-09T06:17:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0900000000000001e-06 2023-10-09T06:17:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566294+0000", 2023-10-09T06:17:51.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2852e-05 2023-10-09T06:17:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566302+0000", 2023-10-09T06:17:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9920000000000007e-06 2023-10-09T06:17:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.570258+0000", 2023-10-09T06:17:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039559749999999996 2023-10-09T06:17:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:51.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.570366+0000", 2023-10-09T06:17:51.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000107772 2023-10-09T06:17:51.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:17:51.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.570476+0000", 2023-10-09T06:17:51.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000109956 2023-10-09T06:17:51.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581185+0000", 2023-10-09T06:17:51.890 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010708746999999999 2023-10-09T06:17:51.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581197+0000", 2023-10-09T06:17:51.890 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2275e-05 2023-10-09T06:17:51.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.599254+0000", 2023-10-09T06:17:51.890 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0180574899999999 2023-10-09T06:17:51.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.599267+0000", 2023-10-09T06:17:51.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2661e-05 2023-10-09T06:17:51.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.802421+0000", 2023-10-09T06:17:51.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.203153666 2023-10-09T06:17:51.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:51.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.802497+0000", 2023-10-09T06:17:51.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5834999999999995e-05 2023-10-09T06:17:51.892 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.892 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:51.892 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.892 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:114774 11.1 11:81618b5d:::benchmark_data_smithi064_631903_object5860:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-10-09T06:17:51.892 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:17:46.566314+0000", 2023-10-09T06:17:51.893 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 5.2163390789999999, 2023-10-09T06:17:51.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2080191130000002, 2023-10-09T06:17:51.893 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:51.893 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:17:51.893 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:51.893 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:17:51.893 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:17:51.893 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 114774 2023-10-09T06:17:51.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.893 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:51.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:51.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566314+0000", 2023-10-09T06:17:51.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:51.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566314+0000", 2023-10-09T06:17:51.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:51.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566314+0000", 2023-10-09T06:17:51.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:17:51.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:51.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566317+0000", 2023-10-09T06:17:51.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7070000000000001e-06 2023-10-09T06:17:51.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:51.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566323+0000", 2023-10-09T06:17:51.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8699999999999997e-06 2023-10-09T06:17:51.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.896 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:51.896 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566328+0000", 2023-10-09T06:17:51.896 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0470000000000001e-06 2023-10-09T06:17:51.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.896 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.896 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566453+0000", 2023-10-09T06:17:51.896 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012555500000000001 2023-10-09T06:17:51.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:51.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566572+0000", 2023-10-09T06:17:51.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118514 2023-10-09T06:17:51.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2023-10-09T06:17:51.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566699+0000", 2023-10-09T06:17:51.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012719000000000001 2023-10-09T06:17:51.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:51.898 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.571304+0000", 2023-10-09T06:17:51.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046050659999999997 2023-10-09T06:17:51.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.898 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.898 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.587174+0000", 2023-10-09T06:17:51.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015869695 2023-10-09T06:17:51.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.898 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.587186+0000", 2023-10-09T06:17:51.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2255e-05 2023-10-09T06:17:51.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.600012+0000", 2023-10-09T06:17:51.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0128258990000001 2023-10-09T06:17:51.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.600023+0000", 2023-10-09T06:17:51.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1344e-05 2023-10-09T06:17:51.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.774248+0000", 2023-10-09T06:17:51.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.17422523 2023-10-09T06:17:51.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:51.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.774333+0000", 2023-10-09T06:17:51.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5148999999999994e-05 2023-10-09T06:17:51.901 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.901 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:51.901 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.901 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.901 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:114776 11.9 11:90bf3a7c:::benchmark_data_smithi064_631903_object5862:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-10-09T06:17:51.901 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:17:46.580825+0000", 2023-10-09T06:17:51.901 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 5.2018287829999998, 2023-10-09T06:17:51.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2761611980000001, 2023-10-09T06:17:51.901 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:51.901 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:51.901 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:51.902 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:17:51.902 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:17:51.902 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 114776 2023-10-09T06:17:51.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.902 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:51.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:51.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.580825+0000", 2023-10-09T06:17:51.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.580825+0000", 2023-10-09T06:17:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.580823+0000", 2023-10-09T06:17:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:17:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:51.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.580830+0000", 2023-10-09T06:17:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9840000000000004e-06 2023-10-09T06:17:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.580841+0000", 2023-10-09T06:17:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1916000000000001e-05 2023-10-09T06:17:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.580850+0000", 2023-10-09T06:17:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2479999999999996e-06 2023-10-09T06:17:51.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.905 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.905 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.905 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.587212+0000", 2023-10-09T06:17:51.905 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063625649999999997 2023-10-09T06:17:51.905 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.905 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.905 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.905 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.587220+0000", 2023-10-09T06:17:51.905 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1410000000000005e-06 2023-10-09T06:17:51.905 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.905 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.905 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.906 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.603487+0000", 2023-10-09T06:17:51.906 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0162665470000001 2023-10-09T06:17:51.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.906 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.906 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.603496+0000", 2023-10-09T06:17:51.906 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3589999999999997e-06 2023-10-09T06:17:51.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.906 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.823894+0000", 2023-10-09T06:17:51.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2203975890000001 2023-10-09T06:17:51.907 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.907 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:51.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.824020+0000", 2023-10-09T06:17:51.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000125685 2023-10-09T06:17:51.907 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.907 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1,6", 2023-10-09T06:17:51.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.824551+0000", 2023-10-09T06:17:51.908 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00053100900000000004 2023-10-09T06:17:51.908 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.908 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.908 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.908 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.847637+0000", 2023-10-09T06:17:51.908 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023086654000000002 2023-10-09T06:17:51.908 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.908 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.908 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.908 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.847674+0000", 2023-10-09T06:17:51.908 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6338999999999999e-05 2023-10-09T06:17:51.909 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.909 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.909 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.909 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.847897+0000", 2023-10-09T06:17:51.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022390999999999999 2023-10-09T06:17:51.909 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.909 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.909 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:51.909 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.856840+0000", 2023-10-09T06:17:51.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0089426409999999994 2023-10-09T06:17:51.909 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.910 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:51.910 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.856893+0000", 2023-10-09T06:17:51.910 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2531999999999998e-05 2023-10-09T06:17:51.910 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.910 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:51.910 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.856986+0000", 2023-10-09T06:17:51.910 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3219000000000003e-05 2023-10-09T06:17:51.910 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.910 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:51.911 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.911 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.911 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.911 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:114779 11.a 11:51407f43:::benchmark_data_smithi064_631903_object5865:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-10-09T06:17:51.911 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:17:46.581278+0000", 2023-10-09T06:17:51.911 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 5.2013758929999998, 2023-10-09T06:17:51.911 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2608115089999998, 2023-10-09T06:17:51.911 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:51.911 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:51.911 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:51.911 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:17:51.912 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:17:51.912 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 114779 2023-10-09T06:17:51.912 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.912 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:51.912 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.912 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:51.912 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581278+0000", 2023-10-09T06:17:51.912 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.912 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.912 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.912 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:51.912 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581278+0000", 2023-10-09T06:17:51.913 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.913 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.913 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.913 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:51.913 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581277+0000", 2023-10-09T06:17:51.913 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:17:51.913 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.913 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.913 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:51.913 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581281+0000", 2023-10-09T06:17:51.913 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1339999999999997e-06 2023-10-09T06:17:51.913 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.914 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:51.914 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581291+0000", 2023-10-09T06:17:51.914 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9820000000000003e-06 2023-10-09T06:17:51.914 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.914 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:51.914 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581298+0000", 2023-10-09T06:17:51.917 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6989999999999997e-06 2023-10-09T06:17:51.917 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.917 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.917 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.918 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581500+0000", 2023-10-09T06:17:51.918 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020228499999999999 2023-10-09T06:17:51.918 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.918 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.918 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.918 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581509+0000", 2023-10-09T06:17:51.918 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8130000000000002e-06 2023-10-09T06:17:51.918 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.918 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.918 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.918 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.601561+0000", 2023-10-09T06:17:51.918 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0200516829999999 2023-10-09T06:17:51.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.919 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.919 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.601573+0000", 2023-10-09T06:17:51.919 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2702e-05 2023-10-09T06:17:51.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.919 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.919 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.802518+0000", 2023-10-09T06:17:51.919 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2009447630000001 2023-10-09T06:17:51.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.920 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:51.920 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.802650+0000", 2023-10-09T06:17:51.920 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013167699999999999 2023-10-09T06:17:51.920 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.920 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.920 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T06:17:51.920 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.803198+0000", 2023-10-09T06:17:51.920 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00054843800000000005 2023-10-09T06:17:51.920 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.920 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.920 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.920 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.841745+0000", 2023-10-09T06:17:51.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.038546634000000003 2023-10-09T06:17:51.921 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.921 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.921 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.841775+0000", 2023-10-09T06:17:51.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0762999999999997e-05 2023-10-09T06:17:51.921 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.921 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.921 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.841910+0000", 2023-10-09T06:17:51.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000134735 2023-10-09T06:17:51.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.922 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:51.922 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.841945+0000", 2023-10-09T06:17:51.922 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.468e-05 2023-10-09T06:17:51.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.922 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:51.922 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.841990+0000", 2023-10-09T06:17:51.922 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4659000000000001e-05 2023-10-09T06:17:51.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.923 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.923 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:51.923 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.842089+0000", 2023-10-09T06:17:51.923 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9487000000000005e-05 2023-10-09T06:17:51.923 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.923 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:51.923 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.923 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.923 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.923 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:114781 11.1 11:882127a0:::benchmark_data_smithi064_631903_object5867:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-10-09T06:17:51.923 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:17:46.581303+0000", 2023-10-09T06:17:51.924 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 5.2013505450000004, 2023-10-09T06:17:51.924 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2665437900000001, 2023-10-09T06:17:51.924 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:51.924 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:51.924 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:51.924 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:17:51.924 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:17:51.924 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 114781 2023-10-09T06:17:51.924 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.924 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:51.924 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.925 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:51.925 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581303+0000", 2023-10-09T06:17:51.925 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.925 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.925 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.925 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:51.925 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581303+0000", 2023-10-09T06:17:51.925 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.925 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.925 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.925 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:51.926 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581303+0000", 2023-10-09T06:17:51.926 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:17:51.926 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.926 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.926 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:51.926 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581305+0000", 2023-10-09T06:17:51.926 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4499999999999998e-06 2023-10-09T06:17:51.926 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.926 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.926 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:51.926 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581311+0000", 2023-10-09T06:17:51.926 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0789999999999997e-06 2023-10-09T06:17:51.927 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.927 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.927 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:51.927 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581316+0000", 2023-10-09T06:17:51.927 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7840000000000003e-06 2023-10-09T06:17:51.927 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.927 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.927 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.927 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.587243+0000", 2023-10-09T06:17:51.927 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059268660000000003 2023-10-09T06:17:51.927 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.928 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.928 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.928 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.587250+0000", 2023-10-09T06:17:51.928 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2099999999999996e-06 2023-10-09T06:17:51.928 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.928 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.928 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.928 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.603461+0000", 2023-10-09T06:17:51.928 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.016211451 2023-10-09T06:17:51.928 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.928 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.928 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.929 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.603469+0000", 2023-10-09T06:17:51.929 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8679999999999999e-06 2023-10-09T06:17:51.929 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.929 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.929 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.929 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.774358+0000", 2023-10-09T06:17:51.929 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.170888288 2023-10-09T06:17:51.929 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.929 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.929 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:51.929 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.774485+0000", 2023-10-09T06:17:51.929 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000127532 2023-10-09T06:17:51.930 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.930 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.930 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,6", 2023-10-09T06:17:51.930 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.775115+0000", 2023-10-09T06:17:51.930 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00062980000000000002 2023-10-09T06:17:51.930 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.930 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.930 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.930 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.791339+0000", 2023-10-09T06:17:51.930 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016224072999999999 2023-10-09T06:17:51.930 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.931 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.931 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:51.931 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.794574+0000", 2023-10-09T06:17:51.931 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032347840000000001 2023-10-09T06:17:51.931 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.931 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.931 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.931 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.806294+0000", 2023-10-09T06:17:51.931 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011720563 2023-10-09T06:17:51.931 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.931 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.932 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.932 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.847650+0000", 2023-10-09T06:17:51.932 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.041356182999999998 2023-10-09T06:17:51.932 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.932 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.932 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:51.932 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.847714+0000", 2023-10-09T06:17:51.932 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3434000000000003e-05 2023-10-09T06:17:51.932 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.932 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.932 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:51.933 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.847847+0000", 2023-10-09T06:17:51.933 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000132812 2023-10-09T06:17:51.933 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.933 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:51.933 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.933 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.933 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.933 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:114783 11.1 11:83e60dcb:::benchmark_data_smithi064_631903_object5869:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-10-09T06:17:51.933 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:17:46.581336+0000", 2023-10-09T06:17:51.933 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 5.2013175360000004, 2023-10-09T06:17:51.933 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2667203570000001, 2023-10-09T06:17:51.933 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:51.934 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:51.934 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:51.934 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:17:51.934 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:17:51.934 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 114783 2023-10-09T06:17:51.934 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.934 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:51.934 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.934 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:51.934 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581336+0000", 2023-10-09T06:17:51.934 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.934 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.935 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:51.935 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581336+0000", 2023-10-09T06:17:51.935 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.935 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.935 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:51.935 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581336+0000", 2023-10-09T06:17:51.935 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:17:51.935 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.935 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:51.936 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581338+0000", 2023-10-09T06:17:51.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3719999999999999e-06 2023-10-09T06:17:51.936 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.936 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.936 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:51.936 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581343+0000", 2023-10-09T06:17:51.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6750000000000001e-06 2023-10-09T06:17:51.936 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.936 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.936 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:51.936 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581347+0000", 2023-10-09T06:17:51.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3889999999999998e-06 2023-10-09T06:17:51.937 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.937 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.937 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.587272+0000", 2023-10-09T06:17:51.937 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059252380000000002 2023-10-09T06:17:51.937 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.937 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.937 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.587281+0000", 2023-10-09T06:17:51.937 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4400000000000005e-06 2023-10-09T06:17:51.937 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.938 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.938 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.603600+0000", 2023-10-09T06:17:51.938 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.016319258 2023-10-09T06:17:51.938 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.938 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.938 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.938 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.603609+0000", 2023-10-09T06:17:51.938 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1779999999999992e-06 2023-10-09T06:17:51.938 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.938 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.938 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.938 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.791551+0000", 2023-10-09T06:17:51.939 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.187941699 2023-10-09T06:17:51.939 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.939 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.939 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:51.939 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.791663+0000", 2023-10-09T06:17:51.939 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000112191 2023-10-09T06:17:51.939 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.939 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.939 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,6", 2023-10-09T06:17:51.939 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.791755+0000", 2023-10-09T06:17:51.939 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2168000000000003e-05 2023-10-09T06:17:51.939 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.940 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.940 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.805268+0000", 2023-10-09T06:17:51.940 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013512266 2023-10-09T06:17:51.940 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.940 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:51.940 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.805290+0000", 2023-10-09T06:17:51.940 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2599000000000002e-05 2023-10-09T06:17:51.940 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.940 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.941 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.847595+0000", 2023-10-09T06:17:51.941 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.042304464 2023-10-09T06:17:51.941 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.941 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.941 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.941 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.847910+0000", 2023-10-09T06:17:51.941 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00031572000000000002 2023-10-09T06:17:51.941 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.941 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.941 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:51.941 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.847951+0000", 2023-10-09T06:17:51.941 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0423000000000002e-05 2023-10-09T06:17:51.942 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.942 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.942 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:51.942 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.848056+0000", 2023-10-09T06:17:51.942 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010554500000000001 2023-10-09T06:17:51.942 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.942 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:51.942 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.942 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.943 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.943 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:114785 11.9 11:9e2e2456:::benchmark_data_smithi064_631903_object5871:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-10-09T06:17:51.943 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:17:46.585044+0000", 2023-10-09T06:17:51.943 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 5.1976094049999997, 2023-10-09T06:17:51.943 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2886440160000001, 2023-10-09T06:17:51.943 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:51.943 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:51.943 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:51.943 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:17:51.943 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:17:51.943 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 114785 2023-10-09T06:17:51.944 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.944 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:51.944 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.944 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:51.944 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.585044+0000", 2023-10-09T06:17:51.944 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.944 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.944 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.944 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:51.944 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.585044+0000", 2023-10-09T06:17:51.944 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.944 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.945 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.945 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:51.945 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.585042+0000", 2023-10-09T06:17:51.945 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:17:51.945 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.945 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.945 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:51.945 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.585050+0000", 2023-10-09T06:17:51.945 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3350000000000001e-06 2023-10-09T06:17:51.945 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.945 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.945 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:51.945 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.585073+0000", 2023-10-09T06:17:51.946 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3804000000000001e-05 2023-10-09T06:17:51.946 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.946 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.946 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:51.946 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.585083+0000", 2023-10-09T06:17:51.946 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3919999999999992e-06 2023-10-09T06:17:51.946 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.946 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.946 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.947 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.587303+0000", 2023-10-09T06:17:51.947 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002220559 2023-10-09T06:17:51.947 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.947 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.947 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.947 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.587311+0000", 2023-10-09T06:17:51.947 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7919999999999993e-06 2023-10-09T06:17:51.947 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.947 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.947 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.947 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.603524+0000", 2023-10-09T06:17:51.948 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0162130540000001 2023-10-09T06:17:51.948 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.948 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.948 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.948 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.603534+0000", 2023-10-09T06:17:51.948 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0051000000000001e-05 2023-10-09T06:17:51.948 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.948 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.948 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.948 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.847924+0000", 2023-10-09T06:17:51.948 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.244389792 2023-10-09T06:17:51.949 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.949 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.949 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:51.949 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.848069+0000", 2023-10-09T06:17:51.949 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014492500000000001 2023-10-09T06:17:51.949 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.949 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.949 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1,6", 2023-10-09T06:17:51.949 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.848220+0000", 2023-10-09T06:17:51.950 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015109300000000001 2023-10-09T06:17:51.950 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.950 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.950 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.950 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.856828+0000", 2023-10-09T06:17:51.950 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086078479999999995 2023-10-09T06:17:51.950 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.950 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.950 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:51.950 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.856995+0000", 2023-10-09T06:17:51.950 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016734700000000001 2023-10-09T06:17:51.951 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.951 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.951 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.951 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.873490+0000", 2023-10-09T06:17:51.951 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016494463000000001 2023-10-09T06:17:51.951 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.951 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.951 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.951 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.873518+0000", 2023-10-09T06:17:51.951 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8430000000000001e-05 2023-10-09T06:17:51.952 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.952 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.952 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:51.952 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.873572+0000", 2023-10-09T06:17:51.952 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3742999999999997e-05 2023-10-09T06:17:51.952 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.952 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.952 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:51.952 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.873688+0000", 2023-10-09T06:17:51.952 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116193 2023-10-09T06:17:51.953 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.953 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:51.953 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.953 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.953 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.953 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:114789 11.a 11:59c1771f:::benchmark_data_smithi064_631903_object5875:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-10-09T06:17:51.953 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:17:46.591783+0000", 2023-10-09T06:17:51.953 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 5.1908707820000002, 2023-10-09T06:17:51.953 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2665931619999999, 2023-10-09T06:17:51.953 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:51.953 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:51.954 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:51.954 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:17:51.954 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:17:51.954 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 114789 2023-10-09T06:17:51.954 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.954 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:51.954 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.954 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:51.954 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.591783+0000", 2023-10-09T06:17:51.954 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.954 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.954 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.955 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:51.955 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.591783+0000", 2023-10-09T06:17:51.955 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.955 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.955 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.955 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:51.955 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.591781+0000", 2023-10-09T06:17:51.955 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:17:51.955 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.955 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.955 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:51.956 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.591789+0000", 2023-10-09T06:17:51.956 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0150000000000003e-06 2023-10-09T06:17:51.956 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.956 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.956 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:51.956 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.591805+0000", 2023-10-09T06:17:51.956 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6209999999999999e-05 2023-10-09T06:17:51.956 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.956 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.956 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:51.957 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.591814+0000", 2023-10-09T06:17:51.957 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4509999999999998e-06 2023-10-09T06:17:51.957 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.957 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.957 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.957 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.591936+0000", 2023-10-09T06:17:51.957 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012205 2023-10-09T06:17:51.957 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.957 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.957 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.957 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.591946+0000", 2023-10-09T06:17:51.957 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6660000000000003e-06 2023-10-09T06:17:51.958 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.958 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.958 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.958 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.602104+0000", 2023-10-09T06:17:51.958 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0101581639999999 2023-10-09T06:17:51.958 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.958 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.958 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.958 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.602116+0000", 2023-10-09T06:17:51.958 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1389999999999999e-05 2023-10-09T06:17:51.959 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.959 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.959 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.959 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.838602+0000", 2023-10-09T06:17:51.959 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2364861620000001 2023-10-09T06:17:51.959 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.959 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.959 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:51.959 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.838773+0000", 2023-10-09T06:17:51.959 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017120899999999999 2023-10-09T06:17:51.959 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.960 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.960 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T06:17:51.960 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.838924+0000", 2023-10-09T06:17:51.960 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015081799999999999 2023-10-09T06:17:51.960 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.960 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.960 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.960 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.841938+0000", 2023-10-09T06:17:51.960 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030146560000000001 2023-10-09T06:17:51.960 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.960 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.961 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:51.961 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.842098+0000", 2023-10-09T06:17:51.961 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000159207 2023-10-09T06:17:51.961 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.961 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.961 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.961 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.851850+0000", 2023-10-09T06:17:51.961 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0097527059999999999 2023-10-09T06:17:51.961 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.961 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.962 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.962 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.858207+0000", 2023-10-09T06:17:51.962 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006356535 2023-10-09T06:17:51.962 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.962 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.962 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:51.962 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.858254+0000", 2023-10-09T06:17:51.962 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7511e-05 2023-10-09T06:17:51.962 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.962 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.962 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:51.962 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.858376+0000", 2023-10-09T06:17:51.963 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000121488 2023-10-09T06:17:51.963 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.963 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:51.963 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.963 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.963 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.963 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:114796 11.9 11:95741733:::benchmark_data_smithi064_631903_object5882:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-10-09T06:17:51.963 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:17:46.615408+0000", 2023-10-09T06:17:51.963 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 5.1672453440000004, 2023-10-09T06:17:51.963 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.277054293, 2023-10-09T06:17:51.963 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:51.964 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:51.964 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:51.964 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:17:51.964 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:17:51.964 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 114796 2023-10-09T06:17:51.964 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.964 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:51.964 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.964 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:51.964 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615408+0000", 2023-10-09T06:17:51.965 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.965 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.965 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.965 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:51.965 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615408+0000", 2023-10-09T06:17:51.965 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.965 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.965 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.965 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:51.965 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615407+0000", 2023-10-09T06:17:51.965 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:17:51.966 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.966 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.966 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:51.966 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615412+0000", 2023-10-09T06:17:51.966 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0309999999999998e-06 2023-10-09T06:17:51.966 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.966 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.966 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:51.966 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615423+0000", 2023-10-09T06:17:51.966 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1311000000000001e-05 2023-10-09T06:17:51.966 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.966 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.967 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:51.967 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615430+0000", 2023-10-09T06:17:51.967 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0589999999999997e-06 2023-10-09T06:17:51.967 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.967 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.967 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.967 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615497+0000", 2023-10-09T06:17:51.967 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6364000000000004e-05 2023-10-09T06:17:51.967 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.967 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.967 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.968 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615513+0000", 2023-10-09T06:17:51.968 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6416999999999999e-05 2023-10-09T06:17:51.968 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.968 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.968 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.968 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.603565+0000", 2023-10-09T06:17:51.968 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98805211299999995 2023-10-09T06:17:51.968 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.968 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.968 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.968 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.603575+0000", 2023-10-09T06:17:51.968 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2830000000000007e-06 2023-10-09T06:17:51.969 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.969 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.969 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.969 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.853707+0000", 2023-10-09T06:17:51.969 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2501325670000001 2023-10-09T06:17:51.969 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.969 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.969 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:51.969 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.853838+0000", 2023-10-09T06:17:51.969 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013037199999999999 2023-10-09T06:17:51.969 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.970 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.970 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1,6", 2023-10-09T06:17:51.970 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.853976+0000", 2023-10-09T06:17:51.970 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000138868 2023-10-09T06:17:51.970 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.970 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.970 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:51.970 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.857759+0000", 2023-10-09T06:17:51.970 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037828190000000002 2023-10-09T06:17:51.970 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.970 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.971 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.971 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.866740+0000", 2023-10-09T06:17:51.971 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0089810029999999996 2023-10-09T06:17:51.971 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.971 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.971 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.971 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.880757+0000", 2023-10-09T06:17:51.971 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014017201 2023-10-09T06:17:51.971 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.971 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.971 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.971 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.892300+0000", 2023-10-09T06:17:51.972 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011542663999999999 2023-10-09T06:17:51.972 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.972 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.972 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:51.972 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.892351+0000", 2023-10-09T06:17:51.972 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0479999999999998e-05 2023-10-09T06:17:51.972 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.972 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.972 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:51.972 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.892462+0000", 2023-10-09T06:17:51.973 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011181 2023-10-09T06:17:51.973 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.973 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:51.973 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.973 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.973 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.973 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:114798 11.8 11:18050a8d:::benchmark_data_smithi064_631903_object5884:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-10-09T06:17:51.973 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:17:46.615516+0000", 2023-10-09T06:17:51.973 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 5.1671371349999999, 2023-10-09T06:17:51.973 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2070446920000002, 2023-10-09T06:17:51.973 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:17:51.974 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:17:51.974 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:17:51.974 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:17:51.974 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:17:51.974 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 114798 2023-10-09T06:17:51.974 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.974 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:17:51.974 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.974 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:17:51.975 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615516+0000", 2023-10-09T06:17:51.975 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.975 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.975 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.975 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:17:51.975 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615516+0000", 2023-10-09T06:17:51.975 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:17:51.975 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.975 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.975 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:17:51.975 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615516+0000", 2023-10-09T06:17:51.975 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:17:51.976 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.976 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.976 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:17:51.976 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615520+0000", 2023-10-09T06:17:51.976 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5029999999999999e-06 2023-10-09T06:17:51.976 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.976 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.976 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:17:51.976 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615537+0000", 2023-10-09T06:17:51.976 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6646999999999998e-05 2023-10-09T06:17:51.976 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.976 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.977 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:17:51.977 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615542+0000", 2023-10-09T06:17:51.977 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9989999999999999e-06 2023-10-09T06:17:51.977 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.977 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.977 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.977 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615600+0000", 2023-10-09T06:17:51.977 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8559e-05 2023-10-09T06:17:51.977 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.977 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.977 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.977 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615614+0000", 2023-10-09T06:17:51.978 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3192e-05 2023-10-09T06:17:51.978 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.978 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.978 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.978 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.599195+0000", 2023-10-09T06:17:51.978 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98358131199999999 2023-10-09T06:17:51.978 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.978 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.978 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:17:51.978 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.599208+0000", 2023-10-09T06:17:51.978 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3329999999999999e-05 2023-10-09T06:17:51.979 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.979 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.979 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:17:51.979 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.803002+0000", 2023-10-09T06:17:51.979 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2037934539999999 2023-10-09T06:17:51.979 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.979 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.979 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:17:51.979 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.803120+0000", 2023-10-09T06:17:51.979 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118214 2023-10-09T06:17:51.979 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.979 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.979 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T06:17:51.980 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.803665+0000", 2023-10-09T06:17:51.980 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00054509399999999998 2023-10-09T06:17:51.980 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.980 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.980 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.980 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.820553+0000", 2023-10-09T06:17:51.980 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016888027999999999 2023-10-09T06:17:51.980 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.980 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.980 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:17:51.980 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.820799+0000", 2023-10-09T06:17:51.981 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00024631399999999997 2023-10-09T06:17:51.981 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.981 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.981 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.981 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.822353+0000", 2023-10-09T06:17:51.981 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001553376 2023-10-09T06:17:51.981 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.981 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.981 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:17:51.981 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.822391+0000", 2023-10-09T06:17:51.981 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7916000000000001e-05 2023-10-09T06:17:51.982 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.982 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.982 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:17:51.982 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.822450+0000", 2023-10-09T06:17:51.982 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9490999999999997e-05 2023-10-09T06:17:51.982 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:17:51.982 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:17:51.982 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:17:51.982 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.822561+0000", 2023-10-09T06:17:51.982 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0001109 2023-10-09T06:17:51.982 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.982 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:51.983 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.983 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:17:51.983 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:17:51.983 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:17:51.983 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:51.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:51 smithi064 ceph-mon[220044]: osdmap e598: 8 total, 8 up, 6 in 2023-10-09T06:17:51.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:51 smithi064 ceph-mon[220044]: failed to encode map e598 with expected crc 2023-10-09T06:17:51.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:51 smithi064 ceph-mon[220044]: failed to encode map e598 with expected crc 2023-10-09T06:17:51.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:51 smithi064 ceph-mon[220044]: failed to encode map e598 with expected crc 2023-10-09T06:17:51.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:51 smithi064 ceph-mon[220044]: failed to encode map e598 with expected crc 2023-10-09T06:17:51.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:51 smithi064 ceph-mon[220044]: failed to encode map e598 with expected crc 2023-10-09T06:17:51.985 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:51 smithi064 ceph-mon[220044]: failed to encode map e598 with expected crc 2023-10-09T06:17:51.985 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:51 smithi064 ceph-mon[220044]: failed to encode map e598 with expected crc 2023-10-09T06:17:51.985 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:51 smithi064 ceph-mon[220044]: failed to encode map e598 with expected crc 2023-10-09T06:17:51.985 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:51 smithi064 ceph-mon[212750]: osdmap e598: 8 total, 8 up, 6 in 2023-10-09T06:17:51.985 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:51 smithi064 ceph-mon[212750]: failed to encode map e598 with expected crc 2023-10-09T06:17:51.985 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:51 smithi064 ceph-mon[212750]: failed to encode map e598 with expected crc 2023-10-09T06:17:51.985 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:51 smithi064 ceph-mon[212750]: failed to encode map e598 with expected crc 2023-10-09T06:17:51.985 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:51 smithi064 ceph-mon[212750]: failed to encode map e598 with expected crc 2023-10-09T06:17:51.986 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:51 smithi064 ceph-mon[212750]: failed to encode map e598 with expected crc 2023-10-09T06:17:51.986 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:51 smithi064 ceph-mon[212750]: failed to encode map e598 with expected crc 2023-10-09T06:17:51.986 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:51 smithi064 ceph-mon[212750]: failed to encode map e598 with expected crc 2023-10-09T06:17:51.986 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:51 smithi064 ceph-mon[212750]: failed to encode map e598 with expected crc 2023-10-09T06:17:52.061 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:52.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:51 smithi156 ceph-mon[195101]: osdmap e598: 8 total, 8 up, 6 in 2023-10-09T06:17:52.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:51 smithi156 ceph-mon[195101]: failed to encode map e598 with expected crc 2023-10-09T06:17:52.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:51 smithi156 ceph-mon[195101]: failed to encode map e598 with expected crc 2023-10-09T06:17:52.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:51 smithi156 ceph-mon[195101]: failed to encode map e598 with expected crc 2023-10-09T06:17:52.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:51 smithi156 ceph-mon[195101]: failed to encode map e598 with expected crc 2023-10-09T06:17:52.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:51 smithi156 ceph-mon[195101]: failed to encode map e598 with expected crc 2023-10-09T06:17:52.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:51 smithi156 ceph-mon[195101]: failed to encode map e598 with expected crc 2023-10-09T06:17:52.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:51 smithi156 ceph-mon[195101]: failed to encode map e598 with expected crc 2023-10-09T06:17:52.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:51 smithi156 ceph-mon[195101]: failed to encode map e598 with expected crc 2023-10-09T06:17:52.199 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:52.207 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:17:52.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:52.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:52.253 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:17:52.291 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:17:52.314 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:52.326 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:17:52.362 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:17:52.394 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:52.401 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:17:52.446 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:17:52.500 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:17:52.501 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:17:52.513 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:17:52.513 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:17:52.513 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:17:52.513 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:17:52.513 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:17:52.513 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:17:52.513 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:17:52.513 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:17:52.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:52.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:52.615 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:52.672 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:52.788 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:17:52.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:52.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:52.903 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:52.951 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:52 smithi156 ceph-mon[195101]: pgmap v1537: 119 pgs: 2 active+remapped+backfilling, 6 active+remapped+backfill_wait, 3 active+undersized+remapped, 9 peering, 99 active+clean; 6.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 361 op/s; 49527/307176 objects misplaced (16.123%); 23 MiB/s, 2 keys/s, 371 objects/s recovering 2023-10-09T06:17:52.951 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:52 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3027555140' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:17:52.958 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:52.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:52 smithi064 ceph-mon[212750]: pgmap v1537: 119 pgs: 2 active+remapped+backfilling, 6 active+remapped+backfill_wait, 3 active+undersized+remapped, 9 peering, 99 active+clean; 6.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 361 op/s; 49527/307176 objects misplaced (16.123%); 23 MiB/s, 2 keys/s, 371 objects/s recovering 2023-10-09T06:17:52.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:52 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3027555140' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:17:52.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:52 smithi064 ceph-mon[220044]: pgmap v1537: 119 pgs: 2 active+remapped+backfilling, 6 active+remapped+backfill_wait, 3 active+undersized+remapped, 9 peering, 99 active+clean; 6.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 361 op/s; 49527/307176 objects misplaced (16.123%); 23 MiB/s, 2 keys/s, 371 objects/s recovering 2023-10-09T06:17:52.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:52 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3027555140' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:17:53.062 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:17:53.067 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:17:53.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:53.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:53.187 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:53.223 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 7, 1] out_osds: [2, 6] dead_osds: [] live_osds: [1, 2, 3, 5, 6, 7, 4, 0] 2023-10-09T06:17:53.223 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:17:53.223 INFO:tasks.thrashosds.thrasher:Killing osd 5, live_osds are [1, 2, 3, 5, 6, 7, 4, 0] 2023-10-09T06:17:53.223 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-10-09T06:17:53.223 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.5 2023-10-09T06:17:53.243 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:53.348 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:17:53.620 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:17:53 smithi156 systemd[1]: Stopping Ceph osd.5 for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T06:17:53.728 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:17:53.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:53.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:53.761 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:17:53.836 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:53.876 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:53.932 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:17:53.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:53.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:53.968 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:54.023 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:53 smithi156 ceph-mon[195101]: osd.5 marked itself down and dead 2023-10-09T06:17:54.023 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:17:53 smithi156 conmon[123547]: 2023-10-09T06:17:53.703+0000 7fd2fac26700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:17:54.023 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:17:53 smithi156 conmon[123547]: 2023-10-09T06:17:53.703+0000 7fd2fac26700 -1 osd.5 598 *** Got signal Terminated *** 2023-10-09T06:17:54.023 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:17:53 smithi156 conmon[123547]: 2023-10-09T06:17:53.703+0000 7fd2fac26700 -1 osd.5 598 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:17:54.037 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:54.070 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:54.123 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:17:54.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:54.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:54.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:53 smithi064 ceph-mon[212750]: osd.5 marked itself down and dead 2023-10-09T06:17:54.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:53 smithi064 ceph-mon[220044]: osd.5 marked itself down and dead 2023-10-09T06:17:54.228 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:54.266 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:54.321 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:54.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:54.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:54.425 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:54.461 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:54.622 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:17:54.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:54.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:54.726 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:54.762 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:54 smithi156 ceph-mon[195101]: pgmap v1538: 119 pgs: 2 active+remapped+backfilling, 6 active+remapped+backfill_wait, 3 active+undersized+remapped, 108 active+clean; 6.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 352 op/s; 49527/305484 objects misplaced (16.213%); 16 MiB/s, 1 keys/s, 262 objects/s recovering 2023-10-09T06:17:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:54 smithi156 ceph-mon[195101]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:17:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:54 smithi156 ceph-mon[195101]: failed to encode map e599 with expected crc 2023-10-09T06:17:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:54 smithi156 ceph-mon[195101]: osdmap e599: 8 total, 7 up, 6 in 2023-10-09T06:17:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:54 smithi156 ceph-mon[195101]: failed to encode map e599 with expected crc 2023-10-09T06:17:55.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:54 smithi156 ceph-mon[195101]: failed to encode map e599 with expected crc 2023-10-09T06:17:55.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:54 smithi064 ceph-mon[212750]: pgmap v1538: 119 pgs: 2 active+remapped+backfilling, 6 active+remapped+backfill_wait, 3 active+undersized+remapped, 108 active+clean; 6.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 352 op/s; 49527/305484 objects misplaced (16.213%); 16 MiB/s, 1 keys/s, 262 objects/s recovering 2023-10-09T06:17:55.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:54 smithi064 ceph-mon[212750]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:17:55.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:54 smithi064 ceph-mon[212750]: failed to encode map e599 with expected crc 2023-10-09T06:17:55.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:54 smithi064 ceph-mon[212750]: osdmap e599: 8 total, 7 up, 6 in 2023-10-09T06:17:55.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:54 smithi064 ceph-mon[212750]: failed to encode map e599 with expected crc 2023-10-09T06:17:55.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:54 smithi064 ceph-mon[212750]: failed to encode map e599 with expected crc 2023-10-09T06:17:55.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:54 smithi064 ceph-mon[220044]: pgmap v1538: 119 pgs: 2 active+remapped+backfilling, 6 active+remapped+backfill_wait, 3 active+undersized+remapped, 108 active+clean; 6.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 352 op/s; 49527/305484 objects misplaced (16.213%); 16 MiB/s, 1 keys/s, 262 objects/s recovering 2023-10-09T06:17:55.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:54 smithi064 ceph-mon[220044]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:17:55.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:54 smithi064 ceph-mon[220044]: failed to encode map e599 with expected crc 2023-10-09T06:17:55.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:54 smithi064 ceph-mon[220044]: osdmap e599: 8 total, 7 up, 6 in 2023-10-09T06:17:55.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:54 smithi064 ceph-mon[220044]: failed to encode map e599 with expected crc 2023-10-09T06:17:55.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:54 smithi064 ceph-mon[220044]: failed to encode map e599 with expected crc 2023-10-09T06:17:55.620 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:17:55 smithi156 bash[401268]: 40d9335e2ab7f04041f18b24a9171003d4f9d045eda69d83536b097c3b4add6e 2023-10-09T06:17:55.721 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:17:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:17:55] "GET /metrics HTTP/1.1" 200 36451 "" "Prometheus/2.43.0" 2023-10-09T06:17:55.862 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:17:56.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:55 smithi156 ceph-mon[195101]: osdmap e600: 8 total, 7 up, 6 in 2023-10-09T06:17:56.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:55 smithi156 ceph-mon[195101]: failed to encode map e600 with expected crc 2023-10-09T06:17:56.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:55 smithi156 ceph-mon[195101]: 11.a starting backfill to osd.1 from (580'2366,598'7308] MIN to 598'7309 2023-10-09T06:17:56.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:55 smithi156 ceph-mon[195101]: failed to encode map e600 with expected crc 2023-10-09T06:17:56.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:55 smithi156 ceph-mon[195101]: 11.8 continuing backfill to osd.0 from (581'2539,598'7470] MIN to 598'7470 2023-10-09T06:17:56.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:55 smithi156 ceph-mon[195101]: failed to encode map e600 with expected crc 2023-10-09T06:17:56.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:55 smithi156 ceph-mon[195101]: failed to encode map e600 with expected crc 2023-10-09T06:17:56.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:55 smithi156 ceph-mon[195101]: failed to encode map e600 with expected crc 2023-10-09T06:17:56.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:55 smithi156 ceph-mon[195101]: failed to encode map e600 with expected crc 2023-10-09T06:17:56.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:55 smithi156 ceph-mon[195101]: 11.5 continuing backfill to osd.3 from (580'2250,598'7445] MIN to 598'7445 2023-10-09T06:17:56.132 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:17:56.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:56.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:56.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:55 smithi064 ceph-mon[212750]: osdmap e600: 8 total, 7 up, 6 in 2023-10-09T06:17:56.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:55 smithi064 ceph-mon[212750]: failed to encode map e600 with expected crc 2023-10-09T06:17:56.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:55 smithi064 ceph-mon[212750]: 11.a starting backfill to osd.1 from (580'2366,598'7308] MIN to 598'7309 2023-10-09T06:17:56.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:55 smithi064 ceph-mon[212750]: failed to encode map e600 with expected crc 2023-10-09T06:17:56.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:55 smithi064 ceph-mon[212750]: 11.8 continuing backfill to osd.0 from (581'2539,598'7470] MIN to 598'7470 2023-10-09T06:17:56.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:55 smithi064 ceph-mon[212750]: failed to encode map e600 with expected crc 2023-10-09T06:17:56.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:55 smithi064 ceph-mon[212750]: failed to encode map e600 with expected crc 2023-10-09T06:17:56.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:55 smithi064 ceph-mon[212750]: failed to encode map e600 with expected crc 2023-10-09T06:17:56.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:55 smithi064 ceph-mon[212750]: failed to encode map e600 with expected crc 2023-10-09T06:17:56.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:55 smithi064 ceph-mon[212750]: 11.5 continuing backfill to osd.3 from (580'2250,598'7445] MIN to 598'7445 2023-10-09T06:17:56.224 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:55 smithi064 ceph-mon[220044]: osdmap e600: 8 total, 7 up, 6 in 2023-10-09T06:17:56.225 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:55 smithi064 ceph-mon[220044]: failed to encode map e600 with expected crc 2023-10-09T06:17:56.225 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:55 smithi064 ceph-mon[220044]: 11.a starting backfill to osd.1 from (580'2366,598'7308] MIN to 598'7309 2023-10-09T06:17:56.225 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:55 smithi064 ceph-mon[220044]: failed to encode map e600 with expected crc 2023-10-09T06:17:56.225 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:55 smithi064 ceph-mon[220044]: 11.8 continuing backfill to osd.0 from (581'2539,598'7470] MIN to 598'7470 2023-10-09T06:17:56.225 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:55 smithi064 ceph-mon[220044]: failed to encode map e600 with expected crc 2023-10-09T06:17:56.225 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:55 smithi064 ceph-mon[220044]: failed to encode map e600 with expected crc 2023-10-09T06:17:56.225 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:55 smithi064 ceph-mon[220044]: failed to encode map e600 with expected crc 2023-10-09T06:17:56.225 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:55 smithi064 ceph-mon[220044]: failed to encode map e600 with expected crc 2023-10-09T06:17:56.226 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:55 smithi064 ceph-mon[220044]: 11.5 continuing backfill to osd.3 from (580'2250,598'7445] MIN to 598'7445 2023-10-09T06:17:56.237 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:56.275 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:56.599 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:56.889 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:17:57.107 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:56 smithi064 ceph-mon[212750]: pgmap v1541: 119 pgs: 2 remapped+peering, 12 peering, 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 99 active+clean; 6.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 403 op/s; 35943/302874 objects misplaced (11.867%); 20 MiB/s, 327 objects/s recovering 2023-10-09T06:17:57.277 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:56 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3420945112' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:17:57.278 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:56 smithi064 ceph-mon[220044]: pgmap v1541: 119 pgs: 2 remapped+peering, 12 peering, 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 99 active+clean; 6.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 403 op/s; 35943/302874 objects misplaced (11.867%); 20 MiB/s, 327 objects/s recovering 2023-10-09T06:17:57.278 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:56 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3420945112' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:17:57.278 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:56 smithi156 ceph-mon[195101]: pgmap v1541: 119 pgs: 2 remapped+peering, 12 peering, 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 99 active+clean; 6.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 403 op/s; 35943/302874 objects misplaced (11.867%); 20 MiB/s, 327 objects/s recovering 2023-10-09T06:17:57.279 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:56 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3420945112' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:17:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:57 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:17:58.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:57 smithi156 ceph-mon[195101]: failed to encode map e601 with expected crc 2023-10-09T06:17:58.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:57 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3420945112' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:17:58.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:57 smithi156 ceph-mon[195101]: osdmap e601: 8 total, 7 up, 6 in 2023-10-09T06:17:58.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:57 smithi156 ceph-mon[195101]: failed to encode map e601 with expected crc 2023-10-09T06:17:58.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:57 smithi156 ceph-mon[195101]: failed to encode map e601 with expected crc 2023-10-09T06:17:58.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:57 smithi156 ceph-mon[195101]: failed to encode map e601 with expected crc 2023-10-09T06:17:58.621 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:57 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:17:58.621 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:57 smithi064 ceph-mon[212750]: failed to encode map e601 with expected crc 2023-10-09T06:17:58.621 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:57 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3420945112' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:17:58.621 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:57 smithi064 ceph-mon[212750]: osdmap e601: 8 total, 7 up, 6 in 2023-10-09T06:17:58.621 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:57 smithi064 ceph-mon[212750]: failed to encode map e601 with expected crc 2023-10-09T06:17:58.622 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:57 smithi064 ceph-mon[212750]: failed to encode map e601 with expected crc 2023-10-09T06:17:58.622 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:57 smithi064 ceph-mon[212750]: failed to encode map e601 with expected crc 2023-10-09T06:17:58.622 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:57 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:17:58.622 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:57 smithi064 ceph-mon[220044]: failed to encode map e601 with expected crc 2023-10-09T06:17:58.622 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:57 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3420945112' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:17:58.622 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:57 smithi064 ceph-mon[220044]: osdmap e601: 8 total, 7 up, 6 in 2023-10-09T06:17:58.622 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:57 smithi064 ceph-mon[220044]: failed to encode map e601 with expected crc 2023-10-09T06:17:58.622 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:57 smithi064 ceph-mon[220044]: failed to encode map e601 with expected crc 2023-10-09T06:17:58.622 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:57 smithi064 ceph-mon[220044]: failed to encode map e601 with expected crc 2023-10-09T06:17:58.633 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:17:58.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:58.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:58.876 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:58.922 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:58.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:58 smithi064 ceph-mon[212750]: pgmap v1543: 119 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 30 active+undersized, 2 remapped+peering, 12 peering, 1 active+remapped+backfilling, 5 active+undersized+degraded, 2 active+remapped+backfill_wait, 64 active+clean; 6.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 655 op/s; 18259/295374 objects degraded (6.182%); 34941/295374 objects misplaced (11.829%); 22 MiB/s, 358 objects/s recovering 2023-10-09T06:17:58.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:58 smithi064 ceph-mon[212750]: Health check failed: Degraded data redundancy: 18259/295374 objects degraded (6.182%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:17:58.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:58 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:17:58.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:17:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:58.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:58 smithi064 ceph-mon[220044]: pgmap v1543: 119 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 30 active+undersized, 2 remapped+peering, 12 peering, 1 active+remapped+backfilling, 5 active+undersized+degraded, 2 active+remapped+backfill_wait, 64 active+clean; 6.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 655 op/s; 18259/295374 objects degraded (6.182%); 34941/295374 objects misplaced (11.829%); 22 MiB/s, 358 objects/s recovering 2023-10-09T06:17:58.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:58 smithi064 ceph-mon[220044]: Health check failed: Degraded data redundancy: 18259/295374 objects degraded (6.182%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:17:58.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:58 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:17:58.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:17:58 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:59.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:58 smithi156 ceph-mon[195101]: pgmap v1543: 119 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 30 active+undersized, 2 remapped+peering, 12 peering, 1 active+remapped+backfilling, 5 active+undersized+degraded, 2 active+remapped+backfill_wait, 64 active+clean; 6.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 655 op/s; 18259/295374 objects degraded (6.182%); 34941/295374 objects misplaced (11.829%); 22 MiB/s, 358 objects/s recovering 2023-10-09T06:17:59.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:58 smithi156 ceph-mon[195101]: Health check failed: Degraded data redundancy: 18259/295374 objects degraded (6.182%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:17:59.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:58 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:17:59.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:17:58 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:59.422 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:17:59.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:59.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:59.534 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:59.593 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:59.739 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:17:59.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:59.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:59.798 INFO:teuthology.orchestra.run.smithi156.stderr:admin_socket: exception getting command descriptions: [Errno 111] Connection refused 2023-10-09T06:17:59.846 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:59.903 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:00.107 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:18:00.150 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:18:00.172 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:00.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:00.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:00.193 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:18:00.229 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:18:00.267 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:18:00.279 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:00.304 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:18:00.345 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:18:00.345 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:18:00.345 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:18:00.345 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:18:00.345 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:18:00.345 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:18:00.345 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-09T06:18:00.345 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:18:00.345 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:18:00.368 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:00.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:00 smithi156 ceph-mon[195101]: pgmap v1544: 119 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 30 active+undersized, 2 remapped+peering, 12 peering, 1 active+remapped+backfilling, 5 active+undersized+degraded, 2 active+remapped+backfill_wait, 64 active+clean; 6.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 561 op/s; 18259/295374 objects degraded (6.182%); 34941/295374 objects misplaced (11.829%); 22 MiB/s, 357 objects/s recovering 2023-10-09T06:18:00.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:00 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:00.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:00 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1229067608' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:00.561 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:00 smithi064 ceph-mon[212750]: pgmap v1544: 119 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 30 active+undersized, 2 remapped+peering, 12 peering, 1 active+remapped+backfilling, 5 active+undersized+degraded, 2 active+remapped+backfill_wait, 64 active+clean; 6.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 561 op/s; 18259/295374 objects degraded (6.182%); 34941/295374 objects misplaced (11.829%); 22 MiB/s, 357 objects/s recovering 2023-10-09T06:18:00.561 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:00 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:00.561 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:00 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1229067608' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:00.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:00 smithi064 ceph-mon[220044]: pgmap v1544: 119 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 30 active+undersized, 2 remapped+peering, 12 peering, 1 active+remapped+backfilling, 5 active+undersized+degraded, 2 active+remapped+backfill_wait, 64 active+clean; 6.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 561 op/s; 18259/295374 objects degraded (6.182%); 34941/295374 objects misplaced (11.829%); 22 MiB/s, 357 objects/s recovering 2023-10-09T06:18:00.562 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:00 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:00.562 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:00 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1229067608' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:00.602 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:00.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:00.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:00.706 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:00.753 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:01.020 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T06:18:01.020 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:18:01.103 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:01.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:01.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:01.209 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:18:01.216 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:01.297 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:01.355 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:01.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:01.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:01.459 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:01.533 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:01.616 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:18:01.758 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T06:18:01.758 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T06:18:01.897 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:01.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:01.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:01.969 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:02.001 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:02.030 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:02.325 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:02 smithi156 ceph-mon[195101]: pgmap v1545: 119 pgs: 2 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 7 active+undersized+degraded, 3 active+remapped+backfill_wait, 64 active+clean; 5.8 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 879 op/s; 22276/285711 objects degraded (7.797%); 45123/285711 objects misplaced (15.793%); 32 MiB/s, 510 objects/s recovering 2023-10-09T06:18:02.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:02 smithi064 ceph-mon[212750]: pgmap v1545: 119 pgs: 2 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 7 active+undersized+degraded, 3 active+remapped+backfill_wait, 64 active+clean; 5.8 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 879 op/s; 22276/285711 objects degraded (7.797%); 45123/285711 objects misplaced (15.793%); 32 MiB/s, 510 objects/s recovering 2023-10-09T06:18:02.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:02 smithi064 ceph-mon[220044]: pgmap v1545: 119 pgs: 2 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 7 active+undersized+degraded, 3 active+remapped+backfill_wait, 64 active+clean; 5.8 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 879 op/s; 22276/285711 objects degraded (7.797%); 45123/285711 objects misplaced (15.793%); 32 MiB/s, 510 objects/s recovering 2023-10-09T06:18:02.485 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:02.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:02.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:02.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:02.630 DEBUG:teuthology.orchestra.run.smithi156:> sudo pkill -f 'journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.5.service' 2023-10-09T06:18:02.664 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:02.685 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:18:02.686 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-10-09T06:18:02.776 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:02.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:02.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:02.890 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:02.961 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:03.037 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:03.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:03.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:03.140 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:03.174 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:03.223 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:03.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:03.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:03.328 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:03.381 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:03.603 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:03.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:03.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:03.714 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:03.753 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:03.760 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:18:03.888 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:03.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:03.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:04.005 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:04.072 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:04.204 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:04.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:04.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:04.311 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:04.383 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:04.428 INFO:teuthology.orchestra.run.smithi156.stderr:admin_socket: exception getting command descriptions: [Errno 111] Connection refused 2023-10-09T06:18:04.511 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:04.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:04.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:04.561 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:04.616 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:04.675 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:04.825 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:18:04.851 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:04.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:04.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:04.956 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:05.004 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:05.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:04 smithi156 ceph-mon[195101]: pgmap v1546: 119 pgs: 2 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 7 active+undersized+degraded, 3 active+remapped+backfill_wait, 64 active+clean; 5.7 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 808 op/s; 21999/282291 objects degraded (7.793%); 45123/282291 objects misplaced (15.985%); 25 MiB/s, 407 objects/s recovering 2023-10-09T06:18:05.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:04 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 21999/282291 objects degraded (7.793%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:05.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:04 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/284403349' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:18:05.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:04 smithi064 ceph-mon[212750]: pgmap v1546: 119 pgs: 2 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 7 active+undersized+degraded, 3 active+remapped+backfill_wait, 64 active+clean; 5.7 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 808 op/s; 21999/282291 objects degraded (7.793%); 45123/282291 objects misplaced (15.985%); 25 MiB/s, 407 objects/s recovering 2023-10-09T06:18:05.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:04 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 21999/282291 objects degraded (7.793%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:05.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:04 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/284403349' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:18:05.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:04 smithi064 ceph-mon[220044]: pgmap v1546: 119 pgs: 2 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 7 active+undersized+degraded, 3 active+remapped+backfill_wait, 64 active+clean; 5.7 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 808 op/s; 21999/282291 objects degraded (7.793%); 45123/282291 objects misplaced (15.985%); 25 MiB/s, 407 objects/s recovering 2023-10-09T06:18:05.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:04 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 21999/282291 objects degraded (7.793%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:05.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:04 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/284403349' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:18:05.402 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T06:18:05.402 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:18:05.414 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:05.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:05.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:05.524 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:05.594 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:18:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:18:05] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2023-10-09T06:18:05.601 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:05.930 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:18:05.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:05 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:05.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:05 smithi064 ceph-mon[212750]: failed to encode map e602 with expected crc 2023-10-09T06:18:05.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:05 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/284403349' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:18:05.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:05 smithi064 ceph-mon[212750]: osdmap e602: 8 total, 7 up, 6 in 2023-10-09T06:18:05.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:05 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:05.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:05 smithi064 ceph-mon[220044]: failed to encode map e602 with expected crc 2023-10-09T06:18:05.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:05 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/284403349' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:18:05.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:05 smithi064 ceph-mon[220044]: osdmap e602: 8 total, 7 up, 6 in 2023-10-09T06:18:06.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:05 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:06.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:05 smithi156 ceph-mon[195101]: failed to encode map e602 with expected crc 2023-10-09T06:18:06.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:05 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/284403349' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:18:06.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:05 smithi156 ceph-mon[195101]: osdmap e602: 8 total, 7 up, 6 in 2023-10-09T06:18:07.073 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:07.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:07.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:07.092 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:06 smithi064 ceph-mon[212750]: failed to encode map e602 with expected crc 2023-10-09T06:18:07.092 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:06 smithi064 ceph-mon[212750]: failed to encode map e602 with expected crc 2023-10-09T06:18:07.092 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:06 smithi064 ceph-mon[212750]: pgmap v1548: 119 pgs: 2 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 7 active+undersized+degraded, 3 active+remapped+backfill_wait, 64 active+clean; 5.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 742 op/s; 20967/279177 objects degraded (7.510%); 44398/279177 objects misplaced (15.903%); 19 MiB/s, 298 objects/s recovering 2023-10-09T06:18:07.092 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:06 smithi064 ceph-mon[220044]: failed to encode map e602 with expected crc 2023-10-09T06:18:07.092 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:06 smithi064 ceph-mon[220044]: failed to encode map e602 with expected crc 2023-10-09T06:18:07.092 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:06 smithi064 ceph-mon[220044]: pgmap v1548: 119 pgs: 2 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 7 active+undersized+degraded, 3 active+remapped+backfill_wait, 64 active+clean; 5.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 742 op/s; 20967/279177 objects degraded (7.510%); 44398/279177 objects misplaced (15.903%); 19 MiB/s, 298 objects/s recovering 2023-10-09T06:18:07.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:06 smithi156 ceph-mon[195101]: failed to encode map e602 with expected crc 2023-10-09T06:18:07.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:06 smithi156 ceph-mon[195101]: failed to encode map e602 with expected crc 2023-10-09T06:18:07.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:06 smithi156 ceph-mon[195101]: pgmap v1548: 119 pgs: 2 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 7 active+undersized+degraded, 3 active+remapped+backfill_wait, 64 active+clean; 5.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 742 op/s; 20967/279177 objects degraded (7.510%); 44398/279177 objects misplaced (15.903%); 19 MiB/s, 298 objects/s recovering 2023-10-09T06:18:07.190 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:07.241 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:07.327 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:07.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:07.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:07.447 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:07.495 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:07.641 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:07.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:07.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:07.686 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 7, 1] out_osds: [2, 6] dead_osds: [5] live_osds: [1, 2, 3, 6, 7, 4, 0] 2023-10-09T06:18:07.686 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:18:07.687 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-10-09T06:18:07.687 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd in 6 2023-10-09T06:18:07.745 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:07.786 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:07.833 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:18:07.845 INFO:teuthology.orchestra.run.smithi156.stderr:admin_socket: exception getting command descriptions: [Errno 111] Connection refused 2023-10-09T06:18:07.857 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:07.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:07.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:07.876 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:18:07.912 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:18:07.953 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:18:07.961 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:07.994 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:08.002 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:18:08.043 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:18:08.060 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:07 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3410369684' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:08.093 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:18:08.095 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:18:08.095 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:18:08.095 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:18:08.095 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:18:08.095 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:18:08.095 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-09T06:18:08.095 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:18:08.095 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:18:08.112 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:07 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3410369684' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:08.113 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:07 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3410369684' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:08.113 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:08.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:08.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:08.217 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:08.297 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:08.353 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:08.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:08.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:08.457 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:08.502 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:08.835 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T06:18:08.836 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:18:08.840 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:08.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:08.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:08.945 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:08.992 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:09.015 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:18:09.108 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:09.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:09.118 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:08 smithi156 ceph-mon[195101]: pgmap v1549: 119 pgs: 2 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 7 active+undersized+degraded, 3 active+remapped+backfill_wait, 64 active+clean; 5.5 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 833 op/s; 19560/270354 objects degraded (7.235%); 43173/270354 objects misplaced (15.969%); 20 MiB/s, 324 objects/s recovering 2023-10-09T06:18:09.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:09.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:08 smithi064 ceph-mon[212750]: pgmap v1549: 119 pgs: 2 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 7 active+undersized+degraded, 3 active+remapped+backfill_wait, 64 active+clean; 5.5 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 833 op/s; 19560/270354 objects degraded (7.235%); 43173/270354 objects misplaced (15.969%); 20 MiB/s, 324 objects/s recovering 2023-10-09T06:18:09.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:08 smithi064 ceph-mon[220044]: pgmap v1549: 119 pgs: 2 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 7 active+undersized+degraded, 3 active+remapped+backfill_wait, 64 active+clean; 5.5 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 833 op/s; 19560/270354 objects degraded (7.235%); 43173/270354 objects misplaced (15.969%); 20 MiB/s, 324 objects/s recovering 2023-10-09T06:18:09.219 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:09.262 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:09.325 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:09.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:09.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:09.335 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:18:09.360 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T06:18:09.360 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T06:18:09.431 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:09.469 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:09.470 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:09.579 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:09.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:09.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:09.686 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:09.756 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:09.852 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:09.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:09.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:09.964 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:10.030 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:10.087 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:10.213 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:10.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:10.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:10.320 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:10.348 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:10.391 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:10.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:10.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:10.494 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:10.527 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:10.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:10 smithi156 ceph-mon[195101]: pgmap v1550: 119 pgs: 2 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 7 active+undersized+degraded, 3 active+remapped+backfill_wait, 64 active+clean; 5.5 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 833 op/s; 19560/270354 objects degraded (7.235%); 43173/270354 objects misplaced (15.969%); 20 MiB/s, 324 objects/s recovering 2023-10-09T06:18:10.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:10 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:10.626 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:10.631 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:10 smithi064 ceph-mon[212750]: pgmap v1550: 119 pgs: 2 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 7 active+undersized+degraded, 3 active+remapped+backfill_wait, 64 active+clean; 5.5 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 833 op/s; 19560/270354 objects degraded (7.235%); 43173/270354 objects misplaced (15.969%); 20 MiB/s, 324 objects/s recovering 2023-10-09T06:18:10.631 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:10 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:10.631 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:10 smithi064 ceph-mon[220044]: pgmap v1550: 119 pgs: 2 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 7 active+undersized+degraded, 3 active+remapped+backfill_wait, 64 active+clean; 5.5 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 833 op/s; 19560/270354 objects degraded (7.235%); 43173/270354 objects misplaced (15.969%); 20 MiB/s, 324 objects/s recovering 2023-10-09T06:18:10.631 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:10 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:10.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:10.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:10.733 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:10.762 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:10.811 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:10.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:10.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:10.915 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:11.001 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:11.361 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:18:11.366 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:11.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:11.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:11.482 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:11.490 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:18:11.490 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:18:11.521 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:11 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 17647/262080 objects degraded (6.733%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:11.580 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:11.684 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:11 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 17647/262080 objects degraded (6.733%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:11.684 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:11 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 17647/262080 objects degraded (6.733%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:11.773 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:11.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:11.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:11.883 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:11.912 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:11.955 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:11.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:11.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:12.059 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:12.095 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:12.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:12 smithi064 conmon[212726]: 2023-10-09T06:18:12.371+0000 7f8823f23700 -1 mon.a@0(leader).osd e602 definitely_dead 0 2023-10-09T06:18:12.458 INFO:teuthology.orchestra.run.smithi064.stderr:marked in osd.6. 2023-10-09T06:18:12.507 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:18:12.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:12 smithi064 ceph-mon[212750]: pgmap v1551: 119 pgs: 2 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 7 active+undersized+degraded, 3 active+remapped+backfill_wait, 64 active+clean; 5.3 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 786 op/s; 17647/262080 objects degraded (6.733%); 41448/262080 objects misplaced (15.815%); 18 MiB/s, 295 objects/s recovering 2023-10-09T06:18:12.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:12 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/961748507' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T06:18:12.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:12 smithi064 ceph-mon[220044]: pgmap v1551: 119 pgs: 2 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 7 active+undersized+degraded, 3 active+remapped+backfill_wait, 64 active+clean; 5.3 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 786 op/s; 17647/262080 objects degraded (6.733%); 41448/262080 objects misplaced (15.815%); 18 MiB/s, 295 objects/s recovering 2023-10-09T06:18:12.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:12 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/961748507' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T06:18:12.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:12 smithi156 ceph-mon[195101]: pgmap v1551: 119 pgs: 2 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 7 active+undersized+degraded, 3 active+remapped+backfill_wait, 64 active+clean; 5.3 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 786 op/s; 17647/262080 objects degraded (6.733%); 41448/262080 objects misplaced (15.815%); 18 MiB/s, 295 objects/s recovering 2023-10-09T06:18:12.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:12 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/961748507' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T06:18:12.873 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:18:12.972 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:12.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:12.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:13.076 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:13.105 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:13.148 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:13.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:13.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:13.252 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:13.282 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:13.473 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:13.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:13.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:13.534 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/961748507' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T06:18:13.534 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[212750]: osdmap e603: 8 total, 7 up, 7 in 2023-10-09T06:18:13.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[212750]: failed to encode map e603 with expected crc 2023-10-09T06:18:13.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[212750]: failed to encode map e603 with expected crc 2023-10-09T06:18:13.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[212750]: failed to encode map e603 with expected crc 2023-10-09T06:18:13.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[212750]: failed to encode map e603 with expected crc 2023-10-09T06:18:13.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[212750]: failed to encode map e603 with expected crc 2023-10-09T06:18:13.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[212750]: failed to encode map e604 with expected crc 2023-10-09T06:18:13.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[212750]: osdmap e604: 8 total, 7 up, 7 in 2023-10-09T06:18:13.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[212750]: failed to encode map e604 with expected crc 2023-10-09T06:18:13.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[212750]: failed to encode map e604 with expected crc 2023-10-09T06:18:13.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[212750]: failed to encode map e604 with expected crc 2023-10-09T06:18:13.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[212750]: failed to encode map e604 with expected crc 2023-10-09T06:18:13.536 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[212750]: failed to encode map e604 with expected crc 2023-10-09T06:18:13.536 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/961748507' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T06:18:13.536 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[220044]: osdmap e603: 8 total, 7 up, 7 in 2023-10-09T06:18:13.536 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[220044]: failed to encode map e603 with expected crc 2023-10-09T06:18:13.536 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[220044]: failed to encode map e603 with expected crc 2023-10-09T06:18:13.536 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[220044]: failed to encode map e603 with expected crc 2023-10-09T06:18:13.536 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[220044]: failed to encode map e603 with expected crc 2023-10-09T06:18:13.536 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[220044]: failed to encode map e603 with expected crc 2023-10-09T06:18:13.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[220044]: failed to encode map e604 with expected crc 2023-10-09T06:18:13.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[220044]: osdmap e604: 8 total, 7 up, 7 in 2023-10-09T06:18:13.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[220044]: failed to encode map e604 with expected crc 2023-10-09T06:18:13.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[220044]: failed to encode map e604 with expected crc 2023-10-09T06:18:13.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[220044]: failed to encode map e604 with expected crc 2023-10-09T06:18:13.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[220044]: failed to encode map e604 with expected crc 2023-10-09T06:18:13.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:13 smithi064 ceph-mon[220044]: failed to encode map e604 with expected crc 2023-10-09T06:18:13.538 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-10-09T06:18:13.577 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:13.608 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:13.658 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:13.757 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:13.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:13.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:13.862 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:13.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:13 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/961748507' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T06:18:13.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:13 smithi156 ceph-mon[195101]: osdmap e603: 8 total, 7 up, 7 in 2023-10-09T06:18:13.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:13 smithi156 ceph-mon[195101]: failed to encode map e603 with expected crc 2023-10-09T06:18:13.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:13 smithi156 ceph-mon[195101]: failed to encode map e603 with expected crc 2023-10-09T06:18:13.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:13 smithi156 ceph-mon[195101]: failed to encode map e603 with expected crc 2023-10-09T06:18:13.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:13 smithi156 ceph-mon[195101]: failed to encode map e603 with expected crc 2023-10-09T06:18:13.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:13 smithi156 ceph-mon[195101]: failed to encode map e603 with expected crc 2023-10-09T06:18:13.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:13 smithi156 ceph-mon[195101]: failed to encode map e604 with expected crc 2023-10-09T06:18:13.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:13 smithi156 ceph-mon[195101]: osdmap e604: 8 total, 7 up, 7 in 2023-10-09T06:18:13.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:13 smithi156 ceph-mon[195101]: failed to encode map e604 with expected crc 2023-10-09T06:18:13.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:13 smithi156 ceph-mon[195101]: failed to encode map e604 with expected crc 2023-10-09T06:18:13.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:13 smithi156 ceph-mon[195101]: failed to encode map e604 with expected crc 2023-10-09T06:18:13.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:13 smithi156 ceph-mon[195101]: failed to encode map e604 with expected crc 2023-10-09T06:18:13.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:13 smithi156 ceph-mon[195101]: failed to encode map e604 with expected crc 2023-10-09T06:18:13.913 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:14.069 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:14.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:14.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:14.178 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:14.222 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:14.407 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:14.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:14.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:14.520 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:14.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:14 smithi064 ceph-mon[212750]: pgmap v1554: 119 pgs: 2 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 7 active+undersized+degraded, 3 active+remapped+backfill_wait, 64 active+clean; 5.3 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 831 op/s; 17412/259185 objects degraded (6.718%); 41448/259185 objects misplaced (15.992%); 17 MiB/s, 264 objects/s recovering 2023-10-09T06:18:14.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:14.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:14.539 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:14 smithi064 ceph-mon[212750]: failed to encode map e605 with expected crc 2023-10-09T06:18:14.539 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:14 smithi064 ceph-mon[220044]: pgmap v1554: 119 pgs: 2 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 7 active+undersized+degraded, 3 active+remapped+backfill_wait, 64 active+clean; 5.3 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 831 op/s; 17412/259185 objects degraded (6.718%); 41448/259185 objects misplaced (15.992%); 17 MiB/s, 264 objects/s recovering 2023-10-09T06:18:14.539 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:14.539 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:14.539 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:14 smithi064 ceph-mon[220044]: failed to encode map e605 with expected crc 2023-10-09T06:18:14.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:14 smithi064 ceph-mon[220044]: failed to encode map e605 with expected crc 2023-10-09T06:18:14.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:14 smithi064 ceph-mon[220044]: failed to encode map e605 with expected crc 2023-10-09T06:18:14.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:14 smithi064 ceph-mon[220044]: failed to encode map e605 with expected crc 2023-10-09T06:18:14.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:14 smithi064 ceph-mon[220044]: osdmap e605: 8 total, 7 up, 7 in 2023-10-09T06:18:14.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:14 smithi064 ceph-mon[220044]: failed to encode map e605 with expected crc 2023-10-09T06:18:14.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:14 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3749698147' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:14.599 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:14.622 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:18:14.623 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:18:14.652 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:14.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:14.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:14.758 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:14.787 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:14.820 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:14 smithi156 ceph-mon[195101]: pgmap v1554: 119 pgs: 2 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 7 active+undersized+degraded, 3 active+remapped+backfill_wait, 64 active+clean; 5.3 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 831 op/s; 17412/259185 objects degraded (6.718%); 41448/259185 objects misplaced (15.992%); 17 MiB/s, 264 objects/s recovering 2023-10-09T06:18:14.821 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:14.821 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:14.821 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:14 smithi156 ceph-mon[195101]: failed to encode map e605 with expected crc 2023-10-09T06:18:14.821 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:14 smithi156 ceph-mon[195101]: failed to encode map e605 with expected crc 2023-10-09T06:18:14.821 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:14 smithi156 ceph-mon[195101]: failed to encode map e605 with expected crc 2023-10-09T06:18:14.821 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:14 smithi156 ceph-mon[195101]: failed to encode map e605 with expected crc 2023-10-09T06:18:14.821 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:14 smithi156 ceph-mon[195101]: osdmap e605: 8 total, 7 up, 7 in 2023-10-09T06:18:14.821 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:14 smithi156 ceph-mon[195101]: failed to encode map e605 with expected crc 2023-10-09T06:18:14.821 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:14 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3749698147' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:14.835 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:18:14.866 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:14 smithi064 ceph-mon[212750]: failed to encode map e605 with expected crc 2023-10-09T06:18:14.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:14 smithi064 ceph-mon[212750]: failed to encode map e605 with expected crc 2023-10-09T06:18:14.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:14 smithi064 ceph-mon[212750]: failed to encode map e605 with expected crc 2023-10-09T06:18:14.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:14 smithi064 ceph-mon[212750]: osdmap e605: 8 total, 7 up, 7 in 2023-10-09T06:18:14.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:14 smithi064 ceph-mon[212750]: failed to encode map e605 with expected crc 2023-10-09T06:18:14.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:14 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3749698147' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:14.887 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:14.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:14.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:14.999 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:15.026 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:15.068 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:15.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:15.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:15.172 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:15.229 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:15.541 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:18:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:18:15] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.43.0" 2023-10-09T06:18:15.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:15 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:15.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:15 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3749698147' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:18:15.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:15 smithi064 ceph-mon[212750]: osdmap e606: 8 total, 7 up, 7 in 2023-10-09T06:18:15.543 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:15 smithi064 ceph-mon[212750]: failed to encode map e606 with expected crc 2023-10-09T06:18:15.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:15 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:15.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:15 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3749698147' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:18:15.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:15 smithi064 ceph-mon[220044]: osdmap e606: 8 total, 7 up, 7 in 2023-10-09T06:18:15.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:15 smithi064 ceph-mon[220044]: failed to encode map e606 with expected crc 2023-10-09T06:18:15.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:15 smithi064 ceph-mon[220044]: failed to encode map e606 with expected crc 2023-10-09T06:18:15.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:15 smithi064 ceph-mon[220044]: failed to encode map e606 with expected crc 2023-10-09T06:18:15.562 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:18:15.746 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:15.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:15.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:15.849 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:15 smithi064 ceph-mon[212750]: failed to encode map e606 with expected crc 2023-10-09T06:18:15.849 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:15 smithi064 ceph-mon[212750]: failed to encode map e606 with expected crc 2023-10-09T06:18:15.856 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:15.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:15 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:15.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:15 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3749698147' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:18:15.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:15 smithi156 ceph-mon[195101]: osdmap e606: 8 total, 7 up, 7 in 2023-10-09T06:18:15.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:15 smithi156 ceph-mon[195101]: failed to encode map e606 with expected crc 2023-10-09T06:18:15.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:15 smithi156 ceph-mon[195101]: failed to encode map e606 with expected crc 2023-10-09T06:18:15.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:15 smithi156 ceph-mon[195101]: failed to encode map e606 with expected crc 2023-10-09T06:18:15.909 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:16.008 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:16.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:16.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:16.117 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:16.142 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:18:16.181 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:16.279 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:16.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:16.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:16.373 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:18:16.384 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:16.409 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:18:16.450 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:16.458 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:18:16.494 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:18:16.528 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:18:16.572 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:18:16.573 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:16.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:16.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:16.602 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:18:16.603 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:18:16.603 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:18:16.603 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:18:16.603 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:18:16.603 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:18:16.603 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-09T06:18:16.603 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:18:16.603 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:18:16.679 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:16.742 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:16.881 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:16.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:16.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:16.992 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:17.020 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:17.065 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:17.067 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:16 smithi064 ceph-mon[212750]: pgmap v1557: 119 pgs: 1 remapped+peering, 1 peering, 5 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 3 active+undersized+remapped, 7 active+undersized+degraded, 2 active+remapped+backfill_wait, 67 active+clean; 5.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 19283/255951 objects degraded (7.534%); 25524/255951 objects misplaced (9.972%); 7.1 MiB/s, 2 keys/s, 114 objects/s recovering 2023-10-09T06:18:17.067 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:16 smithi064 ceph-mon[212750]: failed to encode map e607 with expected crc 2023-10-09T06:18:17.067 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:16 smithi064 ceph-mon[212750]: osdmap e607: 8 total, 7 up, 7 in 2023-10-09T06:18:17.067 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:16 smithi064 ceph-mon[212750]: failed to encode map e607 with expected crc 2023-10-09T06:18:17.067 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:16 smithi064 ceph-mon[212750]: failed to encode map e607 with expected crc 2023-10-09T06:18:17.067 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:16 smithi064 ceph-mon[212750]: failed to encode map e607 with expected crc 2023-10-09T06:18:17.067 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:16 smithi064 ceph-mon[212750]: failed to encode map e607 with expected crc 2023-10-09T06:18:17.067 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:16 smithi064 ceph-mon[212750]: failed to encode map e607 with expected crc 2023-10-09T06:18:17.067 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:16 smithi064 ceph-mon[212750]: failed to encode map e607 with expected crc 2023-10-09T06:18:17.068 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:16 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1704843736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:17.068 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:16 smithi064 ceph-mon[220044]: pgmap v1557: 119 pgs: 1 remapped+peering, 1 peering, 5 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 3 active+undersized+remapped, 7 active+undersized+degraded, 2 active+remapped+backfill_wait, 67 active+clean; 5.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 19283/255951 objects degraded (7.534%); 25524/255951 objects misplaced (9.972%); 7.1 MiB/s, 2 keys/s, 114 objects/s recovering 2023-10-09T06:18:17.068 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:16 smithi064 ceph-mon[220044]: failed to encode map e607 with expected crc 2023-10-09T06:18:17.068 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:16 smithi064 ceph-mon[220044]: osdmap e607: 8 total, 7 up, 7 in 2023-10-09T06:18:17.068 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:16 smithi064 ceph-mon[220044]: failed to encode map e607 with expected crc 2023-10-09T06:18:17.068 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:16 smithi064 ceph-mon[220044]: failed to encode map e607 with expected crc 2023-10-09T06:18:17.068 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:16 smithi064 ceph-mon[220044]: failed to encode map e607 with expected crc 2023-10-09T06:18:17.068 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:16 smithi064 ceph-mon[220044]: failed to encode map e607 with expected crc 2023-10-09T06:18:17.069 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:16 smithi064 ceph-mon[220044]: failed to encode map e607 with expected crc 2023-10-09T06:18:17.069 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:16 smithi064 ceph-mon[220044]: failed to encode map e607 with expected crc 2023-10-09T06:18:17.069 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:16 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1704843736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:17.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:17.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:17.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:16 smithi156 ceph-mon[195101]: pgmap v1557: 119 pgs: 1 remapped+peering, 1 peering, 5 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 31 active+undersized, 3 active+undersized+remapped, 7 active+undersized+degraded, 2 active+remapped+backfill_wait, 67 active+clean; 5.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 19283/255951 objects degraded (7.534%); 25524/255951 objects misplaced (9.972%); 7.1 MiB/s, 2 keys/s, 114 objects/s recovering 2023-10-09T06:18:17.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:16 smithi156 ceph-mon[195101]: failed to encode map e607 with expected crc 2023-10-09T06:18:17.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:16 smithi156 ceph-mon[195101]: osdmap e607: 8 total, 7 up, 7 in 2023-10-09T06:18:17.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:16 smithi156 ceph-mon[195101]: failed to encode map e607 with expected crc 2023-10-09T06:18:17.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:16 smithi156 ceph-mon[195101]: failed to encode map e607 with expected crc 2023-10-09T06:18:17.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:16 smithi156 ceph-mon[195101]: failed to encode map e607 with expected crc 2023-10-09T06:18:17.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:16 smithi156 ceph-mon[195101]: failed to encode map e607 with expected crc 2023-10-09T06:18:17.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:16 smithi156 ceph-mon[195101]: failed to encode map e607 with expected crc 2023-10-09T06:18:17.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:16 smithi156 ceph-mon[195101]: failed to encode map e607 with expected crc 2023-10-09T06:18:17.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:16 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1704843736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:17.171 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:17.198 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:17.263 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:17.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:17.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:17.374 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:17.429 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:17.607 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:17.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:17.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:17.616 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T06:18:17.616 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T06:18:17.716 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:17.770 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:17.870 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:18:17.883 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:17.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:17.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:18.004 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:18.066 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:17 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 19283/255951 objects degraded (7.534%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:18.066 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:17 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 19283/255951 objects degraded (7.534%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:18.072 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:18.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:17 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 19283/255951 objects degraded (7.534%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:18.151 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:18:18.151 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T06:18:18.152 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T06:18:18.152 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:18:18.152 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.152 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:116286 9.9 9:91b2f3bd:::benchmark_data_smithi064_511911_object15184:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e450)", 2023-10-09T06:18:18.152 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:03.586689+0000", 2023-10-09T06:18:18.152 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 554.56091180400006, 2023-10-09T06:18:18.152 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1393943420000001, 2023-10-09T06:18:18.152 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:18.152 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:18:18.152 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:18.152 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:18:18.153 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:18:18.153 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 116286 2023-10-09T06:18:18.153 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.153 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:18.153 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.153 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:18.153 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586689+0000", 2023-10-09T06:18:18.153 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.153 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.153 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.153 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:18.154 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586689+0000", 2023-10-09T06:18:18.154 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.154 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.154 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.154 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:18.154 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586687+0000", 2023-10-09T06:18:18.154 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:18.154 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.154 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.154 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:18.155 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586695+0000", 2023-10-09T06:18:18.155 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0700000000000007e-06 2023-10-09T06:18:18.155 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.155 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.155 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:18.155 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586708+0000", 2023-10-09T06:18:18.155 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2928999999999999e-05 2023-10-09T06:18:18.155 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.155 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.155 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:18.155 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586715+0000", 2023-10-09T06:18:18.155 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1629999999999999e-06 2023-10-09T06:18:18.156 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.156 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.156 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.156 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.590775+0000", 2023-10-09T06:18:18.156 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0040602310000000001 2023-10-09T06:18:18.156 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.156 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.156 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:18.156 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.590785+0000", 2023-10-09T06:18:18.156 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2529999999999993e-06 2023-10-09T06:18:18.156 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.156 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.157 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.157 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.708849+0000", 2023-10-09T06:18:18.157 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1180647349999999 2023-10-09T06:18:18.157 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.157 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.157 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for missing object", 2023-10-09T06:18:18.157 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.708984+0000", 2023-10-09T06:18:18.157 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000134136 2023-10-09T06:18:18.157 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.157 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.157 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.157 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.725947+0000", 2023-10-09T06:18:18.158 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.016963676 2023-10-09T06:18:18.158 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.158 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.158 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:18.158 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.726083+0000", 2023-10-09T06:18:18.158 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000135699 2023-10-09T06:18:18.158 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.158 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:18.158 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.158 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.158 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.159 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:116302 9.9 9:97b6a430:::benchmark_data_smithi064_511911_object15200:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e450)", 2023-10-09T06:18:18.159 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:03.586724+0000", 2023-10-09T06:18:18.159 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 554.56087650799998, 2023-10-09T06:18:18.159 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.139970148, 2023-10-09T06:18:18.159 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:18.159 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:18.159 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:18.159 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:18:18.159 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:18:18.159 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 116302 2023-10-09T06:18:18.159 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.160 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:18.160 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.160 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:18.160 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586724+0000", 2023-10-09T06:18:18.160 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.160 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.160 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.160 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:18.160 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586724+0000", 2023-10-09T06:18:18.160 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.160 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.160 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.161 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:18.161 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586723+0000", 2023-10-09T06:18:18.161 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:18.161 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.161 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.161 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:18.161 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586726+0000", 2023-10-09T06:18:18.161 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.8909999999999999e-06 2023-10-09T06:18:18.161 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.161 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.161 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:18.161 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586733+0000", 2023-10-09T06:18:18.162 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.131e-06 2023-10-09T06:18:18.162 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.162 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.162 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:18.162 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.586739+0000", 2023-10-09T06:18:18.162 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.485e-06 2023-10-09T06:18:18.162 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.162 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.162 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.162 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.599094+0000", 2023-10-09T06:18:18.162 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012355138 2023-10-09T06:18:18.162 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.163 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.163 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:18.163 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.599106+0000", 2023-10-09T06:18:18.163 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1565e-05 2023-10-09T06:18:18.163 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.163 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.163 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.163 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.709008+0000", 2023-10-09T06:18:18.163 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1099019999999999 2023-10-09T06:18:18.163 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.163 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.163 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:18.164 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.709136+0000", 2023-10-09T06:18:18.164 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012808900000000001 2023-10-09T06:18:18.164 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.164 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.164 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:18:18.164 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.709273+0000", 2023-10-09T06:18:18.164 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000137584 2023-10-09T06:18:18.164 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.164 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.164 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:18.164 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.725163+0000", 2023-10-09T06:18:18.164 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015889678000000001 2023-10-09T06:18:18.165 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.165 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.165 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.165 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.725700+0000", 2023-10-09T06:18:18.165 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00053697700000000001 2023-10-09T06:18:18.165 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.165 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.165 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.165 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.726552+0000", 2023-10-09T06:18:18.165 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000852399 2023-10-09T06:18:18.165 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.165 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.166 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:18.166 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.726596+0000", 2023-10-09T06:18:18.166 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3451999999999997e-05 2023-10-09T06:18:18.166 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.166 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.166 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:18.166 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.726694+0000", 2023-10-09T06:18:18.166 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.8176000000000003e-05 2023-10-09T06:18:18.166 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.167 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:18.167 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.167 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.167 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.168 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:132900 9.9 9:9c50d9a2:::benchmark_data_smithi064_511911_object31798:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e455)", 2023-10-09T06:18:18.168 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:26.559784+0000", 2023-10-09T06:18:18.168 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 531.58781632, 2023-10-09T06:18:18.168 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2243415390000001, 2023-10-09T06:18:18.168 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:18.168 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:18:18.168 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:18.168 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:18:18.168 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:18:18.168 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 132900 2023-10-09T06:18:18.168 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.169 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:18.169 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.169 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:18.169 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.559784+0000", 2023-10-09T06:18:18.169 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.169 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.169 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.169 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:18.169 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.559784+0000", 2023-10-09T06:18:18.169 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.169 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.170 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.170 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:18.170 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.559783+0000", 2023-10-09T06:18:18.170 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:18.170 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.170 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.170 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:18.170 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.559789+0000", 2023-10-09T06:18:18.170 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5690000000000001e-06 2023-10-09T06:18:18.170 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.170 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.171 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:18.171 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.559800+0000", 2023-10-09T06:18:18.171 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0288e-05 2023-10-09T06:18:18.171 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.171 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.171 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:18.171 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.562277+0000", 2023-10-09T06:18:18.171 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002477103 2023-10-09T06:18:18.171 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.171 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.171 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.171 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.565865+0000", 2023-10-09T06:18:18.172 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0035886989999999999 2023-10-09T06:18:18.172 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.172 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.172 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:18.172 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.566006+0000", 2023-10-09T06:18:18.172 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014012900000000001 2023-10-09T06:18:18.172 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.172 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.172 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:18:18.172 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.566132+0000", 2023-10-09T06:18:18.172 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012682599999999999 2023-10-09T06:18:18.172 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.173 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.173 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.173 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572376+0000", 2023-10-09T06:18:18.173 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0062437980000000001 2023-10-09T06:18:18.173 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.173 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.173 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:18.173 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.572387+0000", 2023-10-09T06:18:18.173 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0492999999999999e-05 2023-10-09T06:18:18.173 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.173 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.173 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.173 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.587508+0000", 2023-10-09T06:18:18.174 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015120971 2023-10-09T06:18:18.174 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.174 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.174 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:18.174 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.587518+0000", 2023-10-09T06:18:18.174 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0197000000000001e-05 2023-10-09T06:18:18.174 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.174 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.174 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.174 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.761187+0000", 2023-10-09T06:18:18.174 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.17366953 2023-10-09T06:18:18.175 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.175 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.175 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for missing object", 2023-10-09T06:18:18.175 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.761345+0000", 2023-10-09T06:18:18.175 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015735400000000001 2023-10-09T06:18:18.175 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.175 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.175 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.175 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.784048+0000", 2023-10-09T06:18:18.175 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.022703204000000001 2023-10-09T06:18:18.176 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.176 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.176 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:18.176 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.784126+0000", 2023-10-09T06:18:18.176 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7649000000000001e-05 2023-10-09T06:18:18.176 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.176 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:18.176 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.176 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.176 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.176 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:132902 9.9 9:90d35044:::benchmark_data_smithi064_511911_object31800:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e455)", 2023-10-09T06:18:18.177 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:26.562296+0000", 2023-10-09T06:18:18.177 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 531.58530431700001, 2023-10-09T06:18:18.177 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2265417860000001, 2023-10-09T06:18:18.177 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:18.177 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:18:18.177 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:18.177 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:18:18.177 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:18:18.177 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 132902 2023-10-09T06:18:18.177 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.178 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:18.178 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.178 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:18.178 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.562296+0000", 2023-10-09T06:18:18.178 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.178 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.178 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.178 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:18.178 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.562296+0000", 2023-10-09T06:18:18.178 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.178 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.178 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.179 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:18.179 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.562295+0000", 2023-10-09T06:18:18.179 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:18.179 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.179 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.179 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:18.179 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.562300+0000", 2023-10-09T06:18:18.179 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5339999999999999e-06 2023-10-09T06:18:18.179 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.179 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.180 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:18.180 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.562309+0000", 2023-10-09T06:18:18.180 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.3470000000000005e-06 2023-10-09T06:18:18.180 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.180 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.180 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:18.180 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.562314+0000", 2023-10-09T06:18:18.180 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7299999999999996e-06 2023-10-09T06:18:18.180 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.180 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.180 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.181 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.569303+0000", 2023-10-09T06:18:18.181 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0069888600000000004 2023-10-09T06:18:18.181 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.181 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.181 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:18.181 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.569430+0000", 2023-10-09T06:18:18.181 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012732600000000001 2023-10-09T06:18:18.181 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.181 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.181 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:18:18.181 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.569519+0000", 2023-10-09T06:18:18.182 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8524999999999997e-05 2023-10-09T06:18:18.182 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.182 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.182 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.182 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.582104+0000", 2023-10-09T06:18:18.182 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012585202 2023-10-09T06:18:18.182 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.182 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.182 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:18.182 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.582122+0000", 2023-10-09T06:18:18.182 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8048999999999999e-05 2023-10-09T06:18:18.182 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.183 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.183 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.183 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.587546+0000", 2023-10-09T06:18:18.183 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0054235079999999996 2023-10-09T06:18:18.183 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.183 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.183 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:18.183 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:26.587553+0000", 2023-10-09T06:18:18.183 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.99e-06 2023-10-09T06:18:18.183 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.183 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.183 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.184 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.761400+0000", 2023-10-09T06:18:18.184 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.173847133 2023-10-09T06:18:18.184 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.184 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.184 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for missing object", 2023-10-09T06:18:18.184 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.761507+0000", 2023-10-09T06:18:18.184 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000106971 2023-10-09T06:18:18.184 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.184 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.184 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.184 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.788758+0000", 2023-10-09T06:18:18.184 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.027251068999999999 2023-10-09T06:18:18.185 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.185 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.185 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:18.185 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:27.788838+0000", 2023-10-09T06:18:18.185 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9989999999999998e-05 2023-10-09T06:18:18.185 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.185 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:18.185 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.185 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.185 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.185 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:131778 10.9 10:909f8957:::benchmark_data_smithi064_573750_object35563:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-10-09T06:18:18.185 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:14:20.794429+0000", 2023-10-09T06:18:18.186 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 237.35317161, 2023-10-09T06:18:18.186 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99906589199999996, 2023-10-09T06:18:18.186 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:18.186 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:18.186 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:18.186 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:18:18.186 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:18.186 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 131778 2023-10-09T06:18:18.186 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.186 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:18.186 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.186 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:18.187 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.794429+0000", 2023-10-09T06:18:18.187 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.187 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.187 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.187 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:18.187 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.794429+0000", 2023-10-09T06:18:18.187 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.187 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.187 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.187 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:18.187 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.794427+0000", 2023-10-09T06:18:18.188 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:18.188 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.188 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.188 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:18.188 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.794434+0000", 2023-10-09T06:18:18.188 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.934e-06 2023-10-09T06:18:18.188 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.188 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.188 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:18.188 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.794452+0000", 2023-10-09T06:18:18.188 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7730000000000001e-05 2023-10-09T06:18:18.188 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.189 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.189 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:18.189 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.794460+0000", 2023-10-09T06:18:18.189 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6469999999999998e-06 2023-10-09T06:18:18.189 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.189 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.189 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.189 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.794511+0000", 2023-10-09T06:18:18.189 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1496999999999998e-05 2023-10-09T06:18:18.189 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.189 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.189 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:18.190 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.794520+0000", 2023-10-09T06:18:18.190 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8130000000000002e-06 2023-10-09T06:18:18.190 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.190 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.190 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.190 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.759431+0000", 2023-10-09T06:18:18.190 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96491159000000004 2023-10-09T06:18:18.190 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.190 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.190 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:18.190 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.759582+0000", 2023-10-09T06:18:18.191 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015018799999999999 2023-10-09T06:18:18.191 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.191 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.191 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:18:18.191 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.759718+0000", 2023-10-09T06:18:18.191 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000136608 2023-10-09T06:18:18.191 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.191 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.191 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:18.191 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.775116+0000", 2023-10-09T06:18:18.191 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015397622 2023-10-09T06:18:18.191 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.192 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.192 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.192 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793364+0000", 2023-10-09T06:18:18.192 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018247780000000002 2023-10-09T06:18:18.192 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.192 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.192 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.192 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793393+0000", 2023-10-09T06:18:18.192 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9689999999999999e-05 2023-10-09T06:18:18.192 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.192 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.192 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:18.193 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793447+0000", 2023-10-09T06:18:18.193 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3127999999999997e-05 2023-10-09T06:18:18.193 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.193 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.193 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:18.193 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793495+0000", 2023-10-09T06:18:18.193 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8112999999999998e-05 2023-10-09T06:18:18.193 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.193 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:18.193 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.193 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.193 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.194 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:131788 10.9 10:9b77a1ac:::benchmark_data_smithi064_573750_object35573:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-10-09T06:18:18.194 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:14:20.803992+0000", 2023-10-09T06:18:18.194 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 237.34360870099999, 2023-10-09T06:18:18.194 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.98965816100000004, 2023-10-09T06:18:18.194 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:18.194 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:18.194 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:18.194 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:18:18.194 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:18.194 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 131788 2023-10-09T06:18:18.194 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.195 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:18.195 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.195 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:18.195 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.803992+0000", 2023-10-09T06:18:18.195 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.195 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.195 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.195 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:18.195 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.803992+0000", 2023-10-09T06:18:18.195 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.195 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.195 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.196 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:18.196 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.803990+0000", 2023-10-09T06:18:18.196 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:18.196 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.196 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.196 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:18.196 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.803997+0000", 2023-10-09T06:18:18.196 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2729999999999997e-06 2023-10-09T06:18:18.196 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.196 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.196 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:18.197 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804012+0000", 2023-10-09T06:18:18.197 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4501000000000001e-05 2023-10-09T06:18:18.197 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.197 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.197 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:18.197 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804018+0000", 2023-10-09T06:18:18.197 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1940000000000003e-06 2023-10-09T06:18:18.197 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.197 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.197 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.197 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804073+0000", 2023-10-09T06:18:18.197 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5361000000000001e-05 2023-10-09T06:18:18.198 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.198 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.198 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:18.198 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804081+0000", 2023-10-09T06:18:18.198 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5190000000000003e-06 2023-10-09T06:18:18.198 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.198 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.198 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.198 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.762019+0000", 2023-10-09T06:18:18.198 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.95793824800000005 2023-10-09T06:18:18.198 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.199 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.199 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:18.199 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.762126+0000", 2023-10-09T06:18:18.199 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000107262 2023-10-09T06:18:18.199 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.199 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.199 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:18:18.199 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.762202+0000", 2023-10-09T06:18:18.199 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5018000000000006e-05 2023-10-09T06:18:18.199 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.199 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.199 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:18.200 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.775126+0000", 2023-10-09T06:18:18.200 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012924542000000001 2023-10-09T06:18:18.200 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.200 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.200 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.200 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793550+0000", 2023-10-09T06:18:18.200 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018423946 2023-10-09T06:18:18.200 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.200 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.201 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.201 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793588+0000", 2023-10-09T06:18:18.201 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7679000000000003e-05 2023-10-09T06:18:18.201 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.201 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.201 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:18.201 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793627+0000", 2023-10-09T06:18:18.201 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9410999999999999e-05 2023-10-09T06:18:18.201 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.201 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.201 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:18.201 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793650+0000", 2023-10-09T06:18:18.202 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2719999999999999e-05 2023-10-09T06:18:18.202 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.202 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:18.202 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.202 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.202 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.202 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:131789 10.9 10:95bec615:::benchmark_data_smithi064_573750_object35574:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-10-09T06:18:18.202 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:14:20.804037+0000", 2023-10-09T06:18:18.202 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 237.343562926, 2023-10-09T06:18:18.202 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.98989388300000003, 2023-10-09T06:18:18.202 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:18.203 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:18.203 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:18.203 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:18:18.203 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:18.203 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 131789 2023-10-09T06:18:18.203 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.203 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:18.203 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.203 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:18.203 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804037+0000", 2023-10-09T06:18:18.203 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.203 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.204 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.204 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:18.204 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804037+0000", 2023-10-09T06:18:18.204 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.204 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.204 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.204 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:18.204 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804037+0000", 2023-10-09T06:18:18.204 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:18.204 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.204 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.204 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:18.205 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804040+0000", 2023-10-09T06:18:18.205 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9730000000000002e-06 2023-10-09T06:18:18.205 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.205 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.205 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:18.205 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804046+0000", 2023-10-09T06:18:18.205 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3249999999999996e-06 2023-10-09T06:18:18.205 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.205 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.205 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:18.205 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804050+0000", 2023-10-09T06:18:18.206 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2239999999999997e-06 2023-10-09T06:18:18.206 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.206 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.206 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.206 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804140+0000", 2023-10-09T06:18:18.206 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9087999999999998e-05 2023-10-09T06:18:18.206 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.206 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.206 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:18.206 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804145+0000", 2023-10-09T06:18:18.206 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0889999999999999e-06 2023-10-09T06:18:18.207 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.207 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.207 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.207 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.764349+0000", 2023-10-09T06:18:18.207 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96020412399999999 2023-10-09T06:18:18.207 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.207 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.207 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:18.207 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.764467+0000", 2023-10-09T06:18:18.207 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011781300000000001 2023-10-09T06:18:18.207 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.207 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.208 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:18:18.208 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.764590+0000", 2023-10-09T06:18:18.208 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012318400000000001 2023-10-09T06:18:18.208 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.208 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.208 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:18.208 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.775135+0000", 2023-10-09T06:18:18.208 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010544992 2023-10-09T06:18:18.208 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.209 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.209 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.209 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793727+0000", 2023-10-09T06:18:18.209 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018592457 2023-10-09T06:18:18.209 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.209 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.209 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.209 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793848+0000", 2023-10-09T06:18:18.209 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000120865 2023-10-09T06:18:18.209 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.209 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.210 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:18.210 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793888+0000", 2023-10-09T06:18:18.210 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0070000000000001e-05 2023-10-09T06:18:18.210 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.210 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.210 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:18.210 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793931+0000", 2023-10-09T06:18:18.210 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3118000000000003e-05 2023-10-09T06:18:18.210 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.210 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:18.210 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.211 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.211 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.211 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:131809 10.9 10:93420ba2:::benchmark_data_smithi064_573750_object35594:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-10-09T06:18:18.211 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:14:20.820581+0000", 2023-10-09T06:18:18.211 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 237.32701965999999, 2023-10-09T06:18:18.211 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.97345599199999999, 2023-10-09T06:18:18.211 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:18.211 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:18.211 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:18.211 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:18:18.211 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:18.211 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 131809 2023-10-09T06:18:18.212 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.212 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:18.212 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.212 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:18.212 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.820581+0000", 2023-10-09T06:18:18.212 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.212 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.212 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.212 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:18.212 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.820581+0000", 2023-10-09T06:18:18.212 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.213 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.213 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.213 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:18.213 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.820579+0000", 2023-10-09T06:18:18.213 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:18.213 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.213 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.213 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:18.213 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.820586+0000", 2023-10-09T06:18:18.213 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6000000000000003e-06 2023-10-09T06:18:18.213 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.213 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.214 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:18.214 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.820603+0000", 2023-10-09T06:18:18.214 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7329999999999998e-05 2023-10-09T06:18:18.214 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.214 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.214 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:18.214 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.820610+0000", 2023-10-09T06:18:18.214 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2079999999999997e-06 2023-10-09T06:18:18.214 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.214 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.214 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.214 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.820665+0000", 2023-10-09T06:18:18.215 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5900999999999999e-05 2023-10-09T06:18:18.215 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.215 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.215 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:18.215 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.820673+0000", 2023-10-09T06:18:18.215 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7780000000000007e-06 2023-10-09T06:18:18.215 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.215 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.215 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.215 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.766668+0000", 2023-10-09T06:18:18.215 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.94599452299999998 2023-10-09T06:18:18.216 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.216 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.216 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:18.216 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.766781+0000", 2023-10-09T06:18:18.216 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011279499999999999 2023-10-09T06:18:18.216 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.216 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.216 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:18:18.216 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.766894+0000", 2023-10-09T06:18:18.216 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000113927 2023-10-09T06:18:18.216 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.216 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.217 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:18.217 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.775144+0000", 2023-10-09T06:18:18.217 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.008249269 2023-10-09T06:18:18.217 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.217 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.217 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.217 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793750+0000", 2023-10-09T06:18:18.217 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018606527000000001 2023-10-09T06:18:18.217 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.217 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.217 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.217 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793979+0000", 2023-10-09T06:18:18.218 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000229175 2023-10-09T06:18:18.218 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.218 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.218 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:18.218 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.794008+0000", 2023-10-09T06:18:18.218 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.8907e-05 2023-10-09T06:18:18.218 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.218 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.218 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:18.218 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.794037+0000", 2023-10-09T06:18:18.218 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.8339000000000001e-05 2023-10-09T06:18:18.218 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.219 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:18.219 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.219 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.219 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.219 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:131814 10.9 10:98c8ec43:::benchmark_data_smithi064_573750_object35599:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-10-09T06:18:18.219 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:14:20.832064+0000", 2023-10-09T06:18:18.219 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 237.315536458, 2023-10-09T06:18:18.219 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96215577299999999, 2023-10-09T06:18:18.219 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:18.219 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:18.219 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:18.220 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:18:18.220 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:18.220 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 131814 2023-10-09T06:18:18.220 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.220 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:18.220 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.220 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:18.220 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.832064+0000", 2023-10-09T06:18:18.220 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.220 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.220 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.220 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:18.221 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.832064+0000", 2023-10-09T06:18:18.221 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.221 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.221 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.221 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:18.221 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.832063+0000", 2023-10-09T06:18:18.221 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:18.221 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.221 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.221 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:18.221 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.832073+0000", 2023-10-09T06:18:18.221 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0720000000000001e-05 2023-10-09T06:18:18.222 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.222 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.222 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:18.222 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.832085+0000", 2023-10-09T06:18:18.222 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2157000000000001e-05 2023-10-09T06:18:18.222 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.222 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.222 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:18.222 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.832091+0000", 2023-10-09T06:18:18.222 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4519999999999996e-06 2023-10-09T06:18:18.222 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.223 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.223 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.223 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.832140+0000", 2023-10-09T06:18:18.223 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9036999999999998e-05 2023-10-09T06:18:18.223 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.223 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.223 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:18.223 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.832146+0000", 2023-10-09T06:18:18.223 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4880000000000004e-06 2023-10-09T06:18:18.223 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.223 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.223 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.223 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.769195+0000", 2023-10-09T06:18:18.224 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.93704818499999998 2023-10-09T06:18:18.224 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.224 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.224 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:18.224 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.769310+0000", 2023-10-09T06:18:18.224 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000115648 2023-10-09T06:18:18.224 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.224 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.224 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:18:18.224 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.769403+0000", 2023-10-09T06:18:18.224 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2818999999999993e-05 2023-10-09T06:18:18.225 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.225 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.225 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:18.225 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.781216+0000", 2023-10-09T06:18:18.225 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011812977000000001 2023-10-09T06:18:18.225 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.225 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.225 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.225 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.794083+0000", 2023-10-09T06:18:18.225 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012867286 2023-10-09T06:18:18.225 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.226 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.226 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.226 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.794159+0000", 2023-10-09T06:18:18.226 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5951999999999994e-05 2023-10-09T06:18:18.226 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.226 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.226 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:18.226 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.794186+0000", 2023-10-09T06:18:18.226 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.6866999999999999e-05 2023-10-09T06:18:18.227 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.227 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.227 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:18.227 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.794220+0000", 2023-10-09T06:18:18.227 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.3445000000000003e-05 2023-10-09T06:18:18.227 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.227 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:18.227 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.227 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.227 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.227 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:114784 11.3 11:c3e96248:::benchmark_data_smithi064_631903_object5870:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-10-09T06:18:18.228 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:17:46.582244+0000", 2023-10-09T06:18:18.228 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 31.565356778000002, 2023-10-09T06:18:18.228 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.171536272, 2023-10-09T06:18:18.228 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:18.228 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:18:18.228 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:18.228 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:18:18.228 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:18:18.228 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 114784 2023-10-09T06:18:18.228 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.229 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:18.229 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.229 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:18.229 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.582244+0000", 2023-10-09T06:18:18.229 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.229 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.229 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.229 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:18.229 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.582244+0000", 2023-10-09T06:18:18.229 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.229 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.229 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.230 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:18.230 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.582242+0000", 2023-10-09T06:18:18.230 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:18.230 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.230 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.230 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:18.230 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.582250+0000", 2023-10-09T06:18:18.230 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3799999999999996e-06 2023-10-09T06:18:18.230 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.230 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.230 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:18.231 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.582263+0000", 2023-10-09T06:18:18.231 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3191e-05 2023-10-09T06:18:18.231 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.231 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.231 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:18.231 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.582273+0000", 2023-10-09T06:18:18.231 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0380999999999999e-05 2023-10-09T06:18:18.231 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.231 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.231 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.231 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.582334+0000", 2023-10-09T06:18:18.231 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0584999999999997e-05 2023-10-09T06:18:18.232 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.232 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.232 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:18.232 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.582435+0000", 2023-10-09T06:18:18.232 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000101631 2023-10-09T06:18:18.232 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.232 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.232 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:18:18.232 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.582509+0000", 2023-10-09T06:18:18.232 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3464e-05 2023-10-09T06:18:18.232 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.232 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.232 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.233 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.594177+0000", 2023-10-09T06:18:18.233 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011668227 2023-10-09T06:18:18.233 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.233 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.233 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:18.233 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.594188+0000", 2023-10-09T06:18:18.233 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0857e-05 2023-10-09T06:18:18.233 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.233 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.233 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.233 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:47.610233+0000", 2023-10-09T06:18:18.233 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0160452019999999 2023-10-09T06:18:18.234 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.234 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.234 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:18.234 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:47.613670+0000", 2023-10-09T06:18:18.234 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0034372809999999999 2023-10-09T06:18:18.234 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.234 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.234 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.234 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.719998+0000", 2023-10-09T06:18:18.234 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.10632803 2023-10-09T06:18:18.234 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.235 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.235 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for missing object", 2023-10-09T06:18:18.235 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.720141+0000", 2023-10-09T06:18:18.235 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014227900000000001 2023-10-09T06:18:18.235 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.235 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.235 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.235 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.753714+0000", 2023-10-09T06:18:18.235 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.033572777999999998 2023-10-09T06:18:18.235 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.235 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.236 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:18.236 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.753780+0000", 2023-10-09T06:18:18.236 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6303000000000005e-05 2023-10-09T06:18:18.236 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.236 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:18.236 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.236 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.236 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.236 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:114793 11.5 11:afddf1af:::benchmark_data_smithi064_631903_object5879:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-10-09T06:18:18.236 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:17:46.609723+0000", 2023-10-09T06:18:18.236 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 31.537877045999998, 2023-10-09T06:18:18.236 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2334629800000001, 2023-10-09T06:18:18.237 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:18.237 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:18.237 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:18.237 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:18:18.237 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:18:18.237 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 114793 2023-10-09T06:18:18.237 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.237 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:18.237 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.237 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:18.237 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.609723+0000", 2023-10-09T06:18:18.238 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.238 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.238 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.238 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:18.238 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.609723+0000", 2023-10-09T06:18:18.238 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.238 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.238 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.238 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:18.238 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.609722+0000", 2023-10-09T06:18:18.238 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:18:18.239 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.239 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.239 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:18.239 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.609728+0000", 2023-10-09T06:18:18.239 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7039999999999999e-06 2023-10-09T06:18:18.239 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.239 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.239 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:18.239 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.609739+0000", 2023-10-09T06:18:18.239 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0693000000000001e-05 2023-10-09T06:18:18.239 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.239 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.240 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:18.240 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.609745+0000", 2023-10-09T06:18:18.240 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7279999999999998e-06 2023-10-09T06:18:18.240 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.240 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.240 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.240 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.609813+0000", 2023-10-09T06:18:18.240 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8081000000000001e-05 2023-10-09T06:18:18.240 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.240 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.240 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:18.240 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.609830+0000", 2023-10-09T06:18:18.241 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6167e-05 2023-10-09T06:18:18.241 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.241 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.241 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.241 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:47.613206+0000", 2023-10-09T06:18:18.241 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.003376144 2023-10-09T06:18:18.241 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.241 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.241 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:18.241 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:47.613220+0000", 2023-10-09T06:18:18.241 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4768e-05 2023-10-09T06:18:18.241 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.242 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.242 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.242 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.801212+0000", 2023-10-09T06:18:18.242 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.187991953 2023-10-09T06:18:18.242 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.242 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.242 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:18.242 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.801309+0000", 2023-10-09T06:18:18.242 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.6799999999999995e-05 2023-10-09T06:18:18.242 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.242 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.242 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,5,6", 2023-10-09T06:18:18.243 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.801755+0000", 2023-10-09T06:18:18.243 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00044560900000000002 2023-10-09T06:18:18.243 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.243 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.243 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:18.243 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.812096+0000", 2023-10-09T06:18:18.243 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010341331 2023-10-09T06:18:18.243 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.243 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.243 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.243 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.812130+0000", 2023-10-09T06:18:18.244 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.4175999999999999e-05 2023-10-09T06:18:18.244 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.244 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.244 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.244 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.812167+0000", 2023-10-09T06:18:18.244 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6251999999999998e-05 2023-10-09T06:18:18.244 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.244 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.244 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.244 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.843000+0000", 2023-10-09T06:18:18.244 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.030833257999999999 2023-10-09T06:18:18.244 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.245 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.245 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:18.245 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.843063+0000", 2023-10-09T06:18:18.245 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3318999999999995e-05 2023-10-09T06:18:18.245 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.245 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.245 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:18.245 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.843186+0000", 2023-10-09T06:18:18.245 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012310399999999999 2023-10-09T06:18:18.245 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.245 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:18.245 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.246 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.246 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.246 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:114794 11.3 11:c4d965d1:::benchmark_data_smithi064_631903_object5880:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-10-09T06:18:18.246 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:17:46.611601+0000", 2023-10-09T06:18:18.246 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 31.535999660000002, 2023-10-09T06:18:18.246 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.130719021, 2023-10-09T06:18:18.246 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:18.246 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:18.246 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:18.246 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:18:18.247 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:18:18.247 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 114794 2023-10-09T06:18:18.247 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.247 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:18.247 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.247 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:18.247 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.611601+0000", 2023-10-09T06:18:18.247 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.247 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.247 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.247 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:18.248 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.611601+0000", 2023-10-09T06:18:18.248 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.248 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.248 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.248 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:18.248 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.611599+0000", 2023-10-09T06:18:18.248 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:18.248 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.248 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.248 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:18.248 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.611605+0000", 2023-10-09T06:18:18.248 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9379999999999997e-06 2023-10-09T06:18:18.248 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.249 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.249 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:18.249 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.611617+0000", 2023-10-09T06:18:18.249 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1223000000000001e-05 2023-10-09T06:18:18.249 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.249 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.249 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:18.249 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.611623+0000", 2023-10-09T06:18:18.249 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1970000000000001e-06 2023-10-09T06:18:18.249 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.249 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.249 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.250 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.611674+0000", 2023-10-09T06:18:18.250 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0757999999999998e-05 2023-10-09T06:18:18.250 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.250 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.250 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:18.250 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.611682+0000", 2023-10-09T06:18:18.250 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8089999999999999e-06 2023-10-09T06:18:18.250 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.250 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.250 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.250 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:47.613718+0000", 2023-10-09T06:18:18.251 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0020357479999999 2023-10-09T06:18:18.251 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.251 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.251 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:18.251 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:47.613728+0000", 2023-10-09T06:18:18.251 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.9019999999999994e-06 2023-10-09T06:18:18.251 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.251 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.251 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.251 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.720188+0000", 2023-10-09T06:18:18.251 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.106459514 2023-10-09T06:18:18.252 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.252 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.252 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:18.252 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.720322+0000", 2023-10-09T06:18:18.252 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000134122 2023-10-09T06:18:18.252 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.252 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.252 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T06:18:18.252 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.720877+0000", 2023-10-09T06:18:18.252 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00055509600000000004 2023-10-09T06:18:18.252 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.253 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.253 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.253 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.731856+0000", 2023-10-09T06:18:18.253 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010979569 2023-10-09T06:18:18.253 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.253 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.253 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:18.253 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.731867+0000", 2023-10-09T06:18:18.253 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0282000000000001e-05 2023-10-09T06:18:18.253 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.253 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.253 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.254 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.731902+0000", 2023-10-09T06:18:18.254 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5599000000000003e-05 2023-10-09T06:18:18.254 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.254 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.254 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.254 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.742157+0000", 2023-10-09T06:18:18.254 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010254776 2023-10-09T06:18:18.254 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.254 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.254 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:18.254 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.742218+0000", 2023-10-09T06:18:18.255 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0603000000000002e-05 2023-10-09T06:18:18.255 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.255 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.255 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:18.255 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.742320+0000", 2023-10-09T06:18:18.255 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000102086 2023-10-09T06:18:18.255 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.255 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:18.255 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.255 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.255 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.256 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:114797 11.4 11:282f7824:::benchmark_data_smithi064_631903_object5883:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-10-09T06:18:18.256 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:17:46.615504+0000", 2023-10-09T06:18:18.256 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 31.532096195000001, 2023-10-09T06:18:18.256 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.230211868, 2023-10-09T06:18:18.256 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:18.256 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:18.256 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:18.256 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:18:18.256 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:18:18.256 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 114797 2023-10-09T06:18:18.256 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.257 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:18.257 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.257 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:18.257 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615504+0000", 2023-10-09T06:18:18.257 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.257 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.257 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.257 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:18.257 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615504+0000", 2023-10-09T06:18:18.257 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.257 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.257 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.258 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:18.258 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615503+0000", 2023-10-09T06:18:18.258 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:18.258 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.258 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.258 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:18.258 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615509+0000", 2023-10-09T06:18:18.258 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0399999999999998e-06 2023-10-09T06:18:18.258 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.258 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.258 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:18.259 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615521+0000", 2023-10-09T06:18:18.259 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2072000000000001e-05 2023-10-09T06:18:18.259 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.259 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.259 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:18.259 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615526+0000", 2023-10-09T06:18:18.259 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3510000000000001e-06 2023-10-09T06:18:18.259 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.259 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.259 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.259 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615579+0000", 2023-10-09T06:18:18.260 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3093999999999997e-05 2023-10-09T06:18:18.260 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.260 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.260 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:18.260 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615588+0000", 2023-10-09T06:18:18.260 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1749999999999994e-06 2023-10-09T06:18:18.260 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.260 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.260 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.260 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:47.616324+0000", 2023-10-09T06:18:18.260 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.000735191 2023-10-09T06:18:18.260 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.261 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.261 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:18.261 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:47.616333+0000", 2023-10-09T06:18:18.261 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.5899999999999997e-06 2023-10-09T06:18:18.261 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.261 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.261 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.261 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.757868+0000", 2023-10-09T06:18:18.261 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1415350879999999 2023-10-09T06:18:18.261 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.261 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.261 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:18.262 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.757956+0000", 2023-10-09T06:18:18.262 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.7495000000000002e-05 2023-10-09T06:18:18.262 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.262 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.262 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T06:18:18.262 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.758426+0000", 2023-10-09T06:18:18.262 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00047022900000000001 2023-10-09T06:18:18.262 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.262 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.262 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:18.262 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.767516+0000", 2023-10-09T06:18:18.263 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0090905719999999999 2023-10-09T06:18:18.263 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.263 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.263 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.263 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.779063+0000", 2023-10-09T06:18:18.263 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011546172 2023-10-09T06:18:18.263 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.263 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.263 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.263 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.779515+0000", 2023-10-09T06:18:18.263 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00045276600000000001 2023-10-09T06:18:18.264 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.264 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.264 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.264 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.845550+0000", 2023-10-09T06:18:18.264 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.066034143000000003 2023-10-09T06:18:18.264 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.264 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.264 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:18.264 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.845613+0000", 2023-10-09T06:18:18.264 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3727999999999997e-05 2023-10-09T06:18:18.264 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.264 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.265 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:18.265 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.845716+0000", 2023-10-09T06:18:18.265 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010272000000000001 2023-10-09T06:18:18.265 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.265 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:18.265 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.265 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.265 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.265 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:114799 11.3 11:cd132cdc:::benchmark_data_smithi064_631903_object5885:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-10-09T06:18:18.265 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:17:46.615538+0000", 2023-10-09T06:18:18.265 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 31.532062672999999, 2023-10-09T06:18:18.266 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1269155259999999, 2023-10-09T06:18:18.266 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:18.266 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:18.266 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:18.266 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:18:18.266 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:18:18.266 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 114799 2023-10-09T06:18:18.266 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.266 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:18.266 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.266 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:18.267 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615538+0000", 2023-10-09T06:18:18.267 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.267 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.267 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.267 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:18.267 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615538+0000", 2023-10-09T06:18:18.267 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.267 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.267 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.267 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:18.267 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615537+0000", 2023-10-09T06:18:18.267 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:18.267 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.268 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.268 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:18.268 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615540+0000", 2023-10-09T06:18:18.268 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2180000000000002e-06 2023-10-09T06:18:18.268 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.268 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.268 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:18.268 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615551+0000", 2023-10-09T06:18:18.268 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0963e-05 2023-10-09T06:18:18.268 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.268 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.269 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:18.269 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615554+0000", 2023-10-09T06:18:18.269 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1030000000000002e-06 2023-10-09T06:18:18.269 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.269 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.269 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.269 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615596+0000", 2023-10-09T06:18:18.269 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2035e-05 2023-10-09T06:18:18.269 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.269 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.269 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:18.270 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615602+0000", 2023-10-09T06:18:18.270 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.959e-06 2023-10-09T06:18:18.270 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.270 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.270 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.270 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:47.613755+0000", 2023-10-09T06:18:18.270 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99815351699999999 2023-10-09T06:18:18.270 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.270 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.270 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:18.270 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:47.613764+0000", 2023-10-09T06:18:18.270 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1039999999999996e-06 2023-10-09T06:18:18.271 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.271 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.271 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.271 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.728423+0000", 2023-10-09T06:18:18.271 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1146591880000001 2023-10-09T06:18:18.271 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.271 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.271 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:18.271 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.728559+0000", 2023-10-09T06:18:18.271 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013571699999999999 2023-10-09T06:18:18.271 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.272 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.272 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T06:18:18.272 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.728700+0000", 2023-10-09T06:18:18.272 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014088100000000001 2023-10-09T06:18:18.272 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.272 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.272 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:18.272 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.732137+0000", 2023-10-09T06:18:18.272 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0034370870000000001 2023-10-09T06:18:18.272 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.272 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.273 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.273 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.732626+0000", 2023-10-09T06:18:18.273 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00048875199999999996 2023-10-09T06:18:18.273 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.273 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.273 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.273 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.734156+0000", 2023-10-09T06:18:18.273 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001529982 2023-10-09T06:18:18.273 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.273 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.273 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.273 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.742358+0000", 2023-10-09T06:18:18.274 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0082017779999999998 2023-10-09T06:18:18.274 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.274 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.274 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:18.274 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.742393+0000", 2023-10-09T06:18:18.274 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5673999999999998e-05 2023-10-09T06:18:18.274 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.274 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.274 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:18.274 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.742453+0000", 2023-10-09T06:18:18.275 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9889000000000002e-05 2023-10-09T06:18:18.275 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.275 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:18.275 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.275 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.275 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.275 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:114800 11.4 11:23e3be1d:::benchmark_data_smithi064_631903_object5886:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-10-09T06:18:18.275 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:17:46.616114+0000", 2023-10-09T06:18:18.275 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 31.531486260000001, 2023-10-09T06:18:18.275 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2298217999999999, 2023-10-09T06:18:18.275 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:18.275 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:18.276 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:18.276 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:18:18.276 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:18:18.276 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 114800 2023-10-09T06:18:18.276 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.276 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:18.276 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.276 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:18.276 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.616114+0000", 2023-10-09T06:18:18.276 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.276 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.277 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.277 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:18.277 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.616114+0000", 2023-10-09T06:18:18.277 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.277 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.277 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.277 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:18.277 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.616113+0000", 2023-10-09T06:18:18.277 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:18.277 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.277 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.277 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:18.278 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.616118+0000", 2023-10-09T06:18:18.278 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0690000000000003e-06 2023-10-09T06:18:18.278 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.278 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.278 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:18.278 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.616127+0000", 2023-10-09T06:18:18.278 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7089999999999997e-06 2023-10-09T06:18:18.278 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.278 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.278 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:18.278 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.616132+0000", 2023-10-09T06:18:18.279 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2810000000000002e-06 2023-10-09T06:18:18.279 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.279 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.279 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.279 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.616181+0000", 2023-10-09T06:18:18.279 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9543000000000003e-05 2023-10-09T06:18:18.279 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.279 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.279 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:18.279 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.616191+0000", 2023-10-09T06:18:18.279 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0176999999999999e-05 2023-10-09T06:18:18.279 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.280 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.280 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.280 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:47.616355+0000", 2023-10-09T06:18:18.280 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.000163873 2023-10-09T06:18:18.280 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.280 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.280 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:18.280 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:47.616363+0000", 2023-10-09T06:18:18.280 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8709999999999997e-06 2023-10-09T06:18:18.280 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.280 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.280 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.281 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.764770+0000", 2023-10-09T06:18:18.281 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1484067330000001 2023-10-09T06:18:18.281 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.281 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.281 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:18.281 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.764891+0000", 2023-10-09T06:18:18.281 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012121900000000001 2023-10-09T06:18:18.281 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.281 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.281 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T06:18:18.281 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.765029+0000", 2023-10-09T06:18:18.281 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013798900000000001 2023-10-09T06:18:18.282 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.282 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.282 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:18.282 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.767525+0000", 2023-10-09T06:18:18.282 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0024960149999999999 2023-10-09T06:18:18.282 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.282 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.282 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.282 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.779447+0000", 2023-10-09T06:18:18.282 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011922532 2023-10-09T06:18:18.282 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.282 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.283 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.283 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.785755+0000", 2023-10-09T06:18:18.283 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0063076829999999997 2023-10-09T06:18:18.283 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.283 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.283 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.283 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.845767+0000", 2023-10-09T06:18:18.283 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.060011768 2023-10-09T06:18:18.283 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.283 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.283 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:18.284 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.845817+0000", 2023-10-09T06:18:18.284 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0129000000000001e-05 2023-10-09T06:18:18.284 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.284 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.284 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:18.284 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.845936+0000", 2023-10-09T06:18:18.284 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000118819 2023-10-09T06:18:18.284 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.284 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:18.284 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.284 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.284 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.285 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:124811 11.f 11:f42f0b99:::benchmark_data_smithi064_631903_object15897:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e601)", 2023-10-09T06:18:18.285 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:18:01.994785+0000", 2023-10-09T06:18:18.285 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 16.152815197999999, 2023-10-09T06:18:18.285 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24886887699999999, 2023-10-09T06:18:18.285 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:18.285 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:18.285 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:18.285 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:18:18.285 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:18:18.285 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 124811 2023-10-09T06:18:18.285 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.285 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:18.286 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.286 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:18.286 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:01.994785+0000", 2023-10-09T06:18:18.286 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.286 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.286 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.286 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:18.286 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:01.994785+0000", 2023-10-09T06:18:18.286 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.286 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.286 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.286 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:18.287 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:01.994784+0000", 2023-10-09T06:18:18.287 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:18.287 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.287 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.287 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:18.287 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:01.994790+0000", 2023-10-09T06:18:18.287 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1709999999999999e-06 2023-10-09T06:18:18.287 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.287 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.287 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:18.287 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:01.994800+0000", 2023-10-09T06:18:18.287 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.859e-06 2023-10-09T06:18:18.288 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.288 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.288 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:18.288 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:01.994805+0000", 2023-10-09T06:18:18.288 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5550000000000002e-06 2023-10-09T06:18:18.288 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.288 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.288 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.288 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:01.994850+0000", 2023-10-09T06:18:18.288 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4125000000000002e-05 2023-10-09T06:18:18.288 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.288 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.289 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:18.289 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:01.994958+0000", 2023-10-09T06:18:18.289 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000108615 2023-10-09T06:18:18.289 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.289 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.289 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:18:18.289 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:02.235229+0000", 2023-10-09T06:18:18.289 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.240271137 2023-10-09T06:18:18.289 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.289 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.289 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.289 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:02.243508+0000", 2023-10-09T06:18:18.289 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0082784910000000007 2023-10-09T06:18:18.290 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.290 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.290 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.290 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:02.243568+0000", 2023-10-09T06:18:18.290 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0288999999999998e-05 2023-10-09T06:18:18.290 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.290 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.290 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:18.290 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:02.243576+0000", 2023-10-09T06:18:18.290 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.3100000000000001e-06 2023-10-09T06:18:18.290 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.291 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.291 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:18.291 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:02.243618+0000", 2023-10-09T06:18:18.291 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2058999999999998e-05 2023-10-09T06:18:18.291 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.291 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.291 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:18.291 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:02.243654+0000", 2023-10-09T06:18:18.291 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5555000000000001e-05 2023-10-09T06:18:18.291 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.291 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:18.291 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.292 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.292 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.292 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:125610 11.3 11:c75f2463:::benchmark_data_smithi064_631903_object16696:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e601)", 2023-10-09T06:18:18.292 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:18:03.001485+0000", 2023-10-09T06:18:18.292 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 15.146115653000001, 2023-10-09T06:18:18.292 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.27016290500000001, 2023-10-09T06:18:18.292 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:18.292 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:18.292 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:18.292 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:18:18.292 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:18:18.292 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 125610 2023-10-09T06:18:18.292 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.293 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:18.293 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.293 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:18.293 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.001485+0000", 2023-10-09T06:18:18.293 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.293 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.293 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.293 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:18.293 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.001485+0000", 2023-10-09T06:18:18.293 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.293 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.293 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.294 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:18.294 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.001483+0000", 2023-10-09T06:18:18.294 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:18.294 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.294 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.294 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:18.294 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.001490+0000", 2023-10-09T06:18:18.294 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6889999999999998e-06 2023-10-09T06:18:18.294 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.294 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.294 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:18.294 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.001501+0000", 2023-10-09T06:18:18.295 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.094e-05 2023-10-09T06:18:18.295 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.295 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.295 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:18.295 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.001505+0000", 2023-10-09T06:18:18.295 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6619999999999996e-06 2023-10-09T06:18:18.295 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.295 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.295 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.295 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.001548+0000", 2023-10-09T06:18:18.295 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2907000000000001e-05 2023-10-09T06:18:18.295 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.296 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.296 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:18.296 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.001670+0000", 2023-10-09T06:18:18.296 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000122082 2023-10-09T06:18:18.296 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.296 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.296 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:18:18.296 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.001800+0000", 2023-10-09T06:18:18.296 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000129414 2023-10-09T06:18:18.296 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.296 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.296 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:18.297 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.007109+0000", 2023-10-09T06:18:18.297 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0053092900000000004 2023-10-09T06:18:18.297 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.297 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.297 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.297 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.008100+0000", 2023-10-09T06:18:18.297 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00099065799999999995 2023-10-09T06:18:18.297 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.297 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.297 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.297 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.271482+0000", 2023-10-09T06:18:18.297 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.26338230200000001 2023-10-09T06:18:18.298 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.298 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.298 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:18.298 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.271531+0000", 2023-10-09T06:18:18.298 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9255000000000002e-05 2023-10-09T06:18:18.298 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.298 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.298 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:18.298 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.271648+0000", 2023-10-09T06:18:18.298 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000116285 2023-10-09T06:18:18.298 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.298 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:18.299 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.299 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.299 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.299 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:125614 11.f 11:fd5edeb2:::benchmark_data_smithi064_631903_object16700:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e601)", 2023-10-09T06:18:18.299 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:18:03.008153+0000", 2023-10-09T06:18:18.299 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 15.139446949, 2023-10-09T06:18:18.299 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.26429092700000001, 2023-10-09T06:18:18.299 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:18.299 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:18.300 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:18.300 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:18:18.300 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:18:18.300 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 125614 2023-10-09T06:18:18.300 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.300 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:18.300 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.300 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:18.300 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.008153+0000", 2023-10-09T06:18:18.300 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.300 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.301 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.301 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:18.301 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.008153+0000", 2023-10-09T06:18:18.301 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.301 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.301 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.301 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:18.301 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.008152+0000", 2023-10-09T06:18:18.301 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T06:18:18.301 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.302 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.302 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:18.302 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.008159+0000", 2023-10-09T06:18:18.302 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8310000000000001e-06 2023-10-09T06:18:18.302 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.302 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.302 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:18.302 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.008177+0000", 2023-10-09T06:18:18.302 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7048999999999999e-05 2023-10-09T06:18:18.302 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.303 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.303 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:18.303 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.008185+0000", 2023-10-09T06:18:18.303 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1780000000000003e-06 2023-10-09T06:18:18.303 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.303 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.303 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.303 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.008231+0000", 2023-10-09T06:18:18.303 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6712999999999998e-05 2023-10-09T06:18:18.303 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.303 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.303 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:18.303 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.008329+0000", 2023-10-09T06:18:18.304 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7122999999999998e-05 2023-10-09T06:18:18.304 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.304 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.304 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:18:18.304 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.008444+0000", 2023-10-09T06:18:18.304 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011537899999999999 2023-10-09T06:18:18.304 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.304 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.304 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:18.304 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.011226+0000", 2023-10-09T06:18:18.304 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0027819260000000001 2023-10-09T06:18:18.304 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.305 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.305 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.305 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.013933+0000", 2023-10-09T06:18:18.305 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0027066740000000001 2023-10-09T06:18:18.305 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.305 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.305 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.305 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.272378+0000", 2023-10-09T06:18:18.305 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.25844518500000002 2023-10-09T06:18:18.305 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.305 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.305 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:18.306 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.272409+0000", 2023-10-09T06:18:18.306 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0893999999999999e-05 2023-10-09T06:18:18.306 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.306 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.306 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:18.306 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.272444+0000", 2023-10-09T06:18:18.306 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5735000000000003e-05 2023-10-09T06:18:18.306 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.306 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:18.306 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.307 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.307 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.307 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:125617 11.4 11:2d1d3d5d:::benchmark_data_smithi064_631903_object16703:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e601)", 2023-10-09T06:18:18.307 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:18:03.010308+0000", 2023-10-09T06:18:18.307 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 15.13729283, 2023-10-09T06:18:18.307 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.26221362199999998, 2023-10-09T06:18:18.307 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:18.307 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:18.307 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:18.307 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:18:18.307 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:18:18.307 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 125617 2023-10-09T06:18:18.308 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.308 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:18.308 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.308 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:18.308 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.010308+0000", 2023-10-09T06:18:18.308 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.308 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.308 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.308 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:18.308 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.010308+0000", 2023-10-09T06:18:18.308 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.308 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.309 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.309 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:18.309 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.010307+0000", 2023-10-09T06:18:18.309 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:18:18.309 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.309 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.309 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:18.309 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.010312+0000", 2023-10-09T06:18:18.309 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3480000000000003e-06 2023-10-09T06:18:18.309 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.309 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.310 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:18.310 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.010323+0000", 2023-10-09T06:18:18.310 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0742e-05 2023-10-09T06:18:18.310 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.310 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.310 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:18.310 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.010329+0000", 2023-10-09T06:18:18.310 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1519999999999997e-06 2023-10-09T06:18:18.310 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.311 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.311 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.311 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.010385+0000", 2023-10-09T06:18:18.311 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6169999999999999e-05 2023-10-09T06:18:18.311 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.311 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.311 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:18.311 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.010525+0000", 2023-10-09T06:18:18.311 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000139944 2023-10-09T06:18:18.311 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.311 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.311 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T06:18:18.311 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.010662+0000", 2023-10-09T06:18:18.312 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000136701 2023-10-09T06:18:18.312 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.312 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.312 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:18.312 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.013159+0000", 2023-10-09T06:18:18.312 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0024969139999999998 2023-10-09T06:18:18.312 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.312 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.312 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.312 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.014073+0000", 2023-10-09T06:18:18.312 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00091416900000000005 2023-10-09T06:18:18.313 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.313 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.313 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.313 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.017553+0000", 2023-10-09T06:18:18.313 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0034799779999999999 2023-10-09T06:18:18.313 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.313 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.313 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.313 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.272344+0000", 2023-10-09T06:18:18.314 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.254790923 2023-10-09T06:18:18.314 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.314 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.314 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:18.314 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.272402+0000", 2023-10-09T06:18:18.314 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.8559e-05 2023-10-09T06:18:18.314 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.314 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.314 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:18.314 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.272521+0000", 2023-10-09T06:18:18.314 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000118861 2023-10-09T06:18:18.315 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.315 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:18.315 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.315 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.315 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.315 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:125621 11.3 11:c6a9b270:::benchmark_data_smithi064_631903_object16707:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e601)", 2023-10-09T06:18:18.315 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:18:03.021236+0000", 2023-10-09T06:18:18.315 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 15.126364074, 2023-10-09T06:18:18.315 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.25134390400000001, 2023-10-09T06:18:18.315 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:18.315 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:18.315 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:18.316 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:18:18.316 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:18:18.316 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 125621 2023-10-09T06:18:18.316 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.316 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:18.316 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.316 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:18.316 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.021236+0000", 2023-10-09T06:18:18.316 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.316 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.316 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.317 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:18.317 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.021236+0000", 2023-10-09T06:18:18.317 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:18.317 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.317 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.317 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:18.317 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.021236+0000", 2023-10-09T06:18:18.317 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:18.317 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.317 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.317 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:18.318 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.021239+0000", 2023-10-09T06:18:18.318 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.4520000000000002e-06 2023-10-09T06:18:18.318 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.318 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.318 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:18.318 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.021248+0000", 2023-10-09T06:18:18.318 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.003e-06 2023-10-09T06:18:18.318 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.318 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.318 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:18.318 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.021252+0000", 2023-10-09T06:18:18.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8489999999999999e-06 2023-10-09T06:18:18.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.319 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.319 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:18.319 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.021309+0000", 2023-10-09T06:18:18.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6876999999999998e-05 2023-10-09T06:18:18.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.319 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.319 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:18.319 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.021432+0000", 2023-10-09T06:18:18.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012299899999999999 2023-10-09T06:18:18.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.320 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.320 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:18:18.320 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.021566+0000", 2023-10-09T06:18:18.320 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000134275 2023-10-09T06:18:18.320 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.320 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.320 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:18.320 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.024109+0000", 2023-10-09T06:18:18.320 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0025423189999999999 2023-10-09T06:18:18.320 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.320 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.320 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.320 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.024336+0000", 2023-10-09T06:18:18.321 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00022785899999999999 2023-10-09T06:18:18.321 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.321 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.321 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:18.321 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.272494+0000", 2023-10-09T06:18:18.321 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24815743000000001 2023-10-09T06:18:18.321 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.321 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.321 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:18.321 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.272529+0000", 2023-10-09T06:18:18.321 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5092999999999998e-05 2023-10-09T06:18:18.321 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:18.322 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:18.322 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:18.322 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:03.272580+0000", 2023-10-09T06:18:18.322 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1288000000000001e-05 2023-10-09T06:18:18.322 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.322 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:18.322 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.322 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:18.322 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:18.322 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:18:18.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:18.324 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:18.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:18.425 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:18.469 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:18.520 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:18.537 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:18.538 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 7, 1, 6] out_osds: [2] dead_osds: [5] live_osds: [1, 2, 3, 6, 7, 4, 0] 2023-10-09T06:18:18.539 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:18:18.539 INFO:tasks.thrashosds.thrasher:Reviving osd 5 2023-10-09T06:18:18.539 INFO:tasks.cephadm.osd.5:Restarting osd.5 (starting--it wasn't running)... 2023-10-09T06:18:18.539 DEBUG:teuthology.orchestra.run.smithi156:osd.5> sudo journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.5.service 2023-10-09T06:18:18.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:18.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:18.542 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl start ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.5 2023-10-09T06:18:18.571 INFO:journalctl@ceph.osd.5.smithi156.stdout:-- Logs begin at Mon 2023-10-09 05:16:12 UTC. -- 2023-10-09T06:18:18.642 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:18.678 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:18.851 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:18 smithi156 ceph-mon[195101]: pgmap v1559: 119 pgs: 1 remapped+peering, 1 peering, 18 active+clean+remapped, 17 active+undersized, 3 active+undersized+remapped, 6 active+undersized+degraded, 73 active+clean; 5.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 566 op/s; 20911/251514 objects degraded (8.314%); 8/251514 objects misplaced (0.003%); 9.7 MiB/s, 2 keys/s, 157 objects/s recovering 2023-10-09T06:18:18.851 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:18:18 smithi156 systemd[1]: Starting Ceph osd.5 for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T06:18:18.938 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:18.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:18.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:19.044 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:19.069 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:18:19.106 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:19.117 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:18 smithi064 ceph-mon[212750]: pgmap v1559: 119 pgs: 1 remapped+peering, 1 peering, 18 active+clean+remapped, 17 active+undersized, 3 active+undersized+remapped, 6 active+undersized+degraded, 73 active+clean; 5.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 566 op/s; 20911/251514 objects degraded (8.314%); 8/251514 objects misplaced (0.003%); 9.7 MiB/s, 2 keys/s, 157 objects/s recovering 2023-10-09T06:18:19.117 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:18 smithi064 ceph-mon[220044]: pgmap v1559: 119 pgs: 1 remapped+peering, 1 peering, 18 active+clean+remapped, 17 active+undersized, 3 active+undersized+remapped, 6 active+undersized+degraded, 73 active+clean; 5.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 566 op/s; 20911/251514 objects degraded (8.314%); 8/251514 objects misplaced (0.003%); 9.7 MiB/s, 2 keys/s, 157 objects/s recovering 2023-10-09T06:18:19.238 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:19.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:19.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:19.352 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:19.408 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:19.475 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:19.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:19.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:19.579 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:19.613 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:19.617 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:18:19.860 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:18:20.002 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:20.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:20.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:20.121 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:20.184 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:20.329 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:20.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:20.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:20.436 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:20.503 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:20.527 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:20.543 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:18:20 smithi156 bash[405918]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T06:18:20.543 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:18:20 smithi156 bash[405918]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2023-10-09T06:18:20.543 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:18:20 smithi156 bash[405918]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-5 --no-mon-config 2023-10-09T06:18:20.543 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:18:20 smithi156 bash[405918]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-5/block 2023-10-09T06:18:20.543 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:18:20 smithi156 bash[405918]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-5/block 2023-10-09T06:18:20.543 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:18:20 smithi156 bash[405918]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-10-09T06:18:20.543 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:18:20 smithi156 bash[405918]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2023-10-09T06:18:20.544 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:18:20 smithi156 bash[405918]: --> ceph-volume lvm activate successful for osd ID: 5 2023-10-09T06:18:20.606 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:20.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:20.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:20.711 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:20.794 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:20.858 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:20 smithi156 ceph-mon[195101]: pgmap v1560: 119 pgs: 1 remapped+peering, 1 peering, 18 active+clean+remapped, 17 active+undersized, 3 active+undersized+remapped, 6 active+undersized+degraded, 73 active+clean; 5.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 426 op/s; 20911/251514 objects degraded (8.314%); 8/251514 objects misplaced (0.003%); 7.3 MiB/s, 1 keys/s, 118 objects/s recovering 2023-10-09T06:18:20.861 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:20 smithi064 ceph-mon[212750]: pgmap v1560: 119 pgs: 1 remapped+peering, 1 peering, 18 active+clean+remapped, 17 active+undersized, 3 active+undersized+remapped, 6 active+undersized+degraded, 73 active+clean; 5.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 426 op/s; 20911/251514 objects degraded (8.314%); 8/251514 objects misplaced (0.003%); 7.3 MiB/s, 1 keys/s, 118 objects/s recovering 2023-10-09T06:18:20.862 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:20 smithi064 ceph-mon[220044]: pgmap v1560: 119 pgs: 1 remapped+peering, 1 peering, 18 active+clean+remapped, 17 active+undersized, 3 active+undersized+remapped, 6 active+undersized+degraded, 73 active+clean; 5.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 426 op/s; 20911/251514 objects degraded (8.314%); 8/251514 objects misplaced (0.003%); 7.3 MiB/s, 1 keys/s, 118 objects/s recovering 2023-10-09T06:18:21.189 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:21.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:21.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:21.298 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:21.371 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:21.464 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:21.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:21.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:21.568 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:21.601 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:21.858 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:21 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4032892675' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:18:21.887 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:18:21.900 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:21.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:21.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:22.011 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:22.066 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:22.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:21 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4032892675' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:18:22.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:21 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4032892675' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:18:22.324 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:22.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:22.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:22.430 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:22.457 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:22.525 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:18:22.525 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:18:22.526 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 0 2023-10-09T06:18:22.526 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:18:22.770 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:22.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:22.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:22.876 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:22.909 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:22.973 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:22.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:22.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:23.039 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:22 smithi064 ceph-mon[212750]: pgmap v1561: 119 pgs: 20 active+clean+remapped, 17 active+undersized, 6 active+undersized+degraded, 76 active+clean; 4.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 973 op/s; 19666/237246 objects degraded (8.289%); 8/237246 objects misplaced (0.003%); 5.8 MiB/s, 1 keys/s, 94 objects/s recovering 2023-10-09T06:18:23.039 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:22 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:23.039 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:22 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4032892675' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:18:23.039 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:22 smithi064 ceph-mon[212750]: osdmap e608: 8 total, 7 up, 7 in 2023-10-09T06:18:23.039 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:22 smithi064 ceph-mon[212750]: failed to encode map e608 with expected crc 2023-10-09T06:18:23.039 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:22 smithi064 ceph-mon[212750]: failed to encode map e608 with expected crc 2023-10-09T06:18:23.039 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:22 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 19666/237246 objects degraded (8.289%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:23.041 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:22 smithi064 ceph-mon[220044]: pgmap v1561: 119 pgs: 20 active+clean+remapped, 17 active+undersized, 6 active+undersized+degraded, 76 active+clean; 4.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 973 op/s; 19666/237246 objects degraded (8.289%); 8/237246 objects misplaced (0.003%); 5.8 MiB/s, 1 keys/s, 94 objects/s recovering 2023-10-09T06:18:23.041 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:22 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:23.041 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:22 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4032892675' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:18:23.041 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:22 smithi064 ceph-mon[220044]: osdmap e608: 8 total, 7 up, 7 in 2023-10-09T06:18:23.042 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:22 smithi064 ceph-mon[220044]: failed to encode map e608 with expected crc 2023-10-09T06:18:23.042 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:22 smithi064 ceph-mon[220044]: failed to encode map e608 with expected crc 2023-10-09T06:18:23.042 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:22 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 19666/237246 objects degraded (8.289%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:23.079 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:23.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:22 smithi156 ceph-mon[195101]: pgmap v1561: 119 pgs: 20 active+clean+remapped, 17 active+undersized, 6 active+undersized+degraded, 76 active+clean; 4.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 973 op/s; 19666/237246 objects degraded (8.289%); 8/237246 objects misplaced (0.003%); 5.8 MiB/s, 1 keys/s, 94 objects/s recovering 2023-10-09T06:18:23.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:22 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:23.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:22 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4032892675' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:18:23.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:22 smithi156 ceph-mon[195101]: osdmap e608: 8 total, 7 up, 7 in 2023-10-09T06:18:23.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:22 smithi156 ceph-mon[195101]: failed to encode map e608 with expected crc 2023-10-09T06:18:23.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:22 smithi156 ceph-mon[195101]: failed to encode map e608 with expected crc 2023-10-09T06:18:23.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:22 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 19666/237246 objects degraded (8.289%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:23.150 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:23.272 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:23.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:23.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:23.354 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:18:23.379 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:23.395 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:18:23.439 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:18:23.462 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:23.471 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:18:23.484 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:18:23.486 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:18:23.514 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:18:23 smithi156 bash[406663]: 1d1e8c86f3c535bea0f4ec175890ed20f8d2c6ebec7dc46a1d634c53b676f0b7 2023-10-09T06:18:23.514 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:18:23 smithi156 systemd[1]: Started Ceph osd.5 for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T06:18:23.532 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:18:23.566 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:23.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:23.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:23.577 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:18:23.610 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:18:23.610 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:18:23.610 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:18:23.610 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:18:23.610 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:18:23.611 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:18:23.611 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-09T06:18:23.611 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:18:23.611 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:18:23.674 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:23.746 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:23.835 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:23.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:23.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:23.939 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:24.041 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:24.087 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:18:24.093 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:24.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:24.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:24.148 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:18:24.199 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:24.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:23 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/911248046' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:24.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:23 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/911248046' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:24.233 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:24.368 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:23 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/911248046' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:24.369 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:18:23 smithi156 conmon[406738]: 2023-10-09T06:18:23.991+0000 7f1448ba4540 -1 Falling back to public interface 2023-10-09T06:18:24.535 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:24.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:24.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:24.574 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T06:18:24.575 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T06:18:24.647 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:24.713 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:24.800 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:24.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:24.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:24.887 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:24 smithi156 ceph-mon[195101]: pgmap v1563: 119 pgs: 20 active+clean+remapped, 17 active+undersized, 6 active+undersized+degraded, 76 active+clean; 4.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 779 op/s; 19666/237246 objects degraded (8.289%); 8/237246 objects misplaced (0.003%); 1.9 MiB/s, 31 objects/s recovering 2023-10-09T06:18:24.909 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:24.917 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:18:24.969 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:25.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:24 smithi064 ceph-mon[212750]: pgmap v1563: 119 pgs: 20 active+clean+remapped, 17 active+undersized, 6 active+undersized+degraded, 76 active+clean; 4.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 779 op/s; 19666/237246 objects degraded (8.289%); 8/237246 objects misplaced (0.003%); 1.9 MiB/s, 31 objects/s recovering 2023-10-09T06:18:25.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:24 smithi064 ceph-mon[220044]: pgmap v1563: 119 pgs: 20 active+clean+remapped, 17 active+undersized, 6 active+undersized+degraded, 76 active+clean; 4.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 779 op/s; 19666/237246 objects degraded (8.289%); 8/237246 objects misplaced (0.003%); 1.9 MiB/s, 31 objects/s recovering 2023-10-09T06:18:25.579 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:25.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:18:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:18:25] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.43.0" 2023-10-09T06:18:25.893 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:25.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:25.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:26.000 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:26.063 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:26.150 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:26.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:26.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:26.264 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:26.325 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:26.480 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:26.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:26.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:26.576 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:18:26.586 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:26.661 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:26.723 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:26.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:26.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:26.827 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:26.896 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:26.993 INFO:teuthology.orchestra.run.smithi156.stderr:no valid command found; 10 closest matches: 2023-10-09T06:18:26.993 INFO:teuthology.orchestra.run.smithi156.stderr:0 2023-10-09T06:18:26.993 INFO:teuthology.orchestra.run.smithi156.stderr:1 2023-10-09T06:18:26.993 INFO:teuthology.orchestra.run.smithi156.stderr:2 2023-10-09T06:18:26.993 INFO:teuthology.orchestra.run.smithi156.stderr:abort 2023-10-09T06:18:26.993 INFO:teuthology.orchestra.run.smithi156.stderr:assert 2023-10-09T06:18:26.994 INFO:teuthology.orchestra.run.smithi156.stderr:bluefs debug_inject_read_zeros 2023-10-09T06:18:26.994 INFO:teuthology.orchestra.run.smithi156.stderr:bluefs files list 2023-10-09T06:18:26.994 INFO:teuthology.orchestra.run.smithi156.stderr:bluefs stats 2023-10-09T06:18:26.994 INFO:teuthology.orchestra.run.smithi156.stderr:bluestore bluefs device info [] 2023-10-09T06:18:26.994 INFO:teuthology.orchestra.run.smithi156.stderr:config diff 2023-10-09T06:18:26.994 INFO:teuthology.orchestra.run.smithi156.stderr:admin_socket: invalid command 2023-10-09T06:18:26.995 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:26 smithi156 ceph-mon[195101]: pgmap v1564: 119 pgs: 20 active+clean+remapped, 17 active+undersized, 6 active+undersized+degraded, 76 active+clean; 4.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 898 op/s; 19131/230391 objects degraded (8.304%); 8/230391 objects misplaced (0.003%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T06:18:27.015 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:27.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:27.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:27.124 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:27.212 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:27.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:26 smithi064 ceph-mon[220044]: pgmap v1564: 119 pgs: 20 active+clean+remapped, 17 active+undersized, 6 active+undersized+degraded, 76 active+clean; 4.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 898 op/s; 19131/230391 objects degraded (8.304%); 8/230391 objects misplaced (0.003%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T06:18:27.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:26 smithi064 ceph-mon[212750]: pgmap v1564: 119 pgs: 20 active+clean+remapped, 17 active+undersized, 6 active+undersized+degraded, 76 active+clean; 4.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 898 op/s; 19131/230391 objects degraded (8.304%); 8/230391 objects misplaced (0.003%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T06:18:27.418 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:27.729 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:18:27.729 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:18:27.736 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:27.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:27.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:27.839 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:27.953 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:28.206 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:27 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 19131/230391 objects degraded (8.304%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:28.207 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:27 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 19131/230391 objects degraded (8.304%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:28.242 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:28.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:28.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:28.346 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:28.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:27 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 19131/230391 objects degraded (8.304%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:28.382 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:28.493 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T06:18:28.494 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-5, ['dump_ops_in_flight'] 2023-10-09T06:18:28.832 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:18:28.858 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:28.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:28.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:28.946 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:18:28.966 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:29.049 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:29.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:28 smithi156 ceph-mon[195101]: pgmap v1565: 119 pgs: 20 active+clean+remapped, 17 active+undersized, 6 active+undersized+degraded, 76 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 946 op/s; 18521/223098 objects degraded (8.302%); 8/223098 objects misplaced (0.004%) 2023-10-09T06:18:29.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:28 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1256692992' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:29.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:28 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:29.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:28 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:29.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:28 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:18:29.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:28 smithi064 ceph-mon[212750]: pgmap v1565: 119 pgs: 20 active+clean+remapped, 17 active+undersized, 6 active+undersized+degraded, 76 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 946 op/s; 18521/223098 objects degraded (8.302%); 8/223098 objects misplaced (0.004%) 2023-10-09T06:18:29.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:28 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1256692992' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:29.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:28 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:29.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:28 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:29.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:28 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:18:29.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:28 smithi064 ceph-mon[220044]: pgmap v1565: 119 pgs: 20 active+clean+remapped, 17 active+undersized, 6 active+undersized+degraded, 76 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 946 op/s; 18521/223098 objects degraded (8.302%); 8/223098 objects misplaced (0.004%) 2023-10-09T06:18:29.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:28 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1256692992' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:29.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:28 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:29.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:28 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:29.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:28 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:18:29.357 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:29.359 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:18:29.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:29.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:29.464 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:29.522 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:29.951 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:29 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:18:29.951 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:29 smithi156 ceph-mon[195101]: failed to encode map e609 with expected crc 2023-10-09T06:18:29.952 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:29 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:18:29.952 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:29 smithi156 ceph-mon[195101]: osdmap e609: 8 total, 7 up, 7 in 2023-10-09T06:18:29.952 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:29 smithi156 ceph-mon[195101]: failed to encode map e609 with expected crc 2023-10-09T06:18:29.952 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:29 smithi156 ceph-mon[195101]: failed to encode map e609 with expected crc 2023-10-09T06:18:29.952 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:29 smithi156 ceph-mon[195101]: failed to encode map e609 with expected crc 2023-10-09T06:18:30.029 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:30.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:30.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:30.135 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:30.189 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:30.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:29 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:18:30.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:29 smithi064 ceph-mon[220044]: failed to encode map e609 with expected crc 2023-10-09T06:18:30.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:29 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:18:30.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:29 smithi064 ceph-mon[220044]: osdmap e609: 8 total, 7 up, 7 in 2023-10-09T06:18:30.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:29 smithi064 ceph-mon[220044]: failed to encode map e609 with expected crc 2023-10-09T06:18:30.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:29 smithi064 ceph-mon[220044]: failed to encode map e609 with expected crc 2023-10-09T06:18:30.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:29 smithi064 ceph-mon[220044]: failed to encode map e609 with expected crc 2023-10-09T06:18:30.221 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:18:30 smithi156 conmon[406738]: 2023-10-09T06:18:30.215+0000 7f1448ba4540 -1 osd.5 598 log_to_monitors true 2023-10-09T06:18:30.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:29 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:18:30.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:29 smithi064 ceph-mon[212750]: failed to encode map e609 with expected crc 2023-10-09T06:18:30.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:29 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:18:30.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:29 smithi064 ceph-mon[212750]: osdmap e609: 8 total, 7 up, 7 in 2023-10-09T06:18:30.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:29 smithi064 ceph-mon[212750]: failed to encode map e609 with expected crc 2023-10-09T06:18:30.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:29 smithi064 ceph-mon[212750]: failed to encode map e609 with expected crc 2023-10-09T06:18:30.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:29 smithi064 ceph-mon[212750]: failed to encode map e609 with expected crc 2023-10-09T06:18:30.280 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:30.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:30.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:30.389 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:30.434 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:30.601 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:18:30.644 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:18:30.682 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:18:30.737 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:18:30.770 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:18:30.795 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:18:30.854 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:18:30.855 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:18:30.855 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:18:30.855 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:18:30.855 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:18:30.855 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:18:30.855 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-09T06:18:30.855 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:18:30.855 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:18:31.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:30 smithi064 ceph-mon[212750]: pgmap v1567: 119 pgs: 20 active+clean+remapped, 17 active+undersized, 6 active+undersized+degraded, 76 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 18521/223098 objects degraded (8.302%); 8/223098 objects misplaced (0.004%) 2023-10-09T06:18:31.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:30 smithi064 ceph-mon[212750]: from='osd.5 [v2:172.21.15.156:6808/507344993,v1:172.21.15.156:6809/507344993]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T06:18:31.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:30 smithi064 ceph-mon[212750]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T06:18:31.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:30 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3079816861' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:31.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:30 smithi064 ceph-mon[220044]: pgmap v1567: 119 pgs: 20 active+clean+remapped, 17 active+undersized, 6 active+undersized+degraded, 76 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 18521/223098 objects degraded (8.302%); 8/223098 objects misplaced (0.004%) 2023-10-09T06:18:31.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:30 smithi064 ceph-mon[220044]: from='osd.5 [v2:172.21.15.156:6808/507344993,v1:172.21.15.156:6809/507344993]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T06:18:31.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:30 smithi064 ceph-mon[220044]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T06:18:31.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:30 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3079816861' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:31.335 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:30 smithi156 ceph-mon[195101]: pgmap v1567: 119 pgs: 20 active+clean+remapped, 17 active+undersized, 6 active+undersized+degraded, 76 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 18521/223098 objects degraded (8.302%); 8/223098 objects misplaced (0.004%) 2023-10-09T06:18:31.336 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:30 smithi156 ceph-mon[195101]: from='osd.5 [v2:172.21.15.156:6808/507344993,v1:172.21.15.156:6809/507344993]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T06:18:31.336 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:30 smithi156 ceph-mon[195101]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T06:18:31.336 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:30 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3079816861' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:31.336 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:18:30 smithi156 conmon[406738]: 2023-10-09T06:18:30.992+0000 7f1438981700 -1 osd.5 598 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T06:18:31.424 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:31.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:31.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:31.536 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:31.584 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:31.983 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:31 smithi156 ceph-mon[195101]: failed to encode map e610 with expected crc 2023-10-09T06:18:31.984 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:31 smithi156 ceph-mon[195101]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T06:18:31.984 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:31 smithi156 ceph-mon[195101]: osdmap e610: 8 total, 7 up, 7 in 2023-10-09T06:18:31.984 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:31 smithi156 ceph-mon[195101]: from='osd.5 [v2:172.21.15.156:6808/507344993,v1:172.21.15.156:6809/507344993]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T06:18:31.984 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:31 smithi156 ceph-mon[195101]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T06:18:31.984 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:31 smithi156 ceph-mon[195101]: failed to encode map e610 with expected crc 2023-10-09T06:18:31.984 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:31 smithi156 ceph-mon[195101]: failed to encode map e610 with expected crc 2023-10-09T06:18:31.984 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:31 smithi156 ceph-mon[195101]: failed to encode map e610 with expected crc 2023-10-09T06:18:31.984 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:31 smithi156 ceph-mon[195101]: pgmap v1569: 119 pgs: 20 active+clean+remapped, 17 active+undersized, 6 active+undersized+degraded, 76 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 17479/210357 objects degraded (8.309%); 8/210357 objects misplaced (0.004%) 2023-10-09T06:18:32.021 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:32.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:32.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:32.089 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:18:32.089 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T06:18:32.089 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T06:18:32.089 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:18:32.089 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.089 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:116291 9.e 9:7a5f1263:::benchmark_data_smithi064_511911_object15189:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e449)", 2023-10-09T06:18:32.089 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:09:02.760137+0000", 2023-10-09T06:18:32.089 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 569.32448616199997, 2023-10-09T06:18:32.090 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.891237442, 2023-10-09T06:18:32.090 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:32.090 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:18:32.090 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:32.090 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:18:32.090 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:18:32.090 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 116291 2023-10-09T06:18:32.090 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.090 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:32.090 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.091 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:32.091 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.760137+0000", 2023-10-09T06:18:32.091 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.091 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.091 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.091 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:32.091 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.760137+0000", 2023-10-09T06:18:32.091 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.091 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.091 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.091 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:32.091 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.760137+0000", 2023-10-09T06:18:32.092 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:32.092 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.092 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.092 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:32.092 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.760139+0000", 2023-10-09T06:18:32.092 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2390000000000001e-06 2023-10-09T06:18:32.092 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.092 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.092 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:32.092 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.760145+0000", 2023-10-09T06:18:32.092 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4589999999999997e-06 2023-10-09T06:18:32.092 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.093 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.093 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:32.093 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.760148+0000", 2023-10-09T06:18:32.093 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5870000000000001e-06 2023-10-09T06:18:32.093 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.093 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.093 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.093 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.761468+0000", 2023-10-09T06:18:32.093 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001319473 2023-10-09T06:18:32.093 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.093 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.094 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.094 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:02.761474+0000", 2023-10-09T06:18:32.094 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9789999999999998e-06 2023-10-09T06:18:32.094 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.094 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.094 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.094 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.589356+0000", 2023-10-09T06:18:32.094 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.82788202 2023-10-09T06:18:32.094 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.094 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.094 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.094 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:03.589363+0000", 2023-10-09T06:18:32.095 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.973e-06 2023-10-09T06:18:32.095 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.095 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.095 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.095 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.626391+0000", 2023-10-09T06:18:32.095 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.037028439 2023-10-09T06:18:32.095 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.095 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.095 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for missing object", 2023-10-09T06:18:32.095 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.626524+0000", 2023-10-09T06:18:32.095 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000132395 2023-10-09T06:18:32.095 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.096 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.096 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.096 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.651273+0000", 2023-10-09T06:18:32.096 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.024749376999999999 2023-10-09T06:18:32.096 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.096 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.096 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:32.096 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:09:04.651375+0000", 2023-10-09T06:18:32.096 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000101801 2023-10-09T06:18:32.096 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.096 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:32.096 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.097 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.097 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.097 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:95146 10.d 10:bfa5e452:::benchmark_data_smithi064_573750_object95145:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-10-09T06:18:32.097 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:20.510076+0000", 2023-10-09T06:18:32.097 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 311.57454751699998, 2023-10-09T06:18:32.097 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3209230409999999, 2023-10-09T06:18:32.097 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:32.097 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:18:32.097 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:32.097 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:18:32.097 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:32.097 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 95146 2023-10-09T06:18:32.098 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.098 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:32.098 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.098 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:32.098 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.510076+0000", 2023-10-09T06:18:32.098 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.098 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.098 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.098 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:32.098 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.510076+0000", 2023-10-09T06:18:32.098 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.099 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.099 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.099 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:32.099 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.510075+0000", 2023-10-09T06:18:32.099 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:32.099 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.099 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.099 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:32.099 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.510114+0000", 2023-10-09T06:18:32.099 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8423000000000001e-05 2023-10-09T06:18:32.099 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.099 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.100 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:32.100 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.510121+0000", 2023-10-09T06:18:32.100 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6890000000000004e-06 2023-10-09T06:18:32.100 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.100 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.100 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:32.100 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.510126+0000", 2023-10-09T06:18:32.100 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.707e-06 2023-10-09T06:18:32.100 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.100 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.100 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.100 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.511105+0000", 2023-10-09T06:18:32.101 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00097909999999999989 2023-10-09T06:18:32.101 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.101 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.101 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:32.101 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.511225+0000", 2023-10-09T06:18:32.101 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011972999999999999 2023-10-09T06:18:32.101 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.101 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.101 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:18:32.101 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.511369+0000", 2023-10-09T06:18:32.101 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000143654 2023-10-09T06:18:32.101 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.101 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.102 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:32.102 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.513770+0000", 2023-10-09T06:18:32.102 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002400994 2023-10-09T06:18:32.102 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.102 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.102 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.102 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.519773+0000", 2023-10-09T06:18:32.102 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0060033789999999997 2023-10-09T06:18:32.102 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.102 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.102 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.103 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.711202+0000", 2023-10-09T06:18:32.103 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.191429196 2023-10-09T06:18:32.103 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.103 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.103 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.103 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.711214+0000", 2023-10-09T06:18:32.103 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1576999999999999e-05 2023-10-09T06:18:32.103 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.103 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.103 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.103 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.706110+0000", 2023-10-09T06:18:32.103 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99489621699999997 2023-10-09T06:18:32.104 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.104 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.104 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.104 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.706122+0000", 2023-10-09T06:18:32.104 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.205e-05 2023-10-09T06:18:32.104 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.104 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.104 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.104 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.830928+0000", 2023-10-09T06:18:32.104 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1248056710000001 2023-10-09T06:18:32.104 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.104 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.105 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:32.105 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.830999+0000", 2023-10-09T06:18:32.105 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1289000000000001e-05 2023-10-09T06:18:32.105 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.105 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:32.105 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.105 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.105 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.105 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:95160 10.d 10:b709447e:::benchmark_data_smithi064_573750_object95159:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-10-09T06:18:32.105 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:20.519974+0000", 2023-10-09T06:18:32.105 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 311.564649578, 2023-10-09T06:18:32.105 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3111124510000001, 2023-10-09T06:18:32.106 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:32.106 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:18:32.106 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:32.106 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:18:32.106 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:32.106 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 95160 2023-10-09T06:18:32.106 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.106 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:32.106 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.106 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:32.106 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.519974+0000", 2023-10-09T06:18:32.107 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.107 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.107 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.107 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:32.107 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.519974+0000", 2023-10-09T06:18:32.107 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.107 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.107 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.107 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:32.107 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.519973+0000", 2023-10-09T06:18:32.107 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:32.107 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.108 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.108 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:32.108 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.520009+0000", 2023-10-09T06:18:32.108 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5522999999999999e-05 2023-10-09T06:18:32.108 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.108 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.108 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:32.108 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.520016+0000", 2023-10-09T06:18:32.108 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.17e-06 2023-10-09T06:18:32.108 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.108 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.108 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:32.109 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.520033+0000", 2023-10-09T06:18:32.109 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7465e-05 2023-10-09T06:18:32.109 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.109 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.109 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.109 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.520076+0000", 2023-10-09T06:18:32.109 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2305000000000002e-05 2023-10-09T06:18:32.109 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.109 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.109 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:32.109 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.520170+0000", 2023-10-09T06:18:32.109 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.4647000000000004e-05 2023-10-09T06:18:32.110 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.110 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.110 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:18:32.110 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.520272+0000", 2023-10-09T06:18:32.110 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010146500000000001 2023-10-09T06:18:32.110 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.110 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.110 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:32.110 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.523122+0000", 2023-10-09T06:18:32.110 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0028502169999999999 2023-10-09T06:18:32.110 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.111 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.111 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.111 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.526664+0000", 2023-10-09T06:18:32.111 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.003541485 2023-10-09T06:18:32.111 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.111 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.111 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.111 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.711273+0000", 2023-10-09T06:18:32.111 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.184609416 2023-10-09T06:18:32.111 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.111 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.111 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.111 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.711282+0000", 2023-10-09T06:18:32.112 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0680000000000003e-06 2023-10-09T06:18:32.112 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.112 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.112 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.112 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.706164+0000", 2023-10-09T06:18:32.112 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99488170600000003 2023-10-09T06:18:32.112 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.112 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.112 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.112 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.706171+0000", 2023-10-09T06:18:32.112 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0779999999999998e-06 2023-10-09T06:18:32.113 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.113 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.113 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.113 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.831040+0000", 2023-10-09T06:18:32.113 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.12486949 2023-10-09T06:18:32.113 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.113 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.113 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:32.113 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.831086+0000", 2023-10-09T06:18:32.113 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5976999999999997e-05 2023-10-09T06:18:32.113 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.113 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:32.114 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.114 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.114 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.114 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:95167 10.d 10:bd2a0a0e:::benchmark_data_smithi064_573750_object95166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-10-09T06:18:32.114 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:20.530287+0000", 2023-10-09T06:18:32.114 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 311.554336073, 2023-10-09T06:18:32.114 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3008641750000001, 2023-10-09T06:18:32.114 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:32.114 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:18:32.114 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:32.114 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:18:32.115 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:32.115 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 95167 2023-10-09T06:18:32.115 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.115 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:32.115 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.115 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:32.115 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.530287+0000", 2023-10-09T06:18:32.115 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.115 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.115 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.115 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:32.115 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.530287+0000", 2023-10-09T06:18:32.116 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.116 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.116 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.116 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:32.116 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.530286+0000", 2023-10-09T06:18:32.116 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:32.116 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.116 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.116 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:32.116 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.530311+0000", 2023-10-09T06:18:32.116 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4980000000000001e-05 2023-10-09T06:18:32.116 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.117 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.117 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:32.117 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.530328+0000", 2023-10-09T06:18:32.117 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7280000000000001e-05 2023-10-09T06:18:32.117 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.117 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.117 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:32.117 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.530336+0000", 2023-10-09T06:18:32.117 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4490000000000002e-06 2023-10-09T06:18:32.117 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.117 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.118 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.118 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.530388+0000", 2023-10-09T06:18:32.118 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2438000000000002e-05 2023-10-09T06:18:32.118 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.118 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.118 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:32.118 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.530567+0000", 2023-10-09T06:18:32.118 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00017908900000000001 2023-10-09T06:18:32.118 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.118 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.118 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:18:32.119 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.530723+0000", 2023-10-09T06:18:32.119 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000155833 2023-10-09T06:18:32.119 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.119 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.119 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.119 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.533279+0000", 2023-10-09T06:18:32.119 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0025553939999999999 2023-10-09T06:18:32.119 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.119 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.119 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:32.119 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.536005+0000", 2023-10-09T06:18:32.119 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0027268000000000001 2023-10-09T06:18:32.120 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.120 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.120 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.120 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.711335+0000", 2023-10-09T06:18:32.120 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.175330085 2023-10-09T06:18:32.120 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.120 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.120 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.120 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.711343+0000", 2023-10-09T06:18:32.120 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3679999999999996e-06 2023-10-09T06:18:32.120 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.120 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.121 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.121 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.706203+0000", 2023-10-09T06:18:32.121 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99486017599999998 2023-10-09T06:18:32.121 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.121 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.121 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.121 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.706210+0000", 2023-10-09T06:18:32.121 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8709999999999999e-06 2023-10-09T06:18:32.121 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.121 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.121 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.121 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.831108+0000", 2023-10-09T06:18:32.122 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.124898585 2023-10-09T06:18:32.122 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.122 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.122 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:32.122 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.831152+0000", 2023-10-09T06:18:32.122 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3028000000000002e-05 2023-10-09T06:18:32.122 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.122 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:32.122 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.122 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.122 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.123 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:95169 10.d 10:b585993a:::benchmark_data_smithi064_573750_object95168:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-10-09T06:18:32.123 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:20.533138+0000", 2023-10-09T06:18:32.123 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 311.55148555900001, 2023-10-09T06:18:32.123 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2980775480000002, 2023-10-09T06:18:32.123 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:32.123 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:18:32.123 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:32.123 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:18:32.123 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:32.123 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 95169 2023-10-09T06:18:32.123 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.123 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:32.124 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.124 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:32.124 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.533138+0000", 2023-10-09T06:18:32.124 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.124 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.124 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.124 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:32.124 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.533138+0000", 2023-10-09T06:18:32.124 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.124 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.124 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.124 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:32.125 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.533137+0000", 2023-10-09T06:18:32.125 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:32.125 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.125 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.125 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:32.125 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.533186+0000", 2023-10-09T06:18:32.125 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8749999999999999e-05 2023-10-09T06:18:32.125 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.125 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.125 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:32.125 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.533194+0000", 2023-10-09T06:18:32.126 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4880000000000001e-06 2023-10-09T06:18:32.126 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.126 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.126 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:32.126 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.533198+0000", 2023-10-09T06:18:32.126 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1860000000000002e-06 2023-10-09T06:18:32.126 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.126 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.126 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.126 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.533299+0000", 2023-10-09T06:18:32.126 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000101405 2023-10-09T06:18:32.127 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.127 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.127 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:32.127 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.533405+0000", 2023-10-09T06:18:32.127 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000105908 2023-10-09T06:18:32.127 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.127 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.127 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:18:32.127 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.533538+0000", 2023-10-09T06:18:32.127 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013334600000000001 2023-10-09T06:18:32.127 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.127 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.128 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:32.128 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.536181+0000", 2023-10-09T06:18:32.128 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0026423530000000001 2023-10-09T06:18:32.128 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.128 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.128 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.128 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.536314+0000", 2023-10-09T06:18:32.128 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013354 2023-10-09T06:18:32.128 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.128 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.128 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.129 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.711380+0000", 2023-10-09T06:18:32.129 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.17506608100000001 2023-10-09T06:18:32.129 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.129 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.129 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.129 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.711388+0000", 2023-10-09T06:18:32.129 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6890000000000004e-06 2023-10-09T06:18:32.129 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.129 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.129 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.129 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.706239+0000", 2023-10-09T06:18:32.129 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99485103799999997 2023-10-09T06:18:32.130 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.130 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.130 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.130 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.706246+0000", 2023-10-09T06:18:32.130 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6959999999999999e-06 2023-10-09T06:18:32.130 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.130 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.130 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.130 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.831173+0000", 2023-10-09T06:18:32.130 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.124926898 2023-10-09T06:18:32.130 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.131 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.131 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:32.131 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.831215+0000", 2023-10-09T06:18:32.131 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2629000000000001e-05 2023-10-09T06:18:32.131 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.131 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:32.131 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.131 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.131 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.131 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:95177 10.d 10:bcc2ceed:::benchmark_data_smithi064_573750_object95176:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e521)", 2023-10-09T06:18:32.131 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:20.795717+0000", 2023-10-09T06:18:32.131 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 311.288906908, 2023-10-09T06:18:32.132 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.050342831, 2023-10-09T06:18:32.132 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:32.132 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:32.132 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:32.132 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:18:32.132 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:32.132 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 95177 2023-10-09T06:18:32.132 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.132 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:32.132 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.132 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:32.132 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.795717+0000", 2023-10-09T06:18:32.132 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.133 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.133 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.133 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:32.133 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.795717+0000", 2023-10-09T06:18:32.133 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.133 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.133 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.133 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:32.133 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.795715+0000", 2023-10-09T06:18:32.133 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:32.133 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.133 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.134 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:32.134 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.795769+0000", 2023-10-09T06:18:32.134 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3454e-05 2023-10-09T06:18:32.134 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.134 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.134 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:32.134 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.795778+0000", 2023-10-09T06:18:32.134 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.5650000000000007e-06 2023-10-09T06:18:32.134 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.134 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.134 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:32.135 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.795785+0000", 2023-10-09T06:18:32.135 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3380000000000001e-06 2023-10-09T06:18:32.135 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.135 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.135 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.135 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.795855+0000", 2023-10-09T06:18:32.135 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0159000000000004e-05 2023-10-09T06:18:32.135 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.135 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.135 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.135 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.795862+0000", 2023-10-09T06:18:32.135 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1409999999999999e-06 2023-10-09T06:18:32.136 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.136 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.136 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.136 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.706269+0000", 2023-10-09T06:18:32.136 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.91040671500000003 2023-10-09T06:18:32.136 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.136 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.136 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.136 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.706275+0000", 2023-10-09T06:18:32.136 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6220000000000003e-06 2023-10-09T06:18:32.136 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.136 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.137 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.137 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.831232+0000", 2023-10-09T06:18:32.137 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1249570550000001 2023-10-09T06:18:32.137 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.137 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.137 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:32.137 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.831347+0000", 2023-10-09T06:18:32.137 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011471799999999999 2023-10-09T06:18:32.137 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.137 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.137 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T06:18:32.138 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.831506+0000", 2023-10-09T06:18:32.138 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015921500000000001 2023-10-09T06:18:32.138 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.138 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.138 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.138 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.837687+0000", 2023-10-09T06:18:32.138 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0061805480000000001 2023-10-09T06:18:32.138 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.138 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.138 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.138 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.844625+0000", 2023-10-09T06:18:32.138 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0069379949999999998 2023-10-09T06:18:32.139 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.139 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.139 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:32.139 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.844653+0000", 2023-10-09T06:18:32.139 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.8000999999999998e-05 2023-10-09T06:18:32.139 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.139 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.139 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.139 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.845869+0000", 2023-10-09T06:18:32.139 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001216808 2023-10-09T06:18:32.139 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.139 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.140 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:32.140 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.845925+0000", 2023-10-09T06:18:32.140 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5971999999999999e-05 2023-10-09T06:18:32.140 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.140 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.140 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:32.140 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.846059+0000", 2023-10-09T06:18:32.140 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013388500000000001 2023-10-09T06:18:32.140 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.140 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:32.140 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.140 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.141 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.141 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:95183 10.d 10:baf23edc:::benchmark_data_smithi064_573750_object95182:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e521)", 2023-10-09T06:18:32.141 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:20.803711+0000", 2023-10-09T06:18:32.141 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 311.28091224399998, 2023-10-09T06:18:32.141 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0426046769999999, 2023-10-09T06:18:32.141 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:32.141 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:32.141 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:32.141 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:18:32.141 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:32.141 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 95183 2023-10-09T06:18:32.141 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.142 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:32.142 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.142 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:32.142 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.803711+0000", 2023-10-09T06:18:32.142 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.142 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.142 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.142 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:32.142 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.803711+0000", 2023-10-09T06:18:32.142 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.142 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.142 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.143 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:32.143 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.803710+0000", 2023-10-09T06:18:32.143 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:32.143 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.143 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.143 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:32.143 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.803818+0000", 2023-10-09T06:18:32.143 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000108274 2023-10-09T06:18:32.143 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.143 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.143 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:32.143 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.803836+0000", 2023-10-09T06:18:32.144 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8085999999999999e-05 2023-10-09T06:18:32.144 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.144 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.144 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:32.144 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.803842+0000", 2023-10-09T06:18:32.144 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3759999999999999e-06 2023-10-09T06:18:32.144 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.144 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.144 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.144 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.804153+0000", 2023-10-09T06:18:32.144 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00031180800000000001 2023-10-09T06:18:32.144 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.145 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.145 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.145 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.804163+0000", 2023-10-09T06:18:32.145 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7079999999999992e-06 2023-10-09T06:18:32.145 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.145 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.145 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.145 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.706371+0000", 2023-10-09T06:18:32.145 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.90220791199999995 2023-10-09T06:18:32.145 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.145 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.145 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.146 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.706378+0000", 2023-10-09T06:18:32.146 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4399999999999999e-06 2023-10-09T06:18:32.146 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.146 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.146 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.146 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.834223+0000", 2023-10-09T06:18:32.146 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.12784415 2023-10-09T06:18:32.146 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.146 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.146 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:32.146 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.834343+0000", 2023-10-09T06:18:32.147 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000119918 2023-10-09T06:18:32.147 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.147 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.147 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T06:18:32.147 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.834495+0000", 2023-10-09T06:18:32.147 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015243099999999999 2023-10-09T06:18:32.147 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.147 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.147 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:32.147 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.844659+0000", 2023-10-09T06:18:32.147 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010163693 2023-10-09T06:18:32.147 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.148 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.148 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.148 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.845481+0000", 2023-10-09T06:18:32.148 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00082213700000000002 2023-10-09T06:18:32.148 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.148 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.148 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.148 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.846101+0000", 2023-10-09T06:18:32.148 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00062066300000000003 2023-10-09T06:18:32.148 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.148 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.148 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.148 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.846167+0000", 2023-10-09T06:18:32.149 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5611000000000006e-05 2023-10-09T06:18:32.149 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.149 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.149 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:32.149 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.846211+0000", 2023-10-09T06:18:32.149 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4091000000000002e-05 2023-10-09T06:18:32.149 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.149 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.149 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:32.149 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.846316+0000", 2023-10-09T06:18:32.149 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000104678 2023-10-09T06:18:32.149 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.150 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:32.150 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.150 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.150 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.150 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:95193 10.e 10:7b249de2:::benchmark_data_smithi064_573750_object95192:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e521)", 2023-10-09T06:18:32.150 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:20.813524+0000", 2023-10-09T06:18:32.150 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 311.27109972, 2023-10-09T06:18:32.150 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0199568299999999, 2023-10-09T06:18:32.150 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:32.150 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:32.150 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:32.151 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:18:32.151 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:32.151 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 95193 2023-10-09T06:18:32.151 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.151 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:32.151 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.151 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:32.151 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.813524+0000", 2023-10-09T06:18:32.151 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.151 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.151 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.152 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:32.152 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.813524+0000", 2023-10-09T06:18:32.152 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.152 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.152 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.152 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:32.152 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.813523+0000", 2023-10-09T06:18:32.152 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:32.152 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.152 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.152 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:32.152 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.813586+0000", 2023-10-09T06:18:32.153 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2964000000000002e-05 2023-10-09T06:18:32.153 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.153 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.153 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:32.153 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.813594+0000", 2023-10-09T06:18:32.153 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.559e-06 2023-10-09T06:18:32.153 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.153 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.153 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:32.153 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.813598+0000", 2023-10-09T06:18:32.153 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7870000000000001e-06 2023-10-09T06:18:32.153 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.153 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.154 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.154 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.813652+0000", 2023-10-09T06:18:32.154 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3785000000000001e-05 2023-10-09T06:18:32.154 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.154 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.154 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.154 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.813661+0000", 2023-10-09T06:18:32.154 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2900000000000008e-06 2023-10-09T06:18:32.154 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.154 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.154 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.155 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.711452+0000", 2023-10-09T06:18:32.155 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.89779104499999995 2023-10-09T06:18:32.155 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.155 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.155 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.155 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.711465+0000", 2023-10-09T06:18:32.155 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2836e-05 2023-10-09T06:18:32.155 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.155 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.155 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.155 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.819147+0000", 2023-10-09T06:18:32.155 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1076818529999999 2023-10-09T06:18:32.156 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.156 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.156 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:32.156 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.819276+0000", 2023-10-09T06:18:32.156 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012921800000000001 2023-10-09T06:18:32.156 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.156 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.156 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:18:32.156 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.819444+0000", 2023-10-09T06:18:32.156 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000167467 2023-10-09T06:18:32.156 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.156 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.157 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.157 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.822863+0000", 2023-10-09T06:18:32.157 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0034189469999999999 2023-10-09T06:18:32.157 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.157 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.157 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:32.157 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.832250+0000", 2023-10-09T06:18:32.157 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0093869650000000006 2023-10-09T06:18:32.157 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.157 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.157 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.157 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.833224+0000", 2023-10-09T06:18:32.158 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00097416699999999996 2023-10-09T06:18:32.158 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.158 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.158 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.158 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.833304+0000", 2023-10-09T06:18:32.158 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0150999999999999e-05 2023-10-09T06:18:32.158 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.158 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.158 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:32.158 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.833360+0000", 2023-10-09T06:18:32.158 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5630000000000001e-05 2023-10-09T06:18:32.158 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.159 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.159 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:32.159 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.833481+0000", 2023-10-09T06:18:32.159 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000120816 2023-10-09T06:18:32.159 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.159 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:32.159 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.159 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.159 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.159 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:95195 10.e 10:7171b345:::benchmark_data_smithi064_573750_object95194:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e521)", 2023-10-09T06:18:32.159 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:20.817270+0000", 2023-10-09T06:18:32.160 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 311.26735345100002, 2023-10-09T06:18:32.160 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0173483069999998, 2023-10-09T06:18:32.160 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:32.160 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:32.160 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:32.160 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:18:32.160 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:32.160 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 95195 2023-10-09T06:18:32.160 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.160 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:32.160 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.160 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:32.161 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.817270+0000", 2023-10-09T06:18:32.161 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.161 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.161 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.161 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:32.161 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.817270+0000", 2023-10-09T06:18:32.161 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.161 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.161 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.161 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:32.161 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.817269+0000", 2023-10-09T06:18:32.161 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:18:32.161 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.162 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.162 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:32.162 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.817331+0000", 2023-10-09T06:18:32.162 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2089999999999997e-05 2023-10-09T06:18:32.162 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.162 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.162 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:32.162 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.817345+0000", 2023-10-09T06:18:32.162 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3637e-05 2023-10-09T06:18:32.162 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.162 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.163 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:32.163 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.817351+0000", 2023-10-09T06:18:32.163 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3879999999999997e-06 2023-10-09T06:18:32.163 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.163 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.163 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.163 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.817417+0000", 2023-10-09T06:18:32.163 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6390999999999995e-05 2023-10-09T06:18:32.163 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.163 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.163 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.163 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.817431+0000", 2023-10-09T06:18:32.164 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3669000000000001e-05 2023-10-09T06:18:32.164 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.164 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.164 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.164 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.711479+0000", 2023-10-09T06:18:32.164 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.89404781099999997 2023-10-09T06:18:32.164 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.164 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.164 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.164 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.711483+0000", 2023-10-09T06:18:32.164 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0339999999999998e-06 2023-10-09T06:18:32.164 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.165 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.165 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.165 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.822898+0000", 2023-10-09T06:18:32.165 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1114156399999999 2023-10-09T06:18:32.165 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.165 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.165 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:32.165 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.823064+0000", 2023-10-09T06:18:32.165 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00016534300000000001 2023-10-09T06:18:32.165 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.165 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.165 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:18:32.165 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.823258+0000", 2023-10-09T06:18:32.166 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000193838 2023-10-09T06:18:32.166 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.166 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.166 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.166 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.829210+0000", 2023-10-09T06:18:32.166 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0059525369999999999 2023-10-09T06:18:32.166 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.166 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.166 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:32.166 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.832254+0000", 2023-10-09T06:18:32.166 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0030442149999999999 2023-10-09T06:18:32.167 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.167 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.167 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.167 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.833254+0000", 2023-10-09T06:18:32.167 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00099923599999999992 2023-10-09T06:18:32.167 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.167 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.167 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.167 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.834458+0000", 2023-10-09T06:18:32.167 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0012047939999999999 2023-10-09T06:18:32.167 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.167 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.168 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:32.168 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.834511+0000", 2023-10-09T06:18:32.168 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2994000000000001e-05 2023-10-09T06:18:32.168 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.168 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.168 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:32.168 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.834618+0000", 2023-10-09T06:18:32.168 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010686799999999999 2023-10-09T06:18:32.168 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.168 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:32.168 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.168 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.169 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.169 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:95207 10.e 10:72955c88:::benchmark_data_smithi064_573750_object95206:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e521)", 2023-10-09T06:18:32.169 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:20.937758+0000", 2023-10-09T06:18:32.169 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 311.146865177, 2023-10-09T06:18:32.169 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.90852591, 2023-10-09T06:18:32.169 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:32.169 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:32.169 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:32.169 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:18:32.169 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:32.169 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 95207 2023-10-09T06:18:32.169 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.170 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:32.170 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.170 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:32.170 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.937758+0000", 2023-10-09T06:18:32.170 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.170 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.170 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.170 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:32.170 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.937758+0000", 2023-10-09T06:18:32.170 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.170 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.171 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.171 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:32.171 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.937757+0000", 2023-10-09T06:18:32.171 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:32.171 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.171 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.171 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:32.171 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.937794+0000", 2023-10-09T06:18:32.171 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7237999999999998e-05 2023-10-09T06:18:32.171 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.171 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.171 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:32.172 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.937809+0000", 2023-10-09T06:18:32.172 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4823e-05 2023-10-09T06:18:32.172 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.172 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.172 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:32.172 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.937816+0000", 2023-10-09T06:18:32.172 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.759e-06 2023-10-09T06:18:32.172 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.172 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.172 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.172 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.937875+0000", 2023-10-09T06:18:32.172 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9568999999999998e-05 2023-10-09T06:18:32.172 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.173 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.173 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.173 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.937886+0000", 2023-10-09T06:18:32.173 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0387999999999999e-05 2023-10-09T06:18:32.173 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.173 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.173 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.173 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.711493+0000", 2023-10-09T06:18:32.173 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.77360711900000001 2023-10-09T06:18:32.173 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.173 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.173 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.174 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.711496+0000", 2023-10-09T06:18:32.174 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6210000000000001e-06 2023-10-09T06:18:32.174 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.174 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.174 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.174 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.825988+0000", 2023-10-09T06:18:32.174 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1144915289999999 2023-10-09T06:18:32.174 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.174 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.174 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:32.174 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.826139+0000", 2023-10-09T06:18:32.175 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015136500000000001 2023-10-09T06:18:32.175 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.175 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.175 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:18:32.175 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.826297+0000", 2023-10-09T06:18:32.175 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015811199999999999 2023-10-09T06:18:32.175 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.175 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.175 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:32.175 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.832258+0000", 2023-10-09T06:18:32.175 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.005960641 2023-10-09T06:18:32.175 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.176 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.176 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.176 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.845122+0000", 2023-10-09T06:18:32.176 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.01286373 2023-10-09T06:18:32.176 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.176 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.176 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.176 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.845151+0000", 2023-10-09T06:18:32.176 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9362000000000001e-05 2023-10-09T06:18:32.176 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.176 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.176 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.177 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.846123+0000", 2023-10-09T06:18:32.177 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00097178499999999997 2023-10-09T06:18:32.177 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.177 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.177 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:32.177 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.846173+0000", 2023-10-09T06:18:32.177 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0454000000000002e-05 2023-10-09T06:18:32.177 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.177 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.177 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:32.177 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.846284+0000", 2023-10-09T06:18:32.177 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000110895 2023-10-09T06:18:32.178 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.178 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:32.178 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.178 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.178 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.178 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:171686 10.e 10:7b905e5f:::benchmark_data_smithi064_573750_object75471:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:18:32.178 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:15.818949+0000", 2023-10-09T06:18:32.178 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 196.265674193, 2023-10-09T06:18:32.178 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1270911259999998, 2023-10-09T06:18:32.178 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:32.178 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:32.178 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:32.179 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:18:32.179 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:32.179 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 171686 2023-10-09T06:18:32.179 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.179 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:32.179 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.179 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:32.179 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.818949+0000", 2023-10-09T06:18:32.179 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.179 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.179 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.179 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:32.180 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.818949+0000", 2023-10-09T06:18:32.180 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.180 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.180 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.180 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:32.180 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.818947+0000", 2023-10-09T06:18:32.180 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999976 2023-10-09T06:18:32.180 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.180 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.180 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:32.180 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.818955+0000", 2023-10-09T06:18:32.180 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.191e-06 2023-10-09T06:18:32.181 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.181 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.181 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:32.181 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.818973+0000", 2023-10-09T06:18:32.181 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7476999999999999e-05 2023-10-09T06:18:32.181 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.181 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.181 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:32.181 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.818979+0000", 2023-10-09T06:18:32.181 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5810000000000002e-06 2023-10-09T06:18:32.181 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.181 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.182 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.182 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.819044+0000", 2023-10-09T06:18:32.182 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4527000000000001e-05 2023-10-09T06:18:32.182 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.182 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.182 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.182 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.819053+0000", 2023-10-09T06:18:32.182 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1670000000000005e-06 2023-10-09T06:18:32.182 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.182 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.182 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.182 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.807971+0000", 2023-10-09T06:18:32.183 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.98891850400000003 2023-10-09T06:18:32.183 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.183 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.183 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.183 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.807984+0000", 2023-10-09T06:18:32.183 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3021e-05 2023-10-09T06:18:32.183 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.183 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.183 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.183 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.937604+0000", 2023-10-09T06:18:32.183 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1296198289999999 2023-10-09T06:18:32.183 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.184 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.184 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:32.184 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.937709+0000", 2023-10-09T06:18:32.184 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000104522 2023-10-09T06:18:32.184 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.184 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.184 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:18:32.184 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.937809+0000", 2023-10-09T06:18:32.184 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010076700000000001 2023-10-09T06:18:32.184 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.184 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.184 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.185 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.940904+0000", 2023-10-09T06:18:32.185 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0030950190000000001 2023-10-09T06:18:32.185 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.185 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.185 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:32.185 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.940912+0000", 2023-10-09T06:18:32.185 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8650000000000001e-06 2023-10-09T06:18:32.185 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.185 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.185 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.185 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.940952+0000", 2023-10-09T06:18:32.185 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9570999999999998e-05 2023-10-09T06:18:32.186 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.186 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.186 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.186 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.945895+0000", 2023-10-09T06:18:32.186 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0049432770000000003 2023-10-09T06:18:32.186 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.244 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.244 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:32.245 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.945945+0000", 2023-10-09T06:18:32.245 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9401999999999999e-05 2023-10-09T06:18:32.245 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.245 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.245 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:32.245 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946040+0000", 2023-10-09T06:18:32.245 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.5735000000000004e-05 2023-10-09T06:18:32.245 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.245 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:32.245 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.245 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.245 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.246 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:171688 10.e 10:735f8ea9:::benchmark_data_smithi064_573750_object75473:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:18:32.246 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:15.825750+0000", 2023-10-09T06:18:32.246 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 196.258873115, 2023-10-09T06:18:32.246 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1206339970000001, 2023-10-09T06:18:32.246 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:32.246 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:32.246 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:32.246 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:18:32.246 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:32.246 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 171688 2023-10-09T06:18:32.246 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.247 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:32.247 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.247 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:32.247 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825750+0000", 2023-10-09T06:18:32.247 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.247 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.247 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.247 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:32.247 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825750+0000", 2023-10-09T06:18:32.247 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.247 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.247 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.248 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:32.248 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825749+0000", 2023-10-09T06:18:32.248 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:18:32.248 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.248 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.248 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:32.248 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825755+0000", 2023-10-09T06:18:32.248 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5539999999999997e-06 2023-10-09T06:18:32.248 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.248 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.248 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:32.248 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825767+0000", 2023-10-09T06:18:32.249 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2213e-05 2023-10-09T06:18:32.249 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.249 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.249 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:32.249 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825772+0000", 2023-10-09T06:18:32.249 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2499999999999997e-06 2023-10-09T06:18:32.249 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.249 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.249 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.249 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825826+0000", 2023-10-09T06:18:32.249 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3579999999999999e-05 2023-10-09T06:18:32.249 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.249 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.250 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.250 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825837+0000", 2023-10-09T06:18:32.250 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1008e-05 2023-10-09T06:18:32.250 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.250 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.250 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.250 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808012+0000", 2023-10-09T06:18:32.250 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.98217509800000002 2023-10-09T06:18:32.250 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.250 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.250 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.250 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808019+0000", 2023-10-09T06:18:32.251 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8419999999999999e-06 2023-10-09T06:18:32.251 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.251 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.251 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.251 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.940977+0000", 2023-10-09T06:18:32.251 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1329584779999999 2023-10-09T06:18:32.251 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.251 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.251 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:32.251 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.941068+0000", 2023-10-09T06:18:32.251 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0706999999999997e-05 2023-10-09T06:18:32.251 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.252 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.252 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:18:32.252 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.941149+0000", 2023-10-09T06:18:32.252 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1558999999999996e-05 2023-10-09T06:18:32.252 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.252 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.252 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.252 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946099+0000", 2023-10-09T06:18:32.252 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0049497769999999998 2023-10-09T06:18:32.252 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.252 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.252 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.253 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946121+0000", 2023-10-09T06:18:32.253 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2038000000000001e-05 2023-10-09T06:18:32.253 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.253 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.253 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:32.253 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946127+0000", 2023-10-09T06:18:32.253 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1120000000000001e-06 2023-10-09T06:18:32.253 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.253 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.253 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.253 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946295+0000", 2023-10-09T06:18:32.253 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00016754399999999999 2023-10-09T06:18:32.254 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.254 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.254 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:32.254 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946327+0000", 2023-10-09T06:18:32.254 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.2239000000000001e-05 2023-10-09T06:18:32.254 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.254 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.254 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:32.254 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946384+0000", 2023-10-09T06:18:32.254 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7123000000000002e-05 2023-10-09T06:18:32.254 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.254 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:32.255 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.255 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.255 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.255 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:171689 10.e 10:7e719361:::benchmark_data_smithi064_573750_object75474:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:18:32.255 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:15.825780+0000", 2023-10-09T06:18:32.255 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 196.258843254, 2023-10-09T06:18:32.255 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1234250769999998, 2023-10-09T06:18:32.255 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:32.255 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:32.255 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:32.255 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:18:32.255 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:32.256 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 171689 2023-10-09T06:18:32.256 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.256 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:32.256 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.256 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:32.256 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825780+0000", 2023-10-09T06:18:32.256 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.256 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.256 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.256 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:32.256 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825780+0000", 2023-10-09T06:18:32.256 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.257 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.257 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.257 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:32.257 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825780+0000", 2023-10-09T06:18:32.257 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:32.257 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.257 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.257 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:32.257 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825783+0000", 2023-10-09T06:18:32.257 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.8200000000000001e-06 2023-10-09T06:18:32.257 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.257 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.258 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:32.258 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825788+0000", 2023-10-09T06:18:32.258 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1989999999999997e-06 2023-10-09T06:18:32.258 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.258 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.258 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:32.258 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825790+0000", 2023-10-09T06:18:32.258 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.5799999999999999e-06 2023-10-09T06:18:32.258 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.258 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.258 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.258 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825860+0000", 2023-10-09T06:18:32.259 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9343000000000003e-05 2023-10-09T06:18:32.259 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.259 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.259 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.259 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825867+0000", 2023-10-09T06:18:32.259 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4259999999999997e-06 2023-10-09T06:18:32.259 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.259 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.259 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.259 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808061+0000", 2023-10-09T06:18:32.259 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.98219415399999999 2023-10-09T06:18:32.259 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.260 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.260 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.260 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808068+0000", 2023-10-09T06:18:32.260 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7649999999999996e-06 2023-10-09T06:18:32.260 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.260 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.260 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.260 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.943145+0000", 2023-10-09T06:18:32.260 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1350769110000001 2023-10-09T06:18:32.260 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.260 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.260 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:32.261 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.943234+0000", 2023-10-09T06:18:32.261 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8616999999999995e-05 2023-10-09T06:18:32.261 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.261 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.261 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:18:32.261 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.943309+0000", 2023-10-09T06:18:32.261 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5770000000000001e-05 2023-10-09T06:18:32.261 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.261 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.261 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:32.261 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946132+0000", 2023-10-09T06:18:32.261 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0028229879999999998 2023-10-09T06:18:32.262 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.262 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.262 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.262 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946158+0000", 2023-10-09T06:18:32.262 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.5457999999999999e-05 2023-10-09T06:18:32.262 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.262 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.262 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.262 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946263+0000", 2023-10-09T06:18:32.262 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000104824 2023-10-09T06:18:32.262 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.262 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.263 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.263 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.949092+0000", 2023-10-09T06:18:32.263 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0028290469999999999 2023-10-09T06:18:32.263 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.263 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.263 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:32.263 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.949126+0000", 2023-10-09T06:18:32.263 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.4226000000000001e-05 2023-10-09T06:18:32.263 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.263 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.263 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:32.263 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.949205+0000", 2023-10-09T06:18:32.264 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9340999999999998e-05 2023-10-09T06:18:32.264 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.264 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:32.264 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.264 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.264 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.264 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:171708 10.e 10:7c12aa3b:::benchmark_data_smithi064_573750_object75493:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:18:32.264 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:15.851084+0000", 2023-10-09T06:18:32.264 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 196.233539466, 2023-10-09T06:18:32.264 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.109213767, 2023-10-09T06:18:32.264 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:32.264 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:32.265 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:32.265 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:18:32.265 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:32.265 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 171708 2023-10-09T06:18:32.265 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.265 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:32.265 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.265 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:32.265 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.851084+0000", 2023-10-09T06:18:32.265 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.265 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.265 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.266 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:32.266 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.851084+0000", 2023-10-09T06:18:32.266 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.266 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.266 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.266 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:32.266 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.851082+0000", 2023-10-09T06:18:32.266 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:32.266 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.266 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.266 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:32.266 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.851089+0000", 2023-10-09T06:18:32.267 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.144e-06 2023-10-09T06:18:32.267 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.267 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.267 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:32.267 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.851099+0000", 2023-10-09T06:18:32.267 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0579e-05 2023-10-09T06:18:32.267 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.267 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.267 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:32.267 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.851106+0000", 2023-10-09T06:18:32.267 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7309999999999996e-06 2023-10-09T06:18:32.267 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.268 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.268 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.268 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.851155+0000", 2023-10-09T06:18:32.268 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8820999999999999e-05 2023-10-09T06:18:32.268 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.268 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.268 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.268 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.851163+0000", 2023-10-09T06:18:32.268 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1799999999999996e-06 2023-10-09T06:18:32.268 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.268 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.268 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.269 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808090+0000", 2023-10-09T06:18:32.269 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.95692664800000005 2023-10-09T06:18:32.269 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.269 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.269 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.269 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808096+0000", 2023-10-09T06:18:32.269 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5899999999999996e-06 2023-10-09T06:18:32.269 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.269 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.269 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.269 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946407+0000", 2023-10-09T06:18:32.269 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.138310487 2023-10-09T06:18:32.270 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.270 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.270 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:32.270 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946493+0000", 2023-10-09T06:18:32.270 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6648000000000001e-05 2023-10-09T06:18:32.270 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.270 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.270 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:18:32.270 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946574+0000", 2023-10-09T06:18:32.270 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0939000000000006e-05 2023-10-09T06:18:32.270 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.271 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.271 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.271 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.949254+0000", 2023-10-09T06:18:32.271 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0026798500000000001 2023-10-09T06:18:32.271 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.271 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.271 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.271 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.949278+0000", 2023-10-09T06:18:32.271 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4196e-05 2023-10-09T06:18:32.271 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.271 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.271 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:32.271 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.949284+0000", 2023-10-09T06:18:32.272 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.835e-06 2023-10-09T06:18:32.272 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.272 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.272 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.272 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.960139+0000", 2023-10-09T06:18:32.272 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010854508000000001 2023-10-09T06:18:32.272 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.272 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.272 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:32.272 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.960183+0000", 2023-10-09T06:18:32.272 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4882000000000002e-05 2023-10-09T06:18:32.272 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.273 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.273 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:32.273 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.960298+0000", 2023-10-09T06:18:32.273 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000114261 2023-10-09T06:18:32.273 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.273 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:32.273 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.273 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.273 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.273 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:171724 10.e 10:74fab5e2:::benchmark_data_smithi064_573750_object75509:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:18:32.273 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:15.873407+0000", 2023-10-09T06:18:32.273 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 196.21121647800001, 2023-10-09T06:18:32.274 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0927997860000001, 2023-10-09T06:18:32.274 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:32.274 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:32.274 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:32.274 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:18:32.274 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:32.274 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 171724 2023-10-09T06:18:32.274 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.274 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:32.274 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.274 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:32.274 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.873407+0000", 2023-10-09T06:18:32.275 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.275 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.275 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.275 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:32.275 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.873407+0000", 2023-10-09T06:18:32.275 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.275 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.275 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.275 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:32.275 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.873406+0000", 2023-10-09T06:18:32.275 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:32.275 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.276 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.276 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:32.276 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.873413+0000", 2023-10-09T06:18:32.276 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.418e-06 2023-10-09T06:18:32.276 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.276 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.276 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:32.276 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.873427+0000", 2023-10-09T06:18:32.276 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4362000000000001e-05 2023-10-09T06:18:32.276 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.276 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.276 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:32.277 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.873434+0000", 2023-10-09T06:18:32.277 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9419999999999998e-06 2023-10-09T06:18:32.277 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.277 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.277 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.277 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.873487+0000", 2023-10-09T06:18:32.277 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2324000000000003e-05 2023-10-09T06:18:32.277 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.277 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.277 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.277 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.873493+0000", 2023-10-09T06:18:32.278 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3740000000000003e-06 2023-10-09T06:18:32.278 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.278 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.278 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.278 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808117+0000", 2023-10-09T06:18:32.278 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.93462358599999995 2023-10-09T06:18:32.278 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.278 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.278 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.278 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808123+0000", 2023-10-09T06:18:32.278 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1809999999999998e-06 2023-10-09T06:18:32.279 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.279 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.279 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.279 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.949312+0000", 2023-10-09T06:18:32.279 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.141189486 2023-10-09T06:18:32.279 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.279 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.279 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:32.279 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.949425+0000", 2023-10-09T06:18:32.279 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000112353 2023-10-09T06:18:32.279 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.279 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.280 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:18:32.280 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.949554+0000", 2023-10-09T06:18:32.280 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012921800000000001 2023-10-09T06:18:32.280 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.280 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.280 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.280 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.960103+0000", 2023-10-09T06:18:32.280 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010548963 2023-10-09T06:18:32.280 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.280 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.280 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.280 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.960335+0000", 2023-10-09T06:18:32.281 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000232432 2023-10-09T06:18:32.281 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.281 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.281 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.281 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.960365+0000", 2023-10-09T06:18:32.281 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9901000000000001e-05 2023-10-09T06:18:32.281 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.281 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.281 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:32.281 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.966031+0000", 2023-10-09T06:18:32.281 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0056662750000000001 2023-10-09T06:18:32.281 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.281 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.282 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:32.282 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.966084+0000", 2023-10-09T06:18:32.282 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2261e-05 2023-10-09T06:18:32.282 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.282 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.282 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:32.282 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.966207+0000", 2023-10-09T06:18:32.282 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012310700000000001 2023-10-09T06:18:32.282 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.282 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:32.282 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.283 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.283 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.283 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:171731 10.e 10:73794b97:::benchmark_data_smithi064_573750_object75516:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:18:32.283 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:15.882331+0000", 2023-10-09T06:18:32.283 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 196.20229211399999, 2023-10-09T06:18:32.283 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.094724646, 2023-10-09T06:18:32.283 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:32.283 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:32.283 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:32.283 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:18:32.283 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:32.283 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 171731 2023-10-09T06:18:32.284 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.284 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:32.284 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.284 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:32.284 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.882331+0000", 2023-10-09T06:18:32.284 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.284 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.284 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.284 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:32.284 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.882331+0000", 2023-10-09T06:18:32.284 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.284 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.285 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.285 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:32.285 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.882330+0000", 2023-10-09T06:18:32.285 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T06:18:32.285 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.285 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.285 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:32.285 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.882340+0000", 2023-10-09T06:18:32.285 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0457e-05 2023-10-09T06:18:32.285 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.285 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.285 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:32.286 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.882363+0000", 2023-10-09T06:18:32.286 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.281e-05 2023-10-09T06:18:32.286 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.286 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.286 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:32.286 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.882371+0000", 2023-10-09T06:18:32.286 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0679999999999997e-06 2023-10-09T06:18:32.286 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.286 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.286 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.286 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.882441+0000", 2023-10-09T06:18:32.286 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0616000000000003e-05 2023-10-09T06:18:32.287 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.287 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.287 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.287 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.882454+0000", 2023-10-09T06:18:32.287 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2328e-05 2023-10-09T06:18:32.287 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.287 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.287 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.287 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808143+0000", 2023-10-09T06:18:32.287 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.92568909300000002 2023-10-09T06:18:32.287 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.287 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.288 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.288 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808149+0000", 2023-10-09T06:18:32.288 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3210000000000001e-06 2023-10-09T06:18:32.288 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.288 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.288 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.288 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.952065+0000", 2023-10-09T06:18:32.288 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1439156129999999 2023-10-09T06:18:32.288 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.288 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.288 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:32.289 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.952142+0000", 2023-10-09T06:18:32.289 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7379000000000006e-05 2023-10-09T06:18:32.289 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.289 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.289 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:18:32.289 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.952248+0000", 2023-10-09T06:18:32.289 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010558800000000001 2023-10-09T06:18:32.290 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.290 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.290 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.291 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.960396+0000", 2023-10-09T06:18:32.291 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0081483519999999993 2023-10-09T06:18:32.291 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.291 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.292 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.292 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.960429+0000", 2023-10-09T06:18:32.292 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.2342999999999998e-05 2023-10-09T06:18:32.293 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.293 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.293 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.293 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.966264+0000", 2023-10-09T06:18:32.294 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0058356329999999998 2023-10-09T06:18:32.294 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.294 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.295 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:32.295 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.976865+0000", 2023-10-09T06:18:32.296 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010601169000000001 2023-10-09T06:18:32.296 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.296 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.297 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:32.297 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.976922+0000", 2023-10-09T06:18:32.297 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6378999999999997e-05 2023-10-09T06:18:32.297 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.298 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.298 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:32.298 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977056+0000", 2023-10-09T06:18:32.298 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013425500000000001 2023-10-09T06:18:32.299 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.299 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:32.299 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.299 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.300 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.300 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:171732 10.e 10:72905dd9:::benchmark_data_smithi064_573750_object75517:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:18:32.300 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:15.884530+0000", 2023-10-09T06:18:32.300 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 196.20009389000001, 2023-10-09T06:18:32.301 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.092684894, 2023-10-09T06:18:32.301 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:32.301 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:32.301 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:32.302 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:18:32.302 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:32.302 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 171732 2023-10-09T06:18:32.302 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.303 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:32.303 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.303 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:32.303 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.884530+0000", 2023-10-09T06:18:32.303 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.304 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.304 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.304 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:32.304 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.884530+0000", 2023-10-09T06:18:32.304 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.305 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.305 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.305 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:32.305 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.884529+0000", 2023-10-09T06:18:32.306 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:18:32.306 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.306 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.306 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:32.306 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.884538+0000", 2023-10-09T06:18:32.307 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.4760000000000004e-06 2023-10-09T06:18:32.307 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.307 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.307 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:32.308 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.884553+0000", 2023-10-09T06:18:32.308 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4623999999999999e-05 2023-10-09T06:18:32.308 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.308 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.308 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:32.308 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.884561+0000", 2023-10-09T06:18:32.309 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.2749999999999995e-06 2023-10-09T06:18:32.309 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.309 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.309 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.309 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.884625+0000", 2023-10-09T06:18:32.310 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4389999999999998e-05 2023-10-09T06:18:32.310 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.310 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.310 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.310 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.884634+0000", 2023-10-09T06:18:32.310 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0960000000000007e-06 2023-10-09T06:18:32.310 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.310 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.311 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.311 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808169+0000", 2023-10-09T06:18:32.311 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.92353482799999997 2023-10-09T06:18:32.311 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.311 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.311 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.311 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808176+0000", 2023-10-09T06:18:32.311 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3280000000000002e-06 2023-10-09T06:18:32.311 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.311 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.312 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.312 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.954504+0000", 2023-10-09T06:18:32.312 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.146328733 2023-10-09T06:18:32.312 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.312 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.312 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:32.312 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.954581+0000", 2023-10-09T06:18:32.312 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6789e-05 2023-10-09T06:18:32.312 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.312 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.312 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:18:32.313 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.954659+0000", 2023-10-09T06:18:32.313 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7904000000000005e-05 2023-10-09T06:18:32.313 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.313 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.313 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.313 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.966305+0000", 2023-10-09T06:18:32.313 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011645983 2023-10-09T06:18:32.313 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.313 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.313 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.313 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.966337+0000", 2023-10-09T06:18:32.314 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1948e-05 2023-10-09T06:18:32.314 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.314 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.314 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.314 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.966370+0000", 2023-10-09T06:18:32.314 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.3121999999999999e-05 2023-10-09T06:18:32.314 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.314 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.314 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:32.314 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977071+0000", 2023-10-09T06:18:32.314 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010701207000000001 2023-10-09T06:18:32.315 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.315 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.315 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:32.315 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977122+0000", 2023-10-09T06:18:32.315 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1038000000000002e-05 2023-10-09T06:18:32.315 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.315 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.315 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:32.315 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977214+0000", 2023-10-09T06:18:32.315 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2127999999999996e-05 2023-10-09T06:18:32.316 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.316 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:32.316 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.316 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.316 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.316 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:171742 10.e 10:72249b2b:::benchmark_data_smithi064_573750_object75527:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:18:32.316 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:15.902945+0000", 2023-10-09T06:18:32.316 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 196.181677923, 2023-10-09T06:18:32.316 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.074354617, 2023-10-09T06:18:32.316 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:32.317 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:32.317 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:32.317 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:18:32.317 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:32.317 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 171742 2023-10-09T06:18:32.317 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.317 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:32.317 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.317 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:32.318 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.902945+0000", 2023-10-09T06:18:32.318 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.318 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.318 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.318 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:32.318 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.902945+0000", 2023-10-09T06:18:32.318 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.318 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.318 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.319 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:32.319 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.902944+0000", 2023-10-09T06:18:32.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:18:32.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.319 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.319 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:32.319 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.902950+0000", 2023-10-09T06:18:32.319 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3120000000000001e-06 2023-10-09T06:18:32.319 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.320 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.320 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:32.320 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.902962+0000", 2023-10-09T06:18:32.320 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2581e-05 2023-10-09T06:18:32.320 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.320 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.320 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:32.320 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.902967+0000", 2023-10-09T06:18:32.320 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9620000000000001e-06 2023-10-09T06:18:32.321 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.321 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.321 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.321 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.903030+0000", 2023-10-09T06:18:32.321 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2533999999999995e-05 2023-10-09T06:18:32.321 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.321 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.321 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.321 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.903039+0000", 2023-10-09T06:18:32.321 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.7299999999999994e-06 2023-10-09T06:18:32.322 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.322 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.322 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.322 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808196+0000", 2023-10-09T06:18:32.322 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.90515772699999997 2023-10-09T06:18:32.322 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.322 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.322 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.322 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808203+0000", 2023-10-09T06:18:32.322 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2380000000000002e-06 2023-10-09T06:18:32.323 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.323 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.323 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.323 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.957318+0000", 2023-10-09T06:18:32.323 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.149115659 2023-10-09T06:18:32.323 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.323 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.323 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:32.323 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.957450+0000", 2023-10-09T06:18:32.323 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013214699999999999 2023-10-09T06:18:32.324 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.324 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.324 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:18:32.324 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.957605+0000", 2023-10-09T06:18:32.324 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000154578 2023-10-09T06:18:32.324 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.324 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.324 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.324 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.966399+0000", 2023-10-09T06:18:32.324 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0087941670000000003 2023-10-09T06:18:32.325 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.325 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.325 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.325 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.966430+0000", 2023-10-09T06:18:32.325 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0504000000000001e-05 2023-10-09T06:18:32.325 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.325 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.325 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.325 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.966462+0000", 2023-10-09T06:18:32.325 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1952999999999997e-05 2023-10-09T06:18:32.325 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.326 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.326 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:32.326 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977221+0000", 2023-10-09T06:18:32.326 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010759866 2023-10-09T06:18:32.326 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.326 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.326 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:32.326 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977246+0000", 2023-10-09T06:18:32.327 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4289000000000001e-05 2023-10-09T06:18:32.327 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.327 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.327 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:32.327 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977300+0000", 2023-10-09T06:18:32.327 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4431000000000001e-05 2023-10-09T06:18:32.327 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.327 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:32.327 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.327 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.328 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.328 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:171744 10.e 10:7d1e64fd:::benchmark_data_smithi064_573750_object75529:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:18:32.328 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:15.906916+0000", 2023-10-09T06:18:32.328 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 196.177707591, 2023-10-09T06:18:32.328 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0709302470000002, 2023-10-09T06:18:32.328 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:32.328 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:32.328 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:32.328 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:18:32.328 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:32.329 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 171744 2023-10-09T06:18:32.329 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.329 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:32.329 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.329 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:32.329 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.906916+0000", 2023-10-09T06:18:32.329 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.329 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.329 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.329 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:32.330 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.906916+0000", 2023-10-09T06:18:32.330 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.330 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.330 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.330 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:32.330 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.906915+0000", 2023-10-09T06:18:32.330 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:18:32.330 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.331 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.331 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:32.331 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.906922+0000", 2023-10-09T06:18:32.331 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.737e-06 2023-10-09T06:18:32.331 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.331 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.331 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:32.331 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.906932+0000", 2023-10-09T06:18:32.331 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.8630000000000003e-06 2023-10-09T06:18:32.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.332 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.332 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:32.332 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.906936+0000", 2023-10-09T06:18:32.332 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.245e-06 2023-10-09T06:18:32.332 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.332 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.333 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.333 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.907001+0000", 2023-10-09T06:18:32.333 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5140000000000003e-05 2023-10-09T06:18:32.333 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.333 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.333 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.334 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.907009+0000", 2023-10-09T06:18:32.334 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6510000000000001e-06 2023-10-09T06:18:32.334 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.334 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.334 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.335 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808223+0000", 2023-10-09T06:18:32.335 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.90121384900000001 2023-10-09T06:18:32.335 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.335 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.335 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.335 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808229+0000", 2023-10-09T06:18:32.335 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0260000000000004e-06 2023-10-09T06:18:32.335 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.335 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.336 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.336 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.960451+0000", 2023-10-09T06:18:32.336 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1522222259999999 2023-10-09T06:18:32.336 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.336 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.336 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:32.336 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.960567+0000", 2023-10-09T06:18:32.336 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000116549 2023-10-09T06:18:32.336 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.337 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.337 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:18:32.337 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.960697+0000", 2023-10-09T06:18:32.337 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012991599999999999 2023-10-09T06:18:32.337 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.337 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.337 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.337 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.976854+0000", 2023-10-09T06:18:32.337 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.016157082999999999 2023-10-09T06:18:32.337 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.338 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.338 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:32.338 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977304+0000", 2023-10-09T06:18:32.338 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00044994099999999998 2023-10-09T06:18:32.338 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.338 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.338 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.338 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977648+0000", 2023-10-09T06:18:32.339 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00034370499999999998 2023-10-09T06:18:32.339 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.339 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.339 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.339 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977681+0000", 2023-10-09T06:18:32.339 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.3488000000000002e-05 2023-10-09T06:18:32.339 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.339 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.339 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:32.340 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977730+0000", 2023-10-09T06:18:32.340 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8832999999999998e-05 2023-10-09T06:18:32.340 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.340 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.340 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:32.340 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977846+0000", 2023-10-09T06:18:32.340 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011575500000000001 2023-10-09T06:18:32.340 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.340 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:32.340 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.341 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.341 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.341 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:171757 10.e 10:72a212b1:::benchmark_data_smithi064_573750_object75542:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:18:32.341 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:15.960756+0000", 2023-10-09T06:18:32.341 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 196.123867316, 2023-10-09T06:18:32.341 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.017502049, 2023-10-09T06:18:32.341 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:32.341 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:32.341 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:18:32.341 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:18:32.341 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:32.342 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 171757 2023-10-09T06:18:32.342 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.342 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:32.342 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.342 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:32.342 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.960756+0000", 2023-10-09T06:18:32.342 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.342 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.342 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.342 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:32.342 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.960756+0000", 2023-10-09T06:18:32.343 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:32.343 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.343 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.343 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:32.343 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.960755+0000", 2023-10-09T06:18:32.343 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:32.343 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.343 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.344 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:32.344 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.960762+0000", 2023-10-09T06:18:32.344 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3220000000000004e-06 2023-10-09T06:18:32.344 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.344 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.344 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:32.344 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.960778+0000", 2023-10-09T06:18:32.344 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5835e-05 2023-10-09T06:18:32.344 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.345 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.345 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:32.345 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.960800+0000", 2023-10-09T06:18:32.345 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2507e-05 2023-10-09T06:18:32.345 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.345 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.345 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.345 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.960845+0000", 2023-10-09T06:18:32.345 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4177000000000001e-05 2023-10-09T06:18:32.345 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.346 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.346 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.346 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.960853+0000", 2023-10-09T06:18:32.346 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9829999999999996e-06 2023-10-09T06:18:32.346 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.346 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.346 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.346 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808250+0000", 2023-10-09T06:18:32.346 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.847397921 2023-10-09T06:18:32.346 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.346 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.347 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:18:32.347 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808257+0000", 2023-10-09T06:18:32.347 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2179999999999996e-06 2023-10-09T06:18:32.347 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.347 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.347 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:32.347 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.963449+0000", 2023-10-09T06:18:32.347 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1551920849999999 2023-10-09T06:18:32.347 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.347 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.348 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:32.348 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.963546+0000", 2023-10-09T06:18:32.348 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.6785000000000003e-05 2023-10-09T06:18:32.348 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.348 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.348 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:18:32.348 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.963659+0000", 2023-10-09T06:18:32.348 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000113554 2023-10-09T06:18:32.348 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.348 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.348 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:18:32.349 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977308+0000", 2023-10-09T06:18:32.349 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013648680999999999 2023-10-09T06:18:32.349 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.349 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.349 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.349 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977474+0000", 2023-10-09T06:18:32.349 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000165973 2023-10-09T06:18:32.349 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.349 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.349 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.349 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.978068+0000", 2023-10-09T06:18:32.350 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000594708 2023-10-09T06:18:32.350 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.350 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.350 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:32.350 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.978106+0000", 2023-10-09T06:18:32.350 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7033999999999998e-05 2023-10-09T06:18:32.350 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.350 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.350 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:18:32.350 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.978151+0000", 2023-10-09T06:18:32.351 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5046999999999997e-05 2023-10-09T06:18:32.351 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:32.351 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:32.351 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:18:32.351 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.978258+0000", 2023-10-09T06:18:32.351 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000107577 2023-10-09T06:18:32.351 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.351 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:32.351 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.351 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:32.351 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:32.352 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:18:32.352 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:32.352 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:18:32.353 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T06:18:32.354 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T06:18:32.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:31 smithi064 ceph-mon[212750]: failed to encode map e610 with expected crc 2023-10-09T06:18:32.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:31 smithi064 ceph-mon[212750]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T06:18:32.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:31 smithi064 ceph-mon[212750]: osdmap e610: 8 total, 7 up, 7 in 2023-10-09T06:18:32.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:31 smithi064 ceph-mon[212750]: from='osd.5 [v2:172.21.15.156:6808/507344993,v1:172.21.15.156:6809/507344993]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T06:18:32.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:31 smithi064 ceph-mon[212750]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T06:18:32.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:31 smithi064 ceph-mon[212750]: failed to encode map e610 with expected crc 2023-10-09T06:18:32.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:31 smithi064 ceph-mon[212750]: failed to encode map e610 with expected crc 2023-10-09T06:18:32.425 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:31 smithi064 ceph-mon[212750]: failed to encode map e610 with expected crc 2023-10-09T06:18:32.425 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:31 smithi064 ceph-mon[212750]: pgmap v1569: 119 pgs: 20 active+clean+remapped, 17 active+undersized, 6 active+undersized+degraded, 76 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 17479/210357 objects degraded (8.309%); 8/210357 objects misplaced (0.004%) 2023-10-09T06:18:32.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:31 smithi064 ceph-mon[220044]: failed to encode map e610 with expected crc 2023-10-09T06:18:32.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:31 smithi064 ceph-mon[220044]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T06:18:32.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:31 smithi064 ceph-mon[220044]: osdmap e610: 8 total, 7 up, 7 in 2023-10-09T06:18:32.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:31 smithi064 ceph-mon[220044]: from='osd.5 [v2:172.21.15.156:6808/507344993,v1:172.21.15.156:6809/507344993]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T06:18:32.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:31 smithi064 ceph-mon[220044]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2023-10-09T06:18:32.426 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:31 smithi064 ceph-mon[220044]: failed to encode map e610 with expected crc 2023-10-09T06:18:32.426 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:31 smithi064 ceph-mon[220044]: failed to encode map e610 with expected crc 2023-10-09T06:18:32.426 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:31 smithi064 ceph-mon[220044]: failed to encode map e610 with expected crc 2023-10-09T06:18:32.426 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:31 smithi064 ceph-mon[220044]: pgmap v1569: 119 pgs: 20 active+clean+remapped, 17 active+undersized, 6 active+undersized+degraded, 76 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 17479/210357 objects degraded (8.309%); 8/210357 objects misplaced (0.004%) 2023-10-09T06:18:32.427 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:32.485 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:32.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:32.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:32.596 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:32.626 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:33.021 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:33.203 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:33.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:33.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:33.232 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:32 smithi064 ceph-mon[212750]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:18:33.232 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:32 smithi064 ceph-mon[212750]: failed to encode map e611 with expected crc 2023-10-09T06:18:33.232 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:32 smithi064 ceph-mon[212750]: failed to encode map e611 with expected crc 2023-10-09T06:18:33.232 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:32 smithi064 ceph-mon[212750]: failed to encode map e611 with expected crc 2023-10-09T06:18:33.232 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:32 smithi064 ceph-mon[212750]: failed to encode map e611 with expected crc 2023-10-09T06:18:33.232 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:32 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T06:18:33.233 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:32 smithi064 ceph-mon[212750]: osd.5 [v2:172.21.15.156:6808/507344993,v1:172.21.15.156:6809/507344993] boot 2023-10-09T06:18:33.233 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:32 smithi064 ceph-mon[212750]: osdmap e611: 8 total, 8 up, 7 in 2023-10-09T06:18:33.233 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:32 smithi064 ceph-mon[212750]: failed to encode map e611 with expected crc 2023-10-09T06:18:33.233 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:32 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 17479/210357 objects degraded (8.309%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:33.233 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:32 smithi064 ceph-mon[212750]: osdmap e612: 8 total, 8 up, 7 in 2023-10-09T06:18:33.233 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:32 smithi064 ceph-mon[220044]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:18:33.233 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:32 smithi064 ceph-mon[220044]: failed to encode map e611 with expected crc 2023-10-09T06:18:33.234 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:32 smithi064 ceph-mon[220044]: failed to encode map e611 with expected crc 2023-10-09T06:18:33.234 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:32 smithi064 ceph-mon[220044]: failed to encode map e611 with expected crc 2023-10-09T06:18:33.234 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:32 smithi064 ceph-mon[220044]: failed to encode map e611 with expected crc 2023-10-09T06:18:33.234 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:32 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T06:18:33.234 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:32 smithi064 ceph-mon[220044]: osd.5 [v2:172.21.15.156:6808/507344993,v1:172.21.15.156:6809/507344993] boot 2023-10-09T06:18:33.234 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:32 smithi064 ceph-mon[220044]: osdmap e611: 8 total, 8 up, 7 in 2023-10-09T06:18:33.234 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:32 smithi064 ceph-mon[220044]: failed to encode map e611 with expected crc 2023-10-09T06:18:33.234 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:32 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 17479/210357 objects degraded (8.309%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:33.234 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:32 smithi064 ceph-mon[220044]: osdmap e612: 8 total, 8 up, 7 in 2023-10-09T06:18:33.254 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:32 smithi156 ceph-mon[195101]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:18:33.254 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:32 smithi156 ceph-mon[195101]: failed to encode map e611 with expected crc 2023-10-09T06:18:33.255 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:32 smithi156 ceph-mon[195101]: failed to encode map e611 with expected crc 2023-10-09T06:18:33.255 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:32 smithi156 ceph-mon[195101]: failed to encode map e611 with expected crc 2023-10-09T06:18:33.255 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:32 smithi156 ceph-mon[195101]: failed to encode map e611 with expected crc 2023-10-09T06:18:33.255 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:32 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T06:18:33.255 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:32 smithi156 ceph-mon[195101]: osd.5 [v2:172.21.15.156:6808/507344993,v1:172.21.15.156:6809/507344993] boot 2023-10-09T06:18:33.255 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:32 smithi156 ceph-mon[195101]: osdmap e611: 8 total, 8 up, 7 in 2023-10-09T06:18:33.255 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:32 smithi156 ceph-mon[195101]: failed to encode map e611 with expected crc 2023-10-09T06:18:33.255 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:32 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 17479/210357 objects degraded (8.309%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:33.255 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:32 smithi156 ceph-mon[195101]: osdmap e612: 8 total, 8 up, 7 in 2023-10-09T06:18:33.274 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:18:33.309 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:33.347 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:33.448 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:33.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:33.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:33.494 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:18:33.551 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:33.585 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:33.730 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:33.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:33.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:33.834 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:33.873 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:33.987 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:18:33.990 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:33.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:33.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:34.097 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:34.210 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:33 smithi064 ceph-mon[212750]: failed to encode map e599 with expected crc 2023-10-09T06:18:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:33 smithi064 ceph-mon[212750]: failed to encode map e611 with expected crc 2023-10-09T06:18:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:33 smithi064 ceph-mon[212750]: failed to encode map e612 with expected crc 2023-10-09T06:18:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:33 smithi064 ceph-mon[212750]: failed to encode map e612 with expected crc 2023-10-09T06:18:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:33 smithi064 ceph-mon[212750]: failed to encode map e612 with expected crc 2023-10-09T06:18:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:33 smithi064 ceph-mon[212750]: failed to encode map e612 with expected crc 2023-10-09T06:18:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:33 smithi064 ceph-mon[212750]: failed to encode map e612 with expected crc 2023-10-09T06:18:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:33 smithi064 ceph-mon[212750]: failed to encode map e612 with expected crc 2023-10-09T06:18:34.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:33 smithi064 ceph-mon[212750]: pgmap v1572: 119 pgs: 20 active+clean+remapped, 17 active+undersized, 6 active+undersized+degraded, 76 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 966 op/s; 17479/210357 objects degraded (8.309%); 8/210357 objects misplaced (0.004%) 2023-10-09T06:18:34.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:33 smithi064 ceph-mon[212750]: osdmap e613: 8 total, 8 up, 7 in 2023-10-09T06:18:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:33 smithi064 ceph-mon[220044]: failed to encode map e599 with expected crc 2023-10-09T06:18:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:33 smithi064 ceph-mon[220044]: failed to encode map e611 with expected crc 2023-10-09T06:18:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:33 smithi064 ceph-mon[220044]: failed to encode map e612 with expected crc 2023-10-09T06:18:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:33 smithi064 ceph-mon[220044]: failed to encode map e612 with expected crc 2023-10-09T06:18:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:33 smithi064 ceph-mon[220044]: failed to encode map e612 with expected crc 2023-10-09T06:18:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:33 smithi064 ceph-mon[220044]: failed to encode map e612 with expected crc 2023-10-09T06:18:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:33 smithi064 ceph-mon[220044]: failed to encode map e612 with expected crc 2023-10-09T06:18:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:33 smithi064 ceph-mon[220044]: failed to encode map e612 with expected crc 2023-10-09T06:18:34.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:33 smithi064 ceph-mon[220044]: pgmap v1572: 119 pgs: 20 active+clean+remapped, 17 active+undersized, 6 active+undersized+degraded, 76 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 966 op/s; 17479/210357 objects degraded (8.309%); 8/210357 objects misplaced (0.004%) 2023-10-09T06:18:34.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:33 smithi064 ceph-mon[220044]: osdmap e613: 8 total, 8 up, 7 in 2023-10-09T06:18:34.325 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:34.326 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:34 smithi156 ceph-mon[195101]: failed to encode map e599 with expected crc 2023-10-09T06:18:34.326 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:34 smithi156 ceph-mon[195101]: failed to encode map e611 with expected crc 2023-10-09T06:18:34.326 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:34 smithi156 ceph-mon[195101]: failed to encode map e612 with expected crc 2023-10-09T06:18:34.326 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:34 smithi156 ceph-mon[195101]: failed to encode map e612 with expected crc 2023-10-09T06:18:34.326 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:34 smithi156 ceph-mon[195101]: failed to encode map e612 with expected crc 2023-10-09T06:18:34.326 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:34 smithi156 ceph-mon[195101]: failed to encode map e612 with expected crc 2023-10-09T06:18:34.326 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:34 smithi156 ceph-mon[195101]: failed to encode map e612 with expected crc 2023-10-09T06:18:34.326 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:34 smithi156 ceph-mon[195101]: failed to encode map e612 with expected crc 2023-10-09T06:18:34.326 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:34 smithi156 ceph-mon[195101]: pgmap v1572: 119 pgs: 20 active+clean+remapped, 17 active+undersized, 6 active+undersized+degraded, 76 active+clean; 4.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 966 op/s; 17479/210357 objects degraded (8.309%); 8/210357 objects misplaced (0.004%) 2023-10-09T06:18:34.326 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:34 smithi156 ceph-mon[195101]: osdmap e613: 8 total, 8 up, 7 in 2023-10-09T06:18:34.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:34.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:34.355 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:18:34.428 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:34.483 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:34.789 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:34.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:34.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:34.896 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:34.948 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:35.059 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:35.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:35 smithi064 ceph-mon[212750]: failed to encode map e613 with expected crc 2023-10-09T06:18:35.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:35 smithi064 ceph-mon[212750]: failed to encode map e613 with expected crc 2023-10-09T06:18:35.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:35 smithi064 ceph-mon[212750]: failed to encode map e613 with expected crc 2023-10-09T06:18:35.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:35 smithi064 ceph-mon[212750]: failed to encode map e613 with expected crc 2023-10-09T06:18:35.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:35 smithi064 ceph-mon[220044]: failed to encode map e613 with expected crc 2023-10-09T06:18:35.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:35 smithi064 ceph-mon[220044]: failed to encode map e613 with expected crc 2023-10-09T06:18:35.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:35 smithi064 ceph-mon[220044]: failed to encode map e613 with expected crc 2023-10-09T06:18:35.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:35 smithi064 ceph-mon[220044]: failed to encode map e613 with expected crc 2023-10-09T06:18:35.250 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:35.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:35 smithi156 ceph-mon[195101]: failed to encode map e613 with expected crc 2023-10-09T06:18:35.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:35 smithi156 ceph-mon[195101]: failed to encode map e613 with expected crc 2023-10-09T06:18:35.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:35 smithi156 ceph-mon[195101]: failed to encode map e613 with expected crc 2023-10-09T06:18:35.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:35 smithi156 ceph-mon[195101]: failed to encode map e613 with expected crc 2023-10-09T06:18:35.721 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:18:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:18:35] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-10-09T06:18:36.167 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:36.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:36.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:36.277 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:36.326 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:36.368 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:18:36.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:36 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:36.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:36 smithi064 ceph-mon[212750]: pgmap v1574: 119 pgs: 14 active+clean+remapped, 1 active+recovering+undersized+remapped, 7 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 93 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 786 op/s; 10520/206124 objects degraded (5.104%); 7/206124 objects misplaced (0.003%); 0 B/s, 15 objects/s recovering 2023-10-09T06:18:36.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:36 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:36.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:36 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3531078571' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:18:36.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:36 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:18:36.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:36 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:36.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:36 smithi064 ceph-mon[220044]: pgmap v1574: 119 pgs: 14 active+clean+remapped, 1 active+recovering+undersized+remapped, 7 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 93 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 786 op/s; 10520/206124 objects degraded (5.104%); 7/206124 objects misplaced (0.003%); 0 B/s, 15 objects/s recovering 2023-10-09T06:18:36.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:36 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:36.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:36 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3531078571' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:18:36.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:36 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:18:36.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:36 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:36.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:36 smithi156 ceph-mon[195101]: pgmap v1574: 119 pgs: 14 active+clean+remapped, 1 active+recovering+undersized+remapped, 7 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 93 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 786 op/s; 10520/206124 objects degraded (5.104%); 7/206124 objects misplaced (0.003%); 0 B/s, 15 objects/s recovering 2023-10-09T06:18:36.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:36 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:36.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:36 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3531078571' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:18:36.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:36 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:18:36.782 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:18:36.782 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:18:36.782 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.782 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.36591.0:151491 11.a e614/612 11:525620df:::benchmark_data_smithi064_631903_object42577:head v 614'9402, mlcod=598'7308)", 2023-10-09T06:18:36.783 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:18:36.767966+0000", 2023-10-09T06:18:36.783 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0092794439999999995, 2023-10-09T06:18:36.785 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0093126370000000003, 2023-10-09T06:18:36.785 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:36.785 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:18:36.785 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:36.785 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.786 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:36.786 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:36.767966+0000", 2023-10-09T06:18:36.786 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:36.786 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:36.786 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.786 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:36.786 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:36.767966+0000", 2023-10-09T06:18:36.786 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:36.786 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:36.786 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.786 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:36.787 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:36.767965+0000", 2023-10-09T06:18:36.787 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:18:36.787 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:36.787 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.787 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:36.787 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:36.767972+0000", 2023-10-09T06:18:36.787 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.489e-06 2023-10-09T06:18:36.787 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:36.787 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.787 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:36.787 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:36.767985+0000", 2023-10-09T06:18:36.787 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3213999999999999e-05 2023-10-09T06:18:36.788 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:36.788 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.788 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:36.788 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:36.767991+0000", 2023-10-09T06:18:36.788 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4860000000000002e-06 2023-10-09T06:18:36.788 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:36.788 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.788 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:36.788 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:36.770169+0000", 2023-10-09T06:18:36.788 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0021780670000000001 2023-10-09T06:18:36.788 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:36.788 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.789 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:36.789 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:36.770192+0000", 2023-10-09T06:18:36.789 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3070000000000001e-05 2023-10-09T06:18:36.789 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:36.789 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:36.789 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:36.789 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:36.789 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.789 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.36591.0:151494 11.a e614/612 11:597d7722:::benchmark_data_smithi064_631903_object42580:head v 614'9403, mlcod=598'7308)", 2023-10-09T06:18:36.789 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:18:36.771355+0000", 2023-10-09T06:18:36.789 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0058906189999999997, 2023-10-09T06:18:36.789 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0059918660000000002, 2023-10-09T06:18:36.790 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:36.790 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:18:36.790 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:36.790 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.790 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:36.790 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:36.771355+0000", 2023-10-09T06:18:36.790 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:36.790 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:36.790 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.790 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:36.790 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:36.771355+0000", 2023-10-09T06:18:36.790 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:36.791 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:36.791 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.791 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:36.791 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:36.771354+0000", 2023-10-09T06:18:36.791 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:18:36.791 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:36.791 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.791 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:36.791 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:36.771362+0000", 2023-10-09T06:18:36.791 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0090000000000007e-06 2023-10-09T06:18:36.791 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:36.791 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.792 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:36.792 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:36.771372+0000", 2023-10-09T06:18:36.792 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7720000000000006e-06 2023-10-09T06:18:36.792 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:36.792 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.792 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:36.792 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:36.771377+0000", 2023-10-09T06:18:36.792 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9300000000000002e-06 2023-10-09T06:18:36.792 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:36.792 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.792 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:18:36.792 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:36.775015+0000", 2023-10-09T06:18:36.793 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0036379699999999999 2023-10-09T06:18:36.793 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:36.793 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.793 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:18:36.793 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:36.775041+0000", 2023-10-09T06:18:36.793 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.6038e-05 2023-10-09T06:18:36.793 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:36.793 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:36.793 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:36.793 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:36.793 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.793 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.36591.0:151497 11.a e614/612)", 2023-10-09T06:18:36.794 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:18:36.774933+0000", 2023-10-09T06:18:36.794 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0023125210000000001, 2023-10-09T06:18:36.794 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0024696219999999999, 2023-10-09T06:18:36.794 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:18:36.794 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "queued for pg", 2023-10-09T06:18:36.794 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:18:36.794 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.794 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:18:36.794 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:36.774933+0000", 2023-10-09T06:18:36.794 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:36.794 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:36.794 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.795 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:18:36.795 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:36.774933+0000", 2023-10-09T06:18:36.795 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:18:36.795 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:36.795 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.795 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:18:36.795 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:36.774932+0000", 2023-10-09T06:18:36.795 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:18:36.795 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:36.795 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.795 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:18:36.795 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:36.774940+0000", 2023-10-09T06:18:36.796 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.2490000000000001e-06 2023-10-09T06:18:36.796 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:36.796 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.796 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:18:36.796 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:36.774952+0000", 2023-10-09T06:18:36.796 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.129e-05 2023-10-09T06:18:36.796 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:18:36.796 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:18:36.796 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:18:36.796 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:36.774958+0000", 2023-10-09T06:18:36.796 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0469999999999999e-06 2023-10-09T06:18:36.796 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:36.797 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:18:36.797 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:36.797 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:18:36.797 INFO:teuthology.orchestra.run.smithi156.stdout: ], 2023-10-09T06:18:36.797 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 3 2023-10-09T06:18:36.797 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:18:36.830 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:36.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:36.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:36.935 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:37.009 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:37.097 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:37.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:37.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:37.210 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:37.236 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:37.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:37 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:37.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:37 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:18:37.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:37 smithi156 ceph-mon[195101]: osdmap e614: 8 total, 8 up, 7 in 2023-10-09T06:18:37.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:37 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:37.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:37 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:37.699 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:37.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:37.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:37.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:37 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:37.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:37 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:18:37.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:37 smithi064 ceph-mon[212750]: osdmap e614: 8 total, 8 up, 7 in 2023-10-09T06:18:37.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:37 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:37.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:37 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:37.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:37 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:37.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:37 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:18:37.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:37 smithi064 ceph-mon[220044]: osdmap e614: 8 total, 8 up, 7 in 2023-10-09T06:18:37.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:37 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:37.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:37 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:37.810 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:37.874 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:38.262 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:38.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:38.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:38.369 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:38.396 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:38.602 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:38 smithi064 ceph-mon[212750]: failed to encode map e614 with expected crc 2023-10-09T06:18:38.602 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:38 smithi064 ceph-mon[212750]: failed to encode map e614 with expected crc 2023-10-09T06:18:38.602 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:38 smithi064 ceph-mon[212750]: pgmap v1576: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 5801/202224 objects degraded (2.869%); 0 B/s, 12 objects/s recovering 2023-10-09T06:18:38.602 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:38 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:18:38.602 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:38 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:18:38.602 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:38 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:38.603 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:38 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 5801/202224 objects degraded (2.869%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:38.603 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:38 smithi064 ceph-mon[220044]: failed to encode map e614 with expected crc 2023-10-09T06:18:38.603 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:38 smithi064 ceph-mon[220044]: failed to encode map e614 with expected crc 2023-10-09T06:18:38.603 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:38 smithi064 ceph-mon[220044]: pgmap v1576: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 5801/202224 objects degraded (2.869%); 0 B/s, 12 objects/s recovering 2023-10-09T06:18:38.604 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:38 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:18:38.604 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:38 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:18:38.604 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:38 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:38.604 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:38 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 5801/202224 objects degraded (2.869%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:38.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:38 smithi156 ceph-mon[195101]: failed to encode map e614 with expected crc 2023-10-09T06:18:38.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:38 smithi156 ceph-mon[195101]: failed to encode map e614 with expected crc 2023-10-09T06:18:38.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:38 smithi156 ceph-mon[195101]: pgmap v1576: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 5801/202224 objects degraded (2.869%); 0 B/s, 12 objects/s recovering 2023-10-09T06:18:38.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:38 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:18:38.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:38 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:18:38.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:38 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:38.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:38 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 5801/202224 objects degraded (2.869%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:39.831 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:39.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:39.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:39.947 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:40.006 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:40.114 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:18:40.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:40.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:40.222 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:40.291 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:40.339 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:40 smithi156 ceph-mon[195101]: pgmap v1577: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 416 op/s; 5801/202224 objects degraded (2.869%); 0 B/s, 10 objects/s recovering 2023-10-09T06:18:40.342 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:40 smithi064 ceph-mon[220044]: pgmap v1577: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 416 op/s; 5801/202224 objects degraded (2.869%); 0 B/s, 10 objects/s recovering 2023-10-09T06:18:40.473 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:18:40.473 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:18:40.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.473 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36591.0:154316 11.2 e614/572 11:4f194741:::benchmark_data_smithi064_631903_object45402:head v 614'9605, mlcod=614'9603)", 2023-10-09T06:18:40.473 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:18:40.466051+0000", 2023-10-09T06:18:40.473 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0033170349999999999, 2023-10-09T06:18:40.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033479009999999999, 2023-10-09T06:18:40.473 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:18:40.473 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:40.474 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:18:40.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:18:40.474 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:40.466051+0000", 2023-10-09T06:18:40.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:40.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:40.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:18:40.474 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:40.466051+0000", 2023-10-09T06:18:40.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:40.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:40.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:18:40.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:40.466050+0000", 2023-10-09T06:18:40.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:18:40.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:40.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:18:40.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:40.466058+0000", 2023-10-09T06:18:40.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0199999999999994e-06 2023-10-09T06:18:40.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:40.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:18:40.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:40.466069+0000", 2023-10-09T06:18:40.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1429e-05 2023-10-09T06:18:40.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:40.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:18:40.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:40.466077+0000", 2023-10-09T06:18:40.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0050000000000004e-06 2023-10-09T06:18:40.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:40.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:18:40.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:40.466209+0000", 2023-10-09T06:18:40.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013204300000000001 2023-10-09T06:18:40.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:40.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:18:40.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:40.466230+0000", 2023-10-09T06:18:40.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0699999999999998e-05 2023-10-09T06:18:40.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:40.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:18:40.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:40.469381+0000", 2023-10-09T06:18:40.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031511040000000001 2023-10-09T06:18:40.477 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:40.478 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:18:40.478 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:40.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:40.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.478 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36591.0:154323 11.9 e614/604 11:94f78e1c:::benchmark_data_smithi064_631903_object45409:head v 614'9549, mlcod=614'9547)", 2023-10-09T06:18:40.478 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:18:40.466386+0000", 2023-10-09T06:18:40.478 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0029816119999999998, 2023-10-09T06:18:40.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031170450000000001, 2023-10-09T06:18:40.478 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:18:40.478 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:18:40.478 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:18:40.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:18:40.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:40.466386+0000", 2023-10-09T06:18:40.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:40.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:40.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:18:40.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:40.466386+0000", 2023-10-09T06:18:40.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:40.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:40.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:18:40.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:40.466386+0000", 2023-10-09T06:18:40.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:40.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:40.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:18:40.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:40.466392+0000", 2023-10-09T06:18:40.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1469999999999998e-06 2023-10-09T06:18:40.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:40.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:18:40.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:40.466402+0000", 2023-10-09T06:18:40.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8500000000000006e-06 2023-10-09T06:18:40.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:40.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:18:40.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:40.466408+0000", 2023-10-09T06:18:40.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2840000000000003e-06 2023-10-09T06:18:40.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:40.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:18:40.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:40.466470+0000", 2023-10-09T06:18:40.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2323000000000006e-05 2023-10-09T06:18:40.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:40.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.482 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:18:40.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:40.466490+0000", 2023-10-09T06:18:40.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.965e-05 2023-10-09T06:18:40.482 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:40.482 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:18:40.482 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:40.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:40.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.482 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36591.0:154317 11.f e614/572)", 2023-10-09T06:18:40.482 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:18:40.469466+0000", 2023-10-09T06:18:40.483 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 4294967295.9999013, 2023-10-09T06:18:40.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000113089, 2023-10-09T06:18:40.483 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:18:40.483 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:18:40.483 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:18:40.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:18:40.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:40.469466+0000", 2023-10-09T06:18:40.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:40.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:40.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:18:40.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:40.469466+0000", 2023-10-09T06:18:40.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:40.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:40.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:18:40.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:40.469465+0000", 2023-10-09T06:18:40.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:18:40.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:40.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:18:40.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:40.469473+0000", 2023-10-09T06:18:40.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0749999999999998e-06 2023-10-09T06:18:40.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:40.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:18:40.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:40.469484+0000", 2023-10-09T06:18:40.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0524e-05 2023-10-09T06:18:40.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:40.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:18:40.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:40.469492+0000", 2023-10-09T06:18:40.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6490000000000008e-06 2023-10-09T06:18:40.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:40.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:40.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:18:40.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:40.469565+0000", 2023-10-09T06:18:40.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3503000000000004e-05 2023-10-09T06:18:40.486 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:40.486 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:18:40.486 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:40.486 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:40.486 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-10-09T06:18:40.486 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 3 2023-10-09T06:18:40.486 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:18:40.503 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:40.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:40.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:40.613 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:40.637 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:18:40.671 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:40 smithi064 ceph-mon[212750]: pgmap v1577: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 416 op/s; 5801/202224 objects degraded (2.869%); 0 B/s, 10 objects/s recovering 2023-10-09T06:18:40.674 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:41.156 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:41.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:41.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:41.268 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:41.319 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:41.354 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:18:41.354 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:18:41.355 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:18:41.355 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:18:41.398 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:18:41.401 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:41.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:41.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:41.422 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:18:41.456 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:18:41.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:41 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3690365371' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:41.475 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:41 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3690365371' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:41.488 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:18:41.492 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:18:41.493 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:18:41.493 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:18:41.493 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:18:41.493 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:18:41.493 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:18:41.493 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:18:41.493 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:18:41.509 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:41.560 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:41.625 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:41 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3690365371' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:41.651 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:41.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:41.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:41.759 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:41.833 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:41.911 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:41.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:41.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:42.023 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:42.102 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:42.209 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:42.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:42.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:42.327 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:42.363 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:42 smithi064 ceph-mon[220044]: pgmap v1578: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 822 op/s; 5801/190605 objects degraded (3.043%); 0 B/s, 58 objects/s recovering 2023-10-09T06:18:42.393 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:42.473 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:18:42.622 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:42 smithi156 ceph-mon[195101]: pgmap v1578: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 822 op/s; 5801/190605 objects degraded (3.043%); 0 B/s, 58 objects/s recovering 2023-10-09T06:18:42.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:42 smithi064 ceph-mon[212750]: pgmap v1578: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 822 op/s; 5801/190605 objects degraded (3.043%); 0 B/s, 58 objects/s recovering 2023-10-09T06:18:42.790 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:42.821 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:42.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:42.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:42.848 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 5, 7, 1, 6] out_osds: [2] dead_osds: [] live_osds: [1, 2, 3, 6, 7, 4, 0, 5] 2023-10-09T06:18:42.848 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T06:18:42.934 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:43.009 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:43.167 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:43.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:43.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:43.289 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:43.376 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:43.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:43 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 5801/190605 objects degraded (3.043%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:43.440 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:43 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 5801/190605 objects degraded (3.043%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:43.482 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:43.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:43.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:43.601 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:43.626 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:43 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 5801/190605 objects degraded (3.043%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:43.666 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:43.755 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:43.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:43.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:43.868 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:43.930 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:44.035 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:44.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:44.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:44.144 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:44.211 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:44.310 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:18:44.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:44.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:44.418 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:44.501 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:44.545 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:44 smithi156 ceph-mon[195101]: pgmap v1579: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 5801/190605 objects degraded (3.043%); 0 B/s, 48 objects/s recovering 2023-10-09T06:18:44.545 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:44.545 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:44.656 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:44.659 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:44.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:44.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:44.757 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:44.785 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:44.795 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:18:44.833 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:44.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:44 smithi064 ceph-mon[212750]: pgmap v1579: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 5801/190605 objects degraded (3.043%); 0 B/s, 48 objects/s recovering 2023-10-09T06:18:44.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:44.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:44.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:44 smithi064 ceph-mon[220044]: pgmap v1579: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 5801/190605 objects degraded (3.043%); 0 B/s, 48 objects/s recovering 2023-10-09T06:18:44.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:18:44.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:44.921 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:44.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:44.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:45.030 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:45.080 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:45.170 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:45.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:45.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:45.278 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:45.336 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:45.438 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:18:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:18:45] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.43.0" 2023-10-09T06:18:45.620 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:45.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:45.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:45.732 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:45.790 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:46.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:46 smithi156 ceph-mon[195101]: pgmap v1580: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 743 op/s; 5801/183798 objects degraded (3.156%); 0 B/s, 81 objects/s recovering 2023-10-09T06:18:46.905 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:46 smithi064 ceph-mon[212750]: pgmap v1580: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 743 op/s; 5801/183798 objects degraded (3.156%); 0 B/s, 81 objects/s recovering 2023-10-09T06:18:46.905 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:46 smithi064 ceph-mon[220044]: pgmap v1580: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 743 op/s; 5801/183798 objects degraded (3.156%); 0 B/s, 81 objects/s recovering 2023-10-09T06:18:47.253 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:47.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:47.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:47.360 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:47.386 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:47.428 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:47.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:47.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:47.535 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:47.587 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:18:47.594 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:47.680 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:47.694 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:47 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2089887234' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:47.694 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:47 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:47.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:47.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:47.796 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:47.828 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:47 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2089887234' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:47.829 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:47 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:47.832 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:47 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2089887234' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:47.832 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:47 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:47.880 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:48.378 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:48.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:48.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:48.488 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:48.520 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:48.573 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:48.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:48 smithi064 ceph-mon[212750]: pgmap v1581: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 764 op/s; 5801/177072 objects degraded (3.276%); 0 B/s, 74 objects/s recovering 2023-10-09T06:18:48.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:48 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:48.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:48 smithi064 ceph-mon[212750]: failed to encode map e615 with expected crc 2023-10-09T06:18:48.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:48 smithi064 ceph-mon[212750]: failed to encode map e615 with expected crc 2023-10-09T06:18:48.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:48 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:18:48.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:48 smithi064 ceph-mon[212750]: osdmap e615: 8 total, 8 up, 7 in 2023-10-09T06:18:48.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:48 smithi064 ceph-mon[212750]: failed to encode map e615 with expected crc 2023-10-09T06:18:48.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:48 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 5801/177072 objects degraded (3.276%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:48.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:48 smithi064 ceph-mon[220044]: pgmap v1581: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 764 op/s; 5801/177072 objects degraded (3.276%); 0 B/s, 74 objects/s recovering 2023-10-09T06:18:48.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:48 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:48.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:48 smithi064 ceph-mon[220044]: failed to encode map e615 with expected crc 2023-10-09T06:18:48.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:48 smithi064 ceph-mon[220044]: failed to encode map e615 with expected crc 2023-10-09T06:18:48.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:48 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:18:48.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:48 smithi064 ceph-mon[220044]: osdmap e615: 8 total, 8 up, 7 in 2023-10-09T06:18:48.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:48 smithi064 ceph-mon[220044]: failed to encode map e615 with expected crc 2023-10-09T06:18:48.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:48 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 5801/177072 objects degraded (3.276%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:48.837 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:48.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:48.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:48.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:48 smithi156 ceph-mon[195101]: pgmap v1581: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 764 op/s; 5801/177072 objects degraded (3.276%); 0 B/s, 74 objects/s recovering 2023-10-09T06:18:48.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:48 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:48.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:48 smithi156 ceph-mon[195101]: failed to encode map e615 with expected crc 2023-10-09T06:18:48.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:48 smithi156 ceph-mon[195101]: failed to encode map e615 with expected crc 2023-10-09T06:18:48.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:48 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:18:48.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:48 smithi156 ceph-mon[195101]: osdmap e615: 8 total, 8 up, 7 in 2023-10-09T06:18:48.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:48 smithi156 ceph-mon[195101]: failed to encode map e615 with expected crc 2023-10-09T06:18:48.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:48 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 5801/177072 objects degraded (3.276%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:48.951 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:49.004 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:49.080 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:49.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:49.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:49.149 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:18:49.149 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:18:49.189 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:49.239 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:49.313 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:49.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:49.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:49.419 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:49.490 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:49.590 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:49.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:49.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:49.603 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:49 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4003811265' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:18:49.603 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:49 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:18:49.605 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:49 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4003811265' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:18:49.605 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:49 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:18:49.605 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:49 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4003811265' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:18:49.605 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:49 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:18:49.615 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T06:18:49.704 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:49.766 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:49.892 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:49.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:49.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:50.007 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:50.059 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:50.565 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:18:50.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:50 smithi156 ceph-mon[195101]: pgmap v1583: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 837 op/s; 5801/177072 objects degraded (3.276%); 0 B/s, 81 objects/s recovering 2023-10-09T06:18:50.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:50 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:18:50.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:50 smithi156 ceph-mon[195101]: osdmap e616: 8 total, 8 up, 7 in 2023-10-09T06:18:50.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:50 smithi156 ceph-mon[195101]: failed to encode map e616 with expected crc 2023-10-09T06:18:50.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:50 smithi156 ceph-mon[195101]: failed to encode map e616 with expected crc 2023-10-09T06:18:50.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:50 smithi064 ceph-mon[212750]: pgmap v1583: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 837 op/s; 5801/177072 objects degraded (3.276%); 0 B/s, 81 objects/s recovering 2023-10-09T06:18:50.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:50 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:18:50.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:50 smithi064 ceph-mon[212750]: osdmap e616: 8 total, 8 up, 7 in 2023-10-09T06:18:50.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:50 smithi064 ceph-mon[212750]: failed to encode map e616 with expected crc 2023-10-09T06:18:50.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:50 smithi064 ceph-mon[212750]: failed to encode map e616 with expected crc 2023-10-09T06:18:50.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:50 smithi064 ceph-mon[220044]: pgmap v1583: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 837 op/s; 5801/177072 objects degraded (3.276%); 0 B/s, 81 objects/s recovering 2023-10-09T06:18:50.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:50 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:18:50.973 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:50 smithi064 ceph-mon[220044]: osdmap e616: 8 total, 8 up, 7 in 2023-10-09T06:18:50.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:50 smithi064 ceph-mon[220044]: failed to encode map e616 with expected crc 2023-10-09T06:18:50.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:50 smithi064 ceph-mon[220044]: failed to encode map e616 with expected crc 2023-10-09T06:18:51.950 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:51.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:51.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:52.001 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T06:18:52.054 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:18:52.056 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:52.123 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:52.143 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:52.189 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:18:52.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:52.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:52.300 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:52.448 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:52.573 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:52.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:52.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:52.680 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:52.694 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:18:52.742 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:52.750 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:18:52.792 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:18:52.829 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:18:52.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:52 smithi156 ceph-mon[195101]: pgmap v1585: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 5801/163719 objects degraded (3.543%); 0 B/s, 112 objects/s recovering 2023-10-09T06:18:52.874 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:18:52.909 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:18:52.924 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:52 smithi064 ceph-mon[212750]: pgmap v1585: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 5801/163719 objects degraded (3.543%); 0 B/s, 112 objects/s recovering 2023-10-09T06:18:52.926 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:52 smithi064 ceph-mon[220044]: pgmap v1585: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 5801/163719 objects degraded (3.543%); 0 B/s, 112 objects/s recovering 2023-10-09T06:18:52.955 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:18:53.004 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:18:53.005 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:53.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:53.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:53.026 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:18:53.026 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:18:53.026 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:18:53.027 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:18:53.027 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:18:53.027 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:18:53.027 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:18:53.027 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:18:53.114 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:53.177 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:53.286 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:53.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:53.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:53.397 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:53.464 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:53.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:53 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2156976704' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:53.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:53 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 5801/163719 objects degraded (3.543%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:53.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:53 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2156976704' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:53.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:53 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 5801/163719 objects degraded (3.543%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:53.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:53 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2156976704' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:53.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:53 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 5801/163719 objects degraded (3.543%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:54.154 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:54.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:54.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:54.258 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:54.324 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:54.583 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:54.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:54.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:54.634 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:54.651 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:54.691 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:54.752 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:54.819 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:18:54.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:54.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:54.926 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:54.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:54 smithi064 ceph-mon[212750]: pgmap v1586: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 836 op/s; 5801/163719 objects degraded (3.543%); 0 B/s, 59 objects/s recovering 2023-10-09T06:18:54.938 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:54 smithi064 ceph-mon[212750]: osdmap e617: 8 total, 8 up, 7 in 2023-10-09T06:18:54.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:54 smithi064 ceph-mon[220044]: pgmap v1586: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 836 op/s; 5801/163719 objects degraded (3.543%); 0 B/s, 59 objects/s recovering 2023-10-09T06:18:54.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:54 smithi064 ceph-mon[220044]: osdmap e617: 8 total, 8 up, 7 in 2023-10-09T06:18:54.955 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:55.005 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:18:55.009 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:54 smithi156 ceph-mon[195101]: pgmap v1586: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 836 op/s; 5801/163719 objects degraded (3.543%); 0 B/s, 59 objects/s recovering 2023-10-09T06:18:55.009 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:54 smithi156 ceph-mon[195101]: osdmap e617: 8 total, 8 up, 7 in 2023-10-09T06:18:55.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:55.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:55.111 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:55.137 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:55.183 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:55.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:55.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:55.289 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:55.315 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:55.358 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:18:55.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:55.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:55.466 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:55.470 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:18:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:18:55] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.43.0" 2023-10-09T06:18:55.501 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:56.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:55 smithi156 ceph-mon[195101]: failed to encode map e617 with expected crc 2023-10-09T06:18:56.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:55 smithi156 ceph-mon[195101]: failed to encode map e617 with expected crc 2023-10-09T06:18:56.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:55 smithi156 ceph-mon[195101]: failed to encode map e617 with expected crc 2023-10-09T06:18:56.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:55 smithi156 ceph-mon[195101]: osdmap e618: 8 total, 8 up, 7 in 2023-10-09T06:18:56.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:55 smithi064 ceph-mon[212750]: failed to encode map e617 with expected crc 2023-10-09T06:18:56.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:55 smithi064 ceph-mon[212750]: failed to encode map e617 with expected crc 2023-10-09T06:18:56.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:55 smithi064 ceph-mon[212750]: failed to encode map e617 with expected crc 2023-10-09T06:18:56.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:55 smithi064 ceph-mon[212750]: osdmap e618: 8 total, 8 up, 7 in 2023-10-09T06:18:56.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:55 smithi064 ceph-mon[220044]: failed to encode map e617 with expected crc 2023-10-09T06:18:56.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:55 smithi064 ceph-mon[220044]: failed to encode map e617 with expected crc 2023-10-09T06:18:56.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:55 smithi064 ceph-mon[220044]: failed to encode map e617 with expected crc 2023-10-09T06:18:56.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:55 smithi064 ceph-mon[220044]: osdmap e618: 8 total, 8 up, 7 in 2023-10-09T06:18:56.675 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:18:56.860 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:18:56.918 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:56.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:56.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:57.022 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:57.100 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:56 smithi156 ceph-mon[195101]: pgmap v1588: 119 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 822 op/s; 5801/157956 objects degraded (3.673%); 0 B/s, 129 objects/s recovering 2023-10-09T06:18:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:56 smithi156 ceph-mon[195101]: failed to encode map e618 with expected crc 2023-10-09T06:18:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:56 smithi156 ceph-mon[195101]: failed to encode map e618 with expected crc 2023-10-09T06:18:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:56 smithi156 ceph-mon[195101]: failed to encode map e618 with expected crc 2023-10-09T06:18:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:56 smithi156 ceph-mon[195101]: failed to encode map e618 with expected crc 2023-10-09T06:18:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:56 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1498138243' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:18:57.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:56 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:18:57.198 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:57.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:57.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:57.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:56 smithi064 ceph-mon[212750]: pgmap v1588: 119 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 822 op/s; 5801/157956 objects degraded (3.673%); 0 B/s, 129 objects/s recovering 2023-10-09T06:18:57.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:56 smithi064 ceph-mon[212750]: failed to encode map e618 with expected crc 2023-10-09T06:18:57.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:56 smithi064 ceph-mon[212750]: failed to encode map e618 with expected crc 2023-10-09T06:18:57.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:56 smithi064 ceph-mon[212750]: failed to encode map e618 with expected crc 2023-10-09T06:18:57.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:56 smithi064 ceph-mon[212750]: failed to encode map e618 with expected crc 2023-10-09T06:18:57.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:56 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1498138243' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:18:57.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:56 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:18:57.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:56 smithi064 ceph-mon[220044]: pgmap v1588: 119 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 822 op/s; 5801/157956 objects degraded (3.673%); 0 B/s, 129 objects/s recovering 2023-10-09T06:18:57.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:56 smithi064 ceph-mon[220044]: failed to encode map e618 with expected crc 2023-10-09T06:18:57.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:56 smithi064 ceph-mon[220044]: failed to encode map e618 with expected crc 2023-10-09T06:18:57.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:56 smithi064 ceph-mon[220044]: failed to encode map e618 with expected crc 2023-10-09T06:18:57.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:56 smithi064 ceph-mon[220044]: failed to encode map e618 with expected crc 2023-10-09T06:18:57.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:56 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1498138243' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:18:57.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:56 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:18:57.308 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:57.363 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:58.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:57 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:58.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:57 smithi156 ceph-mon[195101]: failed to encode map e619 with expected crc 2023-10-09T06:18:58.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:57 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:18:58.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:57 smithi156 ceph-mon[195101]: osdmap e619: 8 total, 8 up, 7 in 2023-10-09T06:18:58.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:57 smithi156 ceph-mon[195101]: failed to encode map e619 with expected crc 2023-10-09T06:18:58.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:57 smithi156 ceph-mon[195101]: failed to encode map e619 with expected crc 2023-10-09T06:18:58.123 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:57 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 5801/157956 objects degraded (3.673%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:58.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:57 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:58.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:57 smithi064 ceph-mon[212750]: failed to encode map e619 with expected crc 2023-10-09T06:18:58.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:57 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:18:58.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:57 smithi064 ceph-mon[212750]: osdmap e619: 8 total, 8 up, 7 in 2023-10-09T06:18:58.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:57 smithi064 ceph-mon[212750]: failed to encode map e619 with expected crc 2023-10-09T06:18:58.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:57 smithi064 ceph-mon[212750]: failed to encode map e619 with expected crc 2023-10-09T06:18:58.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:57 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 5801/157956 objects degraded (3.673%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:58.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:57 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:58.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:57 smithi064 ceph-mon[220044]: failed to encode map e619 with expected crc 2023-10-09T06:18:58.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:57 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:18:58.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:57 smithi064 ceph-mon[220044]: osdmap e619: 8 total, 8 up, 7 in 2023-10-09T06:18:58.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:57 smithi064 ceph-mon[220044]: failed to encode map e619 with expected crc 2023-10-09T06:18:58.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:57 smithi064 ceph-mon[220044]: failed to encode map e619 with expected crc 2023-10-09T06:18:58.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:57 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 5801/157956 objects degraded (3.673%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:58.798 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:58.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:58.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:58.902 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:58.940 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:58.995 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:58 smithi064 ceph-mon[212750]: pgmap v1591: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 3885/153429 objects degraded (2.532%); 0 B/s, 143 objects/s recovering 2023-10-09T06:18:58.996 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:18:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:58.996 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:58 smithi064 ceph-mon[220044]: pgmap v1591: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 3885/153429 objects degraded (2.532%); 0 B/s, 143 objects/s recovering 2023-10-09T06:18:58.996 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:18:58 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:59.024 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:59.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:59.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:59.132 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:59.182 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:58 smithi156 ceph-mon[195101]: pgmap v1591: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 3885/153429 objects degraded (2.532%); 0 B/s, 143 objects/s recovering 2023-10-09T06:18:59.182 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:18:58 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:59.185 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:59.269 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:59.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:59.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:59.283 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:59.376 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:59.431 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:18:59.431 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T06:18:59.431 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T06:18:59.431 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:18:59.432 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.432 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:83031 10.1 e514/490 10:8db48241:::benchmark_data_smithi064_573750_object83030:head v 514'5214, mlcod=514'5211)", 2023-10-09T06:18:59.432 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.829326+0000", 2023-10-09T06:18:59.432 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 357.59372567000003, 2023-10-09T06:18:59.432 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.310894585, 2023-10-09T06:18:59.432 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:18:59.432 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:59.432 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:18:59.432 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.432 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:18:59.433 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.829326+0000", 2023-10-09T06:18:59.433 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.433 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.433 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.433 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:18:59.433 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.829326+0000", 2023-10-09T06:18:59.433 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.433 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.433 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.433 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:18:59.433 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.829325+0000", 2023-10-09T06:18:59.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:18:59.434 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.434 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.434 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:18:59.434 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.829372+0000", 2023-10-09T06:18:59.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6585999999999997e-05 2023-10-09T06:18:59.434 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.434 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.434 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:18:59.434 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.829381+0000", 2023-10-09T06:18:59.435 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7739999999999999e-06 2023-10-09T06:18:59.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.435 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:18:59.435 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.829388+0000", 2023-10-09T06:18:59.435 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2870000000000001e-06 2023-10-09T06:18:59.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:18:59.436 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832680+0000", 2023-10-09T06:18:59.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003291913 2023-10-09T06:18:59.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:18:59.436 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832702+0000", 2023-10-09T06:18:59.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2847999999999999e-05 2023-10-09T06:18:59.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:18:59.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835347+0000", 2023-10-09T06:18:59.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026447810000000001 2023-10-09T06:18:59.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:18:59.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140221+0000", 2023-10-09T06:18:59.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30487336999999998 2023-10-09T06:18:59.437 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.437 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:18:59.438 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.438 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.438 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:83032 10.6 10:60352a27:::benchmark_data_smithi064_573750_object83031:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e514)", 2023-10-09T06:18:59.438 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.829973+0000", 2023-10-09T06:18:59.438 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 357.593078171, 2023-10-09T06:18:59.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.310193414, 2023-10-09T06:18:59.438 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:18:59.438 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:59.439 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:18:59.439 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:18:59.439 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:59.439 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 83032 2023-10-09T06:18:59.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.439 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:18:59.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.439 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:18:59.439 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.829973+0000", 2023-10-09T06:18:59.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:18:59.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.829973+0000", 2023-10-09T06:18:59.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:18:59.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.829972+0000", 2023-10-09T06:18:59.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:18:59.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:18:59.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.829991+0000", 2023-10-09T06:18:59.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9579e-05 2023-10-09T06:18:59.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:18:59.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.830001+0000", 2023-10-09T06:18:59.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5210000000000008e-06 2023-10-09T06:18:59.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:18:59.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.830007+0000", 2023-10-09T06:18:59.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3640000000000004e-06 2023-10-09T06:18:59.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.442 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.442 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:18:59.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.830079+0000", 2023-10-09T06:18:59.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2119000000000005e-05 2023-10-09T06:18:59.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.442 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.442 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:18:59.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.830202+0000", 2023-10-09T06:18:59.443 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012245100000000001 2023-10-09T06:18:59.443 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.443 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.443 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:18:59.443 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.830327+0000", 2023-10-09T06:18:59.443 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000125127 2023-10-09T06:18:59.443 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.443 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.443 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:18:59.443 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833007+0000", 2023-10-09T06:18:59.443 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026800029999999998 2023-10-09T06:18:59.443 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.444 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.444 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140016+0000", 2023-10-09T06:18:59.444 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30700909700000001 2023-10-09T06:18:59.444 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.444 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.444 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140075+0000", 2023-10-09T06:18:59.444 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9342999999999997e-05 2023-10-09T06:18:59.444 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:18:59.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140125+0000", 2023-10-09T06:18:59.445 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0037000000000002e-05 2023-10-09T06:18:59.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:18:59.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140167+0000", 2023-10-09T06:18:59.445 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1477999999999998e-05 2023-10-09T06:18:59.445 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.445 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:18:59.445 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.446 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:83033 10.2 e514/490 10:4f3784a2:::benchmark_data_smithi064_573750_object83032:head v 514'5188, mlcod=513'5184)", 2023-10-09T06:18:59.446 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.832428+0000", 2023-10-09T06:18:59.446 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 357.59062354100001, 2023-10-09T06:18:59.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.31206429899999999, 2023-10-09T06:18:59.446 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:18:59.446 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:59.446 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:18:59.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.446 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:18:59.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832428+0000", 2023-10-09T06:18:59.447 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:18:59.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832428+0000", 2023-10-09T06:18:59.447 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:18:59.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832427+0000", 2023-10-09T06:18:59.447 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:59.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.448 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.448 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:18:59.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832452+0000", 2023-10-09T06:18:59.448 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4193e-05 2023-10-09T06:18:59.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.448 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.448 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:18:59.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832462+0000", 2023-10-09T06:18:59.448 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0392e-05 2023-10-09T06:18:59.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:18:59.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832470+0000", 2023-10-09T06:18:59.449 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.61e-06 2023-10-09T06:18:59.449 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:18:59.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832509+0000", 2023-10-09T06:18:59.449 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9552999999999999e-05 2023-10-09T06:18:59.449 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.450 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.450 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:18:59.450 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832530+0000", 2023-10-09T06:18:59.450 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0924000000000002e-05 2023-10-09T06:18:59.450 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.450 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.450 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:18:59.450 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.144443+0000", 2023-10-09T06:18:59.450 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.31191265200000001 2023-10-09T06:18:59.450 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.450 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.451 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:18:59.451 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.144492+0000", 2023-10-09T06:18:59.451 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9591e-05 2023-10-09T06:18:59.451 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.451 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:18:59.451 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.451 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.451 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.451 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:83037 10.5 10:ab60acc0:::benchmark_data_smithi064_573750_object83036:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e514)", 2023-10-09T06:18:59.451 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.833694+0000", 2023-10-09T06:18:59.451 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 357.58935763400001, 2023-10-09T06:18:59.452 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30705259600000001, 2023-10-09T06:18:59.452 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:18:59.452 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:59.452 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:18:59.452 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:18:59.452 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:59.452 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 83037 2023-10-09T06:18:59.452 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.452 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:18:59.452 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.452 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:18:59.452 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833694+0000", 2023-10-09T06:18:59.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.453 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.453 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.453 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:18:59.453 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833694+0000", 2023-10-09T06:18:59.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.453 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.453 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.453 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:18:59.453 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833692+0000", 2023-10-09T06:18:59.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T06:18:59.454 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.454 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:18:59.454 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833721+0000", 2023-10-09T06:18:59.454 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9408999999999999e-05 2023-10-09T06:18:59.454 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.454 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:18:59.454 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833737+0000", 2023-10-09T06:18:59.454 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5554e-05 2023-10-09T06:18:59.454 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.455 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:18:59.455 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833746+0000", 2023-10-09T06:18:59.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0620000000000007e-06 2023-10-09T06:18:59.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.455 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:18:59.455 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833815+0000", 2023-10-09T06:18:59.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9517999999999994e-05 2023-10-09T06:18:59.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.456 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:18:59.456 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833944+0000", 2023-10-09T06:18:59.456 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012879100000000001 2023-10-09T06:18:59.456 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.456 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:18:59.456 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834108+0000", 2023-10-09T06:18:59.456 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000164346 2023-10-09T06:18:59.456 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.456 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.456 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.137909+0000", 2023-10-09T06:18:59.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.303800554 2023-10-09T06:18:59.457 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:18:59.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.137917+0000", 2023-10-09T06:18:59.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0900000000000005e-06 2023-10-09T06:18:59.457 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140650+0000", 2023-10-09T06:18:59.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027334120000000002 2023-10-09T06:18:59.458 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.458 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.458 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:18:59.458 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140707+0000", 2023-10-09T06:18:59.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6308999999999999e-05 2023-10-09T06:18:59.458 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.458 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.458 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:18:59.458 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140747+0000", 2023-10-09T06:18:59.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9807e-05 2023-10-09T06:18:59.459 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.459 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:18:59.459 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.459 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.459 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.459 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:83039 10.6 10:6ebb2182:::benchmark_data_smithi064_573750_object83038:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e514)", 2023-10-09T06:18:59.459 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.833781+0000", 2023-10-09T06:18:59.459 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 357.58927028300002, 2023-10-09T06:18:59.459 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30797565900000001, 2023-10-09T06:18:59.459 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:18:59.459 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:59.459 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:18:59.460 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:18:59.460 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:59.460 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 83039 2023-10-09T06:18:59.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.460 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:18:59.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:18:59.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833781+0000", 2023-10-09T06:18:59.460 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:18:59.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833781+0000", 2023-10-09T06:18:59.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.461 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:18:59.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833781+0000", 2023-10-09T06:18:59.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:59.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.461 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:18:59.462 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833803+0000", 2023-10-09T06:18:59.462 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1861e-05 2023-10-09T06:18:59.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.462 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:18:59.462 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833812+0000", 2023-10-09T06:18:59.462 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6080000000000002e-06 2023-10-09T06:18:59.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.462 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:18:59.462 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833819+0000", 2023-10-09T06:18:59.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9999999999999999e-06 2023-10-09T06:18:59.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.463 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:18:59.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833905+0000", 2023-10-09T06:18:59.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5968000000000001e-05 2023-10-09T06:18:59.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.463 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:18:59.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834028+0000", 2023-10-09T06:18:59.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012332900000000001 2023-10-09T06:18:59.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.464 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:18:59.464 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834128+0000", 2023-10-09T06:18:59.464 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9903999999999998e-05 2023-10-09T06:18:59.464 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.464 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.464 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140047+0000", 2023-10-09T06:18:59.464 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30591839300000001 2023-10-09T06:18:59.464 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.465 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.465 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.141647+0000", 2023-10-09T06:18:59.465 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016003650000000001 2023-10-09T06:18:59.465 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.465 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.465 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:18:59.465 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.141663+0000", 2023-10-09T06:18:59.465 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6113e-05 2023-10-09T06:18:59.465 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.465 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.465 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:18:59.466 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.141712+0000", 2023-10-09T06:18:59.466 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8925000000000003e-05 2023-10-09T06:18:59.466 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.466 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:18:59.466 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.141757+0000", 2023-10-09T06:18:59.466 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4807e-05 2023-10-09T06:18:59.466 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.466 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:18:59.466 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.466 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.467 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.467 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:83040 10.a 10:5efad431:::benchmark_data_smithi064_573750_object83039:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e514)", 2023-10-09T06:18:59.467 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.833834+0000", 2023-10-09T06:18:59.467 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 357.58921736299999, 2023-10-09T06:18:59.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32167670500000001, 2023-10-09T06:18:59.467 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:18:59.467 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:59.467 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:18:59.467 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:18:59.467 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:59.467 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 83040 2023-10-09T06:18:59.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.468 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:18:59.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:18:59.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833834+0000", 2023-10-09T06:18:59.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:18:59.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833834+0000", 2023-10-09T06:18:59.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:18:59.469 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833834+0000", 2023-10-09T06:18:59.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:59.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:18:59.469 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833855+0000", 2023-10-09T06:18:59.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1135999999999999e-05 2023-10-09T06:18:59.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.470 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:18:59.470 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833862+0000", 2023-10-09T06:18:59.470 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.481e-06 2023-10-09T06:18:59.470 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.470 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:18:59.470 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833868+0000", 2023-10-09T06:18:59.470 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5090000000000001e-06 2023-10-09T06:18:59.470 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.470 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:18:59.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835333+0000", 2023-10-09T06:18:59.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001465093 2023-10-09T06:18:59.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:18:59.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.137972+0000", 2023-10-09T06:18:59.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30263898299999997 2023-10-09T06:18:59.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:18:59.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.138144+0000", 2023-10-09T06:18:59.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017225399999999999 2023-10-09T06:18:59.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.142232+0000", 2023-10-09T06:18:59.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.004087355 2023-10-09T06:18:59.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.142266+0000", 2023-10-09T06:18:59.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4878e-05 2023-10-09T06:18:59.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.473 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:18:59.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.155431+0000", 2023-10-09T06:18:59.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013164821 2023-10-09T06:18:59.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.473 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:18:59.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.155475+0000", 2023-10-09T06:18:59.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4078e-05 2023-10-09T06:18:59.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:18:59.474 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.155511+0000", 2023-10-09T06:18:59.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5658000000000003e-05 2023-10-09T06:18:59.474 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.474 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:18:59.474 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.474 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:83034 10.2 e514/490 10:4840dafe:::benchmark_data_smithi064_573750_object83033:head v 514'5189, mlcod=514'5185)", 2023-10-09T06:18:59.474 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.834233+0000", 2023-10-09T06:18:59.474 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 357.58881878599999, 2023-10-09T06:18:59.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.31657803899999998, 2023-10-09T06:18:59.475 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:18:59.475 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:59.475 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:18:59.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:18:59.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834233+0000", 2023-10-09T06:18:59.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:18:59.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834233+0000", 2023-10-09T06:18:59.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:18:59.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834232+0000", 2023-10-09T06:18:59.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:59.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:18:59.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834257+0000", 2023-10-09T06:18:59.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4641e-05 2023-10-09T06:18:59.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:18:59.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834264+0000", 2023-10-09T06:18:59.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.554e-06 2023-10-09T06:18:59.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:18:59.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834270+0000", 2023-10-09T06:18:59.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.242e-06 2023-10-09T06:18:59.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:18:59.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835340+0000", 2023-10-09T06:18:59.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010699220000000001 2023-10-09T06:18:59.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:18:59.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.137949+0000", 2023-10-09T06:18:59.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.302608936 2023-10-09T06:18:59.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:18:59.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.150776+0000", 2023-10-09T06:18:59.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012827191999999999 2023-10-09T06:18:59.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:18:59.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.150811+0000", 2023-10-09T06:18:59.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5076000000000001e-05 2023-10-09T06:18:59.479 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.479 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:18:59.480 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.480 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:83036 10.4 e514/490 10:22f39e74:::benchmark_data_smithi064_573750_object83035:head v 514'5125, mlcod=513'5123)", 2023-10-09T06:18:59.480 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.835143+0000", 2023-10-09T06:18:59.480 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 357.58790846900001, 2023-10-09T06:18:59.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30683402399999998, 2023-10-09T06:18:59.480 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:18:59.480 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:59.480 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:18:59.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:18:59.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835143+0000", 2023-10-09T06:18:59.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:18:59.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835143+0000", 2023-10-09T06:18:59.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:18:59.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835143+0000", 2023-10-09T06:18:59.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:59.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.482 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:18:59.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835177+0000", 2023-10-09T06:18:59.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4174000000000002e-05 2023-10-09T06:18:59.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.482 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:18:59.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835188+0000", 2023-10-09T06:18:59.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0804e-05 2023-10-09T06:18:59.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:18:59.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835195+0000", 2023-10-09T06:18:59.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1239999999999999e-06 2023-10-09T06:18:59.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:18:59.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835246+0000", 2023-10-09T06:18:59.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1143999999999997e-05 2023-10-09T06:18:59.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:18:59.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835268+0000", 2023-10-09T06:18:59.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1865000000000002e-05 2023-10-09T06:18:59.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:18:59.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.141950+0000", 2023-10-09T06:18:59.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30668228400000003 2023-10-09T06:18:59.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:18:59.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.141977+0000", 2023-10-09T06:18:59.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7175e-05 2023-10-09T06:18:59.485 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.485 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:18:59.485 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.485 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.46283.0:83032 10.6 e514/490 ondisk, result = 0)", 2023-10-09T06:18:59.485 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.835214+0000", 2023-10-09T06:18:59.485 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 357.58783762799999, 2023-10-09T06:18:59.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30480471100000001, 2023-10-09T06:18:59.486 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:18:59.486 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:18:59.486 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:18:59.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:18:59.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835214+0000", 2023-10-09T06:18:59.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:18:59.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835214+0000", 2023-10-09T06:18:59.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:18:59.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835214+0000", 2023-10-09T06:18:59.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:59.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:18:59.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835217+0000", 2023-10-09T06:18:59.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3239999999999999e-06 2023-10-09T06:18:59.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:18:59.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835224+0000", 2023-10-09T06:18:59.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7859999999999999e-06 2023-10-09T06:18:59.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:18:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835229+0000", 2023-10-09T06:18:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3519999999999997e-06 2023-10-09T06:18:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:18:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140002+0000", 2023-10-09T06:18:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30477259499999998 2023-10-09T06:18:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:18:59.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140010+0000", 2023-10-09T06:18:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1359999999999997e-06 2023-10-09T06:18:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:18:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140019+0000", 2023-10-09T06:18:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8799999999999997e-06 2023-10-09T06:18:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:18:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.46283.0:83032 10.6 e514/490 ondisk, result = 0)", 2023-10-09T06:18:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.837038+0000", 2023-10-09T06:18:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 357.58601361799998, 2023-10-09T06:18:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30313466700000002, 2023-10-09T06:18:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:18:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:18:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:18:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:18:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.837038+0000", 2023-10-09T06:18:59.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:18:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.837038+0000", 2023-10-09T06:18:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:18:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.837037+0000", 2023-10-09T06:18:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:59.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:18:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.837042+0000", 2023-10-09T06:18:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1970000000000004e-06 2023-10-09T06:18:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:18:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.138000+0000", 2023-10-09T06:18:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.300957961 2023-10-09T06:18:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:18:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.138006+0000", 2023-10-09T06:18:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8449999999999999e-06 2023-10-09T06:18:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:18:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140066+0000", 2023-10-09T06:18:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020593970000000001 2023-10-09T06:18:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:18:59.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140071+0000", 2023-10-09T06:18:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4020000000000001e-06 2023-10-09T06:18:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:18:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140173+0000", 2023-10-09T06:18:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101576 2023-10-09T06:18:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:18:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.495 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.46283.0:83039 10.6 e514/490 ondisk, result = 0)", 2023-10-09T06:18:59.497 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.837813+0000", 2023-10-09T06:18:59.497 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 357.585238452, 2023-10-09T06:18:59.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.302235578, 2023-10-09T06:18:59.497 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:18:59.497 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:18:59.497 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:18:59.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:18:59.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.837813+0000", 2023-10-09T06:18:59.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.498 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.498 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:18:59.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.837813+0000", 2023-10-09T06:18:59.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.498 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.498 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:18:59.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.837813+0000", 2023-10-09T06:18:59.499 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:59.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.499 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:18:59.499 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.837818+0000", 2023-10-09T06:18:59.499 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0440000000000003e-06 2023-10-09T06:18:59.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.499 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:18:59.499 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.137814+0000", 2023-10-09T06:18:59.499 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.299996771 2023-10-09T06:18:59.500 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.500 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.500 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:18:59.500 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.137832+0000", 2023-10-09T06:18:59.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7198999999999999e-05 2023-10-09T06:18:59.500 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:18:59.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140037+0000", 2023-10-09T06:18:59.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0022049740000000002 2023-10-09T06:18:59.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:18:59.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140042+0000", 2023-10-09T06:18:59.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7189999999999998e-06 2023-10-09T06:18:59.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:18:59.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140049+0000", 2023-10-09T06:18:59.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5010000000000001e-06 2023-10-09T06:18:59.503 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.503 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:18:59.503 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.503 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:117969 10.c 10:31f340c0:::benchmark_data_smithi064_573750_object21754:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e536)", 2023-10-09T06:18:59.503 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:59.578658+0000", 2023-10-09T06:18:59.504 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 299.844393647, 2023-10-09T06:18:59.504 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.3295903, 2023-10-09T06:18:59.504 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:18:59.504 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:59.504 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:18:59.504 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:18:59.504 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:59.504 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 117969 2023-10-09T06:18:59.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.504 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:18:59.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:18:59.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.578658+0000", 2023-10-09T06:18:59.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:18:59.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.578658+0000", 2023-10-09T06:18:59.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:18:59.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.578657+0000", 2023-10-09T06:18:59.506 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:18:59.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:18:59.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.578662+0000", 2023-10-09T06:18:59.506 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.101e-06 2023-10-09T06:18:59.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:18:59.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.795223+0000", 2023-10-09T06:18:59.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21656120300000001 2023-10-09T06:18:59.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.507 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.507 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:18:59.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.795235+0000", 2023-10-09T06:18:59.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1979e-05 2023-10-09T06:18:59.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.507 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.507 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:18:59.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.795411+0000", 2023-10-09T06:18:59.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000175599 2023-10-09T06:18:59.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:18:59.508 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.795540+0000", 2023-10-09T06:18:59.508 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012918599999999999 2023-10-09T06:18:59.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,7", 2023-10-09T06:18:59.508 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.795671+0000", 2023-10-09T06:18:59.508 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000130681 2023-10-09T06:18:59.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:18:59.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.799483+0000", 2023-10-09T06:18:59.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038128010000000002 2023-10-09T06:18:59.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.907634+0000", 2023-10-09T06:18:59.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.108151071 2023-10-09T06:18:59.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.907752+0000", 2023-10-09T06:18:59.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011757 2023-10-09T06:18:59.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.908098+0000", 2023-10-09T06:18:59.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000346378 2023-10-09T06:18:59.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:18:59.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.908142+0000", 2023-10-09T06:18:59.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3297999999999998e-05 2023-10-09T06:18:59.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:18:59.511 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.908248+0000", 2023-10-09T06:18:59.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010660800000000001 2023-10-09T06:18:59.511 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.511 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:18:59.511 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.511 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:117972 10.d e536/522 10:b67d4907:::benchmark_data_smithi064_573750_object21757:head v 536'7359, mlcod=536'7355)", 2023-10-09T06:18:59.512 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:59.578779+0000", 2023-10-09T06:18:59.512 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 299.84427296899997, 2023-10-09T06:18:59.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33589861999999998, 2023-10-09T06:18:59.512 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:18:59.512 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:59.512 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:18:59.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:18:59.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.578779+0000", 2023-10-09T06:18:59.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.512 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.513 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:18:59.513 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.578779+0000", 2023-10-09T06:18:59.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.513 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.513 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:18:59.513 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.578778+0000", 2023-10-09T06:18:59.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:59.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.513 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.513 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:18:59.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.578786+0000", 2023-10-09T06:18:59.514 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9000000000000006e-06 2023-10-09T06:18:59.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:18:59.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.795187+0000", 2023-10-09T06:18:59.514 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21640057800000001 2023-10-09T06:18:59.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:18:59.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.795679+0000", 2023-10-09T06:18:59.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00049279300000000005 2023-10-09T06:18:59.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:18:59.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.796083+0000", 2023-10-09T06:18:59.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00040412299999999999 2023-10-09T06:18:59.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:18:59.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.796107+0000", 2023-10-09T06:18:59.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3122e-05 2023-10-09T06:18:59.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.516 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:18:59.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.914650+0000", 2023-10-09T06:18:59.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.118543438 2023-10-09T06:18:59.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.516 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:18:59.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.914677+0000", 2023-10-09T06:18:59.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7251e-05 2023-10-09T06:18:59.516 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.517 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:18:59.517 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.517 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:149281 10.6 10:61516434:::benchmark_data_smithi064_573750_object53066:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e550)", 2023-10-09T06:18:59.517 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:47.178748+0000", 2023-10-09T06:18:59.517 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 252.24430352300001, 2023-10-09T06:18:59.517 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.52331555600000002, 2023-10-09T06:18:59.517 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:18:59.517 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:59.517 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:18:59.517 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:18:59.518 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:59.518 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149281 2023-10-09T06:18:59.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.518 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:18:59.518 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.518 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:18:59.518 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.178748+0000", 2023-10-09T06:18:59.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.518 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.518 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:18:59.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.178748+0000", 2023-10-09T06:18:59.519 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:18:59.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.178747+0000", 2023-10-09T06:18:59.519 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:59.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:18:59.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.178752+0000", 2023-10-09T06:18:59.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.933e-06 2023-10-09T06:18:59.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.520 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:18:59.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.178764+0000", 2023-10-09T06:18:59.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2456999999999999e-05 2023-10-09T06:18:59.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.520 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:18:59.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.178771+0000", 2023-10-09T06:18:59.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8990000000000004e-06 2023-10-09T06:18:59.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:18:59.521 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.178818+0000", 2023-10-09T06:18:59.521 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6588999999999997e-05 2023-10-09T06:18:59.521 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:18:59.521 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.178961+0000", 2023-10-09T06:18:59.521 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000143387 2023-10-09T06:18:59.521 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:18:59.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.179114+0000", 2023-10-09T06:18:59.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015322099999999999 2023-10-09T06:18:59.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.190948+0000", 2023-10-09T06:18:59.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011833532000000001 2023-10-09T06:18:59.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:18:59.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.190956+0000", 2023-10-09T06:18:59.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9189999999999999e-06 2023-10-09T06:18:59.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.701981+0000", 2023-10-09T06:18:59.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.51102548599999997 2023-10-09T06:18:59.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:18:59.524 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.702025+0000", 2023-10-09T06:18:59.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3761999999999999e-05 2023-10-09T06:18:59.524 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.524 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.524 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:18:59.524 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.702064+0000", 2023-10-09T06:18:59.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8792999999999999e-05 2023-10-09T06:18:59.524 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.524 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:18:59.524 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.524 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.525 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:149284 10.a 10:5302fd94:::benchmark_data_smithi064_573750_object53069:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e550)", 2023-10-09T06:18:59.525 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:47.182910+0000", 2023-10-09T06:18:59.525 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 252.24014174000001, 2023-10-09T06:18:59.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.51486690099999999, 2023-10-09T06:18:59.525 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:18:59.525 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:59.525 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:18:59.525 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:18:59.525 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:59.525 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149284 2023-10-09T06:18:59.526 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.526 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:18:59.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:18:59.526 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.182910+0000", 2023-10-09T06:18:59.526 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.526 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:18:59.526 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.182910+0000", 2023-10-09T06:18:59.526 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.527 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.527 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.527 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:18:59.527 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.182908+0000", 2023-10-09T06:18:59.527 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:59.527 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.527 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.527 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:18:59.527 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.182919+0000", 2023-10-09T06:18:59.527 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0054999999999999e-05 2023-10-09T06:18:59.527 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.528 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:18:59.528 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.182931+0000", 2023-10-09T06:18:59.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2755000000000001e-05 2023-10-09T06:18:59.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.528 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:18:59.528 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.182942+0000", 2023-10-09T06:18:59.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1113e-05 2023-10-09T06:18:59.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:18:59.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.183001+0000", 2023-10-09T06:18:59.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8242999999999998e-05 2023-10-09T06:18:59.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:18:59.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.183149+0000", 2023-10-09T06:18:59.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014812700000000001 2023-10-09T06:18:59.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:18:59.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.183280+0000", 2023-10-09T06:18:59.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013154799999999999 2023-10-09T06:18:59.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.190818+0000", 2023-10-09T06:18:59.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0075373949999999997 2023-10-09T06:18:59.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:18:59.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.191769+0000", 2023-10-09T06:18:59.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00095116099999999996 2023-10-09T06:18:59.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.697692+0000", 2023-10-09T06:18:59.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.505923558 2023-10-09T06:18:59.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:18:59.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.697730+0000", 2023-10-09T06:18:59.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7725999999999998e-05 2023-10-09T06:18:59.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.532 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:18:59.532 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.697777+0000", 2023-10-09T06:18:59.532 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6589999999999999e-05 2023-10-09T06:18:59.532 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.532 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:18:59.532 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.532 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.532 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:149285 10.a 10:5b3d29d3:::benchmark_data_smithi064_573750_object53070:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e550)", 2023-10-09T06:18:59.532 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:47.182959+0000", 2023-10-09T06:18:59.533 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 252.24009237999999, 2023-10-09T06:18:59.533 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.51908162700000005, 2023-10-09T06:18:59.533 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:18:59.533 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:59.533 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:18:59.533 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:18:59.533 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:59.533 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149285 2023-10-09T06:18:59.533 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.533 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:18:59.533 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.534 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:18:59.534 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.182959+0000", 2023-10-09T06:18:59.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.534 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:18:59.534 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.182959+0000", 2023-10-09T06:18:59.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.534 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:18:59.535 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.182959+0000", 2023-10-09T06:18:59.535 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:59.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.535 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:18:59.535 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.182962+0000", 2023-10-09T06:18:59.535 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1590000000000002e-06 2023-10-09T06:18:59.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.535 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:18:59.535 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.182970+0000", 2023-10-09T06:18:59.536 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2849999999999995e-06 2023-10-09T06:18:59.536 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.536 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.536 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:18:59.536 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.182976+0000", 2023-10-09T06:18:59.536 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7640000000000002e-06 2023-10-09T06:18:59.536 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.536 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.536 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:18:59.536 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.186933+0000", 2023-10-09T06:18:59.536 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039569169999999999 2023-10-09T06:18:59.537 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.537 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.537 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:18:59.537 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.187059+0000", 2023-10-09T06:18:59.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012567899999999999 2023-10-09T06:18:59.537 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.537 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.537 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:18:59.537 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.187183+0000", 2023-10-09T06:18:59.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012484799999999999 2023-10-09T06:18:59.537 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.538 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.538 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.190863+0000", 2023-10-09T06:18:59.538 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0036794879999999999 2023-10-09T06:18:59.538 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.538 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:18:59.538 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.195731+0000", 2023-10-09T06:18:59.538 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048682120000000002 2023-10-09T06:18:59.538 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.539 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.539 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.701948+0000", 2023-10-09T06:18:59.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.50621636000000003 2023-10-09T06:18:59.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.539 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.539 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:18:59.539 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.701995+0000", 2023-10-09T06:18:59.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7537000000000002e-05 2023-10-09T06:18:59.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.539 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.539 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:18:59.539 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.702041+0000", 2023-10-09T06:18:59.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5812000000000001e-05 2023-10-09T06:18:59.540 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.540 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:18:59.540 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.540 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.540 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.540 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:149305 10.6 10:6e58fed9:::benchmark_data_smithi064_573750_object53090:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e550)", 2023-10-09T06:18:59.540 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:47.217073+0000", 2023-10-09T06:18:59.540 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 252.205978149, 2023-10-09T06:18:59.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.485167132, 2023-10-09T06:18:59.540 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:18:59.541 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:59.541 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:18:59.541 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:18:59.541 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:59.541 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149305 2023-10-09T06:18:59.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.541 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:18:59.541 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.541 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:18:59.541 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.217073+0000", 2023-10-09T06:18:59.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.542 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:18:59.542 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.217073+0000", 2023-10-09T06:18:59.542 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.542 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:18:59.542 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.217060+0000", 2023-10-09T06:18:59.542 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999862 2023-10-09T06:18:59.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.543 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:18:59.543 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.217081+0000", 2023-10-09T06:18:59.543 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0916000000000001e-05 2023-10-09T06:18:59.543 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.543 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.543 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:18:59.543 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.217096+0000", 2023-10-09T06:18:59.543 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5381000000000001e-05 2023-10-09T06:18:59.543 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.543 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.543 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:18:59.543 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.217106+0000", 2023-10-09T06:18:59.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0044000000000001e-05 2023-10-09T06:18:59.544 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.544 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.544 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:18:59.544 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.217167+0000", 2023-10-09T06:18:59.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1054000000000002e-05 2023-10-09T06:18:59.544 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.544 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.544 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:18:59.544 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.217309+0000", 2023-10-09T06:18:59.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014138000000000001 2023-10-09T06:18:59.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.545 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:18:59.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.217451+0000", 2023-10-09T06:18:59.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014241899999999999 2023-10-09T06:18:59.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.545 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:18:59.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.221048+0000", 2023-10-09T06:18:59.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035971169999999999 2023-10-09T06:18:59.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.546 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.546 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.223777+0000", 2023-10-09T06:18:59.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027287029999999999 2023-10-09T06:18:59.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.546 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.546 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.702136+0000", 2023-10-09T06:18:59.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.47835958699999998 2023-10-09T06:18:59.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.547 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:18:59.547 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.702187+0000", 2023-10-09T06:18:59.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0717999999999998e-05 2023-10-09T06:18:59.547 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.547 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.547 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:18:59.547 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.702241+0000", 2023-10-09T06:18:59.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3541000000000001e-05 2023-10-09T06:18:59.547 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.547 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:18:59.547 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.547 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.548 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.548 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:149306 10.a 10:57b3c501:::benchmark_data_smithi064_573750_object53091:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e550)", 2023-10-09T06:18:59.548 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:47.221272+0000", 2023-10-09T06:18:59.548 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 252.201779571, 2023-10-09T06:18:59.548 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.48510954299999998, 2023-10-09T06:18:59.548 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:18:59.548 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:59.548 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:18:59.548 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:18:59.548 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:59.548 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149306 2023-10-09T06:18:59.548 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.549 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:18:59.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.549 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:18:59.549 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.221272+0000", 2023-10-09T06:18:59.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.549 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:18:59.549 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.221272+0000", 2023-10-09T06:18:59.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:18:59.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.221270+0000", 2023-10-09T06:18:59.550 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:18:59.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:18:59.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.221278+0000", 2023-10-09T06:18:59.550 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7079999999999997e-06 2023-10-09T06:18:59.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:18:59.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.221294+0000", 2023-10-09T06:18:59.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6487e-05 2023-10-09T06:18:59.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:18:59.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.221306+0000", 2023-10-09T06:18:59.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1522000000000001e-05 2023-10-09T06:18:59.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:18:59.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.221362+0000", 2023-10-09T06:18:59.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6122000000000002e-05 2023-10-09T06:18:59.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:18:59.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.221498+0000", 2023-10-09T06:18:59.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013583599999999999 2023-10-09T06:18:59.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:18:59.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.221641+0000", 2023-10-09T06:18:59.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014281799999999999 2023-10-09T06:18:59.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.225057+0000", 2023-10-09T06:18:59.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034167659999999999 2023-10-09T06:18:59.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:18:59.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.225187+0000", 2023-10-09T06:18:59.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012954899999999999 2023-10-09T06:18:59.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.706283+0000", 2023-10-09T06:18:59.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.48109601400000002 2023-10-09T06:18:59.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:18:59.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.706340+0000", 2023-10-09T06:18:59.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7290999999999997e-05 2023-10-09T06:18:59.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:18:59.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.706382+0000", 2023-10-09T06:18:59.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1409000000000003e-05 2023-10-09T06:18:59.555 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.555 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:18:59.555 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.555 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:149312 10.6 10:601aa7eb:::benchmark_data_smithi064_573750_object53097:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e550)", 2023-10-09T06:18:59.555 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:47.229107+0000", 2023-10-09T06:18:59.555 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 252.19394448700001, 2023-10-09T06:18:59.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.47326104800000002, 2023-10-09T06:18:59.555 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:18:59.556 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:59.556 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:18:59.556 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:18:59.556 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:59.556 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149312 2023-10-09T06:18:59.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.556 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:18:59.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:18:59.556 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.229107+0000", 2023-10-09T06:18:59.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.557 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.557 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.557 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:18:59.557 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.229107+0000", 2023-10-09T06:18:59.557 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.557 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.557 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.557 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:18:59.557 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.229106+0000", 2023-10-09T06:18:59.557 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:59.557 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.557 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.558 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:18:59.558 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.229114+0000", 2023-10-09T06:18:59.558 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0900000000000005e-06 2023-10-09T06:18:59.558 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.558 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.558 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:18:59.558 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.229128+0000", 2023-10-09T06:18:59.558 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4729000000000001e-05 2023-10-09T06:18:59.558 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.558 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.558 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:18:59.559 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.229138+0000", 2023-10-09T06:18:59.559 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3959999999999995e-06 2023-10-09T06:18:59.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.559 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:18:59.559 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.229200+0000", 2023-10-09T06:18:59.559 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2131000000000005e-05 2023-10-09T06:18:59.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.559 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:18:59.559 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.229360+0000", 2023-10-09T06:18:59.559 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000159937 2023-10-09T06:18:59.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:18:59.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.229510+0000", 2023-10-09T06:18:59.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015014999999999999 2023-10-09T06:18:59.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:18:59.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.233925+0000", 2023-10-09T06:18:59.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044149879999999999 2023-10-09T06:18:59.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.561 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.561 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.561 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.234917+0000", 2023-10-09T06:18:59.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0009924720000000001 2023-10-09T06:18:59.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.561 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.561 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.561 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.702286+0000", 2023-10-09T06:18:59.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.467368693 2023-10-09T06:18:59.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.561 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.561 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:18:59.562 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.702328+0000", 2023-10-09T06:18:59.562 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1792000000000002e-05 2023-10-09T06:18:59.562 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.562 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.562 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:18:59.562 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.702368+0000", 2023-10-09T06:18:59.562 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0249e-05 2023-10-09T06:18:59.562 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.562 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:18:59.562 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.562 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.563 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:149321 10.6 10:61199dae:::benchmark_data_smithi064_573750_object53106:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e550)", 2023-10-09T06:18:59.563 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:47.248777+0000", 2023-10-09T06:18:59.563 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 252.174274588, 2023-10-09T06:18:59.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.46306102300000002, 2023-10-09T06:18:59.563 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:18:59.563 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:59.563 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:18:59.563 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:18:59.563 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:18:59.563 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149321 2023-10-09T06:18:59.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.564 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:18:59.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:18:59.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.248777+0000", 2023-10-09T06:18:59.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:18:59.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.248777+0000", 2023-10-09T06:18:59.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:18:59.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:18:59.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.248775+0000", 2023-10-09T06:18:59.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:59.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:18:59.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.248783+0000", 2023-10-09T06:18:59.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8979999999999996e-06 2023-10-09T06:18:59.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.566 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:18:59.566 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.248798+0000", 2023-10-09T06:18:59.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4795e-05 2023-10-09T06:18:59.566 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.566 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:18:59.566 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.248806+0000", 2023-10-09T06:18:59.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3180000000000007e-06 2023-10-09T06:18:59.566 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.567 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:18:59.567 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.248857+0000", 2023-10-09T06:18:59.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0485999999999997e-05 2023-10-09T06:18:59.567 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.567 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:18:59.567 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.248969+0000", 2023-10-09T06:18:59.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000112391 2023-10-09T06:18:59.567 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.567 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:18:59.567 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.249059+0000", 2023-10-09T06:18:59.568 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9460000000000001e-05 2023-10-09T06:18:59.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.568 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.568 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:18:59.568 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.251756+0000", 2023-10-09T06:18:59.568 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002697554 2023-10-09T06:18:59.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.568 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.568 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.568 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.252079+0000", 2023-10-09T06:18:59.568 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00032264400000000002 2023-10-09T06:18:59.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:59.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.711752+0000", 2023-10-09T06:18:59.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.45967268500000003 2023-10-09T06:18:59.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:18:59.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.711794+0000", 2023-10-09T06:18:59.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2370999999999998e-05 2023-10-09T06:18:59.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:18:59.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:18:59.570 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:18:59.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.711838+0000", 2023-10-09T06:18:59.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3999999999999999e-05 2023-10-09T06:18:59.570 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.570 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:18:59.570 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.570 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:18:59.570 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:18:59.570 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:18:59.571 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:00.806 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:19:00.909 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T06:19:00.956 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:00.959 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:19:00.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:00.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:01.063 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:01.092 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:01.201 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:01.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:01.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:01.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:00 smithi064 ceph-mon[220044]: pgmap v1592: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 3885/153429 objects degraded (2.532%); 0 B/s, 143 objects/s recovering 2023-10-09T06:19:01.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:00 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4075578347' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:19:01.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:00 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:19:01.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:00 smithi064 ceph-mon[212750]: pgmap v1592: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 3885/153429 objects degraded (2.532%); 0 B/s, 143 objects/s recovering 2023-10-09T06:19:01.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:00 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4075578347' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:19:01.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:00 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:19:01.220 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:00 smithi156 ceph-mon[195101]: pgmap v1592: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 3885/153429 objects degraded (2.532%); 0 B/s, 143 objects/s recovering 2023-10-09T06:19:01.220 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:00 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4075578347' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:19:01.220 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:00 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:19:01.318 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:01.386 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:01.528 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:19:02.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:01 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:19:02.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:01 smithi156 ceph-mon[195101]: osdmap e620: 8 total, 8 up, 7 in 2023-10-09T06:19:02.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:01 smithi156 ceph-mon[195101]: failed to encode map e620 with expected crc 2023-10-09T06:19:02.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:01 smithi156 ceph-mon[195101]: failed to encode map e620 with expected crc 2023-10-09T06:19:02.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:01 smithi156 ceph-mon[195101]: failed to encode map e620 with expected crc 2023-10-09T06:19:02.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:01 smithi156 ceph-mon[195101]: failed to encode map e620 with expected crc 2023-10-09T06:19:02.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:01 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:19:02.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:01 smithi064 ceph-mon[212750]: osdmap e620: 8 total, 8 up, 7 in 2023-10-09T06:19:02.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:01 smithi064 ceph-mon[212750]: failed to encode map e620 with expected crc 2023-10-09T06:19:02.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:01 smithi064 ceph-mon[212750]: failed to encode map e620 with expected crc 2023-10-09T06:19:02.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:01 smithi064 ceph-mon[212750]: failed to encode map e620 with expected crc 2023-10-09T06:19:02.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:01 smithi064 ceph-mon[212750]: failed to encode map e620 with expected crc 2023-10-09T06:19:02.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:01 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:19:02.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:01 smithi064 ceph-mon[220044]: osdmap e620: 8 total, 8 up, 7 in 2023-10-09T06:19:02.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:01 smithi064 ceph-mon[220044]: failed to encode map e620 with expected crc 2023-10-09T06:19:02.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:01 smithi064 ceph-mon[220044]: failed to encode map e620 with expected crc 2023-10-09T06:19:02.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:01 smithi064 ceph-mon[220044]: failed to encode map e620 with expected crc 2023-10-09T06:19:02.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:01 smithi064 ceph-mon[220044]: failed to encode map e620 with expected crc 2023-10-09T06:19:02.271 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:02.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:02.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:02.378 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:02.438 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:02.991 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T06:19:03.185 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:03.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:03.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:03.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:02 smithi064 ceph-mon[212750]: pgmap v1594: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 908 op/s; 3885/141606 objects degraded (2.744%); 0 B/s, 120 objects/s recovering 2023-10-09T06:19:03.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:02 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 3885/141606 objects degraded (2.744%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:03.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:02 smithi064 ceph-mon[220044]: pgmap v1594: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 908 op/s; 3885/141606 objects degraded (2.744%); 0 B/s, 120 objects/s recovering 2023-10-09T06:19:03.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:02 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 3885/141606 objects degraded (2.744%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:03.291 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:03.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:02 smithi156 ceph-mon[195101]: pgmap v1594: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 908 op/s; 3885/141606 objects degraded (2.744%); 0 B/s, 120 objects/s recovering 2023-10-09T06:19:03.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:02 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 3885/141606 objects degraded (2.744%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:03.409 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:03.472 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:03.526 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:03.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:03.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:03.630 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:03.699 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:03.760 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:19:03.760 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:19:03.760 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:19:03.761 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.36591.0:173339 11.a e620/612 11:5cf7400e:::benchmark_data_smithi064_631903_object64425:head v 620'10745, mlcod=598'7308)", 2023-10-09T06:19:03.761 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:19:03.754484+0000", 2023-10-09T06:19:03.761 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 0.0044238890000000003, 2023-10-09T06:19:03.761 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0044572220000000003, 2023-10-09T06:19:03.761 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:19:03.761 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "started", 2023-10-09T06:19:03.761 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:19:03.761 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:19:03.761 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:19:03.761 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:03.754484+0000", 2023-10-09T06:19:03.761 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:19:03.761 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:19:03.762 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:19:03.762 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:19:03.762 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:03.754484+0000", 2023-10-09T06:19:03.762 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:19:03.762 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:19:03.762 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:19:03.762 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:19:03.762 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:03.754483+0000", 2023-10-09T06:19:03.762 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:19:03.762 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:19:03.762 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:19:03.762 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:19:03.763 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:03.754492+0000", 2023-10-09T06:19:03.763 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1980000000000007e-06 2023-10-09T06:19:03.763 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:19:03.763 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:19:03.763 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:19:03.763 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:03.754511+0000", 2023-10-09T06:19:03.763 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8663000000000001e-05 2023-10-09T06:19:03.763 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:19:03.763 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:19:03.763 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:19:03.763 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:03.754518+0000", 2023-10-09T06:19:03.763 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0879999999999997e-06 2023-10-09T06:19:03.764 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:19:03.764 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:19:03.764 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:19:03.764 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:03.754573+0000", 2023-10-09T06:19:03.764 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4463999999999999e-05 2023-10-09T06:19:03.764 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:19:03.764 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:19:03.764 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:19:03.764 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:03.754596+0000", 2023-10-09T06:19:03.764 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3183000000000001e-05 2023-10-09T06:19:03.764 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:19:03.764 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:19:03.765 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:19:03.765 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:19:03.765 INFO:teuthology.orchestra.run.smithi156.stdout: ], 2023-10-09T06:19:03.765 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 1 2023-10-09T06:19:03.765 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:19:04.137 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:04.138 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:04.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:04.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:04.215 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:19:04.244 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:04.250 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:19:04.301 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:19:04.343 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:19:04.344 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:04.375 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:19:04.417 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:19:04.453 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:19:04.496 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:19:04.507 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:19:04.508 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:19:04.508 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:19:04.508 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:19:04.508 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:19:04.508 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:19:04.508 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:19:04.508 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:19:04.702 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:19:05.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:04 smithi156 ceph-mon[195101]: pgmap v1595: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 707 op/s; 3885/141606 objects degraded (2.744%); 0 B/s, 94 objects/s recovering 2023-10-09T06:19:05.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:04 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1962036423' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:05.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:04 smithi064 ceph-mon[212750]: pgmap v1595: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 707 op/s; 3885/141606 objects degraded (2.744%); 0 B/s, 94 objects/s recovering 2023-10-09T06:19:05.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:04 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1962036423' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:05.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:04 smithi064 ceph-mon[220044]: pgmap v1595: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 707 op/s; 3885/141606 objects degraded (2.744%); 0 B/s, 94 objects/s recovering 2023-10-09T06:19:05.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:04 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1962036423' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:05.282 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:05.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:05.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:05.387 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:05.413 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:19:05.475 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:05.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:19:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:19:05] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.43.0" 2023-10-09T06:19:06.246 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:06.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:06.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:06.352 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:06.381 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:06.424 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:06.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:06.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:06.527 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:06.562 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:06.883 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:06.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:06.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:06.992 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:07.083 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:06 smithi064 ceph-mon[212750]: pgmap v1596: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 707 op/s; 3885/135381 objects degraded (2.870%); 0 B/s, 45 objects/s recovering 2023-10-09T06:19:07.087 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:06 smithi064 ceph-mon[220044]: pgmap v1596: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 707 op/s; 3885/135381 objects degraded (2.870%); 0 B/s, 45 objects/s recovering 2023-10-09T06:19:07.097 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:07.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:06 smithi156 ceph-mon[195101]: pgmap v1596: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 707 op/s; 3885/135381 objects degraded (2.870%); 0 B/s, 45 objects/s recovering 2023-10-09T06:19:07.197 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:07.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:07.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:07.304 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:07.354 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:07.538 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:07.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:07.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:07.596 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:19:07.596 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:19:07.645 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:07.727 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:07.886 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:07.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:07.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:07.956 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:19:07.994 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:08.029 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:08.087 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:08.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:08.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:08.191 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:08.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:07 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1794216470' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:08.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:07 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:08.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:07 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 3885/135381 objects degraded (2.870%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:08.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:07 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1794216470' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:08.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:07 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:08.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:07 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 3885/135381 objects degraded (2.870%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:08.223 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:07 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1794216470' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:08.223 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:07 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:08.223 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:07 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 3885/135381 objects degraded (2.870%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:08.226 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:08.269 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:08.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:08.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:08.373 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:08.434 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:08.449 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:19:08.665 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:19:08.679 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:08.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:08.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:08.786 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:08.877 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:08.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:08 smithi064 ceph-mon[212750]: pgmap v1597: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 796 op/s; 3885/129501 objects degraded (3.000%); 0 B/s, 81 objects/s recovering 2023-10-09T06:19:08.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:08 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:19:08.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:08 smithi064 ceph-mon[212750]: failed to encode map e621 with expected crc 2023-10-09T06:19:08.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:08 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:19:08.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:08 smithi064 ceph-mon[212750]: osdmap e621: 8 total, 8 up, 7 in 2023-10-09T06:19:08.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:08 smithi064 ceph-mon[212750]: failed to encode map e621 with expected crc 2023-10-09T06:19:08.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:08 smithi064 ceph-mon[212750]: failed to encode map e621 with expected crc 2023-10-09T06:19:08.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:08 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/656179660' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:19:08.950 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:08 smithi156 ceph-mon[195101]: pgmap v1597: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 796 op/s; 3885/129501 objects degraded (3.000%); 0 B/s, 81 objects/s recovering 2023-10-09T06:19:08.950 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:08 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:19:08.950 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:08 smithi156 ceph-mon[195101]: failed to encode map e621 with expected crc 2023-10-09T06:19:08.950 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:08 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:19:08.950 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:08 smithi156 ceph-mon[195101]: osdmap e621: 8 total, 8 up, 7 in 2023-10-09T06:19:08.950 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:08 smithi156 ceph-mon[195101]: failed to encode map e621 with expected crc 2023-10-09T06:19:08.951 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:08 smithi156 ceph-mon[195101]: failed to encode map e621 with expected crc 2023-10-09T06:19:08.951 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:08 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/656179660' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:19:08.966 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T06:19:09.005 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:09.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:09.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:09.118 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:09.197 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:09.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:08 smithi064 ceph-mon[220044]: pgmap v1597: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 796 op/s; 3885/129501 objects degraded (3.000%); 0 B/s, 81 objects/s recovering 2023-10-09T06:19:09.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:08 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:19:09.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:08 smithi064 ceph-mon[220044]: failed to encode map e621 with expected crc 2023-10-09T06:19:09.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:08 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:19:09.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:08 smithi064 ceph-mon[220044]: osdmap e621: 8 total, 8 up, 7 in 2023-10-09T06:19:09.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:08 smithi064 ceph-mon[220044]: failed to encode map e621 with expected crc 2023-10-09T06:19:09.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:08 smithi064 ceph-mon[220044]: failed to encode map e621 with expected crc 2023-10-09T06:19:09.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:08 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/656179660' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:19:09.299 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:09.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:09.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:09.356 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:19:09.415 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:09.448 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:09.506 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:09.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:09.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:09.610 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:09.655 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:09.757 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:09.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:09.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:09.870 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:09.934 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:09.936 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T06:19:10.114 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:10.157 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:10.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:10.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:10.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:09 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/656179660' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:19:10.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:09 smithi064 ceph-mon[212750]: osdmap e622: 8 total, 8 up, 7 in 2023-10-09T06:19:10.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:09 smithi064 ceph-mon[212750]: failed to encode map e622 with expected crc 2023-10-09T06:19:10.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:09 smithi064 ceph-mon[212750]: failed to encode map e622 with expected crc 2023-10-09T06:19:10.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:09 smithi064 ceph-mon[212750]: failed to encode map e622 with expected crc 2023-10-09T06:19:10.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:09 smithi064 ceph-mon[212750]: failed to encode map e622 with expected crc 2023-10-09T06:19:10.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:09 smithi064 ceph-mon[212750]: failed to encode map e622 with expected crc 2023-10-09T06:19:10.231 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:09 smithi064 ceph-mon[212750]: failed to encode map e622 with expected crc 2023-10-09T06:19:10.231 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:09 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/656179660' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:19:10.231 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:09 smithi064 ceph-mon[220044]: osdmap e622: 8 total, 8 up, 7 in 2023-10-09T06:19:10.231 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:09 smithi064 ceph-mon[220044]: failed to encode map e622 with expected crc 2023-10-09T06:19:10.231 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:09 smithi064 ceph-mon[220044]: failed to encode map e622 with expected crc 2023-10-09T06:19:10.231 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:09 smithi064 ceph-mon[220044]: failed to encode map e622 with expected crc 2023-10-09T06:19:10.231 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:09 smithi064 ceph-mon[220044]: failed to encode map e622 with expected crc 2023-10-09T06:19:10.231 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:09 smithi064 ceph-mon[220044]: failed to encode map e622 with expected crc 2023-10-09T06:19:10.231 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:09 smithi064 ceph-mon[220044]: failed to encode map e622 with expected crc 2023-10-09T06:19:10.265 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:10.290 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:09 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/656179660' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:19:10.290 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:09 smithi156 ceph-mon[195101]: osdmap e622: 8 total, 8 up, 7 in 2023-10-09T06:19:10.290 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:09 smithi156 ceph-mon[195101]: failed to encode map e622 with expected crc 2023-10-09T06:19:10.290 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:09 smithi156 ceph-mon[195101]: failed to encode map e622 with expected crc 2023-10-09T06:19:10.290 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:09 smithi156 ceph-mon[195101]: failed to encode map e622 with expected crc 2023-10-09T06:19:10.290 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:09 smithi156 ceph-mon[195101]: failed to encode map e622 with expected crc 2023-10-09T06:19:10.290 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:09 smithi156 ceph-mon[195101]: failed to encode map e622 with expected crc 2023-10-09T06:19:10.291 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:09 smithi156 ceph-mon[195101]: failed to encode map e622 with expected crc 2023-10-09T06:19:10.292 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:10.335 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:10.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:10.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:10.440 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:10.466 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:10.509 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:10.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:10.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:10.613 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:10.697 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:10.907 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:10.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:10.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:11.011 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:11.113 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:11.176 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:11.178 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:10 smithi156 ceph-mon[195101]: pgmap v1600: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 503 op/s; 3885/129501 objects degraded (3.000%); 0 B/s, 53 objects/s recovering 2023-10-09T06:19:11.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:11.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:11.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:10 smithi064 ceph-mon[212750]: pgmap v1600: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 503 op/s; 3885/129501 objects degraded (3.000%); 0 B/s, 53 objects/s recovering 2023-10-09T06:19:11.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:10 smithi064 ceph-mon[220044]: pgmap v1600: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 503 op/s; 3885/129501 objects degraded (3.000%); 0 B/s, 53 objects/s recovering 2023-10-09T06:19:11.280 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:11.318 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:11.579 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:19:11.580 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:19:11.661 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:19:11.670 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:11.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:11.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:11.776 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:11.779 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:11.825 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:11.916 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:11.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:11.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:12.026 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:12.072 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:12.174 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:12.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:12.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:12.280 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:12.334 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:12.574 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:12.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:12.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:12.588 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:19:12.683 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:12.780 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:12.787 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:19:12.855 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:19:12.893 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:19:12.937 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:19:12.975 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:19:13.012 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:19:13.055 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:19:13.090 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:19:13.094 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:19:13.094 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:19:13.094 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:19:13.094 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:19:13.094 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:19:13.095 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:19:13.095 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:19:13.095 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:19:13.126 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:12 smithi064 ceph-mon[212750]: pgmap v1601: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 983 op/s; 3885/117975 objects degraded (3.293%); 0 B/s, 112 objects/s recovering 2023-10-09T06:19:13.127 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:12 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2296150948' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:13.127 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:12 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 3885/117975 objects degraded (3.293%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:13.127 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:12 smithi064 ceph-mon[220044]: pgmap v1601: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 983 op/s; 3885/117975 objects degraded (3.293%); 0 B/s, 112 objects/s recovering 2023-10-09T06:19:13.127 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:12 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2296150948' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:13.127 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:12 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 3885/117975 objects degraded (3.293%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:13.241 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:13.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:13.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:13.269 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:13.348 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:13.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:12 smithi156 ceph-mon[195101]: pgmap v1601: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 983 op/s; 3885/117975 objects degraded (3.293%); 0 B/s, 112 objects/s recovering 2023-10-09T06:19:13.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:12 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2296150948' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:13.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:12 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 3885/117975 objects degraded (3.293%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:13.376 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:14.522 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:14.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:14.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:14.629 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:14.667 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:14.761 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:14.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:14.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:14.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:14 smithi156 ceph-mon[195101]: pgmap v1602: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 725 op/s; 3885/117975 objects degraded (3.293%); 0 B/s, 112 objects/s recovering 2023-10-09T06:19:14.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:14.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:14.876 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:14.881 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:14 smithi064 ceph-mon[212750]: pgmap v1602: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 725 op/s; 3885/117975 objects degraded (3.293%); 0 B/s, 112 objects/s recovering 2023-10-09T06:19:14.881 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:14.881 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:14.881 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:14 smithi064 ceph-mon[220044]: pgmap v1602: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 725 op/s; 3885/117975 objects degraded (3.293%); 0 B/s, 112 objects/s recovering 2023-10-09T06:19:14.881 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:14.881 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:14.917 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:15.021 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:15.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:15.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:15.124 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:15.155 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:15.305 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:15.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:15.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:15.409 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:15.434 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:19:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:19:15] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.43.0" 2023-10-09T06:19:15.436 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:15.481 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:15.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:15.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:15.563 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T06:19:15.584 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:15.621 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:15.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:15 smithi064 ceph-mon[212750]: osdmap e623: 8 total, 8 up, 7 in 2023-10-09T06:19:15.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:15 smithi064 ceph-mon[212750]: failed to encode map e623 with expected crc 2023-10-09T06:19:15.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:15 smithi064 ceph-mon[212750]: failed to encode map e623 with expected crc 2023-10-09T06:19:15.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:15 smithi064 ceph-mon[212750]: failed to encode map e623 with expected crc 2023-10-09T06:19:15.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:15 smithi064 ceph-mon[212750]: failed to encode map e623 with expected crc 2023-10-09T06:19:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:15 smithi064 ceph-mon[212750]: failed to encode map e623 with expected crc 2023-10-09T06:19:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:15 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/576393742' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:19:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:15 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:19:15.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:15 smithi064 ceph-mon[220044]: osdmap e623: 8 total, 8 up, 7 in 2023-10-09T06:19:15.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:15 smithi064 ceph-mon[220044]: failed to encode map e623 with expected crc 2023-10-09T06:19:15.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:15 smithi064 ceph-mon[220044]: failed to encode map e623 with expected crc 2023-10-09T06:19:15.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:15 smithi064 ceph-mon[220044]: failed to encode map e623 with expected crc 2023-10-09T06:19:15.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:15 smithi064 ceph-mon[220044]: failed to encode map e623 with expected crc 2023-10-09T06:19:15.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:15 smithi064 ceph-mon[220044]: failed to encode map e623 with expected crc 2023-10-09T06:19:15.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:15 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/576393742' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:19:15.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:15 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:19:15.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:15 smithi156 ceph-mon[195101]: osdmap e623: 8 total, 8 up, 7 in 2023-10-09T06:19:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:15 smithi156 ceph-mon[195101]: failed to encode map e623 with expected crc 2023-10-09T06:19:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:15 smithi156 ceph-mon[195101]: failed to encode map e623 with expected crc 2023-10-09T06:19:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:15 smithi156 ceph-mon[195101]: failed to encode map e623 with expected crc 2023-10-09T06:19:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:15 smithi156 ceph-mon[195101]: failed to encode map e623 with expected crc 2023-10-09T06:19:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:15 smithi156 ceph-mon[195101]: failed to encode map e623 with expected crc 2023-10-09T06:19:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:15 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/576393742' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:19:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:15 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:19:16.314 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:16.493 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:16.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:16.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:16.561 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:16 smithi064 ceph-mon[212750]: pgmap v1604: 119 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 789 op/s; 1971/111984 objects degraded (1.760%); 0 B/s, 69 objects/s recovering 2023-10-09T06:19:16.561 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:16 smithi064 ceph-mon[212750]: failed to encode map e624 with expected crc 2023-10-09T06:19:16.561 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:16 smithi064 ceph-mon[212750]: failed to encode map e624 with expected crc 2023-10-09T06:19:16.562 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:16 smithi064 ceph-mon[212750]: failed to encode map e624 with expected crc 2023-10-09T06:19:16.562 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:16 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:19:16.562 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:16 smithi064 ceph-mon[212750]: osdmap e624: 8 total, 8 up, 7 in 2023-10-09T06:19:16.562 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:16 smithi064 ceph-mon[212750]: failed to encode map e624 with expected crc 2023-10-09T06:19:16.562 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:16 smithi064 ceph-mon[212750]: failed to encode map e624 with expected crc 2023-10-09T06:19:16.562 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:16 smithi064 ceph-mon[220044]: pgmap v1604: 119 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 789 op/s; 1971/111984 objects degraded (1.760%); 0 B/s, 69 objects/s recovering 2023-10-09T06:19:16.562 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:16 smithi064 ceph-mon[220044]: failed to encode map e624 with expected crc 2023-10-09T06:19:16.562 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:16 smithi064 ceph-mon[220044]: failed to encode map e624 with expected crc 2023-10-09T06:19:16.562 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:16 smithi064 ceph-mon[220044]: failed to encode map e624 with expected crc 2023-10-09T06:19:16.563 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:16 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:19:16.563 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:16 smithi064 ceph-mon[220044]: osdmap e624: 8 total, 8 up, 7 in 2023-10-09T06:19:16.563 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:16 smithi064 ceph-mon[220044]: failed to encode map e624 with expected crc 2023-10-09T06:19:16.600 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:16.602 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T06:19:16.610 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:19:16.713 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:16.826 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:16 smithi064 ceph-mon[220044]: failed to encode map e624 with expected crc 2023-10-09T06:19:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:16 smithi156 ceph-mon[195101]: pgmap v1604: 119 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 789 op/s; 1971/111984 objects degraded (1.760%); 0 B/s, 69 objects/s recovering 2023-10-09T06:19:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:16 smithi156 ceph-mon[195101]: failed to encode map e624 with expected crc 2023-10-09T06:19:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:16 smithi156 ceph-mon[195101]: failed to encode map e624 with expected crc 2023-10-09T06:19:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:16 smithi156 ceph-mon[195101]: failed to encode map e624 with expected crc 2023-10-09T06:19:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:16 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:19:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:16 smithi156 ceph-mon[195101]: osdmap e624: 8 total, 8 up, 7 in 2023-10-09T06:19:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:16 smithi156 ceph-mon[195101]: failed to encode map e624 with expected crc 2023-10-09T06:19:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:16 smithi156 ceph-mon[195101]: failed to encode map e624 with expected crc 2023-10-09T06:19:17.248 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:17.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:17.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:17.354 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:17.412 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:17.534 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:17.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:17.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:17.644 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:17.706 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:17.993 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:17.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:17.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:18.098 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:18.213 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:18.264 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:18.299 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:18.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:18.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:18.405 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:18.452 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:18.565 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:18 smithi064 ceph-mon[212750]: pgmap v1606: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 2.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 964 op/s; 1971/106350 objects degraded (1.853%); 0 B/s, 100 objects/s recovering 2023-10-09T06:19:18.565 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:18 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3501018493' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:19:18.565 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:18 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:19:18.565 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:18 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 1971/106350 objects degraded (1.853%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:19:18.571 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:18.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:18.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:18.602 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:19:18.688 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:18.705 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:18.797 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:18.826 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:18 smithi064 ceph-mon[220044]: pgmap v1606: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 2.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 964 op/s; 1971/106350 objects degraded (1.853%); 0 B/s, 100 objects/s recovering 2023-10-09T06:19:18.827 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:18 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3501018493' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:19:18.827 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:18 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:19:18.827 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:18 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 1971/106350 objects degraded (1.853%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:19:18.867 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:19:18.867 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:19:18.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.867 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36591.0:185190 11.3 e625/604 11:cca925fe:::benchmark_data_smithi064_631903_object76276:head v 625'11801, mlcod=625'11799)", 2023-10-09T06:19:18.867 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:19:18.857960+0000", 2023-10-09T06:19:18.868 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0087334990000000005, 2023-10-09T06:19:18.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0087584940000000003, 2023-10-09T06:19:18.868 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:19:18.868 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:19:18.868 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:19:18.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:19:18.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.857960+0000", 2023-10-09T06:19:18.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:18.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:19:18.869 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.857960+0000", 2023-10-09T06:19:18.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:18.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.869 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:19:18.869 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.857959+0000", 2023-10-09T06:19:18.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:19:18.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.869 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:19:18.869 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.857971+0000", 2023-10-09T06:19:18.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.254e-05 2023-10-09T06:19:18.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:19:18.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.857992+0000", 2023-10-09T06:19:18.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1231000000000001e-05 2023-10-09T06:19:18.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:19:18.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.858002+0000", 2023-10-09T06:19:18.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3530000000000001e-06 2023-10-09T06:19:18.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:18.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.858143+0000", 2023-10-09T06:19:18.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000141438 2023-10-09T06:19:18.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:19:18.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.858168+0000", 2023-10-09T06:19:18.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4913e-05 2023-10-09T06:19:18.871 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:18.871 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:18.872 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:18.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.872 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:185188 11.c 11:3c929ac9:::benchmark_data_smithi064_631903_object76274:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e625)", 2023-10-09T06:19:18.872 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:19:18.859810+0000", 2023-10-09T06:19:18.872 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0068835720000000001, 2023-10-09T06:19:18.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0069648280000000002, 2023-10-09T06:19:18.872 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:19:18.872 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:19:18.872 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:19:18.872 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:19:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:19:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 185188 2023-10-09T06:19:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:19:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:19:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.859810+0000", 2023-10-09T06:19:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:19:18.873 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.859810+0000", 2023-10-09T06:19:18.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:18.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.874 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.874 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:19:18.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.859808+0000", 2023-10-09T06:19:18.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T06:19:18.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.874 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.874 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:19:18.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.859818+0000", 2023-10-09T06:19:18.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0395e-05 2023-10-09T06:19:18.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:19:18.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.859838+0000", 2023-10-09T06:19:18.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0154999999999999e-05 2023-10-09T06:19:18.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:19:18.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.859850+0000", 2023-10-09T06:19:18.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1494e-05 2023-10-09T06:19:18.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:18.876 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.859928+0000", 2023-10-09T06:19:18.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8654999999999998e-05 2023-10-09T06:19:18.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:19:18.876 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.860055+0000", 2023-10-09T06:19:18.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012679400000000001 2023-10-09T06:19:18.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,7", 2023-10-09T06:19:18.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.860166+0000", 2023-10-09T06:19:18.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110559 2023-10-09T06:19:18.877 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:18.877 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:18.877 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:18.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.877 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36591.0:185192 11.3 e625/604 11:cd742de2:::benchmark_data_smithi064_631903_object76278:head v 625'11802, mlcod=625'11799)", 2023-10-09T06:19:18.877 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:19:18.863046+0000", 2023-10-09T06:19:18.877 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0036474770000000001, 2023-10-09T06:19:18.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037883819999999999, 2023-10-09T06:19:18.877 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:19:18.878 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:19:18.878 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:19:18.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:19:18.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.863046+0000", 2023-10-09T06:19:18.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:18.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:19:18.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.863046+0000", 2023-10-09T06:19:18.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:18.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:19:18.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.863045+0000", 2023-10-09T06:19:18.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:19:18.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:19:18.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.863058+0000", 2023-10-09T06:19:18.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3207000000000001e-05 2023-10-09T06:19:18.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:19:18.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.863069+0000", 2023-10-09T06:19:18.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0674e-05 2023-10-09T06:19:18.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:19:18.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.863077+0000", 2023-10-09T06:19:18.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1799999999999996e-06 2023-10-09T06:19:18.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:18.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.863204+0000", 2023-10-09T06:19:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000127145 2023-10-09T06:19:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:19:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.863225+0000", 2023-10-09T06:19:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0322999999999998e-05 2023-10-09T06:19:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.881 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.36591.0:185188 11.c e625/572)", 2023-10-09T06:19:18.882 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:19:18.866762+0000", 2023-10-09T06:19:18.882 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 4294967295.9999318, 2023-10-09T06:19:18.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012707800000000001, 2023-10-09T06:19:18.882 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:19:18.882 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-10-09T06:19:18.882 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:19:18.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:19:18.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.866762+0000", 2023-10-09T06:19:18.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:18.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:19:18.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.866762+0000", 2023-10-09T06:19:18.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:18.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:19:18.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.866761+0000", 2023-10-09T06:19:18.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:19:18.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:19:18.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.866767+0000", 2023-10-09T06:19:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8699999999999997e-06 2023-10-09T06:19:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:19:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.866784+0000", 2023-10-09T06:19:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7173999999999999e-05 2023-10-09T06:19:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:19:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:18.866792+0000", 2023-10-09T06:19:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0639999999999994e-06 2023-10-09T06:19:18.884 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:18.885 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:18.885 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:18.885 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:18.885 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-10-09T06:19:18.885 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 4 2023-10-09T06:19:18.885 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:19:18.885 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:18 smithi156 ceph-mon[195101]: pgmap v1606: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 2.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 964 op/s; 1971/106350 objects degraded (1.853%); 0 B/s, 100 objects/s recovering 2023-10-09T06:19:18.886 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:18 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3501018493' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:19:18.886 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:18 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:19:18.886 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:18 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 1971/106350 objects degraded (1.853%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:19:19.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:19 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:19.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:19 smithi156 ceph-mon[195101]: failed to encode map e625 with expected crc 2023-10-09T06:19:19.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:19 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:19:19.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:19 smithi156 ceph-mon[195101]: osdmap e625: 8 total, 8 up, 7 in 2023-10-09T06:19:19.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:19 smithi156 ceph-mon[195101]: failed to encode map e625 with expected crc 2023-10-09T06:19:19.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:19 smithi156 ceph-mon[195101]: failed to encode map e625 with expected crc 2023-10-09T06:19:19.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:19 smithi156 ceph-mon[195101]: failed to encode map e625 with expected crc 2023-10-09T06:19:19.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:19 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:19.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:19 smithi064 ceph-mon[212750]: failed to encode map e625 with expected crc 2023-10-09T06:19:19.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:19 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:19:19.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:19 smithi064 ceph-mon[212750]: osdmap e625: 8 total, 8 up, 7 in 2023-10-09T06:19:19.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:19 smithi064 ceph-mon[212750]: failed to encode map e625 with expected crc 2023-10-09T06:19:19.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:19 smithi064 ceph-mon[212750]: failed to encode map e625 with expected crc 2023-10-09T06:19:19.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:19 smithi064 ceph-mon[212750]: failed to encode map e625 with expected crc 2023-10-09T06:19:19.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:19 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:19.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:19 smithi064 ceph-mon[220044]: failed to encode map e625 with expected crc 2023-10-09T06:19:19.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:19 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:19:19.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:19 smithi064 ceph-mon[220044]: osdmap e625: 8 total, 8 up, 7 in 2023-10-09T06:19:19.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:19 smithi064 ceph-mon[220044]: failed to encode map e625 with expected crc 2023-10-09T06:19:19.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:19 smithi064 ceph-mon[220044]: failed to encode map e625 with expected crc 2023-10-09T06:19:19.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:19 smithi064 ceph-mon[220044]: failed to encode map e625 with expected crc 2023-10-09T06:19:20.668 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:20.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:20.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:20.772 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:20.814 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:20 smithi064 ceph-mon[212750]: pgmap v1608: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 2.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 1971/106350 objects degraded (1.853%); 0 B/s, 55 objects/s recovering 2023-10-09T06:19:20.814 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:20 smithi064 ceph-mon[220044]: pgmap v1608: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 2.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 1971/106350 objects degraded (1.853%); 0 B/s, 55 objects/s recovering 2023-10-09T06:19:20.853 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:20.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:20 smithi156 ceph-mon[195101]: pgmap v1608: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 2.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 1971/106350 objects degraded (1.853%); 0 B/s, 55 objects/s recovering 2023-10-09T06:19:21.435 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:21.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:21.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:21.474 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:19:21.541 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:21.595 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:21.677 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:21.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:21.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:21.792 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:21.844 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:21.899 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:21.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:21.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:22.007 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:22.035 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:22.111 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:19:22.135 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:22.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:22.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:22.249 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:22.319 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:22.590 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:22.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:22.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:22.714 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:22.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:22 smithi064 ceph-mon[212750]: pgmap v1609: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 1971/94008 objects degraded (2.097%); 0 B/s, 142 objects/s recovering 2023-10-09T06:19:22.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:22 smithi064 ceph-mon[220044]: pgmap v1609: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 1971/94008 objects degraded (2.097%); 0 B/s, 142 objects/s recovering 2023-10-09T06:19:22.793 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:22.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:22 smithi156 ceph-mon[195101]: pgmap v1609: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 1971/94008 objects degraded (2.097%); 0 B/s, 142 objects/s recovering 2023-10-09T06:19:22.906 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:19:23.015 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:19:23.060 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:19:23.093 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:19:23.128 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:19:23.164 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:19:23.218 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:19:23.268 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:19:23.302 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:19:23.302 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:19:23.303 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:19:23.303 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:19:23.303 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:19:23.303 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:19:23.303 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:19:23.303 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:19:23.303 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:23.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:23.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:23.320 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:23.411 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:23.499 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:23.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:23 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 1971/94008 objects degraded (2.097%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:19:23.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:23 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/24649500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:23.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:23 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 1971/94008 objects degraded (2.097%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:19:23.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:23 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/24649500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:23.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:23 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 1971/94008 objects degraded (2.097%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:19:23.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:23 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/24649500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:24.304 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:24.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:24.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:24.411 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:24.482 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:24.567 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:24.753 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:24.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:24.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:24.861 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:24.873 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T06:19:24.935 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:25.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:24 smithi156 ceph-mon[195101]: pgmap v1610: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 748 op/s; 1971/94008 objects degraded (2.097%); 0 B/s, 120 objects/s recovering 2023-10-09T06:19:25.122 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:24 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1887451113' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:19:25.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:24 smithi064 ceph-mon[212750]: pgmap v1610: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 748 op/s; 1971/94008 objects degraded (2.097%); 0 B/s, 120 objects/s recovering 2023-10-09T06:19:25.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:24 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1887451113' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:19:25.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:24 smithi064 ceph-mon[220044]: pgmap v1610: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 748 op/s; 1971/94008 objects degraded (2.097%); 0 B/s, 120 objects/s recovering 2023-10-09T06:19:25.223 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:24 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1887451113' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:19:25.365 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:25.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:25.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:25.478 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:25.547 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:25.673 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:25.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:25.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:25.693 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:19:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:19:25] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.43.0" 2023-10-09T06:19:25.788 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:25.847 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:25.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:25 smithi064 ceph-mon[212750]: failed to encode map e626 with expected crc 2023-10-09T06:19:25.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:25 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1887451113' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:19:25.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:25 smithi064 ceph-mon[212750]: osdmap e626: 8 total, 8 up, 7 in 2023-10-09T06:19:25.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:25 smithi064 ceph-mon[212750]: failed to encode map e626 with expected crc 2023-10-09T06:19:25.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:25 smithi064 ceph-mon[212750]: failed to encode map e626 with expected crc 2023-10-09T06:19:25.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:25 smithi064 ceph-mon[212750]: failed to encode map e626 with expected crc 2023-10-09T06:19:25.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:25 smithi064 ceph-mon[220044]: failed to encode map e626 with expected crc 2023-10-09T06:19:25.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:25 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1887451113' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:19:25.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:25 smithi064 ceph-mon[220044]: osdmap e626: 8 total, 8 up, 7 in 2023-10-09T06:19:25.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:25 smithi064 ceph-mon[220044]: failed to encode map e626 with expected crc 2023-10-09T06:19:25.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:25 smithi064 ceph-mon[220044]: failed to encode map e626 with expected crc 2023-10-09T06:19:25.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:25 smithi064 ceph-mon[220044]: failed to encode map e626 with expected crc 2023-10-09T06:19:26.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:25 smithi156 ceph-mon[195101]: failed to encode map e626 with expected crc 2023-10-09T06:19:26.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:25 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1887451113' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:19:26.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:25 smithi156 ceph-mon[195101]: osdmap e626: 8 total, 8 up, 7 in 2023-10-09T06:19:26.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:25 smithi156 ceph-mon[195101]: failed to encode map e626 with expected crc 2023-10-09T06:19:26.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:25 smithi156 ceph-mon[195101]: failed to encode map e626 with expected crc 2023-10-09T06:19:26.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:25 smithi156 ceph-mon[195101]: failed to encode map e626 with expected crc 2023-10-09T06:19:26.413 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:19:27.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:26 smithi064 ceph-mon[212750]: pgmap v1612: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 1.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 774 op/s; 1971/87762 objects degraded (2.246%); 0 B/s, 178 objects/s recovering 2023-10-09T06:19:27.222 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:26 smithi064 ceph-mon[220044]: pgmap v1612: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 1.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 774 op/s; 1971/87762 objects degraded (2.246%); 0 B/s, 178 objects/s recovering 2023-10-09T06:19:27.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:26 smithi156 ceph-mon[195101]: pgmap v1612: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 1.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 774 op/s; 1971/87762 objects degraded (2.246%); 0 B/s, 178 objects/s recovering 2023-10-09T06:19:27.377 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:27.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:27.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:27.490 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:27.517 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T06:19:27.606 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:27.747 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:27.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:27.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:27.798 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:19:27.798 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:19:27.857 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:27.904 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:27.968 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:27.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:27.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:28.074 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:28.106 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:28.149 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:27 smithi064 ceph-mon[212750]: Health check update: Degraded data redundancy: 1971/87762 objects degraded (2.246%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:19:28.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:27 smithi064 ceph-mon[212750]: osdmap e627: 8 total, 8 up, 7 in 2023-10-09T06:19:28.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:27 smithi064 ceph-mon[220044]: Health check update: Degraded data redundancy: 1971/87762 objects degraded (2.246%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:19:28.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:27 smithi064 ceph-mon[220044]: osdmap e627: 8 total, 8 up, 7 in 2023-10-09T06:19:28.163 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:28.167 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:27 smithi156 ceph-mon[195101]: Health check update: Degraded data redundancy: 1971/87762 objects degraded (2.246%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:19:28.167 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:27 smithi156 ceph-mon[195101]: osdmap e627: 8 total, 8 up, 7 in 2023-10-09T06:19:28.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:28.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:28.269 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:28.295 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:28.340 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:28.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:28.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:28.448 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:28.479 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:28.851 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:19:29.154 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:29.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:28 smithi064 ceph-mon[212750]: pgmap v1613: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 952 op/s; 1971/81327 objects degraded (2.424%); 0 B/s, 162 objects/s recovering 2023-10-09T06:19:29.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:28 smithi064 ceph-mon[212750]: failed to encode map e627 with expected crc 2023-10-09T06:19:29.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:28 smithi064 ceph-mon[212750]: failed to encode map e627 with expected crc 2023-10-09T06:19:29.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:28 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3116869273' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:29.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:28 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:29.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:28 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:29.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:28 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:29.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:28 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:29.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:28 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:19:29.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:28 smithi064 ceph-mon[212750]: osdmap e628: 8 total, 8 up, 7 in 2023-10-09T06:19:29.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:28 smithi064 ceph-mon[220044]: pgmap v1613: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 952 op/s; 1971/81327 objects degraded (2.424%); 0 B/s, 162 objects/s recovering 2023-10-09T06:19:29.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:28 smithi064 ceph-mon[220044]: failed to encode map e627 with expected crc 2023-10-09T06:19:29.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:28 smithi064 ceph-mon[220044]: failed to encode map e627 with expected crc 2023-10-09T06:19:29.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:28 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3116869273' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:29.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:28 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:29.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:28 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:29.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:28 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:29.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:28 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:29.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:28 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:19:29.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:28 smithi064 ceph-mon[220044]: osdmap e628: 8 total, 8 up, 7 in 2023-10-09T06:19:29.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:29.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:29.266 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:29.305 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:28 smithi156 ceph-mon[195101]: pgmap v1613: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 952 op/s; 1971/81327 objects degraded (2.424%); 0 B/s, 162 objects/s recovering 2023-10-09T06:19:29.305 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:28 smithi156 ceph-mon[195101]: failed to encode map e627 with expected crc 2023-10-09T06:19:29.305 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:28 smithi156 ceph-mon[195101]: failed to encode map e627 with expected crc 2023-10-09T06:19:29.305 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:28 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3116869273' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:29.305 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:28 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:29.305 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:28 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:29.306 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:28 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:29.306 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:28 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:29.306 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:28 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:19:29.306 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:28 smithi156 ceph-mon[195101]: osdmap e628: 8 total, 8 up, 7 in 2023-10-09T06:19:29.314 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:29.380 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:29.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:29.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:29.487 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:29.550 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:30.188 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:30.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:29 smithi064 ceph-mon[212750]: failed to encode map e628 with expected crc 2023-10-09T06:19:30.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:29 smithi064 ceph-mon[212750]: failed to encode map e628 with expected crc 2023-10-09T06:19:30.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:29 smithi064 ceph-mon[212750]: failed to encode map e628 with expected crc 2023-10-09T06:19:30.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:29 smithi064 ceph-mon[212750]: failed to encode map e628 with expected crc 2023-10-09T06:19:30.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:29 smithi064 ceph-mon[220044]: failed to encode map e628 with expected crc 2023-10-09T06:19:30.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:29 smithi064 ceph-mon[220044]: failed to encode map e628 with expected crc 2023-10-09T06:19:30.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:29 smithi064 ceph-mon[220044]: failed to encode map e628 with expected crc 2023-10-09T06:19:30.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:29 smithi064 ceph-mon[220044]: failed to encode map e628 with expected crc 2023-10-09T06:19:30.218 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:19:30.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:29 smithi156 ceph-mon[195101]: failed to encode map e628 with expected crc 2023-10-09T06:19:30.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:29 smithi156 ceph-mon[195101]: failed to encode map e628 with expected crc 2023-10-09T06:19:30.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:29 smithi156 ceph-mon[195101]: failed to encode map e628 with expected crc 2023-10-09T06:19:30.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:29 smithi156 ceph-mon[195101]: failed to encode map e628 with expected crc 2023-10-09T06:19:30.372 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:30.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:30.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:30.481 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:30.565 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:30.713 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:30.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:30.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:30.829 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:30.882 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:30.957 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:30.967 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:30.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:30.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:30.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:30 smithi064 ceph-mon[212750]: pgmap v1616: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 1971/81327 objects degraded (2.424%); 0 B/s, 125 objects/s recovering 2023-10-09T06:19:30.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:30 smithi064 ceph-mon[220044]: pgmap v1616: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 1971/81327 objects degraded (2.424%); 0 B/s, 125 objects/s recovering 2023-10-09T06:19:31.074 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:31.154 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:31.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:30 smithi156 ceph-mon[195101]: pgmap v1616: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 1971/81327 objects degraded (2.424%); 0 B/s, 125 objects/s recovering 2023-10-09T06:19:32.044 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:32.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:32.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:32.149 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:32.178 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:32.193 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:32.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:31 smithi064 ceph-mon[220044]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1971/81327 objects degraded (2.424%), 1 pg degraded) 2023-10-09T06:19:32.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:31 smithi064 ceph-mon[212750]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1971/81327 objects degraded (2.424%), 1 pg degraded) 2023-10-09T06:19:32.266 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:32.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:32.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:32.283 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:31 smithi156 ceph-mon[195101]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1971/81327 objects degraded (2.424%), 1 pg degraded) 2023-10-09T06:19:32.384 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:32.487 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:32.493 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:19:32.786 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:32.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:32.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:32.896 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:32.968 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:33.201 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:32 smithi064 ceph-mon[212750]: pgmap v1617: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s; 0 B/s, 132 objects/s recovering 2023-10-09T06:19:33.203 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:32 smithi064 ceph-mon[220044]: pgmap v1617: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s; 0 B/s, 132 objects/s recovering 2023-10-09T06:19:33.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:32 smithi156 ceph-mon[195101]: pgmap v1617: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s; 0 B/s, 132 objects/s recovering 2023-10-09T06:19:33.643 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:33.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:33.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:33.753 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:33.780 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:33.825 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:33.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:33.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:33.930 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:33.993 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:34.049 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:34.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:34.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:34.092 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:19:34.126 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:19:34.154 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:34.165 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:19:34.190 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:34.196 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:19:34.216 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:19:34.234 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:34.236 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:19:34.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:34.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:34.266 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:19:34.315 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:19:34.326 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:19:34.326 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:19:34.327 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:19:34.327 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:19:34.327 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:19:34.327 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:19:34.327 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:19:34.327 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:19:34.342 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:34.377 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:34.452 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:34.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:34.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:34.558 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:34.584 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:34.626 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:34.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:34.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:34.734 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:34.824 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:34.926 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:34.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:34.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:35.037 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:35.099 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:35.182 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:35.187 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:34 smithi156 ceph-mon[195101]: pgmap v1618: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 761 op/s; 0 B/s, 107 objects/s recovering 2023-10-09T06:19:35.188 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:34 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1265745965' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:35.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:35.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:35.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:34 smithi064 ceph-mon[212750]: pgmap v1618: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 761 op/s; 0 B/s, 107 objects/s recovering 2023-10-09T06:19:35.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:34 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1265745965' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:35.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:34 smithi064 ceph-mon[220044]: pgmap v1618: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 761 op/s; 0 B/s, 107 objects/s recovering 2023-10-09T06:19:35.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:34 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1265745965' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:35.289 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:35.353 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:35.453 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:35.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:35.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:35.569 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:35.629 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:35.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:19:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:19:35] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.43.0" 2023-10-09T06:19:35.762 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:35.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:35.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:35.878 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:35.937 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:36.025 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:36.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:36.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:36.112 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:36 smithi064 ceph-mon[220044]: pgmap v1619: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.3 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 759 op/s; 0 B/s, 107 objects/s recovering 2023-10-09T06:19:36.112 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:36 smithi064 ceph-mon[220044]: Health check failed: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T06:19:36.113 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:36 smithi064 ceph-mon[212750]: pgmap v1619: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.3 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 759 op/s; 0 B/s, 107 objects/s recovering 2023-10-09T06:19:36.113 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:36 smithi064 ceph-mon[212750]: Health check failed: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T06:19:36.135 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:36.182 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:36.201 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:36.269 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:36.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:36.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:36.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:36 smithi156 ceph-mon[195101]: pgmap v1619: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.3 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 759 op/s; 0 B/s, 107 objects/s recovering 2023-10-09T06:19:36.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:36 smithi156 ceph-mon[195101]: Health check failed: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T06:19:36.377 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:36.426 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:36.477 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:19:36.477 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T06:19:36.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T06:19:36.477 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:19:36.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.477 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:95180 10.b 10:dd6284c2:::benchmark_data_smithi064_573750_object95179:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e521)", 2023-10-09T06:19:36.478 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:20.909957+0000", 2023-10-09T06:19:36.478 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 375.55967609700002, 2023-10-09T06:19:36.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.947192375, 2023-10-09T06:19:36.478 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:19:36.478 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:36.478 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:19:36.478 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:19:36.478 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:19:36.478 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 95180 2023-10-09T06:19:36.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.478 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:19:36.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:19:36.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.909957+0000", 2023-10-09T06:19:36.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:19:36.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.909957+0000", 2023-10-09T06:19:36.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:19:36.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.909955+0000", 2023-10-09T06:19:36.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:19:36.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:19:36.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910013+0000", 2023-10-09T06:19:36.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7899000000000002e-05 2023-10-09T06:19:36.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:19:36.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910023+0000", 2023-10-09T06:19:36.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7739999999999999e-06 2023-10-09T06:19:36.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:19:36.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910031+0000", 2023-10-09T06:19:36.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3299999999999999e-06 2023-10-09T06:19:36.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.914656+0000", 2023-10-09T06:19:36.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046253429999999996 2023-10-09T06:19:36.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.482 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.914667+0000", 2023-10-09T06:19:36.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0550000000000001e-05 2023-10-09T06:19:36.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.482 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.708405+0000", 2023-10-09T06:19:36.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.79373826700000005 2023-10-09T06:19:36.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.708416+0000", 2023-10-09T06:19:36.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0716999999999999e-05 2023-10-09T06:19:36.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.841829+0000", 2023-10-09T06:19:36.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.133413338 2023-10-09T06:19:36.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:19:36.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.842001+0000", 2023-10-09T06:19:36.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000171738 2023-10-09T06:19:36.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,6", 2023-10-09T06:19:36.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.842131+0000", 2023-10-09T06:19:36.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000130011 2023-10-09T06:19:36.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:19:36.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.845569+0000", 2023-10-09T06:19:36.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034383439999999999 2023-10-09T06:19:36.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.846409+0000", 2023-10-09T06:19:36.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00083989199999999996 2023-10-09T06:19:36.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.847475+0000", 2023-10-09T06:19:36.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001065727 2023-10-09T06:19:36.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.856993+0000", 2023-10-09T06:19:36.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0095179770000000004 2023-10-09T06:19:36.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:19:36.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.857042+0000", 2023-10-09T06:19:36.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8761000000000003e-05 2023-10-09T06:19:36.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:19:36.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.857149+0000", 2023-10-09T06:19:36.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010726 2023-10-09T06:19:36.487 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.487 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:36.487 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.487 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:95187 10.0 10:0c9801d5:::benchmark_data_smithi064_573750_object95186:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e521)", 2023-10-09T06:19:36.487 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:20.910037+0000", 2023-10-09T06:19:36.487 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 375.55959531600001, 2023-10-09T06:19:36.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9651730569999999, 2023-10-09T06:19:36.487 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:19:36.487 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:36.488 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:19:36.488 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:19:36.488 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:19:36.488 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 95187 2023-10-09T06:19:36.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.488 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:19:36.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:19:36.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910037+0000", 2023-10-09T06:19:36.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:19:36.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910037+0000", 2023-10-09T06:19:36.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:19:36.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910037+0000", 2023-10-09T06:19:36.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:19:36.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:19:36.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910083+0000", 2023-10-09T06:19:36.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6375999999999998e-05 2023-10-09T06:19:36.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:19:36.490 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910091+0000", 2023-10-09T06:19:36.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2160000000000006e-06 2023-10-09T06:19:36.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:19:36.490 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910098+0000", 2023-10-09T06:19:36.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.494e-06 2023-10-09T06:19:36.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.915482+0000", 2023-10-09T06:19:36.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053844110000000004 2023-10-09T06:19:36.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.915491+0000", 2023-10-09T06:19:36.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7260000000000008e-06 2023-10-09T06:19:36.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.706280+0000", 2023-10-09T06:19:36.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.79078928900000001 2023-10-09T06:19:36.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.706293+0000", 2023-10-09T06:19:36.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2764e-05 2023-10-09T06:19:36.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.852061+0000", 2023-10-09T06:19:36.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.145767838 2023-10-09T06:19:36.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:19:36.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.852193+0000", 2023-10-09T06:19:36.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013227700000000001 2023-10-09T06:19:36.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,6", 2023-10-09T06:19:36.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.852336+0000", 2023-10-09T06:19:36.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014296400000000001 2023-10-09T06:19:36.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:19:36.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.866052+0000", 2023-10-09T06:19:36.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013715597 2023-10-09T06:19:36.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.869816+0000", 2023-10-09T06:19:36.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003763775 2023-10-09T06:19:36.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.495 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.869866+0000", 2023-10-09T06:19:36.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0725e-05 2023-10-09T06:19:36.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.495 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.495 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.875094+0000", 2023-10-09T06:19:36.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0052276029999999999 2023-10-09T06:19:36.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.495 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:19:36.495 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.875143+0000", 2023-10-09T06:19:36.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9153999999999997e-05 2023-10-09T06:19:36.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:19:36.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.875210+0000", 2023-10-09T06:19:36.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7352000000000002e-05 2023-10-09T06:19:36.496 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.496 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:36.496 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.496 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:95188 10.0 10:02867f4b:::benchmark_data_smithi064_573750_object95187:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e521)", 2023-10-09T06:19:36.496 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:20.910103+0000", 2023-10-09T06:19:36.496 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 375.55952921300002, 2023-10-09T06:19:36.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.966474829, 2023-10-09T06:19:36.497 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:19:36.497 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:36.497 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:19:36.497 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:19:36.497 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:19:36.497 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 95188 2023-10-09T06:19:36.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.497 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:19:36.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:19:36.497 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910103+0000", 2023-10-09T06:19:36.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.498 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.498 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:19:36.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910103+0000", 2023-10-09T06:19:36.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.498 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.498 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:19:36.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910103+0000", 2023-10-09T06:19:36.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:19:36.498 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.499 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:19:36.499 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910149+0000", 2023-10-09T06:19:36.499 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5704999999999997e-05 2023-10-09T06:19:36.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.499 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:19:36.499 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910156+0000", 2023-10-09T06:19:36.499 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4399999999999999e-06 2023-10-09T06:19:36.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.499 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:19:36.499 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910162+0000", 2023-10-09T06:19:36.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.156e-06 2023-10-09T06:19:36.500 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.500 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.500 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.500 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.915511+0000", 2023-10-09T06:19:36.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053488800000000003 2023-10-09T06:19:36.500 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.500 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.500 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.500 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.915518+0000", 2023-10-09T06:19:36.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0350000000000004e-06 2023-10-09T06:19:36.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.706332+0000", 2023-10-09T06:19:36.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.79081385800000004 2023-10-09T06:19:36.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.706341+0000", 2023-10-09T06:19:36.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.02e-06 2023-10-09T06:19:36.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.855606+0000", 2023-10-09T06:19:36.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.149264721 2023-10-09T06:19:36.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:19:36.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.855749+0000", 2023-10-09T06:19:36.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014338600000000001 2023-10-09T06:19:36.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,6", 2023-10-09T06:19:36.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.855904+0000", 2023-10-09T06:19:36.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000154988 2023-10-09T06:19:36.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:19:36.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.866060+0000", 2023-10-09T06:19:36.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010156072 2023-10-09T06:19:36.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.869906+0000", 2023-10-09T06:19:36.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038456649999999998 2023-10-09T06:19:36.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.869946+0000", 2023-10-09T06:19:36.504 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0096999999999999e-05 2023-10-09T06:19:36.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.876416+0000", 2023-10-09T06:19:36.504 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0064703460000000001 2023-10-09T06:19:36.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:19:36.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.876471+0000", 2023-10-09T06:19:36.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4540000000000003e-05 2023-10-09T06:19:36.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:19:36.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.876578+0000", 2023-10-09T06:19:36.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000107477 2023-10-09T06:19:36.505 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.505 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:36.505 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.505 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:95189 10.0 10:029c3de1:::benchmark_data_smithi064_573750_object95188:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e521)", 2023-10-09T06:19:36.505 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:20.910168+0000", 2023-10-09T06:19:36.506 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 375.55946511399998, 2023-10-09T06:19:36.506 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9667635670000001, 2023-10-09T06:19:36.506 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:19:36.506 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:36.506 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:19:36.506 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:19:36.506 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:19:36.506 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 95189 2023-10-09T06:19:36.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.506 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:19:36.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:19:36.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910168+0000", 2023-10-09T06:19:36.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.507 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.507 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:19:36.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910168+0000", 2023-10-09T06:19:36.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.507 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.507 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:19:36.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910167+0000", 2023-10-09T06:19:36.508 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:19:36.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:19:36.508 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910213+0000", 2023-10-09T06:19:36.508 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5887999999999998e-05 2023-10-09T06:19:36.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:19:36.508 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910221+0000", 2023-10-09T06:19:36.508 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3189999999999995e-06 2023-10-09T06:19:36.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:19:36.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910229+0000", 2023-10-09T06:19:36.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7370000000000006e-06 2023-10-09T06:19:36.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.915537+0000", 2023-10-09T06:19:36.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.005308442 2023-10-09T06:19:36.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.915544+0000", 2023-10-09T06:19:36.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5180000000000001e-06 2023-10-09T06:19:36.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.706486+0000", 2023-10-09T06:19:36.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.79094211699999994 2023-10-09T06:19:36.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.706495+0000", 2023-10-09T06:19:36.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3740000000000004e-06 2023-10-09T06:19:36.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.511 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.859126+0000", 2023-10-09T06:19:36.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1526308730000001 2023-10-09T06:19:36.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:19:36.511 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.859307+0000", 2023-10-09T06:19:36.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000180693 2023-10-09T06:19:36.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,6", 2023-10-09T06:19:36.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.859479+0000", 2023-10-09T06:19:36.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017154599999999999 2023-10-09T06:19:36.512 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:19:36.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.869771+0000", 2023-10-09T06:19:36.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010292088 2023-10-09T06:19:36.512 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.869985+0000", 2023-10-09T06:19:36.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00021465399999999999 2023-10-09T06:19:36.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.513 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.513 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.513 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.870027+0000", 2023-10-09T06:19:36.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2157999999999999e-05 2023-10-09T06:19:36.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.513 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.513 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.513 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.876615+0000", 2023-10-09T06:19:36.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065879279999999998 2023-10-09T06:19:36.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.513 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:19:36.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.876698+0000", 2023-10-09T06:19:36.514 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2490999999999993e-05 2023-10-09T06:19:36.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:19:36.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.876931+0000", 2023-10-09T06:19:36.514 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00023325599999999999 2023-10-09T06:19:36.514 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.514 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:36.514 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.515 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:95194 10.0 10:09ebd0a1:::benchmark_data_smithi064_573750_object95193:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e521)", 2023-10-09T06:19:36.515 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:20.910235+0000", 2023-10-09T06:19:36.515 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 375.55939735300001, 2023-10-09T06:19:36.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9669212979999999, 2023-10-09T06:19:36.515 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:19:36.515 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:36.515 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:19:36.515 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:19:36.515 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:19:36.515 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 95194 2023-10-09T06:19:36.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.516 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:19:36.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.516 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:19:36.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910235+0000", 2023-10-09T06:19:36.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.516 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:19:36.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910235+0000", 2023-10-09T06:19:36.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.517 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:19:36.517 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910235+0000", 2023-10-09T06:19:36.517 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:19:36.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.517 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:19:36.517 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910285+0000", 2023-10-09T06:19:36.517 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0624000000000002e-05 2023-10-09T06:19:36.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.517 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:19:36.517 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910295+0000", 2023-10-09T06:19:36.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5200000000000003e-06 2023-10-09T06:19:36.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.518 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.518 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:19:36.518 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.910302+0000", 2023-10-09T06:19:36.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5800000000000003e-06 2023-10-09T06:19:36.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.518 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.518 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.518 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.915563+0000", 2023-10-09T06:19:36.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0052606229999999999 2023-10-09T06:19:36.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.915570+0000", 2023-10-09T06:19:36.519 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6250000000000001e-06 2023-10-09T06:19:36.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.706519+0000", 2023-10-09T06:19:36.519 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.79094940499999999 2023-10-09T06:19:36.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:21.706528+0000", 2023-10-09T06:19:36.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7579999999999998e-06 2023-10-09T06:19:36.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.520 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.862573+0000", 2023-10-09T06:19:36.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1560447110000001 2023-10-09T06:19:36.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.520 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:19:36.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.862770+0000", 2023-10-09T06:19:36.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019734200000000001 2023-10-09T06:19:36.521 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,6", 2023-10-09T06:19:36.521 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.862983+0000", 2023-10-09T06:19:36.521 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00021307699999999999 2023-10-09T06:19:36.521 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:19:36.521 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.869826+0000", 2023-10-09T06:19:36.521 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0068432709999999997 2023-10-09T06:19:36.521 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.874683+0000", 2023-10-09T06:19:36.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048568780000000002 2023-10-09T06:19:36.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.874719+0000", 2023-10-09T06:19:36.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5805e-05 2023-10-09T06:19:36.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.877016+0000", 2023-10-09T06:19:36.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0022973009999999999 2023-10-09T06:19:36.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:19:36.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.877069+0000", 2023-10-09T06:19:36.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3176e-05 2023-10-09T06:19:36.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:19:36.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:22.877157+0000", 2023-10-09T06:19:36.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7130000000000001e-05 2023-10-09T06:19:36.524 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.524 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:36.524 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.524 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.524 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.524 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:149238 11.7 11:ea0e1c7c:::benchmark_data_smithi064_631903_object40324:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e611)", 2023-10-09T06:19:36.524 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:18:32.009851+0000", 2023-10-09T06:19:36.524 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 64.459781484999993, 2023-10-09T06:19:36.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9125338629999999, 2023-10-09T06:19:36.524 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:19:36.524 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:36.524 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:19:36.525 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:19:36.525 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:19:36.525 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149238 2023-10-09T06:19:36.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.525 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:19:36.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.525 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:19:36.525 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.009851+0000", 2023-10-09T06:19:36.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.525 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:19:36.526 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.009851+0000", 2023-10-09T06:19:36.526 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.526 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:19:36.526 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.009850+0000", 2023-10-09T06:19:36.526 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:19:36.526 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:19:36.526 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.009855+0000", 2023-10-09T06:19:36.526 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7529999999999998e-06 2023-10-09T06:19:36.527 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.527 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.527 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:19:36.527 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.009867+0000", 2023-10-09T06:19:36.527 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2046e-05 2023-10-09T06:19:36.527 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.527 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.527 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:19:36.527 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.009876+0000", 2023-10-09T06:19:36.527 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.799e-06 2023-10-09T06:19:36.527 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.527 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.528 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.528 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.018722+0000", 2023-10-09T06:19:36.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0088460139999999993 2023-10-09T06:19:36.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.528 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.528 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.018732+0000", 2023-10-09T06:19:36.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5419999999999994e-06 2023-10-09T06:19:36.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.528 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.528 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.809671+0000", 2023-10-09T06:19:36.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.79093902199999999 2023-10-09T06:19:36.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.809680+0000", 2023-10-09T06:19:36.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1840000000000005e-06 2023-10-09T06:19:36.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.912542+0000", 2023-10-09T06:19:36.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.102862429 2023-10-09T06:19:36.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:19:36.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.912736+0000", 2023-10-09T06:19:36.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000193196 2023-10-09T06:19:36.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:19:36.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.912827+0000", 2023-10-09T06:19:36.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1614999999999996e-05 2023-10-09T06:19:36.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:19:36.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.922015+0000", 2023-10-09T06:19:36.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0091882019999999995 2023-10-09T06:19:36.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.922102+0000", 2023-10-09T06:19:36.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6458000000000005e-05 2023-10-09T06:19:36.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.922124+0000", 2023-10-09T06:19:36.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.181e-05 2023-10-09T06:19:36.532 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.532 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:19:36.532 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.922179+0000", 2023-10-09T06:19:36.532 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5220000000000003e-05 2023-10-09T06:19:36.532 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.532 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:19:36.532 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.922385+0000", 2023-10-09T06:19:36.532 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000206157 2023-10-09T06:19:36.532 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.532 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:36.532 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.533 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.533 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.533 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:149239 11.7 11:ebb20dd1:::benchmark_data_smithi064_631903_object40325:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e611)", 2023-10-09T06:19:36.533 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:18:32.009886+0000", 2023-10-09T06:19:36.533 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 64.459746908, 2023-10-09T06:19:36.533 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9213866159999999, 2023-10-09T06:19:36.533 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:19:36.533 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:36.533 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:19:36.533 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:19:36.533 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:19:36.533 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149239 2023-10-09T06:19:36.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.534 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:19:36.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.534 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:19:36.534 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.009886+0000", 2023-10-09T06:19:36.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.534 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:19:36.534 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.009886+0000", 2023-10-09T06:19:36.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.535 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:19:36.535 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.009885+0000", 2023-10-09T06:19:36.535 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:19:36.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.535 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:19:36.535 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.009888+0000", 2023-10-09T06:19:36.535 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.464e-06 2023-10-09T06:19:36.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.535 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:19:36.536 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.009893+0000", 2023-10-09T06:19:36.536 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5550000000000002e-06 2023-10-09T06:19:36.536 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.536 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.536 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:19:36.536 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.009898+0000", 2023-10-09T06:19:36.536 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7790000000000003e-06 2023-10-09T06:19:36.536 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.536 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.536 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.536 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.018759+0000", 2023-10-09T06:19:36.536 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0088610540000000002 2023-10-09T06:19:36.537 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.537 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.537 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.537 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.018767+0000", 2023-10-09T06:19:36.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9470000000000003e-06 2023-10-09T06:19:36.537 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.537 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.537 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.537 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.809716+0000", 2023-10-09T06:19:36.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.79094924099999997 2023-10-09T06:19:36.537 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.537 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.537 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.538 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.809721+0000", 2023-10-09T06:19:36.538 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7199999999999997e-06 2023-10-09T06:19:36.538 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.538 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.538 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.915871+0000", 2023-10-09T06:19:36.538 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1061499109999999 2023-10-09T06:19:36.538 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.538 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:19:36.538 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.916045+0000", 2023-10-09T06:19:36.538 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017384900000000001 2023-10-09T06:19:36.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.539 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.539 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:19:36.539 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.916208+0000", 2023-10-09T06:19:36.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016319300000000001 2023-10-09T06:19:36.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.539 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.539 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:19:36.539 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.922077+0000", 2023-10-09T06:19:36.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058689980000000003 2023-10-09T06:19:36.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.539 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.539 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.540 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.928531+0000", 2023-10-09T06:19:36.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0064541700000000004 2023-10-09T06:19:36.540 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.540 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.540 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.540 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.930960+0000", 2023-10-09T06:19:36.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024283849999999999 2023-10-09T06:19:36.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.541 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.541 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:19:36.542 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.931022+0000", 2023-10-09T06:19:36.542 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2762999999999995e-05 2023-10-09T06:19:36.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.542 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:19:36.542 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.931272+0000", 2023-10-09T06:19:36.542 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00024991800000000002 2023-10-09T06:19:36.542 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.542 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:36.542 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.543 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.543 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:149243 11.7 11:e22d562a:::benchmark_data_smithi064_631903_object40329:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e611)", 2023-10-09T06:19:36.543 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:18:32.010018+0000", 2023-10-09T06:19:36.543 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 64.459614778000002, 2023-10-09T06:19:36.543 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9214904229999998, 2023-10-09T06:19:36.543 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:19:36.543 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:36.543 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:19:36.543 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:19:36.543 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:19:36.543 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149243 2023-10-09T06:19:36.543 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.544 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:19:36.544 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.544 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:19:36.544 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.010018+0000", 2023-10-09T06:19:36.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.544 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.544 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.544 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:19:36.544 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.010018+0000", 2023-10-09T06:19:36.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.544 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.544 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.544 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:19:36.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.010017+0000", 2023-10-09T06:19:36.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:19:36.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.545 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:19:36.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.010021+0000", 2023-10-09T06:19:36.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3459999999999998e-06 2023-10-09T06:19:36.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.545 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:19:36.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.010028+0000", 2023-10-09T06:19:36.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2749999999999998e-06 2023-10-09T06:19:36.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.546 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:19:36.546 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.010034+0000", 2023-10-09T06:19:36.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1129999999999997e-06 2023-10-09T06:19:36.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.546 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.546 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.018953+0000", 2023-10-09T06:19:36.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0089192019999999993 2023-10-09T06:19:36.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.547 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.547 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.018963+0000", 2023-10-09T06:19:36.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0261000000000001e-05 2023-10-09T06:19:36.547 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.547 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.547 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.547 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.809734+0000", 2023-10-09T06:19:36.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.790770325 2023-10-09T06:19:36.547 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.547 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.547 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.548 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.809738+0000", 2023-10-09T06:19:36.548 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0029999999999996e-06 2023-10-09T06:19:36.548 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.548 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.548 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.548 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.919138+0000", 2023-10-09T06:19:36.548 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1094004529999999 2023-10-09T06:19:36.548 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.548 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.548 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:19:36.548 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.919309+0000", 2023-10-09T06:19:36.548 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017084700000000001 2023-10-09T06:19:36.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.549 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:19:36.549 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.919467+0000", 2023-10-09T06:19:36.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015822500000000001 2023-10-09T06:19:36.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.549 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:19:36.549 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.925115+0000", 2023-10-09T06:19:36.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056481609999999996 2023-10-09T06:19:36.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.928588+0000", 2023-10-09T06:19:36.550 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034726710000000001 2023-10-09T06:19:36.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.931327+0000", 2023-10-09T06:19:36.550 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027390679999999999 2023-10-09T06:19:36.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:19:36.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.931381+0000", 2023-10-09T06:19:36.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3792000000000003e-05 2023-10-09T06:19:36.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:19:36.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.931508+0000", 2023-10-09T06:19:36.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000127257 2023-10-09T06:19:36.551 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.551 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:36.551 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.551 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:149278 11.7 11:ed17d593:::benchmark_data_smithi064_631903_object40364:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e611)", 2023-10-09T06:19:36.552 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:18:32.062819+0000", 2023-10-09T06:19:36.552 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 64.406813540000002, 2023-10-09T06:19:36.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8781434099999998, 2023-10-09T06:19:36.552 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:19:36.552 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:36.552 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:19:36.552 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:19:36.552 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:19:36.552 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149278 2023-10-09T06:19:36.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.552 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:19:36.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:19:36.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.062819+0000", 2023-10-09T06:19:36.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:19:36.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.062819+0000", 2023-10-09T06:19:36.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:19:36.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.062818+0000", 2023-10-09T06:19:36.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:19:36.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:19:36.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.062824+0000", 2023-10-09T06:19:36.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5649999999999998e-06 2023-10-09T06:19:36.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:19:36.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.062843+0000", 2023-10-09T06:19:36.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8559e-05 2023-10-09T06:19:36.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:19:36.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.062851+0000", 2023-10-09T06:19:36.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9959999999999993e-06 2023-10-09T06:19:36.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.063417+0000", 2023-10-09T06:19:36.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00056580599999999997 2023-10-09T06:19:36.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.556 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.063427+0000", 2023-10-09T06:19:36.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0443e-05 2023-10-09T06:19:36.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.556 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.809752+0000", 2023-10-09T06:19:36.557 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.74632495499999996 2023-10-09T06:19:36.557 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.557 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.557 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.557 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:32.809756+0000", 2023-10-09T06:19:36.557 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1110000000000001e-06 2023-10-09T06:19:36.557 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.557 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.557 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.557 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.922417+0000", 2023-10-09T06:19:36.558 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1126606610000001 2023-10-09T06:19:36.558 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.558 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.558 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:19:36.558 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.922510+0000", 2023-10-09T06:19:36.558 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2847000000000001e-05 2023-10-09T06:19:36.558 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.558 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.558 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:19:36.558 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.922589+0000", 2023-10-09T06:19:36.558 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8893000000000005e-05 2023-10-09T06:19:36.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.559 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:19:36.559 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.925206+0000", 2023-10-09T06:19:36.559 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026177380000000001 2023-10-09T06:19:36.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.559 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.559 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.931900+0000", 2023-10-09T06:19:36.559 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0066942039999999996 2023-10-09T06:19:36.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.940732+0000", 2023-10-09T06:19:36.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0088312990000000008 2023-10-09T06:19:36.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:19:36.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.940773+0000", 2023-10-09T06:19:36.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1662000000000002e-05 2023-10-09T06:19:36.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:19:36.561 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:33.940962+0000", 2023-10-09T06:19:36.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018906700000000001 2023-10-09T06:19:36.561 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.561 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:36.561 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.561 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.561 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:166553 11.7 11:e2d99c95:::benchmark_data_smithi064_631903_object57639:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e616)", 2023-10-09T06:19:36.561 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:18:54.626587+0000", 2023-10-09T06:19:36.561 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 41.843045850999999, 2023-10-09T06:19:36.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.378149399, 2023-10-09T06:19:36.562 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:19:36.562 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:19:36.562 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:19:36.562 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:19:36.562 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:19:36.562 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 166553 2023-10-09T06:19:36.562 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.562 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:19:36.562 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.562 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:19:36.562 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.626587+0000", 2023-10-09T06:19:36.562 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:19:36.563 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.626587+0000", 2023-10-09T06:19:36.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:19:36.563 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.626586+0000", 2023-10-09T06:19:36.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:19:36.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:19:36.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.626590+0000", 2023-10-09T06:19:36.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1439999999999996e-06 2023-10-09T06:19:36.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:19:36.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.626599+0000", 2023-10-09T06:19:36.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2800000000000003e-06 2023-10-09T06:19:36.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:19:36.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.626605+0000", 2023-10-09T06:19:36.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0669999999999997e-06 2023-10-09T06:19:36.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.626669+0000", 2023-10-09T06:19:36.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4738000000000003e-05 2023-10-09T06:19:36.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:19:36.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.626834+0000", 2023-10-09T06:19:36.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000164568 2023-10-09T06:19:36.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.566 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:19:36.566 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.627010+0000", 2023-10-09T06:19:36.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000175799 2023-10-09T06:19:36.566 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.566 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.566 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.629949+0000", 2023-10-09T06:19:36.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002939601 2023-10-09T06:19:36.566 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.566 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:19:36.567 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.633998+0000", 2023-10-09T06:19:36.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040484249999999996 2023-10-09T06:19:36.567 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.567 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.567 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.644691+0000", 2023-10-09T06:19:36.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010692904 2023-10-09T06:19:36.567 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.567 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.567 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.644709+0000", 2023-10-09T06:19:36.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8233999999999999e-05 2023-10-09T06:19:36.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.568 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.568 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.568 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:55.999061+0000", 2023-10-09T06:19:36.568 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3543524200000001 2023-10-09T06:19:36.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.568 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.568 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-10-09T06:19:36.568 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:55.999203+0000", 2023-10-09T06:19:36.568 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000141707 2023-10-09T06:19:36.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.568 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.004666+0000", 2023-10-09T06:19:36.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0054626859999999996 2023-10-09T06:19:36.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:19:36.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.004736+0000", 2023-10-09T06:19:36.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0368000000000001e-05 2023-10-09T06:19:36.569 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.569 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:36.569 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.570 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.570 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:166570 11.7 11:e0083d6a:::benchmark_data_smithi064_631903_object57656:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e617)", 2023-10-09T06:19:36.570 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:18:54.645925+0000", 2023-10-09T06:19:36.570 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 41.823707957000003, 2023-10-09T06:19:36.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.363002528, 2023-10-09T06:19:36.570 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:19:36.570 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:36.570 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:19:36.570 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:19:36.570 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:19:36.570 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 166570 2023-10-09T06:19:36.571 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.571 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:19:36.571 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.571 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:19:36.571 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.645925+0000", 2023-10-09T06:19:36.571 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.571 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.571 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.571 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:19:36.571 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.645925+0000", 2023-10-09T06:19:36.571 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.571 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.572 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.572 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:19:36.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.645924+0000", 2023-10-09T06:19:36.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:19:36.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.572 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.572 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:19:36.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.645928+0000", 2023-10-09T06:19:36.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8229999999999997e-06 2023-10-09T06:19:36.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.572 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.572 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:19:36.573 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.645936+0000", 2023-10-09T06:19:36.573 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2290000000000003e-06 2023-10-09T06:19:36.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:19:36.573 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.645943+0000", 2023-10-09T06:19:36.573 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6989999999999997e-06 2023-10-09T06:19:36.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.573 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.646436+0000", 2023-10-09T06:19:36.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00049284299999999997 2023-10-09T06:19:36.574 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.574 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.574 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.646442+0000", 2023-10-09T06:19:36.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5640000000000002e-06 2023-10-09T06:19:36.574 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.574 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.574 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:55.999498+0000", 2023-10-09T06:19:36.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.353055906 2023-10-09T06:19:36.575 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.575 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.575 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:19:36.575 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:55.999613+0000", 2023-10-09T06:19:36.575 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000115201 2023-10-09T06:19:36.575 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.575 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.575 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:19:36.575 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:55.999731+0000", 2023-10-09T06:19:36.575 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000117328 2023-10-09T06:19:36.575 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.576 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.576 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.576 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.004444+0000", 2023-10-09T06:19:36.576 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047133039999999998 2023-10-09T06:19:36.576 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.576 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.576 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:19:36.576 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.004456+0000", 2023-10-09T06:19:36.576 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2284e-05 2023-10-09T06:19:36.576 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.576 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.576 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.577 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.008762+0000", 2023-10-09T06:19:36.577 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043062200000000004 2023-10-09T06:19:36.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:19:36.577 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.008819+0000", 2023-10-09T06:19:36.577 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6044000000000001e-05 2023-10-09T06:19:36.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:19:36.577 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.008927+0000", 2023-10-09T06:19:36.577 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108682 2023-10-09T06:19:36.578 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.578 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:36.578 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.578 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.578 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:166596 11.7 11:ed2a48c8:::benchmark_data_smithi064_631903_object57682:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e617)", 2023-10-09T06:19:36.578 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:18:54.695216+0000", 2023-10-09T06:19:36.578 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 41.774416303999999, 2023-10-09T06:19:36.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.313895875, 2023-10-09T06:19:36.578 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:19:36.578 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:36.579 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:19:36.579 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:19:36.579 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:19:36.579 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 166596 2023-10-09T06:19:36.579 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.579 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:19:36.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.579 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:19:36.579 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.695216+0000", 2023-10-09T06:19:36.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.579 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.580 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:19:36.580 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.695216+0000", 2023-10-09T06:19:36.580 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.580 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.580 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:19:36.580 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.695214+0000", 2023-10-09T06:19:36.580 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:19:36.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.580 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.580 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:19:36.580 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.695223+0000", 2023-10-09T06:19:36.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6030000000000005e-06 2023-10-09T06:19:36.581 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.581 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:19:36.581 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.695239+0000", 2023-10-09T06:19:36.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5654e-05 2023-10-09T06:19:36.581 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.581 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:19:36.581 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.695248+0000", 2023-10-09T06:19:36.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6220000000000004e-06 2023-10-09T06:19:36.581 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.582 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.582 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.696954+0000", 2023-10-09T06:19:36.582 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0017056790000000001 2023-10-09T06:19:36.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.582 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.582 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.696965+0000", 2023-10-09T06:19:36.582 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1429e-05 2023-10-09T06:19:36.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.582 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.002170+0000", 2023-10-09T06:19:36.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.305204389 2023-10-09T06:19:36.583 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.583 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.583 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:19:36.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.002316+0000", 2023-10-09T06:19:36.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000146386 2023-10-09T06:19:36.583 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.583 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.583 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:19:36.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.002463+0000", 2023-10-09T06:19:36.584 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014666700000000001 2023-10-09T06:19:36.584 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.584 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.584 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:19:36.584 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.004733+0000", 2023-10-09T06:19:36.584 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0022705059999999998 2023-10-09T06:19:36.584 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.584 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.584 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.584 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.004768+0000", 2023-10-09T06:19:36.584 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5227e-05 2023-10-09T06:19:36.584 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.585 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.585 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.008988+0000", 2023-10-09T06:19:36.585 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042200479999999997 2023-10-09T06:19:36.585 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.585 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:19:36.585 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.009020+0000", 2023-10-09T06:19:36.585 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1715999999999999e-05 2023-10-09T06:19:36.585 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.585 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:19:36.586 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.009112+0000", 2023-10-09T06:19:36.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2017999999999999e-05 2023-10-09T06:19:36.586 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.586 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:36.586 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.586 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.586 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.586 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:166611 11.7 11:e077589f:::benchmark_data_smithi064_631903_object57697:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e617)", 2023-10-09T06:19:36.586 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:18:54.716256+0000", 2023-10-09T06:19:36.586 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 41.753376447999997, 2023-10-09T06:19:36.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.293209754, 2023-10-09T06:19:36.587 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:19:36.587 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:36.587 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:19:36.587 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:19:36.587 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:19:36.587 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 166611 2023-10-09T06:19:36.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.587 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:19:36.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:19:36.587 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.716256+0000", 2023-10-09T06:19:36.587 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:19:36.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.716256+0000", 2023-10-09T06:19:36.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:19:36.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.716256+0000", 2023-10-09T06:19:36.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:19:36.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:19:36.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.716259+0000", 2023-10-09T06:19:36.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6279999999999998e-06 2023-10-09T06:19:36.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:19:36.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.716268+0000", 2023-10-09T06:19:36.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.799e-06 2023-10-09T06:19:36.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:19:36.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.716274+0000", 2023-10-09T06:19:36.590 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9619999999999998e-06 2023-10-09T06:19:36.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.590 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.590 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.716703+0000", 2023-10-09T06:19:36.590 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00042924700000000001 2023-10-09T06:19:36.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.590 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.590 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.716713+0000", 2023-10-09T06:19:36.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6099999999999995e-06 2023-10-09T06:19:36.591 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.591 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.591 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.591 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.004792+0000", 2023-10-09T06:19:36.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.288078815 2023-10-09T06:19:36.591 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.591 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.591 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:19:36.591 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.004893+0000", 2023-10-09T06:19:36.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010157000000000001 2023-10-09T06:19:36.592 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.592 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.592 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:19:36.592 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.005004+0000", 2023-10-09T06:19:36.592 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110516 2023-10-09T06:19:36.592 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.592 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.592 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:19:36.592 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.009132+0000", 2023-10-09T06:19:36.592 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041282599999999999 2023-10-09T06:19:36.592 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.593 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.593 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.009327+0000", 2023-10-09T06:19:36.593 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019468900000000001 2023-10-09T06:19:36.593 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.593 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.593 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.009386+0000", 2023-10-09T06:19:36.593 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8953999999999998e-05 2023-10-09T06:19:36.593 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.593 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:19:36.594 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.009415+0000", 2023-10-09T06:19:36.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9054e-05 2023-10-09T06:19:36.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.594 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.594 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:19:36.594 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.009466+0000", 2023-10-09T06:19:36.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1128000000000002e-05 2023-10-09T06:19:36.594 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.594 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:36.594 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.594 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.595 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:166654 11.7 11:e9ef1eda:::benchmark_data_smithi064_631903_object57740:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e617)", 2023-10-09T06:19:36.595 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:18:54.780572+0000", 2023-10-09T06:19:36.595 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 41.689060818000002, 2023-10-09T06:19:36.595 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2291889490000001, 2023-10-09T06:19:36.595 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:19:36.595 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:36.595 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:19:36.595 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:19:36.595 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:19:36.595 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 166654 2023-10-09T06:19:36.595 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.595 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:19:36.596 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.596 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:19:36.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.780572+0000", 2023-10-09T06:19:36.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.596 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.596 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.596 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:19:36.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.780572+0000", 2023-10-09T06:19:36.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.596 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.596 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.596 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:19:36.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.780571+0000", 2023-10-09T06:19:36.597 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:19:36.597 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.597 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.597 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:19:36.597 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.780577+0000", 2023-10-09T06:19:36.597 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.246e-06 2023-10-09T06:19:36.597 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.597 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.597 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:19:36.597 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.780589+0000", 2023-10-09T06:19:36.597 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2021999999999999e-05 2023-10-09T06:19:36.597 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.598 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.598 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:19:36.598 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.780596+0000", 2023-10-09T06:19:36.598 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4109999999999999e-06 2023-10-09T06:19:36.598 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.598 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.598 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.598 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.780668+0000", 2023-10-09T06:19:36.598 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1873999999999997e-05 2023-10-09T06:19:36.598 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.598 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.598 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.598 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.780679+0000", 2023-10-09T06:19:36.599 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0614e-05 2023-10-09T06:19:36.599 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.599 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.599 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.006588+0000", 2023-10-09T06:19:36.599 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.225909522 2023-10-09T06:19:36.599 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.599 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:19:36.599 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.006722+0000", 2023-10-09T06:19:36.599 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000133342 2023-10-09T06:19:36.599 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.600 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.600 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:19:36.600 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.006814+0000", 2023-10-09T06:19:36.600 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2009000000000006e-05 2023-10-09T06:19:36.600 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.600 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.600 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:19:36.600 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.009205+0000", 2023-10-09T06:19:36.600 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0023909230000000001 2023-10-09T06:19:36.600 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.600 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.600 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.601 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.009509+0000", 2023-10-09T06:19:36.601 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000304374 2023-10-09T06:19:36.601 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.601 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.601 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.601 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.009577+0000", 2023-10-09T06:19:36.601 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8152999999999996e-05 2023-10-09T06:19:36.601 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.601 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.601 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:19:36.601 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.009650+0000", 2023-10-09T06:19:36.601 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3248000000000001e-05 2023-10-09T06:19:36.602 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.602 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.602 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:19:36.602 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.009761+0000", 2023-10-09T06:19:36.602 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011042 2023-10-09T06:19:36.602 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.602 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:36.602 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.602 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.602 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.602 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:166684 11.7 11:e2f45bef:::benchmark_data_smithi064_631903_object57770:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e617)", 2023-10-09T06:19:36.603 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:18:54.810524+0000", 2023-10-09T06:19:36.603 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 41.659108222, 2023-10-09T06:19:36.603 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2131285709999999, 2023-10-09T06:19:36.603 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:19:36.603 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:36.603 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:19:36.603 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:19:36.603 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:19:36.603 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 166684 2023-10-09T06:19:36.603 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.603 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:19:36.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.604 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:19:36.604 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.810524+0000", 2023-10-09T06:19:36.604 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.604 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.604 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.604 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:19:36.604 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.810524+0000", 2023-10-09T06:19:36.604 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.604 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.604 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.604 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:19:36.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.810523+0000", 2023-10-09T06:19:36.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:19:36.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:19:36.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.810530+0000", 2023-10-09T06:19:36.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9399999999999996e-06 2023-10-09T06:19:36.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:19:36.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.810546+0000", 2023-10-09T06:19:36.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6407999999999999e-05 2023-10-09T06:19:36.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:19:36.606 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.810554+0000", 2023-10-09T06:19:36.606 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9559999999999997e-06 2023-10-09T06:19:36.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.606 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.810694+0000", 2023-10-09T06:19:36.606 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013982 2023-10-09T06:19:36.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.607 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.607 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.810706+0000", 2023-10-09T06:19:36.607 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.189e-05 2023-10-09T06:19:36.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.607 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.607 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.009804+0000", 2023-10-09T06:19:36.607 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.19909803 2023-10-09T06:19:36.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.607 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:19:36.608 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.009940+0000", 2023-10-09T06:19:36.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013635000000000001 2023-10-09T06:19:36.608 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.608 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.608 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:19:36.608 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.010068+0000", 2023-10-09T06:19:36.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000127546 2023-10-09T06:19:36.608 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.608 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.608 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.608 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.015190+0000", 2023-10-09T06:19:36.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051225200000000002 2023-10-09T06:19:36.609 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.609 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.609 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:19:36.609 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.015200+0000", 2023-10-09T06:19:36.609 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4010000000000003e-06 2023-10-09T06:19:36.609 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.609 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.609 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.609 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.023491+0000", 2023-10-09T06:19:36.609 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0082914870000000002 2023-10-09T06:19:36.610 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.610 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.610 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:19:36.610 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.023547+0000", 2023-10-09T06:19:36.610 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5939000000000001e-05 2023-10-09T06:19:36.610 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.610 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.610 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:19:36.610 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.023653+0000", 2023-10-09T06:19:36.610 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000105781 2023-10-09T06:19:36.610 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.611 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:36.611 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.611 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:166692 11.7 11:eb7ca5e8:::benchmark_data_smithi064_631903_object57778:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e617)", 2023-10-09T06:19:36.611 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:18:54.816710+0000", 2023-10-09T06:19:36.611 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 41.652922732999997, 2023-10-09T06:19:36.611 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2071931220000001, 2023-10-09T06:19:36.611 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:19:36.611 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:36.611 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:19:36.611 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:19:36.612 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:19:36.612 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 166692 2023-10-09T06:19:36.612 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.612 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:19:36.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:19:36.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.816710+0000", 2023-10-09T06:19:36.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.612 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:19:36.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.816710+0000", 2023-10-09T06:19:36.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:19:36.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.816709+0000", 2023-10-09T06:19:36.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:19:36.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:19:36.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.816712+0000", 2023-10-09T06:19:36.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7410000000000001e-06 2023-10-09T06:19:36.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:19:36.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.816721+0000", 2023-10-09T06:19:36.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6680000000000007e-06 2023-10-09T06:19:36.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:19:36.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.816726+0000", 2023-10-09T06:19:36.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3970000000000002e-06 2023-10-09T06:19:36.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.818511+0000", 2023-10-09T06:19:36.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0017845630000000001 2023-10-09T06:19:36.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.615 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:19:36.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:54.818539+0000", 2023-10-09T06:19:36.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8263e-05 2023-10-09T06:19:36.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.615 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.013018+0000", 2023-10-09T06:19:36.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1944784310000001 2023-10-09T06:19:36.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:19:36.616 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.013137+0000", 2023-10-09T06:19:36.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011945 2023-10-09T06:19:36.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:19:36.616 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.013208+0000", 2023-10-09T06:19:36.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0566000000000002e-05 2023-10-09T06:19:36.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.617 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:19:36.617 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.023407+0000", 2023-10-09T06:19:36.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0101995 2023-10-09T06:19:36.617 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.617 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.617 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.617 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.023452+0000", 2023-10-09T06:19:36.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4672999999999997e-05 2023-10-09T06:19:36.617 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.617 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.617 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.617 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.023741+0000", 2023-10-09T06:19:36.618 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00028945199999999999 2023-10-09T06:19:36.618 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.618 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.618 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:19:36.618 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.023804+0000", 2023-10-09T06:19:36.618 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3146999999999997e-05 2023-10-09T06:19:36.618 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.618 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.618 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:19:36.618 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:18:56.023903+0000", 2023-10-09T06:19:36.618 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8690999999999994e-05 2023-10-09T06:19:36.618 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.619 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:36.619 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.619 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.619 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:172496 11.6 11:6e164ee2:::benchmark_data_smithi064_631903_object63582:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e620)", 2023-10-09T06:19:36.619 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:19:02.536514+0000", 2023-10-09T06:19:36.619 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 33.93311911, 2023-10-09T06:19:36.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26415169900000002, 2023-10-09T06:19:36.619 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:19:36.619 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:36.619 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:19:36.619 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:19:36.620 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:19:36.620 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 172496 2023-10-09T06:19:36.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.620 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:19:36.620 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:19:36.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.536514+0000", 2023-10-09T06:19:36.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.620 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:19:36.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.536514+0000", 2023-10-09T06:19:36.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:19:36.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.536512+0000", 2023-10-09T06:19:36.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:19:36.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:19:36.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.536522+0000", 2023-10-09T06:19:36.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8970000000000003e-06 2023-10-09T06:19:36.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:19:36.622 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.536533+0000", 2023-10-09T06:19:36.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1231e-05 2023-10-09T06:19:36.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:19:36.622 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.536541+0000", 2023-10-09T06:19:36.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4649999999999997e-06 2023-10-09T06:19:36.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.536595+0000", 2023-10-09T06:19:36.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4545e-05 2023-10-09T06:19:36.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.623 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:19:36.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.536746+0000", 2023-10-09T06:19:36.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000150729 2023-10-09T06:19:36.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.623 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:19:36.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.536869+0000", 2023-10-09T06:19:36.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012299000000000001 2023-10-09T06:19:36.624 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.624 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.624 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.624 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.540458+0000", 2023-10-09T06:19:36.624 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035885259999999999 2023-10-09T06:19:36.624 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.624 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.624 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:19:36.624 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.540699+0000", 2023-10-09T06:19:36.624 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00024089599999999999 2023-10-09T06:19:36.624 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.624 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.625 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.625 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.740796+0000", 2023-10-09T06:19:36.625 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20009765299999999 2023-10-09T06:19:36.625 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.625 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.625 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:19:36.625 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.740825+0000", 2023-10-09T06:19:36.625 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8691e-05 2023-10-09T06:19:36.625 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.625 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.625 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:19:36.625 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.800665+0000", 2023-10-09T06:19:36.626 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.059840352999999999 2023-10-09T06:19:36.626 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.626 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:36.626 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.626 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.626 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.626 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:172500 11.6 11:6f5393fe:::benchmark_data_smithi064_631903_object63586:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e620)", 2023-10-09T06:19:36.626 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:19:02.540828+0000", 2023-10-09T06:19:36.626 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 33.928804710999998, 2023-10-09T06:19:36.626 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25996235699999998, 2023-10-09T06:19:36.626 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:19:36.627 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:36.627 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:19:36.627 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:19:36.627 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:19:36.627 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 172500 2023-10-09T06:19:36.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.627 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:19:36.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:19:36.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.540828+0000", 2023-10-09T06:19:36.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.628 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:19:36.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.540828+0000", 2023-10-09T06:19:36.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.628 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:19:36.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.540827+0000", 2023-10-09T06:19:36.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:19:36.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.628 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:19:36.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.540833+0000", 2023-10-09T06:19:36.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4840000000000001e-06 2023-10-09T06:19:36.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:19:36.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.540848+0000", 2023-10-09T06:19:36.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5182e-05 2023-10-09T06:19:36.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:19:36.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.540856+0000", 2023-10-09T06:19:36.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0700000000000007e-06 2023-10-09T06:19:36.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.540895+0000", 2023-10-09T06:19:36.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8429000000000001e-05 2023-10-09T06:19:36.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:19:36.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.541020+0000", 2023-10-09T06:19:36.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012526599999999999 2023-10-09T06:19:36.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.631 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:19:36.631 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.541141+0000", 2023-10-09T06:19:36.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000121245 2023-10-09T06:19:36.631 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.631 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.631 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:19:36.631 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.543777+0000", 2023-10-09T06:19:36.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026353729999999999 2023-10-09T06:19:36.631 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.631 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.631 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.631 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.544273+0000", 2023-10-09T06:19:36.632 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00049645199999999998 2023-10-09T06:19:36.632 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.632 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.632 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.800716+0000", 2023-10-09T06:19:36.632 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25644280600000002 2023-10-09T06:19:36.632 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.632 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:19:36.632 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.800755+0000", 2023-10-09T06:19:36.632 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9314999999999999e-05 2023-10-09T06:19:36.632 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.633 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:19:36.633 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:02.800790+0000", 2023-10-09T06:19:36.633 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5139999999999999e-05 2023-10-09T06:19:36.633 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.633 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:36.633 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.633 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.633 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.633 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:185608 11.6 11:66bdd741:::benchmark_data_smithi064_631903_object76694:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e625)", 2023-10-09T06:19:36.633 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:19:19.262224+0000", 2023-10-09T06:19:36.633 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 17.207409032000001, 2023-10-09T06:19:36.634 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25244052099999997, 2023-10-09T06:19:36.634 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:19:36.634 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:36.634 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:19:36.634 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:19:36.634 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:19:36.634 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 185608 2023-10-09T06:19:36.634 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.634 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:19:36.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.634 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:19:36.634 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.262224+0000", 2023-10-09T06:19:36.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.635 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:19:36.635 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.262224+0000", 2023-10-09T06:19:36.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.635 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:19:36.635 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.262223+0000", 2023-10-09T06:19:36.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:19:36.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.636 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:19:36.636 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.262226+0000", 2023-10-09T06:19:36.636 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8550000000000002e-06 2023-10-09T06:19:36.636 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.636 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:19:36.636 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.262233+0000", 2023-10-09T06:19:36.636 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5570000000000001e-06 2023-10-09T06:19:36.636 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.637 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:19:36.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.262238+0000", 2023-10-09T06:19:36.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.519e-06 2023-10-09T06:19:36.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.637 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.267901+0000", 2023-10-09T06:19:36.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.005663347 2023-10-09T06:19:36.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.637 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:19:36.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.268020+0000", 2023-10-09T06:19:36.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118439 2023-10-09T06:19:36.638 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.638 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:19:36.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.268115+0000", 2023-10-09T06:19:36.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5317999999999998e-05 2023-10-09T06:19:36.638 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.638 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:19:36.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.271766+0000", 2023-10-09T06:19:36.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00365103 2023-10-09T06:19:36.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.639 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.639 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.274539+0000", 2023-10-09T06:19:36.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027729080000000001 2023-10-09T06:19:36.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.639 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.639 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.514575+0000", 2023-10-09T06:19:36.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24003560199999999 2023-10-09T06:19:36.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.640 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:19:36.640 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.514630+0000", 2023-10-09T06:19:36.640 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4962e-05 2023-10-09T06:19:36.640 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.640 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:19:36.640 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.514664+0000", 2023-10-09T06:19:36.640 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4391999999999999e-05 2023-10-09T06:19:36.640 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.640 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:36.640 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.641 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.641 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.641 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:185626 11.6 11:6eaa079c:::benchmark_data_smithi064_631903_object76712:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e625)", 2023-10-09T06:19:36.641 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:19:19.277362+0000", 2023-10-09T06:19:36.641 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 17.192270561000001, 2023-10-09T06:19:36.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25135929699999998, 2023-10-09T06:19:36.641 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:19:36.641 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:36.641 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:19:36.641 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:19:36.641 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:19:36.642 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 185626 2023-10-09T06:19:36.642 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.642 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:19:36.642 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.642 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:19:36.642 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.277362+0000", 2023-10-09T06:19:36.642 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.642 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.642 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.642 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:19:36.642 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.277362+0000", 2023-10-09T06:19:36.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:19:36.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:19:36.643 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.277362+0000", 2023-10-09T06:19:36.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:19:36.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:19:36.643 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.277364+0000", 2023-10-09T06:19:36.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6089999999999999e-06 2023-10-09T06:19:36.644 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:19:36.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.277370+0000", 2023-10-09T06:19:36.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9179999999999999e-06 2023-10-09T06:19:36.644 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:19:36.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.277375+0000", 2023-10-09T06:19:36.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.276e-06 2023-10-09T06:19:36.644 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:19:36.645 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.517727+0000", 2023-10-09T06:19:36.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.240351546 2023-10-09T06:19:36.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:19:36.645 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.517837+0000", 2023-10-09T06:19:36.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010958100000000001 2023-10-09T06:19:36.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:19:36.645 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.517965+0000", 2023-10-09T06:19:36.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012799799999999999 2023-10-09T06:19:36.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.646 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.646 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.524244+0000", 2023-10-09T06:19:36.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006279359 2023-10-09T06:19:36.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.646 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:19:36.646 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.526958+0000", 2023-10-09T06:19:36.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027138489999999999 2023-10-09T06:19:36.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:36.647 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.528649+0000", 2023-10-09T06:19:36.647 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001691629 2023-10-09T06:19:36.647 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:19:36.647 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.528690+0000", 2023-10-09T06:19:36.647 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0501999999999999e-05 2023-10-09T06:19:36.647 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:19:36.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:19:36.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:19:36.648 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:19.528721+0000", 2023-10-09T06:19:36.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1430000000000002e-05 2023-10-09T06:19:36.648 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.648 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:36.648 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.648 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:19:36.648 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:19:36.648 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:19:36.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:36.650 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:36.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:36.751 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:36.797 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:36.903 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:36.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:36.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:37.021 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:37.064 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T06:19:37.121 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:37.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:37 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2916392159' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:19:37.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:37 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:19:37.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:37 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2916392159' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:19:37.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:37 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:19:37.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:37 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2916392159' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:19:37.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:37 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:19:37.830 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:19:38.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:38 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:19:38.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:38 smithi064 ceph-mon[220044]: osdmap e629: 8 total, 8 up, 7 in 2023-10-09T06:19:38.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:38 smithi064 ceph-mon[220044]: failed to encode map e629 with expected crc 2023-10-09T06:19:38.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:38 smithi064 ceph-mon[220044]: failed to encode map e629 with expected crc 2023-10-09T06:19:38.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:38 smithi064 ceph-mon[220044]: failed to encode map e629 with expected crc 2023-10-09T06:19:38.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:38 smithi064 ceph-mon[220044]: pgmap v1621: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.2 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 960 op/s; 0 B/s, 152 objects/s recovering 2023-10-09T06:19:38.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:38 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:19:38.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:38 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:19:38.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:38 smithi064 ceph-mon[212750]: osdmap e629: 8 total, 8 up, 7 in 2023-10-09T06:19:38.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:38 smithi064 ceph-mon[212750]: failed to encode map e629 with expected crc 2023-10-09T06:19:38.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:38 smithi064 ceph-mon[212750]: failed to encode map e629 with expected crc 2023-10-09T06:19:38.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:38 smithi064 ceph-mon[212750]: failed to encode map e629 with expected crc 2023-10-09T06:19:38.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:38 smithi064 ceph-mon[212750]: pgmap v1621: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.2 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 960 op/s; 0 B/s, 152 objects/s recovering 2023-10-09T06:19:38.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:38 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:19:38.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:38 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:19:38.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:38 smithi156 ceph-mon[195101]: osdmap e629: 8 total, 8 up, 7 in 2023-10-09T06:19:38.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:38 smithi156 ceph-mon[195101]: failed to encode map e629 with expected crc 2023-10-09T06:19:38.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:38 smithi156 ceph-mon[195101]: failed to encode map e629 with expected crc 2023-10-09T06:19:38.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:38 smithi156 ceph-mon[195101]: failed to encode map e629 with expected crc 2023-10-09T06:19:38.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:38 smithi156 ceph-mon[195101]: pgmap v1621: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.2 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 960 op/s; 0 B/s, 152 objects/s recovering 2023-10-09T06:19:38.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:38 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:19:38.578 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:38.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:38.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:38.694 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:38.722 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:39.353 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:19:39.630 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:39.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:39.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:39.688 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T06:19:39.747 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:39.780 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:39.860 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:39.961 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:39.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:39.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:40.071 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:40.162 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:40.358 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:40.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:40.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:40.463 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:40.564 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:40.597 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:40 smithi064 ceph-mon[212750]: pgmap v1622: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.2 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 815 op/s; 0 B/s, 129 objects/s recovering 2023-10-09T06:19:40.597 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:40 smithi064 ceph-mon[220044]: pgmap v1622: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.2 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 815 op/s; 0 B/s, 129 objects/s recovering 2023-10-09T06:19:40.675 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:40.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:40.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:40.785 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:40.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:40 smithi156 ceph-mon[195101]: pgmap v1622: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.2 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 815 op/s; 0 B/s, 129 objects/s recovering 2023-10-09T06:19:40.882 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:41.427 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:19:41.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:41 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3585526197' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:19:41.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:41 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:19:41.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:41 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3585526197' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:19:41.723 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:41 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:19:41.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:41 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3585526197' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:19:41.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:41 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:19:42.042 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:42.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:42.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:42.148 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:42.217 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:42.309 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:42.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:42.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:42.414 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:42.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:42 smithi064 ceph-mon[212750]: pgmap v1623: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 947 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 798 op/s; 0 B/s, 87 objects/s recovering 2023-10-09T06:19:42.422 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:42 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:42.422 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:42 smithi064 ceph-mon[212750]: failed to encode map e630 with expected crc 2023-10-09T06:19:42.422 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:42 smithi064 ceph-mon[212750]: failed to encode map e630 with expected crc 2023-10-09T06:19:42.422 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:42 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:19:42.422 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:42 smithi064 ceph-mon[212750]: osdmap e630: 8 total, 8 up, 7 in 2023-10-09T06:19:42.422 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:42 smithi064 ceph-mon[212750]: failed to encode map e630 with expected crc 2023-10-09T06:19:42.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:42 smithi064 ceph-mon[220044]: pgmap v1623: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 947 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 798 op/s; 0 B/s, 87 objects/s recovering 2023-10-09T06:19:42.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:42 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:42.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:42 smithi064 ceph-mon[220044]: failed to encode map e630 with expected crc 2023-10-09T06:19:42.423 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:42 smithi064 ceph-mon[220044]: failed to encode map e630 with expected crc 2023-10-09T06:19:42.423 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:42 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:19:42.423 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:42 smithi064 ceph-mon[220044]: osdmap e630: 8 total, 8 up, 7 in 2023-10-09T06:19:42.423 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:42 smithi064 ceph-mon[220044]: failed to encode map e630 with expected crc 2023-10-09T06:19:42.478 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:42.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:42 smithi156 ceph-mon[195101]: pgmap v1623: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 947 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 798 op/s; 0 B/s, 87 objects/s recovering 2023-10-09T06:19:42.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:42 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:42.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:42 smithi156 ceph-mon[195101]: failed to encode map e630 with expected crc 2023-10-09T06:19:42.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:42 smithi156 ceph-mon[195101]: failed to encode map e630 with expected crc 2023-10-09T06:19:42.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:42 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:19:42.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:42 smithi156 ceph-mon[195101]: osdmap e630: 8 total, 8 up, 7 in 2023-10-09T06:19:42.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:42 smithi156 ceph-mon[195101]: failed to encode map e630 with expected crc 2023-10-09T06:19:42.838 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:42.877 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:42.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:42.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:42.992 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:43.069 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:43.302 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:43.370 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:43.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:43.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:43.481 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:43.499 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:19:43.524 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:43.535 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:19:43.577 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:19:43.615 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:19:43.616 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:43.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:43.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:43.646 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:19:43.692 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:19:43.713 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:43 smithi064 ceph-mon[220044]: failed to encode map e630 with expected crc 2023-10-09T06:19:43.713 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:43 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:43.713 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:43 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:43.718 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:43 smithi064 ceph-mon[212750]: failed to encode map e630 with expected crc 2023-10-09T06:19:43.718 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:43 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:43.718 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:43 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:43.728 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:43.730 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:19:43.760 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:19:43.777 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:19:43.777 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:19:43.778 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:19:43.778 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:19:43.778 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:19:43.778 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:19:43.778 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:19:43.778 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:19:43.811 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:43.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:43 smithi156 ceph-mon[195101]: failed to encode map e630 with expected crc 2023-10-09T06:19:43.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:43 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:43.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:43 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:44.281 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:44.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:44.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:44.397 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:44.424 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:44.478 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:44.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:44.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:44.583 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:44.609 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:44.710 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:44.715 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:44 smithi156 ceph-mon[195101]: pgmap v1625: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 947 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 731 op/s; 0 B/s, 109 objects/s recovering 2023-10-09T06:19:44.715 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:44 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2082394881' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:44.715 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:44.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:44.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:44.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:44 smithi064 ceph-mon[212750]: pgmap v1625: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 947 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 731 op/s; 0 B/s, 109 objects/s recovering 2023-10-09T06:19:44.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:44 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2082394881' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:44.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:44.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:44 smithi064 ceph-mon[220044]: pgmap v1625: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 947 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 731 op/s; 0 B/s, 109 objects/s recovering 2023-10-09T06:19:44.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:44 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2082394881' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:44.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:44.817 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:44.891 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:19:44.894 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:45.617 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:19:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:19:45] "GET /metrics HTTP/1.1" 200 36447 "" "Prometheus/2.43.0" 2023-10-09T06:19:45.629 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:45.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:45.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:45.742 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:45.770 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:45.817 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:45.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:45.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:45.925 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:46.005 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:46.115 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:46.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:46.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:46.223 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:46.266 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:46.324 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:46.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:46.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:46.431 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:46.453 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:46 smithi064 ceph-mon[220044]: pgmap v1626: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 814 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 711 op/s; 0 B/s, 52 objects/s recovering 2023-10-09T06:19:46.456 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:46 smithi156 ceph-mon[195101]: pgmap v1626: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 814 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 711 op/s; 0 B/s, 52 objects/s recovering 2023-10-09T06:19:46.535 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:46.692 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:46.722 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:46 smithi064 ceph-mon[212750]: pgmap v1626: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 814 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 711 op/s; 0 B/s, 52 objects/s recovering 2023-10-09T06:19:46.830 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:46.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:46.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:46.945 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:46.971 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:19:46.971 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:19:46.990 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:47.075 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:47.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:47.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:47.189 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:47.232 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:47.316 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:47.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:47.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:47.424 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:47.520 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:48.242 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:19:48.478 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T06:19:48.566 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:48.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:48.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:48.573 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:48 smithi064 ceph-mon[212750]: pgmap v1627: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 682 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 802 op/s; 0 B/s, 100 objects/s recovering 2023-10-09T06:19:48.573 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:48 smithi064 ceph-mon[212750]: failed to encode map e631 with expected crc 2023-10-09T06:19:48.573 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:48 smithi064 ceph-mon[212750]: osdmap e631: 8 total, 8 up, 7 in 2023-10-09T06:19:48.573 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:48 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3877185081' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:19:48.583 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:48 smithi064 ceph-mon[220044]: pgmap v1627: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 682 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 802 op/s; 0 B/s, 100 objects/s recovering 2023-10-09T06:19:48.583 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:48 smithi064 ceph-mon[220044]: failed to encode map e631 with expected crc 2023-10-09T06:19:48.583 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:48 smithi064 ceph-mon[220044]: osdmap e631: 8 total, 8 up, 7 in 2023-10-09T06:19:48.583 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:48 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3877185081' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:19:48.673 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:48.732 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:48.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:48 smithi156 ceph-mon[195101]: pgmap v1627: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 682 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 802 op/s; 0 B/s, 100 objects/s recovering 2023-10-09T06:19:48.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:48 smithi156 ceph-mon[195101]: failed to encode map e631 with expected crc 2023-10-09T06:19:48.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:48 smithi156 ceph-mon[195101]: osdmap e631: 8 total, 8 up, 7 in 2023-10-09T06:19:48.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:48 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3877185081' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:19:49.467 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:19:49.563 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:49.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:49.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:49.669 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:49.706 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:49.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:49 smithi064 ceph-mon[212750]: failed to encode map e632 with expected crc 2023-10-09T06:19:49.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:49 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3877185081' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:19:49.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:49 smithi064 ceph-mon[212750]: osdmap e632: 8 total, 8 up, 7 in 2023-10-09T06:19:49.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:49 smithi064 ceph-mon[212750]: failed to encode map e632 with expected crc 2023-10-09T06:19:49.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:49 smithi064 ceph-mon[212750]: failed to encode map e632 with expected crc 2023-10-09T06:19:49.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:49 smithi064 ceph-mon[220044]: failed to encode map e632 with expected crc 2023-10-09T06:19:49.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:49 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3877185081' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:19:49.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:49 smithi064 ceph-mon[220044]: osdmap e632: 8 total, 8 up, 7 in 2023-10-09T06:19:49.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:49 smithi064 ceph-mon[220044]: failed to encode map e632 with expected crc 2023-10-09T06:19:49.724 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:49 smithi064 ceph-mon[220044]: failed to encode map e632 with expected crc 2023-10-09T06:19:49.775 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T06:19:49.775 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:19:49.795 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:49.801 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:49 smithi156 ceph-mon[195101]: failed to encode map e632 with expected crc 2023-10-09T06:19:49.801 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:49 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3877185081' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:19:49.802 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:49 smithi156 ceph-mon[195101]: osdmap e632: 8 total, 8 up, 7 in 2023-10-09T06:19:49.802 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:49 smithi156 ceph-mon[195101]: failed to encode map e632 with expected crc 2023-10-09T06:19:49.802 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:49 smithi156 ceph-mon[195101]: failed to encode map e632 with expected crc 2023-10-09T06:19:49.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:49.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:49.904 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:49.945 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:49.945 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:50.014 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:50.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:50.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:50.121 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:50.205 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:50.357 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:50.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:50.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:50.465 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:50.508 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:50.609 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:50.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:50.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:50.718 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:50.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:50 smithi064 ceph-mon[212750]: pgmap v1630: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 682 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 0 B/s, 71 objects/s recovering 2023-10-09T06:19:50.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:50 smithi064 ceph-mon[220044]: pgmap v1630: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 682 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 0 B/s, 71 objects/s recovering 2023-10-09T06:19:50.805 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:50.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:50 smithi156 ceph-mon[195101]: pgmap v1630: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 682 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 0 B/s, 71 objects/s recovering 2023-10-09T06:19:50.945 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:50.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:50.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:51.058 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:51.090 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:51.197 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:51.527 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:19:51.656 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:51.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:51.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:51.765 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:51.812 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:51 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2721772640' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:51.812 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:51 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:51.813 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:51 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2721772640' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:51.813 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:51 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:51.820 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:51.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:51 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2721772640' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:51.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:51 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:52.475 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:52.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:52.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:52.588 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:52.647 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:52.690 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:52.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:52.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:52.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:52 smithi064 ceph-mon[212750]: pgmap v1631: 119 pgs: 119 active+clean; 467 MiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 960 op/s; 0 B/s, 85 objects/s recovering 2023-10-09T06:19:52.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:52 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:19:52.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:52 smithi064 ceph-mon[212750]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T06:19:52.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:52 smithi064 ceph-mon[212750]: failed to encode map e633 with expected crc 2023-10-09T06:19:52.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:52 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:19:52.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:52 smithi064 ceph-mon[212750]: osdmap e633: 8 total, 8 up, 7 in 2023-10-09T06:19:52.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:52 smithi064 ceph-mon[212750]: failed to encode map e633 with expected crc 2023-10-09T06:19:52.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:52 smithi064 ceph-mon[212750]: failed to encode map e633 with expected crc 2023-10-09T06:19:52.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:52 smithi064 ceph-mon[212750]: failed to encode map e633 with expected crc 2023-10-09T06:19:52.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:52 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:52.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:52 smithi064 ceph-mon[220044]: pgmap v1631: 119 pgs: 119 active+clean; 467 MiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 960 op/s; 0 B/s, 85 objects/s recovering 2023-10-09T06:19:52.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:52 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:19:52.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:52 smithi064 ceph-mon[220044]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T06:19:52.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:52 smithi064 ceph-mon[220044]: failed to encode map e633 with expected crc 2023-10-09T06:19:52.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:52 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:19:52.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:52 smithi064 ceph-mon[220044]: osdmap e633: 8 total, 8 up, 7 in 2023-10-09T06:19:52.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:52 smithi064 ceph-mon[220044]: failed to encode map e633 with expected crc 2023-10-09T06:19:52.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:52 smithi064 ceph-mon[220044]: failed to encode map e633 with expected crc 2023-10-09T06:19:52.722 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:52 smithi064 ceph-mon[220044]: failed to encode map e633 with expected crc 2023-10-09T06:19:52.723 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:52 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:52.797 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:52.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:52 smithi156 ceph-mon[195101]: pgmap v1631: 119 pgs: 119 active+clean; 467 MiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 960 op/s; 0 B/s, 85 objects/s recovering 2023-10-09T06:19:52.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:52 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:19:52.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:52 smithi156 ceph-mon[195101]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T06:19:52.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:52 smithi156 ceph-mon[195101]: failed to encode map e633 with expected crc 2023-10-09T06:19:52.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:52 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:19:52.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:52 smithi156 ceph-mon[195101]: osdmap e633: 8 total, 8 up, 7 in 2023-10-09T06:19:52.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:52 smithi156 ceph-mon[195101]: failed to encode map e633 with expected crc 2023-10-09T06:19:52.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:52 smithi156 ceph-mon[195101]: failed to encode map e633 with expected crc 2023-10-09T06:19:52.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:52 smithi156 ceph-mon[195101]: failed to encode map e633 with expected crc 2023-10-09T06:19:52.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:52 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:52.878 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:53.315 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:53.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:53.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:53.401 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:53.421 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:53.511 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:53.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:53 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:53.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:53 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:53.947 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:53 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:53.952 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:19:53.996 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:54.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:54.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:54.003 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:19:54.049 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:19:54.091 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:19:54.103 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:54.114 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:19:54.129 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:54.133 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:19:54.152 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:19:54.171 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:54.171 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:19:54.173 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:19:54.173 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:19:54.173 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:19:54.173 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:19:54.173 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:19:54.173 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:19:54.173 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:19:54.173 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:19:54.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:54.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:54.276 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:54.306 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:54.829 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:54.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:54.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:54.938 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:54.975 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:55.017 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:55.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:55.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:54 smithi156 ceph-mon[195101]: pgmap v1633: 119 pgs: 119 active+clean; 467 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 0 B/s, 19 objects/s recovering 2023-10-09T06:19:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:54 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2242531574' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:55.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:54 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:55.123 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:55.178 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:54 smithi064 ceph-mon[220044]: pgmap v1633: 119 pgs: 119 active+clean; 467 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 0 B/s, 19 objects/s recovering 2023-10-09T06:19:55.178 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:54 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2242531574' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:55.178 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:54 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:55.178 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:54 smithi064 ceph-mon[212750]: pgmap v1633: 119 pgs: 119 active+clean; 467 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 0 B/s, 19 objects/s recovering 2023-10-09T06:19:55.178 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:54 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2242531574' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:55.178 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:54 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:55.182 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:55.344 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:19:55.481 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:19:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:19:55] "GET /metrics HTTP/1.1" 200 36447 "" "Prometheus/2.43.0" 2023-10-09T06:19:55.628 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:55.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:55.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:55.737 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:55.796 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:55.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:55 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:56.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:55 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:56.165 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:56.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:56.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:56.278 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:56.327 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:55 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:56.330 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:56.406 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:56.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:56.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:56.513 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:56.553 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:56.622 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:56.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:56.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:56.733 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:56.737 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:19:56.737 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T06:19:56.737 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T06:19:56.737 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T06:19:56.738 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:19:56.781 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:56.882 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:56.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:56.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:56.989 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:57.051 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:56 smithi064 ceph-mon[212750]: pgmap v1634: 119 pgs: 119 active+clean; 332 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 0 B/s, 14 objects/s recovering 2023-10-09T06:19:57.051 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:56 smithi064 ceph-mon[220044]: pgmap v1634: 119 pgs: 119 active+clean; 332 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 0 B/s, 14 objects/s recovering 2023-10-09T06:19:57.059 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:57.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:56 smithi156 ceph-mon[195101]: pgmap v1634: 119 pgs: 119 active+clean; 332 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 0 B/s, 14 objects/s recovering 2023-10-09T06:19:57.576 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:19:57.596 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T06:19:57.938 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:19:58.180 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:58.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:58.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:58.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:19:58.285 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:58.376 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:58.546 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:58.548 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1635,"stamp":"2023-10-09T06:19:57.349972+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":211883960,"num_objects":3249,"num_object_clones":0,"num_object_copies":9747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3249,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":214660,"num_write_kb":6970334,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16749,"num_bytes_recovered":589718190,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95199,"ondisk_log_size":95199,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":359,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6701444,"kb_used_data":965068,"kb_used_omap":182,"kb_used_meta":5736137,"kb_avail":649514620,"statfs":{"total":671965249536,"available":665102970880,"internally_reserved":0,"allocated":988229632,"data_stored":970973590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":186958,"internal_metadata":5873804722},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-502726656,"num_objects":-7671,"num_object_clones":0,"num_object_copies":-23013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7671,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.880633"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":913,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.520084+0000","last_change":"2023-10-09T06:18:32.883970+0000","last_active":"2023-10-09T06:19:48.520084+0000","last_peered":"2023-10-09T06:19:48.520084+0000","last_clean":"2023-10-09T06:19:48.520084+0000","last_became_active":"2023-10-09T06:18:32.883509+0000","last_became_peered":"2023-10-09T06:18:32.883509+0000","last_unstale":"2023-10-09T06:19:48.520084+0000","last_undegraded":"2023-10-09T06:19:48.520084+0000","last_fullsized":"2023-10-09T06:19:48.520084+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:05.412941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:05.412941+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:30.926648+0000","scrub_duration":0.0061014329999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":711,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517813+0000","last_change":"2023-10-09T06:16:02.748447+0000","last_active":"2023-10-09T06:19:48.517813+0000","last_peered":"2023-10-09T06:19:48.517813+0000","last_clean":"2023-10-09T06:19:48.517813+0000","last_became_active":"2023-10-09T06:16:02.747672+0000","last_became_peered":"2023-10-09T06:16:02.747672+0000","last_unstale":"2023-10-09T06:19:48.517813+0000","last_undegraded":"2023-10-09T06:19:48.517813+0000","last_fullsized":"2023-10-09T06:19:48.517813+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:31.326263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":446,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.523886+0000","last_change":"2023-10-09T06:05:11.451972+0000","last_active":"2023-10-09T06:19:51.523886+0000","last_peered":"2023-10-09T06:19:51.523886+0000","last_clean":"2023-10-09T06:19:51.523886+0000","last_became_active":"2023-10-09T06:05:11.451448+0000","last_became_peered":"2023-10-09T06:05:11.451448+0000","last_unstale":"2023-10-09T06:19:51.523886+0000","last_undegraded":"2023-10-09T06:19:51.523886+0000","last_fullsized":"2023-10-09T06:19:51.523886+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:39.183621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":949,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517918+0000","last_change":"2023-10-09T06:18:32.879870+0000","last_active":"2023-10-09T06:19:48.517918+0000","last_peered":"2023-10-09T06:19:48.517918+0000","last_clean":"2023-10-09T06:19:48.517918+0000","last_became_active":"2023-10-09T06:18:32.879518+0000","last_became_peered":"2023-10-09T06:18:32.879518+0000","last_unstale":"2023-10-09T06:19:48.517918+0000","last_undegraded":"2023-10-09T06:19:48.517918+0000","last_fullsized":"2023-10-09T06:19:48.517918+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:01.468346+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:01.468346+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:28.527869+0000","scrub_duration":0.0016258220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":317,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.470783+0000","last_change":"2023-10-09T06:01:54.762060+0000","last_active":"2023-10-09T06:19:48.470783+0000","last_peered":"2023-10-09T06:19:48.470783+0000","last_clean":"2023-10-09T06:19:48.470783+0000","last_became_active":"2023-10-09T06:01:54.761757+0000","last_became_peered":"2023-10-09T06:01:54.761757+0000","last_unstale":"2023-10-09T06:19:48.470783+0000","last_undegraded":"2023-10-09T06:19:48.470783+0000","last_fullsized":"2023-10-09T06:19:48.470783+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:40.294232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":116,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.522548+0000","last_change":"2023-10-09T06:13:30.789834+0000","last_active":"2023-10-09T06:19:48.522548+0000","last_peered":"2023-10-09T06:19:48.522548+0000","last_clean":"2023-10-09T06:19:48.522548+0000","last_became_active":"2023-10-09T06:13:30.789517+0000","last_became_peered":"2023-10-09T06:13:30.789517+0000","last_unstale":"2023-10-09T06:19:48.522548+0000","last_undegraded":"2023-10-09T06:19:48.522548+0000","last_fullsized":"2023-10-09T06:19:48.522548+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:45.467922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":917,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517607+0000","last_change":"2023-10-09T06:12:07.739121+0000","last_active":"2023-10-09T06:19:48.517607+0000","last_peered":"2023-10-09T06:19:48.517607+0000","last_clean":"2023-10-09T06:19:48.517607+0000","last_became_active":"2023-10-09T06:12:07.738739+0000","last_became_peered":"2023-10-09T06:12:07.738739+0000","last_unstale":"2023-10-09T06:19:48.517607+0000","last_undegraded":"2023-10-09T06:19:48.517607+0000","last_fullsized":"2023-10-09T06:19:48.517607+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.293426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.293426+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:19.815422+0000","scrub_duration":0.001587946,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":428,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.472939+0000","last_change":"2023-10-09T06:09:27.564377+0000","last_active":"2023-10-09T06:19:48.472939+0000","last_peered":"2023-10-09T06:19:48.472939+0000","last_clean":"2023-10-09T06:19:48.472939+0000","last_became_active":"2023-10-09T06:09:27.563710+0000","last_became_peered":"2023-10-09T06:09:27.563710+0000","last_unstale":"2023-10-09T06:19:48.472939+0000","last_undegraded":"2023-10-09T06:19:48.472939+0000","last_fullsized":"2023-10-09T06:19:48.472939+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:48.168447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":107,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.472130+0000","last_change":"2023-10-09T06:18:33.820081+0000","last_active":"2023-10-09T06:19:48.472130+0000","last_peered":"2023-10-09T06:19:48.472130+0000","last_clean":"2023-10-09T06:19:48.472130+0000","last_became_active":"2023-10-09T06:18:33.819790+0000","last_became_peered":"2023-10-09T06:18:33.819790+0000","last_unstale":"2023-10-09T06:19:48.472130+0000","last_undegraded":"2023-10-09T06:19:48.472130+0000","last_fullsized":"2023-10-09T06:19:48.472130+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:18.166606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":868,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.522315+0000","last_change":"2023-10-09T06:18:32.886406+0000","last_active":"2023-10-09T06:19:48.522315+0000","last_peered":"2023-10-09T06:19:48.522315+0000","last_clean":"2023-10-09T06:19:48.522315+0000","last_became_active":"2023-10-09T06:18:32.886142+0000","last_became_peered":"2023-10-09T06:18:32.886142+0000","last_unstale":"2023-10-09T06:19:48.522315+0000","last_undegraded":"2023-10-09T06:19:48.522315+0000","last_fullsized":"2023-10-09T06:19:48.522315+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.384407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.384407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:47:35.682411+0000","scrub_duration":0.0040651960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":548,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473934+0000","last_change":"2023-10-09T06:18:32.819012+0000","last_active":"2023-10-09T06:19:48.473934+0000","last_peered":"2023-10-09T06:19:48.473934+0000","last_clean":"2023-10-09T06:19:48.473934+0000","last_became_active":"2023-10-09T06:18:32.818711+0000","last_became_peered":"2023-10-09T06:18:32.818711+0000","last_unstale":"2023-10-09T06:19:48.473934+0000","last_undegraded":"2023-10-09T06:19:48.473934+0000","last_fullsized":"2023-10-09T06:19:48.473934+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:17:48.281174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":482,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473058+0000","last_change":"2023-10-09T06:18:12.816530+0000","last_active":"2023-10-09T06:19:48.473058+0000","last_peered":"2023-10-09T06:19:48.473058+0000","last_clean":"2023-10-09T06:19:48.473058+0000","last_became_active":"2023-10-09T06:18:12.815392+0000","last_became_peered":"2023-10-09T06:18:12.815392+0000","last_unstale":"2023-10-09T06:19:48.473058+0000","last_undegraded":"2023-10-09T06:19:48.473058+0000","last_fullsized":"2023-10-09T06:19:48.473058+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:18.350974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":777,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.521090+0000","last_change":"2023-10-09T06:16:05.761750+0000","last_active":"2023-10-09T06:19:48.521090+0000","last_peered":"2023-10-09T06:19:48.521090+0000","last_clean":"2023-10-09T06:19:48.521090+0000","last_became_active":"2023-10-09T06:16:05.761436+0000","last_became_peered":"2023-10-09T06:16:05.761436+0000","last_unstale":"2023-10-09T06:19:48.521090+0000","last_undegraded":"2023-10-09T06:19:48.521090+0000","last_fullsized":"2023-10-09T06:19:48.521090+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:20.742561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:20.742561+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:13.021950+0000","scrub_duration":0.0027005039999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":133,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.522521+0000","last_change":"2023-10-09T06:18:32.877651+0000","last_active":"2023-10-09T06:19:48.522521+0000","last_peered":"2023-10-09T06:19:48.522521+0000","last_clean":"2023-10-09T06:19:48.522521+0000","last_became_active":"2023-10-09T06:18:32.877371+0000","last_became_peered":"2023-10-09T06:18:32.877371+0000","last_unstale":"2023-10-09T06:19:48.522521+0000","last_undegraded":"2023-10-09T06:19:48.522521+0000","last_fullsized":"2023-10-09T06:19:48.522521+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:23:59.684830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":646,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473601+0000","last_change":"2023-10-09T06:18:12.816564+0000","last_active":"2023-10-09T06:19:48.473601+0000","last_peered":"2023-10-09T06:19:48.473601+0000","last_clean":"2023-10-09T06:19:48.473601+0000","last_became_active":"2023-10-09T06:18:12.815943+0000","last_became_peered":"2023-10-09T06:18:12.815943+0000","last_unstale":"2023-10-09T06:19:48.473601+0000","last_undegraded":"2023-10-09T06:19:48.473601+0000","last_fullsized":"2023-10-09T06:19:48.473601+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:10.114324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":837,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473376+0000","last_change":"2023-10-09T06:18:33.826699+0000","last_active":"2023-10-09T06:19:48.473376+0000","last_peered":"2023-10-09T06:19:48.473376+0000","last_clean":"2023-10-09T06:19:48.473376+0000","last_became_active":"2023-10-09T06:18:33.826349+0000","last_became_peered":"2023-10-09T06:18:33.826349+0000","last_unstale":"2023-10-09T06:19:48.473376+0000","last_undegraded":"2023-10-09T06:19:48.473376+0000","last_fullsized":"2023-10-09T06:19:48.473376+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:38.588641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:38.588641+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:30:50.340037+0000","scrub_duration":0.010116385,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":122,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.522228+0000","last_change":"2023-10-09T06:13:30.826588+0000","last_active":"2023-10-09T06:19:48.522228+0000","last_peered":"2023-10-09T06:19:48.522228+0000","last_clean":"2023-10-09T06:19:48.522228+0000","last_became_active":"2023-10-09T06:13:30.799199+0000","last_became_peered":"2023-10-09T06:13:30.799199+0000","last_unstale":"2023-10-09T06:19:48.522228+0000","last_undegraded":"2023-10-09T06:19:48.522228+0000","last_fullsized":"2023-10-09T06:19:48.522228+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:38.703864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":82,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.531733+0000","last_change":"2023-10-09T06:18:33.812722+0000","last_active":"2023-10-09T06:19:48.531733+0000","last_peered":"2023-10-09T06:19:48.531733+0000","last_clean":"2023-10-09T06:19:48.531733+0000","last_became_active":"2023-10-09T06:18:33.812386+0000","last_became_peered":"2023-10-09T06:18:33.812386+0000","last_unstale":"2023-10-09T06:19:48.531733+0000","last_undegraded":"2023-10-09T06:19:48.531733+0000","last_fullsized":"2023-10-09T06:19:48.531733+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:26.259509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":807,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.472191+0000","last_change":"2023-10-09T06:09:27.574336+0000","last_active":"2023-10-09T06:19:48.472191+0000","last_peered":"2023-10-09T06:19:48.472191+0000","last_clean":"2023-10-09T06:19:48.472191+0000","last_became_active":"2023-10-09T06:09:27.573974+0000","last_became_peered":"2023-10-09T06:09:27.573974+0000","last_unstale":"2023-10-09T06:19:48.472191+0000","last_undegraded":"2023-10-09T06:19:48.472191+0000","last_fullsized":"2023-10-09T06:19:48.472191+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.339121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.339121+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:45.908579+0000","scrub_duration":0.010474344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":707,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517910+0000","last_change":"2023-10-09T06:18:12.829164+0000","last_active":"2023-10-09T06:19:48.517910+0000","last_peered":"2023-10-09T06:19:48.517910+0000","last_clean":"2023-10-09T06:19:48.517910+0000","last_became_active":"2023-10-09T06:18:12.818219+0000","last_became_peered":"2023-10-09T06:18:12.818219+0000","last_unstale":"2023-10-09T06:19:48.517910+0000","last_undegraded":"2023-10-09T06:19:48.517910+0000","last_fullsized":"2023-10-09T06:19:48.517910+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:14.546650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":672,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.518220+0000","last_change":"2023-10-09T06:18:12.818275+0000","last_active":"2023-10-09T06:19:48.518220+0000","last_peered":"2023-10-09T06:19:48.518220+0000","last_clean":"2023-10-09T06:19:48.518220+0000","last_became_active":"2023-10-09T06:18:12.817978+0000","last_became_peered":"2023-10-09T06:18:12.817978+0000","last_unstale":"2023-10-09T06:19:48.518220+0000","last_undegraded":"2023-10-09T06:19:48.518220+0000","last_fullsized":"2023-10-09T06:19:48.518220+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:52:24.157690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":878,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473241+0000","last_change":"2023-10-09T06:18:32.817454+0000","last_active":"2023-10-09T06:19:48.473241+0000","last_peered":"2023-10-09T06:19:48.473241+0000","last_clean":"2023-10-09T06:19:48.473241+0000","last_became_active":"2023-10-09T06:18:32.817081+0000","last_became_peered":"2023-10-09T06:18:32.817081+0000","last_unstale":"2023-10-09T06:19:48.473241+0000","last_undegraded":"2023-10-09T06:19:48.473241+0000","last_fullsized":"2023-10-09T06:19:48.473241+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:42.900876+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:42.900876+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:50.579398+0000","scrub_duration":0.0014928649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":170,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.521042+0000","last_change":"2023-10-09T06:18:12.833338+0000","last_active":"2023-10-09T06:19:48.521042+0000","last_peered":"2023-10-09T06:19:48.521042+0000","last_clean":"2023-10-09T06:19:48.521042+0000","last_became_active":"2023-10-09T06:18:12.818967+0000","last_became_peered":"2023-10-09T06:18:12.818967+0000","last_unstale":"2023-10-09T06:19:48.521042+0000","last_undegraded":"2023-10-09T06:19:48.521042+0000","last_fullsized":"2023-10-09T06:19:48.521042+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:55.022219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":681,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473581+0000","last_change":"2023-10-09T06:01:54.761558+0000","last_active":"2023-10-09T06:19:48.473581+0000","last_peered":"2023-10-09T06:19:48.473581+0000","last_clean":"2023-10-09T06:19:48.473581+0000","last_became_active":"2023-10-09T06:01:54.761251+0000","last_became_peered":"2023-10-09T06:01:54.761251+0000","last_unstale":"2023-10-09T06:19:48.473581+0000","last_undegraded":"2023-10-09T06:19:48.473581+0000","last_fullsized":"2023-10-09T06:19:48.473581+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:53.245267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":1195,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.520168+0000","last_change":"2023-10-09T06:18:33.824142+0000","last_active":"2023-10-09T06:19:48.520168+0000","last_peered":"2023-10-09T06:19:48.520168+0000","last_clean":"2023-10-09T06:19:48.520168+0000","last_became_active":"2023-10-09T06:18:33.823799+0000","last_became_peered":"2023-10-09T06:18:33.823799+0000","last_unstale":"2023-10-09T06:19:48.520168+0000","last_undegraded":"2023-10-09T06:19:48.520168+0000","last_fullsized":"2023-10-09T06:19:48.520168+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:54:59.270975+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":116,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.521302+0000","last_change":"2023-10-09T06:13:30.788160+0000","last_active":"2023-10-09T06:19:48.521302+0000","last_peered":"2023-10-09T06:19:48.521302+0000","last_clean":"2023-10-09T06:19:48.521302+0000","last_became_active":"2023-10-09T06:13:30.787724+0000","last_became_peered":"2023-10-09T06:13:30.787724+0000","last_unstale":"2023-10-09T06:19:48.521302+0000","last_undegraded":"2023-10-09T06:19:48.521302+0000","last_fullsized":"2023-10-09T06:19:48.521302+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:38.376770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":133,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.522532+0000","last_change":"2023-10-09T06:18:32.884499+0000","last_active":"2023-10-09T06:19:48.522532+0000","last_peered":"2023-10-09T06:19:48.522532+0000","last_clean":"2023-10-09T06:19:48.522532+0000","last_became_active":"2023-10-09T06:18:32.884232+0000","last_became_peered":"2023-10-09T06:18:32.884232+0000","last_unstale":"2023-10-09T06:19:48.522532+0000","last_undegraded":"2023-10-09T06:19:48.522532+0000","last_fullsized":"2023-10-09T06:19:48.522532+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:26.922060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":875,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.472483+0000","last_change":"2023-10-09T06:12:55.750923+0000","last_active":"2023-10-09T06:19:48.472483+0000","last_peered":"2023-10-09T06:19:48.472483+0000","last_clean":"2023-10-09T06:19:48.472483+0000","last_became_active":"2023-10-09T06:12:55.750521+0000","last_became_peered":"2023-10-09T06:12:55.750521+0000","last_unstale":"2023-10-09T06:19:48.472483+0000","last_undegraded":"2023-10-09T06:19:48.472483+0000","last_fullsized":"2023-10-09T06:19:48.472483+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.316697+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.316697+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:23.591439+0000","scrub_duration":0.001666561,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":506,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.520325+0000","last_change":"2023-10-09T06:18:12.813660+0000","last_active":"2023-10-09T06:19:48.520325+0000","last_peered":"2023-10-09T06:19:48.520325+0000","last_clean":"2023-10-09T06:19:48.520325+0000","last_became_active":"2023-10-09T06:18:12.813327+0000","last_became_peered":"2023-10-09T06:18:12.813327+0000","last_unstale":"2023-10-09T06:19:48.520325+0000","last_undegraded":"2023-10-09T06:19:48.520325+0000","last_fullsized":"2023-10-09T06:19:48.520325+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:55.382525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":597,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473837+0000","last_change":"2023-10-09T06:13:30.788284+0000","last_active":"2023-10-09T06:19:48.473837+0000","last_peered":"2023-10-09T06:19:48.473837+0000","last_clean":"2023-10-09T06:19:48.473837+0000","last_became_active":"2023-10-09T06:13:30.787933+0000","last_became_peered":"2023-10-09T06:13:30.787933+0000","last_unstale":"2023-10-09T06:19:48.473837+0000","last_undegraded":"2023-10-09T06:19:48.473837+0000","last_fullsized":"2023-10-09T06:19:48.473837+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:10.735438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":975,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.471365+0000","last_change":"2023-10-09T06:18:33.815711+0000","last_active":"2023-10-09T06:19:48.471365+0000","last_peered":"2023-10-09T06:19:48.471365+0000","last_clean":"2023-10-09T06:19:48.471365+0000","last_became_active":"2023-10-09T06:18:33.815143+0000","last_became_peered":"2023-10-09T06:18:33.815143+0000","last_unstale":"2023-10-09T06:19:48.471365+0000","last_undegraded":"2023-10-09T06:19:48.471365+0000","last_fullsized":"2023-10-09T06:19:48.471365+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:40.388125+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:40.388125+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:43.731337+0000","scrub_duration":0.0055576990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":262,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.530945+0000","last_change":"2023-10-09T06:18:33.812677+0000","last_active":"2023-10-09T06:19:48.530945+0000","last_peered":"2023-10-09T06:19:48.530945+0000","last_clean":"2023-10-09T06:19:48.530945+0000","last_became_active":"2023-10-09T06:18:33.812289+0000","last_became_peered":"2023-10-09T06:18:33.812289+0000","last_unstale":"2023-10-09T06:19:48.530945+0000","last_undegraded":"2023-10-09T06:19:48.530945+0000","last_fullsized":"2023-10-09T06:19:48.530945+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:02.279506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":160,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473827+0000","last_change":"2023-10-09T06:18:32.878768+0000","last_active":"2023-10-09T06:19:48.473827+0000","last_peered":"2023-10-09T06:19:48.473827+0000","last_clean":"2023-10-09T06:19:48.473827+0000","last_became_active":"2023-10-09T06:18:32.878461+0000","last_became_peered":"2023-10-09T06:18:32.878461+0000","last_unstale":"2023-10-09T06:19:48.473827+0000","last_undegraded":"2023-10-09T06:19:48.473827+0000","last_fullsized":"2023-10-09T06:19:48.473827+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:55.024045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"633'13389","reported_seq":17262,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:55.261750+0000","last_change":"2023-10-09T06:19:27.377538+0000","last_active":"2023-10-09T06:19:55.261750+0000","last_peered":"2023-10-09T06:19:55.261750+0000","last_clean":"2023-10-09T06:19:55.261750+0000","last_became_active":"2023-10-09T06:18:55.965695+0000","last_became_peered":"2023-10-09T06:18:55.965695+0000","last_unstale":"2023-10-09T06:19:55.261750+0000","last_undegraded":"2023-10-09T06:19:55.261750+0000","last_fullsized":"2023-10-09T06:19:55.261750+0000","mapping_epoch":617,"log_start":"602'8180","ondisk_log_start":"602'8180","created":572,"last_epoch_clean":626,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5209,"log_dups_size":0,"ondisk_log_size":5209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:44.689199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14352384,"num_objects":219,"num_object_clones":0,"num_object_copies":657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":219,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13389,"num_write_kb":435456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1887,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":814,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.522565+0000","last_change":"2023-10-09T06:13:33.701521+0000","last_active":"2023-10-09T06:19:48.522565+0000","last_peered":"2023-10-09T06:19:48.522565+0000","last_clean":"2023-10-09T06:19:48.522565+0000","last_became_active":"2023-10-09T06:13:33.700870+0000","last_became_peered":"2023-10-09T06:13:33.700870+0000","last_unstale":"2023-10-09T06:19:48.522565+0000","last_undegraded":"2023-10-09T06:19:48.522565+0000","last_fullsized":"2023-10-09T06:19:48.522565+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:10:59.422551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:10:59.422551+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:54.402494+0000","scrub_duration":0.0059716659999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":120,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.520453+0000","last_change":"2023-10-09T06:13:30.822903+0000","last_active":"2023-10-09T06:19:48.520453+0000","last_peered":"2023-10-09T06:19:48.520453+0000","last_clean":"2023-10-09T06:19:48.520453+0000","last_became_active":"2023-10-09T06:13:30.799013+0000","last_became_peered":"2023-10-09T06:13:30.799013+0000","last_unstale":"2023-10-09T06:19:48.520453+0000","last_undegraded":"2023-10-09T06:19:48.520453+0000","last_fullsized":"2023-10-09T06:19:48.520453+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:44.271821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":157,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.520031+0000","last_change":"2023-10-09T06:18:33.824502+0000","last_active":"2023-10-09T06:19:48.520031+0000","last_peered":"2023-10-09T06:19:48.520031+0000","last_clean":"2023-10-09T06:19:48.520031+0000","last_became_active":"2023-10-09T06:18:33.824063+0000","last_became_peered":"2023-10-09T06:18:33.824063+0000","last_unstale":"2023-10-09T06:19:48.520031+0000","last_undegraded":"2023-10-09T06:19:48.520031+0000","last_fullsized":"2023-10-09T06:19:48.520031+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:43.724163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.6","version":"633'13305","reported_seq":13376,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:55.260397+0000","last_change":"2023-10-09T06:16:02.756362+0000","last_active":"2023-10-09T06:19:55.260397+0000","last_peered":"2023-10-09T06:19:55.260397+0000","last_clean":"2023-10-09T06:19:55.260397+0000","last_became_active":"2023-10-09T06:16:02.756033+0000","last_became_peered":"2023-10-09T06:16:02.756033+0000","last_unstale":"2023-10-09T06:19:55.260397+0000","last_undegraded":"2023-10-09T06:19:55.260397+0000","last_fullsized":"2023-10-09T06:19:55.260397+0000","mapping_epoch":572,"log_start":"602'8114","ondisk_log_start":"602'8114","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5191,"log_dups_size":0,"ondisk_log_size":5191,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:07.492688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13172736,"num_objects":201,"num_object_clones":0,"num_object_copies":603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13305,"num_write_kb":432192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":1049,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473357+0000","last_change":"2023-10-09T06:18:15.867832+0000","last_active":"2023-10-09T06:19:48.473357+0000","last_peered":"2023-10-09T06:19:48.473357+0000","last_clean":"2023-10-09T06:19:48.473357+0000","last_became_active":"2023-10-09T06:18:15.864512+0000","last_became_peered":"2023-10-09T06:18:15.864512+0000","last_unstale":"2023-10-09T06:19:48.473357+0000","last_undegraded":"2023-10-09T06:19:48.473357+0000","last_fullsized":"2023-10-09T06:19:48.473357+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:03.241335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:03.241335+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:45.197013+0000","scrub_duration":0.017780547000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":120,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.522467+0000","last_change":"2023-10-09T06:13:30.829763+0000","last_active":"2023-10-09T06:19:48.522467+0000","last_peered":"2023-10-09T06:19:48.522467+0000","last_clean":"2023-10-09T06:19:48.522467+0000","last_became_active":"2023-10-09T06:13:30.802486+0000","last_became_peered":"2023-10-09T06:13:30.802486+0000","last_unstale":"2023-10-09T06:19:48.522467+0000","last_undegraded":"2023-10-09T06:19:48.522467+0000","last_fullsized":"2023-10-09T06:19:48.522467+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:05.468205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":671,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517618+0000","last_change":"2023-10-09T06:01:54.760367+0000","last_active":"2023-10-09T06:19:48.517618+0000","last_peered":"2023-10-09T06:19:48.517618+0000","last_clean":"2023-10-09T06:19:48.517618+0000","last_became_active":"2023-10-09T06:01:54.760087+0000","last_became_peered":"2023-10-09T06:01:54.760087+0000","last_unstale":"2023-10-09T06:19:48.517618+0000","last_undegraded":"2023-10-09T06:19:48.517618+0000","last_fullsized":"2023-10-09T06:19:48.517618+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:26.000104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"633'13391","reported_seq":22801,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:54.500691+0000","last_change":"2023-10-09T06:19:28.923381+0000","last_active":"2023-10-09T06:19:54.500691+0000","last_peered":"2023-10-09T06:19:54.500691+0000","last_clean":"2023-10-09T06:19:54.500691+0000","last_became_active":"2023-10-09T06:19:28.854095+0000","last_became_peered":"2023-10-09T06:19:28.854095+0000","last_unstale":"2023-10-09T06:19:54.500691+0000","last_undegraded":"2023-10-09T06:19:54.500691+0000","last_fullsized":"2023-10-09T06:19:54.500691+0000","mapping_epoch":627,"log_start":"600'7550","ondisk_log_start":"600'7550","created":572,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5841,"log_dups_size":0,"ondisk_log_size":5841,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:20.378253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16711680,"num_objects":255,"num_object_clones":0,"num_object_copies":765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13391,"num_write_kb":436672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4645,"num_bytes_recovered":178978816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":955,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.518235+0000","last_change":"2023-10-09T06:18:33.817134+0000","last_active":"2023-10-09T06:19:48.518235+0000","last_peered":"2023-10-09T06:19:48.518235+0000","last_clean":"2023-10-09T06:19:48.518235+0000","last_became_active":"2023-10-09T06:18:33.816772+0000","last_became_peered":"2023-10-09T06:18:33.816772+0000","last_unstale":"2023-10-09T06:19:48.518235+0000","last_undegraded":"2023-10-09T06:19:48.518235+0000","last_fullsized":"2023-10-09T06:19:48.518235+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:48.495739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:48.495739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:24.990051+0000","scrub_duration":0.0016345509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":169,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.522466+0000","last_change":"2023-10-09T06:18:33.830787+0000","last_active":"2023-10-09T06:19:48.522466+0000","last_peered":"2023-10-09T06:19:48.522466+0000","last_clean":"2023-10-09T06:19:48.522466+0000","last_became_active":"2023-10-09T06:18:33.830538+0000","last_became_peered":"2023-10-09T06:18:33.830538+0000","last_unstale":"2023-10-09T06:19:48.522466+0000","last_undegraded":"2023-10-09T06:19:48.522466+0000","last_fullsized":"2023-10-09T06:19:48.522466+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:54.818646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":684,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.471070+0000","last_change":"2023-10-09T06:18:12.818137+0000","last_active":"2023-10-09T06:19:48.471070+0000","last_peered":"2023-10-09T06:19:48.471070+0000","last_clean":"2023-10-09T06:19:48.471070+0000","last_became_active":"2023-10-09T06:18:12.817853+0000","last_became_peered":"2023-10-09T06:18:12.817853+0000","last_unstale":"2023-10-09T06:19:48.471070+0000","last_undegraded":"2023-10-09T06:19:48.471070+0000","last_fullsized":"2023-10-09T06:19:48.471070+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:14:01.939320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.4","version":"633'13316","reported_seq":14730,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:54.481048+0000","last_change":"2023-10-09T06:18:13.830297+0000","last_active":"2023-10-09T06:19:54.481048+0000","last_peered":"2023-10-09T06:19:54.481048+0000","last_clean":"2023-10-09T06:19:54.481048+0000","last_became_active":"2023-10-09T06:18:13.814894+0000","last_became_peered":"2023-10-09T06:18:13.814894+0000","last_unstale":"2023-10-09T06:19:54.481048+0000","last_undegraded":"2023-10-09T06:19:54.481048+0000","last_fullsized":"2023-10-09T06:19:54.481048+0000","mapping_epoch":604,"log_start":"600'7462","ondisk_log_start":"600'7462","created":572,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5854,"log_dups_size":0,"ondisk_log_size":5854,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:32.472441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16646144,"num_objects":254,"num_object_clones":0,"num_object_copies":762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13316,"num_write_kb":434240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":659,"num_bytes_recovered":43188224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":848,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.518111+0000","last_change":"2023-10-09T06:13:33.708291+0000","last_active":"2023-10-09T06:19:48.518111+0000","last_peered":"2023-10-09T06:19:48.518111+0000","last_clean":"2023-10-09T06:19:48.518111+0000","last_became_active":"2023-10-09T06:13:33.707930+0000","last_became_peered":"2023-10-09T06:13:33.707930+0000","last_unstale":"2023-10-09T06:19:48.518111+0000","last_undegraded":"2023-10-09T06:19:48.518111+0000","last_fullsized":"2023-10-09T06:19:48.518111+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:16.361097+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:16.361097+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:18.111404+0000","scrub_duration":0.028533749000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":440,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.513437+0000","last_change":"2023-10-09T06:18:33.818994+0000","last_active":"2023-10-09T06:19:51.513437+0000","last_peered":"2023-10-09T06:19:51.513437+0000","last_clean":"2023-10-09T06:19:51.513437+0000","last_became_active":"2023-10-09T06:18:33.818598+0000","last_became_peered":"2023-10-09T06:18:33.818598+0000","last_unstale":"2023-10-09T06:19:51.513437+0000","last_undegraded":"2023-10-09T06:19:51.513437+0000","last_fullsized":"2023-10-09T06:19:51.513437+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:59.239942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":648,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.531358+0000","last_change":"2023-10-09T06:09:27.558353+0000","last_active":"2023-10-09T06:19:48.531358+0000","last_peered":"2023-10-09T06:19:48.531358+0000","last_clean":"2023-10-09T06:19:48.531358+0000","last_became_active":"2023-10-09T06:09:27.557233+0000","last_became_peered":"2023-10-09T06:09:27.557233+0000","last_unstale":"2023-10-09T06:19:48.531358+0000","last_undegraded":"2023-10-09T06:19:48.531358+0000","last_fullsized":"2023-10-09T06:19:48.531358+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:53.149350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"633'13688","reported_seq":13793,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:54.494923+0000","last_change":"2023-10-09T06:18:13.827731+0000","last_active":"2023-10-09T06:19:54.494923+0000","last_peered":"2023-10-09T06:19:54.494923+0000","last_clean":"2023-10-09T06:19:54.494923+0000","last_became_active":"2023-10-09T06:18:13.813485+0000","last_became_peered":"2023-10-09T06:18:13.813485+0000","last_unstale":"2023-10-09T06:19:54.494923+0000","last_undegraded":"2023-10-09T06:19:54.494923+0000","last_fullsized":"2023-10-09T06:19:54.494923+0000","mapping_epoch":604,"log_start":"601'7864","ondisk_log_start":"601'7864","created":572,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5824,"log_dups_size":0,"ondisk_log_size":5824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:57.856421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15597568,"num_objects":238,"num_object_clones":0,"num_object_copies":714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13688,"num_write_kb":445632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":1056,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.520240+0000","last_change":"2023-10-09T06:18:32.884000+0000","last_active":"2023-10-09T06:19:48.520240+0000","last_peered":"2023-10-09T06:19:48.520240+0000","last_clean":"2023-10-09T06:19:48.520240+0000","last_became_active":"2023-10-09T06:18:32.883640+0000","last_became_peered":"2023-10-09T06:18:32.883640+0000","last_unstale":"2023-10-09T06:19:48.520240+0000","last_undegraded":"2023-10-09T06:19:48.520240+0000","last_fullsized":"2023-10-09T06:19:48.520240+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:42.795781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:42.795781+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:39.855289+0000","scrub_duration":0.0091262979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":439,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.513767+0000","last_change":"2023-10-09T06:09:27.565666+0000","last_active":"2023-10-09T06:19:51.513767+0000","last_peered":"2023-10-09T06:19:51.513767+0000","last_clean":"2023-10-09T06:19:51.513767+0000","last_became_active":"2023-10-09T06:09:27.565324+0000","last_became_peered":"2023-10-09T06:09:27.565324+0000","last_unstale":"2023-10-09T06:19:51.513767+0000","last_undegraded":"2023-10-09T06:19:51.513767+0000","last_fullsized":"2023-10-09T06:19:51.513767+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:51.216947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":648,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517930+0000","last_change":"2023-10-09T06:18:33.812578+0000","last_active":"2023-10-09T06:19:48.517930+0000","last_peered":"2023-10-09T06:19:48.517930+0000","last_clean":"2023-10-09T06:19:48.517930+0000","last_became_active":"2023-10-09T06:18:33.812233+0000","last_became_peered":"2023-10-09T06:18:33.812233+0000","last_unstale":"2023-10-09T06:19:48.517930+0000","last_undegraded":"2023-10-09T06:19:48.517930+0000","last_fullsized":"2023-10-09T06:19:48.517930+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:49.671676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"633'13440","reported_seq":13511,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:56.160519+0000","last_change":"2023-10-09T06:16:02.747886+0000","last_active":"2023-10-09T06:19:56.160519+0000","last_peered":"2023-10-09T06:19:56.160519+0000","last_clean":"2023-10-09T06:19:56.160519+0000","last_became_active":"2023-10-09T06:16:02.747569+0000","last_became_peered":"2023-10-09T06:16:02.747569+0000","last_unstale":"2023-10-09T06:19:56.160519+0000","last_undegraded":"2023-10-09T06:19:56.160519+0000","last_fullsized":"2023-10-09T06:19:56.160519+0000","mapping_epoch":572,"log_start":"601'7757","ondisk_log_start":"601'7757","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5683,"log_dups_size":0,"ondisk_log_size":5683,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:18.720375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11141120,"num_objects":170,"num_object_clones":0,"num_object_copies":510,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":170,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13440,"num_write_kb":435520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":965,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.472828+0000","last_change":"2023-10-09T06:18:33.829178+0000","last_active":"2023-10-09T06:19:48.472828+0000","last_peered":"2023-10-09T06:19:48.472828+0000","last_clean":"2023-10-09T06:19:48.472828+0000","last_became_active":"2023-10-09T06:18:33.828843+0000","last_became_peered":"2023-10-09T06:18:33.828843+0000","last_unstale":"2023-10-09T06:19:48.472828+0000","last_undegraded":"2023-10-09T06:19:48.472828+0000","last_fullsized":"2023-10-09T06:19:48.472828+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.282393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.282393+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:15:35.723179+0000","scrub_duration":0.01402403,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":572,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517714+0000","last_change":"2023-10-09T06:18:12.815749+0000","last_active":"2023-10-09T06:19:48.517714+0000","last_peered":"2023-10-09T06:19:48.517714+0000","last_clean":"2023-10-09T06:19:48.517714+0000","last_became_active":"2023-10-09T06:18:12.815447+0000","last_became_peered":"2023-10-09T06:18:12.815447+0000","last_unstale":"2023-10-09T06:19:48.517714+0000","last_undegraded":"2023-10-09T06:19:48.517714+0000","last_fullsized":"2023-10-09T06:19:48.517714+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:42.083129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":711,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.518293+0000","last_change":"2023-10-09T06:18:12.820093+0000","last_active":"2023-10-09T06:19:48.518293+0000","last_peered":"2023-10-09T06:19:48.518293+0000","last_clean":"2023-10-09T06:19:48.518293+0000","last_became_active":"2023-10-09T06:18:12.819805+0000","last_became_peered":"2023-10-09T06:18:12.819805+0000","last_unstale":"2023-10-09T06:19:48.518293+0000","last_undegraded":"2023-10-09T06:19:48.518293+0000","last_fullsized":"2023-10-09T06:19:48.518293+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:54:00.429779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"633'13181","reported_seq":16209,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:56.118516+0000","last_change":"2023-10-09T06:18:13.823458+0000","last_active":"2023-10-09T06:19:56.118516+0000","last_peered":"2023-10-09T06:19:56.118516+0000","last_clean":"2023-10-09T06:19:56.118516+0000","last_became_active":"2023-10-09T06:18:13.813080+0000","last_became_peered":"2023-10-09T06:18:13.813080+0000","last_unstale":"2023-10-09T06:19:56.118516+0000","last_undegraded":"2023-10-09T06:19:56.118516+0000","last_fullsized":"2023-10-09T06:19:56.118516+0000","mapping_epoch":604,"log_start":"601'7511","ondisk_log_start":"601'7511","created":572,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5670,"log_dups_size":0,"ondisk_log_size":5670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:48:07.669455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12255232,"num_objects":187,"num_object_clones":0,"num_object_copies":561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13181,"num_write_kb":427776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1468,"num_bytes_recovered":96206848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":984,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.472312+0000","last_change":"2023-10-09T06:18:32.812418+0000","last_active":"2023-10-09T06:19:48.472312+0000","last_peered":"2023-10-09T06:19:48.472312+0000","last_clean":"2023-10-09T06:19:48.472312+0000","last_became_active":"2023-10-09T06:18:32.812079+0000","last_became_peered":"2023-10-09T06:18:32.812079+0000","last_unstale":"2023-10-09T06:19:48.472312+0000","last_undegraded":"2023-10-09T06:19:48.472312+0000","last_fullsized":"2023-10-09T06:19:48.472312+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.373748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.373748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:22.098164+0000","scrub_duration":0.0024576450000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":633,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517730+0000","last_change":"2023-10-09T06:16:02.744252+0000","last_active":"2023-10-09T06:19:48.517730+0000","last_peered":"2023-10-09T06:19:48.517730+0000","last_clean":"2023-10-09T06:19:48.517730+0000","last_became_active":"2023-10-09T06:16:02.743947+0000","last_became_peered":"2023-10-09T06:16:02.743947+0000","last_unstale":"2023-10-09T06:19:48.517730+0000","last_undegraded":"2023-10-09T06:19:48.517730+0000","last_fullsized":"2023-10-09T06:19:48.517730+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:52.293310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":684,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.518215+0000","last_change":"2023-10-09T06:18:12.819118+0000","last_active":"2023-10-09T06:19:48.518215+0000","last_peered":"2023-10-09T06:19:48.518215+0000","last_clean":"2023-10-09T06:19:48.518215+0000","last_became_active":"2023-10-09T06:18:12.818820+0000","last_became_peered":"2023-10-09T06:18:12.818820+0000","last_unstale":"2023-10-09T06:19:48.518215+0000","last_undegraded":"2023-10-09T06:19:48.518215+0000","last_fullsized":"2023-10-09T06:19:48.518215+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:26.399422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"633'13692","reported_seq":13763,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:55.253820+0000","last_change":"2023-10-09T06:16:02.744178+0000","last_active":"2023-10-09T06:19:55.253820+0000","last_peered":"2023-10-09T06:19:55.253820+0000","last_clean":"2023-10-09T06:19:55.253820+0000","last_became_active":"2023-10-09T06:16:02.743835+0000","last_became_peered":"2023-10-09T06:16:02.743835+0000","last_unstale":"2023-10-09T06:19:55.253820+0000","last_undegraded":"2023-10-09T06:19:55.253820+0000","last_fullsized":"2023-10-09T06:19:55.253820+0000","mapping_epoch":572,"log_start":"602'8450","ondisk_log_start":"602'8450","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5242,"log_dups_size":0,"ondisk_log_size":5242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:02.861316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13762560,"num_objects":210,"num_object_clones":0,"num_object_copies":630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13692,"num_write_kb":444864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":751,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517614+0000","last_change":"2023-10-09T06:18:33.812696+0000","last_active":"2023-10-09T06:19:48.517614+0000","last_peered":"2023-10-09T06:19:48.517614+0000","last_clean":"2023-10-09T06:19:48.517614+0000","last_became_active":"2023-10-09T06:18:33.812371+0000","last_became_peered":"2023-10-09T06:18:33.812371+0000","last_unstale":"2023-10-09T06:19:48.517614+0000","last_undegraded":"2023-10-09T06:19:48.517614+0000","last_fullsized":"2023-10-09T06:19:48.517614+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:39.633410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:39.633410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:05.464966+0000","scrub_duration":0.0039938459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":447,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.472935+0000","last_change":"2023-10-09T06:13:30.781849+0000","last_active":"2023-10-09T06:19:48.472935+0000","last_peered":"2023-10-09T06:19:48.472935+0000","last_clean":"2023-10-09T06:19:48.472935+0000","last_became_active":"2023-10-09T06:13:30.781084+0000","last_became_peered":"2023-10-09T06:13:30.781084+0000","last_unstale":"2023-10-09T06:19:48.472935+0000","last_undegraded":"2023-10-09T06:19:48.472935+0000","last_fullsized":"2023-10-09T06:19:48.472935+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:02.199298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":317,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.472215+0000","last_change":"2023-10-09T06:01:54.761768+0000","last_active":"2023-10-09T06:19:48.472215+0000","last_peered":"2023-10-09T06:19:48.472215+0000","last_clean":"2023-10-09T06:19:48.472215+0000","last_became_active":"2023-10-09T06:01:54.761443+0000","last_became_peered":"2023-10-09T06:01:54.761443+0000","last_unstale":"2023-10-09T06:19:48.472215+0000","last_undegraded":"2023-10-09T06:19:48.472215+0000","last_fullsized":"2023-10-09T06:19:48.472215+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:51.555987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"633'13286","reported_seq":16788,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:56.114623+0000","last_change":"2023-10-09T06:18:13.836219+0000","last_active":"2023-10-09T06:19:56.114623+0000","last_peered":"2023-10-09T06:19:56.114623+0000","last_clean":"2023-10-09T06:19:56.114623+0000","last_became_active":"2023-10-09T06:18:13.812037+0000","last_became_peered":"2023-10-09T06:18:13.812037+0000","last_unstale":"2023-10-09T06:19:56.114623+0000","last_undegraded":"2023-10-09T06:19:56.114623+0000","last_fullsized":"2023-10-09T06:19:56.114623+0000","mapping_epoch":604,"log_start":"600'7382","ondisk_log_start":"600'7382","created":572,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5904,"log_dups_size":0,"ondisk_log_size":5904,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:58:37.778303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13565976,"num_objects":208,"num_object_clones":0,"num_object_copies":624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":208,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13286,"num_write_kb":431745,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1698,"num_bytes_recovered":111280128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":878,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473884+0000","last_change":"2023-10-09T06:18:15.863360+0000","last_active":"2023-10-09T06:19:48.473884+0000","last_peered":"2023-10-09T06:19:48.473884+0000","last_clean":"2023-10-09T06:19:48.473884+0000","last_became_active":"2023-10-09T06:18:15.863003+0000","last_became_peered":"2023-10-09T06:18:15.863003+0000","last_unstale":"2023-10-09T06:19:48.473884+0000","last_undegraded":"2023-10-09T06:19:48.473884+0000","last_fullsized":"2023-10-09T06:19:48.473884+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:12:22.289803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:12:22.289803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:40.618745+0000","scrub_duration":0.0076565050000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":380,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.471236+0000","last_change":"2023-10-09T06:12:04.680247+0000","last_active":"2023-10-09T06:19:48.471236+0000","last_peered":"2023-10-09T06:19:48.471236+0000","last_clean":"2023-10-09T06:19:48.471236+0000","last_became_active":"2023-10-09T06:12:04.679943+0000","last_became_peered":"2023-10-09T06:12:04.679943+0000","last_unstale":"2023-10-09T06:19:48.471236+0000","last_undegraded":"2023-10-09T06:19:48.471236+0000","last_fullsized":"2023-10-09T06:19:48.471236+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:56.692774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":467,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473377+0000","last_change":"2023-10-09T06:18:33.813934+0000","last_active":"2023-10-09T06:19:48.473377+0000","last_peered":"2023-10-09T06:19:48.473377+0000","last_clean":"2023-10-09T06:19:48.473377+0000","last_became_active":"2023-10-09T06:18:33.813570+0000","last_became_peered":"2023-10-09T06:18:33.813570+0000","last_unstale":"2023-10-09T06:19:48.473377+0000","last_undegraded":"2023-10-09T06:19:48.473377+0000","last_fullsized":"2023-10-09T06:19:48.473377+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:20.477422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":366,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.471208+0000","last_change":"2023-10-09T06:18:32.877171+0000","last_active":"2023-10-09T06:19:48.471208+0000","last_peered":"2023-10-09T06:19:48.471208+0000","last_clean":"2023-10-09T06:19:48.471208+0000","last_became_active":"2023-10-09T06:18:32.876933+0000","last_became_peered":"2023-10-09T06:18:32.876933+0000","last_unstale":"2023-10-09T06:19:48.471208+0000","last_undegraded":"2023-10-09T06:19:48.471208+0000","last_fullsized":"2023-10-09T06:19:48.471208+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:10.529213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"633'13163","reported_seq":13232,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:56.160956+0000","last_change":"2023-10-09T06:16:02.749724+0000","last_active":"2023-10-09T06:19:56.160956+0000","last_peered":"2023-10-09T06:19:56.160956+0000","last_clean":"2023-10-09T06:19:56.160956+0000","last_became_active":"2023-10-09T06:16:02.749368+0000","last_became_peered":"2023-10-09T06:16:02.749368+0000","last_unstale":"2023-10-09T06:19:56.160956+0000","last_undegraded":"2023-10-09T06:19:56.160956+0000","last_fullsized":"2023-10-09T06:19:56.160956+0000","mapping_epoch":572,"log_start":"601'7497","ondisk_log_start":"601'7497","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5666,"log_dups_size":0,"ondisk_log_size":5666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:24.189151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10682368,"num_objects":163,"num_object_clones":0,"num_object_copies":489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13163,"num_write_kb":426432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":903,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.575304+0000","last_change":"2023-10-09T06:18:32.874923+0000","last_active":"2023-10-09T06:19:51.575304+0000","last_peered":"2023-10-09T06:19:51.575304+0000","last_clean":"2023-10-09T06:19:51.575304+0000","last_became_active":"2023-10-09T06:18:32.874574+0000","last_became_peered":"2023-10-09T06:18:32.874574+0000","last_unstale":"2023-10-09T06:19:51.575304+0000","last_undegraded":"2023-10-09T06:19:51.575304+0000","last_fullsized":"2023-10-09T06:19:51.575304+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:40.807378+0000","scrub_duration":0.0016660799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":371,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.471283+0000","last_change":"2023-10-09T06:18:33.815495+0000","last_active":"2023-10-09T06:19:48.471283+0000","last_peered":"2023-10-09T06:19:48.471283+0000","last_clean":"2023-10-09T06:19:48.471283+0000","last_became_active":"2023-10-09T06:18:33.814689+0000","last_became_peered":"2023-10-09T06:18:33.814689+0000","last_unstale":"2023-10-09T06:19:48.471283+0000","last_undegraded":"2023-10-09T06:19:48.471283+0000","last_fullsized":"2023-10-09T06:19:48.471283+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:54.929165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":633,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517691+0000","last_change":"2023-10-09T06:18:12.816566+0000","last_active":"2023-10-09T06:19:48.517691+0000","last_peered":"2023-10-09T06:19:48.517691+0000","last_clean":"2023-10-09T06:19:48.517691+0000","last_became_active":"2023-10-09T06:18:12.816245+0000","last_became_peered":"2023-10-09T06:18:12.816245+0000","last_unstale":"2023-10-09T06:19:48.517691+0000","last_undegraded":"2023-10-09T06:19:48.517691+0000","last_fullsized":"2023-10-09T06:19:48.517691+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:49.240586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":488,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.471324+0000","last_change":"2023-10-09T06:18:12.818212+0000","last_active":"2023-10-09T06:19:48.471324+0000","last_peered":"2023-10-09T06:19:48.471324+0000","last_clean":"2023-10-09T06:19:48.471324+0000","last_became_active":"2023-10-09T06:18:12.817911+0000","last_became_peered":"2023-10-09T06:18:12.817911+0000","last_unstale":"2023-10-09T06:19:48.471324+0000","last_undegraded":"2023-10-09T06:19:48.471324+0000","last_fullsized":"2023-10-09T06:19:48.471324+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:48.259119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"633'13593","reported_seq":22353,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:56.144757+0000","last_change":"2023-10-09T06:19:15.586290+0000","last_active":"2023-10-09T06:19:56.144757+0000","last_peered":"2023-10-09T06:19:56.144757+0000","last_clean":"2023-10-09T06:19:56.144757+0000","last_became_active":"2023-10-09T06:19:15.562959+0000","last_became_peered":"2023-10-09T06:19:15.562959+0000","last_unstale":"2023-10-09T06:19:56.144757+0000","last_undegraded":"2023-10-09T06:19:56.144757+0000","last_fullsized":"2023-10-09T06:19:56.144757+0000","mapping_epoch":623,"log_start":"601'7839","ondisk_log_start":"601'7839","created":572,"last_epoch_clean":624,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5754,"log_dups_size":0,"ondisk_log_size":5754,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:37:06.168866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11206656,"num_objects":171,"num_object_clones":0,"num_object_copies":513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":171,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13593,"num_write_kb":440448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4323,"num_bytes_recovered":157679616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":688,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.521357+0000","last_change":"2023-10-09T06:14:47.339271+0000","last_active":"2023-10-09T06:19:48.521357+0000","last_peered":"2023-10-09T06:19:48.521357+0000","last_clean":"2023-10-09T06:19:48.521357+0000","last_became_active":"2023-10-09T06:13:33.705351+0000","last_became_peered":"2023-10-09T06:13:33.705351+0000","last_unstale":"2023-10-09T06:19:48.521357+0000","last_undegraded":"2023-10-09T06:19:48.521357+0000","last_fullsized":"2023-10-09T06:19:48.521357+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:14:47.339163+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T06:14:47.339163+0000","last_clean_scrub_stamp":"2023-10-09T06:14:47.339163+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:34.887758+0000","scrub_duration":0.0016607729999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":425,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.176488+0000","last_change":"2023-10-09T06:15:16.802788+0000","last_active":"2023-10-09T06:19:51.176488+0000","last_peered":"2023-10-09T06:19:51.176488+0000","last_clean":"2023-10-09T06:19:51.176488+0000","last_became_active":"2023-10-09T06:15:16.802449+0000","last_became_peered":"2023-10-09T06:15:16.802449+0000","last_unstale":"2023-10-09T06:19:51.176488+0000","last_undegraded":"2023-10-09T06:19:51.176488+0000","last_fullsized":"2023-10-09T06:19:51.176488+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:55:30.488174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":438,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.470724+0000","last_change":"2023-10-09T06:18:12.818545+0000","last_active":"2023-10-09T06:19:48.470724+0000","last_peered":"2023-10-09T06:19:48.470724+0000","last_clean":"2023-10-09T06:19:48.470724+0000","last_became_active":"2023-10-09T06:18:12.818281+0000","last_became_peered":"2023-10-09T06:18:12.818281+0000","last_unstale":"2023-10-09T06:19:48.470724+0000","last_undegraded":"2023-10-09T06:19:48.470724+0000","last_fullsized":"2023-10-09T06:19:48.470724+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:31:33.668786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":348,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.472170+0000","last_change":"2023-10-09T06:18:32.818287+0000","last_active":"2023-10-09T06:19:48.472170+0000","last_peered":"2023-10-09T06:19:48.472170+0000","last_clean":"2023-10-09T06:19:48.472170+0000","last_became_active":"2023-10-09T06:18:32.817946+0000","last_became_peered":"2023-10-09T06:18:32.817946+0000","last_unstale":"2023-10-09T06:19:48.472170+0000","last_undegraded":"2023-10-09T06:19:48.472170+0000","last_fullsized":"2023-10-09T06:19:48.472170+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:17.156210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"633'13246","reported_seq":17299,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:56.148502+0000","last_change":"2023-10-09T06:19:48.514562+0000","last_active":"2023-10-09T06:19:56.148502+0000","last_peered":"2023-10-09T06:19:56.148502+0000","last_clean":"2023-10-09T06:19:56.148502+0000","last_became_active":"2023-10-09T06:19:48.476882+0000","last_became_peered":"2023-10-09T06:19:48.476882+0000","last_unstale":"2023-10-09T06:19:56.148502+0000","last_undegraded":"2023-10-09T06:19:56.148502+0000","last_fullsized":"2023-10-09T06:19:56.148502+0000","mapping_epoch":631,"log_start":"601'7566","ondisk_log_start":"601'7566","created":572,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5680,"log_dups_size":0,"ondisk_log_size":5680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:11.550660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8912896,"num_objects":136,"num_object_clones":0,"num_object_copies":408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":136,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13246,"num_write_kb":428224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1971,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":143,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.521121+0000","last_change":"2023-10-09T06:18:32.880094+0000","last_active":"2023-10-09T06:19:48.521121+0000","last_peered":"2023-10-09T06:19:48.521121+0000","last_clean":"2023-10-09T06:19:48.521121+0000","last_became_active":"2023-10-09T06:18:32.879777+0000","last_became_peered":"2023-10-09T06:18:32.879777+0000","last_unstale":"2023-10-09T06:19:48.521121+0000","last_undegraded":"2023-10-09T06:19:48.521121+0000","last_fullsized":"2023-10-09T06:19:48.521121+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:54.918094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":716,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.519993+0000","last_change":"2023-10-09T06:18:15.852226+0000","last_active":"2023-10-09T06:19:48.519993+0000","last_peered":"2023-10-09T06:19:48.519993+0000","last_clean":"2023-10-09T06:19:48.519993+0000","last_became_active":"2023-10-09T06:18:15.851868+0000","last_became_peered":"2023-10-09T06:18:15.851868+0000","last_unstale":"2023-10-09T06:19:48.519993+0000","last_undegraded":"2023-10-09T06:19:48.519993+0000","last_fullsized":"2023-10-09T06:19:48.519993+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:35.783434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:35.783434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:57.511684+0000","scrub_duration":0.0043794799999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":458,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473369+0000","last_change":"2023-10-09T06:18:33.827842+0000","last_active":"2023-10-09T06:19:48.473369+0000","last_peered":"2023-10-09T06:19:48.473369+0000","last_clean":"2023-10-09T06:19:48.473369+0000","last_became_active":"2023-10-09T06:18:33.827482+0000","last_became_peered":"2023-10-09T06:18:33.827482+0000","last_unstale":"2023-10-09T06:19:48.473369+0000","last_undegraded":"2023-10-09T06:19:48.473369+0000","last_fullsized":"2023-10-09T06:19:48.473369+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:46.769907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":583,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.518214+0000","last_change":"2023-10-09T06:18:12.820039+0000","last_active":"2023-10-09T06:19:48.518214+0000","last_peered":"2023-10-09T06:19:48.518214+0000","last_clean":"2023-10-09T06:19:48.518214+0000","last_became_active":"2023-10-09T06:18:12.819410+0000","last_became_peered":"2023-10-09T06:18:12.819410+0000","last_unstale":"2023-10-09T06:19:48.518214+0000","last_undegraded":"2023-10-09T06:19:48.518214+0000","last_fullsized":"2023-10-09T06:19:48.518214+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:24.514809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"633'13358","reported_seq":13447,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:56.143482+0000","last_change":"2023-10-09T06:18:13.823451+0000","last_active":"2023-10-09T06:19:56.143482+0000","last_peered":"2023-10-09T06:19:56.143482+0000","last_clean":"2023-10-09T06:19:56.143482+0000","last_became_active":"2023-10-09T06:18:13.811684+0000","last_became_peered":"2023-10-09T06:18:13.811684+0000","last_unstale":"2023-10-09T06:19:56.143482+0000","last_undegraded":"2023-10-09T06:19:56.143482+0000","last_fullsized":"2023-10-09T06:19:56.143482+0000","mapping_epoch":604,"log_start":"601'7696","ondisk_log_start":"601'7696","created":572,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5662,"log_dups_size":0,"ondisk_log_size":5662,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:01:16.295158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11010048,"num_objects":168,"num_object_clones":0,"num_object_copies":504,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":168,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13358,"num_write_kb":432832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":944,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.472392+0000","last_change":"2023-10-09T06:18:15.847552+0000","last_active":"2023-10-09T06:19:48.472392+0000","last_peered":"2023-10-09T06:19:48.472392+0000","last_clean":"2023-10-09T06:19:48.472392+0000","last_became_active":"2023-10-09T06:18:15.847229+0000","last_became_peered":"2023-10-09T06:18:15.847229+0000","last_unstale":"2023-10-09T06:19:48.472392+0000","last_undegraded":"2023-10-09T06:19:48.472392+0000","last_fullsized":"2023-10-09T06:19:48.472392+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:57.627296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:57.627296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:02:19.535788+0000","scrub_duration":0.001913166,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":526,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.520110+0000","last_change":"2023-10-09T06:09:27.544373+0000","last_active":"2023-10-09T06:19:48.520110+0000","last_peered":"2023-10-09T06:19:48.520110+0000","last_clean":"2023-10-09T06:19:48.520110+0000","last_became_active":"2023-10-09T06:09:27.544121+0000","last_became_peered":"2023-10-09T06:09:27.544121+0000","last_unstale":"2023-10-09T06:19:48.520110+0000","last_undegraded":"2023-10-09T06:19:48.520110+0000","last_fullsized":"2023-10-09T06:19:48.520110+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:40.429491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":573,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.575137+0000","last_change":"2023-10-09T06:12:52.670573+0000","last_active":"2023-10-09T06:19:51.575137+0000","last_peered":"2023-10-09T06:19:51.575137+0000","last_clean":"2023-10-09T06:19:51.575137+0000","last_became_active":"2023-10-09T06:12:52.670242+0000","last_became_peered":"2023-10-09T06:12:52.670242+0000","last_unstale":"2023-10-09T06:19:51.575137+0000","last_undegraded":"2023-10-09T06:19:51.575137+0000","last_fullsized":"2023-10-09T06:19:51.575137+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:25.237580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":720,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.531533+0000","last_change":"2023-10-09T06:15:16.805514+0000","last_active":"2023-10-09T06:19:48.531533+0000","last_peered":"2023-10-09T06:19:48.531533+0000","last_clean":"2023-10-09T06:19:48.531533+0000","last_became_active":"2023-10-09T06:15:16.805160+0000","last_became_peered":"2023-10-09T06:15:16.805160+0000","last_unstale":"2023-10-09T06:19:48.531533+0000","last_undegraded":"2023-10-09T06:19:48.531533+0000","last_fullsized":"2023-10-09T06:19:48.531533+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:30.153699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"633'13443","reported_seq":13514,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:55.217663+0000","last_change":"2023-10-09T06:16:02.755231+0000","last_active":"2023-10-09T06:19:55.217663+0000","last_peered":"2023-10-09T06:19:55.217663+0000","last_clean":"2023-10-09T06:19:55.217663+0000","last_became_active":"2023-10-09T06:16:02.754877+0000","last_became_peered":"2023-10-09T06:16:02.754877+0000","last_unstale":"2023-10-09T06:19:55.217663+0000","last_undegraded":"2023-10-09T06:19:55.217663+0000","last_fullsized":"2023-10-09T06:19:55.217663+0000","mapping_epoch":572,"log_start":"602'8246","ondisk_log_start":"602'8246","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5197,"log_dups_size":0,"ondisk_log_size":5197,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:16:25.397496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14483456,"num_objects":221,"num_object_clones":0,"num_object_copies":663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13443,"num_write_kb":437248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":869,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473837+0000","last_change":"2023-10-09T06:18:15.863482+0000","last_active":"2023-10-09T06:19:48.473837+0000","last_peered":"2023-10-09T06:19:48.473837+0000","last_clean":"2023-10-09T06:19:48.473837+0000","last_became_active":"2023-10-09T06:18:15.863069+0000","last_became_peered":"2023-10-09T06:18:15.863069+0000","last_unstale":"2023-10-09T06:19:48.473837+0000","last_undegraded":"2023-10-09T06:19:48.473837+0000","last_fullsized":"2023-10-09T06:19:48.473837+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:45.844892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:45.844892+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:50.117604+0000","scrub_duration":0.0017521489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":116,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.522121+0000","last_change":"2023-10-09T06:13:30.799535+0000","last_active":"2023-10-09T06:19:48.522121+0000","last_peered":"2023-10-09T06:19:48.522121+0000","last_clean":"2023-10-09T06:19:48.522121+0000","last_became_active":"2023-10-09T06:13:30.799227+0000","last_became_peered":"2023-10-09T06:13:30.799227+0000","last_unstale":"2023-10-09T06:19:48.522121+0000","last_undegraded":"2023-10-09T06:19:48.522121+0000","last_fullsized":"2023-10-09T06:19:48.522121+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:14.741742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":378,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.471508+0000","last_change":"2023-10-09T06:18:32.812317+0000","last_active":"2023-10-09T06:19:48.471508+0000","last_peered":"2023-10-09T06:19:48.471508+0000","last_clean":"2023-10-09T06:19:48.471508+0000","last_became_active":"2023-10-09T06:18:32.812010+0000","last_became_peered":"2023-10-09T06:18:32.812010+0000","last_unstale":"2023-10-09T06:19:48.471508+0000","last_undegraded":"2023-10-09T06:19:48.471508+0000","last_fullsized":"2023-10-09T06:19:48.471508+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:31.460164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":133,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.520952+0000","last_change":"2023-10-09T06:18:32.879994+0000","last_active":"2023-10-09T06:19:48.520952+0000","last_peered":"2023-10-09T06:19:48.520952+0000","last_clean":"2023-10-09T06:19:48.520952+0000","last_became_active":"2023-10-09T06:18:32.879702+0000","last_became_peered":"2023-10-09T06:18:32.879702+0000","last_unstale":"2023-10-09T06:19:48.520952+0000","last_undegraded":"2023-10-09T06:19:48.520952+0000","last_fullsized":"2023-10-09T06:19:48.520952+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:56.654232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.f","version":"633'13427","reported_seq":13496,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:54.467100+0000","last_change":"2023-10-09T06:16:02.755513+0000","last_active":"2023-10-09T06:19:54.467100+0000","last_peered":"2023-10-09T06:19:54.467100+0000","last_clean":"2023-10-09T06:19:54.467100+0000","last_became_active":"2023-10-09T06:16:02.755072+0000","last_became_peered":"2023-10-09T06:16:02.755072+0000","last_unstale":"2023-10-09T06:19:54.467100+0000","last_undegraded":"2023-10-09T06:19:54.467100+0000","last_fullsized":"2023-10-09T06:19:54.467100+0000","mapping_epoch":572,"log_start":"600'7571","ondisk_log_start":"600'7571","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5856,"log_dups_size":0,"ondisk_log_size":5856,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:11.106599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16318464,"num_objects":249,"num_object_clones":0,"num_object_copies":747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13427,"num_write_kb":437632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":1185,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.518378+0000","last_change":"2023-10-09T06:18:33.816307+0000","last_active":"2023-10-09T06:19:48.518378+0000","last_peered":"2023-10-09T06:19:48.518378+0000","last_clean":"2023-10-09T06:19:48.518378+0000","last_became_active":"2023-10-09T06:18:33.815766+0000","last_became_peered":"2023-10-09T06:18:33.815766+0000","last_unstale":"2023-10-09T06:19:48.518378+0000","last_undegraded":"2023-10-09T06:19:48.518378+0000","last_fullsized":"2023-10-09T06:19:48.518378+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:17.344518+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:17.344518+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:59.097601+0000","scrub_duration":0.0015914099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":692,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473972+0000","last_change":"2023-10-09T06:13:30.798394+0000","last_active":"2023-10-09T06:19:48.473972+0000","last_peered":"2023-10-09T06:19:48.473972+0000","last_clean":"2023-10-09T06:19:48.473972+0000","last_became_active":"2023-10-09T06:13:30.790913+0000","last_became_peered":"2023-10-09T06:13:30.790913+0000","last_unstale":"2023-10-09T06:19:48.473972+0000","last_undegraded":"2023-10-09T06:19:48.473972+0000","last_fullsized":"2023-10-09T06:19:48.473972+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:26.508690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":470,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.472121+0000","last_change":"2023-10-09T06:18:12.818370+0000","last_active":"2023-10-09T06:19:48.472121+0000","last_peered":"2023-10-09T06:19:48.472121+0000","last_clean":"2023-10-09T06:19:48.472121+0000","last_became_active":"2023-10-09T06:18:12.818031+0000","last_became_peered":"2023-10-09T06:18:12.818031+0000","last_unstale":"2023-10-09T06:19:48.472121+0000","last_undegraded":"2023-10-09T06:19:48.472121+0000","last_fullsized":"2023-10-09T06:19:48.472121+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:20:31.304808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":441,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473146+0000","last_change":"2023-10-09T06:13:30.782038+0000","last_active":"2023-10-09T06:19:48.473146+0000","last_peered":"2023-10-09T06:19:48.473146+0000","last_clean":"2023-10-09T06:19:48.473146+0000","last_became_active":"2023-10-09T06:13:30.781500+0000","last_became_peered":"2023-10-09T06:13:30.781500+0000","last_unstale":"2023-10-09T06:19:48.473146+0000","last_undegraded":"2023-10-09T06:19:48.473146+0000","last_fullsized":"2023-10-09T06:19:48.473146+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:43.663265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"633'13681","reported_seq":13750,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:56.068924+0000","last_change":"2023-10-09T06:16:02.755624+0000","last_active":"2023-10-09T06:19:56.068924+0000","last_peered":"2023-10-09T06:19:56.068924+0000","last_clean":"2023-10-09T06:19:56.068924+0000","last_became_active":"2023-10-09T06:16:02.755333+0000","last_became_peered":"2023-10-09T06:16:02.755333+0000","last_unstale":"2023-10-09T06:19:56.068924+0000","last_undegraded":"2023-10-09T06:19:56.068924+0000","last_fullsized":"2023-10-09T06:19:56.068924+0000","mapping_epoch":572,"log_start":"601'7761","ondisk_log_start":"601'7761","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5920,"log_dups_size":0,"ondisk_log_size":5920,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:59.742519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11468800,"num_objects":175,"num_object_clones":0,"num_object_copies":525,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":175,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13681,"num_write_kb":443392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":982,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473253+0000","last_change":"2023-10-09T06:18:32.879060+0000","last_active":"2023-10-09T06:19:48.473253+0000","last_peered":"2023-10-09T06:19:48.473253+0000","last_clean":"2023-10-09T06:19:48.473253+0000","last_became_active":"2023-10-09T06:18:32.878836+0000","last_became_peered":"2023-10-09T06:18:32.878836+0000","last_unstale":"2023-10-09T06:19:48.473253+0000","last_undegraded":"2023-10-09T06:19:48.473253+0000","last_fullsized":"2023-10-09T06:19:48.473253+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:06:22.900486+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":146,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.520568+0000","last_change":"2023-10-09T06:18:12.818437+0000","last_active":"2023-10-09T06:19:48.520568+0000","last_peered":"2023-10-09T06:19:48.520568+0000","last_clean":"2023-10-09T06:19:48.520568+0000","last_became_active":"2023-10-09T06:18:12.818126+0000","last_became_peered":"2023-10-09T06:18:12.818126+0000","last_unstale":"2023-10-09T06:19:48.520568+0000","last_undegraded":"2023-10-09T06:19:48.520568+0000","last_fullsized":"2023-10-09T06:19:48.520568+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:00:02.738584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":681,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.531643+0000","last_change":"2023-10-09T06:09:27.578756+0000","last_active":"2023-10-09T06:19:48.531643+0000","last_peered":"2023-10-09T06:19:48.531643+0000","last_clean":"2023-10-09T06:19:48.531643+0000","last_became_active":"2023-10-09T06:09:27.578384+0000","last_became_peered":"2023-10-09T06:09:27.578384+0000","last_unstale":"2023-10-09T06:19:48.531643+0000","last_undegraded":"2023-10-09T06:19:48.531643+0000","last_fullsized":"2023-10-09T06:19:48.531643+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:22.182630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":620,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517734+0000","last_change":"2023-10-09T06:18:33.812523+0000","last_active":"2023-10-09T06:19:48.517734+0000","last_peered":"2023-10-09T06:19:48.517734+0000","last_clean":"2023-10-09T06:19:48.517734+0000","last_became_active":"2023-10-09T06:18:33.812188+0000","last_became_peered":"2023-10-09T06:18:33.812188+0000","last_unstale":"2023-10-09T06:19:48.517734+0000","last_undegraded":"2023-10-09T06:19:48.517734+0000","last_fullsized":"2023-10-09T06:19:48.517734+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:35.343481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":915,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473118+0000","last_change":"2023-10-09T06:18:15.860341+0000","last_active":"2023-10-09T06:19:48.473118+0000","last_peered":"2023-10-09T06:19:48.473118+0000","last_clean":"2023-10-09T06:19:48.473118+0000","last_became_active":"2023-10-09T06:18:15.860008+0000","last_became_peered":"2023-10-09T06:18:15.860008+0000","last_unstale":"2023-10-09T06:19:48.473118+0000","last_undegraded":"2023-10-09T06:19:48.473118+0000","last_fullsized":"2023-10-09T06:19:48.473118+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.181811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.181811+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:37.378828+0000","scrub_duration":0.011553313000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":564,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.531555+0000","last_change":"2023-10-09T06:13:30.791343+0000","last_active":"2023-10-09T06:19:48.531555+0000","last_peered":"2023-10-09T06:19:48.531555+0000","last_clean":"2023-10-09T06:19:48.531555+0000","last_became_active":"2023-10-09T06:13:30.790945+0000","last_became_peered":"2023-10-09T06:13:30.790945+0000","last_unstale":"2023-10-09T06:19:48.531555+0000","last_undegraded":"2023-10-09T06:19:48.531555+0000","last_fullsized":"2023-10-09T06:19:48.531555+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:14.208391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":346,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.471593+0000","last_change":"2023-10-09T06:15:16.803457+0000","last_active":"2023-10-09T06:19:48.471593+0000","last_peered":"2023-10-09T06:19:48.471593+0000","last_clean":"2023-10-09T06:19:48.471593+0000","last_became_active":"2023-10-09T06:15:16.803097+0000","last_became_peered":"2023-10-09T06:15:16.803097+0000","last_unstale":"2023-10-09T06:19:48.471593+0000","last_undegraded":"2023-10-09T06:19:48.471593+0000","last_fullsized":"2023-10-09T06:19:48.471593+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:45.511763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":1019,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473742+0000","last_change":"2023-10-09T06:18:15.867807+0000","last_active":"2023-10-09T06:19:48.473742+0000","last_peered":"2023-10-09T06:19:48.473742+0000","last_clean":"2023-10-09T06:19:48.473742+0000","last_became_active":"2023-10-09T06:18:15.867453+0000","last_became_peered":"2023-10-09T06:18:15.867453+0000","last_unstale":"2023-10-09T06:19:48.473742+0000","last_undegraded":"2023-10-09T06:19:48.473742+0000","last_fullsized":"2023-10-09T06:19:48.473742+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:04.215270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:04.215270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:50:08.865357+0000","scrub_duration":0.002850652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":421,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.518056+0000","last_change":"2023-10-09T06:18:32.874770+0000","last_active":"2023-10-09T06:19:48.518056+0000","last_peered":"2023-10-09T06:19:48.518056+0000","last_clean":"2023-10-09T06:19:48.518056+0000","last_became_active":"2023-10-09T06:18:32.874452+0000","last_became_peered":"2023-10-09T06:18:32.874452+0000","last_unstale":"2023-10-09T06:19:48.518056+0000","last_undegraded":"2023-10-09T06:19:48.518056+0000","last_fullsized":"2023-10-09T06:19:48.518056+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:30.049140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":651,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517931+0000","last_change":"2023-10-09T06:01:54.764989+0000","last_active":"2023-10-09T06:19:48.517931+0000","last_peered":"2023-10-09T06:19:48.517931+0000","last_clean":"2023-10-09T06:19:48.517931+0000","last_became_active":"2023-10-09T06:01:54.764790+0000","last_became_peered":"2023-10-09T06:01:54.764790+0000","last_unstale":"2023-10-09T06:19:48.517931+0000","last_undegraded":"2023-10-09T06:19:48.517931+0000","last_fullsized":"2023-10-09T06:19:48.517931+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:14:07.208858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":467,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.518499+0000","last_change":"2023-10-09T06:18:33.816108+0000","last_active":"2023-10-09T06:19:48.518499+0000","last_peered":"2023-10-09T06:19:48.518499+0000","last_clean":"2023-10-09T06:19:48.518499+0000","last_became_active":"2023-10-09T06:18:33.815292+0000","last_became_peered":"2023-10-09T06:18:33.815292+0000","last_unstale":"2023-10-09T06:19:48.518499+0000","last_undegraded":"2023-10-09T06:19:48.518499+0000","last_fullsized":"2023-10-09T06:19:48.518499+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:53.643181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":648,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517771+0000","last_change":"2023-10-09T06:09:27.565198+0000","last_active":"2023-10-09T06:19:48.517771+0000","last_peered":"2023-10-09T06:19:48.517771+0000","last_clean":"2023-10-09T06:19:48.517771+0000","last_became_active":"2023-10-09T06:09:27.564909+0000","last_became_peered":"2023-10-09T06:09:27.564909+0000","last_unstale":"2023-10-09T06:19:48.517771+0000","last_undegraded":"2023-10-09T06:19:48.517771+0000","last_fullsized":"2023-10-09T06:19:48.517771+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:46.101014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":845,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.520847+0000","last_change":"2023-10-09T06:18:15.861806+0000","last_active":"2023-10-09T06:19:48.520847+0000","last_peered":"2023-10-09T06:19:48.520847+0000","last_clean":"2023-10-09T06:19:48.520847+0000","last_became_active":"2023-10-09T06:18:15.861466+0000","last_became_peered":"2023-10-09T06:18:15.861466+0000","last_unstale":"2023-10-09T06:19:48.520847+0000","last_undegraded":"2023-10-09T06:19:48.520847+0000","last_fullsized":"2023-10-09T06:19:48.520847+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:00.456132+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:00.456132+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:05.355171+0000","scrub_duration":0.0014992199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":469,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473944+0000","last_change":"2023-10-09T06:18:12.816330+0000","last_active":"2023-10-09T06:19:48.473944+0000","last_peered":"2023-10-09T06:19:48.473944+0000","last_clean":"2023-10-09T06:19:48.473944+0000","last_became_active":"2023-10-09T06:18:12.816040+0000","last_became_peered":"2023-10-09T06:18:12.816040+0000","last_unstale":"2023-10-09T06:19:48.473944+0000","last_undegraded":"2023-10-09T06:19:48.473944+0000","last_fullsized":"2023-10-09T06:19:48.473944+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:42.012205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":461,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473410+0000","last_change":"2023-10-09T06:18:32.816250+0000","last_active":"2023-10-09T06:19:48.473410+0000","last_peered":"2023-10-09T06:19:48.473410+0000","last_clean":"2023-10-09T06:19:48.473410+0000","last_became_active":"2023-10-09T06:18:32.815934+0000","last_became_peered":"2023-10-09T06:18:32.815934+0000","last_unstale":"2023-10-09T06:19:48.473410+0000","last_undegraded":"2023-10-09T06:19:48.473410+0000","last_fullsized":"2023-10-09T06:19:48.473410+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:38.533197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":959,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.471530+0000","last_change":"2023-10-09T06:12:55.753877+0000","last_active":"2023-10-09T06:19:48.471530+0000","last_peered":"2023-10-09T06:19:48.471530+0000","last_clean":"2023-10-09T06:19:48.471530+0000","last_became_active":"2023-10-09T06:12:55.753512+0000","last_became_peered":"2023-10-09T06:12:55.753512+0000","last_unstale":"2023-10-09T06:19:48.471530+0000","last_undegraded":"2023-10-09T06:19:48.471530+0000","last_fullsized":"2023-10-09T06:19:48.471530+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:46.290510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:46.290510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:30.368031+0000","scrub_duration":0.002627819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":690,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.579784+0000","last_change":"2023-10-09T06:18:33.821539+0000","last_active":"2023-10-09T06:19:51.579784+0000","last_peered":"2023-10-09T06:19:51.579784+0000","last_clean":"2023-10-09T06:19:51.579784+0000","last_became_active":"2023-10-09T06:18:33.821093+0000","last_became_peered":"2023-10-09T06:18:33.821093+0000","last_unstale":"2023-10-09T06:19:51.579784+0000","last_undegraded":"2023-10-09T06:19:51.579784+0000","last_fullsized":"2023-10-09T06:19:51.579784+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:25.387520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":684,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517488+0000","last_change":"2023-10-09T06:18:12.815933+0000","last_active":"2023-10-09T06:19:48.517488+0000","last_peered":"2023-10-09T06:19:48.517488+0000","last_clean":"2023-10-09T06:19:48.517488+0000","last_became_active":"2023-10-09T06:18:12.815693+0000","last_became_peered":"2023-10-09T06:18:12.815693+0000","last_unstale":"2023-10-09T06:19:48.517488+0000","last_undegraded":"2023-10-09T06:19:48.517488+0000","last_fullsized":"2023-10-09T06:19:48.517488+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:45.158405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":211288088,"num_objects":3225,"num_object_clones":0,"num_object_copies":9675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3225,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":214599,"num_write_kb":6970305,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16652,"num_bytes_recovered":587333632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":858140672,"data_stored":858128456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90153,"ondisk_log_size":90153,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":89,"num_bytes_recovered":23086,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13410,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234483,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1027200,"kb_used_data":142776,"kb_used_omap":0,"kb_used_meta":884416,"kb_avail":92717952,"statfs":{"total":95995035648,"available":94943182848,"internally_reserved":0,"allocated":146202624,"data_stored":144499137,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":905641984},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:19:41 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":4.7359999999999998,"15min":3.8279999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.19500000000000001},"max":{"1min":5.0119999999999996,"5min":166.94,"15min":166.94},"last":5.4480000000000004},{"interface":"front","average":{"1min":1.8540000000000001,"5min":5.1020000000000003,"15min":3.9900000000000002},"min":{"1min":0.26900000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.2549999999999999,"5min":166.923,"15min":166.923},"last":5.4669999999999996}]},{"osd":1,"last update":"Mon Oct 9 06:19:14 2023","interfaces":[{"interface":"back","average":{"1min":10.066000000000001,"5min":7.399,"15min":5.1390000000000002},"min":{"1min":0.25600000000000001,"5min":0.252,"15min":0.218},"max":{"1min":111.414,"5min":111.414,"15min":124.66200000000001},"last":5.5529999999999999},{"interface":"front","average":{"1min":10.144,"5min":7.1369999999999996,"15min":5.0919999999999996},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.22700000000000001},"max":{"1min":111.767,"5min":111.767,"15min":125.023},"last":2.0510000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:19:41 2023","interfaces":[{"interface":"back","average":{"1min":1.5169999999999999,"5min":1.573,"15min":1.901},"min":{"1min":0.315,"5min":0.29099999999999998,"15min":0.24299999999999999},"max":{"1min":4.9429999999999996,"5min":9.3900000000000006,"15min":47.503},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.8720000000000001,"5min":1.573,"15min":1.927},"min":{"1min":0.35099999999999998,"5min":0.29899999999999999,"15min":0.27000000000000002},"max":{"1min":9.5600000000000005,"5min":9.5600000000000005,"15min":47.545000000000002},"last":0.37}]},{"osd":3,"last update":"Mon Oct 9 06:19:14 2023","interfaces":[{"interface":"back","average":{"1min":12.624000000000001,"5min":5.7549999999999999,"15min":5.4569999999999999},"min":{"1min":0.372,"5min":0.33800000000000002,"15min":0.29899999999999999},"max":{"1min":184.804,"5min":184.804,"15min":184.804},"last":4.9080000000000004},{"interface":"front","average":{"1min":12.515000000000001,"5min":5.1870000000000003,"15min":5.4029999999999996},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32100000000000001},"max":{"1min":186.86099999999999,"5min":186.86099999999999,"15min":186.86099999999999},"last":3.7669999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:18:52 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":1.268,"15min":1.0980000000000001},"min":{"1min":0.27600000000000002,"5min":0.26400000000000001,"15min":0.23100000000000001},"max":{"1min":8.4860000000000007,"5min":10.257,"15min":14.371},"last":0.32200000000000001},{"interface":"front","average":{"1min":1.2310000000000001,"5min":1.569,"15min":1.175},"min":{"1min":0.254,"5min":0.246,"15min":0.246},"max":{"1min":9.6210000000000004,"5min":19.117000000000001,"15min":19.117000000000001},"last":0.307}]},{"osd":5,"last update":"Mon Oct 9 06:19:33 2023","interfaces":[{"interface":"back","average":{"1min":1.893,"5min":1.893,"15min":1.893},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":12.951000000000001,"5min":12.951000000000001,"15min":12.951000000000001},"last":0.34100000000000003},{"interface":"front","average":{"1min":1.4790000000000001,"5min":1.4790000000000001,"15min":1.4790000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.0369999999999999,"5min":5.0369999999999999,"15min":5.0369999999999999},"last":1.226}]},{"osd":6,"last update":"Mon Oct 9 06:19:01 2023","interfaces":[{"interface":"back","average":{"1min":1.6779999999999999,"5min":3.573,"15min":1.8979999999999999},"min":{"1min":0.311,"5min":0.308,"15min":0.26600000000000001},"max":{"1min":9.2599999999999998,"5min":233.56700000000001,"15min":233.56700000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":1.103,"5min":3.1739999999999999,"15min":1.7669999999999999},"min":{"1min":0.33000000000000002,"5min":0.32200000000000001,"15min":0.30599999999999999},"max":{"1min":4.4889999999999999,"5min":232.477,"15min":232.477},"last":0.49199999999999999}]}]},{"osd":6,"up_from":46,"seq":197568496120,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1339832,"kb_used_data":158000,"kb_used_omap":0,"kb_used_meta":1181824,"kb_avail":92405320,"statfs":{"total":95995035648,"available":94623047680,"internally_reserved":0,"allocated":161792000,"data_stored":160096362,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1210187776},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:19:34 2023","interfaces":[{"interface":"back","average":{"1min":5.6710000000000003,"5min":2.907,"15min":4.6849999999999996},"min":{"1min":0.222,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":85.311000000000007,"5min":85.311000000000007,"15min":191.50800000000001},"last":2.2069999999999999},{"interface":"front","average":{"1min":5.9569999999999999,"5min":3.4740000000000002,"15min":4.8440000000000003},"min":{"1min":0.27100000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":85.322999999999993,"5min":85.322999999999993,"15min":191.52600000000001},"last":3.024}]},{"osd":1,"last update":"Mon Oct 9 06:19:34 2023","interfaces":[{"interface":"back","average":{"1min":3.4089999999999998,"5min":5.6749999999999998,"15min":7.1420000000000003},"min":{"1min":0.378,"5min":0.23799999999999999,"15min":0.23200000000000001},"max":{"1min":15.815,"5min":183.458,"15min":328.04399999999998},"last":6.5090000000000003},{"interface":"front","average":{"1min":2.6930000000000001,"5min":4.8490000000000002,"15min":6.8449999999999998},"min":{"1min":0.30499999999999999,"5min":0.253,"15min":0.253},"max":{"1min":16.132999999999999,"5min":183.51499999999999,"15min":328.10000000000002},"last":2.6030000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:19:34 2023","interfaces":[{"interface":"back","average":{"1min":1.6359999999999999,"5min":1.399,"15min":2.758},"min":{"1min":0.35099999999999998,"5min":0.28100000000000003,"15min":0.26100000000000001},"max":{"1min":5.4950000000000001,"5min":9.1500000000000004,"15min":163.04900000000001},"last":2.9529999999999998},{"interface":"front","average":{"1min":1.885,"5min":1.361,"15min":2.7210000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":12.083,"5min":12.083,"15min":162.93000000000001},"last":0.44600000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:19:23 2023","interfaces":[{"interface":"back","average":{"1min":1.5860000000000001,"5min":5.8730000000000002,"15min":7.3769999999999998},"min":{"1min":0.314,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":5.5209999999999999,"5min":208.494,"15min":250.744},"last":3.9769999999999999},{"interface":"front","average":{"1min":2.2120000000000002,"5min":6.1639999999999997,"15min":7.2430000000000003},"min":{"1min":0.39100000000000001,"5min":0.309,"15min":0.309},"max":{"1min":12.536,"5min":211.661,"15min":250.83799999999999},"last":2.153}]},{"osd":4,"last update":"Mon Oct 9 06:18:56 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.754,"15min":1.548},"min":{"1min":0.29899999999999999,"5min":0.251,"15min":0.24399999999999999},"max":{"1min":1.621,"5min":6.0979999999999999,"15min":151.35599999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":1.2290000000000001,"5min":1.3720000000000001,"15min":1.8540000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.23999999999999999},"max":{"1min":5.5270000000000001,"5min":9.2710000000000008,"15min":149.15799999999999},"last":2.9929999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:19:34 2023","interfaces":[{"interface":"back","average":{"1min":1.794,"5min":1.794,"15min":1.794},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":16.032,"5min":16.032,"15min":16.032},"last":2.5819999999999999},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.1040000000000001,"15min":1.1040000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":5.8090000000000002,"5min":5.8090000000000002,"15min":5.8090000000000002},"last":2.1829999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:18:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.3260000000000001,"15min":1.2749999999999999},"min":{"1min":0.35399999999999998,"5min":0.32400000000000001,"15min":0.30299999999999999},"max":{"1min":5.4299999999999997,"5min":9.2989999999999995,"15min":21.98},"last":3.052},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.153,"15min":1.2030000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.33300000000000002},"max":{"1min":4.4039999999999999,"5min":7.1390000000000002,"15min":18.241},"last":2.6200000000000001}]}]},{"osd":1,"up_from":89,"seq":382252089708,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":994992,"kb_used_data":154088,"kb_used_omap":9,"kb_used_meta":840886,"kb_avail":92750160,"statfs":{"total":95995035648,"available":94976163840,"internally_reserved":0,"allocated":157786112,"data_stored":154223014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":861068265},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:18:52 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":3.9199999999999999,"15min":2.714},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.20999999999999999},"max":{"1min":9.6899999999999995,"5min":118.17,"15min":118.17},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.90900000000000003,"5min":3.9809999999999999,"15min":2.9039999999999999},"min":{"1min":0.23200000000000001,"5min":0.20699999999999999,"15min":0.20200000000000001},"max":{"1min":2.968,"5min":117.634,"15min":117.634},"last":1.9350000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:18:52 2023","interfaces":[{"interface":"back","average":{"1min":1.6120000000000001,"5min":1.45,"15min":1.778},"min":{"1min":0.28299999999999997,"5min":0.245,"15min":0.21099999999999999},"max":{"1min":9.7200000000000006,"5min":9.7200000000000006,"15min":17.681000000000001},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.90100000000000002,"5min":1.8759999999999999,"15min":2.044},"min":{"1min":0.27300000000000002,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":3.0710000000000002,"5min":55.694000000000003,"15min":55.694000000000003},"last":1.4399999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:19:23 2023","interfaces":[{"interface":"back","average":{"1min":5.9889999999999999,"5min":5.8380000000000001,"15min":4.9489999999999998},"min":{"1min":0.317,"5min":0.30399999999999999,"15min":0.24199999999999999},"max":{"1min":72.266000000000005,"5min":95.489999999999995,"15min":142.97},"last":1.3899999999999999},{"interface":"front","average":{"1min":5.9560000000000004,"5min":5.4770000000000003,"15min":4.8239999999999998},"min":{"1min":0.28199999999999997,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":72.311000000000007,"5min":95.436999999999998,"15min":142.94800000000001},"last":1.3480000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:18:52 2023","interfaces":[{"interface":"back","average":{"1min":8.9600000000000009,"5min":4.1580000000000004,"15min":3.157},"min":{"1min":0.38200000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":147.822,"5min":147.822,"15min":147.822},"last":0.79200000000000004},{"interface":"front","average":{"1min":9.6699999999999999,"5min":4.5279999999999996,"15min":3.6240000000000001},"min":{"1min":0.41399999999999998,"5min":0.34399999999999997,"15min":0.308},"max":{"1min":147.96199999999999,"5min":147.96199999999999,"15min":147.96199999999999},"last":4.6200000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:19:34 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":1.48,"15min":1.48},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":4.819,"5min":4.819,"15min":4.819},"last":4.5679999999999996},{"interface":"front","average":{"1min":1.3220000000000001,"5min":1.3220000000000001,"15min":1.3220000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.9769999999999999,"5min":3.9769999999999999,"15min":3.9769999999999999},"last":0.57599999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:19:06 2023","interfaces":[{"interface":"back","average":{"1min":6.2679999999999998,"5min":2.4039999999999999,"15min":2.1989999999999998},"min":{"1min":0.42899999999999999,"5min":0.34599999999999997,"15min":0.33700000000000002},"max":{"1min":98.433000000000007,"5min":98.433000000000007,"15min":98.433000000000007},"last":0.52900000000000003},{"interface":"front","average":{"1min":6.4809999999999999,"5min":2.6269999999999998,"15min":2.677},"min":{"1min":0.377,"5min":0.36199999999999999,"15min":0.34999999999999998},"max":{"1min":98.456000000000003,"5min":98.456000000000003,"15min":98.456000000000003},"last":1.004}]},{"osd":7,"last update":"Mon Oct 9 06:19:06 2023","interfaces":[{"interface":"back","average":{"1min":15.449,"5min":4.5380000000000003,"15min":3.5910000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":154.34800000000001,"5min":154.34800000000001,"15min":154.34800000000001},"last":0.95899999999999996},{"interface":"front","average":{"1min":15.24,"5min":4.2830000000000004,"15min":3.0539999999999998},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.38},"max":{"1min":152.20599999999999,"5min":152.20599999999999,"15min":152.20599999999999},"last":0.73599999999999999}]}]},{"osd":0,"up_from":456,"seq":1958505087103,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":566664,"kb_used_data":142144,"kb_used_omap":32,"kb_used_meta":424479,"kb_avail":93178488,"statfs":{"total":95995035648,"available":95414771712,"internally_reserved":0,"allocated":145555456,"data_stored":142211610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32794,"internal_metadata":434667494},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:19:49 2023","interfaces":[{"interface":"back","average":{"1min":4.9800000000000004,"5min":4.8120000000000003,"15min":7.5129999999999999},"min":{"1min":0.28399999999999997,"5min":0.189,"15min":0.17299999999999999},"max":{"1min":43.316000000000003,"5min":137.452,"15min":166.833},"last":0.94699999999999995},{"interface":"front","average":{"1min":4.5679999999999996,"5min":4.7140000000000004,"15min":7.2300000000000004},"min":{"1min":0.254,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":43.401000000000003,"5min":137.10499999999999,"15min":166.81100000000001},"last":0.96199999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:19:49 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":1.6950000000000001,"15min":2.8010000000000002},"min":{"1min":0.246,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":13.071,"5min":13.071,"15min":21.648},"last":0.23300000000000001},{"interface":"front","average":{"1min":1.732,"5min":1.5009999999999999,"15min":2.5750000000000002},"min":{"1min":0.247,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.173,"5min":10.154999999999999,"15min":20.152000000000001},"last":0.42799999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:19:46 2023","interfaces":[{"interface":"back","average":{"1min":16.856999999999999,"5min":5.1980000000000004,"15min":8.7080000000000002},"min":{"1min":0.41899999999999998,"5min":0.214,"15min":0.214},"max":{"1min":138.52699999999999,"5min":138.52699999999999,"15min":203.59299999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":16.591000000000001,"5min":5.3789999999999996,"15min":8.5670000000000002},"min":{"1min":0.39300000000000002,"5min":0.29499999999999998,"15min":0.23899999999999999},"max":{"1min":138.48500000000001,"5min":138.48500000000001,"15min":202.96600000000001},"last":0.68300000000000005}]},{"osd":4,"last update":"Mon Oct 9 06:19:49 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":1.9410000000000001,"15min":2.2400000000000002},"min":{"1min":0.38800000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.5840000000000001,"5min":60.587000000000003,"15min":60.587000000000003},"last":1.7869999999999999},{"interface":"front","average":{"1min":1.4410000000000001,"5min":2.0569999999999999,"15min":2.278},"min":{"1min":0.32400000000000001,"5min":0.254,"15min":0.254},"max":{"1min":4.234,"5min":60.567,"15min":60.567},"last":1.8080000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:19:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":2.1019999999999999,"15min":2.1019999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":8.048,"5min":8.048,"15min":8.048},"last":0.91100000000000003},{"interface":"front","average":{"1min":1.857,"5min":1.857,"15min":1.857},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":13.007,"5min":13.007,"15min":13.007},"last":8.8510000000000009}]},{"osd":6,"last update":"Mon Oct 9 06:19:49 2023","interfaces":[{"interface":"back","average":{"1min":2.3719999999999999,"5min":1.827,"15min":2.286},"min":{"1min":0.39200000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":13.029999999999999,"5min":13.029999999999999,"15min":31.648},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.704,"5min":1.3779999999999999,"15min":2.0990000000000002},"min":{"1min":0.47699999999999998,"5min":0.36399999999999999,"15min":0.36299999999999999},"max":{"1min":7.5640000000000001,"5min":10.051,"15min":24.43},"last":5.2679999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:19:49 2023","interfaces":[{"interface":"back","average":{"1min":2.3759999999999999,"5min":5.7110000000000003,"15min":3.9380000000000002},"min":{"1min":0.40899999999999997,"5min":0.376,"15min":0.376},"max":{"1min":12.957000000000001,"5min":248.99100000000001,"15min":248.99100000000001},"last":2.1589999999999998},{"interface":"front","average":{"1min":2.0419999999999998,"5min":5.5190000000000001,"15min":3.8079999999999998},"min":{"1min":0.505,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.5279999999999996,"5min":249.00800000000001,"15min":249.00800000000001},"last":2.1890000000000001}]}]},{"osd":2,"up_from":0,"seq":403726926200,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":99,"seq":425201762673,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1360228,"kb_used_data":239708,"kb_used_omap":7,"kb_used_meta":1120504,"kb_avail":92384924,"statfs":{"total":95995035648,"available":94602162176,"internally_reserved":0,"allocated":245460992,"data_stored":241905879,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1147396712},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:19:40 2023","interfaces":[{"interface":"back","average":{"1min":2.5510000000000002,"5min":3.3460000000000001,"15min":3.7149999999999999},"min":{"1min":0.23100000000000001,"5min":0.182,"15min":0.17299999999999999},"max":{"1min":11.308999999999999,"5min":117.473,"15min":220.45400000000001},"last":5.4800000000000004},{"interface":"front","average":{"1min":1.2769999999999999,"5min":3.7599999999999998,"15min":4.5949999999999998},"min":{"1min":0.219,"5min":0.219,"15min":0.19600000000000001},"max":{"1min":4.944,"5min":118.873,"15min":216.44200000000001},"last":0.42499999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:19:18 2023","interfaces":[{"interface":"back","average":{"1min":1.982,"5min":3.6629999999999998,"15min":3.516},"min":{"1min":0.29899999999999999,"5min":0.20300000000000001,"15min":0.19800000000000001},"max":{"1min":6.9950000000000001,"5min":167.768,"15min":207.75299999999999},"last":5.6070000000000002},{"interface":"front","average":{"1min":1.4039999999999999,"5min":4.2069999999999999,"15min":4.29},"min":{"1min":0.28499999999999998,"5min":0.218,"15min":0.218},"max":{"1min":8.9450000000000003,"5min":172.143,"15min":207.12200000000001},"last":0.73199999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:19:18 2023","interfaces":[{"interface":"back","average":{"1min":1.746,"5min":1.5349999999999999,"15min":1.9279999999999999},"min":{"1min":0.35599999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.5940000000000003,"5min":6.5940000000000003,"15min":30.547999999999998},"last":0.46400000000000002},{"interface":"front","average":{"1min":2.3079999999999998,"5min":2.2120000000000002,"15min":2.589},"min":{"1min":0.377,"5min":0.29699999999999999,"15min":0.23899999999999999},"max":{"1min":8.9949999999999992,"5min":11.542,"15min":25.984000000000002},"last":0.66800000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:19:02 2023","interfaces":[{"interface":"back","average":{"1min":7.7850000000000001,"5min":3.1899999999999999,"15min":2.6579999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":84.369,"5min":84.369,"15min":95.739000000000004},"last":0.501},{"interface":"front","average":{"1min":8.173,"5min":3.5950000000000002,"15min":3.0840000000000001},"min":{"1min":0.34899999999999998,"5min":0.32600000000000001,"15min":0.28899999999999998},"max":{"1min":87.176000000000002,"5min":87.176000000000002,"15min":97.941999999999993},"last":0.628}]},{"osd":5,"last update":"Mon Oct 9 06:19:32 2023","interfaces":[{"interface":"back","average":{"1min":1.8100000000000001,"5min":1.8100000000000001,"15min":1.8100000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":7.8550000000000004,"5min":7.8550000000000004,"15min":7.8550000000000004},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.911,"5min":1.911,"15min":1.911},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":6.4829999999999997,"5min":6.4829999999999997,"15min":6.4829999999999997},"last":0.55500000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:19:18 2023","interfaces":[{"interface":"back","average":{"1min":1.897,"5min":2.238,"15min":3},"min":{"1min":0.42699999999999999,"5min":0.32600000000000001,"15min":0.26100000000000001},"max":{"1min":5.6619999999999999,"5min":49.526000000000003,"15min":233.26599999999999},"last":5.5759999999999996},{"interface":"front","average":{"1min":2.573,"5min":2.4969999999999999,"15min":3.4510000000000001},"min":{"1min":0.44600000000000001,"5min":0.40000000000000002,"15min":0.33600000000000002},"max":{"1min":8.5150000000000006,"5min":49.429000000000002,"15min":233.24100000000001},"last":0.68600000000000005}]},{"osd":7,"last update":"Mon Oct 9 06:19:18 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":2.726,"15min":3.899},"min":{"1min":0.48599999999999999,"5min":0.36599999999999999,"15min":0.35699999999999998},"max":{"1min":6.8449999999999998,"5min":77.525000000000006,"15min":230.69399999999999},"last":0.58799999999999997},{"interface":"front","average":{"1min":1.7290000000000001,"5min":3.407,"15min":4.3419999999999996},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.32600000000000001},"max":{"1min":5.0810000000000004,"5min":77.438000000000002,"15min":230.67500000000001},"last":1.016}]}]},{"osd":4,"up_from":322,"seq":1382979469536,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":995392,"kb_used_data":68344,"kb_used_omap":35,"kb_used_meta":927004,"kb_avail":92749760,"statfs":{"total":95995035648,"available":94975754240,"internally_reserved":0,"allocated":69984256,"data_stored":68280502,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36552,"internal_metadata":949252408},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:19:38 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":2.234,"15min":1.8979999999999999},"min":{"1min":0.23100000000000001,"5min":0.22700000000000001,"15min":0.216},"max":{"1min":10.574,"5min":81.066000000000003,"15min":81.066000000000003},"last":0.88},{"interface":"front","average":{"1min":2.5169999999999999,"5min":2.5529999999999999,"15min":1.97},"min":{"1min":0.313,"5min":0.27800000000000002,"15min":0.22800000000000001},"max":{"1min":10.695,"5min":81.603999999999999,"15min":81.603999999999999},"last":5.3700000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:18:58 2023","interfaces":[{"interface":"back","average":{"1min":10.872,"5min":5.734,"15min":3.6320000000000001},"min":{"1min":0.29999999999999999,"5min":0.24099999999999999,"15min":0.20999999999999999},"max":{"1min":160.35400000000001,"5min":160.35400000000001,"15min":160.35400000000001},"last":0.75800000000000001},{"interface":"front","average":{"1min":11.196,"5min":5.9329999999999998,"15min":3.7440000000000002},"min":{"1min":0.26700000000000002,"5min":0.26500000000000001,"15min":0.22800000000000001},"max":{"1min":160.333,"5min":160.333,"15min":160.333},"last":8.4440000000000008}]},{"osd":2,"last update":"Mon Oct 9 06:18:58 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":1.726,"15min":2.6299999999999999},"min":{"1min":0.316,"5min":0.311,"15min":0.27200000000000002},"max":{"1min":7.5570000000000004,"5min":10.084,"15min":245.166},"last":0.42799999999999999},{"interface":"front","average":{"1min":4.3780000000000001,"5min":2.3599999999999999,"15min":2.6909999999999998},"min":{"1min":0.33600000000000002,"5min":0.28699999999999998,"15min":0.27900000000000003},"max":{"1min":52.920999999999999,"5min":52.920999999999999,"15min":245.119},"last":5.4500000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:18:58 2023","interfaces":[{"interface":"back","average":{"1min":12.802,"5min":7.9720000000000004,"15min":5.8979999999999997},"min":{"1min":0.33100000000000002,"5min":0.29299999999999998,"15min":0.26800000000000002},"max":{"1min":164.89699999999999,"5min":197.31899999999999,"15min":246.39400000000001},"last":0.53900000000000003},{"interface":"front","average":{"1min":12.507999999999999,"5min":7.4710000000000001,"15min":5.3840000000000003},"min":{"1min":0.47099999999999997,"5min":0.32300000000000001,"15min":0.314},"max":{"1min":164.88300000000001,"5min":198.565,"15min":246.99000000000001},"last":3.8650000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:19:33 2023","interfaces":[{"interface":"back","average":{"1min":2.04,"5min":2.04,"15min":2.04},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":10.659000000000001,"5min":10.659000000000001,"15min":10.659000000000001},"last":5.4809999999999999},{"interface":"front","average":{"1min":1.96,"5min":1.96,"15min":1.96},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":11.779,"5min":11.779,"15min":11.779},"last":0.68000000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:18:58 2023","interfaces":[{"interface":"back","average":{"1min":2.3279999999999998,"5min":1.347,"15min":1.569},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26400000000000001},"max":{"1min":23.712,"5min":23.712,"15min":95.492999999999995},"last":5.4100000000000001},{"interface":"front","average":{"1min":2.379,"5min":1.546,"15min":1.71},"min":{"1min":0.312,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":18.021000000000001,"5min":18.021000000000001,"15min":95.664000000000001},"last":0.51200000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:18:58 2023","interfaces":[{"interface":"back","average":{"1min":4.7690000000000001,"5min":3.1080000000000001,"15min":1.9490000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":61.811999999999998,"5min":95.167000000000002,"15min":95.167000000000002},"last":1.3899999999999999},{"interface":"front","average":{"1min":4.2990000000000004,"5min":2.9710000000000001,"15min":1.841},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.251},"max":{"1min":61.759,"5min":94.811999999999998,"15min":94.811999999999998},"last":0.39400000000000002}]}]},{"osd":5,"up_from":611,"seq":2624225017874,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":417136,"kb_used_data":60008,"kb_used_omap":98,"kb_used_meta":357021,"kb_avail":93328016,"statfs":{"total":95995035648,"available":95567888384,"internally_reserved":0,"allocated":61448192,"data_stored":59757086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":100797,"internal_metadata":365590083},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:19:33 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":1.3129999999999999,"15min":1.3129999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":8.3260000000000005,"5min":8.3260000000000005,"15min":8.3260000000000005},"last":0.29299999999999998},{"interface":"front","average":{"1min":1.681,"5min":1.681,"15min":1.681},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":7.577,"5min":7.577,"15min":7.577},"last":2.8799999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:19:33 2023","interfaces":[{"interface":"back","average":{"1min":4.9790000000000001,"5min":4.9790000000000001,"15min":4.9790000000000001},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":50.430999999999997,"5min":50.430999999999997,"15min":50.430999999999997},"last":3.1269999999999998},{"interface":"front","average":{"1min":3.6280000000000001,"5min":3.6280000000000001,"15min":3.6280000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":50.445,"5min":50.445,"15min":50.445},"last":0.27800000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:19:33 2023","interfaces":[{"interface":"back","average":{"1min":1.371,"5min":1.371,"15min":1.371},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.9980000000000002,"5min":5.9980000000000002,"15min":5.9980000000000002},"last":3.7719999999999998},{"interface":"front","average":{"1min":1.4610000000000001,"5min":1.4610000000000001,"15min":1.4610000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":5.9560000000000004,"5min":5.9560000000000004,"15min":5.9560000000000004},"last":3.5529999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:19:33 2023","interfaces":[{"interface":"back","average":{"1min":9.9749999999999996,"5min":9.9749999999999996,"15min":9.9749999999999996},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":120.357,"5min":120.357,"15min":120.357},"last":2.9489999999999998},{"interface":"front","average":{"1min":9.4619999999999997,"5min":9.4619999999999997,"15min":9.4619999999999997},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":120.759,"5min":120.759,"15min":120.759},"last":0.40799999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:19:33 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":1.012,"15min":1.012},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.6890000000000001,"5min":5.6890000000000001,"15min":5.6890000000000001},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.7350000000000001,"5min":1.7350000000000001,"15min":1.7350000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.6130000000000004,"5min":7.6130000000000004,"15min":7.6130000000000004},"last":2.9140000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:19:33 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":5.9400000000000004,"5min":5.9400000000000004,"15min":5.9400000000000004},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.997,"5min":0.997,"15min":0.997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.8120000000000003,"5min":5.8120000000000003,"15min":5.8120000000000003},"last":0.998}]},{"osd":7,"last update":"Mon Oct 9 06:19:33 2023","interfaces":[{"interface":"back","average":{"1min":1.55,"5min":1.55,"15min":1.55},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":7.6120000000000001,"5min":7.6120000000000001,"15min":7.6120000000000001},"last":2.9769999999999999},{"interface":"front","average":{"1min":1.2729999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":5.8920000000000003,"5min":5.8920000000000003,"15min":5.8920000000000003},"last":3.738}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4889,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":119472128,"data_stored":119472128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":142282752,"data_stored":142278680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":223608832,"data_stored":223608832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":56889344,"data_stored":56885272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":44367872,"data_stored":44367872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":136843264,"data_stored":136839192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":134676480,"data_stored":134676480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:19:58.556 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:19:58.611 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:58.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:58.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:58.716 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:58.752 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:58.843 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:58.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:58.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:58.957 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:58.989 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:59.086 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:59.102 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:58 smithi156 ceph-mon[195101]: pgmap v1635: 119 pgs: 119 active+clean; 202 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 863 op/s; 0 B/s, 13 objects/s recovering 2023-10-09T06:19:59.102 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:58 smithi156 ceph-mon[195101]: from='client.46946 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:19:59.102 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:58 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:59.102 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:19:58 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:59.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:59.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:59.118 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Removed 108911 objects 2023-10-09T06:19:59.118 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Clean up completed and total clean up time :139.052 2023-10-09T06:19:59.206 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:59.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:58 smithi064 ceph-mon[212750]: pgmap v1635: 119 pgs: 119 active+clean; 202 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 863 op/s; 0 B/s, 13 objects/s recovering 2023-10-09T06:19:59.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:58 smithi064 ceph-mon[212750]: from='client.46946 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:19:59.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:58 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:59.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:19:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:59.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:58 smithi064 ceph-mon[220044]: pgmap v1635: 119 pgs: 119 active+clean; 202 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 863 op/s; 0 B/s, 13 objects/s recovering 2023-10-09T06:19:59.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:58 smithi064 ceph-mon[220044]: from='client.46946 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:19:59.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:58 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:19:59.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:19:58 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:59.232 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:59.273 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:59.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:59.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:59.379 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:59.405 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:59.447 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:59.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:59.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:59.552 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:59.581 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:00.120 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:00.121 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:00.125 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:00.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:00.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:00.231 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:00.263 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:00.363 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:00.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:00.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:00.466 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:00.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:00 smithi064 ceph-mon[220044]: pgmap v1636: 119 pgs: 119 active+clean; 202 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 766 op/s; 0 B/s, 11 objects/s recovering 2023-10-09T06:20:00.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:00 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:00.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:00 smithi064 ceph-mon[220044]: overall HEALTH_OK 2023-10-09T06:20:00.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:00 smithi064 ceph-mon[212750]: pgmap v1636: 119 pgs: 119 active+clean; 202 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 766 op/s; 0 B/s, 11 objects/s recovering 2023-10-09T06:20:00.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:00 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:00.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:00 smithi064 ceph-mon[212750]: overall HEALTH_OK 2023-10-09T06:20:00.499 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:00.550 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:00.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:00.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:00.659 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:00.697 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:00.844 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:00.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:00.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:00.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:00 smithi156 ceph-mon[195101]: pgmap v1636: 119 pgs: 119 active+clean; 202 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 766 op/s; 0 B/s, 11 objects/s recovering 2023-10-09T06:20:00.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:00 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:00.873 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:00 smithi156 ceph-mon[195101]: overall HEALTH_OK 2023-10-09T06:20:00.949 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:00.984 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:01.034 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:01.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:01.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:01.068 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:01.141 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:01.179 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:01.235 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:01.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:01.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:01.340 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:01.371 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:01.686 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:01.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:01.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:01.790 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:01.821 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:01.942 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-10-09T06:20:01.943 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-10-09T06:20:02.627 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:02.629 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:02 smithi064 ceph-mon[212750]: pgmap v1637: 119 pgs: 119 active+clean; 582 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 746 op/s 2023-10-09T06:20:02.629 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:02 smithi064 ceph-mon[220044]: pgmap v1637: 119 pgs: 119 active+clean; 582 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 746 op/s 2023-10-09T06:20:02.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:02.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:02.730 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:02.760 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:02 smithi156 ceph-mon[195101]: pgmap v1637: 119 pgs: 119 active+clean; 582 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 746 op/s 2023-10-09T06:20:02.763 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:02.812 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:02.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:02.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:02.919 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:02.946 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:02.990 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:02.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:02.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:03.095 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:03.122 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:03.164 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:03.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:03.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:03.269 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:03.302 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:03.857 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:20:04.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:03 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1363935991' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:20:04.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:03 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:20:04.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:03 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1363935991' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:20:04.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:03 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:20:04.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:03 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1363935991' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:20:04.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:03 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:20:04.624 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:04.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:04.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:04.725 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:20:04.725 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T06:20:04.725 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T06:20:04.726 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:20:04.726 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.726 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:81471 10.3 10:cc6caecd:::benchmark_data_smithi064_573750_object81470:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-10-09T06:20:04.726 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:12:59.854905+0000", 2023-10-09T06:20:04.726 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 424.86764240999997, 2023-10-09T06:20:04.726 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.214265604, 2023-10-09T06:20:04.726 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:04.726 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:04.726 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:04.726 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:20:04.726 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:04.727 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 81471 2023-10-09T06:20:04.727 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.727 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:04.727 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.727 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:04.727 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.854905+0000", 2023-10-09T06:20:04.727 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.727 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.727 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.727 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:04.727 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.854905+0000", 2023-10-09T06:20:04.727 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.728 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.728 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.728 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:04.728 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.854903+0000", 2023-10-09T06:20:04.728 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T06:20:04.728 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.728 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.728 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:04.728 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.854933+0000", 2023-10-09T06:20:04.728 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0031000000000001e-05 2023-10-09T06:20:04.728 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.728 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.729 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:04.729 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.854950+0000", 2023-10-09T06:20:04.729 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6651e-05 2023-10-09T06:20:04.729 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.729 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.729 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:04.729 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.854978+0000", 2023-10-09T06:20:04.729 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7837000000000001e-05 2023-10-09T06:20:04.729 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.729 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.729 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:04.729 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.859094+0000", 2023-10-09T06:20:04.730 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041166450000000004 2023-10-09T06:20:04.730 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.730 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.730 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:04.730 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.859292+0000", 2023-10-09T06:20:04.730 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019750099999999999 2023-10-09T06:20:04.730 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.730 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.730 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:20:04.730 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.859461+0000", 2023-10-09T06:20:04.730 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000168717 2023-10-09T06:20:04.731 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.731 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.731 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.731 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.862286+0000", 2023-10-09T06:20:04.731 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028254970000000002 2023-10-09T06:20:04.731 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.731 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.731 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:04.731 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.862633+0000", 2023-10-09T06:20:04.731 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00034674599999999999 2023-10-09T06:20:04.731 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.731 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.732 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.732 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:12:59.864600+0000", 2023-10-09T06:20:04.732 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019670030000000002 2023-10-09T06:20:04.732 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.732 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.732 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.732 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:00.068977+0000", 2023-10-09T06:20:04.732 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.204377264 2023-10-09T06:20:04.732 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.732 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.732 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:04.732 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:00.069046+0000", 2023-10-09T06:20:04.733 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8472000000000004e-05 2023-10-09T06:20:04.733 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.733 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.733 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:04.733 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:00.069171+0000", 2023-10-09T06:20:04.733 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012542099999999999 2023-10-09T06:20:04.733 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.733 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:04.733 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.733 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.733 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.733 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:83019 10.3 10:ce4365a3:::benchmark_data_smithi064_573750_object83018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e514)", 2023-10-09T06:20:04.733 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.816350+0000", 2023-10-09T06:20:04.734 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 422.906198217, 2023-10-09T06:20:04.734 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27937314000000002, 2023-10-09T06:20:04.734 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:04.734 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:04.734 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:04.734 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:20:04.734 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:04.734 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 83019 2023-10-09T06:20:04.734 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.734 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:04.734 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.735 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:04.735 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.816350+0000", 2023-10-09T06:20:04.735 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.735 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.735 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.735 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:04.735 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.816350+0000", 2023-10-09T06:20:04.735 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.735 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.735 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.735 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:04.735 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.816349+0000", 2023-10-09T06:20:04.736 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:04.736 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.736 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.736 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:04.736 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.816373+0000", 2023-10-09T06:20:04.736 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3547999999999999e-05 2023-10-09T06:20:04.736 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.736 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.736 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:04.736 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.816379+0000", 2023-10-09T06:20:04.736 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7440000000000001e-06 2023-10-09T06:20:04.736 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.736 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.737 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:04.737 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.816385+0000", 2023-10-09T06:20:04.737 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2959999999999998e-06 2023-10-09T06:20:04.737 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.737 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.737 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:04.737 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.820672+0000", 2023-10-09T06:20:04.737 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.004287155 2023-10-09T06:20:04.737 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.737 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.737 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:04.737 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.820810+0000", 2023-10-09T06:20:04.738 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000137787 2023-10-09T06:20:04.738 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.738 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.738 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:20:04.738 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.820957+0000", 2023-10-09T06:20:04.738 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014771599999999999 2023-10-09T06:20:04.738 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.738 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.738 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.738 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.827525+0000", 2023-10-09T06:20:04.738 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065673570000000002 2023-10-09T06:20:04.739 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.739 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.739 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.827554+0000", 2023-10-09T06:20:04.739 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9244e-05 2023-10-09T06:20:04.739 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.739 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.739 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832230+0000", 2023-10-09T06:20:04.739 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046755659999999999 2023-10-09T06:20:04.739 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.740 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:04.740 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832551+0000", 2023-10-09T06:20:04.740 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00032125600000000002 2023-10-09T06:20:04.740 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.740 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.740 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:04.740 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832599+0000", 2023-10-09T06:20:04.740 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8233999999999999e-05 2023-10-09T06:20:04.740 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.740 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.740 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:04.740 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095723+0000", 2023-10-09T06:20:04.741 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26312367800000003 2023-10-09T06:20:04.741 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.741 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:04.741 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.741 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.741 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.741 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:83027 10.3 10:ce4b8369:::benchmark_data_smithi064_573750_object83026:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e514)", 2023-10-09T06:20:04.741 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.823657+0000", 2023-10-09T06:20:04.741 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 422.89889047299999, 2023-10-09T06:20:04.741 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27225728399999999, 2023-10-09T06:20:04.741 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:04.741 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:04.742 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:04.742 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:20:04.742 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:04.742 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 83027 2023-10-09T06:20:04.742 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.742 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:04.742 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.742 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:04.742 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.823657+0000", 2023-10-09T06:20:04.742 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.742 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.743 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.743 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:04.743 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.823657+0000", 2023-10-09T06:20:04.743 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.743 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.743 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.743 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:04.743 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.823656+0000", 2023-10-09T06:20:04.743 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:04.743 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.743 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.743 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:04.744 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.823681+0000", 2023-10-09T06:20:04.744 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4947999999999999e-05 2023-10-09T06:20:04.744 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.744 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.744 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:04.744 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.823690+0000", 2023-10-09T06:20:04.744 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0059999999999998e-06 2023-10-09T06:20:04.744 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.744 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.744 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:04.744 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.823696+0000", 2023-10-09T06:20:04.745 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1600000000000003e-06 2023-10-09T06:20:04.745 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.745 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.745 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:04.745 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824488+0000", 2023-10-09T06:20:04.745 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00079232200000000003 2023-10-09T06:20:04.745 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.745 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.745 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:04.745 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824646+0000", 2023-10-09T06:20:04.745 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015738899999999999 2023-10-09T06:20:04.745 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.746 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.746 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:20:04.746 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824798+0000", 2023-10-09T06:20:04.746 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015239999999999999 2023-10-09T06:20:04.746 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.746 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.746 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.746 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832263+0000", 2023-10-09T06:20:04.746 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0074651350000000003 2023-10-09T06:20:04.746 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.746 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.746 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.747 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832298+0000", 2023-10-09T06:20:04.747 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4591999999999997e-05 2023-10-09T06:20:04.747 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.747 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.747 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.747 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832327+0000", 2023-10-09T06:20:04.747 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8731999999999999e-05 2023-10-09T06:20:04.747 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.747 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.747 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:04.747 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095736+0000", 2023-10-09T06:20:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26340953700000003 2023-10-09T06:20:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095794+0000", 2023-10-09T06:20:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7886000000000001e-05 2023-10-09T06:20:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095915+0000", 2023-10-09T06:20:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012050200000000001 2023-10-09T06:20:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:83026 10.7 e514/490 10:e8d46d43:::benchmark_data_smithi064_573750_object83025:head v 514'5204, mlcod=514'5202)", 2023-10-09T06:20:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.824237+0000", 2023-10-09T06:20:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 422.89831098500002, 2023-10-09T06:20:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27149754799999998, 2023-10-09T06:20:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.750 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:04.750 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824237+0000", 2023-10-09T06:20:04.750 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.750 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.750 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.750 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:04.750 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824237+0000", 2023-10-09T06:20:04.750 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.750 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.750 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.750 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:04.751 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824236+0000", 2023-10-09T06:20:04.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:20:04.751 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.751 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.751 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:04.751 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824266+0000", 2023-10-09T06:20:04.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9991000000000001e-05 2023-10-09T06:20:04.751 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.751 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.751 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:04.751 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824275+0000", 2023-10-09T06:20:04.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7860000000000002e-06 2023-10-09T06:20:04.752 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.752 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.752 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:04.752 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.824281+0000", 2023-10-09T06:20:04.752 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9020000000000004e-06 2023-10-09T06:20:04.752 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.752 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.752 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:04.752 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.826984+0000", 2023-10-09T06:20:04.752 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027032750000000002 2023-10-09T06:20:04.752 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.752 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:04.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.827009+0000", 2023-10-09T06:20:04.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4987e-05 2023-10-09T06:20:04.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.753 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:04.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095693+0000", 2023-10-09T06:20:04.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.268683799 2023-10-09T06:20:04.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.753 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:04.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095734+0000", 2023-10-09T06:20:04.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1600000000000002e-05 2023-10-09T06:20:04.754 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.754 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:04.754 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.754 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.754 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:83030 10.7 e514/490 10:e508533f:::benchmark_data_smithi064_573750_object83029:head v 514'5205, mlcod=514'5202)", 2023-10-09T06:20:04.754 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.831823+0000", 2023-10-09T06:20:04.754 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 422.890724557, 2023-10-09T06:20:04.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26393689799999998, 2023-10-09T06:20:04.755 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:04.755 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:04.755 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:04.755 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.755 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:04.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.831823+0000", 2023-10-09T06:20:04.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.755 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.755 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.755 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:04.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.831823+0000", 2023-10-09T06:20:04.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.756 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.756 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.756 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:04.756 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.831822+0000", 2023-10-09T06:20:04.756 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:20:04.756 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.756 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.756 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:04.756 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.831849+0000", 2023-10-09T06:20:04.757 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7291e-05 2023-10-09T06:20:04.757 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.757 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.757 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:04.757 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.831862+0000", 2023-10-09T06:20:04.757 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2138e-05 2023-10-09T06:20:04.757 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.757 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.757 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:04.757 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.831870+0000", 2023-10-09T06:20:04.757 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4020000000000002e-06 2023-10-09T06:20:04.757 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.758 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.758 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:04.758 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.831944+0000", 2023-10-09T06:20:04.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3606e-05 2023-10-09T06:20:04.758 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.758 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.758 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:04.758 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.831966+0000", 2023-10-09T06:20:04.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2504999999999999e-05 2023-10-09T06:20:04.758 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.759 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:04.759 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095739+0000", 2023-10-09T06:20:04.759 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.263772805 2023-10-09T06:20:04.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.759 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:04.759 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095760+0000", 2023-10-09T06:20:04.759 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1154000000000001e-05 2023-10-09T06:20:04.759 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.759 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:04.759 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.760 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.760 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:83038 10.9 e514/490 10:99b1b726:::benchmark_data_smithi064_573750_object83037:head v 514'5168, mlcod=513'5166)", 2023-10-09T06:20:04.760 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.834529+0000", 2023-10-09T06:20:04.760 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 422.88801839899998, 2023-10-09T06:20:04.760 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26413685199999998, 2023-10-09T06:20:04.760 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:04.760 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:04.760 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:04.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.760 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:04.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834529+0000", 2023-10-09T06:20:04.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:04.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834529+0000", 2023-10-09T06:20:04.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:04.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834529+0000", 2023-10-09T06:20:04.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:20:04.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:04.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834561+0000", 2023-10-09T06:20:04.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2317999999999998e-05 2023-10-09T06:20:04.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:04.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095659+0000", 2023-10-09T06:20:04.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26109802799999998 2023-10-09T06:20:04.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:04.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095671+0000", 2023-10-09T06:20:04.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2126e-05 2023-10-09T06:20:04.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:04.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095734+0000", 2023-10-09T06:20:04.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3114000000000006e-05 2023-10-09T06:20:04.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:04.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.095761+0000", 2023-10-09T06:20:04.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7072000000000001e-05 2023-10-09T06:20:04.764 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.764 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:04.764 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.098631+0000", 2023-10-09T06:20:04.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002870147 2023-10-09T06:20:04.764 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.764 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:04.764 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.098666+0000", 2023-10-09T06:20:04.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4882999999999998e-05 2023-10-09T06:20:04.765 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.765 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:04.765 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.765 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPushReply(10.3 514/510 [PushReplyOp(10:c32ef31e:::benchmark_data_smithi064_573750_object32521:head)])", 2023-10-09T06:20:04.765 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:05.610248+0000", 2023-10-09T06:20:04.765 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 419.11230002299999, 2023-10-09T06:20:04.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29548134799999998, 2023-10-09T06:20:04.765 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:04.765 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:20:04.765 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:04.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:04.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.610248+0000", 2023-10-09T06:20:04.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:04.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.610248+0000", 2023-10-09T06:20:04.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:04.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.610247+0000", 2023-10-09T06:20:04.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:04.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:04.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.610251+0000", 2023-10-09T06:20:04.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2650000000000001e-06 2023-10-09T06:20:04.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:04.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.610258+0000", 2023-10-09T06:20:04.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.695e-06 2023-10-09T06:20:04.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.768 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:04.768 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.610274+0000", 2023-10-09T06:20:04.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5656000000000001e-05 2023-10-09T06:20:04.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.768 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:04.768 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.610296+0000", 2023-10-09T06:20:04.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1787999999999999e-05 2023-10-09T06:20:04.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.769 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.769 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:04.769 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.905729+0000", 2023-10-09T06:20:04.769 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29543320099999998 2023-10-09T06:20:04.769 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.769 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:04.769 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.769 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.769 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.769 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:85642 10.f e514/490 10:f2b65d04:::benchmark_data_smithi064_573750_object85641:head v 514'5293, mlcod=514'5291)", 2023-10-09T06:20:04.769 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:05.624785+0000", 2023-10-09T06:20:04.769 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 419.09776285200002, 2023-10-09T06:20:04.770 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29269213900000002, 2023-10-09T06:20:04.770 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:04.770 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:04.770 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:04.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.770 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:04.770 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.624785+0000", 2023-10-09T06:20:04.770 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.770 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:04.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.624785+0000", 2023-10-09T06:20:04.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.771 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:04.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.624784+0000", 2023-10-09T06:20:04.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:04.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.771 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:04.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.624814+0000", 2023-10-09T06:20:04.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0809999999999998e-05 2023-10-09T06:20:04.772 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:04.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.905640+0000", 2023-10-09T06:20:04.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28082586199999998 2023-10-09T06:20:04.772 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:04.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.905655+0000", 2023-10-09T06:20:04.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5089000000000001e-05 2023-10-09T06:20:04.772 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.773 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:04.773 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.906522+0000", 2023-10-09T06:20:04.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00086632699999999996 2023-10-09T06:20:04.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.773 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:04.773 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.906542+0000", 2023-10-09T06:20:04.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0621000000000001e-05 2023-10-09T06:20:04.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:04.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.917427+0000", 2023-10-09T06:20:04.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010885212 2023-10-09T06:20:04.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.774 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:04.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:05.917477+0000", 2023-10-09T06:20:04.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9546999999999998e-05 2023-10-09T06:20:04.774 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.774 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:04.774 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.775 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:86106 10.3 10:c9bcf3d5:::benchmark_data_smithi064_573750_object86105:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e515)", 2023-10-09T06:20:04.775 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:06.327720+0000", 2023-10-09T06:20:04.775 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 418.39482784400002, 2023-10-09T06:20:04.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18239918699999999, 2023-10-09T06:20:04.775 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:04.775 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:04.775 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:04.775 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:20:04.775 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:04.775 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 86106 2023-10-09T06:20:04.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.776 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:04.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:04.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.327720+0000", 2023-10-09T06:20:04.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:04.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.327720+0000", 2023-10-09T06:20:04.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.777 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:04.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.327718+0000", 2023-10-09T06:20:04.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:04.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.777 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:04.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.327749+0000", 2023-10-09T06:20:04.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0487000000000001e-05 2023-10-09T06:20:04.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.777 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:04.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.327763+0000", 2023-10-09T06:20:04.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4093e-05 2023-10-09T06:20:04.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:04.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.327772+0000", 2023-10-09T06:20:04.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1929999999999999e-06 2023-10-09T06:20:04.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:04.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.327970+0000", 2023-10-09T06:20:04.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019824600000000001 2023-10-09T06:20:04.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:04.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.328115+0000", 2023-10-09T06:20:04.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000144511 2023-10-09T06:20:04.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:20:04.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.328262+0000", 2023-10-09T06:20:04.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014681199999999999 2023-10-09T06:20:04.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:04.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.332137+0000", 2023-10-09T06:20:04.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038754219999999999 2023-10-09T06:20:04.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.332173+0000", 2023-10-09T06:20:04.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5808e-05 2023-10-09T06:20:04.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.334060+0000", 2023-10-09T06:20:04.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018872299999999999 2023-10-09T06:20:04.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.509949+0000", 2023-10-09T06:20:04.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.17588947499999999 2023-10-09T06:20:04.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:04.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.510005+0000", 2023-10-09T06:20:04.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5696999999999999e-05 2023-10-09T06:20:04.782 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:04.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.510119+0000", 2023-10-09T06:20:04.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000113927 2023-10-09T06:20:04.782 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.782 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:04.782 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.783 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:86114 10.3 10:ce2d7be7:::benchmark_data_smithi064_573750_object86113:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e515)", 2023-10-09T06:20:04.783 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:06.335134+0000", 2023-10-09T06:20:04.783 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 418.38741329499999, 2023-10-09T06:20:04.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18034898099999999, 2023-10-09T06:20:04.783 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:04.783 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:04.783 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:04.783 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:20:04.783 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:04.784 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 86114 2023-10-09T06:20:04.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.784 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:04.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.784 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:04.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.335134+0000", 2023-10-09T06:20:04.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.784 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:04.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.335134+0000", 2023-10-09T06:20:04.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.785 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:04.785 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.335133+0000", 2023-10-09T06:20:04.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:04.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.785 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:04.785 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.335155+0000", 2023-10-09T06:20:04.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1685e-05 2023-10-09T06:20:04.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:04.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.335164+0000", 2023-10-09T06:20:04.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5359999999999998e-06 2023-10-09T06:20:04.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:04.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.335170+0000", 2023-10-09T06:20:04.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9390000000000002e-06 2023-10-09T06:20:04.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:04.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.335383+0000", 2023-10-09T06:20:04.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00021336900000000001 2023-10-09T06:20:04.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:04.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.506890+0000", 2023-10-09T06:20:04.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.17150653399999999 2023-10-09T06:20:04.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:20:04.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.507101+0000", 2023-10-09T06:20:04.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00021099199999999999 2023-10-09T06:20:04.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:04.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.511984+0000", 2023-10-09T06:20:04.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048832479999999998 2023-10-09T06:20:04.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.512022+0000", 2023-10-09T06:20:04.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7540000000000003e-05 2023-10-09T06:20:04.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.512315+0000", 2023-10-09T06:20:04.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000293481 2023-10-09T06:20:04.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.515299+0000", 2023-10-09T06:20:04.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029837219999999999 2023-10-09T06:20:04.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:04.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.515362+0000", 2023-10-09T06:20:04.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2644999999999994e-05 2023-10-09T06:20:04.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:04.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:06.515483+0000", 2023-10-09T06:20:04.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012182500000000001 2023-10-09T06:20:04.790 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.790 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:04.790 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.790 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:92993 10.3 10:cbd0b40f:::benchmark_data_smithi064_573750_object92992:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-10-09T06:20:04.790 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:17.279671+0000", 2023-10-09T06:20:04.791 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 407.44287625800001, 2023-10-09T06:20:04.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20967518099999999, 2023-10-09T06:20:04.791 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:04.791 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:04.791 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:04.791 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:20:04.791 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:04.791 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 92993 2023-10-09T06:20:04.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.791 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:04.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:04.792 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.279671+0000", 2023-10-09T06:20:04.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.792 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.792 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:04.792 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.279671+0000", 2023-10-09T06:20:04.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.792 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.792 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:04.792 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.279670+0000", 2023-10-09T06:20:04.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:04.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:04.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.279696+0000", 2023-10-09T06:20:04.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6265999999999999e-05 2023-10-09T06:20:04.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:04.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.279708+0000", 2023-10-09T06:20:04.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1521e-05 2023-10-09T06:20:04.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:04.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.279716+0000", 2023-10-09T06:20:04.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0399999999999993e-06 2023-10-09T06:20:04.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:04.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.280201+0000", 2023-10-09T06:20:04.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00048531699999999997 2023-10-09T06:20:04.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:04.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.280360+0000", 2023-10-09T06:20:04.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015919400000000001 2023-10-09T06:20:04.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:20:04.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.280515+0000", 2023-10-09T06:20:04.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015432400000000001 2023-10-09T06:20:04.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.287237+0000", 2023-10-09T06:20:04.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067219150000000002 2023-10-09T06:20:04.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.287420+0000", 2023-10-09T06:20:04.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018320200000000001 2023-10-09T06:20:04.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:04.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.289348+0000", 2023-10-09T06:20:04.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001928206 2023-10-09T06:20:04.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.489167+0000", 2023-10-09T06:20:04.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.19981912800000001 2023-10-09T06:20:04.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:04.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.489228+0000", 2023-10-09T06:20:04.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0606999999999998e-05 2023-10-09T06:20:04.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:04.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.489347+0000", 2023-10-09T06:20:04.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118967 2023-10-09T06:20:04.798 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.798 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:04.798 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.798 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:93019 10.3 10:c9254536:::benchmark_data_smithi064_573750_object93018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-10-09T06:20:04.798 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:17.310056+0000", 2023-10-09T06:20:04.798 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 407.41249212399998, 2023-10-09T06:20:04.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.183025682, 2023-10-09T06:20:04.798 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:04.799 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:04.799 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:04.799 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:20:04.799 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:04.799 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 93019 2023-10-09T06:20:04.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.799 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:04.799 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.799 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:04.799 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310056+0000", 2023-10-09T06:20:04.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:04.800 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310056+0000", 2023-10-09T06:20:04.800 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.800 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:04.800 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310054+0000", 2023-10-09T06:20:04.800 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:04.800 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:04.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310075+0000", 2023-10-09T06:20:04.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0990999999999999e-05 2023-10-09T06:20:04.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:04.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310090+0000", 2023-10-09T06:20:04.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4449e-05 2023-10-09T06:20:04.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:04.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310112+0000", 2023-10-09T06:20:04.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2952999999999999e-05 2023-10-09T06:20:04.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:04.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310155+0000", 2023-10-09T06:20:04.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2534000000000003e-05 2023-10-09T06:20:04.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:04.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310286+0000", 2023-10-09T06:20:04.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013121699999999999 2023-10-09T06:20:04.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.803 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:20:04.803 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.310404+0000", 2023-10-09T06:20:04.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011796899999999999 2023-10-09T06:20:04.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.803 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:04.803 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.312787+0000", 2023-10-09T06:20:04.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0023830919999999998 2023-10-09T06:20:04.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.803 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.313853+0000", 2023-10-09T06:20:04.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001066132 2023-10-09T06:20:04.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.313932+0000", 2023-10-09T06:20:04.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8236000000000001e-05 2023-10-09T06:20:04.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.492876+0000", 2023-10-09T06:20:04.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.17894394499999999 2023-10-09T06:20:04.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:04.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.492941+0000", 2023-10-09T06:20:04.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5642999999999995e-05 2023-10-09T06:20:04.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:04.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:17.493081+0000", 2023-10-09T06:20:04.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014008799999999999 2023-10-09T06:20:04.805 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.805 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:04.806 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.806 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:94743 10.3 10:c7ba2f96:::benchmark_data_smithi064_573750_object94742:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-10-09T06:20:04.806 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:19.875917+0000", 2023-10-09T06:20:04.806 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 404.846630597, 2023-10-09T06:20:04.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.22279960200000001, 2023-10-09T06:20:04.806 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:04.806 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:04.806 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:04.806 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:20:04.807 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:04.807 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 94743 2023-10-09T06:20:04.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.807 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:04.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:04.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.875917+0000", 2023-10-09T06:20:04.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:04.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.875917+0000", 2023-10-09T06:20:04.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:04.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.875916+0000", 2023-10-09T06:20:04.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:04.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:04.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.875940+0000", 2023-10-09T06:20:04.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4561000000000001e-05 2023-10-09T06:20:04.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:04.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.875953+0000", 2023-10-09T06:20:04.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3276e-05 2023-10-09T06:20:04.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:04.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.875962+0000", 2023-10-09T06:20:04.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1340000000000004e-06 2023-10-09T06:20:04.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:04.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.876240+0000", 2023-10-09T06:20:04.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00027892999999999999 2023-10-09T06:20:04.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:04.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.876436+0000", 2023-10-09T06:20:04.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000195317 2023-10-09T06:20:04.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:20:04.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.876653+0000", 2023-10-09T06:20:04.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000216918 2023-10-09T06:20:04.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.881334+0000", 2023-10-09T06:20:04.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046809870000000002 2023-10-09T06:20:04.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:04.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.881349+0000", 2023-10-09T06:20:04.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5311999999999998e-05 2023-10-09T06:20:04.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:19.884746+0000", 2023-10-09T06:20:04.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033974000000000001 2023-10-09T06:20:04.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.098510+0000", 2023-10-09T06:20:04.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.213763589 2023-10-09T06:20:04.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:04.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.098578+0000", 2023-10-09T06:20:04.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8112000000000001e-05 2023-10-09T06:20:04.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:04.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:20.098717+0000", 2023-10-09T06:20:04.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013864199999999999 2023-10-09T06:20:04.813 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.813 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:04.813 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.814 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:95775 10.3 10:c599e099:::benchmark_data_smithi064_573750_object95774:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-10-09T06:20:04.814 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:23.554330+0000", 2023-10-09T06:20:04.814 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 401.16821817099998, 2023-10-09T06:20:04.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26772326899999999, 2023-10-09T06:20:04.814 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:04.814 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:04.814 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:04.814 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:20:04.814 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:04.814 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 95775 2023-10-09T06:20:04.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.815 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:04.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:04.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.554330+0000", 2023-10-09T06:20:04.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:04.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.554330+0000", 2023-10-09T06:20:04.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:04.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.554328+0000", 2023-10-09T06:20:04.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:04.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:04.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.554352+0000", 2023-10-09T06:20:04.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4216999999999999e-05 2023-10-09T06:20:04.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:04.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.554365+0000", 2023-10-09T06:20:04.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2947e-05 2023-10-09T06:20:04.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:04.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.554373+0000", 2023-10-09T06:20:04.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2120000000000003e-06 2023-10-09T06:20:04.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:04.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.557893+0000", 2023-10-09T06:20:04.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035198059999999999 2023-10-09T06:20:04.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:04.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.558059+0000", 2023-10-09T06:20:04.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000165971 2023-10-09T06:20:04.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,5", 2023-10-09T06:20:04.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.558259+0000", 2023-10-09T06:20:04.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020019300000000001 2023-10-09T06:20:04.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.561632+0000", 2023-10-09T06:20:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033720709999999999 2023-10-09T06:20:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.561647+0000", 2023-10-09T06:20:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5965e-05 2023-10-09T06:20:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821847+0000", 2023-10-09T06:20:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26019937599999998 2023-10-09T06:20:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821877+0000", 2023-10-09T06:20:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9989e-05 2023-10-09T06:20:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821916+0000", 2023-10-09T06:20:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8781e-05 2023-10-09T06:20:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.822053+0000", 2023-10-09T06:20:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013723500000000001 2023-10-09T06:20:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPushReply(10.3 523/510 [PushReplyOp(10:cc80a1b0:::benchmark_data_smithi064_573750_object75768:head)])", 2023-10-09T06:20:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:23.562483+0000", 2023-10-09T06:20:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 401.16006517099999, 2023-10-09T06:20:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25932817600000002, 2023-10-09T06:20:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:20:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562483+0000", 2023-10-09T06:20:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562483+0000", 2023-10-09T06:20:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562482+0000", 2023-10-09T06:20:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:20:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562488+0000", 2023-10-09T06:20:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6699999999999997e-06 2023-10-09T06:20:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562501+0000", 2023-10-09T06:20:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3005e-05 2023-10-09T06:20:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562511+0000", 2023-10-09T06:20:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6679999999999996e-06 2023-10-09T06:20:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562563+0000", 2023-10-09T06:20:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1659000000000001e-05 2023-10-09T06:20:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821811+0000", 2023-10-09T06:20:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25924797700000002 2023-10-09T06:20:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.46283.0:95775 10.3 e523/510 ondisk, result = 0)", 2023-10-09T06:20:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:23.562660+0000", 2023-10-09T06:20:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 401.159887465, 2023-10-09T06:20:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25919107400000002, 2023-10-09T06:20:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:20:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562660+0000", 2023-10-09T06:20:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562660+0000", 2023-10-09T06:20:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562660+0000", 2023-10-09T06:20:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:20:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562665+0000", 2023-10-09T06:20:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.682e-06 2023-10-09T06:20:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562675+0000", 2023-10-09T06:20:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7510000000000003e-06 2023-10-09T06:20:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.562682+0000", 2023-10-09T06:20:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.391e-06 2023-10-09T06:20:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821834+0000", 2023-10-09T06:20:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25915209099999997 2023-10-09T06:20:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821842+0000", 2023-10-09T06:20:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4719999999999998e-06 2023-10-09T06:20:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821851+0000", 2023-10-09T06:20:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4269999999999997e-06 2023-10-09T06:20:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.46283.0:95775 10.3 e523/510 ondisk, result = 0)", 2023-10-09T06:20:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:23.563416+0000", 2023-10-09T06:20:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 401.15913154499998, 2023-10-09T06:20:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25864464399999998, 2023-10-09T06:20:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:20:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.563416+0000", 2023-10-09T06:20:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.563416+0000", 2023-10-09T06:20:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.563416+0000", 2023-10-09T06:20:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.563421+0000", 2023-10-09T06:20:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1250000000000001e-06 2023-10-09T06:20:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821790+0000", 2023-10-09T06:20:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25836899800000002 2023-10-09T06:20:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821803+0000", 2023-10-09T06:20:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.326e-05 2023-10-09T06:20:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821867+0000", 2023-10-09T06:20:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3613000000000002e-05 2023-10-09T06:20:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.821873+0000", 2023-10-09T06:20:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2090000000000002e-06 2023-10-09T06:20:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:23.822061+0000", 2023-10-09T06:20:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018808799999999999 2023-10-09T06:20:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:117026 10.f e536/490 10:f52e132e:::benchmark_data_smithi064_573750_object20811:head v 536'7258, mlcod=536'7256)", 2023-10-09T06:20:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:57.908348+0000", 2023-10-09T06:20:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 366.81420021700001, 2023-10-09T06:20:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.213648951, 2023-10-09T06:20:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908348+0000", 2023-10-09T06:20:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908348+0000", 2023-10-09T06:20:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908347+0000", 2023-10-09T06:20:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:20:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908353+0000", 2023-10-09T06:20:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7190000000000004e-06 2023-10-09T06:20:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908365+0000", 2023-10-09T06:20:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1433999999999999e-05 2023-10-09T06:20:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908372+0000", 2023-10-09T06:20:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4540000000000001e-06 2023-10-09T06:20:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908494+0000", 2023-10-09T06:20:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000121434 2023-10-09T06:20:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908513+0000", 2023-10-09T06:20:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9556999999999999e-05 2023-10-09T06:20:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.121931+0000", 2023-10-09T06:20:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21341768699999999 2023-10-09T06:20:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.121996+0000", 2023-10-09T06:20:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5518000000000005e-05 2023-10-09T06:20:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:117014 10.5 e536/490 10:a26da6a0:::benchmark_data_smithi064_573750_object20799:head v 536'7222, mlcod=536'7220)", 2023-10-09T06:20:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:57.908415+0000", 2023-10-09T06:20:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 366.81413283699999, 2023-10-09T06:20:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21368757199999999, 2023-10-09T06:20:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908415+0000", 2023-10-09T06:20:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908415+0000", 2023-10-09T06:20:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908415+0000", 2023-10-09T06:20:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908419+0000", 2023-10-09T06:20:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6480000000000002e-06 2023-10-09T06:20:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908427+0000", 2023-10-09T06:20:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6469999999999998e-06 2023-10-09T06:20:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908432+0000", 2023-10-09T06:20:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0239999999999997e-06 2023-10-09T06:20:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908476+0000", 2023-10-09T06:20:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4288e-05 2023-10-09T06:20:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.908497+0000", 2023-10-09T06:20:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1121999999999999e-05 2023-10-09T06:20:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.122071+0000", 2023-10-09T06:20:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21357372499999999 2023-10-09T06:20:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.122102+0000", 2023-10-09T06:20:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1529000000000003e-05 2023-10-09T06:20:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:117020 10.5 e536/490 10:a87652b9:::benchmark_data_smithi064_573750_object20805:head v 536'7223, mlcod=536'7220)", 2023-10-09T06:20:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:57.910671+0000", 2023-10-09T06:20:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 366.81187710900002, 2023-10-09T06:20:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.22155350400000001, 2023-10-09T06:20:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.910671+0000", 2023-10-09T06:20:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.910671+0000", 2023-10-09T06:20:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.910670+0000", 2023-10-09T06:20:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:20:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:57.910676+0000", 2023-10-09T06:20:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9460000000000003e-06 2023-10-09T06:20:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.121961+0000", 2023-10-09T06:20:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21128570499999999 2023-10-09T06:20:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.121974+0000", 2023-10-09T06:20:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2244e-05 2023-10-09T06:20:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.122133+0000", 2023-10-09T06:20:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015961999999999999 2023-10-09T06:20:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.122152+0000", 2023-10-09T06:20:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9131e-05 2023-10-09T06:20:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.132185+0000", 2023-10-09T06:20:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010032961 2023-10-09T06:20:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:58.132224+0000", 2023-10-09T06:20:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8819999999999998e-05 2023-10-09T06:20:04.853 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.853 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:04.853 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.853 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:04.853 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:04.853 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:20:04.853 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:04.884 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:04.987 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:04.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:04.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:05.091 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:05.117 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:04 smithi064 ceph-mon[212750]: pgmap v1638: 119 pgs: 119 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 629 op/s 2023-10-09T06:20:05.117 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:04 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:05.117 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:04 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:20:05.117 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:04 smithi064 ceph-mon[212750]: osdmap e634: 8 total, 8 up, 7 in 2023-10-09T06:20:05.117 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:04 smithi064 ceph-mon[212750]: failed to encode map e634 with expected crc 2023-10-09T06:20:05.117 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:04 smithi064 ceph-mon[212750]: failed to encode map e634 with expected crc 2023-10-09T06:20:05.117 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:04 smithi064 ceph-mon[212750]: failed to encode map e634 with expected crc 2023-10-09T06:20:05.117 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:04 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:05.118 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:04 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:05.118 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:04 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:20:05.118 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:04 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:20:05.118 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:04 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:05.118 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:04 smithi064 ceph-mon[220044]: pgmap v1638: 119 pgs: 119 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 629 op/s 2023-10-09T06:20:05.118 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:04 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:05.118 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:04 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:20:05.118 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:04 smithi064 ceph-mon[220044]: osdmap e634: 8 total, 8 up, 7 in 2023-10-09T06:20:05.118 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:04 smithi064 ceph-mon[220044]: failed to encode map e634 with expected crc 2023-10-09T06:20:05.119 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:04 smithi064 ceph-mon[220044]: failed to encode map e634 with expected crc 2023-10-09T06:20:05.119 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:04 smithi064 ceph-mon[220044]: failed to encode map e634 with expected crc 2023-10-09T06:20:05.119 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:04 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:05.119 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:04 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:05.119 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:04 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:20:05.119 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:04 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:20:05.119 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:04 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:05.120 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:05.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:04 smithi156 ceph-mon[195101]: pgmap v1638: 119 pgs: 119 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 629 op/s 2023-10-09T06:20:05.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:04 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:05.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:04 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:20:05.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:04 smithi156 ceph-mon[195101]: osdmap e634: 8 total, 8 up, 7 in 2023-10-09T06:20:05.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:04 smithi156 ceph-mon[195101]: failed to encode map e634 with expected crc 2023-10-09T06:20:05.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:04 smithi156 ceph-mon[195101]: failed to encode map e634 with expected crc 2023-10-09T06:20:05.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:04 smithi156 ceph-mon[195101]: failed to encode map e634 with expected crc 2023-10-09T06:20:05.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:04 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:05.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:04 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:05.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:04 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:20:05.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:04 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:20:05.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:04 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:05.431 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:20:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:20:05] "GET /metrics HTTP/1.1" 200 36444 "" "Prometheus/2.43.0" 2023-10-09T06:20:05.576 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:20:05.577 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:20:05.577 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:20:05.577 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:20:05.577 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:20:05.577 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:20:05.577 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:20:05.577 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:20:05.577 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:20:05.577 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:20:05.578 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:20:05.578 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:20:05.578 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:20:05.578 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:20:05.578 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:20:05.578 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:20:06.040 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:05 smithi064 ceph-mon[212750]: failed to encode map e634 with expected crc 2023-10-09T06:20:06.040 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:05 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1636167351' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:06.040 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:05 smithi064 ceph-mon[220044]: failed to encode map e634 with expected crc 2023-10-09T06:20:06.040 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:05 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1636167351' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:06.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:05 smithi156 ceph-mon[195101]: failed to encode map e634 with expected crc 2023-10-09T06:20:06.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:05 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1636167351' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:06.148 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:20:06.150 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T06:20:06.290 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:20:06.475 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:06.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:06.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:06.580 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:06.612 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:06.661 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:06.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:06.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:06.767 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:06.797 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:06.843 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:06.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:06.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:06.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:06 smithi156 ceph-mon[195101]: pgmap v1640: 119 pgs: 119 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 530 op/s 2023-10-09T06:20:06.951 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:06.980 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:07.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:06 smithi064 ceph-mon[212750]: pgmap v1640: 119 pgs: 119 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 530 op/s 2023-10-09T06:20:07.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:06 smithi064 ceph-mon[220044]: pgmap v1640: 119 pgs: 119 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 530 op/s 2023-10-09T06:20:07.244 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1640,"stamp":"2023-10-09T06:20:05.352072+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":217885,"num_write_kb":6970334,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16749,"num_bytes_recovered":589718190,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95424,"ondisk_log_size":95424,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":359,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5984652,"kb_used_data":244180,"kb_used_omap":182,"kb_used_meta":5740233,"kb_avail":650231412,"statfs":{"total":671965249536,"available":665836965888,"internally_reserved":0,"allocated":250040320,"data_stored":232792605,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":186958,"internal_metadata":5877999026},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-347799576,"num_objects":-5308,"num_object_clones":0,"num_object_copies":-15924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5308,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002714"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":914,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.519768+0000","last_change":"2023-10-09T06:18:32.883970+0000","last_active":"2023-10-09T06:19:51.519768+0000","last_peered":"2023-10-09T06:19:51.519768+0000","last_clean":"2023-10-09T06:19:51.519768+0000","last_became_active":"2023-10-09T06:18:32.883509+0000","last_became_peered":"2023-10-09T06:18:32.883509+0000","last_unstale":"2023-10-09T06:19:51.519768+0000","last_undegraded":"2023-10-09T06:19:51.519768+0000","last_fullsized":"2023-10-09T06:19:51.519768+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:05.412941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:05.412941+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:30.926648+0000","scrub_duration":0.0061014329999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":711,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517813+0000","last_change":"2023-10-09T06:16:02.748447+0000","last_active":"2023-10-09T06:19:48.517813+0000","last_peered":"2023-10-09T06:19:48.517813+0000","last_clean":"2023-10-09T06:19:48.517813+0000","last_became_active":"2023-10-09T06:16:02.747672+0000","last_became_peered":"2023-10-09T06:16:02.747672+0000","last_unstale":"2023-10-09T06:19:48.517813+0000","last_undegraded":"2023-10-09T06:19:48.517813+0000","last_fullsized":"2023-10-09T06:19:48.517813+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:31.326263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":446,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.523886+0000","last_change":"2023-10-09T06:05:11.451972+0000","last_active":"2023-10-09T06:19:51.523886+0000","last_peered":"2023-10-09T06:19:51.523886+0000","last_clean":"2023-10-09T06:19:51.523886+0000","last_became_active":"2023-10-09T06:05:11.451448+0000","last_became_peered":"2023-10-09T06:05:11.451448+0000","last_unstale":"2023-10-09T06:19:51.523886+0000","last_undegraded":"2023-10-09T06:19:51.523886+0000","last_fullsized":"2023-10-09T06:19:51.523886+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:39.183621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":949,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517918+0000","last_change":"2023-10-09T06:18:32.879870+0000","last_active":"2023-10-09T06:19:48.517918+0000","last_peered":"2023-10-09T06:19:48.517918+0000","last_clean":"2023-10-09T06:19:48.517918+0000","last_became_active":"2023-10-09T06:18:32.879518+0000","last_became_peered":"2023-10-09T06:18:32.879518+0000","last_unstale":"2023-10-09T06:19:48.517918+0000","last_undegraded":"2023-10-09T06:19:48.517918+0000","last_fullsized":"2023-10-09T06:19:48.517918+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:01.468346+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:01.468346+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:28.527869+0000","scrub_duration":0.0016258220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":318,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.506495+0000","last_change":"2023-10-09T06:01:54.762060+0000","last_active":"2023-10-09T06:19:51.506495+0000","last_peered":"2023-10-09T06:19:51.506495+0000","last_clean":"2023-10-09T06:19:51.506495+0000","last_became_active":"2023-10-09T06:01:54.761757+0000","last_became_peered":"2023-10-09T06:01:54.761757+0000","last_unstale":"2023-10-09T06:19:51.506495+0000","last_undegraded":"2023-10-09T06:19:51.506495+0000","last_fullsized":"2023-10-09T06:19:51.506495+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:40.294232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":117,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.520806+0000","last_change":"2023-10-09T06:13:30.789834+0000","last_active":"2023-10-09T06:19:51.520806+0000","last_peered":"2023-10-09T06:19:51.520806+0000","last_clean":"2023-10-09T06:19:51.520806+0000","last_became_active":"2023-10-09T06:13:30.789517+0000","last_became_peered":"2023-10-09T06:13:30.789517+0000","last_unstale":"2023-10-09T06:19:51.520806+0000","last_undegraded":"2023-10-09T06:19:51.520806+0000","last_fullsized":"2023-10-09T06:19:51.520806+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:45.467922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":917,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517607+0000","last_change":"2023-10-09T06:12:07.739121+0000","last_active":"2023-10-09T06:19:48.517607+0000","last_peered":"2023-10-09T06:19:48.517607+0000","last_clean":"2023-10-09T06:19:48.517607+0000","last_became_active":"2023-10-09T06:12:07.738739+0000","last_became_peered":"2023-10-09T06:12:07.738739+0000","last_unstale":"2023-10-09T06:19:48.517607+0000","last_undegraded":"2023-10-09T06:19:48.517607+0000","last_fullsized":"2023-10-09T06:19:48.517607+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.293426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.293426+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:19.815422+0000","scrub_duration":0.001587946,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":428,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.472939+0000","last_change":"2023-10-09T06:09:27.564377+0000","last_active":"2023-10-09T06:19:48.472939+0000","last_peered":"2023-10-09T06:19:48.472939+0000","last_clean":"2023-10-09T06:19:48.472939+0000","last_became_active":"2023-10-09T06:09:27.563710+0000","last_became_peered":"2023-10-09T06:09:27.563710+0000","last_unstale":"2023-10-09T06:19:48.472939+0000","last_undegraded":"2023-10-09T06:19:48.472939+0000","last_fullsized":"2023-10-09T06:19:48.472939+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:48.168447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":108,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.506894+0000","last_change":"2023-10-09T06:18:33.820081+0000","last_active":"2023-10-09T06:19:51.506894+0000","last_peered":"2023-10-09T06:19:51.506894+0000","last_clean":"2023-10-09T06:19:51.506894+0000","last_became_active":"2023-10-09T06:18:33.819790+0000","last_became_peered":"2023-10-09T06:18:33.819790+0000","last_unstale":"2023-10-09T06:19:51.506894+0000","last_undegraded":"2023-10-09T06:19:51.506894+0000","last_fullsized":"2023-10-09T06:19:51.506894+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:18.166606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":869,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.519859+0000","last_change":"2023-10-09T06:18:32.886406+0000","last_active":"2023-10-09T06:19:51.519859+0000","last_peered":"2023-10-09T06:19:51.519859+0000","last_clean":"2023-10-09T06:19:51.519859+0000","last_became_active":"2023-10-09T06:18:32.886142+0000","last_became_peered":"2023-10-09T06:18:32.886142+0000","last_unstale":"2023-10-09T06:19:51.519859+0000","last_undegraded":"2023-10-09T06:19:51.519859+0000","last_fullsized":"2023-10-09T06:19:51.519859+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.384407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.384407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:47:35.682411+0000","scrub_duration":0.0040651960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":548,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473934+0000","last_change":"2023-10-09T06:18:32.819012+0000","last_active":"2023-10-09T06:19:48.473934+0000","last_peered":"2023-10-09T06:19:48.473934+0000","last_clean":"2023-10-09T06:19:48.473934+0000","last_became_active":"2023-10-09T06:18:32.818711+0000","last_became_peered":"2023-10-09T06:18:32.818711+0000","last_unstale":"2023-10-09T06:19:48.473934+0000","last_undegraded":"2023-10-09T06:19:48.473934+0000","last_fullsized":"2023-10-09T06:19:48.473934+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:17:48.281174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":482,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473058+0000","last_change":"2023-10-09T06:18:12.816530+0000","last_active":"2023-10-09T06:19:48.473058+0000","last_peered":"2023-10-09T06:19:48.473058+0000","last_clean":"2023-10-09T06:19:48.473058+0000","last_became_active":"2023-10-09T06:18:12.815392+0000","last_became_peered":"2023-10-09T06:18:12.815392+0000","last_unstale":"2023-10-09T06:19:48.473058+0000","last_undegraded":"2023-10-09T06:19:48.473058+0000","last_fullsized":"2023-10-09T06:19:48.473058+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:18.350974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":778,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.520401+0000","last_change":"2023-10-09T06:16:05.761750+0000","last_active":"2023-10-09T06:19:51.520401+0000","last_peered":"2023-10-09T06:19:51.520401+0000","last_clean":"2023-10-09T06:19:51.520401+0000","last_became_active":"2023-10-09T06:16:05.761436+0000","last_became_peered":"2023-10-09T06:16:05.761436+0000","last_unstale":"2023-10-09T06:19:51.520401+0000","last_undegraded":"2023-10-09T06:19:51.520401+0000","last_fullsized":"2023-10-09T06:19:51.520401+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:20.742561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:20.742561+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:13.021950+0000","scrub_duration":0.0027005039999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":134,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.520487+0000","last_change":"2023-10-09T06:18:32.877651+0000","last_active":"2023-10-09T06:19:51.520487+0000","last_peered":"2023-10-09T06:19:51.520487+0000","last_clean":"2023-10-09T06:19:51.520487+0000","last_became_active":"2023-10-09T06:18:32.877371+0000","last_became_peered":"2023-10-09T06:18:32.877371+0000","last_unstale":"2023-10-09T06:19:51.520487+0000","last_undegraded":"2023-10-09T06:19:51.520487+0000","last_fullsized":"2023-10-09T06:19:51.520487+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:23:59.684830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":646,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473601+0000","last_change":"2023-10-09T06:18:12.816564+0000","last_active":"2023-10-09T06:19:48.473601+0000","last_peered":"2023-10-09T06:19:48.473601+0000","last_clean":"2023-10-09T06:19:48.473601+0000","last_became_active":"2023-10-09T06:18:12.815943+0000","last_became_peered":"2023-10-09T06:18:12.815943+0000","last_unstale":"2023-10-09T06:19:48.473601+0000","last_undegraded":"2023-10-09T06:19:48.473601+0000","last_fullsized":"2023-10-09T06:19:48.473601+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:10.114324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":837,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473376+0000","last_change":"2023-10-09T06:18:33.826699+0000","last_active":"2023-10-09T06:19:48.473376+0000","last_peered":"2023-10-09T06:19:48.473376+0000","last_clean":"2023-10-09T06:19:48.473376+0000","last_became_active":"2023-10-09T06:18:33.826349+0000","last_became_peered":"2023-10-09T06:18:33.826349+0000","last_unstale":"2023-10-09T06:19:48.473376+0000","last_undegraded":"2023-10-09T06:19:48.473376+0000","last_fullsized":"2023-10-09T06:19:48.473376+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:38.588641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:38.588641+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:30:50.340037+0000","scrub_duration":0.010116385,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":123,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.519699+0000","last_change":"2023-10-09T06:13:30.826588+0000","last_active":"2023-10-09T06:19:51.519699+0000","last_peered":"2023-10-09T06:19:51.519699+0000","last_clean":"2023-10-09T06:19:51.519699+0000","last_became_active":"2023-10-09T06:13:30.799199+0000","last_became_peered":"2023-10-09T06:13:30.799199+0000","last_unstale":"2023-10-09T06:19:51.519699+0000","last_undegraded":"2023-10-09T06:19:51.519699+0000","last_fullsized":"2023-10-09T06:19:51.519699+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:38.703864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":82,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.531733+0000","last_change":"2023-10-09T06:18:33.812722+0000","last_active":"2023-10-09T06:19:48.531733+0000","last_peered":"2023-10-09T06:19:48.531733+0000","last_clean":"2023-10-09T06:19:48.531733+0000","last_became_active":"2023-10-09T06:18:33.812386+0000","last_became_peered":"2023-10-09T06:18:33.812386+0000","last_unstale":"2023-10-09T06:19:48.531733+0000","last_undegraded":"2023-10-09T06:19:48.531733+0000","last_fullsized":"2023-10-09T06:19:48.531733+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:26.259509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":808,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.507467+0000","last_change":"2023-10-09T06:09:27.574336+0000","last_active":"2023-10-09T06:19:51.507467+0000","last_peered":"2023-10-09T06:19:51.507467+0000","last_clean":"2023-10-09T06:19:51.507467+0000","last_became_active":"2023-10-09T06:09:27.573974+0000","last_became_peered":"2023-10-09T06:09:27.573974+0000","last_unstale":"2023-10-09T06:19:51.507467+0000","last_undegraded":"2023-10-09T06:19:51.507467+0000","last_fullsized":"2023-10-09T06:19:51.507467+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.339121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.339121+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:45.908579+0000","scrub_duration":0.010474344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":707,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517910+0000","last_change":"2023-10-09T06:18:12.829164+0000","last_active":"2023-10-09T06:19:48.517910+0000","last_peered":"2023-10-09T06:19:48.517910+0000","last_clean":"2023-10-09T06:19:48.517910+0000","last_became_active":"2023-10-09T06:18:12.818219+0000","last_became_peered":"2023-10-09T06:18:12.818219+0000","last_unstale":"2023-10-09T06:19:48.517910+0000","last_undegraded":"2023-10-09T06:19:48.517910+0000","last_fullsized":"2023-10-09T06:19:48.517910+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:14.546650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":672,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.518220+0000","last_change":"2023-10-09T06:18:12.818275+0000","last_active":"2023-10-09T06:19:48.518220+0000","last_peered":"2023-10-09T06:19:48.518220+0000","last_clean":"2023-10-09T06:19:48.518220+0000","last_became_active":"2023-10-09T06:18:12.817978+0000","last_became_peered":"2023-10-09T06:18:12.817978+0000","last_unstale":"2023-10-09T06:19:48.518220+0000","last_undegraded":"2023-10-09T06:19:48.518220+0000","last_fullsized":"2023-10-09T06:19:48.518220+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:52:24.157690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":878,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473241+0000","last_change":"2023-10-09T06:18:32.817454+0000","last_active":"2023-10-09T06:19:48.473241+0000","last_peered":"2023-10-09T06:19:48.473241+0000","last_clean":"2023-10-09T06:19:48.473241+0000","last_became_active":"2023-10-09T06:18:32.817081+0000","last_became_peered":"2023-10-09T06:18:32.817081+0000","last_unstale":"2023-10-09T06:19:48.473241+0000","last_undegraded":"2023-10-09T06:19:48.473241+0000","last_fullsized":"2023-10-09T06:19:48.473241+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:42.900876+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:42.900876+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:50.579398+0000","scrub_duration":0.0014928649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":171,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.520445+0000","last_change":"2023-10-09T06:18:12.833338+0000","last_active":"2023-10-09T06:19:51.520445+0000","last_peered":"2023-10-09T06:19:51.520445+0000","last_clean":"2023-10-09T06:19:51.520445+0000","last_became_active":"2023-10-09T06:18:12.818967+0000","last_became_peered":"2023-10-09T06:18:12.818967+0000","last_unstale":"2023-10-09T06:19:51.520445+0000","last_undegraded":"2023-10-09T06:19:51.520445+0000","last_fullsized":"2023-10-09T06:19:51.520445+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:55.022219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":681,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473581+0000","last_change":"2023-10-09T06:01:54.761558+0000","last_active":"2023-10-09T06:19:48.473581+0000","last_peered":"2023-10-09T06:19:48.473581+0000","last_clean":"2023-10-09T06:19:48.473581+0000","last_became_active":"2023-10-09T06:01:54.761251+0000","last_became_peered":"2023-10-09T06:01:54.761251+0000","last_unstale":"2023-10-09T06:19:48.473581+0000","last_undegraded":"2023-10-09T06:19:48.473581+0000","last_fullsized":"2023-10-09T06:19:48.473581+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:53.245267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":1196,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.520320+0000","last_change":"2023-10-09T06:18:33.824142+0000","last_active":"2023-10-09T06:19:51.520320+0000","last_peered":"2023-10-09T06:19:51.520320+0000","last_clean":"2023-10-09T06:19:51.520320+0000","last_became_active":"2023-10-09T06:18:33.823799+0000","last_became_peered":"2023-10-09T06:18:33.823799+0000","last_unstale":"2023-10-09T06:19:51.520320+0000","last_undegraded":"2023-10-09T06:19:51.520320+0000","last_fullsized":"2023-10-09T06:19:51.520320+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:54:59.270975+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":117,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.519460+0000","last_change":"2023-10-09T06:13:30.788160+0000","last_active":"2023-10-09T06:19:51.519460+0000","last_peered":"2023-10-09T06:19:51.519460+0000","last_clean":"2023-10-09T06:19:51.519460+0000","last_became_active":"2023-10-09T06:13:30.787724+0000","last_became_peered":"2023-10-09T06:13:30.787724+0000","last_unstale":"2023-10-09T06:19:51.519460+0000","last_undegraded":"2023-10-09T06:19:51.519460+0000","last_fullsized":"2023-10-09T06:19:51.519460+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:38.376770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":134,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.521097+0000","last_change":"2023-10-09T06:18:32.884499+0000","last_active":"2023-10-09T06:19:51.521097+0000","last_peered":"2023-10-09T06:19:51.521097+0000","last_clean":"2023-10-09T06:19:51.521097+0000","last_became_active":"2023-10-09T06:18:32.884232+0000","last_became_peered":"2023-10-09T06:18:32.884232+0000","last_unstale":"2023-10-09T06:19:51.521097+0000","last_undegraded":"2023-10-09T06:19:51.521097+0000","last_fullsized":"2023-10-09T06:19:51.521097+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:26.922060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":876,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.507873+0000","last_change":"2023-10-09T06:12:55.750923+0000","last_active":"2023-10-09T06:19:51.507873+0000","last_peered":"2023-10-09T06:19:51.507873+0000","last_clean":"2023-10-09T06:19:51.507873+0000","last_became_active":"2023-10-09T06:12:55.750521+0000","last_became_peered":"2023-10-09T06:12:55.750521+0000","last_unstale":"2023-10-09T06:19:51.507873+0000","last_undegraded":"2023-10-09T06:19:51.507873+0000","last_fullsized":"2023-10-09T06:19:51.507873+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.316697+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.316697+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:23.591439+0000","scrub_duration":0.001666561,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":507,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.519603+0000","last_change":"2023-10-09T06:18:12.813660+0000","last_active":"2023-10-09T06:19:51.519603+0000","last_peered":"2023-10-09T06:19:51.519603+0000","last_clean":"2023-10-09T06:19:51.519603+0000","last_became_active":"2023-10-09T06:18:12.813327+0000","last_became_peered":"2023-10-09T06:18:12.813327+0000","last_unstale":"2023-10-09T06:19:51.519603+0000","last_undegraded":"2023-10-09T06:19:51.519603+0000","last_fullsized":"2023-10-09T06:19:51.519603+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:55.382525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":597,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473837+0000","last_change":"2023-10-09T06:13:30.788284+0000","last_active":"2023-10-09T06:19:48.473837+0000","last_peered":"2023-10-09T06:19:48.473837+0000","last_clean":"2023-10-09T06:19:48.473837+0000","last_became_active":"2023-10-09T06:13:30.787933+0000","last_became_peered":"2023-10-09T06:13:30.787933+0000","last_unstale":"2023-10-09T06:19:48.473837+0000","last_undegraded":"2023-10-09T06:19:48.473837+0000","last_fullsized":"2023-10-09T06:19:48.473837+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:10.735438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":976,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.509038+0000","last_change":"2023-10-09T06:18:33.815711+0000","last_active":"2023-10-09T06:19:51.509038+0000","last_peered":"2023-10-09T06:19:51.509038+0000","last_clean":"2023-10-09T06:19:51.509038+0000","last_became_active":"2023-10-09T06:18:33.815143+0000","last_became_peered":"2023-10-09T06:18:33.815143+0000","last_unstale":"2023-10-09T06:19:51.509038+0000","last_undegraded":"2023-10-09T06:19:51.509038+0000","last_fullsized":"2023-10-09T06:19:51.509038+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:40.388125+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:40.388125+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:43.731337+0000","scrub_duration":0.0055576990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":262,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.530945+0000","last_change":"2023-10-09T06:18:33.812677+0000","last_active":"2023-10-09T06:19:48.530945+0000","last_peered":"2023-10-09T06:19:48.530945+0000","last_clean":"2023-10-09T06:19:48.530945+0000","last_became_active":"2023-10-09T06:18:33.812289+0000","last_became_peered":"2023-10-09T06:18:33.812289+0000","last_unstale":"2023-10-09T06:19:48.530945+0000","last_undegraded":"2023-10-09T06:19:48.530945+0000","last_fullsized":"2023-10-09T06:19:48.530945+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:02.279506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":160,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473827+0000","last_change":"2023-10-09T06:18:32.878768+0000","last_active":"2023-10-09T06:19:48.473827+0000","last_peered":"2023-10-09T06:19:48.473827+0000","last_clean":"2023-10-09T06:19:48.473827+0000","last_became_active":"2023-10-09T06:18:32.878461+0000","last_became_peered":"2023-10-09T06:18:32.878461+0000","last_unstale":"2023-10-09T06:19:48.473827+0000","last_undegraded":"2023-10-09T06:19:48.473827+0000","last_fullsized":"2023-10-09T06:19:48.473827+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:55.024045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"633'13608","reported_seq":17482,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:59.096326+0000","last_change":"2023-10-09T06:19:27.377538+0000","last_active":"2023-10-09T06:19:59.096326+0000","last_peered":"2023-10-09T06:19:59.096326+0000","last_clean":"2023-10-09T06:19:59.096326+0000","last_became_active":"2023-10-09T06:18:55.965695+0000","last_became_peered":"2023-10-09T06:18:55.965695+0000","last_unstale":"2023-10-09T06:19:59.096326+0000","last_undegraded":"2023-10-09T06:19:59.096326+0000","last_fullsized":"2023-10-09T06:19:59.096326+0000","mapping_epoch":617,"log_start":"606'8380","ondisk_log_start":"606'8380","created":572,"last_epoch_clean":626,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5228,"log_dups_size":0,"ondisk_log_size":5228,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:44.689199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13608,"num_write_kb":435456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1887,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":815,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.520384+0000","last_change":"2023-10-09T06:13:33.701521+0000","last_active":"2023-10-09T06:19:51.520384+0000","last_peered":"2023-10-09T06:19:51.520384+0000","last_clean":"2023-10-09T06:19:51.520384+0000","last_became_active":"2023-10-09T06:13:33.700870+0000","last_became_peered":"2023-10-09T06:13:33.700870+0000","last_unstale":"2023-10-09T06:19:51.520384+0000","last_undegraded":"2023-10-09T06:19:51.520384+0000","last_fullsized":"2023-10-09T06:19:51.520384+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:10:59.422551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:10:59.422551+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:54.402494+0000","scrub_duration":0.0059716659999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":121,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.520920+0000","last_change":"2023-10-09T06:13:30.822903+0000","last_active":"2023-10-09T06:19:51.520920+0000","last_peered":"2023-10-09T06:19:51.520920+0000","last_clean":"2023-10-09T06:19:51.520920+0000","last_became_active":"2023-10-09T06:13:30.799013+0000","last_became_peered":"2023-10-09T06:13:30.799013+0000","last_unstale":"2023-10-09T06:19:51.520920+0000","last_undegraded":"2023-10-09T06:19:51.520920+0000","last_fullsized":"2023-10-09T06:19:51.520920+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:44.271821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":158,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.519665+0000","last_change":"2023-10-09T06:18:33.824502+0000","last_active":"2023-10-09T06:19:51.519665+0000","last_peered":"2023-10-09T06:19:51.519665+0000","last_clean":"2023-10-09T06:19:51.519665+0000","last_became_active":"2023-10-09T06:18:33.824063+0000","last_became_peered":"2023-10-09T06:18:33.824063+0000","last_unstale":"2023-10-09T06:19:51.519665+0000","last_undegraded":"2023-10-09T06:19:51.519665+0000","last_fullsized":"2023-10-09T06:19:51.519665+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:43.724163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.6","version":"633'13506","reported_seq":13578,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:59.117689+0000","last_change":"2023-10-09T06:16:02.756362+0000","last_active":"2023-10-09T06:19:59.117689+0000","last_peered":"2023-10-09T06:19:59.117689+0000","last_clean":"2023-10-09T06:19:59.117689+0000","last_became_active":"2023-10-09T06:16:02.756033+0000","last_became_peered":"2023-10-09T06:16:02.756033+0000","last_unstale":"2023-10-09T06:19:59.117689+0000","last_undegraded":"2023-10-09T06:19:59.117689+0000","last_fullsized":"2023-10-09T06:19:59.117689+0000","mapping_epoch":572,"log_start":"606'8314","ondisk_log_start":"606'8314","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5192,"log_dups_size":0,"ondisk_log_size":5192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:07.492688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13506,"num_write_kb":432192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":1049,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473357+0000","last_change":"2023-10-09T06:18:15.867832+0000","last_active":"2023-10-09T06:19:48.473357+0000","last_peered":"2023-10-09T06:19:48.473357+0000","last_clean":"2023-10-09T06:19:48.473357+0000","last_became_active":"2023-10-09T06:18:15.864512+0000","last_became_peered":"2023-10-09T06:18:15.864512+0000","last_unstale":"2023-10-09T06:19:48.473357+0000","last_undegraded":"2023-10-09T06:19:48.473357+0000","last_fullsized":"2023-10-09T06:19:48.473357+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:03.241335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:03.241335+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:45.197013+0000","scrub_duration":0.017780547000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":121,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.520415+0000","last_change":"2023-10-09T06:13:30.829763+0000","last_active":"2023-10-09T06:19:51.520415+0000","last_peered":"2023-10-09T06:19:51.520415+0000","last_clean":"2023-10-09T06:19:51.520415+0000","last_became_active":"2023-10-09T06:13:30.802486+0000","last_became_peered":"2023-10-09T06:13:30.802486+0000","last_unstale":"2023-10-09T06:19:51.520415+0000","last_undegraded":"2023-10-09T06:19:51.520415+0000","last_fullsized":"2023-10-09T06:19:51.520415+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:05.468205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":671,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517618+0000","last_change":"2023-10-09T06:01:54.760367+0000","last_active":"2023-10-09T06:19:48.517618+0000","last_peered":"2023-10-09T06:19:48.517618+0000","last_clean":"2023-10-09T06:19:48.517618+0000","last_became_active":"2023-10-09T06:01:54.760087+0000","last_became_peered":"2023-10-09T06:01:54.760087+0000","last_unstale":"2023-10-09T06:19:48.517618+0000","last_undegraded":"2023-10-09T06:19:48.517618+0000","last_fullsized":"2023-10-09T06:19:48.517618+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:26.000104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"633'13646","reported_seq":23057,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:59.111213+0000","last_change":"2023-10-09T06:19:28.923381+0000","last_active":"2023-10-09T06:19:59.111213+0000","last_peered":"2023-10-09T06:19:59.111213+0000","last_clean":"2023-10-09T06:19:59.111213+0000","last_became_active":"2023-10-09T06:19:28.854095+0000","last_became_peered":"2023-10-09T06:19:28.854095+0000","last_unstale":"2023-10-09T06:19:59.111213+0000","last_undegraded":"2023-10-09T06:19:59.111213+0000","last_fullsized":"2023-10-09T06:19:59.111213+0000","mapping_epoch":627,"log_start":"601'7850","ondisk_log_start":"601'7850","created":572,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5796,"log_dups_size":0,"ondisk_log_size":5796,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:20.378253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13646,"num_write_kb":436672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4645,"num_bytes_recovered":178978816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":955,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.518235+0000","last_change":"2023-10-09T06:18:33.817134+0000","last_active":"2023-10-09T06:19:48.518235+0000","last_peered":"2023-10-09T06:19:48.518235+0000","last_clean":"2023-10-09T06:19:48.518235+0000","last_became_active":"2023-10-09T06:18:33.816772+0000","last_became_peered":"2023-10-09T06:18:33.816772+0000","last_unstale":"2023-10-09T06:19:48.518235+0000","last_undegraded":"2023-10-09T06:19:48.518235+0000","last_fullsized":"2023-10-09T06:19:48.518235+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:48.495739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:48.495739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:24.990051+0000","scrub_duration":0.0016345509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":170,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.520295+0000","last_change":"2023-10-09T06:18:33.830787+0000","last_active":"2023-10-09T06:19:51.520295+0000","last_peered":"2023-10-09T06:19:51.520295+0000","last_clean":"2023-10-09T06:19:51.520295+0000","last_became_active":"2023-10-09T06:18:33.830538+0000","last_became_peered":"2023-10-09T06:18:33.830538+0000","last_unstale":"2023-10-09T06:19:51.520295+0000","last_undegraded":"2023-10-09T06:19:51.520295+0000","last_fullsized":"2023-10-09T06:19:51.520295+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:54.818646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":685,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.508738+0000","last_change":"2023-10-09T06:18:12.818137+0000","last_active":"2023-10-09T06:19:51.508738+0000","last_peered":"2023-10-09T06:19:51.508738+0000","last_clean":"2023-10-09T06:19:51.508738+0000","last_became_active":"2023-10-09T06:18:12.817853+0000","last_became_peered":"2023-10-09T06:18:12.817853+0000","last_unstale":"2023-10-09T06:19:51.508738+0000","last_undegraded":"2023-10-09T06:19:51.508738+0000","last_fullsized":"2023-10-09T06:19:51.508738+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:14:01.939320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.4","version":"633'13570","reported_seq":14985,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:59.110110+0000","last_change":"2023-10-09T06:18:13.830297+0000","last_active":"2023-10-09T06:19:59.110110+0000","last_peered":"2023-10-09T06:19:59.110110+0000","last_clean":"2023-10-09T06:19:59.110110+0000","last_became_active":"2023-10-09T06:18:13.814894+0000","last_became_peered":"2023-10-09T06:18:13.814894+0000","last_unstale":"2023-10-09T06:19:59.110110+0000","last_undegraded":"2023-10-09T06:19:59.110110+0000","last_fullsized":"2023-10-09T06:19:59.110110+0000","mapping_epoch":604,"log_start":"601'7762","ondisk_log_start":"601'7762","created":572,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5808,"log_dups_size":0,"ondisk_log_size":5808,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:32.472441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13570,"num_write_kb":434240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":659,"num_bytes_recovered":43188224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":848,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.518111+0000","last_change":"2023-10-09T06:13:33.708291+0000","last_active":"2023-10-09T06:19:48.518111+0000","last_peered":"2023-10-09T06:19:48.518111+0000","last_clean":"2023-10-09T06:19:48.518111+0000","last_became_active":"2023-10-09T06:13:33.707930+0000","last_became_peered":"2023-10-09T06:13:33.707930+0000","last_unstale":"2023-10-09T06:19:48.518111+0000","last_undegraded":"2023-10-09T06:19:48.518111+0000","last_fullsized":"2023-10-09T06:19:48.518111+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:16.361097+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:16.361097+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:18.111404+0000","scrub_duration":0.028533749000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":440,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.513437+0000","last_change":"2023-10-09T06:18:33.818994+0000","last_active":"2023-10-09T06:19:51.513437+0000","last_peered":"2023-10-09T06:19:51.513437+0000","last_clean":"2023-10-09T06:19:51.513437+0000","last_became_active":"2023-10-09T06:18:33.818598+0000","last_became_peered":"2023-10-09T06:18:33.818598+0000","last_unstale":"2023-10-09T06:19:51.513437+0000","last_undegraded":"2023-10-09T06:19:51.513437+0000","last_fullsized":"2023-10-09T06:19:51.513437+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:59.239942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":648,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.531358+0000","last_change":"2023-10-09T06:09:27.558353+0000","last_active":"2023-10-09T06:19:48.531358+0000","last_peered":"2023-10-09T06:19:48.531358+0000","last_clean":"2023-10-09T06:19:48.531358+0000","last_became_active":"2023-10-09T06:09:27.557233+0000","last_became_peered":"2023-10-09T06:09:27.557233+0000","last_unstale":"2023-10-09T06:19:48.531358+0000","last_undegraded":"2023-10-09T06:19:48.531358+0000","last_fullsized":"2023-10-09T06:19:48.531358+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:53.149350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"633'13926","reported_seq":14032,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:59.100204+0000","last_change":"2023-10-09T06:18:13.827731+0000","last_active":"2023-10-09T06:19:59.100204+0000","last_peered":"2023-10-09T06:19:59.100204+0000","last_clean":"2023-10-09T06:19:59.100204+0000","last_became_active":"2023-10-09T06:18:13.813485+0000","last_became_peered":"2023-10-09T06:18:13.813485+0000","last_unstale":"2023-10-09T06:19:59.100204+0000","last_undegraded":"2023-10-09T06:19:59.100204+0000","last_fullsized":"2023-10-09T06:19:59.100204+0000","mapping_epoch":604,"log_start":"601'8064","ondisk_log_start":"601'8064","created":572,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5862,"log_dups_size":0,"ondisk_log_size":5862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:57.856421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13926,"num_write_kb":445632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":1057,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.519445+0000","last_change":"2023-10-09T06:18:32.884000+0000","last_active":"2023-10-09T06:19:51.519445+0000","last_peered":"2023-10-09T06:19:51.519445+0000","last_clean":"2023-10-09T06:19:51.519445+0000","last_became_active":"2023-10-09T06:18:32.883640+0000","last_became_peered":"2023-10-09T06:18:32.883640+0000","last_unstale":"2023-10-09T06:19:51.519445+0000","last_undegraded":"2023-10-09T06:19:51.519445+0000","last_fullsized":"2023-10-09T06:19:51.519445+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:42.795781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:42.795781+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:39.855289+0000","scrub_duration":0.0091262979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":439,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.513767+0000","last_change":"2023-10-09T06:09:27.565666+0000","last_active":"2023-10-09T06:19:51.513767+0000","last_peered":"2023-10-09T06:19:51.513767+0000","last_clean":"2023-10-09T06:19:51.513767+0000","last_became_active":"2023-10-09T06:09:27.565324+0000","last_became_peered":"2023-10-09T06:09:27.565324+0000","last_unstale":"2023-10-09T06:19:51.513767+0000","last_undegraded":"2023-10-09T06:19:51.513767+0000","last_fullsized":"2023-10-09T06:19:51.513767+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:51.216947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":648,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517930+0000","last_change":"2023-10-09T06:18:33.812578+0000","last_active":"2023-10-09T06:19:48.517930+0000","last_peered":"2023-10-09T06:19:48.517930+0000","last_clean":"2023-10-09T06:19:48.517930+0000","last_became_active":"2023-10-09T06:18:33.812233+0000","last_became_peered":"2023-10-09T06:18:33.812233+0000","last_unstale":"2023-10-09T06:19:48.517930+0000","last_undegraded":"2023-10-09T06:19:48.517930+0000","last_fullsized":"2023-10-09T06:19:48.517930+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:49.671676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"633'13610","reported_seq":13681,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:59.070676+0000","last_change":"2023-10-09T06:16:02.747886+0000","last_active":"2023-10-09T06:19:59.070676+0000","last_peered":"2023-10-09T06:19:59.070676+0000","last_clean":"2023-10-09T06:19:59.070676+0000","last_became_active":"2023-10-09T06:16:02.747569+0000","last_became_peered":"2023-10-09T06:16:02.747569+0000","last_unstale":"2023-10-09T06:19:59.070676+0000","last_undegraded":"2023-10-09T06:19:59.070676+0000","last_fullsized":"2023-10-09T06:19:59.070676+0000","mapping_epoch":572,"log_start":"601'7857","ondisk_log_start":"601'7857","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5753,"log_dups_size":0,"ondisk_log_size":5753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:18.720375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13610,"num_write_kb":435520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":965,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.472828+0000","last_change":"2023-10-09T06:18:33.829178+0000","last_active":"2023-10-09T06:19:48.472828+0000","last_peered":"2023-10-09T06:19:48.472828+0000","last_clean":"2023-10-09T06:19:48.472828+0000","last_became_active":"2023-10-09T06:18:33.828843+0000","last_became_peered":"2023-10-09T06:18:33.828843+0000","last_unstale":"2023-10-09T06:19:48.472828+0000","last_undegraded":"2023-10-09T06:19:48.472828+0000","last_fullsized":"2023-10-09T06:19:48.472828+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.282393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.282393+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:15:35.723179+0000","scrub_duration":0.01402403,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":572,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517714+0000","last_change":"2023-10-09T06:18:12.815749+0000","last_active":"2023-10-09T06:19:48.517714+0000","last_peered":"2023-10-09T06:19:48.517714+0000","last_clean":"2023-10-09T06:19:48.517714+0000","last_became_active":"2023-10-09T06:18:12.815447+0000","last_became_peered":"2023-10-09T06:18:12.815447+0000","last_unstale":"2023-10-09T06:19:48.517714+0000","last_undegraded":"2023-10-09T06:19:48.517714+0000","last_fullsized":"2023-10-09T06:19:48.517714+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:42.083129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":711,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.518293+0000","last_change":"2023-10-09T06:18:12.820093+0000","last_active":"2023-10-09T06:19:48.518293+0000","last_peered":"2023-10-09T06:19:48.518293+0000","last_clean":"2023-10-09T06:19:48.518293+0000","last_became_active":"2023-10-09T06:18:12.819805+0000","last_became_peered":"2023-10-09T06:18:12.819805+0000","last_unstale":"2023-10-09T06:19:48.518293+0000","last_undegraded":"2023-10-09T06:19:48.518293+0000","last_fullsized":"2023-10-09T06:19:48.518293+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:54:00.429779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"633'13368","reported_seq":16396,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:59.113482+0000","last_change":"2023-10-09T06:18:13.823458+0000","last_active":"2023-10-09T06:19:59.113482+0000","last_peered":"2023-10-09T06:19:59.113482+0000","last_clean":"2023-10-09T06:19:59.113482+0000","last_became_active":"2023-10-09T06:18:13.813080+0000","last_became_peered":"2023-10-09T06:18:13.813080+0000","last_unstale":"2023-10-09T06:19:59.113482+0000","last_undegraded":"2023-10-09T06:19:59.113482+0000","last_fullsized":"2023-10-09T06:19:59.113482+0000","mapping_epoch":604,"log_start":"601'7611","ondisk_log_start":"601'7611","created":572,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5757,"log_dups_size":0,"ondisk_log_size":5757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:48:07.669455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13368,"num_write_kb":427776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1468,"num_bytes_recovered":96206848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":986,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:20:00.270134+0000","last_change":"2023-10-09T06:18:32.812418+0000","last_active":"2023-10-09T06:20:00.270134+0000","last_peered":"2023-10-09T06:20:00.270134+0000","last_clean":"2023-10-09T06:20:00.270134+0000","last_became_active":"2023-10-09T06:18:32.812079+0000","last_became_peered":"2023-10-09T06:18:32.812079+0000","last_unstale":"2023-10-09T06:20:00.270134+0000","last_undegraded":"2023-10-09T06:20:00.270134+0000","last_fullsized":"2023-10-09T06:20:00.270134+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.373748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.373748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:22.098164+0000","scrub_duration":0.0024576450000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":633,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517730+0000","last_change":"2023-10-09T06:16:02.744252+0000","last_active":"2023-10-09T06:19:48.517730+0000","last_peered":"2023-10-09T06:19:48.517730+0000","last_clean":"2023-10-09T06:19:48.517730+0000","last_became_active":"2023-10-09T06:16:02.743947+0000","last_became_peered":"2023-10-09T06:16:02.743947+0000","last_unstale":"2023-10-09T06:19:48.517730+0000","last_undegraded":"2023-10-09T06:19:48.517730+0000","last_fullsized":"2023-10-09T06:19:48.517730+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:14:52.293310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":684,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.518215+0000","last_change":"2023-10-09T06:18:12.819118+0000","last_active":"2023-10-09T06:19:48.518215+0000","last_peered":"2023-10-09T06:19:48.518215+0000","last_clean":"2023-10-09T06:19:48.518215+0000","last_became_active":"2023-10-09T06:18:12.818820+0000","last_became_peered":"2023-10-09T06:18:12.818820+0000","last_unstale":"2023-10-09T06:19:48.518215+0000","last_undegraded":"2023-10-09T06:19:48.518215+0000","last_fullsized":"2023-10-09T06:19:48.518215+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:26.399422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"633'13902","reported_seq":13974,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:59.118314+0000","last_change":"2023-10-09T06:16:02.744178+0000","last_active":"2023-10-09T06:19:59.118314+0000","last_peered":"2023-10-09T06:19:59.118314+0000","last_clean":"2023-10-09T06:19:59.118314+0000","last_became_active":"2023-10-09T06:16:02.743835+0000","last_became_peered":"2023-10-09T06:16:02.743835+0000","last_unstale":"2023-10-09T06:19:59.118314+0000","last_undegraded":"2023-10-09T06:19:59.118314+0000","last_fullsized":"2023-10-09T06:19:59.118314+0000","mapping_epoch":572,"log_start":"607'8650","ondisk_log_start":"607'8650","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5252,"log_dups_size":0,"ondisk_log_size":5252,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:02.861316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13902,"num_write_kb":444864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":751,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517614+0000","last_change":"2023-10-09T06:18:33.812696+0000","last_active":"2023-10-09T06:19:48.517614+0000","last_peered":"2023-10-09T06:19:48.517614+0000","last_clean":"2023-10-09T06:19:48.517614+0000","last_became_active":"2023-10-09T06:18:33.812371+0000","last_became_peered":"2023-10-09T06:18:33.812371+0000","last_unstale":"2023-10-09T06:19:48.517614+0000","last_undegraded":"2023-10-09T06:19:48.517614+0000","last_fullsized":"2023-10-09T06:19:48.517614+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:39.633410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:39.633410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:05.464966+0000","scrub_duration":0.0039938459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":447,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.472935+0000","last_change":"2023-10-09T06:13:30.781849+0000","last_active":"2023-10-09T06:19:48.472935+0000","last_peered":"2023-10-09T06:19:48.472935+0000","last_clean":"2023-10-09T06:19:48.472935+0000","last_became_active":"2023-10-09T06:13:30.781084+0000","last_became_peered":"2023-10-09T06:13:30.781084+0000","last_unstale":"2023-10-09T06:19:48.472935+0000","last_undegraded":"2023-10-09T06:19:48.472935+0000","last_fullsized":"2023-10-09T06:19:48.472935+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:02.199298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":319,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:20:00.270248+0000","last_change":"2023-10-09T06:01:54.761768+0000","last_active":"2023-10-09T06:20:00.270248+0000","last_peered":"2023-10-09T06:20:00.270248+0000","last_clean":"2023-10-09T06:20:00.270248+0000","last_became_active":"2023-10-09T06:01:54.761443+0000","last_became_peered":"2023-10-09T06:01:54.761443+0000","last_unstale":"2023-10-09T06:20:00.270248+0000","last_undegraded":"2023-10-09T06:20:00.270248+0000","last_fullsized":"2023-10-09T06:20:00.270248+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:51.555987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"633'13494","reported_seq":16996,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:59.110175+0000","last_change":"2023-10-09T06:18:13.836219+0000","last_active":"2023-10-09T06:19:59.110175+0000","last_peered":"2023-10-09T06:19:59.110175+0000","last_clean":"2023-10-09T06:19:59.110175+0000","last_became_active":"2023-10-09T06:18:13.812037+0000","last_became_peered":"2023-10-09T06:18:13.812037+0000","last_unstale":"2023-10-09T06:19:59.110175+0000","last_undegraded":"2023-10-09T06:19:59.110175+0000","last_fullsized":"2023-10-09T06:19:59.110175+0000","mapping_epoch":604,"log_start":"601'7582","ondisk_log_start":"601'7582","created":572,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5912,"log_dups_size":0,"ondisk_log_size":5912,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:58:37.778303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13494,"num_write_kb":431745,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1698,"num_bytes_recovered":111280128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":878,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473884+0000","last_change":"2023-10-09T06:18:15.863360+0000","last_active":"2023-10-09T06:19:48.473884+0000","last_peered":"2023-10-09T06:19:48.473884+0000","last_clean":"2023-10-09T06:19:48.473884+0000","last_became_active":"2023-10-09T06:18:15.863003+0000","last_became_peered":"2023-10-09T06:18:15.863003+0000","last_unstale":"2023-10-09T06:19:48.473884+0000","last_undegraded":"2023-10-09T06:19:48.473884+0000","last_fullsized":"2023-10-09T06:19:48.473884+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:12:22.289803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:12:22.289803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:40.618745+0000","scrub_duration":0.0076565050000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":381,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.506740+0000","last_change":"2023-10-09T06:12:04.680247+0000","last_active":"2023-10-09T06:19:51.506740+0000","last_peered":"2023-10-09T06:19:51.506740+0000","last_clean":"2023-10-09T06:19:51.506740+0000","last_became_active":"2023-10-09T06:12:04.679943+0000","last_became_peered":"2023-10-09T06:12:04.679943+0000","last_unstale":"2023-10-09T06:19:51.506740+0000","last_undegraded":"2023-10-09T06:19:51.506740+0000","last_fullsized":"2023-10-09T06:19:51.506740+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:56.692774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":467,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473377+0000","last_change":"2023-10-09T06:18:33.813934+0000","last_active":"2023-10-09T06:19:48.473377+0000","last_peered":"2023-10-09T06:19:48.473377+0000","last_clean":"2023-10-09T06:19:48.473377+0000","last_became_active":"2023-10-09T06:18:33.813570+0000","last_became_peered":"2023-10-09T06:18:33.813570+0000","last_unstale":"2023-10-09T06:19:48.473377+0000","last_undegraded":"2023-10-09T06:19:48.473377+0000","last_fullsized":"2023-10-09T06:19:48.473377+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:20.477422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":367,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.508878+0000","last_change":"2023-10-09T06:18:32.877171+0000","last_active":"2023-10-09T06:19:51.508878+0000","last_peered":"2023-10-09T06:19:51.508878+0000","last_clean":"2023-10-09T06:19:51.508878+0000","last_became_active":"2023-10-09T06:18:32.876933+0000","last_became_peered":"2023-10-09T06:18:32.876933+0000","last_unstale":"2023-10-09T06:19:51.508878+0000","last_undegraded":"2023-10-09T06:19:51.508878+0000","last_fullsized":"2023-10-09T06:19:51.508878+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:10.529213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"633'13326","reported_seq":13397,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:59.100911+0000","last_change":"2023-10-09T06:16:02.749724+0000","last_active":"2023-10-09T06:19:59.100911+0000","last_peered":"2023-10-09T06:19:59.100911+0000","last_clean":"2023-10-09T06:19:59.100911+0000","last_became_active":"2023-10-09T06:16:02.749368+0000","last_became_peered":"2023-10-09T06:16:02.749368+0000","last_unstale":"2023-10-09T06:19:59.100911+0000","last_undegraded":"2023-10-09T06:19:59.100911+0000","last_fullsized":"2023-10-09T06:19:59.100911+0000","mapping_epoch":572,"log_start":"601'7597","ondisk_log_start":"601'7597","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5729,"log_dups_size":0,"ondisk_log_size":5729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:24.189151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13326,"num_write_kb":426432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":903,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.575304+0000","last_change":"2023-10-09T06:18:32.874923+0000","last_active":"2023-10-09T06:19:51.575304+0000","last_peered":"2023-10-09T06:19:51.575304+0000","last_clean":"2023-10-09T06:19:51.575304+0000","last_became_active":"2023-10-09T06:18:32.874574+0000","last_became_peered":"2023-10-09T06:18:32.874574+0000","last_unstale":"2023-10-09T06:19:51.575304+0000","last_undegraded":"2023-10-09T06:19:51.575304+0000","last_fullsized":"2023-10-09T06:19:51.575304+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:40.807378+0000","scrub_duration":0.0016660799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":372,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.508953+0000","last_change":"2023-10-09T06:18:33.815495+0000","last_active":"2023-10-09T06:19:51.508953+0000","last_peered":"2023-10-09T06:19:51.508953+0000","last_clean":"2023-10-09T06:19:51.508953+0000","last_became_active":"2023-10-09T06:18:33.814689+0000","last_became_peered":"2023-10-09T06:18:33.814689+0000","last_unstale":"2023-10-09T06:19:51.508953+0000","last_undegraded":"2023-10-09T06:19:51.508953+0000","last_fullsized":"2023-10-09T06:19:51.508953+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:54.929165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":633,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517691+0000","last_change":"2023-10-09T06:18:12.816566+0000","last_active":"2023-10-09T06:19:48.517691+0000","last_peered":"2023-10-09T06:19:48.517691+0000","last_clean":"2023-10-09T06:19:48.517691+0000","last_became_active":"2023-10-09T06:18:12.816245+0000","last_became_peered":"2023-10-09T06:18:12.816245+0000","last_unstale":"2023-10-09T06:19:48.517691+0000","last_undegraded":"2023-10-09T06:19:48.517691+0000","last_fullsized":"2023-10-09T06:19:48.517691+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:49.240586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":489,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.506823+0000","last_change":"2023-10-09T06:18:12.818212+0000","last_active":"2023-10-09T06:19:51.506823+0000","last_peered":"2023-10-09T06:19:51.506823+0000","last_clean":"2023-10-09T06:19:51.506823+0000","last_became_active":"2023-10-09T06:18:12.817911+0000","last_became_peered":"2023-10-09T06:18:12.817911+0000","last_unstale":"2023-10-09T06:19:51.506823+0000","last_undegraded":"2023-10-09T06:19:51.506823+0000","last_fullsized":"2023-10-09T06:19:51.506823+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:48.259119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"633'13764","reported_seq":22524,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:59.090540+0000","last_change":"2023-10-09T06:19:15.586290+0000","last_active":"2023-10-09T06:19:59.090540+0000","last_peered":"2023-10-09T06:19:59.090540+0000","last_clean":"2023-10-09T06:19:59.090540+0000","last_became_active":"2023-10-09T06:19:15.562959+0000","last_became_peered":"2023-10-09T06:19:15.562959+0000","last_unstale":"2023-10-09T06:19:59.090540+0000","last_undegraded":"2023-10-09T06:19:59.090540+0000","last_fullsized":"2023-10-09T06:19:59.090540+0000","mapping_epoch":623,"log_start":"602'8039","ondisk_log_start":"602'8039","created":572,"last_epoch_clean":624,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5725,"log_dups_size":0,"ondisk_log_size":5725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:37:06.168866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13764,"num_write_kb":440448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4323,"num_bytes_recovered":157679616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":689,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.519507+0000","last_change":"2023-10-09T06:14:47.339271+0000","last_active":"2023-10-09T06:19:51.519507+0000","last_peered":"2023-10-09T06:19:51.519507+0000","last_clean":"2023-10-09T06:19:51.519507+0000","last_became_active":"2023-10-09T06:13:33.705351+0000","last_became_peered":"2023-10-09T06:13:33.705351+0000","last_unstale":"2023-10-09T06:19:51.519507+0000","last_undegraded":"2023-10-09T06:19:51.519507+0000","last_fullsized":"2023-10-09T06:19:51.519507+0000","mapping_epoch":529,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:14:47.339163+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T06:14:47.339163+0000","last_clean_scrub_stamp":"2023-10-09T06:14:47.339163+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:34.887758+0000","scrub_duration":0.0016607729999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":425,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.176488+0000","last_change":"2023-10-09T06:15:16.802788+0000","last_active":"2023-10-09T06:19:51.176488+0000","last_peered":"2023-10-09T06:19:51.176488+0000","last_clean":"2023-10-09T06:19:51.176488+0000","last_became_active":"2023-10-09T06:15:16.802449+0000","last_became_peered":"2023-10-09T06:15:16.802449+0000","last_unstale":"2023-10-09T06:19:51.176488+0000","last_undegraded":"2023-10-09T06:19:51.176488+0000","last_fullsized":"2023-10-09T06:19:51.176488+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:55:30.488174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":440,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:20:00.270063+0000","last_change":"2023-10-09T06:18:12.818545+0000","last_active":"2023-10-09T06:20:00.270063+0000","last_peered":"2023-10-09T06:20:00.270063+0000","last_clean":"2023-10-09T06:20:00.270063+0000","last_became_active":"2023-10-09T06:18:12.818281+0000","last_became_peered":"2023-10-09T06:18:12.818281+0000","last_unstale":"2023-10-09T06:20:00.270063+0000","last_undegraded":"2023-10-09T06:20:00.270063+0000","last_fullsized":"2023-10-09T06:20:00.270063+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:31:33.668786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":349,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.507061+0000","last_change":"2023-10-09T06:18:32.818287+0000","last_active":"2023-10-09T06:19:51.507061+0000","last_peered":"2023-10-09T06:19:51.507061+0000","last_clean":"2023-10-09T06:19:51.507061+0000","last_became_active":"2023-10-09T06:18:32.817946+0000","last_became_peered":"2023-10-09T06:18:32.817946+0000","last_unstale":"2023-10-09T06:19:51.507061+0000","last_undegraded":"2023-10-09T06:19:51.507061+0000","last_fullsized":"2023-10-09T06:19:51.507061+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:17.156210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"633'13382","reported_seq":17435,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:59.046463+0000","last_change":"2023-10-09T06:19:48.514562+0000","last_active":"2023-10-09T06:19:59.046463+0000","last_peered":"2023-10-09T06:19:59.046463+0000","last_clean":"2023-10-09T06:19:59.046463+0000","last_became_active":"2023-10-09T06:19:48.476882+0000","last_became_peered":"2023-10-09T06:19:48.476882+0000","last_unstale":"2023-10-09T06:19:59.046463+0000","last_undegraded":"2023-10-09T06:19:59.046463+0000","last_fullsized":"2023-10-09T06:19:59.046463+0000","mapping_epoch":631,"log_start":"601'7666","ondisk_log_start":"601'7666","created":572,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5716,"log_dups_size":0,"ondisk_log_size":5716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:11.550660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13382,"num_write_kb":428224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1971,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":144,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.520527+0000","last_change":"2023-10-09T06:18:32.880094+0000","last_active":"2023-10-09T06:19:51.520527+0000","last_peered":"2023-10-09T06:19:51.520527+0000","last_clean":"2023-10-09T06:19:51.520527+0000","last_became_active":"2023-10-09T06:18:32.879777+0000","last_became_peered":"2023-10-09T06:18:32.879777+0000","last_unstale":"2023-10-09T06:19:51.520527+0000","last_undegraded":"2023-10-09T06:19:51.520527+0000","last_fullsized":"2023-10-09T06:19:51.520527+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:54.918094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":717,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.519530+0000","last_change":"2023-10-09T06:18:15.852226+0000","last_active":"2023-10-09T06:19:51.519530+0000","last_peered":"2023-10-09T06:19:51.519530+0000","last_clean":"2023-10-09T06:19:51.519530+0000","last_became_active":"2023-10-09T06:18:15.851868+0000","last_became_peered":"2023-10-09T06:18:15.851868+0000","last_unstale":"2023-10-09T06:19:51.519530+0000","last_undegraded":"2023-10-09T06:19:51.519530+0000","last_fullsized":"2023-10-09T06:19:51.519530+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:35.783434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:35.783434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:57.511684+0000","scrub_duration":0.0043794799999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":458,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473369+0000","last_change":"2023-10-09T06:18:33.827842+0000","last_active":"2023-10-09T06:19:48.473369+0000","last_peered":"2023-10-09T06:19:48.473369+0000","last_clean":"2023-10-09T06:19:48.473369+0000","last_became_active":"2023-10-09T06:18:33.827482+0000","last_became_peered":"2023-10-09T06:18:33.827482+0000","last_unstale":"2023-10-09T06:19:48.473369+0000","last_undegraded":"2023-10-09T06:19:48.473369+0000","last_fullsized":"2023-10-09T06:19:48.473369+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:46.769907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":583,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.518214+0000","last_change":"2023-10-09T06:18:12.820039+0000","last_active":"2023-10-09T06:19:48.518214+0000","last_peered":"2023-10-09T06:19:48.518214+0000","last_clean":"2023-10-09T06:19:48.518214+0000","last_became_active":"2023-10-09T06:18:12.819410+0000","last_became_peered":"2023-10-09T06:18:12.819410+0000","last_unstale":"2023-10-09T06:19:48.518214+0000","last_undegraded":"2023-10-09T06:19:48.518214+0000","last_fullsized":"2023-10-09T06:19:48.518214+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:24.514809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"633'13526","reported_seq":13615,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:59.070332+0000","last_change":"2023-10-09T06:18:13.823451+0000","last_active":"2023-10-09T06:19:59.070332+0000","last_peered":"2023-10-09T06:19:59.070332+0000","last_clean":"2023-10-09T06:19:59.070332+0000","last_became_active":"2023-10-09T06:18:13.811684+0000","last_became_peered":"2023-10-09T06:18:13.811684+0000","last_unstale":"2023-10-09T06:19:59.070332+0000","last_undegraded":"2023-10-09T06:19:59.070332+0000","last_fullsized":"2023-10-09T06:19:59.070332+0000","mapping_epoch":604,"log_start":"601'7796","ondisk_log_start":"601'7796","created":572,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5730,"log_dups_size":0,"ondisk_log_size":5730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:01:16.295158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13526,"num_write_kb":432832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":945,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.507805+0000","last_change":"2023-10-09T06:18:15.847552+0000","last_active":"2023-10-09T06:19:51.507805+0000","last_peered":"2023-10-09T06:19:51.507805+0000","last_clean":"2023-10-09T06:19:51.507805+0000","last_became_active":"2023-10-09T06:18:15.847229+0000","last_became_peered":"2023-10-09T06:18:15.847229+0000","last_unstale":"2023-10-09T06:19:51.507805+0000","last_undegraded":"2023-10-09T06:19:51.507805+0000","last_fullsized":"2023-10-09T06:19:51.507805+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:57.627296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:57.627296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:02:19.535788+0000","scrub_duration":0.001913166,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":527,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.520254+0000","last_change":"2023-10-09T06:09:27.544373+0000","last_active":"2023-10-09T06:19:51.520254+0000","last_peered":"2023-10-09T06:19:51.520254+0000","last_clean":"2023-10-09T06:19:51.520254+0000","last_became_active":"2023-10-09T06:09:27.544121+0000","last_became_peered":"2023-10-09T06:09:27.544121+0000","last_unstale":"2023-10-09T06:19:51.520254+0000","last_undegraded":"2023-10-09T06:19:51.520254+0000","last_fullsized":"2023-10-09T06:19:51.520254+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:40.429491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":573,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.575137+0000","last_change":"2023-10-09T06:12:52.670573+0000","last_active":"2023-10-09T06:19:51.575137+0000","last_peered":"2023-10-09T06:19:51.575137+0000","last_clean":"2023-10-09T06:19:51.575137+0000","last_became_active":"2023-10-09T06:12:52.670242+0000","last_became_peered":"2023-10-09T06:12:52.670242+0000","last_unstale":"2023-10-09T06:19:51.575137+0000","last_undegraded":"2023-10-09T06:19:51.575137+0000","last_fullsized":"2023-10-09T06:19:51.575137+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:25.237580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":720,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.531533+0000","last_change":"2023-10-09T06:15:16.805514+0000","last_active":"2023-10-09T06:19:48.531533+0000","last_peered":"2023-10-09T06:19:48.531533+0000","last_clean":"2023-10-09T06:19:48.531533+0000","last_became_active":"2023-10-09T06:15:16.805160+0000","last_became_peered":"2023-10-09T06:15:16.805160+0000","last_unstale":"2023-10-09T06:19:48.531533+0000","last_undegraded":"2023-10-09T06:19:48.531533+0000","last_fullsized":"2023-10-09T06:19:48.531533+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:30.153699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"633'13664","reported_seq":13736,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:59.116528+0000","last_change":"2023-10-09T06:16:02.755231+0000","last_active":"2023-10-09T06:19:59.116528+0000","last_peered":"2023-10-09T06:19:59.116528+0000","last_clean":"2023-10-09T06:19:59.116528+0000","last_became_active":"2023-10-09T06:16:02.754877+0000","last_became_peered":"2023-10-09T06:16:02.754877+0000","last_unstale":"2023-10-09T06:19:59.116528+0000","last_undegraded":"2023-10-09T06:19:59.116528+0000","last_fullsized":"2023-10-09T06:19:59.116528+0000","mapping_epoch":572,"log_start":"606'8446","ondisk_log_start":"606'8446","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5218,"log_dups_size":0,"ondisk_log_size":5218,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:16:25.397496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13664,"num_write_kb":437248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":869,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473837+0000","last_change":"2023-10-09T06:18:15.863482+0000","last_active":"2023-10-09T06:19:48.473837+0000","last_peered":"2023-10-09T06:19:48.473837+0000","last_clean":"2023-10-09T06:19:48.473837+0000","last_became_active":"2023-10-09T06:18:15.863069+0000","last_became_peered":"2023-10-09T06:18:15.863069+0000","last_unstale":"2023-10-09T06:19:48.473837+0000","last_undegraded":"2023-10-09T06:19:48.473837+0000","last_fullsized":"2023-10-09T06:19:48.473837+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:45.844892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:45.844892+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:50.117604+0000","scrub_duration":0.0017521489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":117,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.519603+0000","last_change":"2023-10-09T06:13:30.799535+0000","last_active":"2023-10-09T06:19:51.519603+0000","last_peered":"2023-10-09T06:19:51.519603+0000","last_clean":"2023-10-09T06:19:51.519603+0000","last_became_active":"2023-10-09T06:13:30.799227+0000","last_became_peered":"2023-10-09T06:13:30.799227+0000","last_unstale":"2023-10-09T06:19:51.519603+0000","last_undegraded":"2023-10-09T06:19:51.519603+0000","last_fullsized":"2023-10-09T06:19:51.519603+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:14.741742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":379,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.506912+0000","last_change":"2023-10-09T06:18:32.812317+0000","last_active":"2023-10-09T06:19:51.506912+0000","last_peered":"2023-10-09T06:19:51.506912+0000","last_clean":"2023-10-09T06:19:51.506912+0000","last_became_active":"2023-10-09T06:18:32.812010+0000","last_became_peered":"2023-10-09T06:18:32.812010+0000","last_unstale":"2023-10-09T06:19:51.506912+0000","last_undegraded":"2023-10-09T06:19:51.506912+0000","last_fullsized":"2023-10-09T06:19:51.506912+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:31.460164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":134,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.520355+0000","last_change":"2023-10-09T06:18:32.879994+0000","last_active":"2023-10-09T06:19:51.520355+0000","last_peered":"2023-10-09T06:19:51.520355+0000","last_clean":"2023-10-09T06:19:51.520355+0000","last_became_active":"2023-10-09T06:18:32.879702+0000","last_became_peered":"2023-10-09T06:18:32.879702+0000","last_unstale":"2023-10-09T06:19:51.520355+0000","last_undegraded":"2023-10-09T06:19:51.520355+0000","last_fullsized":"2023-10-09T06:19:51.520355+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:56.654232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.f","version":"633'13676","reported_seq":13746,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:59.060601+0000","last_change":"2023-10-09T06:16:02.755513+0000","last_active":"2023-10-09T06:19:59.060601+0000","last_peered":"2023-10-09T06:19:59.060601+0000","last_clean":"2023-10-09T06:19:59.060601+0000","last_became_active":"2023-10-09T06:16:02.755072+0000","last_became_peered":"2023-10-09T06:16:02.755072+0000","last_unstale":"2023-10-09T06:19:59.060601+0000","last_undegraded":"2023-10-09T06:19:59.060601+0000","last_fullsized":"2023-10-09T06:19:59.060601+0000","mapping_epoch":572,"log_start":"601'7871","ondisk_log_start":"601'7871","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5805,"log_dups_size":0,"ondisk_log_size":5805,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:11.106599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13676,"num_write_kb":437632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":1185,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.518378+0000","last_change":"2023-10-09T06:18:33.816307+0000","last_active":"2023-10-09T06:19:48.518378+0000","last_peered":"2023-10-09T06:19:48.518378+0000","last_clean":"2023-10-09T06:19:48.518378+0000","last_became_active":"2023-10-09T06:18:33.815766+0000","last_became_peered":"2023-10-09T06:18:33.815766+0000","last_unstale":"2023-10-09T06:19:48.518378+0000","last_undegraded":"2023-10-09T06:19:48.518378+0000","last_fullsized":"2023-10-09T06:19:48.518378+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:17.344518+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:17.344518+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:59.097601+0000","scrub_duration":0.0015914099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":692,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473972+0000","last_change":"2023-10-09T06:13:30.798394+0000","last_active":"2023-10-09T06:19:48.473972+0000","last_peered":"2023-10-09T06:19:48.473972+0000","last_clean":"2023-10-09T06:19:48.473972+0000","last_became_active":"2023-10-09T06:13:30.790913+0000","last_became_peered":"2023-10-09T06:13:30.790913+0000","last_unstale":"2023-10-09T06:19:48.473972+0000","last_undegraded":"2023-10-09T06:19:48.473972+0000","last_fullsized":"2023-10-09T06:19:48.473972+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:26.508690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":471,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.507424+0000","last_change":"2023-10-09T06:18:12.818370+0000","last_active":"2023-10-09T06:19:51.507424+0000","last_peered":"2023-10-09T06:19:51.507424+0000","last_clean":"2023-10-09T06:19:51.507424+0000","last_became_active":"2023-10-09T06:18:12.818031+0000","last_became_peered":"2023-10-09T06:18:12.818031+0000","last_unstale":"2023-10-09T06:19:51.507424+0000","last_undegraded":"2023-10-09T06:19:51.507424+0000","last_fullsized":"2023-10-09T06:19:51.507424+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:20:31.304808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":441,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473146+0000","last_change":"2023-10-09T06:13:30.782038+0000","last_active":"2023-10-09T06:19:48.473146+0000","last_peered":"2023-10-09T06:19:48.473146+0000","last_clean":"2023-10-09T06:19:48.473146+0000","last_became_active":"2023-10-09T06:13:30.781500+0000","last_became_peered":"2023-10-09T06:13:30.781500+0000","last_unstale":"2023-10-09T06:19:48.473146+0000","last_undegraded":"2023-10-09T06:19:48.473146+0000","last_fullsized":"2023-10-09T06:19:48.473146+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:43.663265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"633'13856","reported_seq":13925,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:59.118915+0000","last_change":"2023-10-09T06:16:02.755624+0000","last_active":"2023-10-09T06:19:59.118915+0000","last_peered":"2023-10-09T06:19:59.118915+0000","last_clean":"2023-10-09T06:19:59.118915+0000","last_became_active":"2023-10-09T06:16:02.755333+0000","last_became_peered":"2023-10-09T06:16:02.755333+0000","last_unstale":"2023-10-09T06:19:59.118915+0000","last_undegraded":"2023-10-09T06:19:59.118915+0000","last_fullsized":"2023-10-09T06:19:59.118915+0000","mapping_epoch":572,"log_start":"601'7961","ondisk_log_start":"601'7961","created":572,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:16:02.213829+0000","last_clean_scrub_stamp":"2023-10-09T06:16:02.213829+0000","objects_scrubbed":0,"log_size":5895,"log_dups_size":0,"ondisk_log_size":5895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:59.742519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13856,"num_write_kb":443392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":982,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473253+0000","last_change":"2023-10-09T06:18:32.879060+0000","last_active":"2023-10-09T06:19:48.473253+0000","last_peered":"2023-10-09T06:19:48.473253+0000","last_clean":"2023-10-09T06:19:48.473253+0000","last_became_active":"2023-10-09T06:18:32.878836+0000","last_became_peered":"2023-10-09T06:18:32.878836+0000","last_unstale":"2023-10-09T06:19:48.473253+0000","last_undegraded":"2023-10-09T06:19:48.473253+0000","last_fullsized":"2023-10-09T06:19:48.473253+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:06:22.900486+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":147,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.521042+0000","last_change":"2023-10-09T06:18:12.818437+0000","last_active":"2023-10-09T06:19:51.521042+0000","last_peered":"2023-10-09T06:19:51.521042+0000","last_clean":"2023-10-09T06:19:51.521042+0000","last_became_active":"2023-10-09T06:18:12.818126+0000","last_became_peered":"2023-10-09T06:18:12.818126+0000","last_unstale":"2023-10-09T06:19:51.521042+0000","last_undegraded":"2023-10-09T06:19:51.521042+0000","last_fullsized":"2023-10-09T06:19:51.521042+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:00:02.738584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":681,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.531643+0000","last_change":"2023-10-09T06:09:27.578756+0000","last_active":"2023-10-09T06:19:48.531643+0000","last_peered":"2023-10-09T06:19:48.531643+0000","last_clean":"2023-10-09T06:19:48.531643+0000","last_became_active":"2023-10-09T06:09:27.578384+0000","last_became_peered":"2023-10-09T06:09:27.578384+0000","last_unstale":"2023-10-09T06:19:48.531643+0000","last_undegraded":"2023-10-09T06:19:48.531643+0000","last_fullsized":"2023-10-09T06:19:48.531643+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:22.182630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":620,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517734+0000","last_change":"2023-10-09T06:18:33.812523+0000","last_active":"2023-10-09T06:19:48.517734+0000","last_peered":"2023-10-09T06:19:48.517734+0000","last_clean":"2023-10-09T06:19:48.517734+0000","last_became_active":"2023-10-09T06:18:33.812188+0000","last_became_peered":"2023-10-09T06:18:33.812188+0000","last_unstale":"2023-10-09T06:19:48.517734+0000","last_undegraded":"2023-10-09T06:19:48.517734+0000","last_fullsized":"2023-10-09T06:19:48.517734+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:35.343481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":915,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473118+0000","last_change":"2023-10-09T06:18:15.860341+0000","last_active":"2023-10-09T06:19:48.473118+0000","last_peered":"2023-10-09T06:19:48.473118+0000","last_clean":"2023-10-09T06:19:48.473118+0000","last_became_active":"2023-10-09T06:18:15.860008+0000","last_became_peered":"2023-10-09T06:18:15.860008+0000","last_unstale":"2023-10-09T06:19:48.473118+0000","last_undegraded":"2023-10-09T06:19:48.473118+0000","last_fullsized":"2023-10-09T06:19:48.473118+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.181811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.181811+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:37.378828+0000","scrub_duration":0.011553313000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":564,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.531555+0000","last_change":"2023-10-09T06:13:30.791343+0000","last_active":"2023-10-09T06:19:48.531555+0000","last_peered":"2023-10-09T06:19:48.531555+0000","last_clean":"2023-10-09T06:19:48.531555+0000","last_became_active":"2023-10-09T06:13:30.790945+0000","last_became_peered":"2023-10-09T06:13:30.790945+0000","last_unstale":"2023-10-09T06:19:48.531555+0000","last_undegraded":"2023-10-09T06:19:48.531555+0000","last_fullsized":"2023-10-09T06:19:48.531555+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:14.208391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":347,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.506915+0000","last_change":"2023-10-09T06:15:16.803457+0000","last_active":"2023-10-09T06:19:51.506915+0000","last_peered":"2023-10-09T06:19:51.506915+0000","last_clean":"2023-10-09T06:19:51.506915+0000","last_became_active":"2023-10-09T06:15:16.803097+0000","last_became_peered":"2023-10-09T06:15:16.803097+0000","last_unstale":"2023-10-09T06:19:51.506915+0000","last_undegraded":"2023-10-09T06:19:51.506915+0000","last_fullsized":"2023-10-09T06:19:51.506915+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:45.511763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":1019,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473742+0000","last_change":"2023-10-09T06:18:15.867807+0000","last_active":"2023-10-09T06:19:48.473742+0000","last_peered":"2023-10-09T06:19:48.473742+0000","last_clean":"2023-10-09T06:19:48.473742+0000","last_became_active":"2023-10-09T06:18:15.867453+0000","last_became_peered":"2023-10-09T06:18:15.867453+0000","last_unstale":"2023-10-09T06:19:48.473742+0000","last_undegraded":"2023-10-09T06:19:48.473742+0000","last_fullsized":"2023-10-09T06:19:48.473742+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:04.215270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:04.215270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:50:08.865357+0000","scrub_duration":0.002850652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":421,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.518056+0000","last_change":"2023-10-09T06:18:32.874770+0000","last_active":"2023-10-09T06:19:48.518056+0000","last_peered":"2023-10-09T06:19:48.518056+0000","last_clean":"2023-10-09T06:19:48.518056+0000","last_became_active":"2023-10-09T06:18:32.874452+0000","last_became_peered":"2023-10-09T06:18:32.874452+0000","last_unstale":"2023-10-09T06:19:48.518056+0000","last_undegraded":"2023-10-09T06:19:48.518056+0000","last_fullsized":"2023-10-09T06:19:48.518056+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:30.049140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":651,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517931+0000","last_change":"2023-10-09T06:01:54.764989+0000","last_active":"2023-10-09T06:19:48.517931+0000","last_peered":"2023-10-09T06:19:48.517931+0000","last_clean":"2023-10-09T06:19:48.517931+0000","last_became_active":"2023-10-09T06:01:54.764790+0000","last_became_peered":"2023-10-09T06:01:54.764790+0000","last_unstale":"2023-10-09T06:19:48.517931+0000","last_undegraded":"2023-10-09T06:19:48.517931+0000","last_fullsized":"2023-10-09T06:19:48.517931+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:14:07.208858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":467,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.518499+0000","last_change":"2023-10-09T06:18:33.816108+0000","last_active":"2023-10-09T06:19:48.518499+0000","last_peered":"2023-10-09T06:19:48.518499+0000","last_clean":"2023-10-09T06:19:48.518499+0000","last_became_active":"2023-10-09T06:18:33.815292+0000","last_became_peered":"2023-10-09T06:18:33.815292+0000","last_unstale":"2023-10-09T06:19:48.518499+0000","last_undegraded":"2023-10-09T06:19:48.518499+0000","last_fullsized":"2023-10-09T06:19:48.518499+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:53.643181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":648,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517771+0000","last_change":"2023-10-09T06:09:27.565198+0000","last_active":"2023-10-09T06:19:48.517771+0000","last_peered":"2023-10-09T06:19:48.517771+0000","last_clean":"2023-10-09T06:19:48.517771+0000","last_became_active":"2023-10-09T06:09:27.564909+0000","last_became_peered":"2023-10-09T06:09:27.564909+0000","last_unstale":"2023-10-09T06:19:48.517771+0000","last_undegraded":"2023-10-09T06:19:48.517771+0000","last_fullsized":"2023-10-09T06:19:48.517771+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:46.101014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":846,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.519749+0000","last_change":"2023-10-09T06:18:15.861806+0000","last_active":"2023-10-09T06:19:51.519749+0000","last_peered":"2023-10-09T06:19:51.519749+0000","last_clean":"2023-10-09T06:19:51.519749+0000","last_became_active":"2023-10-09T06:18:15.861466+0000","last_became_peered":"2023-10-09T06:18:15.861466+0000","last_unstale":"2023-10-09T06:19:51.519749+0000","last_undegraded":"2023-10-09T06:19:51.519749+0000","last_fullsized":"2023-10-09T06:19:51.519749+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:00.456132+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:00.456132+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:05.355171+0000","scrub_duration":0.0014992199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":469,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473944+0000","last_change":"2023-10-09T06:18:12.816330+0000","last_active":"2023-10-09T06:19:48.473944+0000","last_peered":"2023-10-09T06:19:48.473944+0000","last_clean":"2023-10-09T06:19:48.473944+0000","last_became_active":"2023-10-09T06:18:12.816040+0000","last_became_peered":"2023-10-09T06:18:12.816040+0000","last_unstale":"2023-10-09T06:19:48.473944+0000","last_undegraded":"2023-10-09T06:19:48.473944+0000","last_fullsized":"2023-10-09T06:19:48.473944+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:42.012205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":461,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.473410+0000","last_change":"2023-10-09T06:18:32.816250+0000","last_active":"2023-10-09T06:19:48.473410+0000","last_peered":"2023-10-09T06:19:48.473410+0000","last_clean":"2023-10-09T06:19:48.473410+0000","last_became_active":"2023-10-09T06:18:32.815934+0000","last_became_peered":"2023-10-09T06:18:32.815934+0000","last_unstale":"2023-10-09T06:19:48.473410+0000","last_undegraded":"2023-10-09T06:19:48.473410+0000","last_fullsized":"2023-10-09T06:19:48.473410+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:38.533197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":960,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.506742+0000","last_change":"2023-10-09T06:12:55.753877+0000","last_active":"2023-10-09T06:19:51.506742+0000","last_peered":"2023-10-09T06:19:51.506742+0000","last_clean":"2023-10-09T06:19:51.506742+0000","last_became_active":"2023-10-09T06:12:55.753512+0000","last_became_peered":"2023-10-09T06:12:55.753512+0000","last_unstale":"2023-10-09T06:19:51.506742+0000","last_undegraded":"2023-10-09T06:19:51.506742+0000","last_fullsized":"2023-10-09T06:19:51.506742+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:46.290510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:46.290510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:30.368031+0000","scrub_duration":0.002627819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":690,"reported_epoch":633,"state":"active+clean","last_fresh":"2023-10-09T06:19:51.579784+0000","last_change":"2023-10-09T06:18:33.821539+0000","last_active":"2023-10-09T06:19:51.579784+0000","last_peered":"2023-10-09T06:19:51.579784+0000","last_clean":"2023-10-09T06:19:51.579784+0000","last_became_active":"2023-10-09T06:18:33.821093+0000","last_became_peered":"2023-10-09T06:18:33.821093+0000","last_unstale":"2023-10-09T06:19:51.579784+0000","last_undegraded":"2023-10-09T06:19:51.579784+0000","last_fullsized":"2023-10-09T06:19:51.579784+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:25.387520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":684,"reported_epoch":632,"state":"active+clean","last_fresh":"2023-10-09T06:19:48.517488+0000","last_change":"2023-10-09T06:18:12.815933+0000","last_active":"2023-10-09T06:19:48.517488+0000","last_peered":"2023-10-09T06:19:48.517488+0000","last_clean":"2023-10-09T06:19:48.517488+0000","last_became_active":"2023-10-09T06:18:12.815693+0000","last_became_peered":"2023-10-09T06:18:12.815693+0000","last_unstale":"2023-10-09T06:19:48.517488+0000","last_undegraded":"2023-10-09T06:19:48.517488+0000","last_fullsized":"2023-10-09T06:19:48.517488+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:45.158405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":217824,"num_write_kb":6970305,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16652,"num_bytes_recovered":587333632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":195559424,"data_stored":195559424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90378,"ondisk_log_size":90378,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":89,"num_bytes_recovered":23086,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13410,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234485,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":899400,"kb_used_data":6784,"kb_used_omap":0,"kb_used_meta":892608,"kb_avail":92845752,"statfs":{"total":95995035648,"available":95074050048,"internally_reserved":0,"allocated":6946816,"data_stored":5243267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":914030592},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:19:41 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":4.7359999999999998,"15min":3.8279999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.19500000000000001},"max":{"1min":5.0119999999999996,"5min":166.94,"15min":166.94},"last":0.502},{"interface":"front","average":{"1min":1.8540000000000001,"5min":5.1020000000000003,"15min":3.9900000000000002},"min":{"1min":0.26900000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.2549999999999999,"5min":166.923,"15min":166.923},"last":0.41199999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:19:14 2023","interfaces":[{"interface":"back","average":{"1min":10.066000000000001,"5min":7.399,"15min":5.1390000000000002},"min":{"1min":0.25600000000000001,"5min":0.252,"15min":0.218},"max":{"1min":111.414,"5min":111.414,"15min":124.66200000000001},"last":0.55800000000000005},{"interface":"front","average":{"1min":10.144,"5min":7.1369999999999996,"15min":5.0919999999999996},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.22700000000000001},"max":{"1min":111.767,"5min":111.767,"15min":125.023},"last":0.52400000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:19:41 2023","interfaces":[{"interface":"back","average":{"1min":1.5169999999999999,"5min":1.573,"15min":1.901},"min":{"1min":0.315,"5min":0.29099999999999998,"15min":0.24299999999999999},"max":{"1min":4.9429999999999996,"5min":9.3900000000000006,"15min":47.503},"last":0.442},{"interface":"front","average":{"1min":1.8720000000000001,"5min":1.573,"15min":1.927},"min":{"1min":0.35099999999999998,"5min":0.29899999999999999,"15min":0.27000000000000002},"max":{"1min":9.5600000000000005,"5min":9.5600000000000005,"15min":47.545000000000002},"last":0.59399999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:19:14 2023","interfaces":[{"interface":"back","average":{"1min":12.624000000000001,"5min":5.7549999999999999,"15min":5.4569999999999999},"min":{"1min":0.372,"5min":0.33800000000000002,"15min":0.29899999999999999},"max":{"1min":184.804,"5min":184.804,"15min":184.804},"last":0.64300000000000002},{"interface":"front","average":{"1min":12.515000000000001,"5min":5.1870000000000003,"15min":5.4029999999999996},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32100000000000001},"max":{"1min":186.86099999999999,"5min":186.86099999999999,"15min":186.86099999999999},"last":0.54400000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:19:55 2023","interfaces":[{"interface":"back","average":{"1min":5.859,"5min":2.1829999999999998,"15min":1.4319999999999999},"min":{"1min":0.32200000000000001,"5min":0.26400000000000001,"15min":0.23100000000000001},"max":{"1min":67.186999999999998,"5min":67.186999999999998,"15min":67.186999999999998},"last":0.61399999999999999},{"interface":"front","average":{"1min":4.6849999999999996,"5min":2.2349999999999999,"15min":1.4319999999999999},"min":{"1min":0.307,"5min":0.246,"15min":0.246},"max":{"1min":67.218000000000004,"5min":67.218000000000004,"15min":67.218000000000004},"last":0.72199999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:19:33 2023","interfaces":[{"interface":"back","average":{"1min":1.893,"5min":1.893,"15min":1.893},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":12.951000000000001,"5min":12.951000000000001,"15min":12.951000000000001},"last":0.69099999999999995},{"interface":"front","average":{"1min":1.4790000000000001,"5min":1.4790000000000001,"15min":1.4790000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.0369999999999999,"5min":5.0369999999999999,"15min":5.0369999999999999},"last":0.57599999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:19:01 2023","interfaces":[{"interface":"back","average":{"1min":1.6779999999999999,"5min":3.573,"15min":1.8979999999999999},"min":{"1min":0.311,"5min":0.308,"15min":0.26600000000000001},"max":{"1min":9.2599999999999998,"5min":233.56700000000001,"15min":233.56700000000001},"last":0.63},{"interface":"front","average":{"1min":1.103,"5min":3.1739999999999999,"15min":1.7669999999999999},"min":{"1min":0.33000000000000002,"5min":0.32200000000000001,"15min":0.30599999999999999},"max":{"1min":4.4889999999999999,"5min":232.477,"15min":232.477},"last":0.67500000000000004}]}]},{"osd":6,"up_from":46,"seq":197568496122,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1196852,"kb_used_data":6828,"kb_used_omap":0,"kb_used_meta":1190016,"kb_avail":92548300,"statfs":{"total":95995035648,"available":94769459200,"internally_reserved":0,"allocated":6991872,"data_stored":5300306,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1218576384},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:19:34 2023","interfaces":[{"interface":"back","average":{"1min":5.6710000000000003,"5min":2.907,"15min":4.6849999999999996},"min":{"1min":0.222,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":85.311000000000007,"5min":85.311000000000007,"15min":191.50800000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":5.9569999999999999,"5min":3.4740000000000002,"15min":4.8440000000000003},"min":{"1min":0.27100000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":85.322999999999993,"5min":85.322999999999993,"15min":191.52600000000001},"last":0.57399999999999995}]},{"osd":1,"last update":"Mon Oct 9 06:19:34 2023","interfaces":[{"interface":"back","average":{"1min":3.4089999999999998,"5min":5.6749999999999998,"15min":7.1420000000000003},"min":{"1min":0.378,"5min":0.23799999999999999,"15min":0.23200000000000001},"max":{"1min":15.815,"5min":183.458,"15min":328.04399999999998},"last":0.504},{"interface":"front","average":{"1min":2.6930000000000001,"5min":4.8490000000000002,"15min":6.8449999999999998},"min":{"1min":0.30499999999999999,"5min":0.253,"15min":0.253},"max":{"1min":16.132999999999999,"5min":183.51499999999999,"15min":328.10000000000002},"last":0.39900000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:19:34 2023","interfaces":[{"interface":"back","average":{"1min":1.6359999999999999,"5min":1.399,"15min":2.758},"min":{"1min":0.35099999999999998,"5min":0.28100000000000003,"15min":0.26100000000000001},"max":{"1min":5.4950000000000001,"5min":9.1500000000000004,"15min":163.04900000000001},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.885,"5min":1.361,"15min":2.7210000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":12.083,"5min":12.083,"15min":162.93000000000001},"last":0.45600000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:19:23 2023","interfaces":[{"interface":"back","average":{"1min":1.5860000000000001,"5min":5.8730000000000002,"15min":7.3769999999999998},"min":{"1min":0.314,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":5.5209999999999999,"5min":208.494,"15min":250.744},"last":0.41899999999999998},{"interface":"front","average":{"1min":2.2120000000000002,"5min":6.1639999999999997,"15min":7.2430000000000003},"min":{"1min":0.39100000000000001,"5min":0.309,"15min":0.309},"max":{"1min":12.536,"5min":211.661,"15min":250.83799999999999},"last":0.59099999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:19:58 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.76600000000000001,"15min":1.5209999999999999},"min":{"1min":0.26800000000000002,"5min":0.251,"15min":0.24399999999999999},"max":{"1min":5.8789999999999996,"5min":6.0979999999999999,"15min":151.35599999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":1.823,"5min":1.518,"15min":1.9239999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.23999999999999999},"max":{"1min":14.009,"5min":14.009,"15min":149.15799999999999},"last":0.61499999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:19:34 2023","interfaces":[{"interface":"back","average":{"1min":1.794,"5min":1.794,"15min":1.794},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":16.032,"5min":16.032,"15min":16.032},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.1040000000000001,"15min":1.1040000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":5.8090000000000002,"5min":5.8090000000000002,"15min":5.8090000000000002},"last":0.627}]},{"osd":7,"last update":"Mon Oct 9 06:19:58 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":1.4810000000000001,"15min":1.3460000000000001},"min":{"1min":0.32700000000000001,"5min":0.32400000000000001,"15min":0.30299999999999999},"max":{"1min":17.803999999999998,"5min":17.803999999999998,"15min":21.98},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.698,"5min":1.2909999999999999,"15min":1.2370000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33300000000000002},"max":{"1min":16.119,"5min":16.119,"15min":18.241},"last":0.56100000000000005}]}]},{"osd":1,"up_from":89,"seq":382252089710,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":837432,"kb_used_data":8816,"kb_used_omap":9,"kb_used_meta":828598,"kb_avail":92907720,"statfs":{"total":95995035648,"available":95137505280,"internally_reserved":0,"allocated":9027584,"data_stored":5464731,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":848485353},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:19:55 2023","interfaces":[{"interface":"back","average":{"1min":1.528,"5min":3.3839999999999999,"15min":2.6640000000000001},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":4.8230000000000004,"5min":118.17,"15min":118.17},"last":0.72599999999999998},{"interface":"front","average":{"1min":1.9810000000000001,"5min":3.5019999999999998,"15min":2.8799999999999999},"min":{"1min":0.24099999999999999,"5min":0.20699999999999999,"15min":0.20200000000000001},"max":{"1min":5.2590000000000003,"5min":117.634,"15min":117.634},"last":0.83399999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:19:55 2023","interfaces":[{"interface":"back","average":{"1min":1.909,"5min":1.575,"15min":1.7290000000000001},"min":{"1min":0.247,"5min":0.245,"15min":0.21099999999999999},"max":{"1min":7.3239999999999998,"5min":9.7200000000000006,"15min":17.681000000000001},"last":1.0429999999999999},{"interface":"front","average":{"1min":1.907,"5min":1.286,"15min":2.0179999999999998},"min":{"1min":0.27400000000000002,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":4.8380000000000001,"5min":6.6799999999999997,"15min":55.694000000000003},"last":0.67500000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:19:23 2023","interfaces":[{"interface":"back","average":{"1min":5.9889999999999999,"5min":5.8380000000000001,"15min":4.9489999999999998},"min":{"1min":0.317,"5min":0.30399999999999999,"15min":0.24199999999999999},"max":{"1min":72.266000000000005,"5min":95.489999999999995,"15min":142.97},"last":0.77300000000000002},{"interface":"front","average":{"1min":5.9560000000000004,"5min":5.4770000000000003,"15min":4.8239999999999998},"min":{"1min":0.28199999999999997,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":72.311000000000007,"5min":95.436999999999998,"15min":142.94800000000001},"last":0.71599999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:19:55 2023","interfaces":[{"interface":"back","average":{"1min":9.6890000000000001,"5min":5.3449999999999998,"15min":3.7080000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.27700000000000002},"max":{"1min":190.523,"5min":190.523,"15min":190.523},"last":0.79200000000000004},{"interface":"front","average":{"1min":10.565,"5min":5.6239999999999997,"15min":4.1619999999999999},"min":{"1min":0.371,"5min":0.34399999999999997,"15min":0.308},"max":{"1min":190.774,"5min":190.774,"15min":190.774},"last":2.0299999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:19:34 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":1.48,"15min":1.48},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":4.819,"5min":4.819,"15min":4.819},"last":1.9359999999999999},{"interface":"front","average":{"1min":1.3220000000000001,"5min":1.3220000000000001,"15min":1.3220000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.9769999999999999,"5min":3.9769999999999999,"15min":3.9769999999999999},"last":0.92400000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:19:06 2023","interfaces":[{"interface":"back","average":{"1min":6.2679999999999998,"5min":2.4039999999999999,"15min":2.1989999999999998},"min":{"1min":0.42899999999999999,"5min":0.34599999999999997,"15min":0.33700000000000002},"max":{"1min":98.433000000000007,"5min":98.433000000000007,"15min":98.433000000000007},"last":0.88},{"interface":"front","average":{"1min":6.4809999999999999,"5min":2.6269999999999998,"15min":2.677},"min":{"1min":0.377,"5min":0.36199999999999999,"15min":0.34999999999999998},"max":{"1min":98.456000000000003,"5min":98.456000000000003,"15min":98.456000000000003},"last":1.976}]},{"osd":7,"last update":"Mon Oct 9 06:19:06 2023","interfaces":[{"interface":"back","average":{"1min":15.449,"5min":4.5380000000000003,"15min":3.5910000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":154.34800000000001,"5min":154.34800000000001,"15min":154.34800000000001},"last":2.0089999999999999},{"interface":"front","average":{"1min":15.24,"5min":4.2830000000000004,"15min":3.0539999999999998},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.38},"max":{"1min":152.20599999999999,"5min":152.20599999999999,"15min":152.20599999999999},"last":1.0900000000000001}]}]},{"osd":0,"up_from":456,"seq":1958505087104,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":493064,"kb_used_data":64448,"kb_used_omap":32,"kb_used_meta":428575,"kb_avail":93252088,"statfs":{"total":95995035648,"available":95490138112,"internally_reserved":0,"allocated":65994752,"data_stored":62650906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32794,"internal_metadata":438861798},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:19:49 2023","interfaces":[{"interface":"back","average":{"1min":4.9800000000000004,"5min":4.8120000000000003,"15min":7.5129999999999999},"min":{"1min":0.28399999999999997,"5min":0.189,"15min":0.17299999999999999},"max":{"1min":43.316000000000003,"5min":137.452,"15min":166.833},"last":1.3600000000000001},{"interface":"front","average":{"1min":4.5679999999999996,"5min":4.7140000000000004,"15min":7.2300000000000004},"min":{"1min":0.254,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":43.401000000000003,"5min":137.10499999999999,"15min":166.81100000000001},"last":1.427}]},{"osd":2,"last update":"Mon Oct 9 06:19:49 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":1.6950000000000001,"15min":2.8010000000000002},"min":{"1min":0.246,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":13.071,"5min":13.071,"15min":21.648},"last":0.50600000000000001},{"interface":"front","average":{"1min":1.732,"5min":1.5009999999999999,"15min":2.5750000000000002},"min":{"1min":0.247,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.173,"5min":10.154999999999999,"15min":20.152000000000001},"last":0.52700000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:19:46 2023","interfaces":[{"interface":"back","average":{"1min":16.856999999999999,"5min":5.1980000000000004,"15min":8.7080000000000002},"min":{"1min":0.41899999999999998,"5min":0.214,"15min":0.214},"max":{"1min":138.52699999999999,"5min":138.52699999999999,"15min":203.59299999999999},"last":0.72499999999999998},{"interface":"front","average":{"1min":16.591000000000001,"5min":5.3789999999999996,"15min":8.5670000000000002},"min":{"1min":0.39300000000000002,"5min":0.29499999999999998,"15min":0.23899999999999999},"max":{"1min":138.48500000000001,"5min":138.48500000000001,"15min":202.96600000000001},"last":0.67000000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:19:49 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":1.9410000000000001,"15min":2.2400000000000002},"min":{"1min":0.38800000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.5840000000000001,"5min":60.587000000000003,"15min":60.587000000000003},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.4410000000000001,"5min":2.0569999999999999,"15min":2.278},"min":{"1min":0.32400000000000001,"5min":0.254,"15min":0.254},"max":{"1min":4.234,"5min":60.567,"15min":60.567},"last":0.41599999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:19:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":2.1019999999999999,"15min":2.1019999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":8.048,"5min":8.048,"15min":8.048},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.857,"5min":1.857,"15min":1.857},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":13.007,"5min":13.007,"15min":13.007},"last":0.54100000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:19:49 2023","interfaces":[{"interface":"back","average":{"1min":2.3719999999999999,"5min":1.827,"15min":2.286},"min":{"1min":0.39200000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":13.029999999999999,"5min":13.029999999999999,"15min":31.648},"last":1.349},{"interface":"front","average":{"1min":1.704,"5min":1.3779999999999999,"15min":2.0990000000000002},"min":{"1min":0.47699999999999998,"5min":0.36399999999999999,"15min":0.36299999999999999},"max":{"1min":7.5640000000000001,"5min":10.051,"15min":24.43},"last":0.68200000000000005}]},{"osd":7,"last update":"Mon Oct 9 06:19:49 2023","interfaces":[{"interface":"back","average":{"1min":2.3759999999999999,"5min":5.7110000000000003,"15min":3.9380000000000002},"min":{"1min":0.40899999999999997,"5min":0.376,"15min":0.376},"max":{"1min":12.957000000000001,"5min":248.99100000000001,"15min":248.99100000000001},"last":0.64000000000000001},{"interface":"front","average":{"1min":2.0419999999999998,"5min":5.5190000000000001,"15min":3.8079999999999998},"min":{"1min":0.505,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.5279999999999996,"5min":249.00800000000001,"15min":249.00800000000001},"last":0.63100000000000001}]}]},{"osd":2,"up_from":0,"seq":403726926201,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":99,"seq":425201762674,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1268644,"kb_used_data":144028,"kb_used_omap":7,"kb_used_meta":1124600,"kb_avail":92476508,"statfs":{"total":95995035648,"available":94695944192,"internally_reserved":0,"allocated":147484672,"data_stored":143929559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1151591016},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:19:40 2023","interfaces":[{"interface":"back","average":{"1min":2.5510000000000002,"5min":3.3460000000000001,"15min":3.7149999999999999},"min":{"1min":0.23100000000000001,"5min":0.182,"15min":0.17299999999999999},"max":{"1min":11.308999999999999,"5min":117.473,"15min":220.45400000000001},"last":0.27100000000000002},{"interface":"front","average":{"1min":1.2769999999999999,"5min":3.7599999999999998,"15min":4.5949999999999998},"min":{"1min":0.219,"5min":0.219,"15min":0.19600000000000001},"max":{"1min":4.944,"5min":118.873,"15min":216.44200000000001},"last":0.309}]},{"osd":1,"last update":"Mon Oct 9 06:19:18 2023","interfaces":[{"interface":"back","average":{"1min":1.982,"5min":3.6629999999999998,"15min":3.516},"min":{"1min":0.29899999999999999,"5min":0.20300000000000001,"15min":0.19800000000000001},"max":{"1min":6.9950000000000001,"5min":167.768,"15min":207.75299999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.4039999999999999,"5min":4.2069999999999999,"15min":4.29},"min":{"1min":0.28499999999999998,"5min":0.218,"15min":0.218},"max":{"1min":8.9450000000000003,"5min":172.143,"15min":207.12200000000001},"last":0.29399999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:19:18 2023","interfaces":[{"interface":"back","average":{"1min":1.746,"5min":1.5349999999999999,"15min":1.9279999999999999},"min":{"1min":0.35599999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.5940000000000003,"5min":6.5940000000000003,"15min":30.547999999999998},"last":0.433},{"interface":"front","average":{"1min":2.3079999999999998,"5min":2.2120000000000002,"15min":2.589},"min":{"1min":0.377,"5min":0.29699999999999999,"15min":0.23899999999999999},"max":{"1min":8.9949999999999992,"5min":11.542,"15min":25.984000000000002},"last":0.38600000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:19:02 2023","interfaces":[{"interface":"back","average":{"1min":7.7850000000000001,"5min":3.1899999999999999,"15min":2.6579999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":84.369,"5min":84.369,"15min":95.739000000000004},"last":0.57399999999999995},{"interface":"front","average":{"1min":8.173,"5min":3.5950000000000002,"15min":3.0840000000000001},"min":{"1min":0.34899999999999998,"5min":0.32600000000000001,"15min":0.28899999999999998},"max":{"1min":87.176000000000002,"5min":87.176000000000002,"15min":97.941999999999993},"last":0.45700000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:19:32 2023","interfaces":[{"interface":"back","average":{"1min":1.8100000000000001,"5min":1.8100000000000001,"15min":1.8100000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":7.8550000000000004,"5min":7.8550000000000004,"15min":7.8550000000000004},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.911,"5min":1.911,"15min":1.911},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":6.4829999999999997,"5min":6.4829999999999997,"15min":6.4829999999999997},"last":0.54300000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:19:18 2023","interfaces":[{"interface":"back","average":{"1min":1.897,"5min":2.238,"15min":3},"min":{"1min":0.42699999999999999,"5min":0.32600000000000001,"15min":0.26100000000000001},"max":{"1min":5.6619999999999999,"5min":49.526000000000003,"15min":233.26599999999999},"last":0.44700000000000001},{"interface":"front","average":{"1min":2.573,"5min":2.4969999999999999,"15min":3.4510000000000001},"min":{"1min":0.44600000000000001,"5min":0.40000000000000002,"15min":0.33600000000000002},"max":{"1min":8.5150000000000006,"5min":49.429000000000002,"15min":233.24100000000001},"last":0.50800000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:19:18 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":2.726,"15min":3.899},"min":{"1min":0.48599999999999999,"5min":0.36599999999999999,"15min":0.35699999999999998},"max":{"1min":6.8449999999999998,"5min":77.525000000000006,"15min":230.69399999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.7290000000000001,"5min":3.407,"15min":4.3419999999999996},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.32600000000000001},"max":{"1min":5.0810000000000004,"5min":77.438000000000002,"15min":230.67500000000001},"last":0.495}]}]},{"osd":4,"up_from":322,"seq":1382979469537,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":921276,"kb_used_data":6516,"kb_used_omap":35,"kb_used_meta":914716,"kb_avail":92823876,"statfs":{"total":95995035648,"available":95051649024,"internally_reserved":0,"allocated":6672384,"data_stored":4972702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36552,"internal_metadata":936669496},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:19:38 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":2.234,"15min":1.8979999999999999},"min":{"1min":0.23100000000000001,"5min":0.22700000000000001,"15min":0.216},"max":{"1min":10.574,"5min":81.066000000000003,"15min":81.066000000000003},"last":5.5359999999999996},{"interface":"front","average":{"1min":2.5169999999999999,"5min":2.5529999999999999,"15min":1.97},"min":{"1min":0.313,"5min":0.27800000000000002,"15min":0.22800000000000001},"max":{"1min":10.695,"5min":81.603999999999999,"15min":81.603999999999999},"last":0.39600000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:18:58 2023","interfaces":[{"interface":"back","average":{"1min":10.872,"5min":5.734,"15min":3.6320000000000001},"min":{"1min":0.29999999999999999,"5min":0.24099999999999999,"15min":0.20999999999999999},"max":{"1min":160.35400000000001,"5min":160.35400000000001,"15min":160.35400000000001},"last":0.93799999999999994},{"interface":"front","average":{"1min":11.196,"5min":5.9329999999999998,"15min":3.7440000000000002},"min":{"1min":0.26700000000000002,"5min":0.26500000000000001,"15min":0.22800000000000001},"max":{"1min":160.333,"5min":160.333,"15min":160.333},"last":0.33100000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:18:58 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":1.726,"15min":2.6299999999999999},"min":{"1min":0.316,"5min":0.311,"15min":0.27200000000000002},"max":{"1min":7.5570000000000004,"5min":10.084,"15min":245.166},"last":0.378},{"interface":"front","average":{"1min":4.3780000000000001,"5min":2.3599999999999999,"15min":2.6909999999999998},"min":{"1min":0.33600000000000002,"5min":0.28699999999999998,"15min":0.27900000000000003},"max":{"1min":52.920999999999999,"5min":52.920999999999999,"15min":245.119},"last":0.79900000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:18:58 2023","interfaces":[{"interface":"back","average":{"1min":12.802,"5min":7.9720000000000004,"15min":5.8979999999999997},"min":{"1min":0.33100000000000002,"5min":0.29299999999999998,"15min":0.26800000000000002},"max":{"1min":164.89699999999999,"5min":197.31899999999999,"15min":246.39400000000001},"last":0.67300000000000004},{"interface":"front","average":{"1min":12.507999999999999,"5min":7.4710000000000001,"15min":5.3840000000000003},"min":{"1min":0.47099999999999997,"5min":0.32300000000000001,"15min":0.314},"max":{"1min":164.88300000000001,"5min":198.565,"15min":246.99000000000001},"last":0.40999999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:19:33 2023","interfaces":[{"interface":"back","average":{"1min":2.04,"5min":2.04,"15min":2.04},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":10.659000000000001,"5min":10.659000000000001,"15min":10.659000000000001},"last":0.36199999999999999},{"interface":"front","average":{"1min":1.96,"5min":1.96,"15min":1.96},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":11.779,"5min":11.779,"15min":11.779},"last":0.51300000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:18:58 2023","interfaces":[{"interface":"back","average":{"1min":2.3279999999999998,"5min":1.347,"15min":1.569},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26400000000000001},"max":{"1min":23.712,"5min":23.712,"15min":95.492999999999995},"last":0.61899999999999999},{"interface":"front","average":{"1min":2.379,"5min":1.546,"15min":1.71},"min":{"1min":0.312,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":18.021000000000001,"5min":18.021000000000001,"15min":95.664000000000001},"last":1.2090000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:18:58 2023","interfaces":[{"interface":"back","average":{"1min":4.7690000000000001,"5min":3.1080000000000001,"15min":1.9490000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":61.811999999999998,"5min":95.167000000000002,"15min":95.167000000000002},"last":0.47999999999999998},{"interface":"front","average":{"1min":4.2990000000000004,"5min":2.9710000000000001,"15min":1.841},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.251},"max":{"1min":61.759,"5min":94.811999999999998,"15min":94.811999999999998},"last":0.438}]}]},{"osd":5,"up_from":611,"seq":2624225017875,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":367984,"kb_used_data":6760,"kb_used_omap":98,"kb_used_meta":361117,"kb_avail":93377168,"statfs":{"total":95995035648,"available":95618220032,"internally_reserved":0,"allocated":6922240,"data_stored":5231134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":100797,"internal_metadata":369784387},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:19:33 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":1.3129999999999999,"15min":1.3129999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":8.3260000000000005,"5min":8.3260000000000005,"15min":8.3260000000000005},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.681,"5min":1.681,"15min":1.681},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":7.577,"5min":7.577,"15min":7.577},"last":0.623}]},{"osd":1,"last update":"Mon Oct 9 06:19:33 2023","interfaces":[{"interface":"back","average":{"1min":4.9790000000000001,"5min":4.9790000000000001,"15min":4.9790000000000001},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":50.430999999999997,"5min":50.430999999999997,"15min":50.430999999999997},"last":0.46400000000000002},{"interface":"front","average":{"1min":3.6280000000000001,"5min":3.6280000000000001,"15min":3.6280000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":50.445,"5min":50.445,"15min":50.445},"last":0.56999999999999995}]},{"osd":2,"last update":"Mon Oct 9 06:19:33 2023","interfaces":[{"interface":"back","average":{"1min":1.371,"5min":1.371,"15min":1.371},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.9980000000000002,"5min":5.9980000000000002,"15min":5.9980000000000002},"last":0.63600000000000001},{"interface":"front","average":{"1min":1.4610000000000001,"5min":1.4610000000000001,"15min":1.4610000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":5.9560000000000004,"5min":5.9560000000000004,"15min":5.9560000000000004},"last":0.502}]},{"osd":3,"last update":"Mon Oct 9 06:19:33 2023","interfaces":[{"interface":"back","average":{"1min":9.9749999999999996,"5min":9.9749999999999996,"15min":9.9749999999999996},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":120.357,"5min":120.357,"15min":120.357},"last":0.51400000000000001},{"interface":"front","average":{"1min":9.4619999999999997,"5min":9.4619999999999997,"15min":9.4619999999999997},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":120.759,"5min":120.759,"15min":120.759},"last":0.52900000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:19:33 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":1.012,"15min":1.012},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.6890000000000001,"5min":5.6890000000000001,"15min":5.6890000000000001},"last":0.439},{"interface":"front","average":{"1min":1.7350000000000001,"5min":1.7350000000000001,"15min":1.7350000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.6130000000000004,"5min":7.6130000000000004,"15min":7.6130000000000004},"last":0.55100000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:19:33 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":5.9400000000000004,"5min":5.9400000000000004,"15min":5.9400000000000004},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.997,"5min":0.997,"15min":0.997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.8120000000000003,"5min":5.8120000000000003,"15min":5.8120000000000003},"last":0.58099999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:19:33 2023","interfaces":[{"interface":"back","average":{"1min":1.55,"5min":1.55,"15min":1.55},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":7.6120000000000001,"5min":7.6120000000000001,"15min":7.6120000000000001},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.2729999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":5.8920000000000003,"5min":5.8920000000000003,"15min":5.8920000000000003},"last":0.60499999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4889,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57147392,"data_stored":57147392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":138412032,"data_stored":138412032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:20:07.252 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T06:20:07.253 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:20:07.253 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [0, 3, 4, 5, 7, 1, 6] 2023-10-09T06:20:07.253 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd out 4 2023-10-09T06:20:07.254 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:07.284 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:07.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:07.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:07.391 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:07.407 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:07.421 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:07.488 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:07.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:07.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:07.592 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:07.624 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:07.674 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:07.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:07.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:07.779 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:07.808 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:07.852 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:07.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:07.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:07.958 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:07.988 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:08.059 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:20:08.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:07 smithi156 ceph-mon[195101]: from='client.36900 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:20:08.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:07 smithi064 ceph-mon[212750]: from='client.36900 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:20:08.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:07 smithi064 ceph-mon[220044]: from='client.36900 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:20:08.488 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:08.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:08.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:08.591 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:08.624 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:08.966 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:08.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:08.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:09.070 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:09.102 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:09.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:08 smithi156 ceph-mon[195101]: pgmap v1641: 119 pgs: 119 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 322 op/s 2023-10-09T06:20:09.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:08 smithi064 ceph-mon[212750]: pgmap v1641: 119 pgs: 119 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 322 op/s 2023-10-09T06:20:09.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:08 smithi064 ceph-mon[220044]: pgmap v1641: 119 pgs: 119 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 322 op/s 2023-10-09T06:20:09.278 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:20:09.376 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:09.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:09.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:09.394 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:09.397 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:09.482 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:09.514 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:09.563 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:09.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:09.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:09.670 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:09.698 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:09.742 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:09.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:09.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:09.781 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:20:09.781 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:20:09.849 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:09.875 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:09.917 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_5' removed 2023-10-09T06:20:09.938 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:09.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:09.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:10.045 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:10.098 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:09 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3440411337' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:20:10.098 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:09 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:20:10.098 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:09 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3440411337' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:20:10.098 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:09 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:20:10.101 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:10.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:09 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3440411337' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:20:10.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:09 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:20:10.933 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:20:11.105 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:11.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:11.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:11.132 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:10 smithi064 ceph-mon[212750]: pgmap v1642: 119 pgs: 119 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 322 op/s 2023-10-09T06:20:11.132 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:10 smithi064 ceph-mon[212750]: failed to encode map e635 with expected crc 2023-10-09T06:20:11.132 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:10 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:20:11.132 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:10 smithi064 ceph-mon[212750]: osdmap e635: 8 total, 8 up, 7 in 2023-10-09T06:20:11.132 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:10 smithi064 ceph-mon[220044]: pgmap v1642: 119 pgs: 119 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 322 op/s 2023-10-09T06:20:11.132 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:10 smithi064 ceph-mon[220044]: failed to encode map e635 with expected crc 2023-10-09T06:20:11.132 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:10 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:20:11.133 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:10 smithi064 ceph-mon[220044]: osdmap e635: 8 total, 8 up, 7 in 2023-10-09T06:20:11.155 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T06:20:11.155 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-10-09T06:20:11.155 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T06:20:11.156 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-10-09T06:20:11.156 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T06:20:11.156 DEBUG:teuthology.orchestra.run.smithi064:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:20:11.209 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:11.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:11.238 INFO:teuthology.orchestra.run.smithi064.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T06:20:11.239 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:20:11.239 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:10 smithi156 ceph-mon[195101]: pgmap v1642: 119 pgs: 119 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 322 op/s 2023-10-09T06:20:11.239 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:10 smithi156 ceph-mon[195101]: failed to encode map e635 with expected crc 2023-10-09T06:20:11.240 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:10 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:20:11.240 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:10 smithi156 ceph-mon[195101]: osdmap e635: 8 total, 8 up, 7 in 2023-10-09T06:20:11.241 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:11.289 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:11.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:11.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:11.397 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:11.425 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:11.442 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T06:20:11.442 DEBUG:teuthology.orchestra.run.smithi064:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T06:20:11.459 INFO:tasks.workunit:timeout=3h 2023-10-09T06:20:11.459 INFO:tasks.workunit:cleanup=True 2023-10-09T06:20:11.460 DEBUG:teuthology.orchestra.run.smithi064:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T06:20:11.470 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:11.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:11.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:11.577 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:11.603 INFO:tasks.workunit.client.0.smithi064.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T06:20:11.606 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:11.661 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:12.456 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:12.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:12.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:12.542 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:12.562 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:12.599 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:12.664 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:12.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:12.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:12.768 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:12.797 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:12.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:12 smithi064 ceph-mon[212750]: pgmap v1644: 103 pgs: 103 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-10-09T06:20:12.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:12 smithi064 ceph-mon[220044]: pgmap v1644: 103 pgs: 103 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-10-09T06:20:13.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:12 smithi156 ceph-mon[195101]: pgmap v1644: 103 pgs: 103 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-10-09T06:20:13.449 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:13.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:13.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:13.554 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:13.586 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:13.634 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:13.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:13.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:13.742 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:13.770 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:13.814 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:13.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:13.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:13.922 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:13.948 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:13.991 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:13.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:13.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:14.034 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:13 smithi064 conmon[212726]: 2023-10-09T06:20:13.958+0000 7f8823f23700 -1 mon.a@0(leader).osd e635 definitely_dead 0 2023-10-09T06:20:14.100 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:14.297 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:14.647 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:14.650 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:14 smithi064 ceph-mon[212750]: pgmap v1645: 103 pgs: 103 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-10-09T06:20:14.650 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:14.650 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:14.650 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:14 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2932986100' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T06:20:14.650 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:14 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T06:20:14.650 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:14 smithi064 ceph-mon[212750]: Client client.admin marked osd.4 out, while it was still marked up 2023-10-09T06:20:14.650 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:14 smithi064 ceph-mon[220044]: pgmap v1645: 103 pgs: 103 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-10-09T06:20:14.651 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:14.651 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:14.651 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:14 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2932986100' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T06:20:14.651 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:14 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T06:20:14.651 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:14 smithi064 ceph-mon[220044]: Client client.admin marked osd.4 out, while it was still marked up 2023-10-09T06:20:14.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:14.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:14.752 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:14.784 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:14.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:14 smithi156 ceph-mon[195101]: pgmap v1645: 103 pgs: 103 active+clean; 582 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail 2023-10-09T06:20:14.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:14.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:14.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:14 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2932986100' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T06:20:14.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:14 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T06:20:14.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:14 smithi156 ceph-mon[195101]: Client client.admin marked osd.4 out, while it was still marked up 2023-10-09T06:20:14.908 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:20:14.909 INFO:teuthology.orchestra.run.smithi064.stderr:marked out osd.4. 2023-10-09T06:20:15.009 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:15.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:15.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:15.117 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:15.148 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:15.159 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:20:15.192 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:20:15.210 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:20:15.367 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:20:15.367 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:20:15.367 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:20:15.367 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:20:15.367 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:20:15.367 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:20:15.367 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:20:15.367 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:20:15.368 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:20:15.368 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:20:15.368 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:20:15.368 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:20:15.368 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:20:15.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:20:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:20:15] "GET /metrics HTTP/1.1" 200 34042 "" "Prometheus/2.43.0" 2023-10-09T06:20:15.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:15 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1062073369' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:15.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:15 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:15.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:15 smithi064 ceph-mon[220044]: failed to encode map e636 with expected crc 2023-10-09T06:20:15.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:15 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T06:20:15.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:15 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1062073369' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:20:15.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:15 smithi064 ceph-mon[220044]: osdmap e636: 8 total, 8 up, 6 in 2023-10-09T06:20:15.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:15 smithi064 ceph-mon[220044]: failed to encode map e636 with expected crc 2023-10-09T06:20:15.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:15 smithi064 ceph-mon[220044]: failed to encode map e636 with expected crc 2023-10-09T06:20:15.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:15 smithi064 ceph-mon[220044]: failed to encode map e636 with expected crc 2023-10-09T06:20:15.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:15 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/670307889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:15 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1062073369' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:15 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:15.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:15 smithi064 ceph-mon[212750]: failed to encode map e636 with expected crc 2023-10-09T06:20:15.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:15 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T06:20:15.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:15 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1062073369' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:20:15.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:15 smithi064 ceph-mon[212750]: osdmap e636: 8 total, 8 up, 6 in 2023-10-09T06:20:15.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:15 smithi064 ceph-mon[212750]: failed to encode map e636 with expected crc 2023-10-09T06:20:15.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:15 smithi064 ceph-mon[212750]: failed to encode map e636 with expected crc 2023-10-09T06:20:15.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:15 smithi064 ceph-mon[212750]: failed to encode map e636 with expected crc 2023-10-09T06:20:15.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:15 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/670307889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:15.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:15 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1062073369' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:15.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:15 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:15.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:15 smithi156 ceph-mon[195101]: failed to encode map e636 with expected crc 2023-10-09T06:20:15.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:15 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T06:20:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:15 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1062073369' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:20:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:15 smithi156 ceph-mon[195101]: osdmap e636: 8 total, 8 up, 6 in 2023-10-09T06:20:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:15 smithi156 ceph-mon[195101]: failed to encode map e636 with expected crc 2023-10-09T06:20:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:15 smithi156 ceph-mon[195101]: failed to encode map e636 with expected crc 2023-10-09T06:20:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:15 smithi156 ceph-mon[195101]: failed to encode map e636 with expected crc 2023-10-09T06:20:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:15 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/670307889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:16.257 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:16.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:16.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:16.361 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:16.389 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:16.431 DEBUG:teuthology.orchestra.run.smithi064:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T06:20:16.552 INFO:tasks.workunit.client.0.smithi064.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T06:20:16.554 INFO:tasks.workunit.client.0.smithi064.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T06:20:16.554 INFO:tasks.workunit.client.0.smithi064.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T06:20:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:16 smithi064 ceph-mon[212750]: pgmap v1647: 103 pgs: 18 peering, 85 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:20:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:16 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 9 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2023-10-09T06:20:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:16 smithi064 ceph-mon[212750]: failed to encode map e637 with expected crc 2023-10-09T06:20:16.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:16 smithi064 ceph-mon[212750]: osdmap e637: 8 total, 8 up, 6 in 2023-10-09T06:20:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:16 smithi064 ceph-mon[212750]: failed to encode map e637 with expected crc 2023-10-09T06:20:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:16 smithi064 ceph-mon[212750]: failed to encode map e637 with expected crc 2023-10-09T06:20:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:16 smithi064 ceph-mon[212750]: failed to encode map e637 with expected crc 2023-10-09T06:20:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:16 smithi064 ceph-mon[220044]: pgmap v1647: 103 pgs: 18 peering, 85 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:20:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:16 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 9 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2023-10-09T06:20:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:16 smithi064 ceph-mon[220044]: failed to encode map e637 with expected crc 2023-10-09T06:20:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:16 smithi064 ceph-mon[220044]: osdmap e637: 8 total, 8 up, 6 in 2023-10-09T06:20:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:16 smithi064 ceph-mon[220044]: failed to encode map e637 with expected crc 2023-10-09T06:20:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:16 smithi064 ceph-mon[220044]: failed to encode map e637 with expected crc 2023-10-09T06:20:16.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:16 smithi064 ceph-mon[220044]: failed to encode map e637 with expected crc 2023-10-09T06:20:16.749 INFO:tasks.workunit.client.0.smithi064.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T06:20:16.801 INFO:tasks.workunit.client.0.smithi064.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T06:20:16.842 INFO:tasks.workunit.client.0.smithi064.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T06:20:16.843 INFO:tasks.workunit.client.0.smithi064.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T06:20:16.843 INFO:tasks.workunit.client.0.smithi064.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T06:20:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:16 smithi156 ceph-mon[195101]: pgmap v1647: 103 pgs: 18 peering, 85 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:20:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:16 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 9 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2023-10-09T06:20:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:16 smithi156 ceph-mon[195101]: failed to encode map e637 with expected crc 2023-10-09T06:20:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:16 smithi156 ceph-mon[195101]: osdmap e637: 8 total, 8 up, 6 in 2023-10-09T06:20:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:16 smithi156 ceph-mon[195101]: failed to encode map e637 with expected crc 2023-10-09T06:20:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:16 smithi156 ceph-mon[195101]: failed to encode map e637 with expected crc 2023-10-09T06:20:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:16 smithi156 ceph-mon[195101]: failed to encode map e637 with expected crc 2023-10-09T06:20:16.947 INFO:tasks.workunit.client.0.smithi064.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T06:20:17.063 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-10-09T06:20:17.064 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T06:20:17.623 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-10-09T06:20:17.624 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-10-09T06:20:17.624 DEBUG:teuthology.orchestra.run.smithi064:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-10-09T06:20:17.857 INFO:tasks.workunit.client.0.smithi064.stdout:Running main() from gmock_main.cc 2023-10-09T06:20:17.857 INFO:tasks.workunit.client.0.smithi064.stdout:[==========] Running 70 tests from 1 test suite. 2023-10-09T06:20:17.857 INFO:tasks.workunit.client.0.smithi064.stdout:[----------] Global test environment set-up. 2023-10-09T06:20:17.857 INFO:tasks.workunit.client.0.smithi064.stdout:[----------] 70 tests from TestClsRbd 2023-10-09T06:20:18.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:17 smithi064 ceph-mon[212750]: failed to encode map e638 with expected crc 2023-10-09T06:20:18.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:17 smithi064 ceph-mon[212750]: osdmap e638: 8 total, 8 up, 6 in 2023-10-09T06:20:18.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:17 smithi064 ceph-mon[212750]: failed to encode map e638 with expected crc 2023-10-09T06:20:18.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:17 smithi064 ceph-mon[212750]: failed to encode map e638 with expected crc 2023-10-09T06:20:18.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:17 smithi064 ceph-mon[212750]: failed to encode map e638 with expected crc 2023-10-09T06:20:18.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:17 smithi064 ceph-mon[212750]: failed to encode map e638 with expected crc 2023-10-09T06:20:18.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:17 smithi064 ceph-mon[212750]: failed to encode map e638 with expected crc 2023-10-09T06:20:18.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:17 smithi064 ceph-mon[212750]: osdmap e639: 8 total, 8 up, 6 in 2023-10-09T06:20:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:17 smithi064 ceph-mon[220044]: failed to encode map e638 with expected crc 2023-10-09T06:20:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:17 smithi064 ceph-mon[220044]: osdmap e638: 8 total, 8 up, 6 in 2023-10-09T06:20:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:17 smithi064 ceph-mon[220044]: failed to encode map e638 with expected crc 2023-10-09T06:20:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:17 smithi064 ceph-mon[220044]: failed to encode map e638 with expected crc 2023-10-09T06:20:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:17 smithi064 ceph-mon[220044]: failed to encode map e638 with expected crc 2023-10-09T06:20:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:17 smithi064 ceph-mon[220044]: failed to encode map e638 with expected crc 2023-10-09T06:20:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:17 smithi064 ceph-mon[220044]: failed to encode map e638 with expected crc 2023-10-09T06:20:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:17 smithi064 ceph-mon[220044]: osdmap e639: 8 total, 8 up, 6 in 2023-10-09T06:20:18.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:17 smithi156 ceph-mon[195101]: failed to encode map e638 with expected crc 2023-10-09T06:20:18.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:17 smithi156 ceph-mon[195101]: osdmap e638: 8 total, 8 up, 6 in 2023-10-09T06:20:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:17 smithi156 ceph-mon[195101]: failed to encode map e638 with expected crc 2023-10-09T06:20:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:17 smithi156 ceph-mon[195101]: failed to encode map e638 with expected crc 2023-10-09T06:20:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:17 smithi156 ceph-mon[195101]: failed to encode map e638 with expected crc 2023-10-09T06:20:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:17 smithi156 ceph-mon[195101]: failed to encode map e638 with expected crc 2023-10-09T06:20:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:17 smithi156 ceph-mon[195101]: failed to encode map e638 with expected crc 2023-10-09T06:20:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:17 smithi156 ceph-mon[195101]: osdmap e639: 8 total, 8 up, 6 in 2023-10-09T06:20:19.062 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:19.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:19.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:19.173 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:19.212 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[212750]: pgmap v1650: 103 pgs: 5 remapped+peering, 18 peering, 80 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:20:19.212 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[212750]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.212 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[212750]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[212750]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[212750]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[212750]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[212750]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[212750]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[212750]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[212750]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[212750]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[212750]: osdmap e640: 8 total, 8 up, 6 in 2023-10-09T06:20:19.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/464305212' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-691499-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:20:19.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[220044]: pgmap v1650: 103 pgs: 5 remapped+peering, 18 peering, 80 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:20:19.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[220044]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[220044]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[220044]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[220044]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[220044]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[220044]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[220044]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[220044]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[220044]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[220044]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.215 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[220044]: osdmap e640: 8 total, 8 up, 6 in 2023-10-09T06:20:19.215 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:18 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/464305212' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-691499-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:20:19.216 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:19.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:18 smithi156 ceph-mon[195101]: pgmap v1650: 103 pgs: 5 remapped+peering, 18 peering, 80 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:20:19.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:18 smithi156 ceph-mon[195101]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:18 smithi156 ceph-mon[195101]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:18 smithi156 ceph-mon[195101]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:18 smithi156 ceph-mon[195101]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:18 smithi156 ceph-mon[195101]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:18 smithi156 ceph-mon[195101]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:18 smithi156 ceph-mon[195101]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:18 smithi156 ceph-mon[195101]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:18 smithi156 ceph-mon[195101]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:18 smithi156 ceph-mon[195101]: failed to encode map e639 with expected crc 2023-10-09T06:20:19.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:18 smithi156 ceph-mon[195101]: osdmap e640: 8 total, 8 up, 6 in 2023-10-09T06:20:19.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:18 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/464305212' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-691499-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:20:19.381 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:19.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:19.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:19.486 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:19.518 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:19.572 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:19.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:19.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:19.576 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:20:19.576 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T06:20:19.576 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-10-09T06:20:19.576 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-10-09T06:20:19.577 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:20:19.676 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:19.705 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:19.750 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:19.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:19.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:19.854 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_all_features 2023-10-09T06:20:19.856 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:19.889 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:19.936 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:19.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:19.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:19.949 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_all_features (94 ms) 2023-10-09T06:20:19.949 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.copyup 2023-10-09T06:20:20.040 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:20.065 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:19 smithi156 ceph-mon[195101]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.065 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:19 smithi156 ceph-mon[195101]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.065 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:19 smithi156 ceph-mon[195101]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.065 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:19 smithi156 ceph-mon[195101]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.065 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:19 smithi156 ceph-mon[195101]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.065 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:19 smithi156 ceph-mon[195101]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.065 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:19 smithi156 ceph-mon[195101]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.065 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:19 smithi156 ceph-mon[195101]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.066 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:19 smithi156 ceph-mon[195101]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.066 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:19 smithi156 ceph-mon[195101]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.066 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:19 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/464305212' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-691499-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T06:20:20.066 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:19 smithi156 ceph-mon[195101]: osdmap e641: 8 total, 8 up, 6 in 2023-10-09T06:20:20.067 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:20.111 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:20.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:20.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:20.124 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.copyup (176 ms) 2023-10-09T06:20:20.124 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-10-09T06:20:20.136 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.sparse_copyup (12 ms) 2023-10-09T06:20:20.137 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-10-09T06:20:20.143 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_and_set_id (7 ms) 2023-10-09T06:20:20.143 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-10-09T06:20:20.153 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.add_remove_child (10 ms) 2023-10-09T06:20:20.153 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.directory_methods 2023-10-09T06:20:20.180 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.directory_methods (26 ms) 2023-10-09T06:20:20.180 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.create 2023-10-09T06:20:20.207 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.create (28 ms) 2023-10-09T06:20:20.208 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_features 2023-10-09T06:20:20.209 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_features (1 ms) 2023-10-09T06:20:20.209 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-10-09T06:20:20.211 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2023-10-09T06:20:20.211 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-10-09T06:20:20.212 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2023-10-09T06:20:20.212 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-10-09T06:20:20.213 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2023-10-09T06:20:20.213 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-10-09T06:20:20.214 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2023-10-09T06:20:20.215 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-10-09T06:20:20.216 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:20.218 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_data_pool (4 ms) 2023-10-09T06:20:20.219 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_size 2023-10-09T06:20:20.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:19 smithi064 ceph-mon[212750]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:19 smithi064 ceph-mon[212750]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:19 smithi064 ceph-mon[212750]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:19 smithi064 ceph-mon[212750]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:19 smithi064 ceph-mon[212750]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:19 smithi064 ceph-mon[212750]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:19 smithi064 ceph-mon[212750]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:19 smithi064 ceph-mon[212750]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:19 smithi064 ceph-mon[212750]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:19 smithi064 ceph-mon[212750]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:19 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/464305212' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-691499-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T06:20:20.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:19 smithi064 ceph-mon[212750]: osdmap e641: 8 total, 8 up, 6 in 2023-10-09T06:20:20.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:19 smithi064 ceph-mon[220044]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:19 smithi064 ceph-mon[220044]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:19 smithi064 ceph-mon[220044]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:19 smithi064 ceph-mon[220044]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:19 smithi064 ceph-mon[220044]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:19 smithi064 ceph-mon[220044]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:19 smithi064 ceph-mon[220044]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:19 smithi064 ceph-mon[220044]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:19 smithi064 ceph-mon[220044]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:19 smithi064 ceph-mon[220044]: failed to encode map e640 with expected crc 2023-10-09T06:20:20.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:19 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/464305212' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-691499-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T06:20:20.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:19 smithi064 ceph-mon[220044]: osdmap e641: 8 total, 8 up, 6 in 2023-10-09T06:20:20.222 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_size (4 ms) 2023-10-09T06:20:20.222 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.set_size 2023-10-09T06:20:20.227 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.set_size (5 ms) 2023-10-09T06:20:20.227 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.protection_status 2023-10-09T06:20:20.242 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:20.244 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.protection_status (17 ms) 2023-10-09T06:20:20.245 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-10-09T06:20:20.252 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.snapshot_limits (7 ms) 2023-10-09T06:20:20.252 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.parents_v1 2023-10-09T06:20:20.287 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:20.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:20.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:20.294 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.parents_v1 (43 ms) 2023-10-09T06:20:20.295 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.parents_v2 2023-10-09T06:20:20.327 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.parents_v2 (32 ms) 2023-10-09T06:20:20.327 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.snapshots 2023-10-09T06:20:20.345 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:20:20.350 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.snapshots (23 ms) 2023-10-09T06:20:20.350 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.snapid_race 2023-10-09T06:20:20.354 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.snapid_race (5 ms) 2023-10-09T06:20:20.354 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.stripingv2 2023-10-09T06:20:20.364 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.stripingv2 (9 ms) 2023-10-09T06:20:20.364 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.object_map_save 2023-10-09T06:20:20.366 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.object_map_save (3 ms) 2023-10-09T06:20:20.366 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-10-09T06:20:20.374 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.object_map_resize (8 ms) 2023-10-09T06:20:20.374 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.object_map_update 2023-10-09T06:20:20.381 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.object_map_update (7 ms) 2023-10-09T06:20:20.381 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-10-09T06:20:20.381 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-10-09T06:20:20.381 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-10-09T06:20:20.386 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.object_map_snap_add (4 ms) 2023-10-09T06:20:20.386 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-10-09T06:20:20.390 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.object_map_snap_remove (4 ms) 2023-10-09T06:20:20.390 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.flags 2023-10-09T06:20:20.393 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:20.395 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.flags (5 ms) 2023-10-09T06:20:20.395 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.metadata 2023-10-09T06:20:20.420 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:20.426 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.metadata (31 ms) 2023-10-09T06:20:20.426 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.set_features 2023-10-09T06:20:20.434 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.set_features (7 ms) 2023-10-09T06:20:20.434 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.mirror 2023-10-09T06:20:20.465 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:20.470 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.mirror (37 ms) 2023-10-09T06:20:20.470 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.mirror_image 2023-10-09T06:20:20.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:20.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:20.486 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.mirror_image (15 ms) 2023-10-09T06:20:20.486 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-10-09T06:20:20.572 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:20.599 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:20.643 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:20.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:20.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:20.749 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:20.776 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:20.784 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:20.846 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:20.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:20.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:20.950 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:20.982 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:21.047 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:21.049 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:20 smithi064 ceph-mon[212750]: pgmap v1653: 135 pgs: 32 unknown, 5 remapped+peering, 18 peering, 80 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:20:21.049 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:20 smithi064 ceph-mon[212750]: failed to encode map e641 with expected crc 2023-10-09T06:20:21.049 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:20 smithi064 ceph-mon[212750]: failed to encode map e641 with expected crc 2023-10-09T06:20:21.049 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:20 smithi064 ceph-mon[212750]: failed to encode map e641 with expected crc 2023-10-09T06:20:21.049 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:20 smithi064 ceph-mon[212750]: failed to encode map e641 with expected crc 2023-10-09T06:20:21.049 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:20 smithi064 ceph-mon[212750]: osdmap e642: 8 total, 8 up, 6 in 2023-10-09T06:20:21.049 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:20 smithi064 ceph-mon[220044]: pgmap v1653: 135 pgs: 32 unknown, 5 remapped+peering, 18 peering, 80 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:20:21.050 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:20 smithi064 ceph-mon[220044]: failed to encode map e641 with expected crc 2023-10-09T06:20:21.050 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:20 smithi064 ceph-mon[220044]: failed to encode map e641 with expected crc 2023-10-09T06:20:21.050 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:20 smithi064 ceph-mon[220044]: failed to encode map e641 with expected crc 2023-10-09T06:20:21.050 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:20 smithi064 ceph-mon[220044]: failed to encode map e641 with expected crc 2023-10-09T06:20:21.050 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:20 smithi064 ceph-mon[220044]: osdmap e642: 8 total, 8 up, 6 in 2023-10-09T06:20:21.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:21.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:21.081 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:20:21.152 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:21.176 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:20 smithi156 ceph-mon[195101]: pgmap v1653: 135 pgs: 32 unknown, 5 remapped+peering, 18 peering, 80 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:20:21.176 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:20 smithi156 ceph-mon[195101]: failed to encode map e641 with expected crc 2023-10-09T06:20:21.176 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:20 smithi156 ceph-mon[195101]: failed to encode map e641 with expected crc 2023-10-09T06:20:21.176 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:20 smithi156 ceph-mon[195101]: failed to encode map e641 with expected crc 2023-10-09T06:20:21.176 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:20 smithi156 ceph-mon[195101]: failed to encode map e641 with expected crc 2023-10-09T06:20:21.176 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:20 smithi156 ceph-mon[195101]: osdmap e642: 8 total, 8 up, 6 in 2023-10-09T06:20:21.178 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:21.221 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:21.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:21.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:21.237 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:20:21.330 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:21.368 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:21.445 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:21.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:21.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:21.549 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:21.576 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:22.075 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:22.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:22.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:22.179 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:22.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:21 smithi064 ceph-mon[212750]: failed to encode map e642 with expected crc 2023-10-09T06:20:22.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:21 smithi064 ceph-mon[212750]: failed to encode map e642 with expected crc 2023-10-09T06:20:22.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:21 smithi064 ceph-mon[220044]: failed to encode map e642 with expected crc 2023-10-09T06:20:22.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:21 smithi064 ceph-mon[220044]: failed to encode map e642 with expected crc 2023-10-09T06:20:22.218 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:22.289 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:22.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:22.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:22.297 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:22.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:21 smithi156 ceph-mon[195101]: failed to encode map e642 with expected crc 2023-10-09T06:20:22.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:21 smithi156 ceph-mon[195101]: failed to encode map e642 with expected crc 2023-10-09T06:20:22.396 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:22.416 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:22.423 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:22.467 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:22.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:22.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:22.573 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:22.610 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:22.772 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:20:22.773 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T06:20:22.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T06:20:22.773 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:20:22.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.773 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:171672 10.3 10:c82752d7:::benchmark_data_smithi064_573750_object75457:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e557)", 2023-10-09T06:20:22.773 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:15:15.799808+0000", 2023-10-09T06:20:22.773 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 306.96829712800002, 2023-10-09T06:20:22.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.162648323, 2023-10-09T06:20:22.773 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:22.773 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:20:22.773 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:22.774 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:20:22.774 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:22.774 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171672 2023-10-09T06:20:22.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.774 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:22.774 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:22.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799808+0000", 2023-10-09T06:20:22.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:22.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799808+0000", 2023-10-09T06:20:22.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:22.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799805+0000", 2023-10-09T06:20:22.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:22.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:22.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799813+0000", 2023-10-09T06:20:22.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9149999999999996e-06 2023-10-09T06:20:22.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:22.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799826+0000", 2023-10-09T06:20:22.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2824e-05 2023-10-09T06:20:22.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799834+0000", 2023-10-09T06:20:22.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8990000000000001e-06 2023-10-09T06:20:22.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.777 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.800015+0000", 2023-10-09T06:20:22.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018057300000000001 2023-10-09T06:20:22.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.777 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:22.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.800155+0000", 2023-10-09T06:20:22.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014034099999999999 2023-10-09T06:20:22.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:20:22.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.800283+0000", 2023-10-09T06:20:22.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000127945 2023-10-09T06:20:22.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:22.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.808806+0000", 2023-10-09T06:20:22.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085225639999999998 2023-10-09T06:20:22.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812281+0000", 2023-10-09T06:20:22.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003475653 2023-10-09T06:20:22.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812302+0000", 2023-10-09T06:20:22.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0441e-05 2023-10-09T06:20:22.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794653+0000", 2023-10-09T06:20:22.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98235096899999996 2023-10-09T06:20:22.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794672+0000", 2023-10-09T06:20:22.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9063e-05 2023-10-09T06:20:22.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.950344+0000", 2023-10-09T06:20:22.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.155671983 2023-10-09T06:20:22.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-10-09T06:20:22.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.950466+0000", 2023-10-09T06:20:22.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000122123 2023-10-09T06:20:22.782 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.962321+0000", 2023-10-09T06:20:22.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01185547 2023-10-09T06:20:22.782 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:22.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.962456+0000", 2023-10-09T06:20:22.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013469099999999999 2023-10-09T06:20:22.782 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.783 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:22.783 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.783 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:171676 10.3 10:c58e27ea:::benchmark_data_smithi064_573750_object75461:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e557)", 2023-10-09T06:20:22.783 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:15:15.799859+0000", 2023-10-09T06:20:22.783 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 306.96824565100002, 2023-10-09T06:20:22.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1745875799999999, 2023-10-09T06:20:22.783 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:22.783 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:20:22.783 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:22.784 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:20:22.784 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:22.784 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171676 2023-10-09T06:20:22.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.784 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:22.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.784 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:22.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799859+0000", 2023-10-09T06:20:22.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.785 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:22.785 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799859+0000", 2023-10-09T06:20:22.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.785 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:22.785 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799859+0000", 2023-10-09T06:20:22.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:22.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.785 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:22.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799862+0000", 2023-10-09T06:20:22.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0750000000000002e-06 2023-10-09T06:20:22.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:22.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799875+0000", 2023-10-09T06:20:22.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.31e-05 2023-10-09T06:20:22.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.799881+0000", 2023-10-09T06:20:22.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.801e-06 2023-10-09T06:20:22.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.803918+0000", 2023-10-09T06:20:22.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040373400000000004 2023-10-09T06:20:22.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:22.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.804065+0000", 2023-10-09T06:20:22.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014711600000000001 2023-10-09T06:20:22.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:20:22.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.804246+0000", 2023-10-09T06:20:22.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018100599999999999 2023-10-09T06:20:22.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812631+0000", 2023-10-09T06:20:22.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0083849679999999996 2023-10-09T06:20:22.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812642+0000", 2023-10-09T06:20:22.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0681e-05 2023-10-09T06:20:22.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794709+0000", 2023-10-09T06:20:22.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98206730200000003 2023-10-09T06:20:22.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794722+0000", 2023-10-09T06:20:22.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2621e-05 2023-10-09T06:20:22.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.950773+0000", 2023-10-09T06:20:22.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.156051175 2023-10-09T06:20:22.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-10-09T06:20:22.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.950916+0000", 2023-10-09T06:20:22.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000142726 2023-10-09T06:20:22.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974353+0000", 2023-10-09T06:20:22.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023437021999999998 2023-10-09T06:20:22.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:22.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974447+0000", 2023-10-09T06:20:22.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4092000000000006e-05 2023-10-09T06:20:22.792 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.792 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:22.792 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.792 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.792 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:171677 10.3 10:cc911201:::benchmark_data_smithi064_573750_object75462:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:20:22.792 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:15:15.812441+0000", 2023-10-09T06:20:22.792 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 306.95566420699998, 2023-10-09T06:20:22.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.162245993, 2023-10-09T06:20:22.792 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:22.792 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:22.793 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:22.793 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:20:22.793 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:22.793 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171677 2023-10-09T06:20:22.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.793 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:22.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:22.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812441+0000", 2023-10-09T06:20:22.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:22.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812441+0000", 2023-10-09T06:20:22.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:22.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812439+0000", 2023-10-09T06:20:22.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:22.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:22.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812444+0000", 2023-10-09T06:20:22.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1150000000000002e-06 2023-10-09T06:20:22.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:22.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812455+0000", 2023-10-09T06:20:22.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1742999999999999e-05 2023-10-09T06:20:22.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812464+0000", 2023-10-09T06:20:22.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4910000000000005e-06 2023-10-09T06:20:22.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812964+0000", 2023-10-09T06:20:22.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00049951399999999995 2023-10-09T06:20:22.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.812975+0000", 2023-10-09T06:20:22.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1063999999999999e-05 2023-10-09T06:20:22.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794751+0000", 2023-10-09T06:20:22.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.981776023 2023-10-09T06:20:22.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794760+0000", 2023-10-09T06:20:22.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9590000000000001e-06 2023-10-09T06:20:22.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.950951+0000", 2023-10-09T06:20:22.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.156191089 2023-10-09T06:20:22.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:22.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.951073+0000", 2023-10-09T06:20:22.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012265999999999999 2023-10-09T06:20:22.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.799 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.799 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T06:20:22.799 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.951219+0000", 2023-10-09T06:20:22.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014580999999999999 2023-10-09T06:20:22.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.799 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.799 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:22.799 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.961907+0000", 2023-10-09T06:20:22.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010687962000000001 2023-10-09T06:20:22.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.800 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.962502+0000", 2023-10-09T06:20:22.800 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000594646 2023-10-09T06:20:22.800 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.800 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.973943+0000", 2023-10-09T06:20:22.800 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011441019 2023-10-09T06:20:22.800 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974516+0000", 2023-10-09T06:20:22.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00057290800000000001 2023-10-09T06:20:22.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:22.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974566+0000", 2023-10-09T06:20:22.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0080999999999997e-05 2023-10-09T06:20:22.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:22.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974687+0000", 2023-10-09T06:20:22.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000120795 2023-10-09T06:20:22.802 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.802 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:22.802 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.802 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:171684 10.3 10:cf644d59:::benchmark_data_smithi064_573750_object75469:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:20:22.802 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:15:15.815765+0000", 2023-10-09T06:20:22.802 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 306.95233942999999, 2023-10-09T06:20:22.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1591680040000001, 2023-10-09T06:20:22.803 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:22.803 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:22.803 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:22.803 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:20:22.803 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:22.803 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171684 2023-10-09T06:20:22.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.803 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:22.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.803 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:22.803 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815765+0000", 2023-10-09T06:20:22.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:22.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815765+0000", 2023-10-09T06:20:22.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:22.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815764+0000", 2023-10-09T06:20:22.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:22.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:22.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815771+0000", 2023-10-09T06:20:22.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3080000000000002e-06 2023-10-09T06:20:22.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:22.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815785+0000", 2023-10-09T06:20:22.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3903000000000001e-05 2023-10-09T06:20:22.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815795+0000", 2023-10-09T06:20:22.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8209999999999997e-06 2023-10-09T06:20:22.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815830+0000", 2023-10-09T06:20:22.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4987999999999998e-05 2023-10-09T06:20:22.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.815839+0000", 2023-10-09T06:20:22.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5189999999999998e-06 2023-10-09T06:20:22.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794786+0000", 2023-10-09T06:20:22.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97894675900000006 2023-10-09T06:20:22.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794794+0000", 2023-10-09T06:20:22.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5590000000000006e-06 2023-10-09T06:20:22.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.954768+0000", 2023-10-09T06:20:22.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.159973972 2023-10-09T06:20:22.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:22.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.954905+0000", 2023-10-09T06:20:22.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013690299999999999 2023-10-09T06:20:22.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T06:20:22.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.955044+0000", 2023-10-09T06:20:22.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013847499999999999 2023-10-09T06:20:22.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:22.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.962281+0000", 2023-10-09T06:20:22.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0072369289999999996 2023-10-09T06:20:22.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.962546+0000", 2023-10-09T06:20:22.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00026517000000000001 2023-10-09T06:20:22.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974728+0000", 2023-10-09T06:20:22.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012181772 2023-10-09T06:20:22.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974764+0000", 2023-10-09T06:20:22.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6820000000000003e-05 2023-10-09T06:20:22.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:22.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974817+0000", 2023-10-09T06:20:22.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2811e-05 2023-10-09T06:20:22.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:22.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974933+0000", 2023-10-09T06:20:22.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116087 2023-10-09T06:20:22.811 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.812 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:22.812 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.812 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:171685 10.3 10:cb5292a6:::benchmark_data_smithi064_573750_object75470:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:20:22.812 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:15:15.816204+0000", 2023-10-09T06:20:22.812 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 306.951901228, 2023-10-09T06:20:22.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1602268950000001, 2023-10-09T06:20:22.812 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:22.812 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:22.812 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:22.813 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:20:22.813 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:22.813 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 171685 2023-10-09T06:20:22.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.813 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:22.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:22.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816204+0000", 2023-10-09T06:20:22.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:22.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816204+0000", 2023-10-09T06:20:22.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:22.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816203+0000", 2023-10-09T06:20:22.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:22.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:22.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816207+0000", 2023-10-09T06:20:22.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3000000000000003e-06 2023-10-09T06:20:22.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:22.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816220+0000", 2023-10-09T06:20:22.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3208e-05 2023-10-09T06:20:22.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816228+0000", 2023-10-09T06:20:22.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6869999999999994e-06 2023-10-09T06:20:22.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816276+0000", 2023-10-09T06:20:22.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7632000000000001e-05 2023-10-09T06:20:22.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:15.816287+0000", 2023-10-09T06:20:22.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0908e-05 2023-10-09T06:20:22.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794918+0000", 2023-10-09T06:20:22.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97863145399999996 2023-10-09T06:20:22.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:16.794928+0000", 2023-10-09T06:20:22.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.005e-05 2023-10-09T06:20:22.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.958279+0000", 2023-10-09T06:20:22.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.163350514 2023-10-09T06:20:22.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:22.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.958431+0000", 2023-10-09T06:20:22.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000151946 2023-10-09T06:20:22.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T06:20:22.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.958587+0000", 2023-10-09T06:20:22.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015680800000000001 2023-10-09T06:20:22.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:22.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.962454+0000", 2023-10-09T06:20:22.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038670100000000002 2023-10-09T06:20:22.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974038+0000", 2023-10-09T06:20:22.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01158314 2023-10-09T06:20:22.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.974974+0000", 2023-10-09T06:20:22.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00093642900000000002 2023-10-09T06:20:22.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.976244+0000", 2023-10-09T06:20:22.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001270265 2023-10-09T06:20:22.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:22.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.976304+0000", 2023-10-09T06:20:22.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9621999999999999e-05 2023-10-09T06:20:22.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:22.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:15:17.976430+0000", 2023-10-09T06:20:22.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012658100000000001 2023-10-09T06:20:22.822 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.822 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:22.822 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.822 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:114773 11.a 11:59d5ab77:::benchmark_data_smithi064_631903_object5859:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-10-09T06:20:22.822 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:17:46.566276+0000", 2023-10-09T06:20:22.822 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 156.20182838599999, 2023-10-09T06:20:22.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2362202039999999, 2023-10-09T06:20:22.822 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:22.822 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:20:22.823 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:22.823 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:20:22.823 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:22.823 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 114773 2023-10-09T06:20:22.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.823 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:22.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:22.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566276+0000", 2023-10-09T06:20:22.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:22.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566276+0000", 2023-10-09T06:20:22.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:22.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566275+0000", 2023-10-09T06:20:22.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:20:22.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:22.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566281+0000", 2023-10-09T06:20:22.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0900000000000001e-06 2023-10-09T06:20:22.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:22.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566294+0000", 2023-10-09T06:20:22.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2852e-05 2023-10-09T06:20:22.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566302+0000", 2023-10-09T06:20:22.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9920000000000007e-06 2023-10-09T06:20:22.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.570258+0000", 2023-10-09T06:20:22.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039559749999999996 2023-10-09T06:20:22.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:22.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.570366+0000", 2023-10-09T06:20:22.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000107772 2023-10-09T06:20:22.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:20:22.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.570476+0000", 2023-10-09T06:20:22.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000109956 2023-10-09T06:20:22.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581185+0000", 2023-10-09T06:20:22.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010708746999999999 2023-10-09T06:20:22.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581197+0000", 2023-10-09T06:20:22.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2275e-05 2023-10-09T06:20:22.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.599254+0000", 2023-10-09T06:20:22.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0180574899999999 2023-10-09T06:20:22.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.599267+0000", 2023-10-09T06:20:22.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2661e-05 2023-10-09T06:20:22.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.802421+0000", 2023-10-09T06:20:22.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.203153666 2023-10-09T06:20:22.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:22.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.802497+0000", 2023-10-09T06:20:22.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5834999999999995e-05 2023-10-09T06:20:22.830 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.830 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:22.830 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.830 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:114774 11.1 11:81618b5d:::benchmark_data_smithi064_631903_object5860:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-10-09T06:20:22.830 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:17:46.566314+0000", 2023-10-09T06:20:22.830 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 156.20179037299999, 2023-10-09T06:20:22.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2080191130000002, 2023-10-09T06:20:22.830 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:22.831 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:20:22.831 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:22.831 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:20:22.831 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:22.831 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 114774 2023-10-09T06:20:22.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.831 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:22.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:22.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566314+0000", 2023-10-09T06:20:22.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:22.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566314+0000", 2023-10-09T06:20:22.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:22.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566314+0000", 2023-10-09T06:20:22.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:22.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:22.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566317+0000", 2023-10-09T06:20:22.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7070000000000001e-06 2023-10-09T06:20:22.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:22.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566323+0000", 2023-10-09T06:20:22.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8699999999999997e-06 2023-10-09T06:20:22.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566328+0000", 2023-10-09T06:20:22.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0470000000000001e-06 2023-10-09T06:20:22.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566453+0000", 2023-10-09T06:20:22.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012555500000000001 2023-10-09T06:20:22.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:22.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566572+0000", 2023-10-09T06:20:22.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118514 2023-10-09T06:20:22.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2023-10-09T06:20:22.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.566699+0000", 2023-10-09T06:20:22.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012719000000000001 2023-10-09T06:20:22.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:22.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.571304+0000", 2023-10-09T06:20:22.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046050659999999997 2023-10-09T06:20:22.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.587174+0000", 2023-10-09T06:20:22.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015869695 2023-10-09T06:20:22.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.587186+0000", 2023-10-09T06:20:22.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2255e-05 2023-10-09T06:20:22.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.600012+0000", 2023-10-09T06:20:22.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0128258990000001 2023-10-09T06:20:22.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.600023+0000", 2023-10-09T06:20:22.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1344e-05 2023-10-09T06:20:22.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.774248+0000", 2023-10-09T06:20:22.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.17422523 2023-10-09T06:20:22.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:22.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.774333+0000", 2023-10-09T06:20:22.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5148999999999994e-05 2023-10-09T06:20:22.838 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.838 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:22.838 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.839 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:114776 11.9 11:90bf3a7c:::benchmark_data_smithi064_631903_object5862:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-10-09T06:20:22.839 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:17:46.580825+0000", 2023-10-09T06:20:22.839 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 156.187280077, 2023-10-09T06:20:22.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2761611980000001, 2023-10-09T06:20:22.839 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:22.839 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:22.839 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:22.839 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:20:22.839 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:22.840 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 114776 2023-10-09T06:20:22.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.840 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:22.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:22.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.580825+0000", 2023-10-09T06:20:22.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:22.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.580825+0000", 2023-10-09T06:20:22.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:22.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.580823+0000", 2023-10-09T06:20:22.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:22.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:22.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.580830+0000", 2023-10-09T06:20:22.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9840000000000004e-06 2023-10-09T06:20:22.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:22.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.580841+0000", 2023-10-09T06:20:22.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1916000000000001e-05 2023-10-09T06:20:22.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.580850+0000", 2023-10-09T06:20:22.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2479999999999996e-06 2023-10-09T06:20:22.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.587212+0000", 2023-10-09T06:20:22.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063625649999999997 2023-10-09T06:20:22.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.587220+0000", 2023-10-09T06:20:22.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1410000000000005e-06 2023-10-09T06:20:22.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.603487+0000", 2023-10-09T06:20:22.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0162665470000001 2023-10-09T06:20:22.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.603496+0000", 2023-10-09T06:20:22.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3589999999999997e-06 2023-10-09T06:20:22.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.823894+0000", 2023-10-09T06:20:22.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2203975890000001 2023-10-09T06:20:22.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:22.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.824020+0000", 2023-10-09T06:20:22.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000125685 2023-10-09T06:20:22.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1,6", 2023-10-09T06:20:22.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.824551+0000", 2023-10-09T06:20:22.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00053100900000000004 2023-10-09T06:20:22.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.847637+0000", 2023-10-09T06:20:22.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023086654000000002 2023-10-09T06:20:22.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.847674+0000", 2023-10-09T06:20:22.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6338999999999999e-05 2023-10-09T06:20:22.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.847897+0000", 2023-10-09T06:20:22.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022390999999999999 2023-10-09T06:20:22.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:22.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.856840+0000", 2023-10-09T06:20:22.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0089426409999999994 2023-10-09T06:20:22.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:22.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.856893+0000", 2023-10-09T06:20:22.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2531999999999998e-05 2023-10-09T06:20:22.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:22.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.856986+0000", 2023-10-09T06:20:22.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3219000000000003e-05 2023-10-09T06:20:22.848 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.848 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:22.848 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.848 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:114779 11.a 11:51407f43:::benchmark_data_smithi064_631903_object5865:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-10-09T06:20:22.848 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:17:46.581278+0000", 2023-10-09T06:20:22.848 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 156.18682718700001, 2023-10-09T06:20:22.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2608115089999998, 2023-10-09T06:20:22.848 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:22.848 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:22.849 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:22.849 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:20:22.849 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:22.849 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 114779 2023-10-09T06:20:22.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.849 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:22.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:22.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581278+0000", 2023-10-09T06:20:22.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:22.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581278+0000", 2023-10-09T06:20:22.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:22.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581277+0000", 2023-10-09T06:20:22.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:22.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:22.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581281+0000", 2023-10-09T06:20:22.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1339999999999997e-06 2023-10-09T06:20:22.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:22.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581291+0000", 2023-10-09T06:20:22.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9820000000000003e-06 2023-10-09T06:20:22.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581298+0000", 2023-10-09T06:20:22.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6989999999999997e-06 2023-10-09T06:20:22.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581500+0000", 2023-10-09T06:20:22.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020228499999999999 2023-10-09T06:20:22.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581509+0000", 2023-10-09T06:20:22.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8130000000000002e-06 2023-10-09T06:20:22.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.601561+0000", 2023-10-09T06:20:22.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0200516829999999 2023-10-09T06:20:22.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.601573+0000", 2023-10-09T06:20:22.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2702e-05 2023-10-09T06:20:22.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.802518+0000", 2023-10-09T06:20:22.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2009447630000001 2023-10-09T06:20:22.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:22.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.802650+0000", 2023-10-09T06:20:22.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013167699999999999 2023-10-09T06:20:22.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T06:20:22.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.803198+0000", 2023-10-09T06:20:22.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00054843800000000005 2023-10-09T06:20:22.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.855 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.841745+0000", 2023-10-09T06:20:22.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.038546634000000003 2023-10-09T06:20:22.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.855 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.841775+0000", 2023-10-09T06:20:22.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0762999999999997e-05 2023-10-09T06:20:22.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.855 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.841910+0000", 2023-10-09T06:20:22.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000134735 2023-10-09T06:20:22.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:22.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.841945+0000", 2023-10-09T06:20:22.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.468e-05 2023-10-09T06:20:22.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:22.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.841990+0000", 2023-10-09T06:20:22.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4659000000000001e-05 2023-10-09T06:20:22.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:22.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.842089+0000", 2023-10-09T06:20:22.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9487000000000005e-05 2023-10-09T06:20:22.857 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.857 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:22.857 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.857 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:114781 11.1 11:882127a0:::benchmark_data_smithi064_631903_object5867:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-10-09T06:20:22.857 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:17:46.581303+0000", 2023-10-09T06:20:22.858 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 156.186801839, 2023-10-09T06:20:22.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2665437900000001, 2023-10-09T06:20:22.858 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:22.858 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:22.858 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:22.858 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:20:22.858 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:22.858 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 114781 2023-10-09T06:20:22.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.858 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:22.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.859 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:22.859 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581303+0000", 2023-10-09T06:20:22.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.859 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:22.859 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581303+0000", 2023-10-09T06:20:22.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.859 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:22.859 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581303+0000", 2023-10-09T06:20:22.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:22.860 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:22.860 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581305+0000", 2023-10-09T06:20:22.860 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4499999999999998e-06 2023-10-09T06:20:22.860 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:22.860 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581311+0000", 2023-10-09T06:20:22.860 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0789999999999997e-06 2023-10-09T06:20:22.860 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581316+0000", 2023-10-09T06:20:22.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7840000000000003e-06 2023-10-09T06:20:22.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.587243+0000", 2023-10-09T06:20:22.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059268660000000003 2023-10-09T06:20:22.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.587250+0000", 2023-10-09T06:20:22.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2099999999999996e-06 2023-10-09T06:20:22.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.603461+0000", 2023-10-09T06:20:22.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.016211451 2023-10-09T06:20:22.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.603469+0000", 2023-10-09T06:20:22.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8679999999999999e-06 2023-10-09T06:20:22.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.774358+0000", 2023-10-09T06:20:22.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.170888288 2023-10-09T06:20:22.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:22.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.774485+0000", 2023-10-09T06:20:22.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000127532 2023-10-09T06:20:22.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,6", 2023-10-09T06:20:22.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.775115+0000", 2023-10-09T06:20:22.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00062980000000000002 2023-10-09T06:20:22.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.791339+0000", 2023-10-09T06:20:22.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016224072999999999 2023-10-09T06:20:22.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:22.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.794574+0000", 2023-10-09T06:20:22.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032347840000000001 2023-10-09T06:20:22.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.806294+0000", 2023-10-09T06:20:22.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011720563 2023-10-09T06:20:22.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.847650+0000", 2023-10-09T06:20:22.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.041356182999999998 2023-10-09T06:20:22.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:22.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.847714+0000", 2023-10-09T06:20:22.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3434000000000003e-05 2023-10-09T06:20:22.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.869 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:22.869 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.847847+0000", 2023-10-09T06:20:22.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000132812 2023-10-09T06:20:22.869 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.869 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:22.869 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.870 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:114783 11.1 11:83e60dcb:::benchmark_data_smithi064_631903_object5869:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-10-09T06:20:22.870 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:17:46.581336+0000", 2023-10-09T06:20:22.870 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 156.18676883000001, 2023-10-09T06:20:22.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2667203570000001, 2023-10-09T06:20:22.870 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:22.870 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:22.870 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:22.870 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:20:22.870 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:22.870 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 114783 2023-10-09T06:20:22.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.871 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:22.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:22.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581336+0000", 2023-10-09T06:20:22.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:22.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581336+0000", 2023-10-09T06:20:22.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:22.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581336+0000", 2023-10-09T06:20:22.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:22.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:22.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581338+0000", 2023-10-09T06:20:22.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3719999999999999e-06 2023-10-09T06:20:22.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:22.873 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581343+0000", 2023-10-09T06:20:22.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6750000000000001e-06 2023-10-09T06:20:22.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.873 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.873 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.581347+0000", 2023-10-09T06:20:22.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3889999999999998e-06 2023-10-09T06:20:22.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.873 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.873 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.587272+0000", 2023-10-09T06:20:22.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059252380000000002 2023-10-09T06:20:22.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.874 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.874 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.587281+0000", 2023-10-09T06:20:22.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4400000000000005e-06 2023-10-09T06:20:22.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.874 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.874 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.603600+0000", 2023-10-09T06:20:22.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.016319258 2023-10-09T06:20:22.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.874 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.603609+0000", 2023-10-09T06:20:22.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1779999999999992e-06 2023-10-09T06:20:22.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.791551+0000", 2023-10-09T06:20:22.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.187941699 2023-10-09T06:20:22.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:22.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.791663+0000", 2023-10-09T06:20:22.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000112191 2023-10-09T06:20:22.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,4,6", 2023-10-09T06:20:22.876 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.791755+0000", 2023-10-09T06:20:22.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2168000000000003e-05 2023-10-09T06:20:22.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.876 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.805268+0000", 2023-10-09T06:20:22.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013512266 2023-10-09T06:20:22.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:22.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.805290+0000", 2023-10-09T06:20:22.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2599000000000002e-05 2023-10-09T06:20:22.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.847595+0000", 2023-10-09T06:20:22.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.042304464 2023-10-09T06:20:22.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.847910+0000", 2023-10-09T06:20:22.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00031572000000000002 2023-10-09T06:20:22.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:22.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.847951+0000", 2023-10-09T06:20:22.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0423000000000002e-05 2023-10-09T06:20:22.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:22.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.848056+0000", 2023-10-09T06:20:22.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010554500000000001 2023-10-09T06:20:22.879 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.879 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:22.879 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.879 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:114785 11.9 11:9e2e2456:::benchmark_data_smithi064_631903_object5871:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-10-09T06:20:22.879 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:17:46.585044+0000", 2023-10-09T06:20:22.879 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 156.18306069900001, 2023-10-09T06:20:22.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2886440160000001, 2023-10-09T06:20:22.879 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:22.879 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:22.880 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:22.880 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:20:22.880 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:22.880 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 114785 2023-10-09T06:20:22.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.880 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:22.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:22.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.585044+0000", 2023-10-09T06:20:22.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:22.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.585044+0000", 2023-10-09T06:20:22.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:22.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.585042+0000", 2023-10-09T06:20:22.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:22.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:22.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.585050+0000", 2023-10-09T06:20:22.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3350000000000001e-06 2023-10-09T06:20:22.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:22.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.585073+0000", 2023-10-09T06:20:22.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3804000000000001e-05 2023-10-09T06:20:22.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.585083+0000", 2023-10-09T06:20:22.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3919999999999992e-06 2023-10-09T06:20:22.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.587303+0000", 2023-10-09T06:20:22.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002220559 2023-10-09T06:20:22.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.587311+0000", 2023-10-09T06:20:22.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7919999999999993e-06 2023-10-09T06:20:22.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.603524+0000", 2023-10-09T06:20:22.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0162130540000001 2023-10-09T06:20:22.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.603534+0000", 2023-10-09T06:20:22.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0051000000000001e-05 2023-10-09T06:20:22.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.847924+0000", 2023-10-09T06:20:22.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.244389792 2023-10-09T06:20:22.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:22.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.848069+0000", 2023-10-09T06:20:22.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014492500000000001 2023-10-09T06:20:22.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1,6", 2023-10-09T06:20:22.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.848220+0000", 2023-10-09T06:20:22.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015109300000000001 2023-10-09T06:20:22.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.886 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.856828+0000", 2023-10-09T06:20:22.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086078479999999995 2023-10-09T06:20:22.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:22.886 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.856995+0000", 2023-10-09T06:20:22.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016734700000000001 2023-10-09T06:20:22.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.887 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.873490+0000", 2023-10-09T06:20:22.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016494463000000001 2023-10-09T06:20:22.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.887 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.873518+0000", 2023-10-09T06:20:22.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8430000000000001e-05 2023-10-09T06:20:22.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:22.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.873572+0000", 2023-10-09T06:20:22.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3742999999999997e-05 2023-10-09T06:20:22.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:22.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.873688+0000", 2023-10-09T06:20:22.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116193 2023-10-09T06:20:22.888 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.888 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:22.888 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.889 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:114789 11.a 11:59c1771f:::benchmark_data_smithi064_631903_object5875:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-10-09T06:20:22.889 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:17:46.591783+0000", 2023-10-09T06:20:22.889 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 156.17632207599999, 2023-10-09T06:20:22.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2665931619999999, 2023-10-09T06:20:22.889 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:22.889 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:22.889 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:22.889 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:20:22.889 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:22.889 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 114789 2023-10-09T06:20:22.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.890 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:22.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:22.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.591783+0000", 2023-10-09T06:20:22.890 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:22.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.591783+0000", 2023-10-09T06:20:22.890 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:22.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.591781+0000", 2023-10-09T06:20:22.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:22.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:22.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.591789+0000", 2023-10-09T06:20:22.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0150000000000003e-06 2023-10-09T06:20:22.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:22.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.591805+0000", 2023-10-09T06:20:22.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6209999999999999e-05 2023-10-09T06:20:22.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.591814+0000", 2023-10-09T06:20:22.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4509999999999998e-06 2023-10-09T06:20:22.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.591936+0000", 2023-10-09T06:20:22.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012205 2023-10-09T06:20:22.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.591946+0000", 2023-10-09T06:20:22.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6660000000000003e-06 2023-10-09T06:20:22.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.602104+0000", 2023-10-09T06:20:22.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0101581639999999 2023-10-09T06:20:22.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.602116+0000", 2023-10-09T06:20:22.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1389999999999999e-05 2023-10-09T06:20:22.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.838602+0000", 2023-10-09T06:20:22.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2364861620000001 2023-10-09T06:20:22.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:22.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.838773+0000", 2023-10-09T06:20:22.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017120899999999999 2023-10-09T06:20:22.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T06:20:22.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.838924+0000", 2023-10-09T06:20:22.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015081799999999999 2023-10-09T06:20:22.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.841938+0000", 2023-10-09T06:20:22.896 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030146560000000001 2023-10-09T06:20:22.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.896 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:22.896 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.842098+0000", 2023-10-09T06:20:22.896 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000159207 2023-10-09T06:20:22.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.896 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.896 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.851850+0000", 2023-10-09T06:20:22.896 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0097527059999999999 2023-10-09T06:20:22.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.858207+0000", 2023-10-09T06:20:22.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006356535 2023-10-09T06:20:22.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:22.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.858254+0000", 2023-10-09T06:20:22.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7511e-05 2023-10-09T06:20:22.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.898 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:22.898 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.858376+0000", 2023-10-09T06:20:22.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000121488 2023-10-09T06:20:22.898 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.898 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:22.898 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.898 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:114796 11.9 11:95741733:::benchmark_data_smithi064_631903_object5882:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-10-09T06:20:22.898 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:17:46.615408+0000", 2023-10-09T06:20:22.898 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 156.15269663800001, 2023-10-09T06:20:22.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.277054293, 2023-10-09T06:20:22.899 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:22.899 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:22.899 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:22.899 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:20:22.899 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:22.899 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 114796 2023-10-09T06:20:22.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.899 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:22.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:22.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615408+0000", 2023-10-09T06:20:22.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:22.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615408+0000", 2023-10-09T06:20:22.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:22.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615407+0000", 2023-10-09T06:20:22.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:20:22.901 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.901 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:22.901 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615412+0000", 2023-10-09T06:20:22.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0309999999999998e-06 2023-10-09T06:20:22.901 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.901 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:22.901 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615423+0000", 2023-10-09T06:20:22.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1311000000000001e-05 2023-10-09T06:20:22.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615430+0000", 2023-10-09T06:20:22.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0589999999999997e-06 2023-10-09T06:20:22.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615497+0000", 2023-10-09T06:20:22.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6364000000000004e-05 2023-10-09T06:20:22.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615513+0000", 2023-10-09T06:20:22.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6416999999999999e-05 2023-10-09T06:20:22.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.603565+0000", 2023-10-09T06:20:22.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98805211299999995 2023-10-09T06:20:22.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.904 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.603575+0000", 2023-10-09T06:20:22.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2830000000000007e-06 2023-10-09T06:20:22.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.904 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.904 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.853707+0000", 2023-10-09T06:20:22.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2501325670000001 2023-10-09T06:20:22.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.904 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:22.904 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.853838+0000", 2023-10-09T06:20:22.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013037199999999999 2023-10-09T06:20:22.905 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.905 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.905 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1,6", 2023-10-09T06:20:22.905 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.853976+0000", 2023-10-09T06:20:22.905 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000138868 2023-10-09T06:20:22.905 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.905 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.905 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:22.905 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.857759+0000", 2023-10-09T06:20:22.905 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037828190000000002 2023-10-09T06:20:22.905 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.905 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.906 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.906 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.866740+0000", 2023-10-09T06:20:22.906 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0089810029999999996 2023-10-09T06:20:22.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.906 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.906 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.880757+0000", 2023-10-09T06:20:22.906 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014017201 2023-10-09T06:20:22.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.906 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.906 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.892300+0000", 2023-10-09T06:20:22.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011542663999999999 2023-10-09T06:20:22.907 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.907 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:22.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.892351+0000", 2023-10-09T06:20:22.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0479999999999998e-05 2023-10-09T06:20:22.907 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.907 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:22.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.892462+0000", 2023-10-09T06:20:22.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011181 2023-10-09T06:20:22.908 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.908 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:22.908 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.908 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.908 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.908 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:114798 11.8 11:18050a8d:::benchmark_data_smithi064_631903_object5884:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-10-09T06:20:22.908 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:17:46.615516+0000", 2023-10-09T06:20:22.908 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 156.15258842899999, 2023-10-09T06:20:22.908 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2070446920000002, 2023-10-09T06:20:22.908 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:22.908 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:22.908 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:22.909 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:20:22.909 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:22.909 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 114798 2023-10-09T06:20:22.909 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.909 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:22.909 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.909 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:22.909 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615516+0000", 2023-10-09T06:20:22.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.909 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.909 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.910 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:22.910 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615516+0000", 2023-10-09T06:20:22.910 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.910 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.910 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:22.910 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615516+0000", 2023-10-09T06:20:22.910 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:22.910 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.911 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:22.911 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615520+0000", 2023-10-09T06:20:22.911 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5029999999999999e-06 2023-10-09T06:20:22.911 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.911 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.911 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:22.911 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615537+0000", 2023-10-09T06:20:22.911 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6646999999999998e-05 2023-10-09T06:20:22.911 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.911 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.911 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.911 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615542+0000", 2023-10-09T06:20:22.912 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9989999999999999e-06 2023-10-09T06:20:22.912 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.912 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.912 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.912 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615600+0000", 2023-10-09T06:20:22.912 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8559e-05 2023-10-09T06:20:22.912 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.912 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.912 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.912 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:46.615614+0000", 2023-10-09T06:20:22.912 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3192e-05 2023-10-09T06:20:22.913 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.913 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.913 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.913 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.599195+0000", 2023-10-09T06:20:22.913 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98358131199999999 2023-10-09T06:20:22.913 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.913 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.913 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.913 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:47.599208+0000", 2023-10-09T06:20:22.913 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3329999999999999e-05 2023-10-09T06:20:22.913 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.913 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.914 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.914 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.803002+0000", 2023-10-09T06:20:22.914 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2037934539999999 2023-10-09T06:20:22.914 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.914 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:22.914 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.803120+0000", 2023-10-09T06:20:22.914 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118214 2023-10-09T06:20:22.914 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.914 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T06:20:22.914 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.803665+0000", 2023-10-09T06:20:22.915 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00054509399999999998 2023-10-09T06:20:22.915 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.915 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.915 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.915 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.820553+0000", 2023-10-09T06:20:22.915 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016888027999999999 2023-10-09T06:20:22.915 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.915 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.915 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:22.915 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.820799+0000", 2023-10-09T06:20:22.915 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00024631399999999997 2023-10-09T06:20:22.916 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.916 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.916 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.916 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.822353+0000", 2023-10-09T06:20:22.916 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001553376 2023-10-09T06:20:22.916 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.916 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.916 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.916 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.822391+0000", 2023-10-09T06:20:22.916 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7916000000000001e-05 2023-10-09T06:20:22.916 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.916 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.917 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:22.917 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.822450+0000", 2023-10-09T06:20:22.917 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9490999999999997e-05 2023-10-09T06:20:22.917 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.917 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.917 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:22.917 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:17:48.822561+0000", 2023-10-09T06:20:22.917 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0001109 2023-10-09T06:20:22.917 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.917 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:22.917 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.917 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.918 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.918 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:181985 11.8 11:12f0e639:::benchmark_data_smithi064_631903_object73071:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e623)", 2023-10-09T06:20:22.918 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:19:14.583243+0000", 2023-10-09T06:20:22.918 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 68.184861701000003, 2023-10-09T06:20:22.918 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0144160870000001, 2023-10-09T06:20:22.918 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:22.918 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:22.918 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:22.918 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:20:22.918 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:22.919 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 181985 2023-10-09T06:20:22.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.919 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:22.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.919 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:22.919 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:14.583243+0000", 2023-10-09T06:20:22.919 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.919 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:22.919 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:14.583243+0000", 2023-10-09T06:20:22.919 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.920 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.920 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.920 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:22.920 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:14.583242+0000", 2023-10-09T06:20:22.920 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:22.920 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.920 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.920 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:22.920 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:14.583248+0000", 2023-10-09T06:20:22.920 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4080000000000003e-06 2023-10-09T06:20:22.920 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.920 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.921 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:22.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:14.583266+0000", 2023-10-09T06:20:22.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8039000000000001e-05 2023-10-09T06:20:22.921 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.921 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.921 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:14.583274+0000", 2023-10-09T06:20:22.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3159999999999997e-06 2023-10-09T06:20:22.921 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.921 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.921 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.922 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:14.583337+0000", 2023-10-09T06:20:22.922 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2786000000000005e-05 2023-10-09T06:20:22.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.922 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.922 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:14.583350+0000", 2023-10-09T06:20:22.922 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2985e-05 2023-10-09T06:20:22.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.922 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.922 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:15.583668+0000", 2023-10-09T06:20:22.922 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.000317417 2023-10-09T06:20:22.923 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.923 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.923 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:22.923 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:15.583784+0000", 2023-10-09T06:20:22.923 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116629 2023-10-09T06:20:22.923 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.923 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.923 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:20:22.923 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:15.583863+0000", 2023-10-09T06:20:22.923 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8775000000000004e-05 2023-10-09T06:20:22.923 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.924 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.924 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.924 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:15.591995+0000", 2023-10-09T06:20:22.924 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0081324169999999994 2023-10-09T06:20:22.924 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.924 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.924 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:22.924 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:15.592006+0000", 2023-10-09T06:20:22.924 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0181e-05 2023-10-09T06:20:22.924 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.924 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.924 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.925 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:15.597596+0000", 2023-10-09T06:20:22.925 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0055901290000000001 2023-10-09T06:20:22.925 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.925 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.925 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:22.925 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:15.597633+0000", 2023-10-09T06:20:22.925 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7329000000000001e-05 2023-10-09T06:20:22.925 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.925 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.925 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:22.925 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:15.597659+0000", 2023-10-09T06:20:22.926 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.614e-05 2023-10-09T06:20:22.926 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.926 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:22.926 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.926 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.926 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.926 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:181992 11.8 11:138f47b9:::benchmark_data_smithi064_631903_object73078:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e623)", 2023-10-09T06:20:22.926 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:19:14.584853+0000", 2023-10-09T06:20:22.926 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 68.183251628999997, 2023-10-09T06:20:22.926 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.013040148, 2023-10-09T06:20:22.926 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:22.926 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:22.927 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:22.927 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:20:22.927 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:22.927 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 181992 2023-10-09T06:20:22.927 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.927 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:22.927 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.927 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:22.927 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:14.584853+0000", 2023-10-09T06:20:22.927 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.927 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.928 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.928 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:22.928 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:14.584853+0000", 2023-10-09T06:20:22.928 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.928 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.928 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.928 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:22.928 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:14.584852+0000", 2023-10-09T06:20:22.928 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:20:22.928 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.928 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.928 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:22.929 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:14.584858+0000", 2023-10-09T06:20:22.929 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9120000000000003e-06 2023-10-09T06:20:22.929 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.929 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.929 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:22.929 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:14.584870+0000", 2023-10-09T06:20:22.929 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1905e-05 2023-10-09T06:20:22.929 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.929 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.929 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.929 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:14.584876+0000", 2023-10-09T06:20:22.930 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8390000000000003e-06 2023-10-09T06:20:22.930 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.930 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.930 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.930 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:14.584935+0000", 2023-10-09T06:20:22.930 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8959999999999998e-05 2023-10-09T06:20:22.930 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.930 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.930 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.930 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:14.584947+0000", 2023-10-09T06:20:22.930 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2311e-05 2023-10-09T06:20:22.931 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.931 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.931 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.931 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:15.586384+0000", 2023-10-09T06:20:22.931 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0014373569999999 2023-10-09T06:20:22.931 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.931 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.931 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:22.931 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:15.586459+0000", 2023-10-09T06:20:22.931 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5035000000000002e-05 2023-10-09T06:20:22.931 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.931 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.932 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:20:22.932 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:15.586534+0000", 2023-10-09T06:20:22.932 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4528000000000002e-05 2023-10-09T06:20:22.932 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.932 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.932 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.932 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:15.592066+0000", 2023-10-09T06:20:22.932 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0055320730000000002 2023-10-09T06:20:22.932 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.932 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.932 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:22.932 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:15.595671+0000", 2023-10-09T06:20:22.933 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0036052789999999999 2023-10-09T06:20:22.933 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.933 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.933 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.933 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:15.597777+0000", 2023-10-09T06:20:22.933 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0021057039999999999 2023-10-09T06:20:22.933 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.933 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.933 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:22.933 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:15.597841+0000", 2023-10-09T06:20:22.933 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3733999999999997e-05 2023-10-09T06:20:22.933 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.934 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.934 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:22.934 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:15.597893+0000", 2023-10-09T06:20:22.934 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2537999999999997e-05 2023-10-09T06:20:22.934 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.934 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:22.934 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.934 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.934 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.934 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:182035 11.8 11:18fec023:::benchmark_data_smithi064_631903_object73121:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e623)", 2023-10-09T06:20:22.934 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:19:14.633336+0000", 2023-10-09T06:20:22.934 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 68.134768797999996, 2023-10-09T06:20:22.935 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96470482099999999, 2023-10-09T06:20:22.935 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:22.935 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:22.935 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:22.935 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:20:22.935 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:22.935 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 182035 2023-10-09T06:20:22.935 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.935 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:22.935 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:22.935 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:14.633336+0000", 2023-10-09T06:20:22.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.936 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.936 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.936 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:22.936 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:14.633336+0000", 2023-10-09T06:20:22.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.936 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.936 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.936 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:22.936 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:14.633335+0000", 2023-10-09T06:20:22.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:22.936 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.937 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:22.937 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:14.633339+0000", 2023-10-09T06:20:22.937 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1300000000000003e-06 2023-10-09T06:20:22.937 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.937 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:22.937 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:14.633355+0000", 2023-10-09T06:20:22.937 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5624000000000002e-05 2023-10-09T06:20:22.937 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.938 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.938 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:14.633362+0000", 2023-10-09T06:20:22.938 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3640000000000001e-06 2023-10-09T06:20:22.938 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.938 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.938 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.938 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:14.633412+0000", 2023-10-09T06:20:22.938 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9670999999999999e-05 2023-10-09T06:20:22.938 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.938 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.938 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.938 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:14.633422+0000", 2023-10-09T06:20:22.939 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0186999999999999e-05 2023-10-09T06:20:22.939 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.939 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.939 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.939 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:15.588640+0000", 2023-10-09T06:20:22.939 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95521758400000001 2023-10-09T06:20:22.939 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.939 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.939 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:22.939 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:15.588784+0000", 2023-10-09T06:20:22.939 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000143786 2023-10-09T06:20:22.940 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.940 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:20:22.940 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:15.588913+0000", 2023-10-09T06:20:22.940 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012964300000000001 2023-10-09T06:20:22.940 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.940 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.940 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:15.592214+0000", 2023-10-09T06:20:22.940 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003301048 2023-10-09T06:20:22.940 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.941 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:22.941 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:15.595681+0000", 2023-10-09T06:20:22.941 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034666860000000001 2023-10-09T06:20:22.941 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.941 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.941 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.941 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:15.597954+0000", 2023-10-09T06:20:22.941 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0022729199999999999 2023-10-09T06:20:22.941 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.941 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.941 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:22.941 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:15.598001+0000", 2023-10-09T06:20:22.942 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7481000000000002e-05 2023-10-09T06:20:22.942 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.942 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.942 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:22.942 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:15.598041+0000", 2023-10-09T06:20:22.942 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9273999999999997e-05 2023-10-09T06:20:22.942 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.942 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:22.942 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.942 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.943 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.943 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGRecoveryDeleteReply(11.a e629,612 11:58151a52:::benchmark_data_smithi064_631903_object27097:head,607'8445)", 2023-10-09T06:20:22.943 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:19:38.992322+0000", 2023-10-09T06:20:22.943 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 43.775782958000001, 2023-10-09T06:20:22.943 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26333904899999999, 2023-10-09T06:20:22.943 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:22.943 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-10-09T06:20:22.943 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:22.943 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.943 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:22.943 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:38.992322+0000", 2023-10-09T06:20:22.943 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.944 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.944 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.944 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:22.944 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:38.992322+0000", 2023-10-09T06:20:22.944 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.944 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.944 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.944 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:22.944 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:38.992321+0000", 2023-10-09T06:20:22.944 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:20:22.944 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.944 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.945 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:22.945 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:38.992330+0000", 2023-10-09T06:20:22.945 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8289999999999997e-06 2023-10-09T06:20:22.945 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.945 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.945 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:22.945 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:39.148815+0000", 2023-10-09T06:20:22.945 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.156485666 2023-10-09T06:20:22.945 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.945 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.945 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.946 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:39.148826+0000", 2023-10-09T06:20:22.946 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0927999999999999e-05 2023-10-09T06:20:22.946 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.946 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.946 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.946 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:39.255321+0000", 2023-10-09T06:20:22.946 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.106494677 2023-10-09T06:20:22.946 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.946 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.946 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:22.946 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:39.255661+0000", 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00033992099999999997 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36591.0:213527 11.1 11:818a5502:::benchmark_data_smithi064_631903_object104613:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e633)", 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:19:54.089069+0000", 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 28.679035846000001, 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26195664299999999, 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:22.948 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:20:22.948 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36591", 2023-10-09T06:20:22.948 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:22.948 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 213527 2023-10-09T06:20:22.948 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.948 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:20:22.948 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.948 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:20:22.948 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:54.089069+0000", 2023-10-09T06:20:22.948 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.948 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.948 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.949 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:20:22.949 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:54.089069+0000", 2023-10-09T06:20:22.949 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:20:22.949 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.949 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.949 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:20:22.949 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:54.089067+0000", 2023-10-09T06:20:22.949 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:22.949 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.949 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.949 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:20:22.950 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:54.089078+0000", 2023-10-09T06:20:22.950 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0314e-05 2023-10-09T06:20:22.950 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.950 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.950 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:20:22.950 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:54.089091+0000", 2023-10-09T06:20:22.950 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2978e-05 2023-10-09T06:20:22.950 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.950 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.950 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.950 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:54.089099+0000", 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8270000000000004e-06 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:54.091321+0000", 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0022214299999999999 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:54.091460+0000", 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013862999999999999 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.952 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.952 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2023-10-09T06:20:22.952 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:54.091581+0000", 2023-10-09T06:20:22.952 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000121369 2023-10-09T06:20:22.952 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.952 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.952 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:20:22.952 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:54.350724+0000", 2023-10-09T06:20:22.952 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25914343699999998 2023-10-09T06:20:22.952 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.952 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.952 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.953 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:54.350758+0000", 2023-10-09T06:20:22.953 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3543000000000001e-05 2023-10-09T06:20:22.953 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.953 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.953 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.953 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:54.350933+0000", 2023-10-09T06:20:22.953 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000175169 2023-10-09T06:20:22.953 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.953 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.953 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:20:22.954 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:54.350985+0000", 2023-10-09T06:20:22.954 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1480000000000002e-05 2023-10-09T06:20:22.954 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:20:22.954 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:20:22.954 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:20:22.954 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:19:54.351026+0000", 2023-10-09T06:20:22.954 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0976000000000002e-05 2023-10-09T06:20:22.954 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.954 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:22.954 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.954 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:20:22.954 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:20:22.955 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:20:22.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:22.956 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:22.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:23.057 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:23.085 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:23.101 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:22 smithi064 ceph-mon[212750]: pgmap v1656: 135 pgs: 6 creating+peering, 6 unknown, 2 remapped+peering, 121 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 79 KiB/s rd, 21 KiB/s wr, 119 op/s; 0 B/s, 2 keys/s, 0 objects/s recovering 2023-10-09T06:20:23.101 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:22 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 11 pgs peering) 2023-10-09T06:20:23.102 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:22 smithi064 ceph-mon[220044]: pgmap v1656: 135 pgs: 6 creating+peering, 6 unknown, 2 remapped+peering, 121 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 79 KiB/s rd, 21 KiB/s wr, 119 op/s; 0 B/s, 2 keys/s, 0 objects/s recovering 2023-10-09T06:20:23.102 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:22 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 11 pgs peering) 2023-10-09T06:20:23.129 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:23.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:23.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:23.236 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:23.267 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:23.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:22 smithi156 ceph-mon[195101]: pgmap v1656: 135 pgs: 6 creating+peering, 6 unknown, 2 remapped+peering, 121 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 79 KiB/s rd, 21 KiB/s wr, 119 op/s; 0 B/s, 2 keys/s, 0 objects/s recovering 2023-10-09T06:20:23.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:22 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 11 pgs peering) 2023-10-09T06:20:23.571 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:23.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:23.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:23.676 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:23.707 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:23.957 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.mirror_image_status (3472 ms) 2023-10-09T06:20:23.957 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-10-09T06:20:24.042 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.mirror_image_map (85 ms) 2023-10-09T06:20:24.042 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-10-09T06:20:24.050 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.mirror_instances (8 ms) 2023-10-09T06:20:24.050 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-10-09T06:20:24.061 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.mirror_snapshot (11 ms) 2023-10-09T06:20:24.062 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-10-09T06:20:24.063 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2023-10-09T06:20:24.063 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-10-09T06:20:24.065 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2023-10-09T06:20:24.065 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-10-09T06:20:24.068 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2023-10-09T06:20:24.069 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-10-09T06:20:24.074 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_dir_rename (6 ms) 2023-10-09T06:20:24.075 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-10-09T06:20:24.078 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2023-10-09T06:20:24.078 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-10-09T06:20:24.082 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2023-10-09T06:20:24.082 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_image_add 2023-10-09T06:20:24.084 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_image_add (3 ms) 2023-10-09T06:20:24.084 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-10-09T06:20:24.087 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2023-10-09T06:20:24.087 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_image_list 2023-10-09T06:20:24.090 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2023-10-09T06:20:24.090 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-10-09T06:20:24.094 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2023-10-09T06:20:24.094 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.image_group_add 2023-10-09T06:20:24.096 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2023-10-09T06:20:24.097 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-10-09T06:20:24.099 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2023-10-09T06:20:24.099 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.image_group_get 2023-10-09T06:20:24.102 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2023-10-09T06:20:24.102 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-10-09T06:20:24.103 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2023-10-09T06:20:24.104 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-10-09T06:20:24.105 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2023-10-09T06:20:24.105 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-10-09T06:20:24.108 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (2 ms) 2023-10-09T06:20:24.108 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-10-09T06:20:24.110 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2023-10-09T06:20:24.110 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-10-09T06:20:24.112 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_set (2 ms) 2023-10-09T06:20:24.112 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-10-09T06:20:24.115 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2023-10-09T06:20:24.115 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-10-09T06:20:24.130 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (15 ms) 2023-10-09T06:20:24.130 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-10-09T06:20:24.296 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 5, 7, 1, 6] out_osds: [2, 4] dead_osds: [] live_osds: [1, 2, 3, 6, 7, 4, 0, 5] 2023-10-09T06:20:24.352 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:20:24.353 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:20:24.353 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 0.892980 2023-10-09T06:20:24.353 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 0.8929800346024579 2023-10-09T06:20:24.427 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (297 ms) 2023-10-09T06:20:24.427 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-10-09T06:20:24.431 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_remove (3 ms) 2023-10-09T06:20:24.431 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-10-09T06:20:24.433 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2023-10-09T06:20:24.433 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.trash_methods 2023-10-09T06:20:24.441 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.trash_methods (8 ms) 2023-10-09T06:20:24.441 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.op_features 2023-10-09T06:20:24.449 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.op_features (9 ms) 2023-10-09T06:20:24.449 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.clone_parent 2023-10-09T06:20:24.470 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.clone_parent (21 ms) 2023-10-09T06:20:24.470 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-10-09T06:20:24.478 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.clone_parent_ns (7 ms) 2023-10-09T06:20:24.478 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.clone_child 2023-10-09T06:20:24.490 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.clone_child (13 ms) 2023-10-09T06:20:24.490 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-10-09T06:20:24.496 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.namespace_methods (6 ms) 2023-10-09T06:20:24.496 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.migration 2023-10-09T06:20:24.505 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.migration (9 ms) 2023-10-09T06:20:24.505 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.migration_v1 2023-10-09T06:20:24.514 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.migration_v1 (9 ms) 2023-10-09T06:20:24.514 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-10-09T06:20:24.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:24 smithi064 ceph-mon[212750]: pgmap v1657: 135 pgs: 6 creating+peering, 129 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 70 KiB/s rd, 23 KiB/s wr, 98 op/s; 0 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T06:20:24.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:24 smithi064 ceph-mon[220044]: pgmap v1657: 135 pgs: 6 creating+peering, 129 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 70 KiB/s rd, 23 KiB/s wr, 98 op/s; 0 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T06:20:25.211 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:25.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:25.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:25.271 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:20:25.297 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:24 smithi156 ceph-mon[195101]: pgmap v1657: 135 pgs: 6 creating+peering, 129 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 70 KiB/s rd, 23 KiB/s wr, 98 op/s; 0 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T06:20:25.317 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:25.344 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:25.388 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:25.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:25.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:25.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:20:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:20:25] "GET /metrics HTTP/1.1" 200 34042 "" "Prometheus/2.43.0" 2023-10-09T06:20:25.495 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:25.522 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:25.589 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:25.619 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:25.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:25.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:25.632 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:20:25.663 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:20:25.685 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:20:25.706 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:20:25.725 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:25.726 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:20:25.745 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:20:25.751 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:25.764 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:20:25.784 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:20:25.803 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:20:25.804 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:20:25.804 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:20:25.804 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:20:25.804 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:20:25.804 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:20:25.805 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:20:25.805 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:20:25.805 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:20:25.846 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:25.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:25.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:25.937 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1422 ms) 2023-10-09T06:20:25.937 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.sparsify 2023-10-09T06:20:25.951 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:25.982 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.sparsify (46 ms) 2023-10-09T06:20:25.990 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:26.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:25 smithi064 ceph-mon[212750]: osdmap e643: 8 total, 8 up, 6 in 2023-10-09T06:20:26.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:25 smithi064 ceph-mon[212750]: failed to encode map e643 with expected crc 2023-10-09T06:20:26.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:25 smithi064 ceph-mon[212750]: failed to encode map e643 with expected crc 2023-10-09T06:20:26.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:25 smithi064 ceph-mon[212750]: failed to encode map e643 with expected crc 2023-10-09T06:20:26.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:25 smithi064 ceph-mon[212750]: failed to encode map e643 with expected crc 2023-10-09T06:20:26.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:25 smithi064 ceph-mon[212750]: failed to encode map e643 with expected crc 2023-10-09T06:20:26.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:25 smithi064 ceph-mon[212750]: failed to encode map e643 with expected crc 2023-10-09T06:20:26.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:25 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3065977369' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:26.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:25 smithi064 ceph-mon[220044]: osdmap e643: 8 total, 8 up, 6 in 2023-10-09T06:20:26.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:25 smithi064 ceph-mon[220044]: failed to encode map e643 with expected crc 2023-10-09T06:20:26.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:25 smithi064 ceph-mon[220044]: failed to encode map e643 with expected crc 2023-10-09T06:20:26.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:25 smithi064 ceph-mon[220044]: failed to encode map e643 with expected crc 2023-10-09T06:20:26.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:25 smithi064 ceph-mon[220044]: failed to encode map e643 with expected crc 2023-10-09T06:20:26.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:25 smithi064 ceph-mon[220044]: failed to encode map e643 with expected crc 2023-10-09T06:20:26.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:25 smithi064 ceph-mon[220044]: failed to encode map e643 with expected crc 2023-10-09T06:20:26.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:25 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3065977369' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:26.247 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:25 smithi156 ceph-mon[195101]: osdmap e643: 8 total, 8 up, 6 in 2023-10-09T06:20:26.247 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:25 smithi156 ceph-mon[195101]: failed to encode map e643 with expected crc 2023-10-09T06:20:26.247 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:25 smithi156 ceph-mon[195101]: failed to encode map e643 with expected crc 2023-10-09T06:20:26.247 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:25 smithi156 ceph-mon[195101]: failed to encode map e643 with expected crc 2023-10-09T06:20:26.247 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:25 smithi156 ceph-mon[195101]: failed to encode map e643 with expected crc 2023-10-09T06:20:26.247 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:25 smithi156 ceph-mon[195101]: failed to encode map e643 with expected crc 2023-10-09T06:20:26.247 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:25 smithi156 ceph-mon[195101]: failed to encode map e643 with expected crc 2023-10-09T06:20:26.247 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:25 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3065977369' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:26.291 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:26.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:26.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:26.395 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:26.424 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:26.949 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:20:26.950 INFO:tasks.workunit.client.0.smithi064.stdout:[----------] 70 tests from TestClsRbd (6128 ms total) 2023-10-09T06:20:26.950 INFO:tasks.workunit.client.0.smithi064.stdout: 2023-10-09T06:20:26.950 INFO:tasks.workunit.client.0.smithi064.stdout:[----------] Global test environment tear-down 2023-10-09T06:20:26.950 INFO:tasks.workunit.client.0.smithi064.stdout:[==========] 70 tests from 1 test suite ran. (9091 ms total) 2023-10-09T06:20:26.950 INFO:tasks.workunit.client.0.smithi064.stdout:[ PASSED ] 70 tests. 2023-10-09T06:20:26.951 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T06:20:26.951 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T06:20:27.017 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:26 smithi064 ceph-mon[220044]: pgmap v1659: 135 pgs: 135 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 2.9 MiB/s rd, 1.4 MiB/s wr, 2.32k op/s; 0 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T06:20:27.017 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:26 smithi064 ceph-mon[220044]: failed to encode map e643 with expected crc 2023-10-09T06:20:27.018 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:26 smithi064 ceph-mon[220044]: osdmap e644: 8 total, 8 up, 6 in 2023-10-09T06:20:27.018 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:26 smithi064 ceph-mon[220044]: failed to encode map e644 with expected crc 2023-10-09T06:20:27.018 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:26 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1556634265' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:20:27.018 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:26 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:20:27.018 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:26 smithi064 ceph-mon[220044]: failed to encode map e644 with expected crc 2023-10-09T06:20:27.018 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:26 smithi064 ceph-mon[212750]: pgmap v1659: 135 pgs: 135 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 2.9 MiB/s rd, 1.4 MiB/s wr, 2.32k op/s; 0 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T06:20:27.018 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:26 smithi064 ceph-mon[212750]: failed to encode map e643 with expected crc 2023-10-09T06:20:27.018 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:26 smithi064 ceph-mon[212750]: osdmap e644: 8 total, 8 up, 6 in 2023-10-09T06:20:27.018 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:26 smithi064 ceph-mon[212750]: failed to encode map e644 with expected crc 2023-10-09T06:20:27.018 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:26 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1556634265' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:20:27.019 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:26 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:20:27.019 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:26 smithi064 ceph-mon[212750]: failed to encode map e644 with expected crc 2023-10-09T06:20:27.021 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-10-09T06:20:27.022 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T06:20:27.208 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:26 smithi156 ceph-mon[195101]: pgmap v1659: 135 pgs: 135 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 2.9 MiB/s rd, 1.4 MiB/s wr, 2.32k op/s; 0 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T06:20:27.208 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:26 smithi156 ceph-mon[195101]: failed to encode map e643 with expected crc 2023-10-09T06:20:27.208 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:26 smithi156 ceph-mon[195101]: osdmap e644: 8 total, 8 up, 6 in 2023-10-09T06:20:27.208 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:26 smithi156 ceph-mon[195101]: failed to encode map e644 with expected crc 2023-10-09T06:20:27.208 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:26 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1556634265' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:20:27.209 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:26 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:20:27.209 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:26 smithi156 ceph-mon[195101]: failed to encode map e644 with expected crc 2023-10-09T06:20:27.417 DEBUG:teuthology.parallel:result is None 2023-10-09T06:20:27.417 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:20:27.429 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:27.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:27.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:27.446 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:20:27.446 DEBUG:teuthology.orchestra.run.smithi064:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:20:27.502 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:20:27.503 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T06:20:27.503 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-10-09T06:20:27.503 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T06:20:27.503 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-10-09T06:20:27.503 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T06:20:27.504 DEBUG:teuthology.orchestra.run.smithi064:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:20:27.533 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:27.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:27.560 INFO:teuthology.orchestra.run.smithi064.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T06:20:27.560 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:20:27.566 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:27.617 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T06:20:27.617 DEBUG:teuthology.orchestra.run.smithi064:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T06:20:27.675 INFO:tasks.workunit:timeout=3h 2023-10-09T06:20:27.675 INFO:tasks.workunit:cleanup=True 2023-10-09T06:20:27.676 DEBUG:teuthology.orchestra.run.smithi064:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T06:20:27.712 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:27.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:27.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:27.734 INFO:tasks.workunit.client.0.smithi064.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T06:20:27.779 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:20:27.779 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:20:27.816 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:27.844 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:28.042 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:28.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:28.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:28.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:27 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:28.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:27 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:20:28.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:27 smithi156 ceph-mon[195101]: osdmap e645: 8 total, 8 up, 6 in 2023-10-09T06:20:28.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:27 smithi156 ceph-mon[195101]: failed to encode map e645 with expected crc 2023-10-09T06:20:28.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:27 smithi156 ceph-mon[195101]: failed to encode map e645 with expected crc 2023-10-09T06:20:28.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:27 smithi156 ceph-mon[195101]: failed to encode map e645 with expected crc 2023-10-09T06:20:28.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:27 smithi156 ceph-mon[195101]: failed to encode map e645 with expected crc 2023-10-09T06:20:28.130 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:27 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:28.130 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:27 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:20:28.130 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:27 smithi064 ceph-mon[212750]: osdmap e645: 8 total, 8 up, 6 in 2023-10-09T06:20:28.130 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:27 smithi064 ceph-mon[212750]: failed to encode map e645 with expected crc 2023-10-09T06:20:28.130 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:27 smithi064 ceph-mon[212750]: failed to encode map e645 with expected crc 2023-10-09T06:20:28.130 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:27 smithi064 ceph-mon[212750]: failed to encode map e645 with expected crc 2023-10-09T06:20:28.131 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:27 smithi064 ceph-mon[212750]: failed to encode map e645 with expected crc 2023-10-09T06:20:28.131 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:27 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:28.131 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:27 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:20:28.131 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:27 smithi064 ceph-mon[220044]: osdmap e645: 8 total, 8 up, 6 in 2023-10-09T06:20:28.131 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:27 smithi064 ceph-mon[220044]: failed to encode map e645 with expected crc 2023-10-09T06:20:28.131 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:27 smithi064 ceph-mon[220044]: failed to encode map e645 with expected crc 2023-10-09T06:20:28.131 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:27 smithi064 ceph-mon[220044]: failed to encode map e645 with expected crc 2023-10-09T06:20:28.132 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:27 smithi064 ceph-mon[220044]: failed to encode map e645 with expected crc 2023-10-09T06:20:28.146 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:28.177 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:28.276 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:28.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:28.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:28.381 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:28.409 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:28.516 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:28.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:28.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:28.570 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:20:28.621 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:28.649 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:28.692 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:28.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:28.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:28.799 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:28.827 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:28.923 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:28.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:28.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:28.946 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.6 primary-affinity to 0.89298 (8585222) 2023-10-09T06:20:28.983 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:20:29.027 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:29.051 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:28 smithi156 ceph-mon[195101]: pgmap v1662: 103 pgs: 103 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:20:29.051 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:28 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/847000183' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.8929800346024579}]: dispatch 2023-10-09T06:20:29.051 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:28 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:29.054 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:29.117 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:29.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:29.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:29.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:28 smithi064 ceph-mon[212750]: pgmap v1662: 103 pgs: 103 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:20:29.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:28 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/847000183' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.8929800346024579}]: dispatch 2023-10-09T06:20:29.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:28 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:29.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:28 smithi064 ceph-mon[220044]: pgmap v1662: 103 pgs: 103 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:20:29.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:28 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/847000183' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.8929800346024579}]: dispatch 2023-10-09T06:20:29.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:28 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:29.221 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:29.250 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:29.295 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:29.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:29.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:29.399 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:29.428 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:29.524 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:29.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:29.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:29.627 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:29.655 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:29.665 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:20:29.731 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:29.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:29.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:29.835 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:29.864 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:29.897 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:20:29.917 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:29.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:29.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:30.027 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:30.062 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:30.124 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:30.171 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:30.176 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:29 smithi064 ceph-mon[212750]: failed to encode map e646 with expected crc 2023-10-09T06:20:30.176 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:29 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/847000183' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.8929800346024579}]': finished 2023-10-09T06:20:30.176 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:29 smithi064 ceph-mon[212750]: osdmap e646: 8 total, 8 up, 6 in 2023-10-09T06:20:30.176 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:29 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:30.177 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:29 smithi064 ceph-mon[220044]: failed to encode map e646 with expected crc 2023-10-09T06:20:30.177 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:29 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/847000183' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.8929800346024579}]': finished 2023-10-09T06:20:30.177 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:29 smithi064 ceph-mon[220044]: osdmap e646: 8 total, 8 up, 6 in 2023-10-09T06:20:30.177 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:29 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:30.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:30.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:30.279 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:30.305 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:29 smithi156 ceph-mon[195101]: failed to encode map e646 with expected crc 2023-10-09T06:20:30.305 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:29 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/847000183' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.8929800346024579}]': finished 2023-10-09T06:20:30.305 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:29 smithi156 ceph-mon[195101]: osdmap e646: 8 total, 8 up, 6 in 2023-10-09T06:20:30.305 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:29 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:30.308 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:30.356 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:30.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:30.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:30.460 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:30.492 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:30.542 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:20:30.542 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:20:31.057 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:31.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:31.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:31.161 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:31.178 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:20:31.191 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:31.204 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:30 smithi156 ceph-mon[195101]: pgmap v1664: 103 pgs: 103 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:20:31.204 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:30 smithi156 ceph-mon[195101]: failed to encode map e647 with expected crc 2023-10-09T06:20:31.204 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:30 smithi156 ceph-mon[195101]: osdmap e647: 8 total, 8 up, 6 in 2023-10-09T06:20:31.204 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:30 smithi156 ceph-mon[195101]: failed to encode map e647 with expected crc 2023-10-09T06:20:31.204 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:30 smithi156 ceph-mon[195101]: failed to encode map e647 with expected crc 2023-10-09T06:20:31.204 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:30 smithi156 ceph-mon[195101]: failed to encode map e647 with expected crc 2023-10-09T06:20:31.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:30 smithi064 ceph-mon[220044]: pgmap v1664: 103 pgs: 103 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:20:31.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:30 smithi064 ceph-mon[220044]: failed to encode map e647 with expected crc 2023-10-09T06:20:31.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:30 smithi064 ceph-mon[220044]: osdmap e647: 8 total, 8 up, 6 in 2023-10-09T06:20:31.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:30 smithi064 ceph-mon[220044]: failed to encode map e647 with expected crc 2023-10-09T06:20:31.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:30 smithi064 ceph-mon[220044]: failed to encode map e647 with expected crc 2023-10-09T06:20:31.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:30 smithi064 ceph-mon[220044]: failed to encode map e647 with expected crc 2023-10-09T06:20:31.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:30 smithi064 ceph-mon[212750]: pgmap v1664: 103 pgs: 103 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:20:31.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:30 smithi064 ceph-mon[212750]: failed to encode map e647 with expected crc 2023-10-09T06:20:31.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:30 smithi064 ceph-mon[212750]: osdmap e647: 8 total, 8 up, 6 in 2023-10-09T06:20:31.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:30 smithi064 ceph-mon[212750]: failed to encode map e647 with expected crc 2023-10-09T06:20:31.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:30 smithi064 ceph-mon[212750]: failed to encode map e647 with expected crc 2023-10-09T06:20:31.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:30 smithi064 ceph-mon[212750]: failed to encode map e647 with expected crc 2023-10-09T06:20:31.241 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:31.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:31.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:31.345 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:31.372 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:31.418 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:31.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:31.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:31.522 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:31.526 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:20:31.551 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:31.659 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:31.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:31.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:31.763 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:31.792 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:31.824 DEBUG:teuthology.orchestra.run.smithi064:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T06:20:31.835 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:31.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:31.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:31.841 INFO:tasks.workunit.client.0.smithi064.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T06:20:31.843 INFO:tasks.workunit.client.0.smithi064.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T06:20:31.844 INFO:tasks.workunit.client.0.smithi064.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T06:20:31.859 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:31.903 INFO:tasks.workunit.client.0.smithi064.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T06:20:31.939 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:31.965 INFO:tasks.workunit.client.0.smithi064.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T06:20:31.970 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:32.012 INFO:tasks.workunit.client.0.smithi064.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T06:20:32.014 INFO:tasks.workunit.client.0.smithi064.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T06:20:32.014 INFO:tasks.workunit.client.0.smithi064.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T06:20:32.058 INFO:tasks.workunit.client.0.smithi064.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T06:20:32.061 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-10-09T06:20:32.061 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T06:20:32.119 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-10-09T06:20:32.120 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-10-09T06:20:32.121 DEBUG:teuthology.orchestra.run.smithi064:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-10-09T06:20:32.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:31 smithi156 ceph-mon[195101]: failed to encode map e647 with expected crc 2023-10-09T06:20:32.124 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:31 smithi064 ceph-mon[220044]: failed to encode map e647 with expected crc 2023-10-09T06:20:32.125 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:31 smithi064 ceph-mon[212750]: failed to encode map e647 with expected crc 2023-10-09T06:20:32.137 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:32.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:32.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:32.147 INFO:tasks.workunit.client.0.smithi064.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-10-09T06:20:32.147 INFO:tasks.workunit.client.0.smithi064.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-10-09T06:20:32.147 INFO:tasks.workunit.client.0.smithi064.stderr:+ TMPDIR=/tmp/rbd_import_export_695299 2023-10-09T06:20:32.147 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm -rf /tmp/rbd_import_export_695299 2023-10-09T06:20:32.147 INFO:tasks.workunit.client.0.smithi064.stderr:+ mkdir /tmp/rbd_import_export_695299 2023-10-09T06:20:32.148 INFO:tasks.workunit.client.0.smithi064.stderr:+ trap 'rm -rf /tmp/rbd_import_export_695299' INT TERM EXIT 2023-10-09T06:20:32.148 INFO:tasks.workunit.client.0.smithi064.stderr:+ mkdir foo.695299 2023-10-09T06:20:32.149 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import foo.695299 foo.dir 2023-10-09T06:20:32.242 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:32.270 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:32.370 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:32.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:32.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:32.397 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: cannot import a directory 2023-10-09T06:20:32.397 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 0% complete...failed. 2023-10-09T06:20:32.397 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: import failed: (21) Is a directory 2023-10-09T06:20:32.399 INFO:tasks.workunit.client.0.smithi064.stderr:+ true 2023-10-09T06:20:32.400 INFO:tasks.workunit.client.0.smithi064.stderr:+ rmdir foo.695299 2023-10-09T06:20:32.400 INFO:tasks.workunit.client.0.smithi064.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_695299/img bs=1k count=1 seek=10 2023-10-09T06:20:32.401 INFO:tasks.workunit.client.0.smithi064.stderr:1+0 records in 2023-10-09T06:20:32.402 INFO:tasks.workunit.client.0.smithi064.stderr:1+0 records out 2023-10-09T06:20:32.402 INFO:tasks.workunit.client.0.smithi064.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 5.6108e-05 s, 18.3 MB/s 2023-10-09T06:20:32.402 INFO:tasks.workunit.client.0.smithi064.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_695299/img bs=1k count=10 seek=100 2023-10-09T06:20:32.402 INFO:tasks.workunit.client.0.smithi064.stderr:10+0 records in 2023-10-09T06:20:32.402 INFO:tasks.workunit.client.0.smithi064.stderr:10+0 records out 2023-10-09T06:20:32.403 INFO:tasks.workunit.client.0.smithi064.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.2102e-05 s, 165 MB/s 2023-10-09T06:20:32.403 INFO:tasks.workunit.client.0.smithi064.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_695299/img bs=1k count=100 seek=1000 2023-10-09T06:20:32.404 INFO:tasks.workunit.client.0.smithi064.stderr:70+1 records in 2023-10-09T06:20:32.404 INFO:tasks.workunit.client.0.smithi064.stderr:70+1 records out 2023-10-09T06:20:32.404 INFO:tasks.workunit.client.0.smithi064.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.000289878 s, 249 MB/s 2023-10-09T06:20:32.404 INFO:tasks.workunit.client.0.smithi064.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_695299/img bs=1k seek=10000 2023-10-09T06:20:32.405 INFO:tasks.workunit.client.0.smithi064.stderr:139+1 records in 2023-10-09T06:20:32.405 INFO:tasks.workunit.client.0.smithi064.stderr:139+1 records out 2023-10-09T06:20:32.405 INFO:tasks.workunit.client.0.smithi064.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.000368566 s, 389 MB/s 2023-10-09T06:20:32.405 INFO:tasks.workunit.client.0.smithi064.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_695299/img bs=1k seek=100000 2023-10-09T06:20:32.439 INFO:tasks.workunit.client.0.smithi064.stderr:70+1 records in 2023-10-09T06:20:32.439 INFO:tasks.workunit.client.0.smithi064.stderr:70+1 records out 2023-10-09T06:20:32.439 INFO:tasks.workunit.client.0.smithi064.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.0331852 s, 2.2 MB/s 2023-10-09T06:20:32.439 INFO:tasks.workunit.client.0.smithi064.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_695299/img bs=1k seek=1000000 2023-10-09T06:20:32.440 INFO:tasks.workunit.client.0.smithi064.stderr:189+1 records in 2023-10-09T06:20:32.440 INFO:tasks.workunit.client.0.smithi064.stderr:189+1 records out 2023-10-09T06:20:32.440 INFO:tasks.workunit.client.0.smithi064.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.000486017 s, 399 MB/s 2023-10-09T06:20:32.440 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm testimg 2023-10-09T06:20:32.476 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:32.478 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 0% complete...failed. 2023-10-09T06:20:32.478 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T06:20:32.481 INFO:tasks.workunit.client.0.smithi064.stderr:+ true 2023-10-09T06:20:32.481 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --new-format /tmp/rbd_import_export_695299/img testimg 2023-10-09T06:20:32.505 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:32.608 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:32.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:32.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:32.685 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:20:32.704 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:20:32.712 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:32.722 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:20:32.742 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:32.742 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:20:32.761 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:20:32.779 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:20:32.785 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:32.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:32.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:32.797 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:20:32.816 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:20:32.818 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:20:32.818 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:20:32.818 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:20:32.818 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:20:32.818 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:20:32.818 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:20:32.818 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:20:32.819 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:20:32.889 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:32.917 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:33.015 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:33.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:33.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:33.072 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:20:33.073 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T06:20:33.073 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T06:20:33.073 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:20:33.073 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.073 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:182287 9.b 9:ddfaf3ee:::benchmark_data_smithi064_511911_object81185:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T06:20:33.073 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:10:49.739834+0000", 2023-10-09T06:20:33.073 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 583.33098339100002, 2023-10-09T06:20:33.073 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0692159409999999, 2023-10-09T06:20:33.073 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:33.073 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:33.074 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:33.074 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:20:33.074 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:20:33.074 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 182287 2023-10-09T06:20:33.074 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.074 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:33.074 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.074 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:33.074 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.739834+0000", 2023-10-09T06:20:33.074 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.074 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.075 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.075 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:33.075 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.739834+0000", 2023-10-09T06:20:33.075 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.075 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.075 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.075 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:33.075 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.739833+0000", 2023-10-09T06:20:33.075 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:33.075 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.075 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.075 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:33.076 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.739838+0000", 2023-10-09T06:20:33.076 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1120000000000004e-06 2023-10-09T06:20:33.076 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.076 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.076 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:33.076 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.739847+0000", 2023-10-09T06:20:33.076 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1349999999999998e-06 2023-10-09T06:20:33.076 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.076 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.076 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:33.076 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.739853+0000", 2023-10-09T06:20:33.077 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1639999999999998e-06 2023-10-09T06:20:33.077 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.077 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.077 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.077 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.743061+0000", 2023-10-09T06:20:33.077 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0032083229999999999 2023-10-09T06:20:33.077 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.077 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.077 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:33.077 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.743073+0000", 2023-10-09T06:20:33.077 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2092e-05 2023-10-09T06:20:33.077 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.078 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.078 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.078 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.791644+0000", 2023-10-09T06:20:33.078 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0485706990000001 2023-10-09T06:20:33.078 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.078 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.078 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:33.078 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.791803+0000", 2023-10-09T06:20:33.078 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015846600000000001 2023-10-09T06:20:33.078 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.078 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.079 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:20:33.079 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.791940+0000", 2023-10-09T06:20:33.079 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013713000000000001 2023-10-09T06:20:33.079 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.079 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.079 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:33.079 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.803203+0000", 2023-10-09T06:20:33.079 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011263451000000001 2023-10-09T06:20:33.079 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.079 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.079 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:33.079 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.807914+0000", 2023-10-09T06:20:33.080 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0047103630000000004 2023-10-09T06:20:33.080 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.080 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.080 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:33.080 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.807982+0000", 2023-10-09T06:20:33.080 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8454999999999994e-05 2023-10-09T06:20:33.080 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.080 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.080 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:33.081 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.808045+0000", 2023-10-09T06:20:33.081 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2638000000000006e-05 2023-10-09T06:20:33.081 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.081 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.081 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:33.081 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.809050+0000", 2023-10-09T06:20:33.081 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0010052349999999999 2023-10-09T06:20:33.081 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.081 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:33.081 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.081 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.081 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.082 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:182292 9.b 9:dc1a8787:::benchmark_data_smithi064_511911_object81190:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T06:20:33.082 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:10:49.753105+0000", 2023-10-09T06:20:33.082 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 583.31771242100001, 2023-10-09T06:20:33.082 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.072077706, 2023-10-09T06:20:33.082 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:33.082 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:33.082 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:33.082 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:20:33.082 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:20:33.082 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 182292 2023-10-09T06:20:33.083 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.083 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:33.083 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.083 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:33.083 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.753105+0000", 2023-10-09T06:20:33.083 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.083 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.083 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.083 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:33.083 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.753105+0000", 2023-10-09T06:20:33.083 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.083 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.084 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.084 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:33.084 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.753103+0000", 2023-10-09T06:20:33.084 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:33.084 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.084 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.084 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:33.084 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.753110+0000", 2023-10-09T06:20:33.084 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6379999999999998e-06 2023-10-09T06:20:33.084 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.084 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.084 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:33.085 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.753123+0000", 2023-10-09T06:20:33.085 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2986999999999999e-05 2023-10-09T06:20:33.085 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.085 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.085 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:33.085 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.753130+0000", 2023-10-09T06:20:33.085 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0899999999999999e-06 2023-10-09T06:20:33.085 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.085 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.085 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.085 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.753584+0000", 2023-10-09T06:20:33.085 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00045353800000000002 2023-10-09T06:20:33.086 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.086 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.086 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:33.086 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.753594+0000", 2023-10-09T06:20:33.086 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0025e-05 2023-10-09T06:20:33.086 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.086 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.086 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.086 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.798330+0000", 2023-10-09T06:20:33.086 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.044735881 2023-10-09T06:20:33.086 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.086 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.087 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:33.087 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.798467+0000", 2023-10-09T06:20:33.087 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000137341 2023-10-09T06:20:33.087 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.087 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.087 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:20:33.087 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.798603+0000", 2023-10-09T06:20:33.087 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013654499999999999 2023-10-09T06:20:33.087 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.087 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.087 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:33.088 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.809190+0000", 2023-10-09T06:20:33.088 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0105863 2023-10-09T06:20:33.088 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.088 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.088 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:33.088 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.809228+0000", 2023-10-09T06:20:33.088 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8352000000000001e-05 2023-10-09T06:20:33.088 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.088 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.088 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:33.088 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.824086+0000", 2023-10-09T06:20:33.088 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.014858356 2023-10-09T06:20:33.088 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.089 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.089 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:33.089 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.824141+0000", 2023-10-09T06:20:33.089 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4466999999999999e-05 2023-10-09T06:20:33.089 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.089 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.089 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:33.089 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.825183+0000", 2023-10-09T06:20:33.089 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0010417 2023-10-09T06:20:33.089 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.089 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:33.090 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.090 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.090 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.090 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:182294 9.b 9:d6e67af7:::benchmark_data_smithi064_511911_object81192:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T06:20:33.090 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:10:49.757688+0000", 2023-10-09T06:20:33.090 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 583.31312902100001, 2023-10-09T06:20:33.090 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.068185304, 2023-10-09T06:20:33.090 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:33.090 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:33.090 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:33.090 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:20:33.091 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:20:33.091 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 182294 2023-10-09T06:20:33.091 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.091 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:33.091 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.091 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:33.091 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.757688+0000", 2023-10-09T06:20:33.091 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.091 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.091 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.091 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:33.091 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.757688+0000", 2023-10-09T06:20:33.092 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.092 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.092 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.092 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:33.092 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.757687+0000", 2023-10-09T06:20:33.092 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:33.092 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.092 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.092 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:33.092 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.757698+0000", 2023-10-09T06:20:33.092 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1630999999999999e-05 2023-10-09T06:20:33.093 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.093 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.093 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:33.093 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.757717+0000", 2023-10-09T06:20:33.093 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8512000000000002e-05 2023-10-09T06:20:33.093 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.093 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.093 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:33.093 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.757724+0000", 2023-10-09T06:20:33.093 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0140000000000001e-06 2023-10-09T06:20:33.093 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.093 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.094 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.094 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.757787+0000", 2023-10-09T06:20:33.094 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3429000000000005e-05 2023-10-09T06:20:33.094 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.094 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.094 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:33.094 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.757800+0000", 2023-10-09T06:20:33.094 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2876e-05 2023-10-09T06:20:33.094 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.094 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.095 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.095 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.803261+0000", 2023-10-09T06:20:33.095 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0454610019999999 2023-10-09T06:20:33.095 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.095 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.095 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:33.095 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.803436+0000", 2023-10-09T06:20:33.095 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000174768 2023-10-09T06:20:33.095 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.095 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.095 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:20:33.095 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.803598+0000", 2023-10-09T06:20:33.096 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000162452 2023-10-09T06:20:33.096 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.096 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.096 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:33.096 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.809288+0000", 2023-10-09T06:20:33.096 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.005689805 2023-10-09T06:20:33.096 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.096 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.096 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:33.096 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.809301+0000", 2023-10-09T06:20:33.096 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3229e-05 2023-10-09T06:20:33.096 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.097 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.097 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:33.097 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.825263+0000", 2023-10-09T06:20:33.097 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015961921 2023-10-09T06:20:33.097 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.097 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.097 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:33.097 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.825310+0000", 2023-10-09T06:20:33.097 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6968000000000002e-05 2023-10-09T06:20:33.098 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.098 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.098 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:33.098 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.825874+0000", 2023-10-09T06:20:33.098 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00056341400000000004 2023-10-09T06:20:33.098 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.098 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:33.098 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.098 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.098 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.098 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:182320 9.b 9:d7a47e95:::benchmark_data_smithi064_511911_object81218:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T06:20:33.099 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:10:49.824803+0000", 2023-10-09T06:20:33.099 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 583.24601417199995, 2023-10-09T06:20:33.099 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.001582857, 2023-10-09T06:20:33.099 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:33.099 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:33.099 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:33.099 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:20:33.099 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:20:33.099 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 182320 2023-10-09T06:20:33.099 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.100 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:33.100 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.100 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:33.100 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.824803+0000", 2023-10-09T06:20:33.100 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.100 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.100 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.100 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:33.100 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.824803+0000", 2023-10-09T06:20:33.100 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.100 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.100 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.101 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:33.101 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.824802+0000", 2023-10-09T06:20:33.101 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:33.101 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.101 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.101 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:33.101 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.824810+0000", 2023-10-09T06:20:33.101 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.2539999999999992e-06 2023-10-09T06:20:33.101 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.101 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.101 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:33.101 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.824832+0000", 2023-10-09T06:20:33.102 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1909e-05 2023-10-09T06:20:33.102 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.102 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.102 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:33.102 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.824839+0000", 2023-10-09T06:20:33.102 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4819999999999997e-06 2023-10-09T06:20:33.102 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.102 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.102 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.102 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.824891+0000", 2023-10-09T06:20:33.102 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2064999999999997e-05 2023-10-09T06:20:33.102 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.103 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.103 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:33.103 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.824904+0000", 2023-10-09T06:20:33.103 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2578999999999999e-05 2023-10-09T06:20:33.103 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.103 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.103 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.103 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.809326+0000", 2023-10-09T06:20:33.103 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.98442220000000002 2023-10-09T06:20:33.103 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.103 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.103 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:33.104 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.809438+0000", 2023-10-09T06:20:33.104 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000111531 2023-10-09T06:20:33.104 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.104 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.104 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:20:33.104 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.809544+0000", 2023-10-09T06:20:33.104 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000105988 2023-10-09T06:20:33.104 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.104 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.104 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:33.105 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.824045+0000", 2023-10-09T06:20:33.105 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.014501328000000001 2023-10-09T06:20:33.105 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.105 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.105 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:33.105 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.825199+0000", 2023-10-09T06:20:33.105 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0011542740000000001 2023-10-09T06:20:33.105 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.105 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.105 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:33.105 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.825941+0000", 2023-10-09T06:20:33.106 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00074225400000000003 2023-10-09T06:20:33.106 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.106 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.106 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:33.106 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.825973+0000", 2023-10-09T06:20:33.106 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1418000000000003e-05 2023-10-09T06:20:33.106 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.106 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.106 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:33.106 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.826386+0000", 2023-10-09T06:20:33.107 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00041313299999999999 2023-10-09T06:20:33.107 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.107 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:33.107 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.107 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.107 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.107 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.35940.0:182329 9.b 9:ddc708a0:::benchmark_data_smithi064_511911_object81227:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T06:20:33.107 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:10:49.834744+0000", 2023-10-09T06:20:33.107 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 583.23607359100004, 2023-10-09T06:20:33.107 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0013349039999999, 2023-10-09T06:20:33.107 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:33.107 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:33.108 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:33.108 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.35940", 2023-10-09T06:20:33.108 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/512876221", 2023-10-09T06:20:33.108 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 182329 2023-10-09T06:20:33.108 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.108 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:33.108 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.108 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:33.108 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.834744+0000", 2023-10-09T06:20:33.108 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.108 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.109 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.109 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:33.109 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.834744+0000", 2023-10-09T06:20:33.109 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.109 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.109 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.109 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:33.109 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.834742+0000", 2023-10-09T06:20:33.109 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:33.109 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.109 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.109 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:33.110 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.834753+0000", 2023-10-09T06:20:33.110 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0342e-05 2023-10-09T06:20:33.110 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.110 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.110 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:33.110 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.834774+0000", 2023-10-09T06:20:33.110 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0925999999999999e-05 2023-10-09T06:20:33.110 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.110 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.110 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:33.111 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.834782+0000", 2023-10-09T06:20:33.111 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0930000000000003e-06 2023-10-09T06:20:33.111 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.111 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.111 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.111 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.834853+0000", 2023-10-09T06:20:33.111 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1280999999999997e-05 2023-10-09T06:20:33.111 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.111 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.111 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:33.111 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:49.834866+0000", 2023-10-09T06:20:33.111 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3122000000000001e-05 2023-10-09T06:20:33.112 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.112 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.112 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.112 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.814103+0000", 2023-10-09T06:20:33.112 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.97923662199999995 2023-10-09T06:20:33.112 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.112 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.112 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:33.112 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.814260+0000", 2023-10-09T06:20:33.112 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015774800000000001 2023-10-09T06:20:33.112 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.112 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.113 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:20:33.113 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.814363+0000", 2023-10-09T06:20:33.113 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000102413 2023-10-09T06:20:33.113 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.113 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.113 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:33.113 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.825207+0000", 2023-10-09T06:20:33.113 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010844457 2023-10-09T06:20:33.113 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.113 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.113 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:33.113 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.825238+0000", 2023-10-09T06:20:33.114 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0802000000000001e-05 2023-10-09T06:20:33.114 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.114 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.114 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:33.114 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.834895+0000", 2023-10-09T06:20:33.114 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.009657367 2023-10-09T06:20:33.114 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.114 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.114 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:33.114 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.834957+0000", 2023-10-09T06:20:33.114 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1943e-05 2023-10-09T06:20:33.114 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.115 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.115 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:33.115 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:10:50.836079+0000", 2023-10-09T06:20:33.115 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0011211979999999999 2023-10-09T06:20:33.115 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.115 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:33.115 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.115 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.115 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.115 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.36591.0:114851 11.8 e596/595 11:1d9f57ad:::benchmark_data_smithi064_631903_object5937:head v 596'7260, mlcod=596'7258)", 2023-10-09T06:20:33.115 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:17:48.899251+0000", 2023-10-09T06:20:33.116 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 164.17156630900001, 2023-10-09T06:20:33.116 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.25212368699999999, 2023-10-09T06:20:33.116 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:33.116 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:33.116 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:33.116 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.116 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:33.116 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.899251+0000", 2023-10-09T06:20:33.116 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.116 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.116 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.116 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:33.117 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.899251+0000", 2023-10-09T06:20:33.117 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.117 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.117 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.117 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:33.117 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.899250+0000", 2023-10-09T06:20:33.117 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:33.117 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.117 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.117 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:33.117 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.899258+0000", 2023-10-09T06:20:33.117 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6270000000000006e-06 2023-10-09T06:20:33.118 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.118 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.118 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:33.118 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.899270+0000", 2023-10-09T06:20:33.118 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2034e-05 2023-10-09T06:20:33.118 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.118 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.118 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:33.118 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.899276+0000", 2023-10-09T06:20:33.118 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0249999999999999e-06 2023-10-09T06:20:33.118 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.118 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.119 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.119 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.899365+0000", 2023-10-09T06:20:33.119 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8238000000000005e-05 2023-10-09T06:20:33.119 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.119 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.119 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:33.119 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.899389+0000", 2023-10-09T06:20:33.119 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4104000000000002e-05 2023-10-09T06:20:33.119 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.119 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.119 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:33.119 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.151329+0000", 2023-10-09T06:20:33.120 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.251939723 2023-10-09T06:20:33.120 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.120 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.120 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:33.120 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.151375+0000", 2023-10-09T06:20:33.120 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6142e-05 2023-10-09T06:20:33.120 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.120 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:33.120 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.120 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.120 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.120 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.36591.0:114855 11.9 e596/595 11:95ebbca1:::benchmark_data_smithi064_631903_object5941:head v 596'7122, mlcod=596'7120)", 2023-10-09T06:20:33.121 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:17:48.913078+0000", 2023-10-09T06:20:33.121 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 164.15773980099999, 2023-10-09T06:20:33.121 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24096907300000001, 2023-10-09T06:20:33.121 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:33.121 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:33.121 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:33.121 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.121 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:33.121 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.913078+0000", 2023-10-09T06:20:33.121 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.121 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.121 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.122 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:33.122 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.913078+0000", 2023-10-09T06:20:33.122 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.122 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.122 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.122 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:33.122 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.913076+0000", 2023-10-09T06:20:33.122 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:20:33.122 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.122 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.122 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:33.123 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.913087+0000", 2023-10-09T06:20:33.123 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0672999999999999e-05 2023-10-09T06:20:33.123 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.123 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.123 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:33.123 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.913106+0000", 2023-10-09T06:20:33.123 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.859e-05 2023-10-09T06:20:33.123 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.123 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.123 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:33.123 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.913114+0000", 2023-10-09T06:20:33.123 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.2719999999999997e-06 2023-10-09T06:20:33.124 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.124 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.124 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.124 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.913166+0000", 2023-10-09T06:20:33.124 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1953999999999998e-05 2023-10-09T06:20:33.124 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.124 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.124 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:33.124 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.913190+0000", 2023-10-09T06:20:33.124 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3669999999999999e-05 2023-10-09T06:20:33.124 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.125 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.125 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:33.125 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.154026+0000", 2023-10-09T06:20:33.125 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24083628200000001 2023-10-09T06:20:33.125 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.125 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.125 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:33.125 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.154047+0000", 2023-10-09T06:20:33.125 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0800000000000001e-05 2023-10-09T06:20:33.125 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.125 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:33.125 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.126 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.126 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.126 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.36591.0:114858 11.1 e596/595 11:8e3e3b2a:::benchmark_data_smithi064_631903_object5944:head v 596'7032, mlcod=596'7030)", 2023-10-09T06:20:33.126 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:17:48.913128+0000", 2023-10-09T06:20:33.126 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 164.157688968, 2023-10-09T06:20:33.126 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24088709999999999, 2023-10-09T06:20:33.126 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:33.126 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:33.126 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:33.126 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.126 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:33.126 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.913128+0000", 2023-10-09T06:20:33.127 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.127 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.127 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.127 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:33.127 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.913128+0000", 2023-10-09T06:20:33.127 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.127 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.127 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.127 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:33.127 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.913128+0000", 2023-10-09T06:20:33.127 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:33.127 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.128 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.128 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:33.128 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.913133+0000", 2023-10-09T06:20:33.128 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7199999999999997e-06 2023-10-09T06:20:33.128 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.128 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.128 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:33.128 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.913148+0000", 2023-10-09T06:20:33.128 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5048999999999999e-05 2023-10-09T06:20:33.128 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.128 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.129 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:33.129 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.913152+0000", 2023-10-09T06:20:33.129 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.284e-06 2023-10-09T06:20:33.129 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.129 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.129 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.129 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.913188+0000", 2023-10-09T06:20:33.129 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5394e-05 2023-10-09T06:20:33.129 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.129 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.129 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:33.129 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.913212+0000", 2023-10-09T06:20:33.129 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4161000000000001e-05 2023-10-09T06:20:33.130 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.130 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.130 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:33.130 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.153976+0000", 2023-10-09T06:20:33.130 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.240763963 2023-10-09T06:20:33.130 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.130 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.130 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:33.130 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.154015+0000", 2023-10-09T06:20:33.130 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9780000000000002e-05 2023-10-09T06:20:33.130 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.131 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:33.131 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.131 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.131 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.131 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.36591.0:114860 11.9 e596/595 11:9a2c6559:::benchmark_data_smithi064_631903_object5946:head v 596'7123, mlcod=596'7120)", 2023-10-09T06:20:33.131 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:17:48.913399+0000", 2023-10-09T06:20:33.131 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 164.157418514, 2023-10-09T06:20:33.131 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.24066686000000001, 2023-10-09T06:20:33.131 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:33.131 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:33.131 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:33.131 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.131 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:33.132 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.913399+0000", 2023-10-09T06:20:33.132 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.132 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.132 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.132 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:33.132 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.913399+0000", 2023-10-09T06:20:33.132 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.132 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.132 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.132 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:33.132 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.913398+0000", 2023-10-09T06:20:33.133 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:33.133 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.133 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.133 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:33.133 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.913403+0000", 2023-10-09T06:20:33.133 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6580000000000001e-06 2023-10-09T06:20:33.133 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.133 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.133 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:33.133 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.913412+0000", 2023-10-09T06:20:33.133 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6340000000000007e-06 2023-10-09T06:20:33.133 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.134 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.134 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:33.134 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.913415+0000", 2023-10-09T06:20:33.134 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.4359999999999998e-06 2023-10-09T06:20:33.134 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.134 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.134 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.134 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.974135+0000", 2023-10-09T06:20:33.134 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.060720424000000002 2023-10-09T06:20:33.134 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.134 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.135 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:33.135 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.974155+0000", 2023-10-09T06:20:33.135 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9851e-05 2023-10-09T06:20:33.135 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.135 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.135 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:33.135 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.154049+0000", 2023-10-09T06:20:33.135 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.179893419 2023-10-09T06:20:33.135 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.135 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.135 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:33.135 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.154066+0000", 2023-10-09T06:20:33.135 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6934000000000001e-05 2023-10-09T06:20:33.136 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.136 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:33.136 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.136 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.136 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.136 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.36591.0:114867 11.9 e596/595 11:95b47e26:::benchmark_data_smithi064_631903_object5953:head v 596'7124, mlcod=596'7120)", 2023-10-09T06:20:33.136 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:17:48.928014+0000", 2023-10-09T06:20:33.136 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 164.142803157, 2023-10-09T06:20:33.136 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.22606910399999999, 2023-10-09T06:20:33.136 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:33.136 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:33.136 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:33.137 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.137 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:33.137 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.928014+0000", 2023-10-09T06:20:33.137 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.137 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.137 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.137 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:33.137 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.928014+0000", 2023-10-09T06:20:33.137 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.137 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.137 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.137 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:33.138 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.928013+0000", 2023-10-09T06:20:33.138 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:20:33.138 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.138 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.138 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:33.138 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.928031+0000", 2023-10-09T06:20:33.138 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7621999999999998e-05 2023-10-09T06:20:33.138 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.138 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.138 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:33.138 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.928048+0000", 2023-10-09T06:20:33.138 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7047000000000001e-05 2023-10-09T06:20:33.139 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.139 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.139 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:33.139 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.928054+0000", 2023-10-09T06:20:33.139 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5999999999999997e-06 2023-10-09T06:20:33.139 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.139 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.139 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.139 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.976669+0000", 2023-10-09T06:20:33.139 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.048615111000000003 2023-10-09T06:20:33.140 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.140 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.140 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:33.140 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.976690+0000", 2023-10-09T06:20:33.140 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1035000000000001e-05 2023-10-09T06:20:33.140 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.140 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.140 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:33.140 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.154067+0000", 2023-10-09T06:20:33.140 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.177377759 2023-10-09T06:20:33.140 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.141 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.141 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:33.141 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.154083+0000", 2023-10-09T06:20:33.141 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5846999999999999e-05 2023-10-09T06:20:33.141 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.141 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:33.141 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.141 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.141 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.141 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.36591.0:114868 11.9 e596/595 11:9d653849:::benchmark_data_smithi064_631903_object5954:head v 596'7125, mlcod=596'7120)", 2023-10-09T06:20:33.141 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:17:48.928696+0000", 2023-10-09T06:20:33.142 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 164.14212132, 2023-10-09T06:20:33.142 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.22540703300000001, 2023-10-09T06:20:33.142 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:33.142 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:33.142 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:33.142 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.142 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:33.142 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.928696+0000", 2023-10-09T06:20:33.142 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.142 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.143 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.143 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:33.143 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.928696+0000", 2023-10-09T06:20:33.143 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.143 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.143 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.143 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:33.143 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.928695+0000", 2023-10-09T06:20:33.143 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:33.143 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.143 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.144 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:33.144 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.928703+0000", 2023-10-09T06:20:33.144 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7000000000000008e-06 2023-10-09T06:20:33.144 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.144 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.144 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:33.144 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.928718+0000", 2023-10-09T06:20:33.144 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5126000000000001e-05 2023-10-09T06:20:33.144 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.144 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.144 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:33.145 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.928725+0000", 2023-10-09T06:20:33.145 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1760000000000004e-06 2023-10-09T06:20:33.145 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.145 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.145 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.145 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.086256+0000", 2023-10-09T06:20:33.145 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.157530216 2023-10-09T06:20:33.145 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.145 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.145 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:33.145 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.086280+0000", 2023-10-09T06:20:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4845e-05 2023-10-09T06:20:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.154087+0000", 2023-10-09T06:20:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.067806456000000001 2023-10-09T06:20:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.154103+0000", 2023-10-09T06:20:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6226e-05 2023-10-09T06:20:33.147 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.147 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:33.147 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.147 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.147 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.147 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.36591.0:114872 11.8 e596/595 11:11b0fa3f:::benchmark_data_smithi064_631903_object5958:head v 596'7261, mlcod=596'7258)", 2023-10-09T06:20:33.147 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:17:48.931240+0000", 2023-10-09T06:20:33.147 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 164.13957780800001, 2023-10-09T06:20:33.147 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.22016063299999999, 2023-10-09T06:20:33.147 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:33.147 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:33.148 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:33.148 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.148 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:33.148 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.931240+0000", 2023-10-09T06:20:33.148 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.148 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.148 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.148 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:33.148 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.931240+0000", 2023-10-09T06:20:33.148 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.148 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.148 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.149 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:33.149 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.931239+0000", 2023-10-09T06:20:33.149 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:33.149 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.149 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.149 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:33.149 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.931249+0000", 2023-10-09T06:20:33.149 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0438999999999999e-05 2023-10-09T06:20:33.149 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.149 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.149 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:33.149 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.931261+0000", 2023-10-09T06:20:33.150 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1970000000000001e-05 2023-10-09T06:20:33.150 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.150 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.150 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:33.150 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.931267+0000", 2023-10-09T06:20:33.150 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4099999999999999e-06 2023-10-09T06:20:33.150 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.150 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.150 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.150 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.973763+0000", 2023-10-09T06:20:33.150 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.042496760000000001 2023-10-09T06:20:33.151 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.151 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.151 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:33.151 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.973790+0000", 2023-10-09T06:20:33.151 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.6940999999999998e-05 2023-10-09T06:20:33.151 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.151 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.151 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:33.151 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.151379+0000", 2023-10-09T06:20:33.151 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.17758866300000001 2023-10-09T06:20:33.151 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.151 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.152 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:33.152 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.151400+0000", 2023-10-09T06:20:33.152 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1124e-05 2023-10-09T06:20:33.152 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.152 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:33.152 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.152 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.152 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.152 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.36591.0:114871 11.9 e596/595 11:9dbf24f4:::benchmark_data_smithi064_631903_object5957:head v 596'7126, mlcod=596'7120)", 2023-10-09T06:20:33.152 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:17:48.931602+0000", 2023-10-09T06:20:33.152 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 164.13921562100001, 2023-10-09T06:20:33.153 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.224474743, 2023-10-09T06:20:33.153 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:33.153 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:33.153 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:33.153 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.153 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:33.153 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.931602+0000", 2023-10-09T06:20:33.153 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.153 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.153 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.153 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:33.153 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.931602+0000", 2023-10-09T06:20:33.154 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.154 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.154 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.154 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:33.154 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.931601+0000", 2023-10-09T06:20:33.154 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:33.154 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.154 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.154 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:33.154 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.931606+0000", 2023-10-09T06:20:33.154 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2820000000000004e-06 2023-10-09T06:20:33.155 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.155 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.155 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:33.155 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.931617+0000", 2023-10-09T06:20:33.155 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.045e-05 2023-10-09T06:20:33.155 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.155 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.155 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:33.155 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.931621+0000", 2023-10-09T06:20:33.155 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3839999999999999e-06 2023-10-09T06:20:33.155 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.156 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.156 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.156 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.151416+0000", 2023-10-09T06:20:33.156 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.21979505599999999 2023-10-09T06:20:33.156 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.156 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.156 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:33.156 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.151439+0000", 2023-10-09T06:20:33.156 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2790999999999999e-05 2023-10-09T06:20:33.156 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.156 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.157 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:33.157 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.156037+0000", 2023-10-09T06:20:33.157 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.004597683 2023-10-09T06:20:33.157 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.157 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.157 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:33.157 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.156076+0000", 2023-10-09T06:20:33.157 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9656000000000001e-05 2023-10-09T06:20:33.157 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.157 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:33.157 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.158 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.158 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.158 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.36591.0:114873 11.8 e596/595 11:130ee4ee:::benchmark_data_smithi064_631903_object5959:head v 596'7262, mlcod=596'7258)", 2023-10-09T06:20:33.158 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:17:48.937846+0000", 2023-10-09T06:20:33.158 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 164.132970855, 2023-10-09T06:20:33.158 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.213575557, 2023-10-09T06:20:33.158 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:33.158 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:33.158 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:33.158 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.158 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:33.158 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.937846+0000", 2023-10-09T06:20:33.159 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.159 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.159 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.159 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:33.159 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.937846+0000", 2023-10-09T06:20:33.159 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.159 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.159 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.159 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:33.159 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.937846+0000", 2023-10-09T06:20:33.159 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:20:33.160 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.160 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.160 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:33.160 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.937857+0000", 2023-10-09T06:20:33.160 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1151e-05 2023-10-09T06:20:33.160 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.160 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.160 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:33.160 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.937870+0000", 2023-10-09T06:20:33.160 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3773e-05 2023-10-09T06:20:33.160 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.160 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.161 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:33.161 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.937876+0000", 2023-10-09T06:20:33.161 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5790000000000003e-06 2023-10-09T06:20:33.161 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.161 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.161 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.161 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.976289+0000", 2023-10-09T06:20:33.161 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.038413347 2023-10-09T06:20:33.161 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.161 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.161 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:33.161 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.976331+0000", 2023-10-09T06:20:33.162 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1146999999999998e-05 2023-10-09T06:20:33.162 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.162 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.162 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:33.162 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.151403+0000", 2023-10-09T06:20:33.162 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.17507209200000001 2023-10-09T06:20:33.162 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.162 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.162 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:33.162 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.151422+0000", 2023-10-09T06:20:33.162 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9409e-05 2023-10-09T06:20:33.163 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.163 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:33.163 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.163 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.163 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.163 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.36591.0:114875 11.1 e596/595 11:892cf210:::benchmark_data_smithi064_631903_object5961:head v 596'7033, mlcod=596'7030)", 2023-10-09T06:20:33.163 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:17:48.941364+0000", 2023-10-09T06:20:33.163 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 164.12945346500001, 2023-10-09T06:20:33.163 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.21503449199999999, 2023-10-09T06:20:33.163 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:33.163 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:33.163 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:33.163 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.164 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:33.164 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.941364+0000", 2023-10-09T06:20:33.164 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.164 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.164 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.164 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:33.164 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.941364+0000", 2023-10-09T06:20:33.164 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.164 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.164 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.164 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:33.164 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.941363+0000", 2023-10-09T06:20:33.165 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:20:33.165 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.165 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.165 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:33.165 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.941374+0000", 2023-10-09T06:20:33.165 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1112e-05 2023-10-09T06:20:33.165 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.165 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.165 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:33.165 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.941390+0000", 2023-10-09T06:20:33.165 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6008999999999999e-05 2023-10-09T06:20:33.165 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.166 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.166 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:33.166 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.941397+0000", 2023-10-09T06:20:33.166 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.691e-06 2023-10-09T06:20:33.166 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.166 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.166 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.166 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.151354+0000", 2023-10-09T06:20:33.166 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.209956949 2023-10-09T06:20:33.166 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.166 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.166 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:33.167 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.151375+0000", 2023-10-09T06:20:33.167 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.084e-05 2023-10-09T06:20:33.167 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.167 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.167 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:33.167 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.156369+0000", 2023-10-09T06:20:33.167 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0049942069999999996 2023-10-09T06:20:33.167 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.167 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.167 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:33.167 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.156398+0000", 2023-10-09T06:20:33.167 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9578e-05 2023-10-09T06:20:33.167 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.168 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:33.168 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.168 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.168 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.168 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.36591.0:114880 11.8 e596/595 11:18cfba7c:::benchmark_data_smithi064_631903_object5966:head v 596'7263, mlcod=596'7258)", 2023-10-09T06:20:33.168 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:17:48.958055+0000", 2023-10-09T06:20:33.168 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 164.11276215500001, 2023-10-09T06:20:33.168 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.193640217, 2023-10-09T06:20:33.168 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:33.168 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:33.168 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:33.168 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.169 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:33.169 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.958055+0000", 2023-10-09T06:20:33.169 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.169 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.169 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.169 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:33.169 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.958055+0000", 2023-10-09T06:20:33.169 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.169 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.169 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.169 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:33.169 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.958054+0000", 2023-10-09T06:20:33.170 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:20:33.170 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.170 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.170 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:33.170 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.958067+0000", 2023-10-09T06:20:33.170 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2863e-05 2023-10-09T06:20:33.170 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.170 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.170 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:33.170 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.958086+0000", 2023-10-09T06:20:33.170 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8950999999999999e-05 2023-10-09T06:20:33.171 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.171 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.171 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:33.171 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.958093+0000", 2023-10-09T06:20:33.171 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2919999999999998e-06 2023-10-09T06:20:33.171 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.171 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.171 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.171 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.086244+0000", 2023-10-09T06:20:33.171 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.128150715 2023-10-09T06:20:33.171 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.171 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.172 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:33.172 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.086269+0000", 2023-10-09T06:20:33.172 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4836000000000001e-05 2023-10-09T06:20:33.172 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.172 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.172 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:33.172 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.151666+0000", 2023-10-09T06:20:33.172 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.065396924999999995 2023-10-09T06:20:33.172 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.172 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.172 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:33.172 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:49.151695+0000", 2023-10-09T06:20:33.172 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9683999999999999e-05 2023-10-09T06:20:33.173 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.173 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:33.173 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.173 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.173 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.173 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.36591.0:198288 11.5 e628/627 11:a6fef892:::benchmark_data_smithi064_631903_object89374:head v 628'12412, mlcod=628'12409)", 2023-10-09T06:20:33.173 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:19:34.898120+0000", 2023-10-09T06:20:33.173 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 58.17269701, 2023-10-09T06:20:33.173 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.023046077000000002, 2023-10-09T06:20:33.173 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:33.173 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:33.173 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:33.174 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.174 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:33.174 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:34.898120+0000", 2023-10-09T06:20:33.174 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.174 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.174 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.174 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:33.174 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:34.898120+0000", 2023-10-09T06:20:33.174 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.174 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.174 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.175 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:33.175 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:34.898120+0000", 2023-10-09T06:20:33.175 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967296 2023-10-09T06:20:33.175 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.175 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.175 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:33.175 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:34.898126+0000", 2023-10-09T06:20:33.175 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2809999999999997e-06 2023-10-09T06:20:33.175 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.175 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.175 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:33.175 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:34.898131+0000", 2023-10-09T06:20:33.176 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9429999999999999e-06 2023-10-09T06:20:33.176 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.176 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.176 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:33.176 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:34.898135+0000", 2023-10-09T06:20:33.176 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.2430000000000001e-06 2023-10-09T06:20:33.176 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.176 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.176 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.176 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:34.902674+0000", 2023-10-09T06:20:33.176 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0045393489999999998 2023-10-09T06:20:33.176 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.177 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.177 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:33.177 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:34.902692+0000", 2023-10-09T06:20:33.177 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7668000000000001e-05 2023-10-09T06:20:33.177 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.177 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.177 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:33.177 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:34.921125+0000", 2023-10-09T06:20:33.177 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018433405999999999 2023-10-09T06:20:33.177 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.177 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.177 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:33.178 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:34.921166+0000", 2023-10-09T06:20:33.178 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1350999999999998e-05 2023-10-09T06:20:33.178 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.178 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:33.178 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.178 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.178 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.178 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.36591.0:215432 11.5 e633/627 11:aed172c9:::benchmark_data_smithi064_631903_object106518:head v 633'13500, mlcod=633'13498)", 2023-10-09T06:20:33.178 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:19:56.457576+0000", 2023-10-09T06:20:33.178 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 36.613241035000001, 2023-10-09T06:20:33.178 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.025968519999999998, 2023-10-09T06:20:33.179 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:33.179 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:33.179 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:33.179 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.179 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:33.179 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.457576+0000", 2023-10-09T06:20:33.179 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.179 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.179 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.179 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:33.179 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.457576+0000", 2023-10-09T06:20:33.180 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.180 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.180 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.180 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:33.180 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.457575+0000", 2023-10-09T06:20:33.180 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:33.180 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.180 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.180 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:33.180 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.457585+0000", 2023-10-09T06:20:33.180 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0547000000000001e-05 2023-10-09T06:20:33.181 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.181 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.181 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:33.181 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.457597+0000", 2023-10-09T06:20:33.181 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1568e-05 2023-10-09T06:20:33.181 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.181 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.181 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:33.181 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.457602+0000", 2023-10-09T06:20:33.181 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3210000000000004e-06 2023-10-09T06:20:33.182 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.182 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.182 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.182 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.457642+0000", 2023-10-09T06:20:33.182 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9737000000000003e-05 2023-10-09T06:20:33.182 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.182 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.182 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:33.182 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.457663+0000", 2023-10-09T06:20:33.183 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0669000000000001e-05 2023-10-09T06:20:33.183 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.183 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.183 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:33.183 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.483506+0000", 2023-10-09T06:20:33.183 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.025842948000000001 2023-10-09T06:20:33.183 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.183 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.183 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:33.183 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.483545+0000", 2023-10-09T06:20:33.183 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9286999999999998e-05 2023-10-09T06:20:33.184 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.184 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:33.184 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.184 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.184 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.184 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.36591.0:215433 11.5 e633/627 11:a33a30bc:::benchmark_data_smithi064_631903_object106519:head v 633'13501, mlcod=633'13498)", 2023-10-09T06:20:33.184 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:19:56.457643+0000", 2023-10-09T06:20:33.184 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 36.613174006999998, 2023-10-09T06:20:33.184 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.025926125000000001, 2023-10-09T06:20:33.184 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:33.184 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:33.185 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:33.185 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.185 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:33.185 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.457643+0000", 2023-10-09T06:20:33.185 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.185 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.185 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.185 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:33.185 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.457643+0000", 2023-10-09T06:20:33.185 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.185 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.185 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.186 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:33.186 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.457643+0000", 2023-10-09T06:20:33.186 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967296 2023-10-09T06:20:33.186 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.186 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.186 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:33.186 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.457648+0000", 2023-10-09T06:20:33.186 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4429999999999996e-06 2023-10-09T06:20:33.186 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.186 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.186 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:33.186 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.457653+0000", 2023-10-09T06:20:33.187 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5199999999999997e-06 2023-10-09T06:20:33.187 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.187 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.187 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:33.187 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.457657+0000", 2023-10-09T06:20:33.187 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8319999999999999e-06 2023-10-09T06:20:33.187 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.187 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.187 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.187 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.460775+0000", 2023-10-09T06:20:33.187 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0031181099999999999 2023-10-09T06:20:33.187 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.188 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.188 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:33.188 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.460794+0000", 2023-10-09T06:20:33.188 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9398999999999998e-05 2023-10-09T06:20:33.188 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.188 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.188 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:33.188 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.483550+0000", 2023-10-09T06:20:33.188 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.022755503999999999 2023-10-09T06:20:33.188 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.188 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.188 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:33.188 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.483569+0000", 2023-10-09T06:20:33.189 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9553000000000001e-05 2023-10-09T06:20:33.189 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.189 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:33.189 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.189 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.189 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.189 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_repop(client.36591.0:215696 11.5 e633/627 11:a1b95423:::benchmark_data_smithi064_631903_object106782:head v 633'13518, mlcod=633'13516)", 2023-10-09T06:20:33.189 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:19:56.741342+0000", 2023-10-09T06:20:33.189 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 36.329475635999998, 2023-10-09T06:20:33.189 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.023486258999999999, 2023-10-09T06:20:33.189 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:33.190 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:33.190 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:33.190 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.190 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:33.190 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.741342+0000", 2023-10-09T06:20:33.190 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.190 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.190 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.190 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:33.190 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.741342+0000", 2023-10-09T06:20:33.190 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:33.190 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.191 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.191 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:33.191 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.741341+0000", 2023-10-09T06:20:33.191 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:20:33.191 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.191 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.191 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:33.191 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.741350+0000", 2023-10-09T06:20:33.191 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9250000000000001e-06 2023-10-09T06:20:33.191 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.191 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.191 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:33.192 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.741366+0000", 2023-10-09T06:20:33.192 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.685e-05 2023-10-09T06:20:33.192 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.192 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.192 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:33.192 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.741373+0000", 2023-10-09T06:20:33.192 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8000000000000001e-06 2023-10-09T06:20:33.192 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.192 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.192 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:33.192 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.744075+0000", 2023-10-09T06:20:33.192 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0027017080000000001 2023-10-09T06:20:33.193 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.193 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.193 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:33.193 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.744093+0000", 2023-10-09T06:20:33.193 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8335999999999999e-05 2023-10-09T06:20:33.193 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.193 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.193 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:33.193 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.764778+0000", 2023-10-09T06:20:33.193 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.020684726000000001 2023-10-09T06:20:33.193 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:33.193 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:33.193 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:33.194 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:56.764828+0000", 2023-10-09T06:20:33.194 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9907999999999997e-05 2023-10-09T06:20:33.194 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.194 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:33.194 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.194 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:33.194 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:33.194 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:20:33.194 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:33.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:32 smithi064 ceph-mon[212750]: pgmap v1666: 103 pgs: 19 peering, 84 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:20:33.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:32 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 17 pgs inactive, 17 pgs peering (PG_AVAILABILITY) 2023-10-09T06:20:33.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:32 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2186534665' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:33.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:32 smithi064 ceph-mon[220044]: pgmap v1666: 103 pgs: 19 peering, 84 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:20:33.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:32 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 17 pgs inactive, 17 pgs peering (PG_AVAILABILITY) 2023-10-09T06:20:33.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:32 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2186534665' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:33.224 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:33.304 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:32 smithi156 ceph-mon[195101]: pgmap v1666: 103 pgs: 19 peering, 84 active+clean; 582 KiB data, 4.8 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:20:33.304 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:32 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 17 pgs inactive, 17 pgs peering (PG_AVAILABILITY) 2023-10-09T06:20:33.304 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:32 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2186534665' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:33.470 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:33.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:33.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:33.574 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:33.601 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:33.647 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:33.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:33.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:33.750 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:33.777 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:33.812 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:20:33.822 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:33.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:33.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:33.925 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:33.951 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:34.003 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:34.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:34.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:34.111 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:34.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:33 smithi156 ceph-mon[195101]: failed to encode map e648 with expected crc 2023-10-09T06:20:34.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:33 smithi156 ceph-mon[195101]: osdmap e648: 8 total, 8 up, 6 in 2023-10-09T06:20:34.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:33 smithi156 ceph-mon[195101]: failed to encode map e648 with expected crc 2023-10-09T06:20:34.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:33 smithi156 ceph-mon[195101]: failed to encode map e648 with expected crc 2023-10-09T06:20:34.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:33 smithi156 ceph-mon[195101]: failed to encode map e648 with expected crc 2023-10-09T06:20:34.140 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:33 smithi064 ceph-mon[212750]: failed to encode map e648 with expected crc 2023-10-09T06:20:34.140 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:33 smithi064 ceph-mon[212750]: osdmap e648: 8 total, 8 up, 6 in 2023-10-09T06:20:34.140 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:33 smithi064 ceph-mon[212750]: failed to encode map e648 with expected crc 2023-10-09T06:20:34.140 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:33 smithi064 ceph-mon[212750]: failed to encode map e648 with expected crc 2023-10-09T06:20:34.140 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:33 smithi064 ceph-mon[212750]: failed to encode map e648 with expected crc 2023-10-09T06:20:34.140 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:33 smithi064 ceph-mon[220044]: failed to encode map e648 with expected crc 2023-10-09T06:20:34.141 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:33 smithi064 ceph-mon[220044]: osdmap e648: 8 total, 8 up, 6 in 2023-10-09T06:20:34.141 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:33 smithi064 ceph-mon[220044]: failed to encode map e648 with expected crc 2023-10-09T06:20:34.141 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:33 smithi064 ceph-mon[220044]: failed to encode map e648 with expected crc 2023-10-09T06:20:34.141 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:33 smithi064 ceph-mon[220044]: failed to encode map e648 with expected crc 2023-10-09T06:20:34.143 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:34.175 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:20:34.204 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:34.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:34.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:34.309 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:34.341 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:34.469 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:34.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:34.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:34.522 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-10-09T06:20:34.523 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:20:34.525 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export testimg /tmp/rbd_import_export_695299/img2 2023-10-09T06:20:34.574 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:34.603 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:34.791 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 5, 7, 1, 6] out_osds: [2, 4] dead_osds: [] live_osds: [1, 2, 3, 6, 7, 4, 0, 5] 2023-10-09T06:20:34.792 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:20:34.792 INFO:tasks.thrashosds.thrasher:Reweighting osd 1 to 0.35111489604760215 2023-10-09T06:20:34.792 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 0.35111489604760215 2023-10-09T06:20:34.836 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T06:20:34.840 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export testimg - 2023-10-09T06:20:34.985 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:34.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:34.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:35.091 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:35.118 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:34 smithi064 ceph-mon[212750]: pgmap v1668: 103 pgs: 19 peering, 84 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:20:35.118 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:34 smithi064 ceph-mon[212750]: osdmap e649: 8 total, 8 up, 6 in 2023-10-09T06:20:35.118 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:34 smithi064 ceph-mon[212750]: failed to encode map e649 with expected crc 2023-10-09T06:20:35.118 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:34 smithi064 ceph-mon[212750]: failed to encode map e649 with expected crc 2023-10-09T06:20:35.119 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:34 smithi064 ceph-mon[212750]: failed to encode map e649 with expected crc 2023-10-09T06:20:35.119 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:34 smithi064 ceph-mon[220044]: pgmap v1668: 103 pgs: 19 peering, 84 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:20:35.119 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:34 smithi064 ceph-mon[220044]: osdmap e649: 8 total, 8 up, 6 in 2023-10-09T06:20:35.119 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:34 smithi064 ceph-mon[220044]: failed to encode map e649 with expected crc 2023-10-09T06:20:35.119 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:34 smithi064 ceph-mon[220044]: failed to encode map e649 with expected crc 2023-10-09T06:20:35.119 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:34 smithi064 ceph-mon[220044]: failed to encode map e649 with expected crc 2023-10-09T06:20:35.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:34 smithi156 ceph-mon[195101]: pgmap v1668: 103 pgs: 19 peering, 84 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:20:35.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:34 smithi156 ceph-mon[195101]: osdmap e649: 8 total, 8 up, 6 in 2023-10-09T06:20:35.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:34 smithi156 ceph-mon[195101]: failed to encode map e649 with expected crc 2023-10-09T06:20:35.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:34 smithi156 ceph-mon[195101]: failed to encode map e649 with expected crc 2023-10-09T06:20:35.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:34 smithi156 ceph-mon[195101]: failed to encode map e649 with expected crc 2023-10-09T06:20:35.122 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:35.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:20:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:20:35] "GET /metrics HTTP/1.1" 200 34028 "" "Prometheus/2.43.0" 2023-10-09T06:20:35.636 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T06:20:35.640 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm testimg 2023-10-09T06:20:35.722 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T06:20:35.725 INFO:tasks.workunit.client.0.smithi064.stderr:+ cmp /tmp/rbd_import_export_695299/img /tmp/rbd_import_export_695299/img2 2023-10-09T06:20:35.771 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:20:35.772 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:20:35.772 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 0 2023-10-09T06:20:35.772 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:20:36.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:36 smithi156 ceph-mon[195101]: pgmap v1670: 103 pgs: 103 active+clean; 5.2 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 16 KiB/s rd, 52 KiB/s wr, 21 op/s 2023-10-09T06:20:36.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:36 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 17 pgs inactive, 17 pgs peering) 2023-10-09T06:20:36.411 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:20:36.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:36 smithi064 ceph-mon[212750]: pgmap v1670: 103 pgs: 103 active+clean; 5.2 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 16 KiB/s rd, 52 KiB/s wr, 21 op/s 2023-10-09T06:20:36.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:36 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 17 pgs inactive, 17 pgs peering) 2023-10-09T06:20:36.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:36 smithi064 ceph-mon[220044]: pgmap v1670: 103 pgs: 103 active+clean; 5.2 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 16 KiB/s rd, 52 KiB/s wr, 21 op/s 2023-10-09T06:20:36.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:36 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 17 pgs inactive, 17 pgs peering) 2023-10-09T06:20:36.516 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:36.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:36.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:36.620 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:36.648 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:36.745 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:36.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:36.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:36.765 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:20:36.780 INFO:tasks.workunit.client.0.smithi064.stderr:+ cmp /tmp/rbd_import_export_695299/img /tmp/rbd_import_export_695299/img3 2023-10-09T06:20:36.851 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:36.880 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:36.929 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:36.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:36.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:37.033 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:37.061 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:37.205 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:37.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:37.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:37.296 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_695299/img2 /tmp/rbd_import_export_695299/img3 2023-10-09T06:20:37.310 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:37.338 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:37.381 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:37.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:37.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:37.484 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:37.512 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:37.557 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:37.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:37.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:37.661 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:37.690 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:37.734 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:37.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:37.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:37.837 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:37.868 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:37.913 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:37.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:37.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:38.017 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:38.046 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:38.247 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:38.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:38.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:38.287 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:20:38.287 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:20:38.288 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T06:20:38.288 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T06:20:38.288 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:20:38.352 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:38.381 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:38.524 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:38.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:38.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:38.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:38 smithi156 ceph-mon[195101]: pgmap v1671: 103 pgs: 103 active+clean; 5.2 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 13 KiB/s rd, 42 KiB/s wr, 19 op/s 2023-10-09T06:20:38.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:38 smithi156 ceph-mon[195101]: 1.1b scrub starts 2023-10-09T06:20:38.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:38 smithi156 ceph-mon[195101]: 1.1b scrub ok 2023-10-09T06:20:38.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:38 smithi156 ceph-mon[195101]: failed to encode map e650 with expected crc 2023-10-09T06:20:38.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:38 smithi156 ceph-mon[195101]: osdmap e650: 8 total, 8 up, 6 in 2023-10-09T06:20:38.628 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:38.662 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:38.715 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:38.716 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:38 smithi064 ceph-mon[220044]: pgmap v1671: 103 pgs: 103 active+clean; 5.2 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 13 KiB/s rd, 42 KiB/s wr, 19 op/s 2023-10-09T06:20:38.717 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:38 smithi064 ceph-mon[220044]: 1.1b scrub starts 2023-10-09T06:20:38.717 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:38 smithi064 ceph-mon[220044]: 1.1b scrub ok 2023-10-09T06:20:38.717 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:38 smithi064 ceph-mon[220044]: failed to encode map e650 with expected crc 2023-10-09T06:20:38.717 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:38 smithi064 ceph-mon[220044]: osdmap e650: 8 total, 8 up, 6 in 2023-10-09T06:20:38.717 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:38 smithi064 ceph-mon[212750]: pgmap v1671: 103 pgs: 103 active+clean; 5.2 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 13 KiB/s rd, 42 KiB/s wr, 19 op/s 2023-10-09T06:20:38.717 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:38 smithi064 ceph-mon[212750]: 1.1b scrub starts 2023-10-09T06:20:38.717 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:38 smithi064 ceph-mon[212750]: 1.1b scrub ok 2023-10-09T06:20:38.717 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:38 smithi064 ceph-mon[212750]: failed to encode map e650 with expected crc 2023-10-09T06:20:38.718 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:38 smithi064 ceph-mon[212750]: osdmap e650: 8 total, 8 up, 6 in 2023-10-09T06:20:38.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:38.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:38.819 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:38.850 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:38.898 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:38.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:38.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:39.001 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:39.030 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:39.047 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:20:39.074 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:39.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:39.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:39.179 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:39.204 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:39.254 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:39.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:39.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:39.362 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:39.395 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:39.434 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:20:39.449 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:39.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:39.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:39.553 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:39.580 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:39.677 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:39.679 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:39 smithi156 ceph-mon[195101]: failed to encode map e650 with expected crc 2023-10-09T06:20:39.679 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:39 smithi156 ceph-mon[195101]: failed to encode map e650 with expected crc 2023-10-09T06:20:39.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:39.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:39.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:39 smithi064 ceph-mon[212750]: failed to encode map e650 with expected crc 2023-10-09T06:20:39.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:39 smithi064 ceph-mon[212750]: failed to encode map e650 with expected crc 2023-10-09T06:20:39.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:39 smithi064 ceph-mon[220044]: failed to encode map e650 with expected crc 2023-10-09T06:20:39.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:39 smithi064 ceph-mon[220044]: failed to encode map e650 with expected crc 2023-10-09T06:20:39.781 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:39.808 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:39.853 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:39.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:39.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:39.957 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:39.984 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:40.026 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:40.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:40.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:40.130 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:40.163 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:40.216 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:40.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:40.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:40.319 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:40.347 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:40.539 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:40.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:40.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:40.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:40 smithi156 ceph-mon[195101]: pgmap v1673: 103 pgs: 103 active+clean; 5.2 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 15 KiB/s rd, 49 KiB/s wr, 22 op/s 2023-10-09T06:20:40.646 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:40.673 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:40.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:40 smithi064 ceph-mon[212750]: pgmap v1673: 103 pgs: 103 active+clean; 5.2 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 15 KiB/s rd, 49 KiB/s wr, 22 op/s 2023-10-09T06:20:40.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:40 smithi064 ceph-mon[220044]: pgmap v1673: 103 pgs: 103 active+clean; 5.2 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 15 KiB/s rd, 49 KiB/s wr, 22 op/s 2023-10-09T06:20:40.769 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:40.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:40.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:40.874 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:40.908 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:40.959 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:20:40.959 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T06:20:40.960 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T06:20:40.960 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:20:40.960 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.960 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:131778 10.9 10:909f8957:::benchmark_data_smithi064_573750_object35563:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-10-09T06:20:40.960 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:14:20.794429+0000", 2023-10-09T06:20:40.960 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 380.16198659399998, 2023-10-09T06:20:40.960 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99906589199999996, 2023-10-09T06:20:40.960 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:40.960 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:40.960 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:40.960 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:20:40.961 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:40.961 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 131778 2023-10-09T06:20:40.961 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.961 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:40.961 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.961 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:40.961 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.794429+0000", 2023-10-09T06:20:40.961 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:40.961 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.961 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.961 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:40.962 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.794429+0000", 2023-10-09T06:20:40.962 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:40.962 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.962 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.962 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:40.962 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.794427+0000", 2023-10-09T06:20:40.962 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:40.962 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.962 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.962 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:40.962 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.794434+0000", 2023-10-09T06:20:40.963 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.934e-06 2023-10-09T06:20:40.963 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.963 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.963 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:40.963 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.794452+0000", 2023-10-09T06:20:40.963 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7730000000000001e-05 2023-10-09T06:20:40.963 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.963 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.963 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:40.963 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.794460+0000", 2023-10-09T06:20:40.963 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6469999999999998e-06 2023-10-09T06:20:40.964 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.964 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.964 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:40.964 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.794511+0000", 2023-10-09T06:20:40.964 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1496999999999998e-05 2023-10-09T06:20:40.964 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.964 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.964 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:40.964 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.794520+0000", 2023-10-09T06:20:40.964 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8130000000000002e-06 2023-10-09T06:20:40.964 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.965 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.965 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:40.965 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.759431+0000", 2023-10-09T06:20:40.965 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96491159000000004 2023-10-09T06:20:40.965 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.965 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.965 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:40.965 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.759582+0000", 2023-10-09T06:20:40.965 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015018799999999999 2023-10-09T06:20:40.965 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.965 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.965 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:20:40.966 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.759718+0000", 2023-10-09T06:20:40.966 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000136608 2023-10-09T06:20:40.966 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.966 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.966 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:40.966 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.775116+0000", 2023-10-09T06:20:40.966 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.015397622 2023-10-09T06:20:40.966 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.966 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.966 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:40.966 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793364+0000", 2023-10-09T06:20:40.967 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018247780000000002 2023-10-09T06:20:40.967 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.967 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.967 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:40.967 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793393+0000", 2023-10-09T06:20:40.967 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9689999999999999e-05 2023-10-09T06:20:40.967 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.967 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.967 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:40.967 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793447+0000", 2023-10-09T06:20:40.967 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3127999999999997e-05 2023-10-09T06:20:40.967 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.968 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.968 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:40.968 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793495+0000", 2023-10-09T06:20:40.968 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8112999999999998e-05 2023-10-09T06:20:40.968 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:40.968 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:40.968 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:40.968 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.968 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.968 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:131788 10.9 10:9b77a1ac:::benchmark_data_smithi064_573750_object35573:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-10-09T06:20:40.968 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:14:20.803992+0000", 2023-10-09T06:20:40.968 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 380.15242368499997, 2023-10-09T06:20:40.969 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.98965816100000004, 2023-10-09T06:20:40.969 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:40.969 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:40.969 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:40.969 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:20:40.969 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:40.969 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 131788 2023-10-09T06:20:40.969 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.969 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:40.969 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.970 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:40.970 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.803992+0000", 2023-10-09T06:20:40.970 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:40.970 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.970 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.970 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:40.970 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.803992+0000", 2023-10-09T06:20:40.970 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:40.970 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.970 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.970 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:40.970 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.803990+0000", 2023-10-09T06:20:40.971 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:40.971 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.971 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.971 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:40.971 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.803997+0000", 2023-10-09T06:20:40.971 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2729999999999997e-06 2023-10-09T06:20:40.971 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.971 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.971 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:40.971 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804012+0000", 2023-10-09T06:20:40.971 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4501000000000001e-05 2023-10-09T06:20:40.972 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.972 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.972 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:40.972 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804018+0000", 2023-10-09T06:20:40.972 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1940000000000003e-06 2023-10-09T06:20:40.972 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.972 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.972 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:40.972 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804073+0000", 2023-10-09T06:20:40.972 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5361000000000001e-05 2023-10-09T06:20:40.972 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.972 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.973 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:40.973 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804081+0000", 2023-10-09T06:20:40.973 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5190000000000003e-06 2023-10-09T06:20:40.973 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.973 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.973 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:40.973 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.762019+0000", 2023-10-09T06:20:40.973 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.95793824800000005 2023-10-09T06:20:40.973 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.973 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.973 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:40.974 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.762126+0000", 2023-10-09T06:20:40.974 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000107262 2023-10-09T06:20:40.974 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.974 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.974 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:20:40.974 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.762202+0000", 2023-10-09T06:20:40.974 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5018000000000006e-05 2023-10-09T06:20:40.974 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.974 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.974 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:40.974 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.775126+0000", 2023-10-09T06:20:40.975 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012924542000000001 2023-10-09T06:20:40.975 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.975 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.975 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:40.975 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793550+0000", 2023-10-09T06:20:40.975 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018423946 2023-10-09T06:20:40.975 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.975 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.975 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:40.975 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793588+0000", 2023-10-09T06:20:40.975 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7679000000000003e-05 2023-10-09T06:20:40.975 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.976 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.976 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:40.976 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793627+0000", 2023-10-09T06:20:40.976 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9410999999999999e-05 2023-10-09T06:20:40.976 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.976 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.976 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:40.976 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793650+0000", 2023-10-09T06:20:40.976 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2719999999999999e-05 2023-10-09T06:20:40.976 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:40.976 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:40.977 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:40.977 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.977 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.977 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:131789 10.9 10:95bec615:::benchmark_data_smithi064_573750_object35574:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-10-09T06:20:40.977 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:14:20.804037+0000", 2023-10-09T06:20:40.977 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 380.15237790999998, 2023-10-09T06:20:40.977 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.98989388300000003, 2023-10-09T06:20:40.977 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:40.977 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:40.977 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:40.977 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:20:40.977 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:40.978 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 131789 2023-10-09T06:20:40.978 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.978 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:40.978 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.978 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:40.978 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804037+0000", 2023-10-09T06:20:40.978 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:40.978 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.978 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.978 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:40.978 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804037+0000", 2023-10-09T06:20:40.978 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:40.979 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.979 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.979 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:40.979 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804037+0000", 2023-10-09T06:20:40.979 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:40.979 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.979 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.979 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:40.979 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804040+0000", 2023-10-09T06:20:40.979 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9730000000000002e-06 2023-10-09T06:20:40.979 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.979 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.980 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:40.980 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804046+0000", 2023-10-09T06:20:40.980 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3249999999999996e-06 2023-10-09T06:20:40.980 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.980 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.980 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:40.980 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804050+0000", 2023-10-09T06:20:40.980 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2239999999999997e-06 2023-10-09T06:20:40.980 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.980 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.980 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:40.980 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804140+0000", 2023-10-09T06:20:40.981 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9087999999999998e-05 2023-10-09T06:20:40.981 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.981 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.981 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:40.981 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.804145+0000", 2023-10-09T06:20:40.981 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0889999999999999e-06 2023-10-09T06:20:40.981 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.981 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.981 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:40.981 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.764349+0000", 2023-10-09T06:20:40.981 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96020412399999999 2023-10-09T06:20:40.981 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.982 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.982 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:40.982 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.764467+0000", 2023-10-09T06:20:40.982 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011781300000000001 2023-10-09T06:20:40.982 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.982 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.982 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:20:40.982 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.764590+0000", 2023-10-09T06:20:40.982 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012318400000000001 2023-10-09T06:20:40.982 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.982 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.983 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:40.983 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.775135+0000", 2023-10-09T06:20:40.983 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010544992 2023-10-09T06:20:40.983 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.983 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.983 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:40.983 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793727+0000", 2023-10-09T06:20:40.983 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018592457 2023-10-09T06:20:40.983 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.983 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.983 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:40.983 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793848+0000", 2023-10-09T06:20:40.984 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000120865 2023-10-09T06:20:40.984 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.984 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.984 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:40.984 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793888+0000", 2023-10-09T06:20:40.984 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0070000000000001e-05 2023-10-09T06:20:40.984 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.984 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.984 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:40.984 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793931+0000", 2023-10-09T06:20:40.984 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3118000000000003e-05 2023-10-09T06:20:40.984 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:40.985 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:40.985 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:40.985 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.985 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.985 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:131809 10.9 10:93420ba2:::benchmark_data_smithi064_573750_object35594:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-10-09T06:20:40.985 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:14:20.820581+0000", 2023-10-09T06:20:40.985 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 380.135834644, 2023-10-09T06:20:40.985 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.97345599199999999, 2023-10-09T06:20:40.985 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:40.985 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:40.985 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:40.985 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:20:40.986 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:40.986 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 131809 2023-10-09T06:20:40.986 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.986 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:40.986 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.986 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:40.986 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.820581+0000", 2023-10-09T06:20:40.986 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:40.986 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.986 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.986 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:40.986 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.820581+0000", 2023-10-09T06:20:40.987 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:40.987 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.987 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.987 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:40.987 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.820579+0000", 2023-10-09T06:20:40.987 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:40.987 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.987 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.987 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:40.987 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.820586+0000", 2023-10-09T06:20:40.987 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6000000000000003e-06 2023-10-09T06:20:40.987 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.988 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.988 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:40.988 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.820603+0000", 2023-10-09T06:20:40.988 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7329999999999998e-05 2023-10-09T06:20:40.988 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.988 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.988 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:40.988 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.820610+0000", 2023-10-09T06:20:40.988 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2079999999999997e-06 2023-10-09T06:20:40.988 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.988 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.988 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:40.989 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.820665+0000", 2023-10-09T06:20:40.989 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5900999999999999e-05 2023-10-09T06:20:40.989 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.989 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.989 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:40.989 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.820673+0000", 2023-10-09T06:20:40.989 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7780000000000007e-06 2023-10-09T06:20:40.989 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.989 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.989 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:40.989 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.766668+0000", 2023-10-09T06:20:40.989 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.94599452299999998 2023-10-09T06:20:40.990 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.990 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.990 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:40.990 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.766781+0000", 2023-10-09T06:20:40.990 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011279499999999999 2023-10-09T06:20:40.990 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.990 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.990 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:20:40.990 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.766894+0000", 2023-10-09T06:20:40.990 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000113927 2023-10-09T06:20:40.990 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.990 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.991 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:40.991 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.775144+0000", 2023-10-09T06:20:40.991 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.008249269 2023-10-09T06:20:40.991 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.991 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.991 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:40.991 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793750+0000", 2023-10-09T06:20:40.991 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018606527000000001 2023-10-09T06:20:40.991 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.991 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.991 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:40.991 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.793979+0000", 2023-10-09T06:20:40.992 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000229175 2023-10-09T06:20:40.992 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.992 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.992 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:40.992 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.794008+0000", 2023-10-09T06:20:40.992 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.8907e-05 2023-10-09T06:20:40.992 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.992 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.992 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:40.992 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.794037+0000", 2023-10-09T06:20:40.992 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.8339000000000001e-05 2023-10-09T06:20:40.993 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:40.993 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:40.993 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:40.993 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.993 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.993 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:131814 10.9 10:98c8ec43:::benchmark_data_smithi064_573750_object35599:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-10-09T06:20:40.993 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:14:20.832064+0000", 2023-10-09T06:20:40.993 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 380.12435144199998, 2023-10-09T06:20:40.993 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.96215577299999999, 2023-10-09T06:20:40.993 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:40.993 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:40.993 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:40.994 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:20:40.994 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:40.994 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 131814 2023-10-09T06:20:40.994 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.994 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:40.994 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.994 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:40.994 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.832064+0000", 2023-10-09T06:20:40.994 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:40.994 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.994 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.994 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:40.995 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.832064+0000", 2023-10-09T06:20:40.995 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:40.995 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.995 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.995 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:40.995 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.832063+0000", 2023-10-09T06:20:40.995 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:40.995 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.995 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.995 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:40.995 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.832073+0000", 2023-10-09T06:20:40.995 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0720000000000001e-05 2023-10-09T06:20:40.996 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.996 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.996 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:40.996 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.832085+0000", 2023-10-09T06:20:40.996 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2157000000000001e-05 2023-10-09T06:20:40.996 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.996 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.996 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:40.996 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.832091+0000", 2023-10-09T06:20:40.996 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4519999999999996e-06 2023-10-09T06:20:40.996 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.997 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.997 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:40.997 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.832140+0000", 2023-10-09T06:20:40.997 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9036999999999998e-05 2023-10-09T06:20:40.997 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.997 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.997 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:40.997 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:20.832146+0000", 2023-10-09T06:20:40.997 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4880000000000004e-06 2023-10-09T06:20:40.997 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.997 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.997 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:40.998 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.769195+0000", 2023-10-09T06:20:40.998 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.93704818499999998 2023-10-09T06:20:40.998 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.998 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.998 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:40.998 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.769310+0000", 2023-10-09T06:20:40.998 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000115648 2023-10-09T06:20:40.998 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.998 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.998 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:20:40.998 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.769403+0000", 2023-10-09T06:20:40.999 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2818999999999993e-05 2023-10-09T06:20:40.999 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.999 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.999 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:40.999 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.781216+0000", 2023-10-09T06:20:40.999 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011812977000000001 2023-10-09T06:20:40.999 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:40.999 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:40.999 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:40.999 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.794083+0000", 2023-10-09T06:20:40.999 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012867286 2023-10-09T06:20:41.000 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.000 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.000 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.000 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.794159+0000", 2023-10-09T06:20:41.000 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5951999999999994e-05 2023-10-09T06:20:41.000 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.000 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.000 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:41.000 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.794186+0000", 2023-10-09T06:20:41.000 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.6866999999999999e-05 2023-10-09T06:20:41.000 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.000 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.001 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:41.001 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:14:21.794220+0000", 2023-10-09T06:20:41.001 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.3445000000000003e-05 2023-10-09T06:20:41.001 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.001 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:41.001 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.001 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.001 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.001 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:114784 11.3 11:c3e96248:::benchmark_data_smithi064_631903_object5870:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-10-09T06:20:41.001 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:17:46.582244+0000", 2023-10-09T06:20:41.001 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 174.374171762, 2023-10-09T06:20:41.002 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.171536272, 2023-10-09T06:20:41.002 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:41.002 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:20:41.002 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:41.002 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:20:41.002 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:41.002 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 114784 2023-10-09T06:20:41.002 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.002 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:41.002 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.002 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:41.002 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.582244+0000", 2023-10-09T06:20:41.003 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.003 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.003 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.003 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:41.003 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.582244+0000", 2023-10-09T06:20:41.003 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.003 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.003 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.003 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:41.003 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.582242+0000", 2023-10-09T06:20:41.003 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:41.003 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.004 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.004 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:41.004 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.582250+0000", 2023-10-09T06:20:41.004 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3799999999999996e-06 2023-10-09T06:20:41.004 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.004 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.004 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:41.004 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.582263+0000", 2023-10-09T06:20:41.004 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3191e-05 2023-10-09T06:20:41.004 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.004 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.004 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:41.005 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.582273+0000", 2023-10-09T06:20:41.005 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0380999999999999e-05 2023-10-09T06:20:41.005 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.005 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.005 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.005 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.582334+0000", 2023-10-09T06:20:41.005 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0584999999999997e-05 2023-10-09T06:20:41.005 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.005 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.005 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:41.005 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.582435+0000", 2023-10-09T06:20:41.005 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000101631 2023-10-09T06:20:41.006 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.006 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.006 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:20:41.006 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.582509+0000", 2023-10-09T06:20:41.006 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3464e-05 2023-10-09T06:20:41.006 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.006 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.006 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.006 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.594177+0000", 2023-10-09T06:20:41.006 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011668227 2023-10-09T06:20:41.006 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.007 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.007 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:41.007 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.594188+0000", 2023-10-09T06:20:41.007 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0857e-05 2023-10-09T06:20:41.007 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.007 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.007 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.007 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:47.610233+0000", 2023-10-09T06:20:41.007 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0160452019999999 2023-10-09T06:20:41.007 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.007 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.007 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:41.008 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:47.613670+0000", 2023-10-09T06:20:41.008 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0034372809999999999 2023-10-09T06:20:41.008 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.008 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.008 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.008 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.719998+0000", 2023-10-09T06:20:41.008 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.10632803 2023-10-09T06:20:41.008 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.008 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.008 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for missing object", 2023-10-09T06:20:41.008 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.720141+0000", 2023-10-09T06:20:41.008 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014227900000000001 2023-10-09T06:20:41.009 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.009 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.009 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.009 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.753714+0000", 2023-10-09T06:20:41.009 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.033572777999999998 2023-10-09T06:20:41.009 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.009 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.009 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:41.009 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.753780+0000", 2023-10-09T06:20:41.009 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6303000000000005e-05 2023-10-09T06:20:41.009 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.010 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:41.010 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.010 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.010 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.010 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:114793 11.5 11:afddf1af:::benchmark_data_smithi064_631903_object5879:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-10-09T06:20:41.010 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:17:46.609723+0000", 2023-10-09T06:20:41.010 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 174.34669203000001, 2023-10-09T06:20:41.010 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2334629800000001, 2023-10-09T06:20:41.010 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:41.010 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:41.010 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:41.010 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:20:41.011 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:41.011 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 114793 2023-10-09T06:20:41.011 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.011 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:41.011 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.011 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:41.011 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.609723+0000", 2023-10-09T06:20:41.011 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.011 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.011 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.011 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:41.011 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.609723+0000", 2023-10-09T06:20:41.012 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.012 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.012 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.012 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:41.012 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.609722+0000", 2023-10-09T06:20:41.012 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:20:41.012 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.012 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.012 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:41.012 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.609728+0000", 2023-10-09T06:20:41.012 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7039999999999999e-06 2023-10-09T06:20:41.012 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.013 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.013 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:41.013 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.609739+0000", 2023-10-09T06:20:41.013 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0693000000000001e-05 2023-10-09T06:20:41.013 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.013 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.013 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:41.013 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.609745+0000", 2023-10-09T06:20:41.013 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7279999999999998e-06 2023-10-09T06:20:41.013 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.013 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.013 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.014 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.609813+0000", 2023-10-09T06:20:41.014 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8081000000000001e-05 2023-10-09T06:20:41.014 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.014 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.014 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:41.014 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.609830+0000", 2023-10-09T06:20:41.014 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6167e-05 2023-10-09T06:20:41.014 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.014 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.014 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.014 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:47.613206+0000", 2023-10-09T06:20:41.015 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.003376144 2023-10-09T06:20:41.015 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.015 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.015 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:41.015 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:47.613220+0000", 2023-10-09T06:20:41.015 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4768e-05 2023-10-09T06:20:41.015 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.015 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.015 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.015 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.801212+0000", 2023-10-09T06:20:41.015 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.187991953 2023-10-09T06:20:41.015 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.016 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.016 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:41.016 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.801309+0000", 2023-10-09T06:20:41.016 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.6799999999999995e-05 2023-10-09T06:20:41.016 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.016 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.016 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 3,5,6", 2023-10-09T06:20:41.016 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.801755+0000", 2023-10-09T06:20:41.016 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00044560900000000002 2023-10-09T06:20:41.016 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.016 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.016 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:41.017 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.812096+0000", 2023-10-09T06:20:41.017 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010341331 2023-10-09T06:20:41.017 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.017 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.017 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.017 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.812130+0000", 2023-10-09T06:20:41.017 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.4175999999999999e-05 2023-10-09T06:20:41.017 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.017 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.017 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.017 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.812167+0000", 2023-10-09T06:20:41.017 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6251999999999998e-05 2023-10-09T06:20:41.018 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.018 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.018 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.018 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.843000+0000", 2023-10-09T06:20:41.018 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.030833257999999999 2023-10-09T06:20:41.018 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.018 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.018 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:41.018 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.843063+0000", 2023-10-09T06:20:41.018 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3318999999999995e-05 2023-10-09T06:20:41.018 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.018 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.019 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:41.019 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.843186+0000", 2023-10-09T06:20:41.019 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012310399999999999 2023-10-09T06:20:41.019 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.019 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:41.019 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.019 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.019 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.019 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:114794 11.3 11:c4d965d1:::benchmark_data_smithi064_631903_object5880:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-10-09T06:20:41.019 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:17:46.611601+0000", 2023-10-09T06:20:41.019 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 174.344814644, 2023-10-09T06:20:41.019 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.130719021, 2023-10-09T06:20:41.020 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:41.020 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:41.020 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:41.020 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:20:41.020 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:41.020 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 114794 2023-10-09T06:20:41.020 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.020 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:41.020 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.020 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:41.020 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.611601+0000", 2023-10-09T06:20:41.020 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.021 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.021 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.021 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:41.021 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.611601+0000", 2023-10-09T06:20:41.021 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.021 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.021 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.021 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:41.021 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.611599+0000", 2023-10-09T06:20:41.021 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:41.021 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.021 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.022 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:41.022 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.611605+0000", 2023-10-09T06:20:41.022 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9379999999999997e-06 2023-10-09T06:20:41.022 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.022 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.022 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:41.022 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.611617+0000", 2023-10-09T06:20:41.022 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1223000000000001e-05 2023-10-09T06:20:41.022 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.022 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.022 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:41.022 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.611623+0000", 2023-10-09T06:20:41.023 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1970000000000001e-06 2023-10-09T06:20:41.023 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.023 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.023 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.023 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.611674+0000", 2023-10-09T06:20:41.023 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0757999999999998e-05 2023-10-09T06:20:41.023 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.023 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.023 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:41.023 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.611682+0000", 2023-10-09T06:20:41.023 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8089999999999999e-06 2023-10-09T06:20:41.024 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.024 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.024 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.024 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:47.613718+0000", 2023-10-09T06:20:41.024 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0020357479999999 2023-10-09T06:20:41.024 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.024 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.024 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:41.024 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:47.613728+0000", 2023-10-09T06:20:41.024 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.9019999999999994e-06 2023-10-09T06:20:41.024 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.024 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.025 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.025 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.720188+0000", 2023-10-09T06:20:41.025 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.106459514 2023-10-09T06:20:41.025 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.025 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.025 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:41.025 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.720322+0000", 2023-10-09T06:20:41.025 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000134122 2023-10-09T06:20:41.025 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.025 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.025 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T06:20:41.026 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.720877+0000", 2023-10-09T06:20:41.026 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00055509600000000004 2023-10-09T06:20:41.026 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.026 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.026 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.026 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.731856+0000", 2023-10-09T06:20:41.026 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010979569 2023-10-09T06:20:41.026 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.026 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.026 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:41.026 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.731867+0000", 2023-10-09T06:20:41.027 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0282000000000001e-05 2023-10-09T06:20:41.027 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.027 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.027 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.027 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.731902+0000", 2023-10-09T06:20:41.027 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5599000000000003e-05 2023-10-09T06:20:41.027 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.027 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.027 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.027 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.742157+0000", 2023-10-09T06:20:41.027 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010254776 2023-10-09T06:20:41.028 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.028 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.028 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:41.028 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.742218+0000", 2023-10-09T06:20:41.028 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0603000000000002e-05 2023-10-09T06:20:41.028 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.028 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.028 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:41.028 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.742320+0000", 2023-10-09T06:20:41.028 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000102086 2023-10-09T06:20:41.028 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.029 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:41.029 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.029 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.029 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.029 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:114797 11.4 11:282f7824:::benchmark_data_smithi064_631903_object5883:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-10-09T06:20:41.029 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:17:46.615504+0000", 2023-10-09T06:20:41.029 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 174.34091117899999, 2023-10-09T06:20:41.029 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.230211868, 2023-10-09T06:20:41.029 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:41.029 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:41.029 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:41.029 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:20:41.030 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:41.030 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 114797 2023-10-09T06:20:41.030 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.030 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:41.030 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.030 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:41.030 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615504+0000", 2023-10-09T06:20:41.030 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.030 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.030 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.030 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:41.031 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615504+0000", 2023-10-09T06:20:41.031 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.031 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.031 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.031 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:41.031 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615503+0000", 2023-10-09T06:20:41.031 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:41.031 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.031 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.031 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:41.031 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615509+0000", 2023-10-09T06:20:41.032 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0399999999999998e-06 2023-10-09T06:20:41.032 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.032 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.032 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:41.032 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615521+0000", 2023-10-09T06:20:41.032 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2072000000000001e-05 2023-10-09T06:20:41.032 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.032 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.032 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:41.032 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615526+0000", 2023-10-09T06:20:41.032 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3510000000000001e-06 2023-10-09T06:20:41.032 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.033 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.033 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.033 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615579+0000", 2023-10-09T06:20:41.033 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3093999999999997e-05 2023-10-09T06:20:41.033 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.033 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.033 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:41.033 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615588+0000", 2023-10-09T06:20:41.033 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1749999999999994e-06 2023-10-09T06:20:41.033 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.033 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.034 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.034 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:47.616324+0000", 2023-10-09T06:20:41.034 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.000735191 2023-10-09T06:20:41.034 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.034 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.034 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:41.034 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:47.616333+0000", 2023-10-09T06:20:41.034 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.5899999999999997e-06 2023-10-09T06:20:41.034 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.034 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.034 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.035 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.757868+0000", 2023-10-09T06:20:41.035 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1415350879999999 2023-10-09T06:20:41.035 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.035 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.035 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:41.035 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.757956+0000", 2023-10-09T06:20:41.035 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.7495000000000002e-05 2023-10-09T06:20:41.035 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.035 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.035 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T06:20:41.035 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.758426+0000", 2023-10-09T06:20:41.036 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00047022900000000001 2023-10-09T06:20:41.036 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.036 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.036 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:41.036 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.767516+0000", 2023-10-09T06:20:41.036 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0090905719999999999 2023-10-09T06:20:41.036 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.036 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.036 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.036 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.779063+0000", 2023-10-09T06:20:41.036 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011546172 2023-10-09T06:20:41.036 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.037 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.037 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.037 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.779515+0000", 2023-10-09T06:20:41.037 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00045276600000000001 2023-10-09T06:20:41.037 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.037 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.037 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.037 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.845550+0000", 2023-10-09T06:20:41.037 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.066034143000000003 2023-10-09T06:20:41.037 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.037 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.038 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:41.038 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.845613+0000", 2023-10-09T06:20:41.038 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3727999999999997e-05 2023-10-09T06:20:41.038 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.038 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.038 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:41.038 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.845716+0000", 2023-10-09T06:20:41.038 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010272000000000001 2023-10-09T06:20:41.038 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.038 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:41.038 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.039 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.039 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.039 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:114799 11.3 11:cd132cdc:::benchmark_data_smithi064_631903_object5885:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-10-09T06:20:41.039 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:17:46.615538+0000", 2023-10-09T06:20:41.039 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 174.34087765699999, 2023-10-09T06:20:41.039 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1269155259999999, 2023-10-09T06:20:41.039 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:41.039 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:41.039 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:41.039 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:20:41.039 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:41.039 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 114799 2023-10-09T06:20:41.040 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.040 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:41.040 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.040 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:41.040 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615538+0000", 2023-10-09T06:20:41.040 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.040 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.040 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.040 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:41.040 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615538+0000", 2023-10-09T06:20:41.040 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.041 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.041 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.041 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:41.041 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615537+0000", 2023-10-09T06:20:41.041 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:41.041 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.041 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.041 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:41.041 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615540+0000", 2023-10-09T06:20:41.041 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2180000000000002e-06 2023-10-09T06:20:41.041 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.042 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.042 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:41.042 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615551+0000", 2023-10-09T06:20:41.042 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0963e-05 2023-10-09T06:20:41.042 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.042 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.042 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:41.042 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615554+0000", 2023-10-09T06:20:41.042 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1030000000000002e-06 2023-10-09T06:20:41.042 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.042 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.043 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.043 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615596+0000", 2023-10-09T06:20:41.043 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2035e-05 2023-10-09T06:20:41.043 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.043 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.043 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:41.043 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.615602+0000", 2023-10-09T06:20:41.043 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.959e-06 2023-10-09T06:20:41.043 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.043 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.043 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.043 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:47.613755+0000", 2023-10-09T06:20:41.044 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99815351699999999 2023-10-09T06:20:41.044 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.044 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.044 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:41.044 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:47.613764+0000", 2023-10-09T06:20:41.044 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1039999999999996e-06 2023-10-09T06:20:41.044 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.044 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.044 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.044 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.728423+0000", 2023-10-09T06:20:41.044 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1146591880000001 2023-10-09T06:20:41.045 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.045 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.045 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:41.045 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.728559+0000", 2023-10-09T06:20:41.045 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013571699999999999 2023-10-09T06:20:41.045 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.045 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.045 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T06:20:41.045 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.728700+0000", 2023-10-09T06:20:41.045 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00014088100000000001 2023-10-09T06:20:41.045 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.046 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.046 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:41.046 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.732137+0000", 2023-10-09T06:20:41.046 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0034370870000000001 2023-10-09T06:20:41.046 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.046 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.046 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.046 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.732626+0000", 2023-10-09T06:20:41.046 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00048875199999999996 2023-10-09T06:20:41.046 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.046 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.046 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.047 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.734156+0000", 2023-10-09T06:20:41.047 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001529982 2023-10-09T06:20:41.047 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.047 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.047 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.047 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.742358+0000", 2023-10-09T06:20:41.047 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0082017779999999998 2023-10-09T06:20:41.047 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.047 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.047 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:41.047 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.742393+0000", 2023-10-09T06:20:41.048 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5673999999999998e-05 2023-10-09T06:20:41.048 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.048 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.048 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:41.048 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.742453+0000", 2023-10-09T06:20:41.048 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9889000000000002e-05 2023-10-09T06:20:41.048 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.048 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:41.048 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.048 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.048 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.049 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:114800 11.4 11:23e3be1d:::benchmark_data_smithi064_631903_object5886:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-10-09T06:20:41.049 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:17:46.616114+0000", 2023-10-09T06:20:41.049 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 174.34030124399999, 2023-10-09T06:20:41.049 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2298217999999999, 2023-10-09T06:20:41.049 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:41.049 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:41.049 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:41.049 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:20:41.049 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:41.049 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 114800 2023-10-09T06:20:41.049 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.049 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:41.050 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.050 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:41.050 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.616114+0000", 2023-10-09T06:20:41.050 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.050 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.050 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.050 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:41.050 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.616114+0000", 2023-10-09T06:20:41.050 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.050 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.050 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.051 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:41.051 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.616113+0000", 2023-10-09T06:20:41.051 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:41.051 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.051 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.051 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:41.051 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.616118+0000", 2023-10-09T06:20:41.051 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0690000000000003e-06 2023-10-09T06:20:41.051 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.051 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.051 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:41.051 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.616127+0000", 2023-10-09T06:20:41.052 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7089999999999997e-06 2023-10-09T06:20:41.052 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.052 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.052 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:41.052 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.616132+0000", 2023-10-09T06:20:41.052 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2810000000000002e-06 2023-10-09T06:20:41.052 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.052 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.052 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.052 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.616181+0000", 2023-10-09T06:20:41.052 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9543000000000003e-05 2023-10-09T06:20:41.053 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.053 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.053 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:41.053 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:46.616191+0000", 2023-10-09T06:20:41.053 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0176999999999999e-05 2023-10-09T06:20:41.053 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.053 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.053 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.053 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:47.616355+0000", 2023-10-09T06:20:41.053 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.000163873 2023-10-09T06:20:41.053 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.054 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.054 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:41.054 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:47.616363+0000", 2023-10-09T06:20:41.054 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8709999999999997e-06 2023-10-09T06:20:41.054 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.054 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.054 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.054 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.764770+0000", 2023-10-09T06:20:41.054 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1484067330000001 2023-10-09T06:20:41.054 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.054 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.054 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:41.055 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.764891+0000", 2023-10-09T06:20:41.055 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012121900000000001 2023-10-09T06:20:41.055 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.055 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.055 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T06:20:41.055 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.765029+0000", 2023-10-09T06:20:41.055 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013798900000000001 2023-10-09T06:20:41.055 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.055 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.055 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:41.055 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.767525+0000", 2023-10-09T06:20:41.056 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0024960149999999999 2023-10-09T06:20:41.056 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.056 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.056 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.056 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.779447+0000", 2023-10-09T06:20:41.056 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011922532 2023-10-09T06:20:41.056 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.056 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.056 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.056 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.785755+0000", 2023-10-09T06:20:41.056 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0063076829999999997 2023-10-09T06:20:41.056 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.057 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.057 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.057 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.845767+0000", 2023-10-09T06:20:41.057 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.060011768 2023-10-09T06:20:41.057 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.057 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.057 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:41.057 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.845817+0000", 2023-10-09T06:20:41.057 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0129000000000001e-05 2023-10-09T06:20:41.057 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.057 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.058 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:41.058 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:17:48.845936+0000", 2023-10-09T06:20:41.058 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000118819 2023-10-09T06:20:41.058 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.058 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:41.058 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.058 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.058 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.058 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:149236 11.5 11:a978e662:::benchmark_data_smithi064_631903_object40322:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e611)", 2023-10-09T06:20:41.058 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:18:31.999271+0000", 2023-10-09T06:20:41.058 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 128.957144473, 2023-10-09T06:20:41.059 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.95864518, 2023-10-09T06:20:41.059 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:41.059 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:41.059 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:41.059 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:20:41.059 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:41.059 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149236 2023-10-09T06:20:41.059 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.059 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:41.059 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.059 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:41.060 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:31.999271+0000", 2023-10-09T06:20:41.060 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.060 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.060 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.060 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:41.060 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:31.999271+0000", 2023-10-09T06:20:41.060 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.060 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.060 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.060 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:41.060 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:31.999269+0000", 2023-10-09T06:20:41.060 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:41.061 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.061 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.061 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:41.061 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:31.999281+0000", 2023-10-09T06:20:41.061 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2279000000000001e-05 2023-10-09T06:20:41.061 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.061 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.061 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:41.061 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:31.999296+0000", 2023-10-09T06:20:41.061 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4355000000000001e-05 2023-10-09T06:20:41.061 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.061 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.062 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:41.062 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:31.999303+0000", 2023-10-09T06:20:41.062 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9249999999999998e-06 2023-10-09T06:20:41.062 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.062 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.062 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.062 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.000192+0000", 2023-10-09T06:20:41.062 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00088970499999999997 2023-10-09T06:20:41.062 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.062 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.062 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op must wait for map", 2023-10-09T06:20:41.063 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.000199+0000", 2023-10-09T06:20:41.063 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.2089999999999999e-06 2023-10-09T06:20:41.063 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.063 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.063 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.063 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.029567+0000", 2023-10-09T06:20:41.063 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.029367606000000001 2023-10-09T06:20:41.063 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.063 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.063 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:41.063 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.029576+0000", 2023-10-09T06:20:41.063 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0399999999999998e-06 2023-10-09T06:20:41.064 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.064 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.064 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.064 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.877682+0000", 2023-10-09T06:20:41.064 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.84810606799999999 2023-10-09T06:20:41.064 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.064 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.064 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:41.064 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.877695+0000", 2023-10-09T06:20:41.064 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2714000000000001e-05 2023-10-09T06:20:41.064 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.064 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.065 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.065 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.925938+0000", 2023-10-09T06:20:41.065 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0482431720000001 2023-10-09T06:20:41.065 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.065 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.065 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:41.065 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.926107+0000", 2023-10-09T06:20:41.065 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000169149 2023-10-09T06:20:41.065 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.065 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.065 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:20:41.066 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.926239+0000", 2023-10-09T06:20:41.066 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013144 2023-10-09T06:20:41.066 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.066 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.066 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.066 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.939732+0000", 2023-10-09T06:20:41.066 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013493392 2023-10-09T06:20:41.066 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.066 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.066 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:41.066 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.939791+0000", 2023-10-09T06:20:41.066 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9382999999999997e-05 2023-10-09T06:20:41.067 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.067 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.067 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.067 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.957603+0000", 2023-10-09T06:20:41.067 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.017811616999999998 2023-10-09T06:20:41.067 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.067 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.067 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:41.067 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.957662+0000", 2023-10-09T06:20:41.067 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9200000000000002e-05 2023-10-09T06:20:41.067 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.068 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.068 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:41.068 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.957916+0000", 2023-10-09T06:20:41.068 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00025382099999999999 2023-10-09T06:20:41.068 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.068 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:41.068 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.068 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.068 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.068 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:149254 11.5 11:afb97216:::benchmark_data_smithi064_631903_object40340:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e611)", 2023-10-09T06:20:41.068 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:18:32.038422+0000", 2023-10-09T06:20:41.068 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 128.91799358200001, 2023-10-09T06:20:41.069 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.9197044619999999, 2023-10-09T06:20:41.069 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:41.069 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:41.069 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:41.069 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:20:41.069 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:41.069 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149254 2023-10-09T06:20:41.069 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.069 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:41.069 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.069 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:41.069 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.038422+0000", 2023-10-09T06:20:41.070 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.070 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.070 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.070 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:41.070 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.038422+0000", 2023-10-09T06:20:41.070 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.070 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.070 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.070 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:41.070 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.038420+0000", 2023-10-09T06:20:41.070 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:41.071 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.071 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.071 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:41.071 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.038427+0000", 2023-10-09T06:20:41.071 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.742e-06 2023-10-09T06:20:41.071 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.071 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.071 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:41.071 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.038440+0000", 2023-10-09T06:20:41.071 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3239e-05 2023-10-09T06:20:41.071 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.071 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.072 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:41.072 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.038448+0000", 2023-10-09T06:20:41.072 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0740000000000004e-06 2023-10-09T06:20:41.072 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.072 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.072 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.072 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.040109+0000", 2023-10-09T06:20:41.072 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0016614010000000001 2023-10-09T06:20:41.072 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.072 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.072 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:41.072 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.040119+0000", 2023-10-09T06:20:41.073 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0343000000000001e-05 2023-10-09T06:20:41.073 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.073 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.073 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.073 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.877719+0000", 2023-10-09T06:20:41.073 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.83759995099999995 2023-10-09T06:20:41.073 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.073 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.073 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:41.073 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.877728+0000", 2023-10-09T06:20:41.073 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6100000000000006e-06 2023-10-09T06:20:41.073 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.074 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.074 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.074 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.929371+0000", 2023-10-09T06:20:41.074 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0516434079999999 2023-10-09T06:20:41.074 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.074 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.074 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:41.074 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.929476+0000", 2023-10-09T06:20:41.074 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010446399999999999 2023-10-09T06:20:41.074 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.074 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.077 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:20:41.077 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.929566+0000", 2023-10-09T06:20:41.077 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0167999999999995e-05 2023-10-09T06:20:41.077 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.078 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.078 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.078 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.939782+0000", 2023-10-09T06:20:41.078 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010215805999999999 2023-10-09T06:20:41.078 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.078 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.078 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:41.078 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.939800+0000", 2023-10-09T06:20:41.078 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7825999999999998e-05 2023-10-09T06:20:41.078 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.078 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.079 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.079 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.957959+0000", 2023-10-09T06:20:41.079 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018159412 2023-10-09T06:20:41.079 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.079 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.079 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:41.079 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.957998+0000", 2023-10-09T06:20:41.079 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8588999999999999e-05 2023-10-09T06:20:41.079 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.079 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.079 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:41.080 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.958126+0000", 2023-10-09T06:20:41.080 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000128699 2023-10-09T06:20:41.080 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.080 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:41.080 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.080 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.080 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.080 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:149260 11.5 11:aae66956:::benchmark_data_smithi064_631903_object40346:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e611)", 2023-10-09T06:20:41.080 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:18:32.044475+0000", 2023-10-09T06:20:41.080 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 128.91194080099999, 2023-10-09T06:20:41.080 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.913840333, 2023-10-09T06:20:41.081 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:41.081 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:41.081 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:41.081 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:20:41.081 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:41.081 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149260 2023-10-09T06:20:41.081 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.081 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:41.081 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.081 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:41.081 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.044475+0000", 2023-10-09T06:20:41.082 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.082 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.082 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.082 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:41.082 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.044475+0000", 2023-10-09T06:20:41.082 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.082 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.082 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.082 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:41.082 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.044473+0000", 2023-10-09T06:20:41.082 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:20:41.083 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.083 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.083 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:41.083 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.044479+0000", 2023-10-09T06:20:41.083 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3630000000000002e-06 2023-10-09T06:20:41.083 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.083 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.083 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:41.083 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.044490+0000", 2023-10-09T06:20:41.083 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1437000000000001e-05 2023-10-09T06:20:41.083 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.084 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.084 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:41.084 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.044495+0000", 2023-10-09T06:20:41.084 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7149999999999997e-06 2023-10-09T06:20:41.084 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.084 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.084 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.084 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.044547+0000", 2023-10-09T06:20:41.084 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1544e-05 2023-10-09T06:20:41.084 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.084 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.084 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:41.085 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.044553+0000", 2023-10-09T06:20:41.085 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6229999999999999e-06 2023-10-09T06:20:41.085 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.085 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.085 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.085 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.877748+0000", 2023-10-09T06:20:41.085 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.83319441400000005 2023-10-09T06:20:41.085 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.085 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.085 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:41.085 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.877755+0000", 2023-10-09T06:20:41.086 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1219999999999997e-06 2023-10-09T06:20:41.086 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.086 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.086 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.086 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.932780+0000", 2023-10-09T06:20:41.086 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0550253350000001 2023-10-09T06:20:41.086 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.086 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.086 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:41.086 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.932927+0000", 2023-10-09T06:20:41.086 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000146718 2023-10-09T06:20:41.086 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.087 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.087 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:20:41.087 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.933080+0000", 2023-10-09T06:20:41.087 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015322799999999999 2023-10-09T06:20:41.087 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.087 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.087 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:41.087 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.939807+0000", 2023-10-09T06:20:41.087 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0067272399999999998 2023-10-09T06:20:41.087 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.087 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.088 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.088 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.945252+0000", 2023-10-09T06:20:41.088 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.005444979 2023-10-09T06:20:41.088 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.088 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.088 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.088 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.958161+0000", 2023-10-09T06:20:41.088 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.012909109 2023-10-09T06:20:41.088 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.088 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.088 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:41.089 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.958199+0000", 2023-10-09T06:20:41.089 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7379000000000002e-05 2023-10-09T06:20:41.089 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.089 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.089 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:41.089 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.958315+0000", 2023-10-09T06:20:41.089 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000116152 2023-10-09T06:20:41.089 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.089 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:41.089 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.089 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.090 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.090 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:149290 11.5 11:a82ec06a:::benchmark_data_smithi064_631903_object40376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e611)", 2023-10-09T06:20:41.090 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:18:32.083507+0000", 2023-10-09T06:20:41.090 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 128.872908497, 2023-10-09T06:20:41.090 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8750918620000001, 2023-10-09T06:20:41.090 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:41.090 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:41.090 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:41.090 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:20:41.090 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:41.090 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 149290 2023-10-09T06:20:41.091 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.091 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:41.091 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.091 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:41.091 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.083507+0000", 2023-10-09T06:20:41.091 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.091 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.091 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.091 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:41.091 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.083507+0000", 2023-10-09T06:20:41.091 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.091 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.092 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.092 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:41.092 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.083506+0000", 2023-10-09T06:20:41.092 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:20:41.092 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.092 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.092 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:41.092 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.083511+0000", 2023-10-09T06:20:41.092 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2689999999999999e-06 2023-10-09T06:20:41.092 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.092 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.092 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:41.093 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.083522+0000", 2023-10-09T06:20:41.093 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0564000000000001e-05 2023-10-09T06:20:41.093 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.093 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.093 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:41.093 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.083528+0000", 2023-10-09T06:20:41.093 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0959999999999997e-06 2023-10-09T06:20:41.093 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.093 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.093 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.093 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.083590+0000", 2023-10-09T06:20:41.094 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1603e-05 2023-10-09T06:20:41.094 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.094 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.094 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:41.094 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.083600+0000", 2023-10-09T06:20:41.094 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0655999999999999e-05 2023-10-09T06:20:41.094 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.094 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.094 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.094 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.877776+0000", 2023-10-09T06:20:41.095 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.79417600499999996 2023-10-09T06:20:41.095 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.095 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.095 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:41.095 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:32.877783+0000", 2023-10-09T06:20:41.095 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.046e-06 2023-10-09T06:20:41.095 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.095 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.095 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.095 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.936613+0000", 2023-10-09T06:20:41.095 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.058829324 2023-10-09T06:20:41.096 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.096 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.096 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:41.096 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.936715+0000", 2023-10-09T06:20:41.096 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010241 2023-10-09T06:20:41.096 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.096 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.096 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:20:41.096 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.936798+0000", 2023-10-09T06:20:41.096 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.3375999999999996e-05 2023-10-09T06:20:41.096 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.096 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.097 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:41.097 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.939886+0000", 2023-10-09T06:20:41.097 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0030878120000000001 2023-10-09T06:20:41.097 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.097 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.097 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.097 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.958350+0000", 2023-10-09T06:20:41.097 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.018463632000000001 2023-10-09T06:20:41.097 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.097 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.097 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.097 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.958436+0000", 2023-10-09T06:20:41.098 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6231999999999997e-05 2023-10-09T06:20:41.098 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.098 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.098 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:41.098 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.958480+0000", 2023-10-09T06:20:41.098 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4036999999999999e-05 2023-10-09T06:20:41.098 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.098 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.098 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:41.098 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:33.958599+0000", 2023-10-09T06:20:41.098 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011859 2023-10-09T06:20:41.098 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.099 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:41.099 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.099 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.099 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.099 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:151821 11.5 11:aef9b580:::benchmark_data_smithi064_631903_object42907:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e614)", 2023-10-09T06:20:41.099 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:18:37.129723+0000", 2023-10-09T06:20:41.099 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 123.826692167, 2023-10-09T06:20:41.099 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.28569557099999998, 2023-10-09T06:20:41.099 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:41.099 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:41.099 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:41.100 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:20:41.100 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:41.100 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 151821 2023-10-09T06:20:41.100 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.100 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:41.100 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.100 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:41.100 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:37.129723+0000", 2023-10-09T06:20:41.100 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.100 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.100 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.101 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:41.101 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:37.129723+0000", 2023-10-09T06:20:41.101 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.101 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.101 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.101 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:41.101 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:37.129723+0000", 2023-10-09T06:20:41.101 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:41.101 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.101 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.101 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:41.101 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:37.129725+0000", 2023-10-09T06:20:41.102 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4420000000000001e-06 2023-10-09T06:20:41.102 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.102 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.102 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:41.102 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:37.129731+0000", 2023-10-09T06:20:41.102 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4360000000000001e-06 2023-10-09T06:20:41.102 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.102 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.102 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:41.102 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:37.129734+0000", 2023-10-09T06:20:41.102 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.36e-06 2023-10-09T06:20:41.102 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.103 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.103 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.103 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:37.398068+0000", 2023-10-09T06:20:41.103 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.268333504 2023-10-09T06:20:41.103 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.103 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.103 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:41.103 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:37.398197+0000", 2023-10-09T06:20:41.103 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000129171 2023-10-09T06:20:41.103 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.103 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.103 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:20:41.104 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:37.398315+0000", 2023-10-09T06:20:41.104 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011855500000000001 2023-10-09T06:20:41.104 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.104 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.104 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:41.104 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:37.404922+0000", 2023-10-09T06:20:41.104 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0066069650000000002 2023-10-09T06:20:41.104 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.104 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.104 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.104 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:37.407448+0000", 2023-10-09T06:20:41.104 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0025257869999999998 2023-10-09T06:20:41.105 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.105 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.105 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.105 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:37.415254+0000", 2023-10-09T06:20:41.105 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0078058240000000003 2023-10-09T06:20:41.105 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.105 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.105 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:41.105 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:37.415301+0000", 2023-10-09T06:20:41.105 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.6925000000000002e-05 2023-10-09T06:20:41.105 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.106 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.106 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:41.106 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:18:37.415419+0000", 2023-10-09T06:20:41.106 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011794300000000001 2023-10-09T06:20:41.106 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.106 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:41.106 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.106 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.106 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.106 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:185635 11.3 11:c014a9ac:::benchmark_data_smithi064_631903_object76721:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e625)", 2023-10-09T06:20:41.106 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:19:19.288350+0000", 2023-10-09T06:20:41.106 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 81.668065088999995, 2023-10-09T06:20:41.107 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.23022004099999999, 2023-10-09T06:20:41.107 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:41.107 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:41.107 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:41.107 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:20:41.107 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:41.107 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 185635 2023-10-09T06:20:41.107 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.107 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:41.107 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.107 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:41.108 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.288350+0000", 2023-10-09T06:20:41.108 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.108 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.108 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.108 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:41.108 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.288350+0000", 2023-10-09T06:20:41.108 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.108 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.108 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.108 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:41.108 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.288349+0000", 2023-10-09T06:20:41.109 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:41.109 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.109 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.109 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:41.109 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.288356+0000", 2023-10-09T06:20:41.109 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1600000000000001e-06 2023-10-09T06:20:41.109 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.109 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.109 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:41.109 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.288373+0000", 2023-10-09T06:20:41.109 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7204e-05 2023-10-09T06:20:41.110 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.110 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.110 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:41.110 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.288380+0000", 2023-10-09T06:20:41.110 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4740000000000002e-06 2023-10-09T06:20:41.110 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.110 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.110 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.110 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.288504+0000", 2023-10-09T06:20:41.110 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012409300000000001 2023-10-09T06:20:41.110 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.110 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.111 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:41.111 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.288593+0000", 2023-10-09T06:20:41.111 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9639e-05 2023-10-09T06:20:41.111 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.111 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.111 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:20:41.111 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.288672+0000", 2023-10-09T06:20:41.111 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8109999999999996e-05 2023-10-09T06:20:41.111 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.111 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.111 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:41.111 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.291782+0000", 2023-10-09T06:20:41.112 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0031104370000000002 2023-10-09T06:20:41.112 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.112 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.112 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.112 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.292168+0000", 2023-10-09T06:20:41.112 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000385714 2023-10-09T06:20:41.112 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.112 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.112 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.112 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.518505+0000", 2023-10-09T06:20:41.112 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.22633707 2023-10-09T06:20:41.112 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.113 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.113 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:41.113 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.518542+0000", 2023-10-09T06:20:41.113 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7651999999999998e-05 2023-10-09T06:20:41.113 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.113 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.113 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:41.113 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.518570+0000", 2023-10-09T06:20:41.113 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.7801999999999999e-05 2023-10-09T06:20:41.113 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.113 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:41.113 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.114 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.114 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.114 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:185637 11.f 11:f6db4a7e:::benchmark_data_smithi064_631903_object76723:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e625)", 2023-10-09T06:20:41.114 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:19:19.289688+0000", 2023-10-09T06:20:41.114 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 81.666727703000006, 2023-10-09T06:20:41.114 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.23706406599999999, 2023-10-09T06:20:41.114 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:41.114 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:41.114 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:41.114 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:20:41.114 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:41.115 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 185637 2023-10-09T06:20:41.115 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.115 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:41.115 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.115 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:41.115 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.289688+0000", 2023-10-09T06:20:41.115 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.115 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.115 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.115 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:41.115 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.289688+0000", 2023-10-09T06:20:41.115 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.116 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.116 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.116 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:41.116 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.289687+0000", 2023-10-09T06:20:41.116 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:20:41.116 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.116 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.116 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:41.116 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.289695+0000", 2023-10-09T06:20:41.116 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.5750000000000001e-06 2023-10-09T06:20:41.116 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.117 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.117 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:41.117 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.289710+0000", 2023-10-09T06:20:41.117 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5017e-05 2023-10-09T06:20:41.117 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.117 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.117 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:41.117 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.289716+0000", 2023-10-09T06:20:41.117 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2389999999999999e-06 2023-10-09T06:20:41.117 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.117 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.117 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.118 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.289771+0000", 2023-10-09T06:20:41.118 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4199000000000001e-05 2023-10-09T06:20:41.118 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.118 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.118 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:41.118 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.289905+0000", 2023-10-09T06:20:41.118 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000134647 2023-10-09T06:20:41.118 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.118 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.118 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:20:41.118 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.290037+0000", 2023-10-09T06:20:41.119 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013184899999999999 2023-10-09T06:20:41.119 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.119 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.119 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.119 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.292900+0000", 2023-10-09T06:20:41.119 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0028627660000000001 2023-10-09T06:20:41.119 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.119 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.119 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:41.119 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.292979+0000", 2023-10-09T06:20:41.119 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8776000000000006e-05 2023-10-09T06:20:41.119 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.120 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.120 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.120 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.524136+0000", 2023-10-09T06:20:41.120 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.23115707399999999 2023-10-09T06:20:41.120 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.120 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.120 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:41.120 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.526708+0000", 2023-10-09T06:20:41.120 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00257186 2023-10-09T06:20:41.120 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.120 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.121 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:41.121 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.526752+0000", 2023-10-09T06:20:41.121 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4023999999999997e-05 2023-10-09T06:20:41.121 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.121 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:41.121 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.121 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.121 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.121 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:200519 11.f 11:fdc8ce9e:::benchmark_data_smithi064_631903_object91605:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e629)", 2023-10-09T06:20:41.121 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:19:37.383308+0000", 2023-10-09T06:20:41.121 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 63.573107014000001, 2023-10-09T06:20:41.121 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.19787400099999999, 2023-10-09T06:20:41.122 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:41.122 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:41.122 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:41.122 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:20:41.122 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:41.122 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 200519 2023-10-09T06:20:41.122 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.122 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:41.122 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.122 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:41.122 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:37.383308+0000", 2023-10-09T06:20:41.123 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.123 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.123 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.123 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:41.123 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:37.383308+0000", 2023-10-09T06:20:41.123 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.123 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.123 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.123 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:41.123 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:37.383307+0000", 2023-10-09T06:20:41.123 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:41.124 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.124 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.124 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:41.124 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:37.383314+0000", 2023-10-09T06:20:41.124 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7009999999999999e-06 2023-10-09T06:20:41.124 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.124 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.124 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:41.124 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:37.383329+0000", 2023-10-09T06:20:41.124 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5545000000000001e-05 2023-10-09T06:20:41.124 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.124 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.125 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:41.125 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:37.383336+0000", 2023-10-09T06:20:41.125 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0419999999999997e-06 2023-10-09T06:20:41.125 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.125 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.125 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.125 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:37.383388+0000", 2023-10-09T06:20:41.125 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1527000000000003e-05 2023-10-09T06:20:41.125 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.125 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.125 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:41.126 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:37.383522+0000", 2023-10-09T06:20:41.126 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013400799999999999 2023-10-09T06:20:41.126 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.126 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.126 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:20:41.126 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:37.383648+0000", 2023-10-09T06:20:41.126 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012629300000000001 2023-10-09T06:20:41.126 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.126 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.126 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:41.126 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:37.393001+0000", 2023-10-09T06:20:41.127 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0093529930000000004 2023-10-09T06:20:41.127 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.127 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.127 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.127 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:37.396214+0000", 2023-10-09T06:20:41.127 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0032127850000000001 2023-10-09T06:20:41.127 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.127 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.127 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.127 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:37.581101+0000", 2023-10-09T06:20:41.127 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.18488727399999999 2023-10-09T06:20:41.127 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.128 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.128 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:41.128 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:37.581141+0000", 2023-10-09T06:20:41.128 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0135000000000001e-05 2023-10-09T06:20:41.128 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.128 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.128 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:41.128 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:37.581182+0000", 2023-10-09T06:20:41.128 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1158000000000001e-05 2023-10-09T06:20:41.128 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.128 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:41.129 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.129 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.129 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.129 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:201513 11.3 11:c8ea5233:::benchmark_data_smithi064_631903_object92599:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e629)", 2023-10-09T06:20:41.129 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:19:38.673268+0000", 2023-10-09T06:20:41.129 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 62.283147433000003, 2023-10-09T06:20:41.129 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.19511990100000001, 2023-10-09T06:20:41.129 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:41.129 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:41.129 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:41.129 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:20:41.130 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:41.130 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 201513 2023-10-09T06:20:41.130 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.130 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:41.130 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.130 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:41.130 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:38.673268+0000", 2023-10-09T06:20:41.130 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.130 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.130 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.130 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:41.131 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:38.673268+0000", 2023-10-09T06:20:41.131 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:41.131 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.131 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.131 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:41.131 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:38.673266+0000", 2023-10-09T06:20:41.131 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:41.131 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.131 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.131 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:41.131 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:38.673277+0000", 2023-10-09T06:20:41.132 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1246999999999999e-05 2023-10-09T06:20:41.132 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.132 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.132 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:41.132 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:38.673294+0000", 2023-10-09T06:20:41.132 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.6909999999999999e-05 2023-10-09T06:20:41.132 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.132 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.132 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:41.132 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:38.673301+0000", 2023-10-09T06:20:41.132 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9929999999999998e-06 2023-10-09T06:20:41.133 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.133 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.133 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:41.133 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:38.673353+0000", 2023-10-09T06:20:41.133 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1272e-05 2023-10-09T06:20:41.133 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.133 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.133 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:41.133 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:38.673485+0000", 2023-10-09T06:20:41.133 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013280900000000001 2023-10-09T06:20:41.133 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.133 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.134 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:20:41.134 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:38.673615+0000", 2023-10-09T06:20:41.134 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000129173 2023-10-09T06:20:41.134 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.134 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.134 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:41.134 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:38.676292+0000", 2023-10-09T06:20:41.134 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0026776460000000001 2023-10-09T06:20:41.134 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.134 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.134 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.135 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:38.677591+0000", 2023-10-09T06:20:41.135 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0012983840000000001 2023-10-09T06:20:41.135 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.135 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.135 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:41.135 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:38.868298+0000", 2023-10-09T06:20:41.135 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.19070709799999999 2023-10-09T06:20:41.135 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.135 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.135 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:41.135 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:38.868351+0000", 2023-10-09T06:20:41.136 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3192000000000002e-05 2023-10-09T06:20:41.136 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:41.136 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:41.136 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:41.136 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:38.868388+0000", 2023-10-09T06:20:41.136 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6893000000000001e-05 2023-10-09T06:20:41.136 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.136 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:41.136 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.136 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:41.136 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:41.136 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:20:41.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:41.138 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:41.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:41.239 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:41.279 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:41.323 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:41.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:41.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:41.393 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --new-format - testimg 2023-10-09T06:20:41.426 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:41.456 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:41.633 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:20:41.802 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 100% complete...done. 2023-10-09T06:20:41.811 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:20:41.811 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export testimg /tmp/rbd_import_export_695299/img2 2023-10-09T06:20:41.812 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:41.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:41.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:41.919 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:41.949 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:41.989 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:20:41.998 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:42.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:42.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:42.016 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:42.102 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:42.127 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T06:20:42.131 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export testimg - 2023-10-09T06:20:42.132 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:42.228 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:42.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:42.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:42.332 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:42.359 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:42.551 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:42 smithi156 ceph-mon[195101]: pgmap v1674: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 148 KiB/s rd, 64 KiB/s wr, 38 op/s 2023-10-09T06:20:42.606 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:42.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:42.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:42.710 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:42.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:42 smithi064 ceph-mon[212750]: pgmap v1674: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 148 KiB/s rd, 64 KiB/s wr, 38 op/s 2023-10-09T06:20:42.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:42 smithi064 ceph-mon[220044]: pgmap v1674: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 148 KiB/s rd, 64 KiB/s wr, 38 op/s 2023-10-09T06:20:42.745 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:42.803 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:42.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:42.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:42.878 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T06:20:42.881 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm testimg 2023-10-09T06:20:42.907 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:42.938 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:42.976 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T06:20:42.979 INFO:tasks.workunit.client.0.smithi064.stderr:+ cmp /tmp/rbd_import_export_695299/img /tmp/rbd_import_export_695299/img2 2023-10-09T06:20:42.991 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:42.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:42.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:43.095 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:43.125 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:43.174 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:43.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:43.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:43.278 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:43.309 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:43.366 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:43.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:43.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:43.444 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:20:43.470 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:43.500 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:43.548 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:20:43.548 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:20:43.548 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 0 2023-10-09T06:20:43.549 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:20:43.561 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:43.564 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:43 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/183031104' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:43.564 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:43 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:43.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:43 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/183031104' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:43.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:43 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:43.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:43.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:43.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:43 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/183031104' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:43.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:43 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:43.666 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:43.696 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:43.702 INFO:tasks.workunit.client.0.smithi064.stderr:+ cmp /tmp/rbd_import_export_695299/img /tmp/rbd_import_export_695299/img3 2023-10-09T06:20:43.792 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:43.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:43.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:43.885 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:20:43.896 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:43.923 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:43.965 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:43.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:43.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:44.069 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:44.095 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:44.237 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_695299/img /tmp/rbd_import_export_695299/img2 /tmp/rbd_import_export_695299/img3 2023-10-09T06:20:44.268 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:44.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:44.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:44.323 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:20:44.372 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:44.399 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:44.444 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:44.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:44.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:44.549 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:44.577 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:44.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:44 smithi156 ceph-mon[195101]: pgmap v1675: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 126 KiB/s rd, 54 KiB/s wr, 32 op/s 2023-10-09T06:20:44.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:44 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:20:44.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:44 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T06:20:44.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:44 smithi156 ceph-mon[195101]: failed to encode map e651 with expected crc 2023-10-09T06:20:44.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:44 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:20:44.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:44 smithi156 ceph-mon[195101]: osdmap e651: 8 total, 8 up, 6 in 2023-10-09T06:20:44.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:44.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:44 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:44.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:44 smithi156 ceph-mon[195101]: failed to encode map e651 with expected crc 2023-10-09T06:20:44.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:44 smithi156 ceph-mon[195101]: failed to encode map e651 with expected crc 2023-10-09T06:20:44.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:44 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:44.673 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:20:44.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:44 smithi064 ceph-mon[212750]: pgmap v1675: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 126 KiB/s rd, 54 KiB/s wr, 32 op/s 2023-10-09T06:20:44.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:44 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:20:44.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:44 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:20:44.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:44 smithi064 ceph-mon[212750]: failed to encode map e651 with expected crc 2023-10-09T06:20:44.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:44 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:20:44.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:44 smithi064 ceph-mon[212750]: osdmap e651: 8 total, 8 up, 6 in 2023-10-09T06:20:44.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:44.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:44 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:44.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:44 smithi064 ceph-mon[212750]: failed to encode map e651 with expected crc 2023-10-09T06:20:44.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:44 smithi064 ceph-mon[212750]: failed to encode map e651 with expected crc 2023-10-09T06:20:44.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:44 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:44.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:44 smithi064 ceph-mon[220044]: pgmap v1675: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 126 KiB/s rd, 54 KiB/s wr, 32 op/s 2023-10-09T06:20:44.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:44 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:20:44.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:44 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T06:20:44.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:44 smithi064 ceph-mon[220044]: failed to encode map e651 with expected crc 2023-10-09T06:20:44.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:44 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:20:44.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:44 smithi064 ceph-mon[220044]: osdmap e651: 8 total, 8 up, 6 in 2023-10-09T06:20:44.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:44.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:44 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:44.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:44 smithi064 ceph-mon[220044]: failed to encode map e651 with expected crc 2023-10-09T06:20:44.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:44 smithi064 ceph-mon[220044]: failed to encode map e651 with expected crc 2023-10-09T06:20:44.721 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:44 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:45.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:45 smithi156 ceph-mon[195101]: failed to encode map e651 with expected crc 2023-10-09T06:20:45.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:45 smithi156 ceph-mon[195101]: failed to encode map e651 with expected crc 2023-10-09T06:20:45.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:45 smithi156 ceph-mon[195101]: failed to encode map e651 with expected crc 2023-10-09T06:20:45.620 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:45 smithi156 ceph-mon[195101]: failed to encode map e651 with expected crc 2023-10-09T06:20:45.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:20:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:20:45] "GET /metrics HTTP/1.1" 200 34080 "" "Prometheus/2.43.0" 2023-10-09T06:20:45.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:45 smithi064 ceph-mon[212750]: failed to encode map e651 with expected crc 2023-10-09T06:20:45.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:45 smithi064 ceph-mon[212750]: failed to encode map e651 with expected crc 2023-10-09T06:20:45.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:45 smithi064 ceph-mon[212750]: failed to encode map e651 with expected crc 2023-10-09T06:20:45.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:45 smithi064 ceph-mon[212750]: failed to encode map e651 with expected crc 2023-10-09T06:20:45.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:45 smithi064 ceph-mon[220044]: failed to encode map e651 with expected crc 2023-10-09T06:20:45.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:45 smithi064 ceph-mon[220044]: failed to encode map e651 with expected crc 2023-10-09T06:20:45.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:45 smithi064 ceph-mon[220044]: failed to encode map e651 with expected crc 2023-10-09T06:20:45.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:45 smithi064 ceph-mon[220044]: failed to encode map e651 with expected crc 2023-10-09T06:20:46.207 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:20:46.208 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:20:46.208 INFO:teuthology.orchestra.run.smithi156.stdout: "complaint_time": 30, 2023-10-09T06:20:46.208 INFO:teuthology.orchestra.run.smithi156.stdout: "num_blocked_ops": 0 2023-10-09T06:20:46.208 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:20:46.621 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:46 smithi156 ceph-mon[195101]: pgmap v1677: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 261 KiB/s rd, 70 KiB/s wr, 66 op/s 2023-10-09T06:20:46.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:46 smithi064 ceph-mon[212750]: pgmap v1677: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 261 KiB/s rd, 70 KiB/s wr, 66 op/s 2023-10-09T06:20:46.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:46 smithi064 ceph-mon[220044]: pgmap v1677: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 261 KiB/s rd, 70 KiB/s wr, 66 op/s 2023-10-09T06:20:46.935 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:20:47.278 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:20:48.807 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:20:48.807 INFO:teuthology.orchestra.run.smithi156.stdout: "size": 20, 2023-10-09T06:20:48.807 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 600, 2023-10-09T06:20:48.807 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [ 2023-10-09T06:20:48.807 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.807 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:95146 10.d 10:bfa5e452:::benchmark_data_smithi064_573750_object95145:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-10-09T06:20:48.807 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:20.510076+0000", 2023-10-09T06:20:48.807 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 448.29389454800003, 2023-10-09T06:20:48.808 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3209230409999999, 2023-10-09T06:20:48.808 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:48.808 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:20:48.808 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:48.808 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:20:48.808 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:48.808 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 95146 2023-10-09T06:20:48.808 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.808 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:48.808 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.808 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:48.809 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.510076+0000", 2023-10-09T06:20:48.809 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.809 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.809 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.809 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:48.809 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.510076+0000", 2023-10-09T06:20:48.809 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.809 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.809 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.809 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:48.809 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.510075+0000", 2023-10-09T06:20:48.810 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:48.810 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.810 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.810 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:48.810 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.510114+0000", 2023-10-09T06:20:48.810 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.8423000000000001e-05 2023-10-09T06:20:48.810 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.810 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.810 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:48.810 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.510121+0000", 2023-10-09T06:20:48.810 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6890000000000004e-06 2023-10-09T06:20:48.811 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.811 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.811 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:48.811 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.510126+0000", 2023-10-09T06:20:48.811 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.707e-06 2023-10-09T06:20:48.811 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.811 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.811 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.811 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.511105+0000", 2023-10-09T06:20:48.811 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00097909999999999989 2023-10-09T06:20:48.812 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.812 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.812 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:48.812 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.511225+0000", 2023-10-09T06:20:48.812 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011972999999999999 2023-10-09T06:20:48.812 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.812 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.812 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:20:48.812 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.511369+0000", 2023-10-09T06:20:48.812 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000143654 2023-10-09T06:20:48.812 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.812 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.813 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:48.813 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.513770+0000", 2023-10-09T06:20:48.813 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.002400994 2023-10-09T06:20:48.813 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.813 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.813 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.813 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.519773+0000", 2023-10-09T06:20:48.813 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0060033789999999997 2023-10-09T06:20:48.813 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.813 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.813 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.813 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.711202+0000", 2023-10-09T06:20:48.814 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.191429196 2023-10-09T06:20:48.814 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.814 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.814 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.814 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.711214+0000", 2023-10-09T06:20:48.814 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1576999999999999e-05 2023-10-09T06:20:48.814 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.814 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.814 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.814 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.706110+0000", 2023-10-09T06:20:48.814 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99489621699999997 2023-10-09T06:20:48.815 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.815 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.815 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.815 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.706122+0000", 2023-10-09T06:20:48.815 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.205e-05 2023-10-09T06:20:48.815 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.815 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.815 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.815 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.830928+0000", 2023-10-09T06:20:48.815 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1248056710000001 2023-10-09T06:20:48.815 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.815 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.816 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:48.816 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.830999+0000", 2023-10-09T06:20:48.816 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1289000000000001e-05 2023-10-09T06:20:48.816 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.816 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:48.816 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.816 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.816 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.816 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:95160 10.d 10:b709447e:::benchmark_data_smithi064_573750_object95159:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-10-09T06:20:48.816 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:20.519974+0000", 2023-10-09T06:20:48.816 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 448.28399660899998, 2023-10-09T06:20:48.817 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3111124510000001, 2023-10-09T06:20:48.817 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:48.817 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:20:48.817 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:48.817 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:20:48.817 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:48.817 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 95160 2023-10-09T06:20:48.817 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.817 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:48.817 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.817 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:48.817 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.519974+0000", 2023-10-09T06:20:48.818 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.818 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.818 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.818 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:48.818 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.519974+0000", 2023-10-09T06:20:48.818 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.818 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.818 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.818 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:48.818 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.519973+0000", 2023-10-09T06:20:48.818 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:48.819 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.819 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.819 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:48.819 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.520009+0000", 2023-10-09T06:20:48.819 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5522999999999999e-05 2023-10-09T06:20:48.819 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.819 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.819 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:48.819 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.520016+0000", 2023-10-09T06:20:48.819 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.17e-06 2023-10-09T06:20:48.819 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.819 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.820 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:48.820 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.520033+0000", 2023-10-09T06:20:48.820 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7465e-05 2023-10-09T06:20:48.820 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.820 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.820 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.820 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.520076+0000", 2023-10-09T06:20:48.820 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2305000000000002e-05 2023-10-09T06:20:48.820 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.820 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.820 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:48.821 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.520170+0000", 2023-10-09T06:20:48.821 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.4647000000000004e-05 2023-10-09T06:20:48.821 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.821 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.821 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:20:48.821 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.520272+0000", 2023-10-09T06:20:48.821 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010146500000000001 2023-10-09T06:20:48.821 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.821 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.821 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:48.821 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.523122+0000", 2023-10-09T06:20:48.821 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0028502169999999999 2023-10-09T06:20:48.822 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.822 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.822 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.822 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.526664+0000", 2023-10-09T06:20:48.822 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.003541485 2023-10-09T06:20:48.822 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.822 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.822 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.822 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.711273+0000", 2023-10-09T06:20:48.822 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.184609416 2023-10-09T06:20:48.822 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.823 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.823 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.823 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.711282+0000", 2023-10-09T06:20:48.823 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0680000000000003e-06 2023-10-09T06:20:48.823 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.823 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.823 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.823 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.706164+0000", 2023-10-09T06:20:48.823 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99488170600000003 2023-10-09T06:20:48.823 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.823 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.823 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.824 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.706171+0000", 2023-10-09T06:20:48.824 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0779999999999998e-06 2023-10-09T06:20:48.824 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.824 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.824 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.824 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.831040+0000", 2023-10-09T06:20:48.824 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.12486949 2023-10-09T06:20:48.824 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.824 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.824 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:48.824 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.831086+0000", 2023-10-09T06:20:48.825 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5976999999999997e-05 2023-10-09T06:20:48.825 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.825 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:48.825 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.825 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.825 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.825 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:95167 10.d 10:bd2a0a0e:::benchmark_data_smithi064_573750_object95166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-10-09T06:20:48.825 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:20.530287+0000", 2023-10-09T06:20:48.825 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 448.27368310399999, 2023-10-09T06:20:48.825 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.3008641750000001, 2023-10-09T06:20:48.825 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:48.825 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:20:48.826 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:48.826 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:20:48.826 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:48.826 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 95167 2023-10-09T06:20:48.826 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.826 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:48.826 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.826 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:48.826 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.530287+0000", 2023-10-09T06:20:48.826 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.826 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.827 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.827 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:48.827 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.530287+0000", 2023-10-09T06:20:48.827 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.827 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.827 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.827 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:48.827 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.530286+0000", 2023-10-09T06:20:48.828 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:48.828 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.828 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.828 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:48.828 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.530311+0000", 2023-10-09T06:20:48.828 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4980000000000001e-05 2023-10-09T06:20:48.828 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.828 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.828 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:48.828 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.530328+0000", 2023-10-09T06:20:48.829 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7280000000000001e-05 2023-10-09T06:20:48.829 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.829 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.829 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:48.829 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.530336+0000", 2023-10-09T06:20:48.829 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4490000000000002e-06 2023-10-09T06:20:48.829 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.829 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.829 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.829 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.530388+0000", 2023-10-09T06:20:48.829 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2438000000000002e-05 2023-10-09T06:20:48.830 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.830 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.830 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:48.830 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.530567+0000", 2023-10-09T06:20:48.830 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00017908900000000001 2023-10-09T06:20:48.830 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.830 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.830 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:20:48.830 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.530723+0000", 2023-10-09T06:20:48.830 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000155833 2023-10-09T06:20:48.830 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.831 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.831 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.831 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.533279+0000", 2023-10-09T06:20:48.831 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0025553939999999999 2023-10-09T06:20:48.831 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.831 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.831 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:48.831 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.536005+0000", 2023-10-09T06:20:48.831 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0027268000000000001 2023-10-09T06:20:48.831 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.831 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.832 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.832 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.711335+0000", 2023-10-09T06:20:48.832 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.175330085 2023-10-09T06:20:48.832 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.832 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.832 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.832 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.711343+0000", 2023-10-09T06:20:48.832 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3679999999999996e-06 2023-10-09T06:20:48.832 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.832 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.832 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.833 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.706203+0000", 2023-10-09T06:20:48.833 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99486017599999998 2023-10-09T06:20:48.833 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.833 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.833 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.833 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.706210+0000", 2023-10-09T06:20:48.833 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8709999999999999e-06 2023-10-09T06:20:48.833 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.833 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.833 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.833 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.831108+0000", 2023-10-09T06:20:48.834 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.124898585 2023-10-09T06:20:48.834 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.834 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.834 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:48.834 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.831152+0000", 2023-10-09T06:20:48.834 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.3028000000000002e-05 2023-10-09T06:20:48.834 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.834 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:48.834 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.834 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.834 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.835 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:95169 10.d 10:b585993a:::benchmark_data_smithi064_573750_object95168:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-10-09T06:20:48.835 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:20.533138+0000", 2023-10-09T06:20:48.835 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 448.27083259, 2023-10-09T06:20:48.835 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2980775480000002, 2023-10-09T06:20:48.835 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:48.835 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "reached pg", 2023-10-09T06:20:48.835 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:48.835 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:20:48.835 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:48.835 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 95169 2023-10-09T06:20:48.835 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.836 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:48.836 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.836 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:48.836 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.533138+0000", 2023-10-09T06:20:48.836 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.836 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.836 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.836 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:48.836 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.533138+0000", 2023-10-09T06:20:48.836 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.836 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.836 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.837 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:48.837 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.533137+0000", 2023-10-09T06:20:48.837 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:48.837 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.837 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.837 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:48.837 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.533186+0000", 2023-10-09T06:20:48.837 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8749999999999999e-05 2023-10-09T06:20:48.837 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.837 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.837 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:48.838 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.533194+0000", 2023-10-09T06:20:48.838 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4880000000000001e-06 2023-10-09T06:20:48.838 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.838 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.838 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:48.838 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.533198+0000", 2023-10-09T06:20:48.838 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.1860000000000002e-06 2023-10-09T06:20:48.838 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.838 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.838 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.838 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.533299+0000", 2023-10-09T06:20:48.839 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000101405 2023-10-09T06:20:48.839 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.839 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.839 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:48.839 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.533405+0000", 2023-10-09T06:20:48.839 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000105908 2023-10-09T06:20:48.839 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.839 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.839 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:20:48.839 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.533538+0000", 2023-10-09T06:20:48.839 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013334600000000001 2023-10-09T06:20:48.840 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.840 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.840 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:48.840 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.536181+0000", 2023-10-09T06:20:48.840 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0026423530000000001 2023-10-09T06:20:48.840 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.840 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.840 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.840 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.536314+0000", 2023-10-09T06:20:48.840 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013354 2023-10-09T06:20:48.840 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.840 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.841 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.841 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.711380+0000", 2023-10-09T06:20:48.841 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.17506608100000001 2023-10-09T06:20:48.841 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.841 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.841 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.841 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.711388+0000", 2023-10-09T06:20:48.841 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6890000000000004e-06 2023-10-09T06:20:48.841 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.841 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.841 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.842 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.706239+0000", 2023-10-09T06:20:48.842 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.99485103799999997 2023-10-09T06:20:48.842 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.842 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.842 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.842 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.706246+0000", 2023-10-09T06:20:48.842 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6959999999999999e-06 2023-10-09T06:20:48.842 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.842 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.842 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.842 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.831173+0000", 2023-10-09T06:20:48.843 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.124926898 2023-10-09T06:20:48.843 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.843 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.843 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:48.843 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.831215+0000", 2023-10-09T06:20:48.843 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.2629000000000001e-05 2023-10-09T06:20:48.843 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.843 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:48.843 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.843 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.843 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.844 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:95177 10.d 10:bcc2ceed:::benchmark_data_smithi064_573750_object95176:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e521)", 2023-10-09T06:20:48.844 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:20.795717+0000", 2023-10-09T06:20:48.844 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 448.00825393899999, 2023-10-09T06:20:48.844 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.050342831, 2023-10-09T06:20:48.844 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:48.844 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:48.844 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:48.844 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:20:48.844 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:48.844 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 95177 2023-10-09T06:20:48.844 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.844 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:48.845 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.845 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:48.845 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.795717+0000", 2023-10-09T06:20:48.845 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.845 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.845 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.845 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:48.845 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.795717+0000", 2023-10-09T06:20:48.845 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.845 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.845 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.846 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:48.846 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.795715+0000", 2023-10-09T06:20:48.846 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:48.846 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.846 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.846 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:48.846 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.795769+0000", 2023-10-09T06:20:48.846 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3454e-05 2023-10-09T06:20:48.846 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.846 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.846 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:48.847 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.795778+0000", 2023-10-09T06:20:48.847 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.5650000000000007e-06 2023-10-09T06:20:48.847 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.847 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.847 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:48.847 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.795785+0000", 2023-10-09T06:20:48.847 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3380000000000001e-06 2023-10-09T06:20:48.847 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.847 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.847 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.847 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.795855+0000", 2023-10-09T06:20:48.848 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0159000000000004e-05 2023-10-09T06:20:48.848 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.848 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.848 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.848 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.795862+0000", 2023-10-09T06:20:48.848 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.1409999999999999e-06 2023-10-09T06:20:48.848 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.848 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.848 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.848 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.706269+0000", 2023-10-09T06:20:48.848 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.91040671500000003 2023-10-09T06:20:48.849 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.849 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.849 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.849 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.706275+0000", 2023-10-09T06:20:48.849 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6220000000000003e-06 2023-10-09T06:20:48.849 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.849 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.849 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.849 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.831232+0000", 2023-10-09T06:20:48.849 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1249570550000001 2023-10-09T06:20:48.849 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.849 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.850 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:48.850 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.831347+0000", 2023-10-09T06:20:48.850 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011471799999999999 2023-10-09T06:20:48.850 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.850 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.850 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T06:20:48.850 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.831506+0000", 2023-10-09T06:20:48.850 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015921500000000001 2023-10-09T06:20:48.851 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.851 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.851 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.851 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.837687+0000", 2023-10-09T06:20:48.851 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0061805480000000001 2023-10-09T06:20:48.851 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.851 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.851 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.851 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.844625+0000", 2023-10-09T06:20:48.851 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0069379949999999998 2023-10-09T06:20:48.851 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.852 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.852 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:48.852 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.844653+0000", 2023-10-09T06:20:48.852 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.8000999999999998e-05 2023-10-09T06:20:48.852 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.852 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.852 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.852 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.845869+0000", 2023-10-09T06:20:48.852 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.001216808 2023-10-09T06:20:48.852 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.852 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.853 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:48.853 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.845925+0000", 2023-10-09T06:20:48.853 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5971999999999999e-05 2023-10-09T06:20:48.853 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.853 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.853 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:48.853 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.846059+0000", 2023-10-09T06:20:48.853 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013388500000000001 2023-10-09T06:20:48.853 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.853 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:48.853 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.854 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.854 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.854 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:95183 10.d 10:baf23edc:::benchmark_data_smithi064_573750_object95182:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e521)", 2023-10-09T06:20:48.854 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:20.803711+0000", 2023-10-09T06:20:48.854 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 448.00025927500002, 2023-10-09T06:20:48.854 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0426046769999999, 2023-10-09T06:20:48.854 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:48.854 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:48.854 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:48.854 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:20:48.854 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:48.854 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 95183 2023-10-09T06:20:48.855 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.855 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:48.855 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.855 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:48.855 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.803711+0000", 2023-10-09T06:20:48.855 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.855 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.855 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.855 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:48.855 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.803711+0000", 2023-10-09T06:20:48.855 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.855 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.857 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.857 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:48.857 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.803710+0000", 2023-10-09T06:20:48.857 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:48.857 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.857 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.857 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:48.857 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.803818+0000", 2023-10-09T06:20:48.857 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000108274 2023-10-09T06:20:48.857 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.857 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.858 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:48.858 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.803836+0000", 2023-10-09T06:20:48.858 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.8085999999999999e-05 2023-10-09T06:20:48.858 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.858 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.858 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:48.858 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.803842+0000", 2023-10-09T06:20:48.858 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3759999999999999e-06 2023-10-09T06:20:48.858 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.858 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.858 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.859 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.804153+0000", 2023-10-09T06:20:48.859 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00031180800000000001 2023-10-09T06:20:48.859 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.859 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.859 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.859 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.804163+0000", 2023-10-09T06:20:48.859 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.7079999999999992e-06 2023-10-09T06:20:48.859 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.859 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.859 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.859 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.706371+0000", 2023-10-09T06:20:48.860 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.90220791199999995 2023-10-09T06:20:48.860 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.860 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.860 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.860 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.706378+0000", 2023-10-09T06:20:48.860 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4399999999999999e-06 2023-10-09T06:20:48.860 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.860 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.860 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.860 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.834223+0000", 2023-10-09T06:20:48.860 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.12784415 2023-10-09T06:20:48.860 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.861 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.861 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:48.861 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.834343+0000", 2023-10-09T06:20:48.861 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000119918 2023-10-09T06:20:48.861 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.861 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.861 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T06:20:48.861 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.834495+0000", 2023-10-09T06:20:48.861 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015243099999999999 2023-10-09T06:20:48.861 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.861 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.862 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:48.862 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.844659+0000", 2023-10-09T06:20:48.862 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010163693 2023-10-09T06:20:48.862 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.862 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.862 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.862 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.845481+0000", 2023-10-09T06:20:48.862 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00082213700000000002 2023-10-09T06:20:48.862 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.862 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.862 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.863 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.846101+0000", 2023-10-09T06:20:48.863 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00062066300000000003 2023-10-09T06:20:48.863 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.863 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.863 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.863 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.846167+0000", 2023-10-09T06:20:48.863 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5611000000000006e-05 2023-10-09T06:20:48.863 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.863 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.863 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:48.863 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.846211+0000", 2023-10-09T06:20:48.863 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4091000000000002e-05 2023-10-09T06:20:48.864 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.864 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.864 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:48.864 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.846316+0000", 2023-10-09T06:20:48.864 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000104678 2023-10-09T06:20:48.864 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.864 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:48.864 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.864 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.864 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.864 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:95193 10.e 10:7b249de2:::benchmark_data_smithi064_573750_object95192:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e521)", 2023-10-09T06:20:48.865 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:20.813524+0000", 2023-10-09T06:20:48.865 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 447.99044675099998, 2023-10-09T06:20:48.865 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0199568299999999, 2023-10-09T06:20:48.865 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:48.865 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:48.865 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:48.865 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:20:48.865 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:48.865 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 95193 2023-10-09T06:20:48.865 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.865 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:48.865 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.866 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:48.866 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.813524+0000", 2023-10-09T06:20:48.866 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.866 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.866 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.866 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:48.866 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.813524+0000", 2023-10-09T06:20:48.866 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.866 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.866 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.866 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:48.867 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.813523+0000", 2023-10-09T06:20:48.867 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:48.867 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.867 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.867 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:48.867 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.813586+0000", 2023-10-09T06:20:48.867 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2964000000000002e-05 2023-10-09T06:20:48.867 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.867 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.867 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:48.867 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.813594+0000", 2023-10-09T06:20:48.867 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.559e-06 2023-10-09T06:20:48.868 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.868 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.868 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:48.868 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.813598+0000", 2023-10-09T06:20:48.868 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.7870000000000001e-06 2023-10-09T06:20:48.868 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.868 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.868 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.868 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.813652+0000", 2023-10-09T06:20:48.868 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3785000000000001e-05 2023-10-09T06:20:48.868 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.869 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.869 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.869 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.813661+0000", 2023-10-09T06:20:48.869 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2900000000000008e-06 2023-10-09T06:20:48.869 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.869 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.869 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.869 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.711452+0000", 2023-10-09T06:20:48.869 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.89779104499999995 2023-10-09T06:20:48.869 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.869 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.869 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.870 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.711465+0000", 2023-10-09T06:20:48.870 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2836e-05 2023-10-09T06:20:48.870 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.870 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.870 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.870 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.819147+0000", 2023-10-09T06:20:48.870 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1076818529999999 2023-10-09T06:20:48.870 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.870 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.870 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:48.870 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.819276+0000", 2023-10-09T06:20:48.871 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012921800000000001 2023-10-09T06:20:48.871 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.871 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.871 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:20:48.871 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.819444+0000", 2023-10-09T06:20:48.871 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000167467 2023-10-09T06:20:48.871 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.871 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.871 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.871 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.822863+0000", 2023-10-09T06:20:48.871 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0034189469999999999 2023-10-09T06:20:48.872 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.872 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.872 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:48.872 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.832250+0000", 2023-10-09T06:20:48.872 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0093869650000000006 2023-10-09T06:20:48.872 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.872 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.872 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.872 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.833224+0000", 2023-10-09T06:20:48.872 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00097416699999999996 2023-10-09T06:20:48.872 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.872 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.873 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.873 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.833304+0000", 2023-10-09T06:20:48.873 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0150999999999999e-05 2023-10-09T06:20:48.873 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.873 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.873 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:48.873 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.833360+0000", 2023-10-09T06:20:48.873 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5630000000000001e-05 2023-10-09T06:20:48.873 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.873 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.873 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:48.874 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.833481+0000", 2023-10-09T06:20:48.874 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000120816 2023-10-09T06:20:48.874 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.874 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:48.874 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.874 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.874 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.874 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:95195 10.e 10:7171b345:::benchmark_data_smithi064_573750_object95194:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e521)", 2023-10-09T06:20:48.874 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:20.817270+0000", 2023-10-09T06:20:48.874 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 447.986700482, 2023-10-09T06:20:48.874 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0173483069999998, 2023-10-09T06:20:48.875 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:48.875 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:48.875 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:48.875 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:20:48.875 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:48.875 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 95195 2023-10-09T06:20:48.875 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.875 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:48.875 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.875 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:48.876 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.817270+0000", 2023-10-09T06:20:48.876 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.876 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.876 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.876 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:48.876 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.817270+0000", 2023-10-09T06:20:48.876 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.876 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.876 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.876 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:48.876 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.817269+0000", 2023-10-09T06:20:48.877 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:20:48.877 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.877 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.877 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:48.877 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.817331+0000", 2023-10-09T06:20:48.877 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2089999999999997e-05 2023-10-09T06:20:48.877 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.877 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.877 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:48.877 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.817345+0000", 2023-10-09T06:20:48.877 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3637e-05 2023-10-09T06:20:48.877 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.878 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.878 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:48.878 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.817351+0000", 2023-10-09T06:20:48.878 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3879999999999997e-06 2023-10-09T06:20:48.878 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.878 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.878 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.878 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.817417+0000", 2023-10-09T06:20:48.878 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.6390999999999995e-05 2023-10-09T06:20:48.878 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.878 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.879 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.879 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.817431+0000", 2023-10-09T06:20:48.879 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3669000000000001e-05 2023-10-09T06:20:48.879 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.879 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.879 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.879 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.711479+0000", 2023-10-09T06:20:48.879 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.89404781099999997 2023-10-09T06:20:48.879 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.879 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.879 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.879 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.711483+0000", 2023-10-09T06:20:48.880 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.0339999999999998e-06 2023-10-09T06:20:48.880 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.880 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.880 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.880 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.822898+0000", 2023-10-09T06:20:48.880 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1114156399999999 2023-10-09T06:20:48.880 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.880 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.880 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:48.880 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.823064+0000", 2023-10-09T06:20:48.880 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00016534300000000001 2023-10-09T06:20:48.881 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.881 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.881 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:20:48.881 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.823258+0000", 2023-10-09T06:20:48.881 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000193838 2023-10-09T06:20:48.881 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.881 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.881 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.881 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.829210+0000", 2023-10-09T06:20:48.881 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0059525369999999999 2023-10-09T06:20:48.881 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.881 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.882 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:48.882 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.832254+0000", 2023-10-09T06:20:48.882 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0030442149999999999 2023-10-09T06:20:48.882 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.882 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.882 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.882 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.833254+0000", 2023-10-09T06:20:48.882 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00099923599999999992 2023-10-09T06:20:48.882 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.882 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.882 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.883 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.834458+0000", 2023-10-09T06:20:48.883 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0012047939999999999 2023-10-09T06:20:48.883 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.883 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.883 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:48.883 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.834511+0000", 2023-10-09T06:20:48.883 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2994000000000001e-05 2023-10-09T06:20:48.883 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.883 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.883 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:48.883 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.834618+0000", 2023-10-09T06:20:48.883 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010686799999999999 2023-10-09T06:20:48.884 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.884 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:48.884 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.884 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.884 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.884 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:95207 10.e 10:72955c88:::benchmark_data_smithi064_573750_object95206:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e521)", 2023-10-09T06:20:48.884 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:13:20.937758+0000", 2023-10-09T06:20:48.884 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 447.86621220799998, 2023-10-09T06:20:48.884 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.90852591, 2023-10-09T06:20:48.884 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:48.884 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:48.885 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:48.885 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:20:48.885 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:48.885 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 95207 2023-10-09T06:20:48.885 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.885 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:48.885 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.885 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:48.885 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.937758+0000", 2023-10-09T06:20:48.885 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.885 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.885 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.886 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:48.886 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.937758+0000", 2023-10-09T06:20:48.886 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.886 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.886 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.886 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:48.886 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.937757+0000", 2023-10-09T06:20:48.886 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:48.886 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.886 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.886 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:48.887 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.937794+0000", 2023-10-09T06:20:48.887 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7237999999999998e-05 2023-10-09T06:20:48.887 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.887 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.887 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:48.887 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.937809+0000", 2023-10-09T06:20:48.887 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4823e-05 2023-10-09T06:20:48.887 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.887 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.887 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:48.887 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.937816+0000", 2023-10-09T06:20:48.887 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.759e-06 2023-10-09T06:20:48.888 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.888 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.888 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.888 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.937875+0000", 2023-10-09T06:20:48.888 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.9568999999999998e-05 2023-10-09T06:20:48.888 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.888 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.888 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.888 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:20.937886+0000", 2023-10-09T06:20:48.888 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0387999999999999e-05 2023-10-09T06:20:48.888 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.888 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.889 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.889 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.711493+0000", 2023-10-09T06:20:48.889 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.77360711900000001 2023-10-09T06:20:48.889 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.889 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.889 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.889 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:21.711496+0000", 2023-10-09T06:20:48.889 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.6210000000000001e-06 2023-10-09T06:20:48.889 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.889 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.889 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.889 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.825988+0000", 2023-10-09T06:20:48.890 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1144915289999999 2023-10-09T06:20:48.890 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.890 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.890 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:48.890 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.826139+0000", 2023-10-09T06:20:48.890 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015136500000000001 2023-10-09T06:20:48.890 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.890 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.890 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:20:48.890 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.826297+0000", 2023-10-09T06:20:48.890 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00015811199999999999 2023-10-09T06:20:48.890 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.891 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.891 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:48.891 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.832258+0000", 2023-10-09T06:20:48.891 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.005960641 2023-10-09T06:20:48.891 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.891 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.891 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.891 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.845122+0000", 2023-10-09T06:20:48.891 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.01286373 2023-10-09T06:20:48.891 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.891 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.891 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.892 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.845151+0000", 2023-10-09T06:20:48.892 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9362000000000001e-05 2023-10-09T06:20:48.892 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.892 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.892 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.892 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.846123+0000", 2023-10-09T06:20:48.892 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00097178499999999997 2023-10-09T06:20:48.892 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.892 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.892 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:48.892 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.846173+0000", 2023-10-09T06:20:48.893 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.0454000000000002e-05 2023-10-09T06:20:48.893 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.893 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.893 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:48.893 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:13:22.846284+0000", 2023-10-09T06:20:48.893 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000110895 2023-10-09T06:20:48.893 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.893 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:48.893 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.893 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.893 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.893 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:171686 10.e 10:7b905e5f:::benchmark_data_smithi064_573750_object75471:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:20:48.894 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:15.818949+0000", 2023-10-09T06:20:48.894 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 332.98502122399998, 2023-10-09T06:20:48.894 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1270911259999998, 2023-10-09T06:20:48.894 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:48.894 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:48.894 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:48.894 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:20:48.894 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:48.894 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 171686 2023-10-09T06:20:48.894 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.894 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:48.894 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.895 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:48.895 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.818949+0000", 2023-10-09T06:20:48.895 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.895 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.895 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.895 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:48.895 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.818949+0000", 2023-10-09T06:20:48.895 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.895 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.895 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.896 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:48.896 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.818947+0000", 2023-10-09T06:20:48.896 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999976 2023-10-09T06:20:48.896 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.896 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.896 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:48.896 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.818955+0000", 2023-10-09T06:20:48.896 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.191e-06 2023-10-09T06:20:48.896 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.896 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.896 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:48.896 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.818973+0000", 2023-10-09T06:20:48.897 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.7476999999999999e-05 2023-10-09T06:20:48.897 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.897 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.897 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:48.897 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.818979+0000", 2023-10-09T06:20:48.897 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5810000000000002e-06 2023-10-09T06:20:48.897 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.897 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.897 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.897 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.819044+0000", 2023-10-09T06:20:48.897 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4527000000000001e-05 2023-10-09T06:20:48.897 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.897 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.898 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.898 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.819053+0000", 2023-10-09T06:20:48.898 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.1670000000000005e-06 2023-10-09T06:20:48.898 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.898 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.898 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.898 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.807971+0000", 2023-10-09T06:20:48.898 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.98891850400000003 2023-10-09T06:20:48.898 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.898 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.898 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.899 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.807984+0000", 2023-10-09T06:20:48.899 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.3021e-05 2023-10-09T06:20:48.899 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.899 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.899 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.899 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.937604+0000", 2023-10-09T06:20:48.899 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1296198289999999 2023-10-09T06:20:48.899 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.899 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.899 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:48.899 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.937709+0000", 2023-10-09T06:20:48.900 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000104522 2023-10-09T06:20:48.900 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.900 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.900 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:20:48.900 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.937809+0000", 2023-10-09T06:20:48.900 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010076700000000001 2023-10-09T06:20:48.900 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.900 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.900 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.900 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.940904+0000", 2023-10-09T06:20:48.900 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0030950190000000001 2023-10-09T06:20:48.900 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.901 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.901 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:48.901 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.940912+0000", 2023-10-09T06:20:48.901 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.8650000000000001e-06 2023-10-09T06:20:48.901 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.901 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.901 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.901 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.940952+0000", 2023-10-09T06:20:48.901 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.9570999999999998e-05 2023-10-09T06:20:48.901 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.901 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.901 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.902 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.945895+0000", 2023-10-09T06:20:48.902 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0049432770000000003 2023-10-09T06:20:48.902 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.902 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.902 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:48.902 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.945945+0000", 2023-10-09T06:20:48.902 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9401999999999999e-05 2023-10-09T06:20:48.902 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.902 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.902 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:48.902 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946040+0000", 2023-10-09T06:20:48.903 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.5735000000000004e-05 2023-10-09T06:20:48.903 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.903 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:48.903 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.903 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.903 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.903 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:171688 10.e 10:735f8ea9:::benchmark_data_smithi064_573750_object75473:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:20:48.903 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:15.825750+0000", 2023-10-09T06:20:48.903 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 332.97822014600001, 2023-10-09T06:20:48.903 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1206339970000001, 2023-10-09T06:20:48.903 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:48.904 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:48.904 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:48.904 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:20:48.904 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:48.904 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 171688 2023-10-09T06:20:48.904 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.904 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:48.904 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.904 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:48.904 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825750+0000", 2023-10-09T06:20:48.904 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.904 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.904 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.905 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:48.905 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825750+0000", 2023-10-09T06:20:48.905 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.905 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.905 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.905 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:48.905 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825749+0000", 2023-10-09T06:20:48.905 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:20:48.905 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.905 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.905 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:48.906 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825755+0000", 2023-10-09T06:20:48.906 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.5539999999999997e-06 2023-10-09T06:20:48.906 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.906 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.906 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:48.906 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825767+0000", 2023-10-09T06:20:48.906 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2213e-05 2023-10-09T06:20:48.906 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.906 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.906 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:48.906 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825772+0000", 2023-10-09T06:20:48.906 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2499999999999997e-06 2023-10-09T06:20:48.906 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.907 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.907 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.907 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825826+0000", 2023-10-09T06:20:48.907 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3579999999999999e-05 2023-10-09T06:20:48.907 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.907 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.907 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.907 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825837+0000", 2023-10-09T06:20:48.907 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1008e-05 2023-10-09T06:20:48.907 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.907 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.908 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.908 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808012+0000", 2023-10-09T06:20:48.908 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.98217509800000002 2023-10-09T06:20:48.908 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.908 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.908 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.908 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808019+0000", 2023-10-09T06:20:48.908 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.8419999999999999e-06 2023-10-09T06:20:48.908 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.908 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.909 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.909 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.940977+0000", 2023-10-09T06:20:48.909 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1329584779999999 2023-10-09T06:20:48.909 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.909 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.909 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:48.909 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.941068+0000", 2023-10-09T06:20:48.909 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0706999999999997e-05 2023-10-09T06:20:48.909 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.909 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.909 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:20:48.909 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.941149+0000", 2023-10-09T06:20:48.910 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1558999999999996e-05 2023-10-09T06:20:48.910 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.910 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.910 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.910 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946099+0000", 2023-10-09T06:20:48.910 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0049497769999999998 2023-10-09T06:20:48.910 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.910 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.910 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.910 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946121+0000", 2023-10-09T06:20:48.910 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2038000000000001e-05 2023-10-09T06:20:48.911 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.911 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.911 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:48.911 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946127+0000", 2023-10-09T06:20:48.911 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1120000000000001e-06 2023-10-09T06:20:48.911 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.911 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.911 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.911 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946295+0000", 2023-10-09T06:20:48.911 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00016754399999999999 2023-10-09T06:20:48.911 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.911 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.912 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:48.912 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946327+0000", 2023-10-09T06:20:48.912 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.2239000000000001e-05 2023-10-09T06:20:48.912 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.912 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.912 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:48.912 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946384+0000", 2023-10-09T06:20:48.912 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.7123000000000002e-05 2023-10-09T06:20:48.912 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.912 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:48.912 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.912 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.913 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.913 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:171689 10.e 10:7e719361:::benchmark_data_smithi064_573750_object75474:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:20:48.913 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:15.825780+0000", 2023-10-09T06:20:48.913 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 332.97819028499998, 2023-10-09T06:20:48.913 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1234250769999998, 2023-10-09T06:20:48.913 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:48.913 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:48.913 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:48.913 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:20:48.913 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:48.913 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 171689 2023-10-09T06:20:48.914 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.914 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:48.914 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.914 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:48.914 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825780+0000", 2023-10-09T06:20:48.914 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.914 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.914 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.914 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:48.914 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825780+0000", 2023-10-09T06:20:48.914 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.915 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.915 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.915 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:48.915 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825780+0000", 2023-10-09T06:20:48.915 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:48.915 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.915 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.915 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:48.915 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825783+0000", 2023-10-09T06:20:48.915 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.8200000000000001e-06 2023-10-09T06:20:48.915 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.915 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.916 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:48.916 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825788+0000", 2023-10-09T06:20:48.916 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1989999999999997e-06 2023-10-09T06:20:48.916 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.916 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.916 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:48.916 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825790+0000", 2023-10-09T06:20:48.916 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.5799999999999999e-06 2023-10-09T06:20:48.916 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.916 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.916 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.916 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825860+0000", 2023-10-09T06:20:48.917 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9343000000000003e-05 2023-10-09T06:20:48.917 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.917 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.917 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.917 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.825867+0000", 2023-10-09T06:20:48.917 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.4259999999999997e-06 2023-10-09T06:20:48.917 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.917 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.917 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.917 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808061+0000", 2023-10-09T06:20:48.917 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.98219415399999999 2023-10-09T06:20:48.918 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.918 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.918 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.918 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808068+0000", 2023-10-09T06:20:48.918 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7649999999999996e-06 2023-10-09T06:20:48.918 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.918 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.918 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.918 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.943145+0000", 2023-10-09T06:20:48.918 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1350769110000001 2023-10-09T06:20:48.918 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.918 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.919 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:48.919 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.943234+0000", 2023-10-09T06:20:48.919 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.8616999999999995e-05 2023-10-09T06:20:48.919 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.919 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.919 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:20:48.919 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.943309+0000", 2023-10-09T06:20:48.919 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.5770000000000001e-05 2023-10-09T06:20:48.919 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.919 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.920 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:48.920 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946132+0000", 2023-10-09T06:20:48.920 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0028229879999999998 2023-10-09T06:20:48.920 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.920 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.920 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.920 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946158+0000", 2023-10-09T06:20:48.920 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.5457999999999999e-05 2023-10-09T06:20:48.920 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.920 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.920 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.920 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946263+0000", 2023-10-09T06:20:48.921 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000104824 2023-10-09T06:20:48.921 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.921 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.921 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.921 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.949092+0000", 2023-10-09T06:20:48.921 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0028290469999999999 2023-10-09T06:20:48.921 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.921 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.921 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:48.921 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.949126+0000", 2023-10-09T06:20:48.921 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.4226000000000001e-05 2023-10-09T06:20:48.922 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.922 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.922 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:48.922 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.949205+0000", 2023-10-09T06:20:48.922 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9340999999999998e-05 2023-10-09T06:20:48.922 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.922 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:48.922 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.922 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.922 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.922 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:171708 10.e 10:7c12aa3b:::benchmark_data_smithi064_573750_object75493:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:20:48.922 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:15.851084+0000", 2023-10-09T06:20:48.923 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 332.95288649700001, 2023-10-09T06:20:48.923 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.109213767, 2023-10-09T06:20:48.923 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:48.923 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:48.923 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:48.923 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:20:48.923 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:48.923 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 171708 2023-10-09T06:20:48.923 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.923 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:48.923 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.923 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:48.924 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.851084+0000", 2023-10-09T06:20:48.924 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.924 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.924 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.924 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:48.924 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.851084+0000", 2023-10-09T06:20:48.924 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.924 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.924 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.924 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:48.924 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.851082+0000", 2023-10-09T06:20:48.925 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:48.925 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.925 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.925 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:48.925 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.851089+0000", 2023-10-09T06:20:48.925 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.144e-06 2023-10-09T06:20:48.925 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.925 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.925 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:48.925 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.851099+0000", 2023-10-09T06:20:48.925 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0579e-05 2023-10-09T06:20:48.925 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.926 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.926 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:48.926 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.851106+0000", 2023-10-09T06:20:48.926 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.7309999999999996e-06 2023-10-09T06:20:48.926 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.926 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.926 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.926 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.851155+0000", 2023-10-09T06:20:48.926 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8820999999999999e-05 2023-10-09T06:20:48.926 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.926 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.926 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.927 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.851163+0000", 2023-10-09T06:20:48.927 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.1799999999999996e-06 2023-10-09T06:20:48.927 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.927 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.927 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.927 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808090+0000", 2023-10-09T06:20:48.927 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.95692664800000005 2023-10-09T06:20:48.927 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.927 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.927 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.927 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808096+0000", 2023-10-09T06:20:48.927 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5899999999999996e-06 2023-10-09T06:20:48.928 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.928 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.928 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.928 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946407+0000", 2023-10-09T06:20:48.928 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.138310487 2023-10-09T06:20:48.928 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.928 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.928 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:48.928 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946493+0000", 2023-10-09T06:20:48.928 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.6648000000000001e-05 2023-10-09T06:20:48.928 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.929 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.929 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:20:48.929 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.946574+0000", 2023-10-09T06:20:48.929 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0939000000000006e-05 2023-10-09T06:20:48.929 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.929 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.929 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.929 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.949254+0000", 2023-10-09T06:20:48.929 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0026798500000000001 2023-10-09T06:20:48.929 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.929 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.930 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.930 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.949278+0000", 2023-10-09T06:20:48.930 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4196e-05 2023-10-09T06:20:48.930 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.930 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.930 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:48.930 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.949284+0000", 2023-10-09T06:20:48.930 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.835e-06 2023-10-09T06:20:48.930 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.930 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.930 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.931 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.960139+0000", 2023-10-09T06:20:48.931 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010854508000000001 2023-10-09T06:20:48.931 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.931 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.931 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:48.931 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.960183+0000", 2023-10-09T06:20:48.931 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4882000000000002e-05 2023-10-09T06:20:48.931 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.931 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.931 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:48.931 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.960298+0000", 2023-10-09T06:20:48.931 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000114261 2023-10-09T06:20:48.932 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.932 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:48.932 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.932 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.932 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.932 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:171724 10.e 10:74fab5e2:::benchmark_data_smithi064_573750_object75509:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:20:48.932 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:15.873407+0000", 2023-10-09T06:20:48.932 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 332.93056350900002, 2023-10-09T06:20:48.932 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0927997860000001, 2023-10-09T06:20:48.932 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:48.932 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:48.932 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:48.933 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:20:48.933 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:48.933 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 171724 2023-10-09T06:20:48.933 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.933 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:48.933 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.933 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:48.933 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.873407+0000", 2023-10-09T06:20:48.933 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.933 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.933 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.933 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:48.934 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.873407+0000", 2023-10-09T06:20:48.934 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.934 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.934 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.934 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:48.934 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.873406+0000", 2023-10-09T06:20:48.934 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:48.934 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.934 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.934 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:48.934 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.873413+0000", 2023-10-09T06:20:48.935 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.418e-06 2023-10-09T06:20:48.935 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.935 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.935 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:48.935 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.873427+0000", 2023-10-09T06:20:48.935 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4362000000000001e-05 2023-10-09T06:20:48.935 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.935 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.935 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:48.935 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.873434+0000", 2023-10-09T06:20:48.935 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.9419999999999998e-06 2023-10-09T06:20:48.935 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.936 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.936 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.936 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.873487+0000", 2023-10-09T06:20:48.936 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2324000000000003e-05 2023-10-09T06:20:48.936 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.936 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.936 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.936 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.873493+0000", 2023-10-09T06:20:48.936 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3740000000000003e-06 2023-10-09T06:20:48.936 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.936 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.936 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.936 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808117+0000", 2023-10-09T06:20:48.937 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.93462358599999995 2023-10-09T06:20:48.937 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.937 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.937 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.937 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808123+0000", 2023-10-09T06:20:48.937 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1809999999999998e-06 2023-10-09T06:20:48.937 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.937 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.937 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.937 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.949312+0000", 2023-10-09T06:20:48.937 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.141189486 2023-10-09T06:20:48.937 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.938 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.938 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:48.938 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.949425+0000", 2023-10-09T06:20:48.938 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000112353 2023-10-09T06:20:48.938 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.938 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.938 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:20:48.938 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.949554+0000", 2023-10-09T06:20:48.938 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012921800000000001 2023-10-09T06:20:48.938 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.938 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.938 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.939 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.960103+0000", 2023-10-09T06:20:48.939 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010548963 2023-10-09T06:20:48.939 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.939 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.939 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.939 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.960335+0000", 2023-10-09T06:20:48.939 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000232432 2023-10-09T06:20:48.939 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.939 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.939 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.939 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.960365+0000", 2023-10-09T06:20:48.939 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.9901000000000001e-05 2023-10-09T06:20:48.940 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.940 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.940 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:48.940 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.966031+0000", 2023-10-09T06:20:48.940 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0056662750000000001 2023-10-09T06:20:48.940 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.940 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.940 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:48.940 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.966084+0000", 2023-10-09T06:20:48.940 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.2261e-05 2023-10-09T06:20:48.940 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.941 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.941 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:48.941 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.966207+0000", 2023-10-09T06:20:48.941 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012310700000000001 2023-10-09T06:20:48.941 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.941 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:48.941 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.941 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.941 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.941 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:171731 10.e 10:73794b97:::benchmark_data_smithi064_573750_object75516:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:20:48.941 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:15.882331+0000", 2023-10-09T06:20:48.942 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 332.92163914499997, 2023-10-09T06:20:48.942 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.094724646, 2023-10-09T06:20:48.942 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:48.942 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:48.942 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:48.942 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:20:48.942 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:48.942 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 171731 2023-10-09T06:20:48.942 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.942 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:48.942 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.943 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:48.943 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.882331+0000", 2023-10-09T06:20:48.943 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.943 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.943 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.943 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:48.943 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.882331+0000", 2023-10-09T06:20:48.943 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.943 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.943 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.943 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:48.944 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.882330+0000", 2023-10-09T06:20:48.944 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:48.944 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.944 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.944 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:48.944 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.882340+0000", 2023-10-09T06:20:48.944 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0457e-05 2023-10-09T06:20:48.944 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.944 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.944 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:48.944 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.882363+0000", 2023-10-09T06:20:48.945 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.281e-05 2023-10-09T06:20:48.945 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.945 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.945 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:48.945 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.882371+0000", 2023-10-09T06:20:48.945 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.0679999999999997e-06 2023-10-09T06:20:48.945 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.945 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.945 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.945 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.882441+0000", 2023-10-09T06:20:48.945 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.0616000000000003e-05 2023-10-09T06:20:48.945 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.946 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.946 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.946 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.882454+0000", 2023-10-09T06:20:48.946 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2328e-05 2023-10-09T06:20:48.946 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.946 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.946 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.946 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808143+0000", 2023-10-09T06:20:48.946 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.92568909300000002 2023-10-09T06:20:48.946 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.946 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.947 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.947 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808149+0000", 2023-10-09T06:20:48.947 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3210000000000001e-06 2023-10-09T06:20:48.947 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.947 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.947 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.947 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.952065+0000", 2023-10-09T06:20:48.947 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1439156129999999 2023-10-09T06:20:48.947 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.947 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.947 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:48.948 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.952142+0000", 2023-10-09T06:20:48.948 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7379000000000006e-05 2023-10-09T06:20:48.948 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.948 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.948 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:20:48.948 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.952248+0000", 2023-10-09T06:20:48.948 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00010558800000000001 2023-10-09T06:20:48.948 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.948 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.948 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.948 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.960396+0000", 2023-10-09T06:20:48.948 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0081483519999999993 2023-10-09T06:20:48.949 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.949 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.949 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.949 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.960429+0000", 2023-10-09T06:20:48.949 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.2342999999999998e-05 2023-10-09T06:20:48.949 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.949 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.949 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.949 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.966264+0000", 2023-10-09T06:20:48.949 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0058356329999999998 2023-10-09T06:20:48.949 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.950 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.950 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:48.950 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.976865+0000", 2023-10-09T06:20:48.950 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010601169000000001 2023-10-09T06:20:48.950 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.950 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.950 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:48.950 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.976922+0000", 2023-10-09T06:20:48.950 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.6378999999999997e-05 2023-10-09T06:20:48.950 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.950 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.951 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:48.951 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977056+0000", 2023-10-09T06:20:48.951 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013425500000000001 2023-10-09T06:20:48.951 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.951 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:48.951 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.951 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.951 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.951 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:171732 10.e 10:72905dd9:::benchmark_data_smithi064_573750_object75517:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:20:48.951 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:15.884530+0000", 2023-10-09T06:20:48.951 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 332.91944092099999, 2023-10-09T06:20:48.952 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.092684894, 2023-10-09T06:20:48.952 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:48.952 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:48.952 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:48.952 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:20:48.952 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:48.952 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 171732 2023-10-09T06:20:48.952 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.952 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:48.952 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.952 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:48.952 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.884530+0000", 2023-10-09T06:20:48.953 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.953 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.953 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.953 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:48.953 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.884530+0000", 2023-10-09T06:20:48.953 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.953 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.953 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.953 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:48.953 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.884529+0000", 2023-10-09T06:20:48.953 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:20:48.954 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.954 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.954 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:48.954 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.884538+0000", 2023-10-09T06:20:48.954 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.4760000000000004e-06 2023-10-09T06:20:48.954 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.954 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.954 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:48.954 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.884553+0000", 2023-10-09T06:20:48.954 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.4623999999999999e-05 2023-10-09T06:20:48.954 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.954 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.955 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:48.955 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.884561+0000", 2023-10-09T06:20:48.955 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.2749999999999995e-06 2023-10-09T06:20:48.955 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.955 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.955 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.955 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.884625+0000", 2023-10-09T06:20:48.955 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.4389999999999998e-05 2023-10-09T06:20:48.955 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.955 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.955 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.956 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.884634+0000", 2023-10-09T06:20:48.956 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.0960000000000007e-06 2023-10-09T06:20:48.956 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.956 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.956 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.956 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808169+0000", 2023-10-09T06:20:48.956 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.92353482799999997 2023-10-09T06:20:48.956 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.956 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.956 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.956 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808176+0000", 2023-10-09T06:20:48.957 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.3280000000000002e-06 2023-10-09T06:20:48.957 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.957 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.957 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.957 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.954504+0000", 2023-10-09T06:20:48.957 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.146328733 2023-10-09T06:20:48.957 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.957 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.957 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:48.957 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.954581+0000", 2023-10-09T06:20:48.957 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6789e-05 2023-10-09T06:20:48.957 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.958 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.958 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:20:48.958 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.954659+0000", 2023-10-09T06:20:48.958 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.7904000000000005e-05 2023-10-09T06:20:48.958 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.958 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.958 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.958 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.966305+0000", 2023-10-09T06:20:48.958 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.011645983 2023-10-09T06:20:48.958 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.959 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.959 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.959 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.966337+0000", 2023-10-09T06:20:48.959 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1948e-05 2023-10-09T06:20:48.959 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.959 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.959 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.959 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.966370+0000", 2023-10-09T06:20:48.959 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.3121999999999999e-05 2023-10-09T06:20:48.959 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.959 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.960 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:48.960 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977071+0000", 2023-10-09T06:20:48.960 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010701207000000001 2023-10-09T06:20:48.960 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.960 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.960 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:48.960 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977122+0000", 2023-10-09T06:20:48.960 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.1038000000000002e-05 2023-10-09T06:20:48.960 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.960 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.960 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:48.960 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977214+0000", 2023-10-09T06:20:48.961 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.2127999999999996e-05 2023-10-09T06:20:48.961 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.961 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:48.961 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.961 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.961 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.961 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:171742 10.e 10:72249b2b:::benchmark_data_smithi064_573750_object75527:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:20:48.961 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:15.902945+0000", 2023-10-09T06:20:48.961 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 332.90102495399998, 2023-10-09T06:20:48.961 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.074354617, 2023-10-09T06:20:48.961 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:48.961 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:48.962 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:48.962 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:20:48.962 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:48.962 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 171742 2023-10-09T06:20:48.962 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.962 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:48.962 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.962 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:48.962 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.902945+0000", 2023-10-09T06:20:48.962 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.962 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.963 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.963 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:48.963 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.902945+0000", 2023-10-09T06:20:48.963 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.963 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.963 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.963 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:48.963 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.902944+0000", 2023-10-09T06:20:48.963 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:20:48.963 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.963 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.963 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:48.963 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.902950+0000", 2023-10-09T06:20:48.964 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.3120000000000001e-06 2023-10-09T06:20:48.964 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.964 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.964 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:48.964 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.902962+0000", 2023-10-09T06:20:48.964 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.2581e-05 2023-10-09T06:20:48.964 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.964 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.964 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:48.964 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.902967+0000", 2023-10-09T06:20:48.964 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.9620000000000001e-06 2023-10-09T06:20:48.964 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.965 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.965 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.965 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.903030+0000", 2023-10-09T06:20:48.965 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2533999999999995e-05 2023-10-09T06:20:48.965 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.965 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.965 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.965 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.903039+0000", 2023-10-09T06:20:48.965 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.7299999999999994e-06 2023-10-09T06:20:48.965 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.965 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.966 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.966 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808196+0000", 2023-10-09T06:20:48.966 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.90515772699999997 2023-10-09T06:20:48.966 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.966 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.966 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.966 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808203+0000", 2023-10-09T06:20:48.966 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2380000000000002e-06 2023-10-09T06:20:48.966 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.966 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.966 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.966 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.957318+0000", 2023-10-09T06:20:48.967 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.149115659 2023-10-09T06:20:48.967 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.967 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.967 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:48.967 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.957450+0000", 2023-10-09T06:20:48.967 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00013214699999999999 2023-10-09T06:20:48.967 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.967 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.967 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:20:48.967 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.957605+0000", 2023-10-09T06:20:48.967 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000154578 2023-10-09T06:20:48.968 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.968 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.968 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.968 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.966399+0000", 2023-10-09T06:20:48.968 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0087941670000000003 2023-10-09T06:20:48.968 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.968 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.968 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.968 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.966430+0000", 2023-10-09T06:20:48.968 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.0504000000000001e-05 2023-10-09T06:20:48.968 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.969 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.969 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.969 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.966462+0000", 2023-10-09T06:20:48.969 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.1952999999999997e-05 2023-10-09T06:20:48.969 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.969 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.969 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:48.969 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977221+0000", 2023-10-09T06:20:48.969 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.010759866 2023-10-09T06:20:48.969 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.969 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.970 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:48.970 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977246+0000", 2023-10-09T06:20:48.970 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.4289000000000001e-05 2023-10-09T06:20:48.970 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.970 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.970 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:48.970 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977300+0000", 2023-10-09T06:20:48.970 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 5.4431000000000001e-05 2023-10-09T06:20:48.970 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.970 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:48.971 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.971 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.971 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.971 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:171744 10.e 10:7d1e64fd:::benchmark_data_smithi064_573750_object75529:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:20:48.971 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:15.906916+0000", 2023-10-09T06:20:48.971 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 332.89705462199998, 2023-10-09T06:20:48.971 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.0709302470000002, 2023-10-09T06:20:48.971 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:48.971 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:48.971 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:48.971 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:20:48.972 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:48.972 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 171744 2023-10-09T06:20:48.972 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.972 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:48.972 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.972 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:48.972 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.906916+0000", 2023-10-09T06:20:48.972 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.972 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.972 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.972 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:48.973 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.906916+0000", 2023-10-09T06:20:48.973 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.973 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.973 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.973 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:48.973 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.906915+0000", 2023-10-09T06:20:48.973 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.999999 2023-10-09T06:20:48.973 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.973 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.973 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:48.973 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.906922+0000", 2023-10-09T06:20:48.974 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.737e-06 2023-10-09T06:20:48.974 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.974 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.974 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:48.974 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.906932+0000", 2023-10-09T06:20:48.974 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.8630000000000003e-06 2023-10-09T06:20:48.974 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.974 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.974 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:48.974 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.906936+0000", 2023-10-09T06:20:48.974 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.245e-06 2023-10-09T06:20:48.975 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.975 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.975 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.975 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.907001+0000", 2023-10-09T06:20:48.975 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.5140000000000003e-05 2023-10-09T06:20:48.975 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.975 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.975 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.975 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.907009+0000", 2023-10-09T06:20:48.975 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.6510000000000001e-06 2023-10-09T06:20:48.975 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.976 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.976 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.976 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808223+0000", 2023-10-09T06:20:48.976 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.90121384900000001 2023-10-09T06:20:48.976 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.976 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.976 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.976 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808229+0000", 2023-10-09T06:20:48.976 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.0260000000000004e-06 2023-10-09T06:20:48.976 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.976 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.977 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.977 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.960451+0000", 2023-10-09T06:20:48.977 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1522222259999999 2023-10-09T06:20:48.977 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.977 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.977 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:48.977 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.960567+0000", 2023-10-09T06:20:48.977 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000116549 2023-10-09T06:20:48.977 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.977 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.977 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:20:48.978 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.960697+0000", 2023-10-09T06:20:48.978 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00012991599999999999 2023-10-09T06:20:48.978 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.978 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.978 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.978 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.976854+0000", 2023-10-09T06:20:48.978 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.016157082999999999 2023-10-09T06:20:48.978 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.978 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.978 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:48.978 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977304+0000", 2023-10-09T06:20:48.979 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00044994099999999998 2023-10-09T06:20:48.979 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.979 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.979 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.979 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977648+0000", 2023-10-09T06:20:48.979 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00034370499999999998 2023-10-09T06:20:48.979 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.979 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.979 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.979 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977681+0000", 2023-10-09T06:20:48.979 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.3488000000000002e-05 2023-10-09T06:20:48.979 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.980 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.980 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:48.980 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977730+0000", 2023-10-09T06:20:48.980 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.8832999999999998e-05 2023-10-09T06:20:48.980 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.980 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.980 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:48.980 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977846+0000", 2023-10-09T06:20:48.980 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011575500000000001 2023-10-09T06:20:48.980 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.980 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:48.980 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.981 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.981 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.981 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.46283.0:171757 10.e 10:72a212b1:::benchmark_data_smithi064_573750_object75542:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e558)", 2023-10-09T06:20:48.981 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:15:15.960756+0000", 2023-10-09T06:20:48.981 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 332.84321434700001, 2023-10-09T06:20:48.981 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.017502049, 2023-10-09T06:20:48.981 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:48.981 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:48.981 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:48.981 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.46283", 2023-10-09T06:20:48.981 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:20:48.982 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 171757 2023-10-09T06:20:48.982 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.982 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:48.982 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.982 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:48.982 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.960756+0000", 2023-10-09T06:20:48.982 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.982 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.982 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.982 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:48.982 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.960756+0000", 2023-10-09T06:20:48.983 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.983 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.983 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.983 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:48.983 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.960755+0000", 2023-10-09T06:20:48.983 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:48.983 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.983 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.983 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:48.983 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.960762+0000", 2023-10-09T06:20:48.983 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.3220000000000004e-06 2023-10-09T06:20:48.984 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.984 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.984 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:48.984 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.960778+0000", 2023-10-09T06:20:48.984 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.5835e-05 2023-10-09T06:20:48.984 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.984 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.984 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:48.984 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.960800+0000", 2023-10-09T06:20:48.984 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.2507e-05 2023-10-09T06:20:48.984 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.985 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.985 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.985 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.960845+0000", 2023-10-09T06:20:48.985 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.4177000000000001e-05 2023-10-09T06:20:48.985 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.985 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.985 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.985 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:15.960853+0000", 2023-10-09T06:20:48.985 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 7.9829999999999996e-06 2023-10-09T06:20:48.985 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.985 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.985 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.986 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808250+0000", 2023-10-09T06:20:48.986 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.847397921 2023-10-09T06:20:48.986 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.986 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.986 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for peered", 2023-10-09T06:20:48.986 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:16.808257+0000", 2023-10-09T06:20:48.986 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.2179999999999996e-06 2023-10-09T06:20:48.986 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.986 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.986 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.987 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.963449+0000", 2023-10-09T06:20:48.987 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.1551920849999999 2023-10-09T06:20:48.987 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.987 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.987 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:48.987 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.963546+0000", 2023-10-09T06:20:48.987 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 9.6785000000000003e-05 2023-10-09T06:20:48.987 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.987 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.987 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T06:20:48.987 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.963659+0000", 2023-10-09T06:20:48.987 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000113554 2023-10-09T06:20:48.988 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.988 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.988 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:48.988 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977308+0000", 2023-10-09T06:20:48.988 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.013648680999999999 2023-10-09T06:20:48.988 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.988 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.988 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.988 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.977474+0000", 2023-10-09T06:20:48.988 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000165973 2023-10-09T06:20:48.988 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.988 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.989 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.989 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.978068+0000", 2023-10-09T06:20:48.989 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000594708 2023-10-09T06:20:48.989 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.989 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.989 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.989 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.978106+0000", 2023-10-09T06:20:48.989 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.7033999999999998e-05 2023-10-09T06:20:48.989 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.989 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.989 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:48.989 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.978151+0000", 2023-10-09T06:20:48.990 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4.5046999999999997e-05 2023-10-09T06:20:48.990 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.990 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.990 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:48.990 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:15:17.978258+0000", 2023-10-09T06:20:48.990 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000107577 2023-10-09T06:20:48.990 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.990 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:48.990 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.990 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.990 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.991 INFO:teuthology.orchestra.run.smithi156.stdout: "description": "osd_op(client.36591.0:185638 11.d 11:be827db7:::benchmark_data_smithi064_631903_object76724:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e625)", 2023-10-09T06:20:48.991 INFO:teuthology.orchestra.run.smithi156.stdout: "initiated_at": "2023-10-09T06:19:19.289809+0000", 2023-10-09T06:20:48.991 INFO:teuthology.orchestra.run.smithi156.stdout: "age": 89.514161836, 2023-10-09T06:20:48.991 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.229769209, 2023-10-09T06:20:48.991 INFO:teuthology.orchestra.run.smithi156.stdout: "type_data": { 2023-10-09T06:20:48.991 INFO:teuthology.orchestra.run.smithi156.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:48.991 INFO:teuthology.orchestra.run.smithi156.stdout: "client_info": { 2023-10-09T06:20:48.991 INFO:teuthology.orchestra.run.smithi156.stdout: "client": "client.36591", 2023-10-09T06:20:48.991 INFO:teuthology.orchestra.run.smithi156.stdout: "client_addr": "172.21.15.64:0/2306923614", 2023-10-09T06:20:48.991 INFO:teuthology.orchestra.run.smithi156.stdout: "tid": 185638 2023-10-09T06:20:48.991 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.991 INFO:teuthology.orchestra.run.smithi156.stdout: "events": [ 2023-10-09T06:20:48.992 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.992 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "initiated", 2023-10-09T06:20:48.992 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.289809+0000", 2023-10-09T06:20:48.992 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.992 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.992 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.992 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "throttled", 2023-10-09T06:20:48.992 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.289809+0000", 2023-10-09T06:20:48.992 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0 2023-10-09T06:20:48.992 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.992 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.993 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "header_read", 2023-10-09T06:20:48.993 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.289806+0000", 2023-10-09T06:20:48.993 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:48.993 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.993 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.993 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "all_read", 2023-10-09T06:20:48.993 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.289817+0000", 2023-10-09T06:20:48.993 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 1.0349999999999999e-05 2023-10-09T06:20:48.993 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.993 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.993 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "dispatched", 2023-10-09T06:20:48.993 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.289839+0000", 2023-10-09T06:20:48.994 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 2.1736999999999999e-05 2023-10-09T06:20:48.994 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.994 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.994 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "queued_for_pg", 2023-10-09T06:20:48.994 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.289848+0000", 2023-10-09T06:20:48.994 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9309999999999997e-06 2023-10-09T06:20:48.994 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.994 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.994 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "reached_pg", 2023-10-09T06:20:48.994 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.289909+0000", 2023-10-09T06:20:48.994 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 6.1693999999999996e-05 2023-10-09T06:20:48.995 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.995 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.995 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "started", 2023-10-09T06:20:48.995 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.290046+0000", 2023-10-09T06:20:48.995 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.000137183 2023-10-09T06:20:48.995 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.995 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.995 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:20:48.995 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.290163+0000", 2023-10-09T06:20:48.995 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.00011616300000000001 2023-10-09T06:20:48.995 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.995 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.996 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.996 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.293126+0000", 2023-10-09T06:20:48.996 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.0029639269999999999 2023-10-09T06:20:48.996 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.996 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.996 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "op_commit", 2023-10-09T06:20:48.996 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.293216+0000", 2023-10-09T06:20:48.996 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 8.9729999999999996e-05 2023-10-09T06:20:48.996 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.996 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.996 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:48.996 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.519510+0000", 2023-10-09T06:20:48.997 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 0.226293889 2023-10-09T06:20:48.997 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.997 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.997 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "commit_sent", 2023-10-09T06:20:48.997 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.519542+0000", 2023-10-09T06:20:48.997 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.2060999999999997e-05 2023-10-09T06:20:48.997 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2023-10-09T06:20:48.997 INFO:teuthology.orchestra.run.smithi156.stdout: { 2023-10-09T06:20:48.997 INFO:teuthology.orchestra.run.smithi156.stdout: "event": "done", 2023-10-09T06:20:48.997 INFO:teuthology.orchestra.run.smithi156.stdout: "time": "2023-10-09T06:19:19.519578+0000", 2023-10-09T06:20:48.997 INFO:teuthology.orchestra.run.smithi156.stdout: "duration": 3.5667000000000003e-05 2023-10-09T06:20:48.997 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.998 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:48.998 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.998 INFO:teuthology.orchestra.run.smithi156.stdout: } 2023-10-09T06:20:48.998 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2023-10-09T06:20:48.998 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:20:48.998 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:48 smithi156 ceph-mon[195101]: pgmap v1678: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 256 KiB/s rd, 76 KiB/s wr, 69 op/s 2023-10-09T06:20:49.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:48 smithi064 ceph-mon[212750]: pgmap v1678: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 256 KiB/s rd, 76 KiB/s wr, 69 op/s 2023-10-09T06:20:49.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:48 smithi064 ceph-mon[220044]: pgmap v1678: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 256 KiB/s rd, 76 KiB/s wr, 69 op/s 2023-10-09T06:20:49.316 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd help export 2023-10-09T06:20:49.316 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep -q export-format 2023-10-09T06:20:49.336 INFO:tasks.workunit.client.0.smithi064.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_695299/img bs=1k count=10 seek=100 2023-10-09T06:20:49.337 INFO:tasks.workunit.client.0.smithi064.stderr:10+0 records in 2023-10-09T06:20:49.337 INFO:tasks.workunit.client.0.smithi064.stderr:10+0 records out 2023-10-09T06:20:49.337 INFO:tasks.workunit.client.0.smithi064.stderr:10240 bytes (10 kB, 10 KiB) copied, 7.194e-05 s, 142 MB/s 2023-10-09T06:20:49.337 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --new-format /tmp/rbd_import_export_695299/img testimg 2023-10-09T06:20:49.391 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 100% complete...done. 2023-10-09T06:20:49.391 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:20:49.394 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd snap create testimg@snap 2023-10-09T06:20:49.433 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:20:49.875 INFO:tasks.workunit.client.0.smithi064.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T06:20:49.882 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd image-meta set testimg key1 value1 2023-10-09T06:20:49.931 INFO:tasks.workunit.client.0.smithi064.stderr:++ rbd image-meta list testimg 2023-10-09T06:20:49.975 INFO:tasks.workunit.client.0.smithi064.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-10-09T06:20:49.975 INFO:tasks.workunit.client.0.smithi064.stderr: 2023-10-09T06:20:49.975 INFO:tasks.workunit.client.0.smithi064.stderr:Key Value 2023-10-09T06:20:49.975 INFO:tasks.workunit.client.0.smithi064.stderr:key1 value1' 2023-10-09T06:20:49.975 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_695299/img_v2 2023-10-09T06:20:50.019 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:50.019 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:20:50.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:50.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:50.023 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_695299/img_v2 testimg_import 2023-10-09T06:20:50.124 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:50.132 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:50.162 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:50.189 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:50.218 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:50.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:50.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:50.322 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:50.353 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:50.398 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:50.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:50.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:50.503 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:50.530 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:50.571 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:50.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:50.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:50.678 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:50.710 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:50.883 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:20:50.886 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd info testimg_import 2023-10-09T06:20:50.922 INFO:tasks.workunit.client.0.smithi064.stdout:rbd image 'testimg_import': 2023-10-09T06:20:50.923 INFO:tasks.workunit.client.0.smithi064.stdout: size 110 KiB in 1 objects 2023-10-09T06:20:50.923 INFO:tasks.workunit.client.0.smithi064.stdout: order 22 (4 MiB objects) 2023-10-09T06:20:50.923 INFO:tasks.workunit.client.0.smithi064.stdout: snapshot_count: 1 2023-10-09T06:20:50.923 INFO:tasks.workunit.client.0.smithi064.stdout: id: b80d87dc70ca 2023-10-09T06:20:50.923 INFO:tasks.workunit.client.0.smithi064.stdout: block_name_prefix: rbd_data.b80d87dc70ca 2023-10-09T06:20:50.923 INFO:tasks.workunit.client.0.smithi064.stdout: format: 2 2023-10-09T06:20:50.923 INFO:tasks.workunit.client.0.smithi064.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T06:20:50.923 INFO:tasks.workunit.client.0.smithi064.stdout: op_features: 2023-10-09T06:20:50.923 INFO:tasks.workunit.client.0.smithi064.stdout: flags: 2023-10-09T06:20:50.923 INFO:tasks.workunit.client.0.smithi064.stdout: create_timestamp: Mon Oct 9 06:20:50 2023 2023-10-09T06:20:50.924 INFO:tasks.workunit.client.0.smithi064.stdout: access_timestamp: Mon Oct 9 06:20:50 2023 2023-10-09T06:20:50.924 INFO:tasks.workunit.client.0.smithi064.stdout: modify_timestamp: Mon Oct 9 06:20:50 2023 2023-10-09T06:20:50.925 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd info testimg_import@snap 2023-10-09T06:20:50.959 INFO:tasks.workunit.client.0.smithi064.stdout:rbd image 'testimg_import': 2023-10-09T06:20:50.959 INFO:tasks.workunit.client.0.smithi064.stdout: size 110 KiB in 1 objects 2023-10-09T06:20:50.959 INFO:tasks.workunit.client.0.smithi064.stdout: order 22 (4 MiB objects) 2023-10-09T06:20:50.960 INFO:tasks.workunit.client.0.smithi064.stdout: snapshot_count: 1 2023-10-09T06:20:50.960 INFO:tasks.workunit.client.0.smithi064.stdout: id: b80d87dc70ca 2023-10-09T06:20:50.960 INFO:tasks.workunit.client.0.smithi064.stdout: block_name_prefix: rbd_data.b80d87dc70ca 2023-10-09T06:20:50.960 INFO:tasks.workunit.client.0.smithi064.stdout: format: 2 2023-10-09T06:20:50.960 INFO:tasks.workunit.client.0.smithi064.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T06:20:50.960 INFO:tasks.workunit.client.0.smithi064.stdout: op_features: 2023-10-09T06:20:50.960 INFO:tasks.workunit.client.0.smithi064.stdout: flags: 2023-10-09T06:20:50.960 INFO:tasks.workunit.client.0.smithi064.stdout: create_timestamp: Mon Oct 9 06:20:50 2023 2023-10-09T06:20:50.960 INFO:tasks.workunit.client.0.smithi064.stdout: access_timestamp: Mon Oct 9 06:20:50 2023 2023-10-09T06:20:50.960 INFO:tasks.workunit.client.0.smithi064.stdout: modify_timestamp: Mon Oct 9 06:20:50 2023 2023-10-09T06:20:50.960 INFO:tasks.workunit.client.0.smithi064.stdout: protected: False 2023-10-09T06:20:50.962 INFO:tasks.workunit.client.0.smithi064.stderr:++ rbd image-meta list testimg_import 2023-10-09T06:20:50.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:50 smithi064 ceph-mon[220044]: pgmap v1679: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 243 KiB/s rd, 72 KiB/s wr, 65 op/s 2023-10-09T06:20:50.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:50 smithi064 ceph-mon[220044]: failed to encode map e652 with expected crc 2023-10-09T06:20:50.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:50 smithi064 ceph-mon[220044]: failed to encode map e652 with expected crc 2023-10-09T06:20:50.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:50 smithi064 ceph-mon[220044]: osdmap e652: 8 total, 8 up, 6 in 2023-10-09T06:20:50.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:50 smithi064 ceph-mon[212750]: pgmap v1679: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 243 KiB/s rd, 72 KiB/s wr, 65 op/s 2023-10-09T06:20:50.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:50 smithi064 ceph-mon[212750]: failed to encode map e652 with expected crc 2023-10-09T06:20:50.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:50 smithi064 ceph-mon[212750]: failed to encode map e652 with expected crc 2023-10-09T06:20:50.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:50 smithi064 ceph-mon[212750]: osdmap e652: 8 total, 8 up, 6 in 2023-10-09T06:20:51.002 INFO:tasks.workunit.client.0.smithi064.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-10-09T06:20:51.002 INFO:tasks.workunit.client.0.smithi064.stderr: 2023-10-09T06:20:51.002 INFO:tasks.workunit.client.0.smithi064.stderr:Key Value 2023-10-09T06:20:51.002 INFO:tasks.workunit.client.0.smithi064.stderr:key1 value1' 2023-10-09T06:20:51.003 INFO:tasks.workunit.client.0.smithi064.stderr:+ '[' 'There is 1 metadatum on this image: 2023-10-09T06:20:51.003 INFO:tasks.workunit.client.0.smithi064.stderr: 2023-10-09T06:20:51.003 INFO:tasks.workunit.client.0.smithi064.stderr:Key Value 2023-10-09T06:20:51.003 INFO:tasks.workunit.client.0.smithi064.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-10-09T06:20:51.003 INFO:tasks.workunit.client.0.smithi064.stderr: 2023-10-09T06:20:51.003 INFO:tasks.workunit.client.0.smithi064.stderr:Key Value 2023-10-09T06:20:51.003 INFO:tasks.workunit.client.0.smithi064.stderr:key1 value1' ']' 2023-10-09T06:20:51.003 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export testimg_import /tmp/rbd_import_export_695299/img_import 2023-10-09T06:20:51.037 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 100% complete...done. 2023-10-09T06:20:51.040 INFO:tasks.workunit.client.0.smithi064.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_695299/img /tmp/rbd_import_export_695299/img_import 2023-10-09T06:20:51.040 INFO:tasks.workunit.client.0.smithi064.stderr:+ cmp -l /tmp/rbd_import_export_695299/img /tmp/rbd_import_export_695299/img_import 2023-10-09T06:20:51.041 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_695299/img --format %b 2023-10-09T06:20:51.042 INFO:tasks.workunit.client.0.smithi064.stderr:+ origsize=24 2023-10-09T06:20:51.042 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_695299/img_import --format %b 2023-10-09T06:20:51.043 INFO:tasks.workunit.client.0.smithi064.stderr:+ exportsize=224 2023-10-09T06:20:51.043 INFO:tasks.workunit.client.0.smithi064.stderr:+ difference=200 2023-10-09T06:20:51.043 INFO:tasks.workunit.client.0.smithi064.stderr:+ difference=200 2023-10-09T06:20:51.043 INFO:tasks.workunit.client.0.smithi064.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-10-09T06:20:51.043 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_695299/img_snap 2023-10-09T06:20:51.060 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:51.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:51.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:51.079 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 100% complete...done. 2023-10-09T06:20:51.082 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_695299/img_snap_import 2023-10-09T06:20:51.117 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 100% complete...done. 2023-10-09T06:20:51.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:50 smithi156 ceph-mon[195101]: pgmap v1679: 103 pgs: 103 active+clean; 582 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 243 KiB/s rd, 72 KiB/s wr, 65 op/s 2023-10-09T06:20:51.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:50 smithi156 ceph-mon[195101]: failed to encode map e652 with expected crc 2023-10-09T06:20:51.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:50 smithi156 ceph-mon[195101]: failed to encode map e652 with expected crc 2023-10-09T06:20:51.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:50 smithi156 ceph-mon[195101]: osdmap e652: 8 total, 8 up, 6 in 2023-10-09T06:20:51.121 INFO:tasks.workunit.client.0.smithi064.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_695299/img_snap /tmp/rbd_import_export_695299/img_snap_import 2023-10-09T06:20:51.121 INFO:tasks.workunit.client.0.smithi064.stderr:+ cmp -l /tmp/rbd_import_export_695299/img_snap /tmp/rbd_import_export_695299/img_snap_import 2023-10-09T06:20:51.122 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_695299/img_snap --format %b 2023-10-09T06:20:51.123 INFO:tasks.workunit.client.0.smithi064.stderr:+ origsize=224 2023-10-09T06:20:51.123 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_695299/img_snap_import --format %b 2023-10-09T06:20:51.124 INFO:tasks.workunit.client.0.smithi064.stderr:+ exportsize=224 2023-10-09T06:20:51.124 INFO:tasks.workunit.client.0.smithi064.stderr:+ difference=0 2023-10-09T06:20:51.124 INFO:tasks.workunit.client.0.smithi064.stderr:+ difference=0 2023-10-09T06:20:51.124 INFO:tasks.workunit.client.0.smithi064.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:20:51.124 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_695299/img_v2 2023-10-09T06:20:51.125 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_695299/img_import 2023-10-09T06:20:51.125 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_695299/img_snap 2023-10-09T06:20:51.126 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_695299/img_snap_import 2023-10-09T06:20:51.126 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd snap rm testimg_import@snap 2023-10-09T06:20:51.166 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:51.192 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:51.234 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:51.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:51.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:51.339 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:51.365 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:51.407 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:51.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:51.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:51.511 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:51.538 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:51.579 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:51.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:51.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:51.684 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:51.715 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:51.770 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:51.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:51.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:51.874 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:51.894 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.1 to 0.351115 (59e2) 2023-10-09T06:20:51.899 INFO:tasks.workunit.client.0.smithi064.stderr: Removing snap: 100% complete...done. 2023-10-09T06:20:51.906 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd remove testimg_import 2023-10-09T06:20:51.908 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:51.949 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:51.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:51.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:51.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:51 smithi064 ceph-mon[220044]: failed to encode map e653 with expected crc 2023-10-09T06:20:51.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:51 smithi064 ceph-mon[220044]: osdmap e653: 8 total, 8 up, 6 in 2023-10-09T06:20:51.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:51 smithi064 ceph-mon[220044]: failed to encode map e653 with expected crc 2023-10-09T06:20:51.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:51 smithi064 ceph-mon[220044]: failed to encode map e653 with expected crc 2023-10-09T06:20:51.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:51 smithi064 ceph-mon[220044]: failed to encode map e653 with expected crc 2023-10-09T06:20:51.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:51 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/805714539' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 0.35111489604760215}]: dispatch 2023-10-09T06:20:51.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:51 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 0.35111489604760215}]: dispatch 2023-10-09T06:20:51.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:51 smithi064 ceph-mon[212750]: failed to encode map e653 with expected crc 2023-10-09T06:20:51.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:51 smithi064 ceph-mon[212750]: osdmap e653: 8 total, 8 up, 6 in 2023-10-09T06:20:51.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:51 smithi064 ceph-mon[212750]: failed to encode map e653 with expected crc 2023-10-09T06:20:51.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:51 smithi064 ceph-mon[212750]: failed to encode map e653 with expected crc 2023-10-09T06:20:51.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:51 smithi064 ceph-mon[212750]: failed to encode map e653 with expected crc 2023-10-09T06:20:51.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:51 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/805714539' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 0.35111489604760215}]: dispatch 2023-10-09T06:20:51.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:51 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 0.35111489604760215}]: dispatch 2023-10-09T06:20:52.026 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:20:52.055 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:52.081 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:52.123 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:52.130 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:51 smithi156 ceph-mon[195101]: failed to encode map e653 with expected crc 2023-10-09T06:20:52.130 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:51 smithi156 ceph-mon[195101]: osdmap e653: 8 total, 8 up, 6 in 2023-10-09T06:20:52.130 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:51 smithi156 ceph-mon[195101]: failed to encode map e653 with expected crc 2023-10-09T06:20:52.130 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:51 smithi156 ceph-mon[195101]: failed to encode map e653 with expected crc 2023-10-09T06:20:52.130 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:51 smithi156 ceph-mon[195101]: failed to encode map e653 with expected crc 2023-10-09T06:20:52.130 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:51 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/805714539' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 0.35111489604760215}]: dispatch 2023-10-09T06:20:52.130 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:51 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 0.35111489604760215}]: dispatch 2023-10-09T06:20:52.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:52.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:52.232 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:52.259 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:52.300 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:52.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:52.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:52.405 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:52.431 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:52.472 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:52.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:52.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:52.580 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:52.613 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:52.997 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:52 smithi064 ceph-mon[212750]: pgmap v1682: 103 pgs: 103 active+clean; 692 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 199 KiB/s rd, 70 KiB/s wr, 101 op/s 2023-10-09T06:20:52.997 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:52 smithi064 ceph-mon[212750]: failed to encode map e654 with expected crc 2023-10-09T06:20:52.997 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:52 smithi064 ceph-mon[212750]: failed to encode map e654 with expected crc 2023-10-09T06:20:52.997 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:52 smithi064 ceph-mon[212750]: failed to encode map e654 with expected crc 2023-10-09T06:20:52.997 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:52 smithi064 ceph-mon[212750]: failed to encode map e654 with expected crc 2023-10-09T06:20:52.997 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:52 smithi064 ceph-mon[212750]: failed to encode map e654 with expected crc 2023-10-09T06:20:52.997 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:52 smithi064 ceph-mon[212750]: failed to encode map e654 with expected crc 2023-10-09T06:20:52.998 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:52 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 0.35111489604760215}]': finished 2023-10-09T06:20:52.998 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:52 smithi064 ceph-mon[212750]: osdmap e654: 8 total, 8 up, 6 in 2023-10-09T06:20:52.998 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:52 smithi064 ceph-mon[212750]: osdmap e655: 8 total, 8 up, 6 in 2023-10-09T06:20:52.998 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:52 smithi064 ceph-mon[220044]: pgmap v1682: 103 pgs: 103 active+clean; 692 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 199 KiB/s rd, 70 KiB/s wr, 101 op/s 2023-10-09T06:20:52.998 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:52 smithi064 ceph-mon[220044]: failed to encode map e654 with expected crc 2023-10-09T06:20:52.998 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:52 smithi064 ceph-mon[220044]: failed to encode map e654 with expected crc 2023-10-09T06:20:52.998 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:52 smithi064 ceph-mon[220044]: failed to encode map e654 with expected crc 2023-10-09T06:20:52.998 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:52 smithi064 ceph-mon[220044]: failed to encode map e654 with expected crc 2023-10-09T06:20:52.999 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:52 smithi064 ceph-mon[220044]: failed to encode map e654 with expected crc 2023-10-09T06:20:52.999 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:52 smithi064 ceph-mon[220044]: failed to encode map e654 with expected crc 2023-10-09T06:20:52.999 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:52 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 0.35111489604760215}]': finished 2023-10-09T06:20:52.999 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:52 smithi064 ceph-mon[220044]: osdmap e654: 8 total, 8 up, 6 in 2023-10-09T06:20:52.999 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:52 smithi064 ceph-mon[220044]: osdmap e655: 8 total, 8 up, 6 in 2023-10-09T06:20:53.070 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 100% complete...done. 2023-10-09T06:20:53.073 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd snap rm testimg@snap 2023-10-09T06:20:53.078 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:20:53.078 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-10-09T06:20:53.078 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2023-10-09T06:20:53.079 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:20:53.237 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:53.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:53.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:53.342 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:53.366 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:52 smithi156 ceph-mon[195101]: pgmap v1682: 103 pgs: 103 active+clean; 692 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 199 KiB/s rd, 70 KiB/s wr, 101 op/s 2023-10-09T06:20:53.366 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:52 smithi156 ceph-mon[195101]: failed to encode map e654 with expected crc 2023-10-09T06:20:53.366 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:52 smithi156 ceph-mon[195101]: failed to encode map e654 with expected crc 2023-10-09T06:20:53.366 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:52 smithi156 ceph-mon[195101]: failed to encode map e654 with expected crc 2023-10-09T06:20:53.366 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:52 smithi156 ceph-mon[195101]: failed to encode map e654 with expected crc 2023-10-09T06:20:53.366 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:52 smithi156 ceph-mon[195101]: failed to encode map e654 with expected crc 2023-10-09T06:20:53.367 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:52 smithi156 ceph-mon[195101]: failed to encode map e654 with expected crc 2023-10-09T06:20:53.367 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:52 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 0.35111489604760215}]': finished 2023-10-09T06:20:53.367 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:52 smithi156 ceph-mon[195101]: osdmap e654: 8 total, 8 up, 6 in 2023-10-09T06:20:53.367 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:52 smithi156 ceph-mon[195101]: osdmap e655: 8 total, 8 up, 6 in 2023-10-09T06:20:53.368 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:53.410 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:53.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:53.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:53.516 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:53.542 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:53.583 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:53.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:53.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:53.688 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:53.718 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:53.815 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:20:53.834 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:20:53.853 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:20:53.875 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:20:53.878 INFO:tasks.workunit.client.0.smithi064.stderr: Removing snap: 100% complete...done. 2023-10-09T06:20:53.891 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm testimg 2023-10-09T06:20:53.900 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:20:53.933 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:20:53.953 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:53.959 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:20:53.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:53.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:53.967 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:53 smithi064 ceph-mon[220044]: failed to encode map e655 with expected crc 2023-10-09T06:20:53.967 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:53 smithi064 ceph-mon[220044]: failed to encode map e655 with expected crc 2023-10-09T06:20:53.967 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:53 smithi064 ceph-mon[220044]: failed to encode map e655 with expected crc 2023-10-09T06:20:53.968 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:53 smithi064 ceph-mon[220044]: failed to encode map e655 with expected crc 2023-10-09T06:20:53.968 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:53 smithi064 ceph-mon[220044]: failed to encode map e655 with expected crc 2023-10-09T06:20:53.968 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:53 smithi064 ceph-mon[220044]: failed to encode map e655 with expected crc 2023-10-09T06:20:53.968 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:53 smithi064 ceph-mon[220044]: failed to encode map e655 with expected crc 2023-10-09T06:20:53.968 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:53 smithi064 ceph-mon[220044]: failed to encode map e655 with expected crc 2023-10-09T06:20:53.968 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:53 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1603357145' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:53.968 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:53 smithi064 ceph-mon[220044]: osdmap e656: 8 total, 8 up, 6 in 2023-10-09T06:20:53.968 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:53 smithi064 ceph-mon[212750]: failed to encode map e655 with expected crc 2023-10-09T06:20:53.968 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:53 smithi064 ceph-mon[212750]: failed to encode map e655 with expected crc 2023-10-09T06:20:53.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:53 smithi064 ceph-mon[212750]: failed to encode map e655 with expected crc 2023-10-09T06:20:53.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:53 smithi064 ceph-mon[212750]: failed to encode map e655 with expected crc 2023-10-09T06:20:53.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:53 smithi064 ceph-mon[212750]: failed to encode map e655 with expected crc 2023-10-09T06:20:53.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:53 smithi064 ceph-mon[212750]: failed to encode map e655 with expected crc 2023-10-09T06:20:53.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:53 smithi064 ceph-mon[212750]: failed to encode map e655 with expected crc 2023-10-09T06:20:53.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:53 smithi064 ceph-mon[212750]: failed to encode map e655 with expected crc 2023-10-09T06:20:53.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:53 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1603357145' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:53.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:53 smithi064 ceph-mon[212750]: osdmap e656: 8 total, 8 up, 6 in 2023-10-09T06:20:53.970 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:20:53.983 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:20:53.984 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:20:53.985 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:20:53.985 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:20:53.985 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:20:53.985 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:20:53.985 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:20:53.985 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:20:53.985 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:20:54.004 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 100% complete...done. 2023-10-09T06:20:54.008 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --order 20 /tmp/rbd_import_export_695299/img testimg 2023-10-09T06:20:54.062 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:54.076 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 100% complete...done. 2023-10-09T06:20:54.077 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:20:54.080 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_695299/img_v2 2023-10-09T06:20:54.089 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:54.122 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:20:54.125 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_695299/img_v2 testimg_import 2023-10-09T06:20:54.130 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:54.135 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:53 smithi156 ceph-mon[195101]: failed to encode map e655 with expected crc 2023-10-09T06:20:54.136 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:53 smithi156 ceph-mon[195101]: failed to encode map e655 with expected crc 2023-10-09T06:20:54.136 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:53 smithi156 ceph-mon[195101]: failed to encode map e655 with expected crc 2023-10-09T06:20:54.136 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:53 smithi156 ceph-mon[195101]: failed to encode map e655 with expected crc 2023-10-09T06:20:54.136 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:53 smithi156 ceph-mon[195101]: failed to encode map e655 with expected crc 2023-10-09T06:20:54.136 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:53 smithi156 ceph-mon[195101]: failed to encode map e655 with expected crc 2023-10-09T06:20:54.136 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:53 smithi156 ceph-mon[195101]: failed to encode map e655 with expected crc 2023-10-09T06:20:54.136 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:53 smithi156 ceph-mon[195101]: failed to encode map e655 with expected crc 2023-10-09T06:20:54.137 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:53 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1603357145' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:54.137 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:53 smithi156 ceph-mon[195101]: osdmap e656: 8 total, 8 up, 6 in 2023-10-09T06:20:54.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:54.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:54.166 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:54.188 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:20:54.192 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd info testimg_import 2023-10-09T06:20:54.193 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep order 2023-10-09T06:20:54.193 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep 20 2023-10-09T06:20:54.193 INFO:tasks.workunit.client.0.smithi064.stderr:+ awk '{print $2}' 2023-10-09T06:20:54.235 INFO:tasks.workunit.client.0.smithi064.stdout:20 2023-10-09T06:20:54.235 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_695299/img_v2 2023-10-09T06:20:54.236 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd remove testimg_import 2023-10-09T06:20:54.238 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:54.269 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:54.335 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 100% complete...done. 2023-10-09T06:20:54.339 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd remove testimg 2023-10-09T06:20:54.435 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 100% complete...done. 2023-10-09T06:20:54.438 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_695299/img testimg 2023-10-09T06:20:54.492 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 100% complete...done. 2023-10-09T06:20:54.496 INFO:tasks.workunit.client.0.smithi064.stderr:++ rbd info testimg 2023-10-09T06:20:54.496 INFO:tasks.workunit.client.0.smithi064.stderr:++ grep features 2023-10-09T06:20:54.531 INFO:tasks.workunit.client.0.smithi064.stderr:+ FEATURES_BEFORE=' features: layering 2023-10-09T06:20:54.531 INFO:tasks.workunit.client.0.smithi064.stderr: op_features: ' 2023-10-09T06:20:54.532 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_695299/img_v2 2023-10-09T06:20:54.624 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:54.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:54.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:54.713 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:20:54.716 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_695299/img_v2 testimg_import 2023-10-09T06:20:54.728 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:54.759 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:54.772 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:20:54.776 INFO:tasks.workunit.client.0.smithi064.stderr:++ rbd info testimg_import 2023-10-09T06:20:54.776 INFO:tasks.workunit.client.0.smithi064.stderr:++ grep features 2023-10-09T06:20:54.813 INFO:tasks.workunit.client.0.smithi064.stderr:+ FEATURES_AFTER=' features: layering 2023-10-09T06:20:54.813 INFO:tasks.workunit.client.0.smithi064.stderr: op_features: ' 2023-10-09T06:20:54.813 INFO:tasks.workunit.client.0.smithi064.stderr:+ '[' ' features: layering 2023-10-09T06:20:54.813 INFO:tasks.workunit.client.0.smithi064.stderr: op_features: ' '!=' ' features: layering 2023-10-09T06:20:54.813 INFO:tasks.workunit.client.0.smithi064.stderr: op_features: ' ']' 2023-10-09T06:20:54.814 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_695299/img_v2 2023-10-09T06:20:54.814 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd remove testimg_import 2023-10-09T06:20:54.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:54 smithi064 ceph-mon[212750]: pgmap v1685: 103 pgs: 103 active+clean; 692 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 59 KiB/s rd, 6.2 KiB/s wr, 76 op/s 2023-10-09T06:20:54.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:54 smithi064 ceph-mon[212750]: failed to encode map e656 with expected crc 2023-10-09T06:20:54.898 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 100% complete...done. 2023-10-09T06:20:54.901 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd remove testimg 2023-10-09T06:20:54.971 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 100% complete...done. 2023-10-09T06:20:54.974 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_695299/img testimg 2023-10-09T06:20:55.033 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 100% complete...done. 2023-10-09T06:20:55.036 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_695299/img_v2 2023-10-09T06:20:55.075 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:20:55.078 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_695299/img_v2 testimg_import 2023-10-09T06:20:55.113 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:55.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:55.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:55.143 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:20:55.146 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd info testimg_import 2023-10-09T06:20:55.146 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep 'stripe unit' 2023-10-09T06:20:55.146 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep -Ei '(4 KiB|4096)' 2023-10-09T06:20:55.184 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:54 smithi064 ceph-mon[220044]: pgmap v1685: 103 pgs: 103 active+clean; 692 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 59 KiB/s rd, 6.2 KiB/s wr, 76 op/s 2023-10-09T06:20:55.184 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:54 smithi064 ceph-mon[220044]: failed to encode map e656 with expected crc 2023-10-09T06:20:55.184 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:54 smithi064 ceph-mon[220044]: failed to encode map e656 with expected crc 2023-10-09T06:20:55.184 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:54 smithi064 ceph-mon[220044]: failed to encode map e656 with expected crc 2023-10-09T06:20:55.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:54 smithi064 ceph-mon[220044]: failed to encode map e656 with expected crc 2023-10-09T06:20:55.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:54 smithi064 ceph-mon[220044]: failed to encode map e656 with expected crc 2023-10-09T06:20:55.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:54 smithi064 ceph-mon[220044]: failed to encode map e656 with expected crc 2023-10-09T06:20:55.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:54 smithi064 ceph-mon[220044]: failed to encode map e656 with expected crc 2023-10-09T06:20:55.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:54 smithi064 ceph-mon[220044]: osdmap e657: 8 total, 8 up, 6 in 2023-10-09T06:20:55.185 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:54 smithi064 ceph-mon[212750]: failed to encode map e656 with expected crc 2023-10-09T06:20:55.185 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:54 smithi064 ceph-mon[212750]: failed to encode map e656 with expected crc 2023-10-09T06:20:55.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:54 smithi064 ceph-mon[212750]: failed to encode map e656 with expected crc 2023-10-09T06:20:55.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:54 smithi064 ceph-mon[212750]: failed to encode map e656 with expected crc 2023-10-09T06:20:55.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:54 smithi064 ceph-mon[212750]: failed to encode map e656 with expected crc 2023-10-09T06:20:55.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:54 smithi064 ceph-mon[212750]: failed to encode map e656 with expected crc 2023-10-09T06:20:55.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:54 smithi064 ceph-mon[212750]: osdmap e657: 8 total, 8 up, 6 in 2023-10-09T06:20:55.186 INFO:tasks.workunit.client.0.smithi064.stdout: stripe unit: 4 KiB 2023-10-09T06:20:55.187 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd info testimg_import 2023-10-09T06:20:55.187 INFO:tasks.workunit.client.0.smithi064.stderr:+ awk '{print $3}' 2023-10-09T06:20:55.187 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep 1000 2023-10-09T06:20:55.187 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep 'stripe count' 2023-10-09T06:20:55.218 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:55.224 INFO:tasks.workunit.client.0.smithi064.stdout:1000 2023-10-09T06:20:55.224 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_695299/img_v2 2023-10-09T06:20:55.225 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd remove testimg_import 2023-10-09T06:20:55.241 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:54 smithi156 ceph-mon[195101]: pgmap v1685: 103 pgs: 103 active+clean; 692 KiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 59 KiB/s rd, 6.2 KiB/s wr, 76 op/s 2023-10-09T06:20:55.242 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:54 smithi156 ceph-mon[195101]: failed to encode map e656 with expected crc 2023-10-09T06:20:55.242 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:54 smithi156 ceph-mon[195101]: failed to encode map e656 with expected crc 2023-10-09T06:20:55.242 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:54 smithi156 ceph-mon[195101]: failed to encode map e656 with expected crc 2023-10-09T06:20:55.242 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:54 smithi156 ceph-mon[195101]: failed to encode map e656 with expected crc 2023-10-09T06:20:55.242 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:54 smithi156 ceph-mon[195101]: failed to encode map e656 with expected crc 2023-10-09T06:20:55.242 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:54 smithi156 ceph-mon[195101]: failed to encode map e656 with expected crc 2023-10-09T06:20:55.242 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:54 smithi156 ceph-mon[195101]: failed to encode map e656 with expected crc 2023-10-09T06:20:55.242 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:54 smithi156 ceph-mon[195101]: osdmap e657: 8 total, 8 up, 6 in 2023-10-09T06:20:55.244 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:55.287 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:55.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:55.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:55.317 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T06:20:55.320 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd remove testimg 2023-10-09T06:20:55.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:55.410 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T06:20:55.415 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_695299/img testimg 2023-10-09T06:20:55.426 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:55.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:20:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:20:55] "GET /metrics HTTP/1.1" 200 34080 "" "Prometheus/2.43.0" 2023-10-09T06:20:55.480 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 100% complete...done. 2023-10-09T06:20:55.480 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:55.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:55.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:55.484 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd snap create testimg@snap1 2023-10-09T06:20:55.584 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:55.610 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:55.651 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:55.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:55.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:55.756 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:55.784 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:55.904 INFO:tasks.workunit.client.0.smithi064.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T06:20:55.911 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd snap create testimg@snap2 2023-10-09T06:20:56.034 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:56.037 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:55 smithi064 ceph-mon[220044]: failed to encode map e657 with expected crc 2023-10-09T06:20:56.037 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:55 smithi064 ceph-mon[220044]: failed to encode map e657 with expected crc 2023-10-09T06:20:56.038 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:55 smithi064 ceph-mon[220044]: failed to encode map e657 with expected crc 2023-10-09T06:20:56.038 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:55 smithi064 ceph-mon[220044]: failed to encode map e657 with expected crc 2023-10-09T06:20:56.038 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:55 smithi064 ceph-mon[220044]: failed to encode map e657 with expected crc 2023-10-09T06:20:56.038 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:55 smithi064 ceph-mon[220044]: failed to encode map e657 with expected crc 2023-10-09T06:20:56.038 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:55 smithi064 ceph-mon[220044]: failed to encode map e657 with expected crc 2023-10-09T06:20:56.038 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:55 smithi064 ceph-mon[220044]: osdmap e658: 8 total, 8 up, 6 in 2023-10-09T06:20:56.039 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:55 smithi064 ceph-mon[212750]: failed to encode map e657 with expected crc 2023-10-09T06:20:56.039 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:55 smithi064 ceph-mon[212750]: failed to encode map e657 with expected crc 2023-10-09T06:20:56.039 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:55 smithi064 ceph-mon[212750]: failed to encode map e657 with expected crc 2023-10-09T06:20:56.039 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:55 smithi064 ceph-mon[212750]: failed to encode map e657 with expected crc 2023-10-09T06:20:56.039 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:55 smithi064 ceph-mon[212750]: failed to encode map e657 with expected crc 2023-10-09T06:20:56.039 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:55 smithi064 ceph-mon[212750]: failed to encode map e657 with expected crc 2023-10-09T06:20:56.039 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:55 smithi064 ceph-mon[212750]: failed to encode map e657 with expected crc 2023-10-09T06:20:56.039 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:55 smithi064 ceph-mon[212750]: osdmap e658: 8 total, 8 up, 6 in 2023-10-09T06:20:56.067 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:56.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:56.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:56.177 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:56.206 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:56.345 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:56.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:56.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:56.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:55 smithi156 ceph-mon[195101]: failed to encode map e657 with expected crc 2023-10-09T06:20:56.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:55 smithi156 ceph-mon[195101]: failed to encode map e657 with expected crc 2023-10-09T06:20:56.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:55 smithi156 ceph-mon[195101]: failed to encode map e657 with expected crc 2023-10-09T06:20:56.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:55 smithi156 ceph-mon[195101]: failed to encode map e657 with expected crc 2023-10-09T06:20:56.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:55 smithi156 ceph-mon[195101]: failed to encode map e657 with expected crc 2023-10-09T06:20:56.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:55 smithi156 ceph-mon[195101]: failed to encode map e657 with expected crc 2023-10-09T06:20:56.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:55 smithi156 ceph-mon[195101]: failed to encode map e657 with expected crc 2023-10-09T06:20:56.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:55 smithi156 ceph-mon[195101]: osdmap e658: 8 total, 8 up, 6 in 2023-10-09T06:20:56.449 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:56.477 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:56.520 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:56.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:56.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:56.629 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:56.659 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:56.943 INFO:tasks.workunit.client.0.smithi064.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T06:20:56.950 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd snap protect testimg@snap2 2023-10-09T06:20:56.994 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_695299/snap_protect 2023-10-09T06:20:57.042 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:20:57.045 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_695299/snap_protect testimg_import 2023-10-09T06:20:57.205 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:20:57.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:56 smithi064 ceph-mon[220044]: pgmap v1688: 103 pgs: 4 active+undersized+remapped, 99 active+clean; 692 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 267 KiB/s rd, 26 KiB/s wr, 342 op/s; 67 KiB/s, 0 keys/s, 2 objects/s recovering 2023-10-09T06:20:57.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:56 smithi064 ceph-mon[220044]: failed to encode map e658 with expected crc 2023-10-09T06:20:57.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:56 smithi064 ceph-mon[220044]: failed to encode map e658 with expected crc 2023-10-09T06:20:57.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:56 smithi064 ceph-mon[220044]: failed to encode map e658 with expected crc 2023-10-09T06:20:57.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:56 smithi064 ceph-mon[220044]: failed to encode map e658 with expected crc 2023-10-09T06:20:57.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:56 smithi064 ceph-mon[220044]: failed to encode map e658 with expected crc 2023-10-09T06:20:57.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:56 smithi064 ceph-mon[212750]: pgmap v1688: 103 pgs: 4 active+undersized+remapped, 99 active+clean; 692 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 267 KiB/s rd, 26 KiB/s wr, 342 op/s; 67 KiB/s, 0 keys/s, 2 objects/s recovering 2023-10-09T06:20:57.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:56 smithi064 ceph-mon[212750]: failed to encode map e658 with expected crc 2023-10-09T06:20:57.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:56 smithi064 ceph-mon[212750]: failed to encode map e658 with expected crc 2023-10-09T06:20:57.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:56 smithi064 ceph-mon[212750]: failed to encode map e658 with expected crc 2023-10-09T06:20:57.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:56 smithi064 ceph-mon[212750]: failed to encode map e658 with expected crc 2023-10-09T06:20:57.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:56 smithi064 ceph-mon[212750]: failed to encode map e658 with expected crc 2023-10-09T06:20:57.282 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:57.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:57.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:57.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:56 smithi156 ceph-mon[195101]: pgmap v1688: 103 pgs: 4 active+undersized+remapped, 99 active+clean; 692 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 267 KiB/s rd, 26 KiB/s wr, 342 op/s; 67 KiB/s, 0 keys/s, 2 objects/s recovering 2023-10-09T06:20:57.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:56 smithi156 ceph-mon[195101]: failed to encode map e658 with expected crc 2023-10-09T06:20:57.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:56 smithi156 ceph-mon[195101]: failed to encode map e658 with expected crc 2023-10-09T06:20:57.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:56 smithi156 ceph-mon[195101]: failed to encode map e658 with expected crc 2023-10-09T06:20:57.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:56 smithi156 ceph-mon[195101]: failed to encode map e658 with expected crc 2023-10-09T06:20:57.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:56 smithi156 ceph-mon[195101]: failed to encode map e658 with expected crc 2023-10-09T06:20:57.386 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:57.418 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:57.466 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:57.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:57.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:57.571 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:57.601 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:57.699 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:57.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:57.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:57.750 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:57.792 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-10-09T06:20:57.792 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:20:57.803 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:57.836 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:57.884 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:57.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:57.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:57.923 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-10-09T06:20:57.990 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:58.000 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 5, 7, 1, 6] out_osds: [2, 4] dead_osds: [] live_osds: [1, 2, 3, 6, 7, 4, 0, 5] 2023-10-09T06:20:58.000 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:20:58.001 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [0, 3, 5, 7, 1, 6] 2023-10-09T06:20:58.001 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd out 5 2023-10-09T06:20:58.018 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:58.061 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:58.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:58.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:57 smithi156 ceph-mon[195101]: failed to encode map e659 with expected crc 2023-10-09T06:20:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:57 smithi156 ceph-mon[195101]: osdmap e659: 8 total, 8 up, 6 in 2023-10-09T06:20:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:57 smithi156 ceph-mon[195101]: failed to encode map e659 with expected crc 2023-10-09T06:20:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:57 smithi156 ceph-mon[195101]: failed to encode map e659 with expected crc 2023-10-09T06:20:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:57 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3614759924' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:20:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:57 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:20:58.167 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:58.192 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:58.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:57 smithi064 ceph-mon[220044]: failed to encode map e659 with expected crc 2023-10-09T06:20:58.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:57 smithi064 ceph-mon[220044]: osdmap e659: 8 total, 8 up, 6 in 2023-10-09T06:20:58.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:57 smithi064 ceph-mon[220044]: failed to encode map e659 with expected crc 2023-10-09T06:20:58.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:57 smithi064 ceph-mon[220044]: failed to encode map e659 with expected crc 2023-10-09T06:20:58.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:57 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3614759924' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:20:58.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:57 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:20:58.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:57 smithi064 ceph-mon[212750]: failed to encode map e659 with expected crc 2023-10-09T06:20:58.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:57 smithi064 ceph-mon[212750]: osdmap e659: 8 total, 8 up, 6 in 2023-10-09T06:20:58.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:57 smithi064 ceph-mon[212750]: failed to encode map e659 with expected crc 2023-10-09T06:20:58.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:57 smithi064 ceph-mon[212750]: failed to encode map e659 with expected crc 2023-10-09T06:20:58.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:57 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3614759924' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:20:58.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:57 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:20:58.233 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:58.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:58.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:58.338 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:58.367 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:58.716 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:58.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:58.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:58.820 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:58.848 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:58.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:58.934 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:58.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:58.952 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:20:59.144 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:59.145 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:58 smithi156 ceph-mon[195101]: pgmap v1691: 103 pgs: 4 active+undersized+remapped, 99 active+clean; 692 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 348 KiB/s rd, 29 KiB/s wr, 449 op/s; 67 KiB/s, 0 keys/s, 3 objects/s recovering 2023-10-09T06:20:59.146 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:58 smithi156 ceph-mon[195101]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:59.146 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:58 smithi156 ceph-mon[195101]: failed to encode map e660 with expected crc 2023-10-09T06:20:59.146 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:58 smithi156 ceph-mon[195101]: failed to encode map e660 with expected crc 2023-10-09T06:20:59.146 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:58 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:20:59.146 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:58 smithi156 ceph-mon[195101]: osdmap e660: 8 total, 8 up, 6 in 2023-10-09T06:20:59.146 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:58 smithi156 ceph-mon[195101]: failed to encode map e660 with expected crc 2023-10-09T06:20:59.147 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:58 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:59.147 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:58 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:59.147 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd info testimg_import@snap1 2023-10-09T06:20:59.148 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep 'protected: False' 2023-10-09T06:20:59.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:58 smithi064 ceph-mon[220044]: pgmap v1691: 103 pgs: 4 active+undersized+remapped, 99 active+clean; 692 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 348 KiB/s rd, 29 KiB/s wr, 449 op/s; 67 KiB/s, 0 keys/s, 3 objects/s recovering 2023-10-09T06:20:59.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:58 smithi064 ceph-mon[220044]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:59.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:58 smithi064 ceph-mon[220044]: failed to encode map e660 with expected crc 2023-10-09T06:20:59.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:58 smithi064 ceph-mon[220044]: failed to encode map e660 with expected crc 2023-10-09T06:20:59.148 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:58 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:20:59.149 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:58 smithi064 ceph-mon[220044]: osdmap e660: 8 total, 8 up, 6 in 2023-10-09T06:20:59.149 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:58 smithi064 ceph-mon[220044]: failed to encode map e660 with expected crc 2023-10-09T06:20:59.149 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:58 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:59.149 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:58 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:59.149 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:58 smithi064 ceph-mon[212750]: pgmap v1691: 103 pgs: 4 active+undersized+remapped, 99 active+clean; 692 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 348 KiB/s rd, 29 KiB/s wr, 449 op/s; 67 KiB/s, 0 keys/s, 3 objects/s recovering 2023-10-09T06:20:59.149 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:58 smithi064 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:59.149 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:58 smithi064 ceph-mon[212750]: failed to encode map e660 with expected crc 2023-10-09T06:20:59.149 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:58 smithi064 ceph-mon[212750]: failed to encode map e660 with expected crc 2023-10-09T06:20:59.150 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:58 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:20:59.150 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:58 smithi064 ceph-mon[212750]: osdmap e660: 8 total, 8 up, 6 in 2023-10-09T06:20:59.150 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:58 smithi064 ceph-mon[212750]: failed to encode map e660 with expected crc 2023-10-09T06:20:59.150 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:58 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:20:59.150 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:58 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:59.151 INFO:tasks.workunit.client.0.smithi064.stdout: protected: False 2023-10-09T06:20:59.151 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd info testimg_import@snap2 2023-10-09T06:20:59.151 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep 'protected: True' 2023-10-09T06:20:59.151 INFO:tasks.workunit.client.0.smithi064.stdout: protected: True 2023-10-09T06:20:59.152 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_695299/snap_protect 2023-10-09T06:20:59.152 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd snap unprotect testimg@snap2 2023-10-09T06:20:59.152 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd snap unprotect testimg_import@snap2 2023-10-09T06:20:59.152 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd snap purge testimg 2023-10-09T06:20:59.187 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:59.585 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:59.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:59.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:59.631 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:20:59.705 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:59.821 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:59.912 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:59.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:59.913 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:59.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:00.014 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:00.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:59 smithi156 ceph-mon[195101]: failed to encode map e661 with expected crc 2023-10-09T06:21:00.268 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:59 smithi156 ceph-mon[195101]: osdmap e661: 8 total, 8 up, 6 in 2023-10-09T06:21:00.268 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:20:59 smithi156 ceph-mon[195101]: failed to encode map e661 with expected crc 2023-10-09T06:21:00.269 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:59 smithi064 ceph-mon[212750]: failed to encode map e661 with expected crc 2023-10-09T06:21:00.269 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:59 smithi064 ceph-mon[212750]: osdmap e661: 8 total, 8 up, 6 in 2023-10-09T06:21:00.269 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:20:59 smithi064 ceph-mon[212750]: failed to encode map e661 with expected crc 2023-10-09T06:21:00.269 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:59 smithi064 ceph-mon[220044]: failed to encode map e661 with expected crc 2023-10-09T06:21:00.269 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:59 smithi064 ceph-mon[220044]: osdmap e661: 8 total, 8 up, 6 in 2023-10-09T06:21:00.269 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:20:59 smithi064 ceph-mon[220044]: failed to encode map e661 with expected crc 2023-10-09T06:21:00.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:00.270 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:00.426 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:21:00.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:00.427 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:00.528 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:00.540 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:21:00.558 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:00.560 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:21:00.579 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:21:00.600 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:21:00.600 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:21:00.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:00.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:00.619 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:21:00.641 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:21:00.661 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:21:00.680 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:21:00.682 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:21:00.682 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:21:00.682 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:21:00.682 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:21:00.682 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:21:00.682 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:21:00.682 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:21:00.682 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:21:00.705 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:00.731 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:00.772 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:21:00.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:00.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:00.879 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:00.949 INFO:tasks.workunit.client.0.smithi064.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T06:21:00.956 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd snap purge testimg_import 2023-10-09T06:21:00.966 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:01.217 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:21:01.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:00 smithi064 ceph-mon[212750]: pgmap v1694: 103 pgs: 4 active+undersized+remapped, 99 active+clean; 692 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 81 KiB/s rd, 3.2 KiB/s wr, 106 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T06:21:01.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:00 smithi064 ceph-mon[212750]: failed to encode map e662 with expected crc 2023-10-09T06:21:01.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:00 smithi064 ceph-mon[212750]: osdmap e662: 8 total, 8 up, 6 in 2023-10-09T06:21:01.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:00 smithi064 ceph-mon[212750]: failed to encode map e662 with expected crc 2023-10-09T06:21:01.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:00 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/75185816' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:01.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:00 smithi064 ceph-mon[220044]: pgmap v1694: 103 pgs: 4 active+undersized+remapped, 99 active+clean; 692 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 81 KiB/s rd, 3.2 KiB/s wr, 106 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T06:21:01.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:00 smithi064 ceph-mon[220044]: failed to encode map e662 with expected crc 2023-10-09T06:21:01.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:00 smithi064 ceph-mon[220044]: osdmap e662: 8 total, 8 up, 6 in 2023-10-09T06:21:01.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:00 smithi064 ceph-mon[220044]: failed to encode map e662 with expected crc 2023-10-09T06:21:01.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:00 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/75185816' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:01.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:01.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:01.322 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:01.351 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:01.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:00 smithi156 ceph-mon[195101]: pgmap v1694: 103 pgs: 4 active+undersized+remapped, 99 active+clean; 692 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 81 KiB/s rd, 3.2 KiB/s wr, 106 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T06:21:01.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:00 smithi156 ceph-mon[195101]: failed to encode map e662 with expected crc 2023-10-09T06:21:01.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:00 smithi156 ceph-mon[195101]: osdmap e662: 8 total, 8 up, 6 in 2023-10-09T06:21:01.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:00 smithi156 ceph-mon[195101]: failed to encode map e662 with expected crc 2023-10-09T06:21:01.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:00 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/75185816' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:01.793 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:21:02.152 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:21:02.154 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:01 smithi064 ceph-mon[212750]: failed to encode map e662 with expected crc 2023-10-09T06:21:02.154 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:01 smithi064 ceph-mon[212750]: failed to encode map e663 with expected crc 2023-10-09T06:21:02.154 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:01 smithi064 ceph-mon[212750]: osdmap e663: 8 total, 8 up, 6 in 2023-10-09T06:21:02.154 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:01 smithi064 ceph-mon[212750]: failed to encode map e663 with expected crc 2023-10-09T06:21:02.154 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:01 smithi064 ceph-mon[220044]: failed to encode map e662 with expected crc 2023-10-09T06:21:02.154 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:01 smithi064 ceph-mon[220044]: failed to encode map e663 with expected crc 2023-10-09T06:21:02.155 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:01 smithi064 ceph-mon[220044]: osdmap e663: 8 total, 8 up, 6 in 2023-10-09T06:21:02.155 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:01 smithi064 ceph-mon[220044]: failed to encode map e663 with expected crc 2023-10-09T06:21:02.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:02.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:02.256 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:02.288 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:02.343 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:21:02.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:02.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:02.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:01 smithi156 ceph-mon[195101]: failed to encode map e662 with expected crc 2023-10-09T06:21:02.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:01 smithi156 ceph-mon[195101]: failed to encode map e663 with expected crc 2023-10-09T06:21:02.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:01 smithi156 ceph-mon[195101]: osdmap e663: 8 total, 8 up, 6 in 2023-10-09T06:21:02.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:01 smithi156 ceph-mon[195101]: failed to encode map e663 with expected crc 2023-10-09T06:21:02.441 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:02.451 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:02.483 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:02.531 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:21:02.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:02.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:02.635 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:02.667 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:02.714 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:02.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:02.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:02.821 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:02.858 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:02.869 INFO:tasks.workunit.client.0.smithi064.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T06:21:02.880 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd remove testimg 2023-10-09T06:21:02.948 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:02 smithi156 ceph-mon[195101]: pgmap v1697: 103 pgs: 103 active+clean; 802 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 208 KiB/s rd, 31 KiB/s wr, 287 op/s 2023-10-09T06:21:02.948 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:02 smithi156 ceph-mon[195101]: failed to encode map e664 with expected crc 2023-10-09T06:21:02.948 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:02 smithi156 ceph-mon[195101]: osdmap e664: 8 total, 8 up, 6 in 2023-10-09T06:21:02.948 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:02 smithi156 ceph-mon[195101]: failed to encode map e664 with expected crc 2023-10-09T06:21:02.948 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:02 smithi156 ceph-mon[195101]: osdmap e665: 8 total, 8 up, 6 in 2023-10-09T06:21:02.993 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 100% complete...done. 2023-10-09T06:21:02.996 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd remove testimg_import 2023-10-09T06:21:03.068 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:21:03.071 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:02 smithi064 ceph-mon[212750]: pgmap v1697: 103 pgs: 103 active+clean; 802 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 208 KiB/s rd, 31 KiB/s wr, 287 op/s 2023-10-09T06:21:03.071 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:02 smithi064 ceph-mon[212750]: failed to encode map e664 with expected crc 2023-10-09T06:21:03.071 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:02 smithi064 ceph-mon[212750]: osdmap e664: 8 total, 8 up, 6 in 2023-10-09T06:21:03.071 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:02 smithi064 ceph-mon[212750]: failed to encode map e664 with expected crc 2023-10-09T06:21:03.072 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:02 smithi064 ceph-mon[212750]: osdmap e665: 8 total, 8 up, 6 in 2023-10-09T06:21:03.072 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:03 smithi064 conmon[212726]: 2023-10-09T06:21:03.023+0000 7f8823f23700 -1 mon.a@0(leader).osd e665 definitely_dead 0 2023-10-09T06:21:03.072 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:02 smithi064 ceph-mon[220044]: pgmap v1697: 103 pgs: 103 active+clean; 802 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 208 KiB/s rd, 31 KiB/s wr, 287 op/s 2023-10-09T06:21:03.072 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:02 smithi064 ceph-mon[220044]: failed to encode map e664 with expected crc 2023-10-09T06:21:03.072 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:02 smithi064 ceph-mon[220044]: osdmap e664: 8 total, 8 up, 6 in 2023-10-09T06:21:03.072 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:02 smithi064 ceph-mon[220044]: failed to encode map e664 with expected crc 2023-10-09T06:21:03.072 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:02 smithi064 ceph-mon[220044]: osdmap e665: 8 total, 8 up, 6 in 2023-10-09T06:21:03.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:03.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:03.090 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 100% complete...done. 2023-10-09T06:21:03.094 INFO:tasks.workunit.client.0.smithi064.stderr:+ tiered=0 2023-10-09T06:21:03.094 INFO:tasks.workunit.client.0.smithi064.stderr:+ ceph osd dump 2023-10-09T06:21:03.094 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep '^pool' 2023-10-09T06:21:03.094 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep ''\''rbd'\''' 2023-10-09T06:21:03.094 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep tier 2023-10-09T06:21:03.174 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-10-09T06:21:03.174 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-10-09T06:21:03.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-10-09T06:21:03.174 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-10-09T06:21:03.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.174 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:83031 10.1 e514/490 10:8db48241:::benchmark_data_smithi064_573750_object83030:head v 514'5214, mlcod=514'5211)", 2023-10-09T06:21:03.174 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.829326+0000", 2023-10-09T06:21:03.174 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 481.34274348299999, 2023-10-09T06:21:03.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.310894585, 2023-10-09T06:21:03.175 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:21:03.175 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:03.175 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:21:03.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:21:03.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.829326+0000", 2023-10-09T06:21:03.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:21:03.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.829326+0000", 2023-10-09T06:21:03.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:21:03.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.829325+0000", 2023-10-09T06:21:03.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:21:03.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:21:03.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.829372+0000", 2023-10-09T06:21:03.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6585999999999997e-05 2023-10-09T06:21:03.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:21:03.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.829381+0000", 2023-10-09T06:21:03.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7739999999999999e-06 2023-10-09T06:21:03.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:21:03.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.829388+0000", 2023-10-09T06:21:03.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2870000000000001e-06 2023-10-09T06:21:03.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:21:03.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832680+0000", 2023-10-09T06:21:03.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003291913 2023-10-09T06:21:03.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:21:03.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832702+0000", 2023-10-09T06:21:03.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2847999999999999e-05 2023-10-09T06:21:03.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:21:03.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835347+0000", 2023-10-09T06:21:03.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026447810000000001 2023-10-09T06:21:03.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:21:03.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140221+0000", 2023-10-09T06:21:03.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30487336999999998 2023-10-09T06:21:03.181 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.181 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:21:03.181 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.181 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:83032 10.6 10:60352a27:::benchmark_data_smithi064_573750_object83031:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e514)", 2023-10-09T06:21:03.181 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.829973+0000", 2023-10-09T06:21:03.182 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 481.34209598400003, 2023-10-09T06:21:03.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.310193414, 2023-10-09T06:21:03.182 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:21:03.182 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:03.182 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:21:03.182 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:21:03.182 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:21:03.182 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 83032 2023-10-09T06:21:03.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.182 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:21:03.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:21:03.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.829973+0000", 2023-10-09T06:21:03.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:21:03.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.829973+0000", 2023-10-09T06:21:03.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:21:03.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.829972+0000", 2023-10-09T06:21:03.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:21:03.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:21:03.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.829991+0000", 2023-10-09T06:21:03.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9579e-05 2023-10-09T06:21:03.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:21:03.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.830001+0000", 2023-10-09T06:21:03.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5210000000000008e-06 2023-10-09T06:21:03.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:21:03.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.830007+0000", 2023-10-09T06:21:03.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3640000000000004e-06 2023-10-09T06:21:03.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:21:03.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.830079+0000", 2023-10-09T06:21:03.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2119000000000005e-05 2023-10-09T06:21:03.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:21:03.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.830202+0000", 2023-10-09T06:21:03.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012245100000000001 2023-10-09T06:21:03.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:21:03.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.830327+0000", 2023-10-09T06:21:03.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000125127 2023-10-09T06:21:03.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:21:03.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833007+0000", 2023-10-09T06:21:03.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026800029999999998 2023-10-09T06:21:03.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140016+0000", 2023-10-09T06:21:03.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30700909700000001 2023-10-09T06:21:03.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140075+0000", 2023-10-09T06:21:03.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9342999999999997e-05 2023-10-09T06:21:03.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:21:03.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140125+0000", 2023-10-09T06:21:03.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0037000000000002e-05 2023-10-09T06:21:03.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:21:03.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140167+0000", 2023-10-09T06:21:03.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1477999999999998e-05 2023-10-09T06:21:03.189 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.189 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:21:03.189 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.189 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:83033 10.2 e514/490 10:4f3784a2:::benchmark_data_smithi064_573750_object83032:head v 514'5188, mlcod=513'5184)", 2023-10-09T06:21:03.189 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.832428+0000", 2023-10-09T06:21:03.189 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 481.33964135399998, 2023-10-09T06:21:03.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.31206429899999999, 2023-10-09T06:21:03.190 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:21:03.190 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:03.190 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:21:03.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:21:03.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832428+0000", 2023-10-09T06:21:03.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.190 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:21:03.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832428+0000", 2023-10-09T06:21:03.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:21:03.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832427+0000", 2023-10-09T06:21:03.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:03.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:21:03.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832452+0000", 2023-10-09T06:21:03.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4193e-05 2023-10-09T06:21:03.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:21:03.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832462+0000", 2023-10-09T06:21:03.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0392e-05 2023-10-09T06:21:03.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:21:03.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832470+0000", 2023-10-09T06:21:03.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.61e-06 2023-10-09T06:21:03.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:21:03.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832509+0000", 2023-10-09T06:21:03.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9552999999999999e-05 2023-10-09T06:21:03.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:21:03.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.832530+0000", 2023-10-09T06:21:03.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0924000000000002e-05 2023-10-09T06:21:03.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:21:03.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.144443+0000", 2023-10-09T06:21:03.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.31191265200000001 2023-10-09T06:21:03.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:21:03.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.144492+0000", 2023-10-09T06:21:03.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9591e-05 2023-10-09T06:21:03.194 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.194 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:21:03.194 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.195 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:83037 10.5 10:ab60acc0:::benchmark_data_smithi064_573750_object83036:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e514)", 2023-10-09T06:21:03.195 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.833694+0000", 2023-10-09T06:21:03.195 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 481.33837544699998, 2023-10-09T06:21:03.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30705259600000001, 2023-10-09T06:21:03.195 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:21:03.195 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:03.195 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:21:03.195 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:21:03.195 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:21:03.196 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 83037 2023-10-09T06:21:03.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.196 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:21:03.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:21:03.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833694+0000", 2023-10-09T06:21:03.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:21:03.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833694+0000", 2023-10-09T06:21:03.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:21:03.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833692+0000", 2023-10-09T06:21:03.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-10-09T06:21:03.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:21:03.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833721+0000", 2023-10-09T06:21:03.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9408999999999999e-05 2023-10-09T06:21:03.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:21:03.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833737+0000", 2023-10-09T06:21:03.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5554e-05 2023-10-09T06:21:03.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:21:03.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833746+0000", 2023-10-09T06:21:03.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0620000000000007e-06 2023-10-09T06:21:03.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:21:03.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833815+0000", 2023-10-09T06:21:03.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9517999999999994e-05 2023-10-09T06:21:03.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:21:03.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833944+0000", 2023-10-09T06:21:03.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012879100000000001 2023-10-09T06:21:03.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:21:03.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834108+0000", 2023-10-09T06:21:03.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000164346 2023-10-09T06:21:03.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.137909+0000", 2023-10-09T06:21:03.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.303800554 2023-10-09T06:21:03.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:21:03.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.137917+0000", 2023-10-09T06:21:03.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0900000000000005e-06 2023-10-09T06:21:03.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140650+0000", 2023-10-09T06:21:03.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027334120000000002 2023-10-09T06:21:03.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:21:03.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140707+0000", 2023-10-09T06:21:03.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6308999999999999e-05 2023-10-09T06:21:03.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:21:03.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140747+0000", 2023-10-09T06:21:03.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9807e-05 2023-10-09T06:21:03.202 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.202 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:21:03.202 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.203 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:83039 10.6 10:6ebb2182:::benchmark_data_smithi064_573750_object83038:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e514)", 2023-10-09T06:21:03.203 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.833781+0000", 2023-10-09T06:21:03.203 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 481.33828809599999, 2023-10-09T06:21:03.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30797565900000001, 2023-10-09T06:21:03.203 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:21:03.203 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:03.203 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:21:03.203 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:21:03.203 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:21:03.203 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 83039 2023-10-09T06:21:03.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.204 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:21:03.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:21:03.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833781+0000", 2023-10-09T06:21:03.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:21:03.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833781+0000", 2023-10-09T06:21:03.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.205 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:21:03.205 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833781+0000", 2023-10-09T06:21:03.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:03.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.205 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:21:03.205 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833803+0000", 2023-10-09T06:21:03.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1861e-05 2023-10-09T06:21:03.206 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.206 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:21:03.206 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833812+0000", 2023-10-09T06:21:03.206 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6080000000000002e-06 2023-10-09T06:21:03.206 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.206 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:21:03.206 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833819+0000", 2023-10-09T06:21:03.206 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9999999999999999e-06 2023-10-09T06:21:03.206 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.207 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:21:03.207 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833905+0000", 2023-10-09T06:21:03.207 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5968000000000001e-05 2023-10-09T06:21:03.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.207 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:21:03.207 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834028+0000", 2023-10-09T06:21:03.207 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012332900000000001 2023-10-09T06:21:03.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.207 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:21:03.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834128+0000", 2023-10-09T06:21:03.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9903999999999998e-05 2023-10-09T06:21:03.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140047+0000", 2023-10-09T06:21:03.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30591839300000001 2023-10-09T06:21:03.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.141647+0000", 2023-10-09T06:21:03.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016003650000000001 2023-10-09T06:21:03.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:21:03.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.141663+0000", 2023-10-09T06:21:03.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6113e-05 2023-10-09T06:21:03.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:21:03.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.141712+0000", 2023-10-09T06:21:03.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8925000000000003e-05 2023-10-09T06:21:03.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.210 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:21:03.210 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.141757+0000", 2023-10-09T06:21:03.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4807e-05 2023-10-09T06:21:03.210 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.210 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:21:03.210 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.210 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:83040 10.a 10:5efad431:::benchmark_data_smithi064_573750_object83039:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e514)", 2023-10-09T06:21:03.210 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.833834+0000", 2023-10-09T06:21:03.211 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 481.33823517600001, 2023-10-09T06:21:03.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32167670500000001, 2023-10-09T06:21:03.211 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:21:03.211 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:03.211 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:21:03.211 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:21:03.211 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:21:03.211 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 83040 2023-10-09T06:21:03.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.211 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:21:03.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:21:03.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833834+0000", 2023-10-09T06:21:03.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:21:03.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833834+0000", 2023-10-09T06:21:03.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:21:03.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833834+0000", 2023-10-09T06:21:03.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:03.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:21:03.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833855+0000", 2023-10-09T06:21:03.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1135999999999999e-05 2023-10-09T06:21:03.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:21:03.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833862+0000", 2023-10-09T06:21:03.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.481e-06 2023-10-09T06:21:03.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.214 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:21:03.214 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.833868+0000", 2023-10-09T06:21:03.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5090000000000001e-06 2023-10-09T06:21:03.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.214 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:21:03.214 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835333+0000", 2023-10-09T06:21:03.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001465093 2023-10-09T06:21:03.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:21:03.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.137972+0000", 2023-10-09T06:21:03.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30263898299999997 2023-10-09T06:21:03.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:21:03.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.138144+0000", 2023-10-09T06:21:03.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017225399999999999 2023-10-09T06:21:03.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.142232+0000", 2023-10-09T06:21:03.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.004087355 2023-10-09T06:21:03.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.142266+0000", 2023-10-09T06:21:03.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4878e-05 2023-10-09T06:21:03.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:21:03.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.155431+0000", 2023-10-09T06:21:03.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013164821 2023-10-09T06:21:03.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:21:03.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.155475+0000", 2023-10-09T06:21:03.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4078e-05 2023-10-09T06:21:03.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:21:03.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.155511+0000", 2023-10-09T06:21:03.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5658000000000003e-05 2023-10-09T06:21:03.218 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.218 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:21:03.218 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.218 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:83034 10.2 e514/490 10:4840dafe:::benchmark_data_smithi064_573750_object83033:head v 514'5189, mlcod=514'5185)", 2023-10-09T06:21:03.218 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.834233+0000", 2023-10-09T06:21:03.218 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 481.33783659900001, 2023-10-09T06:21:03.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.31657803899999998, 2023-10-09T06:21:03.218 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:21:03.219 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:03.219 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:21:03.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:21:03.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834233+0000", 2023-10-09T06:21:03.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:21:03.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834233+0000", 2023-10-09T06:21:03.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.220 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:21:03.220 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834232+0000", 2023-10-09T06:21:03.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:03.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.220 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:21:03.220 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834257+0000", 2023-10-09T06:21:03.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4641e-05 2023-10-09T06:21:03.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.220 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:21:03.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834264+0000", 2023-10-09T06:21:03.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.554e-06 2023-10-09T06:21:03.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:21:03.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.834270+0000", 2023-10-09T06:21:03.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.242e-06 2023-10-09T06:21:03.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:21:03.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835340+0000", 2023-10-09T06:21:03.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010699220000000001 2023-10-09T06:21:03.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:21:03.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.137949+0000", 2023-10-09T06:21:03.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.302608936 2023-10-09T06:21:03.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:21:03.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.150776+0000", 2023-10-09T06:21:03.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012827191999999999 2023-10-09T06:21:03.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:21:03.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.150811+0000", 2023-10-09T06:21:03.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5076000000000001e-05 2023-10-09T06:21:03.223 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.223 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:21:03.223 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.223 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:83036 10.4 e514/490 10:22f39e74:::benchmark_data_smithi064_573750_object83035:head v 514'5125, mlcod=513'5123)", 2023-10-09T06:21:03.224 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.835143+0000", 2023-10-09T06:21:03.224 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 481.33692628199998, 2023-10-09T06:21:03.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30683402399999998, 2023-10-09T06:21:03.224 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:21:03.224 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:03.224 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:21:03.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:21:03.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835143+0000", 2023-10-09T06:21:03.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.224 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:21:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835143+0000", 2023-10-09T06:21:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:21:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835143+0000", 2023-10-09T06:21:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:21:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835177+0000", 2023-10-09T06:21:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4174000000000002e-05 2023-10-09T06:21:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:21:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835188+0000", 2023-10-09T06:21:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0804e-05 2023-10-09T06:21:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:21:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835195+0000", 2023-10-09T06:21:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1239999999999999e-06 2023-10-09T06:21:03.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.227 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:21:03.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835246+0000", 2023-10-09T06:21:03.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1143999999999997e-05 2023-10-09T06:21:03.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.227 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:21:03.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835268+0000", 2023-10-09T06:21:03.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1865000000000002e-05 2023-10-09T06:21:03.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:21:03.228 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.141950+0000", 2023-10-09T06:21:03.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30668228400000003 2023-10-09T06:21:03.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:21:03.228 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.141977+0000", 2023-10-09T06:21:03.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7175e-05 2023-10-09T06:21:03.228 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.228 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:21:03.228 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.229 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.229 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.229 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.46283.0:83032 10.6 e514/490 ondisk, result = 0)", 2023-10-09T06:21:03.229 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.835214+0000", 2023-10-09T06:21:03.229 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 481.33685544100001, 2023-10-09T06:21:03.229 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30480471100000001, 2023-10-09T06:21:03.229 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:21:03.229 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:21:03.229 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:21:03.229 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.229 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:21:03.229 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835214+0000", 2023-10-09T06:21:03.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:21:03.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835214+0000", 2023-10-09T06:21:03.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:21:03.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835214+0000", 2023-10-09T06:21:03.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:03.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:21:03.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835217+0000", 2023-10-09T06:21:03.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3239999999999999e-06 2023-10-09T06:21:03.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:21:03.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835224+0000", 2023-10-09T06:21:03.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7859999999999999e-06 2023-10-09T06:21:03.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.232 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.232 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:21:03.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.835229+0000", 2023-10-09T06:21:03.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3519999999999997e-06 2023-10-09T06:21:03.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.232 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.232 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:21:03.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140002+0000", 2023-10-09T06:21:03.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30477259499999998 2023-10-09T06:21:03.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.233 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:21:03.233 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140010+0000", 2023-10-09T06:21:03.233 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1359999999999997e-06 2023-10-09T06:21:03.233 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.233 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:21:03.233 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140019+0000", 2023-10-09T06:21:03.233 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8799999999999997e-06 2023-10-09T06:21:03.233 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.233 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:21:03.234 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.234 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.234 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.46283.0:83032 10.6 e514/490 ondisk, result = 0)", 2023-10-09T06:21:03.234 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.837038+0000", 2023-10-09T06:21:03.234 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 481.335031431, 2023-10-09T06:21:03.234 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30313466700000002, 2023-10-09T06:21:03.234 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:21:03.234 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:21:03.234 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:21:03.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.235 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:21:03.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.837038+0000", 2023-10-09T06:21:03.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.235 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:21:03.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.837038+0000", 2023-10-09T06:21:03.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.235 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:21:03.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.837037+0000", 2023-10-09T06:21:03.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:03.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:21:03.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.837042+0000", 2023-10-09T06:21:03.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1970000000000004e-06 2023-10-09T06:21:03.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:21:03.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.138000+0000", 2023-10-09T06:21:03.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.300957961 2023-10-09T06:21:03.237 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.237 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.237 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:21:03.237 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.138006+0000", 2023-10-09T06:21:03.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8449999999999999e-06 2023-10-09T06:21:03.237 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.237 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.237 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:21:03.237 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140066+0000", 2023-10-09T06:21:03.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020593970000000001 2023-10-09T06:21:03.238 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.238 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.238 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:21:03.238 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140071+0000", 2023-10-09T06:21:03.238 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4020000000000001e-06 2023-10-09T06:21:03.238 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.238 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.238 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:21:03.238 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140173+0000", 2023-10-09T06:21:03.238 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101576 2023-10-09T06:21:03.238 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.238 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:21:03.239 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.239 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.239 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.239 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.46283.0:83039 10.6 e514/490 ondisk, result = 0)", 2023-10-09T06:21:03.239 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:01.837813+0000", 2023-10-09T06:21:03.239 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 481.33425626500002, 2023-10-09T06:21:03.239 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.302235578, 2023-10-09T06:21:03.239 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:21:03.239 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-10-09T06:21:03.239 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:21:03.239 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.239 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:21:03.240 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.837813+0000", 2023-10-09T06:21:03.240 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.240 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.240 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.240 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:21:03.240 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.837813+0000", 2023-10-09T06:21:03.240 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.240 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.240 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.240 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:21:03.240 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.837813+0000", 2023-10-09T06:21:03.241 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:03.241 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.241 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.241 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:21:03.241 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:01.837818+0000", 2023-10-09T06:21:03.241 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0440000000000003e-06 2023-10-09T06:21:03.241 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.241 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.241 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:21:03.241 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.137814+0000", 2023-10-09T06:21:03.242 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.299996771 2023-10-09T06:21:03.242 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.242 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.242 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:21:03.242 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.137832+0000", 2023-10-09T06:21:03.242 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7198999999999999e-05 2023-10-09T06:21:03.242 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.242 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.242 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:21:03.242 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140037+0000", 2023-10-09T06:21:03.242 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0022049740000000002 2023-10-09T06:21:03.242 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.243 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.243 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:21:03.243 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140042+0000", 2023-10-09T06:21:03.243 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7189999999999998e-06 2023-10-09T06:21:03.243 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.243 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.243 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:21:03.243 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:02.140049+0000", 2023-10-09T06:21:03.243 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5010000000000001e-06 2023-10-09T06:21:03.243 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.243 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:21:03.243 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.244 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.244 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.244 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:117969 10.c 10:31f340c0:::benchmark_data_smithi064_573750_object21754:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e536)", 2023-10-09T06:21:03.244 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:59.578658+0000", 2023-10-09T06:21:03.244 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 423.59341146000003, 2023-10-09T06:21:03.244 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.3295903, 2023-10-09T06:21:03.244 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:21:03.244 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:03.244 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:21:03.244 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:21:03.244 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:21:03.244 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 117969 2023-10-09T06:21:03.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.245 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:21:03.245 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.245 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:21:03.245 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.578658+0000", 2023-10-09T06:21:03.245 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.245 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.245 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:21:03.245 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.578658+0000", 2023-10-09T06:21:03.245 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.246 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.246 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:21:03.246 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.578657+0000", 2023-10-09T06:21:03.246 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-10-09T06:21:03.246 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.246 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:21:03.246 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.578662+0000", 2023-10-09T06:21:03.246 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.101e-06 2023-10-09T06:21:03.246 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.247 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.247 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:21:03.247 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.795223+0000", 2023-10-09T06:21:03.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21656120300000001 2023-10-09T06:21:03.247 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.247 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.247 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:21:03.247 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.795235+0000", 2023-10-09T06:21:03.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1979e-05 2023-10-09T06:21:03.247 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.247 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.247 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:21:03.248 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.795411+0000", 2023-10-09T06:21:03.248 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000175599 2023-10-09T06:21:03.248 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.248 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.248 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:21:03.248 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.795540+0000", 2023-10-09T06:21:03.248 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012918599999999999 2023-10-09T06:21:03.248 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.248 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.248 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4,7", 2023-10-09T06:21:03.248 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.795671+0000", 2023-10-09T06:21:03.248 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000130681 2023-10-09T06:21:03.249 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.249 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.249 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:21:03.249 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.799483+0000", 2023-10-09T06:21:03.249 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038128010000000002 2023-10-09T06:21:03.249 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.249 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.249 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.249 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.907634+0000", 2023-10-09T06:21:03.249 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.108151071 2023-10-09T06:21:03.250 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.250 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.250 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.250 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.907752+0000", 2023-10-09T06:21:03.250 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011757 2023-10-09T06:21:03.250 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.250 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.250 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.250 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.908098+0000", 2023-10-09T06:21:03.250 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000346378 2023-10-09T06:21:03.251 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.251 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.251 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:21:03.251 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.908142+0000", 2023-10-09T06:21:03.251 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3297999999999998e-05 2023-10-09T06:21:03.251 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.251 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.251 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:21:03.251 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.908248+0000", 2023-10-09T06:21:03.251 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010660800000000001 2023-10-09T06:21:03.251 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.252 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:21:03.252 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.252 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.252 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.252 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46283.0:117972 10.d e536/522 10:b67d4907:::benchmark_data_smithi064_573750_object21757:head v 536'7359, mlcod=536'7355)", 2023-10-09T06:21:03.252 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:13:59.578779+0000", 2023-10-09T06:21:03.252 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 423.593290782, 2023-10-09T06:21:03.252 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33589861999999998, 2023-10-09T06:21:03.252 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:21:03.252 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:03.252 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:21:03.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.253 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:21:03.253 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.578779+0000", 2023-10-09T06:21:03.253 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.253 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:21:03.253 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.578779+0000", 2023-10-09T06:21:03.253 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:21:03.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.578778+0000", 2023-10-09T06:21:03.254 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:03.254 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.254 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:21:03.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.578786+0000", 2023-10-09T06:21:03.254 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9000000000000006e-06 2023-10-09T06:21:03.254 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.254 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.255 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:21:03.255 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.795187+0000", 2023-10-09T06:21:03.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21640057800000001 2023-10-09T06:21:03.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.255 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:21:03.255 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.795679+0000", 2023-10-09T06:21:03.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00049279300000000005 2023-10-09T06:21:03.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.256 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:21:03.256 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.796083+0000", 2023-10-09T06:21:03.256 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00040412299999999999 2023-10-09T06:21:03.256 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.256 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.256 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:21:03.256 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.796107+0000", 2023-10-09T06:21:03.256 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3122e-05 2023-10-09T06:21:03.256 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.256 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.256 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:21:03.256 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.914650+0000", 2023-10-09T06:21:03.257 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.118543438 2023-10-09T06:21:03.257 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.257 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.257 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:21:03.257 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:13:59.914677+0000", 2023-10-09T06:21:03.257 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7251e-05 2023-10-09T06:21:03.257 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.257 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:21:03.257 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.257 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.257 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.257 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:149281 10.6 10:61516434:::benchmark_data_smithi064_573750_object53066:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e550)", 2023-10-09T06:21:03.258 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:47.178748+0000", 2023-10-09T06:21:03.258 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 375.99332133600001, 2023-10-09T06:21:03.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.52331555600000002, 2023-10-09T06:21:03.258 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:21:03.258 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:03.258 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:21:03.258 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:21:03.258 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:21:03.258 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149281 2023-10-09T06:21:03.258 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.258 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:21:03.259 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.259 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:21:03.259 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.178748+0000", 2023-10-09T06:21:03.259 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.259 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.259 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.259 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:21:03.259 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.178748+0000", 2023-10-09T06:21:03.259 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.259 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.259 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.260 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:21:03.260 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.178747+0000", 2023-10-09T06:21:03.260 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:03.260 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.260 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.260 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:21:03.260 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.178752+0000", 2023-10-09T06:21:03.260 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.933e-06 2023-10-09T06:21:03.260 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.260 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.260 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:21:03.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.178764+0000", 2023-10-09T06:21:03.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2456999999999999e-05 2023-10-09T06:21:03.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.261 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.261 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:21:03.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.178771+0000", 2023-10-09T06:21:03.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8990000000000004e-06 2023-10-09T06:21:03.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.261 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.261 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:21:03.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.178818+0000", 2023-10-09T06:21:03.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6588999999999997e-05 2023-10-09T06:21:03.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:21:03.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.178961+0000", 2023-10-09T06:21:03.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000143387 2023-10-09T06:21:03.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:21:03.263 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.179114+0000", 2023-10-09T06:21:03.263 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015322099999999999 2023-10-09T06:21:03.263 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.263 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.263 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.263 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.190948+0000", 2023-10-09T06:21:03.263 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011833532000000001 2023-10-09T06:21:03.263 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.263 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.263 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:21:03.264 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.190956+0000", 2023-10-09T06:21:03.264 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9189999999999999e-06 2023-10-09T06:21:03.264 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.264 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.264 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.264 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.701981+0000", 2023-10-09T06:21:03.264 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.51102548599999997 2023-10-09T06:21:03.264 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.264 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.264 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:21:03.264 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.702025+0000", 2023-10-09T06:21:03.265 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3761999999999999e-05 2023-10-09T06:21:03.265 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.265 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.265 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:21:03.265 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.702064+0000", 2023-10-09T06:21:03.265 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8792999999999999e-05 2023-10-09T06:21:03.265 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.265 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:21:03.265 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.265 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.265 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.266 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:149284 10.a 10:5302fd94:::benchmark_data_smithi064_573750_object53069:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e550)", 2023-10-09T06:21:03.266 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:47.182910+0000", 2023-10-09T06:21:03.266 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 375.98915955299998, 2023-10-09T06:21:03.266 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.51486690099999999, 2023-10-09T06:21:03.266 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:21:03.266 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:03.266 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:21:03.266 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:21:03.266 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:21:03.266 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149284 2023-10-09T06:21:03.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.267 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:21:03.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:21:03.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.182910+0000", 2023-10-09T06:21:03.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:21:03.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.182910+0000", 2023-10-09T06:21:03.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.268 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.268 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:21:03.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.182908+0000", 2023-10-09T06:21:03.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:03.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.268 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.268 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:21:03.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.182919+0000", 2023-10-09T06:21:03.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0054999999999999e-05 2023-10-09T06:21:03.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:21:03.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.182931+0000", 2023-10-09T06:21:03.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2755000000000001e-05 2023-10-09T06:21:03.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:21:03.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.182942+0000", 2023-10-09T06:21:03.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1113e-05 2023-10-09T06:21:03.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:21:03.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.183001+0000", 2023-10-09T06:21:03.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8242999999999998e-05 2023-10-09T06:21:03.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:21:03.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.183149+0000", 2023-10-09T06:21:03.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014812700000000001 2023-10-09T06:21:03.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:21:03.271 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.183280+0000", 2023-10-09T06:21:03.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013154799999999999 2023-10-09T06:21:03.271 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.271 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.271 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.190818+0000", 2023-10-09T06:21:03.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0075373949999999997 2023-10-09T06:21:03.271 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.271 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:21:03.271 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.191769+0000", 2023-10-09T06:21:03.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00095116099999999996 2023-10-09T06:21:03.272 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.272 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.272 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.272 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.697692+0000", 2023-10-09T06:21:03.272 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.505923558 2023-10-09T06:21:03.272 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.272 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.272 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:21:03.272 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.697730+0000", 2023-10-09T06:21:03.272 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7725999999999998e-05 2023-10-09T06:21:03.272 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.272 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.273 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:21:03.273 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.697777+0000", 2023-10-09T06:21:03.273 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6589999999999999e-05 2023-10-09T06:21:03.273 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.273 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:21:03.273 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.273 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.273 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.273 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:149285 10.a 10:5b3d29d3:::benchmark_data_smithi064_573750_object53070:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e550)", 2023-10-09T06:21:03.273 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:47.182959+0000", 2023-10-09T06:21:03.273 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 375.98911019299999, 2023-10-09T06:21:03.273 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.51908162700000005, 2023-10-09T06:21:03.273 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:21:03.274 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:03.274 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:21:03.274 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:21:03.274 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:21:03.274 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149285 2023-10-09T06:21:03.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.274 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:21:03.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.274 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:21:03.274 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.182959+0000", 2023-10-09T06:21:03.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:21:03.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.182959+0000", 2023-10-09T06:21:03.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:21:03.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.182959+0000", 2023-10-09T06:21:03.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:03.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:21:03.276 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.182962+0000", 2023-10-09T06:21:03.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1590000000000002e-06 2023-10-09T06:21:03.276 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:21:03.276 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.182970+0000", 2023-10-09T06:21:03.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2849999999999995e-06 2023-10-09T06:21:03.276 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.277 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:21:03.277 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.182976+0000", 2023-10-09T06:21:03.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7640000000000002e-06 2023-10-09T06:21:03.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.277 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:21:03.277 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.186933+0000", 2023-10-09T06:21:03.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039569169999999999 2023-10-09T06:21:03.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.277 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:21:03.277 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.187059+0000", 2023-10-09T06:21:03.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012567899999999999 2023-10-09T06:21:03.278 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.278 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:21:03.278 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.187183+0000", 2023-10-09T06:21:03.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012484799999999999 2023-10-09T06:21:03.278 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.278 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.278 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.190863+0000", 2023-10-09T06:21:03.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0036794879999999999 2023-10-09T06:21:03.278 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:21:03.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.195731+0000", 2023-10-09T06:21:03.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048682120000000002 2023-10-09T06:21:03.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.701948+0000", 2023-10-09T06:21:03.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.50621636000000003 2023-10-09T06:21:03.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:21:03.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.701995+0000", 2023-10-09T06:21:03.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7537000000000002e-05 2023-10-09T06:21:03.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:21:03.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.702041+0000", 2023-10-09T06:21:03.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5812000000000001e-05 2023-10-09T06:21:03.280 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.280 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:21:03.280 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.280 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:149305 10.6 10:6e58fed9:::benchmark_data_smithi064_573750_object53090:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e550)", 2023-10-09T06:21:03.281 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:47.217073+0000", 2023-10-09T06:21:03.281 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 375.954995962, 2023-10-09T06:21:03.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.485167132, 2023-10-09T06:21:03.281 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:21:03.281 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:03.281 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:21:03.281 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:21:03.281 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:21:03.281 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149305 2023-10-09T06:21:03.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.281 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:21:03.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:21:03.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.217073+0000", 2023-10-09T06:21:03.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:21:03.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.217073+0000", 2023-10-09T06:21:03.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:21:03.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.217060+0000", 2023-10-09T06:21:03.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999862 2023-10-09T06:21:03.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:21:03.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.217081+0000", 2023-10-09T06:21:03.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0916000000000001e-05 2023-10-09T06:21:03.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:21:03.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.217096+0000", 2023-10-09T06:21:03.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5381000000000001e-05 2023-10-09T06:21:03.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:21:03.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.217106+0000", 2023-10-09T06:21:03.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0044000000000001e-05 2023-10-09T06:21:03.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:21:03.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.217167+0000", 2023-10-09T06:21:03.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1054000000000002e-05 2023-10-09T06:21:03.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:21:03.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.217309+0000", 2023-10-09T06:21:03.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014138000000000001 2023-10-09T06:21:03.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:21:03.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.217451+0000", 2023-10-09T06:21:03.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014241899999999999 2023-10-09T06:21:03.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:21:03.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.221048+0000", 2023-10-09T06:21:03.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035971169999999999 2023-10-09T06:21:03.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.223777+0000", 2023-10-09T06:21:03.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027287029999999999 2023-10-09T06:21:03.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.702136+0000", 2023-10-09T06:21:03.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.47835958699999998 2023-10-09T06:21:03.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:21:03.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.702187+0000", 2023-10-09T06:21:03.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0717999999999998e-05 2023-10-09T06:21:03.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.287 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:21:03.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.702241+0000", 2023-10-09T06:21:03.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3541000000000001e-05 2023-10-09T06:21:03.287 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.287 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:21:03.287 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.287 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:149306 10.a 10:57b3c501:::benchmark_data_smithi064_573750_object53091:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e550)", 2023-10-09T06:21:03.288 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:47.221272+0000", 2023-10-09T06:21:03.288 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 375.950797384, 2023-10-09T06:21:03.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.48510954299999998, 2023-10-09T06:21:03.288 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:21:03.288 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:03.288 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:21:03.288 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:21:03.288 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:21:03.288 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149306 2023-10-09T06:21:03.288 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.288 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:21:03.288 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.288 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:21:03.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.221272+0000", 2023-10-09T06:21:03.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:21:03.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.221272+0000", 2023-10-09T06:21:03.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:21:03.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.221270+0000", 2023-10-09T06:21:03.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-10-09T06:21:03.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:21:03.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.221278+0000", 2023-10-09T06:21:03.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7079999999999997e-06 2023-10-09T06:21:03.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:21:03.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.221294+0000", 2023-10-09T06:21:03.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6487e-05 2023-10-09T06:21:03.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.291 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:21:03.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.221306+0000", 2023-10-09T06:21:03.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1522000000000001e-05 2023-10-09T06:21:03.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.291 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:21:03.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.221362+0000", 2023-10-09T06:21:03.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6122000000000002e-05 2023-10-09T06:21:03.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:21:03.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.221498+0000", 2023-10-09T06:21:03.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013583599999999999 2023-10-09T06:21:03.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:21:03.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.221641+0000", 2023-10-09T06:21:03.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014281799999999999 2023-10-09T06:21:03.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.225057+0000", 2023-10-09T06:21:03.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034167659999999999 2023-10-09T06:21:03.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:21:03.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.225187+0000", 2023-10-09T06:21:03.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012954899999999999 2023-10-09T06:21:03.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.706283+0000", 2023-10-09T06:21:03.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.48109601400000002 2023-10-09T06:21:03.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:21:03.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.706340+0000", 2023-10-09T06:21:03.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7290999999999997e-05 2023-10-09T06:21:03.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:21:03.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.706382+0000", 2023-10-09T06:21:03.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1409000000000003e-05 2023-10-09T06:21:03.294 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.295 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:21:03.295 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.295 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:149312 10.6 10:601aa7eb:::benchmark_data_smithi064_573750_object53097:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e550)", 2023-10-09T06:21:03.295 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:47.229107+0000", 2023-10-09T06:21:03.295 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 375.94296229999998, 2023-10-09T06:21:03.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.47326104800000002, 2023-10-09T06:21:03.295 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:21:03.295 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:03.295 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:21:03.296 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:21:03.296 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:21:03.296 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149312 2023-10-09T06:21:03.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.296 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:21:03.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:21:03.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.229107+0000", 2023-10-09T06:21:03.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:21:03.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.229107+0000", 2023-10-09T06:21:03.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:21:03.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.229106+0000", 2023-10-09T06:21:03.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:03.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:21:03.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.229114+0000", 2023-10-09T06:21:03.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0900000000000005e-06 2023-10-09T06:21:03.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:21:03.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.229128+0000", 2023-10-09T06:21:03.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4729000000000001e-05 2023-10-09T06:21:03.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:21:03.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.229138+0000", 2023-10-09T06:21:03.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3959999999999995e-06 2023-10-09T06:21:03.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:21:03.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.229200+0000", 2023-10-09T06:21:03.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2131000000000005e-05 2023-10-09T06:21:03.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:21:03.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.229360+0000", 2023-10-09T06:21:03.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000159937 2023-10-09T06:21:03.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:21:03.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.229510+0000", 2023-10-09T06:21:03.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015014999999999999 2023-10-09T06:21:03.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:21:03.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.233925+0000", 2023-10-09T06:21:03.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044149879999999999 2023-10-09T06:21:03.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.234917+0000", 2023-10-09T06:21:03.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0009924720000000001 2023-10-09T06:21:03.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.702286+0000", 2023-10-09T06:21:03.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.467368693 2023-10-09T06:21:03.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:21:03.302 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.702328+0000", 2023-10-09T06:21:03.302 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1792000000000002e-05 2023-10-09T06:21:03.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.302 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:21:03.302 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.702368+0000", 2023-10-09T06:21:03.302 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0249e-05 2023-10-09T06:21:03.302 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.302 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:21:03.302 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.303 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46283.0:149321 10.6 10:61199dae:::benchmark_data_smithi064_573750_object53106:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e550)", 2023-10-09T06:21:03.303 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-10-09T06:14:47.248777+0000", 2023-10-09T06:21:03.303 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 375.92329240100003, 2023-10-09T06:21:03.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.46306102300000002, 2023-10-09T06:21:03.303 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-10-09T06:21:03.303 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:03.303 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-10-09T06:21:03.303 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46283", 2023-10-09T06:21:03.303 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2193403684", 2023-10-09T06:21:03.303 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 149321 2023-10-09T06:21:03.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.304 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-10-09T06:21:03.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-10-09T06:21:03.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.248777+0000", 2023-10-09T06:21:03.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-10-09T06:21:03.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.248777+0000", 2023-10-09T06:21:03.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-10-09T06:21:03.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-10-09T06:21:03.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.248775+0000", 2023-10-09T06:21:03.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:03.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-10-09T06:21:03.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.248783+0000", 2023-10-09T06:21:03.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8979999999999996e-06 2023-10-09T06:21:03.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-10-09T06:21:03.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.248798+0000", 2023-10-09T06:21:03.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4795e-05 2023-10-09T06:21:03.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-10-09T06:21:03.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.248806+0000", 2023-10-09T06:21:03.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3180000000000007e-06 2023-10-09T06:21:03.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-10-09T06:21:03.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.248857+0000", 2023-10-09T06:21:03.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0485999999999997e-05 2023-10-09T06:21:03.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-10-09T06:21:03.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.248969+0000", 2023-10-09T06:21:03.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000112391 2023-10-09T06:21:03.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:21:03.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.249059+0000", 2023-10-09T06:21:03.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9460000000000001e-05 2023-10-09T06:21:03.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-10-09T06:21:03.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.251756+0000", 2023-10-09T06:21:03.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002697554 2023-10-09T06:21:03.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.252079+0000", 2023-10-09T06:21:03.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00032264400000000002 2023-10-09T06:21:03.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:03.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.711752+0000", 2023-10-09T06:21:03.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.45967268500000003 2023-10-09T06:21:03.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-10-09T06:21:03.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.711794+0000", 2023-10-09T06:21:03.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2370999999999998e-05 2023-10-09T06:21:03.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-10-09T06:21:03.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-10-09T06:21:03.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-10-09T06:21:03.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-10-09T06:14:47.711838+0000", 2023-10-09T06:21:03.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3999999999999999e-05 2023-10-09T06:21:03.310 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.310 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:21:03.310 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.310 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-10-09T06:21:03.310 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-10-09T06:21:03.310 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-10-09T06:21:03.311 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:03.339 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:03.959 INFO:tasks.workunit.client.0.smithi064.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_695299/sparse1 2023-10-09T06:21:03.969 INFO:tasks.workunit.client.0.smithi064.stderr:1+0 records in 2023-10-09T06:21:03.969 INFO:tasks.workunit.client.0.smithi064.stderr:1+0 records out 2023-10-09T06:21:03.969 INFO:tasks.workunit.client.0.smithi064.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00692563 s, 151 MB/s 2023-10-09T06:21:03.969 INFO:tasks.workunit.client.0.smithi064.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_695299/sparse2 2023-10-09T06:21:03.974 INFO:teuthology.orchestra.run.smithi064.stderr:marked out osd.5. 2023-10-09T06:21:03.976 INFO:tasks.workunit.client.0.smithi064.stderr:1+0 records in 2023-10-09T06:21:03.976 INFO:tasks.workunit.client.0.smithi064.stderr:1+0 records out 2023-10-09T06:21:03.976 INFO:tasks.workunit.client.0.smithi064.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00632825 s, 166 MB/s 2023-10-09T06:21:03.977 INFO:tasks.workunit.client.0.smithi064.stderr:+ truncate /tmp/rbd_import_export_695299/sparse2 -s 2M 2023-10-09T06:21:04.123 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm sparse1 2023-10-09T06:21:04.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:03 smithi064 ceph-mon[212750]: failed to encode map e665 with expected crc 2023-10-09T06:21:04.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:03 smithi064 ceph-mon[212750]: failed to encode map e665 with expected crc 2023-10-09T06:21:04.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:03 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1520919876' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T06:21:04.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:03 smithi064 ceph-mon[212750]: Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T06:21:04.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:03 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/97313672' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:21:04.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:03 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/97313672' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:21:04.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:03 smithi064 ceph-mon[220044]: failed to encode map e665 with expected crc 2023-10-09T06:21:04.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:03 smithi064 ceph-mon[220044]: failed to encode map e665 with expected crc 2023-10-09T06:21:04.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:03 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1520919876' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T06:21:04.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:03 smithi064 ceph-mon[220044]: Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T06:21:04.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:03 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/97313672' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:21:04.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:03 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/97313672' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:21:04.305 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:21:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:03 smithi156 ceph-mon[195101]: failed to encode map e665 with expected crc 2023-10-09T06:21:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:03 smithi156 ceph-mon[195101]: failed to encode map e665 with expected crc 2023-10-09T06:21:04.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:03 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1520919876' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T06:21:04.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:03 smithi156 ceph-mon[195101]: Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T06:21:04.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:03 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/97313672' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:21:04.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:03 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/97313672' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:21:04.743 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:21:04.942 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:21:04.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:04.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:05.024 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 0% complete...failed. 2023-10-09T06:21:05.024 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T06:21:05.028 INFO:tasks.workunit.client.0.smithi064.stderr:+ true 2023-10-09T06:21:05.028 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_695299/sparse1 2023-10-09T06:21:05.046 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:05.047 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:04 smithi064 ceph-mon[220044]: pgmap v1700: 103 pgs: 103 active+clean; 802 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 208 KiB/s rd, 31 KiB/s wr, 287 op/s 2023-10-09T06:21:05.047 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:04 smithi064 ceph-mon[220044]: failed to encode map e666 with expected crc 2023-10-09T06:21:05.047 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:04 smithi064 ceph-mon[220044]: failed to encode map e666 with expected crc 2023-10-09T06:21:05.047 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:04 smithi064 ceph-mon[220044]: failed to encode map e666 with expected crc 2023-10-09T06:21:05.047 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:04 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1520919876' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T06:21:05.047 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:04 smithi064 ceph-mon[220044]: osdmap e666: 8 total, 8 up, 5 in 2023-10-09T06:21:05.048 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:04 smithi064 ceph-mon[220044]: failed to encode map e666 with expected crc 2023-10-09T06:21:05.048 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:04 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:21:05.048 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:04 smithi064 ceph-mon[212750]: pgmap v1700: 103 pgs: 103 active+clean; 802 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 208 KiB/s rd, 31 KiB/s wr, 287 op/s 2023-10-09T06:21:05.048 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:04 smithi064 ceph-mon[212750]: failed to encode map e666 with expected crc 2023-10-09T06:21:05.048 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:04 smithi064 ceph-mon[212750]: failed to encode map e666 with expected crc 2023-10-09T06:21:05.048 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:04 smithi064 ceph-mon[212750]: failed to encode map e666 with expected crc 2023-10-09T06:21:05.048 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:04 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1520919876' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T06:21:05.048 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:04 smithi064 ceph-mon[212750]: osdmap e666: 8 total, 8 up, 5 in 2023-10-09T06:21:05.049 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:04 smithi064 ceph-mon[212750]: failed to encode map e666 with expected crc 2023-10-09T06:21:05.049 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:04 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:21:05.082 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:05.114 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T06:21:05.115 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:21:05.115 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:21:05.119 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd ls -l 2023-10-09T06:21:05.119 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep sparse1 2023-10-09T06:21:05.119 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-10-09T06:21:05.126 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:04 smithi156 ceph-mon[195101]: pgmap v1700: 103 pgs: 103 active+clean; 802 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 208 KiB/s rd, 31 KiB/s wr, 287 op/s 2023-10-09T06:21:05.126 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:04 smithi156 ceph-mon[195101]: failed to encode map e666 with expected crc 2023-10-09T06:21:05.127 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:04 smithi156 ceph-mon[195101]: failed to encode map e666 with expected crc 2023-10-09T06:21:05.127 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:04 smithi156 ceph-mon[195101]: failed to encode map e666 with expected crc 2023-10-09T06:21:05.127 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:04 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1520919876' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T06:21:05.127 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:04 smithi156 ceph-mon[195101]: osdmap e666: 8 total, 8 up, 5 in 2023-10-09T06:21:05.127 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:04 smithi156 ceph-mon[195101]: failed to encode map e666 with expected crc 2023-10-09T06:21:05.127 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:04 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:21:05.143 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:21:05.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:05.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:05.166 INFO:tasks.workunit.client.0.smithi064.stdout:sparse1 2 MiB 2 2023-10-09T06:21:05.167 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects sparse1 2023-10-09T06:21:05.167 INFO:tasks.workunit.client.0.smithi064.stderr:++ image=sparse1 2023-10-09T06:21:05.167 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rbd info sparse1 2023-10-09T06:21:05.167 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep block_name_prefix 2023-10-09T06:21:05.167 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:21:05.171 INFO:tasks.workunit.client.0.smithi064.stderr:+++ awk '{print $NF;}' 2023-10-09T06:21:05.211 INFO:tasks.workunit.client.0.smithi064.stderr:++ prefix=rbd_data.df12397f7f30 2023-10-09T06:21:05.212 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep rbd_data.df12397f7f30 2023-10-09T06:21:05.212 INFO:tasks.workunit.client.0.smithi064.stderr:++++ get_image_data_pool sparse1 2023-10-09T06:21:05.213 INFO:tasks.workunit.client.0.smithi064.stderr:++++ image=sparse1 2023-10-09T06:21:05.213 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sed -e 's/rbd_data.df12397f7f30\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T06:21:05.213 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sort -u 2023-10-09T06:21:05.213 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ rbd info sparse1 2023-10-09T06:21:05.213 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ grep 'data_pool: ' 2023-10-09T06:21:05.213 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T06:21:05.248 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:05.252 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool= 2023-10-09T06:21:05.252 INFO:tasks.workunit.client.0.smithi064.stderr:++++ '[' -z ']' 2023-10-09T06:21:05.252 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool=rbd 2023-10-09T06:21:05.252 INFO:tasks.workunit.client.0.smithi064.stderr:++++ echo rbd 2023-10-09T06:21:05.252 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rados ls -p rbd 2023-10-09T06:21:05.276 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:05.287 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects=1 2023-10-09T06:21:05.288 INFO:tasks.workunit.client.0.smithi064.stderr:++ echo 1 2023-10-09T06:21:05.288 INFO:tasks.workunit.client.0.smithi064.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-09T06:21:05.288 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export sparse1 /tmp/rbd_import_export_695299/sparse1.out 2023-10-09T06:21:05.336 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T06:21:05.340 INFO:tasks.workunit.client.0.smithi064.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_695299/sparse1 /tmp/rbd_import_export_695299/sparse1.out 2023-10-09T06:21:05.340 INFO:tasks.workunit.client.0.smithi064.stderr:+ cmp -l /tmp/rbd_import_export_695299/sparse1 /tmp/rbd_import_export_695299/sparse1.out 2023-10-09T06:21:05.342 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_695299/sparse1 --format %b 2023-10-09T06:21:05.343 INFO:tasks.workunit.client.0.smithi064.stderr:+ origsize=2048 2023-10-09T06:21:05.343 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_695299/sparse1.out --format %b 2023-10-09T06:21:05.344 INFO:tasks.workunit.client.0.smithi064.stderr:+ exportsize=2048 2023-10-09T06:21:05.344 INFO:tasks.workunit.client.0.smithi064.stderr:+ difference=0 2023-10-09T06:21:05.345 INFO:tasks.workunit.client.0.smithi064.stderr:+ difference=0 2023-10-09T06:21:05.345 INFO:tasks.workunit.client.0.smithi064.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:21:05.345 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_695299/sparse1.out 2023-10-09T06:21:05.345 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm sparse1 2023-10-09T06:21:05.352 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:21:05.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:05.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:05.443 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T06:21:05.447 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm sparse2 2023-10-09T06:21:05.456 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:05.476 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:05.485 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:05.494 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 0% complete...failed. 2023-10-09T06:21:05.494 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T06:21:05.498 INFO:tasks.workunit.client.0.smithi064.stderr:+ true 2023-10-09T06:21:05.498 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_695299/sparse2 2023-10-09T06:21:05.531 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:05.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:05.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:05.577 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T06:21:05.578 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:21:05.578 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:21:05.581 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd ls -l 2023-10-09T06:21:05.581 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep sparse2 2023-10-09T06:21:05.581 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-10-09T06:21:05.618 INFO:tasks.workunit.client.0.smithi064.stdout:sparse2 2 MiB 2 2023-10-09T06:21:05.619 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects sparse2 2023-10-09T06:21:05.619 INFO:tasks.workunit.client.0.smithi064.stderr:++ image=sparse2 2023-10-09T06:21:05.620 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rbd info sparse2 2023-10-09T06:21:05.620 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep block_name_prefix 2023-10-09T06:21:05.620 INFO:tasks.workunit.client.0.smithi064.stderr:+++ awk '{print $NF;}' 2023-10-09T06:21:05.635 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:05.664 INFO:tasks.workunit.client.0.smithi064.stderr:++ prefix=rbd_data.b97be22895ff 2023-10-09T06:21:05.665 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep rbd_data.b97be22895ff 2023-10-09T06:21:05.665 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sed -e 's/rbd_data.b97be22895ff\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T06:21:05.666 INFO:tasks.workunit.client.0.smithi064.stderr:++++ get_image_data_pool sparse2 2023-10-09T06:21:05.666 INFO:tasks.workunit.client.0.smithi064.stderr:++++ image=sparse2 2023-10-09T06:21:05.666 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sort -u 2023-10-09T06:21:05.666 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:05.667 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ rbd info sparse2 2023-10-09T06:21:05.668 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T06:21:05.669 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ grep 'data_pool: ' 2023-10-09T06:21:05.698 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:21:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:21:05] "GET /metrics HTTP/1.1" 200 34109 "" "Prometheus/2.43.0" 2023-10-09T06:21:05.717 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool= 2023-10-09T06:21:05.717 INFO:tasks.workunit.client.0.smithi064.stderr:++++ '[' -z ']' 2023-10-09T06:21:05.717 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool=rbd 2023-10-09T06:21:05.718 INFO:tasks.workunit.client.0.smithi064.stderr:++++ echo rbd 2023-10-09T06:21:05.718 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rados ls -p rbd 2023-10-09T06:21:05.753 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects=0 2023-10-09T06:21:05.753 INFO:tasks.workunit.client.0.smithi064.stderr:++ echo 0 2023-10-09T06:21:05.754 INFO:tasks.workunit.client.0.smithi064.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-10-09T06:21:05.754 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export sparse2 /tmp/rbd_import_export_695299/sparse2.out 2023-10-09T06:21:05.793 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T06:21:05.799 INFO:tasks.workunit.client.0.smithi064.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_695299/sparse2 /tmp/rbd_import_export_695299/sparse2.out 2023-10-09T06:21:05.799 INFO:tasks.workunit.client.0.smithi064.stderr:+ cmp -l /tmp/rbd_import_export_695299/sparse2 /tmp/rbd_import_export_695299/sparse2.out 2023-10-09T06:21:05.802 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_695299/sparse2 --format %b 2023-10-09T06:21:05.803 INFO:tasks.workunit.client.0.smithi064.stderr:+ origsize=2048 2023-10-09T06:21:05.803 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_695299/sparse2.out --format %b 2023-10-09T06:21:05.804 INFO:tasks.workunit.client.0.smithi064.stderr:+ exportsize=2048 2023-10-09T06:21:05.804 INFO:tasks.workunit.client.0.smithi064.stderr:+ difference=0 2023-10-09T06:21:05.805 INFO:tasks.workunit.client.0.smithi064.stderr:+ difference=0 2023-10-09T06:21:05.805 INFO:tasks.workunit.client.0.smithi064.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:21:05.805 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_695299/sparse2.out 2023-10-09T06:21:05.805 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm sparse2 2023-10-09T06:21:05.902 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T06:21:05.905 INFO:tasks.workunit.client.0.smithi064.stderr:+ truncate /tmp/rbd_import_export_695299/sparse1 -s 10M 2023-10-09T06:21:05.906 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --new-format --order 20 - sparse1 2023-10-09T06:21:05.967 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-10-09T06:21:05.976 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:21:05.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:05.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:06.021 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 100% complete...done. 2023-10-09T06:21:06.022 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:21:06.022 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:21:06.025 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd ls -l 2023-10-09T06:21:06.025 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep sparse1 2023-10-09T06:21:06.025 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-10-09T06:21:06.064 INFO:tasks.workunit.client.0.smithi064.stdout:sparse1 10 MiB 2 2023-10-09T06:21:06.064 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects sparse1 2023-10-09T06:21:06.064 INFO:tasks.workunit.client.0.smithi064.stderr:++ image=sparse1 2023-10-09T06:21:06.065 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rbd info sparse1 2023-10-09T06:21:06.065 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep block_name_prefix 2023-10-09T06:21:06.065 INFO:tasks.workunit.client.0.smithi064.stderr:+++ awk '{print $NF;}' 2023-10-09T06:21:06.082 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:06.108 INFO:tasks.workunit.client.0.smithi064.stderr:++ prefix=rbd_data.924f14469d67 2023-10-09T06:21:06.108 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep rbd_data.924f14469d67 2023-10-09T06:21:06.109 INFO:tasks.workunit.client.0.smithi064.stderr:++++ get_image_data_pool sparse1 2023-10-09T06:21:06.109 INFO:tasks.workunit.client.0.smithi064.stderr:++++ image=sparse1 2023-10-09T06:21:06.109 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sort -u 2023-10-09T06:21:06.109 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sed -e 's/rbd_data.924f14469d67\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T06:21:06.111 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ rbd info sparse1 2023-10-09T06:21:06.111 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ grep 'data_pool: ' 2023-10-09T06:21:06.111 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T06:21:06.122 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:06.165 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool= 2023-10-09T06:21:06.165 INFO:tasks.workunit.client.0.smithi064.stderr:++++ '[' -z ']' 2023-10-09T06:21:06.165 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool=rbd 2023-10-09T06:21:06.165 INFO:tasks.workunit.client.0.smithi064.stderr:++++ echo rbd 2023-10-09T06:21:06.165 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rados ls -p rbd 2023-10-09T06:21:06.198 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects=1 2023-10-09T06:21:06.198 INFO:tasks.workunit.client.0.smithi064.stderr:++ echo 1 2023-10-09T06:21:06.199 INFO:tasks.workunit.client.0.smithi064.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-09T06:21:06.199 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export sparse1 /tmp/rbd_import_export_695299/sparse1.out 2023-10-09T06:21:06.236 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-10-09T06:21:06.241 INFO:tasks.workunit.client.0.smithi064.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_695299/sparse1 /tmp/rbd_import_export_695299/sparse1.out 2023-10-09T06:21:06.241 INFO:tasks.workunit.client.0.smithi064.stderr:+ cmp -l /tmp/rbd_import_export_695299/sparse1 /tmp/rbd_import_export_695299/sparse1.out 2023-10-09T06:21:06.249 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_695299/sparse1 --format %b 2023-10-09T06:21:06.250 INFO:tasks.workunit.client.0.smithi064.stderr:+ origsize=2048 2023-10-09T06:21:06.250 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_695299/sparse1.out --format %b 2023-10-09T06:21:06.251 INFO:tasks.workunit.client.0.smithi064.stderr:+ exportsize=2048 2023-10-09T06:21:06.251 INFO:tasks.workunit.client.0.smithi064.stderr:+ difference=0 2023-10-09T06:21:06.251 INFO:tasks.workunit.client.0.smithi064.stderr:+ difference=0 2023-10-09T06:21:06.251 INFO:tasks.workunit.client.0.smithi064.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:21:06.251 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_695299/sparse1.out 2023-10-09T06:21:06.252 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm sparse1 2023-10-09T06:21:06.285 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:21:06.287 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[212750]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.287 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[212750]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.287 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[212750]: osdmap e667: 8 total, 8 up, 5 in 2023-10-09T06:21:06.287 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[212750]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.287 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[212750]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.287 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[212750]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.287 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[212750]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.287 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[212750]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.287 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[212750]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.287 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[212750]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.288 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/945340209' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:06.288 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:06.288 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:06.288 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:21:06.288 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[212750]: osdmap e668: 8 total, 8 up, 5 in 2023-10-09T06:21:06.288 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[220044]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.288 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[220044]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.288 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[220044]: osdmap e667: 8 total, 8 up, 5 in 2023-10-09T06:21:06.289 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[220044]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.289 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[220044]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.289 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[220044]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.289 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[220044]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.289 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[220044]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.289 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[220044]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.289 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[220044]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.289 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/945340209' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:06.289 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:06.289 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[220044]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:06.289 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:21:06.290 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:05 smithi064 ceph-mon[220044]: osdmap e668: 8 total, 8 up, 5 in 2023-10-09T06:21:06.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:06.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:05 smithi156 ceph-mon[195101]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:05 smithi156 ceph-mon[195101]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:05 smithi156 ceph-mon[195101]: osdmap e667: 8 total, 8 up, 5 in 2023-10-09T06:21:06.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:05 smithi156 ceph-mon[195101]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:05 smithi156 ceph-mon[195101]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:05 smithi156 ceph-mon[195101]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:05 smithi156 ceph-mon[195101]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:05 smithi156 ceph-mon[195101]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:05 smithi156 ceph-mon[195101]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:05 smithi156 ceph-mon[195101]: failed to encode map e667 with expected crc 2023-10-09T06:21:06.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:05 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/945340209' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:06.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:05 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:06.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:05 smithi156 ceph-mon[195101]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:06.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:05 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:21:06.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:05 smithi156 ceph-mon[195101]: osdmap e668: 8 total, 8 up, 5 in 2023-10-09T06:21:06.391 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:06.418 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:06.585 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T06:21:06.588 INFO:tasks.workunit.client.0.smithi064.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_695299/sparse2 oflag=append conv=notrunc 2023-10-09T06:21:06.598 INFO:tasks.workunit.client.0.smithi064.stderr:1+0 records in 2023-10-09T06:21:06.598 INFO:tasks.workunit.client.0.smithi064.stderr:1+0 records out 2023-10-09T06:21:06.599 INFO:tasks.workunit.client.0.smithi064.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.00954225 s, 220 MB/s 2023-10-09T06:21:06.599 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --new-format --order 20 - sparse2 2023-10-09T06:21:06.681 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 100% complete...done. 2023-10-09T06:21:06.683 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:21:06.683 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:21:06.686 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd ls -l 2023-10-09T06:21:06.686 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep sparse2 2023-10-09T06:21:06.686 INFO:tasks.workunit.client.0.smithi064.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-10-09T06:21:06.723 INFO:tasks.workunit.client.0.smithi064.stdout:sparse2 4 MiB 2 2023-10-09T06:21:06.724 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects sparse2 2023-10-09T06:21:06.724 INFO:tasks.workunit.client.0.smithi064.stderr:++ image=sparse2 2023-10-09T06:21:06.725 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rbd info sparse2 2023-10-09T06:21:06.725 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep block_name_prefix 2023-10-09T06:21:06.725 INFO:tasks.workunit.client.0.smithi064.stderr:+++ awk '{print $NF;}' 2023-10-09T06:21:06.763 INFO:tasks.workunit.client.0.smithi064.stderr:++ prefix=rbd_data.b9d531e9511c 2023-10-09T06:21:06.763 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep rbd_data.b9d531e9511c 2023-10-09T06:21:06.763 INFO:tasks.workunit.client.0.smithi064.stderr:++++ get_image_data_pool sparse2 2023-10-09T06:21:06.764 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sed -e 's/rbd_data.b9d531e9511c\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T06:21:06.764 INFO:tasks.workunit.client.0.smithi064.stderr:++++ image=sparse2 2023-10-09T06:21:06.764 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sort -u 2023-10-09T06:21:06.764 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ rbd info sparse2 2023-10-09T06:21:06.764 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ grep 'data_pool: ' 2023-10-09T06:21:06.765 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T06:21:06.801 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool= 2023-10-09T06:21:06.801 INFO:tasks.workunit.client.0.smithi064.stderr:++++ '[' -z ']' 2023-10-09T06:21:06.801 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool=rbd 2023-10-09T06:21:06.802 INFO:tasks.workunit.client.0.smithi064.stderr:++++ echo rbd 2023-10-09T06:21:06.802 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rados ls -p rbd 2023-10-09T06:21:06.831 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:21:06.834 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects='0 2023-10-09T06:21:06.834 INFO:tasks.workunit.client.0.smithi064.stderr:2 2023-10-09T06:21:06.834 INFO:tasks.workunit.client.0.smithi064.stderr:3' 2023-10-09T06:21:06.834 INFO:tasks.workunit.client.0.smithi064.stderr:++ echo 0 2 3 2023-10-09T06:21:06.834 INFO:tasks.workunit.client.0.smithi064.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-10-09T06:21:06.834 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export sparse2 /tmp/rbd_import_export_695299/sparse2.out 2023-10-09T06:21:06.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:06.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:06.876 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-10-09T06:21:06.879 INFO:tasks.workunit.client.0.smithi064.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_695299/sparse2 /tmp/rbd_import_export_695299/sparse2.out 2023-10-09T06:21:06.879 INFO:tasks.workunit.client.0.smithi064.stderr:+ cmp -l /tmp/rbd_import_export_695299/sparse2 /tmp/rbd_import_export_695299/sparse2.out 2023-10-09T06:21:06.883 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_695299/sparse2 --format %b 2023-10-09T06:21:06.884 INFO:tasks.workunit.client.0.smithi064.stderr:+ origsize=6144 2023-10-09T06:21:06.884 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_695299/sparse2.out --format %b 2023-10-09T06:21:06.885 INFO:tasks.workunit.client.0.smithi064.stderr:+ exportsize=6144 2023-10-09T06:21:06.885 INFO:tasks.workunit.client.0.smithi064.stderr:+ difference=0 2023-10-09T06:21:06.885 INFO:tasks.workunit.client.0.smithi064.stderr:+ difference=0 2023-10-09T06:21:06.885 INFO:tasks.workunit.client.0.smithi064.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:21:06.885 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_695299/sparse2.out 2023-10-09T06:21:06.886 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm sparse2 2023-10-09T06:21:06.935 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:06.965 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:07.010 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-10-09T06:21:07.013 INFO:tasks.workunit.client.0.smithi064.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-10-09T06:21:07.013 INFO:tasks.workunit.client.0.smithi064.stdout:partially-sparse file imports to partially-sparse image 2023-10-09T06:21:07.013 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_695299/sparse1 sparse 2023-10-09T06:21:07.074 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-10-09T06:21:07.075 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:21:07.075 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:21:07.078 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects sparse 2023-10-09T06:21:07.078 INFO:tasks.workunit.client.0.smithi064.stderr:++ image=sparse 2023-10-09T06:21:07.078 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rbd info sparse 2023-10-09T06:21:07.079 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep block_name_prefix 2023-10-09T06:21:07.079 INFO:tasks.workunit.client.0.smithi064.stderr:+++ awk '{print $NF;}' 2023-10-09T06:21:07.114 INFO:tasks.workunit.client.0.smithi064.stderr:++ prefix=rbd_data.ba02bbd6ae09 2023-10-09T06:21:07.114 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep rbd_data.ba02bbd6ae09 2023-10-09T06:21:07.115 INFO:tasks.workunit.client.0.smithi064.stderr:++++ get_image_data_pool sparse 2023-10-09T06:21:07.115 INFO:tasks.workunit.client.0.smithi064.stderr:++++ image=sparse 2023-10-09T06:21:07.115 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sed -e 's/rbd_data.ba02bbd6ae09\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T06:21:07.115 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sort -u 2023-10-09T06:21:07.115 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ rbd info sparse 2023-10-09T06:21:07.116 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ grep 'data_pool: ' 2023-10-09T06:21:07.116 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T06:21:07.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:06 smithi156 ceph-mon[195101]: pgmap v1703: 103 pgs: 2 remapped+peering, 31 peering, 70 active+clean; 582 KiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 85 KiB/s rd, 7.5 KiB/s wr, 117 op/s; 4 B/s, 0 objects/s recovering 2023-10-09T06:21:07.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:06 smithi156 ceph-mon[195101]: failed to encode map e668 with expected crc 2023-10-09T06:21:07.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:06 smithi156 ceph-mon[195101]: failed to encode map e668 with expected crc 2023-10-09T06:21:07.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:06 smithi156 ceph-mon[195101]: failed to encode map e668 with expected crc 2023-10-09T06:21:07.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:06 smithi156 ceph-mon[195101]: failed to encode map e668 with expected crc 2023-10-09T06:21:07.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:06 smithi156 ceph-mon[195101]: failed to encode map e668 with expected crc 2023-10-09T06:21:07.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:06 smithi156 ceph-mon[195101]: failed to encode map e668 with expected crc 2023-10-09T06:21:07.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:06 smithi156 ceph-mon[195101]: failed to encode map e668 with expected crc 2023-10-09T06:21:07.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:06 smithi156 ceph-mon[195101]: osdmap e669: 8 total, 8 up, 5 in 2023-10-09T06:21:07.152 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool= 2023-10-09T06:21:07.152 INFO:tasks.workunit.client.0.smithi064.stderr:++++ '[' -z ']' 2023-10-09T06:21:07.152 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool=rbd 2023-10-09T06:21:07.152 INFO:tasks.workunit.client.0.smithi064.stderr:++++ echo rbd 2023-10-09T06:21:07.152 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rados ls -p rbd 2023-10-09T06:21:07.184 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects=1 2023-10-09T06:21:07.185 INFO:tasks.workunit.client.0.smithi064.stderr:++ echo 1 2023-10-09T06:21:07.185 INFO:tasks.workunit.client.0.smithi064.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-09T06:21:07.185 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm sparse 2023-10-09T06:21:07.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:06 smithi064 ceph-mon[220044]: pgmap v1703: 103 pgs: 2 remapped+peering, 31 peering, 70 active+clean; 582 KiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 85 KiB/s rd, 7.5 KiB/s wr, 117 op/s; 4 B/s, 0 objects/s recovering 2023-10-09T06:21:07.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:06 smithi064 ceph-mon[220044]: failed to encode map e668 with expected crc 2023-10-09T06:21:07.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:06 smithi064 ceph-mon[220044]: failed to encode map e668 with expected crc 2023-10-09T06:21:07.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:06 smithi064 ceph-mon[220044]: failed to encode map e668 with expected crc 2023-10-09T06:21:07.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:06 smithi064 ceph-mon[220044]: failed to encode map e668 with expected crc 2023-10-09T06:21:07.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:06 smithi064 ceph-mon[220044]: failed to encode map e668 with expected crc 2023-10-09T06:21:07.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:06 smithi064 ceph-mon[220044]: failed to encode map e668 with expected crc 2023-10-09T06:21:07.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:06 smithi064 ceph-mon[220044]: failed to encode map e668 with expected crc 2023-10-09T06:21:07.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:06 smithi064 ceph-mon[220044]: osdmap e669: 8 total, 8 up, 5 in 2023-10-09T06:21:07.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:06 smithi064 ceph-mon[212750]: pgmap v1703: 103 pgs: 2 remapped+peering, 31 peering, 70 active+clean; 582 KiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 85 KiB/s rd, 7.5 KiB/s wr, 117 op/s; 4 B/s, 0 objects/s recovering 2023-10-09T06:21:07.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:06 smithi064 ceph-mon[212750]: failed to encode map e668 with expected crc 2023-10-09T06:21:07.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:06 smithi064 ceph-mon[212750]: failed to encode map e668 with expected crc 2023-10-09T06:21:07.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:06 smithi064 ceph-mon[212750]: failed to encode map e668 with expected crc 2023-10-09T06:21:07.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:06 smithi064 ceph-mon[212750]: failed to encode map e668 with expected crc 2023-10-09T06:21:07.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:06 smithi064 ceph-mon[212750]: failed to encode map e668 with expected crc 2023-10-09T06:21:07.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:06 smithi064 ceph-mon[212750]: failed to encode map e668 with expected crc 2023-10-09T06:21:07.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:06 smithi064 ceph-mon[212750]: failed to encode map e668 with expected crc 2023-10-09T06:21:07.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:06 smithi064 ceph-mon[212750]: osdmap e669: 8 total, 8 up, 5 in 2023-10-09T06:21:07.282 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T06:21:07.285 INFO:tasks.workunit.client.0.smithi064.stderr:+ echo 'zeros import through stdin to sparse image' 2023-10-09T06:21:07.286 INFO:tasks.workunit.client.0.smithi064.stdout:zeros import through stdin to sparse image 2023-10-09T06:21:07.286 INFO:tasks.workunit.client.0.smithi064.stderr:+ dd if=/dev/zero bs=1M count=4 2023-10-09T06:21:07.286 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd import --new-format - sparse 2023-10-09T06:21:07.334 INFO:tasks.workunit.client.0.smithi064.stderr:4+0 records in 2023-10-09T06:21:07.335 INFO:tasks.workunit.client.0.smithi064.stderr:4+0 records out 2023-10-09T06:21:07.335 INFO:tasks.workunit.client.0.smithi064.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.04794 s, 87.5 MB/s 2023-10-09T06:21:07.345 INFO:tasks.workunit.client.0.smithi064.stderr: Importing image: 100% complete...done. 2023-10-09T06:21:07.346 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:21:07.349 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects sparse 2023-10-09T06:21:07.349 INFO:tasks.workunit.client.0.smithi064.stderr:++ image=sparse 2023-10-09T06:21:07.349 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rbd info sparse 2023-10-09T06:21:07.350 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep block_name_prefix 2023-10-09T06:21:07.350 INFO:tasks.workunit.client.0.smithi064.stderr:+++ awk '{print $NF;}' 2023-10-09T06:21:07.386 INFO:tasks.workunit.client.0.smithi064.stderr:++ prefix=rbd_data.92ca4e9ba433 2023-10-09T06:21:07.387 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep rbd_data.92ca4e9ba433 2023-10-09T06:21:07.387 INFO:tasks.workunit.client.0.smithi064.stderr:++++ get_image_data_pool sparse 2023-10-09T06:21:07.387 INFO:tasks.workunit.client.0.smithi064.stderr:++++ image=sparse 2023-10-09T06:21:07.388 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sed -e 's/rbd_data.92ca4e9ba433\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T06:21:07.388 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sort -u 2023-10-09T06:21:07.388 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ rbd info sparse 2023-10-09T06:21:07.388 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ grep 'data_pool: ' 2023-10-09T06:21:07.388 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T06:21:07.412 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:21:07.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:07.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:07.426 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool= 2023-10-09T06:21:07.426 INFO:tasks.workunit.client.0.smithi064.stderr:++++ '[' -z ']' 2023-10-09T06:21:07.426 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool=rbd 2023-10-09T06:21:07.426 INFO:tasks.workunit.client.0.smithi064.stderr:++++ echo rbd 2023-10-09T06:21:07.426 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rados ls -p rbd 2023-10-09T06:21:07.462 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects= 2023-10-09T06:21:07.462 INFO:tasks.workunit.client.0.smithi064.stderr:++ echo 2023-10-09T06:21:07.462 INFO:tasks.workunit.client.0.smithi064.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-10-09T06:21:07.462 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm sparse 2023-10-09T06:21:07.516 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:07.555 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:07.564 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 100% complete...done. 2023-10-09T06:21:07.565 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2023-10-09T06:21:07.565 INFO:teuthology.orchestra.run.smithi156.stdout: "ops": [], 2023-10-09T06:21:07.565 INFO:teuthology.orchestra.run.smithi156.stdout: "num_ops": 0 2023-10-09T06:21:07.565 INFO:teuthology.orchestra.run.smithi156.stdout:} 2023-10-09T06:21:07.570 INFO:tasks.workunit.client.0.smithi064.stdout:zeros export to sparse file 2023-10-09T06:21:07.570 INFO:tasks.workunit.client.0.smithi064.stderr:+ echo 'zeros export to sparse file' 2023-10-09T06:21:07.570 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd create --new-format sparse --size 4 2023-10-09T06:21:07.619 INFO:tasks.workunit.client.0.smithi064.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:21:07.630 INFO:tasks.workunit.client.0.smithi064.stderr:++ rbd info sparse 2023-10-09T06:21:07.630 INFO:tasks.workunit.client.0.smithi064.stderr:++ grep block_name_prefix 2023-10-09T06:21:07.630 INFO:tasks.workunit.client.0.smithi064.stderr:++ awk '{print $NF;}' 2023-10-09T06:21:07.660 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:21:07.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:07.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:07.665 INFO:tasks.workunit.client.0.smithi064.stderr:+ prefix=rbd_data.92ebcad7cd24 2023-10-09T06:21:07.665 INFO:tasks.workunit.client.0.smithi064.stderr:+ dd if=/dev/zero bs=4M count=1 2023-10-09T06:21:07.665 INFO:tasks.workunit.client.0.smithi064.stderr:++ get_image_data_pool sparse 2023-10-09T06:21:07.665 INFO:tasks.workunit.client.0.smithi064.stderr:++ image=sparse 2023-10-09T06:21:07.666 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rbd info sparse 2023-10-09T06:21:07.666 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep 'data_pool: ' 2023-10-09T06:21:07.666 INFO:tasks.workunit.client.0.smithi064.stderr:+++ awk -F: '{ print $NF }' 2023-10-09T06:21:07.711 INFO:tasks.workunit.client.0.smithi064.stderr:++ data_pool= 2023-10-09T06:21:07.712 INFO:tasks.workunit.client.0.smithi064.stderr:++ '[' -z ']' 2023-10-09T06:21:07.712 INFO:tasks.workunit.client.0.smithi064.stderr:++ data_pool=rbd 2023-10-09T06:21:07.712 INFO:tasks.workunit.client.0.smithi064.stderr:++ echo rbd 2023-10-09T06:21:07.712 INFO:tasks.workunit.client.0.smithi064.stderr:+ rados -p rbd put rbd_data.92ebcad7cd24.000000000000 - 2023-10-09T06:21:07.743 INFO:tasks.workunit.client.0.smithi064.stderr:1+0 records in 2023-10-09T06:21:07.743 INFO:tasks.workunit.client.0.smithi064.stderr:1+0 records out 2023-10-09T06:21:07.743 INFO:tasks.workunit.client.0.smithi064.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0783269 s, 53.5 MB/s 2023-10-09T06:21:07.765 DEBUG:teuthology.orchestra.run.smithi156:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:07.775 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects sparse 2023-10-09T06:21:07.775 INFO:tasks.workunit.client.0.smithi064.stderr:++ image=sparse 2023-10-09T06:21:07.776 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rbd info sparse 2023-10-09T06:21:07.776 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep block_name_prefix 2023-10-09T06:21:07.776 INFO:tasks.workunit.client.0.smithi064.stderr:+++ awk '{print $NF;}' 2023-10-09T06:21:07.796 INFO:teuthology.orchestra.run.smithi156.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:07.822 INFO:tasks.workunit.client.0.smithi064.stderr:++ prefix=rbd_data.92ebcad7cd24 2023-10-09T06:21:07.823 INFO:tasks.workunit.client.0.smithi064.stderr:+++ grep rbd_data.92ebcad7cd24 2023-10-09T06:21:07.823 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sed -e 's/rbd_data.92ebcad7cd24\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T06:21:07.824 INFO:tasks.workunit.client.0.smithi064.stderr:++++ get_image_data_pool sparse 2023-10-09T06:21:07.824 INFO:tasks.workunit.client.0.smithi064.stderr:++++ image=sparse 2023-10-09T06:21:07.824 INFO:tasks.workunit.client.0.smithi064.stderr:+++ sort -u 2023-10-09T06:21:07.824 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ rbd info sparse 2023-10-09T06:21:07.824 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ grep 'data_pool: ' 2023-10-09T06:21:07.825 INFO:tasks.workunit.client.0.smithi064.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T06:21:07.866 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool= 2023-10-09T06:21:07.867 INFO:tasks.workunit.client.0.smithi064.stderr:++++ '[' -z ']' 2023-10-09T06:21:07.867 INFO:tasks.workunit.client.0.smithi064.stderr:++++ data_pool=rbd 2023-10-09T06:21:07.867 INFO:tasks.workunit.client.0.smithi064.stderr:++++ echo rbd 2023-10-09T06:21:07.870 INFO:tasks.workunit.client.0.smithi064.stderr:+++ rados ls -p rbd 2023-10-09T06:21:08.017 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-10-09T06:21:08.042 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-10-09T06:21:08.051 INFO:tasks.workunit.client.0.smithi064.stderr:++ objects=0 2023-10-09T06:21:08.051 INFO:tasks.workunit.client.0.smithi064.stderr:++ echo 0 2023-10-09T06:21:08.051 INFO:tasks.workunit.client.0.smithi064.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-10-09T06:21:08.052 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_695299/sparse 2023-10-09T06:21:08.052 INFO:tasks.workunit.client.0.smithi064.stderr:rm: cannot remove '/tmp/rbd_import_export_695299/sparse': No such file or directory 2023-10-09T06:21:08.052 INFO:tasks.workunit.client.0.smithi064.stderr:+ true 2023-10-09T06:21:08.052 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd export sparse /tmp/rbd_import_export_695299/sparse 2023-10-09T06:21:08.066 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-10-09T06:21:08.098 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-10-09T06:21:08.099 INFO:tasks.workunit.client.0.smithi064.stderr: Exporting image: 100% complete...done. 2023-10-09T06:21:08.102 INFO:tasks.workunit.client.0.smithi064.stderr:++ stat /tmp/rbd_import_export_695299/sparse --format=%b 2023-10-09T06:21:08.104 INFO:tasks.workunit.client.0.smithi064.stderr:+ '[' 0 = 0 ']' 2023-10-09T06:21:08.104 INFO:tasks.workunit.client.0.smithi064.stderr:+ rbd rm sparse 2023-10-09T06:21:08.120 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-10-09T06:21:08.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:07 smithi156 ceph-mon[195101]: failed to encode map e669 with expected crc 2023-10-09T06:21:08.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:07 smithi156 ceph-mon[195101]: failed to encode map e669 with expected crc 2023-10-09T06:21:08.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:07 smithi156 ceph-mon[195101]: failed to encode map e669 with expected crc 2023-10-09T06:21:08.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:07 smithi156 ceph-mon[195101]: failed to encode map e669 with expected crc 2023-10-09T06:21:08.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:07 smithi156 ceph-mon[195101]: failed to encode map e669 with expected crc 2023-10-09T06:21:08.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:07 smithi156 ceph-mon[195101]: failed to encode map e669 with expected crc 2023-10-09T06:21:08.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:07 smithi156 ceph-mon[195101]: failed to encode map e669 with expected crc 2023-10-09T06:21:08.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:07 smithi156 ceph-mon[195101]: osdmap e670: 8 total, 8 up, 5 in 2023-10-09T06:21:08.145 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-10-09T06:21:08.171 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-10-09T06:21:08.194 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-10-09T06:21:08.196 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:21:08.196 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:21:08.196 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:21:08.196 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:21:08.196 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:21:08.197 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:21:08.197 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:21:08.197 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:21:08.201 INFO:tasks.workunit.client.0.smithi064.stderr: Removing image: 100% complete...done. 2023-10-09T06:21:08.205 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm /tmp/rbd_import_export_695299/sparse /tmp/rbd_import_export_695299/sparse1 /tmp/rbd_import_export_695299/sparse2 /tmp/rbd_import_export_695299/sparse3 2023-10-09T06:21:08.208 INFO:tasks.workunit.client.0.smithi064.stderr:rm: cannot remove '/tmp/rbd_import_export_695299/sparse3': No such file or directory 2023-10-09T06:21:08.208 INFO:tasks.workunit.client.0.smithi064.stderr:+ true 2023-10-09T06:21:08.208 INFO:tasks.workunit.client.0.smithi064.stdout:OK 2023-10-09T06:21:08.208 INFO:tasks.workunit.client.0.smithi064.stderr:+ echo OK 2023-10-09T06:21:08.208 INFO:tasks.workunit.client.0.smithi064.stderr:+ rm -rf /tmp/rbd_import_export_695299 2023-10-09T06:21:08.209 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T06:21:08.210 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T06:21:08.279 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:07 smithi064 ceph-mon[212750]: failed to encode map e669 with expected crc 2023-10-09T06:21:08.279 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:07 smithi064 ceph-mon[212750]: failed to encode map e669 with expected crc 2023-10-09T06:21:08.279 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:07 smithi064 ceph-mon[212750]: failed to encode map e669 with expected crc 2023-10-09T06:21:08.279 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:07 smithi064 ceph-mon[212750]: failed to encode map e669 with expected crc 2023-10-09T06:21:08.279 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:07 smithi064 ceph-mon[212750]: failed to encode map e669 with expected crc 2023-10-09T06:21:08.279 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:07 smithi064 ceph-mon[212750]: failed to encode map e669 with expected crc 2023-10-09T06:21:08.280 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:07 smithi064 ceph-mon[212750]: failed to encode map e669 with expected crc 2023-10-09T06:21:08.280 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:07 smithi064 ceph-mon[212750]: osdmap e670: 8 total, 8 up, 5 in 2023-10-09T06:21:08.280 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:07 smithi064 ceph-mon[220044]: failed to encode map e669 with expected crc 2023-10-09T06:21:08.280 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:07 smithi064 ceph-mon[220044]: failed to encode map e669 with expected crc 2023-10-09T06:21:08.280 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:07 smithi064 ceph-mon[220044]: failed to encode map e669 with expected crc 2023-10-09T06:21:08.280 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:07 smithi064 ceph-mon[220044]: failed to encode map e669 with expected crc 2023-10-09T06:21:08.280 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:07 smithi064 ceph-mon[220044]: failed to encode map e669 with expected crc 2023-10-09T06:21:08.280 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:07 smithi064 ceph-mon[220044]: failed to encode map e669 with expected crc 2023-10-09T06:21:08.281 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:07 smithi064 ceph-mon[220044]: failed to encode map e669 with expected crc 2023-10-09T06:21:08.281 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:07 smithi064 ceph-mon[220044]: osdmap e670: 8 total, 8 up, 5 in 2023-10-09T06:21:08.283 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-10-09T06:21:08.284 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T06:21:08.460 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:21:08.469 INFO:teuthology.orchestra.run.smithi156.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:21:08.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:08.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi156 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:08.575 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:08.603 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:08.804 DEBUG:teuthology.parallel:result is None 2023-10-09T06:21:08.805 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:21:08.852 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:21:08.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:08.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:08.878 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:21:08.878 DEBUG:teuthology.orchestra.run.smithi064:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:21:08.935 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:21:08.935 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T06:21:08.935 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-10-09T06:21:08.936 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T06:21:08.936 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-10-09T06:21:08.936 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T06:21:08.936 DEBUG:teuthology.orchestra.run.smithi064:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:21:08.956 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:08.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:08.995 INFO:teuthology.orchestra.run.smithi064.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T06:21:08.995 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:21:09.005 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:09.023 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T06:21:09.023 DEBUG:teuthology.orchestra.run.smithi064:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T06:21:09.080 INFO:tasks.thrashosds:joining thrashosds 2023-10-09T06:21:09.101 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:21:09.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:08 smithi064 ceph-mon[212750]: pgmap v1706: 103 pgs: 5 active+undersized+remapped, 2 remapped+peering, 31 peering, 65 active+clean; 582 KiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 178 KiB/s rd, 11 KiB/s wr, 235 op/s; 143 B/s, 1 objects/s recovering 2023-10-09T06:21:09.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:08 smithi064 ceph-mon[212750]: failed to encode map e670 with expected crc 2023-10-09T06:21:09.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:08 smithi064 ceph-mon[212750]: failed to encode map e670 with expected crc 2023-10-09T06:21:09.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:08 smithi064 ceph-mon[212750]: failed to encode map e670 with expected crc 2023-10-09T06:21:09.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:08 smithi064 ceph-mon[212750]: failed to encode map e670 with expected crc 2023-10-09T06:21:09.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:08 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3242491089' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:09.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:08 smithi064 ceph-mon[220044]: pgmap v1706: 103 pgs: 5 active+undersized+remapped, 2 remapped+peering, 31 peering, 65 active+clean; 582 KiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 178 KiB/s rd, 11 KiB/s wr, 235 op/s; 143 B/s, 1 objects/s recovering 2023-10-09T06:21:09.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:08 smithi064 ceph-mon[220044]: failed to encode map e670 with expected crc 2023-10-09T06:21:09.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:08 smithi064 ceph-mon[220044]: failed to encode map e670 with expected crc 2023-10-09T06:21:09.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:08 smithi064 ceph-mon[220044]: failed to encode map e670 with expected crc 2023-10-09T06:21:09.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:08 smithi064 ceph-mon[220044]: failed to encode map e670 with expected crc 2023-10-09T06:21:09.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:08 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3242491089' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:09.284 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:21:09.304 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:21:09.306 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:08 smithi156 ceph-mon[195101]: pgmap v1706: 103 pgs: 5 active+undersized+remapped, 2 remapped+peering, 31 peering, 65 active+clean; 582 KiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 178 KiB/s rd, 11 KiB/s wr, 235 op/s; 143 B/s, 1 objects/s recovering 2023-10-09T06:21:09.306 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:08 smithi156 ceph-mon[195101]: failed to encode map e670 with expected crc 2023-10-09T06:21:09.306 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:08 smithi156 ceph-mon[195101]: failed to encode map e670 with expected crc 2023-10-09T06:21:09.306 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:08 smithi156 ceph-mon[195101]: failed to encode map e670 with expected crc 2023-10-09T06:21:09.307 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:08 smithi156 ceph-mon[195101]: failed to encode map e670 with expected crc 2023-10-09T06:21:09.307 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:08 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3242491089' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:09.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:09.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-44c77acc-6665-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:09.766 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:10.050 INFO:tasks.thrashosds.thrasher:inning osd 2023-10-09T06:21:10.050 INFO:tasks.thrashosds.thrasher:Adding osd 4 2023-10-09T06:21:10.050 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd in 4 2023-10-09T06:21:10.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:10 smithi064 ceph-mon[212750]: pgmap v1708: 103 pgs: 5 active+undersized+remapped, 2 remapped+peering, 31 peering, 65 active+clean; 582 KiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 87 KiB/s rd, 3.4 KiB/s wr, 108 op/s; 129 B/s, 1 objects/s recovering 2023-10-09T06:21:10.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:10 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:10.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:10 smithi064 ceph-mon[220044]: pgmap v1708: 103 pgs: 5 active+undersized+remapped, 2 remapped+peering, 31 peering, 65 active+clean; 582 KiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 87 KiB/s rd, 3.4 KiB/s wr, 108 op/s; 129 B/s, 1 objects/s recovering 2023-10-09T06:21:10.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:10 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:10.776 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:10 smithi156 ceph-mon[195101]: pgmap v1708: 103 pgs: 5 active+undersized+remapped, 2 remapped+peering, 31 peering, 65 active+clean; 582 KiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 87 KiB/s rd, 3.4 KiB/s wr, 108 op/s; 129 B/s, 1 objects/s recovering 2023-10-09T06:21:10.776 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:10 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:11.085 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:21:11.086 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:21:11.207 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:11.705 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:21:11.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:11 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:11.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:11 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:11.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:11 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:11.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:11 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:11.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:11 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:11.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:11 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:11.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:11 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:11.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:11 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:11.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:11 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:11.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:11 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:11.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:11 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:11.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:11 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:12.048 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:21:12.497 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:12 smithi156 ceph-mon[195101]: pgmap v1709: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 4.6 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 1.3 MiB/s rd, 1.8 MiB/s wr, 379 op/s; 43 KiB/s, 3 objects/s recovering 2023-10-09T06:21:12.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:12 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:21:12.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:12 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:21:12.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:12 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:12.874 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-10-09T06:21:12.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:12 smithi064 ceph-mon[212750]: pgmap v1709: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 4.6 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 1.3 MiB/s rd, 1.8 MiB/s wr, 379 op/s; 43 KiB/s, 3 objects/s recovering 2023-10-09T06:21:12.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:12 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:21:12.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:12 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:21:12.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:12 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:12.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:12 smithi064 ceph-mon[220044]: pgmap v1709: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 4.6 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 1.3 MiB/s rd, 1.8 MiB/s wr, 379 op/s; 43 KiB/s, 3 objects/s recovering 2023-10-09T06:21:12.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:12 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:21:12.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:12 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:21:12.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:12 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:13.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:13 smithi064 conmon[212726]: 2023-10-09T06:21:13.001+0000 7f8823f23700 -1 mon.a@0(leader).osd e671 definitely_dead 0 2023-10-09T06:21:13.567 INFO:teuthology.orchestra.run.smithi156.stdout:ERROR: (22) Invalid argument 2023-10-09T06:21:13.568 INFO:teuthology.orchestra.run.smithi156.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:21:13.593 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:13 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1994668791' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:21:13.593 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:13 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:21:13.594 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:13 smithi156 ceph-mon[195101]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:13.594 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:13 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:21:13.594 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:13 smithi156 ceph-mon[195101]: osdmap e671: 8 total, 8 up, 5 in 2023-10-09T06:21:13.594 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:13 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1166572631' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T06:21:13.594 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:13 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T06:21:13.608 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:21:13.896 INFO:teuthology.orchestra.run.smithi064.stderr:marked in osd.4. 2023-10-09T06:21:13.921 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:13 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1994668791' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:21:13.921 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:13 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:21:13.921 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:13 smithi064 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:13.921 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:13 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:21:13.921 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:13 smithi064 ceph-mon[212750]: osdmap e671: 8 total, 8 up, 5 in 2023-10-09T06:21:13.922 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:13 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1166572631' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T06:21:13.922 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:13 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T06:21:13.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:13 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1994668791' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:21:13.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:13 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:21:13.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:13 smithi064 ceph-mon[220044]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:13.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:13 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:21:13.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:13 smithi064 ceph-mon[220044]: osdmap e671: 8 total, 8 up, 5 in 2023-10-09T06:21:13.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:13 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1166572631' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T06:21:13.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:13 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T06:21:14.572 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:14.597 INFO:tasks.thrashosds.thrasher:Added osd 4 2023-10-09T06:21:14.597 INFO:tasks.thrashosds.thrasher:inning osd 2023-10-09T06:21:14.597 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-10-09T06:21:14.597 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd in 2 2023-10-09T06:21:14.776 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[212750]: pgmap v1711: 103 pgs: 103 active+clean; 4.6 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 1.1 MiB/s rd, 1.7 MiB/s wr, 282 op/s; 40 KiB/s, 2 objects/s recovering 2023-10-09T06:21:14.776 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[212750]: failed to encode map e671 with expected crc 2023-10-09T06:21:14.776 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[212750]: failed to encode map e671 with expected crc 2023-10-09T06:21:14.776 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[212750]: failed to encode map e671 with expected crc 2023-10-09T06:21:14.776 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[212750]: failed to encode map e671 with expected crc 2023-10-09T06:21:14.776 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:14.777 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[212750]: failed to encode map e672 with expected crc 2023-10-09T06:21:14.777 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T06:21:14.777 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[212750]: osdmap e672: 8 total, 8 up, 6 in 2023-10-09T06:21:14.777 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[212750]: failed to encode map e672 with expected crc 2023-10-09T06:21:14.777 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[212750]: failed to encode map e672 with expected crc 2023-10-09T06:21:14.777 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[212750]: failed to encode map e672 with expected crc 2023-10-09T06:21:14.777 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[212750]: failed to encode map e672 with expected crc 2023-10-09T06:21:14.777 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[220044]: pgmap v1711: 103 pgs: 103 active+clean; 4.6 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 1.1 MiB/s rd, 1.7 MiB/s wr, 282 op/s; 40 KiB/s, 2 objects/s recovering 2023-10-09T06:21:14.778 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[220044]: failed to encode map e671 with expected crc 2023-10-09T06:21:14.778 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[220044]: failed to encode map e671 with expected crc 2023-10-09T06:21:14.778 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[220044]: failed to encode map e671 with expected crc 2023-10-09T06:21:14.778 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[220044]: failed to encode map e671 with expected crc 2023-10-09T06:21:14.778 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:14.778 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[220044]: failed to encode map e672 with expected crc 2023-10-09T06:21:14.778 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T06:21:14.778 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[220044]: osdmap e672: 8 total, 8 up, 6 in 2023-10-09T06:21:14.778 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[220044]: failed to encode map e672 with expected crc 2023-10-09T06:21:14.778 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[220044]: failed to encode map e672 with expected crc 2023-10-09T06:21:14.778 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[220044]: failed to encode map e672 with expected crc 2023-10-09T06:21:14.779 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:14 smithi064 ceph-mon[220044]: failed to encode map e672 with expected crc 2023-10-09T06:21:14.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:14 smithi156 ceph-mon[195101]: pgmap v1711: 103 pgs: 103 active+clean; 4.6 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 1.1 MiB/s rd, 1.7 MiB/s wr, 282 op/s; 40 KiB/s, 2 objects/s recovering 2023-10-09T06:21:14.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:14 smithi156 ceph-mon[195101]: failed to encode map e671 with expected crc 2023-10-09T06:21:14.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:14 smithi156 ceph-mon[195101]: failed to encode map e671 with expected crc 2023-10-09T06:21:14.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:14 smithi156 ceph-mon[195101]: failed to encode map e671 with expected crc 2023-10-09T06:21:14.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:14 smithi156 ceph-mon[195101]: failed to encode map e671 with expected crc 2023-10-09T06:21:14.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:14 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:14.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:14 smithi156 ceph-mon[195101]: failed to encode map e672 with expected crc 2023-10-09T06:21:14.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:14 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T06:21:14.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:14 smithi156 ceph-mon[195101]: osdmap e672: 8 total, 8 up, 6 in 2023-10-09T06:21:14.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:14 smithi156 ceph-mon[195101]: failed to encode map e672 with expected crc 2023-10-09T06:21:14.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:14 smithi156 ceph-mon[195101]: failed to encode map e672 with expected crc 2023-10-09T06:21:14.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:14 smithi156 ceph-mon[195101]: failed to encode map e672 with expected crc 2023-10-09T06:21:14.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:14 smithi156 ceph-mon[195101]: failed to encode map e672 with expected crc 2023-10-09T06:21:15.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:21:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:21:15] "GET /metrics HTTP/1.1" 200 34112 "" "Prometheus/2.43.0" 2023-10-09T06:21:15.697 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:15.751 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:15 smithi064 ceph-mon[220044]: failed to encode map e672 with expected crc 2023-10-09T06:21:15.751 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:15 smithi064 ceph-mon[220044]: failed to encode map e672 with expected crc 2023-10-09T06:21:15.751 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:15 smithi064 ceph-mon[220044]: failed to encode map e673 with expected crc 2023-10-09T06:21:15.751 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:15 smithi064 ceph-mon[220044]: failed to encode map e673 with expected crc 2023-10-09T06:21:15.751 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:15 smithi064 ceph-mon[220044]: osdmap e673: 8 total, 8 up, 6 in 2023-10-09T06:21:15.751 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:15 smithi064 ceph-mon[220044]: failed to encode map e673 with expected crc 2023-10-09T06:21:15.751 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:15 smithi064 ceph-mon[220044]: failed to encode map e673 with expected crc 2023-10-09T06:21:15.752 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:15 smithi064 ceph-mon[212750]: failed to encode map e672 with expected crc 2023-10-09T06:21:15.752 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:15 smithi064 ceph-mon[212750]: failed to encode map e672 with expected crc 2023-10-09T06:21:15.752 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:15 smithi064 ceph-mon[212750]: failed to encode map e673 with expected crc 2023-10-09T06:21:15.752 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:15 smithi064 ceph-mon[212750]: failed to encode map e673 with expected crc 2023-10-09T06:21:15.752 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:15 smithi064 ceph-mon[212750]: osdmap e673: 8 total, 8 up, 6 in 2023-10-09T06:21:15.753 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:15 smithi064 ceph-mon[212750]: failed to encode map e673 with expected crc 2023-10-09T06:21:15.753 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:15 smithi064 ceph-mon[212750]: failed to encode map e673 with expected crc 2023-10-09T06:21:15.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:15 smithi156 ceph-mon[195101]: failed to encode map e672 with expected crc 2023-10-09T06:21:15.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:15 smithi156 ceph-mon[195101]: failed to encode map e672 with expected crc 2023-10-09T06:21:15.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:15 smithi156 ceph-mon[195101]: failed to encode map e673 with expected crc 2023-10-09T06:21:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:15 smithi156 ceph-mon[195101]: failed to encode map e673 with expected crc 2023-10-09T06:21:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:15 smithi156 ceph-mon[195101]: osdmap e673: 8 total, 8 up, 6 in 2023-10-09T06:21:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:15 smithi156 ceph-mon[195101]: failed to encode map e673 with expected crc 2023-10-09T06:21:15.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:15 smithi156 ceph-mon[195101]: failed to encode map e673 with expected crc 2023-10-09T06:21:16.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:16 smithi064 ceph-mon[220044]: failed to encode map e673 with expected crc 2023-10-09T06:21:16.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:16 smithi064 ceph-mon[220044]: failed to encode map e673 with expected crc 2023-10-09T06:21:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:16 smithi064 ceph-mon[220044]: pgmap v1714: 103 pgs: 3 remapped+peering, 16 peering, 84 active+clean; 4.6 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 1.2 MiB/s rd, 1.8 MiB/s wr, 300 op/s; 43 KiB/s, 2 objects/s recovering 2023-10-09T06:21:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:16 smithi064 ceph-mon[220044]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:21:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:16 smithi064 ceph-mon[220044]: failed to encode map e674 with expected crc 2023-10-09T06:21:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:16 smithi064 ceph-mon[220044]: osdmap e674: 8 total, 8 up, 6 in 2023-10-09T06:21:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:16 smithi064 ceph-mon[220044]: failed to encode map e674 with expected crc 2023-10-09T06:21:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:16 smithi064 ceph-mon[220044]: failed to encode map e674 with expected crc 2023-10-09T06:21:16.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:16 smithi064 ceph-mon[220044]: failed to encode map e674 with expected crc 2023-10-09T06:21:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:16 smithi064 ceph-mon[212750]: failed to encode map e673 with expected crc 2023-10-09T06:21:16.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:16 smithi064 ceph-mon[212750]: failed to encode map e673 with expected crc 2023-10-09T06:21:16.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:16 smithi064 ceph-mon[212750]: pgmap v1714: 103 pgs: 3 remapped+peering, 16 peering, 84 active+clean; 4.6 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 1.2 MiB/s rd, 1.8 MiB/s wr, 300 op/s; 43 KiB/s, 2 objects/s recovering 2023-10-09T06:21:16.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:16 smithi064 ceph-mon[212750]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:21:16.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:16 smithi064 ceph-mon[212750]: failed to encode map e674 with expected crc 2023-10-09T06:21:16.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:16 smithi064 ceph-mon[212750]: osdmap e674: 8 total, 8 up, 6 in 2023-10-09T06:21:16.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:16 smithi064 ceph-mon[212750]: failed to encode map e674 with expected crc 2023-10-09T06:21:16.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:16 smithi064 ceph-mon[212750]: failed to encode map e674 with expected crc 2023-10-09T06:21:16.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:16 smithi064 ceph-mon[212750]: failed to encode map e674 with expected crc 2023-10-09T06:21:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:16 smithi156 ceph-mon[195101]: failed to encode map e673 with expected crc 2023-10-09T06:21:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:16 smithi156 ceph-mon[195101]: failed to encode map e673 with expected crc 2023-10-09T06:21:16.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:16 smithi156 ceph-mon[195101]: pgmap v1714: 103 pgs: 3 remapped+peering, 16 peering, 84 active+clean; 4.6 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 1.2 MiB/s rd, 1.8 MiB/s wr, 300 op/s; 43 KiB/s, 2 objects/s recovering 2023-10-09T06:21:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:16 smithi156 ceph-mon[195101]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:21:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:16 smithi156 ceph-mon[195101]: failed to encode map e674 with expected crc 2023-10-09T06:21:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:16 smithi156 ceph-mon[195101]: osdmap e674: 8 total, 8 up, 6 in 2023-10-09T06:21:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:16 smithi156 ceph-mon[195101]: failed to encode map e674 with expected crc 2023-10-09T06:21:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:16 smithi156 ceph-mon[195101]: failed to encode map e674 with expected crc 2023-10-09T06:21:16.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:16 smithi156 ceph-mon[195101]: failed to encode map e674 with expected crc 2023-10-09T06:21:16.895 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-10-09T06:21:17.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:17 smithi156 ceph-mon[195101]: failed to encode map e674 with expected crc 2023-10-09T06:21:17.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:17 smithi156 ceph-mon[195101]: failed to encode map e674 with expected crc 2023-10-09T06:21:17.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:17 smithi156 ceph-mon[195101]: failed to encode map e674 with expected crc 2023-10-09T06:21:17.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:17 smithi156 ceph-mon[195101]: failed to encode map e674 with expected crc 2023-10-09T06:21:17.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:17 smithi156 ceph-mon[195101]: failed to encode map e674 with expected crc 2023-10-09T06:21:17.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:17 smithi156 ceph-mon[195101]: failed to encode map e674 with expected crc 2023-10-09T06:21:17.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:17 smithi156 ceph-mon[195101]: failed to encode map e674 with expected crc 2023-10-09T06:21:17.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:17 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3214476040' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:17.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:17 smithi156 ceph-mon[195101]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:21:17.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:17 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3214476040' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:21:17.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:17 smithi156 ceph-mon[195101]: osdmap e675: 8 total, 8 up, 6 in 2023-10-09T06:21:17.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:17 smithi156 ceph-mon[195101]: failed to encode map e675 with expected crc 2023-10-09T06:21:17.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:17 smithi156 ceph-mon[195101]: failed to encode map e675 with expected crc 2023-10-09T06:21:17.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:17 smithi156 ceph-mon[195101]: failed to encode map e675 with expected crc 2023-10-09T06:21:17.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[220044]: failed to encode map e674 with expected crc 2023-10-09T06:21:17.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[220044]: failed to encode map e674 with expected crc 2023-10-09T06:21:17.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[220044]: failed to encode map e674 with expected crc 2023-10-09T06:21:17.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[220044]: failed to encode map e674 with expected crc 2023-10-09T06:21:17.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[220044]: failed to encode map e674 with expected crc 2023-10-09T06:21:17.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[220044]: failed to encode map e674 with expected crc 2023-10-09T06:21:17.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[220044]: failed to encode map e674 with expected crc 2023-10-09T06:21:17.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3214476040' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:17.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[220044]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:21:17.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3214476040' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:21:17.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[220044]: osdmap e675: 8 total, 8 up, 6 in 2023-10-09T06:21:17.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[220044]: failed to encode map e675 with expected crc 2023-10-09T06:21:17.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[220044]: failed to encode map e675 with expected crc 2023-10-09T06:21:17.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[220044]: failed to encode map e675 with expected crc 2023-10-09T06:21:17.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[212750]: failed to encode map e674 with expected crc 2023-10-09T06:21:17.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[212750]: failed to encode map e674 with expected crc 2023-10-09T06:21:17.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[212750]: failed to encode map e674 with expected crc 2023-10-09T06:21:17.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[212750]: failed to encode map e674 with expected crc 2023-10-09T06:21:17.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[212750]: failed to encode map e674 with expected crc 2023-10-09T06:21:17.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[212750]: failed to encode map e674 with expected crc 2023-10-09T06:21:17.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[212750]: failed to encode map e674 with expected crc 2023-10-09T06:21:17.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3214476040' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:17.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:21:17.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3214476040' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:21:17.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[212750]: osdmap e675: 8 total, 8 up, 6 in 2023-10-09T06:21:17.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[212750]: failed to encode map e675 with expected crc 2023-10-09T06:21:17.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[212750]: failed to encode map e675 with expected crc 2023-10-09T06:21:17.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:17 smithi064 ceph-mon[212750]: failed to encode map e675 with expected crc 2023-10-09T06:21:18.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:18 smithi064 conmon[212726]: 2023-10-09T06:21:18.060+0000 7f8823f23700 -1 mon.a@0(leader).osd e676 definitely_dead 0 2023-10-09T06:21:18.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:18 smithi156 ceph-mon[195101]: failed to encode map e675 with expected crc 2023-10-09T06:21:18.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:18 smithi156 ceph-mon[195101]: failed to encode map e675 with expected crc 2023-10-09T06:21:18.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:18 smithi156 ceph-mon[195101]: pgmap v1717: 103 pgs: 2 activating+undersized+remapped, 3 active+undersized+remapped, 3 remapped+peering, 16 peering, 1 active+remapped, 78 active+clean; 4.6 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 138 B/s, 1 objects/s recovering 2023-10-09T06:21:18.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:18 smithi156 ceph-mon[195101]: osdmap e676: 8 total, 8 up, 6 in 2023-10-09T06:21:18.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:18 smithi156 ceph-mon[195101]: failed to encode map e676 with expected crc 2023-10-09T06:21:18.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:18 smithi156 ceph-mon[195101]: failed to encode map e676 with expected crc 2023-10-09T06:21:18.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:18 smithi156 ceph-mon[195101]: failed to encode map e676 with expected crc 2023-10-09T06:21:18.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:18 smithi156 ceph-mon[195101]: failed to encode map e676 with expected crc 2023-10-09T06:21:18.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:18 smithi156 ceph-mon[195101]: failed to encode map e676 with expected crc 2023-10-09T06:21:18.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:18 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1743599400' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T06:21:18.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:18 smithi156 ceph-mon[195101]: 1.1b deep-scrub starts 2023-10-09T06:21:18.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:18 smithi156 ceph-mon[195101]: 1.1b deep-scrub ok 2023-10-09T06:21:18.910 INFO:teuthology.orchestra.run.smithi064.stderr:marked in osd.2. 2023-10-09T06:21:18.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:18 smithi064 ceph-mon[212750]: failed to encode map e675 with expected crc 2023-10-09T06:21:18.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:18 smithi064 ceph-mon[212750]: failed to encode map e675 with expected crc 2023-10-09T06:21:18.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:18 smithi064 ceph-mon[212750]: pgmap v1717: 103 pgs: 2 activating+undersized+remapped, 3 active+undersized+remapped, 3 remapped+peering, 16 peering, 1 active+remapped, 78 active+clean; 4.6 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 138 B/s, 1 objects/s recovering 2023-10-09T06:21:18.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:18 smithi064 ceph-mon[212750]: osdmap e676: 8 total, 8 up, 6 in 2023-10-09T06:21:18.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:18 smithi064 ceph-mon[212750]: failed to encode map e676 with expected crc 2023-10-09T06:21:18.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:18 smithi064 ceph-mon[212750]: failed to encode map e676 with expected crc 2023-10-09T06:21:18.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:18 smithi064 ceph-mon[212750]: failed to encode map e676 with expected crc 2023-10-09T06:21:18.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:18 smithi064 ceph-mon[212750]: failed to encode map e676 with expected crc 2023-10-09T06:21:18.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:18 smithi064 ceph-mon[212750]: failed to encode map e676 with expected crc 2023-10-09T06:21:18.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:18 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1743599400' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T06:21:18.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:18 smithi064 ceph-mon[212750]: 1.1b deep-scrub starts 2023-10-09T06:21:18.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:18 smithi064 ceph-mon[212750]: 1.1b deep-scrub ok 2023-10-09T06:21:18.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:18 smithi064 ceph-mon[220044]: failed to encode map e675 with expected crc 2023-10-09T06:21:18.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:18 smithi064 ceph-mon[220044]: failed to encode map e675 with expected crc 2023-10-09T06:21:18.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:18 smithi064 ceph-mon[220044]: pgmap v1717: 103 pgs: 2 activating+undersized+remapped, 3 active+undersized+remapped, 3 remapped+peering, 16 peering, 1 active+remapped, 78 active+clean; 4.6 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 138 B/s, 1 objects/s recovering 2023-10-09T06:21:18.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:18 smithi064 ceph-mon[220044]: osdmap e676: 8 total, 8 up, 6 in 2023-10-09T06:21:18.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:18 smithi064 ceph-mon[220044]: failed to encode map e676 with expected crc 2023-10-09T06:21:18.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:18 smithi064 ceph-mon[220044]: failed to encode map e676 with expected crc 2023-10-09T06:21:18.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:18 smithi064 ceph-mon[220044]: failed to encode map e676 with expected crc 2023-10-09T06:21:18.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:18 smithi064 ceph-mon[220044]: failed to encode map e676 with expected crc 2023-10-09T06:21:18.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:18 smithi064 ceph-mon[220044]: failed to encode map e676 with expected crc 2023-10-09T06:21:18.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:18 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1743599400' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T06:21:18.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:18 smithi064 ceph-mon[220044]: 1.1b deep-scrub starts 2023-10-09T06:21:18.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:18 smithi064 ceph-mon[220044]: 1.1b deep-scrub ok 2023-10-09T06:21:19.662 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-10-09T06:21:19.662 INFO:tasks.thrashosds.thrasher:inning osd 2023-10-09T06:21:19.662 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-10-09T06:21:19.662 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd in 5 2023-10-09T06:21:19.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:19 smithi156 ceph-mon[195101]: failed to encode map e675 with expected crc 2023-10-09T06:21:19.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:19 smithi156 ceph-mon[195101]: failed to encode map e676 with expected crc 2023-10-09T06:21:19.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:19 smithi156 ceph-mon[195101]: failed to encode map e676 with expected crc 2023-10-09T06:21:19.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:19 smithi156 ceph-mon[195101]: failed to encode map e676 with expected crc 2023-10-09T06:21:19.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:19 smithi156 ceph-mon[195101]: failed to encode map e677 with expected crc 2023-10-09T06:21:19.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:19 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1743599400' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T06:21:19.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:19 smithi156 ceph-mon[195101]: osdmap e677: 8 total, 8 up, 7 in 2023-10-09T06:21:19.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:19 smithi156 ceph-mon[195101]: failed to encode map e677 with expected crc 2023-10-09T06:21:19.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:19 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:19.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:19 smithi064 ceph-mon[212750]: failed to encode map e675 with expected crc 2023-10-09T06:21:19.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:19 smithi064 ceph-mon[212750]: failed to encode map e676 with expected crc 2023-10-09T06:21:19.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:19 smithi064 ceph-mon[212750]: failed to encode map e676 with expected crc 2023-10-09T06:21:19.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:19 smithi064 ceph-mon[212750]: failed to encode map e676 with expected crc 2023-10-09T06:21:19.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:19 smithi064 ceph-mon[212750]: failed to encode map e677 with expected crc 2023-10-09T06:21:19.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:19 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1743599400' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T06:21:19.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:19 smithi064 ceph-mon[212750]: osdmap e677: 8 total, 8 up, 7 in 2023-10-09T06:21:19.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:19 smithi064 ceph-mon[212750]: failed to encode map e677 with expected crc 2023-10-09T06:21:19.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:19 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:19.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:19 smithi064 ceph-mon[220044]: failed to encode map e675 with expected crc 2023-10-09T06:21:19.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:19 smithi064 ceph-mon[220044]: failed to encode map e676 with expected crc 2023-10-09T06:21:19.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:19 smithi064 ceph-mon[220044]: failed to encode map e676 with expected crc 2023-10-09T06:21:19.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:19 smithi064 ceph-mon[220044]: failed to encode map e676 with expected crc 2023-10-09T06:21:19.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:19 smithi064 ceph-mon[220044]: failed to encode map e677 with expected crc 2023-10-09T06:21:19.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:19 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1743599400' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T06:21:19.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:19 smithi064 ceph-mon[220044]: osdmap e677: 8 total, 8 up, 7 in 2023-10-09T06:21:19.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:19 smithi064 ceph-mon[220044]: failed to encode map e677 with expected crc 2023-10-09T06:21:19.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:19 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:20.037 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:20.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:20 smithi156 ceph-mon[195101]: failed to encode map e677 with expected crc 2023-10-09T06:21:20.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:20 smithi156 ceph-mon[195101]: pgmap v1720: 103 pgs: 2 activating+undersized+remapped, 3 active+undersized+remapped, 3 remapped+peering, 16 peering, 1 active+remapped, 78 active+clean; 4.6 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 138 B/s, 1 objects/s recovering 2023-10-09T06:21:20.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:20 smithi156 ceph-mon[195101]: osdmap e678: 8 total, 8 up, 7 in 2023-10-09T06:21:20.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:20 smithi156 ceph-mon[195101]: failed to encode map e678 with expected crc 2023-10-09T06:21:20.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:20 smithi156 ceph-mon[195101]: failed to encode map e678 with expected crc 2023-10-09T06:21:20.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:20 smithi156 ceph-mon[195101]: failed to encode map e678 with expected crc 2023-10-09T06:21:20.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:20 smithi156 ceph-mon[195101]: failed to encode map e678 with expected crc 2023-10-09T06:21:20.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:20 smithi156 ceph-mon[195101]: failed to encode map e678 with expected crc 2023-10-09T06:21:20.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:20 smithi064 ceph-mon[212750]: failed to encode map e677 with expected crc 2023-10-09T06:21:20.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:20 smithi064 ceph-mon[212750]: pgmap v1720: 103 pgs: 2 activating+undersized+remapped, 3 active+undersized+remapped, 3 remapped+peering, 16 peering, 1 active+remapped, 78 active+clean; 4.6 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 138 B/s, 1 objects/s recovering 2023-10-09T06:21:20.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:20 smithi064 ceph-mon[212750]: osdmap e678: 8 total, 8 up, 7 in 2023-10-09T06:21:20.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:20 smithi064 ceph-mon[212750]: failed to encode map e678 with expected crc 2023-10-09T06:21:20.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:20 smithi064 ceph-mon[212750]: failed to encode map e678 with expected crc 2023-10-09T06:21:20.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:20 smithi064 ceph-mon[212750]: failed to encode map e678 with expected crc 2023-10-09T06:21:20.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:20 smithi064 ceph-mon[212750]: failed to encode map e678 with expected crc 2023-10-09T06:21:20.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:20 smithi064 ceph-mon[212750]: failed to encode map e678 with expected crc 2023-10-09T06:21:20.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:20 smithi064 ceph-mon[220044]: failed to encode map e677 with expected crc 2023-10-09T06:21:20.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:20 smithi064 ceph-mon[220044]: pgmap v1720: 103 pgs: 2 activating+undersized+remapped, 3 active+undersized+remapped, 3 remapped+peering, 16 peering, 1 active+remapped, 78 active+clean; 4.6 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 138 B/s, 1 objects/s recovering 2023-10-09T06:21:20.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:20 smithi064 ceph-mon[220044]: osdmap e678: 8 total, 8 up, 7 in 2023-10-09T06:21:20.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:20 smithi064 ceph-mon[220044]: failed to encode map e678 with expected crc 2023-10-09T06:21:20.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:20 smithi064 ceph-mon[220044]: failed to encode map e678 with expected crc 2023-10-09T06:21:20.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:20 smithi064 ceph-mon[220044]: failed to encode map e678 with expected crc 2023-10-09T06:21:20.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:20 smithi064 ceph-mon[220044]: failed to encode map e678 with expected crc 2023-10-09T06:21:20.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:20 smithi064 ceph-mon[220044]: failed to encode map e678 with expected crc 2023-10-09T06:21:21.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[220044]: failed to encode map e678 with expected crc 2023-10-09T06:21:21.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[220044]: failed to encode map e678 with expected crc 2023-10-09T06:21:21.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[220044]: failed to encode map e678 with expected crc 2023-10-09T06:21:21.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[220044]: failed to encode map e678 with expected crc 2023-10-09T06:21:21.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[220044]: failed to encode map e678 with expected crc 2023-10-09T06:21:21.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[220044]: failed to encode map e679 with expected crc 2023-10-09T06:21:21.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[220044]: osdmap e679: 8 total, 8 up, 7 in 2023-10-09T06:21:21.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[220044]: failed to encode map e679 with expected crc 2023-10-09T06:21:21.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[220044]: failed to encode map e679 with expected crc 2023-10-09T06:21:21.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[220044]: failed to encode map e679 with expected crc 2023-10-09T06:21:21.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[220044]: failed to encode map e679 with expected crc 2023-10-09T06:21:21.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[220044]: failed to encode map e679 with expected crc 2023-10-09T06:21:21.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[220044]: failed to encode map e679 with expected crc 2023-10-09T06:21:21.720 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[212750]: failed to encode map e678 with expected crc 2023-10-09T06:21:21.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[212750]: failed to encode map e678 with expected crc 2023-10-09T06:21:21.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[212750]: failed to encode map e678 with expected crc 2023-10-09T06:21:21.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[212750]: failed to encode map e678 with expected crc 2023-10-09T06:21:21.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[212750]: failed to encode map e678 with expected crc 2023-10-09T06:21:21.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[212750]: failed to encode map e679 with expected crc 2023-10-09T06:21:21.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[212750]: osdmap e679: 8 total, 8 up, 7 in 2023-10-09T06:21:21.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[212750]: failed to encode map e679 with expected crc 2023-10-09T06:21:21.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[212750]: failed to encode map e679 with expected crc 2023-10-09T06:21:21.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[212750]: failed to encode map e679 with expected crc 2023-10-09T06:21:21.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[212750]: failed to encode map e679 with expected crc 2023-10-09T06:21:21.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[212750]: failed to encode map e679 with expected crc 2023-10-09T06:21:21.721 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:21 smithi064 ceph-mon[212750]: failed to encode map e679 with expected crc 2023-10-09T06:21:21.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:21 smithi156 ceph-mon[195101]: failed to encode map e678 with expected crc 2023-10-09T06:21:21.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:21 smithi156 ceph-mon[195101]: failed to encode map e678 with expected crc 2023-10-09T06:21:21.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:21 smithi156 ceph-mon[195101]: failed to encode map e678 with expected crc 2023-10-09T06:21:21.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:21 smithi156 ceph-mon[195101]: failed to encode map e678 with expected crc 2023-10-09T06:21:21.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:21 smithi156 ceph-mon[195101]: failed to encode map e678 with expected crc 2023-10-09T06:21:21.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:21 smithi156 ceph-mon[195101]: failed to encode map e679 with expected crc 2023-10-09T06:21:21.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:21 smithi156 ceph-mon[195101]: osdmap e679: 8 total, 8 up, 7 in 2023-10-09T06:21:21.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:21 smithi156 ceph-mon[195101]: failed to encode map e679 with expected crc 2023-10-09T06:21:21.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:21 smithi156 ceph-mon[195101]: failed to encode map e679 with expected crc 2023-10-09T06:21:21.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:21 smithi156 ceph-mon[195101]: failed to encode map e679 with expected crc 2023-10-09T06:21:21.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:21 smithi156 ceph-mon[195101]: failed to encode map e679 with expected crc 2023-10-09T06:21:21.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:21 smithi156 ceph-mon[195101]: failed to encode map e679 with expected crc 2023-10-09T06:21:21.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:21 smithi156 ceph-mon[195101]: failed to encode map e679 with expected crc 2023-10-09T06:21:21.932 INFO:teuthology.orchestra.run.smithi064.stderr:marked in osd.5. 2023-10-09T06:21:22.125 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:21 smithi064 conmon[212726]: 2023-10-09T06:21:21.839+0000 7f8823f23700 -1 mon.a@0(leader).osd e679 definitely_dead 0 2023-10-09T06:21:22.725 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-10-09T06:21:22.725 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-10-09T06:21:22.726 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 81 2023-10-09T06:21:22.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:22 smithi156 ceph-mon[195101]: failed to encode map e679 with expected crc 2023-10-09T06:21:22.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:22 smithi156 ceph-mon[195101]: failed to encode map e679 with expected crc 2023-10-09T06:21:22.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:22 smithi156 ceph-mon[195101]: pgmap v1723: 103 pgs: 4 active+undersized+remapped, 7 peering, 92 active+clean; 4.6 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 1024 KiB/s, 2 keys/s, 6 objects/s recovering 2023-10-09T06:21:22.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:22 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3977297324' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T06:21:22.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:22 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T06:21:22.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:22 smithi156 ceph-mon[195101]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:21:22.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:22 smithi156 ceph-mon[195101]: Cluster is now healthy 2023-10-09T06:21:22.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:22 smithi156 ceph-mon[195101]: failed to encode map e680 with expected crc 2023-10-09T06:21:22.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:22 smithi156 ceph-mon[195101]: failed to encode map e680 with expected crc 2023-10-09T06:21:22.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:22 smithi156 ceph-mon[195101]: failed to encode map e680 with expected crc 2023-10-09T06:21:22.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:22 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T06:21:22.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:22 smithi156 ceph-mon[195101]: osdmap e680: 8 total, 8 up, 8 in 2023-10-09T06:21:22.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:22 smithi156 ceph-mon[195101]: failed to encode map e680 with expected crc 2023-10-09T06:21:22.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:22 smithi156 ceph-mon[195101]: failed to encode map e680 with expected crc 2023-10-09T06:21:22.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[212750]: failed to encode map e679 with expected crc 2023-10-09T06:21:22.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[212750]: failed to encode map e679 with expected crc 2023-10-09T06:21:22.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[212750]: pgmap v1723: 103 pgs: 4 active+undersized+remapped, 7 peering, 92 active+clean; 4.6 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 1024 KiB/s, 2 keys/s, 6 objects/s recovering 2023-10-09T06:21:22.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3977297324' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T06:21:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T06:21:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:21:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:21:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[212750]: failed to encode map e680 with expected crc 2023-10-09T06:21:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[212750]: failed to encode map e680 with expected crc 2023-10-09T06:21:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[212750]: failed to encode map e680 with expected crc 2023-10-09T06:21:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T06:21:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[212750]: osdmap e680: 8 total, 8 up, 8 in 2023-10-09T06:21:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[212750]: failed to encode map e680 with expected crc 2023-10-09T06:21:22.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[212750]: failed to encode map e680 with expected crc 2023-10-09T06:21:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[220044]: failed to encode map e679 with expected crc 2023-10-09T06:21:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[220044]: failed to encode map e679 with expected crc 2023-10-09T06:21:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[220044]: pgmap v1723: 103 pgs: 4 active+undersized+remapped, 7 peering, 92 active+clean; 4.6 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 1024 KiB/s, 2 keys/s, 6 objects/s recovering 2023-10-09T06:21:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3977297324' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T06:21:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T06:21:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[220044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:21:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[220044]: Cluster is now healthy 2023-10-09T06:21:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[220044]: failed to encode map e680 with expected crc 2023-10-09T06:21:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[220044]: failed to encode map e680 with expected crc 2023-10-09T06:21:22.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[220044]: failed to encode map e680 with expected crc 2023-10-09T06:21:22.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T06:21:22.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[220044]: osdmap e680: 8 total, 8 up, 8 in 2023-10-09T06:21:22.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[220044]: failed to encode map e680 with expected crc 2023-10-09T06:21:22.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:22 smithi064 ceph-mon[220044]: failed to encode map e680 with expected crc 2023-10-09T06:21:23.129 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:23.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:23 smithi064 ceph-mon[212750]: failed to encode map e680 with expected crc 2023-10-09T06:21:23.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:23 smithi064 ceph-mon[212750]: failed to encode map e680 with expected crc 2023-10-09T06:21:23.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:23 smithi064 ceph-mon[212750]: failed to encode map e680 with expected crc 2023-10-09T06:21:23.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:23 smithi064 ceph-mon[212750]: failed to encode map e681 with expected crc 2023-10-09T06:21:23.691 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:23 smithi064 ceph-mon[212750]: osdmap e681: 8 total, 8 up, 8 in 2023-10-09T06:21:23.692 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:23 smithi064 ceph-mon[212750]: failed to encode map e681 with expected crc 2023-10-09T06:21:23.692 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:23 smithi064 ceph-mon[212750]: failed to encode map e681 with expected crc 2023-10-09T06:21:23.692 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:23 smithi064 ceph-mon[212750]: failed to encode map e681 with expected crc 2023-10-09T06:21:23.692 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:23 smithi064 ceph-mon[212750]: failed to encode map e681 with expected crc 2023-10-09T06:21:23.692 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:23 smithi064 ceph-mon[212750]: failed to encode map e681 with expected crc 2023-10-09T06:21:23.692 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:23 smithi064 ceph-mon[212750]: failed to encode map e681 with expected crc 2023-10-09T06:21:23.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:23 smithi064 ceph-mon[220044]: failed to encode map e680 with expected crc 2023-10-09T06:21:23.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:23 smithi064 ceph-mon[220044]: failed to encode map e680 with expected crc 2023-10-09T06:21:23.692 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:23 smithi064 ceph-mon[220044]: failed to encode map e680 with expected crc 2023-10-09T06:21:23.693 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:23 smithi064 ceph-mon[220044]: failed to encode map e681 with expected crc 2023-10-09T06:21:23.693 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:23 smithi064 ceph-mon[220044]: osdmap e681: 8 total, 8 up, 8 in 2023-10-09T06:21:23.693 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:23 smithi064 ceph-mon[220044]: failed to encode map e681 with expected crc 2023-10-09T06:21:23.693 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:23 smithi064 ceph-mon[220044]: failed to encode map e681 with expected crc 2023-10-09T06:21:23.693 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:23 smithi064 ceph-mon[220044]: failed to encode map e681 with expected crc 2023-10-09T06:21:23.693 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:23 smithi064 ceph-mon[220044]: failed to encode map e681 with expected crc 2023-10-09T06:21:23.693 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:23 smithi064 ceph-mon[220044]: failed to encode map e681 with expected crc 2023-10-09T06:21:23.693 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:23 smithi064 ceph-mon[220044]: failed to encode map e681 with expected crc 2023-10-09T06:21:23.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:23 smithi156 ceph-mon[195101]: failed to encode map e680 with expected crc 2023-10-09T06:21:23.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:23 smithi156 ceph-mon[195101]: failed to encode map e680 with expected crc 2023-10-09T06:21:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:23 smithi156 ceph-mon[195101]: failed to encode map e680 with expected crc 2023-10-09T06:21:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:23 smithi156 ceph-mon[195101]: failed to encode map e681 with expected crc 2023-10-09T06:21:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:23 smithi156 ceph-mon[195101]: osdmap e681: 8 total, 8 up, 8 in 2023-10-09T06:21:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:23 smithi156 ceph-mon[195101]: failed to encode map e681 with expected crc 2023-10-09T06:21:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:23 smithi156 ceph-mon[195101]: failed to encode map e681 with expected crc 2023-10-09T06:21:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:23 smithi156 ceph-mon[195101]: failed to encode map e681 with expected crc 2023-10-09T06:21:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:23 smithi156 ceph-mon[195101]: failed to encode map e681 with expected crc 2023-10-09T06:21:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:23 smithi156 ceph-mon[195101]: failed to encode map e681 with expected crc 2023-10-09T06:21:23.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:23 smithi156 ceph-mon[195101]: failed to encode map e681 with expected crc 2023-10-09T06:21:24.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:24 smithi156 ceph-mon[195101]: failed to encode map e681 with expected crc 2023-10-09T06:21:24.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:24 smithi156 ceph-mon[195101]: failed to encode map e681 with expected crc 2023-10-09T06:21:24.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:24 smithi156 ceph-mon[195101]: failed to encode map e681 with expected crc 2023-10-09T06:21:24.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:24 smithi156 ceph-mon[195101]: failed to encode map e681 with expected crc 2023-10-09T06:21:24.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:24 smithi156 ceph-mon[195101]: pgmap v1726: 103 pgs: 4 active+undersized+remapped, 7 peering, 92 active+clean; 4.6 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 1024 KiB/s, 2 keys/s, 6 objects/s recovering 2023-10-09T06:21:24.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:24 smithi156 ceph-mon[195101]: failed to encode map e682 with expected crc 2023-10-09T06:21:24.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:24 smithi156 ceph-mon[195101]: osdmap e682: 8 total, 8 up, 8 in 2023-10-09T06:21:24.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:24 smithi156 ceph-mon[195101]: failed to encode map e682 with expected crc 2023-10-09T06:21:24.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:24 smithi156 ceph-mon[195101]: failed to encode map e682 with expected crc 2023-10-09T06:21:24.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:24 smithi156 ceph-mon[195101]: failed to encode map e682 with expected crc 2023-10-09T06:21:24.923 INFO:teuthology.orchestra.run.smithi064.stderr:Error EINVAL: specified pgp_num 81 > pg_num 31 2023-10-09T06:21:24.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:24 smithi064 ceph-mon[212750]: failed to encode map e681 with expected crc 2023-10-09T06:21:24.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:24 smithi064 ceph-mon[212750]: failed to encode map e681 with expected crc 2023-10-09T06:21:24.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:24 smithi064 ceph-mon[212750]: failed to encode map e681 with expected crc 2023-10-09T06:21:24.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:24 smithi064 ceph-mon[212750]: failed to encode map e681 with expected crc 2023-10-09T06:21:24.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:24 smithi064 ceph-mon[212750]: pgmap v1726: 103 pgs: 4 active+undersized+remapped, 7 peering, 92 active+clean; 4.6 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 1024 KiB/s, 2 keys/s, 6 objects/s recovering 2023-10-09T06:21:24.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:24 smithi064 ceph-mon[212750]: failed to encode map e682 with expected crc 2023-10-09T06:21:24.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:24 smithi064 ceph-mon[212750]: osdmap e682: 8 total, 8 up, 8 in 2023-10-09T06:21:24.941 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:24 smithi064 ceph-mon[212750]: failed to encode map e682 with expected crc 2023-10-09T06:21:24.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:24 smithi064 ceph-mon[212750]: failed to encode map e682 with expected crc 2023-10-09T06:21:24.942 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:24 smithi064 ceph-mon[212750]: failed to encode map e682 with expected crc 2023-10-09T06:21:24.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:24 smithi064 ceph-mon[220044]: failed to encode map e681 with expected crc 2023-10-09T06:21:24.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:24 smithi064 ceph-mon[220044]: failed to encode map e681 with expected crc 2023-10-09T06:21:24.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:24 smithi064 ceph-mon[220044]: failed to encode map e681 with expected crc 2023-10-09T06:21:24.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:24 smithi064 ceph-mon[220044]: failed to encode map e681 with expected crc 2023-10-09T06:21:24.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:24 smithi064 ceph-mon[220044]: pgmap v1726: 103 pgs: 4 active+undersized+remapped, 7 peering, 92 active+clean; 4.6 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 1024 KiB/s, 2 keys/s, 6 objects/s recovering 2023-10-09T06:21:24.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:24 smithi064 ceph-mon[220044]: failed to encode map e682 with expected crc 2023-10-09T06:21:24.942 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:24 smithi064 ceph-mon[220044]: osdmap e682: 8 total, 8 up, 8 in 2023-10-09T06:21:24.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:24 smithi064 ceph-mon[220044]: failed to encode map e682 with expected crc 2023-10-09T06:21:24.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:24 smithi064 ceph-mon[220044]: failed to encode map e682 with expected crc 2023-10-09T06:21:24.943 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:24 smithi064 ceph-mon[220044]: failed to encode map e682 with expected crc 2023-10-09T06:21:25.650 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:21:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:21:25] "GET /metrics HTTP/1.1" 200 34112 "" "Prometheus/2.43.0" 2023-10-09T06:21:25.651 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:25 smithi064 ceph-mon[212750]: failed to encode map e682 with expected crc 2023-10-09T06:21:25.651 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:25 smithi064 ceph-mon[212750]: failed to encode map e682 with expected crc 2023-10-09T06:21:25.651 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:25 smithi064 ceph-mon[212750]: failed to encode map e683 with expected crc 2023-10-09T06:21:25.651 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:25 smithi064 ceph-mon[212750]: osdmap e683: 8 total, 8 up, 8 in 2023-10-09T06:21:25.651 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:25 smithi064 ceph-mon[212750]: failed to encode map e683 with expected crc 2023-10-09T06:21:25.651 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:25 smithi064 ceph-mon[212750]: failed to encode map e683 with expected crc 2023-10-09T06:21:25.651 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:25 smithi064 ceph-mon[212750]: failed to encode map e683 with expected crc 2023-10-09T06:21:25.652 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:25 smithi064 ceph-mon[212750]: failed to encode map e683 with expected crc 2023-10-09T06:21:25.652 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:25 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3237629294' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "81"}]: dispatch 2023-10-09T06:21:25.652 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:25 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "81"}]: dispatch 2023-10-09T06:21:25.652 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:25 smithi064 ceph-mon[220044]: failed to encode map e682 with expected crc 2023-10-09T06:21:25.652 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:25 smithi064 ceph-mon[220044]: failed to encode map e682 with expected crc 2023-10-09T06:21:25.652 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:25 smithi064 ceph-mon[220044]: failed to encode map e683 with expected crc 2023-10-09T06:21:25.652 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:25 smithi064 ceph-mon[220044]: osdmap e683: 8 total, 8 up, 8 in 2023-10-09T06:21:25.653 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:25 smithi064 ceph-mon[220044]: failed to encode map e683 with expected crc 2023-10-09T06:21:25.653 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:25 smithi064 ceph-mon[220044]: failed to encode map e683 with expected crc 2023-10-09T06:21:25.653 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:25 smithi064 ceph-mon[220044]: failed to encode map e683 with expected crc 2023-10-09T06:21:25.653 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:25 smithi064 ceph-mon[220044]: failed to encode map e683 with expected crc 2023-10-09T06:21:25.653 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:25 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3237629294' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "81"}]: dispatch 2023-10-09T06:21:25.653 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:25 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "81"}]: dispatch 2023-10-09T06:21:25.666 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T06:21:25.667 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-10-09T06:21:25.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:25 smithi156 ceph-mon[195101]: failed to encode map e682 with expected crc 2023-10-09T06:21:25.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:25 smithi156 ceph-mon[195101]: failed to encode map e682 with expected crc 2023-10-09T06:21:25.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:25 smithi156 ceph-mon[195101]: failed to encode map e683 with expected crc 2023-10-09T06:21:25.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:25 smithi156 ceph-mon[195101]: osdmap e683: 8 total, 8 up, 8 in 2023-10-09T06:21:25.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:25 smithi156 ceph-mon[195101]: failed to encode map e683 with expected crc 2023-10-09T06:21:25.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:25 smithi156 ceph-mon[195101]: failed to encode map e683 with expected crc 2023-10-09T06:21:25.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:25 smithi156 ceph-mon[195101]: failed to encode map e683 with expected crc 2023-10-09T06:21:25.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:25 smithi156 ceph-mon[195101]: failed to encode map e683 with expected crc 2023-10-09T06:21:25.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:25 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3237629294' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "81"}]: dispatch 2023-10-09T06:21:25.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:25 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "81"}]: dispatch 2023-10-09T06:21:26.043 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:26.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:26 smithi156 ceph-mon[195101]: failed to encode map e683 with expected crc 2023-10-09T06:21:26.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:26 smithi156 ceph-mon[195101]: failed to encode map e683 with expected crc 2023-10-09T06:21:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:26 smithi156 ceph-mon[195101]: failed to encode map e683 with expected crc 2023-10-09T06:21:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:26 smithi156 ceph-mon[195101]: failed to encode map e683 with expected crc 2023-10-09T06:21:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:26 smithi156 ceph-mon[195101]: failed to encode map e683 with expected crc 2023-10-09T06:21:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:26 smithi156 ceph-mon[195101]: pgmap v1729: 103 pgs: 2 peering, 13 active+undersized+remapped, 88 active+clean; 4.6 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 64 KiB/s, 2 keys/s, 14 objects/s recovering 2023-10-09T06:21:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:26 smithi156 ceph-mon[195101]: osdmap e684: 8 total, 8 up, 8 in 2023-10-09T06:21:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:26 smithi156 ceph-mon[195101]: failed to encode map e684 with expected crc 2023-10-09T06:21:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:26 smithi156 ceph-mon[195101]: failed to encode map e684 with expected crc 2023-10-09T06:21:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:26 smithi156 ceph-mon[195101]: failed to encode map e684 with expected crc 2023-10-09T06:21:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:26 smithi156 ceph-mon[195101]: failed to encode map e684 with expected crc 2023-10-09T06:21:26.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:26 smithi156 ceph-mon[195101]: failed to encode map e684 with expected crc 2023-10-09T06:21:26.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:26 smithi064 ceph-mon[212750]: failed to encode map e683 with expected crc 2023-10-09T06:21:26.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:26 smithi064 ceph-mon[212750]: failed to encode map e683 with expected crc 2023-10-09T06:21:26.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:26 smithi064 ceph-mon[212750]: failed to encode map e683 with expected crc 2023-10-09T06:21:26.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:26 smithi064 ceph-mon[212750]: failed to encode map e683 with expected crc 2023-10-09T06:21:26.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:26 smithi064 ceph-mon[212750]: failed to encode map e683 with expected crc 2023-10-09T06:21:26.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:26 smithi064 ceph-mon[212750]: pgmap v1729: 103 pgs: 2 peering, 13 active+undersized+remapped, 88 active+clean; 4.6 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 64 KiB/s, 2 keys/s, 14 objects/s recovering 2023-10-09T06:21:26.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:26 smithi064 ceph-mon[212750]: osdmap e684: 8 total, 8 up, 8 in 2023-10-09T06:21:26.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:26 smithi064 ceph-mon[212750]: failed to encode map e684 with expected crc 2023-10-09T06:21:26.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:26 smithi064 ceph-mon[212750]: failed to encode map e684 with expected crc 2023-10-09T06:21:26.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:26 smithi064 ceph-mon[212750]: failed to encode map e684 with expected crc 2023-10-09T06:21:26.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:26 smithi064 ceph-mon[212750]: failed to encode map e684 with expected crc 2023-10-09T06:21:26.971 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:26 smithi064 ceph-mon[212750]: failed to encode map e684 with expected crc 2023-10-09T06:21:26.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:26 smithi064 ceph-mon[220044]: failed to encode map e683 with expected crc 2023-10-09T06:21:26.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:26 smithi064 ceph-mon[220044]: failed to encode map e683 with expected crc 2023-10-09T06:21:26.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:26 smithi064 ceph-mon[220044]: failed to encode map e683 with expected crc 2023-10-09T06:21:26.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:26 smithi064 ceph-mon[220044]: failed to encode map e683 with expected crc 2023-10-09T06:21:26.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:26 smithi064 ceph-mon[220044]: failed to encode map e683 with expected crc 2023-10-09T06:21:26.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:26 smithi064 ceph-mon[220044]: pgmap v1729: 103 pgs: 2 peering, 13 active+undersized+remapped, 88 active+clean; 4.6 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 64 KiB/s, 2 keys/s, 14 objects/s recovering 2023-10-09T06:21:26.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:26 smithi064 ceph-mon[220044]: osdmap e684: 8 total, 8 up, 8 in 2023-10-09T06:21:26.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:26 smithi064 ceph-mon[220044]: failed to encode map e684 with expected crc 2023-10-09T06:21:26.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:26 smithi064 ceph-mon[220044]: failed to encode map e684 with expected crc 2023-10-09T06:21:26.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:26 smithi064 ceph-mon[220044]: failed to encode map e684 with expected crc 2023-10-09T06:21:26.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:26 smithi064 ceph-mon[220044]: failed to encode map e684 with expected crc 2023-10-09T06:21:26.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:26 smithi064 ceph-mon[220044]: failed to encode map e684 with expected crc 2023-10-09T06:21:27.560 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a: {} 2023-10-09T06:21:27.571 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b: {} 2023-10-09T06:21:27.580 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c: {} 2023-10-09T06:21:27.581 INFO:teuthology.orchestra.run.smithi064.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T06:21:27.581 INFO:teuthology.orchestra.run.smithi064.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T06:21:27.582 INFO:teuthology.orchestra.run.smithi064.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T06:21:27.602 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:27 smithi064 ceph-mon[212750]: failed to encode map e684 with expected crc 2023-10-09T06:21:27.602 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:27 smithi064 ceph-mon[212750]: failed to encode map e684 with expected crc 2023-10-09T06:21:27.602 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:27 smithi064 ceph-mon[212750]: failed to encode map e684 with expected crc 2023-10-09T06:21:27.603 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:27 smithi064 ceph-mon[212750]: failed to encode map e684 with expected crc 2023-10-09T06:21:27.603 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:27 smithi064 ceph-mon[212750]: failed to encode map e684 with expected crc 2023-10-09T06:21:27.603 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:27 smithi064 ceph-mon[212750]: failed to encode map e684 with expected crc 2023-10-09T06:21:27.603 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:27 smithi064 ceph-mon[212750]: failed to encode map e684 with expected crc 2023-10-09T06:21:27.603 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:27 smithi064 ceph-mon[220044]: failed to encode map e684 with expected crc 2023-10-09T06:21:27.603 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:27 smithi064 ceph-mon[220044]: failed to encode map e684 with expected crc 2023-10-09T06:21:27.603 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:27 smithi064 ceph-mon[220044]: failed to encode map e684 with expected crc 2023-10-09T06:21:27.603 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:27 smithi064 ceph-mon[220044]: failed to encode map e684 with expected crc 2023-10-09T06:21:27.603 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:27 smithi064 ceph-mon[220044]: failed to encode map e684 with expected crc 2023-10-09T06:21:27.603 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:27 smithi064 ceph-mon[220044]: failed to encode map e684 with expected crc 2023-10-09T06:21:27.604 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:27 smithi064 ceph-mon[220044]: failed to encode map e684 with expected crc 2023-10-09T06:21:27.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:27 smithi156 ceph-mon[195101]: failed to encode map e684 with expected crc 2023-10-09T06:21:27.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:27 smithi156 ceph-mon[195101]: failed to encode map e684 with expected crc 2023-10-09T06:21:27.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:27 smithi156 ceph-mon[195101]: failed to encode map e684 with expected crc 2023-10-09T06:21:27.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:27 smithi156 ceph-mon[195101]: failed to encode map e684 with expected crc 2023-10-09T06:21:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:27 smithi156 ceph-mon[195101]: failed to encode map e684 with expected crc 2023-10-09T06:21:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:27 smithi156 ceph-mon[195101]: failed to encode map e684 with expected crc 2023-10-09T06:21:27.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:27 smithi156 ceph-mon[195101]: failed to encode map e684 with expected crc 2023-10-09T06:21:28.300 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T06:21:28.669 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:28.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:28 smithi064 ceph-mon[212750]: pgmap v1731: 103 pgs: 2 peering, 9 active+undersized+remapped, 92 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 57 KiB/s, 2 keys/s, 14 objects/s recovering 2023-10-09T06:21:28.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:28 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3543365271' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T06:21:28.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:28 smithi064 ceph-mon[220044]: pgmap v1731: 103 pgs: 2 peering, 9 active+undersized+remapped, 92 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 57 KiB/s, 2 keys/s, 14 objects/s recovering 2023-10-09T06:21:28.720 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:28 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3543365271' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T06:21:28.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:28 smithi156 ceph-mon[195101]: pgmap v1731: 103 pgs: 2 peering, 9 active+undersized+remapped, 92 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 57 KiB/s, 2 keys/s, 14 objects/s recovering 2023-10-09T06:21:28.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:28 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3543365271' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T06:21:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:29 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:29.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:29 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:29.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:29 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:29.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:29 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:29.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:29 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:29.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:29 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:29.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:29 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:29.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:29 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:29.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:29 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:30.567 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T06:21:30.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:30 smithi156 ceph-mon[195101]: pgmap v1732: 103 pgs: 2 peering, 9 active+undersized+remapped, 92 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 43 KiB/s, 1 keys/s, 10 objects/s recovering 2023-10-09T06:21:30.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:30 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3001296952' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:21:30.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:30 smithi064 ceph-mon[212750]: pgmap v1732: 103 pgs: 2 peering, 9 active+undersized+remapped, 92 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 43 KiB/s, 1 keys/s, 10 objects/s recovering 2023-10-09T06:21:30.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:30 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3001296952' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:21:30.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:30 smithi064 ceph-mon[220044]: pgmap v1732: 103 pgs: 2 peering, 9 active+undersized+remapped, 92 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 43 KiB/s, 1 keys/s, 10 objects/s recovering 2023-10-09T06:21:30.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:30 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3001296952' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:21:31.264 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-10-09T06:21:31.657 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:31.857 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:31 smithi064 ceph-mon[212750]: failed to encode map e685 with expected crc 2023-10-09T06:21:31.857 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:31 smithi064 ceph-mon[212750]: failed to encode map e685 with expected crc 2023-10-09T06:21:31.857 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:31 smithi064 ceph-mon[212750]: failed to encode map e685 with expected crc 2023-10-09T06:21:31.857 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:31 smithi064 ceph-mon[212750]: failed to encode map e685 with expected crc 2023-10-09T06:21:31.857 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:31 smithi064 ceph-mon[212750]: failed to encode map e685 with expected crc 2023-10-09T06:21:31.857 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:31 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3001296952' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:21:31.857 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:31 smithi064 ceph-mon[212750]: osdmap e685: 8 total, 8 up, 8 in 2023-10-09T06:21:31.858 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:31 smithi064 ceph-mon[212750]: failed to encode map e685 with expected crc 2023-10-09T06:21:31.858 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:31 smithi064 ceph-mon[212750]: osdmap e686: 8 total, 8 up, 8 in 2023-10-09T06:21:31.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:31 smithi064 ceph-mon[220044]: failed to encode map e685 with expected crc 2023-10-09T06:21:31.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:31 smithi064 ceph-mon[220044]: failed to encode map e685 with expected crc 2023-10-09T06:21:31.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:31 smithi064 ceph-mon[220044]: failed to encode map e685 with expected crc 2023-10-09T06:21:31.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:31 smithi064 ceph-mon[220044]: failed to encode map e685 with expected crc 2023-10-09T06:21:31.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:31 smithi064 ceph-mon[220044]: failed to encode map e685 with expected crc 2023-10-09T06:21:31.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:31 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3001296952' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:21:31.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:31 smithi064 ceph-mon[220044]: osdmap e685: 8 total, 8 up, 8 in 2023-10-09T06:21:31.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:31 smithi064 ceph-mon[220044]: failed to encode map e685 with expected crc 2023-10-09T06:21:31.859 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:31 smithi064 ceph-mon[220044]: osdmap e686: 8 total, 8 up, 8 in 2023-10-09T06:21:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:31 smithi156 ceph-mon[195101]: failed to encode map e685 with expected crc 2023-10-09T06:21:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:31 smithi156 ceph-mon[195101]: failed to encode map e685 with expected crc 2023-10-09T06:21:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:31 smithi156 ceph-mon[195101]: failed to encode map e685 with expected crc 2023-10-09T06:21:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:31 smithi156 ceph-mon[195101]: failed to encode map e685 with expected crc 2023-10-09T06:21:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:31 smithi156 ceph-mon[195101]: failed to encode map e685 with expected crc 2023-10-09T06:21:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:31 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3001296952' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:21:31.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:31 smithi156 ceph-mon[195101]: osdmap e685: 8 total, 8 up, 8 in 2023-10-09T06:21:31.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:31 smithi156 ceph-mon[195101]: failed to encode map e685 with expected crc 2023-10-09T06:21:31.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:31 smithi156 ceph-mon[195101]: osdmap e686: 8 total, 8 up, 8 in 2023-10-09T06:21:32.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:32 smithi156 ceph-mon[195101]: pgmap v1734: 103 pgs: 12 peering, 91 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 88 B/s, 1 objects/s recovering 2023-10-09T06:21:32.870 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:32 smithi156 ceph-mon[195101]: failed to encode map e686 with expected crc 2023-10-09T06:21:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:32 smithi156 ceph-mon[195101]: failed to encode map e686 with expected crc 2023-10-09T06:21:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:32 smithi156 ceph-mon[195101]: failed to encode map e686 with expected crc 2023-10-09T06:21:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:32 smithi156 ceph-mon[195101]: failed to encode map e686 with expected crc 2023-10-09T06:21:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:32 smithi156 ceph-mon[195101]: failed to encode map e686 with expected crc 2023-10-09T06:21:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:32 smithi156 ceph-mon[195101]: failed to encode map e686 with expected crc 2023-10-09T06:21:32.871 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:32 smithi156 ceph-mon[195101]: failed to encode map e686 with expected crc 2023-10-09T06:21:32.872 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:32 smithi156 ceph-mon[195101]: osdmap e687: 8 total, 8 up, 8 in 2023-10-09T06:21:32.967 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:32 smithi064 ceph-mon[212750]: pgmap v1734: 103 pgs: 12 peering, 91 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 88 B/s, 1 objects/s recovering 2023-10-09T06:21:32.967 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:32 smithi064 ceph-mon[212750]: failed to encode map e686 with expected crc 2023-10-09T06:21:32.967 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:32 smithi064 ceph-mon[212750]: failed to encode map e686 with expected crc 2023-10-09T06:21:32.968 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:32 smithi064 ceph-mon[212750]: failed to encode map e686 with expected crc 2023-10-09T06:21:32.968 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:32 smithi064 ceph-mon[212750]: failed to encode map e686 with expected crc 2023-10-09T06:21:32.968 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:32 smithi064 ceph-mon[212750]: failed to encode map e686 with expected crc 2023-10-09T06:21:32.968 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:32 smithi064 ceph-mon[212750]: failed to encode map e686 with expected crc 2023-10-09T06:21:32.968 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:32 smithi064 ceph-mon[212750]: failed to encode map e686 with expected crc 2023-10-09T06:21:32.968 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:32 smithi064 ceph-mon[212750]: osdmap e687: 8 total, 8 up, 8 in 2023-10-09T06:21:32.968 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:32 smithi064 ceph-mon[220044]: pgmap v1734: 103 pgs: 12 peering, 91 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 88 B/s, 1 objects/s recovering 2023-10-09T06:21:32.968 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:32 smithi064 ceph-mon[220044]: failed to encode map e686 with expected crc 2023-10-09T06:21:32.968 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:32 smithi064 ceph-mon[220044]: failed to encode map e686 with expected crc 2023-10-09T06:21:32.968 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:32 smithi064 ceph-mon[220044]: failed to encode map e686 with expected crc 2023-10-09T06:21:32.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:32 smithi064 ceph-mon[220044]: failed to encode map e686 with expected crc 2023-10-09T06:21:32.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:32 smithi064 ceph-mon[220044]: failed to encode map e686 with expected crc 2023-10-09T06:21:32.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:32 smithi064 ceph-mon[220044]: failed to encode map e686 with expected crc 2023-10-09T06:21:32.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:32 smithi064 ceph-mon[220044]: failed to encode map e686 with expected crc 2023-10-09T06:21:32.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:32 smithi064 ceph-mon[220044]: osdmap e687: 8 total, 8 up, 8 in 2023-10-09T06:21:33.892 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-10-09T06:21:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:33 smithi064 ceph-mon[212750]: failed to encode map e687 with expected crc 2023-10-09T06:21:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:33 smithi064 ceph-mon[212750]: failed to encode map e687 with expected crc 2023-10-09T06:21:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:33 smithi064 ceph-mon[212750]: failed to encode map e687 with expected crc 2023-10-09T06:21:34.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:33 smithi064 ceph-mon[212750]: failed to encode map e687 with expected crc 2023-10-09T06:21:34.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:33 smithi064 ceph-mon[212750]: failed to encode map e687 with expected crc 2023-10-09T06:21:34.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:33 smithi064 ceph-mon[212750]: failed to encode map e687 with expected crc 2023-10-09T06:21:34.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:33 smithi064 ceph-mon[212750]: failed to encode map e687 with expected crc 2023-10-09T06:21:34.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:33 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2258031101' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:21:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:33 smithi064 ceph-mon[220044]: failed to encode map e687 with expected crc 2023-10-09T06:21:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:33 smithi064 ceph-mon[220044]: failed to encode map e687 with expected crc 2023-10-09T06:21:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:33 smithi064 ceph-mon[220044]: failed to encode map e687 with expected crc 2023-10-09T06:21:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:33 smithi064 ceph-mon[220044]: failed to encode map e687 with expected crc 2023-10-09T06:21:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:33 smithi064 ceph-mon[220044]: failed to encode map e687 with expected crc 2023-10-09T06:21:34.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:33 smithi064 ceph-mon[220044]: failed to encode map e687 with expected crc 2023-10-09T06:21:34.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:33 smithi064 ceph-mon[220044]: failed to encode map e687 with expected crc 2023-10-09T06:21:34.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:33 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2258031101' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:21:34.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:33 smithi156 ceph-mon[195101]: failed to encode map e687 with expected crc 2023-10-09T06:21:34.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:33 smithi156 ceph-mon[195101]: failed to encode map e687 with expected crc 2023-10-09T06:21:34.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:33 smithi156 ceph-mon[195101]: failed to encode map e687 with expected crc 2023-10-09T06:21:34.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:33 smithi156 ceph-mon[195101]: failed to encode map e687 with expected crc 2023-10-09T06:21:34.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:33 smithi156 ceph-mon[195101]: failed to encode map e687 with expected crc 2023-10-09T06:21:34.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:33 smithi156 ceph-mon[195101]: failed to encode map e687 with expected crc 2023-10-09T06:21:34.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:33 smithi156 ceph-mon[195101]: failed to encode map e687 with expected crc 2023-10-09T06:21:34.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:33 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2258031101' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:21:34.654 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T06:21:35.030 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:35.215 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:34 smithi064 ceph-mon[220044]: pgmap v1737: 103 pgs: 2 remapped+peering, 12 peering, 89 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:35.215 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:34 smithi064 ceph-mon[220044]: failed to encode map e688 with expected crc 2023-10-09T06:21:35.215 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:34 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2258031101' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:21:35.215 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:34 smithi064 ceph-mon[220044]: osdmap e688: 8 total, 8 up, 8 in 2023-10-09T06:21:35.215 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:34 smithi064 ceph-mon[220044]: failed to encode map e688 with expected crc 2023-10-09T06:21:35.216 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:34 smithi064 ceph-mon[220044]: failed to encode map e688 with expected crc 2023-10-09T06:21:35.216 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:34 smithi064 ceph-mon[220044]: failed to encode map e688 with expected crc 2023-10-09T06:21:35.216 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:34 smithi064 ceph-mon[220044]: failed to encode map e688 with expected crc 2023-10-09T06:21:35.216 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:34 smithi064 ceph-mon[220044]: failed to encode map e688 with expected crc 2023-10-09T06:21:35.216 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:34 smithi064 ceph-mon[220044]: failed to encode map e688 with expected crc 2023-10-09T06:21:35.216 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:34 smithi064 ceph-mon[212750]: pgmap v1737: 103 pgs: 2 remapped+peering, 12 peering, 89 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:35.216 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:34 smithi064 ceph-mon[212750]: failed to encode map e688 with expected crc 2023-10-09T06:21:35.216 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:34 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2258031101' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:21:35.216 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:34 smithi064 ceph-mon[212750]: osdmap e688: 8 total, 8 up, 8 in 2023-10-09T06:21:35.216 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:34 smithi064 ceph-mon[212750]: failed to encode map e688 with expected crc 2023-10-09T06:21:35.217 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:34 smithi064 ceph-mon[212750]: failed to encode map e688 with expected crc 2023-10-09T06:21:35.217 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:34 smithi064 ceph-mon[212750]: failed to encode map e688 with expected crc 2023-10-09T06:21:35.217 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:34 smithi064 ceph-mon[212750]: failed to encode map e688 with expected crc 2023-10-09T06:21:35.217 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:34 smithi064 ceph-mon[212750]: failed to encode map e688 with expected crc 2023-10-09T06:21:35.217 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:34 smithi064 ceph-mon[212750]: failed to encode map e688 with expected crc 2023-10-09T06:21:35.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:34 smithi156 ceph-mon[195101]: pgmap v1737: 103 pgs: 2 remapped+peering, 12 peering, 89 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:35.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:34 smithi156 ceph-mon[195101]: failed to encode map e688 with expected crc 2023-10-09T06:21:35.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:34 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2258031101' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:21:35.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:34 smithi156 ceph-mon[195101]: osdmap e688: 8 total, 8 up, 8 in 2023-10-09T06:21:35.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:34 smithi156 ceph-mon[195101]: failed to encode map e688 with expected crc 2023-10-09T06:21:35.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:34 smithi156 ceph-mon[195101]: failed to encode map e688 with expected crc 2023-10-09T06:21:35.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:34 smithi156 ceph-mon[195101]: failed to encode map e688 with expected crc 2023-10-09T06:21:35.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:34 smithi156 ceph-mon[195101]: failed to encode map e688 with expected crc 2023-10-09T06:21:35.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:34 smithi156 ceph-mon[195101]: failed to encode map e688 with expected crc 2023-10-09T06:21:35.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:34 smithi156 ceph-mon[195101]: failed to encode map e688 with expected crc 2023-10-09T06:21:35.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:21:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:21:35] "GET /metrics HTTP/1.1" 200 34132 "" "Prometheus/2.43.0" 2023-10-09T06:21:36.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:35 smithi064 ceph-mon[212750]: failed to encode map e689 with expected crc 2023-10-09T06:21:36.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:35 smithi064 ceph-mon[212750]: failed to encode map e689 with expected crc 2023-10-09T06:21:36.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:35 smithi064 ceph-mon[212750]: osdmap e689: 8 total, 8 up, 8 in 2023-10-09T06:21:36.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:35 smithi064 ceph-mon[212750]: failed to encode map e689 with expected crc 2023-10-09T06:21:36.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:35 smithi064 ceph-mon[212750]: failed to encode map e689 with expected crc 2023-10-09T06:21:36.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:35 smithi064 ceph-mon[212750]: failed to encode map e689 with expected crc 2023-10-09T06:21:36.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:35 smithi064 ceph-mon[212750]: failed to encode map e689 with expected crc 2023-10-09T06:21:36.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:35 smithi064 ceph-mon[220044]: failed to encode map e689 with expected crc 2023-10-09T06:21:36.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:35 smithi064 ceph-mon[220044]: failed to encode map e689 with expected crc 2023-10-09T06:21:36.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:35 smithi064 ceph-mon[220044]: osdmap e689: 8 total, 8 up, 8 in 2023-10-09T06:21:36.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:35 smithi064 ceph-mon[220044]: failed to encode map e689 with expected crc 2023-10-09T06:21:36.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:35 smithi064 ceph-mon[220044]: failed to encode map e689 with expected crc 2023-10-09T06:21:36.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:35 smithi064 ceph-mon[220044]: failed to encode map e689 with expected crc 2023-10-09T06:21:36.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:35 smithi064 ceph-mon[220044]: failed to encode map e689 with expected crc 2023-10-09T06:21:36.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:35 smithi156 ceph-mon[195101]: failed to encode map e689 with expected crc 2023-10-09T06:21:36.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:35 smithi156 ceph-mon[195101]: failed to encode map e689 with expected crc 2023-10-09T06:21:36.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:35 smithi156 ceph-mon[195101]: osdmap e689: 8 total, 8 up, 8 in 2023-10-09T06:21:36.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:35 smithi156 ceph-mon[195101]: failed to encode map e689 with expected crc 2023-10-09T06:21:36.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:35 smithi156 ceph-mon[195101]: failed to encode map e689 with expected crc 2023-10-09T06:21:36.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:35 smithi156 ceph-mon[195101]: failed to encode map e689 with expected crc 2023-10-09T06:21:36.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:35 smithi156 ceph-mon[195101]: failed to encode map e689 with expected crc 2023-10-09T06:21:36.911 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T06:21:37.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:36 smithi064 ceph-mon[212750]: pgmap v1740: 103 pgs: 13 peering, 90 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 64 KiB/s, 1 objects/s recovering 2023-10-09T06:21:37.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:36 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1031809096' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:21:37.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:36 smithi064 ceph-mon[220044]: pgmap v1740: 103 pgs: 13 peering, 90 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 64 KiB/s, 1 objects/s recovering 2023-10-09T06:21:37.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:36 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1031809096' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:21:37.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:36 smithi156 ceph-mon[195101]: pgmap v1740: 103 pgs: 13 peering, 90 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 64 KiB/s, 1 objects/s recovering 2023-10-09T06:21:37.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:36 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1031809096' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:21:37.706 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-10-09T06:21:37.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:37 smithi064 ceph-mon[212750]: failed to encode map e690 with expected crc 2023-10-09T06:21:37.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:37 smithi064 ceph-mon[212750]: failed to encode map e690 with expected crc 2023-10-09T06:21:37.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:37 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1031809096' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:21:37.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:37 smithi064 ceph-mon[212750]: osdmap e690: 8 total, 8 up, 8 in 2023-10-09T06:21:37.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:37 smithi064 ceph-mon[212750]: failed to encode map e690 with expected crc 2023-10-09T06:21:37.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:37 smithi064 ceph-mon[212750]: failed to encode map e690 with expected crc 2023-10-09T06:21:37.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:37 smithi064 ceph-mon[212750]: failed to encode map e690 with expected crc 2023-10-09T06:21:37.970 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:37 smithi064 ceph-mon[212750]: osdmap e691: 8 total, 8 up, 8 in 2023-10-09T06:21:37.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:37 smithi064 ceph-mon[220044]: failed to encode map e690 with expected crc 2023-10-09T06:21:37.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:37 smithi064 ceph-mon[220044]: failed to encode map e690 with expected crc 2023-10-09T06:21:37.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:37 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1031809096' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:21:37.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:37 smithi064 ceph-mon[220044]: osdmap e690: 8 total, 8 up, 8 in 2023-10-09T06:21:37.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:37 smithi064 ceph-mon[220044]: failed to encode map e690 with expected crc 2023-10-09T06:21:37.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:37 smithi064 ceph-mon[220044]: failed to encode map e690 with expected crc 2023-10-09T06:21:37.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:37 smithi064 ceph-mon[220044]: failed to encode map e690 with expected crc 2023-10-09T06:21:37.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:37 smithi064 ceph-mon[220044]: osdmap e691: 8 total, 8 up, 8 in 2023-10-09T06:21:38.082 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:38.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:37 smithi156 ceph-mon[195101]: failed to encode map e690 with expected crc 2023-10-09T06:21:38.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:37 smithi156 ceph-mon[195101]: failed to encode map e690 with expected crc 2023-10-09T06:21:38.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:37 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1031809096' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:21:38.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:37 smithi156 ceph-mon[195101]: osdmap e690: 8 total, 8 up, 8 in 2023-10-09T06:21:38.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:37 smithi156 ceph-mon[195101]: failed to encode map e690 with expected crc 2023-10-09T06:21:38.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:37 smithi156 ceph-mon[195101]: failed to encode map e690 with expected crc 2023-10-09T06:21:38.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:37 smithi156 ceph-mon[195101]: failed to encode map e690 with expected crc 2023-10-09T06:21:38.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:37 smithi156 ceph-mon[195101]: osdmap e691: 8 total, 8 up, 8 in 2023-10-09T06:21:39.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:38 smithi064 ceph-mon[212750]: pgmap v1742: 103 pgs: 1 peering, 102 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 53 KiB/s, 3 objects/s recovering 2023-10-09T06:21:39.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:38 smithi064 ceph-mon[212750]: failed to encode map e691 with expected crc 2023-10-09T06:21:39.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:38 smithi064 ceph-mon[212750]: failed to encode map e691 with expected crc 2023-10-09T06:21:39.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:38 smithi064 ceph-mon[212750]: failed to encode map e691 with expected crc 2023-10-09T06:21:39.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:38 smithi064 ceph-mon[212750]: failed to encode map e691 with expected crc 2023-10-09T06:21:39.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:38 smithi064 ceph-mon[212750]: failed to encode map e691 with expected crc 2023-10-09T06:21:39.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:38 smithi064 ceph-mon[212750]: failed to encode map e691 with expected crc 2023-10-09T06:21:39.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:38 smithi064 ceph-mon[212750]: failed to encode map e691 with expected crc 2023-10-09T06:21:39.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:38 smithi064 ceph-mon[212750]: osdmap e692: 8 total, 8 up, 8 in 2023-10-09T06:21:39.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:38 smithi064 ceph-mon[220044]: pgmap v1742: 103 pgs: 1 peering, 102 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 53 KiB/s, 3 objects/s recovering 2023-10-09T06:21:39.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:38 smithi064 ceph-mon[220044]: failed to encode map e691 with expected crc 2023-10-09T06:21:39.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:38 smithi064 ceph-mon[220044]: failed to encode map e691 with expected crc 2023-10-09T06:21:39.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:38 smithi064 ceph-mon[220044]: failed to encode map e691 with expected crc 2023-10-09T06:21:39.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:38 smithi064 ceph-mon[220044]: failed to encode map e691 with expected crc 2023-10-09T06:21:39.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:38 smithi064 ceph-mon[220044]: failed to encode map e691 with expected crc 2023-10-09T06:21:39.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:38 smithi064 ceph-mon[220044]: failed to encode map e691 with expected crc 2023-10-09T06:21:39.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:38 smithi064 ceph-mon[220044]: failed to encode map e691 with expected crc 2023-10-09T06:21:39.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:38 smithi064 ceph-mon[220044]: osdmap e692: 8 total, 8 up, 8 in 2023-10-09T06:21:39.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:38 smithi156 ceph-mon[195101]: pgmap v1742: 103 pgs: 1 peering, 102 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 53 KiB/s, 3 objects/s recovering 2023-10-09T06:21:39.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:38 smithi156 ceph-mon[195101]: failed to encode map e691 with expected crc 2023-10-09T06:21:39.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:38 smithi156 ceph-mon[195101]: failed to encode map e691 with expected crc 2023-10-09T06:21:39.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:38 smithi156 ceph-mon[195101]: failed to encode map e691 with expected crc 2023-10-09T06:21:39.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:38 smithi156 ceph-mon[195101]: failed to encode map e691 with expected crc 2023-10-09T06:21:39.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:38 smithi156 ceph-mon[195101]: failed to encode map e691 with expected crc 2023-10-09T06:21:39.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:38 smithi156 ceph-mon[195101]: failed to encode map e691 with expected crc 2023-10-09T06:21:39.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:38 smithi156 ceph-mon[195101]: failed to encode map e691 with expected crc 2023-10-09T06:21:39.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:38 smithi156 ceph-mon[195101]: osdmap e692: 8 total, 8 up, 8 in 2023-10-09T06:21:39.935 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-10-09T06:21:40.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:39 smithi064 ceph-mon[212750]: failed to encode map e692 with expected crc 2023-10-09T06:21:40.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:39 smithi064 ceph-mon[212750]: failed to encode map e692 with expected crc 2023-10-09T06:21:40.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:39 smithi064 ceph-mon[212750]: failed to encode map e692 with expected crc 2023-10-09T06:21:40.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:39 smithi064 ceph-mon[212750]: failed to encode map e692 with expected crc 2023-10-09T06:21:40.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:39 smithi064 ceph-mon[212750]: failed to encode map e692 with expected crc 2023-10-09T06:21:40.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:39 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:40.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:39 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/842401907' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:21:40.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:39 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:21:40.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:39 smithi064 ceph-mon[220044]: failed to encode map e692 with expected crc 2023-10-09T06:21:40.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:39 smithi064 ceph-mon[220044]: failed to encode map e692 with expected crc 2023-10-09T06:21:40.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:39 smithi064 ceph-mon[220044]: failed to encode map e692 with expected crc 2023-10-09T06:21:40.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:39 smithi064 ceph-mon[220044]: failed to encode map e692 with expected crc 2023-10-09T06:21:40.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:39 smithi064 ceph-mon[220044]: failed to encode map e692 with expected crc 2023-10-09T06:21:40.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:39 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:40.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:39 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/842401907' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:21:40.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:39 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:21:40.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:39 smithi156 ceph-mon[195101]: failed to encode map e692 with expected crc 2023-10-09T06:21:40.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:39 smithi156 ceph-mon[195101]: failed to encode map e692 with expected crc 2023-10-09T06:21:40.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:39 smithi156 ceph-mon[195101]: failed to encode map e692 with expected crc 2023-10-09T06:21:40.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:39 smithi156 ceph-mon[195101]: failed to encode map e692 with expected crc 2023-10-09T06:21:40.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:39 smithi156 ceph-mon[195101]: failed to encode map e692 with expected crc 2023-10-09T06:21:40.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:39 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:40.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:39 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/842401907' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:21:40.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:39 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:21:40.811 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T06:21:41.190 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:41.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:40 smithi064 ceph-mon[212750]: pgmap v1745: 103 pgs: 1 peering, 102 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-10-09T06:21:41.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:40 smithi064 ceph-mon[212750]: failed to encode map e693 with expected crc 2023-10-09T06:21:41.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:40 smithi064 ceph-mon[212750]: failed to encode map e693 with expected crc 2023-10-09T06:21:41.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:40 smithi064 ceph-mon[212750]: failed to encode map e693 with expected crc 2023-10-09T06:21:41.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:40 smithi064 ceph-mon[212750]: failed to encode map e693 with expected crc 2023-10-09T06:21:41.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:40 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:21:41.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:40 smithi064 ceph-mon[212750]: osdmap e693: 8 total, 8 up, 8 in 2023-10-09T06:21:41.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:40 smithi064 ceph-mon[212750]: failed to encode map e693 with expected crc 2023-10-09T06:21:41.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:40 smithi064 ceph-mon[212750]: failed to encode map e693 with expected crc 2023-10-09T06:21:41.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:40 smithi064 ceph-mon[212750]: failed to encode map e693 with expected crc 2023-10-09T06:21:41.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:40 smithi064 ceph-mon[220044]: pgmap v1745: 103 pgs: 1 peering, 102 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-10-09T06:21:41.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:40 smithi064 ceph-mon[220044]: failed to encode map e693 with expected crc 2023-10-09T06:21:41.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:40 smithi064 ceph-mon[220044]: failed to encode map e693 with expected crc 2023-10-09T06:21:41.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:40 smithi064 ceph-mon[220044]: failed to encode map e693 with expected crc 2023-10-09T06:21:41.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:40 smithi064 ceph-mon[220044]: failed to encode map e693 with expected crc 2023-10-09T06:21:41.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:40 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:21:41.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:40 smithi064 ceph-mon[220044]: osdmap e693: 8 total, 8 up, 8 in 2023-10-09T06:21:41.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:40 smithi064 ceph-mon[220044]: failed to encode map e693 with expected crc 2023-10-09T06:21:41.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:40 smithi064 ceph-mon[220044]: failed to encode map e693 with expected crc 2023-10-09T06:21:41.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:40 smithi064 ceph-mon[220044]: failed to encode map e693 with expected crc 2023-10-09T06:21:41.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:40 smithi156 ceph-mon[195101]: pgmap v1745: 103 pgs: 1 peering, 102 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-10-09T06:21:41.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:40 smithi156 ceph-mon[195101]: failed to encode map e693 with expected crc 2023-10-09T06:21:41.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:40 smithi156 ceph-mon[195101]: failed to encode map e693 with expected crc 2023-10-09T06:21:41.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:40 smithi156 ceph-mon[195101]: failed to encode map e693 with expected crc 2023-10-09T06:21:41.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:40 smithi156 ceph-mon[195101]: failed to encode map e693 with expected crc 2023-10-09T06:21:41.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:40 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:21:41.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:40 smithi156 ceph-mon[195101]: osdmap e693: 8 total, 8 up, 8 in 2023-10-09T06:21:41.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:40 smithi156 ceph-mon[195101]: failed to encode map e693 with expected crc 2023-10-09T06:21:41.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:40 smithi156 ceph-mon[195101]: failed to encode map e693 with expected crc 2023-10-09T06:21:41.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:40 smithi156 ceph-mon[195101]: failed to encode map e693 with expected crc 2023-10-09T06:21:42.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:41 smithi064 ceph-mon[212750]: failed to encode map e694 with expected crc 2023-10-09T06:21:42.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:41 smithi064 ceph-mon[212750]: failed to encode map e694 with expected crc 2023-10-09T06:21:42.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:41 smithi064 ceph-mon[212750]: osdmap e694: 8 total, 8 up, 8 in 2023-10-09T06:21:42.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:41 smithi064 ceph-mon[212750]: failed to encode map e694 with expected crc 2023-10-09T06:21:42.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:41 smithi064 ceph-mon[212750]: failed to encode map e694 with expected crc 2023-10-09T06:21:42.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:41 smithi064 ceph-mon[212750]: failed to encode map e694 with expected crc 2023-10-09T06:21:42.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:41 smithi064 ceph-mon[212750]: failed to encode map e694 with expected crc 2023-10-09T06:21:42.181 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:41 smithi064 ceph-mon[220044]: failed to encode map e694 with expected crc 2023-10-09T06:21:42.181 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:41 smithi064 ceph-mon[220044]: failed to encode map e694 with expected crc 2023-10-09T06:21:42.181 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:41 smithi064 ceph-mon[220044]: osdmap e694: 8 total, 8 up, 8 in 2023-10-09T06:21:42.181 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:41 smithi064 ceph-mon[220044]: failed to encode map e694 with expected crc 2023-10-09T06:21:42.181 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:41 smithi064 ceph-mon[220044]: failed to encode map e694 with expected crc 2023-10-09T06:21:42.181 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:41 smithi064 ceph-mon[220044]: failed to encode map e694 with expected crc 2023-10-09T06:21:42.181 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:41 smithi064 ceph-mon[220044]: failed to encode map e694 with expected crc 2023-10-09T06:21:42.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:41 smithi156 ceph-mon[195101]: failed to encode map e694 with expected crc 2023-10-09T06:21:42.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:41 smithi156 ceph-mon[195101]: failed to encode map e694 with expected crc 2023-10-09T06:21:42.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:41 smithi156 ceph-mon[195101]: osdmap e694: 8 total, 8 up, 8 in 2023-10-09T06:21:42.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:41 smithi156 ceph-mon[195101]: failed to encode map e694 with expected crc 2023-10-09T06:21:42.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:41 smithi156 ceph-mon[195101]: failed to encode map e694 with expected crc 2023-10-09T06:21:42.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:41 smithi156 ceph-mon[195101]: failed to encode map e694 with expected crc 2023-10-09T06:21:42.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:41 smithi156 ceph-mon[195101]: failed to encode map e694 with expected crc 2023-10-09T06:21:43.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:42 smithi064 ceph-mon[212750]: pgmap v1748: 103 pgs: 2 active+undersized+remapped, 101 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 64 KiB/s, 0 objects/s recovering 2023-10-09T06:21:43.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:42 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1568548919' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:21:43.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:42 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:21:43.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:42 smithi064 ceph-mon[220044]: pgmap v1748: 103 pgs: 2 active+undersized+remapped, 101 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 64 KiB/s, 0 objects/s recovering 2023-10-09T06:21:43.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:42 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1568548919' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:21:43.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:42 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:21:43.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:42 smithi156 ceph-mon[195101]: pgmap v1748: 103 pgs: 2 active+undersized+remapped, 101 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 64 KiB/s, 0 objects/s recovering 2023-10-09T06:21:43.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:42 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1568548919' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:21:43.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:42 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:21:43.891 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T06:21:44.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:43 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:44.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:43 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:21:44.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:43 smithi064 ceph-mon[212750]: osdmap e695: 8 total, 8 up, 8 in 2023-10-09T06:21:44.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:43 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:44.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:43 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:21:44.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:43 smithi064 ceph-mon[220044]: osdmap e695: 8 total, 8 up, 8 in 2023-10-09T06:21:44.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:43 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:44.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:43 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:21:44.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:43 smithi156 ceph-mon[195101]: osdmap e695: 8 total, 8 up, 8 in 2023-10-09T06:21:44.618 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-10-09T06:21:44.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:44 smithi064 ceph-mon[212750]: pgmap v1749: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:44.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:44 smithi064 ceph-mon[212750]: failed to encode map e695 with expected crc 2023-10-09T06:21:44.951 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:44 smithi064 ceph-mon[212750]: failed to encode map e695 with expected crc 2023-10-09T06:21:44.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:44 smithi064 ceph-mon[212750]: failed to encode map e695 with expected crc 2023-10-09T06:21:44.952 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:44 smithi064 ceph-mon[212750]: failed to encode map e695 with expected crc 2023-10-09T06:21:44.987 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:45.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:44 smithi064 ceph-mon[220044]: pgmap v1749: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:45.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:44 smithi064 ceph-mon[220044]: failed to encode map e695 with expected crc 2023-10-09T06:21:45.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:44 smithi064 ceph-mon[220044]: failed to encode map e695 with expected crc 2023-10-09T06:21:45.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:44 smithi064 ceph-mon[220044]: failed to encode map e695 with expected crc 2023-10-09T06:21:45.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:44 smithi064 ceph-mon[220044]: failed to encode map e695 with expected crc 2023-10-09T06:21:45.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:44 smithi156 ceph-mon[195101]: pgmap v1749: 103 pgs: 1 active+undersized+remapped, 102 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:45.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:44 smithi156 ceph-mon[195101]: failed to encode map e695 with expected crc 2023-10-09T06:21:45.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:44 smithi156 ceph-mon[195101]: failed to encode map e695 with expected crc 2023-10-09T06:21:45.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:44 smithi156 ceph-mon[195101]: failed to encode map e695 with expected crc 2023-10-09T06:21:45.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:44 smithi156 ceph-mon[195101]: failed to encode map e695 with expected crc 2023-10-09T06:21:45.655 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:21:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:21:45] "GET /metrics HTTP/1.1" 200 34093 "" "Prometheus/2.43.0" 2023-10-09T06:21:46.960 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-10-09T06:21:47.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:46 smithi064 ceph-mon[212750]: pgmap v1751: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:47.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:46 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/745750813' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:21:47.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:46 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:21:47.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:46 smithi064 ceph-mon[220044]: pgmap v1751: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:47.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:46 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/745750813' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:21:47.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:46 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:21:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:46 smithi156 ceph-mon[195101]: pgmap v1751: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:46 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/745750813' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:21:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:46 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:21:47.696 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T06:21:47.963 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:47 smithi064 ceph-mon[212750]: failed to encode map e696 with expected crc 2023-10-09T06:21:47.963 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:47 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:21:47.963 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:47 smithi064 ceph-mon[212750]: osdmap e696: 8 total, 8 up, 8 in 2023-10-09T06:21:48.061 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:48.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:47 smithi064 ceph-mon[220044]: failed to encode map e696 with expected crc 2023-10-09T06:21:48.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:47 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:21:48.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:47 smithi064 ceph-mon[220044]: osdmap e696: 8 total, 8 up, 8 in 2023-10-09T06:21:48.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:47 smithi156 ceph-mon[195101]: failed to encode map e696 with expected crc 2023-10-09T06:21:48.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:47 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:21:48.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:47 smithi156 ceph-mon[195101]: osdmap e696: 8 total, 8 up, 8 in 2023-10-09T06:21:49.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:48 smithi064 ceph-mon[212750]: pgmap v1753: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:49.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:48 smithi064 ceph-mon[220044]: pgmap v1753: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:49.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:48 smithi156 ceph-mon[195101]: pgmap v1753: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:49.975 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T06:21:50.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:49 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/789636061' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:21:50.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:49 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:21:50.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:49 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/789636061' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:21:50.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:49 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:21:50.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:49 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/789636061' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:21:50.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:49 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:21:50.687 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-10-09T06:21:51.050 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:51.254 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:50 smithi064 ceph-mon[212750]: pgmap v1754: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:51.254 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:50 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:21:51.255 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:50 smithi064 ceph-mon[212750]: osdmap e697: 8 total, 8 up, 8 in 2023-10-09T06:21:51.255 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:50 smithi064 ceph-mon[212750]: failed to encode map e697 with expected crc 2023-10-09T06:21:51.255 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:50 smithi064 ceph-mon[212750]: failed to encode map e697 with expected crc 2023-10-09T06:21:51.255 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:50 smithi064 ceph-mon[212750]: failed to encode map e697 with expected crc 2023-10-09T06:21:51.255 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:50 smithi064 ceph-mon[212750]: failed to encode map e697 with expected crc 2023-10-09T06:21:51.256 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:50 smithi064 ceph-mon[220044]: pgmap v1754: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:51.256 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:50 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:21:51.256 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:50 smithi064 ceph-mon[220044]: osdmap e697: 8 total, 8 up, 8 in 2023-10-09T06:21:51.256 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:50 smithi064 ceph-mon[220044]: failed to encode map e697 with expected crc 2023-10-09T06:21:51.256 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:50 smithi064 ceph-mon[220044]: failed to encode map e697 with expected crc 2023-10-09T06:21:51.257 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:50 smithi064 ceph-mon[220044]: failed to encode map e697 with expected crc 2023-10-09T06:21:51.257 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:50 smithi064 ceph-mon[220044]: failed to encode map e697 with expected crc 2023-10-09T06:21:51.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:50 smithi156 ceph-mon[195101]: pgmap v1754: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:51.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:50 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:21:51.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:50 smithi156 ceph-mon[195101]: osdmap e697: 8 total, 8 up, 8 in 2023-10-09T06:21:51.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:50 smithi156 ceph-mon[195101]: failed to encode map e697 with expected crc 2023-10-09T06:21:51.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:50 smithi156 ceph-mon[195101]: failed to encode map e697 with expected crc 2023-10-09T06:21:51.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:50 smithi156 ceph-mon[195101]: failed to encode map e697 with expected crc 2023-10-09T06:21:51.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:50 smithi156 ceph-mon[195101]: failed to encode map e697 with expected crc 2023-10-09T06:21:52.981 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-10-09T06:21:53.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:52 smithi156 ceph-mon[195101]: pgmap v1756: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:53.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:52 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2980302375' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:21:53.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:52 smithi064 ceph-mon[212750]: pgmap v1756: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:53.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:52 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2980302375' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:21:53.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:52 smithi064 ceph-mon[220044]: pgmap v1756: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:53.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:52 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2980302375' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:21:53.662 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T06:21:54.030 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:54.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:53 smithi064 ceph-mon[212750]: failed to encode map e698 with expected crc 2023-10-09T06:21:54.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:53 smithi064 ceph-mon[212750]: failed to encode map e698 with expected crc 2023-10-09T06:21:54.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:53 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2980302375' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:21:54.238 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:53 smithi064 ceph-mon[212750]: osdmap e698: 8 total, 8 up, 8 in 2023-10-09T06:21:54.238 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:53 smithi064 ceph-mon[220044]: failed to encode map e698 with expected crc 2023-10-09T06:21:54.238 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:53 smithi064 ceph-mon[220044]: failed to encode map e698 with expected crc 2023-10-09T06:21:54.238 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:53 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2980302375' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:21:54.238 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:53 smithi064 ceph-mon[220044]: osdmap e698: 8 total, 8 up, 8 in 2023-10-09T06:21:54.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:53 smithi156 ceph-mon[195101]: failed to encode map e698 with expected crc 2023-10-09T06:21:54.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:53 smithi156 ceph-mon[195101]: failed to encode map e698 with expected crc 2023-10-09T06:21:54.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:53 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2980302375' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:21:54.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:53 smithi156 ceph-mon[195101]: osdmap e698: 8 total, 8 up, 8 in 2023-10-09T06:21:55.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:54 smithi064 ceph-mon[212750]: pgmap v1758: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:55.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:54 smithi064 ceph-mon[212750]: osdmap e699: 8 total, 8 up, 8 in 2023-10-09T06:21:55.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:54 smithi064 ceph-mon[212750]: failed to encode map e699 with expected crc 2023-10-09T06:21:55.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:54 smithi064 ceph-mon[212750]: failed to encode map e699 with expected crc 2023-10-09T06:21:55.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:54 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:55.302 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:54 smithi064 ceph-mon[220044]: pgmap v1758: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:55.303 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:54 smithi064 ceph-mon[220044]: osdmap e699: 8 total, 8 up, 8 in 2023-10-09T06:21:55.303 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:54 smithi064 ceph-mon[220044]: failed to encode map e699 with expected crc 2023-10-09T06:21:55.303 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:54 smithi064 ceph-mon[220044]: failed to encode map e699 with expected crc 2023-10-09T06:21:55.303 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:54 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:55.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:54 smithi156 ceph-mon[195101]: pgmap v1758: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:55.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:54 smithi156 ceph-mon[195101]: osdmap e699: 8 total, 8 up, 8 in 2023-10-09T06:21:55.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:54 smithi156 ceph-mon[195101]: failed to encode map e699 with expected crc 2023-10-09T06:21:55.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:54 smithi156 ceph-mon[195101]: failed to encode map e699 with expected crc 2023-10-09T06:21:55.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:54 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:21:55.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:21:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:21:55] "GET /metrics HTTP/1.1" 200 34093 "" "Prometheus/2.43.0" 2023-10-09T06:21:55.995 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T06:21:56.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:55 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3935346332' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:21:56.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:55 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:21:56.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:55 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3935346332' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:21:56.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:55 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:21:56.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:55 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3935346332' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:21:56.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:55 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:21:56.729 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-10-09T06:21:57.105 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:57.312 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:56 smithi064 ceph-mon[212750]: pgmap v1760: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:57.312 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:56 smithi064 ceph-mon[212750]: failed to encode map e700 with expected crc 2023-10-09T06:21:57.312 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:56 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:21:57.312 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:56 smithi064 ceph-mon[212750]: osdmap e700: 8 total, 8 up, 8 in 2023-10-09T06:21:57.312 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:56 smithi064 ceph-mon[220044]: pgmap v1760: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:57.312 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:56 smithi064 ceph-mon[220044]: failed to encode map e700 with expected crc 2023-10-09T06:21:57.312 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:56 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:21:57.313 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:56 smithi064 ceph-mon[220044]: osdmap e700: 8 total, 8 up, 8 in 2023-10-09T06:21:57.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:56 smithi156 ceph-mon[195101]: pgmap v1760: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:57.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:56 smithi156 ceph-mon[195101]: failed to encode map e700 with expected crc 2023-10-09T06:21:57.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:56 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:21:57.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:56 smithi156 ceph-mon[195101]: osdmap e700: 8 total, 8 up, 8 in 2023-10-09T06:21:58.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:57 smithi156 ceph-mon[195101]: failed to encode map e700 with expected crc 2023-10-09T06:21:58.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:57 smithi156 ceph-mon[195101]: failed to encode map e700 with expected crc 2023-10-09T06:21:58.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:57 smithi156 ceph-mon[195101]: failed to encode map e700 with expected crc 2023-10-09T06:21:58.397 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:57 smithi064 ceph-mon[212750]: failed to encode map e700 with expected crc 2023-10-09T06:21:58.398 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:57 smithi064 ceph-mon[212750]: failed to encode map e700 with expected crc 2023-10-09T06:21:58.398 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:57 smithi064 ceph-mon[212750]: failed to encode map e700 with expected crc 2023-10-09T06:21:58.398 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:58 smithi064 ceph-mon[220044]: failed to encode map e700 with expected crc 2023-10-09T06:21:58.398 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:58 smithi064 ceph-mon[220044]: failed to encode map e700 with expected crc 2023-10-09T06:21:58.398 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:58 smithi064 ceph-mon[220044]: failed to encode map e700 with expected crc 2023-10-09T06:21:59.013 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-10-09T06:21:59.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:59 smithi156 ceph-mon[195101]: pgmap v1762: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:59.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:59 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:59.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:59 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1196321484' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:21:59.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:21:59 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:21:59.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:59 smithi064 ceph-mon[212750]: pgmap v1762: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:59.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:59 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:59.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:59 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1196321484' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:21:59.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:21:59 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:21:59.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:59 smithi064 ceph-mon[220044]: pgmap v1762: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:21:59.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:59 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:59.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:59 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1196321484' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:21:59.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:21:59 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:21:59.738 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T06:22:00.105 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:00.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:00 smithi064 ceph-mon[212750]: failed to encode map e701 with expected crc 2023-10-09T06:22:00.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:00 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:22:00.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:00 smithi064 ceph-mon[212750]: osdmap e701: 8 total, 8 up, 8 in 2023-10-09T06:22:00.304 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:00 smithi064 ceph-mon[220044]: failed to encode map e701 with expected crc 2023-10-09T06:22:00.304 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:00 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:22:00.304 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:00 smithi064 ceph-mon[220044]: osdmap e701: 8 total, 8 up, 8 in 2023-10-09T06:22:00.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:00 smithi156 ceph-mon[195101]: failed to encode map e701 with expected crc 2023-10-09T06:22:00.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:00 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:22:00.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:00 smithi156 ceph-mon[195101]: osdmap e701: 8 total, 8 up, 8 in 2023-10-09T06:22:01.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:01 smithi156 ceph-mon[195101]: pgmap v1764: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:01.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:01 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T06:22:01.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:01 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2023-10-09T06:22:01.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:01 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T06:22:01.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:01 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2023-10-09T06:22:01.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:01 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2023-10-09T06:22:01.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:01 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2023-10-09T06:22:01.448 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:01 smithi064 ceph-mon[212750]: pgmap v1764: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:01.448 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:01 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T06:22:01.448 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:01 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2023-10-09T06:22:01.448 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:01 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T06:22:01.448 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:01 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2023-10-09T06:22:01.448 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:01 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2023-10-09T06:22:01.448 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:01 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2023-10-09T06:22:01.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:01 smithi064 ceph-mon[220044]: pgmap v1764: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:01.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:01 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T06:22:01.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:01 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2023-10-09T06:22:01.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:01 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T06:22:01.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:01 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2023-10-09T06:22:01.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:01 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2023-10-09T06:22:01.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:01 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2023-10-09T06:22:02.048 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T06:22:02.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:02 smithi156 ceph-mon[195101]: failed to encode map e701 with expected crc 2023-10-09T06:22:02.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:02 smithi156 ceph-mon[195101]: failed to encode map e701 with expected crc 2023-10-09T06:22:02.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:02 smithi156 ceph-mon[195101]: failed to encode map e702 with expected crc 2023-10-09T06:22:02.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:02 smithi156 ceph-mon[195101]: failed to encode map e702 with expected crc 2023-10-09T06:22:02.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:02 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T06:22:02.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:02 smithi156 ceph-mon[195101]: failed to encode map e702 with expected crc 2023-10-09T06:22:02.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:02 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2023-10-09T06:22:02.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:02 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2023-10-09T06:22:02.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:02 smithi156 ceph-mon[195101]: osdmap e702: 8 total, 8 up, 8 in 2023-10-09T06:22:02.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:02 smithi156 ceph-mon[195101]: failed to encode map e702 with expected crc 2023-10-09T06:22:02.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:02 smithi156 ceph-mon[195101]: pgmap v1766: 103 pgs: 2 peering, 101 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:02.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:02 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3248341134' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:22:02.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:02 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:22:02.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[212750]: failed to encode map e701 with expected crc 2023-10-09T06:22:02.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[212750]: failed to encode map e701 with expected crc 2023-10-09T06:22:02.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[212750]: failed to encode map e702 with expected crc 2023-10-09T06:22:02.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[212750]: failed to encode map e702 with expected crc 2023-10-09T06:22:02.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T06:22:02.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[212750]: failed to encode map e702 with expected crc 2023-10-09T06:22:02.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2023-10-09T06:22:02.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2023-10-09T06:22:02.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[212750]: osdmap e702: 8 total, 8 up, 8 in 2023-10-09T06:22:02.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[212750]: failed to encode map e702 with expected crc 2023-10-09T06:22:02.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[212750]: pgmap v1766: 103 pgs: 2 peering, 101 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:02.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3248341134' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:22:02.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:22:02.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[220044]: failed to encode map e701 with expected crc 2023-10-09T06:22:02.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[220044]: failed to encode map e701 with expected crc 2023-10-09T06:22:02.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[220044]: failed to encode map e702 with expected crc 2023-10-09T06:22:02.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[220044]: failed to encode map e702 with expected crc 2023-10-09T06:22:02.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T06:22:02.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[220044]: failed to encode map e702 with expected crc 2023-10-09T06:22:02.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2023-10-09T06:22:02.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2023-10-09T06:22:02.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[220044]: osdmap e702: 8 total, 8 up, 8 in 2023-10-09T06:22:02.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[220044]: failed to encode map e702 with expected crc 2023-10-09T06:22:02.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[220044]: pgmap v1766: 103 pgs: 2 peering, 101 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:02.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3248341134' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:22:02.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:02 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:22:02.812 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-10-09T06:22:03.179 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:03.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:03 smithi064 ceph-mon[220044]: failed to encode map e703 with expected crc 2023-10-09T06:22:03.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:03 smithi064 ceph-mon[220044]: failed to encode map e703 with expected crc 2023-10-09T06:22:03.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:03 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:22:03.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:03 smithi064 ceph-mon[220044]: osdmap e703: 8 total, 8 up, 8 in 2023-10-09T06:22:03.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:03 smithi064 ceph-mon[220044]: failed to encode map e703 with expected crc 2023-10-09T06:22:03.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:03 smithi064 ceph-mon[220044]: failed to encode map e703 with expected crc 2023-10-09T06:22:03.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:03 smithi064 ceph-mon[220044]: failed to encode map e703 with expected crc 2023-10-09T06:22:03.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:03 smithi064 ceph-mon[212750]: failed to encode map e703 with expected crc 2023-10-09T06:22:03.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:03 smithi064 ceph-mon[212750]: failed to encode map e703 with expected crc 2023-10-09T06:22:03.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:03 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:22:03.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:03 smithi064 ceph-mon[212750]: osdmap e703: 8 total, 8 up, 8 in 2023-10-09T06:22:03.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:03 smithi064 ceph-mon[212750]: failed to encode map e703 with expected crc 2023-10-09T06:22:03.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:03 smithi064 ceph-mon[212750]: failed to encode map e703 with expected crc 2023-10-09T06:22:03.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:03 smithi064 ceph-mon[212750]: failed to encode map e703 with expected crc 2023-10-09T06:22:03.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:03 smithi156 ceph-mon[195101]: failed to encode map e703 with expected crc 2023-10-09T06:22:03.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:03 smithi156 ceph-mon[195101]: failed to encode map e703 with expected crc 2023-10-09T06:22:03.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:03 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:22:03.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:03 smithi156 ceph-mon[195101]: osdmap e703: 8 total, 8 up, 8 in 2023-10-09T06:22:03.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:03 smithi156 ceph-mon[195101]: failed to encode map e703 with expected crc 2023-10-09T06:22:03.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:03 smithi156 ceph-mon[195101]: failed to encode map e703 with expected crc 2023-10-09T06:22:03.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:03 smithi156 ceph-mon[195101]: failed to encode map e703 with expected crc 2023-10-09T06:22:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:04 smithi156 ceph-mon[195101]: failed to encode map e703 with expected crc 2023-10-09T06:22:04.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:04 smithi156 ceph-mon[195101]: pgmap v1768: 103 pgs: 2 peering, 101 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:04.431 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:04 smithi064 ceph-mon[212750]: failed to encode map e703 with expected crc 2023-10-09T06:22:04.431 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:04 smithi064 ceph-mon[212750]: pgmap v1768: 103 pgs: 2 peering, 101 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:04.431 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:04 smithi064 ceph-mon[220044]: failed to encode map e703 with expected crc 2023-10-09T06:22:04.431 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:04 smithi064 ceph-mon[220044]: pgmap v1768: 103 pgs: 2 peering, 101 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:05.065 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-10-09T06:22:05.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:05 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3294948713' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:22:05.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:05 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:22:05.303 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:05 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3294948713' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:22:05.303 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:05 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:22:05.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:05 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3294948713' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:22:05.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:05 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:22:05.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:22:05 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:22:05] "GET /metrics HTTP/1.1" 200 34085 "" "Prometheus/2.43.0" 2023-10-09T06:22:05.769 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T06:22:06.064 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:06 smithi064 ceph-mon[220044]: failed to encode map e704 with expected crc 2023-10-09T06:22:06.064 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:06 smithi064 ceph-mon[220044]: failed to encode map e704 with expected crc 2023-10-09T06:22:06.064 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:06 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:22:06.064 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:06 smithi064 ceph-mon[220044]: osdmap e704: 8 total, 8 up, 8 in 2023-10-09T06:22:06.064 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:06 smithi064 ceph-mon[220044]: pgmap v1770: 103 pgs: 2 peering, 101 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:06.138 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:06.328 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:06 smithi064 ceph-mon[212750]: failed to encode map e704 with expected crc 2023-10-09T06:22:06.328 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:06 smithi064 ceph-mon[212750]: failed to encode map e704 with expected crc 2023-10-09T06:22:06.328 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:06 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:22:06.328 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:06 smithi064 ceph-mon[212750]: osdmap e704: 8 total, 8 up, 8 in 2023-10-09T06:22:06.328 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:06 smithi064 ceph-mon[212750]: pgmap v1770: 103 pgs: 2 peering, 101 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:06 smithi156 ceph-mon[195101]: failed to encode map e704 with expected crc 2023-10-09T06:22:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:06 smithi156 ceph-mon[195101]: failed to encode map e704 with expected crc 2023-10-09T06:22:06.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:06 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:22:06.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:06 smithi156 ceph-mon[195101]: osdmap e704: 8 total, 8 up, 8 in 2023-10-09T06:22:06.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:06 smithi156 ceph-mon[195101]: pgmap v1770: 103 pgs: 2 peering, 101 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:07.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:07 smithi156 ceph-mon[195101]: failed to encode map e704 with expected crc 2023-10-09T06:22:07.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:07 smithi156 ceph-mon[195101]: failed to encode map e704 with expected crc 2023-10-09T06:22:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:07 smithi064 ceph-mon[212750]: failed to encode map e704 with expected crc 2023-10-09T06:22:07.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:07 smithi064 ceph-mon[212750]: failed to encode map e704 with expected crc 2023-10-09T06:22:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:07 smithi064 ceph-mon[220044]: failed to encode map e704 with expected crc 2023-10-09T06:22:07.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:07 smithi064 ceph-mon[220044]: failed to encode map e704 with expected crc 2023-10-09T06:22:08.894 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T06:22:09.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:08 smithi064 ceph-mon[212750]: pgmap v1771: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:09.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:08 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1483308953' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:22:09.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:08 smithi064 ceph-mon[220044]: pgmap v1771: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:09.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:08 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1483308953' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:22:09.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:08 smithi156 ceph-mon[195101]: pgmap v1771: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:09.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:08 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1483308953' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:22:09.638 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-10-09T06:22:10.005 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:10.201 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:09 smithi064 ceph-mon[212750]: failed to encode map e705 with expected crc 2023-10-09T06:22:10.201 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:09 smithi064 ceph-mon[212750]: failed to encode map e705 with expected crc 2023-10-09T06:22:10.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:09 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1483308953' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:22:10.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:09 smithi064 ceph-mon[212750]: osdmap e705: 8 total, 8 up, 8 in 2023-10-09T06:22:10.202 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:09 smithi064 ceph-mon[220044]: failed to encode map e705 with expected crc 2023-10-09T06:22:10.202 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:09 smithi064 ceph-mon[220044]: failed to encode map e705 with expected crc 2023-10-09T06:22:10.202 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:09 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1483308953' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:22:10.202 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:09 smithi064 ceph-mon[220044]: osdmap e705: 8 total, 8 up, 8 in 2023-10-09T06:22:10.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:09 smithi156 ceph-mon[195101]: failed to encode map e705 with expected crc 2023-10-09T06:22:10.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:09 smithi156 ceph-mon[195101]: failed to encode map e705 with expected crc 2023-10-09T06:22:10.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:09 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1483308953' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:22:10.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:09 smithi156 ceph-mon[195101]: osdmap e705: 8 total, 8 up, 8 in 2023-10-09T06:22:11.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:10 smithi064 ceph-mon[212750]: pgmap v1773: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:11.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:10 smithi064 ceph-mon[212750]: failed to encode map e705 with expected crc 2023-10-09T06:22:11.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:10 smithi064 ceph-mon[220044]: pgmap v1773: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:11.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:10 smithi064 ceph-mon[220044]: failed to encode map e705 with expected crc 2023-10-09T06:22:11.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:10 smithi156 ceph-mon[195101]: pgmap v1773: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:11.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:10 smithi156 ceph-mon[195101]: failed to encode map e705 with expected crc 2023-10-09T06:22:11.907 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-10-09T06:22:12.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:11 smithi064 ceph-mon[212750]: failed to encode map e705 with expected crc 2023-10-09T06:22:12.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:11 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:22:12.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:11 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3778832973' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:22:12.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:11 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:22:12.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:11 smithi064 ceph-mon[220044]: failed to encode map e705 with expected crc 2023-10-09T06:22:12.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:11 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:22:12.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:11 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3778832973' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:22:12.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:11 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:22:12.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:11 smithi156 ceph-mon[195101]: failed to encode map e705 with expected crc 2023-10-09T06:22:12.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:11 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:22:12.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:11 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3778832973' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:22:12.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:11 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:22:12.741 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T06:22:13.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:12 smithi064 ceph-mon[212750]: pgmap v1774: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:13.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:12 smithi064 ceph-mon[212750]: failed to encode map e706 with expected crc 2023-10-09T06:22:13.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:12 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:22:13.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:12 smithi064 ceph-mon[212750]: osdmap e706: 8 total, 8 up, 8 in 2023-10-09T06:22:13.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:12 smithi064 ceph-mon[212750]: failed to encode map e706 with expected crc 2023-10-09T06:22:13.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:12 smithi064 ceph-mon[212750]: failed to encode map e706 with expected crc 2023-10-09T06:22:13.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:12 smithi064 ceph-mon[212750]: failed to encode map e706 with expected crc 2023-10-09T06:22:13.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:12 smithi064 ceph-mon[212750]: failed to encode map e706 with expected crc 2023-10-09T06:22:13.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:12 smithi064 ceph-mon[220044]: pgmap v1774: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:13.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:12 smithi064 ceph-mon[220044]: failed to encode map e706 with expected crc 2023-10-09T06:22:13.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:12 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:22:13.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:12 smithi064 ceph-mon[220044]: osdmap e706: 8 total, 8 up, 8 in 2023-10-09T06:22:13.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:12 smithi064 ceph-mon[220044]: failed to encode map e706 with expected crc 2023-10-09T06:22:13.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:12 smithi064 ceph-mon[220044]: failed to encode map e706 with expected crc 2023-10-09T06:22:13.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:12 smithi064 ceph-mon[220044]: failed to encode map e706 with expected crc 2023-10-09T06:22:13.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:12 smithi064 ceph-mon[220044]: failed to encode map e706 with expected crc 2023-10-09T06:22:13.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:12 smithi156 ceph-mon[195101]: pgmap v1774: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:13.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:12 smithi156 ceph-mon[195101]: failed to encode map e706 with expected crc 2023-10-09T06:22:13.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:12 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:22:13.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:12 smithi156 ceph-mon[195101]: osdmap e706: 8 total, 8 up, 8 in 2023-10-09T06:22:13.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:12 smithi156 ceph-mon[195101]: failed to encode map e706 with expected crc 2023-10-09T06:22:13.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:12 smithi156 ceph-mon[195101]: failed to encode map e706 with expected crc 2023-10-09T06:22:13.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:12 smithi156 ceph-mon[195101]: failed to encode map e706 with expected crc 2023-10-09T06:22:13.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:12 smithi156 ceph-mon[195101]: failed to encode map e706 with expected crc 2023-10-09T06:22:13.413 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:14.051 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:13 smithi064 ceph-mon[212750]: failed to encode map e707 with expected crc 2023-10-09T06:22:14.052 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:13 smithi064 ceph-mon[212750]: failed to encode map e707 with expected crc 2023-10-09T06:22:14.052 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:13 smithi064 ceph-mon[212750]: osdmap e707: 8 total, 8 up, 8 in 2023-10-09T06:22:14.052 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:13 smithi064 ceph-mon[212750]: failed to encode map e707 with expected crc 2023-10-09T06:22:14.052 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:13 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:14.052 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:13 smithi064 ceph-mon[220044]: failed to encode map e707 with expected crc 2023-10-09T06:22:14.052 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:13 smithi064 ceph-mon[220044]: failed to encode map e707 with expected crc 2023-10-09T06:22:14.052 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:13 smithi064 ceph-mon[220044]: osdmap e707: 8 total, 8 up, 8 in 2023-10-09T06:22:14.052 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:13 smithi064 ceph-mon[220044]: failed to encode map e707 with expected crc 2023-10-09T06:22:14.052 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:13 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:14.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:13 smithi156 ceph-mon[195101]: failed to encode map e707 with expected crc 2023-10-09T06:22:14.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:13 smithi156 ceph-mon[195101]: failed to encode map e707 with expected crc 2023-10-09T06:22:14.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:13 smithi156 ceph-mon[195101]: osdmap e707: 8 total, 8 up, 8 in 2023-10-09T06:22:14.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:13 smithi156 ceph-mon[195101]: failed to encode map e707 with expected crc 2023-10-09T06:22:14.373 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:13 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:14.924 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:14 smithi064 ceph-mon[212750]: pgmap v1777: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:14.924 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:14 smithi064 ceph-mon[212750]: failed to encode map e707 with expected crc 2023-10-09T06:22:14.924 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:14 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:22:14.924 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:14 smithi064 ceph-mon[220044]: pgmap v1777: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:14.924 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:14 smithi064 ceph-mon[220044]: failed to encode map e707 with expected crc 2023-10-09T06:22:14.924 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:14 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:22:15.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:14 smithi156 ceph-mon[195101]: pgmap v1777: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:15.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:14 smithi156 ceph-mon[195101]: failed to encode map e707 with expected crc 2023-10-09T06:22:15.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:14 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:22:15.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:22:15 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:22:15] "GET /metrics HTTP/1.1" 200 34088 "" "Prometheus/2.43.0" 2023-10-09T06:22:16.018 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T06:22:16.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:16 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:22:16.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:16 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:22:16.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:16 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/175680710' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:22:16.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:16 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:22:16.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:16 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:22:16.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:16 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:22:16.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:16 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:22:16.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:16 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/175680710' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:22:16.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:16 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:22:16.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:16 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:22:16.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:16 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:22:16.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:16 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:22:16.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:16 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/175680710' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:22:16.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:16 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:22:16.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:16 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:22:16.950 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-10-09T06:22:17.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:17 smithi064 ceph-mon[220044]: pgmap v1778: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:17.372 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:17 smithi064 ceph-mon[220044]: failed to encode map e708 with expected crc 2023-10-09T06:22:17.373 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:17 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/175680710' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:22:17.373 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:17 smithi064 ceph-mon[220044]: osdmap e708: 8 total, 8 up, 8 in 2023-10-09T06:22:17.373 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:17 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:22:17.373 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:17 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:22:17.373 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:17 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:22:17.373 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:17 smithi064 ceph-mon[212750]: pgmap v1778: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:17.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:17 smithi064 ceph-mon[212750]: failed to encode map e708 with expected crc 2023-10-09T06:22:17.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:17 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/175680710' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:22:17.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:17 smithi064 ceph-mon[212750]: osdmap e708: 8 total, 8 up, 8 in 2023-10-09T06:22:17.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:17 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:22:17.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:17 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:22:17.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:17 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:22:17.374 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:17.376 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:17 smithi156 ceph-mon[195101]: pgmap v1778: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:17.376 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:17 smithi156 ceph-mon[195101]: failed to encode map e708 with expected crc 2023-10-09T06:22:17.376 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:17 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/175680710' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:22:17.376 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:17 smithi156 ceph-mon[195101]: osdmap e708: 8 total, 8 up, 8 in 2023-10-09T06:22:17.376 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:17 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:22:17.376 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:17 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:22:17.376 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:17 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:22:18.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:18 smithi064 ceph-mon[220044]: failed to encode map e708 with expected crc 2023-10-09T06:22:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:18 smithi064 ceph-mon[220044]: failed to encode map e708 with expected crc 2023-10-09T06:22:18.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:18 smithi064 ceph-mon[220044]: failed to encode map e708 with expected crc 2023-10-09T06:22:18.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:18 smithi064 ceph-mon[212750]: failed to encode map e708 with expected crc 2023-10-09T06:22:18.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:18 smithi064 ceph-mon[212750]: failed to encode map e708 with expected crc 2023-10-09T06:22:18.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:18 smithi064 ceph-mon[212750]: failed to encode map e708 with expected crc 2023-10-09T06:22:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:18 smithi156 ceph-mon[195101]: failed to encode map e708 with expected crc 2023-10-09T06:22:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:18 smithi156 ceph-mon[195101]: failed to encode map e708 with expected crc 2023-10-09T06:22:18.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:18 smithi156 ceph-mon[195101]: failed to encode map e708 with expected crc 2023-10-09T06:22:19.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:19 smithi156 ceph-mon[195101]: pgmap v1780: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:19.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:19 smithi064 ceph-mon[212750]: pgmap v1780: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:19.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:19 smithi064 ceph-mon[220044]: pgmap v1780: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:20.032 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-10-09T06:22:20.300 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:20 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/890173539' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:22:20.300 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:20 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:22:20.300 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:20 smithi064 ceph-mon[212750]: pgmap v1781: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:20.301 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:20 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/890173539' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:22:20.301 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:20 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:22:20.301 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:20 smithi064 ceph-mon[220044]: pgmap v1781: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:20 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/890173539' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:22:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:20 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:22:20.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:20 smithi156 ceph-mon[195101]: pgmap v1781: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:20.816 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-10-09T06:22:20.816 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-10-09T06:22:20.816 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-10-09T06:22:20.816 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-10-09T06:22:20.816 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T06:22:20.816 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T06:22:21.195 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:21.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:21 smithi064 ceph-mon[212750]: failed to encode map e709 with expected crc 2023-10-09T06:22:21.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:21 smithi064 ceph-mon[212750]: failed to encode map e709 with expected crc 2023-10-09T06:22:21.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:21 smithi064 ceph-mon[212750]: failed to encode map e709 with expected crc 2023-10-09T06:22:21.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:21 smithi064 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:22:21.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:21 smithi064 ceph-mon[212750]: osdmap e709: 8 total, 8 up, 8 in 2023-10-09T06:22:21.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:21 smithi064 ceph-mon[220044]: failed to encode map e709 with expected crc 2023-10-09T06:22:21.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:21 smithi064 ceph-mon[220044]: failed to encode map e709 with expected crc 2023-10-09T06:22:21.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:21 smithi064 ceph-mon[220044]: failed to encode map e709 with expected crc 2023-10-09T06:22:21.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:21 smithi064 ceph-mon[220044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:22:21.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:21 smithi064 ceph-mon[220044]: osdmap e709: 8 total, 8 up, 8 in 2023-10-09T06:22:21.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:21 smithi156 ceph-mon[195101]: failed to encode map e709 with expected crc 2023-10-09T06:22:21.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:21 smithi156 ceph-mon[195101]: failed to encode map e709 with expected crc 2023-10-09T06:22:21.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:21 smithi156 ceph-mon[195101]: failed to encode map e709 with expected crc 2023-10-09T06:22:21.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:21 smithi156 ceph-mon[195101]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:22:21.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:21 smithi156 ceph-mon[195101]: osdmap e709: 8 total, 8 up, 8 in 2023-10-09T06:22:22.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:22 smithi156 ceph-mon[195101]: failed to encode map e710 with expected crc 2023-10-09T06:22:22.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:22 smithi156 ceph-mon[195101]: failed to encode map e710 with expected crc 2023-10-09T06:22:22.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:22 smithi156 ceph-mon[195101]: osdmap e710: 8 total, 8 up, 8 in 2023-10-09T06:22:22.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:22 smithi156 ceph-mon[195101]: failed to encode map e710 with expected crc 2023-10-09T06:22:22.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:22 smithi156 ceph-mon[195101]: failed to encode map e710 with expected crc 2023-10-09T06:22:22.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:22 smithi156 ceph-mon[195101]: pgmap v1784: 103 pgs: 1 active, 102 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:22.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:22 smithi064 ceph-mon[220044]: failed to encode map e710 with expected crc 2023-10-09T06:22:22.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:22 smithi064 ceph-mon[220044]: failed to encode map e710 with expected crc 2023-10-09T06:22:22.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:22 smithi064 ceph-mon[220044]: osdmap e710: 8 total, 8 up, 8 in 2023-10-09T06:22:22.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:22 smithi064 ceph-mon[220044]: failed to encode map e710 with expected crc 2023-10-09T06:22:22.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:22 smithi064 ceph-mon[220044]: failed to encode map e710 with expected crc 2023-10-09T06:22:22.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:22 smithi064 ceph-mon[220044]: pgmap v1784: 103 pgs: 1 active, 102 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:22.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:22 smithi064 ceph-mon[212750]: failed to encode map e710 with expected crc 2023-10-09T06:22:22.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:22 smithi064 ceph-mon[212750]: failed to encode map e710 with expected crc 2023-10-09T06:22:22.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:22 smithi064 ceph-mon[212750]: osdmap e710: 8 total, 8 up, 8 in 2023-10-09T06:22:22.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:22 smithi064 ceph-mon[212750]: failed to encode map e710 with expected crc 2023-10-09T06:22:22.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:22 smithi064 ceph-mon[212750]: failed to encode map e710 with expected crc 2023-10-09T06:22:22.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:22 smithi064 ceph-mon[212750]: pgmap v1784: 103 pgs: 1 active, 102 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:22.950 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:22:22.951 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":710,"fsid":"44c77acc-6665-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:33:40.037198+0000","modified":"2023-10-09T06:22:21.022982+0000","last_up_change":"2023-10-09T06:18:31.967604+0000","last_in_change":"2023-10-09T06:21:21.908025+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":12,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:36:42.276784+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"347","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"299","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:39:17.895056+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"665","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":665,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T05:40:11.204791+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T05:40:12.067097+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"85d0857a-ca32-43d2-8cec-bbd916db9f98","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":129,"last_clean_end":437,"up_from":456,"up_thru":706,"down_at":438,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":119405434},{"type":"v1","addr":"172.21.15.64:6803","nonce":119405434}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":119405434},{"type":"v1","addr":"172.21.15.64:6805","nonce":119405434}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":119405434},{"type":"v1","addr":"172.21.15.64:6809","nonce":119405434}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":119405434},{"type":"v1","addr":"172.21.15.64:6807","nonce":119405434}]},"public_addr":"172.21.15.64:6803/119405434","cluster_addr":"172.21.15.64:6805/119405434","heartbeat_back_addr":"172.21.15.64:6809/119405434","heartbeat_front_addr":"172.21.15.64:6807/119405434","state":["exists","up"]},{"osd":1,"uuid":"05570845-2cd8-4e24-a2f5-f46cf67aa31a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":85,"up_from":89,"up_thru":690,"down_at":86,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":3188144294},{"type":"v1","addr":"172.21.15.64:6811","nonce":3188144294}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":3188144294},{"type":"v1","addr":"172.21.15.64:6813","nonce":3188144294}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":3188144294},{"type":"v1","addr":"172.21.15.64:6817","nonce":3188144294}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":3188144294},{"type":"v1","addr":"172.21.15.64:6815","nonce":3188144294}]},"public_addr":"172.21.15.64:6811/3188144294","cluster_addr":"172.21.15.64:6813/3188144294","heartbeat_back_addr":"172.21.15.64:6817/3188144294","heartbeat_front_addr":"172.21.15.64:6815/3188144294","state":["exists","up"]},{"osd":2,"uuid":"0f306460-9f08-4f15-b573-3587f63be61c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":90,"up_from":94,"up_thru":702,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":3375454963},{"type":"v1","addr":"172.21.15.64:6819","nonce":3375454963}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":3375454963},{"type":"v1","addr":"172.21.15.64:6821","nonce":3375454963}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":3375454963},{"type":"v1","addr":"172.21.15.64:6825","nonce":3375454963}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":3375454963},{"type":"v1","addr":"172.21.15.64:6823","nonce":3375454963}]},"public_addr":"172.21.15.64:6819/3375454963","cluster_addr":"172.21.15.64:6821/3375454963","heartbeat_back_addr":"172.21.15.64:6825/3375454963","heartbeat_front_addr":"172.21.15.64:6823/3375454963","state":["exists","up"]},{"osd":3,"uuid":"4bf38edd-e555-402e-ba4b-c751de06aa6a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":95,"up_from":99,"up_thru":702,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":1043927146},{"type":"v1","addr":"172.21.15.64:6827","nonce":1043927146}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":1043927146},{"type":"v1","addr":"172.21.15.64:6829","nonce":1043927146}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":1043927146},{"type":"v1","addr":"172.21.15.64:6833","nonce":1043927146}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":1043927146},{"type":"v1","addr":"172.21.15.64:6831","nonce":1043927146}]},"public_addr":"172.21.15.64:6827/1043927146","cluster_addr":"172.21.15.64:6829/1043927146","heartbeat_back_addr":"172.21.15.64:6833/1043927146","heartbeat_front_addr":"172.21.15.64:6831/1043927146","state":["exists","up"]},{"osd":4,"uuid":"060e8766-7bf6-42f1-bebe-dae6828f7fb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":315,"up_from":322,"up_thru":702,"down_at":316,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6800","nonce":3173423256},{"type":"v1","addr":"172.21.15.156:6801","nonce":3173423256}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":3173423256},{"type":"v1","addr":"172.21.15.156:6803","nonce":3173423256}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":3173423256},{"type":"v1","addr":"172.21.15.156:6807","nonce":3173423256}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":3173423256},{"type":"v1","addr":"172.21.15.156:6805","nonce":3173423256}]},"public_addr":"172.21.15.156:6801/3173423256","cluster_addr":"172.21.15.156:6803/3173423256","heartbeat_back_addr":"172.21.15.156:6807/3173423256","heartbeat_front_addr":"172.21.15.156:6805/3173423256","state":["exists","up"]},{"osd":5,"uuid":"6b5cf38a-6792-4ac7-9694-268601260ba8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":38,"last_clean_end":598,"up_from":611,"up_thru":709,"down_at":599,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":507344993},{"type":"v1","addr":"172.21.15.156:6809","nonce":507344993}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":507344993},{"type":"v1","addr":"172.21.15.156:6811","nonce":507344993}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":507344993},{"type":"v1","addr":"172.21.15.156:6815","nonce":507344993}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":507344993},{"type":"v1","addr":"172.21.15.156:6813","nonce":507344993}]},"public_addr":"172.21.15.156:6809/507344993","cluster_addr":"172.21.15.156:6811/507344993","heartbeat_back_addr":"172.21.15.156:6815/507344993","heartbeat_front_addr":"172.21.15.156:6813/507344993","state":["exists","up"]},{"osd":6,"uuid":"01deea2d-a225-40b8-a54b-61eee57da527","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":698,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6817","nonce":2369965428}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6819","nonce":2369965428}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6823","nonce":2369965428}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6821","nonce":2369965428}]},"public_addr":"172.21.15.156:6817/2369965428","cluster_addr":"172.21.15.156:6819/2369965428","heartbeat_back_addr":"172.21.15.156:6823/2369965428","heartbeat_front_addr":"172.21.15.156:6821/2369965428","state":["exists","up"]},{"osd":7,"uuid":"693e93b0-c235-47fe-9172-85af68e38634","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":693,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6825","nonce":2767546322}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6827","nonce":2767546322}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6831","nonce":2767546322}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6829","nonce":2767546322}]},"public_addr":"172.21.15.156:6825/2767546322","cluster_addr":"172.21.15.156:6827/2767546322","heartbeat_back_addr":"172.21.15.156:6831/2767546322","heartbeat_front_addr":"172.21.15.156:6829/2767546322","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:35:52.928919+0000","dead_epoch":437},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:36:16.479793+0000","dead_epoch":85},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:36:39.503123+0000","dead_epoch":90},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:03.340353+0000","dead_epoch":95},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:24.645080+0000","dead_epoch":315},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:46.312540+0000","dead_epoch":598},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:38:08.346971+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:38:30.073761+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.9","mappings":[{"from":1,"to":2}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.156:0/3281795067":"2023-10-10T05:43:16.303186+0000","172.21.15.64:0/2115451128":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/1951412611":"2023-10-10T05:42:37.227748+0000","172.21.15.156:0/698211702":"2023-10-10T05:43:16.303186+0000","172.21.15.64:0/514741379":"2023-10-10T05:34:09.281655+0000","172.21.15.64:6800/3793931367":"2023-10-10T05:34:09.281655+0000","172.21.15.64:6801/984813922":"2023-10-10T05:42:37.227748+0000","172.21.15.64:0/3185620259":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/4054596417":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6800/3627852432":"2023-10-10T05:39:05.525329+0000","172.21.15.64:6800/1111090977":"2023-10-10T05:43:58.314769+0000","172.21.15.64:0/1562509700":"2023-10-10T05:34:35.744792+0000","172.21.15.64:0/1571829871":"2023-10-10T05:42:37.227748+0000","172.21.15.64:6800/984813922":"2023-10-10T05:42:37.227748+0000","172.21.15.64:0/3038747921":"2023-10-10T05:42:37.227748+0000","172.21.15.64:0/390193900":"2023-10-10T05:43:58.314769+0000","172.21.15.156:0/2936069725":"2023-10-10T05:43:16.303186+0000","172.21.15.64:0/3346006057":"2023-10-10T05:34:09.281655+0000","172.21.15.156:0/1526589339":"2023-10-10T05:43:16.303186+0000","172.21.15.64:0/1900245998":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6801/1111090977":"2023-10-10T05:43:58.314769+0000","172.21.15.64:0/1336186767":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/2171400226":"2023-10-10T05:34:09.281655+0000","172.21.15.64:6801/3793931367":"2023-10-10T05:34:09.281655+0000","172.21.15.64:0/3707401615":"2023-10-10T05:39:05.525329+0000","172.21.15.64:6801/3627852432":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/584000461":"2023-10-10T05:43:58.314769+0000","172.21.15.64:6800/1833729701":"2023-10-10T05:34:35.744792+0000","172.21.15.64:0/2736837524":"2023-10-10T05:43:58.314769+0000","172.21.15.64:6801/1833729701":"2023-10-10T05:34:35.744792+0000","172.21.15.64:0/3029574061":"2023-10-10T05:42:37.227748+0000","172.21.15.156:6834/3461673683":"2023-10-10T05:43:16.303186+0000","172.21.15.64:0/479050685":"2023-10-10T05:43:58.314769+0000","172.21.15.156:0/4113996764":"2023-10-10T05:43:16.303186+0000","172.21.15.156:6835/3461673683":"2023-10-10T05:43:16.303186+0000","172.21.15.64:0/1766272185":"2023-10-10T05:43:58.314769+0000","172.21.15.64:0/2324320735":"2023-10-10T05:43:58.314769+0000","172.21.15.156:0/1457402607":"2023-10-10T05:43:16.303186+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":12,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T06:22:23.717 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T06:22:23.717 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T06:22:23.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:23 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1919459616' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:22:23.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:23 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1919459616' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:22:24.088 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:24.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:23 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1919459616' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:22:25.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:24 smithi064 ceph-mon[212750]: pgmap v1785: 103 pgs: 1 active, 102 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:25.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:24 smithi064 ceph-mon[220044]: pgmap v1785: 103 pgs: 1 active, 102 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:25.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:24 smithi156 ceph-mon[195101]: pgmap v1785: 103 pgs: 1 active, 102 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:25.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:22:25 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:22:25] "GET /metrics HTTP/1.1" 200 34088 "" "Prometheus/2.43.0" 2023-10-09T06:22:25.797 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:22:25.797 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":710,"fsid":"44c77acc-6665-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:33:40.037198+0000","modified":"2023-10-09T06:22:21.022982+0000","last_up_change":"2023-10-09T06:18:31.967604+0000","last_in_change":"2023-10-09T06:21:21.908025+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":12,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:36:42.276784+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"347","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"299","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:39:17.895056+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"665","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":665,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T05:40:11.204791+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T05:40:12.067097+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"85d0857a-ca32-43d2-8cec-bbd916db9f98","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":129,"last_clean_end":437,"up_from":456,"up_thru":706,"down_at":438,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":119405434},{"type":"v1","addr":"172.21.15.64:6803","nonce":119405434}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":119405434},{"type":"v1","addr":"172.21.15.64:6805","nonce":119405434}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":119405434},{"type":"v1","addr":"172.21.15.64:6809","nonce":119405434}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":119405434},{"type":"v1","addr":"172.21.15.64:6807","nonce":119405434}]},"public_addr":"172.21.15.64:6803/119405434","cluster_addr":"172.21.15.64:6805/119405434","heartbeat_back_addr":"172.21.15.64:6809/119405434","heartbeat_front_addr":"172.21.15.64:6807/119405434","state":["exists","up"]},{"osd":1,"uuid":"05570845-2cd8-4e24-a2f5-f46cf67aa31a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":85,"up_from":89,"up_thru":690,"down_at":86,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":3188144294},{"type":"v1","addr":"172.21.15.64:6811","nonce":3188144294}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":3188144294},{"type":"v1","addr":"172.21.15.64:6813","nonce":3188144294}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":3188144294},{"type":"v1","addr":"172.21.15.64:6817","nonce":3188144294}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":3188144294},{"type":"v1","addr":"172.21.15.64:6815","nonce":3188144294}]},"public_addr":"172.21.15.64:6811/3188144294","cluster_addr":"172.21.15.64:6813/3188144294","heartbeat_back_addr":"172.21.15.64:6817/3188144294","heartbeat_front_addr":"172.21.15.64:6815/3188144294","state":["exists","up"]},{"osd":2,"uuid":"0f306460-9f08-4f15-b573-3587f63be61c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":90,"up_from":94,"up_thru":702,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":3375454963},{"type":"v1","addr":"172.21.15.64:6819","nonce":3375454963}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":3375454963},{"type":"v1","addr":"172.21.15.64:6821","nonce":3375454963}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":3375454963},{"type":"v1","addr":"172.21.15.64:6825","nonce":3375454963}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":3375454963},{"type":"v1","addr":"172.21.15.64:6823","nonce":3375454963}]},"public_addr":"172.21.15.64:6819/3375454963","cluster_addr":"172.21.15.64:6821/3375454963","heartbeat_back_addr":"172.21.15.64:6825/3375454963","heartbeat_front_addr":"172.21.15.64:6823/3375454963","state":["exists","up"]},{"osd":3,"uuid":"4bf38edd-e555-402e-ba4b-c751de06aa6a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":95,"up_from":99,"up_thru":702,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":1043927146},{"type":"v1","addr":"172.21.15.64:6827","nonce":1043927146}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":1043927146},{"type":"v1","addr":"172.21.15.64:6829","nonce":1043927146}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":1043927146},{"type":"v1","addr":"172.21.15.64:6833","nonce":1043927146}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":1043927146},{"type":"v1","addr":"172.21.15.64:6831","nonce":1043927146}]},"public_addr":"172.21.15.64:6827/1043927146","cluster_addr":"172.21.15.64:6829/1043927146","heartbeat_back_addr":"172.21.15.64:6833/1043927146","heartbeat_front_addr":"172.21.15.64:6831/1043927146","state":["exists","up"]},{"osd":4,"uuid":"060e8766-7bf6-42f1-bebe-dae6828f7fb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":315,"up_from":322,"up_thru":702,"down_at":316,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6800","nonce":3173423256},{"type":"v1","addr":"172.21.15.156:6801","nonce":3173423256}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":3173423256},{"type":"v1","addr":"172.21.15.156:6803","nonce":3173423256}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":3173423256},{"type":"v1","addr":"172.21.15.156:6807","nonce":3173423256}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":3173423256},{"type":"v1","addr":"172.21.15.156:6805","nonce":3173423256}]},"public_addr":"172.21.15.156:6801/3173423256","cluster_addr":"172.21.15.156:6803/3173423256","heartbeat_back_addr":"172.21.15.156:6807/3173423256","heartbeat_front_addr":"172.21.15.156:6805/3173423256","state":["exists","up"]},{"osd":5,"uuid":"6b5cf38a-6792-4ac7-9694-268601260ba8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":38,"last_clean_end":598,"up_from":611,"up_thru":709,"down_at":599,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":507344993},{"type":"v1","addr":"172.21.15.156:6809","nonce":507344993}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":507344993},{"type":"v1","addr":"172.21.15.156:6811","nonce":507344993}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":507344993},{"type":"v1","addr":"172.21.15.156:6815","nonce":507344993}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":507344993},{"type":"v1","addr":"172.21.15.156:6813","nonce":507344993}]},"public_addr":"172.21.15.156:6809/507344993","cluster_addr":"172.21.15.156:6811/507344993","heartbeat_back_addr":"172.21.15.156:6815/507344993","heartbeat_front_addr":"172.21.15.156:6813/507344993","state":["exists","up"]},{"osd":6,"uuid":"01deea2d-a225-40b8-a54b-61eee57da527","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":698,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6817","nonce":2369965428}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6819","nonce":2369965428}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6823","nonce":2369965428}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":2369965428},{"type":"v1","addr":"172.21.15.156:6821","nonce":2369965428}]},"public_addr":"172.21.15.156:6817/2369965428","cluster_addr":"172.21.15.156:6819/2369965428","heartbeat_back_addr":"172.21.15.156:6823/2369965428","heartbeat_front_addr":"172.21.15.156:6821/2369965428","state":["exists","up"]},{"osd":7,"uuid":"693e93b0-c235-47fe-9172-85af68e38634","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":693,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6825","nonce":2767546322}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6827","nonce":2767546322}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6831","nonce":2767546322}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":2767546322},{"type":"v1","addr":"172.21.15.156:6829","nonce":2767546322}]},"public_addr":"172.21.15.156:6825/2767546322","cluster_addr":"172.21.15.156:6827/2767546322","heartbeat_back_addr":"172.21.15.156:6831/2767546322","heartbeat_front_addr":"172.21.15.156:6829/2767546322","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:35:52.928919+0000","dead_epoch":437},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:36:16.479793+0000","dead_epoch":85},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:36:39.503123+0000","dead_epoch":90},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:03.340353+0000","dead_epoch":95},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:24.645080+0000","dead_epoch":315},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:37:46.312540+0000","dead_epoch":598},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:38:08.346971+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:38:30.073761+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.9","mappings":[{"from":1,"to":2}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.156:0/3281795067":"2023-10-10T05:43:16.303186+0000","172.21.15.64:0/2115451128":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/1951412611":"2023-10-10T05:42:37.227748+0000","172.21.15.156:0/698211702":"2023-10-10T05:43:16.303186+0000","172.21.15.64:0/514741379":"2023-10-10T05:34:09.281655+0000","172.21.15.64:6800/3793931367":"2023-10-10T05:34:09.281655+0000","172.21.15.64:6801/984813922":"2023-10-10T05:42:37.227748+0000","172.21.15.64:0/3185620259":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/4054596417":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6800/3627852432":"2023-10-10T05:39:05.525329+0000","172.21.15.64:6800/1111090977":"2023-10-10T05:43:58.314769+0000","172.21.15.64:0/1562509700":"2023-10-10T05:34:35.744792+0000","172.21.15.64:0/1571829871":"2023-10-10T05:42:37.227748+0000","172.21.15.64:6800/984813922":"2023-10-10T05:42:37.227748+0000","172.21.15.64:0/3038747921":"2023-10-10T05:42:37.227748+0000","172.21.15.64:0/390193900":"2023-10-10T05:43:58.314769+0000","172.21.15.156:0/2936069725":"2023-10-10T05:43:16.303186+0000","172.21.15.64:0/3346006057":"2023-10-10T05:34:09.281655+0000","172.21.15.156:0/1526589339":"2023-10-10T05:43:16.303186+0000","172.21.15.64:0/1900245998":"2023-10-10T05:34:35.744792+0000","172.21.15.64:6801/1111090977":"2023-10-10T05:43:58.314769+0000","172.21.15.64:0/1336186767":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/2171400226":"2023-10-10T05:34:09.281655+0000","172.21.15.64:6801/3793931367":"2023-10-10T05:34:09.281655+0000","172.21.15.64:0/3707401615":"2023-10-10T05:39:05.525329+0000","172.21.15.64:6801/3627852432":"2023-10-10T05:39:05.525329+0000","172.21.15.64:0/584000461":"2023-10-10T05:43:58.314769+0000","172.21.15.64:6800/1833729701":"2023-10-10T05:34:35.744792+0000","172.21.15.64:0/2736837524":"2023-10-10T05:43:58.314769+0000","172.21.15.64:6801/1833729701":"2023-10-10T05:34:35.744792+0000","172.21.15.64:0/3029574061":"2023-10-10T05:42:37.227748+0000","172.21.15.156:6834/3461673683":"2023-10-10T05:43:16.303186+0000","172.21.15.64:0/479050685":"2023-10-10T05:43:58.314769+0000","172.21.15.156:0/4113996764":"2023-10-10T05:43:16.303186+0000","172.21.15.156:6835/3461673683":"2023-10-10T05:43:16.303186+0000","172.21.15.64:0/1766272185":"2023-10-10T05:43:58.314769+0000","172.21.15.64:0/2324320735":"2023-10-10T05:43:58.314769+0000","172.21.15.156:0/1457402607":"2023-10-10T05:43:16.303186+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":12,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T06:22:26.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:25 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/1368662308' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:22:26.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:25 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/1368662308' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:22:26.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:25 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/1368662308' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:22:26.550 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T06:22:26.551 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T06:22:26.551 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T06:22:26.552 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T06:22:26.552 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T06:22:26.552 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T06:22:26.553 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T06:22:26.553 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T06:22:26.905 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:26 smithi064 ceph-mon[212750]: pgmap v1786: 103 pgs: 1 active, 102 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:27.178 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:27.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:26 smithi064 ceph-mon[220044]: pgmap v1786: 103 pgs: 1 active, 102 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:27.305 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:27.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:26 smithi156 ceph-mon[195101]: pgmap v1786: 103 pgs: 1 active, 102 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:27.399 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:27.695 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:27.695 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:28.654 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:29.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:28 smithi064 ceph-mon[212750]: pgmap v1787: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:29.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:28 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:29.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:28 smithi064 ceph-mon[220044]: pgmap v1787: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:29.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:28 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:29.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:28 smithi156 ceph-mon[195101]: pgmap v1787: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:29.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:28 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:30.768 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:31.032 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:30 smithi064 ceph-mon[212750]: pgmap v1788: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:31.033 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:30 smithi064 ceph-mon[220044]: pgmap v1788: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:31.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:30 smithi156 ceph-mon[195101]: pgmap v1788: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:31.853 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:32.809 INFO:teuthology.orchestra.run.smithi064.stdout:1958505087134 2023-10-09T06:22:32.810 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T06:22:33.068 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:32 smithi064 ceph-mon[212750]: pgmap v1789: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:33.068 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:32 smithi064 ceph-mon[220044]: pgmap v1789: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:33.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:32 smithi156 ceph-mon[195101]: pgmap v1789: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:35.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:34 smithi064 ceph-mon[212750]: pgmap v1790: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:35.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:34 smithi064 ceph-mon[212750]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:22:35.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:34 smithi064 ceph-mon[220044]: pgmap v1790: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:35.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:34 smithi064 ceph-mon[220044]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:22:35.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:34 smithi156 ceph-mon[195101]: pgmap v1790: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:35.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:34 smithi156 ceph-mon[195101]: from='mgr.34107 ' entity='mgr.y' 2023-10-09T06:22:35.504 INFO:teuthology.orchestra.run.smithi064.stdout:231928234515 2023-10-09T06:22:35.504 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T06:22:35.719 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:22:35 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:22:35] "GET /metrics HTTP/1.1" 200 34084 "" "Prometheus/2.43.0" 2023-10-09T06:22:36.505 INFO:teuthology.orchestra.run.smithi064.stdout:197568496153 2023-10-09T06:22:36.505 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T06:22:37.194 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:36 smithi064 ceph-mon[212750]: pgmap v1791: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:37.194 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:36 smithi064 ceph-mon[220044]: pgmap v1791: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:37.211 INFO:teuthology.orchestra.run.smithi064.stdout:425201762705 2023-10-09T06:22:37.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T06:22:37.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:36 smithi156 ceph-mon[195101]: pgmap v1791: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:37.567 INFO:teuthology.orchestra.run.smithi064.stdout:403726926232 2023-10-09T06:22:37.567 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T06:22:38.852 INFO:teuthology.orchestra.run.smithi064.stdout:2624225017906 2023-10-09T06:22:38.852 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T06:22:38.894 INFO:teuthology.orchestra.run.smithi064.stdout:382252089741 2023-10-09T06:22:38.894 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T06:22:38.975 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:39.064 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:39.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:38 smithi064 ceph-mon[212750]: pgmap v1792: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:39.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:38 smithi064 ceph-mon[220044]: pgmap v1792: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:39.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:38 smithi156 ceph-mon[195101]: pgmap v1792: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:40.060 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:40.084 INFO:teuthology.orchestra.run.smithi064.stdout:1382979469569 2023-10-09T06:22:40.084 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T06:22:40.388 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:41.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:40 smithi064 ceph-mon[212750]: pgmap v1793: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:41.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:40 smithi064 ceph-mon[220044]: pgmap v1793: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:41.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:40 smithi156 ceph-mon[195101]: pgmap v1793: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:41.391 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:41.643 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:43.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:42 smithi064 ceph-mon[212750]: pgmap v1794: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:43.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:42 smithi064 ceph-mon[220044]: pgmap v1794: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:43.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:42 smithi156 ceph-mon[195101]: pgmap v1794: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:43.824 INFO:teuthology.orchestra.run.smithi064.stdout:197568496154 2023-10-09T06:22:43.837 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:43.911 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:44.130 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:43 smithi064 ceph-mon[212750]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:44.130 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:43 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2062904694' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T06:22:44.130 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:43 smithi064 ceph-mon[220044]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:44.130 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:43 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2062904694' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T06:22:44.247 INFO:teuthology.orchestra.run.smithi064.stdout:1958505087137 2023-10-09T06:22:44.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:43 smithi156 ceph-mon[195101]: from='mgr.34107 172.21.15.64:0/4182849696' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:44.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:43 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2062904694' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T06:22:44.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:44 smithi064 ceph-mon[212750]: pgmap v1795: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:44.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:44 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/2080053718' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T06:22:44.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:44 smithi064 ceph-mon[220044]: pgmap v1795: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:44.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:44 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/2080053718' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T06:22:45.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:44 smithi156 ceph-mon[195101]: pgmap v1795: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:45.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:44 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/2080053718' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T06:22:45.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:22:45 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:22:45] "GET /metrics HTTP/1.1" 200 34083 "" "Prometheus/2.43.0" 2023-10-09T06:22:45.520 INFO:teuthology.orchestra.run.smithi064.stdout:403726926234 2023-10-09T06:22:45.660 INFO:teuthology.orchestra.run.smithi064.stdout:425201762707 2023-10-09T06:22:45.834 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568496153 got 197568496154 for osd.6 2023-10-09T06:22:45.834 DEBUG:teuthology.parallel:result is None 2023-10-09T06:22:46.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:45 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/878170939' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T06:22:46.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:45 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3222589673' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T06:22:46.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:45 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/878170939' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T06:22:46.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:45 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3222589673' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T06:22:46.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:45 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/878170939' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T06:22:46.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:45 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3222589673' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T06:22:46.391 INFO:teuthology.orchestra.run.smithi064.stdout:231928234518 2023-10-09T06:22:47.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:46 smithi064 ceph-mon[212750]: pgmap v1796: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:47.178 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:46 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/636342659' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T06:22:47.178 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:46 smithi064 ceph-mon[220044]: pgmap v1796: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:47.178 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:46 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/636342659' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T06:22:47.179 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234515 got 231928234518 for osd.7 2023-10-09T06:22:47.179 DEBUG:teuthology.parallel:result is None 2023-10-09T06:22:47.204 INFO:tasks.cephadm.ceph_manager.ceph:need seq 403726926232 got 403726926234 for osd.2 2023-10-09T06:22:47.204 DEBUG:teuthology.parallel:result is None 2023-10-09T06:22:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:46 smithi156 ceph-mon[195101]: pgmap v1796: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:47.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:46 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/636342659' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T06:22:48.892 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1958505087134 got 1958505087137 for osd.0 2023-10-09T06:22:48.893 DEBUG:teuthology.parallel:result is None 2023-10-09T06:22:49.174 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:48 smithi064 ceph-mon[212750]: pgmap v1797: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:49.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:48 smithi064 ceph-mon[220044]: pgmap v1797: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:49.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:48 smithi156 ceph-mon[195101]: pgmap v1797: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:49.521 INFO:tasks.cephadm.ceph_manager.ceph:need seq 425201762705 got 425201762707 for osd.3 2023-10-09T06:22:49.521 DEBUG:teuthology.parallel:result is None 2023-10-09T06:22:49.914 INFO:teuthology.orchestra.run.smithi064.stdout:2624225017909 2023-10-09T06:22:49.994 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:49 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/4100703536' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T06:22:49.994 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:49 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/4100703536' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T06:22:50.154 INFO:teuthology.orchestra.run.smithi064.stdout:382252089743 2023-10-09T06:22:50.372 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:49 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/4100703536' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T06:22:50.824 INFO:teuthology.orchestra.run.smithi064.stdout:1382979469571 2023-10-09T06:22:50.983 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2624225017906 got 2624225017909 for osd.5 2023-10-09T06:22:50.983 DEBUG:teuthology.parallel:result is None 2023-10-09T06:22:51.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:50 smithi064 ceph-mon[212750]: pgmap v1798: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:51.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:50 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/741381862' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T06:22:51.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:50 smithi064 ceph-mon[212750]: from='client.? 172.21.15.64:0/3319376925' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T06:22:51.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:50 smithi064 ceph-mon[220044]: pgmap v1798: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:51.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:50 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/741381862' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T06:22:51.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:50 smithi064 ceph-mon[220044]: from='client.? 172.21.15.64:0/3319376925' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T06:22:51.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:50 smithi156 ceph-mon[195101]: pgmap v1798: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:51.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:50 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/741381862' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T06:22:51.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:50 smithi156 ceph-mon[195101]: from='client.? 172.21.15.64:0/3319376925' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T06:22:51.630 INFO:tasks.cephadm.ceph_manager.ceph:need seq 382252089741 got 382252089743 for osd.1 2023-10-09T06:22:51.631 DEBUG:teuthology.parallel:result is None 2023-10-09T06:22:51.830 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1382979469569 got 1382979469571 for osd.4 2023-10-09T06:22:51.830 DEBUG:teuthology.parallel:result is None 2023-10-09T06:22:51.830 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T06:22:51.831 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:22:52.202 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:53.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:52 smithi064 ceph-mon[212750]: pgmap v1799: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:53.220 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:52 smithi064 ceph-mon[220044]: pgmap v1799: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:53.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:52 smithi156 ceph-mon[195101]: pgmap v1799: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:53.958 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:22:53.960 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T06:22:54.678 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1800,"stamp":"2023-10-09T06:22:53.394146+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5962,"num_read_kb":13177,"num_write":908,"num_write_kb":12725,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":212,"num_bytes_recovered":7107771,"num_keys_recovered":94,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6022,"ondisk_log_size":6022,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6930724,"kb_used_data":79780,"kb_used_omap":264,"kb_used_meta":6850615,"kb_avail":743030492,"statfs":{"total":767960285184,"available":760863223808,"internally_reserved":0,"allocated":81694720,"data_stored":59931436,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":270708,"internal_metadata":7015030412},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003319"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":1026,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.042876+0000","last_change":"2023-10-09T06:22:21.042876+0000","last_active":"2023-10-09T06:22:21.042876+0000","last_peered":"2023-10-09T06:22:21.042876+0000","last_clean":"2023-10-09T06:22:21.042876+0000","last_became_active":"2023-10-09T06:22:21.042521+0000","last_became_peered":"2023-10-09T06:22:21.042521+0000","last_unstale":"2023-10-09T06:22:21.042876+0000","last_undegraded":"2023-10-09T06:22:21.042876+0000","last_fullsized":"2023-10-09T06:22:21.042876+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:05.412941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:05.412941+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:29:23.871949+0000","scrub_duration":0.0061014329999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1576128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":57,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.029403+0000","last_change":"2023-10-09T06:21:19.918486+0000","last_active":"2023-10-09T06:22:20.029403+0000","last_peered":"2023-10-09T06:22:20.029403+0000","last_clean":"2023-10-09T06:22:20.029403+0000","last_became_active":"2023-10-09T06:21:19.918089+0000","last_became_peered":"2023-10-09T06:21:19.918089+0000","last_unstale":"2023-10-09T06:22:20.029403+0000","last_undegraded":"2023-10-09T06:22:20.029403+0000","last_fullsized":"2023-10-09T06:22:20.029403+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:16.089537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":542,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.027225+0000","last_change":"2023-10-09T06:21:14.895796+0000","last_active":"2023-10-09T06:22:21.027225+0000","last_peered":"2023-10-09T06:22:21.027225+0000","last_clean":"2023-10-09T06:22:21.027225+0000","last_became_active":"2023-10-09T06:21:14.895409+0000","last_became_peered":"2023-10-09T06:21:14.895409+0000","last_unstale":"2023-10-09T06:22:21.027225+0000","last_undegraded":"2023-10-09T06:22:21.027225+0000","last_fullsized":"2023-10-09T06:22:21.027225+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:07.852878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":641,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.043336+0000","last_change":"2023-10-09T06:22:21.043336+0000","last_active":"2023-10-09T06:22:21.043336+0000","last_peered":"2023-10-09T06:22:21.043336+0000","last_clean":"2023-10-09T06:22:21.043336+0000","last_became_active":"2023-10-09T06:22:21.042971+0000","last_became_peered":"2023-10-09T06:22:21.042971+0000","last_unstale":"2023-10-09T06:22:21.043336+0000","last_undegraded":"2023-10-09T06:22:21.043336+0000","last_fullsized":"2023-10-09T06:22:21.043336+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:21:18.283899+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T06:21:18.283899+0000","last_clean_scrub_stamp":"2023-10-09T06:21:18.283899+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:36.057623+0000","scrub_duration":0.0018838520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":35,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.039293+0000","last_change":"2023-10-09T06:21:31.568560+0000","last_active":"2023-10-09T06:22:20.039293+0000","last_peered":"2023-10-09T06:22:20.039293+0000","last_clean":"2023-10-09T06:22:20.039293+0000","last_became_active":"2023-10-09T06:21:31.568221+0000","last_became_peered":"2023-10-09T06:21:31.568221+0000","last_unstale":"2023-10-09T06:22:20.039293+0000","last_undegraded":"2023-10-09T06:22:20.039293+0000","last_fullsized":"2023-10-09T06:22:20.039293+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:15.935243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":439,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.027013+0000","last_change":"2023-10-09T06:22:13.707343+0000","last_active":"2023-10-09T06:22:20.027013+0000","last_peered":"2023-10-09T06:22:20.027013+0000","last_clean":"2023-10-09T06:22:20.027013+0000","last_became_active":"2023-10-09T06:22:13.706968+0000","last_became_peered":"2023-10-09T06:22:13.706968+0000","last_unstale":"2023-10-09T06:22:20.027013+0000","last_undegraded":"2023-10-09T06:22:20.027013+0000","last_fullsized":"2023-10-09T06:22:20.027013+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:46.165614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":935,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.029503+0000","last_change":"2023-10-09T06:21:34.892433+0000","last_active":"2023-10-09T06:22:20.029503+0000","last_peered":"2023-10-09T06:22:20.029503+0000","last_clean":"2023-10-09T06:22:20.029503+0000","last_became_active":"2023-10-09T06:21:34.891944+0000","last_became_peered":"2023-10-09T06:21:34.891944+0000","last_unstale":"2023-10-09T06:22:20.029503+0000","last_undegraded":"2023-10-09T06:22:20.029503+0000","last_fullsized":"2023-10-09T06:22:20.029503+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.293426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.293426+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:20.649010+0000","scrub_duration":0.001587946,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":533,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.032754+0000","last_change":"2023-10-09T06:21:31.568601+0000","last_active":"2023-10-09T06:22:20.032754+0000","last_peered":"2023-10-09T06:22:20.032754+0000","last_clean":"2023-10-09T06:22:20.032754+0000","last_became_active":"2023-10-09T06:21:31.568081+0000","last_became_peered":"2023-10-09T06:21:31.568081+0000","last_unstale":"2023-10-09T06:22:20.032754+0000","last_undegraded":"2023-10-09T06:22:20.032754+0000","last_fullsized":"2023-10-09T06:22:20.032754+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:23.144207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":174,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.032457+0000","last_change":"2023-10-09T06:21:22.894700+0000","last_active":"2023-10-09T06:22:20.032457+0000","last_peered":"2023-10-09T06:22:20.032457+0000","last_clean":"2023-10-09T06:22:20.032457+0000","last_became_active":"2023-10-09T06:21:22.894338+0000","last_became_peered":"2023-10-09T06:21:22.894338+0000","last_unstale":"2023-10-09T06:22:20.032457+0000","last_undegraded":"2023-10-09T06:22:20.032457+0000","last_fullsized":"2023-10-09T06:22:20.032457+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:58.574067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":985,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.034793+0000","last_change":"2023-10-09T06:21:25.907079+0000","last_active":"2023-10-09T06:22:20.034793+0000","last_peered":"2023-10-09T06:22:20.034793+0000","last_clean":"2023-10-09T06:22:20.034793+0000","last_became_active":"2023-10-09T06:21:25.906690+0000","last_became_peered":"2023-10-09T06:21:25.906690+0000","last_unstale":"2023-10-09T06:22:20.034793+0000","last_undegraded":"2023-10-09T06:22:20.034793+0000","last_fullsized":"2023-10-09T06:22:20.034793+0000","mapping_epoch":683,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":684,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.384407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.384407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:16:42.288044+0000","scrub_duration":0.0040651960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":621,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.027512+0000","last_change":"2023-10-09T06:22:12.916337+0000","last_active":"2023-10-09T06:22:20.027512+0000","last_peered":"2023-10-09T06:22:20.027512+0000","last_clean":"2023-10-09T06:22:20.027512+0000","last_became_active":"2023-10-09T06:22:12.916026+0000","last_became_peered":"2023-10-09T06:22:12.916026+0000","last_unstale":"2023-10-09T06:22:20.027512+0000","last_undegraded":"2023-10-09T06:22:20.027512+0000","last_fullsized":"2023-10-09T06:22:20.027512+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:05.280276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":596,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.032853+0000","last_change":"2023-10-09T06:21:31.568420+0000","last_active":"2023-10-09T06:22:20.032853+0000","last_peered":"2023-10-09T06:22:20.032853+0000","last_clean":"2023-10-09T06:22:20.032853+0000","last_became_active":"2023-10-09T06:21:31.567473+0000","last_became_peered":"2023-10-09T06:21:31.567473+0000","last_unstale":"2023-10-09T06:22:20.032853+0000","last_undegraded":"2023-10-09T06:22:20.032853+0000","last_fullsized":"2023-10-09T06:22:20.032853+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:29.657914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":914,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.038239+0000","last_change":"2023-10-09T06:21:40.939806+0000","last_active":"2023-10-09T06:22:20.038239+0000","last_peered":"2023-10-09T06:22:20.038239+0000","last_clean":"2023-10-09T06:22:20.038239+0000","last_became_active":"2023-10-09T06:21:40.939463+0000","last_became_peered":"2023-10-09T06:21:40.939463+0000","last_unstale":"2023-10-09T06:22:20.038239+0000","last_undegraded":"2023-10-09T06:22:20.038239+0000","last_fullsized":"2023-10-09T06:22:20.038239+0000","mapping_epoch":693,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":694,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:20.742561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:20.742561+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:35.383613+0000","scrub_duration":0.0027005039999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":230,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.042801+0000","last_change":"2023-10-09T06:22:21.042801+0000","last_active":"2023-10-09T06:22:21.042801+0000","last_peered":"2023-10-09T06:22:21.042801+0000","last_clean":"2023-10-09T06:22:21.042801+0000","last_became_active":"2023-10-09T06:22:21.042476+0000","last_became_peered":"2023-10-09T06:22:21.042476+0000","last_unstale":"2023-10-09T06:22:21.042801+0000","last_undegraded":"2023-10-09T06:22:21.042801+0000","last_fullsized":"2023-10-09T06:22:21.042801+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:37.154983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":724,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.033983+0000","last_change":"2023-10-09T06:18:12.816564+0000","last_active":"2023-10-09T06:22:20.033983+0000","last_peered":"2023-10-09T06:22:20.033983+0000","last_clean":"2023-10-09T06:22:20.033983+0000","last_became_active":"2023-10-09T06:18:12.815943+0000","last_became_peered":"2023-10-09T06:18:12.815943+0000","last_unstale":"2023-10-09T06:22:20.033983+0000","last_undegraded":"2023-10-09T06:22:20.033983+0000","last_fullsized":"2023-10-09T06:22:20.033983+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:10.114324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":778,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.039972+0000","last_change":"2023-10-09T06:22:21.039972+0000","last_active":"2023-10-09T06:22:21.039972+0000","last_peered":"2023-10-09T06:22:21.039972+0000","last_clean":"2023-10-09T06:22:21.039972+0000","last_became_active":"2023-10-09T06:22:21.039641+0000","last_became_peered":"2023-10-09T06:22:21.039641+0000","last_unstale":"2023-10-09T06:22:21.039972+0000","last_undegraded":"2023-10-09T06:22:21.039972+0000","last_fullsized":"2023-10-09T06:22:21.039972+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:38.588641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:38.588641+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:48.531530+0000","scrub_duration":0.010116385,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":231,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.034695+0000","last_change":"2023-10-09T06:21:14.922061+0000","last_active":"2023-10-09T06:22:20.034695+0000","last_peered":"2023-10-09T06:22:20.034695+0000","last_clean":"2023-10-09T06:22:20.034695+0000","last_became_active":"2023-10-09T06:21:14.908746+0000","last_became_peered":"2023-10-09T06:21:14.908746+0000","last_unstale":"2023-10-09T06:22:20.034695+0000","last_undegraded":"2023-10-09T06:22:20.034695+0000","last_fullsized":"2023-10-09T06:22:20.034695+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:06.933730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":35,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.040027+0000","last_change":"2023-10-09T06:21:31.570013+0000","last_active":"2023-10-09T06:22:20.040027+0000","last_peered":"2023-10-09T06:22:20.040027+0000","last_clean":"2023-10-09T06:22:20.040027+0000","last_became_active":"2023-10-09T06:21:31.569705+0000","last_became_peered":"2023-10-09T06:21:31.569705+0000","last_unstale":"2023-10-09T06:22:20.040027+0000","last_undegraded":"2023-10-09T06:22:20.040027+0000","last_fullsized":"2023-10-09T06:22:20.040027+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:52.150192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":885,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.040379+0000","last_change":"2023-10-09T06:09:27.574336+0000","last_active":"2023-10-09T06:22:20.040379+0000","last_peered":"2023-10-09T06:22:20.040379+0000","last_clean":"2023-10-09T06:22:20.040379+0000","last_became_active":"2023-10-09T06:09:27.573974+0000","last_became_peered":"2023-10-09T06:09:27.573974+0000","last_unstale":"2023-10-09T06:22:20.040379+0000","last_undegraded":"2023-10-09T06:22:20.040379+0000","last_fullsized":"2023-10-09T06:22:20.040379+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.339121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.339121+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:45.908579+0000","scrub_duration":0.010474344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":57,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.029706+0000","last_change":"2023-10-09T06:21:14.926090+0000","last_active":"2023-10-09T06:22:20.029706+0000","last_peered":"2023-10-09T06:22:20.029706+0000","last_clean":"2023-10-09T06:22:20.029706+0000","last_became_active":"2023-10-09T06:21:14.904702+0000","last_became_peered":"2023-10-09T06:21:14.904702+0000","last_unstale":"2023-10-09T06:22:20.029706+0000","last_undegraded":"2023-10-09T06:22:20.029706+0000","last_fullsized":"2023-10-09T06:22:20.029706+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:51.732161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":67,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.029730+0000","last_change":"2023-10-09T06:21:31.575158+0000","last_active":"2023-10-09T06:22:20.029730+0000","last_peered":"2023-10-09T06:22:20.029730+0000","last_clean":"2023-10-09T06:22:20.029730+0000","last_became_active":"2023-10-09T06:21:31.572974+0000","last_became_peered":"2023-10-09T06:21:31.572974+0000","last_unstale":"2023-10-09T06:22:20.029730+0000","last_undegraded":"2023-10-09T06:22:20.029730+0000","last_fullsized":"2023-10-09T06:22:20.029730+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:59.423082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":562,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.026772+0000","last_change":"2023-10-09T06:22:12.916424+0000","last_active":"2023-10-09T06:22:20.026772+0000","last_peered":"2023-10-09T06:22:20.026772+0000","last_clean":"2023-10-09T06:22:20.026772+0000","last_became_active":"2023-10-09T06:22:12.916102+0000","last_became_peered":"2023-10-09T06:22:12.916102+0000","last_unstale":"2023-10-09T06:22:20.026772+0000","last_undegraded":"2023-10-09T06:22:20.026772+0000","last_fullsized":"2023-10-09T06:22:20.026772+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:42.900876+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:42.900876+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:44:30.686361+0000","scrub_duration":0.001618488,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":295,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.034557+0000","last_change":"2023-10-09T06:21:19.932732+0000","last_active":"2023-10-09T06:22:20.034557+0000","last_peered":"2023-10-09T06:22:20.034557+0000","last_clean":"2023-10-09T06:22:20.034557+0000","last_became_active":"2023-10-09T06:21:19.928092+0000","last_became_peered":"2023-10-09T06:21:19.928092+0000","last_unstale":"2023-10-09T06:22:20.034557+0000","last_undegraded":"2023-10-09T06:22:20.034557+0000","last_fullsized":"2023-10-09T06:22:20.034557+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:23.967352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":306,"num_keys_recovered":90,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":804,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.034687+0000","last_change":"2023-10-09T06:22:02.796081+0000","last_active":"2023-10-09T06:22:20.034687+0000","last_peered":"2023-10-09T06:22:20.034687+0000","last_clean":"2023-10-09T06:22:20.034687+0000","last_became_active":"2023-10-09T06:22:02.795655+0000","last_became_peered":"2023-10-09T06:22:02.795655+0000","last_unstale":"2023-10-09T06:22:20.034687+0000","last_undegraded":"2023-10-09T06:22:20.034687+0000","last_fullsized":"2023-10-09T06:22:20.034687+0000","mapping_epoch":702,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":703,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:07.057044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":1300,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.045630+0000","last_change":"2023-10-09T06:22:21.045630+0000","last_active":"2023-10-09T06:22:21.045630+0000","last_peered":"2023-10-09T06:22:21.045630+0000","last_clean":"2023-10-09T06:22:21.045630+0000","last_became_active":"2023-10-09T06:22:21.045293+0000","last_became_peered":"2023-10-09T06:22:21.045293+0000","last_unstale":"2023-10-09T06:22:21.045630+0000","last_undegraded":"2023-10-09T06:22:21.045630+0000","last_fullsized":"2023-10-09T06:22:21.045630+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:29.580389+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":402,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.026454+0000","last_change":"2023-10-09T06:22:13.707470+0000","last_active":"2023-10-09T06:22:20.026454+0000","last_peered":"2023-10-09T06:22:20.026454+0000","last_clean":"2023-10-09T06:22:20.026454+0000","last_became_active":"2023-10-09T06:22:13.706954+0000","last_became_peered":"2023-10-09T06:22:13.706954+0000","last_unstale":"2023-10-09T06:22:20.026454+0000","last_undegraded":"2023-10-09T06:22:20.026454+0000","last_fullsized":"2023-10-09T06:22:20.026454+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:54.394842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":205,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.043525+0000","last_change":"2023-10-09T06:22:21.043525+0000","last_active":"2023-10-09T06:22:21.043525+0000","last_peered":"2023-10-09T06:22:21.043525+0000","last_clean":"2023-10-09T06:22:21.043525+0000","last_became_active":"2023-10-09T06:22:21.042362+0000","last_became_peered":"2023-10-09T06:22:21.042362+0000","last_unstale":"2023-10-09T06:22:21.043525+0000","last_undegraded":"2023-10-09T06:22:21.043525+0000","last_fullsized":"2023-10-09T06:22:21.043525+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:19.700258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":987,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.039837+0000","last_change":"2023-10-09T06:21:17.903751+0000","last_active":"2023-10-09T06:22:20.039837+0000","last_peered":"2023-10-09T06:22:20.039837+0000","last_clean":"2023-10-09T06:22:20.039837+0000","last_became_active":"2023-10-09T06:21:17.903469+0000","last_became_peered":"2023-10-09T06:21:17.903469+0000","last_unstale":"2023-10-09T06:22:20.039837+0000","last_undegraded":"2023-10-09T06:22:20.039837+0000","last_fullsized":"2023-10-09T06:22:20.039837+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.316697+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.316697+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:55.762584+0000","scrub_duration":0.001666561,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":603,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.034232+0000","last_change":"2023-10-09T06:21:14.910129+0000","last_active":"2023-10-09T06:22:20.034232+0000","last_peered":"2023-10-09T06:22:20.034232+0000","last_clean":"2023-10-09T06:22:20.034232+0000","last_became_active":"2023-10-09T06:21:14.909795+0000","last_became_peered":"2023-10-09T06:21:14.909795+0000","last_unstale":"2023-10-09T06:22:20.034232+0000","last_undegraded":"2023-10-09T06:22:20.034232+0000","last_fullsized":"2023-10-09T06:22:20.034232+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:28.905834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":494,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.026915+0000","last_change":"2023-10-09T06:22:13.707340+0000","last_active":"2023-10-09T06:22:20.026915+0000","last_peered":"2023-10-09T06:22:20.026915+0000","last_clean":"2023-10-09T06:22:20.026915+0000","last_became_active":"2023-10-09T06:22:13.706991+0000","last_became_peered":"2023-10-09T06:22:13.706991+0000","last_unstale":"2023-10-09T06:22:20.026915+0000","last_undegraded":"2023-10-09T06:22:20.026915+0000","last_fullsized":"2023-10-09T06:22:20.026915+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:01.637273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":1016,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.041404+0000","last_change":"2023-10-09T06:22:21.041404+0000","last_active":"2023-10-09T06:22:21.041404+0000","last_peered":"2023-10-09T06:22:21.041404+0000","last_clean":"2023-10-09T06:22:21.041404+0000","last_became_active":"2023-10-09T06:22:21.040753+0000","last_became_peered":"2023-10-09T06:22:21.040753+0000","last_unstale":"2023-10-09T06:22:21.041404+0000","last_undegraded":"2023-10-09T06:22:21.041404+0000","last_fullsized":"2023-10-09T06:22:21.041404+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:40.388125+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:40.388125+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:10.312209+0000","scrub_duration":0.0055576990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":321,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.032232+0000","last_change":"2023-10-09T06:21:22.914876+0000","last_active":"2023-10-09T06:22:20.032232+0000","last_peered":"2023-10-09T06:22:20.032232+0000","last_clean":"2023-10-09T06:22:20.032232+0000","last_became_active":"2023-10-09T06:21:22.891764+0000","last_became_peered":"2023-10-09T06:21:22.891764+0000","last_unstale":"2023-10-09T06:22:20.032232+0000","last_undegraded":"2023-10-09T06:22:20.032232+0000","last_fullsized":"2023-10-09T06:22:20.032232+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:56.617645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":248,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.026798+0000","last_change":"2023-10-09T06:21:04.990757+0000","last_active":"2023-10-09T06:22:21.026798+0000","last_peered":"2023-10-09T06:22:21.026798+0000","last_clean":"2023-10-09T06:22:21.026798+0000","last_became_active":"2023-10-09T06:21:04.990151+0000","last_became_peered":"2023-10-09T06:21:04.990151+0000","last_unstale":"2023-10-09T06:22:21.026798+0000","last_undegraded":"2023-10-09T06:22:21.026798+0000","last_fullsized":"2023-10-09T06:22:21.026798+0000","mapping_epoch":666,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":667,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:52.630600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":931,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.035012+0000","last_change":"2023-10-09T06:21:17.908395+0000","last_active":"2023-10-09T06:22:20.035012+0000","last_peered":"2023-10-09T06:22:20.035012+0000","last_clean":"2023-10-09T06:22:20.035012+0000","last_became_active":"2023-10-09T06:21:17.907972+0000","last_became_peered":"2023-10-09T06:21:17.907972+0000","last_unstale":"2023-10-09T06:22:20.035012+0000","last_undegraded":"2023-10-09T06:22:20.035012+0000","last_fullsized":"2023-10-09T06:22:20.035012+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:10:59.422551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:10:59.422551+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:31.931097+0000","scrub_duration":0.0059716659999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":253,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.036002+0000","last_change":"2023-10-09T06:21:14.933563+0000","last_active":"2023-10-09T06:22:20.036002+0000","last_peered":"2023-10-09T06:22:20.036002+0000","last_clean":"2023-10-09T06:22:20.036002+0000","last_became_active":"2023-10-09T06:21:14.910247+0000","last_became_peered":"2023-10-09T06:21:14.910247+0000","last_unstale":"2023-10-09T06:22:20.036002+0000","last_undegraded":"2023-10-09T06:22:20.036002+0000","last_fullsized":"2023-10-09T06:22:20.036002+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:55.999558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":239,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.045188+0000","last_change":"2023-10-09T06:22:21.045188+0000","last_active":"2023-10-09T06:22:21.045188+0000","last_peered":"2023-10-09T06:22:21.045188+0000","last_clean":"2023-10-09T06:22:21.045188+0000","last_became_active":"2023-10-09T06:22:21.044873+0000","last_became_peered":"2023-10-09T06:22:21.044873+0000","last_unstale":"2023-10-09T06:22:21.045188+0000","last_undegraded":"2023-10-09T06:22:21.045188+0000","last_fullsized":"2023-10-09T06:22:21.045188+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:55.954049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":1128,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.027449+0000","last_change":"2023-10-09T06:18:15.867832+0000","last_active":"2023-10-09T06:22:21.027449+0000","last_peered":"2023-10-09T06:22:21.027449+0000","last_clean":"2023-10-09T06:22:21.027449+0000","last_became_active":"2023-10-09T06:18:15.864512+0000","last_became_peered":"2023-10-09T06:18:15.864512+0000","last_unstale":"2023-10-09T06:22:21.027449+0000","last_undegraded":"2023-10-09T06:22:21.027449+0000","last_fullsized":"2023-10-09T06:22:21.027449+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:03.241335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:03.241335+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:45.197013+0000","scrub_duration":0.017780547000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":225,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.037050+0000","last_change":"2023-10-09T06:21:14.929007+0000","last_active":"2023-10-09T06:22:20.037050+0000","last_peered":"2023-10-09T06:22:20.037050+0000","last_clean":"2023-10-09T06:22:20.037050+0000","last_became_active":"2023-10-09T06:21:14.909089+0000","last_became_peered":"2023-10-09T06:21:14.909089+0000","last_unstale":"2023-10-09T06:22:20.037050+0000","last_undegraded":"2023-10-09T06:22:20.037050+0000","last_fullsized":"2023-10-09T06:22:20.037050+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:27.294970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":67,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.029347+0000","last_change":"2023-10-09T06:22:02.048965+0000","last_active":"2023-10-09T06:22:20.029347+0000","last_peered":"2023-10-09T06:22:20.029347+0000","last_clean":"2023-10-09T06:22:20.029347+0000","last_became_active":"2023-10-09T06:22:02.048677+0000","last_became_peered":"2023-10-09T06:22:02.048677+0000","last_unstale":"2023-10-09T06:22:20.029347+0000","last_undegraded":"2023-10-09T06:22:20.029347+0000","last_fullsized":"2023-10-09T06:22:20.029347+0000","mapping_epoch":702,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":703,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:15.764062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":1034,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.031792+0000","last_change":"2023-10-09T06:21:25.913480+0000","last_active":"2023-10-09T06:22:20.031792+0000","last_peered":"2023-10-09T06:22:20.031792+0000","last_clean":"2023-10-09T06:22:20.031792+0000","last_became_active":"2023-10-09T06:21:25.913120+0000","last_became_peered":"2023-10-09T06:21:25.913120+0000","last_unstale":"2023-10-09T06:22:20.031792+0000","last_undegraded":"2023-10-09T06:22:20.031792+0000","last_fullsized":"2023-10-09T06:22:20.031792+0000","mapping_epoch":683,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":684,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:48.495739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:48.495739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:28.522756+0000","scrub_duration":0.0016345509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":274,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.034893+0000","last_change":"2023-10-09T06:21:22.919873+0000","last_active":"2023-10-09T06:22:20.034893+0000","last_peered":"2023-10-09T06:22:20.034893+0000","last_clean":"2023-10-09T06:22:20.034893+0000","last_became_active":"2023-10-09T06:21:22.906107+0000","last_became_peered":"2023-10-09T06:21:22.906107+0000","last_unstale":"2023-10-09T06:22:20.034893+0000","last_undegraded":"2023-10-09T06:22:20.034893+0000","last_fullsized":"2023-10-09T06:22:20.034893+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:35.309519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":721,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.033294+0000","last_change":"2023-10-09T06:20:29.964135+0000","last_active":"2023-10-09T06:22:20.033294+0000","last_peered":"2023-10-09T06:22:20.033294+0000","last_clean":"2023-10-09T06:22:20.033294+0000","last_became_active":"2023-10-09T06:20:29.963434+0000","last_became_peered":"2023-10-09T06:20:29.963434+0000","last_unstale":"2023-10-09T06:22:20.033294+0000","last_undegraded":"2023-10-09T06:22:20.033294+0000","last_fullsized":"2023-10-09T06:22:20.033294+0000","mapping_epoch":646,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":647,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:42.376435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":784,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.031964+0000","last_change":"2023-10-09T06:21:17.906807+0000","last_active":"2023-10-09T06:22:20.031964+0000","last_peered":"2023-10-09T06:22:20.031964+0000","last_clean":"2023-10-09T06:22:20.031964+0000","last_became_active":"2023-10-09T06:21:17.906359+0000","last_became_peered":"2023-10-09T06:21:17.906359+0000","last_unstale":"2023-10-09T06:22:20.031964+0000","last_undegraded":"2023-10-09T06:22:20.031964+0000","last_fullsized":"2023-10-09T06:22:20.031964+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:16.361097+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:16.361097+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:34.936406+0000","scrub_duration":0.028533749000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":510,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.036945+0000","last_change":"2023-10-09T06:22:21.036945+0000","last_active":"2023-10-09T06:22:21.036945+0000","last_peered":"2023-10-09T06:22:21.036945+0000","last_clean":"2023-10-09T06:22:21.036945+0000","last_became_active":"2023-10-09T06:22:21.036583+0000","last_became_peered":"2023-10-09T06:22:21.036583+0000","last_unstale":"2023-10-09T06:22:21.036945+0000","last_undegraded":"2023-10-09T06:22:21.036945+0000","last_fullsized":"2023-10-09T06:22:21.036945+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:28.504108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":744,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.026793+0000","last_change":"2023-10-09T06:21:14.889819+0000","last_active":"2023-10-09T06:22:20.026793+0000","last_peered":"2023-10-09T06:22:20.026793+0000","last_clean":"2023-10-09T06:22:20.026793+0000","last_became_active":"2023-10-09T06:21:14.889428+0000","last_became_peered":"2023-10-09T06:21:14.889428+0000","last_unstale":"2023-10-09T06:22:20.026793+0000","last_undegraded":"2023-10-09T06:22:20.026793+0000","last_fullsized":"2023-10-09T06:22:20.026793+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:01.117391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":1193,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.034069+0000","last_change":"2023-10-09T06:21:40.939446+0000","last_active":"2023-10-09T06:22:20.034069+0000","last_peered":"2023-10-09T06:22:20.034069+0000","last_clean":"2023-10-09T06:22:20.034069+0000","last_became_active":"2023-10-09T06:21:40.939117+0000","last_became_peered":"2023-10-09T06:21:40.939117+0000","last_unstale":"2023-10-09T06:22:20.034069+0000","last_undegraded":"2023-10-09T06:22:20.034069+0000","last_fullsized":"2023-10-09T06:22:20.034069+0000","mapping_epoch":693,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":694,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:42.795781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:42.795781+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:29.492945+0000","scrub_duration":0.0091262979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":556,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.033124+0000","last_change":"2023-10-09T06:21:14.911214+0000","last_active":"2023-10-09T06:22:20.033124+0000","last_peered":"2023-10-09T06:22:20.033124+0000","last_clean":"2023-10-09T06:22:20.033124+0000","last_became_active":"2023-10-09T06:21:14.899220+0000","last_became_peered":"2023-10-09T06:21:14.899220+0000","last_unstale":"2023-10-09T06:22:20.033124+0000","last_undegraded":"2023-10-09T06:22:20.033124+0000","last_fullsized":"2023-10-09T06:22:20.033124+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:32.381582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":47,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.029661+0000","last_change":"2023-10-09T06:21:14.906829+0000","last_active":"2023-10-09T06:22:20.029661+0000","last_peered":"2023-10-09T06:22:20.029661+0000","last_clean":"2023-10-09T06:22:20.029661+0000","last_became_active":"2023-10-09T06:21:14.906543+0000","last_became_peered":"2023-10-09T06:21:14.906543+0000","last_unstale":"2023-10-09T06:22:20.029661+0000","last_undegraded":"2023-10-09T06:22:20.029661+0000","last_fullsized":"2023-10-09T06:22:20.029661+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:55:26.944801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":1071,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.037748+0000","last_change":"2023-10-09T06:22:21.037748+0000","last_active":"2023-10-09T06:22:21.037748+0000","last_peered":"2023-10-09T06:22:21.037748+0000","last_clean":"2023-10-09T06:22:21.037748+0000","last_became_active":"2023-10-09T06:22:21.037443+0000","last_became_peered":"2023-10-09T06:22:21.037443+0000","last_unstale":"2023-10-09T06:22:21.037748+0000","last_undegraded":"2023-10-09T06:22:21.037748+0000","last_fullsized":"2023-10-09T06:22:21.037748+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.282393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.282393+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:39.433468+0000","scrub_duration":0.01402403,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":585,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.033766+0000","last_change":"2023-10-09T06:21:54.068630+0000","last_active":"2023-10-09T06:22:20.033766+0000","last_peered":"2023-10-09T06:22:20.033766+0000","last_clean":"2023-10-09T06:22:20.033766+0000","last_became_active":"2023-10-09T06:21:54.068364+0000","last_became_peered":"2023-10-09T06:21:54.068364+0000","last_unstale":"2023-10-09T06:22:20.033766+0000","last_undegraded":"2023-10-09T06:22:20.033766+0000","last_fullsized":"2023-10-09T06:22:20.033766+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:19.998776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":57,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.032301+0000","last_change":"2023-10-09T06:21:37.886287+0000","last_active":"2023-10-09T06:22:20.032301+0000","last_peered":"2023-10-09T06:22:20.032301+0000","last_clean":"2023-10-09T06:22:20.032301+0000","last_became_active":"2023-10-09T06:21:37.886038+0000","last_became_peered":"2023-10-09T06:21:37.886038+0000","last_unstale":"2023-10-09T06:22:20.032301+0000","last_undegraded":"2023-10-09T06:22:20.032301+0000","last_fullsized":"2023-10-09T06:22:20.032301+0000","mapping_epoch":690,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":691,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:06:13.080046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":1097,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.039690+0000","last_change":"2023-10-09T06:21:25.905224+0000","last_active":"2023-10-09T06:22:20.039690+0000","last_peered":"2023-10-09T06:22:20.039690+0000","last_clean":"2023-10-09T06:22:20.039690+0000","last_became_active":"2023-10-09T06:21:25.904818+0000","last_became_peered":"2023-10-09T06:21:25.904818+0000","last_unstale":"2023-10-09T06:22:20.039690+0000","last_undegraded":"2023-10-09T06:22:20.039690+0000","last_fullsized":"2023-10-09T06:22:20.039690+0000","mapping_epoch":683,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":684,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.373748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.373748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:37.543642+0000","scrub_duration":0.0024576450000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":57,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.029248+0000","last_change":"2023-10-09T06:21:19.924713+0000","last_active":"2023-10-09T06:22:20.029248+0000","last_peered":"2023-10-09T06:22:20.029248+0000","last_clean":"2023-10-09T06:22:20.029248+0000","last_became_active":"2023-10-09T06:21:19.924421+0000","last_became_peered":"2023-10-09T06:21:19.924421+0000","last_unstale":"2023-10-09T06:22:20.029248+0000","last_undegraded":"2023-10-09T06:22:20.029248+0000","last_fullsized":"2023-10-09T06:22:20.029248+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:33.242263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":57,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.031873+0000","last_change":"2023-10-09T06:21:31.570579+0000","last_active":"2023-10-09T06:22:20.031873+0000","last_peered":"2023-10-09T06:22:20.031873+0000","last_clean":"2023-10-09T06:22:20.031873+0000","last_became_active":"2023-10-09T06:21:31.570215+0000","last_became_peered":"2023-10-09T06:21:31.570215+0000","last_unstale":"2023-10-09T06:22:20.031873+0000","last_undegraded":"2023-10-09T06:22:20.031873+0000","last_fullsized":"2023-10-09T06:22:20.031873+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:55.013279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":808,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.040514+0000","last_change":"2023-10-09T06:22:21.040514+0000","last_active":"2023-10-09T06:22:21.040514+0000","last_peered":"2023-10-09T06:22:21.040514+0000","last_clean":"2023-10-09T06:22:21.040514+0000","last_became_active":"2023-10-09T06:22:21.039886+0000","last_became_peered":"2023-10-09T06:22:21.039886+0000","last_unstale":"2023-10-09T06:22:21.040514+0000","last_undegraded":"2023-10-09T06:22:21.040514+0000","last_fullsized":"2023-10-09T06:22:21.040514+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:39.633410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:39.633410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.633552+0000","scrub_duration":0.0039938459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":561,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.032641+0000","last_change":"2023-10-09T06:21:31.569751+0000","last_active":"2023-10-09T06:22:20.032641+0000","last_peered":"2023-10-09T06:22:20.032641+0000","last_clean":"2023-10-09T06:22:20.032641+0000","last_became_active":"2023-10-09T06:21:31.569508+0000","last_became_peered":"2023-10-09T06:21:31.569508+0000","last_unstale":"2023-10-09T06:22:20.032641+0000","last_undegraded":"2023-10-09T06:22:20.032641+0000","last_fullsized":"2023-10-09T06:22:20.032641+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:34.063041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":414,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.040477+0000","last_change":"2023-10-09T06:21:14.889635+0000","last_active":"2023-10-09T06:22:20.040477+0000","last_peered":"2023-10-09T06:22:20.040477+0000","last_clean":"2023-10-09T06:22:20.040477+0000","last_became_active":"2023-10-09T06:21:14.889288+0000","last_became_peered":"2023-10-09T06:21:14.889288+0000","last_unstale":"2023-10-09T06:22:20.040477+0000","last_undegraded":"2023-10-09T06:22:20.040477+0000","last_fullsized":"2023-10-09T06:22:20.040477+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:01:23.826329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":909,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.031686+0000","last_change":"2023-10-09T06:20:29.966275+0000","last_active":"2023-10-09T06:22:20.031686+0000","last_peered":"2023-10-09T06:22:20.031686+0000","last_clean":"2023-10-09T06:22:20.031686+0000","last_became_active":"2023-10-09T06:20:29.964913+0000","last_became_peered":"2023-10-09T06:20:29.964913+0000","last_unstale":"2023-10-09T06:22:20.031686+0000","last_undegraded":"2023-10-09T06:22:20.031686+0000","last_fullsized":"2023-10-09T06:22:20.031686+0000","mapping_epoch":646,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":647,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:12:22.289803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:12:22.289803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:17.781708+0000","scrub_duration":0.0076565050000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"670'187","reported_seq":1040,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.039726+0000","last_change":"2023-10-09T06:21:34.889682+0000","last_active":"2023-10-09T06:22:20.039726+0000","last_peered":"2023-10-09T06:22:20.039726+0000","last_clean":"2023-10-09T06:22:20.039726+0000","last_became_active":"2023-10-09T06:21:34.889089+0000","last_became_peered":"2023-10-09T06:21:34.889089+0000","last_unstale":"2023-10-09T06:22:20.039726+0000","last_undegraded":"2023-10-09T06:22:20.039726+0000","last_fullsized":"2023-10-09T06:22:20.039726+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":187,"log_dups_size":0,"ondisk_log_size":187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:20.611558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000879378,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":899,"num_read_kb":1784,"num_write":189,"num_write_kb":1122,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":10240,"num_keys_recovered":4,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":547,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.043579+0000","last_change":"2023-10-09T06:22:21.043579+0000","last_active":"2023-10-09T06:22:21.043579+0000","last_peered":"2023-10-09T06:22:21.043579+0000","last_clean":"2023-10-09T06:22:21.043579+0000","last_became_active":"2023-10-09T06:22:21.043092+0000","last_became_peered":"2023-10-09T06:22:21.043092+0000","last_unstale":"2023-10-09T06:22:21.043579+0000","last_undegraded":"2023-10-09T06:22:21.043579+0000","last_fullsized":"2023-10-09T06:22:21.043579+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:48.367616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":35,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.039909+0000","last_change":"2023-10-09T06:21:31.572397+0000","last_active":"2023-10-09T06:22:20.039909+0000","last_peered":"2023-10-09T06:22:20.039909+0000","last_clean":"2023-10-09T06:22:20.039909+0000","last_became_active":"2023-10-09T06:21:31.572093+0000","last_became_peered":"2023-10-09T06:21:31.572093+0000","last_unstale":"2023-10-09T06:22:20.039909+0000","last_undegraded":"2023-10-09T06:22:20.039909+0000","last_fullsized":"2023-10-09T06:22:20.039909+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:18:56.936089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":1025,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.038680+0000","last_change":"2023-10-09T06:22:21.038680+0000","last_active":"2023-10-09T06:22:21.038680+0000","last_peered":"2023-10-09T06:22:21.038680+0000","last_clean":"2023-10-09T06:22:21.038680+0000","last_became_active":"2023-10-09T06:22:21.038245+0000","last_became_peered":"2023-10-09T06:22:21.038245+0000","last_unstale":"2023-10-09T06:22:21.038680+0000","last_undegraded":"2023-10-09T06:22:21.038680+0000","last_fullsized":"2023-10-09T06:22:21.038680+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:32.125442+0000","scrub_duration":0.0016660799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"669'115","reported_seq":1235,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.041534+0000","last_change":"2023-10-09T06:22:21.041534+0000","last_active":"2023-10-09T06:22:21.041534+0000","last_peered":"2023-10-09T06:22:21.041534+0000","last_clean":"2023-10-09T06:22:21.041534+0000","last_became_active":"2023-10-09T06:22:21.041112+0000","last_became_peered":"2023-10-09T06:22:21.041112+0000","last_unstale":"2023-10-09T06:22:21.041534+0000","last_undegraded":"2023-10-09T06:22:21.041534+0000","last_fullsized":"2023-10-09T06:22:21.041534+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:09.039150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010144900000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1245,"num_read_kb":2142,"num_write":80,"num_write_kb":1053,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":47,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.029896+0000","last_change":"2023-10-09T06:21:14.906777+0000","last_active":"2023-10-09T06:22:20.029896+0000","last_peered":"2023-10-09T06:22:20.029896+0000","last_clean":"2023-10-09T06:22:20.029896+0000","last_became_active":"2023-10-09T06:21:14.906407+0000","last_became_peered":"2023-10-09T06:21:14.906407+0000","last_unstale":"2023-10-09T06:22:20.029896+0000","last_undegraded":"2023-10-09T06:22:20.029896+0000","last_fullsized":"2023-10-09T06:22:20.029896+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:43.590311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":499,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.037548+0000","last_change":"2023-10-09T06:22:21.037548+0000","last_active":"2023-10-09T06:22:21.037548+0000","last_peered":"2023-10-09T06:22:21.037548+0000","last_clean":"2023-10-09T06:22:21.037548+0000","last_became_active":"2023-10-09T06:22:21.037163+0000","last_became_peered":"2023-10-09T06:22:21.037163+0000","last_unstale":"2023-10-09T06:22:21.037548+0000","last_undegraded":"2023-10-09T06:22:21.037548+0000","last_fullsized":"2023-10-09T06:22:21.037548+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:14.397130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":825,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.034460+0000","last_change":"2023-10-09T06:21:17.904458+0000","last_active":"2023-10-09T06:22:20.034460+0000","last_peered":"2023-10-09T06:22:20.034460+0000","last_clean":"2023-10-09T06:22:20.034460+0000","last_became_active":"2023-10-09T06:21:17.904241+0000","last_became_peered":"2023-10-09T06:21:17.904241+0000","last_unstale":"2023-10-09T06:22:20.034460+0000","last_undegraded":"2023-10-09T06:22:20.034460+0000","last_fullsized":"2023-10-09T06:22:20.034460+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:14:47.339163+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T06:14:47.339163+0000","last_clean_scrub_stamp":"2023-10-09T06:14:47.339163+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:46:29.178159+0000","scrub_duration":0.0016607729999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"670'149","reported_seq":1133,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.036767+0000","last_change":"2023-10-09T06:21:40.936384+0000","last_active":"2023-10-09T06:22:20.036767+0000","last_peered":"2023-10-09T06:22:20.036767+0000","last_clean":"2023-10-09T06:22:20.036767+0000","last_became_active":"2023-10-09T06:21:40.935998+0000","last_became_peered":"2023-10-09T06:21:40.935998+0000","last_unstale":"2023-10-09T06:22:20.036767+0000","last_undegraded":"2023-10-09T06:22:20.036767+0000","last_fullsized":"2023-10-09T06:22:20.036767+0000","mapping_epoch":693,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":694,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:57.304826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041077000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1119,"num_read_kb":1169,"num_write":121,"num_write_kb":197,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":526,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.039375+0000","last_change":"2023-10-09T06:21:19.928503+0000","last_active":"2023-10-09T06:22:20.039375+0000","last_peered":"2023-10-09T06:22:20.039375+0000","last_clean":"2023-10-09T06:22:20.039375+0000","last_became_active":"2023-10-09T06:21:19.928155+0000","last_became_peered":"2023-10-09T06:21:19.928155+0000","last_unstale":"2023-10-09T06:22:20.039375+0000","last_undegraded":"2023-10-09T06:22:20.039375+0000","last_fullsized":"2023-10-09T06:22:20.039375+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:50:02.702751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":35,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.040296+0000","last_change":"2023-10-09T06:21:31.571108+0000","last_active":"2023-10-09T06:22:20.040296+0000","last_peered":"2023-10-09T06:22:20.040296+0000","last_clean":"2023-10-09T06:22:20.040296+0000","last_became_active":"2023-10-09T06:21:31.570738+0000","last_became_peered":"2023-10-09T06:21:31.570738+0000","last_unstale":"2023-10-09T06:22:20.040296+0000","last_undegraded":"2023-10-09T06:22:20.040296+0000","last_fullsized":"2023-10-09T06:22:20.040296+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:39.626021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"670'163","reported_seq":769,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.041653+0000","last_change":"2023-10-09T06:22:21.041653+0000","last_active":"2023-10-09T06:22:21.041653+0000","last_peered":"2023-10-09T06:22:21.041653+0000","last_clean":"2023-10-09T06:22:21.041653+0000","last_became_active":"2023-10-09T06:22:21.041160+0000","last_became_peered":"2023-10-09T06:22:21.041160+0000","last_unstale":"2023-10-09T06:22:21.041653+0000","last_undegraded":"2023-10-09T06:22:21.041653+0000","last_fullsized":"2023-10-09T06:22:21.041653+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":163,"log_dups_size":0,"ondisk_log_size":163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:56.685679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063049999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":730,"num_read_kb":2197,"num_write":152,"num_write_kb":1386,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":793,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.037769+0000","last_change":"2023-10-09T06:18:15.852226+0000","last_active":"2023-10-09T06:22:20.037769+0000","last_peered":"2023-10-09T06:22:20.037769+0000","last_clean":"2023-10-09T06:22:20.037769+0000","last_became_active":"2023-10-09T06:18:15.851868+0000","last_became_peered":"2023-10-09T06:18:15.851868+0000","last_unstale":"2023-10-09T06:22:20.037769+0000","last_undegraded":"2023-10-09T06:22:20.037769+0000","last_fullsized":"2023-10-09T06:22:20.037769+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:35.783434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:35.783434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:57.511684+0000","scrub_duration":0.0043794799999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":564,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.026833+0000","last_change":"2023-10-09T06:21:22.889768+0000","last_active":"2023-10-09T06:22:21.026833+0000","last_peered":"2023-10-09T06:22:21.026833+0000","last_clean":"2023-10-09T06:22:21.026833+0000","last_became_active":"2023-10-09T06:21:22.889396+0000","last_became_peered":"2023-10-09T06:21:22.889396+0000","last_unstale":"2023-10-09T06:22:21.026833+0000","last_undegraded":"2023-10-09T06:22:21.026833+0000","last_fullsized":"2023-10-09T06:22:21.026833+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:54.800020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":635,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.031537+0000","last_change":"2023-10-09T06:21:14.913805+0000","last_active":"2023-10-09T06:22:20.031537+0000","last_peered":"2023-10-09T06:22:20.031537+0000","last_clean":"2023-10-09T06:22:20.031537+0000","last_became_active":"2023-10-09T06:21:14.913537+0000","last_became_peered":"2023-10-09T06:21:14.913537+0000","last_unstale":"2023-10-09T06:22:20.031537+0000","last_undegraded":"2023-10-09T06:22:20.031537+0000","last_fullsized":"2023-10-09T06:22:20.031537+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:28.008641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":1074,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.036859+0000","last_change":"2023-10-09T06:21:17.907964+0000","last_active":"2023-10-09T06:22:21.036859+0000","last_peered":"2023-10-09T06:22:21.036859+0000","last_clean":"2023-10-09T06:22:21.036859+0000","last_became_active":"2023-10-09T06:21:17.907678+0000","last_became_peered":"2023-10-09T06:21:17.907678+0000","last_unstale":"2023-10-09T06:22:21.036859+0000","last_undegraded":"2023-10-09T06:22:21.036859+0000","last_fullsized":"2023-10-09T06:22:21.036859+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:57.627296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:57.627296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:29.010811+0000","scrub_duration":0.001913166,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"667'12","reported_seq":684,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.038489+0000","last_change":"2023-10-09T06:21:31.591278+0000","last_active":"2023-10-09T06:22:20.038489+0000","last_peered":"2023-10-09T06:22:20.038489+0000","last_clean":"2023-10-09T06:22:20.038489+0000","last_became_active":"2023-10-09T06:21:31.567099+0000","last_became_peered":"2023-10-09T06:21:31.567099+0000","last_unstale":"2023-10-09T06:22:20.038489+0000","last_undegraded":"2023-10-09T06:22:20.038489+0000","last_fullsized":"2023-10-09T06:22:20.038489+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:11:00.994566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082067299999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":9,"num_read_kb":1407,"num_write":8,"num_write_kb":1216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":678,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.026290+0000","last_change":"2023-10-09T06:21:14.903545+0000","last_active":"2023-10-09T06:22:20.026290+0000","last_peered":"2023-10-09T06:22:20.026290+0000","last_clean":"2023-10-09T06:22:20.026290+0000","last_became_active":"2023-10-09T06:21:14.903292+0000","last_became_peered":"2023-10-09T06:21:14.903292+0000","last_unstale":"2023-10-09T06:22:20.026290+0000","last_undegraded":"2023-10-09T06:22:20.026290+0000","last_fullsized":"2023-10-09T06:22:20.026290+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:35.161316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":808,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.029532+0000","last_change":"2023-10-09T06:21:37.881638+0000","last_active":"2023-10-09T06:22:21.029532+0000","last_peered":"2023-10-09T06:22:21.029532+0000","last_clean":"2023-10-09T06:22:21.029532+0000","last_became_active":"2023-10-09T06:21:37.881262+0000","last_became_peered":"2023-10-09T06:21:37.881262+0000","last_unstale":"2023-10-09T06:22:21.029532+0000","last_undegraded":"2023-10-09T06:22:21.029532+0000","last_fullsized":"2023-10-09T06:22:21.029532+0000","mapping_epoch":690,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":691,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:07:54.050172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":900,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.031070+0000","last_change":"2023-10-09T06:20:29.965059+0000","last_active":"2023-10-09T06:22:20.031070+0000","last_peered":"2023-10-09T06:22:20.031070+0000","last_clean":"2023-10-09T06:22:20.031070+0000","last_became_active":"2023-10-09T06:20:29.962381+0000","last_became_peered":"2023-10-09T06:20:29.962381+0000","last_unstale":"2023-10-09T06:22:20.031070+0000","last_undegraded":"2023-10-09T06:22:20.031070+0000","last_fullsized":"2023-10-09T06:22:20.031070+0000","mapping_epoch":646,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":647,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:45.844892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:45.844892+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:08.335928+0000","scrub_duration":0.0017521489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"669'67","reported_seq":486,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.034580+0000","last_change":"2023-10-09T06:21:14.944527+0000","last_active":"2023-10-09T06:22:20.034580+0000","last_peered":"2023-10-09T06:22:20.034580+0000","last_clean":"2023-10-09T06:22:20.034580+0000","last_became_active":"2023-10-09T06:21:14.909285+0000","last_became_peered":"2023-10-09T06:21:14.909285+0000","last_unstale":"2023-10-09T06:22:20.034580+0000","last_undegraded":"2023-10-09T06:22:20.034580+0000","last_fullsized":"2023-10-09T06:22:20.034580+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":67,"log_dups_size":0,"ondisk_log_size":67,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:44.205011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000602131,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":304,"num_read_kb":884,"num_write":61,"num_write_kb":1396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":44,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.039908+0000","last_change":"2023-10-09T06:21:37.881769+0000","last_active":"2023-10-09T06:22:20.039908+0000","last_peered":"2023-10-09T06:22:20.039908+0000","last_clean":"2023-10-09T06:22:20.039908+0000","last_became_active":"2023-10-09T06:21:37.881393+0000","last_became_peered":"2023-10-09T06:21:37.881393+0000","last_unstale":"2023-10-09T06:22:20.039908+0000","last_undegraded":"2023-10-09T06:22:20.039908+0000","last_fullsized":"2023-10-09T06:22:20.039908+0000","mapping_epoch":690,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":691,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:03.343363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":268,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.027313+0000","last_change":"2023-10-09T06:22:13.707418+0000","last_active":"2023-10-09T06:22:20.027313+0000","last_peered":"2023-10-09T06:22:20.027313+0000","last_clean":"2023-10-09T06:22:20.027313+0000","last_became_active":"2023-10-09T06:22:13.707023+0000","last_became_peered":"2023-10-09T06:22:13.707023+0000","last_unstale":"2023-10-09T06:22:20.027313+0000","last_undegraded":"2023-10-09T06:22:20.027313+0000","last_fullsized":"2023-10-09T06:22:20.027313+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:00.163026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":1297,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.032202+0000","last_change":"2023-10-09T06:21:40.939658+0000","last_active":"2023-10-09T06:22:20.032202+0000","last_peered":"2023-10-09T06:22:20.032202+0000","last_clean":"2023-10-09T06:22:20.032202+0000","last_became_active":"2023-10-09T06:21:40.939238+0000","last_became_peered":"2023-10-09T06:21:40.939238+0000","last_unstale":"2023-10-09T06:22:20.032202+0000","last_undegraded":"2023-10-09T06:22:20.032202+0000","last_fullsized":"2023-10-09T06:22:20.032202+0000","mapping_epoch":693,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":694,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:17.344518+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:17.344518+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:15.256284+0000","scrub_duration":0.0015914099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":770,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.033359+0000","last_change":"2023-10-09T06:13:30.798394+0000","last_active":"2023-10-09T06:22:20.033359+0000","last_peered":"2023-10-09T06:22:20.033359+0000","last_clean":"2023-10-09T06:22:20.033359+0000","last_became_active":"2023-10-09T06:13:30.790913+0000","last_became_peered":"2023-10-09T06:13:30.790913+0000","last_unstale":"2023-10-09T06:22:20.033359+0000","last_undegraded":"2023-10-09T06:22:20.033359+0000","last_fullsized":"2023-10-09T06:22:20.033359+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:26.508690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"670'110","reported_seq":916,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.040170+0000","last_change":"2023-10-09T06:21:34.892140+0000","last_active":"2023-10-09T06:22:20.040170+0000","last_peered":"2023-10-09T06:22:20.040170+0000","last_clean":"2023-10-09T06:22:20.040170+0000","last_became_active":"2023-10-09T06:21:34.891608+0000","last_became_peered":"2023-10-09T06:21:34.891608+0000","last_unstale":"2023-10-09T06:22:20.040170+0000","last_undegraded":"2023-10-09T06:22:20.040170+0000","last_fullsized":"2023-10-09T06:22:20.040170+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:46.255099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043525900000000001,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":561,"num_read_kb":461,"num_write":80,"num_write_kb":4135,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":4194320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":519,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.032947+0000","last_change":"2023-10-09T06:13:30.782038+0000","last_active":"2023-10-09T06:22:20.032947+0000","last_peered":"2023-10-09T06:22:20.032947+0000","last_clean":"2023-10-09T06:22:20.032947+0000","last_became_active":"2023-10-09T06:13:30.781500+0000","last_became_peered":"2023-10-09T06:13:30.781500+0000","last_unstale":"2023-10-09T06:22:20.032947+0000","last_undegraded":"2023-10-09T06:22:20.032947+0000","last_fullsized":"2023-10-09T06:22:20.032947+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:43.663265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":1095,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.027446+0000","last_change":"2023-10-09T06:21:25.906863+0000","last_active":"2023-10-09T06:22:21.027446+0000","last_peered":"2023-10-09T06:22:21.027446+0000","last_clean":"2023-10-09T06:22:21.027446+0000","last_became_active":"2023-10-09T06:21:25.906453+0000","last_became_peered":"2023-10-09T06:21:25.906453+0000","last_unstale":"2023-10-09T06:22:21.027446+0000","last_undegraded":"2023-10-09T06:22:21.027446+0000","last_fullsized":"2023-10-09T06:22:21.027446+0000","mapping_epoch":683,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":684,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:02.748560+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"669'173","reported_seq":824,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.031884+0000","last_change":"2023-10-09T06:21:22.907807+0000","last_active":"2023-10-09T06:22:20.031884+0000","last_peered":"2023-10-09T06:22:20.031884+0000","last_clean":"2023-10-09T06:22:20.031884+0000","last_became_active":"2023-10-09T06:21:22.907273+0000","last_became_peered":"2023-10-09T06:21:22.907273+0000","last_unstale":"2023-10-09T06:22:20.031884+0000","last_undegraded":"2023-10-09T06:22:20.031884+0000","last_fullsized":"2023-10-09T06:22:20.031884+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":173,"log_dups_size":0,"ondisk_log_size":173,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:47.906004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011230579999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1095,"num_read_kb":3133,"num_write":156,"num_write_kb":2191,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":825,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.029756+0000","last_change":"2023-10-09T06:21:31.584018+0000","last_active":"2023-10-09T06:22:21.029756+0000","last_peered":"2023-10-09T06:22:21.029756+0000","last_clean":"2023-10-09T06:22:21.029756+0000","last_became_active":"2023-10-09T06:21:31.568229+0000","last_became_peered":"2023-10-09T06:21:31.568229+0000","last_unstale":"2023-10-09T06:22:21.029756+0000","last_undegraded":"2023-10-09T06:22:21.029756+0000","last_fullsized":"2023-10-09T06:22:21.029756+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:02.652850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":57,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.029251+0000","last_change":"2023-10-09T06:21:22.889114+0000","last_active":"2023-10-09T06:22:20.029251+0000","last_peered":"2023-10-09T06:22:20.029251+0000","last_clean":"2023-10-09T06:22:20.029251+0000","last_became_active":"2023-10-09T06:21:22.888698+0000","last_became_peered":"2023-10-09T06:21:22.888698+0000","last_unstale":"2023-10-09T06:22:20.029251+0000","last_undegraded":"2023-10-09T06:22:20.029251+0000","last_fullsized":"2023-10-09T06:22:20.029251+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:41:02.391445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":993,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.033501+0000","last_change":"2023-10-09T06:18:15.860341+0000","last_active":"2023-10-09T06:22:20.033501+0000","last_peered":"2023-10-09T06:22:20.033501+0000","last_clean":"2023-10-09T06:22:20.033501+0000","last_became_active":"2023-10-09T06:18:15.860008+0000","last_became_peered":"2023-10-09T06:18:15.860008+0000","last_unstale":"2023-10-09T06:22:20.033501+0000","last_undegraded":"2023-10-09T06:22:20.033501+0000","last_fullsized":"2023-10-09T06:22:20.033501+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.181811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.181811+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:37.378828+0000","scrub_duration":0.011553313000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":669,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.027216+0000","last_change":"2023-10-09T06:21:14.902519+0000","last_active":"2023-10-09T06:22:20.027216+0000","last_peered":"2023-10-09T06:22:20.027216+0000","last_clean":"2023-10-09T06:22:20.027216+0000","last_became_active":"2023-10-09T06:21:14.902152+0000","last_became_peered":"2023-10-09T06:21:14.902152+0000","last_unstale":"2023-10-09T06:22:20.027216+0000","last_undegraded":"2023-10-09T06:22:20.027216+0000","last_fullsized":"2023-10-09T06:22:20.027216+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:56:16.979735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":432,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.038013+0000","last_change":"2023-10-09T06:21:41.018573+0000","last_active":"2023-10-09T06:22:20.038013+0000","last_peered":"2023-10-09T06:22:20.038013+0000","last_clean":"2023-10-09T06:22:20.038013+0000","last_became_active":"2023-10-09T06:21:41.018215+0000","last_became_peered":"2023-10-09T06:21:41.018215+0000","last_unstale":"2023-10-09T06:22:20.038013+0000","last_undegraded":"2023-10-09T06:22:20.038013+0000","last_fullsized":"2023-10-09T06:22:20.038013+0000","mapping_epoch":693,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":694,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:22.031166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":1097,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.034788+0000","last_change":"2023-10-09T06:18:15.867807+0000","last_active":"2023-10-09T06:22:20.034788+0000","last_peered":"2023-10-09T06:22:20.034788+0000","last_clean":"2023-10-09T06:22:20.034788+0000","last_became_active":"2023-10-09T06:18:15.867453+0000","last_became_peered":"2023-10-09T06:18:15.867453+0000","last_unstale":"2023-10-09T06:22:20.034788+0000","last_undegraded":"2023-10-09T06:22:20.034788+0000","last_fullsized":"2023-10-09T06:22:20.034788+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:04.215270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:04.215270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:50:08.865357+0000","scrub_duration":0.002850652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":36,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.045231+0000","last_change":"2023-10-09T06:22:21.045231+0000","last_active":"2023-10-09T06:22:21.045231+0000","last_peered":"2023-10-09T06:22:21.045231+0000","last_clean":"2023-10-09T06:22:21.045231+0000","last_became_active":"2023-10-09T06:22:21.044873+0000","last_became_peered":"2023-10-09T06:22:21.044873+0000","last_unstale":"2023-10-09T06:22:21.045231+0000","last_undegraded":"2023-10-09T06:22:21.045231+0000","last_fullsized":"2023-10-09T06:22:21.045231+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:55.616563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":57,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.031561+0000","last_change":"2023-10-09T06:21:31.576153+0000","last_active":"2023-10-09T06:22:20.031561+0000","last_peered":"2023-10-09T06:22:20.031561+0000","last_clean":"2023-10-09T06:22:20.031561+0000","last_became_active":"2023-10-09T06:21:31.575732+0000","last_became_peered":"2023-10-09T06:21:31.575732+0000","last_unstale":"2023-10-09T06:22:20.031561+0000","last_undegraded":"2023-10-09T06:22:20.031561+0000","last_fullsized":"2023-10-09T06:22:20.031561+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:01.829052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":36,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.043369+0000","last_change":"2023-10-09T06:22:21.043369+0000","last_active":"2023-10-09T06:22:21.043369+0000","last_peered":"2023-10-09T06:22:21.043369+0000","last_clean":"2023-10-09T06:22:21.043369+0000","last_became_active":"2023-10-09T06:22:21.042847+0000","last_became_peered":"2023-10-09T06:22:21.042847+0000","last_unstale":"2023-10-09T06:22:21.043369+0000","last_undegraded":"2023-10-09T06:22:21.043369+0000","last_fullsized":"2023-10-09T06:22:21.043369+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:24.163212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":47,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.029335+0000","last_change":"2023-10-09T06:21:14.900160+0000","last_active":"2023-10-09T06:22:20.029335+0000","last_peered":"2023-10-09T06:22:20.029335+0000","last_clean":"2023-10-09T06:22:20.029335+0000","last_became_active":"2023-10-09T06:21:14.899186+0000","last_became_peered":"2023-10-09T06:21:14.899186+0000","last_unstale":"2023-10-09T06:22:20.029335+0000","last_undegraded":"2023-10-09T06:22:20.029335+0000","last_fullsized":"2023-10-09T06:22:20.029335+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:55:28.845262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":922,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.035476+0000","last_change":"2023-10-09T06:18:15.861806+0000","last_active":"2023-10-09T06:22:20.035476+0000","last_peered":"2023-10-09T06:22:20.035476+0000","last_clean":"2023-10-09T06:22:20.035476+0000","last_became_active":"2023-10-09T06:18:15.861466+0000","last_became_peered":"2023-10-09T06:18:15.861466+0000","last_unstale":"2023-10-09T06:22:20.035476+0000","last_undegraded":"2023-10-09T06:22:20.035476+0000","last_fullsized":"2023-10-09T06:22:20.035476+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:00.456132+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:00.456132+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:05.355171+0000","scrub_duration":0.0014992199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":556,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.034989+0000","last_change":"2023-10-09T06:21:37.884340+0000","last_active":"2023-10-09T06:22:20.034989+0000","last_peered":"2023-10-09T06:22:20.034989+0000","last_clean":"2023-10-09T06:22:20.034989+0000","last_became_active":"2023-10-09T06:21:37.883869+0000","last_became_peered":"2023-10-09T06:21:37.883869+0000","last_unstale":"2023-10-09T06:22:20.034989+0000","last_undegraded":"2023-10-09T06:22:20.034989+0000","last_fullsized":"2023-10-09T06:22:20.034989+0000","mapping_epoch":690,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":691,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:52.554008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":557,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.033770+0000","last_change":"2023-10-09T06:21:19.917691+0000","last_active":"2023-10-09T06:22:20.033770+0000","last_peered":"2023-10-09T06:22:20.033770+0000","last_clean":"2023-10-09T06:22:20.033770+0000","last_became_active":"2023-10-09T06:21:19.917360+0000","last_became_peered":"2023-10-09T06:21:19.917360+0000","last_unstale":"2023-10-09T06:22:20.033770+0000","last_undegraded":"2023-10-09T06:22:20.033770+0000","last_fullsized":"2023-10-09T06:22:20.033770+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:56:35.475967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":934,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.039380+0000","last_change":"2023-10-09T06:21:34.894300+0000","last_active":"2023-10-09T06:22:20.039380+0000","last_peered":"2023-10-09T06:22:20.039380+0000","last_clean":"2023-10-09T06:22:20.039380+0000","last_became_active":"2023-10-09T06:21:34.893948+0000","last_became_peered":"2023-10-09T06:21:34.893948+0000","last_unstale":"2023-10-09T06:22:20.039380+0000","last_undegraded":"2023-10-09T06:22:20.039380+0000","last_fullsized":"2023-10-09T06:22:20.039380+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:46.290510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:46.290510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:27.496106+0000","scrub_duration":0.002627819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":828,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.027598+0000","last_change":"2023-10-09T06:21:37.887283+0000","last_active":"2023-10-09T06:22:20.027598+0000","last_peered":"2023-10-09T06:22:20.027598+0000","last_clean":"2023-10-09T06:22:20.027598+0000","last_became_active":"2023-10-09T06:21:37.881566+0000","last_became_peered":"2023-10-09T06:21:37.881566+0000","last_unstale":"2023-10-09T06:22:20.027598+0000","last_undegraded":"2023-10-09T06:22:20.027598+0000","last_fullsized":"2023-10-09T06:22:20.027598+0000","mapping_epoch":690,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":691,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:36:14.630943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":57,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.029109+0000","last_change":"2023-10-09T06:21:31.572882+0000","last_active":"2023-10-09T06:22:20.029109+0000","last_peered":"2023-10-09T06:22:20.029109+0000","last_clean":"2023-10-09T06:22:20.029109+0000","last_became_active":"2023-10-09T06:21:31.572511+0000","last_became_peered":"2023-10-09T06:21:31.572511+0000","last_unstale":"2023-10-09T06:22:20.029109+0000","last_undegraded":"2023-10-09T06:22:20.029109+0000","last_fullsized":"2023-10-09T06:22:20.029109+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:24:13.408389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":122,"num_bytes_recovered":16344,"num_keys_recovered":90,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13410,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2886848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5962,"num_read_kb":13177,"num_write":847,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":4204579,"num_keys_recovered":4,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234519,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1188420,"kb_used_data":7420,"kb_used_omap":0,"kb_used_meta":1180992,"kb_avail":92556732,"statfs":{"total":95995035648,"available":94778093568,"internally_reserved":0,"allocated":7598080,"data_stored":5840641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1209335808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.98499999999999999,"15min":3.7280000000000002},"min":{"1min":0.34999999999999998,"5min":0.221,"15min":0.19500000000000001},"max":{"1min":0.81499999999999995,"5min":5.4480000000000004,"15min":166.94},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.55900000000000005,"5min":1.145,"15min":3.907},"min":{"1min":0.38700000000000001,"5min":0.26900000000000002,"15min":0.23799999999999999},"max":{"1min":0.82299999999999995,"5min":6.2549999999999999,"15min":166.923},"last":0.54300000000000004}]},{"osd":1,"last update":"Mon Oct 9 06:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":6.3869999999999996,"15min":5.0339999999999998},"min":{"1min":0.38100000000000001,"5min":0.25600000000000001,"15min":0.218},"max":{"1min":0.749,"5min":129.17500000000001,"15min":129.17500000000001},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":6.3849999999999998,"15min":4.9779999999999998},"min":{"1min":0.35899999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.79500000000000004,"5min":129.18700000000001,"15min":129.18700000000001},"last":0.47199999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.95799999999999996,"15min":1.6220000000000001},"min":{"1min":0.43099999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.755,"5min":5.0810000000000004,"15min":47.503},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.58099999999999996,"5min":1.0720000000000001,"15min":1.675},"min":{"1min":0.38700000000000001,"5min":0.312,"15min":0.27000000000000002},"max":{"1min":0.70499999999999996,"5min":9.5600000000000005,"15min":47.545000000000002},"last":0.52800000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":4.399,"15min":5.032},"min":{"1min":0.40799999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.79400000000000004,"5min":184.804,"15min":184.804},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.65000000000000002,"5min":4.2270000000000003,"15min":4.976},"min":{"1min":0.46999999999999997,"5min":0.32600000000000001,"15min":0.32100000000000001},"max":{"1min":0.91600000000000004,"5min":186.86099999999999,"15min":186.86099999999999},"last":0.68700000000000006}]},{"osd":4,"last update":"Mon Oct 9 06:22:00 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":1.8919999999999999,"15min":1.385},"min":{"1min":0.40699999999999997,"5min":0.27600000000000002,"15min":0.23100000000000001},"max":{"1min":0.75800000000000001,"5min":67.186999999999998,"15min":67.186999999999998},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.58899999999999997,"5min":1.9390000000000001,"15min":1.381},"min":{"1min":0.34699999999999998,"5min":0.254,"15min":0.246},"max":{"1min":0.77400000000000002,"5min":67.218000000000004,"15min":67.218000000000004},"last":0.629}]},{"osd":5,"last update":"Mon Oct 9 06:22:39 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":1.1479999999999999,"15min":1.6439999999999999},"min":{"1min":0.48899999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.83799999999999997,"5min":12.951000000000001,"15min":12.951000000000001},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.629,"5min":0.995,"15min":1.3169999999999999},"min":{"1min":0.39200000000000002,"5min":0.307,"15min":0.307},"max":{"1min":0.80000000000000004,"5min":5.0369999999999999,"15min":5.0369999999999999},"last":0.74399999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:22:11 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":1.2130000000000001,"15min":1.796},"min":{"1min":0.46800000000000003,"5min":0.311,"15min":0.26600000000000001},"max":{"1min":0.82999999999999996,"5min":9.2599999999999998,"15min":233.56700000000001},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.70199999999999996,"5min":0.97599999999999998,"15min":1.7230000000000001},"min":{"1min":0.48499999999999999,"5min":0.33000000000000002,"15min":0.30599999999999999},"max":{"1min":0.875,"5min":5.0629999999999997,"15min":232.477},"last":0.71399999999999997}]}]},{"osd":6,"up_from":46,"seq":197568496156,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1209780,"kb_used_data":11564,"kb_used_omap":0,"kb_used_meta":1198208,"kb_avail":92535372,"statfs":{"total":95995035648,"available":94756220928,"internally_reserved":0,"allocated":11841536,"data_stored":10099545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1226964992},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:39 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.669,"15min":4.3730000000000002},"min":{"1min":0.33900000000000002,"5min":0.222,"15min":0.21099999999999999},"max":{"1min":0.73499999999999999,"5min":85.311000000000007,"15min":191.50800000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.56499999999999995,"5min":1.976,"15min":4.5960000000000001},"min":{"1min":0.40799999999999997,"5min":0.27100000000000002,"15min":0.24399999999999999},"max":{"1min":0.74399999999999999,"5min":85.322999999999993,"15min":191.52600000000001},"last":0.57999999999999996}]},{"osd":1,"last update":"Mon Oct 9 06:22:39 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":4.7640000000000002,"15min":5.6420000000000003},"min":{"1min":0.27700000000000002,"5min":0.25,"15min":0.23200000000000001},"max":{"1min":0.66800000000000004,"5min":183.458,"15min":328.04399999999998},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.58599999999999997,"5min":4.6020000000000003,"15min":5.3570000000000002},"min":{"1min":0.38800000000000001,"5min":0.28499999999999998,"15min":0.253},"max":{"1min":0.82599999999999996,"5min":183.51499999999999,"15min":328.10000000000002},"last":0.51700000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:22:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":1.0960000000000001,"15min":2.4470000000000001},"min":{"1min":0.38300000000000001,"5min":0.29099999999999998,"15min":0.26100000000000001},"max":{"1min":0.69699999999999995,"5min":9.1500000000000004,"15min":163.04900000000001},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.98099999999999998,"15min":2.4340000000000002},"min":{"1min":0.374,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.89400000000000002,"5min":12.083,"15min":162.93000000000001},"last":0.46800000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:22:26 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":5.7350000000000003,"15min":6.048},"min":{"1min":0.37,"5min":0.314,"15min":0.29199999999999998},"max":{"1min":0.72699999999999998,"5min":208.494,"15min":250.744},"last":0.501},{"interface":"front","average":{"1min":0.64300000000000002,"5min":5.9189999999999996,"15min":5.923},"min":{"1min":0.51000000000000001,"5min":0.33200000000000002,"15min":0.309},"max":{"1min":0.92200000000000004,"5min":211.661,"15min":250.83799999999999},"last":0.373}]},{"osd":4,"last update":"Mon Oct 9 06:22:04 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.68899999999999995,"15min":1.341},"min":{"1min":0.38,"5min":0.26800000000000002,"15min":0.24399999999999999},"max":{"1min":0.91400000000000003,"5min":6.0979999999999999,"15min":151.35599999999999},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.503,"5min":1.173,"15min":1.6599999999999999},"min":{"1min":0.36899999999999999,"5min":0.27700000000000002,"15min":0.23999999999999999},"max":{"1min":0.75700000000000001,"5min":14.009,"15min":149.15799999999999},"last":0.41799999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:22:39 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":1.075,"15min":1.554},"min":{"1min":0.40300000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.76100000000000001,"5min":16.032,"15min":16.032},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.81999999999999995,"15min":1.0089999999999999},"min":{"1min":0.46000000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.95499999999999996,"5min":5.8090000000000002,"15min":5.8090000000000002},"last":0.60299999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:22:04 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":1.248,"15min":1.157},"min":{"1min":0.34499999999999997,"5min":0.32700000000000001,"15min":0.30299999999999999},"max":{"1min":0.70499999999999996,"5min":17.803999999999998,"15min":17.803999999999998},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.091,"15min":1.044},"min":{"1min":0.42599999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.85199999999999998,"5min":16.119,"15min":16.119},"last":0.59299999999999997}]}]},{"osd":1,"up_from":89,"seq":382252089744,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":860240,"kb_used_data":13576,"kb_used_omap":9,"kb_used_meta":846646,"kb_avail":92884912,"statfs":{"total":95995035648,"available":95114149888,"internally_reserved":0,"allocated":13901824,"data_stored":10032213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":866966505},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:00 2023","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":1.121,"15min":2.419},"min":{"1min":0.21099999999999999,"5min":0.187,"15min":0.187},"max":{"1min":0.59799999999999998,"5min":9.6899999999999995,"15min":118.17},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.33000000000000002,"5min":1.2,"15min":2.641},"min":{"1min":0.25800000000000001,"5min":0.23200000000000001,"15min":0.20200000000000001},"max":{"1min":0.63200000000000001,"5min":8.5410000000000004,"15min":117.634},"last":0.29099999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:22:00 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":1.4299999999999999,"15min":1.6279999999999999},"min":{"1min":0.252,"5min":0.247,"15min":0.21099999999999999},"max":{"1min":0.64700000000000002,"5min":19.943999999999999,"15min":19.943999999999999},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.42399999999999999,"5min":1.157,"15min":1.8420000000000001},"min":{"1min":0.29199999999999998,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":1.0229999999999999,"5min":8.5939999999999994,"15min":55.694000000000003},"last":0.33000000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:22:29 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":3.2210000000000001,"15min":4.0990000000000002},"min":{"1min":0.32100000000000001,"5min":0.252,"15min":0.24199999999999999},"max":{"1min":0.64800000000000002,"5min":73.957999999999998,"15min":113.322},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.438,"5min":2.9209999999999998,"15min":3.9620000000000002},"min":{"1min":0.33600000000000002,"5min":0.28199999999999997,"15min":0.21299999999999999},"max":{"1min":0.63500000000000001,"5min":74.709000000000003,"15min":113.38200000000001},"last":0.41499999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:22:00 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":4.4930000000000003,"15min":3.4990000000000001},"min":{"1min":0.42499999999999999,"5min":0.29799999999999999,"15min":0.27700000000000002},"max":{"1min":0.78000000000000003,"5min":190.523,"15min":190.523},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.624,"5min":4.7039999999999997,"15min":4.0069999999999997},"min":{"1min":0.41399999999999998,"5min":0.36299999999999999,"15min":0.308},"max":{"1min":0.96499999999999997,"5min":190.774,"15min":190.774},"last":0.58199999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:22:38 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":1.2969999999999999,"15min":1.419},"min":{"1min":0.46999999999999997,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.214,"5min":11.398999999999999,"15min":11.398999999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.628,"5min":1.0669999999999999,"15min":1.2370000000000001},"min":{"1min":0.40100000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.87,"5min":8.4320000000000004,"15min":8.4320000000000004},"last":0.57099999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:22:12 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":2.3460000000000001,"15min":2.0139999999999998},"min":{"1min":0.39100000000000001,"5min":0.38200000000000001,"15min":0.33700000000000002},"max":{"1min":0.93700000000000006,"5min":98.433000000000007,"15min":98.433000000000007},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.63900000000000001,"5min":2.3730000000000002,"15min":2.4460000000000002},"min":{"1min":0.40500000000000003,"5min":0.377,"15min":0.34999999999999998},"max":{"1min":0.91200000000000003,"5min":98.456000000000003,"15min":98.456000000000003},"last":0.55300000000000005}]},{"osd":7,"last update":"Mon Oct 9 06:22:12 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":4.298,"15min":3.2530000000000001},"min":{"1min":0.437,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.77300000000000002,"5min":154.34800000000001,"15min":154.34800000000001},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.64400000000000002,"5min":4.0670000000000002,"15min":2.7559999999999998},"min":{"1min":0.45100000000000001,"5min":0.40100000000000002,"15min":0.38},"max":{"1min":0.85999999999999999,"5min":152.20599999999999,"15min":152.20599999999999},"last":0.63600000000000001}]}]},{"osd":0,"up_from":456,"seq":1958505087139,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":454976,"kb_used_data":9528,"kb_used_omap":32,"kb_used_meta":445407,"kb_avail":93290176,"statfs":{"total":95995035648,"available":95529140224,"internally_reserved":0,"allocated":9756672,"data_stored":6141356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32794,"internal_metadata":456097766},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:21:51 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":4.1900000000000004,"15min":6.085},"min":{"1min":0.21199999999999999,"5min":0.189,"15min":0.17299999999999999},"max":{"1min":0.46600000000000003,"5min":137.452,"15min":166.833},"last":0.245},{"interface":"front","average":{"1min":0.34499999999999997,"5min":4.0629999999999997,"15min":5.9550000000000001},"min":{"1min":0.28199999999999997,"5min":0.23999999999999999,"15min":0.19700000000000001},"max":{"1min":0.42199999999999999,"5min":137.10499999999999,"15min":166.81100000000001},"last":0.32700000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:21:51 2023","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":1.2829999999999999,"15min":2.3239999999999998},"min":{"1min":0.248,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.50600000000000001,"5min":13.071,"15min":21.648},"last":0.443},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.91900000000000004,"15min":2.109},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.48299999999999998,"5min":6.173,"15min":20.152000000000001},"last":0.40400000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:21:51 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":4.609,"15min":6.649},"min":{"1min":0.28299999999999997,"5min":0.214,"15min":0.214},"max":{"1min":0.78100000000000003,"5min":138.52699999999999,"15min":203.59299999999999},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.44400000000000001,"5min":4.8040000000000003,"15min":6.6269999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.23899999999999999},"max":{"1min":0.79600000000000004,"5min":138.48500000000001,"15min":202.96600000000001},"last":0.374}]},{"osd":4,"last update":"Mon Oct 9 06:21:51 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":1.6870000000000001,"15min":2.0049999999999999},"min":{"1min":0.34799999999999998,"5min":0.313,"15min":0.27800000000000002},"max":{"1min":0.80200000000000005,"5min":60.587000000000003,"15min":60.587000000000003},"last":0.77200000000000002},{"interface":"front","average":{"1min":0.625,"5min":1.7410000000000001,"15min":2.097},"min":{"1min":0.35299999999999998,"5min":0.32400000000000001,"15min":0.254},"max":{"1min":0.82599999999999996,"5min":60.567,"15min":60.567},"last":0.80000000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:22:43 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":1.274,"15min":1.8260000000000001},"min":{"1min":0.38700000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":3.6070000000000002,"5min":8.048,"15min":8.048},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.268,"15min":1.6599999999999999},"min":{"1min":0.45800000000000002,"5min":0.376,"15min":0.376},"max":{"1min":0.77900000000000003,"5min":13.007,"15min":13.007},"last":0.72599999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:21:51 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":1.4330000000000001,"15min":2.0419999999999998},"min":{"1min":0.36599999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.88500000000000001,"5min":13.029999999999999,"15min":31.648},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.626,"5min":1.0960000000000001,"15min":1.823},"min":{"1min":0.432,"5min":0.36499999999999999,"15min":0.36299999999999999},"max":{"1min":0.81999999999999995,"5min":7.5640000000000001,"15min":24.43},"last":0.66800000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:21:51 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":5.3300000000000001,"15min":3.694},"min":{"1min":0.435,"5min":0.376,"15min":0.376},"max":{"1min":0.85099999999999998,"5min":248.99100000000001,"15min":248.99100000000001},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.66600000000000004,"5min":5.1849999999999996,"15min":3.5350000000000001},"min":{"1min":0.52500000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.78000000000000003,"5min":249.00800000000001,"15min":249.00800000000001},"last":0.64400000000000002}]}]},{"osd":2,"up_from":94,"seq":403726926236,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":792360,"kb_used_data":9248,"kb_used_omap":81,"kb_used_meta":783022,"kb_avail":92952792,"statfs":{"total":95995035648,"available":95183659008,"internally_reserved":0,"allocated":9469952,"data_stored":6050318,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":801814746},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:06 2023","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":1.8600000000000001,"15min":3.0840000000000001},"min":{"1min":0.23799999999999999,"5min":0.22900000000000001,"15min":0.17299999999999999},"max":{"1min":0.68600000000000005,"5min":70.218000000000004,"15min":83.652000000000001},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.40000000000000002,"5min":1.718,"15min":2.488},"min":{"1min":0.27000000000000002,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.66000000000000003,"5min":69.968000000000004,"15min":77.578999999999994},"last":0.34200000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:22:06 2023","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":2.0579999999999998,"15min":3.5529999999999999},"min":{"1min":0.29099999999999998,"5min":0.251,"15min":0.19400000000000001},"max":{"1min":0.70599999999999996,"5min":59.331000000000003,"15min":160.45500000000001},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.379,"5min":2.714,"15min":4.3040000000000003},"min":{"1min":0.26000000000000001,"5min":0.22600000000000001,"15min":0.20200000000000001},"max":{"1min":0.71999999999999997,"5min":56.508000000000003,"15min":163.697},"last":0.35999999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:22:19 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":1.7929999999999999,"15min":4.3419999999999996},"min":{"1min":0.29999999999999999,"5min":0.26500000000000001,"15min":0.23000000000000001},"max":{"1min":0.73399999999999999,"5min":43.652999999999999,"15min":122.205},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.44500000000000001,"5min":2.1600000000000001,"15min":4.3890000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.252},"max":{"1min":0.747,"5min":46.792999999999999,"15min":122.45},"last":0.38500000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:22:06 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":2.3180000000000001,"15min":2.4470000000000001},"min":{"1min":0.376,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":0.94099999999999995,"5min":98.644999999999996,"15min":107.51300000000001},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.66200000000000003,"5min":1.9610000000000001,"15min":1.968},"min":{"1min":0.41999999999999998,"5min":0.315,"15min":0.315},"max":{"1min":1.01,"5min":98.668000000000006,"15min":108.768},"last":0.52500000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:22:45 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":1.1180000000000001,"15min":1.534},"min":{"1min":0.47399999999999998,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.93000000000000005,"5min":6.4249999999999998,"15min":6.4249999999999998},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.64000000000000001,"5min":1.407,"15min":2.141},"min":{"1min":0.42399999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.86399999999999999,"5min":18.617000000000001,"15min":18.617000000000001},"last":0.66800000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:22:06 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":1.389,"15min":1.7529999999999999},"min":{"1min":0.42599999999999999,"5min":0.39500000000000002,"15min":0.32000000000000001},"max":{"1min":0.91100000000000003,"5min":7.9720000000000004,"15min":14.962},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.68899999999999995,"5min":1.252,"15min":1.504},"min":{"1min":0.497,"5min":0.38300000000000001,"15min":0.34399999999999997},"max":{"1min":0.995,"5min":6.4400000000000004,"15min":11.103},"last":0.63}]},{"osd":7,"last update":"Mon Oct 9 06:22:06 2023","interfaces":[{"interface":"back","average":{"1min":0.68200000000000005,"5min":4.1459999999999999,"15min":2.9489999999999998},"min":{"1min":0.496,"5min":0.40300000000000002,"15min":0.313},"max":{"1min":0.92200000000000004,"5min":234.26499999999999,"15min":234.26499999999999},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.66800000000000004,"5min":4.0250000000000004,"15min":2.7440000000000002},"min":{"1min":0.52200000000000002,"5min":0.375,"15min":0.33900000000000002},"max":{"1min":1.034,"5min":233.02099999999999,"15min":233.02099999999999},"last":0.64400000000000002}]}]},{"osd":3,"up_from":99,"seq":425201762709,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1126204,"kb_used_data":9780,"kb_used_omap":7,"kb_used_meta":1116408,"kb_avail":92618948,"statfs":{"total":95995035648,"available":94841802752,"internally_reserved":0,"allocated":10014720,"data_stored":6158196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1143202408},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:50 2023","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":2.1040000000000001,"15min":3.4329999999999998},"min":{"1min":0.245,"5min":0.182,"15min":0.17299999999999999},"max":{"1min":0.89700000000000002,"5min":117.473,"15min":220.45400000000001},"last":0.245},{"interface":"front","average":{"1min":0.41299999999999998,"5min":1.766,"15min":4.0019999999999998},"min":{"1min":0.28199999999999997,"5min":0.219,"15min":0.19600000000000001},"max":{"1min":1.133,"5min":118.873,"15min":216.44200000000001},"last":0.33000000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":3.294,"15min":2.6520000000000001},"min":{"1min":0.29399999999999998,"5min":0.22800000000000001,"15min":0.19800000000000001},"max":{"1min":0.51300000000000001,"5min":167.768,"15min":167.768},"last":0.309},{"interface":"front","average":{"1min":0.39100000000000001,"5min":3.1400000000000001,"15min":3.3119999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.218},"max":{"1min":0.71399999999999997,"5min":172.143,"15min":172.143},"last":0.26400000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":1.125,"15min":1.6990000000000001},"min":{"1min":0.29299999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.59699999999999998,"5min":8.5280000000000005,"15min":30.547999999999998},"last":0.38},{"interface":"front","average":{"1min":0.38800000000000001,"5min":1.415,"15min":2.2690000000000001},"min":{"1min":0.34100000000000003,"5min":0.26500000000000001,"15min":0.23899999999999999},"max":{"1min":0.44800000000000001,"5min":10.981,"15min":25.984000000000002},"last":0.371}]},{"osd":4,"last update":"Mon Oct 9 06:22:08 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":2.794,"15min":2.3889999999999998},"min":{"1min":0.45100000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.82499999999999996,"5min":84.369,"15min":95.739000000000004},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.621,"5min":3.0750000000000002,"15min":2.7320000000000002},"min":{"1min":0.46800000000000003,"5min":0.34499999999999997,"15min":0.28899999999999998},"max":{"1min":0.83499999999999996,"5min":87.176000000000002,"15min":97.941999999999993},"last":0.53800000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:22:41 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":1.2789999999999999,"15min":1.633},"min":{"1min":0.42899999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.0800000000000001,"5min":7.8550000000000004,"15min":7.8550000000000004},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.70099999999999996,"5min":1.252,"15min":1.6910000000000001},"min":{"1min":0.52400000000000002,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.2150000000000001,"5min":7.1719999999999997,"15min":7.1719999999999997},"last":0.59499999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":1.542,"15min":1.6379999999999999},"min":{"1min":0.50900000000000001,"5min":0.32600000000000001,"15min":0.26100000000000001},"max":{"1min":0.85599999999999998,"5min":11.183,"15min":49.526000000000003},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.63900000000000001,"5min":1.4419999999999999,"15min":1.9590000000000001},"min":{"1min":0.46500000000000002,"5min":0.39100000000000001,"15min":0.33600000000000002},"max":{"1min":0.83299999999999996,"5min":9.5809999999999995,"15min":49.429000000000002},"last":0.44600000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":2.3319999999999999,"15min":2.9169999999999998},"min":{"1min":0.46600000000000003,"5min":0.36599999999999999,"15min":0.35699999999999998},"max":{"1min":0.84699999999999998,"5min":77.525000000000006,"15min":94.400000000000006},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.626,"5min":2.3559999999999999,"15min":3.3079999999999998},"min":{"1min":0.47899999999999998,"5min":0.40000000000000002,"15min":0.32600000000000001},"max":{"1min":0.81100000000000005,"5min":77.438000000000002,"15min":94.195999999999998},"last":0.502}]}]},{"osd":4,"up_from":322,"seq":1382979469572,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":930112,"kb_used_data":11256,"kb_used_omap":35,"kb_used_meta":918812,"kb_avail":92815040,"statfs":{"total":95995035648,"available":95042600960,"internally_reserved":0,"allocated":11526144,"data_stored":9772510,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36552,"internal_metadata":940863800},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":1.0800000000000001,"15min":1.786},"min":{"1min":0.34499999999999997,"5min":0.23100000000000001,"15min":0.216},"max":{"1min":0.78100000000000003,"5min":18.510999999999999,"15min":81.066000000000003},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.53700000000000003,"5min":1.103,"15min":1.7929999999999999},"min":{"1min":0.379,"5min":0.27800000000000002,"15min":0.22800000000000001},"max":{"1min":0.67100000000000004,"5min":10.695,"15min":81.603999999999999},"last":0.41899999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:22:07 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":5.0700000000000003,"15min":3.528},"min":{"1min":0.36099999999999999,"5min":0.23899999999999999,"15min":0.20999999999999999},"max":{"1min":0.75800000000000001,"5min":160.35400000000001,"15min":160.35400000000001},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.53500000000000003,"5min":5.117,"15min":3.5920000000000001},"min":{"1min":0.377,"5min":0.26500000000000001,"15min":0.22800000000000001},"max":{"1min":0.77100000000000002,"5min":160.333,"15min":160.333},"last":0.5}]},{"osd":2,"last update":"Mon Oct 9 06:22:07 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":1.054,"15min":2.3900000000000001},"min":{"1min":0.41999999999999998,"5min":0.29999999999999999,"15min":0.27200000000000002},"max":{"1min":0.748,"5min":9.9540000000000006,"15min":245.166},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":1.986,"15min":2.569},"min":{"1min":0.39200000000000002,"5min":0.33300000000000002,"15min":0.27900000000000003},"max":{"1min":0.71299999999999997,"5min":52.920999999999999,"15min":245.119},"last":0.55400000000000005}]},{"osd":3,"last update":"Mon Oct 9 06:22:07 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":5.3499999999999996,"15min":5.7389999999999999},"min":{"1min":0.40799999999999997,"5min":0.29299999999999998,"15min":0.26800000000000002},"max":{"1min":0.77200000000000002,"5min":164.89699999999999,"15min":246.39400000000001},"last":0.44},{"interface":"front","average":{"1min":0.64100000000000001,"5min":4.9950000000000001,"15min":5.4279999999999999},"min":{"1min":0.44600000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.95099999999999996,"5min":164.88300000000001,"15min":246.99000000000001},"last":0.54100000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:22:39 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":1.22,"15min":1.766},"min":{"1min":0.34000000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.72199999999999998,"5min":10.659000000000001,"15min":10.659000000000001},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.58599999999999997,"5min":1.1839999999999999,"15min":1.7010000000000001},"min":{"1min":0.36399999999999999,"5min":0.317,"15min":0.317},"max":{"1min":0.84499999999999997,"5min":11.779,"15min":11.779},"last":0.48499999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:22:07 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":1.2649999999999999,"15min":1.444},"min":{"1min":0.40300000000000002,"5min":0.26800000000000002,"15min":0.26400000000000001},"max":{"1min":0.82499999999999996,"5min":23.712,"15min":95.492999999999995},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.625,"5min":1.2809999999999999,"15min":1.54},"min":{"1min":0.45600000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.875,"5min":18.021000000000001,"15min":95.664000000000001},"last":0.52600000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:22:07 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":1.7889999999999999,"15min":1.8240000000000001},"min":{"1min":0.41399999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.84599999999999997,"5min":61.811999999999998,"15min":95.167000000000002},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.64900000000000002,"5min":1.702,"15min":1.744},"min":{"1min":0.41899999999999998,"5min":0.28799999999999998,"15min":0.251},"max":{"1min":0.90700000000000003,"5min":61.759,"15min":94.811999999999998},"last":0.59399999999999997}]}]},{"osd":5,"up_from":611,"seq":2624225017910,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":368632,"kb_used_data":7408,"kb_used_omap":98,"kb_used_meta":361117,"kb_avail":93376520,"statfs":{"total":95995035648,"available":95617556480,"internally_reserved":0,"allocated":7585792,"data_stored":5836657,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":100797,"internal_metadata":369784387},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:35 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.86499999999999999,"15min":1.163},"min":{"1min":0.33100000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.68400000000000005,"5min":8.3260000000000005,"15min":8.3260000000000005},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.56100000000000005,"5min":1.0600000000000001,"15min":1.474},"min":{"1min":0.34200000000000003,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.747,"5min":7.577,"15min":7.577},"last":0.56000000000000005}]},{"osd":1,"last update":"Mon Oct 9 06:22:35 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":2.3999999999999999,"15min":4.1189999999999998},"min":{"1min":0.39600000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.002,"5min":50.430999999999997,"15min":50.430999999999997},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":1.873,"15min":3.0430000000000001},"min":{"1min":0.38100000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.97499999999999998,"5min":50.445,"15min":50.445},"last":0.58399999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:22:35 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.95899999999999996,"15min":1.2330000000000001},"min":{"1min":0.377,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.73399999999999999,"5min":5.9980000000000002,"15min":5.9980000000000002},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":1.0049999999999999,"15min":1.3089999999999999},"min":{"1min":0.33600000000000002,"5min":0.307,"15min":0.307},"max":{"1min":0.90700000000000003,"5min":5.9560000000000004,"15min":5.9560000000000004},"last":0.46999999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:22:35 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":4.6989999999999998,"15min":8.2159999999999993},"min":{"1min":0.47899999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.94299999999999995,"5min":120.357,"15min":120.357},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.63400000000000001,"5min":4.2670000000000003,"15min":7.7300000000000004},"min":{"1min":0.49199999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.083,"5min":120.759,"15min":120.759},"last":0.50700000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:22:35 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.72699999999999998,"15min":0.91700000000000004},"min":{"1min":0.29299999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.0209999999999999,"5min":5.6890000000000001,"15min":5.6890000000000001},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.60399999999999998,"5min":1.087,"15min":1.5189999999999999},"min":{"1min":0.436,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.0409999999999999,"5min":7.6130000000000004,"15min":7.6130000000000004},"last":0.73299999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:22:35 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.61699999999999999,"15min":0.70899999999999996},"min":{"1min":0.36499999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.1359999999999999,"5min":5.9400000000000004,"15min":5.9400000000000004},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.622,"5min":0.76700000000000002,"15min":0.92000000000000004},"min":{"1min":0.443,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.1060000000000001,"5min":5.8120000000000003,"15min":5.8120000000000003},"last":0.63300000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:22:35 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":1.0349999999999999,"15min":1.3779999999999999},"min":{"1min":0.38800000000000001,"5min":0.317,"15min":0.317},"max":{"1min":1.0720000000000001,"5min":7.6120000000000001,"15min":7.6120000000000001},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.91400000000000003,"15min":1.153},"min":{"1min":0.505,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.1859999999999999,"5min":5.8920000000000003,"15min":5.8920000000000003},"last":0.66800000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4889,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:22:54.686 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:22:55.047 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:55.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:54 smithi064 ceph-mon[212750]: pgmap v1800: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:55.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:54 smithi064 ceph-mon[212750]: from='client.37926 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:22:55.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:54 smithi064 ceph-mon[220044]: pgmap v1800: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:55.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:54 smithi064 ceph-mon[220044]: from='client.37926 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:22:55.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:54 smithi156 ceph-mon[195101]: pgmap v1800: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:55.371 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:54 smithi156 ceph-mon[195101]: from='client.37926 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:22:55.612 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:22:55 smithi064 conmon[198859]: ::ffff:172.21.15.156 - - [09/Oct/2023:06:22:55] "GET /metrics HTTP/1.1" 200 34083 "" "Prometheus/2.43.0" 2023-10-09T06:22:56.834 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-10-09T06:22:56.836 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-10-09T06:22:57.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:56 smithi064 ceph-mon[212750]: pgmap v1801: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:57.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:56 smithi064 ceph-mon[220044]: pgmap v1801: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:57.370 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:56 smithi156 ceph-mon[195101]: pgmap v1801: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:22:57.545 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1801,"stamp":"2023-10-09T06:22:55.394580+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5962,"num_read_kb":13177,"num_write":908,"num_write_kb":12725,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":212,"num_bytes_recovered":7107771,"num_keys_recovered":94,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6022,"ondisk_log_size":6022,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6930724,"kb_used_data":79780,"kb_used_omap":264,"kb_used_meta":6850615,"kb_avail":743030492,"statfs":{"total":767960285184,"available":760863223808,"internally_reserved":0,"allocated":81694720,"data_stored":59931436,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":270708,"internal_metadata":7015030412},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003009"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":1026,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.042876+0000","last_change":"2023-10-09T06:22:21.042876+0000","last_active":"2023-10-09T06:22:21.042876+0000","last_peered":"2023-10-09T06:22:21.042876+0000","last_clean":"2023-10-09T06:22:21.042876+0000","last_became_active":"2023-10-09T06:22:21.042521+0000","last_became_peered":"2023-10-09T06:22:21.042521+0000","last_unstale":"2023-10-09T06:22:21.042876+0000","last_undegraded":"2023-10-09T06:22:21.042876+0000","last_fullsized":"2023-10-09T06:22:21.042876+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:05.412941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:05.412941+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:29:23.871949+0000","scrub_duration":0.0061014329999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1576128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":57,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.029403+0000","last_change":"2023-10-09T06:21:19.918486+0000","last_active":"2023-10-09T06:22:20.029403+0000","last_peered":"2023-10-09T06:22:20.029403+0000","last_clean":"2023-10-09T06:22:20.029403+0000","last_became_active":"2023-10-09T06:21:19.918089+0000","last_became_peered":"2023-10-09T06:21:19.918089+0000","last_unstale":"2023-10-09T06:22:20.029403+0000","last_undegraded":"2023-10-09T06:22:20.029403+0000","last_fullsized":"2023-10-09T06:22:20.029403+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:16.089537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":542,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.027225+0000","last_change":"2023-10-09T06:21:14.895796+0000","last_active":"2023-10-09T06:22:21.027225+0000","last_peered":"2023-10-09T06:22:21.027225+0000","last_clean":"2023-10-09T06:22:21.027225+0000","last_became_active":"2023-10-09T06:21:14.895409+0000","last_became_peered":"2023-10-09T06:21:14.895409+0000","last_unstale":"2023-10-09T06:22:21.027225+0000","last_undegraded":"2023-10-09T06:22:21.027225+0000","last_fullsized":"2023-10-09T06:22:21.027225+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:07.852878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":641,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.043336+0000","last_change":"2023-10-09T06:22:21.043336+0000","last_active":"2023-10-09T06:22:21.043336+0000","last_peered":"2023-10-09T06:22:21.043336+0000","last_clean":"2023-10-09T06:22:21.043336+0000","last_became_active":"2023-10-09T06:22:21.042971+0000","last_became_peered":"2023-10-09T06:22:21.042971+0000","last_unstale":"2023-10-09T06:22:21.043336+0000","last_undegraded":"2023-10-09T06:22:21.043336+0000","last_fullsized":"2023-10-09T06:22:21.043336+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:21:18.283899+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T06:21:18.283899+0000","last_clean_scrub_stamp":"2023-10-09T06:21:18.283899+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:36.057623+0000","scrub_duration":0.0018838520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":35,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.039293+0000","last_change":"2023-10-09T06:21:31.568560+0000","last_active":"2023-10-09T06:22:20.039293+0000","last_peered":"2023-10-09T06:22:20.039293+0000","last_clean":"2023-10-09T06:22:20.039293+0000","last_became_active":"2023-10-09T06:21:31.568221+0000","last_became_peered":"2023-10-09T06:21:31.568221+0000","last_unstale":"2023-10-09T06:22:20.039293+0000","last_undegraded":"2023-10-09T06:22:20.039293+0000","last_fullsized":"2023-10-09T06:22:20.039293+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:15.935243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":439,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.027013+0000","last_change":"2023-10-09T06:22:13.707343+0000","last_active":"2023-10-09T06:22:20.027013+0000","last_peered":"2023-10-09T06:22:20.027013+0000","last_clean":"2023-10-09T06:22:20.027013+0000","last_became_active":"2023-10-09T06:22:13.706968+0000","last_became_peered":"2023-10-09T06:22:13.706968+0000","last_unstale":"2023-10-09T06:22:20.027013+0000","last_undegraded":"2023-10-09T06:22:20.027013+0000","last_fullsized":"2023-10-09T06:22:20.027013+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:46.165614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":935,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.029503+0000","last_change":"2023-10-09T06:21:34.892433+0000","last_active":"2023-10-09T06:22:20.029503+0000","last_peered":"2023-10-09T06:22:20.029503+0000","last_clean":"2023-10-09T06:22:20.029503+0000","last_became_active":"2023-10-09T06:21:34.891944+0000","last_became_peered":"2023-10-09T06:21:34.891944+0000","last_unstale":"2023-10-09T06:22:20.029503+0000","last_undegraded":"2023-10-09T06:22:20.029503+0000","last_fullsized":"2023-10-09T06:22:20.029503+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.293426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.293426+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:20.649010+0000","scrub_duration":0.001587946,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":533,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.032754+0000","last_change":"2023-10-09T06:21:31.568601+0000","last_active":"2023-10-09T06:22:20.032754+0000","last_peered":"2023-10-09T06:22:20.032754+0000","last_clean":"2023-10-09T06:22:20.032754+0000","last_became_active":"2023-10-09T06:21:31.568081+0000","last_became_peered":"2023-10-09T06:21:31.568081+0000","last_unstale":"2023-10-09T06:22:20.032754+0000","last_undegraded":"2023-10-09T06:22:20.032754+0000","last_fullsized":"2023-10-09T06:22:20.032754+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:23.144207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":174,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.032457+0000","last_change":"2023-10-09T06:21:22.894700+0000","last_active":"2023-10-09T06:22:20.032457+0000","last_peered":"2023-10-09T06:22:20.032457+0000","last_clean":"2023-10-09T06:22:20.032457+0000","last_became_active":"2023-10-09T06:21:22.894338+0000","last_became_peered":"2023-10-09T06:21:22.894338+0000","last_unstale":"2023-10-09T06:22:20.032457+0000","last_undegraded":"2023-10-09T06:22:20.032457+0000","last_fullsized":"2023-10-09T06:22:20.032457+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:58.574067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":985,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.034793+0000","last_change":"2023-10-09T06:21:25.907079+0000","last_active":"2023-10-09T06:22:20.034793+0000","last_peered":"2023-10-09T06:22:20.034793+0000","last_clean":"2023-10-09T06:22:20.034793+0000","last_became_active":"2023-10-09T06:21:25.906690+0000","last_became_peered":"2023-10-09T06:21:25.906690+0000","last_unstale":"2023-10-09T06:22:20.034793+0000","last_undegraded":"2023-10-09T06:22:20.034793+0000","last_fullsized":"2023-10-09T06:22:20.034793+0000","mapping_epoch":683,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":684,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.384407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.384407+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:16:42.288044+0000","scrub_duration":0.0040651960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":621,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.027512+0000","last_change":"2023-10-09T06:22:12.916337+0000","last_active":"2023-10-09T06:22:20.027512+0000","last_peered":"2023-10-09T06:22:20.027512+0000","last_clean":"2023-10-09T06:22:20.027512+0000","last_became_active":"2023-10-09T06:22:12.916026+0000","last_became_peered":"2023-10-09T06:22:12.916026+0000","last_unstale":"2023-10-09T06:22:20.027512+0000","last_undegraded":"2023-10-09T06:22:20.027512+0000","last_fullsized":"2023-10-09T06:22:20.027512+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:05.280276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":596,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.032853+0000","last_change":"2023-10-09T06:21:31.568420+0000","last_active":"2023-10-09T06:22:20.032853+0000","last_peered":"2023-10-09T06:22:20.032853+0000","last_clean":"2023-10-09T06:22:20.032853+0000","last_became_active":"2023-10-09T06:21:31.567473+0000","last_became_peered":"2023-10-09T06:21:31.567473+0000","last_unstale":"2023-10-09T06:22:20.032853+0000","last_undegraded":"2023-10-09T06:22:20.032853+0000","last_fullsized":"2023-10-09T06:22:20.032853+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:29.657914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":914,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.038239+0000","last_change":"2023-10-09T06:21:40.939806+0000","last_active":"2023-10-09T06:22:20.038239+0000","last_peered":"2023-10-09T06:22:20.038239+0000","last_clean":"2023-10-09T06:22:20.038239+0000","last_became_active":"2023-10-09T06:21:40.939463+0000","last_became_peered":"2023-10-09T06:21:40.939463+0000","last_unstale":"2023-10-09T06:22:20.038239+0000","last_undegraded":"2023-10-09T06:22:20.038239+0000","last_fullsized":"2023-10-09T06:22:20.038239+0000","mapping_epoch":693,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":694,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:20.742561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:20.742561+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:35.383613+0000","scrub_duration":0.0027005039999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":230,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.042801+0000","last_change":"2023-10-09T06:22:21.042801+0000","last_active":"2023-10-09T06:22:21.042801+0000","last_peered":"2023-10-09T06:22:21.042801+0000","last_clean":"2023-10-09T06:22:21.042801+0000","last_became_active":"2023-10-09T06:22:21.042476+0000","last_became_peered":"2023-10-09T06:22:21.042476+0000","last_unstale":"2023-10-09T06:22:21.042801+0000","last_undegraded":"2023-10-09T06:22:21.042801+0000","last_fullsized":"2023-10-09T06:22:21.042801+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:37.154983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":724,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.033983+0000","last_change":"2023-10-09T06:18:12.816564+0000","last_active":"2023-10-09T06:22:20.033983+0000","last_peered":"2023-10-09T06:22:20.033983+0000","last_clean":"2023-10-09T06:22:20.033983+0000","last_became_active":"2023-10-09T06:18:12.815943+0000","last_became_peered":"2023-10-09T06:18:12.815943+0000","last_unstale":"2023-10-09T06:22:20.033983+0000","last_undegraded":"2023-10-09T06:22:20.033983+0000","last_fullsized":"2023-10-09T06:22:20.033983+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:10.114324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":778,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.039972+0000","last_change":"2023-10-09T06:22:21.039972+0000","last_active":"2023-10-09T06:22:21.039972+0000","last_peered":"2023-10-09T06:22:21.039972+0000","last_clean":"2023-10-09T06:22:21.039972+0000","last_became_active":"2023-10-09T06:22:21.039641+0000","last_became_peered":"2023-10-09T06:22:21.039641+0000","last_unstale":"2023-10-09T06:22:21.039972+0000","last_undegraded":"2023-10-09T06:22:21.039972+0000","last_fullsized":"2023-10-09T06:22:21.039972+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:38.588641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:38.588641+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:48.531530+0000","scrub_duration":0.010116385,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":231,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.034695+0000","last_change":"2023-10-09T06:21:14.922061+0000","last_active":"2023-10-09T06:22:20.034695+0000","last_peered":"2023-10-09T06:22:20.034695+0000","last_clean":"2023-10-09T06:22:20.034695+0000","last_became_active":"2023-10-09T06:21:14.908746+0000","last_became_peered":"2023-10-09T06:21:14.908746+0000","last_unstale":"2023-10-09T06:22:20.034695+0000","last_undegraded":"2023-10-09T06:22:20.034695+0000","last_fullsized":"2023-10-09T06:22:20.034695+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:06.933730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":35,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.040027+0000","last_change":"2023-10-09T06:21:31.570013+0000","last_active":"2023-10-09T06:22:20.040027+0000","last_peered":"2023-10-09T06:22:20.040027+0000","last_clean":"2023-10-09T06:22:20.040027+0000","last_became_active":"2023-10-09T06:21:31.569705+0000","last_became_peered":"2023-10-09T06:21:31.569705+0000","last_unstale":"2023-10-09T06:22:20.040027+0000","last_undegraded":"2023-10-09T06:22:20.040027+0000","last_fullsized":"2023-10-09T06:22:20.040027+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:52.150192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":885,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.040379+0000","last_change":"2023-10-09T06:09:27.574336+0000","last_active":"2023-10-09T06:22:20.040379+0000","last_peered":"2023-10-09T06:22:20.040379+0000","last_clean":"2023-10-09T06:22:20.040379+0000","last_became_active":"2023-10-09T06:09:27.573974+0000","last_became_peered":"2023-10-09T06:09:27.573974+0000","last_unstale":"2023-10-09T06:22:20.040379+0000","last_undegraded":"2023-10-09T06:22:20.040379+0000","last_fullsized":"2023-10-09T06:22:20.040379+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:57.339121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:57.339121+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:45.908579+0000","scrub_duration":0.010474344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":57,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.029706+0000","last_change":"2023-10-09T06:21:14.926090+0000","last_active":"2023-10-09T06:22:20.029706+0000","last_peered":"2023-10-09T06:22:20.029706+0000","last_clean":"2023-10-09T06:22:20.029706+0000","last_became_active":"2023-10-09T06:21:14.904702+0000","last_became_peered":"2023-10-09T06:21:14.904702+0000","last_unstale":"2023-10-09T06:22:20.029706+0000","last_undegraded":"2023-10-09T06:22:20.029706+0000","last_fullsized":"2023-10-09T06:22:20.029706+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:51.732161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":67,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.029730+0000","last_change":"2023-10-09T06:21:31.575158+0000","last_active":"2023-10-09T06:22:20.029730+0000","last_peered":"2023-10-09T06:22:20.029730+0000","last_clean":"2023-10-09T06:22:20.029730+0000","last_became_active":"2023-10-09T06:21:31.572974+0000","last_became_peered":"2023-10-09T06:21:31.572974+0000","last_unstale":"2023-10-09T06:22:20.029730+0000","last_undegraded":"2023-10-09T06:22:20.029730+0000","last_fullsized":"2023-10-09T06:22:20.029730+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:59.423082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":562,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.026772+0000","last_change":"2023-10-09T06:22:12.916424+0000","last_active":"2023-10-09T06:22:20.026772+0000","last_peered":"2023-10-09T06:22:20.026772+0000","last_clean":"2023-10-09T06:22:20.026772+0000","last_became_active":"2023-10-09T06:22:12.916102+0000","last_became_peered":"2023-10-09T06:22:12.916102+0000","last_unstale":"2023-10-09T06:22:20.026772+0000","last_undegraded":"2023-10-09T06:22:20.026772+0000","last_fullsized":"2023-10-09T06:22:20.026772+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:42.900876+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:42.900876+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:44:30.686361+0000","scrub_duration":0.001618488,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":295,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.034557+0000","last_change":"2023-10-09T06:21:19.932732+0000","last_active":"2023-10-09T06:22:20.034557+0000","last_peered":"2023-10-09T06:22:20.034557+0000","last_clean":"2023-10-09T06:22:20.034557+0000","last_became_active":"2023-10-09T06:21:19.928092+0000","last_became_peered":"2023-10-09T06:21:19.928092+0000","last_unstale":"2023-10-09T06:22:20.034557+0000","last_undegraded":"2023-10-09T06:22:20.034557+0000","last_fullsized":"2023-10-09T06:22:20.034557+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:23.967352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":306,"num_keys_recovered":90,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":804,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.034687+0000","last_change":"2023-10-09T06:22:02.796081+0000","last_active":"2023-10-09T06:22:20.034687+0000","last_peered":"2023-10-09T06:22:20.034687+0000","last_clean":"2023-10-09T06:22:20.034687+0000","last_became_active":"2023-10-09T06:22:02.795655+0000","last_became_peered":"2023-10-09T06:22:02.795655+0000","last_unstale":"2023-10-09T06:22:20.034687+0000","last_undegraded":"2023-10-09T06:22:20.034687+0000","last_fullsized":"2023-10-09T06:22:20.034687+0000","mapping_epoch":702,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":703,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:07.057044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":1300,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.045630+0000","last_change":"2023-10-09T06:22:21.045630+0000","last_active":"2023-10-09T06:22:21.045630+0000","last_peered":"2023-10-09T06:22:21.045630+0000","last_clean":"2023-10-09T06:22:21.045630+0000","last_became_active":"2023-10-09T06:22:21.045293+0000","last_became_peered":"2023-10-09T06:22:21.045293+0000","last_unstale":"2023-10-09T06:22:21.045630+0000","last_undegraded":"2023-10-09T06:22:21.045630+0000","last_fullsized":"2023-10-09T06:22:21.045630+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:15.170409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:15.170409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:29.580389+0000","scrub_duration":0.0021771239999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":402,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.026454+0000","last_change":"2023-10-09T06:22:13.707470+0000","last_active":"2023-10-09T06:22:20.026454+0000","last_peered":"2023-10-09T06:22:20.026454+0000","last_clean":"2023-10-09T06:22:20.026454+0000","last_became_active":"2023-10-09T06:22:13.706954+0000","last_became_peered":"2023-10-09T06:22:13.706954+0000","last_unstale":"2023-10-09T06:22:20.026454+0000","last_undegraded":"2023-10-09T06:22:20.026454+0000","last_fullsized":"2023-10-09T06:22:20.026454+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:54.394842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":205,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.043525+0000","last_change":"2023-10-09T06:22:21.043525+0000","last_active":"2023-10-09T06:22:21.043525+0000","last_peered":"2023-10-09T06:22:21.043525+0000","last_clean":"2023-10-09T06:22:21.043525+0000","last_became_active":"2023-10-09T06:22:21.042362+0000","last_became_peered":"2023-10-09T06:22:21.042362+0000","last_unstale":"2023-10-09T06:22:21.043525+0000","last_undegraded":"2023-10-09T06:22:21.043525+0000","last_fullsized":"2023-10-09T06:22:21.043525+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:19.700258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":987,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.039837+0000","last_change":"2023-10-09T06:21:17.903751+0000","last_active":"2023-10-09T06:22:20.039837+0000","last_peered":"2023-10-09T06:22:20.039837+0000","last_clean":"2023-10-09T06:22:20.039837+0000","last_became_active":"2023-10-09T06:21:17.903469+0000","last_became_peered":"2023-10-09T06:21:17.903469+0000","last_unstale":"2023-10-09T06:22:20.039837+0000","last_undegraded":"2023-10-09T06:22:20.039837+0000","last_fullsized":"2023-10-09T06:22:20.039837+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:43.316697+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:43.316697+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:55.762584+0000","scrub_duration":0.001666561,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":603,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.034232+0000","last_change":"2023-10-09T06:21:14.910129+0000","last_active":"2023-10-09T06:22:20.034232+0000","last_peered":"2023-10-09T06:22:20.034232+0000","last_clean":"2023-10-09T06:22:20.034232+0000","last_became_active":"2023-10-09T06:21:14.909795+0000","last_became_peered":"2023-10-09T06:21:14.909795+0000","last_unstale":"2023-10-09T06:22:20.034232+0000","last_undegraded":"2023-10-09T06:22:20.034232+0000","last_fullsized":"2023-10-09T06:22:20.034232+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:28.905834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":494,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.026915+0000","last_change":"2023-10-09T06:22:13.707340+0000","last_active":"2023-10-09T06:22:20.026915+0000","last_peered":"2023-10-09T06:22:20.026915+0000","last_clean":"2023-10-09T06:22:20.026915+0000","last_became_active":"2023-10-09T06:22:13.706991+0000","last_became_peered":"2023-10-09T06:22:13.706991+0000","last_unstale":"2023-10-09T06:22:20.026915+0000","last_undegraded":"2023-10-09T06:22:20.026915+0000","last_fullsized":"2023-10-09T06:22:20.026915+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:01.637273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":1016,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.041404+0000","last_change":"2023-10-09T06:22:21.041404+0000","last_active":"2023-10-09T06:22:21.041404+0000","last_peered":"2023-10-09T06:22:21.041404+0000","last_clean":"2023-10-09T06:22:21.041404+0000","last_became_active":"2023-10-09T06:22:21.040753+0000","last_became_peered":"2023-10-09T06:22:21.040753+0000","last_unstale":"2023-10-09T06:22:21.041404+0000","last_undegraded":"2023-10-09T06:22:21.041404+0000","last_fullsized":"2023-10-09T06:22:21.041404+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:40.388125+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:40.388125+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:10.312209+0000","scrub_duration":0.0055576990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":321,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.032232+0000","last_change":"2023-10-09T06:21:22.914876+0000","last_active":"2023-10-09T06:22:20.032232+0000","last_peered":"2023-10-09T06:22:20.032232+0000","last_clean":"2023-10-09T06:22:20.032232+0000","last_became_active":"2023-10-09T06:21:22.891764+0000","last_became_peered":"2023-10-09T06:21:22.891764+0000","last_unstale":"2023-10-09T06:22:20.032232+0000","last_undegraded":"2023-10-09T06:22:20.032232+0000","last_fullsized":"2023-10-09T06:22:20.032232+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:56.617645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":248,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.026798+0000","last_change":"2023-10-09T06:21:04.990757+0000","last_active":"2023-10-09T06:22:21.026798+0000","last_peered":"2023-10-09T06:22:21.026798+0000","last_clean":"2023-10-09T06:22:21.026798+0000","last_became_active":"2023-10-09T06:21:04.990151+0000","last_became_peered":"2023-10-09T06:21:04.990151+0000","last_unstale":"2023-10-09T06:22:21.026798+0000","last_undegraded":"2023-10-09T06:22:21.026798+0000","last_fullsized":"2023-10-09T06:22:21.026798+0000","mapping_epoch":666,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":667,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:52.630600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":931,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.035012+0000","last_change":"2023-10-09T06:21:17.908395+0000","last_active":"2023-10-09T06:22:20.035012+0000","last_peered":"2023-10-09T06:22:20.035012+0000","last_clean":"2023-10-09T06:22:20.035012+0000","last_became_active":"2023-10-09T06:21:17.907972+0000","last_became_peered":"2023-10-09T06:21:17.907972+0000","last_unstale":"2023-10-09T06:22:20.035012+0000","last_undegraded":"2023-10-09T06:22:20.035012+0000","last_fullsized":"2023-10-09T06:22:20.035012+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:10:59.422551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:10:59.422551+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:31.931097+0000","scrub_duration":0.0059716659999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":253,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.036002+0000","last_change":"2023-10-09T06:21:14.933563+0000","last_active":"2023-10-09T06:22:20.036002+0000","last_peered":"2023-10-09T06:22:20.036002+0000","last_clean":"2023-10-09T06:22:20.036002+0000","last_became_active":"2023-10-09T06:21:14.910247+0000","last_became_peered":"2023-10-09T06:21:14.910247+0000","last_unstale":"2023-10-09T06:22:20.036002+0000","last_undegraded":"2023-10-09T06:22:20.036002+0000","last_fullsized":"2023-10-09T06:22:20.036002+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:55.999558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":239,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.045188+0000","last_change":"2023-10-09T06:22:21.045188+0000","last_active":"2023-10-09T06:22:21.045188+0000","last_peered":"2023-10-09T06:22:21.045188+0000","last_clean":"2023-10-09T06:22:21.045188+0000","last_became_active":"2023-10-09T06:22:21.044873+0000","last_became_peered":"2023-10-09T06:22:21.044873+0000","last_unstale":"2023-10-09T06:22:21.045188+0000","last_undegraded":"2023-10-09T06:22:21.045188+0000","last_fullsized":"2023-10-09T06:22:21.045188+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:55.954049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":1128,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.027449+0000","last_change":"2023-10-09T06:18:15.867832+0000","last_active":"2023-10-09T06:22:21.027449+0000","last_peered":"2023-10-09T06:22:21.027449+0000","last_clean":"2023-10-09T06:22:21.027449+0000","last_became_active":"2023-10-09T06:18:15.864512+0000","last_became_peered":"2023-10-09T06:18:15.864512+0000","last_unstale":"2023-10-09T06:22:21.027449+0000","last_undegraded":"2023-10-09T06:22:21.027449+0000","last_fullsized":"2023-10-09T06:22:21.027449+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:03.241335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:03.241335+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:45.197013+0000","scrub_duration":0.017780547000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":225,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.037050+0000","last_change":"2023-10-09T06:21:14.929007+0000","last_active":"2023-10-09T06:22:20.037050+0000","last_peered":"2023-10-09T06:22:20.037050+0000","last_clean":"2023-10-09T06:22:20.037050+0000","last_became_active":"2023-10-09T06:21:14.909089+0000","last_became_peered":"2023-10-09T06:21:14.909089+0000","last_unstale":"2023-10-09T06:22:20.037050+0000","last_undegraded":"2023-10-09T06:22:20.037050+0000","last_fullsized":"2023-10-09T06:22:20.037050+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:27.294970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":67,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.029347+0000","last_change":"2023-10-09T06:22:02.048965+0000","last_active":"2023-10-09T06:22:20.029347+0000","last_peered":"2023-10-09T06:22:20.029347+0000","last_clean":"2023-10-09T06:22:20.029347+0000","last_became_active":"2023-10-09T06:22:02.048677+0000","last_became_peered":"2023-10-09T06:22:02.048677+0000","last_unstale":"2023-10-09T06:22:20.029347+0000","last_undegraded":"2023-10-09T06:22:20.029347+0000","last_fullsized":"2023-10-09T06:22:20.029347+0000","mapping_epoch":702,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":703,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:15.764062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":1034,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.031792+0000","last_change":"2023-10-09T06:21:25.913480+0000","last_active":"2023-10-09T06:22:20.031792+0000","last_peered":"2023-10-09T06:22:20.031792+0000","last_clean":"2023-10-09T06:22:20.031792+0000","last_became_active":"2023-10-09T06:21:25.913120+0000","last_became_peered":"2023-10-09T06:21:25.913120+0000","last_unstale":"2023-10-09T06:22:20.031792+0000","last_undegraded":"2023-10-09T06:22:20.031792+0000","last_fullsized":"2023-10-09T06:22:20.031792+0000","mapping_epoch":683,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":684,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:48.495739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:48.495739+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:28.522756+0000","scrub_duration":0.0016345509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":274,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.034893+0000","last_change":"2023-10-09T06:21:22.919873+0000","last_active":"2023-10-09T06:22:20.034893+0000","last_peered":"2023-10-09T06:22:20.034893+0000","last_clean":"2023-10-09T06:22:20.034893+0000","last_became_active":"2023-10-09T06:21:22.906107+0000","last_became_peered":"2023-10-09T06:21:22.906107+0000","last_unstale":"2023-10-09T06:22:20.034893+0000","last_undegraded":"2023-10-09T06:22:20.034893+0000","last_fullsized":"2023-10-09T06:22:20.034893+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:35.309519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":721,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.033294+0000","last_change":"2023-10-09T06:20:29.964135+0000","last_active":"2023-10-09T06:22:20.033294+0000","last_peered":"2023-10-09T06:22:20.033294+0000","last_clean":"2023-10-09T06:22:20.033294+0000","last_became_active":"2023-10-09T06:20:29.963434+0000","last_became_peered":"2023-10-09T06:20:29.963434+0000","last_unstale":"2023-10-09T06:22:20.033294+0000","last_undegraded":"2023-10-09T06:22:20.033294+0000","last_fullsized":"2023-10-09T06:22:20.033294+0000","mapping_epoch":646,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":647,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:42.376435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":784,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.031964+0000","last_change":"2023-10-09T06:21:17.906807+0000","last_active":"2023-10-09T06:22:20.031964+0000","last_peered":"2023-10-09T06:22:20.031964+0000","last_clean":"2023-10-09T06:22:20.031964+0000","last_became_active":"2023-10-09T06:21:17.906359+0000","last_became_peered":"2023-10-09T06:21:17.906359+0000","last_unstale":"2023-10-09T06:22:20.031964+0000","last_undegraded":"2023-10-09T06:22:20.031964+0000","last_fullsized":"2023-10-09T06:22:20.031964+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:16.361097+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:16.361097+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:34.936406+0000","scrub_duration":0.028533749000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":510,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.036945+0000","last_change":"2023-10-09T06:22:21.036945+0000","last_active":"2023-10-09T06:22:21.036945+0000","last_peered":"2023-10-09T06:22:21.036945+0000","last_clean":"2023-10-09T06:22:21.036945+0000","last_became_active":"2023-10-09T06:22:21.036583+0000","last_became_peered":"2023-10-09T06:22:21.036583+0000","last_unstale":"2023-10-09T06:22:21.036945+0000","last_undegraded":"2023-10-09T06:22:21.036945+0000","last_fullsized":"2023-10-09T06:22:21.036945+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:28.504108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":744,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.026793+0000","last_change":"2023-10-09T06:21:14.889819+0000","last_active":"2023-10-09T06:22:20.026793+0000","last_peered":"2023-10-09T06:22:20.026793+0000","last_clean":"2023-10-09T06:22:20.026793+0000","last_became_active":"2023-10-09T06:21:14.889428+0000","last_became_peered":"2023-10-09T06:21:14.889428+0000","last_unstale":"2023-10-09T06:22:20.026793+0000","last_undegraded":"2023-10-09T06:22:20.026793+0000","last_fullsized":"2023-10-09T06:22:20.026793+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:01.117391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":1193,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.034069+0000","last_change":"2023-10-09T06:21:40.939446+0000","last_active":"2023-10-09T06:22:20.034069+0000","last_peered":"2023-10-09T06:22:20.034069+0000","last_clean":"2023-10-09T06:22:20.034069+0000","last_became_active":"2023-10-09T06:21:40.939117+0000","last_became_peered":"2023-10-09T06:21:40.939117+0000","last_unstale":"2023-10-09T06:22:20.034069+0000","last_undegraded":"2023-10-09T06:22:20.034069+0000","last_fullsized":"2023-10-09T06:22:20.034069+0000","mapping_epoch":693,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":694,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:42.795781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:42.795781+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:29.492945+0000","scrub_duration":0.0091262979999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":556,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.033124+0000","last_change":"2023-10-09T06:21:14.911214+0000","last_active":"2023-10-09T06:22:20.033124+0000","last_peered":"2023-10-09T06:22:20.033124+0000","last_clean":"2023-10-09T06:22:20.033124+0000","last_became_active":"2023-10-09T06:21:14.899220+0000","last_became_peered":"2023-10-09T06:21:14.899220+0000","last_unstale":"2023-10-09T06:22:20.033124+0000","last_undegraded":"2023-10-09T06:22:20.033124+0000","last_fullsized":"2023-10-09T06:22:20.033124+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:32.381582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":47,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.029661+0000","last_change":"2023-10-09T06:21:14.906829+0000","last_active":"2023-10-09T06:22:20.029661+0000","last_peered":"2023-10-09T06:22:20.029661+0000","last_clean":"2023-10-09T06:22:20.029661+0000","last_became_active":"2023-10-09T06:21:14.906543+0000","last_became_peered":"2023-10-09T06:21:14.906543+0000","last_unstale":"2023-10-09T06:22:20.029661+0000","last_undegraded":"2023-10-09T06:22:20.029661+0000","last_fullsized":"2023-10-09T06:22:20.029661+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:55:26.944801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":1071,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.037748+0000","last_change":"2023-10-09T06:22:21.037748+0000","last_active":"2023-10-09T06:22:21.037748+0000","last_peered":"2023-10-09T06:22:21.037748+0000","last_clean":"2023-10-09T06:22:21.037748+0000","last_became_active":"2023-10-09T06:22:21.037443+0000","last_became_peered":"2023-10-09T06:22:21.037443+0000","last_unstale":"2023-10-09T06:22:21.037748+0000","last_undegraded":"2023-10-09T06:22:21.037748+0000","last_fullsized":"2023-10-09T06:22:21.037748+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:58.282393+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:58.282393+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:39.433468+0000","scrub_duration":0.01402403,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":585,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.033766+0000","last_change":"2023-10-09T06:21:54.068630+0000","last_active":"2023-10-09T06:22:20.033766+0000","last_peered":"2023-10-09T06:22:20.033766+0000","last_clean":"2023-10-09T06:22:20.033766+0000","last_became_active":"2023-10-09T06:21:54.068364+0000","last_became_peered":"2023-10-09T06:21:54.068364+0000","last_unstale":"2023-10-09T06:22:20.033766+0000","last_undegraded":"2023-10-09T06:22:20.033766+0000","last_fullsized":"2023-10-09T06:22:20.033766+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:19.998776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":57,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.032301+0000","last_change":"2023-10-09T06:21:37.886287+0000","last_active":"2023-10-09T06:22:20.032301+0000","last_peered":"2023-10-09T06:22:20.032301+0000","last_clean":"2023-10-09T06:22:20.032301+0000","last_became_active":"2023-10-09T06:21:37.886038+0000","last_became_peered":"2023-10-09T06:21:37.886038+0000","last_unstale":"2023-10-09T06:22:20.032301+0000","last_undegraded":"2023-10-09T06:22:20.032301+0000","last_fullsized":"2023-10-09T06:22:20.032301+0000","mapping_epoch":690,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":691,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:06:13.080046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":1097,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.039690+0000","last_change":"2023-10-09T06:21:25.905224+0000","last_active":"2023-10-09T06:22:20.039690+0000","last_peered":"2023-10-09T06:22:20.039690+0000","last_clean":"2023-10-09T06:22:20.039690+0000","last_became_active":"2023-10-09T06:21:25.904818+0000","last_became_peered":"2023-10-09T06:21:25.904818+0000","last_unstale":"2023-10-09T06:22:20.039690+0000","last_undegraded":"2023-10-09T06:22:20.039690+0000","last_fullsized":"2023-10-09T06:22:20.039690+0000","mapping_epoch":683,"log_start":"0'0","ondisk_log_start":"0'0","created":155,"last_epoch_clean":684,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:59:56.373748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:59:56.373748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:37.543642+0000","scrub_duration":0.0024576450000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":57,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.029248+0000","last_change":"2023-10-09T06:21:19.924713+0000","last_active":"2023-10-09T06:22:20.029248+0000","last_peered":"2023-10-09T06:22:20.029248+0000","last_clean":"2023-10-09T06:22:20.029248+0000","last_became_active":"2023-10-09T06:21:19.924421+0000","last_became_peered":"2023-10-09T06:21:19.924421+0000","last_unstale":"2023-10-09T06:22:20.029248+0000","last_undegraded":"2023-10-09T06:22:20.029248+0000","last_fullsized":"2023-10-09T06:22:20.029248+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:33.242263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":57,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.031873+0000","last_change":"2023-10-09T06:21:31.570579+0000","last_active":"2023-10-09T06:22:20.031873+0000","last_peered":"2023-10-09T06:22:20.031873+0000","last_clean":"2023-10-09T06:22:20.031873+0000","last_became_active":"2023-10-09T06:21:31.570215+0000","last_became_peered":"2023-10-09T06:21:31.570215+0000","last_unstale":"2023-10-09T06:22:20.031873+0000","last_undegraded":"2023-10-09T06:22:20.031873+0000","last_fullsized":"2023-10-09T06:22:20.031873+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:55.013279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":808,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.040514+0000","last_change":"2023-10-09T06:22:21.040514+0000","last_active":"2023-10-09T06:22:21.040514+0000","last_peered":"2023-10-09T06:22:21.040514+0000","last_clean":"2023-10-09T06:22:21.040514+0000","last_became_active":"2023-10-09T06:22:21.039886+0000","last_became_peered":"2023-10-09T06:22:21.039886+0000","last_unstale":"2023-10-09T06:22:21.040514+0000","last_undegraded":"2023-10-09T06:22:21.040514+0000","last_fullsized":"2023-10-09T06:22:21.040514+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:39.633410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:39.633410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.633552+0000","scrub_duration":0.0039938459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":561,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.032641+0000","last_change":"2023-10-09T06:21:31.569751+0000","last_active":"2023-10-09T06:22:20.032641+0000","last_peered":"2023-10-09T06:22:20.032641+0000","last_clean":"2023-10-09T06:22:20.032641+0000","last_became_active":"2023-10-09T06:21:31.569508+0000","last_became_peered":"2023-10-09T06:21:31.569508+0000","last_unstale":"2023-10-09T06:22:20.032641+0000","last_undegraded":"2023-10-09T06:22:20.032641+0000","last_fullsized":"2023-10-09T06:22:20.032641+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:34.063041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":414,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.040477+0000","last_change":"2023-10-09T06:21:14.889635+0000","last_active":"2023-10-09T06:22:20.040477+0000","last_peered":"2023-10-09T06:22:20.040477+0000","last_clean":"2023-10-09T06:22:20.040477+0000","last_became_active":"2023-10-09T06:21:14.889288+0000","last_became_peered":"2023-10-09T06:21:14.889288+0000","last_unstale":"2023-10-09T06:22:20.040477+0000","last_undegraded":"2023-10-09T06:22:20.040477+0000","last_fullsized":"2023-10-09T06:22:20.040477+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:01:23.826329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":909,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.031686+0000","last_change":"2023-10-09T06:20:29.966275+0000","last_active":"2023-10-09T06:22:20.031686+0000","last_peered":"2023-10-09T06:22:20.031686+0000","last_clean":"2023-10-09T06:22:20.031686+0000","last_became_active":"2023-10-09T06:20:29.964913+0000","last_became_peered":"2023-10-09T06:20:29.964913+0000","last_unstale":"2023-10-09T06:22:20.031686+0000","last_undegraded":"2023-10-09T06:22:20.031686+0000","last_fullsized":"2023-10-09T06:22:20.031686+0000","mapping_epoch":646,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":647,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:12:22.289803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:12:22.289803+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:17.781708+0000","scrub_duration":0.0076565050000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"670'187","reported_seq":1040,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.039726+0000","last_change":"2023-10-09T06:21:34.889682+0000","last_active":"2023-10-09T06:22:20.039726+0000","last_peered":"2023-10-09T06:22:20.039726+0000","last_clean":"2023-10-09T06:22:20.039726+0000","last_became_active":"2023-10-09T06:21:34.889089+0000","last_became_peered":"2023-10-09T06:21:34.889089+0000","last_unstale":"2023-10-09T06:22:20.039726+0000","last_undegraded":"2023-10-09T06:22:20.039726+0000","last_fullsized":"2023-10-09T06:22:20.039726+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":187,"log_dups_size":0,"ondisk_log_size":187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:20.611558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000879378,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":899,"num_read_kb":1784,"num_write":189,"num_write_kb":1122,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":10240,"num_keys_recovered":4,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":547,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.043579+0000","last_change":"2023-10-09T06:22:21.043579+0000","last_active":"2023-10-09T06:22:21.043579+0000","last_peered":"2023-10-09T06:22:21.043579+0000","last_clean":"2023-10-09T06:22:21.043579+0000","last_became_active":"2023-10-09T06:22:21.043092+0000","last_became_peered":"2023-10-09T06:22:21.043092+0000","last_unstale":"2023-10-09T06:22:21.043579+0000","last_undegraded":"2023-10-09T06:22:21.043579+0000","last_fullsized":"2023-10-09T06:22:21.043579+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:48.367616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":35,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.039909+0000","last_change":"2023-10-09T06:21:31.572397+0000","last_active":"2023-10-09T06:22:20.039909+0000","last_peered":"2023-10-09T06:22:20.039909+0000","last_clean":"2023-10-09T06:22:20.039909+0000","last_became_active":"2023-10-09T06:21:31.572093+0000","last_became_peered":"2023-10-09T06:21:31.572093+0000","last_unstale":"2023-10-09T06:22:20.039909+0000","last_undegraded":"2023-10-09T06:22:20.039909+0000","last_fullsized":"2023-10-09T06:22:20.039909+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:18:56.936089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":1025,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.038680+0000","last_change":"2023-10-09T06:22:21.038680+0000","last_active":"2023-10-09T06:22:21.038680+0000","last_peered":"2023-10-09T06:22:21.038680+0000","last_clean":"2023-10-09T06:22:21.038680+0000","last_became_active":"2023-10-09T06:22:21.038245+0000","last_became_peered":"2023-10-09T06:22:21.038245+0000","last_unstale":"2023-10-09T06:22:21.038680+0000","last_undegraded":"2023-10-09T06:22:21.038680+0000","last_fullsized":"2023-10-09T06:22:21.038680+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:36.224954+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:36.224954+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:32.125442+0000","scrub_duration":0.0016660799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"669'115","reported_seq":1235,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.041534+0000","last_change":"2023-10-09T06:22:21.041534+0000","last_active":"2023-10-09T06:22:21.041534+0000","last_peered":"2023-10-09T06:22:21.041534+0000","last_clean":"2023-10-09T06:22:21.041534+0000","last_became_active":"2023-10-09T06:22:21.041112+0000","last_became_peered":"2023-10-09T06:22:21.041112+0000","last_unstale":"2023-10-09T06:22:21.041534+0000","last_undegraded":"2023-10-09T06:22:21.041534+0000","last_fullsized":"2023-10-09T06:22:21.041534+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:09.039150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010144900000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1245,"num_read_kb":2142,"num_write":80,"num_write_kb":1053,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":47,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.029896+0000","last_change":"2023-10-09T06:21:14.906777+0000","last_active":"2023-10-09T06:22:20.029896+0000","last_peered":"2023-10-09T06:22:20.029896+0000","last_clean":"2023-10-09T06:22:20.029896+0000","last_became_active":"2023-10-09T06:21:14.906407+0000","last_became_peered":"2023-10-09T06:21:14.906407+0000","last_unstale":"2023-10-09T06:22:20.029896+0000","last_undegraded":"2023-10-09T06:22:20.029896+0000","last_fullsized":"2023-10-09T06:22:20.029896+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:43.590311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":499,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.037548+0000","last_change":"2023-10-09T06:22:21.037548+0000","last_active":"2023-10-09T06:22:21.037548+0000","last_peered":"2023-10-09T06:22:21.037548+0000","last_clean":"2023-10-09T06:22:21.037548+0000","last_became_active":"2023-10-09T06:22:21.037163+0000","last_became_peered":"2023-10-09T06:22:21.037163+0000","last_unstale":"2023-10-09T06:22:21.037548+0000","last_undegraded":"2023-10-09T06:22:21.037548+0000","last_fullsized":"2023-10-09T06:22:21.037548+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:14.397130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":825,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.034460+0000","last_change":"2023-10-09T06:21:17.904458+0000","last_active":"2023-10-09T06:22:20.034460+0000","last_peered":"2023-10-09T06:22:20.034460+0000","last_clean":"2023-10-09T06:22:20.034460+0000","last_became_active":"2023-10-09T06:21:17.904241+0000","last_became_peered":"2023-10-09T06:21:17.904241+0000","last_unstale":"2023-10-09T06:22:20.034460+0000","last_undegraded":"2023-10-09T06:22:20.034460+0000","last_fullsized":"2023-10-09T06:22:20.034460+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:14:47.339163+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T06:14:47.339163+0000","last_clean_scrub_stamp":"2023-10-09T06:14:47.339163+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:46:29.178159+0000","scrub_duration":0.0016607729999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"670'149","reported_seq":1133,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.036767+0000","last_change":"2023-10-09T06:21:40.936384+0000","last_active":"2023-10-09T06:22:20.036767+0000","last_peered":"2023-10-09T06:22:20.036767+0000","last_clean":"2023-10-09T06:22:20.036767+0000","last_became_active":"2023-10-09T06:21:40.935998+0000","last_became_peered":"2023-10-09T06:21:40.935998+0000","last_unstale":"2023-10-09T06:22:20.036767+0000","last_undegraded":"2023-10-09T06:22:20.036767+0000","last_fullsized":"2023-10-09T06:22:20.036767+0000","mapping_epoch":693,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":694,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:57.304826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041077000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1119,"num_read_kb":1169,"num_write":121,"num_write_kb":197,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":526,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.039375+0000","last_change":"2023-10-09T06:21:19.928503+0000","last_active":"2023-10-09T06:22:20.039375+0000","last_peered":"2023-10-09T06:22:20.039375+0000","last_clean":"2023-10-09T06:22:20.039375+0000","last_became_active":"2023-10-09T06:21:19.928155+0000","last_became_peered":"2023-10-09T06:21:19.928155+0000","last_unstale":"2023-10-09T06:22:20.039375+0000","last_undegraded":"2023-10-09T06:22:20.039375+0000","last_fullsized":"2023-10-09T06:22:20.039375+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:50:02.702751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":35,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.040296+0000","last_change":"2023-10-09T06:21:31.571108+0000","last_active":"2023-10-09T06:22:20.040296+0000","last_peered":"2023-10-09T06:22:20.040296+0000","last_clean":"2023-10-09T06:22:20.040296+0000","last_became_active":"2023-10-09T06:21:31.570738+0000","last_became_peered":"2023-10-09T06:21:31.570738+0000","last_unstale":"2023-10-09T06:22:20.040296+0000","last_undegraded":"2023-10-09T06:22:20.040296+0000","last_fullsized":"2023-10-09T06:22:20.040296+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:39.626021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"670'163","reported_seq":769,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.041653+0000","last_change":"2023-10-09T06:22:21.041653+0000","last_active":"2023-10-09T06:22:21.041653+0000","last_peered":"2023-10-09T06:22:21.041653+0000","last_clean":"2023-10-09T06:22:21.041653+0000","last_became_active":"2023-10-09T06:22:21.041160+0000","last_became_peered":"2023-10-09T06:22:21.041160+0000","last_unstale":"2023-10-09T06:22:21.041653+0000","last_undegraded":"2023-10-09T06:22:21.041653+0000","last_fullsized":"2023-10-09T06:22:21.041653+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":163,"log_dups_size":0,"ondisk_log_size":163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:56.685679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063049999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":730,"num_read_kb":2197,"num_write":152,"num_write_kb":1386,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":793,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.037769+0000","last_change":"2023-10-09T06:18:15.852226+0000","last_active":"2023-10-09T06:22:20.037769+0000","last_peered":"2023-10-09T06:22:20.037769+0000","last_clean":"2023-10-09T06:22:20.037769+0000","last_became_active":"2023-10-09T06:18:15.851868+0000","last_became_peered":"2023-10-09T06:18:15.851868+0000","last_unstale":"2023-10-09T06:22:20.037769+0000","last_undegraded":"2023-10-09T06:22:20.037769+0000","last_fullsized":"2023-10-09T06:22:20.037769+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:35.783434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:35.783434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:57.511684+0000","scrub_duration":0.0043794799999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":564,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.026833+0000","last_change":"2023-10-09T06:21:22.889768+0000","last_active":"2023-10-09T06:22:21.026833+0000","last_peered":"2023-10-09T06:22:21.026833+0000","last_clean":"2023-10-09T06:22:21.026833+0000","last_became_active":"2023-10-09T06:21:22.889396+0000","last_became_peered":"2023-10-09T06:21:22.889396+0000","last_unstale":"2023-10-09T06:22:21.026833+0000","last_undegraded":"2023-10-09T06:22:21.026833+0000","last_fullsized":"2023-10-09T06:22:21.026833+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:54.800020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":635,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.031537+0000","last_change":"2023-10-09T06:21:14.913805+0000","last_active":"2023-10-09T06:22:20.031537+0000","last_peered":"2023-10-09T06:22:20.031537+0000","last_clean":"2023-10-09T06:22:20.031537+0000","last_became_active":"2023-10-09T06:21:14.913537+0000","last_became_peered":"2023-10-09T06:21:14.913537+0000","last_unstale":"2023-10-09T06:22:20.031537+0000","last_undegraded":"2023-10-09T06:22:20.031537+0000","last_fullsized":"2023-10-09T06:22:20.031537+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:28.008641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":1074,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.036859+0000","last_change":"2023-10-09T06:21:17.907964+0000","last_active":"2023-10-09T06:22:21.036859+0000","last_peered":"2023-10-09T06:22:21.036859+0000","last_clean":"2023-10-09T06:22:21.036859+0000","last_became_active":"2023-10-09T06:21:17.907678+0000","last_became_peered":"2023-10-09T06:21:17.907678+0000","last_unstale":"2023-10-09T06:22:21.036859+0000","last_undegraded":"2023-10-09T06:22:21.036859+0000","last_fullsized":"2023-10-09T06:22:21.036859+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:06:57.627296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:06:57.627296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:29.010811+0000","scrub_duration":0.001913166,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"667'12","reported_seq":684,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.038489+0000","last_change":"2023-10-09T06:21:31.591278+0000","last_active":"2023-10-09T06:22:20.038489+0000","last_peered":"2023-10-09T06:22:20.038489+0000","last_clean":"2023-10-09T06:22:20.038489+0000","last_became_active":"2023-10-09T06:21:31.567099+0000","last_became_peered":"2023-10-09T06:21:31.567099+0000","last_unstale":"2023-10-09T06:22:20.038489+0000","last_undegraded":"2023-10-09T06:22:20.038489+0000","last_fullsized":"2023-10-09T06:22:20.038489+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:11:00.994566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082067299999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":9,"num_read_kb":1407,"num_write":8,"num_write_kb":1216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":678,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.026290+0000","last_change":"2023-10-09T06:21:14.903545+0000","last_active":"2023-10-09T06:22:20.026290+0000","last_peered":"2023-10-09T06:22:20.026290+0000","last_clean":"2023-10-09T06:22:20.026290+0000","last_became_active":"2023-10-09T06:21:14.903292+0000","last_became_peered":"2023-10-09T06:21:14.903292+0000","last_unstale":"2023-10-09T06:22:20.026290+0000","last_undegraded":"2023-10-09T06:22:20.026290+0000","last_fullsized":"2023-10-09T06:22:20.026290+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:35.161316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":808,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.029532+0000","last_change":"2023-10-09T06:21:37.881638+0000","last_active":"2023-10-09T06:22:21.029532+0000","last_peered":"2023-10-09T06:22:21.029532+0000","last_clean":"2023-10-09T06:22:21.029532+0000","last_became_active":"2023-10-09T06:21:37.881262+0000","last_became_peered":"2023-10-09T06:21:37.881262+0000","last_unstale":"2023-10-09T06:22:21.029532+0000","last_undegraded":"2023-10-09T06:22:21.029532+0000","last_fullsized":"2023-10-09T06:22:21.029532+0000","mapping_epoch":690,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":691,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:07:54.050172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":900,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.031070+0000","last_change":"2023-10-09T06:20:29.965059+0000","last_active":"2023-10-09T06:22:20.031070+0000","last_peered":"2023-10-09T06:22:20.031070+0000","last_clean":"2023-10-09T06:22:20.031070+0000","last_became_active":"2023-10-09T06:20:29.962381+0000","last_became_peered":"2023-10-09T06:20:29.962381+0000","last_unstale":"2023-10-09T06:22:20.031070+0000","last_undegraded":"2023-10-09T06:22:20.031070+0000","last_fullsized":"2023-10-09T06:22:20.031070+0000","mapping_epoch":646,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":647,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:11:45.844892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:11:45.844892+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:08.335928+0000","scrub_duration":0.0017521489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"669'67","reported_seq":486,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.034580+0000","last_change":"2023-10-09T06:21:14.944527+0000","last_active":"2023-10-09T06:22:20.034580+0000","last_peered":"2023-10-09T06:22:20.034580+0000","last_clean":"2023-10-09T06:22:20.034580+0000","last_became_active":"2023-10-09T06:21:14.909285+0000","last_became_peered":"2023-10-09T06:21:14.909285+0000","last_unstale":"2023-10-09T06:22:20.034580+0000","last_undegraded":"2023-10-09T06:22:20.034580+0000","last_fullsized":"2023-10-09T06:22:20.034580+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":67,"log_dups_size":0,"ondisk_log_size":67,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:44.205011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000602131,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":304,"num_read_kb":884,"num_write":61,"num_write_kb":1396,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":44,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.039908+0000","last_change":"2023-10-09T06:21:37.881769+0000","last_active":"2023-10-09T06:22:20.039908+0000","last_peered":"2023-10-09T06:22:20.039908+0000","last_clean":"2023-10-09T06:22:20.039908+0000","last_became_active":"2023-10-09T06:21:37.881393+0000","last_became_peered":"2023-10-09T06:21:37.881393+0000","last_unstale":"2023-10-09T06:22:20.039908+0000","last_undegraded":"2023-10-09T06:22:20.039908+0000","last_fullsized":"2023-10-09T06:22:20.039908+0000","mapping_epoch":690,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":691,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:03.343363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":268,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.027313+0000","last_change":"2023-10-09T06:22:13.707418+0000","last_active":"2023-10-09T06:22:20.027313+0000","last_peered":"2023-10-09T06:22:20.027313+0000","last_clean":"2023-10-09T06:22:20.027313+0000","last_became_active":"2023-10-09T06:22:13.707023+0000","last_became_peered":"2023-10-09T06:22:13.707023+0000","last_unstale":"2023-10-09T06:22:20.027313+0000","last_undegraded":"2023-10-09T06:22:20.027313+0000","last_fullsized":"2023-10-09T06:22:20.027313+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:00.163026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":1297,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.032202+0000","last_change":"2023-10-09T06:21:40.939658+0000","last_active":"2023-10-09T06:22:20.032202+0000","last_peered":"2023-10-09T06:22:20.032202+0000","last_clean":"2023-10-09T06:22:20.032202+0000","last_became_active":"2023-10-09T06:21:40.939238+0000","last_became_peered":"2023-10-09T06:21:40.939238+0000","last_unstale":"2023-10-09T06:22:20.032202+0000","last_undegraded":"2023-10-09T06:22:20.032202+0000","last_fullsized":"2023-10-09T06:22:20.032202+0000","mapping_epoch":693,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":694,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:17.344518+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:17.344518+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:15.256284+0000","scrub_duration":0.0015914099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":770,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.033359+0000","last_change":"2023-10-09T06:13:30.798394+0000","last_active":"2023-10-09T06:22:20.033359+0000","last_peered":"2023-10-09T06:22:20.033359+0000","last_clean":"2023-10-09T06:22:20.033359+0000","last_became_active":"2023-10-09T06:13:30.790913+0000","last_became_peered":"2023-10-09T06:13:30.790913+0000","last_unstale":"2023-10-09T06:22:20.033359+0000","last_undegraded":"2023-10-09T06:22:20.033359+0000","last_fullsized":"2023-10-09T06:22:20.033359+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:26.508690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"670'110","reported_seq":916,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.040170+0000","last_change":"2023-10-09T06:21:34.892140+0000","last_active":"2023-10-09T06:22:20.040170+0000","last_peered":"2023-10-09T06:22:20.040170+0000","last_clean":"2023-10-09T06:22:20.040170+0000","last_became_active":"2023-10-09T06:21:34.891608+0000","last_became_peered":"2023-10-09T06:21:34.891608+0000","last_unstale":"2023-10-09T06:22:20.040170+0000","last_undegraded":"2023-10-09T06:22:20.040170+0000","last_fullsized":"2023-10-09T06:22:20.040170+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:46.255099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043525900000000001,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":561,"num_read_kb":461,"num_write":80,"num_write_kb":4135,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":4194320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":519,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.032947+0000","last_change":"2023-10-09T06:13:30.782038+0000","last_active":"2023-10-09T06:22:20.032947+0000","last_peered":"2023-10-09T06:22:20.032947+0000","last_clean":"2023-10-09T06:22:20.032947+0000","last_became_active":"2023-10-09T06:13:30.781500+0000","last_became_peered":"2023-10-09T06:13:30.781500+0000","last_unstale":"2023-10-09T06:22:20.032947+0000","last_undegraded":"2023-10-09T06:22:20.032947+0000","last_fullsized":"2023-10-09T06:22:20.032947+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:43.663265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":1095,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.027446+0000","last_change":"2023-10-09T06:21:25.906863+0000","last_active":"2023-10-09T06:22:21.027446+0000","last_peered":"2023-10-09T06:22:21.027446+0000","last_clean":"2023-10-09T06:22:21.027446+0000","last_became_active":"2023-10-09T06:21:25.906453+0000","last_became_peered":"2023-10-09T06:21:25.906453+0000","last_unstale":"2023-10-09T06:22:21.027446+0000","last_undegraded":"2023-10-09T06:22:21.027446+0000","last_fullsized":"2023-10-09T06:22:21.027446+0000","mapping_epoch":683,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":684,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T05:58:34.332976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T05:58:34.332976+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:02.748560+0000","scrub_duration":0.0023095530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"669'173","reported_seq":824,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.031884+0000","last_change":"2023-10-09T06:21:22.907807+0000","last_active":"2023-10-09T06:22:20.031884+0000","last_peered":"2023-10-09T06:22:20.031884+0000","last_clean":"2023-10-09T06:22:20.031884+0000","last_became_active":"2023-10-09T06:21:22.907273+0000","last_became_peered":"2023-10-09T06:21:22.907273+0000","last_unstale":"2023-10-09T06:22:20.031884+0000","last_undegraded":"2023-10-09T06:22:20.031884+0000","last_fullsized":"2023-10-09T06:22:20.031884+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:39:17.945207+0000","last_clean_scrub_stamp":"2023-10-09T05:39:17.945207+0000","objects_scrubbed":0,"log_size":173,"log_dups_size":0,"ondisk_log_size":173,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:47.906004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011230579999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1095,"num_read_kb":3133,"num_write":156,"num_write_kb":2191,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":825,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.029756+0000","last_change":"2023-10-09T06:21:31.584018+0000","last_active":"2023-10-09T06:22:21.029756+0000","last_peered":"2023-10-09T06:22:21.029756+0000","last_clean":"2023-10-09T06:22:21.029756+0000","last_became_active":"2023-10-09T06:21:31.568229+0000","last_became_peered":"2023-10-09T06:21:31.568229+0000","last_unstale":"2023-10-09T06:22:21.029756+0000","last_undegraded":"2023-10-09T06:22:21.029756+0000","last_fullsized":"2023-10-09T06:22:21.029756+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:02.652850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":57,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.029251+0000","last_change":"2023-10-09T06:21:22.889114+0000","last_active":"2023-10-09T06:22:20.029251+0000","last_peered":"2023-10-09T06:22:20.029251+0000","last_clean":"2023-10-09T06:22:20.029251+0000","last_became_active":"2023-10-09T06:21:22.888698+0000","last_became_peered":"2023-10-09T06:21:22.888698+0000","last_unstale":"2023-10-09T06:22:20.029251+0000","last_undegraded":"2023-10-09T06:22:20.029251+0000","last_fullsized":"2023-10-09T06:22:20.029251+0000","mapping_epoch":680,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":681,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:41:02.391445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":993,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.033501+0000","last_change":"2023-10-09T06:18:15.860341+0000","last_active":"2023-10-09T06:22:20.033501+0000","last_peered":"2023-10-09T06:22:20.033501+0000","last_clean":"2023-10-09T06:22:20.033501+0000","last_became_active":"2023-10-09T06:18:15.860008+0000","last_became_peered":"2023-10-09T06:18:15.860008+0000","last_unstale":"2023-10-09T06:22:20.033501+0000","last_undegraded":"2023-10-09T06:22:20.033501+0000","last_fullsized":"2023-10-09T06:22:20.033501+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:06.181811+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:06.181811+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:37.378828+0000","scrub_duration":0.011553313000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":669,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.027216+0000","last_change":"2023-10-09T06:21:14.902519+0000","last_active":"2023-10-09T06:22:20.027216+0000","last_peered":"2023-10-09T06:22:20.027216+0000","last_clean":"2023-10-09T06:22:20.027216+0000","last_became_active":"2023-10-09T06:21:14.902152+0000","last_became_peered":"2023-10-09T06:21:14.902152+0000","last_unstale":"2023-10-09T06:22:20.027216+0000","last_undegraded":"2023-10-09T06:22:20.027216+0000","last_fullsized":"2023-10-09T06:22:20.027216+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:56:16.979735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":432,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.038013+0000","last_change":"2023-10-09T06:21:41.018573+0000","last_active":"2023-10-09T06:22:20.038013+0000","last_peered":"2023-10-09T06:22:20.038013+0000","last_clean":"2023-10-09T06:22:20.038013+0000","last_became_active":"2023-10-09T06:21:41.018215+0000","last_became_peered":"2023-10-09T06:21:41.018215+0000","last_unstale":"2023-10-09T06:22:20.038013+0000","last_undegraded":"2023-10-09T06:22:20.038013+0000","last_fullsized":"2023-10-09T06:22:20.038013+0000","mapping_epoch":693,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":694,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:22.031166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":1097,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.034788+0000","last_change":"2023-10-09T06:18:15.867807+0000","last_active":"2023-10-09T06:22:20.034788+0000","last_peered":"2023-10-09T06:22:20.034788+0000","last_clean":"2023-10-09T06:22:20.034788+0000","last_became_active":"2023-10-09T06:18:15.867453+0000","last_became_peered":"2023-10-09T06:18:15.867453+0000","last_unstale":"2023-10-09T06:22:20.034788+0000","last_undegraded":"2023-10-09T06:22:20.034788+0000","last_fullsized":"2023-10-09T06:22:20.034788+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:04.215270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:04.215270+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:50:08.865357+0000","scrub_duration":0.002850652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":36,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.045231+0000","last_change":"2023-10-09T06:22:21.045231+0000","last_active":"2023-10-09T06:22:21.045231+0000","last_peered":"2023-10-09T06:22:21.045231+0000","last_clean":"2023-10-09T06:22:21.045231+0000","last_became_active":"2023-10-09T06:22:21.044873+0000","last_became_peered":"2023-10-09T06:22:21.044873+0000","last_unstale":"2023-10-09T06:22:21.045231+0000","last_undegraded":"2023-10-09T06:22:21.045231+0000","last_fullsized":"2023-10-09T06:22:21.045231+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:55.616563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":57,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.031561+0000","last_change":"2023-10-09T06:21:31.576153+0000","last_active":"2023-10-09T06:22:20.031561+0000","last_peered":"2023-10-09T06:22:20.031561+0000","last_clean":"2023-10-09T06:22:20.031561+0000","last_became_active":"2023-10-09T06:21:31.575732+0000","last_became_peered":"2023-10-09T06:21:31.575732+0000","last_unstale":"2023-10-09T06:22:20.031561+0000","last_undegraded":"2023-10-09T06:22:20.031561+0000","last_fullsized":"2023-10-09T06:22:20.031561+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:01.829052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":36,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.043369+0000","last_change":"2023-10-09T06:22:21.043369+0000","last_active":"2023-10-09T06:22:21.043369+0000","last_peered":"2023-10-09T06:22:21.043369+0000","last_clean":"2023-10-09T06:22:21.043369+0000","last_became_active":"2023-10-09T06:22:21.042847+0000","last_became_peered":"2023-10-09T06:22:21.042847+0000","last_unstale":"2023-10-09T06:22:21.043369+0000","last_undegraded":"2023-10-09T06:22:21.043369+0000","last_fullsized":"2023-10-09T06:22:21.043369+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:24.163212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":47,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.029335+0000","last_change":"2023-10-09T06:21:14.900160+0000","last_active":"2023-10-09T06:22:20.029335+0000","last_peered":"2023-10-09T06:22:20.029335+0000","last_clean":"2023-10-09T06:22:20.029335+0000","last_became_active":"2023-10-09T06:21:14.899186+0000","last_became_peered":"2023-10-09T06:21:14.899186+0000","last_unstale":"2023-10-09T06:22:20.029335+0000","last_undegraded":"2023-10-09T06:22:20.029335+0000","last_fullsized":"2023-10-09T06:22:20.029335+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:55:28.845262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":922,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.035476+0000","last_change":"2023-10-09T06:18:15.861806+0000","last_active":"2023-10-09T06:22:20.035476+0000","last_peered":"2023-10-09T06:22:20.035476+0000","last_clean":"2023-10-09T06:22:20.035476+0000","last_became_active":"2023-10-09T06:18:15.861466+0000","last_became_peered":"2023-10-09T06:18:15.861466+0000","last_unstale":"2023-10-09T06:22:20.035476+0000","last_undegraded":"2023-10-09T06:22:20.035476+0000","last_fullsized":"2023-10-09T06:22:20.035476+0000","mapping_epoch":606,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:00:00.456132+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:00:00.456132+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:05.355171+0000","scrub_duration":0.0014992199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":556,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.034989+0000","last_change":"2023-10-09T06:21:37.884340+0000","last_active":"2023-10-09T06:22:20.034989+0000","last_peered":"2023-10-09T06:22:20.034989+0000","last_clean":"2023-10-09T06:22:20.034989+0000","last_became_active":"2023-10-09T06:21:37.883869+0000","last_became_peered":"2023-10-09T06:21:37.883869+0000","last_unstale":"2023-10-09T06:22:20.034989+0000","last_undegraded":"2023-10-09T06:22:20.034989+0000","last_fullsized":"2023-10-09T06:22:20.034989+0000","mapping_epoch":690,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":691,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:52.554008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":557,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.033770+0000","last_change":"2023-10-09T06:21:19.917691+0000","last_active":"2023-10-09T06:22:20.033770+0000","last_peered":"2023-10-09T06:22:20.033770+0000","last_clean":"2023-10-09T06:22:20.033770+0000","last_became_active":"2023-10-09T06:21:19.917360+0000","last_became_peered":"2023-10-09T06:21:19.917360+0000","last_unstale":"2023-10-09T06:22:20.033770+0000","last_undegraded":"2023-10-09T06:22:20.033770+0000","last_fullsized":"2023-10-09T06:22:20.033770+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:56:35.475967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":934,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.039380+0000","last_change":"2023-10-09T06:21:34.894300+0000","last_active":"2023-10-09T06:22:20.039380+0000","last_peered":"2023-10-09T06:22:20.039380+0000","last_clean":"2023-10-09T06:22:20.039380+0000","last_became_active":"2023-10-09T06:21:34.893948+0000","last_became_peered":"2023-10-09T06:21:34.893948+0000","last_unstale":"2023-10-09T06:22:20.039380+0000","last_undegraded":"2023-10-09T06:22:20.039380+0000","last_fullsized":"2023-10-09T06:22:20.039380+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":299,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T06:02:46.290510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:36:42.354759+0000","last_clean_scrub_stamp":"2023-10-09T06:02:46.290510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:27.496106+0000","scrub_duration":0.002627819,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":828,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.027598+0000","last_change":"2023-10-09T06:21:37.887283+0000","last_active":"2023-10-09T06:22:20.027598+0000","last_peered":"2023-10-09T06:22:20.027598+0000","last_clean":"2023-10-09T06:22:20.027598+0000","last_became_active":"2023-10-09T06:21:37.881566+0000","last_became_peered":"2023-10-09T06:21:37.881566+0000","last_unstale":"2023-10-09T06:22:20.027598+0000","last_undegraded":"2023-10-09T06:22:20.027598+0000","last_fullsized":"2023-10-09T06:22:20.027598+0000","mapping_epoch":690,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":691,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:11.603872+0000","last_clean_scrub_stamp":"2023-10-09T05:40:11.603872+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:36:14.630943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":57,"reported_epoch":709,"state":"active+clean","last_fresh":"2023-10-09T06:22:20.029109+0000","last_change":"2023-10-09T06:21:31.572882+0000","last_active":"2023-10-09T06:22:20.029109+0000","last_peered":"2023-10-09T06:22:20.029109+0000","last_clean":"2023-10-09T06:22:20.029109+0000","last_became_active":"2023-10-09T06:21:31.572511+0000","last_became_peered":"2023-10-09T06:21:31.572511+0000","last_unstale":"2023-10-09T06:22:20.029109+0000","last_undegraded":"2023-10-09T06:22:20.029109+0000","last_fullsized":"2023-10-09T06:22:20.029109+0000","mapping_epoch":685,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:40:12.612901+0000","last_clean_scrub_stamp":"2023-10-09T05:40:12.612901+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:24:13.408389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":122,"num_bytes_recovered":16344,"num_keys_recovered":90,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13410,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2886848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5962,"num_read_kb":13177,"num_write":847,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":4204579,"num_keys_recovered":4,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234520,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1188420,"kb_used_data":7420,"kb_used_omap":0,"kb_used_meta":1180992,"kb_avail":92556732,"statfs":{"total":95995035648,"available":94778093568,"internally_reserved":0,"allocated":7598080,"data_stored":5840641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1209335808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.98499999999999999,"15min":3.7280000000000002},"min":{"1min":0.34999999999999998,"5min":0.221,"15min":0.19500000000000001},"max":{"1min":0.81499999999999995,"5min":5.4480000000000004,"15min":166.94},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.55900000000000005,"5min":1.145,"15min":3.907},"min":{"1min":0.38700000000000001,"5min":0.26900000000000002,"15min":0.23799999999999999},"max":{"1min":0.82299999999999995,"5min":6.2549999999999999,"15min":166.923},"last":0.77600000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":6.3869999999999996,"15min":5.0339999999999998},"min":{"1min":0.38100000000000001,"5min":0.25600000000000001,"15min":0.218},"max":{"1min":0.749,"5min":129.17500000000001,"15min":129.17500000000001},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.59399999999999997,"5min":6.3849999999999998,"15min":4.9779999999999998},"min":{"1min":0.35899999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.79500000000000004,"5min":129.18700000000001,"15min":129.18700000000001},"last":0.88}]},{"osd":2,"last update":"Mon Oct 9 06:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.95799999999999996,"15min":1.6220000000000001},"min":{"1min":0.43099999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.755,"5min":5.0810000000000004,"15min":47.503},"last":0.90800000000000003},{"interface":"front","average":{"1min":0.58099999999999996,"5min":1.0720000000000001,"15min":1.675},"min":{"1min":0.38700000000000001,"5min":0.312,"15min":0.27000000000000002},"max":{"1min":0.70499999999999996,"5min":9.5600000000000005,"15min":47.545000000000002},"last":0.82899999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":4.399,"15min":5.032},"min":{"1min":0.40799999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.79400000000000004,"5min":184.804,"15min":184.804},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.65000000000000002,"5min":4.2270000000000003,"15min":4.976},"min":{"1min":0.46999999999999997,"5min":0.32600000000000001,"15min":0.32100000000000001},"max":{"1min":0.91600000000000004,"5min":186.86099999999999,"15min":186.86099999999999},"last":0.96599999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:22:00 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":1.8919999999999999,"15min":1.385},"min":{"1min":0.40699999999999997,"5min":0.27600000000000002,"15min":0.23100000000000001},"max":{"1min":0.75800000000000001,"5min":67.186999999999998,"15min":67.186999999999998},"last":0.86399999999999999},{"interface":"front","average":{"1min":0.58899999999999997,"5min":1.9390000000000001,"15min":1.381},"min":{"1min":0.34699999999999998,"5min":0.254,"15min":0.246},"max":{"1min":0.77400000000000002,"5min":67.218000000000004,"15min":67.218000000000004},"last":0.624}]},{"osd":5,"last update":"Mon Oct 9 06:22:39 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":1.1479999999999999,"15min":1.6439999999999999},"min":{"1min":0.48899999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.83799999999999997,"5min":12.951000000000001,"15min":12.951000000000001},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.629,"5min":0.995,"15min":1.3169999999999999},"min":{"1min":0.39200000000000002,"5min":0.307,"15min":0.307},"max":{"1min":0.80000000000000004,"5min":5.0369999999999999,"15min":5.0369999999999999},"last":0.80400000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:22:11 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":1.2130000000000001,"15min":1.796},"min":{"1min":0.46800000000000003,"5min":0.311,"15min":0.26600000000000001},"max":{"1min":0.82999999999999996,"5min":9.2599999999999998,"15min":233.56700000000001},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.70199999999999996,"5min":0.97599999999999998,"15min":1.7230000000000001},"min":{"1min":0.48499999999999999,"5min":0.33000000000000002,"15min":0.30599999999999999},"max":{"1min":0.875,"5min":5.0629999999999997,"15min":232.477},"last":0.93999999999999995}]}]},{"osd":6,"up_from":46,"seq":197568496157,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1209780,"kb_used_data":11564,"kb_used_omap":0,"kb_used_meta":1198208,"kb_avail":92535372,"statfs":{"total":95995035648,"available":94756220928,"internally_reserved":0,"allocated":11841536,"data_stored":10099545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1226964992},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:39 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.669,"15min":4.3730000000000002},"min":{"1min":0.33900000000000002,"5min":0.222,"15min":0.21099999999999999},"max":{"1min":0.73499999999999999,"5min":85.311000000000007,"15min":191.50800000000001},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.56499999999999995,"5min":1.976,"15min":4.5960000000000001},"min":{"1min":0.40799999999999997,"5min":0.27100000000000002,"15min":0.24399999999999999},"max":{"1min":0.74399999999999999,"5min":85.322999999999993,"15min":191.52600000000001},"last":0.53800000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:22:39 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":4.7640000000000002,"15min":5.6420000000000003},"min":{"1min":0.27700000000000002,"5min":0.25,"15min":0.23200000000000001},"max":{"1min":0.66800000000000004,"5min":183.458,"15min":328.04399999999998},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":4.6020000000000003,"15min":5.3570000000000002},"min":{"1min":0.38800000000000001,"5min":0.28499999999999998,"15min":0.253},"max":{"1min":0.82599999999999996,"5min":183.51499999999999,"15min":328.10000000000002},"last":0.64900000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:22:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":1.0960000000000001,"15min":2.4470000000000001},"min":{"1min":0.38300000000000001,"5min":0.29099999999999998,"15min":0.26100000000000001},"max":{"1min":0.69699999999999995,"5min":9.1500000000000004,"15min":163.04900000000001},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.98099999999999998,"15min":2.4340000000000002},"min":{"1min":0.374,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.89400000000000002,"5min":12.083,"15min":162.93000000000001},"last":0.502}]},{"osd":3,"last update":"Mon Oct 9 06:22:26 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":5.7350000000000003,"15min":6.048},"min":{"1min":0.37,"5min":0.314,"15min":0.29199999999999998},"max":{"1min":0.72699999999999998,"5min":208.494,"15min":250.744},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.64300000000000002,"5min":5.9189999999999996,"15min":5.923},"min":{"1min":0.51000000000000001,"5min":0.33200000000000002,"15min":0.309},"max":{"1min":0.92200000000000004,"5min":211.661,"15min":250.83799999999999},"last":0.56299999999999994}]},{"osd":4,"last update":"Mon Oct 9 06:22:04 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.68899999999999995,"15min":1.341},"min":{"1min":0.38,"5min":0.26800000000000002,"15min":0.24399999999999999},"max":{"1min":0.91400000000000003,"5min":6.0979999999999999,"15min":151.35599999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.503,"5min":1.173,"15min":1.6599999999999999},"min":{"1min":0.36899999999999999,"5min":0.27700000000000002,"15min":0.23999999999999999},"max":{"1min":0.75700000000000001,"5min":14.009,"15min":149.15799999999999},"last":0.47199999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:22:39 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":1.075,"15min":1.554},"min":{"1min":0.40300000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.76100000000000001,"5min":16.032,"15min":16.032},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.81999999999999995,"15min":1.0089999999999999},"min":{"1min":0.46000000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.95499999999999996,"5min":5.8090000000000002,"15min":5.8090000000000002},"last":0.67900000000000005}]},{"osd":7,"last update":"Mon Oct 9 06:22:04 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":1.248,"15min":1.157},"min":{"1min":0.34499999999999997,"5min":0.32700000000000001,"15min":0.30299999999999999},"max":{"1min":0.70499999999999996,"5min":17.803999999999998,"15min":17.803999999999998},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.091,"15min":1.044},"min":{"1min":0.42599999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.85199999999999998,"5min":16.119,"15min":16.119},"last":0.57799999999999996}]}]},{"osd":1,"up_from":89,"seq":382252089745,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":860240,"kb_used_data":13576,"kb_used_omap":9,"kb_used_meta":846646,"kb_avail":92884912,"statfs":{"total":95995035648,"available":95114149888,"internally_reserved":0,"allocated":13901824,"data_stored":10032213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":866966505},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:00 2023","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":1.121,"15min":2.419},"min":{"1min":0.21099999999999999,"5min":0.187,"15min":0.187},"max":{"1min":0.59799999999999998,"5min":9.6899999999999995,"15min":118.17},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.33000000000000002,"5min":1.2,"15min":2.641},"min":{"1min":0.25800000000000001,"5min":0.23200000000000001,"15min":0.20200000000000001},"max":{"1min":0.63200000000000001,"5min":8.5410000000000004,"15min":117.634},"last":0.26700000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:22:00 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":1.4299999999999999,"15min":1.6279999999999999},"min":{"1min":0.252,"5min":0.247,"15min":0.21099999999999999},"max":{"1min":0.64700000000000002,"5min":19.943999999999999,"15min":19.943999999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.42399999999999999,"5min":1.157,"15min":1.8420000000000001},"min":{"1min":0.29199999999999998,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":1.0229999999999999,"5min":8.5939999999999994,"15min":55.694000000000003},"last":0.38400000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:22:29 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":3.2210000000000001,"15min":4.0990000000000002},"min":{"1min":0.32100000000000001,"5min":0.252,"15min":0.24199999999999999},"max":{"1min":0.64800000000000002,"5min":73.957999999999998,"15min":113.322},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.438,"5min":2.9209999999999998,"15min":3.9620000000000002},"min":{"1min":0.33600000000000002,"5min":0.28199999999999997,"15min":0.21299999999999999},"max":{"1min":0.63500000000000001,"5min":74.709000000000003,"15min":113.38200000000001},"last":0.35599999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:22:00 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":4.4930000000000003,"15min":3.4990000000000001},"min":{"1min":0.42499999999999999,"5min":0.29799999999999999,"15min":0.27700000000000002},"max":{"1min":0.78000000000000003,"5min":190.523,"15min":190.523},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.624,"5min":4.7039999999999997,"15min":4.0069999999999997},"min":{"1min":0.41399999999999998,"5min":0.36299999999999999,"15min":0.308},"max":{"1min":0.96499999999999997,"5min":190.774,"15min":190.774},"last":0.752}]},{"osd":5,"last update":"Mon Oct 9 06:22:38 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":1.2969999999999999,"15min":1.419},"min":{"1min":0.46999999999999997,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.214,"5min":11.398999999999999,"15min":11.398999999999999},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.628,"5min":1.0669999999999999,"15min":1.2370000000000001},"min":{"1min":0.40100000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.87,"5min":8.4320000000000004,"15min":8.4320000000000004},"last":0.61399999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:22:12 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":2.3460000000000001,"15min":2.0139999999999998},"min":{"1min":0.39100000000000001,"5min":0.38200000000000001,"15min":0.33700000000000002},"max":{"1min":0.93700000000000006,"5min":98.433000000000007,"15min":98.433000000000007},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.63900000000000001,"5min":2.3730000000000002,"15min":2.4460000000000002},"min":{"1min":0.40500000000000003,"5min":0.377,"15min":0.34999999999999998},"max":{"1min":0.91200000000000003,"5min":98.456000000000003,"15min":98.456000000000003},"last":0.59199999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:22:12 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":4.298,"15min":3.2530000000000001},"min":{"1min":0.437,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.77300000000000002,"5min":154.34800000000001,"15min":154.34800000000001},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.64400000000000002,"5min":4.0670000000000002,"15min":2.7559999999999998},"min":{"1min":0.45100000000000001,"5min":0.40100000000000002,"15min":0.38},"max":{"1min":0.85999999999999999,"5min":152.20599999999999,"15min":152.20599999999999},"last":0.68500000000000005}]}]},{"osd":0,"up_from":456,"seq":1958505087139,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":454976,"kb_used_data":9528,"kb_used_omap":32,"kb_used_meta":445407,"kb_avail":93290176,"statfs":{"total":95995035648,"available":95529140224,"internally_reserved":0,"allocated":9756672,"data_stored":6141356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32794,"internal_metadata":456097766},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:21:51 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":4.1900000000000004,"15min":6.085},"min":{"1min":0.21199999999999999,"5min":0.189,"15min":0.17299999999999999},"max":{"1min":0.46600000000000003,"5min":137.452,"15min":166.833},"last":0.245},{"interface":"front","average":{"1min":0.34499999999999997,"5min":4.0629999999999997,"15min":5.9550000000000001},"min":{"1min":0.28199999999999997,"5min":0.23999999999999999,"15min":0.19700000000000001},"max":{"1min":0.42199999999999999,"5min":137.10499999999999,"15min":166.81100000000001},"last":0.32700000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:21:51 2023","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":1.2829999999999999,"15min":2.3239999999999998},"min":{"1min":0.248,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.50600000000000001,"5min":13.071,"15min":21.648},"last":0.443},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.91900000000000004,"15min":2.109},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.48299999999999998,"5min":6.173,"15min":20.152000000000001},"last":0.40400000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:21:51 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":4.609,"15min":6.649},"min":{"1min":0.28299999999999997,"5min":0.214,"15min":0.214},"max":{"1min":0.78100000000000003,"5min":138.52699999999999,"15min":203.59299999999999},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.44400000000000001,"5min":4.8040000000000003,"15min":6.6269999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.23899999999999999},"max":{"1min":0.79600000000000004,"5min":138.48500000000001,"15min":202.96600000000001},"last":0.374}]},{"osd":4,"last update":"Mon Oct 9 06:21:51 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":1.6870000000000001,"15min":2.0049999999999999},"min":{"1min":0.34799999999999998,"5min":0.313,"15min":0.27800000000000002},"max":{"1min":0.80200000000000005,"5min":60.587000000000003,"15min":60.587000000000003},"last":0.77200000000000002},{"interface":"front","average":{"1min":0.625,"5min":1.7410000000000001,"15min":2.097},"min":{"1min":0.35299999999999998,"5min":0.32400000000000001,"15min":0.254},"max":{"1min":0.82599999999999996,"5min":60.567,"15min":60.567},"last":0.80000000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:22:43 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":1.274,"15min":1.8260000000000001},"min":{"1min":0.38700000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":3.6070000000000002,"5min":8.048,"15min":8.048},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.268,"15min":1.6599999999999999},"min":{"1min":0.45800000000000002,"5min":0.376,"15min":0.376},"max":{"1min":0.77900000000000003,"5min":13.007,"15min":13.007},"last":0.72599999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:21:51 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":1.4330000000000001,"15min":2.0419999999999998},"min":{"1min":0.36599999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.88500000000000001,"5min":13.029999999999999,"15min":31.648},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.626,"5min":1.0960000000000001,"15min":1.823},"min":{"1min":0.432,"5min":0.36499999999999999,"15min":0.36299999999999999},"max":{"1min":0.81999999999999995,"5min":7.5640000000000001,"15min":24.43},"last":0.66800000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:21:51 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":5.3300000000000001,"15min":3.694},"min":{"1min":0.435,"5min":0.376,"15min":0.376},"max":{"1min":0.85099999999999998,"5min":248.99100000000001,"15min":248.99100000000001},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.66600000000000004,"5min":5.1849999999999996,"15min":3.5350000000000001},"min":{"1min":0.52500000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.78000000000000003,"5min":249.00800000000001,"15min":249.00800000000001},"last":0.64400000000000002}]}]},{"osd":2,"up_from":94,"seq":403726926236,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":792360,"kb_used_data":9248,"kb_used_omap":81,"kb_used_meta":783022,"kb_avail":92952792,"statfs":{"total":95995035648,"available":95183659008,"internally_reserved":0,"allocated":9469952,"data_stored":6050318,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":801814746},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:06 2023","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":1.8600000000000001,"15min":3.0840000000000001},"min":{"1min":0.23799999999999999,"5min":0.22900000000000001,"15min":0.17299999999999999},"max":{"1min":0.68600000000000005,"5min":70.218000000000004,"15min":83.652000000000001},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.40000000000000002,"5min":1.718,"15min":2.488},"min":{"1min":0.27000000000000002,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.66000000000000003,"5min":69.968000000000004,"15min":77.578999999999994},"last":0.34200000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:22:06 2023","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":2.0579999999999998,"15min":3.5529999999999999},"min":{"1min":0.29099999999999998,"5min":0.251,"15min":0.19400000000000001},"max":{"1min":0.70599999999999996,"5min":59.331000000000003,"15min":160.45500000000001},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.379,"5min":2.714,"15min":4.3040000000000003},"min":{"1min":0.26000000000000001,"5min":0.22600000000000001,"15min":0.20200000000000001},"max":{"1min":0.71999999999999997,"5min":56.508000000000003,"15min":163.697},"last":0.35999999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:22:19 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":1.7929999999999999,"15min":4.3419999999999996},"min":{"1min":0.29999999999999999,"5min":0.26500000000000001,"15min":0.23000000000000001},"max":{"1min":0.73399999999999999,"5min":43.652999999999999,"15min":122.205},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.44500000000000001,"5min":2.1600000000000001,"15min":4.3890000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.252},"max":{"1min":0.747,"5min":46.792999999999999,"15min":122.45},"last":0.38500000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:22:06 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":2.3180000000000001,"15min":2.4470000000000001},"min":{"1min":0.376,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":0.94099999999999995,"5min":98.644999999999996,"15min":107.51300000000001},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.66200000000000003,"5min":1.9610000000000001,"15min":1.968},"min":{"1min":0.41999999999999998,"5min":0.315,"15min":0.315},"max":{"1min":1.01,"5min":98.668000000000006,"15min":108.768},"last":0.52500000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:22:45 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":1.1180000000000001,"15min":1.534},"min":{"1min":0.47399999999999998,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.93000000000000005,"5min":6.4249999999999998,"15min":6.4249999999999998},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.64000000000000001,"5min":1.407,"15min":2.141},"min":{"1min":0.42399999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.86399999999999999,"5min":18.617000000000001,"15min":18.617000000000001},"last":0.66800000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:22:06 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":1.389,"15min":1.7529999999999999},"min":{"1min":0.42599999999999999,"5min":0.39500000000000002,"15min":0.32000000000000001},"max":{"1min":0.91100000000000003,"5min":7.9720000000000004,"15min":14.962},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.68899999999999995,"5min":1.252,"15min":1.504},"min":{"1min":0.497,"5min":0.38300000000000001,"15min":0.34399999999999997},"max":{"1min":0.995,"5min":6.4400000000000004,"15min":11.103},"last":0.63}]},{"osd":7,"last update":"Mon Oct 9 06:22:06 2023","interfaces":[{"interface":"back","average":{"1min":0.68200000000000005,"5min":4.1459999999999999,"15min":2.9489999999999998},"min":{"1min":0.496,"5min":0.40300000000000002,"15min":0.313},"max":{"1min":0.92200000000000004,"5min":234.26499999999999,"15min":234.26499999999999},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.66800000000000004,"5min":4.0250000000000004,"15min":2.7440000000000002},"min":{"1min":0.52200000000000002,"5min":0.375,"15min":0.33900000000000002},"max":{"1min":1.034,"5min":233.02099999999999,"15min":233.02099999999999},"last":0.64400000000000002}]}]},{"osd":3,"up_from":99,"seq":425201762709,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1126204,"kb_used_data":9780,"kb_used_omap":7,"kb_used_meta":1116408,"kb_avail":92618948,"statfs":{"total":95995035648,"available":94841802752,"internally_reserved":0,"allocated":10014720,"data_stored":6158196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1143202408},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:50 2023","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":2.1040000000000001,"15min":3.4329999999999998},"min":{"1min":0.245,"5min":0.182,"15min":0.17299999999999999},"max":{"1min":0.89700000000000002,"5min":117.473,"15min":220.45400000000001},"last":0.245},{"interface":"front","average":{"1min":0.41299999999999998,"5min":1.766,"15min":4.0019999999999998},"min":{"1min":0.28199999999999997,"5min":0.219,"15min":0.19600000000000001},"max":{"1min":1.133,"5min":118.873,"15min":216.44200000000001},"last":0.33000000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":3.294,"15min":2.6520000000000001},"min":{"1min":0.29399999999999998,"5min":0.22800000000000001,"15min":0.19800000000000001},"max":{"1min":0.51300000000000001,"5min":167.768,"15min":167.768},"last":0.309},{"interface":"front","average":{"1min":0.39100000000000001,"5min":3.1400000000000001,"15min":3.3119999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.218},"max":{"1min":0.71399999999999997,"5min":172.143,"15min":172.143},"last":0.26400000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":1.125,"15min":1.6990000000000001},"min":{"1min":0.29299999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.59699999999999998,"5min":8.5280000000000005,"15min":30.547999999999998},"last":0.38},{"interface":"front","average":{"1min":0.38800000000000001,"5min":1.415,"15min":2.2690000000000001},"min":{"1min":0.34100000000000003,"5min":0.26500000000000001,"15min":0.23899999999999999},"max":{"1min":0.44800000000000001,"5min":10.981,"15min":25.984000000000002},"last":0.371}]},{"osd":4,"last update":"Mon Oct 9 06:22:08 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":2.794,"15min":2.3889999999999998},"min":{"1min":0.45100000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.82499999999999996,"5min":84.369,"15min":95.739000000000004},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.621,"5min":3.0750000000000002,"15min":2.7320000000000002},"min":{"1min":0.46800000000000003,"5min":0.34499999999999997,"15min":0.28899999999999998},"max":{"1min":0.83499999999999996,"5min":87.176000000000002,"15min":97.941999999999993},"last":0.53800000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:22:41 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":1.2789999999999999,"15min":1.633},"min":{"1min":0.42899999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.0800000000000001,"5min":7.8550000000000004,"15min":7.8550000000000004},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.70099999999999996,"5min":1.252,"15min":1.6910000000000001},"min":{"1min":0.52400000000000002,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.2150000000000001,"5min":7.1719999999999997,"15min":7.1719999999999997},"last":0.59499999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":1.542,"15min":1.6379999999999999},"min":{"1min":0.50900000000000001,"5min":0.32600000000000001,"15min":0.26100000000000001},"max":{"1min":0.85599999999999998,"5min":11.183,"15min":49.526000000000003},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.63900000000000001,"5min":1.4419999999999999,"15min":1.9590000000000001},"min":{"1min":0.46500000000000002,"5min":0.39100000000000001,"15min":0.33600000000000002},"max":{"1min":0.83299999999999996,"5min":9.5809999999999995,"15min":49.429000000000002},"last":0.44600000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":2.3319999999999999,"15min":2.9169999999999998},"min":{"1min":0.46600000000000003,"5min":0.36599999999999999,"15min":0.35699999999999998},"max":{"1min":0.84699999999999998,"5min":77.525000000000006,"15min":94.400000000000006},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.626,"5min":2.3559999999999999,"15min":3.3079999999999998},"min":{"1min":0.47899999999999998,"5min":0.40000000000000002,"15min":0.32600000000000001},"max":{"1min":0.81100000000000005,"5min":77.438000000000002,"15min":94.195999999999998},"last":0.502}]}]},{"osd":4,"up_from":322,"seq":1382979469572,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":930112,"kb_used_data":11256,"kb_used_omap":35,"kb_used_meta":918812,"kb_avail":92815040,"statfs":{"total":95995035648,"available":95042600960,"internally_reserved":0,"allocated":11526144,"data_stored":9772510,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36552,"internal_metadata":940863800},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":1.0800000000000001,"15min":1.786},"min":{"1min":0.34499999999999997,"5min":0.23100000000000001,"15min":0.216},"max":{"1min":0.78100000000000003,"5min":18.510999999999999,"15min":81.066000000000003},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.53700000000000003,"5min":1.103,"15min":1.7929999999999999},"min":{"1min":0.379,"5min":0.27800000000000002,"15min":0.22800000000000001},"max":{"1min":0.67100000000000004,"5min":10.695,"15min":81.603999999999999},"last":0.41899999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:22:07 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":5.0700000000000003,"15min":3.528},"min":{"1min":0.36099999999999999,"5min":0.23899999999999999,"15min":0.20999999999999999},"max":{"1min":0.75800000000000001,"5min":160.35400000000001,"15min":160.35400000000001},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.53500000000000003,"5min":5.117,"15min":3.5920000000000001},"min":{"1min":0.377,"5min":0.26500000000000001,"15min":0.22800000000000001},"max":{"1min":0.77100000000000002,"5min":160.333,"15min":160.333},"last":0.5}]},{"osd":2,"last update":"Mon Oct 9 06:22:07 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":1.054,"15min":2.3900000000000001},"min":{"1min":0.41999999999999998,"5min":0.29999999999999999,"15min":0.27200000000000002},"max":{"1min":0.748,"5min":9.9540000000000006,"15min":245.166},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":1.986,"15min":2.569},"min":{"1min":0.39200000000000002,"5min":0.33300000000000002,"15min":0.27900000000000003},"max":{"1min":0.71299999999999997,"5min":52.920999999999999,"15min":245.119},"last":0.55400000000000005}]},{"osd":3,"last update":"Mon Oct 9 06:22:07 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":5.3499999999999996,"15min":5.7389999999999999},"min":{"1min":0.40799999999999997,"5min":0.29299999999999998,"15min":0.26800000000000002},"max":{"1min":0.77200000000000002,"5min":164.89699999999999,"15min":246.39400000000001},"last":0.44},{"interface":"front","average":{"1min":0.64100000000000001,"5min":4.9950000000000001,"15min":5.4279999999999999},"min":{"1min":0.44600000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.95099999999999996,"5min":164.88300000000001,"15min":246.99000000000001},"last":0.54100000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:22:39 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":1.22,"15min":1.766},"min":{"1min":0.34000000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.72199999999999998,"5min":10.659000000000001,"15min":10.659000000000001},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.58599999999999997,"5min":1.1839999999999999,"15min":1.7010000000000001},"min":{"1min":0.36399999999999999,"5min":0.317,"15min":0.317},"max":{"1min":0.84499999999999997,"5min":11.779,"15min":11.779},"last":0.48499999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:22:07 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":1.2649999999999999,"15min":1.444},"min":{"1min":0.40300000000000002,"5min":0.26800000000000002,"15min":0.26400000000000001},"max":{"1min":0.82499999999999996,"5min":23.712,"15min":95.492999999999995},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.625,"5min":1.2809999999999999,"15min":1.54},"min":{"1min":0.45600000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.875,"5min":18.021000000000001,"15min":95.664000000000001},"last":0.52600000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:22:07 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":1.7889999999999999,"15min":1.8240000000000001},"min":{"1min":0.41399999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.84599999999999997,"5min":61.811999999999998,"15min":95.167000000000002},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.64900000000000002,"5min":1.702,"15min":1.744},"min":{"1min":0.41899999999999998,"5min":0.28799999999999998,"15min":0.251},"max":{"1min":0.90700000000000003,"5min":61.759,"15min":94.811999999999998},"last":0.59399999999999997}]}]},{"osd":5,"up_from":611,"seq":2624225017910,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":368632,"kb_used_data":7408,"kb_used_omap":98,"kb_used_meta":361117,"kb_avail":93376520,"statfs":{"total":95995035648,"available":95617556480,"internally_reserved":0,"allocated":7585792,"data_stored":5836657,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":100797,"internal_metadata":369784387},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:35 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.86499999999999999,"15min":1.163},"min":{"1min":0.33100000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.68400000000000005,"5min":8.3260000000000005,"15min":8.3260000000000005},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.56100000000000005,"5min":1.0600000000000001,"15min":1.474},"min":{"1min":0.34200000000000003,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.747,"5min":7.577,"15min":7.577},"last":0.56000000000000005}]},{"osd":1,"last update":"Mon Oct 9 06:22:35 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":2.3999999999999999,"15min":4.1189999999999998},"min":{"1min":0.39600000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.002,"5min":50.430999999999997,"15min":50.430999999999997},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":1.873,"15min":3.0430000000000001},"min":{"1min":0.38100000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.97499999999999998,"5min":50.445,"15min":50.445},"last":0.58399999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:22:35 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.95899999999999996,"15min":1.2330000000000001},"min":{"1min":0.377,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.73399999999999999,"5min":5.9980000000000002,"15min":5.9980000000000002},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":1.0049999999999999,"15min":1.3089999999999999},"min":{"1min":0.33600000000000002,"5min":0.307,"15min":0.307},"max":{"1min":0.90700000000000003,"5min":5.9560000000000004,"15min":5.9560000000000004},"last":0.46999999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:22:35 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":4.6989999999999998,"15min":8.2159999999999993},"min":{"1min":0.47899999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.94299999999999995,"5min":120.357,"15min":120.357},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.63400000000000001,"5min":4.2670000000000003,"15min":7.7300000000000004},"min":{"1min":0.49199999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.083,"5min":120.759,"15min":120.759},"last":0.50700000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:22:35 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.72699999999999998,"15min":0.91700000000000004},"min":{"1min":0.29299999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.0209999999999999,"5min":5.6890000000000001,"15min":5.6890000000000001},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.60399999999999998,"5min":1.087,"15min":1.5189999999999999},"min":{"1min":0.436,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.0409999999999999,"5min":7.6130000000000004,"15min":7.6130000000000004},"last":0.73299999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:22:35 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.61699999999999999,"15min":0.70899999999999996},"min":{"1min":0.36499999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.1359999999999999,"5min":5.9400000000000004,"15min":5.9400000000000004},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.622,"5min":0.76700000000000002,"15min":0.92000000000000004},"min":{"1min":0.443,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.1060000000000001,"5min":5.8120000000000003,"15min":5.8120000000000003},"last":0.63300000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:22:35 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":1.0349999999999999,"15min":1.3779999999999999},"min":{"1min":0.38800000000000001,"5min":0.317,"15min":0.317},"max":{"1min":1.0720000000000001,"5min":7.6120000000000001,"15min":7.6120000000000001},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.91400000000000003,"15min":1.153},"min":{"1min":0.505,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.1859999999999999,"5min":5.8920000000000003,"15min":5.8920000000000003},"last":0.66800000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3946,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4889,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:22:57.552 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T06:22:57.552 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-09T06:22:57.630 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=e368a226f9c246a2b7dd2a80d5aa3389 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-09T06:22:57.633 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-10-09T06:22:57.644 INFO:tasks.cephadm:Teardown begin 2023-10-09T06:22:57.644 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T06:22:57.715 DEBUG:teuthology.orchestra.run.smithi156:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T06:22:57.790 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-10-09T06:22:57.791 DEBUG:teuthology.orchestra.run.smithi064:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T06:22:57.809 DEBUG:teuthology.orchestra.run.smithi156:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T06:22:57.828 INFO:tasks.cephadm:Stopping all daemons... 2023-10-09T06:22:57.828 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-10-09T06:22:57.829 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mon.a 2023-10-09T06:22:58.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:22:57 smithi156 ceph-mon[195101]: from='client.48008 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:22:58.138 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:57 smithi064 systemd[1]: Stopping Ceph mon.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T06:22:58.138 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:57 smithi064 ceph-mon[212750]: from='client.48008 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:22:58.139 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:57 smithi064 ceph-mon[220044]: from='client.48008 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:22:58.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:58 smithi064 conmon[212726]: 2023-10-09T06:22:58.137+0000 7f882972e700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:22:58.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:58 smithi064 conmon[212726]: 2023-10-09T06:22:58.137+0000 7f882972e700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-10-09T06:22:58.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Oct 09 06:22:58 smithi064 bash[725131]: 5d3854b1b1eb71aa2bb759f4396c9cf41c8f6c54083f19a840cd320cab1b52e2 2023-10-09T06:22:59.205 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mon.a.service' 2023-10-09T06:22:59.247 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:22:59.247 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-10-09T06:22:59.247 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-10-09T06:22:59.248 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mon.c 2023-10-09T06:22:59.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:59 smithi064 systemd[1]: Stopping Ceph mon.c for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T06:22:59.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:59 smithi064 conmon[220020]: 2023-10-09T06:22:59.553+0000 7f2ff14dd700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:22:59.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Oct 09 06:22:59 smithi064 conmon[220020]: 2023-10-09T06:22:59.553+0000 7f2ff14dd700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-10-09T06:22:59.913 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:22:59 smithi064 conmon[198859]: [09/Oct/2023:06:22:59] ENGINE Bus STOPPING 2023-10-09T06:22:59.913 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:22:59 smithi064 conmon[198859]: [09/Oct/2023:06:22:59] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T06:22:59.913 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:22:59 smithi064 conmon[198859]: [09/Oct/2023:06:22:59] ENGINE Bus STOPPED 2023-10-09T06:22:59.914 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:22:59 smithi064 conmon[198859]: [09/Oct/2023:06:22:59] ENGINE Bus STARTING 2023-10-09T06:22:59.914 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:22:59 smithi064 conmon[198859]: [09/Oct/2023:06:22:59] ENGINE Serving on http://:::9283 2023-10-09T06:22:59.914 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:22:59 smithi064 conmon[198859]: [09/Oct/2023:06:22:59] ENGINE Bus STARTED 2023-10-09T06:23:00.613 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mon.c.service' 2023-10-09T06:23:00.651 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:23:00.651 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-10-09T06:23:00.652 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-10-09T06:23:00.652 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mon.b 2023-10-09T06:23:01.120 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:23:00 smithi156 systemd[1]: Stopping Ceph mon.b for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T06:23:01.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:23:00 smithi156 conmon[195077]: 2023-10-09T06:23:00.909+0000 7f4c9cc83700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:23:01.121 INFO:journalctl@ceph.mon.b.smithi156.stdout:Oct 09 06:23:00 smithi156 conmon[195077]: 2023-10-09T06:23:00.909+0000 7f4c9cc83700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-10-09T06:23:01.830 DEBUG:teuthology.orchestra.run.smithi156:> sudo pkill -f 'journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mon.b.service' 2023-10-09T06:23:01.870 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:23:01.870 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-10-09T06:23:01.870 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-10-09T06:23:01.871 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mgr.y 2023-10-09T06:23:02.169 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:23:01 smithi064 systemd[1]: Stopping Ceph mgr.y for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T06:23:02.469 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Oct 09 06:23:02 smithi064 conmon[198859]: 2023-10-09T06:23:02.169+0000 7fcbc2622700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:23:03.027 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mgr.y.service' 2023-10-09T06:23:03.067 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:23:03.068 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-10-09T06:23:03.068 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-10-09T06:23:03.068 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mgr.x 2023-10-09T06:23:03.370 INFO:journalctl@ceph.mgr.x.smithi156.stdout:Oct 09 06:23:03 smithi156 systemd[1]: Stopping Ceph mgr.x for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T06:23:04.164 DEBUG:teuthology.orchestra.run.smithi156:> sudo pkill -f 'journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@mgr.x.service' 2023-10-09T06:23:04.202 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:23:04.202 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-10-09T06:23:04.203 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-10-09T06:23:04.203 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.0 2023-10-09T06:23:04.486 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 06:23:04 smithi064 systemd[1]: Stopping Ceph osd.0 for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T06:23:04.969 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 06:23:04 smithi064 conmon[539142]: 2023-10-09T06:23:04.486+0000 7f54e5cf6700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:23:04.969 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 06:23:04 smithi064 conmon[539142]: 2023-10-09T06:23:04.486+0000 7f54e5cf6700 -1 osd.0 710 *** Got signal Terminated *** 2023-10-09T06:23:04.969 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 06:23:04 smithi064 conmon[539142]: 2023-10-09T06:23:04.486+0000 7f54e5cf6700 -1 osd.0 710 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:23:05.370 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 06:23:05 smithi156 conmon[175419]: logger=cleanup t=2023-10-09T06:23:05.024893549Z level=info msg="Completed cleanup jobs" duration=66.649474ms 2023-10-09T06:23:10.969 INFO:journalctl@ceph.osd.0.smithi064.stdout:Oct 09 06:23:10 smithi064 bash[726033]: 631dc6265ff12bb31f544d5b252dfc4348cbd3fe94f57fa1706642287acf3b11 2023-10-09T06:23:12.493 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.0.service' 2023-10-09T06:23:12.534 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:23:12.535 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-10-09T06:23:12.535 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-10-09T06:23:12.536 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.1 2023-10-09T06:23:12.853 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 06:23:12 smithi064 systemd[1]: Stopping Ceph osd.1 for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T06:23:13.219 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 06:23:12 smithi064 conmon[243014]: 2023-10-09T06:23:12.851+0000 7f0e3c6b3700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:23:13.219 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 06:23:12 smithi064 conmon[243014]: 2023-10-09T06:23:12.851+0000 7f0e3c6b3700 -1 osd.1 710 *** Got signal Terminated *** 2023-10-09T06:23:13.220 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 06:23:12 smithi064 conmon[243014]: 2023-10-09T06:23:12.851+0000 7f0e3c6b3700 -1 osd.1 710 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:23:18.970 INFO:journalctl@ceph.osd.1.smithi064.stdout:Oct 09 06:23:18 smithi064 bash[726606]: 2e7bf325c8048e148312d76fb93be30813c876c855b8e0f706e97310b6ee1c08 2023-10-09T06:23:20.675 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.1.service' 2023-10-09T06:23:20.716 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:23:20.717 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-10-09T06:23:20.717 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-10-09T06:23:20.717 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.2 2023-10-09T06:23:20.969 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 06:23:20 smithi064 systemd[1]: Stopping Ceph osd.2 for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T06:23:21.469 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 06:23:21 smithi064 conmon[250796]: 2023-10-09T06:23:21.024+0000 7f29aa938700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:23:21.470 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 06:23:21 smithi064 conmon[250796]: 2023-10-09T06:23:21.024+0000 7f29aa938700 -1 osd.2 710 *** Got signal Terminated *** 2023-10-09T06:23:21.470 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 06:23:21 smithi064 conmon[250796]: 2023-10-09T06:23:21.024+0000 7f29aa938700 -1 osd.2 710 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:23:27.469 INFO:journalctl@ceph.osd.2.smithi064.stdout:Oct 09 06:23:26 smithi064 bash[727186]: 152098b1dc2d3960d2f8f2afcca75cfd06b76ec28308f0b9ec842a6894973a66 2023-10-09T06:23:28.524 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.2.service' 2023-10-09T06:23:28.566 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:23:28.567 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-10-09T06:23:28.567 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-10-09T06:23:28.568 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.3 2023-10-09T06:23:28.892 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 06:23:28 smithi064 systemd[1]: Stopping Ceph osd.3 for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T06:23:29.219 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 06:23:28 smithi064 conmon[259186]: 2023-10-09T06:23:28.890+0000 7fe290134700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:23:29.219 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 06:23:28 smithi064 conmon[259186]: 2023-10-09T06:23:28.890+0000 7fe290134700 -1 osd.3 710 *** Got signal Terminated *** 2023-10-09T06:23:29.219 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 06:23:28 smithi064 conmon[259186]: 2023-10-09T06:23:28.890+0000 7fe290134700 -1 osd.3 710 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:23:30.871 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 06:23:30 smithi156 conmon[179783]: ts=2023-10-09T06:23:30.530Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.64:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.64:8765: connect: connection refused" 2023-10-09T06:23:30.871 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 06:23:30 smithi156 conmon[179783]: ts=2023-10-09T06:23:30.532Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.64:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.64:8765: connect: connection refused" 2023-10-09T06:23:30.872 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 06:23:30 smithi156 conmon[179783]: ts=2023-10-09T06:23:30.534Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.64:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.64:8765: connect: connection refused" 2023-10-09T06:23:30.872 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 06:23:30 smithi156 conmon[179783]: ts=2023-10-09T06:23:30.534Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.64:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.64:8765: connect: connection refused" 2023-10-09T06:23:31.370 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:23:30 smithi156 conmon[281149]: 2023-10-09T06:23:30.947+0000 7fd2950fa700 -1 osd.4 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.929243+0000 front 2023-10-09T06:23:07.929468+0000 (oldest deadline 2023-10-09T06:23:30.229383+0000) 2023-10-09T06:23:31.371 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:31 smithi156 conmon[406738]: 2023-10-09T06:23:31.045+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:09.166396+0000 front 2023-10-09T06:23:09.166360+0000 (oldest deadline 2023-10-09T06:23:30.766982+0000) 2023-10-09T06:23:32.065 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:23:31 smithi156 conmon[281149]: 2023-10-09T06:23:31.915+0000 7fd2950fa700 -1 osd.4 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.929243+0000 front 2023-10-09T06:23:07.929468+0000 (oldest deadline 2023-10-09T06:23:30.229383+0000) 2023-10-09T06:23:32.065 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:31 smithi156 conmon[141346]: 2023-10-09T06:23:31.679+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:32.370 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:32 smithi156 conmon[406738]: 2023-10-09T06:23:32.064+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:09.166396+0000 front 2023-10-09T06:23:09.166360+0000 (oldest deadline 2023-10-09T06:23:30.766982+0000) 2023-10-09T06:23:33.104 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:23:32 smithi156 conmon[281149]: 2023-10-09T06:23:32.870+0000 7fd2950fa700 -1 osd.4 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.929243+0000 front 2023-10-09T06:23:07.929468+0000 (oldest deadline 2023-10-09T06:23:30.229383+0000) 2023-10-09T06:23:33.104 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:32 smithi156 conmon[141346]: 2023-10-09T06:23:32.633+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:33.370 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:33 smithi156 conmon[406738]: 2023-10-09T06:23:33.103+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:09.166396+0000 front 2023-10-09T06:23:09.166360+0000 (oldest deadline 2023-10-09T06:23:30.766982+0000) 2023-10-09T06:23:33.870 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:23:33 smithi156 conmon[281149]: 2023-10-09T06:23:33.844+0000 7fd2950fa700 -1 osd.4 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.929243+0000 front 2023-10-09T06:23:07.929468+0000 (oldest deadline 2023-10-09T06:23:30.229383+0000) 2023-10-09T06:23:33.871 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:33 smithi156 conmon[141346]: 2023-10-09T06:23:33.609+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:34.219 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 06:23:33 smithi064 conmon[259186]: 2023-10-09T06:23:33.832+0000 7fe28bf29700 -1 osd.3 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.945911+0000 front 2023-10-09T06:23:07.945851+0000 (oldest deadline 2023-10-09T06:23:33.245949+0000) 2023-10-09T06:23:34.604 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:34 smithi156 conmon[406738]: 2023-10-09T06:23:34.151+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:09.166396+0000 front 2023-10-09T06:23:09.166360+0000 (oldest deadline 2023-10-09T06:23:30.766982+0000) 2023-10-09T06:23:34.870 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:34 smithi156 conmon[141346]: 2023-10-09T06:23:34.603+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:35.135 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:23:34 smithi156 conmon[281149]: 2023-10-09T06:23:34.879+0000 7fd2950fa700 -1 osd.4 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.929243+0000 front 2023-10-09T06:23:07.929468+0000 (oldest deadline 2023-10-09T06:23:30.229383+0000) 2023-10-09T06:23:35.219 INFO:journalctl@ceph.osd.3.smithi064.stdout:Oct 09 06:23:34 smithi064 bash[727778]: 08e3f2a3f251f126f0ef7fde23e949b597a03b668178087b9fee5ca7d2ff3107 2023-10-09T06:23:35.620 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:35 smithi156 conmon[406738]: 2023-10-09T06:23:35.135+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:09.166396+0000 front 2023-10-09T06:23:09.166360+0000 (oldest deadline 2023-10-09T06:23:30.766982+0000) 2023-10-09T06:23:36.120 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:23:35 smithi156 conmon[281149]: 2023-10-09T06:23:35.868+0000 7fd2950fa700 -1 osd.4 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.929243+0000 front 2023-10-09T06:23:07.929468+0000 (oldest deadline 2023-10-09T06:23:30.229383+0000) 2023-10-09T06:23:36.121 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:35 smithi156 conmon[141346]: 2023-10-09T06:23:35.634+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:36.121 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:35 smithi156 conmon[132493]: 2023-10-09T06:23:35.768+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.876836+0000 front 2023-10-09T06:23:07.876824+0000 (oldest deadline 2023-10-09T06:23:34.877394+0000) 2023-10-09T06:23:36.448 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.3.service' 2023-10-09T06:23:36.491 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:23:36.491 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-10-09T06:23:36.491 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-10-09T06:23:36.492 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.4 2023-10-09T06:23:36.524 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:36 smithi156 conmon[406738]: 2023-10-09T06:23:36.176+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:09.166396+0000 front 2023-10-09T06:23:09.166360+0000 (oldest deadline 2023-10-09T06:23:30.766982+0000) 2023-10-09T06:23:36.775 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:23:36 smithi156 systemd[1]: Stopping Ceph osd.4 for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T06:23:36.775 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:36 smithi156 conmon[141346]: 2023-10-09T06:23:36.677+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:37.121 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:36 smithi156 conmon[132493]: 2023-10-09T06:23:36.775+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.876836+0000 front 2023-10-09T06:23:07.876824+0000 (oldest deadline 2023-10-09T06:23:34.877394+0000) 2023-10-09T06:23:37.123 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:23:36 smithi156 conmon[281149]: 2023-10-09T06:23:36.774+0000 7fd299afe700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:23:37.123 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:23:36 smithi156 conmon[281149]: 2023-10-09T06:23:36.774+0000 7fd299afe700 -1 osd.4 710 *** Got signal Terminated *** 2023-10-09T06:23:37.123 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:23:36 smithi156 conmon[281149]: 2023-10-09T06:23:36.774+0000 7fd299afe700 -1 osd.4 710 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:23:37.124 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:23:36 smithi156 conmon[281149]: 2023-10-09T06:23:36.869+0000 7fd2950fa700 -1 osd.4 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.929243+0000 front 2023-10-09T06:23:07.929468+0000 (oldest deadline 2023-10-09T06:23:30.229383+0000) 2023-10-09T06:23:37.620 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:37 smithi156 conmon[406738]: 2023-10-09T06:23:37.212+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:09.166396+0000 front 2023-10-09T06:23:09.166360+0000 (oldest deadline 2023-10-09T06:23:30.766982+0000) 2023-10-09T06:23:38.120 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:37 smithi156 conmon[132493]: 2023-10-09T06:23:37.730+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.876836+0000 front 2023-10-09T06:23:07.876824+0000 (oldest deadline 2023-10-09T06:23:34.877394+0000) 2023-10-09T06:23:38.121 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:37 smithi156 conmon[141346]: 2023-10-09T06:23:37.661+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:38.121 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:23:37 smithi156 conmon[281149]: 2023-10-09T06:23:37.886+0000 7fd2950fa700 -1 osd.4 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.929243+0000 front 2023-10-09T06:23:07.929468+0000 (oldest deadline 2023-10-09T06:23:30.229383+0000) 2023-10-09T06:23:38.620 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:38 smithi156 conmon[406738]: 2023-10-09T06:23:38.166+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:09.166396+0000 front 2023-10-09T06:23:09.166360+0000 (oldest deadline 2023-10-09T06:23:30.766982+0000) 2023-10-09T06:23:38.912 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:38 smithi156 conmon[141346]: 2023-10-09T06:23:38.627+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:38.913 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:38 smithi156 conmon[132493]: 2023-10-09T06:23:38.729+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.876836+0000 front 2023-10-09T06:23:07.876824+0000 (oldest deadline 2023-10-09T06:23:34.877394+0000) 2023-10-09T06:23:39.212 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:23:38 smithi156 conmon[281149]: 2023-10-09T06:23:38.911+0000 7fd2950fa700 -1 osd.4 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.929243+0000 front 2023-10-09T06:23:07.929468+0000 (oldest deadline 2023-10-09T06:23:30.229383+0000) 2023-10-09T06:23:39.621 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:39 smithi156 conmon[406738]: 2023-10-09T06:23:39.212+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:09.166396+0000 front 2023-10-09T06:23:09.166360+0000 (oldest deadline 2023-10-09T06:23:30.766982+0000) 2023-10-09T06:23:39.937 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:39 smithi156 conmon[132493]: 2023-10-09T06:23:39.705+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.876836+0000 front 2023-10-09T06:23:07.876824+0000 (oldest deadline 2023-10-09T06:23:34.877394+0000) 2023-10-09T06:23:39.938 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:39 smithi156 conmon[132493]: 2023-10-09T06:23:39.705+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:14.878019+0000 front 2023-10-09T06:23:14.878043+0000 (oldest deadline 2023-10-09T06:23:38.977921+0000) 2023-10-09T06:23:39.938 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:39 smithi156 conmon[141346]: 2023-10-09T06:23:39.643+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:40.205 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:23:39 smithi156 conmon[281149]: 2023-10-09T06:23:39.931+0000 7fd2950fa700 -1 osd.4 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.929243+0000 front 2023-10-09T06:23:07.929468+0000 (oldest deadline 2023-10-09T06:23:30.229383+0000) 2023-10-09T06:23:40.206 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:23:39 smithi156 conmon[281149]: 2023-10-09T06:23:39.931+0000 7fd2950fa700 -1 osd.4 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:16.631132+0000 front 2023-10-09T06:23:16.631191+0000 (oldest deadline 2023-10-09T06:23:39.531001+0000) 2023-10-09T06:23:40.621 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:40 smithi156 conmon[406738]: 2023-10-09T06:23:40.200+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:09.166396+0000 front 2023-10-09T06:23:09.166360+0000 (oldest deadline 2023-10-09T06:23:30.766982+0000) 2023-10-09T06:23:40.622 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:40 smithi156 conmon[406738]: 2023-10-09T06:23:40.200+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:15.468171+0000 front 2023-10-09T06:23:15.468328+0000 (oldest deadline 2023-10-09T06:23:40.168238+0000) 2023-10-09T06:23:40.899 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:40 smithi156 conmon[132493]: 2023-10-09T06:23:40.663+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.876836+0000 front 2023-10-09T06:23:07.876824+0000 (oldest deadline 2023-10-09T06:23:34.877394+0000) 2023-10-09T06:23:40.900 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:40 smithi156 conmon[132493]: 2023-10-09T06:23:40.663+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:14.878019+0000 front 2023-10-09T06:23:14.878043+0000 (oldest deadline 2023-10-09T06:23:38.977921+0000) 2023-10-09T06:23:40.900 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:40 smithi156 conmon[141346]: 2023-10-09T06:23:40.634+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:41.247 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:23:40 smithi156 conmon[281149]: 2023-10-09T06:23:40.898+0000 7fd2950fa700 -1 osd.4 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.929243+0000 front 2023-10-09T06:23:07.929468+0000 (oldest deadline 2023-10-09T06:23:30.229383+0000) 2023-10-09T06:23:41.247 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:23:40 smithi156 conmon[281149]: 2023-10-09T06:23:40.898+0000 7fd2950fa700 -1 osd.4 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:16.631132+0000 front 2023-10-09T06:23:16.631191+0000 (oldest deadline 2023-10-09T06:23:39.531001+0000) 2023-10-09T06:23:41.612 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:41 smithi156 conmon[406738]: 2023-10-09T06:23:41.246+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:09.166396+0000 front 2023-10-09T06:23:09.166360+0000 (oldest deadline 2023-10-09T06:23:30.766982+0000) 2023-10-09T06:23:41.613 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:41 smithi156 conmon[406738]: 2023-10-09T06:23:41.246+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:15.468171+0000 front 2023-10-09T06:23:15.468328+0000 (oldest deadline 2023-10-09T06:23:40.168238+0000) 2023-10-09T06:23:41.871 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:41 smithi156 conmon[132493]: 2023-10-09T06:23:41.635+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.876836+0000 front 2023-10-09T06:23:07.876824+0000 (oldest deadline 2023-10-09T06:23:34.877394+0000) 2023-10-09T06:23:41.871 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:41 smithi156 conmon[132493]: 2023-10-09T06:23:41.635+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:14.878019+0000 front 2023-10-09T06:23:14.878043+0000 (oldest deadline 2023-10-09T06:23:38.977921+0000) 2023-10-09T06:23:41.871 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:41 smithi156 conmon[141346]: 2023-10-09T06:23:41.611+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:42.600 INFO:journalctl@ceph.osd.4.smithi156.stdout:Oct 09 06:23:42 smithi156 bash[431325]: 0d0094a29e696a0407d59c08e34f2ab3aa535312d1b2fc9c4626b221cc647b8f 2023-10-09T06:23:42.601 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:42 smithi156 conmon[406738]: 2023-10-09T06:23:42.273+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:09.166396+0000 front 2023-10-09T06:23:09.166360+0000 (oldest deadline 2023-10-09T06:23:30.766982+0000) 2023-10-09T06:23:42.601 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:42 smithi156 conmon[406738]: 2023-10-09T06:23:42.273+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:15.468171+0000 front 2023-10-09T06:23:15.468328+0000 (oldest deadline 2023-10-09T06:23:40.168238+0000) 2023-10-09T06:23:42.870 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:42 smithi156 conmon[141346]: 2023-10-09T06:23:42.606+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:42.870 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:42 smithi156 conmon[132493]: 2023-10-09T06:23:42.642+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.876836+0000 front 2023-10-09T06:23:07.876824+0000 (oldest deadline 2023-10-09T06:23:34.877394+0000) 2023-10-09T06:23:42.871 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:42 smithi156 conmon[132493]: 2023-10-09T06:23:42.642+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:14.878019+0000 front 2023-10-09T06:23:14.878043+0000 (oldest deadline 2023-10-09T06:23:38.977921+0000) 2023-10-09T06:23:43.559 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:43 smithi156 conmon[406738]: 2023-10-09T06:23:43.287+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:09.166396+0000 front 2023-10-09T06:23:09.166360+0000 (oldest deadline 2023-10-09T06:23:30.766982+0000) 2023-10-09T06:23:43.560 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:43 smithi156 conmon[406738]: 2023-10-09T06:23:43.287+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:15.468171+0000 front 2023-10-09T06:23:15.468328+0000 (oldest deadline 2023-10-09T06:23:40.168238+0000) 2023-10-09T06:23:43.870 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:43 smithi156 conmon[141346]: 2023-10-09T06:23:43.595+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:43.870 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:43 smithi156 conmon[132493]: 2023-10-09T06:23:43.679+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.876836+0000 front 2023-10-09T06:23:07.876824+0000 (oldest deadline 2023-10-09T06:23:34.877394+0000) 2023-10-09T06:23:43.871 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:43 smithi156 conmon[132493]: 2023-10-09T06:23:43.679+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:14.878019+0000 front 2023-10-09T06:23:14.878043+0000 (oldest deadline 2023-10-09T06:23:38.977921+0000) 2023-10-09T06:23:44.370 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:44 smithi156 conmon[406738]: 2023-10-09T06:23:44.247+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:09.166396+0000 front 2023-10-09T06:23:09.166360+0000 (oldest deadline 2023-10-09T06:23:30.766982+0000) 2023-10-09T06:23:44.370 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:44 smithi156 conmon[406738]: 2023-10-09T06:23:44.247+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:15.468171+0000 front 2023-10-09T06:23:15.468328+0000 (oldest deadline 2023-10-09T06:23:40.168238+0000) 2023-10-09T06:23:44.657 DEBUG:teuthology.orchestra.run.smithi156:> sudo pkill -f 'journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.4.service' 2023-10-09T06:23:44.739 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:23:44.739 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-10-09T06:23:44.739 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-10-09T06:23:44.740 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.5 2023-10-09T06:23:44.870 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:44 smithi156 systemd[1]: Stopping Ceph osd.5 for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T06:23:44.871 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:44 smithi156 conmon[132493]: 2023-10-09T06:23:44.696+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.876836+0000 front 2023-10-09T06:23:07.876824+0000 (oldest deadline 2023-10-09T06:23:34.877394+0000) 2023-10-09T06:23:44.871 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:44 smithi156 conmon[132493]: 2023-10-09T06:23:44.696+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:14.878019+0000 front 2023-10-09T06:23:14.878043+0000 (oldest deadline 2023-10-09T06:23:38.977921+0000) 2023-10-09T06:23:44.871 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:44 smithi156 conmon[141346]: 2023-10-09T06:23:44.584+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:44.871 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:44 smithi156 conmon[141346]: 2023-10-09T06:23:44.584+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:17.585642+0000 front 2023-10-09T06:23:17.585480+0000 (oldest deadline 2023-10-09T06:23:43.985856+0000) 2023-10-09T06:23:45.370 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:45 smithi156 conmon[406738]: 2023-10-09T06:23:45.047+0000 7f144038b700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:23:45.370 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:45 smithi156 conmon[406738]: 2023-10-09T06:23:45.047+0000 7f144038b700 -1 osd.5 710 *** Got signal Terminated *** 2023-10-09T06:23:45.371 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:45 smithi156 conmon[406738]: 2023-10-09T06:23:45.047+0000 7f144038b700 -1 osd.5 710 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:23:45.371 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:45 smithi156 conmon[406738]: 2023-10-09T06:23:45.210+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:09.166396+0000 front 2023-10-09T06:23:09.166360+0000 (oldest deadline 2023-10-09T06:23:30.766982+0000) 2023-10-09T06:23:45.371 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:45 smithi156 conmon[406738]: 2023-10-09T06:23:45.210+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:15.468171+0000 front 2023-10-09T06:23:15.468328+0000 (oldest deadline 2023-10-09T06:23:40.168238+0000) 2023-10-09T06:23:46.120 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:45 smithi156 conmon[141346]: 2023-10-09T06:23:45.626+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:46.120 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:45 smithi156 conmon[141346]: 2023-10-09T06:23:45.626+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:17.585642+0000 front 2023-10-09T06:23:17.585480+0000 (oldest deadline 2023-10-09T06:23:43.985856+0000) 2023-10-09T06:23:46.121 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:45 smithi156 conmon[132493]: 2023-10-09T06:23:45.661+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.876836+0000 front 2023-10-09T06:23:07.876824+0000 (oldest deadline 2023-10-09T06:23:34.877394+0000) 2023-10-09T06:23:46.121 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:45 smithi156 conmon[132493]: 2023-10-09T06:23:45.661+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:14.878019+0000 front 2023-10-09T06:23:14.878043+0000 (oldest deadline 2023-10-09T06:23:38.977921+0000) 2023-10-09T06:23:46.609 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:46 smithi156 conmon[406738]: 2023-10-09T06:23:46.259+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:09.166396+0000 front 2023-10-09T06:23:09.166360+0000 (oldest deadline 2023-10-09T06:23:30.766982+0000) 2023-10-09T06:23:46.609 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:46 smithi156 conmon[406738]: 2023-10-09T06:23:46.259+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:15.468171+0000 front 2023-10-09T06:23:15.468328+0000 (oldest deadline 2023-10-09T06:23:40.168238+0000) 2023-10-09T06:23:46.870 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:46 smithi156 conmon[141346]: 2023-10-09T06:23:46.608+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:46.870 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:46 smithi156 conmon[141346]: 2023-10-09T06:23:46.608+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:17.585642+0000 front 2023-10-09T06:23:17.585480+0000 (oldest deadline 2023-10-09T06:23:43.985856+0000) 2023-10-09T06:23:46.871 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:46 smithi156 conmon[132493]: 2023-10-09T06:23:46.644+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.876836+0000 front 2023-10-09T06:23:07.876824+0000 (oldest deadline 2023-10-09T06:23:34.877394+0000) 2023-10-09T06:23:46.871 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:46 smithi156 conmon[132493]: 2023-10-09T06:23:46.644+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:14.878019+0000 front 2023-10-09T06:23:14.878043+0000 (oldest deadline 2023-10-09T06:23:38.977921+0000) 2023-10-09T06:23:47.578 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:47 smithi156 conmon[406738]: 2023-10-09T06:23:47.288+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:09.166396+0000 front 2023-10-09T06:23:09.166360+0000 (oldest deadline 2023-10-09T06:23:30.766982+0000) 2023-10-09T06:23:47.578 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:47 smithi156 conmon[406738]: 2023-10-09T06:23:47.288+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:15.468171+0000 front 2023-10-09T06:23:15.468328+0000 (oldest deadline 2023-10-09T06:23:40.168238+0000) 2023-10-09T06:23:47.870 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:47 smithi156 conmon[141346]: 2023-10-09T06:23:47.577+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:47.870 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:47 smithi156 conmon[141346]: 2023-10-09T06:23:47.577+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:17.585642+0000 front 2023-10-09T06:23:17.585480+0000 (oldest deadline 2023-10-09T06:23:43.985856+0000) 2023-10-09T06:23:47.870 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:47 smithi156 conmon[132493]: 2023-10-09T06:23:47.654+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.876836+0000 front 2023-10-09T06:23:07.876824+0000 (oldest deadline 2023-10-09T06:23:34.877394+0000) 2023-10-09T06:23:47.871 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:47 smithi156 conmon[132493]: 2023-10-09T06:23:47.654+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:14.878019+0000 front 2023-10-09T06:23:14.878043+0000 (oldest deadline 2023-10-09T06:23:38.977921+0000) 2023-10-09T06:23:48.608 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:48 smithi156 conmon[406738]: 2023-10-09T06:23:48.263+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:09.166396+0000 front 2023-10-09T06:23:09.166360+0000 (oldest deadline 2023-10-09T06:23:30.766982+0000) 2023-10-09T06:23:48.608 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:48 smithi156 conmon[406738]: 2023-10-09T06:23:48.263+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:15.468171+0000 front 2023-10-09T06:23:15.468328+0000 (oldest deadline 2023-10-09T06:23:40.168238+0000) 2023-10-09T06:23:48.870 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:48 smithi156 conmon[141346]: 2023-10-09T06:23:48.607+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:48.870 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:48 smithi156 conmon[141346]: 2023-10-09T06:23:48.607+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:17.585642+0000 front 2023-10-09T06:23:17.585480+0000 (oldest deadline 2023-10-09T06:23:43.985856+0000) 2023-10-09T06:23:48.870 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:48 smithi156 conmon[141346]: 2023-10-09T06:23:48.607+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:23.986369+0000 front 2023-10-09T06:23:23.986452+0000 (oldest deadline 2023-10-09T06:23:48.086299+0000) 2023-10-09T06:23:48.871 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:48 smithi156 conmon[132493]: 2023-10-09T06:23:48.698+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.876836+0000 front 2023-10-09T06:23:07.876824+0000 (oldest deadline 2023-10-09T06:23:34.877394+0000) 2023-10-09T06:23:48.871 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:48 smithi156 conmon[132493]: 2023-10-09T06:23:48.698+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:14.878019+0000 front 2023-10-09T06:23:14.878043+0000 (oldest deadline 2023-10-09T06:23:38.977921+0000) 2023-10-09T06:23:49.597 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:49 smithi156 conmon[406738]: 2023-10-09T06:23:49.309+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:09.166396+0000 front 2023-10-09T06:23:09.166360+0000 (oldest deadline 2023-10-09T06:23:30.766982+0000) 2023-10-09T06:23:49.598 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:49 smithi156 conmon[406738]: 2023-10-09T06:23:49.309+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:15.468171+0000 front 2023-10-09T06:23:15.468328+0000 (oldest deadline 2023-10-09T06:23:40.168238+0000) 2023-10-09T06:23:49.598 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:49 smithi156 conmon[406738]: 2023-10-09T06:23:49.309+0000 7f143e18c700 -1 osd.5 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:25.969926+0000 front 2023-10-09T06:23:25.969907+0000 (oldest deadline 2023-10-09T06:23:48.269954+0000) 2023-10-09T06:23:49.870 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:49 smithi156 conmon[132493]: 2023-10-09T06:23:49.736+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.876836+0000 front 2023-10-09T06:23:07.876824+0000 (oldest deadline 2023-10-09T06:23:34.877394+0000) 2023-10-09T06:23:49.870 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:49 smithi156 conmon[132493]: 2023-10-09T06:23:49.736+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:14.878019+0000 front 2023-10-09T06:23:14.878043+0000 (oldest deadline 2023-10-09T06:23:38.977921+0000) 2023-10-09T06:23:49.870 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:49 smithi156 conmon[132493]: 2023-10-09T06:23:49.736+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:25.979546+0000 front 2023-10-09T06:23:25.979556+0000 (oldest deadline 2023-10-09T06:23:49.479401+0000) 2023-10-09T06:23:49.871 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:49 smithi156 conmon[141346]: 2023-10-09T06:23:49.597+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:49.871 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:49 smithi156 conmon[141346]: 2023-10-09T06:23:49.597+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:17.585642+0000 front 2023-10-09T06:23:17.585480+0000 (oldest deadline 2023-10-09T06:23:43.985856+0000) 2023-10-09T06:23:49.871 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:49 smithi156 conmon[141346]: 2023-10-09T06:23:49.597+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:23.986369+0000 front 2023-10-09T06:23:23.986452+0000 (oldest deadline 2023-10-09T06:23:48.086299+0000) 2023-10-09T06:23:50.871 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:50 smithi156 conmon[132493]: 2023-10-09T06:23:50.759+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.876836+0000 front 2023-10-09T06:23:07.876824+0000 (oldest deadline 2023-10-09T06:23:34.877394+0000) 2023-10-09T06:23:50.871 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:50 smithi156 conmon[132493]: 2023-10-09T06:23:50.759+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:14.878019+0000 front 2023-10-09T06:23:14.878043+0000 (oldest deadline 2023-10-09T06:23:38.977921+0000) 2023-10-09T06:23:50.871 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:50 smithi156 conmon[132493]: 2023-10-09T06:23:50.759+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:25.979546+0000 front 2023-10-09T06:23:25.979556+0000 (oldest deadline 2023-10-09T06:23:49.479401+0000) 2023-10-09T06:23:50.872 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:50 smithi156 conmon[141346]: 2023-10-09T06:23:50.602+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:50.872 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:50 smithi156 conmon[141346]: 2023-10-09T06:23:50.602+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:17.585642+0000 front 2023-10-09T06:23:17.585480+0000 (oldest deadline 2023-10-09T06:23:43.985856+0000) 2023-10-09T06:23:50.872 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:50 smithi156 conmon[141346]: 2023-10-09T06:23:50.602+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:23.986369+0000 front 2023-10-09T06:23:23.986452+0000 (oldest deadline 2023-10-09T06:23:48.086299+0000) 2023-10-09T06:23:51.370 INFO:journalctl@ceph.osd.5.smithi156.stdout:Oct 09 06:23:51 smithi156 bash[431910]: 1d1e8c86f3c535bea0f4ec175890ed20f8d2c6ebec7dc46a1d634c53b676f0b7 2023-10-09T06:23:52.089 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:51 smithi156 conmon[132493]: 2023-10-09T06:23:51.717+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.876836+0000 front 2023-10-09T06:23:07.876824+0000 (oldest deadline 2023-10-09T06:23:34.877394+0000) 2023-10-09T06:23:52.089 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:51 smithi156 conmon[132493]: 2023-10-09T06:23:51.717+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:14.878019+0000 front 2023-10-09T06:23:14.878043+0000 (oldest deadline 2023-10-09T06:23:38.977921+0000) 2023-10-09T06:23:52.090 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:51 smithi156 conmon[132493]: 2023-10-09T06:23:51.717+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:25.979546+0000 front 2023-10-09T06:23:25.979556+0000 (oldest deadline 2023-10-09T06:23:49.479401+0000) 2023-10-09T06:23:52.090 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:51 smithi156 conmon[141346]: 2023-10-09T06:23:51.642+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:52.090 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:51 smithi156 conmon[141346]: 2023-10-09T06:23:51.642+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:17.585642+0000 front 2023-10-09T06:23:17.585480+0000 (oldest deadline 2023-10-09T06:23:43.985856+0000) 2023-10-09T06:23:52.090 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:51 smithi156 conmon[141346]: 2023-10-09T06:23:51.642+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:23.986369+0000 front 2023-10-09T06:23:23.986452+0000 (oldest deadline 2023-10-09T06:23:48.086299+0000) 2023-10-09T06:23:52.706 DEBUG:teuthology.orchestra.run.smithi156:> sudo pkill -f 'journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.5.service' 2023-10-09T06:23:52.751 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:23:52.751 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-10-09T06:23:52.752 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-10-09T06:23:52.752 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.6 2023-10-09T06:23:52.870 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:52 smithi156 conmon[141346]: 2023-10-09T06:23:52.612+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:52.870 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:52 smithi156 conmon[141346]: 2023-10-09T06:23:52.612+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:17.585642+0000 front 2023-10-09T06:23:17.585480+0000 (oldest deadline 2023-10-09T06:23:43.985856+0000) 2023-10-09T06:23:52.871 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:52 smithi156 conmon[141346]: 2023-10-09T06:23:52.612+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:23.986369+0000 front 2023-10-09T06:23:23.986452+0000 (oldest deadline 2023-10-09T06:23:48.086299+0000) 2023-10-09T06:23:52.871 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:52 smithi156 conmon[132493]: 2023-10-09T06:23:52.698+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.876836+0000 front 2023-10-09T06:23:07.876824+0000 (oldest deadline 2023-10-09T06:23:34.877394+0000) 2023-10-09T06:23:52.871 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:52 smithi156 conmon[132493]: 2023-10-09T06:23:52.698+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:14.878019+0000 front 2023-10-09T06:23:14.878043+0000 (oldest deadline 2023-10-09T06:23:38.977921+0000) 2023-10-09T06:23:52.871 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:52 smithi156 conmon[132493]: 2023-10-09T06:23:52.698+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:25.979546+0000 front 2023-10-09T06:23:25.979556+0000 (oldest deadline 2023-10-09T06:23:49.479401+0000) 2023-10-09T06:23:52.871 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:52 smithi156 systemd[1]: Stopping Ceph osd.6 for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T06:23:53.370 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:53 smithi156 conmon[132493]: 2023-10-09T06:23:53.059+0000 7fa6825a0700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:23:53.370 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:53 smithi156 conmon[132493]: 2023-10-09T06:23:53.059+0000 7fa6825a0700 -1 osd.6 710 *** Got signal Terminated *** 2023-10-09T06:23:53.370 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:53 smithi156 conmon[132493]: 2023-10-09T06:23:53.059+0000 7fa6825a0700 -1 osd.6 710 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:23:54.120 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:53 smithi156 conmon[132493]: 2023-10-09T06:23:53.658+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.876836+0000 front 2023-10-09T06:23:07.876824+0000 (oldest deadline 2023-10-09T06:23:34.877394+0000) 2023-10-09T06:23:54.120 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:53 smithi156 conmon[132493]: 2023-10-09T06:23:53.658+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:14.878019+0000 front 2023-10-09T06:23:14.878043+0000 (oldest deadline 2023-10-09T06:23:38.977921+0000) 2023-10-09T06:23:54.121 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:53 smithi156 conmon[132493]: 2023-10-09T06:23:53.658+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:25.979546+0000 front 2023-10-09T06:23:25.979556+0000 (oldest deadline 2023-10-09T06:23:49.479401+0000) 2023-10-09T06:23:54.121 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:53 smithi156 conmon[141346]: 2023-10-09T06:23:53.638+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:54.121 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:53 smithi156 conmon[141346]: 2023-10-09T06:23:53.638+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:17.585642+0000 front 2023-10-09T06:23:17.585480+0000 (oldest deadline 2023-10-09T06:23:43.985856+0000) 2023-10-09T06:23:54.121 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:53 smithi156 conmon[141346]: 2023-10-09T06:23:53.638+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:23.986369+0000 front 2023-10-09T06:23:23.986452+0000 (oldest deadline 2023-10-09T06:23:48.086299+0000) 2023-10-09T06:23:55.120 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:54 smithi156 conmon[132493]: 2023-10-09T06:23:54.634+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.876836+0000 front 2023-10-09T06:23:07.876824+0000 (oldest deadline 2023-10-09T06:23:34.877394+0000) 2023-10-09T06:23:55.120 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:54 smithi156 conmon[132493]: 2023-10-09T06:23:54.634+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:14.878019+0000 front 2023-10-09T06:23:14.878043+0000 (oldest deadline 2023-10-09T06:23:38.977921+0000) 2023-10-09T06:23:55.120 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:54 smithi156 conmon[132493]: 2023-10-09T06:23:54.634+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:25.979546+0000 front 2023-10-09T06:23:25.979556+0000 (oldest deadline 2023-10-09T06:23:49.479401+0000) 2023-10-09T06:23:55.121 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:54 smithi156 conmon[141346]: 2023-10-09T06:23:54.642+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:55.121 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:54 smithi156 conmon[141346]: 2023-10-09T06:23:54.642+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:17.585642+0000 front 2023-10-09T06:23:17.585480+0000 (oldest deadline 2023-10-09T06:23:43.985856+0000) 2023-10-09T06:23:55.121 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:54 smithi156 conmon[141346]: 2023-10-09T06:23:54.642+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:23.986369+0000 front 2023-10-09T06:23:23.986452+0000 (oldest deadline 2023-10-09T06:23:48.086299+0000) 2023-10-09T06:23:55.870 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:55 smithi156 conmon[141346]: 2023-10-09T06:23:55.675+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:55.871 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:55 smithi156 conmon[141346]: 2023-10-09T06:23:55.675+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:17.585642+0000 front 2023-10-09T06:23:17.585480+0000 (oldest deadline 2023-10-09T06:23:43.985856+0000) 2023-10-09T06:23:55.871 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:55 smithi156 conmon[141346]: 2023-10-09T06:23:55.675+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:23.986369+0000 front 2023-10-09T06:23:23.986452+0000 (oldest deadline 2023-10-09T06:23:48.086299+0000) 2023-10-09T06:23:55.871 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:55 smithi156 conmon[132493]: 2023-10-09T06:23:55.614+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.876836+0000 front 2023-10-09T06:23:07.876824+0000 (oldest deadline 2023-10-09T06:23:34.877394+0000) 2023-10-09T06:23:55.871 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:55 smithi156 conmon[132493]: 2023-10-09T06:23:55.614+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:14.878019+0000 front 2023-10-09T06:23:14.878043+0000 (oldest deadline 2023-10-09T06:23:38.977921+0000) 2023-10-09T06:23:55.871 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:55 smithi156 conmon[132493]: 2023-10-09T06:23:55.614+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:25.979546+0000 front 2023-10-09T06:23:25.979556+0000 (oldest deadline 2023-10-09T06:23:49.479401+0000) 2023-10-09T06:23:55.872 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:55 smithi156 conmon[132493]: 2023-10-09T06:23:55.614+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2023-10-09T06:23:29.480029+0000 front 2023-10-09T06:23:29.479986+0000 (oldest deadline 2023-10-09T06:23:54.680204+0000) 2023-10-09T06:23:57.120 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:56 smithi156 conmon[141346]: 2023-10-09T06:23:56.696+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:57.121 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:56 smithi156 conmon[141346]: 2023-10-09T06:23:56.696+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:17.585642+0000 front 2023-10-09T06:23:17.585480+0000 (oldest deadline 2023-10-09T06:23:43.985856+0000) 2023-10-09T06:23:57.121 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:56 smithi156 conmon[141346]: 2023-10-09T06:23:56.696+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:23.986369+0000 front 2023-10-09T06:23:23.986452+0000 (oldest deadline 2023-10-09T06:23:48.086299+0000) 2023-10-09T06:23:57.121 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:56 smithi156 conmon[132493]: 2023-10-09T06:23:56.621+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.876836+0000 front 2023-10-09T06:23:07.876824+0000 (oldest deadline 2023-10-09T06:23:34.877394+0000) 2023-10-09T06:23:57.121 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:56 smithi156 conmon[132493]: 2023-10-09T06:23:56.621+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:14.878019+0000 front 2023-10-09T06:23:14.878043+0000 (oldest deadline 2023-10-09T06:23:38.977921+0000) 2023-10-09T06:23:57.122 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:56 smithi156 conmon[132493]: 2023-10-09T06:23:56.621+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:25.979546+0000 front 2023-10-09T06:23:25.979556+0000 (oldest deadline 2023-10-09T06:23:49.479401+0000) 2023-10-09T06:23:57.122 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:56 smithi156 conmon[132493]: 2023-10-09T06:23:56.621+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2023-10-09T06:23:29.480029+0000 front 2023-10-09T06:23:29.479986+0000 (oldest deadline 2023-10-09T06:23:54.680204+0000) 2023-10-09T06:23:57.870 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:57 smithi156 conmon[132493]: 2023-10-09T06:23:57.573+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:07.876836+0000 front 2023-10-09T06:23:07.876824+0000 (oldest deadline 2023-10-09T06:23:34.877394+0000) 2023-10-09T06:23:57.870 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:57 smithi156 conmon[132493]: 2023-10-09T06:23:57.573+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:14.878019+0000 front 2023-10-09T06:23:14.878043+0000 (oldest deadline 2023-10-09T06:23:38.977921+0000) 2023-10-09T06:23:57.871 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:57 smithi156 conmon[132493]: 2023-10-09T06:23:57.573+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:25.979546+0000 front 2023-10-09T06:23:25.979556+0000 (oldest deadline 2023-10-09T06:23:49.479401+0000) 2023-10-09T06:23:57.871 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:57 smithi156 conmon[132493]: 2023-10-09T06:23:57.573+0000 7fa6803a1700 -1 osd.6 710 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2023-10-09T06:23:29.480029+0000 front 2023-10-09T06:23:29.479986+0000 (oldest deadline 2023-10-09T06:23:54.680204+0000) 2023-10-09T06:23:57.871 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:57 smithi156 conmon[141346]: 2023-10-09T06:23:57.659+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:57.871 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:57 smithi156 conmon[141346]: 2023-10-09T06:23:57.659+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:17.585642+0000 front 2023-10-09T06:23:17.585480+0000 (oldest deadline 2023-10-09T06:23:43.985856+0000) 2023-10-09T06:23:57.871 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:57 smithi156 conmon[141346]: 2023-10-09T06:23:57.659+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:23.986369+0000 front 2023-10-09T06:23:23.986452+0000 (oldest deadline 2023-10-09T06:23:48.086299+0000) 2023-10-09T06:23:58.988 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:23:58 smithi156 bash[432489]: c8a3e3228b895e51315104823709e855e288d2f33b3262c1c28655d50ced2727 2023-10-09T06:23:59.094 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:58 smithi156 conmon[141346]: 2023-10-09T06:23:58.673+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:23:59.094 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:58 smithi156 conmon[141346]: 2023-10-09T06:23:58.673+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:17.585642+0000 front 2023-10-09T06:23:17.585480+0000 (oldest deadline 2023-10-09T06:23:43.985856+0000) 2023-10-09T06:23:59.094 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:58 smithi156 conmon[141346]: 2023-10-09T06:23:58.673+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:23.986369+0000 front 2023-10-09T06:23:23.986452+0000 (oldest deadline 2023-10-09T06:23:48.086299+0000) 2023-10-09T06:23:59.094 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:58 smithi156 conmon[141346]: 2023-10-09T06:23:58.673+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2023-10-09T06:23:32.187579+0000 front 2023-10-09T06:23:32.187601+0000 (oldest deadline 2023-10-09T06:23:58.087394+0000) 2023-10-09T06:24:00.120 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:59 smithi156 conmon[141346]: 2023-10-09T06:23:59.676+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:24:00.120 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:59 smithi156 conmon[141346]: 2023-10-09T06:23:59.676+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:17.585642+0000 front 2023-10-09T06:23:17.585480+0000 (oldest deadline 2023-10-09T06:23:43.985856+0000) 2023-10-09T06:24:00.120 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:59 smithi156 conmon[141346]: 2023-10-09T06:23:59.676+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:23.986369+0000 front 2023-10-09T06:23:23.986452+0000 (oldest deadline 2023-10-09T06:23:48.086299+0000) 2023-10-09T06:24:00.121 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:23:59 smithi156 conmon[141346]: 2023-10-09T06:23:59.676+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2023-10-09T06:23:32.187579+0000 front 2023-10-09T06:23:32.187601+0000 (oldest deadline 2023-10-09T06:23:58.087394+0000) 2023-10-09T06:24:00.871 DEBUG:teuthology.orchestra.run.smithi156:> sudo pkill -f 'journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.6.service' 2023-10-09T06:24:00.898 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:24:00 smithi156 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.6.service: Succeeded. 2023-10-09T06:24:00.898 INFO:journalctl@ceph.osd.6.smithi156.stdout:Oct 09 06:24:00 smithi156 systemd[1]: Stopped Ceph osd.6 for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T06:24:00.898 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:00 smithi156 conmon[141346]: 2023-10-09T06:24:00.637+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:24:00.898 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:00 smithi156 conmon[141346]: 2023-10-09T06:24:00.637+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:17.585642+0000 front 2023-10-09T06:23:17.585480+0000 (oldest deadline 2023-10-09T06:23:43.985856+0000) 2023-10-09T06:24:00.899 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:00 smithi156 conmon[141346]: 2023-10-09T06:24:00.637+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:23.986369+0000 front 2023-10-09T06:23:23.986452+0000 (oldest deadline 2023-10-09T06:23:48.086299+0000) 2023-10-09T06:24:00.899 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:00 smithi156 conmon[141346]: 2023-10-09T06:24:00.637+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2023-10-09T06:23:32.187579+0000 front 2023-10-09T06:23:32.187601+0000 (oldest deadline 2023-10-09T06:23:58.087394+0000) 2023-10-09T06:24:00.912 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:24:00.912 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-10-09T06:24:00.912 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-10-09T06:24:00.912 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.7 2023-10-09T06:24:01.263 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:00 smithi156 systemd[1]: Stopping Ceph osd.7 for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T06:24:01.620 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:01 smithi156 conmon[141346]: 2023-10-09T06:24:01.258+0000 7f5ca386e700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:24:01.620 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:01 smithi156 conmon[141346]: 2023-10-09T06:24:01.258+0000 7f5ca386e700 -1 osd.7 710 *** Got signal Terminated *** 2023-10-09T06:24:01.620 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:01 smithi156 conmon[141346]: 2023-10-09T06:24:01.258+0000 7f5ca386e700 -1 osd.7 710 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:24:02.120 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:01 smithi156 conmon[141346]: 2023-10-09T06:24:01.657+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:24:02.120 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:01 smithi156 conmon[141346]: 2023-10-09T06:24:01.657+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:17.585642+0000 front 2023-10-09T06:23:17.585480+0000 (oldest deadline 2023-10-09T06:23:43.985856+0000) 2023-10-09T06:24:02.120 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:01 smithi156 conmon[141346]: 2023-10-09T06:24:01.657+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:23.986369+0000 front 2023-10-09T06:23:23.986452+0000 (oldest deadline 2023-10-09T06:23:48.086299+0000) 2023-10-09T06:24:02.121 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:01 smithi156 conmon[141346]: 2023-10-09T06:24:01.657+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2023-10-09T06:23:32.187579+0000 front 2023-10-09T06:23:32.187601+0000 (oldest deadline 2023-10-09T06:23:58.087394+0000) 2023-10-09T06:24:03.120 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:02 smithi156 conmon[141346]: 2023-10-09T06:24:02.635+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:24:03.120 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:02 smithi156 conmon[141346]: 2023-10-09T06:24:02.635+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:17.585642+0000 front 2023-10-09T06:23:17.585480+0000 (oldest deadline 2023-10-09T06:23:43.985856+0000) 2023-10-09T06:24:03.120 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:02 smithi156 conmon[141346]: 2023-10-09T06:24:02.635+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:23.986369+0000 front 2023-10-09T06:23:23.986452+0000 (oldest deadline 2023-10-09T06:23:48.086299+0000) 2023-10-09T06:24:03.121 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:02 smithi156 conmon[141346]: 2023-10-09T06:24:02.635+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2023-10-09T06:23:32.187579+0000 front 2023-10-09T06:23:32.187601+0000 (oldest deadline 2023-10-09T06:23:58.087394+0000) 2023-10-09T06:24:03.871 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:03 smithi156 conmon[141346]: 2023-10-09T06:24:03.596+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:24:03.871 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:03 smithi156 conmon[141346]: 2023-10-09T06:24:03.596+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:17.585642+0000 front 2023-10-09T06:23:17.585480+0000 (oldest deadline 2023-10-09T06:23:43.985856+0000) 2023-10-09T06:24:03.871 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:03 smithi156 conmon[141346]: 2023-10-09T06:24:03.596+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:23.986369+0000 front 2023-10-09T06:23:23.986452+0000 (oldest deadline 2023-10-09T06:23:48.086299+0000) 2023-10-09T06:24:03.871 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:03 smithi156 conmon[141346]: 2023-10-09T06:24:03.596+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2023-10-09T06:23:32.187579+0000 front 2023-10-09T06:23:32.187601+0000 (oldest deadline 2023-10-09T06:23:58.087394+0000) 2023-10-09T06:24:03.871 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:03 smithi156 conmon[141346]: 2023-10-09T06:24:03.596+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.156:6804 osd.4 since back 2023-10-09T06:23:38.087802+0000 front 2023-10-09T06:23:38.087827+0000 (oldest deadline 2023-10-09T06:24:03.387769+0000) 2023-10-09T06:24:04.870 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:04 smithi156 conmon[141346]: 2023-10-09T06:24:04.563+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:24:04.870 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:04 smithi156 conmon[141346]: 2023-10-09T06:24:04.563+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:17.585642+0000 front 2023-10-09T06:23:17.585480+0000 (oldest deadline 2023-10-09T06:23:43.985856+0000) 2023-10-09T06:24:04.870 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:04 smithi156 conmon[141346]: 2023-10-09T06:24:04.563+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:23.986369+0000 front 2023-10-09T06:23:23.986452+0000 (oldest deadline 2023-10-09T06:23:48.086299+0000) 2023-10-09T06:24:04.871 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:04 smithi156 conmon[141346]: 2023-10-09T06:24:04.563+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2023-10-09T06:23:32.187579+0000 front 2023-10-09T06:23:32.187601+0000 (oldest deadline 2023-10-09T06:23:58.087394+0000) 2023-10-09T06:24:04.871 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:04 smithi156 conmon[141346]: 2023-10-09T06:24:04.563+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.156:6804 osd.4 since back 2023-10-09T06:23:38.087802+0000 front 2023-10-09T06:23:38.087827+0000 (oldest deadline 2023-10-09T06:24:03.387769+0000) 2023-10-09T06:24:05.870 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:05 smithi156 conmon[141346]: 2023-10-09T06:24:05.597+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-10-09T06:23:06.484145+0000 front 2023-10-09T06:23:06.484007+0000 (oldest deadline 2023-10-09T06:23:31.183912+0000) 2023-10-09T06:24:05.870 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:05 smithi156 conmon[141346]: 2023-10-09T06:24:05.597+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-10-09T06:23:17.585642+0000 front 2023-10-09T06:23:17.585480+0000 (oldest deadline 2023-10-09T06:23:43.985856+0000) 2023-10-09T06:24:05.870 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:05 smithi156 conmon[141346]: 2023-10-09T06:24:05.597+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-10-09T06:23:23.986369+0000 front 2023-10-09T06:23:23.986452+0000 (oldest deadline 2023-10-09T06:23:48.086299+0000) 2023-10-09T06:24:05.871 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:05 smithi156 conmon[141346]: 2023-10-09T06:24:05.597+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2023-10-09T06:23:32.187579+0000 front 2023-10-09T06:23:32.187601+0000 (oldest deadline 2023-10-09T06:23:58.087394+0000) 2023-10-09T06:24:05.871 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:05 smithi156 conmon[141346]: 2023-10-09T06:24:05.597+0000 7f5c9ce66700 -1 osd.7 710 heartbeat_check: no reply from 172.21.15.156:6804 osd.4 since back 2023-10-09T06:23:38.087802+0000 front 2023-10-09T06:23:38.087827+0000 (oldest deadline 2023-10-09T06:24:03.387769+0000) 2023-10-09T06:24:07.370 INFO:journalctl@ceph.osd.7.smithi156.stdout:Oct 09 06:24:06 smithi156 bash[433072]: df93983742a3bdca2435630ec26a2c0f82f2adb412623fa1380f0752d746a97f 2023-10-09T06:24:09.162 DEBUG:teuthology.orchestra.run.smithi156:> sudo pkill -f 'journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@osd.7.service' 2023-10-09T06:24:09.204 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:24:09.204 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-10-09T06:24:09.204 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-10-09T06:24:09.205 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@prometheus.a 2023-10-09T06:24:09.621 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 06:24:09 smithi156 systemd[1]: Stopping Ceph prometheus.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T06:24:09.621 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 06:24:09 smithi156 conmon[179783]: ts=2023-10-09T06:24:09.518Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T06:24:09.621 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 06:24:09 smithi156 conmon[179783]: ts=2023-10-09T06:24:09.519Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-10-09T06:24:09.621 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 06:24:09 smithi156 conmon[179783]: ts=2023-10-09T06:24:09.519Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-10-09T06:24:09.622 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 06:24:09 smithi156 conmon[179783]: ts=2023-10-09T06:24:09.519Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T06:24:09.622 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 06:24:09 smithi156 conmon[179783]: ts=2023-10-09T06:24:09.519Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-10-09T06:24:09.622 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 06:24:09 smithi156 conmon[179783]: ts=2023-10-09T06:24:09.519Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-10-09T06:24:09.622 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 06:24:09 smithi156 conmon[179783]: ts=2023-10-09T06:24:09.519Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T06:24:09.622 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 06:24:09 smithi156 conmon[179783]: ts=2023-10-09T06:24:09.519Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-10-09T06:24:09.622 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 06:24:09 smithi156 conmon[179783]: ts=2023-10-09T06:24:09.519Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-10-09T06:24:09.622 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 06:24:09 smithi156 conmon[179783]: ts=2023-10-09T06:24:09.520Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T06:24:09.623 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 06:24:09 smithi156 conmon[179783]: ts=2023-10-09T06:24:09.521Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-10-09T06:24:09.623 INFO:journalctl@ceph.prometheus.a.smithi156.stdout:Oct 09 06:24:09 smithi156 conmon[179783]: ts=2023-10-09T06:24:09.521Z caller=main.go:1147 level=info msg="See you next time!" 2023-10-09T06:24:10.255 DEBUG:teuthology.orchestra.run.smithi156:> sudo pkill -f 'journalctl -f -n 0 -u ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@prometheus.a.service' 2023-10-09T06:24:10.293 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:24:10.293 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-10-09T06:24:10.293 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T06:24:11.719 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 06:24:11 smithi064 systemd[1]: Stopping Ceph alertmanager.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T06:24:12.069 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 06:24:11 smithi064 conmon[197067]: ts=2023-10-09T06:24:11.760Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-10-09T06:24:12.069 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 06:24:12 smithi064 bash[728656]: 386f8a4e2221913e01416d141066eeb8280f62201286ec81d1b08f71ca497bb4 2023-10-09T06:24:12.719 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 06:24:12 smithi064 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T06:24:12.719 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Oct 09 06:24:12 smithi064 systemd[1]: Stopped Ceph alertmanager.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T06:24:12.720 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 06:24:12 smithi064 systemd[1]: Stopping Ceph node-exporter.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T06:24:13.551 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 06:24:13 smithi064 bash[728943]: afb28622babd191733cc0f1f8c57cb6135b975673afec1be3854b9e591e706da 2023-10-09T06:24:13.551 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 06:24:13 smithi064 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T06:24:13.552 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 06:24:13 smithi064 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-09T06:24:13.552 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Oct 09 06:24:13 smithi064 systemd[1]: Stopped Ceph node-exporter.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T06:24:24.788 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T06:24:26.120 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 06:24:25 smithi156 systemd[1]: Stopping Ceph node-exporter.b for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T06:24:26.870 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 06:24:26 smithi156 bash[434304]: 1e8c1e28630a753867da56381edc65a074f68c80a2c269177d693db75749592d 2023-10-09T06:24:26.871 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 06:24:26 smithi156 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-09T06:24:26.871 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 06:24:26 smithi156 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-09T06:24:26.871 INFO:journalctl@ceph.node-exporter.b.smithi156.stdout:Oct 09 06:24:26 smithi156 systemd[1]: Stopped Ceph node-exporter.b for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T06:24:27.267 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 06:24:26 smithi156 systemd[1]: Stopping Ceph grafana.a for 44c77acc-6665-11ee-8db5-212e2dc638e7... 2023-10-09T06:24:27.870 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 06:24:27 smithi156 bash[434584]: 365834e48ffb84a2d21b6af747a8343a0bdbfd78c5c7b4b33b418c4fc093050d 2023-10-09T06:24:27.870 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 06:24:27 smithi156 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T06:24:27.871 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 06:24:27 smithi156 systemd[1]: ceph-44c77acc-6665-11ee-8db5-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-10-09T06:24:27.871 INFO:journalctl@ceph.grafana.a.smithi156.stdout:Oct 09 06:24:27 smithi156 systemd[1]: Stopped Ceph grafana.a for 44c77acc-6665-11ee-8db5-212e2dc638e7. 2023-10-09T06:24:33.046 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T06:24:33.086 DEBUG:teuthology.orchestra.run.smithi156:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T06:24:33.116 INFO:tasks.cephadm:Archiving crash dumps... 2023-10-09T06:24:33.118 DEBUG:teuthology.misc:Transferring archived files from smithi064:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417503/remote/smithi064/crash 2023-10-09T06:24:33.118 DEBUG:teuthology.orchestra.run.smithi064:> sudo tar c -f - -C /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T06:24:33.184 INFO:teuthology.orchestra.run.smithi064.stderr:tar: /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T06:24:33.184 INFO:teuthology.orchestra.run.smithi064.stderr:tar: Error is not recoverable: exiting now 2023-10-09T06:24:33.186 DEBUG:teuthology.misc:Transferring archived files from smithi156:/var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417503/remote/smithi156/crash 2023-10-09T06:24:33.186 DEBUG:teuthology.orchestra.run.smithi156:> sudo tar c -f - -C /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T06:24:33.233 INFO:teuthology.orchestra.run.smithi156.stderr:tar: /var/lib/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T06:24:33.233 INFO:teuthology.orchestra.run.smithi156.stderr:tar: Error is not recoverable: exiting now 2023-10-09T06:24:33.234 INFO:tasks.cephadm:Checking cluster log for badness... 2023-10-09T06:24:33.234 DEBUG:teuthology.orchestra.run.smithi064:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-10-09T06:24:33.267 INFO:teuthology.orchestra.run.smithi064.stderr:grep: /var/log/ceph/44c77acc-6665-11ee-8db5-212e2dc638e7/ceph.log: No such file or directory 2023-10-09T06:24:33.268 INFO:tasks.cephadm:Compressing logs... 2023-10-09T06:24:33.269 DEBUG:teuthology.orchestra.run.smithi064:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T06:24:33.311 DEBUG:teuthology.orchestra.run.smithi156:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T06:24:33.348 INFO:teuthology.orchestra.run.smithi064.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-10-09T06:24:33.357 INFO:teuthology.orchestra.run.smithi156.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-10-09T06:34:16.237 INFO:tasks.cephadm:Archiving logs... 2023-10-09T06:34:16.268 DEBUG:teuthology.misc:Transferring archived files from smithi064:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417503/remote/smithi064/log 2023-10-09T06:34:16.269 DEBUG:teuthology.orchestra.run.smithi064:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T06:35:26.203 DEBUG:teuthology.misc:Transferring archived files from smithi156:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417503/remote/smithi156/log 2023-10-09T06:35:26.205 DEBUG:teuthology.orchestra.run.smithi156:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T06:36:38.183 INFO:tasks.cephadm:Removing cluster... 2023-10-09T06:36:38.184 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 --force 2023-10-09T06:36:39.419 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 44c77acc-6665-11ee-8db5-212e2dc638e7 --force 2023-10-09T06:36:40.648 INFO:tasks.cephadm:Removing cephadm ... 2023-10-09T06:36:40.648 DEBUG:teuthology.orchestra.run.smithi064:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T06:36:40.671 DEBUG:teuthology.orchestra.run.smithi156:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T06:36:40.707 INFO:tasks.cephadm:Teardown complete 2023-10-09T06:36:40.707 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-10-09T06:36:40.719 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-10-09T06:36:40.720 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-09T06:36:40.723 DEBUG:teuthology.orchestra.run.smithi156:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-09T06:36:40.844 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-10-09T06:36:40.854 INFO:teuthology.task.clock:Checking final clock skew... 2023-10-09T06:36:40.854 DEBUG:teuthology.orchestra.run.smithi064:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T06:36:40.858 DEBUG:teuthology.orchestra.run.smithi156:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T06:36:40.874 INFO:teuthology.orchestra.run.smithi156.stderr:bash: ntpq: command not found 2023-10-09T06:36:40.884 INFO:teuthology.orchestra.run.smithi064.stderr:bash: ntpq: command not found 2023-10-09T06:36:40.893 INFO:teuthology.orchestra.run.smithi156.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T06:36:40.893 INFO:teuthology.orchestra.run.smithi156.stdout:=============================================================================== 2023-10-09T06:36:40.894 INFO:teuthology.orchestra.run.smithi156.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 63 -698us[ -692us] +/- 77ms 2023-10-09T06:36:40.894 INFO:teuthology.orchestra.run.smithi156.stdout:^* hv02.front.sepia.ceph.com 3 6 377 2 +262us[ +269us] +/- 60ms 2023-10-09T06:36:40.894 INFO:teuthology.orchestra.run.smithi156.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 2 +366us[ +372us] +/- 74ms 2023-10-09T06:36:40.894 INFO:teuthology.orchestra.run.smithi156.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T06:36:40.910 INFO:teuthology.orchestra.run.smithi064.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T06:36:40.910 INFO:teuthology.orchestra.run.smithi064.stdout:=============================================================================== 2023-10-09T06:36:40.910 INFO:teuthology.orchestra.run.smithi064.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 61 -720us[ -724us] +/- 77ms 2023-10-09T06:36:40.910 INFO:teuthology.orchestra.run.smithi064.stdout:^* hv02.front.sepia.ceph.com 3 6 377 60 +237us[ +233us] +/- 59ms 2023-10-09T06:36:40.910 INFO:teuthology.orchestra.run.smithi064.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 61 +360us[ +356us] +/- 73ms 2023-10-09T06:36:40.910 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T06:36:40.911 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-10-09T06:36:40.922 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-10-09T06:36:40.922 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-10-09T06:36:40.949 DEBUG:teuthology.orchestra.run.smithi064:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-10-09T06:36:41.089 DEBUG:teuthology.orchestra.run.smithi156:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-10-09T06:36:41.203 DEBUG:teuthology.orchestra.run.smithi064:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T06:36:41.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:41.238 DEBUG:teuthology.orchestra.run.smithi156:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T06:36:41.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:41.269 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-10-09T06:36:41.281 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-10-09T06:36:41.308 INFO:teuthology.task.internal:Duration was 4762.418057 seconds 2023-10-09T06:36:41.308 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-10-09T06:36:41.342 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-10-09T06:36:41.342 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T06:36:41.345 DEBUG:teuthology.orchestra.run.smithi156:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T06:36:41.426 INFO:teuthology.orchestra.run.smithi064.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T06:36:41.475 INFO:teuthology.orchestra.run.smithi156.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T06:36:41.895 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-10-09T06:36:41.895 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi064.front.sepia.ceph.com 2023-10-09T06:36:41.895 DEBUG:teuthology.orchestra.run.smithi064:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T06:36:41.928 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi156.front.sepia.ceph.com 2023-10-09T06:36:41.928 DEBUG:teuthology.orchestra.run.smithi156:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T06:36:41.958 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-10-09T06:36:41.958 DEBUG:teuthology.orchestra.run.smithi064:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T06:36:41.970 DEBUG:teuthology.orchestra.run.smithi156:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T06:36:42.140 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-10-09T06:36:42.140 DEBUG:teuthology.orchestra.run.smithi064:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T06:36:42.142 DEBUG:teuthology.orchestra.run.smithi156:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T06:36:42.520 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-10-09T06:36:42.532 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-10-09T06:36:42.532 DEBUG:teuthology.orchestra.run.smithi064:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T06:36:42.572 DEBUG:teuthology.orchestra.run.smithi156:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T06:36:42.615 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-10-09T06:36:42.626 DEBUG:teuthology.orchestra.run.smithi064:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T06:36:42.628 DEBUG:teuthology.orchestra.run.smithi156:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T06:36:42.653 INFO:teuthology.orchestra.run.smithi064.stdout:kernel.core_pattern = core 2023-10-09T06:36:42.690 INFO:teuthology.orchestra.run.smithi156.stdout:kernel.core_pattern = core 2023-10-09T06:36:42.705 DEBUG:teuthology.orchestra.run.smithi064:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T06:36:42.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:42.725 DEBUG:teuthology.orchestra.run.smithi156:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T06:36:42.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:42.762 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-10-09T06:36:42.773 INFO:teuthology.task.internal:Transferring archived files... 2023-10-09T06:36:42.774 DEBUG:teuthology.misc:Transferring archived files from smithi064:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417503/remote/smithi064 2023-10-09T06:36:42.775 DEBUG:teuthology.orchestra.run.smithi064:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T06:36:42.839 DEBUG:teuthology.misc:Transferring archived files from smithi156:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417503/remote/smithi156 2023-10-09T06:36:42.839 DEBUG:teuthology.orchestra.run.smithi156:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T06:36:42.890 INFO:teuthology.task.internal:Removing archive directory... 2023-10-09T06:36:42.890 DEBUG:teuthology.orchestra.run.smithi064:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T06:36:42.892 DEBUG:teuthology.orchestra.run.smithi156:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T06:36:42.937 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-10-09T06:36:42.952 INFO:teuthology.task.internal:Not uploading archives. 2023-10-09T06:36:42.952 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-10-09T06:36:42.977 INFO:teuthology.task.internal:Tidying up after the test... 2023-10-09T06:36:42.977 DEBUG:teuthology.orchestra.run.smithi064:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T06:36:42.980 DEBUG:teuthology.orchestra.run.smithi156:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T06:36:42.995 INFO:teuthology.orchestra.run.smithi064.stdout: 262320 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Oct 9 06:36 /home/ubuntu/cephtest 2023-10-09T06:36:42.995 INFO:teuthology.orchestra.run.smithi064.stdout: 4063233 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Oct 9 06:21 /home/ubuntu/cephtest/mnt.0 2023-10-09T06:36:42.995 INFO:teuthology.orchestra.run.smithi064.stdout: 4063234 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Oct 9 06:21 /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:36:42.997 INFO:teuthology.orchestra.run.smithi156.stdout: 262320 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Oct 9 06:36 /home/ubuntu/cephtest 2023-10-09T06:36:43.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:43.008 INFO:teuthology.orchestra.run.smithi064.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-10-09T06:36:43.008 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi064 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-09T06:36:43.015 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-10-09T06:36:43.026 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-10-09T06:36:43.052 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi064 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-09T06:36:43.052 INFO:teuthology.nuke:Checking targets against current locks 2023-10-09T06:36:43.080 DEBUG:teuthology.nuke:shortname: smithi064 2023-10-09T06:36:43.081 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T06:36:43.102 DEBUG:teuthology.nuke:shortname: smithi156 2023-10-09T06:36:43.102 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T06:36:43.106 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi064.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417503', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 05:11:51.397847', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-09T06:36:43.128 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi156.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417503', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 05:11:51.399717', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-09T06:36:43.149 INFO:teuthology.orchestra.console.smithi064:Power off 2023-10-09T06:36:43.149 DEBUG:teuthology.orchestra.console.smithi064:pexpect command: ipmitool -H smithi064.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T06:36:43.175 DEBUG:teuthology.orchestra.console.smithi064:power off output: Chassis Power Control: Down/Off 2023-10-09T06:36:43.175 DEBUG:teuthology.orchestra.console.smithi064:pexpect command: ipmitool -H smithi064.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T06:36:43.192 INFO:teuthology.orchestra.console.smithi156:Power off 2023-10-09T06:36:43.192 DEBUG:teuthology.orchestra.console.smithi156:pexpect command: ipmitool -H smithi156.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T06:36:43.201 DEBUG:teuthology.orchestra.console.smithi064:check power output: Chassis Power is on 2023-10-09T06:36:43.214 DEBUG:teuthology.orchestra.console.smithi156:power off output: Chassis Power Control: Down/Off 2023-10-09T06:36:43.214 DEBUG:teuthology.orchestra.console.smithi156:pexpect command: ipmitool -H smithi156.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T06:36:43.238 DEBUG:teuthology.orchestra.console.smithi156:check power output: Chassis Power is on 2023-10-09T06:36:47.203 DEBUG:teuthology.orchestra.console.smithi064:pexpect command: ipmitool -H smithi064.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T06:36:47.242 DEBUG:teuthology.orchestra.console.smithi156:pexpect command: ipmitool -H smithi156.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T06:36:47.313 DEBUG:teuthology.orchestra.console.smithi064:check power output: Chassis Power is on 2023-10-09T06:36:47.353 DEBUG:teuthology.orchestra.console.smithi156:check power output: Chassis Power is on 2023-10-09T06:36:51.319 DEBUG:teuthology.orchestra.console.smithi064:pexpect command: ipmitool -H smithi064.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T06:36:51.354 DEBUG:teuthology.orchestra.console.smithi156:pexpect command: ipmitool -H smithi156.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T06:36:51.431 DEBUG:teuthology.orchestra.console.smithi064:check power output: Chassis Power is on 2023-10-09T06:36:51.465 DEBUG:teuthology.orchestra.console.smithi156:check power output: Chassis Power is off 2023-10-09T06:36:51.565 INFO:teuthology.orchestra.console.smithi156:Power off completed 2023-10-09T06:36:55.435 DEBUG:teuthology.orchestra.console.smithi064:pexpect command: ipmitool -H smithi064.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T06:36:55.545 DEBUG:teuthology.orchestra.console.smithi064:check power output: Chassis Power is off 2023-10-09T06:36:55.646 INFO:teuthology.orchestra.console.smithi064:Power off completed 2023-10-09T06:36:55.750 INFO:teuthology.run:Summary data: description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} duration: 4762.418057203293 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=e368a226f9c246a2b7dd2a80d5aa3389 status: fail success: false 2023-10-09T06:36:55.751 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T06:36:55.823 INFO:teuthology.run:FAIL